Merge branch 'softparticles_patch' into 'master'

Shadervisitor adjustments and Mac OS fix for soft particles

See merge request OpenMW/openmw!1380

8 files changed, 187 insertions, 41 deletions

diff --git a/apps/openmw/mwrender/postprocessor.cpp b/apps/openmw/mwrender/postprocessor.cpp
index 53fae4cd7d..6db01444ad 100644
--- a/apps/openmw/mwrender/postprocessor.cpp
+++ b/apps/openmw/mwrender/postprocessor.cpp
@@ -101,8 +101,15 @@ namespace
             : mOpaqueDepthFbo(new osg::FrameBufferObject)
             , mSourceFbo(sourceFbo)
             , mOpaqueDepthTex(opaqueDepthTex)
+            , mColorAttached(false)
         {
             mOpaqueDepthFbo->setAttachment(osg::FrameBufferObject::BufferComponent::DEPTH_BUFFER, osg::FrameBufferAttachment(opaqueDepthTex));
+
+#ifdef __APPLE__
+            // Mac OS drivers complain that a FBO is incomplete if it has no color attachment
+            mOpaqueDepthFbo->setAttachment(osg::FrameBufferObject::BufferComponent::COLOR_BUFFER, osg::FrameBufferAttachment(new osg::RenderBuffer(mOpaqueDepthTex->getTextureWidth(), mOpaqueDepthTex->getTextureHeight(), GL_RGB)));
+            mColorAttached = true;
+#endif
         }
 
         void drawImplementation(osgUtil::RenderBin* bin, osg::RenderInfo& renderInfo, osgUtil::RenderLeaf*& previous) override
@@ -113,7 +120,10 @@ namespace
                 osg::GLExtensions* ext = state.get<osg::GLExtensions>();
 
                 mSourceFbo->apply(state, osg::FrameBufferObject::READ_FRAMEBUFFER);
+                postBindOperation(state);
+
                 mOpaqueDepthFbo->apply(state, osg::FrameBufferObject::DRAW_FRAMEBUFFER);
+                postBindOperation(state);
 
                 ext->glBlitFramebuffer(0, 0, mOpaqueDepthTex->getTextureWidth(), mOpaqueDepthTex->getTextureHeight(), 0, 0, mOpaqueDepthTex->getTextureWidth(), mOpaqueDepthTex->getTextureHeight(), GL_DEPTH_BUFFER_BIT, GL_NEAREST);
 
@@ -123,9 +133,20 @@ namespace
             bin->drawImplementation(renderInfo, previous);
         }
     private:
+        void postBindOperation(osg::State& state)
+        {
+            if (mColorAttached)
+                return;
+            #if !defined(OSG_GLES1_AVAILABLE) && !defined(OSG_GLES2_AVAILABLE) && !defined(OSG_GLES3_AVAILABLE)
+            state.glDrawBuffer(GL_NONE);
+            state.glReadBuffer(GL_NONE);
+            #endif
+        }
+
         osg::ref_ptr<osg::FrameBufferObject> mOpaqueDepthFbo;
         osg::ref_ptr<osg::FrameBufferObject> mSourceFbo;
         osg::ref_ptr<osg::Texture2D> mOpaqueDepthTex;
+        bool mColorAttached;
     };
 }
 
diff --git a/apps/openmw/mwrender/renderingmanager.cpp b/apps/openmw/mwrender/renderingmanager.cpp
index 5ebd95ee44..1925494ccd 100644
--- a/apps/openmw/mwrender/renderingmanager.cpp
+++ b/apps/openmw/mwrender/renderingmanager.cpp
@@ -317,6 +317,7 @@ namespace MWRender
         resourceSystem->getSceneManager()->setParticleSystemMask(MWRender::Mask_ParticleSystem);
         // Shadows and radial fog have problems with fixed-function mode
         bool forceShaders = Settings::Manager::getBool("radial fog", "Shaders")
+                            || Settings::Manager::getBool("soft particles", "Shaders")
                             || Settings::Manager::getBool("force shaders", "Shaders")
                             || Settings::Manager::getBool("enable shadows", "Shadows")
                             || lightingMethod != SceneUtil::LightingMethod::FFP
diff --git a/apps/openmw/mwrender/sky.cpp b/apps/openmw/mwrender/sky.cpp
index 7b124b018c..8e4353f2ef 100644
--- a/apps/openmw/mwrender/sky.cpp
+++ b/apps/openmw/mwrender/sky.cpp
@@ -637,7 +637,6 @@ namespace MWRender
                     mParticleNode = new osg::PositionAttitudeTransform;
                     mParticleNode->addCullCallback(mUnderwaterSwitch);
                     mParticleNode->setNodeMask(Mask_WeatherParticles);
-                    mParticleNode->getOrCreateStateSet();
                     mRootNode->addChild(mParticleNode);
                 }
 
@@ -669,7 +668,6 @@ namespace MWRender
                         ps->getParticle(particleIndex)->update(0, true);
                     }
 
-                    ps->getOrCreateStateSet();
                     ps->setUserValue("simpleLighting", true);
                 }
 
diff --git a/components/nifosg/nifloader.cpp b/components/nifosg/nifloader.cpp
index 3148b93cdc..43af68fc90 100644
--- a/components/nifosg/nifloader.cpp
+++ b/components/nifosg/nifloader.cpp
@@ -1029,7 +1029,6 @@ namespace NifOsg
         void handleParticleSystem(const Nif::Node *nifNode, osg::Group *parentNode, SceneUtil::CompositeStateSetUpdater* composite, int animflags)
         {
             osg::ref_ptr<ParticleSystem> partsys (new ParticleSystem);
-            partsys->getOrCreateStateSet();
             partsys->setSortMode(osgParticle::ParticleSystem::SORT_BACK_TO_FRONT);
 
             const Nif::NiParticleSystemController* partctrl = nullptr;
diff --git a/components/shader/shadervisitor.cpp b/components/shader/shadervisitor.cpp
index 95e0a75f18..107665369c 100644
--- a/components/shader/shadervisitor.cpp
+++ b/components/shader/shadervisitor.cpp
@@ -1,6 +1,7 @@
 #include "shadervisitor.hpp"
 
 #include <unordered_set>
+#include <unordered_map>
 #include <set>
 
 #include <osg/AlphaFunc>
@@ -28,6 +29,15 @@
 
 namespace Shader
 {
+    /**
+    * Miniature version of osg::StateSet used to track state added by the shader visitor which should be ignored when
+    * it's applied a second time, and removed when shaders are removed.
+    * Actual StateAttributes aren't kept as they're recoverable from the StateSet this is attached to - we just want
+    * the TypeMemberPair as that uniquely identifies which of those StateAttributes it was we're tracking.
+    * Not all StateSet features have been added yet - we implement an equivalently-named method to each of the StateSet
+    * methods called in createProgram, and implement new ones as they're needed.
+    * When expanding tracking to cover new things, ensure they're accounted for in ensureFFP.
+    */
     class AddedState : public osg::Object
     {
     public:
@@ -37,6 +47,7 @@ namespace Shader
             , mUniforms(rhs.mUniforms)
             , mModes(rhs.mModes)
             , mAttributes(rhs.mAttributes)
+            , mTextureModes(rhs.mTextureModes)
         {
         }
 
@@ -60,16 +71,44 @@ namespace Shader
         template<typename T>
         void setAttributeAndModes(osg::ref_ptr<T> attribute) { setAttributeAndModes(attribute.get()); }
 
+        void setTextureMode(unsigned int unit, osg::StateAttribute::GLMode mode) { mTextureModes[unit].emplace(mode); }
+        void setTextureAttribute(int unit, osg::StateAttribute::TypeMemberPair typeMemberPair) { mTextureAttributes[unit].emplace(typeMemberPair); }
+
+        void setTextureAttribute(unsigned int unit, const osg::StateAttribute* attribute)
+        {
+            mTextureAttributes[unit].emplace(attribute->getTypeMemberPair());
+        }
+        template<typename T>
+        void setTextureAttribute(unsigned int unit, osg::ref_ptr<T> attribute) { setTextureAttribute(unit, attribute.get()); }
+
+        void setTextureAttributeAndModes(unsigned int unit, const osg::StateAttribute* attribute)
+        {
+            setTextureAttribute(unit, attribute);
+            InterrogateModesHelper helper(this, unit);
+            attribute->getModeUsage(helper);
+        }
+        template<typename T>
+        void setTextureAttributeAndModes(unsigned int unit, osg::ref_ptr<T> attribute) { setTextureAttributeAndModes(unit, attribute.get()); }
+
         bool hasUniform(const std::string& name) { return mUniforms.count(name); }
         bool hasMode(osg::StateAttribute::GLMode mode) { return mModes.count(mode); }
         bool hasAttribute(const osg::StateAttribute::TypeMemberPair &typeMemberPair) { return mAttributes.count(typeMemberPair); }
         bool hasAttribute(osg::StateAttribute::Type type, unsigned int member) { return hasAttribute(osg::StateAttribute::TypeMemberPair(type, member)); }
+        bool hasTextureMode(int unit, osg::StateAttribute::GLMode mode)
+        {
+            auto it = mTextureModes.find(unit);
+            if (it == mTextureModes.cend())
+                return false;
+
+            return it->second.count(mode);
+        }
 
         const std::set<osg::StateAttribute::TypeMemberPair>& getAttributes() { return mAttributes; }
+        const std::unordered_map<unsigned int, std::set<osg::StateAttribute::TypeMemberPair>>& getTextureAttributes() { return mTextureAttributes; }
 
         bool empty()
         {
-            return mUniforms.empty() && mModes.empty() && mAttributes.empty();
+            return mUniforms.empty() && mModes.empty() && mAttributes.empty() && mTextureModes.empty() && mTextureAttributes.empty();
         }
 
         META_Object(Shader, AddedState)
@@ -78,17 +117,26 @@ namespace Shader
         class InterrogateModesHelper : public osg::StateAttribute::ModeUsage
         {
         public:
-            InterrogateModesHelper(AddedState* tracker) : mTracker(tracker) {}
+            InterrogateModesHelper(AddedState* tracker, unsigned int textureUnit = 0)
+                : mTracker(tracker)
+                , mTextureUnit(textureUnit)
+            {}
             void usesMode(osg::StateAttribute::GLMode mode) override { mTracker->setMode(mode); }
-            void usesTextureMode(osg::StateAttribute::GLMode mode) override {}
+            void usesTextureMode(osg::StateAttribute::GLMode mode) override { mTracker->setTextureMode(mTextureUnit, mode); }
 
         private:
             AddedState* mTracker;
+            unsigned int mTextureUnit;
         };
 
+        using ModeSet = std::unordered_set<osg::StateAttribute::GLMode>;
+        using AttributeSet = std::set<osg::StateAttribute::TypeMemberPair>;
+
         std::unordered_set<std::string> mUniforms;
-        std::unordered_set<osg::StateAttribute::GLMode> mModes;
-        std::set<osg::StateAttribute::TypeMemberPair> mAttributes;
+        ModeSet mModes;
+        AttributeSet mAttributes;
+        std::unordered_map<unsigned int, ModeSet> mTextureModes;
+        std::unordered_map<unsigned int, AttributeSet> mTextureAttributes;
     };
 
     ShaderVisitor::ShaderRequirements::ShaderRequirements()
@@ -102,6 +150,8 @@ namespace Shader
         , mAlphaBlend(false)
         , mNormalHeight(false)
         , mTexStageRequiringTangents(-1)
+        , mSoftParticles(false)
+        , mSoftParticleSize(0.f)
         , mNode(nullptr)
     {
     }
@@ -213,6 +263,9 @@ namespace Shader
         if (node.getUserValue("shaderRequired", shaderRequired) && shaderRequired)
             mRequirements.back().mShaderRequired = true;
 
+        // Make sure to disregard any state that came from a previous call to createProgram
+        osg::ref_ptr<AddedState> addedState = getAddedState(*stateset);
+
         if (!texAttributes.empty())
         {
             const osg::Texture* diffuseMap = nullptr;
@@ -224,6 +277,11 @@ namespace Shader
                 const osg::StateAttribute *attr = stateset->getTextureAttribute(unit, osg::StateAttribute::TEXTURE);
                 if (attr)
                 {
+                    // If textures ever get removed in createProgram, expand this to check we're operating on main texture attribute list
+                    // rather than the removed list
+                    if (addedState && addedState->hasTextureMode(unit, GL_TEXTURE_2D))
+                        continue;
+
                     const osg::Texture* texture = attr->asTexture();
                     if (texture)
                     {
@@ -350,7 +408,6 @@ namespace Shader
         osg::StateSet::AttributeList removedAttributes;
         if (osg::ref_ptr<osg::StateSet> removedState = getRemovedState(*stateset))
             removedAttributes = removedState->getAttributeList();
-        osg::ref_ptr<AddedState> addedState = getAddedState(*stateset);
 
         for (const auto* attributeMap : std::initializer_list<const osg::StateSet::AttributeList*>{ &attributes, &removedAttributes })
         {
@@ -442,6 +499,22 @@ namespace Shader
             return;
         }
 
+        /**
+        * The shader visitor is supposed to be idempotent and undoable.
+        * That means we need to back up state we've removed (so it can be restored and/or considered by further
+        * applications of the visitor) and track which state we added (so it can be removed and/or ignored by further
+        * applications of the visitor).
+        * Before editing writableStateSet in a way that explicitly removes state or might overwrite existing state, it
+        * should be copied to removedState, another StateSet, unless it's there already or was added by a previous
+        * application of the visitor (is in previousAddedState).
+        * If it's a new class of state that's not already handled by ReinstateRemovedStateVisitor::apply, make sure to
+        * add handling there.
+        * Similarly, any time new state is added to writableStateSet, the equivalent method should be called on
+        * addedState.
+        * If that method doesn't exist yet, implement it - we don't use a full StateSet as we only need to check
+        * existence, not equality, and don't need to actually get the value as we can get it from writableStateSet
+        * instead.
+        */
         osg::Node& node = *reqs.mNode;
         osg::StateSet* writableStateSet = nullptr;
         if (mAllowedToModifyStateSets)
@@ -466,7 +539,10 @@ namespace Shader
         }
 
         if (defineMap["diffuseMap"] == "0")
+        {
             writableStateSet->addUniform(new osg::Uniform("useDiffuseMapForShadowAlpha", false));
+            addedState->addUniform("useDiffuseMapForShadowAlpha");
+        }
 
         defineMap["parallax"] = reqs.mNormalHeight ? "1" : "0";
 
@@ -475,7 +551,6 @@ namespace Shader
 
         defineMap["alphaFunc"] = std::to_string(reqs.mAlphaFunc);
 
-        // back up removed state in case recreateShaders gets rid of the shader later
         osg::ref_ptr<osg::StateSet> removedState;
         if ((removedState = getRemovedState(*writableStateSet)) && !mAllowedToModifyStateSets)
             removedState = new osg::StateSet(*removedState, osg::CopyOp::SHALLOW_COPY);
@@ -545,38 +620,24 @@ namespace Shader
             updateRemovedState(*writableUserData, removedState);
         }
 
-        if (!addedState->empty())
+        if (reqs.mSoftParticles)
         {
-            // user data is normally shallow copied so shared with the original stateset
-            osg::ref_ptr<osg::UserDataContainer> writableUserData;
-            if (mAllowedToModifyStateSets)
-                writableUserData = writableStateSet->getOrCreateUserDataContainer();
-            else
-                writableUserData = getWritableUserDataContainer(*writableStateSet);
-
-            updateAddedState(*writableUserData, addedState);
-        }
+            osg::ref_ptr<osg::Depth> depth = new SceneUtil::AutoDepth;
+            depth->setWriteMask(false);
+            writableStateSet->setAttributeAndModes(depth, osg::StateAttribute::ON|osg::StateAttribute::OVERRIDE);
+            addedState->setAttributeAndModes(depth);
 
-        bool softParticles = false;
+            writableStateSet->addUniform(new osg::Uniform("particleSize", reqs.mSoftParticleSize));
+            addedState->addUniform("particleSize");
 
-        if (mOpaqueDepthTex)
-        {
-            auto partsys = dynamic_cast<osgParticle::ParticleSystem*>(&node);
+            writableStateSet->addUniform(new osg::Uniform("opaqueDepthTex", 2));
+            addedState->addUniform("opaqueDepthTex");
 
-            if (partsys)
-            {
-                softParticles = true;
-
-                auto depth = new SceneUtil::AutoDepth;
-                depth->setWriteMask(false);
-                writableStateSet->setAttributeAndModes(depth, osg::StateAttribute::ON|osg::StateAttribute::OVERRIDE);
-                writableStateSet->addUniform(new osg::Uniform("particleSize", partsys->getDefaultParticleTemplate().getSizeRange().maximum));
-                writableStateSet->addUniform(new osg::Uniform("opaqueDepthTex", 2));
-                writableStateSet->setTextureAttributeAndModes(2, mOpaqueDepthTex, osg::StateAttribute::ON);
-            }
+            writableStateSet->setTextureAttributeAndModes(2, mOpaqueDepthTex, osg::StateAttribute::ON);
+            addedState->setTextureAttributeAndModes(2, mOpaqueDepthTex);
         }
 
-        defineMap["softParticles"] = softParticles ? "1" : "0";
+        defineMap["softParticles"] = reqs.mSoftParticles ? "1" : "0";
 
         std::string shaderPrefix;
         if (!node.getUserValue("shaderPrefix", shaderPrefix))
@@ -597,6 +658,18 @@ namespace Shader
                 addedState->addUniform(texIt->second);
             }
         }
+
+        if (!addedState->empty())
+        {
+            // user data is normally shallow copied so shared with the original stateset
+            osg::ref_ptr<osg::UserDataContainer> writableUserData;
+            if (mAllowedToModifyStateSets)
+                writableUserData = writableStateSet->getOrCreateUserDataContainer();
+            else
+                writableUserData = getWritableUserDataContainer(*writableStateSet);
+
+            updateAddedState(*writableUserData, addedState);
+        }
     }
 
     void ShaderVisitor::ensureFFP(osg::Node& node)
@@ -609,6 +682,18 @@ namespace Shader
         else
             writableStateSet = getWritableStateSet(node);
 
+        /**
+        * We might have been using shaders temporarily with the node (e.g. if a GlowUpdater applied a temporary
+        * environment map for a temporary enchantment).
+        * We therefore need to remove any state doing so added, and restore any that it removed.
+        * This is kept track of in createProgram in the StateSet's userdata.
+        * If new classes of state get added, handling it here is required - not all StateSet features are implemented
+        * in AddedState yet as so far they've not been necessary.
+        * Removed state requires no particular special handling as it's dealt with by merging StateSets.
+        * We don't need to worry about state in writableStateSet having the OVERRIDE flag as if it's in both, it's also
+        * in addedState, and gets removed first.
+        */
+
         // user data is normally shallow copied so shared with the original stateset - we'll need to copy before edits
         osg::ref_ptr<osg::UserDataContainer> writableUserData;
 
@@ -643,6 +728,23 @@ namespace Shader
             // We don't have access to the function to do that, and can't call removeAttribute with an iterator
             for (const auto& [type, member] : addedState->getAttributes())
                 writableStateSet->removeAttribute(type, member);
+
+            for (unsigned int unit = 0; unit < writableStateSet->getTextureModeList().size(); ++unit)
+            {
+                for (auto itr = writableStateSet->getTextureModeList()[unit].begin(); itr != writableStateSet->getTextureModeList()[unit].end();)
+                {
+                    if (addedState->hasTextureMode(unit, itr->first))
+                        writableStateSet->getTextureModeList()[unit].erase(itr++);
+                    else
+                        ++itr;
+                }
+            }
+
+            for (const auto& [unit, attributeList] : addedState->getTextureAttributes())
+            {
+                for (const auto& [type, member] : attributeList)
+                    writableStateSet->removeTextureAttribute(unit, type);
+            }
         }
 
 
@@ -719,13 +821,22 @@ namespace Shader
 
     void ShaderVisitor::apply(osg::Drawable& drawable)
     {
-        // non-Geometry drawable (e.g. particle system)
-        bool needPop = (drawable.getStateSet() != nullptr);
+        auto partsys = dynamic_cast<osgParticle::ParticleSystem*>(&drawable);
 
-        if (drawable.getStateSet())
+        bool needPop = drawable.getStateSet() || partsys;
+
+        if (needPop)
         {
             pushRequirements(drawable);
-            applyStateSet(drawable.getStateSet(), drawable);
+
+            if (partsys && mOpaqueDepthTex)
+            {
+                mRequirements.back().mSoftParticles = true;
+                mRequirements.back().mSoftParticleSize = partsys->getDefaultParticleTemplate().getSizeRange().maximum;
+            }
+
+            if (drawable.getStateSet())
+                applyStateSet(drawable.getStateSet(), drawable);
         }
 
         if (!mRequirements.empty())
@@ -806,6 +917,10 @@ namespace Shader
 
     void ReinstateRemovedStateVisitor::apply(osg::Node& node)
     {
+        // TODO: this may eventually need to remove added state.
+        // If so, we can migrate from explicitly copying removed state to just calling osg::StateSet::merge.
+        // Not everything is transferred from removedState yet - implement more when createProgram starts marking more
+        // as removed.
         if (node.getStateSet())
         {
             osg::ref_ptr<osg::StateSet> removedState = getRemovedState(*node.getStateSet());
@@ -831,6 +946,12 @@ namespace Shader
 
                 for (const auto& attribute : removedState->getAttributeList())
                     writableStateSet->setAttribute(attribute.second.first, attribute.second.second);
+
+                for (unsigned int unit = 0; unit < removedState->getTextureModeList().size(); ++unit)
+                {
+                    for (const auto&[mode, value] : removedState->getTextureModeList()[unit])
+                        writableStateSet->setTextureMode(unit, mode, value);
+                }
             }
         }
 
diff --git a/components/shader/shadervisitor.hpp b/components/shader/shadervisitor.hpp
index d80e697fd8..72dec05b5e 100644
--- a/components/shader/shadervisitor.hpp
+++ b/components/shader/shadervisitor.hpp
@@ -101,6 +101,9 @@ namespace Shader
             // -1 == no tangents required
             int mTexStageRequiringTangents;
 
+            bool mSoftParticles;
+            float mSoftParticleSize;
+
             // the Node that requested these requirements
             osg::Node* mNode;
         };
diff --git a/docs/source/reference/modding/settings/shaders.rst b/docs/source/reference/modding/settings/shaders.rst
index b9d8cfe1b9..5629e321d0 100644
--- a/docs/source/reference/modding/settings/shaders.rst
+++ b/docs/source/reference/modding/settings/shaders.rst
@@ -282,3 +282,6 @@ intersection between individual particles and other opaque geometry by blending
 between them. Note, this relies on overriding specific properties of particle
 systems that potentially differ from the source content, this setting may change
 the look of some particle systems.
+
+Note that the rendering will act as if you have 'force shaders' option enabled.
+This means that shaders will be used to render all objects and the terrain.
\ No newline at end of file
diff --git a/files/shaders/objects_fragment.glsl b/files/shaders/objects_fragment.glsl
index 6e8d929569..9b60e7c6a4 100644
--- a/files/shaders/objects_fragment.glsl
+++ b/files/shaders/objects_fragment.glsl
@@ -226,7 +226,7 @@ void main()
 #endif
     gl_FragData[0].xyz = mix(gl_FragData[0].xyz, gl_Fog.color.xyz, fogValue);
 
-#if @softParticles
+#if !defined(FORCE_OPAQUE) && @softParticles
     gl_FragData[0].a *= calcSoftParticleFade();
 #endif