caching dialog keyword search in Store<ESM::Dialogue>

10 files changed, 175 insertions, 113 deletions

diff --git a/apps/openmw/mwbase/world.hpp b/apps/openmw/mwbase/world.hpp
index d1747a2e39..d0ff851274 100644
--- a/apps/openmw/mwbase/world.hpp
+++ b/apps/openmw/mwbase/world.hpp
@@ -87,6 +87,11 @@ namespace MWWorld
     typedef std::vector<std::pair<MWWorld::Ptr,MWMechanics::Movement> > PtrMovementList;
 }
 
+namespace MWDialogue
+{
+    template <typename string_t, typename value_t> class KeywordSearch;
+}
+
 namespace MWBase
 {
     /// \brief Interface for the World (implemented in MWWorld)
@@ -150,6 +155,7 @@ namespace MWBase
             virtual MWWorld::ConstPtr getPlayerConstPtr() const = 0;
 
             virtual const MWWorld::ESMStore& getStore() const = 0;
+            virtual const MWDialogue::KeywordSearch<std::string, int>& getDialogIdKeywordSearch() = 0;
 
             virtual std::vector<ESM::ESMReader>& getEsmReader() = 0;
 
diff --git a/apps/openmw/mwdialogue/dialoguemanagerimp.cpp b/apps/openmw/mwdialogue/dialoguemanagerimp.cpp
index 0698624f32..9800f1b39c 100644
--- a/apps/openmw/mwdialogue/dialoguemanagerimp.cpp
+++ b/apps/openmw/mwdialogue/dialoguemanagerimp.cpp
@@ -43,6 +43,7 @@
 #include "../mwmechanics/actorutil.hpp"
 
 #include "filter.hpp"
+#include "hypertextparser.hpp"
 
 namespace MWDialogue
 {
@@ -79,7 +80,7 @@ namespace MWDialogue
     {
         std::vector<std::string> topicIdList;
 
-        std::vector<HyperTextParser::Token> hypertext = mHyperTextParser.parseHyperText(text);
+        std::vector<HyperTextParser::Token> hypertext = HyperTextParser::parseHyperText(text);
 
         for (std::vector<HyperTextParser::Token>::iterator tok = hypertext.begin(); tok != hypertext.end(); ++tok)
         {
diff --git a/apps/openmw/mwdialogue/dialoguemanagerimp.hpp b/apps/openmw/mwdialogue/dialoguemanagerimp.hpp
index 1fc80db666..57eb74d0a6 100644
--- a/apps/openmw/mwdialogue/dialoguemanagerimp.hpp
+++ b/apps/openmw/mwdialogue/dialoguemanagerimp.hpp
@@ -16,8 +16,6 @@
 
 #include "../mwscript/compilercontext.hpp"
 
-#include "hypertextparser.hpp"
-
 namespace ESM
 {
     struct Dialogue;
@@ -59,8 +57,6 @@ namespace MWDialogue
             int mCurrentDisposition;
             int mPermanentDispositionChange;
 
-            HyperTextParser mHyperTextParser;
-
             std::vector<std::string> parseTopicIdsFromText (const std::string& text);
             void addTopicsFromText (const std::string& text);
 
diff --git a/apps/openmw/mwdialogue/hypertextparser.cpp b/apps/openmw/mwdialogue/hypertextparser.cpp
index eae36d26d1..c1fb6c9c43 100644
--- a/apps/openmw/mwdialogue/hypertextparser.cpp
+++ b/apps/openmw/mwdialogue/hypertextparser.cpp
@@ -6,89 +6,78 @@
 #include "../mwworld/store.hpp"
 #include "../mwworld/esmstore.hpp"
 
+#include "keywordsearch.hpp"
+
 #include "hypertextparser.hpp"
 
 namespace MWDialogue
 {
-    std::vector<HyperTextParser::Token> HyperTextParser::parseHyperText(const std::string & text)
+    namespace HyperTextParser
     {
-        std::vector<Token> result;
-        size_t pos_end = std::string::npos, iteration_pos = 0;
-        for(;;)
+        std::vector<Token> parseHyperText(const std::string & text)
         {
-            size_t pos_begin = text.find('@', iteration_pos);
-            if (pos_begin != std::string::npos)
-                pos_end = text.find('#', pos_begin);
-
-            if (pos_begin != std::string::npos && pos_end != std::string::npos)
-            {
-                if (pos_begin != iteration_pos)
-                    tokenizeKeywords(text.substr(iteration_pos, pos_begin - iteration_pos), result);
-
-                std::string link = text.substr(pos_begin + 1, pos_end - pos_begin - 1);
-                result.emplace_back(link, Token::ExplicitLink);
-
-                iteration_pos = pos_end + 1;
-            }
-            else
+            std::vector<Token> result;
+            size_t pos_end = std::string::npos, iteration_pos = 0;
+            for(;;)
             {
-                if (iteration_pos != text.size())
-                    tokenizeKeywords(text.substr(iteration_pos), result);
-                break;
+                size_t pos_begin = text.find('@', iteration_pos);
+                if (pos_begin != std::string::npos)
+                    pos_end = text.find('#', pos_begin);
+
+                if (pos_begin != std::string::npos && pos_end != std::string::npos)
+                {
+                    if (pos_begin != iteration_pos)
+                        tokenizeKeywords(text.substr(iteration_pos, pos_begin - iteration_pos), result);
+
+                    std::string link = text.substr(pos_begin + 1, pos_end - pos_begin - 1);
+                    result.emplace_back(link, Token::ExplicitLink);
+
+                    iteration_pos = pos_end + 1;
+                }
+                else
+                {
+                    if (iteration_pos != text.size())
+                        tokenizeKeywords(text.substr(iteration_pos), result);
+                    break;
+                }
             }
-        }
-
-        return result;
-    }
 
-    void HyperTextParser::tokenizeKeywords(const std::string & text, std::vector<Token> & tokens)
-    {
-        const MWWorld::Store<ESM::Dialogue> & dialogs =
-            MWBase::Environment::get().getWorld()->getStore().get<ESM::Dialogue>();
+            return result;
+        }
 
-        if (dialogs.getModPoint() != mKeywordModPoint)
+        void tokenizeKeywords(const std::string & text, std::vector<Token> & tokens)
         {
-            mKeywordSearch.clear();
-
-            std::vector<std::string> keywordList;
-            keywordList.reserve(dialogs.getSize());
-            for (const auto& it : dialogs)
-                keywordList.push_back(Misc::StringUtils::lowerCase(it.mId));
-            sort(keywordList.begin(), keywordList.end());
+            const auto& keywordSearch =
+                MWBase::Environment::get().getWorld()->getDialogIdKeywordSearch();
 
-            for (const auto& it : keywordList)
-                mKeywordSearch.seed(it, 0 /*unused*/);
+            std::vector<KeywordSearch<std::string, int /*unused*/>::Match> matches;
+            keywordSearch.highlightKeywords(text.begin(), text.end(), matches);
 
-            mKeywordModPoint = dialogs.getModPoint();
+            for (std::vector<KeywordSearch<std::string, int /*unused*/>::Match>::const_iterator it = matches.begin(); it != matches.end(); ++it)
+            {
+                tokens.emplace_back(std::string(it->mBeg, it->mEnd), Token::ImplicitKeyword);
+            }
         }
 
-        std::vector<KeywordSearch<std::string, int /*unused*/>::Match> matches;
-        mKeywordSearch.highlightKeywords(text.begin(), text.end(), matches);
-
-        for (std::vector<KeywordSearch<std::string, int /*unused*/>::Match>::const_iterator it = matches.begin(); it != matches.end(); ++it)
+        size_t removePseudoAsterisks(std::string & phrase)
         {
-            tokens.emplace_back(std::string(it->mBeg, it->mEnd), Token::ImplicitKeyword);
-        }
-    }
+            size_t pseudoAsterisksCount = 0;
 
-    size_t HyperTextParser::removePseudoAsterisks(std::string & phrase)
-    {
-        size_t pseudoAsterisksCount = 0;
-
-        if( !phrase.empty() )
-        {
-            std::string::reverse_iterator rit = phrase.rbegin();
-
-            const char specialPseudoAsteriskCharacter = 127;
-            while( rit != phrase.rend() && *rit == specialPseudoAsteriskCharacter )
+            if( !phrase.empty() )
             {
-                pseudoAsterisksCount++;
-                ++rit;
+                std::string::reverse_iterator rit = phrase.rbegin();
+
+                const char specialPseudoAsteriskCharacter = 127;
+                while( rit != phrase.rend() && *rit == specialPseudoAsteriskCharacter )
+                {
+                    pseudoAsterisksCount++;
+                    ++rit;
+                }
             }
-        }
 
-        phrase = phrase.substr(0, phrase.length() - pseudoAsterisksCount);
+            phrase = phrase.substr(0, phrase.length() - pseudoAsterisksCount);
 
-        return pseudoAsterisksCount;
+            return pseudoAsterisksCount;
+        }
     }
 }
diff --git a/apps/openmw/mwdialogue/hypertextparser.hpp b/apps/openmw/mwdialogue/hypertextparser.hpp
index 1e903d5618..4ae0474c42 100644
--- a/apps/openmw/mwdialogue/hypertextparser.hpp
+++ b/apps/openmw/mwdialogue/hypertextparser.hpp
@@ -4,17 +4,10 @@
 #include <string>
 #include <vector>
 
-#include "keywordsearch.hpp"
-
 namespace MWDialogue
 {
-    class HyperTextParser
+    namespace HyperTextParser
     {
-        uint64_t mKeywordModPoint;
-        KeywordSearch<std::string, int /*unused*/> mKeywordSearch;
-
-    public:
-
         struct Token
         {
             enum Type
@@ -31,14 +24,12 @@ namespace MWDialogue
             Type mType;
         };
 
-        HyperTextParser() : mKeywordModPoint(0) {}
-
         // In translations (at least Russian) the links are marked with @#, so
         // it should be a function to parse it
         std::vector<Token> parseHyperText(const std::string & text);
         void tokenizeKeywords(const std::string & text, std::vector<Token> & tokens);
-        static size_t removePseudoAsterisks(std::string & phrase);
-    };
+        size_t removePseudoAsterisks(std::string & phrase);
+    }
 }
 
 #endif
diff --git a/apps/openmw/mwdialogue/keywordsearch.hpp b/apps/openmw/mwdialogue/keywordsearch.hpp
index 39599457ef..3f932084fe 100644
--- a/apps/openmw/mwdialogue/keywordsearch.hpp
+++ b/apps/openmw/mwdialogue/keywordsearch.hpp
@@ -74,13 +74,13 @@ public:
         return left.mBeg < right.mBeg;
     }
 
-    void highlightKeywords (Point beg, Point end, std::vector<Match>& out)
+    void highlightKeywords (Point beg, Point end, std::vector<Match>& out) const
     {
         std::vector<Match> matches;
         for (Point i = beg; i != end; ++i)
         {
             // check first character
-            typename Entry::childen_t::iterator candidate = mRoot.mChildren.find (Misc::StringUtils::toLower (*i));
+            typename Entry::childen_t::const_iterator candidate = mRoot.mChildren.find (Misc::StringUtils::toLower (*i));
 
             // no match, on to next character
             if (candidate == mRoot.mChildren.end ())
@@ -91,11 +91,11 @@ public:
 
             // some keywords might be longer variations of other keywords, so we definitely need a list of candidates
             // the first element in the pair is length of the match, i.e. depth from the first character on
-            std::vector< typename std::pair<int, typename Entry::childen_t::iterator> > candidates;
+            std::vector< typename std::pair<int, typename Entry::childen_t::const_iterator> > candidates;
 
             while ((j + 1) != end)
             {
-                typename Entry::childen_t::iterator next = candidate->second.mChildren.find (Misc::StringUtils::toLower (*++j));
+                typename Entry::childen_t::const_iterator next = candidate->second.mChildren.find (Misc::StringUtils::toLower (*++j));
 
                 if (next == candidate->second.mChildren.end ())
                 {
@@ -116,7 +116,7 @@ public:
             // shorter candidates will be added to the vector first. however, we want to check against longer candidates first
             std::reverse(candidates.begin(), candidates.end());
 
-            for (typename std::vector< std::pair<int, typename Entry::childen_t::iterator> >::iterator it = candidates.begin();
+            for (typename std::vector< std::pair<int, typename Entry::childen_t::const_iterator> >::iterator it = candidates.begin();
                  it != candidates.end(); ++it)
             {
                 candidate = it->second;
diff --git a/apps/openmw/mwworld/esmstore.hpp b/apps/openmw/mwworld/esmstore.hpp
index 8582a1daca..855dd062e9 100644
--- a/apps/openmw/mwworld/esmstore.hpp
+++ b/apps/openmw/mwworld/esmstore.hpp
@@ -284,6 +284,9 @@ namespace MWWorld
         /// Actors with the same ID share spells, abilities, etc.
         /// @return The shared spell list to use for this actor and whether or not it has already been initialized.
         std::pair<std::shared_ptr<MWMechanics::SpellList>, bool> getSpellList(const std::string& id) const;
+
+        const MWDialogue::KeywordSearch<std::string, int>& getDialogIdKeywordSearch() {
+            return mDialogs.getDialogIdKeywordSearch(); }
     };
 
     template <>
diff --git a/apps/openmw/mwworld/store.cpp b/apps/openmw/mwworld/store.cpp
index 5c696e5556..a06bf843cb 100644
--- a/apps/openmw/mwworld/store.cpp
+++ b/apps/openmw/mwworld/store.cpp
@@ -77,13 +77,12 @@ namespace MWWorld
 
     template<typename T>
     Store<T>::Store()
-        : mModPoint(1)
     {
     }
 
     template<typename T>
     Store<T>::Store(const Store<T>& orig)
-        : mStatic(orig.mStatic), mModPoint(orig.mModPoint + 1)
+        : mStatic(orig.mStatic)
     {
     }
 
@@ -94,8 +93,6 @@ namespace MWWorld
         assert(mShared.size() >= mStatic.size());
         mShared.erase(mShared.begin() + mStatic.size(), mShared.end());
         mDynamic.clear();
-
-        mModPoint++;
     }
 
     template<typename T>
@@ -165,8 +162,6 @@ namespace MWWorld
         if (inserted.second)
             mShared.push_back(&inserted.first->second);
 
-        mModPoint++;
-
         return RecordId(record.mId, isDeleted);
     }
     template<typename T>
@@ -218,9 +213,6 @@ namespace MWWorld
         T *ptr = &result.first->second;
         if (result.second)
             mShared.push_back(ptr);
-
-        mModPoint++;
-
         return ptr;
     }
     template<typename T>
@@ -230,9 +222,6 @@ namespace MWWorld
         T *ptr = &result.first->second;
         if (result.second)
             mShared.push_back(ptr);
-
-        mModPoint++;
-
         return ptr;
     }
     template<typename T>
@@ -253,8 +242,6 @@ namespace MWWorld
                 ++sharedIter;
             }
             mStatic.erase(it);
-
-            mModPoint++;
         }
 
         return true;
@@ -272,9 +259,6 @@ namespace MWWorld
         for (auto it = mDynamic.begin(); it != mDynamic.end(); ++it) {
             mShared.push_back(&it->second);
         }
-
-        mModPoint++;
-
         return true;
     }
     template<typename T>
@@ -997,8 +981,11 @@ namespace MWWorld
     // Dialogue
     //=========================================================================
 
+    Store<ESM::Dialogue>::Store()
+        : mKeywordSearchModFlag(true)
+    {
+    }
 
-    template<>
     void Store<ESM::Dialogue>::setUp()
     {
         // DialInfos marked as deleted are kept during the loading phase, so that the linked list
@@ -1014,10 +1001,45 @@ namespace MWWorld
         // TODO: if we require this behaviour, maybe we should move it to the place that requires it
         std::sort(mShared.begin(), mShared.end(), [](const ESM::Dialogue* l, const ESM::Dialogue* r) -> bool { return l->mId < r->mId; });
 
-        mModPoint++;
+        mKeywordSearchModFlag = true;
+    }
+
+    const ESM::Dialogue *Store<ESM::Dialogue>::search(const std::string &id) const
+    {
+        typename Static::const_iterator it = mStatic.find(id);
+        if (it != mStatic.end())
+            return &(it->second);
+
+        return nullptr;
+    }
+
+    const ESM::Dialogue *Store<ESM::Dialogue>::find(const std::string &id) const
+    {
+        const ESM::Dialogue *ptr = search(id);
+        if (ptr == nullptr)
+        {
+            std::stringstream msg;
+            msg << ESM::Dialogue::getRecordType() << " '" << id << "' not found";
+            throw std::runtime_error(msg.str());
+        }
+        return ptr;
+    }
+
+    typename Store<ESM::Dialogue>::iterator Store<ESM::Dialogue>::begin() const
+    {
+        return mShared.begin();
+    }
+
+    typename Store<ESM::Dialogue>::iterator Store<ESM::Dialogue>::end() const
+    {
+        return mShared.end();
+    }
+
+    size_t Store<ESM::Dialogue>::getSize() const
+    {
+        return mShared.size();
     }
 
-    template <>
     inline RecordId Store<ESM::Dialogue>::load(ESM::ESMReader &esm) {
         // The original letter case of a dialogue ID is saved, because it's printed
         ESM::Dialogue dialogue;
@@ -1037,20 +1059,39 @@ namespace MWWorld
             dialogue.mId = found->second.mId;
         }
         
-        mModPoint++;
+        mKeywordSearchModFlag = true;
 
         return RecordId(dialogue.mId, isDeleted);
     }
 
-    template<>
     bool Store<ESM::Dialogue>::eraseStatic(const std::string &id)
     {
         if (mStatic.erase(id))
-            mModPoint++;
+            mKeywordSearchModFlag = true;
 
         return true;
     }
 
+    const MWDialogue::KeywordSearch<std::string, int>& Store<ESM::Dialogue>::getDialogIdKeywordSearch()
+    {
+        if (mKeywordSearchModFlag)
+        {
+            mKeywordSearch.clear();
+
+            std::vector<std::string> keywordList;
+            keywordList.reserve(getSize());
+            for (const auto& it : *this)
+                keywordList.push_back(Misc::StringUtils::lowerCase(it.mId));
+            sort(keywordList.begin(), keywordList.end());
+
+            for (const auto& it : keywordList)
+                mKeywordSearch.seed(it, 0 /*unused*/);
+
+            mKeywordSearchModFlag = false;
+        }
+
+        return mKeywordSearch;
+    }
 }
 
 template class MWWorld::Store<ESM::Activator>;
diff --git a/apps/openmw/mwworld/store.hpp b/apps/openmw/mwworld/store.hpp
index b4ac1056a8..58ac1e6780 100644
--- a/apps/openmw/mwworld/store.hpp
+++ b/apps/openmw/mwworld/store.hpp
@@ -11,6 +11,8 @@
 #include <components/esm/records.hpp>
 #include <components/misc/stringops.hpp>
 
+#include "../mwdialogue/keywordsearch.hpp"
+
 namespace ESM
 {
     struct Land;
@@ -154,13 +156,10 @@ namespace MWWorld
         /// @par mShared usually preserves the record order as it came from the content files (this
         /// is relevant for the spell autocalc code and selection order
         /// for heads/hairs in the character creation)
-        /// @warning ESM::Dialogue Store currently implements a sorted order for unknown reasons.
         std::vector<T*> mShared;
         typedef std::unordered_map<std::string, T, Misc::StringUtils::CiHash, Misc::StringUtils::CiEqual> Dynamic;
         Dynamic mDynamic;
 
-        uint64_t mModPoint;
-
         friend class ESMStore;
 
     public:
@@ -205,8 +204,6 @@ namespace MWWorld
         RecordId load(ESM::ESMReader &esm) override;
         void write(ESM::ESMWriter& writer, Loading::Listener& progress) const override;
         RecordId read(ESM::ESMReader& reader, bool overrideOnly = false) override;
-
-        uint64_t getModPoint() const { return mModPoint; }
     };
 
     template <>
@@ -444,6 +441,41 @@ namespace MWWorld
         iterator end() const;
     };
 
+    template <>
+    class Store<ESM::Dialogue> : public StoreBase
+    {
+        typedef std::unordered_map<std::string, ESM::Dialogue, Misc::StringUtils::CiHash, Misc::StringUtils::CiEqual> Static;
+        Static mStatic;
+        /// @par mShared usually preserves the record order as it came from the content files (this
+        /// is relevant for the spell autocalc code and selection order
+        /// for heads/hairs in the character creation)
+        /// @warning ESM::Dialogue Store currently implements a sorted order for unknown reasons.
+        std::vector<ESM::Dialogue*> mShared;
+
+        bool mKeywordSearchModFlag;
+        MWDialogue::KeywordSearch<std::string, int /*unused*/> mKeywordSearch;
+
+    public:
+        Store();
+
+        typedef SharedIterator<ESM::Dialogue> iterator;
+
+        void setUp() override;
+
+        const ESM::Dialogue *search(const std::string &id) const;
+        const ESM::Dialogue *find(const std::string &id) const;
+
+        iterator begin() const;
+        iterator end() const;
+
+        size_t getSize() const override;
+
+        bool eraseStatic(const std::string &id) override;
+
+        RecordId load(ESM::ESMReader &esm) override;
+
+        const MWDialogue::KeywordSearch<std::string, int>& getDialogIdKeywordSearch();
+    };
 
 } //end namespace
 
diff --git a/apps/openmw/mwworld/worldimp.hpp b/apps/openmw/mwworld/worldimp.hpp
index 84d97ce37c..bea7710a53 100644
--- a/apps/openmw/mwworld/worldimp.hpp
+++ b/apps/openmw/mwworld/worldimp.hpp
@@ -236,6 +236,9 @@ namespace MWWorld
 
             const MWWorld::ESMStore& getStore() const override;
 
+            const MWDialogue::KeywordSearch<std::string, int>& getDialogIdKeywordSearch() override {
+                return mStore.getDialogIdKeywordSearch(); }
+
             std::vector<ESM::ESMReader>& getEsmReader() override;
 
             LocalScripts& getLocalScripts() override;