Add asset loader for localize files

src/ObjLoading/Parsing/LocalizeFile/LocalizeFileLexer.cpp

@@ -0,0 +1,9 @@
+#include "LocalizeFileLexer.h"
+
+LocalizeFileLexer::LocalizeFileLexer(IParserLineStream* stream)
+    : SimpleLexer(stream)
+{
+    SetShouldEmitNewLineTokens(true);
+    SetShouldReadNumbers(false);
+    SetShouldReadStrings(true);
+}

src/ObjLoading/Parsing/LocalizeFile/LocalizeFileLexer.h

@@ -0,0 +1,9 @@
+#pragma once
+
+#include "Parsing/Simple/SimpleLexer.h"
+
+class LocalizeFileLexer final : public SimpleLexer
+{
+public:
+    explicit LocalizeFileLexer(IParserLineStream* stream);
+};

src/ObjLoading/Parsing/LocalizeFile/LocalizeFileParser.cpp

@@ -0,0 +1,38 @@
+#include "LocalizeFileParser.h"
+
+#include "Sequence/SequenceLocalizeFileConfig.h"
+#include "Sequence/SequenceLocalizeFileConsumeEmptyLines.h"
+#include "Sequence/SequenceLocalizeFileEndMarker.h"
+#include "Sequence/SequenceLocalizeFileLanguageValue.h"
+#include "Sequence/SequenceLocalizeFileNotes.h"
+#include "Sequence/SequenceLocalizeFileReference.h"
+#include "Sequence/SequenceLocalizeFileVersion.h"
+
+LocalizeFileParser::LocalizeFileParser(SimpleLexer* lexer, GameLanguage language)
+    : AbstractParser(lexer, std::make_unique<LocalizeFileParserState>(language))
+{
+}
+
+const std::vector<AbstractParser<SimpleParserValue, LocalizeFileParserState>::sequence_t*>& LocalizeFileParser::GetTestsForState()
+{
+    static std::vector<sequence_t*> tests({
+        new SequenceLocalizeFileReference(),
+        new SequenceLocalizeFileConfig(),
+        new SequenceLocalizeFileNotes(),
+        new SequenceLocalizeFileVersion(),
+        new SequenceLocalizeFileEndMarker(),
+        new SequenceLocalizeFileLanguageValue(),
+        new SequenceLocalizeFileConsumeEmptyLines()
+    });
+
+    static std::vector<sequence_t*> noTests({
+        new SequenceLocalizeFileConsumeEmptyLines()
+    });
+
+    return !m_state->m_end ? tests : noTests;
+}
+
+std::vector<LocalizeFileEntry> LocalizeFileParser::GetParsedValues()
+{
+    return std::move(m_state->m_entries);
+}

src/ObjLoading/Parsing/LocalizeFile/LocalizeFileParser.h

@@ -0,0 +1,17 @@
+#pragma once
+
+#include "LocalizeFileParserState.h"
+#include "Game/GameLanguage.h"
+#include "Parsing/Simple/SimpleLexer.h"
+#include "Parsing/Simple/SimpleParserValue.h"
+#include "Parsing/Impl/AbstractParser.h"
+
+class LocalizeFileParser final : public AbstractParser<SimpleParserValue, LocalizeFileParserState>
+{
+protected:
+    const std::vector<sequence_t*>& GetTestsForState() override;
+
+public:
+    LocalizeFileParser(SimpleLexer* lexer, GameLanguage language);
+    std::vector<LocalizeFileEntry> GetParsedValues();
+};

src/ObjLoading/Parsing/LocalizeFile/LocalizeFileParserState.cpp

@@ -0,0 +1,12 @@
+#include "LocalizeFileParserState.h"
+
+#include "Localize/LocalizeCommon.h"
+
+LocalizeFileParserState::LocalizeFileParserState(const GameLanguage language)
+    : m_end(false),
+      m_language(language)
+{
+    m_language_name_caps = LocalizeCommon::GetNameOfLanguage(m_language);
+    for (auto& c : m_language_name_caps)
+        c = static_cast<char>(toupper(c));
+}

src/ObjLoading/Parsing/LocalizeFile/LocalizeFileParserState.h

@@ -0,0 +1,22 @@
+#pragma once
+#include <unordered_set>
+#include <vector>
+
+#include "Game/GameLanguage.h"
+#include "Localize/LocalizeFile.h"
+
+class LocalizeFileParserState
+{
+public:
+    bool m_end;
+
+    std::vector<LocalizeFileEntry> m_entries;
+
+    GameLanguage m_language;
+    std::string m_language_name_caps;
+
+    std::string m_current_reference;
+    std::unordered_set<std::string> m_current_reference_languages;
+
+    explicit LocalizeFileParserState(GameLanguage language);
+};

src/ObjLoading/Parsing/LocalizeFile/LocalizeFileReader.cpp

@@ -0,0 +1,41 @@
+#include "LocalizeFileReader.h"
+
+#include "LocalizeFileLexer.h"
+#include "LocalizeFileParser.h"
+#include "Parsing/Impl/CommentRemovingStreamProxy.h"
+#include "Parsing/Impl/ParserInputStream.h"
+
+LocalizeFileReader::LocalizeFileReader(std::istream& stream, std::string fileName, GameLanguage language)
+    : m_file_name(std::move(fileName)),
+      m_stream(nullptr),
+      m_language(language)
+{
+    OpenBaseStream(stream);
+    SetupStreamProxies();
+    m_stream = m_open_streams.back().get();
+}
+
+bool LocalizeFileReader::OpenBaseStream(std::istream& stream)
+{
+    m_open_streams.emplace_back(std::make_unique<ParserInputStream>(stream, m_file_name));
+    return true;
+}
+
+void LocalizeFileReader::SetupStreamProxies()
+{
+    m_open_streams.emplace_back(std::make_unique<CommentRemovingStreamProxy>(m_open_streams.back().get()));
+
+    m_stream = m_open_streams.back().get();
+}
+
+std::vector<LocalizeFileEntry> LocalizeFileReader::ReadLocalizeFile()
+{
+    const auto lexer = std::make_unique<LocalizeFileLexer>(m_stream);
+    const auto parser = std::make_unique<LocalizeFileParser>(lexer.get(), m_language);
+
+    if (parser->Parse())
+        return parser->GetParsedValues();
+
+    std::cout << "Parsing localization file failed!" << std::endl;
+    return std::vector<LocalizeFileEntry>();
+}

src/ObjLoading/Parsing/LocalizeFile/LocalizeFileReader.h

@@ -0,0 +1,25 @@
+#pragma once
+
+#include <memory>
+#include <string>
+#include <vector>
+
+#include "Game/GameLanguage.h"
+#include "Localize/LocalizeFile.h"
+#include "Parsing/IParserLineStream.h"
+
+class LocalizeFileReader
+{
+    std::string m_file_name;
+    IParserLineStream* m_stream;
+    std::vector<std::unique_ptr<IParserLineStream>> m_open_streams;
+    GameLanguage m_language;
+
+    bool OpenBaseStream(std::istream& stream);
+    void SetupStreamProxies();
+
+public:
+    LocalizeFileReader(std::istream& stream, std::string fileName, GameLanguage language);
+
+    std::vector<LocalizeFileEntry> ReadLocalizeFile();
+};

src/ObjLoading/Parsing/LocalizeFile/Sequence/SequenceLocalizeFileConfig.cpp

@@ -0,0 +1,18 @@
+#include "SequenceLocalizeFileConfig.h"
+
+#include "Parsing/Simple/Matcher/SimpleMatcherFactory.h"
+
+SequenceLocalizeFileConfig::SequenceLocalizeFileConfig()
+{
+    const SimpleMatcherFactory create(this);
+
+    AddMatchers({
+        create.Keyword("CONFIG"),
+        create.String(),
+        create.Type(SimpleParserValueType::NEW_LINE)
+    });
+}
+
+void SequenceLocalizeFileConfig::ProcessMatch(LocalizeFileParserState* state, SequenceResult<SimpleParserValue>& result) const
+{
+}

src/ObjLoading/Parsing/LocalizeFile/Sequence/SequenceLocalizeFileConfig.h

@@ -0,0 +1,12 @@
+#pragma once
+
+#include "Parsing/LocalizeFile/LocalizeFileParser.h"
+
+class SequenceLocalizeFileConfig final : public LocalizeFileParser::sequence_t
+{
+protected:
+    void ProcessMatch(LocalizeFileParserState* state, SequenceResult<SimpleParserValue>& result) const override;
+
+public:
+    SequenceLocalizeFileConfig();
+};

src/ObjLoading/Parsing/LocalizeFile/Sequence/SequenceLocalizeFileConsumeEmptyLines.cpp

@@ -0,0 +1,16 @@
+#include "SequenceLocalizeFileConsumeEmptyLines.h"
+
+#include "Parsing/Simple/Matcher/SimpleMatcherFactory.h"
+
+SequenceLocalizeFileConsumeEmptyLines::SequenceLocalizeFileConsumeEmptyLines()
+{
+    const SimpleMatcherFactory create(this);
+
+    AddMatchers({
+        create.Type(SimpleParserValueType::NEW_LINE)
+    });
+}
+
+void SequenceLocalizeFileConsumeEmptyLines::ProcessMatch(LocalizeFileParserState* state, SequenceResult<SimpleParserValue>& result) const
+{
+}

src/ObjLoading/Parsing/LocalizeFile/Sequence/SequenceLocalizeFileConsumeEmptyLines.h

@@ -0,0 +1,12 @@
+#pragma once
+
+#include "Parsing/LocalizeFile/LocalizeFileParser.h"
+
+class SequenceLocalizeFileConsumeEmptyLines final : public LocalizeFileParser::sequence_t
+{
+protected:
+    void ProcessMatch(LocalizeFileParserState* state, SequenceResult<SimpleParserValue>& result) const override;
+
+public:
+    SequenceLocalizeFileConsumeEmptyLines();
+};

src/ObjLoading/Parsing/LocalizeFile/Sequence/SequenceLocalizeFileEndMarker.cpp

@@ -0,0 +1,17 @@
+#include "SequenceLocalizeFileEndMarker.h"
+
+#include "Parsing/Simple/Matcher/SimpleMatcherFactory.h"
+
+SequenceLocalizeFileEndMarker::SequenceLocalizeFileEndMarker()
+{
+    const SimpleMatcherFactory create(this);
+
+    AddMatchers({
+        create.Keyword("ENDMARKER")
+    });
+}
+
+void SequenceLocalizeFileEndMarker::ProcessMatch(LocalizeFileParserState* state, SequenceResult<SimpleParserValue>& result) const
+{
+    state->m_end = true;
+}

src/ObjLoading/Parsing/LocalizeFile/Sequence/SequenceLocalizeFileEndMarker.h

@@ -0,0 +1,12 @@
+#pragma once
+
+#include "Parsing/LocalizeFile/LocalizeFileParser.h"
+
+class SequenceLocalizeFileEndMarker final : public LocalizeFileParser::sequence_t
+{
+protected:
+    void ProcessMatch(LocalizeFileParserState* state, SequenceResult<SimpleParserValue>& result) const override;
+
+public:
+    SequenceLocalizeFileEndMarker();
+};

src/ObjLoading/Parsing/LocalizeFile/Sequence/SequenceLocalizeFileLanguageValue.cpp

@@ -0,0 +1,72 @@
+#include "SequenceLocalizeFileLanguageValue.h"
+
+#include <sstream>
+
+#include "Parsing/Simple/Matcher/SimpleMatcherFactory.h"
+
+SequenceLocalizeFileLanguageValue::SequenceLocalizeFileLanguageValue()
+{
+    const SimpleMatcherFactory create(this);
+
+    AddMatchers({
+        create.KeywordPrefix("LANG_").Capture(CAPTURE_LANGUAGE_NAME),
+        create.String().Capture(CAPTURE_ENTRY_VALUE),
+        create.Type(SimpleParserValueType::NEW_LINE)
+    });
+}
+
+std::string SequenceLocalizeFileLanguageValue::UnescapeValue(const std::string& value)
+{
+    std::ostringstream str;
+
+    auto isEscaped = false;
+    for(auto c : value)
+    {
+        if(isEscaped)
+        {
+            switch(c)
+            {
+            case 'n':
+                str << '\n';
+                break;
+
+            case 'r':
+                str << '\r';
+                break;
+
+            default:
+                str << c;
+                break;
+            }
+        }
+        else if(c == '\\')
+        {
+            isEscaped = true;
+        }
+        else
+        {
+            str << c;
+        }
+    }
+
+    return str.str();
+}
+
+void SequenceLocalizeFileLanguageValue::ProcessMatch(LocalizeFileParserState* state, SequenceResult<SimpleParserValue>& result) const
+{
+    const auto& langToken = result.NextCapture(CAPTURE_LANGUAGE_NAME);
+    const auto& valueToken = result.NextCapture(CAPTURE_ENTRY_VALUE);
+
+    const auto langName = langToken.IdentifierValue().substr(std::char_traits<char>::length("LANG_"));
+    const auto alreadyDefinedLanguage = state->m_current_reference_languages.find(langName);
+    if(alreadyDefinedLanguage != state->m_current_reference_languages.end())
+    {
+        std::ostringstream str;
+        str << "Value for reference \"" << state->m_current_reference << "\" already defined for language \"" << langToken.IdentifierValue() << "\"";
+        throw ParsingException(langToken.GetPos(), str.str());
+    }
+    state->m_current_reference_languages.emplace(langName);
+
+    if(langName == state->m_language_name_caps)
+        state->m_entries.emplace_back(state->m_current_reference, UnescapeValue(valueToken.StringValue()));
+}

src/ObjLoading/Parsing/LocalizeFile/Sequence/SequenceLocalizeFileLanguageValue.h

@@ -0,0 +1,17 @@
+#pragma once
+
+#include "Parsing/LocalizeFile/LocalizeFileParser.h"
+
+class SequenceLocalizeFileLanguageValue final : public LocalizeFileParser::sequence_t
+{
+    static constexpr auto CAPTURE_LANGUAGE_NAME = 1;
+    static constexpr auto CAPTURE_ENTRY_VALUE = 2;
+
+    static std::string UnescapeValue(const std::string& value);
+
+protected:
+    void ProcessMatch(LocalizeFileParserState* state, SequenceResult<SimpleParserValue>& result) const override;
+
+public:
+    SequenceLocalizeFileLanguageValue();
+};

src/ObjLoading/Parsing/LocalizeFile/Sequence/SequenceLocalizeFileNotes.cpp

@@ -0,0 +1,18 @@
+#include "SequenceLocalizeFileNotes.h"
+
+#include "Parsing/Simple/Matcher/SimpleMatcherFactory.h"
+
+SequenceLocalizeFileNotes::SequenceLocalizeFileNotes()
+{
+    const SimpleMatcherFactory create(this);
+
+    AddMatchers({
+        create.Keyword("FILENOTES"),
+        create.String(),
+        create.Type(SimpleParserValueType::NEW_LINE)
+    });
+}
+
+void SequenceLocalizeFileNotes::ProcessMatch(LocalizeFileParserState* state, SequenceResult<SimpleParserValue>& result) const
+{
+}

src/ObjLoading/Parsing/LocalizeFile/Sequence/SequenceLocalizeFileNotes.h

@@ -0,0 +1,12 @@
+#pragma once
+
+#include "Parsing/LocalizeFile/LocalizeFileParser.h"
+
+class SequenceLocalizeFileNotes final : public LocalizeFileParser::sequence_t
+{
+protected:
+    void ProcessMatch(LocalizeFileParserState* state, SequenceResult<SimpleParserValue>& result) const override;
+
+public:
+    SequenceLocalizeFileNotes();
+};

src/ObjLoading/Parsing/LocalizeFile/Sequence/SequenceLocalizeFileReference.cpp

@@ -0,0 +1,20 @@
+#include "SequenceLocalizeFileReference.h"
+
+#include "Parsing/Simple/Matcher/SimpleMatcherFactory.h"
+
+SequenceLocalizeFileReference::SequenceLocalizeFileReference()
+{
+    const SimpleMatcherFactory create(this);
+
+    AddMatchers({
+        create.Keyword("REFERENCE"),
+        create.Identifier().Capture(CAPTURE_REFERENCE_NAME),
+        create.Type(SimpleParserValueType::NEW_LINE)
+    });
+}
+
+void SequenceLocalizeFileReference::ProcessMatch(LocalizeFileParserState* state, SequenceResult<SimpleParserValue>& result) const
+{
+    state->m_current_reference = result.NextCapture(CAPTURE_REFERENCE_NAME).IdentifierValue();
+    state->m_current_reference_languages.clear();
+}

src/ObjLoading/Parsing/LocalizeFile/Sequence/SequenceLocalizeFileReference.h

@@ -0,0 +1,14 @@
+#pragma once
+
+#include "Parsing/LocalizeFile/LocalizeFileParser.h"
+
+class SequenceLocalizeFileReference final : public LocalizeFileParser::sequence_t
+{
+    static constexpr auto CAPTURE_REFERENCE_NAME = 1;
+
+protected:
+    void ProcessMatch(LocalizeFileParserState* state, SequenceResult<SimpleParserValue>& result) const override;
+
+public:
+    SequenceLocalizeFileReference();
+};

src/ObjLoading/Parsing/LocalizeFile/Sequence/SequenceLocalizeFileVersion.cpp

@@ -0,0 +1,23 @@
+#include "SequenceLocalizeFileVersion.h"
+
+#include "Parsing/Simple/Matcher/SimpleMatcherFactory.h"
+
+SequenceLocalizeFileVersion::SequenceLocalizeFileVersion()
+{
+    const SimpleMatcherFactory create(this);
+
+    AddMatchers({
+        create.Keyword("VERSION"),
+        create.String().Capture(CAPTURE_VERSION),
+        create.Type(SimpleParserValueType::NEW_LINE)
+    });
+}
+
+void SequenceLocalizeFileVersion::ProcessMatch(LocalizeFileParserState* state, SequenceResult<SimpleParserValue>& result) const
+{
+    const auto& versionCapture = result.NextCapture(CAPTURE_VERSION);
+    if(versionCapture.StringValue() != "1")
+    {
+        throw ParsingException(versionCapture.GetPos(), "Localize file needs to be version 1");
+    }
+}

src/ObjLoading/Parsing/LocalizeFile/Sequence/SequenceLocalizeFileVersion.h

@@ -0,0 +1,14 @@
+#pragma once
+
+#include "Parsing/LocalizeFile/LocalizeFileParser.h"
+
+class SequenceLocalizeFileVersion final : public LocalizeFileParser::sequence_t
+{
+    static constexpr auto CAPTURE_VERSION = 1;
+
+protected:
+    void ProcessMatch(LocalizeFileParserState* state, SequenceResult<SimpleParserValue>& result) const override;
+
+public:
+    SequenceLocalizeFileVersion();
+};