Add ZoneDefinition Lexer

src/ZoneCommon/Parsing/ZoneDefinition/Matcher/ZoneDefinitionCommonMatchers.cpp

@@ -0,0 +1,58 @@
+#include "ZoneDefinitionCommonMatchers.h"
+
+#include <sstream>
+
+#include "Parsing/Simple/Matcher/SimpleMatcherFactory.h"
+
+std::unique_ptr<ZoneDefinitionCommonMatchers::matcher_t> ZoneDefinitionCommonMatchers::AssetName(const supplier_t* labelSupplier)
+{
+    const SimpleMatcherFactory create(labelSupplier);
+
+    return create.And({
+        create.Loop(create.Or({
+            create.Identifier(),
+            create.AnyCharBesides({',', '<', '>', '"', '\\', '*', '?', '|', ':'})
+        })),
+        create.Type(SimpleParserValueType::NEW_LINE).NoConsume()
+    }).Transform([](SimpleMatcherFactory::token_list_t& tokens)
+    {
+        std::ostringstream str;
+        auto previousType = SimpleParserValueType::INVALID;
+        auto previousCharacterWasSlash = false;
+
+        for (const auto& token : tokens)
+        {
+            switch (token.get().m_type)
+            {
+            case SimpleParserValueType::IDENTIFIER:
+                if (previousType == SimpleParserValueType::IDENTIFIER)
+                    str << " ";
+                str << token.get().IdentifierValue();
+                break;
+
+            case SimpleParserValueType::CHARACTER:
+                if (token.get().CharacterValue() == '/')
+                {
+                    if (previousType == SimpleParserValueType::CHARACTER && !previousCharacterWasSlash)
+                    {
+                        str << "/";
+                        previousCharacterWasSlash = true;
+                    }
+                }
+                else
+                {
+                    str << token.get().CharacterValue();
+                    previousCharacterWasSlash = false;
+                }
+                break;
+
+            default:
+                break;
+            }
+
+            previousType = token.get().m_type;
+        }
+
+        return SimpleParserValue::Identifier(tokens.front().get().GetPos(), new std::string(str.str()));
+    });
+}

src/ZoneCommon/Parsing/ZoneDefinition/Matcher/ZoneDefinitionCommonMatchers.h

@@ -0,0 +1,19 @@
+#pragma once
+
+#include <limits>
+#include <memory>
+
+#include "Parsing/Simple/SimpleParserValue.h"
+#include "Parsing/Matcher/AbstractMatcher.h"
+#include "Parsing/Matcher/MatcherLabel.h"
+
+class ZoneDefinitionCommonMatchers
+{
+public:
+    typedef AbstractMatcher<SimpleParserValue> matcher_t;
+    typedef IMatcherForLabelSupplier<SimpleParserValue> supplier_t;
+
+    static constexpr int LABEL_ASSET_NAME = std::numeric_limits<int>::max() - 1;
+    
+    static std::unique_ptr<matcher_t> AssetName(const supplier_t* labelSupplier);
+};

src/ZoneCommon/Parsing/ZoneDefinition/Sequence/SequenceZoneDefinitionConsumeEmptyLines.cpp

@@ -0,0 +1,14 @@
+#include "SequenceZoneDefinitionConsumeEmptyLines.h"
+
+#include "Parsing/Simple/Matcher/SimpleMatcherFactory.h"
+
+SequenceZoneDefinitionConsumeEmptyLines::SequenceZoneDefinitionConsumeEmptyLines()
+{
+    const SimpleMatcherFactory create(this);
+
+    AddMatchers(create.Loop(create.Type(SimpleParserValueType::NEW_LINE)));
+}
+
+void SequenceZoneDefinitionConsumeEmptyLines::ProcessMatch(ZoneDefinition* state, SequenceResult<SimpleParserValue>& result) const
+{
+}

src/ZoneCommon/Parsing/ZoneDefinition/Sequence/SequenceZoneDefinitionConsumeEmptyLines.h

@@ -0,0 +1,12 @@
+#pragma once
+
+#include "Parsing/ZoneDefinition/ZoneDefinitionParser.h"
+
+class SequenceZoneDefinitionConsumeEmptyLines final : public ZoneDefinitionParser::sequence_t
+{
+protected:
+    void ProcessMatch(ZoneDefinition* state, SequenceResult<SimpleParserValue>& result) const override;
+
+public:
+    SequenceZoneDefinitionConsumeEmptyLines();
+};

src/ZoneCommon/Parsing/ZoneDefinition/Sequence/SequenceZoneDefinitionEntry.cpp

@@ -1,16 +1,19 @@
 #include "SequenceZoneDefinitionEntry.h"
 
+#include "Parsing/ZoneDefinition/Matcher/ZoneDefinitionCommonMatchers.h"
 #include "Parsing/Simple/Matcher/SimpleMatcherFactory.h"
 
 SequenceZoneDefinitionEntry::SequenceZoneDefinitionEntry()
 {
     const SimpleMatcherFactory create(this);
 
+    AddLabeledMatchers(ZoneDefinitionCommonMatchers::AssetName(this), ZoneDefinitionCommonMatchers::LABEL_ASSET_NAME);
     AddMatchers({
         create.Identifier().Capture(CAPTURE_TYPE_NAME),
         create.Char(','),
         create.Optional(create.Char(',').Tag(TAG_REFERENCE)),
-        create.Identifier().Capture(CAPTURE_ASSET_NAME)
+        create.Label(ZoneDefinitionCommonMatchers::LABEL_ASSET_NAME).Capture(CAPTURE_ASSET_NAME),
+        create.Type(SimpleParserValueType::NEW_LINE)
     });
 }
 

src/ZoneCommon/Parsing/ZoneDefinition/Sequence/SequenceZoneDefinitionIgnore.cpp

@@ -9,8 +9,9 @@ SequenceZoneDefinitionIgnore::SequenceZoneDefinitionIgnore()
     AddMatchers({
         create.Keyword("ignore"),
         create.Char(','),
-        create.Identifier().Capture(CAPTURE_IGNORE_NAME)
-        });
+        create.Identifier().Capture(CAPTURE_IGNORE_NAME),
+        create.Type(SimpleParserValueType::NEW_LINE)
+    });
 }
 
 void SequenceZoneDefinitionIgnore::ProcessMatch(ZoneDefinition* state, SequenceResult<SimpleParserValue>& result) const

src/ZoneCommon/Parsing/ZoneDefinition/Sequence/SequenceZoneDefinitionInclude.cpp

@@ -9,7 +9,8 @@ SequenceZoneDefinitionInclude::SequenceZoneDefinitionInclude()
     AddMatchers({
         create.Keyword("include"),
         create.Char(','),
-        create.Identifier().Capture(CAPTURE_INCLUDE_NAME)
+        create.Identifier().Capture(CAPTURE_INCLUDE_NAME),
+        create.Type(SimpleParserValueType::NEW_LINE)
     });
 }
 

src/ZoneCommon/Parsing/ZoneDefinition/Sequence/SequenceZoneDefinitionMetaData.cpp

@@ -10,7 +10,8 @@ SequenceZoneDefinitionMetaData::SequenceZoneDefinitionMetaData()
         create.Char('>'),
         create.Identifier().Capture(CAPTURE_KEY),
         create.Char(','),
-        create.Identifier().Capture(CAPTURE_VALUE)
+        create.Identifier().Capture(CAPTURE_VALUE),
+        create.Type(SimpleParserValueType::NEW_LINE)
     });
 }
 

src/ZoneCommon/Parsing/ZoneDefinition/ZoneDefinitionLexer.cpp

@@ -0,0 +1,9 @@
+#include "ZoneDefinitionLexer.h"
+
+ZoneDefinitionLexer::ZoneDefinitionLexer(IParserLineStream* stream)
+    : SimpleLexer(stream)
+{
+    SetShouldEmitNewLineTokens(true);
+    SetShouldReadStrings(true);
+    SetShouldReadNumbers(false);
+}

src/ZoneCommon/Parsing/ZoneDefinition/ZoneDefinitionLexer.h

@@ -0,0 +1,9 @@
+#pragma once
+
+#include "Parsing/Simple/SimpleLexer.h"
+
+class ZoneDefinitionLexer final : public SimpleLexer
+{
+public:
+    explicit ZoneDefinitionLexer(IParserLineStream* stream);
+};

src/ZoneCommon/Parsing/ZoneDefinition/ZoneDefinitionParser.cpp

@@ -1,5 +1,7 @@
 #include "ZoneDefinitionParser.h"
 
+
+#include "Sequence/SequenceZoneDefinitionConsumeEmptyLines.h"
 #include "Sequence/SequenceZoneDefinitionEntry.h"
 #include "Sequence/SequenceZoneDefinitionIgnore.h"
 #include "Sequence/SequenceZoneDefinitionInclude.h"
@@ -16,7 +18,8 @@ const std::vector<AbstractParser<SimpleParserValue, ZoneDefinition>::sequence_t*
         new SequenceZoneDefinitionMetaData(),
         new SequenceZoneDefinitionInclude(),
         new SequenceZoneDefinitionIgnore(),
-        new SequenceZoneDefinitionEntry()
+        new SequenceZoneDefinitionEntry(),
+        new SequenceZoneDefinitionConsumeEmptyLines()
     });
 
     return tests;

src/ZoneCommon/Zone/Definition/ZoneDefinitionStream.cpp

@@ -7,6 +7,7 @@
 
 #include "Parsing/Impl/DefinesStreamProxy.h"
 #include "Parsing/Simple/SimpleLexer.h"
+#include "Parsing/ZoneDefinition/ZoneDefinitionLexer.h"
 #include "Parsing/ZoneDefinition/ZoneDefinitionParser.h"
 
 ZoneDefinitionInputStream::ZoneDefinitionInputStream(std::istream& stream, std::string fileName, bool verbose)
@@ -42,7 +43,7 @@ std::unique_ptr<ZoneDefinition> ZoneDefinitionInputStream::ReadDefinition()
         std::cout << "Reading zone definition file: " << m_file_name << std::endl;
     }
 
-    const auto lexer = std::make_unique<SimpleLexer>(m_stream);
+    const auto lexer = std::make_unique<ZoneDefinitionLexer>(m_stream);
     const auto parser = std::make_unique<ZoneDefinitionParser>(lexer.get());
 
     const auto start = std::chrono::steady_clock::now();