INSANEMODE/OpenAssetTools
1
0
Fork 0
mirror of https://github.com/Laupetin/OpenAssetTools.git synced 2025-06-09 22:38:06 -05:00
Code Issues Packages Projects Releases Wiki Activity

Make sure captures work with transforms

Browse Source
This commit is contained in:
Jan
2021-02-13 16:01:58 +01:00
parent 37232e3176
commit 32f815c378
5 changed files with 207 additions and 44 deletions
Download Patch File Download Diff File Expand all files Collapse all files

43
src/ZoneCodeGeneratorLib/Parsing/Matcher/AbstractMatcher.h
View File

@ -15,11 +15,13 @@ class AbstractMatcher
public:
static constexpr int NO_ID = -1;
typedef std::vector<std::reference_wrapper<const TokenType>> token_list_t;
private:
int m_tag_id;
int m_capture_id;
bool m_no_consume;
std::function<TokenType(std::vector<std::reference_wrapper<const TokenType>>&)> m_transform_func;
std::function<TokenType(token_list_t&)> m_transform_func;
protected:
AbstractMatcher()
@ -68,26 +70,29 @@ public:
if (m_tag_id != NO_ID)
result.m_tags.push_back(m_tag_id);
if (m_transform_func)
{
std::vector<std::reference_wrapper<const TokenType>> tokens;
tokens.reserve(result.m_consumed_token_count);
for (auto i = 0u; i < result.m_consumed_token_count; i++)
tokens.emplace_back(lexer->GetToken(tokenOffset + i));
result.m_fabricated_tokens.push_back(m_transform_func(tokens));
result.m_matched_tokens.clear();
result.m_matched_tokens.emplace_back(result.m_fabricated_tokens.size() - 1, true);
}
else if(result.m_matched_tokens.empty())
{
for (auto i = 0u; i < result.m_consumed_token_count; i++)
result.m_matched_tokens.emplace_back(tokenOffset + i, false);
}
if (m_capture_id != NO_ID)
{
if (m_transform_func)
{
std::vector<std::reference_wrapper<const TokenType>> tokens;
tokens.reserve(result.m_consumed_token_count);
for (auto i = 0u; i < result.m_consumed_token_count; i++)
tokens.emplace_back(lexer->GetToken(tokenOffset + i));
result.m_fabricated_tokens.push_back(m_transform_func(tokens));
result.m_captures.emplace_back(m_capture_id, result.m_fabricated_tokens.size() - 1, true);
}
else
{
for (auto i = 0u; i < result.m_consumed_token_count; i++)
{
result.m_captures.emplace_back(m_capture_id, tokenOffset + i);
}
}
for (const auto& match : result.m_matched_tokens)
result.m_captures.emplace_back(m_capture_id, match);
}
return result;

6
src/ZoneCodeGeneratorLib/Parsing/Matcher/AbstractMatcherFactory.h
View File

@ -20,6 +20,8 @@ class MatcherFactoryWrapper
std::unique_ptr<AbstractMatcher<TokenType>> m_matcher;
public:
typedef typename AbstractMatcher<TokenType>::token_list_t token_list_t;
explicit MatcherFactoryWrapper(std::unique_ptr<AbstractMatcher<TokenType>> matcher)
: m_matcher(std::move(matcher))
{
@ -43,7 +45,7 @@ public:
return *this;
}
MatcherFactoryWrapper<TokenType>& Transform(std::function<TokenType(std::vector<std::reference_wrapper<const TokenType>>&)> transform)
MatcherFactoryWrapper<TokenType>& Transform(std::function<TokenType(token_list_t&)> transform)
{
m_matcher->SetTransform(std::move(transform));
return *this;
@ -76,6 +78,8 @@ class AbstractMatcherFactory
const IMatcherForLabelSupplier<TokenType>* m_label_supplier;
public:
typedef typename AbstractMatcher<TokenType>::token_list_t token_list_t;
explicit AbstractMatcherFactory(const IMatcherForLabelSupplier<TokenType>* labelSupplier)
: m_label_supplier(labelSupplier)
{

66
src/ZoneCodeGeneratorLib/Parsing/Matcher/MatcherResult.h
View File

@ -13,29 +13,19 @@ class MatcherResult
static_assert(std::is_base_of<IParserValue, TokenType>::value);
public:
class Capture
class TokenIndex
{
static constexpr unsigned FABRICATED_FLAG_MASK = std::numeric_limits<unsigned>::max() ^ std::numeric_limits<int>::max();
static constexpr unsigned TOKEN_INDEX_MASK = ~FABRICATED_FLAG_MASK;
int m_capture_id;
unsigned m_token_index;
public:
Capture(const int captureId, const unsigned tokenIndex)
: Capture(captureId, tokenIndex, false)
TokenIndex(const unsigned index, const bool isFabricated)
{
}
Capture(const int captureId, const unsigned tokenIndex, const bool isFabricated)
: m_capture_id(captureId),
m_token_index(!isFabricated ? tokenIndex : tokenIndex | FABRICATED_FLAG_MASK)
{
}
_NODISCARD int GetCaptureId() const
{
return m_capture_id;
m_token_index = index & TOKEN_INDEX_MASK;
if (isFabricated)
m_token_index |= FABRICATED_FLAG_MASK;
}
_NODISCARD bool IsFabricated() const
@ -49,10 +39,40 @@ public:
}
};
class Capture
{
public:
int m_capture_id;
TokenIndex m_token_index;
Capture(const int captureId, const unsigned tokenIndex)
: Capture(captureId, tokenIndex, false)
{
}
Capture(const int captureId, const unsigned tokenIndex, const bool isFabricated)
: m_capture_id(captureId),
m_token_index(tokenIndex, isFabricated)
{
}
Capture(const int captureId, const TokenIndex index)
: m_capture_id(captureId),
m_token_index(index)
{
}
_NODISCARD int GetCaptureId() const
{
return m_capture_id;
}
};
bool m_matches;
unsigned m_consumed_token_count;
std::vector<int> m_tags;
std::vector<Capture> m_captures;
std::vector<TokenIndex> m_matched_tokens;
std::vector<TokenType> m_fabricated_tokens;
private:
@ -82,13 +102,21 @@ public:
for (const auto& capture : other.m_captures)
{
if (capture.IsFabricated())
m_captures.emplace_back(capture.GetCaptureId(), m_fabricated_tokens.size() + capture.GetTokenIndex(), true);
if (capture.m_token_index.IsFabricated())
m_captures.emplace_back(capture.GetCaptureId(), TokenIndex(m_fabricated_tokens.size() + capture.m_token_index.GetTokenIndex(), true));
else
m_captures.emplace_back(capture.GetCaptureId(), capture.GetTokenIndex());
m_captures.emplace_back(capture.GetCaptureId(), capture.m_token_index);
}
for(auto& fabricated : other.m_fabricated_tokens)
for (const auto& token : other.m_matched_tokens)
{
if (token.IsFabricated())
m_matched_tokens.emplace_back(m_fabricated_tokens.size() + token.GetTokenIndex(), true);
else
m_matched_tokens.emplace_back(token.GetTokenIndex(), false);
}
for (auto& fabricated : other.m_fabricated_tokens)
{
m_fabricated_tokens.emplace_back(std::move(fabricated));
}

6
src/ZoneCodeGeneratorLib/Parsing/Sequence/SequenceResult.h
View File

@ -37,10 +37,10 @@ public:
{
for (const typename MatcherResult<TokenType>::Capture& capture : result.m_captures)
{
if (capture.IsFabricated())
m_captures[capture.GetCaptureId()].m_tokens.push_back(result.m_fabricated_tokens[capture.GetTokenIndex()]);
if (capture.m_token_index.IsFabricated())
m_captures[capture.GetCaptureId()].m_tokens.push_back(result.m_fabricated_tokens[capture.m_token_index.GetTokenIndex()]);
else
m_captures[capture.GetCaptureId()].m_tokens.push_back(lexer->GetToken(capture.GetTokenIndex()));
m_captures[capture.GetCaptureId()].m_tokens.push_back(lexer->GetToken(capture.m_token_index.GetTokenIndex()));
}
}