INSANEMODE/OpenAssetTools
1
0
Fork 0
mirror of https://github.com/Laupetin/OpenAssetTools.git synced 2025-06-10 14:58:10 -05:00
Code Issues Packages Projects Releases Wiki Activity

Add multicharacter tokens to simple lexer

Browse Source
This commit is contained in:
Jan
2021-11-03 22:02:04 +01:00
parent ab7b516918
commit c5d7d71a51
11 changed files with 181 additions and 11 deletions
Download Patch File Download Diff File Expand all files Collapse all files

7
src/ObjLoading/Parsing/LocalizeFile/LocalizeFileReader.cpp
View File

@ -29,7 +29,12 @@ void LocalizeFileReader::SetupStreamProxies()
std::vector<LocalizeFileEntry> LocalizeFileReader::ReadLocalizeFile()
{
const auto lexer = std::make_unique<SimpleLexer>(m_stream, SimpleLexer::Config{true, true, false});
SimpleLexer::Config lexerConfig;
lexerConfig.m_emit_new_line_tokens = true;
lexerConfig.m_read_strings = true;
lexerConfig.m_read_numbers = false;
const auto lexer = std::make_unique<SimpleLexer>(m_stream, std::move(lexerConfig));
const auto parser = std::make_unique<LocalizeFileParser>(lexer.get(), m_language);
if (parser->Parse())

7
src/ObjLoading/Parsing/Menu/MenuFileReader.cpp
View File

@ -111,7 +111,12 @@ std::unique_ptr<ParsingResult> MenuFileReader::CreateParsingResult(MenuFileParse
std::unique_ptr<ParsingResult> MenuFileReader::ReadMenuFile()
{
const auto lexer = std::make_unique<SimpleLexer>(m_stream, SimpleLexer::Config{false, true, true});
SimpleLexer::Config lexerConfig;
lexerConfig.m_emit_new_line_tokens = false;
lexerConfig.m_read_strings = true;
lexerConfig.m_read_numbers = true;
const auto lexer = std::make_unique<SimpleLexer>(m_stream, std::move(lexerConfig));
const auto parser = std::make_unique<MenuFileParser>(lexer.get(), m_feature_level);
if (!parser->Parse())