Import code from previous AssetBuilder version

This commit is contained in:
Jan
2019-09-24 10:45:09 +02:00
parent 5609557516
commit 0d8432d4f7
919 changed files with 154412 additions and 26 deletions

View File

@ -0,0 +1,353 @@
using System.Collections.Generic;
using Microsoft.VisualStudio.TestTools.UnitTesting;
using ZoneCodeGenerator.Parsing.Impl;
using ZoneCodeGeneratorTests.Parsing.Mock;
namespace ZoneCodeGeneratorTests.Parsing.Impl
{
[TestClass]
public class LexerTest
{
private IncludingParsingStreamTest headerStream;
private Lexer lexer;
[TestInitialize]
public void Setup()
{
headerStream = new IncludingParsingStreamTest("file.h");
lexer = new Lexer(headerStream);
}
private void AssertTokenOutput(params string[] expectedTokens)
{
foreach (var token in expectedTokens)
Assert.AreEqual(token, lexer.NextToken());
}
[TestMethod]
public void EnsureCanSplitWords()
{
headerStream.Lines.AddRange(new List<string>
{
"Some random tokens",
"right here"
});
AssertTokenOutput("Some", "random", "tokens", "right", "here");
Assert.IsTrue(lexer.IsEndOfStream);
}
[TestMethod]
public void EnsureCanSplitSpecialCharactersFromWords()
{
headerStream.Lines.AddRange(new List<string>
{
"word1;word2; word3 ; word4 ;word5",
"word1:word2: word3 : word4 :word5",
"word1{word2{ word3 { word4 {word5",
"word1}word2} word3 } word4 }word5",
"word1,word2, word3 , word4 ,word5",
"word1=word2= word3 = word4 =word5",
"word1(word2( word3 ( word4 (word5",
"word1)word2) word3 ) word4 )word5",
"word1*word2* word3 * word4 *word5",
"word1[word2[ word3 [ word4 [word5",
"word1]word2] word3 ] word4 ]word5",
"word1<word2< word3 < word4 <word5",
"word1>word2> word3 > word4 >word5",
"word1#word2# word3 # word4 #word5",
"word1&word2& word3 & word4 &word5",
"word1%word2% word3 % word4 %word5",
"word1\"word2\" word3 \" word4 \"word5",
"word1?word2? word3 ? word4 ?word5",
"word1!word2! word3 ! word4 !word5",
"word1+word2+ word3 + word4 +word5",
"word1-word2- word3 - word4 -word5",
"word1/word2/ word3 / word4 /word5",
"word1\\word2\\ word3 \\ word4 \\word5",
});
AssertTokenOutput("word1", ";", "word2", ";", "word3", ";", "word4", ";", "word5");
AssertTokenOutput("word1", ":", "word2", ":", "word3", ":", "word4", ":", "word5");
AssertTokenOutput("word1", "{", "word2", "{", "word3", "{", "word4", "{", "word5");
AssertTokenOutput("word1", "}", "word2", "}", "word3", "}", "word4", "}", "word5");
AssertTokenOutput("word1", ",", "word2", ",", "word3", ",", "word4", ",", "word5");
AssertTokenOutput("word1", "=", "word2", "=", "word3", "=", "word4", "=", "word5");
AssertTokenOutput("word1", "(", "word2", "(", "word3", "(", "word4", "(", "word5");
AssertTokenOutput("word1", ")", "word2", ")", "word3", ")", "word4", ")", "word5");
AssertTokenOutput("word1", "*", "word2", "*", "word3", "*", "word4", "*", "word5");
AssertTokenOutput("word1", "[", "word2", "[", "word3", "[", "word4", "[", "word5");
AssertTokenOutput("word1", "]", "word2", "]", "word3", "]", "word4", "]", "word5");
AssertTokenOutput("word1", "<", "word2", "<", "word3", "<", "word4", "<", "word5");
AssertTokenOutput("word1", ">", "word2", ">", "word3", ">", "word4", ">", "word5");
AssertTokenOutput("word1", "#", "word2", "#", "word3", "#", "word4", "#", "word5");
AssertTokenOutput("word1", "&", "word2", "&", "word3", "&", "word4", "&", "word5");
AssertTokenOutput("word1", "%", "word2", "%", "word3", "%", "word4", "%", "word5");
AssertTokenOutput("word1", "\"", "word2", "\"", "word3", "\"", "word4", "\"", "word5");
AssertTokenOutput("word1", "?", "word2", "?", "word3", "?", "word4", "?", "word5");
AssertTokenOutput("word1", "!", "word2", "!", "word3", "!", "word4", "!", "word5");
AssertTokenOutput("word1", "+", "word2", "+", "word3", "+", "word4", "+", "word5");
AssertTokenOutput("word1", "-", "word2", "-", "word3", "-", "word4", "-", "word5");
AssertTokenOutput("word1", "/", "word2", "/", "word3", "/", "word4", "/", "word5");
AssertTokenOutput("word1", "\\", "word2", "\\", "word3", "\\", "word4", "\\", "word5");
Assert.IsTrue(lexer.IsEndOfStream);
}
[TestMethod]
public void EnsurePeekingDoesNotConsumeTokens()
{
headerStream.Lines.AddRange(new List<string>
{
"Some random tokens",
"right here"
});
Assert.AreEqual("Some", lexer.PeekToken());
Assert.AreEqual("tokens", lexer.PeekToken(2));
Assert.AreEqual("here", lexer.PeekToken(4));
Assert.AreEqual("right", lexer.PeekToken(3));
Assert.AreEqual("Some", lexer.NextToken());
Assert.AreEqual("right", lexer.PeekToken(2));
AssertTokenOutput("random", "tokens", "right");
Assert.AreEqual("here", lexer.PeekToken());
Assert.AreEqual("here", lexer.PeekToken());
Assert.AreEqual("here", lexer.PeekToken());
Assert.AreEqual("here", lexer.NextToken());
Assert.IsTrue(lexer.IsEndOfStream);
}
[TestMethod]
public void EnsurePeekingOutOfBoundsDoesNotCauseIssues()
{
headerStream.Lines.AddRange(new List<string>
{
"Some random tokens",
"right here"
});
Assert.AreEqual("", lexer.PeekToken(1337));
Assert.AreEqual("", lexer.PeekToken(5));
Assert.AreEqual("", lexer.PeekToken(-2));
AssertTokenOutput("Some", "random", "tokens", "right", "here");
Assert.IsTrue(lexer.IsEndOfStream);
Assert.AreEqual("", lexer.PeekToken());
Assert.AreEqual("", lexer.PeekToken(2));
}
[TestMethod]
public void EnsureGettingNextTokenAfterStreamEndDoesNotCauseIssues()
{
headerStream.Lines.AddRange(new List<string>
{
"Some random tokens",
"right here"
});
AssertTokenOutput("Some", "random", "tokens", "right", "here");
Assert.IsTrue(lexer.IsEndOfStream);
Assert.AreEqual("", lexer.NextToken());
Assert.AreEqual("", lexer.NextToken());
}
[TestMethod]
public void EnsureSkippingTokensWorksCorrectly()
{
headerStream.Lines.AddRange(new List<string>
{
"Some random tokens",
"right here"
});
AssertTokenOutput("Some", "random");
Assert.IsFalse(lexer.IsEndOfStream);
lexer.SkipTokens(2);
AssertTokenOutput("here");
Assert.IsTrue(lexer.IsEndOfStream);
}
[TestMethod]
public void EnsureSkippingNoTokensWorksCorrectly()
{
headerStream.Lines.AddRange(new List<string>
{
"Some random tokens",
"right here"
});
AssertTokenOutput("Some", "random");
Assert.IsFalse(lexer.IsEndOfStream);
lexer.SkipTokens(0);
AssertTokenOutput("tokens", "right", "here");
Assert.IsTrue(lexer.IsEndOfStream);
}
[TestMethod]
public void EnsureSkippingOutOfBoundsDoesNotCauseIssues()
{
headerStream.Lines.AddRange(new List<string>
{
"Some random tokens",
"right here"
});
AssertTokenOutput("Some", "random");
Assert.IsFalse(lexer.IsEndOfStream);
lexer.SkipTokens(-5);
AssertTokenOutput("tokens", "right");
lexer.SkipTokens(1337);
Assert.IsTrue(lexer.IsEndOfStream);
AssertTokenOutput("");
}
[TestMethod]
public void EnsureSkippingAfterStreamEndDoesNotCauseIssues()
{
headerStream.Lines.AddRange(new List<string>
{
"Some random tokens",
"right here"
});
AssertTokenOutput("Some", "random", "tokens", "right", "here");
Assert.IsTrue(lexer.IsEndOfStream);
lexer.SkipTokens(1);
lexer.SkipTokens(2);
Assert.IsTrue(lexer.IsEndOfStream);
}
[TestMethod]
public void EnsureTheFilenameOfTheTokenIsCorrect()
{
headerStream.Lines.AddRange(new List<string>
{
"Some",
"random",
"tokens",
"right",
"here"
});
Assert.AreEqual("file.h", lexer.CurrentFile);
AssertTokenOutput("Some");
Assert.AreEqual("file.h", lexer.CurrentFile);
headerStream.Filename = "file2.h";
AssertTokenOutput("random");
headerStream.Filename = "file3.h";
Assert.AreEqual("file3.h", lexer.CurrentFile);
}
[TestMethod]
public void EnsureFilenameOfTokenStaysConsistent()
{
headerStream.Lines.AddRange(new List<string>
{
"Some",
"random",
"tokens",
"right",
"here"
});
Assert.AreEqual("file.h", lexer.CurrentFile);
Assert.AreEqual("Some", lexer.PeekToken());
headerStream.Filename = "file2.h";
Assert.AreEqual("file.h", lexer.CurrentFile);
Assert.AreEqual("Some", lexer.PeekToken());
Assert.AreEqual("file.h", lexer.CurrentFile);
Assert.AreEqual("Some", lexer.NextToken());
Assert.AreEqual("file2.h", lexer.CurrentFile);
AssertTokenOutput("random", "tokens", "right", "here");
Assert.IsTrue(lexer.IsEndOfStream);
}
[TestMethod]
public void EnsureTheLineOfTheTokenIsCorrect()
{
headerStream.Lines.AddRange(new List<string>
{
"Some",
"random",
"tokens",
"right",
"here"
});
Assert.AreEqual(0, lexer.CurrentLine);
AssertTokenOutput("Some");
Assert.AreEqual(1, lexer.CurrentLine);
AssertTokenOutput("random");
Assert.AreEqual(2, lexer.CurrentLine);
lexer.SkipTokens(1);
Assert.AreEqual(3, lexer.CurrentLine);
}
[TestMethod]
public void EnsureLineOfTokenStaysConsistent()
{
headerStream.Lines.AddRange(new List<string>
{
"Some",
"random",
"tokens",
"right",
"here"
});
Assert.AreEqual(0, lexer.CurrentLine);
Assert.AreEqual("Some", lexer.PeekToken());
Assert.AreEqual("random", lexer.PeekToken(1));
headerStream.Line = 3;
Assert.AreEqual(0, lexer.CurrentLine);
Assert.AreEqual("Some", lexer.PeekToken());
Assert.AreEqual("Some", lexer.NextToken());
Assert.AreEqual(1, lexer.CurrentLine);
Assert.AreEqual("random", lexer.PeekToken());
Assert.AreEqual("random", lexer.NextToken());
Assert.AreEqual(3, lexer.CurrentLine);
Assert.AreEqual("right", lexer.PeekToken());
Assert.AreEqual("right", lexer.NextToken());
}
[TestMethod]
public void EnsureTreatsUnusedSpecialCharactersAsWordCharacters()
{
headerStream.Lines.AddRange(new List<string>
{
"$ome _random token$",
"right here"
});
AssertTokenOutput("$ome", "_random", "token$", "right", "here");
Assert.IsTrue(lexer.IsEndOfStream);
}
}
}

View File

@ -0,0 +1,242 @@
using System;
using System.Collections.Generic;
using System.Linq;
using Microsoft.VisualStudio.TestTools.UnitTesting;
using ZoneCodeGenerator.Parsing.C_Header;
using ZoneCodeGenerator.Parsing.C_Header.Impl;
using ZoneCodeGenerator.Utils;
using ZoneCodeGeneratorTests.Parsing.Mock;
namespace ZoneCodeGeneratorTests.Parsing.Impl
{
[TestClass]
public class PreprocessorTest
{
private IncludingParsingStreamTest headerStreamTest;
private IHeaderParserState headerParserState;
private Preprocessor preprocessor;
[TestInitialize]
public void Setup()
{
headerStreamTest = new IncludingParsingStreamTest("file.h");
headerParserState = new HeaderParserState();
preprocessor = new Preprocessor(headerStreamTest, headerParserState);
}
[TestMethod]
public void EnsureReturnsUnmodifiedText()
{
string[] stringsThatShouldNotBeModified =
{
"This is a normal string",
"There is nothing to be preprocessed!",
"0124124124 # 124124124",
"...",
"<?php><html>asdf</html>",
""
};
headerStreamTest.Lines.AddRange(stringsThatShouldNotBeModified);
foreach (var stringThatShouldNotBeModified in stringsThatShouldNotBeModified)
{
Assert.AreEqual(stringThatShouldNotBeModified, preprocessor.ReadLine());
}
}
[TestMethod]
public void EnsureDoesRemoveLineComments()
{
var commentStrings = new Dictionary<string, string>()
{
{"// This is a comment at the beginning of the line", ""},
{"Text in front of a comment // Comment", "Text in front of a comment"},
{"/ / Not a comment", "/ / Not a comment"},
{"asdf /// Triple slash is a comment", "asdf"},
{"b2h3 /////////////// In fact after the first two slashes it should always be considered a comment", "b2h3"},
};
headerStreamTest.Lines.AddRange(commentStrings.Keys);
foreach (var (input, expectedResult) in commentStrings)
{
Assert.AreEqual(expectedResult, preprocessor.ReadLine().Trim());
}
}
[TestMethod]
public void EnsureDoesRemoveBlockComments()
{
var commentStrings = new Dictionary<string, string>()
{
{"/* This is a block comment */", ""},
{"Text in front of a comment /** Comment ***/", "Text in front of a comment"},
{"/ * Not a comment */", "/ * Not a comment */"},
{"Text in front of comment /* Comment */ Text after the comment", "Text in front of comment Text after the comment"},
{"Hello/*Hell*/World", "HelloWorld"},
};
headerStreamTest.Lines.AddRange(commentStrings.Keys);
foreach (var (input, expectedResult) in commentStrings)
{
Assert.AreEqual(expectedResult, preprocessor.ReadLine().Trim());
}
}
[TestMethod]
public void EnsureBlockCommentsWorkOverMultipleLines()
{
var commentStrings = new Dictionary<string, string>()
{
{"The start of the comment /* Is now", "The start of the comment"},
{"Nothing to be seen here", ""},
{"* / /* Still nothing", ""},
{"The comment ends */ now", "now"},
{"This line should not cause any issues", "This line should not cause any issues"},
};
headerStreamTest.Lines.AddRange(commentStrings.Keys);
foreach (var (input, expectedResult) in commentStrings)
{
Assert.AreEqual(expectedResult, preprocessor.ReadLine().Trim());
}
}
[TestMethod]
public void EnsureDefinesArePlacedCorrectly()
{
var defineStrings = new Dictionary<string, string>()
{
{"#define World Hell", ""},
{"Hello World!", "Hello Hell!"},
{"World-wide Teamwork", "Hell-wide Teamwork"},
{"#define great pretty bad", ""},
{"Defines are great, right?", "Defines are pretty bad, right?"},
{"Great world", "Great world"}, // Capitalization should matter
};
headerStreamTest.Lines.AddRange(defineStrings.Keys);
foreach (var (input, expectedResult) in defineStrings)
{
Assert.AreEqual(expectedResult, preprocessor.ReadLine().Trim());
}
}
[TestMethod]
public void EnsureDefinesCanBeOverwritten()
{
var defineStrings = new Dictionary<string, string>()
{
{"#define World Hell", ""},
{"#define World Mars", ""},
{"Hello World!", "Hello Mars!"}
};
headerStreamTest.Lines.AddRange(defineStrings.Keys);
foreach (var (input, expectedResult) in defineStrings)
{
Assert.AreEqual(expectedResult, preprocessor.ReadLine().Trim());
}
}
[TestMethod]
public void EnsureEmptyDefinesResolveToEmpty()
{
var defineStrings = new Dictionary<string, string>()
{
{"#define World", ""},
{"Hello World!", "Hello !"}
};
headerStreamTest.Lines.AddRange(defineStrings.Keys);
foreach (var (input, expectedResult) in defineStrings)
{
Assert.AreEqual(expectedResult, preprocessor.ReadLine().Trim());
}
}
[TestMethod]
public void EnsureUndefRemovesDefines()
{
var defineStrings = new Dictionary<string, string>()
{
{"#define World Hell", ""},
{"Hello World!", "Hello Hell!"},
{"#undef World", ""},
{"Hello World 2!", "Hello World 2!"},
{"#define Hell Jupiter", ""},
{"#define Hell Mars", ""},
{"#undef Hell", ""},
{"Hell no", "Hell no"},
};
headerStreamTest.Lines.AddRange(defineStrings.Keys);
foreach (var (input, expectedResult) in defineStrings)
{
Assert.AreEqual(expectedResult, preprocessor.ReadLine().Trim());
}
}
[TestMethod]
public void EnsurePackIsParsedCorrectly()
{
var defaultPack = headerParserState.CurrentPack;
var packs = new List<Tuple<string, int>>
{
new Tuple<string, int>("Test", defaultPack),
new Tuple<string, int>("#pragma pack(push, 16)", 16),
new Tuple<string, int>("Test2", 16),
new Tuple<string, int>("#pragma pack(push, 64)", 64),
new Tuple<string, int>("Test3", 64),
new Tuple<string, int>("Test4", 64),
new Tuple<string, int>("#pragma pack(pop)", 16),
new Tuple<string, int>("Test5", 16),
new Tuple<string, int>("#pragma pack(pop)", defaultPack),
new Tuple<string, int>("Test6", defaultPack)
};
headerStreamTest.Lines.AddRange(packs.Select(tuple => tuple.Item1));
foreach (var (input, expectedPack) in packs)
{
preprocessor.ReadLine();
Assert.AreEqual(expectedPack, headerParserState.CurrentPack);
}
}
[TestMethod]
public void EnsureIncludeChangesFileWhenUsingQuotationMarks()
{
headerStreamTest.Lines.AddRange(new []
{
"#include \"asdf.h\"",
"#include \"file/path/to/header.h\""
});
preprocessor.ReadLine();
Assert.AreEqual(1, headerStreamTest.IncludeCount);
Assert.AreEqual("asdf.h", headerStreamTest.LastInclude);
preprocessor.ReadLine();
Assert.AreEqual(2, headerStreamTest.IncludeCount);
Assert.AreEqual("file/path/to/header.h", headerStreamTest.LastInclude);
}
[TestMethod]
public void EnsureIncludeChangesFileWhenUsingCarets()
{
headerStreamTest.Lines.AddRange(new []
{
"#include <asdf.h>",
"#include <file/path/to/header.h>"
});
preprocessor.ReadLine();
Assert.AreEqual(1, headerStreamTest.IncludeCount);
Assert.AreEqual("asdf.h", headerStreamTest.LastInclude);
preprocessor.ReadLine();
Assert.AreEqual(2, headerStreamTest.IncludeCount);
Assert.AreEqual("file/path/to/header.h", headerStreamTest.LastInclude);
}
}
}