nocturn9x
/
peon-rewrite
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Minor test additions

This commit is contained in:
Mattia Giambirtone 2024-03-04 12:42:50 +01:00
parent ecbdf120e3
commit e7aa19835e
1 changed files with 7 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
tests/tokenize.nim
View File

@ -13,7 +13,7 @@ when isMainModule:
[
testTokenizeSucceeds("emptyFile", "", @[TokenType.EndOfFile]),
testTokenizeSucceeds("newLine", "\n", @[TokenType.EndOfFile]),
testTokenizeSucceeds("CarriageReturn", "\r", @[TokenType.EndOfFile]),
testTokenizeSucceeds("carriageReturn", "\r", @[TokenType.EndOfFile]),
testTokenizeSucceeds("emptyString", "\"\"", @[TokenType.String, TokenType.EndOfFile]),
testTokenizeSucceeds("escapedSingleQuote", "'\\''", @[TokenType.Char, TokenType.EndOfFile]),
testTokenizeSucceeds("escapedDoubleQuote", """ "\"" """, @[TokenType.String, TokenType.EndOfFile]),
@ -28,8 +28,9 @@ when isMainModule:
TokenType.EndOfFile]),
testTokenizeSucceeds("allFloats", "1.0 1e5 1E5 1.5e4 1.5E4", @[TokenType.Float, TokenType.Float, TokenType.Float,
TokenType.Float, TokenType.Float, TokenType.EndOfFile]),
testTokenizeSucceeds("simpleEscapes", """ "\a \b \f \n \r \t \v \" \' \\ " """, @[TokenType.String, TokenType.EndOfFile]),
testTokenizeFails("invalidFloatEndsWithDot", "2.", "invalid float number literal", line=1, location=(0, 1)),
testTokenizeFails("invalidFloatSpuriousChats", "2.f", "invalid float number literal", line=1, location=(0, 1)),
testTokenizeFails("invalidFloatSpuriousChars", "2.f", "invalid float number literal", line=1, location=(0, 1)),
testTokenizeFails("unterminatedChar", "'", "unexpected EOF while parsing character literal", line=1, location=(0, 0)),
testTokenizeFails("emptyChar", "''", "character literal cannot be of length zero", line=1, location=(0, 1)),
testTokenizeFails("charTooLong", "'ab'", "invalid character literal (length must be one!)", line=1, location=(0, 3)),
@ -39,6 +40,8 @@ when isMainModule:
testTokenizeFails("unterminatedCharWithNewline", "'\\n;", "unexpected EOF while parsing character literal", line=1, location=(0, 3)),
testTokenizeFails("unterminatedStringWithNewline", "\"\\n;", "unexpected EOF while parsing string literal", line=1, location=(0, 3)),
testTokenizeFails("illegalTabs", "\t", "tabs are not allowed in peon code, use spaces for indentation instead", line=1, location=(0, 0)),
testTokenizeFails("illegalShortUnicodeEscape", """ "\u123" """, "unicode escape sequences are not supported yet", line=1, location=(1, 2)),
testTokenizeFails("illegalLongUnicodeEscape", """ "\U123" """, "unicode escape sequences are not supported yet", line=1, location=(1, 2))
]
)
var allTokens = ""
@ -53,7 +56,7 @@ when isMainModule:
allTokensList.add(symbols.tokens[lexeme])
allTokensList.add(TokenType.EndOfFile)
suite.addTest(testTokenizeSucceeds("allTokens", allTokens, allTokensList))
const skippedChars = [';', '\'', '\n', '\\', '\t', '\e', '\a', '\r'];
const skippedChars = ['\'', '\n', '\\', '\t', '\e', '\a', '\r'];
var
characters = ""
charTokens = newSeqOfCap[TokenType](256)
@ -64,7 +67,7 @@ when isMainModule:
continue
characters.add(&"'{char(value)}'")
charTokens.add(TokenType.EndOfFile)
characters.add("""';' '\'' '\n' '\\' '\t' '\e' '\a' '\r'""")
characters.add("""'\'' '\n' '\\' '\t' '\e' '\a' '\r'""")
suite.addTest(testTokenizeSucceeds("allCharacters", characters, charTokens))
suite.run()
echo "Tokenization test results: "