nocturn9x
/
peon-rewrite
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Migrate all tokenizer tests to new format

This commit is contained in:
Mattia Giambirtone 2024-04-05 16:24:45 +02:00
parent 9d9093ca6c
commit 3953751cef
3 changed files with 130 additions and 54 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
src/frontend/parsing/lexer.nim
View File

@ -274,7 +274,7 @@ proc error(self: Lexer, message: string) =
proc check(self: Lexer, s: string, distance: int = 0): bool = proc check(self: Lexer, s: string, distance: int = 0): bool =
## Behaves like self.match(), without consuming the ## Behaves Bike self.match(), without consuming the
## token. False is returned if we're at EOF ## token. False is returned if we're at EOF
## regardless of what the token to check is. ## regardless of what the token to check is.
## The distance is passed directly to self.peek() ## The distance is passed directly to self.peek()
@ -570,7 +570,7 @@ proc parseBackticks(self: Lexer) =
if self.match(["\n", "\t", "\e", "\r", "\f"]): if self.match(["\n", "\t", "\e", "\r", "\f"]):
self.error(&"unexpected character in stropped identifier: {self.peek(-1).escape()}") self.error(&"unexpected character in stropped identifier: {self.peek(-1).escape()}")
discard self.step() discard self.step()
if self.done() and not self.match("`"): if self.done() or not self.match("`"):
self.error("unexpected EOF while parsing stropped identifier") self.error("unexpected EOF while parsing stropped identifier")
self.createToken(Identifier) self.createToken(Identifier)
# Strips the backticks # Strips the backticks

51
src/util/testing.nim
View File

@ -120,27 +120,27 @@ proc tokenizeSucceedsRunner(suite: TestSuite, test: Test) =
for (provided, expected) in zip(tokens, test.tokens): for (provided, expected) in zip(tokens, test.tokens):
if provided.kind != expected.kind: if provided.kind != expected.kind:
test.status = Failed test.status = Failed
test.reason = &"Token type mismatch at #{i}: expected {expected.kind}, got {provided.kind}" test.reason = &"Token type mismatch at #{i}: expected {expected.kind}, got {provided.kind}\n Expected: {expected}\n Got: {provided}"
return return
if provided.lexeme != expected.lexeme: if provided.lexeme != expected.lexeme:
test.status = Failed test.status = Failed
test.reason = &"Token lexeme mismatch at #{i}: expected '{expected.lexeme}', got '{provided.lexeme}'" test.reason = &"Token lexeme mismatch at #{i}: expected '{expected.lexeme}', got '{provided.lexeme}'\n Expected: {expected}\n Got: {provided}"
return return
if provided.line != expected.line: if provided.line != expected.line:
test.status = Failed test.status = Failed
test.reason = &"Token line mismatch at #{i}: expected {expected.line}, got {provided.line}" test.reason = &"Token line mismatch at #{i}: expected {expected.line}, got {provided.line}\n Expected: {expected}\n Got: {provided}"
return return
if provided.pos != expected.pos: if provided.pos != expected.pos:
test.status = Failed test.status = Failed
test.reason = &"Token position mismatch at #{i}: expected {expected.pos}, got {provided.pos}" test.reason = &"Token position mismatch at #{i}: expected {expected.pos}, got {provided.pos}\n Expected: {expected}\n Got: {provided}"
return return
if provided.relPos != expected.relPos: if provided.relPos != expected.relPos:
test.status = Failed test.status = Failed
test.reason = &"Token relative position mismatch at #{i}: expected {expected.relPos}, got {provided.relPos}" test.reason = &"Token relative position mismatch at #{i}: expected {expected.relPos}, got {provided.relPos}\n Expected: {expected}\n Got: {provided}"
return return
if provided.spaces != expected.spaces: if provided.spaces != expected.spaces:
test.status = Failed test.status = Failed
test.reason = &"Token spacing mismatch at #{i}: expected {expected.spaces}, got {provided.spaces}" test.reason = &"Token spacing mismatch at #{i}: expected {expected.spaces}, got {provided.spaces}\n Expected: {expected}\n Got: {provided}"
return return
inc(i) inc(i)
except LexingError: except LexingError:
@ -220,17 +220,15 @@ proc removeTests*(self: TestSuite, tests: openarray[Test]) =
proc testTokenizeSucceeds*(name, source: string, tokens: seq[Token], skip = false): Test = proc testTokenizeSucceeds*(name, source: string, tokens: seq[Token], skip = false): Test =
## Creates a new tokenizer test that is expected to succeed. ## Creates a new tokenizer test that is expected to succeed.
## The type of each token returned by the tokenizer is matched ## The type of each token returned by the tokenizer is matched
## against the given list of token types: the test only succeeds ## against the given list of tokens: the test only succeeds
## if no discrepancies are found ## if no discrepancies are found
result = Test(expected: Success) result = Test(expected: Success, kind: Tokenizer)
result.outcome.line = -1 result.outcome.line = -1
result.outcome.location = (-1, -1) result.outcome.location = (-1, -1)
result.name = name result.name = name
result.kind = Tokenizer
result.status = Init result.status = Init
result.source = source result.source = source
result.skip = skip result.skip = skip
result.expected = Success
result.runnerFunc = tokenizeSucceedsRunner result.runnerFunc = tokenizeSucceedsRunner
result.tokens = tokens result.tokens = tokens
@ -238,9 +236,8 @@ proc testTokenizeSucceeds*(name, source: string, tokens: seq[Token], skip = fals
proc testTokenizeFails*(name, source: string, message: string, line: int, location: tuple[start, stop: int], skip = false): Test = proc testTokenizeFails*(name, source: string, message: string, line: int, location: tuple[start, stop: int], skip = false): Test =
## Creates a new tokenizer test that is expected to fail with the ## Creates a new tokenizer test that is expected to fail with the
## given error message and at the given location ## given error message and at the given location
result = Test(expected: Failed) result = Test(expected: Failed, kind: Tokenizer)
result.name = name result.name = name
result.kind = Tokenizer
result.status = Init result.status = Init
result.source = source result.source = source
result.skip = skip result.skip = skip
@ -250,14 +247,37 @@ proc testTokenizeFails*(name, source: string, message: string, line: int, locati
result.line = line result.line = line
proc run*(self: TestSuite) = proc run*(self: TestSuite, verbose: bool = false) =
## Runs the test suite to completion, ## Runs the test suite to completion,
## sequentially ## sequentially
if verbose:
echo "Starting test suite"
var
ran = 0
failed = 0
crashed = 0
successful = 0
skipped = 0
for test in self.tests: for test in self.tests:
if test.skip: if test.skip:
test.status = Skipped test.status = Skipped
inc(skipped)
continue continue
if verbose:
stdout.write(&"Running '{test.name}' ({ran + 1}/{self.tests.len()})\r")
test.runnerFunc(self, test) test.runnerFunc(self, test)
case test.status:
of Success:
inc(successful)
of Failed:
inc(failed)
of Crashed:
inc(crashed)
else:
discard
inc(ran)
if verbose:
echo &"Ran {ran} tests ({skipped} skipped, {successful} successful, {failed} failed, {crashed} crashed)"
proc successful*(self: TestSuite): bool = proc successful*(self: TestSuite): bool =
@ -265,12 +285,11 @@ proc successful*(self: TestSuite): bool =
## successfully or not. If called before run(), ## successfully or not. If called before run(),
## this function returns false. Skipped tests ## this function returns false. Skipped tests
## do not affect the outcome of this function ## do not affect the outcome of this function
result = true
for test in self.tests: for test in self.tests:
if test.status in [Skipped, Success]: if test.status in [Skipped, Success]:
continue continue
result = false return false
break return true
proc getExpectedException(self: Test): ref Exception = proc getExpectedException(self: Test): ref Exception =

129
tests/tokenize.nim
View File

@ -9,21 +9,34 @@ import std/strformat
# Utilities to construct tokens for validation purposes # Utilities to construct tokens for validation purposes
proc makeToken(kind: TokenType, line: int = 1, pos: tuple[start, stop: int] = (0, 0), relPos: tuple[start, stop: int] = (0, 0), lexeme: string, proc makeToken(kind: TokenType, line: int = 1, pos: tuple[start, stop: int] = (0, 0), relPos: tuple[start, stop: int] = (-1, -1), lexeme: string,
spaces: int = 0): Token {.inline.} = spaces: int = 0): Token {.inline.} =
var pos = pos
if pos == (0, 0):
pos = (0, max(lexeme.high(), 0))
var relPos = relPos
if relPos == (-1, -1):
relPos = pos
return Token(kind: kind, line: line, pos: pos, relPos: relPos, lexeme: lexeme, spaces: spaces) return Token(kind: kind, line: line, pos: pos, relPos: relPos, lexeme: lexeme, spaces: spaces)
proc endOfFile(line: int = 1, pos: tuple[start, stop: int] = (0, 0), relPos: tuple[start, stop: int] = (0, 0), spaces: int = 0): Token {.inline.} =
return makeToken(EndOfFile, line, pos, relPos, "", spaces)
proc makeString(line: int = 1, pos: tuple[start, stop: int] = (0, 0), relPos: tuple[start, stop: int] = (0, 0), content: string, delimiter: string, spaces: int = 0): Token {.inline.} = proc endOfFile(line: int = 1, pos: tuple[start, stop: int] = (0, 0), relPos: tuple[start, stop: int] = (-1, -1), spaces: int = 0): Token {.inline.} =
return makeToken(String, line, pos, relPos, &"{delimiter}{content}{delimiter}", spaces) return makeToken(TokenType.EndOfFile, line, pos, relPos, "", spaces)
proc makeChar(line: int = 1, pos: tuple[start, stop: int] = (0, 0), relPos: tuple[start, stop: int] = (0, 0), content: string, spaces: int = 0): Token {.inline.} = proc makeString(line: int = 1, pos: tuple[start, stop: int] = (0, 0), relPos: tuple[start, stop: int] = (-1, -1), content: string, delimiter: string, spaces: int = 0): Token {.inline.} =
return makeToken(Char, line, pos, relPos, &"'{content}'", spaces) return makeToken(TokenType.String, line, pos, relPos, &"{delimiter}{content}{delimiter}", spaces)
proc makeSymbol(line: int = 1, pos: tuple[start, stop: int] = (0, 0), relPos: tuple[start, stop: int] = (0, 0), content: string, spaces: int = 0): Token {.inline.} = proc makeChar(line: int = 1, pos: tuple[start, stop: int] = (0, 0), relPos: tuple[start, stop: int] = (-1, -1), content: string, spaces: int = 0): Token {.inline.} =
return makeToken(Symbol, line, pos, relPos, content, spaces) return makeToken(TokenType.Char, line, pos, relPos, &"'{content}'", spaces)
proc makeSymbol(line: int = 1, pos: tuple[start, stop: int] = (0, 0), relPos: tuple[start, stop: int] = (-1, -1), content: string, spaces: int = 0): Token {.inline.} =
return makeToken(TokenType.Symbol, line, pos, relPos, content, spaces)
proc makeIdent(line: int = 1, pos: tuple[start, stop: int] = (0, 0), relPos: tuple[start, stop: int] = (-1, -1), name: string, spaces: int = 0): Token {.inline.} =
return makeToken(TokenType.Identifier, line, pos, relPos, name, spaces)
proc makeInteger(line: int = 1, pos: tuple[start, stop: int] = (0, 0), relPos: tuple[start, stop: int] = (-1, -1), value: int, spaces: int = 0): Token {.inline.} =
return makeToken(TokenType.Integer, line, pos, relPos, $value, spaces)
when isMainModule: when isMainModule:
@ -31,23 +44,47 @@ when isMainModule:
suite.addTests( suite.addTests(
[ [
testTokenizeSucceeds("emptyFile", "", @[endOfFile()]), testTokenizeSucceeds("emptyFile", "", @[endOfFile()]),
testTokenizeSucceeds("ignoredEscapes", "\r\f\e", @[endOfFile(pos=(3, 3), relPos=(3, 3))]), testTokenizeSucceeds("ignoredEscapes", "\r\f\e", @[endOfFile(pos=(3, 3))]),
testTokenizeSucceeds("newLine", "\n", @[endOfFile(line=2, pos=(1, 1))]), testTokenizeSucceeds("newLine", "\n", @[endOfFile(line=2, pos=(1, 1), relPos=(0, 0))]),
testTokenizeSucceeds("emptyString", """""""", @[makeString(content="", delimiter="\"", pos=(0, 1), relPos=(0, 1)), endOfFile(pos=(2, 2), relPos=(2, 2))]), testTokenizeSucceeds("emptyString", """""""", @[makeString(content="", delimiter="\""), endOfFile(pos=(2, 2))]),
testTokenizeSucceeds("escapedDelimChar", """'\''""", @[makeChar(content="\\'", pos=(0, 3), relPos=(0, 3)), endOfFile(pos=(4, 4), relPos=(4, 4))]), testTokenizeSucceeds("escapedDelimChar", """'\''""", @[makeChar(content="\\'"), endOfFile(pos=(4, 4))]),
testTokenizeSucceeds("escapedDelimString", """"\""""", @[makeString(content="\\\"", delimiter="\"", pos=(0, 3), relPos=(0, 3)), endOfFile(pos=(4, 4), relPos=(4, 4))]), testTokenizeSucceeds("escapedDelimString", """"\""""", @[makeString(content="\\\"", delimiter="\""), endOfFile(pos=(4, 4))]),
#[testTokenizeSucceeds("bareUnicode", "🌎 😂 👩‍👩‍👦‍👦", @[TokenType.Symbol, TokenType.Symbol, TokenType.Symbol, TokenType.EndOfFile]), testTokenizeSucceeds("bareUnicode", "🌎 😂 👩‍👩‍👦‍👦", @[makeSymbol(content="🌎", pos=(0, 3)), makeSymbol(content="😂", pos=(5, 8), spaces=1),
testTokenizeSucceeds("stroppedSingleUnicode", "`🌎` `😂` `👩‍👩‍👦‍👦`", @[TokenType.Identifier, TokenType.Identifier, TokenType.Identifier, TokenType.EndOfFile]), makeSymbol(content="👩‍👩‍👦‍👦", pos=(10, 34), spaces=1), endOfFile(pos=(35, 35))]),
testTokenizeSucceeds("stroppedMultiUnicode", "`🌎🌎` `😂😂` `👩‍👩‍👦‍👦👩‍👩‍👦‍👦`", @[TokenType.Identifier, TokenType.Identifier, TokenType.Identifier, TokenType.EndOfFile]), testTokenizeSucceeds("stroppedSingleUnicode", "`🌎` `😂` `👩‍👩‍👦‍👦`", @[makeIdent(name="🌎", pos=(0, 5)),
testTokenizeSucceeds("stringWithEscapes", """ "\n\t\r\e\f" """, @[TokenType.String, TokenType.EndOfFile]), makeIdent(name="😂", pos=(7, 12), spaces=1),
testTokenizeSucceeds("allIntegers", "1 0x1 0o1 0b1", @[TokenType.Integer, TokenType.Hex, TokenType.Octal, TokenType.Binary, TokenType.EndOfFile]), makeIdent(name="👩‍👩‍👦‍👦", pos=(14, 40), spaces=1),
endOfFile(pos=(41, 41))]),
testTokenizeSucceeds("stroppedMultiUnicode", "`🌎🌎` `😂😂` `👩‍👩‍👦‍👦👩‍👩‍👦‍👦`", @[makeIdent(name="🌎🌎", pos=(0, 9)),
makeIdent(name="😂😂", pos=(11, 20), spaces=1),
makeIdent(name="👩‍👩‍👦‍👦👩‍👩‍👦‍👦", pos=(22, 73), spaces=1),
endOfFile(pos=(74, 74))]),
testTokenizeSucceeds("allIntegers", "1 0x1 0o1 0b1", @[makeInteger(value=1),
makeToken(TokenType.Hex, pos=(2, 4), lexeme="0x1", spaces=1),
makeToken(TokenType.Octal, pos=(6, 8), lexeme="0o1", spaces=1),
makeToken(TokenType.Binary, pos=(10, 12), lexeme="0b1", spaces=1),
endOfFile(pos=(13, 13))
]),
testTokenizeSucceeds("allFloats", "1.0 1e5 1E5 1.5e4 1.5E4", @[makeToken(TokenType.Float, pos=(0, 2), lexeme="1.0"),
makeToken(TokenType.Float, pos=(4, 6), lexeme="1e5", spaces=1),
makeToken(TokenType.Float, pos=(8, 10), lexeme="1E5", spaces=1),
makeToken(TokenType.Float, pos=(12, 16), lexeme="1.5e4", spaces=1),
makeToken(TokenType.Float, pos=(18, 22), lexeme="1.5E4", spaces=1),
endOfFile(pos=(23, 23))]),
testTokenizeSucceeds("sizedNumbers", "1'u8 0x1'i8 0o1'i64 0b1'u32 2.0'f32 1e5'f64 1E5'f32 1.5e4'f64 1.5E4'f32", testTokenizeSucceeds("sizedNumbers", "1'u8 0x1'i8 0o1'i64 0b1'u32 2.0'f32 1e5'f64 1E5'f32 1.5e4'f64 1.5E4'f32",
@[TokenType.Integer, TokenType.Hex, TokenType.Octal, TokenType.Binary, @[makeToken(TokenType.Integer, lexeme="1'u8"),
TokenType.Float, TokenType.Float, TokenType.Float, TokenType.Float, TokenType.Float, makeToken(TokenType.Hex, pos=(5, 10), lexeme="0x1'i8", spaces=1),
TokenType.EndOfFile]), makeToken(TokenType.Octal, pos=(12, 18), lexeme="0o1'i64", spaces=1),
testTokenizeSucceeds("allFloats", "1.0 1e5 1E5 1.5e4 1.5E4", @[TokenType.Float, TokenType.Float, TokenType.Float, makeToken(TokenType.Binary, pos=(20, 26), lexeme="0b1'u32", spaces=1),
TokenType.Float, TokenType.Float, TokenType.EndOfFile]), makeToken(TokenType.Float, pos=(28, 34), lexeme="2.0'f32", spaces=1),
testTokenizeSucceeds("simpleEscapes", """ "\a \b \f \n \r \t \v \" \' \\ " """, @[TokenType.String, TokenType.EndOfFile]),]# makeToken(TokenType.Float, pos=(36, 42), lexeme="1e5'f64", spaces=1),
makeToken(TokenType.Float, pos=(44, 50), lexeme="1E5'f32", spaces=1),
makeToken(TokenType.Float, pos=(52, 60), lexeme="1.5e4'f64", spaces=1),
makeToken(TokenType.Float, pos=(62, 70), lexeme="1.5E4'f32", spaces=1),
endOfFile(pos=(71, 71))]),
testTokenizeSucceeds("simpleEscapes", """ "\a \b \f \n \r \t \v \" \' \\ " """, @[makeString(spaces=1, content="""\a \b \f \n \r \t \v \" \' \\ """,
delimiter="\"", pos=(1, 32)),
endOfFile(pos=(34, 34))]),
testTokenizeFails("invalidFloatEndsWithDot", "2.", "invalid float number literal", line=1, location=(0, 1)), testTokenizeFails("invalidFloatEndsWithDot", "2.", "invalid float number literal", line=1, location=(0, 1)),
testTokenizeFails("invalidFloatSpuriousChars", "2.f", "invalid float number literal", line=1, location=(0, 1)), testTokenizeFails("invalidFloatSpuriousChars", "2.f", "invalid float number literal", line=1, location=(0, 1)),
testTokenizeFails("unterminatedChar", "'", "unexpected EOF while parsing character literal", line=1, location=(0, 0)), testTokenizeFails("unterminatedChar", "'", "unexpected EOF while parsing character literal", line=1, location=(0, 0)),
@ -70,33 +107,53 @@ when isMainModule:
] ]
) )
#[var allTokens = "" var allTokens = ""
var allTokensList = newSeqOfCap[TokenType](symbols.tokens.len()) var allTokensList = newSeqOfCap[Token](symbols.tokens.len())
var i = 0
for lexeme in symbols.tokens.keys(): for lexeme in symbols.tokens.keys():
allTokens.add(&"{lexeme} ") allTokens.add(&"{lexeme} ")
let spaces = if i == 0: 0 else: 1
let pos = (i, i + len(lexeme) - 1)
if lexeme == "_": if lexeme == "_":
# Due to how the lexer is designed, a bare underscore is # Due to how the lexer is designed, a bare underscore is
# parsed as an identifier rather than a symbol # parsed as an identifier rather than a symbol
allTokensList.add(TokenType.Identifier) allTokensList.add(makeIdent(pos=pos, name="_", spaces=spaces))
else: else:
allTokensList.add(symbols.tokens[lexeme]) allTokensList.add(makeToken(symbols.tokens[lexeme], pos=pos, lexeme=lexeme, spaces=spaces))
allTokensList.add(TokenType.EndOfFile) inc(i, len(lexeme) + 1)
allTokensList.add(endOfFile(pos=(i, i)))
suite.addTest(testTokenizeSucceeds("allTokens", allTokens, allTokensList)) suite.addTest(testTokenizeSucceeds("allTokens", allTokens, allTokensList))
const skippedChars = ['\'', '\n', '\\', '\t', '\e', '\a', '\r']; const skippedChars = ['\'', '\n', '\\', '\t', '\e', '\a', '\r'];
var var
characters = "" characters = ""
charTokens = newSeqOfCap[TokenType](256) charTokens = newSeqOfCap[Token](256)
i = 0
for value in 0..255: for value in 0..255:
charTokens.add(Char)
if char(value) in skippedChars: if char(value) in skippedChars:
# These cases are special and we handle them separately # These cases are special and we handle them separately
continue continue
charTokens.add(makeChar(pos=(i, i + 2), content= &"{char(value)}"))
characters.add(&"'{char(value)}'") characters.add(&"'{char(value)}'")
charTokens.add(TokenType.EndOfFile) inc(i, 3)
characters.add("""'\'' '\n' '\\' '\t' '\e' '\a' '\r'""") characters.add("""'\'''\n''\\''\t''\e''\a''\r'""")
suite.addTest(testTokenizeSucceeds("allCharacters", characters, charTokens))]# charTokens.add(makeChar(pos=(i, i + 3), content="""\'"""))
inc(i, 4)
charTokens.add(makeChar(pos=(i, i + 3), content="""\n"""))
inc(i, 4)
charTokens.add(makeChar(pos=(i, i + 3), content="""\\"""))
inc(i, 4)
charTokens.add(makeChar(pos=(i, i + 3), content="""\t"""))
inc(i, 4)
charTokens.add(makeChar(pos=(i, i + 3), content="""\e"""))
inc(i, 4)
charTokens.add(makeChar(pos=(i, i + 3), content="""\a"""))
inc(i, 4)
charTokens.add(makeChar(pos=(i, i + 3), content="""\r"""))
inc(i, 4)
charTokens.add(endOfFile(pos=(i, i)))
suite.addTest(testTokenizeSucceeds("allCharacters", characters, charTokens))
echo "Running tokenizer tests" echo "Running tokenizer tests"
suite.run() suite.run(verbose=true)
if suite.successful(): if suite.successful():
echo "OK: All tokenizer tests were successful" echo "OK: All tokenizer tests were successful"
quit(0) quit(0)