Improve test suite with outcome management
This commit is contained in:
parent
41abf59395
commit
40cbed2b19
GPG Key ID:
8270F9F467971E59
|
|
@ -401,7 +401,10 @@ proc parseString(self: Lexer, delimiter: string, mode: string = "single") =
|
|||
if mode == "multi":
|
||||
self.incLine()
|
||||
else:
|
||||
self.error("unexpected EOL while parsing string literal")
|
||||
if delimiter == "'":
|
||||
self.error("unexpected EOL while parsing character literal")
|
||||
else:
|
||||
self.error("unexpected EOL while parsing string literal")
|
||||
if mode in ["raw", "multi"]:
|
||||
discard self.step()
|
||||
elif self.match("\\"):
|
||||
|
|
|
|||
|
|
@ -12,12 +12,15 @@
|
|||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
## Utilities to print formatted error messages to stderr
|
||||
## Utilities to format peon exceptions into human-readable error messages
|
||||
## and print them
|
||||
import frontend/compiler/typechecker
|
||||
import frontend/parsing/parser
|
||||
import frontend/parsing/lexer
|
||||
import errors
|
||||
|
||||
export errors
|
||||
|
||||
|
||||
import std/os
|
||||
import std/terminal
|
||||
|
|
@ -25,24 +28,26 @@ import std/strutils
|
|||
import std/strformat
|
||||
|
||||
|
||||
proc printError(file, line: string, lineNo: int, pos: tuple[start, stop: int], fn: Declaration, msg: string) =
|
||||
## Internal helper to print a formatted error message
|
||||
## to stderr
|
||||
stderr.styledWrite(fgRed, styleBright, "Error in ", fgYellow, &"{file}:{lineNo}:{pos.start}")
|
||||
proc formatError*(outFile = stderr, file, line: string, lineNo: int, pos: tuple[start, stop: int], fn: Declaration, msg: string, includeSource = true) =
|
||||
## Helper to write a formatted error message to the given file object
|
||||
outFile.styledWrite(fgRed, styleBright, "Error in ", fgYellow, &"{file}:{lineNo}:{pos.start}")
|
||||
if not fn.isNil() and fn.kind == funDecl:
|
||||
# Error occurred inside a (named) function
|
||||
stderr.styledWrite(fgRed, styleBright, " in function ", fgYellow, FunDecl(fn).name.token.lexeme)
|
||||
stderr.styledWriteLine(styleBright, fgDefault, ": ", msg)
|
||||
if line.len() > 0:
|
||||
stderr.styledWrite(fgRed, styleBright, "Source line: ", resetStyle, fgDefault, line[0..<pos.start])
|
||||
outFile.styledWriteLine(styleBright, fgDefault, ": ", msg)
|
||||
if line.len() > 0 and includeSource:
|
||||
# Print the line where the error occurred and underline the exact node that caused
|
||||
# the error. Might be inaccurate, but definitely better than nothing
|
||||
outFile.styledWrite(fgRed, styleBright, "Source line: ", resetStyle, fgDefault, line[0..<pos.start])
|
||||
if pos.stop == line.len():
|
||||
stderr.styledWrite(fgRed, styleUnderscore, line[pos.start..<pos.stop])
|
||||
stderr.styledWriteLine(fgDefault, line[pos.stop..^1])
|
||||
outFile.styledWrite(fgRed, styleUnderscore, line[pos.start..<pos.stop])
|
||||
outFile.styledWriteLine(fgDefault, line[pos.stop..^1])
|
||||
else:
|
||||
stderr.styledWrite(fgRed, styleUnderscore, line[pos.start..pos.stop])
|
||||
stderr.styledWriteLine(fgDefault, line[pos.stop + 1..^1])
|
||||
outFile.styledWrite(fgRed, styleUnderscore, line[pos.start..pos.stop])
|
||||
outFile.styledWriteLine(fgDefault, line[pos.stop + 1..^1])
|
||||
|
||||
|
||||
proc print*(exc: TypeCheckError) =
|
||||
proc print*(exc: TypeCheckError, includeSource = true) =
|
||||
## Prints a formatted error message
|
||||
## for type checking errors to stderr
|
||||
var file = exc.file
|
||||
|
|
@ -51,10 +56,10 @@ proc print*(exc: TypeCheckError) =
|
|||
of -1: discard
|
||||
of 0: contents = exc.instance.getSource().strip(chars={'\n'}).splitLines()[exc.line]
|
||||
else: contents = exc.instance.getSource().strip(chars={'\n'}).splitLines()[exc.line - 1]
|
||||
printError(file, contents, exc.line, exc.node.getRelativeBoundaries(), exc.function, exc.msg)
|
||||
formatError(stderr, file, contents, exc.line, exc.node.getRelativeBoundaries(), exc.function, exc.msg, includeSource)
|
||||
|
||||
|
||||
proc print*(exc: ParseError) =
|
||||
proc print*(exc: ParseError, includeSource = true) =
|
||||
## Prints a formatted error message
|
||||
## for parsing errors to stderr
|
||||
var file = exc.file
|
||||
|
|
@ -65,10 +70,10 @@ proc print*(exc: ParseError) =
|
|||
contents = exc.parser.getSource().strip(chars={'\n'}).splitLines()[exc.line - 1]
|
||||
else:
|
||||
contents = ""
|
||||
printError(file, contents, exc.line, exc.token.relPos, nil, exc.msg)
|
||||
formatError(stderr, file, contents, exc.line, exc.token.relPos, nil, exc.msg, includeSource)
|
||||
|
||||
|
||||
proc print*(exc: LexingError) =
|
||||
proc print*(exc: LexingError, includeSource = true) =
|
||||
## Prints a formatted error message
|
||||
## for lexing errors to stderr
|
||||
var file = exc.file
|
||||
|
|
@ -79,5 +84,5 @@ proc print*(exc: LexingError) =
|
|||
contents = exc.lexer.getSource().strip(chars={'\n'}).splitLines()[exc.line - 1]
|
||||
else:
|
||||
contents = ""
|
||||
printError(file, contents, exc.line, exc.pos, nil, exc.msg)
|
||||
formatError(stderr, file, contents, exc.line, exc.pos, nil, exc.msg, includeSource)
|
||||
|
||||
|
|
|
|||
|
|
@ -42,7 +42,7 @@ type
|
|||
# error field indicates whether the test errored out
|
||||
# or not. If exc is non-null and error is false, this
|
||||
# means the error was expected behavior
|
||||
TestOutcome = tuple[error: bool, exc: ref Exception]
|
||||
TestOutcome = tuple[error: bool, exc: ref Exception, line: int, location: tuple[start, stop: int]]
|
||||
|
||||
Test* {.inheritable.} = ref object
|
||||
## A generic test object
|
||||
|
|
@ -63,20 +63,13 @@ type
|
|||
## upon tokenization failure
|
||||
message: string
|
||||
location: tuple[start, stop: int]
|
||||
line: int
|
||||
lexer: Lexer
|
||||
|
||||
TestSuite* = ref object
|
||||
## A suite of tests
|
||||
tests*: seq[Test]
|
||||
|
||||
|
||||
proc `$`(self: tuple[error: bool, exc: ref Exception]): string =
|
||||
if self.exc.isNil():
|
||||
result = &"Outcome(error={self.error}, exc=nil)"
|
||||
else:
|
||||
var name = ($self.exc.name).split(":")[0]
|
||||
result = &"Outcome(error={self.error}, exc=Error(name='{name}', msg='{self.exc.msg}')"
|
||||
|
||||
|
||||
proc `$`(self: tuple[start, stop: int]): string =
|
||||
if self == (-1, -1):
|
||||
result = "none"
|
||||
|
|
@ -84,26 +77,40 @@ proc `$`(self: tuple[start, stop: int]): string =
|
|||
result = &"Location(start={self.start}, stop={self.stop})"
|
||||
|
||||
|
||||
proc `$`(self: TestOutcome): string =
|
||||
result &= "Outcome(error={self.error}"
|
||||
if self.exc.isNil():
|
||||
result &= &", exc=nil"
|
||||
else:
|
||||
var name = ($self.exc.name).split(":")[0]
|
||||
result = &"exc=Error(name='{name}', msg='{self.exc.msg}'"
|
||||
if self.line != -1:
|
||||
result &= &", line={self.line}"
|
||||
result &= &", location={self.location})"
|
||||
|
||||
|
||||
|
||||
proc `$`*(self: Test): string =
|
||||
case self.kind:
|
||||
of Tokenizer:
|
||||
var self = TokenizerTest(self)
|
||||
return &"TokenizerTest(name='{self.name}', status={self.status}, outcome={self.outcome}, source='{self.source}', location={self.location}, message='{self.message}')"
|
||||
return &"TokenizerTest(name='{self.name}', status={self.status}, outcome={self.outcome}, source='{self.source.escape()}', location={self.location}, message='{self.message}')"
|
||||
else:
|
||||
# TODO
|
||||
return ""
|
||||
|
||||
|
||||
proc createLexer: Lexer =
|
||||
result = newLexer()
|
||||
result.fillSymbolTable()
|
||||
proc setup(self: TokenizerTest) =
|
||||
self.lexer = newLexer()
|
||||
self.lexer.fillSymbolTable()
|
||||
|
||||
|
||||
proc tokenizeSucceedsRunner(suite: TestSuite, test: Test) =
|
||||
## Runs a tokenitazion test that is expected to succeed
|
||||
var test = TokenizerTest(test)
|
||||
test.setup()
|
||||
try:
|
||||
var tokenizer = createLexer()
|
||||
discard tokenizer.lex(test.source, "<string>")
|
||||
discard test.lexer.lex(test.source, test.name)
|
||||
except LexingError:
|
||||
test.status = Failed
|
||||
test.outcome.error = true
|
||||
|
|
@ -119,14 +126,16 @@ proc tokenizeSucceedsRunner(suite: TestSuite, test: Test) =
|
|||
|
||||
proc tokenizeFailsRunner(suite: TestSuite, test: Test) =
|
||||
## Runs a tokenitazion test that is expected to fail
|
||||
## and checks that it fails in the way we expect
|
||||
## and checks that it does so in the way we expect
|
||||
var test = TokenizerTest(test)
|
||||
test.setup()
|
||||
try:
|
||||
var tokenizer = createLexer()
|
||||
discard tokenizer.lex(test.source, "<string>")
|
||||
discard test.lexer.lex(test.source, test.name)
|
||||
except LexingError:
|
||||
var exc = LexingError(getCurrentException())
|
||||
if exc.pos == test.location and exc.msg == test.message:
|
||||
test.outcome.location = exc.pos
|
||||
test.outcome.line = exc.line
|
||||
if exc.pos == test.location and exc.line == test.line and exc.msg == test.message:
|
||||
test.status = Success
|
||||
else:
|
||||
test.status = Failed
|
||||
|
|
@ -175,25 +184,29 @@ proc newTokenizeTest(name, source: string, skip = false): TokenizerTest =
|
|||
result.status = Init
|
||||
result.source = source
|
||||
result.skip = skip
|
||||
result.line = -1
|
||||
result.location = (-1, -1)
|
||||
result.message = ""
|
||||
|
||||
|
||||
proc testTokenizeSucceeds*(name, source: string, skip = false): Test =
|
||||
## Creates a new tokenizer test that is expected to succeed
|
||||
var test = newTokenizeTest(name, source, skip)
|
||||
test.runnerFunc = tokenizeSucceedsRunner
|
||||
test.message = ""
|
||||
test.location = (-1 , -1)
|
||||
result = Test(test)
|
||||
result.expected = Success
|
||||
|
||||
|
||||
proc testTokenizeFails*(name, source: string, message: string, location: tuple[start, stop: int], skip = false): Test =
|
||||
proc testTokenizeFails*(name, source: string, message: string, line: int, location: tuple[start, stop: int], skip = false): Test =
|
||||
## Creates a new tokenizer test that is expected to fail with the
|
||||
## given error message and at the given location
|
||||
var test = newTokenizeTest(name, source, skip)
|
||||
test.runnerFunc = tokenizeFailsRunner
|
||||
test.message = message
|
||||
test.location = location
|
||||
test.line = line
|
||||
result = Test(test)
|
||||
result.expected = Failed
|
||||
|
||||
|
||||
proc run*(self: TestSuite) =
|
||||
|
|
@ -216,4 +229,33 @@ proc successful*(self: TestSuite): bool =
|
|||
if test.status in [Skipped, Success]:
|
||||
continue
|
||||
result = false
|
||||
break
|
||||
break
|
||||
|
||||
|
||||
proc getExpectedException(self: TokenizerTest): ref Exception =
|
||||
## Gets the exception that we expect to be
|
||||
## raised by the test. Could be nil if we
|
||||
## expect no errors
|
||||
if self.expected == Success:
|
||||
return nil
|
||||
return LexingError(msg: self.message, line: self.line, file: self.name, lexer: self.lexer, pos: self.location)
|
||||
|
||||
|
||||
|
||||
proc getExpectedOutcome(self: TokenizerTest): TestOutcome =
|
||||
## Gets the expected outcome of a tokenization test
|
||||
if self.expected == Success:
|
||||
return (false, self.getExpectedException(), -1, (-1, -1))
|
||||
else:
|
||||
return (false, self.getExpectedException, self.line, self.location)
|
||||
|
||||
|
||||
proc getExpectedOutcome*(self: Test): TestOutcome =
|
||||
## Returns the expected outcome of a test
|
||||
doAssert self.expected in [Success, Failed], "expected outcome is neither Success nor Failed: wtf?"
|
||||
case self.kind:
|
||||
of Tokenizer:
|
||||
return TokenizerTest(self).getExpectedOutcome()
|
||||
else:
|
||||
# TODO
|
||||
discard
|
||||
|
|
@ -1,7 +1,11 @@
|
|||
import util/testing
|
||||
import util/fmterr
|
||||
import frontend/parsing/lexer
|
||||
|
||||
|
||||
import std/strformat
|
||||
import std/strutils
|
||||
|
||||
|
||||
|
||||
when isMainModule:
|
||||
|
|
@ -9,18 +13,30 @@ when isMainModule:
|
|||
suite.addTests(
|
||||
[
|
||||
testTokenizeSucceeds("emptyFile", ""),
|
||||
testTokenizeFails("unterminatedChar", "'", "unexpected EOF while parsing character literal", (0, 0)),
|
||||
testTokenizeFails("emptyChar", "''", "character literal cannot be of length zero", (0, 1)),
|
||||
testTokenizeFails("unterminatedString", "\"", "unexpected EOF while parsing string literal", (0, 0)),
|
||||
testTokenizeSucceeds("emptyString", "\"\"")
|
||||
testTokenizeSucceeds("newLine", "\n"),
|
||||
testTokenizeSucceeds("emptyString", "\"\""),
|
||||
testTokenizeFails("unterminatedChar", "'", "unexpected EOF while parsing character literal", line=1, location=(0, 0)),
|
||||
testTokenizeFails("emptyChar", "''", "character literal cannot be of length zero", line=1, location=(0, 1)),
|
||||
testTokenizeFails("unterminatedString", "\"", "unexpected EOF while parsing string literal", line=1, location=(0, 0)),
|
||||
testTokenizeFails("unterminatedCharWithExtraContent", "'\n;", "unexpected EOL while parsing character literal", line=1, location=(0, 1)),
|
||||
testTokenizeFails("unterminatedStringWithExtraContent", "\"\n;", "unexpected EOL while parsing string literal", line=1, location=(0, 1)),
|
||||
]
|
||||
)
|
||||
|
||||
suite.run()
|
||||
echo "Tokenization test results: "
|
||||
for test in suite.tests:
|
||||
echo &" - {test.name} -> {test.status}"
|
||||
if test.status != Success:
|
||||
echo &" Details: {test}"
|
||||
if test.status in [Failed, Crashed]:
|
||||
echo &" Details:"
|
||||
echo &" - Source: {test.source.escape()}"
|
||||
echo &" - Outcome: {test.outcome}"
|
||||
echo &" - Expected state: {test.expected} "
|
||||
echo &" - Expected outcome: {test.getExpectedOutcome()}"
|
||||
if not test.outcome.exc.isNil():
|
||||
echo &"\n Formatted error message follows\n"
|
||||
print(LexingError(test.outcome.exc))
|
||||
echo "\n Formatted error message ends here\n"
|
||||
if suite.successful():
|
||||
quit(0)
|
||||
quit(-1)
|
||||
|
|
|
|||
Loading…
Reference in New Issue