Turn peon into a proper nimble package and add initial test suite

peon.nimble

@@ -0,0 +1,18 @@
+# Package
+
+version       = "0.1.0"
+author        = "nocturn9x"
+description   = "A rewrite of Peon 0.1.x"
+license       = "Apache-2.0"
+srcDir        = "src"
+bin           = @["peon"]
+
+
+# Dependencies
+
+requires "nim >= 2.1.1"
+requires "jale >= 0.1.1"
+
+
+task test, "Runs the test suite":
+  exec "nim r tests/tokenize.nim"

src/frontend/compiler/typechecker.nim

@@ -1,6 +1,6 @@
 import errors
-import frontend/parser/parser
-import frontend/parser/lexer
+import frontend/parsing/parser
+import frontend/parsing/lexer
 import frontend/compiler/typesystem
 
 
@@ -788,7 +788,6 @@ proc stringify*(self: TypeChecker, typ: Type): string =
 
 
 proc stringify*(self: TypeChecker, typ: TypedNode): string =
-
     if typ.node.isConst():
         return self.stringify(TypedExpr(typ).kind)
     case typ.node.kind:

src/frontend/compiler/typesystem.nim

@@ -1,5 +1,5 @@
 import errors
-import frontend/parser/parser
+import frontend/parsing/parser
 
 
 import std/tables

src/frontend/parsing/lexer.nim

@@ -233,8 +233,7 @@ proc peek(self: Lexer, distance: int = 0, length: int = 1): string =
     ## may be empty
     var i = distance
     while len(result) < length:
-        if self.done() or self.current + i > self.source.high() or
-                self.current + i < 0:
+        if self.current + i > self.source.high() or self.current + i < 0:
             break
         else:
             result.add(self.source[self.current + i])
@@ -379,10 +378,10 @@ proc parseEscape(self: Lexer) =
 
 
 proc parseString(self: Lexer, delimiter: string, mode: string = "single") =
-    ## Parses string literals. They can be expressed using matching pairs
-    ## of either single or double quotes. Most C-style escape sequences are
-    ## supported, moreover, a specific prefix may be prepended
-    ## to the string to instruct the lexer on how to parse it:
+    ## Parses string and character literals. They can be expressed using 
+    ## matching pairs of double or single quotes respectivelt. Most C-style 
+    ## escape sequences are supported, moreover, a specific prefix may be 
+    ## prepended to the string to instruct the lexer on how to parse it:
     ## - b -> declares a byte string, where each character is
     ##     interpreted as an integer instead of a character
     ## - r -> declares a raw string literal, where escape sequences
@@ -434,13 +433,18 @@ proc parseString(self: Lexer, delimiter: string, mode: string = "single") =
     if mode == "multi":
         if not self.match(delimiter.repeat(3)):
             self.error("unexpected EOL while parsing multi-line string literal")
-    elif self.done() and self.peek(-1) != delimiter:
-        self.error("unexpected EOF while parsing string literal")
+    elif self.done() and (self.peek(-1) != delimiter or slen == 0):
+        if delimiter == "'":
+            self.error("unexpected EOF while parsing character literal")
+        else:
+            self.error("unexpected EOF while parsing string literal")
     else:
         discard self.step()
     if delimiter == "\"":
         self.createToken(String)
     else:
+        if slen == 0:
+            self.error("character literal cannot be of length zero")
         self.createToken(Char)
 
 

src/main.nim

@@ -15,8 +15,8 @@
 import config
 import util/fmterr
 import util/symbols
-import frontend/parser/lexer
-import frontend/parser/parser
+import frontend/parsing/lexer
+import frontend/parsing/parser
 import frontend/compiler/typechecker
 import backend/bytecode/codegen/generator
 import backend/bytecode/tooling/serializer

src/util/fmterr.nim

@@ -14,8 +14,8 @@
 
 ## Utilities to print formatted error messages to stderr
 import frontend/compiler/typechecker
-import frontend/parser/parser
-import frontend/parser/lexer
+import frontend/parsing/parser
+import frontend/parsing/lexer
 import errors
 
 

src/util/symbols.nim

@@ -1,4 +1,4 @@
-import frontend/parser/lexer
+import frontend/parsing/lexer
 
 
 proc fillSymbolTable*(tokenizer: Lexer) =

src/util/testing.nim

@@ -0,0 +1,180 @@
+import std/strformat
+import std/strutils
+
+import frontend/parsing/lexer
+import util/symbols
+
+
+
+
+type 
+    TestStatus* = enum
+        Init, Running, Success, 
+        Failed, Crashed,
+        TimedOut, Skipped
+
+    TestKind* = enum
+        Tokenizer, Parser, TypeChecker,
+        Runtime
+
+    TestRunner = proc (suite: TestSuite, test: Test)
+    TestOutcome = tuple[error: bool, exc: ref Exception]
+
+    Test* {.inheritable.} = ref object
+        skip*: bool
+        name*: string
+        kind*: TestKind
+        source*: string
+        status*: TestStatus
+        expected*: TestStatus
+        outcome*: TestOutcome
+        runnerFunc: TestRunner
+    
+
+    TokenizerTest* = ref object of Test
+        message: string
+        location: tuple[start, stop: int]
+
+    TestSuite* = ref object
+        tests*: seq[Test]
+
+
+proc `$`(self: tuple[error: bool, exc: ref Exception]): string =
+    if self.exc.isNil():
+        result = &"Outcome(error={self.error}, exc=nil)"
+    else:
+        var name = ($self.exc.name).split(":")[0]
+        result = &"Outcome(error={self.error}, exc=Error(name='{name}', msg='{self.exc.msg}')"
+
+
+proc `$`(self: tuple[start, stop: int]): string = 
+    if self == (-1, -1):
+        result = "none"
+    else:
+        result = &"Location(start={self.start}, stop={self.stop})"
+
+
+proc `$`*(self: Test): string =
+    case self.kind:
+        of Tokenizer:
+            var self = TokenizerTest(self)
+            return &"TokenizerTest(name='{self.name}', status={self.status}, outcome={self.outcome}, source='{self.source}', location={self.location}, message='{self.message}')"
+        else:
+            # TODO
+            return ""
+
+
+proc getTestDetails(self: Test): string =
+    doAssert self.status != Init
+    case self.kind:
+        of Tokenizer:
+            var self = TokenizerTest(self)
+            result = &"expecting "
+        else:
+            # TODO
+            discard
+
+
+proc tokenizeSucceedsRunner(suite: TestSuite, test: Test) =
+    try:
+        var tokenizer = newLexer()
+        tokenizer.fillSymbolTable()
+        discard tokenizer.lex(test.source, "<string>")
+    except LexingError:
+        test.status = Failed
+        test.outcome.error = true
+        test.outcome.exc = getCurrentException()
+        return
+    except CatchableError:
+        test.status = Crashed
+        test.outcome.error = true
+        test.outcome.exc = getCurrentException()
+        return
+    test.status = Success
+
+
+proc tokenizeFailsRunner(suite: TestSuite, test: Test) =
+    var test = TokenizerTest(test)
+    try:
+        var tokenizer = newLexer()
+        tokenizer.fillSymbolTable()
+        discard tokenizer.lex(test.source, "<string>")
+    except LexingError:
+        var exc = LexingError(getCurrentException())
+        if exc.pos == test.location and exc.msg == test.message:
+            test.status = Success
+        else:
+            test.status = Failed
+            test.outcome.error = true
+        test.outcome.exc = getCurrentException()
+        return
+    except CatchableError:
+        test.status = Crashed
+        test.outcome.error = true
+        test.outcome.exc = getCurrentException()
+        return
+    test.status = Failed
+
+
+proc newTestSuite*: TestSuite = new(result)
+
+
+proc addTest*(self: TestSuite, test: Test) =
+    self.tests.add(test)
+
+
+proc addTests*(self: TestSuite, tests: openarray[Test]) =
+    for test in tests:
+        self.addTest(test)
+
+
+proc removeTest*(self: TestSuite, test: Test) =
+    self.tests.delete(self.tests.find(test))
+
+
+proc removeTests*(self: TestSuite, tests: openarray[Test]) =
+    for test in tests:
+        self.removeTest(test)
+
+
+proc newTokenizeTest(name, source: string): TokenizerTest =
+    new(result)
+    result.name = name
+    result.kind = Tokenizer
+    result.status = Init
+    result.source = source
+
+
+proc testTokenizeSucceeds*(name, source: string, skip = false): Test =
+    var test = newTokenizeTest(name, source)
+    test.runnerFunc = tokenizeSucceedsRunner
+    test.message = ""
+    test.location = (-1 , -1)
+    result = Test(test)
+    result.skip = skip
+
+
+proc testTokenizeFails*(name, source: string, message: string, location: tuple[start, stop: int], skip = false): Test =
+    var test = newTokenizeTest(name, source)
+    test.runnerFunc = tokenizeFailsRunner
+    test.message = message
+    test.location = location
+    result = Test(test)
+    result.skip = skip
+
+
+proc run*(self: TestSuite) =
+    for test in self.tests:
+        if test.skip:
+            test.status = Skipped
+            continue
+        test.runnerFunc(self, test)
+    
+
+proc successful*(self: TestSuite): bool =
+    result = true
+    for test in self.tests:
+        if test.status in [Skipped, Success]:
+            continue
+        result = false
+        break

tests/tokenize.nim

@@ -0,0 +1,26 @@
+import util/testing
+
+
+import std/strformat
+
+
+when isMainModule:
+    var suite = newTestSuite()
+    suite.addTests(
+        [
+            testTokenizeSucceeds("emptyFile", ""),
+            testTokenizeFails("unterminatedChar", "'", "unexpected EOF while parsing character literal", (0, 0)),
+            testTokenizeFails("emptyChar", "''", "character literal cannot be of length zero", (0, 1)),
+            testTokenizeFails("unterminatedString", "\"", "unexpected EOF while parsing string literal", (0, 0)),
+            testTokenizeSucceeds("emptyString", "\"\"")
+        ]
+    )
+    suite.run()
+    echo "Tokenization test results: "
+    for test in suite.tests:
+        echo &"  - {test.name} -> {test.status}"
+        if test.status != Success:
+            echo &"    Details: {test}"
+    if suite.successful():
+        quit(0)
+    quit(-1)