nocturn9x
/
peon-rewrite
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Initial work on parser tests. Parser bug fixes. Minor changes to type checker

This commit is contained in:
Mattia Giambirtone 2024-04-08 14:33:14 +02:00
parent 740999bbc2
commit 4fc078d4d8
10 changed files with 183 additions and 111 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
README.md
View File

@ -13,10 +13,11 @@ Work in progress for Peon 0.2.x
## Build ## Build
Just run `nimble build`. It should grab all the dependencies for you and produce a `peon` binary in your current working directory Just run `nimble build`. It should grab all the dependencies, run the test suite and produce a `peon` binary in your current working directory
## Tests ## Tests
Peon is starting to get large enough for it to need an automated test suite (wow, much fancy, such cool), so you can run `nimble test` Peon is starting to get large enough for it to need an automated test suite (wow, much fancy, such cool), so you can run `nimble test`
to run that. The actual tests aren't using testament because I have a severe case of NIH syndrome, sorry folks! to run that (it's also ran automatically at every `nimble build`). The tests aren't using testament because I have a severe case of NIH syndrome,
sorry folks!

3
peon.nimble
View File

@ -12,7 +12,7 @@ binDir = "bin"
# Dependencies # Dependencies
requires "nim >= 2.1.1" requires "nim >= 2.1.1"
requires "jale >= 0.1.1" # requires "jale >= 0.1.1"
before build: before build:
exec "nimble test" exec "nimble test"
@ -20,3 +20,4 @@ before build:
task test, "Runs the test suite": task test, "Runs the test suite":
exec "nim r tests/tokenize.nim" exec "nim r tests/tokenize.nim"
exec "nim r tests/parse.nim"

114
src/frontend/compiler/typechecker.nim
View File

@ -60,7 +60,7 @@ type
## Forward declarations ## Forward declarations
proc validate(self: TypeChecker, node: ASTNode): TypedNode proc check(self: TypeChecker, node: ASTNode): TypedNode
proc toIntrinsic(name: string): Type proc toIntrinsic(name: string): Type
proc done(self: TypeChecker): bool {.inline.} proc done(self: TypeChecker): bool {.inline.}
proc toRef(self: Type): Type {.inline.} proc toRef(self: Type): Type {.inline.}
@ -1015,14 +1015,6 @@ proc match(self: TypeChecker, name: string, sig: TypeSignature, node: ASTNode =
of 1: of 1:
# There's just one match. We're done # There's just one match. We're done
result = matches[0] result = matches[0]
if result.kind == NameKind.Var:
# Variables bound to other names must always
# have this field set
assert not result.assignedName.isNil()
# We found a name bound to a variable, so we
# return the original name object rather than
# the wrapper
result = result.assignedName
# Extra checks # Extra checks
case result.valueType.kind: case result.valueType.kind:
of Function: of Function:
@ -1097,10 +1089,9 @@ proc replaceGenerics(self: TypeChecker, typ: Type, generics: TableRef[string, Ty
else: else:
self.error(&"unable to perform generic instantiation for object of type {self.stringify(typ)}") self.error(&"unable to perform generic instantiation for object of type {self.stringify(typ)}")
proc concretize(self: TypeChecker, name: Name, args: seq[TypedExpr], node: ASTNode = nil): Type = proc concretize(self: TypeChecker, name: Name, args: seq[TypedExpr], node: ASTNode = nil): Type =
## Takes in a generic type and its arguments, returning a concrete instantiation ## Takes in a generic type and its arguments and returns a concrete instantiation
## of it ## of it
let let
typ = name.valueType.unwrapType() typ = name.valueType.unwrapType()
@ -1126,7 +1117,6 @@ proc concretize(self: TypeChecker, name: Name, args: seq[TypedExpr], node: ASTNo
self.replaceGenerics(typ, replaced) self.replaceGenerics(typ, replaced)
proc expandTypeConstraints(self: TypeChecker, condition: Expression, list: var seq[tuple[match: bool, kind: Type, value: Expression]], accept: bool = true) = proc expandTypeConstraints(self: TypeChecker, condition: Expression, list: var seq[tuple[match: bool, kind: Type, value: Expression]], accept: bool = true) =
## Recursively unpacks a type constraint ## Recursively unpacks a type constraint
case condition.kind: case condition.kind:
@ -1249,7 +1239,8 @@ proc declare(self: TypeChecker, node: ASTNode): Name {.discardable.} =
valueType: kind) valueType: kind)
self.addName(name) self.addName(name)
of NodeKind.importStmt: of NodeKind.importStmt:
discard # TODO
self.error("not implemented")
of NodeKind.typeDecl: of NodeKind.typeDecl:
var node = TypeDecl(node) var node = TypeDecl(node)
declaredName = node.name.token.lexeme declaredName = node.name.token.lexeme
@ -1272,7 +1263,8 @@ proc declare(self: TypeChecker, node: ASTNode): Name {.discardable.} =
valueType: kind) valueType: kind)
self.addName(name) self.addName(name)
else: else:
discard # TODO: enums self.error("not implemented")
# TODO: enums
if not name.isNil(): if not name.isNil():
self.dispatchPragmas(name) self.dispatchPragmas(name)
return name return name
@ -1360,7 +1352,7 @@ proc call(self: TypeChecker, node: CallExpr): TypedExpr =
result = newTypedCallExpr(node, impl, typedArgs) result = newTypedCallExpr(node, impl, typedArgs)
else: else:
# TODO? # TODO?
discard self.error("not implemented")
of NodeKind.callExpr: of NodeKind.callExpr:
# Calling a call expression, like hello()() # Calling a call expression, like hello()()
# TODO # TODO
@ -1378,18 +1370,20 @@ proc call(self: TypeChecker, node: CallExpr): TypedExpr =
for exp in all: for exp in all:
result = self.call(exp) result = self.call(exp)
]# ]#
discard self.error("not implemented")
of NodeKind.getterExpr: of NodeKind.getterExpr:
# Calling a.b() # Calling a.b()
# TODO # TODO
let node = GetterExpr(node.callee) let node = GetterExpr(node.callee)
self.error("not implemented")
of NodeKind.lambdaExpr: of NodeKind.lambdaExpr:
# Calling a lambda on the fly # Calling a lambda on the fly
var node = LambdaExpr(node.callee) var node = LambdaExpr(node.callee)
self.error("not implemented")
# TODO # TODO
of NodeKind.genericExpr: of NodeKind.genericExpr:
# Calling a generic expression # Calling a generic expression
# TODO self.error("not implemented")
var node = GenericExpr(node.callee) var node = GenericExpr(node.callee)
# TODO # TODO
else: else:
@ -1400,14 +1394,30 @@ proc call(self: TypeChecker, node: CallExpr): TypedExpr =
self.error(&"object of type '{self.stringify(typ)}' is not callable", node) self.error(&"object of type '{self.stringify(typ)}' is not callable", node)
proc ensureBoolean(self: TypeChecker, node: Expression): TypedExpr {.inline.} =
## Shorthand for checking that a given expression is
## of type bool
return self.check(node, "bool".toIntrinsic())
proc ensureType(self: TypeChecker, node: Expression): TypedExpr {.inline.} =
## Ensures the given expression resolves to a type (rather
## than a value) and returns it
return self.check(node, "typevar".toIntrinsic())
proc refExpr(self: TypeChecker, node: Ref): TypedExpr = proc refExpr(self: TypeChecker, node: Ref): TypedExpr =
## Typechecks ref expressions ## Typechecks ref expressions
result = self.check(node.value, "typevar".toIntrinsic())
# "ref" only makes sense with types
result = self.ensureType(node.value)
result.kind = result.kind.toRef() result.kind = result.kind.toRef()
proc constExpr(self: TypeChecker, node: ast.Const): TypedExpr = proc constExpr(self: TypeChecker, node: ast.Const): TypedExpr =
## Typechecks const expressions ## Typechecks const expressions
# "const" makes a type immutable
var kind = "any".toIntrinsic().toConst() var kind = "any".toIntrinsic().toConst()
result = self.check(node.value, kind) result = self.check(node.value, kind)
result.kind = result.kind.toConst() result.kind = result.kind.toConst()
@ -1416,7 +1426,7 @@ proc constExpr(self: TypeChecker, node: ast.Const): TypedExpr =
proc lentExpr(self: TypeChecker, node: ast.Lent): TypedExpr = proc lentExpr(self: TypeChecker, node: ast.Lent): TypedExpr =
## Typechecks lent expressions ## Typechecks lent expressions
# Only match references # "lent" only makes sense for references
var kind = "any".toIntrinsic().toRef() var kind = "any".toIntrinsic().toRef()
result = self.check(node.value, kind) result = self.check(node.value, kind)
# Wrap the result back # Wrap the result back
@ -1458,29 +1468,23 @@ proc blockStmt(self: TypeChecker, node: BlockStmt): TypedBlockStmt =
## Typechecks block statements ## Typechecks block statements
self.beginScope() self.beginScope()
var body: seq[TypedNode] = @[] var body: seq[TypedNode] = @[]
for decl in node.code: for decl in node.body:
body.add(self.validate(decl)) body.add(self.check(decl))
self.endScope() self.endScope()
result = newTypedBlockStmt(node, body) result = newTypedBlockStmt(node, body)
proc ensureBoolean(self: TypeChecker, node: Expression): TypedExpr {.inline.} =
## Shorthand for checking that a given expression is
## of type bool
return self.check(node, "bool".toIntrinsic())
proc ifStmt(self: TypeChecker, node: IfStmt): TypedNode = proc ifStmt(self: TypeChecker, node: IfStmt): TypedNode =
## Typechecks if/else statements ## Typechecks if/else statements
# Check the condition # Check the condition
let condition = self.ensureBoolean(node.condition) let condition = self.ensureBoolean(node.condition)
# Check the body # Check the body
let then = TypedBlockStmt(self.validate(node.thenBranch)) let then = TypedBlockStmt(self.check(node.thenBranch))
var otherwise: TypedBlockStmt = nil var otherwise: TypedBlockStmt = nil
if not node.elseBranch.isNil(): if not node.elseBranch.isNil():
# Check the else clause, if it exists # Check the else clause, if it exists
otherwise = TypedBlockStmt(self.validate(node.elseBranch)) otherwise = TypedBlockStmt(self.check(node.elseBranch))
# Note: Peon enforces the body of loops and conditionals to # Note: Peon enforces the body of loops and conditionals to
# always be a block statement (for a variety of very good reasons, # always be a block statement (for a variety of very good reasons,
# as it avoids mistakes like the infamous GOTO fail), so the # as it avoids mistakes like the infamous GOTO fail), so the
@ -1490,11 +1494,11 @@ proc ifStmt(self: TypeChecker, node: IfStmt): TypedNode =
proc whileStmt(self: TypeChecker, node: WhileStmt): TypedNode = proc whileStmt(self: TypeChecker, node: WhileStmt): TypedNode =
## Typechecks C-style while loops ## Typechecks C-style while loops
let
condition = self.ensureBoolean(node.condition) # Check the condition
body = TypedBlockStmt(self.check(node.body)) # Check the body
return newTypedWhileStmt(node, body, condition)
# Check the condition
let condition = self.ensureBoolean(node.condition)
# Check the body
return newTypedWhileStmt(node, TypedBlockStmt(self.validate(node.body)), condition)
proc varDecl(self: TypeChecker, node: VarDecl): TypedVarDecl = proc varDecl(self: TypeChecker, node: VarDecl): TypedVarDecl =
@ -1510,8 +1514,8 @@ proc varDecl(self: TypeChecker, node: VarDecl): TypedVarDecl =
self.error("const declaration requires an initializer", node) self.error("const declaration requires an initializer", node)
else: else:
if node.constant and not node.value.isConst(): if node.constant and not node.value.isConst():
self.error("constant initializer is not a constant", node.value) self.error("const initializer must be a value of constant type", node.value)
init = TypedExpr(self.validate(node.value)) init = TypedExpr(self.check(node.value))
typ = init.kind typ = init.kind
if not node.valueType.isNil(): if not node.valueType.isNil():
# Explicit type declaration always takes over # Explicit type declaration always takes over
@ -1519,9 +1523,9 @@ proc varDecl(self: TypeChecker, node: VarDecl): TypedVarDecl =
# Check that the inferred expression represents a type # Check that the inferred expression represents a type
# and not a value. This is to guard against things # and not a value. This is to guard against things
# like "var x: 1 = 1;". We unwrap it immediately because # like "var x: 1 = 1;". We unwrap it immediately because
# we don't actually want the valueType field of the variable # we don't actually want the returned expression to be
# to be a wrapped type, we just do this step as a safety check # a wrapped type, we just do this step as a safety check
typ = self.check(node.valueType, "typevar".toIntrinsic()).kind.unwrapType() typ = self.ensureType(node.valueType).kind.unwrapType()
if typ.isNil(): if typ.isNil():
self.error("expecting either a type declaration or an initializer value, but neither was found", node) self.error("expecting either a type declaration or an initializer value, but neither was found", node)
# Now check that the type of the initializer, if it exists, # Now check that the type of the initializer, if it exists,
@ -1535,9 +1539,9 @@ proc varDecl(self: TypeChecker, node: VarDecl): TypedVarDecl =
proc funDecl(self: TypeChecker, node: FunDecl, name: Name = nil): TypedFunDecl = proc funDecl(self: TypeChecker, node: FunDecl, name: Name = nil): TypedFunDecl =
## Typechecks function declarations ## Typechecks function declarations
# Some things are just not possible # Some things are just not possible in life
if node.token.kind == Operator and node.name.token.lexeme in [".", ]: if node.token.kind == Operator and node.name.token.lexeme in [".", ]:
self.error(&"Due to compiler limitations, the '{node.name.token.lexeme}' operator cannot be currently overridden", node.name) self.error(&"due to compiler limitations, the '{node.name.token.lexeme}' operator cannot be overridden", node.name)
var name = name var name = name
if name.isNil(): if name.isNil():
name = self.declare(node) name = self.declare(node)
@ -1567,17 +1571,17 @@ proc funDecl(self: TypeChecker, node: FunDecl, name: Name = nil): TypedFunDecl =
return return
if node.body.isNil(): if node.body.isNil():
# Forward declaration # Forward declaration
# TODO
self.endScope() self.endScope()
return # TODO
if BlockStmt(node.body).code.len() == 0: self.error("not implemented")
if BlockStmt(node.body).body.len() == 0:
self.error("cannot declare function with empty body") self.error("cannot declare function with empty body")
# We store the current function to restore # We store the current function to restore
# it later # it later
let function = self.currentFunction let function = self.currentFunction
self.currentFunction = name self.currentFunction = name
for decl in BlockStmt(node.body).code: for decl in BlockStmt(node.body).body:
result.body.body.add(self.validate(decl)) result.body.body.add(self.check(decl))
self.endScope() self.endScope()
# Restores the enclosing function (if any). # Restores the enclosing function (if any).
# Makes nested calls work (including recursion) # Makes nested calls work (including recursion)
@ -1684,9 +1688,9 @@ proc typeDecl(self: TypeChecker, node: TypeDecl, name: Name = nil): TypedTypeDec
self.error(&"got node of unexpected type {node.value.kind} at typeDecl()") self.error(&"got node of unexpected type {node.value.kind} at typeDecl()")
if not node.parent.isNil(): if not node.parent.isNil():
# Ensure parent is actually a type # Ensure parent is actually a type
var subtype = self.check(node.parent, "typevar".toIntrinsic()) let subtype = self.ensureType(node.parent)
# Grab its name object # Grab its name object
var parentName = subtype.getName() let parentName = subtype.getName()
# This should *never* be nil # This should *never* be nil
if parentName.isNil(): if parentName.isNil():
self.error(&"could not obtain name information for the given object: is it a type?", node.parent) self.error(&"could not obtain name information for the given object: is it a type?", node.parent)
@ -1713,7 +1717,7 @@ proc pragmaExpr(self: TypeChecker, pragma: Pragma) =
self.pragmas[pragma.name.token.lexeme].handler(self, pragma, nil) self.pragmas[pragma.name.token.lexeme].handler(self, pragma, nil)
proc validate(self: TypeChecker, node: ASTNode): TypedNode = proc check(self: TypeChecker, node: ASTNode): TypedNode =
## Dispatches typeless AST nodes to typecheck them and turn ## Dispatches typeless AST nodes to typecheck them and turn
## them into typed ones ## them into typed ones
case node.kind: case node.kind:
@ -1723,9 +1727,9 @@ proc validate(self: TypeChecker, node: ASTNode): TypedNode =
NodeKind.binExpr, NodeKind.hexExpr, NodeKind.trueExpr, NodeKind.falseExpr, NodeKind.binExpr, NodeKind.hexExpr, NodeKind.trueExpr, NodeKind.falseExpr,
NodeKind.nanExpr, NodeKind.infExpr: NodeKind.nanExpr, NodeKind.infExpr:
result = self.expression(Expression(node)) result = self.expression(Expression(node))
of exprStmt: of NodeKind.exprStmt:
let statement = ExprStmt(node) let statement = ExprStmt(node)
result = TypedExprStmt(node: statement, expression: TypedExpr(self.validate(statement.expression))) result = TypedExprStmt(node: statement, expression: TypedExpr(self.check(statement.expression)))
of NodeKind.whileStmt: of NodeKind.whileStmt:
result = self.whileStmt(WhileStmt(node)) result = self.whileStmt(WhileStmt(node))
of NodeKind.blockStmt: of NodeKind.blockStmt:
@ -1746,7 +1750,7 @@ proc validate(self: TypeChecker, node: ASTNode): TypedNode =
self.error(&"failed to dispatch node of type {node.kind}", node) self.error(&"failed to dispatch node of type {node.kind}", node)
proc validate*(self: TypeChecker, tree: ParseTree, file, source: string, showMismatches: bool = false, proc check*(self: TypeChecker, tree: ParseTree, file, source: string, showMismatches: bool = false,
disabledWarnings: seq[WarningKind] = @[]): seq[TypedNode] = disabledWarnings: seq[WarningKind] = @[]): seq[TypedNode] =
## Transforms a sequence of typeless AST nodes ## Transforms a sequence of typeless AST nodes
## into a sequence of typed AST nodes ## into a sequence of typed AST nodes
@ -1785,10 +1789,12 @@ proc validate*(self: TypeChecker, tree: ParseTree, file, source: string, showMis
ident: newIdentExpr(Token(lexeme: "", kind: Identifier)), ident: newIdentExpr(Token(lexeme: "", kind: Identifier)),
line: 1) line: 1)
self.addName(main) self.addName(main)
var node: TypedNode
while not self.done(): while not self.done():
result.add(self.validate(self.step())) node = self.check(self.step())
if result[^1].isNil(): if node.isNil():
result.delete(result.high()) continue
result.add(node)
assert self.scopeDepth == 0 assert self.scopeDepth == 0
# Do not close the global scope if # Do not close the global scope if
# we're being imported # we're being imported

8
src/frontend/compiler/typesystem.nim
View File

@ -114,14 +114,6 @@ type
# Just for easier lookup, it's all # Just for easier lookup, it's all
# pointers anyway # pointers anyway
names*: TableRef[string, Name] names*: TableRef[string, Name]
of NameKind.Var:
# If the variable's value is another
# name, this attribute contains its
# name object. This is useful for things
# like assigning functions to variables and
# then calling the variable like it's the
# original function
assignedName*: Name
else: else:
discard discard
# The name's identifier # The name's identifier

17
src/frontend/parsing/ast.nim
View File

@ -226,9 +226,8 @@ type
## A block statement. This is basically ## A block statement. This is basically
## the equivalent of a C scope ## the equivalent of a C scope
# We allow declarations inside a block statement.
# Kinda confusing, but it makes sense, I promise (TM) body*: seq[ASTNode]
code*: seq[Declaration]
NamedBlockStmt* = ref object of BlockStmt NamedBlockStmt* = ref object of BlockStmt
## Identical to a block statement, except ## Identical to a block statement, except
@ -580,17 +579,17 @@ proc newAssertStmt*(expression: Expression, token: Token): AssertStmt =
result.token = token result.token = token
proc newBlockStmt*(code: seq[Declaration], token: Token): BlockStmt = proc newBlockStmt*(body: seq[ASTNode], token: Token): BlockStmt =
new(result) new(result)
result.kind = NodeKind.blockStmt result.kind = NodeKind.blockStmt
result.code = code result.body = body
result.token = token result.token = token
proc newNamedBlockStmt*(code: seq[Declaration], name: IdentExpr, token: Token): NamedBlockStmt = proc newNamedBlockStmt*(body: seq[ASTNode], name: IdentExpr, token: Token): NamedBlockStmt =
new(result) new(result)
result.kind = NodeKind.namedBlockStmt result.kind = NodeKind.namedBlockStmt
result.code = code result.body = body
result.token = token result.token = token
result.name = name result.name = name
@ -745,10 +744,10 @@ proc `$`*(self: ASTNode): string =
result &= &"Assert({self.expression})" result &= &"Assert({self.expression})"
of blockStmt: of blockStmt:
var self = BlockStmt(self) var self = BlockStmt(self)
result &= &"""Block([{self.code.join(", ")}])""" result &= &"""Block([{self.body.join(", ")}])"""
of namedBlockStmt: of namedBlockStmt:
var self = NamedBlockStmt(self) var self = NamedBlockStmt(self)
result &= &"""Block(name={self.name}, [{self.code.join(", ")}])""" result &= &"""Block(name={self.name}, [{self.body.join(", ")}])"""
of whileStmt: of whileStmt:
var self = WhileStmt(self) var self = WhileStmt(self)
result &= &"While(condition={self.condition}, body={self.body})" result &= &"While(condition={self.condition}, body={self.body})"

38
src/frontend/parsing/parser.nim
View File

@ -174,7 +174,7 @@ template endOfLine(msg: string, tok: Token = nil) = self.expect(Semicolon, msg,
# Utility functions # Utility functions
proc dispatch(self: Parser): ASTNode
proc beginScope(self: Parser) {.inline.} = proc beginScope(self: Parser) {.inline.} =
## Begins a new lexical scope ## Begins a new lexical scope
@ -449,7 +449,7 @@ proc makeCall(self: Parser, callee: Expression): CallExpr =
if not self.check(RightParen): if not self.check(RightParen):
while true: while true:
if argCount >= 255: if argCount >= 255:
self.error("can not pass more than 255 arguments in function call") self.error("cannot pass more than 255 arguments in call")
argument = self.expression() argument = self.expression()
if argument.kind == assignExpr: if argument.kind == assignExpr:
var assign = AssignExpr(argument) var assign = AssignExpr(argument)
@ -460,7 +460,7 @@ proc makeCall(self: Parser, callee: Expression): CallExpr =
elif arguments.keyword.len() == 0: elif arguments.keyword.len() == 0:
arguments.positionals.add(argument) arguments.positionals.add(argument)
else: else:
self.error("positional arguments cannot follow keyword arguments in function calls") self.error("positional argument cannot follow keyword argument in call", token=argument.token)
if not self.match(Comma): if not self.match(Comma):
break break
argCount += 1 argCount += 1
@ -495,7 +495,7 @@ proc call(self: Parser): Expression =
result.file = self.file result.file = self.file
elif self.match(LeftBracket): elif self.match(LeftBracket):
if self.peek(-2).kind != Identifier: if self.peek(-2).kind != Identifier:
self.error("invalid syntax") self.error("expecting identifier before '['")
result = self.parseGenericArgs() result = self.parseGenericArgs()
else: else:
break break
@ -611,7 +611,7 @@ proc parseAssign(self: Parser): Expression =
result = newSetterExpr(GetterExpr(result).obj, GetterExpr(result).name, value, tok) result = newSetterExpr(GetterExpr(result).obj, GetterExpr(result).name, value, tok)
result.file = self.file result.file = self.file
else: else:
self.error("invalid assignment target", tok) self.error("invalid assignment target", result.token)
proc parseArrow(self: Parser): Expression = proc parseArrow(self: Parser): Expression =
@ -645,13 +645,16 @@ proc blockStmt(self: Parser): BlockStmt =
## scope ## scope
self.beginScope() self.beginScope()
let tok = self.peek(-1) let tok = self.peek(-1)
var code: seq[Declaration] = @[] var
body: seq[ASTNode] = @[]
node: ASTNode
while not self.check(RightBrace) and not self.done(): while not self.check(RightBrace) and not self.done():
code.add(self.declaration()) node = self.dispatch()
if code[^1].isNil(): if node.isNil():
code.delete(code.high()) continue
body.add(node)
self.expect(RightBrace, "expecting '}'") self.expect(RightBrace, "expecting '}'")
result = newBlockStmt(code, tok) result = newBlockStmt(body, tok)
result.file = self.file result.file = self.file
self.endScope() self.endScope()
@ -660,18 +663,21 @@ proc namedBlockStmt(self: Parser): Statement =
## Parses named block statement ## Parses named block statement
self.beginScope() self.beginScope()
let tok = self.peek(-1) let tok = self.peek(-1)
var code: seq[Declaration] = @[]
self.expect(Identifier, "expecting block name after 'block'") self.expect(Identifier, "expecting block name after 'block'")
var name = newIdentExpr(self.peek(-1)) var name = newIdentExpr(self.peek(-1))
name.file = self.file name.file = self.file
inc(self.loopDepth) inc(self.loopDepth)
self.expect(LeftBrace, "expecting '{' after 'block'") self.expect(LeftBrace, "expecting '{' after block name")
var
body: seq[ASTNode] = @[]
node: ASTNode
while not self.check(RightBrace) and not self.done(): while not self.check(RightBrace) and not self.done():
code.add(self.declaration()) node = self.dispatch()
if code[^1].isNil(): if node.isNil():
code.delete(code.high()) continue
body.add(node)
self.expect(RightBrace, "expecting '}'") self.expect(RightBrace, "expecting '}'")
result = newNamedBlockStmt(code, name, tok) result = newNamedBlockStmt(body, name, tok)
result.file = self.file result.file = self.file
self.endScope() self.endScope()
dec(self.loopDepth) dec(self.loopDepth)

2
src/peon.nim
View File

@ -116,7 +116,7 @@ proc runFile(filename: string, fromString: bool = false, dump: bool = true, gene
echo "" echo ""
if not typeCheck: if not typeCheck:
return return
typedNodes = typeChecker.validate(tree, filename, tokenizer.getSource(), mismatches, disabledWarnings) typedNodes = typeChecker.check(tree, filename, tokenizer.getSource(), mismatches, disabledWarnings)
if debugTypeChecker: if debugTypeChecker:
styledEcho fgCyan, "Typechecker output:" styledEcho fgCyan, "Typechecker output:"
for typedNode in typedNodes: for typedNode in typedNodes:

4
src/util/astchecker.nim
View File

@ -59,7 +59,7 @@ proc compare(provided, expected: LiteralExpr): Comparison =
result.match = false result.match = false
result.reason = &"expected lexeme to be {expected.literal.lexeme.escape()}, got {provided.literal.lexeme.escape()}" result.reason = &"expected lexeme to be {expected.literal.lexeme.escape()}, got {provided.literal.lexeme.escape()}"
else: else:
raise newException(ValueError, &"literal comparison attempted with non-literal node of type {provided.kind}") raise newException(ValueError, &"literal comparison attempted with non-literal nodes of type {provided.kind}")
proc compare(provided, expected: ASTNode): Comparison = proc compare(provided, expected: ASTNode): Comparison =
@ -68,6 +68,8 @@ proc compare(provided, expected: ASTNode): Comparison =
ensureEqualType(provided, expected) ensureEqualType(provided, expected)
if provided.isLiteral(): if provided.isLiteral():
return compare(LiteralExpr(provided), LiteralExpr(expected)) return compare(LiteralExpr(provided), LiteralExpr(expected))
else:
raise newException(ValueError, &"comparisons for nodes of type {provided.kind} are not supported yet")
proc check*(self: ASTChecker, code: string, structure: ParseTree): Comparison = proc check*(self: ASTChecker, code: string, structure: ParseTree): Comparison =

2
src/util/fmterr.nim
View File

@ -42,7 +42,7 @@ proc formatError*(errKind: string = "", outFile = stderr, file, line: string, li
# Print the line where the error occurred and underline the exact node that caused # Print the line where the error occurred and underline the exact node that caused
# the error. Might be inaccurate, but definitely better than nothing # the error. Might be inaccurate, but definitely better than nothing
outFile.styledWrite(fgRed, styleBright, "Source line: ", resetStyle, fgDefault, line[0..<pos.start]) outFile.styledWrite(fgRed, styleBright, "Source line: ", resetStyle, fgDefault, line[0..<pos.start])
outFile.styledWrite(fgRed, styleUnderscore, line[pos.start..<pos.stop]) outFile.styledWrite(fgRed, styleUnderscore, line[pos.start..pos.stop])
if pos.stop + 1 <= line.high(): if pos.stop + 1 <= line.high():
outFile.styledWriteLine(fgDefault, line[pos.stop + 1..^1]) outFile.styledWriteLine(fgDefault, line[pos.stop + 1..^1])
else: else:

101
src/util/testing.nim
View File

@ -51,9 +51,10 @@ type
skip*: bool # Skip running this test if true skip*: bool # Skip running this test if true
name*: string # Test name. Only useful for displaying purposes name*: string # Test name. Only useful for displaying purposes
lexer: Lexer # Lexer object used by the test
parser: Parser # Parser object used by the test (may be nil)
case kind*: TestKind: # Test kind (tokenizer, parser, compiler, etc.) case kind*: TestKind: # Test kind (tokenizer, parser, compiler, etc.)
of Tokenizer: of Tokenizer:
lexer: Lexer
tokens: seq[Token] tokens: seq[Token]
of Parser: of Parser:
tree: ParseTree tree: ParseTree
@ -98,12 +99,11 @@ proc `$`*(self: TestOutcome): string =
proc setup(self: Test) = proc setup(self: Test) =
case self.kind: self.lexer = newLexer()
of Tokenizer: self.lexer.fillSymbolTable()
self.lexer = newLexer() if self.kind == Parser:
self.lexer.fillSymbolTable() self.parser = newParser()
else:
discard # TODO
proc tokenizeSucceedsRunner(suite: TestSuite, test: Test) = proc tokenizeSucceedsRunner(suite: TestSuite, test: Test) =
@ -190,6 +190,45 @@ proc tokenizeFailsRunner(suite: TestSuite, test: Test) =
test.status = Failed test.status = Failed
proc parseFailsRunner(suite: TestSuite, test: Test) =
## Runs a parsing test that is expected to fail
## and checks that it does so in the way we expect
test.setup()
try:
var tokens = test.lexer.lex(test.source, test.name)
discard test.parser.parse(tokens, test.name, test.lexer.getLines(), test.source)
except LexingError:
var exc = LexingError(getCurrentException())
test.outcome.location = exc.pos
test.outcome.line = exc.line
test.status = Failed
test.reason = &"Tokenization failed"
test.outcome.error = true
test.outcome.exc = getCurrentException()
return
except ParseError:
var exc = ParseError(getCurrentException())
test.outcome.location = exc.token.pos
test.outcome.line = exc.line
if exc.token.pos == test.location and exc.line == test.line and exc.msg == test.message:
test.status = Success
else:
if test.outcome.location != test.location or exc.line != test.line:
test.reason = &"Expecting failure at {test.line}:({test.location.start}, {test.location.stop}), failed at {exc.line}:({test.outcome.location.start}, {test.outcome.location.stop})"
else:
test.reason = &"Expecting error message to be '{test.message}', got '{exc.msg}'"
test.status = Failed
test.outcome.error = true
test.outcome.exc = getCurrentException()
return
except CatchableError:
test.status = Crashed
test.outcome.error = true
test.outcome.exc = getCurrentException()
return
test.status = Failed
proc newTestSuite*: TestSuite = proc newTestSuite*: TestSuite =
## Creates a new test suite ## Creates a new test suite
new(result) new(result)
@ -219,9 +258,9 @@ proc removeTests*(self: TestSuite, tests: openarray[Test]) =
proc testTokenizeSucceeds*(name, source: string, tokens: seq[Token], skip = false): Test = proc testTokenizeSucceeds*(name, source: string, tokens: seq[Token], skip = false): Test =
## Creates a new tokenizer test that is expected to succeed. ## Creates a new tokenizer test that is expected to succeed.
## The type of each token returned by the tokenizer is matched ## Each token returned by the tokenizer is matched against
## against the given list of tokens: the test only succeeds ## the given list of tokens: the test only succeedscif no
## if no discrepancies are found ## discrepancies are found
result = Test(expected: Success, kind: Tokenizer) result = Test(expected: Success, kind: Tokenizer)
result.outcome.line = -1 result.outcome.line = -1
result.outcome.location = (-1, -1) result.outcome.location = (-1, -1)
@ -247,6 +286,34 @@ proc testTokenizeFails*(name, source: string, message: string, line: int, locati
result.line = line result.line = line
proc testParseSucceeds*(name, source: string, tree: ParseTree, skip = false): Test =
## Creates a new parser test that is expected to succeed and
## return the given tree
result = Test(expected: Success, kind: Parser)
result.outcome.line = -1
result.outcome.location = (-1, -1)
result.name = name
result.status = Init
result.source = source
result.skip = skip
# result.runnerFunc = tokenizeSucceedsRunner
result.tree = tree
proc testParseFails*(name, source: string, message: string, line: int, location: tuple[start, stop: int], skip = false): Test =
## Creates a new parser test that is expected to fail with the
## given error message and at the given location
result = Test(expected: Failed, kind: Parser)
result.name = name
result.status = Init
result.source = source
result.skip = skip
result.runnerFunc = parseFailsRunner
result.message = message
result.location = location
result.line = line
proc run*(self: TestSuite, verbose: bool = false) = proc run*(self: TestSuite, verbose: bool = false) =
## Runs the test suite to completion, ## Runs the test suite to completion,
## sequentially ## sequentially
@ -301,6 +368,8 @@ proc getExpectedException(self: Test): ref Exception =
case self.kind: case self.kind:
of Tokenizer: of Tokenizer:
return LexingError(msg: self.message, line: self.line, file: self.name, lexer: self.lexer, pos: self.location) return LexingError(msg: self.message, line: self.line, file: self.name, lexer: self.lexer, pos: self.location)
of Parser:
return ParseError(msg: self.message, line: self.line, file: self.name, parser: self.parser)
else: else:
discard # TODO discard # TODO
@ -308,11 +377,7 @@ proc getExpectedException(self: Test): ref Exception =
proc getExpectedOutcome*(self: Test): TestOutcome = proc getExpectedOutcome*(self: Test): TestOutcome =
## Gets the expected outcome of a test ## Gets the expected outcome of a test
doAssert self.expected in [Success, Failed], "expected outcome is neither Success nor Failed: wtf?" doAssert self.expected in [Success, Failed], "expected outcome is neither Success nor Failed: wtf?"
case self.kind: if self.expected == Success:
of Tokenizer: return (false, self.getExpectedException(), -1, (-1, -1))
if self.expected == Success: else:
return (false, self.getExpectedException(), -1, (-1, -1)) return (false, self.getExpectedException(), self.line, self.location)
else:
return (false, self.getExpectedException(), self.line, self.location)
else:
discard