nocturn9x
/
peon-rewrite
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Significant parser refactoring and cleanup

This commit is contained in:
Mattia Giambirtone 2023-12-27 13:33:09 +01:00
parent d04f412347
commit c0bd1daebf
Signed by: nocturn9x
GPG Key ID: 8270F9F467971E59
7 changed files with 512 additions and 582 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
.gitignore vendored
View File

@ -143,3 +143,4 @@ dmypy.json
# Cython debug symbols
cython_debug/
tests/test.pn

126
src/frontend/compiler/typechecker.nim
View File

@ -84,6 +84,7 @@ proc specialize(self: TypeChecker, name: Name, args: seq[TypedExpr], node: ASTNo
proc declareGenerics(self: TypeChecker, name: Name)
proc funDecl(self: TypeChecker, node: FunDecl, name: Name = nil): TypedFunDecl
proc getTypeDistance(self: TypeChecker, a, b: Type): int
proc addName(self: TypeChecker, name: Name)
## Public getters for nicer error formatting
proc getCurrentNode*(self: TypeChecker): ASTNode = (if self.done(): self.tree[^1] else: self.tree[self.current - 1])
@ -334,7 +335,6 @@ proc toIntrinsic(name: string): Type =
return Type(kind: Reference, value: "any".toIntrinsic(), intrinsic: true)
proc infer(self: TypeChecker, node: LiteralExpr): TypedExpr =
case node.kind:
of trueExpr, falseExpr:
@ -1076,37 +1076,48 @@ proc specialize(self: TypeChecker, name: Name, args: seq[TypedExpr], node: ASTNo
expectedCount = typ.genericTypes.len() + typ.genericValues.len()
if expectedCount == 0:
self.error(&"cannot create concrete instance of objects of type {self.stringify(typ)} (type is not a generic)")
if len(args) < expectedCount:
self.error(&"partial generic instantiation is not supported (expecting exactly {expectedCount} arguments, got {len(args)} instead)", node=node)
elif len(args) != expectedCount:
self.error(&"invalid number of arguments supplied for generic instantiation (expecting exactly {expectedCount}, got {len(args)} instead)", node=node)
case typ.kind:
of TypeKind.Structure:
result = typ.deepCopy()
result.genericTypes.clear()
result.genericValues.clear()
var replaced = newTable[string, Type]()
var i = 0
for key in typ.genericTypes.keys():
result.genericTypes[key] = self.check(args[i].kind, typ.genericTypes[key], args[i].node)
replaced[key] = result.genericTypes[key]
inc(i)
# Note how we do not reset i!
for key in typ.genericValues.keys():
# echo args[i].kind, "\n"
# echo typ.genericValues[key], "\n\n"
result.genericValues[key] = self.check(args[i].kind, typ.genericValues[key], args[i].node)
replaced[key] = result.genericValues[key]
inc(i)
for field in TypeDecl(name.node).fields:
if field.valueType.kind == identExpr:
let
fieldName = field.name.token.lexeme
fieldType = field.valueType.token.lexeme
if fieldType in replaced:
result.fields[fieldName] = replaced[fieldType]
else:
self.error(&"cannot create concrete instance of objects of type {self.stringify(typ)}")
# Construct a concrete copy of the original generic type
result = typ.deepCopy()
# Create a new hidden scope to declare fresh type variables in
self.beginScope()
var replaced = newTable[string, Type]()
var i = 0
for key in typ.genericTypes.keys():
replaced[key] = self.check(args[i].kind, typ.genericTypes[key], args[i].node)
self.addName(Name(depth: self.scopeDepth,
ident: name.node.genericTypes[key].ident,
isPrivate: true,
module: self.currentModule,
file: self.file,
valueType: replaced[key],
line: node.token.line,
owner: self.currentFunction,
kind: NameKind.Default,
node: name.node,
))
inc(i)
result.genericTypes.clear()
# Note how we do not reset i!
for key in typ.genericValues.keys():
replaced[key] = self.check(args[i].kind, typ.genericValues[key], args[i].node)
self.addName(Name(depth: self.scopeDepth,
ident: name.node.genericValues[key].ident,
isPrivate: true,
module: self.currentModule,
file: self.file,
valueType: replaced[key],
line: node.token.line,
owner: self.currentFunction,
kind: NameKind.Default,
node: name.node,
))
inc(i)
result.genericValues.clear()
# Close the hidden scope once we're done
self.endScope()
proc unpackTypes(self: TypeChecker, condition: Expression, list: var seq[tuple[match: bool, kind: Type, value: Expression]], accept: bool = true) =
@ -1145,10 +1156,8 @@ proc dispatchPragmas(self: TypeChecker, name: Name) =
case name.node.kind:
of NodeKind.funDecl, NodeKind.typeDecl, NodeKind.varDecl:
pragmas = Declaration(name.node).pragmas
of NodeKind.lambdaExpr:
pragmas = LambdaExpr(name.node).pragmas
else:
discard # Unreachable
discard # TODO
var f: PragmaFunc
for pragma in pragmas:
if pragma.name.token.lexeme notin self.pragmas:
@ -1182,7 +1191,8 @@ proc addName(self: TypeChecker, name: Name) =
if scope.hasKey(name.ident.token.lexeme):
for obj in scope[name.ident.token.lexeme]:
if name.valueType.kind == TypeKind.Function:
# We don't check for name clashes for functions because self.match() does that
# We don't check for name clashes for functions because we allow overloading.
# Our match() proc will take care of reporting any ambiguity errors at call time
continue
if (obj.kind in [NameKind.Var, NameKind.Module] or obj.valueType.kind in [TypeKind.Structure, TypeKind.EnumEntry]) and name.module == obj.module:
self.error(&"re-declaration of '{obj.ident.token.lexeme}' is not allowed (previously declared in {obj.module.ident.token.lexeme}:{obj.ident.token.line}:{obj.ident.token.relPos.start})", name.node)
@ -1337,10 +1347,10 @@ proc call(self: TypeChecker, node: CallExpr): TypedExpr =
result = self.call(exp)
]#
discard
of NodeKind.getItemExpr:
of NodeKind.getterExpr:
# Calling a.b()
# TODO
let node = GetItemExpr(node.callee)
let node = GetterExpr(node.callee)
of NodeKind.lambdaExpr:
# Calling a lambda on the fly
var node = LambdaExpr(node.callee)
@ -1533,42 +1543,42 @@ proc declareGenerics(self: TypeChecker, name: Name) =
var
constraints: seq[tuple[match: bool, kind: Type, value: Expression]] = @[]
value: Expression
for gen in name.node.genericTypes:
if gen.cond.isNil():
for gen in name.node.genericTypes.values():
if gen.constr.isNil():
constraints = @[(match: true, kind: "any".toIntrinsic(), value: value)]
else:
self.unpackTypes(gen.cond, constraints)
self.unpackTypes(gen.constr, constraints)
let generic = Name(kind: Default,
ident: gen.name,
ident: gen.ident,
module: self.currentModule,
owner: self.currentFunction,
file: self.currentModule.file,
depth: self.scopeDepth,
isPrivate: true,
valueType: if constraints.len() > 1: Type(kind: Union, types: constraints) else: constraints[0].kind,
line: gen.name.token.line,
line: gen.ident.token.line,
)
self.addName(generic)
name.valueType.genericTypes[gen.name.token.lexeme] = generic.valueType
name.valueType.genericTypes[gen.ident.token.lexeme] = generic.valueType
constraints.setLen(0)
for gen in name.node.genericValues:
if gen.cond.isNil():
for gen in name.node.genericValues.values():
if gen.constr.isNil():
constraints = @[(match: true, kind: "any".toIntrinsic(), value: value)]
else:
self.unpackTypes(gen.cond, constraints)
self.unpackTypes(gen.constr, constraints)
let generic = Name(kind: Default,
ident: gen.name,
ident: gen.ident,
module: self.currentModule,
owner: self.currentFunction,
file: self.currentModule.file,
depth: self.scopeDepth,
isPrivate: true,
valueType: (if constraints.len() > 1: Type(kind: Union, types: constraints) else: constraints[0].kind).unwrapType(),
line: gen.name.token.line,
line: gen.ident.token.line,
)
self.addName(generic)
name.valueType.genericValues[gen.name.token.lexeme] = generic.valueType
name.valueType.genericValues[gen.ident.token.lexeme] = generic.valueType
constraints.setLen(0)
@ -1588,7 +1598,7 @@ proc typeDecl(self: TypeChecker, node: TypeDecl, name: Name = nil): TypedTypeDec
# Type is a structure type
var fieldType: TypedExpr
var n: Name
for field in node.fields:
for field in node.fields.values():
fieldType = self.infer(field.valueType)
if not node.isRef:
# Check for self-recursion of non-ref types (which would require
@ -1598,9 +1608,9 @@ proc typeDecl(self: TypeChecker, node: TypeDecl, name: Name = nil): TypedTypeDec
# Expression has no associated name: cannot self-recurse
continue
if name == n:
self.error(&"illegal self-recursion in member '{field.name.token.lexeme}' for non-ref type '{name.ident.token.lexeme}'", fieldType.node)
result.fields[field.name.token.lexeme] = fieldType
name.valueType.fields[field.name.token.lexeme] = fieldType.kind
self.error(&"illegal self-recursion in member '{field.ident.token.lexeme}' for non-ref type '{name.ident.token.lexeme}'", fieldType.node)
result.fields[field.ident.token.lexeme] = fieldType
name.valueType.fields[field.ident.token.lexeme] = fieldType.kind
else:
# Type is a variant type (aka enum). We'll only declare a single
# object (the enum type itself) so that we don't pollute the
@ -1635,13 +1645,13 @@ proc typeDecl(self: TypeChecker, node: TypeDecl, name: Name = nil): TypedTypeDec
if parentName.isNil():
self.error(&"could not obtain name information for the given object: is it a type?", node.parent)
result.parent = parentName
for field in TypeDecl(result.parent.node).fields:
if result.fields.hasKey(field.name.token.lexeme):
for f in TypeDecl(result.node).fields:
if f.name.token.lexeme == field.name.token.lexeme:
for field in TypeDecl(result.parent.node).fields.values():
if result.fields.hasKey(field.ident.token.lexeme):
for f in TypeDecl(result.node).fields.values():
if f.ident.token.lexeme == field.ident.token.lexeme:
# This always eventually runs
self.error(&"cannot to re-declare type member '{field}'", f.name)
result.fields[field.name.token.lexeme] = newTypedExpr(field.name, result.parent.valueType.fields[field.name.token.lexeme])
self.error(&"cannot to re-declare type member '{field}'", f.ident)
result.fields[field.ident.token.lexeme] = newTypedExpr(field.ident, result.parent.valueType.fields[field.ident.token.lexeme])
# Turn the declared type into a typevar so that future references
# to it will be distinct from its instances
name.valueType = name.valueType.wrapType()

480
src/frontend/parser/ast.nim
View File

@ -18,6 +18,7 @@
import std/strformat
import std/strutils
import std/tables
import token
@ -27,8 +28,7 @@ export token
type
NodeKind* = enum
## Enumeration of the AST
## node types, sorted by
## precedence
## node types
# Declarations
typeDecl = 0'u8
@ -58,12 +58,12 @@ type
lambdaExpr,
awaitExpr,
yieldExpr,
setItemExpr, # Set expressions like a.b = "c"
binaryExpr,
unaryExpr,
sliceExpr,
callExpr,
getItemExpr, # Get expressions like a.b
getterExpr, # Get expressions like a.b
setterExpr, # Set expressions like a.b = "c"
# Primary expressions
groupingExpr, # Parenthesized expressions such as (true) and (3 + 4)
trueExpr,
@ -89,11 +89,10 @@ type
# Here I would've rather used object variants, and in fact that's what was in
# place before, but not being able to re-declare a field of the same type in
# another case branch is kind of a deal breaker long-term, so until that is
# fixed (check out https://github.com/nim-lang/RFCs/issues/368 for more info).
# fixed (check out https://github.com/nim-lang/RFCs/issues/368 for more info)
# I'll stick to using inheritance instead
# Generic AST node types
ASTNode* = ref object of RootObj
## An AST node
kind*: NodeKind
@ -102,173 +101,201 @@ type
# errors accurately even deep in the compilation pipeline
token*: Token
file*: string
# This weird inheritance chain is needed for the parser to
# work properly
# A bunch of aliases and structures for easier typing
Parameter* = ref object of RootObj
ident*: IdentExpr
valueType*: Expression
default*: Expression
Parameters* = TableRef[string, Parameter]
TypeField* = ref object of Parameter
isPrivate*: bool
TypeFields* = TableRef[string, TypeField]
TypeGeneric* = ref object
ident*: IdentExpr
constr*: Expression
TypeGenerics* = TableRef[string, TypeGeneric]
Declaration* = ref object of ASTNode
## A declaration
name*: IdentExpr
isPrivate*: bool
pragmas*: seq[Pragma]
genericTypes*: seq[tuple[name: IdentExpr, cond: Expression]]
genericValues*: seq[tuple[name: IdentExpr, cond: Expression]]
genericTypes*: TypeGenerics
genericValues*: TypeGenerics
Statement* = ref object of Declaration
Statement* = ref object of ASTNode
## A statement
Expression* = ref object of Statement
Expression* = ref object of ASTNode
## An expression
LiteralExpr* = ref object of Expression
# Using a string for literals makes it much easier to handle numeric types, as
# there is no overflow nor underflow or float precision issues during parsing.
# Numbers are just serialized as strings and then converted back to numbers
# before being passed to the VM, which also keeps the door open in the future
# to implementing bignum arithmetic that can take advantage of natively supported
# machine types, meaning that if a numeric type fits into a 64 bit signed/unsigned
# int then it is stored in such a type to save space, otherwise it is just converted
# to a bigint. Bigfloats with arbitrary-precision arithmetic would also be nice,
# although arguably less useful (and probably significantly slower than bigints)
## A literal expression (like a number or a string)
literal*: Token
IdentExpr* = ref object of Expression
## An identifier expression
name*: Token
depth*: int
GroupingExpr* = ref object of Expression
## A parenthesized (or "grouped") expression
expression*: Expression
GetItemExpr* = ref object of Expression
GetterExpr* = ref object of Expression
## A getter expression (e.g. "a.b")
obj*: Expression
name*: IdentExpr
SetItemExpr* = ref object of GetItemExpr
# Since a setItem expression is just
# a getItem one followed by an assignment,
# inheriting it from getItem makes sense
SetterExpr* = ref object of GetterExpr
## A setter expression (e.g. "a.b = c")
# Since a setter expression is just
# a getter one followed by an assignment,
# inheriting from it makes sense
value*: Expression
CallExpr* = ref object of Expression
## A call expression
callee*: Expression # The object being called
arguments*: tuple[positionals: seq[Expression], keyword: seq[tuple[
name: IdentExpr, value: Expression]]]
arguments*: tuple[positionals: seq[Expression], keyword: TableRef[string, tuple[name: IdentExpr, value: Expression]]]
closeParen*: Token # Needed for error reporting
GenericExpr* = ref object of Expression
## A generic instantiation expression
ident*: IdentExpr
args*: seq[Expression]
UnaryExpr* = ref object of Expression
## A unary expression
operator*: Token
a*: Expression
BinaryExpr* = ref object of UnaryExpr
## A binary expression
# Binary expressions can be seen here as unary
# expressions with an extra operand so we just
# inherit from that and add a second operand
# expressions with an extra operand, so we just
# inherit from that and add it
b*: Expression
YieldExpr* = ref object of Expression
expression*: Expression
AwaitExpr* = ref object of Expression
expression*: Expression
LambdaExpr* = ref object of Expression
## A lambda expression. This is basically
## a mirror of FunDecl without a name
body*: Statement
arguments*: seq[tuple[name: IdentExpr, valueType: Expression]]
defaults*: seq[Expression]
isGenerator*: bool
isAsync*: bool
parameters*: Parameters
returnType*: Expression
depth*: int
pragmas*: seq[Pragma]
genericTypes*: TypeGenerics
genericValues*: TypeGenerics
SliceExpr* = ref object of Expression
## A slice expression such as x[b]
expression*: Expression
ends*: seq[Expression]
elements*: seq[Expression]
AssignExpr* = ref object of Expression
## An assignment expression such
## as x = y
name*: IdentExpr
value*: Expression
ExprStmt* = ref object of Statement
## An expression followed by a semicolon
expression*: Expression
ImportStmt* = ref object of Statement
## An import statement
moduleName*: IdentExpr
fromStmt*: bool
names*: seq[IdentExpr]
ExportStmt* = ref object of Statement
## An export statement
name*: IdentExpr
AssertStmt* = ref object of Statement
## An assert statement
expression*: Expression
RaiseStmt* = ref object of Statement
exception*: Expression
BlockStmt* = ref object of Statement
## A block statement. This is basically
## the equivalent of a C scope
# We allow declarations inside a block statement.
# Kinda confusing, but it makes sense, I promise (TM)
code*: seq[Declaration]
NamedBlockStmt* = ref object of BlockStmt
## Identical to a block statement, except
## it also has a name which can be used to
## jump from/to it using break and continue
## statements
name*: IdentExpr
ForStmt* = ref object of Statement
discard # Unused
ForEachStmt* = ref object of Statement
## A foreach statement
identifier*: IdentExpr
expression*: Expression
body*: Statement
WhileStmt* = ref object of Statement
## A C-style while statement
condition*: Expression
body*: BlockStmt
AwaitStmt* = ref object of Statement
expression*: Expression
BreakStmt* = ref object of Statement
label*: IdentExpr
## A break statement
label*: IdentExpr # Optional: which named block should we jump out of?
ContinueStmt* = ref object of Statement
label*: IdentExpr
## A continue statement
label*: IdentExpr # Optional: which named block should we jump to the start of?
ReturnStmt* = ref object of Statement
value*: Expression
## A return statement
value*: Expression # Optional
IfStmt* = ref object of Statement
## An if statement
condition*: Expression
thenBranch*: Statement
elseBranch*: Statement
YieldStmt* = ref object of Statement
expression*: Expression
elseBranch*: Statement # Optional
VarDecl* = ref object of Declaration
name*: IdentExpr
constant*: bool
mutable*: bool
## A variable declaration
constant*: bool # Is this a constant?
mutable*: bool # Is the value mutable?
valueType*: Expression
value*: Expression
FunDecl* = ref object of Declaration
name*: IdentExpr
## A function declaration
body*: Statement
arguments*: seq[tuple[name: IdentExpr, valueType: Expression]]
defaults*: seq[Expression]
isAsync*: bool
isGenerator*: bool
parameters*: Parameters
returnType*: Expression
TypeDecl* = ref object of Declaration
name*: IdentExpr
# Empty if type is an enum
fields*: seq[tuple[name: IdentExpr, valueType: Expression, isPrivate: bool, default: Expression]]
# Empty if type is a structure
## A type declaration
# Empty if type is an enum:
# contains all fields of the
# structure
fields*: TypeFields
# Empty if type is a structure:
# contains all enum members (they
# are separate from fields because
# they are all their own type of sorts)
members*: seq[TypeDecl]
isEnum*: bool
isRef*: bool
isRef*: bool # Is this type a managed reference?
parent*: Expression
# Only filled if it's a type alias
value*: Expression
Pragma* = ref object of Expression
@ -296,6 +323,7 @@ type
default*: BlockStmt
proc isConst*(self: ASTNode): bool =
## Returns true if the given
## AST node represents a value
@ -313,7 +341,7 @@ proc isConst*(self: ASTNode): bool =
proc isDecl*(self: ASTNode): bool =
## Returns true if the given AST node
## represents a declaration
return self.kind in [typeDecl, funDecl, varDecl]
return self.kind in [NodeKind.typeDecl, NodeKind.funDecl, NodeKind.varDecl]
## AST node constructors
@ -326,7 +354,7 @@ proc newASTNode*(kind: NodeKind, token: Token): ASTNode =
proc newPragma*(name: IdentExpr, args: seq[LiteralExpr]): Pragma =
new(result)
result.kind = pragmaExpr
result.kind = NodeKind.pragmaExpr
result.args = args
result.name = name
result.token = name.token
@ -334,28 +362,28 @@ proc newPragma*(name: IdentExpr, args: seq[LiteralExpr]): Pragma =
proc newRefExpr*(expression: Expression, token: Token): Ref =
new(result)
result.kind = refExpr
result.kind = NodeKind.refExpr
result.value = expression
result.token = token
proc newPtrExpr*(expression: Expression, token: Token): Ptr =
new(result)
result.kind = ptrExpr
result.kind = NodeKind.ptrExpr
result.value = expression
result.token = token
proc newLentExpr*(expression: Expression, token: Token): Lent =
new(result)
result.kind = lentExpr
result.kind = NodeKind.lentExpr
result.value = expression
result.token = token
proc newConstExpr*(expression: Expression, token: Token): Const =
new(result)
result.kind = constExpr
result.kind = NodeKind.constExpr
result.value = expression
result.token = token
@ -363,7 +391,7 @@ proc newConstExpr*(expression: Expression, token: Token): Const =
proc newSwitchStmt*(switch: Expression, branches: seq[tuple[cond: Expression, body: BlockStmt]], default: BlockStmt, token: Token): SwitchStmt =
new(result)
result.kind = switchStmt
result.kind = NodeKind.switchStmt
result.switch = switch
result.branches = branches
result.token = token
@ -371,95 +399,100 @@ proc newSwitchStmt*(switch: Expression, branches: seq[tuple[cond: Expression, bo
proc newIntExpr*(literal: Token): LiteralExpr =
result = LiteralExpr(kind: intExpr)
new(result)
result.kind = NodeKind.intExpr
result.literal = literal
result.token = literal
proc newOctExpr*(literal: Token): LiteralExpr =
result = LiteralExpr(kind: octExpr)
new(result)
result.kind = NodeKind.octExpr
result.literal = literal
result.token = literal
proc newHexExpr*(literal: Token): LiteralExpr =
result = LiteralExpr(kind: hexExpr)
new(result)
result.kind = NodeKind.hexExpr
result.literal = literal
result.token = literal
proc newBinExpr*(literal: Token): LiteralExpr =
result = LiteralExpr(kind: binExpr)
new(result)
result.kind = NodeKind.binExpr
result.literal = literal
result.token = literal
proc newFloatExpr*(literal: Token): LiteralExpr =
result = LiteralExpr(kind: floatExpr)
new(result)
result.kind = NodeKind.floatExpr
result.literal = literal
result.token = literal
proc newTrueExpr*(token: Token): LiteralExpr = LiteralExpr(kind: trueExpr, token: token, literal: token)
proc newFalseExpr*(token: Token): LiteralExpr = LiteralExpr(kind: falseExpr, token: token, literal: token)
proc newNanExpr*(token: Token): LiteralExpr = LiteralExpr(kind: nanExpr, token: token, literal: token)
proc newInfExpr*(token: Token): LiteralExpr = LiteralExpr(kind: infExpr, token: token, literal: token)
proc newTrueExpr*(token: Token): LiteralExpr = LiteralExpr(kind: NodeKind.trueExpr, token: token, literal: token)
proc newFalseExpr*(token: Token): LiteralExpr = LiteralExpr(kind: NodeKind.falseExpr, token: token, literal: token)
proc newNanExpr*(token: Token): LiteralExpr = LiteralExpr(kind: NodeKind.nanExpr, token: token, literal: token)
proc newInfExpr*(token: Token): LiteralExpr = LiteralExpr(kind: NodeKind.infExpr, token: token, literal: token)
proc newStrExpr*(literal: Token): LiteralExpr =
result = LiteralExpr(kind: strExpr)
new(result)
result.kind = NodeKind.strExpr
result.literal = literal
result.token = literal
proc newCharExpr*(literal: Token): LiteralExpr =
result = LiteralExpr(kind: charExpr)
new(result)
result.kind = NodeKind.charExpr
result.literal = literal
result.token = literal
proc newIdentExpr*(name: Token, depth: int = 0): IdentExpr =
result = IdentExpr(kind: identExpr)
proc newIdentExpr*(name: Token): IdentExpr =
new(result)
result.kind = NodeKind.identExpr
result.name = name
result.token = name
result.depth = depth
proc newGroupingExpr*(expression: Expression, token: Token): GroupingExpr =
result = GroupingExpr(kind: groupingExpr)
new(result)
result.kind = NodeKind.groupingExpr
result.expression = expression
result.token = token
proc newLambdaExpr*(arguments: seq[tuple[name: IdentExpr, valueType: Expression]], defaults: seq[Expression],
body: Statement, isAsync, isGenerator: bool,
token: Token, depth: int, pragmas: seq[Pragma] = @[],
returnType: Expression, genericTypes: seq[tuple[name: IdentExpr, cond: Expression]] = @[],
genericValues: seq[tuple[name: IdentExpr, cond: Expression]] = @[]): LambdaExpr =
result = LambdaExpr(kind: lambdaExpr)
proc newLambdaExpr*(parameters: Parameters = newTable[string, Parameter](),
body: Statement = nil, token: Token = nil, pragmas: seq[Pragma] = @[],
returnType: Expression = nil, genericTypes: TypeGenerics = newTable[string, TypeGeneric](),
genericValues: TypeGenerics = newTable[string, TypeGeneric]()): LambdaExpr =
new(result)
result.kind = NodeKind.lambdaExpr
result.body = body
result.arguments = arguments
result.defaults = defaults
result.isGenerator = isGenerator
result.isAsync = isAsync
result.parameters = parameters
result.token = token
result.returnType = returnType
result.pragmas = pragmas
result.genericTypes = genericTypes
result.genericValues = genericValues
result.depth = depth
proc newGetItemExpr*(obj: Expression, name: IdentExpr,
token: Token): GetItemExpr =
result = GetItemExpr(kind: getItemExpr)
proc newGetterExpr*(obj: Expression, name: IdentExpr, token: Token): GetterExpr =
new(result)
result.kind = NodeKind.getterExpr
result.obj = obj
result.name = name
result.token = token
proc newSetItemExpr*(obj: Expression, name: IdentExpr, value: Expression,
token: Token): SetItemExpr =
result = SetItemExpr(kind: setItemExpr)
proc newSetterExpr*(obj: Expression, name: IdentExpr, value: Expression, token: Token): SetterExpr =
new(result)
result.kind = NodeKind.setterExpr
result.obj = obj
result.name = name
result.value = value
@ -467,131 +500,112 @@ proc newSetItemExpr*(obj: Expression, name: IdentExpr, value: Expression,
proc newCallExpr*(callee: Expression, arguments: tuple[positionals: seq[
Expression], keyword: seq[tuple[name: IdentExpr, value: Expression]]],
Expression], keyword: TableRef[string, tuple[name: IdentExpr, value: Expression]]],
token: Token): CallExpr =
result = CallExpr(kind: callExpr)
new(result)
result.kind = NodeKind.callExpr
result.callee = callee
result.arguments = arguments
result.token = token
proc newGenericExpr*(ident: IdentExpr, args: seq[Expression]): GenericExpr =
result = GenericExpr(kind: genericExpr)
new(result)
result.kind = NodeKind.genericExpr
result.ident = ident
result.args = args
result.token = ident.token
proc newSliceExpr*(expression: Expression, ends: seq[Expression], token: Token): SliceExpr =
result = SliceExpr(kind: sliceExpr)
proc newSliceExpr*(expression: Expression, elements: seq[Expression], token: Token): SliceExpr =
new(result)
result.kind = NodeKind.sliceExpr
result.expression = expression
result.ends = ends
result.elements = elements
result.token = token
proc newUnaryExpr*(operator: Token, a: Expression): UnaryExpr =
result = UnaryExpr(kind: unaryExpr)
new(result)
result.kind = NodeKind.unaryExpr
result.operator = operator
result.a = a
result.token = result.operator
proc newBinaryExpr*(a: Expression, operator: Token, b: Expression): BinaryExpr =
result = BinaryExpr(kind: binaryExpr)
new(result)
result.kind = NodeKind.binaryExpr
result.operator = operator
result.a = a
result.b = b
result.token = operator
proc newYieldExpr*(expression: Expression, token: Token): YieldExpr =
result = YieldExpr(kind: yieldExpr)
result.expression = expression
result.token = token
proc newAssignExpr*(name: IdentExpr, value: Expression,
token: Token): AssignExpr =
result = AssignExpr(kind: assignExpr)
new(result)
result.kind = NodeKind.assignExpr
result.name = name
result.value = value
result.token = token
proc newAwaitExpr*(expression: Expression, token: Token): AwaitExpr =
result = AwaitExpr(kind: awaitExpr)
result.expression = expression
result.token = token
proc newExprStmt*(expression: Expression, token: Token): ExprStmt =
result = ExprStmt(kind: exprStmt)
new(result)
result.kind = NodeKind.exprStmt
result.expression = expression
result.token = token
proc newImportStmt*(moduleName: IdentExpr, fromStmt: bool, names: seq[IdentExpr], token: Token): ImportStmt =
result = ImportStmt(kind: importStmt)
proc newImportStmt*(moduleName: IdentExpr, names: seq[IdentExpr], token: Token): ImportStmt =
new(result)
result.kind = NodeKind.importStmt
result.moduleName = moduleName
result.token = token
result.fromStmt = fromStmt
result.names = names
proc newExportStmt*(name: IdentExpr, token: Token): ExportStmt =
result = ExportStmt(kind: exportStmt)
new(result)
result.kind = NodeKind.exportStmt
result.name = name
result.token = token
proc newYieldStmt*(expression: Expression, token: Token): YieldStmt =
result = YieldStmt(kind: yieldStmt)
result.expression = expression
result.token = token
proc newAwaitStmt*(expression: Expression, token: Token): AwaitStmt =
result = AwaitStmt(kind: awaitStmt)
result.expression = expression
result.token = token
proc newAssertStmt*(expression: Expression, token: Token): AssertStmt =
result = AssertStmt(kind: assertStmt)
new(result)
result.kind = NodeKind.assertStmt
result.expression = expression
result.token = token
proc newRaiseStmt*(exception: Expression, token: Token): RaiseStmt =
result = RaiseStmt(kind: raiseStmt)
result.exception = exception
result.token = token
proc newBlockStmt*(code: seq[Declaration], token: Token): BlockStmt =
result = BlockStmt(kind: blockStmt)
new(result)
result.kind = NodeKind.blockStmt
result.code = code
result.token = token
proc newNamedBlockStmt*(code: seq[Declaration], name: IdentExpr, token: Token): NamedBlockStmt =
result = NamedBlockStmt(kind: namedBlockStmt)
new(result)
result.kind = NodeKind.namedBlockStmt
result.code = code
result.token = token
result.name = name
proc newWhileStmt*(condition: Expression, body: BlockStmt,
token: Token): WhileStmt =
result = WhileStmt(kind: whileStmt)
proc newWhileStmt*(condition: Expression, body: BlockStmt, token: Token): WhileStmt =
new(result)
result.kind = NodeKind.whileStmt
result.condition = condition
result.body = body
result.token = token
proc newForEachStmt*(identifier: IdentExpr, expression: Expression,
body: Statement, token: Token): ForEachStmt =
result = ForEachStmt(kind: forEachStmt)
proc newForEachStmt*(identifier: IdentExpr, expression: Expression, body: Statement, token: Token): ForEachStmt =
new(result)
result.kind = NodeKind.forEachStmt
result.identifier = identifier
result.expression = expression
result.body = body
@ -599,26 +613,29 @@ proc newForEachStmt*(identifier: IdentExpr, expression: Expression,
proc newBreakStmt*(token: Token, label: IdentExpr = nil): BreakStmt =
result = BreakStmt(kind: breakStmt)
new(result)
result.kind = NodeKind.breakStmt
result.token = token
result.label = label
proc newContinueStmt*(token: Token, label: IdentExpr = nil): ContinueStmt =
result = ContinueStmt(kind: continueStmt)
new(result)
result.kind = NodeKind.continueStmt
result.token = token
result.label = label
proc newReturnStmt*(value: Expression, token: Token): ReturnStmt =
result = ReturnStmt(kind: returnStmt)
new(result)
result.kind = NodeKind.returnStmt
result.value = value
result.token = token
proc newIfStmt*(condition: Expression, thenBranch, elseBranch: Statement,
token: Token): IfStmt =
result = IfStmt(kind: ifStmt)
proc newIfStmt*(condition: Expression, thenBranch, elseBranch: Statement, token: Token): IfStmt =
new(result)
result.kind = NodeKind.ifStmt
result.condition = condition
result.thenBranch = thenBranch
result.elseBranch = elseBranch
@ -639,17 +656,14 @@ proc newVarDecl*(name: IdentExpr, valueType, value: Expression, token: Token,
result.pragmas = pragmas
proc newFunDecl*(name: IdentExpr, arguments: seq[tuple[name: IdentExpr, valueType: Expression]], defaults: seq[Expression],
body: Statement, isAsync, isGenerator: bool, isPrivate: bool, token: Token, pragmas: seq[Pragma] = @[],
returnType: Expression, genericTypes: seq[tuple[name: IdentExpr, cond: Expression]] = @[],
genericValues: seq[tuple[name: IdentExpr, cond: Expression]] = @[]): FunDecl =
result = FunDecl(kind: funDecl)
proc newFunDecl*(name: IdentExpr, parameters: Parameters, body: Statement, isPrivate: bool, token: Token, pragmas: seq[Pragma] = @[],
returnType: Expression, genericTypes: TypeGenerics = newTable[string, TypeGeneric](),
genericValues: TypeGenerics = newTable[string, TypeGeneric]()): FunDecl =
new(result)
result.kind = NodeKind.funDecl
result.name = name
result.arguments = arguments
result.defaults = defaults
result.parameters = parameters
result.body = body
result.isAsync = isAsync
result.isGenerator = isGenerator
result.isPrivate = isPrivate
result.token = token
result.pragmas = pragmas
@ -658,12 +672,10 @@ proc newFunDecl*(name: IdentExpr, arguments: seq[tuple[name: IdentExpr, valueTyp
result.genericValues = genericValues
proc newTypeDecl*(name: IdentExpr, fields: seq[tuple[name: IdentExpr, valueType: Expression, isPrivate: bool, default: Expression]],
defaults: seq[Expression], isPrivate: bool, token: Token, pragmas: seq[Pragma],
genericTypes: seq[tuple[name: IdentExpr, cond: Expression]],
genericValues: seq[tuple[name: IdentExpr, cond: Expression]],
parent: IdentExpr, isEnum: bool, isRef: bool): TypeDecl =
result = TypeDecl(kind: typeDecl)
proc newTypeDecl*(name: IdentExpr, fields: TypeFields, isPrivate: bool, token: Token, pragmas: seq[Pragma], parent: IdentExpr, isEnum: bool, isRef: bool,
genericTypes: TypeGenerics = newTable[string, TypeGeneric](), genericValues: TypeGenerics = newTable[string, TypeGeneric]()): TypeDecl =
new(result)
result.kind = NodeKind.typeDecl
result.name = name
result.fields = fields
result.isPrivate = isPrivate
@ -677,13 +689,18 @@ proc newTypeDecl*(name: IdentExpr, fields: seq[tuple[name: IdentExpr, valueType:
result.members = @[]
proc `$`*(self: Parameter): string
proc `$`*(self: TypeField): string
proc `$`*(self: TypeGeneric): string
proc `$`*(self: ASTNode): string =
if self.isNil():
return "nil"
case self.kind:
of intExpr, floatExpr, hexExpr, binExpr, octExpr, strExpr, trueExpr,
falseExpr:
if self.kind in {trueExpr, falseExpr}:
falseExpr, infExpr, nanExpr:
if self.kind in {trueExpr, falseExpr, infExpr, nanExpr}:
result &= &"Literal({($self.kind)[0..^5]})"
elif self.kind == strExpr:
result &= &"Literal({LiteralExpr(self).literal.lexeme[1..^2].escape()})"
@ -693,15 +710,15 @@ proc `$`*(self: ASTNode): string =
result &= &"Identifier('{IdentExpr(self).name.lexeme}')"
of groupingExpr:
result &= &"Grouping({GroupingExpr(self).expression})"
of getItemExpr:
var self = GetItemExpr(self)
result &= &"GetItem(obj={self.obj}, name={self.name})"
of setItemExpr:
var self = SetItemExpr(self)
result &= &"SetItem(obj={self.obj}, name={self.value}, value={self.value})"
of getterExpr:
var self = GetterExpr(self)
result &= &"Getter(obj={self.obj}, name={self.name})"
of setterExpr:
var self = SetterExpr(self)
result &= &"Setter(obj={self.obj}, name={self.value}, value={self.value})"
of callExpr:
var self = CallExpr(self)
result &= &"""Call({self.callee}, arguments=(positionals=[{self.arguments.positionals.join(", ")}], keyword=[{self.arguments.keyword.join(", ")}]))"""
result &= &"""Call({self.callee}, arguments=(positionals={self.arguments.positionals}, keyword={self.arguments.keyword}))"""
of unaryExpr:
var self = UnaryExpr(self)
result &= &"Unary(Operator('{self.operator.lexeme}'), {self.a})"
@ -722,13 +739,10 @@ proc `$`*(self: ASTNode): string =
result = &"Break({self.label})"
of importStmt:
var self = ImportStmt(self)
result &= &"Import({self.moduleName}, names={self.names}, fromStmt={self.fromStmt})"
result &= &"Import({self.moduleName}, names={self.names})"
of assertStmt:
var self = AssertStmt(self)
result &= &"Assert({self.expression})"
of raiseStmt:
var self = RaiseStmt(self)
result &= &"Raise({self.exception})"
of blockStmt:
var self = BlockStmt(self)
result &= &"""Block([{self.code.join(", ")}])"""
@ -744,39 +758,27 @@ proc `$`*(self: ASTNode): string =
of returnStmt:
var self = ReturnStmt(self)
result &= &"Return({self.value})"
of yieldExpr:
var self = YieldExpr(self)
result &= &"Yield({self.expression})"
of awaitExpr:
var self = AwaitExpr(self)
result &= &"Await({self.expression})"
of ifStmt:
var self = IfStmt(self)
if self.elseBranch == nil:
result &= &"If(condition={self.condition}, thenBranch={self.thenBranch}, elseBranch=nil)"
else:
result &= &"If(condition={self.condition}, thenBranch={self.thenBranch}, elseBranch={self.elseBranch})"
of yieldStmt:
var self = YieldStmt(self)
result &= &"YieldStmt({self.expression})"
of awaitStmt:
var self = AwaitStmt(self)
result &= &"AwaitStmt({self.expression})"
of varDecl:
var self = VarDecl(self)
result &= &"Var(name={self.name}, type={self.valueType}, value={self.value}, mutable={self.mutable}, constant={self.constant}, private={self.isPrivate}, pragmas={self.pragmas})"
of funDecl:
var self = FunDecl(self)
result &= &"""FunDecl(name={self.name}, body={self.body}, type={self.returnType}, arguments=[{self.arguments.join(", ")}], defaults=[{self.defaults.join(", ")}], genericTypes={self.genericTypes}, genericValues={self.genericValues}, async={self.isAsync}, generator={self.isGenerator}, private={self.isPrivate}, pragmas={self.pragmas})"""
result &= &"""FunDecl(name={self.name}, body={self.body}, returnType={self.returnType}, parameters={self.parameters}, genericTypes={self.genericTypes}, genericValues={self.genericValues}, private={self.isPrivate}, pragmas={self.pragmas})"""
of typeDecl:
var self = TypeDecl(self)
result &= &"""TypeDecl(name={self.name}, fields={self.fields}, members={self.members}, private={self.isPrivate}, pragmas={self.pragmas}, genericTypes={self.genericTypes}, genericValues={self.genericValues}, parent={self.parent}, ref={self.isRef}, enum={self.isEnum}, value={self.value})"""
of lambdaExpr:
var self = LambdaExpr(self)
result &= &"""Lambda(body={self.body}, type={self.returnType}, arguments=[{self.arguments.join(", ")}], defaults=[{self.defaults.join(", ")}], generator={self.isGenerator}, async={self.isAsync}, pragmas={self.pragmas})"""
result &= &"""Lambda(body={self.body}, returnType={self.returnType}, parameters={self.parameters}, pragmas={self.pragmas})"""
of sliceExpr:
var self = SliceExpr(self)
result &= &"""Slice({self.expression}, ends=[{self.ends.join(", ")}])"""
result &= &"""Slice({self.expression}, elements=[{self.elements.join(", ")}])"""
of pragmaExpr:
var self = Pragma(self)
result &= &"Pragma(name={self.name}, args={self.args})"
@ -794,6 +796,10 @@ proc `$`*(self: ASTNode): string =
else:
discard
proc `$`*(self: Parameter): string = &"Parameter(name={self.ident}, type={self.valueType}, default={self.default})"
proc `$`*(self: TypeField): string = &"Field(name={self.ident}, type={self.valueType}, default={self.default}, private={self.isPrivate})"
proc `$`*(self: TypeGeneric): string = &"Parameter(name={self.ident}, constraint={self.constr})"
proc `==`*(self, other: IdentExpr): bool {.inline.} = self.token == other.token
@ -802,7 +808,7 @@ proc getRelativeBoundaries*(self: ASTNode): tuple[start, stop: int] =
## Recursively computes the position of a node relative
## to its containing line
case self.kind:
of varDecl:
of NodeKind.varDecl:
var self = VarDecl(self)
let start = self.token.relPos.start
var stop = self.name.token.relPos.stop
@ -813,36 +819,28 @@ proc getRelativeBoundaries*(self: ASTNode): tuple[start, stop: int] =
if self.pragmas.len() > 0:
stop = getRelativeBoundaries(self.pragmas[^1]).stop
result = (start, stop)
of typeDecl:
of NodeKind.typeDecl:
result = (self.token.relPos.start, TypeDecl(self).name.getRelativeBoundaries().stop)
of breakStmt, returnStmt, continueStmt:
result = self.token.relPos
of importStmt:
of NodeKind.importStmt:
result = (self.token.relPos.start, getRelativeBoundaries(ImportStmt(self).moduleName).stop)
of exprStmt:
of NodeKind.exprStmt:
result = getRelativeBoundaries(ExprStmt(self).expression)
of unaryExpr:
of NodeKind.unaryExpr:
var self = UnaryExpr(self)
result = (self.operator.relPos.start, getRelativeBoundaries(self.a).stop)
of binaryExpr:
of NodeKind.binaryExpr:
var self = BinaryExpr(self)
result = (getRelativeBoundaries(self.a).start, getRelativeBoundaries(self.b).stop)
of intExpr, binExpr, hexExpr, octExpr, strExpr, floatExpr, trueExpr, falseExpr:
var self = LiteralExpr(self)
result = self.literal.relPos
of identExpr:
var self = IdentExpr(self)
result = self.token.relPos
of assignExpr:
of NodeKind.assignExpr:
var self = AssignExpr(self)
result = (getRelativeBoundaries(self.name).start, getRelativeBoundaries(self.value).stop)
of callExpr:
of NodeKind.callExpr:
var self = CallExpr(self)
result = (getRelativeBoundaries(self.callee).start, self.closeParen.relPos.stop)
of getItemExpr:
var self = GetItemExpr(self)
of NodeKind.getterExpr:
var self = GetterExpr(self)
result = (getRelativeBoundaries(self.obj).start, getRelativeBoundaries(self.name).stop)
of pragmaExpr:
of NodeKind.pragmaExpr:
var self = Pragma(self)
let start = self.token.relPos.start
var stop = 0
@ -852,7 +850,7 @@ proc getRelativeBoundaries*(self: ASTNode): tuple[start, stop: int] =
stop = self.token.relPos.stop + 1
# -8 so the error highlights the #pragma[ part as well
result = (self.token.relPos.start - 8, stop)
of genericExpr:
of NodeKind.genericExpr:
var self = GenericExpr(self)
let ident = getRelativeBoundaries(self.ident)
var stop: int = ident.stop
@ -861,15 +859,17 @@ proc getRelativeBoundaries*(self: ASTNode): tuple[start, stop: int] =
# Take the "]" into account
inc(stop)
result = (ident.start, stop)
of refExpr:
of NodeKind.refExpr:
var self = Ref(self)
result = (self.token.relPos.start, self.value.getRelativeBoundaries().stop)
of ptrExpr:
of NodeKind.ptrExpr:
var self = Ptr(self)
result = (self.token.relPos.start, self.value.getRelativeBoundaries().stop)
of constExpr:
of NodeKind.constExpr:
var self = Const(self)
result = (self.token.relPos.start, self.value.getRelativeBoundaries().stop)
else:
result = (0, 0)
# A good chunk of node types has enough information
# in their token object already
result = self.token.relPos

477
src/frontend/parser/parser.nim
View File

@ -78,14 +78,6 @@ type
# the break statement being used outside
# loops. Maybe a bit overkill for a parser?
loopDepth: int
# Stores the current function
# being parsed. This is a reference
# to either a FunDecl or LambdaExpr
# AST node and is nil when the parser
# is at the top-level. It allows the
# parser to detect errors like using
# return outside functions
currentFunction: Declaration
# Stores the current scope depth (0 = global, > 0 local)
scopeDepth: int
# Operator table
@ -175,7 +167,6 @@ proc getCurrentToken*(self: Parser): Token {.inline.} = (if self.getCurrent() >=
self.tokens.high() or
self.getCurrent() - 1 < 0: self.tokens[^1] else: self.tokens[self.current - 1])
proc getSource*(self: Parser): string {.inline.} = self.source
proc getCurrentFunction*(self: Parser): Declaration {.inline.} = self.currentFunction
template endOfFile: Token = Token(kind: EndOfFile, lexeme: "", line: -1)
template endOfLine(msg: string, tok: Token = nil) = self.expect(Semicolon, msg, tok)
@ -238,21 +229,23 @@ proc error(self: Parser, message: string, token: Token = nil) {.raises: [ParseEr
# tell at tokenization time which of the two contexts we're in, we just treat everything
# as a symbol and in the cases where we need a specific token we just match the string
# directly
func check[T: TokenType or string](self: Parser, kind: T, distance: int = 0): bool {.inline.} =
func check(self: Parser, kind: TokenType, distance: int = 0): bool {.inline.} =
## Checks if the given token at the given distance
## matches the expected kind and returns a boolean.
## The distance parameter is passed directly to
## self.peek()
when T is TokenType:
# Usually I'm not a fan of templates, but
# this is kind of nice
self.peek(distance).kind == kind
else:
when T is string:
self.peek(distance).lexeme == kind
self.peek(distance).kind == kind
proc check[T: TokenType or string](self: Parser, kind: openarray[T]): bool {.inline.} =
func check(self: Parser, kind: string, distance: int = 0): bool {.inline.} =
## Checks if the given token at the given distance
## matches the expected kind and returns a boolean.
## The distance parameter is passed directly to
## self.peek()
self.peek(distance).lexeme == kind
proc check(self: Parser, kind: openarray[TokenType]): bool {.inline.} =
## Calls self.check() in a loop with each element of
## the given openarray of token kinds and returns
## at the first match. Note that this assumes
@ -264,7 +257,7 @@ proc check[T: TokenType or string](self: Parser, kind: openarray[T]): bool {.inl
return false
proc match[T: TokenType or string](self: Parser, kind: T): bool {.inline.} =
proc match(self: Parser, kind: TokenType): bool {.inline.} =
## Behaves like self.check(), except that when a token
## matches it is also consumed
if self.check(kind):
@ -274,7 +267,7 @@ proc match[T: TokenType or string](self: Parser, kind: T): bool {.inline.} =
result = false
proc match[T: TokenType or string](self: Parser, kind: openarray[T]): bool {.inline.} =
proc match(self: Parser, kind: openarray[TokenType]): bool {.inline.} =
## Calls self.match() in a loop with each element of
## the given openarray of token kinds and returns
## at the first match. Note that this assumes
@ -286,7 +279,7 @@ proc match[T: TokenType or string](self: Parser, kind: openarray[T]): bool {.inl
result = false
proc expect[T: TokenType or string](self: Parser, kind: T, message: string = "", token: Token = nil) {.inline.} =
proc expect(self: Parser, kind: TokenType, message: string = "", token: Token = nil) {.inline.} =
## Behaves like self.match(), except that
## when a token doesn't match, an error
## is raised. If no error message is
@ -298,7 +291,7 @@ proc expect[T: TokenType or string](self: Parser, kind: T, message: string = "",
self.error(message)
proc expect[T: TokenType or string](self: Parser, kind: openarray[T], message: string = "", token: Token = nil) {.inline, used.} =
proc expect(self: Parser, kind: openarray[TokenType], message: string = "", token: Token = nil) {.inline, used.} =
## Behaves like self.expect(), except that
## an error is raised only if none of the
## given token kinds matches
@ -309,13 +302,70 @@ proc expect[T: TokenType or string](self: Parser, kind: openarray[T], message: s
self.error(&"""expecting any of the following tokens: {kind.join(", ")}, but got {self.peek().kind} instead""", token)
proc check(self: Parser, kind: openarray[string]): bool {.inline.} =
## Calls self.check() in a loop with each element of
## the given openarray of strings and returns
## at the first match. Note that this assumes
## that only one token may match at a given
## position
for k in kind:
if self.check(k):
return true
return false
proc match(self: Parser, kind: string): bool {.inline.} =
## Behaves like self.check(), except that when a string
## matches it is also consumed
if self.check(kind):
discard self.step()
result = true
else:
result = false
proc match(self: Parser, kind: openarray[string]): bool {.inline.} =
## Calls self.match() in a loop with each element of
## the given openarray of strings and returns
## at the first match. Note that this assumes
## that only one token may exist at a given
## position
for k in kind:
if self.match(k):
return true
result = false
proc expect(self: Parser, kind: string, message: string = "", token: Token = nil) {.inline.} =
## Behaves like self.match(), except that
## when a string doesn't match, an error
## is raised. If no error message is
## given, a default one is used
if not self.match(kind):
if message.len() == 0:
self.error(&"expecting token of kind {kind}, found {self.peek().kind} instead", token)
else:
self.error(message)
proc expect(self: Parser, kind: openarray[string], message: string = "", token: Token = nil) {.inline, used.} =
## Behaves like self.expect(), except that
## an error is raised only if none of the
## given strings matches
for k in kind:
if self.match(kind):
return
if message.len() == 0:
self.error(&"""expecting any of the following tokens: {kind.join(", ")}, but got {self.peek().kind} instead""", token)
# Forward declarations
proc expression(self: Parser): Expression
proc expressionStatement(self: Parser): Statement
proc statement(self: Parser): Statement
proc varDecl(self: Parser): Declaration
proc parseFunExpr(self: Parser): LambdaExpr
proc funDecl(self: Parser, isAsync: bool = false, isGenerator: bool = false, isLambda: bool = false, isOperator: bool = false): Declaration
proc funDecl(self: Parser, isOperator: bool = false): FunDecl
proc declaration(self: Parser): Declaration
proc parse*(self: Parser, tokens: seq[Token], file: string, lines: seq[tuple[start, stop: int]], source: string, persist: bool = false): seq[ASTNode]
proc findOperators(self: Parser, tokens: seq[Token])
@ -339,31 +389,11 @@ proc primary(self: Parser): Expression =
of Integer:
result = newIntExpr(self.step())
of Identifier:
result = newIdentExpr(self.step(), self.scopeDepth)
result = newIdentExpr(self.step())
of LeftParen:
let tok = self.step()
result = newGroupingExpr(self.expression(), tok)
self.expect(RightParen, "unterminated parenthesized expression")
of Yield:
let tok = self.step()
if self.currentFunction.isNil():
self.error("'yield' cannot be used outside functions", tok)
elif self.currentFunction.token.kind != Generator:
# It's easier than doing conversions for lambda/funDecl
self.error("'yield' cannot be used outside generators", tok)
if not self.check([RightBrace, RightBracket, RightParen, Comma, Semicolon]):
# Expression delimiters
result = newYieldExpr(self.expression(), tok)
else:
# Empty yield
result = newYieldExpr(nil, tok)
of Await:
let tok = self.step()
if self.currentFunction.isNil():
self.error("'await' cannot be used outside functions", tok)
if self.currentFunction.token.kind != Coroutine:
self.error("'await' can only be used inside coroutines", tok)
result = newAwaitExpr(self.expression(), tok)
of RightParen, RightBracket, RightBrace:
# This is *technically* unnecessary: the parser would
# throw an error regardless, but it's a little bit nicer
@ -381,15 +411,6 @@ proc primary(self: Parser): Expression =
result = newInfExpr(self.step())
of TokenType.Nan:
result = newNanExpr(self.step())
of Function:
discard self.step()
result = Expression(self.funDecl(isLambda=true))
of Coroutine:
discard self.step()
result = Expression(self.funDecl(isAsync=true, isLambda=true))
of Generator:
discard self.step()
result = Expression(self.funDecl(isGenerator=true, isLambda=true))
# We only allow expressions with precedence lower than assignment
# inside ref/ptr/lent/const expressions because this allows us to
# parse variable declarations such as var x: ref type = value; without
@ -417,9 +438,8 @@ proc makeCall(self: Parser, callee: Expression): CallExpr =
## to parse a function call
let tok = self.peek(-1)
var argNames: seq[IdentExpr] = @[]
var arguments: tuple[positionals: seq[Expression], keyword: seq[tuple[
name: IdentExpr, value: Expression]]] = (positionals: @[],
keyword: @[])
var arguments: tuple[positionals: seq[Expression], keyword: TableRef[string, tuple[name: IdentExpr, value: Expression]]] = (positionals: @[],
keyword: newTable[string, tuple[name: IdentExpr, value: Expression]]())
var argument: Expression = nil
var argCount = 0
if not self.check(RightParen):
@ -432,7 +452,7 @@ proc makeCall(self: Parser, callee: Expression): CallExpr =
if assign.name in argNames:
self.error("duplicate keyword arguments are not allowed", assign.name.token)
argNames.add(assign.name)
arguments.keyword.add((name: assign.name, value: assign.value))
arguments.keyword[assign.name.token.lexeme] = (name: assign.name, value: assign.value)
elif arguments.keyword.len() == 0:
arguments.positionals.add(argument)
else:
@ -448,7 +468,7 @@ proc makeCall(self: Parser, callee: Expression): CallExpr =
proc parseGenericArgs(self: Parser): Expression =
## Parses generic instantiation expressions
var item = newIdentExpr(self.peek(-2), self.scopeDepth)
var item = newIdentExpr(self.peek(-2))
var types: seq[Expression] = @[]
while not self.check(RightBracket) and not self.done():
types.add(self.expression())
@ -467,7 +487,7 @@ proc call(self: Parser): Expression =
result = self.makeCall(result)
elif self.match(Dot):
self.expect(Identifier, "expecting attribute name after '.'")
result = newGetItemExpr(result, newIdentExpr(self.peek(-1), self.scopeDepth), self.peek(-1))
result = newGetterExpr(result, newIdentExpr(self.peek(-1)), self.peek(-1))
result.file = self.file
elif self.match(LeftBracket):
if self.peek(-2).kind != Identifier:
@ -583,8 +603,8 @@ proc parseAssign(self: Parser): Expression =
of identExpr, sliceExpr:
result = newAssignExpr(IdentExpr(result), value, tok)
result.file = self.file
of getItemExpr:
result = newSetItemExpr(GetItemExpr(result).obj, GetItemExpr(result).name, value, tok)
of getterExpr:
result = newSetterExpr(GetterExpr(result).obj, GetterExpr(result).name, value, tok)
result.file = self.file
else:
self.error("invalid assignment target", tok)
@ -638,7 +658,7 @@ proc namedBlockStmt(self: Parser): Statement =
let tok = self.peek(-1)
var code: seq[Declaration] = @[]
self.expect(Identifier, "expecting block name after 'block'")
var name = newIdentExpr(self.peek(-1), self.scopeDepth)
var name = newIdentExpr(self.peek(-1))
name.file = self.file
inc(self.loopDepth)
self.expect(LeftBrace, "expecting '{' after 'block'")
@ -660,7 +680,7 @@ proc breakStmt(self: Parser): Statement =
if self.loopDepth == 0:
self.error("'break' cannot be used outside loops")
if self.match(Identifier):
label = newIdentExpr(self.peek(-1), self.scopeDepth)
label = newIdentExpr(self.peek(-1))
label.file = self.file
endOfLine("missing semicolon after 'break'")
result = newBreakStmt(tok, label)
@ -674,7 +694,7 @@ proc continueStmt(self: Parser): Statement =
if self.loopDepth == 0:
self.error("'continue' cannot be used outside loops")
if self.match(Identifier):
label = newIdentExpr(self.peek(-1), self.scopeDepth)
label = newIdentExpr(self.peek(-1))
label.file = self.file
endOfLine("missing semicolon after 'continue'")
result = newContinueStmt(tok, label)
@ -684,8 +704,6 @@ proc continueStmt(self: Parser): Statement =
proc returnStmt(self: Parser): Statement =
## Parses return statements
let tok = self.peek(-1)
if self.currentFunction.isNil():
self.error("'return' cannot be used outside functions")
var value: Expression
if not self.check(Semicolon):
# Since return can be used on its own too
@ -697,52 +715,12 @@ proc returnStmt(self: Parser): Statement =
result.file = self.file
proc yieldStmt(self: Parser): Statement =
## Parses yield statements
let tok = self.peek(-1)
if self.currentFunction.isNil():
self.error("'yield' cannot be outside functions")
elif self.currentFunction.token.kind != Generator:
self.error("'yield' can only be used inside generators")
if not self.check(Semicolon):
result = newYieldStmt(self.expression(), tok)
else:
result = newYieldStmt(nil, tok)
result.file = self.file
endOfLine("missing semicolon after 'yield'")
proc awaitStmt(self: Parser): Statement =
## Parses await statements
let tok = self.peek(-1)
if self.currentFunction.isNil():
self.error("'await' cannot be used outside functions")
if self.currentFunction.token.kind != Coroutine:
self.error("'await' can only be used inside coroutines")
endOfLine("missing semicolon after 'await'")
result = newAwaitStmt(self.expression(), tok)
result.file = self.file
proc raiseStmt(self: Parser): Statement =
## Parses raise statements
var exception: Expression
let tok = self.peek(-1)
if not self.check(Semicolon):
# Raise can be used on its own, in which
# case it re-raises the last active exception
exception = self.expression()
endOfLine("missing semicolon after 'raise'")
result = newRaiseStmt(exception, tok)
result.file = self.file
proc forEachStmt(self: Parser): Statement =
## Parses C#-like foreach loops
let tok = self.peek(-1)
inc(self.loopDepth)
self.expect(Identifier)
let identifier = newIdentExpr(self.peek(-1), self.scopeDepth)
let identifier = newIdentExpr(self.peek(-1))
self.expect("in")
let expression = self.expression()
self.expect(LeftBrace)
@ -751,7 +729,7 @@ proc forEachStmt(self: Parser): Statement =
dec(self.loopDepth)
proc importStmt(self: Parser, fromStmt: bool = false): Statement =
proc importStmt(self: Parser): Statement =
## Parses import statements. This is a little
## convoluted because we need to pre-parse the
## module to import the operators from it
@ -773,8 +751,6 @@ proc importStmt(self: Parser, fromStmt: bool = false): Statement =
moduleName &= self.peek(-1).lexeme
else:
break
if fromStmt:
self.expect(Import)
while not self.check(Semicolon) and not self.done():
self.expect(Identifier, "expecting identifier after 'import'")
names.add(newIdentExpr(self.peek(-1)))
@ -784,8 +760,7 @@ proc importStmt(self: Parser, fromStmt: bool = false): Statement =
result = newImportStmt(newIdentExpr(Token(kind: Identifier, lexeme: moduleName,
line: self.peek(-1).line,
pos: (tok.pos.stop + 1, (tok.pos.stop + 1) + len(moduleName)),
relPos: (tok.relPos.stop + 1, (tok.relPos.stop + 1) + len(moduleName))),
self.scopeDepth), fromStmt, names, tok)
relPos: (tok.relPos.stop + 1, (tok.relPos.stop + 1) + len(moduleName)))), names, tok)
result.file = self.file
moduleName &= ".pn"
var lexer = newLexer()
@ -887,7 +862,7 @@ proc parsePragmas(self: Parser): seq[Pragma] =
if self.peek(-1).lexeme in names:
self.error("duplicate pragmas are not allowed")
names.add(self.peek(-1).lexeme)
name = newIdentExpr(self.peek(-1), self.scopeDepth)
name = newIdentExpr(self.peek(-1))
name.file = self.file
if self.match("]"):
result.add(newPragma(name, @[]))
@ -919,7 +894,7 @@ proc varDecl(self: Parser): Declaration =
var tok = self.peek(-1)
self.expect(Identifier, &"expecting identifier after '{tok.lexeme}'")
var
name = newIdentExpr(self.peek(-1), self.scopeDepth)
name = newIdentExpr(self.peek(-1))
value: Expression
valueType: Expression
let isPrivate = not self.match("*")
@ -949,58 +924,52 @@ proc varDecl(self: Parser): Declaration =
result.file = self.file
proc parseDeclArguments(self: Parser, arguments: var seq[tuple[name: IdentExpr, valueType: Expression]],
parameter: var tuple[name: IdentExpr, valueType: Expression],
defaults: var seq[Expression]) =
## Helper to parse declaration arguments and avoid code duplication
proc parseDeclParams(self: Parser, parameters: Parameters) =
## Helper to parse declaration parameters and avoid code duplication
var
ident: IdentExpr
valueType: Expression
default: Expression
metDefaults = false
while not self.check(RightParen):
if arguments.len > 255:
if parameters.len() > 255:
self.error("cannot have more than 255 arguments in function declaration", self.peek(-1))
self.expect(Identifier, "expecting parameter name")
parameter.name = newIdentExpr(self.peek(-1), self.scopeDepth)
parameter.name.file = self.file
ident = newIdentExpr(self.peek(-1))
ident.file = self.file
if self.match(":"):
parameter.valueType = self.expression()
for i in countdown(arguments.high(), 0):
if arguments[i].valueType != nil:
break
arguments[i].valueType = parameter.valueType
else:
parameter.valueType = nil
if parameter in arguments:
self.error("duplicate parameter name in function declaration", parameter.name.token)
arguments.add(parameter)
valueType = self.expression()
if ident.token.lexeme in parameters:
self.error("duplicate parameter name in function declaration is not allowed", ident.token)
if self.match("="):
defaults.add(self.expression())
elif defaults.len() > 0:
self.error("positional argument cannot follow default argument in function declaration", parameter.name.token)
default = self.expression()
metDefaults = true
else:
default = nil
if default.isNil() and metDefaults:
self.error("positional argument cannot follow default argument in function declaration", ident.token)
parameters[ident.token.lexeme] = Parameter(ident: ident, valueType: valueType, default: default)
if not self.match(Comma):
break
self.expect(RightParen)
for argument in arguments:
if argument.valueType.isNil():
self.error(&"missing type declaration for '{argument.name.token.lexeme}' in function declaration")
for parameter in parameters.values():
if parameter.valueType.isNil():
self.error(&"missing type declaration for '{parameter.ident.token.lexeme}' in function declaration")
proc parseFunExpr(self: Parser): LambdaExpr =
## Parses the return value of a function
## when it is another function. Works
## recursively
var arguments: seq[tuple[name: IdentExpr, valueType: Expression]] = @[]
var defaults: seq[Expression] = @[]
result = newLambdaExpr(arguments, defaults, nil, isGenerator=self.peek(-1).kind == Generator,
isAsync=self.peek(-1).kind == Coroutine, token=self.peek(-1),
returnType=nil, depth=self.scopeDepth)
var parameter: tuple[name: IdentExpr, valueType: Expression]
let tok = self.peek(-1)
result = newLambdaExpr(token=tok)
if self.match(LeftParen):
self.parseDeclArguments(arguments, parameter, defaults)
self.parseDeclParams(result.parameters)
if self.match(":"):
if self.match([Function, Coroutine, Generator]):
result.returnType = self.parseFunExpr()
else:
result.returnType = self.expression()
result.arguments = arguments
result.defaults = defaults
result.file = self.file
@ -1024,82 +993,54 @@ proc parseGenericConstraint(self: Parser, endToken: TokenType or string): Expres
proc parseGenerics(self: Parser, decl: Declaration) =
## Parses generics in declarations
var gen: tuple[name: IdentExpr, cond: Expression]
var
ident: IdentExpr
constr: Expression
if self.match("<"):
while not self.check(">") and not self.done():
self.expect(Identifier, "expecting generic type name")
gen.name = newIdentExpr(self.peek(-1), self.scopeDepth)
gen.name.file = self.file
ident = newIdentExpr(self.peek(-1))
ident.file = self.file
if self.match(":"):
gen.cond = self.parseGenericConstraint(">")
constr = self.parseGenericConstraint(">")
else:
gen.cond = nil
decl.genericTypes.add(gen)
constr = nil
decl.genericTypes[ident.token.lexeme] = TypeGeneric(ident: ident, constr: constr)
if not self.match(Comma):
break
self.expect(">", "unterminated generic declaration")
if self.match(LeftBracket):
while not self.check(RightBracket) and not self.done():
self.expect(Identifier, "expecting generic type name")
gen.name = newIdentExpr(self.peek(-1), self.scopeDepth)
gen.name.file = self.file
ident = newIdentExpr(self.peek(-1))
ident.file = self.file
if self.match(":"):
gen.cond = self.parseGenericConstraint(RightBracket)
constr = self.parseGenericConstraint(RightBracket)
else:
gen.cond = nil
decl.genericValues.add(gen)
constr = nil
decl.genericValues[ident.token.lexeme] = TypeGeneric(ident: ident, constr: constr)
if not self.match(Comma):
break
self.expect(RightBracket, "unterminated generic declaration")
proc funDecl(self: Parser, isAsync: bool = false, isGenerator: bool = false,
isLambda: bool = false, isOperator: bool = false): Declaration = # Can't use just FunDecl because it can also return LambdaExpr!
## Parses all types of functions, coroutines, generators and operators
## (with or without a name, where applicable)
proc funDecl(self: Parser, isOperator: bool = false): FunDecl =
## Parses named function declarations
let tok = self.peek(-1)
var enclosingFunction = self.currentFunction
var arguments: seq[tuple[name: IdentExpr, valueType: Expression]] = @[]
var defaults: seq[Expression] = @[]
var returnType: Expression
var pragmas: seq[Pragma] = @[]
if not isLambda and self.match(Identifier):
# We do this extra check because we might
# be called from a context where it's
# ambiguous whether we're parsing a declaration
# or an expression. Fortunately anonymous functions
# are nameless, so we can sort the ambiguity by checking
# if there's an identifier after the keyword
self.currentFunction = newFunDecl(newIdentExpr(self.peek(-1), self.scopeDepth), arguments, defaults, newBlockStmt(@[], Token()),
isAsync=isAsync,
isGenerator=isGenerator,
isPrivate=true,
token=tok,
returnType=nil)
if self.match("*"):
FunDecl(self.currentFunction).isPrivate = false
self.checkDecl(FunDecl(self.currentFunction).isPrivate)
if self.check(["<", "["]):
self.parseGenerics(self.currentFunction)
elif not isLambda and (self.check([LeftBrace, LeftParen]) or self.check(":")):
# We do a bit of hacking to pretend we never
# wanted to parse this as a declaration in
# the first place and pass control over to
# expressionStatement(), which will in turn
# go all the way up to primary(), which will
# call us back with isLambda=true, allowing us
# to actually parse the function as an expression
while not self.check(tok.kind): # We rewind back to the token that caused us to be called
dec(self.current)
result = Declaration(self.expressionStatement())
self.currentFunction = enclosingFunction
return result
elif isLambda:
self.currentFunction = newLambdaExpr(arguments, defaults, newBlockStmt(@[], Token()), isGenerator=isGenerator, isAsync=isAsync, token=tok,
returnType=nil, depth=self.scopeDepth)
var
parameters: Parameters = newTable[string, Parameter]()
returnType: Expression
pragmas: seq[Pragma] = @[]
function = newFunDecl(newIdentExpr(self.peek(-1)), parameters, nil, true, tok, pragmas, returnType)
function.file = self.file
if self.match("*"):
function.isPrivate = true
self.checkDecl(function.isPrivate)
if self.check(["<", "["]):
self.parseGenerics(function)
if self.match(LeftParen):
var parameter: tuple[name: IdentExpr, valueType: Expression]
self.parseDeclArguments(arguments, parameter, defaults)
self.parseDeclParams(parameters)
if self.match(":"):
# Function returns a value
if self.match([Function, Coroutine, Generator]):
@ -1111,44 +1052,30 @@ proc funDecl(self: Parser, isAsync: bool = false, isGenerator: bool = false,
returnType = self.parseFunExpr()
else:
returnType = self.expression()
if self.currentFunction.kind == funDecl:
if not self.match(Semicolon):
# If we don't find a semicolon,
# it's not a forward declaration
self.expect(LeftBrace)
if self.match(TokenType.Pragma):
for pragma in self.parsePragmas():
pragmas.add(pragma)
FunDecl(self.currentFunction).body = self.blockStmt()
else:
# This is a forward declaration, so we explicitly
# nullify the function's body to tell the compiler
# to look for it elsewhere in the file later
FunDecl(self.currentFunction).body = nil
if self.match(TokenType.Pragma):
for pragma in self.parsePragmas():
pragmas.add(pragma)
FunDecl(self.currentFunction).arguments = arguments
FunDecl(self.currentFunction).returnType = returnType
else:
if not self.match(Semicolon):
# If we don't find a semicolon,
# it's not a forward declaration
self.expect(LeftBrace)
if self.match(TokenType.Pragma):
for pragma in self.parsePragmas():
pragmas.add(pragma)
LambdaExpr(Expression(self.currentFunction)).body = self.blockStmt()
LambdaExpr(Expression(self.currentFunction)).arguments = arguments
LambdaExpr(Expression(self.currentFunction)).returnType = returnType
result = self.currentFunction
function.body = self.blockStmt()
else:
# This is a forward declaration, so we keep the
# function body null so the compiler knows to treat
# it as such
if self.match(TokenType.Pragma):
for pragma in self.parsePragmas():
pragmas.add(pragma)
function.returnType = returnType
result = function
if isOperator:
if arguments.len() == 0:
if parameters.len() == 0:
self.error("cannot declare operator without arguments")
elif isLambda:
self.error("cannot declare anonymous operator")
for argument in arguments:
if argument.valueType == nil:
self.error(&"missing type declaration for '{argument.name.token.lexeme}' in function declaration")
self.currentFunction = enclosingFunction
result.pragmas = pragmas
for name in parameters.keys():
let parameter = parameters[name]
if parameter.valueType == nil:
self.error(&"missing type declaration for '{name}' in function declaration")
result.file = self.file
@ -1192,52 +1119,40 @@ proc switchStmt(self: Parser): Statement =
proc statement(self: Parser): Statement =
## Parses statements
case self.peek().kind:
of If:
of TokenType.If:
discard self.step()
result = self.ifStmt()
of Switch:
of TokenType.Switch:
discard self.step()
result = self.switchStmt()
of Assert:
of TokenType.Assert:
discard self.step()
result = self.assertStmt()
of Raise:
discard self.step()
result = self.raiseStmt()
of Break:
of TokenType.Break:
discard self.step()
result = self.breakStmt()
of Continue:
of TokenType.Continue:
discard self.step()
result = self.continueStmt()
of Return:
of TokenType.Return:
discard self.step()
result = self.returnStmt()
of Import:
of TokenType.Import:
discard self.step()
result = self.importStmt()
of Export:
of TokenType.Export:
discard self.step()
result = self.exportStmt()
of From:
discard self.step()
result = self.importStmt(fromStmt=true)
of While:
of TokenType.While:
discard self.step()
result = self.whileStmt()
of Foreach:
of TokenType.Foreach:
discard self.step()
result = self.forEachStmt()
of LeftBrace:
of TokenType.LeftBrace:
discard self.step()
result = self.blockStmt()
of Yield:
discard self.step()
result = self.yieldStmt()
of Await:
discard self.step()
result = self.awaitStmt()
of Block:
of TokenType.Block:
discard self.step()
result = self.namedBlockStmt()
else:
@ -1245,9 +1160,9 @@ proc statement(self: Parser): Statement =
result.file = self.file
proc parseTypeFields(self: Parser): seq[tuple[name: IdentExpr, valueType: Expression, isPrivate: bool, default: Expression]] =
proc parseTypeFields(self: Parser): TypeFields =
## Parses type fields
result = @[]
result = newTable[string, TypeField]()
var
argName: IdentExpr
argPrivate: bool
@ -1255,13 +1170,13 @@ proc parseTypeFields(self: Parser): seq[tuple[name: IdentExpr, valueType: Expres
argDefault: Expression
while not self.check(RightBrace) and not self.done():
self.expect(Identifier, "expecting type member name")
argName = newIdentExpr(self.peek(-1), self.scopeDepth)
argName = newIdentExpr(self.peek(-1))
argPrivate = not self.match("*")
self.expect(":", "expecting ':' after type member name")
argType = self.expression()
if self.match("="):
argDefault = self.expression()
result.add((argName, argType, argPrivate, argDefault))
result[argName.token.lexeme] = TypeField(ident: argName, valueType: argType, default: argDefault, isPrivate: argPrivate)
if not self.check([";", "}"]):
if self.peek().kind == Semicolon:
discard self.step()
@ -1274,8 +1189,8 @@ proc typeDecl(self: Parser): TypeDecl =
## Parses type declarations
let token = self.peek(-1)
self.expect(Identifier, "expecting type name after 'type'")
var name = newIdentExpr(self.peek(-1), self.scopeDepth)
result = newTypeDecl(name, @[], @[], true, token, @[], @[], @[], nil, false, false)
var name = newIdentExpr(self.peek(-1))
result = newTypeDecl(name, newTable[string, TypeField](), true, token, @[], nil, false, false)
result.file = self.file
if self.check(["<", "["]):
self.parseGenerics(result)
@ -1318,7 +1233,7 @@ proc typeDecl(self: Parser): TypeDecl =
else:
var variant: TypeDecl
while not self.done():
variant = newTypeDecl(nil, @[], @[], true, nil, @[], @[], @[], nil, false, false)
variant = newTypeDecl(nil, nil, true, nil, @[], nil, false, false)
self.expect(Identifier, "expecting variant name")
variant.name = newIdentExpr(self.peek(-1))
variant.token = variant.name.token
@ -1338,32 +1253,39 @@ proc typeDecl(self: Parser): TypeDecl =
proc declaration(self: Parser): Declaration =
## Parses declarations
case self.peek().kind:
of TokenType.Var, TokenType.Const, Let:
of TokenType.Var, TokenType.Const, TokenType.Let:
discard self.step()
result = self.varDecl()
of Function:
of TokenType.Function:
discard self.step()
result = self.funDecl()
of Coroutine:
discard self.step()
result = self.funDecl(isAsync=true)
of Generator:
discard self.step()
result = self.funDecl(isGenerator=true)
of Operator:
of TokenType.Operator:
discard self.step()
result = self.funDecl(isOperator=true)
of TokenType.Pragma:
discard self.step()
for p in self.parsePragmas():
self.tree.add(p)
of Type:
of TokenType.Type:
discard self.step()
result = self.typeDecl()
of Comment:
of TokenType.Comment:
discard self.step() # TODO: Docstrings and stuff
else:
result = Declaration(self.statement())
self.error(&"unknown token type {self.peek().kind} at declaration()")
proc dispatch(self: Parser): ASTNode =
case self.peek().kind:
of TokenType.Var, TokenType.Const, TokenType.Let, TokenType.Function,
TokenType.Operator, TokenType.Pragma, TokenType.Type:
return self.declaration()
of TokenType.If, TokenType.Switch, TokenType.Assert, TokenType.While,
TokenType.Foreach, TokenType.Break, TokenType.Continue, TokenType.Return,
TokenType.Import, TokenType.Export, TokenType.LeftBrace, TokenType.Block:
return self.statement()
else:
return self.expression()
proc findOperators(self: Parser, tokens: seq[Token]) =
@ -1391,7 +1313,6 @@ proc parse*(self: Parser, tokens: seq[Token], file: string, lines: seq[tuple[sta
self.current = 0
self.scopeDepth = 0
self.loopDepth = 0
self.currentFunction = nil
self.tree = @[]
if not persist:
self.operators = newOperatorTable()
@ -1399,7 +1320,7 @@ proc parse*(self: Parser, tokens: seq[Token], file: string, lines: seq[tuple[sta
self.findOperators(tokens)
var node: ASTNode
while not self.done():
node = self.declaration()
node = self.dispatch()
if not node.isNil():
# This only happens because we haven't implemented
# all of our grammar yet. Will be removed soon

6
src/frontend/parser/token.nim
View File

@ -34,9 +34,9 @@ type
Function, Break, Continue,
Var, Let, Const, Return,
Coroutine, Generator, Import,
Raise, Assert, Await, Foreach,
Assert, Await, Foreach,
Yield, Type, Operator, Case,
Enum, From, Ptr, Ref, Object,
Enum, Ptr, Ref, Object,
Export, Block, Switch, Lent
# Literal types
@ -80,4 +80,4 @@ proc `$`*(self: Token): string =
proc `==`*(self, other: Token): bool =
## Returns self == other
return self.kind == other.kind and self.lexeme == other.lexeme
return self.kind == other.kind and self.lexeme == other.lexeme

2
src/util/fmterr.nim
View File

@ -65,7 +65,7 @@ proc print*(exc: ParseError) =
contents = exc.parser.getSource().strip(chars={'\n'}).splitLines()[exc.line - 1]
else:
contents = ""
printError(file, contents, exc.line, exc.token.relPos, exc.parser.getCurrentFunction(), exc.msg)
printError(file, contents, exc.line, exc.token.relPos, nil, exc.msg)
proc print*(exc: LexingError) =

2
src/util/symbols.nim
View File

@ -32,7 +32,6 @@ proc fillSymbolTable*(tokenizer: Lexer) =
tokenizer.symbols.addKeyword("if", TokenType.If)
tokenizer.symbols.addKeyword("else", TokenType.Else)
tokenizer.symbols.addKeyword("await", TokenType.Await)
tokenizer.symbols.addKeyword("raise", TokenType.Raise)
tokenizer.symbols.addKeyword("assert", TokenType.Assert)
tokenizer.symbols.addKeyword("const", TokenType.Const)
tokenizer.symbols.addKeyword("let", TokenType.Let)
@ -45,7 +44,6 @@ proc fillSymbolTable*(tokenizer: Lexer) =
tokenizer.symbols.addKeyword("block", TokenType.Block)
tokenizer.symbols.addKeyword("switch", TokenType.Switch)
tokenizer.symbols.addKeyword("lent", TokenType.Lent)
tokenizer.symbols.addKeyword("from", TokenType.From)
# These are more like expressions with a reserved
# name that produce a value of a builtin type,
# but we don't need to care about that until