nocturn9x
/
peon
2
2
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

BROKEN: Initial work on closed-over variables and attempt to cleanup the AST type structure

This commit is contained in:
Nocturn9x 2022-04-27 16:03:48 +02:00
parent 43040df9f9
commit 78d955541d
7 changed files with 360 additions and 286 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

161
src/frontend/compiler.nim
View File

@ -45,6 +45,7 @@ type
isPrivate: bool isPrivate: bool
isConst: bool isConst: bool
valueType: IdentExpr valueType: IdentExpr
codePos: int
Loop = object Loop = object
## A "loop object" used ## A "loop object" used
@ -470,8 +471,8 @@ proc binary(self: Compiler, node: BinaryExpr) =
proc declareName(self: Compiler, node: ASTNode, kind: IdentExpr) = proc declareName(self: Compiler, node: Declaration, kind: IdentExpr) =
## Compiles all name declarations ## Statically declares a name into the current scope
case node.kind: case node.kind:
of NodeKind.varDecl: of NodeKind.varDecl:
var node = VarDecl(node) var node = VarDecl(node)
@ -480,18 +481,27 @@ proc declareName(self: Compiler, node: ASTNode, kind: IdentExpr) =
# If someone ever hits this limit in real-world scenarios, I swear I'll # If someone ever hits this limit in real-world scenarios, I swear I'll
# slap myself 100 times with a sign saying "I'm dumb". Mark my words # slap myself 100 times with a sign saying "I'm dumb". Mark my words
self.error("cannot declare more than 16777216 variables at a time") self.error("cannot declare more than 16777216 variables at a time")
self.names.add(Name(depth: self.scopeDepth, name: IdentExpr(node.name), self.names.add(Name(depth: self.scopeDepth,
name: node.name,
isPrivate: node.isPrivate, isPrivate: node.isPrivate,
owner: self.currentModule, owner: self.currentModule,
isConst: node.isConst, isConst: node.isConst,
valueType: kind)) valueType: kind,
codePos: self.chunk.code.len()))
self.emitByte(StoreVar) self.emitByte(StoreVar)
self.emitBytes(self.names.high().toTriple()) self.emitBytes(self.names.high().toTriple())
of funDecl: of funDecl:
var node = FunDecl(node) var node = FunDecl(node)
# Declares the function's name in the # Declares the function's name in the
# current (outer) scope... # current (outer) scope...
self.declareName(node.name, IdentExpr(node.returnType)) self.names.add(Name(depth: self.scopeDepth,
isPrivate: node.isPrivate,
isConst: false,
owner: self.currentModule,
valueType: node.returnType,
codePos: self.chunk.code.len()))
self.emitByte(StoreVar)
self.emitBytes(self.names.high().toTriple())
# ... but its arguments in an inner one! # ... but its arguments in an inner one!
self.scopeDepth += 1 self.scopeDepth += 1
# (this ugly part is needed because # (this ugly part is needed because
@ -500,35 +510,42 @@ proc declareName(self: Compiler, node: ASTNode, kind: IdentExpr) =
for argument in node.arguments: for argument in node.arguments:
if self.names.high() > 16777215: if self.names.high() > 16777215:
self.error("cannot declare more than 16777216 variables at a time") self.error("cannot declare more than 16777216 variables at a time")
self.names.add(Name(depth: self.scopeDepth + 1, isPrivate: true, owner: self.currentModule, isConst: false, name: IdentExpr(argument.name), valueType: kind)) self.names.add(Name(depth: self.scopeDepth + 1,
self.emitByte(LoadVar) isPrivate: true,
owner: self.currentModule,
isConst: false,
name: argument.name,
valueType: kind,
codePos: self.chunk.code.len()))
self.emitByte(StoreVar)
self.emitBytes(self.names.high().toTriple()) self.emitBytes(self.names.high().toTriple())
self.scopeDepth -= 1 self.scopeDepth -= 1
# TODO: Default arguments and unpacking # TODO: Default arguments and unpacking
else: else:
discard # TODO: Classes discard # Unreachable
proc resolveStatic(self: Compiler, name: IdentExpr, proc resolve(self: Compiler, name: IdentExpr,
depth: int = self.scopeDepth): Name = depth: int = self.scopeDepth): Name =
## Traverses self.staticNames backwards and returns the ## Traverses self.names backwards and returns the
## first name object with the given name. Returns ## first name object with the given name. Returns
## nil when the name can't be found. This function ## nil when the name can't be found. This function
## has no concept of scope depth, because getStaticIndex ## has no concept of scope depth, because getStackPos
## does that job. Note that private names declared in ## does that job. Note that private names declared in
## other modules will not be resolved! ## other modules will not be resolved!
for obj in reversed(self.names): for obj in reversed(self.names):
if obj.name.token.lexeme == name.token.lexeme: if obj.name.token.lexeme == name.token.lexeme:
if obj.isPrivate and obj.owner != self.currentModule: if obj.isPrivate and obj.owner != self.currentModule:
return nil continue # There may be a name in the current module that
# matches, so we skip this
return obj return obj
return nil return nil
proc getStaticIndex(self: Compiler, name: IdentExpr, depth: int = self.scopeDepth): tuple[closedOver: bool, pos: int] = proc getStackPos(self: Compiler, name: IdentExpr, depth: int = self.scopeDepth): tuple[closedOver: bool, pos: int] =
## Gets the predicted stack position of the given variable and ## Iterates the internal list of declared names backwards and
## returns a tuple (closedOver, pos) that tells the caller whether ## returns a tuple (closedOver, pos) that tells the caller whether the
## the variable is to be emitted as a closure as well as its predicted ## the name is to be emitted as a closure as well as its predicted
## stack/closure array position. Returns (false, -1) if the variable's ## stack/closure array position. Returns (false, -1) if the variable's
## location can not be determined at compile time (this is an error!). ## location can not be determined at compile time (this is an error!).
## Note that private names declared in other modules will not be resolved! ## Note that private names declared in other modules will not be resolved!
@ -536,7 +553,7 @@ proc getStaticIndex(self: Compiler, name: IdentExpr, depth: int = self.scopeDept
for variable in reversed(self.names): for variable in reversed(self.names):
if name.name.lexeme == variable.name.name.lexeme: if name.name.lexeme == variable.name.name.lexeme:
if variable.isPrivate and variable.owner != self.currentModule: if variable.isPrivate and variable.owner != self.currentModule:
return (false, -1) continue
if variable.depth == depth or variable.depth == 0: if variable.depth == depth or variable.depth == 0:
# variable.depth == 0 for globals! # variable.depth == 0 for globals!
return (false, i) return (false, i)
@ -548,32 +565,69 @@ proc getStaticIndex(self: Compiler, name: IdentExpr, depth: int = self.scopeDept
return (false, -1) return (false, -1)
proc detectClosureVariable(self: Compiler, name: IdentExpr, depth: int = self.scopeDepth) =
## Detects if the given name is used in a local scope deeper
## than the given one and modifies the code emitted for it
## to store it as a closure variable if it is. Does nothing if the name
## hasn't been declared yet or is unreachable (for example if it's
## declared as private in another module), if the name itself is a
## global variable and if either the current or the outer scope are
## the global (outermost) one. This function must be called each
## time a name is referenced in order for closed-over variables
## to be emitted properly, otherwise the runtime may behave
## unpredictably or crash
if depth == 0 or depth - 1 == 0:
return
var entry = self.resolve(name)
if entry == nil:
return
if entry.depth == 0:
return
if entry.depth < depth:
# Ding! The given name is closed over: we need to
# change the StoreVar instruction that created this
# name entry into a StoreHeap. We don't need to change
# other pieces of code because self.identifier() already
# emits LoadHeap if it detects the variable is closed over,
# whether or not this function is called
self.closedOver.add(entry.name)
if self.closedOver.len() >= 16777216:
self.error("too many consecutive closure-over variables (max is 16777216)")
let idx = self.closedOver.high().toTriple()
self.chunk.code[entry.codePos] = StoreHeap.uint8
self.chunk.code[entry.codePos + 1] = idx[0]
self.chunk.code[entry.codePos + 2] = idx[1]
self.chunk.code[entry.codePos + 3] = idx[2]
proc identifier(self: Compiler, node: IdentExpr) = proc identifier(self: Compiler, node: IdentExpr) =
## Compiles access to identifiers ## Compiles access to identifiers
let s = self.resolveStatic(node) let s = self.resolve(node)
if s != nil: if s == nil:
if s.isConst:
# Constants are emitted as, you guessed it, constant instructions
# no matter the scope depth. Also, name resolution specifiers do not
# apply to them (because what would it mean for a constant to be dynamic
# anyway?)
self.emitConstant(node)
else:
let t = self.getStaticIndex(node)
let index = t.pos
if index != -1:
if not t.closedOver:
self.emitByte(LoadVar) # Static name resolution, loads value at index in the stack. Very fast. Much wow.
self.emitBytes(index.toTriple())
else:
if self.closedOver.len() == 0:
self.error("error: closure variable array is empty but LoadHeap would be emitted (this is an internal error and most likely a bug)")
if self.closedOver.len() >= 16777216:
self.error("too many consecutive closure-over variables (max is 16777216)")
self.emitByte(LoadHeap) # Heap-allocated closure variable. Stored in a separate "closure array" in the VM that does not have stack semantics
self.emitBytes(self.closedOver.high().toTriple())
else:
self.error(&"reference to undeclared name '{node.token.lexeme}'") self.error(&"reference to undeclared name '{node.token.lexeme}'")
elif s.isConst:
# Constants are emitted as, you guessed it, LoadConstant instructions
# no matter the scope depth. If optimizations are enabled, the compiler
# will reuse the same constant every time it is referenced instead
self.emitConstant(node)
else:
self.detectClosureVariable(s.name)
let t = self.getStackPos(node)
let index = t.pos
# We don't check if index is -1 because if it
# were, self.resolve() would have returned nil
if not t.closedOver:
# Static name resolution, loads value at index in the stack. Very fast. Much wow.
self.emitByte(LoadVar)
self.emitBytes(index.toTriple())
else:
if self.closedOver.len() == 0:
self.error("error: closure variable array is empty but LoadHeap would be emitted (this is an internal error and most likely a bug)")
# Heap-allocated closure variable. Stored in a separate "closure array" in the VM that does not have stack semantics.
# This makes closures work as expected and is not comparatively slower than indexing our stack (since they're both
# dynamic arrays at runtime anyway)
self.emitByte(LoadHeap)
self.emitBytes(self.closedOver.high().toTriple())
proc assignment(self: Compiler, node: ASTNode) = proc assignment(self: Compiler, node: ASTNode) =
@ -581,12 +635,13 @@ proc assignment(self: Compiler, node: ASTNode) =
case node.kind: case node.kind:
of assignExpr: of assignExpr:
var node = AssignExpr(node) var node = AssignExpr(node)
# TODO: This will explode with slicing!
var name = IdentExpr(node.name) var name = IdentExpr(node.name)
let r = self.resolveStatic(name) let r = self.resolve(name)
if r != nil and r.isConst: if r != nil and r.isConst:
self.error("cannot assign to constant") self.error("cannot assign to constant")
self.expression(node.value) self.expression(node.value)
let t = self.getStaticIndex(name) let t = self.getStackPos(name)
let index = t.pos let index = t.pos
case node.token.kind: case node.token.kind:
of InplaceAdd: of InplaceAdd:
@ -615,13 +670,17 @@ proc assignment(self: Compiler, node: ASTNode) =
discard # Unreachable discard # Unreachable
# In-place operators just change # In-place operators just change
# what values is set to a given # what values is set to a given
# stack offset/name, so we only # offset in a dynamic array, so we only
# need to perform the operation # need to perform the operation as usual
# as usual and then store it. # and then store it. We could combine
# TODO: A better optimization would # everything into a single opcode, but
# be to have everything in one opcode, # that would require variants of each
# but that requires variants for stack, # one for regular stack variables as
# heap, and closure variables and I cba # well as closed-over ones
# Cannot be nil, we already resolved this!
if self.resolve(name).isConst:
self.error("cannot mutate constant")
if index != -1: if index != -1:
if not t.closedOver: if not t.closedOver:
self.emitByte(StoreVar) self.emitByte(StoreVar)
@ -782,7 +841,7 @@ proc inferExprType(self: Compiler, node: ASTNode): ASTNode =
case node.kind: case node.kind:
of identExpr: of identExpr:
var node = IdentExpr(node) var node = IdentExpr(node)
var name = self.resolveStatic(node) var name = self.resolve(node)
if name == nil: if name == nil:
return nil return nil
return name.valueType return name.valueType
@ -1019,7 +1078,7 @@ proc funDecl(self: Compiler, node: FunDecl) =
# A function's code is just compiled linearly # A function's code is just compiled linearly
# and then jumped over # and then jumped over
let jmp = self.emitJump(JumpForwards) let jmp = self.emitJump(JumpForwards)
self.declareName(node, IdentExpr(node.returnType)) self.declareName(node, node.returnType)
# Since the deferred array is a linear # Since the deferred array is a linear
# sequence of instructions and we want # sequence of instructions and we want

13
src/frontend/lexer.nim
View File

@ -368,6 +368,7 @@ proc parseString(self: Lexer, delimiter: string, mode: string = "single") =
## lines and escape sequences in them are not parsed, like in raw ## lines and escape sequences in them are not parsed, like in raw
## strings, so a multi-line string prefixed with the "r" modifier ## strings, so a multi-line string prefixed with the "r" modifier
## is redundant, although multi-line byte/format strings are supported ## is redundant, although multi-line byte/format strings are supported
var slen = 0
while not self.check(delimiter) and not self.done(): while not self.check(delimiter) and not self.done():
if self.match("\n"): if self.match("\n"):
if mode == "multi": if mode == "multi":
@ -399,6 +400,9 @@ proc parseString(self: Lexer, delimiter: string, mode: string = "single") =
self.source = self.source[0..<self.current] & self.source[ self.source = self.source[0..<self.current] & self.source[
self.current + 1..^1] self.current + 1..^1]
discard self.step() discard self.step()
inc(slen)
if slen > 1 and delimiter == "'":
self.error("invalid character literal (length must be one!)")
if mode == "multi": if mode == "multi":
if not self.match(delimiter.repeat(3)): if not self.match(delimiter.repeat(3)):
self.error("unexpected EOL while parsing multi-line string literal") self.error("unexpected EOL while parsing multi-line string literal")
@ -406,7 +410,10 @@ proc parseString(self: Lexer, delimiter: string, mode: string = "single") =
self.error("unexpected EOF while parsing string literal") self.error("unexpected EOF while parsing string literal")
else: else:
discard self.step() discard self.step()
self.createToken(String) if delimiter == "\"":
self.createToken(String)
else:
self.createToken(Char)
proc parseBinary(self: Lexer) = proc parseBinary(self: Lexer) =
@ -528,9 +535,9 @@ proc next(self: Lexer) =
# New line # New line
self.incLine() self.incLine()
elif self.match(["\"", "'"]): elif self.match(["\"", "'"]):
# String literal # String or character literal
var mode = "single" var mode = "single"
if self.check(self.peek(-1)) and self.check(self.peek(-1), 1): if self.peek(-1) != "'" and self.check(self.peek(-1)) and self.check(self.peek(-1), 1):
# Multiline strings start with 3 quotes # Multiline strings start with 3 quotes
discard self.step(2) discard self.step(2)
mode = "multi" mode = "multi"

263
src/frontend/meta/ast.nim
View File

@ -71,6 +71,7 @@ type
setExpr, setExpr,
falseExpr, falseExpr,
strExpr, strExpr,
charExpr,
intExpr, intExpr,
floatExpr, floatExpr,
hexExpr, hexExpr,
@ -81,7 +82,14 @@ type
infExpr, infExpr,
identExpr, # Identifier identExpr, # Identifier
# Here I would've rather used object variants, and in fact that's what was in
# place before, but not being able to re-declare a field of the same type in
# another case branch is kind of a deal breaker long-term, so until that is
# fixed (check out https://github.com/nim-lang/RFCs/issues/368 for more info).
# I'll stick to using inheritance instead
# Generic AST node types
ASTNode* = ref object of RootObj ASTNode* = ref object of RootObj
## An AST node ## An AST node
kind*: NodeKind kind*: NodeKind
@ -89,14 +97,19 @@ type
# This is not shown when the node is printed, but makes it a heck of a lot easier to report # This is not shown when the node is printed, but makes it a heck of a lot easier to report
# errors accurately even deep in the compilation pipeline # errors accurately even deep in the compilation pipeline
token*: Token token*: Token
# This weird inheritance chain is just a map of
# our grammar's precedence rules (expressions first,
# then statements and declarations last)
Expression* = ref object of ASTNode
## An expression
Statement* = ref object of Expression
## A statement
Declaration* = ref object of Statement
## A declaration
closedOver*: bool
pragmas*: seq[Token]
# Here I would've rather used object variants, and in fact that's what was in LiteralExpr* = ref object of Expression
# place before, but not being able to re-declare a field of the same type in
# another case branch is kind of a deal breaker long-term, so until that is
# fixed (check out https://github.com/nim-lang/RFCs/issues/368 for more info).
# I'll stick to using inheritance instead
LiteralExpr* = ref object of ASTNode
# Using a string for literals makes it much easier to handle numeric types, as # Using a string for literals makes it much easier to handle numeric types, as
# there is no overflow nor underflow or float precision issues during parsing. # there is no overflow nor underflow or float precision issues during parsing.
# Numbers are just serialized as strings and then converted back to numbers # Numbers are just serialized as strings and then converted back to numbers
@ -114,6 +127,7 @@ type
BinExpr* = ref object of LiteralExpr BinExpr* = ref object of LiteralExpr
FloatExpr* = ref object of LiteralExpr FloatExpr* = ref object of LiteralExpr
StrExpr* = ref object of LiteralExpr StrExpr* = ref object of LiteralExpr
CharExpr* = ref object of LiteralExpr
TrueExpr* = ref object of LiteralExpr TrueExpr* = ref object of LiteralExpr
FalseExpr* = ref object of LiteralExpr FalseExpr* = ref object of LiteralExpr
@ -122,156 +136,143 @@ type
InfExpr* = ref object of LiteralExpr InfExpr* = ref object of LiteralExpr
ListExpr* = ref object of LiteralExpr ListExpr* = ref object of LiteralExpr
members*: seq[ASTNode] members*: seq[Expression]
valueType*: IdentExpr valueType*: IdentExpr
SetExpr* = ref object of ListExpr SetExpr* = ref object of ListExpr
TupleExpr* = ref object of ListExpr TupleExpr* = ref object of ListExpr
DictExpr* = ref object of ASTNode DictExpr* = ref object of Expression
keys*: seq[ASTNode] keys*: seq[Expression]
values*: seq[ASTNode] values*: seq[Expression]
keyType*: IdentExpr keyType*: IdentExpr
valueType*: IdentExpr valueType*: IdentExpr
IdentExpr* = ref object of ASTNode IdentExpr* = ref object of Expression
name*: Token name*: Token
GroupingExpr* = ref object of ASTNode GroupingExpr* = ref object of Expression
expression*: ASTNode expression*: Expression
GetItemExpr* = ref object of ASTNode GetItemExpr* = ref object of Expression
obj*: ASTNode obj*: Expression
name*: ASTNode name*: IdentExpr
SetItemExpr* = ref object of GetItemExpr SetItemExpr* = ref object of GetItemExpr
# Since a setItem expression is just # Since a setItem expression is just
# a getItem one followed by an assignment, # a getItem one followed by an assignment,
# inheriting it from getItem makes sense # inheriting it from getItem makes sense
value*: ASTNode value*: Expression
CallExpr* = ref object of ASTNode CallExpr* = ref object of Expression
callee*: ASTNode # The thing being called callee*: ASTNode # The thing being called
arguments*: tuple[positionals: seq[ASTNode], keyword: seq[tuple[ arguments*: tuple[positionals: seq[Expression], keyword: seq[tuple[
name: ASTNode, value: ASTNode]]] name: IdentExpr, value: Expression]]]
UnaryExpr* = ref object of ASTNode UnaryExpr* = ref object of Expression
operator*: Token operator*: Token
a*: ASTNode a*: Expression
BinaryExpr* = ref object of UnaryExpr BinaryExpr* = ref object of UnaryExpr
# Binary expressions can be seen here as unary # Binary expressions can be seen here as unary
# expressions with an extra operand so we just # expressions with an extra operand so we just
# inherit from that and add a second operand # inherit from that and add a second operand
b*: ASTNode b*: Expression
YieldExpr* = ref object of ASTNode YieldExpr* = ref object of Expression
expression*: ASTNode expression*: Expression
AwaitExpr* = ref object of ASTNode AwaitExpr* = ref object of Expression
expression*: ASTNode expression*: Expression
LambdaExpr* = ref object of ASTNode LambdaExpr* = ref object of Expression
body*: ASTNode body*: Statement
arguments*: seq[tuple[name: ASTNode, valueType: ASTNode]] arguments*: seq[tuple[name: IdentExpr, valueType: IdentExpr]]
defaults*: seq[ASTNode] defaults*: seq[Expression]
isGenerator*: bool isGenerator*: bool
isAsync*: bool isAsync*: bool
returnType*: ASTNode returnType*: IdentExpr
SliceExpr* = ref object of ASTNode SliceExpr* = ref object of Expression
expression*: ASTNode expression*: Expression
ends*: seq[ASTNode] ends*: seq[ASTNode]
AssignExpr* = ref object of ASTNode AssignExpr* = ref object of Expression
name*: ASTNode name*: Expression
value*: ASTNode value*: Expression
ExprStmt* = ref object of ASTNode ExprStmt* = ref object of Statement
expression*: ASTNode expression*: Expression
ImportStmt* = ref object of ASTNode ImportStmt* = ref object of Statement
moduleName*: ASTNode moduleName*: IdentExpr
AssertStmt* = ref object of ASTNode AssertStmt* = ref object of Statement
expression*: ASTNode expression*: Expression
RaiseStmt* = ref object of ASTNode RaiseStmt* = ref object of Statement
exception*: ASTNode exception*: Expression
BlockStmt* = ref object of ASTNode BlockStmt* = ref object of Statement
code*: seq[ASTNode] code*: seq[Declaration]
ForStmt* = ref object of ASTNode ForStmt* = ref object of Statement
discard # Unused discard # Unused
ForEachStmt* = ref object of ASTNode ForEachStmt* = ref object of Statement
identifier*: ASTNode identifier*: ASTNode
expression*: ASTNode expression*: ASTNode
body*: ASTNode body*: ASTNode
DeferStmt* = ref object of ASTNode DeferStmt* = ref object of Statement
expression*: ASTNode expression*: ASTNode
TryStmt* = ref object of ASTNode TryStmt* = ref object of Statement
body*: ASTNode body*: Statement
handlers*: seq[tuple[body: ASTNode, exc: ASTNode, name: ASTNode]] handlers*: seq[tuple[body: Statement, exc: IdentExpr, name: IdentExpr]]
finallyClause*: ASTNode finallyClause*: Statement
elseClause*: ASTNode elseClause*: Statement
WhileStmt* = ref object of ASTNode WhileStmt* = ref object of Statement
condition*: ASTNode condition*: Expression
body*: ASTNode body*: Statement
AwaitStmt* = ref object of ASTNode AwaitStmt* = ref object of Statement
expression*: ASTNode expression*: Expression
BreakStmt* = ref object of ASTNode BreakStmt* = ref object of Statement
ContinueStmt* = ref object of ASTNode ContinueStmt* = ref object of Statement
ReturnStmt* = ref object of ASTNode ReturnStmt* = ref object of Statement
value*: ASTNode value*: Expression
IfStmt* = ref object of ASTNode IfStmt* = ref object of Statement
condition*: ASTNode condition*: Expression
thenBranch*: ASTNode thenBranch*: Statement
elseBranch*: ASTNode elseBranch*: Statement
YieldStmt* = ref object of ASTNode YieldStmt* = ref object of Statement
expression*: ASTNode expression*: Expression
Declaration* = ref object of ASTNode
closedOver*: bool
pragmas*: seq[Token]
VarDecl* = ref object of Declaration VarDecl* = ref object of Declaration
name*: ASTNode name*: IdentExpr
value*: ASTNode value*: Expression
isConst*: bool isConst*: bool
isPrivate*: bool isPrivate*: bool
isLet*: bool isLet*: bool
valueType*: ASTNode valueType*: IdentExpr
FunDecl* = ref object of Declaration FunDecl* = ref object of Declaration
name*: ASTNode name*: IdentExpr
body*: ASTNode body*: Statement
arguments*: seq[tuple[name: ASTNode, valueType: ASTNode]] arguments*: seq[tuple[name: IdentExpr, valueType: IdentExpr]]
defaults*: seq[ASTNode] defaults*: seq[Expression]
isAsync*: bool isAsync*: bool
isGenerator*: bool isGenerator*: bool
isPrivate*: bool isPrivate*: bool
returnType*: ASTNode returnType*: IdentExpr
Expression* = LiteralExpr | ListExpr | GetItemExpr | SetItemExpr | UnaryExpr | BinaryExpr | CallExpr | AssignExpr |
GroupingExpr | IdentExpr | DictExpr | TupleExpr | SetExpr | TrueExpr | FalseExpr | NilExpr |
NanExpr | InfExpr
Statement* = ExprStmt | ImportStmt | AssertStmt | RaiseStmt | BlockStmt | ForStmt | WhileStmt |
ForStmt | BreakStmt | ContinueStmt | ReturnStmt | IfStmt
proc newASTNode*(kind: NodeKind, token: Token): ASTNode = proc newASTNode*(kind: NodeKind, token: Token): ASTNode =
@ -356,19 +357,25 @@ proc newStrExpr*(literal: Token): StrExpr =
result.token = literal result.token = literal
proc newCharExpr*(literal: Token): CharExpr =
result = CharExpr(kind: charExpr)
result.literal = literal
result.token = literal
proc newIdentExpr*(name: Token): IdentExpr = proc newIdentExpr*(name: Token): IdentExpr =
result = IdentExpr(kind: identExpr) result = IdentExpr(kind: identExpr)
result.name = name result.name = name
result.token = name result.token = name
proc newGroupingExpr*(expression: ASTNode, token: Token): GroupingExpr = proc newGroupingExpr*(expression: Expression, token: Token): GroupingExpr =
result = GroupingExpr(kind: groupingExpr) result = GroupingExpr(kind: groupingExpr)
result.expression = expression result.expression = expression
result.token = token result.token = token
proc newLambdaExpr*(arguments: seq[tuple[name: ASTNode, valueType: ASTNode]], defaults: seq[ASTNode], body: ASTNode, proc newLambdaExpr*(arguments: seq[tuple[name: IdentExpr, valueType: IdentExpr]], defaults: seq[Expression], body: Declaration,
isGenerator: bool, isAsync: bool, token: Token): LambdaExpr = isGenerator: bool, isAsync: bool, token: Token): LambdaExpr =
result = LambdaExpr(kind: lambdaExpr) result = LambdaExpr(kind: lambdaExpr)
result.body = body result.body = body
@ -379,42 +386,42 @@ proc newLambdaExpr*(arguments: seq[tuple[name: ASTNode, valueType: ASTNode]], de
result.token = token result.token = token
proc newGetItemExpr*(obj: ASTNode, name: ASTNode, token: Token): GetItemExpr = proc newGetItemExpr*(obj: Expression, name: IdentExpr, token: Token): GetItemExpr =
result = GetItemExpr(kind: getItemExpr) result = GetItemExpr(kind: getItemExpr)
result.obj = obj result.obj = obj
result.name = name result.name = name
result.token = token result.token = token
proc newListExpr*(members: seq[ASTNode], token: Token): ListExpr = proc newListExpr*(members: seq[Expression], token: Token): ListExpr =
result = ListExpr(kind: listExpr) result = ListExpr(kind: listExpr)
result.members = members result.members = members
result.token = token result.token = token
result.literal = result.token result.literal = result.token
proc newSetExpr*(members: seq[ASTNode], token: Token): SetExpr = proc newSetExpr*(members: seq[Expression], token: Token): SetExpr =
result = SetExpr(kind: setExpr) result = SetExpr(kind: setExpr)
result.members = members result.members = members
result.token = token result.token = token
result.literal = result.token result.literal = result.token
proc newTupleExpr*(members: seq[ASTNode], token: Token): TupleExpr = proc newTupleExpr*(members: seq[Expression], token: Token): TupleExpr =
result = TupleExpr(kind: tupleExpr) result = TupleExpr(kind: tupleExpr)
result.members = members result.members = members
result.token = token result.token = token
result.literal = result.token result.literal = result.token
proc newDictExpr*(keys, values: seq[ASTNode], token: Token): DictExpr = proc newDictExpr*(keys, values: seq[Expression], token: Token): DictExpr =
result = DictExpr(kind: dictExpr) result = DictExpr(kind: dictExpr)
result.keys = keys result.keys = keys
result.values = values result.values = values
result.token = token result.token = token
proc newSetItemExpr*(obj, name, value: ASTNode, token: Token): SetItemExpr = proc newSetItemExpr*(obj: Expression, name: IdentExpr, value: Expression, token: Token): SetItemExpr =
result = SetItemExpr(kind: setItemExpr) result = SetItemExpr(kind: setItemExpr)
result.obj = obj result.obj = obj
result.name = name result.name = name
@ -422,8 +429,8 @@ proc newSetItemExpr*(obj, name, value: ASTNode, token: Token): SetItemExpr =
result.token = token result.token = token
proc newCallExpr*(callee: ASTNode, arguments: tuple[positionals: seq[ASTNode], proc newCallExpr*(callee: ASTNode, arguments: tuple[positionals: seq[Expression],
keyword: seq[tuple[name: ASTNode, value: ASTNode]]], keyword: seq[tuple[name: IdentExpr, value: Expression]]],
token: Token): CallExpr = token: Token): CallExpr =
result = CallExpr(kind: callExpr) result = CallExpr(kind: callExpr)
result.callee = callee result.callee = callee
@ -431,7 +438,7 @@ proc newCallExpr*(callee: ASTNode, arguments: tuple[positionals: seq[ASTNode],
result.token = token result.token = token
proc newSliceExpr*(expression: ASTNode, ends: seq[ASTNode], proc newSliceExpr*(expression: Expression, ends: seq[ASTNode],
token: Token): SliceExpr = token: Token): SliceExpr =
result = SliceExpr(kind: sliceExpr) result = SliceExpr(kind: sliceExpr)
result.expression = expression result.expression = expression
@ -439,14 +446,14 @@ proc newSliceExpr*(expression: ASTNode, ends: seq[ASTNode],
result.token = token result.token = token
proc newUnaryExpr*(operator: Token, a: ASTNode): UnaryExpr = proc newUnaryExpr*(operator: Token, a: Expression): UnaryExpr =
result = UnaryExpr(kind: unaryExpr) result = UnaryExpr(kind: unaryExpr)
result.operator = operator result.operator = operator
result.a = a result.a = a
result.token = result.operator result.token = result.operator
proc newBinaryExpr*(a: ASTNode, operator: Token, b: ASTNode): BinaryExpr = proc newBinaryExpr*(a: Expression, operator: Token, b: Expression): BinaryExpr =
result = BinaryExpr(kind: binaryExpr) result = BinaryExpr(kind: binaryExpr)
result.operator = operator result.operator = operator
result.a = a result.a = a
@ -454,70 +461,70 @@ proc newBinaryExpr*(a: ASTNode, operator: Token, b: ASTNode): BinaryExpr =
result.token = operator result.token = operator
proc newYieldExpr*(expression: ASTNode, token: Token): YieldExpr = proc newYieldExpr*(expression: Expression, token: Token): YieldExpr =
result = YieldExpr(kind: yieldExpr) result = YieldExpr(kind: yieldExpr)
result.expression = expression result.expression = expression
result.token = token result.token = token
proc newAssignExpr*(name, value: ASTNode, token: Token): AssignExpr = proc newAssignExpr*(name: Expression, value: Expression, token: Token): AssignExpr =
result = AssignExpr(kind: assignExpr) result = AssignExpr(kind: assignExpr)
result.name = name result.name = name
result.value = value result.value = value
result.token = token result.token = token
proc newAwaitExpr*(expression: ASTNode, token: Token): AwaitExpr = proc newAwaitExpr*(expression: Expression, token: Token): AwaitExpr =
result = AwaitExpr(kind: awaitExpr) result = AwaitExpr(kind: awaitExpr)
result.expression = expression result.expression = expression
result.token = token result.token = token
proc newExprStmt*(expression: ASTNode, token: Token): ExprStmt = proc newExprStmt*(expression: Expression, token: Token): ExprStmt =
result = ExprStmt(kind: exprStmt) result = ExprStmt(kind: exprStmt)
result.expression = expression result.expression = expression
result.token = token result.token = token
proc newImportStmt*(moduleName: ASTNode, token: Token): ImportStmt = proc newImportStmt*(moduleName: IdentExpr, token: Token): ImportStmt =
result = ImportStmt(kind: importStmt) result = ImportStmt(kind: importStmt)
result.moduleName = moduleName result.moduleName = moduleName
result.token = token result.token = token
proc newYieldStmt*(expression: ASTNode, token: Token): YieldStmt = proc newYieldStmt*(expression: Expression, token: Token): YieldStmt =
result = YieldStmt(kind: yieldStmt) result = YieldStmt(kind: yieldStmt)
result.expression = expression result.expression = expression
result.token = token result.token = token
proc newAwaitStmt*(expression: ASTNode, token: Token): AwaitExpr = proc newAwaitStmt*(expression: Expression, token: Token): AwaitStmt =
result = AwaitExpr(kind: awaitExpr) result = AwaitStmt(kind: awaitStmt)
result.expression = expression result.expression = expression
result.token = token result.token = token
proc newAssertStmt*(expression: ASTNode, token: Token): AssertStmt = proc newAssertStmt*(expression: Expression, token: Token): AssertStmt =
result = AssertStmt(kind: assertStmt) result = AssertStmt(kind: assertStmt)
result.expression = expression result.expression = expression
result.token = token result.token = token
proc newDeferStmt*(expression: ASTNode, token: Token): DeferStmt = proc newDeferStmt*(expression: Expression, token: Token): DeferStmt =
result = DeferStmt(kind: deferStmt) result = DeferStmt(kind: deferStmt)
result.expression = expression result.expression = expression
result.token = token result.token = token
proc newRaiseStmt*(exception: ASTNode, token: Token): RaiseStmt = proc newRaiseStmt*(exception: Expression, token: Token): RaiseStmt =
result = RaiseStmt(kind: raiseStmt) result = RaiseStmt(kind: raiseStmt)
result.exception = exception result.exception = exception
result.token = token result.token = token
proc newTryStmt*(body: ASTNode, handlers: seq[tuple[body: ASTNode, exc: ASTNode, name: ASTNode]], proc newTryStmt*(body: Statement, handlers: seq[tuple[body: Statement, exc: IdentExpr, name: IdentExpr]],
finallyClause: ASTNode, finallyClause: Statement,
elseClause: ASTNode, token: Token): TryStmt = elseClause: Statement, token: Token): TryStmt =
result = TryStmt(kind: tryStmt) result = TryStmt(kind: tryStmt)
result.body = body result.body = body
result.handlers = handlers result.handlers = handlers
@ -526,20 +533,20 @@ proc newTryStmt*(body: ASTNode, handlers: seq[tuple[body: ASTNode, exc: ASTNode,
result.token = token result.token = token
proc newBlockStmt*(code: seq[ASTNode], token: Token): BlockStmt = proc newBlockStmt*(code: seq[Declaration], token: Token): BlockStmt =
result = BlockStmt(kind: blockStmt) result = BlockStmt(kind: blockStmt)
result.code = code result.code = code
result.token = token result.token = token
proc newWhileStmt*(condition: ASTNode, body: ASTNode, token: Token): WhileStmt = proc newWhileStmt*(condition: Expression, body: Statement, token: Token): WhileStmt =
result = WhileStmt(kind: whileStmt) result = WhileStmt(kind: whileStmt)
result.condition = condition result.condition = condition
result.body = body result.body = body
result.token = token result.token = token
proc newForEachStmt*(identifier: ASTNode, expression, body: ASTNode, proc newForEachStmt*(identifier: IdentExpr, expression: Expression, body: Statement,
token: Token): ForEachStmt = token: Token): ForEachStmt =
result = ForEachStmt(kind: forEachStmt) result = ForEachStmt(kind: forEachStmt)
result.identifier = identifier result.identifier = identifier
@ -558,13 +565,13 @@ proc newContinueStmt*(token: Token): ContinueStmt =
result.token = token result.token = token
proc newReturnStmt*(value: ASTNode, token: Token): ReturnStmt = proc newReturnStmt*(value: Expression, token: Token): ReturnStmt =
result = ReturnStmt(kind: returnStmt) result = ReturnStmt(kind: returnStmt)
result.value = value result.value = value
result.token = token result.token = token
proc newIfStmt*(condition: ASTNode, thenBranch, elseBranch: ASTNode, proc newIfStmt*(condition: Expression, thenBranch, elseBranch: Statement,
token: Token): IfStmt = token: Token): IfStmt =
result = IfStmt(kind: ifStmt) result = IfStmt(kind: ifStmt)
result.condition = condition result.condition = condition
@ -573,9 +580,9 @@ proc newIfStmt*(condition: ASTNode, thenBranch, elseBranch: ASTNode,
result.token = token result.token = token
proc newVarDecl*(name: ASTNode, value: ASTNode, isConst: bool = false, proc newVarDecl*(name: IdentExpr, value: Expression, isConst: bool = false,
isPrivate: bool = true, token: Token, closedOver: bool, isPrivate: bool = true, token: Token, closedOver: bool,
isLet: bool = false, valueType: ASTNode): VarDecl = isLet: bool = false, valueType: IdentExpr): VarDecl =
result = VarDecl(kind: varDecl) result = VarDecl(kind: varDecl)
result.name = name result.name = name
result.value = value result.value = value
@ -586,8 +593,8 @@ proc newVarDecl*(name: ASTNode, value: ASTNode, isConst: bool = false,
result.valueType = valueType result.valueType = valueType
proc newFunDecl*(name: ASTNode, arguments: seq[tuple[name: ASTNode, valueType: ASTNode]], defaults: seq[ASTNode], proc newFunDecl*(name: IdentExpr, arguments: seq[tuple[name: IdentExpr, valueType: IdentExpr]], defaults: seq[Expression],
body: ASTNode, isAsync, isGenerator: bool, body: Statement, isAsync, isGenerator: bool,
isPrivate: bool, token: Token, closedOver: bool): FunDecl = isPrivate: bool, token: Token, closedOver: bool): FunDecl =
result = FunDecl(kind: funDecl) result = FunDecl(kind: funDecl)
result.name = name result.name = name

6
src/frontend/meta/bytecode.nim
View File

@ -57,9 +57,7 @@ type
# used for more complex opcodes. All # used for more complex opcodes. All
# arguments to opcodes (if they take # arguments to opcodes (if they take
# arguments) come from popping off the # arguments) come from popping off the
# stack. Unsupported operations will # stack
# raise TypeError or ValueError exceptions
# and never fail silently
LoadConstant = 0u8, # Pushes constant at position x in the constant table onto the stack LoadConstant = 0u8, # Pushes constant at position x in the constant table onto the stack
## Binary operators ## Binary operators
UnaryNegate, # Pushes the result of -x onto the stack UnaryNegate, # Pushes the result of -x onto the stack
@ -169,7 +167,7 @@ const simpleInstructions* = {OpCode.Return, BinaryAdd, BinaryMultiply,
OpCode.LessOrEqual, BinaryOr, BinaryAnd, OpCode.LessOrEqual, BinaryOr, BinaryAnd,
UnaryNot, BinaryFloorDiv, BinaryOf, OpCode.Raise, UnaryNot, BinaryFloorDiv, BinaryOf, OpCode.Raise,
ReRaise, BeginTry, FinishTry, OpCode.Yield, OpCode.Await, ReRaise, BeginTry, FinishTry, OpCode.Yield, OpCode.Await,
MakeClass, ImplicitReturn} MakeClass, ImplicitReturn, UnaryPlus}
# Constant instructions are instructions that operate on the bytecode constant table # Constant instructions are instructions that operate on the bytecode constant table
const constantInstructions* = {LoadConstant, } const constantInstructions* = {LoadConstant, }

2
src/frontend/meta/token.nim
View File

@ -42,7 +42,7 @@ type
# Literal types # Literal types
Integer, Float, String, Identifier, Integer, Float, String, Identifier,
Binary, Octal, Hex Binary, Octal, Hex, Char
# Brackets, parentheses, # Brackets, parentheses,
# operators and others # operators and others

196
src/frontend/parser.nim
View File

@ -60,7 +60,7 @@ type
# is at the top-level. It allows the # is at the top-level. It allows the
# parser to detect errors like return # parser to detect errors like return
# outside functions # outside functions
currentFunction: ASTNode currentFunction: Declaration
# Stores the current scope depth (0 = global, > 0 local) # Stores the current scope depth (0 = global, > 0 local)
scopeDepth: int scopeDepth: int
# We store user-defined operators for later use # We store user-defined operators for later use
@ -194,15 +194,15 @@ proc expect(self: Parser, kinds: openarray[TokenType], message: string = "") =
# Forward declarations # Forward declarations
proc expression(self: Parser): ASTNode proc expression(self: Parser): Expression
proc expressionStatement(self: Parser): ASTNode proc expressionStatement(self: Parser): Statement
proc statement(self: Parser): ASTNode proc statement(self: Parser): Statement
proc varDecl(self: Parser, isLet: bool = false, isConst: bool = false): ASTNode proc varDecl(self: Parser, isLet: bool = false, isConst: bool = false): Declaration
proc funDecl(self: Parser, isAsync: bool = false, isGenerator: bool = false, isLambda: bool = false, isOperator: bool = false): ASTNode proc funDecl(self: Parser, isAsync: bool = false, isGenerator: bool = false, isLambda: bool = false, isOperator: bool = false): Declaration
proc declaration(self: Parser): ASTNode proc declaration(self: Parser): Declaration
proc primary(self: Parser): ASTNode = proc primary(self: Parser): Expression =
## Parses primary expressions such ## Parses primary expressions such
## as integer literals and keywords ## as integer literals and keywords
## that map to builtin types (true, ## that map to builtin types (true,
@ -331,13 +331,13 @@ proc primary(self: Parser): ASTNode =
self.error("invalid syntax") self.error("invalid syntax")
proc makeCall(self: Parser, callee: ASTNode): ASTNode = proc makeCall(self: Parser, callee: Expression): Expression =
## Utility function called iteratively by self.call() ## Utility function called iteratively by self.call()
## to parse a function call ## to parse a function call
let tok = self.peek(-1) let tok = self.peek(-1)
var argNames: seq[ASTNode] = @[] var argNames: seq[IdentExpr] = @[]
var arguments: tuple[positionals: seq[ASTNode], keyword: seq[tuple[name: ASTNode, value: ASTNode]]] = (positionals: @[], keyword: @[]) var arguments: tuple[positionals: seq[Expression], keyword: seq[tuple[name: IdentExpr, value: Expression]]] = (positionals: @[], keyword: @[])
var argument: ASTNode = nil var argument: Expression = nil
var argCount = 0 var argCount = 0
if not self.check(RightParen): if not self.check(RightParen):
while true: while true:
@ -346,10 +346,11 @@ proc makeCall(self: Parser, callee: ASTNode): ASTNode =
break break
argument = self.expression() argument = self.expression()
if argument.kind == assignExpr: if argument.kind == assignExpr:
if AssignExpr(argument).name in argNames: # TODO: This will explode with slices!
if IdentExpr(AssignExpr(argument).name) in argNames:
self.error("duplicate keyword argument in call") self.error("duplicate keyword argument in call")
argNames.add(AssignExpr(argument).name) argNames.add(IdentExpr(AssignExpr(argument).name))
arguments.keyword.add((name: AssignExpr(argument).name, value: AssignExpr(argument).value)) arguments.keyword.add((name: IdentExpr(AssignExpr(argument).name), value: AssignExpr(argument).value))
elif arguments.keyword.len() == 0: elif arguments.keyword.len() == 0:
arguments.positionals.add(argument) arguments.positionals.add(argument)
else: else:
@ -361,8 +362,8 @@ proc makeCall(self: Parser, callee: ASTNode): ASTNode =
result = newCallExpr(callee, arguments, tok) result = newCallExpr(callee, arguments, tok)
proc call(self: Parser): ASTNode = proc call(self: Parser): Expression =
## Parses func5ion calls, object field ## Parses function calls, object field
## accessing and slicing expressions ## accessing and slicing expressions
result = self.primary() result = self.primary()
while true: while true:
@ -388,7 +389,7 @@ proc call(self: Parser): ASTNode =
break break
proc unary(self: Parser): ASTNode = proc unary(self: Parser): Expression =
## Parses unary expressions ## Parses unary expressions
if self.match([Minus, Tilde, LogicalNot, Plus]): if self.match([Minus, Tilde, LogicalNot, Plus]):
result = newUnaryExpr(self.peek(-1), self.unary()) result = newUnaryExpr(self.peek(-1), self.unary())
@ -396,7 +397,7 @@ proc unary(self: Parser): ASTNode =
result = self.call() result = self.call()
proc customUnaryOperator(self: Parser): ASTNode = proc customUnaryOperator(self: Parser): Expression =
## Parses user-defined unary expressions ## Parses user-defined unary expressions
if self.peek().lexeme in self.operators: if self.peek().lexeme in self.operators:
discard self.step() discard self.step()
@ -405,120 +406,120 @@ proc customUnaryOperator(self: Parser): ASTNode =
result = self.unary() result = self.unary()
proc pow(self: Parser): ASTNode = proc pow(self: Parser): Expression =
## Parses exponentiation expressions ## Parses exponentiation expressions
result = self.customUnaryOperator() result = self.customUnaryOperator()
var operator: Token var operator: Token
var right: ASTNode var right: Expression
while self.match(DoubleStar): while self.match(DoubleStar):
operator = self.peek(-1) operator = self.peek(-1)
right = self.customUnaryOperator() right = self.customUnaryOperator()
result = newBinaryExpr(result, operator, right) result = newBinaryExpr(result, operator, right)
proc mul(self: Parser): ASTNode = proc mul(self: Parser): Expression =
## Parses multiplication and division expressions ## Parses multiplication and division expressions
result = self.pow() result = self.pow()
var operator: Token var operator: Token
var right: ASTNode var right: Expression
while self.match([Slash, Percentage, FloorDiv, Star]): while self.match([Slash, Percentage, FloorDiv, Star]):
operator = self.peek(-1) operator = self.peek(-1)
right = self.pow() right = self.pow()
result = newBinaryExpr(result, operator, right) result = newBinaryExpr(result, operator, right)
proc add(self: Parser): ASTNode = proc add(self: Parser): Expression =
## Parses addition and subtraction expressions ## Parses addition and subtraction expressions
result = self.mul() result = self.mul()
var operator: Token var operator: Token
var right: ASTNode var right: Expression
while self.match([Plus, Minus]): while self.match([Plus, Minus]):
operator = self.peek(-1) operator = self.peek(-1)
right = self.mul() right = self.mul()
result = newBinaryExpr(result, operator, right) result = newBinaryExpr(result, operator, right)
proc comparison(self: Parser): ASTNode = proc comparison(self: Parser): Expression =
## Parses other comparison expressions ## Parses other comparison expressions
## and some other operators ## and some other operators
result = self.add() result = self.add()
var operator: Token var operator: Token
var right: ASTNode var right: Expression
while self.match([LessThan, GreaterThan, LessOrEqual, GreaterOrEqual, Is, As, Of, IsNot]): while self.match([LessThan, GreaterThan, LessOrEqual, GreaterOrEqual, Is, As, Of, IsNot]):
operator = self.peek(-1) operator = self.peek(-1)
right = self.add() right = self.add()
result = newBinaryExpr(result, operator, right) result = newBinaryExpr(result, operator, right)
proc equality(self: Parser): ASTNode = proc equality(self: Parser): Expression =
## Parses equality expressions ## Parses equality expressions
result = self.comparison() result = self.comparison()
var operator: Token var operator: Token
var right: ASTNode var right: Expression
while self.match([DoubleEqual, NotEqual]): while self.match([DoubleEqual, NotEqual]):
operator = self.peek(-1) operator = self.peek(-1)
right = self.comparison() right = self.comparison()
result = newBinaryExpr(result, operator, right) result = newBinaryExpr(result, operator, right)
proc logicalAnd(self: Parser): ASTNode = proc logicalAnd(self: Parser): Expression =
## Parses logical and expressions ## Parses logical and expressions
## (a and b) ## (a and b)
result = self.equality() result = self.equality()
var operator: Token var operator: Token
var right: ASTNode var right: Expression
while self.match(LogicalAnd): while self.match(LogicalAnd):
operator = self.peek(-1) operator = self.peek(-1)
right = self.equality() right = self.equality()
result = newBinaryExpr(result, operator, right) result = newBinaryExpr(result, operator, right)
proc logicalOr(self: Parser): ASTNode = proc logicalOr(self: Parser): Expression =
## Parses logical or expressions ## Parses logical or expressions
## (a or b) ## (a or b)
result = self.logicalAnd() result = self.logicalAnd()
var operator: Token var operator: Token
var right: ASTNode var right: Expression
while self.match(LogicalOr): while self.match(LogicalOr):
operator = self.peek(-1) operator = self.peek(-1)
right = self.logicalAnd() right = self.logicalAnd()
result = newBinaryExpr(result, operator, right) result = newBinaryExpr(result, operator, right)
proc bitwiseAnd(self: Parser): ASTNode = proc bitwiseAnd(self: Parser): Expression =
## Parses a & b expressions ## Parses a & b expressions
result = self.logicalOr() result = self.logicalOr()
var operator: Token var operator: Token
var right: ASTNode var right: Expression
while self.match(Pipe): while self.match(Pipe):
operator = self.peek(-1) operator = self.peek(-1)
right = self.logicalOr() right = self.logicalOr()
result = newBinaryExpr(result, operator, right) result = newBinaryExpr(result, operator, right)
proc bitwiseOr(self: Parser): ASTNode = proc bitwiseOr(self: Parser): Expression =
## Parses a | b expressions ## Parses a | b expressions
result = self.bitwiseAnd() result = self.bitwiseAnd()
var operator: Token var operator: Token
var right: ASTNode var right: Expression
while self.match(Ampersand): while self.match(Ampersand):
operator = self.peek(-1) operator = self.peek(-1)
right = self.bitwiseAnd() right = self.bitwiseAnd()
result = newBinaryExpr(result, operator, right) result = newBinaryExpr(result, operator, right)
proc customBinaryOperator(self: Parser): ASTNode = proc customBinaryOperator(self: Parser): Expression =
## Parses user-defined binary operators ## Parses user-defined binary operators
result = self.bitwiseOr() result = self.bitwiseOr()
var operator: Token var operator: Token
var right: ASTNode var right: Expression
while self.peek().lexeme in self.operators: while self.peek().lexeme in self.operators:
operator = self.step() operator = self.step()
right = self.bitwiseOr() right = self.bitwiseOr()
result = newBinaryExpr(result, operator, right) result = newBinaryExpr(result, operator, right)
proc assignment(self: Parser): ASTNode = proc assignment(self: Parser): Expression =
## Parses assignment, the highest-level ## Parses assignment, the highest-level
## expression (including stuff like a.b = 1). ## expression (including stuff like a.b = 1).
## Slice assignments are also parsed here ## Slice assignments are also parsed here
@ -536,7 +537,7 @@ proc assignment(self: Parser): ASTNode =
self.error("invalid assignment target") self.error("invalid assignment target")
proc assertStmt(self: Parser): ASTNode = proc assertStmt(self: Parser): Statement =
## Parses "assert" statements, which ## Parses "assert" statements, which
## raise an error if the expression ## raise an error if the expression
## fed into them is falsey ## fed into them is falsey
@ -556,13 +557,13 @@ proc endScope(self: Parser) =
dec(self.scopeDepth) dec(self.scopeDepth)
proc blockStmt(self: Parser): ASTNode = proc blockStmt(self: Parser): Statement =
## Parses block statements. A block ## Parses block statements. A block
## statement simply opens a new local ## statement simply opens a new local
## scope ## scope
self.beginScope() self.beginScope()
let tok = self.peek(-1) let tok = self.peek(-1)
var code: seq[ASTNode] = @[] var code: seq[Declaration] = @[]
while not self.check(RightBrace) and not self.done(): while not self.check(RightBrace) and not self.done():
code.add(self.declaration()) code.add(self.declaration())
self.expect(RightBrace, "expecting '}'") self.expect(RightBrace, "expecting '}'")
@ -570,7 +571,7 @@ proc blockStmt(self: Parser): ASTNode =
self.endScope() self.endScope()
proc breakStmt(self: Parser): ASTNode = proc breakStmt(self: Parser): Statement =
## Parses break statements ## Parses break statements
let tok = self.peek(-1) let tok = self.peek(-1)
if self.currentLoop != Loop: if self.currentLoop != Loop:
@ -579,7 +580,7 @@ proc breakStmt(self: Parser): ASTNode =
result = newBreakStmt(tok) result = newBreakStmt(tok)
proc deferStmt(self: Parser): ASTNode = proc deferStmt(self: Parser): Statement =
## Parses defer statements ## Parses defer statements
let tok = self.peek(-1) let tok = self.peek(-1)
if self.currentFunction == nil: if self.currentFunction == nil:
@ -588,7 +589,7 @@ proc deferStmt(self: Parser): ASTNode =
endOfLine("missing semicolon after defer statement") endOfLine("missing semicolon after defer statement")
proc continueStmt(self: Parser): ASTNode = proc continueStmt(self: Parser): Statement =
## Parses continue statements ## Parses continue statements
let tok = self.peek(-1) let tok = self.peek(-1)
if self.currentLoop != Loop: if self.currentLoop != Loop:
@ -597,12 +598,12 @@ proc continueStmt(self: Parser): ASTNode =
result = newContinueStmt(tok) result = newContinueStmt(tok)
proc returnStmt(self: Parser): ASTNode = proc returnStmt(self: Parser): Statement =
## Parses return statements ## Parses return statements
let tok = self.peek(-1) let tok = self.peek(-1)
if self.currentFunction == nil: if self.currentFunction == nil:
self.error("'return' cannot be used outside functions") self.error("'return' cannot be used outside functions")
var value: ASTNode = newNilExpr(Token(lexeme: "nil")) var value: Expression = newNilExpr(Token(lexeme: "nil"))
if not self.check(Semicolon): if not self.check(Semicolon):
# Since return can be used on its own too # Since return can be used on its own too
# (in which case it implicitly returns nil), # (in which case it implicitly returns nil),
@ -613,7 +614,7 @@ proc returnStmt(self: Parser): ASTNode =
result = newReturnStmt(value, tok) result = newReturnStmt(value, tok)
proc yieldStmt(self: Parser): ASTNode = proc yieldStmt(self: Parser): Statement =
## Parses yield statements ## Parses yield statements
let tok = self.peek(-1) let tok = self.peek(-1)
if self.currentFunction == nil: if self.currentFunction == nil:
@ -627,7 +628,7 @@ proc yieldStmt(self: Parser): ASTNode =
endOfLine("missing semicolon after yield statement") endOfLine("missing semicolon after yield statement")
proc awaitStmt(self: Parser): ASTNode = proc awaitStmt(self: Parser): Statement =
## Parses await statements ## Parses await statements
let tok = self.peek(-1) let tok = self.peek(-1)
if self.currentFunction == nil: if self.currentFunction == nil:
@ -638,9 +639,9 @@ proc awaitStmt(self: Parser): ASTNode =
endOfLine("missing semicolon after await statement") endOfLine("missing semicolon after await statement")
proc raiseStmt(self: Parser): ASTNode = proc raiseStmt(self: Parser): Statement =
## Parses raise statements ## Parses raise statements
var exception: ASTNode var exception: Expression
let tok = self.peek(-1) let tok = self.peek(-1)
if not self.check(Semicolon): if not self.check(Semicolon):
# Raise can be used on its own, in which # Raise can be used on its own, in which
@ -650,7 +651,7 @@ proc raiseStmt(self: Parser): ASTNode =
result = newRaiseStmt(exception, tok) result = newRaiseStmt(exception, tok)
proc forEachStmt(self: Parser): ASTNode = proc forEachStmt(self: Parser): Statement =
## Parses C#-like foreach loops ## Parses C#-like foreach loops
let tok = self.peek(-1) let tok = self.peek(-1)
var enclosingLoop = self.currentLoop var enclosingLoop = self.currentLoop
@ -666,7 +667,7 @@ proc forEachStmt(self: Parser): ASTNode =
self.currentLoop = enclosingLoop self.currentLoop = enclosingLoop
proc importStmt(self: Parser, fromStmt: bool = false): ASTNode = proc importStmt(self: Parser, fromStmt: bool = false): Statement =
## Parses import statements ## Parses import statements
var tok: Token var tok: Token
if fromStmt: if fromStmt:
@ -680,26 +681,28 @@ proc importStmt(self: Parser, fromStmt: bool = false): ASTNode =
proc tryStmt(self: Parser): ASTNode = proc tryStmt(self: Parser): Statement =
## Parses try/except/else/finally blocks ## Parses try/except/else/finally blocks
let tok = self.peek(-1) let tok = self.peek(-1)
var body = self.statement() var body = self.statement()
var handlers: seq[tuple[body, exc, name: ASTNode]] = @[] var handlers: seq[tuple[body: Statement, exc: IdentExpr, name: IdentExpr]] = @[]
var finallyClause: ASTNode var finallyClause: Statement
var elseClause: ASTNode var elseClause: Statement
var asName: ASTNode var asName: IdentExpr
var excName: ASTNode var excName: Expression
var handlerBody: ASTNode var handlerBody: Statement
while self.match(Except): while self.match(Except):
excName = self.expression() excName = self.expression()
if excName.kind == identExpr: if excName.kind == identExpr:
handlerBody = self.statement() handlerBody = self.statement()
handlers.add((body: handlerBody, exc: excName, name: asName)) handlers.add((body: handlerBody, exc: IdentExpr(excName), name: asName))
asName = nil asName = nil
elif excName.kind == binaryExpr and BinaryExpr(excName).operator.kind == As: elif excName.kind == binaryExpr and BinaryExpr(excName).operator.kind == As:
asName = BinaryExpr(excName).b asName = IdentExpr(BinaryExpr(excName).b)
if BinaryExpr(excName).b.kind != identExpr: if BinaryExpr(excName).b.kind != identExpr:
self.error("expecting alias name after 'except ... as'") self.error("expecting alias name after 'except ... as'")
elif BinaryExpr(excName).a.kind != identExpr:
self.error("expecting exception name")
excName = BinaryExpr(excName).a excName = BinaryExpr(excName).a
# Note how we don't use elif here: when the if above sets excName to As' # Note how we don't use elif here: when the if above sets excName to As'
# first operand, that might be a tuple, which we unpack below # first operand, that might be a tuple, which we unpack below
@ -710,7 +713,9 @@ proc tryStmt(self: Parser): ASTNode =
# types each time # types each time
handlerBody = self.statement() handlerBody = self.statement()
for element in TupleExpr(excName).members: for element in TupleExpr(excName).members:
handlers.add((body: handlerBody, exc: element, name: asName)) if element.kind != identExpr:
self.error("expecting exception name")
handlers.add((body: handlerBody, exc: IdentExpr(element), name: asName))
continue continue
else: else:
excName = nil excName = nil
@ -726,7 +731,7 @@ proc tryStmt(self: Parser): ASTNode =
result = newTryStmt(body, handlers, finallyClause, elseClause, tok) result = newTryStmt(body, handlers, finallyClause, elseClause, tok)
proc whileStmt(self: Parser): ASTNode = proc whileStmt(self: Parser): Statement =
## Parses a C-style while loop statement ## Parses a C-style while loop statement
let tok = self.peek(-1) let tok = self.peek(-1)
self.beginScope() self.beginScope()
@ -740,7 +745,7 @@ proc whileStmt(self: Parser): ASTNode =
self.endScope() self.endScope()
proc forStmt(self: Parser): ASTNode = proc forStmt(self: Parser): Statement =
## Parses a C-style for loop ## Parses a C-style for loop
self.beginScope() self.beginScope()
let tok = self.peek(-1) let tok = self.peek(-1)
@ -748,8 +753,8 @@ proc forStmt(self: Parser): ASTNode =
self.currentLoop = Loop self.currentLoop = Loop
self.expect(LeftParen, "expecting '(' after 'for'") self.expect(LeftParen, "expecting '(' after 'for'")
var initializer: ASTNode = nil var initializer: ASTNode = nil
var condition: ASTNode = nil var condition: Expression = nil
var increment: ASTNode = nil var increment: Expression = nil
if self.match(Semicolon): if self.match(Semicolon):
discard discard
elif self.match(Var): elif self.match(Var):
@ -768,7 +773,7 @@ proc forStmt(self: Parser): ASTNode =
if increment != nil: if increment != nil:
# The increment runs after each iteration, so we # The increment runs after each iteration, so we
# inject it into the block as the last statement # inject it into the block as the last statement
body = newBlockStmt(@[body, newExprStmt(increment, increment.token)], tok) body = newBlockStmt(@[Declaration(body), Declaration(Expression(newExprStmt(increment, increment.token)))], tok)
if condition == nil: if condition == nil:
## An empty condition is functionally ## An empty condition is functionally
## equivalent to "true" ## equivalent to "true"
@ -778,7 +783,7 @@ proc forStmt(self: Parser): ASTNode =
if initializer != nil: if initializer != nil:
# Nested blocks, so the initializer is # Nested blocks, so the initializer is
# only executed once # only executed once
body = newBlockStmt(@[initializer, body], tok) body = newBlockStmt(@[Declaration(initializer), Declaration(body)], tok)
# This desgugars the following code: # This desgugars the following code:
# for (var i = 0; i < 10; i += 1) { # for (var i = 0; i < 10; i += 1) {
# print(i); # print(i);
@ -797,14 +802,14 @@ proc forStmt(self: Parser): ASTNode =
self.endScope() self.endScope()
proc ifStmt(self: Parser): ASTNode = proc ifStmt(self: Parser): Statement =
## Parses if statements ## Parses if statements
let tok = self.peek(-1) let tok = self.peek(-1)
self.expect(LeftParen, "expecting '(' before if condition") self.expect(LeftParen, "expecting '(' before if condition")
var condition = self.expression() var condition = self.expression()
self.expect(RightParen, "expecting ')' after if condition") self.expect(RightParen, "expecting ')' after if condition")
var thenBranch = self.statement() var thenBranch = self.statement()
var elseBranch: ASTNode = nil var elseBranch: Statement = nil
if self.match(Else): if self.match(Else):
elseBranch = self.statement() elseBranch = self.statement()
result = newIfStmt(condition, thenBranch, elseBranch, tok) result = newIfStmt(condition, thenBranch, elseBranch, tok)
@ -819,15 +824,15 @@ template checkDecl(self: Parser, isPrivate: bool) =
self.error("cannot bind public names inside local scopes") self.error("cannot bind public names inside local scopes")
proc varDecl(self: Parser, isLet: bool = false, isConst: bool = false): ASTNode = proc varDecl(self: Parser, isLet: bool = false, isConst: bool = false): Declaration =
## Parses variable declarations ## Parses variable declarations
var tok = self.peek(-1) var tok = self.peek(-1)
var value: ASTNode var value: Expression
self.expect(Identifier, &"expecting identifier after '{tok.lexeme}'") self.expect(Identifier, &"expecting identifier after '{tok.lexeme}'")
var name = newIdentExpr(self.peek(-1)) var name = newIdentExpr(self.peek(-1))
let isPrivate = not self.match(Star) let isPrivate = not self.match(Star)
self.checkDecl(isPrivate) self.checkDecl(isPrivate)
var valueType: ASTNode var valueType: IdentExpr
if self.match(Colon): if self.match(Colon):
# We don't enforce it here because # We don't enforce it here because
# the compiler may be able to infer # the compiler may be able to infer
@ -854,13 +859,13 @@ proc varDecl(self: Parser, isLet: bool = false, isConst: bool = false): ASTNode
discard # Unreachable discard # Unreachable
proc funDecl(self: Parser, isAsync: bool = false, isGenerator: bool = false, isLambda: bool = false, isOperator: bool = false): ASTNode = proc funDecl(self: Parser, isAsync: bool = false, isGenerator: bool = false, isLambda: bool = false, isOperator: bool = false): Declaration =
## Parses functions, coroutines, generators, anonymous functions and custom operators ## Parses functions, coroutines, generators, anonymous functions and custom operators
let tok = self.peek(-1) let tok = self.peek(-1)
var enclosingFunction = self.currentFunction var enclosingFunction = self.currentFunction
var arguments: seq[tuple[name: ASTNode, valueType: ASTNode]] = @[] var arguments: seq[tuple[name: IdentExpr, valueType: IdentExpr]] = @[]
var defaults: seq[ASTNode] = @[] var defaults: seq[Expression] = @[]
var returnType: ASTNode var returnType: IdentExpr
if not isLambda and self.check(Identifier): if not isLambda and self.check(Identifier):
# We do this extra check because we might # We do this extra check because we might
# be called from a contexst where it's # be called from a contexst where it's
@ -885,11 +890,11 @@ proc funDecl(self: Parser, isAsync: bool = false, isGenerator: bool = false, isL
# call us back with isLambda=true, allowing us # call us back with isLambda=true, allowing us
# to actually parse the function as an expression # to actually parse the function as an expression
dec(self.current) dec(self.current)
result = self.expressionStatement() result = Declaration(self.expressionStatement())
self.currentFunction = enclosingFunction self.currentFunction = enclosingFunction
return result return result
elif isLambda: elif isLambda:
self.currentFunction = newLambdaExpr(arguments, defaults, newBlockStmt(@[], Token()), isGenerator=isGenerator, isAsync=isAsync, token=tok) self.currentFunction = Declaration(Expression(newLambdaExpr(arguments, defaults, Declaration(Statement(newBlockStmt(@[], Token()))), isGenerator=isGenerator, isAsync=isAsync, token=tok)))
elif not isOperator: elif not isOperator:
self.error("funDecl: invalid state") self.error("funDecl: invalid state")
if self.match(Colon): if self.match(Colon):
@ -900,7 +905,7 @@ proc funDecl(self: Parser, isAsync: bool = false, isGenerator: bool = false, isL
returnType = newIdentExpr(self.peek(-1)) returnType = newIdentExpr(self.peek(-1))
if not self.match(LeftBrace): if not self.match(LeftBrace):
# Argument-less function # Argument-less function
var parameter: tuple[name: ASTNode, valueType: ASTNode] var parameter: tuple[name: IdentExpr, valueType: IdentExpr]
self.expect(LeftParen) self.expect(LeftParen)
while not self.check(RightParen): while not self.check(RightParen):
if arguments.len > 255: if arguments.len > 255:
@ -938,9 +943,9 @@ proc funDecl(self: Parser, isAsync: bool = false, isGenerator: bool = false, isL
FunDecl(self.currentFunction).arguments = arguments FunDecl(self.currentFunction).arguments = arguments
FunDecl(self.currentFunction).returnType = returnType FunDecl(self.currentFunction).returnType = returnType
else: else:
LambdaExpr(self.currentFunction).body = self.blockStmt() LambdaExpr(Expression(self.currentFunction)).body = self.blockStmt()
LambdaExpr(self.currentFunction).arguments = arguments LambdaExpr(Expression(self.currentFunction)).arguments = arguments
LambdaExpr(self.currentFunction).returnType = returnType LambdaExpr(Expression(self.currentFunction)).returnType = returnType
result = self.currentFunction result = self.currentFunction
if isOperator: if isOperator:
# isOperator is only true for functions # isOperator is only true for functions
@ -955,20 +960,20 @@ proc funDecl(self: Parser, isAsync: bool = false, isGenerator: bool = false, isL
self.currentFunction = enclosingFunction self.currentFunction = enclosingFunction
proc expression(self: Parser): ASTNode = proc expression(self: Parser): Expression =
## Parses expressions ## Parses expressions
result = self.assignment() result = self.assignment()
proc expressionStatement(self: Parser): ASTNode = proc expressionStatement(self: Parser): Statement =
## Parses expression statements, which ## Parses expression statements, which
## are expressions followed by a semicolon ## are expressions followed by a semicolon
var expression = self.expression() var expression = self.expression()
endOfLine("missing semicolon after expression") endOfLine("missing semicolon after expression")
result = newExprStmt(expression, expression.token) result = Statement(newExprStmt(expression, expression.token))
proc statement(self: Parser): ASTNode = proc statement(self: Parser): Statement =
## Parses statements ## Parses statements
case self.peek().kind: case self.peek().kind:
of If: of If:
@ -1025,7 +1030,7 @@ proc statement(self: Parser): ASTNode =
result = self.expressionStatement() result = self.expressionStatement()
proc declaration(self: Parser): ASTNode = proc declaration(self: Parser): Declaration =
## Parses declarations ## Parses declarations
case self.peek().kind: case self.peek().kind:
of Var, Const, Let: of Var, Const, Let:
@ -1046,7 +1051,7 @@ proc declaration(self: Parser): ASTNode =
of Type, Comment, TokenType.Whitespace, TokenType.Tab: of Type, Comment, TokenType.Whitespace, TokenType.Tab:
discard self.step() # TODO discard self.step() # TODO
else: else:
result = self.statement() result = Declaration(self.statement())
proc parse*(self: Parser, tokens: seq[Token], file: string): seq[ASTNode] = proc parse*(self: Parser, tokens: seq[Token], file: string): seq[ASTNode] =
@ -1066,13 +1071,10 @@ proc parse*(self: Parser, tokens: seq[Token], file: string): seq[ASTNode] =
if token.kind == Operator: if token.kind == Operator:
if i == self.tokens.high(): if i == self.tokens.high():
self.error("invalid state: found malformed tokenizer input while looking for operators (missing EOF)") self.error("invalid state: found malformed tokenizer input while looking for operators (missing EOF)")
elif self.tokens[i + 1].kind != Identifier: self.operators.add(self.tokens[i + 1].lexeme)
# The input is probably malformed,
# we'll catch this error later
break
else:
self.operators.add(self.tokens[i + 1].lexeme)
if i == self.tokens.high() and token.kind != EndOfFile: if i == self.tokens.high() and token.kind != EndOfFile:
# Since we're iterating this list anyway might as
# well perform some extra checks
self.error("invalid state: found malformed tokenizer input while looking for operators (missing EOF)") self.error("invalid state: found malformed tokenizer input while looking for operators (missing EOF)")
while not self.done(): while not self.done():
result.add(self.declaration()) result.add(self.declaration())

5
src/test.nim
View File

@ -16,7 +16,8 @@ import jale/multiline
import frontend/lexer as l import frontend/lexer as l
import frontend/parser as p import frontend/parser as p
import frontend/compiler as c import frontend/compiler as c
import frontend/optimizer as o # TODO: Broken
# import frontend/optimizer as o
import util/serializer as s import util/serializer as s
import util/debugger import util/debugger
@ -41,7 +42,7 @@ when isMainModule:
tokens: seq[Token] = @[] tokens: seq[Token] = @[]
tree: seq[ASTNode] = @[] tree: seq[ASTNode] = @[]
compiled: Chunk compiled: Chunk
optimized: tuple[tree: seq[ASTNode], warnings: seq[Warning]] # optimized: tuple[tree: seq[ASTNode], warnings: seq[Warning]]
serialized: Serialized serialized: Serialized
serializedRaw: seq[byte] serializedRaw: seq[byte]
tokenizer = newLexer() tokenizer = newLexer()