nocturn9x
/
peon
2
2
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Initial work on adding type support to the parser for declarations + fixes for lambdas

This commit is contained in:
Mattia Giambirtone 2022-04-05 15:52:58 +02:00
parent 7f950893c5
commit dbab30952b
4 changed files with 201 additions and 211 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

200
src/frontend/meta/ast.nim
View File

@ -18,10 +18,11 @@
import strformat import strformat
import strutils import strutils
import sequtils
import token import token
export token
type type
NodeKind* = enum NodeKind* = enum
@ -30,8 +31,7 @@ type
## precedence ## precedence
# Declarations # Declarations
classDecl = 0u8, funDecl = 0'u8,
funDecl,
varDecl, varDecl,
# Statements # Statements
forStmt, # Unused for now (for loops are compiled to while loops) forStmt, # Unused for now (for loops are compiled to while loops)
@ -44,11 +44,9 @@ type
blockStmt, blockStmt,
raiseStmt, raiseStmt,
assertStmt, assertStmt,
delStmt,
tryStmt, tryStmt,
yieldStmt, yieldStmt,
awaitStmt, awaitStmt,
fromImportStmt,
importStmt, importStmt,
deferStmt, deferStmt,
# An expression followed by a semicolon # An expression followed by a semicolon
@ -64,8 +62,8 @@ type
sliceExpr, sliceExpr,
callExpr, callExpr,
getItemExpr, # Get expressions like a.b getItemExpr, # Get expressions like a.b
# Primary expressions # Primary expressions
groupingExpr, # Parenthesized expressions such as (true) and (3 + 4) groupingExpr, # Parenthesized expressions such as (true) and (3 + 4)
trueExpr, trueExpr,
listExpr, listExpr,
tupleExpr, tupleExpr,
@ -95,7 +93,7 @@ type
# Here I would've rather used object variants, and in fact that's what was in # Here I would've rather used object variants, and in fact that's what was in
# place before, but not being able to re-declare a field of the same type in # place before, but not being able to re-declare a field of the same type in
# another case branch is kind of a deal breaker long-term, so until that is # another case branch is kind of a deal breaker long-term, so until that is
# fixed (check out https://github.com/nim-lang/RFCs/issues/368 for more info) # fixed (check out https://github.com/nim-lang/RFCs/issues/368 for more info).
# I'll stick to using inheritance instead # I'll stick to using inheritance instead
LiteralExpr* = ref object of ASTNode LiteralExpr* = ref object of ASTNode
@ -129,6 +127,7 @@ type
# a tough luck for us # a tough luck for us
ListExpr* = ref object of ASTNode ListExpr* = ref object of ASTNode
members*: seq[ASTNode] members*: seq[ASTNode]
valueType*: IdentExpr
SetExpr* = ref object of ListExpr SetExpr* = ref object of ListExpr
@ -137,6 +136,8 @@ type
DictExpr* = ref object of ASTNode DictExpr* = ref object of ASTNode
keys*: seq[ASTNode] keys*: seq[ASTNode]
values*: seq[ASTNode] values*: seq[ASTNode]
keyType*: IdentExpr
valueType*: IdentExpr
IdentExpr* = ref object of ASTNode IdentExpr* = ref object of ASTNode
name*: Token name*: Token
@ -173,21 +174,18 @@ type
expression*: ASTNode expression*: ASTNode
AwaitExpr* = ref object of ASTNode AwaitExpr* = ref object of ASTNode
awaitee*: ASTNode expression*: ASTNode
LambdaExpr* = ref object of ASTNode LambdaExpr* = ref object of ASTNode
body*: ASTNode body*: ASTNode
arguments*: seq[ASTNode] arguments*: seq[tuple[name: ASTNode, valueType: ASTNode]]
# This is, in order, the list of each default argument
# the function takes. It maps 1:1 with self.arguments
# although it may be shorter (in which case this maps
# 1:1 with what's left of self.arguments after all
# positional arguments have been consumed)
defaults*: seq[ASTNode] defaults*: seq[ASTNode]
isGenerator*: bool isGenerator*: bool
isAsync*: bool
returnType*: ASTNode
SliceExpr* = ref object of ASTNode SliceExpr* = ref object of ASTNode
slicee*: ASTNode expression*: ASTNode
ends*: seq[ASTNode] ends*: seq[ASTNode]
AssignExpr* = ref object of ASTNode AssignExpr* = ref object of ASTNode
@ -200,13 +198,6 @@ type
ImportStmt* = ref object of ASTNode ImportStmt* = ref object of ASTNode
moduleName*: ASTNode moduleName*: ASTNode
FromImportStmt* = ref object of ASTNode
fromModule*: ASTNode
fromAttributes*: seq[ASTNode]
DelStmt* = ref object of ASTNode
name*: ASTNode
AssertStmt* = ref object of ASTNode AssertStmt* = ref object of ASTNode
expression*: ASTNode expression*: ASTNode
@ -225,7 +216,7 @@ type
body*: ASTNode body*: ASTNode
DeferStmt* = ref object of ASTNode DeferStmt* = ref object of ASTNode
deferred*: ASTNode expression*: ASTNode
TryStmt* = ref object of ASTNode TryStmt* = ref object of ASTNode
body*: ASTNode body*: ASTNode
@ -238,7 +229,7 @@ type
body*: ASTNode body*: ASTNode
AwaitStmt* = ref object of ASTNode AwaitStmt* = ref object of ASTNode
awaitee*: ASTNode expression*: ASTNode
BreakStmt* = ref object of ASTNode BreakStmt* = ref object of ASTNode
@ -256,7 +247,6 @@ type
expression*: ASTNode expression*: ASTNode
Declaration* = ref object of ASTNode Declaration* = ref object of ASTNode
owner*: string # Used for determining if a module can access a given field
closedOver*: bool closedOver*: bool
VarDecl* = ref object of Declaration VarDecl* = ref object of Declaration
@ -265,36 +255,25 @@ type
isConst*: bool isConst*: bool
isStatic*: bool isStatic*: bool
isPrivate*: bool isPrivate*: bool
isLet*: bool
valueType*: ASTNode
FunDecl* = ref object of Declaration FunDecl* = ref object of Declaration
name*: ASTNode name*: ASTNode
body*: ASTNode body*: ASTNode
arguments*: seq[ASTNode] arguments*: seq[tuple[name: ASTNode, valueType: ASTNode]]
# This is, in order, the list of each default argument defaults: seq[ASTNode]
# the function takes. It maps 1:1 with self.arguments
# although it may be shorter (in which case this maps
# 1:1 with what's left of self.arguments after all
# positional arguments have been consumed)
defaults*: seq[ASTNode]
isAsync*: bool isAsync*: bool
isGenerator*: bool isGenerator*: bool
isStatic*: bool
isPrivate*: bool isPrivate*: bool
returnType*: ASTNode
ClassDecl* = ref object of Declaration Expression* = LiteralExpr | ListExpr | GetItemExpr | SetItemExpr | UnaryExpr | BinaryExpr | CallExpr | AssignExpr |
name*: ASTNode GroupingExpr | IdentExpr | DictExpr | TupleExpr | SetExpr | TrueExpr | FalseExpr | NilExpr |
body*: ASTNode NanExpr | InfExpr
parents*: seq[ASTNode]
isStatic*: bool
isPrivate*: bool
Expression* = LiteralExpr | ListExpr | GetItemExpr | SetItemExpr | UnaryExpr | BinaryExpr | CallExpr | AssignExpr | Statement* = ExprStmt | ImportStmt | AssertStmt | RaiseStmt | BlockStmt | ForStmt | WhileStmt |
GroupingExpr | IdentExpr | DictExpr | TupleExpr | SetExpr | ForStmt | BreakStmt | ContinueStmt | ReturnStmt | IfStmt
TrueExpr | FalseExpr | NilExpr |
NanExpr | InfExpr
Statement* = ExprStmt | ImportStmt | FromImportStmt | DelStmt | AssertStmt | RaiseStmt | BlockStmt | ForStmt | WhileStmt |
ForStmt | BreakStmt | ContinueStmt | ReturnStmt | IfStmt
@ -306,15 +285,32 @@ proc newASTNode*(kind: NodeKind, token: Token): ASTNode =
result.token = token result.token = token
proc isConst*(self: ASTNode): bool {.inline.} = self.kind in {intExpr, hexExpr, binExpr, octExpr, strExpr, proc isConst*(self: ASTNode): bool =
falseExpr, ## Returns true if the given
trueExpr, infExpr, ## AST node represents a value
nanExpr, ## of constant type. All integers,
floatExpr, nilExpr} ## strings and singletons count as
## constants, as well as collections
## comprised only of those types.
case self.kind:
of intExpr, hexExpr, binExpr, octExpr, strExpr, falseExpr, trueExpr, infExpr, nanExpr, floatExpr, nilExpr:
return true
of tupleExpr, setExpr, listExpr:
for item in ListExpr(self).members:
if not item.isConst():
return false
return true
of dictExpr:
for (key, value) in zip(DictExpr(self).keys, DictExpr(self).values):
if not key.isConst() or not value.isConst():
return false
return true
else:
return false
# TODO: Fix (not all literals are constants)
proc isLiteral*(self: ASTNode): bool {.inline.} = self.isConst() or self.kind in proc isLiteral*(self: ASTNode): bool {.inline.} = self.isConst()
{tupleExpr, dictExpr, setExpr, listExpr}
proc newIntExpr*(literal: Token): IntExpr = proc newIntExpr*(literal: Token): IntExpr =
@ -372,13 +368,14 @@ proc newGroupingExpr*(expression: ASTNode, token: Token): GroupingExpr =
result.token = token result.token = token
proc newLambdaExpr*(arguments, defaults: seq[ASTNode], body: ASTNode, proc newLambdaExpr*(arguments: seq[tuple[name: ASTNode, valueType: ASTNode]], defaults: seq[ASTNode], body: ASTNode,
isGenerator: bool, token: Token): LambdaExpr = isGenerator: bool, isAsync: bool, token: Token): LambdaExpr =
result = LambdaExpr(kind: lambdaExpr) result = LambdaExpr(kind: lambdaExpr)
result.body = body result.body = body
result.arguments = arguments result.arguments = arguments
result.defaults = defaults result.defaults = defaults
result.isGenerator = isGenerator result.isGenerator = isGenerator
result.isAsync = isAsync
result.token = token result.token = token
@ -431,10 +428,10 @@ proc newCallExpr*(callee: ASTNode, arguments: tuple[positionals: seq[ASTNode],
result.token = token result.token = token
proc newSliceExpr*(slicee: ASTNode, ends: seq[ASTNode], proc newSliceExpr*(expression: ASTNode, ends: seq[ASTNode],
token: Token): SliceExpr = token: Token): SliceExpr =
result = SliceExpr(kind: sliceExpr) result = SliceExpr(kind: sliceExpr)
result.slicee = slicee result.expression = expression
result.ends = ends result.ends = ends
result.token = token result.token = token
@ -467,9 +464,9 @@ proc newAssignExpr*(name, value: ASTNode, token: Token): AssignExpr =
result.token = token result.token = token
proc newAwaitExpr*(awaitee: ASTNode, token: Token): AwaitExpr = proc newAwaitExpr*(expression: ASTNode, token: Token): AwaitExpr =
result = AwaitExpr(kind: awaitExpr) result = AwaitExpr(kind: awaitExpr)
result.awaitee = awaitee result.expression = expression
result.token = token result.token = token
@ -485,29 +482,15 @@ proc newImportStmt*(moduleName: ASTNode, token: Token): ImportStmt =
result.token = token result.token = token
proc newFromImportStmt*(fromModule: ASTNode, fromAttributes: seq[ASTNode],
token: Token): FromImportStmt =
result = FromImportStmt(kind: fromImportStmt)
result.fromModule = fromModule
result.fromAttributes = fromAttributes
result.token = token
proc newDelStmt*(name: ASTNode, token: Token): DelStmt =
result = DelStmt(kind: delStmt)
result.name = name
result.token = token
proc newYieldStmt*(expression: ASTNode, token: Token): YieldStmt = proc newYieldStmt*(expression: ASTNode, token: Token): YieldStmt =
result = YieldStmt(kind: yieldStmt) result = YieldStmt(kind: yieldStmt)
result.expression = expression result.expression = expression
result.token = token result.token = token
proc newAwaitStmt*(awaitee: ASTNode, token: Token): AwaitExpr = proc newAwaitStmt*(expression: ASTNode, token: Token): AwaitExpr =
result = AwaitExpr(kind: awaitExpr) result = AwaitExpr(kind: awaitExpr)
result.awaitee = awaitee result.expression = expression
result.token = token result.token = token
@ -517,9 +500,9 @@ proc newAssertStmt*(expression: ASTNode, token: Token): AssertStmt =
result.token = token result.token = token
proc newDeferStmt*(deferred: ASTNode, token: Token): DeferStmt = proc newDeferStmt*(expression: ASTNode, token: Token): DeferStmt =
result = DeferStmt(kind: deferStmt) result = DeferStmt(kind: deferStmt)
result.deferred = deferred result.expression = expression
result.token = token result.token = token
@ -587,24 +570,22 @@ proc newIfStmt*(condition: ASTNode, thenBranch, elseBranch: ASTNode,
result.token = token result.token = token
proc newVarDecl*(name: ASTNode, value: ASTNode = newNilExpr(Token()), proc newVarDecl*(name: ASTNode, value: ASTNode, isConst: bool = false,
isStatic: bool = true, isConst: bool = false, isPrivate: bool = true, token: Token, closedOver: bool,
isPrivate: bool = true, token: Token, owner: string, isLet: bool = false, valueType: ASTNode): VarDecl =
closedOver: bool): VarDecl =
result = VarDecl(kind: varDecl) result = VarDecl(kind: varDecl)
result.name = name result.name = name
result.value = value result.value = value
result.isConst = isConst result.isConst = isConst
result.isStatic = isStatic
result.isPrivate = isPrivate result.isPrivate = isPrivate
result.token = token result.token = token
result.owner = owner result.isLet = isLet
result.valueType = valueType
proc newFunDecl*(name: ASTNode, arguments, defaults: seq[ASTNode], proc newFunDecl*(name: ASTNode, arguments: seq[tuple[name: ASTNode, valueType: ASTNode]], defaults: seq[ASTNode],
body: ASTNode, isStatic: bool = true, isAsync, body: ASTNode, isAsync, isGenerator: bool,
isGenerator: bool, isPrivate: bool = true, token: Token, isPrivate: bool, token: Token, closedOver: bool): FunDecl =
owner: string, closedOver: bool): FunDecl =
result = FunDecl(kind: funDecl) result = FunDecl(kind: funDecl)
result.name = name result.name = name
result.arguments = arguments result.arguments = arguments
@ -612,34 +593,18 @@ proc newFunDecl*(name: ASTNode, arguments, defaults: seq[ASTNode],
result.body = body result.body = body
result.isAsync = isAsync result.isAsync = isAsync
result.isGenerator = isGenerator result.isGenerator = isGenerator
result.isStatic = isStatic
result.isPrivate = isPrivate result.isPrivate = isPrivate
result.token = token result.token = token
result.owner = owner
result.closedOver = closedOver result.closedOver = closedOver
proc newClassDecl*(name: ASTNode, body: ASTNode,
parents: seq[ASTNode], isStatic: bool = true,
isPrivate: bool = true, token: Token,
owner: string, closedOver: bool): ClassDecl =
result = ClassDecl(kind: classDecl)
result.name = name
result.body = body
result.parents = parents
result.isStatic = isStatic
result.isPrivate = isPrivate
result.token = token
result.owner = owner
result.closedOver = closedOver
proc `$`*(self: ASTNode): string = proc `$`*(self: ASTNode): string =
if self == nil: if self == nil:
return "nil" return "nil"
case self.kind: case self.kind:
of intExpr, floatExpr, hexExpr, binExpr, octExpr, strExpr, trueExpr, of intExpr, floatExpr, hexExpr, binExpr, octExpr, strExpr, trueExpr,
falseExpr, nanExpr, nilExpr, infExpr: falseExpr, nanExpr, nilExpr, infExpr:
if self.kind in {trueExpr, falseExpr, nanExpr, nilExpr, infExpr}: if self.kind in {trueExpr, falseExpr, nanExpr, nilExpr, infExpr}:
result &= &"Literal({($self.kind)[0..^5]})" result &= &"Literal({($self.kind)[0..^5]})"
elif self.kind == strExpr: elif self.kind == strExpr:
@ -676,12 +641,6 @@ proc `$`*(self: ASTNode): string =
of importStmt: of importStmt:
var self = ImportStmt(self) var self = ImportStmt(self)
result &= &"Import({self.moduleName})" result &= &"Import({self.moduleName})"
of fromImportStmt:
var self = FromImportStmt(self)
result &= &"""FromImport(fromModule={self.fromModule}, fromAttributes=[{self.fromAttributes.join(", ")}])"""
of delStmt:
var self = DelStmt(self)
result &= &"Del({self.name})"
of assertStmt: of assertStmt:
var self = AssertStmt(self) var self = AssertStmt(self)
result &= &"Assert({self.expression})" result &= &"Assert({self.expression})"
@ -705,7 +664,7 @@ proc `$`*(self: ASTNode): string =
result &= &"Yield({self.expression})" result &= &"Yield({self.expression})"
of awaitExpr: of awaitExpr:
var self = AwaitExpr(self) var self = AwaitExpr(self)
result &= &"Await({self.awaitee})" result &= &"Await({self.expression})"
of ifStmt: of ifStmt:
var self = IfStmt(self) var self = IfStmt(self)
if self.elseBranch == nil: if self.elseBranch == nil:
@ -717,16 +676,13 @@ proc `$`*(self: ASTNode): string =
result &= &"YieldStmt({self.expression})" result &= &"YieldStmt({self.expression})"
of awaitStmt: of awaitStmt:
var self = AwaitStmt(self) var self = AwaitStmt(self)
result &= &"AwaitStmt({self.awaitee})" result &= &"AwaitStmt({self.expression})"
of varDecl: of varDecl:
var self = VarDecl(self) var self = VarDecl(self)
result &= &"Var(name={self.name}, value={self.value}, const={self.isConst}, static={self.isStatic}, private={self.isPrivate})" result &= &"Var(name={self.name}, value={self.value}, const={self.isConst}, private={self.isPrivate}, type={self.valueType})"
of funDecl: of funDecl:
var self = FunDecl(self) var self = FunDecl(self)
result &= &"""FunDecl(name={self.name}, body={self.body}, arguments=[{self.arguments.join(", ")}], defaults=[{self.defaults.join(", ")}], async={self.isAsync}, generator={self.isGenerator}, static={self.isStatic}, private={self.isPrivate})""" result &= &"""FunDecl(name={self.name}, body={self.body}, type={self.returnType}, arguments=[{self.arguments.join(", ")}], defaults=[{self.defaults.join(", ")}], async={self.isAsync}, generator={self.isGenerator}, private={self.isPrivate})"""
of classDecl:
var self = ClassDecl(self)
result &= &"""Class(name={self.name}, body={self.body}, parents=[{self.parents.join(", ")}], static={self.isStatic}, private={self.isPrivate})"""
of tupleExpr: of tupleExpr:
var self = TupleExpr(self) var self = TupleExpr(self)
result &= &"""Tuple([{self.members.join(", ")}])""" result &= &"""Tuple([{self.members.join(", ")}])"""
@ -741,13 +697,13 @@ proc `$`*(self: ASTNode): string =
result &= &"""Dict(keys=[{self.keys.join(", ")}], values=[{self.values.join(", ")}])""" result &= &"""Dict(keys=[{self.keys.join(", ")}], values=[{self.values.join(", ")}])"""
of lambdaExpr: of lambdaExpr:
var self = LambdaExpr(self) var self = LambdaExpr(self)
result &= &"""Lambda(body={self.body}, arguments=[{self.arguments.join(", ")}], defaults=[{self.defaults.join(", ")}], generator={self.isGenerator})""" result &= &"""Lambda(body={self.body}, type={self.returnType}, arguments=[{self.arguments.join(", ")}], defaults=[{self.defaults.join(", ")}], generator={self.isGenerator}, async={self.isAsync})"""
of deferStmt: of deferStmt:
var self = DeferStmt(self) var self = DeferStmt(self)
result &= &"Defer({self.deferred})" result &= &"Defer({self.expression})"
of sliceExpr: of sliceExpr:
var self = SliceExpr(self) var self = SliceExpr(self)
result &= &"""Slice({self.slicee}, ends=[{self.ends.join(", ")}])""" result &= &"""Slice({self.expression}, ends=[{self.ends.join(", ")}])"""
of tryStmt: of tryStmt:
var self = TryStmt(self) var self = TryStmt(self)
result &= &"TryStmt(body={self.body}, handlers={self.handlers}" result &= &"TryStmt(body={self.body}, handlers={self.handlers}"

2
src/frontend/meta/token.nim
View File

@ -33,7 +33,7 @@ type
While, For, While, For,
# Keywords # Keywords
Function, Break, Lambda, Continue, Function, Break, Continue,
Var, Let, Const, Is, Return, Var, Let, Const, Is, Return,
Coroutine, Generator, Import, Coroutine, Generator, Import,
IsNot, Raise, Assert, Await, IsNot, Raise, Assert, Await,

206
src/frontend/parser.nim
View File

@ -55,20 +55,10 @@ type
# Stores the current function # Stores the current function
# being parsed. This is a reference # being parsed. This is a reference
# to either a FunDecl or LambdaExpr # to either a FunDecl or LambdaExpr
# AST node and is mostly used to allow # AST node and is nil when the parser
# implicit generators to work. What that # is at the top-level. It allows the
# means is that there is no need for the # parser to detect errors like return
# programmer to specifiy a function is a # outside functions
# generator like in nim, (which uses the
# 'iterator' keyword): any function is
# automatically a generator if it contains
# any number of yield statement(s) or
# yield expression(s). This attribute
# is nil when the parser is at the top-level
# code and is what allows the parser to detect
# errors like return outside functions and attempts
# to declare public names inside them before
# compilation even begins
currentFunction: ASTNode currentFunction: ASTNode
# Stores the current scope depth (0 = global, > 0 local) # Stores the current scope depth (0 = global, > 0 local)
scopeDepth: int scopeDepth: int
@ -89,7 +79,6 @@ proc getCurrent*(self: Parser): int = self.current
proc getCurrentToken*(self: Parser): Token = (if self.getCurrent() >= self.tokens.len(): self.tokens[^1] else: self.tokens[self.current - 1]) proc getCurrentToken*(self: Parser): Token = (if self.getCurrent() >= self.tokens.len(): self.tokens[^1] else: self.tokens[self.current - 1])
# Handy templates to make our life easier, thanks nim! # Handy templates to make our life easier, thanks nim!
template endOfFile: Token = Token(kind: EndOfFile, lexeme: "", line: -1) template endOfFile: Token = Token(kind: EndOfFile, lexeme: "", line: -1)
template endOfLine(msg: string) = self.expect(Semicolon, msg) template endOfLine(msg: string) = self.expect(Semicolon, msg)
@ -145,7 +134,7 @@ proc check(self: Parser, kind: openarray[TokenType]): bool =
## Calls self.check() in a loop with each entry of ## Calls self.check() in a loop with each entry of
## the given openarray of token kinds and returns ## the given openarray of token kinds and returns
## at the first match. Note that this assumes ## at the first match. Note that this assumes
## that only one token may exist at a given ## that only one token may match at a given
## position ## position
for k in kind: for k in kind:
if self.check(k): if self.check(k):
@ -153,10 +142,10 @@ proc check(self: Parser, kind: openarray[TokenType]): bool =
return false return false
proc match(self: Parser, kind: TokenType, distance: int = 0): bool = proc match(self: Parser, kind: TokenType): bool =
## Behaves like self.check(), except that when a token ## Behaves like self.check(), except that when a token
## matches it is consumed ## matches it is also consumed
if self.check(kind, distance): if self.check(kind,):
discard self.step() discard self.step()
result = true result = true
else: else:
@ -189,18 +178,16 @@ proc expect(self: Parser, kind: TokenType, message: string = "") =
proc unnest(self: Parser, node: ASTNode): ASTNode = proc unnest(self: Parser, node: ASTNode): ASTNode =
## Unpacks an arbitrarily nested grouping expression ## Unpacks an arbitrarily nested grouping expression
var node = node
while node.kind == groupingExpr and GroupingExpr(node).expression != nil:
node = GroupingExpr(node).expression
result = node result = node
while result.kind == groupingExpr and GroupingExpr(result).expression != nil:
result = GroupingExpr(result).expression
# Forward declarations # Forward declarations
proc expression(self: Parser): ASTNode proc expression(self: Parser): ASTNode
proc expressionStatement(self: Parser): ASTNode proc expressionStatement(self: Parser): ASTNode
proc statement(self: Parser): ASTNode proc statement(self: Parser): ASTNode
proc varDecl(self: Parser, isStatic: bool = true, isPrivate: bool = true): ASTNode proc varDecl(self: Parser, isLet: bool = false, isConst: bool = false): ASTNode
proc funDecl(self: Parser, isAsync: bool = false, isStatic: bool = true, isPrivate: bool = true, isLambda: bool = false): ASTNode proc funDecl(self: Parser, isAsync: bool = false, isGenerator: bool = false, isLambda: bool = false): ASTNode
proc declaration(self: Parser): ASTNode proc declaration(self: Parser): ASTNode
@ -287,25 +274,22 @@ proc primary(self: Parser): ASTNode =
of Yield: of Yield:
let tok = self.step() let tok = self.step()
if self.currentFunction == nil: if self.currentFunction == nil:
self.error("'yield' cannot be outside functions") self.error("'yield' cannot be used outside functions")
if self.currentFunction.kind == NodeKind.funDecl: elif self.currentFunction.kind == lambdaExpr or not FunDecl(self.currentFunction).isGenerator:
FunDecl(self.currentFunction).isGenerator = true self.error("'yield' cannot be used outside generators")
else:
LambdaExpr(self.currentFunction).isGenerator = true
if not self.check([RightBrace, RightBracket, RightParen, Comma, Semicolon]): if not self.check([RightBrace, RightBracket, RightParen, Comma, Semicolon]):
# Expression delimiters
result = newYieldExpr(self.expression(), tok) result = newYieldExpr(self.expression(), tok)
else: else:
# Empty yield
result = newYieldExpr(newNilExpr(Token()), tok) result = newYieldExpr(newNilExpr(Token()), tok)
of Await: of Await:
let tok = self.step() let tok = self.step()
if self.currentFunction == nil: if self.currentFunction == nil:
self.error("'await' cannot be used outside functions") self.error("'await' cannot be used outside functions")
if self.currentFunction.kind == lambdaExpr or not FunDecl(self.currentFunction).isAsync: if self.currentFunction.kind == lambdaExpr or not FunDecl(self.currentFunction).isAsync:
self.error("'await' can only be used inside async functions") self.error("'await' can only be used inside coroutines")
result = newAwaitExpr(self.expression(), tok) result = newAwaitExpr(self.expression(), tok)
of Lambda:
discard self.step()
result = self.funDecl(isLambda=true)
of RightParen, RightBracket, RightBrace: of RightParen, RightBracket, RightBrace:
# This is *technically* unnecessary: the parser would # This is *technically* unnecessary: the parser would
# throw an error regardless, but it's a little bit nicer # throw an error regardless, but it's a little bit nicer
@ -321,6 +305,15 @@ proc primary(self: Parser): ASTNode =
result = newStrExpr(self.step()) result = newStrExpr(self.step())
of Infinity: of Infinity:
result = newInfExpr(self.step()) result = newInfExpr(self.step())
of Function:
discard self.step()
result = self.funDecl(isLambda=true)
of Coroutine:
discard self.step()
result = self.funDecl(isAsync=true, isLambda=true)
of Generator:
discard self.step()
result = self.funDecl(isGenerator=true, isLambda=true)
else: else:
self.error("invalid syntax") self.error("invalid syntax")
@ -730,7 +723,7 @@ proc forStmt(self: Parser): ASTNode =
if self.match(Semicolon): if self.match(Semicolon):
discard discard
elif self.match(Var): elif self.match(Var):
initializer = self.varDecl(isStatic=true, isPrivate=true) initializer = self.varDecl()
else: else:
initializer = self.expressionStatement() initializer = self.expressionStatement()
if not self.check(Semicolon): if not self.check(Semicolon):
@ -761,7 +754,7 @@ proc forStmt(self: Parser): ASTNode =
# To the semantically equivalent snippet # To the semantically equivalent snippet
# below: # below:
# { # {
# private static var i = 0; # var i = 0;
# while (i < 10) { # while (i < 10) {
# print(i); # print(i);
# i += 1; # i += 1;
@ -785,95 +778,128 @@ proc ifStmt(self: Parser): ASTNode =
result = newIfStmt(condition, thenBranch, elseBranch, tok) result = newIfStmt(condition, thenBranch, elseBranch, tok)
template checkDecl(self: Parser, isStatic, isPrivate: bool) = template checkDecl(self: Parser, isPrivate: bool) =
## Handy utility function that avoids us from copy ## Handy utility template that avoids us from copy
## pasting the same checks to all declaration handlers ## pasting the same checks to all declaration handlers
if not isStatic and self.currentFunction != nil:
self.error("dynamic declarations are not allowed inside functions")
if not isStatic and self.scopeDepth > 0:
self.error("dynamic declarations are not allowed inside local scopes")
if not isPrivate and self.currentFunction != nil: if not isPrivate and self.currentFunction != nil:
self.error("cannot bind public names inside functions") self.error("cannot bind public names inside functions")
if not isPrivate and self.scopeDepth > 0: if not isPrivate and self.scopeDepth > 0:
self.error("cannot bind public names inside local scopes") self.error("cannot bind public names inside local scopes")
proc varDecl(self: Parser, isStatic: bool = true, isPrivate: bool = true): ASTNode = proc varDecl(self: Parser, isLet: bool = false, isConst: bool = false): ASTNode =
## Parses variable declarations ## Parses variable declarations
self.checkDecl(isStatic, isPrivate) var tok = self.peek(-1)
var varKind = self.peek(-1)
var keyword = ""
var value: ASTNode var value: ASTNode
case varKind.kind: self.expect(Identifier, &"expecting identifier after '{tok.lexeme}'")
of Const:
# Note that isStatic being false is an error, because constants are replaced at compile-time
if not isStatic:
self.error("constant declarations cannot be dynamic")
keyword = "constant"
else:
keyword = "variable"
self.expect(Identifier, &"expecting {keyword} name after '{varKind.lexeme}'")
var name = newIdentExpr(self.peek(-1)) var name = newIdentExpr(self.peek(-1))
let isPrivate = not self.match(Star)
self.checkDecl(isPrivate)
var valueType: ASTNode
if self.match(Colon):
# We don't enforce it here because
# the compiler may be able to infer
# the type later!
self.expect(Identifier, "expecting type name after ':'")
valueType = newIdentExpr(self.peek(-1))
if self.match(Equal): if self.match(Equal):
value = self.expression() value = self.expression()
if varKind.kind == Const and not value.isConst(): if isConst and not value.isConst():
self.error("the initializer for constant declarations must be a primitive and constant type") self.error("constant initializer is not a constant")
else: else:
if varKind.kind == Const: if tok.kind != Var:
self.error("constant declaration requires an explicit initializer") self.error(&"{tok.lexeme} declaration requires an initializer")
value = newNilExpr(Token()) value = newNilExpr(Token())
self.expect(Semicolon, &"expecting semicolon after {keyword} declaration") self.expect(Semicolon, &"expecting semicolon after declaration")
case varKind.kind: case tok.kind:
of Var: of Var:
result = newVarDecl(name, value, isStatic=isStatic, isPrivate=isPrivate, token=varKind, owner=self.file, closedOver=false) result = newVarDecl(name, value, isPrivate=isPrivate, token=tok, closedOver=false, valueType=valueType)
of Const: of Const:
result = newVarDecl(name, value, isConst=true, isPrivate=isPrivate, isStatic=true, token=varKind, owner=self.file, closedOver=false) result = newVarDecl(name, value, isPrivate=isPrivate, token=tok, isConst=true, closedOver=false, valueType=valueType)
of Let:
result = newVarDecl(name, value, isPrivate=isPrivate, token=tok, isLet=isLet, closedOver=false, valueType=valueType)
else: else:
discard # Unreachable discard # Unreachable
proc funDecl(self: Parser, isAsync: bool = false, isStatic: bool = true, isPrivate: bool = true, isLambda: bool = false): ASTNode = proc funDecl(self: Parser, isAsync: bool = false, isGenerator: bool = false, isLambda: bool = false): ASTNode =
## Parses function and lambda declarations. Note that lambdas count as expressions! ## Parses functions, coroutines, generators and anonymous functions declarations (the latter is
self.checkDecl(isStatic, isPrivate) ## technically an expression, not a declaration)
let tok = self.peek(-1) let tok = self.peek(-1)
var enclosingFunction = self.currentFunction var enclosingFunction = self.currentFunction
var arguments: seq[ASTNode] = @[] var arguments: seq[tuple[name: ASTNode, valueType: ASTNode]] = @[]
var defaults: seq[ASTNode] = @[] var defaults: seq[ASTNode] = @[]
if not isLambda: var returnType: ASTNode
self.currentFunction = newFunDecl(nil, arguments, defaults, newBlockStmt(@[], Token()), isAsync=isAsync, isGenerator=false, isStatic=isStatic, isPrivate=isPrivate, token=tok, owner=self.file, closedOver=false) if not isLambda and self.check(Identifier):
else: # We do this extra check because we might
self.currentFunction = newLambdaExpr(arguments, defaults, newBlockStmt(@[], Token()), isGenerator=false, token=tok) # be called from a contexst where it's
if not isLambda: # ambiguous whether we're parsing a declaration
self.expect(Identifier, "expecting function name after 'fun'") # or an expression. Fortunately anonymous functions
# are nameless, so we can sort the ambiguity by checking
# if there's an identifier after the keyword
self.expect(Identifier, &"expecting function name after '{tok.lexeme}'")
self.checkDecl(not self.check(Star))
self.currentFunction = newFunDecl(nil, arguments, defaults, newBlockStmt(@[], Token()),
isAsync=isAsync, isGenerator=isGenerator, isPrivate=true,
token=tok, closedOver=false)
FunDecl(self.currentFunction).name = newIdentExpr(self.peek(-1)) FunDecl(self.currentFunction).name = newIdentExpr(self.peek(-1))
if self.match(LeftBrace): if self.match(Star):
# Argument-less function FunDecl(self.currentFunction).isPrivate = false
discard elif not isLambda and self.check([LeftBrace, Colon]):
# We do a bit of hacking to pretend we never
# wanted to parse this as a declaration in
# the first place and pass control over to
# expressionStatement(), which will in turn
# go all the way up to primary(), which will
# call us back with isLambda=true, allowing us
# to actually parse the function as an expression
dec(self.current)
result = self.expressionStatement()
self.currentFunction = enclosingFunction
return result
elif isLambda:
self.currentFunction = newLambdaExpr(arguments, defaults, newBlockStmt(@[], Token()), isGenerator=isGenerator, isAsync=isAsync, token=tok)
else: else:
var parameter: IdentExpr self.error("funDecl: invalid state")
if self.match(Colon):
# A function without an explicit
# return type is the same as a void
# function in C (i.e. no return type)
self.expect(Identifier, "expecting function return type after ':'")
returnType = newIdentExpr(self.peek(-1))
if not self.match(LeftBrace):
# Argument-less function
var parameter: tuple[name: ASTNode, valueType: ASTNode]
self.expect(LeftParen) self.expect(LeftParen)
while not self.check(RightParen): while not self.check(RightParen):
if arguments.len > 255: if arguments.len > 255:
self.error("cannot have more than 255 arguments in function declaration") self.error("cannot have more than 255 arguments in function declaration")
self.expect(Identifier) self.expect(Identifier, "expecting parameter name")
parameter = newIdentExpr(self.peek(-1)) parameter.name = newIdentExpr(self.peek(-1))
self.expect(Identifier, "expecting parameter type")
parameter.valueType = newIdentExpr(self.peek(-1))
if parameter in arguments: if parameter in arguments:
self.error("duplicate parameter name in function declaration") self.error("duplicate parameter name in function declaration")
arguments.add(parameter) arguments.add(parameter)
if self.match(Equal): if self.match(Equal):
defaults.add(self.expression()) defaults.add(self.expression())
elif defaults.len() > 0: elif defaults.len() > 0:
self.error("positional argument(s) cannot follow default argument(s) in function declaration") self.error("positional argument cannot follow default argument in function declaration")
if not self.match(Comma): if not self.match(Comma):
break break
self.expect(RightParen) self.expect(RightParen)
self.expect(LeftBrace) self.expect(LeftBrace)
if not isLambda: if self.currentFunction.kind == funDecl:
FunDecl(self.currentFunction).body = self.blockStmt() if not self.match(Semicolon):
# Forward declaration!
FunDecl(self.currentFunction).body = self.blockStmt()
FunDecl(self.currentFunction).arguments = arguments FunDecl(self.currentFunction).arguments = arguments
FunDecl(self.currentFunction).returnType = returnType
else: else:
LambdaExpr(self.currentFunction).body = self.blockStmt() if not self.match(Semicolon):
LambdaExpr(self.currentFunction).body = self.blockStmt()
LambdaExpr(self.currentFunction).arguments = arguments LambdaExpr(self.currentFunction).arguments = arguments
LambdaExpr(self.currentFunction).returnType = returnType
result = self.currentFunction result = self.currentFunction
self.currentFunction = enclosingFunction self.currentFunction = enclosingFunction
@ -947,13 +973,19 @@ proc declaration(self: Parser): ASTNode =
## Parses declarations ## Parses declarations
case self.peek().kind: case self.peek().kind:
of Var, Const, Let: of Var, Const, Let:
discard self.step() let keyword = self.step()
result = self.varDecl() result = self.varDecl(isLet=keyword.kind == Let, isConst=keyword.kind == Const)
of Function: of Function:
discard self.step() discard self.step()
result = self.funDecl() result = self.funDecl()
of Type: of Coroutine:
discard # TODO discard self.step()
result = self.funDecl(isAsync=true)
of Generator:
discard self.step()
result = self.funDecl(isGenerator=true)
of Type, Comment, Whitespace, Tab:
discard self.step() # TODO
else: else:
result = self.statement() result = self.statement()

4
src/test.nim
View File

@ -40,6 +40,7 @@ when isMainModule:
try: try:
input = lineEditor.read() input = lineEditor.read()
if input.len() > 0: if input.len() > 0:
# Currently the parser doesn't handle these tokens well
tokens = filter(tokenizer.lex(input, "<stdin>"), proc (x: Token): bool = x.kind notin {Whitespace, Tab}) tokens = filter(tokenizer.lex(input, "<stdin>"), proc (x: Token): bool = x.kind notin {Whitespace, Tab})
for i, token in tokens: for i, token in tokens:
if i == tokens.high(): if i == tokens.high():
@ -134,8 +135,9 @@ proc fillSymbolTable(tokenizer: Lexer) =
tokenizer.symbols.addKeyword("const", Const) tokenizer.symbols.addKeyword("const", Const)
tokenizer.symbols.addKeyword("let", Let) tokenizer.symbols.addKeyword("let", Let)
tokenizer.symbols.addKeyword("var", Var) tokenizer.symbols.addKeyword("var", Var)
tokenizer.symbols.addKeyword("lambda", Lambda)
tokenizer.symbols.addKeyword("import", Import) tokenizer.symbols.addKeyword("import", Import)
tokenizer.symbols.addKeyword("yield", Yield)
tokenizer.symbols.addKeyword("return", Return)
# These are technically more like expressions # These are technically more like expressions
# with a reserved name that produce a value of a # with a reserved name that produce a value of a
# builtin type, but we don't need to care about # builtin type, but we don't need to care about