From dbab30952b5342bed58300c1a69837b714c7e55f Mon Sep 17 00:00:00 2001 From: Mattia Giambirtone Date: Tue, 5 Apr 2022 15:52:58 +0200 Subject: [PATCH] Initial work on adding type support to the parser for declarations + fixes for lambdas --- src/frontend/meta/ast.nim | 200 ++++++++++++++-------------------- src/frontend/meta/token.nim | 2 +- src/frontend/parser.nim | 206 +++++++++++++++++++++--------------- src/test.nim | 4 +- 4 files changed, 201 insertions(+), 211 deletions(-) diff --git a/src/frontend/meta/ast.nim b/src/frontend/meta/ast.nim index 2f85e52..ed87f3f 100644 --- a/src/frontend/meta/ast.nim +++ b/src/frontend/meta/ast.nim @@ -18,10 +18,11 @@ import strformat import strutils +import sequtils import token - +export token type NodeKind* = enum @@ -30,8 +31,7 @@ type ## precedence # Declarations - classDecl = 0u8, - funDecl, + funDecl = 0'u8, varDecl, # Statements forStmt, # Unused for now (for loops are compiled to while loops) @@ -44,11 +44,9 @@ type blockStmt, raiseStmt, assertStmt, - delStmt, tryStmt, yieldStmt, awaitStmt, - fromImportStmt, importStmt, deferStmt, # An expression followed by a semicolon @@ -64,8 +62,8 @@ type sliceExpr, callExpr, getItemExpr, # Get expressions like a.b - # Primary expressions - groupingExpr, # Parenthesized expressions such as (true) and (3 + 4) + # Primary expressions + groupingExpr, # Parenthesized expressions such as (true) and (3 + 4) trueExpr, listExpr, tupleExpr, @@ -95,7 +93,7 @@ type # Here I would've rather used object variants, and in fact that's what was in # place before, but not being able to re-declare a field of the same type in # another case branch is kind of a deal breaker long-term, so until that is - # fixed (check out https://github.com/nim-lang/RFCs/issues/368 for more info) + # fixed (check out https://github.com/nim-lang/RFCs/issues/368 for more info). # I'll stick to using inheritance instead LiteralExpr* = ref object of ASTNode @@ -129,6 +127,7 @@ type # a tough luck for us ListExpr* = ref object of ASTNode members*: seq[ASTNode] + valueType*: IdentExpr SetExpr* = ref object of ListExpr @@ -137,6 +136,8 @@ type DictExpr* = ref object of ASTNode keys*: seq[ASTNode] values*: seq[ASTNode] + keyType*: IdentExpr + valueType*: IdentExpr IdentExpr* = ref object of ASTNode name*: Token @@ -173,21 +174,18 @@ type expression*: ASTNode AwaitExpr* = ref object of ASTNode - awaitee*: ASTNode + expression*: ASTNode LambdaExpr* = ref object of ASTNode body*: ASTNode - arguments*: seq[ASTNode] - # This is, in order, the list of each default argument - # the function takes. It maps 1:1 with self.arguments - # although it may be shorter (in which case this maps - # 1:1 with what's left of self.arguments after all - # positional arguments have been consumed) + arguments*: seq[tuple[name: ASTNode, valueType: ASTNode]] defaults*: seq[ASTNode] isGenerator*: bool + isAsync*: bool + returnType*: ASTNode SliceExpr* = ref object of ASTNode - slicee*: ASTNode + expression*: ASTNode ends*: seq[ASTNode] AssignExpr* = ref object of ASTNode @@ -200,13 +198,6 @@ type ImportStmt* = ref object of ASTNode moduleName*: ASTNode - FromImportStmt* = ref object of ASTNode - fromModule*: ASTNode - fromAttributes*: seq[ASTNode] - - DelStmt* = ref object of ASTNode - name*: ASTNode - AssertStmt* = ref object of ASTNode expression*: ASTNode @@ -225,7 +216,7 @@ type body*: ASTNode DeferStmt* = ref object of ASTNode - deferred*: ASTNode + expression*: ASTNode TryStmt* = ref object of ASTNode body*: ASTNode @@ -238,7 +229,7 @@ type body*: ASTNode AwaitStmt* = ref object of ASTNode - awaitee*: ASTNode + expression*: ASTNode BreakStmt* = ref object of ASTNode @@ -256,7 +247,6 @@ type expression*: ASTNode Declaration* = ref object of ASTNode - owner*: string # Used for determining if a module can access a given field closedOver*: bool VarDecl* = ref object of Declaration @@ -265,36 +255,25 @@ type isConst*: bool isStatic*: bool isPrivate*: bool + isLet*: bool + valueType*: ASTNode FunDecl* = ref object of Declaration name*: ASTNode body*: ASTNode - arguments*: seq[ASTNode] - # This is, in order, the list of each default argument - # the function takes. It maps 1:1 with self.arguments - # although it may be shorter (in which case this maps - # 1:1 with what's left of self.arguments after all - # positional arguments have been consumed) - defaults*: seq[ASTNode] + arguments*: seq[tuple[name: ASTNode, valueType: ASTNode]] + defaults: seq[ASTNode] isAsync*: bool isGenerator*: bool - isStatic*: bool isPrivate*: bool + returnType*: ASTNode - ClassDecl* = ref object of Declaration - name*: ASTNode - body*: ASTNode - parents*: seq[ASTNode] - isStatic*: bool - isPrivate*: bool + Expression* = LiteralExpr | ListExpr | GetItemExpr | SetItemExpr | UnaryExpr | BinaryExpr | CallExpr | AssignExpr | + GroupingExpr | IdentExpr | DictExpr | TupleExpr | SetExpr | TrueExpr | FalseExpr | NilExpr | + NanExpr | InfExpr - Expression* = LiteralExpr | ListExpr | GetItemExpr | SetItemExpr | UnaryExpr | BinaryExpr | CallExpr | AssignExpr | - GroupingExpr | IdentExpr | DictExpr | TupleExpr | SetExpr | - TrueExpr | FalseExpr | NilExpr | - NanExpr | InfExpr - - Statement* = ExprStmt | ImportStmt | FromImportStmt | DelStmt | AssertStmt | RaiseStmt | BlockStmt | ForStmt | WhileStmt | - ForStmt | BreakStmt | ContinueStmt | ReturnStmt | IfStmt + Statement* = ExprStmt | ImportStmt | AssertStmt | RaiseStmt | BlockStmt | ForStmt | WhileStmt | + ForStmt | BreakStmt | ContinueStmt | ReturnStmt | IfStmt @@ -306,15 +285,32 @@ proc newASTNode*(kind: NodeKind, token: Token): ASTNode = result.token = token -proc isConst*(self: ASTNode): bool {.inline.} = self.kind in {intExpr, hexExpr, binExpr, octExpr, strExpr, - falseExpr, - trueExpr, infExpr, - nanExpr, - floatExpr, nilExpr} +proc isConst*(self: ASTNode): bool = + ## Returns true if the given + ## AST node represents a value + ## of constant type. All integers, + ## strings and singletons count as + ## constants, as well as collections + ## comprised only of those types. + case self.kind: + of intExpr, hexExpr, binExpr, octExpr, strExpr, falseExpr, trueExpr, infExpr, nanExpr, floatExpr, nilExpr: + return true + of tupleExpr, setExpr, listExpr: + for item in ListExpr(self).members: + if not item.isConst(): + return false + return true + of dictExpr: + for (key, value) in zip(DictExpr(self).keys, DictExpr(self).values): + if not key.isConst() or not value.isConst(): + return false + return true + else: + return false + - -proc isLiteral*(self: ASTNode): bool {.inline.} = self.isConst() or self.kind in - {tupleExpr, dictExpr, setExpr, listExpr} +# TODO: Fix (not all literals are constants) +proc isLiteral*(self: ASTNode): bool {.inline.} = self.isConst() proc newIntExpr*(literal: Token): IntExpr = @@ -372,13 +368,14 @@ proc newGroupingExpr*(expression: ASTNode, token: Token): GroupingExpr = result.token = token -proc newLambdaExpr*(arguments, defaults: seq[ASTNode], body: ASTNode, - isGenerator: bool, token: Token): LambdaExpr = +proc newLambdaExpr*(arguments: seq[tuple[name: ASTNode, valueType: ASTNode]], defaults: seq[ASTNode], body: ASTNode, + isGenerator: bool, isAsync: bool, token: Token): LambdaExpr = result = LambdaExpr(kind: lambdaExpr) result.body = body result.arguments = arguments result.defaults = defaults result.isGenerator = isGenerator + result.isAsync = isAsync result.token = token @@ -431,10 +428,10 @@ proc newCallExpr*(callee: ASTNode, arguments: tuple[positionals: seq[ASTNode], result.token = token -proc newSliceExpr*(slicee: ASTNode, ends: seq[ASTNode], +proc newSliceExpr*(expression: ASTNode, ends: seq[ASTNode], token: Token): SliceExpr = result = SliceExpr(kind: sliceExpr) - result.slicee = slicee + result.expression = expression result.ends = ends result.token = token @@ -467,9 +464,9 @@ proc newAssignExpr*(name, value: ASTNode, token: Token): AssignExpr = result.token = token -proc newAwaitExpr*(awaitee: ASTNode, token: Token): AwaitExpr = +proc newAwaitExpr*(expression: ASTNode, token: Token): AwaitExpr = result = AwaitExpr(kind: awaitExpr) - result.awaitee = awaitee + result.expression = expression result.token = token @@ -485,29 +482,15 @@ proc newImportStmt*(moduleName: ASTNode, token: Token): ImportStmt = result.token = token -proc newFromImportStmt*(fromModule: ASTNode, fromAttributes: seq[ASTNode], - token: Token): FromImportStmt = - result = FromImportStmt(kind: fromImportStmt) - result.fromModule = fromModule - result.fromAttributes = fromAttributes - result.token = token - - -proc newDelStmt*(name: ASTNode, token: Token): DelStmt = - result = DelStmt(kind: delStmt) - result.name = name - result.token = token - - proc newYieldStmt*(expression: ASTNode, token: Token): YieldStmt = result = YieldStmt(kind: yieldStmt) result.expression = expression result.token = token -proc newAwaitStmt*(awaitee: ASTNode, token: Token): AwaitExpr = +proc newAwaitStmt*(expression: ASTNode, token: Token): AwaitExpr = result = AwaitExpr(kind: awaitExpr) - result.awaitee = awaitee + result.expression = expression result.token = token @@ -517,9 +500,9 @@ proc newAssertStmt*(expression: ASTNode, token: Token): AssertStmt = result.token = token -proc newDeferStmt*(deferred: ASTNode, token: Token): DeferStmt = +proc newDeferStmt*(expression: ASTNode, token: Token): DeferStmt = result = DeferStmt(kind: deferStmt) - result.deferred = deferred + result.expression = expression result.token = token @@ -587,24 +570,22 @@ proc newIfStmt*(condition: ASTNode, thenBranch, elseBranch: ASTNode, result.token = token -proc newVarDecl*(name: ASTNode, value: ASTNode = newNilExpr(Token()), - isStatic: bool = true, isConst: bool = false, - isPrivate: bool = true, token: Token, owner: string, - closedOver: bool): VarDecl = +proc newVarDecl*(name: ASTNode, value: ASTNode, isConst: bool = false, + isPrivate: bool = true, token: Token, closedOver: bool, + isLet: bool = false, valueType: ASTNode): VarDecl = result = VarDecl(kind: varDecl) result.name = name result.value = value result.isConst = isConst - result.isStatic = isStatic result.isPrivate = isPrivate result.token = token - result.owner = owner + result.isLet = isLet + result.valueType = valueType -proc newFunDecl*(name: ASTNode, arguments, defaults: seq[ASTNode], - body: ASTNode, isStatic: bool = true, isAsync, - isGenerator: bool, isPrivate: bool = true, token: Token, - owner: string, closedOver: bool): FunDecl = +proc newFunDecl*(name: ASTNode, arguments: seq[tuple[name: ASTNode, valueType: ASTNode]], defaults: seq[ASTNode], + body: ASTNode, isAsync, isGenerator: bool, + isPrivate: bool, token: Token, closedOver: bool): FunDecl = result = FunDecl(kind: funDecl) result.name = name result.arguments = arguments @@ -612,34 +593,18 @@ proc newFunDecl*(name: ASTNode, arguments, defaults: seq[ASTNode], result.body = body result.isAsync = isAsync result.isGenerator = isGenerator - result.isStatic = isStatic result.isPrivate = isPrivate result.token = token - result.owner = owner result.closedOver = closedOver -proc newClassDecl*(name: ASTNode, body: ASTNode, - parents: seq[ASTNode], isStatic: bool = true, - isPrivate: bool = true, token: Token, - owner: string, closedOver: bool): ClassDecl = - result = ClassDecl(kind: classDecl) - result.name = name - result.body = body - result.parents = parents - result.isStatic = isStatic - result.isPrivate = isPrivate - result.token = token - result.owner = owner - result.closedOver = closedOver - proc `$`*(self: ASTNode): string = if self == nil: return "nil" case self.kind: of intExpr, floatExpr, hexExpr, binExpr, octExpr, strExpr, trueExpr, - falseExpr, nanExpr, nilExpr, infExpr: + falseExpr, nanExpr, nilExpr, infExpr: if self.kind in {trueExpr, falseExpr, nanExpr, nilExpr, infExpr}: result &= &"Literal({($self.kind)[0..^5]})" elif self.kind == strExpr: @@ -676,12 +641,6 @@ proc `$`*(self: ASTNode): string = of importStmt: var self = ImportStmt(self) result &= &"Import({self.moduleName})" - of fromImportStmt: - var self = FromImportStmt(self) - result &= &"""FromImport(fromModule={self.fromModule}, fromAttributes=[{self.fromAttributes.join(", ")}])""" - of delStmt: - var self = DelStmt(self) - result &= &"Del({self.name})" of assertStmt: var self = AssertStmt(self) result &= &"Assert({self.expression})" @@ -705,7 +664,7 @@ proc `$`*(self: ASTNode): string = result &= &"Yield({self.expression})" of awaitExpr: var self = AwaitExpr(self) - result &= &"Await({self.awaitee})" + result &= &"Await({self.expression})" of ifStmt: var self = IfStmt(self) if self.elseBranch == nil: @@ -717,16 +676,13 @@ proc `$`*(self: ASTNode): string = result &= &"YieldStmt({self.expression})" of awaitStmt: var self = AwaitStmt(self) - result &= &"AwaitStmt({self.awaitee})" + result &= &"AwaitStmt({self.expression})" of varDecl: var self = VarDecl(self) - result &= &"Var(name={self.name}, value={self.value}, const={self.isConst}, static={self.isStatic}, private={self.isPrivate})" + result &= &"Var(name={self.name}, value={self.value}, const={self.isConst}, private={self.isPrivate}, type={self.valueType})" of funDecl: var self = FunDecl(self) - result &= &"""FunDecl(name={self.name}, body={self.body}, arguments=[{self.arguments.join(", ")}], defaults=[{self.defaults.join(", ")}], async={self.isAsync}, generator={self.isGenerator}, static={self.isStatic}, private={self.isPrivate})""" - of classDecl: - var self = ClassDecl(self) - result &= &"""Class(name={self.name}, body={self.body}, parents=[{self.parents.join(", ")}], static={self.isStatic}, private={self.isPrivate})""" + result &= &"""FunDecl(name={self.name}, body={self.body}, type={self.returnType}, arguments=[{self.arguments.join(", ")}], defaults=[{self.defaults.join(", ")}], async={self.isAsync}, generator={self.isGenerator}, private={self.isPrivate})""" of tupleExpr: var self = TupleExpr(self) result &= &"""Tuple([{self.members.join(", ")}])""" @@ -741,13 +697,13 @@ proc `$`*(self: ASTNode): string = result &= &"""Dict(keys=[{self.keys.join(", ")}], values=[{self.values.join(", ")}])""" of lambdaExpr: var self = LambdaExpr(self) - result &= &"""Lambda(body={self.body}, arguments=[{self.arguments.join(", ")}], defaults=[{self.defaults.join(", ")}], generator={self.isGenerator})""" + result &= &"""Lambda(body={self.body}, type={self.returnType}, arguments=[{self.arguments.join(", ")}], defaults=[{self.defaults.join(", ")}], generator={self.isGenerator}, async={self.isAsync})""" of deferStmt: var self = DeferStmt(self) - result &= &"Defer({self.deferred})" + result &= &"Defer({self.expression})" of sliceExpr: var self = SliceExpr(self) - result &= &"""Slice({self.slicee}, ends=[{self.ends.join(", ")}])""" + result &= &"""Slice({self.expression}, ends=[{self.ends.join(", ")}])""" of tryStmt: var self = TryStmt(self) result &= &"TryStmt(body={self.body}, handlers={self.handlers}" diff --git a/src/frontend/meta/token.nim b/src/frontend/meta/token.nim index 65e6e2c..911d9a1 100644 --- a/src/frontend/meta/token.nim +++ b/src/frontend/meta/token.nim @@ -33,7 +33,7 @@ type While, For, # Keywords - Function, Break, Lambda, Continue, + Function, Break, Continue, Var, Let, Const, Is, Return, Coroutine, Generator, Import, IsNot, Raise, Assert, Await, diff --git a/src/frontend/parser.nim b/src/frontend/parser.nim index 112f216..3be46d9 100644 --- a/src/frontend/parser.nim +++ b/src/frontend/parser.nim @@ -55,20 +55,10 @@ type # Stores the current function # being parsed. This is a reference # to either a FunDecl or LambdaExpr - # AST node and is mostly used to allow - # implicit generators to work. What that - # means is that there is no need for the - # programmer to specifiy a function is a - # generator like in nim, (which uses the - # 'iterator' keyword): any function is - # automatically a generator if it contains - # any number of yield statement(s) or - # yield expression(s). This attribute - # is nil when the parser is at the top-level - # code and is what allows the parser to detect - # errors like return outside functions and attempts - # to declare public names inside them before - # compilation even begins + # AST node and is nil when the parser + # is at the top-level. It allows the + # parser to detect errors like return + # outside functions currentFunction: ASTNode # Stores the current scope depth (0 = global, > 0 local) scopeDepth: int @@ -89,7 +79,6 @@ proc getCurrent*(self: Parser): int = self.current proc getCurrentToken*(self: Parser): Token = (if self.getCurrent() >= self.tokens.len(): self.tokens[^1] else: self.tokens[self.current - 1]) # Handy templates to make our life easier, thanks nim! - template endOfFile: Token = Token(kind: EndOfFile, lexeme: "", line: -1) template endOfLine(msg: string) = self.expect(Semicolon, msg) @@ -145,7 +134,7 @@ proc check(self: Parser, kind: openarray[TokenType]): bool = ## Calls self.check() in a loop with each entry of ## the given openarray of token kinds and returns ## at the first match. Note that this assumes - ## that only one token may exist at a given + ## that only one token may match at a given ## position for k in kind: if self.check(k): @@ -153,10 +142,10 @@ proc check(self: Parser, kind: openarray[TokenType]): bool = return false -proc match(self: Parser, kind: TokenType, distance: int = 0): bool = +proc match(self: Parser, kind: TokenType): bool = ## Behaves like self.check(), except that when a token - ## matches it is consumed - if self.check(kind, distance): + ## matches it is also consumed + if self.check(kind,): discard self.step() result = true else: @@ -189,18 +178,16 @@ proc expect(self: Parser, kind: TokenType, message: string = "") = proc unnest(self: Parser, node: ASTNode): ASTNode = ## Unpacks an arbitrarily nested grouping expression - var node = node - while node.kind == groupingExpr and GroupingExpr(node).expression != nil: - node = GroupingExpr(node).expression result = node - + while result.kind == groupingExpr and GroupingExpr(result).expression != nil: + result = GroupingExpr(result).expression # Forward declarations proc expression(self: Parser): ASTNode proc expressionStatement(self: Parser): ASTNode proc statement(self: Parser): ASTNode -proc varDecl(self: Parser, isStatic: bool = true, isPrivate: bool = true): ASTNode -proc funDecl(self: Parser, isAsync: bool = false, isStatic: bool = true, isPrivate: bool = true, isLambda: bool = false): ASTNode +proc varDecl(self: Parser, isLet: bool = false, isConst: bool = false): ASTNode +proc funDecl(self: Parser, isAsync: bool = false, isGenerator: bool = false, isLambda: bool = false): ASTNode proc declaration(self: Parser): ASTNode @@ -287,25 +274,22 @@ proc primary(self: Parser): ASTNode = of Yield: let tok = self.step() if self.currentFunction == nil: - self.error("'yield' cannot be outside functions") - if self.currentFunction.kind == NodeKind.funDecl: - FunDecl(self.currentFunction).isGenerator = true - else: - LambdaExpr(self.currentFunction).isGenerator = true + self.error("'yield' cannot be used outside functions") + elif self.currentFunction.kind == lambdaExpr or not FunDecl(self.currentFunction).isGenerator: + self.error("'yield' cannot be used outside generators") if not self.check([RightBrace, RightBracket, RightParen, Comma, Semicolon]): + # Expression delimiters result = newYieldExpr(self.expression(), tok) else: + # Empty yield result = newYieldExpr(newNilExpr(Token()), tok) of Await: let tok = self.step() if self.currentFunction == nil: self.error("'await' cannot be used outside functions") if self.currentFunction.kind == lambdaExpr or not FunDecl(self.currentFunction).isAsync: - self.error("'await' can only be used inside async functions") + self.error("'await' can only be used inside coroutines") result = newAwaitExpr(self.expression(), tok) - of Lambda: - discard self.step() - result = self.funDecl(isLambda=true) of RightParen, RightBracket, RightBrace: # This is *technically* unnecessary: the parser would # throw an error regardless, but it's a little bit nicer @@ -321,6 +305,15 @@ proc primary(self: Parser): ASTNode = result = newStrExpr(self.step()) of Infinity: result = newInfExpr(self.step()) + of Function: + discard self.step() + result = self.funDecl(isLambda=true) + of Coroutine: + discard self.step() + result = self.funDecl(isAsync=true, isLambda=true) + of Generator: + discard self.step() + result = self.funDecl(isGenerator=true, isLambda=true) else: self.error("invalid syntax") @@ -730,7 +723,7 @@ proc forStmt(self: Parser): ASTNode = if self.match(Semicolon): discard elif self.match(Var): - initializer = self.varDecl(isStatic=true, isPrivate=true) + initializer = self.varDecl() else: initializer = self.expressionStatement() if not self.check(Semicolon): @@ -761,7 +754,7 @@ proc forStmt(self: Parser): ASTNode = # To the semantically equivalent snippet # below: # { - # private static var i = 0; + # var i = 0; # while (i < 10) { # print(i); # i += 1; @@ -785,95 +778,128 @@ proc ifStmt(self: Parser): ASTNode = result = newIfStmt(condition, thenBranch, elseBranch, tok) -template checkDecl(self: Parser, isStatic, isPrivate: bool) = - ## Handy utility function that avoids us from copy +template checkDecl(self: Parser, isPrivate: bool) = + ## Handy utility template that avoids us from copy ## pasting the same checks to all declaration handlers - if not isStatic and self.currentFunction != nil: - self.error("dynamic declarations are not allowed inside functions") - if not isStatic and self.scopeDepth > 0: - self.error("dynamic declarations are not allowed inside local scopes") if not isPrivate and self.currentFunction != nil: self.error("cannot bind public names inside functions") if not isPrivate and self.scopeDepth > 0: self.error("cannot bind public names inside local scopes") -proc varDecl(self: Parser, isStatic: bool = true, isPrivate: bool = true): ASTNode = +proc varDecl(self: Parser, isLet: bool = false, isConst: bool = false): ASTNode = ## Parses variable declarations - self.checkDecl(isStatic, isPrivate) - var varKind = self.peek(-1) - var keyword = "" + var tok = self.peek(-1) var value: ASTNode - case varKind.kind: - of Const: - # Note that isStatic being false is an error, because constants are replaced at compile-time - if not isStatic: - self.error("constant declarations cannot be dynamic") - keyword = "constant" - else: - keyword = "variable" - self.expect(Identifier, &"expecting {keyword} name after '{varKind.lexeme}'") + self.expect(Identifier, &"expecting identifier after '{tok.lexeme}'") var name = newIdentExpr(self.peek(-1)) + let isPrivate = not self.match(Star) + self.checkDecl(isPrivate) + var valueType: ASTNode + if self.match(Colon): + # We don't enforce it here because + # the compiler may be able to infer + # the type later! + self.expect(Identifier, "expecting type name after ':'") + valueType = newIdentExpr(self.peek(-1)) if self.match(Equal): value = self.expression() - if varKind.kind == Const and not value.isConst(): - self.error("the initializer for constant declarations must be a primitive and constant type") + if isConst and not value.isConst(): + self.error("constant initializer is not a constant") else: - if varKind.kind == Const: - self.error("constant declaration requires an explicit initializer") + if tok.kind != Var: + self.error(&"{tok.lexeme} declaration requires an initializer") value = newNilExpr(Token()) - self.expect(Semicolon, &"expecting semicolon after {keyword} declaration") - case varKind.kind: + self.expect(Semicolon, &"expecting semicolon after declaration") + case tok.kind: of Var: - result = newVarDecl(name, value, isStatic=isStatic, isPrivate=isPrivate, token=varKind, owner=self.file, closedOver=false) + result = newVarDecl(name, value, isPrivate=isPrivate, token=tok, closedOver=false, valueType=valueType) of Const: - result = newVarDecl(name, value, isConst=true, isPrivate=isPrivate, isStatic=true, token=varKind, owner=self.file, closedOver=false) + result = newVarDecl(name, value, isPrivate=isPrivate, token=tok, isConst=true, closedOver=false, valueType=valueType) + of Let: + result = newVarDecl(name, value, isPrivate=isPrivate, token=tok, isLet=isLet, closedOver=false, valueType=valueType) else: discard # Unreachable -proc funDecl(self: Parser, isAsync: bool = false, isStatic: bool = true, isPrivate: bool = true, isLambda: bool = false): ASTNode = - ## Parses function and lambda declarations. Note that lambdas count as expressions! - self.checkDecl(isStatic, isPrivate) +proc funDecl(self: Parser, isAsync: bool = false, isGenerator: bool = false, isLambda: bool = false): ASTNode = + ## Parses functions, coroutines, generators and anonymous functions declarations (the latter is + ## technically an expression, not a declaration) let tok = self.peek(-1) var enclosingFunction = self.currentFunction - var arguments: seq[ASTNode] = @[] + var arguments: seq[tuple[name: ASTNode, valueType: ASTNode]] = @[] var defaults: seq[ASTNode] = @[] - if not isLambda: - self.currentFunction = newFunDecl(nil, arguments, defaults, newBlockStmt(@[], Token()), isAsync=isAsync, isGenerator=false, isStatic=isStatic, isPrivate=isPrivate, token=tok, owner=self.file, closedOver=false) - else: - self.currentFunction = newLambdaExpr(arguments, defaults, newBlockStmt(@[], Token()), isGenerator=false, token=tok) - if not isLambda: - self.expect(Identifier, "expecting function name after 'fun'") + var returnType: ASTNode + if not isLambda and self.check(Identifier): + # We do this extra check because we might + # be called from a contexst where it's + # ambiguous whether we're parsing a declaration + # or an expression. Fortunately anonymous functions + # are nameless, so we can sort the ambiguity by checking + # if there's an identifier after the keyword + self.expect(Identifier, &"expecting function name after '{tok.lexeme}'") + self.checkDecl(not self.check(Star)) + self.currentFunction = newFunDecl(nil, arguments, defaults, newBlockStmt(@[], Token()), + isAsync=isAsync, isGenerator=isGenerator, isPrivate=true, + token=tok, closedOver=false) FunDecl(self.currentFunction).name = newIdentExpr(self.peek(-1)) - if self.match(LeftBrace): - # Argument-less function - discard + if self.match(Star): + FunDecl(self.currentFunction).isPrivate = false + elif not isLambda and self.check([LeftBrace, Colon]): + # We do a bit of hacking to pretend we never + # wanted to parse this as a declaration in + # the first place and pass control over to + # expressionStatement(), which will in turn + # go all the way up to primary(), which will + # call us back with isLambda=true, allowing us + # to actually parse the function as an expression + dec(self.current) + result = self.expressionStatement() + self.currentFunction = enclosingFunction + return result + elif isLambda: + self.currentFunction = newLambdaExpr(arguments, defaults, newBlockStmt(@[], Token()), isGenerator=isGenerator, isAsync=isAsync, token=tok) else: - var parameter: IdentExpr + self.error("funDecl: invalid state") + if self.match(Colon): + # A function without an explicit + # return type is the same as a void + # function in C (i.e. no return type) + self.expect(Identifier, "expecting function return type after ':'") + returnType = newIdentExpr(self.peek(-1)) + if not self.match(LeftBrace): + # Argument-less function + var parameter: tuple[name: ASTNode, valueType: ASTNode] self.expect(LeftParen) while not self.check(RightParen): if arguments.len > 255: self.error("cannot have more than 255 arguments in function declaration") - self.expect(Identifier) - parameter = newIdentExpr(self.peek(-1)) + self.expect(Identifier, "expecting parameter name") + parameter.name = newIdentExpr(self.peek(-1)) + self.expect(Identifier, "expecting parameter type") + parameter.valueType = newIdentExpr(self.peek(-1)) if parameter in arguments: self.error("duplicate parameter name in function declaration") arguments.add(parameter) if self.match(Equal): defaults.add(self.expression()) elif defaults.len() > 0: - self.error("positional argument(s) cannot follow default argument(s) in function declaration") + self.error("positional argument cannot follow default argument in function declaration") if not self.match(Comma): break self.expect(RightParen) self.expect(LeftBrace) - if not isLambda: - FunDecl(self.currentFunction).body = self.blockStmt() + if self.currentFunction.kind == funDecl: + if not self.match(Semicolon): + # Forward declaration! + FunDecl(self.currentFunction).body = self.blockStmt() FunDecl(self.currentFunction).arguments = arguments + FunDecl(self.currentFunction).returnType = returnType else: - LambdaExpr(self.currentFunction).body = self.blockStmt() + if not self.match(Semicolon): + LambdaExpr(self.currentFunction).body = self.blockStmt() LambdaExpr(self.currentFunction).arguments = arguments + LambdaExpr(self.currentFunction).returnType = returnType result = self.currentFunction self.currentFunction = enclosingFunction @@ -947,13 +973,19 @@ proc declaration(self: Parser): ASTNode = ## Parses declarations case self.peek().kind: of Var, Const, Let: - discard self.step() - result = self.varDecl() + let keyword = self.step() + result = self.varDecl(isLet=keyword.kind == Let, isConst=keyword.kind == Const) of Function: discard self.step() result = self.funDecl() - of Type: - discard # TODO + of Coroutine: + discard self.step() + result = self.funDecl(isAsync=true) + of Generator: + discard self.step() + result = self.funDecl(isGenerator=true) + of Type, Comment, Whitespace, Tab: + discard self.step() # TODO else: result = self.statement() diff --git a/src/test.nim b/src/test.nim index a39d9d4..125f250 100644 --- a/src/test.nim +++ b/src/test.nim @@ -40,6 +40,7 @@ when isMainModule: try: input = lineEditor.read() if input.len() > 0: + # Currently the parser doesn't handle these tokens well tokens = filter(tokenizer.lex(input, ""), proc (x: Token): bool = x.kind notin {Whitespace, Tab}) for i, token in tokens: if i == tokens.high(): @@ -134,8 +135,9 @@ proc fillSymbolTable(tokenizer: Lexer) = tokenizer.symbols.addKeyword("const", Const) tokenizer.symbols.addKeyword("let", Let) tokenizer.symbols.addKeyword("var", Var) - tokenizer.symbols.addKeyword("lambda", Lambda) tokenizer.symbols.addKeyword("import", Import) + tokenizer.symbols.addKeyword("yield", Yield) + tokenizer.symbols.addKeyword("return", Return) # These are technically more like expressions # with a reserved name that produce a value of a # builtin type, but we don't need to care about