diff --git a/docs/grammar.md b/docs/grammar.md index 81de100..bfa995e 100644 --- a/docs/grammar.md +++ b/docs/grammar.md @@ -90,6 +90,7 @@ importStmt -> ("from" IDENTIFIER)? "import" (IDENTIFIER ("as" IDENTIFIER)? " assertStmt → "assert" expression ";"; delStmt → "del" expression ";"; yieldStmt → "yield" expression ";"; +awaitStmt → "await" expression ";"; continueStmt → "continue" ";"; blockStmt → "{" declaration* "}"; // Blocks create a new scope that lasts until they're closed ifStmt → "if" "(" expression ")" statement ("else" statement)?; // If statements are conditional jumps @@ -100,15 +101,16 @@ foreachStmt → "foreach" "(" (IDENTIFIER ":" expression) ")" statement; // Expressions (rules that produce a value, but also have side effects) expression → assignment; -assignment → (call ".")? IDENTIFIER "=" yield; // Assignment is the highest-level expression -yield → "(" "yield" expression ")"; +assignment → (call ".")? IDENTIFIER "=" expression; // Assignment is the highest-level expression +yieldExpr → "yield" expression; +awaitExpr → "await" expression; logic_or → logic_and ("and" logic_and)*; logic_and → equality ("or" equality)*; equality → comparison (( "!=" | "==" ) comparison )*; comparison → term (( ">" | ">=" | "<" | "<=" ) term )*; term → factor (( "-" | "+" ) factor )*; // Precedence for + and - in operations factor → unary (("/" | "*" | "**" | "^" | "&") unary)*; // All other binary operators have the same precedence -unary → ("!" | "-" | "~" | "await") unary | call; +unary → ("!" | "-" | "~") unary | call; call → primary ("(" arguments? ")" | "." IDENTIFIER)*; // Below are some collection literals: lists, sets, dictionaries and tuples listExpr → "[" arguments? "]" diff --git a/src/backend/lexer.nim b/src/backend/lexer.nim index 0762af2..dc583e6 100644 --- a/src/backend/lexer.nim +++ b/src/backend/lexer.nim @@ -457,7 +457,8 @@ proc next(self: Lexer) = self.line += 1 elif single in ['"', '\'']: if self.check(single) and self.check(single, 1): - # Multiline strings start with 3 apexes + # Multiline strings start with 3 quotes + discard self.step(2) self.parseString(single, "multi") else: self.parseString(single) @@ -479,7 +480,7 @@ proc next(self: Lexer) = else: # Comments are a special case if single == '#': - while not self.check('\n'): + while not (self.check('\n') or self.done()): discard self.step() return # We start by checking for multi-character tokens, diff --git a/src/backend/meta/ast.nim b/src/backend/meta/ast.nim index c412c89..ade0f21 100644 --- a/src/backend/meta/ast.nim +++ b/src/backend/meta/ast.nim @@ -45,12 +45,15 @@ type raiseStmt, assertStmt, delStmt, + yieldStmt, + awaitStmt, fromImportStmt, importStmt, # An expression followed by a semicolon exprStmt, # Expressions assignExpr, + awaitExpr, yieldExpr, setItemExpr, # Set expressions like a.b = "c" binaryExpr, @@ -160,6 +163,9 @@ type YieldExpr* = ref object of ASTNode expression*: ASTNode + AwaitExpr* = ref object of ASTNode + awaitee*: ASTNode + AssignExpr* = ref object of ASTNode name*: ASTNode value*: ASTNode @@ -198,6 +204,9 @@ type condition*: ASTNode body*: ASTNode + AwaitStmt* = ref object of ASTNode + awaitee*: ASTNode + BreakStmt* = ref object of ASTNode ContinueStmt* = ref object of ASTNode @@ -210,6 +219,9 @@ type thenBranch*: ASTNode elseBranch*: ASTNode + YieldStmt* = ref object of ASTNode + expression*: ASTNode + VarDecl* = ref object of ASTNode name*: ASTNode value*: ASTNode @@ -373,6 +385,11 @@ proc newAssignExpr*(name, value: ASTNode): AssignExpr = result.value = value +proc newAwaitExpr*(awaitee: ASTNode): AwaitExpr = + result = AwaitExpr(kind: awaitExpr) + result.awaitee = awaitee + + proc newExprStmt*(expression: ASTNode): ExprStmt = result = ExprStmt(kind: exprStmt) result.expression = expression @@ -394,6 +411,16 @@ proc newDelStmt*(name: ASTNode): DelStmt = result.name = name +proc newYieldStmt*(expression: ASTNode): YieldStmt = + result = YieldStmt(kind: yieldStmt) + result.expression = expression + + +proc newAwaitStmt*(awaitee: ASTNode): AwaitExpr = + result = AwaitExpr(kind: awaitExpr) + result.awaitee = awaitee + + proc newAssertStmt*(expression: ASTNode): AssertStmt = result = AssertStmt(kind: assertStmt) result.expression = expression @@ -439,7 +466,7 @@ proc newIfStmt*(condition: ASTNode, thenBranch, elseBranch: ASTNode): IfStmt = proc newVarDecl*(name: ASTNode, value: ASTNode = newNilExpr(), - isStatic: bool = true, isConst, + isStatic: bool = true, isConst: bool = false, isPrivate: bool = true): VarDecl = result = VarDecl(kind: varDecl) result.name = name @@ -479,6 +506,8 @@ proc `$`*(self: ASTNode): string = of intExpr, floatExpr, hexExpr, binExpr, octExpr, strExpr, trueExpr, falseExpr, nanExpr, nilExpr, infExpr: if self.kind in {trueExpr, falseExpr, nanExpr, nilExpr, infExpr}: result &= &"Literal({($self.kind)[0..^5]})" + elif self.kind == strExpr: + result &= &"Literal({LiteralExpr(self).literal.lexeme.escape()})" else: result &= &"Literal({LiteralExpr(self).literal.lexeme})" of identExpr: @@ -502,7 +531,7 @@ proc `$`*(self: ASTNode): string = result &= &"Binary({self.a}, Operator('{self.operator.lexeme}'), {self.b})" of assignExpr: var self = AssignExpr(self) - result &= &"Assign(name='{self.namec}', value={self.value})" + result &= &"Assign(name='{self.name}', value={self.value})" of exprStmt: var self = ExprStmt(self) result &= &"ExpressionStatement({self.expression})" @@ -533,12 +562,18 @@ proc `$`*(self: ASTNode): string = of returnStmt: var self = ReturnStmt(self) result &= &"Return({self.value})" + of yieldExpr: + var self = YieldExpr(self) + result &= &"Yield(expression={self.expression})" of ifStmt: var self = IfStmt(self) if self.elseBranch == nil: result &= &"If(condition={self.condition}, thenBranch={self.thenBranch}, elseBranch=nil)" else: result &= &"If(condition={self.condition}, thenBranch={self.thenBranch}, elseBranch={self.elseBranch})" + of yieldStmt: + var self = YieldStmt(self) + result &= &"YieldStmt(expression={self.expression})" of varDecl: var self = VarDecl(self) result &= &"Var(name={self.name}, value={self.value}, const={self.isConst}, static={self.isStatic}, private={self.isPrivate})" diff --git a/src/backend/optimizer.nim b/src/backend/optimizer.nim index 2d65084..1d2dd06 100644 --- a/src/backend/optimizer.nim +++ b/src/backend/optimizer.nim @@ -37,15 +37,14 @@ type Optimizer* = ref object warnings: seq[Warning] - dryRun: bool + foldConstants: bool -proc initOptimizer*(self: Optimizer = nil): Optimizer = +proc initOptimizer*(foldConstants: bool = true): Optimizer = ## Initializes a new optimizer object - ## or resets the state of an existing one - if self != nil: - result = self new(result) + result.foldConstants = foldConstants + result.warnings = @[] proc newWarning(self: Optimizer, kind: WarningKind, node: ASTNode) = @@ -63,6 +62,8 @@ proc optimizeConstant(self: Optimizer, node: ASTNode): ASTNode = ## integers. This method converts all of the different ## integer forms (binary, octal and hexadecimal) to ## decimal integers. Overflows are checked here too + if not self.foldConstants: + return node case node.kind: of intExpr: var x: int @@ -272,6 +273,8 @@ proc optimizeNode(self: Optimizer, node: ASTNode): ASTNode = ## Analyzes an AST node and attempts to perform ## optimizations on it. If no optimization can be ## applied, the same node is returned + if not self.foldConstants: + return node case node.kind: of exprStmt: result = newExprStmt(self.optimizeNode(ExprStmt(node).expression)) @@ -321,7 +324,9 @@ proc optimize*(self: Optimizer, tree: seq[ASTNode]): tuple[tree: seq[ASTNode], w ## as well as a list of warnings that may ## be of interest. The input tree may be ## identical to the output tree if no optimization - ## could be performed + ## could be performed. Constant folding can be + ## turned off by setting foldConstants to false + ## when initializing the optimizer object var newTree: seq[ASTNode] = @[] for node in tree: newTree.add(self.optimizeNode(node)) diff --git a/src/backend/parser.nim b/src/backend/parser.nim index f4d1fcc..f3cfc75 100644 --- a/src/backend/parser.nim +++ b/src/backend/parser.nim @@ -196,10 +196,6 @@ proc primary(self: Parser): ASTNode = if self.match(RightParen): # This yields an empty tuple result = newTupleExpr(@[]) - elif self.match(Yield): - if self.context != Function: - self.error("'yield' outside function") - result = newYieldExpr(self.expression()) else: result = self.expression() if self.match(Comma): @@ -213,6 +209,14 @@ proc primary(self: Parser): ASTNode = else: self.expect(RightParen, "unterminated parenthesized expression") result = newGroupingExpr(result) + of Yield: + if self.context != Function: + self.error("'yield' cannot be outside functions") + result = newYieldExpr(self.expression()) + of Await: + if self.context != Function: + self.error("'await' cannot be used outside functions") + result = newAwaitExpr(self.expression()) of RightParen: # This is *technically* unnecessary: the parser would # throw an error regardless, but it's a little bit nicer @@ -280,9 +284,7 @@ proc call(self: Parser): ASTNode = proc unary(self: Parser): ASTNode = ## Parses unary expressions - if self.match([Minus, Tilde, Await]): - if self.peek(-1).kind == Await and self.context != Function: - self.error("'await' cannot be used outside functions") + if self.match([Minus, Tilde]): result = newUnaryExpr(self.peek(-1), self.unary()) else: result = self.call() @@ -476,11 +478,19 @@ proc returnStmt(self: Parser): ASTNode = proc yieldStmt(self: Parser): ASTNode = ## Parses yield Statements if self.context != Function: - self.error("'yield' outside function") + self.error("'yield' cannot be used outside functions") result = newYieldExpr(self.expression) endOfLine("missing semicolon after yield statement") +proc awaitStmt(self: Parser): ASTNode = + ## Parses yield Statements + if self.context != Function: + self.error("'await' cannot be used outside functions") + result = newAwaitExpr(self.expression) + endOfLine("missing semicolon after yield statement") + + proc raiseStmt(self: Parser): ASTNode = ## Parses raise statements var exception: ASTNode @@ -653,20 +663,20 @@ proc funDecl(self: Parser, isAsync: bool = false, isStatic: bool = true, isPriva # We do some analysis on the code of the function. Namely we check # if the user used 'await' in a non-async function and if the # function has any yield expressions in them, making it a - # generator. Async generators are also (syntactically) supported + # generator. Async generators are also supported for line in BlockStmt(result.body).code: if line.kind == exprStmt: - if ExprStmt(line).expression.kind == unaryExpr: - if not result.isGenerator and UnaryExpr(ExprStmt(line).expression).operator.kind == Yield: - # If the function has the isGenerator field already set, checking again - # is redundant - result.isGenerator = true - elif UnaryExpr(ExprStmt(line).expression).operator.kind == Await and not result.isAsync: - self.error("'await' outside async function") - elif not result.isGenerator and line.kind == NodeKind.funDecl: + echo ExprStmt(line).expression.kind + if ExprStmt(line).expression.kind == yieldExpr: + result.isGenerator = true + elif not result.isAsync and ExprStmt(line).expression.kind == awaitExpr: + self.error("'await' cannot be used outside async functions") + elif line.kind == NodeKind.funDecl: # Nested function declarations with yield expressions as # default arguments are valid, but for them to work we - # need to make the containing function a generator! + # need to make the containing function a generator. These + # yield expressions will run in the outer function's scope + # and are lazily evaluated at runtime for default in FunDecl(line).defaults: if default.kind == NodeKind.yieldExpr: result.isGenerator = true @@ -745,6 +755,9 @@ proc statement(self: Parser): ASTNode = of Yield: discard self.step() result = self.yieldStmt() + of Await: + discard self.step() + result = self.awaitStmt() else: result = self.expressionStatement() @@ -796,7 +809,9 @@ proc declaration(self: Parser): ASTNode = let isStatic: bool = if self.peek(-1).kind == Static: true else: false if self.match(Async): self.expect(Fun) + self.context = Function result = self.funDecl(isStatic=isStatic, isPrivate=true, isAsync=true) + self.context = Script else: case self.peek().kind: of Var, Const: @@ -815,7 +830,9 @@ proc declaration(self: Parser): ASTNode = of Async: discard self.step() self.expect(Fun) + self.context = Function result = self.funDecl(isAsync=true) + self.context = Script else: result = self.statement()