Initial (semi-broken) support for await and yield statements/expressions. Minor changes to the lexer and grammar

docs/grammar.md

@@ -90,6 +90,7 @@ importStmt     -> ("from" IDENTIFIER)? "import" (IDENTIFIER ("as" IDENTIFIER)? "
 assertStmt     → "assert" expression ";";
 delStmt        → "del" expression ";";
 yieldStmt      → "yield" expression ";"; 
+awaitStmt      → "await" expression ";";
 continueStmt   → "continue" ";";
 blockStmt      → "{" declaration* "}";  // Blocks create a new scope that lasts until they're closed
 ifStmt         → "if" "(" expression ")" statement ("else" statement)?;  // If statements are conditional jumps
@@ -100,15 +101,16 @@ foreachStmt    → "foreach" "(" (IDENTIFIER ":" expression) ")" statement;
 
 // Expressions (rules that produce a value, but also have side effects)
 expression     → assignment;
-assignment     → (call ".")? IDENTIFIER "=" yield;  // Assignment is the highest-level expression
+assignment     → (call ".")? IDENTIFIER "=" expression;  // Assignment is the highest-level expression
-yield          → "(" "yield" expression ")";
+yieldExpr      → "yield" expression;
+awaitExpr      → "await" expression;
 logic_or       → logic_and ("and" logic_and)*;
 logic_and      → equality ("or" equality)*;
 equality       → comparison (( "!=" | "==" ) comparison )*;
 comparison     → term (( ">" | ">=" | "<" | "<=" ) term )*;
 term           → factor (( "-" | "+" ) factor )*;  // Precedence for + and - in operations
 factor         → unary (("/" | "*" | "**" | "^" | "&") unary)*;  // All other binary operators have the same precedence
-unary          → ("!" | "-" | "~" | "await") unary | call;
+unary          → ("!" | "-" | "~") unary | call;
 call           → primary ("(" arguments? ")" | "." IDENTIFIER)*;
 // Below are some collection literals: lists, sets, dictionaries and tuples
 listExpr       → "[" arguments? "]"

src/backend/lexer.nim

@@ -457,7 +457,8 @@ proc next(self: Lexer) =
         self.line += 1
     elif single in ['"', '\'']:
         if self.check(single) and self.check(single, 1):
-            # Multiline strings start with 3 apexes
+            # Multiline strings start with 3 quotes
+            discard self.step(2)
             self.parseString(single, "multi")
         else:
             self.parseString(single)
@@ -479,7 +480,7 @@ proc next(self: Lexer) =
     else:
         # Comments are a special case
         if single == '#':
-            while not self.check('\n'):
+            while not (self.check('\n') or self.done()):
                 discard self.step()
             return
         # We start by checking for multi-character tokens,

src/backend/meta/ast.nim

@@ -45,12 +45,15 @@ type
         raiseStmt,
         assertStmt,
         delStmt,
+        yieldStmt,
+        awaitStmt,
         fromImportStmt,
         importStmt,
         # An expression followed by a semicolon
         exprStmt,
         # Expressions
         assignExpr,
+        awaitExpr,
         yieldExpr,
         setItemExpr,  # Set expressions like a.b = "c"
         binaryExpr,
@@ -160,6 +163,9 @@ type
     YieldExpr* = ref object of ASTNode
         expression*: ASTNode
 
+    AwaitExpr* = ref object of ASTNode
+        awaitee*: ASTNode
+
     AssignExpr* = ref object of ASTNode
         name*: ASTNode
         value*: ASTNode
@@ -198,6 +204,9 @@ type
         condition*: ASTNode
         body*: ASTNode
     
+    AwaitStmt* = ref object of ASTNode
+        awaitee*: ASTNode
+
     BreakStmt* = ref object of ASTNode
     
     ContinueStmt* = ref object of ASTNode
@@ -210,6 +219,9 @@ type
         thenBranch*: ASTNode
         elseBranch*: ASTNode
 
+    YieldStmt* = ref object of ASTNode
+        expression*: ASTNode
+
     VarDecl* = ref object of ASTNode
         name*: ASTNode
         value*: ASTNode
@@ -373,6 +385,11 @@ proc newAssignExpr*(name, value: ASTNode): AssignExpr =
     result.value = value
 
 
+proc newAwaitExpr*(awaitee: ASTNode): AwaitExpr =
+    result = AwaitExpr(kind: awaitExpr)
+    result.awaitee = awaitee
+
+
 proc newExprStmt*(expression: ASTNode): ExprStmt =
     result = ExprStmt(kind: exprStmt)
     result.expression = expression
@@ -394,6 +411,16 @@ proc newDelStmt*(name: ASTNode): DelStmt =
     result.name = name
 
 
+proc newYieldStmt*(expression: ASTNode): YieldStmt =
+    result = YieldStmt(kind: yieldStmt)
+    result.expression = expression
+
+
+proc newAwaitStmt*(awaitee: ASTNode): AwaitExpr =
+    result = AwaitExpr(kind: awaitExpr)
+    result.awaitee = awaitee
+
+
 proc newAssertStmt*(expression: ASTNode): AssertStmt =
     result = AssertStmt(kind: assertStmt)
     result.expression = expression
@@ -439,7 +466,7 @@ proc newIfStmt*(condition: ASTNode, thenBranch, elseBranch: ASTNode): IfStmt =
 
 
 proc newVarDecl*(name: ASTNode, value: ASTNode = newNilExpr(),
-                 isStatic: bool = true, isConst,
+                 isStatic: bool = true, isConst: bool = false,
                  isPrivate: bool = true): VarDecl =
     result = VarDecl(kind: varDecl)
     result.name = name
@@ -479,6 +506,8 @@ proc `$`*(self: ASTNode): string =
         of intExpr, floatExpr, hexExpr, binExpr, octExpr, strExpr, trueExpr, falseExpr, nanExpr, nilExpr, infExpr:
             if self.kind in {trueExpr, falseExpr, nanExpr, nilExpr, infExpr}:
                 result &= &"Literal({($self.kind)[0..^5]})"
+            elif self.kind == strExpr:
+                result &= &"Literal({LiteralExpr(self).literal.lexeme.escape()})"
             else:
                 result &= &"Literal({LiteralExpr(self).literal.lexeme})"
         of identExpr:
@@ -502,7 +531,7 @@ proc `$`*(self: ASTNode): string =
             result &= &"Binary({self.a}, Operator('{self.operator.lexeme}'), {self.b})"
         of assignExpr:
             var self = AssignExpr(self)
-            result &= &"Assign(name='{self.namec}', value={self.value})"
+            result &= &"Assign(name='{self.name}', value={self.value})"
         of exprStmt:
             var self = ExprStmt(self)
             result &= &"ExpressionStatement({self.expression})"
@@ -533,12 +562,18 @@ proc `$`*(self: ASTNode): string =
         of returnStmt:
             var self = ReturnStmt(self)
             result &= &"Return({self.value})"
+        of yieldExpr:
+            var self = YieldExpr(self)
+            result &= &"Yield(expression={self.expression})"
         of ifStmt:
             var self = IfStmt(self)
             if self.elseBranch == nil:
                 result &= &"If(condition={self.condition}, thenBranch={self.thenBranch}, elseBranch=nil)"
             else:
                 result &= &"If(condition={self.condition}, thenBranch={self.thenBranch}, elseBranch={self.elseBranch})"
+        of yieldStmt:
+            var self = YieldStmt(self)
+            result &= &"YieldStmt(expression={self.expression})"
         of varDecl:
             var self = VarDecl(self)
             result &= &"Var(name={self.name}, value={self.value}, const={self.isConst}, static={self.isStatic}, private={self.isPrivate})"

src/backend/optimizer.nim

@@ -37,15 +37,14 @@ type
 
     Optimizer* = ref object
         warnings: seq[Warning]
-        dryRun: bool
+        foldConstants: bool
 
 
-proc initOptimizer*(self: Optimizer = nil): Optimizer =
+proc initOptimizer*(foldConstants: bool = true): Optimizer =
     ## Initializes a new optimizer object
-    ## or resets the state of an existing one
-    if self != nil:
-        result = self
     new(result)
+    result.foldConstants = foldConstants
+    result.warnings = @[]
 
 
 proc newWarning(self: Optimizer, kind: WarningKind, node: ASTNode) =
@@ -63,6 +62,8 @@ proc optimizeConstant(self: Optimizer, node: ASTNode): ASTNode =
     ## integers. This method converts all of the different
     ## integer forms (binary, octal and hexadecimal) to
     ## decimal integers. Overflows are checked here too
+    if not self.foldConstants:
+        return node
     case node.kind:
         of intExpr:
             var x: int
@@ -272,6 +273,8 @@ proc optimizeNode(self: Optimizer, node: ASTNode): ASTNode =
     ## Analyzes an AST node and attempts to perform
     ## optimizations on it. If no optimization can be
     ## applied, the same node is returned
+    if not self.foldConstants:
+        return node
     case node.kind:
         of exprStmt:
             result = newExprStmt(self.optimizeNode(ExprStmt(node).expression))
@@ -321,7 +324,9 @@ proc optimize*(self: Optimizer, tree: seq[ASTNode]): tuple[tree: seq[ASTNode], w
     ## as well as a list of warnings that may
     ## be of interest. The input tree may be
     ## identical to the output tree if no optimization
-    ## could be performed
+    ## could be performed. Constant folding can be
+    ## turned off by setting foldConstants to false
+    ## when initializing the optimizer object
     var newTree: seq[ASTNode] = @[]
     for node in tree:
         newTree.add(self.optimizeNode(node))

src/backend/parser.nim

@@ -196,10 +196,6 @@ proc primary(self: Parser): ASTNode =
             if self.match(RightParen):
                 # This yields an empty tuple
                 result = newTupleExpr(@[])
-            elif self.match(Yield):
-                if self.context != Function:
-                    self.error("'yield' outside function")
-                result = newYieldExpr(self.expression())
             else:
                 result = self.expression()
                 if self.match(Comma):
@@ -213,6 +209,14 @@ proc primary(self: Parser): ASTNode =
                 else:
                     self.expect(RightParen, "unterminated parenthesized expression")
                     result = newGroupingExpr(result)
+        of Yield:
+            if self.context != Function:
+                self.error("'yield' cannot be outside functions")
+            result = newYieldExpr(self.expression())
+        of Await:
+            if self.context != Function:
+                self.error("'await' cannot be used outside functions")
+            result = newAwaitExpr(self.expression())
         of RightParen:
             # This is *technically* unnecessary: the parser would
             # throw an error regardless, but it's a little bit nicer
@@ -280,9 +284,7 @@ proc call(self: Parser): ASTNode =
 
 proc unary(self: Parser): ASTNode = 
     ## Parses unary expressions
-    if self.match([Minus, Tilde, Await]):
+    if self.match([Minus, Tilde]):
-        if self.peek(-1).kind == Await and self.context != Function:
-            self.error("'await' cannot be used outside functions")
         result = newUnaryExpr(self.peek(-1), self.unary())
     else:
         result = self.call()
@@ -476,11 +478,19 @@ proc returnStmt(self: Parser): ASTNode =
 proc yieldStmt(self: Parser): ASTNode =
     ## Parses yield Statements
     if self.context != Function:
-        self.error("'yield' outside function")
+        self.error("'yield' cannot be used outside functions")
     result = newYieldExpr(self.expression)
     endOfLine("missing semicolon after yield statement")
 
 
+proc awaitStmt(self: Parser): ASTNode =
+    ## Parses yield Statements
+    if self.context != Function:
+        self.error("'await' cannot be used outside functions")
+    result = newAwaitExpr(self.expression)
+    endOfLine("missing semicolon after yield statement")
+
+
 proc raiseStmt(self: Parser): ASTNode =
     ## Parses raise statements
     var exception: ASTNode
@@ -653,20 +663,20 @@ proc funDecl(self: Parser, isAsync: bool = false, isStatic: bool = true, isPriva
     # We do some analysis on the code of the function. Namely we check
     # if the user used 'await' in a non-async function and if the
     # function has any yield expressions in them, making it a 
-    # generator. Async generators are also (syntactically) supported
+    # generator. Async generators are also supported
     for line in BlockStmt(result.body).code:
         if line.kind == exprStmt:
-            if ExprStmt(line).expression.kind == unaryExpr:
+            echo ExprStmt(line).expression.kind
-                if not result.isGenerator and UnaryExpr(ExprStmt(line).expression).operator.kind == Yield:
+            if ExprStmt(line).expression.kind == yieldExpr:
-                    # If the function has the isGenerator field already set, checking again
+                result.isGenerator = true
-                    # is redundant
+            elif not result.isAsync and ExprStmt(line).expression.kind == awaitExpr:
-                    result.isGenerator = true
+                self.error("'await' cannot be used outside async functions")
-                elif UnaryExpr(ExprStmt(line).expression).operator.kind == Await and not result.isAsync:
+        elif line.kind == NodeKind.funDecl:
-                    self.error("'await' outside async function")
-        elif not result.isGenerator and line.kind == NodeKind.funDecl:
             # Nested function declarations with yield expressions as
             # default arguments are valid, but for them to work we
-            # need to make the containing function a generator!
+            # need to make the containing function a generator. These
+            # yield expressions will run in the outer function's scope
+            # and are lazily evaluated at runtime
             for default in FunDecl(line).defaults:
                 if default.kind == NodeKind.yieldExpr:
                     result.isGenerator = true
@@ -745,6 +755,9 @@ proc statement(self: Parser): ASTNode =
         of Yield:
             discard self.step()
             result = self.yieldStmt()
+        of Await:
+            discard self.step()
+            result = self.awaitStmt()
         else:
             result = self.expressionStatement()
 
@@ -796,7 +809,9 @@ proc declaration(self: Parser): ASTNode =
             let isStatic: bool = if self.peek(-1).kind == Static: true else: false
             if self.match(Async):
                 self.expect(Fun)
+                self.context = Function
                 result = self.funDecl(isStatic=isStatic, isPrivate=true, isAsync=true)
+                self.context = Script
             else:
                 case self.peek().kind:
                     of Var, Const:
@@ -815,7 +830,9 @@ proc declaration(self: Parser): ASTNode =
         of Async:
             discard self.step()
             self.expect(Fun)
+            self.context = Function
             result = self.funDecl(isAsync=true)
+            self.context = Script
         else:
             result = self.statement()