japl
/
JustAnotherJAPL
mirror of https://github.com/japl-lang/JustAnotherJAPL.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

Initial (semi-broken) support for await and yield statements/expressions. Minor changes to the lexer and grammar

This commit is contained in:
Nocturn9x 2021-10-25 11:50:12 +02:00
parent 74b8328bf7
commit 151b8da0d5
5 changed files with 91 additions and 31 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
docs/grammar.md
View File

@ -90,6 +90,7 @@ importStmt -> ("from" IDENTIFIER)? "import" (IDENTIFIER ("as" IDENTIFIER)? "
assertStmt → "assert" expression ";";
delStmt → "del" expression ";";
yieldStmt → "yield" expression ";";
awaitStmt → "await" expression ";";
continueStmt → "continue" ";";
blockStmt → "{" declaration* "}"; // Blocks create a new scope that lasts until they're closed
ifStmt → "if" "(" expression ")" statement ("else" statement)?; // If statements are conditional jumps
@ -100,15 +101,16 @@ foreachStmt → "foreach" "(" (IDENTIFIER ":" expression) ")" statement;
// Expressions (rules that produce a value, but also have side effects)
expression → assignment;
assignment → (call ".")? IDENTIFIER "=" yield; // Assignment is the highest-level expression
yield → "(" "yield" expression ")";
assignment → (call ".")? IDENTIFIER "=" expression; // Assignment is the highest-level expression
yieldExpr → "yield" expression;
awaitExpr → "await" expression;
logic_or → logic_and ("and" logic_and)*;
logic_and → equality ("or" equality)*;
equality → comparison (( "!=" | "==" ) comparison )*;
comparison → term (( ">" | ">=" | "<" | "<=" ) term )*;
term → factor (( "-" | "+" ) factor )*; // Precedence for + and - in operations
factor → unary (("/" | "*" | "**" | "^" | "&") unary)*; // All other binary operators have the same precedence
unary → ("!" | "-" | "~" | "await") unary | call;
unary → ("!" | "-" | "~") unary | call;
call → primary ("(" arguments? ")" | "." IDENTIFIER)*;
// Below are some collection literals: lists, sets, dictionaries and tuples
listExpr → "[" arguments? "]"

5
src/backend/lexer.nim
View File

@ -457,7 +457,8 @@ proc next(self: Lexer) =
self.line += 1
elif single in ['"', '\'']:
if self.check(single) and self.check(single, 1):
# Multiline strings start with 3 apexes
# Multiline strings start with 3 quotes
discard self.step(2)
self.parseString(single, "multi")
else:
self.parseString(single)
@ -479,7 +480,7 @@ proc next(self: Lexer) =
else:
# Comments are a special case
if single == '#':
while not self.check('\n'):
while not (self.check('\n') or self.done()):
discard self.step()
return
# We start by checking for multi-character tokens,

39
src/backend/meta/ast.nim
View File

@ -45,12 +45,15 @@ type
raiseStmt,
assertStmt,
delStmt,
yieldStmt,
awaitStmt,
fromImportStmt,
importStmt,
# An expression followed by a semicolon
exprStmt,
# Expressions
assignExpr,
awaitExpr,
yieldExpr,
setItemExpr, # Set expressions like a.b = "c"
binaryExpr,
@ -160,6 +163,9 @@ type
YieldExpr* = ref object of ASTNode
expression*: ASTNode
AwaitExpr* = ref object of ASTNode
awaitee*: ASTNode
AssignExpr* = ref object of ASTNode
name*: ASTNode
value*: ASTNode
@ -198,6 +204,9 @@ type
condition*: ASTNode
body*: ASTNode
AwaitStmt* = ref object of ASTNode
awaitee*: ASTNode
BreakStmt* = ref object of ASTNode
ContinueStmt* = ref object of ASTNode
@ -210,6 +219,9 @@ type
thenBranch*: ASTNode
elseBranch*: ASTNode
YieldStmt* = ref object of ASTNode
expression*: ASTNode
VarDecl* = ref object of ASTNode
name*: ASTNode
value*: ASTNode
@ -373,6 +385,11 @@ proc newAssignExpr*(name, value: ASTNode): AssignExpr =
result.value = value
proc newAwaitExpr*(awaitee: ASTNode): AwaitExpr =
result = AwaitExpr(kind: awaitExpr)
result.awaitee = awaitee
proc newExprStmt*(expression: ASTNode): ExprStmt =
result = ExprStmt(kind: exprStmt)
result.expression = expression
@ -394,6 +411,16 @@ proc newDelStmt*(name: ASTNode): DelStmt =
result.name = name
proc newYieldStmt*(expression: ASTNode): YieldStmt =
result = YieldStmt(kind: yieldStmt)
result.expression = expression
proc newAwaitStmt*(awaitee: ASTNode): AwaitExpr =
result = AwaitExpr(kind: awaitExpr)
result.awaitee = awaitee
proc newAssertStmt*(expression: ASTNode): AssertStmt =
result = AssertStmt(kind: assertStmt)
result.expression = expression
@ -439,7 +466,7 @@ proc newIfStmt*(condition: ASTNode, thenBranch, elseBranch: ASTNode): IfStmt =
proc newVarDecl*(name: ASTNode, value: ASTNode = newNilExpr(),
isStatic: bool = true, isConst,
isStatic: bool = true, isConst: bool = false,
isPrivate: bool = true): VarDecl =
result = VarDecl(kind: varDecl)
result.name = name
@ -479,6 +506,8 @@ proc `$`*(self: ASTNode): string =
of intExpr, floatExpr, hexExpr, binExpr, octExpr, strExpr, trueExpr, falseExpr, nanExpr, nilExpr, infExpr:
if self.kind in {trueExpr, falseExpr, nanExpr, nilExpr, infExpr}:
result &= &"Literal({($self.kind)[0..^5]})"
elif self.kind == strExpr:
result &= &"Literal({LiteralExpr(self).literal.lexeme.escape()})"
else:
result &= &"Literal({LiteralExpr(self).literal.lexeme})"
of identExpr:
@ -502,7 +531,7 @@ proc `$`*(self: ASTNode): string =
result &= &"Binary({self.a}, Operator('{self.operator.lexeme}'), {self.b})"
of assignExpr:
var self = AssignExpr(self)
result &= &"Assign(name='{self.namec}', value={self.value})"
result &= &"Assign(name='{self.name}', value={self.value})"
of exprStmt:
var self = ExprStmt(self)
result &= &"ExpressionStatement({self.expression})"
@ -533,12 +562,18 @@ proc `$`*(self: ASTNode): string =
of returnStmt:
var self = ReturnStmt(self)
result &= &"Return({self.value})"
of yieldExpr:
var self = YieldExpr(self)
result &= &"Yield(expression={self.expression})"
of ifStmt:
var self = IfStmt(self)
if self.elseBranch == nil:
result &= &"If(condition={self.condition}, thenBranch={self.thenBranch}, elseBranch=nil)"
else:
result &= &"If(condition={self.condition}, thenBranch={self.thenBranch}, elseBranch={self.elseBranch})"
of yieldStmt:
var self = YieldStmt(self)
result &= &"YieldStmt(expression={self.expression})"
of varDecl:
var self = VarDecl(self)
result &= &"Var(name={self.name}, value={self.value}, const={self.isConst}, static={self.isStatic}, private={self.isPrivate})"

17
src/backend/optimizer.nim
View File

@ -37,15 +37,14 @@ type
Optimizer* = ref object
warnings: seq[Warning]
dryRun: bool
foldConstants: bool
proc initOptimizer*(self: Optimizer = nil): Optimizer =
proc initOptimizer*(foldConstants: bool = true): Optimizer =
## Initializes a new optimizer object
## or resets the state of an existing one
if self != nil:
result = self
new(result)
result.foldConstants = foldConstants
result.warnings = @[]
proc newWarning(self: Optimizer, kind: WarningKind, node: ASTNode) =
@ -63,6 +62,8 @@ proc optimizeConstant(self: Optimizer, node: ASTNode): ASTNode =
## integers. This method converts all of the different
## integer forms (binary, octal and hexadecimal) to
## decimal integers. Overflows are checked here too
if not self.foldConstants:
return node
case node.kind:
of intExpr:
var x: int
@ -272,6 +273,8 @@ proc optimizeNode(self: Optimizer, node: ASTNode): ASTNode =
## Analyzes an AST node and attempts to perform
## optimizations on it. If no optimization can be
## applied, the same node is returned
if not self.foldConstants:
return node
case node.kind:
of exprStmt:
result = newExprStmt(self.optimizeNode(ExprStmt(node).expression))
@ -321,7 +324,9 @@ proc optimize*(self: Optimizer, tree: seq[ASTNode]): tuple[tree: seq[ASTNode], w
## as well as a list of warnings that may
## be of interest. The input tree may be
## identical to the output tree if no optimization
## could be performed
## could be performed. Constant folding can be
## turned off by setting foldConstants to false
## when initializing the optimizer object
var newTree: seq[ASTNode] = @[]
for node in tree:
newTree.add(self.optimizeNode(node))

53
src/backend/parser.nim
View File

@ -196,10 +196,6 @@ proc primary(self: Parser): ASTNode =
if self.match(RightParen):
# This yields an empty tuple
result = newTupleExpr(@[])
elif self.match(Yield):
if self.context != Function:
self.error("'yield' outside function")
result = newYieldExpr(self.expression())
else:
result = self.expression()
if self.match(Comma):
@ -213,6 +209,14 @@ proc primary(self: Parser): ASTNode =
else:
self.expect(RightParen, "unterminated parenthesized expression")
result = newGroupingExpr(result)
of Yield:
if self.context != Function:
self.error("'yield' cannot be outside functions")
result = newYieldExpr(self.expression())
of Await:
if self.context != Function:
self.error("'await' cannot be used outside functions")
result = newAwaitExpr(self.expression())
of RightParen:
# This is *technically* unnecessary: the parser would
# throw an error regardless, but it's a little bit nicer
@ -280,9 +284,7 @@ proc call(self: Parser): ASTNode =
proc unary(self: Parser): ASTNode =
## Parses unary expressions
if self.match([Minus, Tilde, Await]):
if self.peek(-1).kind == Await and self.context != Function:
self.error("'await' cannot be used outside functions")
if self.match([Minus, Tilde]):
result = newUnaryExpr(self.peek(-1), self.unary())
else:
result = self.call()
@ -476,11 +478,19 @@ proc returnStmt(self: Parser): ASTNode =
proc yieldStmt(self: Parser): ASTNode =
## Parses yield Statements
if self.context != Function:
self.error("'yield' outside function")
self.error("'yield' cannot be used outside functions")
result = newYieldExpr(self.expression)
endOfLine("missing semicolon after yield statement")
proc awaitStmt(self: Parser): ASTNode =
## Parses yield Statements
if self.context != Function:
self.error("'await' cannot be used outside functions")
result = newAwaitExpr(self.expression)
endOfLine("missing semicolon after yield statement")
proc raiseStmt(self: Parser): ASTNode =
## Parses raise statements
var exception: ASTNode
@ -653,20 +663,20 @@ proc funDecl(self: Parser, isAsync: bool = false, isStatic: bool = true, isPriva
# We do some analysis on the code of the function. Namely we check
# if the user used 'await' in a non-async function and if the
# function has any yield expressions in them, making it a
# generator. Async generators are also (syntactically) supported
# generator. Async generators are also supported
for line in BlockStmt(result.body).code:
if line.kind == exprStmt:
if ExprStmt(line).expression.kind == unaryExpr:
if not result.isGenerator and UnaryExpr(ExprStmt(line).expression).operator.kind == Yield:
# If the function has the isGenerator field already set, checking again
# is redundant
result.isGenerator = true
elif UnaryExpr(ExprStmt(line).expression).operator.kind == Await and not result.isAsync:
self.error("'await' outside async function")
elif not result.isGenerator and line.kind == NodeKind.funDecl:
echo ExprStmt(line).expression.kind
if ExprStmt(line).expression.kind == yieldExpr:
result.isGenerator = true
elif not result.isAsync and ExprStmt(line).expression.kind == awaitExpr:
self.error("'await' cannot be used outside async functions")
elif line.kind == NodeKind.funDecl:
# Nested function declarations with yield expressions as
# default arguments are valid, but for them to work we
# need to make the containing function a generator!
# need to make the containing function a generator. These
# yield expressions will run in the outer function's scope
# and are lazily evaluated at runtime
for default in FunDecl(line).defaults:
if default.kind == NodeKind.yieldExpr:
result.isGenerator = true
@ -745,6 +755,9 @@ proc statement(self: Parser): ASTNode =
of Yield:
discard self.step()
result = self.yieldStmt()
of Await:
discard self.step()
result = self.awaitStmt()
else:
result = self.expressionStatement()
@ -796,7 +809,9 @@ proc declaration(self: Parser): ASTNode =
let isStatic: bool = if self.peek(-1).kind == Static: true else: false
if self.match(Async):
self.expect(Fun)
self.context = Function
result = self.funDecl(isStatic=isStatic, isPrivate=true, isAsync=true)
self.context = Script
else:
case self.peek().kind:
of Var, Const:
@ -815,7 +830,9 @@ proc declaration(self: Parser): ASTNode =
of Async:
discard self.step()
self.expect(Fun)
self.context = Function
result = self.funDecl(isAsync=true)
self.context = Script
else:
result = self.statement()