Minor fixes and changes to the parser
This commit is contained in:
parent
37f35bec08
commit
68a6e2e83b
|
@ -21,12 +21,9 @@ import std/times
|
||||||
import config
|
import config
|
||||||
import errors
|
import errors
|
||||||
import multibyte
|
import multibyte
|
||||||
import frontend/parsing/ast
|
|
||||||
import frontend/compiler/targets/bytecode/opcodes
|
import frontend/compiler/targets/bytecode/opcodes
|
||||||
|
|
||||||
|
|
||||||
export ast
|
|
||||||
|
|
||||||
type
|
type
|
||||||
Serializer* = ref object
|
Serializer* = ref object
|
||||||
file: string
|
file: string
|
||||||
|
|
|
@ -24,7 +24,7 @@ import ast
|
||||||
import token
|
import token
|
||||||
import errors
|
import errors
|
||||||
import config
|
import config
|
||||||
import lexer as l
|
import lexer
|
||||||
import util/symbols
|
import util/symbols
|
||||||
|
|
||||||
|
|
||||||
|
@ -79,14 +79,14 @@ type
|
||||||
# to either a FunDecl or LambdaExpr
|
# to either a FunDecl or LambdaExpr
|
||||||
# AST node and is nil when the parser
|
# AST node and is nil when the parser
|
||||||
# is at the top-level. It allows the
|
# is at the top-level. It allows the
|
||||||
# parser to detect errors like return
|
# parser to detect errors like using
|
||||||
# outside functions
|
# return outside functions
|
||||||
currentFunction: Declaration
|
currentFunction: Declaration
|
||||||
# Stores the current scope depth (0 = global, > 0 local)
|
# Stores the current scope depth (0 = global, > 0 local)
|
||||||
scopeDepth: int
|
scopeDepth: int
|
||||||
# Operator table
|
# Operator table
|
||||||
operators: OperatorTable
|
operators: OperatorTable
|
||||||
# The AST node
|
# The AST we're producing
|
||||||
tree: seq[Declaration]
|
tree: seq[Declaration]
|
||||||
# Stores line data
|
# Stores line data
|
||||||
lines: seq[tuple[start, stop: int]]
|
lines: seq[tuple[start, stop: int]]
|
||||||
|
@ -99,6 +99,7 @@ type
|
||||||
# to avoid importing a module twice and to
|
# to avoid importing a module twice and to
|
||||||
# detect recursive dependency cycles
|
# detect recursive dependency cycles
|
||||||
modules: TableRef[string, bool]
|
modules: TableRef[string, bool]
|
||||||
|
|
||||||
ParseError* = ref object of PeonException
|
ParseError* = ref object of PeonException
|
||||||
## A parsing exception
|
## A parsing exception
|
||||||
parser*: Parser
|
parser*: Parser
|
||||||
|
@ -203,9 +204,8 @@ proc done(self: Parser): bool {.inline.} =
|
||||||
result = self.peek().kind == EndOfFile
|
result = self.peek().kind == EndOfFile
|
||||||
|
|
||||||
|
|
||||||
proc step(self: Parser, n: int = 1): Token {.inline.} =
|
proc step(self: Parser): Token {.inline.} =
|
||||||
## Steps n tokens into the input,
|
## Consumes a token and returns it
|
||||||
## returning the last consumed one
|
|
||||||
if self.done():
|
if self.done():
|
||||||
result = self.peek()
|
result = self.peek()
|
||||||
else:
|
else:
|
||||||
|
@ -228,8 +228,7 @@ proc error(self: Parser, message: string, token: Token = nil) {.raises: [ParseEr
|
||||||
# tell at tokenization time which of the two contexts we're in, we just treat everything
|
# tell at tokenization time which of the two contexts we're in, we just treat everything
|
||||||
# as a symbol and in the cases where we need a specific token we just match the string
|
# as a symbol and in the cases where we need a specific token we just match the string
|
||||||
# directly
|
# directly
|
||||||
proc check[T: TokenType or string](self: Parser, kind: T,
|
proc check[T: TokenType or string](self: Parser, kind: T, distance: int = 0): bool {.inline.} =
|
||||||
distance: int = 0): bool {.inline.} =
|
|
||||||
## Checks if the given token at the given distance
|
## Checks if the given token at the given distance
|
||||||
## matches the expected kind and returns a boolean.
|
## matches the expected kind and returns a boolean.
|
||||||
## The distance parameter is passed directly to
|
## The distance parameter is passed directly to
|
||||||
|
@ -302,11 +301,9 @@ proc expect[T: TokenType or string](self: Parser, kind: openarray[T], message: s
|
||||||
proc expression(self: Parser): Expression
|
proc expression(self: Parser): Expression
|
||||||
proc expressionStatement(self: Parser): Statement
|
proc expressionStatement(self: Parser): Statement
|
||||||
proc statement(self: Parser): Statement
|
proc statement(self: Parser): Statement
|
||||||
proc varDecl(self: Parser, isLet: bool = false,
|
proc varDecl(self: Parser, isLet: bool = false, isConst: bool = false): Declaration
|
||||||
isConst: bool = false): Declaration
|
|
||||||
proc parseFunExpr(self: Parser): LambdaExpr
|
proc parseFunExpr(self: Parser): LambdaExpr
|
||||||
proc funDecl(self: Parser, isAsync: bool = false, isGenerator: bool = false,
|
proc funDecl(self: Parser, isAsync: bool = false, isGenerator: bool = false, isLambda: bool = false, isOperator: bool = false): Declaration
|
||||||
isLambda: bool = false, isOperator: bool = false): Declaration
|
|
||||||
proc declaration(self: Parser): Declaration
|
proc declaration(self: Parser): Declaration
|
||||||
proc parse*(self: Parser, tokens: seq[Token], file: string, lines: seq[tuple[start, stop: int]], source: string, persist: bool = false): seq[Declaration]
|
proc parse*(self: Parser, tokens: seq[Token], file: string, lines: seq[tuple[start, stop: int]], source: string, persist: bool = false): seq[Declaration]
|
||||||
proc findOperators(self: Parser, tokens: seq[Token])
|
proc findOperators(self: Parser, tokens: seq[Token])
|
||||||
|
@ -416,7 +413,6 @@ proc makeCall(self: Parser, callee: Expression): CallExpr =
|
||||||
while true:
|
while true:
|
||||||
if argCount >= 255:
|
if argCount >= 255:
|
||||||
self.error("can not pass more than 255 arguments in function call")
|
self.error("can not pass more than 255 arguments in function call")
|
||||||
break
|
|
||||||
argument = self.expression()
|
argument = self.expression()
|
||||||
if argument.kind == assignExpr:
|
if argument.kind == assignExpr:
|
||||||
var assign = AssignExpr(argument)
|
var assign = AssignExpr(argument)
|
||||||
|
@ -547,7 +543,7 @@ proc parseAnd(self: Parser): Expression =
|
||||||
result = self.parseCmp()
|
result = self.parseCmp()
|
||||||
var operator: Token
|
var operator: Token
|
||||||
var right: Expression
|
var right: Expression
|
||||||
while self.check([Identifier, Symbol]) and self.operators.getPrecedence(self.peek().lexeme) == Precedence.And:
|
while self.check([Identifier, Symbol]) and self.operators.getPrecedence(self.peek().lexeme) == And:
|
||||||
operator = self.step()
|
operator = self.step()
|
||||||
right = self.parseCmp()
|
right = self.parseCmp()
|
||||||
result = newBinaryExpr(result, operator, right)
|
result = newBinaryExpr(result, operator, right)
|
||||||
|
@ -559,7 +555,7 @@ proc parseOr(self: Parser): Expression =
|
||||||
result = self.parseAnd()
|
result = self.parseAnd()
|
||||||
var operator: Token
|
var operator: Token
|
||||||
var right: Expression
|
var right: Expression
|
||||||
while self.check([Identifier, Symbol]) and self.operators.getPrecedence(self.peek().lexeme) == Precedence.Or:
|
while self.check([Identifier, Symbol]) and self.operators.getPrecedence(self.peek().lexeme) == Or:
|
||||||
operator = self.step()
|
operator = self.step()
|
||||||
right = self.parseAnd()
|
right = self.parseAnd()
|
||||||
result = newBinaryExpr(result, operator, right)
|
result = newBinaryExpr(result, operator, right)
|
||||||
|
@ -588,7 +584,7 @@ proc parseArrow(self: Parser): Expression =
|
||||||
result = self.parseAssign()
|
result = self.parseAssign()
|
||||||
var operator: Token
|
var operator: Token
|
||||||
var right: Expression
|
var right: Expression
|
||||||
while self.check([Identifier, Symbol]) and self.operators.getPrecedence(self.peek().lexeme) == Precedence.Arrow:
|
while self.check([Identifier, Symbol]) and self.operators.getPrecedence(self.peek().lexeme) == Arrow:
|
||||||
operator = self.step()
|
operator = self.step()
|
||||||
right = self.parseAssign()
|
right = self.parseAssign()
|
||||||
result = newBinaryExpr(result, operator, right)
|
result = newBinaryExpr(result, operator, right)
|
||||||
|
@ -815,7 +811,7 @@ proc importStmt(self: Parser, fromStmt: bool = false): Statement =
|
||||||
elif i == searchPath.high():
|
elif i == searchPath.high():
|
||||||
self.error(&"""could not import '{path}': module not found""")
|
self.error(&"""could not import '{path}': module not found""")
|
||||||
if not self.modules.getOrDefault(path, true):
|
if not self.modules.getOrDefault(path, true):
|
||||||
self.error(&"coult not import '{path}' (recursive dependency detected)")
|
self.error(&"coult not import '{path}' from '{self.file}' due to a cyclic dependency")
|
||||||
else:
|
else:
|
||||||
self.modules[path] = false
|
self.modules[path] = false
|
||||||
try:
|
try:
|
||||||
|
@ -940,8 +936,7 @@ proc parsePragmas(self: Parser): seq[Pragma] =
|
||||||
elif self.match("("):
|
elif self.match("("):
|
||||||
while not self.match(")") and not self.done():
|
while not self.match(")") and not self.done():
|
||||||
exp = self.primary()
|
exp = self.primary()
|
||||||
if exp.kind notin {strExpr, intExpr, octExpr, binExpr, hexExpr, floatExpr,
|
if not exp.isConst():
|
||||||
trueExpr, falseExpr}:
|
|
||||||
self.error("pragma arguments can only be literals", exp.token)
|
self.error("pragma arguments can only be literals", exp.token)
|
||||||
args.add(LiteralExpr(exp))
|
args.add(LiteralExpr(exp))
|
||||||
if not self.match(","):
|
if not self.match(","):
|
||||||
|
@ -949,8 +944,7 @@ proc parsePragmas(self: Parser): seq[Pragma] =
|
||||||
self.expect(LeftParen, "unterminated parenthesis in pragma arguments")
|
self.expect(LeftParen, "unterminated parenthesis in pragma arguments")
|
||||||
else:
|
else:
|
||||||
exp = self.primary()
|
exp = self.primary()
|
||||||
if exp.kind notin {strExpr, intExpr, octExpr, binExpr, hexExpr, floatExpr,
|
if not exp.isConst():
|
||||||
trueExpr, falseExpr}:
|
|
||||||
self.error("pragma arguments can only be literals", exp.token)
|
self.error("pragma arguments can only be literals", exp.token)
|
||||||
args.add(LiteralExpr(exp))
|
args.add(LiteralExpr(exp))
|
||||||
result.add(newPragma(name, args))
|
result.add(newPragma(name, args))
|
||||||
|
@ -990,14 +984,11 @@ proc varDecl(self: Parser, isLet: bool = false,
|
||||||
pragmas.add(pragma)
|
pragmas.add(pragma)
|
||||||
case tok.kind:
|
case tok.kind:
|
||||||
of TokenType.Var:
|
of TokenType.Var:
|
||||||
result = newVarDecl(name, value, isPrivate = isPrivate, token = tok,
|
result = newVarDecl(name, value, isPrivate=isPrivate, token=tok, valueType=valueType, pragmas=(@[]))
|
||||||
valueType = valueType, pragmas = (@[]))
|
|
||||||
of Const:
|
of Const:
|
||||||
result = newVarDecl(name, value, isPrivate = isPrivate, token = tok,
|
result = newVarDecl(name, value, isPrivate=isPrivate, token=tok, isConst=true, valueType=valueType, pragmas=(@[]))
|
||||||
isConst = true, valueType = valueType, pragmas = (@[]))
|
|
||||||
of Let:
|
of Let:
|
||||||
result = newVarDecl(name, value, isPrivate = isPrivate, token = tok,
|
result = newVarDecl(name, value, isPrivate=isPrivate, token=tok, isLet=isLet, valueType=valueType, pragmas=(@[]))
|
||||||
isLet = isLet, valueType = valueType, pragmas = (@[]))
|
|
||||||
else:
|
else:
|
||||||
discard # Unreachable
|
discard # Unreachable
|
||||||
if not hasInit and VarDecl(result).valueType.isNil():
|
if not hasInit and VarDecl(result).valueType.isNil():
|
||||||
|
@ -1064,7 +1055,7 @@ proc parseFunExpr(self: Parser): LambdaExpr =
|
||||||
proc parseGenericConstraint(self: Parser): Expression =
|
proc parseGenericConstraint(self: Parser): Expression =
|
||||||
## Recursively parses a generic constraint
|
## Recursively parses a generic constraint
|
||||||
## and returns it as an expression
|
## and returns it as an expression
|
||||||
result = self.expression() # First value is always an identifier of some sort
|
result = self.expression()
|
||||||
if not self.check(RightBracket):
|
if not self.check(RightBracket):
|
||||||
case self.peek().lexeme:
|
case self.peek().lexeme:
|
||||||
of "|":
|
of "|":
|
||||||
|
@ -1139,8 +1130,11 @@ proc funDecl(self: Parser, isAsync: bool = false, isGenerator: bool = false,
|
||||||
elif isLambda:
|
elif isLambda:
|
||||||
self.currentFunction = newLambdaExpr(arguments, defaults, newBlockStmt(@[], Token()), isGenerator=isGenerator, isAsync=isAsync, token=tok,
|
self.currentFunction = newLambdaExpr(arguments, defaults, newBlockStmt(@[], Token()), isGenerator=isGenerator, isAsync=isAsync, token=tok,
|
||||||
returnType=nil, depth=self.scopeDepth)
|
returnType=nil, depth=self.scopeDepth)
|
||||||
|
if self.match(LeftParen):
|
||||||
|
var parameter: tuple[name: IdentExpr, valueType: Expression]
|
||||||
|
self.parseDeclArguments(arguments, parameter, defaults)
|
||||||
if self.match(":"):
|
if self.match(":"):
|
||||||
# Function has explicit return type
|
# Function returns a value
|
||||||
if self.match([Function, Coroutine, Generator]):
|
if self.match([Function, Coroutine, Generator]):
|
||||||
# The function's return type is another
|
# The function's return type is another
|
||||||
# function. We specialize this case because
|
# function. We specialize this case because
|
||||||
|
@ -1150,15 +1144,6 @@ proc funDecl(self: Parser, isAsync: bool = false, isGenerator: bool = false,
|
||||||
returnType = self.parseFunExpr()
|
returnType = self.parseFunExpr()
|
||||||
else:
|
else:
|
||||||
returnType = self.expression()
|
returnType = self.expression()
|
||||||
if self.match(LeftParen):
|
|
||||||
var parameter: tuple[name: IdentExpr, valueType: Expression]
|
|
||||||
self.parseDeclArguments(arguments, parameter, defaults)
|
|
||||||
if self.match(":"):
|
|
||||||
# Function's return type
|
|
||||||
if self.match([Function, Coroutine, Generator]):
|
|
||||||
returnType = self.parseFunExpr()
|
|
||||||
else:
|
|
||||||
returnType = self.expression()
|
|
||||||
if self.currentFunction.kind == funDecl:
|
if self.currentFunction.kind == funDecl:
|
||||||
if not self.match(Semicolon):
|
if not self.match(Semicolon):
|
||||||
# If we don't find a semicolon,
|
# If we don't find a semicolon,
|
||||||
|
|
Loading…
Reference in New Issue