Fixed and improved error reporting and made the parser ignore whitespace/tab (again)
This commit is contained in:
parent
7cf69cf0cf
commit
6d6ae3ee7a
|
@ -163,6 +163,9 @@ proc inferType(self: Compiler, node: Expression): Type
|
||||||
## Public getter for nicer error formatting
|
## Public getter for nicer error formatting
|
||||||
proc getCurrentNode*(self: Compiler): ASTNode = (if self.current >=
|
proc getCurrentNode*(self: Compiler): ASTNode = (if self.current >=
|
||||||
self.ast.len(): self.ast[^1] else: self.ast[self.current - 1])
|
self.ast.len(): self.ast[^1] else: self.ast[self.current - 1])
|
||||||
|
proc getCurrentFunction*(self: Compiler): Declaration {.inline.} = self.currentFunction
|
||||||
|
proc getFile*(self: COmpiler): string {.inline.} = self.file
|
||||||
|
proc getModule*(self: COmpiler): string {.inline.} = self.currentModule
|
||||||
|
|
||||||
|
|
||||||
## Utility functions
|
## Utility functions
|
||||||
|
@ -185,10 +188,9 @@ proc done(self: Compiler): bool =
|
||||||
result = self.current > self.ast.high()
|
result = self.current > self.ast.high()
|
||||||
|
|
||||||
|
|
||||||
proc error(self: Compiler, message: string) {.raises: [CompileError, ValueError].} =
|
proc error(self: Compiler, message: string) {.raises: [CompileError].} =
|
||||||
## Raises a formatted CompileError exception
|
## Raises a CompileError exception
|
||||||
var tok = self.getCurrentNode().token
|
raise newException(CompileError, message)
|
||||||
raise newException(CompileError, &"A fatal error occurred while compiling '{self.file}', module '{self.currentModule}' line {tok.line} at '{tok.lexeme}' -> {message}")
|
|
||||||
|
|
||||||
|
|
||||||
proc step(self: Compiler): ASTNode =
|
proc step(self: Compiler): ASTNode =
|
||||||
|
@ -601,7 +603,6 @@ proc inferType(self: Compiler, node: Declaration): Type =
|
||||||
var node = FunDecl(node)
|
var node = FunDecl(node)
|
||||||
let resolved = self.resolve(node.name)
|
let resolved = self.resolve(node.name)
|
||||||
if resolved != nil:
|
if resolved != nil:
|
||||||
echo resolved[]
|
|
||||||
return resolved.valueType
|
return resolved.valueType
|
||||||
of NodeKind.varDecl:
|
of NodeKind.varDecl:
|
||||||
var node = VarDecl(node)
|
var node = VarDecl(node)
|
||||||
|
|
|
@ -19,6 +19,8 @@ import strutils
|
||||||
import parseutils
|
import parseutils
|
||||||
import strformat
|
import strformat
|
||||||
import tables
|
import tables
|
||||||
|
import terminal
|
||||||
|
|
||||||
|
|
||||||
import meta/token
|
import meta/token
|
||||||
import meta/errors
|
import meta/errors
|
||||||
|
@ -142,14 +144,19 @@ proc isAlphaNumeric(s: string): bool =
|
||||||
return false
|
return false
|
||||||
return true
|
return true
|
||||||
|
|
||||||
|
proc incLine(self: Lexer)
|
||||||
|
|
||||||
# Simple public getters used for error
|
# Simple public getters used for error
|
||||||
# formatting and whatnot
|
# formatting and whatnot
|
||||||
proc getStart*(self: Lexer): int = self.start
|
proc getStart*(self: Lexer): int = self.start
|
||||||
|
proc getFile*(self: Lexer): string = self.file
|
||||||
proc getCurrent*(self: Lexer): int = self.current
|
proc getCurrent*(self: Lexer): int = self.current
|
||||||
proc getLine*(self: Lexer): int = self.line
|
proc getLine*(self: Lexer): int = self.line
|
||||||
proc getSource*(self: Lexer): string = self.source
|
proc getSource*(self: Lexer): string = self.source
|
||||||
proc getRelPos*(self: Lexer, line: int): tuple[start, stop: int] = (if line >
|
proc getRelPos*(self: Lexer, line: int): tuple[start, stop: int] =
|
||||||
1: self.lines[line - 2] else: (start: 0, stop: self.current))
|
if self.tokens.len() == 0 or self.tokens[^1].kind != EndOfFile:
|
||||||
|
self.incLine()
|
||||||
|
return self.lines[line - 1]
|
||||||
|
|
||||||
|
|
||||||
proc newLexer*(self: Lexer = nil): Lexer =
|
proc newLexer*(self: Lexer = nil): Lexer =
|
||||||
|
@ -178,9 +185,9 @@ proc incLine(self: Lexer) =
|
||||||
## Increments the lexer's line
|
## Increments the lexer's line
|
||||||
## and updates internal line
|
## and updates internal line
|
||||||
## metadata
|
## metadata
|
||||||
self.lines.add((start: self.lastLine, stop: self.current))
|
self.lines.add((self.lastLine, self.current))
|
||||||
self.line += 1
|
|
||||||
self.lastLine = self.current
|
self.lastLine = self.current
|
||||||
|
self.line += 1
|
||||||
|
|
||||||
|
|
||||||
proc step(self: Lexer, n: int = 1): string =
|
proc step(self: Lexer, n: int = 1): string =
|
||||||
|
@ -196,7 +203,7 @@ proc step(self: Lexer, n: int = 1): string =
|
||||||
inc(self.current)
|
inc(self.current)
|
||||||
|
|
||||||
|
|
||||||
proc peek(self: Lexer, distance: int = 0, length: int = 1): string =
|
proc peek*(self: Lexer, distance: int = 0, length: int = 1): string =
|
||||||
## Returns a stream of characters of
|
## Returns a stream of characters of
|
||||||
## at most length bytes from the source
|
## at most length bytes from the source
|
||||||
## file, starting at the given distance,
|
## file, starting at the given distance,
|
||||||
|
@ -219,7 +226,7 @@ proc peek(self: Lexer, distance: int = 0, length: int = 1): string =
|
||||||
proc error(self: Lexer, message: string) =
|
proc error(self: Lexer, message: string) =
|
||||||
## Raises a lexing error with a formatted
|
## Raises a lexing error with a formatted
|
||||||
## error message
|
## error message
|
||||||
raise newException(LexingError, &"A fatal error occurred while parsing '{self.file}', line {self.line} at '{self.peek()}' -> {message}")
|
raise newException(LexingError, message)
|
||||||
|
|
||||||
|
|
||||||
proc check(self: Lexer, s: string, distance: int = 0): bool =
|
proc check(self: Lexer, s: string, distance: int = 0): bool =
|
||||||
|
@ -625,9 +632,11 @@ proc lex*(self: Lexer, source, file: string): seq[Token] =
|
||||||
self.symbols = symbols
|
self.symbols = symbols
|
||||||
self.source = source
|
self.source = source
|
||||||
self.file = file
|
self.file = file
|
||||||
|
self.lines = @[]
|
||||||
while not self.done():
|
while not self.done():
|
||||||
self.next()
|
self.next()
|
||||||
self.start = self.current
|
self.start = self.current
|
||||||
self.tokens.add(Token(kind: EndOfFile, lexeme: "",
|
self.tokens.add(Token(kind: EndOfFile, lexeme: "",
|
||||||
line: self.line, pos: (self.current, self.current)))
|
line: self.line, pos: (self.current, self.current)))
|
||||||
|
self.incLine()
|
||||||
return self.tokens
|
return self.tokens
|
||||||
|
|
|
@ -12,10 +12,11 @@
|
||||||
# See the License for the specific language governing permissions and
|
# See the License for the specific language governing permissions and
|
||||||
# limitations under the License.
|
# limitations under the License.
|
||||||
|
|
||||||
|
|
||||||
type
|
type
|
||||||
## Nim exceptions for internal JAPL failures
|
## Nim exceptions for internal Peon failures
|
||||||
NimVMException* = object of CatchableError
|
PeonException* = object of CatchableError
|
||||||
LexingError* = object of NimVMException
|
LexingError* = object of PeonException
|
||||||
ParseError* = object of NimVMException
|
ParseError* = object of PeonException
|
||||||
CompileError* = object of NimVMException
|
CompileError* = object of PeonException
|
||||||
SerializationError* = object of NimVMException
|
SerializationError* = object of PeonException
|
||||||
|
|
|
@ -150,6 +150,8 @@ proc getCurrent*(self: Parser): int {.inline.} = self.current
|
||||||
proc getCurrentToken*(self: Parser): Token {.inline.} = (if self.getCurrent() >=
|
proc getCurrentToken*(self: Parser): Token {.inline.} = (if self.getCurrent() >=
|
||||||
self.tokens.high() or
|
self.tokens.high() or
|
||||||
self.getCurrent() - 1 < 0: self.tokens[^1] else: self.tokens[self.current - 1])
|
self.getCurrent() - 1 < 0: self.tokens[^1] else: self.tokens[self.current - 1])
|
||||||
|
proc getCurrentFunction*(self: Parser): Declaration {.inline.} = self.currentFunction
|
||||||
|
proc getFile*(self: Parser): string {.inline.} = self.file
|
||||||
|
|
||||||
# Handy templates to make our life easier, thanks nim!
|
# Handy templates to make our life easier, thanks nim!
|
||||||
template endOfFile: Token = Token(kind: EndOfFile, lexeme: "", line: -1)
|
template endOfFile: Token = Token(kind: EndOfFile, lexeme: "", line: -1)
|
||||||
|
@ -167,10 +169,6 @@ proc peek(self: Parser, distance: int = 0): Token =
|
||||||
result = endOfFile
|
result = endOfFile
|
||||||
else:
|
else:
|
||||||
result = self.tokens[self.current + distance]
|
result = self.tokens[self.current + distance]
|
||||||
## Hack to ignore whitespace/tab
|
|
||||||
if result.kind in {TokenType.Whitespace, Tab}:
|
|
||||||
# self.current += 1
|
|
||||||
result = self.peek(distance + 1)
|
|
||||||
|
|
||||||
|
|
||||||
proc done(self: Parser): bool {.inline.} =
|
proc done(self: Parser): bool {.inline.} =
|
||||||
|
@ -192,15 +190,9 @@ proc step(self: Parser, n: int = 1): Token =
|
||||||
self.current += 1
|
self.current += 1
|
||||||
|
|
||||||
|
|
||||||
proc error(self: Parser, message: string) {.raises: [ParseError, ValueError].} =
|
proc error(self: Parser, message: string) {.raises: [ParseError].} =
|
||||||
## Raises a formatted ParseError exception
|
## Raises a ParseError exception
|
||||||
var lexeme = self.peek().lexeme
|
raise newException(ParseError, message)
|
||||||
var fn = ""
|
|
||||||
if self.currentFunction != nil:
|
|
||||||
if self.currentFunction.kind == NodeKind.funDecl:
|
|
||||||
fn = &"inside function '{FunDecl(self.currentFunction).name.token.lexeme}'"
|
|
||||||
var errorMessage = &"A fatal error occurred while parsing '{self.file}', {fn} line {self.peek().line} at '{lexeme}' -> {message}"
|
|
||||||
raise newException(ParseError, errorMessage)
|
|
||||||
|
|
||||||
|
|
||||||
# Why do we allow strings or enum members of TokenType? Well, it's simple:
|
# Why do we allow strings or enum members of TokenType? Well, it's simple:
|
||||||
|
@ -882,32 +874,31 @@ proc parseDeclArguments(self: Parser, arguments: var seq[tuple[name: IdentExpr,
|
||||||
|
|
||||||
|
|
||||||
proc funDecl(self: Parser, isAsync: bool = false, isGenerator: bool = false,
|
proc funDecl(self: Parser, isAsync: bool = false, isGenerator: bool = false,
|
||||||
isLambda: bool = false, isOperator: bool = false): Declaration =
|
isLambda: bool = false, isOperator: bool = false): Declaration = # Can't use just FunDecl because it can also return LambdaExpr!
|
||||||
## Parses functions, coroutines, generators, anonymous functions and operators
|
## Parses all types of functions, coroutines, generators and operators
|
||||||
|
## (with or without a name, where applicable)
|
||||||
let tok = self.peek(-1)
|
let tok = self.peek(-1)
|
||||||
var enclosingFunction = self.currentFunction
|
var enclosingFunction = self.currentFunction
|
||||||
var arguments: seq[tuple[name: IdentExpr, valueType: Expression,
|
var arguments: seq[tuple[name: IdentExpr, valueType: Expression,
|
||||||
mutable: bool, isRef: bool, isPtr: bool]] = @[]
|
mutable: bool, isRef: bool, isPtr: bool]] = @[]
|
||||||
var defaults: seq[Expression] = @[]
|
var defaults: seq[Expression] = @[]
|
||||||
var returnType: Expression
|
var returnType: Expression
|
||||||
if not isLambda and self.check(Identifier):
|
if not isLambda and self.match(Identifier):
|
||||||
# We do this extra check because we might
|
# We do this extra check because we might
|
||||||
# be called from a context where it's
|
# be called from a context where it's
|
||||||
# ambiguous whether we're parsing a declaration
|
# ambiguous whether we're parsing a declaration
|
||||||
# or an expression. Fortunately anonymous functions
|
# or an expression. Fortunately anonymous functions
|
||||||
# are nameless, so we can sort the ambiguity by checking
|
# are nameless, so we can sort the ambiguity by checking
|
||||||
# if there's an identifier after the keyword
|
# if there's an identifier after the keyword
|
||||||
self.expect(Identifier, &"expecting identifier after '{tok.lexeme}'")
|
self.currentFunction = newFunDecl(newIdentExpr(self.peek(-1)), arguments, defaults, newBlockStmt(@[], Token()),
|
||||||
self.checkDecl(not self.check("*"))
|
|
||||||
self.currentFunction = newFunDecl(nil, arguments, defaults, newBlockStmt(@[], Token()),
|
|
||||||
isAsync = isAsync,
|
isAsync = isAsync,
|
||||||
isGenerator = isGenerator,
|
isGenerator = isGenerator,
|
||||||
isPrivate = true,
|
isPrivate = true,
|
||||||
token = tok, pragmas = (@[]),
|
token = tok, pragmas = (@[]),
|
||||||
returnType = nil)
|
returnType = nil)
|
||||||
FunDecl(self.currentFunction).name = newIdentExpr(self.peek(-1))
|
|
||||||
if self.match("*"):
|
if self.match("*"):
|
||||||
FunDecl(self.currentFunction).isPrivate = false
|
FunDecl(self.currentFunction).isPrivate = false
|
||||||
|
self.checkDecl(FunDecl(self.currentFunction).isPrivate)
|
||||||
elif not isLambda and (self.check([LeftBrace, LeftParen]) or self.check(":")):
|
elif not isLambda and (self.check([LeftBrace, LeftParen]) or self.check(":")):
|
||||||
# We do a bit of hacking to pretend we never
|
# We do a bit of hacking to pretend we never
|
||||||
# wanted to parse this as a declaration in
|
# wanted to parse this as a declaration in
|
||||||
|
@ -916,7 +907,7 @@ proc funDecl(self: Parser, isAsync: bool = false, isGenerator: bool = false,
|
||||||
# go all the way up to primary(), which will
|
# go all the way up to primary(), which will
|
||||||
# call us back with isLambda=true, allowing us
|
# call us back with isLambda=true, allowing us
|
||||||
# to actually parse the function as an expression
|
# to actually parse the function as an expression
|
||||||
while not self.check(tok.kind):
|
while not self.check(tok.kind): # We rewind back to the token that caused us to be called
|
||||||
dec(self.current)
|
dec(self.current)
|
||||||
result = Declaration(self.expressionStatement())
|
result = Declaration(self.expressionStatement())
|
||||||
self.currentFunction = enclosingFunction
|
self.currentFunction = enclosingFunction
|
||||||
|
@ -1003,6 +994,8 @@ proc funDecl(self: Parser, isAsync: bool = false, isGenerator: bool = false,
|
||||||
self.error("cannot declare operator without arguments")
|
self.error("cannot declare operator without arguments")
|
||||||
elif FunDecl(result).returnType == nil:
|
elif FunDecl(result).returnType == nil:
|
||||||
self.error("operators must have a return type")
|
self.error("operators must have a return type")
|
||||||
|
elif isLambda:
|
||||||
|
self.error("cannot declare anonymous operator")
|
||||||
for argument in arguments:
|
for argument in arguments:
|
||||||
if argument.valueType == nil:
|
if argument.valueType == nil:
|
||||||
self.error(&"missing type declaration for '{argument.name.token.lexeme}' in function declaration")
|
self.error(&"missing type declaration for '{argument.name.token.lexeme}' in function declaration")
|
||||||
|
@ -1126,8 +1119,13 @@ proc declaration(self: Parser): Declaration =
|
||||||
|
|
||||||
|
|
||||||
proc parse*(self: Parser, tokens: seq[Token], file: string): seq[Declaration] =
|
proc parse*(self: Parser, tokens: seq[Token], file: string): seq[Declaration] =
|
||||||
## Parses a series of tokens into an AST node
|
## Parses a sequence of tokens into a sequence of AST nodes
|
||||||
self.tokens = tokens
|
self.tokens = @[]
|
||||||
|
# The parser is not designed to handle these tokens.
|
||||||
|
# Maybe create a separate syntax checker module?
|
||||||
|
for token in tokens:
|
||||||
|
if token.kind notin {TokenType.Whitespace, Tab}:
|
||||||
|
self.tokens.add(token)
|
||||||
self.file = file
|
self.file = file
|
||||||
self.current = 0
|
self.current = 0
|
||||||
self.currentLoop = LoopContext.None
|
self.currentLoop = LoopContext.None
|
||||||
|
|
63
src/main.nim
63
src/main.nim
|
@ -1,5 +1,4 @@
|
||||||
# Builtins & external libs
|
# Builtins & external libs
|
||||||
import sequtils
|
|
||||||
import strformat
|
import strformat
|
||||||
import strutils
|
import strutils
|
||||||
import terminal
|
import terminal
|
||||||
|
@ -25,11 +24,11 @@ proc fillSymbolTable(tokenizer: Lexer)
|
||||||
proc getLineEditor: LineEditor
|
proc getLineEditor: LineEditor
|
||||||
|
|
||||||
# Handy dandy compile-time constants
|
# Handy dandy compile-time constants
|
||||||
const debugLexer = true
|
const debugLexer = false
|
||||||
const debugParser = true
|
const debugParser = false
|
||||||
const debugCompiler = true
|
const debugCompiler = false
|
||||||
const debugSerializer = true
|
const debugSerializer = false
|
||||||
const debugRuntime = true
|
const debugRuntime = false
|
||||||
|
|
||||||
when debugSerializer:
|
when debugSerializer:
|
||||||
import nimSHA2
|
import nimSHA2
|
||||||
|
@ -125,26 +124,42 @@ when isMainModule:
|
||||||
# TODO: The code for error reporting completely
|
# TODO: The code for error reporting completely
|
||||||
# breaks down with multiline input, fix it
|
# breaks down with multiline input, fix it
|
||||||
except LexingError:
|
except LexingError:
|
||||||
# let lineNo = tokenizer.getLine()
|
let lineNo = tokenizer.getLine()
|
||||||
# let relPos = tokenizer.getRelPos(lineNo)
|
let relPos = tokenizer.getRelPos(lineNo)
|
||||||
# let line = tokenizer.getSource().splitLines()[lineNo - 1].strip()
|
let line = tokenizer.getSource().splitLines()[lineNo - 1].strip()
|
||||||
stderr.styledWriteLine(fgRed, getCurrentExceptionMsg())
|
stderr.styledWriteLine(fgRed, "A fatal error occurred while parsing ", fgYellow, &"'{tokenizer.getFile()}'", fgRed, ", module ",
|
||||||
# echo &"Source line: {line}"
|
fgYellow, &"'{tokenizer.getFile()}'", fgRed, ", line ", fgYellow, $lineNo, fgRed, " at ", fgYellow, &"'{tokenizer.peek()}'",
|
||||||
# echo " ".repeat(relPos.start + len("Source line: ")) & "^".repeat(relPos.stop - relPos.start)
|
fgRed, ": ", fgGreen , getCurrentExceptionMsg())
|
||||||
|
styledEcho fgBlue, "Source line: " , fgDefault, line
|
||||||
|
styledEcho fgCyan, " ".repeat(len("Source line: ")) & "^".repeat(relPos.stop - relPos.start)
|
||||||
except ParseError:
|
except ParseError:
|
||||||
# let lineNo = parser.getCurrentToken().line
|
let lexeme = parser.getCurrentToken().lexeme
|
||||||
# let relPos = tokenizer.getRelPos(lineNo)
|
let lineNo = parser.getCurrentToken().line
|
||||||
# let line = tokenizer.getSource().splitLines()[lineNo - 1].strip()
|
let relPos = tokenizer.getRelPos(lineNo)
|
||||||
stderr.styledWriteLine(fgRed, getCurrentExceptionMsg())
|
let fn = parser.getCurrentFunction()
|
||||||
# echo &"Source line: {line}"
|
let line = tokenizer.getSource().splitLines()[lineNo - 1].strip()
|
||||||
# echo " ".repeat(relPos.start + len("Source line: ")) & "^".repeat(relPos.stop - parser.getCurrentToken().lexeme.len())
|
var fnMsg = ""
|
||||||
|
if fn != nil and fn.kind == funDecl:
|
||||||
|
fnMsg &= &"in function '{FunDecl(fn).name.token.lexeme}'"
|
||||||
|
stderr.styledWriteLine(fgRed, "A fatal error occurred while parsing ", fgYellow, &"'{parser.getFile()}'", fgRed, ", module ",
|
||||||
|
fgYellow, &"'{parser.getFile()}'", fgRed, ", line ", fgYellow, $lineNo, fgRed, " at ", fgYellow, &"'{lexeme}'",
|
||||||
|
fgRed, ": ", fgGreen , getCurrentExceptionMsg())
|
||||||
|
styledEcho fgBlue, "Source line: " , fgDefault, line
|
||||||
|
styledEcho fgCyan, " ".repeat(len("Source line: ")) & "^".repeat(relPos.stop - relPos.start)
|
||||||
except CompileError:
|
except CompileError:
|
||||||
# let lineNo = compiler.getCurrentNode().token.line
|
let lexeme = compiler.getCurrentNode().token.lexeme
|
||||||
# let relPos = tokenizer.getRelPos(lineNo)
|
let lineNo = compiler.getCurrentNode().token.line
|
||||||
# let line = tokenizer.getSource().splitLines()[lineNo - 1].strip()
|
let relPos = tokenizer.getRelPos(lineNo)
|
||||||
stderr.styledWriteLine(fgRed, getCurrentExceptionMsg())
|
let line = tokenizer.getSource().splitLines()[lineNo - 1].strip()
|
||||||
# echo &"Source line: {line}"
|
var fn = compiler.getCurrentFunction()
|
||||||
# echo " ".repeat(relPos.start + len("Source line: ")) & "^".repeat(relPos.stop - compiler.getCurrentNode().token.lexeme.len())
|
var fnMsg = ""
|
||||||
|
if fn != nil and fn.kind == funDecl:
|
||||||
|
fnMsg &= &"in function '{FunDecl(fn).name.token.lexeme}'"
|
||||||
|
stderr.styledWriteLine(fgRed, "A fatal error occurred while compiling ", fgYellow, &"'{compiler.getFile()}'", fgRed, ", module ",
|
||||||
|
fgYellow, &"'{compiler.getModule()}'", fgRed, ", line ", fgYellow, $lineNo, fgRed, " at ", fgYellow, &"'{lexeme}'",
|
||||||
|
fgRed, ": ", fgGreen , getCurrentExceptionMsg())
|
||||||
|
styledEcho fgBlue, "Source line: " , fgDefault, line
|
||||||
|
styledEcho fgCyan, " ".repeat(len("Source line: ")) & "^".repeat(relPos.stop - relPos.start)
|
||||||
except SerializationError:
|
except SerializationError:
|
||||||
stderr.styledWriteLine(fgRed, getCurrentExceptionMsg())
|
stderr.styledWriteLine(fgRed, getCurrentExceptionMsg())
|
||||||
quit(0)
|
quit(0)
|
||||||
|
|
Loading…
Reference in New Issue