diff --git a/src/frontend/compiler.nim b/src/frontend/compiler.nim index c9736a7..c7693f5 100644 --- a/src/frontend/compiler.nim +++ b/src/frontend/compiler.nim @@ -176,10 +176,20 @@ type frames: seq[int] # Compiler procedures called by pragmas compilerProcs: TableRef[string, proc (self: Compiler, pragma: Pragma, node: ASTNode)] + # Stores line data + lines: seq[tuple[start, stop: int]] + # The source of the current module + source: string + CompileError* = ref object of PeonException + compiler*: Compiler + node*: ASTNode + file*: string + module*: string ## Forward declarations -proc compile*(self: Compiler, ast: seq[Declaration], file: string): Chunk +proc compile*(self: Compiler, ast: seq[Declaration], file: string, lines: seq[tuple[start, stop: int]], source: string, chunk: Chunk = nil, + standalone: bool = true): Chunk proc expression(self: Compiler, node: Expression) proc statement(self: Compiler, node: Statement) proc declaration(self: Compiler, node: Declaration) @@ -207,6 +217,8 @@ proc newCompiler*(enableOptimizations: bool = true, replMode: bool = false): Com result.file = "" result.names = @[] result.scopeDepth = 0 + result.frames = @[0] + result.lines = @[] result.currentFunction = nil result.enableOptimizations = enableOptimizations result.replMode = replMode @@ -214,7 +226,7 @@ proc newCompiler*(enableOptimizations: bool = true, replMode: bool = false): Com result.compilerProcs = newTable[string, proc (self: Compiler, pragma: Pragma, node: ASTNode)]() result.compilerProcs["magic"] = handleMagicPragma result.compilerProcs["pure"] = handlePurePragma - + result.source = "" proc compileModule(self: Compiler, filename: string) @@ -225,7 +237,9 @@ proc getCurrentNode*(self: Compiler): ASTNode = (if self.current >= proc getCurrentFunction*(self: Compiler): Declaration {.inline.} = (if self.currentFunction.isNil(): nil else: self.currentFunction.fun) proc getFile*(self: Compiler): string {.inline.} = self.file proc getModule*(self: Compiler): string {.inline.} = self.currentModule - +proc getLines*(self: Compiler): seq[tuple[start, stop: int]] = self.lines +proc getSource*(self: Compiler): string = self.source +proc getRelPos*(self: Compiler, line: int): tuple[start, stop: int] = self.lines[line - 1] ## Utility functions @@ -249,7 +263,7 @@ proc done(self: Compiler): bool {.inline.} = proc error(self: Compiler, message: string, node: ASTNode = nil) {.raises: [CompileError], inline.} = ## Raises a CompileError exception - raise CompileError(msg: message, node: if node.isNil(): self.getCurrentNode() else: node, file: self.file, module: self.currentModule) + raise CompileError(msg: message, node: if node.isNil(): self.getCurrentNode() else: node, file: self.file, module: self.currentModule, compiler: self) proc step(self: Compiler): ASTNode {.inline.} = @@ -329,7 +343,7 @@ proc emitConstant(self: Compiler, obj: Expression, kind: Type) = let str = LiteralExpr(obj).literal.lexeme if str.len() >= 16777216: self.error("string constants cannot be larger than 16777215 bytes") - self.emitBytes(LiteralExpr(obj).literal.lexeme.len().toTriple()) + self.emitBytes(str.len().toTriple()) of Float32: self.emitByte(LoadFloat32) of Float64: @@ -550,6 +564,8 @@ proc toIntrinsic(name: string): Type = return Type(kind: Bool) elif name == "typevar": return Type(kind: Typevar) + elif name == "string": + return Type(kind: String) else: return nil @@ -591,6 +607,8 @@ proc inferType(self: Compiler, node: LiteralExpr): Type = return Type(kind: TypeKind.Nan) of infExpr: return Type(kind: TypeKind.Inf) + of strExpr: + return Type(kind: String) else: discard # TODO @@ -1519,6 +1537,8 @@ proc importStmt(self: Compiler, node: ImportStmt) = self.compileModule(filename) except IOError: self.error(&"""could not import '{filename}': {getCurrentExceptionMsg()}""") + except OSError: + self.error(&"""could not import '{filename}': {getCurrentExceptionMsg()} [errno {osLastError()}]""") proc statement(self: Compiler, node: Statement) = @@ -1797,43 +1817,22 @@ proc declaration(self: Compiler, node: Declaration) = self.statement(Statement(node)) -proc compileModule(self: Compiler, filename: string) = - ## Compiles an imported module into an existing chunk. - ## A temporary compiler object is initialized internally - ## and its state is copied from the self argument - var lexer = newLexer() - var parser = newParser() - var compiler = newCompiler() - lexer.fillSymbolTable() - compiler.chunk = self.chunk - compiler.ast = parser.parse(lexer.lex(readFile(filename), filename), filename) - compiler.file = filename - compiler.names = @[] - compiler.scopeDepth = 0 - compiler.currentFunction = nil - compiler.currentModule = compiler.file.extractFilename() - compiler.current = 0 - compiler.frames = @[0] - while not compiler.done(): - compiler.declaration(Declaration(compiler.step())) - self.names &= compiler.names - self.closedOver &= compiler.closedOver - compiler.endScope() +proc terminateProgram(self: Compiler, pos: int) = + ## Utility to terminate a peon program + ## compiled as the main module + self.endScope() + self.patchReturnAddress(pos) + self.emitByte(OpCode.Return) + self.emitByte(0) # Entry point has no return value (TODO: Add easter eggs, cuz why not) +proc beginProgram(self: Compiler): int = + ## Utility to begin a peon program + ## compiled as the main module. + ## Returns a dummy return address of + ## the implicit main to be patched by + ## terminateProgram -proc compile*(self: Compiler, ast: seq[Declaration], file: string): Chunk = - ## Compiles a sequence of AST nodes into a chunk - ## object - self.chunk = newChunk() - self.ast = ast - self.file = file - self.names = @[] - self.scopeDepth = 0 - self.currentFunction = nil - self.currentModule = self.file.extractFilename() - self.current = 0 - self.frames = @[0] # Every peon program has a hidden entry point in # which user code is wrapped. Think of it as if # peon is implicitly writing the main() function @@ -1863,14 +1862,51 @@ proc compile*(self: Compiler, ast: seq[Declaration], file: string): Chunk = self.emitByte(LoadFunction) self.emitBytes(main.codePos.toTriple()) self.emitByte(LoadReturnAddress) - let pos = self.chunk.code.len() + result = self.chunk.code.len() self.emitBytes(0.toQuad()) self.emitByte(Call) self.emitBytes(0.toTriple()) + + +proc compile*(self: Compiler, ast: seq[Declaration], file: string, lines: seq[tuple[start, stop: int]], source: string, chunk: Chunk = nil, + standalone: bool = true): Chunk = + ## Compiles a sequence of AST nodes into a chunk + ## object + if chunk.isNil(): + self.chunk = newChunk() + else: + self.chunk = chunk + self.ast = ast + self.file = file + self.names = @[] + self.scopeDepth = 0 + self.currentFunction = nil + self.currentModule = self.file.extractFilename() + self.current = 0 + self.frames = @[0] + self.lines = lines + self.source = source + var pos: int + if standalone: + pos = self.beginProgram() while not self.done(): self.declaration(Declaration(self.step())) - self.endScope() - self.patchReturnAddress(pos) - self.emitByte(OpCode.Return) - self.emitByte(0) # Entry point has no return value (TODO: Add easter eggs, cuz why not) + if standalone: + self.terminateProgram(pos) result = self.chunk + + +proc compileModule(self: Compiler, filename: string) = + ## Compiles an imported module into an existing chunk. + ## A temporary compiler object is initialized internally + var lexer = newLexer() + var parser = newParser() + var compiler = newCompiler() + lexer.fillSymbolTable() + let source = readFile(joinPath(splitPath(self.file).head, filename)) + let tokens = lexer.lex(source, filename) + let ast = parser.parse(tokens, filename, lexer.getLines(), source) + discard compiler.compile(ast, filename, lexer.getLines(), source, standalone=false) + self.names &= compiler.names + self.closedOver &= compiler.closedOver + compiler.endScope() \ No newline at end of file diff --git a/src/frontend/lexer.nim b/src/frontend/lexer.nim index 3e02341..d6218ad 100644 --- a/src/frontend/lexer.nim +++ b/src/frontend/lexer.nim @@ -52,6 +52,12 @@ type lines: seq[tuple[start, stop: int]] lastLine: int spaces: int + LexingError* = ref object of PeonException + ## A lexing error + lexer*: Lexer + file*: string + lexeme*: string + line*: int proc newSymbolTable: SymbolTable = @@ -154,6 +160,7 @@ proc getStart*(self: Lexer): int = self.start proc getFile*(self: Lexer): string = self.file proc getCurrent*(self: Lexer): int = self.current proc getLine*(self: Lexer): int = self.line +proc getLines*(self: Lexer): seq[tuple[start, stop: int]] = self.lines proc getSource*(self: Lexer): string = self.source proc getRelPos*(self: Lexer, line: int): tuple[start, stop: int] = if self.tokens.len() == 0 or self.tokens[^1].kind != EndOfFile: @@ -186,9 +193,9 @@ proc done(self: Lexer): bool = proc incLine(self: Lexer) = ## Increments the lexer's line - ## and updates internal line - ## metadata - self.lines.add((self.lastLine, self.current)) + ## counter and updates internal + ## line metadata + self.lines.add((self.lastLine, self.current - 1)) self.lastLine = self.current self.line += 1 @@ -230,7 +237,7 @@ proc peek(self: Lexer, distance: int = 0, length: int = 1): string = proc error(self: Lexer, message: string) = ## Raises a lexing error with info ## for error messages - raise LexingError(msg: message, line: self.line, file: self.file, lexeme: self.peek()) + raise LexingError(msg: message, line: self.line, file: self.file, lexeme: self.peek(), lexer: self) proc check(self: Lexer, s: string, distance: int = 0): bool = diff --git a/src/frontend/meta/errors.nim b/src/frontend/meta/errors.nim index 3b6d57c..f89a075 100644 --- a/src/frontend/meta/errors.nim +++ b/src/frontend/meta/errors.nim @@ -11,24 +11,10 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -import token -import ast type ## Nim exceptions for internal Peon failures PeonException* = ref object of CatchableError - LexingError* = ref object of PeonException - file*: string - lexeme*: string - line*: int - ParseError* = ref object of PeonException - file*: string - token*: Token - module*: string - CompileError* = ref object of PeonException - node*: ASTNode - file*: string - module*: string SerializationError* = ref object of PeonException file*: string diff --git a/src/frontend/parser.nim b/src/frontend/parser.nim index a8b197d..ebd4ee4 100644 --- a/src/frontend/parser.nim +++ b/src/frontend/parser.nim @@ -89,6 +89,15 @@ type operators: OperatorTable # The AST node tree: seq[Declaration] + # Stores line data + lines: seq[tuple[start, stop: int]] + # The source of the current module + source: string + ParseError* = ref object of PeonException + parser*: Parser + file*: string + token*: Token + module*: string proc newOperatorTable: OperatorTable = @@ -146,6 +155,7 @@ proc newParser*: Parser = result.scopeDepth = 0 result.operators = newOperatorTable() result.tree = @[] + result.source = "" # Public getters for improved error formatting @@ -155,8 +165,10 @@ proc getCurrentToken*(self: Parser): Token {.inline.} = (if self.getCurrent() >= self.getCurrent() - 1 < 0: self.tokens[^1] else: self.tokens[self.current - 1]) proc getCurrentFunction*(self: Parser): Declaration {.inline.} = self.currentFunction proc getFile*(self: Parser): string {.inline.} = self.file -proc getModule*(self: Parser): string {.inline.} = self.getFile().extractFilename() - +proc getModule*(self: Parser): string {.inline.} = self.getFile().splitFile().name +proc getLines*(self: Parser): seq[tuple[start, stop: int]] = self.lines +proc getSource*(self: Parser): string = self.source +proc getRelPos*(self: Parser, line: int): tuple[start, stop: int] = self.lines[line - 1] template endOfFile: Token = Token(kind: EndOfFile, lexeme: "", line: -1) template endOfLine(msg: string) = self.expect(Semicolon, msg) @@ -196,7 +208,7 @@ proc step(self: Parser, n: int = 1): Token = proc error(self: Parser, message: string, token: Token = nil) {.raises: [ParseError].} = ## Raises a ParseError exception - raise ParseError(msg: message, token: if token.isNil(): self.getCurrentToken() else: token, file: self.file, module: self.getModule()) + raise ParseError(msg: message, token: if token.isNil(): self.getCurrentToken() else: token, file: self.file, module: self.getModule(), parser: self) # Why do we allow strings or enum members of TokenType? Well, it's simple: @@ -664,7 +676,7 @@ proc forEachStmt(self: Parser): Statement = self.currentLoop = enclosingLoop -proc parse*(self: Parser, tokens: seq[Token], file: string): seq[Declaration] +proc parse*(self: Parser, tokens: seq[Token], file: string, lines: seq[tuple[start, stop: int]], source: string): seq[Declaration] proc findOperators(self: Parser, tokens: seq[Token]) @@ -684,12 +696,15 @@ proc importStmt(self: Parser, fromStmt: bool = false): Statement = var filename = ImportStmt(result).moduleName.token.lexeme & ".pn" var lexer = newLexer() lexer.fillSymbolTable() + let path = joinPath(splitPath(self.file).head, filename) # TODO: This is obviously horrible. It's just a test try: - self.findOperators(lexer.lex(readFile(filename), filename)) + self.findOperators(lexer.lex(readFile(path), filename)) except IOError: self.error(&"""could not import '{filename}': {getCurrentExceptionMsg()}""") - + except OSError: + self.error(&"""could not import '{filename}': {getCurrentExceptionMsg()} [errno {osLastError()}]""") + proc tryStmt(self: Parser): Statement = ## Parses try/except/else/finally blocks @@ -1223,7 +1238,7 @@ proc findOperators(self: Parser, tokens: seq[Token]) = self.error("invalid state: found malformed tokenizer input while looking for operators (missing EOF)", token) -proc parse*(self: Parser, tokens: seq[Token], file: string): seq[Declaration] = +proc parse*(self: Parser, tokens: seq[Token], file: string, lines: seq[tuple[start, stop: int]], source: string): seq[Declaration] = ## Parses a sequence of tokens into a sequence of AST nodes self.tokens = tokens self.file = file @@ -1233,6 +1248,8 @@ proc parse*(self: Parser, tokens: seq[Token], file: string): seq[Declaration] = self.scopeDepth = 0 self.operators = newOperatorTable() self.tree = @[] + self.source = source + self.lines = lines self.findOperators(tokens) while not self.done(): self.tree.add(self.declaration()) diff --git a/src/main.nim b/src/main.nim index 05617c6..ee701fb 100644 --- a/src/main.nim +++ b/src/main.nim @@ -95,7 +95,7 @@ proc repl(vm: PeonVM = newPeonVM()) = break styledEcho fgGreen, "\t", $token echo "" - tree = parser.parse(tokens, "stdin") + tree = parser.parse(tokens, "stdin", tokenizer.getLines(), input) if tree.len() == 0: continue when debugParser: @@ -103,7 +103,7 @@ proc repl(vm: PeonVM = newPeonVM()) = for node in tree: styledEcho fgGreen, "\t", $node echo "" - compiled = compiler.compile(tree, "stdin") + compiled = compiler.compile(tree, "stdin", tokenizer.getLines(), input) when debugCompiler: styledEcho fgCyan, "Compilation step:\n" debugger.disassembleChunk(compiled, "stdin") @@ -218,7 +218,7 @@ proc runFile(f: string, interactive: bool = false, fromString: bool = false) = break styledEcho fgGreen, "\t", $token echo "" - tree = parser.parse(tokens, f) + tree = parser.parse(tokens, f, tokenizer.getLines(), input) if tree.len() == 0: return when debugParser: @@ -226,7 +226,7 @@ proc runFile(f: string, interactive: bool = false, fromString: bool = false) = for node in tree: styledEcho fgGreen, "\t", $node echo "" - compiled = compiler.compile(tree, f) + compiled = compiler.compile(tree, f, tokenizer.getLines(), input) when debugCompiler: styledEcho fgCyan, "Compilation step:\n" debugger.disassembleChunk(compiled, f) @@ -267,8 +267,8 @@ proc runFile(f: string, interactive: bool = false, fromString: bool = false) = except LexingError: input = "" let exc = LexingError(getCurrentException()) - let relPos = tokenizer.getRelPos(exc.line) - let line = tokenizer.getSource().splitLines()[exc.line - 1].strip(chars={'\n'}) + let relPos = exc.lexer.getRelPos(exc.line) + let line = exc.lexer.getSource().splitLines()[exc.line - 1].strip(chars={'\n'}) stderr.styledWriteLine(fgRed, "A fatal error occurred while parsing ", fgYellow, &"'{exc.file}'", fgRed, ", module ", fgYellow, &"'{exc.file.extractFilename()}'", fgRed, ", line ", fgYellow, $exc.line, fgRed, " at ", fgYellow, &"'{exc.lexeme.escape()}'", fgRed, ": ", fgGreen , getCurrentExceptionMsg()) @@ -279,14 +279,14 @@ proc runFile(f: string, interactive: bool = false, fromString: bool = false) = let exc = ParseError(getCurrentException()) let lexeme = exc.token.lexeme let lineNo = exc.token.line - let relPos = tokenizer.getRelPos(lineNo) + let relPos = exc.parser.getRelPos(lineNo) let fn = parser.getCurrentFunction() - let line = tokenizer.getSource().splitLines()[lineNo - 1].strip(chars={'\n'}) + let line = exc.parser.getSource().splitLines()[lineNo - 1].strip(chars={'\n'}) var fnMsg = "" if fn != nil and fn.kind == funDecl: fnMsg &= &"in function '{FunDecl(fn).name.token.lexeme}'" stderr.styledWriteLine(fgRed, "A fatal error occurred while parsing ", fgYellow, &"'{exc.file}'", fgRed, ", module ", - fgYellow, &"'{exc.file}'", fgRed, ", line ", fgYellow, $lineNo, fgRed, " at ", fgYellow, &"'{lexeme}'", + fgYellow, &"'{exc.module}'", fgRed, ", line ", fgYellow, $lineNo, fgRed, " at ", fgYellow, &"'{lexeme}'", fgRed, ": ", fgGreen , getCurrentExceptionMsg()) styledEcho fgBlue, "Source line: " , fgDefault, line styledEcho fgCyan, " ".repeat(len("Source line: ")) & "^".repeat(relPos.stop - relPos.start) @@ -294,9 +294,9 @@ proc runFile(f: string, interactive: bool = false, fromString: bool = false) = let exc = CompileError(getCurrentException()) let lexeme = exc.node.token.lexeme let lineNo = exc.node.token.line - let relPos = tokenizer.getRelPos(lineNo) - let line = tokenizer.getSource().splitLines()[lineNo - 1].strip(chars={'\n'}) - var fn = compiler.getCurrentFunction() + let relPos = exc.compiler.getRelPos(lineNo) + let line = exc.compiler.getSource().splitLines()[lineNo - 1].strip(chars={'\n'}) + var fn = exc.compiler.getCurrentFunction() var fnMsg = "" if fn != nil and fn.kind == funDecl: fnMsg &= &"in function '{FunDecl(fn).name.token.lexeme}'" diff --git a/tests/import/lib.pn b/tests/import/lib.pn index b19abf8..45393f4 100644 --- a/tests/import/lib.pn +++ b/tests/import/lib.pn @@ -36,4 +36,3 @@ fn print*(x: float) { fn print*(x: int) { #pragma[magic: "GenericPrint"] } -