From 13b432b2d24b244a83a19224ecbe75fcd0f91671 Mon Sep 17 00:00:00 2001 From: Mattia Giambirtone Date: Tue, 16 Aug 2022 12:20:07 +0200 Subject: [PATCH] Basic support for actual incremental compilation --- src/frontend/compiler.nim | 59 ++++++++++++++++++++-------------- src/frontend/meta/bytecode.nim | 2 +- src/frontend/parser.nim | 15 +++++++-- src/main.nim | 32 +++++++++--------- tests/std.pn | 1 + 5 files changed, 64 insertions(+), 45 deletions(-) diff --git a/src/frontend/compiler.nim b/src/frontend/compiler.nim index 26fcc8b..67b8454 100644 --- a/src/frontend/compiler.nim +++ b/src/frontend/compiler.nim @@ -26,6 +26,7 @@ import algorithm import parseutils import strutils import sequtils +import sets import os @@ -180,6 +181,8 @@ type lines: seq[tuple[start, stop: int]] # The source of the current module source: string + # Currently imported modules + modules: HashSet[string] CompileError* = ref object of PeonException compiler*: Compiler node*: ASTNode @@ -189,7 +192,7 @@ type ## Forward declarations proc compile*(self: Compiler, ast: seq[Declaration], file: string, lines: seq[tuple[start, stop: int]], source: string, chunk: Chunk = nil, - standalone: bool = true): Chunk + terminateScope: bool = true, incremental: bool = false): Chunk proc expression(self: Compiler, node: Expression) proc statement(self: Compiler, node: Statement) proc declaration(self: Compiler, node: Declaration) @@ -1302,7 +1305,6 @@ proc declareName(self: Compiler, node: Declaration, mutable: bool = false) = name.valueType = self.inferType(argument.valueType) # If it's still nil, it's an error! if name.valueType.isNil(): - echo argument.name.token self.error(&"cannot determine the type of argument '{argument.name.token.lexeme}'", argument.name) fn.valueType.args.add((argument.name.token.lexeme, name.valueType)) else: @@ -1936,18 +1938,18 @@ proc declaration(self: Compiler, node: Declaration) = self.statement(Statement(node)) -proc terminateProgram(self: Compiler, pos: int) = +proc terminateProgram(self: Compiler, pos: int, terminateScope: bool = true) = ## Utility to terminate a peon program - ## compiled as the main module - self.endScope() + if terminateScope: + self.endScope() self.patchReturnAddress(pos) self.emitByte(OpCode.Return) self.emitByte(0) # Entry point has no return value (TODO: Add easter eggs, cuz why not) -proc beginProgram(self: Compiler): int = +proc beginProgram(self: Compiler, incremental: bool = false): int = ## Utility to begin a peon program - ## compiled as the main module. + ## compiled. ## Returns a dummy return address of ## the implicit main to be patched by ## terminateProgram @@ -1962,7 +1964,11 @@ proc beginProgram(self: Compiler): int = # of where our program ends (which we don't know yet). # To fix this, we emit dummy offsets and patch them # later, once we know the boundaries of our hidden main() - var main = Name(depth: 0, + var main: Name + if incremental: + main = self.names[0] + else: + main = Name(depth: 0, isPrivate: true, isConst: false, isLet: false, @@ -1977,18 +1983,19 @@ proc beginProgram(self: Compiler): int = name: newIdentExpr(Token(lexeme: "", kind: Identifier)), isFunDecl: true, line: -1) - self.names.add(main) - self.emitByte(LoadFunction) - self.emitBytes(main.codePos.toTriple()) - self.emitByte(LoadReturnAddress) - result = self.chunk.code.len() - self.emitBytes(0.toQuad()) - self.emitByte(Call) - self.emitBytes(0.toTriple()) + self.names.add(main) + self.emitByte(LoadFunction) + self.emitBytes(main.codePos.toTriple()) + self.emitByte(LoadReturnAddress) + # result = self.chunk.code.len() + self.emitBytes(0.toQuad()) + self.emitByte(Call) + self.emitBytes(0.toTriple()) + result = 5 -proc compile*(self: Compiler, ast: seq[Declaration], file: string, lines: seq[tuple[start, stop: int]], source: string, chunk: Chunk = nil, - standalone: bool = true): Chunk = +proc compile*(self: Compiler, ast: seq[Declaration], file: string, lines: seq[tuple[start, stop: int]], source: string, chunk: Chunk = nil, + terminateScope: bool = true, incremental: bool = false): Chunk = ## Compiles a sequence of AST nodes into a chunk ## object if chunk.isNil(): @@ -1997,7 +2004,9 @@ proc compile*(self: Compiler, ast: seq[Declaration], file: string, lines: seq[tu self.chunk = chunk self.ast = ast self.file = file - self.names = @[] + var terminateScope = terminateScope + if incremental: + terminateScope = false self.scopeDepth = 0 self.currentFunction = nil self.currentModule = self.file.extractFilename() @@ -2005,13 +2014,13 @@ proc compile*(self: Compiler, ast: seq[Declaration], file: string, lines: seq[tu self.frames = @[0] self.lines = lines self.source = source - var pos: int - if standalone: - pos = self.beginProgram() + let pos = self.beginProgram(incremental=incremental) + if incremental: + for i in countup(1, 2): + discard self.chunk.code.pop() while not self.done(): self.declaration(Declaration(self.step())) - if standalone: - self.terminateProgram(pos) + self.terminateProgram(pos, terminateScope) result = self.chunk @@ -2025,7 +2034,7 @@ proc compileModule(self: Compiler, filename: string) = let source = readFile(joinPath(splitPath(self.file).head, filename)) let tokens = lexer.lex(source, filename) let ast = parser.parse(tokens, filename, lexer.getLines(), source) - discard compiler.compile(ast, filename, lexer.getLines(), source, standalone=false) + discard compiler.compile(ast, filename, lexer.getLines(), source, terminateScope=false) self.names &= compiler.names self.closedOver &= compiler.closedOver compiler.endScope() \ No newline at end of file diff --git a/src/frontend/meta/bytecode.nim b/src/frontend/meta/bytecode.nim index 0291441..9e83e44 100644 --- a/src/frontend/meta/bytecode.nim +++ b/src/frontend/meta/bytecode.nim @@ -293,7 +293,7 @@ const simpleInstructions* = {Return, LoadNil, LessOrEqualFloat64, GreaterOrEqualFloat32, LessOrEqualFloat32, - SysClock64, + SysClock64, NegInt16 } # Constant instructions are instructions that operate on the bytecode constant table diff --git a/src/frontend/parser.nim b/src/frontend/parser.nim index 3dc1165..03eaaf3 100644 --- a/src/frontend/parser.nim +++ b/src/frontend/parser.nim @@ -277,7 +277,7 @@ proc expect[T: TokenType or string](self: Parser, kind: T, message: string = "") self.error(message) -proc expect[T: TokenType or string](self: Parser, kind: openarray[T], message: string = "") = +proc expect[T: TokenType or string](self: Parser, kind: openarray[T], message: string = "") {.used.} = ## Behaves like self.expect(), except that ## an error is raised only if none of the ## given token kinds matches @@ -450,6 +450,7 @@ proc call(self: Parser): Expression = ## Operator parsing handlers proc unary(self: Parser): Expression = + ## Parses unary expressions if self.peek().kind == Symbol and self.peek().lexeme in self.operators.tokens: result = newUnaryExpr(self.step(), self.unary()) else: @@ -457,6 +458,7 @@ proc unary(self: Parser): Expression = proc parsePow(self: Parser): Expression = + ## Parses power expressions result = self.unary() var operator: Token var right: Expression @@ -467,6 +469,8 @@ proc parsePow(self: Parser): Expression = proc parseMul(self: Parser): Expression = + ## Parses multiplication and division + ## expressions result = self.parsePow() var operator: Token var right: Expression @@ -477,6 +481,8 @@ proc parseMul(self: Parser): Expression = proc parseAdd(self: Parser): Expression = + ## Parses addition and subtraction + ## expressions result = self.parseMul() var operator: Token var right: Expression @@ -487,6 +493,7 @@ proc parseAdd(self: Parser): Expression = proc parseCmp(self: Parser): Expression = + ## Parses comparison expressions result = self.parseAdd() var operator: Token var right: Expression @@ -497,6 +504,7 @@ proc parseCmp(self: Parser): Expression = proc parseAnd(self: Parser): Expression = + ## Parses logical and expressions result = self.parseCmp() var operator: Token var right: Expression @@ -507,6 +515,7 @@ proc parseAnd(self: Parser): Expression = proc parseOr(self: Parser): Expression = + ## Parses logical or expressions result = self.parseAnd() var operator: Token var right: Expression @@ -517,6 +526,7 @@ proc parseOr(self: Parser): Expression = proc parseAssign(self: Parser): Expression = + ## Parses assignment expressions result = self.parseOr() if self.peek().kind == Symbol and self.operators.getPrecedence(self.peek().lexeme) == Assign: let tok = self.step() @@ -531,6 +541,7 @@ proc parseAssign(self: Parser): Expression = proc parseArrow(self: Parser): Expression = + ## Parses arrow expressions result = self.parseAssign() var operator: Token var right: Expression @@ -546,7 +557,7 @@ proc parseArrow(self: Parser): Expression = proc assertStmt(self: Parser): Statement = ## Parses "assert" statements, which ## raise an error if the expression - ## fed into them is falsey + ## fed into them is false let tok = self.peek(-1) var expression = self.expression() endOfLine("missing statement terminator after 'assert'") diff --git a/src/main.nim b/src/main.nim index 99b7104..212bdb0 100644 --- a/src/main.nim +++ b/src/main.nim @@ -35,9 +35,6 @@ const debugSerializer {.booldefine.} = false const debugRuntime {.booldefine.} = false -when debugSerializer: - import nimSHA2 - proc repl(vm: PeonVM = newPeonVM()) = styledEcho fgMagenta, "Welcome into the peon REPL!" @@ -45,7 +42,7 @@ proc repl(vm: PeonVM = newPeonVM()) = keep = true tokens: seq[Token] = @[] tree: seq[Declaration] = @[] - compiled: Chunk + compiled: Chunk = newChunk() serialized: Serialized tokenizer = newLexer() parser = newParser() @@ -55,6 +52,7 @@ proc repl(vm: PeonVM = newPeonVM()) = editor = getLineEditor() input: string current: string + incremental: bool = false tokenizer.fillSymbolTable() editor.bindEvent(jeQuit): stdout.styledWriteLine(fgGreen, "Goodbye!") @@ -71,19 +69,16 @@ proc repl(vm: PeonVM = newPeonVM()) = # so that you can, for example, define a function # then press enter and use it at the next iteration # of the read loop - current = editor.read() - if current.len() == 0: + input = editor.read() + if input.len() == 0: continue - elif current == "#clearInput": - input = "" + elif input == "#reset": + compiled = newChunk() + incremental = false continue - elif current == "#clear": + elif input == "#clear": stdout.write("\x1Bc") continue - elif current == "#showInput": - echo input - continue - input &= &"{current}\n" tokens = tokenizer.lex(input, "stdin") if tokens.len() == 0: continue @@ -103,14 +98,14 @@ proc repl(vm: PeonVM = newPeonVM()) = for node in tree: styledEcho fgGreen, "\t", $node echo "" - compiled = compiler.compile(tree, "stdin", tokenizer.getLines(), input) + discard compiler.compile(tree, "stdin", tokenizer.getLines(), input, chunk=compiled, incremental=incremental, terminateScope=false) + incremental = true when debugCompiler: styledEcho fgCyan, "Compilation step:\n" debugger.disassembleChunk(compiled, "stdin") echo "" - serializer.dumpFile(compiled, "stdin", "stdin.pbc") - serialized = serializer.loadFile("stdin.pbc") + serialized = serializer.loadBytes(serializer.dumpBytes(compiled, "stdin")) when debugSerializer: styledEcho fgCyan, "Serialization step: " styledEcho fgBlue, "\t- Peon version: ", fgYellow, &"{serialized.version.major}.{serialized.version.minor}.{serialized.version.patch}", fgBlue, " (commit ", fgYellow, serialized.commit[0..8], fgBlue, ") on branch ", fgYellow, serialized.branch @@ -207,6 +202,7 @@ proc runFile(f: string, interactive: bool = false, fromString: bool = false) = input = readFile(f) else: input = f + f = "" tokens = tokenizer.lex(input, f) if tokens.len() == 0: return @@ -237,6 +233,8 @@ proc runFile(f: string, interactive: bool = false, fromString: bool = false) = path &= splitFile(f).name & ".pbc" serializer.dumpFile(compiled, f, path) serialized = serializer.loadFile(path) + if fromString: + discard tryRemoveFile(".pbc") when debugSerializer: styledEcho fgCyan, "Serialization step: " styledEcho fgBlue, "\t- Peon version: ", fgYellow, &"{serialized.version.major}.{serialized.version.minor}.{serialized.version.patch}", fgBlue, " (commit ", fgYellow, serialized.commit[0..8], fgBlue, ") on branch ", fgYellow, serialized.branch @@ -361,7 +359,7 @@ when isMainModule: else: echo "usage: peon [options] [filename.pn]" quit() - # TODO: Use interactive/fromString options + # TODO: Use interactive if file == "": repl() else: diff --git a/tests/std.pn b/tests/std.pn index 47ce228..82cea2e 100644 --- a/tests/std.pn +++ b/tests/std.pn @@ -222,6 +222,7 @@ operator `<`*(a, b: int): bool { operator `==`*(a, b: int): bool { #pragma[magic: "EqualInt64", pure] + } operator `!=`*(a, b: int): bool {