Incremental compilation now works in the REPL

README.md

@@ -28,15 +28,7 @@ In peon, all objects are first-class (this includes functions, iterators, closur
 **Disclaimer 1**: The project is still in its very early days: lots of stuff is not implemented, a work in progress or 
 otherwise outright broken. Feel free to report bugs!
 
-
+**Disclaimer 2**: Currently, the `std` module has to be _always_ imported explicitly for even the most basic snippets to work. This is because intrinsic types and builtin operators are defined within it: if it is not imported, peon won't even know how to parse `2 + 2` (and even if it could, it would have no idea what the type of the expression would be). You can have a look at the [peon standard library](src/peon/stdlib) to see how the builtins are defined (be aware that they heavily rely on compiler black magic to work) and can even provide your own implementation if you're so inclined.
-**Disclaimer 2**: Currently the REPL is very basic (it adds your code to previous input plus a newline, as if it was compiling a new file every time),
-because incremental compilation is designed for modules and it doesn't play well with the interactive nature of a REPL session. To show the current state
-of the REPL, type `#show` (this will print all the code that has been typed so far), while to reset everything, type `#reset`. You can also type
-`#clear` if you want a clean slate to type in, but note that it won't reset the REPL state. If adding a new piece of code causes compilation to fail, the REPL will not add the last piece of code to the input so you can type it again and recompile without having to exit the program and start from scratch. You can move through the code using left/right arrows and go to a new line by pressing Ctrl+Enter. Using the up/down keys on your keyboard
-will move through the input history (which is never reset). Also note that UTF-8 is currently unsupported in the REPL (it will be soon though!)
-
-
-**Disclaimer 3**: Currently, the `std` module has to be _always_ imported explicitly for even the most basic snippets to work. This is because intrinsic types and builtin operators are defined within it: if it is not imported, peon won't even know how to parse `2 + 2` (and even if it could, it would have no idea what the type of the expression would be). You can have a look at the [peon standard library](src/peon/stdlib) to see how the builtins are defined (be aware that they heavily rely on compiler black magic to work) and can even provide your own implementation if you're so inclined.
 
 
 ### TODO List

src/backend/vm.nim

@@ -768,6 +768,8 @@ proc dispatch*(self: var PeonVM) =
                 # not needed there anymore
                 discard self.pop()
                 discard self.pop()
+            of ReplExit:
+                return
             of Return:
                 # Returns from a function.
                 # Every peon program is wrapped
@@ -1050,7 +1052,7 @@ proc dispatch*(self: var PeonVM) =
                 discard
 
 
-proc run*(self: var PeonVM, chunk: Chunk, breakpoints: seq[uint64] = @[]) =
+proc run*(self: var PeonVM, chunk: Chunk, breakpoints: seq[uint64] = @[], repl: bool = false) =
     ## Executes a piece of Peon bytecode
     self.chunk = chunk
     self.frames = @[]
@@ -1065,8 +1067,26 @@ proc run*(self: var PeonVM, chunk: Chunk, breakpoints: seq[uint64] = @[]) =
     except NilAccessDefect:
         stderr.writeLine("Memory Access Violation: SIGSEGV")
         quit(1)
-    # We clean up after ourselves!
+    if not repl:
-    self.collect()
+        # We clean up after ourselves!
+        self.collect()
+
+
+proc resume*(self: var PeonVM, chunk: Chunk) =
+    ## Resumes execution of the given chunk (which
+    ## may have changed since the last call to run()).
+    ## The instruction pointer is set to address 12,
+    ## as it is assumed that the compiler has produced
+    ## new code in the chunk, replacing the previous one.
+    ## No other state mutation occurs and all stacks as 
+    ## well as other metadata are left intact
+    try:
+        self.ip = 12
+        self.chunk = chunk
+        self.dispatch()
+    except NilAccessDefect:
+        stderr.writeLine("Memory Access Violation: SIGSEGV")
+        quit(1)
 
 
 {.pop.}

src/frontend/compiler/targets/bytecode/opcodes.nim

@@ -188,7 +188,8 @@ type
         PushC,       # Pop off the operand stack onto the call stack
         SysClock64,  # Pushes the output of a monotonic clock on the stack
         LoadTOS,     # Pushes the top of the call stack onto the operand stack
-        DupTop       # Duplicates the top of the operand stack onto the operand stack
+        DupTop,      # Duplicates the top of the operand stack onto the operand stack
+        ReplExit     # Exits the VM immediately, leaving its state intact. Used in the REPL 
 
 
 # We group instructions by their operation/operand types for easier handling when debugging
@@ -267,7 +268,8 @@ const simpleInstructions* = {Return, LoadNil,
                              Float32LessThan,
                              Float32GreaterOrEqual,
                              Float32LessOrEqual,
-                             DupTop
+                             DupTop,
+                             ReplExit
                             }
 
 # Constant instructions are instructions that operate on the bytecode constant table

src/frontend/compiler/targets/bytecode/target.nim

@@ -565,6 +565,8 @@ proc endScope(self: BytecodeCompiler) =
     var names: seq[Name] = @[]
     var popCount = 0
     for name in self.names:
+        if self.replMode and name.depth == 0:
+            continue
         # We only pop names in scopes deeper than ours
         if name.depth > self.depth:
             if name.depth == 0 and not self.isMainModule:
@@ -999,9 +1001,12 @@ proc terminateProgram(self: BytecodeCompiler, pos: int) =
     ## Utility to terminate a peon program
     self.patchForwardDeclarations()
     self.endScope()
-    self.emitByte(OpCode.Return, self.peek().token.line)
+    if self.replMode:
-    self.emitByte(0, self.peek().token.line)   # Entry point has no return value (TODO: Add easter eggs, cuz why not)
+        self.emitByte(ReplExit, self.peek().token.line)
-    self.patchReturnAddress(pos)
+    else:
+        self.emitByte(OpCode.Return, self.peek().token.line)
+        self.emitByte(0, self.peek().token.line)   # Entry point has no return value (TODO: Add easter eggs, cuz why not)
+        self.patchReturnAddress(pos)
 
 
 proc beginProgram(self: BytecodeCompiler): int =
@@ -2046,18 +2051,23 @@ proc compile*(self: BytecodeCompiler, ast: seq[Declaration], file: string, lines
         self.chunk = newChunk()
     else:
         self.chunk = chunk
-    self.ast = ast
     self.file = file
     self.depth = 0
     self.currentFunction = nil
-    self.current = 0
+    if self.replMode:
-    self.lines = lines
+        self.ast &= ast
-    self.source = source
+        self.source &= "\n" & source
+        self.lines &= lines
+    else:
+        self.ast = ast
+        self.current = 0
+        self.stackIndex = 1
+        self.lines = lines
+        self.source = source
     self.isMainModule = isMainModule
     self.disabledWarnings = disabledWarnings
     self.showMismatches = showMismatches
     self.mode = mode
-    self.stackIndex = 1
     if not incremental:
         self.jumps = @[]
     let pos = self.beginProgram()
@@ -2094,6 +2104,8 @@ proc compileModule(self: BytecodeCompiler, module: Name) =
     let currentModule = self.currentModule
     let mainModule = self.isMainModule
     let parentModule = self.parentModule
+    let replMode = self.replMode
+    self.replMode = false
     self.parentModule = currentModule
     self.currentModule = module
     discard self.compile(self.parser.parse(self.lexer.lex(source, path), 
@@ -2111,6 +2123,7 @@ proc compileModule(self: BytecodeCompiler, module: Name) =
     self.currentModule = currentModule
     self.isMainModule = mainModule
     self.parentModule = parentModule
+    self.replMode = replMode
     self.lines = lines
     self.source = src
     self.modules.incl(module)

src/main.nim

@@ -62,17 +62,17 @@ proc repl(warnings: seq[WarningKind] = @[], mismatches: bool = false, mode: Comp
         serialized: Serialized
         tokenizer = newLexer()
         vm = newPeonVM()
+        parser = newParser()
         debugger = newDebugger()
         serializer = newSerializer()
         editor = getLineEditor()
         input: string
-        current: string
+        first: bool = false
     tokenizer.fillSymbolTable()
     editor.bindEvent(jeQuit):
         stdout.styledWriteLine(fgGreen, "Goodbye!")
         keep = false
         input = ""
-        current = ""
     editor.bindKey("ctrl+a"):
         editor.content.home()
     editor.bindKey("ctrl+e"):
@@ -80,18 +80,12 @@ proc repl(warnings: seq[WarningKind] = @[], mismatches: bool = false, mode: Comp
     while keep:
         try:
             input = editor.read()
-            if input == "#reset":
+            if input == "#clear":
-                compiled = newChunk()
-                current = ""
-                continue
-            elif input == "#show":
-                echo current
-            elif input == "#clear":
                 stdout.write("\x1Bc")
                 continue
             elif input == "":
                 continue
-            tokens = tokenizer.lex(current & input & "\n", "stdin")
+            tokens = tokenizer.lex(input, "stdin")
             if tokens.len() == 0:
                 continue
             when debugLexer:
@@ -102,7 +96,7 @@ proc repl(warnings: seq[WarningKind] = @[], mismatches: bool = false, mode: Comp
                         break
                     styledEcho fgGreen, "\t", $token
                 echo ""
-            tree = newParser().parse(tokens, "stdin", tokenizer.getLines(), current & input & "\n")
+            tree = parser.parse(tokens, "stdin", tokenizer.getLines(), input, persist=true)
             if tree.len() == 0:
                 continue
             when debugParser:
@@ -110,7 +104,7 @@ proc repl(warnings: seq[WarningKind] = @[], mismatches: bool = false, mode: Comp
                 for node in tree:
                     styledEcho fgGreen, "\t", $node
                 echo ""
-            compiled = newBytecodeCompiler(replMode=true).compile(tree, "stdin", tokenizer.getLines(), current & input & "\n", showMismatches=mismatches, disabledWarnings=warnings, mode=mode)
+            compiled = compiler.compile(tree, "stdin", tokenizer.getLines(), input, showMismatches=mismatches, disabledWarnings=warnings, mode=mode)
             when debugCompiler:
                 styledEcho fgCyan, "Compilation step:\n"
                 debugger.disassembleChunk(compiled, "stdin")
@@ -141,8 +135,11 @@ proc repl(warnings: seq[WarningKind] = @[], mismatches: bool = false, mode: Comp
                     styledEcho fgGreen, "OK"
                 else:
                     styledEcho fgRed, "Corrupted"
-            vm.run(serialized.chunk)
+            if not first:
-            current &= input & "\n"
+                vm.run(serialized.chunk, repl=true)
+                first = true
+            else:
+                vm.resume(serialized.chunk)
         except LexingError:
             print(LexingError(getCurrentException()))
         except ParseError: