diff --git a/src/backend/vm.nim b/src/backend/vm.nim
index d20ca42..27d39ad 100644
--- a/src/backend/vm.nim
+++ b/src/backend/vm.nim
@@ -13,14 +13,14 @@
# limitations under the License.
## The Peon runtime environment
+import strutils
+import strformat
+
import types
import ../config
-when DEBUG_TRACE_VM:
- import strformat
import ../frontend/meta/bytecode
import ../util/multibyte
-
export types
type
@@ -30,9 +30,8 @@ type
ip: int # Instruction pointer
cache: array[6, PeonObject] # Singletons cache
chunk: Chunk # Piece of bytecode to execute
- frames: seq[int] # Stores the initial index of stack frames
- heapVars: seq[PeonObject] # Stores variables that do not have stack semantics (i.e. "static")
- lastPop*: PeonObject # Used in the REPL
+ frames: seq[int] # Stores the bottom of stack frames
+ heapVars: seq[PeonObject] # Stores variables that do not have stack semantics
proc initCache*(self: PeonVM) =
@@ -58,19 +57,19 @@ proc newPeonVM*: PeonVM =
## Getters for singleton types (they are cached!)
-proc getNil*(self: PeonVM): PeonObject = self.cache[0]
+proc getNil*(self: PeonVM): PeonObject {.inline.} = self.cache[0]
-proc getBool*(self: PeonVM, value: bool): PeonObject =
+proc getBool*(self: PeonVM, value: bool): PeonObject {.inline.} =
if value:
return self.cache[1]
return self.cache[2]
-proc getInf*(self: PeonVM, positive: bool): PeonObject =
+proc getInf*(self: PeonVM, positive: bool): PeonObject {.inline.} =
if positive:
return self.cache[3]
return self.cache[4]
-proc getNan*(self: PeonVM): PeonObject = self.cache[5]
+proc getNan*(self: PeonVM): PeonObject {.inline.} = self.cache[5]
## Stack primitives. Note: all stack accessing that goes
## through the get/set wrappers is frame-relative, meaning
@@ -99,18 +98,21 @@ proc peek(self: PeonVM): PeonObject =
proc get(self: PeonVM, idx: int): PeonObject =
## Accessor method that abstracts
- ## stack accessing through stack
+ ## stack indexing through stack
## frames
return self.stack[idx + self.frames[^1]]
proc set(self: PeonVM, idx: int, val: PeonObject) =
## Setter method that abstracts
- ## stack accessing through stack
+ ## stack indexing through stack
## frames
self.stack[idx + self.frames[^1]] = val
+## Byte-level primitives to read/decode
+## bytecode
+
proc readByte(self: PeonVM): uint8 =
## Reads a single byte from the
## bytecode and returns it as an
@@ -197,6 +199,7 @@ proc dispatch*(self: PeonVM) =
echo &"Instruction: {instruction}"
echo &"Stack: {self.stack}"
echo &"Current Frame: {self.stack[self.frames[^1]..^1]}"
+ echo &"Heap Vars: {self.heapVars}"
discard readLine stdin
case instruction:
# Constant loading
@@ -216,44 +219,48 @@ proc dispatch*(self: PeonVM) =
self.push(self.readUInt64(int(self.readLong())))
of LoadUInt32:
self.push(self.readUInt32(int(self.readLong())))
+ of LoadInt32:
+ self.push(self.readInt32(int(self.readLong())))
of Call:
# Calls a function. The calling convention for peon
# functions is pretty simple: the return address sits
# at the bottom of the stack frame, then follow the
# arguments and all temporaries/local variables
let newIp = self.readLong()
- # We do this because if we immediately changed
+ # We store it because if we immediately changed
# the instruction pointer, we'd read the wrong
# value for the argument count. Storing it and
# changing it later fixes this issue
self.frames.add(int(self.readLong()))
self.ip = int(newIp)
of OpCode.Return:
- # Returns from a void function or terminates the
- # program entirely if we're at the topmost frame
+ # Returns from a void function
let frame = self.frames.pop()
- if self.frames.len() > 1:
- for i in countdown(self.stack.high(), frame):
- discard self.pop()
- self.ip = int(self.pop().uInt)
- else:
- while self.stack.len() > 0:
- discard self.pop()
- return
+ for i in 0.. 0:
+ discard self.pop()
+ discard self.frames.pop()
+ return
of ReturnValue:
# Returns from a function which has a return value,
# pushing it on the stack
let retVal = self.pop()
let frame = self.frames.pop()
- for i in countdown(frame, 1):
+ for i in 0.. 0 and name.depth < depth:
# Ding! The given name is closed over: we need to
- # change the NoOp instructions that self.declareName
+ # change the dummy Jump instruction that self.declareName
# put in place for us into a StoreHeap. We don't need to change
# other pieces of code because self.identifier() already
# emits LoadHeap if it detects the variable is closed over,
@@ -414,7 +420,7 @@ proc detectClosureVariable(self: Compiler, name: Name,
let idx = self.closedOver.high().toTriple()
if self.closedOver.len() >= 16777216:
self.error("too many consecutive closed-over variables (max is 16777216)")
- self.chunk.code[name.codePos] = StoreHeap.uint8
+ self.chunk.code[name.codePos] = StoreClosure.uint8
self.chunk.code[name.codePos + 1] = idx[0]
self.chunk.code[name.codePos + 2] = idx[1]
self.chunk.code[name.codePos + 3] = idx[2]
@@ -428,9 +434,9 @@ proc compareTypes(self: Compiler, a, b: Type): bool =
# The nil code here is for void functions (when
# we compare their return types)
if a == nil:
- return b == nil
+ return b == nil or b.kind == Any
elif b == nil:
- return a == nil
+ return a == nil or a.kind == Any
elif a.kind == Any or b.kind == Any:
# This is needed internally: user code
# cannot generate code for matching
@@ -598,6 +604,8 @@ proc inferType(self: Compiler, node: Expression): Type =
let resolved = self.resolve(IdentExpr(node.callee))
if resolved != nil:
result = resolved.valueType.returnType
+ if result == nil:
+ result = Type(kind: Any)
else:
result = nil
of lambdaExpr:
@@ -790,33 +798,31 @@ proc matchImpl(self: Compiler, name: string, kind: Type): Name =
return impl[0]
-proc callUnaryOp(self: Compiler, fn: Name, op: UnaryExpr) =
- ## Emits the code to call a unary operator
- # Pushes the return address
+proc generateCall(self: Compiler, fn: Name, args: seq[Expression]) =
+ ## Small wrapper that abstracts emitting a call instruction
+ ## for a given function
+ self.emitByte(LoadNil) # Stack alignment
self.emitByte(LoadUInt32)
# We patch it later!
let idx = self.chunk.consts.len()
self.emitBytes(self.chunk.writeConstant((0xffffffff'u32).toQuad()))
- self.expression(op.a) # Pushes the arguments onto the stack
+ for argument in args:
+ self.expression(argument) # Pushes the arguments onto the stack
self.emitByte(Call) # Creates a stack frame
self.emitBytes(fn.codePos.toTriple())
- self.emitBytes(1.toTriple())
+ self.emitBytes((args.len()).toTriple())
self.patchReturnAddress(idx)
+proc callUnaryOp(self: Compiler, fn: Name, op: UnaryExpr) =
+ ## Emits the code to call a unary operator
+ self.generateCall(fn, @[op.a])
+
+
proc callBinaryOp(self: Compiler, fn: Name, op: BinaryExpr) =
## Emits the code to call a binary operator
# Pushes the return address
- self.emitByte(LoadUInt32)
- # We patch it later!
- let idx = self.chunk.consts.len()
- self.emitBytes(self.chunk.writeConstant((0xffffffff'u32).toQuad()))
- self.expression(op.a) # Pushes the arguments onto the stack
- self.expression(op.b)
- self.emitByte(Call) # Creates a stack frame
- self.emitBytes(fn.codePos.toTriple())
- self.emitBytes(2.toTriple())
- self.patchReturnAddress(idx)
+ self.generateCall(fn, @[op.a, op.b])
proc unary(self: Compiler, node: UnaryExpr) =
@@ -888,11 +894,21 @@ proc declareName(self: Compiler, node: Declaration) =
isLet: node.isLet,
isClosedOver: false,
line: node.token.line))
- # We emit 4 No-Ops because they may become a
- # StoreHeap instruction. If not, they'll be
- # removed before the compiler is finished
- # TODO: This may break CFI offsets
- self.emitBytes([NoOp, NoOp, NoOp, NoOp])
+ # We emit a jump of 0 because this may become a
+ # StoreHeap instruction. If they variable is
+ # not closed over, we'll sadly be wasting a
+ # VM cycle. The previous implementation used 4 no-op
+ # instructions, which wasted 4 times as many clock
+ # cycles.
+ # TODO: Optimize this. It's a bit tricky because
+ # deleting bytecode would render all of our
+ # jump offsets and other absolute indeces in the
+ # bytecode wrong
+ if self.scopeDepth > 0:
+ # Closure variables are only used in local
+ # scopes
+ self.emitByte(LongJumpForwards)
+ self.emitBytes(0.toTriple())
of NodeKind.funDecl:
var node = FunDecl(node)
self.names.add(Name(depth: self.scopeDepth,
@@ -965,15 +981,15 @@ proc identifier(self: Compiler, node: IdentExpr) =
# were, self.resolve() would have returned nil
if not t.closedOver:
# Static name resolution, loads value at index in the stack. Very fast. Much wow.
+ if self.scopeDepth > 0:
+ inc(index, 1)
self.emitByte(LoadVar)
- if self.frames.len() > 1:
- inc(index, 2)
self.emitBytes((index - self.frames[^1]).toTriple())
else:
- # Heap-allocated closure variable. Stored in a separate "closure array" in the VM that does not have stack semantics.
- # This makes closures work as expected and is not much slower than indexing our stack (since they're both
- # dynamic arrays at runtime anyway)
- self.emitByte(LoadHeap)
+ # Heap-allocated closure variable. Stored in a separate "closure array" in the VM that does not have stack semantics
+ # and where the no-effect invariant is not kept. This makes closures work as expected and is not much slower than
+ # indexing our stack (since they're both dynamic arrays at runtime anyway)
+ self.emitByte(LoadClosure)
self.emitBytes(self.closedOver.high().toTriple())
@@ -997,7 +1013,7 @@ proc assignment(self: Compiler, node: ASTNode) =
if not t.closedOver:
self.emitByte(StoreVar)
else:
- self.emitByte(StoreHeap)
+ self.emitByte(StoreClosure)
self.emitBytes(index.toTriple())
else:
self.error(&"reference to undeclared name '{node.token.lexeme}'")
@@ -1017,7 +1033,7 @@ proc beginScope(self: Compiler) =
inc(self.scopeDepth)
-proc endScope(self: Compiler) =
+proc endScope(self: Compiler, fromFunc: bool = false) =
## Ends the current local scope
if self.scopeDepth < 0:
self.error("cannot call endScope with scopeDepth < 0 (This is an internal error and most likely a bug)")
@@ -1025,18 +1041,12 @@ proc endScope(self: Compiler) =
var names: seq[Name] = @[]
for name in self.names:
if name.depth > self.scopeDepth:
- if name.valueType.kind != Function and OpCode(self.chunk.code[name.codePos]) == NoOp:
- for _ in countup(0, 3):
- # Since by deleting it the size of the
- # sequence decreases, we don't need to
- # increase the index
- self.chunk.code.delete(name.codePos)
names.add(name)
- if not self.enableOptimizations:
+ if not self.enableOptimizations and not fromFunc:
# All variables with a scope depth larger than the current one
# are now out of scope. Begone, you're now homeless!
self.emitByte(Pop)
- if self.enableOptimizations and len(names) > 1:
+ if self.enableOptimizations and len(names) > 1 and not fromFunc:
# If we're popping less than 65535 variables, then
# we can emit a PopN instruction. This is true for
# 99.99999% of the use cases of the language (who the
@@ -1050,7 +1060,7 @@ proc endScope(self: Compiler) =
for i in countdown(self.names.high(), len(names) - uint16.high().int()):
if self.names[i].depth > self.scopeDepth:
self.emitByte(Pop)
- elif len(names) == 1:
+ elif len(names) == 1 and not fromFunc:
# We only emit PopN if we're popping more than one value
self.emitByte(Pop)
# This seems *really* slow, but
@@ -1068,6 +1078,7 @@ proc endScope(self: Compiler) =
for name in names:
if name.isClosedOver:
self.closedOver.delete(idx)
+ self.emitByte(PopClosure)
inc(idx)
@@ -1136,9 +1147,10 @@ proc whileStmt(self: Compiler, node: WhileStmt) =
self.emitLoop(start)
-proc callFunction(self: Compiler, node: CallExpr) =
+proc callExpr(self: Compiler, node: CallExpr) =
## Compiles code to call a function
var args: seq[tuple[name: string, kind: Type]] = @[]
+ var argExpr: seq[Expression] = @[]
var kind: Type
# TODO: Keyword arguments
for i, argument in node.arguments.positionals:
@@ -1148,6 +1160,7 @@ proc callFunction(self: Compiler, node: CallExpr) =
self.error(&"reference to undeclared identifier '{IdentExpr(argument).name.lexeme}'")
self.error(&"cannot infer the type of argument {i + 1} in function call")
args.add(("", kind))
+ argExpr.add(argument)
for argument in node.arguments.keyword:
discard
if args.len() >= 16777216:
@@ -1158,16 +1171,7 @@ proc callFunction(self: Compiler, node: CallExpr) =
funct = self.matchImpl(IdentExpr(node.callee).name.lexeme, Type(kind: Function, returnType: Type(kind: Any), args: args))
else:
discard # TODO: Lambdas
- self.emitByte(LoadUInt32)
- # We patch it later!
- let idx = self.chunk.consts.len()
- self.emitBytes(self.chunk.writeConstant((0xffffffff'u32).toQuad()))
- for argument in node.arguments.positionals:
- self.expression(argument)
- self.emitByte(Call) # Creates a stack frame
- self.emitBytes(funct.codePos.toTriple())
- self.emitBytes(args.len().toTriple())
- self.patchReturnAddress(idx)
+ self.generateCall(funct, argExpr)
proc expression(self: Compiler, node: Expression) =
@@ -1178,8 +1182,8 @@ proc expression(self: Compiler, node: Expression) =
# error in self.identifier()
self.error("expression has no type")
case node.kind:
- of callExpr:
- self.callFunction(CallExpr(node)) # TODO
+ of NodeKind.callExpr:
+ self.callExpr(CallExpr(node)) # TODO
of getItemExpr:
discard # TODO
# Note that for setItem and assign we don't convert
@@ -1243,9 +1247,11 @@ proc returnStmt(self: Compiler, node: ReturnStmt) =
let typ = self.inferType(self.currentFunction)
## Having the return type
if returnType == nil and typ.returnType != nil:
+ if node.value.kind == identExpr:
+ self.error(&"reference to undeclared identifier '{node.value.token.lexeme}'")
self.error(&"expected return value of type '{self.typeToStr(typ.returnType)}', but expression has no type")
elif typ.returnType == nil and returnType != nil:
- self.error("empty return statement is not allowed in non-void functions")
+ self.error("non-empty return statement is not allowed in void functions")
elif not self.compareTypes(returnType, typ.returnType):
self.error(&"expected return value of type '{self.typeToStr(typ.returnType)}', got '{self.typeToStr(returnType)}' instead")
if node.value != nil:
@@ -1316,7 +1322,11 @@ proc statement(self: Compiler, node: Statement) =
of exprStmt:
var expression = ExprStmt(node).expression
self.expression(expression)
- self.emitByte(Pop) # Expression statements discard their value. Their main use case is side effects in function calls
+ # We only print top-level expressions
+ if self.replMode and self.scopeDepth == 0:
+ self.emitByte(PopRepl)
+ else:
+ self.emitByte(Pop) # Expression statements discard their value. Their main use case is side effects in function calls
of NodeKind.ifStmt:
self.ifStmt(IfStmt(node))
of NodeKind.assertStmt:
@@ -1375,11 +1385,21 @@ proc funDecl(self: Compiler, node: FunDecl) =
## Compiles function declarations
# A function's code is just compiled linearly
# and then jumped over
- self.emitByte(LoadNil) # Aligns the stack
+
+ # While the compiler stores functions as if they
+ # were on the stack, in reality there is nothing
+ # at runtime that represents them (instead, we
+ # compile everything in a single blob and jump back
+ # and forth). If we didn't align the stack to account
+ # for this behavior, all of our offsets would be off
+ # by how many functions we have declared. We could fix
+ # this by storing functions in a separate list of identifiers,
+ # but that's rather unelegant and requires specialized functions
+ # for looking identifiers vs functions, which is not ideal
let jmp = self.emitJump(JumpForwards)
var function = self.currentFunction
self.declareName(node)
- self.frames.add(self.names.len())
+ self.frames.add(self.names.len() - 1)
# TODO: Forward declarations
if node.body != nil:
if BlockStmt(node.body).code.len() == 0:
@@ -1409,18 +1429,11 @@ proc funDecl(self: Compiler, node: FunDecl) =
# We let our debugger know a function is starting
let start = self.chunk.code.high()
- self.blockStmt(BlockStmt(node.body))
- # Yup, we're done. That was easy, huh?
- # But, after all, functions are just named
- # scopes, and we compile them just like that:
- # we declare their name and arguments (before
- # their body so recursion works) and then just
- # handle them as a block statement (which takes
- # care of incrementing self.scopeDepth so locals
- # are resolved properly). There's a need for a bit
- # of boilerplate code to make closures work, but
- # that's about it
-
+ self.beginScope()
+ # self.emitByte(LoadNil)
+ for decl in BlockStmt(node.body).code:
+ self.declaration(decl)
+ self.endScope(fromFunc=true)
case self.currentFunction.kind:
of NodeKind.funDecl:
if not self.currentFunction.hasExplicitReturn:
@@ -1497,8 +1510,7 @@ proc compile*(self: Compiler, ast: seq[Declaration], file: string): Chunk =
self.declaration(Declaration(self.step()))
if self.ast.len() > 0:
# *Technically* an empty program is a valid program
- self.endScope()
- self.emitByte(OpCode.Return) # Exits the VM's main loop when used at the global scope
+ self.emitByte(ProgExit)
result = self.chunk
- if self.ast.len() > 0 and self.scopeDepth != -1:
- self.error(&"invalid state: invalid scopeDepth value (expected -1, got {self.scopeDepth}), did you forget to call endScope/beginScope?")
+ if self.ast.len() > 0 and self.scopeDepth != 0:
+ self.error(&"invalid state: invalid scopeDepth value (expected 0, got {self.scopeDepth}), did you forget to call endScope/beginScope?")
diff --git a/src/frontend/meta/bytecode.nim b/src/frontend/meta/bytecode.nim
index 5a206f0..211fc23 100644
--- a/src/frontend/meta/bytecode.nim
+++ b/src/frontend/meta/bytecode.nim
@@ -90,14 +90,16 @@ type
LoadInf,
## Basic stack operations
Pop, # Pops an element off the stack and discards it
+ PopRepl, # Same as Pop, but also prints the value of what's popped (used in REPL mode)
Push, # Pushes x onto the stack
PopN, # Pops x elements off the stack (optimization for exiting local scopes which usually pop many elements)
## Name resolution/handling
LoadAttribute, # Pushes the attribute b of object a onto the stack
LoadVar, # Pushes the object at position x in the stack onto the stack
StoreVar, # Stores the value of b at position a in the stack
- LoadHeap, # Pushes the object position x in the closure array onto the stack
- StoreHeap, # Stores the value of b at position a in the closure array
+ LoadClosure, # Pushes the object position x in the closure array onto the stack
+ StoreClosure, # Stores the value of b at position a in the closure array
+ PopClosure, # Pops a closed-over variable from the closure array
## Looping and jumping
Jump, # Absolute, unconditional jump into the bytecode
JumpForwards, # Relative, unconditional, positive jump in the bytecode
@@ -129,6 +131,7 @@ type
## Misc
Assert, # Raises an AssertionFailed exception if x is false
NoOp, # Just a no-op
+ ProgExit, # Terminates the whole program
# We group instructions by their operation/operand types for easier handling when debugging
@@ -137,9 +140,10 @@ type
const simpleInstructions* = {Return, LoadNil,
LoadTrue, LoadFalse,
LoadNan, LoadInf,
- Pop, Raise, BeginTry,
- FinishTry, Yield,
- Await, NoOp, ReturnValue}
+ Pop, PopRepl, Raise,
+ BeginTry, FinishTry, Yield,
+ Await, NoOp, ReturnValue,
+ PopClosure, ProgExit}
# Constant instructions are instructions that operate on the bytecode constant table
const constantInstructions* = {LoadInt64, LoadUInt64,
@@ -151,7 +155,7 @@ const constantInstructions* = {LoadInt64, LoadUInt64,
# Stack triple instructions operate on the stack at arbitrary offsets and pop arguments off of it in the form
# of 24 bit integers
-const stackTripleInstructions* = {StoreVar, LoadVar, LoadHeap, StoreHeap}
+const stackTripleInstructions* = {StoreVar, LoadVar, LoadCLosure, StoreClosure}
# Stack double instructions operate on the stack at arbitrary offsets and pop arguments off of it in the form
# of 16 bit integers
diff --git a/src/main.nim b/src/main.nim
index 5d22fd6..479e01f 100644
--- a/src/main.nim
+++ b/src/main.nim
@@ -47,7 +47,7 @@ proc repl =
serialized: Serialized
tokenizer = newLexer()
parser = newParser()
- compiler = newCompiler()
+ compiler = newCompiler(replMode=true)
debugger = newDebugger()
serializer = newSerializer()
vm = newPeonVM()
@@ -136,33 +136,6 @@ proc repl =
when debugRuntime:
styledEcho fgCyan, "\n\nExecution step: "
vm.run(serialized.chunk)
- var popped = vm.lastPop
- case popped.kind:
- of Int64:
- echo &"{popped.long}'i64"
- of UInt64:
- echo &"{popped.uLong}'u64"
- of Int32:
- echo &"{popped.`int`}'i32"
- of UInt32:
- echo &"{popped.uInt}'u32"
- of Int16:
- echo &"{popped.short}'i16"
- of UInt16:
- echo &"{popped.uShort}'u16"
- of Int8:
- echo &"{popped.tiny}'i8"
- of UInt8:
- echo &"{popped.uTiny}'u8"
- of ObjectKind.Inf:
- if popped.positive:
- echo "inf"
- else:
- echo "-inf"
- of ObjectKind.Nan, Nil:
- echo ($popped.kind).toLowerAscii()
- else:
- discard
except LexingError:
input = ""
let exc = LexingError(getCurrentException())
diff --git a/src/util/debugger.nim b/src/util/debugger.nim
index 51cd058..812dd2e 100644
--- a/src/util/debugger.nim
+++ b/src/util/debugger.nim
@@ -131,7 +131,7 @@ proc callInstruction(self: Debugger, instruction: OpCode) =
var args = [self.chunk.code[self.current + 4], self.chunk.code[self.current + 5], self.chunk.code[self.current + 6]].fromTriple()
printInstruction(instruction)
stdout.styledWrite(fgGreen, &", jumps to address ", fgYellow, $slot, fgGreen, " with ", fgYellow, $args, fgGreen, " argument")
- if args > 1:
+ if args > 1 or args == 0:
stdout.styledWrite(fgGreen, "s")
nl()
self.current += 7
diff --git a/tests/closures.pn b/tests/closures.pn
index e54ebb5..9059841 100644
--- a/tests/closures.pn
+++ b/tests/closures.pn
@@ -3,12 +3,7 @@ fn outer {
fn inner {
var y = x;
}
+ inner();
}
-
-fn outerTwo: fn: int {
- fn inner: int {
- return 0;
- }
- return inner;
-}
+outer();