diff --git a/src/backend/vm.nim b/src/backend/vm.nim index d20ca42..27d39ad 100644 --- a/src/backend/vm.nim +++ b/src/backend/vm.nim @@ -13,14 +13,14 @@ # limitations under the License. ## The Peon runtime environment +import strutils +import strformat + import types import ../config -when DEBUG_TRACE_VM: - import strformat import ../frontend/meta/bytecode import ../util/multibyte - export types type @@ -30,9 +30,8 @@ type ip: int # Instruction pointer cache: array[6, PeonObject] # Singletons cache chunk: Chunk # Piece of bytecode to execute - frames: seq[int] # Stores the initial index of stack frames - heapVars: seq[PeonObject] # Stores variables that do not have stack semantics (i.e. "static") - lastPop*: PeonObject # Used in the REPL + frames: seq[int] # Stores the bottom of stack frames + heapVars: seq[PeonObject] # Stores variables that do not have stack semantics proc initCache*(self: PeonVM) = @@ -58,19 +57,19 @@ proc newPeonVM*: PeonVM = ## Getters for singleton types (they are cached!) -proc getNil*(self: PeonVM): PeonObject = self.cache[0] +proc getNil*(self: PeonVM): PeonObject {.inline.} = self.cache[0] -proc getBool*(self: PeonVM, value: bool): PeonObject = +proc getBool*(self: PeonVM, value: bool): PeonObject {.inline.} = if value: return self.cache[1] return self.cache[2] -proc getInf*(self: PeonVM, positive: bool): PeonObject = +proc getInf*(self: PeonVM, positive: bool): PeonObject {.inline.} = if positive: return self.cache[3] return self.cache[4] -proc getNan*(self: PeonVM): PeonObject = self.cache[5] +proc getNan*(self: PeonVM): PeonObject {.inline.} = self.cache[5] ## Stack primitives. Note: all stack accessing that goes ## through the get/set wrappers is frame-relative, meaning @@ -99,18 +98,21 @@ proc peek(self: PeonVM): PeonObject = proc get(self: PeonVM, idx: int): PeonObject = ## Accessor method that abstracts - ## stack accessing through stack + ## stack indexing through stack ## frames return self.stack[idx + self.frames[^1]] proc set(self: PeonVM, idx: int, val: PeonObject) = ## Setter method that abstracts - ## stack accessing through stack + ## stack indexing through stack ## frames self.stack[idx + self.frames[^1]] = val +## Byte-level primitives to read/decode +## bytecode + proc readByte(self: PeonVM): uint8 = ## Reads a single byte from the ## bytecode and returns it as an @@ -197,6 +199,7 @@ proc dispatch*(self: PeonVM) = echo &"Instruction: {instruction}" echo &"Stack: {self.stack}" echo &"Current Frame: {self.stack[self.frames[^1]..^1]}" + echo &"Heap Vars: {self.heapVars}" discard readLine stdin case instruction: # Constant loading @@ -216,44 +219,48 @@ proc dispatch*(self: PeonVM) = self.push(self.readUInt64(int(self.readLong()))) of LoadUInt32: self.push(self.readUInt32(int(self.readLong()))) + of LoadInt32: + self.push(self.readInt32(int(self.readLong()))) of Call: # Calls a function. The calling convention for peon # functions is pretty simple: the return address sits # at the bottom of the stack frame, then follow the # arguments and all temporaries/local variables let newIp = self.readLong() - # We do this because if we immediately changed + # We store it because if we immediately changed # the instruction pointer, we'd read the wrong # value for the argument count. Storing it and # changing it later fixes this issue self.frames.add(int(self.readLong())) self.ip = int(newIp) of OpCode.Return: - # Returns from a void function or terminates the - # program entirely if we're at the topmost frame + # Returns from a void function let frame = self.frames.pop() - if self.frames.len() > 1: - for i in countdown(self.stack.high(), frame): - discard self.pop() - self.ip = int(self.pop().uInt) - else: - while self.stack.len() > 0: - discard self.pop() - return + for i in 0.. 0: + discard self.pop() + discard self.frames.pop() + return of ReturnValue: # Returns from a function which has a return value, # pushing it on the stack let retVal = self.pop() let frame = self.frames.pop() - for i in countdown(frame, 1): + for i in 0.. 0 and name.depth < depth: # Ding! The given name is closed over: we need to - # change the NoOp instructions that self.declareName + # change the dummy Jump instruction that self.declareName # put in place for us into a StoreHeap. We don't need to change # other pieces of code because self.identifier() already # emits LoadHeap if it detects the variable is closed over, @@ -414,7 +420,7 @@ proc detectClosureVariable(self: Compiler, name: Name, let idx = self.closedOver.high().toTriple() if self.closedOver.len() >= 16777216: self.error("too many consecutive closed-over variables (max is 16777216)") - self.chunk.code[name.codePos] = StoreHeap.uint8 + self.chunk.code[name.codePos] = StoreClosure.uint8 self.chunk.code[name.codePos + 1] = idx[0] self.chunk.code[name.codePos + 2] = idx[1] self.chunk.code[name.codePos + 3] = idx[2] @@ -428,9 +434,9 @@ proc compareTypes(self: Compiler, a, b: Type): bool = # The nil code here is for void functions (when # we compare their return types) if a == nil: - return b == nil + return b == nil or b.kind == Any elif b == nil: - return a == nil + return a == nil or a.kind == Any elif a.kind == Any or b.kind == Any: # This is needed internally: user code # cannot generate code for matching @@ -598,6 +604,8 @@ proc inferType(self: Compiler, node: Expression): Type = let resolved = self.resolve(IdentExpr(node.callee)) if resolved != nil: result = resolved.valueType.returnType + if result == nil: + result = Type(kind: Any) else: result = nil of lambdaExpr: @@ -790,33 +798,31 @@ proc matchImpl(self: Compiler, name: string, kind: Type): Name = return impl[0] -proc callUnaryOp(self: Compiler, fn: Name, op: UnaryExpr) = - ## Emits the code to call a unary operator - # Pushes the return address +proc generateCall(self: Compiler, fn: Name, args: seq[Expression]) = + ## Small wrapper that abstracts emitting a call instruction + ## for a given function + self.emitByte(LoadNil) # Stack alignment self.emitByte(LoadUInt32) # We patch it later! let idx = self.chunk.consts.len() self.emitBytes(self.chunk.writeConstant((0xffffffff'u32).toQuad())) - self.expression(op.a) # Pushes the arguments onto the stack + for argument in args: + self.expression(argument) # Pushes the arguments onto the stack self.emitByte(Call) # Creates a stack frame self.emitBytes(fn.codePos.toTriple()) - self.emitBytes(1.toTriple()) + self.emitBytes((args.len()).toTriple()) self.patchReturnAddress(idx) +proc callUnaryOp(self: Compiler, fn: Name, op: UnaryExpr) = + ## Emits the code to call a unary operator + self.generateCall(fn, @[op.a]) + + proc callBinaryOp(self: Compiler, fn: Name, op: BinaryExpr) = ## Emits the code to call a binary operator # Pushes the return address - self.emitByte(LoadUInt32) - # We patch it later! - let idx = self.chunk.consts.len() - self.emitBytes(self.chunk.writeConstant((0xffffffff'u32).toQuad())) - self.expression(op.a) # Pushes the arguments onto the stack - self.expression(op.b) - self.emitByte(Call) # Creates a stack frame - self.emitBytes(fn.codePos.toTriple()) - self.emitBytes(2.toTriple()) - self.patchReturnAddress(idx) + self.generateCall(fn, @[op.a, op.b]) proc unary(self: Compiler, node: UnaryExpr) = @@ -888,11 +894,21 @@ proc declareName(self: Compiler, node: Declaration) = isLet: node.isLet, isClosedOver: false, line: node.token.line)) - # We emit 4 No-Ops because they may become a - # StoreHeap instruction. If not, they'll be - # removed before the compiler is finished - # TODO: This may break CFI offsets - self.emitBytes([NoOp, NoOp, NoOp, NoOp]) + # We emit a jump of 0 because this may become a + # StoreHeap instruction. If they variable is + # not closed over, we'll sadly be wasting a + # VM cycle. The previous implementation used 4 no-op + # instructions, which wasted 4 times as many clock + # cycles. + # TODO: Optimize this. It's a bit tricky because + # deleting bytecode would render all of our + # jump offsets and other absolute indeces in the + # bytecode wrong + if self.scopeDepth > 0: + # Closure variables are only used in local + # scopes + self.emitByte(LongJumpForwards) + self.emitBytes(0.toTriple()) of NodeKind.funDecl: var node = FunDecl(node) self.names.add(Name(depth: self.scopeDepth, @@ -965,15 +981,15 @@ proc identifier(self: Compiler, node: IdentExpr) = # were, self.resolve() would have returned nil if not t.closedOver: # Static name resolution, loads value at index in the stack. Very fast. Much wow. + if self.scopeDepth > 0: + inc(index, 1) self.emitByte(LoadVar) - if self.frames.len() > 1: - inc(index, 2) self.emitBytes((index - self.frames[^1]).toTriple()) else: - # Heap-allocated closure variable. Stored in a separate "closure array" in the VM that does not have stack semantics. - # This makes closures work as expected and is not much slower than indexing our stack (since they're both - # dynamic arrays at runtime anyway) - self.emitByte(LoadHeap) + # Heap-allocated closure variable. Stored in a separate "closure array" in the VM that does not have stack semantics + # and where the no-effect invariant is not kept. This makes closures work as expected and is not much slower than + # indexing our stack (since they're both dynamic arrays at runtime anyway) + self.emitByte(LoadClosure) self.emitBytes(self.closedOver.high().toTriple()) @@ -997,7 +1013,7 @@ proc assignment(self: Compiler, node: ASTNode) = if not t.closedOver: self.emitByte(StoreVar) else: - self.emitByte(StoreHeap) + self.emitByte(StoreClosure) self.emitBytes(index.toTriple()) else: self.error(&"reference to undeclared name '{node.token.lexeme}'") @@ -1017,7 +1033,7 @@ proc beginScope(self: Compiler) = inc(self.scopeDepth) -proc endScope(self: Compiler) = +proc endScope(self: Compiler, fromFunc: bool = false) = ## Ends the current local scope if self.scopeDepth < 0: self.error("cannot call endScope with scopeDepth < 0 (This is an internal error and most likely a bug)") @@ -1025,18 +1041,12 @@ proc endScope(self: Compiler) = var names: seq[Name] = @[] for name in self.names: if name.depth > self.scopeDepth: - if name.valueType.kind != Function and OpCode(self.chunk.code[name.codePos]) == NoOp: - for _ in countup(0, 3): - # Since by deleting it the size of the - # sequence decreases, we don't need to - # increase the index - self.chunk.code.delete(name.codePos) names.add(name) - if not self.enableOptimizations: + if not self.enableOptimizations and not fromFunc: # All variables with a scope depth larger than the current one # are now out of scope. Begone, you're now homeless! self.emitByte(Pop) - if self.enableOptimizations and len(names) > 1: + if self.enableOptimizations and len(names) > 1 and not fromFunc: # If we're popping less than 65535 variables, then # we can emit a PopN instruction. This is true for # 99.99999% of the use cases of the language (who the @@ -1050,7 +1060,7 @@ proc endScope(self: Compiler) = for i in countdown(self.names.high(), len(names) - uint16.high().int()): if self.names[i].depth > self.scopeDepth: self.emitByte(Pop) - elif len(names) == 1: + elif len(names) == 1 and not fromFunc: # We only emit PopN if we're popping more than one value self.emitByte(Pop) # This seems *really* slow, but @@ -1068,6 +1078,7 @@ proc endScope(self: Compiler) = for name in names: if name.isClosedOver: self.closedOver.delete(idx) + self.emitByte(PopClosure) inc(idx) @@ -1136,9 +1147,10 @@ proc whileStmt(self: Compiler, node: WhileStmt) = self.emitLoop(start) -proc callFunction(self: Compiler, node: CallExpr) = +proc callExpr(self: Compiler, node: CallExpr) = ## Compiles code to call a function var args: seq[tuple[name: string, kind: Type]] = @[] + var argExpr: seq[Expression] = @[] var kind: Type # TODO: Keyword arguments for i, argument in node.arguments.positionals: @@ -1148,6 +1160,7 @@ proc callFunction(self: Compiler, node: CallExpr) = self.error(&"reference to undeclared identifier '{IdentExpr(argument).name.lexeme}'") self.error(&"cannot infer the type of argument {i + 1} in function call") args.add(("", kind)) + argExpr.add(argument) for argument in node.arguments.keyword: discard if args.len() >= 16777216: @@ -1158,16 +1171,7 @@ proc callFunction(self: Compiler, node: CallExpr) = funct = self.matchImpl(IdentExpr(node.callee).name.lexeme, Type(kind: Function, returnType: Type(kind: Any), args: args)) else: discard # TODO: Lambdas - self.emitByte(LoadUInt32) - # We patch it later! - let idx = self.chunk.consts.len() - self.emitBytes(self.chunk.writeConstant((0xffffffff'u32).toQuad())) - for argument in node.arguments.positionals: - self.expression(argument) - self.emitByte(Call) # Creates a stack frame - self.emitBytes(funct.codePos.toTriple()) - self.emitBytes(args.len().toTriple()) - self.patchReturnAddress(idx) + self.generateCall(funct, argExpr) proc expression(self: Compiler, node: Expression) = @@ -1178,8 +1182,8 @@ proc expression(self: Compiler, node: Expression) = # error in self.identifier() self.error("expression has no type") case node.kind: - of callExpr: - self.callFunction(CallExpr(node)) # TODO + of NodeKind.callExpr: + self.callExpr(CallExpr(node)) # TODO of getItemExpr: discard # TODO # Note that for setItem and assign we don't convert @@ -1243,9 +1247,11 @@ proc returnStmt(self: Compiler, node: ReturnStmt) = let typ = self.inferType(self.currentFunction) ## Having the return type if returnType == nil and typ.returnType != nil: + if node.value.kind == identExpr: + self.error(&"reference to undeclared identifier '{node.value.token.lexeme}'") self.error(&"expected return value of type '{self.typeToStr(typ.returnType)}', but expression has no type") elif typ.returnType == nil and returnType != nil: - self.error("empty return statement is not allowed in non-void functions") + self.error("non-empty return statement is not allowed in void functions") elif not self.compareTypes(returnType, typ.returnType): self.error(&"expected return value of type '{self.typeToStr(typ.returnType)}', got '{self.typeToStr(returnType)}' instead") if node.value != nil: @@ -1316,7 +1322,11 @@ proc statement(self: Compiler, node: Statement) = of exprStmt: var expression = ExprStmt(node).expression self.expression(expression) - self.emitByte(Pop) # Expression statements discard their value. Their main use case is side effects in function calls + # We only print top-level expressions + if self.replMode and self.scopeDepth == 0: + self.emitByte(PopRepl) + else: + self.emitByte(Pop) # Expression statements discard their value. Their main use case is side effects in function calls of NodeKind.ifStmt: self.ifStmt(IfStmt(node)) of NodeKind.assertStmt: @@ -1375,11 +1385,21 @@ proc funDecl(self: Compiler, node: FunDecl) = ## Compiles function declarations # A function's code is just compiled linearly # and then jumped over - self.emitByte(LoadNil) # Aligns the stack + + # While the compiler stores functions as if they + # were on the stack, in reality there is nothing + # at runtime that represents them (instead, we + # compile everything in a single blob and jump back + # and forth). If we didn't align the stack to account + # for this behavior, all of our offsets would be off + # by how many functions we have declared. We could fix + # this by storing functions in a separate list of identifiers, + # but that's rather unelegant and requires specialized functions + # for looking identifiers vs functions, which is not ideal let jmp = self.emitJump(JumpForwards) var function = self.currentFunction self.declareName(node) - self.frames.add(self.names.len()) + self.frames.add(self.names.len() - 1) # TODO: Forward declarations if node.body != nil: if BlockStmt(node.body).code.len() == 0: @@ -1409,18 +1429,11 @@ proc funDecl(self: Compiler, node: FunDecl) = # We let our debugger know a function is starting let start = self.chunk.code.high() - self.blockStmt(BlockStmt(node.body)) - # Yup, we're done. That was easy, huh? - # But, after all, functions are just named - # scopes, and we compile them just like that: - # we declare their name and arguments (before - # their body so recursion works) and then just - # handle them as a block statement (which takes - # care of incrementing self.scopeDepth so locals - # are resolved properly). There's a need for a bit - # of boilerplate code to make closures work, but - # that's about it - + self.beginScope() + # self.emitByte(LoadNil) + for decl in BlockStmt(node.body).code: + self.declaration(decl) + self.endScope(fromFunc=true) case self.currentFunction.kind: of NodeKind.funDecl: if not self.currentFunction.hasExplicitReturn: @@ -1497,8 +1510,7 @@ proc compile*(self: Compiler, ast: seq[Declaration], file: string): Chunk = self.declaration(Declaration(self.step())) if self.ast.len() > 0: # *Technically* an empty program is a valid program - self.endScope() - self.emitByte(OpCode.Return) # Exits the VM's main loop when used at the global scope + self.emitByte(ProgExit) result = self.chunk - if self.ast.len() > 0 and self.scopeDepth != -1: - self.error(&"invalid state: invalid scopeDepth value (expected -1, got {self.scopeDepth}), did you forget to call endScope/beginScope?") + if self.ast.len() > 0 and self.scopeDepth != 0: + self.error(&"invalid state: invalid scopeDepth value (expected 0, got {self.scopeDepth}), did you forget to call endScope/beginScope?") diff --git a/src/frontend/meta/bytecode.nim b/src/frontend/meta/bytecode.nim index 5a206f0..211fc23 100644 --- a/src/frontend/meta/bytecode.nim +++ b/src/frontend/meta/bytecode.nim @@ -90,14 +90,16 @@ type LoadInf, ## Basic stack operations Pop, # Pops an element off the stack and discards it + PopRepl, # Same as Pop, but also prints the value of what's popped (used in REPL mode) Push, # Pushes x onto the stack PopN, # Pops x elements off the stack (optimization for exiting local scopes which usually pop many elements) ## Name resolution/handling LoadAttribute, # Pushes the attribute b of object a onto the stack LoadVar, # Pushes the object at position x in the stack onto the stack StoreVar, # Stores the value of b at position a in the stack - LoadHeap, # Pushes the object position x in the closure array onto the stack - StoreHeap, # Stores the value of b at position a in the closure array + LoadClosure, # Pushes the object position x in the closure array onto the stack + StoreClosure, # Stores the value of b at position a in the closure array + PopClosure, # Pops a closed-over variable from the closure array ## Looping and jumping Jump, # Absolute, unconditional jump into the bytecode JumpForwards, # Relative, unconditional, positive jump in the bytecode @@ -129,6 +131,7 @@ type ## Misc Assert, # Raises an AssertionFailed exception if x is false NoOp, # Just a no-op + ProgExit, # Terminates the whole program # We group instructions by their operation/operand types for easier handling when debugging @@ -137,9 +140,10 @@ type const simpleInstructions* = {Return, LoadNil, LoadTrue, LoadFalse, LoadNan, LoadInf, - Pop, Raise, BeginTry, - FinishTry, Yield, - Await, NoOp, ReturnValue} + Pop, PopRepl, Raise, + BeginTry, FinishTry, Yield, + Await, NoOp, ReturnValue, + PopClosure, ProgExit} # Constant instructions are instructions that operate on the bytecode constant table const constantInstructions* = {LoadInt64, LoadUInt64, @@ -151,7 +155,7 @@ const constantInstructions* = {LoadInt64, LoadUInt64, # Stack triple instructions operate on the stack at arbitrary offsets and pop arguments off of it in the form # of 24 bit integers -const stackTripleInstructions* = {StoreVar, LoadVar, LoadHeap, StoreHeap} +const stackTripleInstructions* = {StoreVar, LoadVar, LoadCLosure, StoreClosure} # Stack double instructions operate on the stack at arbitrary offsets and pop arguments off of it in the form # of 16 bit integers diff --git a/src/main.nim b/src/main.nim index 5d22fd6..479e01f 100644 --- a/src/main.nim +++ b/src/main.nim @@ -47,7 +47,7 @@ proc repl = serialized: Serialized tokenizer = newLexer() parser = newParser() - compiler = newCompiler() + compiler = newCompiler(replMode=true) debugger = newDebugger() serializer = newSerializer() vm = newPeonVM() @@ -136,33 +136,6 @@ proc repl = when debugRuntime: styledEcho fgCyan, "\n\nExecution step: " vm.run(serialized.chunk) - var popped = vm.lastPop - case popped.kind: - of Int64: - echo &"{popped.long}'i64" - of UInt64: - echo &"{popped.uLong}'u64" - of Int32: - echo &"{popped.`int`}'i32" - of UInt32: - echo &"{popped.uInt}'u32" - of Int16: - echo &"{popped.short}'i16" - of UInt16: - echo &"{popped.uShort}'u16" - of Int8: - echo &"{popped.tiny}'i8" - of UInt8: - echo &"{popped.uTiny}'u8" - of ObjectKind.Inf: - if popped.positive: - echo "inf" - else: - echo "-inf" - of ObjectKind.Nan, Nil: - echo ($popped.kind).toLowerAscii() - else: - discard except LexingError: input = "" let exc = LexingError(getCurrentException()) diff --git a/src/util/debugger.nim b/src/util/debugger.nim index 51cd058..812dd2e 100644 --- a/src/util/debugger.nim +++ b/src/util/debugger.nim @@ -131,7 +131,7 @@ proc callInstruction(self: Debugger, instruction: OpCode) = var args = [self.chunk.code[self.current + 4], self.chunk.code[self.current + 5], self.chunk.code[self.current + 6]].fromTriple() printInstruction(instruction) stdout.styledWrite(fgGreen, &", jumps to address ", fgYellow, $slot, fgGreen, " with ", fgYellow, $args, fgGreen, " argument") - if args > 1: + if args > 1 or args == 0: stdout.styledWrite(fgGreen, "s") nl() self.current += 7 diff --git a/tests/closures.pn b/tests/closures.pn index e54ebb5..9059841 100644 --- a/tests/closures.pn +++ b/tests/closures.pn @@ -3,12 +3,7 @@ fn outer { fn inner { var y = x; } + inner(); } - -fn outerTwo: fn: int { - fn inner: int { - return 0; - } - return inner; -} +outer();