# Copyright 2022 Mattia Giambirtone & All Contributors # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. ## The Peon runtime environment import std/monotimes import std/math import std/segfaults import std/strutils import std/sequtils import std/sets import ../config import ../frontend/meta/bytecode import ../util/multibyte when debugVM or debugMem or debugGC: import std/strformat import std/terminal {.push checks:on.} # The VM is a critical point where checks are deleterious type PeonVM* = ref object ## The Peon Virtual Machine. ## Note how the only data ## type we handle here is ## a 64-bit unsigned integer: ## This is to allow the use ## of unboxed primitive types. ## For more complex types, the ## value represents a pointer to ## some stack- or heap-allocated ## object. The VM has no concept ## of type by itself and it relies ## on the compiler to produce the ## correct results ip: uint64 # Instruction pointer chunk: Chunk # Piece of bytecode to execute calls: seq[uint64] # Our call stack operands: seq[uint64] # Our operand stack cache: array[6, uint64] # Singletons cache frames: seq[uint64] # Stores the bottom of stack frames closures: seq[uint64] # Stores closure offsets envs: seq[uint64] # Stores variables that do not have stack semantics results: seq[uint64] # Stores function's results (return values) gc: PeonGC ObjectKind* = enum ## A tag for heap-allocated ## peon objects String, List, Dict, Tuple, CustomType, Closure HeapObject* = object ## A tagged box for a heap-allocated ## peon object marked*: bool case kind*: ObjectKind of String: str*: ptr UncheckedArray[char] len*: int else: discard # TODO PeonGC* = ref object ## A simple Mark&Sweep collector ## to manage peon's heap space vm: PeonVM bytesAllocated: tuple[total, current: int] nextGC: int pointers: HashSet[uint64] objects: seq[ptr HeapObject] # Implementation of peon's memory manager proc newPeonGC*: PeonGC = ## Initializes a new, blank ## garbage collector new(result) result.bytesAllocated = (0, 0) result.objects = @[] result.nextGC = FirstGC proc collect*(self: PeonGC) proc reallocate*(self: PeonGC, p: pointer, oldSize: int, newSize: int): pointer = ## Simple wrapper around realloc/dealloc with ## built-in garbage collection self.bytesAllocated.current += newSize - oldSize try: if newSize == 0 and not p.isNil(): when debugMem: if oldSize > 1: echo &"DEBUG - Memory manager: Deallocating {oldSize} bytes of memory" else: echo "DEBUG - Memory manager: Deallocating 1 byte of memory" dealloc(p) elif (oldSize > 0 and not p.isNil() and newSize > oldSize) or oldSize == 0: self.bytesAllocated.total += newSize - oldSize when debugStressGC: self.collect() else: if self.bytesAllocated.current > self.nextGC: self.collect() when debugMem: if oldSize == 0: if newSize > 1: echo &"DEBUG - Memory manager: Allocating {newSize} bytes of memory" else: echo "DEBUG - Memory manager: Allocating 1 byte of memory" else: echo &"DEBUG - Memory manager: Resizing {oldSize} bytes of memory to {newSize} bytes" result = realloc(p, newSize) when debugMem: if p.isNil() and newSize == 0: echo &"DEBUG - Memory manager: Warning, asked to dealloc() nil pointer from {oldSize} to {newSize} bytes, ignoring request" elif oldSize > 0 and p.isNil(): echo &"DEBUG - Memory manager: Warning, asked to realloc() nil pointer from {oldSize} to {newSize} bytes, ignoring request" except NilAccessDefect: stderr.write("Peon: could not manage memory, segmentation fault\n") quit(139) # For now, there's not much we can do if we can't get the memory we need, so we exit template resizeArray*(self: PeonGC, kind: untyped, p: pointer, oldCount, newCount: int): untyped = ## Handy template to resize a dynamic array cast[ptr UncheckedArray[kind]](reallocate(self, p, sizeof(kind) * oldCount, sizeof(kind) * newCount)) template freeArray*(self: PeonGC, kind: untyped, p: pointer, size: int): untyped = ## Frees a dynamic array discard reallocate(self, p, sizeof(kind) * size, 0) template free*(self: PeonGC, kind: typedesc, p: pointer): untyped = ## Frees a pointer by reallocating its ## size to 0 discard reallocate(self, p, sizeof(kind), 0) proc allocate*(self: PeonGC, kind: ObjectKind, size: typedesc, count: int): ptr HeapObject {.inline.} = ## Allocates aobject on the heap result = cast[ptr HeapObject](self.reallocate(nil, 0, sizeof(HeapObject) * 1)) result.marked = false self.bytesAllocated.total += sizeof(result) self.bytesAllocated.current += sizeof(result) case kind: of String: result.str = cast[ptr UncheckedArray[char]](self.reallocate(nil, 0, sizeof(size) * count)) result.len = count self.bytesAllocated.current += sizeof(size) * count else: discard # TODO self.objects.add(result) self.pointers.incl(cast[uint64](result)) proc mark(self: ptr HeapObject): bool = ## Marks a single object if self.marked: return false self.marked = true return true proc markRoots(self: PeonGC): seq[ptr HeapObject] = ## Marks root objects *not* to be ## collected by the GC and returns ## their addresses # Unlike what bob does in his book, # we keep track of objects in a different # way due to how the whole thing is designed. # Specifically, we don't have neat structs for # all peon objects: When we allocate() an object, # we keep track of the small wrapper it created # along with its type and other metadata. Then, # we can go through the various sources of roots # in the VM, see if they match any pointers we # already know about (we store them in a hash set so # it's really fast), and then we can be sure that # anything that's in the difference (i.e. mathematical # set difference) between our full list of pointers # and the live ones is not a root object, so if it's # not indirectly reachable through a root itself, it # can be freed. I'm not sure if I can call this GC # strategy precise, since technically there is a chance # for a regular value to collide with one of the pointers # we allocated and that would cause a memory leak, but # with a 64-bit address-space it probably hardly matters, # so I guess this is a mostly-precise Mark&Sweep collector when debugGC: echo "DEBUG - GC: Starting mark phase" var live = initHashSet[uint64]() for obj in self.vm.calls: if obj in self.pointers: live.incl(obj) for obj in self.vm.operands: if obj in self.pointers: live.incl(obj) for obj in self.vm.envs: if obj in self.pointers: live.incl(obj) # We preallocate the space on the seq result = newSeqOfCap[ptr HeapObject](len(live)) var obj: ptr HeapObject for p in live: obj = cast[ptr HeapObject](p) if obj.mark(): when debugGC: echo &"DEBUG - GC: Marking object: {obj[]}" result.add(obj) when debugGC: echo "DEBUG - GC: Mark phase complete" proc trace(self: PeonGC, roots: seq[ptr HeapObject]) = ## Traces references to other ## objects starting from the ## roots. The second argument ## is the output of the mark ## phase. To speak in terms ## of the tricolor abstraction, ## this is where we blacken gray ## objects when debugGC: echo &"DEBUG - GC: Tracing indirect references from {len(roots)} roots" for root in roots: case root.kind: of String: discard # Strings hold no additional references else: discard # TODO: Other types when debugGC: echo &"DEBUG - GC: Tracing phase complete" proc free(self: PeonGC, obj: ptr HeapObject) = ## Frees a single heap-allocated ## peon object and all the memory ## it directly or indirectly owns when debugAlloc: echo &"DEBUG - GC: Freeing object: {obj[]}" case obj.kind: of String: # Strings only own their # underlying character array if obj.len > 0 and not obj.str.isNil(): self.freeArray(char, obj.str, obj.len) else: discard # TODO self.free(HeapObject, obj) self.pointers.excl(cast[uint64](obj)) proc sweep(self: PeonGC) = ## Sweeps unmarked objects ## that have been left behind ## during the mark phase. ## This is more convoluted ## than it needs to be because ## nim disallows changing the ## size of a sequence during ## iteration when debugGC: echo "DEBUG - GC: Beginning sweeping phase" var j = -1 var idx = 0 var count = 0 while j < self.objects.high(): inc(j) if self.objects[j].marked: # Object is marked: don't touch it, # but reset its mark so that it doesn't # stay alive forever self.objects[j].marked = false when debugGC: echo &"DEBUG - GC: Unmarking object: {self.objects[j][]}" inc(idx) else: # Object is unmarked: its memory is # fair game self.free(self.objects[idx]) self.objects.delete(idx) inc(idx) inc(count) when debugGC: echo &"DEBUG - GC: Swept {count} objects" proc collect(self: PeonGC) = ## Attempts to reclaim some ## memory from unreachable ## objects onto the heap let before {.used.} = self.bytesAllocated.current let time {.used.} = getMonoTime().ticks().float() / 1_000_000 when debugGC: echo &"DEBUG - GC: Starting collection cycle at heap size {self.bytesAllocated.current}" self.trace(self.markRoots()) self.sweep() self.nextGC = self.bytesAllocated.current * HeapGrowFactor when debugGC: echo &"DEBUG - GC: Collection cycle has terminated in {getMonoTime().ticks().float() / 1_000_000 - time:.2f} ms, collected {before - self.bytesAllocated.current} bytes of memory in total" echo &"DEBUG - GC: Next cycle at {self.nextGC} bytes" proc initCache*(self: PeonVM) = ## Initializes the VM's ## singletons cache self.cache[0] = 0x0 # False self.cache[1] = 0x1 # True self.cache[2] = 0x2 # Nil self.cache[3] = 0x3 # Positive inf self.cache[4] = 0x4 # Negative inf self.cache[5] = 0x5 # NaN proc newPeonVM*: PeonVM = ## Initializes a new, blank VM ## for executing Peon bytecode new(result) result.ip = 0 result.initCache() result.gc = newPeonGC() result.frames = @[] result.calls = @[] result.operands = @[] result.results = @[] result.envs = @[] result.gc.vm = result # Getters for singleton types {.push inline.} proc getNil*(self: PeonVM): uint64 = self.cache[2] proc getBool*(self: PeonVM, value: bool): uint64 = if value: return self.cache[1] return self.cache[0] proc getInf*(self: PeonVM, positive: bool): uint64 = if positive: return self.cache[3] return self.cache[4] proc getNan*(self: PeonVM): uint64 = self.cache[5] # Thanks to nim's *genius* idea of making x !> y a template # for y < x (which by itself is fine) together with the fact # that the order of evaluation of templates with the same # expression is fucking stupid (see https://nim-lang.org/docs/manual.html#order-of-evaluation # and https://github.com/nim-lang/Nim/issues/10425 and try not to # bang your head against the nearest wall), we need a custom operator # that preserves the natural order of evaluation proc `!>`[T](a, b: T): auto {.inline.} = b < a proc `!>=`[T](a, b: T): auto {.inline, used.} = b <= a # Stack primitives. Note: all accesses to the call stack # that go through the getc/setc wrappers is frame-relative, # meaning that the index is added to the current stack frame's # bottom to obtain an absolute stack index proc push(self: PeonVM, obj: uint64) = ## Pushes a value object onto the ## operand stack self.operands.add(obj) proc pop(self: PeonVM): uint64 = ## Pops a value off the operand ## stack and returns it return self.operands.pop() proc peekb(self: PeonVM, distance: BackwardsIndex = ^1): uint64 = ## Returns the value at the given (backwards) ## distance from the top of the operand stack ## without consuming it return self.operands[distance] proc peek(self: PeonVM, distance: int = 0): uint64 = ## Returns the value at the given ## distance from the top of the ## operand stack without consuming it if distance < 0: return self.peekb(^(-int(distance))) return self.operands[self.operands.high() + distance] proc pushc(self: PeonVM, val: uint64) = ## Pushes a value onto the ## call stack self.calls.add(val) proc popc(self: PeonVM): uint64 = ## Pops a value off the call ## stack and returns it return self.calls.pop() proc peekc(self: PeonVM, distance: int = 0): uint64 {.used.} = ## Returns the value at the given ## distance from the top of the ## call stack without consuming it return self.calls[self.calls.high() + distance] proc getc(self: PeonVM, idx: int): uint64 = ## Getter method that abstracts ## indexing our call stack through ## stack frames return self.calls[idx.uint64 + self.frames[^1]] proc setc(self: PeonVM, idx: int, val: uint64) = ## Setter method that abstracts ## indexing our call stack through ## stack frames self.calls[idx.uint + self.frames[^1]] = val proc getClosure(self: PeonVM, idx: int): uint64 = ## Getter method that abstracts ## indexing closure environments return self.envs[idx.uint + self.closures[^1]] proc setClosure(self: PeonVM, idx: int, val: uint64) = ## Setter method that abstracts ## indexing closure environments if idx == self.envs.len(): self.envs.add(val) else: self.envs[idx.uint + self.closures[^1]] = val proc popClosure(self: PeonVM, idx: int): uint64 = ## Pop method that abstracts ## popping values off closure ## environments var idx = idx.uint + self.closures[^1] result = self.envs[idx] self.envs.delete(idx) # Byte-level primitives to read and decode # bytecode proc readByte(self: PeonVM): uint8 = ## Reads a single byte from the ## bytecode and returns it as an ## unsigned 8 bit integer inc(self.ip) return self.chunk.code[self.ip - 1] proc readShort(self: PeonVM): uint16 = ## Reads two bytes from the ## bytecode and returns them ## as an unsigned 16 bit ## integer return [self.readByte(), self.readByte()].fromDouble() proc readLong(self: PeonVM): uint32 = ## Reads three bytes from the ## bytecode and returns them ## as an unsigned 32 bit ## integer. Note however that ## the boundary is capped at ## 24 bits instead of 32 return uint32([self.readByte(), self.readByte(), self.readByte()].fromTriple()) proc readUInt(self: PeonVM): uint32 = ## Reads three bytes from the ## bytecode and returns them ## as an unsigned 32 bit ## integer return uint32([self.readByte(), self.readByte(), self.readByte(), self.readByte()].fromQuad()) # Functions to read primitives from the chunk's # constants table proc constReadInt64(self: PeonVM, idx: int): int64 = ## Reads a constant from the ## chunk's constant table and ## returns it as an int64 var arr = [self.chunk.consts[idx], self.chunk.consts[idx + 1], self.chunk.consts[idx + 2], self.chunk.consts[idx + 3], self.chunk.consts[idx + 4], self.chunk.consts[idx + 5], self.chunk.consts[idx + 6], self.chunk.consts[idx + 7], ] copyMem(result.addr, arr.addr, sizeof(arr)) proc constReadUInt64(self: PeonVM, idx: int): uint64 = ## Reads a constant from the ## chunk's constant table and ## returns it as an uint64 var arr = [self.chunk.consts[idx], self.chunk.consts[idx + 1], self.chunk.consts[idx + 2], self.chunk.consts[idx + 3], self.chunk.consts[idx + 4], self.chunk.consts[idx + 5], self.chunk.consts[idx + 6], self.chunk.consts[idx + 7], ] copyMem(result.addr, arr.addr, sizeof(arr)) proc constReadUInt32(self: PeonVM, idx: int): uint32 = ## Reads a constant from the ## chunk's constant table and ## returns it as an int32 var arr = [self.chunk.consts[idx], self.chunk.consts[idx + 1], self.chunk.consts[idx + 2], self.chunk.consts[idx + 3]] copyMem(result.addr, arr.addr, sizeof(arr)) proc constReadInt32(self: PeonVM, idx: int): int32 = ## Reads a constant from the ## chunk's constant table and ## returns it as an uint32 var arr = [self.chunk.consts[idx], self.chunk.consts[idx + 1], self.chunk.consts[idx + 2], self.chunk.consts[idx + 3]] copyMem(result.addr, arr.addr, sizeof(arr)) proc constReadInt16(self: PeonVM, idx: int): int16 = ## Reads a constant from the ## chunk's constant table and ## returns it as an int16 var arr = [self.chunk.consts[idx], self.chunk.consts[idx + 1]] copyMem(result.addr, arr.addr, sizeof(arr)) proc constReadUInt16(self: PeonVM, idx: int): uint16 = ## Reads a constant from the ## chunk's constant table and ## returns it as an uint16 var arr = [self.chunk.consts[idx], self.chunk.consts[idx + 1]] copyMem(result.addr, arr.addr, sizeof(arr)) proc constReadInt8(self: PeonVM, idx: int): int8 = ## Reads a constant from the ## chunk's constant table and ## returns it as an int8 result = int8(self.chunk.consts[idx]) proc constReadUInt8(self: PeonVM, idx: int): uint8 = ## Reads a constant from the ## chunk's constant table and ## returns it as an uint8 result = self.chunk.consts[idx] proc constReadFloat32(self: PeonVM, idx: int): float32 = ## Reads a constant from the ## chunk's constant table and ## returns it as a float32 var arr = [self.chunk.consts[idx], self.chunk.consts[idx + 1], self.chunk.consts[idx + 2], self.chunk.consts[idx + 3]] copyMem(result.addr, arr.addr, sizeof(arr)) proc constReadFloat64(self: PeonVM, idx: int): float = ## Reads a constant from the ## chunk's constant table and ## returns it as a float var arr = [self.chunk.consts[idx], self.chunk.consts[idx + 1], self.chunk.consts[idx + 2], self.chunk.consts[idx + 3], self.chunk.consts[idx + 4], self.chunk.consts[idx + 5], self.chunk.consts[idx + 6], self.chunk.consts[idx + 7]] copyMem(result.addr, arr.addr, sizeof(arr)) proc constReadString(self: PeonVM, size, idx: int): ptr HeapObject = ## Reads a constant from the ## chunk's constant table and ## returns it as a pointer to ## a heap-allocated string let str = self.chunk.consts[idx.. 0: stdout.styledWrite(fgGreen, "Call Stack: ", fgMagenta, "[") for i, e in self.calls: stdout.styledWrite(fgYellow, $e) if i < self.calls.high(): stdout.styledWrite(fgYellow, ", ") styledEcho fgMagenta, "]" if self.operands.len() !> 0: stdout.styledWrite(fgBlue, "Operand Stack: ", fgMagenta, "[") for i, e in self.operands: stdout.styledWrite(fgYellow, $e) if i < self.operands.high(): stdout.styledWrite(fgYellow, ", ") styledEcho fgMagenta, "]" if self.frames.len() !> 0: stdout.styledWrite(fgCyan, "Current Frame: ", fgMagenta, "[") for i, e in self.calls[self.frames[^1]..^1]: stdout.styledWrite(fgYellow, $e) if i < (self.calls.high() - self.frames[^1].int): stdout.styledWrite(fgYellow, ", ") styledEcho fgMagenta, "]", fgCyan stdout.styledWrite(fgRed, "Live stack frames: ", fgMagenta, "[") for i, e in self.frames: stdout.styledWrite(fgYellow, $e) if i < self.frames.high(): stdout.styledWrite(fgYellow, ", ") styledEcho fgMagenta, "]" if self.envs.len() !> 0: stdout.styledWrite(fgGreen, "Environments: ", fgMagenta, "[") for i, e in self.envs: stdout.styledWrite(fgYellow, $e) if i < self.envs.high(): stdout.styledWrite(fgYellow, ", ") styledEcho fgMagenta, "]" if self.closures.len() !> 0: stdout.styledWrite(fgGreen, "Environment offsets: ", fgMagenta, "[") for i, e in self.closures: stdout.styledWrite(fgYellow, $e) if i < self.closures.high(): stdout.styledWrite(fgYellow, ", ") styledEcho fgMagenta, "]" if self.results.len() !> 0: stdout.styledWrite(fgYellow, "Function Results: ", fgMagenta, "[") for i, e in self.results: stdout.styledWrite(fgYellow, $e) if i < self.results.high(): stdout.styledWrite(fgYellow, ", ") styledEcho fgMagenta, "]" if self.closures.len() !> 0: stdout.styledWrite(fgBlue, "Closure offsets: ", fgMagenta, "[") for i, e in self.closures: stdout.styledWrite(fgYellow, $e) if i < self.closures.high(): stdout.styledWrite(fgYellow, ", ") styledEcho fgMagenta, "]" discard readLine stdin proc dispatch*(self: PeonVM) = ## Main bytecode dispatch loop var instruction {.register.}: OpCode while true: {.computedgoto.} # https://nim-lang.org/docs/manual.html#pragmas-computedgoto-pragma when debugVM: self.debug() instruction = OpCode(self.readByte()) case instruction: # Constant loading instructions of LoadTrue: self.push(self.getBool(true)) of LoadFalse: self.push(self.getBool(false)) of LoadNan: self.push(self.getNan()) of LoadNil: self.push(self.getNil()) of LoadInf: self.push(self.getInf(true)) of LoadInt64: self.push(uint64(self.constReadInt64(int(self.readLong())))) of LoadUInt64: self.push(uint64(self.constReadUInt64(int(self.readLong())))) of LoadUInt32: self.push(uint64(self.constReadUInt32(int(self.readLong())))) of LoadInt32: self.push(uint64(self.constReadInt32(int(self.readLong())))) of LoadInt16: self.push(uint64(self.constReadInt16(int(self.readLong())))) of LoadUInt16: self.push(uint64(self.constReadUInt16(int(self.readLong())))) of LoadInt8: self.push(uint64(self.constReadInt8(int(self.readLong())))) of LoadUInt8: self.push(uint64(self.constReadUInt8(int(self.readLong())))) of LoadString: # TODO: Use constReadString with own memory manager # Strings are broken rn!! self.push(cast[uint64](self.constReadString(int(self.readLong()), int(self.readLong())))) # We cast instead of converting because, unlike with integers, # we don't want nim to touch any of the bits of the underlying # value! of LoadFloat32: self.push(cast[uint64](self.constReadFloat32(int(self.readLong())))) of LoadFloat64: self.push(cast[uint64](self.constReadFloat64(int(self.readLong())))) of Call: # Calls a peon function. The calling convention here # is pretty simple: the first value in the frame is # the new instruction pointer to jump to, then a # 32-bit return address follows. After that, all # arguments and locals follow. Note that, due to # how the stack works, all arguments before the call # are in the reverse order in which they are passed # to the function let argc = self.readLong().int let retAddr = self.peek(-argc - 1) # Return address let jmpAddr = self.peek(-argc - 2) # Function address self.ip = jmpAddr self.pushc(jmpAddr) self.pushc(retAddr) # Creates a new result slot for the # function's return value self.results.add(self.getNil()) # Creates a new call frame self.frames.add(uint64(self.calls.len() - 2)) # Loads the arguments onto the stack for _ in 0..