WIP on closures: single layer closures work, nested closures still broken

.gitignore

@@ -6,4 +6,7 @@ bin/nds
 *.lua
 *.py
 benchmarkgen
-test
+test
+perf.data
+test.perf
+perf.data.old

examples/test.nds

@@ -0,0 +1,17 @@
+outer = funct() {
+  var a = 1;
+  var b = 2;
+  var result;
+  var middle = funct() {
+    var c = 3;
+    var d = 4;
+    var inner = funct() {
+      print a + c + b + d;
+    };
+    result = inner;
+  };
+  :result = result;
+};
+
+//expect:10.0
+outer()();

perf.sh

@@ -0,0 +1,6 @@
+# change this to the path to nds and to the path to the program profiled
+sudo perf record -g -F 999 /home/user/Projects/nondescript/bin/nds benchmarks/fib.nds
+
+sudo perf script -F +pid > test.perf
+
+# https://profiler.firefox.com/

src/ndspkg/chunk.nim

@@ -15,7 +15,7 @@ type
     opConstant, # constant
     opDefineGlobal, opGetGlobal, opSetGlobal, # globals (uses constants)
     opGetLocal, opSetLocal, # locals
-    opGetUpvalue, opSetUpvalue, # upvalues
+    opGetUpvalue, opSetUpvalue, opCloseUpvalue, # upvalues
     opJumpIfFalse, opJump, opLoop, opJumpIfFalsePop, # jumps
     opCreateList, opCreateTable, # collection creation
     opLen, opSetIndex, opGetIndex, # collection operators
@@ -115,7 +115,7 @@ const argInstructions = {
   opJumpIfFalse, opJump, opLoop, opJumpIfFalsePop,
   opFunctionDef,
   opCreateList, opCreateTable,
-  opGetUpvalue, opSetUpvalue,
+  opGetUpvalue, opSetUpvalue, opCloseUpvalue,
 }
 
 proc disassembleInstruction*(ch: Chunk, c: var int, lastLine: var int) =

src/ndspkg/compiler.nim

@@ -17,6 +17,7 @@ type
     # if depth is -1, the variable cannot be referenced yet
     # its depth will be set once its first ever value is determined
     scope: Scope # the innermost scope of this local
+    captured: bool
   
   Upvalue = ref object
     index: int
@@ -181,7 +182,7 @@ proc addLocal(comp: Compiler, name: string, delta: int) =
 
   # if delta is 0 or negative, it means that it is already on the stack when addLocal is called  
   # if delta is positive, the first ever value of the local is to the right
-  comp.locals.add(Local(name: name, depth: if delta > 0: -1 else: comp.scopes.high, index: comp.stackIndex + delta, scope: comp.scopes[comp.scopes.high()]))
+  comp.locals.add(Local(name: name, depth: if delta > 0: -1 else: comp.scopes.high, index: comp.stackIndex + delta, scope: comp.scopes[comp.scopes.high()], captured: false))
 
 proc markInitialized(comp: Compiler) =
   comp.locals[comp.locals.high].depth = comp.scopes.high
@@ -311,6 +312,17 @@ proc endScope(comp: Compiler): Scope =
   if popped.parentFunction == popped:
     popped.parentFunction = nil # cleanup cycles
   let function = popped.function
+  # close upvalues
+  var i = comp.locals.high()
+  while i >= 0:
+    let local = comp.locals[i]
+    if local.scope == popped and local.captured:
+      comp.writeChunk(0, opCloseUpvalue)
+      comp.writeChunk(0, local.index.toDU8())
+    if local.depth < comp.scopes.len():
+      break
+    i.dec()
+
   # restore the stackIndex, emit pops
   when debugCompiler:
     debugEcho &"End scope called for depth {comp.scopes.len} function? {function}"
@@ -451,6 +463,8 @@ proc resolveLocal(comp: Compiler, name: string): tuple[index: int, upvalue: bool
       if not upvalue:
         return (local.index, false)
       else:
+        # resolveUpvalue
+        local.captured = true
         return (comp.addUpvalue(i), true)
     i.dec
   return (index: -1, upvalue: false)

src/ndspkg/config.nim

@@ -12,6 +12,7 @@ const assertionsCompiler* = true # sanity checks in the compiler
 const debugVM* = defined(debug)
 const assertionsVM* = defined(debug) # sanity checks in the VM, such as the stack being empty at the end
 const profileInstructions* = defined(ndsprofile) # if true, the time spent on every opcode is measured
+const debugClosures* = defined(debug) # specific closure debug switches
 
 # choose a line editor for the repl
 const lineEditor = leRdstdin

src/ndspkg/pointerutils.nim

@@ -9,4 +9,10 @@ template pdiff*[T](x: ptr T, y: ptr T): int =
   cast[int](x) - cast[int](y)
 
 template pless*[T](x: ptr T, y: ptr T): bool =
-  cast[uint](x) < cast[uint](y)
+  cast[uint](x) < cast[uint](y)
+
+template pgreater*[T](x: ptr T, y: ptr T): bool =
+  cast[uint](x) > cast[uint](y)
+
+template pge*[T](x: ptr T, y: ptr T): bool =
+  cast[uint](x) >= cast[uint](y)

src/ndspkg/types/closure.nim

@@ -1,12 +1,17 @@
+import strformat
+import strutils
+
 type
   UpvalueObj[T] = object
-    location: ptr T
+    location*: ptr T
+    next*: Upvalue[T]
+    closed*: T
   
   Upvalue*[T] = ptr UpvalueObj[T]
 
   ClosureObj[T] = object
     start: ptr uint8
-    upvalueCount: int
+    upvalueCount*: int
     upvalues: UncheckedArray[Upvalue[T]]
     
   Closure*[T] = ptr ClosureObj[T]
@@ -15,6 +20,8 @@ proc newClosure*[T](start: ptr uint8, upvalueCount: int): Closure[T] =
   result = cast[Closure[T]](alloc0(8 * upvalueCount + sizeof(ClosureObj[T])))
   result.start = start
   result.upvalueCount = upvalueCount
+  for i in 0 .. upvalueCount:
+    result.upvalues[i] = nil
 
 proc getIp*[T](clos: Closure[T]): ptr uint8 {.inline.} =
   clos.start
@@ -25,9 +32,23 @@ proc set*[T](clos: Closure[T], index: int, val: Upvalue[T]) =
 proc get*[T](clos: Closure[T], index: int): Upvalue[T] =
   clos.upvalues[index]
 
-proc captureUpvalue*[T](location: ptr T): Upvalue[T] =
+proc debugStr*[T](clos: Closure[T]): string =
+  let addrUint: uint = cast[uint](clos.start)
+  let addrStr: string = addrUint.toHex()
+  let upvalCountStr: string = $clos.upvalueCount
+  result = &"Closure(start: {addrStr}, length: {upvalCountStr}, upvalues: "
+  mixin `$`
+  for i in 0 .. clos.upvalueCount:
+    if clos.upvalues[i] != nil:
+      result &= &"{$(clos.upvalues[i].location[])}, "
+    else:
+      result &= "<unset>, "
+  result &= ")"
+
+proc newUpvalue*[T](location: ptr T): Upvalue[T] =
   result = cast[Upvalue[T]](alloc0(sizeof(UpvalueObj[T])))
   result.location = location
+  result.next = nil
 
 proc read*[T](upval: Upvalue[T]): T =
   upval.location[]

src/ndspkg/types/stack.nim

@@ -89,7 +89,7 @@ proc getIndexNeg*[T](stack: Stack[T], index: int): var T =
       raise newException(Defect, "Attempt to getIndexNeg with an index out of bounds.")
   stack.top.psub(index * sizeof(T))[]
 
-template `[]`*[T](stack: Stack[T], index: int): T =
+template `[]`*[T](stack: Stack[T], index: int): var T =
   stack.getIndex(index)
 
 proc setIndex*[T](stack: var Stack[T], index: int, item: T) =

src/ndspkg/vm.nim

@@ -12,6 +12,8 @@ import types/hashtable
 import types/ndlist
 import types/closure
 
+when debugVM:
+  import terminal
 
 when profileInstructions:
   import times
@@ -27,7 +29,6 @@ type
   Frame = object
     stackBottom: int # the absolute index of where 0 inside the frame is
     returnIp: ptr uint8
-    closure: Closure[NdValue]
 
   InterpretResult* = enum
     irOK, irRuntimeError
@@ -40,6 +41,8 @@ proc run*(chunk: Chunk): InterpretResult =
     hadError: bool
     globals: Table[NdValue, NdValue]
     frames: Stack[Frame] = newStack[Frame](4)
+    closures: Stack[Closure[NdValue]] = newStack[Closure[NdValue]](4)
+    openUpvalues: Upvalue[NdValue] = nil
 
   proc runtimeError(msg: string) =
     let ii = ip.pdiff(chunk.code[0].unsafeAddr)
@@ -73,11 +76,41 @@ proc run*(chunk: Chunk): InterpretResult =
       frames.add(Frame(stackBottom: stack.high - argcount, returnIp: ip))
       ip = funct.asFunct() # jump to the entry point
     elif funct.isClosure():
-      frames.add(Frame(stackBottom: stack.high - argcount, returnIp: ip, closure: funct.asClosure()))
+      frames.add(Frame(stackBottom: stack.high - argcount, returnIp: ip))
+      closures.add(funct.asClosure())
       ip = funct.asClosure().getIp()
     else:
       error
 
+  proc captureUpvalue(location: ptr NdValue): Upvalue[NdValue] =
+    var prev: Upvalue[NdValue]
+    var upvalue = openUpvalues
+    while upvalue != nil and upvalue.location.pgreater(location):
+      prev = upvalue
+      upvalue = upvalue.next
+    
+    # existing upvalue
+    if upvalue != nil and upvalue.location == location:
+      return upvalue
+
+    # new upvalue
+    result = newUpvalue(location)
+    result.next = upvalue
+
+    if prev == nil:
+      openUpvalues = result
+    else:
+      prev.next = result
+
+  proc closeUpvalues(last: ptr NdValue) =
+    while openUpvalues != nil and openUpvalues.location.pge(last):
+      let upval = openUpvalues
+      upval.closed = upval.location[]
+      when debugClosures:
+        echo &"CLOSURES - closeUpvalues: closing on {upval.closed}"
+      upval.location = upval.closed.addr
+      openUpvalues = upval.next
+
   while true:
     {.computedgoto.} # See https://nim-lang.org/docs/manual.html#pragmas-computedgoto-pragma
 
@@ -85,9 +118,6 @@ proc run*(chunk: Chunk): InterpretResult =
     ip = ip.padd(1)
 
     when debugVM:
-      var ii = ip.pdiff(chunk.code[0].unsafeAddr) - 1
-      var ll = -1
-      disassembleInstruction(chunk, ii, ll)
       var msg = ""
       msg &= " Stack: [ "
       for i in 0 .. stack.high():
@@ -98,6 +128,22 @@ proc run*(chunk: Chunk): InterpretResult =
           msg &= &"{e} "
       msg &= "]"
       echo msg
+
+      when debugClosures:
+        if closures.len() > 0:
+          msg = " Closures: [ "
+          for i in 0 .. closures.high():
+            msg &= debugStr(closures[i]) & " "
+          msg &= "]"
+          echo msg
+          
+
+      var ii = ip.pdiff(chunk.code[0].unsafeAddr) - 1
+      var ll = -1
+      setForegroundColor(fgYellow)
+      disassembleInstruction(chunk, ii, ll)
+      setForegroundColor(fgDefault)
+      write stdout, " "
     
     when profileInstructions:
       let startTime = getMonoTime()
@@ -192,7 +238,7 @@ proc run*(chunk: Chunk): InterpretResult =
         let name = ip.readConstant(chunk)
         let existed = globals.tableSet(name, stack.peek())
         if not existed:
-          runtimeError("Attempt to redefine an existing global variable.")
+          runtimeError("Attempt to assign to undefined global variable.")
           break
       of opGetLocal:
         let slot = ip.readDU8()
@@ -202,10 +248,18 @@ proc run*(chunk: Chunk): InterpretResult =
         stack[slot + frameBottom] = stack.peek()
       of opGetUpvalue:
         let slot = ip.readDU8()
-        stack.push(frames.peek().closure.get(slot).read())
+        let val = closures.peek().get(slot).read()
+        when debugClosures:
+          echo &"CLOSURES - getupvalue got {val} from slot {slot}"
+        stack.push(val)
       of opSetUpvalue:
         let slot = ip.readDU8()
-        frames.peek().closure.get(slot).write(stack.peek())
+        when debugClosures:
+          echo &"CLOSURES - setupvalue is setting {$stack.peek} to slot {slot}, number of slots: {closures.peek().upvalueCount}"
+        closures.peek().get(slot).write(stack.peek())
+      of opCloseUpvalue:
+        let slot = ip.readDU8()
+        stack[slot + frameBottom].addr.closeUpvalues()
       of opJumpIfFalse:
         let offset = ip.readDU8()
         if stack.peek().isFalsey():
@@ -234,9 +288,15 @@ proc run*(chunk: Chunk): InterpretResult =
           let slot = ip.readDU8()
           let isLocal = ip.readUI8() == 0
           if isLocal:
-            closure.set(i, stack[slot + frameBottom].addr.captureUpvalue())
+            let loc = stack[slot + frameBottom].addr
+            when debugClosures:
+              echo &"CLOSURES - opClosure: local upvalue {loc[]} from local slot {slot} to slot {i}"
+            closure.set(i, loc.captureUpvalue())
           else:
-            closure.set(i, frames.peek().closure.get(slot))
+            let val = closures.peek().get(slot)
+            when debugClosures:
+              echo &"CLOSURES - opClosure: non local upvalue {val.location[]} from slot {slot} to slot {i}"
+            closure.set(i, val)
 
       of opCheckArity:
         let arity = ip.readUI8()
@@ -257,7 +317,6 @@ proc run*(chunk: Chunk): InterpretResult =
         funct.call(argcount):
           runtimeError("Attempt to call a non-funct (a defunct?).") # here is a bad defunct joke
           break
-
       of opCreateList:
         let listLen = ip.readDU8()
         if listLen == 0:
@@ -319,9 +378,7 @@ proc run*(chunk: Chunk): InterpretResult =
         let code = val.asString().getIndexAsChar(0).float()
         stack.settip(code.fromFloat())
       of opPutchar:
-        write stdout, $stack.peek()
-        
-        
+        write stdout, $stack.peek() 
 
     when profileInstructions:
       durations[ins] += getMonoTime() - startTime
@@ -339,6 +396,7 @@ proc run*(chunk: Chunk): InterpretResult =
   stack.free()
   frames.free()
   globals.free()
+  closures.free()
 
   if hadError:
     irRuntimeError

tests/closures.nds

@@ -1,3 +1,36 @@
+// cascade
+
+var f1 = funct() {
+  var x = 1;
+  var y = 5;
+  :result = funct() {
+    var z = 8;
+    print x;
+    x = x + 1;
+    :result = funct() {
+      print x;
+      x = x + 1;
+      :result = funct() {
+        print x;
+        print y;
+        print z;
+        x = x + 1;
+        :result = funct() {
+          print x;
+          x = x + 1;
+        };
+      };
+    };
+  };
+};
+f1()()()()();
+//expect:1.0
+//expect:2.0
+//expect:3.0
+//expect:5.0
+//expect:8.0
+//expect:4.0
+
 // capturing closures in lists:
 
 var f = funct() {
@@ -153,7 +186,7 @@ var main5 = funct() {
   globalGet = get;
 };
 
-main();
+main5();
 globalGet();
 //expect:initial
 globalSet();
@@ -164,15 +197,15 @@ globalGet();
 
 {
   var a = 1;
-  f = funct() {
+  var f = funct() {
     print a;
   };
   var b = 2;
-  g = funct() {
+  var g = funct() {
     print b;
   };
   var c = 3;
-  h = funct() {
+  var h = funct() {
     print c;
   };
 
@@ -188,15 +221,15 @@ globalGet();
 
 var bonus = funct() {
   var a = 1;
-  f = funct() {
+  var f = funct() {
     print a;
   };
   var b = 2;
-  g = funct() {
+  var g = funct() {
     print b;
   };
   var c = 3;
-  h = funct() {
+  var h = funct() {
     print c;
   };