Browse Source

It is now possible to close over function arguments

custom-types
Mattia Giambirtone 4 months ago
parent
commit
85de75a50a
  1. 32
      src/backend/vm.nim
  2. 105
      src/frontend/compiler.nim
  3. 7
      src/frontend/meta/bytecode.nim
  4. 10
      src/util/debugger.nim
  5. 11
      tests/closures.pn

32
src/backend/vm.nim

@ -31,7 +31,7 @@ when debugVM or debugMem or debugGC:
import std/terminal
{.push checks:off.} # The VM is a critical point where checks are deleterious
{.push checks:on.} # The VM is a critical point where checks are deleterious
type
PeonVM* = ref object
@ -54,6 +54,7 @@ type
operands: seq[uint64] # Our operand stack
cache: array[6, uint64] # Singletons cache
frames: seq[uint64] # Stores the bottom of stack frames
closures: seq[uint64] # Stores closure offsets
closedOver: seq[uint64] # Stores variables that do not have stack semantics
results: seq[uint64] # Stores function's results (return values)
gc: PeonGC
@ -62,7 +63,8 @@ type
## peon objects
String, List,
Dict, Tuple,
CustomType
CustomType,
Closure
HeapObject* = object
## A tagged box for a heap-allocated
## peon object
@ -429,11 +431,11 @@ proc peekc(self: PeonVM, distance: int = 0): uint64 {.used.} =
return self.calls[self.calls.high() + distance]
proc getc(self: PeonVM, idx: uint64): uint64 =
proc getc(self: PeonVM, idx: int): uint64 =
## Accessor method that abstracts
## indexing our call stack through stack
## frames
return self.calls[idx + self.frames[^1]]
return self.calls[idx.uint64 + self.frames[^1]]
proc setc(self: PeonVM, idx: uint, val: uint64) =
@ -636,6 +638,13 @@ when debugVM: # So nim shuts up
if i < self.results.high():
stdout.styledWrite(fgYellow, ", ")
styledEcho fgMagenta, "]"
if self.closures.len() !> 0:
stdout.styledWrite(fgBlue, "Closure offsets: ", fgMagenta, "[")
for i, e in self.closures:
stdout.styledWrite(fgYellow, $e)
if i < self.closures.high():
stdout.styledWrite(fgYellow, ", ")
styledEcho fgMagenta, "]"
discard readLine stdin
@ -713,6 +722,7 @@ proc dispatch*(self: PeonVM) =
self.results.add(self.getNil())
# Creates a new call frame
self.frames.add(uint64(self.calls.len() - 2))
self.closures.add(self.closedOver.len().uint64)
# Loads the arguments onto the stack
for _ in 0..<argc:
self.pushc(self.pop())
@ -726,7 +736,7 @@ proc dispatch*(self: PeonVM) =
# Every peon program is wrapped
# in a hidden function, so this
# will also exit the VM if we're
# at the end of the program
# at the end of the program
while self.calls.len().uint64 !> self.frames[^1] + 2'u64:
# Discards the function's local variables,
# if there is any
@ -740,6 +750,7 @@ proc dispatch*(self: PeonVM) =
discard self.results.pop()
# Discard the topmost stack frame
discard self.frames.pop()
discard self.closures.pop()
if self.frames.len() == 0:
# End of the program!
return
@ -757,7 +768,7 @@ proc dispatch*(self: PeonVM) =
# into the given call stack index
let idx = self.readLong()
when debugVM:
assert idx - self.calls.high() <= 1, "StoreVar index is bigger than the length of the call stack"
assert idx.int - self.calls.high() <= 1, "StoreVar index is bigger than the length of the call stack"
if idx + self.frames[^1] <= self.calls.high().uint:
self.setc(idx, self.pop())
else:
@ -775,11 +786,16 @@ proc dispatch*(self: PeonVM) =
of LoadClosure:
# Loads a closed-over variable onto the
# stack
self.push(self.closedOver[self.readLong()])
self.push(self.closedOver[self.readLong() + self.closures[^1] - 1])
of PopClosure:
self.closedOver.delete(self.readLong())
of LiftArgument:
# Lifts a function argument onto the stack
self.closedOver.add(self.getc(self.readLong().int))
of LoadVar:
# Pushes a variable onto the operand
# stack
self.push(self.getc(self.readLong()))
self.push(self.getc(self.readLong().int))
of NoOp:
# Does nothing
continue

105
src/frontend/compiler.nim

@ -14,7 +14,6 @@
import meta/token
import meta/ast
import meta/errors
import ../config
import ../util/multibyte
import ../util/symbols
import lexer as l
@ -63,6 +62,9 @@ type
isBuiltinFunction: bool
builtinOp: string
fun: FunDecl
isClosure: bool
closureBounds: tuple[start, stop: int]
childFunc: Type
of Reference, Pointer:
value: Type
of Generic:
@ -100,6 +102,8 @@ type
codePos: int
# Is the name closed over (i.e. used in a closure)?
isClosedOver: bool
# The function that owns this variable (may be nil!)
belongsTo: Name
# Is this a function argument?
isFunctionArgument: bool
# Where is this node declared in the file?
@ -143,7 +147,7 @@ type
# in a local scope, otherwise it's global
scopeDepth: int
# The current function being compiled
currentFunction: Type
currentFunction: Name
# Are optimizations turned on?
enableOptimizations: bool
# The current loop being compiled (used to
@ -172,7 +176,7 @@ type
# be empty)
deferred: seq[uint8]
# List of closed-over variables
closedOver: seq[Name]
closedOver: seq[tuple[name: Name, count: int]]
# Keeps track of stack frames
frames: seq[int]
# Compiler procedures called by pragmas
@ -239,7 +243,7 @@ proc compileModule(self: Compiler, filename: string)
## Public getter for nicer error formatting
proc getCurrentNode*(self: Compiler): ASTNode = (if self.current >=
self.ast.len(): self.ast[^1] else: self.ast[self.current - 1])
proc getCurrentFunction*(self: Compiler): Declaration {.inline.} = (if self.currentFunction.isNil(): nil else: self.currentFunction.fun)
proc getCurrentFunction*(self: Compiler): Declaration {.inline.} = (if self.currentFunction.valueType.isNil(): nil else: self.currentFunction.valueType.fun)
proc getFile*(self: Compiler): string {.inline.} = self.file
proc getModule*(self: Compiler): string {.inline.} = self.currentModule
proc getLines*(self: Compiler): seq[tuple[start, stop: int]] = self.lines
@ -428,7 +432,7 @@ proc getClosurePos(self: Compiler, name: Name): int =
result = self.closedOver.high()
var found = false
for variable in reversed(self.closedOver):
if name == variable:
if name == variable.name:
found = true
break
dec(result)
@ -464,23 +468,29 @@ proc detectClosureVariable(self: Compiler, name: var Name, depth: int = self.sco
## unpredictably or crash
if name.isNil() or name.depth == 0 or name.isClosedOver:
return
elif name.depth < depth and self.scopes[name.depth - 1] != self.scopes[depth - 1]:
elif name.depth < depth and self.scopes[name.depth - 1] != self.scopes[depth - 1]:
# Ding! The given name is closed over in another function:
# we need to change the Jump instruction that self.declareName
# put in place for us into a StoreClosure. We also update
# the name's isClosedOver field so that self.identifier()
# can emit a LoadClosure instruction instead of a LoadVar
# once this name is referenced in the future
self.closedOver.add((name, 0))
name.isClosedOver = true
if not self.currentFunction.valueType.isClosure:
self.currentFunction.valueType.isClosure = true
self.currentFunction.valueType.closureBounds.start = self.closedOver.high()
self.currentFunction.valueType.closureBounds.stop = self.closedOver.high()
if self.closedOver.len() >= 16777216:
self.error("too many consecutive closed-over variables (max is 16777215)")
if not name.isFunctionArgument:
# We handle closed-over function arguments later
self.closedOver.add(name)
if self.closedOver.len() >= 16777216:
self.error("too many consecutive closed-over variables (max is 16777215)")
name.isClosedOver = true
self.chunk.code[name.codePos] = StoreClosure.uint8()
for i, b in self.closedOver.high().toTriple():
self.chunk.code[name.codePos + i + 1] = b
else:
self.chunk.code[name.codePos] = LiftArgument.uint8()
for i, b in self.getStackPos(name).toTriple():
self.chunk.code[name.codePos + i + 1] = b
proc compareTypes(self: Compiler, a, b: Type): bool =
@ -920,7 +930,7 @@ proc literal(self: Compiler, node: ASTNode) =
var x: float
var y = FloatExpr(node)
try:
discard parseFloat(y.literal.lexeme)
discard parseFloat(y.literal.lexeme, x)
except ValueError:
self.error("floating point value out of range")
self.emitConstant(y, self.inferType(y))
@ -1036,6 +1046,12 @@ proc generateCall(self: Compiler, fn: Name, args: seq[Expression], onStack: bool
# because of how stack semantics
# work. They'll be fixed at runtime
self.expression(argument)
var f = fn.valueType
while not f.isNil():
if f.isClosure:
for i in f.closureBounds.start..f.closureBounds.stop:
self.closedOver[i].count += 1
f = f.childFunc
# Creates a new call frame and jumps
# to the function's first instruction
# in the code
@ -1100,7 +1116,9 @@ proc declareName(self: Compiler, node: Declaration, mutable: bool = false) =
codePos: self.chunk.code.len(),
isLet: node.isLet,
isClosedOver: false,
line: node.token.line))
line: node.token.line,
belongsTo: self.currentFunction
))
if mutable:
self.names[^1].valueType.mutable = true
# We emit a jump of 0 because this may become a
@ -1163,7 +1181,9 @@ proc declareName(self: Compiler, node: Declaration, mutable: bool = false) =
isLet: false,
isClosedOver: false,
line: argument.name.token.line,
isFunctionArgument: true)
isFunctionArgument: true,
belongsTo: fn
)
self.names.add(name)
name.valueType = self.inferType(argument.valueType)
# If it's still nil, it's an error!
@ -1201,7 +1221,7 @@ proc identifier(self: Compiler, node: IdentExpr) =
# align its semantics with the call stack. This makes closures work as expected and is
# not much slower than indexing our stack (since they're both dynamic arrays at runtime anyway)
self.emitByte(LoadClosure)
self.emitBytes(self.getClosurePos(s).toTriple())
self.emitBytes((self.getClosurePos(s)).toTriple())
proc assignment(self: Compiler, node: ASTNode) =
@ -1242,7 +1262,7 @@ proc beginScope(self: Compiler) =
## Begins a new local scope by incrementing the current
## scope's depth
inc(self.scopeDepth)
self.scopes.add(self.currentFunction)
self.scopes.add(self.currentFunction.valueType)
proc endScope(self: Compiler) =
@ -1261,6 +1281,16 @@ proc endScope(self: Compiler) =
# We don't increase the pop count for these kinds of objects
# because they're not stored the same way as regular variables
inc(popCount)
if name.isFunDecl and not name.valueType.childFunc.isNil() and name.valueType.childFunc.isClosure:
var i = 0
var closure: tuple[name: Name, count: int]
for y in name.valueType.childFunc.closureBounds.start..name.valueType.childFunc.closureBounds.stop:
closure = self.closedOver[y + i]
self.closedOver.delete(y + i)
for _ in 0..<closure.count:
self.emitByte(PopClosure)
self.emitBytes((y + i).toTriple())
inc(i)
if popCount > 1:
# If we're popping less than 65535 variables, then
# we can emit a PopN instruction. This is true for
@ -1379,8 +1409,8 @@ proc callExpr(self: Compiler, node: CallExpr): Name {.discardable.} =
result = funct
self.generateCall(funct, argExpr, onStack)
if not self.checkCallIsPure(node.callee):
if self.currentFunction.name != "":
self.error(&"cannot make sure that calls to '{self.currentFunction.name}' are side-effect free")
if self.currentFunction.valueType.name != "":
self.error(&"cannot make sure that calls to '{self.currentFunction.valueType.name}' are side-effect free")
else:
self.error(&"cannot make sure that call is side-effect free")
@ -1451,7 +1481,7 @@ proc deferStmt(self: Compiler, node: DeferStmt) =
proc returnStmt(self: Compiler, node: ReturnStmt) =
## Compiles return statements
var expected = self.currentFunction.returnType
var expected = self.currentFunction.valueType.returnType
self.check(node.value, expected)
if not node.value.isNil():
self.expression(node.value)
@ -1683,21 +1713,20 @@ proc dispatchPragmas(self: Compiler, node: ASTnode) =
self.compilerProcs[pragma.name.token.lexeme](self, pragma, node)
proc fixGenericFunc(self: Compiler, name: Name, args: seq[Expression]): Type =
proc fixGenericFunc(self: Compiler, name: Name, args: seq[Expression]): Name =
## Specializes generic arguments in functions
var fn = name.valueType.deepCopy()
var fn = name.deepCopy()
result = fn
var typ: Type
for i in 0..args.high():
if fn.args[i].kind.kind == Generic:
if fn.valueType.args[i].kind.kind == Generic:
typ = self.inferType(args[i])
fn.args[i].kind = typ
self.resolve(fn.args[i].name).valueType = typ
if fn.args[i].kind.isNil():
fn.valueType.args[i].kind = typ
self.resolve(fn.valueType.args[i].name).valueType = typ
if fn.valueType.args[i].kind.isNil():
self.error(&"cannot specialize generic function: argument {i + 1} has no type")
proc funDecl(self: Compiler, node: FunDecl, fn: Name = nil, args: seq[Expression] = @[]) =
## Compiles function declarations
#[if not node.isNil():
@ -1710,6 +1739,7 @@ proc funDecl(self: Compiler, node: FunDecl, fn: Name = nil, args: seq[Expression
self.dispatchPragmas(node)
var node = node
var fn = if fn.isNil(): self.names[^(node.arguments.len() + 1)] else: fn
var names = self.names[^(node.arguments.len())..^1]
if fn.valueType.isBuiltinFunction:
# We take the arguments off of our name list
# because they become temporaries on the stack.
@ -1727,13 +1757,20 @@ proc funDecl(self: Compiler, node: FunDecl, fn: Name = nil, args: seq[Expression
jmp = self.emitJump(JumpForwards)
# Function's code starts after the jump
fn.codePos = self.chunk.code.len()
# We let our debugger know a function is starting
let start = self.chunk.code.high()
for name in names:
self.emitBytes([NoOp, NoOp, NoOp, NoOp])
name.codePos = self.chunk.code.len() - 4
# We store the current function
self.currentFunction = fn.valueType
if not self.currentFunction.isNil():
self.currentFunction.valueType.childFunc = fn.valueType
self.currentFunction = fn
if node.isNil():
# We got called back with more specific type
# arguments: time to fix them!
self.currentFunction = self.fixGenericFunc(fn, args)
node = self.currentFunction.fun
node = self.currentFunction.valueType.fun
elif not node.body.isNil():
if BlockStmt(node.body).code.len() == 0:
self.error("cannot declare function with empty body")
@ -1758,18 +1795,16 @@ proc funDecl(self: Compiler, node: FunDecl, fn: Name = nil, args: seq[Expression
# the try/finally block with the deferred
# code
var deferStart = self.deferred.len()
# We let our debugger know a function is starting
let start = self.chunk.code.high()
self.beginScope()
for decl in BlockStmt(node.body).code:
self.declaration(decl)
let typ = self.currentFunction.returnType
let typ = self.currentFunction.valueType.returnType
var hasVal: bool = false
case self.currentFunction.fun.kind:
case self.currentFunction.valueType.fun.kind:
of NodeKind.funDecl:
hasVal = self.currentFunction.fun.hasExplicitReturn
hasVal = self.currentFunction.valueType.fun.hasExplicitReturn
of NodeKind.lambdaExpr:
hasVal = LambdaExpr(Declaration(self.currentFunction.fun)).hasExplicitReturn
hasVal = LambdaExpr(Declaration(self.currentFunction.valueType.fun)).hasExplicitReturn
else:
discard # Unreachable
if not hasVal and not typ.isNil():

7
src/frontend/meta/bytecode.nim

@ -152,6 +152,8 @@ type
StoreVar, # Stores the value of b at position a in the stack
LoadClosure, # Pushes the object position x in the closure array onto the stack
StoreClosure, # Stores the value of b at position a in the closure array
LiftArgument, # Closes over a function argument
PopClosure,
## Looping and jumping
Jump, # Absolute, unconditional jump into the bytecode
JumpForwards, # Relative, unconditional, positive jump in the bytecode
@ -253,7 +255,7 @@ const constantInstructions* = {LoadInt64, LoadUInt64,
# Stack triple instructions operate on the stack at arbitrary offsets and pop arguments off of it in the form
# of 24 bit integers
const stackTripleInstructions* = {StoreVar, LoadVar, LoadCLosure, }
const stackTripleInstructions* = {StoreVar, LoadVar, LoadCLosure, LiftArgument, PopClosure}
# Stack double instructions operate on the stack at arbitrary offsets and pop arguments off of it in the form
# of 16 bit integers
@ -265,9 +267,6 @@ const argumentDoubleInstructions* = {PopN, }
# Argument double argument instructions take hardcoded arguments as 24 bit integers
const argumentTripleInstructions* = {StoreClosure}
# Instructions that call functions
const callInstructions* = {Call, }
# Jump instructions jump at relative or absolute bytecode offsets
const jumpInstructions* = {Jump, JumpIfFalse, JumpIfFalsePop,
JumpForwards, JumpBackwards,

10
src/util/debugger.nim

@ -99,7 +99,6 @@ proc simpleInstruction(self: Debugger, instruction: OpCode) =
self.checkFrameEnd(self.current)
proc stackTripleInstruction(self: Debugger, instruction: OpCode) =
## Debugs instructions that operate on a single value on the stack using a 24-bit operand
var slot = [self.chunk.code[self.current + 1], self.chunk.code[self.current + 2], self.chunk.code[self.current + 3]].fromTriple()
@ -136,9 +135,10 @@ proc argumentTripleInstruction(self: Debugger, instruction: OpCode) =
proc callInstruction(self: Debugger, instruction: OpCode) =
## Debugs function calls
var size = [self.chunk.code[self.current + 1], self.chunk.code[self.current + 2], self.chunk.code[self.current + 3]].fromTriple()
self.current += 3
printInstruction(instruction)
styledEcho fgGreen, &", creates frame of size ", fgYellow, $(size + 2)
self.current += 4
styledEcho fgGreen, &", creates frame of size ", fgYellow, $(size + 2), fgGreen
self.current += 1
proc functionInstruction(self: Debugger, instruction: OpCode) =
@ -183,6 +183,8 @@ proc jumpInstruction(self: Debugger, instruction: OpCode) =
stdout.styledWrite(fgYellow, $jump)
nl()
self.current += 4
while self.chunk.code[self.current] == NoOp.uint8:
inc(self.current)
for i in countup(orig, self.current + 1):
self.checkFrameStart(i)
@ -207,7 +209,7 @@ proc disassembleInstruction*(self: Debugger) =
self.argumentDoubleInstruction(opcode)
of argumentTripleInstructions:
self.argumentTripleInstruction(opcode)
of callInstructions:
of Call:
self.callInstruction(opcode)
of jumpInstructions:
self.jumpInstruction(opcode)

11
tests/closures.pn

@ -3,14 +3,15 @@ import std;
fn makeClosure(n: int): fn: int {
let n = n; # Workaround
fn inner: int {
return n;
}
return inner;
}
var closure = makeClosure(1)();
print(closure); # 1
print(makeClosure(2)()); # 2
var closed = makeClosure(1)();
print(closed); # 1
print(makeClosure(2)()); # 2
var closure = makeClosure(3);
print(closure()); # 3
print(closure()); # 3

Loading…
Cancel
Save