Browse Source

Broken work with closures and chained calls

master
Mattia Giambirtone 1 month ago
parent
commit
c230142378
  1. 162
      src/backend/vm.nim
  2. 1342
      src/frontend/compiler.nim
  3. 8
      src/frontend/meta/ast.nim
  4. 25
      src/frontend/meta/bytecode.nim
  5. 27
      src/frontend/parser.nim
  6. 36
      src/util/debugger.nim
  7. 18
      tests/closures.pn

162
src/backend/vm.nim

@ -55,7 +55,7 @@ type
cache: array[6, uint64] # Singletons cache
frames: seq[uint64] # Stores the bottom of stack frames
closures: seq[uint64] # Stores closure offsets
closedOver: seq[uint64] # Stores variables that do not have stack semantics
envs: seq[uint64] # Stores variables that do not have stack semantics
results: seq[uint64] # Stores function's results (return values)
gc: PeonGC
ObjectKind* = enum
@ -213,7 +213,7 @@ proc markRoots(self: PeonGC): seq[ptr HeapObject] =
for obj in self.vm.operands:
if obj in self.pointers:
live.incl(obj)
for obj in self.vm.closedOver:
for obj in self.vm.envs:
if obj in self.pointers:
live.incl(obj)
# We preallocate the space on the seq
@ -342,7 +342,7 @@ proc newPeonVM*: PeonVM =
result.calls = @[]
result.operands = @[]
result.results = @[]
result.closedOver = @[]
result.envs = @[]
result.gc.vm = result
@ -383,66 +383,90 @@ proc `!>=`[T](a, b: T): auto {.inline, used.} =
# that go through the getc/setc wrappers is frame-relative,
# meaning that the index is added to the current stack frame's
# bottom to obtain an absolute stack index
proc push(self: PeonVM, obj: uint64) =
proc push(self: PeonVM, obj: uint64) =
## Pushes a value object onto the
## operand stack
self.operands.add(obj)
proc pop(self: PeonVM): uint64 =
## Pops a value off the
## operand stack and
## returns it
proc pop(self: PeonVM): uint64 =
## Pops a value off the operand
## stack and returns it
return self.operands.pop()
proc peekb(self: PeonVM, distance: BackwardsIndex = ^1): uint64 =
## Returns the value at the
## given (backwards) distance from the top of
## the operand stack without consuming it
proc peekb(self: PeonVM, distance: BackwardsIndex = ^1): uint64 =
## Returns the value at the given (backwards)
## distance from the top of the operand stack
## without consuming it
return self.operands[distance]
proc peek(self: PeonVM, distance: int = 0): uint64 =
## Returns the value at the
## given distance from the top of
## the operand stack without consuming it
proc peek(self: PeonVM, distance: int = 0): uint64 =
## Returns the value at the given
## distance from the top of the
## operand stack without consuming it
if distance < 0:
return self.peekb(^(-int(distance)))
return self.operands[self.operands.high() + distance]
proc pushc(self: PeonVM, val: uint64) =
## Pushes a value to the
proc pushc(self: PeonVM, val: uint64) =
## Pushes a value onto the
## call stack
self.calls.add(val)
proc popc(self: PeonVM): uint64 =
proc popc(self: PeonVM): uint64 =
## Pops a value off the call
## stack and returns it
return self.calls.pop()
proc peekc(self: PeonVM, distance: int = 0): uint64 {.used.} =
## Returns the value at the
## given distance from the top of
## the call stack without consuming it
## Returns the value at the given
## distance from the top of the
## call stack without consuming it
return self.calls[self.calls.high() + distance]
proc getc(self: PeonVM, idx: int): uint64 =
## Accessor method that abstracts
## indexing our call stack through stack
## frames
## Getter method that abstracts
## indexing our call stack through
## stack frames
return self.calls[idx.uint64 + self.frames[^1]]
proc setc(self: PeonVM, idx: uint, val: uint64) =
proc setc(self: PeonVM, idx: int, val: uint64) =
## Setter method that abstracts
## indexing our call stack through
## stack frames
self.calls[idx.uint + self.frames[^1]] = val
proc getClosure(self: PeonVM, idx: int): uint64 =
## Getter method that abstracts
## indexing closure environments
return self.envs[idx.uint + self.closures[^1]]
proc setClosure(self: PeonVM, idx: int, val: uint64) =
## Setter method that abstracts
## indexing our call stack through stack
## frames
self.calls[idx + self.frames[^1]] = val
## indexing closure environments
if idx == self.envs.len():
self.envs.add(val)
else:
self.envs[idx.uint + self.closures[^1]] = val
proc popClosure(self: PeonVM, idx: int): uint64 =
## Pop method that abstracts
## popping values off closure
## environments
var idx = idx.uint + self.closures[^1]
result = self.envs[idx]
self.envs.delete(idx)
# Byte-level primitives to read and decode
# bytecode
@ -613,22 +637,29 @@ when debugVM: # So nim shuts up
styledEcho fgMagenta, "]"
if self.frames.len() !> 0:
stdout.styledWrite(fgCyan, "Current Frame: ", fgMagenta, "[")
for i, e in self.calls[self.frames[^1]..self.calls.high()]:
for i, e in self.calls[self.frames[^1]..^1]:
stdout.styledWrite(fgYellow, $e)
if i < self.calls.high():
if i < (self.calls.high() - self.frames[^1].int):
stdout.styledWrite(fgYellow, ", ")
styledEcho fgMagenta, "]"
styledEcho fgMagenta, "]", fgCyan
stdout.styledWrite(fgRed, "Live stack frames: ", fgMagenta, "[")
for i, e in self.frames:
stdout.styledWrite(fgYellow, $e)
if i < self.frames.high():
stdout.styledWrite(fgYellow, ", ")
styledEcho fgMagenta, "]"
if self.closedOver.len() !> 0:
stdout.styledWrite(fgGreen, "Closure Array: ", fgMagenta, "[")
for i, e in self.closedOver:
if self.envs.len() !> 0:
stdout.styledWrite(fgGreen, "Environments: ", fgMagenta, "[")
for i, e in self.envs:
stdout.styledWrite(fgYellow, $e)
if i < self.closedOver.high():
if i < self.envs.high():
stdout.styledWrite(fgYellow, ", ")
styledEcho fgMagenta, "]"
if self.closures.len() !> 0:
stdout.styledWrite(fgGreen, "Environment offsets: ", fgMagenta, "[")
for i, e in self.closures:
stdout.styledWrite(fgYellow, $e)
if i < self.closures.high():
stdout.styledWrite(fgYellow, ", ")
styledEcho fgMagenta, "]"
if self.results.len() !> 0:
@ -695,13 +726,6 @@ proc dispatch*(self: PeonVM) =
self.push(cast[uint64](self.constReadFloat32(int(self.readLong()))))
of LoadFloat64:
self.push(cast[uint64](self.constReadFloat64(int(self.readLong()))))
of LoadFunction:
# Loads a function address onto the operand stack
self.push(uint64(self.readLong()))
of LoadReturnAddress:
# Loads a 32-bit unsigned integer onto the operand stack.
# Used to load function return addresses
self.push(uint64(self.readUInt()))
of Call:
# Calls a peon function. The calling convention here
# is pretty simple: the first value in the frame is
@ -722,7 +746,32 @@ proc dispatch*(self: PeonVM) =
self.results.add(self.getNil())
# Creates a new call frame
self.frames.add(uint64(self.calls.len() - 2))
self.closures.add(self.closedOver.len().uint64)
# Loads the arguments onto the stack
for _ in 0..<argc:
self.pushc(self.pop())
# Pops the function and return address
# off the operand stack since they're
# not needed there anymore
discard self.pop()
discard self.pop()
of CallClosure:
# Calls a peon closure. The code here is
# mostly identical to the one for Call,
# but we also create a new environment
# containing the function's closed-over variables
let argc = self.readLong().int
let offset = self.readLong().uint64
let retAddr = self.peek(-argc - 1) # Return address
let jmpAddr = self.peek(-argc - 2) # Function address
self.ip = jmpAddr
self.pushc(jmpAddr)
self.pushc(retAddr)
# Creates a new result slot for the
# function's return value
self.results.add(self.getNil())
# Creates a new call frame
self.frames.add(uint64(self.calls.len() - 2))
self.closures.add(offset)
# Loads the arguments onto the stack
for _ in 0..<argc:
self.pushc(self.pop())
@ -750,7 +799,6 @@ proc dispatch*(self: PeonVM) =
discard self.results.pop()
# Discard the topmost stack frame
discard self.frames.pop()
discard self.closures.pop()
if self.frames.len() == 0:
# End of the program!
return
@ -770,28 +818,20 @@ proc dispatch*(self: PeonVM) =
when debugVM:
assert idx.int - self.calls.high() <= 1, "StoreVar index is bigger than the length of the call stack"
if idx + self.frames[^1] <= self.calls.high().uint:
self.setc(idx, self.pop())
self.setc(idx.int, self.pop())
else:
self.pushc(self.pop())
of StoreClosure:
# Stores/updates the value of a closed-over
# variable
let idx = self.readLong().int
if idx !> self.closedOver.high():
# Note: we *peek* the stack, but we
# don't pop!
self.closedOver.add(self.peek())
else:
self.closedOver[idx] = self.peek()
of LoadClosure:
# Loads a closed-over variable onto the
# stack
self.push(self.closedOver[self.readLong() + self.closures[^1] - 1])
self.push(self.getClosure(self.readLong().int))
of PopClosure:
self.closedOver.delete(self.readLong())
of LiftArgument:
# Lifts a function argument onto the stack
self.closedOver.add(self.getc(self.readLong().int))
discard self.popClosure(self.readLong().int)
of StoreClosure:
# Stores/updates the value of a closed-over
# variable
let item = self.getc(self.readLong().int)
self.setClosure(self.readLong().int, item)
of LoadVar:
# Pushes a variable onto the operand
# stack

1342
src/frontend/compiler.nim

File diff suppressed because it is too large

8
src/frontend/meta/ast.nim

@ -182,7 +182,6 @@ type
isPure*: bool
returnType*: Expression
hasExplicitReturn*: bool
freeVars*: seq[IdentExpr]
depth*: int
SliceExpr* = ref object of Expression
@ -264,7 +263,6 @@ type
isPure*: bool
returnType*: Expression
hasExplicitReturn*: bool
freeVars*: seq[IdentExpr]
depth*: int
TypeDecl* = ref object of Declaration
@ -428,7 +426,6 @@ proc newLambdaExpr*(arguments: seq[tuple[name: IdentExpr, valueType: Expression]
result.isPure = false
result.pragmas = pragmas
result.generics = generics
result.freeVars = freeVars
result.depth = depth
@ -634,7 +631,6 @@ proc newFunDecl*(name: IdentExpr, arguments: seq[tuple[name: IdentExpr, valueTyp
result.returnType = returnType
result.isPure = false
result.generics = generics
result.freeVars = freeVars
result.depth = depth
@ -735,13 +731,13 @@ proc `$`*(self: ASTNode): string =
result &= &"Var(name={self.name}, value={self.value}, const={self.isConst}, private={self.isPrivate}, type={self.valueType}, pragmas={self.pragmas})"
of funDecl:
var self = FunDecl(self)
result &= &"""FunDecl(name={self.name}, body={self.body}, type={self.returnType}, arguments=[{self.arguments.join(", ")}], defaults=[{self.defaults.join(", ")}], generics=[{self.generics.join(", ")}], async={self.isAsync}, generator={self.isGenerator}, private={self.isPrivate}, pragmas={self.pragmas}, vars=[{self.freeVars.join(", ")}])"""
result &= &"""FunDecl(name={self.name}, body={self.body}, type={self.returnType}, arguments=[{self.arguments.join(", ")}], defaults=[{self.defaults.join(", ")}], generics=[{self.generics.join(", ")}], async={self.isAsync}, generator={self.isGenerator}, private={self.isPrivate}, pragmas={self.pragmas})"""
of typeDecl:
var self = TypeDecl(self)
result &= &"""TypeDecl(name={self.name}, fields={self.fields}, defaults={self.defaults}, private={self.isPrivate}, pragmas={self.pragmas}, generics={self.generics}, pragmas={self.pragmas}, type={self.valueType})"""
of lambdaExpr:
var self = LambdaExpr(self)
result &= &"""Lambda(body={self.body}, type={self.returnType}, arguments=[{self.arguments.join(", ")}], defaults=[{self.defaults.join(", ")}], generator={self.isGenerator}, async={self.isAsync}, pragmas={self.pragmas}, vars=[{self.freeVars.join(", ")}])"""
result &= &"""Lambda(body={self.body}, type={self.returnType}, arguments=[{self.arguments.join(", ")}], defaults=[{self.defaults.join(", ")}], generator={self.isGenerator}, async={self.isAsync}, pragmas={self.pragmas})"""
of deferStmt:
var self = DeferStmt(self)
result &= &"Defer({self.expression})"

25
src/frontend/meta/bytecode.nim

@ -29,13 +29,10 @@ type
## Length Encoding. Instructions are encoded in groups whose structure
## follows the following schema:
## - The first integer represents the line number
## - The second integer represents the count of whatever comes after it
## (let's call it c)
## - After c, a sequence of c integers follows
##
## A visual representation may be easier to understand: [1, 2, 3, 4]
## This is to be interpreted as "there are 2 instructions at line 1 whose values
## are 3 and 4"
## - The second integer represents the number of
## instructions on that line
## For example, if lines equals [1, 5], it means that there are 5 instructions
## at line 1, meaning that all instructions in code[0..4] belong to the same line.
## This is more efficient than using the naive approach, which would encode
## the same line number multiple times and waste considerable amounts of space.
## cfi represents Call Frame Information and encodes the following information:
@ -81,8 +78,6 @@ type
LoadFloat64,
LoadFloat32,
LoadString,
LoadFunction,
LoadReturnAddress,
## Singleton opcodes (each of them pushes a constant singleton on the stack)
LoadNil,
LoadTrue,
@ -164,6 +159,7 @@ type
JumpIfFalseOrPop, # Jumps to an absolute index in the bytecode if x is false and pops otherwise (used for logical and)
## Functions
Call, # Calls a function and initiates a new stack frame
CallClosure, # Calls a closure
Return, # Terminates the current function
SetResult, # Sets the result of the current function
## Exception handling
@ -264,8 +260,6 @@ const stackDoubleInstructions* = {}
# Argument double argument instructions take hardcoded arguments as 16 bit integers
const argumentDoubleInstructions* = {PopN, }
# Argument double argument instructions take hardcoded arguments as 24 bit integers
const argumentTripleInstructions* = {StoreClosure}
# Jump instructions jump at relative or absolute bytecode offsets
const jumpInstructions* = {Jump, JumpIfFalse, JumpIfFalsePop,
@ -329,6 +323,15 @@ proc getLine*(self: Chunk, idx: int): int =
raise newException(IndexDefect, "index out of range")
proc getIdx*(self: Chunk, line: int): int =
## Gets the index into self.lines
## where the line counter for the given
## line is located
for i, v in self.lines:
if (i and 1) != 0 and v == line:
return i
proc writeConstant*(self: Chunk, data: openarray[uint8]): array[3, uint8] =
## Writes a series of bytes to the chunk's constant
## table and returns the index of the first byte as

27
src/frontend/parser.nim

@ -86,6 +86,8 @@ type
currentFunction: Declaration
# Stores the current scope depth (0 = global, > 0 local)
scopeDepth: int
# TODO
scopes: seq[Declaration]
operators: OperatorTable
# The AST node
tree: seq[Declaration]
@ -170,7 +172,7 @@ proc getLines*(self: Parser): seq[tuple[start, stop: int]] = self.lines
proc getSource*(self: Parser): string = self.source
proc getRelPos*(self: Parser, line: int): tuple[start, stop: int] = self.lines[line - 1]
template endOfFile: Token = Token(kind: EndOfFile, lexeme: "", line: -1)
template endOfLine(msg: string) = self.expect(Semicolon, msg)
template endOfLine(msg: string, tok: Token = nil) = self.expect(Semicolon, msg, tok)
@ -265,19 +267,19 @@ proc match[T: TokenType or string](self: Parser, kind: openarray[T]): bool =
result = false
proc expect[T: TokenType or string](self: Parser, kind: T, message: string = "") =
proc expect[T: TokenType or string](self: Parser, kind: T, message: string = "", token: Token = nil) =
## Behaves like self.match(), except that
## when a token doesn't match, an error
## is raised. If no error message is
## given, a default one is used
if not self.match(kind):
if message.len() == 0:
self.error(&"expecting token of kind {kind}, found {self.peek().kind} instead")
self.error(&"expecting token of kind {kind}, found {self.peek().kind} instead", token)
else:
self.error(message)
proc expect[T: TokenType or string](self: Parser, kind: openarray[T], message: string = "") {.used.} =
proc expect[T: TokenType or string](self: Parser, kind: openarray[T], message: string = "", token: Token = nil) {.used.} =
## Behaves like self.expect(), except that
## an error is raised only if none of the
## given token kinds matches
@ -285,7 +287,7 @@ proc expect[T: TokenType or string](self: Parser, kind: openarray[T], message: s
if self.match(kind):
return
if message.len() == 0:
self.error(&"""expecting any of the following tokens: {kind.join(", ")}, but got {self.peek().kind} instead""")
self.error(&"""expecting any of the following tokens: {kind.join(", ")}, but got {self.peek().kind} instead""", token)
# Forward declarations
@ -320,16 +322,6 @@ proc primary(self: Parser): Expression =
result = newIntExpr(self.step())
of Identifier:
result = newIdentExpr(self.step(), self.scopeDepth)
if not self.currentFunction.isNil() and self.scopeDepth > 0:
case self.currentFunction.kind:
of NodeKind.funDecl:
if FunDecl(self.currentFunction).depth != self.scopeDepth:
FunDecl(self.currentFunction).freeVars.add(IdentExpr(result))
of NodeKind.lambdaExpr:
if LambdaExpr(self.currentFunction).depth != self.scopeDepth:
LambdaExpr(self.currentFunction).freeVars.add(IdentExpr(result))
else:
discard # Unreachable
of LeftParen:
let tok = self.step()
result = newGroupingExpr(self.expression(), tok)
@ -1036,6 +1028,7 @@ proc funDecl(self: Parser, isAsync: bool = false, isGenerator: bool = false,
elif isLambda:
self.currentFunction = newLambdaExpr(arguments, defaults, newBlockStmt(@[], Token()), isGenerator=isGenerator, isAsync=isAsync, token=tok,
returnType=nil, depth=self.scopeDepth)
self.scopes.add(FunDecl(self.currentFunction))
if self.match(":"):
# Function has explicit return type
if self.match([Function, Coroutine, Generator]):
@ -1101,11 +1094,12 @@ proc expression(self: Parser): Expression =
result = self.parseArrow() # Highest-level expression
proc expressionStatement(self: Parser): Statement =
## Parses expression statements, which
## are expressions followed by a semicolon
var expression = self.expression()
endOfLine("missing expression terminator")
endOfLine("missing expression terminator", expression.token)
result = Statement(newExprStmt(expression, expression.token))
@ -1254,6 +1248,7 @@ proc findOperators(self: Parser, tokens: seq[Token]) =
self.error("invalid state: found malformed tokenizer input while looking for operators (missing EOF)", token)
proc parse*(self: Parser, tokens: seq[Token], file: string, lines: seq[tuple[start, stop: int]], source: string, persist: bool = false): seq[Declaration] =
## Parses a sequence of tokens into a sequence of AST nodes
self.tokens = tokens

36
src/util/debugger.nim

@ -124,7 +124,7 @@ proc argumentDoubleInstruction(self: Debugger, instruction: OpCode) =
self.current += 3
proc argumentTripleInstruction(self: Debugger, instruction: OpCode) =
proc argumentTripleInstruction(self: Debugger, instruction: OpCode) {.used.} =
## Debugs instructions that operate on a hardcoded value on the stack using a 24-bit operand
var slot = [self.chunk.code[self.current + 1], self.chunk.code[self.current + 2], self.chunk.code[self.current + 3]].fromTriple()
printInstruction(instruction)
@ -132,6 +132,15 @@ proc argumentTripleInstruction(self: Debugger, instruction: OpCode) =
self.current += 4
proc storeClosureInstruction(self: Debugger, instruction: OpCode) =
## Debugs instructions that operate on a hardcoded value on the stack using a 24-bit operand
var idx = [self.chunk.code[self.current + 1], self.chunk.code[self.current + 2], self.chunk.code[self.current + 3]].fromTriple()
var idx2 = [self.chunk.code[self.current + 4], self.chunk.code[self.current + 5], self.chunk.code[self.current + 6]].fromTriple()
printInstruction(instruction)
stdout.styledWriteLine(fgGreen, ", stores element at position ", fgYellow, $idx, fgGreen, " into position ", fgYellow, $idx2)
self.current += 7
proc callInstruction(self: Debugger, instruction: OpCode) =
## Debugs function calls
var size = [self.chunk.code[self.current + 1], self.chunk.code[self.current + 2], self.chunk.code[self.current + 3]].fromTriple()
@ -141,23 +150,6 @@ proc callInstruction(self: Debugger, instruction: OpCode) =
self.current += 1
proc functionInstruction(self: Debugger, instruction: OpCode) =
## Debugs function calls
var address = [self.chunk.code[self.current + 1], self.chunk.code[self.current + 2], self.chunk.code[self.current + 3]].fromTriple()
printInstruction(instruction)
styledEcho fgGreen, &", loads function at address ", fgYellow, $address
self.current += 4
proc loadAddressInstruction(self: Debugger, instruction: OpCode) =
## Debugs LoadReturnAddress instructions
var address = [self.chunk.code[self.current + 1], self.chunk.code[self.current + 2], self.chunk.code[self.current + 3],
self.chunk.code[self.current + 4]].fromQuad()
printInstruction(instruction)
styledEcho fgGreen, &", loads address ", fgYellow, $address
self.current += 5
proc constantInstruction(self: Debugger, instruction: OpCode) =
## Debugs instructions that operate on the constant table
var size: uint
@ -207,16 +199,12 @@ proc disassembleInstruction*(self: Debugger) =
self.stackTripleInstruction(opcode)
of argumentDoubleInstructions:
self.argumentDoubleInstruction(opcode)
of argumentTripleInstructions:
self.argumentTripleInstruction(opcode)
of StoreClosure:
self.storeClosureInstruction(opcode)
of Call:
self.callInstruction(opcode)
of jumpInstructions:
self.jumpInstruction(opcode)
of LoadFunction:
self.functionInstruction(opcode)
of LoadReturnAddress:
self.loadAddressInstruction(opcode)
else:
echo &"DEBUG - Unknown opcode {opcode} at index {self.current}"
self.current += 1

18
tests/closures.pn

@ -2,16 +2,16 @@
import std;
fn makeClosure(n: int): fn: int {
fn inner: int {
return n;
fn makeClosure(n: int): fn: fn: int {
fn inner: fn: int {
fn deep: int {
return n;
}
return deep;
}
return inner;
}
var closed = makeClosure(1)();
print(closed); # 1
print(makeClosure(2)()); # 2
var closure = makeClosure(3);
print(closure()); # 3
print(closure()); # 3
print(makeClosure(1)()()); # 1

Loading…
Cancel
Save