nocturn9x
/
peon
2
2
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Added Makefile, prettyfied code, initial work on pragmas

This commit is contained in:
Mattia Giambirtone 2022-05-18 13:32:32 +02:00
parent e8cbec94bd
commit cb21af4aa3
14 changed files with 448 additions and 317 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
Makefile Normal file
View File

@ -0,0 +1,5 @@
run:
nim --hints:off --warnings:off r src/test.nim
pretty:
nimpretty src/*.nim src/backend/*.nim src/frontend/*.nim src/frontend/meta/*.nim src/memory/*.nim src/util/*.nim

5
src/backend/types.nim
View File

@ -13,7 +13,7 @@
# limitations under the License. # limitations under the License.
type type
ObjectKind* = enum ObjectKind* = enum
## Enumeration of Peon ## Enumeration of Peon
## types ## types
@ -51,5 +51,4 @@ type
of CustomType: of CustomType:
fields*: seq[PeonObject] fields*: seq[PeonObject]
else: else:
discard # TODO discard # TODO

16
src/backend/vm.nim
View File

@ -18,14 +18,14 @@ import ../config
import ../frontend/meta/bytecode import ../frontend/meta/bytecode
type type
PeonVM* = ref object PeonVM* = ref object
## The Peon Virtual Machine ## The Peon Virtual Machine
stack: seq[PeonObject] stack: seq[PeonObject]
ip: int # Instruction pointer ip: int # Instruction pointer
sp: int # Stack pointer sp: int # Stack pointer
cache: array[6, PeonObject] # Singletons cache cache: array[6, PeonObject] # Singletons cache
chunk: Chunk # Piece of bytecode to execute chunk: Chunk # Piece of bytecode to execute
proc initCache*(self: PeonVM) = proc initCache*(self: PeonVM) =
@ -126,7 +126,8 @@ proc readInt64(self: PeonVM, idx: int): PeonObject =
## chunk's constant table and ## chunk's constant table and
## returns a Peon object. Assumes ## returns a Peon object. Assumes
## the constant is an Int64 ## the constant is an Int64
var arr = [self.chunk.byteConsts[idx], self.chunk.byteConsts[idx + 1], self.chunk.byteConsts[idx + 2], self.chunk.byteConsts[idx + 3]] var arr = [self.chunk.byteConsts[idx], self.chunk.byteConsts[idx + 1],
self.chunk.byteConsts[idx + 2], self.chunk.byteConsts[idx + 3]]
result = PeonObject(kind: Int64) result = PeonObject(kind: Int64)
copyMem(result.long.addr, arr.addr, sizeof(arr)) copyMem(result.long.addr, arr.addr, sizeof(arr))
@ -136,7 +137,8 @@ proc readUInt64(self: PeonVM, idx: int): PeonObject =
## chunk's constant table and ## chunk's constant table and
## returns a Peon object. Assumes ## returns a Peon object. Assumes
## the constant is an UInt64 ## the constant is an UInt64
var arr = [self.chunk.byteConsts[idx], self.chunk.byteConsts[idx + 1], self.chunk.byteConsts[idx + 2], self.chunk.byteConsts[idx + 3]] var arr = [self.chunk.byteConsts[idx], self.chunk.byteConsts[idx + 1],
self.chunk.byteConsts[idx + 2], self.chunk.byteConsts[idx + 3]]
result = PeonObject(kind: UInt64) result = PeonObject(kind: UInt64)
copyMem(result.uLong.addr, arr.addr, sizeof(arr)) copyMem(result.uLong.addr, arr.addr, sizeof(arr))

16
src/config.nim
View File

@ -15,13 +15,13 @@
import strformat import strformat
const BYTECODE_MARKER* = "PEON_BYTECODE" const BYTECODE_MARKER* = "PEON_BYTECODE"
const MAP_LOAD_FACTOR* = 0.75 # Load factor for builtin hashmaps const MAP_LOAD_FACTOR* = 0.75 # Load factor for builtin hashmaps
when MAP_LOAD_FACTOR >= 1.0: when MAP_LOAD_FACTOR >= 1.0:
{.fatal: "Hashmap load factor must be < 1".} {.fatal: "Hashmap load factor must be < 1".}
const HEAP_GROW_FACTOR* = 2 # How much extra memory to allocate for dynamic arrays and garbage collection when resizing const HEAP_GROW_FACTOR* = 2 # How much extra memory to allocate for dynamic arrays and garbage collection when resizing
when HEAP_GROW_FACTOR <= 1: when HEAP_GROW_FACTOR <= 1:
{.fatal: "Heap growth factor must be > 1".} {.fatal: "Heap growth factor must be > 1".}
const MAX_STACK_FRAMES* = 800 # The maximum number of stack frames at any one time. Acts as a recursion limiter (1 frame = 1 call) const MAX_STACK_FRAMES* = 800 # The maximum number of stack frames at any one time. Acts as a recursion limiter (1 frame = 1 call)
when MAX_STACK_FRAMES <= 0: when MAX_STACK_FRAMES <= 0:
{.fatal: "The frame limit must be > 0".} {.fatal: "The frame limit must be > 0".}
const PEON_VERSION* = (major: 0, minor: 4, patch: 0) const PEON_VERSION* = (major: 0, minor: 4, patch: 0)
@ -32,11 +32,11 @@ when len(PEON_COMMIT_HASH) != 40:
const PEON_BRANCH* = "master" const PEON_BRANCH* = "master"
when len(PEON_BRANCH) >= 255: when len(PEON_BRANCH) >= 255:
{.fatal: "The git branch name's length must be less than or equal to 255 characters".} {.fatal: "The git branch name's length must be less than or equal to 255 characters".}
const DEBUG_TRACE_VM* = false # Traces VM execution const DEBUG_TRACE_VM* = false # Traces VM execution
const SKIP_STDLIB_INIT* = false # Skips stdlib initialization (can be imported manually) const SKIP_STDLIB_INIT* = false # Skips stdlib initialization (can be imported manually)
const DEBUG_TRACE_GC* = false # Traces the garbage collector (TODO) const DEBUG_TRACE_GC* = false # Traces the garbage collector (TODO)
const DEBUG_TRACE_ALLOCATION* = false # Traces memory allocation/deallocation const DEBUG_TRACE_ALLOCATION* = false # Traces memory allocation/deallocation
const DEBUG_TRACE_COMPILER* = false # Traces the compiler const DEBUG_TRACE_COMPILER* = false # Traces the compiler
const INITIAL_STACK_SIZE* = 0 const INITIAL_STACK_SIZE* = 0
const PEON_VERSION_STRING* = &"Peon {PEON_VERSION.major}.{PEON_VERSION.minor}.{PEON_VERSION.patch} {PEON_RELEASE} ({PEON_BRANCH}, {CompileDate}, {CompileTime}, {PEON_COMMIT_HASH[0..8]}) [Nim {NimVersion}] on {hostOS} ({hostCPU})" const PEON_VERSION_STRING* = &"Peon {PEON_VERSION.major}.{PEON_VERSION.minor}.{PEON_VERSION.patch} {PEON_RELEASE} ({PEON_BRANCH}, {CompileDate}, {CompileTime}, {PEON_COMMIT_HASH[0..8]}) [Nim {NimVersion}] on {hostOS} ({hostCPU})"
const HELP_MESSAGE* = """The peon programming language, Copyright (C) 2022 Mattia Giambirtone & All Contributors const HELP_MESSAGE* = """The peon programming language, Copyright (C) 2022 Mattia Giambirtone & All Contributors

124
src/frontend/compiler.nim
View File

@ -96,7 +96,7 @@ type
# The bytecode chunk where we write code to # The bytecode chunk where we write code to
chunk: Chunk chunk: Chunk
# The output of our parser (AST) # The output of our parser (AST)
ast: seq[ASTNode] ast: seq[Declaration]
# The current AST node we're looking at # The current AST node we're looking at
current: int current: int
# The current file being compiled (used only for # The current file being compiled (used only for
@ -185,7 +185,8 @@ proc done(self: Compiler): bool =
result = self.current > self.ast.high() result = self.current > self.ast.high()
proc error(self: Compiler, message: string) {.raises: [CompileError, ValueError].} = proc error(self: Compiler, message: string) {.raises: [CompileError,
ValueError].} =
## Raises a formatted CompileError exception ## Raises a formatted CompileError exception
var tok = self.getCurrentNode().token var tok = self.getCurrentNode().token
raise newException(CompileError, &"A fatal error occurred while compiling '{self.file}', module '{self.currentModule}' line {tok.line} at '{tok.lexeme}' -> {message}") raise newException(CompileError, &"A fatal error occurred while compiling '{self.file}', module '{self.currentModule}' line {tok.line} at '{tok.lexeme}' -> {message}")
@ -243,7 +244,7 @@ proc emitConstant(self: Compiler, obj: Expression, kind: Type) =
of Int64: of Int64:
self.emitByte(LoadInt64) self.emitByte(LoadInt64)
else: else:
discard # TODO discard # TODO
self.emitBytes(self.makeConstant(obj, kind)) self.emitBytes(self.makeConstant(obj, kind))
@ -261,7 +262,7 @@ proc emitJump(self: Compiler, opcode: OpCode): int =
proc patchJump(self: Compiler, offset: int) = proc patchJump(self: Compiler, offset: int) =
## Patches a previously emitted relative ## Patches a previously emitted relative
## jump using emitJump. Since emitJump assumes ## jump using emitJump. Since emitJump assumes
## a long jump, this also shrinks the jump ## a long jump, this also shrinks the jump
## offset and changes the bytecode instruction if possible ## offset and changes the bytecode instruction if possible
@ -284,7 +285,7 @@ proc patchJump(self: Compiler, offset: int) =
self.chunk.code[offset] = JumpIfFalseOrPop.uint8() self.chunk.code[offset] = JumpIfFalseOrPop.uint8()
else: else:
discard discard
self.chunk.code.delete(offset + 1) # Discards the first 8 bits of the jump offset (which are empty) self.chunk.code.delete(offset + 1) # Discards the first 8 bits of the jump offset (which are empty)
let offsetArray = (jump - 1).toDouble() # -1 since we got rid of 1 byte! let offsetArray = (jump - 1).toDouble() # -1 since we got rid of 1 byte!
self.chunk.code[offset + 1] = offsetArray[0] self.chunk.code[offset + 1] = offsetArray[0]
self.chunk.code[offset + 2] = offsetArray[1] self.chunk.code[offset + 2] = offsetArray[1]
@ -319,17 +320,18 @@ proc resolve(self: Compiler, name: IdentExpr,
for obj in reversed(self.names): for obj in reversed(self.names):
if obj.name.token.lexeme == name.token.lexeme: if obj.name.token.lexeme == name.token.lexeme:
if obj.isPrivate and obj.owner != self.currentModule: if obj.isPrivate and obj.owner != self.currentModule:
continue # There may be a name in the current module that continue # There may be a name in the current module that
# matches, so we skip this # matches, so we skip this
return obj return obj
return nil return nil
proc getStackPos(self: Compiler, name: IdentExpr, depth: int = self.scopeDepth): tuple[closedOver: bool, pos: int] = proc getStackPos(self: Compiler, name: IdentExpr,
depth: int = self.scopeDepth): tuple[closedOver: bool, pos: int] =
## Iterates the internal list of declared names backwards and ## Iterates the internal list of declared names backwards and
## returns a tuple (closedOver, pos) that tells the caller whether the ## returns a tuple (closedOver, pos) that tells the caller whether the
## the name is to be emitted as a closure as well as its predicted ## the name is to be emitted as a closure as well as its predicted
## stack/closure array position. Returns (false, -1) if the variable's ## stack/closure array position. Returns (false, -1) if the variable's
## location can not be determined at compile time (this is an error!). ## location can not be determined at compile time (this is an error!).
## Note that private names declared in other modules will not be resolved! ## Note that private names declared in other modules will not be resolved!
var i: int = self.names.high() var i: int = self.names.high()
@ -348,14 +350,15 @@ proc getStackPos(self: Compiler, name: IdentExpr, depth: int = self.scopeDepth):
return (false, -1) return (false, -1)
proc detectClosureVariable(self: Compiler, name: IdentExpr, depth: int = self.scopeDepth) = proc detectClosureVariable(self: Compiler, name: IdentExpr,
depth: int = self.scopeDepth) =
## Detects if the given name is used in a local scope deeper ## Detects if the given name is used in a local scope deeper
## than the given one and modifies the code emitted for it ## than the given one and modifies the code emitted for it
## to store it as a closure variable if it is. Does nothing if the name ## to store it as a closure variable if it is. Does nothing if the name
## hasn't been declared yet or is unreachable (for example if it's ## hasn't been declared yet or is unreachable (for example if it's
## declared as private in another module). This function must be called ## declared as private in another module). This function must be called
## each time a name is referenced in order for closed-over variables ## each time a name is referenced in order for closed-over variables
## to be emitted properly, otherwise the runtime may behave ## to be emitted properly, otherwise the runtime may behave
## unpredictably or crash ## unpredictably or crash
let entry = self.resolve(name) let entry = self.resolve(name)
if entry == nil: if entry == nil:
@ -392,14 +395,15 @@ proc compareTypes(self: Compiler, a, b: Type): bool =
Char, Byte, String, Nil, Nan, Bool, Inf: Char, Byte, String, Nil, Nan, Bool, Inf:
return true return true
of Function: of Function:
let let
a = FunDecl(a.node) a = FunDecl(a.node)
b = FunDecl(b.node) b = FunDecl(b.node)
if a.name.token.lexeme != b.name.token.lexeme: if a.name.token.lexeme != b.name.token.lexeme:
return false return false
elif a.arguments.len() != b.arguments.len(): elif a.arguments.len() != b.arguments.len():
return false return false
elif not self.compareTypes(self.inferType(a.returnType), self.inferType(b.returnType)): elif not self.compareTypes(self.inferType(a.returnType),
self.inferType(b.returnType)):
return false return false
for (argA, argB) in zip(a.arguments, b.arguments): for (argA, argB) in zip(a.arguments, b.arguments):
if argA.mutable != argB.mutable: if argA.mutable != argB.mutable:
@ -408,7 +412,8 @@ proc compareTypes(self: Compiler, a, b: Type): bool =
return false return false
elif argA.isPtr != argB.isPtr: elif argA.isPtr != argB.isPtr:
return false return false
elif not self.compareTypes(self.inferType(argA.valueType), self.inferType(argB.valueType)): elif not self.compareTypes(self.inferType(argA.valueType),
self.inferType(argB.valueType)):
return false return false
return true return true
else: else:
@ -416,9 +421,9 @@ proc compareTypes(self: Compiler, a, b: Type): bool =
proc toIntrinsic(name: string): Type = proc toIntrinsic(name: string): Type =
## Converts a string to an intrinsic ## Converts a string to an intrinsic
## type if it is valid and returns nil ## type if it is valid and returns nil
## otherwise ## otherwise
if name in ["int", "int64", "i64"]: if name in ["int", "int64", "i64"]:
return Type(kind: Int64) return Type(kind: Int64)
elif name in ["uint64", "u64"]: elif name in ["uint64", "u64"]:
@ -493,11 +498,11 @@ proc inferType(self: Compiler, node: LiteralExpr): Type =
of infExpr: of infExpr:
return Type(node: node, kind: TypeKind.Inf) return Type(node: node, kind: TypeKind.Inf)
else: else:
discard # TODO discard # TODO
proc toIntrinsic(self: Compiler, typ: Expression): Type = proc toIntrinsic(self: Compiler, typ: Expression): Type =
## Gets an expression's ## Gets an expression's
## intrinsic type, if possible ## intrinsic type, if possible
if typ == nil: if typ == nil:
return nil return nil
@ -535,13 +540,13 @@ proc inferType(self: Compiler, node: Expression): Type =
if not self.compareTypes(a, b): if not self.compareTypes(a, b):
return nil return nil
return a return a
of {intExpr, hexExpr, binExpr, octExpr, of {intExpr, hexExpr, binExpr, octExpr,
strExpr, falseExpr, trueExpr, infExpr, strExpr, falseExpr, trueExpr, infExpr,
nanExpr, floatExpr, nilExpr nanExpr, floatExpr, nilExpr
}: }:
return self.inferType(LiteralExpr(node)) return self.inferType(LiteralExpr(node))
else: else:
discard # Unreachable discard # Unreachable
proc typeToStr(self: Compiler, typ: Type): string = proc typeToStr(self: Compiler, typ: Type): string =
@ -550,7 +555,7 @@ proc typeToStr(self: Compiler, typ: Type): string =
case typ.kind: case typ.kind:
of Int8, UInt8, Int16, UInt16, Int32, of Int8, UInt8, Int16, UInt16, Int32,
UInt32, Int64, UInt64, Float32, Float64, UInt32, Int64, UInt64, Float32, Float64,
Char, Byte, String, Nil, TypeKind.Nan, Bool, Char, Byte, String, Nil, TypeKind.Nan, Bool,
TypeKind.Inf: TypeKind.Inf:
return ($typ.kind).toLowerAscii() return ($typ.kind).toLowerAscii()
of Function: of Function:
@ -571,11 +576,11 @@ proc typeToStr(self: Compiler, typ: Type): string =
result &= ", " result &= ", "
result &= ")" result &= ")"
else: else:
discard # Unreachable discard # Unreachable
result &= &": {self.typeToStr(typ.returnType)}" result &= &": {self.typeToStr(typ.returnType)}"
else: else:
discard discard
proc inferType(self: Compiler, node: Declaration): Type = proc inferType(self: Compiler, node: Declaration): Type =
## Infers the type of a given declaration ## Infers the type of a given declaration
@ -596,7 +601,7 @@ proc inferType(self: Compiler, node: Declaration): Type =
else: else:
return self.inferType(node.value) return self.inferType(node.value)
else: else:
return # Unreachable return # Unreachable
## End of utility functions ## End of utility functions
@ -634,11 +639,11 @@ proc literal(self: Compiler, node: ASTNode) =
discard parseHex(y.literal.lexeme, x) discard parseHex(y.literal.lexeme, x)
except ValueError: except ValueError:
self.error("integer value out of range") self.error("integer value out of range")
let node = newIntExpr(Token(lexeme: $x, line: y.token.line, let node = newIntExpr(Token(lexeme: $x, line: y.token.line,
pos: (start: y.token.pos.start, pos: (start: y.token.pos.start,
stop: y.token.pos.start + len($x)) stop: y.token.pos.start + len($x))
) )
) )
self.emitConstant(node, Type(kind: Int64)) self.emitConstant(node, Type(kind: Int64))
of binExpr: of binExpr:
var x: int var x: int
@ -647,11 +652,11 @@ proc literal(self: Compiler, node: ASTNode) =
discard parseBin(y.literal.lexeme, x) discard parseBin(y.literal.lexeme, x)
except ValueError: except ValueError:
self.error("integer value out of range") self.error("integer value out of range")
let node = newIntExpr(Token(lexeme: $x, line: y.token.line, let node = newIntExpr(Token(lexeme: $x, line: y.token.line,
pos: (start: y.token.pos.start, pos: (start: y.token.pos.start,
stop: y.token.pos.start + len($x)) stop: y.token.pos.start + len($x))
) )
) )
self.emitConstant(node, Type(kind: Int64)) self.emitConstant(node, Type(kind: Int64))
of octExpr: of octExpr:
var x: int var x: int
@ -660,11 +665,11 @@ proc literal(self: Compiler, node: ASTNode) =
discard parseOct(y.literal.lexeme, x) discard parseOct(y.literal.lexeme, x)
except ValueError: except ValueError:
self.error("integer value out of range") self.error("integer value out of range")
let node = newIntExpr(Token(lexeme: $x, line: y.token.line, let node = newIntExpr(Token(lexeme: $x, line: y.token.line,
pos: (start: y.token.pos.start, pos: (start: y.token.pos.start,
stop: y.token.pos.start + len($x)) stop: y.token.pos.start + len($x))
) )
) )
self.emitConstant(node, Type(kind: Int64)) self.emitConstant(node, Type(kind: Int64))
of floatExpr: of floatExpr:
var x: float var x: float
@ -685,7 +690,7 @@ proc literal(self: Compiler, node: ASTNode) =
proc unary(self: Compiler, node: UnaryExpr) = proc unary(self: Compiler, node: UnaryExpr) =
## Compiles unary expressions such as decimal ## Compiles unary expressions such as decimal
## and bitwise negation ## and bitwise negation
self.expression(node.a) # Pushes the operand onto the stack self.expression(node.a) # Pushes the operand onto the stack
# TODO: Find implementation of # TODO: Find implementation of
# the given operator and call it # the given operator and call it
@ -731,12 +736,13 @@ proc declareName(self: Compiler, node: Declaration) =
# If someone ever hits this limit in real-world scenarios, I swear I'll # If someone ever hits this limit in real-world scenarios, I swear I'll
# slap myself 100 times with a sign saying "I'm dumb". Mark my words # slap myself 100 times with a sign saying "I'm dumb". Mark my words
self.error("cannot declare more than 16777216 variables at a time") self.error("cannot declare more than 16777216 variables at a time")
self.names.add(Name(depth: self.scopeDepth, self.names.add(Name(depth: self.scopeDepth,
name: node.name, name: node.name,
isPrivate: node.isPrivate, isPrivate: node.isPrivate,
owner: self.currentModule, owner: self.currentModule,
isConst: node.isConst, isConst: node.isConst,
valueType: Type(kind: self.inferType(node.value).kind, node: node), valueType: Type(kind: self.inferType(
node.value).kind, node: node),
codePos: self.chunk.code.len(), codePos: self.chunk.code.len(),
isLet: node.isLet)) isLet: node.isLet))
self.emitByte(StoreVar) self.emitByte(StoreVar)
@ -754,18 +760,20 @@ proc declareName(self: Compiler, node: Declaration) =
isPrivate: node.isPrivate, isPrivate: node.isPrivate,
isConst: false, isConst: false,
owner: self.currentModule, owner: self.currentModule,
valueType: Type(kind: Function, node: node, returnType: self.inferType(node.returnType)), valueType: Type(kind: Function, node: node,
returnType: self.inferType(
node.returnType)),
codePos: self.chunk.code.len(), codePos: self.chunk.code.len(),
name: node.name, name: node.name,
isLet: false)) isLet: false))
for argument in node.arguments: for argument in node.arguments:
if self.names.high() > 16777215: if self.names.high() > 16777215:
self.error("cannot declare more than 16777216 variables at a time") self.error("cannot declare more than 16777216 variables at a time")
self.names.add(Name(depth: self.scopeDepth + 1, self.names.add(Name(depth: self.scopeDepth + 1,
isPrivate: true, isPrivate: true,
owner: self.currentModule, owner: self.currentModule,
isConst: false, isConst: false,
name: argument.name, name: argument.name,
valueType: nil, valueType: nil,
codePos: self.chunk.code.len(), codePos: self.chunk.code.len(),
isLet: false)) isLet: false))
@ -774,8 +782,8 @@ proc declareName(self: Compiler, node: Declaration) =
self.emitByte(LoadVar) self.emitByte(LoadVar)
self.emitBytes(self.names.high().toTriple()) self.emitBytes(self.names.high().toTriple())
else: else:
discard # Unreachable discard # Unreachable
proc identifier(self: Compiler, node: IdentExpr) = proc identifier(self: Compiler, node: IdentExpr) =
## Compiles access to identifiers ## Compiles access to identifiers
@ -792,7 +800,7 @@ proc identifier(self: Compiler, node: IdentExpr) =
self.detectClosureVariable(s.name) self.detectClosureVariable(s.name)
let t = self.getStackPos(node) let t = self.getStackPos(node)
let index = t.pos let index = t.pos
# We don't check if index is -1 because if it # We don't check if index is -1 because if it
# were, self.resolve() would have returned nil # were, self.resolve() would have returned nil
if not t.closedOver: if not t.closedOver:
# Static name resolution, loads value at index in the stack. Very fast. Much wow. # Static name resolution, loads value at index in the stack. Very fast. Much wow.
@ -802,7 +810,7 @@ proc identifier(self: Compiler, node: IdentExpr) =
if self.closedOver.len() == 0: if self.closedOver.len() == 0:
self.error("error: closure variable array is empty but LoadHeap would be emitted (this is an internal error and most likely a bug)") self.error("error: closure variable array is empty but LoadHeap would be emitted (this is an internal error and most likely a bug)")
# Heap-allocated closure variable. Stored in a separate "closure array" in the VM that does not have stack semantics. # Heap-allocated closure variable. Stored in a separate "closure array" in the VM that does not have stack semantics.
# This makes closures work as expected and is not comparatively slower than indexing our stack (since they're both # This makes closures work as expected and is not comparatively slower than indexing our stack (since they're both
# dynamic arrays at runtime anyway) # dynamic arrays at runtime anyway)
self.emitByte(LoadHeap) self.emitByte(LoadHeap)
self.emitBytes(self.closedOver.high().toTriple()) self.emitBytes(self.closedOver.high().toTriple())
@ -989,9 +997,9 @@ proc expression(self: Compiler, node: Expression) =
self.error("expression has no type") self.error("expression has no type")
case node.kind: case node.kind:
of callExpr: of callExpr:
discard # TODO discard # TODO
of getItemExpr: of getItemExpr:
discard # TODO discard # TODO
# Note that for setItem and assign we don't convert # Note that for setItem and assign we don't convert
# the node to its true type because that type information # the node to its true type because that type information
# would be lost in the call anyway. The differentiation # would be lost in the call anyway. The differentiation
@ -1014,7 +1022,7 @@ proc expression(self: Compiler, node: Expression) =
# Since all of these AST nodes share the # Since all of these AST nodes share the
# same overall structure and the kind # same overall structure and the kind
# field is enough to tell one from the # field is enough to tell one from the
# other, why bother with specialized # other, why bother with specialized
# cases when one is enough? # cases when one is enough?
self.literal(node) self.literal(node)
else: else:
@ -1127,7 +1135,7 @@ proc statement(self: Compiler, node: Statement) =
of exprStmt: of exprStmt:
var expression = ExprStmt(node).expression var expression = ExprStmt(node).expression
self.expression(expression) self.expression(expression)
self.emitByte(Pop) # Expression statements discard their value. Their main use case is side effects in function calls self.emitByte(Pop) # Expression statements discard their value. Their main use case is side effects in function calls
of NodeKind.ifStmt: of NodeKind.ifStmt:
self.ifStmt(IfStmt(node)) self.ifStmt(IfStmt(node))
of NodeKind.assertStmt: of NodeKind.assertStmt:
@ -1204,7 +1212,7 @@ proc funDecl(self: Compiler, node: FunDecl) =
var function = self.currentFunction var function = self.currentFunction
self.currentFunction = node self.currentFunction = node
# Since the deferred array is a linear # Since the deferred array is a linear
# sequence of instructions and we want # sequence of instructions and we want
# to keep track to whose function's each # to keep track to whose function's each
# set of deferred instruction belongs, # set of deferred instruction belongs,
@ -1250,7 +1258,7 @@ proc declaration(self: Compiler, node: Declaration) =
self.statement(Statement(node)) self.statement(Statement(node))
proc compile*(self: Compiler, ast: seq[ASTNode], file: string): Chunk = proc compile*(self: Compiler, ast: seq[Declaration], file: string): Chunk =
## Compiles a sequence of AST nodes into a chunk ## Compiles a sequence of AST nodes into a chunk
## object ## object
self.chunk = newChunk() self.chunk = newChunk()
@ -1269,4 +1277,4 @@ proc compile*(self: Compiler, ast: seq[ASTNode], file: string): Chunk =
self.emitByte(OpCode.Return) # Exits the VM's main loop when used at the global scope self.emitByte(OpCode.Return) # Exits the VM's main loop when used at the global scope
result = self.chunk result = self.chunk
if self.ast.len() > 0 and self.scopeDepth != -1: if self.ast.len() > 0 and self.scopeDepth != -1:
self.error(&"invalid state: invalid scopeDepth value (expected -1, got {self.scopeDepth}), did you forget to call endScope/beginScope?") self.error(&"invalid state: invalid scopeDepth value (expected -1, got {self.scopeDepth}), did you forget to call endScope/beginScope?")

55
src/frontend/lexer.nim
View File

@ -32,7 +32,7 @@ type
SymbolTable* = ref object SymbolTable* = ref object
## A table of symbols used ## A table of symbols used
## to lex a source file ## to lex a source file
# Although we don't parse keywords # Although we don't parse keywords
# as symbols, but rather as identifiers, # as symbols, but rather as identifiers,
# we keep them here for consistency # we keep them here for consistency
@ -64,7 +64,7 @@ proc addSymbol*(self: SymbolTable, lexeme: string, token: TokenType) =
self.symbols[lexeme] = token self.symbols[lexeme] = token
proc removeSymbol*(self: SymbolTable, lexeme: string) = proc removeSymbol*(self: SymbolTable, lexeme: string) =
## Removes a symbol from the symbol table ## Removes a symbol from the symbol table
## (does nothing if it does not exist) ## (does nothing if it does not exist)
self.symbols.del(lexeme) self.symbols.del(lexeme)
@ -76,30 +76,31 @@ proc addKeyword*(self: SymbolTable, lexeme: string, token: TokenType) =
self.keywords[lexeme] = token self.keywords[lexeme] = token
proc removeKeyword*(self: SymbolTable, lexeme: string) = proc removeKeyword*(self: SymbolTable, lexeme: string) =
## Removes a keyword from the symbol table ## Removes a keyword from the symbol table
## (does nothing if it does not exist) ## (does nothing if it does not exist)
self.keywords.del(lexeme) self.keywords.del(lexeme)
proc existsSymbol*(self: SymbolTable, lexeme: string): bool {.inline.} = proc existsSymbol*(self: SymbolTable, lexeme: string): bool {.inline.} =
## Returns true if a given symbol exists ## Returns true if a given symbol exists
## in the symbol table already ## in the symbol table already
lexeme in self.symbols lexeme in self.symbols
proc existsKeyword*(self: SymbolTable, lexeme: string): bool {.inline.} = proc existsKeyword*(self: SymbolTable, lexeme: string): bool {.inline.} =
## Returns true if a given keyword exists ## Returns true if a given keyword exists
## in the symbol table already ## in the symbol table already
lexeme in self.keywords lexeme in self.keywords
proc getToken(self: Lexer, lexeme: string): Token = proc getToken(self: Lexer, lexeme: string): Token =
## Gets the matching token object for a given ## Gets the matching token object for a given
## string according to the symbol table or ## string according to the symbol table or
## returns nil if there's no match ## returns nil if there's no match
let table = self.symbols let table = self.symbols
var kind = table.symbols.getOrDefault(lexeme, table.keywords.getOrDefault(lexeme, NoMatch)) var kind = table.symbols.getOrDefault(lexeme, table.keywords.getOrDefault(
lexeme, NoMatch))
if kind == NoMatch: if kind == NoMatch:
return nil return nil
new(result) new(result)
@ -125,7 +126,7 @@ proc getSymbols(self: SymbolTable, n: int): seq[string] =
for lexeme in self.symbols.keys(): for lexeme in self.symbols.keys():
if len(lexeme) == n: if len(lexeme) == n:
result.add(lexeme) result.add(lexeme)
# Wrappers around isDigit and isAlphanumeric for # Wrappers around isDigit and isAlphanumeric for
# strings # strings
proc isDigit(s: string): bool = proc isDigit(s: string): bool =
@ -147,7 +148,8 @@ proc getStart*(self: Lexer): int = self.start
proc getCurrent*(self: Lexer): int = self.current proc getCurrent*(self: Lexer): int = self.current
proc getLine*(self: Lexer): int = self.line proc getLine*(self: Lexer): int = self.line
proc getSource*(self: Lexer): string = self.source proc getSource*(self: Lexer): string = self.source
proc getRelPos*(self: Lexer, line: int): tuple[start, stop: int] = (if line > 1: self.lines[line - 2] else: (start: 0, stop: self.current)) proc getRelPos*(self: Lexer, line: int): tuple[start, stop: int] = (if line >
1: self.lines[line - 2] else: (start: 0, stop: self.current))
proc newLexer*(self: Lexer = nil): Lexer = proc newLexer*(self: Lexer = nil): Lexer =
@ -183,7 +185,7 @@ proc incLine(self: Lexer) =
proc step(self: Lexer, n: int = 1): string = proc step(self: Lexer, n: int = 1): string =
## Steps n characters forward in the ## Steps n characters forward in the
## source file (default = 1). A string ## source file (default = 1). A string
## of at most n bytes is returned. If n ## of at most n bytes is returned. If n
## exceeds EOF, the string will be shorter ## exceeds EOF, the string will be shorter
while len(result) < n: while len(result) < n:
@ -196,17 +198,18 @@ proc step(self: Lexer, n: int = 1): string =
proc peek(self: Lexer, distance: int = 0, length: int = 1): string = proc peek(self: Lexer, distance: int = 0, length: int = 1): string =
## Returns a stream of characters of ## Returns a stream of characters of
## at most length bytes from the source ## at most length bytes from the source
## file, starting at the given distance, ## file, starting at the given distance,
## without consuming it. The distance ## without consuming it. The distance
## parameter may be negative to retrieve ## parameter may be negative to retrieve
## previously consumed tokens. If the ## previously consumed tokens. If the
## distance and/or the length are beyond ## distance and/or the length are beyond
## EOF (even partially), the resulting string ## EOF (even partially), the resulting string
## will be shorter than length bytes ## will be shorter than length bytes
var i = distance var i = distance
while len(result) < length: while len(result) < length:
if self.done() or self.current + i > self.source.high() or self.current + i < 0: if self.done() or self.current + i > self.source.high() or
self.current + i < 0:
break break
else: else:
result.add(self.source[self.current + i]) result.add(self.source[self.current + i])
@ -242,9 +245,9 @@ proc check(self: Lexer, args: openarray[string], distance: int = 0): bool =
proc match(self: Lexer, s: string): bool = proc match(self: Lexer, s: string): bool =
## Returns true if the next len(s) bytes ## Returns true if the next len(s) bytes
## of the source file match the provided ## of the source file match the provided
## string. If the match is successful, ## string. If the match is successful,
## len(s) bytes are consumed, otherwise ## len(s) bytes are consumed, otherwise
## false is returned ## false is returned
if not self.check(s): if not self.check(s):
@ -286,7 +289,7 @@ proc parseEscape(self: Lexer) =
# likely be soon. Another notable limitation is that # likely be soon. Another notable limitation is that
# \xhhh and \nnn are limited to the size of a char # \xhhh and \nnn are limited to the size of a char
# (i.e. uint8, or 256 values) # (i.e. uint8, or 256 values)
case self.peek()[0]: # We use a char instead of a string because of how case statements handle ranges with strings case self.peek()[0]: # We use a char instead of a string because of how case statements handle ranges with strings
# (i.e. not well, given they crash the C code generator) # (i.e. not well, given they crash the C code generator)
of 'a': of 'a':
self.source[self.current] = cast[char](0x07) self.source[self.current] = cast[char](0x07)
@ -319,7 +322,7 @@ proc parseEscape(self: Lexer) =
self.source[self.current] = '\'' self.source[self.current] = '\''
of '\\': of '\\':
self.source[self.current] = cast[char](0x5C) self.source[self.current] = cast[char](0x5C)
of '0'..'9': # This is the reason we're using char instead of string. See https://github.com/nim-lang/Nim/issues/19678 of '0'..'9': # This is the reason we're using char instead of string. See https://github.com/nim-lang/Nim/issues/19678
var code = "" var code = ""
var value = 0 var value = 0
var i = self.current var i = self.current
@ -490,7 +493,8 @@ proc parseNumber(self: Lexer) =
discard self.step() discard self.step()
if self.match("'"): if self.match("'"):
# Could be a size specifier, better catch it # Could be a size specifier, better catch it
while (self.peek().isAlphaNumeric() or self.check("_")) and not self.done(): while (self.peek().isAlphaNumeric() or self.check("_")) and
not self.done():
discard self.step() discard self.step()
self.createToken(kind) self.createToken(kind)
if kind == Binary: if kind == Binary:
@ -558,13 +562,14 @@ proc next(self: Lexer) =
elif self.match(["\"", "'"]): elif self.match(["\"", "'"]):
# String or character literal # String or character literal
var mode = "single" var mode = "single"
if self.peek(-1) != "'" and self.check(self.peek(-1)) and self.check(self.peek(-1), 1): if self.peek(-1) != "'" and self.check(self.peek(-1)) and self.check(
self.peek(-1), 1):
# Multiline strings start with 3 quotes # Multiline strings start with 3 quotes
discard self.step(2) discard self.step(2)
mode = "multi" mode = "multi"
self.parseString(self.peek(-1), mode) self.parseString(self.peek(-1), mode)
elif self.peek().isDigit(): elif self.peek().isDigit():
discard self.step() # Needed because parseNumber reads the next discard self.step() # Needed because parseNumber reads the next
# character to tell the base of the number # character to tell the base of the number
# Number literal # Number literal
self.parseNumber() self.parseNumber()

84
src/frontend/meta/ast.nim
View File

@ -33,7 +33,7 @@ type
funDecl = 0'u8, funDecl = 0'u8,
varDecl, varDecl,
# Statements # Statements
forStmt, # Unused for now (for loops are compiled to while loops) forStmt, # Unused for now (for loops are compiled to while loops)
ifStmt, ifStmt,
returnStmt, returnStmt,
breakStmt, breakStmt,
@ -61,8 +61,8 @@ type
sliceExpr, sliceExpr,
callExpr, callExpr,
getItemExpr, # Get expressions like a.b getItemExpr, # Get expressions like a.b
# Primary expressions # Primary expressions
groupingExpr, # Parenthesized expressions such as (true) and (3 + 4) groupingExpr, # Parenthesized expressions such as (true) and (3 + 4)
trueExpr, trueExpr,
falseExpr, falseExpr,
strExpr, strExpr,
@ -76,6 +76,7 @@ type
nanExpr, nanExpr,
infExpr, infExpr,
identExpr, # Identifier identExpr, # Identifier
pragmaExpr
# Here I would've rather used object variants, and in fact that's what was in # Here I would've rather used object variants, and in fact that's what was in
# place before, but not being able to re-declare a field of the same type in # place before, but not being able to re-declare a field of the same type in
@ -97,6 +98,8 @@ type
Declaration* = ref object of ASTNode Declaration* = ref object of ASTNode
## A declaration ## A declaration
pragmas*: seq[Pragma] pragmas*: seq[Pragma]
generics*: seq[tuple[name: IdentExpr, cond: Expression]]
Statement* = ref object of Declaration Statement* = ref object of Declaration
## A statement ## A statement
Expression* = ref object of Statement Expression* = ref object of Statement
@ -145,7 +148,8 @@ type
CallExpr* = ref object of Expression CallExpr* = ref object of Expression
callee*: Expression # The object being called callee*: Expression # The object being called
arguments*: tuple[positionals: seq[Expression], keyword: seq[tuple[name: IdentExpr, value: Expression]]] arguments*: tuple[positionals: seq[Expression], keyword: seq[tuple[
name: IdentExpr, value: Expression]]]
UnaryExpr* = ref object of Expression UnaryExpr* = ref object of Expression
operator*: Token operator*: Token
@ -165,7 +169,8 @@ type
LambdaExpr* = ref object of Expression LambdaExpr* = ref object of Expression
body*: Statement body*: Statement
arguments*: seq[tuple[name: IdentExpr, valueType: Expression, mutable: bool, isRef: bool, isPtr: bool]] arguments*: seq[tuple[name: IdentExpr, valueType: Expression,
mutable: bool, isRef: bool, isPtr: bool]]
defaults*: seq[Expression] defaults*: seq[Expression]
isGenerator*: bool isGenerator*: bool
isAsync*: bool isAsync*: bool
@ -245,7 +250,8 @@ type
FunDecl* = ref object of Declaration FunDecl* = ref object of Declaration
name*: IdentExpr name*: IdentExpr
body*: Statement body*: Statement
arguments*: seq[tuple[name: IdentExpr, valueType: Expression, mutable: bool, isRef: bool, isPtr: bool]] arguments*: seq[tuple[name: IdentExpr, valueType: Expression,
mutable: bool, isRef: bool, isPtr: bool]]
defaults*: seq[Expression] defaults*: seq[Expression]
isAsync*: bool isAsync*: bool
isGenerator*: bool isGenerator*: bool
@ -264,18 +270,19 @@ proc isConst*(self: ASTNode): bool =
## strings and singletons count as ## strings and singletons count as
## constants ## constants
case self.kind: case self.kind:
of intExpr, hexExpr, binExpr, octExpr, strExpr, falseExpr, trueExpr, infExpr, nanExpr, floatExpr, nilExpr: of intExpr, hexExpr, binExpr, octExpr, strExpr, falseExpr, trueExpr,
infExpr, nanExpr, floatExpr, nilExpr:
return true return true
else: else:
return false return false
proc isLiteral*(self: ASTNode): bool {.inline.} =
proc isLiteral*(self: ASTNode): bool {.inline.} =
## Returns if the AST node represents a literal ## Returns if the AST node represents a literal
self.kind in {intExpr, hexExpr, binExpr, octExpr, self.kind in {intExpr, hexExpr, binExpr, octExpr,
strExpr, falseExpr, trueExpr, infExpr, strExpr, falseExpr, trueExpr, infExpr,
nanExpr, floatExpr, nilExpr nanExpr, floatExpr, nilExpr
} }
## AST node constructors ## AST node constructors
proc newASTNode*(kind: NodeKind, token: Token): ASTNode = proc newASTNode*(kind: NodeKind, token: Token): ASTNode =
@ -285,6 +292,13 @@ proc newASTNode*(kind: NodeKind, token: Token): ASTNode =
result.token = token result.token = token
proc newPragma*(name: IdentExpr, args: seq[LiteralExpr]): Pragma =
new(result)
result.kind = pragmaExpr
result.args = args
result.name = name
proc newIntExpr*(literal: Token): IntExpr = proc newIntExpr*(literal: Token): IntExpr =
result = IntExpr(kind: intExpr) result = IntExpr(kind: intExpr)
result.literal = literal result.literal = literal
@ -315,11 +329,16 @@ proc newFloatExpr*(literal: Token): FloatExpr =
result.token = literal result.token = literal
proc newTrueExpr*(token: Token): LiteralExpr = LiteralExpr(kind: trueExpr, token: token, literal: token) proc newTrueExpr*(token: Token): LiteralExpr = LiteralExpr(kind: trueExpr,
proc newFalseExpr*(token: Token): LiteralExpr = LiteralExpr(kind: falseExpr, token: token, literal: token) token: token, literal: token)
proc newNaNExpr*(token: Token): LiteralExpr = LiteralExpr(kind: nanExpr, token: token, literal: token) proc newFalseExpr*(token: Token): LiteralExpr = LiteralExpr(kind: falseExpr,
proc newNilExpr*(token: Token): LiteralExpr = LiteralExpr(kind: nilExpr, token: token, literal: token) token: token, literal: token)
proc newInfExpr*(token: Token): LiteralExpr = LiteralExpr(kind: infExpr, token: token, literal: token) proc newNaNExpr*(token: Token): LiteralExpr = LiteralExpr(kind: nanExpr,
token: token, literal: token)
proc newNilExpr*(token: Token): LiteralExpr = LiteralExpr(kind: nilExpr,
token: token, literal: token)
proc newInfExpr*(token: Token): LiteralExpr = LiteralExpr(kind: infExpr,
token: token, literal: token)
proc newStrExpr*(literal: Token): StrExpr = proc newStrExpr*(literal: Token): StrExpr =
@ -346,8 +365,10 @@ proc newGroupingExpr*(expression: Expression, token: Token): GroupingExpr =
result.token = token result.token = token
proc newLambdaExpr*(arguments: seq[tuple[name: IdentExpr, valueType: Expression, mutable: bool, isRef: bool, isPtr: bool]], defaults: seq[Expression], body: Statement, proc newLambdaExpr*(arguments: seq[tuple[name: IdentExpr, valueType: Expression,
isGenerator: bool, isAsync: bool, token: Token, returnType: Expression): LambdaExpr = mutable: bool, isRef: bool, isPtr: bool]], defaults: seq[Expression],
body: Statement, isGenerator: bool, isAsync: bool, token: Token,
returnType: Expression, pragmas: seq[Pragma]): LambdaExpr =
result = LambdaExpr(kind: lambdaExpr) result = LambdaExpr(kind: lambdaExpr)
result.body = body result.body = body
result.arguments = arguments result.arguments = arguments
@ -357,16 +378,19 @@ proc newLambdaExpr*(arguments: seq[tuple[name: IdentExpr, valueType: Expression,
result.token = token result.token = token
result.returnType = returnType result.returnType = returnType
result.isPure = false result.isPure = false
result.pragmas = pragmas
proc newGetItemExpr*(obj: Expression, name: IdentExpr, token: Token): GetItemExpr = proc newGetItemExpr*(obj: Expression, name: IdentExpr,
token: Token): GetItemExpr =
result = GetItemExpr(kind: getItemExpr) result = GetItemExpr(kind: getItemExpr)
result.obj = obj result.obj = obj
result.name = name result.name = name
result.token = token result.token = token
proc newSetItemExpr*(obj: Expression, name: IdentExpr, value: Expression, token: Token): SetItemExpr = proc newSetItemExpr*(obj: Expression, name: IdentExpr, value: Expression,
token: Token): SetItemExpr =
result = SetItemExpr(kind: setItemExpr) result = SetItemExpr(kind: setItemExpr)
result.obj = obj result.obj = obj
result.name = name result.name = name
@ -374,8 +398,8 @@ proc newSetItemExpr*(obj: Expression, name: IdentExpr, value: Expression, token:
result.token = token result.token = token
proc newCallExpr*(callee: Expression, arguments: tuple[positionals: seq[Expression], proc newCallExpr*(callee: Expression, arguments: tuple[positionals: seq[
keyword: seq[tuple[name: IdentExpr, value: Expression]]], Expression], keyword: seq[tuple[name: IdentExpr, value: Expression]]],
token: Token): CallExpr = token: Token): CallExpr =
result = CallExpr(kind: callExpr) result = CallExpr(kind: callExpr)
result.callee = callee result.callee = callee
@ -412,7 +436,8 @@ proc newYieldExpr*(expression: Expression, token: Token): YieldExpr =
result.token = token result.token = token
proc newAssignExpr*(name: Expression, value: Expression, token: Token): AssignExpr = proc newAssignExpr*(name: Expression, value: Expression,
token: Token): AssignExpr =
result = AssignExpr(kind: assignExpr) result = AssignExpr(kind: assignExpr)
result.name = name result.name = name
result.value = value result.value = value
@ -484,15 +509,16 @@ proc newBlockStmt*(code: seq[Declaration], token: Token): BlockStmt =
result.token = token result.token = token
proc newWhileStmt*(condition: Expression, body: Statement, token: Token): WhileStmt = proc newWhileStmt*(condition: Expression, body: Statement,
token: Token): WhileStmt =
result = WhileStmt(kind: whileStmt) result = WhileStmt(kind: whileStmt)
result.condition = condition result.condition = condition
result.body = body result.body = body
result.token = token result.token = token
proc newForEachStmt*(identifier: IdentExpr, expression: Expression, body: Statement, proc newForEachStmt*(identifier: IdentExpr, expression: Expression,
token: Token): ForEachStmt = body: Statement, token: Token): ForEachStmt =
result = ForEachStmt(kind: forEachStmt) result = ForEachStmt(kind: forEachStmt)
result.identifier = identifier result.identifier = identifier
result.expression = expression result.expression = expression
@ -540,7 +566,7 @@ proc newVarDecl*(name: IdentExpr, value: Expression, isConst: bool = false,
proc newFunDecl*(name: IdentExpr, arguments: seq[tuple[name: IdentExpr, valueType: Expression, mutable: bool, isRef: bool, isPtr: bool]], defaults: seq[Expression], proc newFunDecl*(name: IdentExpr, arguments: seq[tuple[name: IdentExpr, valueType: Expression, mutable: bool, isRef: bool, isPtr: bool]], defaults: seq[Expression],
body: Statement, isAsync, isGenerator: bool, body: Statement, isAsync, isGenerator: bool,
isPrivate: bool, token: Token, pragmas: seq[Pragma], isPrivate: bool, token: Token, pragmas: seq[Pragma],
returnType: Expression): FunDecl = returnType: Expression): FunDecl =
result = FunDecl(kind: funDecl) result = FunDecl(kind: funDecl)
@ -667,4 +693,4 @@ proc `$`*(self: ASTNode): string =
discard discard
proc `==`*(self, other: IdentExpr): bool {.inline.} = self.token == other.token proc `==`*(self, other: IdentExpr): bool {.inline.} = self.token == other.token

73
src/frontend/meta/bytecode.nim
View File

@ -55,15 +55,15 @@ type
## Enum of Peon's bytecode opcodes ## Enum of Peon's bytecode opcodes
# Note: x represents the argument # Note: x represents the argument
# to unary opcodes, while a and b # to unary opcodes, while a and b
# represent arguments to binary # represent arguments to binary
# opcodes. Other variable names (c, d, ...) # opcodes. Other variable names (c, d, ...)
# may be used for more complex opcodes. If # may be used for more complex opcodes. If
# an opcode takes any arguments at runtime, # an opcode takes any arguments at runtime,
# they come from either the stack or the VM's # they come from either the stack or the VM's
# closure array. Some other opcodes (e.g. # closure array. Some other opcodes (e.g.
# jumps), take arguments in the form of 16 # jumps), take arguments in the form of 16
# or 24 bit numbers that are defined statically # or 24 bit numbers that are defined statically
# at compilation time into the bytecode # at compilation time into the bytecode
# These push a constant onto the stack # These push a constant onto the stack
@ -85,23 +85,23 @@ type
LoadNan, LoadNan,
LoadInf, LoadInf,
## Basic stack operations ## Basic stack operations
Pop, # Pops an element off the stack and discards it Pop, # Pops an element off the stack and discards it
Push, # Pushes x onto the stack Push, # Pushes x onto the stack
PopN, # Pops x elements off the stack (optimization for exiting scopes and returning from functions) PopN, # Pops x elements off the stack (optimization for exiting scopes and returning from functions)
## Name resolution/handling ## Name resolution/handling
LoadAttribute, # Pushes the attribute b of object a onto the stack LoadAttribute, # Pushes the attribute b of object a onto the stack
LoadVar, # Pushes the object at position x in the stack onto the stack LoadVar, # Pushes the object at position x in the stack onto the stack
StoreVar, # Stores the value of b at position a in the stack StoreVar, # Stores the value of b at position a in the stack
LoadHeap, # Pushes the object position x in the closure array onto the stack LoadHeap, # Pushes the object position x in the closure array onto the stack
StoreHeap, # Stores the value of b at position a in the closure array StoreHeap, # Stores the value of b at position a in the closure array
## Looping and jumping ## Looping and jumping
Jump, # Absolute, unconditional jump into the bytecode Jump, # Absolute, unconditional jump into the bytecode
JumpForwards, # Relative, unconditional, positive jump in the bytecode JumpForwards, # Relative, unconditional, positive jump in the bytecode
JumpBackwards, # Relative, unconditional, negative jump in the bytecode JumpBackwards, # Relative, unconditional, negative jump in the bytecode
JumpIfFalse, # Jumps to a relative index in the bytecode if x is false JumpIfFalse, # Jumps to a relative index in the bytecode if x is false
JumpIfTrue, # Jumps to a relative index in the bytecode if x is true JumpIfTrue, # Jumps to a relative index in the bytecode if x is true
JumpIfFalsePop, # Like JumpIfFalse, but also pops off the stack (regardless of truthyness). Optimization for if statements JumpIfFalsePop, # Like JumpIfFalse, but also pops off the stack (regardless of truthyness). Optimization for if statements
JumpIfFalseOrPop, # Jumps to an absolute index in the bytecode if x is false and pops otherwise (used for logical and) JumpIfFalseOrPop, # Jumps to an absolute index in the bytecode if x is false and pops otherwise (used for logical and)
## Long variants of jumps (they use a 24-bit operand instead of a 16-bit one) ## Long variants of jumps (they use a 24-bit operand instead of a 16-bit one)
LongJump, LongJump,
LongJumpIfFalse, LongJumpIfFalse,
@ -111,29 +111,29 @@ type
LongJumpForwards, LongJumpForwards,
LongJumpBackwards, LongJumpBackwards,
## Functions ## Functions
Call, # Calls a function and initiates a new stack frame Call, # Calls a function and initiates a new stack frame
Return, # Terminates the current function without popping off the stack Return, # Terminates the current function without popping off the stack
ReturnPop, # Pops a return value off the stack and terminates the current function ReturnPop, # Pops a return value off the stack and terminates the current function
## Exception handling ## Exception handling
Raise, # Raises exception x or re-raises active exception if x is nil Raise, # Raises exception x or re-raises active exception if x is nil
BeginTry, # Initiates an exception handling context BeginTry, # Initiates an exception handling context
FinishTry, # Closes the current exception handling context FinishTry, # Closes the current exception handling context
## Generators ## Generators
Yield, # Yields control from a generator back to the caller Yield, # Yields control from a generator back to the caller
## Coroutines ## Coroutines
Await, # Calls an asynchronous function Await, # Calls an asynchronous function
## Misc ## Misc
Assert, # Raises an AssertionFailed exception if x is false Assert, # Raises an AssertionFailed exception if x is false
NoOp, # Just a no-op NoOp, # Just a no-op
# We group instructions by their operation/operand types for easier handling when debugging # We group instructions by their operation/operand types for easier handling when debugging
# Simple instructions encompass instructions that push onto/pop off the stack unconditionally (True, False, Pop, etc.) # Simple instructions encompass instructions that push onto/pop off the stack unconditionally (True, False, Pop, etc.)
const simpleInstructions* = {OpCode.Return, LoadNil, const simpleInstructions* = {OpCode.Return, LoadNil,
LoadTrue, LoadFalse, LoadTrue, LoadFalse,
LoadNan, LoadInf, LoadNan, LoadInf,
Pop, OpCode.Raise, Pop, OpCode.Raise,
BeginTry, FinishTry, BeginTry, FinishTry,
OpCode.Yield, OpCode.Await, OpCode.Yield, OpCode.Await,
OpCode.NoOp, OpCode.Return, OpCode.NoOp, OpCode.Return,
@ -159,10 +159,10 @@ const stackDoubleInstructions* = {}
const argumentDoubleInstructions* = {PopN, } const argumentDoubleInstructions* = {PopN, }
# Jump instructions jump at relative or absolute bytecode offsets # Jump instructions jump at relative or absolute bytecode offsets
const jumpInstructions* = {Jump, LongJump, JumpIfFalse, JumpIfFalsePop, const jumpInstructions* = {Jump, LongJump, JumpIfFalse, JumpIfFalsePop,
JumpForwards, JumpBackwards, JumpForwards, JumpBackwards,
LongJumpIfFalse, LongJumpIfFalsePop, LongJumpIfFalse, LongJumpIfFalsePop,
LongJumpForwards, LongJumpBackwards, LongJumpForwards, LongJumpBackwards,
JumpIfTrue, LongJumpIfTrue} JumpIfTrue, LongJumpIfTrue}
@ -234,7 +234,8 @@ proc findOrAddConstant(self: Chunk, constant: Expression, kind: Type): int =
if c.kind != constant.kind: if c.kind != constant.kind:
continue continue
if constant.isConst(): if constant.isConst():
if LiteralExpr(c).literal.lexeme == LiteralExpr(constant).literal.lexeme: if LiteralExpr(c).literal.lexeme == LiteralExpr(
constant).literal.lexeme:
# This wouldn't work for stuff like 2e3 and 2000.0, but those # This wouldn't work for stuff like 2e3 and 2000.0, but those
# forms are collapsed in the compiler before being written # forms are collapsed in the compiler before being written
# to the constants table # to the constants table
@ -251,7 +252,7 @@ proc findOrAddConstant(self: Chunk, constant: Expression, kind: Type): int =
proc addConstant*(self: Chunk, constant: Expression, kind: Type): array[3, uint8] = proc addConstant*(self: Chunk, constant: Expression, kind: Type): array[3, uint8] =
## Writes a constant of the given type in the chunk's constant ## Writes a constant of the given type in the chunk's constant
## table. Returns its index as an array of 3 unsigned 8 bit integers. ## table. Returns its index as an array of 3 unsigned 8 bit integers.
## Constant indexes are reused if a constant is used more than once ## Constant indexes are reused if a constant is used more than once
## and self.reuseConsts equals true ## and self.reuseConsts equals true
if self.consts.high() == 16777215: if self.consts.high() == 16777215:
# The constant index is a 24 bit unsigned integer, so that's as far # The constant index is a 24 bit unsigned integer, so that's as far

98
src/frontend/meta/token.nim
View File

@ -18,72 +18,72 @@ import strformat
type type
TokenType* {.pure.} = enum TokenType* {.pure.} = enum
## Token types enumeration ## Token types enumeration
# Booleans # Booleans
True, False, True, False,
# Other singleton types # Other singleton types
Infinity, NotANumber, Nil Infinity, NotANumber, Nil
# Control flow statements # Control flow statements
If, Else, If, Else,
# Looping statements # Looping statements
While, For, While, For,
# Keywords # Keywords
Function, Break, Continue, Function, Break, Continue,
Var, Let, Const, Return, Var, Let, Const, Return,
Coroutine, Generator, Import, Coroutine, Generator, Import,
Raise, Assert, Await, Foreach, Raise, Assert, Await, Foreach,
Yield, Defer, Try, Except, Yield, Defer, Try, Except,
Finally, Type, Operator, Case, Finally, Type, Operator, Case,
Enum, From, Ptr, Ref Enum, From, Ptr, Ref
# Literal types # Literal types
Integer, Float, String, Identifier, Integer, Float, String, Identifier,
Binary, Octal, Hex, Char Binary, Octal, Hex, Char
# Brackets, parentheses, # Brackets, parentheses,
# operators and others # operators and others
LeftParen, RightParen, # () LeftParen, RightParen, # ()
LeftBrace, RightBrace, # {} LeftBrace, RightBrace, # {}
LeftBracket, RightBracket, # [] LeftBracket, RightBracket, # []
Dot, Semicolon, Comma, # . ; , Dot, Semicolon, Comma, # . ; ,
# Miscellaneous # Miscellaneous
EndOfFile, # Marks the end of the token stream EndOfFile, # Marks the end of the token stream
NoMatch, # Used internally by the symbol table NoMatch, # Used internally by the symbol table
Comment, # Useful for documentation comments, pragmas, etc. Comment, # Useful for documentation comments, pragmas, etc.
Symbol, # A generic symbol Symbol, # A generic symbol
# These are not used at the moment but may be # These are not used at the moment but may be
# employed to enforce indentation or other neat # employed to enforce indentation or other neat
# stuff I haven't thought about yet # stuff I haven't thought about yet
Whitespace, Whitespace,
Tab, Tab,
Token* = ref object Token* = ref object
## A token object ## A token object
kind*: TokenType # Type of the token kind*: TokenType # Type of the token
lexeme*: string # The lexeme associated to the token lexeme*: string # The lexeme associated to the token
line*: int # The line where the token appears line*: int # The line where the token appears
pos*: tuple[start, stop: int] # The absolute position in the source file pos*: tuple[start, stop: int] # The absolute position in the source file
# (0-indexed and inclusive at the beginning) # (0-indexed and inclusive at the beginning)
proc `$`*(self: Token): string = proc `$`*(self: Token): string =
## Strinfifies ## Strinfifies
if self != nil: if self != nil:
result = &"Token(kind={self.kind}, lexeme='{$(self.lexeme)}', line={self.line}, pos=({self.pos.start}, {self.pos.stop}))" result = &"Token(kind={self.kind}, lexeme='{$(self.lexeme)}', line={self.line}, pos=({self.pos.start}, {self.pos.stop}))"
else: else:
result = "nil" result = "nil"
proc `==`*(self, other: Token): bool = proc `==`*(self, other: Token): bool =
## Returns self == other ## Returns self == other
return self.kind == other.kind and self.lexeme == other.lexeme return self.kind == other.kind and self.lexeme == other.lexeme

217
src/frontend/parser.nim
View File

@ -26,8 +26,8 @@ import meta/errors
export token, ast, errors export token, ast, errors
type type
LoopContext {.pure.} = enum LoopContext {.pure.} = enum
Loop, None Loop, None
Precedence {.pure.} = enum Precedence {.pure.} = enum
@ -42,7 +42,7 @@ type
Addition, Addition,
Multiplication, Multiplication,
Power, Power,
None # Used for stuff that isn't an operator None # Used for stuff that isn't an operator
OperatorTable = ref object OperatorTable = ref object
## A table for storing and ## A table for storing and
@ -76,14 +76,16 @@ type
# Stores the current function # Stores the current function
# being parsed. This is a reference # being parsed. This is a reference
# to either a FunDecl or LambdaExpr # to either a FunDecl or LambdaExpr
# AST node and is nil when the parser # AST node and is nil when the parser
# is at the top-level. It allows the # is at the top-level. It allows the
# parser to detect errors like return # parser to detect errors like return
# outside functions # outside functions
currentFunction: Declaration currentFunction: Declaration
# Stores the current scope depth (0 = global, > 0 local) # Stores the current scope depth (0 = global, > 0 local)
scopeDepth: int scopeDepth: int
operators: OperatorTable operators: OperatorTable
# The AST node
tree: seq[Declaration]
proc newOperatorTable: OperatorTable = proc newOperatorTable: OperatorTable =
@ -101,7 +103,7 @@ proc addOperator(self: OperatorTable, lexeme: string) =
## is inferred from the operator's lexeme (the ## is inferred from the operator's lexeme (the
## criteria are similar to Nim's) ## criteria are similar to Nim's)
if lexeme in self.tokens: if lexeme in self.tokens:
return # We've already added it! return # We've already added it!
var prec = Precedence.high() var prec = Precedence.high()
if lexeme.len() >= 2 and lexeme[^2..^1] in ["->", "~>", "=>"]: if lexeme.len() >= 2 and lexeme[^2..^1] in ["->", "~>", "=>"]:
prec = Arrow prec = Arrow
@ -128,9 +130,9 @@ proc getPrecedence(self: OperatorTable, lexeme: string): Precedence =
for (prec, operators) in self.precedence.pairs(): for (prec, operators) in self.precedence.pairs():
if lexeme in operators: if lexeme in operators:
return prec return prec
proc newParser*: Parser =
proc newParser*: Parser =
## Initializes a new Parser object ## Initializes a new Parser object
new(result) new(result)
result.current = 0 result.current = 0
@ -140,11 +142,13 @@ proc newParser*: Parser =
result.currentLoop = LoopContext.None result.currentLoop = LoopContext.None
result.scopeDepth = 0 result.scopeDepth = 0
result.operators = newOperatorTable() result.operators = newOperatorTable()
result.tree = @[]
# Public getters for improved error formatting # Public getters for improved error formatting
proc getCurrent*(self: Parser): int {.inline.} = self.current proc getCurrent*(self: Parser): int {.inline.} = self.current
proc getCurrentToken*(self: Parser): Token {.inline.} = (if self.getCurrent() >= self.tokens.high() or proc getCurrentToken*(self: Parser): Token {.inline.} = (if self.getCurrent() >=
self.tokens.high() or
self.getCurrent() - 1 < 0: self.tokens[^1] else: self.tokens[self.current - 1]) self.getCurrent() - 1 < 0: self.tokens[^1] else: self.tokens[self.current - 1])
# Handy templates to make our life easier, thanks nim! # Handy templates to make our life easier, thanks nim!
@ -158,7 +162,8 @@ proc peek(self: Parser, distance: int = 0): Token =
## token is returned. A negative distance may ## token is returned. A negative distance may
## be used to retrieve previously consumed ## be used to retrieve previously consumed
## tokens ## tokens
if self.tokens.high() == -1 or self.current + distance > self.tokens.high() or self.current + distance < 0: if self.tokens.high() == -1 or self.current + distance > self.tokens.high(
) or self.current + distance < 0:
result = endOfFile result = endOfFile
else: else:
result = self.tokens[self.current + distance] result = self.tokens[self.current + distance]
@ -173,7 +178,7 @@ proc done(self: Parser): bool =
result = self.peek().kind == EndOfFile result = self.peek().kind == EndOfFile
proc step(self: Parser, n: int = 1): Token = proc step(self: Parser, n: int = 1): Token =
## Steps n tokens into the input, ## Steps n tokens into the input,
## returning the last consumed one ## returning the last consumed one
if self.done(): if self.done():
@ -197,7 +202,8 @@ proc error(self: Parser, message: string) {.raises: [ParseError, ValueError].} =
# tell at tokenization time which of the two contexts we're in, we just treat everything # tell at tokenization time which of the two contexts we're in, we just treat everything
# as a symbol and in the cases where we need a specific token we just match the string # as a symbol and in the cases where we need a specific token we just match the string
# directly # directly
proc check[T: TokenType or string](self: Parser, kind: T, distance: int = 0): bool = proc check[T: TokenType or string](self: Parser, kind: T,
distance: int = 0): bool =
## Checks if the given token at the given distance ## Checks if the given token at the given distance
## matches the expected kind and returns a boolean. ## matches the expected kind and returns a boolean.
## The distance parameter is passed directly to ## The distance parameter is passed directly to
@ -207,7 +213,7 @@ proc check[T: TokenType or string](self: Parser, kind: T, distance: int = 0): bo
else: else:
when T is string: when T is string:
self.peek(distance).lexeme == kind self.peek(distance).lexeme == kind
proc check[T: TokenType or string](self: Parser, kind: openarray[T]): bool = proc check[T: TokenType or string](self: Parser, kind: openarray[T]): bool =
## Calls self.check() in a loop with each entry of ## Calls self.check() in a loop with each entry of
@ -243,7 +249,8 @@ proc match[T: TokenType or string](self: Parser, kind: openarray[T]): bool =
result = false result = false
proc expect[T: TokenType or string](self: Parser, kind: T, message: string = "") = proc expect[T: TokenType or string](self: Parser, kind: T,
message: string = "") =
## Behaves like self.match(), except that ## Behaves like self.match(), except that
## when a token doesn't match, an error ## when a token doesn't match, an error
## is raised. If no error message is ## is raised. If no error message is
@ -255,7 +262,8 @@ proc expect[T: TokenType or string](self: Parser, kind: T, message: string = "")
self.error(message) self.error(message)
proc expect[T: TokenType or string](self: Parser, kind: openarray[T], message: string = "") = proc expect[T: TokenType or string](self: Parser, kind: openarray[T],
message: string = "") =
## Behaves like self.expect(), except that ## Behaves like self.expect(), except that
## an error is raised only if none of the ## an error is raised only if none of the
## given token kinds matches ## given token kinds matches
@ -264,22 +272,24 @@ proc expect[T: TokenType or string](self: Parser, kind: openarray[T], message: s
return return
if message.len() == 0: if message.len() == 0:
self.error(&"""expecting any of the following tokens: {kinds.join(", ")}, but got {self.peek().kind} instead""") self.error(&"""expecting any of the following tokens: {kinds.join(", ")}, but got {self.peek().kind} instead""")
# Forward declarations # Forward declarations
proc expression(self: Parser): Expression proc expression(self: Parser): Expression
proc expressionStatement(self: Parser): Statement proc expressionStatement(self: Parser): Statement
proc statement(self: Parser): Statement proc statement(self: Parser): Statement
proc varDecl(self: Parser, isLet: bool = false, isConst: bool = false): Declaration proc varDecl(self: Parser, isLet: bool = false,
proc funDecl(self: Parser, isAsync: bool = false, isGenerator: bool = false, isLambda: bool = false, isOperator: bool = false): Declaration isConst: bool = false): Declaration
proc funDecl(self: Parser, isAsync: bool = false, isGenerator: bool = false,
isLambda: bool = false, isOperator: bool = false): Declaration
proc declaration(self: Parser): Declaration proc declaration(self: Parser): Declaration
# End of forward declarations # End of forward declarations
proc primary(self: Parser): Expression = proc primary(self: Parser): Expression =
## Parses primary expressions such ## Parses primary expressions such
## as integer literals and keywords ## as integer literals and keywords
## that map to builtin types (true, ## that map to builtin types (true,
## false, nil, etc.) ## false, nil, etc.)
case self.peek().kind: case self.peek().kind:
of True: of True:
@ -337,13 +347,14 @@ proc primary(self: Parser): Expression =
result = newInfExpr(self.step()) result = newInfExpr(self.step())
of Function: of Function:
discard self.step() discard self.step()
result = Expression(self.funDecl(isLambda=true)) result = Expression(self.funDecl(isLambda = true))
of Coroutine: of Coroutine:
discard self.step() discard self.step()
result = Expression(self.funDecl(isAsync=true, isLambda=true)) result = Expression(self.funDecl(isAsync = true, isLambda = true))
of Generator: of Generator:
discard self.step() discard self.step()
result = Expression(self.funDecl(isGenerator=true, isLambda=true)) result = Expression(self.funDecl(isGenerator = true,
isLambda = true))
else: else:
self.error("invalid syntax") self.error("invalid syntax")
@ -353,7 +364,9 @@ proc makeCall(self: Parser, callee: Expression): Expression =
## to parse a function call ## to parse a function call
let tok = self.peek(-1) let tok = self.peek(-1)
var argNames: seq[IdentExpr] = @[] var argNames: seq[IdentExpr] = @[]
var arguments: tuple[positionals: seq[Expression], keyword: seq[tuple[name: IdentExpr, value: Expression]]] = (positionals: @[], keyword: @[]) var arguments: tuple[positionals: seq[Expression], keyword: seq[tuple[
name: IdentExpr, value: Expression]]] = (positionals: @[],
keyword: @[])
var argument: Expression = nil var argument: Expression = nil
var argCount = 0 var argCount = 0
if not self.check(RightParen): if not self.check(RightParen):
@ -367,7 +380,8 @@ proc makeCall(self: Parser, callee: Expression): Expression =
if IdentExpr(AssignExpr(argument).name) in argNames: if IdentExpr(AssignExpr(argument).name) in argNames:
self.error("duplicate keyword argument in call") self.error("duplicate keyword argument in call")
argNames.add(IdentExpr(AssignExpr(argument).name)) argNames.add(IdentExpr(AssignExpr(argument).name))
arguments.keyword.add((name: IdentExpr(AssignExpr(argument).name), value: AssignExpr(argument).value)) arguments.keyword.add((name: IdentExpr(AssignExpr(
argument).name), value: AssignExpr(argument).value))
elif arguments.keyword.len() == 0: elif arguments.keyword.len() == 0:
arguments.positionals.add(argument) arguments.positionals.add(argument)
else: else:
@ -379,7 +393,7 @@ proc makeCall(self: Parser, callee: Expression): Expression =
result = newCallExpr(callee, arguments, tok) result = newCallExpr(callee, arguments, tok)
proc call(self: Parser): Expression = proc call(self: Parser): Expression =
## Parses function calls, object field ## Parses function calls, object field
## accessing and slicing expressions ## accessing and slicing expressions
result = self.primary() result = self.primary()
@ -388,7 +402,8 @@ proc call(self: Parser): Expression =
result = self.makeCall(result) result = self.makeCall(result)
elif self.match(Dot): elif self.match(Dot):
self.expect(Identifier, "expecting attribute name after '.'") self.expect(Identifier, "expecting attribute name after '.'")
result = newGetItemExpr(result, newIdentExpr(self.peek(-1)), self.peek(-1)) result = newGetItemExpr(result, newIdentExpr(self.peek(-1)),
self.peek(-1))
elif self.match(LeftBracket): elif self.match(LeftBracket):
# Slicing such as a[1:2], which is then # Slicing such as a[1:2], which is then
# translated to `[]`(a, 1, 2) # translated to `[]`(a, 1, 2)
@ -408,7 +423,7 @@ proc call(self: Parser): Expression =
## Operator parsing handlers ## Operator parsing handlers
proc unary(self: Parser): Expression = proc unary(self: Parser): Expression =
if self.peek().lexeme in self.operators.tokens: if self.peek().lexeme in self.operators.tokens:
result = newUnaryExpr(self.step(), self.unary()) result = newUnaryExpr(self.step(), self.unary())
else: else:
@ -484,7 +499,8 @@ proc parseAssign(self: Parser): Expression =
of identExpr, sliceExpr: of identExpr, sliceExpr:
result = newAssignExpr(result, value, tok) result = newAssignExpr(result, value, tok)
of getItemExpr: of getItemExpr:
result = newSetItemExpr(GetItemExpr(result).obj, GetItemExpr(result).name, value, tok) result = newSetItemExpr(GetItemExpr(result).obj, GetItemExpr(
result).name, value, tok)
else: else:
self.error("invalid assignment target") self.error("invalid assignment target")
@ -531,6 +547,8 @@ proc blockStmt(self: Parser): Statement =
var code: seq[Declaration] = @[] var code: seq[Declaration] = @[]
while not self.check(RightBrace) and not self.done(): while not self.check(RightBrace) and not self.done():
code.add(self.declaration()) code.add(self.declaration())
if self.tree[^1] == nil:
self.tree.delete(self.tree.high())
self.expect(RightBrace, "expecting '}'") self.expect(RightBrace, "expecting '}'")
result = newBlockStmt(code, tok) result = newBlockStmt(code, tok)
self.endScope() self.endScope()
@ -687,7 +705,7 @@ proc whileStmt(self: Parser): Statement =
self.endScope() self.endScope()
proc forStmt(self: Parser): Statement = proc forStmt(self: Parser): Statement =
## Parses a C-style for loop ## Parses a C-style for loop
self.beginScope() self.beginScope()
let tok = self.peek(-1) let tok = self.peek(-1)
@ -715,7 +733,8 @@ proc forStmt(self: Parser): Statement =
if increment != nil: if increment != nil:
# The increment runs after each iteration, so we # The increment runs after each iteration, so we
# inject it into the block as the last statement # inject it into the block as the last statement
body = newBlockStmt(@[Declaration(body), newExprStmt(increment, increment.token)], tok) body = newBlockStmt(@[Declaration(body), newExprStmt(increment,
increment.token)], tok)
if condition == nil: if condition == nil:
## An empty condition is functionally ## An empty condition is functionally
## equivalent to "true" ## equivalent to "true"
@ -766,7 +785,8 @@ template checkDecl(self: Parser, isPrivate: bool) =
self.error("cannot bind public names inside local scopes") self.error("cannot bind public names inside local scopes")
proc varDecl(self: Parser, isLet: bool = false, isConst: bool = false): Declaration = proc varDecl(self: Parser, isLet: bool = false,
isConst: bool = false): Declaration =
## Parses variable declarations ## Parses variable declarations
var tok = self.peek(-1) var tok = self.peek(-1)
var value: Expression var value: Expression
@ -792,17 +812,22 @@ proc varDecl(self: Parser, isLet: bool = false, isConst: bool = false): Declarat
self.expect(Semicolon, &"expecting semicolon after declaration") self.expect(Semicolon, &"expecting semicolon after declaration")
case tok.kind: case tok.kind:
of Var: of Var:
result = newVarDecl(name, value, isPrivate=isPrivate, token=tok, valueType=valueType, pragmas=(@[])) result = newVarDecl(name, value, isPrivate = isPrivate, token = tok,
valueType = valueType, pragmas = (@[]))
of Const: of Const:
result = newVarDecl(name, value, isPrivate=isPrivate, token=tok, isConst=true, valueType=valueType, pragmas=(@[])) result = newVarDecl(name, value, isPrivate = isPrivate, token = tok,
isConst = true, valueType = valueType, pragmas = (@[]))
of Let: of Let:
result = newVarDecl(name, value, isPrivate=isPrivate, token=tok, isLet=isLet, valueType=valueType, pragmas=(@[])) result = newVarDecl(name, value, isPrivate = isPrivate, token = tok,
isLet = isLet, valueType = valueType, pragmas = (@[]))
else: else:
discard # Unreachable discard # Unreachable
proc parseDeclArguments(self: Parser, arguments: var seq[tuple[name: IdentExpr, valueType: Expression, mutable: bool, isRef: bool, isPtr: bool]], proc parseDeclArguments(self: Parser, arguments: var seq[tuple[name: IdentExpr, valueType: Expression, mutable: bool, isRef: bool, isPtr: bool]],
parameter: var tuple[name: IdentExpr, valueType: Expression, mutable: bool, isRef: bool, isPtr: bool], parameter: var tuple[name: IdentExpr,
valueType: Expression, mutable: bool,
isRef: bool, isPtr: bool],
defaults: var seq[Expression]) = defaults: var seq[Expression]) =
## Helper to parse declaration arguments and avoid code duplication ## Helper to parse declaration arguments and avoid code duplication
while not self.check(RightParen): while not self.check(RightParen):
@ -843,11 +868,13 @@ proc parseDeclArguments(self: Parser, arguments: var seq[tuple[name: IdentExpr,
self.error(&"missing type declaration for '{argument.name.token.lexeme}' in function declaration") self.error(&"missing type declaration for '{argument.name.token.lexeme}' in function declaration")
proc funDecl(self: Parser, isAsync: bool = false, isGenerator: bool = false, isLambda: bool = false, isOperator: bool = false): Declaration = proc funDecl(self: Parser, isAsync: bool = false, isGenerator: bool = false,
isLambda: bool = false, isOperator: bool = false): Declaration =
## Parses functions, coroutines, generators, anonymous functions and operators ## Parses functions, coroutines, generators, anonymous functions and operators
let tok = self.peek(-1) let tok = self.peek(-1)
var enclosingFunction = self.currentFunction var enclosingFunction = self.currentFunction
var arguments: seq[tuple[name: IdentExpr, valueType: Expression, mutable: bool, isRef: bool, isPtr: bool]] = @[] var arguments: seq[tuple[name: IdentExpr, valueType: Expression,
mutable: bool, isRef: bool, isPtr: bool]] = @[]
var defaults: seq[Expression] = @[] var defaults: seq[Expression] = @[]
var returnType: Expression var returnType: Expression
if not isLambda and self.check(Identifier): if not isLambda and self.check(Identifier):
@ -859,9 +886,12 @@ proc funDecl(self: Parser, isAsync: bool = false, isGenerator: bool = false, isL
# if there's an identifier after the keyword # if there's an identifier after the keyword
self.expect(Identifier, &"expecting identifier after '{tok.lexeme}'") self.expect(Identifier, &"expecting identifier after '{tok.lexeme}'")
self.checkDecl(not self.check("*")) self.checkDecl(not self.check("*"))
self.currentFunction = newFunDecl(nil, arguments, defaults, newBlockStmt(@[], Token()), self.currentFunction = newFunDecl(nil, arguments, defaults, newBlockStmt(@[], Token()),
isAsync=isAsync, isGenerator=isGenerator, isPrivate=true, isAsync = isAsync,
token=tok, pragmas=(@[]), returnType=nil) isGenerator = isGenerator,
isPrivate = true,
token = tok, pragmas = (@[]),
returnType = nil)
FunDecl(self.currentFunction).name = newIdentExpr(self.peek(-1)) FunDecl(self.currentFunction).name = newIdentExpr(self.peek(-1))
if self.match("*"): if self.match("*"):
FunDecl(self.currentFunction).isPrivate = false FunDecl(self.currentFunction).isPrivate = false
@ -878,8 +908,8 @@ proc funDecl(self: Parser, isAsync: bool = false, isGenerator: bool = false, isL
self.currentFunction = enclosingFunction self.currentFunction = enclosingFunction
return result return result
elif isLambda: elif isLambda:
self.currentFunction = newLambdaExpr(arguments, defaults, newBlockStmt(@[], Token()), isGenerator=isGenerator, isAsync=isAsync, token=tok, self.currentFunction = newLambdaExpr(arguments, defaults, newBlockStmt(@[], Token()), isGenerator = isGenerator, isAsync = isAsync, token = tok,
returnType=nil) returnType = nil, pragmas = (@[]))
elif not isOperator: elif not isOperator:
self.error("funDecl: invalid state") self.error("funDecl: invalid state")
if self.match(":"): if self.match(":"):
@ -890,12 +920,20 @@ proc funDecl(self: Parser, isAsync: bool = false, isGenerator: bool = false, isL
# the type declaration for a function lacks # the type declaration for a function lacks
# the braces that would qualify it as an # the braces that would qualify it as an
# expression # expression
var arguments: seq[tuple[name: IdentExpr, valueType: Expression, mutable: bool, isRef: bool, isPtr: bool]] = @[] var arguments: seq[tuple[name: IdentExpr, valueType: Expression,
mutable: bool, isRef: bool, isPtr: bool]] = @[]
var defaults: seq[Expression] = @[] var defaults: seq[Expression] = @[]
returnType = newLambdaExpr(arguments, defaults, nil, isGenerator=self.peek(-1).kind == Generator, returnType = newLambdaExpr(arguments, defaults, nil, isGenerator = self.peek(-1).kind == Generator,
isAsync=self.peek(-1).kind == Coroutine, isAsync = self.peek(
token=self.peek(-1), returnType=nil) -1).kind ==
var parameter: tuple[name: IdentExpr, valueType: Expression, mutable: bool, isRef: bool, isPtr: bool] Coroutine,
token = self.peek(
-1),
returnType = nil,
pragmas = (
@[]))
var parameter: tuple[name: IdentExpr, valueType: Expression,
mutable: bool, isRef: bool, isPtr: bool]
if self.match(LeftParen): if self.match(LeftParen):
self.parseDeclArguments(arguments, parameter, defaults) self.parseDeclArguments(arguments, parameter, defaults)
if self.match(":"): if self.match(":"):
@ -904,17 +942,26 @@ proc funDecl(self: Parser, isAsync: bool = false, isGenerator: bool = false, isL
returnType = self.expression() returnType = self.expression()
if not self.match(LeftBrace): if not self.match(LeftBrace):
self.expect(LeftParen) self.expect(LeftParen)
var parameter: tuple[name: IdentExpr, valueType: Expression, mutable: bool, isRef: bool, isPtr: bool] var parameter: tuple[name: IdentExpr, valueType: Expression,
mutable: bool, isRef: bool, isPtr: bool]
self.parseDeclArguments(arguments, parameter, defaults) self.parseDeclArguments(arguments, parameter, defaults)
if self.match(":"): if self.match(":"):
# Function's return type # Function's return type
if self.match([Function, Coroutine, Generator]): if self.match([Function, Coroutine, Generator]):
var arguments: seq[tuple[name: IdentExpr, valueType: Expression, mutable: bool, isRef: bool, isPtr: bool]] = @[] var arguments: seq[tuple[name: IdentExpr, valueType: Expression,
mutable: bool, isRef: bool, isPtr: bool]] = @[]
var defaults: seq[Expression] = @[] var defaults: seq[Expression] = @[]
returnType = newLambdaExpr(arguments, defaults, nil, isGenerator=self.peek(-1).kind == Generator, returnType = newLambdaExpr(arguments, defaults, nil, isGenerator = self.peek(-1).kind == Generator,
isAsync=self.peek(-1).kind == Coroutine, isAsync = self.peek(
token=self.peek(-1), returnType=nil) -1).kind ==
var parameter: tuple[name: IdentExpr, valueType: Expression, mutable: bool, isRef: bool, isPtr: bool] Coroutine,
token = self.peek(
-1),
returnType = nil,
pragmas = (
@[]))
var parameter: tuple[name: IdentExpr, valueType: Expression,
mutable: bool, isRef: bool, isPtr: bool]
if self.match(LeftParen): if self.match(LeftParen):
self.parseDeclArguments(arguments, parameter, defaults) self.parseDeclArguments(arguments, parameter, defaults)
if self.match(":"): if self.match(":"):
@ -924,7 +971,7 @@ proc funDecl(self: Parser, isAsync: bool = false, isGenerator: bool = false, isL
self.expect(LeftBrace) self.expect(LeftBrace)
if self.currentFunction.kind == funDecl: if self.currentFunction.kind == funDecl:
if not self.match(Semicolon): if not self.match(Semicolon):
# If we don't find a semicolon, # If we don't find a semicolon,
# it's not a forward declaration # it's not a forward declaration
FunDecl(self.currentFunction).body = self.blockStmt() FunDecl(self.currentFunction).body = self.blockStmt()
else: else:
@ -946,13 +993,13 @@ proc funDecl(self: Parser, isAsync: bool = false, isGenerator: bool = false, isL
self.error("operators must have a return type") self.error("operators must have a return type")
for argument in arguments: for argument in arguments:
if argument.valueType == nil: if argument.valueType == nil:
self.error(&"missing type declaration for '{argument.name.token.lexeme}' in function declaration") self.error(&"missing type declaration for '{argument.name.token.lexeme}' in function declaration")
self.currentFunction = enclosingFunction self.currentFunction = enclosingFunction
proc expression(self: Parser): Expression = proc expression(self: Parser): Expression =
## Parses expressions ## Parses expressions
result = self.parseArrow() # Highest-level expression result = self.parseArrow() # Highest-level expression
proc expressionStatement(self: Parser): Statement = proc expressionStatement(self: Parser): Statement =
@ -991,7 +1038,7 @@ proc statement(self: Parser): Statement =
# TODO # TODO
# from module import a [, b, c as d] # from module import a [, b, c as d]
discard self.step() discard self.step()
result = self.importStmt(fromStmt=true) result = self.importStmt(fromStmt = true)
of While: of While:
discard self.step() discard self.step()
result = self.whileStmt() result = self.whileStmt()
@ -1020,33 +1067,53 @@ proc statement(self: Parser): Statement =
result = self.expressionStatement() result = self.expressionStatement()
proc parsePragma(self: Parser): Pragma =
## Parses pragmas
if self.scopeDepth == 0:
## Pragmas used at the
## top level are either
## used for compile-time
## switches or for variable
## declarations
var decl: VarDecl
for node in self.tree:
if node.token.line == self.peek(-1).line and node.kind == varDecl:
decl = VarDecl(node)
break
else:
var decl = self.currentFunction
proc declaration(self: Parser): Declaration = proc declaration(self: Parser): Declaration =
## Parses declarations ## Parses declarations
case self.peek().kind: case self.peek().kind:
of Var, Const, Let: of Var, Const, Let:
let keyword = self.step() let keyword = self.step()
result = self.varDecl(isLet=keyword.kind == Let, isConst=keyword.kind == Const) result = self.varDecl(isLet = keyword.kind == Let,
isConst = keyword.kind == Const)
of Function: of Function:
discard self.step() discard self.step()
result = self.funDecl() result = self.funDecl()
of Coroutine: of Coroutine:
discard self.step() discard self.step()
result = self.funDecl(isAsync=true) result = self.funDecl(isAsync = true)
of Generator: of Generator:
discard self.step() discard self.step()
result = self.funDecl(isGenerator=true) result = self.funDecl(isGenerator = true)
of Operator: of Operator:
discard self.step() discard self.step()
result = self.funDecl(isOperator=true) result = self.funDecl(isOperator = true)
of Type, TokenType.Whitespace, TokenType.Tab, Comment: of TokenType.Comment:
# TODO: Comments, pragmas, docstrings let tok = self.step()
discard self.step() # TODO if tok.lexeme.startsWith("#pragma["):
return newNilExpr(Token(lexeme: "nil")) result = self.parsePragma()
of Type, TokenType.Whitespace, TokenType.Tab:
discard self.step() # TODO
else: else:
result = Declaration(self.statement()) result = Declaration(self.statement())
proc parse*(self: Parser, tokens: seq[Token], file: string): seq[ASTNode] = proc parse*(self: Parser, tokens: seq[Token], file: string): seq[Declaration] =
## Parses a series of tokens into an AST node ## Parses a series of tokens into an AST node
self.tokens = tokens self.tokens = tokens
self.file = file self.file = file
@ -1055,10 +1122,11 @@ proc parse*(self: Parser, tokens: seq[Token], file: string): seq[ASTNode] =
self.currentFunction = nil self.currentFunction = nil
self.scopeDepth = 0 self.scopeDepth = 0
self.operators = newOperatorTable() self.operators = newOperatorTable()
self.tree = @[]
for i, token in self.tokens: for i, token in self.tokens:
# We do a first pass over the tokens # We do a first pass over the tokens
# to find operators. Note that this # to find operators. Note that this
# relies on the lexer ending the input # relies on the lexer ending the input
# with an EOF token # with an EOF token
if token.kind == Operator: if token.kind == Operator:
if i == self.tokens.high(): if i == self.tokens.high():
@ -1069,4 +1137,7 @@ proc parse*(self: Parser, tokens: seq[Token], file: string): seq[ASTNode] =
# well perform some extra checks # well perform some extra checks
self.error("invalid state: found malformed tokenizer input while looking for operators (missing EOF)") self.error("invalid state: found malformed tokenizer input while looking for operators (missing EOF)")
while not self.done(): while not self.done():
result.add(self.declaration()) self.tree.add(self.declaration())
if self.tree[^1] == nil:
self.tree.delete(self.tree.high())
result = self.tree

8
src/memory/allocator.nim
View File

@ -51,12 +51,14 @@ proc reallocate*(p: pointer, oldSize: int, newSize: int): pointer =
echo &"DEBUG - Memory manager: Warning, asked to realloc() nil pointer from {oldSize} to {newSize} bytes, ignoring request" echo &"DEBUG - Memory manager: Warning, asked to realloc() nil pointer from {oldSize} to {newSize} bytes, ignoring request"
except NilAccessDefect: except NilAccessDefect:
stderr.write("JAPL: could not manage memory, segmentation fault\n") stderr.write("JAPL: could not manage memory, segmentation fault\n")
quit(139) # For now, there's not much we can do if we can't get the memory we need, so we exit quit(139) # For now, there's not much we can do if we can't get the memory we need, so we exit
template resizeArray*(kind: untyped, pointr: pointer, oldCount, newCount: int): untyped = template resizeArray*(kind: untyped, pointr: pointer, oldCount,
newCount: int): untyped =
## Handy macro (in the C sense of macro, not nim's) to resize a dynamic array ## Handy macro (in the C sense of macro, not nim's) to resize a dynamic array
cast[ptr UncheckedArray[kind]](reallocate(pointr, sizeof(kind) * oldCount, sizeof(kind) * newCount)) cast[ptr UncheckedArray[kind]](reallocate(pointr, sizeof(kind) * oldCount,
sizeof(kind) * newCount))
template freeArray*(kind: untyped, pointr: pointer, oldCount: int): untyped = template freeArray*(kind: untyped, pointr: pointer, oldCount: int): untyped =

17
src/test.nim
View File

@ -42,7 +42,7 @@ when isMainModule:
var var
keep = true keep = true
tokens: seq[Token] = @[] tokens: seq[Token] = @[]
tree: seq[ASTNode] = @[] tree: seq[Declaration] = @[]
compiled: Chunk compiled: Chunk
serialized: Serialized serialized: Serialized
serializedRaw: seq[byte] serializedRaw: seq[byte]
@ -66,7 +66,8 @@ when isMainModule:
if input.len() == 0: if input.len() == 0:
continue continue
# Currently the parser doesn't handle these tokens well # Currently the parser doesn't handle these tokens well
tokens = filter(tokenizer.lex(input, "stdin"), proc (x: Token): bool = x.kind notin {TokenType.Whitespace, Tab}) tokens = filter(tokenizer.lex(input, "stdin"), proc (
x: Token): bool = x.kind notin {TokenType.Whitespace, Tab})
if tokens.len() == 0: if tokens.len() == 0:
continue continue
when debugLexer: when debugLexer:
@ -157,7 +158,7 @@ proc fillSymbolTable(tokenizer: Lexer) =
## Initializes the Lexer's symbol ## Initializes the Lexer's symbol
## table with the builtin symbols ## table with the builtin symbols
## and keywords ## and keywords
# 1-byte symbols # 1-byte symbols
tokenizer.symbols.addSymbol("{", LeftBrace) tokenizer.symbols.addSymbol("{", LeftBrace)
tokenizer.symbols.addSymbol("}", RightBrace) tokenizer.symbols.addSymbol("}", RightBrace)
@ -196,10 +197,10 @@ proc fillSymbolTable(tokenizer: Lexer) =
tokenizer.symbols.addKeyword("import", Import) tokenizer.symbols.addKeyword("import", Import)
tokenizer.symbols.addKeyword("yield", TokenType.Yield) tokenizer.symbols.addKeyword("yield", TokenType.Yield)
tokenizer.symbols.addKeyword("return", TokenType.Return) tokenizer.symbols.addKeyword("return", TokenType.Return)
# These are more like expressions with a reserved # These are more like expressions with a reserved
# name that produce a value of a builtin type, # name that produce a value of a builtin type,
# but we don't need to care about that until # but we don't need to care about that until
# we're in the parsing/ compilation steps so # we're in the parsing/ compilation steps so
# it's fine # it's fine
tokenizer.symbols.addKeyword("nan", NotANumber) tokenizer.symbols.addKeyword("nan", NotANumber)
tokenizer.symbols.addKeyword("inf", Infinity) tokenizer.symbols.addKeyword("inf", Infinity)
@ -217,4 +218,4 @@ proc getLineEditor: LineEditor =
result.prompt = "=> " result.prompt = "=> "
result.populateDefaults() result.populateDefaults()
let history = result.plugHistory() let history = result.plugHistory()
result.bindHistory(history) result.bindHistory(history)

25
src/util/debugger.nim
View File

@ -52,9 +52,11 @@ proc simpleInstruction(instruction: OpCode, offset: int): int =
return offset + 1 return offset + 1
proc stackTripleInstruction(instruction: OpCode, chunk: Chunk, offset: int): int = proc stackTripleInstruction(instruction: OpCode, chunk: Chunk,
offset: int): int =
## Debugs instructions that operate on a single value on the stack using a 24-bit operand ## Debugs instructions that operate on a single value on the stack using a 24-bit operand
var slot = [chunk.code[offset + 1], chunk.code[offset + 2], chunk.code[offset + 3]].fromTriple() var slot = [chunk.code[offset + 1], chunk.code[offset + 2], chunk.code[
offset + 3]].fromTriple()
printInstruction(instruction) printInstruction(instruction)
stdout.write(&", points to index ") stdout.write(&", points to index ")
setForegroundColor(fgYellow) setForegroundColor(fgYellow)
@ -63,7 +65,8 @@ proc stackTripleInstruction(instruction: OpCode, chunk: Chunk, offset: int): int
return offset + 4 return offset + 4
proc stackDoubleInstruction(instruction: OpCode, chunk: Chunk, offset: int): int = proc stackDoubleInstruction(instruction: OpCode, chunk: Chunk,
offset: int): int =
## Debugs instructions that operate on a single value on the stack using a 16-bit operand ## Debugs instructions that operate on a single value on the stack using a 16-bit operand
var slot = [chunk.code[offset + 1], chunk.code[offset + 2]].fromDouble() var slot = [chunk.code[offset + 1], chunk.code[offset + 2]].fromDouble()
printInstruction(instruction) printInstruction(instruction)
@ -74,7 +77,8 @@ proc stackDoubleInstruction(instruction: OpCode, chunk: Chunk, offset: int): int
return offset + 3 return offset + 3
proc argumentDoubleInstruction(instruction: OpCode, chunk: Chunk, offset: int): int = proc argumentDoubleInstruction(instruction: OpCode, chunk: Chunk,
offset: int): int =
## Debugs instructions that operate on a hardcoded value on the stack using a 16-bit operand ## Debugs instructions that operate on a hardcoded value on the stack using a 16-bit operand
var slot = [chunk.code[offset + 1], chunk.code[offset + 2]].fromDouble() var slot = [chunk.code[offset + 1], chunk.code[offset + 2]].fromDouble()
printInstruction(instruction) printInstruction(instruction)
@ -87,7 +91,8 @@ proc argumentDoubleInstruction(instruction: OpCode, chunk: Chunk, offset: int):
proc constantInstruction(instruction: OpCode, chunk: Chunk, offset: int): int = proc constantInstruction(instruction: OpCode, chunk: Chunk, offset: int): int =
## Debugs instructions that operate on the constant table ## Debugs instructions that operate on the constant table
var constant = [chunk.code[offset + 1], chunk.code[offset + 2], chunk.code[offset + 3]].fromTriple() var constant = [chunk.code[offset + 1], chunk.code[offset + 2], chunk.code[
offset + 3]].fromTriple()
printInstruction(instruction) printInstruction(instruction)
stdout.write(&", points to constant at position ") stdout.write(&", points to constant at position ")
setForegroundColor(fgYellow) setForegroundColor(fgYellow)
@ -95,7 +100,7 @@ proc constantInstruction(instruction: OpCode, chunk: Chunk, offset: int): int =
nl() nl()
let obj = chunk.consts[constant] let obj = chunk.consts[constant]
setForegroundColor(fgGreen) setForegroundColor(fgGreen)
printDebug("Operand: ") printDebug("Operand: ")
setForegroundColor(fgYellow) setForegroundColor(fgYellow)
stdout.write(&"{obj}\n") stdout.write(&"{obj}\n")
setForegroundColor(fgGreen) setForegroundColor(fgGreen)
@ -111,10 +116,12 @@ proc jumpInstruction(instruction: OpCode, chunk: Chunk, offset: int): int =
case instruction: case instruction:
of Jump, JumpIfFalse, JumpIfTrue, JumpIfFalsePop, JumpForwards, JumpBackwards: of Jump, JumpIfFalse, JumpIfTrue, JumpIfFalsePop, JumpForwards, JumpBackwards:
jump = [chunk.code[offset + 1], chunk.code[offset + 2]].fromDouble().int() jump = [chunk.code[offset + 1], chunk.code[offset + 2]].fromDouble().int()
of LongJump, LongJumpIfFalse, LongJumpIfTrue, LongJumpIfFalsePop, LongJumpForwards, LongJumpBackwards: of LongJump, LongJumpIfFalse, LongJumpIfTrue, LongJumpIfFalsePop,
jump = [chunk.code[offset + 1], chunk.code[offset + 2], chunk.code[offset + 3]].fromTriple().int() LongJumpForwards, LongJumpBackwards:
jump = [chunk.code[offset + 1], chunk.code[offset + 2], chunk.code[
offset + 3]].fromTriple().int()
else: else:
discard # Unreachable discard # Unreachable
printInstruction(instruction, true) printInstruction(instruction, true)
printDebug("Jump size: ") printDebug("Jump size: ")
setForegroundColor(fgYellow) setForegroundColor(fgYellow)

22
src/util/serializer.nim
View File

@ -33,7 +33,7 @@ type
filename: string filename: string
chunk: Chunk chunk: Chunk
Serialized* = ref object Serialized* = ref object
## Wrapper returned by ## Wrapper returned by
## the Serializer.read* ## the Serializer.read*
## procedures to store ## procedures to store
## metadata ## metadata
@ -51,7 +51,8 @@ proc `$`*(self: Serialized): string =
proc error(self: Serializer, message: string) = proc error(self: Serializer, message: string) =
## Raises a formatted SerializationError exception ## Raises a formatted SerializationError exception
raise newException(SerializationError, &"A fatal error occurred while (de)serializing '{self.filename}' -> {message}") raise newException(SerializationError,
&"A fatal error occurred while (de)serializing '{self.filename}' -> {message}")
proc newSerializer*(self: Serializer = nil): Serializer = proc newSerializer*(self: Serializer = nil): Serializer =
@ -98,7 +99,7 @@ proc extend[T](s: var seq[T], a: openarray[T]) =
s.add(e) s.add(e)
proc writeHeaders(self: Serializer, stream: var seq[byte], file: string) = proc writeHeaders(self: Serializer, stream: var seq[byte], file: string) =
## Writes the Peon bytecode headers in-place into a byte stream ## Writes the Peon bytecode headers in-place into a byte stream
stream.extend(self.toBytes(BYTECODE_MARKER)) stream.extend(self.toBytes(BYTECODE_MARKER))
stream.add(byte(PEON_VERSION.major)) stream.add(byte(PEON_VERSION.major))
@ -138,7 +139,7 @@ proc writeConstants(self: Serializer, stream: var seq[byte]) =
else: else:
strip = 2 strip = 2
temp = 0x0 temp = 0x0
stream.extend((len(constant.token.lexeme) - strip).toTriple()) # Removes the quotes from the length count as they're not written stream.extend((len(constant.token.lexeme) - strip).toTriple()) # Removes the quotes from the length count as they're not written
stream.add(temp) stream.add(temp)
stream.add(self.toBytes(constant.token.lexeme[offset..^2])) stream.add(self.toBytes(constant.token.lexeme[offset..^2]))
of identExpr: of identExpr:
@ -147,7 +148,7 @@ proc writeConstants(self: Serializer, stream: var seq[byte]) =
stream.add(self.toBytes(constant.token.lexeme)) stream.add(self.toBytes(constant.token.lexeme))
else: else:
self.error(&"unknown constant kind in chunk table ({constant.kind})") self.error(&"unknown constant kind in chunk table ({constant.kind})")
stream.add(0x59) # End marker stream.add(0x59) # End marker
proc readConstants(self: Serializer, stream: seq[byte]): int = proc readConstants(self: Serializer, stream: seq[byte]): int =
@ -204,7 +205,8 @@ proc readConstants(self: Serializer, stream: seq[byte]): int =
stream = stream[1..^1] stream = stream[1..^1]
let size = self.bytesToInt([stream[0], stream[1], stream[2]]) let size = self.bytesToInt([stream[0], stream[1], stream[2]])
stream = stream[3..^1] stream = stream[3..^1]
self.chunk.consts.add(newIdentExpr(Token(lexeme: self.bytesToString(stream[0..<size])))) self.chunk.consts.add(newIdentExpr(Token(
lexeme: self.bytesToString(stream[0..<size]))))
# TODO # TODO
# discard self.chunk.addConstant(newIdentExpr(Token(lexeme: self.bytesToString(stream[0..<size])))) # discard self.chunk.addConstant(newIdentExpr(Token(lexeme: self.bytesToString(stream[0..<size]))))
stream = stream[size..^1] stream = stream[size..^1]
@ -263,7 +265,8 @@ proc loadBytes*(self: Serializer, stream: seq[byte]): Serialized =
if stream[0..<len(BYTECODE_MARKER)] != self.toBytes(BYTECODE_MARKER): if stream[0..<len(BYTECODE_MARKER)] != self.toBytes(BYTECODE_MARKER):
self.error("malformed bytecode marker") self.error("malformed bytecode marker")
stream = stream[len(BYTECODE_MARKER)..^1] stream = stream[len(BYTECODE_MARKER)..^1]
result.peonVer = (major: int(stream[0]), minor: int(stream[1]), patch: int(stream[2])) result.peonVer = (major: int(stream[0]), minor: int(stream[1]),
patch: int(stream[2]))
stream = stream[3..^1] stream = stream[3..^1]
let branchLength = stream[0] let branchLength = stream[0]
stream = stream[1..^1] stream = stream[1..^1]
@ -271,7 +274,8 @@ proc loadBytes*(self: Serializer, stream: seq[byte]): Serialized =
stream = stream[branchLength..^1] stream = stream[branchLength..^1]
result.commitHash = self.bytesToString(stream[0..<40]).toLowerAscii() result.commitHash = self.bytesToString(stream[0..<40]).toLowerAscii()
stream = stream[40..^1] stream = stream[40..^1]
result.compileDate = self.bytesToInt([stream[0], stream[1], stream[2], stream[3], stream[4], stream[5], stream[6], stream[7]]) result.compileDate = self.bytesToInt([stream[0], stream[1], stream[2],
stream[3], stream[4], stream[5], stream[6], stream[7]])
stream = stream[8..^1] stream = stream[8..^1]
result.fileHash = self.bytesToString(stream[0..<32]).toHex().toLowerAscii() result.fileHash = self.bytesToString(stream[0..<32]).toHex().toLowerAscii()
stream = stream[32..^1] stream = stream[32..^1]
@ -295,4 +299,4 @@ proc loadFile*(self: Serializer, src: string): Serialized =
while pos < size: while pos < size:
discard fp.readBytes(data, pos, size) discard fp.readBytes(data, pos, size)
pos = fp.getFilePos() pos = fp.getFilePos()
return self.loadBytes(data) return self.loadBytes(data)