nocturn9x
/
peon
2
2
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Added Makefile, prettyfied code, initial work on pragmas

This commit is contained in:
Mattia Giambirtone 2022-05-18 13:32:32 +02:00
parent e8cbec94bd
commit cb21af4aa3
14 changed files with 448 additions and 317 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
Makefile Normal file
View File

@ -0,0 +1,5 @@
run:
nim --hints:off --warnings:off r src/test.nim
pretty:
nimpretty src/*.nim src/backend/*.nim src/frontend/*.nim src/frontend/meta/*.nim src/memory/*.nim src/util/*.nim

5
src/backend/types.nim
View File

@ -13,7 +13,7 @@
# limitations under the License.
type
type
ObjectKind* = enum
## Enumeration of Peon
## types
@ -51,5 +51,4 @@ type
of CustomType:
fields*: seq[PeonObject]
else:
discard # TODO
discard # TODO

16
src/backend/vm.nim
View File

@ -18,14 +18,14 @@ import ../config
import ../frontend/meta/bytecode
type
type
PeonVM* = ref object
## The Peon Virtual Machine
stack: seq[PeonObject]
ip: int # Instruction pointer
sp: int # Stack pointer
cache: array[6, PeonObject] # Singletons cache
chunk: Chunk # Piece of bytecode to execute
ip: int # Instruction pointer
sp: int # Stack pointer
cache: array[6, PeonObject] # Singletons cache
chunk: Chunk # Piece of bytecode to execute
proc initCache*(self: PeonVM) =
@ -126,7 +126,8 @@ proc readInt64(self: PeonVM, idx: int): PeonObject =
## chunk's constant table and
## returns a Peon object. Assumes
## the constant is an Int64
var arr = [self.chunk.byteConsts[idx], self.chunk.byteConsts[idx + 1], self.chunk.byteConsts[idx + 2], self.chunk.byteConsts[idx + 3]]
var arr = [self.chunk.byteConsts[idx], self.chunk.byteConsts[idx + 1],
self.chunk.byteConsts[idx + 2], self.chunk.byteConsts[idx + 3]]
result = PeonObject(kind: Int64)
copyMem(result.long.addr, arr.addr, sizeof(arr))
@ -136,7 +137,8 @@ proc readUInt64(self: PeonVM, idx: int): PeonObject =
## chunk's constant table and
## returns a Peon object. Assumes
## the constant is an UInt64
var arr = [self.chunk.byteConsts[idx], self.chunk.byteConsts[idx + 1], self.chunk.byteConsts[idx + 2], self.chunk.byteConsts[idx + 3]]
var arr = [self.chunk.byteConsts[idx], self.chunk.byteConsts[idx + 1],
self.chunk.byteConsts[idx + 2], self.chunk.byteConsts[idx + 3]]
result = PeonObject(kind: UInt64)
copyMem(result.uLong.addr, arr.addr, sizeof(arr))

16
src/config.nim
View File

@ -15,13 +15,13 @@
import strformat
const BYTECODE_MARKER* = "PEON_BYTECODE"
const MAP_LOAD_FACTOR* = 0.75 # Load factor for builtin hashmaps
const MAP_LOAD_FACTOR* = 0.75 # Load factor for builtin hashmaps
when MAP_LOAD_FACTOR >= 1.0:
{.fatal: "Hashmap load factor must be < 1".}
const HEAP_GROW_FACTOR* = 2 # How much extra memory to allocate for dynamic arrays and garbage collection when resizing
const HEAP_GROW_FACTOR* = 2 # How much extra memory to allocate for dynamic arrays and garbage collection when resizing
when HEAP_GROW_FACTOR <= 1:
{.fatal: "Heap growth factor must be > 1".}
const MAX_STACK_FRAMES* = 800 # The maximum number of stack frames at any one time. Acts as a recursion limiter (1 frame = 1 call)
const MAX_STACK_FRAMES* = 800 # The maximum number of stack frames at any one time. Acts as a recursion limiter (1 frame = 1 call)
when MAX_STACK_FRAMES <= 0:
{.fatal: "The frame limit must be > 0".}
const PEON_VERSION* = (major: 0, minor: 4, patch: 0)
@ -32,11 +32,11 @@ when len(PEON_COMMIT_HASH) != 40:
const PEON_BRANCH* = "master"
when len(PEON_BRANCH) >= 255:
{.fatal: "The git branch name's length must be less than or equal to 255 characters".}
const DEBUG_TRACE_VM* = false # Traces VM execution
const SKIP_STDLIB_INIT* = false # Skips stdlib initialization (can be imported manually)
const DEBUG_TRACE_GC* = false # Traces the garbage collector (TODO)
const DEBUG_TRACE_ALLOCATION* = false # Traces memory allocation/deallocation
const DEBUG_TRACE_COMPILER* = false # Traces the compiler
const DEBUG_TRACE_VM* = false # Traces VM execution
const SKIP_STDLIB_INIT* = false # Skips stdlib initialization (can be imported manually)
const DEBUG_TRACE_GC* = false # Traces the garbage collector (TODO)
const DEBUG_TRACE_ALLOCATION* = false # Traces memory allocation/deallocation
const DEBUG_TRACE_COMPILER* = false # Traces the compiler
const INITIAL_STACK_SIZE* = 0
const PEON_VERSION_STRING* = &"Peon {PEON_VERSION.major}.{PEON_VERSION.minor}.{PEON_VERSION.patch} {PEON_RELEASE} ({PEON_BRANCH}, {CompileDate}, {CompileTime}, {PEON_COMMIT_HASH[0..8]}) [Nim {NimVersion}] on {hostOS} ({hostCPU})"
const HELP_MESSAGE* = """The peon programming language, Copyright (C) 2022 Mattia Giambirtone & All Contributors

124
src/frontend/compiler.nim
View File

@ -96,7 +96,7 @@ type
# The bytecode chunk where we write code to
chunk: Chunk
# The output of our parser (AST)
ast: seq[ASTNode]
ast: seq[Declaration]
# The current AST node we're looking at
current: int
# The current file being compiled (used only for
@ -185,7 +185,8 @@ proc done(self: Compiler): bool =
result = self.current > self.ast.high()
proc error(self: Compiler, message: string) {.raises: [CompileError, ValueError].} =
proc error(self: Compiler, message: string) {.raises: [CompileError,
ValueError].} =
## Raises a formatted CompileError exception
var tok = self.getCurrentNode().token
raise newException(CompileError, &"A fatal error occurred while compiling '{self.file}', module '{self.currentModule}' line {tok.line} at '{tok.lexeme}' -> {message}")
@ -243,7 +244,7 @@ proc emitConstant(self: Compiler, obj: Expression, kind: Type) =
of Int64:
self.emitByte(LoadInt64)
else:
discard # TODO
discard # TODO
self.emitBytes(self.makeConstant(obj, kind))
@ -261,7 +262,7 @@ proc emitJump(self: Compiler, opcode: OpCode): int =
proc patchJump(self: Compiler, offset: int) =
## Patches a previously emitted relative
## Patches a previously emitted relative
## jump using emitJump. Since emitJump assumes
## a long jump, this also shrinks the jump
## offset and changes the bytecode instruction if possible
@ -284,7 +285,7 @@ proc patchJump(self: Compiler, offset: int) =
self.chunk.code[offset] = JumpIfFalseOrPop.uint8()
else:
discard
self.chunk.code.delete(offset + 1) # Discards the first 8 bits of the jump offset (which are empty)
self.chunk.code.delete(offset + 1) # Discards the first 8 bits of the jump offset (which are empty)
let offsetArray = (jump - 1).toDouble() # -1 since we got rid of 1 byte!
self.chunk.code[offset + 1] = offsetArray[0]
self.chunk.code[offset + 2] = offsetArray[1]
@ -319,17 +320,18 @@ proc resolve(self: Compiler, name: IdentExpr,
for obj in reversed(self.names):
if obj.name.token.lexeme == name.token.lexeme:
if obj.isPrivate and obj.owner != self.currentModule:
continue # There may be a name in the current module that
continue # There may be a name in the current module that
# matches, so we skip this
return obj
return nil
proc getStackPos(self: Compiler, name: IdentExpr, depth: int = self.scopeDepth): tuple[closedOver: bool, pos: int] =
proc getStackPos(self: Compiler, name: IdentExpr,
depth: int = self.scopeDepth): tuple[closedOver: bool, pos: int] =
## Iterates the internal list of declared names backwards and
## returns a tuple (closedOver, pos) that tells the caller whether the
## the name is to be emitted as a closure as well as its predicted
## stack/closure array position. Returns (false, -1) if the variable's
## stack/closure array position. Returns (false, -1) if the variable's
## location can not be determined at compile time (this is an error!).
## Note that private names declared in other modules will not be resolved!
var i: int = self.names.high()
@ -348,14 +350,15 @@ proc getStackPos(self: Compiler, name: IdentExpr, depth: int = self.scopeDepth):
return (false, -1)
proc detectClosureVariable(self: Compiler, name: IdentExpr, depth: int = self.scopeDepth) =
proc detectClosureVariable(self: Compiler, name: IdentExpr,
depth: int = self.scopeDepth) =
## Detects if the given name is used in a local scope deeper
## than the given one and modifies the code emitted for it
## to store it as a closure variable if it is. Does nothing if the name
## hasn't been declared yet or is unreachable (for example if it's
## declared as private in another module). This function must be called
## each time a name is referenced in order for closed-over variables
## to be emitted properly, otherwise the runtime may behave
## each time a name is referenced in order for closed-over variables
## to be emitted properly, otherwise the runtime may behave
## unpredictably or crash
let entry = self.resolve(name)
if entry == nil:
@ -392,14 +395,15 @@ proc compareTypes(self: Compiler, a, b: Type): bool =
Char, Byte, String, Nil, Nan, Bool, Inf:
return true
of Function:
let
let
a = FunDecl(a.node)
b = FunDecl(b.node)
if a.name.token.lexeme != b.name.token.lexeme:
return false
elif a.arguments.len() != b.arguments.len():
return false
elif not self.compareTypes(self.inferType(a.returnType), self.inferType(b.returnType)):
elif not self.compareTypes(self.inferType(a.returnType),
self.inferType(b.returnType)):
return false
for (argA, argB) in zip(a.arguments, b.arguments):
if argA.mutable != argB.mutable:
@ -408,7 +412,8 @@ proc compareTypes(self: Compiler, a, b: Type): bool =
return false
elif argA.isPtr != argB.isPtr:
return false
elif not self.compareTypes(self.inferType(argA.valueType), self.inferType(argB.valueType)):
elif not self.compareTypes(self.inferType(argA.valueType),
self.inferType(argB.valueType)):
return false
return true
else:
@ -416,9 +421,9 @@ proc compareTypes(self: Compiler, a, b: Type): bool =
proc toIntrinsic(name: string): Type =
## Converts a string to an intrinsic
## Converts a string to an intrinsic
## type if it is valid and returns nil
## otherwise
## otherwise
if name in ["int", "int64", "i64"]:
return Type(kind: Int64)
elif name in ["uint64", "u64"]:
@ -493,11 +498,11 @@ proc inferType(self: Compiler, node: LiteralExpr): Type =
of infExpr:
return Type(node: node, kind: TypeKind.Inf)
else:
discard # TODO
discard # TODO
proc toIntrinsic(self: Compiler, typ: Expression): Type =
## Gets an expression's
proc toIntrinsic(self: Compiler, typ: Expression): Type =
## Gets an expression's
## intrinsic type, if possible
if typ == nil:
return nil
@ -535,13 +540,13 @@ proc inferType(self: Compiler, node: Expression): Type =
if not self.compareTypes(a, b):
return nil
return a
of {intExpr, hexExpr, binExpr, octExpr,
strExpr, falseExpr, trueExpr, infExpr,
of {intExpr, hexExpr, binExpr, octExpr,
strExpr, falseExpr, trueExpr, infExpr,
nanExpr, floatExpr, nilExpr
}:
return self.inferType(LiteralExpr(node))
else:
discard # Unreachable
discard # Unreachable
proc typeToStr(self: Compiler, typ: Type): string =
@ -550,7 +555,7 @@ proc typeToStr(self: Compiler, typ: Type): string =
case typ.kind:
of Int8, UInt8, Int16, UInt16, Int32,
UInt32, Int64, UInt64, Float32, Float64,
Char, Byte, String, Nil, TypeKind.Nan, Bool,
Char, Byte, String, Nil, TypeKind.Nan, Bool,
TypeKind.Inf:
return ($typ.kind).toLowerAscii()
of Function:
@ -571,11 +576,11 @@ proc typeToStr(self: Compiler, typ: Type): string =
result &= ", "
result &= ")"
else:
discard # Unreachable
discard # Unreachable
result &= &": {self.typeToStr(typ.returnType)}"
else:
discard
proc inferType(self: Compiler, node: Declaration): Type =
## Infers the type of a given declaration
@ -596,7 +601,7 @@ proc inferType(self: Compiler, node: Declaration): Type =
else:
return self.inferType(node.value)
else:
return # Unreachable
return # Unreachable
## End of utility functions
@ -634,11 +639,11 @@ proc literal(self: Compiler, node: ASTNode) =
discard parseHex(y.literal.lexeme, x)
except ValueError:
self.error("integer value out of range")
let node = newIntExpr(Token(lexeme: $x, line: y.token.line,
pos: (start: y.token.pos.start,
let node = newIntExpr(Token(lexeme: $x, line: y.token.line,
pos: (start: y.token.pos.start,
stop: y.token.pos.start + len($x))
)
)
)
)
self.emitConstant(node, Type(kind: Int64))
of binExpr:
var x: int
@ -647,11 +652,11 @@ proc literal(self: Compiler, node: ASTNode) =
discard parseBin(y.literal.lexeme, x)
except ValueError:
self.error("integer value out of range")
let node = newIntExpr(Token(lexeme: $x, line: y.token.line,
pos: (start: y.token.pos.start,
let node = newIntExpr(Token(lexeme: $x, line: y.token.line,
pos: (start: y.token.pos.start,
stop: y.token.pos.start + len($x))
)
)
)
)
self.emitConstant(node, Type(kind: Int64))
of octExpr:
var x: int
@ -660,11 +665,11 @@ proc literal(self: Compiler, node: ASTNode) =
discard parseOct(y.literal.lexeme, x)
except ValueError:
self.error("integer value out of range")
let node = newIntExpr(Token(lexeme: $x, line: y.token.line,
pos: (start: y.token.pos.start,
let node = newIntExpr(Token(lexeme: $x, line: y.token.line,
pos: (start: y.token.pos.start,
stop: y.token.pos.start + len($x))
)
)
)
)
self.emitConstant(node, Type(kind: Int64))
of floatExpr:
var x: float
@ -685,7 +690,7 @@ proc literal(self: Compiler, node: ASTNode) =
proc unary(self: Compiler, node: UnaryExpr) =
## Compiles unary expressions such as decimal
## and bitwise negation
self.expression(node.a) # Pushes the operand onto the stack
self.expression(node.a) # Pushes the operand onto the stack
# TODO: Find implementation of
# the given operator and call it
@ -731,12 +736,13 @@ proc declareName(self: Compiler, node: Declaration) =
# If someone ever hits this limit in real-world scenarios, I swear I'll
# slap myself 100 times with a sign saying "I'm dumb". Mark my words
self.error("cannot declare more than 16777216 variables at a time")
self.names.add(Name(depth: self.scopeDepth,
self.names.add(Name(depth: self.scopeDepth,
name: node.name,
isPrivate: node.isPrivate,
owner: self.currentModule,
isConst: node.isConst,
valueType: Type(kind: self.inferType(node.value).kind, node: node),
valueType: Type(kind: self.inferType(
node.value).kind, node: node),
codePos: self.chunk.code.len(),
isLet: node.isLet))
self.emitByte(StoreVar)
@ -754,18 +760,20 @@ proc declareName(self: Compiler, node: Declaration) =
isPrivate: node.isPrivate,
isConst: false,
owner: self.currentModule,
valueType: Type(kind: Function, node: node, returnType: self.inferType(node.returnType)),
valueType: Type(kind: Function, node: node,
returnType: self.inferType(
node.returnType)),
codePos: self.chunk.code.len(),
name: node.name,
isLet: false))
for argument in node.arguments:
if self.names.high() > 16777215:
self.error("cannot declare more than 16777216 variables at a time")
self.names.add(Name(depth: self.scopeDepth + 1,
isPrivate: true,
owner: self.currentModule,
isConst: false,
name: argument.name,
self.names.add(Name(depth: self.scopeDepth + 1,
isPrivate: true,
owner: self.currentModule,
isConst: false,
name: argument.name,
valueType: nil,
codePos: self.chunk.code.len(),
isLet: false))
@ -774,8 +782,8 @@ proc declareName(self: Compiler, node: Declaration) =
self.emitByte(LoadVar)
self.emitBytes(self.names.high().toTriple())
else:
discard # Unreachable
discard # Unreachable
proc identifier(self: Compiler, node: IdentExpr) =
## Compiles access to identifiers
@ -792,7 +800,7 @@ proc identifier(self: Compiler, node: IdentExpr) =
self.detectClosureVariable(s.name)
let t = self.getStackPos(node)
let index = t.pos
# We don't check if index is -1 because if it
# We don't check if index is -1 because if it
# were, self.resolve() would have returned nil
if not t.closedOver:
# Static name resolution, loads value at index in the stack. Very fast. Much wow.
@ -802,7 +810,7 @@ proc identifier(self: Compiler, node: IdentExpr) =
if self.closedOver.len() == 0:
self.error("error: closure variable array is empty but LoadHeap would be emitted (this is an internal error and most likely a bug)")
# Heap-allocated closure variable. Stored in a separate "closure array" in the VM that does not have stack semantics.
# This makes closures work as expected and is not comparatively slower than indexing our stack (since they're both
# This makes closures work as expected and is not comparatively slower than indexing our stack (since they're both
# dynamic arrays at runtime anyway)
self.emitByte(LoadHeap)
self.emitBytes(self.closedOver.high().toTriple())
@ -989,9 +997,9 @@ proc expression(self: Compiler, node: Expression) =
self.error("expression has no type")
case node.kind:
of callExpr:
discard # TODO
discard # TODO
of getItemExpr:
discard # TODO
discard # TODO
# Note that for setItem and assign we don't convert
# the node to its true type because that type information
# would be lost in the call anyway. The differentiation
@ -1014,7 +1022,7 @@ proc expression(self: Compiler, node: Expression) =
# Since all of these AST nodes share the
# same overall structure and the kind
# field is enough to tell one from the
# other, why bother with specialized
# other, why bother with specialized
# cases when one is enough?
self.literal(node)
else:
@ -1127,7 +1135,7 @@ proc statement(self: Compiler, node: Statement) =
of exprStmt:
var expression = ExprStmt(node).expression
self.expression(expression)
self.emitByte(Pop) # Expression statements discard their value. Their main use case is side effects in function calls
self.emitByte(Pop) # Expression statements discard their value. Their main use case is side effects in function calls
of NodeKind.ifStmt:
self.ifStmt(IfStmt(node))
of NodeKind.assertStmt:
@ -1204,7 +1212,7 @@ proc funDecl(self: Compiler, node: FunDecl) =
var function = self.currentFunction
self.currentFunction = node
# Since the deferred array is a linear
# Since the deferred array is a linear
# sequence of instructions and we want
# to keep track to whose function's each
# set of deferred instruction belongs,
@ -1250,7 +1258,7 @@ proc declaration(self: Compiler, node: Declaration) =
self.statement(Statement(node))
proc compile*(self: Compiler, ast: seq[ASTNode], file: string): Chunk =
proc compile*(self: Compiler, ast: seq[Declaration], file: string): Chunk =
## Compiles a sequence of AST nodes into a chunk
## object
self.chunk = newChunk()
@ -1269,4 +1277,4 @@ proc compile*(self: Compiler, ast: seq[ASTNode], file: string): Chunk =
self.emitByte(OpCode.Return) # Exits the VM's main loop when used at the global scope
result = self.chunk
if self.ast.len() > 0 and self.scopeDepth != -1:
self.error(&"invalid state: invalid scopeDepth value (expected -1, got {self.scopeDepth}), did you forget to call endScope/beginScope?")
self.error(&"invalid state: invalid scopeDepth value (expected -1, got {self.scopeDepth}), did you forget to call endScope/beginScope?")

55
src/frontend/lexer.nim
View File

@ -32,7 +32,7 @@ type
SymbolTable* = ref object
## A table of symbols used
## to lex a source file
# Although we don't parse keywords
# as symbols, but rather as identifiers,
# we keep them here for consistency
@ -64,7 +64,7 @@ proc addSymbol*(self: SymbolTable, lexeme: string, token: TokenType) =
self.symbols[lexeme] = token
proc removeSymbol*(self: SymbolTable, lexeme: string) =
proc removeSymbol*(self: SymbolTable, lexeme: string) =
## Removes a symbol from the symbol table
## (does nothing if it does not exist)
self.symbols.del(lexeme)
@ -76,30 +76,31 @@ proc addKeyword*(self: SymbolTable, lexeme: string, token: TokenType) =
self.keywords[lexeme] = token
proc removeKeyword*(self: SymbolTable, lexeme: string) =
proc removeKeyword*(self: SymbolTable, lexeme: string) =
## Removes a keyword from the symbol table
## (does nothing if it does not exist)
self.keywords.del(lexeme)
proc existsSymbol*(self: SymbolTable, lexeme: string): bool {.inline.} =
## Returns true if a given symbol exists
proc existsSymbol*(self: SymbolTable, lexeme: string): bool {.inline.} =
## Returns true if a given symbol exists
## in the symbol table already
lexeme in self.symbols
proc existsKeyword*(self: SymbolTable, lexeme: string): bool {.inline.} =
## Returns true if a given keyword exists
proc existsKeyword*(self: SymbolTable, lexeme: string): bool {.inline.} =
## Returns true if a given keyword exists
## in the symbol table already
lexeme in self.keywords
proc getToken(self: Lexer, lexeme: string): Token =
## Gets the matching token object for a given
## string according to the symbol table or
## string according to the symbol table or
## returns nil if there's no match
let table = self.symbols
var kind = table.symbols.getOrDefault(lexeme, table.keywords.getOrDefault(lexeme, NoMatch))
var kind = table.symbols.getOrDefault(lexeme, table.keywords.getOrDefault(
lexeme, NoMatch))
if kind == NoMatch:
return nil
new(result)
@ -125,7 +126,7 @@ proc getSymbols(self: SymbolTable, n: int): seq[string] =
for lexeme in self.symbols.keys():
if len(lexeme) == n:
result.add(lexeme)
# Wrappers around isDigit and isAlphanumeric for
# strings
proc isDigit(s: string): bool =
@ -147,7 +148,8 @@ proc getStart*(self: Lexer): int = self.start
proc getCurrent*(self: Lexer): int = self.current
proc getLine*(self: Lexer): int = self.line
proc getSource*(self: Lexer): string = self.source
proc getRelPos*(self: Lexer, line: int): tuple[start, stop: int] = (if line > 1: self.lines[line - 2] else: (start: 0, stop: self.current))
proc getRelPos*(self: Lexer, line: int): tuple[start, stop: int] = (if line >
1: self.lines[line - 2] else: (start: 0, stop: self.current))
proc newLexer*(self: Lexer = nil): Lexer =
@ -183,7 +185,7 @@ proc incLine(self: Lexer) =
proc step(self: Lexer, n: int = 1): string =
## Steps n characters forward in the
## source file (default = 1). A string
## source file (default = 1). A string
## of at most n bytes is returned. If n
## exceeds EOF, the string will be shorter
while len(result) < n:
@ -196,17 +198,18 @@ proc step(self: Lexer, n: int = 1): string =
proc peek(self: Lexer, distance: int = 0, length: int = 1): string =
## Returns a stream of characters of
## at most length bytes from the source
## file, starting at the given distance,
## without consuming it. The distance
## parameter may be negative to retrieve
## previously consumed tokens. If the
## at most length bytes from the source
## file, starting at the given distance,
## without consuming it. The distance
## parameter may be negative to retrieve
## previously consumed tokens. If the
## distance and/or the length are beyond
## EOF (even partially), the resulting string
## will be shorter than length bytes
var i = distance
while len(result) < length:
if self.done() or self.current + i > self.source.high() or self.current + i < 0:
if self.done() or self.current + i > self.source.high() or
self.current + i < 0:
break
else:
result.add(self.source[self.current + i])
@ -242,9 +245,9 @@ proc check(self: Lexer, args: openarray[string], distance: int = 0): bool =
proc match(self: Lexer, s: string): bool =
## Returns true if the next len(s) bytes
## Returns true if the next len(s) bytes
## of the source file match the provided
## string. If the match is successful,
## string. If the match is successful,
## len(s) bytes are consumed, otherwise
## false is returned
if not self.check(s):
@ -286,7 +289,7 @@ proc parseEscape(self: Lexer) =
# likely be soon. Another notable limitation is that
# \xhhh and \nnn are limited to the size of a char
# (i.e. uint8, or 256 values)
case self.peek()[0]: # We use a char instead of a string because of how case statements handle ranges with strings
case self.peek()[0]: # We use a char instead of a string because of how case statements handle ranges with strings
# (i.e. not well, given they crash the C code generator)
of 'a':
self.source[self.current] = cast[char](0x07)
@ -319,7 +322,7 @@ proc parseEscape(self: Lexer) =
self.source[self.current] = '\''
of '\\':
self.source[self.current] = cast[char](0x5C)
of '0'..'9': # This is the reason we're using char instead of string. See https://github.com/nim-lang/Nim/issues/19678
of '0'..'9': # This is the reason we're using char instead of string. See https://github.com/nim-lang/Nim/issues/19678
var code = ""
var value = 0
var i = self.current
@ -490,7 +493,8 @@ proc parseNumber(self: Lexer) =
discard self.step()
if self.match("'"):
# Could be a size specifier, better catch it
while (self.peek().isAlphaNumeric() or self.check("_")) and not self.done():
while (self.peek().isAlphaNumeric() or self.check("_")) and
not self.done():
discard self.step()
self.createToken(kind)
if kind == Binary:
@ -558,13 +562,14 @@ proc next(self: Lexer) =
elif self.match(["\"", "'"]):
# String or character literal
var mode = "single"
if self.peek(-1) != "'" and self.check(self.peek(-1)) and self.check(self.peek(-1), 1):
if self.peek(-1) != "'" and self.check(self.peek(-1)) and self.check(
self.peek(-1), 1):
# Multiline strings start with 3 quotes
discard self.step(2)
mode = "multi"
self.parseString(self.peek(-1), mode)
elif self.peek().isDigit():
discard self.step() # Needed because parseNumber reads the next
discard self.step() # Needed because parseNumber reads the next
# character to tell the base of the number
# Number literal
self.parseNumber()

84
src/frontend/meta/ast.nim
View File

@ -33,7 +33,7 @@ type
funDecl = 0'u8,
varDecl,
# Statements
forStmt, # Unused for now (for loops are compiled to while loops)
forStmt, # Unused for now (for loops are compiled to while loops)
ifStmt,
returnStmt,
breakStmt,
@ -61,8 +61,8 @@ type
sliceExpr,
callExpr,
getItemExpr, # Get expressions like a.b
# Primary expressions
groupingExpr, # Parenthesized expressions such as (true) and (3 + 4)
# Primary expressions
groupingExpr, # Parenthesized expressions such as (true) and (3 + 4)
trueExpr,
falseExpr,
strExpr,
@ -76,6 +76,7 @@ type
nanExpr,
infExpr,
identExpr, # Identifier
pragmaExpr
# Here I would've rather used object variants, and in fact that's what was in
# place before, but not being able to re-declare a field of the same type in
@ -97,6 +98,8 @@ type
Declaration* = ref object of ASTNode
## A declaration
pragmas*: seq[Pragma]
generics*: seq[tuple[name: IdentExpr, cond: Expression]]
Statement* = ref object of Declaration
## A statement
Expression* = ref object of Statement
@ -145,7 +148,8 @@ type
CallExpr* = ref object of Expression
callee*: Expression # The object being called
arguments*: tuple[positionals: seq[Expression], keyword: seq[tuple[name: IdentExpr, value: Expression]]]
arguments*: tuple[positionals: seq[Expression], keyword: seq[tuple[
name: IdentExpr, value: Expression]]]
UnaryExpr* = ref object of Expression
operator*: Token
@ -165,7 +169,8 @@ type
LambdaExpr* = ref object of Expression
body*: Statement
arguments*: seq[tuple[name: IdentExpr, valueType: Expression, mutable: bool, isRef: bool, isPtr: bool]]
arguments*: seq[tuple[name: IdentExpr, valueType: Expression,
mutable: bool, isRef: bool, isPtr: bool]]
defaults*: seq[Expression]
isGenerator*: bool
isAsync*: bool
@ -245,7 +250,8 @@ type
FunDecl* = ref object of Declaration
name*: IdentExpr
body*: Statement
arguments*: seq[tuple[name: IdentExpr, valueType: Expression, mutable: bool, isRef: bool, isPtr: bool]]
arguments*: seq[tuple[name: IdentExpr, valueType: Expression,
mutable: bool, isRef: bool, isPtr: bool]]
defaults*: seq[Expression]
isAsync*: bool
isGenerator*: bool
@ -264,18 +270,19 @@ proc isConst*(self: ASTNode): bool =
## strings and singletons count as
## constants
case self.kind:
of intExpr, hexExpr, binExpr, octExpr, strExpr, falseExpr, trueExpr, infExpr, nanExpr, floatExpr, nilExpr:
of intExpr, hexExpr, binExpr, octExpr, strExpr, falseExpr, trueExpr,
infExpr, nanExpr, floatExpr, nilExpr:
return true
else:
return false
proc isLiteral*(self: ASTNode): bool {.inline.} =
proc isLiteral*(self: ASTNode): bool {.inline.} =
## Returns if the AST node represents a literal
self.kind in {intExpr, hexExpr, binExpr, octExpr,
strExpr, falseExpr, trueExpr, infExpr,
self.kind in {intExpr, hexExpr, binExpr, octExpr,
strExpr, falseExpr, trueExpr, infExpr,
nanExpr, floatExpr, nilExpr
}
}
## AST node constructors
proc newASTNode*(kind: NodeKind, token: Token): ASTNode =
@ -285,6 +292,13 @@ proc newASTNode*(kind: NodeKind, token: Token): ASTNode =
result.token = token
proc newPragma*(name: IdentExpr, args: seq[LiteralExpr]): Pragma =
new(result)
result.kind = pragmaExpr
result.args = args
result.name = name
proc newIntExpr*(literal: Token): IntExpr =
result = IntExpr(kind: intExpr)
result.literal = literal
@ -315,11 +329,16 @@ proc newFloatExpr*(literal: Token): FloatExpr =
result.token = literal
proc newTrueExpr*(token: Token): LiteralExpr = LiteralExpr(kind: trueExpr, token: token, literal: token)
proc newFalseExpr*(token: Token): LiteralExpr = LiteralExpr(kind: falseExpr, token: token, literal: token)
proc newNaNExpr*(token: Token): LiteralExpr = LiteralExpr(kind: nanExpr, token: token, literal: token)
proc newNilExpr*(token: Token): LiteralExpr = LiteralExpr(kind: nilExpr, token: token, literal: token)
proc newInfExpr*(token: Token): LiteralExpr = LiteralExpr(kind: infExpr, token: token, literal: token)
proc newTrueExpr*(token: Token): LiteralExpr = LiteralExpr(kind: trueExpr,
token: token, literal: token)
proc newFalseExpr*(token: Token): LiteralExpr = LiteralExpr(kind: falseExpr,
token: token, literal: token)
proc newNaNExpr*(token: Token): LiteralExpr = LiteralExpr(kind: nanExpr,
token: token, literal: token)
proc newNilExpr*(token: Token): LiteralExpr = LiteralExpr(kind: nilExpr,
token: token, literal: token)
proc newInfExpr*(token: Token): LiteralExpr = LiteralExpr(kind: infExpr,
token: token, literal: token)
proc newStrExpr*(literal: Token): StrExpr =
@ -346,8 +365,10 @@ proc newGroupingExpr*(expression: Expression, token: Token): GroupingExpr =
result.token = token
proc newLambdaExpr*(arguments: seq[tuple[name: IdentExpr, valueType: Expression, mutable: bool, isRef: bool, isPtr: bool]], defaults: seq[Expression], body: Statement,
isGenerator: bool, isAsync: bool, token: Token, returnType: Expression): LambdaExpr =
proc newLambdaExpr*(arguments: seq[tuple[name: IdentExpr, valueType: Expression,
mutable: bool, isRef: bool, isPtr: bool]], defaults: seq[Expression],
body: Statement, isGenerator: bool, isAsync: bool, token: Token,
returnType: Expression, pragmas: seq[Pragma]): LambdaExpr =
result = LambdaExpr(kind: lambdaExpr)
result.body = body
result.arguments = arguments
@ -357,16 +378,19 @@ proc newLambdaExpr*(arguments: seq[tuple[name: IdentExpr, valueType: Expression,
result.token = token
result.returnType = returnType
result.isPure = false
result.pragmas = pragmas
proc newGetItemExpr*(obj: Expression, name: IdentExpr, token: Token): GetItemExpr =
proc newGetItemExpr*(obj: Expression, name: IdentExpr,
token: Token): GetItemExpr =
result = GetItemExpr(kind: getItemExpr)
result.obj = obj
result.name = name
result.token = token
proc newSetItemExpr*(obj: Expression, name: IdentExpr, value: Expression, token: Token): SetItemExpr =
proc newSetItemExpr*(obj: Expression, name: IdentExpr, value: Expression,
token: Token): SetItemExpr =
result = SetItemExpr(kind: setItemExpr)
result.obj = obj
result.name = name
@ -374,8 +398,8 @@ proc newSetItemExpr*(obj: Expression, name: IdentExpr, value: Expression, token:
result.token = token
proc newCallExpr*(callee: Expression, arguments: tuple[positionals: seq[Expression],
keyword: seq[tuple[name: IdentExpr, value: Expression]]],
proc newCallExpr*(callee: Expression, arguments: tuple[positionals: seq[
Expression], keyword: seq[tuple[name: IdentExpr, value: Expression]]],
token: Token): CallExpr =
result = CallExpr(kind: callExpr)
result.callee = callee
@ -412,7 +436,8 @@ proc newYieldExpr*(expression: Expression, token: Token): YieldExpr =
result.token = token
proc newAssignExpr*(name: Expression, value: Expression, token: Token): AssignExpr =
proc newAssignExpr*(name: Expression, value: Expression,
token: Token): AssignExpr =
result = AssignExpr(kind: assignExpr)
result.name = name
result.value = value
@ -484,15 +509,16 @@ proc newBlockStmt*(code: seq[Declaration], token: Token): BlockStmt =
result.token = token
proc newWhileStmt*(condition: Expression, body: Statement, token: Token): WhileStmt =
proc newWhileStmt*(condition: Expression, body: Statement,
token: Token): WhileStmt =
result = WhileStmt(kind: whileStmt)
result.condition = condition
result.body = body
result.token = token
proc newForEachStmt*(identifier: IdentExpr, expression: Expression, body: Statement,
token: Token): ForEachStmt =
proc newForEachStmt*(identifier: IdentExpr, expression: Expression,
body: Statement, token: Token): ForEachStmt =
result = ForEachStmt(kind: forEachStmt)
result.identifier = identifier
result.expression = expression
@ -540,7 +566,7 @@ proc newVarDecl*(name: IdentExpr, value: Expression, isConst: bool = false,
proc newFunDecl*(name: IdentExpr, arguments: seq[tuple[name: IdentExpr, valueType: Expression, mutable: bool, isRef: bool, isPtr: bool]], defaults: seq[Expression],
body: Statement, isAsync, isGenerator: bool,
body: Statement, isAsync, isGenerator: bool,
isPrivate: bool, token: Token, pragmas: seq[Pragma],
returnType: Expression): FunDecl =
result = FunDecl(kind: funDecl)
@ -667,4 +693,4 @@ proc `$`*(self: ASTNode): string =
discard
proc `==`*(self, other: IdentExpr): bool {.inline.} = self.token == other.token
proc `==`*(self, other: IdentExpr): bool {.inline.} = self.token == other.token

73
src/frontend/meta/bytecode.nim
View File

@ -55,15 +55,15 @@ type
## Enum of Peon's bytecode opcodes
# Note: x represents the argument
# to unary opcodes, while a and b
# to unary opcodes, while a and b
# represent arguments to binary
# opcodes. Other variable names (c, d, ...)
# may be used for more complex opcodes. If
# an opcode takes any arguments at runtime,
# they come from either the stack or the VM's
# closure array. Some other opcodes (e.g.
# jumps), take arguments in the form of 16
# or 24 bit numbers that are defined statically
# closure array. Some other opcodes (e.g.
# jumps), take arguments in the form of 16
# or 24 bit numbers that are defined statically
# at compilation time into the bytecode
# These push a constant onto the stack
@ -85,23 +85,23 @@ type
LoadNan,
LoadInf,
## Basic stack operations
Pop, # Pops an element off the stack and discards it
Push, # Pushes x onto the stack
PopN, # Pops x elements off the stack (optimization for exiting scopes and returning from functions)
Pop, # Pops an element off the stack and discards it
Push, # Pushes x onto the stack
PopN, # Pops x elements off the stack (optimization for exiting scopes and returning from functions)
## Name resolution/handling
LoadAttribute, # Pushes the attribute b of object a onto the stack
LoadVar, # Pushes the object at position x in the stack onto the stack
StoreVar, # Stores the value of b at position a in the stack
LoadHeap, # Pushes the object position x in the closure array onto the stack
StoreHeap, # Stores the value of b at position a in the closure array
LoadAttribute, # Pushes the attribute b of object a onto the stack
LoadVar, # Pushes the object at position x in the stack onto the stack
StoreVar, # Stores the value of b at position a in the stack
LoadHeap, # Pushes the object position x in the closure array onto the stack
StoreHeap, # Stores the value of b at position a in the closure array
## Looping and jumping
Jump, # Absolute, unconditional jump into the bytecode
JumpForwards, # Relative, unconditional, positive jump in the bytecode
JumpBackwards, # Relative, unconditional, negative jump in the bytecode
JumpIfFalse, # Jumps to a relative index in the bytecode if x is false
JumpIfTrue, # Jumps to a relative index in the bytecode if x is true
JumpIfFalsePop, # Like JumpIfFalse, but also pops off the stack (regardless of truthyness). Optimization for if statements
JumpIfFalseOrPop, # Jumps to an absolute index in the bytecode if x is false and pops otherwise (used for logical and)
Jump, # Absolute, unconditional jump into the bytecode
JumpForwards, # Relative, unconditional, positive jump in the bytecode
JumpBackwards, # Relative, unconditional, negative jump in the bytecode
JumpIfFalse, # Jumps to a relative index in the bytecode if x is false
JumpIfTrue, # Jumps to a relative index in the bytecode if x is true
JumpIfFalsePop, # Like JumpIfFalse, but also pops off the stack (regardless of truthyness). Optimization for if statements
JumpIfFalseOrPop, # Jumps to an absolute index in the bytecode if x is false and pops otherwise (used for logical and)
## Long variants of jumps (they use a 24-bit operand instead of a 16-bit one)
LongJump,
LongJumpIfFalse,
@ -111,29 +111,29 @@ type
LongJumpForwards,
LongJumpBackwards,
## Functions
Call, # Calls a function and initiates a new stack frame
Return, # Terminates the current function without popping off the stack
ReturnPop, # Pops a return value off the stack and terminates the current function
Call, # Calls a function and initiates a new stack frame
Return, # Terminates the current function without popping off the stack
ReturnPop, # Pops a return value off the stack and terminates the current function
## Exception handling
Raise, # Raises exception x or re-raises active exception if x is nil
BeginTry, # Initiates an exception handling context
FinishTry, # Closes the current exception handling context
Raise, # Raises exception x or re-raises active exception if x is nil
BeginTry, # Initiates an exception handling context
FinishTry, # Closes the current exception handling context
## Generators
Yield, # Yields control from a generator back to the caller
Yield, # Yields control from a generator back to the caller
## Coroutines
Await, # Calls an asynchronous function
Await, # Calls an asynchronous function
## Misc
Assert, # Raises an AssertionFailed exception if x is false
NoOp, # Just a no-op
Assert, # Raises an AssertionFailed exception if x is false
NoOp, # Just a no-op
# We group instructions by their operation/operand types for easier handling when debugging
# We group instructions by their operation/operand types for easier handling when debugging
# Simple instructions encompass instructions that push onto/pop off the stack unconditionally (True, False, Pop, etc.)
# Simple instructions encompass instructions that push onto/pop off the stack unconditionally (True, False, Pop, etc.)
const simpleInstructions* = {OpCode.Return, LoadNil,
LoadTrue, LoadFalse,
LoadNan, LoadInf,
Pop, OpCode.Raise,
Pop, OpCode.Raise,
BeginTry, FinishTry,
OpCode.Yield, OpCode.Await,
OpCode.NoOp, OpCode.Return,
@ -159,10 +159,10 @@ const stackDoubleInstructions* = {}
const argumentDoubleInstructions* = {PopN, }
# Jump instructions jump at relative or absolute bytecode offsets
const jumpInstructions* = {Jump, LongJump, JumpIfFalse, JumpIfFalsePop,
const jumpInstructions* = {Jump, LongJump, JumpIfFalse, JumpIfFalsePop,
JumpForwards, JumpBackwards,
LongJumpIfFalse, LongJumpIfFalsePop,
LongJumpForwards, LongJumpBackwards,
LongJumpForwards, LongJumpBackwards,
JumpIfTrue, LongJumpIfTrue}
@ -234,7 +234,8 @@ proc findOrAddConstant(self: Chunk, constant: Expression, kind: Type): int =
if c.kind != constant.kind:
continue
if constant.isConst():
if LiteralExpr(c).literal.lexeme == LiteralExpr(constant).literal.lexeme:
if LiteralExpr(c).literal.lexeme == LiteralExpr(
constant).literal.lexeme:
# This wouldn't work for stuff like 2e3 and 2000.0, but those
# forms are collapsed in the compiler before being written
# to the constants table
@ -251,7 +252,7 @@ proc findOrAddConstant(self: Chunk, constant: Expression, kind: Type): int =
proc addConstant*(self: Chunk, constant: Expression, kind: Type): array[3, uint8] =
## Writes a constant of the given type in the chunk's constant
## table. Returns its index as an array of 3 unsigned 8 bit integers.
## Constant indexes are reused if a constant is used more than once
## Constant indexes are reused if a constant is used more than once
## and self.reuseConsts equals true
if self.consts.high() == 16777215:
# The constant index is a 24 bit unsigned integer, so that's as far

98
src/frontend/meta/token.nim
View File

@ -18,72 +18,72 @@ import strformat
type
TokenType* {.pure.} = enum
## Token types enumeration
## Token types enumeration
# Booleans
True, False,
# Booleans
True, False,
# Other singleton types
Infinity, NotANumber, Nil
# Other singleton types
Infinity, NotANumber, Nil
# Control flow statements
If, Else,
# Control flow statements
If, Else,
# Looping statements
While, For,
# Looping statements
While, For,
# Keywords
Function, Break, Continue,
Var, Let, Const, Return,
Coroutine, Generator, Import,
Raise, Assert, Await, Foreach,
Yield, Defer, Try, Except,
Finally, Type, Operator, Case,
Enum, From, Ptr, Ref
# Keywords
Function, Break, Continue,
Var, Let, Const, Return,
Coroutine, Generator, Import,
Raise, Assert, Await, Foreach,
Yield, Defer, Try, Except,
Finally, Type, Operator, Case,
Enum, From, Ptr, Ref
# Literal types
Integer, Float, String, Identifier,
Binary, Octal, Hex, Char
# Literal types
Integer, Float, String, Identifier,
Binary, Octal, Hex, Char
# Brackets, parentheses,
# operators and others
# Brackets, parentheses,
# operators and others
LeftParen, RightParen, # ()
LeftBrace, RightBrace, # {}
LeftBracket, RightBracket, # []
Dot, Semicolon, Comma, # . ; ,
LeftParen, RightParen, # ()
LeftBrace, RightBrace, # {}
LeftBracket, RightBracket, # []
Dot, Semicolon, Comma, # . ; ,
# Miscellaneous
EndOfFile, # Marks the end of the token stream
NoMatch, # Used internally by the symbol table
Comment, # Useful for documentation comments, pragmas, etc.
Symbol, # A generic symbol
# These are not used at the moment but may be
# employed to enforce indentation or other neat
# stuff I haven't thought about yet
Whitespace,
Tab,
EndOfFile, # Marks the end of the token stream
NoMatch, # Used internally by the symbol table
Comment, # Useful for documentation comments, pragmas, etc.
Symbol, # A generic symbol
# These are not used at the moment but may be
# employed to enforce indentation or other neat
# stuff I haven't thought about yet
Whitespace,
Tab,
Token* = ref object
## A token object
kind*: TokenType # Type of the token
lexeme*: string # The lexeme associated to the token
line*: int # The line where the token appears
pos*: tuple[start, stop: int] # The absolute position in the source file
# (0-indexed and inclusive at the beginning)
## A token object
kind*: TokenType # Type of the token
lexeme*: string # The lexeme associated to the token
line*: int # The line where the token appears
pos*: tuple[start, stop: int] # The absolute position in the source file
# (0-indexed and inclusive at the beginning)
proc `$`*(self: Token): string =
## Strinfifies
if self != nil:
result = &"Token(kind={self.kind}, lexeme='{$(self.lexeme)}', line={self.line}, pos=({self.pos.start}, {self.pos.stop}))"
else:
result = "nil"
## Strinfifies
if self != nil:
result = &"Token(kind={self.kind}, lexeme='{$(self.lexeme)}', line={self.line}, pos=({self.pos.start}, {self.pos.stop}))"
else:
result = "nil"
proc `==`*(self, other: Token): bool =
## Returns self == other
return self.kind == other.kind and self.lexeme == other.lexeme
## Returns self == other
return self.kind == other.kind and self.lexeme == other.lexeme

217
src/frontend/parser.nim
View File

@ -26,8 +26,8 @@ import meta/errors
export token, ast, errors
type
type
LoopContext {.pure.} = enum
Loop, None
Precedence {.pure.} = enum
@ -42,7 +42,7 @@ type
Addition,
Multiplication,
Power,
None # Used for stuff that isn't an operator
None # Used for stuff that isn't an operator
OperatorTable = ref object
## A table for storing and
@ -76,14 +76,16 @@ type
# Stores the current function
# being parsed. This is a reference
# to either a FunDecl or LambdaExpr
# AST node and is nil when the parser
# is at the top-level. It allows the
# AST node and is nil when the parser
# is at the top-level. It allows the
# parser to detect errors like return
# outside functions
currentFunction: Declaration
# Stores the current scope depth (0 = global, > 0 local)
scopeDepth: int
operators: OperatorTable
# The AST node
tree: seq[Declaration]
proc newOperatorTable: OperatorTable =
@ -101,7 +103,7 @@ proc addOperator(self: OperatorTable, lexeme: string) =
## is inferred from the operator's lexeme (the
## criteria are similar to Nim's)
if lexeme in self.tokens:
return # We've already added it!
return # We've already added it!
var prec = Precedence.high()
if lexeme.len() >= 2 and lexeme[^2..^1] in ["->", "~>", "=>"]:
prec = Arrow
@ -128,9 +130,9 @@ proc getPrecedence(self: OperatorTable, lexeme: string): Precedence =
for (prec, operators) in self.precedence.pairs():
if lexeme in operators:
return prec
proc newParser*: Parser =
proc newParser*: Parser =
## Initializes a new Parser object
new(result)
result.current = 0
@ -140,11 +142,13 @@ proc newParser*: Parser =
result.currentLoop = LoopContext.None
result.scopeDepth = 0
result.operators = newOperatorTable()
result.tree = @[]
# Public getters for improved error formatting
proc getCurrent*(self: Parser): int {.inline.} = self.current
proc getCurrentToken*(self: Parser): Token {.inline.} = (if self.getCurrent() >= self.tokens.high() or
proc getCurrentToken*(self: Parser): Token {.inline.} = (if self.getCurrent() >=
self.tokens.high() or
self.getCurrent() - 1 < 0: self.tokens[^1] else: self.tokens[self.current - 1])
# Handy templates to make our life easier, thanks nim!
@ -158,7 +162,8 @@ proc peek(self: Parser, distance: int = 0): Token =
## token is returned. A negative distance may
## be used to retrieve previously consumed
## tokens
if self.tokens.high() == -1 or self.current + distance > self.tokens.high() or self.current + distance < 0:
if self.tokens.high() == -1 or self.current + distance > self.tokens.high(
) or self.current + distance < 0:
result = endOfFile
else:
result = self.tokens[self.current + distance]
@ -173,7 +178,7 @@ proc done(self: Parser): bool =
result = self.peek().kind == EndOfFile
proc step(self: Parser, n: int = 1): Token =
proc step(self: Parser, n: int = 1): Token =
## Steps n tokens into the input,
## returning the last consumed one
if self.done():
@ -197,7 +202,8 @@ proc error(self: Parser, message: string) {.raises: [ParseError, ValueError].} =
# tell at tokenization time which of the two contexts we're in, we just treat everything
# as a symbol and in the cases where we need a specific token we just match the string
# directly
proc check[T: TokenType or string](self: Parser, kind: T, distance: int = 0): bool =
proc check[T: TokenType or string](self: Parser, kind: T,
distance: int = 0): bool =
## Checks if the given token at the given distance
## matches the expected kind and returns a boolean.
## The distance parameter is passed directly to
@ -207,7 +213,7 @@ proc check[T: TokenType or string](self: Parser, kind: T, distance: int = 0): bo
else:
when T is string:
self.peek(distance).lexeme == kind
proc check[T: TokenType or string](self: Parser, kind: openarray[T]): bool =
## Calls self.check() in a loop with each entry of
@ -243,7 +249,8 @@ proc match[T: TokenType or string](self: Parser, kind: openarray[T]): bool =
result = false
proc expect[T: TokenType or string](self: Parser, kind: T, message: string = "") =
proc expect[T: TokenType or string](self: Parser, kind: T,
message: string = "") =
## Behaves like self.match(), except that
## when a token doesn't match, an error
## is raised. If no error message is
@ -255,7 +262,8 @@ proc expect[T: TokenType or string](self: Parser, kind: T, message: string = "")
self.error(message)
proc expect[T: TokenType or string](self: Parser, kind: openarray[T], message: string = "") =
proc expect[T: TokenType or string](self: Parser, kind: openarray[T],
message: string = "") =
## Behaves like self.expect(), except that
## an error is raised only if none of the
## given token kinds matches
@ -264,22 +272,24 @@ proc expect[T: TokenType or string](self: Parser, kind: openarray[T], message: s
return
if message.len() == 0:
self.error(&"""expecting any of the following tokens: {kinds.join(", ")}, but got {self.peek().kind} instead""")
# Forward declarations
proc expression(self: Parser): Expression
proc expressionStatement(self: Parser): Statement
proc statement(self: Parser): Statement
proc varDecl(self: Parser, isLet: bool = false, isConst: bool = false): Declaration
proc funDecl(self: Parser, isAsync: bool = false, isGenerator: bool = false, isLambda: bool = false, isOperator: bool = false): Declaration
proc varDecl(self: Parser, isLet: bool = false,
isConst: bool = false): Declaration
proc funDecl(self: Parser, isAsync: bool = false, isGenerator: bool = false,
isLambda: bool = false, isOperator: bool = false): Declaration
proc declaration(self: Parser): Declaration
# End of forward declarations
proc primary(self: Parser): Expression =
proc primary(self: Parser): Expression =
## Parses primary expressions such
## as integer literals and keywords
## that map to builtin types (true,
## that map to builtin types (true,
## false, nil, etc.)
case self.peek().kind:
of True:
@ -337,13 +347,14 @@ proc primary(self: Parser): Expression =
result = newInfExpr(self.step())
of Function:
discard self.step()
result = Expression(self.funDecl(isLambda=true))
result = Expression(self.funDecl(isLambda = true))
of Coroutine:
discard self.step()
result = Expression(self.funDecl(isAsync=true, isLambda=true))
result = Expression(self.funDecl(isAsync = true, isLambda = true))
of Generator:
discard self.step()
result = Expression(self.funDecl(isGenerator=true, isLambda=true))
result = Expression(self.funDecl(isGenerator = true,
isLambda = true))
else:
self.error("invalid syntax")
@ -353,7 +364,9 @@ proc makeCall(self: Parser, callee: Expression): Expression =
## to parse a function call
let tok = self.peek(-1)
var argNames: seq[IdentExpr] = @[]
var arguments: tuple[positionals: seq[Expression], keyword: seq[tuple[name: IdentExpr, value: Expression]]] = (positionals: @[], keyword: @[])
var arguments: tuple[positionals: seq[Expression], keyword: seq[tuple[
name: IdentExpr, value: Expression]]] = (positionals: @[],
keyword: @[])
var argument: Expression = nil
var argCount = 0
if not self.check(RightParen):
@ -367,7 +380,8 @@ proc makeCall(self: Parser, callee: Expression): Expression =
if IdentExpr(AssignExpr(argument).name) in argNames:
self.error("duplicate keyword argument in call")
argNames.add(IdentExpr(AssignExpr(argument).name))
arguments.keyword.add((name: IdentExpr(AssignExpr(argument).name), value: AssignExpr(argument).value))
arguments.keyword.add((name: IdentExpr(AssignExpr(
argument).name), value: AssignExpr(argument).value))
elif arguments.keyword.len() == 0:
arguments.positionals.add(argument)
else:
@ -379,7 +393,7 @@ proc makeCall(self: Parser, callee: Expression): Expression =
result = newCallExpr(callee, arguments, tok)
proc call(self: Parser): Expression =
proc call(self: Parser): Expression =
## Parses function calls, object field
## accessing and slicing expressions
result = self.primary()
@ -388,7 +402,8 @@ proc call(self: Parser): Expression =
result = self.makeCall(result)
elif self.match(Dot):
self.expect(Identifier, "expecting attribute name after '.'")
result = newGetItemExpr(result, newIdentExpr(self.peek(-1)), self.peek(-1))
result = newGetItemExpr(result, newIdentExpr(self.peek(-1)),
self.peek(-1))
elif self.match(LeftBracket):
# Slicing such as a[1:2], which is then
# translated to `[]`(a, 1, 2)
@ -408,7 +423,7 @@ proc call(self: Parser): Expression =
## Operator parsing handlers
proc unary(self: Parser): Expression =
proc unary(self: Parser): Expression =
if self.peek().lexeme in self.operators.tokens:
result = newUnaryExpr(self.step(), self.unary())
else:
@ -484,7 +499,8 @@ proc parseAssign(self: Parser): Expression =
of identExpr, sliceExpr:
result = newAssignExpr(result, value, tok)
of getItemExpr:
result = newSetItemExpr(GetItemExpr(result).obj, GetItemExpr(result).name, value, tok)
result = newSetItemExpr(GetItemExpr(result).obj, GetItemExpr(
result).name, value, tok)
else:
self.error("invalid assignment target")
@ -531,6 +547,8 @@ proc blockStmt(self: Parser): Statement =
var code: seq[Declaration] = @[]
while not self.check(RightBrace) and not self.done():
code.add(self.declaration())
if self.tree[^1] == nil:
self.tree.delete(self.tree.high())
self.expect(RightBrace, "expecting '}'")
result = newBlockStmt(code, tok)
self.endScope()
@ -687,7 +705,7 @@ proc whileStmt(self: Parser): Statement =
self.endScope()
proc forStmt(self: Parser): Statement =
proc forStmt(self: Parser): Statement =
## Parses a C-style for loop
self.beginScope()
let tok = self.peek(-1)
@ -715,7 +733,8 @@ proc forStmt(self: Parser): Statement =
if increment != nil:
# The increment runs after each iteration, so we
# inject it into the block as the last statement
body = newBlockStmt(@[Declaration(body), newExprStmt(increment, increment.token)], tok)
body = newBlockStmt(@[Declaration(body), newExprStmt(increment,
increment.token)], tok)
if condition == nil:
## An empty condition is functionally
## equivalent to "true"
@ -766,7 +785,8 @@ template checkDecl(self: Parser, isPrivate: bool) =
self.error("cannot bind public names inside local scopes")
proc varDecl(self: Parser, isLet: bool = false, isConst: bool = false): Declaration =
proc varDecl(self: Parser, isLet: bool = false,
isConst: bool = false): Declaration =
## Parses variable declarations
var tok = self.peek(-1)
var value: Expression
@ -792,17 +812,22 @@ proc varDecl(self: Parser, isLet: bool = false, isConst: bool = false): Declarat
self.expect(Semicolon, &"expecting semicolon after declaration")
case tok.kind:
of Var:
result = newVarDecl(name, value, isPrivate=isPrivate, token=tok, valueType=valueType, pragmas=(@[]))
result = newVarDecl(name, value, isPrivate = isPrivate, token = tok,
valueType = valueType, pragmas = (@[]))
of Const:
result = newVarDecl(name, value, isPrivate=isPrivate, token=tok, isConst=true, valueType=valueType, pragmas=(@[]))
result = newVarDecl(name, value, isPrivate = isPrivate, token = tok,
isConst = true, valueType = valueType, pragmas = (@[]))
of Let:
result = newVarDecl(name, value, isPrivate=isPrivate, token=tok, isLet=isLet, valueType=valueType, pragmas=(@[]))
result = newVarDecl(name, value, isPrivate = isPrivate, token = tok,
isLet = isLet, valueType = valueType, pragmas = (@[]))
else:
discard # Unreachable
discard # Unreachable
proc parseDeclArguments(self: Parser, arguments: var seq[tuple[name: IdentExpr, valueType: Expression, mutable: bool, isRef: bool, isPtr: bool]],
parameter: var tuple[name: IdentExpr, valueType: Expression, mutable: bool, isRef: bool, isPtr: bool],
proc parseDeclArguments(self: Parser, arguments: var seq[tuple[name: IdentExpr, valueType: Expression, mutable: bool, isRef: bool, isPtr: bool]],
parameter: var tuple[name: IdentExpr,
valueType: Expression, mutable: bool,
isRef: bool, isPtr: bool],
defaults: var seq[Expression]) =
## Helper to parse declaration arguments and avoid code duplication
while not self.check(RightParen):
@ -843,11 +868,13 @@ proc parseDeclArguments(self: Parser, arguments: var seq[tuple[name: IdentExpr,
self.error(&"missing type declaration for '{argument.name.token.lexeme}' in function declaration")
proc funDecl(self: Parser, isAsync: bool = false, isGenerator: bool = false, isLambda: bool = false, isOperator: bool = false): Declaration =
proc funDecl(self: Parser, isAsync: bool = false, isGenerator: bool = false,
isLambda: bool = false, isOperator: bool = false): Declaration =
## Parses functions, coroutines, generators, anonymous functions and operators
let tok = self.peek(-1)
var enclosingFunction = self.currentFunction
var arguments: seq[tuple[name: IdentExpr, valueType: Expression, mutable: bool, isRef: bool, isPtr: bool]] = @[]
var arguments: seq[tuple[name: IdentExpr, valueType: Expression,
mutable: bool, isRef: bool, isPtr: bool]] = @[]
var defaults: seq[Expression] = @[]
var returnType: Expression
if not isLambda and self.check(Identifier):
@ -859,9 +886,12 @@ proc funDecl(self: Parser, isAsync: bool = false, isGenerator: bool = false, isL
# if there's an identifier after the keyword
self.expect(Identifier, &"expecting identifier after '{tok.lexeme}'")
self.checkDecl(not self.check("*"))
self.currentFunction = newFunDecl(nil, arguments, defaults, newBlockStmt(@[], Token()),
isAsync=isAsync, isGenerator=isGenerator, isPrivate=true,
token=tok, pragmas=(@[]), returnType=nil)
self.currentFunction = newFunDecl(nil, arguments, defaults, newBlockStmt(@[], Token()),
isAsync = isAsync,
isGenerator = isGenerator,
isPrivate = true,
token = tok, pragmas = (@[]),
returnType = nil)
FunDecl(self.currentFunction).name = newIdentExpr(self.peek(-1))
if self.match("*"):
FunDecl(self.currentFunction).isPrivate = false
@ -878,8 +908,8 @@ proc funDecl(self: Parser, isAsync: bool = false, isGenerator: bool = false, isL
self.currentFunction = enclosingFunction
return result
elif isLambda:
self.currentFunction = newLambdaExpr(arguments, defaults, newBlockStmt(@[], Token()), isGenerator=isGenerator, isAsync=isAsync, token=tok,
returnType=nil)
self.currentFunction = newLambdaExpr(arguments, defaults, newBlockStmt(@[], Token()), isGenerator = isGenerator, isAsync = isAsync, token = tok,
returnType = nil, pragmas = (@[]))
elif not isOperator:
self.error("funDecl: invalid state")
if self.match(":"):
@ -890,12 +920,20 @@ proc funDecl(self: Parser, isAsync: bool = false, isGenerator: bool = false, isL
# the type declaration for a function lacks
# the braces that would qualify it as an
# expression
var arguments: seq[tuple[name: IdentExpr, valueType: Expression, mutable: bool, isRef: bool, isPtr: bool]] = @[]
var arguments: seq[tuple[name: IdentExpr, valueType: Expression,
mutable: bool, isRef: bool, isPtr: bool]] = @[]
var defaults: seq[Expression] = @[]
returnType = newLambdaExpr(arguments, defaults, nil, isGenerator=self.peek(-1).kind == Generator,
isAsync=self.peek(-1).kind == Coroutine,
token=self.peek(-1), returnType=nil)
var parameter: tuple[name: IdentExpr, valueType: Expression, mutable: bool, isRef: bool, isPtr: bool]
returnType = newLambdaExpr(arguments, defaults, nil, isGenerator = self.peek(-1).kind == Generator,
isAsync = self.peek(
-1).kind ==
Coroutine,
token = self.peek(
-1),
returnType = nil,
pragmas = (
@[]))
var parameter: tuple[name: IdentExpr, valueType: Expression,
mutable: bool, isRef: bool, isPtr: bool]
if self.match(LeftParen):
self.parseDeclArguments(arguments, parameter, defaults)
if self.match(":"):
@ -904,17 +942,26 @@ proc funDecl(self: Parser, isAsync: bool = false, isGenerator: bool = false, isL
returnType = self.expression()
if not self.match(LeftBrace):
self.expect(LeftParen)
var parameter: tuple[name: IdentExpr, valueType: Expression, mutable: bool, isRef: bool, isPtr: bool]
var parameter: tuple[name: IdentExpr, valueType: Expression,
mutable: bool, isRef: bool, isPtr: bool]
self.parseDeclArguments(arguments, parameter, defaults)
if self.match(":"):
# Function's return type
if self.match([Function, Coroutine, Generator]):
var arguments: seq[tuple[name: IdentExpr, valueType: Expression, mutable: bool, isRef: bool, isPtr: bool]] = @[]
var arguments: seq[tuple[name: IdentExpr, valueType: Expression,
mutable: bool, isRef: bool, isPtr: bool]] = @[]
var defaults: seq[Expression] = @[]
returnType = newLambdaExpr(arguments, defaults, nil, isGenerator=self.peek(-1).kind == Generator,
isAsync=self.peek(-1).kind == Coroutine,
token=self.peek(-1), returnType=nil)
var parameter: tuple[name: IdentExpr, valueType: Expression, mutable: bool, isRef: bool, isPtr: bool]
returnType = newLambdaExpr(arguments, defaults, nil, isGenerator = self.peek(-1).kind == Generator,
isAsync = self.peek(
-1).kind ==
Coroutine,
token = self.peek(
-1),
returnType = nil,
pragmas = (
@[]))
var parameter: tuple[name: IdentExpr, valueType: Expression,
mutable: bool, isRef: bool, isPtr: bool]
if self.match(LeftParen):
self.parseDeclArguments(arguments, parameter, defaults)
if self.match(":"):
@ -924,7 +971,7 @@ proc funDecl(self: Parser, isAsync: bool = false, isGenerator: bool = false, isL
self.expect(LeftBrace)
if self.currentFunction.kind == funDecl:
if not self.match(Semicolon):
# If we don't find a semicolon,
# If we don't find a semicolon,
# it's not a forward declaration
FunDecl(self.currentFunction).body = self.blockStmt()
else:
@ -946,13 +993,13 @@ proc funDecl(self: Parser, isAsync: bool = false, isGenerator: bool = false, isL
self.error("operators must have a return type")
for argument in arguments:
if argument.valueType == nil:
self.error(&"missing type declaration for '{argument.name.token.lexeme}' in function declaration")
self.error(&"missing type declaration for '{argument.name.token.lexeme}' in function declaration")
self.currentFunction = enclosingFunction
proc expression(self: Parser): Expression =
proc expression(self: Parser): Expression =
## Parses expressions
result = self.parseArrow() # Highest-level expression
result = self.parseArrow() # Highest-level expression
proc expressionStatement(self: Parser): Statement =
@ -991,7 +1038,7 @@ proc statement(self: Parser): Statement =
# TODO
# from module import a [, b, c as d]
discard self.step()
result = self.importStmt(fromStmt=true)
result = self.importStmt(fromStmt = true)
of While:
discard self.step()
result = self.whileStmt()
@ -1020,33 +1067,53 @@ proc statement(self: Parser): Statement =
result = self.expressionStatement()
proc parsePragma(self: Parser): Pragma =
## Parses pragmas
if self.scopeDepth == 0:
## Pragmas used at the
## top level are either
## used for compile-time
## switches or for variable
## declarations
var decl: VarDecl
for node in self.tree:
if node.token.line == self.peek(-1).line and node.kind == varDecl:
decl = VarDecl(node)
break
else:
var decl = self.currentFunction
proc declaration(self: Parser): Declaration =
## Parses declarations
case self.peek().kind:
of Var, Const, Let:
let keyword = self.step()
result = self.varDecl(isLet=keyword.kind == Let, isConst=keyword.kind == Const)
result = self.varDecl(isLet = keyword.kind == Let,
isConst = keyword.kind == Const)
of Function:
discard self.step()
result = self.funDecl()
of Coroutine:
discard self.step()
result = self.funDecl(isAsync=true)
result = self.funDecl(isAsync = true)
of Generator:
discard self.step()
result = self.funDecl(isGenerator=true)
result = self.funDecl(isGenerator = true)
of Operator:
discard self.step()
result = self.funDecl(isOperator=true)
of Type, TokenType.Whitespace, TokenType.Tab, Comment:
# TODO: Comments, pragmas, docstrings
discard self.step() # TODO
return newNilExpr(Token(lexeme: "nil"))
result = self.funDecl(isOperator = true)
of TokenType.Comment:
let tok = self.step()
if tok.lexeme.startsWith("#pragma["):
result = self.parsePragma()
of Type, TokenType.Whitespace, TokenType.Tab:
discard self.step() # TODO
else:
result = Declaration(self.statement())
proc parse*(self: Parser, tokens: seq[Token], file: string): seq[ASTNode] =
proc parse*(self: Parser, tokens: seq[Token], file: string): seq[Declaration] =
## Parses a series of tokens into an AST node
self.tokens = tokens
self.file = file
@ -1055,10 +1122,11 @@ proc parse*(self: Parser, tokens: seq[Token], file: string): seq[ASTNode] =
self.currentFunction = nil
self.scopeDepth = 0
self.operators = newOperatorTable()
self.tree = @[]
for i, token in self.tokens:
# We do a first pass over the tokens
# to find operators. Note that this
# relies on the lexer ending the input
# to find operators. Note that this
# relies on the lexer ending the input
# with an EOF token
if token.kind == Operator:
if i == self.tokens.high():
@ -1069,4 +1137,7 @@ proc parse*(self: Parser, tokens: seq[Token], file: string): seq[ASTNode] =
# well perform some extra checks
self.error("invalid state: found malformed tokenizer input while looking for operators (missing EOF)")
while not self.done():
result.add(self.declaration())
self.tree.add(self.declaration())
if self.tree[^1] == nil:
self.tree.delete(self.tree.high())
result = self.tree

8
src/memory/allocator.nim
View File

@ -51,12 +51,14 @@ proc reallocate*(p: pointer, oldSize: int, newSize: int): pointer =
echo &"DEBUG - Memory manager: Warning, asked to realloc() nil pointer from {oldSize} to {newSize} bytes, ignoring request"
except NilAccessDefect:
stderr.write("JAPL: could not manage memory, segmentation fault\n")
quit(139) # For now, there's not much we can do if we can't get the memory we need, so we exit
quit(139) # For now, there's not much we can do if we can't get the memory we need, so we exit
template resizeArray*(kind: untyped, pointr: pointer, oldCount, newCount: int): untyped =
template resizeArray*(kind: untyped, pointr: pointer, oldCount,
newCount: int): untyped =
## Handy macro (in the C sense of macro, not nim's) to resize a dynamic array
cast[ptr UncheckedArray[kind]](reallocate(pointr, sizeof(kind) * oldCount, sizeof(kind) * newCount))
cast[ptr UncheckedArray[kind]](reallocate(pointr, sizeof(kind) * oldCount,
sizeof(kind) * newCount))
template freeArray*(kind: untyped, pointr: pointer, oldCount: int): untyped =

17
src/test.nim
View File

@ -42,7 +42,7 @@ when isMainModule:
var
keep = true
tokens: seq[Token] = @[]
tree: seq[ASTNode] = @[]
tree: seq[Declaration] = @[]
compiled: Chunk
serialized: Serialized
serializedRaw: seq[byte]
@ -66,7 +66,8 @@ when isMainModule:
if input.len() == 0:
continue
# Currently the parser doesn't handle these tokens well
tokens = filter(tokenizer.lex(input, "stdin"), proc (x: Token): bool = x.kind notin {TokenType.Whitespace, Tab})
tokens = filter(tokenizer.lex(input, "stdin"), proc (
x: Token): bool = x.kind notin {TokenType.Whitespace, Tab})
if tokens.len() == 0:
continue
when debugLexer:
@ -157,7 +158,7 @@ proc fillSymbolTable(tokenizer: Lexer) =
## Initializes the Lexer's symbol
## table with the builtin symbols
## and keywords
# 1-byte symbols
tokenizer.symbols.addSymbol("{", LeftBrace)
tokenizer.symbols.addSymbol("}", RightBrace)
@ -196,10 +197,10 @@ proc fillSymbolTable(tokenizer: Lexer) =
tokenizer.symbols.addKeyword("import", Import)
tokenizer.symbols.addKeyword("yield", TokenType.Yield)
tokenizer.symbols.addKeyword("return", TokenType.Return)
# These are more like expressions with a reserved
# name that produce a value of a builtin type,
# but we don't need to care about that until
# we're in the parsing/ compilation steps so
# These are more like expressions with a reserved
# name that produce a value of a builtin type,
# but we don't need to care about that until
# we're in the parsing/ compilation steps so
# it's fine
tokenizer.symbols.addKeyword("nan", NotANumber)
tokenizer.symbols.addKeyword("inf", Infinity)
@ -217,4 +218,4 @@ proc getLineEditor: LineEditor =
result.prompt = "=> "
result.populateDefaults()
let history = result.plugHistory()
result.bindHistory(history)
result.bindHistory(history)

25
src/util/debugger.nim
View File

@ -52,9 +52,11 @@ proc simpleInstruction(instruction: OpCode, offset: int): int =
return offset + 1
proc stackTripleInstruction(instruction: OpCode, chunk: Chunk, offset: int): int =
proc stackTripleInstruction(instruction: OpCode, chunk: Chunk,
offset: int): int =
## Debugs instructions that operate on a single value on the stack using a 24-bit operand
var slot = [chunk.code[offset + 1], chunk.code[offset + 2], chunk.code[offset + 3]].fromTriple()
var slot = [chunk.code[offset + 1], chunk.code[offset + 2], chunk.code[
offset + 3]].fromTriple()
printInstruction(instruction)
stdout.write(&", points to index ")
setForegroundColor(fgYellow)
@ -63,7 +65,8 @@ proc stackTripleInstruction(instruction: OpCode, chunk: Chunk, offset: int): int
return offset + 4
proc stackDoubleInstruction(instruction: OpCode, chunk: Chunk, offset: int): int =
proc stackDoubleInstruction(instruction: OpCode, chunk: Chunk,
offset: int): int =
## Debugs instructions that operate on a single value on the stack using a 16-bit operand
var slot = [chunk.code[offset + 1], chunk.code[offset + 2]].fromDouble()
printInstruction(instruction)
@ -74,7 +77,8 @@ proc stackDoubleInstruction(instruction: OpCode, chunk: Chunk, offset: int): int
return offset + 3
proc argumentDoubleInstruction(instruction: OpCode, chunk: Chunk, offset: int): int =
proc argumentDoubleInstruction(instruction: OpCode, chunk: Chunk,
offset: int): int =
## Debugs instructions that operate on a hardcoded value on the stack using a 16-bit operand
var slot = [chunk.code[offset + 1], chunk.code[offset + 2]].fromDouble()
printInstruction(instruction)
@ -87,7 +91,8 @@ proc argumentDoubleInstruction(instruction: OpCode, chunk: Chunk, offset: int):
proc constantInstruction(instruction: OpCode, chunk: Chunk, offset: int): int =
## Debugs instructions that operate on the constant table
var constant = [chunk.code[offset + 1], chunk.code[offset + 2], chunk.code[offset + 3]].fromTriple()
var constant = [chunk.code[offset + 1], chunk.code[offset + 2], chunk.code[
offset + 3]].fromTriple()
printInstruction(instruction)
stdout.write(&", points to constant at position ")
setForegroundColor(fgYellow)
@ -95,7 +100,7 @@ proc constantInstruction(instruction: OpCode, chunk: Chunk, offset: int): int =
nl()
let obj = chunk.consts[constant]
setForegroundColor(fgGreen)
printDebug("Operand: ")
printDebug("Operand: ")
setForegroundColor(fgYellow)
stdout.write(&"{obj}\n")
setForegroundColor(fgGreen)
@ -111,10 +116,12 @@ proc jumpInstruction(instruction: OpCode, chunk: Chunk, offset: int): int =
case instruction:
of Jump, JumpIfFalse, JumpIfTrue, JumpIfFalsePop, JumpForwards, JumpBackwards:
jump = [chunk.code[offset + 1], chunk.code[offset + 2]].fromDouble().int()
of LongJump, LongJumpIfFalse, LongJumpIfTrue, LongJumpIfFalsePop, LongJumpForwards, LongJumpBackwards:
jump = [chunk.code[offset + 1], chunk.code[offset + 2], chunk.code[offset + 3]].fromTriple().int()
of LongJump, LongJumpIfFalse, LongJumpIfTrue, LongJumpIfFalsePop,
LongJumpForwards, LongJumpBackwards:
jump = [chunk.code[offset + 1], chunk.code[offset + 2], chunk.code[
offset + 3]].fromTriple().int()
else:
discard # Unreachable
discard # Unreachable
printInstruction(instruction, true)
printDebug("Jump size: ")
setForegroundColor(fgYellow)

22
src/util/serializer.nim
View File

@ -33,7 +33,7 @@ type
filename: string
chunk: Chunk
Serialized* = ref object
## Wrapper returned by
## Wrapper returned by
## the Serializer.read*
## procedures to store
## metadata
@ -51,7 +51,8 @@ proc `$`*(self: Serialized): string =
proc error(self: Serializer, message: string) =
## Raises a formatted SerializationError exception
raise newException(SerializationError, &"A fatal error occurred while (de)serializing '{self.filename}' -> {message}")
raise newException(SerializationError,
&"A fatal error occurred while (de)serializing '{self.filename}' -> {message}")
proc newSerializer*(self: Serializer = nil): Serializer =
@ -98,7 +99,7 @@ proc extend[T](s: var seq[T], a: openarray[T]) =
s.add(e)
proc writeHeaders(self: Serializer, stream: var seq[byte], file: string) =
proc writeHeaders(self: Serializer, stream: var seq[byte], file: string) =
## Writes the Peon bytecode headers in-place into a byte stream
stream.extend(self.toBytes(BYTECODE_MARKER))
stream.add(byte(PEON_VERSION.major))
@ -138,7 +139,7 @@ proc writeConstants(self: Serializer, stream: var seq[byte]) =
else:
strip = 2
temp = 0x0
stream.extend((len(constant.token.lexeme) - strip).toTriple()) # Removes the quotes from the length count as they're not written
stream.extend((len(constant.token.lexeme) - strip).toTriple()) # Removes the quotes from the length count as they're not written
stream.add(temp)
stream.add(self.toBytes(constant.token.lexeme[offset..^2]))
of identExpr:
@ -147,7 +148,7 @@ proc writeConstants(self: Serializer, stream: var seq[byte]) =
stream.add(self.toBytes(constant.token.lexeme))
else:
self.error(&"unknown constant kind in chunk table ({constant.kind})")
stream.add(0x59) # End marker
stream.add(0x59) # End marker
proc readConstants(self: Serializer, stream: seq[byte]): int =
@ -204,7 +205,8 @@ proc readConstants(self: Serializer, stream: seq[byte]): int =
stream = stream[1..^1]
let size = self.bytesToInt([stream[0], stream[1], stream[2]])
stream = stream[3..^1]
self.chunk.consts.add(newIdentExpr(Token(lexeme: self.bytesToString(stream[0..<size]))))
self.chunk.consts.add(newIdentExpr(Token(
lexeme: self.bytesToString(stream[0..<size]))))
# TODO
# discard self.chunk.addConstant(newIdentExpr(Token(lexeme: self.bytesToString(stream[0..<size]))))
stream = stream[size..^1]
@ -263,7 +265,8 @@ proc loadBytes*(self: Serializer, stream: seq[byte]): Serialized =
if stream[0..<len(BYTECODE_MARKER)] != self.toBytes(BYTECODE_MARKER):
self.error("malformed bytecode marker")
stream = stream[len(BYTECODE_MARKER)..^1]
result.peonVer = (major: int(stream[0]), minor: int(stream[1]), patch: int(stream[2]))
result.peonVer = (major: int(stream[0]), minor: int(stream[1]),
patch: int(stream[2]))
stream = stream[3..^1]
let branchLength = stream[0]
stream = stream[1..^1]
@ -271,7 +274,8 @@ proc loadBytes*(self: Serializer, stream: seq[byte]): Serialized =
stream = stream[branchLength..^1]
result.commitHash = self.bytesToString(stream[0..<40]).toLowerAscii()
stream = stream[40..^1]
result.compileDate = self.bytesToInt([stream[0], stream[1], stream[2], stream[3], stream[4], stream[5], stream[6], stream[7]])
result.compileDate = self.bytesToInt([stream[0], stream[1], stream[2],
stream[3], stream[4], stream[5], stream[6], stream[7]])
stream = stream[8..^1]
result.fileHash = self.bytesToString(stream[0..<32]).toHex().toLowerAscii()
stream = stream[32..^1]
@ -295,4 +299,4 @@ proc loadFile*(self: Serializer, src: string): Serialized =
while pos < size:
discard fp.readBytes(data, pos, size)
pos = fp.getFilePos()
return self.loadBytes(data)
return self.loadBytes(data)