nocturn9x
/
peon
2
2
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Various fixes to the GC, added a few more tests

This commit is contained in:
Mattia Giambirtone 2022-08-18 19:18:29 +02:00
parent 95315a0094
commit 4fdd90614a
10 changed files with 329 additions and 191 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

137
src/backend/vm.nim
View File

@ -17,6 +17,7 @@ import std/monotimes
import std/math import std/math
import std/segfaults import std/segfaults
import std/strutils import std/strutils
import std/sequtils
import std/sets import std/sets
@ -81,6 +82,7 @@ type
pointers: HashSet[uint64] pointers: HashSet[uint64]
objects: seq[ptr HeapObject] objects: seq[ptr HeapObject]
# Implementation of peon's memory manager # Implementation of peon's memory manager
proc newPeonGC*: PeonGC = proc newPeonGC*: PeonGC =
@ -96,10 +98,9 @@ proc collect*(self: PeonGC)
proc reallocate*(self: PeonGC, p: pointer, oldSize: int, newSize: int): pointer = proc reallocate*(self: PeonGC, p: pointer, oldSize: int, newSize: int): pointer =
## Simple wrapper around realloc/dealloc ## Simple wrapper around realloc/dealloc with
## built-in garbage collection
self.bytesAllocated.current += newSize - oldSize self.bytesAllocated.current += newSize - oldSize
if self.bytesAllocated.current > self.nextGC:
self.collect()
try: try:
if newSize == 0 and not p.isNil(): if newSize == 0 and not p.isNil():
when debugMem: when debugMem:
@ -112,6 +113,9 @@ proc reallocate*(self: PeonGC, p: pointer, oldSize: int, newSize: int): pointer
self.bytesAllocated.total += newSize - oldSize self.bytesAllocated.total += newSize - oldSize
when debugStressGC: when debugStressGC:
self.collect() self.collect()
else:
if self.bytesAllocated.current > self.nextGC:
self.collect()
when debugMem: when debugMem:
if oldSize == 0: if oldSize == 0:
if newSize > 1: if newSize > 1:
@ -166,7 +170,7 @@ proc allocate*(self: PeonGC, kind: ObjectKind, size: typedesc, count: int): ptr
proc mark(self: ptr HeapObject): bool = proc mark(self: ptr HeapObject): bool =
## Marks a single object ## Marks a single object
if self.isNil() or self.marked: if self.marked:
return false return false
self.marked = true self.marked = true
return true return true
@ -186,17 +190,20 @@ proc markRoots(self: PeonGC): seq[ptr HeapObject] =
# along with its type and other metadata. Then, # along with its type and other metadata. Then,
# we can go through the various sources of roots # we can go through the various sources of roots
# in the VM, see if they match any pointers we # in the VM, see if they match any pointers we
# already know about (we store them a hash set so # already know about (we store them in a hash set so
# it's really fast), and then we can be sure that # it's really fast), and then we can be sure that
# anything that's in the difference (i.e. mathematical # anything that's in the difference (i.e. mathematical
# set difference) between our full list of pointers # set difference) between our full list of pointers
# and the live ones is not a root object, so if it's not # and the live ones is not a root object, so if it's
# indirectly reachable through a root itself, it can be # not indirectly reachable through a root itself, it
# freed. I'm not sure if I can call this GC strategy precise, # can be freed. I'm not sure if I can call this GC
# since technically there is a chance for a regular value to # strategy precise, since technically there is a chance
# collide with one of the pointers we allocated and that would # for a regular value to collide with one of the pointers
# cause a memory leak, but with a 64-bit address-space it probably # we allocated and that would cause a memory leak, but
# hardly matters, so I guess this is a mostly-precise Mark&Sweep collector # with a 64-bit address-space it probably hardly matters,
# so I guess this is a mostly-precise Mark&Sweep collector
when debugGC:
echo "DEBUG - GC: Starting mark phase"
var live = initHashSet[uint64]() var live = initHashSet[uint64]()
for obj in self.vm.calls: for obj in self.vm.calls:
if obj in self.pointers: if obj in self.pointers:
@ -213,10 +220,11 @@ proc markRoots(self: PeonGC): seq[ptr HeapObject] =
for p in live: for p in live:
obj = cast[ptr HeapObject](p) obj = cast[ptr HeapObject](p)
if obj.mark(): if obj.mark():
result.add(obj) when debugGC:
when debugMem:
if obj.marked:
echo &"DEBUG - GC: Marking object: {obj[]}" echo &"DEBUG - GC: Marking object: {obj[]}"
result.add(obj)
when debugGC:
echo "DEBUG - GC: Mark phase complete"
proc trace(self: PeonGC, roots: seq[ptr HeapObject]) = proc trace(self: PeonGC, roots: seq[ptr HeapObject]) =
@ -224,24 +232,34 @@ proc trace(self: PeonGC, roots: seq[ptr HeapObject]) =
## objects starting from the ## objects starting from the
## roots. The second argument ## roots. The second argument
## is the output of the mark ## is the output of the mark
## phase ## phase. To speak in terms
## of the tricolor abstraction,
## this is where we blacken gray
## objects
when debugGC:
echo &"DEBUG - GC: Tracing indirect references from {len(roots)} roots"
for root in roots: for root in roots:
case root.kind: case root.kind:
of String: of String:
discard # No additional references discard # Strings hold no additional references
else: else:
discard # TODO discard # TODO: Other types
when debugGC:
echo &"DEBUG - GC: Tracing phase complete"
proc free(self: PeonGC, obj: ptr HeapObject) = proc free(self: PeonGC, obj: ptr HeapObject) =
## Frees a single heap-allocated ## Frees a single heap-allocated
## peon object and all the memory ## peon object and all the memory
## it directly or indirectly owns ## it directly or indirectly owns
when debugAlloc:
echo &"DEBUG - GC: Freeing object: {obj[]}"
case obj.kind: case obj.kind:
of String: of String:
# Strings only own their # Strings only own their
# underlying character array # underlying character array
self.freeArray(char, obj.str, obj.len) if obj.len > 0 and not obj.str.isNil():
self.freeArray(char, obj.str, obj.len)
else: else:
discard # TODO discard # TODO
self.free(HeapObject, obj) self.free(HeapObject, obj)
@ -254,9 +272,15 @@ proc sweep(self: PeonGC) =
## during the mark phase. ## during the mark phase.
## This is more convoluted ## This is more convoluted
## than it needs to be because ## than it needs to be because
## nim disallows ## nim disallows changing the
## size of a sequence during
## iteration
when debugGC:
echo "DEBUG - GC: Beginning sweeping phase"
var j = -1 var j = -1
var idx = 0 var idx = 0
var count = 0
while j < self.objects.high(): while j < self.objects.high():
inc(j) inc(j)
if self.objects[j].marked: if self.objects[j].marked:
@ -264,27 +288,34 @@ proc sweep(self: PeonGC) =
# but reset its mark so that it doesn't # but reset its mark so that it doesn't
# stay alive forever # stay alive forever
self.objects[j].marked = false self.objects[j].marked = false
continue when debugGC:
echo &"DEBUG - GC: Unmarking object: {self.objects[j][]}"
inc(idx)
else: else:
# Object is unmarked: its memory is # Object is unmarked: its memory is
# fair game # fair game
self.free(self.objects[idx]) self.free(self.objects[idx])
self.objects.delete(idx) self.objects.delete(idx)
idx += 1 inc(idx)
inc(count)
when debugGC:
echo &"DEBUG - GC: Swept {count} objects"
proc collect(self: PeonGC) = proc collect(self: PeonGC) =
## Attempts to reclaim some ## Attempts to reclaim some
## memory from unreachable ## memory from unreachable
## objects onto the heap ## objects onto the heap
let before = self.bytesAllocated.current let before {.used.} = self.bytesAllocated.current
let time {.used.} = getMonoTime().ticks().float() / 1_000_000
when debugGC: when debugGC:
echo "DEBUG - GC: Starting collection cycle" echo &"DEBUG - GC: Starting collection cycle at heap size {self.bytesAllocated.current}"
self.trace(self.markRoots()) self.trace(self.markRoots())
self.sweep() self.sweep()
self.nextGC = self.bytesAllocated.current * HeapGrowFactor self.nextGC = self.bytesAllocated.current * HeapGrowFactor
when debugGC: when debugGC:
echo &"DEBUG - GC: Collection cycle has terminated, collected {before - self.bytesAllocated.current} bytes of memory in total" echo &"DEBUG - GC: Collection cycle has terminated in {getMonoTime().ticks().float() / 1_000_000 - time:.2f} ms, collected {before - self.bytesAllocated.current} bytes of memory in total"
echo &"DEBUG - GC: Next cycle at {self.nextGC} bytes"
proc initCache*(self: PeonVM) = proc initCache*(self: PeonVM) =
@ -303,11 +334,13 @@ proc newPeonVM*: PeonVM =
## for executing Peon bytecode ## for executing Peon bytecode
new(result) new(result)
result.ip = 0 result.ip = 0
result.frames = @[]
result.calls = newSeq[uint64]()
result.operands = newSeq[uint64]()
result.initCache() result.initCache()
result.gc = newPeonGC() result.gc = newPeonGC()
result.frames = @[]
result.calls = @[]
result.operands = @[]
result.results = @[]
result.closedOver = @[]
result.gc.vm = result result.gc.vm = result
@ -373,7 +406,7 @@ proc peek(self: PeonVM, distance: int = 0): uint64 =
## given distance from the top of ## given distance from the top of
## the operand stack without consuming it ## the operand stack without consuming it
if distance < 0: if distance < 0:
return self.peekb(^(-distance)) return self.peekb(^(-int(distance)))
return self.operands[self.operands.high() + distance] return self.operands[self.operands.high() + distance]
@ -396,7 +429,7 @@ proc peekc(self: PeonVM, distance: int = 0): uint64 {.used.} =
return self.calls[self.calls.high() + distance] return self.calls[self.calls.high() + distance]
proc getc(self: PeonVM, idx: uint): uint64 = proc getc(self: PeonVM, idx: uint64): uint64 =
## Accessor method that abstracts ## Accessor method that abstracts
## indexing our call stack through stack ## indexing our call stack through stack
## frames ## frames
@ -550,6 +583,8 @@ proc constReadString(self: PeonVM, size, idx: int): ptr HeapObject =
result = self.gc.allocate(String, char, len(str)) result = self.gc.allocate(String, char, len(str))
for i, c in str: for i, c in str:
result.str[i] = c result.str[i] = c
when debugAlloc:
echo &"DEBUG - GC: Allocated new object: {result[]}"
{.pop.} {.pop.}
@ -576,7 +611,7 @@ when debugVM: # So nim shuts up
styledEcho fgMagenta, "]" styledEcho fgMagenta, "]"
if self.frames.len() !> 0: if self.frames.len() !> 0:
stdout.styledWrite(fgCyan, "Current Frame: ", fgMagenta, "[") stdout.styledWrite(fgCyan, "Current Frame: ", fgMagenta, "[")
for i, e in self.calls[self.frames[^1]..^1]: for i, e in self.calls[self.frames[^1]..self.calls.high()]:
stdout.styledWrite(fgYellow, $e) stdout.styledWrite(fgYellow, $e)
if i < self.calls.high(): if i < self.calls.high():
stdout.styledWrite(fgYellow, ", ") stdout.styledWrite(fgYellow, ", ")
@ -722,7 +757,7 @@ proc dispatch*(self: PeonVM) =
# into the given call stack index # into the given call stack index
let idx = self.readLong() let idx = self.readLong()
when debugVM: when debugVM:
assert idx.int - self.calls.high() <= 1, "StoreVar index is bigger than the length of the call stack" assert idx - self.calls.high() <= 1, "StoreVar index is bigger than the length of the call stack"
if idx + self.frames[^1] <= self.calls.high().uint: if idx + self.frames[^1] <= self.calls.high().uint:
self.setc(idx, self.pop()) self.setc(idx, self.pop())
else: else:
@ -786,16 +821,17 @@ proc dispatch*(self: PeonVM) =
self.ip += self.readLong() self.ip += self.readLong()
of JumpIfTrue: of JumpIfTrue:
# Conditional positive jump # Conditional positive jump
let ip = self.readLong()
if self.peek().bool: if self.peek().bool:
self.ip += self.readLong() self.ip += ip
of JumpIfFalsePop: of JumpIfFalsePop:
let ip = self.readLong()
if not self.pop().bool:
self.ip += ip
of JumpIfFalseOrPop:
let ip = self.readLong() let ip = self.readLong()
if not self.peek().bool: if not self.peek().bool:
self.ip += ip self.ip += ip
discard self.pop()
of JumpIfFalseOrPop:
if not self.peek().bool:
self.ip += self.readLong()
else: else:
discard self.pop() discard self.pop()
# Built-in operations on primitive types. # Built-in operations on primitive types.
@ -934,7 +970,32 @@ proc run*(self: PeonVM, chunk: Chunk) =
self.frames = @[] self.frames = @[]
self.calls = @[] self.calls = @[]
self.operands = @[] self.operands = @[]
self.results = @[]
self.ip = 0 self.ip = 0
self.dispatch() #[
# Sorry, but there only is enough space
# for one GC in this VM :(
when defined(gcOrc):
GC_disableOrc()
when not defined(gcArc):
GC_disable()
GC_disableMarkAndSweep()
]#
try:
self.dispatch()
except NilAccessDefect:
stderr.writeLine("Memory Access Violation: SIGSEGV")
quit(1)
# We clean up after ourselves!
self.gc.collect()
#[
# This is unnecessary if we use ARC,
# but *just in case*
when defined(gcOrc):
GC_enable_Orc()
when not defined(gcArc):
GC_enable()
GC_enableMarkAndSweep()
]#
{.pop.} {.pop.}

25
src/config.nim
View File

@ -14,18 +14,19 @@
import strformat import strformat
# Debug various components of peon # These variables can be tweaked to debug and test various components of the toolchain
const debugLexer* {.booldefine.} = false const debugLexer* {.booldefine.} = false # Print the tokenizer's output
const debugParser* {.booldefine.} = false const debugParser* {.booldefine.} = false # Print the AST generated by the parser
const debugCompiler* {.booldefine.} = false const debugCompiler* {.booldefine.} = false # Disassemble and print the bytecode generated by the compiler
const debugVM* {.booldefine.} = false const debugVM* {.booldefine.} = false # Run the VM in debug mode and show stack and instruction info
const debugGC* {.booldefine.} = false const debugGC* {.booldefine.} = false # Debug the Garbage Collector (extremely verbose)
const debugMem* {.booldefine.} = false const debugAlloc* {.booldefine.} = false # Trace object allocation (extremely verbose)
const debugSerializer* {.booldefine.} = false const debugMem* {.booldefine.} = false # Debug the memory allocator (extremely verbose)
const debugStressGC* {.booldefine.} = false const debugSerializer* {.booldefine.} = false # Validate the bytecode serializer's output
const PeonBytecodeMarker* = "PEON_BYTECODE" const debugStressGC* {.booldefine.} = false # Make the GC run a collection at every allocation (VERY SLOW!)
const HeapGrowFactor* = 2 # How much extra memory to allocate for dynamic arrays and garbage collection when resizing const PeonBytecodeMarker* = "PEON_BYTECODE" # Magic value at the beginning of bytecode files
const FirstGC* = 1024 * 1024; const HeapGrowFactor* = 2 # The growth factor used by the GC to schedule the next collection
const FirstGC* = 1024 * 1024; # How many bytes to allocate before running the first GC
when HeapGrowFactor <= 1: when HeapGrowFactor <= 1:
{.fatal: "Heap growth factor must be > 1".} {.fatal: "Heap growth factor must be > 1".}
const PeonVersion* = (major: 0, minor: 1, patch: 0) const PeonVersion* = (major: 0, minor: 1, patch: 0)

66
src/frontend/compiler.nim
View File

@ -296,17 +296,26 @@ proc emitBytes(self: Compiler, bytarr: openarray[OpCode | uint8]) {.inline.} =
proc makeConstant(self: Compiler, val: Expression, typ: Type): array[3, uint8] = proc makeConstant(self: Compiler, val: Expression, typ: Type): array[3, uint8] =
## Adds a constant to the current chunk's constant table ## Adds a constant to the current chunk's constant table
## and returns its index as a 3-byte array of uint8s ## and returns its index as a 3-byte array of uint8s
var v: int var lit: string
discard parseInt(val.token.lexeme, v) if typ.kind in [UInt8, Int8, Int16, UInt16, Int32, UInt32, Int64, UInt64]:
lit = val.token.lexeme
if "'" in lit:
var idx = lit.high()
while lit[idx] != '\'':
lit = lit[0..^2]
dec(idx)
lit = lit[0..^2]
case typ.kind: case typ.kind:
of UInt8, Int8: of UInt8, Int8:
result = self.chunk.writeConstant([uint8(v)]) result = self.chunk.writeConstant([uint8(parseInt(lit))])
of Int16, UInt16: of Int16, UInt16:
result = self.chunk.writeConstant(v.toDouble()) result = self.chunk.writeConstant(parseInt(lit).toDouble())
of Int32, UInt32: of Int32, UInt32:
result = self.chunk.writeConstant(v.toQuad()) result = self.chunk.writeConstant(parseInt(lit).toQuad())
of Int64, UInt64: of Int64:
result = self.chunk.writeConstant(v.toLong()) result = self.chunk.writeConstant(parseInt(lit).toLong())
of UInt64:
result = self.chunk.writeConstant(parseBiggestUInt(lit).toLong())
of String: of String:
result = self.chunk.writeConstant(val.token.lexeme[1..^1].toBytes()) result = self.chunk.writeConstant(val.token.lexeme[1..^1].toBytes())
of Float32: of Float32:
@ -372,10 +381,16 @@ proc patchJump(self: Compiler, offset: int) =
var jump: int = self.chunk.code.len() - offset var jump: int = self.chunk.code.len() - offset
if jump > 16777215: if jump > 16777215:
self.error("cannot jump more than 16777215 instructions") self.error("cannot jump more than 16777215 instructions")
# We subtract 4 because that's the size of our jump instruction case OpCode(self.chunk.code[offset]):
# which the caller of patchJump doesn't take into account (and of JumpBackwards, Jump, JumpIfFalsePop, JumpIfFalse:
# that's by design) # We subtract 4 because backwards
let offsetArray = (jump - 4).toTriple() # and absolute jumps don't take
# the size of the jump offset
# into account
jump -= 4
else:
discard
let offsetArray = jump.toTriple()
self.chunk.code[offset + 1] = offsetArray[0] self.chunk.code[offset + 1] = offsetArray[0]
self.chunk.code[offset + 2] = offsetArray[1] self.chunk.code[offset + 2] = offsetArray[1]
self.chunk.code[offset + 3] = offsetArray[2] self.chunk.code[offset + 3] = offsetArray[2]
@ -852,14 +867,21 @@ proc literal(self: Compiler, node: ASTNode) =
of strExpr: of strExpr:
self.emitConstant(LiteralExpr(node), Type(kind: String)) self.emitConstant(LiteralExpr(node), Type(kind: String))
of intExpr: of intExpr:
var x: int let y = IntExpr(node)
var y = IntExpr(node) let kind = self.inferType(y)
try: if kind.kind in [Int64, Int32, Int16, Int8]:
discard parseInt(y.literal.lexeme, x) var x: int
except ValueError: try:
self.error("integer value out of range") discard parseInt(y.literal.lexeme, x)
except ValueError:
self.emitConstant(y, self.inferType(y)) self.error("integer value out of range")
else:
var x: uint64
try:
discard parseBiggestUInt(y.literal.lexeme, x)
except ValueError:
self.error("integer value out of range")
self.emitConstant(y, kind)
of hexExpr: of hexExpr:
var x: int var x: int
var y = HexExpr(node) var y = HexExpr(node)
@ -903,7 +925,7 @@ proc literal(self: Compiler, node: ASTNode) =
var x: float var x: float
var y = FloatExpr(node) var y = FloatExpr(node)
try: try:
discard parseFloat(y.literal.lexeme, x) discard parseFloat(y.literal.lexeme)
except ValueError: except ValueError:
self.error("floating point value out of range") self.error("floating point value out of range")
self.emitConstant(y, self.inferType(y)) self.emitConstant(y, self.inferType(y))
@ -919,7 +941,7 @@ proc literal(self: Compiler, node: ASTNode) =
proc handleBuiltinFunction(self: Compiler, fn: Name, args: seq[Expression]) = proc handleBuiltinFunction(self: Compiler, fn: Name, args: seq[Expression]) =
## Emits instructions for builtin functions ## Emits instructions for builtin functions
## such as addition or subtraction ## such as addition or subtraction
if fn.valueType.builtinOp notin ["GenericLogicalOr", "GenericLogicalAnd"]: if fn.valueType.builtinOp notin ["LogicalOr", "LogicalAnd"]:
if len(args) == 2: if len(args) == 2:
self.expression(args[1]) self.expression(args[1])
self.expression(args[0]) self.expression(args[0])
@ -1314,8 +1336,8 @@ proc whileStmt(self: Compiler, node: WhileStmt) =
self.expression(node.condition) self.expression(node.condition)
let jump = self.emitJump(JumpIfFalsePop) let jump = self.emitJump(JumpIfFalsePop)
self.statement(node.body) self.statement(node.body)
self.patchJump(jump)
self.emitLoop(start) self.emitLoop(start)
self.patchJump(jump)
proc checkCallIsPure(self: Compiler, node: ASTnode): bool = proc checkCallIsPure(self: Compiler, node: ASTnode): bool =

18
src/frontend/parser.nim
View File

@ -451,7 +451,7 @@ proc call(self: Parser): Expression =
proc unary(self: Parser): Expression = proc unary(self: Parser): Expression =
## Parses unary expressions ## Parses unary expressions
if self.peek().kind == Symbol and self.peek().lexeme in self.operators.tokens: if self.peek().kind in [Identifier, Symbol] and self.peek().lexeme in self.operators.tokens:
result = newUnaryExpr(self.step(), self.unary()) result = newUnaryExpr(self.step(), self.unary())
else: else:
result = self.call() result = self.call()
@ -462,7 +462,7 @@ proc parsePow(self: Parser): Expression =
result = self.unary() result = self.unary()
var operator: Token var operator: Token
var right: Expression var right: Expression
while self.peek().kind == Symbol and self.operators.getPrecedence(self.peek().lexeme) == Power: while self.peek().kind in [Identifier, Symbol] and self.operators.getPrecedence(self.peek().lexeme) == Power:
operator = self.step() operator = self.step()
right = self.unary() right = self.unary()
result = newBinaryExpr(result, operator, right) result = newBinaryExpr(result, operator, right)
@ -474,7 +474,7 @@ proc parseMul(self: Parser): Expression =
result = self.parsePow() result = self.parsePow()
var operator: Token var operator: Token
var right: Expression var right: Expression
while self.peek().kind == Symbol and self.operators.getPrecedence(self.peek().lexeme) == Multiplication: while self.peek().kind in [Identifier, Symbol] and self.operators.getPrecedence(self.peek().lexeme) == Multiplication:
operator = self.step() operator = self.step()
right = self.parsePow() right = self.parsePow()
result = newBinaryExpr(result, operator, right) result = newBinaryExpr(result, operator, right)
@ -486,7 +486,7 @@ proc parseAdd(self: Parser): Expression =
result = self.parseMul() result = self.parseMul()
var operator: Token var operator: Token
var right: Expression var right: Expression
while self.peek().kind == Symbol and self.operators.getPrecedence(self.peek().lexeme) == Addition: while self.peek().kind in [Identifier, Symbol] and self.operators.getPrecedence(self.peek().lexeme) == Addition:
operator = self.step() operator = self.step()
right = self.parseMul() right = self.parseMul()
result = newBinaryExpr(result, operator, right) result = newBinaryExpr(result, operator, right)
@ -497,7 +497,7 @@ proc parseCmp(self: Parser): Expression =
result = self.parseAdd() result = self.parseAdd()
var operator: Token var operator: Token
var right: Expression var right: Expression
while self.peek().kind == Symbol and self.operators.getPrecedence(self.peek().lexeme) == Compare: while self.peek().kind in [Identifier, Symbol] and self.operators.getPrecedence(self.peek().lexeme) == Compare:
operator = self.step() operator = self.step()
right = self.parseAdd() right = self.parseAdd()
result = newBinaryExpr(result, operator, right) result = newBinaryExpr(result, operator, right)
@ -508,7 +508,7 @@ proc parseAnd(self: Parser): Expression =
result = self.parseCmp() result = self.parseCmp()
var operator: Token var operator: Token
var right: Expression var right: Expression
while self.peek().kind == Symbol and self.operators.getPrecedence(self.peek().lexeme) == Precedence.And: while self.peek().kind in [Identifier, Symbol] and self.operators.getPrecedence(self.peek().lexeme) == Precedence.And:
operator = self.step() operator = self.step()
right = self.parseCmp() right = self.parseCmp()
result = newBinaryExpr(result, operator, right) result = newBinaryExpr(result, operator, right)
@ -519,7 +519,7 @@ proc parseOr(self: Parser): Expression =
result = self.parseAnd() result = self.parseAnd()
var operator: Token var operator: Token
var right: Expression var right: Expression
while self.peek().kind == Symbol and self.operators.getPrecedence(self.peek().lexeme) == Precedence.Or: while self.peek().kind in [Identifier, Symbol] and self.operators.getPrecedence(self.peek().lexeme) == Precedence.Or:
operator = self.step() operator = self.step()
right = self.parseAnd() right = self.parseAnd()
result = newBinaryExpr(result, operator, right) result = newBinaryExpr(result, operator, right)
@ -528,7 +528,7 @@ proc parseOr(self: Parser): Expression =
proc parseAssign(self: Parser): Expression = proc parseAssign(self: Parser): Expression =
## Parses assignment expressions ## Parses assignment expressions
result = self.parseOr() result = self.parseOr()
if self.peek().kind == Symbol and self.operators.getPrecedence(self.peek().lexeme) == Assign: if self.peek().kind in [Identifier, Symbol] and self.operators.getPrecedence(self.peek().lexeme) == Assign:
let tok = self.step() let tok = self.step()
var value = self.expression() var value = self.expression()
case result.kind: case result.kind:
@ -545,7 +545,7 @@ proc parseArrow(self: Parser): Expression =
result = self.parseAssign() result = self.parseAssign()
var operator: Token var operator: Token
var right: Expression var right: Expression
while self.peek().kind == Symbol and self.operators.getPrecedence(self.peek().lexeme) == Precedence.Or: while self.peek().kind in [Identifier, Symbol] and self.operators.getPrecedence(self.peek().lexeme) == Precedence.Or:
operator = self.step() operator = self.step()
right = self.parseAssign() right = self.parseAssign()
result = newBinaryExpr(result, operator, right) result = newBinaryExpr(result, operator, right)

8
src/util/multibyte.nim
View File

@ -75,9 +75,11 @@ proc toBytes*(s: int): array[8, uint8] =
proc fromBytes*(input: seq[byte]): string = proc fromBytes*(input: seq[byte]): string =
## Converts a sequence of bytes to ## Converts a sequence of bytes to
## a string ## a string
for b in input: var i = 0
result.add(char(b)) while i < input.len():
result.add(char(input[i]))
inc(i)
proc extend*[T](s: var seq[T], a: openarray[T]) = proc extend*[T](s: var seq[T], a: openarray[T]) =
## Extends s with the elements of a ## Extends s with the elements of a

16
tests/gc.pn Normal file
View File

@ -0,0 +1,16 @@
import std;
var x: uint64 = 1000000'u64;
var y = "just a test";
print(y);
print("Starting GC torture test");
print(x);
while x > 0'u64 {
"hello";
x = x - 1'u64;
}
print("END");
print(y);
y = "test";
print(y);
"";

6
tests/import_a.pn Normal file
View File

@ -0,0 +1,6 @@
# Tests importing another module and executing it
import std;
import import_b;
print("a");

3
tests/import_b.pn Normal file
View File

@ -0,0 +1,3 @@
import std;
print("b");

22
tests/loops.pn Normal file
View File

@ -0,0 +1,22 @@
import std;
print("Counting down...");
var from = 10;
let to = 0;
while from > to {
print(from);
from = from - 1;
}
print("Done!");
print("Counting up...");
var start = 0;
let stop = 10;
while start < stop {
print(start);
start = start + 1;
}
print("Done!");

219
tests/std.pn
View File

@ -8,8 +8,10 @@
# - It makes the implementation easier and more flexible # - It makes the implementation easier and more flexible
# TODO: Use generics
operator `+`*(a, b: int): int { operator `+`*(a, b: int): int {
#pragma[magic: "SignedAdd", pure] #pragma[magic: "Add", pure]
} }
@ -19,7 +21,7 @@ operator `+`*(a, b: uint64): uint64 {
operator `+`*(a, b: int32): int32 { operator `+`*(a, b: int32): int32 {
#pragma[magic: "SignedAdd", pure] #pragma[magic: "Add", pure]
} }
@ -29,7 +31,7 @@ operator `+`*(a, b: uint32): uint32 {
operator `+`*(a, b: int16): int16 { operator `+`*(a, b: int16): int16 {
#pragma[magic: "SignedAdd", pure] #pragma[magic: "Add", pure]
} }
@ -44,157 +46,157 @@ operator `+`*(a, b: int8): int8 {
operator `+`*(a, b: uint8): uint8 { operator `+`*(a, b: uint8): uint8 {
#pragma[magic: "AddUInt8", pure] #pragma[magic: "Add", pure]
} }
operator `+`*(a, b: float64): float64 { operator `+`*(a, b: float64): float64 {
#pragma[magic: "AddFloat64", pure] #pragma[magic: "Add", pure]
} }
operator `+`*(a, b: float32): float32 { operator `+`*(a, b: float32): float32 {
#pragma[magic: "AddFloat32", pure] #pragma[magic: "Add", pure]
} }
operator `-`*(a, b: int): int { operator `-`*(a, b: int): int {
#pragma[magic: "SubInt64", pure] #pragma[magic: "Subtract", pure]
} }
operator `-`*(a, b: uint64): uint64 { operator `-`*(a, b: uint64): uint64 {
#pragma[magic: "SubUInt64", pure] #pragma[magic: "Subtract", pure]
} }
operator `-`*(a, b: int32): int32 { operator `-`*(a, b: int32): int32 {
#pragma[magic: "SubInt32", pure] #pragma[magic: "Subtract", pure]
} }
operator `-`*(a, b: uint32): uint32 { operator `-`*(a, b: uint32): uint32 {
#pragma[magic: "SubUInt32", pure] #pragma[magic: "Subtract", pure]
} }
operator `-`*(a, b: int16): int16 { operator `-`*(a, b: int16): int16 {
#pragma[magic: "SubInt16", pure] #pragma[magic: "Subtract", pure]
} }
operator `-`*(a, b: uint16): uint16 { operator `-`*(a, b: uint16): uint16 {
#pragma[magic: "SubUInt16", pure] #pragma[magic: "Subtract", pure]
} }
operator `-`*(a, b: int8): int8 { operator `-`*(a, b: int8): int8 {
#pragma[magic: "SubInt8", pure] #pragma[magic: "Subtract", pure]
} }
operator `-`*(a, b: uint8): uint8 { operator `-`*(a, b: uint8): uint8 {
#pragma[magic: "SubUInt8", pure] #pragma[magic: "Subtract", pure]
} }
operator `-`*(a, b: float64): float64 { operator `-`*(a, b: float64): float64 {
#pragma[magic: "SubFloat64", pure] #pragma[magic: "SubtractFloat64", pure]
} }
operator `-`*(a, b: float32): float32 { operator `-`*(a, b: float32): float32 {
#pragma[magic: "SubFloat32", pure] #pragma[magic: "SubtractFloat32", pure]
} }
operator `*`*(a, b: int): int { operator `*`*(a, b: int): int {
#pragma[magic: "SignedMultiply", pure] #pragma[magic: "Multiply", pure]
} }
operator `*`*(a, b: uint64): uint64 { operator `*`*(a, b: uint64): uint64 {
#pragma[magic: "MulUInt64", pure] #pragma[magic: "Multiply", pure]
} }
operator `*`*(a, b: int32): int32 { operator `*`*(a, b: int32): int32 {
#pragma[magic: "SignedMultiply", pure] #pragma[magic: "Multiply", pure]
} }
operator `*`*(a, b: uint32): uint32 { operator `*`*(a, b: uint32): uint32 {
#pragma[magic: "MulUInt32", pure] #pragma[magic: "Multiply", pure]
} }
operator `*`*(a, b: int16): int16 { operator `*`*(a, b: int16): int16 {
#pragma[magic: "SignedMultiply", pure] #pragma[magic: "Multiply", pure]
} }
operator `*`*(a, b: uint16): uint16 { operator `*`*(a, b: uint16): uint16 {
#pragma[magic: "MulUInt16", pure] #pragma[magic: "Multiply", pure]
} }
operator `*`*(a, b: int8): int8 { operator `*`*(a, b: int8): int8 {
#pragma[magic: "SignedMultiply", pure] #pragma[magic: "Multiply", pure]
} }
operator `*`*(a, b: uint8): uint8 { operator `*`*(a, b: uint8): uint8 {
#pragma[magic: "MulUInt8", pure] #pragma[magic: "Multiply", pure]
} }
operator `*`*(a, b: float64): float64 { operator `*`*(a, b: float64): float64 {
#pragma[magic: "MulFloat64", pure] #pragma[magic: "MultiplyFloat64", pure]
} }
operator `*`*(a, b: float32): float32 { operator `*`*(a, b: float32): float32 {
#pragma[magic: "MulFloat32", pure] #pragma[magic: "MultiplyFloat32", pure]
} }
operator `/`*(a, b: int): int { operator `/`*(a, b: int): int {
#pragma[magic: "DivInt64", pure] #pragma[magic: "SignedDivide", pure]
} }
operator `/`*(a, b: uint64): uint64 { operator `/`*(a, b: uint64): uint64 {
#pragma[magic: "DivUInt64", pure] #pragma[magic: "Divide", pure]
} }
operator `/`*(a, b: int32): int32 { operator `/`*(a, b: int32): int32 {
#pragma[magic: "DivInt32", pure] #pragma[magic: "SignedDivide", pure]
} }
operator `/`*(a, b: uint32): uint32 { operator `/`*(a, b: uint32): uint32 {
#pragma[magic: "DivUInt32", pure] #pragma[magic: "Divide", pure]
} }
operator `/`*(a, b: int16): int16 { operator `/`*(a, b: int16): int16 {
#pragma[magic: "DivInt16", pure] #pragma[magic: "SignedDivide", pure]
} }
operator `/`*(a, b: uint16): uint16 { operator `/`*(a, b: uint16): uint16 {
#pragma[magic: "DivUInt16", pure] #pragma[magic: "Divide", pure]
} }
operator `/`*(a, b: int8): int8 { operator `/`*(a, b: int8): int8 {
#pragma[magic: "DivInt8", pure] #pragma[magic: "SignedDivide", pure]
} }
operator `/`*(a, b: uint8): uint8 { operator `/`*(a, b: uint8): uint8 {
#pragma[magic: "DivUInt8", pure] #pragma[magic: "Divide", pure]
} }
@ -209,308 +211,313 @@ operator `/`*(a, b: float32): float32 {
operator `**`*(a, b: int64): int64 { operator `**`*(a, b: int64): int64 {
#pragma[magic: "PowInt64", pure] #pragma[magic: "SignedPow", pure]
}
operator `**`*(a, b: uint64): uint64 {
#pragma[magic: "Pow", pure]
} }
# Comparison operators # Comparison operators
operator `>`*(a, b: int): bool { operator `>`*(a, b: int): bool {
#pragma[magic: "GreaterThanInt64", pure] #pragma[magic: "GreaterThan", pure]
} }
operator `<`*(a, b: int): bool { operator `<`*(a, b: int): bool {
#pragma[magic: "LessThanInt64", pure] #pragma[magic: "LessThan", pure]
} }
operator `==`*(a, b: int): bool { operator `==`*(a, b: int): bool {
#pragma[magic: "EqualInt64", pure] #pragma[magic: "Equal", pure]
} }
operator `!=`*(a, b: int): bool { operator `!=`*(a, b: int): bool {
#pragma[magic: "NotEqualInt64", pure] #pragma[magic: "NotEqual", pure]
} }
operator `>`*(a, b: uint64): bool { operator `>`*(a, b: uint64): bool {
#pragma[magic: "GreaterThanUInt64", pure] #pragma[magic: "GreaterThan", pure]
} }
operator `<`*(a, b: uint64): bool { operator `<`*(a, b: uint64): bool {
#pragma[magic: "LessThanUInt64", pure] #pragma[magic: "LessThan", pure]
} }
operator `==`*(a, b: uint64): bool { operator `==`*(a, b: uint64): bool {
#pragma[magic: "EqualUInt64", pure] #pragma[magic: "Equal", pure]
} }
operator `!=`*(a, b: uint64): bool { operator `!=`*(a, b: uint64): bool {
#pragma[magic: "NotEqualUInt64", pure] #pragma[magic: "NotEqual", pure]
} }
operator `>`*(a, b: int32): bool { operator `>`*(a, b: int32): bool {
#pragma[magic: "GreaterThanInt32", pure] #pragma[magic: "GreaterThan", pure]
} }
operator `<`*(a, b: int32): bool { operator `<`*(a, b: int32): bool {
#pragma[magic: "LessThanInt32", pure] #pragma[magic: "LessThan", pure]
} }
operator `==`*(a, b: int32): bool { operator `==`*(a, b: int32): bool {
#pragma[magic: "EqualInt32", pure] #pragma[magic: "Equal", pure]
} }
operator `!=`*(a, b: int32): bool { operator `!=`*(a, b: int32): bool {
#pragma[magic: "NotEqualInt32", pure] #pragma[magic: "NotEqual", pure]
} }
operator `>`*(a, b: uint32): bool { operator `>`*(a, b: uint32): bool {
#pragma[magic: "GreaterThanUInt32", pure] #pragma[magic: "GreaterThan", pure]
} }
operator `<`*(a, b: uint32): bool { operator `<`*(a, b: uint32): bool {
#pragma[magic: "LessThanUInt32", pure] #pragma[magic: "LessThan", pure]
} }
operator `==`*(a, b: uint32): bool { operator `==`*(a, b: uint32): bool {
#pragma[magic: "EqualUInt32", pure] #pragma[magic: "Equal", pure]
} }
operator `!=`*(a, b: uint32): bool { operator `!=`*(a, b: uint32): bool {
#pragma[magic: "NotEqualUInt32", pure] #pragma[magic: "NotEqual", pure]
} }
operator `>`*(a, b: int16): bool { operator `>`*(a, b: int16): bool {
#pragma[magic: "GreaterThanInt16", pure] #pragma[magic: "GreaterThan", pure]
} }
operator `<`*(a, b: int16): bool { operator `<`*(a, b: int16): bool {
#pragma[magic: "LessThanInt16", pure] #pragma[magic: "LessThan", pure]
} }
operator `==`*(a, b: int16): bool { operator `==`*(a, b: int16): bool {
#pragma[magic: "EqualInt16", pure] #pragma[magic: "Equal", pure]
} }
operator `!=`*(a, b: int16): bool { operator `!=`*(a, b: int16): bool {
#pragma[magic: "NotEqualInt16", pure] #pragma[magic: "NotEqual", pure]
} }
operator `>`*(a, b: uint16): bool { operator `>`*(a, b: uint16): bool {
#pragma[magic: "GreaterThanUInt16", pure] #pragma[magic: "GreaterThan", pure]
} }
operator `<`*(a, b: uint16): bool { operator `<`*(a, b: uint16): bool {
#pragma[magic: "LessThanUInt16", pure] #pragma[magic: "LessThan", pure]
} }
operator `==`*(a, b: uint16): bool { operator `==`*(a, b: uint16): bool {
#pragma[magic: "EqualUInt16", pure] #pragma[magic: "Equal", pure]
} }
operator `!=`*(a, b: uint16): bool { operator `!=`*(a, b: uint16): bool {
#pragma[magic: "NotEqualUInt16", pure] #pragma[magic: "NotEqual", pure]
} }
operator `>`*(a, b: int8): bool { operator `>`*(a, b: int8): bool {
#pragma[magic: "GreaterThanInt8", pure] #pragma[magic: "GreaterThan", pure]
} }
operator `<`*(a, b: int8): bool { operator `<`*(a, b: int8): bool {
#pragma[magic: "LessThanInt8", pure] #pragma[magic: "LessThan", pure]
} }
operator `==`*(a, b: int8): bool { operator `==`*(a, b: int8): bool {
#pragma[magic: "EqualInt8", pure] #pragma[magic: "Equal", pure]
} }
operator `!=`*(a, b: int8): bool { operator `!=`*(a, b: int8): bool {
#pragma[magic: "NotEqualInt8", pure] #pragma[magic: "NotEqual", pure]
} }
operator `>`*(a, b: uint8): bool { operator `>`*(a, b: uint8): bool {
#pragma[magic: "GreaterThanUInt8", pure] #pragma[magic: "GreaterThan", pure]
} }
operator `<`*(a, b: uint8): bool { operator `<`*(a, b: uint8): bool {
#pragma[magic: "LessThanUInt8", pure] #pragma[magic: "LessThan", pure]
} }
operator `==`*(a, b: uint8): bool { operator `==`*(a, b: uint8): bool {
#pragma[magic: "EqualUInt8", pure] #pragma[magic: "Equal", pure]
} }
operator `!=`*(a, b: uint8): bool { operator `!=`*(a, b: uint8): bool {
#pragma[magic: "NotEqualUInt8", pure] #pragma[magic: "NotEqual", pure]
} }
operator `>`*(a, b: float): bool { operator `>`*(a, b: float): bool {
#pragma[magic: "GreaterThanFloat64", pure] #pragma[magic: "GreaterThan", pure]
} }
operator `<`*(a, b: float): bool { operator `<`*(a, b: float): bool {
#pragma[magic: "LessThanFloat64", pure] #pragma[magic: "LessThan", pure]
} }
operator `==`*(a, b: float): bool { operator `==`*(a, b: float): bool {
#pragma[magic: "EqualFloat64", pure] #pragma[magic: "Equal", pure]
} }
operator `!=`*(a, b: float): bool { operator `!=`*(a, b: float): bool {
#pragma[magic: "NotEqualFloat64", pure] #pragma[magic: "NotEqual", pure]
} }
operator `>`*(a, b: float32): bool { operator `>`*(a, b: float32): bool {
#pragma[magic: "GreaterThanFloat32", pure] #pragma[magic: "GreaterThan", pure]
} }
operator `<`*(a, b: float32): bool { operator `<`*(a, b: float32): bool {
#pragma[magic: "LessThanFloat32", pure] #pragma[magic: "LessThan", pure]
} }
operator `==`*(a, b: float32): bool { operator `==`*(a, b: float32): bool {
#pragma[magic: "EqualFloat32", pure] #pragma[magic: "Equal", pure]
} }
operator `!=`*(a, b: float32): bool { operator `!=`*(a, b: float32): bool {
#pragma[magic: "NotEqualFloat32", pure] #pragma[magic: "NotEqual", pure]
} }
operator `>=`*(a, b: int): bool { operator `>=`*(a, b: int): bool {
#pragma[magic: "GreaterOrEqualInt64", pure] #pragma[magic: "GreaterOrEqual", pure]
} }
operator `<=`*(a, b: int): bool { operator `<=`*(a, b: int): bool {
#pragma[magic: "LessOrEqualInt64", pure] #pragma[magic: "LessOrEqual", pure]
} }
operator `>=`*(a, b: uint64): bool { operator `>=`*(a, b: uint64): bool {
#pragma[magic: "GreaterOrEqualUInt64", pure] #pragma[magic: "GreaterOrEqual", pure]
} }
operator `<=`*(a, b: uint64): bool { operator `<=`*(a, b: uint64): bool {
#pragma[magic: "LessOrEqualUInt64", pure] #pragma[magic: "LessOrEqual", pure]
} }
operator `>=`*(a, b: int32): bool { operator `>=`*(a, b: int32): bool {
#pragma[magic: "GreaterOrEqualInt32", pure] #pragma[magic: "GreaterOrEqual", pure]
} }
operator `<=`*(a, b: int32): bool { operator `<=`*(a, b: int32): bool {
#pragma[magic: "LessOrEqualInt32", pure] #pragma[magic: "LessOrEqual", pure]
} }
operator `>=`*(a, b: uint32): bool { operator `>=`*(a, b: uint32): bool {
#pragma[magic: "GreaterOrEqualUInt32", pure] #pragma[magic: "GreaterOrEqual", pure]
} }
operator `<=`*(a, b: uint32): bool { operator `<=`*(a, b: uint32): bool {
#pragma[magic: "LessOrEqualUInt32", pure] #pragma[magic: "LessOrEqual", pure]
} }
operator `>=`*(a, b: int16): bool { operator `>=`*(a, b: int16): bool {
#pragma[magic: "GreaterOrEqualInt16", pure] #pragma[magic: "GreaterOrEqual", pure]
} }
operator `<=`*(a, b: int16): bool { operator `<=`*(a, b: int16): bool {
#pragma[magic: "LessOrEqualInt16", pure] #pragma[magic: "LessOrEqual", pure]
} }
operator `>=`*(a, b: uint16): bool { operator `>=`*(a, b: uint16): bool {
#pragma[magic: "GreaterOrEqualUInt16", pure] #pragma[magic: "GreaterOrEqual", pure]
} }
operator `<=`*(a, b: uint16): bool { operator `<=`*(a, b: uint16): bool {
#pragma[magic: "LessOrEqualUInt16", pure] #pragma[magic: "LessOrEqual", pure]
} }
operator `>=`*(a, b: int8): bool { operator `>=`*(a, b: int8): bool {
#pragma[magic: "GreaterOrEqualInt8", pure] #pragma[magic: "GreaterOrEqual", pure]
} }
operator `<=`*(a, b: int8): bool { operator `<=`*(a, b: int8): bool {
#pragma[magic: "LessOrEqualInt8", pure] #pragma[magic: "LessOrEqual", pure]
} }
operator `>=`*(a, b: uint8): bool { operator `>=`*(a, b: uint8): bool {
#pragma[magic: "GreaterOrEqualUInt8", pure] #pragma[magic: "GreaterOrEqual", pure]
} }
operator `<=`*(a, b: uint8): bool { operator `<=`*(a, b: uint8): bool {
#pragma[magic: "LessOrEqualUInt8", pure] #pragma[magic: "LessOrEqual", pure]
} }
operator `>=`*(a, b: float): bool { operator `>=`*(a, b: float): bool {
#pragma[magic: "GreaterOrEqualFloat64", pure] #pragma[magic: "GreaterOrEqual", pure]
} }
operator `<=`*(a, b: float): bool { operator `<=`*(a, b: float): bool {
#pragma[magic: "LessOrEqualFloat64", pure] #pragma[magic: "LessOrEqual", pure]
} }
operator `>=`*(a, b: float32): bool { operator `>=`*(a, b: float32): bool {
#pragma[magic: "GreaterOrEqualFloat32", pure] #pragma[magic: "GreaterOrEqual", pure]
} }
operator `<=`*(a, b: float32): bool { operator `<=`*(a, b: float32): bool {
#pragma[magic: "LessOrEqualFloat32", pure] #pragma[magic: "LessOrEqual", pure]
} }
@ -538,23 +545,21 @@ fn clock*: float {
} }
# TODO: Replace with generics fn print*(x: int) {
#pragma[magic: "PrintInt64"]
fn print*(x: float) {
#pragma[magic: "GenericPrint"]
} }
fn print*(x: int) { fn print*(x: uint64) {
#pragma[magic: "GenericPrint"] #pragma[magic: "PrintUInt64"]
}
fn print*(x: float) {
#pragma[magic: "PrintFloat64"]
} }
fn print*(x: string) { fn print*(x: string) {
#pragma[magic: "GenericPrint"] #pragma[magic: "PrintString"]
}
fn print*(x: bool) {
#pragma[magic: "GenericPrint"]
} }