wip lists and tables

This commit is contained in:
prod2 2022-02-03 03:18:11 +01:00
parent 788ca7eec0
commit 0ec5bfb60a
13 changed files with 538 additions and 25 deletions

View File

@ -16,6 +16,8 @@ type
opDefineGlobal, opGetGlobal, opSetGlobal, # globals (uses constants)
opGetLocal, opSetLocal, # locals
opJumpIfFalse, opJump, opLoop, opJumpIfFalsePop, # jumps
opCreateList, opCreateTable, # collection creation
opLen, opSetIndex, opGetIndex, # collection operators
Chunk* = object
code*: seq[uint8]
@ -78,7 +80,8 @@ const simpleInstructions = {
opNegate, opNot,
opAdd, opSubtract, opMultiply, opDivide,
opEqual, opGreater, opLess,
opTrue, opFalse, opNil
opTrue, opFalse, opNil,
opLen, opSetIndex, opGetIndex,
}
const constantInstructions = {
opConstant,
@ -94,6 +97,7 @@ const argInstructions = {
opGetLocal, opSetLocal,
opJumpIfFalse, opJump, opLoop, opJumpIfFalsePop,
opFunctionDef,
opCreateList, opCreateTable,
}

View File

@ -49,7 +49,7 @@ type
Precedence = enum
pcNone, pcAssignment, pcOr, pcAnd, pcEquality, pcComparison,
pcTerm, pcFactor, pcUnary, pcCall, pcPrimary
pcTerm, pcFactor, pcUnary, pcIndex, pcCall, pcPrimary
# pcUnary applies to all prefix operators regardless of this enum's value
# changing pcUnary's position can change the priority of all unary ops
#
@ -651,6 +651,64 @@ proc parseFunct(comp: Compiler) =
tkFunct.genRule(parseFunct, nop, pcNone)
# lists
proc parseList(comp: Compiler) =
var count: int
while comp.current.tokenType != tkRightBracket:
comp.expression()
count.inc()
if comp.current.tokenType != tkRightBracket or comp.current.tokenType == tkComma:
comp.consume(tkComma, "Comma expected after list member.")
comp.consume(tkRightBracket, "Right bracket expected after list members.")
if count > argMax:
comp.error("Maximum list length exceeded.")
comp.writeChunk(1 - count, opCreateList)
comp.writeChunk(0, count.toDU8())
tkStartList.genRule(parseList, nop, pcNone)
# tables
proc parseTable(comp: Compiler) =
var count: int
while comp.current.tokenType != tkRightBrace:
comp.expression()
comp.consume(tkEqual, "Equal sign expected after key.")
comp.expression()
count.inc()
if comp.current.tokenType != tkRightBrace or comp.current.tokenType == tkComma:
comp.consume(tkComma, "Comma expected after key-value pair.")
comp.consume(tkRightBrace, "Right brace expected after list members.")
if count > argMax:
comp.error("Maximum table length exceeded.")
comp.writeChunk(1 - 2 * count, opCreateTable)
comp.writeChunk(0, count.toDU8())
tkStartTable.genRule(parseTable, nop, pcNone)
# len op
proc parseLen(comp: Compiler) =
comp.expression()
comp.writeChunk(0, opLen)
tkHashtag.genRule(parseLen, nop, pcNone)
# get/set index
proc parseIndex(comp: Compiler) =
# the index
comp.expression()
comp.consume(tkRightBracket, "Right bracket expected after index.")
if comp.match(tkEqual):
comp.expression()
comp.writeChunk(-2, opSetIndex)
else:
comp.writeChunk(-1, opGetIndex)
tkLeftBracket.genRule(nop, parseIndex, pcIndex)
# below are the expressions that can contain statements in some way
# the only expressions that can contain a statement are:

View File

@ -14,6 +14,8 @@ type
tkLeftParen, tkRightParen, tkLeftBrace, tkRightBrace, tkComma, tkDot,
tkMinus, tkPlus, tkSemicolon, tkSlash, tkStar, tkBang, tkBangEqual,
tkGreater, tkGreaterEqual, tkLess, tkLessEqual, tkEqual, tkEqualEqual,
tkStartList, tkStartTable, tkLeftBracket, tkRightBracket,
tkHashtag,
tkIdentifier, tkString,
tkNumber, tkAnd, tkElse, tkFalse, tkFor, tkFunct, tkGoto, tkIf, tkNil,
tkOr, tkPrint, tkLabel, tkBreak, tkTrue, tkVar, tkWhile,
@ -169,6 +171,8 @@ proc scanToken*(scanner: Scanner): Token =
of ')': return scanner.makeToken(tkRightParen)
of '{': return scanner.makeToken(tkLeftBrace)
of '}': return scanner.makeToken(tkRightBrace)
of '[': return scanner.makeToken(tkLeftBracket)
of ']': return scanner.makeToken(tkRightBracket)
of ';': return scanner.makeToken(tkSemicolon)
of ',': return scanner.makeToken(tkComma)
of '.': return scanner.makeToken(tkDot)
@ -176,6 +180,7 @@ proc scanToken*(scanner: Scanner): Token =
of '+': return scanner.makeToken(tkPlus)
of '/': return scanner.makeToken(tkSlash)
of '*': return scanner.makeToken(tkStar)
of '#': return scanner.makeToken(tkHashtag)
of '!':
return if scanner.match('='): scanner.makeToken(tkBangEqual) else: scanner.makeToken(tkBang)
of '=':
@ -189,7 +194,9 @@ proc scanToken*(scanner: Scanner): Token =
of Digits:
return scanner.scanNumber()
of '@':
return scanner.scanLabel()
if scanner.match('['): return scanner.makeToken(tkStartList)
elif scanner.match('{'): return scanner.makeToken(tkStartTable)
else: return scanner.scanLabel()
else:
if c.canStartIdent():
return scanner.scanIdentifier()

View File

@ -1,4 +1,4 @@
# The hash table implementation for string interning
# The hash table implementation for string interning and globals
import ndstring
@ -11,7 +11,7 @@ type
EntryStatus = enum
esNil, esAlive, esTombstone
Entry*[U, V] = object
Entry[U, V] = object
entryStatus: EntryStatus
key: U
value: V

View File

@ -0,0 +1,89 @@
import ../pointerutils
import strformat
# configure ndlist here
const boundsChecks = defined(debug)
# boundsChecks default: false, true has a large performance impact, and emitting correct code is on the compiler's job
# boundsChecking is only meant for debugging
const growthFactor = 2
# should be a natural number larger than 1
const startCap = 8
type
ListObj[T] = object
len: int
cap: int
entries: UncheckedArray[T]
List*[T] = ptr ListObj[T]
template allocSize[T](list: List[T], cap: int): int =
sizeof(ListObj[T]) + sizeof(T) * cap
proc newList*[T](): List[T] =
return cast[List[T]](nil)
proc free*[T](list: var List[T]) =
## dealloc's the list object
if list != nil:
list.dealloc()
proc grow[T](list: var List[T]) {.inline.} =
## growth the list's capacity
let newcap = if list == nil: startCap else: list.cap * growthFactor
let size = list.allocSize(newcap)
if list == nil:
list = cast[List[T]](size.alloc())
list.cap = newcap
list.len = 0
else:
list = cast[List[T]](list.realloc(size))
list.cap = newcap
proc add*[T](list: var List[T], item: T) {.inline.} =
if list == nil or list.len == list.cap:
list.grow()
list.entries[list.len] = item
list.len.inc()
proc getIndex*[T](list: List[T], index: int): T =
when boundsChecks:
if index < 0 or index >= list.len:
raise newException(Defect, &"Attempt to getIndex with an index {index} which is out of bounds (len: {$list.len}).")
list.entries[index]
proc getIndexNeg*[T](list: List[T], index: int): T =
## warning: -1 is the top value, using 0 is invalid (unlike stack.getIndexNeg)
when boundsChecks:
if index <= 0 or index > list.len:
raise newException(Defect, "Attempt to getIndexNeg with an index out of bounds.")
list.entries[list.len - index]
proc setIndex*[T](list: var List[T], index: int, item: T) =
when boundsChecks:
if index < 0 or index >= list.len:
raise newException(Defect, "Attempt to getIndex with an index out of bounds.")
list.entries[index] = item
proc setIndexNeg*[T](list: List[T], index: int, item: T) =
## warning: -1 is the top value, using 0 is invalid (unlike stack.setIndexNeg)
when boundsChecks:
if index <= 0 or index > list.len:
raise newException(Defect, "Attempt to setIndexNeg with an index out of bounds.")
list.entries[list.len - index] = item
proc getLength*[T](list: List[T]): int {.inline.} =
if list == nil:
0
else:
list.len
proc `$`*[T](list: List[T]): string =
result = "@[ "
for i in countup(0, list.len - 1):
mixin `$`
result &= $list.getIndex(i)
if i < list.len - 1:
result &= ", "
result &= " ]"

View File

@ -0,0 +1,129 @@
# The hash table implementation for table types
import ndstring
import bitops
const tableMaxLoad = 0.75
const tableInitSize = 8
type
EntryStatus = enum
esNil, esAlive, esTombstone
Entry[U, V] = object
entryStatus: EntryStatus
key: U
value: V
NdTable*[U, V] = ptr object
count: uint32
cap: uint32
entries: UncheckedArray[Entry[U, V]]
template allocSize[U, V](tbl: NdTable[U, V], cap: int): int =
8 + cap * sizeof(Entry[U, V])
proc newNdTable*[U, V]: NdTable[U, V] =
result = cast[NdTable[U,V]](result.allocSize(tableInitSize).alloc0())
result[].cap = tableInitSize
result[].count = 0
proc free*[U, V](tbl: var NdTable[U, V]) =
dealloc(tbl)
proc findEntry[U, V](entries: ptr UncheckedArray[Entry[U, V]], cap: int, key: U): ptr Entry[U, V] =
mixin fnv1a, equal
var index = key.fnv1a().bitand(cap - 1)
var tombstone: ptr Entry[U, V] = nil
while true:
let entry: ptr Entry[U, V] = entries[index].addr # TODO: check the performance impact of this line
case entry[].entryStatus:
of esNil:
return if tombstone != nil: tombstone else: entry
of esTombstone:
if tombstone == nil:
tombstone = entry
of esAlive:
if entry[].key.equal(key):
return entry
index = (index + 1).bitand(cap - 1)
proc grow[U, V](tbl: var NdTable[U, V]): int {.inline.} =
## Calculates the new capacity
tbl[].cap.int * 2
proc adjustCapacity[U, V](tbl: var NdTable[U, V], newcap: int) =
let tblnew = cast[NdTable[U,V]](tbl.allocSize(newcap).alloc0())
tblnew[].count = tbl.count
tblnew[].cap = newcap.uint32
for i in countup(0, tbl.cap.int - 1):
let entry = tbl[].entries[i]
if entry.entryStatus == esAlive:
var dest = findEntry(tblnew.entries.addr, newcap, entry.key)
dest[].key = entry.key
dest[].value = entry.value
dest[].entryStatus = esAlive
tbl.count.inc
if tbl != nil:
dealloc(tbl)
tbl = tblnew
proc tableSet*[U, V](tbl: var NdTable[U, V], key: U, val: V): bool =
## Returns false if new value is entered
## True if the value entered already existed and is overwritten
if tbl.count.int + 1 > int(tbl.cap.float * tableMaxLoad):
let cap = tbl.grow()
tbl.adjustCapacity(cap)
let entry: ptr Entry[U, V] = findEntry(tbl.entries.addr, tbl.cap.int, key)
let status = entry[].entryStatus
if status == esNil:
tbl.count.inc
elif status == esAlive:
result = true
entry[].key = key
entry[].value = val
entry[].entryStatus = esAlive
proc tableGet*[U, V](tbl: NdTable[U, V], key: U, val: var V): bool =
## Returns false if not in table
## Returns true if in the table, sets value to val
if tbl.count == 0:
return false
let entry = findEntry(tbl.entries.addr, tbl.cap.int, key)
if not (entry[].entryStatus == esAlive):
return false
val = entry[].value
return true
proc tableDelete*[U, V](tbl: NdTable[U, V], key: U): bool =
if tbl.count == 0:
return false
let entry = findEntry(tbl.entries.addr, tbl.cap.int, key)
if entry[].entryStatus != esAlive:
return false
entry[].entryStatus = esTombstone
return true
proc `$`*[U, V](tbl: NdTable[U, V]): string =
result = "@{ "
for i in countup(0, tbl.cap.int - 1):
let entry = tbl[].entries[i]
if entry.entryStatus == esAlive:
mixin `$`
result &= $entry.key
result &= " = "
result &= $entry.value
result &= ", "
result[^2] = ' '
result[^1] = '}'
proc getLength*[U, V](tbl: NdTable[U, V]): int =
tbl.count.int

View File

@ -27,6 +27,10 @@ proc newString*(str: string): NdString =
discard ndStrings.tableSet(result, nil)
proc resetInternedStrings* =
ndStrings.free()
ndStrings = newTable[NdString, NdString]()
proc `$`*(ndStr: NdString): string =
result = newString(ndStr.len.int)
copyMem(result[0].unsafeAddr, ndStr.chars[0].unsafeAddr, ndStr.len.int)
@ -34,3 +38,10 @@ proc `$`*(ndStr: NdString): string =
proc `&`*(left, right: NdString): NdString =
# TODO optimize this later when strings will be benchmarked
newString($left & $right)
proc getLength*(ndStr: NdString): int =
ndStr.len.int
proc getIndex*(ndStr: NdString, index: int): NdString =
# TODO optimize this later
newString($($ndStr)[index])

View File

@ -3,6 +3,8 @@ import bitops
import ndstring
import stringutils
import ndtable
import ndlist
type
NdValue* = uint
@ -15,12 +17,19 @@ type
# see https://craftinginterpreters.com/optimization.html
# bit layout:
# bit 63 is unused (can be used for more types in the future)
# bit 63 is used for type information as well
# bits 62-50 are all 1 if it's not a float
# bits 49-48 determine type:
# if bit 63 is 0:
# 00 -> nil or bool (singletons)
# 01 -> string
# 10 -> funct
# 11 -> closure (to be implemented later)
# if bit 63 is 1:
# 00 -> list
# 01 -> table
# 10 -> native funct (to be implemented later)
# 11 -> unused for now
const qNan* = 0x7ffc000000000000'u
@ -34,6 +43,9 @@ const tagString* = 0x7ffd000000000000'u
# 0111 1111 1111 11*10* 0000 0000 0000 0000 0000 0000 0000 0000 0000 0000 0000 0000
const tagFunct* = 0x7ffe000000000000'u
const tagList* = 0xfffc000000000000'u
const tagTable* = 0xfffd000000000000'u
const mask48* = 0xffff000000000000'u
# converters
@ -53,6 +65,12 @@ template isString*(val: NdValue): bool =
template isFunct*(val: NdValue): bool =
val.bitand(mask48) == tagFunct
template isList*(val: NdValue): bool =
val.bitand(mask48) == tagList
template isTable*(val: NdValue): bool =
val.bitand(mask48) == tagTable
# these assume that the type has been previously determined
template asBool*(val: NdValue): bool =
val == ndTrue
@ -66,6 +84,12 @@ template asString*(val: NdValue): NdString =
template asFunct*(val: NdValue): ptr uint8 =
cast[ptr uint8](val.bitand(mask48.bitnot()))
template asList*(val: NdValue): List[NdValue] =
cast[List[NdValue]](val.bitand(mask48.bitnot()))
template asTable*(val: NdValue): NdTable[NdValue, NdValue] =
cast[NdTable[NdValue, NdValue]](val.bitand(mask48.bitnot()))
template fromNil*(): NdValue =
ndNil
@ -84,6 +108,29 @@ template fromNimString*(sval: string): NdValue =
template fromFunct*(val: ptr uint8): NdValue =
cast[uint](val).bitor(tagFunct)
template fromList*(val: List[NdValue]): NdValue =
cast[uint](val).bitor(tagList)
template fromTable*(val: NdTable[NdValue, NdValue]): NdValue =
cast[uint](val).bitor(tagTable)
# for hashtables
proc fnv1a*(ndval: NdValue): int =
var hash = 2166136261'u32
var val = ndval
for i in countup(0, 7):
hash = hash xor val.uint8.uint32
hash *= 16777619
val = val shr 8
return hash.int
proc equal*(val, right: NdValue): bool =
if val.isFloat() and right.isFloat():
val.asFloat() == right.asFloat()
else:
val == right
# KON VALUE HELPERS, MUST BE DEFINED FOR EVERY KONVALUE
@ -100,6 +147,10 @@ proc `$`*(val: NdValue): string =
return &"Function object {cast[uint](val.asFunct())}"
elif val.isString():
return $val.asString()
elif val.isTable():
return $val.asTable()
elif val.isList():
return $val.asList()
proc isFalsey*(val: NdValue): bool =
val == ndNil or val == ndFalse
@ -118,6 +169,10 @@ proc friendlyType*(val: NdValue): string =
"string"
elif val.isFunct():
"function"
elif val.isTable():
"table"
elif val.isList():
"list"
else:
"unknown"
@ -181,19 +236,66 @@ proc greater*(val: var NdValue, right: NdValue): NatReturn {.inline.} =
return natError(&"Attempt to compare types {val.friendlyType()} and {right.friendlyType()}.")
return natOk
# for hashtables
proc fnv1a*(ndval: NdValue): int =
var hash = 2166136261'u32
var val = ndval
for i in countup(0, 7):
hash = hash xor val.uint8.uint32
hash *= 16777619
val = val shr 8
return hash.int
proc equal*(val, right: NdValue): bool =
if val.isFloat() and right.isFloat():
val.asFloat() == right.asFloat()
proc getLength*(val: var NdValue): NatReturn {.inline.} =
if val.isList():
val = fromFloat(val.asList().getLength().float)
elif val.isTable():
val = fromFloat(val.asTable().getLength().float)
elif val.isString():
val = fromFloat(val.asString().getLength().float)
else:
val == right
return natError(&"Attempt to get length of type {val.friendlyType()}, this operator is only available for lists, tables and strings.")
return natOk
proc getIndex*(val: var NdValue, index: NdValue): NatReturn {.inline.} =
if val.isTable():
var tbl = val.asTable()
if not tbl.tableGet(index, val):
val = fromNil()
return natOk
template indexInt: int =
index.asFloat().int
template checkBounds(len: int) =
if not index.isFloat():
return natError(&"Attempt to index using an index of type {index.friendlyType()}: only numerical indexes are allowed.")
if indexInt() < 0 or indexInt() >= len:
return natError(&"Index out of bounds. Index used: {index}; valid range: 0 - {len - 1} inclusive.")
if val.isList():
var list = val.asList()
let len = list.getLength()
len.checkBounds()
val = list.getIndex(indexInt)
elif val.isString():
var str = val.asString()
let len = str.getLength()
len.checkBounds()
val = str.getIndex(indexInt).fromNdString()
else:
return natError(&"Attempt to index a type {val.friendlyType()}, this operator is only available for lists, tables and strings.")
return natOk
proc setIndex*(val: var NdValue, index: NdValue, newval: NdValue): NatReturn {.inline.} =
if val.isTable():
var tbl = val.asTable()
discard tbl.tableSet(index, newval)
return natOk
if val.isList():
var list = val.asList()
let len = list.getLength()
let indexint = index.asFloat().int
if not index.isFloat():
return natError(&"Attempt to index using an index of type {index.friendlyType()}: only numerical indexes are allowed.")
if indexInt < 0 or indexInt >= len:
return natError(&"Index out of bounds. Index used: {index}; valid range: 0 - {len - 1} inclusive.")
list.setIndex(indexInt, newval)
else:
return natError(&"Attempt to set-index a type {val.friendlyType()}, this operator is only available for lists and tables.")
return natOk

View File

@ -10,6 +10,8 @@ import types/stringutils
import bitops # needed for value's templates
import types/value
import types/hashtable
import types/ndtable
import types/ndlist
when profileInstructions:
@ -228,6 +230,39 @@ proc run*(chunk: Chunk): InterpretResult =
frames.add(Frame(stackBottom: stack.high - argcount, returnIp: ip))
ip = funct.asFunct() # jump to the entry point
of opCreateList:
let listLen = readDU8()
var list = newList[NdValue]()
for i in countup(0, listLen - 1): # TODO optimize: some copyMem might be possible, which will also fix the reversing order issue
list.add(stack.pop())
stack.push(list.fromList())
of opCreateTable:
let tblLen = readDU8()
var tbl = newNdTable[NdValue, NdValue]()
for i in countup(0, tblLen - 1): # TODO optimize: allocate the right size right upfront
let val = stack.pop()
let key = stack.pop()
discard tbl.tableSet(key, val) # TODO check for duplicates and disallow
stack.push(tbl.fromTable())
of opLen:
let res = stack.peek().getLength()
if not res.ok:
runtimeError(res.msg)
break
of opGetIndex:
let index = stack.pop()
let res = stack.peek().getIndex(index)
if not res.ok:
runtimeError(res.msg)
break
of opSetIndex:
let value = stack.pop()
let index = stack.pop()
let res = stack.peek().setIndex(index, value)
if not res.ok:
runtimeError(res.msg)
break
when profileInstructions:
durations[ins] += getMonoTime() - startTime

View File

@ -1,6 +1,6 @@
import ndspkg/types/hashtable
import ndspkg/types/ndstring
import ndspkg/types/stringutils
import ../src/ndspkg/types/hashtable
import ../src/ndspkg/types/ndstring
import ../src/ndspkg/types/stringutils
import strformat

21
tests/ndlist.nim Normal file
View File

@ -0,0 +1,21 @@
import ../src/ndspkg/types/ndlist
proc testNdlist* =
var list = newList[int]()
var simlen = 0 # simulated length
for i in countup(0, 10000):
assert list.getLength() == simlen
list.add(i)
simlen.inc
assert list.getLength() == simlen
assert list.getIndex(i) == i
list.setIndex(i, i*2)
assert list.getLength() == simlen
assert list.getIndex(i) == i * 2
list.free()
echo "ndlist test finished"

49
tests/ndtable.nim Normal file
View File

@ -0,0 +1,49 @@
import ../src/ndspkg/types/ndtable
import ../src/ndspkg/types/ndstring
import ../src/ndspkg/types/stringutils
proc equal*(l, r: string): bool =
l == r
proc testNdtables* =
var tbl = newNdTable[string, int]()
var val: int
assert tbl.tableSet("hello", 1) == false
assert tbl.tableGet("hello", val) == true
assert val == 1
assert tbl.tableSet("hello", 4) == true
assert tbl.tableGet("hello", val) == true
assert val == 4
assert tbl.tableGet("hellw", val) == false
assert val == 4
assert tbl.tableDelete("hello") == true
val = 0
assert tbl.tableGet("hello", val) == false
assert val == 0
for i in countup(0, 10000):
assert tbl.tableSet($i, i) == false
assert tbl.tableget($i, val) == true
assert val == i
assert tbl.tableSet($i, i * 2) == true
assert tbl.tableget($i, val) == true
assert val == i * 2
assert tbl.tableSet($i, i * 4) == true
assert tbl.tableget($i, val) == true
assert val == i * 4
if i mod 5 == 0:
assert tbl.tableDelete($i) == true
assert tbl.tableDelete($i) == false
for i in countup(0, 10000):
if i mod 5 == 0:
assert tbl.tableGet($i, val) == false
else:
assert tbl.tableGet($i, val) == true
assert val == i * 4
tbl.free()
echo "ndtable test finished"

View File

@ -1,3 +1,11 @@
import hashtable
testHashtables()
testHashtables()
import ndtable
testNdtables()
import ndlist
testNdlist()