japl
/
japl
mirror of https://github.com/japl-lang/japl.git
1
1
Fork 0
Code Issues Projects Releases Wiki Activity

Added interning for compile-time strings, made gc:markAndSweep the default option in the build tool and fixed a bug with the compiler where it would not free its memory if it encountered an error

This commit is contained in:
nocturn9x 2020-12-28 10:09:52 +01:00
parent b895399e01
commit cead00061b
5 changed files with 112 additions and 20 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

37
README.md
View File

@ -33,8 +33,9 @@ of what's done in JAPL:
- Basic arithmetic (`+`, `-`, `/`, `*`) :heavy_check_mark:
- Modulo division (`%`) and exponentiation (`**`) :heavy_check_mark:
- Bitwise operators (AND, OR, XOR, NOT) :heavy_check_mark:
- Global and local variables (__WIP__)
- Explicit scopes using brackets (__WIP__)
- Global and local variables :heavy_check_mark:
- Explicit scopes using brackets :heavy_check_mark:
- Simple optimizations (constant string interning, singletons caching) :heavy_check_mark:
- Garbage collector (__Coming soon__)
- String slicing, with start:end syntax as well :heavy_check_mark:
- Operations on strings (addition, multiplication) :heavy_check_mark:
@ -112,9 +113,37 @@ can be tweaked with command-line options, for more information, run `python3 bui
To compile the JAPL runtime, you'll first need to move into the project's directory you cloned before,
so run `cd japl`, then `python build.py ./src --flags gc:markAndSweep,d:release` and wait for it
to complete. You should now find an executable named `japl` (or `japl.exe` on windows) inside the `src` folder.
so run `cd japl`, then `python3 build.py ./src` and wait for it to complete. You should now find an
executable named `japl` (or `japl.exe` on windows) inside the `src` folder.
If you're running under windows, you might encounter some issues when using forward-slashes as opposed to back-slashes in paths,
so you should replace `./src` with `.\src`
If you're running under linux, you can also call the build script with `./build.py` (assuming python is installed in the directory indicated by the shebang at the top of the file)
## Advanced builds
If you need more customizability or want to enable debugging for JAPL, there's a few things you can do.
### Nim compiler options
The build tool calls the system's nim compiler to build JAPL and by default, the only extra flag that's passed
to it is `--gc:markAndSweep`. If you want to customize the options passed to the compiler, you can pass a comma
separated list of key:value options (spaces are not allowed). For example, doing `python3 build.py src --flags d:release,threads:on`
will call `nim compile src/japl --gc:markAndSweep -d:release --threads:on`.
### JAPL Debugging options
JAPL has some (still very beta) internal tooling to debug various parts of its ecosystem (compiler, runtime, GC, etc).
There are also some compile-time constants (such as the heap grow factor for the garbage collector) that can be set via the
`--options` parameter in the same fashion as the nim's compiler options. The available options are:
- `debug_vm` -> Debugs the runtime, instruction by instruction, showing the effects of the bytecode on the VM's stack and scopes in real time (beware of bugs!)
- `debug_gc` -> Debugs the garbage collector (once we have one)
- `debug_alloc` -> Debugs memory allocation/deallocation
- `debug_compiler` -> Debugs the compiler, showing each bytes that is spit into the bytecode
Each of these options is independent of the others and can be enabled/disabled at will. To enable an option, pass `option_name:true` to `--options` while to disable it, replace `true` with `false`.
Note that the build tool will generate a file named `config.nim` inside the `src` directory and will use that for subsequent builds, so if you want to override it you'll have to pass `--override-config` as a command-line options. Passing it without any option will fallback to (somewhat) sensible defaults

4
build.py
View File

@ -143,7 +143,9 @@ if __name__ == "__main__":
"Note that if a config.nim file exists in the destination directory, that will override any setting defined here unless --override-config is used")
parser.add_argument("--override-config", help="Overrides the setting of an already existing config.nim file in the destination directory", action="store_true")
args = parser.parse_args()
flags = {}
flags = {
"gc": "markAndSweep",
}
options = {
"debug_vm": "false",
"debug_gc": "false",

56
src/compiler.nim
View File

@ -27,8 +27,10 @@ import types/baseObject
import types/function
import types/numbers
import types/japlString
import types/methods
import tables
import config
import memory
when isMainModule:
import util/debug
@ -46,6 +48,7 @@ type
loop*: Loop
objects*: seq[ptr Obj]
file*: string
interned*: Table[string, ptr Obj]
Local* = ref object # A local variable
name*: Token
@ -312,7 +315,14 @@ proc strVal(self: Compiler, canAssign: bool) =
var str = self.parser.previous().lexeme
var delimiter = &"{str[0]}" # TODO: Add proper escape sequences support
str = str.unescape(delimiter, delimiter)
self.emitConstant(self.markObject(asStr(str)))
if str notin self.interned:
self.interned[str] = str.asStr()
self.emitConstant(self.markObject(self.interned[str]))
else:
# We intern only constant strings!
# Note that we don't call self.markObject on an already
# interned string because that has already been marked
self.emitConstant(self.interned[str])
proc bracketAssign(self: Compiler, canAssign: bool) =
@ -1025,6 +1035,46 @@ proc declaration(self: Compiler) =
if self.parser.panicMode:
self.synchronize()
proc freeObject(self: Compiler, obj: ptr Obj) =
## Frees the associated memory
## of an object
case obj.kind:
of ObjectType.String:
var str = cast[ptr String](obj)
when DEBUG_TRACE_ALLOCATION:
echo &"DEBUG - Compiler: Freeing string object of length {str.len}"
discard freeArray(char, str.str, str.len)
discard free(ObjectType.String, obj)
of ObjectType.Exception, ObjectType.Class,
ObjectType.Module, ObjectType.BaseObject, ObjectType.Integer,
ObjectType.Float, ObjectType.Bool, ObjectType.NotANumber,
ObjectType.Infinity, ObjectType.Nil:
when DEBUG_TRACE_ALLOCATION:
echo &"DEBUG - Compiler: Freeing {obj.typeName()} object with value '{stringify(obj)}'"
discard free(obj.kind, obj)
of ObjectType.Function:
var fun = cast[ptr Function](obj)
when DEBUG_TRACE_ALLOCATION:
if fun.name == nil:
echo &"DEBUG - Compiler: Freeing global code object"
else:
echo &"DEBUG - Compiler: Freeing function object with name '{stringify(fun)}'"
fun.chunk.freeChunk()
discard free(ObjectType.Function, fun)
proc freeCompiler*(self: Compiler) =
## Frees all the allocated objects
## from the compiler
var objCount = len(self.objects)
var objFreed = 0
for obj in reversed(self.objects):
self.freeObject(obj)
discard self.objects.pop()
objFreed += 1
when DEBUG_TRACE_ALLOCATION:
echo &"DEBUG - Compiler: Freed {objFreed} objects out of {objCount} compile-time objects"
# The array of all parse rules
var rules: array[TokenType, ParseRule] = [
@ -1104,12 +1154,10 @@ proc compile*(self: Compiler, source: string): ptr Function =
self.parser = initParser(tokens, self.file)
while not self.parser.match(EOF):
self.declaration()
var function = self.endCompiler()
when DEBUG_TRACE_COMPILER:
echo "\n==== COMPILER debugger ends ===="
echo ""
if not self.parser.hadError:
when DEBUG_TRACE_COMPILER:
echo "Result: Ok"
@ -1117,6 +1165,7 @@ proc compile*(self: Compiler, source: string): ptr Function =
else:
when DEBUG_TRACE_COMPILER:
echo "Result: Fail"
# self.freeCompiler()
return nil
else:
return nil
@ -1125,6 +1174,7 @@ proc compile*(self: Compiler, source: string): ptr Function =
proc initParser*(tokens: seq[Token], file: string): Parser =
result = Parser(current: 0, tokens: tokens, hadError: false, panicMode: false, file: file)
proc initCompiler*(context: FunctionType, enclosing: Compiler = nil, parser: Parser = initParser(@[], ""), file: string): Compiler =
## Initializes a new compiler object and returns a reference
## to it

12
src/memory.nim
View File

@ -24,19 +24,25 @@
import segfaults
import config
# when DEBUG_TRACE_ALLOCATION:
# import util/debug # TODO: Add memory debugging
import strformat
proc reallocate*(pointr: pointer, oldSize: int, newSize: int): pointer =
## Wrapper around realloc/dealloc
try:
if newSize == 0 and pointr != nil: # pointr is awful, but clashing with builtins is even more awful
when DEBUG_TRACE_ALLOCATION:
echo &"DEBUG - Memory manager: Deallocating {oldSize} bytes"
dealloc(pointr)
return nil
when DEBUG_TRACE_ALLOCATION:
if oldSize == 0:
echo &"DEBUG - Memory manager: Allocating {newSize} bytes of memory"
else:
echo &"DEBUG - Memory manager: Resizing {oldSize} bytes of memory to {newSize} bytes"
result = realloc(pointr, newSize)
except NilAccessError:
stderr.write("A fatal error occurred -> could not allocate memory, segmentation fault\n")
stderr.write("A fatal error occurred -> could not manage memory, segmentation fault\n")
quit(71) # For now, there's not much we can do if we can't get the memory we need

23
src/vm.nim
View File

@ -592,7 +592,7 @@ proc freeObject(self: VM, obj: ptr Obj) =
of ObjectType.String:
var str = cast[ptr String](obj)
when DEBUG_TRACE_ALLOCATION:
echo &"DEBUG: Freeing string object of length {str.len}"
echo &"DEBUG - VM: Freeing string object of length {str.len}"
discard freeArray(char, str.str, str.len)
discard free(ObjectType.String, obj)
of ObjectType.Exception, ObjectType.Class,
@ -601,17 +601,17 @@ proc freeObject(self: VM, obj: ptr Obj) =
ObjectType.Infinity, ObjectType.Nil:
when DEBUG_TRACE_ALLOCATION:
if obj notin self.cached:
echo &"DEBUG: Freeing {obj.typeName()} object with value '{stringify(obj)}'"
echo &"DEBUG- VM: Freeing {obj.typeName()} object with value '{stringify(obj)}'"
else:
echo &"DEBUG: Freeing cached {obj.typeName()} object with value '{stringify(obj)}'"
echo &"DEBUG - VM: Freeing cached {obj.typeName()} object with value '{stringify(obj)}'"
discard free(obj.kind, obj)
of ObjectType.Function:
var fun = cast[ptr Function](obj)
when DEBUG_TRACE_ALLOCATION:
if fun.name == nil:
echo &"DEBUG: Freeing global code object"
echo &"DEBUG - VM: Freeing global code object"
else:
echo &"DEBUG: Freeing function object with name '{stringify(fun)}'"
echo &"DEBUG - VM: Freeing function object with name '{stringify(fun)}'"
fun.chunk.freeChunk()
discard free(ObjectType.Function, fun)
@ -631,7 +631,7 @@ proc freeObjects(self: var VM) =
self.freeObject(cached_obj)
cachedFreed += 1
when DEBUG_TRACE_ALLOCATION:
echo &"DEBUG: Freed {runtimeFreed + cachedFreed} objects out of {runtimeObjCount + cacheCount} ({cachedFreed}/{cacheCount} cached objects, {runtimeFreed}/{runtimeObjCount} runtime objects)"
echo &"DEBUG - VM: Freed {runtimeFreed + cachedFreed} objects out of {runtimeObjCount + cacheCount} ({cachedFreed}/{cacheCount} cached objects, {runtimeFreed}/{runtimeObjCount} runtime objects)"
proc freeVM*(self: var VM) =
@ -644,7 +644,7 @@ proc freeVM*(self: var VM) =
quit(71)
when DEBUG_TRACE_ALLOCATION:
if self.objects.len > 0:
echo &"DEBUG: Warning, {self.objects.len} objects were not freed"
echo &"DEBUG - VM: Warning, {self.objects.len} objects were not freed"
proc initCache(self: var VM) =
@ -671,14 +671,19 @@ proc initVM*(): VM =
proc interpret*(self: var VM, source: string, repl: bool = false, file: string): InterpretResult =
## Interprets a source string containing JAPL code
self.resetStack()
var compiler = initCompiler(SCRIPT, file=file)
var compiled = compiler.compile(source)
self.source = source
self.file = file
var compiler = initCompiler(SCRIPT, file=file)
var compiled = compiler.compile(source)
# Here we take into account that self.interpret() might
# get called multiple times and we don't wanna loose
# what we allocated before, so we merge everything we
# allocated + everything the compiler allocated at compile time
self.objects = self.objects & compiler.objects # TODO:
# revisit the best way to transfer marked objects from the compiler
# to the vm
if compiled == nil:
compiler.freeCompiler()
return CompileError
self.push(compiled)
discard self.callObject(compiled, 0)