Fixed various minor bugs, improved debugger output and consistency

This commit is contained in:
nocturn9x 2021-01-16 18:14:22 +01:00
parent e15e03764e
commit 71f79d8174
10 changed files with 138 additions and 81 deletions

View File

@ -24,7 +24,6 @@ import logging
import argparse
from time import time
from typing import Dict
from pprint import pformat
from subprocess import Popen, PIPE, DEVNULL, run
@ -52,6 +51,7 @@ const FRAMES_MAX* = {frames_max} # The maximum recursion limit
const JAPL_VERSION* = "0.3.0"
const JAPL_RELEASE* = "alpha"
const DEBUG_TRACE_VM* = {debug_vm} # Traces VM execution
const SKIP_STDLIB_INIT* = {skip_stdlib_init} # Skips stdlib initialization in debug mode
const DEBUG_TRACE_GC* = {debug_gc} # Traces the garbage collector (TODO)
const DEBUG_TRACE_ALLOCATION* = {debug_alloc} # Traces memory allocation/deallocation (WIP)
const DEBUG_TRACE_COMPILER* = {debug_compiler} # Traces the compiler
@ -120,7 +120,8 @@ def build(path: str, flags: Dict[str, str] = {}, options: Dict[str, bool] = {},
config_path = os.path.join(path, "config.nim")
main_path = os.path.join(path, "japl.nim")
logging.info("Just Another Build Tool, version 0.3.2")
logging.info("Just Another Build Tool, version 0.3.3")
listing = "\n- {} = {}"
if not os.path.exists(path):
logging.error(f"Input path '{path}' does not exist")
return
@ -136,7 +137,7 @@ def build(path: str, flags: Dict[str, str] = {}, options: Dict[str, bool] = {},
logging.error(f"A fatal unhandled exception occurred -> {type(fatal).__name__}: {fatal}")
return
else:
logging.debug(f"Config file has been generated, compiling with options as follows: \n{pformat(options, indent=2)}")
logging.debug(f"Config file has been generated, compiling with options as follows: {''.join(listing.format(k, v) for k, v in options.items())}")
logging.debug(f"Compiling '{main_path}'")
nim_flags = " ".join(f"-{name}:{value}" if len(name) == 1 else f"--{name}:{value}" for name, value in flags.items())
command = "nim {flags} compile {path}".format(flags=nim_flags, path=main_path)
@ -176,10 +177,9 @@ def build(path: str, flags: Dict[str, str] = {}, options: Dict[str, bool] = {},
else:
# TODO -> Is PATH defined on all linux distros?
logging.info(f"Installing JAPL at PATH")
if any(os.path.exists(os.path.join(path, "jpl")) for path in os.getenv("PATH").split(":")) and not ignore_binary:
if not ignore_binary and any(os.path.exists(os.path.join(path, "jpl")) for path in os.getenv("PATH").split(":")):
logging.error("Could not install JAPL because a binary already exists in PATH")
return
install_path = os.path.join(os.getenv("PATH").split(":")[0], "jpl")
for path in os.getenv("PATH").split(":"):
install_path = os.path.join(path, "jpl")
logging.debug(f"Attempting to install JAPL at '{install_path}'")
@ -193,8 +193,7 @@ def build(path: str, flags: Dict[str, str] = {}, options: Dict[str, bool] = {},
logging.debug(f"JAPL installed at '{path}', setting executable permissions")
# TODO: Use external oschmod library once we support windows!
try:
perms = os.stat(install_path)
os.chmod(install_path, perms.st_mode | stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH)
os.chmod(install_path, os.stat(install_path).st_mode | stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH)
except Exception as fatal:
logging.error(f"A fatal unhandled exception occurred -> {type(fatal).__name__}: {fatal}")
break
@ -218,6 +217,7 @@ if __name__ == "__main__":
}
options = {
"debug_vm": "false",
"skip_stdlib_init": "false",
"debug_gc": "false",
"debug_compiler": "false",
"debug_alloc": "false",

View File

@ -33,6 +33,8 @@ import memory
import multibyte
when isMainModule:
import util/debug
when DEBUG_TRACE_COMPILER:
import types/methods
type
@ -101,6 +103,14 @@ proc peek(self: Parser): Token =
return self.tokens[self.current]
proc peekNext(self: Parser): Token =
## Returns the next token without consuming it
## or an EOF token if we're at the end of the file
if self.current <= len(self.tokens) - 1:
return self.tokens[self.current + 1]
return Token(kind: EOF, lexeme: "")
proc previous(self: Parser): Token =
## Returns the previously consumed token
return self.tokens[self.current - 1]
@ -112,6 +122,12 @@ proc check(self: Parser, kind: TokenType): bool =
return self.peek().kind == kind
proc checkNext(self: Parser, kind: TokenType): bool =
## Checks if the next token is of the expected type
## without consuming it
return self.peekNext().kind == kind
proc match(self: Parser, kind: TokenType): bool =
## Calls self.check() and consumes a token if the expected
## token type is encountered, in which case true
@ -194,7 +210,7 @@ proc emitByte(self: Compiler, byt: OpCode|uint8) =
## Emits a single bytecode instruction and writes it
## to the current chunk being compiled
when DEBUG_TRACE_COMPILER:
echo "Compiler.emitByte byt:" & $byt & " (uint8 value of " & $(uint8 byt) & ")"
echo "DEBUG - Compiler: Emitting " & $byt & " (uint8 value of " & $(uint8 byt) & ")"
self.currentChunk.writeChunk(uint8 byt, self.parser.previous.line)
@ -878,7 +894,7 @@ proc parseFunction(self: Compiler, funType: FunctionType) =
self.function.arity -= 1
self.function.optionals += 1
self.expression()
# self.function.defaults.add(self.parser.previous.lexeme) # TODO
self.function.defaults.add(self.parser.previous.lexeme)
defaultFollows = true
elif defaultFollows:
self.compileError("non-default argument follows default argument")
@ -911,25 +927,49 @@ proc funDeclaration(self: Compiler, named: bool = true) =
self.parseFunction(FunctionType.LAMBDA)
proc argumentList(self: Compiler): uint8 =
proc argumentList(self: Compiler): tuple[pos: uint8, kw: uint8] =
## Parses arguments passed to function calls
result = 0
result.pos = 0
result.kw = 0
if not self.parser.check(RP):
while true:
self.expression()
if result == 255:
self.compileError("cannot pass more than 255 arguments")
return
result += 1
if not self.parser.match(COMMA):
break
if self.parser.check(ID) and self.parser.checkNext(TokenType.EQ):
discard self.parser.advance()
discard self.parser.advance()
if self.parser.check(EOF):
self.parser.parseError(self.parser.previous, "Unexpected EOF")
return
else:
self.expression()
if result.pos + result.kw == 255:
self.compileError("cannot pass more than 255 arguments")
return
if not self.parser.match(COMMA):
break
result.kw += 1
else:
if self.parser.check(EOF):
self.parser.parseError(self.parser.previous, "Unexpected EOF")
return
if result.kw > 0:
self.parser.parseError(self.parser.peek, "positional argument follows default argument")
return
self.expression()
if result.pos == 255:
self.compileError("cannot pass more than 255 arguments")
return
result.pos += 1
if not self.parser.match(COMMA):
break
self.parser.consume(RP, "Expecting ')' after arguments")
proc call(self: Compiler, canAssign: bool) =
## Emits appropriate bytecode to call
## a function with its arguments
self.emitBytes(OpCode.Call, self.argumentList())
# TODO -> Keyword arguments
let args = self.argumentList()
self.emitBytes(OpCode.Call, args.pos)
proc returnStatement(self: Compiler) =
@ -1098,9 +1138,6 @@ proc getRule(kind: TokenType): ParseRule =
proc compile*(self: Compiler, source: string): ptr Function =
when DEBUG_TRACE_COMPILER:
echo "==== COMPILER debugger starts ===="
echo ""
## Compiles a source string into a function
## object. This wires up all the code
## inside the parser and the lexer
@ -1111,19 +1148,17 @@ proc compile*(self: Compiler, source: string): ptr Function =
while not self.parser.match(EOF):
self.declaration()
var function = self.endCompiler()
when DEBUG_TRACE_COMPILER:
echo "\n==== COMPILER debugger ends ===="
echo ""
if not self.parser.hadError:
when DEBUG_TRACE_COMPILER:
echo "Result: Ok"
echo "DEBUG - Compiler: Result -> Ok"
return function
else:
when DEBUG_TRACE_COMPILER:
echo "Result: Fail"
# self.freeCompiler()
echo "DEBUG - Compiler: Result -> ParseError"
return nil
else:
when DEBUG_TRACE_COMPILER:
echo "DEBUG - Compiler: Result -> LexingError"
return nil

View File

@ -31,7 +31,6 @@ proc repl() =
echo JAPL_VERSION_STRING
echo &"[Nim {NimVersion} on {hostOs} ({hostCPU})]"
when DEBUG_TRACE_VM:
echo "Debugger enabled, expect verbose output\n"
echo "==== Runtime Constants ====\n"
echo &"- FRAMES_MAX -> {FRAMES_MAX}"
echo "==== Debugger started ====\n"
@ -54,14 +53,10 @@ proc repl() =
echo &"[Nim {NimVersion} on {hostOs} ({hostCPU})]"
continue
elif source != "":
let result = bytecodeVM.interpret(source, "stdin")
discard bytecodeVM.interpret(source, "stdin")
if not bytecodeVM.lastPop.isNil():
echo stringify(bytecodeVM.lastPop)
bytecodeVM.lastPop = cast[ptr Nil](bytecodeVM.cached[2])
when DEBUG_TRACE_VM:
echo &"Result: {result}"
when DEBUG_TRACE_VM:
echo "==== Debugger exits ===="
proc main(file: var string = "", fromString: bool = false) =
@ -84,15 +79,7 @@ proc main(file: var string = "", fromString: bool = false) =
source = file
file = "<string>"
var bytecodeVM = initVM()
when DEBUG_TRACE_VM:
echo "Debugger enabled, expect verbose output\n"
echo "==== VM Constants ====\n"
echo &"- FRAMES_MAX -> {FRAMES_MAX}"
echo "==== Code starts ====\n"
let result = bytecodeVM.interpret(source, file)
echo &"Result: {result}"
when not DEBUG_TRACE_VM:
discard bytecodeVM.interpret(source, file)
discard bytecodeVM.interpret(source, file)
bytecodeVM.freeVM()

View File

@ -41,7 +41,8 @@ proc reallocate*(pointr: pointer, oldSize: int, newSize: int): pointer =
echo &"DEBUG - Memory manager: Allocating {newSize} bytes of memory"
else:
echo &"DEBUG - Memory manager: Resizing {oldSize} bytes of memory to {newSize} bytes"
result = realloc(pointr, newSize)
if oldSize > 0 and pointr != nil or oldSize == 0:
result = realloc(pointr, newSize)
except NilAccessDefect:
stderr.write("A fatal error occurred -> could not manage memory, segmentation fault\n")
quit(71) # For now, there's not much we can do if we can't get the memory we need

View File

@ -19,7 +19,7 @@ type
name*: ptr String
arity*: int # The number of required parameters
optionals*: int # The number of optional parameters
defaults*: seq[ptr Obj] # List of default arguments, in order
defaults*: seq[string]
chunk*: Chunk # The function's body

View File

@ -24,13 +24,13 @@ import strformat
proc simpleInstruction(name: string, index: int): int =
echo &"\tInstruction at IP: {name}\n"
echo &"DEBUG - VM:\tInstruction -> {name}"
return index + 1
proc byteInstruction(name: string, chunk: Chunk, offset: int): int =
var slot = chunk.code[offset + 1]
echo &"\tInstruction at IP: {name}, points to slot {slot}\n"
echo &"DEBUG - VM:\tInstruction -> {name}, points to slot {slot}"
return offset + 2
@ -38,24 +38,24 @@ proc constantInstruction(name: string, chunk: Chunk, offset: int): int =
# Rebuild the index
var constantArray: array[3, uint8] = [chunk.code[offset + 1], chunk.code[offset + 2], chunk.code[offset + 3]]
var constant: int
copyMem(constant.addr, unsafeAddr(constantArray), sizeof(constantArray))
echo &"\tInstruction at IP: {name}, points to slot {constant}"
copyMem(constant.addr, constantArray.addr, sizeof(constantArray))
echo &"DEBUG - VM:\tInstruction -> {name}, points to slot {constant}"
let obj = chunk.consts[constant]
echo &"\tOperand: {stringify(obj)}\n\tValue kind: {obj.kind}\n"
echo &"DEBUG - VM:\tOperand -> {stringify(obj)}\nDEBUG - VM:\tValue kind -> {obj.kind}"
return offset + 4
proc jumpInstruction(name: string, chunk: Chunk, offset: int): int =
var jumpArray: array[2, uint8] = [chunk.code[offset + 1], chunk.code[offset + 2]]
var jump: int
copyMem(jump.addr, unsafeAddr(jumpArray), sizeof(uint16))
echo &"\tInstruction at IP: {name}\n\tJump offset: {jump}\n"
copyMem(jump.addr, jumpArray.addr, sizeof(uint16))
echo &"DEBUG - VM:\tInstruction -> {name}\nDEBUG - VM:\tJump size -> {jump}"
return offset + 3
proc disassembleInstruction*(chunk: Chunk, offset: int): int =
## Takes one bytecode instruction and prints it
echo &"Current IP position: {offset}\nCurrent line: {chunk.lines[offset]}"
echo &"DEBUG - VM:\tOffset: {offset}\nDEBUG - VM:\tLine: {chunk.lines[offset]}"
var opcode = OpCode(chunk.code[offset])
case opcode:
of simpleInstructions:

View File

@ -77,6 +77,8 @@ func handleInterrupt() {.noconv.} =
proc resetStack*(self: VM) =
## Resets the VM stack to a blank state
when DEBUG_TRACE_VM:
echo "DEBUG - VM: Resetting the stack"
self.stack = new(seq[ptr Obj])
self.frames = @[]
self.frameCount = 0
@ -179,8 +181,14 @@ proc peek*(self: VM, distance: int): ptr Obj =
proc call(self: VM, function: ptr Function, argCount: int): bool =
## Sets up the call frame and performs error checking
## when calling callables
if argCount != function.arity:
self.error(newTypeError(&"function '{stringify(function.name)}' takes {function.arity} argument(s), got {argCount}"))
if argCount < function.arity:
var arg: string
if function.arity > 1:
arg = "s"
self.error(newTypeError(&"function '{stringify(function.name)}' takes at least {function.arity} argument{arg}, got {argCount}"))
return false
elif argCount > function.arity and (argCount - function.arity) - function.optionals != 0:
self.error(newTypeError(&"function '{stringify(function.name)}' takes at least {function.arity} arguments and at most {function.arity + function.optionals}, got {argCount}"))
return false
if self.frameCount == FRAMES_MAX:
self.error(newRecursionError("max recursion depth exceeded"))
@ -250,8 +258,8 @@ proc defineGlobal*(self: VM, name: string, value: ptr Obj) =
proc readByte(self: CallFrame): uint8 =
## Reads a single byte from the given
## frame's chunk of bytecode
result = self.function.chunk.code[self.ip]
inc(self.ip)
result = self.function.chunk.code[self.ip - 1]
proc readBytes(self: CallFrame): int =
@ -281,36 +289,39 @@ proc readConstant(self: CallFrame): ptr Obj =
proc showRuntime*(self: VM, frame: CallFrame, iteration: uint64) =
## Shows debug information about the current
## state of the virtual machine
stdout.write(&"Iteration N. {iteration}\nCurrent VM stack status: [")
stdout.write("DEBUG - VM: General information\n")
stdout.write(&"DEBUG - VM:\tIteration -> {iteration}\nDEBUG - VM:\tStack -> [")
for i, v in self.stack:
stdout.write(stringify(v))
if i < self.stack.high():
stdout.write(", ")
stdout.write("]\nCurrent global scope status: {")
stdout.write("]\nDEBUG - VM: \tGlobals -> {")
for i, (k, v) in enumerate(self.globals.pairs()):
stdout.write(&"'{k}': {stringify(v)}")
if i < self.globals.len() - 1:
stdout.write(", ")
stdout.write("}\nCurrent frame type: ")
stdout.write("}\nDEBUG - VM: Frame information\n")
stdout.write("DEBUG - VM:\tType -> ")
if frame.function.name == nil:
stdout.write("main\n")
else:
stdout.write(&"function, '{frame.function.name.stringify()}'\n")
echo &"Current frame count: {self.frameCount}"
echo &"Current frame length: {frame.len}"
stdout.write("Current frame constants table: ")
echo &"DEBUG - VM:\tCount -> {self.frameCount}"
echo &"DEBUG - VM:\tLength -> {frame.len}"
stdout.write("DEBUG - VM:\tTable -> ")
stdout.write("[")
for i, e in frame.function.chunk.consts:
stdout.write(stringify(e))
if i < frame.function.chunk.consts.high():
stdout.write(", ")
stdout.write("]\nCurrent frame stack status: ")
stdout.write("]\nDEBUG - VM:\tStack view -> ")
stdout.write("[")
for i, e in frame.getView():
stdout.write(stringify(e))
if i < len(frame) - 1:
stdout.write(", ")
stdout.write("]\n")
echo "DEBUG - VM: Current instruction"
discard disassembleInstruction(frame.function.chunk, frame.ip - 1)
@ -319,14 +330,16 @@ proc run(self: VM): InterpretResult =
## them one at a time: this is the runtime's
## main loop
var frame = self.frames[self.frameCount - 1]
var instruction: OpCode
when DEBUG_TRACE_VM:
var iteration: uint64 = 0
while true:
instruction = OpCode(frame.readByte())
{.computedgoto.} # See https://nim-lang.org/docs/manual.html#pragmas-computedgoto-pragma
when DEBUG_TRACE_VM: # Insight inside the VM
iteration += 1
self.showRuntime(frame, iteration)
case OpCode(frame.readByte()): # Main OpCodes dispatcher
case instruction: # Main OpCodes dispatcher
of OpCode.Constant:
# Loads a constant from the chunk's constant
# table
@ -635,7 +648,7 @@ proc freeObject(self: VM, obj: ptr Obj) =
ObjectType.Infinity, ObjectType.Nil, ObjectType.Native:
when DEBUG_TRACE_ALLOCATION:
if obj notin self.cached:
echo &"DEBUG- VM: Freeing {obj.typeName()} object with value '{stringify(obj)}'"
echo &"DEBUG - VM: Freeing {obj.typeName()} object with value '{stringify(obj)}'"
else:
echo &"DEBUG - VM: Freeing cached {obj.typeName()} object with value '{stringify(obj)}'"
discard free(obj.kind, obj)
@ -682,6 +695,7 @@ proc freeVM*(self: VM) =
when DEBUG_TRACE_ALLOCATION:
if self.objects.len > 0:
echo &"DEBUG - VM: Warning, {self.objects.len} objects were not freed"
echo "DEBUG - VM: The virtual machine has shut down"
proc initCache(self: VM) =
@ -694,6 +708,8 @@ proc initCache(self: VM) =
# implement proper object identity
# in a quicker way than it is done
# for equality
when DEBUG_TRACE_VM:
echo "DEBUG - VM: Initializing singletons cache"
self.cached =
[
true.asBool().asObj(),
@ -716,12 +732,17 @@ proc stdlibInit*(vm: VM) =
## take a different number of arguments according to
## how it's called) and should be handled by the nim
## procedure accordingly
vm.defineGlobal("print", newNative("print", natPrint, -1))
vm.defineGlobal("clock", newNative("clock", natClock, 0))
vm.defineGlobal("round", newNative("round", natRound, -1))
vm.defineGlobal("toInt", newNative("toInt", natToInt, 1))
vm.defineGlobal("toString", newNative("toString", natToString, 1))
vm.defineGlobal("type", newNative("type", natType, 1))
when DEBUG_TRACE_VM and not SKIP_STDLIB_INIT or not DEBUG_TRACE_VM:
when DEBUG_TRACE_VM:
echo "DEBUG - VM: Initializing stdlib"
vm.defineGlobal("print", newNative("print", natPrint, -1))
vm.defineGlobal("clock", newNative("clock", natClock, 0))
vm.defineGlobal("round", newNative("round", natRound, -1))
vm.defineGlobal("toInt", newNative("toInt", natToInt, 1))
vm.defineGlobal("toString", newNative("toString", natToString, 1))
vm.defineGlobal("type", newNative("type", natType, 1))
when DEBUG_TRACE_VM and SKIP_STDLIB_INIT:
echo "DEBUG - VM: Skipping stdlib initialization"
proc initVM*(): VM =
@ -730,21 +751,28 @@ proc initVM*(): VM =
## handlers, loading the standard
## library and preparing the stack
## and internal data structures
when DEBUG_TRACE_VM:
echo &"DEBUG - VM: Initializing the virtual machine, {JAPL_VERSION_STRING}"
result = VM(objects: @[], globals: initTable[string, ptr Obj](), source: "", file: "")
result.initCache()
result.stdlibInit()
result.resetStack()
setControlCHook(handleInterrupt)
result.lastPop = cast[ptr Nil](result.cached[2])
when DEBUG_TRACE_VM:
echo &"DEBUG - VM: Initialization complete, compiled with the following constants: FRAMES_MAX={FRAMES_MAX}, ARRAY_GROW_FACTOR={ARRAY_GROW_FACTOR}, MAP_LOAD_FACTOR={MAP_LOAD_FACTOR}"
proc interpret*(self: VM, source: string, file: string): InterpretResult =
## Interprets a source string containing JAPL code
when DEBUG_TRACE_VM:
echo &"DEBUG - VM: Preparing to run '{file}'"
self.resetStack()
self.source = source
self.file = file
when DEBUG_TRACE_VM:
echo &"DEBUG - VM: Compiling '{file}'"
var compiler = initCompiler(SCRIPT, file=file)
var compiled = compiler.compile(source)
# Here we take into account that self.interpret() might
@ -756,19 +784,23 @@ proc interpret*(self: VM, source: string, file: string): InterpretResult =
if compiled == nil:
# Compile-time error
compiler.freeCompiler()
when DEBUG_TRACE_VM:
echo "DEBUG - VM: Result -> CompileError"
return CompileError
when DEBUG_TRACE_VM:
echo "DEBUG - VM: Compilation successful"
# Since in JAPL all code runs in some
# sort of function, we push our global
# "code object" and call it like any
# other function
self.push(compiled)
discard self.callObject(compiled, 0)
when DEBUG_TRACE_VM:
echo "==== VM debugger starts ====\n"
try:
result = self.run()
except KeyboardInterrupt: # TODO: Better handling
self.error(newInterruptedError(""))
when DEBUG_TRACE_VM:
echo "DEBUG - VM: Result -> RuntimeError"
return RuntimeError
when DEBUG_TRACE_VM:
echo "==== VM debugger ends ====\n"
echo &"DEBUG - VM: Result -> {result}"

View File

@ -1,7 +1,7 @@
var a = b;
//output:Traceback (most recent call last):
//output:An unhandled exception occurred, traceback below:
//output: File '', line 1, in '<module>':
//output: File '', line 1, in <module>:
//output:ReferenceError: undefined name 'b'

View File

@ -1,7 +1,7 @@
var a = 2 + "hey";
//output:Traceback (most recent call last):
//output:An unhandled exception occurred, traceback below:
//output: File '', line 1, in '<module>':
//output: File '', line 1, in <module>:
//output:TypeError: unsupported binary operator '+' for objects of type 'integer' and 'string'

View File

@ -22,14 +22,16 @@
# Imports nim tests as well
import multibyte, os, strformat, times, re, terminal, strutils
import multibyte
import os, strformat, times, re, terminal, strutils
const tempOutputFile = ".testoutput.txt"
const testResultsPath = "testresults.txt"
# Exceptions for tests that represent not-yet implemented behaviour
const exceptions = ["all.jpl", "for_with_function.jpl", "runtime_interning.jpl"]
const exceptions = ["all.jpl", "for_with_function.jpl", "runtime_interning.jpl", "problem4.jpl"]
# for_with_function.jpl probably contains an algorithmic error too
# TODO: fix that test