diff --git a/README.md b/README.md index 7fd9c4d..b3ff47a 100644 --- a/README.md +++ b/README.md @@ -62,8 +62,6 @@ of what's done in JAPL: If you want to contribute, feel free to open a PR! -Right now there are some major issues with the virtual machine which need to be addressed before the development can proceed, and some help is ~~desperately needed~~ greatly appreciated! - To get started, you might want to have a look at the currently open issues and start from there ### Community @@ -72,7 +70,7 @@ Our first goal is to create a welcoming and helpful community, so if you are so ### A special thanks -JAPL is born thanks to the amazing work of Bob Nystrom that wrote a book available completely for free at [this](https://craftinginterpreters.com) link, where he describes the implementation of a simple language called Lox. +JAPL is born thanks to the amazing work of Bob Nystrom that wrote a book available completely for free at [this](https://craftinginterpreters.com) link, where he describes the implementation of a simple language called Lox ## JAPL - Installing @@ -97,7 +95,7 @@ git clone https://github.com/japl-lang/japl ### Running the build script -As a next step, you need to run the build script. This will generate the required configuration files, compile the JAPL runtime and run tests (unless `--skip-tests` is passed). There are some options that can be tweaked with command-line options, for more information, run `python3 build.py --help`. +As a next step, you need to run the build script. This will generate the required configuration files, compile the JAPL runtime and run tests (unless `--skip-tests` is used). There are some settings that can be tweaked with command-line options (or environment variables), for more information, run `python3 build.py --help`. To compile the JAPL runtime, you'll first need to move into the project's directory you cloned before, so run `cd japl`, then `python3 build.py ./src` and wait for it to complete. You should now find an executable named `japl` (or `japl.exe` on windows) inside the `src` folder. @@ -111,14 +109,13 @@ If you need more customizability or want to enable debugging for JAPL, there's a ### Nim compiler options -The build tool calls the system's nim compiler to build JAPL. If you want to customize the options passed to the compiler, you can pass a comma separated list of key:value options (spaces are not allowed). For example, doing `python3 build.py src --flags d:release,threads:on` will call `nim compile src/japl -d:release --threads:on`. +The build tool calls the system's nim compiler to build JAPL. If you want to customize the options passed to the compiler, you can pass a comma separated list of key:value pairs (spaces not allowed) via the `--flags` option. For example, doing `python3 build.py src --flags d:release,threads:on` will call `nim compile src/japl -d:release --threads:on`. #### Known issues Right now JAPL is in its very early stages and we've encountered a series of issues related to nim's garbage collection implementations. Some of them -seem to clash with JAPL's own memory management and cause random `NilAccessDefects` because the GC frees stuff that JAPL needs. If the test suite shows -weird crashes try changing the `gc` option to `boehm` (particularly recommended since it seems to cause very little interference with JAPL), or `regions` -to see if this mitigates the problem; this is a temporary solution until JAPL becomes fully independent from nim's runtime memory management. +seem to clash with JAPL's own memory management and cause random `NilAccessDefect`s because the GC frees stuff that JAPL needs. If the test suite shows +weird crashes try changing (via `--flags`) the `gc` option to `boehm` (particularly recommended since it seems to cause very little interference with JAPL), or `regions` to see if this mitigates the problem; this is a temporary solution until the JAPL VM becomes fully independent from nim's runtime memory management. ### JAPL Debugging options @@ -129,12 +126,16 @@ There are also some compile-time constants (such as the heap grow factor for the - `debug_gc` -> Debugs the garbage collector (once we have one) - `debug_alloc` -> Debugs memory allocation/deallocation - `debug_compiler` -> Debugs the compiler, showing each byte that is spit into the bytecode +- `skip_stdlib_init` -> Skips the initialization of the standard library (useful to reduce the amount of unneeded output in debug logs) +- `array_grow_factor` -> Sets the multiplicative factor by which JAPL's dynamic array implementation will increase its size when it becomes full +- `map_load_factor` -> A real value between 0 and 1 that indicates the max. % of full buckets in JAPL's hashmap implementation that are needed to trigger a resize +- `frames_max` - The max. number of call frames allowed, used to limit recursion depth -Each of these options is independent of the others and can be enabled/disabled at will. To enable an option, pass `option_name:true` to `--options` while to disable it, replace `true` with `false`. +Each of these options is independent of the others and can be enabled/disabled at will. Except for `array_grow_factor`, `map_load_factor` and `frames_max` (which take integers and a real values respectively), all other options require boolean parameters; to enable an option, pass `option_name:true` to `--options` while to disable it, replace `true` with `false`. Note that the build tool will generate a file named `config.nim` inside the `src` directory and will use that for subsequent builds, so if you want to override it you'll have to pass `--override-config` as a command-line options. Passing it without any option will fallback to (somewhat) sensible defaults -**P.S.**: The test suite assumes that all debugging options are turned off, so for development/debug builds we recommend skipping the test suite by passing `--skip-tests` to the build script +**P.S.**: For now the test suite assumes that all debugging options are turned off, so for development/debug builds we recommend skipping the test suite by passing `--skip-tests` to the build script. This will be fixed soon (the test suite will ignore debugging output) ### Installing on Linux @@ -148,5 +149,5 @@ the already existing data unless `--ignore-binary` is passed! ### Environment variables On both Windows and Linux, the build script supports reading parameters from environment variables if they are not specified via the command line. -All options follow the same naming scheme: `JAPL_OPTION_NAME=value` and will only be applied only if no explicit override for them is passed +All options follow the same naming scheme: `JAPL_OPTION_NAME=value` and will only be applied if no explicit override for them is passed when running the script diff --git a/build.py b/build.py index 5f8a475..d41ea95 100755 --- a/build.py +++ b/build.py @@ -23,7 +23,7 @@ import shutil import logging import argparse from time import time -from typing import Dict +from typing import Dict, Optional from subprocess import Popen, PIPE, DEVNULL, run @@ -46,14 +46,14 @@ import strformat const MAP_LOAD_FACTOR* = {map_load_factor} # Load factor for builtin hashmaps -const ARRAY_GROW_FACTOR* = {array_grow_factor} # How much extra memory to allocate for dynamic arrays +const ARRAY_GROW_FACTOR* = {array_grow_factor} # How much extra memory to allocate for dynamic arrays when resizing const FRAMES_MAX* = {frames_max} # The maximum recursion limit const JAPL_VERSION* = "0.3.0" const JAPL_RELEASE* = "alpha" const DEBUG_TRACE_VM* = {debug_vm} # Traces VM execution const SKIP_STDLIB_INIT* = {skip_stdlib_init} # Skips stdlib initialization in debug mode const DEBUG_TRACE_GC* = {debug_gc} # Traces the garbage collector (TODO) -const DEBUG_TRACE_ALLOCATION* = {debug_alloc} # Traces memory allocation/deallocation (WIP) +const DEBUG_TRACE_ALLOCATION* = {debug_alloc} # Traces memory allocation/deallocation const DEBUG_TRACE_COMPILER* = {debug_compiler} # Traces the compiler const JAPL_VERSION_STRING* = &"JAPL {{JAPL_VERSION}} ({{JAPL_RELEASE}}, {{CompileDate}} {{CompileTime}})" const HELP_MESSAGE* = """The JAPL runtime interface, Copyright (C) 2020 Mattia Giambirtone @@ -87,23 +87,23 @@ def run_command(command: str, mode: str = "Popen", **kwargs): if mode == "Popen": process = Popen(shlex.split(command, posix=os.name != "nt"), **kwargs) - else: - process = run(command, **kwargs) - if mode == "Popen": stdout, stderr = process.communicate() else: + process = run(command, **kwargs) stdout, stderr = None, None return stdout, stderr, process.returncode -def build(path: str, flags: Dict[str, str] = {}, options: Dict[str, bool] = {}, override: bool = False, - skip_tests: bool = False, install: bool = False, ignore_binary: bool = False): +def build(path: str, flags: Optional[Dict[str, str]] = {}, options: Optional[Dict[str, bool]] = {}, + override: Optional[bool] = False, skip_tests: Optional[bool] = False, + install: Optional[bool] = False, ignore_binary: Optional[bool] = False, + verbose: Optional[bool] = False): """ - Compiles the JAPL runtime, generating the appropriate - configuration needed for compilation to succeed, - runs tests and performs installation - when possible. - Nim 1.2 or above is required to build JAPL + Builds the JAPL runtime. + + This function generates the required configuration + according to the user's choice, runs tests and + performs installation when possible. :param path: The path to JAPL's main source directory :type path: string, optional @@ -134,6 +134,10 @@ def build(path: str, flags: Dict[str, str] = {}, options: Dict[str, bool] = {}, or folder already named "jpl" in ANY entry in PATH so this option allows to overwrite whatever data is there. Note that JAPL right now isn't aware of what it is replacing so make sure you don't lose any sensitive data! + :type ignore_binary: bool, optional + :param verbose: This parameter tells the test suite to use verbose logs, + defaults to False + :type verbose: bool, optional """ @@ -272,13 +276,20 @@ if __name__ == "__main__": for value in args.options.split(","): k, v = value.split(":", maxsplit=2) if k not in options: - logging.error("Invalid compile-time option '{key}'") + logging.error(f"Invalid compile-time option '{k}'") exit() options[k] = v except Exception: logging.error("Invalid parameter for --options") exit() - build(args.path, flags, options, args.override_config, args.skip_tests, args.install, args.ignore_binary) + build(args.path, + flags, + options, + args.override_config, + args.skip_tests, + args.install, + args.ignore_binary, + args.verbose) logging.debug("Build tool exited") except KeyboardInterrupt: logging.info("Interrupted by the user") diff --git a/src/memory.nim b/src/memory.nim index e603b2a..a6b38bb 100644 --- a/src/memory.nim +++ b/src/memory.nim @@ -33,13 +33,22 @@ proc reallocate*(pointr: pointer, oldSize: int, newSize: int): pointer = try: if newSize == 0 and pointr != nil: # pointr is awful, but clashing with builtins is even more awful when DEBUG_TRACE_ALLOCATION: - echo &"DEBUG - Memory manager: Deallocating {oldSize} bytes" + if oldSize > 1: + echo &"DEBUG - Memory manager: Deallocating {oldSize} bytes" + else: + echo "DEBUG - Memory manager: Deallocating 1 byte" dealloc(pointr) return nil + when DEBUG_TRACE_ALLOCATION: + if pointr == nil and newSize == 0: + echo &"DEBUG - Memory manager: Warning, asked to dealloc() nil pointer from {oldSize} to {newSize} bytes, ignoring request" if oldSize > 0 and pointr != nil or oldSize == 0: when DEBUG_TRACE_ALLOCATION: if oldSize == 0: - echo &"DEBUG - Memory manager: Allocating {newSize} bytes of memory" + if newSize > 1: + echo &"DEBUG - Memory manager: Allocating {newSize} bytes of memory" + else: + echo "DEBUG - Memory manager: Allocating 1 byte of memory" else: echo &"DEBUG - Memory manager: Resizing {oldSize} bytes of memory to {newSize} bytes" result = realloc(pointr, newSize) diff --git a/src/types/hashmap.nim b/src/types/hashmap.nim index 1b34d4c..9d8bd1c 100644 --- a/src/types/hashmap.nim +++ b/src/types/hashmap.nim @@ -213,6 +213,10 @@ proc `$`*[K, V](self: ptr HashMap[K, V]): string = result &= "}" +proc typeName*[K, V](self: ptr HashMap[K, V]): string = + result = "dict" + + var d = newHashMap[int, int]() d[1] = 55 d[2] = 876 diff --git a/src/types/methods.nim b/src/types/methods.nim index a7e96bd..606bd8e 100644 --- a/src/types/methods.nim +++ b/src/types/methods.nim @@ -58,6 +58,8 @@ proc typeName*(self: ptr Obj): string = result = cast[ptr Nil](self).typeName() of ObjectType.Native: result = cast[ptr Native](self).typeName() + of ObjectType.List: + result = "list" # We can't do casting here since it's not a concrete type! else: discard diff --git a/src/vm.nim b/src/vm.nim index 5d9ef30..243e255 100644 --- a/src/vm.nim +++ b/src/vm.nim @@ -21,8 +21,9 @@ import strformat import tables import std/enumerate ## Our modules -import stdlib import config +when not SKIP_STDLIB_INIT: + import stdlib import compiler import meta/opcode import meta/frame @@ -294,6 +295,7 @@ proc readConstant(self: CallFrame): ptr Obj = proc showRuntime*(self: VM, frame: CallFrame, iteration: uint64) = ## Shows debug information about the current ## state of the virtual machine + let view = frame.getView() stdout.write("DEBUG - VM: General information\n") stdout.write(&"DEBUG - VM:\tIteration -> {iteration}\nDEBUG - VM:\tStack -> [") for i, v in self.stack: @@ -312,18 +314,18 @@ proc showRuntime*(self: VM, frame: CallFrame, iteration: uint64) = else: stdout.write(&"function, '{frame.function.name.stringify()}'\n") echo &"DEBUG - VM:\tCount -> {self.frames.len()}" - echo &"DEBUG - VM:\tLength -> {frame.len}" + echo &"DEBUG - VM:\tLength -> {view.len}" stdout.write("DEBUG - VM:\tTable -> ") stdout.write("[") for i, e in frame.function.chunk.consts: stdout.write(stringify(e)) - if i < frame.function.chunk.consts.high(): + if i < len(frame.function.chunk.consts) - 1: stdout.write(", ") stdout.write("]\nDEBUG - VM:\tStack view -> ") stdout.write("[") - for i, e in frame.getView(): + for i, e in view: stdout.write(stringify(e)) - if i < len(frame) - 1: + if i < len(view) - 1: stdout.write(", ") stdout.write("]\n") echo "DEBUG - VM: Current instruction" diff --git a/tests/jatr.nim b/tests/jatr.nim index 8158719..bc89a1b 100644 --- a/tests/jatr.nim +++ b/tests/jatr.nim @@ -17,7 +17,8 @@ # a testrunner process import ../src/vm -import os, strformat +import os + var btvm = initVM() try: diff --git a/tests/jats.nim b/tests/jats.nim index abe8f20..7e90a06 100644 --- a/tests/jats.nim +++ b/tests/jats.nim @@ -15,28 +15,34 @@ # Just Another Test Suite for running JAPL tests import nim/nimtests +import testobject +import testutils +import logutils -import testobject, testutils, logutils -import os, strformat, parseopt, strutils +import os +import strformat +import parseopt +import strutils + + +const jatsVersion = "(dev)" +type + Action {.pure.} = enum + Run, Help, Version + DebugAction {.pure.} = enum + Interactive, Stdout + QuitValue {.pure.} = enum + Success, Failure, ArgParseErr, InternalErr + when isMainModule: - const jatsVersion = "(dev)" - var optparser = initOptParser(commandLineParams()) - type Action {.pure.} = enum - Run, Help, Version var action: Action = Action.Run - type DebugAction {.pure.} = enum - Interactive, Stdout var debugActions: seq[DebugAction] var targetFiles: seq[string] var verbose = true - - type QuitValue {.pure.} = enum - Success, Failure, ArgParseErr, InternalErr var quitVal = QuitValue.Success - proc evalKey(key: string) = let key = key.toLower() if key == "h" or key == "help": @@ -54,6 +60,7 @@ when isMainModule: action = Action.Help quitVal = QuitValue.ArgParseErr + proc evalKeyVal(key: string, val: string) = let key = key.toLower() if key == "o" or key == "output": @@ -63,6 +70,7 @@ when isMainModule: action = Action.Help quitVal = QuitValue.ArgParseErr + proc evalArg(key: string) = echo &"Unexpected argument" action = Action.Help @@ -80,6 +88,7 @@ when isMainModule: of cmdArgument: evalArg(optparser.key) + proc printUsage = echo """ JATS - Just Another Test Suite @@ -95,9 +104,12 @@ Flags: -h (or --help) displays this help message -v (or --version) displays the version number of JATS """ + + proc printVersion = echo &"JATS - Just Another Test Suite version {jatsVersion}" + if action == Action.Help: printUsage() quit int(quitVal) @@ -109,15 +121,11 @@ Flags: else: echo &"Unknown action {action}, please contact the devs to fix this." quit int(QuitValue.InternalErr) - setVerbosity(verbose) setLogfiles(targetFiles) - # start of JATS - try: log(LogLevel.Debug, &"Welcome to JATS") - runNimTests() var jatr = "jatr" var testDir = "japl" @@ -125,7 +133,6 @@ Flags: log(LogLevel.Debug, &"Must be in root: prepending \"tests\" to paths") jatr = "tests" / jatr testDir = "tests" / testDir - log(LogLevel.Info, &"Running JAPL tests.") log(LogLevel.Info, &"Building tests...") let tests: seq[Test] = buildTests(testDir) @@ -138,9 +145,7 @@ Flags: log(LogLevel.Debug, &"Tests evaluated.") if not tests.printResults(): quitVal = QuitValue.Failure - log(LogLevel.Debug, &"Quitting JATS.") - # special options to view the entire debug log finally: let logs = getTotalLog() @@ -158,6 +163,5 @@ Flags: write stderr, "Interactive mode not supported on your platform, try --stdout and piping, or install/alias 'more' or 'less' to a terminal pager.\n" of DebugAction.Stdout: echo logs - quit int(quitVal) diff --git a/tests/maketest.nim b/tests/maketest.nim index 1c8d787..3d062e0 100644 --- a/tests/maketest.nim +++ b/tests/maketest.nim @@ -14,19 +14,22 @@ # Test creation tool, use mainly for exceptions -# +import os +import strformat +import re +import strutils -# Imports nim tests as well -import multibyte, os, strformat, times, re, terminal, strutils const tempCodeFile = ".tempcode_drEHdZuwNYLqsQaMDMqeNRtmqoqXBXfnCfeqEcmcUYJToBVQkF.jpl" const tempOutputFile = ".tempoutput.txt" -proc autoremove(path: string) = + +proc autoRemove(path: string) = if fileExists(path): removeFile(path) + when isMainModule: var testsDir = "tests" / "japl" var japlExec = "src" / "japl" @@ -41,16 +44,13 @@ when isMainModule: if not dirExists(testsDir): echo "Tests dir not found" quit(1) - echo "Please enter the JAPL code or specify a file containing it with file:" - let response = stdin.readLine() if response =~ re"^file:(.*)$": let codepath = matches[0] writeFile(tempCodeFile, readFile(codepath)) else: writeFile(tempCodeFile, response) - let japlCode = readFile(tempCodeFile) discard execShellCmd(&"{japlExec} {tempCodeFile} > {tempOutputFile} 2>&1") var output: string @@ -59,9 +59,8 @@ when isMainModule: else: echo "Temporary output file not detected, aborting" quit(1) - autoremove(tempCodeFile) - autoremove(tempOutputFile) - + autoRemove(tempCodeFile) + autoRemove(tempOutputFile) echo "Got the following output:" echo output echo "Do you want to keep it as a test? [y/N]" diff --git a/tests/nim/nimtests.nim b/tests/nim/nimtests.nim index 9ba03e5..b36f321 100644 --- a/tests/nim/nimtests.nim +++ b/tests/nim/nimtests.nim @@ -1,10 +1,12 @@ import multibyte import ../logutils + proc runNimTests* = log(LogLevel.Info, "Running nim tests.") testMultibyte() log(LogLevel.Debug, "Nim tests finished") + when isMainModule: runNimTests() diff --git a/tests/testobject.nim b/tests/testobject.nim index b4c62ed..a947a6e 100644 --- a/tests/testobject.nim +++ b/tests/testobject.nim @@ -40,6 +40,7 @@ proc compileExpectedOutput*(source: string): string = if line =~ re"^.*//stdout:[ ]?(.*)$": result &= matches[0] & "\n" + proc compileExpectedError*(source: string): string = for line in source.split('\n'): if line =~ re"^.*//stderr:[ ]?(.*)$": diff --git a/tests/testutils.nim b/tests/testutils.nim index ce6160d..f5aad95 100644 --- a/tests/testutils.nim +++ b/tests/testutils.nim @@ -17,8 +17,9 @@ import testobject, logutils, os, osproc, streams, strformat # Tests that represent not-yet implemented behaviour -const exceptions = ["all.jpl", "for_with_function.jpl", "runtime_interning.jpl", "problem4.jpl"] -# TODO: for_with_function.jpl and problem4.jpl should already be implemented, check on them +const exceptions = ["all.jpl", "for_with_function.jpl", "runtime_interning.jpl"] +# TODO: for_with_function.jpl should already be implemented, check on it + proc buildTest(path: string): Test = log(LogLevel.Debug, &"Building test {path}") @@ -32,6 +33,7 @@ proc buildTest(path: string): Test = input: compileInput(source) ) + proc buildTests*(testDir: string): seq[Test] = for candidateObj in walkDir(testDir): let candidate = candidateObj.path @@ -41,6 +43,7 @@ proc buildTests*(testDir: string): seq[Test] = else: result.add buildTest(candidate) + proc runTest(test: Test, runner: string) = log(LogLevel.Debug, &"Starting test {test.path}.") let process = startProcess(runner, args = @[test.path]) @@ -56,6 +59,7 @@ proc runTest(test: Test, runner: string) = test.result = TestResult.Running + proc tryFinishTest(test: Test): bool = if test.process.running(): return false @@ -69,6 +73,7 @@ proc tryFinishTest(test: Test): bool = log(LogLevel.Debug, &"Test {test.path} finished.") return true + proc killTest(test: Test) = if test.process.running(): test.process.kill() @@ -76,10 +81,12 @@ proc killTest(test: Test) = log(LogLevel.Error, &"Test {test.path} was killed for taking too long.") discard test.tryFinishTest() + const maxAliveTests = 16 const testWait = 100 const timeout = 100 # number of cycles after which a test is killed for timeout + proc runTests*(tests: seq[Test], runner: string) = var aliveTests = 0 @@ -87,7 +94,6 @@ proc runTests*(tests: seq[Test], runner: string) = finishedTests = 0 buffer = newBuffer() let totalTests = tests.len() - buffer.updateProgressBar(&"", totalTests, finishedTests) buffer.render() while aliveTests > 0 or currentTest < tests.len(): @@ -117,6 +123,7 @@ proc runTests*(tests: seq[Test], runner: string) = buffer.render() buffer.endBuffer() + proc evalTest(test: Test) = test.output = test.output.tuStrip() test.error = test.error.tuStrip() @@ -127,18 +134,19 @@ proc evalTest(test: Test) = else: test.result = TestResult.Success + proc evalTests*(tests: seq[Test]) = for test in tests: if test.result == TestResult.ToEval: evalTest(test) + proc printResults*(tests: seq[Test]): bool = var skipped = 0 success = 0 fail = 0 crash = 0 - for test in tests: log(LogLevel.Debug, &"Test {test.path} result: {test.result}") case test.result: @@ -156,7 +164,6 @@ proc printResults*(tests: seq[Test]): bool = log(LogLevel.Error, &"Probably a testing suite bug: test {test.path} has result {test.result}") let finalLevel = if fail == 0 and crash == 0: LogLevel.Info else: LogLevel.Error log(finalLevel, &"{tests.len()} tests: {success} succeeded, {skipped} skipped, {fail} failed, {crash} crashed.") - - fail == 0 and crash == 0 + result = fail == 0 and crash == 0