diff --git a/.gitignore b/.gitignore index baf9fb5..2af0abf 100644 --- a/.gitignore +++ b/.gitignore @@ -24,6 +24,8 @@ testresults.txt .tempcode_drEHdZuwNYLqsQaMDMqeNRtmqoqXBXfnCfeqEcmcUYJToBVQkF.jpl .tempoutput.txt config.nim +jatr +jats # MacOS diff --git a/build.py b/build.py index 121aaa1..d175825 100755 --- a/build.py +++ b/build.py @@ -157,20 +157,27 @@ def build(path: str, flags: Dict[str, str] = {}, options: Dict[str, bool] = {}, logging.info("Running tests under tests/") logging.debug("Compiling test suite") start = time() - tests_path = "./tests/runtests" if os.name != "nt" else ".\tests\runtests" - _, stderr, status = run_command(f"nim compile {tests_path}", stdout=DEVNULL, stderr=PIPE) + test_runner_path = "./tests/jatr" if os.name != "nt" else ".\tests\jatr" + tests_path = "./tests/jats" if os.name != "nt" else ".\tests\jats" + command = "nim {flags} compile {path}".format(flags=nim_flags, path=test_runner_path) + _, stderr, status = run_command(command, stdout=DEVNULL, stderr=PIPE) if status != 0: logging.error(f"Command '{command}' exited with non-0 exit code {status}, output below:\n{stderr.decode()}") else: - logging.debug(f"Test suite compilation completed in {time() - start:.2f} seconds") - logging.debug("Running tests") - start = time() - # TODO: Find a better way of running the test suite - process = run_command(f"{tests_path}", mode="run", shell=True, stderr=PIPE) + command = f"nim compile {tests_path}" + _, stderr, status = run_command(command, stdout=DEVNULL, stderr=PIPE) if status != 0: logging.error(f"Command '{command}' exited with non-0 exit code {status}, output below:\n{stderr.decode()}") else: - logging.debug(f"Test suite ran in {time() - start:.2f} seconds") + logging.debug(f"Test suite compilation completed in {time() - start:.2f} seconds") + logging.debug("Running tests") + start = time() + # TODO: Find a better way of running the test suite + process = run_command(f"{tests_path}", mode="run", shell=True, stderr=PIPE) + if status != 0: + logging.error(f"Command '{command}' exited with non-0 exit code {status}, output below:\n{stderr.decode()}") + else: + logging.debug(f"Test suite ran in {time() - start:.2f} seconds") logging.info("Test suite completed!") if args.install: if os.name == "nt": @@ -263,4 +270,4 @@ if __name__ == "__main__": logging.warning(f"Could not remove test results file due to a {type(error).__name__}: {error}") logging.debug("Build tool exited") except KeyboardInterrupt: - logging.info("Interrupted by the user") \ No newline at end of file + logging.info("Interrupted by the user") diff --git a/tests/japl/errors/read_in_own_init.jpl b/tests/japl/errors/read_in_own_init.jpl new file mode 100644 index 0000000..47da9d3 --- /dev/null +++ b/tests/japl/errors/read_in_own_init.jpl @@ -0,0 +1,4 @@ +var a = 1; { var a = a; } +//output:A fatal error occurred while compiling '''', line 1, at ';' -> cannot read local variable in its own initializer + +//output: diff --git a/tests/japl/undefname.jpl b/tests/japl/errors/undefname.jpl similarity index 74% rename from tests/japl/undefname.jpl rename to tests/japl/errors/undefname.jpl index 72cbcea..9aa1325 100644 --- a/tests/japl/undefname.jpl +++ b/tests/japl/errors/undefname.jpl @@ -1,7 +1,7 @@ var a = b; //output:An unhandled exception occurred, traceback below: -//output: File '', line 1, in : +//output: File '''', line 1, in : //output:ReferenceError: undefined name 'b' diff --git a/tests/japl/unsup_binary_intstr.jpl b/tests/japl/errors/unsup_binary_intstr.jpl similarity index 80% rename from tests/japl/unsup_binary_intstr.jpl rename to tests/japl/errors/unsup_binary_intstr.jpl index a4046d2..186c6a8 100644 --- a/tests/japl/unsup_binary_intstr.jpl +++ b/tests/japl/errors/unsup_binary_intstr.jpl @@ -1,7 +1,7 @@ var a = 2 + "hey"; //output:An unhandled exception occurred, traceback below: -//output: File '', line 1, in : +//output: File '''', line 1, in : //output:TypeError: unsupported binary operator '+' for objects of type 'integer' and 'string' diff --git a/tests/japl/read_in_own_init.jpl b/tests/japl/read_in_own_init.jpl deleted file mode 100644 index 70a301f..0000000 --- a/tests/japl/read_in_own_init.jpl +++ /dev/null @@ -1,4 +0,0 @@ -var a = 1; { var a = a; } -//output:A fatal error occurred while compiling '', line 1, at ';' -> cannot read local variable in its own initializer - -//output: diff --git a/tests/jatr.nim b/tests/jatr.nim new file mode 100644 index 0000000..4cbd25b --- /dev/null +++ b/tests/jatr.nim @@ -0,0 +1,15 @@ +# Just Another Test Runner for running JAPL tests +# a testrunner process + +import ../src/vm +import os, strformat + +var btvm = initVM() +try: + discard btvm.interpret(readFile(paramStr(1)), "") + quit(0) +except: + let error = getCurrentException() + writeLine stderr, error.msg + quit(1) + diff --git a/tests/jats.nim b/tests/jats.nim new file mode 100644 index 0000000..94cdfbe --- /dev/null +++ b/tests/jats.nim @@ -0,0 +1,172 @@ +# Just Another Test Suite for running JAPL tests + +import nim/nimtests + +import ../src/vm +import testutils + +import os, osproc, strformat, streams + +# Tests that represent not-yet implemented behaviour +const exceptions = ["all.jpl", "for_with_function.jpl", "runtime_interning.jpl", "problem4.jpl"] +# TODO: for_with_function.jpl and problem4.jpl should already be implemented, check on them + +proc buildTest(path: string): Test = + log(LogLevel.Debug, &"Building test {path}") + let source = readFile(path) + result = Test( + path: path, + result: if path.extractFilename in exceptions: TestResult.Skip + else: TestResult.Unstarted, + expectedOutput: compileExpectedOutput(source), + expectedError: compileExpectedError(source) + ) + +proc buildTests(testDir: string): seq[Test] = + for candidateObj in walkDir(testDir): + let candidate = candidateObj.path + if dirExists(candidate): + log(LogLevel.Debug, &"Descending into dir {candidate}") + result &= buildTests(candidate) + else: + result.add buildTest(candidate) + +proc runTest(test: Test, runner: string) = + log(LogLevel.Debug, &"Starting test {test.path}.") + let process = startProcess(runner, args = @[test.path]) + test.process = process + test.result = TestResult.Running + +proc tryFinishTest(test: Test): bool = + if test.process.running(): + return false + test.output = test.process.outputStream.readAll() + test.error = test.process.errorStream.readAll() + if test.process.peekExitCode() == 0: + test.result = TestResult.ToEval + else: + test.result = TestResult.Crash + test.process.close() + log(LogLevel.Debug, &"Test {test.path} finished.") + return true + +const maxAliveTests = 8 +const testWait = 10 + +proc runTests(tests: seq[Test], runner: string) = + var + aliveTests = 0 + currentTest = 0 + finishedTests = 0 + buffer = newBuffer() + let totalTests = tests.len() + + buffer.updateProgressBar(&"", totalTests, finishedTests) + buffer.render() + while aliveTests > 0 or currentTest < tests.len(): + buffer.render() + sleep(testWait) + if aliveTests < maxAliveTests and currentTest < tests.len(): + if tests[currentTest].result == TestResult.Unstarted: + tests[currentTest].runTest(runner) + inc aliveTests + inc currentTest + else: + inc currentTest + inc finishedTests + for i in countup(0, min(currentTest, tests.high())): + if tests[i].result == TestResult.Running: + if tryFinishTest(tests[i]): + inc finishedTests + buffer.updateProgressBar(&"", totalTests, finishedTests) + dec aliveTests + else: + inc tests[i].cycles + buffer.render() + +proc evalTest(test: Test) = + test.output = test.output.strip() + test.error = test.error.strip() + test.expectedOutput = test.expectedOutput.strip() + test.expectedError = test.expectedError.strip() + if test.output != test.expectedOutput or test.error != test.expectedError: + test.result = TestResult.Mismatch + else: + test.result = TestResult.Success + +proc evalTests(tests: seq[Test]) = + for test in tests: + if test.result == TestResult.ToEval: + evalTest(test) + +proc printResults(tests: seq[Test]) = + var + skipped = 0 + success = 0 + fail = 0 + crash = 0 + + for test in tests: + log(LogLevel.Debug, &"Test {test.path} result: {test.result}") + case test.result: + of TestResult.Skip: + inc skipped + of TestResult.Mismatch: + inc fail + log(LogLevel.Debug, &"[{test.path}\noutput:\n{test.output}\nerror:\n{test.error}\nexpected output:\n{test.expectedOutput}\nexpectedError:\n{test.expectedError}\n]") + of TestResult.Crash: + inc crash + log(LogLevel.Debug, &"{test.path} \ncrash:\n{test.error}") + of TestResult.Success: + inc success + else: + log(LogLevel.Error, &"Probably a testing suite bug: test {test.path} has result {test.result}") + let finalLevel = if fail == 0 and crash == 0: LogLevel.Info else: LogLevel.Error + log(finalLevel, &"{tests.len()} tests: {success} succeeded, {skipped} skipped, {fail} failed, {crash} crashed.") + +when isMainModule: + const jatsVersion = "(dev)" + + if paramCount() > 0: + if paramStr(1) == "-h": + echo "Usage: jats [-h | -v | -i | -o filename.txt]" + quit(0) + elif paramStr(1) == "-v": + echo "JATS v" & $jatsVersion + quit(0) + + log(LogLevel.Debug, &"Welcome to JATS") + + runNimTests() + var jatr = "jatr" + var testDir = "japl" + if not fileExists(jatr) and fileExists("tests" / jatr): + log(LogLevel.Debug, &"Must be in root: prepending \"tests\" to paths") + jatr = "tests" / jatr + testDir = "tests" / testDir + + log(LogLevel.Info, &"Running JAPL tests.") + log(LogLevel.Info, &"Building tests...") + let tests: seq[Test] = buildTests(testDir) + log(LogLevel.Debug, &"Tests built.") + log(LogLevel.Info, &"Running tests...") + tests.runTests(jatr) + log(LogLevel.Debug, &"Tests ran.") + log(LogLevel.Debug, &"Evaluating tests...") + tests.evalTests() + log(LogLevel.Debug, &"Tests evaluated.") + tests.printResults() + log(LogLevel.Debug, &"Quitting JATS.") + + # special options to view the entire debug log + + if paramCount() > 0: + if paramStr(1) == "-i": + writeFile("testresults.txt", getTotalLog()) + discard execShellCmd("less testresults.txt") + removeFile("testresults.txt") + if paramStr(1) == "-o": + writeFile(paramStr(2), getTotalLog()) + + + diff --git a/tests/multibyte.nim b/tests/nim/multibyte.nim similarity index 96% rename from tests/multibyte.nim rename to tests/nim/multibyte.nim index 2cd2b93..ecf2a15 100644 --- a/tests/multibyte.nim +++ b/tests/nim/multibyte.nim @@ -15,7 +15,7 @@ # Tests that our multibyte module works -import ../src/multibyte +import ../../src/multibyte template testMultibyte* = diff --git a/tests/nim/nimtests.nim b/tests/nim/nimtests.nim new file mode 100644 index 0000000..964e0bd --- /dev/null +++ b/tests/nim/nimtests.nim @@ -0,0 +1,10 @@ +import multibyte +import ../testutils + +proc runNimTests* = + log(LogLevel.Info, "Running nim tests.") + testMultibyte() + log(LogLevel.Debug, "Nim tests finished") + +when isMainModule: + runNimTests() diff --git a/tests/runtests.nim b/tests/runtests.nim deleted file mode 100644 index 6e88ed9..0000000 --- a/tests/runtests.nim +++ /dev/null @@ -1,182 +0,0 @@ -# Copyright 2020 Mattia Giambirtone -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - - - -# Common entry point to run JAPL's tests -# -# - Assumes "japl" binary in ../src/japl built with all debugging off -# - Goes through all tests in (/tests/) -# - Runs all tests in (/tests/)japl/ and checks their output (marked by `//output:{output}`) - - -# Imports nim tests as well -import multibyte - -import os, strformat, times, re, terminal, strutils - -const tempOutputFile = ".testoutput.txt" -const testResultsPath = "testresults.txt" - - -# Exceptions for tests that represent not-yet implemented behaviour -const exceptions = ["all.jpl", "for_with_function.jpl", "runtime_interning.jpl", "problem4.jpl"] -# for_with_function.jpl probably contains an algorithmic error too -# TODO: fix that test - -type LogLevel {.pure.} = enum - Debug, # always written to file only (large outputs, such as the entire output of the failing test or stacktrace) - Info, # important information about the progress of the test suite - Error, # failing tests (printed with red) - Stdout, # always printed to stdout only (for cli experience) - - -const echoedLogs = {LogLevel.Info, LogLevel.Error, LogLevel.Stdout} -const savedLogs = {LogLevel.Debug, LogLevel.Info, LogLevel.Error} - - -proc compileExpectedOutput(path: string): string = - for line in path.lines(): - if line =~ re"^.*//output:(.*)$": - result &= matches[0] & "\n" - - -proc deepComp(left, right: string, path: string): tuple[same: bool, place: int] = - var mleft, mright: string - result.same = true - if left.high() != right.high(): - if left.replace(path, "").high() == right.replace(path, "").high(): - mleft = left.replace(path, "") - mright = right.replace(path, "") - else: - result.same = false - else: - mleft = left - mright = right - for i in countup(0, mleft.high()): - result.place = i - if i > mright.high(): - # already false because of the len check at the beginning - # already correct place because it's updated every i - return - if mleft[i] != mright[i]: - result.same = false - return - - -proc logWithLevel(level: LogLevel, file: File, msg: string) = - let msg = &"[{$level} - {$getTime()}] {msg}" - if level in savedLogs: - file.writeLine(msg) - if level in echoedLogs: - if level == LogLevel.Error: - setForegroundColor(fgRed) - elif level == LogLevel.Info: - setForegroundColor(fgGreen) - elif level == LogLevel.Stdout: - setForegroundColor(fgYellow) - echo msg - setForegroundColor(fgDefault) - - -proc main(testsDir: string, japlExec: string, testResultsFile: File): tuple[numOfTests: int, successTests: int, failedTests: int, skippedTests: int] = - template detail(msg: string) = - logWithLevel(LogLevel.Debug, testResultsFile, msg) - template log(msg: string) = - logWithLevel(LogLevel.Info, testResultsFile, msg) - template error(msg: string) = - logWithLevel(LogLevel.Error, testResultsFile, msg) - var numOfTests = 0 - var successTests = 0 - var failedTests = 0 - var skippedTests = 0 - try: - for file in walkDir(testsDir): - block singleTest: - if file.path.extractFilename in exceptions: - detail(&"Skipping '{file.path}'") - numOfTests += 1 - skippedTests += 1 - break singleTest - elif file.path.dirExists(): - detail(&"Descending into '" & file.path & "'") - var subTestResult = main(file.path, japlExec, testResultsFile) - numOfTests += subTestResult.numOfTests - successTests += subTestResult.successTests - failedTests += subTestResult.failedTests - skippedTests += subTestResult.skippedTests - break singleTest - detail(&"Running test '{file.path}'") - if fileExists(tempOutputFile): - removeFile(tempOutputFile) # in case this crashed - let retCode = execShellCmd(&"{japlExec} {file.path} > {tempOutputFile} 2>&1") - numOfTests += 1 - if retCode != 0: - failedTests += 1 - error(&"Test '{file.path}' has crashed!") - else: - let expectedOutput = compileExpectedOutput(file.path).replace(re"(\n*)$", "") - let realOutputFile = open(tempOutputFile, fmRead) - let realOutput = realOutputFile.readAll().replace(re"([\n\r]*)$", "") - realOutputFile.close() - removeFile(tempOutputFile) - let comparison = deepComp(expectedOutput, realOutput, file.path) - if comparison.same: - successTests += 1 - log(&"Test '{file.path}' was successful") - else: - failedTests += 1 - detail(&"Expected output:\n{expectedOutput}\n") - detail(&"Received output:\n{realOutput}\n") - detail(&"Mismatch at pos {comparison.place}") - if comparison.place > expectedOutput.high() or comparison.place > realOutput.high(): - detail(&"Length mismatch") - else: - detail(&"Expected is '{expectedOutput[comparison.place]}' while received '{realOutput[comparison.place]}'") - error(&"Test '{file.path}' failed") - result = (numOfTests: numOfTests, successTests: successTests, failedTests: failedTests, skippedTests: skippedTests) - except IOError: - stderr.write(&"Fatal IO error encountered while running tests -> {getCurrentExceptionMsg()}") - - -when isMainModule: - let testResultsFile = open(testResultsPath, fmWrite) - template log (msg: string) = - logWithLevel(LogLevel.Stdout, testResultsFile, msg) - log("Running Nim tests") - # Nim tests - logWithLevel(LogLevel.Debug, testResultsFile, "Running testMultiByte") - testMultiByte() - # JAPL tests - log("Running JAPL tests") - var testsDir = "tests" / "japl" - var japlExec = "src" / "japl" - var currentDir = getCurrentDir() - # Supports running from both the project root and the tests dir itself - if currentDir.lastPathPart() == "tests": - testsDir = "japl" - japlExec = ".." / japlExec - log(&"Looking for JAPL tests in {testsDir}") - log(&"Looking for JAPL executable at {japlExec}") - if not fileExists(japlExec): - log("JAPL executable not found") - quit(1) - if not dirExists(testsDir): - log("Tests dir not found") - quit(1) - let testResult = main(testsDir, japlExec, testResultsFile) - log(&"Found {testResult.numOfTests} tests: {testResult.successTests} were successful, {testResult.failedTests} failed and {testResult.skippedTests} were skipped.") - logWithLevel(LogLevel.Stdout, testResultsFile, "Check 'testresults.txt' for details") - testResultsfile.close() - diff --git a/tests/testutils.nim b/tests/testutils.nim new file mode 100644 index 0000000..6501659 --- /dev/null +++ b/tests/testutils.nim @@ -0,0 +1,92 @@ +# Common code from between the JAPL testing suites +# (during transition from runtests -> Just Another Test Runner + +import re, strutils, terminal, osproc, strformat, times, os + +# types + +type + TestResult* {.pure.} = enum + Unstarted, Running, ToEval, Success, Skip, Mismatch, Crash + + Test* = ref object + result*: TestResult + path*: string + expectedOutput*: string + expectedError*: string + output*: string + error*: string + process*: Process + cycles*: int + +# logging stuff + +type LogLevel* {.pure.} = enum + Debug, # always written to file only (large outputs, such as the entire output of the failing test or stacktrace) + Info, # important information about the progress of the test suite + Error, # failing tests (printed with red) + Stdout, # always printed to stdout only (for cli experience) + + +const echoedLogs = {LogLevel.Info, LogLevel.Error, LogLevel.Stdout} +const savedLogs = {LogLevel.Debug, LogLevel.Info, LogLevel.Error} + +const logColors = [LogLevel.Debug: fgDefault, LogLevel.Info: fgGreen, LogLevel.Error: fgRed, LogLevel.Stdout: fgYellow] + +var totalLog = "" + +proc log*(level: LogLevel, msg: string) = + let msg = &"[{$level} - {$getTime()}] {msg}" + if level in savedLogs: + totalLog &= msg & "\n" + if level in echoedLogs: + setForegroundColor(logColors[level]) + echo msg + setForegroundColor(fgDefault) + +proc getTotalLog*: string = + totalLog + +const progbarLength = 25 +type Buffer* = ref object + contents: string + previous: string + +proc newBuffer*: Buffer = + new(result) + +proc updateProgressBar*(buf: Buffer, text: string, total: int, current: int) = + var newline = "" + newline &= "[" + let ratio = current / total + let filledCount = int(ratio * progbarLength) + for i in countup(1, filledCount): + newline &= "=" + for i in countup(filledCount + 1, progbarLength): + newline &= " " + newline &= &"] ({current}/{total}) {text}" + # to avoid process switching during half-written progress bars and whatnot all terminal editing happens at the end + buf.contents = newline + +proc render*(buf: Buffer) = + if buf.previous != buf.contents: + echo buf.contents + buf.previous = buf.contents + +# parsing the test notation + +proc compileExpectedOutput*(source: string): string = + for line in source.split('\n'): + if line =~ re"^.*//output:[ ]?(.*)$": + result &= matches[0] & "\n" + +proc compileExpectedError*(source: string): string = + for line in source.split('\n'): + if line =~ re"^.*//error:[ ]?(.*)$": + result &= matches[0] & "\n" + +# stuff for cleaning test output + +proc strip*(input: string): string = + return input.replace(re"[\n\r]*$", "") +