diff --git a/tests/japl/errors/read_in_own_init.jpl b/tests/japl/errors/read_in_own_init.jpl new file mode 100644 index 0000000..47da9d3 --- /dev/null +++ b/tests/japl/errors/read_in_own_init.jpl @@ -0,0 +1,4 @@ +var a = 1; { var a = a; } +//output:A fatal error occurred while compiling '''', line 1, at ';' -> cannot read local variable in its own initializer + +//output: diff --git a/tests/japl/undefname.jpl b/tests/japl/errors/undefname.jpl similarity index 74% rename from tests/japl/undefname.jpl rename to tests/japl/errors/undefname.jpl index 72cbcea..9aa1325 100644 --- a/tests/japl/undefname.jpl +++ b/tests/japl/errors/undefname.jpl @@ -1,7 +1,7 @@ var a = b; //output:An unhandled exception occurred, traceback below: -//output: File '', line 1, in : +//output: File '''', line 1, in : //output:ReferenceError: undefined name 'b' diff --git a/tests/japl/unsup_binary_intstr.jpl b/tests/japl/errors/unsup_binary_intstr.jpl similarity index 80% rename from tests/japl/unsup_binary_intstr.jpl rename to tests/japl/errors/unsup_binary_intstr.jpl index a4046d2..186c6a8 100644 --- a/tests/japl/unsup_binary_intstr.jpl +++ b/tests/japl/errors/unsup_binary_intstr.jpl @@ -1,7 +1,7 @@ var a = 2 + "hey"; //output:An unhandled exception occurred, traceback below: -//output: File '', line 1, in : +//output: File '''', line 1, in : //output:TypeError: unsupported binary operator '+' for objects of type 'integer' and 'string' diff --git a/tests/japl/read_in_own_init.jpl b/tests/japl/read_in_own_init.jpl deleted file mode 100644 index 70a301f..0000000 --- a/tests/japl/read_in_own_init.jpl +++ /dev/null @@ -1,4 +0,0 @@ -var a = 1; { var a = a; } -//output:A fatal error occurred while compiling '', line 1, at ';' -> cannot read local variable in its own initializer - -//output: diff --git a/tests/jats.nim b/tests/jats.nim index bfde950..2da3da5 100644 --- a/tests/jats.nim +++ b/tests/jats.nim @@ -10,22 +10,27 @@ const exceptions = ["all.jpl", "for_with_function.jpl", "runtime_interning.jpl", # TODO: for_with_function.jpl and problem4.jpl should already be implemented, check on them proc buildTest(path: string): Test = + log(LogLevel.Debug, &"Building test {path}") + let source = readFile(path) result = Test( path: path, result: if path.extractFilename in exceptions: TestResult.Skip else: TestResult.Unstarted, - expectedOutput: compileExpectedOutput(readFile(path)) + expectedOutput: compileExpectedOutput(source), + expectedError: compileExpectedError(source) ) proc buildTests(testDir: string): seq[Test] = for candidateObj in walkDir(testDir): let candidate = candidateObj.path if dirExists(candidate): + log(LogLevel.Debug, &"Descending into dir {candidate}") result &= buildTests(candidate) else: result.add buildTest(candidate) proc runTest(test: Test, runner: string) = + log(LogLevel.Debug, &"Starting test {test.path}.") let process = startProcess(runner, args = @[test.path]) test.process = process test.result = TestResult.Running @@ -40,38 +45,42 @@ proc tryFinishTest(test: Test): bool = else: test.result = TestResult.Crash test.process.close() + log(LogLevel.Debug, &"Test {test.path} finished.") return true const maxAliveTests = 8 -const testWait = 100 +const testWait = 10 proc runTests(tests: seq[Test], runner: string) = var aliveTests = 0 currentTest = 0 + finishedTests = 0 + buffer = newBuffer() + let totalTests = tests.len() - while true: + buffer.updateProgressBar(&"", totalTests, finishedTests) + buffer.render() + while aliveTests > 0 or currentTest < tests.len(): + buffer.render() + sleep(testWait) if aliveTests < maxAliveTests and currentTest < tests.len(): if tests[currentTest].result == TestResult.Unstarted: - echo &"Doing test {$currentTest}" tests[currentTest].runTest(runner) inc aliveTests inc currentTest else: - echo &"Skipping test {$currentTest}" inc currentTest - continue - if aliveTests == 0 and currentTest >= tests.len(): - break + inc finishedTests for i in countup(0, min(currentTest, tests.high())): if tests[i].result == TestResult.Running: if tryFinishTest(tests[i]): + inc finishedTests + buffer.updateProgressBar(&"", totalTests, finishedTests) dec aliveTests - echo &"finished running {tests[i].path}" else: - echo &"{tests[i].path} still running" inc tests[i].cycles - sleep(testWait) + buffer.render() proc evalTest(test: Test) = test.output = test.output.strip() @@ -89,18 +98,67 @@ proc evalTests(tests: seq[Test]) = evalTest(test) proc printResults(tests: seq[Test]) = + var + skipped = 0 + success = 0 + fail = 0 + crash = 0 + for test in tests: - echo &"Test {test.path} {test.result}" + log(LogLevel.Debug, &"Test {test.path} result: {test.result}") + case test.result: + of TestResult.Skip: + inc skipped + of TestResult.Mismatch: + inc fail + log(LogLevel.Debug, &"[{test.path}\noutput:\n{test.output}\nerror:\n{test.error}\nexpected output:\n{test.expectedOutput}\nexpectedError:\n{test.expectedError}\n]") + of TestResult.Crash: + inc crash + log(LogLevel.Debug, &"{test.path} \ncrash:\n{test.error}") + of TestResult.Success: + inc success + else: + log(LogLevel.Error, &"Probably a testing suite bug: test {test.path} has result {test.result}") + let finalLevel = if fail == 0 and crash == 0: LogLevel.Info else: LogLevel.Error + log(finalLevel, &"{tests.len()} tests: {success} succeeded, {skipped} skipped, {fail} failed, {crash} crashed.") when isMainModule: + const jatsVersion = "(dev)" + + if paramCount() > 0: + if paramStr(1) == "-h": + echo "Usage: jats [-h | -v | -i | -o filename.txt]" + elif paramStr(1) == "-v": + echo "JATS v" & $jatsVersion + log(LogLevel.Debug, &"Welcome to JATS") var jatr = "jatr" var testDir = "japl" if not fileExists(jatr) and fileExists("tests" / jatr): + log(LogLevel.Debug, &"Must be in root: prepending \"tests\" to paths") jatr = "tests" / jatr testDir = "tests" / testDir + log(LogLevel.Info, &"Building tests...") let tests: seq[Test] = buildTests(testDir) + log(LogLevel.Debug, &"Tests built.") + log(LogLevel.Info, &"Running tests...") tests.runTests(jatr) + log(LogLevel.Debug, &"Tests ran.") + log(LogLevel.Debug, &"Evaluating tests...") tests.evalTests() + log(LogLevel.Debug, &"Tests evaluated.") tests.printResults() + log(LogLevel.Debug, &"Quitting JATS.") + + # special options to view the entire debug log + + if paramCount() > 0: + if paramStr(1) == "-i": + writeFile("testresults.txt", getTotalLog()) + discard execShellCmd("less testresults.txt") + removeFile("testresults.txt") + if paramStr(1) == "-o": + writeFile(paramStr(2), getTotalLog()) + + diff --git a/tests/runtests.nim b/tests/runtests.nim deleted file mode 100644 index 59df0a7..0000000 --- a/tests/runtests.nim +++ /dev/null @@ -1,184 +0,0 @@ -# Copyright 2020 Mattia Giambirtone -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - - - -# Common entry point to run JAPL's tests -# -# - Assumes "japl" binary in ../src/japl built with all debugging off -# - Goes through all tests in (/tests/) -# - Runs all tests in (/tests/)japl/ and checks their output (marked by `//output:{output}`) - - -# Imports nim tests as well -import multibyte - -import os, strformat, times, re, terminal, strutils - -import testutils - -const tempOutputFile = ".testoutput.txt" -const testResultsPath = "testresults.txt" - - -# Exceptions for tests that represent not-yet implemented behaviour -const exceptions = ["all.jpl", "for_with_function.jpl", "runtime_interning.jpl", "problem4.jpl"] -# for_with_function.jpl probably contains an algorithmic error too -# TODO: fix that test - -type LogLevel {.pure.} = enum - Debug, # always written to file only (large outputs, such as the entire output of the failing test or stacktrace) - Info, # important information about the progress of the test suite - Error, # failing tests (printed with red) - Stdout, # always printed to stdout only (for cli experience) - - -const echoedLogs = {LogLevel.Info, LogLevel.Error, LogLevel.Stdout} -const savedLogs = {LogLevel.Debug, LogLevel.Info, LogLevel.Error} - - -proc compileExpectedOutput(path: string): string = - for line in path.lines(): - if line =~ re"^.*//output:(.*)$": - result &= matches[0] & "\n" - - -proc deepComp(left, right: string, path: string): tuple[same: bool, place: int] = - var mleft, mright: string - result.same = true - if left.high() != right.high(): - if left.replace(path, "").high() == right.replace(path, "").high(): - mleft = left.replace(path, "") - mright = right.replace(path, "") - else: - result.same = false - else: - mleft = left - mright = right - for i in countup(0, mleft.high()): - result.place = i - if i > mright.high(): - # already false because of the len check at the beginning - # already correct place because it's updated every i - return - if mleft[i] != mright[i]: - result.same = false - return - - -proc logWithLevel(level: LogLevel, file: File, msg: string) = - let msg = &"[{$level} - {$getTime()}] {msg}" - if level in savedLogs: - file.writeLine(msg) - if level in echoedLogs: - if level == LogLevel.Error: - setForegroundColor(fgRed) - elif level == LogLevel.Info: - setForegroundColor(fgGreen) - elif level == LogLevel.Stdout: - setForegroundColor(fgYellow) - echo msg - setForegroundColor(fgDefault) - - -proc main(testsDir: string, japlExec: string, testResultsFile: File): tuple[numOfTests: int, successTests: int, failedTests: int, skippedTests: int] = - template detail(msg: string) = - logWithLevel(LogLevel.Debug, testResultsFile, msg) - template log(msg: string) = - logWithLevel(LogLevel.Info, testResultsFile, msg) - template error(msg: string) = - logWithLevel(LogLevel.Error, testResultsFile, msg) - var numOfTests = 0 - var successTests = 0 - var failedTests = 0 - var skippedTests = 0 - try: - for file in walkDir(testsDir): - block singleTest: - if file.path.extractFilename in exceptions: - detail(&"Skipping '{file.path}'") - numOfTests += 1 - skippedTests += 1 - break singleTest - elif file.path.dirExists(): - detail(&"Descending into '" & file.path & "'") - var subTestResult = main(file.path, japlExec, testResultsFile) - numOfTests += subTestResult.numOfTests - successTests += subTestResult.successTests - failedTests += subTestResult.failedTests - skippedTests += subTestResult.skippedTests - break singleTest - detail(&"Running test '{file.path}'") - if fileExists(tempOutputFile): - removeFile(tempOutputFile) # in case this crashed - let retCode = execShellCmd(&"{japlExec} {file.path} > {tempOutputFile} 2>&1") - numOfTests += 1 - if retCode != 0: - failedTests += 1 - error(&"Test '{file.path}' has crashed!") - else: - let expectedOutput = compileExpectedOutput(file.path).replace(re"(\n*)$", "") - let realOutputFile = open(tempOutputFile, fmRead) - let realOutput = realOutputFile.readAll().replace(re"([\n\r]*)$", "") - realOutputFile.close() - removeFile(tempOutputFile) - let comparison = deepComp(expectedOutput, realOutput, file.path) - if comparison.same: - successTests += 1 - log(&"Test '{file.path}' was successful") - else: - failedTests += 1 - detail(&"Expected output:\n{expectedOutput}\n") - detail(&"Received output:\n{realOutput}\n") - detail(&"Mismatch at pos {comparison.place}") - if comparison.place > expectedOutput.high() or comparison.place > realOutput.high(): - detail(&"Length mismatch") - else: - detail(&"Expected is '{expectedOutput[comparison.place]}' while received '{realOutput[comparison.place]}'") - error(&"Test '{file.path}' failed") - result = (numOfTests: numOfTests, successTests: successTests, failedTests: failedTests, skippedTests: skippedTests) - except IOError: - stderr.write(&"Fatal IO error encountered while running tests -> {getCurrentExceptionMsg()}") - - -when isMainModule: - let testResultsFile = open(testResultsPath, fmWrite) - template log (msg: string) = - logWithLevel(LogLevel.Stdout, testResultsFile, msg) - log("Running Nim tests") - # Nim tests - logWithLevel(LogLevel.Debug, testResultsFile, "Running testMultiByte") - testMultiByte() - # JAPL tests - log("Running JAPL tests") - var testsDir = "tests" / "japl" - var japlExec = "src" / "japl" - var currentDir = getCurrentDir() - # Supports running from both the project root and the tests dir itself - if currentDir.lastPathPart() == "tests": - testsDir = "japl" - japlExec = ".." / japlExec - log(&"Looking for JAPL tests in {testsDir}") - log(&"Looking for JAPL executable at {japlExec}") - if not fileExists(japlExec): - log("JAPL executable not found") - quit(1) - if not dirExists(testsDir): - log("Tests dir not found") - quit(1) - let testResult = main(testsDir, japlExec, testResultsFile) - log(&"Found {testResult.numOfTests} tests: {testResult.successTests} were successful, {testResult.failedTests} failed and {testResult.skippedTests} were skipped.") - logWithLevel(LogLevel.Stdout, testResultsFile, "Check 'testresults.txt' for details") - testResultsfile.close() - diff --git a/tests/testutils.nim b/tests/testutils.nim index eec4282..6501659 100644 --- a/tests/testutils.nim +++ b/tests/testutils.nim @@ -1,7 +1,7 @@ # Common code from between the JAPL testing suites # (during transition from runtests -> Just Another Test Runner -import re, strutils, terminal, osproc, strformat, times +import re, strutils, terminal, osproc, strformat, times, os # types @@ -19,8 +19,6 @@ type process*: Process cycles*: int - - # logging stuff type LogLevel* {.pure.} = enum @@ -35,21 +33,56 @@ const savedLogs = {LogLevel.Debug, LogLevel.Info, LogLevel.Error} const logColors = [LogLevel.Debug: fgDefault, LogLevel.Info: fgGreen, LogLevel.Error: fgRed, LogLevel.Stdout: fgYellow] -proc log*(level: LogLevel, file: File, msg: string) = +var totalLog = "" + +proc log*(level: LogLevel, msg: string) = let msg = &"[{$level} - {$getTime()}] {msg}" if level in savedLogs: - file.writeLine(msg) + totalLog &= msg & "\n" if level in echoedLogs: setForegroundColor(logColors[level]) echo msg setForegroundColor(fgDefault) +proc getTotalLog*: string = + totalLog + +const progbarLength = 25 +type Buffer* = ref object + contents: string + previous: string + +proc newBuffer*: Buffer = + new(result) + +proc updateProgressBar*(buf: Buffer, text: string, total: int, current: int) = + var newline = "" + newline &= "[" + let ratio = current / total + let filledCount = int(ratio * progbarLength) + for i in countup(1, filledCount): + newline &= "=" + for i in countup(filledCount + 1, progbarLength): + newline &= " " + newline &= &"] ({current}/{total}) {text}" + # to avoid process switching during half-written progress bars and whatnot all terminal editing happens at the end + buf.contents = newline + +proc render*(buf: Buffer) = + if buf.previous != buf.contents: + echo buf.contents + buf.previous = buf.contents # parsing the test notation proc compileExpectedOutput*(source: string): string = for line in source.split('\n'): - if line =~ re"^.*//output:(.*)$": + if line =~ re"^.*//output:[ ]?(.*)$": + result &= matches[0] & "\n" + +proc compileExpectedError*(source: string): string = + for line in source.split('\n'): + if line =~ re"^.*//error:[ ]?(.*)$": result &= matches[0] & "\n" # stuff for cleaning test output