diff --git a/tests/japl/fib.jpl b/tests/japl/fib.jpl index 99a8236..b42cdf2 100644 --- a/tests/japl/fib.jpl +++ b/tests/japl/fib.jpl @@ -5,15 +5,15 @@ fun fib(n) { return n; return fib(n-2) + fib(n-1); } -print(fib(1));//stdout:1 -print(fib(2));//stdout:1 -print(fib(3));//stdout:2 -print(fib(4));//stdout:3 -print(fib(5));//stdout:5 -print(fib(6));//stdout:8 -print(fib(7));//stdout:13 -print(fib(8));//stdout:21 -print(fib(9));//stdout:3 +print(fib(1)); +print(fib(2)); +print(fib(3)); +print(fib(4)); +print(fib(5)); +print(fib(6)); +print(fib(7)); +print(fib(8)); +print(fib(9)); [end] [stdout] 1 diff --git a/tests/jats.nim b/tests/jats.nim index da7f494..372c8d7 100644 --- a/tests/jats.nim +++ b/tests/jats.nim @@ -54,6 +54,7 @@ when isMainModule: var targetFiles: seq[string] var verbose = true var quitVal = QuitValue.Success + var testDir = "japl" proc evalKey(key: string) = ## Modifies the globals that define what JATS does based on the @@ -69,6 +70,10 @@ when isMainModule: verbose = false elif key == "stdout": debugActions.add(DebugAction.Stdout) + elif key == "f" or key == "force": + force = true + elif key == "e" or key == "enumerate": + enumerate = true else: echo &"Unknown flag: {key}" action = Action.Help @@ -88,6 +93,24 @@ when isMainModule: echo "Can't parse non-integer option passed to -j/--jobs." action = Action.Help quitVal = QuitValue.ArgParseErr + elif key == "t" or key == "test" or key == "tests": + testDir = val + elif key == "timeout": + try: + var timeoutSeconds = parseFloat(val) + # a round is 100 ms, so let's not get close to that + if timeoutSeconds < 0.3: + timeoutSeconds = 0.3 + # I don't want anything not nicely convertible to int, + # so how about cut it off at 10 hours. Open an issue + # if that's not enough... or just tweak it you lunatic + if timeoutSeconds > 36000.0: + timeoutSeconds = 36000.0 + timeout = (timeoutSeconds * 10).int + except ValueError: + echo "Can't parse invalid timeout value " & val + action = Action.Help + quitVal = QuitValue.ArgParseErr else: echo &"Unknown option: {key}" action = Action.Help @@ -119,15 +142,18 @@ when isMainModule: echo """ JATS - Just Another Test Suite -Usage: -jats -Runs the tests -Flags: +Usage: ./jats +Debug output flags: -i (or --interactive) displays all debug info -o: (or --output:) saves debug info to a file -s (or --silent) will disable all output (except --stdout) --stdout will put all debug info to stdout +-e (or --enumerate) will list all tests that fail, crash or get killed +Test behavior flags: -j: (or --jobs:) to specify number of tests to run parallel +-t: (or --test: or --tests:) to specify where tests are +-f (or --force) will run skipped tests +Miscellaneous flags: -h (or --help) displays this help message -v (or --version) displays the version number of JATS """ @@ -167,18 +193,24 @@ Flags: # the second half of the test suite defined in ~japl/tests/japl # Find ~japl/tests/japl and the test runner JATR var jatr = "jatr" - var testDir = "japl" if not fileExists(jatr): if fileExists("tests" / jatr): log(LogLevel.Debug, - &"Must be in root: prepending \"tests\" to paths") + &"Must be in root: prepending \"tests\" to jatr path") jatr = "tests" / jatr - testDir = "tests" / testDir else: # only those two dirs are realistically useful for now, - echo "The tests directory couldn't be found." + echo "The test runner was not found." quit int(QuitValue.JatrNotFound) + if not dirExists(testDir) and not fileExists(testDir): + if dirExists("tests" / testDir) or fileExists("tests" / testDir): + log(LogLevel.Debug, "Prepending \"tests\" to test path") + testDir = "tests" / testDir + else: + echo "The test dir/file was not found." + quit int(QuitValue.JatrNotFound) + # set the global var which specifies the path to the test runner testRunner = jatr log(LogLevel.Info, &"Running JAPL tests.") @@ -196,7 +228,7 @@ Flags: setControlCHook(ctrlc) log(LogLevel.Info, &"Running tests...") # run tests (see testrun.nim) - tests.runTests(jatr) + tests.runTests() log(LogLevel.Debug, &"Tests ran.") log(LogLevel.Debug, &"Evaluating tests...") # evaluate tests (see testeval.nim) diff --git a/tests/logutils.nim b/tests/logutils.nim index 5c55f15..2df5d86 100644 --- a/tests/logutils.nim +++ b/tests/logutils.nim @@ -35,20 +35,24 @@ type LogLevel* {.pure.} = enum ## All the different possible log levels Debug, # always written to file only (large outputs, such as the entire output of the failing test or stacktrace) Info, # important information about the progress of the test suite + Enumeration, # a white output for the enumerate option Error, # failing tests (printed with yellow) Fatal # always printed with red, halts the entire suite (test parsing errors, printed with red) # log config: which log levels to show, show in silent mode and save to the # detailed debug logs -const echoedLogs = {LogLevel.Info, LogLevel.Error, LogLevel.Fatal} -const echoedLogsSilent = {LogLevel.Fatal} # will be echoed even if test suite is silent -const savedLogs = {LogLevel.Debug, LogLevel.Info, LogLevel.Error, LogLevel.Fatal} +const echoedLogs = {LogLevel.Info, LogLevel.Error, LogLevel.Fatal, + LogLevel.Enumeration} +const echoedLogsSilent = {LogLevel.Fatal, LogLevel.Enumeration} # will be echoed even if test suite is silent +const savedLogs = {LogLevel.Debug, LogLevel.Info, LogLevel.Error, + LogLevel.Fatal, LogLevel.Enumeration} # aesthetic config: # progress bar length const progbarLength = 25 # log level colors const logColors = [LogLevel.Debug: fgDefault, LogLevel.Info: fgGreen, + LogLevel.Enumeration: fgDefault, LogLevel.Error: fgYellow, LogLevel.Fatal: fgRed] # global vars for the proc log diff --git a/tests/testbuilder.nim b/tests/testbuilder.nim index c7dd7d2..42289bf 100644 --- a/tests/testbuilder.nim +++ b/tests/testbuilder.nim @@ -14,17 +14,24 @@ import testobject import logutils +import testconfig import os import strutils import strformat +## A rudimentary test builder. Converts test directories to test +## sequences. proc parseModalLine(line: string): tuple[modal: bool, mode: string, detail: string, comment: bool] = - + ## parses one line. If it's a line that's a mode (in format [modename: detail] it returns modal: true, else modal: false. + ## mode contains the content of the line, if it's modal the mode name + ## detail contains the content of the detail of the modename. If empty or non modal "" + ## if comment is true, the returned value has to be ignored # when non modal, mode becomes the line # when comment is true, it must not do anything to whenever it is exported let line = line + # initialize result result.modal = false result.mode = "" result.detail = "" @@ -32,25 +39,32 @@ proc parseModalLine(line: string): tuple[modal: bool, mode: string, detail: stri if line.len() > 0 and line[0] == '[': if line.len() > 1: if line[1] == '[': + # escaped early return result.mode = line[1..line.high()] return result elif line[1] == ';': + # comment early return result.comment = true result.modal = true return result result.modal = true + # not modal line early return else: result.mode = line return result - var colon = false + + # normal modal line: + var colon = false # if there has been a colon already for i in countup(0, line.high()): let ch = line[i] if ch in Letters or ch in Digits or ch in {'_', '-'}: + # legal characters if colon: result.detail &= ($ch).toLower() else: result.mode &= ($ch).toLower() elif ch == ':': + # colon if not colon: colon = true else: @@ -58,18 +72,22 @@ proc parseModalLine(line: string): tuple[modal: bool, mode: string, detail: stri elif ch in Whitespace: discard elif ch == ']': + # closing can only come at the end if i != line.high(): fatal &"] is only allowed to close the line <{line}>." elif ch == '[': + # can only start with it if i > 0: fatal &"[ is only allowed to open the modal line <{line}>." else: fatal &"Illegal character in <{line}>: {ch}." + # must be closed by it if line[line.high()] != ']': fatal &"Line <{line}> must be closed off by ']'." proc buildTest(lines: seq[string], i: var int, name: string, path: string): Test = + ## Builds a single test (starting with index i in lines, while i is modified) result = newTest(name, path) # since this is a very simple parser, some state can reduce code length inc i # to discard the first "test" mode @@ -119,7 +137,9 @@ proc buildTest(lines: seq[string], i: var int, name: string, path: string): Test fatal &"Invalid mode {parsed.mode} when inside a block (currently in mode {mode}) at line {i} in {path}." else: # still if modal, but not inside if parsed.mode == "skip": - result.skip() + result.m_skipped = true + if not force: + result.skip() elif parsed.mode == "end": # end of test return result @@ -138,6 +158,7 @@ proc buildTest(lines: seq[string], i: var int, name: string, path: string): Test proc buildTestFile(path: string): seq[Test] = + ## Builds a test file consisting of multiple tests log(LogLevel.Debug, &"Checking {path} for tests") let lines = path.readFile().split('\n') var i = 0 @@ -156,6 +177,16 @@ proc buildTestFile(path: string): seq[Test] = proc buildTests*(testDir: string): seq[Test] = + ## Builds all test within the directory testDir + ## if testDir is a file, only build that one file + if not dirExists(testDir): + if fileExists(testDir): + result &= buildTestFile(testDir) + for test in result: + test.important = true + else: + fatal "test dir/file doesn't exist" + for candidateObj in walkDir(testDir): let candidate = candidateObj.path if dirExists(candidate): diff --git a/tests/testconfig.nim b/tests/testconfig.nim index 327e4e9..5728100 100644 --- a/tests/testconfig.nim +++ b/tests/testconfig.nim @@ -16,8 +16,11 @@ const jatsVersion* = "(dev)" var maxAliveTests* = 16 # number of tests that can run parallel const testWait* = 100 # number of milliseconds per cycle -const timeout* = 50 # number of cycles after which a test is killed for timeout +var timeout* = 50 # number of cycles after which a test is killed for timeout var testRunner* = "jatr" +var force*: bool = false # if skipped tests get executed +var enumerate*: bool = false # if true, all failed/crashed and killed tests + # are enumerated to stdout const outputIgnore* = [ "^DEBUG.*$" ] diff --git a/tests/testeval.nim b/tests/testeval.nim index e72b3f6..1ee3b13 100644 --- a/tests/testeval.nim +++ b/tests/testeval.nim @@ -24,12 +24,18 @@ import strformat import testconfig proc evalTests*(tests: seq[Test]) = + ## Goes through every test in tests and evaluates all finished + ## tests to success or mismatch for test in tests: if test.result == TestResult.ToEval: test.result = if test.eval(): TestResult.Success else: TestResult.Mismatch proc printResults*(tests: seq[Test]): bool = + ## Goes through every test in tests and prints the number of good/ + ## skipped/failed/crashed/killed tests to the screen. It also debug + ## logs all failed test details and crash messages. It returns + ## true if no tests {failed | crashed | got killed}. var skipped = 0 success = 0 @@ -37,18 +43,31 @@ proc printResults*(tests: seq[Test]): bool = crash = 0 killed = 0 for test in tests: - log(LogLevel.Debug, &"Test {test.name}@{test.path} result: {test.result}") + var level = LogLevel.Debug + var detailLevel = LogLevel.Debug + + if test.important: + level = LogLevel.Info + detailLevel = LogLevel.Info + if (test.result in {TestResult.Crash, TestResult.Mismatch, TestResult.Killed} and enumerate): + level = LogLevel.Enumeration + + + log(level, &"Test {test.name}@{test.path} result: {test.result}") + case test.result: of TestResult.Skip: inc skipped of TestResult.Mismatch: inc fail - log(LogLevel.Debug, &"[{test.name}@{test.path}\nstdout:\n{test.output}\nstderr:\n{test.error}\nexpected stdout:\n{test.expectedOutput}\nexpected stderr:\n{test.expectedError}\n]") - log(LogLevel.Debug, &"\nMismatch pos for stdout: {test.mismatchPos}\nMismatch pos for stderr: {test.errorMismatchPos}") + log(detailLevel, &"[{test.name}@{test.path}\nstdout:\n{test.output}\nstderr:\n{test.error}\nexpected stdout:\n{test.expectedOutput}\nexpected stderr:\n{test.expectedError}\n]") + log(detailLevel, &"\nMismatch pos for stdout: {test.mismatchPos}\nMismatch pos for stderr: {test.errorMismatchPos}") of TestResult.Crash: inc crash - log(LogLevel.Debug, &"{test.name}@{test.path} \ncrash:\n{test.error}") + log(detailLevel, &"{test.name}@{test.path} \ncrash:\n{test.error}") of TestResult.Success: + if test.m_skipped: + log(LogLevel.Info, &"Test {test.name}@{test.path} succeeded, despite being marked to be skipped.") inc success of TestResult.Killed: inc killed diff --git a/tests/testobject.nim b/tests/testobject.nim index eea7e37..1c34320 100644 --- a/tests/testobject.nim +++ b/tests/testobject.nim @@ -40,6 +40,10 @@ type source*: string path*: string name*: string + important*: bool # if set to true, the stdout/stderr and extra debug + # will get printed when finished + m_skipped*: bool # metadata, whether the skipped mode is in the file + # NOT WHETHER THE TEST IS ACTUALLY SKIPPED # generated after building expectedOutput*: seq[ExpectedLine] expectedError*: seq[ExpectedLine] @@ -122,6 +126,8 @@ proc newTest*(name: string, path: string): Test = result.name = name result.mismatchPos = -1 result.errorMismatchPos = -1 + result.important = false + result.m_skipped = false proc skip*(test: Test) = test.result = TestResult.Skip diff --git a/tests/testrun.nim b/tests/testrun.nim index 259eea4..644379f 100644 --- a/tests/testrun.nim +++ b/tests/testrun.nim @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -# Test runner supervisor/manager +## Test runner supervisor/manager import testobject import logutils @@ -22,10 +22,13 @@ import strformat import os proc runTest(test: Test) = + ## Starts running a test log(LogLevel.Debug, &"Starting test {test.path}.") test.start() proc tryFinishTest(test: Test): bool = + ## Attempts to finish a test and returns true if it finished. + ## False otherwise. if test.running(): return false test.finish() @@ -33,16 +36,21 @@ proc tryFinishTest(test: Test): bool = return true proc killTest(test: Test) = + ## Kills the test, logs kill reason as taking too long if test.running(): test.kill() log(LogLevel.Error, &"Test {test.path} was killed for taking too long.") proc killTests*(tests: seq[Test]) = + ## kills all running tests in tests sequence for test in tests: if test.running(): test.kill() -proc runTests*(tests: seq[Test], runner: string) = +proc runTests*(tests: seq[Test]) = + ## Runs all tests tests in tests, manages the maximum alive tests + ## and launching of tests parallel. Also writes progress to the + ## screen var aliveTests = 0 currentTest = 0