diff --git a/tests/jats.nim b/tests/jats.nim index bdc8388..372c8d7 100644 --- a/tests/jats.nim +++ b/tests/jats.nim @@ -54,6 +54,7 @@ when isMainModule: var targetFiles: seq[string] var verbose = true var quitVal = QuitValue.Success + var testDir = "japl" proc evalKey(key: string) = ## Modifies the globals that define what JATS does based on the @@ -69,6 +70,10 @@ when isMainModule: verbose = false elif key == "stdout": debugActions.add(DebugAction.Stdout) + elif key == "f" or key == "force": + force = true + elif key == "e" or key == "enumerate": + enumerate = true else: echo &"Unknown flag: {key}" action = Action.Help @@ -88,6 +93,24 @@ when isMainModule: echo "Can't parse non-integer option passed to -j/--jobs." action = Action.Help quitVal = QuitValue.ArgParseErr + elif key == "t" or key == "test" or key == "tests": + testDir = val + elif key == "timeout": + try: + var timeoutSeconds = parseFloat(val) + # a round is 100 ms, so let's not get close to that + if timeoutSeconds < 0.3: + timeoutSeconds = 0.3 + # I don't want anything not nicely convertible to int, + # so how about cut it off at 10 hours. Open an issue + # if that's not enough... or just tweak it you lunatic + if timeoutSeconds > 36000.0: + timeoutSeconds = 36000.0 + timeout = (timeoutSeconds * 10).int + except ValueError: + echo "Can't parse invalid timeout value " & val + action = Action.Help + quitVal = QuitValue.ArgParseErr else: echo &"Unknown option: {key}" action = Action.Help @@ -119,15 +142,18 @@ when isMainModule: echo """ JATS - Just Another Test Suite -Usage: -jats -Runs the tests -Flags: +Usage: ./jats +Debug output flags: -i (or --interactive) displays all debug info -o: (or --output:) saves debug info to a file -s (or --silent) will disable all output (except --stdout) --stdout will put all debug info to stdout +-e (or --enumerate) will list all tests that fail, crash or get killed +Test behavior flags: -j: (or --jobs:) to specify number of tests to run parallel +-t: (or --test: or --tests:) to specify where tests are +-f (or --force) will run skipped tests +Miscellaneous flags: -h (or --help) displays this help message -v (or --version) displays the version number of JATS """ @@ -167,18 +193,24 @@ Flags: # the second half of the test suite defined in ~japl/tests/japl # Find ~japl/tests/japl and the test runner JATR var jatr = "jatr" - var testDir = "japl" if not fileExists(jatr): if fileExists("tests" / jatr): log(LogLevel.Debug, - &"Must be in root: prepending \"tests\" to paths") + &"Must be in root: prepending \"tests\" to jatr path") jatr = "tests" / jatr - testDir = "tests" / testDir else: # only those two dirs are realistically useful for now, - echo "The tests directory couldn't be found." + echo "The test runner was not found." quit int(QuitValue.JatrNotFound) + if not dirExists(testDir) and not fileExists(testDir): + if dirExists("tests" / testDir) or fileExists("tests" / testDir): + log(LogLevel.Debug, "Prepending \"tests\" to test path") + testDir = "tests" / testDir + else: + echo "The test dir/file was not found." + quit int(QuitValue.JatrNotFound) + # set the global var which specifies the path to the test runner testRunner = jatr log(LogLevel.Info, &"Running JAPL tests.") diff --git a/tests/logutils.nim b/tests/logutils.nim index 5c55f15..2df5d86 100644 --- a/tests/logutils.nim +++ b/tests/logutils.nim @@ -35,20 +35,24 @@ type LogLevel* {.pure.} = enum ## All the different possible log levels Debug, # always written to file only (large outputs, such as the entire output of the failing test or stacktrace) Info, # important information about the progress of the test suite + Enumeration, # a white output for the enumerate option Error, # failing tests (printed with yellow) Fatal # always printed with red, halts the entire suite (test parsing errors, printed with red) # log config: which log levels to show, show in silent mode and save to the # detailed debug logs -const echoedLogs = {LogLevel.Info, LogLevel.Error, LogLevel.Fatal} -const echoedLogsSilent = {LogLevel.Fatal} # will be echoed even if test suite is silent -const savedLogs = {LogLevel.Debug, LogLevel.Info, LogLevel.Error, LogLevel.Fatal} +const echoedLogs = {LogLevel.Info, LogLevel.Error, LogLevel.Fatal, + LogLevel.Enumeration} +const echoedLogsSilent = {LogLevel.Fatal, LogLevel.Enumeration} # will be echoed even if test suite is silent +const savedLogs = {LogLevel.Debug, LogLevel.Info, LogLevel.Error, + LogLevel.Fatal, LogLevel.Enumeration} # aesthetic config: # progress bar length const progbarLength = 25 # log level colors const logColors = [LogLevel.Debug: fgDefault, LogLevel.Info: fgGreen, + LogLevel.Enumeration: fgDefault, LogLevel.Error: fgYellow, LogLevel.Fatal: fgRed] # global vars for the proc log diff --git a/tests/testbuilder.nim b/tests/testbuilder.nim index 71ca0c5..42289bf 100644 --- a/tests/testbuilder.nim +++ b/tests/testbuilder.nim @@ -14,6 +14,7 @@ import testobject import logutils +import testconfig import os import strutils @@ -136,7 +137,9 @@ proc buildTest(lines: seq[string], i: var int, name: string, path: string): Test fatal &"Invalid mode {parsed.mode} when inside a block (currently in mode {mode}) at line {i} in {path}." else: # still if modal, but not inside if parsed.mode == "skip": - result.skip() + result.m_skipped = true + if not force: + result.skip() elif parsed.mode == "end": # end of test return result @@ -175,6 +178,15 @@ proc buildTestFile(path: string): seq[Test] = proc buildTests*(testDir: string): seq[Test] = ## Builds all test within the directory testDir + ## if testDir is a file, only build that one file + if not dirExists(testDir): + if fileExists(testDir): + result &= buildTestFile(testDir) + for test in result: + test.important = true + else: + fatal "test dir/file doesn't exist" + for candidateObj in walkDir(testDir): let candidate = candidateObj.path if dirExists(candidate): diff --git a/tests/testconfig.nim b/tests/testconfig.nim index 327e4e9..5728100 100644 --- a/tests/testconfig.nim +++ b/tests/testconfig.nim @@ -16,8 +16,11 @@ const jatsVersion* = "(dev)" var maxAliveTests* = 16 # number of tests that can run parallel const testWait* = 100 # number of milliseconds per cycle -const timeout* = 50 # number of cycles after which a test is killed for timeout +var timeout* = 50 # number of cycles after which a test is killed for timeout var testRunner* = "jatr" +var force*: bool = false # if skipped tests get executed +var enumerate*: bool = false # if true, all failed/crashed and killed tests + # are enumerated to stdout const outputIgnore* = [ "^DEBUG.*$" ] diff --git a/tests/testeval.nim b/tests/testeval.nim index 989b9f2..1ee3b13 100644 --- a/tests/testeval.nim +++ b/tests/testeval.nim @@ -43,18 +43,31 @@ proc printResults*(tests: seq[Test]): bool = crash = 0 killed = 0 for test in tests: - log(LogLevel.Debug, &"Test {test.name}@{test.path} result: {test.result}") + var level = LogLevel.Debug + var detailLevel = LogLevel.Debug + + if test.important: + level = LogLevel.Info + detailLevel = LogLevel.Info + if (test.result in {TestResult.Crash, TestResult.Mismatch, TestResult.Killed} and enumerate): + level = LogLevel.Enumeration + + + log(level, &"Test {test.name}@{test.path} result: {test.result}") + case test.result: of TestResult.Skip: inc skipped of TestResult.Mismatch: inc fail - log(LogLevel.Debug, &"[{test.name}@{test.path}\nstdout:\n{test.output}\nstderr:\n{test.error}\nexpected stdout:\n{test.expectedOutput}\nexpected stderr:\n{test.expectedError}\n]") - log(LogLevel.Debug, &"\nMismatch pos for stdout: {test.mismatchPos}\nMismatch pos for stderr: {test.errorMismatchPos}") + log(detailLevel, &"[{test.name}@{test.path}\nstdout:\n{test.output}\nstderr:\n{test.error}\nexpected stdout:\n{test.expectedOutput}\nexpected stderr:\n{test.expectedError}\n]") + log(detailLevel, &"\nMismatch pos for stdout: {test.mismatchPos}\nMismatch pos for stderr: {test.errorMismatchPos}") of TestResult.Crash: inc crash - log(LogLevel.Debug, &"{test.name}@{test.path} \ncrash:\n{test.error}") + log(detailLevel, &"{test.name}@{test.path} \ncrash:\n{test.error}") of TestResult.Success: + if test.m_skipped: + log(LogLevel.Info, &"Test {test.name}@{test.path} succeeded, despite being marked to be skipped.") inc success of TestResult.Killed: inc killed diff --git a/tests/testobject.nim b/tests/testobject.nim index eea7e37..1c34320 100644 --- a/tests/testobject.nim +++ b/tests/testobject.nim @@ -40,6 +40,10 @@ type source*: string path*: string name*: string + important*: bool # if set to true, the stdout/stderr and extra debug + # will get printed when finished + m_skipped*: bool # metadata, whether the skipped mode is in the file + # NOT WHETHER THE TEST IS ACTUALLY SKIPPED # generated after building expectedOutput*: seq[ExpectedLine] expectedError*: seq[ExpectedLine] @@ -122,6 +126,8 @@ proc newTest*(name: string, path: string): Test = result.name = name result.mismatchPos = -1 result.errorMismatchPos = -1 + result.important = false + result.m_skipped = false proc skip*(test: Test) = test.result = TestResult.Skip