Merge pull request #41 from Productive2/master

Testing suite improvements

tests/japl/fib.jpl

@@ -5,15 +5,15 @@ fun fib(n) {
         return n;
     return fib(n-2) + fib(n-1);
 }
-print(fib(1));//stdout:1
+print(fib(1));
-print(fib(2));//stdout:1
+print(fib(2));
-print(fib(3));//stdout:2
+print(fib(3));
-print(fib(4));//stdout:3
+print(fib(4));
-print(fib(5));//stdout:5
+print(fib(5));
-print(fib(6));//stdout:8
+print(fib(6));
-print(fib(7));//stdout:13
+print(fib(7));
-print(fib(8));//stdout:21
+print(fib(8));
-print(fib(9));//stdout:3
+print(fib(9));
 [end]
 [stdout]
 1

tests/jats.nim

@@ -54,6 +54,7 @@ when isMainModule:
     var targetFiles: seq[string]
     var verbose = true
     var quitVal = QuitValue.Success
+    var testDir = "japl"
 
     proc evalKey(key: string) =
         ## Modifies the globals that define what JATS does based on the 
@@ -69,6 +70,10 @@ when isMainModule:
             verbose = false
         elif key == "stdout":
             debugActions.add(DebugAction.Stdout)
+        elif key == "f" or key == "force":
+            force = true
+        elif key == "e" or key == "enumerate":
+            enumerate = true
         else:
             echo &"Unknown flag: {key}"
             action = Action.Help
@@ -88,6 +93,24 @@ when isMainModule:
                 echo "Can't parse non-integer option passed to -j/--jobs."
                 action = Action.Help
                 quitVal = QuitValue.ArgParseErr
+        elif key == "t" or key == "test" or key == "tests":
+            testDir = val
+        elif key == "timeout":
+            try:
+                var timeoutSeconds = parseFloat(val)
+                # a round is 100 ms, so let's not get close to that
+                if timeoutSeconds < 0.3:
+                    timeoutSeconds = 0.3
+                # I don't want anything not nicely convertible to int, 
+                # so how about cut it off at 10 hours. Open an issue
+                # if that's not enough... or just tweak it you lunatic
+                if timeoutSeconds > 36000.0:
+                    timeoutSeconds = 36000.0
+                timeout = (timeoutSeconds * 10).int
+            except ValueError:
+                echo "Can't parse invalid timeout value " & val
+                action = Action.Help
+                quitVal = QuitValue.ArgParseErr
         else:
             echo &"Unknown option: {key}"
             action = Action.Help
@@ -119,15 +142,18 @@ when isMainModule:
         echo """
 JATS - Just Another Test Suite
 
-Usage:
+Usage: ./jats <flags>
-jats 
+Debug output flags:
-Runs the tests
-Flags:
 -i (or --interactive) displays all debug info
 -o:<filename> (or --output:<filename>) saves debug info to a file
 -s (or --silent) will disable all output (except --stdout)
 --stdout will put all debug info to stdout
+-e (or --enumerate) will list all tests that fail, crash or get killed
+Test behavior flags:
 -j:<parallel test count> (or --jobs:<parallel test count>) to specify number of tests to run parallel
+-t:<test file or dir> (or --test:<path> or --tests:<path>) to specify where tests are
+-f (or --force) will run skipped tests
+Miscellaneous flags:
 -h (or --help) displays this help message
 -v (or --version) displays the version number of JATS
 """
@@ -167,18 +193,24 @@ Flags:
         # the second half of the test suite defined in ~japl/tests/japl
         # Find ~japl/tests/japl and the test runner JATR
         var jatr = "jatr"
-        var testDir = "japl"
         if not fileExists(jatr):
             if fileExists("tests" / jatr):
                 log(LogLevel.Debug, 
-                    &"Must be in root: prepending \"tests\" to paths")
+                    &"Must be in root: prepending \"tests\" to jatr path")
                 jatr = "tests" / jatr
-                testDir = "tests" / testDir
             else:
                 # only those two dirs are realistically useful for now,
-                echo "The tests directory couldn't be found."
+                echo "The test runner was not found."
                 quit int(QuitValue.JatrNotFound)
 
+        if not dirExists(testDir) and not fileExists(testDir):
+          if dirExists("tests" / testDir) or fileExists("tests" / testDir):
+            log(LogLevel.Debug, "Prepending \"tests\" to test path")
+            testDir = "tests" / testDir
+          else:
+            echo "The test dir/file was not found."
+            quit int(QuitValue.JatrNotFound)
+
         # set the global var which specifies the path to the test runner
         testRunner = jatr
         log(LogLevel.Info, &"Running JAPL tests.")
@@ -196,7 +228,7 @@ Flags:
         setControlCHook(ctrlc)
         log(LogLevel.Info, &"Running tests...")
         # run tests (see testrun.nim)
-        tests.runTests(jatr)
+        tests.runTests()
         log(LogLevel.Debug, &"Tests ran.")
         log(LogLevel.Debug, &"Evaluating tests...")
         # evaluate tests (see testeval.nim)

tests/logutils.nim

@@ -35,20 +35,24 @@ type LogLevel* {.pure.} = enum
     ## All the different possible log levels
     Debug, # always written to file only (large outputs, such as the entire output of the failing test or stacktrace)
     Info, # important information about the progress of the test suite
+    Enumeration, # a white output for the enumerate option
     Error, # failing tests (printed with yellow)
     Fatal # always printed with red, halts the entire suite (test parsing errors, printed with red)
 
 # log config: which log levels to show, show in silent mode and save to the
 # detailed debug logs
-const echoedLogs = {LogLevel.Info, LogLevel.Error, LogLevel.Fatal}
+const echoedLogs = {LogLevel.Info, LogLevel.Error, LogLevel.Fatal, 
-const echoedLogsSilent = {LogLevel.Fatal} # will be echoed even if test suite is silent
+                    LogLevel.Enumeration}
-const savedLogs = {LogLevel.Debug, LogLevel.Info, LogLevel.Error, LogLevel.Fatal}
+const echoedLogsSilent = {LogLevel.Fatal, LogLevel.Enumeration} # will be echoed even if test suite is silent
+const savedLogs = {LogLevel.Debug, LogLevel.Info, LogLevel.Error, 
+                   LogLevel.Fatal, LogLevel.Enumeration}
 
 # aesthetic config:
 # progress bar length
 const progbarLength = 25
 # log level colors
 const logColors = [LogLevel.Debug: fgDefault, LogLevel.Info: fgGreen, 
+                   LogLevel.Enumeration: fgDefault,
                    LogLevel.Error: fgYellow, LogLevel.Fatal: fgRed]
 
 # global vars for the proc log

tests/testbuilder.nim

@@ -14,17 +14,24 @@
 
 import testobject
 import logutils
+import testconfig
 
 import os
 import strutils
 import strformat
 
+## A rudimentary test builder. Converts test directories to test
+## sequences.
 
 proc parseModalLine(line: string): tuple[modal: bool, mode: string, detail: string, comment: bool] =
-
+    ## parses one line. If it's a line that's a mode (in format [modename: detail] it returns modal: true, else modal: false.
+    ## mode contains the content of the line, if it's modal the mode name
+    ## detail contains the content of the detail of the modename. If empty or non modal ""
+    ## if comment is true, the returned value has to be ignored
     # when non modal, mode becomes the line
     # when comment is true, it must not do anything to whenever it is exported
     let line = line
+    # initialize result
     result.modal = false
     result.mode = ""
     result.detail = ""
@@ -32,25 +39,32 @@ proc parseModalLine(line: string): tuple[modal: bool, mode: string, detail: stri
     if line.len() > 0 and line[0] == '[':
         if line.len() > 1:
             if line[1] == '[':
+                # escaped early return
                 result.mode = line[1..line.high()]
                 return result 
             elif line[1] == ';':
+                # comment early return
                 result.comment = true
                 result.modal = true
                 return result
         result.modal = true
+    # not modal line early return
     else:
         result.mode = line
         return result
-    var colon = false
+
+    # normal modal line:
+    var colon = false # if there has been a colon already
     for i in countup(0, line.high()):
         let ch = line[i]
         if ch in Letters or ch in Digits or ch in {'_', '-'}:
+            # legal characters
             if colon:
                 result.detail &= ($ch).toLower()
             else:
                 result.mode &= ($ch).toLower()
         elif ch == ':':
+            # colon
             if not colon:
                 colon = true
             else:
@@ -58,18 +72,22 @@ proc parseModalLine(line: string): tuple[modal: bool, mode: string, detail: stri
         elif ch in Whitespace:
             discard
         elif ch == ']':
+            # closing can only come at the end
             if i != line.high():
                 fatal &"] is only allowed to close the line <{line}>."
         elif ch == '[':
+            # can only start with it
             if i > 0:
                 fatal &"[ is only allowed to open the modal line <{line}>."
         else:
             fatal &"Illegal character in <{line}>: {ch}."
+    # must be closed by it
     if line[line.high()] != ']':
         fatal &"Line <{line}> must be closed off by ']'."    
 
 
 proc buildTest(lines: seq[string], i: var int, name: string, path: string): Test =
+    ## Builds a single test (starting with index i in lines, while i is modified)
     result = newTest(name, path)
     # since this is a very simple parser, some state can reduce code length
     inc i # to discard the first "test" mode
@@ -119,7 +137,9 @@ proc buildTest(lines: seq[string], i: var int, name: string, path: string): Test
                     fatal &"Invalid mode {parsed.mode} when inside a block (currently in mode {mode}) at line {i} in {path}."
             else: # still if modal, but not inside
                 if parsed.mode == "skip":
-                    result.skip()
+                    result.m_skipped = true
+                    if not force:
+                        result.skip()
                 elif parsed.mode == "end":
                     # end of test
                     return result
@@ -138,6 +158,7 @@ proc buildTest(lines: seq[string], i: var int, name: string, path: string): Test
 
 
 proc buildTestFile(path: string): seq[Test] =
+    ## Builds a test file consisting of multiple tests
     log(LogLevel.Debug, &"Checking {path} for tests")
     let lines = path.readFile().split('\n') 
     var i = 0
@@ -156,6 +177,16 @@ proc buildTestFile(path: string): seq[Test] =
 
 
 proc buildTests*(testDir: string): seq[Test] =
+    ## Builds all test within the directory testDir
+    ## if testDir is a file, only build that one file
+    if not dirExists(testDir):
+        if fileExists(testDir):
+            result &= buildTestFile(testDir)
+            for test in result:
+                test.important = true
+        else:
+            fatal "test dir/file doesn't exist"
+
     for candidateObj in walkDir(testDir):
         let candidate = candidateObj.path
         if dirExists(candidate):

tests/testconfig.nim

@@ -16,8 +16,11 @@ const jatsVersion* = "(dev)"
 
 var maxAliveTests* = 16 # number of tests that can run parallel
 const testWait* = 100 # number of milliseconds per cycle
-const timeout* = 50 # number of cycles after which a test is killed for timeout
+var timeout* = 50 # number of cycles after which a test is killed for timeout
 
 var testRunner* = "jatr"
+var force*: bool = false # if skipped tests get executed
+var enumerate*: bool = false # if true, all failed/crashed and killed tests
+                             # are enumerated to stdout
 
 const outputIgnore* = [ "^DEBUG.*$" ]

tests/testeval.nim

@@ -24,12 +24,18 @@ import strformat
 import testconfig
 
 proc evalTests*(tests: seq[Test]) =
+    ## Goes through every test in tests and evaluates all finished
+    ## tests to success or mismatch
     for test in tests:
         if test.result == TestResult.ToEval:
             test.result = if test.eval(): TestResult.Success else: TestResult.Mismatch
 
 
 proc printResults*(tests: seq[Test]): bool =
+    ## Goes through every test in tests and prints the number of good/
+    ## skipped/failed/crashed/killed tests to the screen. It also debug
+    ## logs all failed test details and crash messages. It returns 
+    ## true if no tests {failed | crashed | got killed}.
     var
         skipped = 0
         success = 0
@@ -37,18 +43,31 @@ proc printResults*(tests: seq[Test]): bool =
         crash = 0
         killed = 0
     for test in tests:
-        log(LogLevel.Debug, &"Test {test.name}@{test.path} result: {test.result}")
+        var level = LogLevel.Debug
+        var detailLevel = LogLevel.Debug
+
+        if test.important:
+            level = LogLevel.Info
+            detailLevel = LogLevel.Info
+        if (test.result in {TestResult.Crash, TestResult.Mismatch, TestResult.Killed} and enumerate): 
+            level = LogLevel.Enumeration
+
+
+        log(level, &"Test {test.name}@{test.path} result: {test.result}")
+
         case test.result:
             of TestResult.Skip:
                 inc skipped
             of TestResult.Mismatch:
                 inc fail
-                log(LogLevel.Debug, &"[{test.name}@{test.path}\nstdout:\n{test.output}\nstderr:\n{test.error}\nexpected stdout:\n{test.expectedOutput}\nexpected stderr:\n{test.expectedError}\n]")
+                log(detailLevel, &"[{test.name}@{test.path}\nstdout:\n{test.output}\nstderr:\n{test.error}\nexpected stdout:\n{test.expectedOutput}\nexpected stderr:\n{test.expectedError}\n]")
-                log(LogLevel.Debug, &"\nMismatch pos for stdout: {test.mismatchPos}\nMismatch pos for stderr: {test.errorMismatchPos}")
+                log(detailLevel, &"\nMismatch pos for stdout: {test.mismatchPos}\nMismatch pos for stderr: {test.errorMismatchPos}")
             of TestResult.Crash:
                 inc crash
-                log(LogLevel.Debug, &"{test.name}@{test.path} \ncrash:\n{test.error}")
+                log(detailLevel, &"{test.name}@{test.path} \ncrash:\n{test.error}")
             of TestResult.Success:
+                if test.m_skipped:
+                    log(LogLevel.Info, &"Test {test.name}@{test.path} succeeded, despite being marked to be skipped.")
                 inc success
             of TestResult.Killed:
                 inc killed

tests/testobject.nim

@@ -40,6 +40,10 @@ type
         source*: string
         path*: string
         name*: string
+        important*: bool # if set to true, the stdout/stderr and extra debug
+                         # will get printed when finished
+        m_skipped*: bool # metadata, whether the skipped mode is in the file
+                         # NOT WHETHER THE TEST IS ACTUALLY SKIPPED
         # generated after building
         expectedOutput*: seq[ExpectedLine]
         expectedError*: seq[ExpectedLine]
@@ -122,6 +126,8 @@ proc newTest*(name: string, path: string): Test =
     result.name = name
     result.mismatchPos = -1
     result.errorMismatchPos = -1
+    result.important = false
+    result.m_skipped = false
 
 proc skip*(test: Test) =
     test.result = TestResult.Skip

tests/testrun.nim

@@ -12,7 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-# Test runner supervisor/manager
+## Test runner supervisor/manager
 
 import testobject
 import logutils
@@ -22,10 +22,13 @@ import strformat
 import os
 
 proc runTest(test: Test) =
+    ## Starts running a test
     log(LogLevel.Debug, &"Starting test {test.path}.")
     test.start()
 
 proc tryFinishTest(test: Test): bool =
+    ## Attempts to finish a test and returns true if it finished.
+    ## False otherwise.
     if test.running():
         return false
     test.finish()
@@ -33,16 +36,21 @@ proc tryFinishTest(test: Test): bool =
     return true
 
 proc killTest(test: Test) =
+    ## Kills the test, logs kill reason as taking too long
     if test.running():
         test.kill()
         log(LogLevel.Error, &"Test {test.path} was killed for taking too long.")
 
 proc killTests*(tests: seq[Test]) =
+    ## kills all running tests in tests sequence
     for test in tests:
         if test.running():
             test.kill()
 
-proc runTests*(tests: seq[Test], runner: string) =
+proc runTests*(tests: seq[Test]) =
+    ## Runs all tests tests in tests, manages the maximum alive tests
+    ## and launching of tests parallel. Also writes progress to the
+    ## screen
     var
         aliveTests = 0
         currentTest = 0