japl
/
japl
mirror of https://github.com/japl-lang/japl.git
1
1
Fork 0
Code Issues Projects Releases Wiki Activity

Merge pull request #41 from Productive2/master 

Testing suite improvements
This commit is contained in:
Mattia 2021-02-20 22:55:28 +01:00 committed by GitHub
parent 86fc8b9ca3 b85598dfb6
commit 607d88f0c0
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
8 changed files with 134 additions and 31 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

18
tests/japl/fib.jpl
View File

@ -5,15 +5,15 @@ fun fib(n) {
return n;
return fib(n-2) + fib(n-1);
}
print(fib(1));//stdout:1
print(fib(2));//stdout:1
print(fib(3));//stdout:2
print(fib(4));//stdout:3
print(fib(5));//stdout:5
print(fib(6));//stdout:8
print(fib(7));//stdout:13
print(fib(8));//stdout:21
print(fib(9));//stdout:3
print(fib(1));
print(fib(2));
print(fib(3));
print(fib(4));
print(fib(5));
print(fib(6));
print(fib(7));
print(fib(8));
print(fib(9));
[end]
[stdout]
1

50
tests/jats.nim
View File

@ -54,6 +54,7 @@ when isMainModule:
var targetFiles: seq[string]
var verbose = true
var quitVal = QuitValue.Success
var testDir = "japl"
proc evalKey(key: string) =
## Modifies the globals that define what JATS does based on the
@ -69,6 +70,10 @@ when isMainModule:
verbose = false
elif key == "stdout":
debugActions.add(DebugAction.Stdout)
elif key == "f" or key == "force":
force = true
elif key == "e" or key == "enumerate":
enumerate = true
else:
echo &"Unknown flag: {key}"
action = Action.Help
@ -88,6 +93,24 @@ when isMainModule:
echo "Can't parse non-integer option passed to -j/--jobs."
action = Action.Help
quitVal = QuitValue.ArgParseErr
elif key == "t" or key == "test" or key == "tests":
testDir = val
elif key == "timeout":
try:
var timeoutSeconds = parseFloat(val)
# a round is 100 ms, so let's not get close to that
if timeoutSeconds < 0.3:
timeoutSeconds = 0.3
# I don't want anything not nicely convertible to int,
# so how about cut it off at 10 hours. Open an issue
# if that's not enough... or just tweak it you lunatic
if timeoutSeconds > 36000.0:
timeoutSeconds = 36000.0
timeout = (timeoutSeconds * 10).int
except ValueError:
echo "Can't parse invalid timeout value " & val
action = Action.Help
quitVal = QuitValue.ArgParseErr
else:
echo &"Unknown option: {key}"
action = Action.Help
@ -119,15 +142,18 @@ when isMainModule:
echo """
JATS - Just Another Test Suite
Usage:
jats
Runs the tests
Flags:
Usage: ./jats <flags>
Debug output flags:
-i (or --interactive) displays all debug info
-o:<filename> (or --output:<filename>) saves debug info to a file
-s (or --silent) will disable all output (except --stdout)
--stdout will put all debug info to stdout
-e (or --enumerate) will list all tests that fail, crash or get killed
Test behavior flags:
-j:<parallel test count> (or --jobs:<parallel test count>) to specify number of tests to run parallel
-t:<test file or dir> (or --test:<path> or --tests:<path>) to specify where tests are
-f (or --force) will run skipped tests
Miscellaneous flags:
-h (or --help) displays this help message
-v (or --version) displays the version number of JATS
"""
@ -167,18 +193,24 @@ Flags:
# the second half of the test suite defined in ~japl/tests/japl
# Find ~japl/tests/japl and the test runner JATR
var jatr = "jatr"
var testDir = "japl"
if not fileExists(jatr):
if fileExists("tests" / jatr):
log(LogLevel.Debug,
&"Must be in root: prepending \"tests\" to paths")
&"Must be in root: prepending \"tests\" to jatr path")
jatr = "tests" / jatr
testDir = "tests" / testDir
else:
# only those two dirs are realistically useful for now,
echo "The tests directory couldn't be found."
echo "The test runner was not found."
quit int(QuitValue.JatrNotFound)
if not dirExists(testDir) and not fileExists(testDir):
if dirExists("tests" / testDir) or fileExists("tests" / testDir):
log(LogLevel.Debug, "Prepending \"tests\" to test path")
testDir = "tests" / testDir
else:
echo "The test dir/file was not found."
quit int(QuitValue.JatrNotFound)
# set the global var which specifies the path to the test runner
testRunner = jatr
log(LogLevel.Info, &"Running JAPL tests.")
@ -196,7 +228,7 @@ Flags:
setControlCHook(ctrlc)
log(LogLevel.Info, &"Running tests...")
# run tests (see testrun.nim)
tests.runTests(jatr)
tests.runTests()
log(LogLevel.Debug, &"Tests ran.")
log(LogLevel.Debug, &"Evaluating tests...")
# evaluate tests (see testeval.nim)

10
tests/logutils.nim
View File

@ -35,20 +35,24 @@ type LogLevel* {.pure.} = enum
## All the different possible log levels
Debug, # always written to file only (large outputs, such as the entire output of the failing test or stacktrace)
Info, # important information about the progress of the test suite
Enumeration, # a white output for the enumerate option
Error, # failing tests (printed with yellow)
Fatal # always printed with red, halts the entire suite (test parsing errors, printed with red)
# log config: which log levels to show, show in silent mode and save to the
# detailed debug logs
const echoedLogs = {LogLevel.Info, LogLevel.Error, LogLevel.Fatal}
const echoedLogsSilent = {LogLevel.Fatal} # will be echoed even if test suite is silent
const savedLogs = {LogLevel.Debug, LogLevel.Info, LogLevel.Error, LogLevel.Fatal}
const echoedLogs = {LogLevel.Info, LogLevel.Error, LogLevel.Fatal,
LogLevel.Enumeration}
const echoedLogsSilent = {LogLevel.Fatal, LogLevel.Enumeration} # will be echoed even if test suite is silent
const savedLogs = {LogLevel.Debug, LogLevel.Info, LogLevel.Error,
LogLevel.Fatal, LogLevel.Enumeration}
# aesthetic config:
# progress bar length
const progbarLength = 25
# log level colors
const logColors = [LogLevel.Debug: fgDefault, LogLevel.Info: fgGreen,
LogLevel.Enumeration: fgDefault,
LogLevel.Error: fgYellow, LogLevel.Fatal: fgRed]
# global vars for the proc log

37
tests/testbuilder.nim
View File

@ -14,17 +14,24 @@
import testobject
import logutils
import testconfig
import os
import strutils
import strformat
## A rudimentary test builder. Converts test directories to test
## sequences.
proc parseModalLine(line: string): tuple[modal: bool, mode: string, detail: string, comment: bool] =
## parses one line. If it's a line that's a mode (in format [modename: detail] it returns modal: true, else modal: false.
## mode contains the content of the line, if it's modal the mode name
## detail contains the content of the detail of the modename. If empty or non modal ""
## if comment is true, the returned value has to be ignored
# when non modal, mode becomes the line
# when comment is true, it must not do anything to whenever it is exported
let line = line
# initialize result
result.modal = false
result.mode = ""
result.detail = ""
@ -32,25 +39,32 @@ proc parseModalLine(line: string): tuple[modal: bool, mode: string, detail: stri
if line.len() > 0 and line[0] == '[':
if line.len() > 1:
if line[1] == '[':
# escaped early return
result.mode = line[1..line.high()]
return result
elif line[1] == ';':
# comment early return
result.comment = true
result.modal = true
return result
result.modal = true
# not modal line early return
else:
result.mode = line
return result
var colon = false
# normal modal line:
var colon = false # if there has been a colon already
for i in countup(0, line.high()):
let ch = line[i]
if ch in Letters or ch in Digits or ch in {'_', '-'}:
# legal characters
if colon:
result.detail &= ($ch).toLower()
else:
result.mode &= ($ch).toLower()
elif ch == ':':
# colon
if not colon:
colon = true
else:
@ -58,18 +72,22 @@ proc parseModalLine(line: string): tuple[modal: bool, mode: string, detail: stri
elif ch in Whitespace:
discard
elif ch == ']':
# closing can only come at the end
if i != line.high():
fatal &"] is only allowed to close the line <{line}>."
elif ch == '[':
# can only start with it
if i > 0:
fatal &"[ is only allowed to open the modal line <{line}>."
else:
fatal &"Illegal character in <{line}>: {ch}."
# must be closed by it
if line[line.high()] != ']':
fatal &"Line <{line}> must be closed off by ']'."
proc buildTest(lines: seq[string], i: var int, name: string, path: string): Test =
## Builds a single test (starting with index i in lines, while i is modified)
result = newTest(name, path)
# since this is a very simple parser, some state can reduce code length
inc i # to discard the first "test" mode
@ -119,7 +137,9 @@ proc buildTest(lines: seq[string], i: var int, name: string, path: string): Test
fatal &"Invalid mode {parsed.mode} when inside a block (currently in mode {mode}) at line {i} in {path}."
else: # still if modal, but not inside
if parsed.mode == "skip":
result.skip()
result.m_skipped = true
if not force:
result.skip()
elif parsed.mode == "end":
# end of test
return result
@ -138,6 +158,7 @@ proc buildTest(lines: seq[string], i: var int, name: string, path: string): Test
proc buildTestFile(path: string): seq[Test] =
## Builds a test file consisting of multiple tests
log(LogLevel.Debug, &"Checking {path} for tests")
let lines = path.readFile().split('\n')
var i = 0
@ -156,6 +177,16 @@ proc buildTestFile(path: string): seq[Test] =
proc buildTests*(testDir: string): seq[Test] =
## Builds all test within the directory testDir
## if testDir is a file, only build that one file
if not dirExists(testDir):
if fileExists(testDir):
result &= buildTestFile(testDir)
for test in result:
test.important = true
else:
fatal "test dir/file doesn't exist"
for candidateObj in walkDir(testDir):
let candidate = candidateObj.path
if dirExists(candidate):

5
tests/testconfig.nim
View File

@ -16,8 +16,11 @@ const jatsVersion* = "(dev)"
var maxAliveTests* = 16 # number of tests that can run parallel
const testWait* = 100 # number of milliseconds per cycle
const timeout* = 50 # number of cycles after which a test is killed for timeout
var timeout* = 50 # number of cycles after which a test is killed for timeout
var testRunner* = "jatr"
var force*: bool = false # if skipped tests get executed
var enumerate*: bool = false # if true, all failed/crashed and killed tests
# are enumerated to stdout
const outputIgnore* = [ "^DEBUG.*$" ]

27
tests/testeval.nim
View File

@ -24,12 +24,18 @@ import strformat
import testconfig
proc evalTests*(tests: seq[Test]) =
## Goes through every test in tests and evaluates all finished
## tests to success or mismatch
for test in tests:
if test.result == TestResult.ToEval:
test.result = if test.eval(): TestResult.Success else: TestResult.Mismatch
proc printResults*(tests: seq[Test]): bool =
## Goes through every test in tests and prints the number of good/
## skipped/failed/crashed/killed tests to the screen. It also debug
## logs all failed test details and crash messages. It returns
## true if no tests {failed | crashed | got killed}.
var
skipped = 0
success = 0
@ -37,18 +43,31 @@ proc printResults*(tests: seq[Test]): bool =
crash = 0
killed = 0
for test in tests:
log(LogLevel.Debug, &"Test {test.name}@{test.path} result: {test.result}")
var level = LogLevel.Debug
var detailLevel = LogLevel.Debug
if test.important:
level = LogLevel.Info
detailLevel = LogLevel.Info
if (test.result in {TestResult.Crash, TestResult.Mismatch, TestResult.Killed} and enumerate):
level = LogLevel.Enumeration
log(level, &"Test {test.name}@{test.path} result: {test.result}")
case test.result:
of TestResult.Skip:
inc skipped
of TestResult.Mismatch:
inc fail
log(LogLevel.Debug, &"[{test.name}@{test.path}\nstdout:\n{test.output}\nstderr:\n{test.error}\nexpected stdout:\n{test.expectedOutput}\nexpected stderr:\n{test.expectedError}\n]")
log(LogLevel.Debug, &"\nMismatch pos for stdout: {test.mismatchPos}\nMismatch pos for stderr: {test.errorMismatchPos}")
log(detailLevel, &"[{test.name}@{test.path}\nstdout:\n{test.output}\nstderr:\n{test.error}\nexpected stdout:\n{test.expectedOutput}\nexpected stderr:\n{test.expectedError}\n]")
log(detailLevel, &"\nMismatch pos for stdout: {test.mismatchPos}\nMismatch pos for stderr: {test.errorMismatchPos}")
of TestResult.Crash:
inc crash
log(LogLevel.Debug, &"{test.name}@{test.path} \ncrash:\n{test.error}")
log(detailLevel, &"{test.name}@{test.path} \ncrash:\n{test.error}")
of TestResult.Success:
if test.m_skipped:
log(LogLevel.Info, &"Test {test.name}@{test.path} succeeded, despite being marked to be skipped.")
inc success
of TestResult.Killed:
inc killed

6
tests/testobject.nim
View File

@ -40,6 +40,10 @@ type
source*: string
path*: string
name*: string
important*: bool # if set to true, the stdout/stderr and extra debug
# will get printed when finished
m_skipped*: bool # metadata, whether the skipped mode is in the file
# NOT WHETHER THE TEST IS ACTUALLY SKIPPED
# generated after building
expectedOutput*: seq[ExpectedLine]
expectedError*: seq[ExpectedLine]
@ -122,6 +126,8 @@ proc newTest*(name: string, path: string): Test =
result.name = name
result.mismatchPos = -1
result.errorMismatchPos = -1
result.important = false
result.m_skipped = false
proc skip*(test: Test) =
test.result = TestResult.Skip

12
tests/testrun.nim
View File

@ -12,7 +12,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
# Test runner supervisor/manager
## Test runner supervisor/manager
import testobject
import logutils
@ -22,10 +22,13 @@ import strformat
import os
proc runTest(test: Test) =
## Starts running a test
log(LogLevel.Debug, &"Starting test {test.path}.")
test.start()
proc tryFinishTest(test: Test): bool =
## Attempts to finish a test and returns true if it finished.
## False otherwise.
if test.running():
return false
test.finish()
@ -33,16 +36,21 @@ proc tryFinishTest(test: Test): bool =
return true
proc killTest(test: Test) =
## Kills the test, logs kill reason as taking too long
if test.running():
test.kill()
log(LogLevel.Error, &"Test {test.path} was killed for taking too long.")
proc killTests*(tests: seq[Test]) =
## kills all running tests in tests sequence
for test in tests:
if test.running():
test.kill()
proc runTests*(tests: seq[Test], runner: string) =
proc runTests*(tests: seq[Test]) =
## Runs all tests tests in tests, manages the maximum alive tests
## and launching of tests parallel. Also writes progress to the
## screen
var
aliveTests = 0
currentTest = 0