diff --git a/tests/japl/all.jpl b/tests/japl/all.jpl index 5527309..5a8115d 100644 --- a/tests/japl/all.jpl +++ b/tests/japl/all.jpl @@ -1,3 +1,6 @@ +[Test: all] +[skip] +[source: mixed] // Example file to test JAPL's syntax // Mathematical expressions @@ -137,3 +140,5 @@ mark.greet(); "implicit start"[:5]; // From 0 to 5 "hello" + " world"; // Strings are immutable! "hello" * 3; //hellohellohello +[end] +[end] diff --git a/tests/japl/arithmetic.jpl b/tests/japl/arithmetic.jpl index 6a23318..98de321 100644 --- a/tests/japl/arithmetic.jpl +++ b/tests/japl/arithmetic.jpl @@ -1,3 +1,5 @@ +[Test: arithmetic] +[source: mixed] //int arithmetic print(7+5); //stdout:12 @@ -32,3 +34,5 @@ print(64/-64);//stdout:-1.0 print(8/0);//stdout:inf print(8/-0);//stdout:inf print(-8/0);//stdout:-inf +[end] +[end] diff --git a/tests/japl/bitwise.jpl b/tests/japl/bitwise.jpl index cab7b56..d826179 100644 --- a/tests/japl/bitwise.jpl +++ b/tests/japl/bitwise.jpl @@ -1,3 +1,5 @@ +[Test: bitwise] +[source: mixed] print(~5 | 5);//stdout:-1 print(1 | 2);//stdout:3 print(1 & 2);//stdout:0 @@ -7,3 +9,5 @@ print(32 | 64);//stdout:96 print(96 & 32);//stdout:32 print(~0);//stdout:-1 print(~356);//stdout:-357 +[end] +[end] diff --git a/tests/japl/booleans.jpl b/tests/japl/booleans.jpl index f02cfc4..ac15abd 100644 --- a/tests/japl/booleans.jpl +++ b/tests/japl/booleans.jpl @@ -1,3 +1,5 @@ +[Test: booleans] +[source: mixed] print(2 or 3);//stdout:2 print(2 and 3);//stdout:3 print(false or true);//stdout:true @@ -19,3 +21,5 @@ print(not 0);//stdout:true print(not 1);//stdout:false print(not 1 and not 2);//stdout:false print(not (1 and 0));//stdout:true +[end] +[end] diff --git a/tests/japl/callchain.jpl b/tests/japl/callchain.jpl index eb2d32d..5c4f7f7 100644 --- a/tests/japl/callchain.jpl +++ b/tests/japl/callchain.jpl @@ -1,4 +1,5 @@ - +[Test: callchain] +[source: mixed] fun add2(x) { return x + 2; @@ -16,3 +17,5 @@ print(add2(sub2(mul2(sub2(5))))); //5-2=3 //3*2=6 //stdout:6 +[end] +[end] diff --git a/tests/japl/comparisons.jpl b/tests/japl/comparisons.jpl index 0ce2e87..4ca915d 100644 --- a/tests/japl/comparisons.jpl +++ b/tests/japl/comparisons.jpl @@ -1,3 +1,5 @@ +[Test: comparisons] +[source: mixed] var x = 4; var y = 5; var z = 6; @@ -40,3 +42,5 @@ if (8 <= z) print("15"); else print("16");//stdout:16 +[end] +[end] diff --git a/tests/japl/compile_time_interning.jpl b/tests/japl/compile_time_interning.jpl index 1d2cf0e..24cdeb5 100644 --- a/tests/japl/compile_time_interning.jpl +++ b/tests/japl/compile_time_interning.jpl @@ -1,3 +1,5 @@ +[Test: compile_time_intern] +[source: mixed] //compile time interning var a = "hello"; @@ -9,5 +11,6 @@ print(a is b);//stdout:true var x = "ex"; var y = "ey"; print(x is y);//stdout:false - +[end] +[end] diff --git a/tests/japl/constant_long.jpl b/tests/japl/constant_long.jpl index 416ed75..ca6771b 100644 --- a/tests/japl/constant_long.jpl +++ b/tests/japl/constant_long.jpl @@ -1,3 +1,5 @@ +[Test: constant_long] +[source: mixed] // Test for constants var v_1 = 1; @@ -128,3 +130,5 @@ var v_125 = 1; var v_126 = 1; var v_127 = 1; var v_128 = 1; +[end] +[end] diff --git a/tests/japl/errors/read_in_own_init.jpl b/tests/japl/errors/read_in_own_init.jpl index b33d8b3..855ee55 100644 --- a/tests/japl/errors/read_in_own_init.jpl +++ b/tests/japl/errors/read_in_own_init.jpl @@ -1,4 +1,15 @@ +[Test: read_in_own_init] +[source: mixed] var a = 1; { var a = a; } -//stdout:A fatal error occurred while compiling '''', line 1, at ';' -> cannot read local variable in its own initializer +//stderr:A fatal error occurred while compiling '''', line 1, at ';' -> cannot read local variable in its own initializer +[end] +[end] -//stdout: +[Test: read_in_own_init_regex] +[source: raw] +var a = 1; { var a = a; } +[end] +[stderr: re] +[[^\-]*-> cannot read local variable in its own initializer +[end] +[end] diff --git a/tests/japl/errors/undefname.jpl b/tests/japl/errors/undefname.jpl index 8c2314f..d2005ea 100644 --- a/tests/japl/errors/undefname.jpl +++ b/tests/japl/errors/undefname.jpl @@ -1,8 +1,22 @@ +[Test: undefname] +[source: mixed] var a = b; -//stdout:An unhandled exception occurred, traceback below: +//stderr:An unhandled exception occurred, traceback below: -//stdout: File '''', line 1, in : +//stderr: File '''', line 1, in : -//stdout:ReferenceError: undefined name 'b' +//stderr:ReferenceError: undefined name 'b' -//stdout: +[end] +[end] + +[Test: undefname_raw] +[source: raw] +var a = b; +[end] +[stderr] +An unhandled exception occurred, traceback below: +[] File '''', line 1, in : +ReferenceError: undefined name 'b' +[end] +[end] diff --git a/tests/japl/errors/unsup_binary_intstr.jpl b/tests/japl/errors/unsup_binary_intstr.jpl index c940b98..960c43e 100644 --- a/tests/japl/errors/unsup_binary_intstr.jpl +++ b/tests/japl/errors/unsup_binary_intstr.jpl @@ -1,8 +1,10 @@ +[Test: unsup_binary_instr] +[source: mixed] var a = 2 + "hey"; -//stdout:An unhandled exception occurred, traceback below: +//stderr:An unhandled exception occurred, traceback below: -//stdout: File '''', line 1, in : +//stderr: File '''', line 1, in : -//stdout:TypeError: unsupported binary operator '+' for objects of type 'integer' and 'string' - -//stdout: +//stderr:TypeError: unsupported binary operator '+' for objects of type 'integer' and 'string' +[end] +[end] diff --git a/tests/japl/euler/problem1.jpl b/tests/japl/euler/problem1.jpl index b4d5623..f4425c9 100644 --- a/tests/japl/euler/problem1.jpl +++ b/tests/japl/euler/problem1.jpl @@ -1,3 +1,5 @@ +[Test: problem1] +[source: mixed] // Task: find the multiples of 3 and 5 below 1000, find their sum var sum = 0; @@ -9,3 +11,5 @@ for (var x = 3; x < 1001; x = x + 1) } } print(sum);//stdout:234168 +[end] +[end] diff --git a/tests/japl/euler/problem2.jpl b/tests/japl/euler/problem2.jpl index eceb537..2c2357c 100644 --- a/tests/japl/euler/problem2.jpl +++ b/tests/japl/euler/problem2.jpl @@ -1,3 +1,5 @@ +[Test: problem2] +[source: mixed] // Sum of even valued fibonacci numbers that don't exceed 4M var a = 1; @@ -14,4 +16,5 @@ while (a < 4000000) b = c; } print(sum);//stdout:4613732 - +[end] +[end] diff --git a/tests/japl/euler/problem4.jpl b/tests/japl/euler/problem4.jpl index 831a608..0badc12 100644 --- a/tests/japl/euler/problem4.jpl +++ b/tests/japl/euler/problem4.jpl @@ -1,3 +1,5 @@ +[Test: problem4] +[source: mixed] // Find the largest palindrome that is a product of two 3 digit numbers fun isPalindrome(n) @@ -63,3 +65,5 @@ for (var i = 100; i < 1000; i = i + 1) } } print(largest); +[end] +[end] diff --git a/tests/japl/for.jpl b/tests/japl/for.jpl index f278395..9ea022e 100644 --- a/tests/japl/for.jpl +++ b/tests/japl/for.jpl @@ -1,6 +1,10 @@ +[Test: for] +[source: mixed] for (var x = 0; x < 2; x = x + 1) { print(x); //stdout:0 //stdout:1 } +[end] +[end] diff --git a/tests/japl/for_with_function.jpl b/tests/japl/for_with_function.jpl index 26c3603..5c4b002 100644 --- a/tests/japl/for_with_function.jpl +++ b/tests/japl/for_with_function.jpl @@ -1,4 +1,6 @@ - +[Test: forwithfunction] +[skip] +[source: mixed] var y = 0; //a global to keep track of state //does not need closures for this to work yet @@ -82,3 +84,5 @@ for (var i = 0; i != -1; i = next(i)) // y = 9 //stdout:10 // y = 10 +[end] +[end] diff --git a/tests/japl/hello.jpl b/tests/japl/hello.jpl index fa707d8..545fe1b 100644 --- a/tests/japl/hello.jpl +++ b/tests/japl/hello.jpl @@ -1,4 +1,6 @@ +[Test: hello] +[source:mixed] print("Hello, world."); //stdout:Hello, world. - -//stdout: +[end] +[end] diff --git a/tests/japl/hellojapl.jpl b/tests/japl/hellojapl.jpl index 753b391..2004e39 100644 --- a/tests/japl/hellojapl.jpl +++ b/tests/japl/hellojapl.jpl @@ -1,4 +1,15 @@ +[Test: hellojapl] +[source: mixed] print("Hello, JAPL."); //stdout:Hello, JAPL. +[end] +[end] -//stdout: +[Test: hello_second_way] +[source: raw] +print("Hello, JAPL."); +[end] +[stdout] +Hello, JAPL. +[end] +[end] diff --git a/tests/japl/if.jpl b/tests/japl/if.jpl index aceae0c..06e4f7b 100644 --- a/tests/japl/if.jpl +++ b/tests/japl/if.jpl @@ -1,3 +1,5 @@ +[Test: if] +[source: mixed] var x = 5; if (x > 2) { @@ -26,3 +28,5 @@ if (2 == x) print("2"); else print("not 2");//stdout:not 2 +[end] +[end] diff --git a/tests/japl/ifchain.jpl b/tests/japl/ifchain.jpl index 16a9396..aa86d56 100644 --- a/tests/japl/ifchain.jpl +++ b/tests/japl/ifchain.jpl @@ -1,3 +1,5 @@ +[Test: ifchain] +[source: mixed] fun printInt(x) { if (x == 1) print("one"); @@ -20,3 +22,5 @@ x = 7; printInt(x); x = 1; printInt(x);//stdout:one +[end] +[end] diff --git a/tests/japl/inputtest.jpl b/tests/japl/inputtest.jpl index 1cc66af..f49f2a0 100644 --- a/tests/japl/inputtest.jpl +++ b/tests/japl/inputtest.jpl @@ -1,3 +1,22 @@ +[Test: inputtest] +[source: mixed] //stdin:Hello world! print(readLine()); //stdout:Hello world! +[end] +[end] + +[Test: inputtesttwo] + +[source: raw] +print(readLine()); +[end] + +[stdin] +Hello world! +[end] + +[stdout] +Hello world! +[end] +[end] diff --git a/tests/japl/is.jpl b/tests/japl/is.jpl index 2f1cca1..92bc584 100644 --- a/tests/japl/is.jpl +++ b/tests/japl/is.jpl @@ -1,3 +1,5 @@ +[Test: is] +[source:mixed] var x = 4; var y = x; @@ -19,4 +21,5 @@ print((l is z) is l);//stdout:true var k; print(k is nil);//stdout:true - +[end] +[end] diff --git a/tests/japl/lambdachain.jpl b/tests/japl/lambdachain.jpl index a279488..a19d6b5 100644 --- a/tests/japl/lambdachain.jpl +++ b/tests/japl/lambdachain.jpl @@ -1,4 +1,5 @@ - +[Test: lambdachain] +[source: mixed] var add2 = lambda(x) { return x + 2; @@ -16,3 +17,5 @@ print(add2(sub2(mul2(sub2(5))))); //5-2=3 //3*2=6 //stdout:6 +[end] +[end] diff --git a/tests/japl/longs/globAssgnRead.jpl b/tests/japl/longs/globAssgnRead.jpl index b255eea..bb4d08d 100644 --- a/tests/japl/longs/globAssgnRead.jpl +++ b/tests/japl/longs/globAssgnRead.jpl @@ -1,3 +1,5 @@ +[Test: glob_assgn_read] +[source: mixed] var a0 = 451; var a1 = 5098; var a2 = 469; @@ -1499,3 +1501,5 @@ print(a151);//stdout:4839 print(a975);//stdout:7651 print(a7);//stdout:2979 print(a661);//stdout:8235 +[end] +[end] diff --git a/tests/japl/longs/globWithSets.jpl b/tests/japl/longs/globWithSets.jpl index bcac0c2..f007479 100644 --- a/tests/japl/longs/globWithSets.jpl +++ b/tests/japl/longs/globWithSets.jpl @@ -1,3 +1,5 @@ +[Test: glob_with_sets] +[source: mixed] var a0 = 829; var a1 = 6820; var a2 = 114; @@ -5236,3 +5238,5 @@ print(a87);//stdout:1282 print(a445);//stdout:1726 print(a790);//stdout:1140 print(a961);//stdout:1708 +[end] +[end] diff --git a/tests/japl/longs/locAssgnRead.jpl b/tests/japl/longs/locAssgnRead.jpl index 47b8e28..2b7eaae 100644 --- a/tests/japl/longs/locAssgnRead.jpl +++ b/tests/japl/longs/locAssgnRead.jpl @@ -1,3 +1,5 @@ +[Test: loc_assgn_read] +[source: mixed] { var a0 = 9103; var a1 = 4565; @@ -1500,4 +1502,6 @@ print(a142);//stdout:4255 print(a722);//stdout:5380 print(a538);//stdout:8625 print(a809);//stdout:4506 -} \ No newline at end of file +} +[end] +[end] diff --git a/tests/japl/longs/locWithSets.jpl b/tests/japl/longs/locWithSets.jpl index 24eee6b..0382287 100644 --- a/tests/japl/longs/locWithSets.jpl +++ b/tests/japl/longs/locWithSets.jpl @@ -1,3 +1,5 @@ +[Test: loc_with_sets] +[source: mixed] { var a0 = 8313; var a1 = 3509; @@ -5183,4 +5185,6 @@ print(a729);//stdout:2001 print(a380);//stdout:2145 print(a125);//stdout:4280 print(a55);//stdout:6992 -} \ No newline at end of file +} +[end] +[end] diff --git a/tests/japl/nan.jpl b/tests/japl/nan.jpl index bf80939..c479160 100644 --- a/tests/japl/nan.jpl +++ b/tests/japl/nan.jpl @@ -1,5 +1,9 @@ +[Test: nan] +[source: raw] print((5/0)*0); +[end] +[stdout] +nan +[end] +[end] -//stdout:nan - -//stdout: diff --git a/tests/japl/reassignment.japl b/tests/japl/reassignment.japl index 7c52661..1d87459 100644 --- a/tests/japl/reassignment.japl +++ b/tests/japl/reassignment.japl @@ -1,3 +1,5 @@ +[Test: reassignment] +[source: mixed] { var x = 5; var y = x; @@ -23,3 +25,5 @@ fun resetter(x) { var q = 5; resetter(q);//stdout:7 print(q);//stdout:5 +[end] +[end] diff --git a/tests/japl/runtime_interning.jpl b/tests/japl/runtime_interning.jpl index 970f603..78e6ea9 100644 --- a/tests/japl/runtime_interning.jpl +++ b/tests/japl/runtime_interning.jpl @@ -1,3 +1,6 @@ +[Test: runtimeinterning] +[skip] +[source: mixed] //runtime interning var f = "leafy"; @@ -11,5 +14,5 @@ print(h is j);//stdout:true var x = "ex"; var y = "ey"; print(x is y);//stdout:false - - +[end] +[end] diff --git a/tests/japl/shadowing.jpl b/tests/japl/shadowing.jpl index 6ba0a98..cc52808 100644 --- a/tests/japl/shadowing.jpl +++ b/tests/japl/shadowing.jpl @@ -1,3 +1,5 @@ +[Test: shadowing] +[source: mixed] //similar to vars.jpl, but more focused on shadowing // simple shadowing @@ -74,3 +76,5 @@ eat();//stdout:nom nom nom print(eat);//stdout:5 } eat();//stdout:nom nom nom +[end] +[end] diff --git a/tests/japl/strings.jpl b/tests/japl/strings.jpl index b06e4ac..ae65a48 100644 --- a/tests/japl/strings.jpl +++ b/tests/japl/strings.jpl @@ -1,3 +1,5 @@ +[Test: strings] +[source: mixed] var left = "left"; var right = "right"; var directions = left + " " + right; @@ -10,3 +12,5 @@ left = left + " side"; print(left);//stdout:left side right = "side: " + right; print(right);//stdout:side: right +[end] +[end] diff --git a/tests/japl/vars.jpl b/tests/japl/vars.jpl index ef47730..250fd56 100644 --- a/tests/japl/vars.jpl +++ b/tests/japl/vars.jpl @@ -1,3 +1,5 @@ +[Test: vars] +[source: mixed] var x = 1; var y = 2; print(x);//stdout:1 @@ -28,3 +30,5 @@ longName = "hello"; print(longName); //stdout:hello longName = longName + " world"; print(longName); //stdout:hello world +[end] +[end] diff --git a/tests/japl/while.jpl b/tests/japl/while.jpl index 513d067..75679ca 100644 --- a/tests/japl/while.jpl +++ b/tests/japl/while.jpl @@ -1,3 +1,5 @@ +[Test: while] +[source: mixed] var x = 5; while (x > 0) { @@ -18,3 +20,5 @@ while (x < 10) string = string + "A"; } print(string);//stdout:hAAAAAAAAAA +[end] +[end] diff --git a/tests/jatr.nim b/tests/jatr.nim index a7ee308..aeb893a 100644 --- a/tests/jatr.nim +++ b/tests/jatr.nim @@ -36,5 +36,6 @@ try: except: let error = getCurrentException() writeLine stderr, error.msg + writeLine stderr, error.getStacktrace() quit(1) diff --git a/tests/jats.nim b/tests/jats.nim index 6217c07..7739e88 100644 --- a/tests/jats.nim +++ b/tests/jats.nim @@ -36,7 +36,7 @@ type DebugAction {.pure.} = enum Interactive, Stdout QuitValue {.pure.} = enum - Success, Failure, ArgParseErr, InternalErr, Interrupt + Success, Failure, ArgParseErr, Unreachable, Interrupt, JatrNotFound, UncaughtException when isMainModule: var optparser = initOptParser(commandLineParams()) @@ -132,7 +132,7 @@ Flags: discard else: echo &"Unknown action {action}, please contact the devs to fix this." - quit int(QuitValue.InternalErr) + quit int(QuitValue.Unreachable) setVerbosity(verbose) setLogfiles(targetFiles) # start of JATS @@ -143,10 +143,15 @@ Flags: runNimTests() var jatr = "jatr" var testDir = "japl" - if not fileExists(jatr) and fileExists("tests" / jatr): - log(LogLevel.Debug, &"Must be in root: prepending \"tests\" to paths") - jatr = "tests" / jatr - testDir = "tests" / testDir + if not fileExists(jatr): + if fileExists("tests" / jatr): + log(LogLevel.Debug, &"Must be in root: prepending \"tests\" to paths") + jatr = "tests" / jatr + testDir = "tests" / testDir + else: + echo "The tests directory couldn't be found." + quit int(QuitValue.JatrNotFound) + testRunner = jatr log(LogLevel.Info, &"Running JAPL tests.") log(LogLevel.Info, &"Building tests...") let tests: seq[Test] = buildTests(testDir) @@ -169,8 +174,9 @@ Flags: # special options to view the entire debug log except: errorDisplay() - writeLine stderr, getCurrentExceptionMsg() - writeStacktrace() + writeLine stdout, getCurrentExceptionMsg() + writeLine stdout, getCurrentException().getStackTrace() + quit(int(QuitValue.UncaughtException)) finally: let logs = getTotalLog() diff --git a/tests/logutils.nim b/tests/logutils.nim index 5229e60..29f07cc 100644 --- a/tests/logutils.nim +++ b/tests/logutils.nim @@ -55,9 +55,13 @@ proc log*(level: LogLevel, msg: string) = echo msg setForegroundColor(fgDefault) +type FatalError* = ref object of CatchableError + proc fatal*(msg: string) = log(LogLevel.Fatal, msg) - raise newException(CatchableError, msg) + let e = new(FatalError) + e.msg = msg + raise e proc getTotalLog*: string = totalLog @@ -67,7 +71,7 @@ type Buffer* = ref object previous: string proc newBuffer*: Buffer = - hideCursor() +# hideCursor() new(result) proc updateProgressBar*(buf: Buffer, text: string, total: int, current: int) = @@ -89,7 +93,7 @@ proc updateProgressBar*(buf: Buffer, text: string, total: int, current: int) = buf.contents = newline proc clearLineAndWrite(text: string, oldsize: int) = - write stdout, text & "\r" + write stdout, "\r" & text & "\r" proc render*(buf: Buffer) = if verbose: #and buf.previous != buf.contents: diff --git a/tests/testbuilder.nim b/tests/testbuilder.nim index 60c7846..4808a9a 100644 --- a/tests/testbuilder.nim +++ b/tests/testbuilder.nim @@ -21,21 +21,39 @@ import strutils import sequtils import strformat -proc parseModalLine(line: string): tuple[modal: bool, mode: string, detail: string] = +proc parseModalLine(line: string): tuple[modal: bool, mode: string, detail: string, comment: bool] = + + # when non modal, mode becomes the line + # when comment is true, it must not do anything to whenever it is exported + let line = line.strip() result.modal = false result.mode = "" result.detail = "" + result.comment = false + if line.len() > 0 and line[0] == '[': + if line.len() > 1: + if line[1] == '[': + result.mode = line[1..line.high()] + return result + elif line[1] == ';': + result.comment = true + result.modal = true + return result + elif line[1] == ']': + result.mode = line[2..line.high()] + return result result.modal = true else: + result.mode = line return result - + var colon = false for i in countup(0, line.high()): let ch = line[i] - if ch in Letters: + if ch in Letters or ch in Digits or ch in {'_', '-'}: if colon: result.detail &= ($ch).toLower() else: @@ -67,22 +85,17 @@ proc buildTest(lines: seq[string], i: var int, name: string, path: string): Test var inside: bool = false var body: string while i < lines.len(): - let line = lines[i] - let parsed = parseModalLine(line.strip()) - let modal = parsed.modal - if parsed.modal: + let parsed = parseModalLine(lines[i].strip()) + let line = parsed.mode + if parsed.modal and not parsed.comment: if inside: if parsed.mode == "end": # end inside - echo "end" if mode == "source" and (detail == "" or detail == "mixed"): result.parseMixed(body) - echo "mixed " & body elif mode == "source" and detail == "raw": - echo "parse source " & body result.parseSource(body) elif mode == "stdout" and (detail == ""): - echo "stdout " & body result.parseStdout(body) elif mode == "stdoutre" or (mode == "stdout" and detail == "re"): result.parseStdout(body, true) @@ -106,22 +119,24 @@ proc buildTest(lines: seq[string], i: var int, name: string, path: string): Test detail = "" body = "" else: - fatal &"Invalid mode {parsed.mode} when inside a block." + fatal &"Invalid mode {parsed.mode} when inside a block (currently in mode {mode})." else: # still if modal, but not inside - echo "not inside" - if mode == "skip": + if parsed.mode == "skip": result.skip() - elif mode == "end": + elif parsed.mode == "end": # end of test return result else: - echo "mode " & parsed.mode # start a new mode inside = true mode = parsed.mode detail = parsed.detail + elif parsed.comment: + discard elif inside: # when not modal body &= line & "\n" + elif line.strip().len() == 0: + discard # whitespace else: # invalid fatal &"Invalid code inside a test: {line} in test {name} at {path}" @@ -132,16 +147,17 @@ proc buildTestFile(path: string): seq[Test] = let lines = path.readFile().split('\n') var i = 0 while i < lines.len(): - let line = lines[i].strip() - let parsed = line.parseModalLine() - if parsed.modal: + let parsed = lines[i].strip().parseModalLine() + let line = parsed.mode + if parsed.modal and not parsed.comment: if parsed.mode == "test": let testname = parsed.detail log(LogLevel.Debug, &"Building test {testname} at {path}") result.add buildTest(lines, i, testname, path) else: fatal &"Invalid mode at root-level {parsed.mode} at line {i} of file {path}." - # root can only contain "test" modes, anything else is just a comment + + # root can only contain "test" modes, anything else is just a comment (including modal and non modal comments) inc i proc buildTests*(testDir: string): seq[Test] = @@ -153,6 +169,8 @@ proc buildTests*(testDir: string): seq[Test] = else: try: result &= buildTestFile(candidate) + except FatalError: + discard except: write stderr, getCurrentExceptionMsg() write stderr, getCurrentException().getStacktrace() diff --git a/tests/testconfig.nim b/tests/testconfig.nim index 849a241..f678777 100644 --- a/tests/testconfig.nim +++ b/tests/testconfig.nim @@ -14,15 +14,11 @@ const jatsVersion* = "(dev)" -# Tests that represent not-yet implemented behaviour -const exceptions* = ["all.jpl", "for_with_function.jpl", "runtime_interning.jpl"] -# TODO: for_with_function.jpl should already be implemented, check on it - var maxAliveTests* = 16 # number of tests that can run parallel const testWait* = 100 # number of milliseconds per cycle -const timeout* = 100 # number of cycles after which a test is killed for timeout +const timeout* = 50 # number of cycles after which a test is killed for timeout var testRunner* = "jatr" -const outputStripReplaces* = [ r"\[DEBUG.*\n", r"[\n\r ]*$" ] -const outputStripReplaceTargets* = [ "", "" ] +const outputStripReplaces* = [ r"\[DEBUG[^\n]*$" ] +const outputStripReplaceTargets* = [ "" ] diff --git a/tests/testeval.nim b/tests/testeval.nim index eeda47e..81b93ee 100644 --- a/tests/testeval.nim +++ b/tests/testeval.nim @@ -35,23 +35,26 @@ proc printResults*(tests: seq[Test]): bool = success = 0 fail = 0 crash = 0 + killed = 0 for test in tests: - log(LogLevel.Debug, &"Test {test.path} result: {test.result}") + log(LogLevel.Debug, &"Test {test.name}@{test.path} result: {test.result}") case test.result: of TestResult.Skip: inc skipped of TestResult.Mismatch: inc fail - log(LogLevel.Debug, &"[{test.path}\nstdout:\n{test.output}\nstderr:\n{test.error}\nexpected stdout:\n{test.expectedOutput}\nexpected stderr:\n{test.expectedError}\n]") + log(LogLevel.Debug, &"[{test.name}@{test.path}\nstdout:\n{test.output}\nstderr:\n{test.error}\nexpected stdout:\n{test.expectedOutput}\nexpected stderr:\n{test.expectedError}\n]") log(LogLevel.Debug, &"\nMismatch pos for stdout: {test.mismatchPos}\nMismatch pos for stderr: {test.errorMismatchPos}") of TestResult.Crash: inc crash - log(LogLevel.Debug, &"{test.path} \ncrash:\n{test.output}") + log(LogLevel.Debug, &"{test.name}@{test.path} \ncrash:\n{test.error}") of TestResult.Success: inc success + of TestResult.Killed: + inc killed else: - log(LogLevel.Error, &"Probably a testing suite bug: test {test.path} has result {test.result}") + log(LogLevel.Error, &"Probably a testing suite bug: test {test.path} has result {test.result}. Refer to testeval.nim/printResults.") let finalLevel = if fail == 0 and crash == 0: LogLevel.Info else: LogLevel.Error - log(finalLevel, &"{tests.len()} tests: {success} succeeded, {skipped} skipped, {fail} failed, {crash} crashed.") + log(finalLevel, &"{tests.len()} tests: {success} succeeded, {skipped} skipped, {fail} failed, {killed} killed, {crash} crashed.") result = fail == 0 and crash == 0 diff --git a/tests/testobject.nim b/tests/testobject.nim index 020ddcc..d793df1 100644 --- a/tests/testobject.nim +++ b/tests/testobject.nim @@ -63,7 +63,7 @@ proc compileExpectedOutput(source: string, rawkw: string, rekw: string): seq[Exp for line in source.split('\n'): if line =~ re("^.*//" & rawkw & ":[ ]?(.*)$"): result &= genEL(matches[0], ExpectedLineKind.Raw) - elif line =~ re("^.*//" & rekw & ":[ ]?(.*$"): + elif line =~ re("^.*//" & rekw & ":[ ]?(.*)$"): result &= genEL(matches[0], ExpectedLineKind.Regex) proc compileExpectedOutput(source: string): seq[ExpectedLine] = @@ -82,7 +82,6 @@ proc parseMixed*(test: Test, source: string) = test.expectedOutput = compileExpectedOutput(source) test.expectedError = compileExpectedError(source) test.input = compileInput(source) - test.result = TestResult.Unstarted proc parseSource*(test: Test, source: string) = test.source &= source @@ -100,6 +99,13 @@ proc parseStdout*(test: Test, source: string, regex: bool = false, stderr: bool else: test.expectedOutput.add(genEL(line, kind)) + if stderr: + while test.expectedError.len() > 0 and test.expectedError[test.expectedError.high()].content == "": + discard test.expectedError.pop() + else: + while test.expectedOutput.len() > 0 and test.expectedOutput[test.expectedOutput.high()].content == "": + discard test.expectedOutput.pop() + proc parseStderr*(test: Test, source: string, regex: bool = false) = parseStdout(test, source, regex, true) @@ -108,6 +114,7 @@ proc parsePython*(test: Test, source: string) = proc newTest*(name: string, path: string): Test = new(result) + result.result = TestResult.Unstarted result.path = path result.name = name result.mismatchPos = -1 @@ -160,31 +167,36 @@ proc running*(test: Test): bool = # Helpers for evaluating tests -proc toStrip(input: string): string = - var text = input - for i in countup(0, outputStripReplaces.high()): - text = text.replace(outputStripReplaces[i], outputStripReplaceTargets[i]) +proc stdStrip(input: string): seq[string] = + var lines = input.split('\n') + var toRemove: seq[int] + for i in countup(0, lines.high()): + template line: string = lines[i] + let hadContent = line.len() > 0 + for op in countup(0, outputStripReplaces.high()): + line = line.replace(re(outputStripReplaces[op]), outputStripReplaceTargets[op]) + if hadContent and line.len() == 0: + toRemove.add(i) + + for i in toRemove: + lines.delete(i) + while lines.len() > 0 and lines[lines.high()] == "": + discard lines.pop() + lines proc eval*(test: Test): bool = - echo repr test.output.toStrip().split('\n') - echo repr test.expectedOutput let - outputLines = test.output.toStrip().split('\n') - errorLines = test.error.toStrip().split('\n') + outputLines = test.output.stdStrip() + errorLines = test.error.stdStrip() if test.expectedOutput.len() != outputLines.len(): - if outputLines.len() - 1 == test.expectedOutput.len() and outputLines[outputLines.high()].strip() == "": - discard - else: - test.mismatchPos = outputLines.len() - return false + test.mismatchPos = outputLines.len() + return false if test.expectedError.len() != errorLines.len(): - if errorLines.len() - 1 == test.expectedError.len() and errorLines[errorLines.high()].strip() == "": - discard - else: - test.errorMismatchPos = errorLines.len() - return false + test.errorMismatchPos = errorLines.len() + return false + for i in countup(0, test.expectedOutput.high()): let line = test.expectedOutput[i] case line.kind: