From 390087dab5d51ea9e7fcd08e7acb06efef06a709 Mon Sep 17 00:00:00 2001 From: Productive2 <48047721+Productive2@users.noreply.github.com> Date: Sun, 10 Jan 2021 20:33:29 +0100 Subject: [PATCH] Testing now supports testing for JAPL exceptions... kinda --- .gitignore | 3 ++- tests/japl/hello.jpl | 4 ++++ tests/japl/hellojapl.jpl | 4 ++++ tests/japl/nan.jpl | 5 ++++ tests/japl/read_in_own_init.jpl | 4 ++++ tests/japl/undefname.jpl | 8 +++++++ tests/japl/unsup_binary_intstr.jpl | 8 +++++++ tests/maketest.nim | 38 +++++++++++++++++------------- tests/runtests.nim | 37 +++++++++++++++++++---------- 9 files changed, 81 insertions(+), 30 deletions(-) create mode 100644 tests/japl/hello.jpl create mode 100644 tests/japl/hellojapl.jpl create mode 100644 tests/japl/nan.jpl create mode 100644 tests/japl/read_in_own_init.jpl create mode 100644 tests/japl/undefname.jpl create mode 100644 tests/japl/unsup_binary_intstr.jpl diff --git a/.gitignore b/.gitignore index 6ab8b78..5e10e43 100644 --- a/.gitignore +++ b/.gitignore @@ -18,8 +18,9 @@ src/lexer src/vm tests/runtests testresults.txt -testoutput.txt +.testoutput.txt .tempcode.jpl +.tempcode_drEHdZuwNYLqsQaMDMqeNRtmqoqXBXfnCfeqEcmcUYJToBVQkF.jpl .tempoutput.txt # MacOS diff --git a/tests/japl/hello.jpl b/tests/japl/hello.jpl new file mode 100644 index 0000000..50ad207 --- /dev/null +++ b/tests/japl/hello.jpl @@ -0,0 +1,4 @@ +print("Hello, world."); +//output:Hello, world. + +//output: diff --git a/tests/japl/hellojapl.jpl b/tests/japl/hellojapl.jpl new file mode 100644 index 0000000..b648d48 --- /dev/null +++ b/tests/japl/hellojapl.jpl @@ -0,0 +1,4 @@ +print("Hello, JAPL."); +//output:Hello, JAPL. + +//output: diff --git a/tests/japl/nan.jpl b/tests/japl/nan.jpl new file mode 100644 index 0000000..ee38b1a --- /dev/null +++ b/tests/japl/nan.jpl @@ -0,0 +1,5 @@ +print((5/0)*0); + +//output:nan + +//output: diff --git a/tests/japl/read_in_own_init.jpl b/tests/japl/read_in_own_init.jpl new file mode 100644 index 0000000..70a301f --- /dev/null +++ b/tests/japl/read_in_own_init.jpl @@ -0,0 +1,4 @@ +var a = 1; { var a = a; } +//output:A fatal error occurred while compiling '', line 1, at ';' -> cannot read local variable in its own initializer + +//output: diff --git a/tests/japl/undefname.jpl b/tests/japl/undefname.jpl new file mode 100644 index 0000000..1d24cdb --- /dev/null +++ b/tests/japl/undefname.jpl @@ -0,0 +1,8 @@ +var a = b; +//output:Traceback (most recent call last): + +//output: File '', line 1, in '': + +//output:ReferenceError: undefined name 'b' + +//output: diff --git a/tests/japl/unsup_binary_intstr.jpl b/tests/japl/unsup_binary_intstr.jpl new file mode 100644 index 0000000..4384c8a --- /dev/null +++ b/tests/japl/unsup_binary_intstr.jpl @@ -0,0 +1,8 @@ +var a = 2 + "hey"; +//output:Traceback (most recent call last): + +//output: File '', line 1, in '': + +//output:TypeError: unsupported binary operator '+' for objects of type 'integer' and 'string' + +//output: diff --git a/tests/maketest.nim b/tests/maketest.nim index 166d2b7..1c8d787 100644 --- a/tests/maketest.nim +++ b/tests/maketest.nim @@ -20,7 +20,7 @@ # Imports nim tests as well import multibyte, os, strformat, times, re, terminal, strutils -const tempCodeFile = ".tempcode.jpl" +const tempCodeFile = ".tempcode_drEHdZuwNYLqsQaMDMqeNRtmqoqXBXfnCfeqEcmcUYJToBVQkF.jpl" const tempOutputFile = ".tempoutput.txt" proc autoremove(path: string) = @@ -45,41 +45,47 @@ when isMainModule: echo "Please enter the JAPL code or specify a file containing it with file:" let response = stdin.readLine() - var codepath: string if response =~ re"^file:(.*)$": - codepath = matches[0] + let codepath = matches[0] + writeFile(tempCodeFile, readFile(codepath)) else: writeFile(tempCodeFile, response) - codepath = tempCodeFile - let japlCode = readFile(codepath) - discard execShellCmd(&"{japlExec} {codepath} > testoutput.txt") - let output = readFile(tempOutputFile) + let japlCode = readFile(tempCodeFile) + discard execShellCmd(&"{japlExec} {tempCodeFile} > {tempOutputFile} 2>&1") + var output: string + if fileExists(tempOutputFile): + output = readFile(tempOutputFile) + else: + echo "Temporary output file not detected, aborting" + quit(1) autoremove(tempCodeFile) autoremove(tempOutputFile) echo "Got the following output:" echo output echo "Do you want to keep it as a test? [y/N]" - let keep = ($stdin.readChar()).toLower() == "y" + let keepResponse = ($stdin.readLine()).toLower() + let keep = keepResponse[0] == 'y' if keep: block saving: while true: - echo "Please name the test" + echo "Please name the test (without the .jpl extension)" let testname = stdin.readLine() if testname == "": echo "aborted" break saving # I like to be explicit let testpath = testsDir / testname & ".jpl" + echo &"Generating test at {testpath}" var testContent = japlCode - for line in output.lines: - testContent = testContent & "\n" & "//output:" & line & "\n" + for line in output.split('\n'): + var mline = line + mline = mline.replace(tempCodeFile, "") + testContent = testContent & "\n" & "//output:" & mline & "\n" if fileExists(testpath): echo "Test already exists" else: writeFile(testpath, testContent) - echo &"Test generated at {testpath}" - - - - + break saving + else: + echo "Aborting" diff --git a/tests/runtests.nim b/tests/runtests.nim index eaa17c8..d042e25 100644 --- a/tests/runtests.nim +++ b/tests/runtests.nim @@ -22,7 +22,10 @@ # Imports nim tests as well -import multibyte, os, strformat, times, re, terminal +import multibyte, os, strformat, times, re, terminal, strutils + +const tempOutputFile = ".testoutput.txt" +const testResultsPath = "testresults.txt" # Exceptions for tests that represent not-yet implemented behaviour @@ -47,17 +50,25 @@ proc compileExpectedOutput(path: string): string = result &= matches[0] & "\n" -proc deepComp(left, right: string): tuple[same: bool, place: int] = +proc deepComp(left, right: string, path: string): tuple[same: bool, place: int] = + var mleft, mright: string result.same = true if left.high() != right.high(): - result.same = false - for i in countup(0, left.high()): + if left.replace(path, "").high() == right.replace(path, "").high(): + mleft = left.replace(path, "") + mright = right.replace(path, "") + else: + result.same = false + else: + mleft = left + mright = right + for i in countup(0, mleft.high()): result.place = i - if i > right.high(): + if i > mright.high(): # already false because of the len check at the beginning # already correct place because it's updated every i return - if left[i] != right[i]: + if mleft[i] != mright[i]: result.same = false return @@ -106,20 +117,20 @@ proc main(testsDir: string, japlExec: string, testResultsFile: File): tuple[numO skippedTests += subTestResult.skippedTests break singleTest detail(&"Running test '{file.path}'") - if fileExists("testoutput.txt"): - removeFile("testoutput.txt") # in case this crashed - let retCode = execShellCmd(&"{japlExec} {file.path} >> testoutput.txt") + if fileExists(tempOutputFile): + removeFile(tempOutputFile) # in case this crashed + let retCode = execShellCmd(&"{japlExec} {file.path} > {tempOutputFile} 2>&1") numOfTests += 1 if retCode != 0: failedTests += 1 error(&"Test '{file.path}' has crashed!") else: let expectedOutput = compileExpectedOutput(file.path).replace(re"(\n*)$", "") - let realOutputFile = open("testoutput.txt", fmRead) + let realOutputFile = open(tempOutputFile, fmRead) let realOutput = realOutputFile.readAll().replace(re"([\n\r]*)$", "") realOutputFile.close() - removeFile("testoutput.txt") - let comparison = deepComp(expectedOutput, realOutput) + removeFile(tempOutputFile) + let comparison = deepComp(expectedOutput, realOutput, file.path) if comparison.same: successTests += 1 log(&"Test '{file.path}' was successful") @@ -139,7 +150,7 @@ proc main(testsDir: string, japlExec: string, testResultsFile: File): tuple[numO when isMainModule: - let testResultsFile = open("testresults.txt", fmWrite) + let testResultsFile = open(testResultsPath, fmWrite) template log (msg: string) = logWithLevel(LogLevel.Info, testResultsFile, msg) log("Running Nim tests")