Merge pull request #32 from Productive2/master

Improving the testing

.gitignore

@@ -13,10 +13,15 @@ htmldocs/
 src/japl
 src/compiler
 tests/runtests
+tests/maketest
 src/lexer
 src/vm
 tests/runtests
 testresults.txt
+.testoutput.txt
+.tempcode.jpl
+.tempcode_drEHdZuwNYLqsQaMDMqeNRtmqoqXBXfnCfeqEcmcUYJToBVQkF.jpl
+.tempoutput.txt
 
 # MacOS
 
@@ -36,6 +41,3 @@ main
 
 config.nim
 
-# test results
-
-testresults.txt

tests/japl/callchain.jpl

@@ -0,0 +1,18 @@
+
+fun add2(x)
+{
+    return x + 2;
+}
+fun sub2(x)
+{
+    return x - 2;
+}
+fun mul2(x)
+{
+    return x * 2;
+}
+
+print(add2(sub2(mul2(sub2(5)))));
+//5-2=3
+//3*2=6
+//output:6

tests/japl/comparisons.jpl

@@ -0,0 +1,42 @@
+var x = 4;
+var y = 5;
+var z = 6;
+if (x < y)
+    print("1");//output:1
+else
+    print("2");
+
+if (x == y)
+    print("3");//output:4
+else
+    print("4");
+
+if (x > y)
+    print("5");//output:6
+else if (x < y)
+    print("6");
+
+if (y >= 5)
+    print("7");//output:7
+else
+    print("8");
+
+if (z >= 5)
+    print("9");//output:9
+else
+    print("10");
+
+if (x <= 4)
+    print("11");//output:11
+else
+    print("12");
+
+if (2 <= y)
+    print("13");//output:13
+else
+    print("14");
+
+if (8 <= z)
+    print("15");
+else
+    print("16");//output:16

tests/japl/for_with_function.jpl

@@ -0,0 +1,86 @@
+
+var y = 0; //a global to keep track of state
+//does not need closures for this to work yet
+
+fun next(x) {
+    if (x == 10)
+    {
+        y = y + 1;
+        x = 0;
+    }
+    if (y == 10)
+        return -1;
+    return x+y+1;
+}
+
+var i = 0;
+for (; i != -1; i = next(i))
+    print(i);
+// before using next
+//output:0
+// y = 0
+//output:1
+//output:2
+//output:3
+//output:4
+//output:5
+//output:6
+//output:7
+//output:8
+//output:9
+//output:10
+// y = 1
+//output:2
+//output:3
+//output:4
+//output:5
+//output:6
+//output:7
+//output:8
+//output:9
+//output:10
+// y = 2
+//output:3
+//output:4
+//output:5
+//output:6
+//output:7
+//output:8
+//output:9
+//output:10
+// y = 3
+//output:4
+//output:5
+//output:6
+//output:7
+//output:8
+//output:9
+//output:10
+// y = 4
+//output:5
+//output:6
+//output:7
+//output:8
+//output:9
+//output:10
+// y = 5
+//output:6
+//output:7
+//output:8
+//output:9
+//output:10
+// y = 6
+//output:7
+//output:8
+//output:9
+//output:10
+// y = 7
+//output:8
+//output:9
+//output:10
+// y = 8
+//output:9
+//output:10
+// y = 9
+//output:10
+// y = 10

tests/japl/hello.jpl

@@ -0,0 +1,4 @@
+print("Hello, world.");
+//output:Hello, world.
+
+//output:

tests/japl/hellojapl.jpl

@@ -0,0 +1,4 @@
+print("Hello, JAPL.");
+//output:Hello, JAPL.
+
+//output:

tests/japl/ifchain.jpl

@@ -0,0 +1,22 @@
+fun printInt(x) {
+    if (x == 1)
+        print("one");
+    else if (x == 2)
+        print("two");
+    else if (x == 3)
+        print("three");
+    else if (x == 4)
+        print("four");
+    else if (x == 5)
+        print("five");
+    else if (x == 6)
+        print("six");
+}
+var x = 3;
+printInt(x);//output:three
+x = 5;
+printInt(x);//output:five
+x = 7;
+printInt(x);
+x = 1;
+printInt(x);//output:one

tests/japl/is.jpl

@@ -0,0 +1,22 @@
+var x = 4;
+var y = x;
+
+print(x is y);//output:false
+print(x is x);//output:true
+print(x is 4);//output:false
+
+var z = true;
+var u = true;
+
+print(z is u);//output:true
+print(z is x);//output:false
+print(z is z);//output:true
+
+var l = false;
+print((not l) is z);//output:true
+print(l is z);//output:false
+print((l is z) is l);//output:true
+
+var k;
+print(k is nil);//output:true
+

tests/japl/nan.jpl

@@ -0,0 +1,5 @@
+print((5/0)*0);
+
+//output:nan
+
+//output:

tests/japl/read_in_own_init.jpl

@@ -0,0 +1,4 @@
+var a = 1; { var a = a; }
+//output:A fatal error occurred while compiling '', line 1, at ';' -> cannot read local variable in its own initializer
+
+//output:

tests/japl/reassignment.japl

@@ -0,0 +1,25 @@
+{
+    var x = 5;
+    var y = x;
+    y = 6;
+    print(x);//output:5
+}
+
+var g = 7;
+var p = g;
+{
+    var k = g;
+    p = 3;
+    k = 9;
+    print(g);//output:7
+}
+print(g);//output:7
+
+fun resetter(x) {
+    x = 7;
+    print(x);
+}
+
+var q = 5;
+resetter(q);//output:7
+print(q);//output:5

tests/japl/shadowing.jpl

@@ -0,0 +1,76 @@
+//similar to vars.jpl, but more focused on shadowing
+
+// simple shadowing
+var x = 4;
+{
+    var x = 5;
+    print(x);//output:5
+}
+print(x);//output:4
+
+// type changing shadowing
+var y = true;
+{
+    var y = 2;
+    print(y);//output:2
+}
+print(y);//output:true
+
+// no shadowing here
+var z = 3;
+{
+    z = true;
+    print(z);//output:true
+}
+print(z);//output:true
+
+//in-function shadowing
+fun shadow(x) {
+    //will be called once with the input 3
+    print(x);//output:3
+    {
+        var x = 4;
+        print(x);//output:4
+    }
+    print(x);//output:3
+    x = nil;
+    print(x);//output:nil
+    return x;
+}
+
+print(shadow(3));//output:nil
+
+//shadowing functions
+fun hello() {
+    print("hello");
+}
+hello();//output:hello
+{
+    fun hello() {
+        print("hello in");
+    }
+    hello();//output:hello in
+    {
+        fun hello() {
+            print("hello inmost");
+        }
+        hello();//output:hello inmost
+    }
+    hello();//output:hello in
+}
+hello();//output:hello
+
+//functions shadowing with type change
+fun eat() {
+    print("nom nom nom");
+}
+eat();//output:nom nom nom
+{
+    var eat = 4;
+    print(eat);//output:4
+    {{{{{
+        eat = 5;
+    }}}}} //multiple scopes haha
+    print(eat);//output:5
+}
+eat();//output:nom nom nom

tests/japl/strings.jpl

@@ -0,0 +1,12 @@
+var left = "left";
+var right = "right";
+var directions = left + " " + right;
+print(directions);//output:left right
+
+var longstring = directions * 5;
+print(longstring);//output:left rightleft rightleft rightleft rightleft right
+
+left = left + " side";
+print(left);//output:left side
+right = "side: " + right;
+print(right);//output:side: right

tests/japl/undefname.jpl

@@ -0,0 +1,8 @@
+var a = b;
+//output:Traceback (most recent call last):
+
+//output:  File '', line 1, in '<module>':
+
+//output:ReferenceError: undefined name 'b'
+
+//output:

tests/japl/unsup_binary_intstr.jpl

@@ -0,0 +1,8 @@
+var a = 2 + "hey";
+//output:Traceback (most recent call last):
+
+//output:  File '', line 1, in '<module>':
+
+//output:TypeError: unsupported binary operator '+' for objects of type 'integer' and 'string'
+
+//output:

tests/maketest.nim

@@ -0,0 +1,91 @@
+# Copyright 2020 Mattia Giambirtone
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+# Test creation tool, use mainly for exceptions
+#
+
+
+# Imports nim tests as well
+import multibyte, os, strformat, times, re, terminal, strutils
+
+const tempCodeFile = ".tempcode_drEHdZuwNYLqsQaMDMqeNRtmqoqXBXfnCfeqEcmcUYJToBVQkF.jpl"
+const tempOutputFile = ".tempoutput.txt"
+
+proc autoremove(path: string) =
+    if fileExists(path):
+        removeFile(path)
+
+when isMainModule:
+    var testsDir = "tests" / "japl"
+    var japlExec = "src" / "japl"
+    var currentDir = getCurrentDir()
+    # Supports running from both the project root and the tests dir itself
+    if currentDir.lastPathPart() == "tests":
+        testsDir = "japl"
+        japlExec = ".." / japlExec
+    if not fileExists(japlExec):
+        echo "JAPL executable not found"
+        quit(1)
+    if not dirExists(testsDir):
+        echo "Tests dir not found"
+        quit(1)
+
+    echo "Please enter the JAPL code or specify a file containing it with file:<path>"
+
+    let response = stdin.readLine()
+    if response =~ re"^file:(.*)$":
+        let codepath = matches[0]
+        writeFile(tempCodeFile, readFile(codepath))
+    else:
+        writeFile(tempCodeFile, response)
+
+    let japlCode = readFile(tempCodeFile)
+    discard execShellCmd(&"{japlExec} {tempCodeFile} > {tempOutputFile} 2>&1")
+    var output: string
+    if fileExists(tempOutputFile):
+        output = readFile(tempOutputFile)
+    else:
+        echo "Temporary output file not detected, aborting"
+        quit(1)
+    autoremove(tempCodeFile) 
+    autoremove(tempOutputFile)
+        
+    echo "Got the following output:"
+    echo output                
+    echo "Do you want to keep it as a test? [y/N]"
+    let keepResponse = ($stdin.readLine()).toLower()
+    let keep = keepResponse[0] == 'y'
+    if keep:
+        block saving:
+            while true:
+                echo "Please name the test (without the .jpl extension)"
+                let testname = stdin.readLine()
+                if testname == "":
+                    echo "aborted"
+                    break saving # I like to be explicit
+                let testpath = testsDir / testname & ".jpl"
+                echo &"Generating test at {testpath}"
+                var testContent = japlCode
+                for line in output.split('\n'):
+                    var mline = line
+                    mline = mline.replace(tempCodeFile, "")
+                    testContent = testContent & "\n" & "//output:" & mline & "\n"
+                if fileExists(testpath):
+                    echo "Test already exists"
+                else:
+                    writeFile(testpath, testContent)
+                    break saving
+    else:
+        echo "Aborting"

tests/runtests.nim

@@ -22,11 +22,26 @@
 
 
 # Imports nim tests as well
-import multibyte, os, strformat, times, re
+import multibyte, os, strformat, times, re, terminal, strutils
+
+const tempOutputFile = ".testoutput.txt"
+const testResultsPath = "testresults.txt"
 
 
 # Exceptions for tests that represent not-yet implemented behaviour
-const exceptions = ["all.jpl"]
+const exceptions = ["all.jpl", "for_with_function.jpl"]
+# for_with_function.jpl probably contains an algorithmic error too
+# TODO: fix that test
+
+type LogLevel {.pure.} = enum
+    Debug, # always written to file only (large outputs, such as the entire output of the failing test or stacktrace)
+    Info, # important information about the progress of the test suite
+    Error, # failing tests (printed with red)
+    Stdout, # always printed to stdout only (for cli experience)
+
+
+const echoedLogs = { LogLevel.Info, LogLevel.Error, LogLevel.Stdout }
+const savedLogs = { LogLevel.Debug, LogLevel.Info, LogLevel.Error }
 
 
 proc compileExpectedOutput(path: string): string =
@@ -35,37 +50,52 @@ proc compileExpectedOutput(path: string): string =
             result &= matches[0] & "\n"
 
 
-proc deepComp(left, right: string): tuple[same: bool, place: int] =
+proc deepComp(left, right: string, path: string): tuple[same: bool, place: int] =
+    var mleft, mright: string
     result.same = true
     if left.high() != right.high():
-        result.same = false
-    for i in countup(0, left.high()):
+        if left.replace(path, "").high() == right.replace(path, "").high():
+            mleft = left.replace(path, "")
+            mright = right.replace(path, "")
+        else:
+            result.same = false
+    else:
+        mleft = left
+        mright = right
+    for i in countup(0, mleft.high()):
         result.place = i
-        if i > right.high():
+        if i > mright.high():
             # already false because of the len check at the beginning
             # already correct place because it's updated every i
             return
-        if left[i] != right[i]:
+        if mleft[i] != mright[i]:
             result.same = false
             return
 
 
-# Quick logging levels using procs
+proc logWithLevel(level: LogLevel, file: File, msg: string) =
+    let msg = &"[{$level} - {$getTime()}] {msg}" 
 
-proc log(file: File, msg: string, toFile: bool = true) =
-    ## Logs to stdout and to the log file unless
-    ## toFile == false
-    if toFile:
-        file.writeLine(&"[LOG - {$getTime()}] {msg}")
-    echo &"[LOG - {$getTime()}] {msg}"
+    if level in savedLogs:
+        file.writeLine(msg)
+    if level in echoedLogs:
+        if level == LogLevel.Error:
+            setForegroundColor(fgRed)
+        echo msg
+        if level == LogLevel.Error:
+            setForegroundColor(fgDefault)
 
 
-proc detail(file: File, msg: string) =
-    ## Logs only to the log file
-    file.writeLine(&"[DETAIL - {$getTime()}] {msg}")
 
 
 proc main(testsDir: string, japlExec: string, testResultsFile: File): tuple[numOfTests: int, successTests: int, failedTests: int, skippedTests: int] =
+    template detail(msg: string) =
+        logWithLevel(LogLevel.Debug, testResultsFile, msg)
+    template log(msg: string) =
+        logWithLevel(LogLevel.Info, testResultsFile, msg)
+    template error(msg: string) =
+        logWithLevel(LogLevel.Error, testResultsFile, msg)
+
     var numOfTests = 0
     var successTests = 0
     var failedTests = 0
@@ -74,59 +104,61 @@ proc main(testsDir: string, japlExec: string, testResultsFile: File): tuple[numO
         for file in walkDir(testsDir):
             block singleTest:
                 if file.path.extractFilename in exceptions:
-                    detail(testResultsFile, &"Skipping '{file.path}'")
+                    detail(&"Skipping '{file.path}'")
                     numOfTests += 1
                     skippedTests += 1
                     break singleTest
                 elif file.path.dirExists():
-                    detail(testResultsFile, "Descending into '" & file.path & "'")
+                    detail(&"Descending into '" & file.path & "'")
                     var subTestResult = main(file.path, japlExec, testResultsFile)
                     numOfTests += subTestResult.numOfTests
                     successTests += subTestResult.successTests
                     failedTests += subTestResult.failedTests
                     skippedTests += subTestResult.skippedTests
                     break singleTest
-                detail(testResultsFile, &"Running test '{file.path}'")
-                if fileExists("testoutput.txt"):
-                    removeFile("testoutput.txt") # in case this crashed
-                let retCode = execShellCmd(&"{japlExec} {file.path} >> testoutput.txt")
+                detail(&"Running test '{file.path}'")
+                if fileExists(tempOutputFile):
+                    removeFile(tempOutputFile) # in case this crashed
+                let retCode = execShellCmd(&"{japlExec} {file.path} > {tempOutputFile} 2>&1")
                 numOfTests += 1
                 if retCode != 0:
                     failedTests += 1
-                    log(testResultsFile, &"Test '{file.path}' has crashed!")
+                    error(&"Test '{file.path}' has crashed!")
                 else:
                     let expectedOutput = compileExpectedOutput(file.path).replace(re"(\n*)$", "")
-                    let realOutputFile = open("testoutput.txt", fmRead)
+                    let realOutputFile = open(tempOutputFile, fmRead)
                     let realOutput = realOutputFile.readAll().replace(re"([\n\r]*)$", "")
                     realOutputFile.close()
-                    removeFile("testoutput.txt")
-                    let comparison = deepComp(expectedOutput, realOutput)
+                    removeFile(tempOutputFile)
+                    let comparison = deepComp(expectedOutput, realOutput, file.path)
                     if comparison.same:
                         successTests += 1
-                        log(testResultsFile, &"Test '{file.path}' was successful")
+                        log(&"Test '{file.path}' was successful")
                     else:
                         failedTests += 1
-                        detail(testResultsFile, &"Expected output:\n{expectedOutput}\n")
-                        detail(testResultsFile, &"Received output:\n{realOutput}\n")
-                        detail(testResultsFile, &"Mismatch at pos {comparison.place}")
+                        detail(&"Expected output:\n{expectedOutput}\n")
+                        detail(&"Received output:\n{realOutput}\n")
+                        detail(&"Mismatch at pos {comparison.place}")
                         if comparison.place > expectedOutput.high() or comparison.place > realOutput.high():
-                            detail(testResultsFile, &"Length mismatch")
+                            detail(&"Length mismatch")
                         else:
-                            detail(testResultsFile, &"Expected is '{expectedOutput[comparison.place]}' while received '{realOutput[comparison.place]}'")
-                        log(testResultsFile, &"Test '{file.path}' failed")
+                            detail(&"Expected is '{expectedOutput[comparison.place]}' while received '{realOutput[comparison.place]}'")
+                        error(&"Test '{file.path}' failed")
         result = (numOfTests: numOfTests, successTests: successTests, failedTests: failedTests, skippedTests: skippedTests)
     except IOError:
         stderr.write(&"Fatal IO error encountered while running tests -> {getCurrentExceptionMsg()}")
 
 
 when isMainModule:
-    let testResultsFile = open("testresults.txt", fmWrite)
-    log(testResultsFile, "Running Nim tests")
+    let testResultsFile = open(testResultsPath, fmWrite)
+    template log (msg: string) =
+        logWithLevel(LogLevel.Info, testResultsFile, msg)
+    log("Running Nim tests")
     # Nim tests
-    detail(testResultsFile, "Running testMultiByte")
+    logWithLevel(LogLevel.Debug, testResultsFile, "Running testMultiByte")
     testMultiByte()
     # JAPL tests
-    log(testResultsFile, "Running JAPL tests")
+    log("Running JAPL tests")
     var testsDir = "tests" / "japl"
     var japlExec = "src" / "japl"
     var currentDir = getCurrentDir()
@@ -134,16 +166,16 @@ when isMainModule:
     if currentDir.lastPathPart() == "tests":
         testsDir = "japl"
         japlExec = ".." / japlExec
-    log(testResultsFile, &"Looking for JAPL tests in {testsDir}")
-    log(testResultsFile, &"Looking for JAPL executable at {japlExec}")
+    log(&"Looking for JAPL tests in {testsDir}")
+    log(&"Looking for JAPL executable at {japlExec}")
     if not fileExists(japlExec):
-        log(testResultsFile, "JAPL executable not found")
+        log("JAPL executable not found")
         quit(1)
     if not dirExists(testsDir):
-        log(testResultsFile, "Tests dir not found")
+        log("Tests dir not found")
         quit(1)
     let testResult = main(testsDir, japlExec, testResultsFile)
-    log(testResultsFile, &"Found {testResult.numOfTests} tests: {testResult.successTests} were successful, {testResult.failedTests} failed and {testResult.skippedTests} were skipped.")
-    log(testResultsFile, "Check 'testresults.txt' for details", toFile=false)   
+    log(&"Found {testResult.numOfTests} tests: {testResult.successTests} were successful, {testResult.failedTests} failed and {testResult.skippedTests} were skipped.")
+    logWithLevel(LogLevel.Stdout, testResultsFile, "Check 'testresults.txt' for details")
     testResultsfile.close()