japl/tests/runtests.nim

# Copyright 2020 Mattia Giambirtone
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.



# Common entry point to run JAPL's tests
#
# - Assumes "japl" binary in ../src/japl built with all debugging off
# - Goes through all tests in (/tests/)
# - Runs all tests in (/tests/)japl/ and checks their output (marked by `//output:{output}`)


# Imports nim tests as well
import multibyte

import os, strformat, times, re, terminal, strutils

const tempOutputFile = ".testoutput.txt"
const testResultsPath = "testresults.txt"


# Exceptions for tests that represent not-yet implemented behaviour
const exceptions = ["all.jpl", "for_with_function.jpl", "runtime_interning.jpl", "problem4.jpl"]
# for_with_function.jpl probably contains an algorithmic error too
# TODO: fix that test

type LogLevel {.pure.} = enum
    Debug, # always written to file only (large outputs, such as the entire output of the failing test or stacktrace)
    Info, # important information about the progress of the test suite
    Error, # failing tests (printed with red)
    Stdout, # always printed to stdout only (for cli experience)


const echoedLogs = {LogLevel.Info, LogLevel.Error, LogLevel.Stdout}
const savedLogs = {LogLevel.Debug, LogLevel.Info, LogLevel.Error}


proc compileExpectedOutput(path: string): string =
    for line in path.lines():
        if line =~ re"^.*//output:(.*)$":
            result &= matches[0] & "\n"


proc deepComp(left, right: string, path: string): tuple[same: bool, place: int] =
    var mleft, mright: string
    result.same = true
    if left.high() != right.high():
        if left.replace(path, "").high() == right.replace(path, "").high():
            mleft = left.replace(path, "")
            mright = right.replace(path, "")
        else:
            result.same = false
    else:
        mleft = left
        mright = right
    for i in countup(0, mleft.high()):
        result.place = i
        if i > mright.high():
            # already false because of the len check at the beginning
            # already correct place because it's updated every i
            return
        if mleft[i] != mright[i]:
            result.same = false
            return


proc logWithLevel(level: LogLevel, file: File, msg: string) =
    let msg = &"[{$level} - {$getTime()}] {msg}"
    if level in savedLogs:
        file.writeLine(msg)
    if level in echoedLogs:
        if level == LogLevel.Error:
            setForegroundColor(fgRed)
        elif level == LogLevel.Info:
            setForegroundColor(fgGreen)
        elif level == LogLevel.Stdout:
            setForegroundColor(fgYellow)
        echo msg
        setForegroundColor(fgDefault)


proc main(testsDir: string, japlExec: string, testResultsFile: File): tuple[numOfTests: int, successTests: int, failedTests: int, skippedTests: int] =
    template detail(msg: string) =
        logWithLevel(LogLevel.Debug, testResultsFile, msg)
    template log(msg: string) =
        logWithLevel(LogLevel.Info, testResultsFile, msg)
    template error(msg: string) =
        logWithLevel(LogLevel.Error, testResultsFile, msg)
    var numOfTests = 0
    var successTests = 0
    var failedTests = 0
    var skippedTests = 0
    try:
        for file in walkDir(testsDir):
            block singleTest:
                if file.path.extractFilename in exceptions:
                    detail(&"Skipping '{file.path}'")
                    numOfTests += 1
                    skippedTests += 1
                    break singleTest
                elif file.path.dirExists():
                    detail(&"Descending into '" & file.path & "'")
                    var subTestResult = main(file.path, japlExec, testResultsFile)
                    numOfTests += subTestResult.numOfTests
                    successTests += subTestResult.successTests
                    failedTests += subTestResult.failedTests
                    skippedTests += subTestResult.skippedTests
                    break singleTest
                detail(&"Running test '{file.path}'")
                if fileExists(tempOutputFile):
                    removeFile(tempOutputFile) # in case this crashed
                let retCode = execShellCmd(&"{japlExec} {file.path} > {tempOutputFile} 2>&1")
                numOfTests += 1
                if retCode != 0:
                    failedTests += 1
                    error(&"Test '{file.path}' has crashed!")
                else:
                    let expectedOutput = compileExpectedOutput(file.path).replace(re"(\n*)$", "")
                    let realOutputFile = open(tempOutputFile, fmRead)
                    let realOutput = realOutputFile.readAll().replace(re"([\n\r]*)$", "")
                    realOutputFile.close()
                    removeFile(tempOutputFile)
                    let comparison = deepComp(expectedOutput, realOutput, file.path)
                    if comparison.same:
                        successTests += 1
                        log(&"Test '{file.path}' was successful")
                    else:
                        failedTests += 1
                        detail(&"Expected output:\n{expectedOutput}\n")
                        detail(&"Received output:\n{realOutput}\n")
                        detail(&"Mismatch at pos {comparison.place}")
                        if comparison.place > expectedOutput.high() or comparison.place > realOutput.high():
                            detail(&"Length mismatch")
                        else:
                            detail(&"Expected is '{expectedOutput[comparison.place]}' while received '{realOutput[comparison.place]}'")
                        error(&"Test '{file.path}' failed")
        result = (numOfTests: numOfTests, successTests: successTests, failedTests: failedTests, skippedTests: skippedTests)
    except IOError:
        stderr.write(&"Fatal IO error encountered while running tests -> {getCurrentExceptionMsg()}")


when isMainModule:
    let testResultsFile = open(testResultsPath, fmWrite)
    template log (msg: string) =
        logWithLevel(LogLevel.Stdout, testResultsFile, msg)
    log("Running Nim tests")
    # Nim tests
    logWithLevel(LogLevel.Debug, testResultsFile, "Running testMultiByte")
    testMultiByte()
    # JAPL tests
    log("Running JAPL tests")
    var testsDir = "tests" / "japl"
    var japlExec = "src" / "japl"
    var currentDir = getCurrentDir()
    # Supports running from both the project root and the tests dir itself
    if currentDir.lastPathPart() == "tests":
        testsDir = "japl"
        japlExec = ".." / japlExec
    log(&"Looking for JAPL tests in {testsDir}")
    log(&"Looking for JAPL executable at {japlExec}")
    if not fileExists(japlExec):
        log("JAPL executable not found")
        quit(1)
    if not dirExists(testsDir):
        log("Tests dir not found")
        quit(1)
    let testResult = main(testsDir, japlExec, testResultsFile)
    log(&"Found {testResult.numOfTests} tests: {testResult.successTests} were successful, {testResult.failedTests} failed and {testResult.skippedTests} were skipped.")
    logWithLevel(LogLevel.Stdout, testResultsFile, "Check 'testresults.txt' for details")
    testResultsfile.close()