2021-01-05 16:10:28 +01:00
# Copyright 2020 Mattia Giambirtone
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
2021-01-05 12:21:23 +01:00
# Common entry point to run JAPL's tests
#
# - Assumes "japl" binary in ../src/japl built with all debugging off
# - Goes through all tests in (/tests/)
# - Runs all tests in (/tests/)japl/ and checks their output (marked by `//output:{output}`)
2021-01-10 08:24:34 +01:00
2021-01-05 12:21:23 +01:00
2021-01-05 16:10:28 +01:00
# Imports nim tests as well
2021-01-16 18:14:22 +01:00
import multibyte
import os , strformat , times , re , terminal , strutils
2021-01-10 20:33:29 +01:00
const tempOutputFile = " .testoutput.txt "
const testResultsPath = " testresults.txt "
2021-01-05 12:21:23 +01:00
2021-01-09 18:03:47 +01:00
# Exceptions for tests that represent not-yet implemented behaviour
2021-01-16 18:14:22 +01:00
const exceptions = [ " all.jpl " , " for_with_function.jpl " , " runtime_interning.jpl " , " problem4.jpl " ]
2021-01-10 19:44:00 +01:00
# for_with_function.jpl probably contains an algorithmic error too
# TODO: fix that test
2021-01-09 18:03:47 +01:00
2021-01-10 19:32:28 +01:00
type LogLevel {. pure . } = enum
Debug , # always written to file only (large outputs, such as the entire output of the failing test or stacktrace)
Info , # important information about the progress of the test suite
Error , # failing tests (printed with red)
Stdout , # always printed to stdout only (for cli experience)
2021-01-14 22:37:11 +01:00
const echoedLogs = { LogLevel . Info , LogLevel . Error , LogLevel . Stdout }
const savedLogs = { LogLevel . Debug , LogLevel . Info , LogLevel . Error }
2021-01-10 19:32:28 +01:00
2021-01-09 18:03:47 +01:00
2021-01-05 12:21:23 +01:00
proc compileExpectedOutput ( path : string ) : string =
for line in path . lines ( ) :
if line = ~ re" ^.*//output:(.*) $ " :
result & = matches [ 0 ] & " \n "
2021-01-05 16:10:28 +01:00
2021-01-10 20:33:29 +01:00
proc deepComp ( left , right : string , path : string ) : tuple [ same : bool , place : int ] =
var mleft , mright : string
2021-01-05 12:52:41 +01:00
result . same = true
if left . high ( ) ! = right . high ( ) :
2021-01-10 20:33:29 +01:00
if left . replace ( path , " " ) . high ( ) = = right . replace ( path , " " ) . high ( ) :
mleft = left . replace ( path , " " )
mright = right . replace ( path , " " )
else :
result . same = false
else :
mleft = left
mright = right
for i in countup ( 0 , mleft . high ( ) ) :
2021-01-05 12:52:41 +01:00
result . place = i
2021-01-10 20:33:29 +01:00
if i > mright . high ( ) :
2021-01-09 18:03:47 +01:00
# already false because of the len check at the beginning
# already correct place because it's updated every i
2021-01-05 12:52:41 +01:00
return
2021-01-10 20:33:29 +01:00
if mleft [ i ] ! = mright [ i ] :
2021-01-05 12:52:41 +01:00
result . same = false
return
2021-01-10 19:32:28 +01:00
proc logWithLevel ( level : LogLevel , file : File , msg : string ) =
2021-01-18 11:05:37 +01:00
let msg = & " [{ $level } - { $getTime ()}] {msg} "
2021-01-10 19:32:28 +01:00
if level in savedLogs :
file . writeLine ( msg )
if level in echoedLogs :
if level = = LogLevel . Error :
setForegroundColor ( fgRed )
2021-01-18 11:05:37 +01:00
elif level = = LogLevel . Info :
setForegroundColor ( fgGreen )
elif level = = LogLevel . Stdout :
setForegroundColor ( fgYellow )
2021-01-10 19:32:28 +01:00
echo msg
2021-01-18 11:05:37 +01:00
setForegroundColor ( fgDefault )
2021-01-05 12:21:23 +01:00
2021-01-09 18:03:47 +01:00
proc main ( testsDir : string , japlExec : string , testResultsFile : File ) : tuple [ numOfTests : int , successTests : int , failedTests : int , skippedTests : int ] =
2021-01-10 19:32:28 +01:00
template detail ( msg : string ) =
logWithLevel ( LogLevel . Debug , testResultsFile , msg )
template log ( msg : string ) =
logWithLevel ( LogLevel . Info , testResultsFile , msg )
template error ( msg : string ) =
logWithLevel ( LogLevel . Error , testResultsFile , msg )
2021-01-09 18:03:47 +01:00
var numOfTests = 0
var successTests = 0
var failedTests = 0
var skippedTests = 0
2021-01-05 16:10:28 +01:00
try :
for file in walkDir ( testsDir ) :
block singleTest :
2021-01-10 08:24:34 +01:00
if file . path . extractFilename in exceptions :
2021-01-10 19:32:28 +01:00
detail ( & " Skipping ' {file.path} ' " )
2021-01-10 08:24:34 +01:00
numOfTests + = 1
skippedTests + = 1
break singleTest
elif file . path . dirExists ( ) :
2021-01-10 19:32:28 +01:00
detail ( & " Descending into ' " & file . path & " ' " )
2021-01-09 18:03:47 +01:00
var subTestResult = main ( file . path , japlExec , testResultsFile )
numOfTests + = subTestResult . numOfTests
successTests + = subTestResult . successTests
failedTests + = subTestResult . failedTests
skippedTests + = subTestResult . skippedTests
2021-01-05 20:51:05 +01:00
break singleTest
2021-01-10 19:32:28 +01:00
detail ( & " Running test ' {file.path} ' " )
2021-01-10 20:33:29 +01:00
if fileExists ( tempOutputFile ) :
removeFile ( tempOutputFile ) # in case this crashed
let retCode = execShellCmd ( & " {japlExec} {file.path} > {tempOutputFile} 2>&1 " )
2021-01-09 18:03:47 +01:00
numOfTests + = 1
if retCode ! = 0 :
failedTests + = 1
2021-01-10 19:32:28 +01:00
error ( & " Test ' {file.path} ' has crashed! " )
2021-01-05 16:10:28 +01:00
else :
2021-01-09 18:03:47 +01:00
let expectedOutput = compileExpectedOutput ( file . path ) . replace ( re" ( \ n*) $ " , " " )
2021-01-10 20:33:29 +01:00
let realOutputFile = open ( tempOutputFile , fmRead )
2021-01-09 18:03:47 +01:00
let realOutput = realOutputFile . readAll ( ) . replace ( re" ([ \ n \ r]*) $ " , " " )
realOutputFile . close ( )
2021-01-10 20:33:29 +01:00
removeFile ( tempOutputFile )
let comparison = deepComp ( expectedOutput , realOutput , file . path )
2021-01-09 18:03:47 +01:00
if comparison . same :
2021-01-10 08:24:34 +01:00
successTests + = 1
2021-01-10 19:32:28 +01:00
log ( & " Test ' {file.path} ' was successful " )
2021-01-05 16:10:28 +01:00
else :
2021-01-10 08:24:34 +01:00
failedTests + = 1
2021-01-10 19:32:28 +01:00
detail ( & " Expected output: \n {expectedOutput} \n " )
detail ( & " Received output: \n {realOutput} \n " )
detail ( & " Mismatch at pos {comparison.place} " )
2021-01-10 08:24:34 +01:00
if comparison . place > expectedOutput . high ( ) or comparison . place > realOutput . high ( ) :
2021-01-10 19:32:28 +01:00
detail ( & " Length mismatch " )
2021-01-09 18:03:47 +01:00
else :
2021-01-10 19:32:28 +01:00
detail ( & " Expected is ' {expectedOutput[comparison.place]} ' while received ' {realOutput[comparison.place]} ' " )
error ( & " Test ' {file.path} ' failed " )
2021-01-09 18:03:47 +01:00
result = ( numOfTests : numOfTests , successTests : successTests , failedTests : failedTests , skippedTests : skippedTests )
2021-01-05 16:10:28 +01:00
except IOError :
2021-01-09 18:03:47 +01:00
stderr . write ( & " Fatal IO error encountered while running tests -> {getCurrentExceptionMsg()} " )
2021-01-05 20:51:05 +01:00
2021-01-09 18:03:47 +01:00
when isMainModule :
2021-01-10 20:33:29 +01:00
let testResultsFile = open ( testResultsPath , fmWrite )
2021-01-10 19:32:28 +01:00
template log ( msg : string ) =
2021-01-18 11:05:37 +01:00
logWithLevel ( LogLevel . Stdout , testResultsFile , msg )
2021-01-10 19:32:28 +01:00
log ( " Running Nim tests " )
2021-01-09 18:03:47 +01:00
# Nim tests
2021-01-10 19:32:28 +01:00
logWithLevel ( LogLevel . Debug , testResultsFile , " Running testMultiByte " )
2021-01-09 18:03:47 +01:00
testMultiByte ( )
# JAPL tests
2021-01-10 19:32:28 +01:00
log ( " Running JAPL tests " )
2021-01-05 20:51:05 +01:00
var testsDir = " tests " / " japl "
var japlExec = " src " / " japl "
var currentDir = getCurrentDir ( )
2021-01-09 18:03:47 +01:00
# Supports running from both the project root and the tests dir itself
2021-01-05 20:51:05 +01:00
if currentDir . lastPathPart ( ) = = " tests " :
testsDir = " japl "
japlExec = " .. " / japlExec
2021-01-10 19:32:28 +01:00
log ( & " Looking for JAPL tests in {testsDir} " )
log ( & " Looking for JAPL executable at {japlExec} " )
2021-01-05 20:51:05 +01:00
if not fileExists ( japlExec ) :
2021-01-10 19:32:28 +01:00
log ( " JAPL executable not found " )
2021-01-05 20:51:05 +01:00
quit ( 1 )
if not dirExists ( testsDir ) :
2021-01-10 19:32:28 +01:00
log ( " Tests dir not found " )
2021-01-05 20:51:05 +01:00
quit ( 1 )
2021-01-09 18:03:47 +01:00
let testResult = main ( testsDir , japlExec , testResultsFile )
2021-01-10 19:32:28 +01:00
log ( & " Found {testResult.numOfTests} tests: {testResult.successTests} were successful, {testResult.failedTests} failed and {testResult.skippedTests} were skipped. " )
logWithLevel ( LogLevel . Stdout , testResultsFile , " Check ' testresults.txt ' for details " )
2021-01-05 20:51:05 +01:00
testResultsfile . close ( )
2021-01-09 18:03:47 +01:00