mirror of https://github.com/japl-lang/japl.git
Reorganizing stuff, preparing new test format
This commit is contained in:
Productive2
parent
92348cabd9
commit
4439a24a5b
|
|
@ -1,5 +0,0 @@
|
|||
# Constants for communication between the test runner and the test suite
|
||||
const nextTestPre = "\rNEXTTEST{"
|
||||
const nextTestPost = "}"
|
||||
const nextTestRe = "^\rNEXTTEST{.*}$"
|
||||
const nextTestNameRe = "{(.*)}$"
|
||||
|
|
@ -17,27 +17,17 @@
|
|||
# a testrunner process
|
||||
|
||||
import ../src/vm
|
||||
import directives
|
||||
import os
|
||||
import strformat
|
||||
|
||||
|
||||
var btvm = initVM()
|
||||
if paramCount() > 0 and paramStr(1) == "stdin":
|
||||
block main:
|
||||
while true:
|
||||
block test:
|
||||
var test = ""
|
||||
while true:
|
||||
let nl = stdin.readLine()
|
||||
if
|
||||
|
||||
elif paramCount() > 0:
|
||||
try:
|
||||
discard btvm.interpret(readFile(paramStr(1)), "")
|
||||
quit(0)
|
||||
except:
|
||||
let error = getCurrentException()
|
||||
writeLine stderr, error.msg
|
||||
quit(1)
|
||||
try:
|
||||
discard btvm.interpret(stdin.readAll(), "")
|
||||
quit(0)
|
||||
except:
|
||||
let error = getCurrentException()
|
||||
writeLine stderr, error.msg
|
||||
quit(1)
|
||||
|
||||
|
|
|
|||
|
|
@ -23,8 +23,8 @@ import strutils
|
|||
type LogLevel* {.pure.} = enum
|
||||
Debug, # always written to file only (large outputs, such as the entire output of the failing test or stacktrace)
|
||||
Info, # important information about the progress of the test suite
|
||||
Error, # failing tests (printed with red)
|
||||
Stdout, # always printed to stdout only (for cli experience)
|
||||
Error, # failing tests (printed with yellow)
|
||||
Fatal # always printed with red, halts the entire suite (test parsing errors, printed with red)
|
||||
|
||||
# don't move this to testglobals/testconfig
|
||||
const echoedLogs = {LogLevel.Info, LogLevel.Error, LogLevel.Stdout}
|
||||
|
|
@ -32,7 +32,7 @@ const echoedLogsSilent = {LogLevel.Error}
|
|||
const savedLogs = {LogLevel.Debug, LogLevel.Info, LogLevel.Error}
|
||||
const progbarLength = 25
|
||||
|
||||
const logColors = [LogLevel.Debug: fgDefault, LogLevel.Info: fgGreen, LogLevel.Error: fgRed, LogLevel.Stdout: fgYellow]
|
||||
const logColors = [LogLevel.Debug: fgDefault, LogLevel.Info: fgGreen, LogLevel.Error: fgYellow, LogLevel.Fatal: fgRed]
|
||||
|
||||
var totalLog = ""
|
||||
var verbose = true
|
||||
|
|
@ -55,6 +55,10 @@ proc log*(level: LogLevel, msg: string) =
|
|||
echo msg
|
||||
setForegroundColor(fgDefault)
|
||||
|
||||
proc fatal*(msg: string) =
|
||||
log(LogLevel.Fatal, msg)
|
||||
raise newException(CatchableError, msg)
|
||||
|
||||
proc getTotalLog*: string =
|
||||
totalLog
|
||||
|
||||
|
|
|
|||
|
|
@ -12,40 +12,13 @@
|
|||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
# Test object helpers
|
||||
# Test runner supervisor/manager
|
||||
|
||||
import testobject
|
||||
import logutils
|
||||
|
||||
import os
|
||||
import osproc
|
||||
import streams
|
||||
import strformat
|
||||
import testconfig
|
||||
|
||||
|
||||
|
||||
proc buildTest(path: string): Test =
|
||||
log(LogLevel.Debug, &"Building test {path}")
|
||||
let source = readFile(path)
|
||||
result = Test(
|
||||
path: path,
|
||||
result: if path.extractFilename in exceptions: TestResult.Skip
|
||||
else: TestResult.Unstarted,
|
||||
expectedOutput: compileExpectedOutput(source),
|
||||
expectedError: compileExpectedError(source),
|
||||
input: compileInput(source)
|
||||
)
|
||||
|
||||
|
||||
proc buildTests*(testDir: string): seq[Test] =
|
||||
for candidateObj in walkDir(testDir):
|
||||
let candidate = candidateObj.path
|
||||
if dirExists(candidate):
|
||||
log(LogLevel.Debug, &"Descending into dir {candidate}")
|
||||
result &= buildTests(candidate)
|
||||
else:
|
||||
result.add buildTest(candidate)
|
||||
import strformat
|
||||
|
||||
|
||||
proc runTest(test: Test, runner: string) =
|
||||
|
|
@ -130,47 +103,3 @@ proc runTests*(tests: seq[Test], runner: string) =
|
|||
buffer.render()
|
||||
buffer.endBuffer()
|
||||
|
||||
|
||||
proc evalTest(test: Test) =
|
||||
test.output = test.output.tuStrip()
|
||||
test.error = test.error.tuStrip()
|
||||
test.expectedOutput = test.expectedOutput.tuStrip()
|
||||
test.expectedError = test.expectedError.tuStrip()
|
||||
if test.output != test.expectedOutput or test.error != test.expectedError:
|
||||
test.result = TestResult.Mismatch
|
||||
else:
|
||||
test.result = TestResult.Success
|
||||
|
||||
|
||||
proc evalTests*(tests: seq[Test]) =
|
||||
for test in tests:
|
||||
if test.result == TestResult.ToEval:
|
||||
evalTest(test)
|
||||
|
||||
|
||||
proc printResults*(tests: seq[Test]): bool =
|
||||
var
|
||||
skipped = 0
|
||||
success = 0
|
||||
fail = 0
|
||||
crash = 0
|
||||
for test in tests:
|
||||
log(LogLevel.Debug, &"Test {test.path} result: {test.result}")
|
||||
case test.result:
|
||||
of TestResult.Skip:
|
||||
inc skipped
|
||||
of TestResult.Mismatch:
|
||||
inc fail
|
||||
log(LogLevel.Debug, &"[{test.path}\noutput:\n{test.output}\nerror:\n{test.error}\nexpected output:\n{test.expectedOutput}\nexpectedError:\n{test.expectedError}\n]")
|
||||
of TestResult.Crash:
|
||||
inc crash
|
||||
log(LogLevel.Debug, &"{test.path} \ncrash:\n{test.output}")
|
||||
of TestResult.Success:
|
||||
inc success
|
||||
else:
|
||||
log(LogLevel.Error, &"Probably a testing suite bug: test {test.path} has result {test.result}")
|
||||
let finalLevel = if fail == 0 and crash == 0: LogLevel.Info else: LogLevel.Error
|
||||
log(finalLevel, &"{tests.len()} tests: {success} succeeded, {skipped} skipped, {fail} failed, {crash} crashed.")
|
||||
result = fail == 0 and crash == 0
|
||||
|
||||
|
||||
|
|
@ -0,0 +1,102 @@
|
|||
# Copyright 2020 Mattia Giambirtone
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
import testobject
|
||||
import logutils
|
||||
import testconfig
|
||||
|
||||
import os
|
||||
import strutils
|
||||
import re
|
||||
|
||||
proc buildTest(lines: seq[string], i: var int, name: string, path: string): Test =
|
||||
result = newTest(name, path)
|
||||
# since this is a very simple parser, some state can reduce code length
|
||||
var mode: string
|
||||
var modedetail: string
|
||||
var inside: bool
|
||||
var body: string
|
||||
while i < lines.len():
|
||||
let line = lines[i]
|
||||
if line =~ re"^[ \t]*\[[ \t]*(.*)[ \t]*\][ \t]*$":
|
||||
let content = matches[0]
|
||||
let parts = content.split(':').map(strip)
|
||||
if inside:
|
||||
if parts[0] == "end":
|
||||
# end inside
|
||||
if mode == "source" and (modedetail == "" or modedetail == "mixed"):
|
||||
result.parseMixed(body)
|
||||
elif mode == "source" and modedetail == "raw":
|
||||
result.parseSource(body)
|
||||
elif mode == "stdout" and (modedetail == ""):
|
||||
result.parseStdout(body)
|
||||
elif mode == "stdoutre" or (mode == "stdout" and modedetail == "re"):
|
||||
result.parseStdout(body, true)
|
||||
elif mode == "stderr" and (modedetail == ""):
|
||||
result.parseStderr(body)
|
||||
elif mode == "stderrre" or (mode == "stderr" and modedetail == "re"):
|
||||
result.parseStderr(body, true)
|
||||
elif modedetail != "":
|
||||
fatal &"Invalid mode detail {modedetail} for mode {mode} in test {name} at {path}."
|
||||
# non-modedetail modes below:
|
||||
elif mode == "stdin":
|
||||
result.parseStdin(body)
|
||||
elif mode == "python":
|
||||
result.parsePython(body)
|
||||
elif mode == "comment":
|
||||
discard # just a comment
|
||||
else:
|
||||
fatal &"Invalid mode {mode} for test {name} at {path}."
|
||||
inside = false
|
||||
mode = ""
|
||||
modedetail = ""
|
||||
body = ""
|
||||
else:
|
||||
if parts[0] == "skip":
|
||||
result.skip()
|
||||
else:
|
||||
inside = true
|
||||
mode = parts[0]
|
||||
if parts.len() > 1:
|
||||
modedetail = parts[1]
|
||||
else:
|
||||
modedetail = ""
|
||||
elif line =~ re"^[ \t]*$":
|
||||
discard # nothing interesting
|
||||
elif inside:
|
||||
body &= line & "\n"
|
||||
else:
|
||||
# invalid
|
||||
fatal &"Invalid test code: {line} in test {name} at {path}"
|
||||
|
||||
proc buildTestFile(path: string): seq[Test] =
|
||||
log(LogLevel.Debug, &"Building test {path}")
|
||||
let lines = path.split('\n')
|
||||
var i = 0
|
||||
while i < lines.len():
|
||||
let line = lines[i]
|
||||
if line =~ re"\[Test:[ \t]*(.*)[ \t*]\]":
|
||||
let testname = matches[0]
|
||||
result.add buildTest(lines, i, testname, path)
|
||||
|
||||
proc buildTests*(testDir: string): seq[Test] =
|
||||
for candidateObj in walkDir(testDir):
|
||||
let candidate = candidateObj.path
|
||||
if dirExists(candidate):
|
||||
log(LogLevel.Debug, &"Descending into dir {candidate}")
|
||||
result &= buildTests(candidate)
|
||||
else:
|
||||
result &= buildTestFile(candidate)
|
||||
|
||||
|
||||
|
|
@ -0,0 +1,70 @@
|
|||
# Copyright 2020 Mattia Giambirtone
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
# Test object helpers
|
||||
|
||||
import testobject
|
||||
import logutils
|
||||
|
||||
import os
|
||||
import osproc
|
||||
import streams
|
||||
import strformat
|
||||
import testconfig
|
||||
|
||||
|
||||
|
||||
proc evalTest(test: Test) =
|
||||
test.output = test.output.tuStrip()
|
||||
test.error = test.error.tuStrip()
|
||||
test.expectedOutput = test.expectedOutput.tuStrip()
|
||||
test.expectedError = test.expectedError.tuStrip()
|
||||
if test.output != test.expectedOutput or test.error != test.expectedError:
|
||||
test.result = TestResult.Mismatch
|
||||
else:
|
||||
test.result = TestResult.Success
|
||||
|
||||
|
||||
proc evalTests*(tests: seq[Test]) =
|
||||
for test in tests:
|
||||
if test.result == TestResult.ToEval:
|
||||
evalTest(test)
|
||||
|
||||
|
||||
proc printResults*(tests: seq[Test]): bool =
|
||||
var
|
||||
skipped = 0
|
||||
success = 0
|
||||
fail = 0
|
||||
crash = 0
|
||||
for test in tests:
|
||||
log(LogLevel.Debug, &"Test {test.path} result: {test.result}")
|
||||
case test.result:
|
||||
of TestResult.Skip:
|
||||
inc skipped
|
||||
of TestResult.Mismatch:
|
||||
inc fail
|
||||
log(LogLevel.Debug, &"[{test.path}\noutput:\n{test.output}\nerror:\n{test.error}\nexpected output:\n{test.expectedOutput}\nexpectedError:\n{test.expectedError}\n]")
|
||||
of TestResult.Crash:
|
||||
inc crash
|
||||
log(LogLevel.Debug, &"{test.path} \ncrash:\n{test.output}")
|
||||
of TestResult.Success:
|
||||
inc success
|
||||
else:
|
||||
log(LogLevel.Error, &"Probably a testing suite bug: test {test.path} has result {test.result}")
|
||||
let finalLevel = if fail == 0 and crash == 0: LogLevel.Info else: LogLevel.Error
|
||||
log(finalLevel, &"{tests.len()} tests: {success} succeeded, {skipped} skipped, {fail} failed, {crash} crashed.")
|
||||
result = fail == 0 and crash == 0
|
||||
|
||||
|
||||
|
|
@ -22,38 +22,91 @@ type
|
|||
TestResult* {.pure.} = enum
|
||||
Unstarted, Running, ToEval, Success, Skip, Mismatch, Crash
|
||||
|
||||
ExpectedLineKind* {.pure.} = enum
|
||||
Raw, Regex
|
||||
|
||||
ExpectedLine* = object
|
||||
kind*: ExpectedLineKind
|
||||
content*: string
|
||||
|
||||
Test* = ref object
|
||||
result*: TestResult
|
||||
# test origins
|
||||
source*: string
|
||||
path*: string
|
||||
expectedOutput*: string
|
||||
expectedError*: string
|
||||
name*: string
|
||||
# generated after building
|
||||
expectedOutput*: seq[ExpectedLine]
|
||||
expectedError*: seq[ExpectedLine]
|
||||
input*: string
|
||||
# during running/output of running
|
||||
output*: string
|
||||
error*: string
|
||||
process*: Process
|
||||
cycles*: int
|
||||
# after evaluation
|
||||
result*: TestResult
|
||||
|
||||
# parsing the test notation
|
||||
# Helpers for building tests:
|
||||
|
||||
proc compileExpectedOutput*(source: string): string =
|
||||
proc genEL(content: string, kind: ExpectedLineKind): ExpectedLine =
|
||||
ExpectedLine(kind: kind, content: content)
|
||||
|
||||
proc compileExpectedOutput(source: string, rawkw: string, rekw: string): seq[ExpectedLine] =
|
||||
for line in source.split('\n'):
|
||||
if line =~ re"^.*//stdout:[ ]?(.*)$":
|
||||
result &= matches[0] & "\n"
|
||||
if line =~ re("^.*//" & rawkw & ":[ ]?(.*)$"):
|
||||
result &= genEL(matches[0], ExpectedLineKind.Raw)
|
||||
elif line =~ re("^.*//" & rekw & ":[ ]?(.*$"):
|
||||
result &= genEL(matches[0], ExpectedLineKind.Regex)
|
||||
|
||||
proc compileExpectedOutput(source: string): seq[ExpectedLine] =
|
||||
compileExpectedOutput(source, "stdout", "stdoutre")
|
||||
|
||||
proc compileExpectedError*(source: string): string =
|
||||
for line in source.split('\n'):
|
||||
if line =~ re"^.*//stderr:[ ]?(.*)$":
|
||||
result &= matches[0] & "\n"
|
||||
proc compileExpectedError(source: string): seq[ExpectedLine] =
|
||||
compileExpectedOutput(source, "stderr", "stderrre")
|
||||
|
||||
proc compileInput*(source: string): string =
|
||||
proc compileInput(source: string): string =
|
||||
for line in source.split('\n'):
|
||||
if line =~ re"^.*//stdin:[ ]?(.*)$":
|
||||
result &= matches[0] & "\n"
|
||||
|
||||
proc parseMixed*(test: Test, source: string) =
|
||||
test.source &= source
|
||||
test.expectedOutput = compileExpectedOutput(source)
|
||||
test.expectedError = compileExpectedError(source)
|
||||
test.input = compileInput(source)
|
||||
test.result = TestResult.Unstarted
|
||||
|
||||
# stuff for cleaning test output
|
||||
proc parseSource*(test: Test, source: string) =
|
||||
test.source &= source
|
||||
|
||||
proc parseStdin*(test: Test, source: string) =
|
||||
test.input &= source
|
||||
|
||||
proc parseStdout*(test: Test, source: string, regex: bool = false, stderr: bool = false) =
|
||||
var kind: ExpectedLineKind.Raw
|
||||
if regex:
|
||||
kind = ExpectedLineKind.Regex
|
||||
for line in source.split('\n'):
|
||||
if stderr:
|
||||
test.expectedError.add(genEL(line, kind))
|
||||
else:
|
||||
test.expectedOutput.add(genEL(line, kind))
|
||||
|
||||
proc parseStderr*(test: Test, source: string, regex: bool = false) =
|
||||
parseStdout(test, source, regex, true)
|
||||
|
||||
proc parsePython*(test: Test, source: string) =
|
||||
discard # TODO
|
||||
|
||||
proc newTest*(name: string, path: string): Test =
|
||||
result.path = path
|
||||
result.name = name
|
||||
|
||||
proc skip*(test: Test) =
|
||||
test.result = TestResult.Skip
|
||||
|
||||
# Helpers for running tests
|
||||
|
||||
proc tuStrip*(input: string): string =
|
||||
return input.replace(re"\[DEBUG.*\n","\n").replace(re"[\n\r]*", "\n").replace(re"\n$","")
|
||||
return input.replace(re"\[DEBUG.*\n","\n").strip()
|
||||
|
||||
|
|
|
|||
Loading…
Reference in New Issue