mirror of https://github.com/japl-lang/japl.git
Finished moving around code for better modularity
Added testobject, repurposed testutils
This commit is contained in:
Productive2
parent
e2b45dd1dc
commit
bc2a2ac74d
121
tests/jats.nim
121
tests/jats.nim
|
|
@ -16,129 +16,10 @@
|
|||
|
||||
import nim/nimtests
|
||||
|
||||
import testutils, logutils
|
||||
import testobject, testutils, logutils
|
||||
|
||||
import os, osproc, strformat, streams, parseopt, strutils
|
||||
|
||||
# Tests that represent not-yet implemented behaviour
|
||||
const exceptions = ["all.jpl", "for_with_function.jpl", "runtime_interning.jpl", "problem4.jpl"]
|
||||
# TODO: for_with_function.jpl and problem4.jpl should already be implemented, check on them
|
||||
|
||||
proc buildTest(path: string): Test =
|
||||
log(LogLevel.Debug, &"Building test {path}")
|
||||
let source = readFile(path)
|
||||
result = Test(
|
||||
path: path,
|
||||
result: if path.extractFilename in exceptions: TestResult.Skip
|
||||
else: TestResult.Unstarted,
|
||||
expectedOutput: compileExpectedOutput(source),
|
||||
expectedError: compileExpectedError(source)
|
||||
)
|
||||
|
||||
proc buildTests(testDir: string): seq[Test] =
|
||||
for candidateObj in walkDir(testDir):
|
||||
let candidate = candidateObj.path
|
||||
if dirExists(candidate):
|
||||
log(LogLevel.Debug, &"Descending into dir {candidate}")
|
||||
result &= buildTests(candidate)
|
||||
else:
|
||||
result.add buildTest(candidate)
|
||||
|
||||
proc runTest(test: Test, runner: string) =
|
||||
log(LogLevel.Debug, &"Starting test {test.path}.")
|
||||
let process = startProcess(runner, args = @[test.path])
|
||||
test.process = process
|
||||
test.result = TestResult.Running
|
||||
|
||||
proc tryFinishTest(test: Test): bool =
|
||||
if test.process.running():
|
||||
return false
|
||||
test.output = test.process.outputStream.readAll()
|
||||
test.error = test.process.errorStream.readAll()
|
||||
if test.process.peekExitCode() == 0:
|
||||
test.result = TestResult.ToEval
|
||||
else:
|
||||
test.result = TestResult.Crash
|
||||
test.process.close()
|
||||
log(LogLevel.Debug, &"Test {test.path} finished.")
|
||||
return true
|
||||
|
||||
const maxAliveTests = 8
|
||||
const testWait = 10
|
||||
|
||||
proc runTests(tests: seq[Test], runner: string) =
|
||||
var
|
||||
aliveTests = 0
|
||||
currentTest = 0
|
||||
finishedTests = 0
|
||||
buffer = newBuffer()
|
||||
let totalTests = tests.len()
|
||||
|
||||
buffer.updateProgressBar(&"", totalTests, finishedTests)
|
||||
buffer.render()
|
||||
while aliveTests > 0 or currentTest < tests.len():
|
||||
buffer.render()
|
||||
sleep(testWait)
|
||||
if aliveTests < maxAliveTests and currentTest < tests.len():
|
||||
if tests[currentTest].result == TestResult.Unstarted:
|
||||
tests[currentTest].runTest(runner)
|
||||
inc aliveTests
|
||||
inc currentTest
|
||||
else:
|
||||
inc currentTest
|
||||
inc finishedTests
|
||||
for i in countup(0, min(currentTest, tests.high())):
|
||||
if tests[i].result == TestResult.Running:
|
||||
if tryFinishTest(tests[i]):
|
||||
inc finishedTests
|
||||
buffer.updateProgressBar(&"", totalTests, finishedTests)
|
||||
dec aliveTests
|
||||
else:
|
||||
inc tests[i].cycles
|
||||
buffer.render()
|
||||
|
||||
proc evalTest(test: Test) =
|
||||
test.output = test.output.tuStrip()
|
||||
test.error = test.error.tuStrip()
|
||||
test.expectedOutput = test.expectedOutput.tuStrip()
|
||||
test.expectedError = test.expectedError.tuStrip()
|
||||
if test.output != test.expectedOutput or test.error != test.expectedError:
|
||||
test.result = TestResult.Mismatch
|
||||
else:
|
||||
test.result = TestResult.Success
|
||||
|
||||
proc evalTests(tests: seq[Test]) =
|
||||
for test in tests:
|
||||
if test.result == TestResult.ToEval:
|
||||
evalTest(test)
|
||||
|
||||
proc printResults(tests: seq[Test]): bool =
|
||||
var
|
||||
skipped = 0
|
||||
success = 0
|
||||
fail = 0
|
||||
crash = 0
|
||||
|
||||
for test in tests:
|
||||
log(LogLevel.Debug, &"Test {test.path} result: {test.result}")
|
||||
case test.result:
|
||||
of TestResult.Skip:
|
||||
inc skipped
|
||||
of TestResult.Mismatch:
|
||||
inc fail
|
||||
log(LogLevel.Debug, &"[{test.path}\noutput:\n{test.output}\nerror:\n{test.error}\nexpected output:\n{test.expectedOutput}\nexpectedError:\n{test.expectedError}\n]")
|
||||
of TestResult.Crash:
|
||||
inc crash
|
||||
log(LogLevel.Debug, &"{test.path} \ncrash:\n{test.error}")
|
||||
of TestResult.Success:
|
||||
inc success
|
||||
else:
|
||||
log(LogLevel.Error, &"Probably a testing suite bug: test {test.path} has result {test.result}")
|
||||
let finalLevel = if fail == 0 and crash == 0: LogLevel.Info else: LogLevel.Error
|
||||
log(finalLevel, &"{tests.len()} tests: {success} succeeded, {skipped} skipped, {fail} failed, {crash} crashed.")
|
||||
|
||||
fail == 0 and crash == 0
|
||||
|
||||
when isMainModule:
|
||||
const jatsVersion = "(dev)"
|
||||
|
||||
|
|
|
|||
|
|
@ -1,6 +1,8 @@
|
|||
|
||||
# logging stuff
|
||||
|
||||
import terminal, strformat, times
|
||||
|
||||
type LogLevel* {.pure.} = enum
|
||||
Debug, # always written to file only (large outputs, such as the entire output of the failing test or stacktrace)
|
||||
Info, # important information about the progress of the test suite
|
||||
|
|
|
|||
|
|
@ -1,5 +1,5 @@
|
|||
import multibyte
|
||||
import ../testutils
|
||||
import ../logutils
|
||||
|
||||
proc runNimTests* =
|
||||
log(LogLevel.Info, "Running nim tests.")
|
||||
|
|
|
|||
|
|
@ -0,0 +1,51 @@
|
|||
# Copyright 2020 Mattia Giambirtone
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
# Test object and helpers
|
||||
|
||||
import re, strutils, osproc
|
||||
|
||||
# types
|
||||
|
||||
type
|
||||
TestResult* {.pure.} = enum
|
||||
Unstarted, Running, ToEval, Success, Skip, Mismatch, Crash
|
||||
|
||||
Test* = ref object
|
||||
result*: TestResult
|
||||
path*: string
|
||||
expectedOutput*: string
|
||||
expectedError*: string
|
||||
output*: string
|
||||
error*: string
|
||||
process*: Process
|
||||
cycles*: int
|
||||
|
||||
# parsing the test notation
|
||||
|
||||
proc compileExpectedOutput*(source: string): string =
|
||||
for line in source.split('\n'):
|
||||
if line =~ re"^.*//output:[ ]?(.*)$":
|
||||
result &= matches[0] & "\n"
|
||||
|
||||
proc compileExpectedError*(source: string): string =
|
||||
for line in source.split('\n'):
|
||||
if line =~ re"^.*//error:[ ]?(.*)$":
|
||||
result &= matches[0] & "\n"
|
||||
|
||||
# stuff for cleaning test output
|
||||
|
||||
proc tuStrip*(input: string): string =
|
||||
return input.replace(re"[\n\r]*$", "")
|
||||
|
||||
|
|
@ -12,40 +12,127 @@
|
|||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
# Test object and helpers
|
||||
# Test object helpers
|
||||
|
||||
import re, strutils, terminal, osproc, strformat, times
|
||||
import testobject, logutils, os, osproc, streams, strformat
|
||||
|
||||
# types
|
||||
# Tests that represent not-yet implemented behaviour
|
||||
const exceptions = ["all.jpl", "for_with_function.jpl", "runtime_interning.jpl", "problem4.jpl"]
|
||||
# TODO: for_with_function.jpl and problem4.jpl should already be implemented, check on them
|
||||
|
||||
type
|
||||
TestResult* {.pure.} = enum
|
||||
Unstarted, Running, ToEval, Success, Skip, Mismatch, Crash
|
||||
proc buildTest(path: string): Test =
|
||||
log(LogLevel.Debug, &"Building test {path}")
|
||||
let source = readFile(path)
|
||||
result = Test(
|
||||
path: path,
|
||||
result: if path.extractFilename in exceptions: TestResult.Skip
|
||||
else: TestResult.Unstarted,
|
||||
expectedOutput: compileExpectedOutput(source),
|
||||
expectedError: compileExpectedError(source)
|
||||
)
|
||||
|
||||
Test* = ref object
|
||||
result*: TestResult
|
||||
path*: string
|
||||
expectedOutput*: string
|
||||
expectedError*: string
|
||||
output*: string
|
||||
error*: string
|
||||
process*: Process
|
||||
cycles*: int
|
||||
proc buildTests*(testDir: string): seq[Test] =
|
||||
for candidateObj in walkDir(testDir):
|
||||
let candidate = candidateObj.path
|
||||
if dirExists(candidate):
|
||||
log(LogLevel.Debug, &"Descending into dir {candidate}")
|
||||
result &= buildTests(candidate)
|
||||
else:
|
||||
result.add buildTest(candidate)
|
||||
|
||||
# parsing the test notation
|
||||
proc runTest(test: Test, runner: string) =
|
||||
log(LogLevel.Debug, &"Starting test {test.path}.")
|
||||
let process = startProcess(runner, args = @[test.path])
|
||||
test.process = process
|
||||
test.result = TestResult.Running
|
||||
|
||||
proc compileExpectedOutput*(source: string): string =
|
||||
for line in source.split('\n'):
|
||||
if line =~ re"^.*//output:[ ]?(.*)$":
|
||||
result &= matches[0] & "\n"
|
||||
proc tryFinishTest(test: Test): bool =
|
||||
if test.process.running():
|
||||
return false
|
||||
test.output = test.process.outputStream.readAll()
|
||||
test.error = test.process.errorStream.readAll()
|
||||
if test.process.peekExitCode() == 0:
|
||||
test.result = TestResult.ToEval
|
||||
else:
|
||||
test.result = TestResult.Crash
|
||||
test.process.close()
|
||||
log(LogLevel.Debug, &"Test {test.path} finished.")
|
||||
return true
|
||||
|
||||
proc compileExpectedError*(source: string): string =
|
||||
for line in source.split('\n'):
|
||||
if line =~ re"^.*//error:[ ]?(.*)$":
|
||||
result &= matches[0] & "\n"
|
||||
const maxAliveTests = 8
|
||||
const testWait = 10
|
||||
|
||||
# stuff for cleaning test output
|
||||
proc runTests*(tests: seq[Test], runner: string) =
|
||||
var
|
||||
aliveTests = 0
|
||||
currentTest = 0
|
||||
finishedTests = 0
|
||||
buffer = newBuffer()
|
||||
let totalTests = tests.len()
|
||||
|
||||
buffer.updateProgressBar(&"", totalTests, finishedTests)
|
||||
buffer.render()
|
||||
while aliveTests > 0 or currentTest < tests.len():
|
||||
buffer.render()
|
||||
sleep(testWait)
|
||||
if aliveTests < maxAliveTests and currentTest < tests.len():
|
||||
if tests[currentTest].result == TestResult.Unstarted:
|
||||
tests[currentTest].runTest(runner)
|
||||
inc aliveTests
|
||||
inc currentTest
|
||||
else:
|
||||
inc currentTest
|
||||
inc finishedTests
|
||||
for i in countup(0, min(currentTest, tests.high())):
|
||||
if tests[i].result == TestResult.Running:
|
||||
if tryFinishTest(tests[i]):
|
||||
inc finishedTests
|
||||
buffer.updateProgressBar(&"", totalTests, finishedTests)
|
||||
dec aliveTests
|
||||
else:
|
||||
inc tests[i].cycles
|
||||
buffer.render()
|
||||
|
||||
proc evalTest(test: Test) =
|
||||
test.output = test.output.tuStrip()
|
||||
test.error = test.error.tuStrip()
|
||||
test.expectedOutput = test.expectedOutput.tuStrip()
|
||||
test.expectedError = test.expectedError.tuStrip()
|
||||
if test.output != test.expectedOutput or test.error != test.expectedError:
|
||||
test.result = TestResult.Mismatch
|
||||
else:
|
||||
test.result = TestResult.Success
|
||||
|
||||
proc evalTests*(tests: seq[Test]) =
|
||||
for test in tests:
|
||||
if test.result == TestResult.ToEval:
|
||||
evalTest(test)
|
||||
|
||||
proc printResults*(tests: seq[Test]): bool =
|
||||
var
|
||||
skipped = 0
|
||||
success = 0
|
||||
fail = 0
|
||||
crash = 0
|
||||
|
||||
for test in tests:
|
||||
log(LogLevel.Debug, &"Test {test.path} result: {test.result}")
|
||||
case test.result:
|
||||
of TestResult.Skip:
|
||||
inc skipped
|
||||
of TestResult.Mismatch:
|
||||
inc fail
|
||||
log(LogLevel.Debug, &"[{test.path}\noutput:\n{test.output}\nerror:\n{test.error}\nexpected output:\n{test.expectedOutput}\nexpectedError:\n{test.expectedError}\n]")
|
||||
of TestResult.Crash:
|
||||
inc crash
|
||||
log(LogLevel.Debug, &"{test.path} \ncrash:\n{test.error}")
|
||||
of TestResult.Success:
|
||||
inc success
|
||||
else:
|
||||
log(LogLevel.Error, &"Probably a testing suite bug: test {test.path} has result {test.result}")
|
||||
let finalLevel = if fail == 0 and crash == 0: LogLevel.Info else: LogLevel.Error
|
||||
log(finalLevel, &"{tests.len()} tests: {success} succeeded, {skipped} skipped, {fail} failed, {crash} crashed.")
|
||||
|
||||
fail == 0 and crash == 0
|
||||
|
||||
proc tuStrip*(input: string): string =
|
||||
return input.replace(re"[\n\r]*$", "")
|
||||
|
||||
|
|
|
|||
Loading…
Reference in New Issue