Added test suite
This commit is contained in:
parent
6153112c21
commit
4d4b12a603
|
|
@ -3,4 +3,6 @@ nimcache/
|
|||
nimblecache/
|
||||
htmldocs/
|
||||
nim.cfg
|
||||
bin
|
||||
bin
|
||||
# Python
|
||||
__pycache__
|
||||
|
|
|
|||
|
|
@ -2258,7 +2258,7 @@ proc main: int =
|
|||
echo &"Unknown command '{cmd[0]}'. Type 'help' for more information."
|
||||
except IOError:
|
||||
echo ""
|
||||
return -1
|
||||
return 0
|
||||
except EOFError:
|
||||
echo ""
|
||||
return 0
|
||||
|
|
|
|||
|
|
@ -1,7 +1,5 @@
|
|||
import re
|
||||
import os
|
||||
import sys
|
||||
import time
|
||||
import subprocess
|
||||
from shutil import which
|
||||
from pathlib import Path
|
||||
|
|
@ -10,17 +8,19 @@ from argparse import ArgumentParser, Namespace
|
|||
|
||||
|
||||
def main(args: Namespace) -> int:
|
||||
if args.silent:
|
||||
print = lambda *_: ...
|
||||
print("Nimfish move validator v0.0.1 by nocturn9x")
|
||||
try:
|
||||
STOCKFISH = (args.stockfish or Path(which("stockfish"))).resolve(strict=True)
|
||||
except Exception as e:
|
||||
print(f"Could not locate stockfish executable -> {type(e).__name__}: {e}")
|
||||
return -1
|
||||
return 2
|
||||
try:
|
||||
NIMFISH = (args.nimfish or (Path.cwd() / "bin" / "nimfish")).resolve(strict=True)
|
||||
except Exception as e:
|
||||
print(f"Could not locate nimfish executable -> {type(e).__name__}: {e}")
|
||||
return -1
|
||||
return 2
|
||||
print(f"Starting Stockfish engine at {STOCKFISH.as_posix()!r}")
|
||||
stockfish_process = subprocess.Popen(STOCKFISH,
|
||||
stdout=subprocess.PIPE,
|
||||
|
|
@ -49,8 +49,14 @@ def main(args: Namespace) -> int:
|
|||
print(f"Engines started, beginning search to depth {args.ply}")
|
||||
nimfish_process.stdin.write(f"go perft {args.ply} {'bulk' if args.bulk else ''}\n")
|
||||
stockfish_process.stdin.write(f"go perft {args.ply}\n")
|
||||
stockfish_output = stockfish_process.communicate()[0]
|
||||
nimfish_output = nimfish_process.communicate()[0]
|
||||
stockfish_output, stockfish_error = stockfish_process.communicate()
|
||||
nimfish_output, nimfish_error = nimfish_process.communicate()
|
||||
if nimfish_process.returncode != 0:
|
||||
print(f"Nimfish crashed, stderr output below:\n{nimfish_error}")
|
||||
if stockfish_process.returncode != 0:
|
||||
print(f"Stockfish crashed, stderr below:\n{stockfish_error}")
|
||||
if not all([stockfish_process.returncode == 0, nimfish_process.returncode == 0]):
|
||||
return 3
|
||||
positions = {
|
||||
"all": {},
|
||||
"stockfish": {},
|
||||
|
|
@ -142,18 +148,20 @@ def main(args: Namespace) -> int:
|
|||
for move in mistakes:
|
||||
missed = positions["stockfish"][move] - positions["nimfish"][move]
|
||||
print(f" - {move}: expected {positions['stockfish'][move]}, got {positions['nimfish'][move]} ({'-' if missed > 0 else '+'}{abs(missed)})")
|
||||
return 1
|
||||
else:
|
||||
print("No discrepancies detected")
|
||||
return 0
|
||||
|
||||
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
parser = ArgumentParser(description="Automatically compare perft results between our engine and Stockfish")
|
||||
parser = ArgumentParser(description="Automatically compare perft results between Nimfish and Stockfish")
|
||||
parser.add_argument("--fen", "-f", type=str, default="", help="The FEN string of the position to start from (empty string means the initial one). Defaults to ''")
|
||||
parser.add_argument("--ply", "-d", type=int, required=True, help="The depth to stop at, expressed in plys (half-moves)")
|
||||
parser.add_argument("--bulk", action="store_true", help="Enable bulk-counting for Nimfish (faster, less debuggable)", default=False)
|
||||
parser.add_argument("--bulk", action="store_true", help="Enable bulk-counting for Nimfish (much faster)", default=False)
|
||||
parser.add_argument("--stockfish", type=Path, help="Path to the stockfish executable. Defaults to '' (detected automatically)", default=None)
|
||||
parser.add_argument("--nimfish", type=Path, help="Path to the nimfish executable. Defaults to '' (detected automatically)", default=None)
|
||||
parser.add_argument("--auto-mode", action="store_true", help="Automatically attempt to detect which moves Nimfish got wrong")
|
||||
parser.add_argument("--silent", action="store_true", help="Disable all output (a return code of 0 means the test was successful)", default=False)
|
||||
sys.exit(main(parser.parse_args()))
|
||||
|
|
@ -0,0 +1,128 @@
|
|||
rnbqkbnr/pppppppp/8/8/8/8/PPPPPPPP/RNBQKBNR w KQkq - 0 1
|
||||
r3k2r/p1ppqpb1/bn2pnp1/3PN3/1p2P3/2N2Q1p/PPPBBPPP/R3K2R w KQkq - 0 1
|
||||
4k3/8/8/8/8/8/8/4K2R w K - 0 1
|
||||
4k3/8/8/8/8/8/8/R3K3 w Q - 0 1
|
||||
4k2r/8/8/8/8/8/8/4K3 w k - 0 1
|
||||
r3k3/8/8/8/8/8/8/4K3 w q - 0 1
|
||||
4k3/8/8/8/8/8/8/R3K2R w KQ - 0 1
|
||||
r3k2r/8/8/8/8/8/8/4K3 w kq - 0 1
|
||||
8/8/8/8/8/8/6k1/4K2R w K - 0 1
|
||||
8/8/8/8/8/8/1k6/R3K3 w Q - 0 1
|
||||
4k2r/6K1/8/8/8/8/8/8 w k - 0 1
|
||||
r3k3/1K6/8/8/8/8/8/8 w q - 0 1
|
||||
r3k2r/8/8/8/8/8/8/R3K2R w KQkq - 0 1
|
||||
r3k2r/8/8/8/8/8/8/1R2K2R w Kkq - 0 1
|
||||
r3k2r/8/8/8/8/8/8/2R1K2R w Kkq - 0 1
|
||||
r3k2r/8/8/8/8/8/8/R3K1R1 w Qkq - 0 1
|
||||
1r2k2r/8/8/8/8/8/8/R3K2R w KQk - 0 1
|
||||
2r1k2r/8/8/8/8/8/8/R3K2R w KQk - 0 1
|
||||
r3k1r1/8/8/8/8/8/8/R3K2R w KQq - 0 1
|
||||
4k3/8/8/8/8/8/8/4K2R b K - 0 1
|
||||
4k3/8/8/8/8/8/8/R3K3 b Q - 0 1
|
||||
4k2r/8/8/8/8/8/8/4K3 b k - 0 1
|
||||
r3k3/8/8/8/8/8/8/4K3 b q - 0 1
|
||||
4k3/8/8/8/8/8/8/R3K2R b KQ - 0 1
|
||||
r3k2r/8/8/8/8/8/8/4K3 b kq - 0 1
|
||||
8/8/8/8/8/8/6k1/4K2R b K - 0 1
|
||||
8/8/8/8/8/8/1k6/R3K3 b Q - 0 1
|
||||
4k2r/6K1/8/8/8/8/8/8 b k - 0 1
|
||||
r3k3/1K6/8/8/8/8/8/8 b q - 0 1
|
||||
r3k2r/8/8/8/8/8/8/R3K2R b KQkq - 0 1
|
||||
r3k2r/8/8/8/8/8/8/1R2K2R b Kkq - 0 1
|
||||
r3k2r/8/8/8/8/8/8/2R1K2R b Kkq - 0 1
|
||||
r3k2r/8/8/8/8/8/8/R3K1R1 b Qkq - 0 1
|
||||
1r2k2r/8/8/8/8/8/8/R3K2R b KQk - 0 1
|
||||
2r1k2r/8/8/8/8/8/8/R3K2R b KQk - 0 1
|
||||
r3k1r1/8/8/8/8/8/8/R3K2R b KQq - 0 1
|
||||
8/1n4N1/2k5/8/8/5K2/1N4n1/8 w - - 0 1
|
||||
8/1k6/8/5N2/8/4n3/8/2K5 w - - 0 1
|
||||
8/8/4k3/3Nn3/3nN3/4K3/8/8 w - - 0 1
|
||||
K7/8/2n5/1n6/8/8/8/k6N w - - 0 1
|
||||
k7/8/2N5/1N6/8/8/8/K6n w - - 0 1
|
||||
8/1n4N1/2k5/8/8/5K2/1N4n1/8 b - - 0 1
|
||||
8/1k6/8/5N2/8/4n3/8/2K5 b - - 0 1
|
||||
8/8/3K4/3Nn3/3nN3/4k3/8/8 b - - 0 1
|
||||
K7/8/2n5/1n6/8/8/8/k6N b - - 0 1
|
||||
k7/8/2N5/1N6/8/8/8/K6n b - - 0 1
|
||||
B6b/8/8/8/2K5/4k3/8/b6B w - - 0 1
|
||||
8/8/1B6/7b/7k/8/2B1b3/7K w - - 0 1
|
||||
k7/B7/1B6/1B6/8/8/8/K6b w - - 0 1
|
||||
K7/b7/1b6/1b6/8/8/8/k6B w - - 0 1
|
||||
B6b/8/8/8/2K5/5k2/8/b6B b - - 0 1
|
||||
8/8/1B6/7b/7k/8/2B1b3/7K b - - 0 1
|
||||
k7/B7/1B6/1B6/8/8/8/K6b b - - 0 1
|
||||
K7/b7/1b6/1b6/8/8/8/k6B b - - 0 1
|
||||
7k/RR6/8/8/8/8/rr6/7K w - - 0 1
|
||||
R6r/8/8/2K5/5k2/8/8/r6R w - - 0 1
|
||||
7k/RR6/8/8/8/8/rr6/7K b - - 0 1
|
||||
R6r/8/8/2K5/5k2/8/8/r6R b - - 0 1
|
||||
6kq/8/8/8/8/8/8/7K w - - 0 1
|
||||
6KQ/8/8/8/8/8/8/7k b - - 0 1
|
||||
K7/8/8/3Q4/4q3/8/8/7k w - - 0 1
|
||||
6qk/8/8/8/8/8/8/7K b - - 0 1
|
||||
6KQ/8/8/8/8/8/8/7k b - - 0 1
|
||||
K7/8/8/3Q4/4q3/8/8/7k b - - 0 1
|
||||
8/8/8/8/8/K7/P7/k7 w - - 0 1
|
||||
8/8/8/8/8/7K/7P/7k w - - 0 1
|
||||
K7/p7/k7/8/8/8/8/8 w - - 0 1
|
||||
7K/7p/7k/8/8/8/8/8 w - - 0 1
|
||||
8/2k1p3/3pP3/3P2K1/8/8/8/8 w - - 0 1
|
||||
8/8/8/8/8/K7/P7/k7 b - - 0 1
|
||||
8/8/8/8/8/7K/7P/7k b - - 0 1
|
||||
K7/p7/k7/8/8/8/8/8 b - - 0 1
|
||||
7K/7p/7k/8/8/8/8/8 b - - 0 1
|
||||
8/2k1p3/3pP3/3P2K1/8/8/8/8 b - - 0 1
|
||||
8/8/8/8/8/4k3/4P3/4K3 w - - 0 1
|
||||
4k3/4p3/4K3/8/8/8/8/8 b - - 0 1
|
||||
8/8/7k/7p/7P/7K/8/8 w - - 0 1
|
||||
8/8/k7/p7/P7/K7/8/8 w - - 0 1
|
||||
8/8/3k4/3p4/3P4/3K4/8/8 w - - 0 1
|
||||
8/3k4/3p4/8/3P4/3K4/8/8 w - - 0 1
|
||||
8/8/3k4/3p4/8/3P4/3K4/8 w - - 0 1
|
||||
k7/8/3p4/8/3P4/8/8/7K w - - 0 1
|
||||
8/8/7k/7p/7P/7K/8/8 b - - 0 1
|
||||
8/8/k7/p7/P7/K7/8/8 b - - 0 1
|
||||
8/8/3k4/3p4/3P4/3K4/8/8 b - - 0 1
|
||||
8/3k4/3p4/8/3P4/3K4/8/8 b - - 0 1
|
||||
8/8/3k4/3p4/8/3P4/3K4/8 b - - 0 1
|
||||
k7/8/3p4/8/3P4/8/8/7K b - - 0 1
|
||||
7k/3p4/8/8/3P4/8/8/K7 w - - 0 1
|
||||
7k/8/8/3p4/8/8/3P4/K7 w - - 0 1
|
||||
k7/8/8/7p/6P1/8/8/K7 w - - 0 1
|
||||
k7/8/7p/8/8/6P1/8/K7 w - - 0 1
|
||||
k7/8/8/6p1/7P/8/8/K7 w - - 0 1
|
||||
k7/8/6p1/8/8/7P/8/K7 w - - 0 1
|
||||
k7/8/8/3p4/4p3/8/8/7K w - - 0 1
|
||||
k7/8/3p4/8/8/4P3/8/7K w - - 0 1
|
||||
7k/3p4/8/8/3P4/8/8/K7 b - - 0 1
|
||||
7k/8/8/3p4/8/8/3P4/K7 b - - 0 1
|
||||
k7/8/8/7p/6P1/8/8/K7 b - - 0 1
|
||||
k7/8/7p/8/8/6P1/8/K7 b - - 0 1
|
||||
k7/8/8/6p1/7P/8/8/K7 b - - 0 1
|
||||
k7/8/6p1/8/8/7P/8/K7 b - - 0 1
|
||||
k7/8/8/3p4/4p3/8/8/7K b - - 0 1
|
||||
k7/8/3p4/8/8/4P3/8/7K b - - 0 1
|
||||
7k/8/8/p7/1P6/8/8/7K w - - 0 1
|
||||
7k/8/p7/8/8/1P6/8/7K w - - 0 1
|
||||
7k/8/8/1p6/P7/8/8/7K w - - 0 1
|
||||
7k/8/1p6/8/8/P7/8/7K w - - 0 1
|
||||
k7/7p/8/8/8/8/6P1/K7 w - - 0 1
|
||||
k7/6p1/8/8/8/8/7P/K7 w - - 0 1
|
||||
3k4/3pp3/8/8/8/8/3PP3/3K4 w - - 0 1
|
||||
7k/8/8/p7/1P6/8/8/7K b - - 0 1
|
||||
7k/8/p7/8/8/1P6/8/7K b - - 0 1
|
||||
7k/8/8/1p6/P7/8/8/7K b - - 0 1
|
||||
7k/8/1p6/8/8/P7/8/7K b - - 0 1
|
||||
k7/7p/8/8/8/8/6P1/K7 b - - 0 1
|
||||
k7/6p1/8/8/8/8/7P/K7 b - - 0 1
|
||||
3k4/3pp3/8/8/8/8/3PP3/3K4 b - - 0 1
|
||||
8/Pk6/8/8/8/8/6Kp/8 w - - 0 1
|
||||
n1n5/1Pk5/8/8/8/8/5Kp1/5N1N w - - 0 1
|
||||
8/PPPk4/8/8/8/8/4Kppp/8 w - - 0 1
|
||||
n1n5/PPPk4/8/8/8/8/4Kppp/5N1N w - - 0 1
|
||||
8/Pk6/8/8/8/8/6Kp/8 b - - 0 1
|
||||
n1n5/1Pk5/8/8/8/8/5Kp1/5N1N b - - 0 1
|
||||
8/PPPk4/8/8/8/8/4Kppp/8 b - - 0 1
|
||||
n1n5/PPPk4/8/8/8/8/4Kppp/5N1N b - - 0 1
|
||||
8/2p5/3p4/KP5r/1R3p1k/8/4P1P1/8 w - - 0 1
|
||||
rnbqkb1r/ppppp1pp/7n/4Pp2/8/8/PPPP1PPP/RNBQKBNR w KQkq f6 0 3
|
||||
|
|
@ -0,0 +1,43 @@
|
|||
import sys
|
||||
import timeit
|
||||
from pathlib import Path
|
||||
from argparse import Namespace, ArgumentParser
|
||||
from compare_positions import main as test
|
||||
|
||||
|
||||
|
||||
def main(args: Namespace) -> int:
|
||||
print("[S] Starting test suite")
|
||||
successful = []
|
||||
failed = []
|
||||
positions = args.positions.read_text().splitlines()
|
||||
start = timeit.default_timer()
|
||||
longest_fen = max(sorted([len(fen) for fen in positions]))
|
||||
for i, fen in enumerate(positions):
|
||||
fen = fen.strip(" ")
|
||||
fen += " " * (longest_fen - len(fen))
|
||||
sys.stdout.write(f"\r[S] Testing {fen} ({i + 1}/{len(positions)})\033[K")
|
||||
args.fen = fen
|
||||
args.silent = not args.no_silent
|
||||
if test(args) == 0:
|
||||
successful.append(fen)
|
||||
else:
|
||||
failed.append(fen)
|
||||
stop = timeit.default_timer()
|
||||
print(f"\r[S] Ran {len(positions)} tests at depth {args.ply} in {stop - start:.2f} seconds ({len(successful)} successful, {len(failed)} failed)\033[K")
|
||||
if failed and args.show_failures:
|
||||
print("[S] The following FENs failed to pass the test:", end="")
|
||||
print("\n\t".join(failed))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
parser = ArgumentParser(description="Run a set of tests using compare_positions.py")
|
||||
parser.add_argument("--ply", "-d", type=int, required=True, help="The depth to stop at, expressed in plys (half-moves)")
|
||||
parser.add_argument("--bulk", action="store_true", help="Enable bulk-counting for Nimfish (much faster)", default=False)
|
||||
parser.add_argument("--stockfish", type=Path, help="Path to the stockfish executable. Defaults to '' (detected automatically)", default=None)
|
||||
parser.add_argument("--nimfish", type=Path, help="Path to the nimfish executable. Defaults to '' (detected automatically)", default=None)
|
||||
parser.add_argument("--positions", type=Path, help="Location of the file containing FENs to test, one per line. Defaults to 'tests/positions.txt'",
|
||||
default=Path("tests/positions.txt"))
|
||||
parser.add_argument("--no-silent", action="store_true", help="Do not suppress output from compare_positions.py (defaults)", default=False)
|
||||
parser.add_argument("--show-failures", action="store_true", help="Show which FENs failed to pass the test", default=False)
|
||||
sys.exit(main(parser.parse_args()))
|
||||
Loading…
Reference in New Issue