From 4d4b12a603a42dc096ec359198e6c4e5c4929bba Mon Sep 17 00:00:00 2001 From: nocturn9x Date: Sat, 13 Apr 2024 16:28:48 +0200 Subject: [PATCH] Added test suite --- .gitignore | 4 +- src/Chess/board.nim | 2 +- src/Chess/{ => tests}/compare_positions.py | 26 +++-- src/Chess/tests/positions.txt | 128 +++++++++++++++++++++ src/Chess/tests/suite.py | 43 +++++++ 5 files changed, 192 insertions(+), 11 deletions(-) rename src/Chess/{ => tests}/compare_positions.py (90%) create mode 100644 src/Chess/tests/positions.txt create mode 100644 src/Chess/tests/suite.py diff --git a/.gitignore b/.gitignore index 56408cb..55f666e 100644 --- a/.gitignore +++ b/.gitignore @@ -3,4 +3,6 @@ nimcache/ nimblecache/ htmldocs/ nim.cfg -bin \ No newline at end of file +bin +# Python +__pycache__ diff --git a/src/Chess/board.nim b/src/Chess/board.nim index cfd08dd..48a8776 100644 --- a/src/Chess/board.nim +++ b/src/Chess/board.nim @@ -2258,7 +2258,7 @@ proc main: int = echo &"Unknown command '{cmd[0]}'. Type 'help' for more information." except IOError: echo "" - return -1 + return 0 except EOFError: echo "" return 0 diff --git a/src/Chess/compare_positions.py b/src/Chess/tests/compare_positions.py similarity index 90% rename from src/Chess/compare_positions.py rename to src/Chess/tests/compare_positions.py index a8d41f4..274351f 100644 --- a/src/Chess/compare_positions.py +++ b/src/Chess/tests/compare_positions.py @@ -1,7 +1,5 @@ import re -import os import sys -import time import subprocess from shutil import which from pathlib import Path @@ -10,17 +8,19 @@ from argparse import ArgumentParser, Namespace def main(args: Namespace) -> int: + if args.silent: + print = lambda *_: ... print("Nimfish move validator v0.0.1 by nocturn9x") try: STOCKFISH = (args.stockfish or Path(which("stockfish"))).resolve(strict=True) except Exception as e: print(f"Could not locate stockfish executable -> {type(e).__name__}: {e}") - return -1 + return 2 try: NIMFISH = (args.nimfish or (Path.cwd() / "bin" / "nimfish")).resolve(strict=True) except Exception as e: print(f"Could not locate nimfish executable -> {type(e).__name__}: {e}") - return -1 + return 2 print(f"Starting Stockfish engine at {STOCKFISH.as_posix()!r}") stockfish_process = subprocess.Popen(STOCKFISH, stdout=subprocess.PIPE, @@ -49,8 +49,14 @@ def main(args: Namespace) -> int: print(f"Engines started, beginning search to depth {args.ply}") nimfish_process.stdin.write(f"go perft {args.ply} {'bulk' if args.bulk else ''}\n") stockfish_process.stdin.write(f"go perft {args.ply}\n") - stockfish_output = stockfish_process.communicate()[0] - nimfish_output = nimfish_process.communicate()[0] + stockfish_output, stockfish_error = stockfish_process.communicate() + nimfish_output, nimfish_error = nimfish_process.communicate() + if nimfish_process.returncode != 0: + print(f"Nimfish crashed, stderr output below:\n{nimfish_error}") + if stockfish_process.returncode != 0: + print(f"Stockfish crashed, stderr below:\n{stockfish_error}") + if not all([stockfish_process.returncode == 0, nimfish_process.returncode == 0]): + return 3 positions = { "all": {}, "stockfish": {}, @@ -142,18 +148,20 @@ def main(args: Namespace) -> int: for move in mistakes: missed = positions["stockfish"][move] - positions["nimfish"][move] print(f" - {move}: expected {positions['stockfish'][move]}, got {positions['nimfish'][move]} ({'-' if missed > 0 else '+'}{abs(missed)})") + return 1 else: print("No discrepancies detected") + return 0 if __name__ == "__main__": - parser = ArgumentParser(description="Automatically compare perft results between our engine and Stockfish") + parser = ArgumentParser(description="Automatically compare perft results between Nimfish and Stockfish") parser.add_argument("--fen", "-f", type=str, default="", help="The FEN string of the position to start from (empty string means the initial one). Defaults to ''") parser.add_argument("--ply", "-d", type=int, required=True, help="The depth to stop at, expressed in plys (half-moves)") - parser.add_argument("--bulk", action="store_true", help="Enable bulk-counting for Nimfish (faster, less debuggable)", default=False) + parser.add_argument("--bulk", action="store_true", help="Enable bulk-counting for Nimfish (much faster)", default=False) parser.add_argument("--stockfish", type=Path, help="Path to the stockfish executable. Defaults to '' (detected automatically)", default=None) parser.add_argument("--nimfish", type=Path, help="Path to the nimfish executable. Defaults to '' (detected automatically)", default=None) - parser.add_argument("--auto-mode", action="store_true", help="Automatically attempt to detect which moves Nimfish got wrong") + parser.add_argument("--silent", action="store_true", help="Disable all output (a return code of 0 means the test was successful)", default=False) sys.exit(main(parser.parse_args())) \ No newline at end of file diff --git a/src/Chess/tests/positions.txt b/src/Chess/tests/positions.txt new file mode 100644 index 0000000..f281138 --- /dev/null +++ b/src/Chess/tests/positions.txt @@ -0,0 +1,128 @@ +rnbqkbnr/pppppppp/8/8/8/8/PPPPPPPP/RNBQKBNR w KQkq - 0 1 +r3k2r/p1ppqpb1/bn2pnp1/3PN3/1p2P3/2N2Q1p/PPPBBPPP/R3K2R w KQkq - 0 1 +4k3/8/8/8/8/8/8/4K2R w K - 0 1 +4k3/8/8/8/8/8/8/R3K3 w Q - 0 1 +4k2r/8/8/8/8/8/8/4K3 w k - 0 1 +r3k3/8/8/8/8/8/8/4K3 w q - 0 1 +4k3/8/8/8/8/8/8/R3K2R w KQ - 0 1 +r3k2r/8/8/8/8/8/8/4K3 w kq - 0 1 +8/8/8/8/8/8/6k1/4K2R w K - 0 1 +8/8/8/8/8/8/1k6/R3K3 w Q - 0 1 +4k2r/6K1/8/8/8/8/8/8 w k - 0 1 +r3k3/1K6/8/8/8/8/8/8 w q - 0 1 +r3k2r/8/8/8/8/8/8/R3K2R w KQkq - 0 1 +r3k2r/8/8/8/8/8/8/1R2K2R w Kkq - 0 1 +r3k2r/8/8/8/8/8/8/2R1K2R w Kkq - 0 1 +r3k2r/8/8/8/8/8/8/R3K1R1 w Qkq - 0 1 +1r2k2r/8/8/8/8/8/8/R3K2R w KQk - 0 1 +2r1k2r/8/8/8/8/8/8/R3K2R w KQk - 0 1 +r3k1r1/8/8/8/8/8/8/R3K2R w KQq - 0 1 +4k3/8/8/8/8/8/8/4K2R b K - 0 1 +4k3/8/8/8/8/8/8/R3K3 b Q - 0 1 +4k2r/8/8/8/8/8/8/4K3 b k - 0 1 +r3k3/8/8/8/8/8/8/4K3 b q - 0 1 +4k3/8/8/8/8/8/8/R3K2R b KQ - 0 1 +r3k2r/8/8/8/8/8/8/4K3 b kq - 0 1 +8/8/8/8/8/8/6k1/4K2R b K - 0 1 +8/8/8/8/8/8/1k6/R3K3 b Q - 0 1 +4k2r/6K1/8/8/8/8/8/8 b k - 0 1 +r3k3/1K6/8/8/8/8/8/8 b q - 0 1 +r3k2r/8/8/8/8/8/8/R3K2R b KQkq - 0 1 +r3k2r/8/8/8/8/8/8/1R2K2R b Kkq - 0 1 +r3k2r/8/8/8/8/8/8/2R1K2R b Kkq - 0 1 +r3k2r/8/8/8/8/8/8/R3K1R1 b Qkq - 0 1 +1r2k2r/8/8/8/8/8/8/R3K2R b KQk - 0 1 +2r1k2r/8/8/8/8/8/8/R3K2R b KQk - 0 1 +r3k1r1/8/8/8/8/8/8/R3K2R b KQq - 0 1 +8/1n4N1/2k5/8/8/5K2/1N4n1/8 w - - 0 1 +8/1k6/8/5N2/8/4n3/8/2K5 w - - 0 1 +8/8/4k3/3Nn3/3nN3/4K3/8/8 w - - 0 1 +K7/8/2n5/1n6/8/8/8/k6N w - - 0 1 +k7/8/2N5/1N6/8/8/8/K6n w - - 0 1 +8/1n4N1/2k5/8/8/5K2/1N4n1/8 b - - 0 1 +8/1k6/8/5N2/8/4n3/8/2K5 b - - 0 1 +8/8/3K4/3Nn3/3nN3/4k3/8/8 b - - 0 1 +K7/8/2n5/1n6/8/8/8/k6N b - - 0 1 +k7/8/2N5/1N6/8/8/8/K6n b - - 0 1 +B6b/8/8/8/2K5/4k3/8/b6B w - - 0 1 +8/8/1B6/7b/7k/8/2B1b3/7K w - - 0 1 +k7/B7/1B6/1B6/8/8/8/K6b w - - 0 1 +K7/b7/1b6/1b6/8/8/8/k6B w - - 0 1 +B6b/8/8/8/2K5/5k2/8/b6B b - - 0 1 +8/8/1B6/7b/7k/8/2B1b3/7K b - - 0 1 +k7/B7/1B6/1B6/8/8/8/K6b b - - 0 1 +K7/b7/1b6/1b6/8/8/8/k6B b - - 0 1 +7k/RR6/8/8/8/8/rr6/7K w - - 0 1 +R6r/8/8/2K5/5k2/8/8/r6R w - - 0 1 +7k/RR6/8/8/8/8/rr6/7K b - - 0 1 +R6r/8/8/2K5/5k2/8/8/r6R b - - 0 1 +6kq/8/8/8/8/8/8/7K w - - 0 1 +6KQ/8/8/8/8/8/8/7k b - - 0 1 +K7/8/8/3Q4/4q3/8/8/7k w - - 0 1 +6qk/8/8/8/8/8/8/7K b - - 0 1 +6KQ/8/8/8/8/8/8/7k b - - 0 1 +K7/8/8/3Q4/4q3/8/8/7k b - - 0 1 +8/8/8/8/8/K7/P7/k7 w - - 0 1 +8/8/8/8/8/7K/7P/7k w - - 0 1 +K7/p7/k7/8/8/8/8/8 w - - 0 1 +7K/7p/7k/8/8/8/8/8 w - - 0 1 +8/2k1p3/3pP3/3P2K1/8/8/8/8 w - - 0 1 +8/8/8/8/8/K7/P7/k7 b - - 0 1 +8/8/8/8/8/7K/7P/7k b - - 0 1 +K7/p7/k7/8/8/8/8/8 b - - 0 1 +7K/7p/7k/8/8/8/8/8 b - - 0 1 +8/2k1p3/3pP3/3P2K1/8/8/8/8 b - - 0 1 +8/8/8/8/8/4k3/4P3/4K3 w - - 0 1 +4k3/4p3/4K3/8/8/8/8/8 b - - 0 1 +8/8/7k/7p/7P/7K/8/8 w - - 0 1 +8/8/k7/p7/P7/K7/8/8 w - - 0 1 +8/8/3k4/3p4/3P4/3K4/8/8 w - - 0 1 +8/3k4/3p4/8/3P4/3K4/8/8 w - - 0 1 +8/8/3k4/3p4/8/3P4/3K4/8 w - - 0 1 +k7/8/3p4/8/3P4/8/8/7K w - - 0 1 +8/8/7k/7p/7P/7K/8/8 b - - 0 1 +8/8/k7/p7/P7/K7/8/8 b - - 0 1 +8/8/3k4/3p4/3P4/3K4/8/8 b - - 0 1 +8/3k4/3p4/8/3P4/3K4/8/8 b - - 0 1 +8/8/3k4/3p4/8/3P4/3K4/8 b - - 0 1 +k7/8/3p4/8/3P4/8/8/7K b - - 0 1 +7k/3p4/8/8/3P4/8/8/K7 w - - 0 1 +7k/8/8/3p4/8/8/3P4/K7 w - - 0 1 +k7/8/8/7p/6P1/8/8/K7 w - - 0 1 +k7/8/7p/8/8/6P1/8/K7 w - - 0 1 +k7/8/8/6p1/7P/8/8/K7 w - - 0 1 +k7/8/6p1/8/8/7P/8/K7 w - - 0 1 +k7/8/8/3p4/4p3/8/8/7K w - - 0 1 +k7/8/3p4/8/8/4P3/8/7K w - - 0 1 +7k/3p4/8/8/3P4/8/8/K7 b - - 0 1 +7k/8/8/3p4/8/8/3P4/K7 b - - 0 1 +k7/8/8/7p/6P1/8/8/K7 b - - 0 1 +k7/8/7p/8/8/6P1/8/K7 b - - 0 1 +k7/8/8/6p1/7P/8/8/K7 b - - 0 1 +k7/8/6p1/8/8/7P/8/K7 b - - 0 1 +k7/8/8/3p4/4p3/8/8/7K b - - 0 1 +k7/8/3p4/8/8/4P3/8/7K b - - 0 1 +7k/8/8/p7/1P6/8/8/7K w - - 0 1 +7k/8/p7/8/8/1P6/8/7K w - - 0 1 +7k/8/8/1p6/P7/8/8/7K w - - 0 1 +7k/8/1p6/8/8/P7/8/7K w - - 0 1 +k7/7p/8/8/8/8/6P1/K7 w - - 0 1 +k7/6p1/8/8/8/8/7P/K7 w - - 0 1 +3k4/3pp3/8/8/8/8/3PP3/3K4 w - - 0 1 +7k/8/8/p7/1P6/8/8/7K b - - 0 1 +7k/8/p7/8/8/1P6/8/7K b - - 0 1 +7k/8/8/1p6/P7/8/8/7K b - - 0 1 +7k/8/1p6/8/8/P7/8/7K b - - 0 1 +k7/7p/8/8/8/8/6P1/K7 b - - 0 1 +k7/6p1/8/8/8/8/7P/K7 b - - 0 1 +3k4/3pp3/8/8/8/8/3PP3/3K4 b - - 0 1 +8/Pk6/8/8/8/8/6Kp/8 w - - 0 1 +n1n5/1Pk5/8/8/8/8/5Kp1/5N1N w - - 0 1 +8/PPPk4/8/8/8/8/4Kppp/8 w - - 0 1 +n1n5/PPPk4/8/8/8/8/4Kppp/5N1N w - - 0 1 +8/Pk6/8/8/8/8/6Kp/8 b - - 0 1 +n1n5/1Pk5/8/8/8/8/5Kp1/5N1N b - - 0 1 +8/PPPk4/8/8/8/8/4Kppp/8 b - - 0 1 +n1n5/PPPk4/8/8/8/8/4Kppp/5N1N b - - 0 1 +8/2p5/3p4/KP5r/1R3p1k/8/4P1P1/8 w - - 0 1 +rnbqkb1r/ppppp1pp/7n/4Pp2/8/8/PPPP1PPP/RNBQKBNR w KQkq f6 0 3 \ No newline at end of file diff --git a/src/Chess/tests/suite.py b/src/Chess/tests/suite.py new file mode 100644 index 0000000..877e9a1 --- /dev/null +++ b/src/Chess/tests/suite.py @@ -0,0 +1,43 @@ +import sys +import timeit +from pathlib import Path +from argparse import Namespace, ArgumentParser +from compare_positions import main as test + + + +def main(args: Namespace) -> int: + print("[S] Starting test suite") + successful = [] + failed = [] + positions = args.positions.read_text().splitlines() + start = timeit.default_timer() + longest_fen = max(sorted([len(fen) for fen in positions])) + for i, fen in enumerate(positions): + fen = fen.strip(" ") + fen += " " * (longest_fen - len(fen)) + sys.stdout.write(f"\r[S] Testing {fen} ({i + 1}/{len(positions)})\033[K") + args.fen = fen + args.silent = not args.no_silent + if test(args) == 0: + successful.append(fen) + else: + failed.append(fen) + stop = timeit.default_timer() + print(f"\r[S] Ran {len(positions)} tests at depth {args.ply} in {stop - start:.2f} seconds ({len(successful)} successful, {len(failed)} failed)\033[K") + if failed and args.show_failures: + print("[S] The following FENs failed to pass the test:", end="") + print("\n\t".join(failed)) + + +if __name__ == "__main__": + parser = ArgumentParser(description="Run a set of tests using compare_positions.py") + parser.add_argument("--ply", "-d", type=int, required=True, help="The depth to stop at, expressed in plys (half-moves)") + parser.add_argument("--bulk", action="store_true", help="Enable bulk-counting for Nimfish (much faster)", default=False) + parser.add_argument("--stockfish", type=Path, help="Path to the stockfish executable. Defaults to '' (detected automatically)", default=None) + parser.add_argument("--nimfish", type=Path, help="Path to the nimfish executable. Defaults to '' (detected automatically)", default=None) + parser.add_argument("--positions", type=Path, help="Location of the file containing FENs to test, one per line. Defaults to 'tests/positions.txt'", + default=Path("tests/positions.txt")) + parser.add_argument("--no-silent", action="store_true", help="Do not suppress output from compare_positions.py (defaults)", default=False) + parser.add_argument("--show-failures", action="store_true", help="Show which FENs failed to pass the test", default=False) + sys.exit(main(parser.parse_args())) \ No newline at end of file