Added test suite

.gitignore

@@ -3,4 +3,6 @@ nimcache/
 nimblecache/
 htmldocs/
 nim.cfg
-bin
+bin
+# Python
+__pycache__

src/Chess/board.nim

@@ -2258,7 +2258,7 @@ proc main: int =
                     echo &"Unknown command '{cmd[0]}'. Type 'help' for more information."
         except IOError:
             echo ""
-            return -1
+            return 0
         except EOFError:
             echo ""
             return 0

src/Chess/tests/compare_positions.py

@@ -1,7 +1,5 @@
 import re
-import os
 import sys
-import time
 import subprocess
 from shutil import which
 from pathlib import Path
@@ -10,17 +8,19 @@ from argparse import ArgumentParser, Namespace
 
 
 def main(args: Namespace) -> int:
+    if args.silent:
+        print = lambda *_: ...
     print("Nimfish move validator v0.0.1 by nocturn9x")
     try:
         STOCKFISH = (args.stockfish or Path(which("stockfish"))).resolve(strict=True)
     except Exception as e:
         print(f"Could not locate stockfish executable -> {type(e).__name__}: {e}")
-        return -1
+        return 2
     try:
         NIMFISH = (args.nimfish or (Path.cwd() / "bin" / "nimfish")).resolve(strict=True)
     except Exception as e:
         print(f"Could not locate nimfish executable -> {type(e).__name__}: {e}")
-        return -1
+        return 2
     print(f"Starting Stockfish engine at {STOCKFISH.as_posix()!r}")
     stockfish_process = subprocess.Popen(STOCKFISH,
                                          stdout=subprocess.PIPE,
@@ -49,8 +49,14 @@ def main(args: Namespace) -> int:
     print(f"Engines started, beginning search to depth {args.ply}")
     nimfish_process.stdin.write(f"go perft {args.ply} {'bulk' if args.bulk else ''}\n")
     stockfish_process.stdin.write(f"go perft {args.ply}\n")
-    stockfish_output = stockfish_process.communicate()[0]
+    stockfish_output, stockfish_error = stockfish_process.communicate()
-    nimfish_output = nimfish_process.communicate()[0]
+    nimfish_output, nimfish_error = nimfish_process.communicate()
+    if nimfish_process.returncode != 0:
+        print(f"Nimfish crashed, stderr output below:\n{nimfish_error}")
+    if stockfish_process.returncode != 0:
+        print(f"Stockfish crashed, stderr below:\n{stockfish_error}")
+    if not all([stockfish_process.returncode == 0, nimfish_process.returncode == 0]):
+        return 3
     positions = {
         "all": {},
         "stockfish": {},
@@ -142,18 +148,20 @@ def main(args: Namespace) -> int:
             for move in mistakes:
                 missed = positions["stockfish"][move] - positions["nimfish"][move]
                 print(f"       - {move}: expected {positions['stockfish'][move]}, got {positions['nimfish'][move]} ({'-' if missed > 0 else '+'}{abs(missed)})")
+        return 1
     else:
         print("No discrepancies detected")
+        return 0
 
     
 
 
 if __name__ == "__main__":
-    parser = ArgumentParser(description="Automatically compare perft results between our engine and Stockfish")
+    parser = ArgumentParser(description="Automatically compare perft results between Nimfish and Stockfish")
     parser.add_argument("--fen", "-f", type=str, default="", help="The FEN string of the position to start from (empty string means the initial one). Defaults to ''")
     parser.add_argument("--ply", "-d", type=int, required=True, help="The depth to stop at, expressed in plys (half-moves)")
-    parser.add_argument("--bulk", action="store_true", help="Enable bulk-counting for Nimfish (faster, less debuggable)", default=False)
+    parser.add_argument("--bulk", action="store_true", help="Enable bulk-counting for Nimfish (much faster)", default=False)
     parser.add_argument("--stockfish", type=Path, help="Path to the stockfish executable. Defaults to '' (detected automatically)", default=None)
     parser.add_argument("--nimfish", type=Path, help="Path to the nimfish executable. Defaults to '' (detected automatically)", default=None)
-    parser.add_argument("--auto-mode", action="store_true", help="Automatically attempt to detect which moves Nimfish got wrong")
+    parser.add_argument("--silent", action="store_true", help="Disable all output (a return code of 0 means the test was successful)", default=False)
     sys.exit(main(parser.parse_args()))

src/Chess/tests/positions.txt

@@ -0,0 +1,128 @@
+rnbqkbnr/pppppppp/8/8/8/8/PPPPPPPP/RNBQKBNR w KQkq - 0 1 
+r3k2r/p1ppqpb1/bn2pnp1/3PN3/1p2P3/2N2Q1p/PPPBBPPP/R3K2R w KQkq - 0 1 
+4k3/8/8/8/8/8/8/4K2R w K - 0 1 
+4k3/8/8/8/8/8/8/R3K3 w Q - 0 1 
+4k2r/8/8/8/8/8/8/4K3 w k - 0 1 
+r3k3/8/8/8/8/8/8/4K3 w q - 0 1 
+4k3/8/8/8/8/8/8/R3K2R w KQ - 0 1 
+r3k2r/8/8/8/8/8/8/4K3 w kq - 0 1 
+8/8/8/8/8/8/6k1/4K2R w K - 0 1 
+8/8/8/8/8/8/1k6/R3K3 w Q - 0 1 
+4k2r/6K1/8/8/8/8/8/8 w k - 0 1 
+r3k3/1K6/8/8/8/8/8/8 w q - 0 1 
+r3k2r/8/8/8/8/8/8/R3K2R w KQkq - 0 1 
+r3k2r/8/8/8/8/8/8/1R2K2R w Kkq - 0 1 
+r3k2r/8/8/8/8/8/8/2R1K2R w Kkq - 0 1 
+r3k2r/8/8/8/8/8/8/R3K1R1 w Qkq - 0 1 
+1r2k2r/8/8/8/8/8/8/R3K2R w KQk - 0 1 
+2r1k2r/8/8/8/8/8/8/R3K2R w KQk - 0 1 
+r3k1r1/8/8/8/8/8/8/R3K2R w KQq - 0 1 
+4k3/8/8/8/8/8/8/4K2R b K - 0 1 
+4k3/8/8/8/8/8/8/R3K3 b Q - 0 1 
+4k2r/8/8/8/8/8/8/4K3 b k - 0 1 
+r3k3/8/8/8/8/8/8/4K3 b q - 0 1 
+4k3/8/8/8/8/8/8/R3K2R b KQ - 0 1 
+r3k2r/8/8/8/8/8/8/4K3 b kq - 0 1 
+8/8/8/8/8/8/6k1/4K2R b K - 0 1 
+8/8/8/8/8/8/1k6/R3K3 b Q - 0 1 
+4k2r/6K1/8/8/8/8/8/8 b k - 0 1 
+r3k3/1K6/8/8/8/8/8/8 b q - 0 1 
+r3k2r/8/8/8/8/8/8/R3K2R b KQkq - 0 1 
+r3k2r/8/8/8/8/8/8/1R2K2R b Kkq - 0 1 
+r3k2r/8/8/8/8/8/8/2R1K2R b Kkq - 0 1 
+r3k2r/8/8/8/8/8/8/R3K1R1 b Qkq - 0 1 
+1r2k2r/8/8/8/8/8/8/R3K2R b KQk - 0 1 
+2r1k2r/8/8/8/8/8/8/R3K2R b KQk - 0 1 
+r3k1r1/8/8/8/8/8/8/R3K2R b KQq - 0 1 
+8/1n4N1/2k5/8/8/5K2/1N4n1/8 w - - 0 1 
+8/1k6/8/5N2/8/4n3/8/2K5 w - - 0 1 
+8/8/4k3/3Nn3/3nN3/4K3/8/8 w - - 0 1 
+K7/8/2n5/1n6/8/8/8/k6N w - - 0 1 
+k7/8/2N5/1N6/8/8/8/K6n w - - 0 1 
+8/1n4N1/2k5/8/8/5K2/1N4n1/8 b - - 0 1 
+8/1k6/8/5N2/8/4n3/8/2K5 b - - 0 1 
+8/8/3K4/3Nn3/3nN3/4k3/8/8 b - - 0 1 
+K7/8/2n5/1n6/8/8/8/k6N b - - 0 1 
+k7/8/2N5/1N6/8/8/8/K6n b - - 0 1 
+B6b/8/8/8/2K5/4k3/8/b6B w - - 0 1 
+8/8/1B6/7b/7k/8/2B1b3/7K w - - 0 1 
+k7/B7/1B6/1B6/8/8/8/K6b w - - 0 1 
+K7/b7/1b6/1b6/8/8/8/k6B w - - 0 1 
+B6b/8/8/8/2K5/5k2/8/b6B b - - 0 1 
+8/8/1B6/7b/7k/8/2B1b3/7K b - - 0 1 
+k7/B7/1B6/1B6/8/8/8/K6b b - - 0 1 
+K7/b7/1b6/1b6/8/8/8/k6B b - - 0 1 
+7k/RR6/8/8/8/8/rr6/7K w - - 0 1 
+R6r/8/8/2K5/5k2/8/8/r6R w - - 0 1 
+7k/RR6/8/8/8/8/rr6/7K b - - 0 1 
+R6r/8/8/2K5/5k2/8/8/r6R b - - 0 1 
+6kq/8/8/8/8/8/8/7K w - - 0 1 
+6KQ/8/8/8/8/8/8/7k b - - 0 1 
+K7/8/8/3Q4/4q3/8/8/7k w - - 0 1 
+6qk/8/8/8/8/8/8/7K b - - 0 1 
+6KQ/8/8/8/8/8/8/7k b - - 0 1 
+K7/8/8/3Q4/4q3/8/8/7k b - - 0 1 
+8/8/8/8/8/K7/P7/k7 w - - 0 1 
+8/8/8/8/8/7K/7P/7k w - - 0 1 
+K7/p7/k7/8/8/8/8/8 w - - 0 1 
+7K/7p/7k/8/8/8/8/8 w - - 0 1 
+8/2k1p3/3pP3/3P2K1/8/8/8/8 w - - 0 1 
+8/8/8/8/8/K7/P7/k7 b - - 0 1 
+8/8/8/8/8/7K/7P/7k b - - 0 1 
+K7/p7/k7/8/8/8/8/8 b - - 0 1 
+7K/7p/7k/8/8/8/8/8 b - - 0 1 
+8/2k1p3/3pP3/3P2K1/8/8/8/8 b - - 0 1 
+8/8/8/8/8/4k3/4P3/4K3 w - - 0 1 
+4k3/4p3/4K3/8/8/8/8/8 b - - 0 1 
+8/8/7k/7p/7P/7K/8/8 w - - 0 1 
+8/8/k7/p7/P7/K7/8/8 w - - 0 1 
+8/8/3k4/3p4/3P4/3K4/8/8 w - - 0 1 
+8/3k4/3p4/8/3P4/3K4/8/8 w - - 0 1 
+8/8/3k4/3p4/8/3P4/3K4/8 w - - 0 1 
+k7/8/3p4/8/3P4/8/8/7K w - - 0 1 
+8/8/7k/7p/7P/7K/8/8 b - - 0 1 
+8/8/k7/p7/P7/K7/8/8 b - - 0 1 
+8/8/3k4/3p4/3P4/3K4/8/8 b - - 0 1 
+8/3k4/3p4/8/3P4/3K4/8/8 b - - 0 1 
+8/8/3k4/3p4/8/3P4/3K4/8 b - - 0 1 
+k7/8/3p4/8/3P4/8/8/7K b - - 0 1 
+7k/3p4/8/8/3P4/8/8/K7 w - - 0 1 
+7k/8/8/3p4/8/8/3P4/K7 w - - 0 1 
+k7/8/8/7p/6P1/8/8/K7 w - - 0 1 
+k7/8/7p/8/8/6P1/8/K7 w - - 0 1 
+k7/8/8/6p1/7P/8/8/K7 w - - 0 1 
+k7/8/6p1/8/8/7P/8/K7 w - - 0 1 
+k7/8/8/3p4/4p3/8/8/7K w - - 0 1 
+k7/8/3p4/8/8/4P3/8/7K w - - 0 1 
+7k/3p4/8/8/3P4/8/8/K7 b - - 0 1 
+7k/8/8/3p4/8/8/3P4/K7 b - - 0 1 
+k7/8/8/7p/6P1/8/8/K7 b - - 0 1 
+k7/8/7p/8/8/6P1/8/K7 b - - 0 1 
+k7/8/8/6p1/7P/8/8/K7 b - - 0 1 
+k7/8/6p1/8/8/7P/8/K7 b - - 0 1 
+k7/8/8/3p4/4p3/8/8/7K b - - 0 1 
+k7/8/3p4/8/8/4P3/8/7K b - - 0 1 
+7k/8/8/p7/1P6/8/8/7K w - - 0 1 
+7k/8/p7/8/8/1P6/8/7K w - - 0 1 
+7k/8/8/1p6/P7/8/8/7K w - - 0 1 
+7k/8/1p6/8/8/P7/8/7K w - - 0 1 
+k7/7p/8/8/8/8/6P1/K7 w - - 0 1 
+k7/6p1/8/8/8/8/7P/K7 w - - 0 1 
+3k4/3pp3/8/8/8/8/3PP3/3K4 w - - 0 1 
+7k/8/8/p7/1P6/8/8/7K b - - 0 1 
+7k/8/p7/8/8/1P6/8/7K b - - 0 1 
+7k/8/8/1p6/P7/8/8/7K b - - 0 1 
+7k/8/1p6/8/8/P7/8/7K b - - 0 1 
+k7/7p/8/8/8/8/6P1/K7 b - - 0 1 
+k7/6p1/8/8/8/8/7P/K7 b - - 0 1 
+3k4/3pp3/8/8/8/8/3PP3/3K4 b - - 0 1 
+8/Pk6/8/8/8/8/6Kp/8 w - - 0 1 
+n1n5/1Pk5/8/8/8/8/5Kp1/5N1N w - - 0 1 
+8/PPPk4/8/8/8/8/4Kppp/8 w - - 0 1 
+n1n5/PPPk4/8/8/8/8/4Kppp/5N1N w - - 0 1 
+8/Pk6/8/8/8/8/6Kp/8 b - - 0 1 
+n1n5/1Pk5/8/8/8/8/5Kp1/5N1N b - - 0 1 
+8/PPPk4/8/8/8/8/4Kppp/8 b - - 0 1 
+n1n5/PPPk4/8/8/8/8/4Kppp/5N1N b - - 0 1 
+8/2p5/3p4/KP5r/1R3p1k/8/4P1P1/8 w - - 0 1 
+rnbqkb1r/ppppp1pp/7n/4Pp2/8/8/PPPP1PPP/RNBQKBNR w KQkq f6 0 3 

src/Chess/tests/suite.py

@@ -0,0 +1,43 @@
+import sys
+import timeit
+from pathlib import Path
+from argparse import Namespace, ArgumentParser
+from compare_positions import main as test
+
+
+
+def main(args: Namespace) -> int:
+    print("[S] Starting test suite")
+    successful = []
+    failed = []
+    positions = args.positions.read_text().splitlines()
+    start = timeit.default_timer()
+    longest_fen = max(sorted([len(fen) for fen in positions]))
+    for i, fen in enumerate(positions):
+        fen = fen.strip(" ")
+        fen += " " * (longest_fen - len(fen))
+        sys.stdout.write(f"\r[S] Testing {fen} ({i + 1}/{len(positions)})\033[K")
+        args.fen = fen
+        args.silent = not args.no_silent
+        if test(args) == 0:
+            successful.append(fen)
+        else:
+            failed.append(fen)
+    stop = timeit.default_timer()
+    print(f"\r[S] Ran {len(positions)} tests at depth {args.ply} in {stop - start:.2f} seconds ({len(successful)} successful, {len(failed)} failed)\033[K")
+    if failed and args.show_failures:
+        print("[S] The following FENs failed to pass the test:", end="")
+        print("\n\t".join(failed))
+
+
+if __name__ == "__main__":
+    parser = ArgumentParser(description="Run a set of tests using compare_positions.py")
+    parser.add_argument("--ply", "-d", type=int, required=True, help="The depth to stop at, expressed in plys (half-moves)")
+    parser.add_argument("--bulk", action="store_true", help="Enable bulk-counting for Nimfish (much faster)", default=False)
+    parser.add_argument("--stockfish", type=Path, help="Path to the stockfish executable. Defaults to '' (detected automatically)", default=None)
+    parser.add_argument("--nimfish", type=Path, help="Path to the nimfish executable. Defaults to '' (detected automatically)", default=None)
+    parser.add_argument("--positions", type=Path, help="Location of the file containing FENs to test, one per line. Defaults to 'tests/positions.txt'", 
+                        default=Path("tests/positions.txt"))
+    parser.add_argument("--no-silent", action="store_true", help="Do not suppress output from compare_positions.py (defaults)", default=False)
+    parser.add_argument("--show-failures", action="store_true", help="Show which FENs failed to pass the test", default=False)
+    sys.exit(main(parser.parse_args()))