Improve to testing interface and methodology

This commit is contained in:
Mattia Giambirtone 2023-10-30 14:46:27 +01:00
parent a9a9b917c6
commit 75869357cc
2 changed files with 343 additions and 113 deletions

View File

@ -19,6 +19,7 @@ import std/strutils
import std/strformat
import std/sequtils
type
PieceColor* = enum
@ -742,7 +743,19 @@ proc getCheckResolutions(self: ChessBoard, color: PieceColor): seq[Location] =
if not location.isValid():
break
result.add(location)
func getKing(self: ChessBoard, color: PieceColor): Location {.inline.} =
var color = color
if color == None:
color = self.getActiveColor()
case color:
of White:
return self.position.pieces.white.king
of Black:
return self.position.pieces.black.king
else:
discard
proc generatePawnMoves(self: ChessBoard, location: Location): seq[Move] =
@ -771,8 +784,26 @@ proc generatePawnMoves(self: ChessBoard, location: Location): seq[Move] =
for diagonal in [location + piece.color.topRightDiagonal(), location + piece.color.topLeftDiagonal()]:
if diagonal.isValid():
if diagonal == self.position.enPassantSquare.targetSquare:
locations.add(diagonal)
flags.add(EnPassant)
# Ensure en passant doesn't create a check
let king = self.getKing(piece.color)
var ok = true
if king.row == location.row:
var current = location + piece.color.rightSide()
while true:
current = current + piece.color.rightSide()
if not current.isValid():
break
let p = self.grid[current.row, current.col]
if p.color == piece.color:
break
if p.color == None:
continue
# Bishops can't create checks through en passant (I'm pretty sure at least)
if p.color == piece.color.opposite() and p.kind in [Queen, Rook]:
ok = false
if ok:
locations.add(diagonal)
flags.add(EnPassant)
elif self.grid[diagonal.row, diagonal.col].color == piece.color.opposite() and self.grid[diagonal.row, diagonal.col].kind != King:
locations.add(diagonal)
flags.add(Capture)
@ -1455,21 +1486,6 @@ proc doMove(self: ChessBoard, move: Move) =
move: emptyMove(),
pieces: previous.pieces,
)
if move.isPromotion():
# Move is a pawn promotion: get rid of the pawn
# and spawn a new piece
self.removePiece(move.startSquare)
case move.flag:
of PromoteToBishop:
self.spawnPiece(move.targetSquare, Piece(kind: Bishop, color: piece.color))
of PromoteToKnight:
self.spawnPiece(move.targetSquare, Piece(kind: Knight, color: piece.color))
of PromoteToRook:
self.spawnPiece(move.targetSquare, Piece(kind: Rook, color: piece.color))
of PromoteToQueen:
self.spawnPiece(move.targetSquare, Piece(kind: Queen, color: piece.color))
else:
discard
if move.flag in [CastleShort, CastleLong]:
# Move the rook onto the
# correct file
@ -1486,12 +1502,28 @@ proc doMove(self: ChessBoard, move: Move) =
let move = Move(startSquare: location, targetSquare: location + target, flag: move.flag)
self.movePiece(move, attack=false)
# Update position and attack metadata
self.updateLocations(move)
if move.flag == EnPassant:
self.removePiece(move.targetSquare + piece.color.bottomSide())
# Update position and attack metadata
self.updateLocations(move)
elif move.isPromotion():
# Move is a pawn promotion: get rid of the pawn
# and spawn a new piece
self.removePiece(move.targetSquare, attack=false)
case move.flag:
of PromoteToBishop:
self.spawnPiece(move.targetSquare, Piece(kind: Bishop, color: piece.color))
of PromoteToKnight:
self.spawnPiece(move.targetSquare, Piece(kind: Knight, color: piece.color))
of PromoteToRook:
self.spawnPiece(move.targetSquare, Piece(kind: Rook, color: piece.color))
of PromoteToQueen:
self.spawnPiece(move.targetSquare, Piece(kind: Queen, color: piece.color))
else:
discard
self.updateAttackedSquares()
# Check for double pawn push
if move.flag == DoublePush:
self.position.enPassantSquare = Move(startSquare: (move.startSquare.row, move.startSquare.col),
@ -1732,13 +1764,18 @@ proc toFEN*(self: ChessBoard): string =
result &= $self.getMoveCount()
proc perftBulkCount*(self: ChessBoard, ply: int, verbose: bool = false, divide: bool = false): int =
## Version of perft that implements bulk-counting.
## Only the total number of nodes reached after the
## given number of ply is returned
proc perft*(self: ChessBoard, ply: int, verbose: bool = false, divide: bool = false, bulk: bool = false): CountData =
## Counts (and debugs) the number of legal positions reached after
## the given number of ply
let moves = self.generateAllMoves()
if ply == 1:
return len(moves)
if not bulk and ply == 0:
return (1, 0, 0, 0, 0, 0, 0)
if bulk and ply == 1:
return (uint64(len(moves)), 0, 0, 0, 0, 0, 0)
if len(moves) == 0:
inc(result.checkmates)
for move in moves:
if verbose:
let canCastle = self.canCastle(self.getActiveColor())
@ -1747,99 +1784,160 @@ proc perftBulkCount*(self: ChessBoard, ply: int, verbose: bool = false, divide:
echo &"Turn: {self.getActiveColor()}"
echo &"Piece: {self.grid[move.startSquare.row, move.startSquare.col].kind}"
echo &"Flag: {move.flag}"
echo &"In check: {(if self.inCheck(self.getActiveColor()): \"yes\" else: \"no\")}"
echo &"In check: {(if self.inCheck(): \"yes\" else: \"no\")}"
echo &"Can castle:\n - King side: {(if canCastle.king: \"yes\" else: \"no\")}\n - Queen side: {(if canCastle.queen: \"yes\" else: \"no\")}"
echo &"Before: {self.toFEN()}\n"
echo self.pretty()
echo &"Position before move: {self.toFEN()}"
stdout.write("En Passant target: ")
if self.getEnPassantTarget() != emptyLocation():
echo self.getEnPassantTarget().locationToAlgebraic()
else:
echo "None"
echo "\n", self.pretty()
self.doMove(move)
case move.flag:
of Capture:
inc(result.captures)
of CastleShort, CastleLong:
inc(result.castles)
of PromoteToBishop, PromoteToKnight, PromoteToQueen, PromoteToRook:
inc(result.promotions)
of EnPassant:
inc(result.enPassant)
else:
discard
if self.inCheck():
# Opponent king is in check
inc(result.checks)
if verbose:
echo &"Now: {self.toFEN()}\n"
echo self.pretty()
let canCastle = self.canCastle(self.getActiveColor())
echo "\n"
echo &"Opponent in check: {(if self.inCheck(): \"yes\" else: \"no\")}"
echo &"Opponent can castle:\n - King side: {(if canCastle.king: \"yes\" else: \"no\")}\n - Queen side: {(if canCastle.queen: \"yes\" else: \"no\")}"
echo &"Position after move: {self.toFEN()}"
echo "\n", self.pretty()
stdout.write(">>> ")
try:
discard readLine(stdin)
except IOError:
discard
except EOFError:
discard
let next = self.perftBulkCount(ply - 1, verbose)
result += next
let next = self.perft(ply - 1, verbose, bulk=bulk)
if divide:
echo &"{move.startSquare.locationToAlgebraic()}{move.targetSquare.locationToAlgebraic()}: {next}"
var postfix = ""
case move.flag:
of PromoteToBishop:
postfix = "b"
of PromoteToKnight:
postfix = "n"
of PromoteToRook:
postfix = "r"
of PromoteToQueen:
postfix = "q"
else:
discard
echo &"{move.startSquare.locationToAlgebraic()}{move.targetSquare.locationToAlgebraic()}{postfix}: {next.nodes}"
if verbose:
echo ""
result.nodes += next.nodes
result.captures += next.captures
result.checks += next.checks
result.promotions += next.promotions
result.castles += next.castles
result.enPassant += next.enPassant
result.checkmates += next.checkmates
self.undoMove(move)
proc perft*(self: ChessBoard, ply: int, verbose: bool = false, divide: bool = false): CountData =
## Counts (and debugs) the number of legal positions reached after
## the given number of ply
var verbose = verbose
if ply == 0:
result = (1, 0, 0, 0, 0, 0, 0)
else:
let moves = self.generateAllMoves()
if len(moves) == 0:
inc(result.checkmates)
for move in moves:
if verbose:
let canCastle = self.canCastle(self.getActiveColor())
echo &"Ply: {self.position.plyFromRoot}"
echo &"Move: {move.startSquare.locationToAlgebraic()}{move.targetSquare.locationToAlgebraic()}, from ({move.startSquare.row}, {move.startSquare.col}) to ({move.targetSquare.row}, {move.targetSquare.col})"
echo &"Turn: {self.getActiveColor()}"
echo &"Piece: {self.grid[move.startSquare.row, move.startSquare.col].kind}"
echo &"Flag: {move.flag}"
echo &"In check: {(if self.inCheck(): \"yes\" else: \"no\")}"
echo &"Can castle:\n - King side: {(if canCastle.king: \"yes\" else: \"no\")}\n - Queen side: {(if canCastle.queen: \"yes\" else: \"no\")}"
echo &"Position before move: {self.toFEN()}"
stdout.write("En Passant target: ")
if self.getEnPassantTarget() != emptyLocation():
echo self.getEnPassantTarget().locationToAlgebraic()
else:
echo "None"
echo "\n", self.pretty()
self.doMove(move)
case move.flag:
of Capture:
inc(result.captures)
of CastleShort, CastleLong:
inc(result.castles)
of PromoteToBishop, PromoteToKnight, PromoteToQueen, PromoteToRook:
inc(result.promotions)
of EnPassant:
inc(result.enPassant)
else:
discard
if self.inCheck():
# Opponent king is in check
inc(result.checks)
if verbose:
let canCastle = self.canCastle(self.getActiveColor())
echo "\n"
echo &"Opponent in check: {(if self.inCheck(): \"yes\" else: \"no\")}"
echo &"Opponent can castle:\n - King side: {(if canCastle.king: \"yes\" else: \"no\")}\n - Queen side: {(if canCastle.queen: \"yes\" else: \"no\")}"
echo &"Position after move: {self.toFEN()}"
echo "\n", self.pretty()
stdout.write(">>> ")
try:
discard readLine(stdin)
except IOError:
discard
except EOFError:
discard
let next = self.perft(ply - 1, verbose)
if divide:
echo &"{move.startSquare.locationToAlgebraic()}{move.targetSquare.locationToAlgebraic()}: {next.nodes}"
if verbose:
echo ""
result.nodes += next.nodes
result.captures += next.captures
result.checks += next.checks
result.promotions += next.promotions
result.castles += next.castles
result.enPassant += next.enPassant
result.checkmates += next.checkmates
self.undoMove(move)
proc main: int =
echo "Nimfish by nocturn9x (see LICENSE)"
var board = newDefaultChessboard()
while true:
var cmd: seq[string]
try:
stdout.write(">>> ")
stdout.flushFile()
cmd = readLine(stdin).strip(leading=true, trailing=true, chars={'\t', ' '}).splitWhitespace(maxsplit=2)
case cmd[0]:
of "clear":
echo "\x1Bc"
of "help":
echo "TODO"
of "go":
if len(cmd) < 2:
echo &"Error: go: invalid number of arguments"
continue
case cmd[1]:
of "perft":
if len(cmd) == 2:
echo &"Error: go: perft: invalid number of arguments"
continue
var
args = cmd[2].splitWhitespace()
bulk = false
if args.len() > 1:
case args[1]:
of "bulk-count", "bulk":
bulk = true
else:
echo &"Error: go: perft: invalid argument '{args[1]}'"
continue
try:
let ply = parseInt(args[0])
if bulk:
echo &"\nNodes searched (bulk-counting enabled): {board.perft(ply, divide=true, bulk=true).nodes}\n"
else:
let data = board.perft(ply, divide=true)
echo &"\nNodes searched: {data.nodes}"
echo &" - Captures: {data.captures}"
echo &" - Checks: {data.checks}"
echo &" - E.P: {data.enPassant}"
echo &" - Checkmates: {data.checkmates}"
echo &" - Castles: {data.castles}"
echo &" - Promotions: {data.promotions}"
echo ""
except ValueError:
echo "Error: go: perft: invalid depth"
continue
else:
echo &"Error: go: unknown subcommand '{cmd[1]}'"
continue
of "position":
case len(cmd):
of 2:
case cmd[1]:
of "startpos":
board = newDefaultChessboard()
of "current", "cur":
echo &"Current position: {board.toFEN()}"
of "pretty":
echo board.pretty()
of "print", "show":
echo board
else:
echo &"Error: position: invalid argument '{cmd[1]}'"
continue
of 3:
case cmd[1]:
of "fen":
try:
board = newChessboardFromFEN(cmd[2])
except ValueError:
echo &"Error: position: invalid FEN string '{cmd[2]}': {getCurrentExceptionMsg()}"
else:
echo &"Error: position: unknown subcommand '{cmd[1]}'"
else:
echo &"Error: position: invalid number of arguments"
continue
else:
echo &"Unknown command '{cmd[0]}'. Type 'help' for more information."
except IOError:
echo ""
return -1
except EOFError:
echo ""
return 0
when isMainModule:
@ -1850,7 +1948,6 @@ when isMainModule:
let pieces = board.countPieces(kind, color)
doAssert pieces == count, &"expected {count} pieces of kind {kind} and color {color}, got {pieces} instead"
echo "Running tests"
var b = newDefaultChessboard()
# Ensure correct number of pieces
testPieceCount(b, Pawn, White, 8)
@ -1895,11 +1992,5 @@ when isMainModule:
# Queens
testPiece(b.getPiece("d1"), Queen, White)
testPiece(b.getPiece("d8"), Queen, Black)
when compileOption("profiler"):
import nimprof
b = newChessboardFromFEN("8/2p5/3p4/KP5r/1R3p1k/8/4P1P1/8 w - - ")
#echo b.perftBulkCount(4, divide=true)
echo b.perft(2, verbose=false, divide=true)
echo "All tests were successful"
setControlCHook(proc () {.noconv.} = quit(0))
quit(main())

View File

@ -0,0 +1,139 @@
import re
import sys
import time
import subprocess
from shutil import which
from pathlib import Path
from argparse import ArgumentParser, Namespace
def main(args: Namespace) -> int:
print("Nimfish move validator v0.0.1 by nocturn9x")
try:
STOCKFISH = (args.stockfish or Path(which("stockfish"))).resolve(strict=True)
except Exception as e:
print(f"Could not locate stockfish executable -> {type(e).__name__}: {e}")
return -1
try:
NIMFISH = (args.nimfish or (Path.cwd() / "nimfish")).resolve(strict=True)
except Exception as e:
print(f"Could not locate nimfish executable -> {type(e).__name__}: {e}")
return -1
print(f"Starting Stockfish engine at {STOCKFISH.as_posix()!r}")
stockfish_process = subprocess.Popen(STOCKFISH,
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT,
stdin=subprocess.PIPE,
encoding="u8"
)
print(f"Starting Nimfish engine at {NIMFISH.as_posix()!r}")
nimfish_process = subprocess.Popen(NIMFISH,
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT,
stdin=subprocess.PIPE,
encoding="u8")
print(f"Setting position to {args.fen!r}")
if args.fen:
nimfish_process.stdin.write(f"position fen {args.fen}\n")
stockfish_process.stdin.write(f"position fen {args.fen}\n")
else:
nimfish_process.stdin.write("position startpos\n")
stockfish_process.stdin.write("position startpos\n")
print(f"Engines started, beginning search to depth {args.ply}")
nimfish_process.stdin.write(f"go perft {args.ply} {'bulk' if args.bulk else ''}\n")
stockfish_process.stdin.write(f"go perft {args.ply}\n")
print("Search started, waiting for engine completion")
start_time = time.time()
stockfish_output = stockfish_process.communicate()[0]
stockfish_time = time.time() - start_time
start_time = time.time()
nimfish_output = nimfish_process.communicate()[0]
nimfish_time = time.time() - start_time
positions = {
"all": {},
"stockfish": {},
"nimfish": {}
}
pattern = re.compile(r"(?P<source>[a-h][1-8])(?P<target>[a-h][1-8])(?P<promotion>b|n|q|r)?:\s(?P<nodes>[0-9]+)", re.MULTILINE)
for (source, target, promotion, nodes) in pattern.findall(stockfish_output):
move = f"{source}{target}{promotion}"
positions["all"][move] = [int(nodes)]
positions["stockfish"][move] = int(nodes)
for (source, target, promotion, nodes) in pattern.findall(nimfish_output):
move = f"{source}{target}{promotion}"
positions["all"][move].append(int(nodes))
positions["nimfish"][move] = int(nodes)
differences = {
# Are in nimfish but not in stockfish
"nimfish": [],
# Are in stockfish but not in nimfish
"stockfish": []
}
for move, nodes in positions["all"].items():
if move in positions["stockfish"]:
if move not in positions["nimfish"] or positions["stockfish"][move] != positions["nimfish"][move]:
differences["stockfish"].append(move)
elif move in positions["nimfish"]:
if move not in positions["stockfish"] or positions["nimfish"][move] != positions["stockfish"][move]:
differences["nimfish"].append(move)
total_nodes = {"stockfish": sum(positions["stockfish"][move] for move in positions["stockfish"]),
"nimfish": sum(positions["nimfish"][move] for move in positions["nimfish"])}
total_difference = total_nodes["stockfish"] - total_nodes["nimfish"]
print(f"Stockfish searched {total_nodes['stockfish']} nodes in {stockfish_time:.2f} seconds")
print(f"Nimfish searched {total_nodes['nimfish']} nodes in {nimfish_time:.2f} seconds")
if total_difference > 0:
print(f"Stockfish searched {total_difference} more nodes than Nimfish")
elif total_difference != 0:
print(f"Nimfish searched {-total_difference} more nodes than Stockfish")
if differences["stockfish"] or differences["nimfish"]:
pattern = re.compile(r"(?:\s\s-\sCaptures:\s(?P<captures>[0-9]+))\n"
r"(?:\s\s-\sChecks:\s(?P<checks>[0-9]+))\n"
r"(?:\s\s-\sE\.P:\s(?P<enPassant>[0-9]+))\n"
r"(?:\s\s-\sCheckmates:\s(?P<checkmates>[0-9]+))\n"
r"(?:\s\s-\sCastles:\s(?P<castles>[0-9]+))\n"
r"(?:\s\s-\sPromotions:\s(?P<promotions>[0-9]+))",
re.MULTILINE)
data: re.Match | None = None
if args.bulk:
print("Note: Nimfish was run in bulk-counting mode, so a detailed breakdown of each move type is not available. "
"To fix this, re-run the program without the --bulk option")
else:
data = pattern.search(nimfish_output)
print(f"Found {len(differences)} mismatches, more info below: ")
if data:
print(f" - Breakdown by move type:")
print(f" - Captures: {data.group('captures')}")
print(f" - Checks: {data.group('checks')}")
print(f" - En Passant: {data.group('enPassant')}")
print(f" - Checkmates: {data.group('checkmates')}")
print(f" - Castles: {data.group('castles')}")
print(f" - Promotions: {data.group('promotions')}")
elif not args.bulk:
print("Unable to locate move breakdown in Nimfish output")
if differences["stockfish"]:
print(" - Legal moves missed by Nimfish: ")
for difference in differences["stockfish"]:
print(f" - {difference}: {positions['stockfish'][difference]}")
if differences["nimfish"]:
print(" - Illegal moves missed by Nimfish: ")
for difference in differences["stockfish"]:
print(f" - {difference}: {positions['nimfish'][difference]}")
else:
print("No mismatches in node count or moves were found")
if __name__ == "__main__":
parser = ArgumentParser(description="Automatically compare perft results between our engine and Stockfish")
parser.add_argument("--fen", "-f", type=str, default="", help="The FEN string of the position to start from (empty string means the initial one). Defaults to ''")
parser.add_argument("--ply", "-d", type=int, required=True, help="The depth to stop at, expressed in plys (half-moves)")
parser.add_argument("--bulk", action="store_true", help="Enable bulk-counting for Nimfish (faster, less debuggable)", default=False)
parser.add_argument("--stockfish", type=Path, help="Path to the stockfish executable. Defaults to '' (detected automatically)", default=None)
parser.add_argument("--nimfish", type=Path, help="Path to the nimfish executable. Defaults to 'nimfish'", default=Path("nimfish"))
sys.exit(main(parser.parse_args()))