Fix release versioning and unsupported bench checks (bench 5775715)

This commit is contained in:
2026-06-16 17:17:39 +02:00
parent 230dc3b492
commit 93348fd5f2
4 changed files with 655 additions and 352 deletions

View File

@@ -52,6 +52,8 @@ jobs:
- name: Build release binaries
id: build
env:
BUILD_REF: ${{ github.event.inputs.ref || github.ref }}
run: |
set -euo pipefail
@@ -70,17 +72,40 @@ jobs:
ld.lld --version
git lfs install --local
tag="${GITHUB_REF_NAME:-}"
ref="${BUILD_REF:-}"
ref_name="${GITHUB_REF_NAME:-}"
ref_type="${GITHUB_REF_TYPE:-}"
case "$ref" in
refs/tags/*)
ref_type="tag"
ref_name="${ref#refs/tags/}"
;;
refs/heads/*)
ref_type="branch"
ref_name="${ref#refs/heads/}"
;;
"")
;;
*)
ref_name="$ref"
;;
esac
version="$ref_name"
version="${version#v}"
make_args=()
if [[ "${GITHUB_REF_TYPE:-}" == "tag" && "$tag" =~ ^([0-9]+)\.([0-9]+)\.([0-9]+)$ ]]; then
if [[ "$version" =~ ^([0-9]+)\.([0-9]+)\.([0-9]+)$ ]]; then
echo "Using Makefile version ${BASH_REMATCH[1]}.${BASH_REMATCH[2]}.${BASH_REMATCH[3]} from ref ${ref_name}"
make_args+=(
"MAJOR_VERSION=${BASH_REMATCH[1]}"
"MINOR_VERSION=${BASH_REMATCH[2]}"
"PATCH_VERSION=${BASH_REMATCH[3]}"
)
else
echo "Using Makefile default version for ref ${ref_name:-<unknown>}"
fi
if [[ "${GITHUB_REF_TYPE:-}" == "tag" && ( "$tag" == *alpha* || "$tag" == *beta* || "$tag" == *rc* || "$tag" == *dev* ) ]]; then
if [[ "$ref_type" == "tag" && ( "$ref_name" == *alpha* || "$ref_name" == *beta* || "$ref_name" == *rc* || "$ref_name" == *dev* ) ]]; then
make prereleases
else
make ci-releases "${make_args[@]}"

View File

@@ -60,6 +60,8 @@ jobs:
- name: Build release binaries
id: build
env:
BUILD_REF: ${{ github.event.inputs.ref || github.ref }}
run: |
set -euo pipefail
@@ -79,17 +81,40 @@ jobs:
ld.lld --version
git lfs install --local
tag="${GITHUB_REF_NAME:-}"
ref="${BUILD_REF:-}"
ref_name="${GITHUB_REF_NAME:-}"
ref_type="${GITHUB_REF_TYPE:-}"
case "$ref" in
refs/tags/*)
ref_type="tag"
ref_name="${ref#refs/tags/}"
;;
refs/heads/*)
ref_type="branch"
ref_name="${ref#refs/heads/}"
;;
"")
;;
*)
ref_name="$ref"
;;
esac
version="$ref_name"
version="${version#v}"
make_args=()
if [[ "${GITHUB_REF_TYPE:-}" == "tag" && "$tag" =~ ^([0-9]+)\.([0-9]+)\.([0-9]+)$ ]]; then
if [[ "$version" =~ ^([0-9]+)\.([0-9]+)\.([0-9]+)$ ]]; then
echo "Using Makefile version ${BASH_REMATCH[1]}.${BASH_REMATCH[2]}.${BASH_REMATCH[3]} from ref ${ref_name}"
make_args+=(
"MAJOR_VERSION=${BASH_REMATCH[1]}"
"MINOR_VERSION=${BASH_REMATCH[2]}"
"PATCH_VERSION=${BASH_REMATCH[3]}"
)
else
echo "Using Makefile default version for ref ${ref_name:-<unknown>}"
fi
if [[ "${GITHUB_REF_TYPE:-}" == "tag" && ( "$tag" == *alpha* || "$tag" == *beta* || "$tag" == *rc* || "$tag" == *dev* ) ]]; then
if [[ "$ref_type" == "tag" && ( "$ref_name" == *alpha* || "$ref_name" == *beta* || "$ref_name" == *rc* || "$ref_name" == *dev* ) ]]; then
make prereleases
else
make ci-releases "${make_args[@]}"

View File

@@ -1,11 +1,24 @@
#!/usr/bin/env python3
import argparse
import os
import re
import signal
import shutil
import subprocess
import sys
import tempfile
import textwrap
from pathlib import Path
ILLEGAL_INSTRUCTION_EXIT_CODES = {
-signal.SIGILL,
128 + signal.SIGILL,
0xC000001D,
-1073741795,
}
def parse_args() -> argparse.Namespace:
parser = argparse.ArgumentParser(
description="Verify built binaries match the bench recorded in a commit message"
@@ -36,6 +49,182 @@ def expected_bench(commit: str) -> str:
return match.group(1)
def proc_cpuinfo_flags() -> set[str]:
cpuinfo = Path("/proc/cpuinfo")
if not cpuinfo.is_file():
return set()
flags: set[str] = set()
for line in cpuinfo.read_text(errors="ignore").splitlines():
key, sep, value = line.partition(":")
if sep == "" or key.strip().lower() not in {"flags", "features"}:
continue
flags.update(flag.lower() for flag in value.split())
return flags
def cpu_probe_compiler() -> str | None:
candidates = [os.environ.get("CC"), "cc", "clang", "gcc", "cl"]
for candidate in candidates:
if candidate and shutil.which(candidate):
return candidate
return None
def cpuid_probe_source() -> str:
return textwrap.dedent(
r"""
#include <stdio.h>
#if defined(__i386__) || defined(__x86_64__) || defined(_M_IX86) || defined(_M_X64)
#if defined(_MSC_VER)
#include <intrin.h>
static void cpuidex(unsigned leaf, unsigned subleaf, unsigned regs[4]) {
int out[4];
__cpuidex(out, (int)leaf, (int)subleaf);
regs[0] = (unsigned)out[0];
regs[1] = (unsigned)out[1];
regs[2] = (unsigned)out[2];
regs[3] = (unsigned)out[3];
}
static unsigned long long xgetbv0(void) {
return _xgetbv(0);
}
#else
#include <cpuid.h>
static void cpuidex(unsigned leaf, unsigned subleaf, unsigned regs[4]) {
__cpuid_count(leaf, subleaf, regs[0], regs[1], regs[2], regs[3]);
}
static unsigned long long xgetbv0(void) {
unsigned eax, edx;
__asm__ volatile("xgetbv" : "=a"(eax), "=d"(edx) : "c"(0));
return ((unsigned long long)edx << 32) | eax;
}
#endif
#endif
int main(void) {
#if defined(__i386__) || defined(__x86_64__) || defined(_M_IX86) || defined(_M_X64)
unsigned regs[4] = {0, 0, 0, 0};
cpuidex(1, 0, regs);
unsigned leaf1_ecx = regs[2];
int osxsave = (regs[2] & (1u << 27)) != 0;
if (!osxsave) {
return 0;
}
unsigned long long xcr0 = xgetbv0();
int avx_state = (xcr0 & 0x6u) == 0x6u;
int avx512_state = (xcr0 & 0xe6u) == 0xe6u;
cpuidex(7, 0, regs);
if (avx_state && (regs[1] & (1u << 5))) {
puts("avx2");
}
if (avx_state && (leaf1_ecx & (1u << 12))) {
puts("fma");
}
if (regs[1] & (1u << 3)) {
puts("bmi1");
}
if (regs[1] & (1u << 8)) {
puts("bmi2");
}
if (avx512_state && (regs[1] & (1u << 16))) {
puts("avx512f");
}
if (avx512_state && (regs[1] & (1u << 17))) {
puts("avx512dq");
}
if (avx512_state && (regs[1] & (1u << 28))) {
puts("avx512cd");
}
if (avx512_state && (regs[1] & (1u << 30))) {
puts("avx512bw");
}
if (avx512_state && (regs[1] & (1u << 31))) {
puts("avx512vl");
}
if (avx512_state && (regs[2] & (1u << 11))) {
puts("avx512vnni");
}
#endif
return 0;
}
"""
)
def cpuid_probe_flags() -> set[str]:
compiler = cpu_probe_compiler()
if compiler is None:
return set()
with tempfile.TemporaryDirectory() as tempdir:
temp = Path(tempdir)
source = temp / "cpu_probe.c"
binary = temp / ("cpu_probe.exe" if sys.platform.startswith(("win", "msys", "cygwin")) else "cpu_probe")
source.write_text(cpuid_probe_source())
compile_cmd = [compiler, str(source), "-O2", "-o", str(binary)]
if Path(compiler).name.lower() == "cl":
compile_cmd = [compiler, "/nologo", "/O2", str(source), f"/Fe:{binary}"]
try:
subprocess.run(
compile_cmd,
check=True,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
text=True,
)
result = subprocess.run(
[str(binary)],
check=True,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
text=True,
)
except (OSError, subprocess.CalledProcessError):
return set()
return set(result.stdout.split())
def host_cpu_flags() -> set[str]:
flags = cpuid_probe_flags()
if flags:
return flags
return proc_cpuinfo_flags()
def binary_cpu_requirements(binary: Path) -> set[str]:
name = binary.name.lower()
avx512_v4 = {"avx512f", "avx512bw", "avx512cd", "avx512dq", "avx512vl"}
if "-vnni" in name:
return avx512_v4 | {"avx512vnni"}
if "-avx512" in name:
return avx512_v4
if "-haswell" in name or "-zen2" in name:
return {"avx2", "fma", "bmi1", "bmi2"}
return set()
def unsupported_reason(binary: Path, flags: set[str]) -> str | None:
required = binary_cpu_requirements(binary)
if not required or not flags:
return None
missing = sorted(required - flags)
if missing:
return "host CPU is missing " + ", ".join(missing)
return None
def is_illegal_instruction(returncode: int) -> bool:
return returncode in ILLEGAL_INSTRUCTION_EXIT_CODES
def actual_bench(binary: Path, depth: int) -> tuple[int, str | None, str]:
result = subprocess.run(
[str(binary), "bench", str(depth), "-s"],
@@ -60,6 +249,7 @@ def main() -> int:
return 1
status = 0
flags = host_cpu_flags()
for binary_name in args.binaries:
binary = Path(binary_name)
if not binary.is_file():
@@ -67,6 +257,11 @@ def main() -> int:
status = 1
continue
reason = unsupported_reason(binary, flags)
if reason is not None:
print(f"Skipping {binary}: {reason}")
continue
print(f"Checking {binary} against bench {expected}")
try:
returncode, actual, output = actual_bench(binary, args.depth)
@@ -76,6 +271,9 @@ def main() -> int:
continue
if returncode != 0:
if is_illegal_instruction(returncode):
print(f"Skipping {binary}: failed with illegal instruction exit code {returncode}")
continue
print(
f"Error: '{binary} bench {args.depth} -s' failed with exit code {returncode}",
file=sys.stderr,

View File

@@ -1,425 +1,480 @@
# Copyright 2026 Mattia Giambirtone & All Contributors
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
when defined(linux):
# Copyright 2026 Mattia Giambirtone & All Contributors
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
## NUMA topology detection and best-effort thread binding.
##
## Adapted from Soul's NUMA support:
## - https://github.com/Aethdv/Soul/blob/soul/src/numa.rs
##
## Shamelessly yoinked with GPT 5.5 <3
## NUMA topology detection and best-effort thread binding.
##
## Adapted from Soul's NUMA support:
## - https://github.com/Aethdv/Soul/blob/soul/src/numa.rs
##
## Shamelessly yoinked with GPT 5.5 <3
import std/[cpuinfo, options, strformat, strutils]
import std/[cpuinfo, options, strformat, strutils]
type
CPU = int
type
CPU = int
CPUMask = object
words: array[64, uint64]
wordCount: int
cpuCount: int
CPUMask = object
words: array[64, uint64]
wordCount: int
cpuCount: int
NUMATopology* = object
## The machine's memory and cache locality domains, filtered to CPUs the
## current process is allowed to run on.
nodes: seq[seq[CPU]]
domains: seq[seq[CPU]]
NUMATopology* = object
## The machine's memory and cache locality domains, filtered to CPUs the
## current process is allowed to run on.
nodes: seq[seq[CPU]]
domains: seq[seq[CPU]]
NUMABinding = object
## Fixed-size, unmanaged view of the detected topology for GC-safe thread
## binding.
nodeCount: int
domainCount: int
nodes: array[256, CPUMask]
domains: array[256, CPUMask]
NUMABinding = object
## Fixed-size, unmanaged view of the detected topology for GC-safe thread
## binding.
nodeCount: int
domainCount: int
nodes: array[256, CPUMask]
domains: array[256, CPUMask]
{.emit: """
#include <stdint.h>
{.emit: """
#include <stdint.h>
#if defined(__linux__)
#include <unistd.h>
#include <sys/syscall.h>
#endif
NIM_EXTERNC int heimdall_process_affinity(uint64_t *mask, int words) {
#if defined(__linux__) && defined(SYS_sched_getaffinity)
long ret = syscall(SYS_sched_getaffinity, 0, (size_t)words * sizeof(uint64_t), mask);
return ret >= 0;
#else
(void)mask;
(void)words;
return 0;
#endif
}
NIM_EXTERNC int heimdall_bind_thread(const uint64_t *mask, int words) {
#if defined(__linux__) && defined(SYS_sched_setaffinity)
long ret = syscall(SYS_sched_setaffinity, 0, (size_t)words * sizeof(uint64_t), mask);
if (ret != 0) {
return 0;
}
#if defined(SYS_sched_yield)
(void)syscall(SYS_sched_yield);
#if defined(__linux__)
#include <unistd.h>
#include <sys/syscall.h>
#endif
return 1;
#else
(void)mask;
(void)words;
return 0;
#endif
}
""".}
NIM_EXTERNC int heimdall_process_affinity(uint64_t *mask, int words) {
#if defined(__linux__) && defined(SYS_sched_getaffinity)
long ret = syscall(SYS_sched_getaffinity, 0, (size_t)words * sizeof(uint64_t), mask);
return ret >= 0;
#else
(void)mask;
(void)words;
return 0;
#endif
}
NIM_EXTERNC int heimdall_bind_thread(const uint64_t *mask, int words) {
#if defined(__linux__) && defined(SYS_sched_setaffinity)
long ret = syscall(SYS_sched_setaffinity, 0, (size_t)words * sizeof(uint64_t), mask);
if (ret != 0) {
return 0;
}
#if defined(SYS_sched_yield)
(void)syscall(SYS_sched_yield);
#endif
return 1;
#else
(void)mask;
(void)words;
return 0;
#endif
}
""".}
proc heimdallProcessAffinity(mask: ptr uint64, words: cint): cint {.importc: "heimdall_process_affinity", nodecl, gcsafe.}
proc heimdallBindThread(mask: ptr uint64, words: cint): cint {.importc: "heimdall_bind_thread", nodecl, gcsafe.}
proc heimdallProcessAffinity(mask: ptr uint64, words: cint): cint {.importc: "heimdall_process_affinity", nodecl, gcsafe.}
proc heimdallBindThread(mask: ptr uint64, words: cint): cint {.importc: "heimdall_bind_thread", nodecl, gcsafe.}
func numNodes*(self: NUMATopology): int {.inline.} = self.nodes.len()
func numDomains*(self: NUMATopology): int {.inline.} = self.domains.len()
func numNodes*(self: NUMATopology): int {.inline.} = self.nodes.len()
func numDomains*(self: NUMATopology): int {.inline.} = self.domains.len()
func shouldBind*(self: NUMATopology, threads: int): bool {.inline.} =
## Binding pays only when multiple search threads can be spread over
## multiple cache domains.
self.domains.len() > 1 and threads > 1
func shouldBind*(self: NUMATopology, threads: int): bool {.inline.} =
## Binding pays only when multiple search threads can be spread over
## multiple cache domains.
self.domains.len() > 1 and threads > 1
func shouldDistribute*(self: NUMATopology, threads: int): bool {.inline.} =
## A lone search thread wants local TT memory. Multi-threaded searches on
## multi-node systems benefit from striped first-touch placement.
self.nodes.len() > 1 and threads > 1
func shouldDistribute*(self: NUMATopology, threads: int): bool {.inline.} =
## A lone search thread wants local TT memory. Multi-threaded searches on
## multi-node systems benefit from striped first-touch placement.
self.nodes.len() > 1 and threads > 1
func parseCPUList*(s: string): seq[CPU] =
## Parses Linux cpulist syntax such as "0-15,128-143" or "0,2,4".
for part in s.split(','):
let part = part.strip()
if part.len() == 0:
continue
let bounds = part.split('-', maxsplit=1)
if bounds.len() == 2:
try:
let
lo = bounds[0].parseInt()
hi = bounds[1].parseInt()
if lo <= hi:
for cpu in lo..hi:
result.add(cpu)
except ValueError:
discard
else:
try:
result.add(part.parseInt())
except ValueError:
discard
func parseCPUList*(s: string): seq[CPU] =
## Parses Linux cpulist syntax such as "0-15,128-143" or "0,2,4".
for part in s.split(','):
let part = part.strip()
if part.len() == 0:
continue
let bounds = part.split('-', maxsplit=1)
if bounds.len() == 2:
try:
let
lo = bounds[0].parseInt()
hi = bounds[1].parseInt()
if lo <= hi:
for cpu in lo..hi:
result.add(cpu)
except ValueError:
discard
else:
try:
result.add(part.parseInt())
except ValueError:
discard
func fill(occupied: int, domain: seq[CPU]): float {.inline.} =
(occupied + 1).float / max(domain.len(), 1).float
func fill(occupied: int, domain: seq[CPU]): float {.inline.} =
(occupied + 1).float / max(domain.len(), 1).float
func distribute*(self: NUMATopology, threads: int): seq[int] =
## Assigns search threads to L3 domains, balancing by occupied/available CPU
## ratio so larger domains naturally receive more threads.
var occupied = newSeq[int](max(self.domains.len(), 1))
result = newSeqOfCap[int](threads)
for _ in 0..<threads:
var pick = 0
if self.domains.len() > 0:
var best = fill(occupied[0], self.domains[0])
for i in 1..<self.domains.len():
let candidate = fill(occupied[i], self.domains[i])
if candidate < best:
pick = i
best = candidate
inc(occupied[pick])
result.add(pick)
func distribute*(self: NUMATopology, threads: int): seq[int] =
## Assigns search threads to L3 domains, balancing by occupied/available CPU
## ratio so larger domains naturally receive more threads.
var occupied = newSeq[int](max(self.domains.len(), 1))
result = newSeqOfCap[int](threads)
for _ in 0..<threads:
var pick = 0
if self.domains.len() > 0:
var best = fill(occupied[0], self.domains[0])
for i in 1..<self.domains.len():
let candidate = fill(occupied[i], self.domains[i])
if candidate < best:
pick = i
best = candidate
inc(occupied[pick])
result.add(pick)
proc readTrimmed(path: string): Option[string] =
try:
return some(readFile(path).strip())
except OSError:
return none(string)
proc readTrimmed(path: string): Option[string] =
try:
return some(readFile(path).strip())
except OSError:
return none(string)
proc processAffinity: Option[seq[CPU]] =
var mask: array[64, uint64]
if heimdallProcessAffinity(addr mask[0], mask.len().cint) == 0:
return none(seq[CPU])
proc processAffinity: Option[seq[CPU]] =
var mask: array[64, uint64]
if heimdallProcessAffinity(addr mask[0], mask.len().cint) == 0:
return none(seq[CPU])
var cpus: seq[CPU] = @[]
for word, bits in mask:
for bit in 0..<64:
if (bits and (1'u64 shl bit)) != 0:
cpus.add(word * 64 + bit)
some(cpus)
proc allowedCPUs: seq[CPU] =
let affinity = processAffinity()
if affinity.isSome() and affinity.get().len() > 0:
return affinity.get()
let online = readTrimmed("/sys/devices/system/cpu/online")
if online.isSome():
let cpus = parseCPUList(online.get())
if cpus.len() > 0:
return cpus
let count = countProcessors()
for cpu in 0..<max(count, 1):
result.add(cpu)
proc readNUMANodes(allowed: seq[CPU]): Option[seq[seq[CPU]]] =
let online = readTrimmed("/sys/devices/system/node/online")
if online.isNone():
return none(seq[seq[CPU]])
var nodes: seq[seq[CPU]] = @[]
for node in parseCPUList(online.get()):
let cpulist = readTrimmed(&"/sys/devices/system/node/node{node}/cpulist")
if cpulist.isNone():
return none(seq[seq[CPU]])
var cpus: seq[CPU] = @[]
for cpu in parseCPUList(cpulist.get()):
if cpu in allowed:
cpus.add(cpu)
if cpus.len() > 0:
nodes.add(cpus)
if nodes.len() == 0:
return none(seq[seq[CPU]])
some(nodes)
for word, bits in mask:
for bit in 0..<64:
if (bits and (1'u64 shl bit)) != 0:
cpus.add(word * 64 + bit)
some(cpus)
proc readL3Siblings(cpu: CPU): Option[string] =
for index in 0..<8:
let base = &"/sys/devices/system/cpu/cpu{cpu}/cache/index{index}"
let level = readTrimmed(&"{base}/level")
if level.isNone():
break
if level.get() == "3":
return readTrimmed(&"{base}/shared_cpu_list")
none(string)
proc allowedCPUs: seq[CPU] =
let affinity = processAffinity()
if affinity.isSome() and affinity.get().len() > 0:
return affinity.get()
let online = readTrimmed("/sys/devices/system/cpu/online")
if online.isSome():
let cpus = parseCPUList(online.get())
if cpus.len() > 0:
return cpus
let count = countProcessors()
for cpu in 0..<max(count, 1):
result.add(cpu)
proc readL3Domains(allowed: seq[CPU]): Option[seq[seq[CPU]]] =
var ceiling = 0
for cpu in allowed:
ceiling = max(ceiling, cpu + 1)
var
grouped = newSeq[bool](ceiling)
domains: seq[seq[CPU]] = @[]
for cpu in allowed:
if cpu < grouped.len() and grouped[cpu]:
continue
let shared = readL3Siblings(cpu)
if shared.isNone():
proc readNUMANodes(allowed: seq[CPU]): Option[seq[seq[CPU]]] =
let online = readTrimmed("/sys/devices/system/node/online")
if online.isNone():
return none(seq[seq[CPU]])
var group: seq[CPU] = @[]
for sibling in parseCPUList(shared.get()):
if sibling in allowed:
group.add(sibling)
if sibling < grouped.len():
grouped[sibling] = true
var nodes: seq[seq[CPU]] = @[]
for node in parseCPUList(online.get()):
let cpulist = readTrimmed(&"/sys/devices/system/node/node{node}/cpulist")
if cpulist.isNone():
return none(seq[seq[CPU]])
var cpus: seq[CPU] = @[]
for cpu in parseCPUList(cpulist.get()):
if cpu in allowed:
cpus.add(cpu)
if cpus.len() > 0:
nodes.add(cpus)
if group.len() > 0:
domains.add(group)
if domains.len() == 0:
return none(seq[seq[CPU]])
some(domains)
if nodes.len() == 0:
return none(seq[seq[CPU]])
some(nodes)
proc detectNUMATopology*: NUMATopology =
## Detects NUMA memory nodes and L3 cache domains from Linux /sys. If any
## required topology read fails, the engine falls back to a single domain.
let allowed = allowedCPUs()
# ♪ numa numa numa iei ♪
let nodes = readNUMANodes(allowed)
if nodes.isSome():
result.nodes = nodes.get()
else:
result.nodes = @[allowed]
let domains = readL3Domains(allowed)
if domains.isSome():
result.domains = domains.get()
else:
result.domains = result.nodes
proc readL3Siblings(cpu: CPU): Option[string] =
for index in 0..<8:
let base = &"/sys/devices/system/cpu/cpu{cpu}/cache/index{index}"
let level = readTrimmed(&"{base}/level")
if level.isNone():
break
if level.get() == "3":
return readTrimmed(&"{base}/shared_cpu_list")
none(string)
proc bindThread(cpus: seq[CPU]): bool =
if cpus.len() == 0:
return false
proc readL3Domains(allowed: seq[CPU]): Option[seq[seq[CPU]]] =
var ceiling = 0
for cpu in allowed:
ceiling = max(ceiling, cpu + 1)
var highest = 0
for cpu in cpus:
highest = max(highest, cpu)
var
grouped = newSeq[bool](ceiling)
domains: seq[seq[CPU]] = @[]
let words = highest div 64 + 1
var mask = newSeq[uint64](words)
for cpu in cpus:
if cpu >= 0:
mask[cpu div 64] = mask[cpu div 64] or (1'u64 shl (cpu mod 64))
for cpu in allowed:
if cpu < grouped.len() and grouped[cpu]:
continue
let shared = readL3Siblings(cpu)
if shared.isNone():
return none(seq[seq[CPU]])
heimdallBindThread(addr mask[0], words.cint) != 0
var group: seq[CPU] = @[]
for sibling in parseCPUList(shared.get()):
if sibling in allowed:
group.add(sibling)
if sibling < grouped.len():
grouped[sibling] = true
if group.len() > 0:
domains.add(group)
if domains.len() == 0:
return none(seq[seq[CPU]])
some(domains)
proc bindToDomain*(self: NUMATopology, domain: int): bool =
if domain notin 0..<self.domains.len():
return false
bindThread(self.domains[domain])
proc detectNUMATopology*: NUMATopology =
## Detects NUMA memory nodes and L3 cache domains from Linux /sys. If any
## required topology read fails, the engine falls back to a single domain.
let allowed = allowedCPUs()
# ♪ numa numa numa iei ♪
let nodes = readNUMANodes(allowed)
if nodes.isSome():
result.nodes = nodes.get()
else:
result.nodes = @[allowed]
let domains = readL3Domains(allowed)
if domains.isSome():
result.domains = domains.get()
else:
result.domains = result.nodes
proc bindToNode*(self: NUMATopology, node: int): bool =
if node notin 0..<self.nodes.len():
return false
bindThread(self.nodes[node])
proc bindThread(cpus: seq[CPU]): bool =
if cpus.len() == 0:
return false
func toCPUMask(cpus: seq[CPU]): CPUMask =
var highest = -1
for cpu in cpus:
if cpu in 0..<result.words.len() * 64:
result.words[cpu div 64] = result.words[cpu div 64] or (1'u64 shl (cpu mod 64))
result.cpuCount += 1
var highest = 0
for cpu in cpus:
highest = max(highest, cpu)
result.wordCount = max(highest div 64 + 1, 1)
let words = highest div 64 + 1
var mask = newSeq[uint64](words)
for cpu in cpus:
if cpu >= 0:
mask[cpu div 64] = mask[cpu div 64] or (1'u64 shl (cpu mod 64))
heimdallBindThread(addr mask[0], words.cint) != 0
proc detectNUMABinding: NUMABinding =
let topology = detectNUMATopology()
result.nodeCount = min(topology.nodes.len(), result.nodes.len())
result.domainCount = min(topology.domains.len(), result.domains.len())
for i in 0..<result.nodeCount:
result.nodes[i] = toCPUMask(topology.nodes[i])
for i in 0..<result.domainCount:
result.domains[i] = toCPUMask(topology.domains[i])
proc bindToDomain*(self: NUMATopology, domain: int): bool =
if domain notin 0..<self.domains.len():
return false
bindThread(self.domains[domain])
let detectedBinding = detectNUMABinding()
proc bindToNode*(self: NUMATopology, node: int): bool =
if node notin 0..<self.nodes.len():
return false
bindThread(self.nodes[node])
func maskCPUCount(mask: CPUMask): int {.inline.} = max(mask.cpuCount, 1)
func toCPUMask(cpus: seq[CPU]): CPUMask =
var highest = -1
for cpu in cpus:
if cpu in 0..<result.words.len() * 64:
result.words[cpu div 64] = result.words[cpu div 64] or (1'u64 shl (cpu mod 64))
result.cpuCount += 1
highest = max(highest, cpu)
result.wordCount = max(highest div 64 + 1, 1)
proc bindMask(mask: ptr CPUMask): bool {.gcsafe.} =
if mask == nil or mask.cpuCount == 0:
return false
heimdallBindThread(unsafeAddr mask.words[0], mask.wordCount.cint) != 0
proc detectNUMABinding: NUMABinding =
let topology = detectNUMATopology()
result.nodeCount = min(topology.nodes.len(), result.nodes.len())
result.domainCount = min(topology.domains.len(), result.domains.len())
for i in 0..<result.nodeCount:
result.nodes[i] = toCPUMask(topology.nodes[i])
for i in 0..<result.domainCount:
result.domains[i] = toCPUMask(topology.domains[i])
proc NUMANodeCount*: int {.inline, gcsafe.} = detectedBinding.nodeCount
proc NUMADomainCount*: int {.inline, gcsafe.} = detectedBinding.domainCount
proc NUMAShouldBind*(threads: int): bool {.inline, gcsafe.} = detectedBinding.domainCount > 1 and threads > 1
proc NUMAShouldDistribute*(threads: int): bool {.inline, gcsafe.} = detectedBinding.nodeCount > 1 and threads > 1
let detectedBinding = detectNUMABinding()
proc NUMANodeForThread*(threadId, threads: int): int {.gcsafe.} =
## Returns the NUMA memory node assignment for an init worker without allocating.
if threadId notin 0..<threads or detectedBinding.nodeCount == 0:
return -1
var occupied: array[256, int]
for t in 0..threadId:
var pick = 0
for node in 1..<detectedBinding.nodeCount:
let
lhs = (occupied[node] + 1) * detectedBinding.nodes[pick].maskCPUCount()
rhs = (occupied[pick] + 1) * detectedBinding.nodes[node].maskCPUCount()
if lhs < rhs:
pick = node
inc(occupied[pick])
if t == threadId:
return pick
-1
func maskCPUCount(mask: CPUMask): int {.inline.} = max(mask.cpuCount, 1)
proc NUMADomainForThread*(threadId, threads: int): int {.gcsafe.} =
## Returns the L3 domain assignment for a search thread without allocating.
if threadId notin 0..<threads or detectedBinding.domainCount == 0:
return -1
var occupied: array[256, int]
for t in 0..threadId:
var pick = 0
for domain in 1..<detectedBinding.domainCount:
let
lhs = (occupied[domain] + 1) * detectedBinding.domains[pick].maskCPUCount()
rhs = (occupied[pick] + 1) * detectedBinding.domains[domain].maskCPUCount()
if lhs < rhs:
pick = domain
inc(occupied[pick])
if t == threadId:
return pick
-1
proc bindMask(mask: ptr CPUMask): bool {.gcsafe.} =
if mask == nil or mask.cpuCount == 0:
return false
heimdallBindThread(unsafeAddr mask.words[0], mask.wordCount.cint) != 0
proc bindToNUMADomain*(domain: int): bool {.inline, gcsafe.} =
if domain notin 0..<detectedBinding.domainCount:
return false
bindMask(unsafeAddr detectedBinding.domains[domain])
proc NUMANodeCount*: int {.inline, gcsafe.} = detectedBinding.nodeCount
proc NUMADomainCount*: int {.inline, gcsafe.} = detectedBinding.domainCount
proc NUMAShouldBind*(threads: int): bool {.inline, gcsafe.} = detectedBinding.domainCount > 1 and threads > 1
proc NUMAShouldDistribute*(threads: int): bool {.inline, gcsafe.} = detectedBinding.nodeCount > 1 and threads > 1
proc bindToNUMANode*(node: int): bool {.inline, gcsafe.} =
if node notin 0..<detectedBinding.nodeCount:
return false
bindMask(unsafeAddr detectedBinding.nodes[node])
proc NUMANodeForThread*(threadId, threads: int): int {.gcsafe.} =
## Returns the NUMA memory node assignment for an init worker without allocating.
if threadId notin 0..<threads or detectedBinding.nodeCount == 0:
return -1
var occupied: array[256, int]
for t in 0..threadId:
var pick = 0
for node in 1..<detectedBinding.nodeCount:
let
lhs = (occupied[node] + 1) * detectedBinding.nodes[pick].maskCPUCount()
rhs = (occupied[pick] + 1) * detectedBinding.nodes[node].maskCPUCount()
if lhs < rhs:
pick = node
inc(occupied[pick])
if t == threadId:
return pick
-1
func CPUList(first, last: CPU): seq[CPU] =
for cpu in first..last:
result.add(cpu)
proc NUMADomainForThread*(threadId, threads: int): int {.gcsafe.} =
## Returns the L3 domain assignment for a search thread without allocating.
if threadId notin 0..<threads or detectedBinding.domainCount == 0:
return -1
var occupied: array[256, int]
for t in 0..threadId:
var pick = 0
for domain in 1..<detectedBinding.domainCount:
let
lhs = (occupied[domain] + 1) * detectedBinding.domains[pick].maskCPUCount()
rhs = (occupied[pick] + 1) * detectedBinding.domains[domain].maskCPUCount()
if lhs < rhs:
pick = domain
inc(occupied[pick])
if t == threadId:
return pick
-1
func countAssignments(assignment: seq[int], domain: int): int =
for assigned in assignment:
if assigned == domain:
inc(result)
proc bindToNUMADomain*(domain: int): bool {.inline, gcsafe.} =
if domain notin 0..<detectedBinding.domainCount:
return false
bindMask(unsafeAddr detectedBinding.domains[domain])
proc basicTests* =
## Ported from Soul's NUMA test suite:
## https://github.com/Aethdv/Soul/blob/soul/src/numa.rs
doAssert parseCPUList("0-3") == @[0, 1, 2, 3]
doAssert parseCPUList("0,2,4") == @[0, 2, 4]
doAssert parseCPUList("") == @[]
proc bindToNUMANode*(node: int): bool {.inline, gcsafe.} =
if node notin 0..<detectedBinding.nodeCount:
return false
bindMask(unsafeAddr detectedBinding.nodes[node])
# EPYC 9654 node 0: physical cores plus their SMT siblings, two blocks.
let cpus = parseCPUList("0-95,192-287")
doAssert cpus.len() == 192
doAssert cpus[0] == 0
doAssert cpus[^1] == 287
doAssert 96 notin cpus
let balanced = NUMATopology(nodes: @[CPUList(0, 31)], domains: @[CPUList(0, 15), CPUList(16, 31)])
let assignment = balanced.distribute(4)
doAssert countAssignments(assignment, 0) == 2
doAssert countAssignments(assignment, 1) == 2
func CPUList(first, last: CPU): seq[CPU] =
for cpu in first..last:
result.add(cpu)
let single = NUMATopology(nodes: @[CPUList(0, 7)], domains: @[CPUList(0, 7)])
doAssert not single.shouldBind(8)
doAssert single.distribute(4) == @[0, 0, 0, 0]
let topology = detectNUMATopology()
doAssert topology.numNodes() >= 1
doAssert topology.numDomains() >= 1
func countAssignments(assignment: seq[int], domain: int): int =
for assigned in assignment:
if assigned == domain:
inc(result)
proc basicTests* =
## Ported from Soul's NUMA test suite:
## https://github.com/Aethdv/Soul/blob/soul/src/numa.rs
doAssert parseCPUList("0-3") == @[0, 1, 2, 3]
doAssert parseCPUList("0,2,4") == @[0, 2, 4]
doAssert parseCPUList("") == @[]
# EPYC 9654 node 0: physical cores plus their SMT siblings, two blocks.
let cpus = parseCPUList("0-95,192-287")
doAssert cpus.len() == 192
doAssert cpus[0] == 0
doAssert cpus[^1] == 287
doAssert 96 notin cpus
let balanced = NUMATopology(nodes: @[CPUList(0, 31)], domains: @[CPUList(0, 15), CPUList(16, 31)])
let assignment = balanced.distribute(4)
doAssert countAssignments(assignment, 0) == 2
doAssert countAssignments(assignment, 1) == 2
let single = NUMATopology(nodes: @[CPUList(0, 7)], domains: @[CPUList(0, 7)])
doAssert not single.shouldBind(8)
doAssert single.distribute(4) == @[0, 0, 0, 0]
let topology = detectNUMATopology()
doAssert topology.numNodes() >= 1
doAssert topology.numDomains() >= 1
else:
type
NUMATopology* = object
func numNodes*(self: NUMATopology): int {.inline.} = 1
func numDomains*(self: NUMATopology): int {.inline.} = 1
func shouldBind*(self: NUMATopology, threads: int): bool {.inline.} = false
func shouldDistribute*(self: NUMATopology, threads: int): bool {.inline.} = false
func parseCPUList*(s: string): seq[int] = @[]
func distribute*(self: NUMATopology, threads: int): seq[int] =
result = newSeq[int](threads)
proc detectNUMATopology*: NUMATopology =
discard
proc bindToDomain*(self: NUMATopology, domain: int): bool =
false
proc bindToNode*(self: NUMATopology, node: int): bool =
false
proc NUMANodeCount*: int {.inline, gcsafe.} = 1
proc NUMADomainCount*: int {.inline, gcsafe.} = 1
proc NUMAShouldBind*(threads: int): bool {.inline, gcsafe.} = false
proc NUMAShouldDistribute*(threads: int): bool {.inline, gcsafe.} = false
proc NUMANodeForThread*(threadId, threads: int): int {.gcsafe.} =
-1
proc NUMADomainForThread*(threadId, threads: int): int {.gcsafe.} =
-1
proc bindToNUMADomain*(domain: int): bool {.inline, gcsafe.} = false
proc bindToNUMANode*(node: int): bool {.inline, gcsafe.} = false
proc basicTests* =
discard