#!/usr/bin/env python3
"""Check the PERF_PLAN Phase-0 acceptance criterion.

Runs bench_pipeline several times and reports, per row, how far the passes
spread around their median. The plan's gate is: the same configuration run 7x
lands within +/-5% on every row. Until that holds, no measured difference
between KPN and TBB is worth acting on.

Exits non-zero if any row exceeds the tolerance, so it can gate a session of
performance work rather than merely inform one.

Usage:
    scripts/bench_repro_check.py ./build_bench/benchmarks/bench_pipeline \\
        --passes 7 --tolerance 5 -- --work=10 --topos=chain,wide --reps=5
"""

import argparse
import statistics
import subprocess
import sys

KEY_COLS = ("topology", "size", "work_us", "threads")

# bench_pipeline reports throughput, bench_dispatch reports per-dispatch cost.
# Either is a valid thing to demand reproducibility of; deviation from the
# median is symmetric, so it does not matter which direction is "better".
METRIC_COLS = ("items_per_sec", "ns_per_dispatch")


def parse_csv(text, metric=None):
    """Return ({(topology, size, work_us, threads): value}, metric_name)."""
    rows = {}
    header = None
    for line in text.splitlines():
        line = line.strip()
        if not line or line.startswith("#"):
            continue
        fields = line.split(",")
        if header is None:
            if fields[0] != "topology":
                continue
            header = fields
            if metric is None:
                for cand in METRIC_COLS:
                    if cand in header:
                        metric = cand
                        break
                else:
                    sys.exit(f"no metric column found in header: {header}")
            elif metric not in header:
                sys.exit(f"metric {metric!r} not in header: {header}")
            continue
        rec = dict(zip(header, fields))
        try:
            key = tuple(rec[c] for c in KEY_COLS)
            rows[key] = float(rec[metric])
        except (KeyError, ValueError):
            continue
    return rows, metric


def main():
    ap = argparse.ArgumentParser()
    ap.add_argument("binary", help="path to bench_pipeline")
    ap.add_argument("--passes", type=int, default=7)
    ap.add_argument("--tolerance", type=float, default=5.0,
                    help="max allowed deviation from the median, percent")
    ap.add_argument("--metric", default=None, choices=METRIC_COLS,
                    help="column to check (default: whichever the CSV carries)")

    # Everything after a standalone `--` goes to bench_pipeline verbatim.
    # argparse.REMAINDER would swallow this script's own flags instead.
    argv = sys.argv[1:]
    extra = []
    if "--" in argv:
        cut = argv.index("--")
        argv, extra = argv[:cut], argv[cut + 1:]
    args = ap.parse_args(argv)

    passes = []
    metric = args.metric
    for i in range(args.passes):
        print(f"pass {i + 1}/{args.passes} ...", file=sys.stderr, flush=True)
        proc = subprocess.run([args.binary] + extra,
                              capture_output=True, text=True)
        if proc.returncode != 0:
            print(proc.stderr, file=sys.stderr)
            sys.exit(f"{args.binary} failed with {proc.returncode}")
        rows, metric = parse_csv(proc.stdout, metric)
        passes.append(rows)

    keys = set(passes[0])
    for p in passes[1:]:
        keys &= set(p)
    if not keys:
        sys.exit("no rows common to every pass")

    print(f"\n{'row':<34} {'median ' + metric:>20} {'worst dev':>10}  verdict")
    print("-" * 72)

    failures = 0
    for key in sorted(keys):
        values = [p[key] for p in passes]
        med = statistics.median(values)
        worst = max(abs(v - med) / med * 100 for v in values) if med else 0.0
        ok = worst <= args.tolerance
        failures += not ok
        label = "{}-{} w={} s={}".format(*key)
        print(f"{label:<34} {med:>20.1f} {worst:>9.1f}%  {'ok' if ok else 'NOISY'}")

    print("-" * 72)
    if failures:
        print(f"{failures}/{len(keys)} rows exceed +/-{args.tolerance:g}% — "
              f"the Phase-0 gate is not met.")
        return 1
    print(f"all {len(keys)} rows within +/-{args.tolerance:g}% "
          f"over {args.passes} passes — Phase-0 gate met.")
    return 0


if __name__ == "__main__":
    sys.exit(main())