dtourolle
48332d2041
The sae_kpn module has not compiled since the AR-007/AR-008 tracker redesign, and was switched off at the build rather than patched because the fix is a restructuring. Two failures, one cause. It did not compile: `add_face_tracker` built FaceTrackerFunc from a Config alone, and the tracker has required a TrackRegistry and a calibration since association moved into probability space. And presence was rebuilt in Python. `replay.py::build_minimal` merged per-frame detections into windows by annealing gaps, which is what the pipeline did before AR-012. The sink builds a window from a TrackRegistry claim instead — the extent of a track an actor owned, starting when they appeared rather than when recognition first succeeded. Those answer different questions, so every sweep was tuning against a contract the shipped code had stopped honouring. Both follow from the seam being a factory per node. The chain has a construction order — the matcher fits the calibration, the registry needs a discounter built from it, the tracker needs both, and the sink needs the registry's claims — and independent factories cannot express it, so the tracker kept being built against a signature that no longer existed. One `add_pipeline` mirrors main.cpp exactly and is now the only way to build the chain, so the ordering cannot be got wrong again from Python. DP-001 is the requirement behind it: a replay harness is a front-end, and its job is to supply frames and read the result, not to re-derive presence. Lifetimes needed a home. ResultSinkFunc holds `const Config&` and `std::atomic<bool>&`, which under main() are locals in a frame outliving the pipeline; there is no such frame when the network is built and torn down from Python. ReplaySession owns both for the network's lifetime, keyed by network and released explicitly — a sweep builds one network per replay and the sink retains every annotation, so holding them forever would grow with films x configs. Getting this wrong presented as an empty output_path: the sink announced `[result_sink] writing ` and wrote nothing. test_sae_kpn.py is ported rather than left behind. It called all three removed factories and asserted on SceneAnnotations read back per frame; neither half survives, so it now waits on pipeline_done and asserts on the file the sink writes. Verified against gallery_lvface.h5: three frames through the real chain, timestamps 0/1/2, truth file written. EOF is a control token the sink flushes on and does not record, so three inputs give three frames, never four. SAE_BUILD_KPN_BINDINGS goes back to ON. TRACES: VR-011, VR-002 | DP-001 | PR-002
109 lines
4.9 KiB
Python
109 lines
4.9 KiB
Python
#!/usr/bin/env python3
|
|
"""
|
|
run_holdout_all_models.py — replay each model's own tuned full_exp config
|
|
against the 5 held-out films, score with second_score.py, and dump a combined
|
|
JSON. r50 is excluded (see docs/model-bakeoff.md: dropped from the detailed
|
|
comparison, kept only in the calibration-curve chart).
|
|
|
|
This fills a real gap: the shipped report claimed "nothing in held-out
|
|
validation contradicts the model choice" without ever running mbf/r18 on the
|
|
held-out films — only LVFace had been checked.
|
|
|
|
Usage: python3 scripts/docs/run_holdout_all_models.py --out docs_data/holdout_all_models.json
|
|
"""
|
|
import argparse
|
|
import json
|
|
import subprocess
|
|
import sys
|
|
from pathlib import Path
|
|
|
|
REPO = Path(__file__).resolve().parent.parent.parent
|
|
sys.path.insert(0, str(REPO / "scripts" / "optimizer"))
|
|
sys.path.insert(0, str(REPO / "scripts" / "validation"))
|
|
from second_score import score_seconds # noqa: E402
|
|
from sample_eval import load_gallery_keys # noqa: E402
|
|
|
|
MODELS = ["LVFace-B_Glint360K", "arcface_w600k_mbf", "arcface_r18"]
|
|
|
|
HELDOUT = [
|
|
{"name": "Benny & Joon", "slug": "Benny___Joon",
|
|
"xray": "experiments/xray/scene_level_movie_data_XRay_US/xrays/4808_Benny__Joon"},
|
|
{"name": "Downton Abbey: A New Era", "slug": "Downton_Abbey__A_New_Era",
|
|
"xray": "experiments/xray/scene_level_movie_data_XRay_US/xrays/19_Downton_Abbey_A_New_Era"},
|
|
{"name": "Lovelace", "slug": "Lovelace",
|
|
"xray": "experiments/xray/scene_level_movie_data_XRay_US/xrays/4108_Lovelace"},
|
|
{"name": "The Many Saints of Newark", "slug": "The_Many_Saints_of_Newark",
|
|
"xray": "experiments/xray/scene_level_movie_data_XRay_US/xrays/900_The_Many_Saints_Of_Newark"},
|
|
{"name": "Valerian and the City of a Thousand Planets",
|
|
"slug": "Valerian_and_the_City_of_a_Thousand_Plan",
|
|
"xray": "experiments/xray/scene_level_movie_data_XRay_US/xrays/5312_Valerian_and_the_City_of_a_Thousand_Planets"},
|
|
]
|
|
|
|
TRAINING = [
|
|
{"name": "Café Society", "slug": "Café_Society",
|
|
"xray": "experiments/xray/scene_level_movie_data_XRay_US/xrays/225_Cafe_Society"},
|
|
{"name": "Lord of War", "slug": "Lord_of_War",
|
|
"xray": "experiments/xray/scene_level_movie_data_XRay_US/xrays/2474_Lord_of_War"},
|
|
{"name": "Scarface", "slug": "Scarface",
|
|
"xray": "experiments/xray/scene_level_movie_data_XRay_US/xrays/197_Scarface"},
|
|
{"name": "Sound of Metal", "slug": "Sound_of_Metal",
|
|
"xray": "experiments/xray/scene_level_movie_data_XRay_US/xrays/6278_Sound_of_Metal"},
|
|
]
|
|
|
|
|
|
def main():
|
|
p = argparse.ArgumentParser()
|
|
p.add_argument("--out", required=True)
|
|
p.add_argument("--work-dir", default="/tmp/holdout_all_models")
|
|
p.add_argument("--films", choices=["heldout", "training"], default="heldout")
|
|
args = p.parse_args()
|
|
|
|
work = Path(args.work_dir)
|
|
work.mkdir(parents=True, exist_ok=True)
|
|
film_set = HELDOUT if args.films == "heldout" else TRAINING
|
|
|
|
results = {}
|
|
for model in MODELS:
|
|
cfg = json.load(open(REPO / f"experiments/results/rep4_best_{model}_full_exp.json"))["best"]["config"]
|
|
gallery = REPO / f"experiments/galleries/gallery_{model}.h5"
|
|
results[model] = {"config": cfg, "films": {}}
|
|
|
|
for film in film_set:
|
|
dump = REPO / f"experiments/dumps/{model}/dump_{film['slug']}.h5"
|
|
if not dump.exists():
|
|
print(f"SKIP {model}/{film['slug']}: no dump", file=sys.stderr)
|
|
continue
|
|
pred_path = work / f"pred_{model}_{film['slug']}.json"
|
|
cmd = [
|
|
"python3", "scripts/optimizer/replay.py",
|
|
"--dump", str(dump), "--gallery", str(gallery),
|
|
"--out", str(pred_path),
|
|
"--prob-threshold", str(cfg["prob_threshold"]),
|
|
# anneal_sec and extinction_sec are both gone: presence is
|
|
# the registry's, built from track extents (AR-012/AR-013), and
|
|
# replay.py no longer windows anything itself (VR-011).
|
|
"--expand-gallery",
|
|
]
|
|
print(f"RUN {model}/{film['slug']}...", file=sys.stderr)
|
|
r = subprocess.run(cmd, cwd=REPO, capture_output=True, text=True, timeout=120)
|
|
if r.returncode != 0:
|
|
print(f"FAIL {model}/{film['slug']}: {r.stderr[-800:]}", file=sys.stderr)
|
|
results[model]["films"][film["slug"]] = {"error": r.stderr[-500:]}
|
|
continue
|
|
|
|
gk = load_gallery_keys(str(gallery))
|
|
pred_json = json.loads(pred_path.read_text())
|
|
m = score_seconds(pred_json, str(REPO / film["xray"]), gk)
|
|
results[model]["films"][film["slug"]] = {"name": film["name"], **m}
|
|
print(f" -> F1={m['f1']*100:.1f}% P={m['precision']*100:.1f}% "
|
|
f"R={m['recall']*100:.1f}% misid={m['FPI_misid']}", file=sys.stderr)
|
|
|
|
Path(args.out).parent.mkdir(parents=True, exist_ok=True)
|
|
with open(args.out, "w") as f:
|
|
json.dump(results, f, indent=1)
|
|
print(f"wrote {args.out}", file=sys.stderr)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|