docs: full data-grounded rewrite of the performance report
Replaces narrative claims with verified numbers across all report pages: - Cross-model held-out validation (LVFace/mbf/r18, all 5 held-out films): LVFace wins every film outright, not just "consistent with" the training-set pick. r50 dropped from the detailed comparison (gallery has ~30% fewer reference images per actor than the other three models on identical source photos). - Per-film training breakdown: LVFace does not win every training film (mbf beats it on Lord of War); the 75.3% macro figure hides a 10.7pp spread. - Gallery coverage computed per film (20.3%-78.6%) instead of one flat 67%-missing average. - Found and fixed a real scoring bug in optimize.py: a candidate whose hardest film's replay timed out was averaged over survivors instead of penalized, silently rewarding partial coverage. Affected 3 of 16 training combos; corrected throughout, and optimize.py now scores an incomplete evaluation f1=0.0 instead of averaging over whichever films happened to finish. - Every FPI frame in the deep dive now comes from the proper montage renderer (Onscreen/Offscreen panel, ghosts never drawn as boxes), never the bare-box debug overlay used earlier. - Every distinct out-of-cast name across all 9 films gets its own frame at its first appearance (9 names, 4 films), not a single-example spot check: 2 ground-truth gaps, 1 photograph misread as a person, 6 genuine lookalike confusions. - New methodology.md: the scene-level-vs-per-second scoring mismatch that the rest of the report assumes, written out once. - Cut the deadlock/gdb debugging narrative from the experiment log; kept the one fact that matters (KPN's node/network split lets the expensive GPU stage run once and the cheap stage replay against cached embeddings). - Plain declarative style throughout, no em dashes, no blog voice.
This commit is contained in:
@@ -0,0 +1,107 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
run_holdout_all_models.py — replay each model's own tuned full_exp config
|
||||
against the 5 held-out films, score with second_score.py, and dump a combined
|
||||
JSON. r50 is excluded (see docs/model-bakeoff.md: dropped from the detailed
|
||||
comparison, kept only in the calibration-curve chart).
|
||||
|
||||
This fills a real gap: the shipped report claimed "nothing in held-out
|
||||
validation contradicts the model choice" without ever running mbf/r18 on the
|
||||
held-out films — only LVFace had been checked.
|
||||
|
||||
Usage: python3 scripts/docs/run_holdout_all_models.py --out docs_data/holdout_all_models.json
|
||||
"""
|
||||
import argparse
|
||||
import json
|
||||
import subprocess
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
REPO = Path(__file__).resolve().parent.parent.parent
|
||||
sys.path.insert(0, str(REPO / "scripts" / "optimizer"))
|
||||
sys.path.insert(0, str(REPO / "scripts" / "validation"))
|
||||
from second_score import score_seconds # noqa: E402
|
||||
from sample_eval import load_gallery_keys # noqa: E402
|
||||
|
||||
MODELS = ["LVFace-B_Glint360K", "arcface_w600k_mbf", "arcface_r18"]
|
||||
|
||||
HELDOUT = [
|
||||
{"name": "Benny & Joon", "slug": "Benny___Joon",
|
||||
"xray": "experiments/xray/scene_level_movie_data_XRay_US/xrays/4808_Benny__Joon"},
|
||||
{"name": "Downton Abbey: A New Era", "slug": "Downton_Abbey__A_New_Era",
|
||||
"xray": "experiments/xray/scene_level_movie_data_XRay_US/xrays/19_Downton_Abbey_A_New_Era"},
|
||||
{"name": "Lovelace", "slug": "Lovelace",
|
||||
"xray": "experiments/xray/scene_level_movie_data_XRay_US/xrays/4108_Lovelace"},
|
||||
{"name": "The Many Saints of Newark", "slug": "The_Many_Saints_of_Newark",
|
||||
"xray": "experiments/xray/scene_level_movie_data_XRay_US/xrays/900_The_Many_Saints_Of_Newark"},
|
||||
{"name": "Valerian and the City of a Thousand Planets",
|
||||
"slug": "Valerian_and_the_City_of_a_Thousand_Plan",
|
||||
"xray": "experiments/xray/scene_level_movie_data_XRay_US/xrays/5312_Valerian_and_the_City_of_a_Thousand_Planets"},
|
||||
]
|
||||
|
||||
TRAINING = [
|
||||
{"name": "Café Society", "slug": "Café_Society",
|
||||
"xray": "experiments/xray/scene_level_movie_data_XRay_US/xrays/225_Cafe_Society"},
|
||||
{"name": "Lord of War", "slug": "Lord_of_War",
|
||||
"xray": "experiments/xray/scene_level_movie_data_XRay_US/xrays/2474_Lord_of_War"},
|
||||
{"name": "Scarface", "slug": "Scarface",
|
||||
"xray": "experiments/xray/scene_level_movie_data_XRay_US/xrays/197_Scarface"},
|
||||
{"name": "Sound of Metal", "slug": "Sound_of_Metal",
|
||||
"xray": "experiments/xray/scene_level_movie_data_XRay_US/xrays/6278_Sound_of_Metal"},
|
||||
]
|
||||
|
||||
|
||||
def main():
|
||||
p = argparse.ArgumentParser()
|
||||
p.add_argument("--out", required=True)
|
||||
p.add_argument("--work-dir", default="/tmp/holdout_all_models")
|
||||
p.add_argument("--films", choices=["heldout", "training"], default="heldout")
|
||||
args = p.parse_args()
|
||||
|
||||
work = Path(args.work_dir)
|
||||
work.mkdir(parents=True, exist_ok=True)
|
||||
film_set = HELDOUT if args.films == "heldout" else TRAINING
|
||||
|
||||
results = {}
|
||||
for model in MODELS:
|
||||
cfg = json.load(open(REPO / f"experiments/results/rep4_best_{model}_full_exp.json"))["best"]["config"]
|
||||
gallery = REPO / f"experiments/galleries/gallery_{model}.h5"
|
||||
results[model] = {"config": cfg, "films": {}}
|
||||
|
||||
for film in film_set:
|
||||
dump = REPO / f"experiments/dumps/{model}/dump_{film['slug']}.h5"
|
||||
if not dump.exists():
|
||||
print(f"SKIP {model}/{film['slug']}: no dump", file=sys.stderr)
|
||||
continue
|
||||
pred_path = work / f"pred_{model}_{film['slug']}.json"
|
||||
cmd = [
|
||||
"python3", "scripts/optimizer/replay.py",
|
||||
"--dump", str(dump), "--gallery", str(gallery),
|
||||
"--out", str(pred_path),
|
||||
"--prob-threshold", str(cfg["prob_threshold"]),
|
||||
"--anneal-sec", str(cfg["anneal_sec"]),
|
||||
"--extinction-sec", str(cfg["extinction_sec"]),
|
||||
"--expand-gallery",
|
||||
]
|
||||
print(f"RUN {model}/{film['slug']}...", file=sys.stderr)
|
||||
r = subprocess.run(cmd, cwd=REPO, capture_output=True, text=True, timeout=120)
|
||||
if r.returncode != 0:
|
||||
print(f"FAIL {model}/{film['slug']}: {r.stderr[-800:]}", file=sys.stderr)
|
||||
results[model]["films"][film["slug"]] = {"error": r.stderr[-500:]}
|
||||
continue
|
||||
|
||||
gk = load_gallery_keys(str(gallery))
|
||||
pred_json = json.loads(pred_path.read_text())
|
||||
m = score_seconds(pred_json, str(REPO / film["xray"]), gk)
|
||||
results[model]["films"][film["slug"]] = {"name": film["name"], **m}
|
||||
print(f" -> F1={m['f1']*100:.1f}% P={m['precision']*100:.1f}% "
|
||||
f"R={m['recall']*100:.1f}% misid={m['FPI_misid']}", file=sys.stderr)
|
||||
|
||||
Path(args.out).parent.mkdir(parents=True, exist_ok=True)
|
||||
with open(args.out, "w") as f:
|
||||
json.dump(results, f, indent=1)
|
||||
print(f"wrote {args.out}", file=sys.stderr)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in New Issue
Block a user