#!/usr/bin/env python3 """ gallery_membership.py — definitive per-film gallery coverage of X-Ray cast. For each film, splits the X-Ray cast (people.csv) into those WITH a gallery reference embedding and those WITHOUT. This is the model-independent foundation for honest FP/FN rates: because every model's gallery is built from the SAME TMDB source images (same actors), the membership list is identical across models — only the embedding values differ. So FN can be measured over the recognisable denominator (in-gallery cast) and out-of-cast misIDs (predicted actor not in the film at all) are well defined. Outputs experiments/results/membership.json: { film: { xray_cast: N, in_gallery: M, coverage: M/N, in_gallery_names: [...], missing_names: [...] } } Usage: python scripts/optimizer/gallery_membership.py \ --manifest experiments/manifests/films.json \ --gallery gallery_arcface_w600k_r50.json \ --out experiments/results/membership.json """ from __future__ import annotations import argparse import csv import json import sys from pathlib import Path REPO = Path(__file__).resolve().parent.parent.parent sys.path.insert(0, str(REPO / "scripts" / "validation")) from identity import keys_for # noqa: E402 def gallery_keyset(gallery_path: str) -> set: keys = set() for a in json.loads(Path(gallery_path).read_text())["actors"]: if not a.get("embeddings"): continue # no embedding = not actually recognisable keys |= keys_for(imdb_id=a.get("imdb_id"), tmdb_id=a.get("tmdb_id"), jellyfin_id=a.get("jellyfin_id"), name=a.get("name")) return keys def film_cast(xray_dir: str) -> dict[str, str]: """nm_id → person name from a film's X-Ray people.csv.""" out = {} with open(Path(xray_dir) / "people.csv", newline="", encoding="utf-8") as f: for r in csv.DictReader(f): nm = (r.get("name_id") or "").strip() if nm: out[nm] = (r.get("person") or "").strip() return out def main(): p = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) p.add_argument("--manifest", required=True) p.add_argument("--gallery", required=True) p.add_argument("--out", required=True) args = p.parse_args() gkeys = gallery_keyset(args.gallery) films = json.loads(Path(args.manifest).read_text()) report = {} tot_cast = tot_in = 0 print(f"{'film':32s} {'cast':>5s} {'in-gal':>7s} {'cover':>6s}") for f in films: cast = film_cast(f["xray"]) in_g, miss = [], [] for nm, name in cast.items(): if keys_for(imdb_id=nm, name=name) & gkeys: in_g.append(name) else: miss.append(name) n, m = len(cast), len(in_g) tot_cast += n; tot_in += m report[f["name"]] = {"xray_cast": n, "in_gallery": m, "coverage": round(m / n, 3) if n else 0.0, "in_gallery_names": sorted(in_g), "missing_names": sorted(miss)} print(f"{f['name'][:32]:32s} {n:>5d} {m:>7d} {m/n*100 if n else 0:>5.0f}%") print(f"{'TOTAL':32s} {tot_cast:>5d} {tot_in:>7d} {tot_in/tot_cast*100:>5.0f}%") Path(args.out).parent.mkdir(parents=True, exist_ok=True) Path(args.out).write_text(json.dumps(report, indent=2)) print(f"\n→ {args.out}") if __name__ == "__main__": main()