feat(tooling): X-Ray threshold optimizer, gallery utilities, artifact registry, docs build
Optimizer (scripts/optimizer/): replay.py runs the real C++ tracker/matcher/ scene_tracker chain over a dumped-embeddings HDF5 via sae_kpn, so a threshold sweep never re-decodes video or re-embeds faces. optimize.py drives scipy's differential_evolution over the knob space, with DE-level parallelism (multiple population candidates evaluated concurrently via a ThreadPoolExecutor) on top of per-film replay parallelism. second_score.py is the per-second X-Ray scoring metric (TPI/FPI/FN, out-of-cast misID weighted 10x, fair recall masked to gallery-known cast) that superseded an earlier scene-union metric. dump_error_frames.py / dump_scene_montage.py extract annotated video frames (bounding boxes, TPI/FPI/FN captions, onscreen-vs-offscreen split) for visual review of a replay against ground truth. Gallery utilities: cast_restrict.py, gallery_membership.py, fetch_missing_actors.py, reembed_gallery.py. scripts/validation/: X-Ray ground-truth loading and provider-agnostic identity matching (identity.py's keys_for — an actor is the union of every id we can derive, since pipeline output and ground truth don't share one id space). scripts/artifacts/: push/pull scripts for the Gitea generic package registry — galleries, montage frames, and experiment data (manifests/trajectories/results) are pushed there instead of committed, since none are needed to run the app, only benchmarks. Versioned by git short-SHA. scripts/docs/: MkDocs site build (build_site.sh) and the calibration-curve comparison chart (calibration_chart.py, matplotlib, reads each gallery's embedded calibration). Gallery-building scripts (make_jellyfin_gallery.py, make_gallery.py, filter_gallery.py, run_from_jellyfin.py, movienet_eval.py, movienet_prep.py, sae_gallery.py) updated to read/write HDF5 galleries exclusively, matching the engine-side format switch. run_from_jellyfin.py and the optimizer no longer carry movie source paths in shared manifests (some source filenames include scene-release tags) — resolved locally via a gitignored file-lut.json instead.
This commit is contained in:
@@ -0,0 +1,197 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
second_score.py — uniform per-second agreement with X-Ray.
|
||||
|
||||
Unlike scene_score.py (which unions our detections over a whole X-Ray scene), this
|
||||
samples EVERY SECOND of the film and asks: at second t, do we name the same actors
|
||||
X-Ray says are on screen?
|
||||
|
||||
GT(t) = the cast set of the X-Ray scene containing t (scenes.csv + people_in_scenes)
|
||||
Pred(t) = actors whose presence window [start,end] covers t (the pipeline's output)
|
||||
|
||||
Per second we count instances:
|
||||
TPI = |Pred ∩ GT| true positive instances
|
||||
FPI = |Pred − GT| false positive instances, split into:
|
||||
FPI_misid — actor NOT in the film's cast at all (a real misID, weighted 10×)
|
||||
FPI_incast — actor in the film but not this second (timing/boundary)
|
||||
FN = |GT − Pred|, counting only gallery-known actors (fair recall — ~67% of X-Ray
|
||||
cast have no reference embedding and can never be recognised)
|
||||
agreement at t = Jaccard |Pred ∩ GT| / |Pred ∪ GT| — PARTIAL credit, so naming 2
|
||||
of 3 actors scores 2/3, not 0. Averaged over sampled seconds → the
|
||||
"what fraction of the time do we agree with X-Ray" number. (Exact-set match is
|
||||
reported separately as exact_match_rate; it is far harsher and dominated by
|
||||
recall.)
|
||||
|
||||
Objective (DE): per-second F1 computed with the WEIGHTED FPI, so naming someone who
|
||||
isn't in the film hurts 10× more than a boundary slip.
|
||||
|
||||
Reported: TPI, FPI (+split), FN, precision, recall, F1, and agreement_rate — the
|
||||
fraction of sampled seconds where we exactly matched X-Ray.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import csv
|
||||
import json
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
REPO = Path(__file__).resolve().parent.parent.parent
|
||||
sys.path.insert(0, str(REPO / "scripts" / "validation"))
|
||||
from identity import keys_for # noqa: E402
|
||||
|
||||
|
||||
def load_second_timeline(xray_dir: str):
|
||||
"""Return (timeline, film_cast_keys, duration).
|
||||
|
||||
timeline: dict second -> list of actor key-sets on screen per X-Ray.
|
||||
Each second inside a scene [start,end) inherits that scene's cast set.
|
||||
"""
|
||||
d = Path(xray_dir)
|
||||
id_to_name = {}
|
||||
with open(d / "people.csv", newline="", encoding="utf-8") as f:
|
||||
for r in csv.DictReader(f):
|
||||
nm = (r.get("name_id") or "").strip()
|
||||
if nm:
|
||||
id_to_name[nm] = (r.get("person") or "").strip()
|
||||
|
||||
film_cast = set()
|
||||
for nm, name in id_to_name.items():
|
||||
film_cast |= keys_for(imdb_id=nm, name=name)
|
||||
|
||||
spans = {}
|
||||
with open(d / "scenes.csv", newline="", encoding="utf-8") as f:
|
||||
for r in csv.DictReader(f):
|
||||
sn = (r.get("scene") or "").strip()
|
||||
try:
|
||||
spans[sn] = (float(r["start"]) / 1000.0, float(r["end"]) / 1000.0)
|
||||
except (KeyError, ValueError):
|
||||
continue
|
||||
|
||||
scene_cast: dict[str, list] = {}
|
||||
with open(d / "people_in_scenes.csv", newline="", encoding="utf-8") as f:
|
||||
for r in csv.DictReader(f):
|
||||
sn = (r.get("scene") or "").strip()
|
||||
nm = (r.get("name_id") or "").strip()
|
||||
if sn in spans and nm:
|
||||
scene_cast.setdefault(sn, []).append(
|
||||
frozenset(keys_for(imdb_id=nm, name=id_to_name.get(nm))))
|
||||
|
||||
timeline: dict[int, list] = {}
|
||||
duration = 0.0
|
||||
for sn, (t0, t1) in spans.items():
|
||||
duration = max(duration, t1)
|
||||
cast = scene_cast.get(sn, [])
|
||||
for t in range(int(t0), int(t1)):
|
||||
timeline[t] = cast
|
||||
return timeline, film_cast, duration
|
||||
|
||||
|
||||
def load_pred_intervals(pred_json: dict):
|
||||
"""[(keyset, [(t0,t1),...]), ...] for each actor the pipeline named."""
|
||||
out = []
|
||||
for a in pred_json.get("actors", []):
|
||||
keys = frozenset(keys_for(imdb_id=a.get("imdb_id"), tmdb_id=a.get("tmdb_id"),
|
||||
jellyfin_id=a.get("jellyfin_id"), name=a.get("name")))
|
||||
out.append((keys, [(float(t0), float(t1)) for t0, t1 in a.get("scenes", [])]))
|
||||
return out
|
||||
|
||||
|
||||
def _match(P, G):
|
||||
"""Greedy 1:1 match by key intersection; returns (n_matched, matched_G_mask)."""
|
||||
used = [False] * len(G)
|
||||
n = 0
|
||||
for pa in P:
|
||||
for j, ga in enumerate(G):
|
||||
if not used[j] and (pa & ga):
|
||||
used[j] = True
|
||||
n += 1
|
||||
break
|
||||
return n, used
|
||||
|
||||
|
||||
def score_seconds(pred_json: dict, xray_dir: str, gallery_keys: set | None = None,
|
||||
misid_weight: float = 10.0):
|
||||
timeline, film_cast, duration = load_second_timeline(xray_dir)
|
||||
pred = load_pred_intervals(pred_json)
|
||||
|
||||
TPI = FPI = FN = 0
|
||||
FPI_misid = FPI_incast = 0
|
||||
FPI_w = 0.0
|
||||
jaccard_sum = 0.0 # partial-credit agreement, summed over seconds
|
||||
exact = 0
|
||||
n_sec = 0
|
||||
|
||||
for t in sorted(timeline):
|
||||
G = [set(a) for a in timeline[t]]
|
||||
P = [set(k) for k, wins in pred if any(w0 <= t <= w1 for w0, w1 in wins)]
|
||||
# fair recall: only GT actors we could possibly recognise
|
||||
if gallery_keys is not None:
|
||||
G = [g for g in G if g & gallery_keys]
|
||||
|
||||
tp, matched = _match(P, G)
|
||||
# classify each unmatched prediction
|
||||
fpi_w = 0.0
|
||||
n_fp = 0
|
||||
for pa in P:
|
||||
if any(pa & ga for ga in G):
|
||||
continue
|
||||
n_fp += 1
|
||||
if pa & film_cast:
|
||||
FPI_incast += 1; fpi_w += 1.0
|
||||
else:
|
||||
FPI_misid += 1; fpi_w += misid_weight
|
||||
fn = len(G) - tp
|
||||
|
||||
TPI += tp; FPI += n_fp; FN += fn; FPI_w += fpi_w
|
||||
# partial-credit agreement: |∩| / |∪| at this second
|
||||
union = tp + n_fp + fn
|
||||
if union:
|
||||
jaccard_sum += tp / union
|
||||
else:
|
||||
jaccard_sum += 1.0 # both empty = agreement (nobody on screen)
|
||||
if n_fp == 0 and fn == 0:
|
||||
exact += 1
|
||||
n_sec += 1
|
||||
|
||||
prec = TPI / (TPI + FPI_w) if TPI + FPI_w else 0.0 # weighted (misID hurts 10×)
|
||||
prec_raw = TPI / (TPI + FPI) if TPI + FPI else 0.0
|
||||
rec = TPI / (TPI + FN) if TPI + FN else 0.0
|
||||
f1 = 2 * prec * rec / (prec + rec) if prec + rec else 0.0
|
||||
return {"TPI": TPI, "FPI": FPI, "FPI_misid": FPI_misid, "FPI_incast": FPI_incast,
|
||||
"FN": FN, "precision": prec, "precision_raw": prec_raw, "recall": rec,
|
||||
"f1": f1,
|
||||
# partial-credit: mean per-second Jaccard = "% of actors we agree on, over time"
|
||||
"agreement_rate": jaccard_sum / n_sec if n_sec else 0.0,
|
||||
"exact_match_rate": exact / n_sec if n_sec else 0.0,
|
||||
"n_seconds": n_sec, "duration_sec": duration}
|
||||
|
||||
|
||||
def main():
|
||||
p = argparse.ArgumentParser(description=__doc__,
|
||||
formatter_class=argparse.RawDescriptionHelpFormatter)
|
||||
p.add_argument("--pred", required=True)
|
||||
p.add_argument("--xray", required=True)
|
||||
p.add_argument("--gallery")
|
||||
args = p.parse_args()
|
||||
gk = None
|
||||
if args.gallery:
|
||||
sys.path.insert(0, str(REPO / "scripts" / "validation"))
|
||||
from sample_eval import load_gallery_keys
|
||||
gk = load_gallery_keys(args.gallery)
|
||||
m = score_seconds(json.loads(Path(args.pred).read_text()), args.xray, gk)
|
||||
print(f"seconds sampled : {m['n_seconds']} (film {m['duration_sec']:.0f}s)")
|
||||
print(f"TPI/FPI/FN : {m['TPI']}/{m['FPI']}/{m['FN']}")
|
||||
print(f" FPI misID : {m['FPI_misid']} (actor not in film — weighted 10x)")
|
||||
print(f" FPI in-cast : {m['FPI_incast']}")
|
||||
print(f"precision (w) : {m['precision']*100:.1f}% raw {m['precision_raw']*100:.1f}%")
|
||||
print(f"recall : {m['recall']*100:.1f}%")
|
||||
print(f"F1 (weighted) : {m['f1']*100:.1f}%")
|
||||
print(f"AGREEMENT : {m['agreement_rate']*100:.1f}% (mean per-second % of actors "
|
||||
f"we agree on with X-Ray)")
|
||||
print(f" exact-set match: {m['exact_match_rate']*100:.1f}% of seconds (harsher, "
|
||||
f"all-or-nothing)")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in New Issue
Block a user