feat(tooling): X-Ray threshold optimizer, gallery utilities, artifact registry, docs build
Optimizer (scripts/optimizer/): replay.py runs the real C++ tracker/matcher/ scene_tracker chain over a dumped-embeddings HDF5 via sae_kpn, so a threshold sweep never re-decodes video or re-embeds faces. optimize.py drives scipy's differential_evolution over the knob space, with DE-level parallelism (multiple population candidates evaluated concurrently via a ThreadPoolExecutor) on top of per-film replay parallelism. second_score.py is the per-second X-Ray scoring metric (TPI/FPI/FN, out-of-cast misID weighted 10x, fair recall masked to gallery-known cast) that superseded an earlier scene-union metric. dump_error_frames.py / dump_scene_montage.py extract annotated video frames (bounding boxes, TPI/FPI/FN captions, onscreen-vs-offscreen split) for visual review of a replay against ground truth. Gallery utilities: cast_restrict.py, gallery_membership.py, fetch_missing_actors.py, reembed_gallery.py. scripts/validation/: X-Ray ground-truth loading and provider-agnostic identity matching (identity.py's keys_for — an actor is the union of every id we can derive, since pipeline output and ground truth don't share one id space). scripts/artifacts/: push/pull scripts for the Gitea generic package registry — galleries, montage frames, and experiment data (manifests/trajectories/results) are pushed there instead of committed, since none are needed to run the app, only benchmarks. Versioned by git short-SHA. scripts/docs/: MkDocs site build (build_site.sh) and the calibration-curve comparison chart (calibration_chart.py, matplotlib, reads each gallery's embedded calibration). Gallery-building scripts (make_jellyfin_gallery.py, make_gallery.py, filter_gallery.py, run_from_jellyfin.py, movienet_eval.py, movienet_prep.py, sae_gallery.py) updated to read/write HDF5 galleries exclusively, matching the engine-side format switch. run_from_jellyfin.py and the optimizer no longer carry movie source paths in shared manifests (some source filenames include scene-release tags) — resolved locally via a gitignored file-lut.json instead.
This commit is contained in:
@@ -0,0 +1,148 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Self-contained tests for the presence eval. Builds a synthetic X-Ray fixture and
|
||||
a pipeline-output JSON in a temp dir, then checks scoring, masking, name-fallback
|
||||
matching, and sampling modes. Run: python scripts/validation/test_sample_eval.py
|
||||
"""
|
||||
|
||||
import csv
|
||||
import json
|
||||
import sys
|
||||
import tempfile
|
||||
from pathlib import Path
|
||||
|
||||
HERE = Path(__file__).resolve().parent
|
||||
sys.path.insert(0, str(HERE))
|
||||
|
||||
from identity import keys_for, norm_name # noqa: E402
|
||||
from sample_eval import Prediction, score, sample_points, _match_count # noqa: E402
|
||||
from ground_truth import XRayGroundTruth # noqa: E402
|
||||
from tmdb_imdb_map import CrosswalkTable # noqa: E402
|
||||
|
||||
|
||||
def _write_xray(d: Path):
|
||||
d.mkdir(parents=True, exist_ok=True)
|
||||
with open(d / "scenes.csv", "w", newline="") as f:
|
||||
w = csv.writer(f); w.writerow(["scene", "start_ms", "end_ms"])
|
||||
w.writerows([("1", 0, 60000), ("2", 300000, 340000)])
|
||||
# real Zenodo X-Ray schema: people(name_id,person,character);
|
||||
# people_in_scenes(scene,start,end,name_id,timestamp)
|
||||
with open(d / "people.csv", "w", newline="") as f:
|
||||
w = csv.writer(f); w.writerow(["name_id", "person", "character"])
|
||||
w.writerows([("nm0330687", "Lauren Graham", "Lorelai"),
|
||||
("nm0004754", "Alexis Bledel", "Rory"),
|
||||
("nm0000001", "Ghost Actor", "Ghost")])
|
||||
with open(d / "people_in_scenes.csv", "w", newline="") as f:
|
||||
w = csv.writer(f); w.writerow(["scene", "start", "end", "name_id", "timestamp"])
|
||||
w.writerows([("1", 0, 60000, "nm0330687", 5000),
|
||||
("1", 0, 60000, "nm0000001", 8000),
|
||||
("2", 300000, 340000, "nm0004754", 305000)])
|
||||
|
||||
|
||||
def _write_pred(path: Path):
|
||||
# pipeline output carries tmdb+name but NO nm ids -> name-fallback join to X-Ray
|
||||
doc = {"schema_version": 1, "movie": "x", "sample_fps": 1.0, "anneal_sec": 10.0,
|
||||
"actors": [
|
||||
{"name": "Lauren Graham", "imdb_id": "", "tmdb_id": "16858",
|
||||
"jellyfin_id": "a", "scenes": [[21.0, 31.0], [50.0, 59.0]]},
|
||||
{"name": "Alexis Bledel", "imdb_id": "", "tmdb_id": "6279",
|
||||
"jellyfin_id": "b", "scenes": [[301.0, 311.0]]},
|
||||
{"name": "Edward Herrmann", "imdb_id": "", "tmdb_id": "52995",
|
||||
"jellyfin_id": "c", "scenes": [[4.0, 56.0]]}]}
|
||||
path.write_text(json.dumps(doc))
|
||||
|
||||
|
||||
class T:
|
||||
n = 0
|
||||
def check(self, cond, msg):
|
||||
T.n += 1
|
||||
assert cond, f"FAIL: {msg}"
|
||||
print(f" ok: {msg}")
|
||||
|
||||
|
||||
def main():
|
||||
t = T()
|
||||
|
||||
# -- identity ---------------------------------------------------------------
|
||||
t.check(norm_name("Zöe Saldaña!") == "zoe saldana", "accent/punct normalization")
|
||||
t.check(keys_for(imdb_id="nm1", name="Jo Ann") == {"imdb:nm1", "name:jo ann"},
|
||||
"keys_for builds namespaced tokens")
|
||||
t.check(keys_for(imdb_id="") == set(), "empty ids dropped")
|
||||
|
||||
# -- crosswalk: tmdb-only actor gains an exact imdb: key --------------------
|
||||
xw = CrosswalkTable({"16858": "nm0330687", "999": None})
|
||||
k = keys_for(imdb_id="", tmdb_id="16858", name="Lauren Graham", crosswalk=xw)
|
||||
t.check("imdb:nm0330687" in k, "crosswalk resolves tmdb→imdb key")
|
||||
k_null = keys_for(tmdb_id="999", crosswalk=xw)
|
||||
t.check(not any(x.startswith("imdb:") for x in k_null), "crosswalk null → no imdb key")
|
||||
t.check(len(xw) == 1, "CrosswalkTable len counts non-null entries")
|
||||
|
||||
# -- matching ---------------------------------------------------------------
|
||||
t.check(_match_count([{"name:jo"}, {"name:al"}], [{"imdb:x", "name:jo"}]) == 1,
|
||||
"one match by shared name key")
|
||||
t.check(_match_count([{"name:jo"}], [{"name:jo"}, {"name:jo"}]) == 1,
|
||||
"greedy 1:1 uses each GT once")
|
||||
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
tmp = Path(tmp)
|
||||
_write_xray(tmp / "xray")
|
||||
_write_pred(tmp / "pred.json")
|
||||
|
||||
pred = Prediction(tmp / "pred.json")
|
||||
gt = XRayGroundTruth(tmp / "xray")
|
||||
|
||||
# -- presence lookups ---------------------------------------------------
|
||||
t.check(len(pred.present_at(25)) == 2, "Graham+Herrmann present at 25s")
|
||||
t.check(len(pred.present_at(305)) == 1, "only Bledel present at 305s")
|
||||
t.check(len(gt.present_at(30)) == 2, "X-Ray scene1 has Graham+Ghost at 30s")
|
||||
t.check(len(gt.present_at(320)) == 1, "X-Ray scene2 has Bledel at 320s")
|
||||
|
||||
# -- masking behaviour --------------------------------------------------
|
||||
pts = [30.0]
|
||||
# unmasked: at 30s pred={Graham,Herrmann}, gt={Graham,Ghost}
|
||||
# match Graham -> TP1; Herrmann unmatched -> FP1; Ghost unmatched -> FN1
|
||||
m = score(pred, gt, pts, mask=None)
|
||||
t.check((m["TP"], m["FP"], m["FN"]) == (1, 1, 1), "unmasked 30s = 1/1/1")
|
||||
|
||||
# masked to GT∩pred keys: Ghost & Herrmann are absent from the other side's
|
||||
# key space, so both drop -> only Graham remains on both -> 1/0/0
|
||||
gt_keys = gt.all_keys(); pred_keys = pred.all_keys()
|
||||
mask = gt_keys & pred_keys
|
||||
m2 = score(pred, gt, pts, mask=mask)
|
||||
t.check((m2["TP"], m2["FP"], m2["FN"]) == (1, 0, 0),
|
||||
"masked 30s drops off-gallery actors = 1/0/0")
|
||||
|
||||
# -- crosswalk end-to-end: exact imdb join, names garbled ---------------
|
||||
# Rebuild pred with names that WON'T match X-Ray, but a crosswalk that maps
|
||||
# their tmdb ids to the correct nm ids. Match must survive via imdb key.
|
||||
garbled = {"schema_version": 1, "movie": "x", "actors": [
|
||||
{"name": "WRONG NAME A", "imdb_id": "", "tmdb_id": "16858",
|
||||
"jellyfin_id": "a", "scenes": [[21.0, 31.0]]}, # →nm0330687 Graham
|
||||
{"name": "WRONG NAME B", "imdb_id": "", "tmdb_id": "6279",
|
||||
"jellyfin_id": "b", "scenes": [[301.0, 311.0]]}]} # →nm0004754 Bledel
|
||||
(tmp / "garbled.json").write_text(json.dumps(garbled))
|
||||
xw = CrosswalkTable({"16858": "nm0330687", "6279": "nm0004754"})
|
||||
pred_g = Prediction(tmp / "garbled.json", crosswalk=xw)
|
||||
# at 30s (scene1) Graham should match by imdb despite wrong name
|
||||
m_g = score(pred_g, gt, [30.0], mask=None)
|
||||
t.check(m_g["TP"] == 1, "crosswalk yields exact imdb match despite wrong names")
|
||||
# without crosswalk, wrong names → no match at all
|
||||
pred_bad = Prediction(tmp / "garbled.json")
|
||||
m_bad = score(pred_bad, gt, [30.0], mask=None)
|
||||
t.check(m_bad["TP"] == 0, "no crosswalk + wrong names → no match")
|
||||
|
||||
# -- sampling modes -----------------------------------------------------
|
||||
class A: # arg stub
|
||||
scene_anchored = True; random = None; step = 1.0; end = None; seed = 0
|
||||
sp = sample_points(A, pred, gt)
|
||||
t.check(sp == [30.0, 320.0], "scene-anchored samples scene midpoints")
|
||||
|
||||
A.scene_anchored = False; A.end = 10.0; A.step = 2.0
|
||||
grid = sample_points(A, pred, gt)
|
||||
t.check(grid == [0.0, 2.0, 4.0, 6.0, 8.0, 10.0], "regular grid step")
|
||||
|
||||
print(f"\nALL {T.n} CHECKS PASSED")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in New Issue
Block a user