feat(tooling): X-Ray threshold optimizer, gallery utilities, artifact registry, docs build

Optimizer (scripts/optimizer/): replay.py runs the real C++ tracker/matcher/
scene_tracker chain over a dumped-embeddings HDF5 via sae_kpn, so a threshold
sweep never re-decodes video or re-embeds faces. optimize.py drives scipy's
differential_evolution over the knob space, with DE-level parallelism
(multiple population candidates evaluated concurrently via a ThreadPoolExecutor)
on top of per-film replay parallelism. second_score.py is the per-second X-Ray
scoring metric (TPI/FPI/FN, out-of-cast misID weighted 10x, fair recall masked
to gallery-known cast) that superseded an earlier scene-union metric.
dump_error_frames.py / dump_scene_montage.py extract annotated video frames
(bounding boxes, TPI/FPI/FN captions, onscreen-vs-offscreen split) for visual
review of a replay against ground truth. Gallery utilities: cast_restrict.py,
gallery_membership.py, fetch_missing_actors.py, reembed_gallery.py.

scripts/validation/: X-Ray ground-truth loading and provider-agnostic identity
matching (identity.py's keys_for — an actor is the union of every id we can
derive, since pipeline output and ground truth don't share one id space).

scripts/artifacts/: push/pull scripts for the Gitea generic package registry —
galleries, montage frames, and experiment data (manifests/trajectories/results)
are pushed there instead of committed, since none are needed to run the app,
only benchmarks. Versioned by git short-SHA.

scripts/docs/: MkDocs site build (build_site.sh) and the calibration-curve
comparison chart (calibration_chart.py, matplotlib, reads each gallery's
embedded calibration).

Gallery-building scripts (make_jellyfin_gallery.py, make_gallery.py,
filter_gallery.py, run_from_jellyfin.py, movienet_eval.py, movienet_prep.py,
sae_gallery.py) updated to read/write HDF5 galleries exclusively, matching the
engine-side format switch. run_from_jellyfin.py and the optimizer no longer
carry movie source paths in shared manifests (some source filenames include
scene-release tags) — resolved locally via a gitignored file-lut.json instead.

scripts/sae_gallery.py

@@ -2,7 +2,13 @@
 
 Consolidates the three near-identical download loops (make_gallery.download_images,
 make_jellyfin_gallery.download_urls + download_person_images) and the duplicated
-"write gallery.json + .missing_images.json" tail from both builders.
+"write gallery.h5 + .missing_images.json" tail from both builders.
+
+Galleries are written directly as HDF5 — never JSON. Same layout the C++ side
+reads/writes (src/gallery/gallery_store.cpp): flat [N,512] embeddings + per-actor
+offset/count, parallel imdb_id/tmdb_id/jellyfin_id/name string arrays, and a
+per-embedding-row source_images array. calibration is left absent (calib_hash=0);
+the C++ identity_matcher fits and writes it back into the file on first use.
 """
 
 import io
@@ -10,6 +16,8 @@ import json
 import sys
 from pathlib import Path
 
+import h5py
+import numpy as np
 import requests
 from PIL import Image, UnidentifiedImageError
 
@@ -93,11 +101,81 @@ def download_images(urls: list[str], dest_dir: Path, n: int,
     return paths
 
 
-def save_gallery(gallery: dict, missing: list[dict], output: Path) -> None:
-    """Write gallery.json and, if any actors lack images, a .missing_images.json sidecar."""
+def save_gallery_hdf5(gallery: dict, output: Path) -> None:
+    """Write a gallery dict ({"actors": [...]}) directly as HDF5 — same schema
+    src/gallery/gallery_store.cpp reads/writes. No calibration group; the
+    C++ identity_matcher computes and writes it back into this file on first
+    use against an unseen set of embeddings."""
+    actors = gallery["actors"]
+    embs, offsets, counts = [], [], []
+    imdb, tmdb, jf, name, src_images = [], [], [], [], []
+    row = 0
+    for a in actors:
+        e = a.get("embeddings", [])
+        offsets.append(row)
+        counts.append(len(e))
+        row += len(e)
+        embs.extend(e)
+        si = a.get("source_images", [])
+        for i in range(len(e)):
+            src_images.append(si[i] if i < len(si) else "")
+        imdb.append(a.get("imdb_id", "") or "")
+        tmdb.append(str(a.get("tmdb_id", "") or ""))
+        jf.append(a.get("jellyfin_id", a.get("jellyfin_person_id", "")) or "")
+        name.append(a.get("name", "") or "")
+
+    emb_arr = np.asarray(embs, dtype=np.float32) if embs else np.zeros((0, 512), np.float32)
+    if emb_arr.ndim == 1:
+        emb_arr = emb_arr.reshape(0, 512)
+
     output.parent.mkdir(parents=True, exist_ok=True)
-    output.write_text(json.dumps(gallery, indent=2) + "\n")
-    print(f"Saved: {output}", file=sys.stderr)
+    str_t = h5py.string_dtype("utf-8")
+    with h5py.File(output, "w") as f:
+        f.create_dataset("embeddings", data=emb_arr)
+        f.create_dataset("offset", data=np.asarray(offsets, np.int64))
+        f.create_dataset("count", data=np.asarray(counts, np.int32))
+        f.create_dataset("imdb_id", data=np.asarray(imdb, dtype=object), dtype=str_t)
+        f.create_dataset("tmdb_id", data=np.asarray(tmdb, dtype=object), dtype=str_t)
+        f.create_dataset("jellyfin_id", data=np.asarray(jf, dtype=object), dtype=str_t)
+        f.create_dataset("name", data=np.asarray(name, dtype=object), dtype=str_t)
+        f.create_dataset("source_images", data=np.asarray(src_images, dtype=object), dtype=str_t)
+    print(f"Saved: {output} ({len(actors)} actors, {emb_arr.shape[0]} embeddings)",
+          file=sys.stderr)
+
+
+def load_gallery_hdf5(path: Path) -> dict:
+    """Read an HDF5 gallery back into the same {"actors": [...]} dict shape the
+    builders work with in memory (for --merge). Mirrors save_gallery_hdf5."""
+    with h5py.File(path, "r") as f:
+        emb = f["embeddings"][:]
+        offset = f["offset"][:]
+        count = f["count"][:]
+        imdb = [s.decode() if isinstance(s, bytes) else s for s in f["imdb_id"][:]]
+        tmdb = [s.decode() if isinstance(s, bytes) else s for s in f["tmdb_id"][:]]
+        jf = [s.decode() if isinstance(s, bytes) else s for s in f["jellyfin_id"][:]]
+        name = [s.decode() if isinstance(s, bytes) else s for s in f["name"][:]]
+        src_images = None
+        if "source_images" in f:
+            src_images = [s.decode() if isinstance(s, bytes) else s
+                         for s in f["source_images"][:]]
+
+    actors = []
+    for a in range(len(offset)):
+        s, n = int(offset[a]), int(count[a])
+        actor = {"imdb_id": imdb[a], "tmdb_id": tmdb[a], "jellyfin_id": jf[a],
+                "name": name[a], "embeddings": [emb[s + i].tolist() for i in range(n)]}
+        if src_images is not None:
+            actor["source_images"] = [src_images[s + i] for i in range(n)]
+        actors.append(actor)
+    return {"actors": actors}
+
+
+def save_gallery(gallery: dict, missing: list[dict], output: Path) -> None:
+    """Write the gallery as HDF5 (forcing a .h5 extension) and, if any actors
+    lack images, a .missing_images.json sidecar."""
+    if output.suffix not in (".h5", ".hdf5"):
+        output = output.with_suffix(".h5")
+    save_gallery_hdf5(gallery, output)
 
     if missing:
         missing_path = output.with_name(output.stem + ".missing_images.json")