feat(tooling): X-Ray threshold optimizer, gallery utilities, artifact registry, docs build
Optimizer (scripts/optimizer/): replay.py runs the real C++ tracker/matcher/ scene_tracker chain over a dumped-embeddings HDF5 via sae_kpn, so a threshold sweep never re-decodes video or re-embeds faces. optimize.py drives scipy's differential_evolution over the knob space, with DE-level parallelism (multiple population candidates evaluated concurrently via a ThreadPoolExecutor) on top of per-film replay parallelism. second_score.py is the per-second X-Ray scoring metric (TPI/FPI/FN, out-of-cast misID weighted 10x, fair recall masked to gallery-known cast) that superseded an earlier scene-union metric. dump_error_frames.py / dump_scene_montage.py extract annotated video frames (bounding boxes, TPI/FPI/FN captions, onscreen-vs-offscreen split) for visual review of a replay against ground truth. Gallery utilities: cast_restrict.py, gallery_membership.py, fetch_missing_actors.py, reembed_gallery.py. scripts/validation/: X-Ray ground-truth loading and provider-agnostic identity matching (identity.py's keys_for — an actor is the union of every id we can derive, since pipeline output and ground truth don't share one id space). scripts/artifacts/: push/pull scripts for the Gitea generic package registry — galleries, montage frames, and experiment data (manifests/trajectories/results) are pushed there instead of committed, since none are needed to run the app, only benchmarks. Versioned by git short-SHA. scripts/docs/: MkDocs site build (build_site.sh) and the calibration-curve comparison chart (calibration_chart.py, matplotlib, reads each gallery's embedded calibration). Gallery-building scripts (make_jellyfin_gallery.py, make_gallery.py, filter_gallery.py, run_from_jellyfin.py, movienet_eval.py, movienet_prep.py, sae_gallery.py) updated to read/write HDF5 galleries exclusively, matching the engine-side format switch. run_from_jellyfin.py and the optimizer no longer carry movie source paths in shared manifests (some source filenames include scene-release tags) — resolved locally via a gitignored file-lut.json instead.
This commit is contained in:
Executable
+40
@@ -0,0 +1,40 @@
|
||||
#!/bin/bash
|
||||
# build_site.sh — pull the images the docs reference from the artifact registry
|
||||
# (if not already present locally), stage them under docs/assets/, then build
|
||||
# the MkDocs site. The built site/ output is what gets pushed to gitea-pages —
|
||||
# never the source images themselves (see scripts/artifacts/push_artifacts.sh).
|
||||
set -euo pipefail
|
||||
|
||||
REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
|
||||
cd "$REPO_ROOT"
|
||||
|
||||
ASSETS_DIR="docs/assets/images"
|
||||
mkdir -p "$ASSETS_DIR"
|
||||
|
||||
# Frames referenced by docs/rep4-optimizer-results.md. Pull the film's montage
|
||||
# frames from the registry if this machine doesn't already have them locally.
|
||||
FRAMES_ROOT="experiments/results/holdout/frames"
|
||||
if [ ! -d "$FRAMES_ROOT/many_saints" ] || [ ! -d "$FRAMES_ROOT/downton_abbey" ]; then
|
||||
echo "==> pulling montage frames (not found locally)..."
|
||||
scripts/artifacts/pull_artifacts.sh montage-frames Many_Saints_of_Newark || true
|
||||
scripts/artifacts/pull_artifacts.sh montage-frames Downton_Abbey__A_New_Era || true
|
||||
fi
|
||||
|
||||
echo "==> staging referenced frames into ${ASSETS_DIR}"
|
||||
cp -v "${FRAMES_ROOT}/many_saints/fpi/fpi_t03543.jpg" \
|
||||
"${ASSETS_DIR}/many_saints_ghost_fpi.jpg"
|
||||
cp -v "${FRAMES_ROOT}/downton_abbey/fpi/fpi_t07242.jpg" \
|
||||
"${ASSETS_DIR}/downton_abbey_ghost_fpi.jpg"
|
||||
|
||||
if [ ! -d experiments/galleries ] || [ -z "$(ls -A experiments/galleries 2>/dev/null)" ]; then
|
||||
echo "==> pulling galleries (not found locally)..."
|
||||
scripts/artifacts/pull_artifacts.sh galleries
|
||||
fi
|
||||
|
||||
echo "==> generating calibration curve chart"
|
||||
python3 scripts/docs/calibration_chart.py --out "${ASSETS_DIR}/calibration_curves.png"
|
||||
|
||||
echo "==> building site"
|
||||
mkdocs build
|
||||
|
||||
echo "==> done. site/ is ready to deploy to the gitea-pages branch."
|
||||
@@ -0,0 +1,89 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
calibration_chart.py — plot each model's calibrated P(match|similarity) sigmoid,
|
||||
from the (a, b) fitted into each gallery's HDF5 /calibration group. Shows
|
||||
discriminative power: a steeper curve (larger |a|) separates positive/negative
|
||||
pairs more sharply at the same decision boundary.
|
||||
|
||||
Usage:
|
||||
python scripts/docs/calibration_chart.py --out docs/assets/images/calibration_curves.png
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
from pathlib import Path
|
||||
|
||||
import h5py
|
||||
import numpy as np
|
||||
import matplotlib.pyplot as plt
|
||||
|
||||
MODELS = [
|
||||
("arcface_w600k_r50", "ArcFace w600k-R50"),
|
||||
("arcface_r18", "ArcFace R18"),
|
||||
("arcface_w600k_mbf", "ArcFace w600k-MBF"),
|
||||
("LVFace-B_Glint360K", "LVFace-B Glint360K"),
|
||||
]
|
||||
COLOURS = ["#2a78d6", "#008300", "#e87ba4", "#eda100"]
|
||||
|
||||
REPO = Path(__file__).resolve().parent.parent.parent
|
||||
|
||||
|
||||
def load_calibrations() -> list[dict]:
|
||||
out = []
|
||||
for slug, label in MODELS:
|
||||
path = REPO / f"experiments/galleries/gallery_{slug}.h5"
|
||||
if not path.exists():
|
||||
print(f"[calibration_chart] skip {slug}: gallery not found at {path}")
|
||||
continue
|
||||
with h5py.File(path, "r") as f:
|
||||
if "calibration" not in f:
|
||||
print(f"[calibration_chart] skip {slug}: no calibration in gallery "
|
||||
f"(run a replay against it once to fit and embed one)")
|
||||
continue
|
||||
cal = f["calibration"]
|
||||
out.append({"slug": slug, "label": label,
|
||||
"a": float(cal.attrs["a"]), "b": float(cal.attrs["b"])})
|
||||
return out
|
||||
|
||||
|
||||
def main():
|
||||
p = argparse.ArgumentParser()
|
||||
p.add_argument("--out", required=True)
|
||||
args = p.parse_args()
|
||||
|
||||
models = load_calibrations()
|
||||
if not models:
|
||||
raise SystemExit("no galleries had embedded calibration — run a replay "
|
||||
"against each gallery once first (see identity_matcher_node.hpp)")
|
||||
|
||||
sim = np.linspace(-1, 1, 400)
|
||||
fig, ax = plt.subplots(figsize=(7.5, 4.8), dpi=150)
|
||||
|
||||
for m, colour in zip(models, COLOURS):
|
||||
p_match = 1.0 / (1.0 + np.exp(-(m["a"] * sim + m["b"])))
|
||||
boundary = -m["b"] / m["a"]
|
||||
ax.plot(sim, p_match, color=colour, linewidth=2,
|
||||
label=f"{m['label']} (a={m['a']:.1f}, boundary@P=0.5: sim={boundary:.2f})")
|
||||
|
||||
ax.axhline(0.5, color="#999999", linewidth=1, linestyle="--", zorder=0)
|
||||
ax.set_xlabel("cosine similarity")
|
||||
ax.set_ylabel("P(match)")
|
||||
ax.set_title("Calibrated P(match | similarity), per embedding model")
|
||||
ax.set_xlim(-1, 1)
|
||||
ax.set_ylim(0, 1)
|
||||
ax.legend(loc="upper left", fontsize=8, frameon=False)
|
||||
ax.spines["top"].set_visible(False)
|
||||
ax.spines["right"].set_visible(False)
|
||||
fig.tight_layout()
|
||||
|
||||
out_path = Path(args.out)
|
||||
out_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
fig.savefig(out_path)
|
||||
print(f"[calibration_chart] wrote {out_path} ({len(models)} models)")
|
||||
for m in models:
|
||||
print(f" {m['label']}: a={m['a']:.2f} b={m['b']:.2f} "
|
||||
f"boundary(P=0.5)=sim{-m['b']/m['a']:.3f}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in New Issue
Block a user