From 6f0ad83a55d8f93c45cb2f9f3ce8cfce095da4f6 Mon Sep 17 00:00:00 2001 From: Duncan Tourolle Date: Sun, 19 Jul 2026 19:06:48 +0200 Subject: [PATCH] feat(tooling): X-Ray threshold optimizer, gallery utilities, artifact registry, docs build MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Optimizer (scripts/optimizer/): replay.py runs the real C++ tracker/matcher/ scene_tracker chain over a dumped-embeddings HDF5 via sae_kpn, so a threshold sweep never re-decodes video or re-embeds faces. optimize.py drives scipy's differential_evolution over the knob space, with DE-level parallelism (multiple population candidates evaluated concurrently via a ThreadPoolExecutor) on top of per-film replay parallelism. second_score.py is the per-second X-Ray scoring metric (TPI/FPI/FN, out-of-cast misID weighted 10x, fair recall masked to gallery-known cast) that superseded an earlier scene-union metric. dump_error_frames.py / dump_scene_montage.py extract annotated video frames (bounding boxes, TPI/FPI/FN captions, onscreen-vs-offscreen split) for visual review of a replay against ground truth. Gallery utilities: cast_restrict.py, gallery_membership.py, fetch_missing_actors.py, reembed_gallery.py. scripts/validation/: X-Ray ground-truth loading and provider-agnostic identity matching (identity.py's keys_for — an actor is the union of every id we can derive, since pipeline output and ground truth don't share one id space). scripts/artifacts/: push/pull scripts for the Gitea generic package registry — galleries, montage frames, and experiment data (manifests/trajectories/results) are pushed there instead of committed, since none are needed to run the app, only benchmarks. Versioned by git short-SHA. scripts/docs/: MkDocs site build (build_site.sh) and the calibration-curve comparison chart (calibration_chart.py, matplotlib, reads each gallery's embedded calibration). Gallery-building scripts (make_jellyfin_gallery.py, make_gallery.py, filter_gallery.py, run_from_jellyfin.py, movienet_eval.py, movienet_prep.py, sae_gallery.py) updated to read/write HDF5 galleries exclusively, matching the engine-side format switch. run_from_jellyfin.py and the optimizer no longer carry movie source paths in shared manifests (some source filenames include scene-release tags) — resolved locally via a gitignored file-lut.json instead. --- scripts/artifacts/pull_artifacts.sh | 106 ++++++ scripts/artifacts/push_artifacts.sh | 112 +++++++ scripts/build_trt_engines.sh | 24 +- scripts/convert_transnetv2.py | 137 ++++++++ scripts/docs/build_site.sh | 40 +++ scripts/docs/calibration_chart.py | 89 +++++ scripts/download_models.sh | 17 + scripts/filter_gallery.py | 22 +- scripts/make_gallery.py | 13 +- scripts/make_jellyfin_gallery.py | 19 +- scripts/movienet_eval.py | 6 +- scripts/movienet_prep.py | 10 +- scripts/optimizer/SCHEMA.md | 50 +++ scripts/optimizer/cast_restrict.py | 58 ++++ scripts/optimizer/dump_error_frames.py | 201 ++++++++++++ scripts/optimizer/dump_scene_montage.py | 383 ++++++++++++++++++++++ scripts/optimizer/fetch_missing_actors.py | 154 +++++++++ scripts/optimizer/gallery_membership.py | 94 ++++++ scripts/optimizer/optimize.py | 240 ++++++++++++++ scripts/optimizer/reembed_gallery.py | 94 ++++++ scripts/optimizer/replay.py | 240 ++++++++++++++ scripts/optimizer/second_score.py | 197 +++++++++++ scripts/optimizer/test_sae_kpn.py | 75 +++++ scripts/run_from_jellyfin.py | 31 +- scripts/sae_gallery.py | 88 ++++- scripts/validation/README.md | 87 +++++ scripts/validation/ground_truth.py | 221 +++++++++++++ scripts/validation/identity.py | 54 +++ scripts/validation/sample_eval.py | 313 ++++++++++++++++++ scripts/validation/test_sample_eval.py | 148 +++++++++ scripts/validation/tmdb_imdb_map.py | 135 ++++++++ 31 files changed, 3411 insertions(+), 47 deletions(-) create mode 100755 scripts/artifacts/pull_artifacts.sh create mode 100755 scripts/artifacts/push_artifacts.sh create mode 100644 scripts/convert_transnetv2.py create mode 100755 scripts/docs/build_site.sh create mode 100644 scripts/docs/calibration_chart.py create mode 100644 scripts/optimizer/SCHEMA.md create mode 100644 scripts/optimizer/cast_restrict.py create mode 100644 scripts/optimizer/dump_error_frames.py create mode 100644 scripts/optimizer/dump_scene_montage.py create mode 100644 scripts/optimizer/fetch_missing_actors.py create mode 100644 scripts/optimizer/gallery_membership.py create mode 100644 scripts/optimizer/optimize.py create mode 100644 scripts/optimizer/reembed_gallery.py create mode 100644 scripts/optimizer/replay.py create mode 100644 scripts/optimizer/second_score.py create mode 100644 scripts/optimizer/test_sae_kpn.py create mode 100644 scripts/validation/README.md create mode 100644 scripts/validation/ground_truth.py create mode 100644 scripts/validation/identity.py create mode 100644 scripts/validation/sample_eval.py create mode 100644 scripts/validation/test_sample_eval.py create mode 100644 scripts/validation/tmdb_imdb_map.py diff --git a/scripts/artifacts/pull_artifacts.sh b/scripts/artifacts/pull_artifacts.sh new file mode 100755 index 0000000..b1ee196 --- /dev/null +++ b/scripts/artifacts/pull_artifacts.sh @@ -0,0 +1,106 @@ +#!/bin/bash +# pull_artifacts.sh — download benchmark artifacts (galleries, montage frames) from +# the Gitea generic package registry. Counterpart to push_artifacts.sh. +# +# Downloads are public (no token needed) as long as the repo/packages are public. +# Resolving "latest" needs GITEA_TOKEN (the list-packages endpoint requires auth +# on this instance even for a public account) — export it before using "latest". +# +# Usage: +# scripts/artifacts/pull_artifacts.sh galleries [version] +# scripts/artifacts/pull_artifacts.sh montage-frames [version] +# scripts/artifacts/pull_artifacts.sh experiment-data [version] +# version defaults to "latest" (newest uploaded version, by created_at). +set -euo pipefail + +REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" +OWNER="dtourolle" +# NOTE: file download lives under /api/packages/ (no /v1/); listing/metadata +# lives under the regular /api/v1/packages/ REST API. Different base paths, +# both real — see push_artifacts.sh's comment. +DL_BASE="https://gitea.tourolle.paris/api/packages/${OWNER}" + +resolve_latest_version() { + local package="$1" + if [ -z "${GITEA_TOKEN:-}" ]; then + echo "error: resolving 'latest' needs GITEA_TOKEN (list-packages requires auth here)." >&2 + echo " export GITEA_TOKEN=... or pass an explicit version instead of 'latest'." >&2 + exit 1 + fi + curl -sf "https://gitea.tourolle.paris/api/v1/packages/${OWNER}" \ + -H "Authorization: token ${GITEA_TOKEN}" \ + | python3 -c " +import json, sys +d = json.load(sys.stdin) +matches = [p for p in d if p['name'] == '$package' and p['type'] == 'generic'] +if not matches: + sys.exit('no versions found for package \'$package\'') +matches.sort(key=lambda p: p['created_at']) +print(matches[-1]['version']) +" +} + +pull_galleries() { + local version="$1" + local dest="${REPO_ROOT}/experiments/galleries" + mkdir -p "$dest" + echo "=== galleries (version ${version}) ===" + for model in arcface_w600k_r50 arcface_r18 arcface_w600k_mbf LVFace-B_Glint360K; do + local f="gallery_${model}.h5" + echo " fetching ${f}..." + curl -sf "${DL_BASE}/generic/galleries/${version}/${f}" \ + -o "${dest}/${f}" || echo " [warn] ${f} not found at version ${version}" + done +} + +pull_montage_frames() { + local version="$1" film="$2" + local dest="${REPO_ROOT}/experiments/results/holdout/montage_bestworst" + mkdir -p "$dest" + echo "=== montage-frames/${film} (version ${version}) ===" + local tmp; tmp="$(mktemp)" + curl -sf "${DL_BASE}/generic/montage-frames/${version}/${film}.zip" -o "$tmp" + mkdir -p "${dest}/${film}" + unzip -qo "$tmp" -d "${dest}/${film}" + rm "$tmp" +} + +pull_experiment_data() { + local version="$1" + echo "=== experiment-data (version ${version}) ===" + local tmp; tmp="$(mktemp)" + curl -sf "${DL_BASE}/generic/experiment-data/${version}/experiment-data.zip" -o "$tmp" + unzip -qo "$tmp" -d "$REPO_ROOT" + rm "$tmp" +} + +if [ $# -eq 0 ]; then + echo "usage: $0 galleries [version]" >&2 + echo " $0 montage-frames [version]" >&2 + echo " $0 experiment-data [version]" >&2 + exit 1 +fi + +TARGET="$1" +case "$TARGET" in + galleries) + VERSION="${2:-latest}" + [ "$VERSION" = "latest" ] && VERSION="$(resolve_latest_version galleries)" + pull_galleries "$VERSION" + ;; + montage-frames) + FILM="${2:?usage: $0 montage-frames [version]}" + VERSION="${3:-latest}" + [ "$VERSION" = "latest" ] && VERSION="$(resolve_latest_version montage-frames)" + pull_montage_frames "$VERSION" "$FILM" + ;; + experiment-data) + VERSION="${2:-latest}" + [ "$VERSION" = "latest" ] && VERSION="$(resolve_latest_version experiment-data)" + pull_experiment_data "$VERSION" + ;; + *) + echo "unknown target: $TARGET (expected galleries, montage-frames, or experiment-data)" >&2 + exit 1 + ;; +esac diff --git a/scripts/artifacts/push_artifacts.sh b/scripts/artifacts/push_artifacts.sh new file mode 100755 index 0000000..bf4a707 --- /dev/null +++ b/scripts/artifacts/push_artifacts.sh @@ -0,0 +1,112 @@ +#!/bin/bash +# push_artifacts.sh — upload benchmark artifacts (galleries, montage frames) to the +# Gitea generic package registry, decoupled from git history entirely. These are +# NOT needed to run the main app — only for benchmarks/experiments/reports. +# +# Requires GITEA_TOKEN in the environment (a Gitea access token with package +# read/write scope). Never hardcode the token; export it in your shell: +# export GITEA_TOKEN=... +# +# Usage: +# scripts/artifacts/push_artifacts.sh galleries +# scripts/artifacts/push_artifacts.sh montage-frames +# scripts/artifacts/push_artifacts.sh experiment-data +# scripts/artifacts/push_artifacts.sh galleries montage-frames experiment-data +# +# Package layout (owner=dtourolle, repo=scene-actor-extraction): +# generic/galleries//gallery_.h5 (one file per model) +# generic/montage-frames//.zip (zipped per-film frames) +# generic/experiment-data//experiment-data.zip (manifests/trajectories/results) +# version = current git short SHA, so artifacts are traceable to the code that +# produced them. Re-running with the same SHA overwrites that version's files. +set -euo pipefail + +REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" +OWNER="dtourolle" +PKG_REPO="scene-actor-extraction" +# NOTE: the generic package registry lives under /api/packages/ (no /v1/) — +# distinct from the regular REST API under /api/v1/packages/ used for listing. +BASE_URL="https://gitea.tourolle.paris/api/packages/${OWNER}" +VERSION="$(git -C "$REPO_ROOT" rev-parse --short HEAD)" + +if [ -z "${GITEA_TOKEN:-}" ]; then + echo "error: GITEA_TOKEN is not set. export GITEA_TOKEN= and retry." >&2 + exit 1 +fi + +upload() { + local package="$1" filename="$2" filepath="$3" + local url="${BASE_URL}/generic/${package}/${VERSION}/${filename}" + echo " uploading ${filename} -> ${package}/${VERSION}..." + curl -sf -X PUT "$url" \ + -H "Authorization: token ${GITEA_TOKEN}" \ + -H "Content-Type: application/octet-stream" \ + --upload-file "$filepath" \ + -o /dev/null -w " HTTP %{http_code}\n" +} + +push_galleries() { + echo "=== galleries (version ${VERSION}) ===" + local dir="${REPO_ROOT}/experiments/galleries" + shopt -s nullglob + for f in "$dir"/gallery_*.h5; do + upload "galleries" "$(basename "$f")" "$f" + done + shopt -u nullglob +} + +push_montage_frames() { + echo "=== montage-frames (version ${VERSION}) ===" + local root="${REPO_ROOT}/experiments/results/holdout/montage_bestworst" + if [ ! -d "$root" ]; then + echo " no montage_bestworst dir found, skipping" >&2 + return + fi + local tmp + tmp="$(mktemp -d)" + trap 'rm -rf "$tmp"' RETURN + for filmdir in "$root"/*/; do + [ -d "$filmdir" ] || continue + local slug ascii_slug zipfile + slug="$(basename "$filmdir")" + # Gitea's generic package registry rejects non-ASCII filenames (verified: + # a bare é in the name 400s). Transliterate for the upload name only — + # the local directory name (with accents) is untouched. + ascii_slug="$(echo "$slug" | iconv -f utf-8 -t ascii//translit 2>/dev/null || echo "$slug")" + zipfile="${tmp}/${ascii_slug}.zip" + (cd "$filmdir" && zip -qr "$zipfile" .) + upload "montage-frames" "${ascii_slug}.zip" "$zipfile" + done +} + +push_experiment_data() { + echo "=== experiment-data (version ${VERSION}) ===" + # manifests/trajectories/results are all small text/JSON — no source paths + # (manifests never carry a "movie" field; see experiments/file-lut.json) + # so this is safe to share as one bundle. + local tmp; tmp="$(mktemp -d)" + trap 'rm -rf "$tmp"' RETURN + local zipfile="${tmp}/experiment-data.zip" + (cd "$REPO_ROOT" && zip -qr "$zipfile" \ + experiments/manifests experiments/trajectories experiments/results \ + -x '*.log' -x '*/holdout/montage_bestworst/*' -x '*/holdout/frames/*' \ + -x '*/holdout/montage/*' -x '*/holdout/*.jsonl' -x '*/holdout/pred_*.json' \ + -x '*/holdout/raw_*.jsonl') + upload "experiment-data" "experiment-data.zip" "$zipfile" +} + +if [ $# -eq 0 ]; then + echo "usage: $0 [...]" >&2 + exit 1 +fi + +for target in "$@"; do + case "$target" in + galleries) push_galleries ;; + montage-frames) push_montage_frames ;; + experiment-data) push_experiment_data ;; + *) echo "unknown target: $target (expected galleries, montage-frames, or experiment-data)" >&2; exit 1 ;; + esac +done + +echo "Done. Browse at: https://gitea.tourolle.paris/${OWNER}/${PKG_REPO}/packages" diff --git a/scripts/build_trt_engines.sh b/scripts/build_trt_engines.sh index eed6ab7..781ec5b 100755 --- a/scripts/build_trt_engines.sh +++ b/scripts/build_trt_engines.sh @@ -4,8 +4,9 @@ # block the pipeline; this script does it offline so cold starts are instant. # # Profiles must match src/arcface_embedder.hpp and src/scrfd_decoder.hpp: -# ArcFace : min=1x3x112x112 opt=Nx3x112x112 max=Nx3x112x112 (N = embed batch) -# SCRFD : 1x3x640x640 (fixed; we letterbox to this) +# ArcFace : min=1x3x112x112 opt=Nx3x112x112 max=Nx3x112x112 (N = embed batch) +# SCRFD : 1x3x640x640 (fixed; we letterbox to this) +# TransNetV2 : 1x100x27x48x3 (fixed; scene detector window), input tensor "input" # # These trtexec-built engines are *not* picked up by the ORT TRT EP cache — # ORT uses its own engine format. The point of this script is: @@ -22,6 +23,7 @@ mkdir -p "$OUT" EMBED_BATCH="${EMBED_BATCH:-4}" ARCFACE_MODEL="${ARCFACE_MODEL:-$MODELS/arcface_w600k_r50.onnx}" SCRFD_MODEL="${SCRFD_MODEL:-$MODELS/scrfd_500m_bnkps.onnx}" +SCENE_MODEL="${SCENE_MODEL:-$MODELS/transnetv2.onnx}" run() { echo "+ $*"; "$@"; } @@ -46,6 +48,24 @@ run trtexec \ --saveEngine="$OUT/scrfd.$(basename "$SCRFD_MODEL" .onnx).640.fp16.engine" \ --useCudaGraph +if [[ -f "$SCENE_MODEL" ]]; then + echo + echo "== TransNetV2 (scene detector) ==" + # Fixed 1x100x27x48x3 window. The raw-TRT scene detector backend loads this + # engine directly via --scene-detector-engine; the ORT-TRT EP builds its own. + run trtexec \ + --onnx="$SCENE_MODEL" \ + --fp16 \ + --minShapes=input:1x100x27x48x3 \ + --optShapes=input:1x100x27x48x3 \ + --maxShapes=input:1x100x27x48x3 \ + --saveEngine="$OUT/transnetv2.100x27x48.fp16.engine" \ + --useCudaGraph +else + echo + echo "== TransNetV2 skipped (no $SCENE_MODEL) ==" +fi + echo echo "Engines saved under: $OUT" echo "Look for 'mean: ... ms' in each section for per-call latency." diff --git a/scripts/convert_transnetv2.py b/scripts/convert_transnetv2.py new file mode 100644 index 0000000..e0cf64f --- /dev/null +++ b/scripts/convert_transnetv2.py @@ -0,0 +1,137 @@ +#!/usr/bin/env python3 +""" +convert_transnetv2.py — verify and optimise the TransNetV2 scene-detector model. + +The scene detector (src/nodes/scene_detector_node.hpp → ISceneDetector) consumes +the elya5/transnetv2 ONNX export with a fixed input contract: + + input "input" : float32 [1, 100, 27, 48, 3] RGB, channels-last, 0-255 + output "534" : float32 [1, 100, 1] per-frame boundary logits (used) + output "535" : float32 [1, 100, 1] many-hot head (ignored) + +This script provides the ORT/TRT conversion infra for that model: + + --verify (default) assert the .onnx matches the contract above and run one + dummy inference through onnxruntime, reporting output ranges. + --ort-opt write a graph-optimised .ort next to the model (ORT loads this + faster; the runtime also caches its own under ./ort_cache). + --trt build a TensorRT engine via trtexec with the pinned 1x100x27x48x3 + profile (delegates to scripts/build_trt_engines.sh SCENE_MODEL). + +Usage: + python scripts/convert_transnetv2.py # verify default model + python scripts/convert_transnetv2.py --ort-opt + python scripts/convert_transnetv2.py --trt + python scripts/convert_transnetv2.py --model path/to/transnetv2.onnx --verify +""" +import argparse +import os +import subprocess +import sys + +ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) +DEFAULT_MODEL = os.path.join(ROOT, "models", "transnetv2.onnx") + +# The fixed contract the C++ scene detector depends on. +EXPECTED_INPUT_SHAPE = [1, 100, 27, 48, 3] +EXPECTED_OUTPUT_SHAPE = [1, 100, 1] + + +def verify(model_path: str) -> int: + import numpy as np + import onnx + import onnxruntime as ort + + print(f"[verify] loading {model_path}") + m = onnx.load(model_path, load_external_data=False) + g = m.graph + + def shape(t): + return [d.dim_value if d.HasField("dim_value") else d.dim_param + for d in t.type.tensor_type.shape.dim] + + in_shape = shape(g.input[0]) + print(f"[verify] input '{g.input[0].name}': {in_shape}") + if in_shape != EXPECTED_INPUT_SHAPE: + print(f"[verify] ERROR: input shape {in_shape} != {EXPECTED_INPUT_SHAPE}") + return 1 + + out_names = [o.name for o in g.output] + out0_shape = shape(g.output[0]) + print(f"[verify] outputs: {out_names} primary '{out_names[0]}': {out0_shape}") + if out0_shape != EXPECTED_OUTPUT_SHAPE: + print(f"[verify] ERROR: primary output {out0_shape} != {EXPECTED_OUTPUT_SHAPE}") + return 1 + + # One dummy inference: a mid-grey clip should produce low boundary scores. + sess = ort.InferenceSession(model_path, providers=["CPUExecutionProvider"]) + dummy = np.full(EXPECTED_INPUT_SHAPE, 128.0, dtype=np.float32) + logits = sess.run([out_names[0]], {g.input[0].name: dummy})[0] + probs = 1.0 / (1.0 + np.exp(-logits)) + print(f"[verify] dummy inference OK — boundary prob " + f"min={probs.min():.4f} max={probs.max():.4f} mean={probs.mean():.4f}") + print("[verify] contract matches the C++ ISceneDetector. ✓") + return 0 + + +def ort_opt(model_path: str) -> int: + import onnxruntime as ort + + out_path = os.path.splitext(model_path)[0] + ".ort" + print(f"[ort-opt] writing graph-optimised model → {out_path}") + so = ort.SessionOptions() + so.graph_optimization_level = ort.GraphOptimizationLevel.ORT_ENABLE_ALL + so.optimized_model_filepath = out_path + # Constructing the session triggers optimisation + serialisation. + ort.InferenceSession(model_path, so, providers=["CPUExecutionProvider"]) + print(f"[ort-opt] done: {out_path}") + return 0 + + +def trt(model_path: str) -> int: + script = os.path.join(ROOT, "scripts", "build_trt_engines.sh") + print(f"[trt] delegating to {script} (SCENE_MODEL={model_path})") + env = dict(os.environ, SCENE_MODEL=model_path) + # build_trt_engines.sh also builds ArcFace/SCRFD; that's harmless (and a + # useful sanity check), but if you only want the scene engine, run trtexec + # directly with the profile printed below. + print("[trt] equivalent standalone command:") + print(f" trtexec --onnx={model_path} --fp16 " + f"--minShapes=input:1x100x27x48x3 " + f"--optShapes=input:1x100x27x48x3 " + f"--maxShapes=input:1x100x27x48x3 " + f"--saveEngine=trt_cache/transnetv2.100x27x48.fp16.engine --useCudaGraph") + return subprocess.call(["bash", script], env=env) + + +def main() -> int: + ap = argparse.ArgumentParser(description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter) + ap.add_argument("--model", default=DEFAULT_MODEL, + help=f"path to transnetv2.onnx (default: {DEFAULT_MODEL})") + ap.add_argument("--verify", action="store_true", help="verify I/O contract (default)") + ap.add_argument("--ort-opt", action="store_true", help="write optimised .ort") + ap.add_argument("--trt", action="store_true", help="build a TensorRT engine") + args = ap.parse_args() + + if not os.path.isfile(args.model): + print(f"ERROR: model not found: {args.model}\n" + f"Fetch it with: bash scripts/download_models.sh", file=sys.stderr) + return 2 + + # Default action is verify when nothing else is requested. + if not (args.ort_opt or args.trt): + args.verify = True + + rc = 0 + if args.verify: + rc |= verify(args.model) + if rc == 0 and args.ort_opt: + rc |= ort_opt(args.model) + if rc == 0 and args.trt: + rc |= trt(args.model) + return rc + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/docs/build_site.sh b/scripts/docs/build_site.sh new file mode 100755 index 0000000..62bcbc1 --- /dev/null +++ b/scripts/docs/build_site.sh @@ -0,0 +1,40 @@ +#!/bin/bash +# build_site.sh — pull the images the docs reference from the artifact registry +# (if not already present locally), stage them under docs/assets/, then build +# the MkDocs site. The built site/ output is what gets pushed to gitea-pages — +# never the source images themselves (see scripts/artifacts/push_artifacts.sh). +set -euo pipefail + +REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" +cd "$REPO_ROOT" + +ASSETS_DIR="docs/assets/images" +mkdir -p "$ASSETS_DIR" + +# Frames referenced by docs/rep4-optimizer-results.md. Pull the film's montage +# frames from the registry if this machine doesn't already have them locally. +FRAMES_ROOT="experiments/results/holdout/frames" +if [ ! -d "$FRAMES_ROOT/many_saints" ] || [ ! -d "$FRAMES_ROOT/downton_abbey" ]; then + echo "==> pulling montage frames (not found locally)..." + scripts/artifacts/pull_artifacts.sh montage-frames Many_Saints_of_Newark || true + scripts/artifacts/pull_artifacts.sh montage-frames Downton_Abbey__A_New_Era || true +fi + +echo "==> staging referenced frames into ${ASSETS_DIR}" +cp -v "${FRAMES_ROOT}/many_saints/fpi/fpi_t03543.jpg" \ + "${ASSETS_DIR}/many_saints_ghost_fpi.jpg" +cp -v "${FRAMES_ROOT}/downton_abbey/fpi/fpi_t07242.jpg" \ + "${ASSETS_DIR}/downton_abbey_ghost_fpi.jpg" + +if [ ! -d experiments/galleries ] || [ -z "$(ls -A experiments/galleries 2>/dev/null)" ]; then + echo "==> pulling galleries (not found locally)..." + scripts/artifacts/pull_artifacts.sh galleries +fi + +echo "==> generating calibration curve chart" +python3 scripts/docs/calibration_chart.py --out "${ASSETS_DIR}/calibration_curves.png" + +echo "==> building site" +mkdocs build + +echo "==> done. site/ is ready to deploy to the gitea-pages branch." diff --git a/scripts/docs/calibration_chart.py b/scripts/docs/calibration_chart.py new file mode 100644 index 0000000..20130fc --- /dev/null +++ b/scripts/docs/calibration_chart.py @@ -0,0 +1,89 @@ +#!/usr/bin/env python3 +""" +calibration_chart.py — plot each model's calibrated P(match|similarity) sigmoid, +from the (a, b) fitted into each gallery's HDF5 /calibration group. Shows +discriminative power: a steeper curve (larger |a|) separates positive/negative +pairs more sharply at the same decision boundary. + +Usage: + python scripts/docs/calibration_chart.py --out docs/assets/images/calibration_curves.png +""" +from __future__ import annotations + +import argparse +from pathlib import Path + +import h5py +import numpy as np +import matplotlib.pyplot as plt + +MODELS = [ + ("arcface_w600k_r50", "ArcFace w600k-R50"), + ("arcface_r18", "ArcFace R18"), + ("arcface_w600k_mbf", "ArcFace w600k-MBF"), + ("LVFace-B_Glint360K", "LVFace-B Glint360K"), +] +COLOURS = ["#2a78d6", "#008300", "#e87ba4", "#eda100"] + +REPO = Path(__file__).resolve().parent.parent.parent + + +def load_calibrations() -> list[dict]: + out = [] + for slug, label in MODELS: + path = REPO / f"experiments/galleries/gallery_{slug}.h5" + if not path.exists(): + print(f"[calibration_chart] skip {slug}: gallery not found at {path}") + continue + with h5py.File(path, "r") as f: + if "calibration" not in f: + print(f"[calibration_chart] skip {slug}: no calibration in gallery " + f"(run a replay against it once to fit and embed one)") + continue + cal = f["calibration"] + out.append({"slug": slug, "label": label, + "a": float(cal.attrs["a"]), "b": float(cal.attrs["b"])}) + return out + + +def main(): + p = argparse.ArgumentParser() + p.add_argument("--out", required=True) + args = p.parse_args() + + models = load_calibrations() + if not models: + raise SystemExit("no galleries had embedded calibration — run a replay " + "against each gallery once first (see identity_matcher_node.hpp)") + + sim = np.linspace(-1, 1, 400) + fig, ax = plt.subplots(figsize=(7.5, 4.8), dpi=150) + + for m, colour in zip(models, COLOURS): + p_match = 1.0 / (1.0 + np.exp(-(m["a"] * sim + m["b"]))) + boundary = -m["b"] / m["a"] + ax.plot(sim, p_match, color=colour, linewidth=2, + label=f"{m['label']} (a={m['a']:.1f}, boundary@P=0.5: sim={boundary:.2f})") + + ax.axhline(0.5, color="#999999", linewidth=1, linestyle="--", zorder=0) + ax.set_xlabel("cosine similarity") + ax.set_ylabel("P(match)") + ax.set_title("Calibrated P(match | similarity), per embedding model") + ax.set_xlim(-1, 1) + ax.set_ylim(0, 1) + ax.legend(loc="upper left", fontsize=8, frameon=False) + ax.spines["top"].set_visible(False) + ax.spines["right"].set_visible(False) + fig.tight_layout() + + out_path = Path(args.out) + out_path.parent.mkdir(parents=True, exist_ok=True) + fig.savefig(out_path) + print(f"[calibration_chart] wrote {out_path} ({len(models)} models)") + for m in models: + print(f" {m['label']}: a={m['a']:.2f} b={m['b']:.2f} " + f"boundary(P=0.5)=sim{-m['b']/m['a']:.3f}") + + +if __name__ == "__main__": + main() diff --git a/scripts/download_models.sh b/scripts/download_models.sh index ed2f3d4..0d08413 100755 --- a/scripts/download_models.sh +++ b/scripts/download_models.sh @@ -66,6 +66,23 @@ else echo "SCRFD-500MF already present: $SCRFD_FILE" fi +# ── TransNetV2 shot-boundary detection (scene detector, opt-in) ─────────────── +# ONNX export (elya5/transnetv2, MIT). Fixed input 1x100x27x48x3 (RGB 0-255), +# primary output "534" = per-frame boundary logits. Used only with --scene-detect. +SCENE_FILE="$MODELS_DIR/transnetv2.onnx" +SCENE_SHA="c4d54a682bace32f25136ef83ca2c9d403e8f8193775efeb995172a0d95a8e0c" +if [ ! -f "$SCENE_FILE" ]; then + echo "Downloading TransNetV2…" + curl -L "https://huggingface.co/elya5/transnetv2/resolve/main/transnetv2.onnx" \ + -o "$SCENE_FILE" + if command -v sha256sum >/dev/null; then + echo "$SCENE_SHA $SCENE_FILE" | sha256sum -c - \ + || echo "WARNING: TransNetV2 sha256 mismatch (upstream may have changed)" + fi +else + echo "TransNetV2 already present: $SCENE_FILE" +fi + echo "" echo "Models ready in $MODELS_DIR/:" ls -lh "$MODELS_DIR" diff --git a/scripts/filter_gallery.py b/scripts/filter_gallery.py index 62474ee..8610603 100644 --- a/scripts/filter_gallery.py +++ b/scripts/filter_gallery.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -"""filter_gallery.py — restrict a global gallery.json to one title's known cast. +"""filter_gallery.py — restrict a global gallery.h5 to one title's known cast. make_jellyfin_gallery.py builds a single gallery spanning the whole Jellyfin library, so identity_matcher can recognise any actor from any film. For a @@ -8,28 +8,27 @@ credited cast only — fewer candidates means faster matching and fewer look-alike false positives. This script looks up the title's cast live from Jellyfin and writes a -filtered gallery.json containing only those actors, matched via the +filtered gallery.h5 containing only those actors, matched via the "jellyfin_person_id" field recorded by make_jellyfin_gallery.py. Usage: python scripts/filter_gallery.py \\ - --gallery gallery.json \\ + --gallery gallery.h5 \\ --jellyfin-url http://jellyfin.local:8096 \\ --api-key YOUR_API_KEY \\ --item-id \\ - --output gallery_movie.json + --output gallery_movie.h5 # Or search by title: python scripts/filter_gallery.py \\ - --gallery gallery.json \\ + --gallery gallery.h5 \\ --jellyfin-url http://jellyfin.local:8096 \\ --api-key YOUR_API_KEY \\ --title "The Matrix" \\ - --output gallery_movie.json + --output gallery_movie.h5 """ import argparse -import json import sys from pathlib import Path @@ -42,6 +41,7 @@ from sae_jellyfin import ( # noqa: F401 fetch_cast_person_ids, actor_jellyfin_id, ) +from sae_gallery import load_gallery_hdf5, save_gallery_hdf5 def main(): @@ -50,7 +50,7 @@ def main(): formatter_class=argparse.RawDescriptionHelpFormatter, ) parser.add_argument("--gallery", required=True, - help="Global gallery.json built by make_jellyfin_gallery.py") + help="Global gallery.h5 built by make_jellyfin_gallery.py") parser.add_argument("--jellyfin-url", required=True) parser.add_argument("--api-key", required=True) group = parser.add_mutually_exclusive_group(required=True) @@ -58,10 +58,10 @@ def main(): group.add_argument("--title", help="Title to search for (uses first match)") parser.add_argument("--item-types", default="Movie,Series", help="Item types to search when using --title (default: Movie,Series)") - parser.add_argument("--output", required=True, help="Output filtered gallery.json path") + parser.add_argument("--output", required=True, help="Output filtered gallery.h5 path") args = parser.parse_args() - gallery = json.loads(Path(args.gallery).read_text()) + gallery = load_gallery_hdf5(Path(args.gallery)) item_id = args.item_id if item_id is None: @@ -77,7 +77,7 @@ def main(): if missing > 0: print(f"[warn] {missing} cast member(s) not present in gallery (not yet embedded)", file=sys.stderr) - Path(args.output).write_text(json.dumps({"actors": actors}, indent=2) + "\n") + save_gallery_hdf5({"actors": actors}, Path(args.output)) print(f"Saved {len(actors)} actor(s) to {args.output}", file=sys.stderr) diff --git a/scripts/make_gallery.py b/scripts/make_gallery.py index 1a7e736..7d3a208 100755 --- a/scripts/make_gallery.py +++ b/scripts/make_gallery.py @@ -1,9 +1,9 @@ #!/usr/bin/env python3 -"""make_gallery.py — fetch actor images for a movie and build gallery.json. +"""make_gallery.py — fetch actor images for a movie and build gallery.h5. Fetches the cast from TMDB, downloads actor profile images, embeds them via the sae_embed module (SCRFD + ArcFace, same models as scene_analyze, loaded -once), then writes gallery.json. +once), then writes gallery.h5. Requirements: pip install requests Pillow @@ -13,13 +13,13 @@ Usage: python scripts/make_gallery.py \\ --tmdb-key YOUR_KEY \\ --imdb-id tt0137523 \\ - --output gallery.json + --output gallery.h5 # Or directly with a TMDB movie ID: python scripts/make_gallery.py \\ --tmdb-key YOUR_KEY \\ --movie-id 550 \\ - --output gallery.json + --output gallery.h5 # Additional options: # --build-dir build/ build dir containing sae_embed module @@ -31,7 +31,6 @@ Get a free TMDB API key at: https://www.themoviedb.org/settings/api """ import argparse -import json import sys import time from pathlib import Path @@ -151,7 +150,7 @@ def build_gallery(movie_id: int, key: str, embedder, def main(): parser = argparse.ArgumentParser( - description="Fetch TMDB cast images and build gallery.json via sae_embed") + description="Fetch TMDB cast images and build gallery.h5 via sae_embed") parser.add_argument("--tmdb-key", required=True, help="TMDB Bearer token (API Read Access Token from themoviedb.org/settings/api)") group = parser.add_mutually_exclusive_group(required=True) @@ -159,7 +158,7 @@ def main(): help="IMDB movie ID, e.g. tt0137523 — looked up via TMDB automatically") group.add_argument("--movie-id", type=int, help="TMDB movie ID (alternative to --imdb-id)") - parser.add_argument("--output", required=True, help="Output gallery.json path") + parser.add_argument("--output", required=True, help="Output gallery.h5 path") parser.add_argument("--build-dir", default="build", help="Build directory containing the sae_embed module (default: build)") parser.add_argument("--models-dir", default="models", diff --git a/scripts/make_jellyfin_gallery.py b/scripts/make_jellyfin_gallery.py index d5f8809..0ff4153 100644 --- a/scripts/make_jellyfin_gallery.py +++ b/scripts/make_jellyfin_gallery.py @@ -1,10 +1,10 @@ #!/usr/bin/env python3 -"""make_jellyfin_gallery.py — build a gallery.json spanning an entire Jellyfin library. +"""make_jellyfin_gallery.py — build a gallery.h5 spanning an entire Jellyfin library. Queries the Jellyfin API for every Movie/Series, collects the unique cast across the whole library, downloads each actor's headshot directly from Jellyfin (no TMDB key needed), embeds them with the sae_embed module (SCRFD + -ArcFace, loaded once), and writes one global gallery.json. +ArcFace, loaded once), and writes one global gallery.h5. Because identity_matcher scores every detected face against the whole gallery, scene_analyze can then recognise any actor in your library in any @@ -20,21 +20,21 @@ Usage: python scripts/make_jellyfin_gallery.py \\ --jellyfin-url http://jellyfin.local:8096 \\ --api-key YOUR_API_KEY \\ - --output gallery.json + --output gallery.h5 # Re-run later to pick up newly added titles without re-embedding # actors already in the gallery: python scripts/make_jellyfin_gallery.py \\ --jellyfin-url http://jellyfin.local:8096 \\ --api-key YOUR_API_KEY \\ - --output gallery.json --merge + --output gallery.h5 --merge # Fall back to TMDB profile images for actors with no usable Jellyfin image: python scripts/make_jellyfin_gallery.py \\ --jellyfin-url http://jellyfin.local:8096 \\ --api-key YOUR_API_KEY \\ --tmdb-key YOUR_TMDB_KEY \\ - --output gallery.json + --output gallery.h5 Get a Jellyfin API key from Dashboard → Advanced → API Keys. Get a free TMDB API key at: https://www.themoviedb.org/settings/api @@ -52,7 +52,8 @@ import requests sys.path.insert(0, str(Path(__file__).resolve().parent)) import sae_env # noqa: F401 — loads .env into os.environ on import from sae_embed_loader import load_embedder -from sae_gallery import download_image, download_images, save_gallery, wikidata_image_urls +from sae_gallery import (download_image, download_images, load_gallery_hdf5, + save_gallery, wikidata_image_urls) from sae_jellyfin import actor_jellyfin_id, jf_get, normalize_jellyfin_url from sae_tmdb import ( tmdb_person_by_name, @@ -323,7 +324,7 @@ def build_gallery(base_url: str, api_key: str, embedder, item_types: list[str], def main(): parser = argparse.ArgumentParser( - description="Build a gallery.json spanning an entire Jellyfin library", + description="Build a gallery.h5 spanning an entire Jellyfin library", formatter_class=argparse.RawDescriptionHelpFormatter, ) parser.add_argument("--jellyfin-url", default=os.environ.get("JELLYFIN_URL"), @@ -333,7 +334,7 @@ def main(): parser.add_argument("--api-key", default=os.environ.get("JELLYFIN_API_KEY"), required=not os.environ.get("JELLYFIN_API_KEY"), help="Jellyfin API key (Dashboard → Advanced → API Keys). Env: JELLYFIN_API_KEY") - parser.add_argument("--output", required=True, help="Output gallery.json path") + parser.add_argument("--output", required=True, help="Output gallery.h5 path") parser.add_argument("--item-types", default="Movie,Series", help="Comma-separated Jellyfin item types to scan (default: Movie,Series)") parser.add_argument("--build-dir", default="build", @@ -375,7 +376,7 @@ def main(): existing_actors = {} if args.merge and output.is_file(): - existing = json.loads(output.read_text()) + existing = load_gallery_hdf5(output) for actor in existing.get("actors", []): pid = actor_jellyfin_id(actor) if pid: diff --git a/scripts/movienet_eval.py b/scripts/movienet_eval.py index 9bfd6f5..b94abf8 100644 --- a/scripts/movienet_eval.py +++ b/scripts/movienet_eval.py @@ -4,7 +4,7 @@ movienet_eval.py — embed probe crops and match against a gallery. Usage: python scripts/movienet_eval.py \ - --gallery gallery_r50.json \ + --gallery gallery_r50.h5 \ --arcface models/arcface_w600k_r50.onnx \ --gt eval/gt.json \ --output eval/predictions_r50.json \ @@ -23,12 +23,12 @@ import numpy as np sys.path.insert(0, str(Path(__file__).resolve().parent)) from sae_embed_loader import load_embedder +from sae_gallery import load_gallery_hdf5 def load_gallery(path: str) -> dict[str, dict]: """Return {imdb_id: {"name": str, "refs": np.ndarray[n_refs, dim]}}.""" - with open(path) as f: - data = json.load(f) + data = load_gallery_hdf5(Path(path)) return {a["imdb_id"]: {"name": a["name"], "refs": np.asarray(a["embeddings"], dtype=np.float32)} for a in data["actors"]} diff --git a/scripts/movienet_prep.py b/scripts/movienet_prep.py index 1aff111..27f719e 100644 --- a/scripts/movienet_prep.py +++ b/scripts/movienet_prep.py @@ -5,7 +5,7 @@ movienet_prep.py — extract probe crops from MovieNet-PS for actors in our gall Usage: python scripts/movienet_prep.py \ --movienet \ - --gallery gallery.json \ + --gallery gallery.h5 \ --output eval/ \ [--split Train_app10] \ [--margin 0.2] \ @@ -28,6 +28,9 @@ import zipfile from io import BytesIO from pathlib import Path +sys.path.insert(0, str(Path(__file__).resolve().parent)) +from sae_gallery import load_gallery_hdf5 # noqa: E402 + try: import cv2 import numpy as np @@ -74,8 +77,7 @@ def load_movienet_annotations(movienet_root: Path, split: str) -> list[dict]: def load_gallery_ids(gallery_path: str) -> dict[str, str]: """Return {imdb_id: actor_name} for all actors in the gallery.""" - with open(gallery_path) as f: - data = json.load(f) + data = load_gallery_hdf5(Path(gallery_path)) return {a["imdb_id"]: a["name"] for a in data["actors"]} @@ -100,7 +102,7 @@ def crop_face(img: "np.ndarray", bbox: list[float], margin: float) -> "np.ndarra def main(): p = argparse.ArgumentParser() p.add_argument("--movienet", required=True, help="MovieNet-PS root directory") - p.add_argument("--gallery", required=True, help="gallery.json (for actor list)") + p.add_argument("--gallery", required=True, help="gallery.h5 (for actor list)") p.add_argument("--output", default="eval", help="output directory") p.add_argument("--split", default="Train_app10", help="annotation split to use (default: Train_app10)") diff --git a/scripts/optimizer/SCHEMA.md b/scripts/optimizer/SCHEMA.md new file mode 100644 index 0000000..cf2ba34 --- /dev/null +++ b/scripts/optimizer/SCHEMA.md @@ -0,0 +1,50 @@ +# Embedding-dump HDF5 schema (v1) + +One file per analysed title. Captures the pipeline state at the `EmbeddedSceneFrame` +channel — i.e. after decode → detect → align → embed, but **before** tracking and +identity matching. Everything downstream (face tracker, identity matcher, scene +tracker/anneal) is cheap CPU math, so replaying from this file lets a parameter +sweep re-run the whole downstream tail thousands of times with no GPU and no video. + +Written by the C++ dump sink (`--dump-embeddings out.h5`); read by +`scripts/optimizer/replay.py`. + +## Layout + +The dump is **flat/ragged**: all faces across all frames are concatenated into +per-face arrays, with a per-frame index table pointing into them. This avoids +variable-length HDF5 types and reads straight into numpy. + +``` +/ (root) + attrs: + schema_version : int = 1 + movie : str (source video path) + sample_fps : float + embed_dim : int = 512 + + frames/ group — one row per sampled frame + timestamp_sec : float64 [F] + frame_idx : int64 [F] + is_cut : uint8 [F] (histogram intra-scene cut) + is_scene_boundary : uint8 [F] (TransNetV2 boundary; 0 if scene_detect off) + face_offset : int64 [F] start index into faces/* for this frame + face_count : int32 [F] number of faces in this frame + + faces/ group — one row per detected face, concatenated + embedding : float32 [N, 512] L2-normalised ArcFace embedding + bbox : float32 [N, 4] x, y, w, h in original video pixels + landmarks : float32 [N, 10] 5 (x,y) pairs, SCRFD/ArcFace order + confidence : float32 [N] detector confidence +``` + +`F` = number of sampled frames, `N` = total faces (= sum of face_count). +Frame *i*'s faces are `faces/*[ face_offset[i] : face_offset[i]+face_count[i] ]`. + +## Invariants +- `embedding` rows are unit-norm (cosine == dot product against the gallery). +- `face_offset[0] == 0`; `face_offset[i+1] == face_offset[i] + face_count[i]`. +- `bbox` is already mapped to original resolution (bbox_upscale applied at dump time), + matching what the identity matcher would emit. +- A frame with no faces has `face_count == 0` (still gets a row, so timestamps stay dense). +- EOF sentinel frames are NOT written. diff --git a/scripts/optimizer/cast_restrict.py b/scripts/optimizer/cast_restrict.py new file mode 100644 index 0000000..94e3886 --- /dev/null +++ b/scripts/optimizer/cast_restrict.py @@ -0,0 +1,58 @@ +#!/usr/bin/env python3 +""" +cast_restrict.py — produce a per-film gallery restricted to its credited cast. + +Benchmark arm: instead of matching a face against the WHOLE gallery (2418 actors, +risking cross-film misIDs like naming Archie Yates in a film he's not in), restrict +the matcher's candidate set to the title's credited cast (from Jellyfin — the top +~15 billed actors, exactly what run_from_jellyfin.py does in production). + +Filters a gallery to actors whose jellyfin_id is in the film's cast set, writing a +small gallery JSON the replay can load. Actors are kept if their jellyfin_id (or, as +a fallback, normalized name) matches the cast. + +Used by the full-vs-restricted bake-off. Cached per (gallery, film) so a DE sweep +reuses the restricted gallery. +""" +from __future__ import annotations + +import json +import sys +import tempfile +from pathlib import Path + +REPO = Path(__file__).resolve().parent.parent.parent +sys.path.insert(0, str(REPO / "scripts" / "validation")) +from identity import norm_name # noqa: E402 + +_CACHE: dict = {} + + +def restricted_gallery_path(gallery_path: str, cast_jellyfin_ids: set[str], + cast_names: set[str] | None = None) -> str: + """Write (once, cached) a gallery filtered to the film's credited cast; return path. + + Matches gallery actors to the cast by jellyfin_id first, then normalized name.""" + key = (gallery_path, frozenset(cast_jellyfin_ids)) + if key in _CACHE: + return _CACHE[key] + + gal = json.loads(Path(gallery_path).read_text()) + names = {norm_name(n) for n in (cast_names or set())} + kept = [] + for a in gal["actors"]: + jid = a.get("jellyfin_id", "") + if (jid and jid in cast_jellyfin_ids) or (names and norm_name(a["name"]) in names): + kept.append(a) + + tf = tempfile.NamedTemporaryFile("w", suffix=".json", delete=False, + prefix="castgal_") + json.dump({"actors": kept}, tf) + tf.close() + _CACHE[key] = tf.name + return tf.name + + +def load_casts(casts_json: str) -> dict[str, list[str]]: + """film name → [jellyfin person id, ...] from jellyfin_casts.json.""" + return json.loads(Path(casts_json).read_text()) diff --git a/scripts/optimizer/dump_error_frames.py b/scripts/optimizer/dump_error_frames.py new file mode 100644 index 0000000..2892345 --- /dev/null +++ b/scripts/optimizer/dump_error_frames.py @@ -0,0 +1,201 @@ +#!/usr/bin/env python3 +""" +dump_error_frames.py — extract example video frames for visual inspection of a +replayed prediction vs X-Ray ground truth: best-agreement seconds, FPI (false +identification) seconds, and FN (missed cast) seconds. + +Reuses second_score.py's per-second timeline/prediction loading, but keeps the +per-second classification (score_seconds only returns aggregates) and picks +representative timestamps in each bucket, then pulls single frames from the +source video via ffmpeg -ss (nearest keyframe-independent seek + decode). + +If --raw (the JSONL from `replay.py --raw-out`) is given, also draws each visible +actor's bounding box + name/similarity on the extracted frame — green for +identified, orange for unknown — matching debug_renderer_node.hpp's colour +convention. Without --raw, frames are saved unannotated. + +Usage: + python scripts/optimizer/dump_error_frames.py \ + --pred pred.json --raw raw.jsonl \ + --xray experiments/xray/.../900_The_Many_Saints_Of_Newark \ + --movie "/mnt/movies/The Many Saints Of Newark (2021)/....mp4" \ + --gallery experiments/galleries/gallery_LVFace-B_Glint360K.h5 \ + --out-dir experiments/dump_review/many_saints --n-per-bucket 6 +""" +from __future__ import annotations + +import argparse +import json +import subprocess +import sys +from pathlib import Path + +import cv2 + +REPO = Path(__file__).resolve().parent.parent.parent +sys.path.insert(0, str(REPO / "scripts" / "optimizer")) +sys.path.insert(0, str(REPO / "scripts" / "validation")) + +from second_score import load_second_timeline, load_pred_intervals, _match # noqa: E402 +from sample_eval import load_gallery_keys # noqa: E402 +from identity import keys_for # noqa: E402 + + +def per_second_detail(pred_json: dict, xray_dir: str, gallery_keys: set | None): + """Like second_score.score_seconds, but yields one record per sampled second + instead of collapsing to aggregates.""" + timeline, film_cast, duration = load_second_timeline(xray_dir) + pred = load_pred_intervals(pred_json) + + name_by_keys = {} + for a in pred_json.get("actors", []): + k = frozenset(keys_for(imdb_id=a.get("imdb_id"), tmdb_id=a.get("tmdb_id"), + jellyfin_id=a.get("jellyfin_id"), name=a.get("name"))) + name_by_keys[k] = a.get("name", "?") + + records = [] + for t in sorted(timeline): + G = [set(a) for a in timeline[t]] + P_all = [(k, set(k)) for k, wins in pred if any(w0 <= t <= w1 for w0, w1 in wins)] + if gallery_keys is not None: + G = [g for g in G if g & gallery_keys] + + P = [p for _, p in P_all] + tp, matched = _match(P, G) + fp_names, fn_names = [], [] + for key, pa in P_all: + if not any(pa & ga for ga in G): + fp_names.append(name_by_keys.get(key, "?")) + for j, ga in enumerate(G): + if not matched[j]: + fn_names.append("|".join(sorted(x for x in ga if not x.startswith("imdb:") and not x.startswith("tmdb:"))) or "?") + + union = tp + len(fp_names) + len(fn_names) + jaccard = (tp / union) if union else 1.0 + records.append({"t": t, "tp": tp, "fp": fp_names, "fn": fn_names, "jaccard": jaccard}) + return records + + +def pick_timestamps(records, n_per_bucket): + best = sorted(records, key=lambda r: (-r["jaccard"], -r["tp"])) + best = [r for r in best if r["tp"] > 0][:n_per_bucket] + + fpi = [r for r in records if r["fp"]] + fpi = sorted(fpi, key=lambda r: -len(r["fp"]))[:n_per_bucket] + + fn = [r for r in records if r["fn"]] + fn = sorted(fn, key=lambda r: -len(r["fn"]))[:n_per_bucket] + + return {"best": best, "fpi": fpi, "fn": fn} + + +def pick_by_interval(records, interval_sec): + """One best (highest jaccard) and one worst (lowest jaccard) second per + interval_sec-second window across the whole film, e.g. --interval-sec 600 for + a per-10-minute best/worst sweep. Windows with no sampled seconds are skipped + (X-Ray timelines only cover scenes, so gaps between/after scenes are common).""" + windows: dict[int, list] = {} + for r in records: + windows.setdefault(r["t"] // interval_sec, []).append(r) + + buckets: dict[str, list] = {} + for w in sorted(windows): + wr = windows[w] + best = max(wr, key=lambda r: (r["jaccard"], r["tp"])) + worst = min(wr, key=lambda r: (r["jaccard"], -max(len(r["fp"]), len(r["fn"])))) + buckets[f"w{w:03d}_best"] = [best] + buckets[f"w{w:03d}_worst"] = [worst] + return buckets + + +def load_raw_annotations(raw_path: str): + """second (int, floor) -> list of visible_actors dicts (last frame wins if + several fall in the same second, which is the common case at 1fps sampling).""" + by_second = {} + with open(raw_path) as f: + for line in f: + sa = json.loads(line) + if sa.get("eof"): + continue + by_second[int(sa["timestamp_sec"])] = sa.get("visible_actors", []) + return by_second + + +def draw_annotations(frame_path: Path, actors: list): + img = cv2.imread(str(frame_path)) + if img is None: + return + for a in actors: + known = a.get("actor_idx", -1) >= 0 + colour = (60, 200, 0) if known else (220, 100, 0) # BGR: green / orange + x, y, w, h = a["bbox"] + x, y, w, h = int(x), int(y), int(w), int(h) + cv2.rectangle(img, (x, y), (x + w, y + h), colour, 2) + + label = f"{a['name']} {a['similarity']*100:.0f}%" if known else f"unknown {a['similarity']*100:.0f}%" + (tw, th), baseline = cv2.getTextSize(label, cv2.FONT_HERSHEY_SIMPLEX, 0.5, 1) + strip_y0 = max(0, y - th - 4) + cv2.rectangle(img, (x, strip_y0), (x + tw + 4, y), colour, cv2.FILLED) + cv2.putText(img, label, (x + 2, y - 2), cv2.FONT_HERSHEY_SIMPLEX, 0.5, + (255, 255, 255), 1, cv2.LINE_AA) + cv2.imwrite(str(frame_path), img) + + +def extract_frame(movie: str, t: float, out_path: Path): + out_path.parent.mkdir(parents=True, exist_ok=True) + subprocess.run( + ["ffmpeg", "-y", "-ss", str(t), "-i", movie, "-frames:v", "1", + "-q:v", "2", str(out_path)], + check=True, capture_output=True) + + +def main(): + p = argparse.ArgumentParser(description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter) + p.add_argument("--pred", required=True) + p.add_argument("--raw", help="raw per-frame annotations JSONL (replay.py --raw-out); " + "draws bboxes + names on extracted frames if given") + p.add_argument("--xray", required=True) + p.add_argument("--movie", required=True) + p.add_argument("--gallery") + p.add_argument("--out-dir", required=True) + p.add_argument("--n-per-bucket", type=int, default=6) + p.add_argument("--interval-sec", type=int, + help="instead of global best/fpi/fn buckets, pick one best + one " + "worst (by jaccard) second per interval-sec window across " + "the whole film, e.g. 600 for per-10-minute best/worst") + args = p.parse_args() + + pred_json = json.loads(Path(args.pred).read_text()) + gk = load_gallery_keys(args.gallery) if args.gallery else None + records = per_second_detail(pred_json, args.xray, gk) + buckets = (pick_by_interval(records, args.interval_sec) if args.interval_sec + else pick_timestamps(records, args.n_per_bucket)) + raw_by_second = load_raw_annotations(args.raw) if args.raw else None + + out_dir = Path(args.out_dir) + manifest = [] + for bucket, recs in buckets.items(): + for r in recs: + fname = f"{bucket}_t{r['t']:05d}.jpg" + out_path = out_dir / bucket / fname + try: + extract_frame(args.movie, r["t"], out_path) + ok = True + if raw_by_second is not None: + draw_annotations(out_path, raw_by_second.get(r["t"], [])) + except subprocess.CalledProcessError as e: + ok = False + print(f"[dump_error_frames] ffmpeg failed at t={r['t']}: {e}", file=sys.stderr) + manifest.append({"bucket": bucket, "t": r["t"], "tp": r["tp"], + "fp": r["fp"], "fn": r["fn"], "jaccard": round(r["jaccard"], 3), + "file": str(out_path.relative_to(out_dir)) if ok else None}) + print(f"[{bucket}] t={r['t']}s tp={r['tp']} fp={r['fp']} fn={r['fn']}", file=sys.stderr) + + (out_dir / "manifest.json").write_text(json.dumps(manifest, indent=2, ensure_ascii=False)) + print(f"[dump_error_frames] wrote {len(manifest)} frames + manifest.json to {out_dir}", + file=sys.stderr) + + +if __name__ == "__main__": + main() diff --git a/scripts/optimizer/dump_scene_montage.py b/scripts/optimizer/dump_scene_montage.py new file mode 100644 index 0000000..04dc006 --- /dev/null +++ b/scripts/optimizer/dump_scene_montage.py @@ -0,0 +1,383 @@ +#!/usr/bin/env python3 +""" +dump_scene_montage.py — one BEST and one WORST frame per X-Ray scene, split into +onscreen vs. offscreen actor identification (TPI / FPI / FN). + +For each X-Ray scene (scenes.csv span), scores every sampled second by a simple +per-second Jaccard agreement (TPI / (TPI+FPI+FN), same spirit as second_score.py) +and picks the single best-agreement and single worst-agreement second. Each gets +one output frame: the full frame (not a face crop) with a solid box drawn for +every currently-active TPI/FPI actor who has a REAL detection backing them, plus a +black caption panel below with two columns — Onscreen (has a real detection) and +Offscreen (no real detection: FN misses, and "ghost" detections where the tracker +is re-emitting a frozen last-known bbox with nothing there — see +docs/rep4-optimizer-results.md) — names colour-coded by bucket, with a legend. + +A predicted bbox is checked against the dump's OWN raw per-frame face detections +(IoU) to tell a real detection from a ghost. Ghosts are NEVER drawn as boxes (they +have no real screen position); they only appear as a name in the Offscreen column. + +Frames where at least one FPI name isn't in the film's cast AT ALL (an out-of-cast +misID, not just a right-actor/wrong-scene timing slip) are also copied into +/out_of_cast_fpi/ for quick review of the most confident wrong answers. + +Requires the raw per-frame annotations from `replay.py --raw-out` (bboxes aren't +in the merged pred.json) and the film's HDF5 dump (for ghost-checking against real +detections). + +Usage: + python scripts/optimizer/dump_scene_montage.py \ + --raw raw.jsonl --dump experiments/dumps/.../dump_X.h5 \ + --xray experiments/xray/.../900_The_Many_Saints_Of_Newark \ + --movie "/mnt/movies/.../X.mp4" \ + --gallery experiments/galleries/gallery_LVFace-B_Glint360K.h5 \ + --out-dir experiments/results/holdout/montage/many_saints --scene 5 +""" +from __future__ import annotations + +import argparse +import csv +import json +import subprocess +import sys +from pathlib import Path + +import cv2 +import h5py +import numpy as np + +REPO = Path(__file__).resolve().parent.parent.parent +sys.path.insert(0, str(REPO / "scripts" / "optimizer")) +sys.path.insert(0, str(REPO / "scripts" / "validation")) + +from sample_eval import load_gallery_keys # noqa: E402 +from identity import keys_for # noqa: E402 + +COLOUR_TPI = (60, 200, 0) # green, BGR +COLOUR_FPI = (0, 60, 220) # red, BGR +CAPTION_H = 28 # px per line in the bottom strip + + +def load_scene_spans(xray_dir: str): + """scene_id -> (t0_sec, t1_sec), from scenes.csv (ms).""" + spans = {} + with open(Path(xray_dir) / "scenes.csv", newline="", encoding="utf-8") as f: + for r in csv.DictReader(f): + sn = (r.get("scene") or "").strip() + try: + spans[sn] = (float(r["start"]) / 1000.0, float(r["end"]) / 1000.0) + except (KeyError, ValueError): + continue + return spans + + +def load_film_cast(xray_dir: str) -> set: + """Every actor key X-Ray credits ANYWHERE in the film — used to tell an + out-of-cast misID (named someone who isn't even in this film) apart from an + in-cast timing slip (right actor, wrong scene), same distinction as + second_score.py's FPI_misid vs FPI_incast.""" + keys = set() + with open(Path(xray_dir) / "people.csv", newline="", encoding="utf-8") as f: + for r in csv.DictReader(f): + nm = (r.get("name_id") or "").strip() + person = (r.get("person") or "").strip() + if nm or person: + keys |= keys_for(imdb_id=nm, name=person) + return keys + + +def load_scene_cast(xray_dir: str): + """scene_id -> set of actor key-frozensets X-Ray lists as present.""" + id_to_name = {} + with open(Path(xray_dir) / "people.csv", newline="", encoding="utf-8") as f: + for r in csv.DictReader(f): + nm = (r.get("name_id") or "").strip() + if nm: + id_to_name[nm] = (r.get("person") or "").strip() + + scene_cast: dict[str, set] = {} + with open(Path(xray_dir) / "people_in_scenes.csv", newline="", encoding="utf-8") as f: + for r in csv.DictReader(f): + sn = (r.get("scene") or "").strip() + nm = (r.get("name_id") or "").strip() + if sn and nm: + scene_cast.setdefault(sn, set()).add( + frozenset(keys_for(imdb_id=nm, name=id_to_name.get(nm)))) + return scene_cast + + +def load_raw_by_second(raw_path: str): + by_second: dict[int, list] = {} + with open(raw_path) as f: + for line in f: + sa = json.loads(line) + if sa.get("eof"): + continue + by_second[int(sa["timestamp_sec"])] = sa.get("visible_actors", []) + return by_second + + +def load_dump_faces_by_second(dump_path: str): + """second (int) -> list of raw detected bboxes (x,y,w,h), for ghost-checking. + A predicted actor's bbox is real iff it overlaps one of these; a bbox with no + overlap at all is a frozen/stale re-emission, not an actual detection.""" + by_second: dict[int, list] = {} + with h5py.File(dump_path, "r") as f: + ts = f["frames/timestamp_sec"][:] + off = f["frames/face_offset"][:] + cnt = f["frames/face_count"][:] + bbox = f["faces/bbox"][:] + for i in range(len(ts)): + s, n = int(off[i]), int(cnt[i]) + by_second[int(ts[i])] = [tuple(b) for b in bbox[s:s + n]] + return by_second + + +def iou(a, b): + ax, ay, aw, ah = a + bx, by, bw, bh = b + ix0, iy0 = max(ax, bx), max(ay, by) + ix1, iy1 = min(ax + aw, bx + bw), min(ay + ah, by + bh) + iw, ih = max(0.0, ix1 - ix0), max(0.0, iy1 - iy0) + inter = iw * ih + union = aw * ah + bw * bh - inter + return inter / union if union > 0 else 0.0 + + +def is_ghost(bbox, real_boxes, iou_thresh=0.3): + return not any(iou(bbox, rb) >= iou_thresh for rb in real_boxes) + + +def actor_key(a: dict) -> frozenset: + return frozenset(keys_for(imdb_id=a.get("imdb_id"), tmdb_id=a.get("tmdb_id"), + jellyfin_id=a.get("jellyfin_id"), name=a.get("name"))) + + +COLOUR_FN = (220, 130, 0) # blue, BGR +LEGEND = (("TPI (correct)", COLOUR_TPI), ("FPI (wrong)", COLOUR_FPI), + ("FN (missed)", COLOUR_FN)) + + +def render_frame(frame_path: Path, t: int, tpi_boxes: list, fpi_boxes: list, entries: list): + """entries: list of (name, bucket, onscreen) — bucket in {tpi,fpi,fn}, + onscreen=True iff a real detected face backs this name at this second. Ghost + detections (bucket fpi/tpi but no real face — see is_ghost) are never drawn as + boxes: they have no real screen position, they only ever appear in the + Offscreen column.""" + img = cv2.imread(str(frame_path)) + if img is None: + return None + + for name, bbox, sim in tpi_boxes: + x, y, w, h = (int(v) for v in bbox) + cv2.rectangle(img, (x, y), (x + w, y + h), COLOUR_TPI, 2) + _label(img, (x, y), f"{name} {sim*100:.0f}%", COLOUR_TPI) + for name, bbox, sim in fpi_boxes: + x, y, w, h = (int(v) for v in bbox) + cv2.rectangle(img, (x, y), (x + w, y + h), COLOUR_FPI, 2) + _label(img, (x, y), f"{name} {sim*100:.0f}%", COLOUR_FPI) + + h_img, w_img = img.shape[:2] + bucket_colour = {"tpi": COLOUR_TPI, "fpi": COLOUR_FPI, "fn": COLOUR_FN} + onscreen = [(n, bucket_colour[b]) for n, b, on in entries if on] + offscreen = [(n, bucket_colour[b]) for n, b, on in entries if not on] + + n_rows = max(len(onscreen), len(offscreen), 1) + header_h = 24 + legend_h = CAPTION_H + table_h = header_h + n_rows * CAPTION_H + legend_h + 16 + canvas = np.zeros((h_img + table_h, w_img, 3), dtype=np.uint8) # black bg + canvas[:h_img] = img + + col_x = (8, w_img // 2 + 8) + cv2.putText(canvas, f"t={t}s", (8, 16), cv2.FONT_HERSHEY_SIMPLEX, 0.5, + (255, 255, 255), 1, cv2.LINE_AA) + y0 = h_img + header_h + cv2.putText(canvas, "Onscreen", (col_x[0], y0), cv2.FONT_HERSHEY_SIMPLEX, 0.55, + (255, 255, 255), 1, cv2.LINE_AA) + cv2.putText(canvas, "Offscreen", (col_x[1], y0), cv2.FONT_HERSHEY_SIMPLEX, 0.55, + (255, 255, 255), 1, cv2.LINE_AA) + cv2.line(canvas, (col_x[1] - 8, h_img), (col_x[1] - 8, h_img + table_h), + (90, 90, 90), 1) + + for i in range(n_rows): + y = y0 + CAPTION_H * (i + 1) + if i < len(onscreen): + name, colour = onscreen[i] + cv2.putText(canvas, name, (col_x[0], y), cv2.FONT_HERSHEY_SIMPLEX, 0.5, + colour, 1, cv2.LINE_AA) + if i < len(offscreen): + name, colour = offscreen[i] + cv2.putText(canvas, name, (col_x[1], y), cv2.FONT_HERSHEY_SIMPLEX, 0.5, + colour, 1, cv2.LINE_AA) + + ly = y0 + CAPTION_H * (n_rows + 1) + 4 + lx = 8 + for label, colour in LEGEND: + (tw, _), _ = cv2.getTextSize(label, cv2.FONT_HERSHEY_SIMPLEX, 0.42, 1) + cv2.rectangle(canvas, (lx, ly - 10), (lx + 12, ly + 2), colour, cv2.FILLED) + cv2.putText(canvas, label, (lx + 18, ly), cv2.FONT_HERSHEY_SIMPLEX, 0.42, + (200, 200, 200), 1, cv2.LINE_AA) + lx += tw + 40 + return canvas + + +def _label(img, pt, text, colour): + x, y = pt + (tw, th), _ = cv2.getTextSize(text, cv2.FONT_HERSHEY_SIMPLEX, 0.5, 1) + strip_y0 = max(0, y - th - 4) + cv2.rectangle(img, (x, strip_y0), (x + tw + 4, y), colour, cv2.FILLED) + cv2.putText(img, text, (x + 2, y - 2), cv2.FONT_HERSHEY_SIMPLEX, 0.5, + (255, 255, 255), 1, cv2.LINE_AA) + + +def extract_frame(movie: str, t: float, out_path: Path): + out_path.parent.mkdir(parents=True, exist_ok=True) + subprocess.run( + ["ffmpeg", "-y", "-ss", str(t), "-i", movie, "-frames:v", "1", + "-q:v", "2", str(out_path)], + check=True, capture_output=True) + + +def classify_second(t: int, gt_cast: set, film_cast: set, raw_by_second: dict, + dump_faces_by_second: dict): + """One second's TPI/FPI/FN classification: (score, tpi_boxes, fpi_boxes, + entries, has_outofcast). score = Jaccard-style agreement in [0,1], used to + rank seconds for best/worst picking.""" + actors = raw_by_second.get(t, []) + real_boxes = dump_faces_by_second.get(t, []) + + tpi_boxes, fpi_boxes = [], [] + entries = [] # (name, bucket, onscreen) + cur_state: dict[frozenset, str] = {} + has_outofcast = False + + for a in actors: + if a.get("actor_idx", -1) < 0: + continue + key = actor_key(a) + name = a.get("name", "?") + bbox = tuple(a["bbox"]) + sim = a.get("similarity", 0.0) + ghost = is_ghost(bbox, real_boxes) + hit = any(key & g for g in gt_cast) + + if ghost: + cur_state[key] = "ghost" + entries.append((name, "fpi" if not hit else "tpi", False)) + elif hit: + tpi_boxes.append((name, bbox, sim)) + cur_state[key] = "tpi" + entries.append((name, "tpi", True)) + else: + fpi_boxes.append((name, bbox, sim)) + cur_state[key] = "fpi" + entries.append((name, "fpi", True)) + + if not hit and not (key & film_cast): + has_outofcast = True # named someone not in the film at all (FPI_misid) + + tpi_keys = [k for k, state in cur_state.items() if state in ("tpi", "ghost")] + fn_count = 0 + for g in gt_cast: + if any(g & k for k in tpi_keys): + continue + nm = next((x.split("name:", 1)[1] for x in g if x.startswith("name:")), None) + entries.append((nm or next(iter(g), "?"), "fn", False)) + fn_count += 1 + + tp = sum(1 for _, b, on in entries if b == "tpi" and on) + fp = sum(1 for _, b, on in entries if b == "fpi") + union = tp + fp + fn_count + score = tp / union if union else 1.0 # both-empty = perfect agreement + return score, tpi_boxes, fpi_boxes, entries, has_outofcast + + +def process_scene(scene_id: str, t0: float, t1: float, gt_cast: set, film_cast: set, + raw_by_second: dict, dump_faces_by_second: dict, + gallery_keys: set | None, movie: str, out_dir: Path, + outofcast_dir: Path): + if gallery_keys is not None: + gt_cast = {g for g in gt_cast if g & gallery_keys} + + per_second = {} + for t in range(int(t0), int(t1)): + per_second[t] = classify_second(t, gt_cast, film_cast, raw_by_second, + dump_faces_by_second) + if not per_second: + return [] + + best_t = max(per_second, key=lambda t: per_second[t][0]) + worst_t = min(per_second, key=lambda t: per_second[t][0]) + + manifest = [] + for label, t in (("best", best_t), ("worst", worst_t)): + score, tpi_boxes, fpi_boxes, entries, has_outofcast = per_second[t] + fname = f"{scene_id}_{label}_t{t:06d}.jpg" + out_path = out_dir / fname + try: + extract_frame(movie, t, out_path) + canvas = render_frame(out_path, t, tpi_boxes, fpi_boxes, entries) + if canvas is not None: + cv2.imwrite(str(out_path), canvas) + manifest.append({"label": label, "t": t, "score": round(score, 3), + "entries": entries, "file": fname, + "outofcast": has_outofcast}) + print(f"[scene {scene_id}] {label} t={t}s score={score:.2f} " + f"entries={entries}", file=sys.stderr) + if has_outofcast: + outofcast_dir.mkdir(parents=True, exist_ok=True) + cv2.imwrite(str(outofcast_dir / fname), cv2.imread(str(out_path))) + except subprocess.CalledProcessError as e: + print(f"[dump_scene_montage] ffmpeg failed at t={t}: {e}", file=sys.stderr) + + return manifest + + +def main(): + p = argparse.ArgumentParser(description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter) + p.add_argument("--raw", required=True, help="raw per-frame annotations (replay.py --raw-out)") + p.add_argument("--dump", required=True, help="film's HDF5 embedding dump (for ghost-checking)") + p.add_argument("--xray", required=True) + p.add_argument("--movie", required=True) + p.add_argument("--gallery") + p.add_argument("--out-dir", required=True) + p.add_argument("--scene", help="only process this X-Ray scene id (default: all)") + args = p.parse_args() + + spans = load_scene_spans(args.xray) + scene_cast = load_scene_cast(args.xray) + film_cast = load_film_cast(args.xray) + raw_by_second = load_raw_by_second(args.raw) + dump_faces_by_second = load_dump_faces_by_second(args.dump) + gk = load_gallery_keys(args.gallery) if args.gallery else None + + out_dir = Path(args.out_dir) + out_dir.mkdir(parents=True, exist_ok=True) + outofcast_dir = out_dir / "out_of_cast_fpi" + + scene_ids = [args.scene] if args.scene else sorted(spans, key=lambda s: spans[s][0]) + all_manifest = {} + for sn in scene_ids: + if sn not in spans: + print(f"[dump_scene_montage] unknown scene id: {sn}", file=sys.stderr) + continue + t0, t1 = spans[sn] + gt_cast = scene_cast.get(sn, set()) + scene_dir = out_dir / f"scene_{sn}" + scene_dir.mkdir(parents=True, exist_ok=True) + m = process_scene(sn, t0, t1, gt_cast, film_cast, raw_by_second, + dump_faces_by_second, gk, args.movie, scene_dir, outofcast_dir) + all_manifest[sn] = m + + (out_dir / "manifest.json").write_text(json.dumps(all_manifest, indent=2, ensure_ascii=False)) + total = sum(len(v) for v in all_manifest.values()) + n_outofcast = sum(1 for v in all_manifest.values() for r in v if r.get("outofcast")) + print(f"[dump_scene_montage] wrote {total} best/worst frames across " + f"{len(all_manifest)} scenes to {out_dir} " + f"({n_outofcast} copied to {outofcast_dir})", file=sys.stderr) + + +if __name__ == "__main__": + main() diff --git a/scripts/optimizer/fetch_missing_actors.py b/scripts/optimizer/fetch_missing_actors.py new file mode 100644 index 0000000..f125c6b --- /dev/null +++ b/scripts/optimizer/fetch_missing_actors.py @@ -0,0 +1,154 @@ +#!/usr/bin/env python3 +""" +fetch_missing_actors.py — close the gallery coverage gap. + +X-Ray credits ~67% of each film's cast that our gallery never had a reference +embedding for, making those actors unrecoverable FNs no threshold can fix. This +fetches images for those missing actors (by IMDb nm id → TMDB profile photos), +embeds them with the SAME SCRFD+ArcFace models (sae_embed), and writes gallery +entries. Merge the result into the baseline to make those actors recognisable. + +nm → TMDB person → /person/{id}/images profile photos → download → embed. + +Usage: + python scripts/optimizer/fetch_missing_actors.py \ + --missing missing_actors.json \ + --out gallery_missing.json \ + [--images-per-actor 3] [--build-dir build] + # TMDB_API_KEY from env/.env + +Then merge: + python scripts/optimizer/fetch_missing_actors.py --merge \ + gallery_arcface_w600k_r50.json gallery_missing.json \ + --out gallery_augmented.json +""" +from __future__ import annotations + +import argparse +import json +import os +import sys +import tempfile +from pathlib import Path + +REPO = Path(__file__).resolve().parent.parent.parent +sys.path.insert(0, str(REPO / "scripts")) +import sae_env # noqa: E402 loads .env +from sae_tmdb import tmdb_get, tmdb_person_for_imdb, TMDB_IMG # noqa: E402 +from sae_gallery import download_images, wikidata_image_urls # noqa: E402 +from sae_embed_loader import load_embedder # noqa: E402 + + +def profile_urls_for_imdb(imdb_id: str, token: str, n: int) -> tuple[str | None, list[str]]: + """(tmdb_person_id, [image_url,...]) via /find then /person/{id}/images.""" + data = tmdb_get(f"/find/{imdb_id}", token, external_source="imdb_id") + people = data.get("person_results", []) + if not people: + return None, [] + pid = str(people[0]["id"]) + imgs = tmdb_get(f"/person/{pid}/images", token) + profiles = imgs.get("profiles", [])[:n] + return pid, [TMDB_IMG + p["file_path"] for p in profiles if p.get("file_path")] + + +def fetch(missing_path, out_path, token, build_dir, models_dir, arcface, + images_per_actor, use_wikidata=False): + missing = json.loads(Path(missing_path).read_text()) + src = "TMDB + Wikidata fallback" if use_wikidata else "TMDB" + print(f"[fetch] {len(missing)} missing actors to resolve via {src}", file=sys.stderr) + embedder = load_embedder(build_dir, models_dir, arcface) + + img_root = Path(tempfile.mkdtemp(prefix="missing_gallery_")) + actors = [] + n_resolved = n_no_tmdb = n_no_img = n_no_face = 0 + n_via_wikidata = 0 + + for i, m in enumerate(missing, 1): + nm, name = m["imdb_id"], m.get("name", "") + tmdb_id, urls = None, [] + try: + tmdb_id, urls = profile_urls_for_imdb(nm, token, images_per_actor) + except Exception as e: + print(f" [{i}] {name}: TMDB error {e}", file=sys.stderr) + # Wikidata fallback: keyed cleanly by IMDb nm (P345→P18 Commons photo), + # recovers on-camera character actors TMDB's film-centric DB misses. + if (not urls) and use_wikidata: + wiki_urls = wikidata_image_urls(nm)[:images_per_actor] + if wiki_urls: + urls = wiki_urls + n_via_wikidata += 1 + if not urls: + if tmdb_id is None: + n_no_tmdb += 1 + else: + n_no_img += 1 + continue + dest = img_root / nm + dest.mkdir(parents=True, exist_ok=True) + paths = download_images(urls, dest, images_per_actor) + embeddings = [] + for p in paths: + res = embedder.embed(str(p)) + if res.ok: + embeddings.append(list(res.embedding)) + if not embeddings: + n_no_face += 1 + continue + actors.append({"imdb_id": nm, "tmdb_id": str(tmdb_id) if tmdb_id else "", + "jellyfin_id": "", "name": name, + "embeddings": embeddings, "source_images": []}) + n_resolved += 1 + if i % 20 == 0 or i == len(missing): + print(f" [{i}/{len(missing)}] resolved={n_resolved} " + f"(wiki={n_via_wikidata}) no_tmdb={n_no_tmdb} no_img={n_no_img} " + f"no_face={n_no_face}", file=sys.stderr) + + Path(out_path).write_text(json.dumps({"actors": actors}, indent=2)) + n_emb = sum(len(a["embeddings"]) for a in actors) + print(f"\n[fetch] recovered {n_resolved}/{len(missing)} actors " + f"({n_via_wikidata} via Wikidata), {n_emb} embeddings → {out_path}", + file=sys.stderr) + print(f"[fetch] unrecoverable: no_tmdb={n_no_tmdb} no_img={n_no_img} " + f"no_face={n_no_face}", file=sys.stderr) + + +def merge(base_path, add_path, out_path): + base = json.loads(Path(base_path).read_text()) + add = json.loads(Path(add_path).read_text()) + have = {a.get("imdb_id") for a in base["actors"] if a.get("imdb_id")} + added = [a for a in add["actors"] if a.get("imdb_id") not in have] + base["actors"].extend(added) + Path(out_path).write_text(json.dumps(base, indent=2)) + print(f"[merge] {len(base['actors'])-len(added)} + {len(added)} = " + f"{len(base['actors'])} actors → {out_path}", file=sys.stderr) + + +def main(): + p = argparse.ArgumentParser(description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter) + p.add_argument("--merge", nargs=2, metavar=("BASE", "ADD"), + help="merge ADD gallery into BASE → --out") + p.add_argument("--missing") + p.add_argument("--out", required=True) + p.add_argument("--tmdb-key", default=os.environ.get("TMDB_API_KEY")) + p.add_argument("--build-dir", default=str(REPO / "build")) + p.add_argument("--models-dir", default=str(REPO / "models")) + p.add_argument("--arcface", default=None) + p.add_argument("--images-per-actor", type=int, default=3) + p.add_argument("--wikidata", action="store_true", + help="fall back to Wikidata (P345→P18 Commons photo) when TMDB has no image") + args = p.parse_args() + + if args.merge: + merge(args.merge[0], args.merge[1], args.out) + return + if not args.missing: + sys.exit("--missing required (or use --merge)") + if not args.tmdb_key: + sys.exit("no TMDB key — set TMDB_API_KEY") + fetch(args.missing, args.out, args.tmdb_key, args.build_dir, args.models_dir, + args.arcface, args.images_per_actor, use_wikidata=args.wikidata) + + +if __name__ == "__main__": + main() diff --git a/scripts/optimizer/gallery_membership.py b/scripts/optimizer/gallery_membership.py new file mode 100644 index 0000000..1b4611b --- /dev/null +++ b/scripts/optimizer/gallery_membership.py @@ -0,0 +1,94 @@ +#!/usr/bin/env python3 +""" +gallery_membership.py — definitive per-film gallery coverage of X-Ray cast. + +For each film, splits the X-Ray cast (people.csv) into those WITH a gallery reference +embedding and those WITHOUT. This is the model-independent foundation for honest +FP/FN rates: because every model's gallery is built from the SAME TMDB source images +(same actors), the membership list is identical across models — only the embedding +values differ. So FN can be measured over the recognisable denominator (in-gallery +cast) and out-of-cast misIDs (predicted actor not in the film at all) are well defined. + +Outputs experiments/results/membership.json: + { film: { + xray_cast: N, in_gallery: M, coverage: M/N, + in_gallery_names: [...], missing_names: [...] } } + +Usage: + python scripts/optimizer/gallery_membership.py \ + --manifest experiments/manifests/films.json \ + --gallery gallery_arcface_w600k_r50.json \ + --out experiments/results/membership.json +""" +from __future__ import annotations + +import argparse +import csv +import json +import sys +from pathlib import Path + +REPO = Path(__file__).resolve().parent.parent.parent +sys.path.insert(0, str(REPO / "scripts" / "validation")) +from identity import keys_for # noqa: E402 + + +def gallery_keyset(gallery_path: str) -> set: + keys = set() + for a in json.loads(Path(gallery_path).read_text())["actors"]: + if not a.get("embeddings"): + continue # no embedding = not actually recognisable + keys |= keys_for(imdb_id=a.get("imdb_id"), tmdb_id=a.get("tmdb_id"), + jellyfin_id=a.get("jellyfin_id"), name=a.get("name")) + return keys + + +def film_cast(xray_dir: str) -> dict[str, str]: + """nm_id → person name from a film's X-Ray people.csv.""" + out = {} + with open(Path(xray_dir) / "people.csv", newline="", encoding="utf-8") as f: + for r in csv.DictReader(f): + nm = (r.get("name_id") or "").strip() + if nm: + out[nm] = (r.get("person") or "").strip() + return out + + +def main(): + p = argparse.ArgumentParser(description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter) + p.add_argument("--manifest", required=True) + p.add_argument("--gallery", required=True) + p.add_argument("--out", required=True) + args = p.parse_args() + + gkeys = gallery_keyset(args.gallery) + films = json.loads(Path(args.manifest).read_text()) + + report = {} + tot_cast = tot_in = 0 + print(f"{'film':32s} {'cast':>5s} {'in-gal':>7s} {'cover':>6s}") + for f in films: + cast = film_cast(f["xray"]) + in_g, miss = [], [] + for nm, name in cast.items(): + if keys_for(imdb_id=nm, name=name) & gkeys: + in_g.append(name) + else: + miss.append(name) + n, m = len(cast), len(in_g) + tot_cast += n; tot_in += m + report[f["name"]] = {"xray_cast": n, "in_gallery": m, + "coverage": round(m / n, 3) if n else 0.0, + "in_gallery_names": sorted(in_g), + "missing_names": sorted(miss)} + print(f"{f['name'][:32]:32s} {n:>5d} {m:>7d} {m/n*100 if n else 0:>5.0f}%") + print(f"{'TOTAL':32s} {tot_cast:>5d} {tot_in:>7d} {tot_in/tot_cast*100:>5.0f}%") + + Path(args.out).parent.mkdir(parents=True, exist_ok=True) + Path(args.out).write_text(json.dumps(report, indent=2)) + print(f"\n→ {args.out}") + + +if __name__ == "__main__": + main() diff --git a/scripts/optimizer/optimize.py b/scripts/optimizer/optimize.py new file mode 100644 index 0000000..66413a2 --- /dev/null +++ b/scripts/optimizer/optimize.py @@ -0,0 +1,240 @@ +#!/usr/bin/env python3 +""" +optimize.py — Differential Evolution over pipeline thresholds, scored against X-Ray. + +Replaces the coarse grid sweep with scipy's differential_evolution over the +continuous knob space. Each candidate config is a full-9-film replay (real KPN +nodes) scored against Amazon X-Ray presence, micro-averaged. The gallery is loaded +once per process (binding caches by path), so an evaluation is just N cheap replays. + +Objective: **maximize micro-F1** (DE minimizes, so we return -F1). NOTE: X-Ray recall +is a face-vs-cast-in-scene ceiling (see [[xray-validation-results]]), so unconstrained +F1 tends to push prob_threshold DOWN to recover unreachable recall — trading real +precision for it. We therefore log precision/recall at every evaluation and print +them at the optimum so the trade-off is visible and you can pick another operating +point from the trajectory (--trajectory). + +Usage: + python scripts/optimizer/optimize.py --manifest films.json \ + --gallery gallery_arcface_w600k_r50.json \ + --params prob_threshold:0.5:0.999 anneal_sec:1:30 extinction_sec:1:15 \ + --popsize 20 --maxiter 25 --trajectory traj.json +""" +from __future__ import annotations + +import argparse +import json +import sys +import time +from pathlib import Path + +import numpy as np +from scipy.optimize import differential_evolution + +REPO = Path(__file__).resolve().parent.parent.parent +sys.path.insert(0, str(REPO / "scripts" / "optimizer")) +sys.path.insert(0, str(REPO / "scripts" / "validation")) + +import json as _json +import os +import subprocess +import tempfile +import threading +from concurrent.futures import ThreadPoolExecutor + +# Concurrent per-eval replays. Each replay is an isolated subprocess, so parallelism +# is deadlock-safe; with 9 films/eval, 8 workers replays nearly all at once. Tune via +# REPLAY_WORKERS (8 is the measured sweet spot on this 24GB GPU). +REPLAY_WORKERS = int(os.environ.get("REPLAY_WORKERS", "8")) + +# DE-level parallelism: how many population candidates get evaluated concurrently +# (each spawning its own REPLAY_WORKERS film subprocesses). Total concurrent GPU +# replay processes ≈ DE_WORKERS × min(REPLAY_WORKERS, n_films). Threads, not +# multiprocessing — each objective() call just waits on subprocess.run, so threads +# share the GIL fine and avoid pickling the objective/gallery-key cache. +DE_WORKERS = int(os.environ.get("DE_WORKERS", "1")) + +from second_score import score_seconds # noqa: E402 uniform per-second TPI/FPI scoring +from sample_eval import load_gallery_keys # noqa: E402 + +_GAL_KEYS: dict = {} # gallery path → key set (fair-recall FN mask), loaded once +_REPLAY_TIMEOUT = 45 # seconds per film; a wedged replay is killed, not left to hang + +REPLAY_CLI = str(Path(__file__).resolve().parent / "replay.py") + + +def _gallery_keys(path): + if path not in _GAL_KEYS: + _GAL_KEYS[path] = load_gallery_keys(path) + return _GAL_KEYS[path] + + +def _replay_subprocess(dump, gallery, cfg, build_dir): + """Run one replay in a SUBPROCESS with a timeout, returning its presence JSON. + + In-process replay intermittently DEADLOCKS at network teardown — a KPN worker + stuck mid-rocBLAS GEMM inside the ROCm driver makes ~PyNode's jthread.join() hang + forever (root-caused via gdb, 2026-07-15). Isolating each replay means a wedged + GPU thread only kills that subprocess; the sweep continues. Returns None on + timeout/failure (the caller drops that film from the average).""" + with tempfile.NamedTemporaryFile("r", suffix=".json", delete=False) as tf: + out = tf.name + argv = [sys.executable, REPLAY_CLI, "--dump", dump, "--gallery", gallery, + "--out", out, "--build-dir", build_dir] + for k, v in cfg.items(): + if isinstance(v, bool): # store_true flags: pass the flag, not a value + if v: + argv.append(f"--{k.replace('_', '-')}") + else: + argv += [f"--{k.replace('_', '-')}", str(v)] + try: + subprocess.run(argv, timeout=_REPLAY_TIMEOUT, capture_output=True, check=True) + return _json.loads(Path(out).read_text()) + except (subprocess.TimeoutExpired, subprocess.CalledProcessError, + FileNotFoundError, ValueError) as e: + print(f"[opt] replay failed for {Path(dump).name}: {type(e).__name__}", + file=sys.stderr) + return None + finally: + try: + Path(out).unlink() + except OSError: + pass + + +def evaluate(cfg, films, build_dir, step=None): + """Objective = MACRO-mean over films of each film's duration-weighted per-scene F1. + + Each film's replay runs in a subprocess (timeout-guarded) to survive the + intermittent ROCm teardown deadlock. A film whose replay times out is dropped + from the average rather than hanging the whole sweep. + + UNIFORM PER-SECOND scoring (second_score.py): every second of the film is sampled; + GT(t) = the cast of the X-Ray scene containing t, Pred(t) = actors whose presence + window covers t. Counts instances — TPI / FPI / FN — with FPI weighted 10× when the + named actor isn't in the film's cast at all (a real misID vs a timing slip). FN + counts only gallery-known actors (fair recall). Reports agreement_rate = mean + per-second Jaccard (the "% of on-screen actors we agree with X-Ray about, over + time"). Objective = macro-mean across films of the per-second weighted F1. + + expand_gallery: controlled by env SAE_EXPAND (default on). Set SAE_EXPAND=0 to run + the no-expansion arm — the overnight matrix tests both to quantify what expansion buys. + + The 9 films' replays run CONCURRENTLY (REPLAY_WORKERS) — each is an isolated + subprocess, so parallelism is safe (a wedged one only kills itself).""" + if os.environ.get("SAE_EXPAND", "1") == "1": + cfg = {**cfg, "expand_gallery": True} + + def _one(film): + pj = _replay_subprocess(film["dump"], film.get("gallery"), cfg, build_dir) + if pj is None: + return None + return score_seconds(pj, film["xray"], + gallery_keys=_gallery_keys(film.get("gallery"))) + + with ThreadPoolExecutor(max_workers=REPLAY_WORKERS) as ex: + per_film = [m for m in ex.map(_one, films) if m is not None] + n = len(per_film) + if not n: + return {"precision": 0.0, "recall": 0.0, "f1": 0.0, "agreement": 0.0, + "TPI": 0, "FPI": 0, "FPI_misid": 0, "FN": 0} + return {"precision": sum(m["precision"] for m in per_film) / n, + "recall": sum(m["recall"] for m in per_film) / n, + "f1": sum(m["f1"] for m in per_film) / n, + "agreement": sum(m["agreement_rate"] for m in per_film) / n, + "TPI": sum(m["TPI"] for m in per_film), + "FPI": sum(m["FPI"] for m in per_film), + "FPI_misid": sum(m["FPI_misid"] for m in per_film), + "FN": sum(m["FN"] for m in per_film)} + + +def main(): + p = argparse.ArgumentParser(description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter) + p.add_argument("--manifest", required=True) + p.add_argument("--gallery", help="default gallery if not per-film") + p.add_argument("--params", nargs="+", required=True, + help="knob:lo:hi (e.g. prob_threshold:0.5:0.999). Int knobs kept float, rounded in cfg.") + p.add_argument("--build-dir", default=str(REPO / "build")) + p.add_argument("--step", type=float, default=5.0) + p.add_argument("--popsize", type=int, default=20) + p.add_argument("--maxiter", type=int, default=25) + p.add_argument("--seed", type=int, default=0) + p.add_argument("--trajectory", help="write every evaluation here (JSON lines)") + p.add_argument("--out", help="write best config + metrics") + args = p.parse_args() + + films = json.loads(Path(args.manifest).read_text()) + for f in films: + f.setdefault("gallery", args.gallery) + if not Path(f["dump"]).exists(): + sys.exit(f"[opt] missing dump for {f['name']}: {f['dump']}") + + names, bounds = [], [] + int_knobs = {"track_max_frames_missing", "cut_inactive_max_frames"} + for spec in args.params: + k, lo, hi = spec.split(":") + names.append(k); bounds.append((float(lo), float(hi))) + print(f"[opt] DE over {names} bounds={bounds}", file=sys.stderr) + print(f"[opt] {len(films)} films, popsize={args.popsize}, maxiter={args.maxiter}", file=sys.stderr) + + traj = [] + evals = [0] + t0 = time.time() + traj_lock = threading.Lock() + + def vec_to_cfg(x): + cfg = {} + for k, v in zip(names, x): + cfg[k] = int(round(v)) if k in int_knobs else float(v) + return cfg + + def objective(x): + cfg = vec_to_cfg(x) + m = evaluate(cfg, films, args.build_dir, args.step) + with traj_lock: + evals[0] += 1 + rec = {"eval": evals[0], "config": cfg, **m, "t": round(time.time() - t0, 1)} + traj.append(rec) + print(f"[opt] eval {evals[0]:3d} thr={cfg['prob_threshold']:.2f} " + f"ann={cfg['anneal_sec']:.0f} ext={cfg['extinction_sec']:.1f} → " + f"F1={m['f1']*100:.1f}% P={m['precision']*100:.1f}% R={m['recall']*100:.1f}% " + f"agree={m.get('agreement', 0)*100:.1f}% misID={m.get('FPI_misid', 0)}", + file=sys.stderr) + if args.trajectory: + with open(args.trajectory, "a") as tf: + tf.write(json.dumps(rec) + "\n") + return -m["f1"] + + de_kwargs = dict( + popsize=args.popsize, maxiter=args.maxiter, + seed=args.seed, polish=False, tol=1e-4, mutation=(0.5, 1.0), recombination=0.7, + init="sobol") + if DE_WORKERS > 1: + pool = ThreadPoolExecutor(max_workers=DE_WORKERS) + de_kwargs["workers"] = pool.map + result = differential_evolution(objective, bounds, **de_kwargs) + + best_cfg = vec_to_cfg(result.x) + best = evaluate(best_cfg, films, args.build_dir, args.step) + print("\n══ DE optimum (by F1) ═══════════════════════════") + print(f" config : {best_cfg}") + print(f" F1 : {best['f1']*100:.2f}%") + print(f" precision: {best['precision']*100:.2f}% recall: {best['recall']*100:.2f}%") + print(f" TP/FP/FN: {best['TP']}/{best['FP']}/{best['FN']}") + print(f" evaluations: {evals[0]} time: {time.time()-t0:.0f}s") + + # Also surface the highest-precision config seen (the ship-safe operating point). + if traj: + hp = max(traj, key=lambda r: (r["precision"], r["recall"])) + print("\n── highest-precision config seen (ship-safe) ──") + print(f" config : {hp['config']}") + print(f" P={hp['precision']*100:.2f}% R={hp['recall']*100:.2f}% F1={hp['f1']*100:.2f}%") + + if args.out: + Path(args.out).write_text(json.dumps( + {"best_by_f1": {"config": best_cfg, **best}, "n_evals": evals[0]}, indent=2)) + + +if __name__ == "__main__": + main() diff --git a/scripts/optimizer/reembed_gallery.py b/scripts/optimizer/reembed_gallery.py new file mode 100644 index 0000000..603d809 --- /dev/null +++ b/scripts/optimizer/reembed_gallery.py @@ -0,0 +1,94 @@ +#!/usr/bin/env python3 +""" +reembed_gallery.py — re-embed an existing gallery's actors with a different model. + +For the embedding-model bake-off: take a reference gallery (with all actor ids + +source_images) and produce a new gallery where every actor's embeddings are computed +by a DIFFERENT ArcFace/LVFace model from the SAME cached source images. All identity +keys (imdb/tmdb/jellyfin/name) are preserved, so membership/matching is unchanged — +only the embedding vectors (and hence the model's similarity space) differ. + +Source images live in `--images /_/NN.jpg` (the gallery build +cache). Actors are matched to their image dir by jellyfin_id first, then name. + +Usage: + python scripts/optimizer/reembed_gallery.py \ + --ref gallery_arcface_w600k_r50.h5 \ + --images images \ + --arcface models/arcface_r18.onnx \ + --out experiments/galleries/gallery_arcface_r18.h5 \ + [--build-dir build] +""" +from __future__ import annotations + +import argparse +import sys +from pathlib import Path + +REPO = Path(__file__).resolve().parent.parent.parent +sys.path.insert(0, str(REPO / "scripts")) +from sae_embed_loader import load_embedder # noqa: E402 +from sae_gallery import load_gallery_hdf5, save_gallery_hdf5 # noqa: E402 + + +def find_dir(images_root: Path, jellyfin_id: str, name: str) -> Path | None: + if jellyfin_id: + d = images_root / f"{jellyfin_id}_{name.replace(' ', '_')}" + if d.is_dir(): + return d + # jellyfin_id prefix match (name spelling may differ) + hits = list(images_root.glob(f"{jellyfin_id}_*")) + if hits: + return hits[0] + hits = list(images_root.glob(f"*_{name.replace(' ', '_')}")) + return hits[0] if hits else None + + +def main(): + p = argparse.ArgumentParser(description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter) + p.add_argument("--ref", required=True, help="reference gallery.h5 (ids + source imgs)") + p.add_argument("--images", required=True, help="image cache root") + p.add_argument("--arcface", required=True, help="model ONNX to re-embed with") + p.add_argument("--out", required=True) + p.add_argument("--build-dir", default=str(REPO / "build")) + p.add_argument("--models-dir", default=str(REPO / "models")) + args = p.parse_args() + + ref = load_gallery_hdf5(Path(args.ref)) + images_root = Path(args.images) + embedder = load_embedder(args.build_dir, args.models_dir, args.arcface) + + out_actors = [] + n_ok = n_nodir = n_noemb = 0 + total = len(ref["actors"]) + for i, a in enumerate(ref["actors"], 1): + d = find_dir(images_root, a.get("jellyfin_id", ""), a["name"]) + if d is None: + n_nodir += 1 + continue + embeddings = [] + for img in sorted(d.glob("*.jpg")): + res = embedder.embed(str(img)) + if res.ok: + embeddings.append(list(res.embedding)) + if not embeddings: + n_noemb += 1 + continue + out_actors.append({"imdb_id": a.get("imdb_id", ""), "tmdb_id": a.get("tmdb_id", ""), + "jellyfin_id": a.get("jellyfin_id", ""), "name": a["name"], + "embeddings": embeddings, + "source_images": [p.name for p in sorted(d.glob("*.jpg"))]}) + n_ok += 1 + if i % 200 == 0 or i == total: + print(f" [{i}/{total}] ok={n_ok} no_dir={n_nodir} no_emb={n_noemb}", + file=sys.stderr) + + save_gallery_hdf5({"actors": out_actors}, Path(args.out)) + n_emb = sum(len(a["embeddings"]) for a in out_actors) + print(f"[reembed] {Path(args.arcface).stem}: {n_ok}/{total} actors, {n_emb} embeddings " + f"→ {args.out}", file=sys.stderr) + + +if __name__ == "__main__": + main() diff --git a/scripts/optimizer/replay.py b/scripts/optimizer/replay.py new file mode 100644 index 0000000..0d6ad17 --- /dev/null +++ b/scripts/optimizer/replay.py @@ -0,0 +1,240 @@ +#!/usr/bin/env python3 +""" +replay.py — replay a dumped embedding HDF5 through the real KPN downstream nodes. + +Reads an embedding dump (scripts/optimizer/SCHEMA.md), feeds each frame as an +EmbeddedSceneFrame into a Python-assembled KPN network wiring the *real* C++ +face_tracker → identity_matcher → scene_tracker, and returns the same presence-window +JSON that scene_analyze's result_sink produces (minimal schema). No decode, no GPU +embedding — only the cheap downstream tail runs, so a sweep can vary Config knobs +freely. See [[kpn-python-replay-optimizer]]. + +CLI: + python scripts/optimizer/replay.py --dump film.h5 --gallery gallery.json \ + --out replayed.json [--prob-threshold 0.99] [--anneal 10] ... +""" +from __future__ import annotations + +import argparse +import json +import sys +import time +from pathlib import Path + +import h5py +import numpy as np + +REPO = Path(__file__).resolve().parent.parent.parent + + +def load_frames(dump_path: str, min_conf: float = 0.0): + """Yield EmbeddedSceneFrame dicts from the HDF5 dump, then a trailing EOF. + + `min_conf` drops detections below that detector confidence before they reach the + matcher — an UPWARD-only detector_conf sweep on already-dumped faces (the dump was + made at detector_conf=0.5, so 0.5 is the floor). Lets us test whether near-threshold + detections are real faces (raising min_conf hurts recall) or phantoms (it helps + precision at no recall cost).""" + with h5py.File(dump_path, "r") as f: + ts = f["frames/timestamp_sec"][:] + fidx = f["frames/frame_idx"][:] + cut = f["frames/is_cut"][:] + off = f["frames/face_offset"][:] + cnt = f["frames/face_count"][:] + emb = f["faces/embedding"][:] + bbox = f["faces/bbox"][:] + lmk = f["faces/landmarks"][:] + conf = f["faces/confidence"][:] + movie = f.attrs.get("movie", "") + fps = float(f.attrs.get("sample_fps", 1.0)) + + frames = [] + for i in range(len(ts)): + s, n = int(off[i]), int(cnt[i]) + keep = slice(s, s + n) + c = np.ascontiguousarray(conf[keep], dtype=np.float32) + if min_conf > 0.0 and n: + m = c >= min_conf + sel = np.where(m)[0] + frames.append({ + "timestamp_sec": float(ts[i]), "frame_idx": int(fidx[i]), + "is_cut": bool(cut[i]), "eof": False, + "bbox": np.ascontiguousarray(bbox[keep][sel], dtype=np.float32), + "landmarks": np.ascontiguousarray(lmk[keep][sel], dtype=np.float32), + "confidence": np.ascontiguousarray(c[sel], dtype=np.float32), + "embeddings": np.ascontiguousarray(emb[keep][sel], dtype=np.float32), + }) + else: + frames.append({ + "timestamp_sec": float(ts[i]), "frame_idx": int(fidx[i]), + "is_cut": bool(cut[i]), "eof": False, + "bbox": np.ascontiguousarray(bbox[keep], dtype=np.float32), + "landmarks": np.ascontiguousarray(lmk[keep], dtype=np.float32), + "confidence": c, + "embeddings": np.ascontiguousarray(emb[keep], dtype=np.float32), + }) + last_ts = float(ts[-1]) if len(ts) else 0.0 + frames.append({"timestamp_sec": last_ts, "eof": True}) + return frames, str(movie), fps + + +def replay(dump_path: str, gallery: str, cfg: dict, build_dir: str, stop: bool = True, + raw_out: str | None = None) -> dict: + """Run the dump through the real KPN chain; return minimal-schema presence JSON. + + cfg may include "detector_conf" to prune dumped detections below that confidence + (upward-only from the 0.5 dump floor) before matching. + + raw_out: if set, also write the raw per-frame annotations (timestamp, actor_idx, + name, bbox, similarity — one entry per input frame, before merging into windows) + as JSON lines to this path. Needed to draw bounding boxes on extracted frames; + the merged window schema returned by this function has no per-frame bbox.""" + sys.path.insert(0, build_dir) + import sae_kpn + + frames, movie, fps = load_frames(dump_path, min_conf=float(cfg.get("detector_conf", 0.0))) + + net = sae_kpn.Network() + sae_kpn._register_types(net) + + idx = [0] + eof = {"timestamp_sec": frames[-1]["timestamp_sec"], "eof": True} + + def source(): + # A no-input source node's run_loop calls this in a tight loop. Once frames + # are exhausted we must NOT hot-spin returning EOF — that pegs a core and + # floods the downstream channel with EOFs (livelock that wedged DE). Sleep + # briefly after the single real EOF so net.stop() can tear the thread down. + i = idx[0] + idx[0] += 1 + if i < len(frames): + return frames[i] + time.sleep(0.05) + return eof + + # Channel capacity must exceed the frame count so the fast source can't overflow + # a downstream FIFO before the serial reader drains it — PyNode DROPS on overflow, + # which would silently truncate the replay. Size to the whole film + slack. + # Every channel gets capacity ≥ the whole film so NOTHING can ever overflow-drop: + # the source can push all frames before any downstream node has drained, and a + # dropped frame silently corrupts the score. Memory is cheap (a few k pointers); + # correctness is not. Generous slack on top. + cap = len(frames) * 2 + 64 + sae_kpn.add_node_python(net, "replay", source, [], ["EmbeddedSceneFrame"], cap) + sae_kpn.add_face_tracker(net, "tracker", cfg, cap) + sae_kpn.add_identity_matcher(net, "matcher", gallery, cfg, cap) + sae_kpn.add_scene_tracker(net, "scene", cfg, cap) + net.connect("replay", 0, "tracker", 0) + net.connect("tracker", 0, "matcher", 0) + net.connect("matcher", 0, "scene", 0) + net.build() + net.start() + + # Read exactly one annotation per input frame. The source emits EOF as an ordinary + # value AFTER the last frame, but the concurrent pipeline lets that EOF OVERTAKE + # the last few real frames still flowing tracker→matcher→scene. Breaking on the + # first eof therefore dropped a random tail (~0.5–1%, race-dependent). Instead we + # keep reading past eof until we've collected all n_frames annotations (or hit a + # run of consecutive eofs meaning the pipeline is genuinely drained). + n_expected = len(frames) - 1 # excludes the trailing eof frame + annotations = [] + eof_streak = 0 + max_reads = n_expected * 2 + 32 + for _ in range(max_reads): + sa = net.read("scene", 0) + if sa.get("eof"): + eof_streak += 1 + # stragglers can still arrive after an eof; only stop once we've either + # got everything or seen several eofs in a row (truly drained). + if len(annotations) >= n_expected or eof_streak >= 8: + break + continue + eof_streak = 0 + annotations.append(sa) + if len(annotations) >= n_expected: + break + + if raw_out: + with open(raw_out, "w") as f: + for sa in annotations: + f.write(json.dumps(sa) + "\n") + + result = build_minimal(annotations, movie, fps, cfg) + if stop: + net.stop() + return result + + +def build_minimal(annotations, movie, fps, cfg) -> dict: + """Reproduce result_sink's minimal schema: per-actor annealed [start,end] windows. + + Mirrors ResultSinkFunc::build_actor_windows — merge each actor's detection + timestamps into windows, bridging gaps shorter than anneal_sec. + """ + anneal = float(cfg.get("anneal_sec", 10.0)) + info = {} # actor_idx -> identity fields + times = {} # actor_idx -> [timestamps] + for sa in annotations: + for a in sa["visible_actors"]: + if a["actor_idx"] < 0: + continue + info[a["actor_idx"]] = a + times.setdefault(a["actor_idx"], []).append(sa["timestamp_sec"]) + + actors = [] + for idx, ts in times.items(): + ts.sort() + scenes = [] + ws = we = ts[0] + for t in ts[1:]: + if t - we > anneal: + scenes.append([ws, we]) + ws = t + we = t + scenes.append([ws, we]) + a = info[idx] + actors.append({ + "name": a["name"], "imdb_id": a["imdb_id"], "tmdb_id": a["tmdb_id"], + "jellyfin_id": a["jellyfin_id"], "scenes": scenes, + }) + + return {"schema_version": 1, "movie": movie, "sample_fps": fps, + "anneal_sec": anneal, "actors": actors} + + +CFG_KEYS = ["detector_conf", "prob_threshold", "match_prior", "match_threshold", "match_ratio", + "match_ratio_ceil", "track_alpha", "track_min_iou", "track_max_embed_dist", + "track_max_frames_missing", "cut_revive_sim", "cut_inactive_max_frames", + "extinction_sec", "anneal_sec"] + + +def main(): + p = argparse.ArgumentParser(description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter) + p.add_argument("--dump", required=True, help="embedding HDF5 dump") + p.add_argument("--gallery", required=True) + p.add_argument("--out", required=True, help="output presence JSON") + p.add_argument("--raw-out", help="also write raw per-frame annotations (JSONL, with bboxes) here") + p.add_argument("--build-dir", default=str(REPO / "build")) + for k in CFG_KEYS: + p.add_argument(f"--{k.replace('_','-')}", type=float, default=None) + # per-film gallery expansion: promotes pose-varied views of confidently-identified + # actors into an in-memory annex, recovering ~+4 recall at no precision cost. + p.add_argument("--expand-gallery", action="store_true") + args = p.parse_args() + + cfg = {k: getattr(args, k) for k in CFG_KEYS if getattr(args, k) is not None} + if args.expand_gallery: + cfg["expand_gallery"] = True + # stop=True: PyNode::stop() sets stop_flag_ before joining, so the source + # thread's run_loop actually exits. stop=False skips that, leaving stop_flag_ + # false forever — the PyNode destructor's jthread.join() then blocks forever + # (verified via gdb: stuck in the source node's run_loop, not the GEMM path). + result = replay(args.dump, args.gallery, cfg, args.build_dir, stop=True, + raw_out=args.raw_out) + Path(args.out).write_text(json.dumps(result, indent=2)) + print(f"[replay] {len(result['actors'])} actors → {args.out}", file=sys.stderr) + + +if __name__ == "__main__": + main() diff --git a/scripts/optimizer/second_score.py b/scripts/optimizer/second_score.py new file mode 100644 index 0000000..57fd391 --- /dev/null +++ b/scripts/optimizer/second_score.py @@ -0,0 +1,197 @@ +#!/usr/bin/env python3 +""" +second_score.py — uniform per-second agreement with X-Ray. + +Unlike scene_score.py (which unions our detections over a whole X-Ray scene), this +samples EVERY SECOND of the film and asks: at second t, do we name the same actors +X-Ray says are on screen? + + GT(t) = the cast set of the X-Ray scene containing t (scenes.csv + people_in_scenes) + Pred(t) = actors whose presence window [start,end] covers t (the pipeline's output) + +Per second we count instances: + TPI = |Pred ∩ GT| true positive instances + FPI = |Pred − GT| false positive instances, split into: + FPI_misid — actor NOT in the film's cast at all (a real misID, weighted 10×) + FPI_incast — actor in the film but not this second (timing/boundary) + FN = |GT − Pred|, counting only gallery-known actors (fair recall — ~67% of X-Ray + cast have no reference embedding and can never be recognised) + agreement at t = Jaccard |Pred ∩ GT| / |Pred ∪ GT| — PARTIAL credit, so naming 2 + of 3 actors scores 2/3, not 0. Averaged over sampled seconds → the + "what fraction of the time do we agree with X-Ray" number. (Exact-set match is + reported separately as exact_match_rate; it is far harsher and dominated by + recall.) + +Objective (DE): per-second F1 computed with the WEIGHTED FPI, so naming someone who +isn't in the film hurts 10× more than a boundary slip. + +Reported: TPI, FPI (+split), FN, precision, recall, F1, and agreement_rate — the +fraction of sampled seconds where we exactly matched X-Ray. +""" +from __future__ import annotations + +import argparse +import csv +import json +import sys +from pathlib import Path + +REPO = Path(__file__).resolve().parent.parent.parent +sys.path.insert(0, str(REPO / "scripts" / "validation")) +from identity import keys_for # noqa: E402 + + +def load_second_timeline(xray_dir: str): + """Return (timeline, film_cast_keys, duration). + + timeline: dict second -> list of actor key-sets on screen per X-Ray. + Each second inside a scene [start,end) inherits that scene's cast set. + """ + d = Path(xray_dir) + id_to_name = {} + with open(d / "people.csv", newline="", encoding="utf-8") as f: + for r in csv.DictReader(f): + nm = (r.get("name_id") or "").strip() + if nm: + id_to_name[nm] = (r.get("person") or "").strip() + + film_cast = set() + for nm, name in id_to_name.items(): + film_cast |= keys_for(imdb_id=nm, name=name) + + spans = {} + with open(d / "scenes.csv", newline="", encoding="utf-8") as f: + for r in csv.DictReader(f): + sn = (r.get("scene") or "").strip() + try: + spans[sn] = (float(r["start"]) / 1000.0, float(r["end"]) / 1000.0) + except (KeyError, ValueError): + continue + + scene_cast: dict[str, list] = {} + with open(d / "people_in_scenes.csv", newline="", encoding="utf-8") as f: + for r in csv.DictReader(f): + sn = (r.get("scene") or "").strip() + nm = (r.get("name_id") or "").strip() + if sn in spans and nm: + scene_cast.setdefault(sn, []).append( + frozenset(keys_for(imdb_id=nm, name=id_to_name.get(nm)))) + + timeline: dict[int, list] = {} + duration = 0.0 + for sn, (t0, t1) in spans.items(): + duration = max(duration, t1) + cast = scene_cast.get(sn, []) + for t in range(int(t0), int(t1)): + timeline[t] = cast + return timeline, film_cast, duration + + +def load_pred_intervals(pred_json: dict): + """[(keyset, [(t0,t1),...]), ...] for each actor the pipeline named.""" + out = [] + for a in pred_json.get("actors", []): + keys = frozenset(keys_for(imdb_id=a.get("imdb_id"), tmdb_id=a.get("tmdb_id"), + jellyfin_id=a.get("jellyfin_id"), name=a.get("name"))) + out.append((keys, [(float(t0), float(t1)) for t0, t1 in a.get("scenes", [])])) + return out + + +def _match(P, G): + """Greedy 1:1 match by key intersection; returns (n_matched, matched_G_mask).""" + used = [False] * len(G) + n = 0 + for pa in P: + for j, ga in enumerate(G): + if not used[j] and (pa & ga): + used[j] = True + n += 1 + break + return n, used + + +def score_seconds(pred_json: dict, xray_dir: str, gallery_keys: set | None = None, + misid_weight: float = 10.0): + timeline, film_cast, duration = load_second_timeline(xray_dir) + pred = load_pred_intervals(pred_json) + + TPI = FPI = FN = 0 + FPI_misid = FPI_incast = 0 + FPI_w = 0.0 + jaccard_sum = 0.0 # partial-credit agreement, summed over seconds + exact = 0 + n_sec = 0 + + for t in sorted(timeline): + G = [set(a) for a in timeline[t]] + P = [set(k) for k, wins in pred if any(w0 <= t <= w1 for w0, w1 in wins)] + # fair recall: only GT actors we could possibly recognise + if gallery_keys is not None: + G = [g for g in G if g & gallery_keys] + + tp, matched = _match(P, G) + # classify each unmatched prediction + fpi_w = 0.0 + n_fp = 0 + for pa in P: + if any(pa & ga for ga in G): + continue + n_fp += 1 + if pa & film_cast: + FPI_incast += 1; fpi_w += 1.0 + else: + FPI_misid += 1; fpi_w += misid_weight + fn = len(G) - tp + + TPI += tp; FPI += n_fp; FN += fn; FPI_w += fpi_w + # partial-credit agreement: |∩| / |∪| at this second + union = tp + n_fp + fn + if union: + jaccard_sum += tp / union + else: + jaccard_sum += 1.0 # both empty = agreement (nobody on screen) + if n_fp == 0 and fn == 0: + exact += 1 + n_sec += 1 + + prec = TPI / (TPI + FPI_w) if TPI + FPI_w else 0.0 # weighted (misID hurts 10×) + prec_raw = TPI / (TPI + FPI) if TPI + FPI else 0.0 + rec = TPI / (TPI + FN) if TPI + FN else 0.0 + f1 = 2 * prec * rec / (prec + rec) if prec + rec else 0.0 + return {"TPI": TPI, "FPI": FPI, "FPI_misid": FPI_misid, "FPI_incast": FPI_incast, + "FN": FN, "precision": prec, "precision_raw": prec_raw, "recall": rec, + "f1": f1, + # partial-credit: mean per-second Jaccard = "% of actors we agree on, over time" + "agreement_rate": jaccard_sum / n_sec if n_sec else 0.0, + "exact_match_rate": exact / n_sec if n_sec else 0.0, + "n_seconds": n_sec, "duration_sec": duration} + + +def main(): + p = argparse.ArgumentParser(description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter) + p.add_argument("--pred", required=True) + p.add_argument("--xray", required=True) + p.add_argument("--gallery") + args = p.parse_args() + gk = None + if args.gallery: + sys.path.insert(0, str(REPO / "scripts" / "validation")) + from sample_eval import load_gallery_keys + gk = load_gallery_keys(args.gallery) + m = score_seconds(json.loads(Path(args.pred).read_text()), args.xray, gk) + print(f"seconds sampled : {m['n_seconds']} (film {m['duration_sec']:.0f}s)") + print(f"TPI/FPI/FN : {m['TPI']}/{m['FPI']}/{m['FN']}") + print(f" FPI misID : {m['FPI_misid']} (actor not in film — weighted 10x)") + print(f" FPI in-cast : {m['FPI_incast']}") + print(f"precision (w) : {m['precision']*100:.1f}% raw {m['precision_raw']*100:.1f}%") + print(f"recall : {m['recall']*100:.1f}%") + print(f"F1 (weighted) : {m['f1']*100:.1f}%") + print(f"AGREEMENT : {m['agreement_rate']*100:.1f}% (mean per-second % of actors " + f"we agree on with X-Ray)") + print(f" exact-set match: {m['exact_match_rate']*100:.1f}% of seconds (harsher, " + f"all-or-nothing)") + + +if __name__ == "__main__": + main() diff --git a/scripts/optimizer/test_sae_kpn.py b/scripts/optimizer/test_sae_kpn.py new file mode 100644 index 0000000..ca33455 --- /dev/null +++ b/scripts/optimizer/test_sae_kpn.py @@ -0,0 +1,75 @@ +#!/usr/bin/env python3 +""" +Smoke test for the sae_kpn module: assemble the real downstream pipeline nodes +(face_tracker → identity_matcher → scene_tracker) in a Python-driven KPN network, +fed by a no-input Python source node, and verify SceneAnnotations flow out. + +Proves the KPN-native replay path works without any numpy port of node logic. +Run: python scripts/optimizer/test_sae_kpn.py [gallery.json] [build_dir] +""" +import sys +import queue +import numpy as np +from pathlib import Path + +REPO = Path(__file__).resolve().parent.parent.parent +GAL = sys.argv[1] if len(sys.argv) > 1 else str(REPO / "gallery_arcface_w600k_r50.json") +BUILD = sys.argv[2] if len(sys.argv) > 2 else str(REPO / "build") +sys.path.insert(0, BUILD) +import sae_kpn # noqa: E402 + + +def make_frame(t, n): + e = np.random.randn(n, 512).astype(np.float32) + e /= np.linalg.norm(e, axis=1, keepdims=True) + return {"timestamp_sec": t, "eof": False, + "bbox": np.tile(np.array([10, 10, 50, 50], np.float32), (n, 1)), + "landmarks": np.tile(np.arange(10, dtype=np.float32), (n, 1)), + "confidence": np.full((n,), 0.9, np.float32), "embeddings": e} + + +def main(): + net = sae_kpn.Network() + sae_kpn._register_types(net) + cfg = {"prob_threshold": 0.99, "anneal_sec": 10.0, "extinction_sec": 5.0} + + frames = [make_frame(float(t), 1) for t in range(3)] + frames.append({"timestamp_sec": 3.0, "eof": True}) + idx = [0] + eof_frame = {"timestamp_sec": 3.0, "eof": True} + + def source(): + # Emit each frame once, then keep returning EOF (never block) so the node + # thread stays responsive to stop() after the sink has seen EOF. + i = idx[0] + idx[0] += 1 + return frames[i] if i < len(frames) else eof_frame + + sae_kpn.add_node_python(net, "replay", source, [], ["EmbeddedSceneFrame"], 8) + sae_kpn.add_face_tracker(net, "tracker", cfg, 16) + sae_kpn.add_identity_matcher(net, "matcher", GAL, cfg, 16) + sae_kpn.add_scene_tracker(net, "scene", cfg, 16) + net.connect("replay", 0, "tracker", 0) + net.connect("tracker", 0, "matcher", 0) + net.connect("matcher", 0, "scene", 0) + net.build() + net.start() + + got = [] + for _ in range(4): + sa = net.read("scene", 0) + got.append(sa) + if sa.get("eof"): + break + net.stop() + + non_eof = [g for g in got if not g.get("eof")] + assert len(non_eof) == 3, f"expected 3 annotations, got {len(non_eof)}" + assert got[-1].get("eof"), "expected trailing EOF" + assert [g["timestamp_sec"] for g in non_eof] == [0.0, 1.0, 2.0], "timestamps wrong" + assert all("visible_actors" in g for g in non_eof), "missing visible_actors" + print(f"OK: {len(non_eof)} annotations through the real KPN chain, EOF received") + + +if __name__ == "__main__": + main() diff --git a/scripts/run_from_jellyfin.py b/scripts/run_from_jellyfin.py index 9cb4ba3..86321eb 100644 --- a/scripts/run_from_jellyfin.py +++ b/scripts/run_from_jellyfin.py @@ -11,18 +11,24 @@ Usage: --jellyfin-url http://jellyfin.local:8096 \\ --api-key YOUR_API_KEY \\ --title "The Matrix" \\ - --gallery whole_gallery.json \\ + --gallery whole_gallery.h5 \\ -- --fps 5 --verbosity 2 Anything after "--" is passed through unchanged to scene_analyze. +Add --preview to open the live OpenCV display window: it resolves the media path +from Jellyfin exactly as normal, then launches build/scene_preview instead of the +headless binary (implies --no-push). Works with --worker or a single title: + + python scripts/run_from_jellyfin.py ... --worker --preview -- --fps 5 + Worker mode (--worker) polls the JRay plugin's Tasks/Pending endpoint for a random batch of items with no truth data yet, processing each in turn: python scripts/run_from_jellyfin.py \\ --jellyfin-url http://jellyfin.local:8096 \\ --api-key YOUR_API_KEY \\ - --gallery whole_gallery.json \\ + --gallery whole_gallery.h5 \\ --worker \\ -- --fps 5 """ @@ -40,6 +46,7 @@ import requests sys.path.insert(0, str(Path(__file__).resolve().parent)) import sae_env # noqa: F401 — loads .env into os.environ on import +from sae_gallery import load_gallery_hdf5, save_gallery_hdf5 from sae_jellyfin import ( jf_get, find_item_id, fetch_cast_person_ids, actor_jellyfin_id, fetch_episode_info, ) @@ -171,7 +178,7 @@ def process_item(args, item_id: str, extra: list[str]) -> None: gallery_path = args.gallery filtered_file = None if not args.no_filter: - gallery = json.loads(Path(args.gallery).read_text()) + gallery = load_gallery_hdf5(Path(args.gallery)) actors = episode_cast_actors(args, item_id, gallery) if actors is None: cast_ids = fetch_cast_person_ids(args.jellyfin_url, args.api_key, item_id) @@ -180,9 +187,9 @@ def process_item(args, item_id: str, extra: list[str]) -> None: print(f"Filtered gallery to {len(actors)}/{len(gallery.get('actors', []))} " f"actor(s) credited in {name!r}", file=sys.stderr) filtered_file = tempfile.NamedTemporaryFile( - mode="w", suffix=".json", prefix="sae_gallery_", delete=False) - json.dump({"actors": actors}, filtered_file) + suffix=".h5", prefix="sae_gallery_", delete=False) filtered_file.close() + save_gallery_hdf5({"actors": actors}, Path(filtered_file.name)) gallery_path = filtered_file.name try: @@ -281,7 +288,7 @@ def main(): parser.add_argument("--item-types", default="Movie,Episode", help="Item types to search when using --title (default: Movie,Episode)") parser.add_argument("--gallery", required=True, - help="Global gallery.json built by make_jellyfin_gallery.py") + help="Global gallery.h5 built by make_jellyfin_gallery.py") parser.add_argument("--no-filter", action="store_true", help="Skip per-title cast filtering and pass --gallery through as-is") parser.add_argument("--episode-cast", choices=("tmdb", "series"), default="tmdb", @@ -297,6 +304,12 @@ def main(): "ignored with --worker, which always uses .json)") parser.add_argument("--bin", default="build/scene_analyze", help="Path to scene_analyze binary (default: build/scene_analyze)") + parser.add_argument("--preview", action="store_true", + help="Launch the scene_preview binary (live OpenCV display window) " + "instead of headless scene_analyze. Implies --no-push. Use " + "--preview-bin to override its path. Press q/Esc to close.") + parser.add_argument("--preview-bin", default="build/scene_preview", + help="Path to scene_preview binary (default: build/scene_preview)") parser.add_argument("--dry-run", action="store_true", help="Resolve and print the scene_analyze command without running it") parser.add_argument("--no-push", action="store_true", @@ -306,6 +319,12 @@ def main(): if extra and extra[0] == "--": extra = extra[1:] + # --preview swaps in the display binary and disables pushing truth (a preview + # run is interactive/debug, not a truth-producing analysis). + if args.preview: + args.bin = args.preview_bin + args.no_push = True + if args.worker: if args.output: sys.exit("--output is incompatible with --worker (each item needs its own file)") diff --git a/scripts/sae_gallery.py b/scripts/sae_gallery.py index fa9efe1..ac97d5f 100644 --- a/scripts/sae_gallery.py +++ b/scripts/sae_gallery.py @@ -2,7 +2,13 @@ Consolidates the three near-identical download loops (make_gallery.download_images, make_jellyfin_gallery.download_urls + download_person_images) and the duplicated -"write gallery.json + .missing_images.json" tail from both builders. +"write gallery.h5 + .missing_images.json" tail from both builders. + +Galleries are written directly as HDF5 — never JSON. Same layout the C++ side +reads/writes (src/gallery/gallery_store.cpp): flat [N,512] embeddings + per-actor +offset/count, parallel imdb_id/tmdb_id/jellyfin_id/name string arrays, and a +per-embedding-row source_images array. calibration is left absent (calib_hash=0); +the C++ identity_matcher fits and writes it back into the file on first use. """ import io @@ -10,6 +16,8 @@ import json import sys from pathlib import Path +import h5py +import numpy as np import requests from PIL import Image, UnidentifiedImageError @@ -93,11 +101,81 @@ def download_images(urls: list[str], dest_dir: Path, n: int, return paths -def save_gallery(gallery: dict, missing: list[dict], output: Path) -> None: - """Write gallery.json and, if any actors lack images, a .missing_images.json sidecar.""" +def save_gallery_hdf5(gallery: dict, output: Path) -> None: + """Write a gallery dict ({"actors": [...]}) directly as HDF5 — same schema + src/gallery/gallery_store.cpp reads/writes. No calibration group; the + C++ identity_matcher computes and writes it back into this file on first + use against an unseen set of embeddings.""" + actors = gallery["actors"] + embs, offsets, counts = [], [], [] + imdb, tmdb, jf, name, src_images = [], [], [], [], [] + row = 0 + for a in actors: + e = a.get("embeddings", []) + offsets.append(row) + counts.append(len(e)) + row += len(e) + embs.extend(e) + si = a.get("source_images", []) + for i in range(len(e)): + src_images.append(si[i] if i < len(si) else "") + imdb.append(a.get("imdb_id", "") or "") + tmdb.append(str(a.get("tmdb_id", "") or "")) + jf.append(a.get("jellyfin_id", a.get("jellyfin_person_id", "")) or "") + name.append(a.get("name", "") or "") + + emb_arr = np.asarray(embs, dtype=np.float32) if embs else np.zeros((0, 512), np.float32) + if emb_arr.ndim == 1: + emb_arr = emb_arr.reshape(0, 512) + output.parent.mkdir(parents=True, exist_ok=True) - output.write_text(json.dumps(gallery, indent=2) + "\n") - print(f"Saved: {output}", file=sys.stderr) + str_t = h5py.string_dtype("utf-8") + with h5py.File(output, "w") as f: + f.create_dataset("embeddings", data=emb_arr) + f.create_dataset("offset", data=np.asarray(offsets, np.int64)) + f.create_dataset("count", data=np.asarray(counts, np.int32)) + f.create_dataset("imdb_id", data=np.asarray(imdb, dtype=object), dtype=str_t) + f.create_dataset("tmdb_id", data=np.asarray(tmdb, dtype=object), dtype=str_t) + f.create_dataset("jellyfin_id", data=np.asarray(jf, dtype=object), dtype=str_t) + f.create_dataset("name", data=np.asarray(name, dtype=object), dtype=str_t) + f.create_dataset("source_images", data=np.asarray(src_images, dtype=object), dtype=str_t) + print(f"Saved: {output} ({len(actors)} actors, {emb_arr.shape[0]} embeddings)", + file=sys.stderr) + + +def load_gallery_hdf5(path: Path) -> dict: + """Read an HDF5 gallery back into the same {"actors": [...]} dict shape the + builders work with in memory (for --merge). Mirrors save_gallery_hdf5.""" + with h5py.File(path, "r") as f: + emb = f["embeddings"][:] + offset = f["offset"][:] + count = f["count"][:] + imdb = [s.decode() if isinstance(s, bytes) else s for s in f["imdb_id"][:]] + tmdb = [s.decode() if isinstance(s, bytes) else s for s in f["tmdb_id"][:]] + jf = [s.decode() if isinstance(s, bytes) else s for s in f["jellyfin_id"][:]] + name = [s.decode() if isinstance(s, bytes) else s for s in f["name"][:]] + src_images = None + if "source_images" in f: + src_images = [s.decode() if isinstance(s, bytes) else s + for s in f["source_images"][:]] + + actors = [] + for a in range(len(offset)): + s, n = int(offset[a]), int(count[a]) + actor = {"imdb_id": imdb[a], "tmdb_id": tmdb[a], "jellyfin_id": jf[a], + "name": name[a], "embeddings": [emb[s + i].tolist() for i in range(n)]} + if src_images is not None: + actor["source_images"] = [src_images[s + i] for i in range(n)] + actors.append(actor) + return {"actors": actors} + + +def save_gallery(gallery: dict, missing: list[dict], output: Path) -> None: + """Write the gallery as HDF5 (forcing a .h5 extension) and, if any actors + lack images, a .missing_images.json sidecar.""" + if output.suffix not in (".h5", ".hdf5"): + output = output.with_suffix(".h5") + save_gallery_hdf5(gallery, output) if missing: missing_path = output.with_name(output.stem + ".missing_images.json") diff --git a/scripts/validation/README.md b/scripts/validation/README.md new file mode 100644 index 0000000..0f165bc --- /dev/null +++ b/scripts/validation/README.md @@ -0,0 +1,87 @@ +# scripts/validation — per-scene actor-presence eval + +Validates the pipeline's per-scene "who's on screen" output against external +ground truth, offline. Annealing (`anneal_sec`) means an actor's presence is only +defined *after* the whole file is merged into `[start,end]` windows, so we cannot +score live: process → write the pipeline JSON → **sample timepoints** → compare +predicted vs ground-truth presence sets → micro-sum TP/FP/FN → precision/recall/F1. + +## Ground-truth sources + +| Source | Semantics | Fair to a face pipeline? | What it measures | +| ------ | --------- | ------------------------ | ---------------- | +| **MovieNet-PS** | on-screen **face** presence per shot | yes — like-for-like | recognition accuracy | +| **Amazon X-Ray** (Zenodo) | **cast-in-scene** (incl. off-camera / non-speaking) | no — penalizes by design | coverage ceiling; recall gap = actors we structurally can't see | + +- MovieNet is the honest recognition number. +- X-Ray is an upper bound: its recall gap tells you how much presence is off-camera + cast a face detector can never reach — not a pipeline error. + +X-Ray dataset: Zenodo DOI `10.5281/zenodo.17659734` (CC-BY-4.0). Per movie it ships +`people.csv`, `scenes.csv`, `people_in_scenes.csv`. + +## Usage + +```bash +# against Amazon X-Ray CSVs for one title +python scripts/validation/sample_eval.py \ + --pred "Scene in a Mall.json" \ + --xray /data/xray/<movie_dir> \ + --gallery gallery_arcface_w600k_r50.json \ + --step 1.0 + +# against MovieNet-PS for one title +python scripts/validation/sample_eval.py \ + --pred out.json \ + --movienet /data/movienet --split Train_app10 --title tt0032138 \ + --gallery gallery_arcface_w600k_r50.json +``` + +### Sampling modes +- `--step S` regular grid every S s (default 1.0) — time-weighted headline number. +- `--random N` N uniform-random timepoints (for confidence intervals). +- `--scene-anchored` one timepoint per GT scene midpoint — the literal X-Ray + "did I get this scene's cast right?" question; neutralizes long-scene bias. + +Ground truth is compared **raw** (annealing is *not* applied to GT). + +## Matching & masking + +Identity is provider-agnostic (`identity.py`): each actor is the *set* of every key +we can derive — `imdb:nm…`, `tmdb:…`, `jf:…`, `name:<normalized>`. Predicted and GT +actors match iff their key-sets intersect, so an output carrying only tmdb/jellyfin +ids still joins X-Ray's `nm` ids via the normalized-name fallback. + +Scoring is **masked to `gallery ∩ GT`**: a GT actor absent from the gallery is +ignored (not an FN), so we measure pipeline accuracy, not gallery coverage. Without +`--gallery` the mask falls back to `GT ∩ pred` keys. `--no-mask` disables it. + +### Exact id join via the tmdb→imdb crosswalk (recommended) + +The gallery/pipeline output key actors by **TMDB** id (no `nm…`), while X-Ray and +MovieNet key on **IMDb**. They only overlap on the fuzzy `name:` key by default. +Build a cached `tmdb→imdb` table once and pass it with `--crosswalk` to turn the +name join into an exact id join: + +```bash +# one-time: resolve every gallery tmdb id via TMDB /person/{id}/external_ids +python scripts/validation/tmdb_imdb_map.py \ + --gallery gallery_arcface_w600k_r50.json \ + --out scripts/validation/tmdb_imdb.json # TMDB_API_KEY from env/.env + +# then score with exact ids +python scripts/validation/sample_eval.py --pred out.json --xray <dir> \ + --gallery gallery_arcface_w600k_r50.json \ + --crosswalk scripts/validation/tmdb_imdb.json +``` + +The table caches nulls (tmdb ids TMDB has no IMDb id for) and checkpoints, so a +re-run only resolves new ids. TMDB is authoritative for this crosswalk — there is +no clean free bulk `tmdb_person ↔ nm` file, so we query the API once and cache. + +## Files +- `sample_eval.py` — CLI scorer. +- `ground_truth.py` — `XRayGroundTruth`, `MovieNetGroundTruth` loaders. +- `identity.py` — provider-agnostic match keys. +- `tmdb_imdb_map.py` — build/consult the cached `tmdb→imdb` crosswalk. +- `test_sample_eval.py` — self-contained tests (`python scripts/validation/test_sample_eval.py`). diff --git a/scripts/validation/ground_truth.py b/scripts/validation/ground_truth.py new file mode 100644 index 0000000..174f014 --- /dev/null +++ b/scripts/validation/ground_truth.py @@ -0,0 +1,221 @@ +#!/usr/bin/env python3 +""" +ground_truth.py — pluggable ground-truth loaders for per-scene presence eval. + +A ground truth is a timeline of "who is present when", exposed as: + + GroundTruth.present_at(t: float) -> set[str] # match-keys present at time t + GroundTruth.scene_windows() -> list[(t0, t1)] # scene spans (for --scene-anchored) + GroundTruth.all_keys() -> set[str] # every actor the GT knows (for masking) + +"Match-keys" are provider-agnostic identity tokens (see identity.py): an actor is +represented by *all* the keys we can derive (nm-id, tmdb-id, jellyfin-id, normalized +name), so predicted and GT sets intersect if they agree on *any* shared id space. +This matters because the pipeline output may carry only tmdb/jellyfin ids while +X-Ray/MovieNet key on IMDb nm-ids — see [[per-scene-presence-eval-design]]. + +Two sources implemented: + * XRayGroundTruth — Zenodo scene-level Amazon X-Ray CSVs (cast-in-scene). + * MovieNetGroundTruth — MovieNet-PS per-shot face annotations (on-screen faces). +""" + +from __future__ import annotations + +import csv +import sys +from bisect import bisect_right +from pathlib import Path + +from identity import keys_for + + +class GroundTruth: + """Base: a set of actors, each with presence intervals [(t0,t1), ...] in seconds.""" + + def __init__(self) -> None: + # actor_id (any stable local id) -> {"keys": set[str], "intervals": [(t0,t1)]} + self._actors: dict[str, dict] = {} + self._scene_spans: list[tuple[float, float]] = [] + + # -- construction helpers ------------------------------------------------ + def _add_interval(self, actor_id: str, keys: set[str], t0: float, t1: float) -> None: + a = self._actors.setdefault(actor_id, {"keys": set(), "intervals": []}) + a["keys"] |= keys + a["intervals"].append((float(t0), float(t1))) + + def _finalize(self) -> None: + """Sort intervals and precompute a flat sorted start-array per actor.""" + for a in self._actors.values(): + a["intervals"].sort() + a["_starts"] = [iv[0] for iv in a["intervals"]] + self._scene_spans.sort() + + # -- query API ----------------------------------------------------------- + def present_at(self, t: float) -> set[frozenset[str]]: + """Set of actors present at t; each actor is its (frozen) key-set.""" + out: set[frozenset[str]] = set() + for a in self._actors.values(): + ivs = a["intervals"] + i = bisect_right(a["_starts"], t) # first interval starting after t + # walk back over intervals that started at/before t + j = i - 1 + while j >= 0: + t0, t1 = ivs[j] + if t1 >= t: + out.add(frozenset(a["keys"])) + break + # intervals sorted by start; an earlier one could still cover t, + # but since we only need membership, keep scanning a bounded window. + j -= 1 + if i - j > 8: # bound: overlapping intervals per actor are rare + break + return out + + def scene_windows(self) -> list[tuple[float, float]]: + return self._scene_spans + + def all_keys(self) -> set[str]: + out: set[str] = set() + for a in self._actors.values(): + out |= a["keys"] + return out + + def summary(self) -> str: + n_iv = sum(len(a["intervals"]) for a in self._actors.values()) + return (f"{len(self._actors)} actors, {n_iv} intervals, " + f"{len(self._scene_spans)} scenes") + + +# ── Amazon X-Ray (Zenodo) ───────────────────────────────────────────────────── + +class XRayGroundTruth(GroundTruth): + """ + Load one movie's X-Ray CSVs (Zenodo DOI 10.5281/zenodo.17659734). + Real schema (columns are milliseconds): + people.csv name_id (nm...), person, character + scenes.csv scene, start, end (ms) + people_in_scenes.csv scene, start, end, name_id, timestamp (ms) + + Presence = whole scene span for every character listed in that scene. + Semantics: cast-in-scene (incl. off-camera) — a recall ceiling, not accuracy. + Columns are resolved case-insensitively so minor variants still load. + """ + + def __init__(self, movie_dir: str | Path) -> None: + super().__init__() + d = Path(movie_dir) + people = _read_csv(d / "people.csv") + scenes = _read_csv(d / "scenes.csv") + pis = _read_csv(d / "people_in_scenes.csv") + + # nm-id -> actor name (for building match keys) + nm_col_p = _find_col(people, "name_id", "nm", "imdb") + name_col = _find_col(people, "person", "actor") # actor name lives in "person" + id_to_name: dict[str, str] = {} + for row in people: + nm = (row.get(nm_col_p) or "").strip() + if nm: + id_to_name[nm] = (row.get(name_col) or "").strip() + + # scene number -> (t0_sec, t1_sec) + scene_col_s = _find_col(scenes, "scene") + start_col = _find_col(scenes, "start") + end_col = _find_col(scenes, "end") + span: dict[str, tuple[float, float]] = {} + for row in scenes: + sn = (row.get(scene_col_s) or "").strip() + t0 = _ms_to_sec(row.get(start_col)) + t1 = _ms_to_sec(row.get(end_col)) + if sn and t0 is not None and t1 is not None: + span[sn] = (t0, t1) + self._scene_spans.append((t0, t1)) + + # scene number -> [nm ids present] + scene_col_pis = _find_col(pis, "scene") + nm_col_pis = _find_col(pis, "name_id", "nm", "imdb") + for row in pis: + sn = (row.get(scene_col_pis) or "").strip() + nm = (row.get(nm_col_pis) or "").strip() + if sn not in span or not nm: + continue + t0, t1 = span[sn] + self._add_interval(nm, keys_for(imdb_id=nm, name=id_to_name.get(nm)), t0, t1) + + self._finalize() + + +# ── MovieNet-PS ──────────────────────────────────────────────────────────────── + +class MovieNetGroundTruth(GroundTruth): + """ + Build presence from MovieNet-PS per-shot face annotations for a single title. + + Input: the flat annotation list produced by movienet_prep.load_movienet_annotations + filtered to one movie (tt-id), plus a shot->time map. Because MovieNet frames are + named tt.../shot_XXXX_img_Y.jpg with no absolute timestamp, presence is expressed + in *shot index* units unless a fps/shot-duration map is supplied. For the sampler + we therefore sample at shot granularity (one timepoint per annotated shot). + + Semantics: on-screen face presence per shot — like-for-like fair benchmark. + """ + + def __init__(self, annotations: list[dict], id_to_name: dict[str, str] | None = None, + shot_seconds: float = 1.0) -> None: + super().__init__() + id_to_name = id_to_name or {} + # group by shot index; each annotated shot becomes a unit interval on a + # synthetic timeline (shot_index * shot_seconds). + shots: dict[int, set[str]] = {} + for ann in annotations: + shot = _shot_index(ann["img_path"]) + if shot is None: + continue + shots.setdefault(shot, set()).add(ann["imdb_id"]) + + for shot, nm_ids in shots.items(): + t0 = shot * shot_seconds + t1 = t0 + shot_seconds + self._scene_spans.append((t0, t1)) + for nm in nm_ids: + self._add_interval(nm, keys_for(imdb_id=nm, name=id_to_name.get(nm)), t0, t1) + + self._finalize() + + +# ── small parsing helpers ────────────────────────────────────────────────────── + +def _read_csv(path: Path) -> list[dict]: + if not path.exists(): + raise FileNotFoundError(f"expected X-Ray CSV not found: {path}") + with open(path, newline="", encoding="utf-8") as f: + return list(csv.DictReader(f)) + + +def _find_col(rows: list[dict], *keywords: str) -> str: + """Return the first column whose lowercased name contains all keywords of any + single keyword group. We try each keyword in order and accept the first hit.""" + if not rows: + raise ValueError("empty CSV — cannot resolve columns") + cols = list(rows[0].keys()) + low = {c: c.lower() for c in cols} + for kw in keywords: + for c in cols: + if kw in low[c]: + return c + raise KeyError(f"no column matching {keywords} in {cols}") + + +def _ms_to_sec(v) -> float | None: + if v is None or str(v).strip() == "": + return None + try: + return float(v) / 1000.0 + except ValueError: + return None + + +def _shot_index(img_path: str) -> int | None: + # tt0032138/shot_0003_img_1.jpg -> 3 + import re + m = re.search(r"shot_(\d+)", img_path) + return int(m.group(1)) if m else None diff --git a/scripts/validation/identity.py b/scripts/validation/identity.py new file mode 100644 index 0000000..daad68c --- /dev/null +++ b/scripts/validation/identity.py @@ -0,0 +1,54 @@ +#!/usr/bin/env python3 +""" +identity.py — provider-agnostic match keys. + +The pipeline output and the ground truth may not share one id space: an output +actor can carry only tmdb/jellyfin ids while X-Ray/MovieNet key on IMDb nm-ids. +We represent each actor by the *set* of every key we can derive, and treat two +actors as the same iff their key sets intersect. Namespacing each key by its +provider prevents cross-provider collisions (e.g. an nm-number equalling a +tmdb-number). +""" + +from __future__ import annotations + +import re +import unicodedata + + +def norm_name(name: str | None) -> str | None: + """Lowercased, accent-stripped, punctuation-free name for fuzzy fallback match.""" + if not name: + return None + s = unicodedata.normalize("NFKD", name) + s = "".join(c for c in s if not unicodedata.combining(c)) + s = re.sub(r"[^a-z0-9 ]+", "", s.lower()).strip() + s = re.sub(r"\s+", " ", s) + return s or None + + +def keys_for(imdb_id: str | None = None, + tmdb_id: str | None = None, + jellyfin_id: str | None = None, + name: str | None = None, + crosswalk=None) -> set[str]: + """All identity tokens for one actor. Empty strings are ignored. + + If `crosswalk` (a CrosswalkTable) is given and no imdb_id is present, resolve + tmdb_id → imdb_id through it so a tmdb-only actor still gets an exact `imdb:` + key — turning the fuzzy name join into an exact id join. See tmdb_imdb_map.py. + """ + keys: set[str] = set() + imdb = imdb_id.strip() if (imdb_id and imdb_id.strip()) else None + if not imdb and crosswalk is not None and tmdb_id: + imdb = crosswalk.imdb_for(tmdb_id) + if imdb: + keys.add(f"imdb:{imdb}") + if tmdb_id and str(tmdb_id).strip(): + keys.add(f"tmdb:{str(tmdb_id).strip()}") + if jellyfin_id and jellyfin_id.strip(): + keys.add(f"jf:{jellyfin_id.strip()}") + nn = norm_name(name) + if nn: + keys.add(f"name:{nn}") + return keys diff --git a/scripts/validation/sample_eval.py b/scripts/validation/sample_eval.py new file mode 100644 index 0000000..96ced28 --- /dev/null +++ b/scripts/validation/sample_eval.py @@ -0,0 +1,313 @@ +#!/usr/bin/env python3 +""" +sample_eval.py — offline per-scene presence eval by timepoint sampling. + +Annealing (anneal_sec) means an actor is "present" only after the whole file is +merged into [start,end] windows, so we cannot score live: we process → write the +pipeline JSON → sample timepoints → compare predicted vs ground-truth presence +sets → micro-sum TP/FP/FN → precision / recall / F1. +See [[per-scene-presence-eval-design]]. + +Usage: + # against Amazon X-Ray CSVs (Zenodo) + python scripts/validation/sample_eval.py \ + --pred "Scene in a Mall.json" \ + --xray /data/xray/tt0384766 \ + --step 1.0 + + # against MovieNet-PS (needs the .mat split + a title tt-id) + python scripts/validation/sample_eval.py \ + --pred out.json \ + --movienet /data/movienet --split Train_app10 --title tt0032138 + +Sampling: + --step S regular grid every S seconds (default 1.0) — time-weighted headline + --random N N uniform-random timepoints instead of a grid (for CIs) + --scene-anchored one timepoint at each GT scene midpoint (X-Ray "per-scene" question) + +Masking: scoring is restricted to actors present in BOTH the pipeline gallery +(--gallery) AND the ground truth. A GT actor absent from the gallery is ignored +(not counted as a miss) so we measure pipeline accuracy, not gallery coverage. +Pass --no-mask to disable. +""" + +from __future__ import annotations + +import argparse +import json +import random +import sys +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parent)) +from identity import keys_for # noqa: E402 +from ground_truth import XRayGroundTruth, MovieNetGroundTruth # noqa: E402 +from tmdb_imdb_map import CrosswalkTable # noqa: E402 + + +# ── pipeline output → presence timeline ──────────────────────────────────────── + +class Prediction: + """Pipeline output (minimal/standard schema) as per-actor presence windows.""" + + def __init__(self, path: str | Path, crosswalk=None) -> None: + with open(path) as f: + data = json.load(f) + self.movie = data.get("movie", "") + self.anneal_sec = data.get("anneal_sec") + self.actors: list[dict] = [] + self._max_t = 0.0 + for a in data.get("actors", []): + keys = keys_for(imdb_id=a.get("imdb_id"), tmdb_id=a.get("tmdb_id"), + jellyfin_id=a.get("jellyfin_id"), name=a.get("name"), + crosswalk=crosswalk) + windows = [(float(t0), float(t1)) for t0, t1 in a.get("scenes", [])] + for _, t1 in windows: + self._max_t = max(self._max_t, t1) + self.actors.append({"keys": keys, "windows": windows}) + + def present_at(self, t: float) -> set[frozenset[str]]: + """Set of actors present at t; each actor is its (frozen) key-set.""" + out: set[frozenset[str]] = set() + for a in self.actors: + for t0, t1 in a["windows"]: + if t0 <= t <= t1: + out.add(frozenset(a["keys"])) + break + return out + + def present_in_span(self, s0: float, s1: float) -> set[frozenset[str]]: + """Actors with ANY detection window overlapping [s0,s1]. + + Snaps detections to a scene grid: an actor seen anywhere inside a scene + counts as present for the whole scene. Isolates 'did we see this actor in + this scene at all' (coverage) from exact-timing recall.""" + out: set[frozenset[str]] = set() + for a in self.actors: + for t0, t1 in a["windows"]: + if t0 <= s1 and t1 >= s0: # interval overlap + out.add(frozenset(a["keys"])) + break + return out + + def all_keys(self) -> set[str]: + out: set[str] = set() + for a in self.actors: + out |= a["keys"] + return out + + @property + def max_t(self) -> float: + return self._max_t + + +def load_gallery_keys(path: str | None, crosswalk=None) -> set[str] | None: + """Union of match keys for every actor in the gallery, for masking. + + Accepts either the JSON gallery or the HDF5 fast-load gallery (.h5/.hdf5, + produced by json_to_hdf5_gallery.py) — the matcher reads HDF5, so this side must + too. HDF5 stores ids/names as parallel string datasets.""" + if not path: + return None + out: set[str] = set() + if path.endswith(".h5") or path.endswith(".hdf5"): + import h5py + with h5py.File(path, "r") as f: + def col(name): + return [(v.decode() if isinstance(v, bytes) else str(v)) + for v in f[name][:]] if name in f else [] + imdb, tmdb = col("imdb_id"), col("tmdb_id") + jf, name = col("jellyfin_id"), col("name") + for i in range(len(name)): + out |= keys_for(imdb_id=imdb[i] if i < len(imdb) else "", + tmdb_id=tmdb[i] if i < len(tmdb) else "", + jellyfin_id=jf[i] if i < len(jf) else "", + name=name[i], crosswalk=crosswalk) + return out + with open(path) as f: + data = json.load(f) + for a in data.get("actors", []): + out |= keys_for(imdb_id=a.get("imdb_id"), tmdb_id=a.get("tmdb_id"), + jellyfin_id=a.get("jellyfin_id"), name=a.get("name"), + crosswalk=crosswalk) + return out + + +# ── sampling ──────────────────────────────────────────────────────────────── + +def sample_points(args, pred: Prediction, gt) -> list[float]: + if args.scene_anchored: + spans = gt.scene_windows() + if not spans: + sys.exit("[eval] --scene-anchored: ground truth has no scene spans") + return [(t0 + t1) / 2.0 for t0, t1 in spans] + + end = args.end if args.end is not None else max(pred.max_t, _gt_end(gt)) + if end <= 0: + sys.exit("[eval] could not determine timeline end; pass --end") + + if args.random: + rng = random.Random(args.seed) + return sorted(rng.uniform(0.0, end) for _ in range(args.random)) + + n = int(end / args.step) + 1 + return [i * args.step for i in range(n)] + + +def _gt_end(gt) -> float: + spans = gt.scene_windows() + return max((t1 for _, t1 in spans), default=0.0) + + +# ── scoring ──────────────────────────────────────────────────────────────── + +def score(pred: Prediction, gt, points: list[float], mask: set[str] | None, + count_out_of_cast_fp: bool = False): + """Micro-sum TP/FP/FN over timepoints. + + Each side is a set of actors, an actor being its key-set. Predicted actor P + matches GT actor G iff their key-sets intersect (any shared id/name). We match + greedily so each actor is used once, then: + TP = matched pairs, FP = unmatched predicted, FN = unmatched GT. + + `mask` (gallery∩GT keys) restricts GT so X-Ray cast we can't recognise doesn't + inflate FN. By default predictions are masked the same way — which DROPS a + predicted actor who isn't in this film's cast (a cross-film misidentification), + hiding the pipeline's worst false positives. + + Set count_out_of_cast_fp=True to keep ALL predictions: an actor named who is not + a present GT cast member counts as an FP, including out-of-cast confusions. This + is the honest, ship-relevant precision. GT is still masked for fair recall. + """ + TP = FP = FN = 0 + per_point = [] + + for t in points: + P = [set(a) for a in pred.present_at(t)] + G = [set(a) for a in gt.present_at(t)] + if mask is not None: + G = [a for a in G if a & mask] + if not count_out_of_cast_fp: + P = [a for a in P if a & mask] + + tp = _match_count(P, G) + fp = len(P) - tp + fn = len(G) - tp + TP += tp + FP += fp + FN += fn + per_point.append((t, tp, fp, fn)) + + prec = TP / (TP + FP) if (TP + FP) else 0.0 + rec = TP / (TP + FN) if (TP + FN) else 0.0 + f1 = 2 * prec * rec / (prec + rec) if (prec + rec) else 0.0 + return {"TP": TP, "FP": FP, "FN": FN, "precision": prec, + "recall": rec, "f1": f1, "n_points": len(points), + "per_point": per_point} + + +def _match_count(P: list[set[str]], G: list[set[str]]) -> int: + """Greedy 1:1 matching of predicted↔GT actors by key intersection.""" + used = [False] * len(G) + matched = 0 + for pa in P: + for j, ga in enumerate(G): + if not used[j] and pa & ga: + used[j] = True + matched += 1 + break + return matched + + +# ── main ──────────────────────────────────────────────────────────────────── + +def main(): + p = argparse.ArgumentParser(description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter) + p.add_argument("--pred", required=True, help="pipeline output JSON") + src = p.add_mutually_exclusive_group(required=True) + src.add_argument("--xray", help="dir with people.csv/scenes.csv/people_in_scenes.csv") + src.add_argument("--movienet", help="MovieNet-PS root (needs --split and --title)") + p.add_argument("--split", default="Train_app10", help="MovieNet annotation split") + p.add_argument("--title", help="MovieNet title tt-id to filter to") + p.add_argument("--gallery", help="gallery.json for masking (gallery ∩ GT)") + p.add_argument("--crosswalk", help="tmdb→imdb JSON (tmdb_imdb_map.py) for exact " + "id join when pred/gallery lack imdb_id") + p.add_argument("--no-mask", action="store_true", help="disable gallery∩GT masking") + p.add_argument("--step", type=float, default=1.0, help="regular grid step (s)") + p.add_argument("--random", type=int, help="sample N uniform-random timepoints") + p.add_argument("--scene-anchored", action="store_true", + help="sample GT scene midpoints (one vote per scene)") + p.add_argument("--end", type=float, help="timeline end (s); default = max of pred/GT") + p.add_argument("--seed", type=int, default=0) + p.add_argument("--json-out", help="write full metrics (incl. per-point) here") + args = p.parse_args() + + crosswalk = CrosswalkTable.load(args.crosswalk) if args.crosswalk else None + if crosswalk is not None: + print(f"[eval] crosswalk: {len(crosswalk)} tmdb→imdb entries", file=sys.stderr) + + pred = Prediction(args.pred, crosswalk=crosswalk) + print(f"[eval] pred: {len(pred.actors)} actors, timeline≈{pred.max_t:.0f}s " + f"({pred.movie})", file=sys.stderr) + + if args.xray: + gt = XRayGroundTruth(args.xray) + else: + if not args.title: + sys.exit("[eval] --movienet requires --title tt-id") + gt = _load_movienet(args.movienet, args.split, args.title, args.gallery) + print(f"[eval] GT: {gt.summary()}", file=sys.stderr) + + mask = None + if not args.no_mask: + gkeys = load_gallery_keys(args.gallery, crosswalk=crosswalk) + gt_keys = gt.all_keys() + if gkeys is None: + # no gallery given → mask to GT ∩ pred key spaces so absent-from-gallery + # GT actors don't inflate FN. Fall back to GT keys the pred could name. + mask = gt_keys & pred.all_keys() + print("[eval] no --gallery; masking to GT∩pred keys " + f"({len(mask)})", file=sys.stderr) + else: + mask = gkeys & gt_keys + print(f"[eval] mask = gallery∩GT ({len(mask)} keys)", file=sys.stderr) + + points = sample_points(args, pred, gt) + print(f"[eval] sampling {len(points)} timepoints " + f"({'scene-anchored' if args.scene_anchored else 'random' if args.random else f'grid@{args.step}s'})", + file=sys.stderr) + + m = score(pred, gt, points, mask) + print("\n── presence eval ─────────────────────────────") + print(f" timepoints : {m['n_points']}") + print(f" TP/FP/FN : {m['TP']} / {m['FP']} / {m['FN']}") + print(f" precision : {m['precision']*100:.1f}%") + print(f" recall : {m['recall']*100:.1f}%") + print(f" F1 : {m['f1']*100:.1f}%") + + if args.json_out: + out = {k: v for k, v in m.items() if k != "per_point"} + out["per_point"] = [{"t": t, "tp": tp, "fp": fp, "fn": fn} + for t, tp, fp, fn in m["per_point"]] + Path(args.json_out).write_text(json.dumps(out, indent=2)) + print(f"[eval] wrote {args.json_out}", file=sys.stderr) + + +def _load_movienet(root, split, title, gallery): + sys.path.insert(0, str(Path(__file__).resolve().parent.parent)) + from movienet_prep import load_movienet_annotations + anns = load_movienet_annotations(Path(root), split) + anns = [a for a in anns if a["img_path"].startswith(title)] + if not anns: + sys.exit(f"[eval] no MovieNet annotations for title {title} in {split}") + id_to_name = {} + if gallery: + for a in json.load(open(gallery)).get("actors", []): + if a.get("imdb_id"): + id_to_name[a["imdb_id"]] = a.get("name", "") + return MovieNetGroundTruth(anns, id_to_name) + + +if __name__ == "__main__": + main() diff --git a/scripts/validation/test_sample_eval.py b/scripts/validation/test_sample_eval.py new file mode 100644 index 0000000..dca502d --- /dev/null +++ b/scripts/validation/test_sample_eval.py @@ -0,0 +1,148 @@ +#!/usr/bin/env python3 +""" +Self-contained tests for the presence eval. Builds a synthetic X-Ray fixture and +a pipeline-output JSON in a temp dir, then checks scoring, masking, name-fallback +matching, and sampling modes. Run: python scripts/validation/test_sample_eval.py +""" + +import csv +import json +import sys +import tempfile +from pathlib import Path + +HERE = Path(__file__).resolve().parent +sys.path.insert(0, str(HERE)) + +from identity import keys_for, norm_name # noqa: E402 +from sample_eval import Prediction, score, sample_points, _match_count # noqa: E402 +from ground_truth import XRayGroundTruth # noqa: E402 +from tmdb_imdb_map import CrosswalkTable # noqa: E402 + + +def _write_xray(d: Path): + d.mkdir(parents=True, exist_ok=True) + with open(d / "scenes.csv", "w", newline="") as f: + w = csv.writer(f); w.writerow(["scene", "start_ms", "end_ms"]) + w.writerows([("1", 0, 60000), ("2", 300000, 340000)]) + # real Zenodo X-Ray schema: people(name_id,person,character); + # people_in_scenes(scene,start,end,name_id,timestamp) + with open(d / "people.csv", "w", newline="") as f: + w = csv.writer(f); w.writerow(["name_id", "person", "character"]) + w.writerows([("nm0330687", "Lauren Graham", "Lorelai"), + ("nm0004754", "Alexis Bledel", "Rory"), + ("nm0000001", "Ghost Actor", "Ghost")]) + with open(d / "people_in_scenes.csv", "w", newline="") as f: + w = csv.writer(f); w.writerow(["scene", "start", "end", "name_id", "timestamp"]) + w.writerows([("1", 0, 60000, "nm0330687", 5000), + ("1", 0, 60000, "nm0000001", 8000), + ("2", 300000, 340000, "nm0004754", 305000)]) + + +def _write_pred(path: Path): + # pipeline output carries tmdb+name but NO nm ids -> name-fallback join to X-Ray + doc = {"schema_version": 1, "movie": "x", "sample_fps": 1.0, "anneal_sec": 10.0, + "actors": [ + {"name": "Lauren Graham", "imdb_id": "", "tmdb_id": "16858", + "jellyfin_id": "a", "scenes": [[21.0, 31.0], [50.0, 59.0]]}, + {"name": "Alexis Bledel", "imdb_id": "", "tmdb_id": "6279", + "jellyfin_id": "b", "scenes": [[301.0, 311.0]]}, + {"name": "Edward Herrmann", "imdb_id": "", "tmdb_id": "52995", + "jellyfin_id": "c", "scenes": [[4.0, 56.0]]}]} + path.write_text(json.dumps(doc)) + + +class T: + n = 0 + def check(self, cond, msg): + T.n += 1 + assert cond, f"FAIL: {msg}" + print(f" ok: {msg}") + + +def main(): + t = T() + + # -- identity --------------------------------------------------------------- + t.check(norm_name("Zöe Saldaña!") == "zoe saldana", "accent/punct normalization") + t.check(keys_for(imdb_id="nm1", name="Jo Ann") == {"imdb:nm1", "name:jo ann"}, + "keys_for builds namespaced tokens") + t.check(keys_for(imdb_id="") == set(), "empty ids dropped") + + # -- crosswalk: tmdb-only actor gains an exact imdb: key -------------------- + xw = CrosswalkTable({"16858": "nm0330687", "999": None}) + k = keys_for(imdb_id="", tmdb_id="16858", name="Lauren Graham", crosswalk=xw) + t.check("imdb:nm0330687" in k, "crosswalk resolves tmdb→imdb key") + k_null = keys_for(tmdb_id="999", crosswalk=xw) + t.check(not any(x.startswith("imdb:") for x in k_null), "crosswalk null → no imdb key") + t.check(len(xw) == 1, "CrosswalkTable len counts non-null entries") + + # -- matching --------------------------------------------------------------- + t.check(_match_count([{"name:jo"}, {"name:al"}], [{"imdb:x", "name:jo"}]) == 1, + "one match by shared name key") + t.check(_match_count([{"name:jo"}], [{"name:jo"}, {"name:jo"}]) == 1, + "greedy 1:1 uses each GT once") + + with tempfile.TemporaryDirectory() as tmp: + tmp = Path(tmp) + _write_xray(tmp / "xray") + _write_pred(tmp / "pred.json") + + pred = Prediction(tmp / "pred.json") + gt = XRayGroundTruth(tmp / "xray") + + # -- presence lookups --------------------------------------------------- + t.check(len(pred.present_at(25)) == 2, "Graham+Herrmann present at 25s") + t.check(len(pred.present_at(305)) == 1, "only Bledel present at 305s") + t.check(len(gt.present_at(30)) == 2, "X-Ray scene1 has Graham+Ghost at 30s") + t.check(len(gt.present_at(320)) == 1, "X-Ray scene2 has Bledel at 320s") + + # -- masking behaviour -------------------------------------------------- + pts = [30.0] + # unmasked: at 30s pred={Graham,Herrmann}, gt={Graham,Ghost} + # match Graham -> TP1; Herrmann unmatched -> FP1; Ghost unmatched -> FN1 + m = score(pred, gt, pts, mask=None) + t.check((m["TP"], m["FP"], m["FN"]) == (1, 1, 1), "unmasked 30s = 1/1/1") + + # masked to GT∩pred keys: Ghost & Herrmann are absent from the other side's + # key space, so both drop -> only Graham remains on both -> 1/0/0 + gt_keys = gt.all_keys(); pred_keys = pred.all_keys() + mask = gt_keys & pred_keys + m2 = score(pred, gt, pts, mask=mask) + t.check((m2["TP"], m2["FP"], m2["FN"]) == (1, 0, 0), + "masked 30s drops off-gallery actors = 1/0/0") + + # -- crosswalk end-to-end: exact imdb join, names garbled --------------- + # Rebuild pred with names that WON'T match X-Ray, but a crosswalk that maps + # their tmdb ids to the correct nm ids. Match must survive via imdb key. + garbled = {"schema_version": 1, "movie": "x", "actors": [ + {"name": "WRONG NAME A", "imdb_id": "", "tmdb_id": "16858", + "jellyfin_id": "a", "scenes": [[21.0, 31.0]]}, # →nm0330687 Graham + {"name": "WRONG NAME B", "imdb_id": "", "tmdb_id": "6279", + "jellyfin_id": "b", "scenes": [[301.0, 311.0]]}]} # →nm0004754 Bledel + (tmp / "garbled.json").write_text(json.dumps(garbled)) + xw = CrosswalkTable({"16858": "nm0330687", "6279": "nm0004754"}) + pred_g = Prediction(tmp / "garbled.json", crosswalk=xw) + # at 30s (scene1) Graham should match by imdb despite wrong name + m_g = score(pred_g, gt, [30.0], mask=None) + t.check(m_g["TP"] == 1, "crosswalk yields exact imdb match despite wrong names") + # without crosswalk, wrong names → no match at all + pred_bad = Prediction(tmp / "garbled.json") + m_bad = score(pred_bad, gt, [30.0], mask=None) + t.check(m_bad["TP"] == 0, "no crosswalk + wrong names → no match") + + # -- sampling modes ----------------------------------------------------- + class A: # arg stub + scene_anchored = True; random = None; step = 1.0; end = None; seed = 0 + sp = sample_points(A, pred, gt) + t.check(sp == [30.0, 320.0], "scene-anchored samples scene midpoints") + + A.scene_anchored = False; A.end = 10.0; A.step = 2.0 + grid = sample_points(A, pred, gt) + t.check(grid == [0.0, 2.0, 4.0, 6.0, 8.0, 10.0], "regular grid step") + + print(f"\nALL {T.n} CHECKS PASSED") + + +if __name__ == "__main__": + main() diff --git a/scripts/validation/tmdb_imdb_map.py b/scripts/validation/tmdb_imdb_map.py new file mode 100644 index 0000000..f434329 --- /dev/null +++ b/scripts/validation/tmdb_imdb_map.py @@ -0,0 +1,135 @@ +#!/usr/bin/env python3 +""" +tmdb_imdb_map.py — build & consult a cached tmdb_person_id → imdb_id crosswalk. + +The gallery (and pipeline output) key actors by TMDB person id but carry no IMDb +`nm…` id, while the X-Ray / MovieNet ground truth keys on IMDb. Rather than join on +fuzzy names, we resolve tmdb→imdb once via TMDB's authoritative +`/person/{id}/external_ids` endpoint and cache the result to JSON. The eval loaders +consult this table to add an exact `imdb:` key alongside each `tmdb:` key. + +Table format (JSON): { "<tmdb_person_id>": "nm0000123" | null, ... } +A null means "looked up, TMDB has no IMDb id" — cached so we don't re-query. + +Build / refresh the table: + python scripts/validation/tmdb_imdb_map.py \ + --gallery gallery_arcface_w600k_r50.json \ + --out scripts/validation/tmdb_imdb.json + # TMDB_API_KEY read from env / .env (via sae_env) + +Consult it from code: + from tmdb_imdb_map import CrosswalkTable + tbl = CrosswalkTable.load("scripts/validation/tmdb_imdb.json") + nm = tbl.imdb_for("35467") # -> "nm..." or None +""" + +from __future__ import annotations + +import argparse +import json +import os +import sys +import time +from pathlib import Path + +_HERE = Path(__file__).resolve().parent + + +class CrosswalkTable: + """Read-only view over the cached tmdb→imdb JSON. Missing file → empty table.""" + + def __init__(self, mapping: dict[str, str | None]): + self._m = mapping + + @classmethod + def load(cls, path: str | Path) -> "CrosswalkTable": + p = Path(path) + if not p.exists(): + return cls({}) + return cls(json.loads(p.read_text())) + + def imdb_for(self, tmdb_id) -> str | None: + if tmdb_id is None: + return None + return self._m.get(str(tmdb_id)) + + def __len__(self) -> int: + return sum(1 for v in self._m.values() if v) + + +# ── builder ───────────────────────────────────────────────────────────────── + +def _external_ids(tmdb_get, tmdb_person_id: str, token: str) -> str | None: + data = tmdb_get(f"/person/{tmdb_person_id}/external_ids", token) + imdb = data.get("imdb_id") + return imdb or None # normalize "" → None + + +def build(gallery_path: str, out_path: str, token: str, sleep: float = 0.0) -> None: + # import the existing TMDB helper (scripts/ is the parent dir) + sys.path.insert(0, str(_HERE.parent)) + from sae_tmdb import tmdb_get + + with open(gallery_path) as f: + actors = json.load(f).get("actors", []) + tmdb_ids = sorted({str(a["tmdb_id"]) for a in actors if a.get("tmdb_id")}) + print(f"[map] gallery tmdb ids: {len(tmdb_ids)}", file=sys.stderr) + + out = Path(out_path) + existing: dict[str, str | None] = {} + if out.exists(): + existing = json.loads(out.read_text()) + print(f"[map] resuming from {len(existing)} cached entries", file=sys.stderr) + + todo = [t for t in tmdb_ids if t not in existing] + print(f"[map] to resolve: {len(todo)}", file=sys.stderr) + + n_ok = n_none = n_err = 0 + for i, tid in enumerate(todo, 1): + try: + nm = _external_ids(tmdb_get, tid, token) + existing[tid] = nm + n_ok += (nm is not None) + n_none += (nm is None) + except Exception as e: # network/rate-limit/404 — record nothing, keep going + n_err += 1 + print(f"\n[map] error on tmdb {tid}: {e}", file=sys.stderr) + if i % 25 == 0 or i == len(todo): + print(f"\r[map] {i}/{len(todo)} resolved " + f"(imdb={n_ok} none={n_none} err={n_err})", end="", file=sys.stderr) + out.write_text(json.dumps(existing, indent=2)) # checkpoint + if sleep: + time.sleep(sleep) + + out.write_text(json.dumps(existing, indent=2)) + print(f"\n[map] wrote {out} — {sum(1 for v in existing.values() if v)} imdb ids", + file=sys.stderr) + + +def main(): + # load .env → os.environ (same convention as the gallery builders) + sys.path.insert(0, str(_HERE.parent)) + try: + import sae_env # noqa: F401 (side-effect import) + except Exception: + pass + + p = argparse.ArgumentParser(description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter) + p.add_argument("--gallery", required=True, help="gallery.json (source of tmdb ids)") + p.add_argument("--out", default=str(_HERE / "tmdb_imdb.json"), + help="output crosswalk JSON (default: scripts/validation/tmdb_imdb.json)") + p.add_argument("--tmdb-key", default=os.environ.get("TMDB_API_KEY"), + help="TMDB v3 API key or v4 read token. Env: TMDB_API_KEY") + p.add_argument("--sleep", type=float, default=0.0, + help="seconds between requests (TMDB has no hard limit; use if throttled)") + args = p.parse_args() + + if not args.tmdb_key: + sys.exit("[map] no TMDB key — set TMDB_API_KEY or pass --tmdb-key") + + build(args.gallery, args.out, args.tmdb_key, args.sleep) + + +if __name__ == "__main__": + main()