feat(tooling): X-Ray threshold optimizer, gallery utilities, artifact registry, docs build
Optimizer (scripts/optimizer/): replay.py runs the real C++ tracker/matcher/ scene_tracker chain over a dumped-embeddings HDF5 via sae_kpn, so a threshold sweep never re-decodes video or re-embeds faces. optimize.py drives scipy's differential_evolution over the knob space, with DE-level parallelism (multiple population candidates evaluated concurrently via a ThreadPoolExecutor) on top of per-film replay parallelism. second_score.py is the per-second X-Ray scoring metric (TPI/FPI/FN, out-of-cast misID weighted 10x, fair recall masked to gallery-known cast) that superseded an earlier scene-union metric. dump_error_frames.py / dump_scene_montage.py extract annotated video frames (bounding boxes, TPI/FPI/FN captions, onscreen-vs-offscreen split) for visual review of a replay against ground truth. Gallery utilities: cast_restrict.py, gallery_membership.py, fetch_missing_actors.py, reembed_gallery.py. scripts/validation/: X-Ray ground-truth loading and provider-agnostic identity matching (identity.py's keys_for — an actor is the union of every id we can derive, since pipeline output and ground truth don't share one id space). scripts/artifacts/: push/pull scripts for the Gitea generic package registry — galleries, montage frames, and experiment data (manifests/trajectories/results) are pushed there instead of committed, since none are needed to run the app, only benchmarks. Versioned by git short-SHA. scripts/docs/: MkDocs site build (build_site.sh) and the calibration-curve comparison chart (calibration_chart.py, matplotlib, reads each gallery's embedded calibration). Gallery-building scripts (make_jellyfin_gallery.py, make_gallery.py, filter_gallery.py, run_from_jellyfin.py, movienet_eval.py, movienet_prep.py, sae_gallery.py) updated to read/write HDF5 galleries exclusively, matching the engine-side format switch. run_from_jellyfin.py and the optimizer no longer carry movie source paths in shared manifests (some source filenames include scene-release tags) — resolved locally via a gitignored file-lut.json instead.
This commit is contained in:
Executable
+106
@@ -0,0 +1,106 @@
|
||||
#!/bin/bash
|
||||
# pull_artifacts.sh — download benchmark artifacts (galleries, montage frames) from
|
||||
# the Gitea generic package registry. Counterpart to push_artifacts.sh.
|
||||
#
|
||||
# Downloads are public (no token needed) as long as the repo/packages are public.
|
||||
# Resolving "latest" needs GITEA_TOKEN (the list-packages endpoint requires auth
|
||||
# on this instance even for a public account) — export it before using "latest".
|
||||
#
|
||||
# Usage:
|
||||
# scripts/artifacts/pull_artifacts.sh galleries [version]
|
||||
# scripts/artifacts/pull_artifacts.sh montage-frames <film-slug> [version]
|
||||
# scripts/artifacts/pull_artifacts.sh experiment-data [version]
|
||||
# version defaults to "latest" (newest uploaded version, by created_at).
|
||||
set -euo pipefail
|
||||
|
||||
REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
|
||||
OWNER="dtourolle"
|
||||
# NOTE: file download lives under /api/packages/ (no /v1/); listing/metadata
|
||||
# lives under the regular /api/v1/packages/ REST API. Different base paths,
|
||||
# both real — see push_artifacts.sh's comment.
|
||||
DL_BASE="https://gitea.tourolle.paris/api/packages/${OWNER}"
|
||||
|
||||
resolve_latest_version() {
|
||||
local package="$1"
|
||||
if [ -z "${GITEA_TOKEN:-}" ]; then
|
||||
echo "error: resolving 'latest' needs GITEA_TOKEN (list-packages requires auth here)." >&2
|
||||
echo " export GITEA_TOKEN=... or pass an explicit version instead of 'latest'." >&2
|
||||
exit 1
|
||||
fi
|
||||
curl -sf "https://gitea.tourolle.paris/api/v1/packages/${OWNER}" \
|
||||
-H "Authorization: token ${GITEA_TOKEN}" \
|
||||
| python3 -c "
|
||||
import json, sys
|
||||
d = json.load(sys.stdin)
|
||||
matches = [p for p in d if p['name'] == '$package' and p['type'] == 'generic']
|
||||
if not matches:
|
||||
sys.exit('no versions found for package \'$package\'')
|
||||
matches.sort(key=lambda p: p['created_at'])
|
||||
print(matches[-1]['version'])
|
||||
"
|
||||
}
|
||||
|
||||
pull_galleries() {
|
||||
local version="$1"
|
||||
local dest="${REPO_ROOT}/experiments/galleries"
|
||||
mkdir -p "$dest"
|
||||
echo "=== galleries (version ${version}) ==="
|
||||
for model in arcface_w600k_r50 arcface_r18 arcface_w600k_mbf LVFace-B_Glint360K; do
|
||||
local f="gallery_${model}.h5"
|
||||
echo " fetching ${f}..."
|
||||
curl -sf "${DL_BASE}/generic/galleries/${version}/${f}" \
|
||||
-o "${dest}/${f}" || echo " [warn] ${f} not found at version ${version}"
|
||||
done
|
||||
}
|
||||
|
||||
pull_montage_frames() {
|
||||
local version="$1" film="$2"
|
||||
local dest="${REPO_ROOT}/experiments/results/holdout/montage_bestworst"
|
||||
mkdir -p "$dest"
|
||||
echo "=== montage-frames/${film} (version ${version}) ==="
|
||||
local tmp; tmp="$(mktemp)"
|
||||
curl -sf "${DL_BASE}/generic/montage-frames/${version}/${film}.zip" -o "$tmp"
|
||||
mkdir -p "${dest}/${film}"
|
||||
unzip -qo "$tmp" -d "${dest}/${film}"
|
||||
rm "$tmp"
|
||||
}
|
||||
|
||||
pull_experiment_data() {
|
||||
local version="$1"
|
||||
echo "=== experiment-data (version ${version}) ==="
|
||||
local tmp; tmp="$(mktemp)"
|
||||
curl -sf "${DL_BASE}/generic/experiment-data/${version}/experiment-data.zip" -o "$tmp"
|
||||
unzip -qo "$tmp" -d "$REPO_ROOT"
|
||||
rm "$tmp"
|
||||
}
|
||||
|
||||
if [ $# -eq 0 ]; then
|
||||
echo "usage: $0 galleries [version]" >&2
|
||||
echo " $0 montage-frames <film-slug> [version]" >&2
|
||||
echo " $0 experiment-data [version]" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
TARGET="$1"
|
||||
case "$TARGET" in
|
||||
galleries)
|
||||
VERSION="${2:-latest}"
|
||||
[ "$VERSION" = "latest" ] && VERSION="$(resolve_latest_version galleries)"
|
||||
pull_galleries "$VERSION"
|
||||
;;
|
||||
montage-frames)
|
||||
FILM="${2:?usage: $0 montage-frames <film-slug> [version]}"
|
||||
VERSION="${3:-latest}"
|
||||
[ "$VERSION" = "latest" ] && VERSION="$(resolve_latest_version montage-frames)"
|
||||
pull_montage_frames "$VERSION" "$FILM"
|
||||
;;
|
||||
experiment-data)
|
||||
VERSION="${2:-latest}"
|
||||
[ "$VERSION" = "latest" ] && VERSION="$(resolve_latest_version experiment-data)"
|
||||
pull_experiment_data "$VERSION"
|
||||
;;
|
||||
*)
|
||||
echo "unknown target: $TARGET (expected galleries, montage-frames, or experiment-data)" >&2
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
Executable
+112
@@ -0,0 +1,112 @@
|
||||
#!/bin/bash
|
||||
# push_artifacts.sh — upload benchmark artifacts (galleries, montage frames) to the
|
||||
# Gitea generic package registry, decoupled from git history entirely. These are
|
||||
# NOT needed to run the main app — only for benchmarks/experiments/reports.
|
||||
#
|
||||
# Requires GITEA_TOKEN in the environment (a Gitea access token with package
|
||||
# read/write scope). Never hardcode the token; export it in your shell:
|
||||
# export GITEA_TOKEN=...
|
||||
#
|
||||
# Usage:
|
||||
# scripts/artifacts/push_artifacts.sh galleries
|
||||
# scripts/artifacts/push_artifacts.sh montage-frames
|
||||
# scripts/artifacts/push_artifacts.sh experiment-data
|
||||
# scripts/artifacts/push_artifacts.sh galleries montage-frames experiment-data
|
||||
#
|
||||
# Package layout (owner=dtourolle, repo=scene-actor-extraction):
|
||||
# generic/galleries/<version>/gallery_<model>.h5 (one file per model)
|
||||
# generic/montage-frames/<version>/<film-slug>.zip (zipped per-film frames)
|
||||
# generic/experiment-data/<version>/experiment-data.zip (manifests/trajectories/results)
|
||||
# version = current git short SHA, so artifacts are traceable to the code that
|
||||
# produced them. Re-running with the same SHA overwrites that version's files.
|
||||
set -euo pipefail
|
||||
|
||||
REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
|
||||
OWNER="dtourolle"
|
||||
PKG_REPO="scene-actor-extraction"
|
||||
# NOTE: the generic package registry lives under /api/packages/ (no /v1/) —
|
||||
# distinct from the regular REST API under /api/v1/packages/ used for listing.
|
||||
BASE_URL="https://gitea.tourolle.paris/api/packages/${OWNER}"
|
||||
VERSION="$(git -C "$REPO_ROOT" rev-parse --short HEAD)"
|
||||
|
||||
if [ -z "${GITEA_TOKEN:-}" ]; then
|
||||
echo "error: GITEA_TOKEN is not set. export GITEA_TOKEN=<your token> and retry." >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
upload() {
|
||||
local package="$1" filename="$2" filepath="$3"
|
||||
local url="${BASE_URL}/generic/${package}/${VERSION}/${filename}"
|
||||
echo " uploading ${filename} -> ${package}/${VERSION}..."
|
||||
curl -sf -X PUT "$url" \
|
||||
-H "Authorization: token ${GITEA_TOKEN}" \
|
||||
-H "Content-Type: application/octet-stream" \
|
||||
--upload-file "$filepath" \
|
||||
-o /dev/null -w " HTTP %{http_code}\n"
|
||||
}
|
||||
|
||||
push_galleries() {
|
||||
echo "=== galleries (version ${VERSION}) ==="
|
||||
local dir="${REPO_ROOT}/experiments/galleries"
|
||||
shopt -s nullglob
|
||||
for f in "$dir"/gallery_*.h5; do
|
||||
upload "galleries" "$(basename "$f")" "$f"
|
||||
done
|
||||
shopt -u nullglob
|
||||
}
|
||||
|
||||
push_montage_frames() {
|
||||
echo "=== montage-frames (version ${VERSION}) ==="
|
||||
local root="${REPO_ROOT}/experiments/results/holdout/montage_bestworst"
|
||||
if [ ! -d "$root" ]; then
|
||||
echo " no montage_bestworst dir found, skipping" >&2
|
||||
return
|
||||
fi
|
||||
local tmp
|
||||
tmp="$(mktemp -d)"
|
||||
trap 'rm -rf "$tmp"' RETURN
|
||||
for filmdir in "$root"/*/; do
|
||||
[ -d "$filmdir" ] || continue
|
||||
local slug ascii_slug zipfile
|
||||
slug="$(basename "$filmdir")"
|
||||
# Gitea's generic package registry rejects non-ASCII filenames (verified:
|
||||
# a bare é in the name 400s). Transliterate for the upload name only —
|
||||
# the local directory name (with accents) is untouched.
|
||||
ascii_slug="$(echo "$slug" | iconv -f utf-8 -t ascii//translit 2>/dev/null || echo "$slug")"
|
||||
zipfile="${tmp}/${ascii_slug}.zip"
|
||||
(cd "$filmdir" && zip -qr "$zipfile" .)
|
||||
upload "montage-frames" "${ascii_slug}.zip" "$zipfile"
|
||||
done
|
||||
}
|
||||
|
||||
push_experiment_data() {
|
||||
echo "=== experiment-data (version ${VERSION}) ==="
|
||||
# manifests/trajectories/results are all small text/JSON — no source paths
|
||||
# (manifests never carry a "movie" field; see experiments/file-lut.json)
|
||||
# so this is safe to share as one bundle.
|
||||
local tmp; tmp="$(mktemp -d)"
|
||||
trap 'rm -rf "$tmp"' RETURN
|
||||
local zipfile="${tmp}/experiment-data.zip"
|
||||
(cd "$REPO_ROOT" && zip -qr "$zipfile" \
|
||||
experiments/manifests experiments/trajectories experiments/results \
|
||||
-x '*.log' -x '*/holdout/montage_bestworst/*' -x '*/holdout/frames/*' \
|
||||
-x '*/holdout/montage/*' -x '*/holdout/*.jsonl' -x '*/holdout/pred_*.json' \
|
||||
-x '*/holdout/raw_*.jsonl')
|
||||
upload "experiment-data" "experiment-data.zip" "$zipfile"
|
||||
}
|
||||
|
||||
if [ $# -eq 0 ]; then
|
||||
echo "usage: $0 <galleries|montage-frames|experiment-data> [...]" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
for target in "$@"; do
|
||||
case "$target" in
|
||||
galleries) push_galleries ;;
|
||||
montage-frames) push_montage_frames ;;
|
||||
experiment-data) push_experiment_data ;;
|
||||
*) echo "unknown target: $target (expected galleries, montage-frames, or experiment-data)" >&2; exit 1 ;;
|
||||
esac
|
||||
done
|
||||
|
||||
echo "Done. Browse at: https://gitea.tourolle.paris/${OWNER}/${PKG_REPO}/packages"
|
||||
@@ -4,8 +4,9 @@
|
||||
# block the pipeline; this script does it offline so cold starts are instant.
|
||||
#
|
||||
# Profiles must match src/arcface_embedder.hpp and src/scrfd_decoder.hpp:
|
||||
# ArcFace : min=1x3x112x112 opt=Nx3x112x112 max=Nx3x112x112 (N = embed batch)
|
||||
# SCRFD : 1x3x640x640 (fixed; we letterbox to this)
|
||||
# ArcFace : min=1x3x112x112 opt=Nx3x112x112 max=Nx3x112x112 (N = embed batch)
|
||||
# SCRFD : 1x3x640x640 (fixed; we letterbox to this)
|
||||
# TransNetV2 : 1x100x27x48x3 (fixed; scene detector window), input tensor "input"
|
||||
#
|
||||
# These trtexec-built engines are *not* picked up by the ORT TRT EP cache —
|
||||
# ORT uses its own engine format. The point of this script is:
|
||||
@@ -22,6 +23,7 @@ mkdir -p "$OUT"
|
||||
EMBED_BATCH="${EMBED_BATCH:-4}"
|
||||
ARCFACE_MODEL="${ARCFACE_MODEL:-$MODELS/arcface_w600k_r50.onnx}"
|
||||
SCRFD_MODEL="${SCRFD_MODEL:-$MODELS/scrfd_500m_bnkps.onnx}"
|
||||
SCENE_MODEL="${SCENE_MODEL:-$MODELS/transnetv2.onnx}"
|
||||
|
||||
run() { echo "+ $*"; "$@"; }
|
||||
|
||||
@@ -46,6 +48,24 @@ run trtexec \
|
||||
--saveEngine="$OUT/scrfd.$(basename "$SCRFD_MODEL" .onnx).640.fp16.engine" \
|
||||
--useCudaGraph
|
||||
|
||||
if [[ -f "$SCENE_MODEL" ]]; then
|
||||
echo
|
||||
echo "== TransNetV2 (scene detector) =="
|
||||
# Fixed 1x100x27x48x3 window. The raw-TRT scene detector backend loads this
|
||||
# engine directly via --scene-detector-engine; the ORT-TRT EP builds its own.
|
||||
run trtexec \
|
||||
--onnx="$SCENE_MODEL" \
|
||||
--fp16 \
|
||||
--minShapes=input:1x100x27x48x3 \
|
||||
--optShapes=input:1x100x27x48x3 \
|
||||
--maxShapes=input:1x100x27x48x3 \
|
||||
--saveEngine="$OUT/transnetv2.100x27x48.fp16.engine" \
|
||||
--useCudaGraph
|
||||
else
|
||||
echo
|
||||
echo "== TransNetV2 skipped (no $SCENE_MODEL) =="
|
||||
fi
|
||||
|
||||
echo
|
||||
echo "Engines saved under: $OUT"
|
||||
echo "Look for 'mean: ... ms' in each section for per-call latency."
|
||||
|
||||
@@ -0,0 +1,137 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
convert_transnetv2.py — verify and optimise the TransNetV2 scene-detector model.
|
||||
|
||||
The scene detector (src/nodes/scene_detector_node.hpp → ISceneDetector) consumes
|
||||
the elya5/transnetv2 ONNX export with a fixed input contract:
|
||||
|
||||
input "input" : float32 [1, 100, 27, 48, 3] RGB, channels-last, 0-255
|
||||
output "534" : float32 [1, 100, 1] per-frame boundary logits (used)
|
||||
output "535" : float32 [1, 100, 1] many-hot head (ignored)
|
||||
|
||||
This script provides the ORT/TRT conversion infra for that model:
|
||||
|
||||
--verify (default) assert the .onnx matches the contract above and run one
|
||||
dummy inference through onnxruntime, reporting output ranges.
|
||||
--ort-opt write a graph-optimised .ort next to the model (ORT loads this
|
||||
faster; the runtime also caches its own under ./ort_cache).
|
||||
--trt build a TensorRT engine via trtexec with the pinned 1x100x27x48x3
|
||||
profile (delegates to scripts/build_trt_engines.sh SCENE_MODEL).
|
||||
|
||||
Usage:
|
||||
python scripts/convert_transnetv2.py # verify default model
|
||||
python scripts/convert_transnetv2.py --ort-opt
|
||||
python scripts/convert_transnetv2.py --trt
|
||||
python scripts/convert_transnetv2.py --model path/to/transnetv2.onnx --verify
|
||||
"""
|
||||
import argparse
|
||||
import os
|
||||
import subprocess
|
||||
import sys
|
||||
|
||||
ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
|
||||
DEFAULT_MODEL = os.path.join(ROOT, "models", "transnetv2.onnx")
|
||||
|
||||
# The fixed contract the C++ scene detector depends on.
|
||||
EXPECTED_INPUT_SHAPE = [1, 100, 27, 48, 3]
|
||||
EXPECTED_OUTPUT_SHAPE = [1, 100, 1]
|
||||
|
||||
|
||||
def verify(model_path: str) -> int:
|
||||
import numpy as np
|
||||
import onnx
|
||||
import onnxruntime as ort
|
||||
|
||||
print(f"[verify] loading {model_path}")
|
||||
m = onnx.load(model_path, load_external_data=False)
|
||||
g = m.graph
|
||||
|
||||
def shape(t):
|
||||
return [d.dim_value if d.HasField("dim_value") else d.dim_param
|
||||
for d in t.type.tensor_type.shape.dim]
|
||||
|
||||
in_shape = shape(g.input[0])
|
||||
print(f"[verify] input '{g.input[0].name}': {in_shape}")
|
||||
if in_shape != EXPECTED_INPUT_SHAPE:
|
||||
print(f"[verify] ERROR: input shape {in_shape} != {EXPECTED_INPUT_SHAPE}")
|
||||
return 1
|
||||
|
||||
out_names = [o.name for o in g.output]
|
||||
out0_shape = shape(g.output[0])
|
||||
print(f"[verify] outputs: {out_names} primary '{out_names[0]}': {out0_shape}")
|
||||
if out0_shape != EXPECTED_OUTPUT_SHAPE:
|
||||
print(f"[verify] ERROR: primary output {out0_shape} != {EXPECTED_OUTPUT_SHAPE}")
|
||||
return 1
|
||||
|
||||
# One dummy inference: a mid-grey clip should produce low boundary scores.
|
||||
sess = ort.InferenceSession(model_path, providers=["CPUExecutionProvider"])
|
||||
dummy = np.full(EXPECTED_INPUT_SHAPE, 128.0, dtype=np.float32)
|
||||
logits = sess.run([out_names[0]], {g.input[0].name: dummy})[0]
|
||||
probs = 1.0 / (1.0 + np.exp(-logits))
|
||||
print(f"[verify] dummy inference OK — boundary prob "
|
||||
f"min={probs.min():.4f} max={probs.max():.4f} mean={probs.mean():.4f}")
|
||||
print("[verify] contract matches the C++ ISceneDetector. ✓")
|
||||
return 0
|
||||
|
||||
|
||||
def ort_opt(model_path: str) -> int:
|
||||
import onnxruntime as ort
|
||||
|
||||
out_path = os.path.splitext(model_path)[0] + ".ort"
|
||||
print(f"[ort-opt] writing graph-optimised model → {out_path}")
|
||||
so = ort.SessionOptions()
|
||||
so.graph_optimization_level = ort.GraphOptimizationLevel.ORT_ENABLE_ALL
|
||||
so.optimized_model_filepath = out_path
|
||||
# Constructing the session triggers optimisation + serialisation.
|
||||
ort.InferenceSession(model_path, so, providers=["CPUExecutionProvider"])
|
||||
print(f"[ort-opt] done: {out_path}")
|
||||
return 0
|
||||
|
||||
|
||||
def trt(model_path: str) -> int:
|
||||
script = os.path.join(ROOT, "scripts", "build_trt_engines.sh")
|
||||
print(f"[trt] delegating to {script} (SCENE_MODEL={model_path})")
|
||||
env = dict(os.environ, SCENE_MODEL=model_path)
|
||||
# build_trt_engines.sh also builds ArcFace/SCRFD; that's harmless (and a
|
||||
# useful sanity check), but if you only want the scene engine, run trtexec
|
||||
# directly with the profile printed below.
|
||||
print("[trt] equivalent standalone command:")
|
||||
print(f" trtexec --onnx={model_path} --fp16 "
|
||||
f"--minShapes=input:1x100x27x48x3 "
|
||||
f"--optShapes=input:1x100x27x48x3 "
|
||||
f"--maxShapes=input:1x100x27x48x3 "
|
||||
f"--saveEngine=trt_cache/transnetv2.100x27x48.fp16.engine --useCudaGraph")
|
||||
return subprocess.call(["bash", script], env=env)
|
||||
|
||||
|
||||
def main() -> int:
|
||||
ap = argparse.ArgumentParser(description=__doc__,
|
||||
formatter_class=argparse.RawDescriptionHelpFormatter)
|
||||
ap.add_argument("--model", default=DEFAULT_MODEL,
|
||||
help=f"path to transnetv2.onnx (default: {DEFAULT_MODEL})")
|
||||
ap.add_argument("--verify", action="store_true", help="verify I/O contract (default)")
|
||||
ap.add_argument("--ort-opt", action="store_true", help="write optimised .ort")
|
||||
ap.add_argument("--trt", action="store_true", help="build a TensorRT engine")
|
||||
args = ap.parse_args()
|
||||
|
||||
if not os.path.isfile(args.model):
|
||||
print(f"ERROR: model not found: {args.model}\n"
|
||||
f"Fetch it with: bash scripts/download_models.sh", file=sys.stderr)
|
||||
return 2
|
||||
|
||||
# Default action is verify when nothing else is requested.
|
||||
if not (args.ort_opt or args.trt):
|
||||
args.verify = True
|
||||
|
||||
rc = 0
|
||||
if args.verify:
|
||||
rc |= verify(args.model)
|
||||
if rc == 0 and args.ort_opt:
|
||||
rc |= ort_opt(args.model)
|
||||
if rc == 0 and args.trt:
|
||||
rc |= trt(args.model)
|
||||
return rc
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
Executable
+40
@@ -0,0 +1,40 @@
|
||||
#!/bin/bash
|
||||
# build_site.sh — pull the images the docs reference from the artifact registry
|
||||
# (if not already present locally), stage them under docs/assets/, then build
|
||||
# the MkDocs site. The built site/ output is what gets pushed to gitea-pages —
|
||||
# never the source images themselves (see scripts/artifacts/push_artifacts.sh).
|
||||
set -euo pipefail
|
||||
|
||||
REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
|
||||
cd "$REPO_ROOT"
|
||||
|
||||
ASSETS_DIR="docs/assets/images"
|
||||
mkdir -p "$ASSETS_DIR"
|
||||
|
||||
# Frames referenced by docs/rep4-optimizer-results.md. Pull the film's montage
|
||||
# frames from the registry if this machine doesn't already have them locally.
|
||||
FRAMES_ROOT="experiments/results/holdout/frames"
|
||||
if [ ! -d "$FRAMES_ROOT/many_saints" ] || [ ! -d "$FRAMES_ROOT/downton_abbey" ]; then
|
||||
echo "==> pulling montage frames (not found locally)..."
|
||||
scripts/artifacts/pull_artifacts.sh montage-frames Many_Saints_of_Newark || true
|
||||
scripts/artifacts/pull_artifacts.sh montage-frames Downton_Abbey__A_New_Era || true
|
||||
fi
|
||||
|
||||
echo "==> staging referenced frames into ${ASSETS_DIR}"
|
||||
cp -v "${FRAMES_ROOT}/many_saints/fpi/fpi_t03543.jpg" \
|
||||
"${ASSETS_DIR}/many_saints_ghost_fpi.jpg"
|
||||
cp -v "${FRAMES_ROOT}/downton_abbey/fpi/fpi_t07242.jpg" \
|
||||
"${ASSETS_DIR}/downton_abbey_ghost_fpi.jpg"
|
||||
|
||||
if [ ! -d experiments/galleries ] || [ -z "$(ls -A experiments/galleries 2>/dev/null)" ]; then
|
||||
echo "==> pulling galleries (not found locally)..."
|
||||
scripts/artifacts/pull_artifacts.sh galleries
|
||||
fi
|
||||
|
||||
echo "==> generating calibration curve chart"
|
||||
python3 scripts/docs/calibration_chart.py --out "${ASSETS_DIR}/calibration_curves.png"
|
||||
|
||||
echo "==> building site"
|
||||
mkdocs build
|
||||
|
||||
echo "==> done. site/ is ready to deploy to the gitea-pages branch."
|
||||
@@ -0,0 +1,89 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
calibration_chart.py — plot each model's calibrated P(match|similarity) sigmoid,
|
||||
from the (a, b) fitted into each gallery's HDF5 /calibration group. Shows
|
||||
discriminative power: a steeper curve (larger |a|) separates positive/negative
|
||||
pairs more sharply at the same decision boundary.
|
||||
|
||||
Usage:
|
||||
python scripts/docs/calibration_chart.py --out docs/assets/images/calibration_curves.png
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
from pathlib import Path
|
||||
|
||||
import h5py
|
||||
import numpy as np
|
||||
import matplotlib.pyplot as plt
|
||||
|
||||
MODELS = [
|
||||
("arcface_w600k_r50", "ArcFace w600k-R50"),
|
||||
("arcface_r18", "ArcFace R18"),
|
||||
("arcface_w600k_mbf", "ArcFace w600k-MBF"),
|
||||
("LVFace-B_Glint360K", "LVFace-B Glint360K"),
|
||||
]
|
||||
COLOURS = ["#2a78d6", "#008300", "#e87ba4", "#eda100"]
|
||||
|
||||
REPO = Path(__file__).resolve().parent.parent.parent
|
||||
|
||||
|
||||
def load_calibrations() -> list[dict]:
|
||||
out = []
|
||||
for slug, label in MODELS:
|
||||
path = REPO / f"experiments/galleries/gallery_{slug}.h5"
|
||||
if not path.exists():
|
||||
print(f"[calibration_chart] skip {slug}: gallery not found at {path}")
|
||||
continue
|
||||
with h5py.File(path, "r") as f:
|
||||
if "calibration" not in f:
|
||||
print(f"[calibration_chart] skip {slug}: no calibration in gallery "
|
||||
f"(run a replay against it once to fit and embed one)")
|
||||
continue
|
||||
cal = f["calibration"]
|
||||
out.append({"slug": slug, "label": label,
|
||||
"a": float(cal.attrs["a"]), "b": float(cal.attrs["b"])})
|
||||
return out
|
||||
|
||||
|
||||
def main():
|
||||
p = argparse.ArgumentParser()
|
||||
p.add_argument("--out", required=True)
|
||||
args = p.parse_args()
|
||||
|
||||
models = load_calibrations()
|
||||
if not models:
|
||||
raise SystemExit("no galleries had embedded calibration — run a replay "
|
||||
"against each gallery once first (see identity_matcher_node.hpp)")
|
||||
|
||||
sim = np.linspace(-1, 1, 400)
|
||||
fig, ax = plt.subplots(figsize=(7.5, 4.8), dpi=150)
|
||||
|
||||
for m, colour in zip(models, COLOURS):
|
||||
p_match = 1.0 / (1.0 + np.exp(-(m["a"] * sim + m["b"])))
|
||||
boundary = -m["b"] / m["a"]
|
||||
ax.plot(sim, p_match, color=colour, linewidth=2,
|
||||
label=f"{m['label']} (a={m['a']:.1f}, boundary@P=0.5: sim={boundary:.2f})")
|
||||
|
||||
ax.axhline(0.5, color="#999999", linewidth=1, linestyle="--", zorder=0)
|
||||
ax.set_xlabel("cosine similarity")
|
||||
ax.set_ylabel("P(match)")
|
||||
ax.set_title("Calibrated P(match | similarity), per embedding model")
|
||||
ax.set_xlim(-1, 1)
|
||||
ax.set_ylim(0, 1)
|
||||
ax.legend(loc="upper left", fontsize=8, frameon=False)
|
||||
ax.spines["top"].set_visible(False)
|
||||
ax.spines["right"].set_visible(False)
|
||||
fig.tight_layout()
|
||||
|
||||
out_path = Path(args.out)
|
||||
out_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
fig.savefig(out_path)
|
||||
print(f"[calibration_chart] wrote {out_path} ({len(models)} models)")
|
||||
for m in models:
|
||||
print(f" {m['label']}: a={m['a']:.2f} b={m['b']:.2f} "
|
||||
f"boundary(P=0.5)=sim{-m['b']/m['a']:.3f}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -66,6 +66,23 @@ else
|
||||
echo "SCRFD-500MF already present: $SCRFD_FILE"
|
||||
fi
|
||||
|
||||
# ── TransNetV2 shot-boundary detection (scene detector, opt-in) ───────────────
|
||||
# ONNX export (elya5/transnetv2, MIT). Fixed input 1x100x27x48x3 (RGB 0-255),
|
||||
# primary output "534" = per-frame boundary logits. Used only with --scene-detect.
|
||||
SCENE_FILE="$MODELS_DIR/transnetv2.onnx"
|
||||
SCENE_SHA="c4d54a682bace32f25136ef83ca2c9d403e8f8193775efeb995172a0d95a8e0c"
|
||||
if [ ! -f "$SCENE_FILE" ]; then
|
||||
echo "Downloading TransNetV2…"
|
||||
curl -L "https://huggingface.co/elya5/transnetv2/resolve/main/transnetv2.onnx" \
|
||||
-o "$SCENE_FILE"
|
||||
if command -v sha256sum >/dev/null; then
|
||||
echo "$SCENE_SHA $SCENE_FILE" | sha256sum -c - \
|
||||
|| echo "WARNING: TransNetV2 sha256 mismatch (upstream may have changed)"
|
||||
fi
|
||||
else
|
||||
echo "TransNetV2 already present: $SCENE_FILE"
|
||||
fi
|
||||
|
||||
echo ""
|
||||
echo "Models ready in $MODELS_DIR/:"
|
||||
ls -lh "$MODELS_DIR"
|
||||
|
||||
+11
-11
@@ -1,5 +1,5 @@
|
||||
#!/usr/bin/env python3
|
||||
"""filter_gallery.py — restrict a global gallery.json to one title's known cast.
|
||||
"""filter_gallery.py — restrict a global gallery.h5 to one title's known cast.
|
||||
|
||||
make_jellyfin_gallery.py builds a single gallery spanning the whole Jellyfin
|
||||
library, so identity_matcher can recognise any actor from any film. For a
|
||||
@@ -8,28 +8,27 @@ credited cast only — fewer candidates means faster matching and fewer
|
||||
look-alike false positives.
|
||||
|
||||
This script looks up the title's cast live from Jellyfin and writes a
|
||||
filtered gallery.json containing only those actors, matched via the
|
||||
filtered gallery.h5 containing only those actors, matched via the
|
||||
"jellyfin_person_id" field recorded by make_jellyfin_gallery.py.
|
||||
|
||||
Usage:
|
||||
python scripts/filter_gallery.py \\
|
||||
--gallery gallery.json \\
|
||||
--gallery gallery.h5 \\
|
||||
--jellyfin-url http://jellyfin.local:8096 \\
|
||||
--api-key YOUR_API_KEY \\
|
||||
--item-id <jellyfin item id> \\
|
||||
--output gallery_movie.json
|
||||
--output gallery_movie.h5
|
||||
|
||||
# Or search by title:
|
||||
python scripts/filter_gallery.py \\
|
||||
--gallery gallery.json \\
|
||||
--gallery gallery.h5 \\
|
||||
--jellyfin-url http://jellyfin.local:8096 \\
|
||||
--api-key YOUR_API_KEY \\
|
||||
--title "The Matrix" \\
|
||||
--output gallery_movie.json
|
||||
--output gallery_movie.h5
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
@@ -42,6 +41,7 @@ from sae_jellyfin import ( # noqa: F401
|
||||
fetch_cast_person_ids,
|
||||
actor_jellyfin_id,
|
||||
)
|
||||
from sae_gallery import load_gallery_hdf5, save_gallery_hdf5
|
||||
|
||||
|
||||
def main():
|
||||
@@ -50,7 +50,7 @@ def main():
|
||||
formatter_class=argparse.RawDescriptionHelpFormatter,
|
||||
)
|
||||
parser.add_argument("--gallery", required=True,
|
||||
help="Global gallery.json built by make_jellyfin_gallery.py")
|
||||
help="Global gallery.h5 built by make_jellyfin_gallery.py")
|
||||
parser.add_argument("--jellyfin-url", required=True)
|
||||
parser.add_argument("--api-key", required=True)
|
||||
group = parser.add_mutually_exclusive_group(required=True)
|
||||
@@ -58,10 +58,10 @@ def main():
|
||||
group.add_argument("--title", help="Title to search for (uses first match)")
|
||||
parser.add_argument("--item-types", default="Movie,Series",
|
||||
help="Item types to search when using --title (default: Movie,Series)")
|
||||
parser.add_argument("--output", required=True, help="Output filtered gallery.json path")
|
||||
parser.add_argument("--output", required=True, help="Output filtered gallery.h5 path")
|
||||
args = parser.parse_args()
|
||||
|
||||
gallery = json.loads(Path(args.gallery).read_text())
|
||||
gallery = load_gallery_hdf5(Path(args.gallery))
|
||||
|
||||
item_id = args.item_id
|
||||
if item_id is None:
|
||||
@@ -77,7 +77,7 @@ def main():
|
||||
if missing > 0:
|
||||
print(f"[warn] {missing} cast member(s) not present in gallery (not yet embedded)", file=sys.stderr)
|
||||
|
||||
Path(args.output).write_text(json.dumps({"actors": actors}, indent=2) + "\n")
|
||||
save_gallery_hdf5({"actors": actors}, Path(args.output))
|
||||
print(f"Saved {len(actors)} actor(s) to {args.output}", file=sys.stderr)
|
||||
|
||||
|
||||
|
||||
@@ -1,9 +1,9 @@
|
||||
#!/usr/bin/env python3
|
||||
"""make_gallery.py — fetch actor images for a movie and build gallery.json.
|
||||
"""make_gallery.py — fetch actor images for a movie and build gallery.h5.
|
||||
|
||||
Fetches the cast from TMDB, downloads actor profile images, embeds them via
|
||||
the sae_embed module (SCRFD + ArcFace, same models as scene_analyze, loaded
|
||||
once), then writes gallery.json.
|
||||
once), then writes gallery.h5.
|
||||
|
||||
Requirements:
|
||||
pip install requests Pillow
|
||||
@@ -13,13 +13,13 @@ Usage:
|
||||
python scripts/make_gallery.py \\
|
||||
--tmdb-key YOUR_KEY \\
|
||||
--imdb-id tt0137523 \\
|
||||
--output gallery.json
|
||||
--output gallery.h5
|
||||
|
||||
# Or directly with a TMDB movie ID:
|
||||
python scripts/make_gallery.py \\
|
||||
--tmdb-key YOUR_KEY \\
|
||||
--movie-id 550 \\
|
||||
--output gallery.json
|
||||
--output gallery.h5
|
||||
|
||||
# Additional options:
|
||||
# --build-dir build/ build dir containing sae_embed module
|
||||
@@ -31,7 +31,6 @@ Get a free TMDB API key at: https://www.themoviedb.org/settings/api
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import sys
|
||||
import time
|
||||
from pathlib import Path
|
||||
@@ -151,7 +150,7 @@ def build_gallery(movie_id: int, key: str, embedder,
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(
|
||||
description="Fetch TMDB cast images and build gallery.json via sae_embed")
|
||||
description="Fetch TMDB cast images and build gallery.h5 via sae_embed")
|
||||
parser.add_argument("--tmdb-key", required=True,
|
||||
help="TMDB Bearer token (API Read Access Token from themoviedb.org/settings/api)")
|
||||
group = parser.add_mutually_exclusive_group(required=True)
|
||||
@@ -159,7 +158,7 @@ def main():
|
||||
help="IMDB movie ID, e.g. tt0137523 — looked up via TMDB automatically")
|
||||
group.add_argument("--movie-id", type=int,
|
||||
help="TMDB movie ID (alternative to --imdb-id)")
|
||||
parser.add_argument("--output", required=True, help="Output gallery.json path")
|
||||
parser.add_argument("--output", required=True, help="Output gallery.h5 path")
|
||||
parser.add_argument("--build-dir", default="build",
|
||||
help="Build directory containing the sae_embed module (default: build)")
|
||||
parser.add_argument("--models-dir", default="models",
|
||||
|
||||
@@ -1,10 +1,10 @@
|
||||
#!/usr/bin/env python3
|
||||
"""make_jellyfin_gallery.py — build a gallery.json spanning an entire Jellyfin library.
|
||||
"""make_jellyfin_gallery.py — build a gallery.h5 spanning an entire Jellyfin library.
|
||||
|
||||
Queries the Jellyfin API for every Movie/Series, collects the unique cast
|
||||
across the whole library, downloads each actor's headshot directly from
|
||||
Jellyfin (no TMDB key needed), embeds them with the sae_embed module (SCRFD +
|
||||
ArcFace, loaded once), and writes one global gallery.json.
|
||||
ArcFace, loaded once), and writes one global gallery.h5.
|
||||
|
||||
Because identity_matcher scores every detected face against the whole
|
||||
gallery, scene_analyze can then recognise any actor in your library in any
|
||||
@@ -20,21 +20,21 @@ Usage:
|
||||
python scripts/make_jellyfin_gallery.py \\
|
||||
--jellyfin-url http://jellyfin.local:8096 \\
|
||||
--api-key YOUR_API_KEY \\
|
||||
--output gallery.json
|
||||
--output gallery.h5
|
||||
|
||||
# Re-run later to pick up newly added titles without re-embedding
|
||||
# actors already in the gallery:
|
||||
python scripts/make_jellyfin_gallery.py \\
|
||||
--jellyfin-url http://jellyfin.local:8096 \\
|
||||
--api-key YOUR_API_KEY \\
|
||||
--output gallery.json --merge
|
||||
--output gallery.h5 --merge
|
||||
|
||||
# Fall back to TMDB profile images for actors with no usable Jellyfin image:
|
||||
python scripts/make_jellyfin_gallery.py \\
|
||||
--jellyfin-url http://jellyfin.local:8096 \\
|
||||
--api-key YOUR_API_KEY \\
|
||||
--tmdb-key YOUR_TMDB_KEY \\
|
||||
--output gallery.json
|
||||
--output gallery.h5
|
||||
|
||||
Get a Jellyfin API key from Dashboard → Advanced → API Keys.
|
||||
Get a free TMDB API key at: https://www.themoviedb.org/settings/api
|
||||
@@ -52,7 +52,8 @@ import requests
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parent))
|
||||
import sae_env # noqa: F401 — loads .env into os.environ on import
|
||||
from sae_embed_loader import load_embedder
|
||||
from sae_gallery import download_image, download_images, save_gallery, wikidata_image_urls
|
||||
from sae_gallery import (download_image, download_images, load_gallery_hdf5,
|
||||
save_gallery, wikidata_image_urls)
|
||||
from sae_jellyfin import actor_jellyfin_id, jf_get, normalize_jellyfin_url
|
||||
from sae_tmdb import (
|
||||
tmdb_person_by_name,
|
||||
@@ -323,7 +324,7 @@ def build_gallery(base_url: str, api_key: str, embedder, item_types: list[str],
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(
|
||||
description="Build a gallery.json spanning an entire Jellyfin library",
|
||||
description="Build a gallery.h5 spanning an entire Jellyfin library",
|
||||
formatter_class=argparse.RawDescriptionHelpFormatter,
|
||||
)
|
||||
parser.add_argument("--jellyfin-url", default=os.environ.get("JELLYFIN_URL"),
|
||||
@@ -333,7 +334,7 @@ def main():
|
||||
parser.add_argument("--api-key", default=os.environ.get("JELLYFIN_API_KEY"),
|
||||
required=not os.environ.get("JELLYFIN_API_KEY"),
|
||||
help="Jellyfin API key (Dashboard → Advanced → API Keys). Env: JELLYFIN_API_KEY")
|
||||
parser.add_argument("--output", required=True, help="Output gallery.json path")
|
||||
parser.add_argument("--output", required=True, help="Output gallery.h5 path")
|
||||
parser.add_argument("--item-types", default="Movie,Series",
|
||||
help="Comma-separated Jellyfin item types to scan (default: Movie,Series)")
|
||||
parser.add_argument("--build-dir", default="build",
|
||||
@@ -375,7 +376,7 @@ def main():
|
||||
|
||||
existing_actors = {}
|
||||
if args.merge and output.is_file():
|
||||
existing = json.loads(output.read_text())
|
||||
existing = load_gallery_hdf5(output)
|
||||
for actor in existing.get("actors", []):
|
||||
pid = actor_jellyfin_id(actor)
|
||||
if pid:
|
||||
|
||||
@@ -4,7 +4,7 @@ movienet_eval.py — embed probe crops and match against a gallery.
|
||||
|
||||
Usage:
|
||||
python scripts/movienet_eval.py \
|
||||
--gallery gallery_r50.json \
|
||||
--gallery gallery_r50.h5 \
|
||||
--arcface models/arcface_w600k_r50.onnx \
|
||||
--gt eval/gt.json \
|
||||
--output eval/predictions_r50.json \
|
||||
@@ -23,12 +23,12 @@ import numpy as np
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parent))
|
||||
from sae_embed_loader import load_embedder
|
||||
from sae_gallery import load_gallery_hdf5
|
||||
|
||||
|
||||
def load_gallery(path: str) -> dict[str, dict]:
|
||||
"""Return {imdb_id: {"name": str, "refs": np.ndarray[n_refs, dim]}}."""
|
||||
with open(path) as f:
|
||||
data = json.load(f)
|
||||
data = load_gallery_hdf5(Path(path))
|
||||
return {a["imdb_id"]: {"name": a["name"],
|
||||
"refs": np.asarray(a["embeddings"], dtype=np.float32)}
|
||||
for a in data["actors"]}
|
||||
|
||||
@@ -5,7 +5,7 @@ movienet_prep.py — extract probe crops from MovieNet-PS for actors in our gall
|
||||
Usage:
|
||||
python scripts/movienet_prep.py \
|
||||
--movienet <movienet_root> \
|
||||
--gallery gallery.json \
|
||||
--gallery gallery.h5 \
|
||||
--output eval/ \
|
||||
[--split Train_app10] \
|
||||
[--margin 0.2] \
|
||||
@@ -28,6 +28,9 @@ import zipfile
|
||||
from io import BytesIO
|
||||
from pathlib import Path
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parent))
|
||||
from sae_gallery import load_gallery_hdf5 # noqa: E402
|
||||
|
||||
try:
|
||||
import cv2
|
||||
import numpy as np
|
||||
@@ -74,8 +77,7 @@ def load_movienet_annotations(movienet_root: Path, split: str) -> list[dict]:
|
||||
|
||||
def load_gallery_ids(gallery_path: str) -> dict[str, str]:
|
||||
"""Return {imdb_id: actor_name} for all actors in the gallery."""
|
||||
with open(gallery_path) as f:
|
||||
data = json.load(f)
|
||||
data = load_gallery_hdf5(Path(gallery_path))
|
||||
return {a["imdb_id"]: a["name"] for a in data["actors"]}
|
||||
|
||||
|
||||
@@ -100,7 +102,7 @@ def crop_face(img: "np.ndarray", bbox: list[float], margin: float) -> "np.ndarra
|
||||
def main():
|
||||
p = argparse.ArgumentParser()
|
||||
p.add_argument("--movienet", required=True, help="MovieNet-PS root directory")
|
||||
p.add_argument("--gallery", required=True, help="gallery.json (for actor list)")
|
||||
p.add_argument("--gallery", required=True, help="gallery.h5 (for actor list)")
|
||||
p.add_argument("--output", default="eval", help="output directory")
|
||||
p.add_argument("--split", default="Train_app10",
|
||||
help="annotation split to use (default: Train_app10)")
|
||||
|
||||
@@ -0,0 +1,50 @@
|
||||
# Embedding-dump HDF5 schema (v1)
|
||||
|
||||
One file per analysed title. Captures the pipeline state at the `EmbeddedSceneFrame`
|
||||
channel — i.e. after decode → detect → align → embed, but **before** tracking and
|
||||
identity matching. Everything downstream (face tracker, identity matcher, scene
|
||||
tracker/anneal) is cheap CPU math, so replaying from this file lets a parameter
|
||||
sweep re-run the whole downstream tail thousands of times with no GPU and no video.
|
||||
|
||||
Written by the C++ dump sink (`--dump-embeddings out.h5`); read by
|
||||
`scripts/optimizer/replay.py`.
|
||||
|
||||
## Layout
|
||||
|
||||
The dump is **flat/ragged**: all faces across all frames are concatenated into
|
||||
per-face arrays, with a per-frame index table pointing into them. This avoids
|
||||
variable-length HDF5 types and reads straight into numpy.
|
||||
|
||||
```
|
||||
/ (root)
|
||||
attrs:
|
||||
schema_version : int = 1
|
||||
movie : str (source video path)
|
||||
sample_fps : float
|
||||
embed_dim : int = 512
|
||||
|
||||
frames/ group — one row per sampled frame
|
||||
timestamp_sec : float64 [F]
|
||||
frame_idx : int64 [F]
|
||||
is_cut : uint8 [F] (histogram intra-scene cut)
|
||||
is_scene_boundary : uint8 [F] (TransNetV2 boundary; 0 if scene_detect off)
|
||||
face_offset : int64 [F] start index into faces/* for this frame
|
||||
face_count : int32 [F] number of faces in this frame
|
||||
|
||||
faces/ group — one row per detected face, concatenated
|
||||
embedding : float32 [N, 512] L2-normalised ArcFace embedding
|
||||
bbox : float32 [N, 4] x, y, w, h in original video pixels
|
||||
landmarks : float32 [N, 10] 5 (x,y) pairs, SCRFD/ArcFace order
|
||||
confidence : float32 [N] detector confidence
|
||||
```
|
||||
|
||||
`F` = number of sampled frames, `N` = total faces (= sum of face_count).
|
||||
Frame *i*'s faces are `faces/*[ face_offset[i] : face_offset[i]+face_count[i] ]`.
|
||||
|
||||
## Invariants
|
||||
- `embedding` rows are unit-norm (cosine == dot product against the gallery).
|
||||
- `face_offset[0] == 0`; `face_offset[i+1] == face_offset[i] + face_count[i]`.
|
||||
- `bbox` is already mapped to original resolution (bbox_upscale applied at dump time),
|
||||
matching what the identity matcher would emit.
|
||||
- A frame with no faces has `face_count == 0` (still gets a row, so timestamps stay dense).
|
||||
- EOF sentinel frames are NOT written.
|
||||
@@ -0,0 +1,58 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
cast_restrict.py — produce a per-film gallery restricted to its credited cast.
|
||||
|
||||
Benchmark arm: instead of matching a face against the WHOLE gallery (2418 actors,
|
||||
risking cross-film misIDs like naming Archie Yates in a film he's not in), restrict
|
||||
the matcher's candidate set to the title's credited cast (from Jellyfin — the top
|
||||
~15 billed actors, exactly what run_from_jellyfin.py does in production).
|
||||
|
||||
Filters a gallery to actors whose jellyfin_id is in the film's cast set, writing a
|
||||
small gallery JSON the replay can load. Actors are kept if their jellyfin_id (or, as
|
||||
a fallback, normalized name) matches the cast.
|
||||
|
||||
Used by the full-vs-restricted bake-off. Cached per (gallery, film) so a DE sweep
|
||||
reuses the restricted gallery.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import sys
|
||||
import tempfile
|
||||
from pathlib import Path
|
||||
|
||||
REPO = Path(__file__).resolve().parent.parent.parent
|
||||
sys.path.insert(0, str(REPO / "scripts" / "validation"))
|
||||
from identity import norm_name # noqa: E402
|
||||
|
||||
_CACHE: dict = {}
|
||||
|
||||
|
||||
def restricted_gallery_path(gallery_path: str, cast_jellyfin_ids: set[str],
|
||||
cast_names: set[str] | None = None) -> str:
|
||||
"""Write (once, cached) a gallery filtered to the film's credited cast; return path.
|
||||
|
||||
Matches gallery actors to the cast by jellyfin_id first, then normalized name."""
|
||||
key = (gallery_path, frozenset(cast_jellyfin_ids))
|
||||
if key in _CACHE:
|
||||
return _CACHE[key]
|
||||
|
||||
gal = json.loads(Path(gallery_path).read_text())
|
||||
names = {norm_name(n) for n in (cast_names or set())}
|
||||
kept = []
|
||||
for a in gal["actors"]:
|
||||
jid = a.get("jellyfin_id", "")
|
||||
if (jid and jid in cast_jellyfin_ids) or (names and norm_name(a["name"]) in names):
|
||||
kept.append(a)
|
||||
|
||||
tf = tempfile.NamedTemporaryFile("w", suffix=".json", delete=False,
|
||||
prefix="castgal_")
|
||||
json.dump({"actors": kept}, tf)
|
||||
tf.close()
|
||||
_CACHE[key] = tf.name
|
||||
return tf.name
|
||||
|
||||
|
||||
def load_casts(casts_json: str) -> dict[str, list[str]]:
|
||||
"""film name → [jellyfin person id, ...] from jellyfin_casts.json."""
|
||||
return json.loads(Path(casts_json).read_text())
|
||||
@@ -0,0 +1,201 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
dump_error_frames.py — extract example video frames for visual inspection of a
|
||||
replayed prediction vs X-Ray ground truth: best-agreement seconds, FPI (false
|
||||
identification) seconds, and FN (missed cast) seconds.
|
||||
|
||||
Reuses second_score.py's per-second timeline/prediction loading, but keeps the
|
||||
per-second classification (score_seconds only returns aggregates) and picks
|
||||
representative timestamps in each bucket, then pulls single frames from the
|
||||
source video via ffmpeg -ss (nearest keyframe-independent seek + decode).
|
||||
|
||||
If --raw (the JSONL from `replay.py --raw-out`) is given, also draws each visible
|
||||
actor's bounding box + name/similarity on the extracted frame — green for
|
||||
identified, orange for unknown — matching debug_renderer_node.hpp's colour
|
||||
convention. Without --raw, frames are saved unannotated.
|
||||
|
||||
Usage:
|
||||
python scripts/optimizer/dump_error_frames.py \
|
||||
--pred pred.json --raw raw.jsonl \
|
||||
--xray experiments/xray/.../900_The_Many_Saints_Of_Newark \
|
||||
--movie "/mnt/movies/The Many Saints Of Newark (2021)/....mp4" \
|
||||
--gallery experiments/galleries/gallery_LVFace-B_Glint360K.h5 \
|
||||
--out-dir experiments/dump_review/many_saints --n-per-bucket 6
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import subprocess
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
import cv2
|
||||
|
||||
REPO = Path(__file__).resolve().parent.parent.parent
|
||||
sys.path.insert(0, str(REPO / "scripts" / "optimizer"))
|
||||
sys.path.insert(0, str(REPO / "scripts" / "validation"))
|
||||
|
||||
from second_score import load_second_timeline, load_pred_intervals, _match # noqa: E402
|
||||
from sample_eval import load_gallery_keys # noqa: E402
|
||||
from identity import keys_for # noqa: E402
|
||||
|
||||
|
||||
def per_second_detail(pred_json: dict, xray_dir: str, gallery_keys: set | None):
|
||||
"""Like second_score.score_seconds, but yields one record per sampled second
|
||||
instead of collapsing to aggregates."""
|
||||
timeline, film_cast, duration = load_second_timeline(xray_dir)
|
||||
pred = load_pred_intervals(pred_json)
|
||||
|
||||
name_by_keys = {}
|
||||
for a in pred_json.get("actors", []):
|
||||
k = frozenset(keys_for(imdb_id=a.get("imdb_id"), tmdb_id=a.get("tmdb_id"),
|
||||
jellyfin_id=a.get("jellyfin_id"), name=a.get("name")))
|
||||
name_by_keys[k] = a.get("name", "?")
|
||||
|
||||
records = []
|
||||
for t in sorted(timeline):
|
||||
G = [set(a) for a in timeline[t]]
|
||||
P_all = [(k, set(k)) for k, wins in pred if any(w0 <= t <= w1 for w0, w1 in wins)]
|
||||
if gallery_keys is not None:
|
||||
G = [g for g in G if g & gallery_keys]
|
||||
|
||||
P = [p for _, p in P_all]
|
||||
tp, matched = _match(P, G)
|
||||
fp_names, fn_names = [], []
|
||||
for key, pa in P_all:
|
||||
if not any(pa & ga for ga in G):
|
||||
fp_names.append(name_by_keys.get(key, "?"))
|
||||
for j, ga in enumerate(G):
|
||||
if not matched[j]:
|
||||
fn_names.append("|".join(sorted(x for x in ga if not x.startswith("imdb:") and not x.startswith("tmdb:"))) or "?")
|
||||
|
||||
union = tp + len(fp_names) + len(fn_names)
|
||||
jaccard = (tp / union) if union else 1.0
|
||||
records.append({"t": t, "tp": tp, "fp": fp_names, "fn": fn_names, "jaccard": jaccard})
|
||||
return records
|
||||
|
||||
|
||||
def pick_timestamps(records, n_per_bucket):
|
||||
best = sorted(records, key=lambda r: (-r["jaccard"], -r["tp"]))
|
||||
best = [r for r in best if r["tp"] > 0][:n_per_bucket]
|
||||
|
||||
fpi = [r for r in records if r["fp"]]
|
||||
fpi = sorted(fpi, key=lambda r: -len(r["fp"]))[:n_per_bucket]
|
||||
|
||||
fn = [r for r in records if r["fn"]]
|
||||
fn = sorted(fn, key=lambda r: -len(r["fn"]))[:n_per_bucket]
|
||||
|
||||
return {"best": best, "fpi": fpi, "fn": fn}
|
||||
|
||||
|
||||
def pick_by_interval(records, interval_sec):
|
||||
"""One best (highest jaccard) and one worst (lowest jaccard) second per
|
||||
interval_sec-second window across the whole film, e.g. --interval-sec 600 for
|
||||
a per-10-minute best/worst sweep. Windows with no sampled seconds are skipped
|
||||
(X-Ray timelines only cover scenes, so gaps between/after scenes are common)."""
|
||||
windows: dict[int, list] = {}
|
||||
for r in records:
|
||||
windows.setdefault(r["t"] // interval_sec, []).append(r)
|
||||
|
||||
buckets: dict[str, list] = {}
|
||||
for w in sorted(windows):
|
||||
wr = windows[w]
|
||||
best = max(wr, key=lambda r: (r["jaccard"], r["tp"]))
|
||||
worst = min(wr, key=lambda r: (r["jaccard"], -max(len(r["fp"]), len(r["fn"]))))
|
||||
buckets[f"w{w:03d}_best"] = [best]
|
||||
buckets[f"w{w:03d}_worst"] = [worst]
|
||||
return buckets
|
||||
|
||||
|
||||
def load_raw_annotations(raw_path: str):
|
||||
"""second (int, floor) -> list of visible_actors dicts (last frame wins if
|
||||
several fall in the same second, which is the common case at 1fps sampling)."""
|
||||
by_second = {}
|
||||
with open(raw_path) as f:
|
||||
for line in f:
|
||||
sa = json.loads(line)
|
||||
if sa.get("eof"):
|
||||
continue
|
||||
by_second[int(sa["timestamp_sec"])] = sa.get("visible_actors", [])
|
||||
return by_second
|
||||
|
||||
|
||||
def draw_annotations(frame_path: Path, actors: list):
|
||||
img = cv2.imread(str(frame_path))
|
||||
if img is None:
|
||||
return
|
||||
for a in actors:
|
||||
known = a.get("actor_idx", -1) >= 0
|
||||
colour = (60, 200, 0) if known else (220, 100, 0) # BGR: green / orange
|
||||
x, y, w, h = a["bbox"]
|
||||
x, y, w, h = int(x), int(y), int(w), int(h)
|
||||
cv2.rectangle(img, (x, y), (x + w, y + h), colour, 2)
|
||||
|
||||
label = f"{a['name']} {a['similarity']*100:.0f}%" if known else f"unknown {a['similarity']*100:.0f}%"
|
||||
(tw, th), baseline = cv2.getTextSize(label, cv2.FONT_HERSHEY_SIMPLEX, 0.5, 1)
|
||||
strip_y0 = max(0, y - th - 4)
|
||||
cv2.rectangle(img, (x, strip_y0), (x + tw + 4, y), colour, cv2.FILLED)
|
||||
cv2.putText(img, label, (x + 2, y - 2), cv2.FONT_HERSHEY_SIMPLEX, 0.5,
|
||||
(255, 255, 255), 1, cv2.LINE_AA)
|
||||
cv2.imwrite(str(frame_path), img)
|
||||
|
||||
|
||||
def extract_frame(movie: str, t: float, out_path: Path):
|
||||
out_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
subprocess.run(
|
||||
["ffmpeg", "-y", "-ss", str(t), "-i", movie, "-frames:v", "1",
|
||||
"-q:v", "2", str(out_path)],
|
||||
check=True, capture_output=True)
|
||||
|
||||
|
||||
def main():
|
||||
p = argparse.ArgumentParser(description=__doc__,
|
||||
formatter_class=argparse.RawDescriptionHelpFormatter)
|
||||
p.add_argument("--pred", required=True)
|
||||
p.add_argument("--raw", help="raw per-frame annotations JSONL (replay.py --raw-out); "
|
||||
"draws bboxes + names on extracted frames if given")
|
||||
p.add_argument("--xray", required=True)
|
||||
p.add_argument("--movie", required=True)
|
||||
p.add_argument("--gallery")
|
||||
p.add_argument("--out-dir", required=True)
|
||||
p.add_argument("--n-per-bucket", type=int, default=6)
|
||||
p.add_argument("--interval-sec", type=int,
|
||||
help="instead of global best/fpi/fn buckets, pick one best + one "
|
||||
"worst (by jaccard) second per interval-sec window across "
|
||||
"the whole film, e.g. 600 for per-10-minute best/worst")
|
||||
args = p.parse_args()
|
||||
|
||||
pred_json = json.loads(Path(args.pred).read_text())
|
||||
gk = load_gallery_keys(args.gallery) if args.gallery else None
|
||||
records = per_second_detail(pred_json, args.xray, gk)
|
||||
buckets = (pick_by_interval(records, args.interval_sec) if args.interval_sec
|
||||
else pick_timestamps(records, args.n_per_bucket))
|
||||
raw_by_second = load_raw_annotations(args.raw) if args.raw else None
|
||||
|
||||
out_dir = Path(args.out_dir)
|
||||
manifest = []
|
||||
for bucket, recs in buckets.items():
|
||||
for r in recs:
|
||||
fname = f"{bucket}_t{r['t']:05d}.jpg"
|
||||
out_path = out_dir / bucket / fname
|
||||
try:
|
||||
extract_frame(args.movie, r["t"], out_path)
|
||||
ok = True
|
||||
if raw_by_second is not None:
|
||||
draw_annotations(out_path, raw_by_second.get(r["t"], []))
|
||||
except subprocess.CalledProcessError as e:
|
||||
ok = False
|
||||
print(f"[dump_error_frames] ffmpeg failed at t={r['t']}: {e}", file=sys.stderr)
|
||||
manifest.append({"bucket": bucket, "t": r["t"], "tp": r["tp"],
|
||||
"fp": r["fp"], "fn": r["fn"], "jaccard": round(r["jaccard"], 3),
|
||||
"file": str(out_path.relative_to(out_dir)) if ok else None})
|
||||
print(f"[{bucket}] t={r['t']}s tp={r['tp']} fp={r['fp']} fn={r['fn']}", file=sys.stderr)
|
||||
|
||||
(out_dir / "manifest.json").write_text(json.dumps(manifest, indent=2, ensure_ascii=False))
|
||||
print(f"[dump_error_frames] wrote {len(manifest)} frames + manifest.json to {out_dir}",
|
||||
file=sys.stderr)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,383 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
dump_scene_montage.py — one BEST and one WORST frame per X-Ray scene, split into
|
||||
onscreen vs. offscreen actor identification (TPI / FPI / FN).
|
||||
|
||||
For each X-Ray scene (scenes.csv span), scores every sampled second by a simple
|
||||
per-second Jaccard agreement (TPI / (TPI+FPI+FN), same spirit as second_score.py)
|
||||
and picks the single best-agreement and single worst-agreement second. Each gets
|
||||
one output frame: the full frame (not a face crop) with a solid box drawn for
|
||||
every currently-active TPI/FPI actor who has a REAL detection backing them, plus a
|
||||
black caption panel below with two columns — Onscreen (has a real detection) and
|
||||
Offscreen (no real detection: FN misses, and "ghost" detections where the tracker
|
||||
is re-emitting a frozen last-known bbox with nothing there — see
|
||||
docs/rep4-optimizer-results.md) — names colour-coded by bucket, with a legend.
|
||||
|
||||
A predicted bbox is checked against the dump's OWN raw per-frame face detections
|
||||
(IoU) to tell a real detection from a ghost. Ghosts are NEVER drawn as boxes (they
|
||||
have no real screen position); they only appear as a name in the Offscreen column.
|
||||
|
||||
Frames where at least one FPI name isn't in the film's cast AT ALL (an out-of-cast
|
||||
misID, not just a right-actor/wrong-scene timing slip) are also copied into
|
||||
<out-dir>/out_of_cast_fpi/ for quick review of the most confident wrong answers.
|
||||
|
||||
Requires the raw per-frame annotations from `replay.py --raw-out` (bboxes aren't
|
||||
in the merged pred.json) and the film's HDF5 dump (for ghost-checking against real
|
||||
detections).
|
||||
|
||||
Usage:
|
||||
python scripts/optimizer/dump_scene_montage.py \
|
||||
--raw raw.jsonl --dump experiments/dumps/.../dump_X.h5 \
|
||||
--xray experiments/xray/.../900_The_Many_Saints_Of_Newark \
|
||||
--movie "/mnt/movies/.../X.mp4" \
|
||||
--gallery experiments/galleries/gallery_LVFace-B_Glint360K.h5 \
|
||||
--out-dir experiments/results/holdout/montage/many_saints --scene 5
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import csv
|
||||
import json
|
||||
import subprocess
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
import cv2
|
||||
import h5py
|
||||
import numpy as np
|
||||
|
||||
REPO = Path(__file__).resolve().parent.parent.parent
|
||||
sys.path.insert(0, str(REPO / "scripts" / "optimizer"))
|
||||
sys.path.insert(0, str(REPO / "scripts" / "validation"))
|
||||
|
||||
from sample_eval import load_gallery_keys # noqa: E402
|
||||
from identity import keys_for # noqa: E402
|
||||
|
||||
COLOUR_TPI = (60, 200, 0) # green, BGR
|
||||
COLOUR_FPI = (0, 60, 220) # red, BGR
|
||||
CAPTION_H = 28 # px per line in the bottom strip
|
||||
|
||||
|
||||
def load_scene_spans(xray_dir: str):
|
||||
"""scene_id -> (t0_sec, t1_sec), from scenes.csv (ms)."""
|
||||
spans = {}
|
||||
with open(Path(xray_dir) / "scenes.csv", newline="", encoding="utf-8") as f:
|
||||
for r in csv.DictReader(f):
|
||||
sn = (r.get("scene") or "").strip()
|
||||
try:
|
||||
spans[sn] = (float(r["start"]) / 1000.0, float(r["end"]) / 1000.0)
|
||||
except (KeyError, ValueError):
|
||||
continue
|
||||
return spans
|
||||
|
||||
|
||||
def load_film_cast(xray_dir: str) -> set:
|
||||
"""Every actor key X-Ray credits ANYWHERE in the film — used to tell an
|
||||
out-of-cast misID (named someone who isn't even in this film) apart from an
|
||||
in-cast timing slip (right actor, wrong scene), same distinction as
|
||||
second_score.py's FPI_misid vs FPI_incast."""
|
||||
keys = set()
|
||||
with open(Path(xray_dir) / "people.csv", newline="", encoding="utf-8") as f:
|
||||
for r in csv.DictReader(f):
|
||||
nm = (r.get("name_id") or "").strip()
|
||||
person = (r.get("person") or "").strip()
|
||||
if nm or person:
|
||||
keys |= keys_for(imdb_id=nm, name=person)
|
||||
return keys
|
||||
|
||||
|
||||
def load_scene_cast(xray_dir: str):
|
||||
"""scene_id -> set of actor key-frozensets X-Ray lists as present."""
|
||||
id_to_name = {}
|
||||
with open(Path(xray_dir) / "people.csv", newline="", encoding="utf-8") as f:
|
||||
for r in csv.DictReader(f):
|
||||
nm = (r.get("name_id") or "").strip()
|
||||
if nm:
|
||||
id_to_name[nm] = (r.get("person") or "").strip()
|
||||
|
||||
scene_cast: dict[str, set] = {}
|
||||
with open(Path(xray_dir) / "people_in_scenes.csv", newline="", encoding="utf-8") as f:
|
||||
for r in csv.DictReader(f):
|
||||
sn = (r.get("scene") or "").strip()
|
||||
nm = (r.get("name_id") or "").strip()
|
||||
if sn and nm:
|
||||
scene_cast.setdefault(sn, set()).add(
|
||||
frozenset(keys_for(imdb_id=nm, name=id_to_name.get(nm))))
|
||||
return scene_cast
|
||||
|
||||
|
||||
def load_raw_by_second(raw_path: str):
|
||||
by_second: dict[int, list] = {}
|
||||
with open(raw_path) as f:
|
||||
for line in f:
|
||||
sa = json.loads(line)
|
||||
if sa.get("eof"):
|
||||
continue
|
||||
by_second[int(sa["timestamp_sec"])] = sa.get("visible_actors", [])
|
||||
return by_second
|
||||
|
||||
|
||||
def load_dump_faces_by_second(dump_path: str):
|
||||
"""second (int) -> list of raw detected bboxes (x,y,w,h), for ghost-checking.
|
||||
A predicted actor's bbox is real iff it overlaps one of these; a bbox with no
|
||||
overlap at all is a frozen/stale re-emission, not an actual detection."""
|
||||
by_second: dict[int, list] = {}
|
||||
with h5py.File(dump_path, "r") as f:
|
||||
ts = f["frames/timestamp_sec"][:]
|
||||
off = f["frames/face_offset"][:]
|
||||
cnt = f["frames/face_count"][:]
|
||||
bbox = f["faces/bbox"][:]
|
||||
for i in range(len(ts)):
|
||||
s, n = int(off[i]), int(cnt[i])
|
||||
by_second[int(ts[i])] = [tuple(b) for b in bbox[s:s + n]]
|
||||
return by_second
|
||||
|
||||
|
||||
def iou(a, b):
|
||||
ax, ay, aw, ah = a
|
||||
bx, by, bw, bh = b
|
||||
ix0, iy0 = max(ax, bx), max(ay, by)
|
||||
ix1, iy1 = min(ax + aw, bx + bw), min(ay + ah, by + bh)
|
||||
iw, ih = max(0.0, ix1 - ix0), max(0.0, iy1 - iy0)
|
||||
inter = iw * ih
|
||||
union = aw * ah + bw * bh - inter
|
||||
return inter / union if union > 0 else 0.0
|
||||
|
||||
|
||||
def is_ghost(bbox, real_boxes, iou_thresh=0.3):
|
||||
return not any(iou(bbox, rb) >= iou_thresh for rb in real_boxes)
|
||||
|
||||
|
||||
def actor_key(a: dict) -> frozenset:
|
||||
return frozenset(keys_for(imdb_id=a.get("imdb_id"), tmdb_id=a.get("tmdb_id"),
|
||||
jellyfin_id=a.get("jellyfin_id"), name=a.get("name")))
|
||||
|
||||
|
||||
COLOUR_FN = (220, 130, 0) # blue, BGR
|
||||
LEGEND = (("TPI (correct)", COLOUR_TPI), ("FPI (wrong)", COLOUR_FPI),
|
||||
("FN (missed)", COLOUR_FN))
|
||||
|
||||
|
||||
def render_frame(frame_path: Path, t: int, tpi_boxes: list, fpi_boxes: list, entries: list):
|
||||
"""entries: list of (name, bucket, onscreen) — bucket in {tpi,fpi,fn},
|
||||
onscreen=True iff a real detected face backs this name at this second. Ghost
|
||||
detections (bucket fpi/tpi but no real face — see is_ghost) are never drawn as
|
||||
boxes: they have no real screen position, they only ever appear in the
|
||||
Offscreen column."""
|
||||
img = cv2.imread(str(frame_path))
|
||||
if img is None:
|
||||
return None
|
||||
|
||||
for name, bbox, sim in tpi_boxes:
|
||||
x, y, w, h = (int(v) for v in bbox)
|
||||
cv2.rectangle(img, (x, y), (x + w, y + h), COLOUR_TPI, 2)
|
||||
_label(img, (x, y), f"{name} {sim*100:.0f}%", COLOUR_TPI)
|
||||
for name, bbox, sim in fpi_boxes:
|
||||
x, y, w, h = (int(v) for v in bbox)
|
||||
cv2.rectangle(img, (x, y), (x + w, y + h), COLOUR_FPI, 2)
|
||||
_label(img, (x, y), f"{name} {sim*100:.0f}%", COLOUR_FPI)
|
||||
|
||||
h_img, w_img = img.shape[:2]
|
||||
bucket_colour = {"tpi": COLOUR_TPI, "fpi": COLOUR_FPI, "fn": COLOUR_FN}
|
||||
onscreen = [(n, bucket_colour[b]) for n, b, on in entries if on]
|
||||
offscreen = [(n, bucket_colour[b]) for n, b, on in entries if not on]
|
||||
|
||||
n_rows = max(len(onscreen), len(offscreen), 1)
|
||||
header_h = 24
|
||||
legend_h = CAPTION_H
|
||||
table_h = header_h + n_rows * CAPTION_H + legend_h + 16
|
||||
canvas = np.zeros((h_img + table_h, w_img, 3), dtype=np.uint8) # black bg
|
||||
canvas[:h_img] = img
|
||||
|
||||
col_x = (8, w_img // 2 + 8)
|
||||
cv2.putText(canvas, f"t={t}s", (8, 16), cv2.FONT_HERSHEY_SIMPLEX, 0.5,
|
||||
(255, 255, 255), 1, cv2.LINE_AA)
|
||||
y0 = h_img + header_h
|
||||
cv2.putText(canvas, "Onscreen", (col_x[0], y0), cv2.FONT_HERSHEY_SIMPLEX, 0.55,
|
||||
(255, 255, 255), 1, cv2.LINE_AA)
|
||||
cv2.putText(canvas, "Offscreen", (col_x[1], y0), cv2.FONT_HERSHEY_SIMPLEX, 0.55,
|
||||
(255, 255, 255), 1, cv2.LINE_AA)
|
||||
cv2.line(canvas, (col_x[1] - 8, h_img), (col_x[1] - 8, h_img + table_h),
|
||||
(90, 90, 90), 1)
|
||||
|
||||
for i in range(n_rows):
|
||||
y = y0 + CAPTION_H * (i + 1)
|
||||
if i < len(onscreen):
|
||||
name, colour = onscreen[i]
|
||||
cv2.putText(canvas, name, (col_x[0], y), cv2.FONT_HERSHEY_SIMPLEX, 0.5,
|
||||
colour, 1, cv2.LINE_AA)
|
||||
if i < len(offscreen):
|
||||
name, colour = offscreen[i]
|
||||
cv2.putText(canvas, name, (col_x[1], y), cv2.FONT_HERSHEY_SIMPLEX, 0.5,
|
||||
colour, 1, cv2.LINE_AA)
|
||||
|
||||
ly = y0 + CAPTION_H * (n_rows + 1) + 4
|
||||
lx = 8
|
||||
for label, colour in LEGEND:
|
||||
(tw, _), _ = cv2.getTextSize(label, cv2.FONT_HERSHEY_SIMPLEX, 0.42, 1)
|
||||
cv2.rectangle(canvas, (lx, ly - 10), (lx + 12, ly + 2), colour, cv2.FILLED)
|
||||
cv2.putText(canvas, label, (lx + 18, ly), cv2.FONT_HERSHEY_SIMPLEX, 0.42,
|
||||
(200, 200, 200), 1, cv2.LINE_AA)
|
||||
lx += tw + 40
|
||||
return canvas
|
||||
|
||||
|
||||
def _label(img, pt, text, colour):
|
||||
x, y = pt
|
||||
(tw, th), _ = cv2.getTextSize(text, cv2.FONT_HERSHEY_SIMPLEX, 0.5, 1)
|
||||
strip_y0 = max(0, y - th - 4)
|
||||
cv2.rectangle(img, (x, strip_y0), (x + tw + 4, y), colour, cv2.FILLED)
|
||||
cv2.putText(img, text, (x + 2, y - 2), cv2.FONT_HERSHEY_SIMPLEX, 0.5,
|
||||
(255, 255, 255), 1, cv2.LINE_AA)
|
||||
|
||||
|
||||
def extract_frame(movie: str, t: float, out_path: Path):
|
||||
out_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
subprocess.run(
|
||||
["ffmpeg", "-y", "-ss", str(t), "-i", movie, "-frames:v", "1",
|
||||
"-q:v", "2", str(out_path)],
|
||||
check=True, capture_output=True)
|
||||
|
||||
|
||||
def classify_second(t: int, gt_cast: set, film_cast: set, raw_by_second: dict,
|
||||
dump_faces_by_second: dict):
|
||||
"""One second's TPI/FPI/FN classification: (score, tpi_boxes, fpi_boxes,
|
||||
entries, has_outofcast). score = Jaccard-style agreement in [0,1], used to
|
||||
rank seconds for best/worst picking."""
|
||||
actors = raw_by_second.get(t, [])
|
||||
real_boxes = dump_faces_by_second.get(t, [])
|
||||
|
||||
tpi_boxes, fpi_boxes = [], []
|
||||
entries = [] # (name, bucket, onscreen)
|
||||
cur_state: dict[frozenset, str] = {}
|
||||
has_outofcast = False
|
||||
|
||||
for a in actors:
|
||||
if a.get("actor_idx", -1) < 0:
|
||||
continue
|
||||
key = actor_key(a)
|
||||
name = a.get("name", "?")
|
||||
bbox = tuple(a["bbox"])
|
||||
sim = a.get("similarity", 0.0)
|
||||
ghost = is_ghost(bbox, real_boxes)
|
||||
hit = any(key & g for g in gt_cast)
|
||||
|
||||
if ghost:
|
||||
cur_state[key] = "ghost"
|
||||
entries.append((name, "fpi" if not hit else "tpi", False))
|
||||
elif hit:
|
||||
tpi_boxes.append((name, bbox, sim))
|
||||
cur_state[key] = "tpi"
|
||||
entries.append((name, "tpi", True))
|
||||
else:
|
||||
fpi_boxes.append((name, bbox, sim))
|
||||
cur_state[key] = "fpi"
|
||||
entries.append((name, "fpi", True))
|
||||
|
||||
if not hit and not (key & film_cast):
|
||||
has_outofcast = True # named someone not in the film at all (FPI_misid)
|
||||
|
||||
tpi_keys = [k for k, state in cur_state.items() if state in ("tpi", "ghost")]
|
||||
fn_count = 0
|
||||
for g in gt_cast:
|
||||
if any(g & k for k in tpi_keys):
|
||||
continue
|
||||
nm = next((x.split("name:", 1)[1] for x in g if x.startswith("name:")), None)
|
||||
entries.append((nm or next(iter(g), "?"), "fn", False))
|
||||
fn_count += 1
|
||||
|
||||
tp = sum(1 for _, b, on in entries if b == "tpi" and on)
|
||||
fp = sum(1 for _, b, on in entries if b == "fpi")
|
||||
union = tp + fp + fn_count
|
||||
score = tp / union if union else 1.0 # both-empty = perfect agreement
|
||||
return score, tpi_boxes, fpi_boxes, entries, has_outofcast
|
||||
|
||||
|
||||
def process_scene(scene_id: str, t0: float, t1: float, gt_cast: set, film_cast: set,
|
||||
raw_by_second: dict, dump_faces_by_second: dict,
|
||||
gallery_keys: set | None, movie: str, out_dir: Path,
|
||||
outofcast_dir: Path):
|
||||
if gallery_keys is not None:
|
||||
gt_cast = {g for g in gt_cast if g & gallery_keys}
|
||||
|
||||
per_second = {}
|
||||
for t in range(int(t0), int(t1)):
|
||||
per_second[t] = classify_second(t, gt_cast, film_cast, raw_by_second,
|
||||
dump_faces_by_second)
|
||||
if not per_second:
|
||||
return []
|
||||
|
||||
best_t = max(per_second, key=lambda t: per_second[t][0])
|
||||
worst_t = min(per_second, key=lambda t: per_second[t][0])
|
||||
|
||||
manifest = []
|
||||
for label, t in (("best", best_t), ("worst", worst_t)):
|
||||
score, tpi_boxes, fpi_boxes, entries, has_outofcast = per_second[t]
|
||||
fname = f"{scene_id}_{label}_t{t:06d}.jpg"
|
||||
out_path = out_dir / fname
|
||||
try:
|
||||
extract_frame(movie, t, out_path)
|
||||
canvas = render_frame(out_path, t, tpi_boxes, fpi_boxes, entries)
|
||||
if canvas is not None:
|
||||
cv2.imwrite(str(out_path), canvas)
|
||||
manifest.append({"label": label, "t": t, "score": round(score, 3),
|
||||
"entries": entries, "file": fname,
|
||||
"outofcast": has_outofcast})
|
||||
print(f"[scene {scene_id}] {label} t={t}s score={score:.2f} "
|
||||
f"entries={entries}", file=sys.stderr)
|
||||
if has_outofcast:
|
||||
outofcast_dir.mkdir(parents=True, exist_ok=True)
|
||||
cv2.imwrite(str(outofcast_dir / fname), cv2.imread(str(out_path)))
|
||||
except subprocess.CalledProcessError as e:
|
||||
print(f"[dump_scene_montage] ffmpeg failed at t={t}: {e}", file=sys.stderr)
|
||||
|
||||
return manifest
|
||||
|
||||
|
||||
def main():
|
||||
p = argparse.ArgumentParser(description=__doc__,
|
||||
formatter_class=argparse.RawDescriptionHelpFormatter)
|
||||
p.add_argument("--raw", required=True, help="raw per-frame annotations (replay.py --raw-out)")
|
||||
p.add_argument("--dump", required=True, help="film's HDF5 embedding dump (for ghost-checking)")
|
||||
p.add_argument("--xray", required=True)
|
||||
p.add_argument("--movie", required=True)
|
||||
p.add_argument("--gallery")
|
||||
p.add_argument("--out-dir", required=True)
|
||||
p.add_argument("--scene", help="only process this X-Ray scene id (default: all)")
|
||||
args = p.parse_args()
|
||||
|
||||
spans = load_scene_spans(args.xray)
|
||||
scene_cast = load_scene_cast(args.xray)
|
||||
film_cast = load_film_cast(args.xray)
|
||||
raw_by_second = load_raw_by_second(args.raw)
|
||||
dump_faces_by_second = load_dump_faces_by_second(args.dump)
|
||||
gk = load_gallery_keys(args.gallery) if args.gallery else None
|
||||
|
||||
out_dir = Path(args.out_dir)
|
||||
out_dir.mkdir(parents=True, exist_ok=True)
|
||||
outofcast_dir = out_dir / "out_of_cast_fpi"
|
||||
|
||||
scene_ids = [args.scene] if args.scene else sorted(spans, key=lambda s: spans[s][0])
|
||||
all_manifest = {}
|
||||
for sn in scene_ids:
|
||||
if sn not in spans:
|
||||
print(f"[dump_scene_montage] unknown scene id: {sn}", file=sys.stderr)
|
||||
continue
|
||||
t0, t1 = spans[sn]
|
||||
gt_cast = scene_cast.get(sn, set())
|
||||
scene_dir = out_dir / f"scene_{sn}"
|
||||
scene_dir.mkdir(parents=True, exist_ok=True)
|
||||
m = process_scene(sn, t0, t1, gt_cast, film_cast, raw_by_second,
|
||||
dump_faces_by_second, gk, args.movie, scene_dir, outofcast_dir)
|
||||
all_manifest[sn] = m
|
||||
|
||||
(out_dir / "manifest.json").write_text(json.dumps(all_manifest, indent=2, ensure_ascii=False))
|
||||
total = sum(len(v) for v in all_manifest.values())
|
||||
n_outofcast = sum(1 for v in all_manifest.values() for r in v if r.get("outofcast"))
|
||||
print(f"[dump_scene_montage] wrote {total} best/worst frames across "
|
||||
f"{len(all_manifest)} scenes to {out_dir} "
|
||||
f"({n_outofcast} copied to {outofcast_dir})", file=sys.stderr)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,154 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
fetch_missing_actors.py — close the gallery coverage gap.
|
||||
|
||||
X-Ray credits ~67% of each film's cast that our gallery never had a reference
|
||||
embedding for, making those actors unrecoverable FNs no threshold can fix. This
|
||||
fetches images for those missing actors (by IMDb nm id → TMDB profile photos),
|
||||
embeds them with the SAME SCRFD+ArcFace models (sae_embed), and writes gallery
|
||||
entries. Merge the result into the baseline to make those actors recognisable.
|
||||
|
||||
nm → TMDB person → /person/{id}/images profile photos → download → embed.
|
||||
|
||||
Usage:
|
||||
python scripts/optimizer/fetch_missing_actors.py \
|
||||
--missing missing_actors.json \
|
||||
--out gallery_missing.json \
|
||||
[--images-per-actor 3] [--build-dir build]
|
||||
# TMDB_API_KEY from env/.env
|
||||
|
||||
Then merge:
|
||||
python scripts/optimizer/fetch_missing_actors.py --merge \
|
||||
gallery_arcface_w600k_r50.json gallery_missing.json \
|
||||
--out gallery_augmented.json
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
import tempfile
|
||||
from pathlib import Path
|
||||
|
||||
REPO = Path(__file__).resolve().parent.parent.parent
|
||||
sys.path.insert(0, str(REPO / "scripts"))
|
||||
import sae_env # noqa: E402 loads .env
|
||||
from sae_tmdb import tmdb_get, tmdb_person_for_imdb, TMDB_IMG # noqa: E402
|
||||
from sae_gallery import download_images, wikidata_image_urls # noqa: E402
|
||||
from sae_embed_loader import load_embedder # noqa: E402
|
||||
|
||||
|
||||
def profile_urls_for_imdb(imdb_id: str, token: str, n: int) -> tuple[str | None, list[str]]:
|
||||
"""(tmdb_person_id, [image_url,...]) via /find then /person/{id}/images."""
|
||||
data = tmdb_get(f"/find/{imdb_id}", token, external_source="imdb_id")
|
||||
people = data.get("person_results", [])
|
||||
if not people:
|
||||
return None, []
|
||||
pid = str(people[0]["id"])
|
||||
imgs = tmdb_get(f"/person/{pid}/images", token)
|
||||
profiles = imgs.get("profiles", [])[:n]
|
||||
return pid, [TMDB_IMG + p["file_path"] for p in profiles if p.get("file_path")]
|
||||
|
||||
|
||||
def fetch(missing_path, out_path, token, build_dir, models_dir, arcface,
|
||||
images_per_actor, use_wikidata=False):
|
||||
missing = json.loads(Path(missing_path).read_text())
|
||||
src = "TMDB + Wikidata fallback" if use_wikidata else "TMDB"
|
||||
print(f"[fetch] {len(missing)} missing actors to resolve via {src}", file=sys.stderr)
|
||||
embedder = load_embedder(build_dir, models_dir, arcface)
|
||||
|
||||
img_root = Path(tempfile.mkdtemp(prefix="missing_gallery_"))
|
||||
actors = []
|
||||
n_resolved = n_no_tmdb = n_no_img = n_no_face = 0
|
||||
n_via_wikidata = 0
|
||||
|
||||
for i, m in enumerate(missing, 1):
|
||||
nm, name = m["imdb_id"], m.get("name", "")
|
||||
tmdb_id, urls = None, []
|
||||
try:
|
||||
tmdb_id, urls = profile_urls_for_imdb(nm, token, images_per_actor)
|
||||
except Exception as e:
|
||||
print(f" [{i}] {name}: TMDB error {e}", file=sys.stderr)
|
||||
# Wikidata fallback: keyed cleanly by IMDb nm (P345→P18 Commons photo),
|
||||
# recovers on-camera character actors TMDB's film-centric DB misses.
|
||||
if (not urls) and use_wikidata:
|
||||
wiki_urls = wikidata_image_urls(nm)[:images_per_actor]
|
||||
if wiki_urls:
|
||||
urls = wiki_urls
|
||||
n_via_wikidata += 1
|
||||
if not urls:
|
||||
if tmdb_id is None:
|
||||
n_no_tmdb += 1
|
||||
else:
|
||||
n_no_img += 1
|
||||
continue
|
||||
dest = img_root / nm
|
||||
dest.mkdir(parents=True, exist_ok=True)
|
||||
paths = download_images(urls, dest, images_per_actor)
|
||||
embeddings = []
|
||||
for p in paths:
|
||||
res = embedder.embed(str(p))
|
||||
if res.ok:
|
||||
embeddings.append(list(res.embedding))
|
||||
if not embeddings:
|
||||
n_no_face += 1
|
||||
continue
|
||||
actors.append({"imdb_id": nm, "tmdb_id": str(tmdb_id) if tmdb_id else "",
|
||||
"jellyfin_id": "", "name": name,
|
||||
"embeddings": embeddings, "source_images": []})
|
||||
n_resolved += 1
|
||||
if i % 20 == 0 or i == len(missing):
|
||||
print(f" [{i}/{len(missing)}] resolved={n_resolved} "
|
||||
f"(wiki={n_via_wikidata}) no_tmdb={n_no_tmdb} no_img={n_no_img} "
|
||||
f"no_face={n_no_face}", file=sys.stderr)
|
||||
|
||||
Path(out_path).write_text(json.dumps({"actors": actors}, indent=2))
|
||||
n_emb = sum(len(a["embeddings"]) for a in actors)
|
||||
print(f"\n[fetch] recovered {n_resolved}/{len(missing)} actors "
|
||||
f"({n_via_wikidata} via Wikidata), {n_emb} embeddings → {out_path}",
|
||||
file=sys.stderr)
|
||||
print(f"[fetch] unrecoverable: no_tmdb={n_no_tmdb} no_img={n_no_img} "
|
||||
f"no_face={n_no_face}", file=sys.stderr)
|
||||
|
||||
|
||||
def merge(base_path, add_path, out_path):
|
||||
base = json.loads(Path(base_path).read_text())
|
||||
add = json.loads(Path(add_path).read_text())
|
||||
have = {a.get("imdb_id") for a in base["actors"] if a.get("imdb_id")}
|
||||
added = [a for a in add["actors"] if a.get("imdb_id") not in have]
|
||||
base["actors"].extend(added)
|
||||
Path(out_path).write_text(json.dumps(base, indent=2))
|
||||
print(f"[merge] {len(base['actors'])-len(added)} + {len(added)} = "
|
||||
f"{len(base['actors'])} actors → {out_path}", file=sys.stderr)
|
||||
|
||||
|
||||
def main():
|
||||
p = argparse.ArgumentParser(description=__doc__,
|
||||
formatter_class=argparse.RawDescriptionHelpFormatter)
|
||||
p.add_argument("--merge", nargs=2, metavar=("BASE", "ADD"),
|
||||
help="merge ADD gallery into BASE → --out")
|
||||
p.add_argument("--missing")
|
||||
p.add_argument("--out", required=True)
|
||||
p.add_argument("--tmdb-key", default=os.environ.get("TMDB_API_KEY"))
|
||||
p.add_argument("--build-dir", default=str(REPO / "build"))
|
||||
p.add_argument("--models-dir", default=str(REPO / "models"))
|
||||
p.add_argument("--arcface", default=None)
|
||||
p.add_argument("--images-per-actor", type=int, default=3)
|
||||
p.add_argument("--wikidata", action="store_true",
|
||||
help="fall back to Wikidata (P345→P18 Commons photo) when TMDB has no image")
|
||||
args = p.parse_args()
|
||||
|
||||
if args.merge:
|
||||
merge(args.merge[0], args.merge[1], args.out)
|
||||
return
|
||||
if not args.missing:
|
||||
sys.exit("--missing required (or use --merge)")
|
||||
if not args.tmdb_key:
|
||||
sys.exit("no TMDB key — set TMDB_API_KEY")
|
||||
fetch(args.missing, args.out, args.tmdb_key, args.build_dir, args.models_dir,
|
||||
args.arcface, args.images_per_actor, use_wikidata=args.wikidata)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,94 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
gallery_membership.py — definitive per-film gallery coverage of X-Ray cast.
|
||||
|
||||
For each film, splits the X-Ray cast (people.csv) into those WITH a gallery reference
|
||||
embedding and those WITHOUT. This is the model-independent foundation for honest
|
||||
FP/FN rates: because every model's gallery is built from the SAME TMDB source images
|
||||
(same actors), the membership list is identical across models — only the embedding
|
||||
values differ. So FN can be measured over the recognisable denominator (in-gallery
|
||||
cast) and out-of-cast misIDs (predicted actor not in the film at all) are well defined.
|
||||
|
||||
Outputs experiments/results/membership.json:
|
||||
{ film: {
|
||||
xray_cast: N, in_gallery: M, coverage: M/N,
|
||||
in_gallery_names: [...], missing_names: [...] } }
|
||||
|
||||
Usage:
|
||||
python scripts/optimizer/gallery_membership.py \
|
||||
--manifest experiments/manifests/films.json \
|
||||
--gallery gallery_arcface_w600k_r50.json \
|
||||
--out experiments/results/membership.json
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import csv
|
||||
import json
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
REPO = Path(__file__).resolve().parent.parent.parent
|
||||
sys.path.insert(0, str(REPO / "scripts" / "validation"))
|
||||
from identity import keys_for # noqa: E402
|
||||
|
||||
|
||||
def gallery_keyset(gallery_path: str) -> set:
|
||||
keys = set()
|
||||
for a in json.loads(Path(gallery_path).read_text())["actors"]:
|
||||
if not a.get("embeddings"):
|
||||
continue # no embedding = not actually recognisable
|
||||
keys |= keys_for(imdb_id=a.get("imdb_id"), tmdb_id=a.get("tmdb_id"),
|
||||
jellyfin_id=a.get("jellyfin_id"), name=a.get("name"))
|
||||
return keys
|
||||
|
||||
|
||||
def film_cast(xray_dir: str) -> dict[str, str]:
|
||||
"""nm_id → person name from a film's X-Ray people.csv."""
|
||||
out = {}
|
||||
with open(Path(xray_dir) / "people.csv", newline="", encoding="utf-8") as f:
|
||||
for r in csv.DictReader(f):
|
||||
nm = (r.get("name_id") or "").strip()
|
||||
if nm:
|
||||
out[nm] = (r.get("person") or "").strip()
|
||||
return out
|
||||
|
||||
|
||||
def main():
|
||||
p = argparse.ArgumentParser(description=__doc__,
|
||||
formatter_class=argparse.RawDescriptionHelpFormatter)
|
||||
p.add_argument("--manifest", required=True)
|
||||
p.add_argument("--gallery", required=True)
|
||||
p.add_argument("--out", required=True)
|
||||
args = p.parse_args()
|
||||
|
||||
gkeys = gallery_keyset(args.gallery)
|
||||
films = json.loads(Path(args.manifest).read_text())
|
||||
|
||||
report = {}
|
||||
tot_cast = tot_in = 0
|
||||
print(f"{'film':32s} {'cast':>5s} {'in-gal':>7s} {'cover':>6s}")
|
||||
for f in films:
|
||||
cast = film_cast(f["xray"])
|
||||
in_g, miss = [], []
|
||||
for nm, name in cast.items():
|
||||
if keys_for(imdb_id=nm, name=name) & gkeys:
|
||||
in_g.append(name)
|
||||
else:
|
||||
miss.append(name)
|
||||
n, m = len(cast), len(in_g)
|
||||
tot_cast += n; tot_in += m
|
||||
report[f["name"]] = {"xray_cast": n, "in_gallery": m,
|
||||
"coverage": round(m / n, 3) if n else 0.0,
|
||||
"in_gallery_names": sorted(in_g),
|
||||
"missing_names": sorted(miss)}
|
||||
print(f"{f['name'][:32]:32s} {n:>5d} {m:>7d} {m/n*100 if n else 0:>5.0f}%")
|
||||
print(f"{'TOTAL':32s} {tot_cast:>5d} {tot_in:>7d} {tot_in/tot_cast*100:>5.0f}%")
|
||||
|
||||
Path(args.out).parent.mkdir(parents=True, exist_ok=True)
|
||||
Path(args.out).write_text(json.dumps(report, indent=2))
|
||||
print(f"\n→ {args.out}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,240 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
optimize.py — Differential Evolution over pipeline thresholds, scored against X-Ray.
|
||||
|
||||
Replaces the coarse grid sweep with scipy's differential_evolution over the
|
||||
continuous knob space. Each candidate config is a full-9-film replay (real KPN
|
||||
nodes) scored against Amazon X-Ray presence, micro-averaged. The gallery is loaded
|
||||
once per process (binding caches by path), so an evaluation is just N cheap replays.
|
||||
|
||||
Objective: **maximize micro-F1** (DE minimizes, so we return -F1). NOTE: X-Ray recall
|
||||
is a face-vs-cast-in-scene ceiling (see [[xray-validation-results]]), so unconstrained
|
||||
F1 tends to push prob_threshold DOWN to recover unreachable recall — trading real
|
||||
precision for it. We therefore log precision/recall at every evaluation and print
|
||||
them at the optimum so the trade-off is visible and you can pick another operating
|
||||
point from the trajectory (--trajectory).
|
||||
|
||||
Usage:
|
||||
python scripts/optimizer/optimize.py --manifest films.json \
|
||||
--gallery gallery_arcface_w600k_r50.json \
|
||||
--params prob_threshold:0.5:0.999 anneal_sec:1:30 extinction_sec:1:15 \
|
||||
--popsize 20 --maxiter 25 --trajectory traj.json
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import sys
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
import numpy as np
|
||||
from scipy.optimize import differential_evolution
|
||||
|
||||
REPO = Path(__file__).resolve().parent.parent.parent
|
||||
sys.path.insert(0, str(REPO / "scripts" / "optimizer"))
|
||||
sys.path.insert(0, str(REPO / "scripts" / "validation"))
|
||||
|
||||
import json as _json
|
||||
import os
|
||||
import subprocess
|
||||
import tempfile
|
||||
import threading
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
|
||||
# Concurrent per-eval replays. Each replay is an isolated subprocess, so parallelism
|
||||
# is deadlock-safe; with 9 films/eval, 8 workers replays nearly all at once. Tune via
|
||||
# REPLAY_WORKERS (8 is the measured sweet spot on this 24GB GPU).
|
||||
REPLAY_WORKERS = int(os.environ.get("REPLAY_WORKERS", "8"))
|
||||
|
||||
# DE-level parallelism: how many population candidates get evaluated concurrently
|
||||
# (each spawning its own REPLAY_WORKERS film subprocesses). Total concurrent GPU
|
||||
# replay processes ≈ DE_WORKERS × min(REPLAY_WORKERS, n_films). Threads, not
|
||||
# multiprocessing — each objective() call just waits on subprocess.run, so threads
|
||||
# share the GIL fine and avoid pickling the objective/gallery-key cache.
|
||||
DE_WORKERS = int(os.environ.get("DE_WORKERS", "1"))
|
||||
|
||||
from second_score import score_seconds # noqa: E402 uniform per-second TPI/FPI scoring
|
||||
from sample_eval import load_gallery_keys # noqa: E402
|
||||
|
||||
_GAL_KEYS: dict = {} # gallery path → key set (fair-recall FN mask), loaded once
|
||||
_REPLAY_TIMEOUT = 45 # seconds per film; a wedged replay is killed, not left to hang
|
||||
|
||||
REPLAY_CLI = str(Path(__file__).resolve().parent / "replay.py")
|
||||
|
||||
|
||||
def _gallery_keys(path):
|
||||
if path not in _GAL_KEYS:
|
||||
_GAL_KEYS[path] = load_gallery_keys(path)
|
||||
return _GAL_KEYS[path]
|
||||
|
||||
|
||||
def _replay_subprocess(dump, gallery, cfg, build_dir):
|
||||
"""Run one replay in a SUBPROCESS with a timeout, returning its presence JSON.
|
||||
|
||||
In-process replay intermittently DEADLOCKS at network teardown — a KPN worker
|
||||
stuck mid-rocBLAS GEMM inside the ROCm driver makes ~PyNode's jthread.join() hang
|
||||
forever (root-caused via gdb, 2026-07-15). Isolating each replay means a wedged
|
||||
GPU thread only kills that subprocess; the sweep continues. Returns None on
|
||||
timeout/failure (the caller drops that film from the average)."""
|
||||
with tempfile.NamedTemporaryFile("r", suffix=".json", delete=False) as tf:
|
||||
out = tf.name
|
||||
argv = [sys.executable, REPLAY_CLI, "--dump", dump, "--gallery", gallery,
|
||||
"--out", out, "--build-dir", build_dir]
|
||||
for k, v in cfg.items():
|
||||
if isinstance(v, bool): # store_true flags: pass the flag, not a value
|
||||
if v:
|
||||
argv.append(f"--{k.replace('_', '-')}")
|
||||
else:
|
||||
argv += [f"--{k.replace('_', '-')}", str(v)]
|
||||
try:
|
||||
subprocess.run(argv, timeout=_REPLAY_TIMEOUT, capture_output=True, check=True)
|
||||
return _json.loads(Path(out).read_text())
|
||||
except (subprocess.TimeoutExpired, subprocess.CalledProcessError,
|
||||
FileNotFoundError, ValueError) as e:
|
||||
print(f"[opt] replay failed for {Path(dump).name}: {type(e).__name__}",
|
||||
file=sys.stderr)
|
||||
return None
|
||||
finally:
|
||||
try:
|
||||
Path(out).unlink()
|
||||
except OSError:
|
||||
pass
|
||||
|
||||
|
||||
def evaluate(cfg, films, build_dir, step=None):
|
||||
"""Objective = MACRO-mean over films of each film's duration-weighted per-scene F1.
|
||||
|
||||
Each film's replay runs in a subprocess (timeout-guarded) to survive the
|
||||
intermittent ROCm teardown deadlock. A film whose replay times out is dropped
|
||||
from the average rather than hanging the whole sweep.
|
||||
|
||||
UNIFORM PER-SECOND scoring (second_score.py): every second of the film is sampled;
|
||||
GT(t) = the cast of the X-Ray scene containing t, Pred(t) = actors whose presence
|
||||
window covers t. Counts instances — TPI / FPI / FN — with FPI weighted 10× when the
|
||||
named actor isn't in the film's cast at all (a real misID vs a timing slip). FN
|
||||
counts only gallery-known actors (fair recall). Reports agreement_rate = mean
|
||||
per-second Jaccard (the "% of on-screen actors we agree with X-Ray about, over
|
||||
time"). Objective = macro-mean across films of the per-second weighted F1.
|
||||
|
||||
expand_gallery: controlled by env SAE_EXPAND (default on). Set SAE_EXPAND=0 to run
|
||||
the no-expansion arm — the overnight matrix tests both to quantify what expansion buys.
|
||||
|
||||
The 9 films' replays run CONCURRENTLY (REPLAY_WORKERS) — each is an isolated
|
||||
subprocess, so parallelism is safe (a wedged one only kills itself)."""
|
||||
if os.environ.get("SAE_EXPAND", "1") == "1":
|
||||
cfg = {**cfg, "expand_gallery": True}
|
||||
|
||||
def _one(film):
|
||||
pj = _replay_subprocess(film["dump"], film.get("gallery"), cfg, build_dir)
|
||||
if pj is None:
|
||||
return None
|
||||
return score_seconds(pj, film["xray"],
|
||||
gallery_keys=_gallery_keys(film.get("gallery")))
|
||||
|
||||
with ThreadPoolExecutor(max_workers=REPLAY_WORKERS) as ex:
|
||||
per_film = [m for m in ex.map(_one, films) if m is not None]
|
||||
n = len(per_film)
|
||||
if not n:
|
||||
return {"precision": 0.0, "recall": 0.0, "f1": 0.0, "agreement": 0.0,
|
||||
"TPI": 0, "FPI": 0, "FPI_misid": 0, "FN": 0}
|
||||
return {"precision": sum(m["precision"] for m in per_film) / n,
|
||||
"recall": sum(m["recall"] for m in per_film) / n,
|
||||
"f1": sum(m["f1"] for m in per_film) / n,
|
||||
"agreement": sum(m["agreement_rate"] for m in per_film) / n,
|
||||
"TPI": sum(m["TPI"] for m in per_film),
|
||||
"FPI": sum(m["FPI"] for m in per_film),
|
||||
"FPI_misid": sum(m["FPI_misid"] for m in per_film),
|
||||
"FN": sum(m["FN"] for m in per_film)}
|
||||
|
||||
|
||||
def main():
|
||||
p = argparse.ArgumentParser(description=__doc__,
|
||||
formatter_class=argparse.RawDescriptionHelpFormatter)
|
||||
p.add_argument("--manifest", required=True)
|
||||
p.add_argument("--gallery", help="default gallery if not per-film")
|
||||
p.add_argument("--params", nargs="+", required=True,
|
||||
help="knob:lo:hi (e.g. prob_threshold:0.5:0.999). Int knobs kept float, rounded in cfg.")
|
||||
p.add_argument("--build-dir", default=str(REPO / "build"))
|
||||
p.add_argument("--step", type=float, default=5.0)
|
||||
p.add_argument("--popsize", type=int, default=20)
|
||||
p.add_argument("--maxiter", type=int, default=25)
|
||||
p.add_argument("--seed", type=int, default=0)
|
||||
p.add_argument("--trajectory", help="write every evaluation here (JSON lines)")
|
||||
p.add_argument("--out", help="write best config + metrics")
|
||||
args = p.parse_args()
|
||||
|
||||
films = json.loads(Path(args.manifest).read_text())
|
||||
for f in films:
|
||||
f.setdefault("gallery", args.gallery)
|
||||
if not Path(f["dump"]).exists():
|
||||
sys.exit(f"[opt] missing dump for {f['name']}: {f['dump']}")
|
||||
|
||||
names, bounds = [], []
|
||||
int_knobs = {"track_max_frames_missing", "cut_inactive_max_frames"}
|
||||
for spec in args.params:
|
||||
k, lo, hi = spec.split(":")
|
||||
names.append(k); bounds.append((float(lo), float(hi)))
|
||||
print(f"[opt] DE over {names} bounds={bounds}", file=sys.stderr)
|
||||
print(f"[opt] {len(films)} films, popsize={args.popsize}, maxiter={args.maxiter}", file=sys.stderr)
|
||||
|
||||
traj = []
|
||||
evals = [0]
|
||||
t0 = time.time()
|
||||
traj_lock = threading.Lock()
|
||||
|
||||
def vec_to_cfg(x):
|
||||
cfg = {}
|
||||
for k, v in zip(names, x):
|
||||
cfg[k] = int(round(v)) if k in int_knobs else float(v)
|
||||
return cfg
|
||||
|
||||
def objective(x):
|
||||
cfg = vec_to_cfg(x)
|
||||
m = evaluate(cfg, films, args.build_dir, args.step)
|
||||
with traj_lock:
|
||||
evals[0] += 1
|
||||
rec = {"eval": evals[0], "config": cfg, **m, "t": round(time.time() - t0, 1)}
|
||||
traj.append(rec)
|
||||
print(f"[opt] eval {evals[0]:3d} thr={cfg['prob_threshold']:.2f} "
|
||||
f"ann={cfg['anneal_sec']:.0f} ext={cfg['extinction_sec']:.1f} → "
|
||||
f"F1={m['f1']*100:.1f}% P={m['precision']*100:.1f}% R={m['recall']*100:.1f}% "
|
||||
f"agree={m.get('agreement', 0)*100:.1f}% misID={m.get('FPI_misid', 0)}",
|
||||
file=sys.stderr)
|
||||
if args.trajectory:
|
||||
with open(args.trajectory, "a") as tf:
|
||||
tf.write(json.dumps(rec) + "\n")
|
||||
return -m["f1"]
|
||||
|
||||
de_kwargs = dict(
|
||||
popsize=args.popsize, maxiter=args.maxiter,
|
||||
seed=args.seed, polish=False, tol=1e-4, mutation=(0.5, 1.0), recombination=0.7,
|
||||
init="sobol")
|
||||
if DE_WORKERS > 1:
|
||||
pool = ThreadPoolExecutor(max_workers=DE_WORKERS)
|
||||
de_kwargs["workers"] = pool.map
|
||||
result = differential_evolution(objective, bounds, **de_kwargs)
|
||||
|
||||
best_cfg = vec_to_cfg(result.x)
|
||||
best = evaluate(best_cfg, films, args.build_dir, args.step)
|
||||
print("\n══ DE optimum (by F1) ═══════════════════════════")
|
||||
print(f" config : {best_cfg}")
|
||||
print(f" F1 : {best['f1']*100:.2f}%")
|
||||
print(f" precision: {best['precision']*100:.2f}% recall: {best['recall']*100:.2f}%")
|
||||
print(f" TP/FP/FN: {best['TP']}/{best['FP']}/{best['FN']}")
|
||||
print(f" evaluations: {evals[0]} time: {time.time()-t0:.0f}s")
|
||||
|
||||
# Also surface the highest-precision config seen (the ship-safe operating point).
|
||||
if traj:
|
||||
hp = max(traj, key=lambda r: (r["precision"], r["recall"]))
|
||||
print("\n── highest-precision config seen (ship-safe) ──")
|
||||
print(f" config : {hp['config']}")
|
||||
print(f" P={hp['precision']*100:.2f}% R={hp['recall']*100:.2f}% F1={hp['f1']*100:.2f}%")
|
||||
|
||||
if args.out:
|
||||
Path(args.out).write_text(json.dumps(
|
||||
{"best_by_f1": {"config": best_cfg, **best}, "n_evals": evals[0]}, indent=2))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,94 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
reembed_gallery.py — re-embed an existing gallery's actors with a different model.
|
||||
|
||||
For the embedding-model bake-off: take a reference gallery (with all actor ids +
|
||||
source_images) and produce a new gallery where every actor's embeddings are computed
|
||||
by a DIFFERENT ArcFace/LVFace model from the SAME cached source images. All identity
|
||||
keys (imdb/tmdb/jellyfin/name) are preserved, so membership/matching is unchanged —
|
||||
only the embedding vectors (and hence the model's similarity space) differ.
|
||||
|
||||
Source images live in `--images <root>/<jellyfin_id>_<Name>/NN.jpg` (the gallery build
|
||||
cache). Actors are matched to their image dir by jellyfin_id first, then name.
|
||||
|
||||
Usage:
|
||||
python scripts/optimizer/reembed_gallery.py \
|
||||
--ref gallery_arcface_w600k_r50.h5 \
|
||||
--images images \
|
||||
--arcface models/arcface_r18.onnx \
|
||||
--out experiments/galleries/gallery_arcface_r18.h5 \
|
||||
[--build-dir build]
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
REPO = Path(__file__).resolve().parent.parent.parent
|
||||
sys.path.insert(0, str(REPO / "scripts"))
|
||||
from sae_embed_loader import load_embedder # noqa: E402
|
||||
from sae_gallery import load_gallery_hdf5, save_gallery_hdf5 # noqa: E402
|
||||
|
||||
|
||||
def find_dir(images_root: Path, jellyfin_id: str, name: str) -> Path | None:
|
||||
if jellyfin_id:
|
||||
d = images_root / f"{jellyfin_id}_{name.replace(' ', '_')}"
|
||||
if d.is_dir():
|
||||
return d
|
||||
# jellyfin_id prefix match (name spelling may differ)
|
||||
hits = list(images_root.glob(f"{jellyfin_id}_*"))
|
||||
if hits:
|
||||
return hits[0]
|
||||
hits = list(images_root.glob(f"*_{name.replace(' ', '_')}"))
|
||||
return hits[0] if hits else None
|
||||
|
||||
|
||||
def main():
|
||||
p = argparse.ArgumentParser(description=__doc__,
|
||||
formatter_class=argparse.RawDescriptionHelpFormatter)
|
||||
p.add_argument("--ref", required=True, help="reference gallery.h5 (ids + source imgs)")
|
||||
p.add_argument("--images", required=True, help="image cache root")
|
||||
p.add_argument("--arcface", required=True, help="model ONNX to re-embed with")
|
||||
p.add_argument("--out", required=True)
|
||||
p.add_argument("--build-dir", default=str(REPO / "build"))
|
||||
p.add_argument("--models-dir", default=str(REPO / "models"))
|
||||
args = p.parse_args()
|
||||
|
||||
ref = load_gallery_hdf5(Path(args.ref))
|
||||
images_root = Path(args.images)
|
||||
embedder = load_embedder(args.build_dir, args.models_dir, args.arcface)
|
||||
|
||||
out_actors = []
|
||||
n_ok = n_nodir = n_noemb = 0
|
||||
total = len(ref["actors"])
|
||||
for i, a in enumerate(ref["actors"], 1):
|
||||
d = find_dir(images_root, a.get("jellyfin_id", ""), a["name"])
|
||||
if d is None:
|
||||
n_nodir += 1
|
||||
continue
|
||||
embeddings = []
|
||||
for img in sorted(d.glob("*.jpg")):
|
||||
res = embedder.embed(str(img))
|
||||
if res.ok:
|
||||
embeddings.append(list(res.embedding))
|
||||
if not embeddings:
|
||||
n_noemb += 1
|
||||
continue
|
||||
out_actors.append({"imdb_id": a.get("imdb_id", ""), "tmdb_id": a.get("tmdb_id", ""),
|
||||
"jellyfin_id": a.get("jellyfin_id", ""), "name": a["name"],
|
||||
"embeddings": embeddings,
|
||||
"source_images": [p.name for p in sorted(d.glob("*.jpg"))]})
|
||||
n_ok += 1
|
||||
if i % 200 == 0 or i == total:
|
||||
print(f" [{i}/{total}] ok={n_ok} no_dir={n_nodir} no_emb={n_noemb}",
|
||||
file=sys.stderr)
|
||||
|
||||
save_gallery_hdf5({"actors": out_actors}, Path(args.out))
|
||||
n_emb = sum(len(a["embeddings"]) for a in out_actors)
|
||||
print(f"[reembed] {Path(args.arcface).stem}: {n_ok}/{total} actors, {n_emb} embeddings "
|
||||
f"→ {args.out}", file=sys.stderr)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,240 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
replay.py — replay a dumped embedding HDF5 through the real KPN downstream nodes.
|
||||
|
||||
Reads an embedding dump (scripts/optimizer/SCHEMA.md), feeds each frame as an
|
||||
EmbeddedSceneFrame into a Python-assembled KPN network wiring the *real* C++
|
||||
face_tracker → identity_matcher → scene_tracker, and returns the same presence-window
|
||||
JSON that scene_analyze's result_sink produces (minimal schema). No decode, no GPU
|
||||
embedding — only the cheap downstream tail runs, so a sweep can vary Config knobs
|
||||
freely. See [[kpn-python-replay-optimizer]].
|
||||
|
||||
CLI:
|
||||
python scripts/optimizer/replay.py --dump film.h5 --gallery gallery.json \
|
||||
--out replayed.json [--prob-threshold 0.99] [--anneal 10] ...
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import sys
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
import h5py
|
||||
import numpy as np
|
||||
|
||||
REPO = Path(__file__).resolve().parent.parent.parent
|
||||
|
||||
|
||||
def load_frames(dump_path: str, min_conf: float = 0.0):
|
||||
"""Yield EmbeddedSceneFrame dicts from the HDF5 dump, then a trailing EOF.
|
||||
|
||||
`min_conf` drops detections below that detector confidence before they reach the
|
||||
matcher — an UPWARD-only detector_conf sweep on already-dumped faces (the dump was
|
||||
made at detector_conf=0.5, so 0.5 is the floor). Lets us test whether near-threshold
|
||||
detections are real faces (raising min_conf hurts recall) or phantoms (it helps
|
||||
precision at no recall cost)."""
|
||||
with h5py.File(dump_path, "r") as f:
|
||||
ts = f["frames/timestamp_sec"][:]
|
||||
fidx = f["frames/frame_idx"][:]
|
||||
cut = f["frames/is_cut"][:]
|
||||
off = f["frames/face_offset"][:]
|
||||
cnt = f["frames/face_count"][:]
|
||||
emb = f["faces/embedding"][:]
|
||||
bbox = f["faces/bbox"][:]
|
||||
lmk = f["faces/landmarks"][:]
|
||||
conf = f["faces/confidence"][:]
|
||||
movie = f.attrs.get("movie", "")
|
||||
fps = float(f.attrs.get("sample_fps", 1.0))
|
||||
|
||||
frames = []
|
||||
for i in range(len(ts)):
|
||||
s, n = int(off[i]), int(cnt[i])
|
||||
keep = slice(s, s + n)
|
||||
c = np.ascontiguousarray(conf[keep], dtype=np.float32)
|
||||
if min_conf > 0.0 and n:
|
||||
m = c >= min_conf
|
||||
sel = np.where(m)[0]
|
||||
frames.append({
|
||||
"timestamp_sec": float(ts[i]), "frame_idx": int(fidx[i]),
|
||||
"is_cut": bool(cut[i]), "eof": False,
|
||||
"bbox": np.ascontiguousarray(bbox[keep][sel], dtype=np.float32),
|
||||
"landmarks": np.ascontiguousarray(lmk[keep][sel], dtype=np.float32),
|
||||
"confidence": np.ascontiguousarray(c[sel], dtype=np.float32),
|
||||
"embeddings": np.ascontiguousarray(emb[keep][sel], dtype=np.float32),
|
||||
})
|
||||
else:
|
||||
frames.append({
|
||||
"timestamp_sec": float(ts[i]), "frame_idx": int(fidx[i]),
|
||||
"is_cut": bool(cut[i]), "eof": False,
|
||||
"bbox": np.ascontiguousarray(bbox[keep], dtype=np.float32),
|
||||
"landmarks": np.ascontiguousarray(lmk[keep], dtype=np.float32),
|
||||
"confidence": c,
|
||||
"embeddings": np.ascontiguousarray(emb[keep], dtype=np.float32),
|
||||
})
|
||||
last_ts = float(ts[-1]) if len(ts) else 0.0
|
||||
frames.append({"timestamp_sec": last_ts, "eof": True})
|
||||
return frames, str(movie), fps
|
||||
|
||||
|
||||
def replay(dump_path: str, gallery: str, cfg: dict, build_dir: str, stop: bool = True,
|
||||
raw_out: str | None = None) -> dict:
|
||||
"""Run the dump through the real KPN chain; return minimal-schema presence JSON.
|
||||
|
||||
cfg may include "detector_conf" to prune dumped detections below that confidence
|
||||
(upward-only from the 0.5 dump floor) before matching.
|
||||
|
||||
raw_out: if set, also write the raw per-frame annotations (timestamp, actor_idx,
|
||||
name, bbox, similarity — one entry per input frame, before merging into windows)
|
||||
as JSON lines to this path. Needed to draw bounding boxes on extracted frames;
|
||||
the merged window schema returned by this function has no per-frame bbox."""
|
||||
sys.path.insert(0, build_dir)
|
||||
import sae_kpn
|
||||
|
||||
frames, movie, fps = load_frames(dump_path, min_conf=float(cfg.get("detector_conf", 0.0)))
|
||||
|
||||
net = sae_kpn.Network()
|
||||
sae_kpn._register_types(net)
|
||||
|
||||
idx = [0]
|
||||
eof = {"timestamp_sec": frames[-1]["timestamp_sec"], "eof": True}
|
||||
|
||||
def source():
|
||||
# A no-input source node's run_loop calls this in a tight loop. Once frames
|
||||
# are exhausted we must NOT hot-spin returning EOF — that pegs a core and
|
||||
# floods the downstream channel with EOFs (livelock that wedged DE). Sleep
|
||||
# briefly after the single real EOF so net.stop() can tear the thread down.
|
||||
i = idx[0]
|
||||
idx[0] += 1
|
||||
if i < len(frames):
|
||||
return frames[i]
|
||||
time.sleep(0.05)
|
||||
return eof
|
||||
|
||||
# Channel capacity must exceed the frame count so the fast source can't overflow
|
||||
# a downstream FIFO before the serial reader drains it — PyNode DROPS on overflow,
|
||||
# which would silently truncate the replay. Size to the whole film + slack.
|
||||
# Every channel gets capacity ≥ the whole film so NOTHING can ever overflow-drop:
|
||||
# the source can push all frames before any downstream node has drained, and a
|
||||
# dropped frame silently corrupts the score. Memory is cheap (a few k pointers);
|
||||
# correctness is not. Generous slack on top.
|
||||
cap = len(frames) * 2 + 64
|
||||
sae_kpn.add_node_python(net, "replay", source, [], ["EmbeddedSceneFrame"], cap)
|
||||
sae_kpn.add_face_tracker(net, "tracker", cfg, cap)
|
||||
sae_kpn.add_identity_matcher(net, "matcher", gallery, cfg, cap)
|
||||
sae_kpn.add_scene_tracker(net, "scene", cfg, cap)
|
||||
net.connect("replay", 0, "tracker", 0)
|
||||
net.connect("tracker", 0, "matcher", 0)
|
||||
net.connect("matcher", 0, "scene", 0)
|
||||
net.build()
|
||||
net.start()
|
||||
|
||||
# Read exactly one annotation per input frame. The source emits EOF as an ordinary
|
||||
# value AFTER the last frame, but the concurrent pipeline lets that EOF OVERTAKE
|
||||
# the last few real frames still flowing tracker→matcher→scene. Breaking on the
|
||||
# first eof therefore dropped a random tail (~0.5–1%, race-dependent). Instead we
|
||||
# keep reading past eof until we've collected all n_frames annotations (or hit a
|
||||
# run of consecutive eofs meaning the pipeline is genuinely drained).
|
||||
n_expected = len(frames) - 1 # excludes the trailing eof frame
|
||||
annotations = []
|
||||
eof_streak = 0
|
||||
max_reads = n_expected * 2 + 32
|
||||
for _ in range(max_reads):
|
||||
sa = net.read("scene", 0)
|
||||
if sa.get("eof"):
|
||||
eof_streak += 1
|
||||
# stragglers can still arrive after an eof; only stop once we've either
|
||||
# got everything or seen several eofs in a row (truly drained).
|
||||
if len(annotations) >= n_expected or eof_streak >= 8:
|
||||
break
|
||||
continue
|
||||
eof_streak = 0
|
||||
annotations.append(sa)
|
||||
if len(annotations) >= n_expected:
|
||||
break
|
||||
|
||||
if raw_out:
|
||||
with open(raw_out, "w") as f:
|
||||
for sa in annotations:
|
||||
f.write(json.dumps(sa) + "\n")
|
||||
|
||||
result = build_minimal(annotations, movie, fps, cfg)
|
||||
if stop:
|
||||
net.stop()
|
||||
return result
|
||||
|
||||
|
||||
def build_minimal(annotations, movie, fps, cfg) -> dict:
|
||||
"""Reproduce result_sink's minimal schema: per-actor annealed [start,end] windows.
|
||||
|
||||
Mirrors ResultSinkFunc::build_actor_windows — merge each actor's detection
|
||||
timestamps into windows, bridging gaps shorter than anneal_sec.
|
||||
"""
|
||||
anneal = float(cfg.get("anneal_sec", 10.0))
|
||||
info = {} # actor_idx -> identity fields
|
||||
times = {} # actor_idx -> [timestamps]
|
||||
for sa in annotations:
|
||||
for a in sa["visible_actors"]:
|
||||
if a["actor_idx"] < 0:
|
||||
continue
|
||||
info[a["actor_idx"]] = a
|
||||
times.setdefault(a["actor_idx"], []).append(sa["timestamp_sec"])
|
||||
|
||||
actors = []
|
||||
for idx, ts in times.items():
|
||||
ts.sort()
|
||||
scenes = []
|
||||
ws = we = ts[0]
|
||||
for t in ts[1:]:
|
||||
if t - we > anneal:
|
||||
scenes.append([ws, we])
|
||||
ws = t
|
||||
we = t
|
||||
scenes.append([ws, we])
|
||||
a = info[idx]
|
||||
actors.append({
|
||||
"name": a["name"], "imdb_id": a["imdb_id"], "tmdb_id": a["tmdb_id"],
|
||||
"jellyfin_id": a["jellyfin_id"], "scenes": scenes,
|
||||
})
|
||||
|
||||
return {"schema_version": 1, "movie": movie, "sample_fps": fps,
|
||||
"anneal_sec": anneal, "actors": actors}
|
||||
|
||||
|
||||
CFG_KEYS = ["detector_conf", "prob_threshold", "match_prior", "match_threshold", "match_ratio",
|
||||
"match_ratio_ceil", "track_alpha", "track_min_iou", "track_max_embed_dist",
|
||||
"track_max_frames_missing", "cut_revive_sim", "cut_inactive_max_frames",
|
||||
"extinction_sec", "anneal_sec"]
|
||||
|
||||
|
||||
def main():
|
||||
p = argparse.ArgumentParser(description=__doc__,
|
||||
formatter_class=argparse.RawDescriptionHelpFormatter)
|
||||
p.add_argument("--dump", required=True, help="embedding HDF5 dump")
|
||||
p.add_argument("--gallery", required=True)
|
||||
p.add_argument("--out", required=True, help="output presence JSON")
|
||||
p.add_argument("--raw-out", help="also write raw per-frame annotations (JSONL, with bboxes) here")
|
||||
p.add_argument("--build-dir", default=str(REPO / "build"))
|
||||
for k in CFG_KEYS:
|
||||
p.add_argument(f"--{k.replace('_','-')}", type=float, default=None)
|
||||
# per-film gallery expansion: promotes pose-varied views of confidently-identified
|
||||
# actors into an in-memory annex, recovering ~+4 recall at no precision cost.
|
||||
p.add_argument("--expand-gallery", action="store_true")
|
||||
args = p.parse_args()
|
||||
|
||||
cfg = {k: getattr(args, k) for k in CFG_KEYS if getattr(args, k) is not None}
|
||||
if args.expand_gallery:
|
||||
cfg["expand_gallery"] = True
|
||||
# stop=True: PyNode::stop() sets stop_flag_ before joining, so the source
|
||||
# thread's run_loop actually exits. stop=False skips that, leaving stop_flag_
|
||||
# false forever — the PyNode destructor's jthread.join() then blocks forever
|
||||
# (verified via gdb: stuck in the source node's run_loop, not the GEMM path).
|
||||
result = replay(args.dump, args.gallery, cfg, args.build_dir, stop=True,
|
||||
raw_out=args.raw_out)
|
||||
Path(args.out).write_text(json.dumps(result, indent=2))
|
||||
print(f"[replay] {len(result['actors'])} actors → {args.out}", file=sys.stderr)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,197 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
second_score.py — uniform per-second agreement with X-Ray.
|
||||
|
||||
Unlike scene_score.py (which unions our detections over a whole X-Ray scene), this
|
||||
samples EVERY SECOND of the film and asks: at second t, do we name the same actors
|
||||
X-Ray says are on screen?
|
||||
|
||||
GT(t) = the cast set of the X-Ray scene containing t (scenes.csv + people_in_scenes)
|
||||
Pred(t) = actors whose presence window [start,end] covers t (the pipeline's output)
|
||||
|
||||
Per second we count instances:
|
||||
TPI = |Pred ∩ GT| true positive instances
|
||||
FPI = |Pred − GT| false positive instances, split into:
|
||||
FPI_misid — actor NOT in the film's cast at all (a real misID, weighted 10×)
|
||||
FPI_incast — actor in the film but not this second (timing/boundary)
|
||||
FN = |GT − Pred|, counting only gallery-known actors (fair recall — ~67% of X-Ray
|
||||
cast have no reference embedding and can never be recognised)
|
||||
agreement at t = Jaccard |Pred ∩ GT| / |Pred ∪ GT| — PARTIAL credit, so naming 2
|
||||
of 3 actors scores 2/3, not 0. Averaged over sampled seconds → the
|
||||
"what fraction of the time do we agree with X-Ray" number. (Exact-set match is
|
||||
reported separately as exact_match_rate; it is far harsher and dominated by
|
||||
recall.)
|
||||
|
||||
Objective (DE): per-second F1 computed with the WEIGHTED FPI, so naming someone who
|
||||
isn't in the film hurts 10× more than a boundary slip.
|
||||
|
||||
Reported: TPI, FPI (+split), FN, precision, recall, F1, and agreement_rate — the
|
||||
fraction of sampled seconds where we exactly matched X-Ray.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import csv
|
||||
import json
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
REPO = Path(__file__).resolve().parent.parent.parent
|
||||
sys.path.insert(0, str(REPO / "scripts" / "validation"))
|
||||
from identity import keys_for # noqa: E402
|
||||
|
||||
|
||||
def load_second_timeline(xray_dir: str):
|
||||
"""Return (timeline, film_cast_keys, duration).
|
||||
|
||||
timeline: dict second -> list of actor key-sets on screen per X-Ray.
|
||||
Each second inside a scene [start,end) inherits that scene's cast set.
|
||||
"""
|
||||
d = Path(xray_dir)
|
||||
id_to_name = {}
|
||||
with open(d / "people.csv", newline="", encoding="utf-8") as f:
|
||||
for r in csv.DictReader(f):
|
||||
nm = (r.get("name_id") or "").strip()
|
||||
if nm:
|
||||
id_to_name[nm] = (r.get("person") or "").strip()
|
||||
|
||||
film_cast = set()
|
||||
for nm, name in id_to_name.items():
|
||||
film_cast |= keys_for(imdb_id=nm, name=name)
|
||||
|
||||
spans = {}
|
||||
with open(d / "scenes.csv", newline="", encoding="utf-8") as f:
|
||||
for r in csv.DictReader(f):
|
||||
sn = (r.get("scene") or "").strip()
|
||||
try:
|
||||
spans[sn] = (float(r["start"]) / 1000.0, float(r["end"]) / 1000.0)
|
||||
except (KeyError, ValueError):
|
||||
continue
|
||||
|
||||
scene_cast: dict[str, list] = {}
|
||||
with open(d / "people_in_scenes.csv", newline="", encoding="utf-8") as f:
|
||||
for r in csv.DictReader(f):
|
||||
sn = (r.get("scene") or "").strip()
|
||||
nm = (r.get("name_id") or "").strip()
|
||||
if sn in spans and nm:
|
||||
scene_cast.setdefault(sn, []).append(
|
||||
frozenset(keys_for(imdb_id=nm, name=id_to_name.get(nm))))
|
||||
|
||||
timeline: dict[int, list] = {}
|
||||
duration = 0.0
|
||||
for sn, (t0, t1) in spans.items():
|
||||
duration = max(duration, t1)
|
||||
cast = scene_cast.get(sn, [])
|
||||
for t in range(int(t0), int(t1)):
|
||||
timeline[t] = cast
|
||||
return timeline, film_cast, duration
|
||||
|
||||
|
||||
def load_pred_intervals(pred_json: dict):
|
||||
"""[(keyset, [(t0,t1),...]), ...] for each actor the pipeline named."""
|
||||
out = []
|
||||
for a in pred_json.get("actors", []):
|
||||
keys = frozenset(keys_for(imdb_id=a.get("imdb_id"), tmdb_id=a.get("tmdb_id"),
|
||||
jellyfin_id=a.get("jellyfin_id"), name=a.get("name")))
|
||||
out.append((keys, [(float(t0), float(t1)) for t0, t1 in a.get("scenes", [])]))
|
||||
return out
|
||||
|
||||
|
||||
def _match(P, G):
|
||||
"""Greedy 1:1 match by key intersection; returns (n_matched, matched_G_mask)."""
|
||||
used = [False] * len(G)
|
||||
n = 0
|
||||
for pa in P:
|
||||
for j, ga in enumerate(G):
|
||||
if not used[j] and (pa & ga):
|
||||
used[j] = True
|
||||
n += 1
|
||||
break
|
||||
return n, used
|
||||
|
||||
|
||||
def score_seconds(pred_json: dict, xray_dir: str, gallery_keys: set | None = None,
|
||||
misid_weight: float = 10.0):
|
||||
timeline, film_cast, duration = load_second_timeline(xray_dir)
|
||||
pred = load_pred_intervals(pred_json)
|
||||
|
||||
TPI = FPI = FN = 0
|
||||
FPI_misid = FPI_incast = 0
|
||||
FPI_w = 0.0
|
||||
jaccard_sum = 0.0 # partial-credit agreement, summed over seconds
|
||||
exact = 0
|
||||
n_sec = 0
|
||||
|
||||
for t in sorted(timeline):
|
||||
G = [set(a) for a in timeline[t]]
|
||||
P = [set(k) for k, wins in pred if any(w0 <= t <= w1 for w0, w1 in wins)]
|
||||
# fair recall: only GT actors we could possibly recognise
|
||||
if gallery_keys is not None:
|
||||
G = [g for g in G if g & gallery_keys]
|
||||
|
||||
tp, matched = _match(P, G)
|
||||
# classify each unmatched prediction
|
||||
fpi_w = 0.0
|
||||
n_fp = 0
|
||||
for pa in P:
|
||||
if any(pa & ga for ga in G):
|
||||
continue
|
||||
n_fp += 1
|
||||
if pa & film_cast:
|
||||
FPI_incast += 1; fpi_w += 1.0
|
||||
else:
|
||||
FPI_misid += 1; fpi_w += misid_weight
|
||||
fn = len(G) - tp
|
||||
|
||||
TPI += tp; FPI += n_fp; FN += fn; FPI_w += fpi_w
|
||||
# partial-credit agreement: |∩| / |∪| at this second
|
||||
union = tp + n_fp + fn
|
||||
if union:
|
||||
jaccard_sum += tp / union
|
||||
else:
|
||||
jaccard_sum += 1.0 # both empty = agreement (nobody on screen)
|
||||
if n_fp == 0 and fn == 0:
|
||||
exact += 1
|
||||
n_sec += 1
|
||||
|
||||
prec = TPI / (TPI + FPI_w) if TPI + FPI_w else 0.0 # weighted (misID hurts 10×)
|
||||
prec_raw = TPI / (TPI + FPI) if TPI + FPI else 0.0
|
||||
rec = TPI / (TPI + FN) if TPI + FN else 0.0
|
||||
f1 = 2 * prec * rec / (prec + rec) if prec + rec else 0.0
|
||||
return {"TPI": TPI, "FPI": FPI, "FPI_misid": FPI_misid, "FPI_incast": FPI_incast,
|
||||
"FN": FN, "precision": prec, "precision_raw": prec_raw, "recall": rec,
|
||||
"f1": f1,
|
||||
# partial-credit: mean per-second Jaccard = "% of actors we agree on, over time"
|
||||
"agreement_rate": jaccard_sum / n_sec if n_sec else 0.0,
|
||||
"exact_match_rate": exact / n_sec if n_sec else 0.0,
|
||||
"n_seconds": n_sec, "duration_sec": duration}
|
||||
|
||||
|
||||
def main():
|
||||
p = argparse.ArgumentParser(description=__doc__,
|
||||
formatter_class=argparse.RawDescriptionHelpFormatter)
|
||||
p.add_argument("--pred", required=True)
|
||||
p.add_argument("--xray", required=True)
|
||||
p.add_argument("--gallery")
|
||||
args = p.parse_args()
|
||||
gk = None
|
||||
if args.gallery:
|
||||
sys.path.insert(0, str(REPO / "scripts" / "validation"))
|
||||
from sample_eval import load_gallery_keys
|
||||
gk = load_gallery_keys(args.gallery)
|
||||
m = score_seconds(json.loads(Path(args.pred).read_text()), args.xray, gk)
|
||||
print(f"seconds sampled : {m['n_seconds']} (film {m['duration_sec']:.0f}s)")
|
||||
print(f"TPI/FPI/FN : {m['TPI']}/{m['FPI']}/{m['FN']}")
|
||||
print(f" FPI misID : {m['FPI_misid']} (actor not in film — weighted 10x)")
|
||||
print(f" FPI in-cast : {m['FPI_incast']}")
|
||||
print(f"precision (w) : {m['precision']*100:.1f}% raw {m['precision_raw']*100:.1f}%")
|
||||
print(f"recall : {m['recall']*100:.1f}%")
|
||||
print(f"F1 (weighted) : {m['f1']*100:.1f}%")
|
||||
print(f"AGREEMENT : {m['agreement_rate']*100:.1f}% (mean per-second % of actors "
|
||||
f"we agree on with X-Ray)")
|
||||
print(f" exact-set match: {m['exact_match_rate']*100:.1f}% of seconds (harsher, "
|
||||
f"all-or-nothing)")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,75 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Smoke test for the sae_kpn module: assemble the real downstream pipeline nodes
|
||||
(face_tracker → identity_matcher → scene_tracker) in a Python-driven KPN network,
|
||||
fed by a no-input Python source node, and verify SceneAnnotations flow out.
|
||||
|
||||
Proves the KPN-native replay path works without any numpy port of node logic.
|
||||
Run: python scripts/optimizer/test_sae_kpn.py [gallery.json] [build_dir]
|
||||
"""
|
||||
import sys
|
||||
import queue
|
||||
import numpy as np
|
||||
from pathlib import Path
|
||||
|
||||
REPO = Path(__file__).resolve().parent.parent.parent
|
||||
GAL = sys.argv[1] if len(sys.argv) > 1 else str(REPO / "gallery_arcface_w600k_r50.json")
|
||||
BUILD = sys.argv[2] if len(sys.argv) > 2 else str(REPO / "build")
|
||||
sys.path.insert(0, BUILD)
|
||||
import sae_kpn # noqa: E402
|
||||
|
||||
|
||||
def make_frame(t, n):
|
||||
e = np.random.randn(n, 512).astype(np.float32)
|
||||
e /= np.linalg.norm(e, axis=1, keepdims=True)
|
||||
return {"timestamp_sec": t, "eof": False,
|
||||
"bbox": np.tile(np.array([10, 10, 50, 50], np.float32), (n, 1)),
|
||||
"landmarks": np.tile(np.arange(10, dtype=np.float32), (n, 1)),
|
||||
"confidence": np.full((n,), 0.9, np.float32), "embeddings": e}
|
||||
|
||||
|
||||
def main():
|
||||
net = sae_kpn.Network()
|
||||
sae_kpn._register_types(net)
|
||||
cfg = {"prob_threshold": 0.99, "anneal_sec": 10.0, "extinction_sec": 5.0}
|
||||
|
||||
frames = [make_frame(float(t), 1) for t in range(3)]
|
||||
frames.append({"timestamp_sec": 3.0, "eof": True})
|
||||
idx = [0]
|
||||
eof_frame = {"timestamp_sec": 3.0, "eof": True}
|
||||
|
||||
def source():
|
||||
# Emit each frame once, then keep returning EOF (never block) so the node
|
||||
# thread stays responsive to stop() after the sink has seen EOF.
|
||||
i = idx[0]
|
||||
idx[0] += 1
|
||||
return frames[i] if i < len(frames) else eof_frame
|
||||
|
||||
sae_kpn.add_node_python(net, "replay", source, [], ["EmbeddedSceneFrame"], 8)
|
||||
sae_kpn.add_face_tracker(net, "tracker", cfg, 16)
|
||||
sae_kpn.add_identity_matcher(net, "matcher", GAL, cfg, 16)
|
||||
sae_kpn.add_scene_tracker(net, "scene", cfg, 16)
|
||||
net.connect("replay", 0, "tracker", 0)
|
||||
net.connect("tracker", 0, "matcher", 0)
|
||||
net.connect("matcher", 0, "scene", 0)
|
||||
net.build()
|
||||
net.start()
|
||||
|
||||
got = []
|
||||
for _ in range(4):
|
||||
sa = net.read("scene", 0)
|
||||
got.append(sa)
|
||||
if sa.get("eof"):
|
||||
break
|
||||
net.stop()
|
||||
|
||||
non_eof = [g for g in got if not g.get("eof")]
|
||||
assert len(non_eof) == 3, f"expected 3 annotations, got {len(non_eof)}"
|
||||
assert got[-1].get("eof"), "expected trailing EOF"
|
||||
assert [g["timestamp_sec"] for g in non_eof] == [0.0, 1.0, 2.0], "timestamps wrong"
|
||||
assert all("visible_actors" in g for g in non_eof), "missing visible_actors"
|
||||
print(f"OK: {len(non_eof)} annotations through the real KPN chain, EOF received")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -11,18 +11,24 @@ Usage:
|
||||
--jellyfin-url http://jellyfin.local:8096 \\
|
||||
--api-key YOUR_API_KEY \\
|
||||
--title "The Matrix" \\
|
||||
--gallery whole_gallery.json \\
|
||||
--gallery whole_gallery.h5 \\
|
||||
-- --fps 5 --verbosity 2
|
||||
|
||||
Anything after "--" is passed through unchanged to scene_analyze.
|
||||
|
||||
Add --preview to open the live OpenCV display window: it resolves the media path
|
||||
from Jellyfin exactly as normal, then launches build/scene_preview instead of the
|
||||
headless binary (implies --no-push). Works with --worker or a single title:
|
||||
|
||||
python scripts/run_from_jellyfin.py ... --worker --preview -- --fps 5
|
||||
|
||||
Worker mode (--worker) polls the JRay plugin's Tasks/Pending endpoint for a
|
||||
random batch of items with no truth data yet, processing each in turn:
|
||||
|
||||
python scripts/run_from_jellyfin.py \\
|
||||
--jellyfin-url http://jellyfin.local:8096 \\
|
||||
--api-key YOUR_API_KEY \\
|
||||
--gallery whole_gallery.json \\
|
||||
--gallery whole_gallery.h5 \\
|
||||
--worker \\
|
||||
-- --fps 5
|
||||
"""
|
||||
@@ -40,6 +46,7 @@ import requests
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parent))
|
||||
import sae_env # noqa: F401 — loads .env into os.environ on import
|
||||
from sae_gallery import load_gallery_hdf5, save_gallery_hdf5
|
||||
from sae_jellyfin import (
|
||||
jf_get, find_item_id, fetch_cast_person_ids, actor_jellyfin_id, fetch_episode_info,
|
||||
)
|
||||
@@ -171,7 +178,7 @@ def process_item(args, item_id: str, extra: list[str]) -> None:
|
||||
gallery_path = args.gallery
|
||||
filtered_file = None
|
||||
if not args.no_filter:
|
||||
gallery = json.loads(Path(args.gallery).read_text())
|
||||
gallery = load_gallery_hdf5(Path(args.gallery))
|
||||
actors = episode_cast_actors(args, item_id, gallery)
|
||||
if actors is None:
|
||||
cast_ids = fetch_cast_person_ids(args.jellyfin_url, args.api_key, item_id)
|
||||
@@ -180,9 +187,9 @@ def process_item(args, item_id: str, extra: list[str]) -> None:
|
||||
print(f"Filtered gallery to {len(actors)}/{len(gallery.get('actors', []))} "
|
||||
f"actor(s) credited in {name!r}", file=sys.stderr)
|
||||
filtered_file = tempfile.NamedTemporaryFile(
|
||||
mode="w", suffix=".json", prefix="sae_gallery_", delete=False)
|
||||
json.dump({"actors": actors}, filtered_file)
|
||||
suffix=".h5", prefix="sae_gallery_", delete=False)
|
||||
filtered_file.close()
|
||||
save_gallery_hdf5({"actors": actors}, Path(filtered_file.name))
|
||||
gallery_path = filtered_file.name
|
||||
|
||||
try:
|
||||
@@ -281,7 +288,7 @@ def main():
|
||||
parser.add_argument("--item-types", default="Movie,Episode",
|
||||
help="Item types to search when using --title (default: Movie,Episode)")
|
||||
parser.add_argument("--gallery", required=True,
|
||||
help="Global gallery.json built by make_jellyfin_gallery.py")
|
||||
help="Global gallery.h5 built by make_jellyfin_gallery.py")
|
||||
parser.add_argument("--no-filter", action="store_true",
|
||||
help="Skip per-title cast filtering and pass --gallery through as-is")
|
||||
parser.add_argument("--episode-cast", choices=("tmdb", "series"), default="tmdb",
|
||||
@@ -297,6 +304,12 @@ def main():
|
||||
"ignored with --worker, which always uses <title>.json)")
|
||||
parser.add_argument("--bin", default="build/scene_analyze",
|
||||
help="Path to scene_analyze binary (default: build/scene_analyze)")
|
||||
parser.add_argument("--preview", action="store_true",
|
||||
help="Launch the scene_preview binary (live OpenCV display window) "
|
||||
"instead of headless scene_analyze. Implies --no-push. Use "
|
||||
"--preview-bin to override its path. Press q/Esc to close.")
|
||||
parser.add_argument("--preview-bin", default="build/scene_preview",
|
||||
help="Path to scene_preview binary (default: build/scene_preview)")
|
||||
parser.add_argument("--dry-run", action="store_true",
|
||||
help="Resolve and print the scene_analyze command without running it")
|
||||
parser.add_argument("--no-push", action="store_true",
|
||||
@@ -306,6 +319,12 @@ def main():
|
||||
if extra and extra[0] == "--":
|
||||
extra = extra[1:]
|
||||
|
||||
# --preview swaps in the display binary and disables pushing truth (a preview
|
||||
# run is interactive/debug, not a truth-producing analysis).
|
||||
if args.preview:
|
||||
args.bin = args.preview_bin
|
||||
args.no_push = True
|
||||
|
||||
if args.worker:
|
||||
if args.output:
|
||||
sys.exit("--output is incompatible with --worker (each item needs its own file)")
|
||||
|
||||
+83
-5
@@ -2,7 +2,13 @@
|
||||
|
||||
Consolidates the three near-identical download loops (make_gallery.download_images,
|
||||
make_jellyfin_gallery.download_urls + download_person_images) and the duplicated
|
||||
"write gallery.json + .missing_images.json" tail from both builders.
|
||||
"write gallery.h5 + .missing_images.json" tail from both builders.
|
||||
|
||||
Galleries are written directly as HDF5 — never JSON. Same layout the C++ side
|
||||
reads/writes (src/gallery/gallery_store.cpp): flat [N,512] embeddings + per-actor
|
||||
offset/count, parallel imdb_id/tmdb_id/jellyfin_id/name string arrays, and a
|
||||
per-embedding-row source_images array. calibration is left absent (calib_hash=0);
|
||||
the C++ identity_matcher fits and writes it back into the file on first use.
|
||||
"""
|
||||
|
||||
import io
|
||||
@@ -10,6 +16,8 @@ import json
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
import h5py
|
||||
import numpy as np
|
||||
import requests
|
||||
from PIL import Image, UnidentifiedImageError
|
||||
|
||||
@@ -93,11 +101,81 @@ def download_images(urls: list[str], dest_dir: Path, n: int,
|
||||
return paths
|
||||
|
||||
|
||||
def save_gallery(gallery: dict, missing: list[dict], output: Path) -> None:
|
||||
"""Write gallery.json and, if any actors lack images, a .missing_images.json sidecar."""
|
||||
def save_gallery_hdf5(gallery: dict, output: Path) -> None:
|
||||
"""Write a gallery dict ({"actors": [...]}) directly as HDF5 — same schema
|
||||
src/gallery/gallery_store.cpp reads/writes. No calibration group; the
|
||||
C++ identity_matcher computes and writes it back into this file on first
|
||||
use against an unseen set of embeddings."""
|
||||
actors = gallery["actors"]
|
||||
embs, offsets, counts = [], [], []
|
||||
imdb, tmdb, jf, name, src_images = [], [], [], [], []
|
||||
row = 0
|
||||
for a in actors:
|
||||
e = a.get("embeddings", [])
|
||||
offsets.append(row)
|
||||
counts.append(len(e))
|
||||
row += len(e)
|
||||
embs.extend(e)
|
||||
si = a.get("source_images", [])
|
||||
for i in range(len(e)):
|
||||
src_images.append(si[i] if i < len(si) else "")
|
||||
imdb.append(a.get("imdb_id", "") or "")
|
||||
tmdb.append(str(a.get("tmdb_id", "") or ""))
|
||||
jf.append(a.get("jellyfin_id", a.get("jellyfin_person_id", "")) or "")
|
||||
name.append(a.get("name", "") or "")
|
||||
|
||||
emb_arr = np.asarray(embs, dtype=np.float32) if embs else np.zeros((0, 512), np.float32)
|
||||
if emb_arr.ndim == 1:
|
||||
emb_arr = emb_arr.reshape(0, 512)
|
||||
|
||||
output.parent.mkdir(parents=True, exist_ok=True)
|
||||
output.write_text(json.dumps(gallery, indent=2) + "\n")
|
||||
print(f"Saved: {output}", file=sys.stderr)
|
||||
str_t = h5py.string_dtype("utf-8")
|
||||
with h5py.File(output, "w") as f:
|
||||
f.create_dataset("embeddings", data=emb_arr)
|
||||
f.create_dataset("offset", data=np.asarray(offsets, np.int64))
|
||||
f.create_dataset("count", data=np.asarray(counts, np.int32))
|
||||
f.create_dataset("imdb_id", data=np.asarray(imdb, dtype=object), dtype=str_t)
|
||||
f.create_dataset("tmdb_id", data=np.asarray(tmdb, dtype=object), dtype=str_t)
|
||||
f.create_dataset("jellyfin_id", data=np.asarray(jf, dtype=object), dtype=str_t)
|
||||
f.create_dataset("name", data=np.asarray(name, dtype=object), dtype=str_t)
|
||||
f.create_dataset("source_images", data=np.asarray(src_images, dtype=object), dtype=str_t)
|
||||
print(f"Saved: {output} ({len(actors)} actors, {emb_arr.shape[0]} embeddings)",
|
||||
file=sys.stderr)
|
||||
|
||||
|
||||
def load_gallery_hdf5(path: Path) -> dict:
|
||||
"""Read an HDF5 gallery back into the same {"actors": [...]} dict shape the
|
||||
builders work with in memory (for --merge). Mirrors save_gallery_hdf5."""
|
||||
with h5py.File(path, "r") as f:
|
||||
emb = f["embeddings"][:]
|
||||
offset = f["offset"][:]
|
||||
count = f["count"][:]
|
||||
imdb = [s.decode() if isinstance(s, bytes) else s for s in f["imdb_id"][:]]
|
||||
tmdb = [s.decode() if isinstance(s, bytes) else s for s in f["tmdb_id"][:]]
|
||||
jf = [s.decode() if isinstance(s, bytes) else s for s in f["jellyfin_id"][:]]
|
||||
name = [s.decode() if isinstance(s, bytes) else s for s in f["name"][:]]
|
||||
src_images = None
|
||||
if "source_images" in f:
|
||||
src_images = [s.decode() if isinstance(s, bytes) else s
|
||||
for s in f["source_images"][:]]
|
||||
|
||||
actors = []
|
||||
for a in range(len(offset)):
|
||||
s, n = int(offset[a]), int(count[a])
|
||||
actor = {"imdb_id": imdb[a], "tmdb_id": tmdb[a], "jellyfin_id": jf[a],
|
||||
"name": name[a], "embeddings": [emb[s + i].tolist() for i in range(n)]}
|
||||
if src_images is not None:
|
||||
actor["source_images"] = [src_images[s + i] for i in range(n)]
|
||||
actors.append(actor)
|
||||
return {"actors": actors}
|
||||
|
||||
|
||||
def save_gallery(gallery: dict, missing: list[dict], output: Path) -> None:
|
||||
"""Write the gallery as HDF5 (forcing a .h5 extension) and, if any actors
|
||||
lack images, a .missing_images.json sidecar."""
|
||||
if output.suffix not in (".h5", ".hdf5"):
|
||||
output = output.with_suffix(".h5")
|
||||
save_gallery_hdf5(gallery, output)
|
||||
|
||||
if missing:
|
||||
missing_path = output.with_name(output.stem + ".missing_images.json")
|
||||
|
||||
@@ -0,0 +1,87 @@
|
||||
# scripts/validation — per-scene actor-presence eval
|
||||
|
||||
Validates the pipeline's per-scene "who's on screen" output against external
|
||||
ground truth, offline. Annealing (`anneal_sec`) means an actor's presence is only
|
||||
defined *after* the whole file is merged into `[start,end]` windows, so we cannot
|
||||
score live: process → write the pipeline JSON → **sample timepoints** → compare
|
||||
predicted vs ground-truth presence sets → micro-sum TP/FP/FN → precision/recall/F1.
|
||||
|
||||
## Ground-truth sources
|
||||
|
||||
| Source | Semantics | Fair to a face pipeline? | What it measures |
|
||||
| ------ | --------- | ------------------------ | ---------------- |
|
||||
| **MovieNet-PS** | on-screen **face** presence per shot | yes — like-for-like | recognition accuracy |
|
||||
| **Amazon X-Ray** (Zenodo) | **cast-in-scene** (incl. off-camera / non-speaking) | no — penalizes by design | coverage ceiling; recall gap = actors we structurally can't see |
|
||||
|
||||
- MovieNet is the honest recognition number.
|
||||
- X-Ray is an upper bound: its recall gap tells you how much presence is off-camera
|
||||
cast a face detector can never reach — not a pipeline error.
|
||||
|
||||
X-Ray dataset: Zenodo DOI `10.5281/zenodo.17659734` (CC-BY-4.0). Per movie it ships
|
||||
`people.csv`, `scenes.csv`, `people_in_scenes.csv`.
|
||||
|
||||
## Usage
|
||||
|
||||
```bash
|
||||
# against Amazon X-Ray CSVs for one title
|
||||
python scripts/validation/sample_eval.py \
|
||||
--pred "Scene in a Mall.json" \
|
||||
--xray /data/xray/<movie_dir> \
|
||||
--gallery gallery_arcface_w600k_r50.json \
|
||||
--step 1.0
|
||||
|
||||
# against MovieNet-PS for one title
|
||||
python scripts/validation/sample_eval.py \
|
||||
--pred out.json \
|
||||
--movienet /data/movienet --split Train_app10 --title tt0032138 \
|
||||
--gallery gallery_arcface_w600k_r50.json
|
||||
```
|
||||
|
||||
### Sampling modes
|
||||
- `--step S` regular grid every S s (default 1.0) — time-weighted headline number.
|
||||
- `--random N` N uniform-random timepoints (for confidence intervals).
|
||||
- `--scene-anchored` one timepoint per GT scene midpoint — the literal X-Ray
|
||||
"did I get this scene's cast right?" question; neutralizes long-scene bias.
|
||||
|
||||
Ground truth is compared **raw** (annealing is *not* applied to GT).
|
||||
|
||||
## Matching & masking
|
||||
|
||||
Identity is provider-agnostic (`identity.py`): each actor is the *set* of every key
|
||||
we can derive — `imdb:nm…`, `tmdb:…`, `jf:…`, `name:<normalized>`. Predicted and GT
|
||||
actors match iff their key-sets intersect, so an output carrying only tmdb/jellyfin
|
||||
ids still joins X-Ray's `nm` ids via the normalized-name fallback.
|
||||
|
||||
Scoring is **masked to `gallery ∩ GT`**: a GT actor absent from the gallery is
|
||||
ignored (not an FN), so we measure pipeline accuracy, not gallery coverage. Without
|
||||
`--gallery` the mask falls back to `GT ∩ pred` keys. `--no-mask` disables it.
|
||||
|
||||
### Exact id join via the tmdb→imdb crosswalk (recommended)
|
||||
|
||||
The gallery/pipeline output key actors by **TMDB** id (no `nm…`), while X-Ray and
|
||||
MovieNet key on **IMDb**. They only overlap on the fuzzy `name:` key by default.
|
||||
Build a cached `tmdb→imdb` table once and pass it with `--crosswalk` to turn the
|
||||
name join into an exact id join:
|
||||
|
||||
```bash
|
||||
# one-time: resolve every gallery tmdb id via TMDB /person/{id}/external_ids
|
||||
python scripts/validation/tmdb_imdb_map.py \
|
||||
--gallery gallery_arcface_w600k_r50.json \
|
||||
--out scripts/validation/tmdb_imdb.json # TMDB_API_KEY from env/.env
|
||||
|
||||
# then score with exact ids
|
||||
python scripts/validation/sample_eval.py --pred out.json --xray <dir> \
|
||||
--gallery gallery_arcface_w600k_r50.json \
|
||||
--crosswalk scripts/validation/tmdb_imdb.json
|
||||
```
|
||||
|
||||
The table caches nulls (tmdb ids TMDB has no IMDb id for) and checkpoints, so a
|
||||
re-run only resolves new ids. TMDB is authoritative for this crosswalk — there is
|
||||
no clean free bulk `tmdb_person ↔ nm` file, so we query the API once and cache.
|
||||
|
||||
## Files
|
||||
- `sample_eval.py` — CLI scorer.
|
||||
- `ground_truth.py` — `XRayGroundTruth`, `MovieNetGroundTruth` loaders.
|
||||
- `identity.py` — provider-agnostic match keys.
|
||||
- `tmdb_imdb_map.py` — build/consult the cached `tmdb→imdb` crosswalk.
|
||||
- `test_sample_eval.py` — self-contained tests (`python scripts/validation/test_sample_eval.py`).
|
||||
@@ -0,0 +1,221 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
ground_truth.py — pluggable ground-truth loaders for per-scene presence eval.
|
||||
|
||||
A ground truth is a timeline of "who is present when", exposed as:
|
||||
|
||||
GroundTruth.present_at(t: float) -> set[str] # match-keys present at time t
|
||||
GroundTruth.scene_windows() -> list[(t0, t1)] # scene spans (for --scene-anchored)
|
||||
GroundTruth.all_keys() -> set[str] # every actor the GT knows (for masking)
|
||||
|
||||
"Match-keys" are provider-agnostic identity tokens (see identity.py): an actor is
|
||||
represented by *all* the keys we can derive (nm-id, tmdb-id, jellyfin-id, normalized
|
||||
name), so predicted and GT sets intersect if they agree on *any* shared id space.
|
||||
This matters because the pipeline output may carry only tmdb/jellyfin ids while
|
||||
X-Ray/MovieNet key on IMDb nm-ids — see [[per-scene-presence-eval-design]].
|
||||
|
||||
Two sources implemented:
|
||||
* XRayGroundTruth — Zenodo scene-level Amazon X-Ray CSVs (cast-in-scene).
|
||||
* MovieNetGroundTruth — MovieNet-PS per-shot face annotations (on-screen faces).
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import csv
|
||||
import sys
|
||||
from bisect import bisect_right
|
||||
from pathlib import Path
|
||||
|
||||
from identity import keys_for
|
||||
|
||||
|
||||
class GroundTruth:
|
||||
"""Base: a set of actors, each with presence intervals [(t0,t1), ...] in seconds."""
|
||||
|
||||
def __init__(self) -> None:
|
||||
# actor_id (any stable local id) -> {"keys": set[str], "intervals": [(t0,t1)]}
|
||||
self._actors: dict[str, dict] = {}
|
||||
self._scene_spans: list[tuple[float, float]] = []
|
||||
|
||||
# -- construction helpers ------------------------------------------------
|
||||
def _add_interval(self, actor_id: str, keys: set[str], t0: float, t1: float) -> None:
|
||||
a = self._actors.setdefault(actor_id, {"keys": set(), "intervals": []})
|
||||
a["keys"] |= keys
|
||||
a["intervals"].append((float(t0), float(t1)))
|
||||
|
||||
def _finalize(self) -> None:
|
||||
"""Sort intervals and precompute a flat sorted start-array per actor."""
|
||||
for a in self._actors.values():
|
||||
a["intervals"].sort()
|
||||
a["_starts"] = [iv[0] for iv in a["intervals"]]
|
||||
self._scene_spans.sort()
|
||||
|
||||
# -- query API -----------------------------------------------------------
|
||||
def present_at(self, t: float) -> set[frozenset[str]]:
|
||||
"""Set of actors present at t; each actor is its (frozen) key-set."""
|
||||
out: set[frozenset[str]] = set()
|
||||
for a in self._actors.values():
|
||||
ivs = a["intervals"]
|
||||
i = bisect_right(a["_starts"], t) # first interval starting after t
|
||||
# walk back over intervals that started at/before t
|
||||
j = i - 1
|
||||
while j >= 0:
|
||||
t0, t1 = ivs[j]
|
||||
if t1 >= t:
|
||||
out.add(frozenset(a["keys"]))
|
||||
break
|
||||
# intervals sorted by start; an earlier one could still cover t,
|
||||
# but since we only need membership, keep scanning a bounded window.
|
||||
j -= 1
|
||||
if i - j > 8: # bound: overlapping intervals per actor are rare
|
||||
break
|
||||
return out
|
||||
|
||||
def scene_windows(self) -> list[tuple[float, float]]:
|
||||
return self._scene_spans
|
||||
|
||||
def all_keys(self) -> set[str]:
|
||||
out: set[str] = set()
|
||||
for a in self._actors.values():
|
||||
out |= a["keys"]
|
||||
return out
|
||||
|
||||
def summary(self) -> str:
|
||||
n_iv = sum(len(a["intervals"]) for a in self._actors.values())
|
||||
return (f"{len(self._actors)} actors, {n_iv} intervals, "
|
||||
f"{len(self._scene_spans)} scenes")
|
||||
|
||||
|
||||
# ── Amazon X-Ray (Zenodo) ─────────────────────────────────────────────────────
|
||||
|
||||
class XRayGroundTruth(GroundTruth):
|
||||
"""
|
||||
Load one movie's X-Ray CSVs (Zenodo DOI 10.5281/zenodo.17659734).
|
||||
Real schema (columns are milliseconds):
|
||||
people.csv name_id (nm...), person, character
|
||||
scenes.csv scene, start, end (ms)
|
||||
people_in_scenes.csv scene, start, end, name_id, timestamp (ms)
|
||||
|
||||
Presence = whole scene span for every character listed in that scene.
|
||||
Semantics: cast-in-scene (incl. off-camera) — a recall ceiling, not accuracy.
|
||||
Columns are resolved case-insensitively so minor variants still load.
|
||||
"""
|
||||
|
||||
def __init__(self, movie_dir: str | Path) -> None:
|
||||
super().__init__()
|
||||
d = Path(movie_dir)
|
||||
people = _read_csv(d / "people.csv")
|
||||
scenes = _read_csv(d / "scenes.csv")
|
||||
pis = _read_csv(d / "people_in_scenes.csv")
|
||||
|
||||
# nm-id -> actor name (for building match keys)
|
||||
nm_col_p = _find_col(people, "name_id", "nm", "imdb")
|
||||
name_col = _find_col(people, "person", "actor") # actor name lives in "person"
|
||||
id_to_name: dict[str, str] = {}
|
||||
for row in people:
|
||||
nm = (row.get(nm_col_p) or "").strip()
|
||||
if nm:
|
||||
id_to_name[nm] = (row.get(name_col) or "").strip()
|
||||
|
||||
# scene number -> (t0_sec, t1_sec)
|
||||
scene_col_s = _find_col(scenes, "scene")
|
||||
start_col = _find_col(scenes, "start")
|
||||
end_col = _find_col(scenes, "end")
|
||||
span: dict[str, tuple[float, float]] = {}
|
||||
for row in scenes:
|
||||
sn = (row.get(scene_col_s) or "").strip()
|
||||
t0 = _ms_to_sec(row.get(start_col))
|
||||
t1 = _ms_to_sec(row.get(end_col))
|
||||
if sn and t0 is not None and t1 is not None:
|
||||
span[sn] = (t0, t1)
|
||||
self._scene_spans.append((t0, t1))
|
||||
|
||||
# scene number -> [nm ids present]
|
||||
scene_col_pis = _find_col(pis, "scene")
|
||||
nm_col_pis = _find_col(pis, "name_id", "nm", "imdb")
|
||||
for row in pis:
|
||||
sn = (row.get(scene_col_pis) or "").strip()
|
||||
nm = (row.get(nm_col_pis) or "").strip()
|
||||
if sn not in span or not nm:
|
||||
continue
|
||||
t0, t1 = span[sn]
|
||||
self._add_interval(nm, keys_for(imdb_id=nm, name=id_to_name.get(nm)), t0, t1)
|
||||
|
||||
self._finalize()
|
||||
|
||||
|
||||
# ── MovieNet-PS ────────────────────────────────────────────────────────────────
|
||||
|
||||
class MovieNetGroundTruth(GroundTruth):
|
||||
"""
|
||||
Build presence from MovieNet-PS per-shot face annotations for a single title.
|
||||
|
||||
Input: the flat annotation list produced by movienet_prep.load_movienet_annotations
|
||||
filtered to one movie (tt-id), plus a shot->time map. Because MovieNet frames are
|
||||
named tt.../shot_XXXX_img_Y.jpg with no absolute timestamp, presence is expressed
|
||||
in *shot index* units unless a fps/shot-duration map is supplied. For the sampler
|
||||
we therefore sample at shot granularity (one timepoint per annotated shot).
|
||||
|
||||
Semantics: on-screen face presence per shot — like-for-like fair benchmark.
|
||||
"""
|
||||
|
||||
def __init__(self, annotations: list[dict], id_to_name: dict[str, str] | None = None,
|
||||
shot_seconds: float = 1.0) -> None:
|
||||
super().__init__()
|
||||
id_to_name = id_to_name or {}
|
||||
# group by shot index; each annotated shot becomes a unit interval on a
|
||||
# synthetic timeline (shot_index * shot_seconds).
|
||||
shots: dict[int, set[str]] = {}
|
||||
for ann in annotations:
|
||||
shot = _shot_index(ann["img_path"])
|
||||
if shot is None:
|
||||
continue
|
||||
shots.setdefault(shot, set()).add(ann["imdb_id"])
|
||||
|
||||
for shot, nm_ids in shots.items():
|
||||
t0 = shot * shot_seconds
|
||||
t1 = t0 + shot_seconds
|
||||
self._scene_spans.append((t0, t1))
|
||||
for nm in nm_ids:
|
||||
self._add_interval(nm, keys_for(imdb_id=nm, name=id_to_name.get(nm)), t0, t1)
|
||||
|
||||
self._finalize()
|
||||
|
||||
|
||||
# ── small parsing helpers ──────────────────────────────────────────────────────
|
||||
|
||||
def _read_csv(path: Path) -> list[dict]:
|
||||
if not path.exists():
|
||||
raise FileNotFoundError(f"expected X-Ray CSV not found: {path}")
|
||||
with open(path, newline="", encoding="utf-8") as f:
|
||||
return list(csv.DictReader(f))
|
||||
|
||||
|
||||
def _find_col(rows: list[dict], *keywords: str) -> str:
|
||||
"""Return the first column whose lowercased name contains all keywords of any
|
||||
single keyword group. We try each keyword in order and accept the first hit."""
|
||||
if not rows:
|
||||
raise ValueError("empty CSV — cannot resolve columns")
|
||||
cols = list(rows[0].keys())
|
||||
low = {c: c.lower() for c in cols}
|
||||
for kw in keywords:
|
||||
for c in cols:
|
||||
if kw in low[c]:
|
||||
return c
|
||||
raise KeyError(f"no column matching {keywords} in {cols}")
|
||||
|
||||
|
||||
def _ms_to_sec(v) -> float | None:
|
||||
if v is None or str(v).strip() == "":
|
||||
return None
|
||||
try:
|
||||
return float(v) / 1000.0
|
||||
except ValueError:
|
||||
return None
|
||||
|
||||
|
||||
def _shot_index(img_path: str) -> int | None:
|
||||
# tt0032138/shot_0003_img_1.jpg -> 3
|
||||
import re
|
||||
m = re.search(r"shot_(\d+)", img_path)
|
||||
return int(m.group(1)) if m else None
|
||||
@@ -0,0 +1,54 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
identity.py — provider-agnostic match keys.
|
||||
|
||||
The pipeline output and the ground truth may not share one id space: an output
|
||||
actor can carry only tmdb/jellyfin ids while X-Ray/MovieNet key on IMDb nm-ids.
|
||||
We represent each actor by the *set* of every key we can derive, and treat two
|
||||
actors as the same iff their key sets intersect. Namespacing each key by its
|
||||
provider prevents cross-provider collisions (e.g. an nm-number equalling a
|
||||
tmdb-number).
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
import unicodedata
|
||||
|
||||
|
||||
def norm_name(name: str | None) -> str | None:
|
||||
"""Lowercased, accent-stripped, punctuation-free name for fuzzy fallback match."""
|
||||
if not name:
|
||||
return None
|
||||
s = unicodedata.normalize("NFKD", name)
|
||||
s = "".join(c for c in s if not unicodedata.combining(c))
|
||||
s = re.sub(r"[^a-z0-9 ]+", "", s.lower()).strip()
|
||||
s = re.sub(r"\s+", " ", s)
|
||||
return s or None
|
||||
|
||||
|
||||
def keys_for(imdb_id: str | None = None,
|
||||
tmdb_id: str | None = None,
|
||||
jellyfin_id: str | None = None,
|
||||
name: str | None = None,
|
||||
crosswalk=None) -> set[str]:
|
||||
"""All identity tokens for one actor. Empty strings are ignored.
|
||||
|
||||
If `crosswalk` (a CrosswalkTable) is given and no imdb_id is present, resolve
|
||||
tmdb_id → imdb_id through it so a tmdb-only actor still gets an exact `imdb:`
|
||||
key — turning the fuzzy name join into an exact id join. See tmdb_imdb_map.py.
|
||||
"""
|
||||
keys: set[str] = set()
|
||||
imdb = imdb_id.strip() if (imdb_id and imdb_id.strip()) else None
|
||||
if not imdb and crosswalk is not None and tmdb_id:
|
||||
imdb = crosswalk.imdb_for(tmdb_id)
|
||||
if imdb:
|
||||
keys.add(f"imdb:{imdb}")
|
||||
if tmdb_id and str(tmdb_id).strip():
|
||||
keys.add(f"tmdb:{str(tmdb_id).strip()}")
|
||||
if jellyfin_id and jellyfin_id.strip():
|
||||
keys.add(f"jf:{jellyfin_id.strip()}")
|
||||
nn = norm_name(name)
|
||||
if nn:
|
||||
keys.add(f"name:{nn}")
|
||||
return keys
|
||||
@@ -0,0 +1,313 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
sample_eval.py — offline per-scene presence eval by timepoint sampling.
|
||||
|
||||
Annealing (anneal_sec) means an actor is "present" only after the whole file is
|
||||
merged into [start,end] windows, so we cannot score live: we process → write the
|
||||
pipeline JSON → sample timepoints → compare predicted vs ground-truth presence
|
||||
sets → micro-sum TP/FP/FN → precision / recall / F1.
|
||||
See [[per-scene-presence-eval-design]].
|
||||
|
||||
Usage:
|
||||
# against Amazon X-Ray CSVs (Zenodo)
|
||||
python scripts/validation/sample_eval.py \
|
||||
--pred "Scene in a Mall.json" \
|
||||
--xray /data/xray/tt0384766 \
|
||||
--step 1.0
|
||||
|
||||
# against MovieNet-PS (needs the .mat split + a title tt-id)
|
||||
python scripts/validation/sample_eval.py \
|
||||
--pred out.json \
|
||||
--movienet /data/movienet --split Train_app10 --title tt0032138
|
||||
|
||||
Sampling:
|
||||
--step S regular grid every S seconds (default 1.0) — time-weighted headline
|
||||
--random N N uniform-random timepoints instead of a grid (for CIs)
|
||||
--scene-anchored one timepoint at each GT scene midpoint (X-Ray "per-scene" question)
|
||||
|
||||
Masking: scoring is restricted to actors present in BOTH the pipeline gallery
|
||||
(--gallery) AND the ground truth. A GT actor absent from the gallery is ignored
|
||||
(not counted as a miss) so we measure pipeline accuracy, not gallery coverage.
|
||||
Pass --no-mask to disable.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import random
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parent))
|
||||
from identity import keys_for # noqa: E402
|
||||
from ground_truth import XRayGroundTruth, MovieNetGroundTruth # noqa: E402
|
||||
from tmdb_imdb_map import CrosswalkTable # noqa: E402
|
||||
|
||||
|
||||
# ── pipeline output → presence timeline ────────────────────────────────────────
|
||||
|
||||
class Prediction:
|
||||
"""Pipeline output (minimal/standard schema) as per-actor presence windows."""
|
||||
|
||||
def __init__(self, path: str | Path, crosswalk=None) -> None:
|
||||
with open(path) as f:
|
||||
data = json.load(f)
|
||||
self.movie = data.get("movie", "")
|
||||
self.anneal_sec = data.get("anneal_sec")
|
||||
self.actors: list[dict] = []
|
||||
self._max_t = 0.0
|
||||
for a in data.get("actors", []):
|
||||
keys = keys_for(imdb_id=a.get("imdb_id"), tmdb_id=a.get("tmdb_id"),
|
||||
jellyfin_id=a.get("jellyfin_id"), name=a.get("name"),
|
||||
crosswalk=crosswalk)
|
||||
windows = [(float(t0), float(t1)) for t0, t1 in a.get("scenes", [])]
|
||||
for _, t1 in windows:
|
||||
self._max_t = max(self._max_t, t1)
|
||||
self.actors.append({"keys": keys, "windows": windows})
|
||||
|
||||
def present_at(self, t: float) -> set[frozenset[str]]:
|
||||
"""Set of actors present at t; each actor is its (frozen) key-set."""
|
||||
out: set[frozenset[str]] = set()
|
||||
for a in self.actors:
|
||||
for t0, t1 in a["windows"]:
|
||||
if t0 <= t <= t1:
|
||||
out.add(frozenset(a["keys"]))
|
||||
break
|
||||
return out
|
||||
|
||||
def present_in_span(self, s0: float, s1: float) -> set[frozenset[str]]:
|
||||
"""Actors with ANY detection window overlapping [s0,s1].
|
||||
|
||||
Snaps detections to a scene grid: an actor seen anywhere inside a scene
|
||||
counts as present for the whole scene. Isolates 'did we see this actor in
|
||||
this scene at all' (coverage) from exact-timing recall."""
|
||||
out: set[frozenset[str]] = set()
|
||||
for a in self.actors:
|
||||
for t0, t1 in a["windows"]:
|
||||
if t0 <= s1 and t1 >= s0: # interval overlap
|
||||
out.add(frozenset(a["keys"]))
|
||||
break
|
||||
return out
|
||||
|
||||
def all_keys(self) -> set[str]:
|
||||
out: set[str] = set()
|
||||
for a in self.actors:
|
||||
out |= a["keys"]
|
||||
return out
|
||||
|
||||
@property
|
||||
def max_t(self) -> float:
|
||||
return self._max_t
|
||||
|
||||
|
||||
def load_gallery_keys(path: str | None, crosswalk=None) -> set[str] | None:
|
||||
"""Union of match keys for every actor in the gallery, for masking.
|
||||
|
||||
Accepts either the JSON gallery or the HDF5 fast-load gallery (.h5/.hdf5,
|
||||
produced by json_to_hdf5_gallery.py) — the matcher reads HDF5, so this side must
|
||||
too. HDF5 stores ids/names as parallel string datasets."""
|
||||
if not path:
|
||||
return None
|
||||
out: set[str] = set()
|
||||
if path.endswith(".h5") or path.endswith(".hdf5"):
|
||||
import h5py
|
||||
with h5py.File(path, "r") as f:
|
||||
def col(name):
|
||||
return [(v.decode() if isinstance(v, bytes) else str(v))
|
||||
for v in f[name][:]] if name in f else []
|
||||
imdb, tmdb = col("imdb_id"), col("tmdb_id")
|
||||
jf, name = col("jellyfin_id"), col("name")
|
||||
for i in range(len(name)):
|
||||
out |= keys_for(imdb_id=imdb[i] if i < len(imdb) else "",
|
||||
tmdb_id=tmdb[i] if i < len(tmdb) else "",
|
||||
jellyfin_id=jf[i] if i < len(jf) else "",
|
||||
name=name[i], crosswalk=crosswalk)
|
||||
return out
|
||||
with open(path) as f:
|
||||
data = json.load(f)
|
||||
for a in data.get("actors", []):
|
||||
out |= keys_for(imdb_id=a.get("imdb_id"), tmdb_id=a.get("tmdb_id"),
|
||||
jellyfin_id=a.get("jellyfin_id"), name=a.get("name"),
|
||||
crosswalk=crosswalk)
|
||||
return out
|
||||
|
||||
|
||||
# ── sampling ────────────────────────────────────────────────────────────────
|
||||
|
||||
def sample_points(args, pred: Prediction, gt) -> list[float]:
|
||||
if args.scene_anchored:
|
||||
spans = gt.scene_windows()
|
||||
if not spans:
|
||||
sys.exit("[eval] --scene-anchored: ground truth has no scene spans")
|
||||
return [(t0 + t1) / 2.0 for t0, t1 in spans]
|
||||
|
||||
end = args.end if args.end is not None else max(pred.max_t, _gt_end(gt))
|
||||
if end <= 0:
|
||||
sys.exit("[eval] could not determine timeline end; pass --end")
|
||||
|
||||
if args.random:
|
||||
rng = random.Random(args.seed)
|
||||
return sorted(rng.uniform(0.0, end) for _ in range(args.random))
|
||||
|
||||
n = int(end / args.step) + 1
|
||||
return [i * args.step for i in range(n)]
|
||||
|
||||
|
||||
def _gt_end(gt) -> float:
|
||||
spans = gt.scene_windows()
|
||||
return max((t1 for _, t1 in spans), default=0.0)
|
||||
|
||||
|
||||
# ── scoring ────────────────────────────────────────────────────────────────
|
||||
|
||||
def score(pred: Prediction, gt, points: list[float], mask: set[str] | None,
|
||||
count_out_of_cast_fp: bool = False):
|
||||
"""Micro-sum TP/FP/FN over timepoints.
|
||||
|
||||
Each side is a set of actors, an actor being its key-set. Predicted actor P
|
||||
matches GT actor G iff their key-sets intersect (any shared id/name). We match
|
||||
greedily so each actor is used once, then:
|
||||
TP = matched pairs, FP = unmatched predicted, FN = unmatched GT.
|
||||
|
||||
`mask` (gallery∩GT keys) restricts GT so X-Ray cast we can't recognise doesn't
|
||||
inflate FN. By default predictions are masked the same way — which DROPS a
|
||||
predicted actor who isn't in this film's cast (a cross-film misidentification),
|
||||
hiding the pipeline's worst false positives.
|
||||
|
||||
Set count_out_of_cast_fp=True to keep ALL predictions: an actor named who is not
|
||||
a present GT cast member counts as an FP, including out-of-cast confusions. This
|
||||
is the honest, ship-relevant precision. GT is still masked for fair recall.
|
||||
"""
|
||||
TP = FP = FN = 0
|
||||
per_point = []
|
||||
|
||||
for t in points:
|
||||
P = [set(a) for a in pred.present_at(t)]
|
||||
G = [set(a) for a in gt.present_at(t)]
|
||||
if mask is not None:
|
||||
G = [a for a in G if a & mask]
|
||||
if not count_out_of_cast_fp:
|
||||
P = [a for a in P if a & mask]
|
||||
|
||||
tp = _match_count(P, G)
|
||||
fp = len(P) - tp
|
||||
fn = len(G) - tp
|
||||
TP += tp
|
||||
FP += fp
|
||||
FN += fn
|
||||
per_point.append((t, tp, fp, fn))
|
||||
|
||||
prec = TP / (TP + FP) if (TP + FP) else 0.0
|
||||
rec = TP / (TP + FN) if (TP + FN) else 0.0
|
||||
f1 = 2 * prec * rec / (prec + rec) if (prec + rec) else 0.0
|
||||
return {"TP": TP, "FP": FP, "FN": FN, "precision": prec,
|
||||
"recall": rec, "f1": f1, "n_points": len(points),
|
||||
"per_point": per_point}
|
||||
|
||||
|
||||
def _match_count(P: list[set[str]], G: list[set[str]]) -> int:
|
||||
"""Greedy 1:1 matching of predicted↔GT actors by key intersection."""
|
||||
used = [False] * len(G)
|
||||
matched = 0
|
||||
for pa in P:
|
||||
for j, ga in enumerate(G):
|
||||
if not used[j] and pa & ga:
|
||||
used[j] = True
|
||||
matched += 1
|
||||
break
|
||||
return matched
|
||||
|
||||
|
||||
# ── main ────────────────────────────────────────────────────────────────────
|
||||
|
||||
def main():
|
||||
p = argparse.ArgumentParser(description=__doc__,
|
||||
formatter_class=argparse.RawDescriptionHelpFormatter)
|
||||
p.add_argument("--pred", required=True, help="pipeline output JSON")
|
||||
src = p.add_mutually_exclusive_group(required=True)
|
||||
src.add_argument("--xray", help="dir with people.csv/scenes.csv/people_in_scenes.csv")
|
||||
src.add_argument("--movienet", help="MovieNet-PS root (needs --split and --title)")
|
||||
p.add_argument("--split", default="Train_app10", help="MovieNet annotation split")
|
||||
p.add_argument("--title", help="MovieNet title tt-id to filter to")
|
||||
p.add_argument("--gallery", help="gallery.json for masking (gallery ∩ GT)")
|
||||
p.add_argument("--crosswalk", help="tmdb→imdb JSON (tmdb_imdb_map.py) for exact "
|
||||
"id join when pred/gallery lack imdb_id")
|
||||
p.add_argument("--no-mask", action="store_true", help="disable gallery∩GT masking")
|
||||
p.add_argument("--step", type=float, default=1.0, help="regular grid step (s)")
|
||||
p.add_argument("--random", type=int, help="sample N uniform-random timepoints")
|
||||
p.add_argument("--scene-anchored", action="store_true",
|
||||
help="sample GT scene midpoints (one vote per scene)")
|
||||
p.add_argument("--end", type=float, help="timeline end (s); default = max of pred/GT")
|
||||
p.add_argument("--seed", type=int, default=0)
|
||||
p.add_argument("--json-out", help="write full metrics (incl. per-point) here")
|
||||
args = p.parse_args()
|
||||
|
||||
crosswalk = CrosswalkTable.load(args.crosswalk) if args.crosswalk else None
|
||||
if crosswalk is not None:
|
||||
print(f"[eval] crosswalk: {len(crosswalk)} tmdb→imdb entries", file=sys.stderr)
|
||||
|
||||
pred = Prediction(args.pred, crosswalk=crosswalk)
|
||||
print(f"[eval] pred: {len(pred.actors)} actors, timeline≈{pred.max_t:.0f}s "
|
||||
f"({pred.movie})", file=sys.stderr)
|
||||
|
||||
if args.xray:
|
||||
gt = XRayGroundTruth(args.xray)
|
||||
else:
|
||||
if not args.title:
|
||||
sys.exit("[eval] --movienet requires --title tt-id")
|
||||
gt = _load_movienet(args.movienet, args.split, args.title, args.gallery)
|
||||
print(f"[eval] GT: {gt.summary()}", file=sys.stderr)
|
||||
|
||||
mask = None
|
||||
if not args.no_mask:
|
||||
gkeys = load_gallery_keys(args.gallery, crosswalk=crosswalk)
|
||||
gt_keys = gt.all_keys()
|
||||
if gkeys is None:
|
||||
# no gallery given → mask to GT ∩ pred key spaces so absent-from-gallery
|
||||
# GT actors don't inflate FN. Fall back to GT keys the pred could name.
|
||||
mask = gt_keys & pred.all_keys()
|
||||
print("[eval] no --gallery; masking to GT∩pred keys "
|
||||
f"({len(mask)})", file=sys.stderr)
|
||||
else:
|
||||
mask = gkeys & gt_keys
|
||||
print(f"[eval] mask = gallery∩GT ({len(mask)} keys)", file=sys.stderr)
|
||||
|
||||
points = sample_points(args, pred, gt)
|
||||
print(f"[eval] sampling {len(points)} timepoints "
|
||||
f"({'scene-anchored' if args.scene_anchored else 'random' if args.random else f'grid@{args.step}s'})",
|
||||
file=sys.stderr)
|
||||
|
||||
m = score(pred, gt, points, mask)
|
||||
print("\n── presence eval ─────────────────────────────")
|
||||
print(f" timepoints : {m['n_points']}")
|
||||
print(f" TP/FP/FN : {m['TP']} / {m['FP']} / {m['FN']}")
|
||||
print(f" precision : {m['precision']*100:.1f}%")
|
||||
print(f" recall : {m['recall']*100:.1f}%")
|
||||
print(f" F1 : {m['f1']*100:.1f}%")
|
||||
|
||||
if args.json_out:
|
||||
out = {k: v for k, v in m.items() if k != "per_point"}
|
||||
out["per_point"] = [{"t": t, "tp": tp, "fp": fp, "fn": fn}
|
||||
for t, tp, fp, fn in m["per_point"]]
|
||||
Path(args.json_out).write_text(json.dumps(out, indent=2))
|
||||
print(f"[eval] wrote {args.json_out}", file=sys.stderr)
|
||||
|
||||
|
||||
def _load_movienet(root, split, title, gallery):
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parent.parent))
|
||||
from movienet_prep import load_movienet_annotations
|
||||
anns = load_movienet_annotations(Path(root), split)
|
||||
anns = [a for a in anns if a["img_path"].startswith(title)]
|
||||
if not anns:
|
||||
sys.exit(f"[eval] no MovieNet annotations for title {title} in {split}")
|
||||
id_to_name = {}
|
||||
if gallery:
|
||||
for a in json.load(open(gallery)).get("actors", []):
|
||||
if a.get("imdb_id"):
|
||||
id_to_name[a["imdb_id"]] = a.get("name", "")
|
||||
return MovieNetGroundTruth(anns, id_to_name)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,148 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Self-contained tests for the presence eval. Builds a synthetic X-Ray fixture and
|
||||
a pipeline-output JSON in a temp dir, then checks scoring, masking, name-fallback
|
||||
matching, and sampling modes. Run: python scripts/validation/test_sample_eval.py
|
||||
"""
|
||||
|
||||
import csv
|
||||
import json
|
||||
import sys
|
||||
import tempfile
|
||||
from pathlib import Path
|
||||
|
||||
HERE = Path(__file__).resolve().parent
|
||||
sys.path.insert(0, str(HERE))
|
||||
|
||||
from identity import keys_for, norm_name # noqa: E402
|
||||
from sample_eval import Prediction, score, sample_points, _match_count # noqa: E402
|
||||
from ground_truth import XRayGroundTruth # noqa: E402
|
||||
from tmdb_imdb_map import CrosswalkTable # noqa: E402
|
||||
|
||||
|
||||
def _write_xray(d: Path):
|
||||
d.mkdir(parents=True, exist_ok=True)
|
||||
with open(d / "scenes.csv", "w", newline="") as f:
|
||||
w = csv.writer(f); w.writerow(["scene", "start_ms", "end_ms"])
|
||||
w.writerows([("1", 0, 60000), ("2", 300000, 340000)])
|
||||
# real Zenodo X-Ray schema: people(name_id,person,character);
|
||||
# people_in_scenes(scene,start,end,name_id,timestamp)
|
||||
with open(d / "people.csv", "w", newline="") as f:
|
||||
w = csv.writer(f); w.writerow(["name_id", "person", "character"])
|
||||
w.writerows([("nm0330687", "Lauren Graham", "Lorelai"),
|
||||
("nm0004754", "Alexis Bledel", "Rory"),
|
||||
("nm0000001", "Ghost Actor", "Ghost")])
|
||||
with open(d / "people_in_scenes.csv", "w", newline="") as f:
|
||||
w = csv.writer(f); w.writerow(["scene", "start", "end", "name_id", "timestamp"])
|
||||
w.writerows([("1", 0, 60000, "nm0330687", 5000),
|
||||
("1", 0, 60000, "nm0000001", 8000),
|
||||
("2", 300000, 340000, "nm0004754", 305000)])
|
||||
|
||||
|
||||
def _write_pred(path: Path):
|
||||
# pipeline output carries tmdb+name but NO nm ids -> name-fallback join to X-Ray
|
||||
doc = {"schema_version": 1, "movie": "x", "sample_fps": 1.0, "anneal_sec": 10.0,
|
||||
"actors": [
|
||||
{"name": "Lauren Graham", "imdb_id": "", "tmdb_id": "16858",
|
||||
"jellyfin_id": "a", "scenes": [[21.0, 31.0], [50.0, 59.0]]},
|
||||
{"name": "Alexis Bledel", "imdb_id": "", "tmdb_id": "6279",
|
||||
"jellyfin_id": "b", "scenes": [[301.0, 311.0]]},
|
||||
{"name": "Edward Herrmann", "imdb_id": "", "tmdb_id": "52995",
|
||||
"jellyfin_id": "c", "scenes": [[4.0, 56.0]]}]}
|
||||
path.write_text(json.dumps(doc))
|
||||
|
||||
|
||||
class T:
|
||||
n = 0
|
||||
def check(self, cond, msg):
|
||||
T.n += 1
|
||||
assert cond, f"FAIL: {msg}"
|
||||
print(f" ok: {msg}")
|
||||
|
||||
|
||||
def main():
|
||||
t = T()
|
||||
|
||||
# -- identity ---------------------------------------------------------------
|
||||
t.check(norm_name("Zöe Saldaña!") == "zoe saldana", "accent/punct normalization")
|
||||
t.check(keys_for(imdb_id="nm1", name="Jo Ann") == {"imdb:nm1", "name:jo ann"},
|
||||
"keys_for builds namespaced tokens")
|
||||
t.check(keys_for(imdb_id="") == set(), "empty ids dropped")
|
||||
|
||||
# -- crosswalk: tmdb-only actor gains an exact imdb: key --------------------
|
||||
xw = CrosswalkTable({"16858": "nm0330687", "999": None})
|
||||
k = keys_for(imdb_id="", tmdb_id="16858", name="Lauren Graham", crosswalk=xw)
|
||||
t.check("imdb:nm0330687" in k, "crosswalk resolves tmdb→imdb key")
|
||||
k_null = keys_for(tmdb_id="999", crosswalk=xw)
|
||||
t.check(not any(x.startswith("imdb:") for x in k_null), "crosswalk null → no imdb key")
|
||||
t.check(len(xw) == 1, "CrosswalkTable len counts non-null entries")
|
||||
|
||||
# -- matching ---------------------------------------------------------------
|
||||
t.check(_match_count([{"name:jo"}, {"name:al"}], [{"imdb:x", "name:jo"}]) == 1,
|
||||
"one match by shared name key")
|
||||
t.check(_match_count([{"name:jo"}], [{"name:jo"}, {"name:jo"}]) == 1,
|
||||
"greedy 1:1 uses each GT once")
|
||||
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
tmp = Path(tmp)
|
||||
_write_xray(tmp / "xray")
|
||||
_write_pred(tmp / "pred.json")
|
||||
|
||||
pred = Prediction(tmp / "pred.json")
|
||||
gt = XRayGroundTruth(tmp / "xray")
|
||||
|
||||
# -- presence lookups ---------------------------------------------------
|
||||
t.check(len(pred.present_at(25)) == 2, "Graham+Herrmann present at 25s")
|
||||
t.check(len(pred.present_at(305)) == 1, "only Bledel present at 305s")
|
||||
t.check(len(gt.present_at(30)) == 2, "X-Ray scene1 has Graham+Ghost at 30s")
|
||||
t.check(len(gt.present_at(320)) == 1, "X-Ray scene2 has Bledel at 320s")
|
||||
|
||||
# -- masking behaviour --------------------------------------------------
|
||||
pts = [30.0]
|
||||
# unmasked: at 30s pred={Graham,Herrmann}, gt={Graham,Ghost}
|
||||
# match Graham -> TP1; Herrmann unmatched -> FP1; Ghost unmatched -> FN1
|
||||
m = score(pred, gt, pts, mask=None)
|
||||
t.check((m["TP"], m["FP"], m["FN"]) == (1, 1, 1), "unmasked 30s = 1/1/1")
|
||||
|
||||
# masked to GT∩pred keys: Ghost & Herrmann are absent from the other side's
|
||||
# key space, so both drop -> only Graham remains on both -> 1/0/0
|
||||
gt_keys = gt.all_keys(); pred_keys = pred.all_keys()
|
||||
mask = gt_keys & pred_keys
|
||||
m2 = score(pred, gt, pts, mask=mask)
|
||||
t.check((m2["TP"], m2["FP"], m2["FN"]) == (1, 0, 0),
|
||||
"masked 30s drops off-gallery actors = 1/0/0")
|
||||
|
||||
# -- crosswalk end-to-end: exact imdb join, names garbled ---------------
|
||||
# Rebuild pred with names that WON'T match X-Ray, but a crosswalk that maps
|
||||
# their tmdb ids to the correct nm ids. Match must survive via imdb key.
|
||||
garbled = {"schema_version": 1, "movie": "x", "actors": [
|
||||
{"name": "WRONG NAME A", "imdb_id": "", "tmdb_id": "16858",
|
||||
"jellyfin_id": "a", "scenes": [[21.0, 31.0]]}, # →nm0330687 Graham
|
||||
{"name": "WRONG NAME B", "imdb_id": "", "tmdb_id": "6279",
|
||||
"jellyfin_id": "b", "scenes": [[301.0, 311.0]]}]} # →nm0004754 Bledel
|
||||
(tmp / "garbled.json").write_text(json.dumps(garbled))
|
||||
xw = CrosswalkTable({"16858": "nm0330687", "6279": "nm0004754"})
|
||||
pred_g = Prediction(tmp / "garbled.json", crosswalk=xw)
|
||||
# at 30s (scene1) Graham should match by imdb despite wrong name
|
||||
m_g = score(pred_g, gt, [30.0], mask=None)
|
||||
t.check(m_g["TP"] == 1, "crosswalk yields exact imdb match despite wrong names")
|
||||
# without crosswalk, wrong names → no match at all
|
||||
pred_bad = Prediction(tmp / "garbled.json")
|
||||
m_bad = score(pred_bad, gt, [30.0], mask=None)
|
||||
t.check(m_bad["TP"] == 0, "no crosswalk + wrong names → no match")
|
||||
|
||||
# -- sampling modes -----------------------------------------------------
|
||||
class A: # arg stub
|
||||
scene_anchored = True; random = None; step = 1.0; end = None; seed = 0
|
||||
sp = sample_points(A, pred, gt)
|
||||
t.check(sp == [30.0, 320.0], "scene-anchored samples scene midpoints")
|
||||
|
||||
A.scene_anchored = False; A.end = 10.0; A.step = 2.0
|
||||
grid = sample_points(A, pred, gt)
|
||||
t.check(grid == [0.0, 2.0, 4.0, 6.0, 8.0, 10.0], "regular grid step")
|
||||
|
||||
print(f"\nALL {T.n} CHECKS PASSED")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,135 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
tmdb_imdb_map.py — build & consult a cached tmdb_person_id → imdb_id crosswalk.
|
||||
|
||||
The gallery (and pipeline output) key actors by TMDB person id but carry no IMDb
|
||||
`nm…` id, while the X-Ray / MovieNet ground truth keys on IMDb. Rather than join on
|
||||
fuzzy names, we resolve tmdb→imdb once via TMDB's authoritative
|
||||
`/person/{id}/external_ids` endpoint and cache the result to JSON. The eval loaders
|
||||
consult this table to add an exact `imdb:` key alongside each `tmdb:` key.
|
||||
|
||||
Table format (JSON): { "<tmdb_person_id>": "nm0000123" | null, ... }
|
||||
A null means "looked up, TMDB has no IMDb id" — cached so we don't re-query.
|
||||
|
||||
Build / refresh the table:
|
||||
python scripts/validation/tmdb_imdb_map.py \
|
||||
--gallery gallery_arcface_w600k_r50.json \
|
||||
--out scripts/validation/tmdb_imdb.json
|
||||
# TMDB_API_KEY read from env / .env (via sae_env)
|
||||
|
||||
Consult it from code:
|
||||
from tmdb_imdb_map import CrosswalkTable
|
||||
tbl = CrosswalkTable.load("scripts/validation/tmdb_imdb.json")
|
||||
nm = tbl.imdb_for("35467") # -> "nm..." or None
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
_HERE = Path(__file__).resolve().parent
|
||||
|
||||
|
||||
class CrosswalkTable:
|
||||
"""Read-only view over the cached tmdb→imdb JSON. Missing file → empty table."""
|
||||
|
||||
def __init__(self, mapping: dict[str, str | None]):
|
||||
self._m = mapping
|
||||
|
||||
@classmethod
|
||||
def load(cls, path: str | Path) -> "CrosswalkTable":
|
||||
p = Path(path)
|
||||
if not p.exists():
|
||||
return cls({})
|
||||
return cls(json.loads(p.read_text()))
|
||||
|
||||
def imdb_for(self, tmdb_id) -> str | None:
|
||||
if tmdb_id is None:
|
||||
return None
|
||||
return self._m.get(str(tmdb_id))
|
||||
|
||||
def __len__(self) -> int:
|
||||
return sum(1 for v in self._m.values() if v)
|
||||
|
||||
|
||||
# ── builder ─────────────────────────────────────────────────────────────────
|
||||
|
||||
def _external_ids(tmdb_get, tmdb_person_id: str, token: str) -> str | None:
|
||||
data = tmdb_get(f"/person/{tmdb_person_id}/external_ids", token)
|
||||
imdb = data.get("imdb_id")
|
||||
return imdb or None # normalize "" → None
|
||||
|
||||
|
||||
def build(gallery_path: str, out_path: str, token: str, sleep: float = 0.0) -> None:
|
||||
# import the existing TMDB helper (scripts/ is the parent dir)
|
||||
sys.path.insert(0, str(_HERE.parent))
|
||||
from sae_tmdb import tmdb_get
|
||||
|
||||
with open(gallery_path) as f:
|
||||
actors = json.load(f).get("actors", [])
|
||||
tmdb_ids = sorted({str(a["tmdb_id"]) for a in actors if a.get("tmdb_id")})
|
||||
print(f"[map] gallery tmdb ids: {len(tmdb_ids)}", file=sys.stderr)
|
||||
|
||||
out = Path(out_path)
|
||||
existing: dict[str, str | None] = {}
|
||||
if out.exists():
|
||||
existing = json.loads(out.read_text())
|
||||
print(f"[map] resuming from {len(existing)} cached entries", file=sys.stderr)
|
||||
|
||||
todo = [t for t in tmdb_ids if t not in existing]
|
||||
print(f"[map] to resolve: {len(todo)}", file=sys.stderr)
|
||||
|
||||
n_ok = n_none = n_err = 0
|
||||
for i, tid in enumerate(todo, 1):
|
||||
try:
|
||||
nm = _external_ids(tmdb_get, tid, token)
|
||||
existing[tid] = nm
|
||||
n_ok += (nm is not None)
|
||||
n_none += (nm is None)
|
||||
except Exception as e: # network/rate-limit/404 — record nothing, keep going
|
||||
n_err += 1
|
||||
print(f"\n[map] error on tmdb {tid}: {e}", file=sys.stderr)
|
||||
if i % 25 == 0 or i == len(todo):
|
||||
print(f"\r[map] {i}/{len(todo)} resolved "
|
||||
f"(imdb={n_ok} none={n_none} err={n_err})", end="", file=sys.stderr)
|
||||
out.write_text(json.dumps(existing, indent=2)) # checkpoint
|
||||
if sleep:
|
||||
time.sleep(sleep)
|
||||
|
||||
out.write_text(json.dumps(existing, indent=2))
|
||||
print(f"\n[map] wrote {out} — {sum(1 for v in existing.values() if v)} imdb ids",
|
||||
file=sys.stderr)
|
||||
|
||||
|
||||
def main():
|
||||
# load .env → os.environ (same convention as the gallery builders)
|
||||
sys.path.insert(0, str(_HERE.parent))
|
||||
try:
|
||||
import sae_env # noqa: F401 (side-effect import)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
p = argparse.ArgumentParser(description=__doc__,
|
||||
formatter_class=argparse.RawDescriptionHelpFormatter)
|
||||
p.add_argument("--gallery", required=True, help="gallery.json (source of tmdb ids)")
|
||||
p.add_argument("--out", default=str(_HERE / "tmdb_imdb.json"),
|
||||
help="output crosswalk JSON (default: scripts/validation/tmdb_imdb.json)")
|
||||
p.add_argument("--tmdb-key", default=os.environ.get("TMDB_API_KEY"),
|
||||
help="TMDB v3 API key or v4 read token. Env: TMDB_API_KEY")
|
||||
p.add_argument("--sleep", type=float, default=0.0,
|
||||
help="seconds between requests (TMDB has no hard limit; use if throttled)")
|
||||
args = p.parse_args()
|
||||
|
||||
if not args.tmdb_key:
|
||||
sys.exit("[map] no TMDB key — set TMDB_API_KEY or pass --tmdb-key")
|
||||
|
||||
build(args.gallery, args.out, args.tmdb_key, args.sleep)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in New Issue
Block a user