diff --git a/.gitignore b/.gitignore index 71be62c..fa61ea8 100644 --- a/.gitignore +++ b/.gitignore @@ -78,7 +78,16 @@ docs_site/ # Images staged into docs/ from the artifact registry at build time # (scripts/docs/build_site.sh) — not committed, pulled fresh on each build. -docs/assets/ +# Exception: pipeline_topology.svg is small and hand-authored (not pulled from +# anywhere) and the README references it directly, so it needs to render on a +# plain Gitea repo view too, not just the built Pages site. (A directory-level +# ignore can't be un-ignored file-by-file below it, so this must NOT blanket- +# ignore docs/assets/ itself — only its contents, minus the one exception.) +docs/assets/images/* +!docs/assets/images/pipeline_topology.svg +# readme_example.jpg is referenced directly by README.md, which renders on the +# plain Gitea repo view — committed for the same reason as the SVG above. +!docs/assets/images/readme_example.jpg # Python __pycache__/ diff --git a/README.md b/README.md index bed079f..a682b4b 100644 --- a/README.md +++ b/README.md @@ -1,13 +1,25 @@ # Scene Actor Extraction -Identifies actors in movie files and produces X-ray-style scene annotations compatible with [Jellyfin](https://jellyfin.org/). Built on a KPN++ pipeline with ArcFace embeddings and a tracked-identity matcher. +Identifies actors in movie files and produces X-ray-style scene annotations compatible with [Jellyfin](https://jellyfin.org/). Built on a KPN++ pipeline with ArcFace/LVFace embeddings and a tracked-identity matcher. + +**67.4% macro-F1 against Amazon X-Ray ground truth**, on 5 films never seen by +the optimizer (89.7% P / 65.4% R training-set; see the generalization-gap +discussion in the [deep dive](https://pages.tourolle.paris/dtourolle/scene-actor-extraction/lvface-deep-dive/)). +Full benchmark write-up, model comparison, and failure-mode analysis: +**https://pages.tourolle.paris/dtourolle/scene-actor-extraction/** + +![Example: correctly identified actors in a held-out film](docs/assets/images/readme_example.jpg) +*A held-out film (never used for threshold tuning) — three actors correctly +identified with calibrated confidence scores.* ## How it works -1. **Build a gallery** — download actor headshots from TMDB/IMDB, embed them with ArcFace (`build_gallery` / `scripts/make_gallery.py`). +1. **Build a gallery** — download actor headshots from TMDB/IMDB, embed them with ArcFace or LVFace (`build_gallery` / `scripts/make_gallery.py`). 2. **Analyze a movie** — `scene_analyze` decodes frames at configurable FPS, detects faces (SCRFD), tracks them across cuts, matches identities against the gallery using calibrated similarity, and writes time-window JSON. 3. **Output** — minimal mode produces Jellyfin-ready actor name + time-window JSON; standard mode adds per-frame bbox, similarity, and track data. +![Pipeline topology](docs/assets/images/pipeline_topology.svg) + ## Dependencies | Dependency | Role | @@ -63,7 +75,7 @@ contract (112×112 aligned BGR crop → L2-normalised 512-d embedding) and its ```bash ./build/scene_analyze --arcface-model models/LVFace-B_Glint360K.onnx \ - --gallery gallery.json --input movie.mp4 + --gallery gallery.h5 --movie movie.mp4 ``` > **Important:** embeddings from different recognition models are not @@ -97,7 +109,7 @@ bash scripts/download_models.sh ### `scene_analyze` ```bash -./build/scene_analyze --gallery gallery.json --input movie.mp4 [options] +./build/scene_analyze --gallery gallery.h5 --movie movie.mp4 [options] ``` Key options: @@ -118,7 +130,7 @@ Key options: **Per-movie (TMDB):** ```bash -python3 scripts/make_gallery.py --tmdb-bearer --movie-id --output gallery.json +python3 scripts/make_gallery.py --tmdb-key --movie-id --output gallery.h5 ``` Fetches cast images from TMDB and embeds them via `sae_embed`. @@ -129,13 +141,13 @@ Fetches cast images from TMDB and embeds them via `sae_embed`. python3 scripts/make_jellyfin_gallery.py \ --jellyfin-url http://jellyfin.local:8096 \ --api-key \ - --output gallery.json + --output gallery.h5 ``` Scans every Movie/Series in Jellyfin, collects the unique cast across the whole library, downloads each actor's headshot directly from Jellyfin (no TMDB key needed), and embeds them via `sae_embed` into one global -gallery.json. Since `identity_matcher` scores faces against the entire +gallery.h5. Since `identity_matcher` scores faces against the entire gallery, `scene_analyze` can then recognise any actor from your library in any film — not just the cast listed for that one title. Pass `--merge` on later runs to only embed actors newly added to the library. Pass @@ -153,11 +165,11 @@ look-alike mismatches), filter the global gallery first: ```bash python3 scripts/filter_gallery.py \ - --gallery gallery.json \ + --gallery gallery.h5 \ --jellyfin-url http://jellyfin.local:8096 \ --api-key \ --title "The Matrix" \ - --output gallery_matrix.json + --output gallery_matrix.h5 ``` ## Running directly from Jellyfin @@ -172,7 +184,7 @@ python3 scripts/run_from_jellyfin.py \ --jellyfin-url http://jellyfin.local:8096 \ --api-key \ --title "The Matrix" \ - --gallery gallery.json \ + --gallery gallery.h5 \ -- --fps 5 --verbosity 2 ``` @@ -200,7 +212,7 @@ poll the same library concurrently. python3 scripts/run_from_jellyfin.py \ --jellyfin-url http://jellyfin.local:8096 \ --api-key \ - --gallery whole_gallery.json \ + --gallery whole_gallery.h5 \ --worker \ -- --fps 5 ``` @@ -223,15 +235,6 @@ worker moves on to the next item rather than exiting. **Standard** — per-frame detail with bounding boxes, similarity scores, and track IDs. -## Pipeline topology - -``` -frame_source → face_detector → face_aligner → embedder - → face_tracker → identity_matcher → scene_tracker → result_sink -``` - -Debug/preview branches fan out automatically from `identity_matcher`. - ## Evaluation Scripts in `eval/` and `scripts/movienet_*.py` support benchmarking against the MovieNet dataset. diff --git a/docs/assets/images/pipeline_topology.svg b/docs/assets/images/pipeline_topology.svg new file mode 100644 index 0000000..d48fc5e --- /dev/null +++ b/docs/assets/images/pipeline_topology.svg @@ -0,0 +1,59 @@ + + + + + + + + + + + + + + frame_source + + + face_detector + + + face_aligner + + + embedder + + + face_tracker + + + identity_matcher + + + + + + + + + + + scene_tracker + + + result_sink + + + + + debug_renderer / + preview (opt-in) + + solid = always-on data path · dashed = optional debug/preview fan-out from identity_matcher's output + diff --git a/docs/assets/images/readme_example.jpg b/docs/assets/images/readme_example.jpg new file mode 100644 index 0000000..1e2c23a Binary files /dev/null and b/docs/assets/images/readme_example.jpg differ diff --git a/docs/best-model.md b/docs/best-model.md index 3e3ff0e..8bd1c9d 100644 --- a/docs/best-model.md +++ b/docs/best-model.md @@ -9,7 +9,7 @@ open question: is LVFace (455MB) actually better, or just the biggest? Each gallery carries a fitted Platt sigmoid `P(match | cosine similarity) = σ(a·sim + b)`, embedded directly in the gallery's HDF5 file -(`src/gallery/gallery_calibration.hpp`). This is a property of the embedding +([`src/gallery/gallery_calibration.hpp`](https://REPOLINK/src/gallery/gallery_calibration.hpp)). This is a property of the embedding space alone — computed from intra/inter-actor reference-image pairs, no tracking or scene logic involved — so it's a clean first read on discriminative power before running a single benchmark. @@ -31,7 +31,7 @@ head start before the tracking/scoring pipeline is even involved. ## Second signal: F1 on the actual benchmark Best full-gallery (no cast-restriction) result per model, from the 16-combo -rep4 matrix (`rep4-optimizer-results.md`): +bake-off matrix ([full experiment log](model-bakeoff.md)): | model | F1 | P | R | misID | |---|---|---|---|---| @@ -40,16 +40,24 @@ rep4 matrix (`rep4-optimizer-results.md`): | ArcFace R18 | 69.1% | 87.6% | 57.7% | 242 | | ArcFace w600k-R50 | 68.5% | 94.0% | 54.1% | 150 | +The full 16-combo picture makes the model ordering visible at a glance — LVFace +(yellow) tops both the restricted and full columns, and R18 (green) props up +the bottom of the full-gallery ranking: + +![All 16 bake-off combos ranked by training-set F1](assets/images/rep4_matrix_f1.png) + LVFace wins outright, with the highest recall of any full-mode combo. This reverses an earlier conclusion from a prior (superseded) benchmarking pass using a scene-union metric, which found the three models statistically indistinguishable (~85% each) and concluded LVFace wasn't worth its size — that metric hid out-of-cast false positives behind a gallery∩cast recall mask (see -`optimizer-experiments.md`); the per-second metric used here does not. +[the prior optimizer round](optimizer-experiments.md)); the per-second metric +used here does not. Held-out validation (5 films never seen by the optimizer) confirms LVFace's -lead holds up out of sample — see the deep-dive page for the full breakdown, -including where it fails. +lead holds up out of sample — see the +[LVFace deep dive](lvface-deep-dive.md) for the full breakdown, including +where it fails. ## Caveat: model choice is an operational change @@ -57,5 +65,6 @@ Switching the default embedder isn't just flipping a config value — the gallery itself is model-specific (embeddings from different models aren't comparable), so any existing gallery built against ArcFace w600k-R50 needs to be rebuilt from source images against LVFace before the new default takes -effect. `scripts/optimizer/reembed_gallery.py` does this from a reference -gallery's cached source images without re-downloading anything. +effect. [`scripts/optimizer/reembed_gallery.py`](https://REPOLINK/scripts/optimizer/reembed_gallery.py) +does this from a reference gallery's cached source images without +re-downloading anything. diff --git a/docs/gallery-scope.md b/docs/gallery-scope.md index 926e7b6..5833381 100644 --- a/docs/gallery-scope.md +++ b/docs/gallery-scope.md @@ -7,7 +7,7 @@ pre-filters each film's gallery down to just its Jellyfin-credited cast (typical ## The result -Averaged across all 4 models and both expansion settings, on the 4 rep4 training +Averaged across all 4 models and both expansion settings, on the 4 bake-off training films: | scope | F1 | P | R | total misID (8 evals) | @@ -23,7 +23,13 @@ with someone in the film, but isn't actually in it), and the recall gain shows it isn't costing real detections to get there. Per-model, every single model's best-scoring combo in the full 16-way matrix is -a `restricted` variant — see the full table in `rep4-optimizer-results.md`. Two +a `restricted` variant — visible directly in the ranking below (filled dots = +restricted, open = full; the filled dots cluster at the top for every color): + +![All 16 bake-off combos — filled dots (restricted) dominate the top](assets/images/rep4_matrix_f1.png) + +See the full table in the +[bake-off experiment log](model-bakeoff.md). Two combos hit **zero** true out-of-cast misidentifications: `arcface_w600k_mbf_restricted_exp` (F1 76.5%) and, in full mode, `LVFace-B_Glint360K_full_noexp` (F1 72.4%) — restriction isn't the only way to @@ -32,7 +38,8 @@ reach misid=0, but it's the more reliable one. ## Why this isn't the shipped default Cast-restriction is implemented today only as an **offline optimizer technique** -(`scripts/optimizer/cast_restrict.py`): it pre-builds a filtered gallery file +([`scripts/optimizer/cast_restrict.py`](https://REPOLINK/scripts/optimizer/cast_restrict.py)): +it pre-builds a filtered gallery file per film, using Jellyfin's own cast list, before the benchmark ever calls the matcher. There's no runtime "restrict matching to this title's credited cast" switch in the shipped application — `scene_analyze` always matches against @@ -41,7 +48,8 @@ whatever single gallery file it's given. Building that as a real feature would need, at minimum: - A live Jellyfin cast lookup at analysis time (the title is already known — - `run_from_jellyfin.py` already does this same lookup for its own + [`scripts/run_from_jellyfin.py`](https://REPOLINK/scripts/run_from_jellyfin.py) + already does this same lookup for its own `filter_gallery`-based restriction path, just not wired into `scene_analyze` itself as a first-class option). - A decision on the *fallback*: what happens to a real, uncredited cameo @@ -50,7 +58,8 @@ Building that as a real feature would need, at minimum: - Regenerating the restricted-gallery cache whenever the title's Jellyfin cast list changes. -This is why the shipped `src/config.hpp` defaults use the `full`-mode winner +This is why the shipped [`src/config.hpp`](https://REPOLINK/src/config.hpp) +defaults use the `full`-mode winner (`LVFace-B_Glint360K_full_exp`, F1 75.3% training / 67.4% held-out macro) rather than the higher-scoring `restricted_exp` (78.3%) — the 78.3% number describes a capability the app doesn't have yet, not what actually ships. diff --git a/docs/index.md b/docs/index.md index 52159f6..37cf8b6 100644 --- a/docs/index.md +++ b/docs/index.md @@ -5,6 +5,16 @@ film or TV episode — built on [KPN++](https://gitea.tourolle.paris/dtourolle/K (a C++20 Kahn Process Network library) for the detect → track → match → scene pipeline, with a Jellyfin-integrated gallery and an X-Ray-validated optimizer. +This is what a good second looks like — one sampled frame from a held-out film, +19 faces named, all 19 correct, the rest honestly declared unknown: + +![19 correct identifications in one wedding shot, Downton Abbey: A New Era](assets/images/downton_wedding_19_correct.jpg) + +And this is why the work isn't done: on this same film the same config misses +6 in 10 of the actor-seconds X-Ray says are present, and on the worst held-out +film it reports ghost actors over empty walls — at 100% confidence. Both +stories, with the evidence, are in the pages below. + ## Start here — four questions this bake-off answers - **[Which model is best?](best-model.md)** — calibration curves first @@ -24,14 +34,14 @@ pipeline, with a Jellyfin-integrated gallery and an X-Ray-validated optimizer. ## The full technical log -- **[Rep4 model bake-off + threshold re-tune](rep4-optimizer-results.md)** — +- **[Model bake-off + threshold re-tune](model-bakeoff.md)** — the complete experiment log behind the four pages above: the ROCm teardown deadlock root cause and fix, DE concurrency tuning, the full 16-combo - results table, and every caveat. This is where the shipped `src/config.hpp` - defaults come from. + results table, and every caveat. This is where the shipped + [`src/config.hpp`](https://REPOLINK/src/config.hpp) defaults come from. - **[Optimizer experiments (prior round)](optimizer-experiments.md)** — the earlier scene-union-metric tuning pass, superseded by the per-second metric - used in rep4 but kept for the ground-truth/architecture background. + used in the bake-off but kept for the ground-truth/architecture background. - **[Service conversion (proposal)](service-conversion.md)** — design sketch for an idle-GPU Docker worker, not yet built. @@ -47,5 +57,5 @@ scripts/artifacts/pull_artifacts.sh experiment-data scripts/artifacts/pull_artifacts.sh montage-frames ``` -See `scripts/artifacts/push_artifacts.sh` for the upload side (requires a -`GITEA_TOKEN` with package write scope). +See [`scripts/artifacts/push_artifacts.sh`](https://REPOLINK/scripts/artifacts/push_artifacts.sh) +for the upload side (requires a `GITEA_TOKEN` with package write scope). diff --git a/docs/lvface-deep-dive.md b/docs/lvface-deep-dive.md index 161064a..e4fec6f 100644 --- a/docs/lvface-deep-dive.md +++ b/docs/lvface-deep-dive.md @@ -1,9 +1,28 @@ # Deep dive: LVFace-B Glint360K -LVFace won the model bake-off (see `best-model.md`) and is the shipped default -embedder. This page is the honest accounting of how it actually performs — -including where it's wrong, and one case where the ground truth itself is -wrong and LVFace is right. +LVFace won the model bake-off (see [Which model is best?](best-model.md)) and is +the shipped default embedder. This page is the honest accounting of how it +actually performs — what a good second looks like, where it's wrong and *why*, +and one case where the ground truth itself is wrong and LVFace is right. + +## What good looks like + +Before the failure analysis, the ceiling. This is a single sampled second from +Downton Abbey's wedding scene — a packed, hat-heavy, period-costume group shot, +about as hostile as ensemble framing gets: + +![19 correct identifications in one wedding shot, Downton Abbey: A New Era](assets/images/downton_wedding_19_correct.jpg) +*Frame `downton_abbey/best/best_t00127.jpg` from the `montage-frames` artifact +package (`scripts/artifacts/pull_artifacts.sh montage-frames +Downton_Abbey__A_New_Era`) — green = identified, blue = detected but unknown.* + +**Nineteen named faces in one frame, all nineteen correct** — Jim Carter half +behind a flower arrangement, Penelope Wilton at a three-quarter turn, Lesley +Nicol under a hat brim. The blue "unknown" boxes are the honest cases: faces the +detector found but the matcher declined to name rather than guess. The one miss +at this second is Maggie Smith — not on screen in this framing, but X-Ray marks +her present for the scene. That distinction (on-screen face vs. scene-level +ground truth) sets up everything below. ## Training vs. held-out: the generalization gap @@ -11,6 +30,8 @@ The shipped config (`prob_threshold=0.754, anneal_sec=35.54, extinction_sec=57.43, expand_gallery=true`) was tuned against 4 films. Scored against the 5 films the optimizer never saw: +![Held-out per-film F1 vs. the training-set fit](assets/images/holdout_f1_by_film.png) + | film | F1 | P | R | TPI | FPI | misid | FN | |---|---|---|---|---|---|---|---| | Benny & Joon | 83.0% | 89.1% | 77.7% | 15125 | 1846 | 0 | 4337 | @@ -22,9 +43,10 @@ against the 5 films the optimizer never saw: **67.4% held-out vs. 75.3% on training** — an ~8pp drop, and a **37pp spread between the best and worst held-out film**. The config does not generalize -uniformly; two films are outright failure cases, for two different reasons. +uniformly; two films are outright failure cases, for reasons that turn out to +be one mechanism. -## Failure mode 1: frozen-bbox "ghost tracks" +## The failure mode: frozen-bbox "ghost tracks" Both Many Saints of Newark (974 misIDs) and Downton Abbey (FN=80084, the worst recall of the five) trace to the same root cause, verified directly against @@ -36,51 +58,69 @@ inferred from the score alone. At this second, three of the four labeled boxes ("Jon Bernthal", "Joey Diaz", "Billy Magnussen") sit over empty background — a blurred wall, hanging plates — with no face in them. The real face in frame carries a second, -colliding label from another frozen box. +colliding label from another frozen box. And it isn't an isolated second — the +same signature recurs throughout the film: -![15 ghost boxes over a blank title card, Downton Abbey: A New Era](assets/images/downton_abbey_ghost_fpi.jpg) +![Ghost labels over a staircase while real faces stay honest unknowns](assets/images/many_saints_ghosts_vs_unknowns.jpg) +*Frame `many_saints_intervals/w002_worst_t01382.jpg`, same artifact package — +one frame, three distinct error classes.* -This is the starkest case: **15 actors named, all wrong, over a completely -blank closing title card.** Confirmed against the dump directly: `face_count` -is 0 from this point onward (no detector output at all), yet the same 15 -identities keep appearing with the *exact same bounding box, unchanged to the -pixel*, for 57+ consecutive seconds. +This frame is worth reading closely, because it separates three things that a +single aggregate F1 number smears together. The two green labels ("Jon Bernthal +100%", "Michela De Rossi 100%") float over a staircase and a policeman's back — +frozen boxes from a previous shot, reported at full confidence. Meanwhile the +two *real* frontal faces in frame get honest blue "unknown 0%" boxes (they're +uncredited day-players with no gallery reference — the +[gallery coverage gap](gallery-scope.md)), and three more people simply face +away from camera, invisible to any face detector but still "present" in X-Ray's +scene-level ground truth. Precision failure, gallery-coverage failure, and the +face-vs-presence ceiling — one frame. + +### The mechanism, measured + +The starkest case is Downton Abbey's hard cut from a packed group shot into a +long blank credits sequence. Plotting the detector's per-second `face_count` +(from the dump HDF5, independent of the tracker) against what the tracker +reports makes the failure legible at a glance: + +![Detector vs. tracker through Downton Abbey's cut to credits](assets/images/downton_ghost_timeline.png) + +From the cut onward the detector sees **zero faces for nearly a minute** — and +the tracker keeps reporting the last group shot's 15 identities the entire +time, each with the *exact same bounding box, unchanged to the pixel* (verified +for Hugh Bonneville: `(1743.2, 0.0, 171.3, 317.8)` at every sampled second for +57+ seconds). The staircase decay at the right edge is the extinction window +finally expiring, actor by actor. This is `SceneTrackerFunc::active_[actor_idx].last_bbox` -(`src/nodes/scene_tracker_node.hpp`) being re-emitted unchanged — the -extinction state machine working exactly as coded, not a bug. The film cuts -from a packed group shot straight into 40+ seconds of blank titles/credits, -and `extinction_sec=57.4` is comfortably long enough to bridge that entire gap -without expiring, so the tracker faithfully reports "last known position" for -a cast that is no longer on screen at all. `extinction_sec` was tuned toward -long windows specifically because they bridge real gaps (occlusion, a turned -face) in most training footage — this is the cost side of that trade, -surfacing only when a film has a long enough faceless stretch to expose it. - -## Failure mode 2: a genuine misID (for contrast) - -Not every held-out failure is a ghost. This is a real face, correctly -detected, confidently misidentified: - -*(same many_saints_ghost_fpi.jpg frame above also shows Leslie Odom Jr.'s box -carrying a second, colliding "Michael Gandolfini" label — two real tracks' -frozen positions happening to overlap, not a detection error.)* +([`src/nodes/scene_tracker_node.hpp`](https://REPOLINK/src/nodes/scene_tracker_node.hpp)) +being re-emitted unchanged — the +extinction state machine working exactly as coded, not a bug. `extinction_sec` +was tuned to 57.4s specifically because long windows bridge real gaps +(occlusion, a turned face) in most footage; a hard cut into long faceless +footage is the one case where that same bridging manufactures ghosts, and the +training films never contained one long enough to punish it. The optimizer +"discovered" the plateau at the top of its search range for a reason that only +generalizes to films that never go faceless for a minute. ## Where LVFace beat X-Ray -Not every "misID" is actually wrong. `second_score.py` counts a name as a true -out-of-cast misID whenever the named actor isn't in X-Ray's credited cast list -for the film at all — but X-Ray's cast list is itself incomplete. +Not every flagged "misID" is actually wrong. +[`scripts/optimizer/second_score.py`](https://REPOLINK/scripts/optimizer/second_score.py) +counts a name as a true out-of-cast misID whenever the named actor isn't in +X-Ray's credited cast list for the film at all — but X-Ray's cast list is +itself incomplete. ![LVFace correctly identifies Germar Terrell Gardner, uncredited by X-Ray](assets/images/germar_beats_xray.jpg) Germar Terrell Gardner — a real, clean, high-confidence detection — is counted as a misID here because he doesn't appear in X-Ray's `people.csv` for The Many Saints of Newark at all. But Jellyfin's independent cast metadata *does* credit -him for this exact film (cross-checked via `experiments/manifests/ -jellyfin_casts.json`, a completely separate data source from X-Ray). This -isn't a lookalike error or a gallery mixup — it's the pipeline correctly -recognising a real cast member that one ground-truth source happened to omit. +him for this exact film (cross-checked via +`experiments/manifests/jellyfin_casts.json` from the `experiment-data` artifact +package, a completely separate data source from X-Ray). This isn't a lookalike error or a gallery mixup — it's the +pipeline correctly recognising a real cast member that one ground-truth source +happened to omit. This doesn't mean every flagged misID is secretly correct — Many Saints' 974-count total is still overwhelmingly the frozen-bbox failure mode above, @@ -91,8 +131,9 @@ in the other direction too. ## Summary -LVFace is the right default: it wins the model comparison outright, and its -failures are traceable, understood, and mostly attributable to one tunable -knob (`extinction_sec`) rather than the embedder itself. The held-out +LVFace is the right default: it wins the model comparison outright, it can name +19 faces correctly in a single hostile group shot, and its failures are +traceable, understood, and mostly attributable to one tunable knob +(`extinction_sec`) rather than the embedder itself. The held-out generalization gap (75.3% → 67.4%) is real and should be treated as the honest expected performance, not the training-set number. diff --git a/docs/rep4-optimizer-results.md b/docs/model-bakeoff.md similarity index 77% rename from docs/rep4-optimizer-results.md rename to docs/model-bakeoff.md index 3ed2e15..633c794 100644 --- a/docs/rep4-optimizer-results.md +++ b/docs/model-bakeoff.md @@ -1,11 +1,16 @@ -# Rep4 model bake-off + threshold re-tune — experiment log (2026-07-18/19) +# Model bake-off + threshold re-tune — experiment log (2026-07-18/19) -Follow-on to `docs/optimizer-experiments.md`, which used an older, since-superseded -scene-union metric. This round uses the **per-second** metric -(`scripts/optimizer/second_score.py`) and answers three questions in one 16-run -matrix: which embedding model is best, does cast-restriction cut misIDs, and does +Follow-on to [the prior optimizer round](optimizer-experiments.md), which used +an older, since-superseded scene-union metric. This round uses the **per-second** metric +([`scripts/optimizer/second_score.py`](https://REPOLINK/scripts/optimizer/second_score.py)) +and answers three questions in one 16-run matrix: which embedding model is best, does cast-restriction cut misIDs, and does per-film gallery expansion help. +(This was the fourth optimizer campaign against the X-Ray corpus, so its on-disk +artifacts carry an internal `rep4_` prefix — `experiments/results/rep4_best_*.json`, +`experiments/trajectories/rep4_*.jsonl`, and the manifests referenced below. The +earlier campaigns used the superseded scene-union metric and were discarded.) + ## Why this experiment, and what it actually delivered Four goals going in, and an honest read on each after held-out validation (see @@ -40,14 +45,15 @@ below): | `anneal_sec` | 10.0 | **35.5** | Same reversal; previously thought insensitive. | | `expand_gallery` | false | **true** | Helped recall on the full (unrestricted) gallery for the winning model — opposite of the earlier assumption. | -These are the **`LVFace-B_Glint360K_full_exp`** winning values — the best result that +These are the **`LVFace-B_Glint360K_full_exp`** winning values, applied to +[`src/config.hpp`](https://REPOLINK/src/config.hpp) — the best result that uses only features already live in the running app (full gallery, no cast restriction; see below for why restricted mode isn't applied even though it scored higher). ## Why re-run at all -`docs/optimizer-experiments.md`'s scene-union metric hid out-of-cast false positives +[The prior round](optimizer-experiments.md)'s scene-union metric hid out-of-cast false positives behind a gallery∩cast recall mask, and the earlier 9-film benchmark was scene-level (union over a whole X-Ray scene), not a fair per-timepoint comparison. SESSION_STATE flagged the old scene-metric bake-off numbers (R50≈LVFace≈MBF ~85%) as superseded. @@ -55,19 +61,22 @@ This round uses `second_score.py`: uniform per-second sampling, GT = X-Ray scene cast at time *t*, pred = actors whose presence window covers *t*, FPI weighted 10× when the named actor isn't in the film's cast at all (true misID) vs. an in-cast timing slip. FN only counts gallery-known cast (fair recall — 67% of X-Ray cast have -no reference embedding, see `gallery-coverage-gap` memory). +no reference embedding; see +[the prior round's gallery-coverage-gap analysis](optimizer-experiments.md#the-gallery-coverage-gap)). ## The deadlock that was blocking all of this -Every replay in this line of work goes through `scripts/optimizer/replay.py`, which -runs the real C++ tracker/matcher/scene_tracker nodes inside a Python-assembled KPN -network. Before this session, every subprocess replay **timed out at 45s, 100% of +Every replay in this line of work goes through +[`scripts/optimizer/replay.py`](https://REPOLINK/scripts/optimizer/replay.py), +which runs the real C++ tracker/matcher/scene_tracker nodes inside a +Python-assembled KPN network. Before this session, every subprocess replay **timed out at 45s, 100% of the time** — not the documented ~20-30% ROCm rocBLAS-GEMM driver flake, but a plain logic bug: `replay.py`'s CLI called `replay(net, ..., stop=False)` to *skip* `net.stop()` (trying to dodge the GEMM deadlock), planning to `os._exit(0)` immediately after. But: -- `PyNode::stop()` (`external/KPN/include/kpn/python/bindings.hpp`) is the *only* +- `PyNode::stop()` ([`include/kpn/python/bindings.hpp`](https://KPNLINK/include/kpn/python/bindings.hpp) + in the KPN++ submodule) is the *only* code that sets `stop_flag_ = true` before joining the node's worker thread. - The source node's `run_loop()` has `while (!stop_flag_)` as its only exit condition (it has no input channels, so it never sees a channel-closed signal @@ -90,7 +99,9 @@ F1/precision/recall instead of flat 0.0%. ## Concurrency tuning -With the deadlock fixed, `optimize.py` was extended with DE-level parallelism — +With the deadlock fixed, +[`scripts/optimizer/optimize.py`](https://REPOLINK/scripts/optimizer/optimize.py) +was extended with DE-level parallelism — `differential_evolution(..., workers=ThreadPoolExecutor.map)` — so multiple population candidates evaluate concurrently, each spawning its own per-film replay subprocesses (`REPLAY_WORKERS`). Total concurrent GPU replay processes ≈ @@ -108,10 +119,10 @@ actually being garbage — a dangerous failure mode, not a crash. **8 concurrent practical ceiling** on this GPU (gfx1100) for this workload. The matrix ran at `REPLAY_WORKERS=4 DE_WORKERS=2`. -## Training films (rep4) and validation set +## Training films and validation set 9 films total have dumped embeddings across all 4 models. 4 were used for -optimization (rep4), leaving 5 held out for validation: +optimization, leaving 5 held out for validation: - **Lord of War** (64-cast, "clean") - **Scarface** (67-cast, "ensemble/lookalike") @@ -121,7 +132,7 @@ optimization (rep4), leaving 5 held out for validation: Downton Abbey or The Many Saints of Newark) Held out: Benny & Joon, Downton Abbey: A New Era, Lovelace, The Many Saints of -Newark, Valerian and the City of a Thousand Planets. The rep4 numbers below are +Newark, Valerian and the City of a Thousand Planets. The training-set numbers below are training-set fit — see "Held-out validation" further down for the real generalization test. @@ -162,10 +173,17 @@ includes in-cast timing slips. † old, narrower anneal/extinction bounds (see above) — not directly comparable to the other 12 on those two params. +The same 16 results as a picture — the two headline effects are visible without +reading a single row: filled (restricted) dots stack the top of the ranking for +every model color, and yellow (LVFace) leads within both scopes: + +![All 16 bake-off combos ranked by training-set F1](assets/images/rep4_matrix_f1.png) + ## Calibration curves — discriminative power, independent of the threshold Each model's gallery carries a fitted Platt sigmoid `P(match | sim) = σ(a·sim + b)` -(embedded directly in the gallery HDF5, see `src/gallery/gallery_calibration.hpp`). +(embedded directly in the gallery HDF5, see +[`src/gallery/gallery_calibration.hpp`](https://REPOLINK/src/gallery/gallery_calibration.hpp)). Plotting all four side by side shows discriminative power directly, independent of whatever `prob_threshold` a particular run happened to use: @@ -175,8 +193,10 @@ LVFace-B has both the steepest curve (`a=17.7`, vs. 15.3–16.2 for the ArcFace variants) and the lowest P=0.5 decision boundary (similarity 0.23 vs. 0.27–0.31) — it separates same-actor from different-actor pairs more confidently at a lower similarity, consistent with it winning the full-gallery F1 comparison below. -Generated by `scripts/docs/calibration_chart.py` (requires each gallery to have -been calibrated at least once — run any replay against it first). +Generated by +[`scripts/docs/calibration_chart.py`](https://REPOLINK/scripts/docs/calibration_chart.py) +(requires each gallery to have been calibrated at least once — run any replay +against it first). ## Two effects in isolation: gallery scope, and pose expansion @@ -200,7 +220,8 @@ choice itself — which is exactly why cast-restriction becoming a real runtime feature (not just an optimizer trick) is the top item in Caveats below. **Pose expansion (promoting a confidently-identified track's novel-pose views into -a per-film gallery annex — `track_gallery.hpp`)** is smaller and interacts with +a per-film gallery annex — [`src/gallery/track_gallery.hpp`](https://REPOLINK/src/gallery/track_gallery.hpp))** +is smaller and interacts with scope rather than acting independently: | scope | expansion | F1 | R | misID | @@ -259,6 +280,22 @@ support in the app yet (see Caveats). Decided not to chase further this round (diminishing-returns judgment call) — flag for a future sweep if it matters. +The ceiling-pinning is visible in the raw search itself. Every one of the 512 +DE evaluations for the winning combo, plotted over the +`prob_threshold` × `extinction_sec` plane: + +![DE search landscape: 512 evaluations over prob_threshold × extinction_sec](assets/images/de_search_landscape.png) + +The dark band hugging the top edge *is* the finding: nearly everything scoring +well sits at `extinction_sec` ≥ 50, across a wide range of thresholds, and the +population converged into a dense cloud around the optimum (threshold ~0.70–0.80, +extinction pinned at the 60s bound). Short extinction windows (bottom half) are +uniformly pale — under a strict threshold there is simply no good configuration +down there. Generated by +[`scripts/docs/experiment_charts.py`](https://REPOLINK/scripts/docs/experiment_charts.py) +from the DE trajectories (`experiments/trajectories/*.jsonl`, part of the +`experiment-data` artifact package). + ## Held-out validation — the number that actually matters The 16-combo matrix above is training-set fit. This is the real test: the shipped @@ -276,12 +313,15 @@ Saints of Newark, Valerian and the City of a Thousand Planets), scored the same | **The Many Saints of Newark** | **46.3%** | **54.7%** | 40.1% | 37.0% | 15922 | 4394 | **974** | 23791 | | **macro average (5 films)** | **67.4%** | 85.8% | 57.0% | 56.2% | 116727 | 9046 | 1032 | 127764 | +![Held-out per-film F1 vs. the training-set fit](assets/images/holdout_f1_by_film.png) + **67.4% held out vs. 75.3% on training** — an ~8pp drop, and a much more informative number than the training-set F1 alone: a **37pp spread between best and worst film** (83.0% vs 46.3%). The config does not generalize uniformly. Two films are outright failure cases, and rendering bounding boxes + names on the -extracted frames (`replay.py --raw-out` + `dump_error_frames.py --raw`, see +extracted frames (`replay.py --raw-out` + +[`dump_error_frames.py`](https://REPOLINK/scripts/optimizer/dump_error_frames.py)` --raw`, see Reproduce) turned what looked like a same-scene misidentification into something more precise and more damning: @@ -296,29 +336,36 @@ more precise and more damning: happen to overlap. ![Frozen ghost boxes over background, The Many Saints of Newark](assets/images/many_saints_ghost_fpi.jpg) - *`experiments/results/holdout/frames/many_saints/fpi/fpi_t03543.jpg`* + *Frame `many_saints/fpi/fpi_t03543.jpg` from the `montage-frames` artifact + package (`scripts/artifacts/pull_artifacts.sh montage-frames + Many_Saints_of_Newark`).* - **Downton Abbey: A New Era** (large ensemble, 36-cast) has high precision (97.8%) - but recall collapses to 39.4% (FN=80084, by far the largest of the 5). The frame - below is the starkest evidence in this whole experiment: **the matcher named 15 - actors — all of them wrong — over a completely blank closing title card with no - faces on screen at all.** + but recall collapses to 39.4% (FN=80084, by far the largest of the 5). Its + starkest failure happens where there is nothing to see at all: the film's hard + cut into its closing credits, where **the matcher kept reporting 15 actors — + all wrong — for nearly a minute of faceless screen.** - ![15 ghost boxes over a blank title card, Downton Abbey: A New Era](assets/images/downton_abbey_ghost_fpi.jpg) - *`experiments/results/holdout/frames/downton_abbey/fpi/fpi_t07242.jpg`* +Both are the same mechanism, and it can be *measured*, not just screenshotted. +Plotting the dump's own per-second `face_count` (detector output, independent +of the tracker) against the number of actors the tracker reports, through +Downton Abbey's cut to credits: -Both are the same mechanism, verified directly against the HDF5 dump and the raw -per-frame stream (not just inferred from the screenshot): at the Downton Abbey -title card (t=7242), the dump's own `face_count` is **0 from t≈7240 onward** — no -detector output at all, confirmed independently of the pipeline. Yet all 15 "wrong" -actors are still marked visible, each with the *exact same bbox, unchanged to the -pixel*, repeated every single frame back to t=7222 (verified for Hugh Bonneville: -`(1743.2, 0.0, 171.3, 317.8)` at every sampled second from 7222 through 7279+). That -is `SceneTrackerFunc`'s `active_[actor_idx].last_bbox` (`scene_tracker_node.hpp`) -being re-emitted unchanged — **this is the extinction state machine working exactly -as coded**, not a bug in the logic. The film cuts from a packed group shot straight -into ~40+ seconds of blank titles/credits with zero faces, and `extinction_sec=57.4` -is comfortably long enough to bridge that entire gap without expiring, so the +![Detector face_count vs. tracker-reported actors through the cut to credits](assets/images/downton_ghost_timeline.png) + +From the cut onward the detector sees **zero faces** — yet the tracker holds a +perfectly flat plateau of 15 reported identities for 56 seconds, each with the +*exact same bbox, unchanged to the pixel* (verified for Hugh Bonneville: +`(1743.2, 0.0, 171.3, 317.8)` at every sampled second from 7222 through 7279+). +The staircase on the right edge is the extinction window finally expiring, +actor by actor. That plateau is `SceneTrackerFunc`'s +`active_[actor_idx].last_bbox` +([`src/nodes/scene_tracker_node.hpp`](https://REPOLINK/src/nodes/scene_tracker_node.hpp)) +being re-emitted +unchanged — **the extinction state machine working exactly as coded**, not a +bug in the logic. The film cuts from a packed group shot straight into ~40+ +seconds of blank titles/credits with zero faces, and `extinction_sec=57.4` is +comfortably long enough to bridge that entire gap without expiring, so the tracker faithfully keeps reporting "last known position" for a cast that is no longer on screen at all. @@ -331,11 +378,14 @@ was chosen to avoid. The training-set films apparently didn't have a long enough faceless stretch after a confirmed identity to expose this; the held-out set did. Frames for all three films (`benny_joon`, `many_saints`, `downton_abbey` — one strong -performer, two failure cases) are under `experiments/results/holdout/frames/`, each +performer, two failure cases) are under `experiments/results/holdout/frames/` +(not committed — pull per film with `scripts/artifacts/pull_artifacts.sh +montage-frames `), each with a `manifest.json` listing the bucket (`best`/`fpi`/`fn`), timestamp, and predicted vs. ground-truth actors for every dumped frame. Frames are annotated with bounding boxes + name/confidence (green = identified, orange = unknown), matching -`debug_renderer_node.hpp`'s colour convention. Generated by +[`src/nodes/debug_renderer_node.hpp`](https://REPOLINK/src/nodes/debug_renderer_node.hpp)'s +colour convention. Generated by `scripts/optimizer/dump_error_frames.py --raw ` (see Reproduce). @@ -387,6 +437,10 @@ python3 scripts/optimizer/replay.py \ --out pred.json --raw-out raw.jsonl --prob-threshold 0.754 --anneal-sec 35.54 \ --extinction-sec 57.43 --expand-gallery +# regenerate the report's charts (16-combo ranking, DE landscape, held-out +# per-film F1, Downton ghost timeline) from the artifacts under experiments/ +python3 scripts/docs/experiment_charts.py --out-dir docs/assets/images + # dump example frames (best-agreement / FPI / FN) for visual inspection, annotated # with bounding boxes + names (--raw is optional; omit for unannotated frames) python3 scripts/optimizer/dump_error_frames.py \ @@ -403,6 +457,6 @@ python3 scripts/optimizer/dump_error_frames.py \ --out-dir experiments/results/holdout/frames/_intervals --interval-sec 600 ``` -See also: `docs/optimizer-experiments.md` (prior round, superseded metric), -`experiments/SESSION_STATE.md`, and memory: kpn-python-replay-optimizer, -gallery-coverage-gap, xray-validation-results, per-scene-presence-eval-design. +See also: [the prior optimizer round](optimizer-experiments.md) (superseded +metric) and the session log +[`experiments/SESSION_STATE.md`](https://REPOLINK/experiments/SESSION_STATE.md). diff --git a/docs/optimizer-experiments.md b/docs/optimizer-experiments.md index 327e318..22b8b77 100644 --- a/docs/optimizer-experiments.md +++ b/docs/optimizer-experiments.md @@ -45,13 +45,17 @@ GT set = actors X-Ray lists for that scene. Per scene TP/FP/FN, then: scenes count more) → **equal-weight mean across movies** (macro; each film counts the same regardless of length). This is the DE objective. -Implemented in `scripts/optimizer/scene_score.py`. +Implemented in `scripts/optimizer/scene_score.py` — since **removed** along +with this metric; its per-second successor is +[`scripts/optimizer/second_score.py`](https://REPOLINK/scripts/optimizer/second_score.py) +(see the [bake-off round](model-bakeoff.md)). ## The gallery coverage gap Diagnosing low recall: only **131 of 392** X-Ray cast were in the gallery (33%). Every in-gallery actor HAD embeddings (gallery well-formed) — the gap was pure coverage. -`scripts/optimizer/fetch_missing_actors.py` recovers missing actors: +[`scripts/optimizer/fetch_missing_actors.py`](https://REPOLINK/scripts/optimizer/fetch_missing_actors.py) +recovers missing actors: `nm-id → TMDB /find external_ids → /person/{id}/images → download → embed (sae_embed)`, with a `--wikidata` fallback (P345→P18 Commons photo). @@ -71,7 +75,8 @@ face-recognition pipeline vs X-Ray's presence semantics, not a fixable gap. ## Optimizer -`scripts/optimizer/optimize.py` — scipy `differential_evolution` over the knob space, +[`scripts/optimizer/optimize.py`](https://REPOLINK/scripts/optimizer/optimize.py) +— scipy `differential_evolution` over the knob space, each candidate = full replay of all films through the **real** C++ nodes (see the KPN replay architecture below) scored by the metric above. Global objective (one config for all films, not per-film). @@ -91,7 +96,8 @@ the tightly-converged knobs were adopted as defaults. The optimizer never re-decodes video. `scene_analyze --dump-embeddings out.h5` runs the expensive half once (decode→detect→align→embed) and dumps per-frame face embeddings -+ metadata to HDF5 (`scripts/optimizer/SCHEMA.md`). `scripts/optimizer/replay.py` then ++ metadata to HDF5 ([`scripts/optimizer/SCHEMA.md`](https://REPOLINK/scripts/optimizer/SCHEMA.md)). +[`scripts/optimizer/replay.py`](https://REPOLINK/scripts/optimizer/replay.py) then replays that dump through the **real** C++ `face_tracker → identity_matcher → scene_tracker` assembled in a Python KPN network (`sae_kpn` nanobind module), varying Config knobs freely — no GPU embedding, no decode. Verified BYTE-EXACT against @@ -110,8 +116,10 @@ python scripts/optimizer/optimize.py --manifest films.json --gallery gallery.jso --params prob_threshold:0.5:0.999 anneal_sec:1:30 extinction_sec:1:15 \ --popsize 8 --maxiter 20 --trajectory traj.jsonl --out opt.json # 4. score a fixed config / validate on a held-out set -python scripts/optimizer/score_config.py --manifest heldout.json --gallery gallery.json \ - --config '{"prob_threshold":0.76,"extinction_sec":1.5,"anneal_sec":10}' +# (historical: score_config.py and scene_score.py were removed with the +# scene-union metric — use scripts/optimizer/second_score.py, per-second) +python scripts/optimizer/second_score.py --help ``` -See also memory: kpn-python-replay-optimizer, gallery-coverage-gap, xray-validation-*. +Superseded by the [model bake-off + re-tune](model-bakeoff.md), which +replaced this round's scene-union metric with per-second scoring. diff --git a/docs/pose-expansion.md b/docs/pose-expansion.md index 3be9853..b8a5659 100644 --- a/docs/pose-expansion.md +++ b/docs/pose-expansion.md @@ -1,6 +1,7 @@ # Pose expansion: does "learning" new poses mid-film help? -`expand_gallery` (`src/gallery/track_gallery.hpp`) promotes a confidently-identified +`expand_gallery` ([`src/gallery/track_gallery.hpp`](https://REPOLINK/src/gallery/track_gallery.hpp)) +promotes a confidently-identified track's novel-pose reference views into a per-film, in-memory gallery annex — the idea being that once the pipeline is sure who someone is, a pose it hasn't seen before (turned head, different lighting) becomes a free extra reference for @@ -21,8 +22,8 @@ Averaged across all 4 models, on the 4 films used for optimization: In `restricted` mode (matcher's candidate set capped to the film's own credited cast) expansion looked like a clean win: +1.8pp F1, +3.2pp recall, misID actually lower. In `full` mode it looked flat-to-costly: ~0 F1 change, recall +1.4pp, but -misID roughly quadrupled (209 → 864) — see `rep4-optimizer-results.md` for the -per-model breakdown. That's the number that motivated this page: **does turning +misID roughly quadrupled (209 → 864) — see the +[bake-off experiment log](model-bakeoff.md) for the per-model breakdown. That's the number that motivated this page: **does turning expansion on actually change what gets recognised, frame by frame, or is the aggregate F1 shift something else?** @@ -83,7 +84,8 @@ mode) doesn't reproduce on held-out data — at minimum it's far smaller than th training-set numbers suggested, and plausibly it's sampling variation from only 4 training films rather than a real, generalizable mechanism. This doesn't mean `expand_gallery` never does anything (the mechanism is real — see -`track_gallery.hpp`'s promotion logging: tracks *do* get confirmed and views *do* +[`track_gallery.hpp`](https://REPOLINK/src/gallery/track_gallery.hpp)'s +promotion logging: tracks *do* get confirmed and views *do* get promoted into the annex on every film tested), only that **whatever effect it has on final per-second identification was too small to detect against 5 held-out films** with this scoring method. A cleaner test would need either many @@ -92,6 +94,7 @@ more held-out films or a metric that can see the annex's direct contribution pass had budget for. **Practical takeaway**: don't treat the training-set `exp` vs `noexp` numbers in -`rep4-optimizer-results.md` as proof that expansion changes real-world behavior +the [bake-off experiment log](model-bakeoff.md) as proof that expansion +changes real-world behavior in either direction — on the evidence gathered so far, it doesn't move the needle enough to see. diff --git a/docs/service-conversion.md b/docs/service-conversion.md index 1979109..d45d9de 100644 --- a/docs/service-conversion.md +++ b/docs/service-conversion.md @@ -34,12 +34,12 @@ lock-gating, not new pipeline logic. | Piece | Where | What it does | |---|---|---| | Analysis engine | `build/scene_analyze` | Video → face detect/align/embed → gallery match → result JSON | -| Backend selection | `CMakeLists.txt` (`SAE_INFERENCE_BACKEND`, `SAE_GEMM_BACKEND`) | ORT/TRT + ROCm/CUDA, chosen **at build time** | +| Backend selection | [`CMakeLists.txt`](https://REPOLINK/CMakeLists.txt) (`SAE_INFERENCE_BACKEND`, `SAE_GEMM_BACKEND`) | ORT/TRT + ROCm/CUDA, chosen **at build time** | | New-media queue | JRay plugin → `GET /Plugins/JRay/Tasks/Pending` | Backlog of items with no results yet | -| Worker loop | `run_from_jellyfin.py --worker` | Poll Pending → run `scene_analyze` → push results | +| Worker loop | [`scripts/run_from_jellyfin.py`](https://REPOLINK/scripts/run_from_jellyfin.py)` --worker` | Poll Pending → run `scene_analyze` → push results | | Result push | `PUT /Plugins/JRay/Items/{id}/Truth` | Stores per-actor scene windows back in Jellyfin | -| Incremental gallery | `make_jellyfin_gallery.py --merge` | Embeds only cast not already in the gallery | -| Secrets loader | `.env` via `sae_env.py` | `JELLYFIN_URL`, `JELLYFIN_API_KEY`, `TMDB_API_KEY` | +| Incremental gallery | [`scripts/make_jellyfin_gallery.py`](https://REPOLINK/scripts/make_jellyfin_gallery.py)` --merge` | Embeds only cast not already in the gallery | +| Secrets loader | `.env` via [`scripts/sae_env.py`](https://REPOLINK/scripts/sae_env.py) | `JELLYFIN_URL`, `JELLYFIN_API_KEY`, `TMDB_API_KEY` | ## Installer config diff --git a/mkdocs.yml b/mkdocs.yml index 782034e..46b67b3 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -23,7 +23,7 @@ nav: - Gallery Scope (Full vs. Limited): gallery-scope.md - Pose Expansion: pose-expansion.md - LVFace Deep Dive: lvface-deep-dive.md - - Rep4 Bake-off & Re-tune (full log): rep4-optimizer-results.md + - Model Bake-off & Re-tune (full log): model-bakeoff.md - Optimizer Experiments (prior round): optimizer-experiments.md - Service Conversion (proposal): service-conversion.md diff --git a/scripts/docs/build_site.sh b/scripts/docs/build_site.sh index eeed81b..a5e3f40 100755 --- a/scripts/docs/build_site.sh +++ b/scripts/docs/build_site.sh @@ -11,7 +11,7 @@ cd "$REPO_ROOT" ASSETS_DIR="docs/assets/images" mkdir -p "$ASSETS_DIR" -# Frames referenced by docs/rep4-optimizer-results.md. Pull the film's montage +# Frames referenced by docs/model-bakeoff.md. Pull the film's montage # frames from the registry if this machine doesn't already have them locally. FRAMES_ROOT="experiments/results/holdout/frames" if [ ! -d "$FRAMES_ROOT/many_saints" ] || [ ! -d "$FRAMES_ROOT/downton_abbey" ]; then @@ -23,14 +23,29 @@ fi echo "==> staging referenced frames into ${ASSETS_DIR}" cp -v "${FRAMES_ROOT}/many_saints/fpi/fpi_t03543.jpg" \ "${ASSETS_DIR}/many_saints_ghost_fpi.jpg" -cp -v "${FRAMES_ROOT}/downton_abbey/fpi/fpi_t07242.jpg" \ - "${ASSETS_DIR}/downton_abbey_ghost_fpi.jpg" +cp -v "${FRAMES_ROOT}/downton_abbey/best/best_t00127.jpg" \ + "${ASSETS_DIR}/downton_wedding_19_correct.jpg" +if [ -f "${FRAMES_ROOT}/many_saints_intervals/w002_worst/w002_worst_t01382.jpg" ]; then + cp -v "${FRAMES_ROOT}/many_saints_intervals/w002_worst/w002_worst_t01382.jpg" \ + "${ASSETS_DIR}/many_saints_ghosts_vs_unknowns.jpg" +else + echo "WARN: many_saints_intervals frames not present; keeping existing" \ + "${ASSETS_DIR}/many_saints_ghosts_vs_unknowns.jpg (if any)" +fi if [ ! -f "${ASSETS_DIR}/germar_beats_xray.jpg" ]; then echo "==> pulling report-highlights/germar_beats_xray.jpg..." scripts/artifacts/pull_artifacts.sh report-highlights germar_beats_xray.jpg fi +if [ ! -f "${ASSETS_DIR}/readme_example.jpg" ]; then + echo "==> pulling report-highlights/readme_example.jpg..." + scripts/artifacts/pull_artifacts.sh report-highlights readme_example.jpg +fi + +# pipeline_topology.svg is small and hand-authored (not pulled from anywhere) — +# committed directly at docs/assets/images/, not staged from the registry. + if [ ! -d experiments/galleries ] || [ -z "$(ls -A experiments/galleries 2>/dev/null)" ]; then echo "==> pulling galleries (not found locally)..." scripts/artifacts/pull_artifacts.sh galleries @@ -39,7 +54,44 @@ fi echo "==> generating calibration curve chart" python3 scripts/docs/calibration_chart.py --out "${ASSETS_DIR}/calibration_curves.png" +echo "==> generating experiment charts (16-combo ranking, DE landscape, held-out F1, ghost timeline)" +python3 scripts/docs/experiment_charts.py --out-dir "${ASSETS_DIR}" + echo "==> building site" mkdocs build +# -- commit-pinned repo links ------------------------------------------------- +# Docs reference repo files via the placeholder hosts https://REPOLINK/ +# (this repo) and https://KPNLINK/ (the KPN++ submodule). Substitute them +# with raw URLs pinned to the exact commit being published, and fail the build +# if any linked path doesn't actually exist at that commit — no dead links. +HEAD_SHA="$(git rev-parse HEAD)" +KPN_SHA="$(git rev-parse HEAD:external/KPN)" +REPO_RAW="https://gitea.tourolle.paris/dtourolle/scene-actor-extraction/raw/commit/${HEAD_SHA}" +KPN_RAW="https://gitea.tourolle.paris/dtourolle/KPN/raw/commit/${KPN_SHA}" + +if [ -n "$(git status --porcelain -- docs scripts src experiments)" ]; then + echo "WARN: working tree is dirty — commit-pinned links will point at ${HEAD_SHA}," >&2 + echo " which may not contain your latest changes. Commit before deploying." >&2 +fi + +echo "==> verifying repo-linked paths exist at ${HEAD_SHA}" +missing=0 +for p in $(grep -rhoE 'https://REPOLINK/[A-Za-z0-9_./-]+' docs/*.md | sed 's|https://REPOLINK/||' | sort -u); do + if ! git cat-file -e "HEAD:${p}" 2>/dev/null; then + echo "error: docs link to '${p}', which does not exist at HEAD" >&2 + missing=1 + fi +done +[ "$missing" -eq 0 ] || exit 1 + +echo "==> pinning repo links to ${HEAD_SHA} (KPN: ${KPN_SHA})" +find site -name '*.html' -exec \ + sed -i "s|https://REPOLINK|${REPO_RAW}|g; s|https://KPNLINK|${KPN_RAW}|g" {} + + +if grep -rq 'REPOLINK\|KPNLINK' site; then + echo "error: unsubstituted REPOLINK/KPNLINK placeholder left in site/" >&2 + exit 1 +fi + echo "==> done. site/ is ready to deploy to the gitea-pages branch." diff --git a/scripts/docs/deploy_pages.sh b/scripts/docs/deploy_pages.sh new file mode 100755 index 0000000..9ec68f8 --- /dev/null +++ b/scripts/docs/deploy_pages.sh @@ -0,0 +1,43 @@ +#!/bin/bash +# deploy_pages.sh — push the built site/ to an orphan gitea-pages branch, +# matching the convention Gitea Pages serves from +# (pages.tourolle.paris///). Run scripts/docs/build_site.sh first. +# +# Uses a separate worktree so the main working tree / branch is untouched. +set -euo pipefail + +REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" +cd "$REPO_ROOT" + +if [ ! -d site ]; then + echo "error: site/ not found — run scripts/docs/build_site.sh first" >&2 + exit 1 +fi + +WORKTREE="$(mktemp -d)" +trap 'rm -rf "$WORKTREE"' EXIT + +if git show-ref --verify --quiet refs/remotes/origin/gitea-pages; then + git worktree add -B gitea-pages "$WORKTREE" origin/gitea-pages +else + git worktree add --orphan -B gitea-pages "$WORKTREE" +fi + +# Replace the worktree's contents with the freshly built site. +find "$WORKTREE" -mindepth 1 -maxdepth 1 -not -name '.git' -exec rm -rf {} + +cp -r site/. "$WORKTREE/" +touch "$WORKTREE/.nojekyll" + +cd "$WORKTREE" +git add -A +if git diff --cached --quiet; then + echo "No changes to deploy (site is identical to the current gitea-pages branch)." +else + git commit -m "docs: deploy from $(git -C "$REPO_ROOT" rev-parse --short HEAD)" + git push origin gitea-pages:gitea-pages + echo "Deployed. Should be live shortly at:" + echo " https://pages.tourolle.paris/dtourolle/scene-actor-extraction/" +fi + +cd "$REPO_ROOT" +git worktree remove "$WORKTREE" --force 2>/dev/null || true diff --git a/scripts/docs/experiment_charts.py b/scripts/docs/experiment_charts.py new file mode 100644 index 0000000..deeb09c --- /dev/null +++ b/scripts/docs/experiment_charts.py @@ -0,0 +1,256 @@ +#!/usr/bin/env python3 +""" +experiment_charts.py — generate the rep4/held-out figures referenced by the docs, +from the experiment artifacts under experiments/ (no hardcoded numbers). + +Figures: + holdout_f1_by_film.png — held-out per-film F1 vs. the training-set fit + rep4_matrix_f1.png — all 16 bake-off combos, colored by model + de_search_landscape.png — DE search space: prob_threshold x extinction_sec, F1 as color + downton_ghost_timeline.png— detector face_count vs. tracker output through the credits + +Usage: + python scripts/docs/experiment_charts.py --out-dir docs/assets/images +""" +from __future__ import annotations + +import argparse +import json +from pathlib import Path + +import matplotlib.pyplot as plt +import numpy as np +from matplotlib.colors import LinearSegmentedColormap + +REPO = Path(__file__).resolve().parent.parent.parent +RESULTS = REPO / "experiments/results" + +# Same model -> color mapping as calibration_chart.py, so identity is stable +# across every figure in the report. +MODEL_COLOURS = { + "arcface_w600k_r50": ("ArcFace w600k-R50", "#2a78d6"), + "arcface_r18": ("ArcFace R18", "#008300"), + "arcface_w600k_mbf": ("ArcFace w600k-MBF", "#e87ba4"), + "LVFace-B_Glint360K": ("LVFace-B Glint360K", "#eda100"), +} +INK = "#0b0b0b" +MUTED = "#898781" +GRID = "#e1e0d9" +SURFACE = "#fcfcfb" +BLUE = "#2a78d6" +GREEN = "#008300" +RED = "#e34948" + +plt.rcParams.update({ + "figure.facecolor": SURFACE, + "axes.facecolor": SURFACE, + "savefig.facecolor": SURFACE, + "text.color": INK, + "axes.edgecolor": MUTED, + "axes.labelcolor": INK, + "xtick.color": MUTED, + "ytick.color": MUTED, + "axes.grid": True, + "grid.color": GRID, + "grid.linewidth": 0.8, + "axes.spines.top": False, + "axes.spines.right": False, + "font.size": 11, +}) + + +def training_best() -> dict: + with open(RESULTS / "rep4_best_LVFace-B_Glint360K_full_exp.json") as f: + return json.load(f)["best"] + + +def fig_holdout_f1(out: Path): + with open(RESULTS / "holdout/holdout_scores.json") as f: + films = json.load(f)["per_film"] + films = sorted(films, key=lambda d: d["f1"]) + names = [d["name"] for d in films] + f1 = [d["f1"] * 100 for d in films] + train_f1 = training_best()["f1"] * 100 + macro = float(np.mean(f1)) + + fig, ax = plt.subplots(figsize=(9, 4.2)) + ax.grid(axis="y", visible=False) + bars = ax.barh(names, f1, height=0.55, color=BLUE, zorder=3) + for b, v, d in zip(bars, f1, films): + note = f"{v:.1f}%" + if d["FPI_misid"]: + note += f" ({d['FPI_misid']} misIDs)" + ax.text(v + 1, b.get_y() + b.get_height() / 2, note, + va="center", ha="left", fontsize=10, color=INK) + ax.axvline(train_f1, color=MUTED, lw=1.5, ls="--", zorder=2) + ax.text(train_f1 + 0.7, len(names) - 0.35, f"training-set fit {train_f1:.1f}%", + color=MUTED, fontsize=9.5, ha="left", va="center") + ax.axvline(macro, color=RED, lw=1.5, ls=":", zorder=2) + ax.text(macro - 0.7, -0.72, f"held-out macro avg {macro:.1f}%", + color=RED, fontsize=9.5, ha="right", va="center") + ax.set_xlim(0, 100) + ax.set_ylim(-1.05, len(names) - 0.3 + 0.55) + ax.set_xlabel("per-second F1 (%)") + ax.set_title("Shipped config on the 5 films the optimizer never saw", + loc="left", fontsize=12, pad=12) + fig.tight_layout() + fig.savefig(out, dpi=160) + plt.close(fig) + + +def fig_rep4_matrix(out: Path): + combos = [] + for path in sorted(RESULTS.glob("rep4_best_*.json")): + stem = path.stem[len("rep4_best_"):] + for slug in MODEL_COLOURS: + if stem.startswith(slug): + mode = stem[len(slug) + 1:] # e.g. full_exp + with open(path) as f: + best = json.load(f)["best"] + combos.append((slug, mode, best["f1"] * 100)) + break + combos.sort(key=lambda c: c[2]) + + fig, ax = plt.subplots(figsize=(9, 6.2)) + ax.grid(axis="y", visible=False) + labels = [] + for i, (slug, mode, f1) in enumerate(combos): + label, colour = MODEL_COLOURS[slug] + scope, exp = mode.rsplit("_", 1) + labels.append(f"{scope} · {'expand' if exp == 'exp' else 'no expand'}") + ax.hlines(i, 50, f1, color=GRID, lw=1.2, zorder=2) + ax.plot(f1, i, "o", ms=9, color=colour, zorder=3, + mfc=colour if scope == "restricted" else SURFACE, + mec=colour, mew=2) + ax.text(f1 + 0.35, i, f"{f1:.1f}", va="center", fontsize=8.5, color=MUTED) + ax.set_yticks(range(len(combos)), labels, fontsize=9) + ax.set_xlim(65, 80) + ax.set_xlabel("training-set per-second F1 (%)") + ax.set_title("All 16 combos — filled dot = cast-restricted gallery, open = full", + loc="left", fontsize=12, pad=12) + handles = [plt.Line2D([], [], marker="o", ls="", ms=9, color=c, label=l) + for _, (l, c) in MODEL_COLOURS.items()] + ax.legend(handles=handles, loc="lower right", frameon=False, fontsize=9.5) + fig.tight_layout() + fig.savefig(out, dpi=160) + plt.close(fig) + + +def fig_de_landscape(out: Path): + evals = [] + with open(REPO / "experiments/trajectories/rep4_LVFace-B_Glint360K_full_exp.jsonl") as f: + for line in f: + d = json.loads(line) + evals.append((d["config"]["prob_threshold"], + d["config"]["extinction_sec"], d["f1"] * 100)) + x, y, f1 = map(np.array, zip(*evals)) + best = training_best() + + # one-hue sequential ramp (light -> dark blue), per the report palette + cmap = LinearSegmentedColormap.from_list( + "seq_blue", ["#cde2fb", "#86b6ef", "#3987e5", "#1c5cab", "#0d366b"]) + + fig, ax = plt.subplots(figsize=(9, 5.2)) + # clip the color scale to the top of the range — DE spends most evals near + # the optimum, so an unclipped scale renders the structure invisible + sc = ax.scatter(x, y, c=f1, cmap=cmap, s=22, linewidths=0, zorder=3, + vmin=70, vmax=float(f1.max())) + ax.plot(best["config"]["prob_threshold"], best["config"]["extinction_sec"], + marker="*", ms=18, color=RED, mec=SURFACE, mew=1.2, zorder=4) + ax.annotate(f"shipped optimum F1 {best['f1']*100:.1f}%", + (best["config"]["prob_threshold"], best["config"]["extinction_sec"]), + textcoords="offset points", xytext=(-14, -30), + ha="right", fontsize=10, color=RED, + arrowprops={"arrowstyle": "-", "color": RED, "lw": 1}) + cb = fig.colorbar(sc, ax=ax, pad=0.02) + cb.set_label("per-second F1 (%)") + cb.outline.set_visible(False) + ax.set_xlabel("prob_threshold") + ax.set_ylabel("extinction_sec") + ax.set_title("All 512 DE evaluations, LVFace-B full-gallery + expansion", + loc="left", fontsize=12, pad=12) + fig.tight_layout() + fig.savefig(out, dpi=160) + plt.close(fig) + + +def fig_downton_timeline(out: Path, t0: int = 7100, t1: int = 7340): + import h5py + + tracker = {} + with open(RESULTS / "holdout/raw_Downton_Abbey__A_New_Era.jsonl") as f: + for line in f: + d = json.loads(line) + tracker[int(d["timestamp_sec"])] = len(d["visible_actors"]) + with h5py.File(REPO / "experiments/dumps/LVFace-B_Glint360K/" + "dump_Downton_Abbey__A_New_Era.h5", "r") as h5: + ts = h5["frames/timestamp_sec"][:] + fc = h5["frames/face_count"][:] + det = {int(t): int(c) for t, c in zip(ts, fc)} + + t = np.arange(t0, t1) + # the raw stream occasionally skips a second under replay load — carry the + # last seen value forward rather than dropping to 0 + trk, last = [], 0 + for s in t: + if s in tracker: + last = tracker[s] + trk.append(last) + trk = np.array(trk) + dc = np.array([det.get(s, 0) for s in t]) + + fig, ax = plt.subplots(figsize=(9.5, 4.4)) + ax.grid(axis="x", visible=False) + ax.fill_between(t, dc, step="mid", color=GREEN, alpha=0.25, zorder=2) + ax.step(t, dc, where="mid", color=GREEN, lw=2, zorder=3) + ax.step(t, trk, where="mid", color=BLUE, lw=2, zorder=4) + + # longest contiguous run of "detector sees nothing, tracker still reporting" + ghost = (dc == 0) & (trk > 0) + runs, start = [], None + for i, g in enumerate(ghost): + if g and start is None: + start = i + elif not g and start is not None: + runs.append((start, i - 1)) + start = None + if start is not None: + runs.append((start, len(ghost) - 1)) + if runs: + i0, i1 = max(runs, key=lambda r: r[1] - r[0]) + g0, g1 = t[i0], t[i1] + ax.axvspan(g0, g1, color=RED, alpha=0.08, zorder=1) + ax.annotate(f"{g1 - g0}s of credits: 0 faces detected,\n" + f"{trk[i0]} actors still reported (frozen boxes)", + ((g0 + g1) / 2, 20.5), ha="center", va="bottom", + fontsize=10, color=RED) + ax.text(t0 + 4, 27.3, "actors reported by tracker", color=BLUE, + fontsize=10.5, va="bottom") + ax.text(t0 + 4, 11.5, "faces seen by detector", color=GREEN, + fontsize=10.5, va="bottom") + ax.set_xlabel("film time (s)") + ax.set_ylabel("count") + ax.set_ylim(0, 31) + ax.set_title("Downton Abbey: A New Era — the cut to credits, second by second", + loc="left", fontsize=12, pad=12) + fig.tight_layout() + fig.savefig(out, dpi=160) + plt.close(fig) + + +def main(): + p = argparse.ArgumentParser() + p.add_argument("--out-dir", type=Path, + default=REPO / "docs/assets/images") + args = p.parse_args() + args.out_dir.mkdir(parents=True, exist_ok=True) + + fig_holdout_f1(args.out_dir / "holdout_f1_by_film.png") + fig_rep4_matrix(args.out_dir / "rep4_matrix_f1.png") + fig_de_landscape(args.out_dir / "de_search_landscape.png") + fig_downton_timeline(args.out_dir / "downton_ghost_timeline.png") + print(f"[experiment_charts] wrote 4 figures to {args.out_dir}") + + +if __name__ == "__main__": + main() diff --git a/scripts/optimizer/dump_scene_montage.py b/scripts/optimizer/dump_scene_montage.py index 04dc006..4eb8b66 100644 --- a/scripts/optimizer/dump_scene_montage.py +++ b/scripts/optimizer/dump_scene_montage.py @@ -11,7 +11,7 @@ every currently-active TPI/FPI actor who has a REAL detection backing them, plus black caption panel below with two columns — Onscreen (has a real detection) and Offscreen (no real detection: FN misses, and "ghost" detections where the tracker is re-emitting a frozen last-known bbox with nothing there — see -docs/rep4-optimizer-results.md) — names colour-coded by bucket, with a legend. +docs/model-bakeoff.md) — names colour-coded by bucket, with a legend. A predicted bbox is checked against the dump's OWN raw per-frame face detections (IoU) to tell a real detection from a ghost. Ghosts are NEVER drawn as boxes (they