From 584f23546acd0e702e3ec8d5b2d4136f19580186 Mon Sep 17 00:00:00 2001 From: Duncan Tourolle Date: Sun, 9 Aug 2026 10:21:13 +0200 Subject: [PATCH] feat(presence): flood-fill presence mode Add PresenceMode::flood alongside the default track_extent. In flood mode the result sink snaps each presence claim to the shot it sits in, so an actor seen once anywhere in a shot is reported for the whole shot [prev_boundary, next_boundary]. This trades precision for recall against X-Ray's per-scene cast granularity and is a toggleable knob for the optimizer to weigh rather than a default. Boundaries come from the frame stream, now carried through SceneAnnotation (is_cut and is_scene_boundary). Flood prefers TransNetV2 shot boundaries when a scene detector populated them, otherwise falls back to the always-on histogram cuts (camera_position_change_detector); with no boundaries it degrades to track_extent per claim. The is_scene_boundary path stays dormant so an out-of-process scene detector can be revived later without re-wiring. Selected with --presence-mode flood|track_extent (default track_extent), so existing output is byte-for-byte unchanged. The dump_embeddings header note records why TransNetV2 scene detection is not run in that process. --- src/config.hpp | 18 ++++++++++ src/dump_embeddings.cpp | 6 ++++ src/main.cpp | 1 + src/nodes/frame_annotation_node.hpp | 7 +++- src/nodes/result_sink_node.hpp | 53 +++++++++++++++++++++++++++++ src/types.hpp | 7 ++++ 6 files changed, 91 insertions(+), 1 deletion(-) diff --git a/src/config.hpp b/src/config.hpp index e361ad0..3284f32 100644 --- a/src/config.hpp +++ b/src/config.hpp @@ -11,6 +11,20 @@ enum class Verbosity { standard, // per-frame detail: bbox, similarity, unknowns logged xray, // Jellyfin-Xray format: {"second": ["Actor", ...], ...} }; + +// How a track's accepted frames become a reported presence window. +enum class PresenceMode { + // A claim IS its track's [first_seen, last_seen] (AR-012/AR-013). The + // default and the only mode whose semantics the register validated. + track_extent, + // Flood-fill: snap each claim to the shot it sits in, so an actor seen once + // anywhere in a scene is reported for the whole scene [prev_boundary, + // next_boundary]. Trades precision for recall against X-Ray's per-scene cast + // granularity. Snaps to TransNetV2 shot boundaries (is_scene_boundary) when a + // scene detector populated them, else to the always-on histogram cuts + // (is_cut). With no boundaries at all it degrades to track_extent per claim. + flood, +}; // debug verbosity = compile with -DSAE_DEBUG → scene_analyze_debug binary struct Config { @@ -115,6 +129,10 @@ struct Config { // fallback everywhere, which is at least the same wrong number in every // stage. See identity_matcher_node.hpp. + // ── Presence derivation ────────────────────────────────────────────────── + // How accepted frames become a reported window. flood requires scene_detect. + PresenceMode presence_mode{PresenceMode::track_extent}; + // ── Cut detection ──────────────────────────────────────────────────────── float cut_threshold{0.70f}; // grayscale histogram correlation below this → hard cut diff --git a/src/dump_embeddings.cpp b/src/dump_embeddings.cpp index bce34b7..3269e3a 100644 --- a/src/dump_embeddings.cpp +++ b/src/dump_embeddings.cpp @@ -8,6 +8,12 @@ // gallery file needed. Purpose-built for the optimizer's replay corpus and the // embedding-model bake-off (dump each --arcface model over the film set). // +// The camera-position (histogram) cut detector runs, so frames/is_cut is recorded +// in the dump. Flood-fill presence snaps to those cuts. TransNetV2 scene detection +// is NOT run here: on the ROCm build it needs MIGraphX, whose statically-linked +// LLVM collides with the VAAPI decoder's system libLLVM and crashes at compile, so +// the two cannot share a process. is_scene_boundary therefore stays 0 here. +// // Usage: // dump_embeddings --movie --out [--arcface ] // [--detector ] [--fps 1] [--start S] [--end S] diff --git a/src/main.cpp b/src/main.cpp index 5e23040..21081fa 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -208,6 +208,7 @@ static Config parse_args(int argc, char** argv) { else if (arg("--start")) cfg.start_sec = std::stod(next()); else if (arg("--end")) cfg.end_sec = std::stod(next()); else if (arg("--cut-threshold")) cfg.cut_threshold = std::stof(next()); + else if (arg("--presence-mode")) { std::string m = next(); cfg.presence_mode = (m == "flood") ? PresenceMode::flood : PresenceMode::track_extent; } else if (arg("--scene-detect")) cfg.scene_detect = true; else if (arg("--scene-detector")) cfg.scene_model = next(); else if (arg("--scene-detector-engine")) cfg.scene_engine = next(); diff --git a/src/nodes/frame_annotation_node.hpp b/src/nodes/frame_annotation_node.hpp index c381b46..894b7a7 100644 --- a/src/nodes/frame_annotation_node.hpp +++ b/src/nodes/frame_annotation_node.hpp @@ -38,6 +38,11 @@ struct FrameAnnotationFunc { SceneAnnotation operator()(MatchedSceneFrame mf) { if (mf.source.eof) return {0.0, {}, /*eof=*/true}; - return {mf.source.timestamp_sec, std::move(mf.actors)}; + SceneAnnotation sa; + sa.timestamp_sec = mf.source.timestamp_sec; + sa.visible_actors = std::move(mf.actors); + sa.is_cut = mf.source.is_cut; + sa.is_scene_boundary = mf.source.is_scene_boundary; + return sa; } }; diff --git a/src/nodes/result_sink_node.hpp b/src/nodes/result_sink_node.hpp index 06ffacf..743f574 100644 --- a/src/nodes/result_sink_node.hpp +++ b/src/nodes/result_sink_node.hpp @@ -184,6 +184,20 @@ private: aw.scenes.push_back({c.first_seen, c.last_seen, c.belief, c.route}); } + // Flood-fill: snap each claim to the shot it sits in, so an actor seen + // once in a scene is reported across the whole scene. Bounded by real + // TransNetV2 boundaries — a window never crosses one — and a no-op when + // scene detection found no boundaries (nothing to snap to). + if (cfg_.presence_mode == PresenceMode::flood) { + const std::vector bounds = scene_boundaries(); + if (!bounds.empty()) + for (auto& [idx, aw] : by_actor) + for (auto& w : aw.scenes) { + w.start = boundary_at_or_before(bounds, w.start); + w.end = boundary_after(bounds, w.end); + } + } + std::vector result; for (auto& [idx, aw] : by_actor) { std::sort(aw.scenes.begin(), aw.scenes.end(), @@ -193,6 +207,45 @@ private: return result; } + // Sorted, de-duplicated boundary timestamps seen this run, framed by the + // film's own extent so the first and last shots are closed intervals. Derived + // from frames_ rather than a separate accumulator: the frames are already + // retained and this runs once. + // + // Prefers TransNetV2 shot boundaries (is_scene_boundary) when a scene detector + // populated them; otherwise falls back to the always-on histogram cuts + // (is_cut, camera_position_change_detector). On this ROCm box the scene + // detector cannot run in-process (see the dumper note), so is_cut is what + // flood-fill actually snaps to — coarser than true shot boundaries (cuts also + // fire on in-shot angle changes) but present with no extra pass. + std::vector scene_boundaries() const { + bool have_scene = false; + for (const auto& sa : frames_) + if (sa.is_scene_boundary) { have_scene = true; break; } + + std::vector b; + b.push_back(0.0); + for (const auto& sa : frames_) { + const bool boundary = have_scene ? sa.is_scene_boundary : sa.is_cut; + if (boundary) b.push_back(sa.timestamp_sec); + } + b.push_back(last_ts_ + 1.0); // a right edge past the final sample + std::sort(b.begin(), b.end()); + b.erase(std::unique(b.begin(), b.end()), b.end()); + return b; + } + + // The boundary opening the shot that contains t (largest boundary ≤ t). + static double boundary_at_or_before(const std::vector& b, double t) { + auto it = std::upper_bound(b.begin(), b.end(), t); + return (it == b.begin()) ? b.front() : *(it - 1); + } + // The boundary closing the shot that contains t (smallest boundary > t). + static double boundary_after(const std::vector& b, double t) { + auto it = std::upper_bound(b.begin(), b.end(), t); + return (it == b.end()) ? b.back() : *it; + } + json build_epochs() { json actors = json::array(); for (const auto& aw : build_actor_windows()) { diff --git a/src/types.hpp b/src/types.hpp index 58904d7..a332908 100644 --- a/src/types.hpp +++ b/src/types.hpp @@ -155,6 +155,13 @@ struct SceneAnnotation { double timestamp_sec{0.0}; std::vector visible_actors; bool eof{false}; + // Carried through from Frame so the sink can collect boundaries for flood-fill + // presence (PresenceMode::flood). is_cut is the always-on histogram cut + // (camera_position_change_detector) — the boundary flood-fill uses by default. + // is_scene_boundary is the opt-in TransNetV2 shot boundary (0 unless scene + // detection ran); kept for a future out-of-process scene detector. + bool is_cut{false}; + bool is_scene_boundary{false}; }; // ── Actor gallery ─────────────────────────────────────────────────────────────