feat(presence): flood-fill presence mode

Add PresenceMode::flood alongside the default track_extent. In flood mode
the result sink snaps each presence claim to the shot it sits in, so an
actor seen once anywhere in a shot is reported for the whole shot
[prev_boundary, next_boundary]. This trades precision for recall against
X-Ray's per-scene cast granularity and is a toggleable knob for the
optimizer to weigh rather than a default.

Boundaries come from the frame stream, now carried through SceneAnnotation
(is_cut and is_scene_boundary). Flood prefers TransNetV2 shot boundaries
when a scene detector populated them, otherwise falls back to the
always-on histogram cuts (camera_position_change_detector); with no
boundaries it degrades to track_extent per claim. The is_scene_boundary
path stays dormant so an out-of-process scene detector can be revived
later without re-wiring.

Selected with --presence-mode flood|track_extent (default track_extent),
so existing output is byte-for-byte unchanged. The dump_embeddings header
note records why TransNetV2 scene detection is not run in that process.
This commit is contained in:
2026-08-09 10:21:13 +02:00
parent de02e25e6a
commit 584f23546a
6 changed files with 91 additions and 1 deletions
+18
View File
@@ -11,6 +11,20 @@ enum class Verbosity {
standard, // per-frame detail: bbox, similarity, unknowns logged
xray, // Jellyfin-Xray format: {"second": ["Actor", ...], ...}
};
// How a track's accepted frames become a reported presence window.
enum class PresenceMode {
// A claim IS its track's [first_seen, last_seen] (AR-012/AR-013). The
// default and the only mode whose semantics the register validated.
track_extent,
// Flood-fill: snap each claim to the shot it sits in, so an actor seen once
// anywhere in a scene is reported for the whole scene [prev_boundary,
// next_boundary]. Trades precision for recall against X-Ray's per-scene cast
// granularity. Snaps to TransNetV2 shot boundaries (is_scene_boundary) when a
// scene detector populated them, else to the always-on histogram cuts
// (is_cut). With no boundaries at all it degrades to track_extent per claim.
flood,
};
// debug verbosity = compile with -DSAE_DEBUG → scene_analyze_debug binary
struct Config {
@@ -115,6 +129,10 @@ struct Config {
// fallback everywhere, which is at least the same wrong number in every
// stage. See identity_matcher_node.hpp.
// ── Presence derivation ──────────────────────────────────────────────────
// How accepted frames become a reported window. flood requires scene_detect.
PresenceMode presence_mode{PresenceMode::track_extent};
// ── Cut detection ────────────────────────────────────────────────────────
float cut_threshold{0.70f}; // grayscale histogram correlation below this → hard cut
+6
View File
@@ -8,6 +8,12 @@
// gallery file needed. Purpose-built for the optimizer's replay corpus and the
// embedding-model bake-off (dump each --arcface model over the film set).
//
// The camera-position (histogram) cut detector runs, so frames/is_cut is recorded
// in the dump. Flood-fill presence snaps to those cuts. TransNetV2 scene detection
// is NOT run here: on the ROCm build it needs MIGraphX, whose statically-linked
// LLVM collides with the VAAPI decoder's system libLLVM and crashes at compile, so
// the two cannot share a process. is_scene_boundary therefore stays 0 here.
//
// Usage:
// dump_embeddings --movie <path> --out <dump.h5> [--arcface <model.onnx>]
// [--detector <model.onnx>] [--fps 1] [--start S] [--end S]
+1
View File
@@ -208,6 +208,7 @@ static Config parse_args(int argc, char** argv) {
else if (arg("--start")) cfg.start_sec = std::stod(next());
else if (arg("--end")) cfg.end_sec = std::stod(next());
else if (arg("--cut-threshold")) cfg.cut_threshold = std::stof(next());
else if (arg("--presence-mode")) { std::string m = next(); cfg.presence_mode = (m == "flood") ? PresenceMode::flood : PresenceMode::track_extent; }
else if (arg("--scene-detect")) cfg.scene_detect = true;
else if (arg("--scene-detector")) cfg.scene_model = next();
else if (arg("--scene-detector-engine")) cfg.scene_engine = next();
+6 -1
View File
@@ -38,6 +38,11 @@ struct FrameAnnotationFunc {
SceneAnnotation operator()(MatchedSceneFrame mf) {
if (mf.source.eof) return {0.0, {}, /*eof=*/true};
return {mf.source.timestamp_sec, std::move(mf.actors)};
SceneAnnotation sa;
sa.timestamp_sec = mf.source.timestamp_sec;
sa.visible_actors = std::move(mf.actors);
sa.is_cut = mf.source.is_cut;
sa.is_scene_boundary = mf.source.is_scene_boundary;
return sa;
}
};
+53
View File
@@ -184,6 +184,20 @@ private:
aw.scenes.push_back({c.first_seen, c.last_seen, c.belief, c.route});
}
// Flood-fill: snap each claim to the shot it sits in, so an actor seen
// once in a scene is reported across the whole scene. Bounded by real
// TransNetV2 boundaries — a window never crosses one — and a no-op when
// scene detection found no boundaries (nothing to snap to).
if (cfg_.presence_mode == PresenceMode::flood) {
const std::vector<double> bounds = scene_boundaries();
if (!bounds.empty())
for (auto& [idx, aw] : by_actor)
for (auto& w : aw.scenes) {
w.start = boundary_at_or_before(bounds, w.start);
w.end = boundary_after(bounds, w.end);
}
}
std::vector<ActorWindow> result;
for (auto& [idx, aw] : by_actor) {
std::sort(aw.scenes.begin(), aw.scenes.end(),
@@ -193,6 +207,45 @@ private:
return result;
}
// Sorted, de-duplicated boundary timestamps seen this run, framed by the
// film's own extent so the first and last shots are closed intervals. Derived
// from frames_ rather than a separate accumulator: the frames are already
// retained and this runs once.
//
// Prefers TransNetV2 shot boundaries (is_scene_boundary) when a scene detector
// populated them; otherwise falls back to the always-on histogram cuts
// (is_cut, camera_position_change_detector). On this ROCm box the scene
// detector cannot run in-process (see the dumper note), so is_cut is what
// flood-fill actually snaps to — coarser than true shot boundaries (cuts also
// fire on in-shot angle changes) but present with no extra pass.
std::vector<double> scene_boundaries() const {
bool have_scene = false;
for (const auto& sa : frames_)
if (sa.is_scene_boundary) { have_scene = true; break; }
std::vector<double> b;
b.push_back(0.0);
for (const auto& sa : frames_) {
const bool boundary = have_scene ? sa.is_scene_boundary : sa.is_cut;
if (boundary) b.push_back(sa.timestamp_sec);
}
b.push_back(last_ts_ + 1.0); // a right edge past the final sample
std::sort(b.begin(), b.end());
b.erase(std::unique(b.begin(), b.end()), b.end());
return b;
}
// The boundary opening the shot that contains t (largest boundary ≤ t).
static double boundary_at_or_before(const std::vector<double>& b, double t) {
auto it = std::upper_bound(b.begin(), b.end(), t);
return (it == b.begin()) ? b.front() : *(it - 1);
}
// The boundary closing the shot that contains t (smallest boundary > t).
static double boundary_after(const std::vector<double>& b, double t) {
auto it = std::upper_bound(b.begin(), b.end(), t);
return (it == b.end()) ? b.back() : *it;
}
json build_epochs() {
json actors = json::array();
for (const auto& aw : build_actor_windows()) {
+7
View File
@@ -155,6 +155,13 @@ struct SceneAnnotation {
double timestamp_sec{0.0};
std::vector<IdentifiedActor> visible_actors;
bool eof{false};
// Carried through from Frame so the sink can collect boundaries for flood-fill
// presence (PresenceMode::flood). is_cut is the always-on histogram cut
// (camera_position_change_detector) — the boundary flood-fill uses by default.
// is_scene_boundary is the opt-in TransNetV2 shot boundary (0 unless scene
// detection ran); kept for a future out-of-process scene detector.
bool is_cut{false};
bool is_scene_boundary{false};
};
// ── Actor gallery ─────────────────────────────────────────────────────────────