feat(presence): flood-fill presence mode
Add PresenceMode::flood alongside the default track_extent. In flood mode the result sink snaps each presence claim to the shot it sits in, so an actor seen once anywhere in a shot is reported for the whole shot [prev_boundary, next_boundary]. This trades precision for recall against X-Ray's per-scene cast granularity and is a toggleable knob for the optimizer to weigh rather than a default. Boundaries come from the frame stream, now carried through SceneAnnotation (is_cut and is_scene_boundary). Flood prefers TransNetV2 shot boundaries when a scene detector populated them, otherwise falls back to the always-on histogram cuts (camera_position_change_detector); with no boundaries it degrades to track_extent per claim. The is_scene_boundary path stays dormant so an out-of-process scene detector can be revived later without re-wiring. Selected with --presence-mode flood|track_extent (default track_extent), so existing output is byte-for-byte unchanged. The dump_embeddings header note records why TransNetV2 scene detection is not run in that process.
This commit is contained in:
@@ -11,6 +11,20 @@ enum class Verbosity {
|
|||||||
standard, // per-frame detail: bbox, similarity, unknowns logged
|
standard, // per-frame detail: bbox, similarity, unknowns logged
|
||||||
xray, // Jellyfin-Xray format: {"second": ["Actor", ...], ...}
|
xray, // Jellyfin-Xray format: {"second": ["Actor", ...], ...}
|
||||||
};
|
};
|
||||||
|
|
||||||
|
// How a track's accepted frames become a reported presence window.
|
||||||
|
enum class PresenceMode {
|
||||||
|
// A claim IS its track's [first_seen, last_seen] (AR-012/AR-013). The
|
||||||
|
// default and the only mode whose semantics the register validated.
|
||||||
|
track_extent,
|
||||||
|
// Flood-fill: snap each claim to the shot it sits in, so an actor seen once
|
||||||
|
// anywhere in a scene is reported for the whole scene [prev_boundary,
|
||||||
|
// next_boundary]. Trades precision for recall against X-Ray's per-scene cast
|
||||||
|
// granularity. Snaps to TransNetV2 shot boundaries (is_scene_boundary) when a
|
||||||
|
// scene detector populated them, else to the always-on histogram cuts
|
||||||
|
// (is_cut). With no boundaries at all it degrades to track_extent per claim.
|
||||||
|
flood,
|
||||||
|
};
|
||||||
// debug verbosity = compile with -DSAE_DEBUG → scene_analyze_debug binary
|
// debug verbosity = compile with -DSAE_DEBUG → scene_analyze_debug binary
|
||||||
|
|
||||||
struct Config {
|
struct Config {
|
||||||
@@ -115,6 +129,10 @@ struct Config {
|
|||||||
// fallback everywhere, which is at least the same wrong number in every
|
// fallback everywhere, which is at least the same wrong number in every
|
||||||
// stage. See identity_matcher_node.hpp.
|
// stage. See identity_matcher_node.hpp.
|
||||||
|
|
||||||
|
// ── Presence derivation ──────────────────────────────────────────────────
|
||||||
|
// How accepted frames become a reported window. flood requires scene_detect.
|
||||||
|
PresenceMode presence_mode{PresenceMode::track_extent};
|
||||||
|
|
||||||
// ── Cut detection ────────────────────────────────────────────────────────
|
// ── Cut detection ────────────────────────────────────────────────────────
|
||||||
float cut_threshold{0.70f}; // grayscale histogram correlation below this → hard cut
|
float cut_threshold{0.70f}; // grayscale histogram correlation below this → hard cut
|
||||||
|
|
||||||
|
|||||||
@@ -8,6 +8,12 @@
|
|||||||
// gallery file needed. Purpose-built for the optimizer's replay corpus and the
|
// gallery file needed. Purpose-built for the optimizer's replay corpus and the
|
||||||
// embedding-model bake-off (dump each --arcface model over the film set).
|
// embedding-model bake-off (dump each --arcface model over the film set).
|
||||||
//
|
//
|
||||||
|
// The camera-position (histogram) cut detector runs, so frames/is_cut is recorded
|
||||||
|
// in the dump. Flood-fill presence snaps to those cuts. TransNetV2 scene detection
|
||||||
|
// is NOT run here: on the ROCm build it needs MIGraphX, whose statically-linked
|
||||||
|
// LLVM collides with the VAAPI decoder's system libLLVM and crashes at compile, so
|
||||||
|
// the two cannot share a process. is_scene_boundary therefore stays 0 here.
|
||||||
|
//
|
||||||
// Usage:
|
// Usage:
|
||||||
// dump_embeddings --movie <path> --out <dump.h5> [--arcface <model.onnx>]
|
// dump_embeddings --movie <path> --out <dump.h5> [--arcface <model.onnx>]
|
||||||
// [--detector <model.onnx>] [--fps 1] [--start S] [--end S]
|
// [--detector <model.onnx>] [--fps 1] [--start S] [--end S]
|
||||||
|
|||||||
@@ -208,6 +208,7 @@ static Config parse_args(int argc, char** argv) {
|
|||||||
else if (arg("--start")) cfg.start_sec = std::stod(next());
|
else if (arg("--start")) cfg.start_sec = std::stod(next());
|
||||||
else if (arg("--end")) cfg.end_sec = std::stod(next());
|
else if (arg("--end")) cfg.end_sec = std::stod(next());
|
||||||
else if (arg("--cut-threshold")) cfg.cut_threshold = std::stof(next());
|
else if (arg("--cut-threshold")) cfg.cut_threshold = std::stof(next());
|
||||||
|
else if (arg("--presence-mode")) { std::string m = next(); cfg.presence_mode = (m == "flood") ? PresenceMode::flood : PresenceMode::track_extent; }
|
||||||
else if (arg("--scene-detect")) cfg.scene_detect = true;
|
else if (arg("--scene-detect")) cfg.scene_detect = true;
|
||||||
else if (arg("--scene-detector")) cfg.scene_model = next();
|
else if (arg("--scene-detector")) cfg.scene_model = next();
|
||||||
else if (arg("--scene-detector-engine")) cfg.scene_engine = next();
|
else if (arg("--scene-detector-engine")) cfg.scene_engine = next();
|
||||||
|
|||||||
@@ -38,6 +38,11 @@ struct FrameAnnotationFunc {
|
|||||||
|
|
||||||
SceneAnnotation operator()(MatchedSceneFrame mf) {
|
SceneAnnotation operator()(MatchedSceneFrame mf) {
|
||||||
if (mf.source.eof) return {0.0, {}, /*eof=*/true};
|
if (mf.source.eof) return {0.0, {}, /*eof=*/true};
|
||||||
return {mf.source.timestamp_sec, std::move(mf.actors)};
|
SceneAnnotation sa;
|
||||||
|
sa.timestamp_sec = mf.source.timestamp_sec;
|
||||||
|
sa.visible_actors = std::move(mf.actors);
|
||||||
|
sa.is_cut = mf.source.is_cut;
|
||||||
|
sa.is_scene_boundary = mf.source.is_scene_boundary;
|
||||||
|
return sa;
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
|
|||||||
@@ -184,6 +184,20 @@ private:
|
|||||||
aw.scenes.push_back({c.first_seen, c.last_seen, c.belief, c.route});
|
aw.scenes.push_back({c.first_seen, c.last_seen, c.belief, c.route});
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Flood-fill: snap each claim to the shot it sits in, so an actor seen
|
||||||
|
// once in a scene is reported across the whole scene. Bounded by real
|
||||||
|
// TransNetV2 boundaries — a window never crosses one — and a no-op when
|
||||||
|
// scene detection found no boundaries (nothing to snap to).
|
||||||
|
if (cfg_.presence_mode == PresenceMode::flood) {
|
||||||
|
const std::vector<double> bounds = scene_boundaries();
|
||||||
|
if (!bounds.empty())
|
||||||
|
for (auto& [idx, aw] : by_actor)
|
||||||
|
for (auto& w : aw.scenes) {
|
||||||
|
w.start = boundary_at_or_before(bounds, w.start);
|
||||||
|
w.end = boundary_after(bounds, w.end);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
std::vector<ActorWindow> result;
|
std::vector<ActorWindow> result;
|
||||||
for (auto& [idx, aw] : by_actor) {
|
for (auto& [idx, aw] : by_actor) {
|
||||||
std::sort(aw.scenes.begin(), aw.scenes.end(),
|
std::sort(aw.scenes.begin(), aw.scenes.end(),
|
||||||
@@ -193,6 +207,45 @@ private:
|
|||||||
return result;
|
return result;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Sorted, de-duplicated boundary timestamps seen this run, framed by the
|
||||||
|
// film's own extent so the first and last shots are closed intervals. Derived
|
||||||
|
// from frames_ rather than a separate accumulator: the frames are already
|
||||||
|
// retained and this runs once.
|
||||||
|
//
|
||||||
|
// Prefers TransNetV2 shot boundaries (is_scene_boundary) when a scene detector
|
||||||
|
// populated them; otherwise falls back to the always-on histogram cuts
|
||||||
|
// (is_cut, camera_position_change_detector). On this ROCm box the scene
|
||||||
|
// detector cannot run in-process (see the dumper note), so is_cut is what
|
||||||
|
// flood-fill actually snaps to — coarser than true shot boundaries (cuts also
|
||||||
|
// fire on in-shot angle changes) but present with no extra pass.
|
||||||
|
std::vector<double> scene_boundaries() const {
|
||||||
|
bool have_scene = false;
|
||||||
|
for (const auto& sa : frames_)
|
||||||
|
if (sa.is_scene_boundary) { have_scene = true; break; }
|
||||||
|
|
||||||
|
std::vector<double> b;
|
||||||
|
b.push_back(0.0);
|
||||||
|
for (const auto& sa : frames_) {
|
||||||
|
const bool boundary = have_scene ? sa.is_scene_boundary : sa.is_cut;
|
||||||
|
if (boundary) b.push_back(sa.timestamp_sec);
|
||||||
|
}
|
||||||
|
b.push_back(last_ts_ + 1.0); // a right edge past the final sample
|
||||||
|
std::sort(b.begin(), b.end());
|
||||||
|
b.erase(std::unique(b.begin(), b.end()), b.end());
|
||||||
|
return b;
|
||||||
|
}
|
||||||
|
|
||||||
|
// The boundary opening the shot that contains t (largest boundary ≤ t).
|
||||||
|
static double boundary_at_or_before(const std::vector<double>& b, double t) {
|
||||||
|
auto it = std::upper_bound(b.begin(), b.end(), t);
|
||||||
|
return (it == b.begin()) ? b.front() : *(it - 1);
|
||||||
|
}
|
||||||
|
// The boundary closing the shot that contains t (smallest boundary > t).
|
||||||
|
static double boundary_after(const std::vector<double>& b, double t) {
|
||||||
|
auto it = std::upper_bound(b.begin(), b.end(), t);
|
||||||
|
return (it == b.end()) ? b.back() : *it;
|
||||||
|
}
|
||||||
|
|
||||||
json build_epochs() {
|
json build_epochs() {
|
||||||
json actors = json::array();
|
json actors = json::array();
|
||||||
for (const auto& aw : build_actor_windows()) {
|
for (const auto& aw : build_actor_windows()) {
|
||||||
|
|||||||
@@ -155,6 +155,13 @@ struct SceneAnnotation {
|
|||||||
double timestamp_sec{0.0};
|
double timestamp_sec{0.0};
|
||||||
std::vector<IdentifiedActor> visible_actors;
|
std::vector<IdentifiedActor> visible_actors;
|
||||||
bool eof{false};
|
bool eof{false};
|
||||||
|
// Carried through from Frame so the sink can collect boundaries for flood-fill
|
||||||
|
// presence (PresenceMode::flood). is_cut is the always-on histogram cut
|
||||||
|
// (camera_position_change_detector) — the boundary flood-fill uses by default.
|
||||||
|
// is_scene_boundary is the opt-in TransNetV2 shot boundary (0 unless scene
|
||||||
|
// detection ran); kept for a future out-of-process scene detector.
|
||||||
|
bool is_cut{false};
|
||||||
|
bool is_scene_boundary{false};
|
||||||
};
|
};
|
||||||
|
|
||||||
// ── Actor gallery ─────────────────────────────────────────────────────────────
|
// ── Actor gallery ─────────────────────────────────────────────────────────────
|
||||||
|
|||||||
Reference in New Issue
Block a user