dtourolle/scene-actor-extraction
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(engine): HDF5-native galleries with embedded calibration; TensorRT backends; scene detection

Browse Source 
Gallery format switches from JSON to HDF5 exclusively (JSON read-only kept for
back-compat): save_gallery always writes HDF5, and the fitted Platt-sigmoid
calibration (a, b, valid, hash) is now embedded directly in the gallery file
instead of a sidecar .calib_cache.json — identity_matcher reads it from the
loaded gallery and writes back only when the embeddings actually changed
(hash mismatch), skipping the O(n^2) refit otherwise.

Also includes: TensorRT inference backend support (ort_backend.cpp,
trt_backend.cpp), gemm_backend improvements, TransNetV2-based scene-boundary
detection wired through frame_source/face_tracker/main, and CMake build
target updates for the new sources.

Bumps the KPN submodule to feature/persistent-pipeline-reuse (push_blocking
backpressure, node_ptr/node_stats introspection, ObjectVariantNodeWrapper for
stateful functors) — needed by the optimizer's sae_kpn Python bindings.
This commit is contained in:
dtourolle
2026-07-19 19:04:03 +02:00
parent aca6147d69
commit 41a277bc19
19 changed files with 1151 additions and 216 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/main.cpp
+181 -80
View File
@@ -22,7 +22,23 @@
// --extinction <f> actor extinction window in seconds (default: 5.0)
// --detector <path> override SCRFD detector model path
// --arcface <path> override ArcFace model path
// --scene-detect enable TransNetV2 shot-boundary detection (dense decode;
// writes <output>.scenes.json). Off by default.
// --scene-detector <path> override TransNetV2 .onnx model path
// --scene-detector-engine <path> pre-built TransNetV2 TRT engine (TRT backend)
// --scene-threshold <f> boundary sigmoid prob above this → cut (default: 0.60)
// --scene-stride <N> frames between TransNetV2 windows (default: 50, ≤100)
// --scene-decode-fps <f> dense decode rate in scene-detect mode (default: 12;
// 0 = native fps). Lower = faster, coarser boundaries.
// --dense-scale <f> downscale decoded frames in scene-detect mode (0<f≤1,
// default 1=off). Speeds decode; keep ≥0.5 on 1080p.
// --max-faces <N> max faces kept per frame (default: 10)
// --expand-gallery enable per-film gallery expansion from track continuity
// --expand-buffer <N> per-track diversity buffer size (default: 20)
// --expand-novelty-sim <f> promote only views with best sim < f (default: 0.55)
// --expand-spread-max <f> reject track if buffer spread > f (default: 0.60)
// --expand-min-anchor <N> accepted frames before a track confirms (default: 3)
// --expand-debug-dir <p> dump promoted mugshots + embeddings here (SAE_DEBUG)
// (SAE_DEBUG only)
// --debug-dir <path> debug frames output dir (default: debug_frames)
// --crop-context <f> bbox expansion factor for context crops (default: 1.5)
@@ -31,13 +47,16 @@
#include "types.hpp"
#include "gallery/gallery_store.hpp"
#include "nodes/frame_source_node.hpp"
#include "nodes/camera_position_change_detector_node.hpp"
#include "nodes/face_detector_node.hpp"
#include "nodes/face_aligner_node.hpp"
#include "nodes/embedder_node.hpp"
#include "nodes/face_tracker_node.hpp"
#include "nodes/identity_matcher_node.hpp"
#include "nodes/scene_tracker_node.hpp"
#include "nodes/scene_detector_node.hpp"
#include "nodes/result_sink_node.hpp"
#include "nodes/embedding_dump_node.hpp"
#ifdef SAE_DEBUG
#include "nodes/debug_renderer_node.hpp"
#endif
@@ -61,6 +80,7 @@ static Config parse_args(int argc, char** argv) {
Config cfg;
cfg.detector_model = kDefaultDetectorModel;
cfg.arcface_model = kDefaultArcfaceModel;
cfg.scene_model = kDefaultSceneModel;
cfg.output_path = "annotations.json";
for (int i = 1; i < argc; ++i) {
@@ -73,11 +93,19 @@ static Config parse_args(int argc, char** argv) {
if (arg("--movie")) cfg.movie_path = next();
else if (arg("--gallery")) cfg.gallery_path = next();
else if (arg("--output")) cfg.output_path = next();
else if (arg("--dump-embeddings")) cfg.dump_embeddings_path = next();
else if (arg("--fps")) cfg.sample_fps = std::stof(next());
else if (arg("--max-decode-fps")) cfg.max_decode_fps = std::stof(next());
else if (arg("--start")) cfg.start_sec = std::stod(next());
else if (arg("--end")) cfg.end_sec = std::stod(next());
else if (arg("--cut-threshold")) cfg.cut_threshold = std::stof(next());
else if (arg("--scene-detect")) cfg.scene_detect = true;
else if (arg("--scene-detector")) cfg.scene_model = next();
else if (arg("--scene-detector-engine")) cfg.scene_engine = next();
else if (arg("--scene-threshold")) cfg.scene_threshold = std::stof(next());
else if (arg("--scene-stride")) cfg.scene_stride = std::stoi(next());
else if (arg("--scene-decode-fps")) cfg.scene_decode_fps = std::stof(next());
else if (arg("--dense-scale")) cfg.dense_scale = std::stof(next());
else if (arg("--verbosity")) { int v = std::stoi(next()); cfg.verbosity = v == 2 ? Verbosity::xray : v == 1 ? Verbosity::standard : Verbosity::minimal; }
else if (arg("--prior")) cfg.match_prior = std::stof(next());
else if (arg("--prob-threshold")) cfg.prob_threshold = std::stof(next());
@@ -96,7 +124,15 @@ static Config parse_args(int argc, char** argv) {
else if (arg("--track-min-iou")) cfg.track_min_iou = std::stof(next());
else if (arg("--track-max-embed")) cfg.track_max_embed_dist = std::stof(next());
else if (arg("--track-max-missing")) cfg.track_max_frames_missing = std::stoi(next());
else if (arg("--cut-revive-sim")) cfg.cut_revive_sim = std::stof(next());
else if (arg("--cut-inactive-max")) cfg.cut_inactive_max_frames = std::stoi(next());
else if (arg("--anneal")) cfg.anneal_sec = std::stod(next());
else if (arg("--expand-gallery")) cfg.expand_gallery = true;
else if (arg("--expand-buffer")) cfg.expand_buffer_size = std::stoi(next());
else if (arg("--expand-novelty-sim")) cfg.expand_novelty_sim = std::stof(next());
else if (arg("--expand-spread-max")) cfg.expand_track_spread_max = std::stof(next());
else if (arg("--expand-min-anchor")) cfg.expand_min_anchor_frames = std::stoi(next());
else if (arg("--expand-debug-dir")) cfg.expand_debug_dir = next();
else if (arg("--trt-cache")) cfg.trt.cache_dir = next();
else if (arg("--trt-fp16")) cfg.trt.fp16 = true;
else if (arg("--no-trt-fp16")) cfg.trt.fp16 = false;
@@ -139,16 +175,18 @@ int main(int argc, char** argv) {
// ── Construct node functors ───────────────────────────────────────────────
std::atomic<bool> done{false};
std::atomic<bool> done{false}; // set by result_sink (face branch)
std::atomic<bool> scene_done{true}; // set by scene_detector; true when disabled
FrameSourceFunc source_fn {cfg};
FaceDetectorFunc detector_fn{cfg};
FaceAlignerFunc aligner_fn;
EmbedderFunc embedder_fn{cfg};
FaceTrackerFunc ftracker_fn{cfg};
IdentityMatcherFunc matcher_fn {gallery, cfg};
SceneTrackerFunc tracker_fn {cfg};
ResultSinkFunc sink_fn {cfg, done};
FrameSourceFunc source_fn {cfg};
CameraPositionChangeDetectorFunc campos_fn {cfg};
FaceDetectorFunc detector_fn{cfg};
FaceAlignerFunc aligner_fn;
EmbedderFunc embedder_fn{cfg};
FaceTrackerFunc ftracker_fn{cfg};
IdentityMatcherFunc matcher_fn {gallery, cfg};
SceneTrackerFunc tracker_fn {cfg};
ResultSinkFunc sink_fn {cfg, done};
#ifdef SAE_DEBUG
DebugRendererFunc debug_fn {cfg};
#endif
@@ -158,7 +196,8 @@ int main(int argc, char** argv) {
// Queue sizes tuned to the pipeline's speed profile:
// embedder (16ms) is the slowest GPU node — buffer before it must be largest
// to prevent face_aligner pool overflows and frame drops.
kpn::ObjectNode<FrameSourceFunc, kpn::in<>, kpn::out<"frame">, "frame_source", 0> source (source_fn, 32);
kpn::ObjectNode<FrameSourceFunc, kpn::in<>, kpn::out<"raw">, "frame_source", 0> source (source_fn, 32);
kpn::ObjectNode<CameraPositionChangeDetectorFunc, kpn::in<"raw">, kpn::out<"frame">, "camera_pos", 0> campos (campos_fn, 32);
kpn::ObjectNode<FaceDetectorFunc, kpn::in<"frame">, kpn::out<"scene">, "face_detector", 0> detector (detector_fn, 64);
kpn::ObjectNode<FaceAlignerFunc, kpn::in<"scene">, kpn::out<"aligned">, "face_aligner", 0> aligner (aligner_fn, 64);
kpn::ObjectNode<EmbedderFunc, kpn::in<"aligned">, kpn::out<"embedded">, "embedder", 0> embedder (embedder_fn, 32);
@@ -167,81 +206,143 @@ int main(int argc, char** argv) {
kpn::ObjectNode<SceneTrackerFunc, kpn::in<"matched">, kpn::out<"annotation">, "scene_tracker", 0> tracker (tracker_fn, 16);
kpn::ObjectNode<ResultSinkFunc, kpn::in<"annotation">,kpn::out<>, "result_sink", 0> sink (sink_fn, 16);
// ── Build static network ──────────────────────────────────────────────────
#ifdef SAE_DEBUG
kpn::ObjectNode<DebugRendererFunc, kpn::in<"matched">, kpn::out<>, "debug_renderer", 1> debug_node(debug_fn, 16);
// matcher → FanoutNode<MatchedSceneFrame,2> → scene_tracker + debug_node (auto-inserted)
auto net = kpn::make_network(
kpn::edge(source.output<"frame">(), detector.input<"frame">()),
kpn::edge(detector.output<"scene">(), aligner.input<"scene">()),
kpn::edge(aligner.output<"aligned">(), embedder.input<"aligned">()),
kpn::edge(embedder.output<"embedded">(), ftracker.input<"embedded">()),
kpn::edge(ftracker.output<"tracked">(), matcher.input<"tracked">()),
kpn::edge(matcher.output<"matched">(), tracker.input<"matched">()),
kpn::edge(matcher.output<"matched">(), debug_node.input<"matched">()),
kpn::edge(tracker.output<"annotation">(), sink.input<"annotation">())
);
#else
auto net = kpn::make_network(
kpn::edge(source.output<"frame">(), detector.input<"frame">()),
kpn::edge(detector.output<"scene">(), aligner.input<"scene">()),
kpn::edge(aligner.output<"aligned">(), embedder.input<"aligned">()),
kpn::edge(embedder.output<"embedded">(), ftracker.input<"embedded">()),
kpn::edge(ftracker.output<"tracked">(), matcher.input<"tracked">()),
kpn::edge(matcher.output<"matched">(), tracker.input<"matched">()),
kpn::edge(tracker.output<"annotation">(), sink.input<"annotation">())
);
#endif
// ── Pipeline observability (KPN event handler) ────────────────────────────
// Tally dropped frames per node (channel overflow) and detect a node that
// stops unexpectedly. A Closed event from any node other than result_sink at
// EOF means a stage crashed — without this the main loop below would hang on
// `done` forever, so we trip a flag to unblock it and exit non-zero.
// ── Pipeline observability + run loop (topology-agnostic) ──────────────────
// Factored into a lambda so the two topologies (with/without the scene-detect
// branch) share identical event handling, wait loop, and teardown. Any
// make_network result type binds to `Net&&`.
std::mutex event_mtx;
std::map<std::string, long> overflow_counts;
std::atomic<bool> node_crashed{false};
net.set_event_handler(
[&](std::string_view node_name, kpn::NodeEvent ev,
std::chrono::steady_clock::time_point) {
if (ev == kpn::NodeEvent::Overflow) {
std::lock_guard<std::mutex> lk(event_mtx);
++overflow_counts[std::string(node_name)];
} else { // NodeEvent::Closed
// result_sink closing once EOF has been signalled is the normal
// shutdown path, not a crash.
if (node_name == "result_sink" && done.load(std::memory_order_acquire))
return;
std::cerr << "[main] node '" << node_name
<< "' stopped unexpectedly — aborting pipeline\n";
node_crashed.store(true, std::memory_order_release);
auto run_net = [&](auto&& net) -> int {
// Tally per-node channel overflow, and treat any non-result_sink Closed
// event as a crash so the wait loop below can't hang on `done` forever.
net.set_event_handler(
[&](std::string_view node_name, kpn::NodeEvent ev,
std::chrono::steady_clock::time_point) {
if (ev == kpn::NodeEvent::Overflow) {
std::lock_guard<std::mutex> lk(event_mtx);
++overflow_counts[std::string(node_name)];
} else { // NodeEvent::Closed
if (node_name == "result_sink" && done.load(std::memory_order_acquire))
return;
std::cerr << "[main] node '" << node_name
<< "' stopped unexpectedly — aborting pipeline\n";
node_crashed.store(true, std::memory_order_release);
}
});
std::cerr << "[main] starting pipeline…\n";
net.start();
// Wait until BOTH terminal branches finish: result_sink (face pipeline)
// and, when enabled, scene_detector (the dense TransNetV2 branch, which
// runs much slower and must not be torn down mid-stream). scene_done is
// pre-set true when scene detection is disabled.
while ((!done.load(std::memory_order_acquire) ||
!scene_done.load(std::memory_order_acquire)) &&
!node_crashed.load(std::memory_order_acquire))
std::this_thread::sleep_for(std::chrono::milliseconds(100));
net.stop();
net.print_diagnostics();
{
std::lock_guard<std::mutex> lk(event_mtx);
if (!overflow_counts.empty()) {
std::cerr << "[main] dropped frames (channel overflow):\n";
for (const auto& [name, count] : overflow_counts)
std::cerr << " " << name << ": " << count << "\n";
}
});
// ── Run ───────────────────────────────────────────────────────────────────
std::cerr << "[main] starting pipeline…\n";
net.start();
// Main thread waits until ResultSinkFunc signals EOF completion, or a node
// crash trips node_crashed.
while (!done.load(std::memory_order_acquire) &&
!node_crashed.load(std::memory_order_acquire))
std::this_thread::sleep_for(std::chrono::milliseconds(100));
net.stop();
net.print_diagnostics();
{
std::lock_guard<std::mutex> lk(event_mtx);
if (!overflow_counts.empty()) {
std::cerr << "[main] dropped frames (channel overflow):\n";
for (const auto& [name, count] : overflow_counts)
std::cerr << " " << name << ": " << count << "\n";
}
}
return node_crashed.load(std::memory_order_acquire) ? 1 : 0;
};
return node_crashed.load(std::memory_order_acquire) ? 1 : 0;
// ── Build static network and run ──────────────────────────────────────────
// Common face-analysis chain (campos → … → sink) is identical in all cases;
// the scene-detect branch and the debug fanout are spliced on conditionally.
// Topology:
// plain: source → campos → detector → … → sink
// scene-detect: source ─┬→ campos → decimate(filter) → detector → … → sink
// └→ scene_detector (TransNetV2 sink → scenes.json)
// The fanout after `source` is auto-inserted by make_network when its output
// feeds two edges. In dense mode campos still sees native-rate frames (so it
// detects angle changes correctly); a FilterNode then thins to sample_fps
// before face detection.
#ifdef SAE_DEBUG
kpn::ObjectNode<DebugRendererFunc, kpn::in<"matched">, kpn::out<>, "debug_renderer", 1> debug_node(debug_fn, 16);
#define SAE_DEBUG_EDGE , kpn::edge(matcher.output<"matched">(), debug_node.input<"matched">())
#else
#define SAE_DEBUG_EDGE
#endif
int rc = 0;
if (!cfg.dump_embeddings_path.empty()) {
// Dump-only topology: run the expensive front half and tee the embedder
// output to an HDF5 dump for offline sweep replay (sae_kpn). Downstream
// matching is skipped — the sweep re-runs it from the dump.
EmbeddingDumpFunc dump_fn{cfg, done};
kpn::ObjectNode<EmbeddingDumpFunc, kpn::in<"embedded">, kpn::out<>, "embedding_dump", 0>
dump_node(dump_fn, 32);
auto net = kpn::make_network(
kpn::edge(source.output<"raw">(), campos.input<"raw">()),
kpn::edge(campos.output<"frame">(), detector.input<"frame">()),
kpn::edge(detector.output<"scene">(), aligner.input<"scene">()),
kpn::edge(aligner.output<"aligned">(), embedder.input<"aligned">()),
kpn::edge(embedder.output<"embedded">(), dump_node.input<"embedded">())
);
return run_net(std::move(net));
}
if (cfg.scene_detect) {
scene_done.store(false, std::memory_order_release); // now a real terminal branch
SceneDetectorFunc scene_fn{cfg, scene_done};
kpn::ObjectNode<SceneDetectorFunc, kpn::in<"dense">, kpn::out<>, "scene_detector", 0>
scene_node(scene_fn, 128);
// Decimator: keep frames on the sample_fps cadence, drop the rest.
// eof always passes so downstream shuts down cleanly. Stateful — one
// instance, mutable via shared_ptr so the std::function stays copyable.
auto decim_state = std::make_shared<double>(-1e18);
const double interval = 1.0 / cfg.sample_fps;
auto decimate = kpn::make_filter<Frame>(
[decim_state, interval](const Frame& f) {
if (f.eof) return true;
if (f.timestamp_sec - *decim_state >= interval - 1e-6) {
*decim_state = f.timestamp_sec;
return true;
}
return false;
}, 32);
auto net = kpn::make_network(
kpn::edge(source.output<"raw">(), campos.input<"raw">()),
kpn::edge(source.output<"raw">(), scene_node.input<"dense">()),
kpn::edge(campos.output<"frame">(), decimate.input<0>()),
kpn::edge(decimate.output<0>(), detector.input<"frame">()),
kpn::edge(detector.output<"scene">(), aligner.input<"scene">()),
kpn::edge(aligner.output<"aligned">(), embedder.input<"aligned">()),
kpn::edge(embedder.output<"embedded">(), ftracker.input<"embedded">()),
kpn::edge(ftracker.output<"tracked">(), matcher.input<"tracked">()),
kpn::edge(matcher.output<"matched">(), tracker.input<"matched">()),
kpn::edge(tracker.output<"annotation">(), sink.input<"annotation">())
SAE_DEBUG_EDGE
);
rc = run_net(std::move(net));
} else {
auto net = kpn::make_network(
kpn::edge(source.output<"raw">(), campos.input<"raw">()),
kpn::edge(campos.output<"frame">(), detector.input<"frame">()),
kpn::edge(detector.output<"scene">(), aligner.input<"scene">()),
kpn::edge(aligner.output<"aligned">(), embedder.input<"aligned">()),
kpn::edge(embedder.output<"embedded">(), ftracker.input<"embedded">()),
kpn::edge(ftracker.output<"tracked">(), matcher.input<"tracked">()),
kpn::edge(matcher.output<"matched">(), tracker.input<"matched">()),
kpn::edge(tracker.output<"annotation">(), sink.input<"annotation">())
SAE_DEBUG_EDGE
);
rc = run_net(std::move(net));
}
#undef SAE_DEBUG_EDGE
return rc;
}