dtourolle
e5204a831a
Wire the XGBoost scene-boundary detector into scene_analyze as a post-EOF step in
the result sink (like flood-fill itself — the per-film knee threshold needs the
whole film, so it cannot stream). With --scene-xgb-model set, the camera-position
node stamps a per-frame RGB histogram onto the Frame, it rides through to the
sink, and at EOF the sink runs XGBSceneBoundary over the collected histograms +
the movie's per-second audio log-PSD to produce the flood-fill boundaries. Falls
back to is_scene_boundary / is_cut when no model is configured or inference fails.
Inference is real XGBoost via CMake FetchContent (v2.1.1, static), C API in
src/inference/xgb_scene_boundary.hpp; audio log-PSD in src/inference/
audio_logpsd.hpp (FFTW + ffmpeg full-file 16kHz decode). Feature extraction
matches training exactly — video features verified row-identical to numpy, and to
avoid chasing numpy's every rounding the shipped model is TRAINED on the
C++-extracted features (scene_features_dump exe → train_xgb_cpp.py). The
C++/Python peak-finders differ slightly so boundary counts differ, but what
matters is downstream: flood + C++ detector = 75.8% macro presence F1 vs 64.0%
for the histogram-cut flood and 62.5% for track_extent, and it fixes the Scarface
flood collapse (41 -> 70). All nine films improve.
Guarded by the SAE_SCENE_XGB CMake option (on by default; heavy first build).
xgb_boundary_parity is a diff harness; scene_features_dump writes the C++ feature
matrix so training and inference share one feature implementation.
Verified end to end: scene_analyze --scene-xgb-model on a real movie stamps the
histogram, runs the detector at EOF ("XGBoost scene detector: N boundaries"), and
flood-snaps presence to the learned boundaries.
700 lines
38 KiB
C++
700 lines
38 KiB
C++
// scene_analyze — identify actors in a movie using a KPN pipeline
|
|
//
|
|
// TRACES: DP-001, DP-002 | PR-004
|
|
// One analysis core; the CLI is a front-end over it and must not fork pipeline
|
|
// logic. Other deployment modes (DP-003, DP-004) wrap this same core.
|
|
//
|
|
// KPN topology (release build):
|
|
//
|
|
// [frame_source] ──Frame──► [face_detector] ──SceneFrame──► [face_aligner]
|
|
// ──AlignedSceneFrame──► [embedder] ──EmbeddedSceneFrame──►
|
|
// [identity_matcher] ──MatchedSceneFrame──► [frame_annotation]
|
|
// ──SceneAnnotation──► [result_sink]
|
|
//
|
|
// Debug build (SAE_DEBUG=1):
|
|
// [identity_matcher] output fans out to both [frame_annotation] AND [debug_renderer].
|
|
// FanoutNode<MatchedSceneFrame, 2> is auto-inserted by make_network().
|
|
//
|
|
// Usage:
|
|
// scene_analyze --movie <path> --gallery <gallery.json> [options]
|
|
//
|
|
// Options:
|
|
// --output <path> output JSON (default: annotations.json)
|
|
// --fps <N> sample rate in frames/sec (default: 1.0)
|
|
// --verbosity <0|1|2> 0=minimal, 1=standard, 2=jellyfin-xray (default: 0)
|
|
// --prob-threshold <f> posterior P(match) to accept (default: 0.754)
|
|
// --detector <path> override SCRFD detector model path
|
|
// --arcface <path> override ArcFace model path
|
|
// --scene-detect enable TransNetV2 shot-boundary detection (dense decode;
|
|
// writes <output>.scenes.json). Off by default.
|
|
// --scene-detector <path> override TransNetV2 .onnx model path
|
|
// --scene-detector-engine <path> pre-built TransNetV2 TRT engine (TRT backend)
|
|
// --scene-threshold <f> boundary sigmoid prob above this → cut (default: 0.60)
|
|
// --scene-stride <N> frames between TransNetV2 windows (default: 50, ≤100)
|
|
// --scene-decode-fps <f> dense decode rate in scene-detect mode (default: 0 =
|
|
// native, the only rate TransNetV2 is calibrated for;
|
|
// AR-011). Lowering it runs the model off-distribution.
|
|
// --dense-scale <f> downscale decoded frames in scene-detect mode (0<f≤1,
|
|
// default 1=off). Speeds decode; keep ≥0.5 on 1080p.
|
|
// --max-faces <N> max faces kept per frame (default: 0 = uncapped)
|
|
// --ownership-logodds <f> belief needed to own a track (default: 2.0 ≈ P 0.88).
|
|
// Below it a track makes no presence claim at all.
|
|
// --evidence-rho-max <f> ceiling on correlation between two observations of
|
|
// one track (default: 0.5 = a repeated view is worth
|
|
// at most two independent ones). AR-025.
|
|
// --evidence-admit-below <p> P(same view) under this counts as a new look
|
|
// --evidence-max-views <N> distinct views remembered per track
|
|
// --expand-gallery enable per-film gallery expansion from track continuity
|
|
// --expand-buffer <N> per-track diversity buffer size (default: 20)
|
|
// --expand-band-lo <p> store admission floor, P(same person) (default: 0.90)
|
|
// --expand-band-hi <p> store admission ceiling, P(same person) (default: 0.95)
|
|
// --expand-min-anchor <N> accepted frames before a track confirms (default: 3)
|
|
// --expand-debug-dir <p> dump promoted mugshots + embeddings here (SAE_DEBUG)
|
|
// --benchmark <path> write a per-node timing + bottleneck report (JSON) and
|
|
// print it at shutdown. Says where the run's time went
|
|
// and which node is pacing it. See src/benchmark.hpp.
|
|
// --benchmark-interval-ms <N> channel-occupancy sampling period (default: 100)
|
|
// (SAE_DEBUG only)
|
|
// --debug-dir <path> debug frames output dir (default: debug_frames)
|
|
// --crop-context <f> bbox expansion factor for context crops (default: 1.5)
|
|
|
|
#include "benchmark.hpp"
|
|
#include "config.hpp"
|
|
#include "types.hpp"
|
|
#include "gallery/embedder_stamp.hpp"
|
|
#include "gallery/gallery_store.hpp"
|
|
#include "nodes/frame_source_node.hpp"
|
|
#include "nodes/camera_position_change_detector_node.hpp"
|
|
#include "nodes/face_detector_node.hpp"
|
|
#include "nodes/face_aligner_node.hpp"
|
|
#include "nodes/embedder_node.hpp"
|
|
#include "nodes/face_tracker_node.hpp"
|
|
#include "nodes/identity_matcher_node.hpp"
|
|
#include "nodes/frame_annotation_node.hpp"
|
|
#include "inference/scene_detector.hpp" // ISceneDetector::kWindow, for the join-depth derivation
|
|
#include "nodes/scene_detector_node.hpp"
|
|
#include "scene_boundaries.hpp"
|
|
#include "nodes/scene_boundary_annotator_node.hpp"
|
|
#include "nodes/result_sink_node.hpp"
|
|
#include "nodes/embedding_dump_node.hpp"
|
|
#ifdef SAE_DEBUG
|
|
#include "nodes/debug_renderer_node.hpp"
|
|
#endif
|
|
|
|
#include <kpn/kpn.hpp>
|
|
|
|
#include <opencv2/core/utility.hpp> // cv::setNumThreads (SAE_CV_THREADS)
|
|
|
|
#include <algorithm>
|
|
#include <atomic>
|
|
#include <chrono>
|
|
#include <cmath>
|
|
#include <csignal>
|
|
#include <cstdlib>
|
|
#include <cstring>
|
|
#include <fstream>
|
|
#include <iostream>
|
|
#include <map>
|
|
#include <mutex>
|
|
#include <stdexcept>
|
|
#include <string>
|
|
#include <string_view>
|
|
#include <thread>
|
|
|
|
// ── CLI parsing ───────────────────────────────────────────────────────────────
|
|
|
|
/// TRACES: AR-010, AR-004 | SR-002
|
|
/// Depth of the dense branch's own input queue. Part of how far behind the
|
|
/// fanout head TransNetV2 can be, and therefore an input to the join depth.
|
|
static constexpr std::size_t kSceneInputDepth = 128;
|
|
|
|
/// TRACES: AR-010, AR-004 | SR-002
|
|
/// How far the sampled branch must trail the dense one, in seconds of film.
|
|
///
|
|
/// TransNetV2 needs kWindow (100) dense frames before it can score any of
|
|
/// them, and its input queue can hold kSceneInputDepth more, so in the worst
|
|
/// case it has scored only up to (kSceneInputDepth + kWindow) frames behind
|
|
/// whatever the fanout has just delivered. The face branch must be at least
|
|
/// that far behind, or `scene_annotate` asks about frames nobody has looked at
|
|
/// yet. Backpressure turns depth into lag: the fanout blocks on the slower
|
|
/// branch rather than dropping, so the dense branch simply runs ahead.
|
|
///
|
|
/// Divided by a *lower bound* on native frame rate, because a slower source
|
|
/// makes the same frame count span more film — 24 fps is the floor for the
|
|
/// material this runs on, so it is the conservative choice.
|
|
static constexpr double kMinNativeFps = 24.0;
|
|
static constexpr double kSceneJoinLagSec =
|
|
(kSceneInputDepth + ISceneDetector::kWindow) / kMinNativeFps; // ~9.5 s
|
|
|
|
/// Margin over that minimum, for jitter in TransNetV2's inference time.
|
|
static constexpr double kSceneJoinSafety = 2.0;
|
|
|
|
/// TRACES: AR-004 | SR-002
|
|
/// Slots the sampled branch needs to hold `kSceneJoinLagSec` of film.
|
|
///
|
|
/// This used to be a constant 256, which is the whole bug: the requirement is a
|
|
/// span of *film*, and the slots needed to hold it depend on `sample_fps`.
|
|
/// Pinned at 256 it was ~256 s of lag at 1 fps — 27x what the join needs — and
|
|
/// nothing recomputed it if `sample_fps` changed, so the one number the join's
|
|
/// correctness rests on drifted silently with an unrelated knob.
|
|
///
|
|
/// It is also the largest single memory item in the pipeline. Every message
|
|
/// embeds `Frame source`, so a slot on this branch holds a full decoded image:
|
|
/// 256 of them is ~1.5 GB at 1080p, against ~110 MB for the derived depth at
|
|
/// 1 fps. See AR-004 — capacity is counted in items, and only the byte figure
|
|
/// (now correct, see types.hpp) shows what a slot really costs.
|
|
static std::size_t scene_join_depth(float sample_fps) {
|
|
const double slots = kSceneJoinSafety * kSceneJoinLagSec * sample_fps;
|
|
// Floor of 16: below that the queue stops absorbing ordinary jitter and
|
|
// starts throttling the fanout, which would slow the dense branch it
|
|
// exists to let run ahead.
|
|
return std::max<std::size_t>(16, static_cast<std::size_t>(std::ceil(slots)));
|
|
}
|
|
|
|
/// TRACES: AR-004 | SR-002
|
|
/// The decimator's input, on the *full-rate* stream.
|
|
///
|
|
/// This was also kSceneJoinDepth, which put a 256-slot buffer of full-rate
|
|
/// frames in front of the decimator — and at 1 fps against 24 fps native, 23 of
|
|
/// every 24 of those frames exist only to be discarded a moment later. Holding
|
|
/// ~1.5 GB of decoded images for frames the very next node throws away is the
|
|
/// worst available use of the memory budget.
|
|
///
|
|
/// A filter is a pass-through, not a reservoir: the lag belongs *after*
|
|
/// decimation, where a slot buys `1/sample_fps` seconds of film instead of
|
|
/// `1/native_fps`. Sized only to keep the decimator fed.
|
|
static constexpr std::size_t kDecimatorInputDepth = 16;
|
|
|
|
/// Set when the scene branch is built, so shutdown can report whether the join
|
|
/// actually worked.
|
|
static std::shared_ptr<SceneBoundaries> scene_stats;
|
|
|
|
/// TRACES: VR-015 | AR-004 | PR-004
|
|
/// Set by SIGUSR1, serviced by the wait loop. `kill -USR1 <pid>` on a running
|
|
/// or WEDGED run prints the benchmark table immediately — channel occupancy
|
|
/// names the stalled node (full input, empty output) without a debug build or a
|
|
/// debugger, which is the difference between diagnosing the AR-004 hang in
|
|
/// seconds and reproducing it under gdb.
|
|
///
|
|
/// The handler only stores a flag; all printing happens on the main thread,
|
|
/// since nothing in the report is async-signal-safe.
|
|
static std::atomic<bool> g_dump_request{false};
|
|
extern "C" void sae_on_dump_signal(int) {
|
|
g_dump_request.store(true, std::memory_order_relaxed);
|
|
}
|
|
|
|
static Config parse_args(int argc, char** argv) {
|
|
Config cfg;
|
|
cfg.detector_model = kDefaultDetectorModel;
|
|
cfg.arcface_model = kDefaultArcfaceModel;
|
|
cfg.scene_model = kDefaultSceneModel;
|
|
cfg.output_path = "annotations.json";
|
|
|
|
for (int i = 1; i < argc; ++i) {
|
|
auto arg = [&](const char* flag) { return std::strcmp(argv[i], flag) == 0; };
|
|
auto next = [&]() -> std::string {
|
|
if (++i >= argc) throw std::runtime_error(std::string("missing arg after ") + argv[i-1]);
|
|
return argv[i];
|
|
};
|
|
|
|
if (arg("--movie")) cfg.movie_path = next();
|
|
else if (arg("--gallery")) cfg.gallery_path = next();
|
|
else if (arg("--output")) cfg.output_path = next();
|
|
else if (arg("--dump-embeddings")) cfg.dump_embeddings_path = next();
|
|
else if (arg("--benchmark")) cfg.benchmark_path = next();
|
|
else if (arg("--benchmark-interval-ms")) cfg.benchmark_interval_ms = std::stoi(next());
|
|
else if (arg("--fps")) cfg.sample_fps = std::stof(next());
|
|
else if (arg("--max-decode-fps")) cfg.max_decode_fps = std::stof(next());
|
|
else if (arg("--start")) cfg.start_sec = std::stod(next());
|
|
else if (arg("--end")) cfg.end_sec = std::stod(next());
|
|
else if (arg("--cut-threshold")) cfg.cut_threshold = std::stof(next());
|
|
else if (arg("--presence-mode")) { std::string m = next(); cfg.presence_mode = (m == "flood") ? PresenceMode::flood : PresenceMode::track_extent; }
|
|
else if (arg("--scene-xgb-model")) cfg.scene_xgb_model = next();
|
|
else if (arg("--scene-detect")) cfg.scene_detect = true;
|
|
else if (arg("--scene-detector")) cfg.scene_model = next();
|
|
else if (arg("--scene-detector-engine")) cfg.scene_engine = next();
|
|
else if (arg("--scene-threshold")) cfg.scene_threshold = std::stof(next());
|
|
else if (arg("--scene-stride")) cfg.scene_stride = std::stoi(next());
|
|
else if (arg("--scene-decode-fps")) cfg.scene_decode_fps = std::stof(next());
|
|
else if (arg("--dense-scale")) cfg.dense_scale = std::stof(next());
|
|
else if (arg("--verbosity")) { int v = std::stoi(next()); cfg.verbosity = v == 2 ? Verbosity::xray : v == 1 ? Verbosity::standard : Verbosity::minimal; }
|
|
else if (arg("--prior")) cfg.match_prior = std::stof(next());
|
|
else if (arg("--prob-threshold")) cfg.prob_threshold = std::stof(next());
|
|
else if (arg("--detector")) cfg.detector_model = next();
|
|
else if (arg("--detector-engine")) cfg.detector_engine = next();
|
|
else if (arg("--arcface")) cfg.arcface_model = next();
|
|
else if (arg("--require-gallery-stamp")) cfg.require_gallery_stamp = true;
|
|
else if (arg("--arcface-engine")) cfg.arcface_engine = next();
|
|
else if (arg("--conf")) cfg.detector_conf = std::stof(next());
|
|
else if (arg("--max-faces")) cfg.max_faces = std::stoi(next());
|
|
else if (arg("--min-face-px")) cfg.min_face_px = std::stof(next());
|
|
else if (arg("--track-alpha")) cfg.track_alpha = std::stof(next());
|
|
else if (arg("--track-min-iou")) cfg.track_min_iou = std::stof(next());
|
|
else if (arg("--track-min-prob")) cfg.track_assoc_min_prob = std::stof(next());
|
|
else if (arg("--track-extinction")) cfg.track_extinction_sec = std::stod(next());
|
|
else if (arg("--ownership-logodds")) cfg.ownership_logodds = std::stof(next());
|
|
else if (arg("--evidence-rho-max")) cfg.evidence_rho_max = std::stof(next());
|
|
else if (arg("--evidence-admit-below")) cfg.evidence_admit_below = std::stof(next());
|
|
else if (arg("--evidence-max-views")) cfg.evidence_max_views = std::stoi(next());
|
|
else if (arg("--expand-gallery")) cfg.expand_gallery = true;
|
|
else if (arg("--expand-buffer")) cfg.expand_buffer_size = std::stoi(next());
|
|
else if (arg("--expand-band-lo")) cfg.expand_band_lo = std::stof(next());
|
|
else if (arg("--expand-band-hi")) cfg.expand_band_hi = std::stof(next());
|
|
else if (arg("--expand-min-anchor")) cfg.expand_min_anchor_frames = std::stoi(next());
|
|
else if (arg("--expand-debug-dir")) cfg.expand_debug_dir = next();
|
|
else if (arg("--trt-cache")) cfg.trt.cache_dir = next();
|
|
else if (arg("--trt-fp16")) cfg.trt.fp16 = true;
|
|
else if (arg("--no-trt-fp16")) cfg.trt.fp16 = false;
|
|
else if (arg("--trt-int8")) cfg.trt.int8 = true;
|
|
else if (arg("--embed-batch")) cfg.embed_batch_size = std::stoi(next());
|
|
#ifdef SAE_DEBUG
|
|
else if (arg("--debug-dir")) cfg.debug_dir = next();
|
|
else if (arg("--crop-context")) cfg.crop_context = std::stof(next());
|
|
#endif
|
|
else {
|
|
std::cerr << "[warn] unknown flag: " << argv[i] << "\n";
|
|
}
|
|
}
|
|
|
|
if (cfg.movie_path.empty()) throw std::runtime_error("--movie is required");
|
|
if (cfg.gallery_path.empty()) throw std::runtime_error("--gallery is required");
|
|
return cfg;
|
|
}
|
|
|
|
// ── Main ──────────────────────────────────────────────────────────────────────
|
|
|
|
int main(int argc, char** argv) {
|
|
/// TRACES: VR-015 | PR-004
|
|
// OpenCV here is built against TBB, so cv::parallel_for_ opens an arena of
|
|
// nproc-1 workers (19 on a 20-core box) *on top of* KPN's one thread per
|
|
// node. Two schedulers, neither aware of the other, on the same cores.
|
|
//
|
|
// SAE_CV_THREADS=1 hands concurrency entirely to KPN, which is where this
|
|
// pipeline's parallelism is supposed to come from. Worth measuring rather
|
|
// than assuming: TBB fan-out inside warpAffine is free speed when the
|
|
// pipeline is otherwise idle, so this can cut either way. Unset = default.
|
|
if (const char* t = std::getenv("SAE_CV_THREADS")) {
|
|
const int n = std::atoi(t);
|
|
cv::setNumThreads(n);
|
|
std::cerr << "[opencv] cv::setNumThreads(" << n << ")\n";
|
|
}
|
|
|
|
Config cfg;
|
|
try {
|
|
cfg = parse_args(argc, argv);
|
|
} catch (const std::exception& e) {
|
|
std::cerr << "Usage error: " << e.what() << "\n";
|
|
return 1;
|
|
}
|
|
|
|
// Load actor gallery
|
|
ActorGallery gallery;
|
|
try {
|
|
gallery = load_gallery(cfg.gallery_path);
|
|
/// TRACES: GR-004 | SR-001
|
|
// Hard startup error before a single frame is decoded: a gallery built
|
|
// with another embedder yields plausible-looking, meaningless matches.
|
|
verify_gallery_embedder(gallery, cfg.gallery_path, cfg.arcface_model,
|
|
cfg.require_gallery_stamp);
|
|
} catch (const std::exception& e) {
|
|
std::cerr << "Gallery error: " << e.what() << "\n";
|
|
return 1;
|
|
}
|
|
std::cerr << "[main] gallery loaded: " << gallery.actors.size() << " actors\n";
|
|
|
|
// ── Construct node functors ───────────────────────────────────────────────
|
|
|
|
std::atomic<bool> done{false}; // set by result_sink (face branch)
|
|
std::atomic<bool> scene_done{true}; // set by scene_detector; true when disabled
|
|
|
|
FrameSourceFunc source_fn {cfg};
|
|
CameraPositionChangeDetectorFunc campos_fn {cfg};
|
|
FaceDetectorFunc detector_fn{cfg};
|
|
FaceAlignerFunc aligner_fn;
|
|
EmbedderFunc embedder_fn{cfg};
|
|
// Constructed before the tracker: it fits (or loads) the calibration, and
|
|
// the tracker must decide in that same probability space (AR-024).
|
|
IdentityMatcherFunc matcher_fn {gallery, cfg};
|
|
|
|
/// TRACES: AR-007, AR-008, AR-012, AR-024 | SR-002
|
|
// The registry is created here and shared, not owned by a node: track state
|
|
// is not a stage in the stream, it is state several stages read and write,
|
|
// and its final answer is only known when a track dies.
|
|
auto same_person = same_person_probability(matcher_fn.calibration());
|
|
TrackRegistry::Config reg_cfg;
|
|
reg_cfg.track_extinction_sec = cfg.track_extinction_sec;
|
|
reg_cfg.ownership_logodds = cfg.ownership_logodds;
|
|
/// TRACES: AR-025 | SR-002
|
|
// The discounter's parameters come from Config now. They used to be
|
|
// in-class defaults reached through the one-argument constructor, so the
|
|
// VR-007 sweep that rho_max's own comment defers to could not vary it.
|
|
EvidenceDiscounter::Config disc_cfg;
|
|
disc_cfg.max_views = cfg.evidence_max_views;
|
|
disc_cfg.admit_below = cfg.evidence_admit_below;
|
|
disc_cfg.rho_max = cfg.evidence_rho_max;
|
|
auto registry = std::make_shared<TrackRegistry>(
|
|
reg_cfg, EvidenceDiscounter(same_person, disc_cfg));
|
|
|
|
matcher_fn.set_registry(registry);
|
|
|
|
|
|
FaceTrackerFunc ftracker_fn{cfg, registry, same_person};
|
|
FrameAnnotationFunc tracker_fn {};
|
|
ResultSinkFunc sink_fn {cfg, done};
|
|
|
|
/// TRACES: AR-012, AR-016 | IR-002, IR-003 | SR-002
|
|
// A reaped track goes straight to the aggregator, so the registry holds only
|
|
// live tracks and its size is bounded by concurrent on-screen faces rather
|
|
// than growing with the film.
|
|
registry->on_track_dead([&sink_fn](const DeadTrack& d) { sink_fn.add_claim(d); });
|
|
// AR-016: a film ends with faces on screen and those tracks have not timed
|
|
// out. Without this flush the closing scene's cast is silently never
|
|
// emitted — a loss that reads as a recognition miss, not a bookkeeping bug.
|
|
/// TRACES: VR-015 | PR-004
|
|
// Last timestamp the pipeline reached, latched on the way out. It is what
|
|
// turns wall-clock seconds into the number that matters — seconds of film
|
|
// per second of run — and the sink is the only node that knows it.
|
|
std::atomic<double> film_sec{0.0};
|
|
sink_fn.set_pre_write_hook([registry, &film_sec](double last_ts) {
|
|
film_sec.store(last_ts, std::memory_order_release);
|
|
registry->flush(last_ts);
|
|
});
|
|
#ifdef SAE_DEBUG
|
|
DebugRendererFunc debug_fn {cfg};
|
|
#endif
|
|
|
|
// ── Wrap in KPN ObjectNodes ───────────────────────────────────────────────
|
|
|
|
// Queue sizes tuned to the pipeline's speed profile:
|
|
// embedder (16ms) is the slowest GPU node — buffer before it must be largest
|
|
// to prevent face_aligner pool overflows and frame drops.
|
|
kpn::ObjectNode<FrameSourceFunc, kpn::in<>, kpn::out<"raw">, "frame_source", 0> source (source_fn, 32);
|
|
kpn::ObjectNode<CameraPositionChangeDetectorFunc, kpn::in<"raw">, kpn::out<"frame">, "camera_pos", 0> campos (campos_fn, 32);
|
|
kpn::ObjectNode<FaceDetectorFunc, kpn::in<"frame">, kpn::out<"scene">, "face_detector", 0> detector (detector_fn, 64);
|
|
kpn::ObjectNode<FaceAlignerFunc, kpn::in<"scene">, kpn::out<"aligned">, "face_aligner", 0> aligner (aligner_fn, 64);
|
|
kpn::ObjectNode<EmbedderFunc, kpn::in<"aligned">, kpn::out<"embedded">, "embedder", 0> embedder (embedder_fn, 32);
|
|
kpn::ObjectNode<FaceTrackerFunc, kpn::in<"embedded">, kpn::out<"tracked">, "face_tracker", 0> ftracker (ftracker_fn, 16);
|
|
kpn::ObjectNode<IdentityMatcherFunc, kpn::in<"tracked">, kpn::out<"matched">, "identity_matcher", 0> matcher (matcher_fn, 16);
|
|
kpn::ObjectNode<FrameAnnotationFunc, kpn::in<"matched">, kpn::out<"annotation">, "frame_annotation", 0> tracker (tracker_fn, 16);
|
|
kpn::ObjectNode<ResultSinkFunc, kpn::in<"annotation">,kpn::out<>, "result_sink", 0> sink (sink_fn, 16);
|
|
|
|
// ── Pipeline observability + run loop (topology-agnostic) ──────────────────
|
|
// Factored into a lambda so the two topologies (with/without the scene-detect
|
|
// branch) share identical event handling, wait loop, and teardown. Any
|
|
// make_network result type binds to `Net&&`.
|
|
std::mutex event_mtx;
|
|
std::map<std::string, long> overflow_counts;
|
|
std::atomic<bool> node_crashed{false};
|
|
|
|
auto run_net = [&](auto&& net) -> int {
|
|
// Tally per-node channel overflow, and treat any non-result_sink Closed
|
|
// event as a crash so the wait loop below can't hang on `done` forever.
|
|
net.set_event_handler(
|
|
[&](std::string_view node_name, kpn::NodeEvent ev,
|
|
std::chrono::steady_clock::time_point) {
|
|
if (ev == kpn::NodeEvent::Overflow) {
|
|
std::lock_guard<std::mutex> lk(event_mtx);
|
|
++overflow_counts[std::string(node_name)];
|
|
} else { // NodeEvent::Closed
|
|
if (node_name == "result_sink" && done.load(std::memory_order_acquire))
|
|
return;
|
|
std::cerr << "[main] node '" << node_name
|
|
<< "' stopped unexpectedly — aborting pipeline\n";
|
|
node_crashed.store(true, std::memory_order_release);
|
|
}
|
|
});
|
|
|
|
// Report *why* a node died. A Closed event alone says only that one
|
|
// stopped; the exception it carried is what identifies the fault, and
|
|
// without this listener it is discarded at the node boundary. Returning
|
|
// false keeps the existing semantics — the node still stops and the
|
|
// Closed handler above still aborts the run — but the run now names the
|
|
// cause instead of leaving it to be reconstructed from a debugger.
|
|
net.set_error_handler(
|
|
[&](std::string_view node_name, std::exception_ptr eptr) {
|
|
std::string what = "unknown exception";
|
|
try {
|
|
if (eptr) std::rethrow_exception(eptr);
|
|
} catch (const std::exception& e) {
|
|
what = e.what();
|
|
} catch (...) {
|
|
}
|
|
std::lock_guard<std::mutex> lk(event_mtx);
|
|
std::cerr << "[main] node '" << node_name << "' threw: " << what << "\n";
|
|
return false;
|
|
});
|
|
|
|
/// TRACES: VR-015 | PR-004
|
|
// Sampling must start with the network and stop before it is destroyed:
|
|
// channel fill is instantaneous, and by the time a run ends everything
|
|
// has drained, so a single read at shutdown reports an idle pipeline no
|
|
// matter how congested it was.
|
|
sae::bench::BenchmarkRecorder bench{cfg.benchmark_interval_ms};
|
|
const bool benchmarking = !cfg.benchmark_path.empty();
|
|
|
|
std::cerr << "[main] starting pipeline…\n";
|
|
net.start();
|
|
if (benchmarking) {
|
|
bench.start([&net] { return net.network_snapshot(); });
|
|
std::signal(SIGUSR1, sae_on_dump_signal);
|
|
std::cerr << "[benchmark] sampling every " << cfg.benchmark_interval_ms
|
|
<< "ms — `kill -USR1 " << getpid()
|
|
<< "` to dump the table now (works while hung)\n";
|
|
}
|
|
|
|
// Wait until BOTH terminal branches finish: result_sink (face pipeline)
|
|
// and, when enabled, scene_detector (the dense TransNetV2 branch, which
|
|
// runs much slower and must not be torn down mid-stream). scene_done is
|
|
// pre-set true when scene detection is disabled.
|
|
while ((!done.load(std::memory_order_acquire) ||
|
|
!scene_done.load(std::memory_order_acquire)) &&
|
|
!node_crashed.load(std::memory_order_acquire)) {
|
|
std::this_thread::sleep_for(std::chrono::milliseconds(100));
|
|
/// TRACES: VR-015 | AR-004 | PR-004
|
|
if (g_dump_request.exchange(false, std::memory_order_relaxed))
|
|
bench.dump_live(std::cerr, film_sec.load(std::memory_order_acquire));
|
|
}
|
|
|
|
// Latch the counters before stop(): they stay readable afterwards, but
|
|
// only while the network object is alive, and this keeps the numbers
|
|
// describing the run rather than the teardown.
|
|
if (benchmarking) bench.stop();
|
|
|
|
net.stop();
|
|
net.print_diagnostics();
|
|
|
|
/// TRACES: VR-015 | PR-004
|
|
if (benchmarking && bench.has_data()) {
|
|
const double film = film_sec.load(std::memory_order_acquire);
|
|
bench.print(std::cerr, film);
|
|
|
|
nlohmann::json run_cfg{
|
|
{"movie", cfg.movie_path},
|
|
{"gallery", cfg.gallery_path},
|
|
{"gallery_actors", gallery.actors.size()},
|
|
{"sample_fps", cfg.sample_fps},
|
|
{"min_face_px", cfg.min_face_px},
|
|
{"max_faces", cfg.max_faces},
|
|
{"embed_batch", cfg.embed_batch_size},
|
|
{"expand_gallery", cfg.expand_gallery},
|
|
{"scene_detect", cfg.scene_detect},
|
|
{"detector_engine", cfg.detector_engine},
|
|
{"arcface_engine", cfg.arcface_engine},
|
|
{"detector_model", cfg.detector_model},
|
|
{"arcface_model", cfg.arcface_model},
|
|
};
|
|
std::ofstream bf(cfg.benchmark_path);
|
|
if (bf) {
|
|
bf << bench.to_json(run_cfg, film).dump(2) << "\n";
|
|
std::cerr << "[benchmark] wrote " << cfg.benchmark_path << "\n";
|
|
} else {
|
|
std::cerr << "[benchmark] ERROR: could not write "
|
|
<< cfg.benchmark_path << "\n";
|
|
}
|
|
}
|
|
|
|
/// TRACES: AR-004 | SR-002
|
|
// A dropped frame does not degrade a result, it silently changes one —
|
|
// the output is a claim about footage that was never analysed, and
|
|
// nothing in the file says so. Since AR-004 made data pushes block, a
|
|
// drop can no longer happen on the data path, so any drop here means
|
|
// either that fix regressed (it lives in the KPN submodule, one line,
|
|
// easy to lose in an update) or a channel was disabled mid-run.
|
|
//
|
|
// Reporting it in a footer and exiting 0 made both invisible: the run
|
|
// "succeeded" and the truth file looked complete. Fail instead.
|
|
/// TRACES: AR-010 | SR-002
|
|
if (scene_stats) {
|
|
std::cerr << "[scene_annotate] boundaries=" << scene_stats->count()
|
|
<< " scored_through=" << scene_stats->scored_through() << "s";
|
|
// The tail is expected: frames after the detector's last full
|
|
// window are never covered, and no amount of buffering changes
|
|
// that. They are counted rather than silently treated as
|
|
// boundary-free, which is the distinction that matters.
|
|
if (scene_stats->outran() > 0)
|
|
std::cerr << " unscored=" << scene_stats->outran()
|
|
<< " frame(s) past the detector's last window — treated as"
|
|
" boundary-free, which is unverified rather than known";
|
|
std::cerr << "\n";
|
|
}
|
|
|
|
/// TRACES: AR-025, AR-012 | SR-002
|
|
// How often the registry was asked about a track it had already reaped.
|
|
//
|
|
// A vote is dropped when the matcher lags the tracker by more than
|
|
// track_extinction_sec of FILM time. The two are adjacent nodes with a
|
|
// 16-deep channel between them, and the matcher is much the slower of
|
|
// the pair (a GEMM over the whole gallery against a Hungarian solve over
|
|
// a handful of boxes), so that channel runs full and the lag is close to
|
|
// its depth. In frames:
|
|
//
|
|
// lag_sec ~= channel_depth / sample_fps
|
|
//
|
|
// At the default sample_fps of 1.0 that is ~16 s against a 5 s window,
|
|
// so votes CAN be dropped here, and each one is identity evidence that
|
|
// never reached the track it belonged to -- presence under-reported, in
|
|
// a way that reads as a recognition miss.
|
|
//
|
|
// Reported rather than fatal, deliberately, and the distinction from the
|
|
// dropped-frame case below is real: a dropped frame means the output
|
|
// describes footage nobody analysed, which is always wrong. A dropped
|
|
// vote means one observation of a track went missing, which degrades a
|
|
// claim without falsifying it. There is also no measurement yet of how
|
|
// often it happens on real content -- so this prints the number that
|
|
// would justify a harder line rather than presuming it. See VR-017.
|
|
if (registry) {
|
|
const int dv = registry->dropped_votes();
|
|
if (dv > 0) {
|
|
std::cerr << "[registry] WARNING: " << dv << " identity vote(s) "
|
|
"arrived for already-reaped tracks. The matcher is "
|
|
"lagging the tracker by more than track_extinction_sec ("
|
|
<< cfg.track_extinction_sec << "s) of film; presence is "
|
|
"under-reported. Raise --track-extinction or reduce the "
|
|
"face_tracker/identity_matcher channel depth.\n";
|
|
}
|
|
std::cerr << "[registry] belief_swaps=" << registry->belief_swaps()
|
|
<< " actor_conflicts=" << registry->actor_conflicts()
|
|
<< " dropped_votes=" << dv << "\n";
|
|
}
|
|
|
|
bool dropped = false;
|
|
{
|
|
std::lock_guard<std::mutex> lk(event_mtx);
|
|
if (!overflow_counts.empty()) {
|
|
dropped = true;
|
|
std::cerr << "[main] ERROR: frames were dropped (channel overflow):\n";
|
|
for (const auto& [name, count] : overflow_counts)
|
|
std::cerr << " " << name << ": " << count << "\n";
|
|
std::cerr << "[main] The output would describe footage that was never "
|
|
"analysed. Refusing to report success.\n";
|
|
}
|
|
}
|
|
if (node_crashed.load(std::memory_order_acquire)) return 1;
|
|
return dropped ? 2 : 0;
|
|
};
|
|
|
|
// ── Build static network and run ──────────────────────────────────────────
|
|
// Common face-analysis chain (campos → … → sink) is identical in all cases;
|
|
// the scene-detect branch and the debug fanout are spliced on conditionally.
|
|
// Topology:
|
|
// plain: source → campos → detector → … → sink
|
|
// scene-detect: source ─┬→ campos → decimate(filter) → detector → … → sink
|
|
// └→ scene_detector (TransNetV2 sink → scenes.json)
|
|
// The fanout after `source` is auto-inserted by make_network when its output
|
|
// feeds two edges. In dense mode campos still sees native-rate frames (so it
|
|
// detects angle changes correctly); a FilterNode then thins to sample_fps
|
|
// before face detection.
|
|
#ifdef SAE_DEBUG
|
|
kpn::ObjectNode<DebugRendererFunc, kpn::in<"matched">, kpn::out<>, "debug_renderer", 1> debug_node(debug_fn, 16);
|
|
#define SAE_DEBUG_EDGE , kpn::edge(matcher.output<"matched">(), debug_node.input<"matched">())
|
|
#else
|
|
#define SAE_DEBUG_EDGE
|
|
#endif
|
|
|
|
int rc = 0;
|
|
if (!cfg.dump_embeddings_path.empty()) {
|
|
// Dump-only topology: run the expensive front half and tee the embedder
|
|
// output to an HDF5 dump for offline sweep replay (sae_kpn). Downstream
|
|
// matching is skipped — the sweep re-runs it from the dump.
|
|
EmbeddingDumpFunc dump_fn{cfg, done};
|
|
kpn::ObjectNode<EmbeddingDumpFunc, kpn::in<"embedded">, kpn::out<>, "embedding_dump", 0>
|
|
dump_node(dump_fn, 32);
|
|
auto net = kpn::make_network(
|
|
kpn::edge(source.output<"raw">(), campos.input<"raw">()),
|
|
kpn::edge(campos.output<"frame">(), detector.input<"frame">()),
|
|
kpn::edge(detector.output<"scene">(), aligner.input<"scene">()),
|
|
kpn::edge(aligner.output<"aligned">(), embedder.input<"aligned">()),
|
|
kpn::edge(embedder.output<"embedded">(), dump_node.input<"embedded">())
|
|
);
|
|
return run_net(std::move(net));
|
|
}
|
|
if (cfg.scene_detect) {
|
|
scene_done.store(false, std::memory_order_release); // now a real terminal branch
|
|
SceneDetectorFunc scene_fn{cfg, scene_done};
|
|
|
|
/// TRACES: AR-010 | SR-002
|
|
// The join of the decode butterfly. source fans out to the dense
|
|
// TransNetV2 branch and the sampled face branch; boundaries found on the
|
|
// first have to reach the second, and cannot ride the frames because the
|
|
// branches run in parallel.
|
|
//
|
|
// TransNetV2 buffers kWindow frames before it can score any of them, so
|
|
// the face branch must lag by at least that much or it will ask about
|
|
// frames nobody has looked at yet. Channel depth is what creates the lag:
|
|
// with backpressure (AR-004) the fanout blocks on the slower branch, so
|
|
// a deep face-branch channel lets the detector run ahead by its window
|
|
// rather than dropping anything.
|
|
auto boundaries = std::make_shared<SceneBoundaries>();
|
|
scene_fn.set_boundaries(boundaries);
|
|
kpn::ObjectNode<SceneDetectorFunc, kpn::in<"dense">, kpn::out<>, "scene_detector", 0>
|
|
scene_node(scene_fn, kSceneInputDepth);
|
|
|
|
// Decimator: keep frames on the sample_fps cadence, drop the rest.
|
|
// eof always passes so downstream shuts down cleanly. Stateful — one
|
|
// instance, mutable via shared_ptr so the std::function stays copyable.
|
|
auto decim_state = std::make_shared<double>(-1e18);
|
|
const double interval = 1.0 / cfg.sample_fps;
|
|
auto decimate = kpn::make_filter<Frame>(
|
|
[decim_state, interval](const Frame& f) {
|
|
if (f.eof) return true;
|
|
if (f.timestamp_sec - *decim_state >= interval - 1e-6) {
|
|
*decim_state = f.timestamp_sec;
|
|
return true;
|
|
}
|
|
return false;
|
|
}, kDecimatorInputDepth);
|
|
|
|
/// TRACES: AR-010 | SR-002
|
|
// Stamp is_scene_boundary from the detector's published verdict. tol is
|
|
// half a sample interval: the two branches sample at different rates, so
|
|
// a boundary found on a dense frame rarely lands exactly on a sampled
|
|
// one, and half an interval attributes it to the nearest sampled frame
|
|
// and no further.
|
|
//
|
|
// outran() counts frames that arrived before the detector had scored
|
|
// them. Nonzero means the join depth is too shallow for the window, and
|
|
// those frames were annotated from an incomplete verdict — which would
|
|
// otherwise look exactly like "no boundary here".
|
|
SceneBoundaryAnnotatorFunc annotate_fn{boundaries, 0.5 / cfg.sample_fps};
|
|
kpn::ObjectNode<SceneBoundaryAnnotatorFunc, kpn::in<"frame">, kpn::out<"frame">,
|
|
"scene_annotate", 0> annotate(annotate_fn, scene_join_depth(cfg.sample_fps));
|
|
|
|
// Reported at shutdown: without this the join is unverifiable, and an
|
|
// annotator that never fired looks identical to footage with no
|
|
// boundaries.
|
|
scene_stats = boundaries;
|
|
|
|
auto net = kpn::make_network(
|
|
kpn::edge(source.output<"raw">(), campos.input<"raw">()),
|
|
kpn::edge(source.output<"raw">(), scene_node.input<"dense">()),
|
|
kpn::edge(campos.output<"frame">(), decimate.input<0>()),
|
|
kpn::edge(decimate.output<0>(), annotate.input<"frame">()),
|
|
kpn::edge(annotate.output<"frame">(), detector.input<"frame">()),
|
|
kpn::edge(detector.output<"scene">(), aligner.input<"scene">()),
|
|
kpn::edge(aligner.output<"aligned">(), embedder.input<"aligned">()),
|
|
kpn::edge(embedder.output<"embedded">(), ftracker.input<"embedded">()),
|
|
kpn::edge(ftracker.output<"tracked">(), matcher.input<"tracked">()),
|
|
kpn::edge(matcher.output<"matched">(), tracker.input<"matched">()),
|
|
kpn::edge(tracker.output<"annotation">(), sink.input<"annotation">())
|
|
SAE_DEBUG_EDGE
|
|
);
|
|
rc = run_net(std::move(net));
|
|
} else {
|
|
auto net = kpn::make_network(
|
|
kpn::edge(source.output<"raw">(), campos.input<"raw">()),
|
|
kpn::edge(campos.output<"frame">(), detector.input<"frame">()),
|
|
kpn::edge(detector.output<"scene">(), aligner.input<"scene">()),
|
|
kpn::edge(aligner.output<"aligned">(), embedder.input<"aligned">()),
|
|
kpn::edge(embedder.output<"embedded">(), ftracker.input<"embedded">()),
|
|
kpn::edge(ftracker.output<"tracked">(), matcher.input<"tracked">()),
|
|
kpn::edge(matcher.output<"matched">(), tracker.input<"matched">()),
|
|
kpn::edge(tracker.output<"annotation">(), sink.input<"annotation">())
|
|
SAE_DEBUG_EDGE
|
|
);
|
|
rc = run_net(std::move(net));
|
|
}
|
|
#undef SAE_DEBUG_EDGE
|
|
|
|
return rc;
|
|
}
|