dtourolle/scene-actor-extraction
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(engine): HDF5-native galleries with embedded calibration; TensorRT backends; scene detection

Browse Source 
Gallery format switches from JSON to HDF5 exclusively (JSON read-only kept for
back-compat): save_gallery always writes HDF5, and the fitted Platt-sigmoid
calibration (a, b, valid, hash) is now embedded directly in the gallery file
instead of a sidecar .calib_cache.json — identity_matcher reads it from the
loaded gallery and writes back only when the embeddings actually changed
(hash mismatch), skipping the O(n^2) refit otherwise.

Also includes: TensorRT inference backend support (ort_backend.cpp,
trt_backend.cpp), gemm_backend improvements, TransNetV2-based scene-boundary
detection wired through frame_source/face_tracker/main, and CMake build
target updates for the new sources.

Bumps the KPN submodule to feature/persistent-pipeline-reuse (push_blocking
backpressure, node_ptr/node_stats introspection, ObjectVariantNodeWrapper for
stateful functors) — needed by the optimizer's sae_kpn Python bindings.
This commit is contained in:
dtourolle
2026-07-19 19:04:03 +02:00
parent aca6147d69
commit 41a277bc19
19 changed files with 1151 additions and 216 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/nodes/face_detector_node.hpp
+7 -2
View File
@@ -26,10 +26,15 @@ struct FaceDetectorFunc {
auto faces = detector_->detect(f.image);
// Drop faces below minimum pixel size (too small for reliable ArcFace alignment)
// Drop faces below minimum pixel size (too small for reliable ArcFace
// alignment). Note: when dense_scale downscaled the frame, both the
// detection coords and min_face_px are in downscaled space — so scale
// the threshold down to match, keeping the physical size cutoff constant.
const float min_px = (f.bbox_upscale != 1.f)
? min_face_px_ / f.bbox_upscale : min_face_px_;
faces.erase(
std::remove_if(faces.begin(), faces.end(), [&](const DetectedFace& d) {
return d.bbox.width < min_face_px_ || d.bbox.height < min_face_px_;
return d.bbox.width < min_px || d.bbox.height < min_px;
}),
faces.end());
src/nodes/face_tracker_node.hpp
+75 -4
View File
@@ -23,6 +23,16 @@
//
// Unmatched tracks have their frames_missing counter incremented; they are
// expired once frames_missing > max_frames_missing.
//
// Cross-cut re-association. A camera-angle change (Frame::is_cut, set by
// camera_position_change_detector) destroys spatial (IoU) continuity — the same
// person reappears at a new position — but not identity. On a cut the tracker
// does NOT discard its tracks; it parks them in an inactive pool keyed by their
// last-frame raw embedding. A post-cut detection whose raw cosine similarity to
// a parked track's last-frame embedding is ≥ cut_revive_sim revives that track:
// the original track_id, mean embedding and n_frames are restored (only the bbox
// jumps to the new detection), so identity continuity survives the cut. Parked
// tracks left unrevived for cut_inactive_max_frames are finally dropped.
struct FaceTrackerFunc {
static constexpr std::string_view label() { return "face_tracker"; }
@@ -30,6 +40,7 @@ struct FaceTrackerFunc {
struct TrackState {
cv::Rect2f bbox;
Embedding mean_emb{};
Embedding last_emb{}; // raw embedding of the most recent matched frame
int n_frames{0};
int frames_missing{0};
};
@@ -39,16 +50,21 @@ struct FaceTrackerFunc {
, min_iou_(cfg.track_min_iou)
, max_embed_dist_(cfg.track_max_embed_dist)
, max_missing_(cfg.track_max_frames_missing)
, revive_sim_(cfg.cut_revive_sim)
, inactive_max_(cfg.cut_inactive_max_frames)
{
std::cerr << "[face_tracker] alpha=" << alpha_
<< " min_iou=" << min_iou_
<< " max_embed_dist=" << max_embed_dist_
<< " max_missing=" << max_missing_ << "\n";
<< " max_missing=" << max_missing_
<< " cut_revive_sim=" << revive_sim_
<< " cut_inactive_max=" << inactive_max_ << "\n";
}
TrackedSceneFrame operator()(EmbeddedSceneFrame ef) {
if (ef.source.eof) {
tracks_.clear();
inactive_.clear();
TrackedSceneFrame out;
out.source = std::move(ef.source);
return out;
@@ -56,11 +72,26 @@ struct FaceTrackerFunc {
const int n_det = static_cast<int>(ef.embeddings.size());
// Camera-angle change: park active tracks instead of destroying them so
// they can be revived by identity (raw last-frame embedding cosine) once
// the same people reappear from the new angle.
if (ef.source.is_cut && !tracks_.empty()) {
std::cerr << "[face_tracker] cut — clearing " << tracks_.size() << " tracks\n";
std::cerr << "[face_tracker] cut — parking " << tracks_.size()
<< " track(s) into inactive pool\n";
for (auto& [tid, ts] : tracks_) {
ts.frames_missing = 0; // repurpose as time-since-parked counter
inactive_[tid] = std::move(ts);
}
tracks_.clear();
}
// Age the inactive pool every frame and drop tracks parked too long.
for (auto it = inactive_.begin(); it != inactive_.end(); ) {
it->second.frames_missing++;
it = (it->second.frames_missing > inactive_max_)
? inactive_.erase(it) : std::next(it);
}
// Snapshot active track IDs so the map can be modified safely below
std::vector<int> tids;
tids.reserve(tracks_.size());
@@ -111,6 +142,7 @@ struct FaceTrackerFunc {
continue;
}
update_mean(ts.mean_emb, ts.n_frames, ef.embeddings[di]);
ts.last_emb = ef.embeddings[di];
ts.bbox = ef.faces[di].bbox;
ts.n_frames++;
ts.frames_missing = 0;
@@ -119,13 +151,34 @@ struct FaceTrackerFunc {
out.track_ids[di] = tids[ti];
}
// Create new tracks for unmatched detections
// Handle unmatched detections: first try to revive a parked track by
// identity (raw last-frame embedding cosine), else start a fresh track.
for (int di = 0; di < n_det; ++di) {
if (det_matched[di]) continue;
int tid = next_id_++;
int tid = revive_from_inactive(ef.embeddings[di]);
if (tid >= 0) {
// Restore the parked track: keep its identity statistics
// (mean_emb, n_frames), jump the bbox to the new detection.
TrackState ts = std::move(inactive_[tid]);
inactive_.erase(tid);
update_mean(ts.mean_emb, ts.n_frames, ef.embeddings[di]);
ts.last_emb = ef.embeddings[di];
ts.bbox = ef.faces[di].bbox;
ts.n_frames++;
ts.frames_missing = 0;
tracks_[tid] = std::move(ts);
out.track_ids[di] = tid;
std::cerr << "[face_tracker] revived track " << tid
<< " across cut\n";
continue;
}
tid = next_id_++;
TrackState ts;
ts.bbox = ef.faces[di].bbox;
ts.mean_emb = ef.embeddings[di];
ts.last_emb = ef.embeddings[di];
ts.n_frames = 1;
tracks_[tid] = ts;
out.track_ids[di] = tid;
@@ -141,6 +194,21 @@ struct FaceTrackerFunc {
}
private:
// Pick the parked track whose last-frame embedding is most similar to emb,
// returning its id if that raw cosine similarity clears revive_sim_, else -1.
// The caller removes the returned track from the pool, so a later detection in
// the same frame cannot claim it again.
int revive_from_inactive(const Embedding& emb) const {
int best_tid = -1;
float best_sim = revive_sim_; // threshold is the bar to beat (inclusive)
for (const auto& [tid, ts] : inactive_) {
float sim = cosine_similarity(ts.last_emb, emb);
if (sim >= best_sim) { best_sim = sim; best_tid = tid; }
// subsequent ties keep the later id; harmless, all clear the threshold
}
return best_tid;
}
// IoU of two axis-aligned bounding boxes
static float iou(const cv::Rect2f& a, const cv::Rect2f& b) {
float ix = std::max(0.f, std::min(a.x + a.width, b.x + b.width)
@@ -225,9 +293,12 @@ private:
}
std::map<int, TrackState> tracks_;
std::map<int, TrackState> inactive_; // parked across a cut, keyed by track id
int next_id_{0};
float alpha_;
float min_iou_;
float max_embed_dist_;
int max_missing_;
float revive_sim_;
int inactive_max_;
};
src/nodes/frame_source_node.hpp
+39 -30
View File
@@ -3,7 +3,6 @@
#include "config.hpp"
#include "ffmpeg_decoder.hpp"
#include <opencv2/imgproc.hpp>
#include <chrono>
#include <iostream>
#include <memory>
@@ -27,21 +26,46 @@ struct FrameSourceFunc {
static constexpr std::string_view label() { return "frame_source"; }
explicit FrameSourceFunc(const Config& cfg)
: decoder_(std::make_unique<FFmpegDecoder>(cfg.movie_path))
: decoder_(std::make_unique<FFmpegDecoder>(
cfg.movie_path, /*use_hw=*/true,
/*out_scale=*/cfg.scene_detect ? cfg.dense_scale : 1.0f))
{
sample_interval_sec_ = 1.0 / cfg.sample_fps;
next_pos_sec_ = cfg.start_sec;
end_sec_ = cfg.end_sec;
cut_threshold_ = cfg.cut_threshold;
max_decode_fps_ = cfg.max_decode_fps;
// Dense mode: emit every native-rate frame instead of seeking to each
// 1-FPS sample point. Required by the TransNetV2 scene detector, which
// needs consecutive frames. A downstream decimator drops back to
// sample_fps for the face pipeline. When dense, we advance by the
// decoder's frame period (best effort — read_at decodes forward past the
// last position, so consecutive small steps yield consecutive frames).
dense_ = cfg.scene_detect;
double dense_fps = 0.0;
if (dense_) {
double vfps = decoder_->fps();
if (vfps <= 0.0) vfps = 25.0;
// Dense decode rate: capped at native fps. A lower scene_decode_fps
// decodes fewer frames (big speedup); TransNetV2 still localises cuts
// and boundary timestamps stay exact (keyed off real timestamps).
dense_fps = (cfg.scene_decode_fps > 0.f)
? std::min(static_cast<double>(cfg.scene_decode_fps), vfps)
: vfps;
dense_step_sec_ = 1.0 / dense_fps;
if (cfg.dense_scale > 0.f && cfg.dense_scale < 1.f)
bbox_upscale_ = 1.0f / cfg.dense_scale;
}
double total_s = decoder_->duration_sec();
double span_s = (end_sec_ > 0 ? std::min(end_sec_, total_s) : total_s)
- cfg.start_sec;
int n_frames = static_cast<int>(span_s * cfg.sample_fps);
double emit_fps = dense_ ? dense_fps : cfg.sample_fps;
int n_frames = static_cast<int>(span_s * emit_fps);
std::cerr << "[frame_source] decoder=" << decoder_->codec_name()
<< " (" << decoder_->hw_backend() << ")"
<< " video_fps=" << decoder_->fps()
<< (dense_ ? " DENSE@" + std::to_string(dense_fps) + "fps" : "")
<< " start=" << cfg.start_sec << "s"
<< (end_sec_ > 0 ? " end=" + std::to_string(end_sec_) + "s" : "")
<< " sample_fps=" << cfg.sample_fps
@@ -97,29 +121,14 @@ struct FrameSourceFunc {
return Frame{{}, next_pos_sec_, frame_idx_++, /*eof=*/true};
}
// Cut detection: compare grayscale histogram to previous frame
bool is_cut = false;
cv::Mat gray;
cv::cvtColor(img, gray, cv::COLOR_BGR2GRAY);
cv::Mat hist;
const int bins = 64;
const float range[] = {0.f, 256.f};
const float* ranges = range;
cv::calcHist(&gray, 1, nullptr, cv::Mat(), hist, 1, &bins, &ranges);
cv::normalize(hist, hist, 1.0, 0.0, cv::NORM_L1);
if (prev_hist_valid_) {
double corr = cv::compareHist(prev_hist_, hist, cv::HISTCMP_CORREL);
is_cut = (corr < cut_threshold_);
if (is_cut)
std::cerr << "[frame_source] cut at t=" << next_pos_sec_
<< "s hist_corr=" << corr << "\n";
}
prev_hist_ = hist;
prev_hist_valid_ = true;
Frame f{img, next_pos_sec_, frame_idx_++, /*eof=*/false, is_cut};
next_pos_sec_ += sample_interval_sec_;
// Cut detection is a downstream concern: camera_position_change_detector
// owns the histogram compare and sets Frame::is_cut. The source emits
// is_cut=false and only decodes/samples frames.
Frame f{img, next_pos_sec_, frame_idx_++, /*eof=*/false, /*is_cut=*/false};
// When dense_scale downscaled the decode, tell downstream how to map
// detector coordinates back to original video resolution.
f.bbox_upscale = bbox_upscale_;
next_pos_sec_ += dense_ ? dense_step_sec_ : sample_interval_sec_;
if (end_sec_ > 0 && next_pos_sec_ > end_sec_) {
hit_eof_ = true;
std::cerr << "[frame_source] reached end_sec=" << end_sec_ << "s\n";
@@ -130,16 +139,16 @@ struct FrameSourceFunc {
private:
std::unique_ptr<FFmpegDecoder> decoder_;
double sample_interval_sec_{1.0};
bool dense_{false};
float bbox_upscale_{1.f};
double dense_step_sec_{0.04};
double next_pos_sec_{0.0};
double end_sec_{-1.0};
float cut_threshold_{0.70f};
float max_decode_fps_{0.f};
std::chrono::steady_clock::time_point next_decode_at_{};
bool rate_started_{false};
int64_t frame_idx_{0};
bool hit_eof_{false};
cv::Mat prev_hist_;
bool prev_hist_valid_{false};
double decode_ms_acc_{0.0};
int decode_count_{0};
};
src/nodes/identity_matcher_node.hpp
+60 -2
View File
@@ -4,6 +4,7 @@
#include "inference/similarity.hpp"
#include "gallery/gallery_store.hpp"
#include "gallery/gallery_calibration.hpp"
#include "gallery/track_gallery.hpp"
#include <cstdint>
#include <cstring>
@@ -51,6 +52,7 @@ struct IdentityMatcherFunc {
, threshold_(cfg.match_threshold)
, ratio_(cfg.match_ratio)
, ratio_ceil_(cfg.match_ratio_ceil)
, track_gallery_(cfg)
{
std::cerr << "[identity_matcher] flattening gallery embeddings...\n";
for (int ai = 0; ai < static_cast<int>(gallery_.actors.size()); ++ai) {
@@ -63,8 +65,24 @@ struct IdentityMatcherFunc {
std::cerr << "[identity_matcher] starting calibration ("
<< flat_emb_.size() << " embeddings)...\n";
bool recomputed = false;
cal_ = calibrate_gallery_cached(flat_emb_, flat_actor_,
cfg.gallery_path + ".calib_cache.json");
gallery_.calib_a, gallery_.calib_b,
gallery_.calib_valid, gallery_.calib_hash,
cfg.gallery_path + ".calib_cache", recomputed);
if (recomputed) {
// Persist the freshly-fitted calibration into the gallery file (always
// HDF5 — save_gallery rewrites any other extension, see gallery_store.cpp)
// so the next run against this same, unchanged gallery skips the O(n^2) fit.
ActorGallery to_save = gallery_;
to_save.calib_a = cal_.a;
to_save.calib_b = cal_.b;
to_save.calib_valid = cal_.valid;
to_save.calib_hash = hash_gallery_embeddings(flat_emb_, flat_actor_);
save_gallery(cfg.gallery_path, to_save);
std::cerr << "[identity_matcher] wrote refreshed calibration back to "
<< cfg.gallery_path << "\n";
}
if (cal_.valid) {
std::cerr << "[identity_matcher] calibrated Bayesian matching"
@@ -89,8 +107,23 @@ struct IdentityMatcherFunc {
sim_engine_ = make_similarity_engine(host_gallery.data(), n_gallery_, kMaxFaces);
}
// Runtime setter — lets a persistent pipeline be reused across a threshold sweep
// without rebuilding the (expensive, gallery-resident) matcher. The gallery,
// calibration and GPU sim-engine stay put; only the accept threshold changes.
void set_prob_threshold(float t) { prob_threshold_ = t; }
MatchedSceneFrame operator()(TrackedSceneFrame tf) {
if (tf.source.eof) return {std::move(tf.source), {}};
if (tf.source.eof) {
track_gallery_.clear_tracks();
return {std::move(tf.source), {}};
}
// A hard cut changes the camera viewpoint. The face_tracker may revive a
// track_id across the cut (identity continuity), but promotion must never
// mix embeddings from two viewpoints under one buffer, so we still drop
// every diversity buffer here — a revived track simply re-accumulates its
// buffer from post-cut frames. Stale cross-cut embeddings are never promoted.
if (tf.source.is_cut) track_gallery_.clear_tracks();
const int n_faces = static_cast<int>(tf.embeddings.size());
std::vector<IdentifiedActor> actors;
@@ -120,6 +153,14 @@ struct IdentityMatcherFunc {
if (sim > best_sim[ai]) best_sim[ai] = sim;
}
// Fold in the per-film annex (CPU-side, tens of embeddings). Promoted
// pose-varied views compete for best-of-N exactly like baked refs, so
// a face at a pose the gallery lacked can now win its true actor.
for (const auto& ae : track_gallery_.annex()) {
float sim = cosine_similarity(tf.embeddings[fi], ae.emb);
if (sim > best_sim[ae.actor_idx]) best_sim[ae.actor_idx] = sim;
}
int best_actor = -1;
int second_actor = -1;
float best_s = -std::numeric_limits<float>::max();
@@ -155,7 +196,15 @@ struct IdentityMatcherFunc {
}
IdentifiedActor ia;
// Map bbox back to original video resolution when dense_scale
// downscaled the decoded frame (detection/tracking ran downscaled;
// output bboxes must be in original pixel space).
ia.bbox = tf.faces[fi].bbox;
if (tf.source.bbox_upscale != 1.f) {
const float s = tf.source.bbox_upscale;
ia.bbox.x *= s; ia.bbox.y *= s;
ia.bbox.width *= s; ia.bbox.height *= s;
}
ia.crop = tf.crops[fi];
ia.track_id = tf.track_ids[fi];
@@ -170,6 +219,14 @@ struct IdentityMatcherFunc {
: best_s;
}
// Feed this face into per-film gallery expansion. best_actor/best_s
// reflect the actor with the strongest gallery similarity for this
// face (annex already folded in above); the track's diversity buffer
// keeps the gallery-far views and promotes them once the track is
// confirmed. No-op unless --expand-gallery is set.
track_gallery_.observe(tf.track_ids[fi], tf.embeddings[fi],
best_actor, best_s, accept, tf.crops[fi]);
actors.push_back(std::move(ia));
}
@@ -189,4 +246,5 @@ private:
int n_gallery_{0};
std::unique_ptr<ISimilarityEngine> sim_engine_;
TrackGallery track_gallery_;
};
src/nodes/preview_node.hpp
+41
View File
@@ -7,6 +7,7 @@
#include <opencv2/highgui.hpp>
#include <opencv2/imgproc.hpp>
#include <algorithm>
#include <string>
// ── PreviewNode ───────────────────────────────────────────────────────────────
@@ -35,6 +36,7 @@ public:
cv::Mat display = mf.source.image.clone();
draw_detections(display, mf.actors);
draw_hud(display, mf.source.timestamp_sec, mf.actors);
draw_cut_meter(display, mf.source.cut_score, mf.source.is_cut);
// Fit to display width while keeping aspect ratio
if (display.cols > max_display_w_) {
@@ -132,6 +134,45 @@ private:
}
}
// Running cut-score meter (top-right): a horizontal bar whose fill and colour
// track the histogram cut score in [0,1] (0 = no change, ~1 = hard cut).
// Flashes a red "CUT" tag on frames where the threshold tripped (is_cut).
static void draw_cut_meter(cv::Mat& img, float score, bool is_cut) {
const int bar_w = 220;
const int bar_h = 14;
const int pad = 10;
const int x0 = img.cols - bar_w - pad;
const int y0 = pad + 16; // below any top-left HUD row height
score = std::clamp(score, 0.f, 1.f);
// Label
char lbl[32];
std::snprintf(lbl, sizeof(lbl), "cut %.2f", score);
cv::putText(img, lbl, cv::Point(x0, y0 - 4),
cv::FONT_HERSHEY_SIMPLEX, 0.5,
cv::Scalar(220, 220, 220), 1, cv::LINE_AA);
// Track (dark) + fill (green→red by score)
cv::rectangle(img, cv::Rect(x0, y0, bar_w, bar_h),
cv::Scalar(40, 40, 40), cv::FILLED);
int fill_w = static_cast<int>(bar_w * score);
// BGR: low score → green (0,210,60), high → red (0,0,255)
cv::Scalar fill_col(60.0 * (1.f - score),
210.0 * (1.f - score),
60.0 * (1.f - score) + 255.0 * score);
if (fill_w > 0)
cv::rectangle(img, cv::Rect(x0, y0, fill_w, bar_h), fill_col, cv::FILLED);
cv::rectangle(img, cv::Rect(x0, y0, bar_w, bar_h),
cv::Scalar(120, 120, 120), 1);
if (is_cut) {
cv::putText(img, "CUT", cv::Point(x0 - 52, y0 + bar_h),
cv::FONT_HERSHEY_DUPLEX, 0.6,
cv::Scalar(0, 0, 255), 2, cv::LINE_AA);
}
}
static std::string pct(float v) {
char buf[8];
std::snprintf(buf, sizeof(buf), "%.0f%%", v * 100.f);
src/nodes/scene_tracker_node.hpp
+4
View File
@@ -26,6 +26,10 @@ struct SceneTrackerFunc {
std::cerr << "[scene_tracker] extinction_sec=" << extinction_sec_ << "\n";
}
// Runtime setter for pipeline reuse across a sweep. Also clears the active-actor
// state so a re-run starts clean (no carry-over from the previous config's film).
void set_extinction_sec(double s) { extinction_sec_ = s; active_.clear(); }
SceneAnnotation operator()(MatchedSceneFrame mf) {
if (mf.source.eof) return {0.0, {}, /*eof=*/true};