dtourolle/scene-actor-extraction
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(engine): HDF5-native galleries with embedded calibration; TensorRT backends; scene detection

Browse Source 
Gallery format switches from JSON to HDF5 exclusively (JSON read-only kept for
back-compat): save_gallery always writes HDF5, and the fitted Platt-sigmoid
calibration (a, b, valid, hash) is now embedded directly in the gallery file
instead of a sidecar .calib_cache.json — identity_matcher reads it from the
loaded gallery and writes back only when the embeddings actually changed
(hash mismatch), skipping the O(n^2) refit otherwise.

Also includes: TensorRT inference backend support (ort_backend.cpp,
trt_backend.cpp), gemm_backend improvements, TransNetV2-based scene-boundary
detection wired through frame_source/face_tracker/main, and CMake build
target updates for the new sources.

Bumps the KPN submodule to feature/persistent-pipeline-reuse (push_blocking
backpressure, node_ptr/node_stats introspection, ObjectVariantNodeWrapper for
stateful functors) — needed by the optimizer's sae_kpn Python bindings.
This commit is contained in:
dtourolle
2026-07-19 19:04:03 +02:00
parent aca6147d69
commit 41a277bc19
19 changed files with 1151 additions and 216 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/nodes/identity_matcher_node.hpp
+60 -2
View File
@@ -4,6 +4,7 @@
#include "inference/similarity.hpp"
#include "gallery/gallery_store.hpp"
#include "gallery/gallery_calibration.hpp"
#include "gallery/track_gallery.hpp"
#include <cstdint>
#include <cstring>
@@ -51,6 +52,7 @@ struct IdentityMatcherFunc {
, threshold_(cfg.match_threshold)
, ratio_(cfg.match_ratio)
, ratio_ceil_(cfg.match_ratio_ceil)
, track_gallery_(cfg)
{
std::cerr << "[identity_matcher] flattening gallery embeddings...\n";
for (int ai = 0; ai < static_cast<int>(gallery_.actors.size()); ++ai) {
@@ -63,8 +65,24 @@ struct IdentityMatcherFunc {
std::cerr << "[identity_matcher] starting calibration ("
<< flat_emb_.size() << " embeddings)...\n";
bool recomputed = false;
cal_ = calibrate_gallery_cached(flat_emb_, flat_actor_,
cfg.gallery_path + ".calib_cache.json");
gallery_.calib_a, gallery_.calib_b,
gallery_.calib_valid, gallery_.calib_hash,
cfg.gallery_path + ".calib_cache", recomputed);
if (recomputed) {
// Persist the freshly-fitted calibration into the gallery file (always
// HDF5 — save_gallery rewrites any other extension, see gallery_store.cpp)
// so the next run against this same, unchanged gallery skips the O(n^2) fit.
ActorGallery to_save = gallery_;
to_save.calib_a = cal_.a;
to_save.calib_b = cal_.b;
to_save.calib_valid = cal_.valid;
to_save.calib_hash = hash_gallery_embeddings(flat_emb_, flat_actor_);
save_gallery(cfg.gallery_path, to_save);
std::cerr << "[identity_matcher] wrote refreshed calibration back to "
<< cfg.gallery_path << "\n";
}
if (cal_.valid) {
std::cerr << "[identity_matcher] calibrated Bayesian matching"
@@ -89,8 +107,23 @@ struct IdentityMatcherFunc {
sim_engine_ = make_similarity_engine(host_gallery.data(), n_gallery_, kMaxFaces);
}
// Runtime setter — lets a persistent pipeline be reused across a threshold sweep
// without rebuilding the (expensive, gallery-resident) matcher. The gallery,
// calibration and GPU sim-engine stay put; only the accept threshold changes.
void set_prob_threshold(float t) { prob_threshold_ = t; }
MatchedSceneFrame operator()(TrackedSceneFrame tf) {
if (tf.source.eof) return {std::move(tf.source), {}};
if (tf.source.eof) {
track_gallery_.clear_tracks();
return {std::move(tf.source), {}};
}
// A hard cut changes the camera viewpoint. The face_tracker may revive a
// track_id across the cut (identity continuity), but promotion must never
// mix embeddings from two viewpoints under one buffer, so we still drop
// every diversity buffer here — a revived track simply re-accumulates its
// buffer from post-cut frames. Stale cross-cut embeddings are never promoted.
if (tf.source.is_cut) track_gallery_.clear_tracks();
const int n_faces = static_cast<int>(tf.embeddings.size());
std::vector<IdentifiedActor> actors;
@@ -120,6 +153,14 @@ struct IdentityMatcherFunc {
if (sim > best_sim[ai]) best_sim[ai] = sim;
}
// Fold in the per-film annex (CPU-side, tens of embeddings). Promoted
// pose-varied views compete for best-of-N exactly like baked refs, so
// a face at a pose the gallery lacked can now win its true actor.
for (const auto& ae : track_gallery_.annex()) {
float sim = cosine_similarity(tf.embeddings[fi], ae.emb);
if (sim > best_sim[ae.actor_idx]) best_sim[ae.actor_idx] = sim;
}
int best_actor = -1;
int second_actor = -1;
float best_s = -std::numeric_limits<float>::max();
@@ -155,7 +196,15 @@ struct IdentityMatcherFunc {
}
IdentifiedActor ia;
// Map bbox back to original video resolution when dense_scale
// downscaled the decoded frame (detection/tracking ran downscaled;
// output bboxes must be in original pixel space).
ia.bbox = tf.faces[fi].bbox;
if (tf.source.bbox_upscale != 1.f) {
const float s = tf.source.bbox_upscale;
ia.bbox.x *= s; ia.bbox.y *= s;
ia.bbox.width *= s; ia.bbox.height *= s;
}
ia.crop = tf.crops[fi];
ia.track_id = tf.track_ids[fi];
@@ -170,6 +219,14 @@ struct IdentityMatcherFunc {
: best_s;
}
// Feed this face into per-film gallery expansion. best_actor/best_s
// reflect the actor with the strongest gallery similarity for this
// face (annex already folded in above); the track's diversity buffer
// keeps the gallery-far views and promotes them once the track is
// confirmed. No-op unless --expand-gallery is set.
track_gallery_.observe(tf.track_ids[fi], tf.embeddings[fi],
best_actor, best_s, accept, tf.crops[fi]);
actors.push_back(std::move(ia));
}
@@ -189,4 +246,5 @@ private:
int n_gallery_{0};
std::unique_ptr<ISimilarityEngine> sim_engine_;
TrackGallery track_gallery_;
};