dtourolle/scene-actor-extraction
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(engine): HDF5-native galleries with embedded calibration; TensorRT backends; scene detection

Browse Source 
Gallery format switches from JSON to HDF5 exclusively (JSON read-only kept for
back-compat): save_gallery always writes HDF5, and the fitted Platt-sigmoid
calibration (a, b, valid, hash) is now embedded directly in the gallery file
instead of a sidecar .calib_cache.json — identity_matcher reads it from the
loaded gallery and writes back only when the embeddings actually changed
(hash mismatch), skipping the O(n^2) refit otherwise.

Also includes: TensorRT inference backend support (ort_backend.cpp,
trt_backend.cpp), gemm_backend improvements, TransNetV2-based scene-boundary
detection wired through frame_source/face_tracker/main, and CMake build
target updates for the new sources.

Bumps the KPN submodule to feature/persistent-pipeline-reuse (push_blocking
backpressure, node_ptr/node_stats introspection, ObjectVariantNodeWrapper for
stateful functors) — needed by the optimizer's sae_kpn Python bindings.
This commit is contained in:
dtourolle
2026-07-19 19:04:03 +02:00
parent aca6147d69
commit 41a277bc19
19 changed files with 1151 additions and 216 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/nodes/face_tracker_node.hpp
+75 -4
View File
@@ -23,6 +23,16 @@
//
// Unmatched tracks have their frames_missing counter incremented; they are
// expired once frames_missing > max_frames_missing.
//
// Cross-cut re-association. A camera-angle change (Frame::is_cut, set by
// camera_position_change_detector) destroys spatial (IoU) continuity — the same
// person reappears at a new position — but not identity. On a cut the tracker
// does NOT discard its tracks; it parks them in an inactive pool keyed by their
// last-frame raw embedding. A post-cut detection whose raw cosine similarity to
// a parked track's last-frame embedding is ≥ cut_revive_sim revives that track:
// the original track_id, mean embedding and n_frames are restored (only the bbox
// jumps to the new detection), so identity continuity survives the cut. Parked
// tracks left unrevived for cut_inactive_max_frames are finally dropped.
struct FaceTrackerFunc {
static constexpr std::string_view label() { return "face_tracker"; }
@@ -30,6 +40,7 @@ struct FaceTrackerFunc {
struct TrackState {
cv::Rect2f bbox;
Embedding mean_emb{};
Embedding last_emb{}; // raw embedding of the most recent matched frame
int n_frames{0};
int frames_missing{0};
};
@@ -39,16 +50,21 @@ struct FaceTrackerFunc {
, min_iou_(cfg.track_min_iou)
, max_embed_dist_(cfg.track_max_embed_dist)
, max_missing_(cfg.track_max_frames_missing)
, revive_sim_(cfg.cut_revive_sim)
, inactive_max_(cfg.cut_inactive_max_frames)
{
std::cerr << "[face_tracker] alpha=" << alpha_
<< " min_iou=" << min_iou_
<< " max_embed_dist=" << max_embed_dist_
<< " max_missing=" << max_missing_ << "\n";
<< " max_missing=" << max_missing_
<< " cut_revive_sim=" << revive_sim_
<< " cut_inactive_max=" << inactive_max_ << "\n";
}
TrackedSceneFrame operator()(EmbeddedSceneFrame ef) {
if (ef.source.eof) {
tracks_.clear();
inactive_.clear();
TrackedSceneFrame out;
out.source = std::move(ef.source);
return out;
@@ -56,11 +72,26 @@ struct FaceTrackerFunc {
const int n_det = static_cast<int>(ef.embeddings.size());
// Camera-angle change: park active tracks instead of destroying them so
// they can be revived by identity (raw last-frame embedding cosine) once
// the same people reappear from the new angle.
if (ef.source.is_cut && !tracks_.empty()) {
std::cerr << "[face_tracker] cut — clearing " << tracks_.size() << " tracks\n";
std::cerr << "[face_tracker] cut — parking " << tracks_.size()
<< " track(s) into inactive pool\n";
for (auto& [tid, ts] : tracks_) {
ts.frames_missing = 0; // repurpose as time-since-parked counter
inactive_[tid] = std::move(ts);
}
tracks_.clear();
}
// Age the inactive pool every frame and drop tracks parked too long.
for (auto it = inactive_.begin(); it != inactive_.end(); ) {
it->second.frames_missing++;
it = (it->second.frames_missing > inactive_max_)
? inactive_.erase(it) : std::next(it);
}
// Snapshot active track IDs so the map can be modified safely below
std::vector<int> tids;
tids.reserve(tracks_.size());
@@ -111,6 +142,7 @@ struct FaceTrackerFunc {
continue;
}
update_mean(ts.mean_emb, ts.n_frames, ef.embeddings[di]);
ts.last_emb = ef.embeddings[di];
ts.bbox = ef.faces[di].bbox;
ts.n_frames++;
ts.frames_missing = 0;
@@ -119,13 +151,34 @@ struct FaceTrackerFunc {
out.track_ids[di] = tids[ti];
}
// Create new tracks for unmatched detections
// Handle unmatched detections: first try to revive a parked track by
// identity (raw last-frame embedding cosine), else start a fresh track.
for (int di = 0; di < n_det; ++di) {
if (det_matched[di]) continue;
int tid = next_id_++;
int tid = revive_from_inactive(ef.embeddings[di]);
if (tid >= 0) {
// Restore the parked track: keep its identity statistics
// (mean_emb, n_frames), jump the bbox to the new detection.
TrackState ts = std::move(inactive_[tid]);
inactive_.erase(tid);
update_mean(ts.mean_emb, ts.n_frames, ef.embeddings[di]);
ts.last_emb = ef.embeddings[di];
ts.bbox = ef.faces[di].bbox;
ts.n_frames++;
ts.frames_missing = 0;
tracks_[tid] = std::move(ts);
out.track_ids[di] = tid;
std::cerr << "[face_tracker] revived track " << tid
<< " across cut\n";
continue;
}
tid = next_id_++;
TrackState ts;
ts.bbox = ef.faces[di].bbox;
ts.mean_emb = ef.embeddings[di];
ts.last_emb = ef.embeddings[di];
ts.n_frames = 1;
tracks_[tid] = ts;
out.track_ids[di] = tid;
@@ -141,6 +194,21 @@ struct FaceTrackerFunc {
}
private:
// Pick the parked track whose last-frame embedding is most similar to emb,
// returning its id if that raw cosine similarity clears revive_sim_, else -1.
// The caller removes the returned track from the pool, so a later detection in
// the same frame cannot claim it again.
int revive_from_inactive(const Embedding& emb) const {
int best_tid = -1;
float best_sim = revive_sim_; // threshold is the bar to beat (inclusive)
for (const auto& [tid, ts] : inactive_) {
float sim = cosine_similarity(ts.last_emb, emb);
if (sim >= best_sim) { best_sim = sim; best_tid = tid; }
// subsequent ties keep the later id; harmless, all clear the threshold
}
return best_tid;
}
// IoU of two axis-aligned bounding boxes
static float iou(const cv::Rect2f& a, const cv::Rect2f& b) {
float ix = std::max(0.f, std::min(a.x + a.width, b.x + b.width)
@@ -225,9 +293,12 @@ private:
}
std::map<int, TrackState> tracks_;
std::map<int, TrackState> inactive_; // parked across a cut, keyed by track id
int next_id_{0};
float alpha_;
float min_iou_;
float max_embed_dist_;
int max_missing_;
float revive_sim_;
int inactive_max_;
};