dtourolle/scene-actor-extraction
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(engine): add Python replay bindings, gallery pose-expansion, scene detection, embedding dumps

Browse Source 
New C++ sources:
- kpn_bindings.cpp (sae_kpn): assembles the real face_tracker/identity_matcher/
  scene_tracker nodes inside a Python-driven KPN network via nanobind, for
  offline threshold-sweep replay against dumped embeddings (scripts/optimizer/).
- track_gallery.hpp: per-film gallery expansion — promotes a confidently-
  identified track's novel-pose reference views into an in-memory annex so
  later frames/tracks of that actor at similar poses are recognised, without
  touching the baked gallery.
- dump_embeddings.cpp: standalone exe that runs detect→embed only (no gallery,
  no matching) and dumps per-frame face embeddings + metadata to HDF5, so a
  parameter sweep can replay the expensive half once and vary tracking/matching
  config freely downstream.
- scene_detector.hpp / scene_detector_node.hpp: TransNetV2-based shot-boundary
  detection, opt-in alongside the always-on histogram cut detector.
- camera_position_change_detector_node.hpp, embedding_dump_node.hpp: supporting
  nodes for the above.
This commit is contained in:
dtourolle
2026-07-19 19:05:05 +02:00
parent 41a277bc19
commit 26139ffe8a
7 changed files with 1013 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/gallery/track_gallery.hpp
+243
View File
@@ -0,0 +1,243 @@
#pragma once
#include "types.hpp"
#include "config.hpp"
#include <cmath>
#include <cstdio>
#include <iostream>
#include <limits>
#include <map>
#include <string>
#include <vector>
#ifdef SAE_DEBUG
#include <opencv2/imgcodecs.hpp>
#endif
#include <opencv2/core.hpp>
#include <filesystem>
// ── TrackGallery ──────────────────────────────────────────────────────────────
// Per-film gallery expansion driven by track continuity.
//
// A single uncut face track is, by construction, one physical person: the
// face_tracker links detections frame-to-frame and clears all tracks on a scene
// cut, so a track ID never spans a cut. That continuity is a same-identity label
// the baked gallery does not have. TrackGallery exploits it in two stages:
//
// 1. Diversity buffer (per track). Every frame's embedding is offered to a
// fixed-capacity buffer for that track. When full, the member whose best
// similarity to the *owning actor's* gallery references is HIGHEST is
// dropped — i.e. the pose the gallery already recognises well is the least
// informative, so the buffer is continuously biased toward the gallery-far
// (novel-pose) embeddings on which recognition currently fails.
//
// 2. Promotion (on confirmation). A track is "owned" by actor A once ≥N frames
// have been accepted (by the matcher's calibrated posterior) as A. On
// confirmation the retained buffer — the hard, gallery-far poses — is
// promoted into A's per-film annex, after two safety gates:
// • novelty: only embeddings whose best sim to A's refs is below
// expand_novelty_sim are added (skip poses already covered);
// • spread: if the retained buffer's internal spread (1 − min pairwise
// cosine sim) exceeds expand_track_spread_max the whole track is
// rejected — such spread signals a track-ID collision merging two
// people, whose embeddings must never enter A's annex.
//
// The annex is CPU-side and in-memory: it is small (tens of embeddings) so the
// matcher scans it with a scalar loop, and it is discarded when the process
// exits. Promoted embeddings only help SUBSEQUENT frames and later tracks of A —
// the pipeline stays streaming, no emitted output is buffered or relabelled.
struct TrackGallery {
// One promoted reference view held in the per-actor annex.
struct AnnexEntry {
Embedding emb;
int actor_idx{-1};
};
explicit TrackGallery(const Config& cfg)
: enabled_(cfg.expand_gallery)
, buffer_size_(std::max(1, cfg.expand_buffer_size))
, novelty_sim_(cfg.expand_novelty_sim)
, spread_max_(cfg.expand_track_spread_max)
, min_anchor_frames_(std::max(1, cfg.expand_min_anchor_frames))
, debug_dir_(cfg.expand_debug_dir)
{
if (!enabled_) return;
std::cerr << "[track_gallery] per-film expansion ON"
<< " buffer=" << buffer_size_
<< " novelty_sim<" << novelty_sim_
<< " spread_max=" << spread_max_
<< " min_anchor_frames=" << min_anchor_frames_;
if (!debug_dir_.empty()) {
std::filesystem::create_directories(debug_dir_);
std::cerr << " debug_dir=" << debug_dir_;
}
std::cerr << "\n";
}
bool enabled() const { return enabled_; }
// Current annex contents (empty when disabled). The matcher scans these
// alongside the baked gallery so a promoted view can win best-of-N for its
// actor. Returned by const-ref; only grows, never reordered.
const std::vector<AnnexEntry>& annex() const { return annex_; }
// Offer one observed face to its track's diversity buffer.
// track_id : face_tracker track (−1 = untracked, ignored)
// emb : this frame's raw embedding
// best_actor : actor with the highest gallery similarity for this face
// best_gal_sim : that similarity (best sim to best_actor's baked+annex refs)
// accepted : true if the matcher accepted this face as best_actor
// crop : aligned crop, retained only when debug dumping is on
void observe(int track_id, const Embedding& emb,
int best_actor, float best_gal_sim, bool accepted,
const cv::Mat& crop)
{
if (!enabled_ || track_id < 0) return;
TrackState& ts = tracks_[track_id];
// Vote toward ownership: only accepted frames name an actor, and a track
// that flip-flops between actors is ambiguous, so we tally per actor and
// pick the plurality winner at confirmation time.
if (accepted && best_actor >= 0) {
ts.actor_votes[best_actor]++;
ts.accepted_frames++;
}
insert_into_buffer(ts, emb, best_gal_sim, crop);
// Confirm and promote as soon as the anchor threshold is met, once.
if (!ts.promoted && ts.accepted_frames >= min_anchor_frames_)
promote(track_id, ts);
}
// Drop a track's buffer when the face_tracker expires it or on a scene cut,
// so stale/cross-cut embeddings can never be promoted later. Called by the
// matcher when it observes a cut or track disappearance.
void forget(int track_id) { tracks_.erase(track_id); }
// Drop every track buffer (scene cut / EOF). Mirrors face_tracker's clear.
void clear_tracks() { tracks_.clear(); }
private:
struct BufEntry {
Embedding emb;
float gal_sim{0.f}; // best sim to owning actor's refs when observed
cv::Mat crop; // populated only when debug_dir_ set
};
struct TrackState {
std::vector<BufEntry> buf;
std::map<int, int> actor_votes; // actor_idx → accepted-frame count
int accepted_frames{0};
bool promoted{false};
};
void insert_into_buffer(TrackState& ts, const Embedding& emb,
float gal_sim, const cv::Mat& crop)
{
BufEntry e;
e.emb = emb;
e.gal_sim = gal_sim;
if (!debug_dir_.empty() && !crop.empty()) e.crop = crop.clone();
if (static_cast<int>(ts.buf.size()) < buffer_size_) {
ts.buf.push_back(std::move(e));
return;
}
// Buffer full: evict the member the gallery recognises best (highest
// gal_sim) — least informative — but only if the newcomer is at least as
// novel. Keeping the most gallery-far views is the whole point.
int worst_i = -1;
float worst_sim = e.gal_sim; // newcomer's sim is the bar to beat
for (int i = 0; i < static_cast<int>(ts.buf.size()); ++i) {
if (ts.buf[i].gal_sim > worst_sim) {
worst_sim = ts.buf[i].gal_sim;
worst_i = i;
}
}
// worst_i == −1 → every buffered view is already more novel than the
// newcomer; drop the newcomer instead of a better sample.
if (worst_i >= 0) ts.buf[worst_i] = std::move(e);
}
void promote(int track_id, TrackState& ts) {
ts.promoted = true; // idempotent: never promote a track twice
int actor = plurality_actor(ts);
if (actor < 0) return;
// ── Safety gate: internal spread ─────────────────────────────────────
// A legitimate single-person track varies in pose but stays reasonably
// self-similar. Large spread signals two people merged under one track
// ID — reject the whole track rather than poison the actor's annex.
float spread = buffer_spread(ts.buf);
if (spread > spread_max_) {
std::cerr << "[track_gallery] track " << track_id
<< " → actor " << actor
<< " REJECTED (spread " << spread
<< " > " << spread_max_ << ", likely ID collision)\n";
return;
}
int added = 0;
for (const auto& be : ts.buf) {
// ── Safety gate: novelty ─────────────────────────────────────────
// Skip poses the gallery already covers; only gallery-far views are
// worth the annex slot (and the extra per-frame scan cost).
if (be.gal_sim >= novelty_sim_) continue;
annex_.push_back({be.emb, actor});
if (!debug_dir_.empty() && !be.crop.empty())
dump_mugshot(track_id, actor, added, be);
++added;
}
std::cerr << "[track_gallery] track " << track_id
<< " confirmed actor " << actor
<< " (" << ts.accepted_frames << " accepted frames, spread "
<< spread << ") — promoted " << added << "/"
<< ts.buf.size() << " views; annex now "
<< annex_.size() << "\n";
}
static int plurality_actor(const TrackState& ts) {
int best = -1, best_votes = 0;
for (const auto& [ai, v] : ts.actor_votes) {
if (v > best_votes) { best_votes = v; best = ai; }
}
return best;
}
// Spread = 1 − min pairwise cosine similarity over the buffer (0 when <2).
static float buffer_spread(const std::vector<BufEntry>& buf) {
float min_sim = std::numeric_limits<float>::max();
for (size_t i = 0; i < buf.size(); ++i)
for (size_t j = i + 1; j < buf.size(); ++j)
min_sim = std::min(min_sim, cosine_similarity(buf[i].emb, buf[j].emb));
if (min_sim == std::numeric_limits<float>::max()) return 0.f;
return 1.f - min_sim;
}
void dump_mugshot(int track_id, int actor, int idx, const BufEntry& be) {
#ifdef SAE_DEBUG
char name[64];
std::snprintf(name, sizeof(name), "trk%d_actor%d_%d_sim%.3f.jpg",
track_id, actor, idx, be.gal_sim);
cv::imwrite((std::filesystem::path(debug_dir_) / name).string(), be.crop);
#else
(void)track_id; (void)actor; (void)idx; (void)be;
#endif
}
bool enabled_;
int buffer_size_;
float novelty_sim_;
float spread_max_;
int min_anchor_frames_;
std::string debug_dir_;
std::map<int, TrackState> tracks_;
std::vector<AnnexEntry> annex_;
};