Initial commit: scene-actor-extraction pipeline
Source (KPN++ pipeline nodes, ArcFace embedders, SCRFD/YuNet detectors, gallery builder), build scripts, and eval artifacts. - external/KPN as a git submodule (gitea.tourolle.paris/dtourolle/KPN) - ONNX models tracked via Git LFS (models/*.onnx) - generated outputs, TensorRT engines, reference repos, and media ignored
This commit is contained in:
@@ -0,0 +1,154 @@
|
||||
#pragma once
|
||||
#include "types.hpp"
|
||||
#include "config.hpp"
|
||||
#include "gallery/gallery_store.hpp"
|
||||
#include "gallery/gallery_calibration.hpp"
|
||||
|
||||
#include <cmath>
|
||||
#include <limits>
|
||||
#include <iostream>
|
||||
|
||||
// ── IdentityMatcherFunc ───────────────────────────────────────────────────────
|
||||
// KPN node: compares each embedding against every reference embedding in the
|
||||
// actor gallery using cosine similarity.
|
||||
//
|
||||
// Matching strategy — two modes selected at construction time:
|
||||
//
|
||||
// Calibrated (preferred): gallery calibration fits a sigmoid
|
||||
// P(match) = σ(a·similarity + b) from intra/inter-class pairs.
|
||||
// A face is accepted if P(match | best_actor) > prob_threshold.
|
||||
//
|
||||
// Fallback (no calibration): dual-criterion accept —
|
||||
// (a) best cosine distance < match_threshold, OR
|
||||
// (b) ratio test: best_dist/second_best_dist < match_ratio
|
||||
// AND best_dist < match_ratio_ceil.
|
||||
//
|
||||
// In both modes, per-actor best similarity is determined by scanning all
|
||||
// reference embeddings and taking the closest (best-of-N).
|
||||
|
||||
struct IdentityMatcherFunc {
|
||||
static constexpr std::string_view label() { return "identity_matcher"; }
|
||||
|
||||
IdentityMatcherFunc(const ActorGallery& gallery, const Config& cfg)
|
||||
: gallery_(gallery)
|
||||
, prob_threshold_(cfg.prob_threshold)
|
||||
, log_prior_odds_(std::log(cfg.match_prior / (1.f - cfg.match_prior)))
|
||||
, threshold_(cfg.match_threshold)
|
||||
, ratio_(cfg.match_ratio)
|
||||
, ratio_ceil_(cfg.match_ratio_ceil)
|
||||
{
|
||||
for (int ai = 0; ai < static_cast<int>(gallery_.actors.size()); ++ai) {
|
||||
for (const auto& emb : gallery_.actors[ai].embeddings) {
|
||||
flat_emb_.push_back(emb);
|
||||
flat_actor_.push_back(ai);
|
||||
}
|
||||
}
|
||||
|
||||
cal_ = calibrate_gallery(flat_emb_, flat_actor_);
|
||||
|
||||
if (cal_.valid) {
|
||||
std::cerr << "[identity_matcher] calibrated Bayesian matching"
|
||||
<< " prior=" << cfg.match_prior
|
||||
<< " P_threshold=" << prob_threshold_
|
||||
<< " effective_sim_boundary="
|
||||
<< cal_.boundary_at(prob_threshold_, log_prior_odds_) << "\n";
|
||||
} else {
|
||||
std::cerr << "[identity_matcher] threshold matching (calibration skipped)"
|
||||
<< " threshold=" << threshold_
|
||||
<< " ratio=" << ratio_ << " ratio_ceil=" << ratio_ceil_ << "\n";
|
||||
}
|
||||
std::cerr << "[identity_matcher] gallery: "
|
||||
<< gallery_.actors.size() << " actors, "
|
||||
<< flat_emb_.size() << " reference embeddings\n";
|
||||
}
|
||||
|
||||
MatchedSceneFrame operator()(TrackedSceneFrame tf) {
|
||||
if (tf.source.eof) return {std::move(tf.source), {}};
|
||||
|
||||
std::vector<IdentifiedActor> actors;
|
||||
actors.reserve(tf.embeddings.size());
|
||||
|
||||
for (int fi = 0; fi < static_cast<int>(tf.embeddings.size()); ++fi) {
|
||||
// Prefer the track's accumulated mean embedding when the track is
|
||||
// mature (≥ min_frames observations) — more stable than single-frame.
|
||||
const Embedding& query = tf.track_mature[fi]
|
||||
? tf.track_embeddings[fi]
|
||||
: tf.embeddings[fi];
|
||||
|
||||
// Per-actor best cosine similarity (max dot product)
|
||||
std::vector<float> best_sim(gallery_.actors.size(),
|
||||
-std::numeric_limits<float>::max());
|
||||
|
||||
for (int ei = 0; ei < static_cast<int>(flat_emb_.size()); ++ei) {
|
||||
float sim = cosine_similarity(query, flat_emb_[ei]);
|
||||
int ai = flat_actor_[ei];
|
||||
if (sim > best_sim[ai]) best_sim[ai] = sim;
|
||||
}
|
||||
|
||||
// Find best and second-best actor by similarity
|
||||
int best_actor = -1;
|
||||
int second_actor = -1;
|
||||
float best_s = -std::numeric_limits<float>::max();
|
||||
float second_s = -std::numeric_limits<float>::max();
|
||||
for (int ai = 0; ai < static_cast<int>(best_sim.size()); ++ai) {
|
||||
if (best_sim[ai] > best_s) {
|
||||
second_s = best_s;
|
||||
second_actor = best_actor;
|
||||
best_s = best_sim[ai];
|
||||
best_actor = ai;
|
||||
} else if (best_sim[ai] > second_s) {
|
||||
second_s = best_sim[ai];
|
||||
second_actor = ai;
|
||||
}
|
||||
}
|
||||
(void)second_actor;
|
||||
|
||||
bool accept = false;
|
||||
if (best_actor >= 0) {
|
||||
if (cal_.valid) {
|
||||
accept = cal_.probability(best_s, log_prior_odds_) > prob_threshold_;
|
||||
} else {
|
||||
float best_d = 1.f - best_s;
|
||||
float second_d = (second_s > -std::numeric_limits<float>::max())
|
||||
? 1.f - second_s
|
||||
: std::numeric_limits<float>::max();
|
||||
bool absolute = best_d < threshold_;
|
||||
bool ratio = (best_d < ratio_ceil_) &&
|
||||
(second_d == std::numeric_limits<float>::max() ||
|
||||
best_d / second_d < ratio_);
|
||||
accept = absolute || ratio;
|
||||
}
|
||||
}
|
||||
|
||||
IdentifiedActor ia;
|
||||
ia.bbox = tf.faces[fi].bbox;
|
||||
ia.crop = tf.crops[fi];
|
||||
ia.track_id = tf.track_ids[fi];
|
||||
|
||||
if (accept) {
|
||||
ia.actor_idx = best_actor;
|
||||
ia.name = gallery_.actors[best_actor].name;
|
||||
ia.imdb_id = gallery_.actors[best_actor].imdb_id;
|
||||
ia.similarity = cal_.valid
|
||||
? cal_.probability(best_s, log_prior_odds_)
|
||||
: best_s;
|
||||
}
|
||||
// actor_idx == -1, name == "" → unknown face
|
||||
|
||||
actors.push_back(std::move(ia));
|
||||
}
|
||||
|
||||
return {std::move(tf.source), std::move(actors)};
|
||||
}
|
||||
|
||||
private:
|
||||
ActorGallery gallery_;
|
||||
GalleryCalibration cal_;
|
||||
float prob_threshold_;
|
||||
float log_prior_odds_;
|
||||
float threshold_;
|
||||
float ratio_;
|
||||
float ratio_ceil_;
|
||||
std::vector<Embedding> flat_emb_;
|
||||
std::vector<int> flat_actor_;
|
||||
};
|
||||
Reference in New Issue
Block a user