dtourolle/scene-actor-extraction
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(engine): add Python replay bindings, gallery pose-expansion, scene detection, embedding dumps

Browse Source 
New C++ sources:
- kpn_bindings.cpp (sae_kpn): assembles the real face_tracker/identity_matcher/
  scene_tracker nodes inside a Python-driven KPN network via nanobind, for
  offline threshold-sweep replay against dumped embeddings (scripts/optimizer/).
- track_gallery.hpp: per-film gallery expansion — promotes a confidently-
  identified track's novel-pose reference views into an in-memory annex so
  later frames/tracks of that actor at similar poses are recognised, without
  touching the baked gallery.
- dump_embeddings.cpp: standalone exe that runs detect→embed only (no gallery,
  no matching) and dumps per-frame face embeddings + metadata to HDF5, so a
  parameter sweep can replay the expensive half once and vary tracking/matching
  config freely downstream.
- scene_detector.hpp / scene_detector_node.hpp: TransNetV2-based shot-boundary
  detection, opt-in alongside the always-on histogram cut detector.
- camera_position_change_detector_node.hpp, embedding_dump_node.hpp: supporting
  nodes for the above.
This commit is contained in:
dtourolle
2026-07-19 19:05:05 +02:00
parent 41a277bc19
commit 26139ffe8a
7 changed files with 1013 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/nodes/camera_position_change_detector_node.hpp
+73
View File
@@ -0,0 +1,73 @@
#pragma once
#include "types.hpp"
#include "config.hpp"
#include <opencv2/imgproc.hpp>
#include <algorithm>
#include <iostream>
#include <string_view>
// ── CameraPositionChangeDetectorFunc ──────────────────────────────────────────
// KPN node: flags intra-scene camera-angle changes (hard cuts) by comparing each
// frame's grayscale histogram to the previous frame's. When the normalised
// histogram correlation drops below cut_threshold, the frame is marked with
// Frame::is_cut = true.
//
// This is the pipeline's cut *trigger*: it owns cut detection so a single node
// decides when the camera position has changed, and every downstream stage reads
// the decision off Frame::is_cut (which rides with the frame's timestamp_sec /
// frame_idx). The face_tracker consumes it to re-associate tracks across the cut
// rather than blindly resetting; the track_gallery consumes it to bound
// promotion to a single physical viewpoint.
//
// This is the always-on histogram cut — separate from the opt-in TransNetV2
// scene detector, which localises true shot boundaries as Frame::is_scene_boundary.
//
// The node is a pure pass-through: it forwards the Frame unchanged except for
// is_cut, so it slots between frame_source and face_detector without altering the
// downstream contract. eof frames are forwarded immediately without processing.
struct CameraPositionChangeDetectorFunc {
static constexpr std::string_view label() { return "camera_position_change_detector"; }
explicit CameraPositionChangeDetectorFunc(const Config& cfg)
: cut_threshold_(cfg.cut_threshold)
{
std::cerr << "[camera_position_change_detector] cut_threshold="
<< cut_threshold_ << "\n";
}
Frame operator()(Frame f) {
if (f.eof) return f;
cv::Mat gray;
cv::cvtColor(f.image, gray, cv::COLOR_BGR2GRAY);
cv::Mat hist;
const int bins = 64;
const float range[] = {0.f, 256.f};
const float* ranges = range;
cv::calcHist(&gray, 1, nullptr, cv::Mat(), hist, 1, &bins, &ranges);
cv::normalize(hist, hist, 1.0, 0.0, cv::NORM_L1);
if (prev_hist_valid_) {
double corr = cv::compareHist(prev_hist_, hist, cv::HISTCMP_CORREL);
// Cut score in [0,1]: 0 = identical to previous frame, ~1 = fully
// different. Rides on the Frame for the preview HUD / debugging.
f.cut_score = static_cast<float>(std::clamp(1.0 - corr, 0.0, 1.0));
f.is_cut = (corr < cut_threshold_);
if (f.is_cut)
std::cerr << "[camera_position_change_detector] cut at t="
<< f.timestamp_sec << "s hist_corr=" << corr << "\n";
}
prev_hist_ = hist;
prev_hist_valid_ = true;
return f;
}
private:
float cut_threshold_;
cv::Mat prev_hist_;
bool prev_hist_valid_{false};
};
src/nodes/embedding_dump_node.hpp
+124
View File
@@ -0,0 +1,124 @@
#pragma once
#include "types.hpp"
#include "config.hpp"
#include <H5Cpp.h>
#include <cstdint>
#include <iostream>
#include <string>
#include <vector>
// ── EmbeddingDumpFunc ─────────────────────────────────────────────────────────
// KPN sink that taps the EmbeddedSceneFrame channel and writes the per-frame face
// metadata + embeddings to one HDF5 file (schema: scripts/optimizer/SCHEMA.md).
// The dump is the expensive, parameter-independent half of the pipeline
// (decode→detect→align→embed); replaying it lets a threshold sweep re-run the cheap
// downstream nodes thousands of times with no GPU. See sae_kpn / scripts/optimizer.
//
// Accumulates in flat/ragged arrays and writes once on EOF.
struct EmbeddingDumpFunc {
static constexpr std::string_view label() { return "embedding_dump"; }
EmbeddingDumpFunc(const Config& cfg, std::atomic<bool>& done)
: path_(cfg.dump_embeddings_path), movie_(cfg.movie_path),
sample_fps_(cfg.sample_fps), done_(done)
{
std::cerr << "[embedding_dump] writing " << path_ << "\n";
}
void operator()(EmbeddedSceneFrame ef) {
if (ef.source.eof) { flush(); return; }
const int32_t n = static_cast<int32_t>(ef.faces.size());
ts_.push_back(ef.source.timestamp_sec);
fidx_.push_back(ef.source.frame_idx);
is_cut_.push_back(ef.source.is_cut ? 1 : 0);
is_bnd_.push_back(ef.source.is_scene_boundary ? 1 : 0);
face_off_.push_back(static_cast<int64_t>(conf_.size()));
face_cnt_.push_back(n);
for (int i = 0; i < n; ++i) {
const auto& f = ef.faces[i];
bbox_.insert(bbox_.end(), {f.bbox.x, f.bbox.y, f.bbox.width, f.bbox.height});
for (int k = 0; k < 5; ++k) {
lmk_.push_back(f.landmarks[k].x);
lmk_.push_back(f.landmarks[k].y);
}
conf_.push_back(f.confidence);
const auto& e = ef.embeddings[i];
emb_.insert(emb_.end(), e.begin(), e.end());
}
}
void flush() {
if (written_.exchange(true)) return;
try {
write_hdf5();
} catch (const H5::Exception& e) {
std::cerr << "[embedding_dump] HDF5 error: " << e.getDetailMsg() << "\n";
}
done_.store(true, std::memory_order_release);
}
private:
static constexpr int kSchemaVersion = 1;
static constexpr int kEmbedDim = 512;
template<typename T>
void write_vec(H5::Group& g, const char* name, const std::vector<T>& v,
const H5::PredType& dtype, hsize_t cols = 0) {
hsize_t rows = cols ? v.size() / cols : v.size();
std::vector<hsize_t> dims = cols ? std::vector<hsize_t>{rows, cols}
: std::vector<hsize_t>{rows};
H5::DataSpace space(static_cast<int>(dims.size()), dims.data());
auto ds = g.createDataSet(name, dtype, space);
if (!v.empty()) ds.write(v.data(), dtype);
}
void write_hdf5() {
H5::H5File file(path_, H5F_ACC_TRUNC);
// root attrs
auto scalar = H5::DataSpace(H5S_SCALAR);
auto ver = file.createAttribute("schema_version", H5::PredType::NATIVE_INT, scalar);
int sv = kSchemaVersion; ver.write(H5::PredType::NATIVE_INT, &sv);
auto ed = file.createAttribute("embed_dim", H5::PredType::NATIVE_INT, scalar);
int dim = kEmbedDim; ed.write(H5::PredType::NATIVE_INT, &dim);
auto fps = file.createAttribute("sample_fps", H5::PredType::NATIVE_FLOAT, scalar);
fps.write(H5::PredType::NATIVE_FLOAT, &sample_fps_);
H5::StrType str(H5::PredType::C_S1, H5T_VARIABLE);
auto mv = file.createAttribute("movie", str, scalar);
mv.write(str, movie_);
H5::Group frames = file.createGroup("frames");
write_vec(frames, "timestamp_sec", ts_, H5::PredType::NATIVE_DOUBLE);
write_vec(frames, "frame_idx", fidx_, H5::PredType::NATIVE_INT64);
write_vec(frames, "is_cut", is_cut_, H5::PredType::NATIVE_UINT8);
write_vec(frames, "is_scene_boundary", is_bnd_, H5::PredType::NATIVE_UINT8);
write_vec(frames, "face_offset", face_off_, H5::PredType::NATIVE_INT64);
write_vec(frames, "face_count", face_cnt_, H5::PredType::NATIVE_INT32);
H5::Group faces = file.createGroup("faces");
write_vec(faces, "embedding", emb_, H5::PredType::NATIVE_FLOAT, kEmbedDim);
write_vec(faces, "bbox", bbox_, H5::PredType::NATIVE_FLOAT, 4);
write_vec(faces, "landmarks", lmk_, H5::PredType::NATIVE_FLOAT, 10);
write_vec(faces, "confidence", conf_, H5::PredType::NATIVE_FLOAT);
std::cerr << "[embedding_dump] wrote " << ts_.size() << " frames, "
<< conf_.size() << " faces → " << path_ << "\n";
}
std::string path_, movie_;
float sample_fps_;
std::atomic<bool>& done_;
std::atomic<bool> written_{false};
std::vector<double> ts_;
std::vector<int64_t> fidx_;
std::vector<uint8_t> is_cut_, is_bnd_;
std::vector<int64_t> face_off_;
std::vector<int32_t> face_cnt_;
std::vector<float> emb_, bbox_, lmk_, conf_;
};
src/nodes/scene_detector_node.hpp
+179
View File
@@ -0,0 +1,179 @@
#pragma once
#include "types.hpp"
#include "config.hpp"
#include "inference/scene_detector.hpp"
#include <nlohmann/json.hpp>
#include <algorithm>
#include <atomic>
#include <deque>
#include <fstream>
#include <iostream>
#include <string>
#include <vector>
// ── SceneDetectorFunc ─────────────────────────────────────────────────────────
// KPN sink node: TransNetV2 shot-boundary detection on the dense frame stream.
//
// Buffers incoming (dense, native-rate) Frames into a rolling window of
// ISceneDetector::kWindow (=100) frames. Every `stride` frames it runs one
// inference and reads back per-frame boundary probabilities, but only trusts the
// central region of each window — TransNetV2 (like most sliding-window boundary
// models) is unreliable near the window edges where it lacks temporal context.
// Overlapping windows by (kWindow - stride) frames means every frame is scored
// from at least one window's trusted centre.
//
// Boundaries (prob > scene_threshold, local maxima) are collected with their
// timestamps and written to scenes.json alongside the main annotations output on
// EOF. This branch is terminal: it produces no pipeline messages, only a file.
struct SceneDetectorFunc {
static constexpr std::string_view label() { return "scene_detector"; }
SceneDetectorFunc(const Config& cfg, std::atomic<bool>& done)
: detector_(make_scene_detector(cfg))
, threshold_(cfg.scene_threshold)
, stride_(std::clamp(cfg.scene_stride, 1, ISceneDetector::kWindow))
, output_path_(scenes_path(cfg.output_path))
, movie_path_(cfg.movie_path)
, done_(done)
{
// Trusted centre half of each window. Frames outside [guard, kWindow-guard)
// are re-scored by an adjacent window, so we ignore them here to avoid
// edge artefacts and double-counting.
guard_ = (ISceneDetector::kWindow - stride_) / 2;
std::cerr << "[scene_detector] threshold=" << threshold_
<< " stride=" << stride_
<< " guard=" << guard_
<< " output=" << output_path_ << "\n";
}
void operator()(Frame f) {
if (f.eof) {
flush_remaining();
write_output();
done_.store(true, std::memory_order_release);
return;
}
images_.push_back(f.image);
times_.push_back(f.timestamp_sec);
// Once we have a full window, score it and slide forward by `stride`.
while (static_cast<int>(images_.size()) >= ISceneDetector::kWindow) {
score_window();
for (int i = 0; i < stride_; ++i) {
images_.pop_front();
times_.pop_front();
}
window_base_ += stride_;
}
}
private:
// Run TransNetV2 on the leading kWindow frames of the buffer and record any
// boundaries found within the trusted centre region.
void score_window() {
std::vector<cv::Mat> win(images_.begin(),
images_.begin() + ISceneDetector::kWindow);
std::vector<float> probs = detector_->detect_window(win);
// On the very first window there is no preceding window, so trust from 0;
// otherwise skip the leading guard already covered by the previous window.
const int lo = (window_base_ == 0) ? 0 : guard_;
const int hi = ISceneDetector::kWindow - guard_;
for (int i = lo; i < hi; ++i) {
if (probs[i] <= threshold_) continue;
// Local maximum → the boundary frame (avoid a run of high scores
// registering as several adjacent cuts).
const bool peak =
(i == 0 || probs[i] >= probs[i-1]) &&
(i == kLast_() || probs[i] >= probs[i+1]);
if (peak)
boundaries_.push_back({times_[i], probs[i]});
}
}
// At EOF the tail (< kWindow frames) never formed a full window. Pad it out
// to kWindow by repeating the last frame so the final real frames still get
// scored, then take only the region past what earlier windows covered.
void flush_remaining() {
const int n = static_cast<int>(images_.size());
if (n == 0) return;
std::vector<cv::Mat> win(images_.begin(), images_.end());
cv::Mat last = win.back();
while (static_cast<int>(win.size()) < ISceneDetector::kWindow)
win.push_back(last);
std::vector<float> probs = detector_->detect_window(win);
const int lo = (window_base_ == 0) ? 0 : guard_;
for (int i = lo; i < n; ++i) { // only real (non-padded) frames
if (probs[i] <= threshold_) continue;
const bool peak =
(i == 0 || probs[i] >= probs[i-1]) &&
(i == n - 1 || probs[i] >= probs[i+1]);
if (peak)
boundaries_.push_back({times_[i], probs[i]});
}
}
void write_output() {
if (written_) return;
written_ = true;
// Merge boundaries closer than one frame apart (dedup across window seams).
std::sort(boundaries_.begin(), boundaries_.end(),
[](const Boundary& a, const Boundary& b) {
return a.t < b.t;
});
nlohmann::json root;
root["schema_version"] = 1;
root["movie"] = movie_path_;
root["model"] = "transnetv2";
root["threshold"] = threshold_;
nlohmann::json cuts = nlohmann::json::array();
double last_t = -1e9;
for (const auto& b : boundaries_) {
if (b.t - last_t < 0.04) continue; // ~1 frame @25fps dedup
cuts.push_back({{"t", b.t}, {"probability", b.prob}});
last_t = b.t;
}
root["cuts"] = std::move(cuts);
std::ofstream f(output_path_);
if (!f.is_open()) {
std::cerr << "\n[scene_detector] ERROR: cannot write "
<< output_path_ << "\n";
return;
}
f << root.dump(2) << "\n";
std::cerr << "\n[scene_detector] wrote " << root["cuts"].size()
<< " boundaries → " << output_path_ << "\n";
}
static int kLast_() { return ISceneDetector::kWindow - 1; }
// annotations.json → annotations.scenes.json (or scenes.json for bare names)
static std::string scenes_path(const std::string& out) {
auto dot = out.find_last_of('.');
if (dot == std::string::npos) return out + ".scenes.json";
return out.substr(0, dot) + ".scenes.json";
}
struct Boundary { double t; float prob; };
std::unique_ptr<ISceneDetector> detector_;
float threshold_;
int stride_;
int guard_{0};
std::string output_path_;
std::string movie_path_;
std::atomic<bool>& done_;
std::deque<cv::Mat> images_;
std::deque<double> times_;
int64_t window_base_{0}; // frame index of images_.front()
std::vector<Boundary> boundaries_;
bool written_{false};
};