feat(engine): add Python replay bindings, gallery pose-expansion, scene detection, embedding dumps
New C++ sources: - kpn_bindings.cpp (sae_kpn): assembles the real face_tracker/identity_matcher/ scene_tracker nodes inside a Python-driven KPN network via nanobind, for offline threshold-sweep replay against dumped embeddings (scripts/optimizer/). - track_gallery.hpp: per-film gallery expansion — promotes a confidently- identified track's novel-pose reference views into an in-memory annex so later frames/tracks of that actor at similar poses are recognised, without touching the baked gallery. - dump_embeddings.cpp: standalone exe that runs detect→embed only (no gallery, no matching) and dumps per-frame face embeddings + metadata to HDF5, so a parameter sweep can replay the expensive half once and vary tracking/matching config freely downstream. - scene_detector.hpp / scene_detector_node.hpp: TransNetV2-based shot-boundary detection, opt-in alongside the always-on histogram cut detector. - camera_position_change_detector_node.hpp, embedding_dump_node.hpp: supporting nodes for the above.
This commit is contained in:
@@ -0,0 +1,73 @@
|
||||
#pragma once
|
||||
#include "types.hpp"
|
||||
#include "config.hpp"
|
||||
|
||||
#include <opencv2/imgproc.hpp>
|
||||
#include <algorithm>
|
||||
#include <iostream>
|
||||
#include <string_view>
|
||||
|
||||
// ── CameraPositionChangeDetectorFunc ──────────────────────────────────────────
|
||||
// KPN node: flags intra-scene camera-angle changes (hard cuts) by comparing each
|
||||
// frame's grayscale histogram to the previous frame's. When the normalised
|
||||
// histogram correlation drops below cut_threshold, the frame is marked with
|
||||
// Frame::is_cut = true.
|
||||
//
|
||||
// This is the pipeline's cut *trigger*: it owns cut detection so a single node
|
||||
// decides when the camera position has changed, and every downstream stage reads
|
||||
// the decision off Frame::is_cut (which rides with the frame's timestamp_sec /
|
||||
// frame_idx). The face_tracker consumes it to re-associate tracks across the cut
|
||||
// rather than blindly resetting; the track_gallery consumes it to bound
|
||||
// promotion to a single physical viewpoint.
|
||||
//
|
||||
// This is the always-on histogram cut — separate from the opt-in TransNetV2
|
||||
// scene detector, which localises true shot boundaries as Frame::is_scene_boundary.
|
||||
//
|
||||
// The node is a pure pass-through: it forwards the Frame unchanged except for
|
||||
// is_cut, so it slots between frame_source and face_detector without altering the
|
||||
// downstream contract. eof frames are forwarded immediately without processing.
|
||||
|
||||
struct CameraPositionChangeDetectorFunc {
|
||||
static constexpr std::string_view label() { return "camera_position_change_detector"; }
|
||||
|
||||
explicit CameraPositionChangeDetectorFunc(const Config& cfg)
|
||||
: cut_threshold_(cfg.cut_threshold)
|
||||
{
|
||||
std::cerr << "[camera_position_change_detector] cut_threshold="
|
||||
<< cut_threshold_ << "\n";
|
||||
}
|
||||
|
||||
Frame operator()(Frame f) {
|
||||
if (f.eof) return f;
|
||||
|
||||
cv::Mat gray;
|
||||
cv::cvtColor(f.image, gray, cv::COLOR_BGR2GRAY);
|
||||
|
||||
cv::Mat hist;
|
||||
const int bins = 64;
|
||||
const float range[] = {0.f, 256.f};
|
||||
const float* ranges = range;
|
||||
cv::calcHist(&gray, 1, nullptr, cv::Mat(), hist, 1, &bins, &ranges);
|
||||
cv::normalize(hist, hist, 1.0, 0.0, cv::NORM_L1);
|
||||
|
||||
if (prev_hist_valid_) {
|
||||
double corr = cv::compareHist(prev_hist_, hist, cv::HISTCMP_CORREL);
|
||||
// Cut score in [0,1]: 0 = identical to previous frame, ~1 = fully
|
||||
// different. Rides on the Frame for the preview HUD / debugging.
|
||||
f.cut_score = static_cast<float>(std::clamp(1.0 - corr, 0.0, 1.0));
|
||||
f.is_cut = (corr < cut_threshold_);
|
||||
if (f.is_cut)
|
||||
std::cerr << "[camera_position_change_detector] cut at t="
|
||||
<< f.timestamp_sec << "s hist_corr=" << corr << "\n";
|
||||
}
|
||||
prev_hist_ = hist;
|
||||
prev_hist_valid_ = true;
|
||||
|
||||
return f;
|
||||
}
|
||||
|
||||
private:
|
||||
float cut_threshold_;
|
||||
cv::Mat prev_hist_;
|
||||
bool prev_hist_valid_{false};
|
||||
};
|
||||
Reference in New Issue
Block a user