feat(engine): add Python replay bindings, gallery pose-expansion, scene detection, embedding dumps
New C++ sources: - kpn_bindings.cpp (sae_kpn): assembles the real face_tracker/identity_matcher/ scene_tracker nodes inside a Python-driven KPN network via nanobind, for offline threshold-sweep replay against dumped embeddings (scripts/optimizer/). - track_gallery.hpp: per-film gallery expansion — promotes a confidently- identified track's novel-pose reference views into an in-memory annex so later frames/tracks of that actor at similar poses are recognised, without touching the baked gallery. - dump_embeddings.cpp: standalone exe that runs detect→embed only (no gallery, no matching) and dumps per-frame face embeddings + metadata to HDF5, so a parameter sweep can replay the expensive half once and vary tracking/matching config freely downstream. - scene_detector.hpp / scene_detector_node.hpp: TransNetV2-based shot-boundary detection, opt-in alongside the always-on histogram cut detector. - camera_position_change_detector_node.hpp, embedding_dump_node.hpp: supporting nodes for the above.
This commit is contained in:
@@ -0,0 +1,45 @@
|
||||
#pragma once
|
||||
#include <memory>
|
||||
#include <vector>
|
||||
|
||||
#include <opencv2/core.hpp>
|
||||
|
||||
// ── ISceneDetector ────────────────────────────────────────────────────────────
|
||||
// Backend-agnostic shot-boundary (scene-cut) detector interface, mirroring
|
||||
// IFaceDetector. The concrete implementation wraps TransNetV2 and is selected at
|
||||
// compile time by CMake (SAE_INFERENCE_BACKEND): exactly one of
|
||||
// backends/ort_backend.cpp or backends/trt_backend.cpp provides
|
||||
// make_scene_detector().
|
||||
//
|
||||
// TransNetV2 consumes a window of exactly kWindow consecutive frames, each
|
||||
// downscaled to kFrameW×kFrameH RGB, and predicts a per-frame boundary
|
||||
// probability. It is a DENSE model: the frames it sees must be consecutive at
|
||||
// (near) native frame rate — 1-FPS sampled frames give meaningless output.
|
||||
// Feeding is handled by the pipeline (dense decode + decimator); this interface
|
||||
// only exposes the fixed-size window inference.
|
||||
|
||||
struct Config;
|
||||
|
||||
struct ISceneDetector {
|
||||
virtual ~ISceneDetector() = default;
|
||||
|
||||
// TransNetV2 fixed input contract (from the elya5/transnetv2 ONNX export):
|
||||
// input "input" : float32 [1, 100, 27, 48, 3] (RGB, channels-last, 0-255)
|
||||
// output "534" : float32 [1, 100, 1] single-frame boundary logits
|
||||
// output "535" : float32 [1, 100, 1] "many-hot" auxiliary head (unused)
|
||||
static constexpr int kWindow = 100;
|
||||
static constexpr int kFrameH = 27;
|
||||
static constexpr int kFrameW = 48;
|
||||
|
||||
// Run one window of exactly kWindow frames. Each frame must already be
|
||||
// kFrameW×kFrameH, BGR, CV_8UC3 (the backend handles BGR→RGB). Returns
|
||||
// kWindow boundary probabilities in [0,1] (sigmoid of the primary head),
|
||||
// one per input frame, in order.
|
||||
virtual std::vector<float> detect_window(
|
||||
const std::vector<cv::Mat>& window) = 0;
|
||||
};
|
||||
|
||||
// Construct the scene detector for the compiled-in backend. Reads
|
||||
// cfg.scene_model / cfg.scene_engine and cfg.trt. Only called when scene
|
||||
// detection is enabled (cfg.scene_detect).
|
||||
std::unique_ptr<ISceneDetector> make_scene_detector(const Config& cfg);
|
||||
Reference in New Issue
Block a user