feat(engine): add Python replay bindings, gallery pose-expansion, scene detection, embedding dumps
New C++ sources: - kpn_bindings.cpp (sae_kpn): assembles the real face_tracker/identity_matcher/ scene_tracker nodes inside a Python-driven KPN network via nanobind, for offline threshold-sweep replay against dumped embeddings (scripts/optimizer/). - track_gallery.hpp: per-film gallery expansion — promotes a confidently- identified track's novel-pose reference views into an in-memory annex so later frames/tracks of that actor at similar poses are recognised, without touching the baked gallery. - dump_embeddings.cpp: standalone exe that runs detect→embed only (no gallery, no matching) and dumps per-frame face embeddings + metadata to HDF5, so a parameter sweep can replay the expensive half once and vary tracking/matching config freely downstream. - scene_detector.hpp / scene_detector_node.hpp: TransNetV2-based shot-boundary detection, opt-in alongside the always-on histogram cut detector. - camera_position_change_detector_node.hpp, embedding_dump_node.hpp: supporting nodes for the above.
This commit is contained in:
@@ -0,0 +1,92 @@
|
||||
// dump_embeddings — standalone embedding dumper (NO gallery required).
|
||||
//
|
||||
// Runs only the expensive, gallery-independent front half of the pipeline
|
||||
// (decode → camera-pos → detect → align → embed) and writes per-frame face
|
||||
// embeddings + metadata to HDF5 (scripts/optimizer/SCHEMA.md). Unlike
|
||||
// `scene_analyze --dump-embeddings`, it does NOT construct the identity matcher, so
|
||||
// it neither loads a gallery nor runs calibration — ~24s faster per run and no
|
||||
// gallery file needed. Purpose-built for the optimizer's replay corpus and the
|
||||
// embedding-model bake-off (dump each --arcface model over the film set).
|
||||
//
|
||||
// Usage:
|
||||
// dump_embeddings --movie <path> --out <dump.h5> [--arcface <model.onnx>]
|
||||
// [--detector <model.onnx>] [--fps 1] [--start S] [--end S]
|
||||
// [--conf 0.5] [--max-faces 10] [--min-face-px 40]
|
||||
|
||||
#include "config.hpp"
|
||||
#include "types.hpp"
|
||||
#include "nodes/frame_source_node.hpp"
|
||||
#include "nodes/camera_position_change_detector_node.hpp"
|
||||
#include "nodes/face_detector_node.hpp"
|
||||
#include "nodes/face_aligner_node.hpp"
|
||||
#include "nodes/embedder_node.hpp"
|
||||
#include "nodes/embedding_dump_node.hpp"
|
||||
|
||||
#include <kpn/kpn.hpp>
|
||||
|
||||
#include <atomic>
|
||||
#include <iostream>
|
||||
#include <string>
|
||||
|
||||
int main(int argc, char** argv) {
|
||||
Config cfg;
|
||||
cfg.arcface_model = kDefaultArcfaceModel;
|
||||
cfg.detector_model = std::string(SAE_MODELS_DIR) + "/scrfd_500m_bnkps.onnx";
|
||||
|
||||
for (int i = 1; i < argc; ++i) {
|
||||
std::string a = argv[i];
|
||||
auto next = [&]() -> std::string {
|
||||
if (++i >= argc) throw std::runtime_error("missing arg after " + a);
|
||||
return argv[i];
|
||||
};
|
||||
if (a == "--movie") cfg.movie_path = next();
|
||||
else if (a == "--out") cfg.dump_embeddings_path = next();
|
||||
else if (a == "--arcface") cfg.arcface_model = next();
|
||||
else if (a == "--arcface-engine") cfg.arcface_engine = next();
|
||||
else if (a == "--detector") cfg.detector_model = next();
|
||||
else if (a == "--detector-engine") cfg.detector_engine = next();
|
||||
else if (a == "--fps") cfg.sample_fps = std::stof(next());
|
||||
else if (a == "--start") cfg.start_sec = std::stod(next());
|
||||
else if (a == "--end") cfg.end_sec = std::stod(next());
|
||||
else if (a == "--conf") cfg.detector_conf = std::stof(next());
|
||||
else if (a == "--max-faces") cfg.max_faces = std::stoi(next());
|
||||
else if (a == "--min-face-px") cfg.min_face_px = std::stof(next());
|
||||
else if (a == "--max-decode-fps") cfg.max_decode_fps = std::stof(next());
|
||||
else { std::cerr << "[dump] unknown flag: " << a << "\n"; return 1; }
|
||||
}
|
||||
if (cfg.movie_path.empty() || cfg.dump_embeddings_path.empty()) {
|
||||
std::cerr << "Usage: dump_embeddings --movie <path> --out <dump.h5> "
|
||||
"[--arcface <model.onnx>] [--fps 1] ...\n";
|
||||
return 1;
|
||||
}
|
||||
|
||||
std::atomic<bool> done{false};
|
||||
|
||||
FrameSourceFunc source_fn {cfg};
|
||||
CameraPositionChangeDetectorFunc campos_fn {cfg};
|
||||
FaceDetectorFunc detector_fn{cfg};
|
||||
FaceAlignerFunc aligner_fn;
|
||||
EmbedderFunc embedder_fn{cfg};
|
||||
EmbeddingDumpFunc dump_fn {cfg, done};
|
||||
|
||||
kpn::ObjectNode<FrameSourceFunc, kpn::in<>, kpn::out<"raw">, "frame_source", 0> source (source_fn, 32);
|
||||
kpn::ObjectNode<CameraPositionChangeDetectorFunc, kpn::in<"raw">, kpn::out<"frame">, "camera_pos", 0> campos (campos_fn, 32);
|
||||
kpn::ObjectNode<FaceDetectorFunc, kpn::in<"frame">, kpn::out<"scene">, "face_detector", 0> detector(detector_fn, 64);
|
||||
kpn::ObjectNode<FaceAlignerFunc, kpn::in<"scene">, kpn::out<"aligned">, "face_aligner", 0> aligner (aligner_fn, 64);
|
||||
kpn::ObjectNode<EmbedderFunc, kpn::in<"aligned">, kpn::out<"embedded">, "embedder", 0> embedder(embedder_fn, 32);
|
||||
kpn::ObjectNode<EmbeddingDumpFunc, kpn::in<"embedded">,kpn::out<>, "embedding_dump",0> dump (dump_fn, 32);
|
||||
|
||||
auto net = kpn::make_network(
|
||||
kpn::edge(source.output<"raw">(), campos.input<"raw">()),
|
||||
kpn::edge(campos.output<"frame">(), detector.input<"frame">()),
|
||||
kpn::edge(detector.output<"scene">(), aligner.input<"scene">()),
|
||||
kpn::edge(aligner.output<"aligned">(), embedder.input<"aligned">()),
|
||||
kpn::edge(embedder.output<"embedded">(), dump.input<"embedded">())
|
||||
);
|
||||
|
||||
net.start();
|
||||
using namespace std::chrono_literals;
|
||||
while (!done.load(std::memory_order_acquire)) std::this_thread::sleep_for(50ms);
|
||||
net.stop();
|
||||
return 0;
|
||||
}
|
||||
Reference in New Issue
Block a user