dtourolle
584f23546a
Add PresenceMode::flood alongside the default track_extent. In flood mode the result sink snaps each presence claim to the shot it sits in, so an actor seen once anywhere in a shot is reported for the whole shot [prev_boundary, next_boundary]. This trades precision for recall against X-Ray's per-scene cast granularity and is a toggleable knob for the optimizer to weigh rather than a default. Boundaries come from the frame stream, now carried through SceneAnnotation (is_cut and is_scene_boundary). Flood prefers TransNetV2 shot boundaries when a scene detector populated them, otherwise falls back to the always-on histogram cuts (camera_position_change_detector); with no boundaries it degrades to track_extent per claim. The is_scene_boundary path stays dormant so an out-of-process scene detector can be revived later without re-wiring. Selected with --presence-mode flood|track_extent (default track_extent), so existing output is byte-for-byte unchanged. The dump_embeddings header note records why TransNetV2 scene detection is not run in that process.
99 lines
5.2 KiB
C++
99 lines
5.2 KiB
C++
// dump_embeddings — standalone embedding dumper (NO gallery required).
|
|
//
|
|
// Runs only the expensive, gallery-independent front half of the pipeline
|
|
// (decode → camera-pos → detect → align → embed) and writes per-frame face
|
|
// embeddings + metadata to HDF5 (scripts/optimizer/SCHEMA.md). Unlike
|
|
// `scene_analyze --dump-embeddings`, it does NOT construct the identity matcher, so
|
|
// it neither loads a gallery nor runs calibration — ~24s faster per run and no
|
|
// gallery file needed. Purpose-built for the optimizer's replay corpus and the
|
|
// embedding-model bake-off (dump each --arcface model over the film set).
|
|
//
|
|
// The camera-position (histogram) cut detector runs, so frames/is_cut is recorded
|
|
// in the dump. Flood-fill presence snaps to those cuts. TransNetV2 scene detection
|
|
// is NOT run here: on the ROCm build it needs MIGraphX, whose statically-linked
|
|
// LLVM collides with the VAAPI decoder's system libLLVM and crashes at compile, so
|
|
// the two cannot share a process. is_scene_boundary therefore stays 0 here.
|
|
//
|
|
// Usage:
|
|
// dump_embeddings --movie <path> --out <dump.h5> [--arcface <model.onnx>]
|
|
// [--detector <model.onnx>] [--fps 1] [--start S] [--end S]
|
|
// [--conf 0.5] [--max-faces 10] [--min-face-px 40]
|
|
|
|
#include "config.hpp"
|
|
#include "types.hpp"
|
|
#include "nodes/frame_source_node.hpp"
|
|
#include "nodes/camera_position_change_detector_node.hpp"
|
|
#include "nodes/face_detector_node.hpp"
|
|
#include "nodes/face_aligner_node.hpp"
|
|
#include "nodes/embedder_node.hpp"
|
|
#include "nodes/embedding_dump_node.hpp"
|
|
|
|
#include <kpn/kpn.hpp>
|
|
|
|
#include <atomic>
|
|
#include <iostream>
|
|
#include <string>
|
|
|
|
int main(int argc, char** argv) {
|
|
Config cfg;
|
|
cfg.arcface_model = kDefaultArcfaceModel;
|
|
cfg.detector_model = std::string(SAE_MODELS_DIR) + "/scrfd_500m_bnkps.onnx";
|
|
|
|
for (int i = 1; i < argc; ++i) {
|
|
std::string a = argv[i];
|
|
auto next = [&]() -> std::string {
|
|
if (++i >= argc) throw std::runtime_error("missing arg after " + a);
|
|
return argv[i];
|
|
};
|
|
if (a == "--movie") cfg.movie_path = next();
|
|
else if (a == "--out") cfg.dump_embeddings_path = next();
|
|
else if (a == "--arcface") cfg.arcface_model = next();
|
|
else if (a == "--arcface-engine") cfg.arcface_engine = next();
|
|
else if (a == "--detector") cfg.detector_model = next();
|
|
else if (a == "--detector-engine") cfg.detector_engine = next();
|
|
else if (a == "--fps") cfg.sample_fps = std::stof(next());
|
|
else if (a == "--start") cfg.start_sec = std::stod(next());
|
|
else if (a == "--end") cfg.end_sec = std::stod(next());
|
|
else if (a == "--conf") cfg.detector_conf = std::stof(next());
|
|
else if (a == "--max-faces") cfg.max_faces = std::stoi(next());
|
|
else if (a == "--min-face-px") cfg.min_face_px = std::stof(next());
|
|
else if (a == "--max-decode-fps") cfg.max_decode_fps = std::stof(next());
|
|
else { std::cerr << "[dump] unknown flag: " << a << "\n"; return 1; }
|
|
}
|
|
if (cfg.movie_path.empty() || cfg.dump_embeddings_path.empty()) {
|
|
std::cerr << "Usage: dump_embeddings --movie <path> --out <dump.h5> "
|
|
"[--arcface <model.onnx>] [--fps 1] ...\n";
|
|
return 1;
|
|
}
|
|
|
|
std::atomic<bool> done{false};
|
|
|
|
FrameSourceFunc source_fn {cfg};
|
|
CameraPositionChangeDetectorFunc campos_fn {cfg};
|
|
FaceDetectorFunc detector_fn{cfg};
|
|
FaceAlignerFunc aligner_fn;
|
|
EmbedderFunc embedder_fn{cfg};
|
|
EmbeddingDumpFunc dump_fn {cfg, done};
|
|
|
|
kpn::ObjectNode<FrameSourceFunc, kpn::in<>, kpn::out<"raw">, "frame_source", 0> source (source_fn, 32);
|
|
kpn::ObjectNode<CameraPositionChangeDetectorFunc, kpn::in<"raw">, kpn::out<"frame">, "camera_pos", 0> campos (campos_fn, 32);
|
|
kpn::ObjectNode<FaceDetectorFunc, kpn::in<"frame">, kpn::out<"scene">, "face_detector", 0> detector(detector_fn, 64);
|
|
kpn::ObjectNode<FaceAlignerFunc, kpn::in<"scene">, kpn::out<"aligned">, "face_aligner", 0> aligner (aligner_fn, 64);
|
|
kpn::ObjectNode<EmbedderFunc, kpn::in<"aligned">, kpn::out<"embedded">, "embedder", 0> embedder(embedder_fn, 32);
|
|
kpn::ObjectNode<EmbeddingDumpFunc, kpn::in<"embedded">,kpn::out<>, "embedding_dump",0> dump (dump_fn, 32);
|
|
|
|
auto net = kpn::make_network(
|
|
kpn::edge(source.output<"raw">(), campos.input<"raw">()),
|
|
kpn::edge(campos.output<"frame">(), detector.input<"frame">()),
|
|
kpn::edge(detector.output<"scene">(), aligner.input<"scene">()),
|
|
kpn::edge(aligner.output<"aligned">(), embedder.input<"aligned">()),
|
|
kpn::edge(embedder.output<"embedded">(), dump.input<"embedded">())
|
|
);
|
|
|
|
net.start();
|
|
using namespace std::chrono_literals;
|
|
while (!done.load(std::memory_order_acquire)) std::this_thread::sleep_for(50ms);
|
|
net.stop();
|
|
return 0;
|
|
}
|