dtourolle
41a277bc19
Gallery format switches from JSON to HDF5 exclusively (JSON read-only kept for back-compat): save_gallery always writes HDF5, and the fitted Platt-sigmoid calibration (a, b, valid, hash) is now embedded directly in the gallery file instead of a sidecar .calib_cache.json — identity_matcher reads it from the loaded gallery and writes back only when the embeddings actually changed (hash mismatch), skipping the O(n^2) refit otherwise. Also includes: TensorRT inference backend support (ort_backend.cpp, trt_backend.cpp), gemm_backend improvements, TransNetV2-based scene-boundary detection wired through frame_source/face_tracker/main, and CMake build target updates for the new sources. Bumps the KPN submodule to feature/persistent-pipeline-reuse (push_blocking backpressure, node_ptr/node_stats introspection, ObjectVariantNodeWrapper for stateful functors) — needed by the optimizer's sae_kpn Python bindings.
149 lines
11 KiB
C++
149 lines
11 KiB
C++
#pragma once
|
||
#include "inference/backend_config.hpp"
|
||
#include <string>
|
||
|
||
inline const std::string kDefaultDetectorModel = std::string(SAE_MODELS_DIR) + "/scrfd_500m_bnkps.onnx";
|
||
inline const std::string kDefaultArcfaceModel = std::string(SAE_MODELS_DIR) + "/LVFace-B_Glint360K.onnx";
|
||
inline const std::string kDefaultSceneModel = std::string(SAE_MODELS_DIR) + "/transnetv2.onnx";
|
||
|
||
enum class Verbosity {
|
||
minimal, // actor names + merged time windows only
|
||
standard, // per-frame detail: bbox, similarity, unknowns logged
|
||
xray, // Jellyfin-Xray format: {"second": ["Actor", ...], ...}
|
||
};
|
||
// debug verbosity = compile with -DSAE_DEBUG → scene_analyze_debug binary
|
||
|
||
struct Config {
|
||
// ── Input ─────────────────────────────────────────────────────────────────
|
||
std::string movie_path;
|
||
std::string gallery_path; // gallery.json produced by build_gallery
|
||
|
||
// ── Output ───────────────────────────────────────────────────────────────
|
||
std::string output_path; // annotations.json
|
||
Verbosity verbosity{Verbosity::minimal};
|
||
|
||
// When set, tee the embedder output to an HDF5 dump (schema:
|
||
// scripts/optimizer/SCHEMA.md) for offline threshold-sweep replay via sae_kpn.
|
||
std::string dump_embeddings_path;
|
||
|
||
// ── Sampling ─────────────────────────────────────────────────────────────
|
||
float sample_fps{1.0f}; // frames to analyse per second of movie
|
||
float max_decode_fps{0.f}; // wall-clock cap on source decode rate (0 = uncapped)
|
||
double start_sec{0.0}; // seek to this timestamp before sampling
|
||
double end_sec{-1.0}; // stop at this timestamp (-1 = end of file)
|
||
|
||
// ── Detection (SCRFD-500MF via cv::dnn::Net) ──────────────────────────────
|
||
std::string detector_model;
|
||
std::string detector_engine; // optional path to pre-built TRT engine; bypasses ORT
|
||
int max_faces{10}; // pipeline cap: keep only the N largest faces
|
||
float min_face_px{40.f}; // discard detections narrower or shorter than this
|
||
float detector_conf{0.5f};
|
||
float detector_nms{0.4f};
|
||
|
||
// ── Recognition (ArcFace ONNX) ────────────────────────────────────────────
|
||
std::string arcface_model;
|
||
std::string arcface_engine; // optional path to a pre-built TRT engine; bypasses ORT
|
||
int embed_batch_size{4}; // max faces per ORT Run() call — bounds per-call latency
|
||
float match_prior{0.5f}; // base-rate prior; 0.5 = use calibrated sigmoid directly
|
||
// prob_threshold tuned by Differential Evolution against Amazon X-Ray per-scene
|
||
// presence over 4 films, per-second metric (see docs/rep4-optimizer-results.md).
|
||
// Best model+mode: LVFace-B_Glint360K, full gallery, expansion on. Supersedes the
|
||
// earlier 9-film scene-union-metric tuning (0.76) — that metric is now known to
|
||
// have hidden out-of-cast false positives (see docs/optimizer-experiments.md).
|
||
float prob_threshold{0.754f}; // posterior P(match | sim, prior) threshold
|
||
float match_threshold{0.45f}; // cosine distance hard ceiling fallback (no calibration)
|
||
float match_ratio{0.80f}; // ratio test fallback: accept if best/second < ratio
|
||
float match_ratio_ceil{0.65f}; // ratio test only fires below this absolute distance
|
||
|
||
// ── Cut detection ────────────────────────────────────────────────────────
|
||
float cut_threshold{0.70f}; // grayscale histogram correlation below this → hard cut
|
||
|
||
// ── Scene detection (TransNetV2, opt-in) ─────────────────────────────────
|
||
// When enabled, the source decodes densely (native FPS) and a decimator
|
||
// splits the stream: full-res 1-FPS frames to the face pipeline, and a
|
||
// downscaled dense stream to the TransNetV2 scene detector. Shot boundaries
|
||
// it finds are surfaced as Frame::is_scene_boundary. This is separate from
|
||
// the always-on histogram cut, which flags intra-scene camera-angle changes.
|
||
bool scene_detect{false}; // master switch (--scene-detect)
|
||
std::string scene_model; // TransNetV2 .onnx (default set in main)
|
||
std::string scene_engine; // optional pre-built TRT .engine; bypasses ORT
|
||
float scene_threshold{0.60f}; // sigmoid boundary prob above this → boundary
|
||
// (this export's non-boundary baseline sits
|
||
// at ~0.50; real boundaries spike to ~0.7+)
|
||
int scene_stride{50}; // frames advanced between windows (≤ kWindow)
|
||
|
||
// Dense-decode throughput knobs (only active with scene_detect). Dense decode
|
||
// of every native-rate frame is the pipeline's cost driver; these trade a
|
||
// little boundary precision for a large speedup.
|
||
// scene_decode_fps: rate the source decodes at in dense mode. Lower =
|
||
// fewer frames decoded. TransNetV2 tolerates ~12fps; boundary timestamps
|
||
// stay correct (keyed off each frame's real timestamp). 0 = native fps.
|
||
// dense_scale: downscale factor applied to decoded frames in dense mode
|
||
// (0<f≤1; e.g. 0.5 = half size). Cheaper sws_scale + smaller frames
|
||
// through the fanout. NOTE: also shrinks what the face detector sees —
|
||
// keep ≥0.5 on 1080p sources so SCRFD still resolves small faces. 1 = off.
|
||
float scene_decode_fps{12.0f}; // dense decode rate (0 = native)
|
||
float dense_scale{1.0f}; // dense-mode frame downscale (1 = off)
|
||
|
||
// ── Face tracking (frame-to-frame) ───────────────────────────────────────
|
||
float track_alpha{0.4f}; // cost weight: 0=embedding only, 1=spatial only
|
||
float track_min_iou{0.1f}; // IoU below which spatial link alone is rejected
|
||
float track_max_embed_dist{0.7f}; // cosine dist above which embedding link alone is rejected
|
||
int track_max_frames_missing{5}; // expire track after N consecutive missed frames
|
||
|
||
// ── Cross-cut track re-association ────────────────────────────────────────
|
||
// A camera-angle change (Frame::is_cut) breaks spatial (IoU) continuity but
|
||
// not identity: the same people are usually still on screen from a new angle.
|
||
// Instead of destroying tracks on a cut, the tracker parks them in an
|
||
// inactive pool. A post-cut detection whose raw cosine similarity to a parked
|
||
// track's last-frame embedding is ≥ cut_revive_sim revives that track_id
|
||
// (identity continuity survives the cut); otherwise it starts a fresh track.
|
||
// Parked tracks that go unrevived for cut_inactive_max_frames are dropped.
|
||
float cut_revive_sim{0.50f}; // min raw cosine sim (last-frame emb) to revive across a cut
|
||
int cut_inactive_max_frames{5}; // drop a parked track after N frames without revival
|
||
|
||
// ── Scene tracking ────────────────────────────────────────────────────────
|
||
// extinction_sec re-tuned by DE against X-Ray per-second presence, 4-film rep4
|
||
// matrix (docs/rep4-optimizer-results.md). Reverses the earlier "short is better"
|
||
// finding: with a stricter prob_threshold, a long extinction window bridges real
|
||
// presence gaps (occlusion, turned face) instead of just smearing FPs — every
|
||
// model's best config pushed to ~90%+ of the search ceiling (tried up to 60s).
|
||
// The ceiling kept getting hit, so treat 60 as "good enough", not a proven optimum.
|
||
double extinction_sec{57.4}; // keep actor active this many seconds after last detection
|
||
// anneal_sec: previously found INSENSITIVE at a 1–30s range; the wider rep4 sweep
|
||
// (1–60s) also pushed this to the ceiling alongside extinction_sec (see above).
|
||
double anneal_sec{35.5}; // merge actor windows separated by less than this into one epoch
|
||
|
||
// ── Per-film gallery expansion ────────────────────────────────────────────
|
||
// Within one uncut track every face is the same physical person — a free
|
||
// same-identity label the baked gallery lacks. When a track is confidently
|
||
// owned by an actor, its gallery-far (pose-varied) embeddings are validated
|
||
// new reference views; they are promoted into a per-film, in-memory annex so
|
||
// later frames/tracks of that actor at similar poses recognise. See
|
||
// gallery/track_gallery.hpp.
|
||
// Default ON: rep4 matrix (docs/rep4-optimizer-results.md) found expansion helps
|
||
// recall on the full (unrestricted) gallery for the winning model/mode — the
|
||
// opposite of the earlier assumption that it only helps restricted galleries.
|
||
bool expand_gallery{true}; // master switch
|
||
int expand_buffer_size{20}; // per-track diversity buffer capacity
|
||
float expand_novelty_sim{0.55f}; // promote only embeddings whose best sim to the
|
||
// actor's refs is below this (gallery-far / novel)
|
||
float expand_track_spread_max{0.60f}; // reject promotion if the retained buffer's
|
||
// internal spread (1 - min pairwise sim) exceeds
|
||
// this — guards track-ID collisions / two people
|
||
int expand_min_anchor_frames{3}; // require ≥N accepted frames naming the actor before
|
||
// the track is confirmed and its buffer promoted
|
||
std::string expand_debug_dir; // if set, dump promoted mugshots + embeddings here
|
||
|
||
// ── Inference backend tuning ────────────────────────────────────────────
|
||
// Consumed by the compiled-in inference backend (ORT or TRT).
|
||
// INT8 is unsafe for ArcFace without a calibration table.
|
||
BackendConfig trt{}; // fp16=true, int8=false, cache_dir="./trt_cache"
|
||
|
||
// ── Debug output (only used when SAE_DEBUG is defined) ───────────────────
|
||
#ifdef SAE_DEBUG
|
||
std::string debug_dir{"debug_frames"};
|
||
float crop_context{1.5f}; // bbox expansion factor for context crop
|
||
#endif
|
||
};
|