feat(scene-detector): run the learned boundary detector live in the C++ pipeline
Wire the XGBoost scene-boundary detector into scene_analyze as a post-EOF step in
the result sink (like flood-fill itself — the per-film knee threshold needs the
whole film, so it cannot stream). With --scene-xgb-model set, the camera-position
node stamps a per-frame RGB histogram onto the Frame, it rides through to the
sink, and at EOF the sink runs XGBSceneBoundary over the collected histograms +
the movie's per-second audio log-PSD to produce the flood-fill boundaries. Falls
back to is_scene_boundary / is_cut when no model is configured or inference fails.
Inference is real XGBoost via CMake FetchContent (v2.1.1, static), C API in
src/inference/xgb_scene_boundary.hpp; audio log-PSD in src/inference/
audio_logpsd.hpp (FFTW + ffmpeg full-file 16kHz decode). Feature extraction
matches training exactly — video features verified row-identical to numpy, and to
avoid chasing numpy's every rounding the shipped model is TRAINED on the
C++-extracted features (scene_features_dump exe → train_xgb_cpp.py). The
C++/Python peak-finders differ slightly so boundary counts differ, but what
matters is downstream: flood + C++ detector = 75.8% macro presence F1 vs 64.0%
for the histogram-cut flood and 62.5% for track_extent, and it fixes the Scarface
flood collapse (41 -> 70). All nine films improve.
Guarded by the SAE_SCENE_XGB CMake option (on by default; heavy first build).
xgb_boundary_parity is a diff harness; scene_features_dump writes the C++ feature
matrix so training and inference share one feature implementation.
Verified end to end: scene_analyze --scene-xgb-model on a real movie stamps the
histogram, runs the detector at EOF ("XGBoost scene detector: N boundaries"), and
flood-snaps presence to the learned boundaries.
This commit is contained in:
@@ -3,6 +3,10 @@
|
||||
#include "types.hpp"
|
||||
#include "config.hpp"
|
||||
#include "track_registry.hpp"
|
||||
#ifdef SAE_SCENE_XGB
|
||||
#include "inference/xgb_scene_boundary.hpp"
|
||||
#include "inference/audio_logpsd.hpp"
|
||||
#endif
|
||||
|
||||
#include <nlohmann/json.hpp>
|
||||
#include <algorithm>
|
||||
@@ -219,15 +223,24 @@ private:
|
||||
// flood-fill actually snaps to — coarser than true shot boundaries (cuts also
|
||||
// fire on in-shot angle changes) but present with no extra pass.
|
||||
std::vector<double> scene_boundaries() const {
|
||||
bool have_scene = false;
|
||||
for (const auto& sa : frames_)
|
||||
if (sa.is_scene_boundary) { have_scene = true; break; }
|
||||
|
||||
std::vector<double> b;
|
||||
b.push_back(0.0);
|
||||
for (const auto& sa : frames_) {
|
||||
const bool boundary = have_scene ? sa.is_scene_boundary : sa.is_cut;
|
||||
if (boundary) b.push_back(sa.timestamp_sec);
|
||||
|
||||
// Preferred: the learned XGBoost scene detector, run once here post-EOF
|
||||
// (the knee threshold needs the whole film, so this is inherently a final
|
||||
// step — like flood-fill itself). Measured best flood boundary source.
|
||||
std::vector<double> learned = xgb_boundaries();
|
||||
if (!learned.empty()) {
|
||||
for (double t : learned) b.push_back(t);
|
||||
} else {
|
||||
// Fallback: TransNetV2 shot boundaries if present, else histogram cuts.
|
||||
bool have_scene = false;
|
||||
for (const auto& sa : frames_)
|
||||
if (sa.is_scene_boundary) { have_scene = true; break; }
|
||||
for (const auto& sa : frames_) {
|
||||
const bool boundary = have_scene ? sa.is_scene_boundary : sa.is_cut;
|
||||
if (boundary) b.push_back(sa.timestamp_sec);
|
||||
}
|
||||
}
|
||||
b.push_back(last_ts_ + 1.0); // a right edge past the final sample
|
||||
std::sort(b.begin(), b.end());
|
||||
@@ -235,6 +248,42 @@ private:
|
||||
return b;
|
||||
}
|
||||
|
||||
// Run the learned scene-boundary detector over the collected per-frame RGB
|
||||
// histograms + per-second audio log-PSD (decoded once from the movie). Returns
|
||||
// {} when no model is configured, the build lacks XGBoost, or no rgb_hist was
|
||||
// stamped (camera-position node only does so when a model is set).
|
||||
std::vector<double> xgb_boundaries() const {
|
||||
#ifdef SAE_SCENE_XGB
|
||||
if (cfg_.scene_xgb_model.empty()) return {};
|
||||
std::vector<std::vector<float>> hist;
|
||||
std::vector<double> ts;
|
||||
hist.reserve(frames_.size()); ts.reserve(frames_.size());
|
||||
for (const auto& sa : frames_) {
|
||||
if (sa.rgb_hist.empty()) return {}; // hist not stamped → bail to fallback
|
||||
hist.push_back(sa.rgb_hist);
|
||||
ts.push_back(sa.timestamp_sec);
|
||||
}
|
||||
if (hist.size() < 16) return {};
|
||||
try {
|
||||
auto audio = AudioLogPSD::extract(cfg_.movie_path); // [T'][B], aligned per second
|
||||
if ((int)audio.size() != (int)hist.size())
|
||||
audio.resize(hist.size(),
|
||||
std::vector<float>(audio.empty() ? 57 : audio[0].size(), 0.f));
|
||||
XGBSceneBoundary det(cfg_.scene_xgb_model);
|
||||
auto b = det.boundaries(hist, ts, audio);
|
||||
std::cerr << "[result_sink] XGBoost scene detector: " << b.size()
|
||||
<< " boundaries\n";
|
||||
return b;
|
||||
} catch (const std::exception& e) {
|
||||
std::cerr << "[result_sink] scene detector failed (" << e.what()
|
||||
<< "), falling back to histogram cuts\n";
|
||||
return {};
|
||||
}
|
||||
#else
|
||||
return {};
|
||||
#endif
|
||||
}
|
||||
|
||||
// The boundary opening the shot that contains t (largest boundary ≤ t).
|
||||
static double boundary_at_or_before(const std::vector<double>& b, double t) {
|
||||
auto it = std::upper_bound(b.begin(), b.end(), t);
|
||||
|
||||
Reference in New Issue
Block a user