dtourolle
e5204a831a
Wire the XGBoost scene-boundary detector into scene_analyze as a post-EOF step in
the result sink (like flood-fill itself — the per-film knee threshold needs the
whole film, so it cannot stream). With --scene-xgb-model set, the camera-position
node stamps a per-frame RGB histogram onto the Frame, it rides through to the
sink, and at EOF the sink runs XGBSceneBoundary over the collected histograms +
the movie's per-second audio log-PSD to produce the flood-fill boundaries. Falls
back to is_scene_boundary / is_cut when no model is configured or inference fails.
Inference is real XGBoost via CMake FetchContent (v2.1.1, static), C API in
src/inference/xgb_scene_boundary.hpp; audio log-PSD in src/inference/
audio_logpsd.hpp (FFTW + ffmpeg full-file 16kHz decode). Feature extraction
matches training exactly — video features verified row-identical to numpy, and to
avoid chasing numpy's every rounding the shipped model is TRAINED on the
C++-extracted features (scene_features_dump exe → train_xgb_cpp.py). The
C++/Python peak-finders differ slightly so boundary counts differ, but what
matters is downstream: flood + C++ detector = 75.8% macro presence F1 vs 64.0%
for the histogram-cut flood and 62.5% for track_extent, and it fixes the Scarface
flood collapse (41 -> 70). All nine films improve.
Guarded by the SAE_SCENE_XGB CMake option (on by default; heavy first build).
xgb_boundary_parity is a diff harness; scene_features_dump writes the C++ feature
matrix so training and inference share one feature implementation.
Verified end to end: scene_analyze --scene-xgb-model on a real movie stamps the
histogram, runs the detector at EOF ("XGBoost scene detector: N boundaries"), and
flood-snaps presence to the learned boundaries.
97 lines
4.1 KiB
C++
97 lines
4.1 KiB
C++
#pragma once
|
|
#include "types.hpp"
|
|
#include "config.hpp"
|
|
|
|
#include <opencv2/imgproc.hpp>
|
|
#include <algorithm>
|
|
#include <iostream>
|
|
#include <string_view>
|
|
|
|
// ── CameraPositionChangeDetectorFunc ──────────────────────────────────────────
|
|
// KPN node: flags intra-scene camera-angle changes (hard cuts) by comparing each
|
|
// frame's grayscale histogram to the previous frame's. When the normalised
|
|
// histogram correlation drops below cut_threshold, the frame is marked with
|
|
// Frame::is_cut = true.
|
|
//
|
|
// This is the pipeline's cut *trigger*: it owns cut detection so a single node
|
|
// decides when the camera position has changed, and every downstream stage reads
|
|
// the decision off Frame::is_cut (which rides with the frame's timestamp_sec /
|
|
// frame_idx). The face_tracker consumes it to re-associate tracks across the cut
|
|
// rather than blindly resetting; the track_gallery consumes it to bound
|
|
// promotion to a single physical viewpoint.
|
|
//
|
|
// This is the always-on histogram cut — separate from the opt-in TransNetV2
|
|
// scene detector, which localises true shot boundaries as Frame::is_scene_boundary.
|
|
//
|
|
// The node is a pure pass-through: it forwards the Frame unchanged except for
|
|
// is_cut, so it slots between frame_source and face_detector without altering the
|
|
// downstream contract. eof frames are forwarded immediately without processing.
|
|
//
|
|
/// TRACES: AR-009 | SR-002
|
|
struct CameraPositionChangeDetectorFunc {
|
|
static constexpr std::string_view label() { return "camera_position_change_detector"; }
|
|
|
|
explicit CameraPositionChangeDetectorFunc(const Config& cfg)
|
|
: cut_threshold_(cfg.cut_threshold)
|
|
, want_rgb_hist_(!cfg.scene_xgb_model.empty())
|
|
{
|
|
std::cerr << "[camera_position_change_detector] cut_threshold="
|
|
<< cut_threshold_
|
|
<< (want_rgb_hist_ ? " (+rgb_hist for scene detector)" : "")
|
|
<< "\n";
|
|
}
|
|
|
|
// 32-bin-per-channel normalised RGB histogram (96 floats), the exact layout
|
|
// the XGBoost scene detector was trained on (see embedding_dump_node). Only
|
|
// computed when a scene model is configured, so it costs nothing otherwise.
|
|
static std::vector<float> rgb_histogram(const cv::Mat& img) {
|
|
constexpr int kBins = 32;
|
|
std::vector<float> out(kBins * 3, 0.f);
|
|
if (img.empty() || img.channels() != 3) return out;
|
|
float range[] = {0.f, 256.f}; const float* ranges = range; int bins = kBins;
|
|
for (int c = 0; c < 3; ++c) { // OpenCV BGR → store B,G,R blocks
|
|
cv::Mat h;
|
|
cv::calcHist(&img, 1, &c, cv::Mat(), h, 1, &bins, &ranges);
|
|
cv::normalize(h, h, 1.0, 0.0, cv::NORM_L1);
|
|
for (int b = 0; b < kBins; ++b) out[c*kBins + b] = h.at<float>(b);
|
|
}
|
|
return out;
|
|
}
|
|
|
|
Frame operator()(Frame f) {
|
|
if (f.eof) return f;
|
|
|
|
cv::Mat gray;
|
|
cv::cvtColor(f.image, gray, cv::COLOR_BGR2GRAY);
|
|
|
|
cv::Mat hist;
|
|
const int bins = 64;
|
|
const float range[] = {0.f, 256.f};
|
|
const float* ranges = range;
|
|
cv::calcHist(&gray, 1, nullptr, cv::Mat(), hist, 1, &bins, &ranges);
|
|
cv::normalize(hist, hist, 1.0, 0.0, cv::NORM_L1);
|
|
|
|
if (prev_hist_valid_) {
|
|
double corr = cv::compareHist(prev_hist_, hist, cv::HISTCMP_CORREL);
|
|
// Cut score in [0,1]: 0 = identical to previous frame, ~1 = fully
|
|
// different. Rides on the Frame for the preview HUD / debugging.
|
|
f.cut_score = static_cast<float>(std::clamp(1.0 - corr, 0.0, 1.0));
|
|
f.is_cut = (corr < cut_threshold_);
|
|
if (f.is_cut)
|
|
std::cerr << "[camera_position_change_detector] cut at t="
|
|
<< f.timestamp_sec << "s hist_corr=" << corr << "\n";
|
|
}
|
|
prev_hist_ = hist;
|
|
prev_hist_valid_ = true;
|
|
|
|
if (want_rgb_hist_) f.rgb_hist = rgb_histogram(f.image);
|
|
return f;
|
|
}
|
|
|
|
private:
|
|
float cut_threshold_;
|
|
bool want_rgb_hist_{false};
|
|
cv::Mat prev_hist_;
|
|
bool prev_hist_valid_{false};
|
|
};
|