feat(dump): record a per-frame RGB histogram for scene-boundary training

Add frames/rgb_hist to the embedding dump: a normalised 32-bin-per-channel
RGB histogram (96 floats/frame), computed from the already-decoded frame so
it is nearly free and ~40 KB per film. This is the training signal for the
learned scene-boundary detector — the grayscale-correlation cut detector is
blind on low-contrast grades (Scarface: 1 cut in 10k frames), and the
symmetric RGB-histogram delta separates X-Ray scene boundaries far better.
The dump stays gallery-independent; downstream replay/training consume the
histogram offline.
This commit is contained in:
2026-08-09 19:20:34 +02:00
parent 0e97e532a8
commit 8dd2255125
+36
View File
@@ -5,6 +5,7 @@
#include "gallery/embedder_stamp.hpp" #include "gallery/embedder_stamp.hpp"
#include <H5Cpp.h> #include <H5Cpp.h>
#include <opencv2/imgproc.hpp> // cv::calcHist for the per-frame RGB histogram
#include <atomic> #include <atomic>
#include <cstdint> #include <cstdint>
@@ -167,6 +168,12 @@ struct EmbeddingDumpFunc {
fidx_.push_back(ef.source.frame_idx); fidx_.push_back(ef.source.frame_idx);
is_cut_.push_back(ef.source.is_cut ? 1 : 0); is_cut_.push_back(ef.source.is_cut ? 1 : 0);
is_bnd_.push_back(ef.source.is_scene_boundary ? 1 : 0); is_bnd_.push_back(ef.source.is_scene_boundary ? 1 : 0);
// Per-frame normalised RGB histogram (kHistBins per channel), for offline
// training of a learned scene-boundary detector against X-Ray scene
// boundaries — the grayscale-correlation cut detector is blind on
// low-contrast grades (Scarface: 1 cut in 10k frames). Cheap and the frame
// is already decoded here; empty frame → zeros.
append_rgb_hist(ef.source.image);
face_off_.push_back(static_cast<int64_t>(conf_.size())); face_off_.push_back(static_cast<int64_t>(conf_.size()));
face_cnt_.push_back(n); face_cnt_.push_back(n);
@@ -287,6 +294,11 @@ private:
write_vec(frames, "is_scene_boundary", is_bnd_, H5::PredType::NATIVE_UINT8); write_vec(frames, "is_scene_boundary", is_bnd_, H5::PredType::NATIVE_UINT8);
write_vec(frames, "face_offset", face_off_, H5::PredType::NATIVE_INT64); write_vec(frames, "face_offset", face_off_, H5::PredType::NATIVE_INT64);
write_vec(frames, "face_count", face_cnt_, H5::PredType::NATIVE_INT32); write_vec(frames, "face_count", face_cnt_, H5::PredType::NATIVE_INT32);
// Per-frame normalised RGB histogram, kHistBins per channel laid out
// [R(kHistBins) G(kHistBins) B(kHistBins)] per row. Feeds the learned
// scene-boundary detector (see scripts/scene_detector/).
write_vec(frames, "rgb_hist", rgb_hist_, H5::PredType::NATIVE_FLOAT,
kHistBins * 3);
H5::Group faces = file.createGroup("faces"); H5::Group faces = file.createGroup("faces");
write_vec(faces, "embedding", emb_, H5::PredType::NATIVE_FLOAT, kEmbedDim); write_vec(faces, "embedding", emb_, H5::PredType::NATIVE_FLOAT, kEmbedDim);
@@ -301,6 +313,29 @@ private:
<< conf_.size() << " faces → " << path_ << "\n"; << conf_.size() << " faces → " << path_ << "\n";
} }
// Per-channel bin count for the RGB histogram. 32 → a 96-float row per frame,
// ~40 KB per 10k-frame film: negligible next to the embeddings.
static constexpr int kHistBins = 32;
// Append the frame's normalised per-channel RGB histogram (R,G,B blocks). An
// empty frame (EOF sentinels never reach here) yields a zero row so the array
// stays parallel to ts_.
void append_rgb_hist(const cv::Mat& img) {
const size_t base = rgb_hist_.size();
rgb_hist_.resize(base + kHistBins * 3, 0.f);
if (img.empty() || img.channels() != 3) return;
float range[] = {0.f, 256.f};
const float* ranges[] = {range};
int bins = kHistBins;
for (int c = 0; c < 3; ++c) { // OpenCV is BGR; store as B,G,R blocks
cv::Mat h;
cv::calcHist(&img, 1, &c, cv::Mat(), h, 1, &bins, ranges);
cv::normalize(h, h, 1.0, 0.0, cv::NORM_L1);
for (int b = 0; b < kHistBins; ++b)
rgb_hist_[base + c * kHistBins + b] = h.at<float>(b);
}
}
std::string path_, movie_; std::string path_, movie_;
EmbedderStamp stamp_; EmbedderStamp stamp_;
DumpProvenance prov_; DumpProvenance prov_;
@@ -315,4 +350,5 @@ private:
std::vector<int32_t> face_cnt_; std::vector<int32_t> face_cnt_;
std::vector<float> emb_, bbox_, lmk_, conf_; std::vector<float> emb_, bbox_, lmk_, conf_;
std::vector<float> sharp_, resid_; // AR-028 quality vector, parallel to conf_ std::vector<float> sharp_, resid_; // AR-028 quality vector, parallel to conf_
std::vector<float> rgb_hist_; // kHistBins*3 per frame, parallel to ts_
}; };