feat(dump): record a per-frame RGB histogram for scene-boundary training
Add frames/rgb_hist to the embedding dump: a normalised 32-bin-per-channel RGB histogram (96 floats/frame), computed from the already-decoded frame so it is nearly free and ~40 KB per film. This is the training signal for the learned scene-boundary detector — the grayscale-correlation cut detector is blind on low-contrast grades (Scarface: 1 cut in 10k frames), and the symmetric RGB-histogram delta separates X-Ray scene boundaries far better. The dump stays gallery-independent; downstream replay/training consume the histogram offline.
This commit is contained in:
@@ -5,6 +5,7 @@
|
||||
#include "gallery/embedder_stamp.hpp"
|
||||
|
||||
#include <H5Cpp.h>
|
||||
#include <opencv2/imgproc.hpp> // cv::calcHist for the per-frame RGB histogram
|
||||
|
||||
#include <atomic>
|
||||
#include <cstdint>
|
||||
@@ -167,6 +168,12 @@ struct EmbeddingDumpFunc {
|
||||
fidx_.push_back(ef.source.frame_idx);
|
||||
is_cut_.push_back(ef.source.is_cut ? 1 : 0);
|
||||
is_bnd_.push_back(ef.source.is_scene_boundary ? 1 : 0);
|
||||
// Per-frame normalised RGB histogram (kHistBins per channel), for offline
|
||||
// training of a learned scene-boundary detector against X-Ray scene
|
||||
// boundaries — the grayscale-correlation cut detector is blind on
|
||||
// low-contrast grades (Scarface: 1 cut in 10k frames). Cheap and the frame
|
||||
// is already decoded here; empty frame → zeros.
|
||||
append_rgb_hist(ef.source.image);
|
||||
face_off_.push_back(static_cast<int64_t>(conf_.size()));
|
||||
face_cnt_.push_back(n);
|
||||
|
||||
@@ -287,6 +294,11 @@ private:
|
||||
write_vec(frames, "is_scene_boundary", is_bnd_, H5::PredType::NATIVE_UINT8);
|
||||
write_vec(frames, "face_offset", face_off_, H5::PredType::NATIVE_INT64);
|
||||
write_vec(frames, "face_count", face_cnt_, H5::PredType::NATIVE_INT32);
|
||||
// Per-frame normalised RGB histogram, kHistBins per channel laid out
|
||||
// [R(kHistBins) G(kHistBins) B(kHistBins)] per row. Feeds the learned
|
||||
// scene-boundary detector (see scripts/scene_detector/).
|
||||
write_vec(frames, "rgb_hist", rgb_hist_, H5::PredType::NATIVE_FLOAT,
|
||||
kHistBins * 3);
|
||||
|
||||
H5::Group faces = file.createGroup("faces");
|
||||
write_vec(faces, "embedding", emb_, H5::PredType::NATIVE_FLOAT, kEmbedDim);
|
||||
@@ -301,6 +313,29 @@ private:
|
||||
<< conf_.size() << " faces → " << path_ << "\n";
|
||||
}
|
||||
|
||||
// Per-channel bin count for the RGB histogram. 32 → a 96-float row per frame,
|
||||
// ~40 KB per 10k-frame film: negligible next to the embeddings.
|
||||
static constexpr int kHistBins = 32;
|
||||
|
||||
// Append the frame's normalised per-channel RGB histogram (R,G,B blocks). An
|
||||
// empty frame (EOF sentinels never reach here) yields a zero row so the array
|
||||
// stays parallel to ts_.
|
||||
void append_rgb_hist(const cv::Mat& img) {
|
||||
const size_t base = rgb_hist_.size();
|
||||
rgb_hist_.resize(base + kHistBins * 3, 0.f);
|
||||
if (img.empty() || img.channels() != 3) return;
|
||||
float range[] = {0.f, 256.f};
|
||||
const float* ranges[] = {range};
|
||||
int bins = kHistBins;
|
||||
for (int c = 0; c < 3; ++c) { // OpenCV is BGR; store as B,G,R blocks
|
||||
cv::Mat h;
|
||||
cv::calcHist(&img, 1, &c, cv::Mat(), h, 1, &bins, ranges);
|
||||
cv::normalize(h, h, 1.0, 0.0, cv::NORM_L1);
|
||||
for (int b = 0; b < kHistBins; ++b)
|
||||
rgb_hist_[base + c * kHistBins + b] = h.at<float>(b);
|
||||
}
|
||||
}
|
||||
|
||||
std::string path_, movie_;
|
||||
EmbedderStamp stamp_;
|
||||
DumpProvenance prov_;
|
||||
@@ -315,4 +350,5 @@ private:
|
||||
std::vector<int32_t> face_cnt_;
|
||||
std::vector<float> emb_, bbox_, lmk_, conf_;
|
||||
std::vector<float> sharp_, resid_; // AR-028 quality vector, parallel to conf_
|
||||
std::vector<float> rgb_hist_; // kHistBins*3 per frame, parallel to ts_
|
||||
};
|
||||
|
||||
Reference in New Issue
Block a user