feat(dump): record a per-frame RGB histogram for scene-boundary training
Add frames/rgb_hist to the embedding dump: a normalised 32-bin-per-channel RGB histogram (96 floats/frame), computed from the already-decoded frame so it is nearly free and ~40 KB per film. This is the training signal for the learned scene-boundary detector — the grayscale-correlation cut detector is blind on low-contrast grades (Scarface: 1 cut in 10k frames), and the symmetric RGB-histogram delta separates X-Ray scene boundaries far better. The dump stays gallery-independent; downstream replay/training consume the histogram offline.
This commit is contained in:
@@ -5,6 +5,7 @@
|
|||||||
#include "gallery/embedder_stamp.hpp"
|
#include "gallery/embedder_stamp.hpp"
|
||||||
|
|
||||||
#include <H5Cpp.h>
|
#include <H5Cpp.h>
|
||||||
|
#include <opencv2/imgproc.hpp> // cv::calcHist for the per-frame RGB histogram
|
||||||
|
|
||||||
#include <atomic>
|
#include <atomic>
|
||||||
#include <cstdint>
|
#include <cstdint>
|
||||||
@@ -167,6 +168,12 @@ struct EmbeddingDumpFunc {
|
|||||||
fidx_.push_back(ef.source.frame_idx);
|
fidx_.push_back(ef.source.frame_idx);
|
||||||
is_cut_.push_back(ef.source.is_cut ? 1 : 0);
|
is_cut_.push_back(ef.source.is_cut ? 1 : 0);
|
||||||
is_bnd_.push_back(ef.source.is_scene_boundary ? 1 : 0);
|
is_bnd_.push_back(ef.source.is_scene_boundary ? 1 : 0);
|
||||||
|
// Per-frame normalised RGB histogram (kHistBins per channel), for offline
|
||||||
|
// training of a learned scene-boundary detector against X-Ray scene
|
||||||
|
// boundaries — the grayscale-correlation cut detector is blind on
|
||||||
|
// low-contrast grades (Scarface: 1 cut in 10k frames). Cheap and the frame
|
||||||
|
// is already decoded here; empty frame → zeros.
|
||||||
|
append_rgb_hist(ef.source.image);
|
||||||
face_off_.push_back(static_cast<int64_t>(conf_.size()));
|
face_off_.push_back(static_cast<int64_t>(conf_.size()));
|
||||||
face_cnt_.push_back(n);
|
face_cnt_.push_back(n);
|
||||||
|
|
||||||
@@ -287,6 +294,11 @@ private:
|
|||||||
write_vec(frames, "is_scene_boundary", is_bnd_, H5::PredType::NATIVE_UINT8);
|
write_vec(frames, "is_scene_boundary", is_bnd_, H5::PredType::NATIVE_UINT8);
|
||||||
write_vec(frames, "face_offset", face_off_, H5::PredType::NATIVE_INT64);
|
write_vec(frames, "face_offset", face_off_, H5::PredType::NATIVE_INT64);
|
||||||
write_vec(frames, "face_count", face_cnt_, H5::PredType::NATIVE_INT32);
|
write_vec(frames, "face_count", face_cnt_, H5::PredType::NATIVE_INT32);
|
||||||
|
// Per-frame normalised RGB histogram, kHistBins per channel laid out
|
||||||
|
// [R(kHistBins) G(kHistBins) B(kHistBins)] per row. Feeds the learned
|
||||||
|
// scene-boundary detector (see scripts/scene_detector/).
|
||||||
|
write_vec(frames, "rgb_hist", rgb_hist_, H5::PredType::NATIVE_FLOAT,
|
||||||
|
kHistBins * 3);
|
||||||
|
|
||||||
H5::Group faces = file.createGroup("faces");
|
H5::Group faces = file.createGroup("faces");
|
||||||
write_vec(faces, "embedding", emb_, H5::PredType::NATIVE_FLOAT, kEmbedDim);
|
write_vec(faces, "embedding", emb_, H5::PredType::NATIVE_FLOAT, kEmbedDim);
|
||||||
@@ -301,6 +313,29 @@ private:
|
|||||||
<< conf_.size() << " faces → " << path_ << "\n";
|
<< conf_.size() << " faces → " << path_ << "\n";
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Per-channel bin count for the RGB histogram. 32 → a 96-float row per frame,
|
||||||
|
// ~40 KB per 10k-frame film: negligible next to the embeddings.
|
||||||
|
static constexpr int kHistBins = 32;
|
||||||
|
|
||||||
|
// Append the frame's normalised per-channel RGB histogram (R,G,B blocks). An
|
||||||
|
// empty frame (EOF sentinels never reach here) yields a zero row so the array
|
||||||
|
// stays parallel to ts_.
|
||||||
|
void append_rgb_hist(const cv::Mat& img) {
|
||||||
|
const size_t base = rgb_hist_.size();
|
||||||
|
rgb_hist_.resize(base + kHistBins * 3, 0.f);
|
||||||
|
if (img.empty() || img.channels() != 3) return;
|
||||||
|
float range[] = {0.f, 256.f};
|
||||||
|
const float* ranges[] = {range};
|
||||||
|
int bins = kHistBins;
|
||||||
|
for (int c = 0; c < 3; ++c) { // OpenCV is BGR; store as B,G,R blocks
|
||||||
|
cv::Mat h;
|
||||||
|
cv::calcHist(&img, 1, &c, cv::Mat(), h, 1, &bins, ranges);
|
||||||
|
cv::normalize(h, h, 1.0, 0.0, cv::NORM_L1);
|
||||||
|
for (int b = 0; b < kHistBins; ++b)
|
||||||
|
rgb_hist_[base + c * kHistBins + b] = h.at<float>(b);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
std::string path_, movie_;
|
std::string path_, movie_;
|
||||||
EmbedderStamp stamp_;
|
EmbedderStamp stamp_;
|
||||||
DumpProvenance prov_;
|
DumpProvenance prov_;
|
||||||
@@ -315,4 +350,5 @@ private:
|
|||||||
std::vector<int32_t> face_cnt_;
|
std::vector<int32_t> face_cnt_;
|
||||||
std::vector<float> emb_, bbox_, lmk_, conf_;
|
std::vector<float> emb_, bbox_, lmk_, conf_;
|
||||||
std::vector<float> sharp_, resid_; // AR-028 quality vector, parallel to conf_
|
std::vector<float> sharp_, resid_; // AR-028 quality vector, parallel to conf_
|
||||||
|
std::vector<float> rgb_hist_; // kHistBins*3 per frame, parallel to ts_
|
||||||
};
|
};
|
||||||
|
|||||||
Reference in New Issue
Block a user