A gallery is only valid for the embedder that produced its vectors. Cosine
similarities across models are meaningless but *look* plausible, so the mistake
is silent and every measurement taken afterwards is suspect. Stamp the embedder
identity into the gallery at build; verify it at every load.
The stamp is the model file's basename plus the SHA-256 of its bytes (plus
embed_dim). The hash decides, the name explains. A name alone is a promise
rather than a fact — models get re-exported and overwritten in place under an
unchanged filename, which is exactly the case where the weights differ and
nothing else does. A hash alone is correct but unactionable in an error message.
SHA-256 is derived from the artefact, needs no registry kept current, and costs
~0.1s for a 250MB ONNX, memoised per process.
Mismatch is a hard error in every mode, with no bypass, naming both sides.
Unstamped legacy galleries warn loudly and proceed: unknown is not known-bad,
and hard-failing every pre-existing gallery would turn the check into something
people disable rather than trust. --require-gallery-stamp (or
SAE_REQUIRE_GALLERY_STAMP=1, which propagates to subprocesses) promotes that to
a hard error — the mode measurement work should run in. scripts/stamp_gallery.py
re-binds an existing gallery with no re-embedding, so "warn" is a cheap state to
leave rather than a permanent one.
Embedding dumps carry the same stamp: a replay has no live embedder, so the dump
is the embedder as far as the gallery is concerned. Derived galleries inherit
their source's stamp; --merge and the JSON gallery merge check before writing,
since one file holding two embedding spaces cannot be untangled afterwards.
Verified in: scene_analyze, scene_preview, the sae_kpn matcher binding,
replay.py, optimize.py (once per film at startup, before the first evaluation),
movienet_eval.py and both merge paths.
Stamp logic lives in src/gallery/embedder_stamp.{hpp,cpp} and its Python twin
scripts/sae_stamp.py, kept dependency-light so replay subprocesses do not pay
sae_gallery's requests/Pillow import to ask whether two models match.
Tests: 12 new cases in test_gallery_store.cpp covering the comparison logic,
both round trips, and the SHA-256 vectors that guarantee the C++ and hashlib
stamps agree. No ONNX or GPU required.
Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
125 lines
4.7 KiB
C++
125 lines
4.7 KiB
C++
#include "gallery_builder.hpp"
|
|
#include "config.hpp"
|
|
#include "embedder_stamp.hpp"
|
|
#include "face_utils.hpp"
|
|
#include "inference/face_detector.hpp"
|
|
#include "inference/face_embedder.hpp"
|
|
|
|
#include <opencv2/imgcodecs.hpp>
|
|
|
|
#include <algorithm>
|
|
#include <filesystem>
|
|
#include <iostream>
|
|
#include <stdexcept>
|
|
#include <string>
|
|
|
|
namespace fs = std::filesystem;
|
|
|
|
// ── Parse "nm0000093_Brad_Pitt" → ("nm0000093", "Brad Pitt") ─────────────────
|
|
static std::pair<std::string, std::string> parse_dir_name(const std::string& dirname) {
|
|
auto pos = dirname.find('_');
|
|
if (pos == std::string::npos) return {dirname, dirname};
|
|
|
|
std::string imdb_id = dirname.substr(0, pos);
|
|
std::string raw = dirname.substr(pos + 1);
|
|
std::string name;
|
|
name.reserve(raw.size());
|
|
for (char c : raw)
|
|
name += (c == '_' ? ' ' : c);
|
|
return {imdb_id, name};
|
|
}
|
|
|
|
// ── Public API ────────────────────────────────────────────────────────────────
|
|
|
|
ActorGallery build_gallery(const BuildConfig& cfg) {
|
|
Config icfg;
|
|
icfg.detector_model = cfg.detector_model;
|
|
icfg.arcface_model = cfg.arcface_model;
|
|
icfg.detector_conf = cfg.detector_conf;
|
|
icfg.detector_nms = cfg.detector_nms;
|
|
auto decoder = make_face_detector(icfg);
|
|
auto arcface = make_face_embedder(icfg);
|
|
|
|
ActorGallery gallery;
|
|
|
|
/// TRACES: GR-004 | SR-001
|
|
// Stamp before the first embedding exists, so there is no window in which a
|
|
// gallery holds vectors without recording what produced them.
|
|
gallery.embedder = make_embedder_stamp(cfg.arcface_model);
|
|
std::cerr << "[build_gallery] embedder: " << gallery.embedder.describe() << "\n";
|
|
|
|
for (const auto& actor_dir : fs::directory_iterator(cfg.gallery_root)) {
|
|
if (!actor_dir.is_directory()) continue;
|
|
|
|
auto [imdb_id, name] = parse_dir_name(actor_dir.path().filename().string());
|
|
std::cerr << "[build_gallery] " << name << " (" << imdb_id << ")\n";
|
|
|
|
ActorGallery::Actor actor;
|
|
actor.imdb_id = imdb_id;
|
|
actor.name = name;
|
|
|
|
static const std::vector<std::string> kExts{".jpg", ".jpeg", ".png", ".webp"};
|
|
for (const auto& img_file : fs::directory_iterator(actor_dir.path())) {
|
|
if (!img_file.is_regular_file()) continue;
|
|
std::string ext = img_file.path().extension().string();
|
|
std::transform(ext.begin(), ext.end(), ext.begin(), ::tolower);
|
|
if (std::find(kExts.begin(), kExts.end(), ext) == kExts.end()) continue;
|
|
|
|
cv::Mat img = cv::imread(img_file.path().string());
|
|
if (img.empty()) {
|
|
std::cerr << " [skip] cannot read " << img_file.path().filename() << "\n";
|
|
continue;
|
|
}
|
|
|
|
if (cfg.max_side > 0) {
|
|
const int big = std::max(img.cols, img.rows);
|
|
if (big > cfg.max_side) {
|
|
const double s = static_cast<double>(cfg.max_side) / big;
|
|
cv::resize(img, img, {}, s, s, cv::INTER_AREA);
|
|
}
|
|
}
|
|
|
|
auto faces = decoder->detect(img);
|
|
|
|
if (faces.empty()) {
|
|
std::cerr << " [skip] no face: " << img_file.path().filename() << "\n";
|
|
continue;
|
|
}
|
|
if (faces.size() > 1) {
|
|
std::cerr << " [warn] " << faces.size() << " faces, using highest confidence: "
|
|
<< img_file.path().filename() << "\n";
|
|
}
|
|
|
|
const auto& best = *std::max_element(
|
|
faces.begin(), faces.end(),
|
|
[](const DetectedFace& a, const DetectedFace& b) {
|
|
return a.confidence < b.confidence;
|
|
});
|
|
|
|
cv::Mat crop = align_face(img, best.landmarks);
|
|
if (crop.empty()) {
|
|
std::cerr << " [skip] alignment failed: " << img_file.path().filename() << "\n";
|
|
continue;
|
|
}
|
|
|
|
Embedding emb = arcface->embed_one(crop);
|
|
actor.embeddings.push_back(emb);
|
|
actor.source_images.push_back(img_file.path().filename().string());
|
|
|
|
std::cerr << " [ok] " << img_file.path().filename()
|
|
<< " conf=" << best.confidence << "\n";
|
|
}
|
|
|
|
if (actor.embeddings.empty()) {
|
|
std::cerr << " [warn] no valid embeddings for " << name << " — skipped\n";
|
|
continue;
|
|
}
|
|
|
|
std::cerr << " → " << actor.embeddings.size() << " embeddings\n";
|
|
gallery.actors.push_back(std::move(actor));
|
|
}
|
|
|
|
std::cerr << "[build_gallery] total: " << gallery.actors.size() << " actors\n";
|
|
return gallery;
|
|
}
|