feat: gallery build report
GR-003 — the calibration fit already computed per-actor dedup counts, how many actors are eligible for positive pairs, and a 200-bin histogram of the intra and inter distributions, then discarded all of it to stderr. Nothing persisted, so nobody could audit whether a gallery was any good. The report is written alongside the gallery at build time. That is the right moment: the matcher fits the same sigmoid at analysis time, but by then the answer is per-run and nobody is looking, whereas build time is when a gallery's quality is actually decided. What it surfaces, in order of usefulness: - actors with no usable image — a silent recall ceiling, since the pipeline can never name them and nothing else says why - actors below the positive-pair threshold — not broken, so nothing complains; they just quietly weaken every threshold downstream - near-duplicate references removed, per actor and total - the fitted calibration AND the two distributions behind it That last one is the point. Every threshold in the pipeline is expressed in the probability space this sigmoid defines, so if the distributions overlap heavily the calibration is weak and every downstream decision inherits it — while the gallery still looks fine from the outside. The gallery-derived prior, intra/(intra+inter), is computed and reported but the shipped default of 0.5 is deliberately left alone. The spec records these as disagreeing; now the real value is visible, so the decision can be made on evidence rather than argument. Three tests: a zero-image actor is visible in the report, an under-referenced actor is counted, and the report round-trips through JSON. Suite: 95 cases, 6142 assertions. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com> TRACES: GR-003 | SR-001
This commit is contained in:
@@ -168,3 +168,96 @@ TEST_CASE("calibrate_gallery_cached treats hash=0 as always-recompute", "[calibr
|
||||
CHECK(recomputed);
|
||||
CHECK(cal.valid);
|
||||
}
|
||||
|
||||
// ── GR-003 — the build report ────────────────────────────────────────────────
|
||||
#include "gallery/gallery_report.hpp"
|
||||
|
||||
namespace {
|
||||
// A unit vector on one axis. Distinct axes are orthogonal, which is unrealistic
|
||||
// as a same-actor cluster but irrelevant here: these tests count actors, they do
|
||||
// not assess fit quality.
|
||||
Embedding unit_axis(int slot) {
|
||||
Embedding e{};
|
||||
e[slot % 512] = 1.0f;
|
||||
return e;
|
||||
}
|
||||
} // namespace
|
||||
|
||||
TEST_CASE("report surfaces actors that can never be recognised", "[report][GR-003]") {
|
||||
// An actor with no usable image is a silent recall ceiling: the pipeline
|
||||
// will never name them, and nothing in the gallery says why. This is the
|
||||
// single most useful number in the report.
|
||||
ActorGallery g;
|
||||
for (int a = 0; a < 3; ++a) {
|
||||
ActorGallery::Actor act;
|
||||
act.name = "actor" + std::to_string(a);
|
||||
if (a != 1) // actor1 gets nothing
|
||||
for (int i = 0; i < 6; ++i) act.embeddings.push_back(unit_axis(a * 10 + i));
|
||||
g.actors.push_back(std::move(act));
|
||||
}
|
||||
|
||||
std::vector<Embedding> flat;
|
||||
std::vector<int> flat_actor;
|
||||
for (int a = 0; a < static_cast<int>(g.actors.size()); ++a)
|
||||
for (const auto& e : g.actors[a].embeddings) { flat.push_back(e); flat_actor.push_back(a); }
|
||||
|
||||
GalleryCalibrationStats stats;
|
||||
GalleryCalibration cal = calibrate_gallery(flat, flat_actor, &stats);
|
||||
GalleryReport r = build_gallery_report(g, cal, stats);
|
||||
|
||||
// An actor present in the gallery with no embeddings is counted as
|
||||
// in-gallery but contributes nothing; the zero-usable list is populated
|
||||
// from the build audit, which a stored gallery cannot supply.
|
||||
CHECK(r.actors_in_gallery == 3);
|
||||
CHECK(r.actors[1].references == 0);
|
||||
}
|
||||
|
||||
TEST_CASE("report surfaces actors too thin to calibrate on", "[report][GR-003]") {
|
||||
// Below the positive-pair threshold an actor contributes nothing to the
|
||||
// intra-class side of the fit. They are not broken, so nothing complains —
|
||||
// they just quietly weaken every threshold downstream.
|
||||
ActorGallery g;
|
||||
for (int a = 0; a < 2; ++a) {
|
||||
ActorGallery::Actor act;
|
||||
act.name = "actor" + std::to_string(a);
|
||||
const int n = (a == 0) ? 6 : 2; // actor1 is under-referenced
|
||||
for (int i = 0; i < n; ++i) act.embeddings.push_back(unit_axis(a * 10 + i));
|
||||
g.actors.push_back(std::move(act));
|
||||
}
|
||||
|
||||
std::vector<Embedding> flat;
|
||||
std::vector<int> flat_actor;
|
||||
for (int a = 0; a < static_cast<int>(g.actors.size()); ++a)
|
||||
for (const auto& e : g.actors[a].embeddings) { flat.push_back(e); flat_actor.push_back(a); }
|
||||
|
||||
GalleryCalibrationStats stats;
|
||||
GalleryCalibration cal = calibrate_gallery(flat, flat_actor, &stats);
|
||||
GalleryReport r = build_gallery_report(g, cal, stats);
|
||||
|
||||
CHECK(r.actors_below_positive_threshold >= 1);
|
||||
}
|
||||
|
||||
TEST_CASE("report round-trips", "[report][GR-003]") {
|
||||
ActorGallery g;
|
||||
ActorGallery::Actor act;
|
||||
act.name = "solo";
|
||||
for (int i = 0; i < 6; ++i) act.embeddings.push_back(unit_axis(i));
|
||||
g.actors.push_back(std::move(act));
|
||||
|
||||
std::vector<Embedding> flat;
|
||||
std::vector<int> flat_actor;
|
||||
for (const auto& e : g.actors[0].embeddings) { flat.push_back(e); flat_actor.push_back(0); }
|
||||
|
||||
GalleryCalibrationStats stats;
|
||||
GalleryCalibration cal = calibrate_gallery(flat, flat_actor, &stats);
|
||||
GalleryReport r = build_gallery_report(g, cal, stats);
|
||||
|
||||
const std::string path = "/tmp/gr003_roundtrip.report.json";
|
||||
save_gallery_report(path, r);
|
||||
GalleryReport back = load_gallery_report(path);
|
||||
|
||||
CHECK(back.actors_in_gallery == r.actors_in_gallery);
|
||||
CHECK(back.actors_below_positive_threshold == r.actors_below_positive_threshold);
|
||||
CHECK(back.calib_a == r.calib_a);
|
||||
std::remove(path.c_str());
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user