dtourolle/scene-actor-extraction
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(quality): score every face on sharpness and alignment before it is evidence

Browse Source 
Every embedding now carries the quality of the input it came from. Both
axes fall out of the AR-005 warp for free: crop_sharpness() is the
normalised Laplacian variance over the aligned 112x112, so contrast and
size cannot leak into it, and the alignment residual is the part of the
landmark deformation a similarity transform cannot explain, so in-plane
roll reads as zero and foreshortening does not.

Carried, not consumed. Nothing discounts or thresholds on either number
yet -- that is AR-030 and VR-012, and the knee has to be located against
recorded data before a gate is chosen. What this change buys is that the
data exists to locate it with.

No face is admitted unscored: the -1 sentinel is preserved rather than
clamped, and a degenerate landmark fit is counted rather than silently
dropped.

Takes the VR-001 dump to schema_version 2. The bump is not for readers,
which check for the datasets by name and replay a v1 dump unchanged; it
is so a consumer can tell "never scored" from "scored zero", which is
not recoverable from the arrays afterwards.

TRACES: AR-028, AR-029, AR-030 | VR-001 | SR-002
This commit is contained in:
dtourolle
2026-08-05 14:37:30 +02:00
parent c1155cb607
commit 777c98cb33
10 changed files with 726 additions and 17 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tests/CMakeLists.txt
+1
View File
@@ -25,6 +25,7 @@ add_executable(sae_tests
test_face_detector_node.cpp
test_scene_detector_node.cpp
test_replay_fixtures.cpp
test_embedding_dump.cpp
test_audio_signature.cpp
${CMAKE_SOURCE_DIR}/src/backends/gemm_backend.cpp
${CMAKE_SOURCE_DIR}/src/gallery/gallery_store.cpp
tests/test_embedding_dump.cpp
+169
View File
@@ -0,0 +1,169 @@
// TRACES: AR-028 | VR-001 | UT-139, UT-140, UT-141 | SR-002
//
// The other half of AR-028: the quality vector has to *survive into the dump*.
// Measuring it at inference and then leaving it in a struct that dies at the
// EmbeddedSceneFrame channel would satisfy the letter of "assessed" and none of
// the point — VR-012 sets its knees from recorded data, and what the dump does
// not carry cannot be re-litigated without re-running video on a GPU.
//
// Tier T2, but cheap: EmbeddingDumpFunc is a sink, so it can be driven directly
// with hand-built frames. No model, no video, no gallery — the embedder stamp
// tolerates an unset model path (GR-004 records it as unverifiable).
#include <catch2/catch_test_macros.hpp>
#include <catch2/matchers/catch_matchers_floating_point.hpp>
#include "config.hpp"
#include "nodes/embedding_dump_node.hpp"
#include "types.hpp"
#include <H5Cpp.h>
#include <atomic>
#include <cstdio>
#include <filesystem>
#include <string>
#include <vector>
using Catch::Matchers::WithinAbs;
namespace {
namespace fs = std::filesystem;
// Removes the file on scope exit so a failing assertion cannot leave the next
// run reading a stale dump.
struct TempDump {
fs::path path;
explicit TempDump(const char* stem)
: path(fs::temp_directory_path() / (std::string("sae_") + stem + ".h5")) {
std::remove(path.c_str());
}
~TempDump() { std::error_code ec; fs::remove(path, ec); }
};
EmbeddedSceneFrame frame_with(double ts, const std::vector<std::pair<float, float>>& quality) {
EmbeddedSceneFrame ef;
ef.source.timestamp_sec = ts;
ef.source.frame_idx = static_cast<int64_t>(ts * 5.0);
for (const auto& [sharpness, residual] : quality) {
DetectedFace f;
f.bbox = cv::Rect2f(10.f, 20.f, 60.f, 60.f);
f.confidence = 0.8f;
f.sharpness = sharpness;
f.alignment_residual = residual;
ef.faces.push_back(f);
Embedding e{};
e[0] = 1.f;
ef.embeddings.push_back(e);
}
return ef;
}
EmbeddedSceneFrame eof_frame() {
EmbeddedSceneFrame ef;
ef.source.eof = true;
return ef;
}
std::vector<float> read_face_col(const H5::H5File& f, const char* name) {
H5::DataSet ds = f.openDataSet(std::string("faces/") + name);
hsize_t n = 0;
ds.getSpace().getSimpleExtentDims(&n, nullptr);
std::vector<float> out(n);
if (n) ds.read(out.data(), H5::PredType::NATIVE_FLOAT);
return out;
}
int read_schema_version(const H5::H5File& f) {
int v = 0;
f.openAttribute("schema_version").read(H5::PredType::NATIVE_INT, &v);
return v;
}
} // namespace
TEST_CASE("the quality vector survives into the dump", "[dump][AR-028][UT-139]") {
TempDump tmp("quality_roundtrip");
Config cfg;
cfg.dump_embeddings_path = tmp.path.string();
cfg.movie_path = "synthetic";
cfg.sample_fps = 5.f;
std::atomic<bool> done{false};
{
EmbeddingDumpFunc dump(cfg, done);
dump(frame_with(0.0, {{3.25f, 0.75f}, {0.5f, 4.5f}}));
dump(frame_with(0.2, {})); // a frame with no faces
dump(frame_with(0.4, {{12.0f, 0.0f}}));
dump(eof_frame());
}
REQUIRE(done.load());
REQUIRE(fs::exists(tmp.path));
H5::H5File f(tmp.path.string(), H5F_ACC_RDONLY);
const std::vector<float> sharp = read_face_col(f, "sharpness");
const std::vector<float> resid = read_face_col(f, "alignment_residual");
const std::vector<float> conf = read_face_col(f, "confidence");
// Parallel to every other per-face array, so a consumer can index the
// quality of face i with the same slice it uses for the embedding.
REQUIRE(sharp.size() == conf.size());
REQUIRE(resid.size() == conf.size());
REQUIRE(sharp.size() == 3);
CHECK_THAT(sharp[0], WithinAbs(3.25f, 1e-6f));
CHECK_THAT(sharp[1], WithinAbs(0.50f, 1e-6f));
CHECK_THAT(sharp[2], WithinAbs(12.0f, 1e-6f));
CHECK_THAT(resid[0], WithinAbs(0.75f, 1e-6f));
CHECK_THAT(resid[1], WithinAbs(4.50f, 1e-6f));
CHECK_THAT(resid[2], WithinAbs(0.00f, 1e-6f));
}
TEST_CASE("a dump carrying the quality vector announces itself as v2", "[dump][AR-028][UT-140]") {
// The bump is not for readers — they check for the datasets by name, and a
// v1 dump still replays. It is so a consumer of the vector can tell "these
// faces were never scored" from "these faces scored zero", which is not
// recoverable from the arrays. Same reason scene_detect is an attribute.
TempDump tmp("quality_version");
Config cfg;
cfg.dump_embeddings_path = tmp.path.string();
cfg.movie_path = "synthetic";
std::atomic<bool> done{false};
{
EmbeddingDumpFunc dump(cfg, done);
dump(frame_with(0.0, {{1.f, 1.f}}));
dump(eof_frame());
}
H5::H5File f(tmp.path.string(), H5F_ACC_RDONLY);
CHECK(read_schema_version(f) == 2);
}
TEST_CASE("an unscored face keeps its sentinel through the dump", "[dump][AR-028][UT-141]") {
// The aligner admits no unscored face, so this state should be unreachable.
// The dump still must not clamp it: -1 is how a future path that skipped
// scoring would be caught, and rewriting it to 0 would hide that path behind
// a legitimate-looking "featureless crop" reading.
TempDump tmp("quality_sentinel");
Config cfg;
cfg.dump_embeddings_path = tmp.path.string();
cfg.movie_path = "synthetic";
std::atomic<bool> done{false};
{
EmbeddingDumpFunc dump(cfg, done);
dump(frame_with(0.0, {{-1.f, -1.f}}));
dump(eof_frame());
}
H5::H5File f(tmp.path.string(), H5F_ACC_RDONLY);
CHECK(read_face_col(f, "sharpness")[0] < 0.f);
CHECK(read_face_col(f, "alignment_residual")[0] < 0.f);
}
tests/test_face_utils.cpp
+275 -3
View File
@@ -1,18 +1,26 @@
// TRACES: AR-005, AR-030 | SR-002
// TRACES: AR-005, AR-028, AR-029, AR-030 | UT-130, UT-131, UT-132, UT-133, UT-134, UT-135, UT-136, UT-137, UT-138 | SR-002
//
// Unit tests for the geometric/numeric helpers in types.hpp and face_utils.hpp:
// cosine_similarity, the ArcFace 5-point alignment transform, and the alignment
// residual that AR-030 reads as its visibility measure. GPU-free, model-free.
// cosine_similarity, the ArcFace 5-point alignment transform, and the two
// measured axes of the AR-028 quality vector — the alignment residual AR-030
// reads as visibility, and the normalised Laplacian variance AR-029 reads as
// sharpness. The aligner node is exercised here too, since it is the unit that
// fills the vector in. GPU-free, model-free.
#include <catch2/catch_test_macros.hpp>
#include <catch2/matchers/catch_matchers_floating_point.hpp>
#include "face_utils.hpp"
#include "nodes/face_aligner_node.hpp"
#include "types.hpp"
#include <algorithm>
#include <array>
#include <cmath>
#include <cstdint>
#include <limits>
using Catch::Matchers::WithinAbs;
using Catch::Matchers::WithinRel;
TEST_CASE("cosine_similarity of a unit vector with itself is 1", "[types]") {
std::array<float, 512> raw{};
@@ -179,3 +187,267 @@ TEST_CASE("degenerate landmarks report not-ok rather than a residual", "[face_ut
CHECK_FALSE(a.ok);
CHECK(a.M.empty());
}
// ── AR-029: normalised Laplacian variance as a sharpness measure ──────────────
// As with AR-030 above, these assert the *properties* the measure is relied on
// for rather than magic values: no threshold is set here or anywhere else, so a
// number that drifted with the OpenCV version would still be usable — a number
// that stopped falling with blur, or started tracking exposure, would not.
namespace {
// Pink noise: broadband, but with a 1/f spectrum, so most of the energy sits at
// low frequency the way it does in a photograph. An LCG rather than cv::randu so
// the ladder is identical on every machine and every OpenCV build.
//
// **The 1/f part is load-bearing, not decoration.** On a flat-spectrum texture
// (raw white noise) the Gaussian ladder still falls, but the motion-blur ladder
// *rises* — 80.1 → 90.0 across the same kernel lengths used below. That is not a
// bug in the measure, it is what a normalised measure must do on such an input:
// a horizontal smear takes energy out of the numerator and the denominator
// together, and what survives is vertical detail that really is just as fine.
// Real crops have the low-frequency mass that keeps the denominator steady while
// the numerator falls. See the second hazard note on crop_sharpness().
cv::Mat pink(int size, uint32_t seed = 12345u) {
cv::Mat white(size, size, CV_32F);
uint32_t s = seed;
for (int y = 0; y < size; ++y)
for (int x = 0; x < size; ++x) {
s = s * 1664525u + 1013904223u;
white.at<float>(y, x) = float((s >> 16) & 0xFFFF) / 65535.f - 0.5f;
}
// Octaves weighted 1/f. The sigma=0 band keeps genuine per-pixel detail in,
// so the top of the blur ladder is a sharp image rather than an already-soft
// one.
cv::Mat acc = cv::Mat::zeros(size, size, CV_32F);
const double sigma[] = {0.0, 1.0, 2.0, 4.0, 8.0};
const double weight[] = {1.0, 2.0, 4.0, 8.0, 16.0};
for (int k = 0; k < 5; ++k) {
cv::Mat band;
if (sigma[k] <= 0.0) band = white.clone();
else cv::GaussianBlur(white, band, {0, 0}, sigma[k], sigma[k], cv::BORDER_REPLICATE);
acc += band * weight[k];
}
// Into [40, 215]: 8-bit like a real crop, with headroom at both ends so the
// contrast test can halve it without clipping.
double lo = 0, hi = 0;
cv::minMaxLoc(acc, &lo, &hi);
const double scale = 175.0 / (hi - lo);
cv::Mat out;
acc.convertTo(out, CV_8U, scale, 40.0 - lo * scale);
return out;
}
// One master pattern, resampled. So "the same face at 40 px and at 400 px" is
// literally the same image at two resolutions, and a test about source
// resolution is not accidentally a test about two different textures.
// INTER_AREA because area-averaging is what a sensor does when it images the
// same subject onto fewer pixels.
cv::Mat texture(int size) {
static const cv::Mat master = pink(448);
if (size == master.cols) return master;
cv::Mat out;
cv::resize(master, out, {size, size}, 0, 0, cv::INTER_AREA);
return out;
}
cv::Mat gaussian(const cv::Mat& in, double sigma) {
if (sigma <= 0.0) return in.clone();
cv::Mat out;
cv::GaussianBlur(in, out, {0, 0}, sigma, sigma, cv::BORDER_REPLICATE);
return out;
}
// Horizontal box smear — motion blur, which is anisotropic and so attenuates
// only one axis of the spectrum. A measure tuned to the isotropic case can
// miss it.
cv::Mat motion(const cv::Mat& in, int len) {
if (len <= 1) return in.clone();
const cv::Mat k(1, len, CV_32F, cv::Scalar(1.0 / len));
cv::Mat out;
cv::filter2D(in, out, -1, k, {-1, -1}, 0, cv::BORDER_REPLICATE);
return out;
}
// The pipeline reaches 112×112 through warpAffine's INTER_LINEAR; resize with
// the same interpolation so a test about source resolution is not really a test
// about which resampler was used.
cv::Mat to_crop(const cv::Mat& in) {
cv::Mat out;
cv::resize(in, out, {112, 112}, 0, 0, cv::INTER_LINEAR);
return out;
}
} // namespace
TEST_CASE("sharpness falls monotonically along a Gaussian blur ladder", "[face_utils][AR-029][UT-130]") {
const cv::Mat src = texture(112);
float prev = std::numeric_limits<float>::infinity();
for (double sigma : {0.0, 0.6, 1.0, 1.6, 2.5, 4.0}) {
const float s = crop_sharpness(gaussian(src, sigma));
CHECK(s < prev);
CHECK(s > 0.f);
prev = s;
}
}
TEST_CASE("sharpness falls monotonically under motion blur too", "[face_utils][AR-029][UT-131]") {
// Motion blur is the failure mode that leaves the bounding box looking
// perfectly healthy, so it is the one the measure exists for.
const cv::Mat src = texture(112);
float prev = std::numeric_limits<float>::infinity();
for (int len : {1, 3, 5, 9, 15}) {
const float s = crop_sharpness(motion(src, len));
CHECK(s < prev);
CHECK(s > 0.f);
prev = s;
}
}
TEST_CASE("contrast does not leak into sharpness", "[face_utils][AR-029][UT-132]") {
// The normalisation that makes the axis mean the same thing in a dim scene
// and a bright one. Without it VR-012 would locate a different knee per
// film — a magic number wearing a measurement's clothes (AR-024).
const cv::Mat src = texture(112);
cv::Mat dim;
src.convertTo(dim, CV_8U, 0.5, 64.0); // half contrast, re-centred, no clipping
const float a = crop_sharpness(src);
const float b = crop_sharpness(dim);
REQUIRE(a > 0.f);
CHECK_THAT(b, WithinRel(a, 0.03f));
}
TEST_CASE("the contrast invariance is exact, and 8-bit sampling is what bends it",
"[face_utils][AR-029]") {
// Worth separating because the two have different consequences. The
// algebra is exact — scaling I by α scales the Laplacian by α, so both
// variances scale by α² and cancel — which is why halving a float crop
// changes nothing at all.
//
// What deviates is the 8-bit *round trip*: halving the contrast of a stored
// crop throws away a bit of dynamic range, and the quantisation floor it
// leaves behind is broadband, so it lands almost entirely in the numerator.
// The effect scales with how little signal is left to compete with it —
// measured on this texture, a half-contrast copy reads 0.9% high when sharp,
// 24% high at sigma 1.2 and 148% high at sigma 2.5.
//
// So: a dim *and* soft crop reads sharper than it is, and that is the corner
// of the axis VR-012 has to put a knee in. Asserted here rather than left as
// a comment, because "the measure is contrast-invariant" is the kind of claim
// that gets repeated without its precondition.
cv::Mat src;
gaussian(texture(112), 1.2).convertTo(src, CV_32F);
const cv::Mat half = src * 0.5 + 64.0;
const float a = crop_sharpness(src);
const float b = crop_sharpness(half);
REQUIRE(a > 0.f);
CHECK_THAT(b, WithinRel(a, 1e-5f));
}
TEST_CASE("a small sharp face outscores a large soft one", "[face_utils][AR-029][UT-134]") {
// The register's named edge case: "size must not leak into this axis". What
// that means operationally is that the measure is not a monotone function of
// source face size — it reports the detail present in the embedder's input,
// so the ordering can and must invert when the large face is the blurred one.
//
// Small sharp: 40 px of real detail, upsampled 2.8x → finest scale ~2.8 crop px.
// Large soft: 400 px blurred at sigma 20, downsampled 3.57x → ~5.6 crop px.
const float small_sharp = crop_sharpness(to_crop(texture(40)));
const float large_soft = crop_sharpness(to_crop(gaussian(texture(400), 20.0)));
CHECK(small_sharp > large_soft);
}
TEST_CASE("a flat crop scores zero rather than dividing by zero", "[face_utils][AR-029][UT-135]") {
const cv::Mat flat(112, 112, CV_8UC3, cv::Scalar(90, 90, 90));
const float s = crop_sharpness(flat);
CHECK(std::isfinite(s));
CHECK_THAT(s, WithinAbs(0.0f, 1e-6f));
}
TEST_CASE("an empty crop is unscored, not zero", "[face_utils][AR-029][UT-136]") {
// -1 says "nothing measured this"; 0 says "measured, and there was no
// detail". Collapsing them would put unscored faces at the bottom of the
// quality axis, where VR-012 would read them as the blurriest in the film.
CHECK(crop_sharpness(cv::Mat()) < 0.f);
}
// ── AR-028: the aligner fills the vector, and loses nothing quietly ───────────
namespace {
// A face at `centre` in an image with enough texture for sharpness to be a real
// number rather than the flat-crop zero.
std::array<cv::Point2f, 5> face_at(cv::Point2f centre, float scale) {
std::array<cv::Point2f, 5> lm;
for (int i = 0; i < 5; ++i)
lm[i] = {centre.x + (kArcFaceRef[i][0] - 56.f) * scale,
centre.y + (kArcFaceRef[i][1] - 56.f) * scale};
return lm;
}
cv::Mat textured_frame(int w, int h) {
cv::Mat gray = texture(std::max(w, h));
cv::Mat bgr;
cv::cvtColor(gray(cv::Rect(0, 0, w, h)), bgr, cv::COLOR_GRAY2BGR);
return bgr;
}
} // namespace
TEST_CASE("every face the aligner admits carries a full quality vector", "[face_utils][AR-028][UT-137]") {
SceneFrame sf;
sf.source.image = textured_frame(400, 300);
for (auto c : {cv::Point2f{120.f, 100.f}, cv::Point2f{280.f, 190.f}}) {
DetectedFace f;
f.landmarks = face_at(c, 1.2f);
f.bbox = cv::Rect2f(c.x - 60.f, c.y - 60.f, 120.f, 120.f);
f.confidence = 0.9f;
sf.faces.push_back(f);
}
FaceAlignerFunc aligner;
const AlignedSceneFrame out = aligner(std::move(sf));
REQUIRE(out.faces.size() == 2);
for (const auto& f : out.faces) {
// Not "is it good quality" — that is VR-012's to decide. Only that the
// sentinel is gone, so no embedding reaches the matcher unscored.
CHECK(f.sharpness >= 0.f);
CHECK(f.alignment_residual >= 0.f);
}
CHECK(aligner.scored() == 2);
CHECK(aligner.degenerate() == 0);
}
TEST_CASE("a degenerate detection is counted, not silently vanished", "[face_utils][AR-028][UT-138]") {
// It cannot be scored — there is no crop and no fit to score — so it is
// dropped. The requirement is that the drop leaves a trace: without the
// tally, a detector emitting unusable landmark sets produces a dump that
// looks exactly like footage with fewer faces in it.
SceneFrame sf;
sf.source.image = textured_frame(400, 300);
DetectedFace good;
good.landmarks = face_at({150.f, 140.f}, 1.2f);
good.confidence = 0.9f;
sf.faces.push_back(good);
DetectedFace degenerate;
for (auto& p : degenerate.landmarks) p = {200.f, 200.f};
degenerate.confidence = 0.9f;
sf.faces.push_back(degenerate);
FaceAlignerFunc aligner;
const AlignedSceneFrame out = aligner(std::move(sf));
CHECK(out.faces.size() == 1);
CHECK(out.crops.size() == 1);
CHECK(aligner.scored() == 1);
CHECK(aligner.degenerate() == 1);
}