diff --git a/scripts/optimizer/SCHEMA.md b/scripts/optimizer/SCHEMA.md index e710f79..2f9fc92 100644 --- a/scripts/optimizer/SCHEMA.md +++ b/scripts/optimizer/SCHEMA.md @@ -1,4 +1,4 @@ -# Embedding-dump HDF5 schema (v1) +# Embedding-dump HDF5 schema (v2) One file per analysed title. Captures the pipeline state at the `EmbeddedSceneFrame` channel — i.e. after decode → detect → align → embed, but **before** tracking and @@ -18,7 +18,7 @@ variable-length HDF5 types and reads straight into numpy. ``` / (root) attrs: - schema_version : int = 1 + schema_version : int = 2 embed_dim : int = 512 # ── what produced the vectors (GR-004) ────────────────────────────────── @@ -60,6 +60,12 @@ variable-length HDF5 types and reads straight into numpy. landmarks : float32 [N, 10] 5 (x,y) pairs, SCRFD/ArcFace order, same space as bbox confidence : float32 [N] detector confidence + + # ── embedding input quality (AR-028), v2 onward ───────────────────────── + sharpness : float32 [N] normalised Laplacian variance on the + 112x112 aligned crop (AR-029) + alignment_residual : float32 [N] RMS landmark misfit in canonical px, + after the AR-005 similarity fit (AR-030) ``` `F` = number of sampled frames, `N` = total faces (= sum of face_count). @@ -87,8 +93,9 @@ exactly the fact the committed fixtures needed to state.) Reading is by name with a default or an existence check on **both** sides — `replay.py` (`f.attrs.get(...)`) and `read_dump_provenance()` in `src/nodes/embedding_dump_node.hpp` (`attrExists`). So the attributes are -additive and `schema_version` stays 1: a pre-VR-010 dump still loads, and a -post-VR-010 dump still reads on old code. +additive and did not themselves move `schema_version` off 1: a pre-VR-010 dump +still loads, and a post-VR-010 dump still reads on old code. (AR-028 later took +it to 2 by adding *datasets* — see below.) A missing attribute means **unknown**, never a default value. Substituting `detector_conf = 0.5` for a dump that does not say so manufactures the provenance @@ -97,6 +104,40 @@ provenance is unknown is worse than no fixture, because it will be trusted."* The committed `tests/fixtures/dumps/*.h5` predate VR-010 and carry none of these attributes; re-dump to bind them, as with GR-004. +## Embedding input quality (AR-028) — and why this one bumps the version + +`sharpness` and `alignment_residual` are two of the three AR-028 quality axes, +written beside the embedding they describe. **The third axis, size, is already +here**: it is `bbox`, scaled by `bbox_upscale` to reach the original resolution +AR-002 thresholds in. It is not duplicated into a third column, because that +would put the same quantity in two coordinate spaces inside one file — the trap +the `bbox_upscale` note below records — and the copy is the one that drifts. + +The vector is **carried, not consumed**. Nothing in the pipeline thresholds or +discounts on it yet; VR-012 locates the knees from these columns, which is only +possible if they were recorded at inference. A study cannot recover how sharp a +face was from an embedding, any more than it can recover which model produced it. + +**This is the change that bumps `schema_version` to 2**, where VR-010's +attributes did not. The rule is unchanged — a bump is for the *datasets* — and +so is the reason behind it. Readers are fine either way: `replay.py` and +`test_replay_fixtures.cpp` take these datasets by name with an existence check, +so a v1 dump still replays and loses only what it never had. The version exists +for a *consumer of the quality vector*, which otherwise cannot tell **"this +film's faces were never scored"** from **"this film's faces scored zero"** — +sharpness 0 is a real reading, meaning a featureless crop. That is the same +distinction `scene_detect` exists to make, and it is equally unrecoverable from +the arrays. + +A v1 dump reports the vector as **unknown, never as a default** — `load_frames` +omits the keys rather than filling zeros, and the C++ side leaves the +`DetectedFace` fields at their -1 "unscored" sentinel. Re-dump to acquire it; +there is no migration, for the same reason GR-004 has none. + +> The committed `tests/fixtures/dumps/*.h5` are v1 and carry no quality vector. +> Re-dumping needs a GPU host (`scripts/make_fixtures.sh`), so until that runs, +> anything driven from the fixtures sees the sentinel. + ## Model binding (GR-004) `embedder_model` / `embedder_sha256` record which embedder produced every vector @@ -144,3 +185,7 @@ one never received. Two further reasons: original resolution (see above). - A frame with no faces has `face_count == 0` (still gets a row, so timestamps stay dense). - EOF sentinel frames are NOT written. +- v2 onward: `sharpness` and `alignment_residual` are `[N]`, parallel to + `confidence`, so face *i*'s quality indexes with the same slice as its + embedding. Both are `>= 0` for any face the aligner admitted; a negative value + means unscored and must never be read as a quality. diff --git a/scripts/optimizer/replay.py b/scripts/optimizer/replay.py index 57b7440..6415841 100644 --- a/scripts/optimizer/replay.py +++ b/scripts/optimizer/replay.py @@ -63,6 +63,14 @@ def load_frames(dump_path: str, min_conf: float = 0.0): bbox = f["faces/bbox"][:] lmk = f["faces/landmarks"][:] conf = f["faces/confidence"][:] + # TRACES: AR-028 | SR-002 + # The quality vector, present from schema v2. A v1 dump predates AR-028 + # and simply has no such dataset — read as absent, never as a default, + # so a face from an old dump stays at the C++ -1 "unscored" sentinel + # rather than acquiring a fabricated sharpness of 0 (which is a real + # value on this axis, meaning a featureless crop). + qual = {k: f[f"faces/{k}"][:] for k in ("sharpness", "alignment_residual") + if f"faces/{k}" in f} movie = f.attrs.get("movie", "") fps = float(f.attrs.get("sample_fps", 1.0)) @@ -81,6 +89,8 @@ def load_frames(dump_path: str, min_conf: float = 0.0): "landmarks": np.ascontiguousarray(lmk[keep][sel], dtype=np.float32), "confidence": np.ascontiguousarray(c[sel], dtype=np.float32), "embeddings": np.ascontiguousarray(emb[keep][sel], dtype=np.float32), + **{k: np.ascontiguousarray(v[keep][sel], dtype=np.float32) + for k, v in qual.items()}, }) else: frames.append({ @@ -90,6 +100,8 @@ def load_frames(dump_path: str, min_conf: float = 0.0): "landmarks": np.ascontiguousarray(lmk[keep], dtype=np.float32), "confidence": c, "embeddings": np.ascontiguousarray(emb[keep], dtype=np.float32), + **{k: np.ascontiguousarray(v[keep], dtype=np.float32) + for k, v in qual.items()}, }) last_ts = float(ts[-1]) if len(ts) else 0.0 frames.append({"timestamp_sec": last_ts, "eof": True}) diff --git a/src/face_utils.hpp b/src/face_utils.hpp index c45d87b..ad6812b 100644 --- a/src/face_utils.hpp +++ b/src/face_utils.hpp @@ -1,5 +1,5 @@ #pragma once -/// TRACES: AR-005, AR-030 | SR-002 +/// TRACES: AR-005, AR-029, AR-030 | SR-002 #include "types.hpp" #include @@ -143,6 +143,96 @@ inline cv::Mat align_face(const cv::Mat& img, return crop; } +// ── crop_sharpness ──────────────────────────────────────────────────────────── +/// TRACES: AR-029 | SR-002 +// +// Normalised variance of the Laplacian over the aligned 112×112 crop: the AR-029 +// sharpness axis. Returns -1 for an empty crop (unscored), matching the +// DetectedFace sentinel. +// +// sharpness = Var(∇²I) / Var(I) +// +// Two normalisations, each removing a quantity that would otherwise be read as +// blur: +// +// - **Divided by the image variance, so contrast cannot leak in.** Scaling +// intensity by α scales the Laplacian by α too, so both variances scale by α² +// and the ratio is unchanged. A raw Var(∇²I) — the textbook measure — instead +// falls with exposure, so a dim scene reads as soft and a graded-up one as +// sharp. VR-012 has to locate one knee across films whose grading differs by +// more than their focus does; an uncalibrated measure would put the knee in a +// different place per film, which is the AR-024 failure in another metric. +// - **Measured on the aligned crop, so size cannot leak in.** The destination +// frame is fixed at 112×112 (AR-002 owns size, and double-counting it here +// would make every small face read as blurred). What the ratio reports is the +// detail actually present in the embedder's input — so a small sharp face can +// and does outscore a large soft one. That is the claim; it is *not* a claim +// of invariance to source resolution, because a 40 px face warped up to 112 +// genuinely carries less detail, and hiding that would defeat the point. +// +// Frequency-domain reading of why the blur ladder is monotone: with +// Var(∇²I) = ∫|ω|⁴|F(ω)|² and Var(I) = ∫|F(ω)|², the ratio is E[|ω|⁴] under the +// image's own spectral measure. Gaussian blur multiplies that measure by +// e^{-σ²|ω|²}, concentrating it at low |ω|, so the expectation falls strictly +// with σ. It is a property of the construction, not a fitted behaviour. +// +// **Three known hazards, for VR-012 to check rather than for a threshold to +// absorb.** All are recorded here because they are properties of the measure, +// visible in the dumped distribution, and neither should be papered over by a +// correction chosen before that distribution has been looked at. +// +// 1. **Border fill.** `align_face` warps with BORDER_CONSTANT, so a face +// crossing the frame edge brings a hard black step into the crop, and a +// step edge is high-frequency. The normalisation blunts it — the fill +// inflates Var(I) as well as Var(∇²I) — but does not remove it, so +// heavily-cropped faces may read sharper than they are. The fix is either a +// validity mask or a different border mode, and the second changes what the +// embedder is fed (AR-011). +// +// 2. **The contrast invariance is exact in the algebra and approximate in +// 8 bits.** Scaling I by α cancels exactly; what does not cancel is the +// quantisation floor of a stored crop, which is broadband and so lands in +// the numerator. It matters only where there is little signal left to +// compete with it: on the AR-029 test texture a half-contrast copy reads +// 0.9% high when sharp, 24% high at sigma 1.2 and 148% high at sigma 2.5. +// A crop that is both **dim and soft therefore reads sharper than it is** — +// the low corner of the axis, and the corner VR-012 must put a knee in. +// +// 3. **It reports where the energy sits, not how much there is.** A crop whose +// energy is *already* concentrated at high frequency — dense film grain, +// a face against foliage — loses numerator and denominator together under +// blur, so the ratio moves less than the damage does. Measured on a +// flat-spectrum synthetic, an anisotropic (motion) smear even makes it rise, +// because the surviving perpendicular detail really is as fine as before. +// Natural crops have the low-frequency mass that keeps the denominator +// steady, and on those both ladders fall (see the AR-029 tests, which use a +// 1/f texture for exactly this reason). The same property means the axis +// conflates focus with intrinsic texture — a bearded face outscores a smooth +// one at equal focus — which is true of every no-reference sharpness measure +// and is why AR-028 carries the number instead of thresholding on it. +inline float crop_sharpness(const cv::Mat& crop) { + if (crop.empty()) return -1.f; + + cv::Mat gray; + if (crop.channels() == 3) cv::cvtColor(crop, gray, cv::COLOR_BGR2GRAY); + else gray = crop; + + cv::Mat lap; + cv::Laplacian(gray, lap, CV_32F, 3); + + cv::Scalar mean_i, sd_i, mean_l, sd_l; + cv::meanStdDev(gray, mean_i, sd_i); + cv::meanStdDev(lap, mean_l, sd_l); + + const double var_i = sd_i[0] * sd_i[0]; + // A flat crop has no detail to be sharp or soft about, and the ratio is 0/0. + // Zero is the honest answer and keeps the axis finite; -1 would claim the + // face was never scored, which is a different fact. + if (var_i < 1e-6) return 0.f; + + return static_cast((sd_l[0] * sd_l[0]) / var_i); +} + // ── enhance_for_retry ──────────────────────────────────────────────────────── // Used when initial face detection finds nothing. Pads the image by 50% // (border-replicated, so the detector doesn't see a hard edge) and applies diff --git a/src/kpn_bindings.cpp b/src/kpn_bindings.cpp index f518009..fbc6bd6 100644 --- a/src/kpn_bindings.cpp +++ b/src/kpn_bindings.cpp @@ -28,6 +28,7 @@ #include #include +#include #include namespace nb = nanobind; @@ -69,6 +70,22 @@ template<> struct PythonConverter { auto conf = nb::cast, nb::c_contig>>(d["confidence"]); auto emb = nb::cast, nb::c_contig>>(d["embeddings"]); + // AR-028 quality vector. Optional because a v1 dump predates it — absent + // leaves the DetectedFace sentinels at -1, which reads as *unscored*, not + // as a bad face. There is no live aligner on this path to recompute it: + // the replay starts at the embedded-frame channel, so what the dump does + // not carry is genuinely gone. + // + // Held in named locals, like the four above, because the ndarray owns the + // reference that keeps the buffer alive — reading .data() off a temporary + // would leave the pointer dangling at the end of the statement. + using FloatCol = nb::ndarray, nb::c_contig>; + std::optional sharp_col, resid_col; + if (d.contains("sharpness")) sharp_col = nb::cast(d["sharpness"]); + if (d.contains("alignment_residual")) resid_col = nb::cast(d["alignment_residual"]); + const float* sp = sharp_col ? sharp_col->data() : nullptr; + const float* rp = resid_col ? resid_col->data() : nullptr; + const size_t n = bbox.shape(0); ef.faces.reserve(n); ef.embeddings.reserve(n); @@ -82,6 +99,8 @@ template<> struct PythonConverter { for (int k = 0; k < 5; ++k) f.landmarks[k] = cv::Point2f(lp[i*10 + k*2], lp[i*10 + k*2 + 1]); f.confidence = cp[i]; + if (sp) f.sharpness = sp[i]; + if (rp) f.alignment_residual = rp[i]; ef.faces.push_back(f); Embedding e; diff --git a/src/nodes/embedding_dump_node.hpp b/src/nodes/embedding_dump_node.hpp index 969865d..348bf08 100644 --- a/src/nodes/embedding_dump_node.hpp +++ b/src/nodes/embedding_dump_node.hpp @@ -1,5 +1,5 @@ #pragma once -/// TRACES: VR-001, VR-010 | PR-002 +/// TRACES: AR-028 | VR-001, VR-010 | PR-002 #include "types.hpp" #include "config.hpp" #include "gallery/embedder_stamp.hpp" @@ -178,6 +178,16 @@ struct EmbeddingDumpFunc { lmk_.push_back(f.landmarks[k].y); } conf_.push_back(f.confidence); + /// TRACES: AR-028 | SR-002 + // The quality vector, carried rather than consumed: written beside + // the embedding it describes so VR-012 can locate its knees against + // recorded data instead of by re-running video. Size is the third + // axis and is already here as bbox + the bbox_upscale attribute. + // Both are -1 only if a face reached the dump unscored, which the + // aligner does not allow — the sentinel is preserved rather than + // clamped so that a future path which did would be visible. + sharp_.push_back(f.sharpness); + resid_.push_back(f.alignment_residual); const auto& e = ef.embeddings[i]; emb_.insert(emb_.end(), e.begin(), e.end()); } @@ -194,11 +204,20 @@ struct EmbeddingDumpFunc { } private: - // Root attributes are additive: schema_version stays 1 across VR-010, because + // Root attributes are additive: schema_version stayed 1 across VR-010, because // every reader takes attributes by name with a default (replay.py) or an // existence check (read_dump_provenance), so an old dump loses nothing and a // new dump breaks nothing. A bump is for a change to the *datasets*. - static constexpr int kSchemaVersion = 1; + // + // v2 is that change: AR-028 adds faces/sharpness and faces/alignment_residual. + // The bump is not about readers — those check for the datasets by name, and a + // v1 dump still replays. It is so a *consumer of the quality vector* can tell + // "this film's faces were never scored" from "this film's faces scored zero", + // which is the same distinction scene_detect exists to make and is likewise + // not recoverable from the arrays. A v1 dump reports the vector as unknown; + // re-dump to acquire it, since nobody can assert after the fact how sharp a + // face was. + static constexpr int kSchemaVersion = 2; static constexpr int kEmbedDim = 512; static std::string basename_of(const std::string& path) { @@ -274,6 +293,9 @@ private: write_vec(faces, "bbox", bbox_, H5::PredType::NATIVE_FLOAT, 4); write_vec(faces, "landmarks", lmk_, H5::PredType::NATIVE_FLOAT, 10); write_vec(faces, "confidence", conf_, H5::PredType::NATIVE_FLOAT); + /// TRACES: AR-028 | SR-002 + write_vec(faces, "sharpness", sharp_, H5::PredType::NATIVE_FLOAT); + write_vec(faces, "alignment_residual", resid_, H5::PredType::NATIVE_FLOAT); std::cerr << "[embedding_dump] wrote " << ts_.size() << " frames, " << conf_.size() << " faces → " << path_ << "\n"; @@ -292,4 +314,5 @@ private: std::vector face_off_; std::vector face_cnt_; std::vector emb_, bbox_, lmk_, conf_; + std::vector sharp_, resid_; // AR-028 quality vector, parallel to conf_ }; diff --git a/src/nodes/face_aligner_node.hpp b/src/nodes/face_aligner_node.hpp index f14cda4..afc6fef 100644 --- a/src/nodes/face_aligner_node.hpp +++ b/src/nodes/face_aligner_node.hpp @@ -1,25 +1,55 @@ #pragma once #include "face_utils.hpp" +#include #include // ── FaceAlignerFunc ─────────────────────────────────────────────────────────── -/// TRACES: AR-005, AR-030 | SR-002 +/// TRACES: AR-005, AR-028, AR-029, AR-030 | SR-002 /// // KPN node: applies a 5-point similarity transform to each detected face, // producing a 112×112 BGR crop suitable for ArcFace inference. // // Alignment is an Umeyama least-squares fit over all five landmarks (AR-005), // not a robust one: a RANSAC fit discards the very landmarks AR-030 reads. -// Degenerate detections (where the fit fails) are dropped from the output -// vectors. The fit's residual is the AR-030 visibility measure and comes free, -// since the warp needs the transform anyway. +// +// This is also where the AR-028 quality vector is filled in, because this is +// where the inputs to it already exist: +// +// - **Visibility** (AR-030) is the fit's residual, and is genuinely free — the +// transform is computed for the warp regardless, and the residual is what +// that fit could not explain. +// - **Sharpness** (AR-029) is measured on the crop this node just produced, +// which is the only place it *can* be measured: the aligned canvas is what +// makes the number scale-normalised, and downstream of the embedder the crop +// is only forwarded for debug rendering. It is not free — 33 us per face +// single-threaded (cvtColor, one Laplacian, two meanStdDev over 112x112) — +// but it is two orders below the embedder inference it qualifies, and it +// runs per face rather than per frame, so a landscape shot costs nothing. +// +// Size, the third axis, is `bbox` and needs no work here. +// +// No face is admitted unscored: every face in the output carries both numbers, +// so a negative value downstream is a bug rather than a poor-quality face. +// Nothing is dropped or discounted on quality — that is AR-030's discount and +// VR-012's knee, both still open. +// +// Degenerate detections (where the fit fails) cannot be scored, since there is +// no crop and no residual to score, and are therefore dropped — but they are +// **counted**, not silently discarded. A nonzero tally means the detector is +// emitting landmark sets the aligner cannot use, which is a fact about the +// detector; losing it leaves a hole in the dump that looks like footage with +// no faces in it. struct FaceAlignerFunc { static constexpr std::string_view label() { return "face_aligner"; } AlignedSceneFrame operator()(SceneFrame sf) { - if (sf.source.eof || sf.faces.empty()) + if (sf.source.eof) { + report(); + return {std::move(sf.source), {}, {}}; + } + if (sf.faces.empty()) return {std::move(sf.source), {}, {}}; std::vector good_faces; @@ -33,15 +63,38 @@ struct FaceAlignerFunc { float residual = -1.f; cv::Mat crop = align_face(sf.source.image, face.landmarks, &residual); if (crop.empty()) { - std::cerr << "[face_aligner] degenerate detection skipped\n"; + ++degenerate_; continue; } face.alignment_residual = residual; + face.sharpness = crop_sharpness(crop); good_faces.push_back(face); crops.push_back(std::move(crop)); + ++scored_; } return {std::move(sf.source), std::move(good_faces), std::move(crops)}; } + /// Faces that carry a full quality vector, and faces the fit could not use. + uint64_t scored() const { return scored_; } + uint64_t degenerate() const { return degenerate_; } + +private: + // Reported once at EOF rather than per occurrence: a run with a systematic + // landmark problem would otherwise emit one line per face for the length of + // a film, which is how the count came to be ignored. + void report() { + if (reported_) return; + reported_ = true; + if (degenerate_) + std::cerr << "[face_aligner] " << degenerate_ << " of " + << (degenerate_ + scored_) + << " detections had a degenerate landmark fit and were dropped" + " (no crop, so no embedding and no quality vector)\n"; + } + + uint64_t scored_{0}; + uint64_t degenerate_{0}; + bool reported_{false}; }; diff --git a/src/types.hpp b/src/types.hpp index 8d41a0f..3227a2b 100644 --- a/src/types.hpp +++ b/src/types.hpp @@ -59,11 +59,36 @@ inline constexpr float kArcFaceRef[5][2] = { // Landmark order matches ArcFace convention (same as SCRFD output order): // [0] right-eye-centre [1] left-eye-centre [2] nose // [3] right-mouth [4] left-mouth +/// TRACES: AR-028 | SR-002 struct DetectedFace { cv::Rect2f bbox; std::array landmarks; float confidence{0.f}; + // ── AR-028 quality vector ──────────────────────────────────────────────── + // Three axes, kept separate and never collapsed into one scalar: they fail + // for different reasons, have different remedies, and do not earn the same + // response. Carried, not consumed — the vector travels with the face into + // the VR-001 dump so a threshold can be re-litigated against recorded data + // rather than by re-running video. + // + // **Size is the third axis and is deliberately not a field here.** It is + // `bbox`, which every consumer already has, scaled by the frame's + // `bbox_upscale` to reach the original resolution AR-002 thresholds in. + // Copying it into a second field would put the same quantity in two + // coordinate spaces inside one struct — the trap SCHEMA.md records for + // `bbox_upscale` — and the copy would be the one that drifts. + // + // Both fields below are -1 until the aligner runs, so *unscored* is + // distinguishable from *scored badly*. Nothing downstream may read a + // negative value as a quality. + + // AR-029 sharpness: normalised Laplacian variance over the aligned crop, + // dimensionless. Falls with motion blur and soft focus; invariant to + // contrast, and taken on the fixed 112×112 canvas so it cannot re-measure + // face size. See crop_sharpness() for the construction and its one hazard. + float sharpness{-1.f}; + // AR-030 visibility: RMS landmark misfit, in canonical 112×112 pixels, left // over after the best similarity fit to the ArcFace template. Rises with // out-of-plane pose and with occlusion; blind to in-plane roll and to face diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index fe8a4f2..5f47259 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -25,6 +25,7 @@ add_executable(sae_tests test_face_detector_node.cpp test_scene_detector_node.cpp test_replay_fixtures.cpp + test_embedding_dump.cpp test_audio_signature.cpp ${CMAKE_SOURCE_DIR}/src/backends/gemm_backend.cpp ${CMAKE_SOURCE_DIR}/src/gallery/gallery_store.cpp diff --git a/tests/test_embedding_dump.cpp b/tests/test_embedding_dump.cpp new file mode 100644 index 0000000..ea46a09 --- /dev/null +++ b/tests/test_embedding_dump.cpp @@ -0,0 +1,169 @@ +// TRACES: AR-028 | VR-001 | UT-139, UT-140, UT-141 | SR-002 +// +// The other half of AR-028: the quality vector has to *survive into the dump*. +// Measuring it at inference and then leaving it in a struct that dies at the +// EmbeddedSceneFrame channel would satisfy the letter of "assessed" and none of +// the point — VR-012 sets its knees from recorded data, and what the dump does +// not carry cannot be re-litigated without re-running video on a GPU. +// +// Tier T2, but cheap: EmbeddingDumpFunc is a sink, so it can be driven directly +// with hand-built frames. No model, no video, no gallery — the embedder stamp +// tolerates an unset model path (GR-004 records it as unverifiable). +#include +#include + +#include "config.hpp" +#include "nodes/embedding_dump_node.hpp" +#include "types.hpp" + +#include + +#include +#include +#include +#include +#include + +using Catch::Matchers::WithinAbs; + +namespace { + +namespace fs = std::filesystem; + +// Removes the file on scope exit so a failing assertion cannot leave the next +// run reading a stale dump. +struct TempDump { + fs::path path; + explicit TempDump(const char* stem) + : path(fs::temp_directory_path() / (std::string("sae_") + stem + ".h5")) { + std::remove(path.c_str()); + } + ~TempDump() { std::error_code ec; fs::remove(path, ec); } +}; + +EmbeddedSceneFrame frame_with(double ts, const std::vector>& quality) { + EmbeddedSceneFrame ef; + ef.source.timestamp_sec = ts; + ef.source.frame_idx = static_cast(ts * 5.0); + for (const auto& [sharpness, residual] : quality) { + DetectedFace f; + f.bbox = cv::Rect2f(10.f, 20.f, 60.f, 60.f); + f.confidence = 0.8f; + f.sharpness = sharpness; + f.alignment_residual = residual; + ef.faces.push_back(f); + + Embedding e{}; + e[0] = 1.f; + ef.embeddings.push_back(e); + } + return ef; +} + +EmbeddedSceneFrame eof_frame() { + EmbeddedSceneFrame ef; + ef.source.eof = true; + return ef; +} + +std::vector read_face_col(const H5::H5File& f, const char* name) { + H5::DataSet ds = f.openDataSet(std::string("faces/") + name); + hsize_t n = 0; + ds.getSpace().getSimpleExtentDims(&n, nullptr); + std::vector out(n); + if (n) ds.read(out.data(), H5::PredType::NATIVE_FLOAT); + return out; +} + +int read_schema_version(const H5::H5File& f) { + int v = 0; + f.openAttribute("schema_version").read(H5::PredType::NATIVE_INT, &v); + return v; +} + +} // namespace + +TEST_CASE("the quality vector survives into the dump", "[dump][AR-028][UT-139]") { + TempDump tmp("quality_roundtrip"); + + Config cfg; + cfg.dump_embeddings_path = tmp.path.string(); + cfg.movie_path = "synthetic"; + cfg.sample_fps = 5.f; + + std::atomic done{false}; + { + EmbeddingDumpFunc dump(cfg, done); + dump(frame_with(0.0, {{3.25f, 0.75f}, {0.5f, 4.5f}})); + dump(frame_with(0.2, {})); // a frame with no faces + dump(frame_with(0.4, {{12.0f, 0.0f}})); + dump(eof_frame()); + } + REQUIRE(done.load()); + REQUIRE(fs::exists(tmp.path)); + + H5::H5File f(tmp.path.string(), H5F_ACC_RDONLY); + + const std::vector sharp = read_face_col(f, "sharpness"); + const std::vector resid = read_face_col(f, "alignment_residual"); + const std::vector conf = read_face_col(f, "confidence"); + + // Parallel to every other per-face array, so a consumer can index the + // quality of face i with the same slice it uses for the embedding. + REQUIRE(sharp.size() == conf.size()); + REQUIRE(resid.size() == conf.size()); + REQUIRE(sharp.size() == 3); + + CHECK_THAT(sharp[0], WithinAbs(3.25f, 1e-6f)); + CHECK_THAT(sharp[1], WithinAbs(0.50f, 1e-6f)); + CHECK_THAT(sharp[2], WithinAbs(12.0f, 1e-6f)); + + CHECK_THAT(resid[0], WithinAbs(0.75f, 1e-6f)); + CHECK_THAT(resid[1], WithinAbs(4.50f, 1e-6f)); + CHECK_THAT(resid[2], WithinAbs(0.00f, 1e-6f)); +} + +TEST_CASE("a dump carrying the quality vector announces itself as v2", "[dump][AR-028][UT-140]") { + // The bump is not for readers — they check for the datasets by name, and a + // v1 dump still replays. It is so a consumer of the vector can tell "these + // faces were never scored" from "these faces scored zero", which is not + // recoverable from the arrays. Same reason scene_detect is an attribute. + TempDump tmp("quality_version"); + + Config cfg; + cfg.dump_embeddings_path = tmp.path.string(); + cfg.movie_path = "synthetic"; + + std::atomic done{false}; + { + EmbeddingDumpFunc dump(cfg, done); + dump(frame_with(0.0, {{1.f, 1.f}})); + dump(eof_frame()); + } + + H5::H5File f(tmp.path.string(), H5F_ACC_RDONLY); + CHECK(read_schema_version(f) == 2); +} + +TEST_CASE("an unscored face keeps its sentinel through the dump", "[dump][AR-028][UT-141]") { + // The aligner admits no unscored face, so this state should be unreachable. + // The dump still must not clamp it: -1 is how a future path that skipped + // scoring would be caught, and rewriting it to 0 would hide that path behind + // a legitimate-looking "featureless crop" reading. + TempDump tmp("quality_sentinel"); + + Config cfg; + cfg.dump_embeddings_path = tmp.path.string(); + cfg.movie_path = "synthetic"; + + std::atomic done{false}; + { + EmbeddingDumpFunc dump(cfg, done); + dump(frame_with(0.0, {{-1.f, -1.f}})); + dump(eof_frame()); + } + + H5::H5File f(tmp.path.string(), H5F_ACC_RDONLY); + CHECK(read_face_col(f, "sharpness")[0] < 0.f); + CHECK(read_face_col(f, "alignment_residual")[0] < 0.f); +} diff --git a/tests/test_face_utils.cpp b/tests/test_face_utils.cpp index 08e24bc..5bd2cfe 100644 --- a/tests/test_face_utils.cpp +++ b/tests/test_face_utils.cpp @@ -1,18 +1,26 @@ -// TRACES: AR-005, AR-030 | SR-002 +// TRACES: AR-005, AR-028, AR-029, AR-030 | UT-130, UT-131, UT-132, UT-133, UT-134, UT-135, UT-136, UT-137, UT-138 | SR-002 // // Unit tests for the geometric/numeric helpers in types.hpp and face_utils.hpp: -// cosine_similarity, the ArcFace 5-point alignment transform, and the alignment -// residual that AR-030 reads as its visibility measure. GPU-free, model-free. +// cosine_similarity, the ArcFace 5-point alignment transform, and the two +// measured axes of the AR-028 quality vector — the alignment residual AR-030 +// reads as visibility, and the normalised Laplacian variance AR-029 reads as +// sharpness. The aligner node is exercised here too, since it is the unit that +// fills the vector in. GPU-free, model-free. #include #include #include "face_utils.hpp" +#include "nodes/face_aligner_node.hpp" #include "types.hpp" +#include #include #include +#include +#include using Catch::Matchers::WithinAbs; +using Catch::Matchers::WithinRel; TEST_CASE("cosine_similarity of a unit vector with itself is 1", "[types]") { std::array raw{}; @@ -179,3 +187,267 @@ TEST_CASE("degenerate landmarks report not-ok rather than a residual", "[face_ut CHECK_FALSE(a.ok); CHECK(a.M.empty()); } + +// ── AR-029: normalised Laplacian variance as a sharpness measure ────────────── +// As with AR-030 above, these assert the *properties* the measure is relied on +// for rather than magic values: no threshold is set here or anywhere else, so a +// number that drifted with the OpenCV version would still be usable — a number +// that stopped falling with blur, or started tracking exposure, would not. + +namespace { + +// Pink noise: broadband, but with a 1/f spectrum, so most of the energy sits at +// low frequency the way it does in a photograph. An LCG rather than cv::randu so +// the ladder is identical on every machine and every OpenCV build. +// +// **The 1/f part is load-bearing, not decoration.** On a flat-spectrum texture +// (raw white noise) the Gaussian ladder still falls, but the motion-blur ladder +// *rises* — 80.1 → 90.0 across the same kernel lengths used below. That is not a +// bug in the measure, it is what a normalised measure must do on such an input: +// a horizontal smear takes energy out of the numerator and the denominator +// together, and what survives is vertical detail that really is just as fine. +// Real crops have the low-frequency mass that keeps the denominator steady while +// the numerator falls. See the second hazard note on crop_sharpness(). +cv::Mat pink(int size, uint32_t seed = 12345u) { + cv::Mat white(size, size, CV_32F); + uint32_t s = seed; + for (int y = 0; y < size; ++y) + for (int x = 0; x < size; ++x) { + s = s * 1664525u + 1013904223u; + white.at(y, x) = float((s >> 16) & 0xFFFF) / 65535.f - 0.5f; + } + + // Octaves weighted 1/f. The sigma=0 band keeps genuine per-pixel detail in, + // so the top of the blur ladder is a sharp image rather than an already-soft + // one. + cv::Mat acc = cv::Mat::zeros(size, size, CV_32F); + const double sigma[] = {0.0, 1.0, 2.0, 4.0, 8.0}; + const double weight[] = {1.0, 2.0, 4.0, 8.0, 16.0}; + for (int k = 0; k < 5; ++k) { + cv::Mat band; + if (sigma[k] <= 0.0) band = white.clone(); + else cv::GaussianBlur(white, band, {0, 0}, sigma[k], sigma[k], cv::BORDER_REPLICATE); + acc += band * weight[k]; + } + + // Into [40, 215]: 8-bit like a real crop, with headroom at both ends so the + // contrast test can halve it without clipping. + double lo = 0, hi = 0; + cv::minMaxLoc(acc, &lo, &hi); + const double scale = 175.0 / (hi - lo); + cv::Mat out; + acc.convertTo(out, CV_8U, scale, 40.0 - lo * scale); + return out; +} + +// One master pattern, resampled. So "the same face at 40 px and at 400 px" is +// literally the same image at two resolutions, and a test about source +// resolution is not accidentally a test about two different textures. +// INTER_AREA because area-averaging is what a sensor does when it images the +// same subject onto fewer pixels. +cv::Mat texture(int size) { + static const cv::Mat master = pink(448); + if (size == master.cols) return master; + cv::Mat out; + cv::resize(master, out, {size, size}, 0, 0, cv::INTER_AREA); + return out; +} + +cv::Mat gaussian(const cv::Mat& in, double sigma) { + if (sigma <= 0.0) return in.clone(); + cv::Mat out; + cv::GaussianBlur(in, out, {0, 0}, sigma, sigma, cv::BORDER_REPLICATE); + return out; +} + +// Horizontal box smear — motion blur, which is anisotropic and so attenuates +// only one axis of the spectrum. A measure tuned to the isotropic case can +// miss it. +cv::Mat motion(const cv::Mat& in, int len) { + if (len <= 1) return in.clone(); + const cv::Mat k(1, len, CV_32F, cv::Scalar(1.0 / len)); + cv::Mat out; + cv::filter2D(in, out, -1, k, {-1, -1}, 0, cv::BORDER_REPLICATE); + return out; +} + +// The pipeline reaches 112×112 through warpAffine's INTER_LINEAR; resize with +// the same interpolation so a test about source resolution is not really a test +// about which resampler was used. +cv::Mat to_crop(const cv::Mat& in) { + cv::Mat out; + cv::resize(in, out, {112, 112}, 0, 0, cv::INTER_LINEAR); + return out; +} + +} // namespace + +TEST_CASE("sharpness falls monotonically along a Gaussian blur ladder", "[face_utils][AR-029][UT-130]") { + const cv::Mat src = texture(112); + float prev = std::numeric_limits::infinity(); + for (double sigma : {0.0, 0.6, 1.0, 1.6, 2.5, 4.0}) { + const float s = crop_sharpness(gaussian(src, sigma)); + CHECK(s < prev); + CHECK(s > 0.f); + prev = s; + } +} + +TEST_CASE("sharpness falls monotonically under motion blur too", "[face_utils][AR-029][UT-131]") { + // Motion blur is the failure mode that leaves the bounding box looking + // perfectly healthy, so it is the one the measure exists for. + const cv::Mat src = texture(112); + float prev = std::numeric_limits::infinity(); + for (int len : {1, 3, 5, 9, 15}) { + const float s = crop_sharpness(motion(src, len)); + CHECK(s < prev); + CHECK(s > 0.f); + prev = s; + } +} + +TEST_CASE("contrast does not leak into sharpness", "[face_utils][AR-029][UT-132]") { + // The normalisation that makes the axis mean the same thing in a dim scene + // and a bright one. Without it VR-012 would locate a different knee per + // film — a magic number wearing a measurement's clothes (AR-024). + const cv::Mat src = texture(112); + + cv::Mat dim; + src.convertTo(dim, CV_8U, 0.5, 64.0); // half contrast, re-centred, no clipping + + const float a = crop_sharpness(src); + const float b = crop_sharpness(dim); + REQUIRE(a > 0.f); + CHECK_THAT(b, WithinRel(a, 0.03f)); +} + +TEST_CASE("the contrast invariance is exact, and 8-bit sampling is what bends it", + "[face_utils][AR-029]") { + // Worth separating because the two have different consequences. The + // algebra is exact — scaling I by α scales the Laplacian by α, so both + // variances scale by α² and cancel — which is why halving a float crop + // changes nothing at all. + // + // What deviates is the 8-bit *round trip*: halving the contrast of a stored + // crop throws away a bit of dynamic range, and the quantisation floor it + // leaves behind is broadband, so it lands almost entirely in the numerator. + // The effect scales with how little signal is left to compete with it — + // measured on this texture, a half-contrast copy reads 0.9% high when sharp, + // 24% high at sigma 1.2 and 148% high at sigma 2.5. + // + // So: a dim *and* soft crop reads sharper than it is, and that is the corner + // of the axis VR-012 has to put a knee in. Asserted here rather than left as + // a comment, because "the measure is contrast-invariant" is the kind of claim + // that gets repeated without its precondition. + cv::Mat src; + gaussian(texture(112), 1.2).convertTo(src, CV_32F); + const cv::Mat half = src * 0.5 + 64.0; + + const float a = crop_sharpness(src); + const float b = crop_sharpness(half); + REQUIRE(a > 0.f); + CHECK_THAT(b, WithinRel(a, 1e-5f)); +} + +TEST_CASE("a small sharp face outscores a large soft one", "[face_utils][AR-029][UT-134]") { + // The register's named edge case: "size must not leak into this axis". What + // that means operationally is that the measure is not a monotone function of + // source face size — it reports the detail present in the embedder's input, + // so the ordering can and must invert when the large face is the blurred one. + // + // Small sharp: 40 px of real detail, upsampled 2.8x → finest scale ~2.8 crop px. + // Large soft: 400 px blurred at sigma 20, downsampled 3.57x → ~5.6 crop px. + const float small_sharp = crop_sharpness(to_crop(texture(40))); + const float large_soft = crop_sharpness(to_crop(gaussian(texture(400), 20.0))); + + CHECK(small_sharp > large_soft); +} + +TEST_CASE("a flat crop scores zero rather than dividing by zero", "[face_utils][AR-029][UT-135]") { + const cv::Mat flat(112, 112, CV_8UC3, cv::Scalar(90, 90, 90)); + const float s = crop_sharpness(flat); + CHECK(std::isfinite(s)); + CHECK_THAT(s, WithinAbs(0.0f, 1e-6f)); +} + +TEST_CASE("an empty crop is unscored, not zero", "[face_utils][AR-029][UT-136]") { + // -1 says "nothing measured this"; 0 says "measured, and there was no + // detail". Collapsing them would put unscored faces at the bottom of the + // quality axis, where VR-012 would read them as the blurriest in the film. + CHECK(crop_sharpness(cv::Mat()) < 0.f); +} + +// ── AR-028: the aligner fills the vector, and loses nothing quietly ─────────── + +namespace { + +// A face at `centre` in an image with enough texture for sharpness to be a real +// number rather than the flat-crop zero. +std::array face_at(cv::Point2f centre, float scale) { + std::array lm; + for (int i = 0; i < 5; ++i) + lm[i] = {centre.x + (kArcFaceRef[i][0] - 56.f) * scale, + centre.y + (kArcFaceRef[i][1] - 56.f) * scale}; + return lm; +} + +cv::Mat textured_frame(int w, int h) { + cv::Mat gray = texture(std::max(w, h)); + cv::Mat bgr; + cv::cvtColor(gray(cv::Rect(0, 0, w, h)), bgr, cv::COLOR_GRAY2BGR); + return bgr; +} + +} // namespace + +TEST_CASE("every face the aligner admits carries a full quality vector", "[face_utils][AR-028][UT-137]") { + SceneFrame sf; + sf.source.image = textured_frame(400, 300); + for (auto c : {cv::Point2f{120.f, 100.f}, cv::Point2f{280.f, 190.f}}) { + DetectedFace f; + f.landmarks = face_at(c, 1.2f); + f.bbox = cv::Rect2f(c.x - 60.f, c.y - 60.f, 120.f, 120.f); + f.confidence = 0.9f; + sf.faces.push_back(f); + } + + FaceAlignerFunc aligner; + const AlignedSceneFrame out = aligner(std::move(sf)); + + REQUIRE(out.faces.size() == 2); + for (const auto& f : out.faces) { + // Not "is it good quality" — that is VR-012's to decide. Only that the + // sentinel is gone, so no embedding reaches the matcher unscored. + CHECK(f.sharpness >= 0.f); + CHECK(f.alignment_residual >= 0.f); + } + CHECK(aligner.scored() == 2); + CHECK(aligner.degenerate() == 0); +} + +TEST_CASE("a degenerate detection is counted, not silently vanished", "[face_utils][AR-028][UT-138]") { + // It cannot be scored — there is no crop and no fit to score — so it is + // dropped. The requirement is that the drop leaves a trace: without the + // tally, a detector emitting unusable landmark sets produces a dump that + // looks exactly like footage with fewer faces in it. + SceneFrame sf; + sf.source.image = textured_frame(400, 300); + + DetectedFace good; + good.landmarks = face_at({150.f, 140.f}, 1.2f); + good.confidence = 0.9f; + sf.faces.push_back(good); + + DetectedFace degenerate; + for (auto& p : degenerate.landmarks) p = {200.f, 200.f}; + degenerate.confidence = 0.9f; + sf.faces.push_back(degenerate); + + FaceAlignerFunc aligner; + const AlignedSceneFrame out = aligner(std::move(sf)); + + CHECK(out.faces.size() == 1); + CHECK(out.crops.size() == 1); + CHECK(aligner.scored() == 1); + CHECK(aligner.degenerate() == 1); +}