feat(quality): score every face on sharpness and alignment before it is evidence
Every embedding now carries the quality of the input it came from. Both axes fall out of the AR-005 warp for free: crop_sharpness() is the normalised Laplacian variance over the aligned 112x112, so contrast and size cannot leak into it, and the alignment residual is the part of the landmark deformation a similarity transform cannot explain, so in-plane roll reads as zero and foreshortening does not. Carried, not consumed. Nothing discounts or thresholds on either number yet -- that is AR-030 and VR-012, and the knee has to be located against recorded data before a gate is chosen. What this change buys is that the data exists to locate it with. No face is admitted unscored: the -1 sentinel is preserved rather than clamped, and a degenerate landmark fit is counted rather than silently dropped. Takes the VR-001 dump to schema_version 2. The bump is not for readers, which check for the datasets by name and replay a v1 dump unchanged; it is so a consumer can tell "never scored" from "scored zero", which is not recoverable from the arrays afterwards. TRACES: AR-028, AR-029, AR-030 | VR-001 | SR-002
This commit is contained in:
@@ -1,5 +1,5 @@
|
||||
#pragma once
|
||||
/// TRACES: VR-001, VR-010 | PR-002
|
||||
/// TRACES: AR-028 | VR-001, VR-010 | PR-002
|
||||
#include "types.hpp"
|
||||
#include "config.hpp"
|
||||
#include "gallery/embedder_stamp.hpp"
|
||||
@@ -178,6 +178,16 @@ struct EmbeddingDumpFunc {
|
||||
lmk_.push_back(f.landmarks[k].y);
|
||||
}
|
||||
conf_.push_back(f.confidence);
|
||||
/// TRACES: AR-028 | SR-002
|
||||
// The quality vector, carried rather than consumed: written beside
|
||||
// the embedding it describes so VR-012 can locate its knees against
|
||||
// recorded data instead of by re-running video. Size is the third
|
||||
// axis and is already here as bbox + the bbox_upscale attribute.
|
||||
// Both are -1 only if a face reached the dump unscored, which the
|
||||
// aligner does not allow — the sentinel is preserved rather than
|
||||
// clamped so that a future path which did would be visible.
|
||||
sharp_.push_back(f.sharpness);
|
||||
resid_.push_back(f.alignment_residual);
|
||||
const auto& e = ef.embeddings[i];
|
||||
emb_.insert(emb_.end(), e.begin(), e.end());
|
||||
}
|
||||
@@ -194,11 +204,20 @@ struct EmbeddingDumpFunc {
|
||||
}
|
||||
|
||||
private:
|
||||
// Root attributes are additive: schema_version stays 1 across VR-010, because
|
||||
// Root attributes are additive: schema_version stayed 1 across VR-010, because
|
||||
// every reader takes attributes by name with a default (replay.py) or an
|
||||
// existence check (read_dump_provenance), so an old dump loses nothing and a
|
||||
// new dump breaks nothing. A bump is for a change to the *datasets*.
|
||||
static constexpr int kSchemaVersion = 1;
|
||||
//
|
||||
// v2 is that change: AR-028 adds faces/sharpness and faces/alignment_residual.
|
||||
// The bump is not about readers — those check for the datasets by name, and a
|
||||
// v1 dump still replays. It is so a *consumer of the quality vector* can tell
|
||||
// "this film's faces were never scored" from "this film's faces scored zero",
|
||||
// which is the same distinction scene_detect exists to make and is likewise
|
||||
// not recoverable from the arrays. A v1 dump reports the vector as unknown;
|
||||
// re-dump to acquire it, since nobody can assert after the fact how sharp a
|
||||
// face was.
|
||||
static constexpr int kSchemaVersion = 2;
|
||||
static constexpr int kEmbedDim = 512;
|
||||
|
||||
static std::string basename_of(const std::string& path) {
|
||||
@@ -274,6 +293,9 @@ private:
|
||||
write_vec(faces, "bbox", bbox_, H5::PredType::NATIVE_FLOAT, 4);
|
||||
write_vec(faces, "landmarks", lmk_, H5::PredType::NATIVE_FLOAT, 10);
|
||||
write_vec(faces, "confidence", conf_, H5::PredType::NATIVE_FLOAT);
|
||||
/// TRACES: AR-028 | SR-002
|
||||
write_vec(faces, "sharpness", sharp_, H5::PredType::NATIVE_FLOAT);
|
||||
write_vec(faces, "alignment_residual", resid_, H5::PredType::NATIVE_FLOAT);
|
||||
|
||||
std::cerr << "[embedding_dump] wrote " << ts_.size() << " frames, "
|
||||
<< conf_.size() << " faces → " << path_ << "\n";
|
||||
@@ -292,4 +314,5 @@ private:
|
||||
std::vector<int64_t> face_off_;
|
||||
std::vector<int32_t> face_cnt_;
|
||||
std::vector<float> emb_, bbox_, lmk_, conf_;
|
||||
std::vector<float> sharp_, resid_; // AR-028 quality vector, parallel to conf_
|
||||
};
|
||||
|
||||
@@ -1,25 +1,55 @@
|
||||
#pragma once
|
||||
#include "face_utils.hpp"
|
||||
|
||||
#include <cstdint>
|
||||
#include <iostream>
|
||||
|
||||
// ── FaceAlignerFunc ───────────────────────────────────────────────────────────
|
||||
/// TRACES: AR-005, AR-030 | SR-002
|
||||
/// TRACES: AR-005, AR-028, AR-029, AR-030 | SR-002
|
||||
///
|
||||
// KPN node: applies a 5-point similarity transform to each detected face,
|
||||
// producing a 112×112 BGR crop suitable for ArcFace inference.
|
||||
//
|
||||
// Alignment is an Umeyama least-squares fit over all five landmarks (AR-005),
|
||||
// not a robust one: a RANSAC fit discards the very landmarks AR-030 reads.
|
||||
// Degenerate detections (where the fit fails) are dropped from the output
|
||||
// vectors. The fit's residual is the AR-030 visibility measure and comes free,
|
||||
// since the warp needs the transform anyway.
|
||||
//
|
||||
// This is also where the AR-028 quality vector is filled in, because this is
|
||||
// where the inputs to it already exist:
|
||||
//
|
||||
// - **Visibility** (AR-030) is the fit's residual, and is genuinely free — the
|
||||
// transform is computed for the warp regardless, and the residual is what
|
||||
// that fit could not explain.
|
||||
// - **Sharpness** (AR-029) is measured on the crop this node just produced,
|
||||
// which is the only place it *can* be measured: the aligned canvas is what
|
||||
// makes the number scale-normalised, and downstream of the embedder the crop
|
||||
// is only forwarded for debug rendering. It is not free — 33 us per face
|
||||
// single-threaded (cvtColor, one Laplacian, two meanStdDev over 112x112) —
|
||||
// but it is two orders below the embedder inference it qualifies, and it
|
||||
// runs per face rather than per frame, so a landscape shot costs nothing.
|
||||
//
|
||||
// Size, the third axis, is `bbox` and needs no work here.
|
||||
//
|
||||
// No face is admitted unscored: every face in the output carries both numbers,
|
||||
// so a negative value downstream is a bug rather than a poor-quality face.
|
||||
// Nothing is dropped or discounted on quality — that is AR-030's discount and
|
||||
// VR-012's knee, both still open.
|
||||
//
|
||||
// Degenerate detections (where the fit fails) cannot be scored, since there is
|
||||
// no crop and no residual to score, and are therefore dropped — but they are
|
||||
// **counted**, not silently discarded. A nonzero tally means the detector is
|
||||
// emitting landmark sets the aligner cannot use, which is a fact about the
|
||||
// detector; losing it leaves a hole in the dump that looks like footage with
|
||||
// no faces in it.
|
||||
|
||||
struct FaceAlignerFunc {
|
||||
static constexpr std::string_view label() { return "face_aligner"; }
|
||||
|
||||
AlignedSceneFrame operator()(SceneFrame sf) {
|
||||
if (sf.source.eof || sf.faces.empty())
|
||||
if (sf.source.eof) {
|
||||
report();
|
||||
return {std::move(sf.source), {}, {}};
|
||||
}
|
||||
if (sf.faces.empty())
|
||||
return {std::move(sf.source), {}, {}};
|
||||
|
||||
std::vector<DetectedFace> good_faces;
|
||||
@@ -33,15 +63,38 @@ struct FaceAlignerFunc {
|
||||
float residual = -1.f;
|
||||
cv::Mat crop = align_face(sf.source.image, face.landmarks, &residual);
|
||||
if (crop.empty()) {
|
||||
std::cerr << "[face_aligner] degenerate detection skipped\n";
|
||||
++degenerate_;
|
||||
continue;
|
||||
}
|
||||
face.alignment_residual = residual;
|
||||
face.sharpness = crop_sharpness(crop);
|
||||
good_faces.push_back(face);
|
||||
crops.push_back(std::move(crop));
|
||||
++scored_;
|
||||
}
|
||||
|
||||
return {std::move(sf.source), std::move(good_faces), std::move(crops)};
|
||||
}
|
||||
|
||||
/// Faces that carry a full quality vector, and faces the fit could not use.
|
||||
uint64_t scored() const { return scored_; }
|
||||
uint64_t degenerate() const { return degenerate_; }
|
||||
|
||||
private:
|
||||
// Reported once at EOF rather than per occurrence: a run with a systematic
|
||||
// landmark problem would otherwise emit one line per face for the length of
|
||||
// a film, which is how the count came to be ignored.
|
||||
void report() {
|
||||
if (reported_) return;
|
||||
reported_ = true;
|
||||
if (degenerate_)
|
||||
std::cerr << "[face_aligner] " << degenerate_ << " of "
|
||||
<< (degenerate_ + scored_)
|
||||
<< " detections had a degenerate landmark fit and were dropped"
|
||||
" (no crop, so no embedding and no quality vector)\n";
|
||||
}
|
||||
|
||||
uint64_t scored_{0};
|
||||
uint64_t degenerate_{0};
|
||||
bool reported_{false};
|
||||
};
|
||||
|
||||
Reference in New Issue
Block a user