Keep each face's quality, and never compare against a poor one
The embedder's raw output has a length, and the length is a reading of how recognisable the crop was: a blur, an occlusion or a hard profile comes out short. Normalising threw it away. A short vector sits near the middle of the sphere and matches a little of everyone, which is how one bad crop bridges two people in a grouping pass. So the length is kept — the store now holds the raw vector, re-normalised on load, with the length beside it as `faces.quality` — and a face under MIN_GALLERY_QUALITY (14) is a probe: measured against the gallery and placed where it fits, but never what another face is measured against. Two probes are never paired, and a probe is nobody's evidence for a confidence. The People screen shows the number as "Quality 17.3", dimmed below the floor. Faces indexed before this stored unit vectors and have no reading; they are admitted to the gallery, and schema V14 forgets the run marker of every image holding one so the next indexing pass measures them. A peer's unmeasured shard faces are not adopted, or a sync would write that marker back.
This commit is contained in:
@@ -63,15 +63,16 @@ fn main() {
|
||||
let embed_ms = t.elapsed().as_secs_f64() * 1e3;
|
||||
|
||||
println!(
|
||||
" [{i}] conf {:.3} box {:.0},{:.0} {:.0}×{:.0} crop_px {:.0} embed {embed_ms:.0} ms",
|
||||
" [{i}] conf {:.3} box {:.0},{:.0} {:.0}×{:.0} crop_px {:.0} quality {:.1} embed {embed_ms:.0} ms",
|
||||
d.confidence,
|
||||
d.bbox.0,
|
||||
d.bbox.1,
|
||||
d.width(),
|
||||
d.height(),
|
||||
aligned.source_px(),
|
||||
emb.quality,
|
||||
);
|
||||
all.push((path.clone(), i, emb));
|
||||
all.push((path.clone(), i, emb.embedding));
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -46,11 +46,13 @@ fn main() {
|
||||
);
|
||||
for n in sizes {
|
||||
let (embeddings, crop_px, images) = population(n);
|
||||
let gallery = vec![true; n];
|
||||
let faces = Faces {
|
||||
embeddings: &embeddings,
|
||||
dim: EMBEDDING_DIM,
|
||||
crop_px: &crop_px,
|
||||
images: &images,
|
||||
gallery: &gallery,
|
||||
};
|
||||
|
||||
let start = std::time::Instant::now();
|
||||
|
||||
+48
-11
@@ -136,11 +136,24 @@ pub const RIVAL_FLOOR: f32 = 0.5;
|
||||
///
|
||||
/// A face in no group, or one with no evidence for anybody, scores 0.
|
||||
///
|
||||
/// `gallery` is one flag per face — which faces may be evidence at all
|
||||
/// ([`crate::embedding::MIN_GALLERY_QUALITY`]). Its length is the face count.
|
||||
/// A pair is evidence *about* either face but only *from* a gallery one: a
|
||||
/// probe learns from the references it matched, and a reference learns nothing
|
||||
/// from a probe that happened to match it, however well. Without that, the one
|
||||
/// short vector in a group would be the strongest match every face in it had.
|
||||
///
|
||||
/// `pairs` must be the *evidence* list — scanned at [`RIVAL_FLOOR`], not at the
|
||||
/// merge threshold. Passing the merge list still works but silently removes
|
||||
/// every rival weaker than a merge, which is most of them, and every uniqueness
|
||||
/// collapses to 1.
|
||||
pub fn identity_shares(faces: usize, clusters: &[Cluster], pairs: &[Pair], top: usize) -> Vec<f32> {
|
||||
pub fn identity_shares(
|
||||
gallery: &[bool],
|
||||
clusters: &[Cluster],
|
||||
pairs: &[Pair],
|
||||
top: usize,
|
||||
) -> Vec<f32> {
|
||||
let faces = gallery.len();
|
||||
// An identity is a *person*, not a group. One person routinely holds
|
||||
// several anchored groups — the same reason they hold several unnamed ones
|
||||
// — and keying this by group had Catherine competing with Catherine, which
|
||||
@@ -174,15 +187,16 @@ pub fn identity_shares(faces: usize, clusters: &[Cluster], pairs: &[Pair], top:
|
||||
}
|
||||
// A pair is evidence in both directions: j's identity hears about i,
|
||||
// and i's identity hears about j. The pair list holds each unordered
|
||||
// pair once, so both have to be recorded here.
|
||||
// pair once, so both have to be recorded here — each only where the
|
||||
// face doing the telling is in the gallery.
|
||||
let (gi, gj) = (group_of[p.i], group_of[p.j]);
|
||||
if gj != usize::MAX {
|
||||
if gj != usize::MAX && gallery[p.j] {
|
||||
evidence[p.i]
|
||||
.entry(key_of[gj])
|
||||
.or_default()
|
||||
.push(p.probability);
|
||||
}
|
||||
if gi != usize::MAX {
|
||||
if gi != usize::MAX && gallery[p.i] {
|
||||
evidence[p.j]
|
||||
.entry(key_of[gi])
|
||||
.or_default()
|
||||
@@ -255,6 +269,11 @@ mod tests {
|
||||
Pair { i, j, probability }
|
||||
}
|
||||
|
||||
/// `n` faces, every one of them fit to be compared against.
|
||||
fn all(n: usize) -> Vec<bool> {
|
||||
vec![true; n]
|
||||
}
|
||||
|
||||
/// The failure the module exists to fix: face 0 matches its own group's
|
||||
/// three members strongly, and the group has forty more it is unrelated to.
|
||||
/// The old within-group mean reported ~0.07 for this.
|
||||
@@ -264,7 +283,7 @@ mod tests {
|
||||
let clusters = vec![cluster(&members)];
|
||||
let pairs = vec![pair(0, 1, 0.99), pair(0, 2, 0.97), pair(0, 3, 0.95)];
|
||||
|
||||
let shares = identity_shares(44, &clusters, &pairs, TOP_MATCHES);
|
||||
let shares = identity_shares(&all(44), &clusters, &pairs, TOP_MATCHES);
|
||||
assert!(
|
||||
(shares[0] - 0.97).abs() < 1e-6,
|
||||
"the mean of its three real matches, undiluted: {}",
|
||||
@@ -284,7 +303,7 @@ mod tests {
|
||||
pair(0, 4, 0.90),
|
||||
];
|
||||
|
||||
let shares = identity_shares(5, &clusters, &pairs, TOP_MATCHES);
|
||||
let shares = identity_shares(&all(5), &clusters, &pairs, TOP_MATCHES);
|
||||
// Coherent at 0.90, and only half of the evidence is its own.
|
||||
assert!(
|
||||
(shares[0] - 0.45).abs() < 1e-6,
|
||||
@@ -298,9 +317,9 @@ mod tests {
|
||||
#[test]
|
||||
fn a_rival_too_weak_to_merge_still_lowers_the_confidence() {
|
||||
let clusters = vec![named(&[0, 1], 1), named(&[2, 3], 2)];
|
||||
let sure = identity_shares(4, &clusters, &[pair(0, 1, 0.95)], TOP_MATCHES);
|
||||
let sure = identity_shares(&all(4), &clusters, &[pair(0, 1, 0.95)], TOP_MATCHES);
|
||||
let contested = identity_shares(
|
||||
4,
|
||||
&all(4),
|
||||
&clusters,
|
||||
&[pair(0, 1, 0.95), pair(0, 2, 0.60)],
|
||||
TOP_MATCHES,
|
||||
@@ -321,7 +340,7 @@ mod tests {
|
||||
fn an_unnamed_group_is_not_treated_as_competition() {
|
||||
let clusters = vec![cluster(&[0, 1]), cluster(&[2, 3])];
|
||||
let shares = identity_shares(
|
||||
4,
|
||||
&all(4),
|
||||
&clusters,
|
||||
&[pair(0, 1, 0.95), pair(0, 2, 0.90)],
|
||||
TOP_MATCHES,
|
||||
@@ -343,7 +362,7 @@ mod tests {
|
||||
let mut pairs: Vec<Pair> = (1..11).map(|j| pair(0, j, 0.90)).collect();
|
||||
pairs.extend((11..62).map(|j| pair(0, j, 0.55)));
|
||||
|
||||
let shares = identity_shares(62, &clusters, &pairs, TOP_MATCHES);
|
||||
let shares = identity_shares(&all(62), &clusters, &pairs, TOP_MATCHES);
|
||||
// Ten at 0.90 against ten at 0.55 — not fifty-one at 0.55.
|
||||
assert!(
|
||||
(shares[0] - 0.90 * (9.0 / 14.5)).abs() < 1e-5,
|
||||
@@ -352,11 +371,29 @@ mod tests {
|
||||
);
|
||||
}
|
||||
|
||||
/// A probe learns from the references it matched; a reference learns
|
||||
/// nothing from a probe. The pair is the same pair — what differs is who
|
||||
/// is doing the telling.
|
||||
#[test]
|
||||
fn a_face_outside_the_gallery_is_nobody_s_evidence() {
|
||||
let clusters = vec![named(&[0, 1, 2], 1)];
|
||||
let gallery = vec![true, true, false];
|
||||
let pairs = vec![pair(0, 1, 0.80), pair(0, 2, 0.99), pair(1, 2, 0.99)];
|
||||
|
||||
let shares = identity_shares(&gallery, &clusters, &pairs, TOP_MATCHES);
|
||||
// Faces 0 and 1 hear only from each other: the 0.99 the probe offered
|
||||
// them is not counted.
|
||||
assert!((shares[0] - 0.80).abs() < 1e-6, "{}", shares[0]);
|
||||
assert!((shares[1] - 0.80).abs() < 1e-6, "{}", shares[1]);
|
||||
// The probe hears from both references.
|
||||
assert!((shares[2] - 0.99).abs() < 1e-6, "{}", shares[2]);
|
||||
}
|
||||
|
||||
/// A face nothing has any evidence about claims nothing.
|
||||
#[test]
|
||||
fn a_face_with_no_evidence_reports_no_confidence() {
|
||||
let clusters = vec![cluster(&[0, 1])];
|
||||
let shares = identity_shares(2, &clusters, &[], TOP_MATCHES);
|
||||
let shares = identity_shares(&all(2), &clusters, &[], TOP_MATCHES);
|
||||
assert_eq!(shares, vec![0.0, 0.0]);
|
||||
}
|
||||
}
|
||||
|
||||
+307
-3
@@ -19,6 +19,23 @@
|
||||
//! and clustering never moves it. Two groups holding confirmations of
|
||||
//! *different* people cannot merge, whatever their similarity says.
|
||||
//!
|
||||
//! # The gallery, and the faces that are only ever compared against it
|
||||
//!
|
||||
//! A third defence, and the cheapest of all: **a short embedding is never a
|
||||
//! reference.** The length of the raw vector is the model's own reading of
|
||||
//! how recognisable the crop was ([`crate::embedding::MIN_GALLERY_QUALITY`]),
|
||||
//! and a short one sits near the centre of the sphere, matching a little of
|
||||
//! everybody. One of those in a group is a bridge to the next group over.
|
||||
//!
|
||||
//! So the population is split. Faces at or above the floor are the
|
||||
//! **gallery**, and they cluster exactly as described below. Faces under it
|
||||
//! are **probes**: each is measured against the finished groups and joins the
|
||||
//! one it fits, by the same average-link rule and under the same constraints
|
||||
//! — but it is measured against the gallery members only, never against
|
||||
//! another probe, and once placed it is never part of what the next face is
|
||||
//! measured against. A blurred photograph of a known person is still named;
|
||||
//! it just cannot vouch for anyone else.
|
||||
//!
|
||||
//! # Average link, not single link
|
||||
//!
|
||||
//! Single-link chains: one bad edge welds two identities together, and it is
|
||||
@@ -117,6 +134,11 @@ pub struct Candidate {
|
||||
pub embedding: Vec<f32>,
|
||||
/// Source pixels across the aligned crop, for the calibration's size term.
|
||||
pub crop_px: f32,
|
||||
/// Length of the raw embedding, where it was recorded
|
||||
/// ([`crate::embedding::MIN_GALLERY_QUALITY`]). `None` for a face indexed
|
||||
/// before it was kept, which is admitted to the gallery — see
|
||||
/// [`Candidate::in_gallery`].
|
||||
pub quality: Option<f32>,
|
||||
/// The person this face is *confirmed* to be, if any.
|
||||
///
|
||||
/// Suggestions are deliberately not passed here. They are this function's
|
||||
@@ -125,6 +147,13 @@ pub struct Candidate {
|
||||
pub confirmed_person: Option<u64>,
|
||||
}
|
||||
|
||||
impl Candidate {
|
||||
/// Whether this face may be compared *against*, as well as compared.
|
||||
pub fn in_gallery(&self) -> bool {
|
||||
crate::embedding::in_gallery(self.quality)
|
||||
}
|
||||
}
|
||||
|
||||
/// One group of faces the clusterer believes are one person.
|
||||
#[derive(Debug, Clone, PartialEq)]
|
||||
pub struct Cluster {
|
||||
@@ -202,7 +231,7 @@ pub fn cluster_scored(faces: &[Candidate], cal: &Calibration, min_probability: f
|
||||
|
||||
let clusters = build(faces, cal, min_probability, &merges);
|
||||
let confidence = crate::assign::identity_shares(
|
||||
faces.len(),
|
||||
&columns.gallery,
|
||||
&clusters,
|
||||
&evidence,
|
||||
crate::assign::TOP_MATCHES,
|
||||
@@ -225,6 +254,7 @@ struct Columns {
|
||||
dim: usize,
|
||||
crop_px: Vec<f32>,
|
||||
images: Vec<u64>,
|
||||
gallery: Vec<bool>,
|
||||
}
|
||||
|
||||
impl Columns {
|
||||
@@ -245,6 +275,7 @@ impl Columns {
|
||||
dim,
|
||||
crop_px: faces.iter().map(|f| f.crop_px).collect(),
|
||||
images: faces.iter().map(|f| f.image).collect(),
|
||||
gallery: faces.iter().map(Candidate::in_gallery).collect(),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -254,22 +285,171 @@ impl Columns {
|
||||
dim: self.dim,
|
||||
crop_px: &self.crop_px,
|
||||
images: &self.images,
|
||||
gallery: &self.gallery,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Agglomerate the gallery over its pairs, then place the probes.
|
||||
///
|
||||
/// `pairs` is what [`neighbours::above_threshold`] returned: every pair has a
|
||||
/// gallery side, but a pair with a probe on the other side is not a merge —
|
||||
/// it is the evidence [`place_probes`] works from. Only the gallery-to-gallery
|
||||
/// pairs reach the engine, so a probe enters it as a singleton with no edges
|
||||
/// and comes out exactly as it went in.
|
||||
fn build(
|
||||
faces: &[Candidate],
|
||||
cal: &Calibration,
|
||||
min_probability: f32,
|
||||
pairs: &[neighbours::Pair],
|
||||
) -> Vec<Cluster> {
|
||||
let gallery: Vec<bool> = faces.iter().map(Candidate::in_gallery).collect();
|
||||
let (merges, probe_pairs): (Vec<_>, Vec<_>) = pairs
|
||||
.iter()
|
||||
.copied()
|
||||
.partition(|p| gallery[p.i] && gallery[p.j]);
|
||||
|
||||
let mut engine = Engine::new(faces, cal, min_probability);
|
||||
let parts = components(faces.len(), pairs);
|
||||
let parts = components(faces.len(), &merges);
|
||||
for (component, edges) in parts.members.iter().zip(&parts.edges) {
|
||||
engine.agglomerate(component, edges);
|
||||
}
|
||||
engine.finish()
|
||||
let dot = engine.dot;
|
||||
let clusters = engine.finish();
|
||||
if probe_pairs.is_empty() {
|
||||
return clusters;
|
||||
}
|
||||
place_probes(
|
||||
faces,
|
||||
cal,
|
||||
min_probability,
|
||||
dot,
|
||||
&gallery,
|
||||
clusters,
|
||||
&probe_pairs,
|
||||
)
|
||||
}
|
||||
|
||||
/// Put each probe into the finished group it fits, or leave it alone.
|
||||
///
|
||||
/// The same decision the engine makes for a singleton — average link over the
|
||||
/// group, at or above `min_probability`, subject to [`Engine::can_link`]'s two
|
||||
/// constraints — with one difference that is the whole point: the average is
|
||||
/// over the group's **gallery** members. A probe already placed is not part of
|
||||
/// what the next one is measured against, so a run of short vectors cannot
|
||||
/// pull each other in one after another.
|
||||
///
|
||||
/// Probes are placed in index order and each placement is final, which is
|
||||
/// what keeps this deterministic. The group a probe joins gains its
|
||||
/// photograph, so a second face from the same frame cannot follow it — the
|
||||
/// co-occurrence rule, applied exactly as the engine applies it.
|
||||
fn place_probes(
|
||||
faces: &[Candidate],
|
||||
cal: &Calibration,
|
||||
min_probability: f32,
|
||||
dot: neighbours::DotFn,
|
||||
gallery: &[bool],
|
||||
mut clusters: Vec<Cluster>,
|
||||
probe_pairs: &[neighbours::Pair],
|
||||
) -> Vec<Cluster> {
|
||||
// Where each face sits, and what each group's photographs and gallery
|
||||
// members are. The probe's own singleton is here too, and is dropped once
|
||||
// it has moved.
|
||||
let mut group_of = vec![usize::MAX; faces.len()];
|
||||
for (g, c) in clusters.iter().enumerate() {
|
||||
for &m in &c.members {
|
||||
group_of[m] = g;
|
||||
}
|
||||
}
|
||||
let mut images: Vec<HashSet<u64>> = clusters
|
||||
.iter()
|
||||
.map(|c| c.members.iter().map(|&m| faces[m].image).collect())
|
||||
.collect();
|
||||
let references: Vec<Vec<usize>> = clusters
|
||||
.iter()
|
||||
.map(|c| c.members.iter().copied().filter(|&m| gallery[m]).collect())
|
||||
.collect();
|
||||
|
||||
// Which groups each probe has any above-threshold pair into. Only those
|
||||
// can average above the threshold — the argument the module note makes
|
||||
// for the engine holds here unchanged.
|
||||
let mut candidates: Vec<Vec<usize>> = vec![Vec::new(); faces.len()];
|
||||
for p in probe_pairs {
|
||||
let (probe, reference) = if gallery[p.i] { (p.j, p.i) } else { (p.i, p.j) };
|
||||
candidates[probe].push(group_of[reference]);
|
||||
}
|
||||
|
||||
let mut moved: Vec<usize> = Vec::new();
|
||||
for probe in 0..faces.len() {
|
||||
if gallery[probe] || candidates[probe].is_empty() {
|
||||
continue;
|
||||
}
|
||||
let mut groups = std::mem::take(&mut candidates[probe]);
|
||||
groups.sort_unstable();
|
||||
groups.dedup();
|
||||
|
||||
let face = &faces[probe];
|
||||
let mut best: Option<(f32, usize)> = None;
|
||||
for g in groups {
|
||||
let target = &clusters[g];
|
||||
if let (Some(mine), Some(theirs)) = (face.confirmed_person, target.person) {
|
||||
if mine != theirs {
|
||||
continue;
|
||||
}
|
||||
}
|
||||
if images[g].contains(&face.image) {
|
||||
continue;
|
||||
}
|
||||
let (mut sum, mut count) = (0.0_f64, 0.0_f64);
|
||||
for &r in &references[g] {
|
||||
let cos = dot(&face.embedding, &faces[r].embedding);
|
||||
let min_crop = face.crop_px.min(faces[r].crop_px);
|
||||
sum += cal.probability(cos, min_crop, 0.0) as f64;
|
||||
count += 1.0;
|
||||
}
|
||||
if count == 0.0 {
|
||||
continue;
|
||||
}
|
||||
let p = (sum / count) as f32;
|
||||
// Strictly better wins; on a tie the lowest group index, which is
|
||||
// the engine's own tiebreak.
|
||||
if p >= min_probability && best.is_none_or(|(bp, _)| p > bp) {
|
||||
best = Some((p, g));
|
||||
}
|
||||
}
|
||||
|
||||
let Some((_, g)) = best else { continue };
|
||||
let own = group_of[probe];
|
||||
clusters[g].members.push(probe);
|
||||
clusters[g].members.sort_unstable();
|
||||
clusters[g].person = clusters[g].person.or(face.confirmed_person);
|
||||
images[g].insert(face.image);
|
||||
group_of[probe] = g;
|
||||
moved.push(own);
|
||||
}
|
||||
|
||||
if moved.is_empty() {
|
||||
return clusters;
|
||||
}
|
||||
// The singletons the probes left behind, then the order `Engine::finish`
|
||||
// promises: largest first, lowest member first among equals.
|
||||
let mut vacated = vec![false; clusters.len()];
|
||||
for g in moved {
|
||||
vacated[g] = true;
|
||||
}
|
||||
let mut out: Vec<Cluster> = clusters
|
||||
.into_iter()
|
||||
.zip(vacated)
|
||||
.filter(|(_, gone)| !gone)
|
||||
.map(|(c, _)| c)
|
||||
.collect();
|
||||
out.sort_by(|x, y| {
|
||||
y.members
|
||||
.len()
|
||||
.cmp(&x.members.len())
|
||||
.then(x.members[0].cmp(&y.members[0]))
|
||||
});
|
||||
out
|
||||
}
|
||||
|
||||
/// Split one person's faces into the groups a raised threshold separates them
|
||||
@@ -726,10 +906,19 @@ mod tests {
|
||||
image,
|
||||
embedding: at_cosine(identity, cosine),
|
||||
crop_px: 150.0,
|
||||
quality: None,
|
||||
confirmed_person: None,
|
||||
}
|
||||
}
|
||||
|
||||
/// A face too short to be a reference: compared, never compared against.
|
||||
fn probe(face: u64, image: u64, identity: usize, cosine: f32) -> Candidate {
|
||||
Candidate {
|
||||
quality: Some(crate::embedding::MIN_GALLERY_QUALITY - 5.0),
|
||||
..candidate(face, image, identity, cosine)
|
||||
}
|
||||
}
|
||||
|
||||
/// A calibration steep enough that the test's cosines are unambiguous:
|
||||
/// 0.6 is near-certain, 0.1 is near-impossible.
|
||||
fn cal() -> Calibration {
|
||||
@@ -1099,6 +1288,7 @@ mod tests {
|
||||
image,
|
||||
embedding: at_cosine(p, cosine),
|
||||
crop_px: 60.0 + ((out.len() % 11) as f32) * 25.0,
|
||||
quality: None,
|
||||
confirmed_person: None,
|
||||
});
|
||||
image += 1;
|
||||
@@ -1182,6 +1372,7 @@ mod tests {
|
||||
image: 5_000,
|
||||
embedding: at_cosine(200, 1.0),
|
||||
crop_px: 150.0,
|
||||
quality: None,
|
||||
confirmed_person: None,
|
||||
});
|
||||
let out = cluster(&faces, &cal(), DEFAULT_MERGE_PROBABILITY);
|
||||
@@ -1191,4 +1382,117 @@ mod tests {
|
||||
"the outlier was absorbed"
|
||||
);
|
||||
}
|
||||
|
||||
// ── the gallery ───────────────────────────────────────────────────────
|
||||
|
||||
/// A short vector is still somebody: it joins the group it matches.
|
||||
#[test]
|
||||
fn a_probe_joins_the_group_it_matches() {
|
||||
let faces = vec![
|
||||
candidate(1, 10, 0, 1.0),
|
||||
candidate(2, 11, 0, 0.95),
|
||||
probe(3, 12, 0, 0.92),
|
||||
];
|
||||
let out = cluster(&faces, &cal(), DEFAULT_MERGE_PROBABILITY);
|
||||
assert_eq!(out.len(), 1);
|
||||
assert_eq!(out[0].members, vec![0, 1, 2]);
|
||||
}
|
||||
|
||||
/// Two short vectors that resemble each other are noise agreeing with
|
||||
/// noise, and there is nothing in the gallery for either to be measured
|
||||
/// against.
|
||||
#[test]
|
||||
fn two_probes_are_never_grouped_with_each_other() {
|
||||
let faces = vec![probe(1, 10, 0, 1.0), probe(2, 11, 0, 0.98)];
|
||||
let out = cluster(&faces, &cal(), DEFAULT_MERGE_PROBABILITY);
|
||||
assert_eq!(out.len(), 2, "two probes were grouped: {out:?}");
|
||||
}
|
||||
|
||||
/// The point of measuring against the gallery only: a probe that has been
|
||||
/// placed is not a stepping stone for the next one.
|
||||
#[test]
|
||||
fn a_placed_probe_is_not_what_the_next_probe_is_measured_against() {
|
||||
let mut first = probe(2, 11, 0, 0.6);
|
||||
// 0.6 along identity 0 and 0.8 along its perpendicular: near enough to
|
||||
// the reference to join it, and much nearer to the face below.
|
||||
first.embedding = at_cosine(0, 0.6);
|
||||
let mut second = probe(3, 12, 0, 0.0);
|
||||
second.embedding = at_cosine(0, 0.0);
|
||||
let faces = vec![candidate(1, 10, 0, 1.0), first, second];
|
||||
|
||||
let out = cluster(&faces, &cal(), DEFAULT_MERGE_PROBABILITY);
|
||||
let group = out.iter().find(|c| c.members.contains(&0)).unwrap();
|
||||
assert_eq!(
|
||||
group.members,
|
||||
vec![0, 1],
|
||||
"the first probe should have joined"
|
||||
);
|
||||
assert!(
|
||||
out.iter().any(|c| c.members == vec![2]),
|
||||
"the second probe reached the group through the first: {out:?}"
|
||||
);
|
||||
}
|
||||
|
||||
/// A confirmation on a probe is still the user's word: the group it joins
|
||||
/// becomes that person, and a group already someone else's is closed to it.
|
||||
#[test]
|
||||
fn a_probe_carries_its_confirmation_and_respects_others() {
|
||||
let mut anchored = probe(3, 12, 0, 0.92);
|
||||
anchored.confirmed_person = Some(7);
|
||||
let faces = vec![
|
||||
candidate(1, 10, 0, 1.0),
|
||||
candidate(2, 11, 0, 0.95),
|
||||
anchored,
|
||||
];
|
||||
let out = cluster(&faces, &cal(), DEFAULT_MERGE_PROBABILITY);
|
||||
assert_eq!(out.len(), 1);
|
||||
assert_eq!(out[0].person, Some(7));
|
||||
|
||||
let mut theirs = candidate(1, 10, 0, 1.0);
|
||||
theirs.confirmed_person = Some(8);
|
||||
let faces = vec![theirs, candidate(2, 11, 0, 0.95), {
|
||||
let mut a = probe(3, 12, 0, 0.92);
|
||||
a.confirmed_person = Some(7);
|
||||
a
|
||||
}];
|
||||
let out = cluster(&faces, &cal(), DEFAULT_MERGE_PROBABILITY);
|
||||
assert!(
|
||||
out.iter()
|
||||
.any(|c| c.members == vec![2] && c.person == Some(7)),
|
||||
"a probe confirmed as one person joined another's group: {out:?}"
|
||||
);
|
||||
}
|
||||
|
||||
/// The co-occurrence rule follows a probe in: once it has joined, its
|
||||
/// photograph is the group's.
|
||||
#[test]
|
||||
fn a_probe_cannot_join_a_group_holding_a_face_from_its_own_photograph() {
|
||||
let faces = vec![
|
||||
candidate(1, 10, 0, 1.0),
|
||||
candidate(2, 11, 0, 0.95),
|
||||
probe(3, 10, 0, 0.92),
|
||||
];
|
||||
let out = cluster(&faces, &cal(), DEFAULT_MERGE_PROBABILITY);
|
||||
assert!(out.iter().any(|c| c.members == vec![2]), "{out:?}");
|
||||
}
|
||||
|
||||
/// A probe's placement is scored like anyone else's, from the references
|
||||
/// it matched — and the references' own scores do not hear from it.
|
||||
#[test]
|
||||
fn a_probe_is_scored_but_is_not_evidence() {
|
||||
let gallery_only = vec![candidate(1, 10, 0, 1.0), candidate(2, 11, 0, 0.95)];
|
||||
let without = cluster_scored(&gallery_only, &cal(), DEFAULT_MERGE_PROBABILITY);
|
||||
|
||||
let mut with_probe = gallery_only.clone();
|
||||
with_probe.push(probe(3, 12, 0, 0.99));
|
||||
let with = cluster_scored(&with_probe, &cal(), DEFAULT_MERGE_PROBABILITY);
|
||||
|
||||
assert_eq!(with.clusters[0].members, vec![0, 1, 2]);
|
||||
assert!(with.confidence[2] > 0.9, "{}", with.confidence[2]);
|
||||
assert_eq!(
|
||||
&with.confidence[..2],
|
||||
&without.confidence[..],
|
||||
"a probe changed what the references were sure of"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -16,6 +16,41 @@ use crate::align::{Aligned112, ALIGNED_EDGE};
|
||||
use crate::embedding::{normalise, Embedding, ModelId, EMBEDDING_DIM};
|
||||
use crate::{install_backend, FaceError};
|
||||
|
||||
/// What one pass of the embedder produces: the direction, and the length.
|
||||
///
|
||||
/// Two fields rather than a `quality` on [`Embedding`], because every other
|
||||
/// holder of an `Embedding` relies on it being unit length and compares by
|
||||
/// dot product; the length is a separate fact about the same face, and it is
|
||||
/// stored separately too.
|
||||
#[derive(Debug, Clone, PartialEq)]
|
||||
pub struct Embedded {
|
||||
pub embedding: Embedding,
|
||||
/// L2 norm of the raw model output.
|
||||
///
|
||||
/// The model's own opinion of how recognisable the crop was — see
|
||||
/// [`crate::embedding::MIN_GALLERY_QUALITY`] for what it means and where
|
||||
/// it is used.
|
||||
pub quality: f32,
|
||||
}
|
||||
|
||||
impl Embedded {
|
||||
/// Storage form: the **raw** vector, `512 × f16`.
|
||||
///
|
||||
/// Not the unit vector. The length is the quality, and a store that held
|
||||
/// only the direction would have thrown it away at the one moment it could
|
||||
/// be known — which is what this crate used to do. Readers re-normalise
|
||||
/// ([`Embedding::from_f16_bytes`]), so every comparison is still a dot
|
||||
/// product, and [`crate::embedding::read_f16_bytes`] gives the length back
|
||||
/// to a reader that wants it.
|
||||
///
|
||||
/// f16 costs nothing extra at this scale: its precision is relative, so a
|
||||
/// component of a vector of length 20 is kept to the same three figures as
|
||||
/// the same component scaled to length 1.
|
||||
pub fn to_f16_bytes(&self) -> Vec<u8> {
|
||||
self.embedding.to_f16_bytes_scaled(self.quality)
|
||||
}
|
||||
}
|
||||
|
||||
/// A loaded ArcFace graph.
|
||||
pub struct Embedder {
|
||||
session: ort::session::Session,
|
||||
@@ -63,7 +98,7 @@ impl Embedder {
|
||||
}
|
||||
|
||||
/// Embed one aligned face.
|
||||
pub fn embed(&mut self, face: &Aligned112) -> Result<Embedding, FaceError> {
|
||||
pub fn embed(&mut self, face: &Aligned112) -> Result<Embedded, FaceError> {
|
||||
// `(x·255 − 127.5) / 128` — see the `/128` note in `detect::Letterbox`.
|
||||
let px = face.pixels();
|
||||
let mut input = Array4::<f32>::zeros((1, 3, ALIGNED_EDGE, ALIGNED_EDGE));
|
||||
@@ -95,11 +130,14 @@ impl Embedder {
|
||||
|
||||
let mut v = Box::new([0.0_f32; EMBEDDING_DIM]);
|
||||
v.copy_from_slice(&data[..EMBEDDING_DIM]);
|
||||
normalise(&mut v);
|
||||
let quality = normalise(&mut v);
|
||||
|
||||
Ok(Embedding {
|
||||
model: self.model.clone(),
|
||||
v,
|
||||
Ok(Embedded {
|
||||
embedding: Embedding {
|
||||
model: self.model.clone(),
|
||||
v,
|
||||
},
|
||||
quality,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
+119
-18
@@ -12,6 +12,43 @@
|
||||
/// Embedding dimensionality. Fixed by the model family, not a parameter.
|
||||
pub const EMBEDDING_DIM: usize = 512;
|
||||
|
||||
/// The shortest raw embedding a face may be *compared against*.
|
||||
///
|
||||
/// # What the length of the vector says
|
||||
///
|
||||
/// ArcFace is trained on the direction of its output and nothing else, and
|
||||
/// the length it leaves behind turns out to be a free quality signal: the
|
||||
/// magnitude grows with how recognisable the crop was to the model, and a
|
||||
/// blurred, occluded, badly lit or hard-profile face comes out short. MagFace
|
||||
/// (Meng et al., CVPR 2021) made that the training objective; the plain
|
||||
/// ArcFace heads this crate runs already show it, weaker but usable, which is
|
||||
/// why it is worth keeping the number the normalisation discards.
|
||||
///
|
||||
/// # Why it gates the gallery and not the face
|
||||
///
|
||||
/// A short vector is a bad *reference*: it sits nearer the centre of the
|
||||
/// sphere than a real identity does and matches a little of everyone, which
|
||||
/// is exactly the face that welds two people together in a clustering pass.
|
||||
/// It is not a bad *probe* — the face is still real, still somebody, and
|
||||
/// comparing it against good references is the only way it will ever be named.
|
||||
/// So a face below this floor is compared against the gallery and never
|
||||
/// becomes part of it: see `cluster::Candidate::in_gallery`.
|
||||
///
|
||||
/// 14 is the operating point for `w600k_mbf`, whose norms on the reference
|
||||
/// library run from about 8 on a blur to the high 20s on a clean portrait. A
|
||||
/// face whose quality was never recorded — indexed before the number was kept
|
||||
/// — is not gated, because a rule that cannot be checked should admit, not
|
||||
/// exclude.
|
||||
pub const MIN_GALLERY_QUALITY: f32 = 14.0;
|
||||
|
||||
/// Whether an embedding of this quality may serve as a reference.
|
||||
///
|
||||
/// `None` is "not measured", and is admitted: the rule is about a number that
|
||||
/// was read and found short, not about a number that is missing.
|
||||
pub fn in_gallery(quality: Option<f32>) -> bool {
|
||||
quality.is_none_or(|q| q >= MIN_GALLERY_QUALITY)
|
||||
}
|
||||
|
||||
/// Which model produced an embedding.
|
||||
///
|
||||
/// Embeddings from different models are not comparable, and this is the one
|
||||
@@ -58,10 +95,19 @@ impl Embedding {
|
||||
}
|
||||
|
||||
/// Storage form: `512 × f16`, 1 KB per face (catalog.md §10.1).
|
||||
///
|
||||
/// This writes the unit vector. What the catalog stores is the raw one —
|
||||
/// `embed::Embedded::to_f16_bytes` — because the length is the quality
|
||||
/// and a unit vector has none left to read.
|
||||
pub fn to_f16_bytes(&self) -> Vec<u8> {
|
||||
self.to_f16_bytes_scaled(1.0)
|
||||
}
|
||||
|
||||
/// The unit vector scaled by `length`, as `512 × f16`.
|
||||
pub(crate) fn to_f16_bytes_scaled(&self, length: f32) -> Vec<u8> {
|
||||
let mut out = Vec::with_capacity(EMBEDDING_DIM * 2);
|
||||
for &x in self.v.iter() {
|
||||
out.extend_from_slice(&f32_to_f16_bits(x).to_le_bytes());
|
||||
out.extend_from_slice(&f32_to_f16_bits(x * length).to_le_bytes());
|
||||
}
|
||||
out
|
||||
}
|
||||
@@ -71,25 +117,42 @@ impl Embedding {
|
||||
/// The f16 round-trip perturbs a unit vector by ~1e-3 in cosine — three
|
||||
/// orders below the separation between a match and a non-match — but the
|
||||
/// drift is free to remove and invisible if left, so it is removed here
|
||||
/// rather than remembered at every call site.
|
||||
/// rather than remembered at every call site. The same pass is what turns
|
||||
/// a stored raw vector back into the unit one every comparison expects.
|
||||
pub fn from_f16_bytes(model: ModelId, bytes: &[u8]) -> Option<Self> {
|
||||
if bytes.len() != EMBEDDING_DIM * 2 {
|
||||
return None;
|
||||
}
|
||||
let mut v = Box::new([0.0_f32; EMBEDDING_DIM]);
|
||||
for (i, chunk) in bytes.chunks_exact(2).enumerate() {
|
||||
v[i] = f16_bits_to_f32(u16::from_le_bytes([chunk[0], chunk[1]]));
|
||||
}
|
||||
normalise(&mut v);
|
||||
Some(Self { model, v })
|
||||
read_f16_bytes(model, bytes).map(|(e, _)| e)
|
||||
}
|
||||
}
|
||||
|
||||
/// Read a stored vector back, with the length it was stored at.
|
||||
///
|
||||
/// The length is the quality where the blob is a raw one, and ~1 where it is
|
||||
/// a unit vector from before raw vectors were stored — which is why the
|
||||
/// catalog keeps the quality beside the blob rather than deriving it from
|
||||
/// this: a unit vector reads as a quality of 1, not as "unmeasured".
|
||||
pub fn read_f16_bytes(model: ModelId, bytes: &[u8]) -> Option<(Embedding, f32)> {
|
||||
if bytes.len() != EMBEDDING_DIM * 2 {
|
||||
return None;
|
||||
}
|
||||
let mut v = Box::new([0.0_f32; EMBEDDING_DIM]);
|
||||
for (i, chunk) in bytes.chunks_exact(2).enumerate() {
|
||||
v[i] = f16_bits_to_f32(u16::from_le_bytes([chunk[0], chunk[1]]));
|
||||
}
|
||||
let length = normalise(&mut v);
|
||||
Some((Embedding { model, v }, length))
|
||||
}
|
||||
|
||||
fn dot(a: &[f32; EMBEDDING_DIM], b: &[f32; EMBEDDING_DIM]) -> f32 {
|
||||
a.iter().zip(b.iter()).map(|(x, y)| x * y).sum()
|
||||
}
|
||||
|
||||
pub(crate) fn normalise(v: &mut [f32; EMBEDDING_DIM]) {
|
||||
/// Scale `v` to unit length, and return the length it had.
|
||||
///
|
||||
/// The length is the one thing about the raw output that survives being
|
||||
/// thrown away by everything downstream, and it is a quality signal
|
||||
/// ([`MIN_GALLERY_QUALITY`]) — so it comes back out rather than being lost
|
||||
/// here.
|
||||
pub(crate) fn normalise(v: &mut [f32; EMBEDDING_DIM]) -> f32 {
|
||||
// Clamped rather than checked: a zero-norm embedding is a broken model,
|
||||
// not a runtime condition worth an error path, and dividing by 1e-6 keeps
|
||||
// the NaN out of the catalog.
|
||||
@@ -97,6 +160,7 @@ pub(crate) fn normalise(v: &mut [f32; EMBEDDING_DIM]) {
|
||||
for x in v.iter_mut() {
|
||||
*x /= norm;
|
||||
}
|
||||
norm
|
||||
}
|
||||
|
||||
// ── f16 ───────────────────────────────────────────────────────────────────
|
||||
@@ -113,9 +177,10 @@ fn f32_to_f16_bits(x: f32) -> u16 {
|
||||
let mant = bits & 0x007f_ffff;
|
||||
|
||||
if exp >= 0x1f {
|
||||
// Overflow, inf, or NaN. Embeddings are unit-norm so this is the
|
||||
// broken-model path; infinity is the honest answer, not a clamp that
|
||||
// hides it.
|
||||
// Overflow, inf, or NaN. No component of an embedding exceeds its
|
||||
// length, and the lengths this model produces are in the tens, so
|
||||
// this is the broken-model path; infinity is the honest answer, not a
|
||||
// clamp that hides it.
|
||||
return sign
|
||||
| 0x7c00
|
||||
| if mant != 0 && exp == 0x1f + 112 {
|
||||
@@ -125,9 +190,9 @@ fn f32_to_f16_bits(x: f32) -> u16 {
|
||||
};
|
||||
}
|
||||
if exp <= 0 {
|
||||
// Subnormal or underflow. A component of a unit 512-vector is ~0.04,
|
||||
// nowhere near here, so this branch exists for correctness rather than
|
||||
// for traffic.
|
||||
// Subnormal or underflow. A component of a unit 512-vector is ~0.04
|
||||
// and a stored one is that times the length, nowhere near here, so
|
||||
// this branch exists for correctness rather than for traffic.
|
||||
if exp < -10 {
|
||||
return sign;
|
||||
}
|
||||
@@ -221,6 +286,42 @@ mod tests {
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn normalising_reports_the_length_it_removed() {
|
||||
let mut v = Box::new([0.0_f32; EMBEDDING_DIM]);
|
||||
v[0] = 3.0;
|
||||
v[1] = 4.0;
|
||||
let norm = normalise(&mut v);
|
||||
assert!((norm - 5.0).abs() < 1e-6, "norm {norm}");
|
||||
assert!((v[0] - 0.6).abs() < 1e-6 && (v[1] - 0.8).abs() < 1e-6);
|
||||
}
|
||||
|
||||
/// The gate admits what it cannot measure: a face from before the number
|
||||
/// was kept is not a face that was found wanting.
|
||||
#[test]
|
||||
fn an_unmeasured_quality_is_admitted_to_the_gallery() {
|
||||
assert!(in_gallery(None));
|
||||
assert!(in_gallery(Some(MIN_GALLERY_QUALITY)));
|
||||
assert!(in_gallery(Some(27.5)));
|
||||
assert!(!in_gallery(Some(MIN_GALLERY_QUALITY - 0.01)));
|
||||
assert!(!in_gallery(Some(8.0)));
|
||||
}
|
||||
|
||||
/// The storage form carries the length, and the length comes back out —
|
||||
/// without touching the direction every comparison is made on.
|
||||
#[test]
|
||||
fn a_raw_vector_round_trips_with_its_length() {
|
||||
let e = unit(3);
|
||||
let raw = e.to_f16_bytes_scaled(21.5);
|
||||
let (back, length) = read_f16_bytes(e.model.clone(), &raw).unwrap();
|
||||
assert!((length - 21.5).abs() < 0.05, "length {length}");
|
||||
assert!(e.cosine(&back).unwrap() > 0.9999);
|
||||
// A unit vector from an older store reads as length 1, not as an
|
||||
// error — see `read_f16_bytes` on why that is not "unmeasured".
|
||||
let (_, one) = read_f16_bytes(e.model.clone(), &e.to_f16_bytes()).unwrap();
|
||||
assert!((one - 1.0).abs() < 1e-2, "length {one}");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn f16_round_trip_rejects_a_wrong_length_blob() {
|
||||
assert!(Embedding::from_f16_bytes(ModelId::new("m"), &[0u8; 100]).is_none());
|
||||
|
||||
@@ -75,8 +75,10 @@ pub use cluster::{
|
||||
#[cfg(feature = "inference")]
|
||||
pub use detect::{DetectOptions, Detection, Detector};
|
||||
#[cfg(feature = "inference")]
|
||||
pub use embed::Embedder;
|
||||
pub use embedding::{Embedding, ModelId, EMBEDDING_DIM};
|
||||
pub use embed::{Embedded, Embedder};
|
||||
pub use embedding::{
|
||||
in_gallery, read_f16_bytes, Embedding, ModelId, EMBEDDING_DIM, MIN_GALLERY_QUALITY,
|
||||
};
|
||||
pub use naming::{name_for_instance, name_instances, NamedFace};
|
||||
|
||||
/// What can go wrong between an image and a face.
|
||||
|
||||
@@ -117,6 +117,17 @@ pub struct Faces<'a> {
|
||||
/// Which photograph each face came from. Two faces in one frame are not
|
||||
/// the same person, so those pairs are never returned (docs/faces.md §9).
|
||||
pub images: &'a [u64],
|
||||
/// Which faces may be compared *against* — the gallery
|
||||
/// ([`crate::embedding::MIN_GALLERY_QUALITY`]).
|
||||
///
|
||||
/// A pair needs at least one gallery side: a probe measured against a
|
||||
/// reference is a comparison, two short vectors measured against each
|
||||
/// other is noise agreeing with noise, and those pairs are never returned.
|
||||
/// Filtered here rather than by the caller for the same reason
|
||||
/// co-occurrence is: what this module leaves out of the list stays out of
|
||||
/// the graph, the components and the merge order, so nothing downstream
|
||||
/// has to remember the rule.
|
||||
pub gallery: &'a [bool],
|
||||
}
|
||||
|
||||
impl Faces<'_> {
|
||||
@@ -272,8 +283,9 @@ fn scan_rows(scan: &Scan, from: usize, to: usize, out: &mut Vec<Pair>) {
|
||||
let a = faces.row(i);
|
||||
let crop_a = faces.crop_px[i];
|
||||
let image_a = faces.images[i];
|
||||
let gallery_a = faces.gallery[i];
|
||||
for j in start..tile_end {
|
||||
if image_a == faces.images[j] {
|
||||
if image_a == faces.images[j] || !(gallery_a || faces.gallery[j]) {
|
||||
continue;
|
||||
}
|
||||
let cos = dot(a, faces.row(j));
|
||||
@@ -552,6 +564,7 @@ mod tests {
|
||||
embeddings: Vec<f32>,
|
||||
crop_px: Vec<f32>,
|
||||
images: Vec<u64>,
|
||||
gallery: Vec<bool>,
|
||||
}
|
||||
|
||||
impl Set {
|
||||
@@ -561,6 +574,7 @@ mod tests {
|
||||
dim: DIM,
|
||||
crop_px: &self.crop_px,
|
||||
images: &self.images,
|
||||
gallery: &self.gallery,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -588,10 +602,12 @@ mod tests {
|
||||
}
|
||||
}
|
||||
let crop_px = vec![150.0; embeddings.len()];
|
||||
let gallery = vec![true; embeddings.len()];
|
||||
Set {
|
||||
embeddings: embeddings.concat(),
|
||||
crop_px,
|
||||
images,
|
||||
gallery,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -601,7 +617,7 @@ mod tests {
|
||||
let mut out = Vec::new();
|
||||
for i in 0..n {
|
||||
for j in i + 1..n {
|
||||
if faces.images[i] == faces.images[j] {
|
||||
if faces.images[i] == faces.images[j] || !(faces.gallery[i] || faces.gallery[j]) {
|
||||
continue;
|
||||
}
|
||||
let cos: f32 = faces
|
||||
@@ -750,6 +766,35 @@ mod tests {
|
||||
assert!(above_threshold(&s.faces(), &cal(), 0.9).is_empty());
|
||||
}
|
||||
|
||||
/// A probe against a reference is a comparison; two probes against each
|
||||
/// other is not. The rule lives here so that nothing downstream sees the
|
||||
/// pair at all.
|
||||
#[test]
|
||||
fn two_faces_outside_the_gallery_are_never_paired() {
|
||||
let mut s = population(1, 3, 1.0);
|
||||
s.gallery = vec![false, false, true];
|
||||
let pairs = above_threshold(&s.faces(), &cal(), 0.9);
|
||||
assert!(
|
||||
!pairs.iter().any(|p| p.i == 0 && p.j == 1),
|
||||
"two probes were paired with each other"
|
||||
);
|
||||
// Each probe is still measured against the one reference.
|
||||
assert!(pairs.iter().any(|p| p.i == 0 && p.j == 2));
|
||||
assert!(pairs.iter().any(|p| p.i == 1 && p.j == 2));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn the_gallery_rule_matches_the_reference_at_scale() {
|
||||
let mut s = population(60, 8, 0.97);
|
||||
for (i, g) in s.gallery.iter_mut().enumerate() {
|
||||
*g = i % 3 != 0;
|
||||
}
|
||||
let f = s.faces();
|
||||
let got = above_threshold(&f, &cal(), 0.9);
|
||||
let want = reference(&f, &cal(), 0.9);
|
||||
assert!(same_pairs(&got, &want), "{} vs {}", got.len(), want.len());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn pairs_come_back_in_index_order() {
|
||||
let s = population(300, 8, 0.97);
|
||||
|
||||
Reference in New Issue
Block a user