dtourolle/DarkRoom
1
0
Fork 0
Code Issues 39 Pull Requests Actions Packages Projects Releases 20 Wiki Activity

Keep each face's quality, and never compare against a poor one

Browse Source 
The embedder's raw output has a length, and the length is a reading of
how recognisable the crop was: a blur, an occlusion or a hard profile
comes out short. Normalising threw it away. A short vector sits near
the middle of the sphere and matches a little of everyone, which is how
one bad crop bridges two people in a grouping pass.

So the length is kept — the store now holds the raw vector, re-normalised
on load, with the length beside it as `faces.quality` — and a face under
MIN_GALLERY_QUALITY (14) is a probe: measured against the gallery and
placed where it fits, but never what another face is measured against.
Two probes are never paired, and a probe is nobody's evidence for a
confidence. The People screen shows the number as "Quality 17.3", dimmed
below the floor.

Faces indexed before this stored unit vectors and have no reading; they
are admitted to the gallery, and schema V14 forgets the run marker of
every image holding one so the next indexing pass measures them. A
peer's unmeasured shard faces are not adopted, or a sync would write
that marker back.
This commit is contained in:
dtourolle
2026-09-11 21:50:12 +02:00
parent a87139b838
commit 8b3abdb787
22 changed files with 1070 additions and 149 deletions
Download Patch File Download Diff File Expand all files Collapse all files
core/dr-catalog/examples/face_confidence.rs
+23 -15
View File
@@ -34,6 +34,10 @@ struct Known {
crop_px: f32,
}
/// What the catalog holds per face, decoded: photograph, vector, size,
/// quality.
type Decoded = (u64, Vec<f32>, f32, Option<f32>);
fn main() {
let args: Vec<String> = std::env::args().skip(1).collect();
let Some(path) = args.first() else {
@@ -59,10 +63,10 @@ fn main() {
let model = dr_face::ModelId::new(MODEL_ID.to_string());
let stored = faces::embeddings(conn, MODEL_ID).expect("embeddings");
let mut embedding_of = HashMap::new();
for (id, image, blob, crop_px) in stored {
if let Some(e) = dr_face::Embedding::from_f16_bytes(model.clone(), &blob) {
embedding_of.insert(id, (image.0, e.v.to_vec(), crop_px));
let mut embedding_of: HashMap<faces::FaceId, Decoded> = HashMap::new();
for f in stored {
if let Some(e) = dr_face::Embedding::from_f16_bytes(model.clone(), &f.embedding) {
embedding_of.insert(f.face, (f.image.0, e.v.to_vec(), f.crop_px, f.quality));
}
}
println!("faces with embeddings: {}", embedding_of.len());
@@ -82,7 +86,7 @@ fn main() {
if !f.confirmed {
continue;
}
if let Some((image, embedding, crop_px)) = embedding_of.get(&f.id) {
if let Some((image, embedding, crop_px, _)) = embedding_of.get(&f.id) {
mine.push(Known {
image: *image,
person: p.id,
@@ -288,19 +292,22 @@ fn band(label: &str, v: &[f32]) {
/// The whole library through the real clusterer, for the numbers it would
/// actually write.
fn full_library(
embedding_of: &HashMap<faces::FaceId, (u64, Vec<f32>, f32)>,
embedding_of: &HashMap<faces::FaceId, Decoded>,
confirmed: &HashMap<faces::FaceId, u64>,
cal: &dr_face::Calibration,
) {
let mut candidates: Vec<dr_face::Candidate> = embedding_of
.iter()
.map(|(id, (image, embedding, crop_px))| dr_face::Candidate {
face: id.0,
image: *image,
embedding: embedding.clone(),
crop_px: *crop_px,
confirmed_person: confirmed.get(id).copied(),
})
.map(
|(id, (image, embedding, crop_px, quality))| dr_face::Candidate {
face: id.0,
image: *image,
embedding: embedding.clone(),
crop_px: *crop_px,
quality: *quality,
confirmed_person: confirmed.get(id).copied(),
},
)
.collect();
candidates.sort_by_key(|c| c.face);
@@ -317,11 +324,13 @@ fn full_library(
.collect();
let crop_px: Vec<f32> = candidates.iter().map(|c| c.crop_px).collect();
let images: Vec<u64> = candidates.iter().map(|c| c.image).collect();
let gallery: Vec<bool> = candidates.iter().map(|c| c.in_gallery()).collect();
let view = dr_face::neighbours::Faces {
embeddings: &flat,
dim,
crop_px: &crop_px,
images: &images,
gallery: &gallery,
};
let t = std::time::Instant::now();
@@ -335,8 +344,7 @@ fn full_library(
let agglomerate = t.elapsed().as_secs_f64() - scan;
let t = std::time::Instant::now();
let _ =
dr_face::identity_shares(candidates.len(), &clusters, &evidence, dr_face::TOP_MATCHES);
let _ = dr_face::identity_shares(&gallery, &clusters, &evidence, dr_face::TOP_MATCHES);
println!(
" scan {scan:.2}s ({} evidence pairs) · agglomerate {agglomerate:.2}s · score {:.2}s",
evidence.len(),
core/dr-catalog/src/face_shard.rs
+89 -17
View File
@@ -76,6 +76,9 @@ pub struct SharedFace {
pub confidence: f32,
pub embedding: Vec<u8>,
pub crop_px: f32,
/// See `faces::DetectedFace::quality`. `None` from a shard written before
/// the number was kept.
pub quality: Option<f32>,
/// The face cut out and encoded, or empty where none was kept.
///
/// Travels with the face rather than in the catalog snapshot, which is the
@@ -224,8 +227,8 @@ impl FaceShardStore {
tx.execute(
"INSERT INTO faces
(file_id, model_id, x, y, w, h, landmarks, confidence,
embedding, crop_px, crop)
VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, ?11)",
embedding, crop_px, crop, quality)
VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, ?11, ?12)",
rusqlite::params![
f.file_id as i64,
f.model_id,
@@ -238,6 +241,7 @@ impl FaceShardStore {
f.embedding,
f.crop_px as f64,
(!f.crop.is_empty()).then_some(f.crop.as_slice()),
f.quality.map(f64::from),
],
)?;
}
@@ -442,9 +446,10 @@ impl FaceShardStore {
}
let mut fq = src.prepare(&format!(
"SELECT f.file_id, f.model_id, f.x, f.y, f.w, f.h, f.landmarks,
f.confidence, f.embedding, f.crop_px, {}
f.confidence, f.embedding, f.crop_px, {}, {}
FROM faces f WHERE f.file_id = ?1 AND f.model_id = ?2",
crop_column(&src)
column_or_null(&src, "crop"),
column_or_null(&src, "quality"),
))?;
let faces: Vec<SharedFace> = fq
.query_map(rusqlite::params![file_id, &model_id], read_shared_face)?
@@ -484,7 +489,8 @@ impl FaceShardStore {
let Some(edge) = edge else { return Ok(None) };
let mut q = conn.prepare(
"SELECT file_id, model_id, x, y, w, h, landmarks, confidence, embedding, crop_px, crop
"SELECT file_id, model_id, x, y, w, h, landmarks, confidence, embedding, crop_px,
crop, quality
FROM faces WHERE file_id = ?1 AND model_id = ?2",
)?;
let faces: Vec<SharedFace> = q
@@ -541,6 +547,7 @@ fn upgrade_shard(conn: &Connection) -> Result<(), CatalogError> {
for (table, column, decl) in [
("faces", "crop", "BLOB"),
("indexed", "indexed_at", "INTEGER"),
("faces", "quality", "REAL"),
] {
if !has_column(conn, table, column)? {
conn.execute_batch(&format!("ALTER TABLE {table} ADD COLUMN {column} {decl}"))?;
@@ -555,16 +562,19 @@ fn has_column(conn: &Connection, table: &str, column: &str) -> Result<bool, Cata
Ok(stmt.exists(rusqlite::params![table, column])?)
}
/// `f.crop`, or a `NULL` standing in for it.
/// `f.<column>`, or a `NULL` standing in for it.
///
/// A shard downloaded from a peer is opened **read-only** and cannot be
/// upgraded, so one written before crops existed has to be read as it is rather
/// than repaired. Selecting a literal keeps the column count the same, which is
/// what lets [`read_shared_face`] stay a single function.
fn crop_column(conn: &Connection) -> &'static str {
match has_column(conn, "faces", "crop") {
Ok(true) => "f.crop",
_ => "NULL",
/// upgraded, so one written before a column existed has to be read as it is
/// rather than repaired. Selecting a literal keeps the column count the same,
/// which is what lets [`read_shared_face`] stay a single function.
///
/// `column` is one of this module's own names, never anything read from
/// outside, which is what makes formatting it into SQL acceptable.
fn column_or_null(conn: &Connection, column: &'static str) -> String {
match has_column(conn, "faces", column) {
Ok(true) => format!("f.{column}"),
_ => "NULL".to_string(),
}
}
@@ -637,7 +647,8 @@ pub fn export_to_shards_reporting(
continue;
}
let mut fq = conn.prepare(
"SELECT x, y, w, h, landmarks, detector_confidence, embedding, crop_px, crop
"SELECT x, y, w, h, landmarks, detector_confidence, embedding, crop_px, crop,
quality
FROM faces WHERE image_id = ?1 AND model_id = ?2",
)?;
let faces: Vec<SharedFace> = fq
@@ -654,6 +665,7 @@ pub fn export_to_shards_reporting(
embedding: r.get(6)?,
crop_px: r.get::<_, f64>(7)? as f32,
crop: r.get::<_, Option<Vec<u8>>>(8)?.unwrap_or_default(),
quality: r.get::<_, Option<f64>>(9)?.map(|q| q as f32),
})
})?
.collect::<Result<_, _>>()?;
@@ -710,6 +722,14 @@ pub fn import_from_shards(
let Some((faces, edge)) = store.get_image(file_id as u64, model_id)? else {
continue;
};
// A peer that embedded before the quality was kept has done work this
// device cannot finish: the number exists only at embedding time, and
// adopting the faces would write the run marker that keeps them from
// ever being measured (schema V14). Left for this device's own pass —
// or for the peer's, whose re-export replaces these.
if faces.iter().any(|f| f.quality.is_none()) {
continue;
}
let local: Vec<crate::faces::DetectedFace> = faces
.into_iter()
.map(|f| crate::faces::DetectedFace {
@@ -721,6 +741,7 @@ pub fn import_from_shards(
confidence: f.confidence,
embedding: f.embedding,
crop_px: f.crop_px,
quality: f.quality,
model_id: f.model_id,
// A peer that indexed before crops existed sends none, and the
// reader falls back to the proxy exactly as it does for a face
@@ -766,6 +787,7 @@ fn read_shared_face(r: &rusqlite::Row<'_>) -> rusqlite::Result<SharedFace> {
embedding: r.get(8)?,
crop_px: r.get::<_, f64>(9)? as f32,
crop: r.get::<_, Option<Vec<u8>>>(10)?.unwrap_or_default(),
quality: r.get::<_, Option<f64>>(11)?.map(|q| q as f32),
})
}
@@ -849,7 +871,11 @@ CREATE TABLE IF NOT EXISTS faces (
crop_px REAL NOT NULL,
-- The face, cut out. NULL where the face was found before crops were kept,
-- or adopted from a peer that did not have one.
crop BLOB
crop BLOB,
-- Length of the raw embedding (`faces::DetectedFace::quality`). NULL from
-- a build that did not keep it, and a face the receiving device will not
-- adopt -- see `import_from_shards`.
quality REAL
);
CREATE INDEX IF NOT EXISTS faces_file ON faces(file_id, model_id);
@@ -908,6 +934,7 @@ mod tests {
confidence: 0.87,
embedding: vec![seed; 1024],
crop_px: 180.0,
quality: Some(17.5),
crop: vec![seed; 64],
}
}
@@ -925,6 +952,7 @@ mod tests {
assert_eq!(edge, 1024);
assert_eq!(faces[0].embedding.len(), 1024);
assert!((faces[0].crop_px - 180.0).abs() < 1e-3);
assert_eq!(faces[0].quality, Some(17.5));
}
/// The case the run marker exists for, carried across the wire: an image
@@ -1115,6 +1143,7 @@ mod catalog_round_trip {
confidence: 0.9,
embedding: vec![seed; 1024],
crop_px: 180.0,
quality: Some(20.0),
model_id: "w600k_mbf".into(),
crop: vec![seed; 64],
}
@@ -1162,9 +1191,47 @@ mod catalog_round_trip {
let got = faces::for_image(&b, dr_types::ImageId(90)).unwrap();
assert_eq!(got.len(), 1);
assert!((got[0].crop_px - 180.0).abs() < 1e-3);
assert_eq!(got[0].quality, Some(20.0));
assert!((got[0].landmarks[2].0 - 0.15).abs() < 1e-5);
let emb = faces::embeddings(&b, "w600k_mbf").unwrap();
assert!(emb.iter().any(|(_, _, blob, _)| blob[0] == 1));
assert!(emb.iter().any(|e| e.embedding[0] == 1));
}
/// A face a peer embedded without measuring it is work this device
/// cannot finish, and adopting it would write the marker that stops it
/// ever being measured. The image stays outstanding instead.
#[test]
fn a_peers_unmeasured_faces_are_left_for_this_device_to_index() {
let b = device(&[(90, 5001), (91, 5002)]);
let mut store = FaceShardStore::open(&tempdir("unmeasured")).unwrap();
let shared = |file_id: u64, quality: Option<f32>| SharedFace {
file_id,
model_id: "w600k_mbf".into(),
x: 0.1,
y: 0.2,
w: 0.15,
h: 0.2,
landmarks: vec![1; 40],
confidence: 0.87,
embedding: vec![1; 1024],
crop_px: 180.0,
quality,
crop: Vec::new(),
};
store
.put_image(5001, "w600k_mbf", 2560, &[shared(5001, None)])
.unwrap();
store
.put_image(5002, "w600k_mbf", 2560, &[shared(5002, Some(19.0))])
.unwrap();
assert_eq!(import_from_shards(&b, &store, "w600k_mbf").unwrap(), 1);
let cov = faces::coverage(&b, "w600k_mbf").unwrap();
assert_eq!(cov.indexed, 1);
assert_eq!(cov.outstanding(), 1, "the unmeasured image was adopted");
assert!(faces::for_image(&b, dr_types::ImageId(90))
.unwrap()
.is_empty());
}
#[test]
@@ -1187,7 +1254,7 @@ mod catalog_round_trip {
"a peer's copy replaced work this device had already done"
);
let emb = faces::embeddings(&b, "w600k_mbf").unwrap();
assert_eq!(emb[0].2[0], 9, "B's own embedding was overwritten");
assert_eq!(emb[0].embedding[0], 9, "B's own embedding was overwritten");
}
/// A device holding a subset of the library takes only its own part.
@@ -1281,6 +1348,7 @@ mod catalog_round_trip {
confidence: 0.87,
embedding: vec![seed; 1024],
crop_px: 180.0,
quality: None,
crop: vec![seed; 64],
}
}
@@ -1430,6 +1498,10 @@ mod catalog_round_trip {
let (faces, _) = store.get_image(77, "w600k_mbf").unwrap().unwrap();
assert_eq!(faces.len(), 1);
assert!(faces[0].crop.is_empty(), "a crop was invented from nowhere");
assert_eq!(
faces[0].quality, None,
"a quality was invented from nowhere"
);
let _ = std::fs::remove_dir_all(&dir);
}
core/dr-catalog/src/faces.rs
+57 -19
View File
@@ -61,10 +61,22 @@ pub struct DetectedFace {
/// Five `(x, y)` pairs, normalised the same way.
pub landmarks: [(f32, f32); 5],
pub confidence: f32,
/// 512 × f16, L2-normalised — `dr_face::Embedding::to_f16_bytes`.
/// 512 × f16, the raw model output — `dr_face::Embedded::to_f16_bytes`.
///
/// Raw rather than unit length, so the length ([`Self::quality`]) is in
/// the blob and not only beside it. Readers re-normalise on load.
pub embedding: Vec<u8>,
/// Source pixels across the aligned crop (docs/faces.md §7).
pub crop_px: f32,
/// Length of the raw embedding before normalisation — the model's own
/// reading of how recognisable the crop was, and the gate on whether
/// this face may be compared *against* (`dr_face::MIN_GALLERY_QUALITY`).
///
/// `None` where it was never measured: a face indexed, here or by a peer,
/// before raw vectors were stored. The unit vector those builds kept has
/// no length left to read, so the only way to measure one is to embed it
/// again (schema V14).
pub quality: Option<f32>,
/// Which model produced the embedding. Comparing across models is the one
/// mistake that yields plausible garbage rather than an error.
pub model_id: String,
@@ -94,6 +106,9 @@ pub struct Face {
pub landmarks: [(f32, f32); 5],
pub confidence: f32,
pub crop_px: f32,
/// See [`DetectedFace::quality`]. `None` for a face indexed before it was
/// recorded.
pub quality: Option<f32>,
pub model_id: String,
/// `None` when the face belongs to no one yet.
pub person: Option<PersonId>,
@@ -184,8 +199,8 @@ pub fn record_detections(
tx.execute(
"INSERT INTO faces
(image_id, x, y, w, h, landmarks, detector_confidence,
embedding, crop_px, model_id, detected_at, crop)
VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, ?11, ?12)",
embedding, crop_px, model_id, detected_at, crop, quality)
VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, ?11, ?12, ?13)",
rusqlite::params![
image_id.0 as i64,
f.x as f64,
@@ -202,6 +217,7 @@ pub fn record_detections(
// the database rather than two the readers each have to know
// about.
(!f.crop.is_empty()).then_some(f.crop.as_slice()),
f.quality.map(f64::from),
],
)?;
let id = FaceId(tx.last_insert_rowid() as u64);
@@ -366,7 +382,7 @@ pub fn for_image(conn: &Connection, image_id: ImageId) -> Result<Vec<Face>, Cata
let mut q = conn.prepare(
"SELECT f.id, f.image_id, f.x, f.y, f.w, f.h, f.landmarks,
f.detector_confidence, f.crop_px, f.model_id,
fp.person_id, fp.probability, fp.confirmed
fp.person_id, fp.probability, fp.confirmed, f.quality
FROM faces f
LEFT JOIN face_person fp ON fp.face_id = f.id
WHERE f.image_id = ?1
@@ -393,9 +409,18 @@ pub fn unassigned(conn: &Connection, model_id: &str) -> Result<Vec<FaceId>, Cata
/// One face's stored embedding, as the clustering pass consumes it.
///
/// A named type rather than a tuple because it crosses a crate boundary and
/// "the third element" is not a thing anyone should have to remember.
pub type StoredEmbedding = (FaceId, ImageId, Vec<u8>, f32);
/// A struct rather than a tuple because it crosses a crate boundary and "the
/// fourth element" is not a thing anyone should have to remember.
#[derive(Debug, Clone, PartialEq)]
pub struct StoredEmbedding {
pub face: FaceId,
pub image: ImageId,
/// 512 × f16 — `dr_face::Embedding::from_f16_bytes` reads it.
pub embedding: Vec<u8>,
pub crop_px: f32,
/// See [`DetectedFace::quality`].
pub quality: Option<f32>,
}
/// Embeddings for clustering, oldest first so the pass is deterministic.
///
@@ -404,16 +429,17 @@ pub type StoredEmbedding = (FaceId, ImageId, Vec<u8>, f32);
/// would double the memory of the one operation that holds them all at once.
pub fn embeddings(conn: &Connection, model_id: &str) -> Result<Vec<StoredEmbedding>, CatalogError> {
let mut q = conn.prepare(
"SELECT id, image_id, embedding, crop_px FROM faces
"SELECT id, image_id, embedding, crop_px, quality FROM faces
WHERE model_id = ?1 ORDER BY id",
)?;
let rows = q.query_map([model_id], |r| {
Ok((
FaceId(r.get::<_, i64>(0)? as u64),
ImageId(r.get::<_, i64>(1)? as u64),
r.get::<_, Vec<u8>>(2)?,
r.get::<_, f64>(3)? as f32,
))
Ok(StoredEmbedding {
face: FaceId(r.get::<_, i64>(0)? as u64),
image: ImageId(r.get::<_, i64>(1)? as u64),
embedding: r.get::<_, Vec<u8>>(2)?,
crop_px: r.get::<_, f64>(3)? as f32,
quality: r.get::<_, Option<f64>>(4)?.map(|q| q as f32),
})
})?;
rows.collect::<Result<_, _>>().map_err(Into::into)
}
@@ -643,7 +669,7 @@ pub fn for_person(
let mut q = conn.prepare(
"SELECT f.id, f.image_id, f.x, f.y, f.w, f.h, f.landmarks,
f.detector_confidence, f.crop_px, f.model_id,
fp.person_id, fp.probability, fp.confirmed
fp.person_id, fp.probability, fp.confirmed, f.quality
FROM faces f
JOIN face_person fp ON fp.face_id = f.id
WHERE fp.person_id = ?1 AND (?2 OR fp.confirmed = 1)
@@ -894,6 +920,7 @@ fn read_face(r: &rusqlite::Row<'_>) -> rusqlite::Result<Face> {
landmarks: blob_to_landmarks(&r.get::<_, Vec<u8>>(6)?),
confidence: r.get::<_, f64>(7)? as f32,
crop_px: r.get::<_, f64>(8)? as f32,
quality: r.get::<_, Option<f64>>(13)?.map(|q| q as f32),
model_id: r.get(9)?,
person: person.map(|p| PersonId(p as u64)),
probability: r.get::<_, Option<f64>>(11)?.unwrap_or(0.0) as f32,
@@ -1016,6 +1043,7 @@ mod tests {
confidence: 0.9,
embedding: vec![seed; 1024],
crop_px: 180.0,
quality: Some(10.0 + f32::from(seed)),
model_id: "w600k_mbf".into(),
crop: Vec::new(),
}
@@ -1041,6 +1069,15 @@ mod tests {
assert!((got[0].crop_px - 180.0).abs() < 1e-3);
assert!((got[0].landmarks[2].1 - 0.2).abs() < 1e-5);
assert!(got[0].person.is_none());
// Both readers carry the quality, and the one for the grouping pass
// carries it as the option it is.
let mut qualities: Vec<Option<f32>> = got.iter().map(|f| f.quality).collect();
qualities.sort_by(|a, b| a.partial_cmp(b).unwrap());
assert_eq!(qualities, vec![Some(11.0), Some(12.0)]);
let stored = embeddings(&c, "w600k_mbf").unwrap();
assert_eq!(stored.len(), 2);
assert_eq!(stored[0].quality, Some(11.0), "oldest first");
assert_eq!(stored[1].quality, Some(12.0));
}
/// Re-detection is coalesced per image, so it must replace rather than
@@ -1450,10 +1487,11 @@ mod tests {
record_detections(&c, img, "w600k_mbf", 1024, &[face(7)]).unwrap();
let e = embeddings(&c, "w600k_mbf").unwrap();
assert_eq!(e.len(), 1);
assert_eq!(e[0].1, img);
assert_eq!(e[0].2.len(), 1024);
assert_eq!(e[0].2[0], 7);
assert!((e[0].3 - 180.0).abs() < 1e-3);
assert_eq!(e[0].image, img);
assert_eq!(e[0].embedding.len(), 1024);
assert_eq!(e[0].embedding[0], 7);
assert!((e[0].crop_px - 180.0).abs() < 1e-3);
assert_eq!(e[0].quality, Some(17.0));
}
// ── stored crops ──────────────────────────────────────────────────────
core/dr-catalog/src/schema.rs
+128 -3
View File
@@ -15,7 +15,7 @@ use rusqlite::Connection;
use crate::error::CatalogError;
/// Schema version this build writes and understands.
pub const SCHEMA_VERSION: i64 = 13;
pub const SCHEMA_VERSION: i64 = 14;
/// Apply migrations up to [`SCHEMA_VERSION`].
///
@@ -119,6 +119,23 @@ pub fn migrate(conn: &Connection) -> Result<i64, CatalogError> {
tx.commit()?;
}
if from < 14 {
let tx = conn.unchecked_transaction()?;
// `ALTER TABLE ... ADD COLUMN` has no `IF NOT EXISTS`, and NFR-R5
// wants this re-enterable: a catalog whose `user_version` was rewound
// by a rollback already has the column, and would otherwise fail its
// next open on it.
let has_quality: bool = tx
.prepare("SELECT 1 FROM pragma_table_info('faces') WHERE name = 'quality'")?
.exists([])?;
if !has_quality {
tx.execute_batch("ALTER TABLE faces ADD COLUMN quality REAL;")?;
}
tx.execute_batch(V14)?;
tx.pragma_update(None, "user_version", 14)?;
tx.commit()?;
}
Ok(from)
}
@@ -251,7 +268,8 @@ pub fn for_attached(schema_name: &str) -> String {
format!(
"{}\n{}\n{}\n\
ALTER TABLE {schema_name}.people ADD COLUMN ignored INTEGER NOT NULL DEFAULT 0;\n\
ALTER TABLE {schema_name}.faces ADD COLUMN crop BLOB;",
ALTER TABLE {schema_name}.faces ADD COLUMN crop BLOB;\n\
ALTER TABLE {schema_name}.faces ADD COLUMN quality REAL;",
rewrite_for_attached(V1, schema_name),
rewrite_for_attached(V6, schema_name),
rewrite_for_attached(V8, schema_name),
@@ -569,6 +587,55 @@ CREATE TABLE IF NOT EXISTS sidecars (
);
"#;
const V14: &str = r#"
-- TRACES: FR-CULL-9 | FR-CULL-10
-- How recognisable the model found each face, and a second look at the faces
-- it was never asked about.
--
-- The embedder's raw output has a length, and the length is a quality
-- reading: it grows with how much of a face the model could make out, and a
-- blur, an occlusion or a hard profile comes out short (dr_face::embedding,
-- `MIN_GALLERY_QUALITY`). Normalising threw it away. A short vector sits
-- near the middle of the sphere and matches a little of everyone, which is
-- how one bad crop bridges two people in a grouping pass -- so a face below
-- the floor is compared against the others and never compared *against*.
--
-- Nullable, and NULL means "never measured": every face indexed before this
-- version stored the unit vector, whose length is one whatever the crop was.
-- A face with no reading is admitted to the gallery, because a rule that
-- cannot be checked should admit rather than exclude -- but it is also a
-- face this rule is not yet protecting anyone from, and the only way to
-- measure it is to embed it again.
--
-- So the run markers of every image that holds a face are forgotten, exactly
-- as V12 forgot the runs made against too small a proxy, and for the same
-- reason: the work list is "images with no marker", and an image examined
-- without this number is otherwise indistinguishable from one examined with
-- it. The faces stay where they are and keep drawing the People screen until
-- the next pass replaces them, `record_detections` carries the user's
-- confirmations across by box overlap, and an image that was examined and
-- found empty keeps its marker -- there is nothing on it to re-measure.
--
-- The cost is a re-fetch of every image with a face on it, on the next pass
-- the user starts. That is a whole-library transfer (FR-NC-6), and it starts
-- when they say so, not here.
--
-- From this version the `embedding` blob is the **raw** model output rather
-- than the unit vector V8 describes -- the length is the quality, and a store
-- that kept only the direction had thrown it away. Readers re-normalise on
-- load, so a unit blob from before and a raw blob from now compare alike;
-- `quality` is that length kept beside the blob for the readers that never
-- load the vector, and NULL rather than 1.0 for the old rows, because a unit
-- vector reads as a length of one and one is not "unmeasured".
--
-- The column itself is added in `migrate`, guarded, because ALTER has no
-- IF NOT EXISTS and this step has to be re-enterable (NFR-R5).
DELETE FROM face_index
WHERE EXISTS (SELECT 1 FROM faces f
WHERE f.image_id = face_index.image_id
AND f.model_id = face_index.model_id);
"#;
const V9: &str = r#"
-- TRACES: FR-CULL-8
-- A record that face detection has *run* on an image, distinct from what it
@@ -648,7 +715,7 @@ CREATE TABLE faces (
x REAL NOT NULL, y REAL NOT NULL, w REAL NOT NULL, h REAL NOT NULL,
landmarks BLOB NOT NULL, -- 5 x (x, y) f32, normalised likewise
detector_confidence REAL NOT NULL,
embedding BLOB NOT NULL, -- 512 x f16, L2-normalised
embedding BLOB NOT NULL, -- 512 x f16; unit length until V14, raw since
-- Source pixels across the aligned 112x112 crop (docs/faces.md §7).
--
-- Not cosmetic: it is the honest quality signal for the UI, a feature in
@@ -1405,6 +1472,64 @@ mod tests {
assert_eq!(kept, vec![3, 4]);
}
#[test]
fn v14_forgets_runs_that_found_faces_but_never_measured_them() {
let c = mem();
c.pragma_update(None, "user_version", 0).unwrap();
migrate(&c).unwrap();
c.execute(
"INSERT INTO roots(id, kind, label) VALUES (1, 'local', 'test')",
[],
)
.unwrap();
c.execute(
"INSERT INTO images(id, root_id, source_ref, added_at)
VALUES (1,1,'a',0),(2,1,'b',0),(3,1,'c',0)",
[],
)
.unwrap();
// Image 1 was examined and holds a face; 2 was examined and found
// empty; 3 holds a face found by a different model.
for (image, model) in [(1, "m"), (2, "m"), (3, "m")] {
c.execute(
"INSERT INTO face_index(image_id, model_id, indexed_at, faces_found, source_edge)
VALUES (?1, ?2, 0, 0, 2560)",
rusqlite::params![image, model],
)
.unwrap();
}
for (image, model) in [(1, "m"), (3, "other")] {
c.execute(
"INSERT INTO faces
(image_id, x, y, w, h, landmarks, detector_confidence, embedding,
crop_px, model_id, detected_at)
VALUES (?1, 0.1, 0.1, 0.2, 0.2, X'00', 0.9, X'00', 180.0, ?2, 0)",
rusqlite::params![image, model],
)
.unwrap();
}
c.pragma_update(None, "user_version", 13).unwrap();
migrate(&c).unwrap();
let kept: Vec<i64> = c
.prepare("SELECT image_id FROM face_index ORDER BY image_id")
.unwrap()
.query_map([], |r| r.get(0))
.unwrap()
.map(Result::unwrap)
.collect();
// 1 goes: it has a face with no quality. 2 stays: nothing on it to
// measure. 3 stays: its face belongs to a run this marker does not
// describe.
assert_eq!(kept, vec![2, 3]);
// And the faces themselves are untouched.
let faces: i64 = c
.query_row("SELECT count(*) FROM faces", [], |r| r.get(0))
.unwrap();
assert_eq!(faces, 2);
}
#[test]
fn job_uniqueness_coalesces_rather_than_duplicating() {
let c = mem();
core/dr-face/examples/faces.rs
+3 -2
View File
@@ -63,15 +63,16 @@ fn main() {
let embed_ms = t.elapsed().as_secs_f64() * 1e3;
println!(
" [{i}] conf {:.3} box {:.0},{:.0} {:.0}×{:.0} crop_px {:.0} embed {embed_ms:.0} ms",
" [{i}] conf {:.3} box {:.0},{:.0} {:.0}×{:.0} crop_px {:.0} quality {:.1} embed {embed_ms:.0} ms",
d.confidence,
d.bbox.0,
d.bbox.1,
d.width(),
d.height(),
aligned.source_px(),
emb.quality,
);
all.push((path.clone(), i, emb));
all.push((path.clone(), i, emb.embedding));
}
}
core/dr-face/examples/scan_bench.rs
+2
View File
@@ -46,11 +46,13 @@ fn main() {
);
for n in sizes {
let (embeddings, crop_px, images) = population(n);
let gallery = vec![true; n];
let faces = Faces {
embeddings: &embeddings,
dim: EMBEDDING_DIM,
crop_px: &crop_px,
images: &images,
gallery: &gallery,
};
let start = std::time::Instant::now();
core/dr-face/src/assign.rs
+48 -11
View File
@@ -136,11 +136,24 @@ pub const RIVAL_FLOOR: f32 = 0.5;
///
/// A face in no group, or one with no evidence for anybody, scores 0.
///
/// `gallery` is one flag per face — which faces may be evidence at all
/// ([`crate::embedding::MIN_GALLERY_QUALITY`]). Its length is the face count.
/// A pair is evidence *about* either face but only *from* a gallery one: a
/// probe learns from the references it matched, and a reference learns nothing
/// from a probe that happened to match it, however well. Without that, the one
/// short vector in a group would be the strongest match every face in it had.
///
/// `pairs` must be the *evidence* list — scanned at [`RIVAL_FLOOR`], not at the
/// merge threshold. Passing the merge list still works but silently removes
/// every rival weaker than a merge, which is most of them, and every uniqueness
/// collapses to 1.
pub fn identity_shares(faces: usize, clusters: &[Cluster], pairs: &[Pair], top: usize) -> Vec<f32> {
pub fn identity_shares(
gallery: &[bool],
clusters: &[Cluster],
pairs: &[Pair],
top: usize,
) -> Vec<f32> {
let faces = gallery.len();
// An identity is a *person*, not a group. One person routinely holds
// several anchored groups — the same reason they hold several unnamed ones
// — and keying this by group had Catherine competing with Catherine, which
@@ -174,15 +187,16 @@ pub fn identity_shares(faces: usize, clusters: &[Cluster], pairs: &[Pair], top:
}
// A pair is evidence in both directions: j's identity hears about i,
// and i's identity hears about j. The pair list holds each unordered
// pair once, so both have to be recorded here.
// pair once, so both have to be recorded here — each only where the
// face doing the telling is in the gallery.
let (gi, gj) = (group_of[p.i], group_of[p.j]);
if gj != usize::MAX {
if gj != usize::MAX && gallery[p.j] {
evidence[p.i]
.entry(key_of[gj])
.or_default()
.push(p.probability);
}
if gi != usize::MAX {
if gi != usize::MAX && gallery[p.i] {
evidence[p.j]
.entry(key_of[gi])
.or_default()
@@ -255,6 +269,11 @@ mod tests {
Pair { i, j, probability }
}
/// `n` faces, every one of them fit to be compared against.
fn all(n: usize) -> Vec<bool> {
vec![true; n]
}
/// The failure the module exists to fix: face 0 matches its own group's
/// three members strongly, and the group has forty more it is unrelated to.
/// The old within-group mean reported ~0.07 for this.
@@ -264,7 +283,7 @@ mod tests {
let clusters = vec![cluster(&members)];
let pairs = vec![pair(0, 1, 0.99), pair(0, 2, 0.97), pair(0, 3, 0.95)];
let shares = identity_shares(44, &clusters, &pairs, TOP_MATCHES);
let shares = identity_shares(&all(44), &clusters, &pairs, TOP_MATCHES);
assert!(
(shares[0] - 0.97).abs() < 1e-6,
"the mean of its three real matches, undiluted: {}",
@@ -284,7 +303,7 @@ mod tests {
pair(0, 4, 0.90),
];
let shares = identity_shares(5, &clusters, &pairs, TOP_MATCHES);
let shares = identity_shares(&all(5), &clusters, &pairs, TOP_MATCHES);
// Coherent at 0.90, and only half of the evidence is its own.
assert!(
(shares[0] - 0.45).abs() < 1e-6,
@@ -298,9 +317,9 @@ mod tests {
#[test]
fn a_rival_too_weak_to_merge_still_lowers_the_confidence() {
let clusters = vec![named(&[0, 1], 1), named(&[2, 3], 2)];
let sure = identity_shares(4, &clusters, &[pair(0, 1, 0.95)], TOP_MATCHES);
let sure = identity_shares(&all(4), &clusters, &[pair(0, 1, 0.95)], TOP_MATCHES);
let contested = identity_shares(
4,
&all(4),
&clusters,
&[pair(0, 1, 0.95), pair(0, 2, 0.60)],
TOP_MATCHES,
@@ -321,7 +340,7 @@ mod tests {
fn an_unnamed_group_is_not_treated_as_competition() {
let clusters = vec![cluster(&[0, 1]), cluster(&[2, 3])];
let shares = identity_shares(
4,
&all(4),
&clusters,
&[pair(0, 1, 0.95), pair(0, 2, 0.90)],
TOP_MATCHES,
@@ -343,7 +362,7 @@ mod tests {
let mut pairs: Vec<Pair> = (1..11).map(|j| pair(0, j, 0.90)).collect();
pairs.extend((11..62).map(|j| pair(0, j, 0.55)));
let shares = identity_shares(62, &clusters, &pairs, TOP_MATCHES);
let shares = identity_shares(&all(62), &clusters, &pairs, TOP_MATCHES);
// Ten at 0.90 against ten at 0.55 — not fifty-one at 0.55.
assert!(
(shares[0] - 0.90 * (9.0 / 14.5)).abs() < 1e-5,
@@ -352,11 +371,29 @@ mod tests {
);
}
/// A probe learns from the references it matched; a reference learns
/// nothing from a probe. The pair is the same pair — what differs is who
/// is doing the telling.
#[test]
fn a_face_outside_the_gallery_is_nobody_s_evidence() {
let clusters = vec![named(&[0, 1, 2], 1)];
let gallery = vec![true, true, false];
let pairs = vec![pair(0, 1, 0.80), pair(0, 2, 0.99), pair(1, 2, 0.99)];
let shares = identity_shares(&gallery, &clusters, &pairs, TOP_MATCHES);
// Faces 0 and 1 hear only from each other: the 0.99 the probe offered
// them is not counted.
assert!((shares[0] - 0.80).abs() < 1e-6, "{}", shares[0]);
assert!((shares[1] - 0.80).abs() < 1e-6, "{}", shares[1]);
// The probe hears from both references.
assert!((shares[2] - 0.99).abs() < 1e-6, "{}", shares[2]);
}
/// A face nothing has any evidence about claims nothing.
#[test]
fn a_face_with_no_evidence_reports_no_confidence() {
let clusters = vec![cluster(&[0, 1])];
let shares = identity_shares(2, &clusters, &[], TOP_MATCHES);
let shares = identity_shares(&all(2), &clusters, &[], TOP_MATCHES);
assert_eq!(shares, vec![0.0, 0.0]);
}
}
core/dr-face/src/cluster.rs
+307 -3
View File
@@ -19,6 +19,23 @@
//! and clustering never moves it. Two groups holding confirmations of
//! *different* people cannot merge, whatever their similarity says.
//!
//! # The gallery, and the faces that are only ever compared against it
//!
//! A third defence, and the cheapest of all: **a short embedding is never a
//! reference.** The length of the raw vector is the model's own reading of
//! how recognisable the crop was ([`crate::embedding::MIN_GALLERY_QUALITY`]),
//! and a short one sits near the centre of the sphere, matching a little of
//! everybody. One of those in a group is a bridge to the next group over.
//!
//! So the population is split. Faces at or above the floor are the
//! **gallery**, and they cluster exactly as described below. Faces under it
//! are **probes**: each is measured against the finished groups and joins the
//! one it fits, by the same average-link rule and under the same constraints
//! — but it is measured against the gallery members only, never against
//! another probe, and once placed it is never part of what the next face is
//! measured against. A blurred photograph of a known person is still named;
//! it just cannot vouch for anyone else.
//!
//! # Average link, not single link
//!
//! Single-link chains: one bad edge welds two identities together, and it is
@@ -117,6 +134,11 @@ pub struct Candidate {
pub embedding: Vec<f32>,
/// Source pixels across the aligned crop, for the calibration's size term.
pub crop_px: f32,
/// Length of the raw embedding, where it was recorded
/// ([`crate::embedding::MIN_GALLERY_QUALITY`]). `None` for a face indexed
/// before it was kept, which is admitted to the gallery — see
/// [`Candidate::in_gallery`].
pub quality: Option<f32>,
/// The person this face is *confirmed* to be, if any.
///
/// Suggestions are deliberately not passed here. They are this function's
@@ -125,6 +147,13 @@ pub struct Candidate {
pub confirmed_person: Option<u64>,
}
impl Candidate {
/// Whether this face may be compared *against*, as well as compared.
pub fn in_gallery(&self) -> bool {
crate::embedding::in_gallery(self.quality)
}
}
/// One group of faces the clusterer believes are one person.
#[derive(Debug, Clone, PartialEq)]
pub struct Cluster {
@@ -202,7 +231,7 @@ pub fn cluster_scored(faces: &[Candidate], cal: &Calibration, min_probability: f
let clusters = build(faces, cal, min_probability, &merges);
let confidence = crate::assign::identity_shares(
faces.len(),
&columns.gallery,
&clusters,
&evidence,
crate::assign::TOP_MATCHES,
@@ -225,6 +254,7 @@ struct Columns {
dim: usize,
crop_px: Vec<f32>,
images: Vec<u64>,
gallery: Vec<bool>,
}
impl Columns {
@@ -245,6 +275,7 @@ impl Columns {
dim,
crop_px: faces.iter().map(|f| f.crop_px).collect(),
images: faces.iter().map(|f| f.image).collect(),
gallery: faces.iter().map(Candidate::in_gallery).collect(),
}
}
@@ -254,22 +285,171 @@ impl Columns {
dim: self.dim,
crop_px: &self.crop_px,
images: &self.images,
gallery: &self.gallery,
}
}
}
/// Agglomerate the gallery over its pairs, then place the probes.
///
/// `pairs` is what [`neighbours::above_threshold`] returned: every pair has a
/// gallery side, but a pair with a probe on the other side is not a merge —
/// it is the evidence [`place_probes`] works from. Only the gallery-to-gallery
/// pairs reach the engine, so a probe enters it as a singleton with no edges
/// and comes out exactly as it went in.
fn build(
faces: &[Candidate],
cal: &Calibration,
min_probability: f32,
pairs: &[neighbours::Pair],
) -> Vec<Cluster> {
let gallery: Vec<bool> = faces.iter().map(Candidate::in_gallery).collect();
let (merges, probe_pairs): (Vec<_>, Vec<_>) = pairs
.iter()
.copied()
.partition(|p| gallery[p.i] && gallery[p.j]);
let mut engine = Engine::new(faces, cal, min_probability);
let parts = components(faces.len(), pairs);
let parts = components(faces.len(), &merges);
for (component, edges) in parts.members.iter().zip(&parts.edges) {
engine.agglomerate(component, edges);
}
engine.finish()
let dot = engine.dot;
let clusters = engine.finish();
if probe_pairs.is_empty() {
return clusters;
}
place_probes(
faces,
cal,
min_probability,
dot,
&gallery,
clusters,
&probe_pairs,
)
}
/// Put each probe into the finished group it fits, or leave it alone.
///
/// The same decision the engine makes for a singleton — average link over the
/// group, at or above `min_probability`, subject to [`Engine::can_link`]'s two
/// constraints — with one difference that is the whole point: the average is
/// over the group's **gallery** members. A probe already placed is not part of
/// what the next one is measured against, so a run of short vectors cannot
/// pull each other in one after another.
///
/// Probes are placed in index order and each placement is final, which is
/// what keeps this deterministic. The group a probe joins gains its
/// photograph, so a second face from the same frame cannot follow it — the
/// co-occurrence rule, applied exactly as the engine applies it.
fn place_probes(
faces: &[Candidate],
cal: &Calibration,
min_probability: f32,
dot: neighbours::DotFn,
gallery: &[bool],
mut clusters: Vec<Cluster>,
probe_pairs: &[neighbours::Pair],
) -> Vec<Cluster> {
// Where each face sits, and what each group's photographs and gallery
// members are. The probe's own singleton is here too, and is dropped once
// it has moved.
let mut group_of = vec![usize::MAX; faces.len()];
for (g, c) in clusters.iter().enumerate() {
for &m in &c.members {
group_of[m] = g;
}
}
let mut images: Vec<HashSet<u64>> = clusters
.iter()
.map(|c| c.members.iter().map(|&m| faces[m].image).collect())
.collect();
let references: Vec<Vec<usize>> = clusters
.iter()
.map(|c| c.members.iter().copied().filter(|&m| gallery[m]).collect())
.collect();
// Which groups each probe has any above-threshold pair into. Only those
// can average above the threshold — the argument the module note makes
// for the engine holds here unchanged.
let mut candidates: Vec<Vec<usize>> = vec![Vec::new(); faces.len()];
for p in probe_pairs {
let (probe, reference) = if gallery[p.i] { (p.j, p.i) } else { (p.i, p.j) };
candidates[probe].push(group_of[reference]);
}
let mut moved: Vec<usize> = Vec::new();
for probe in 0..faces.len() {
if gallery[probe] || candidates[probe].is_empty() {
continue;
}
let mut groups = std::mem::take(&mut candidates[probe]);
groups.sort_unstable();
groups.dedup();
let face = &faces[probe];
let mut best: Option<(f32, usize)> = None;
for g in groups {
let target = &clusters[g];
if let (Some(mine), Some(theirs)) = (face.confirmed_person, target.person) {
if mine != theirs {
continue;
}
}
if images[g].contains(&face.image) {
continue;
}
let (mut sum, mut count) = (0.0_f64, 0.0_f64);
for &r in &references[g] {
let cos = dot(&face.embedding, &faces[r].embedding);
let min_crop = face.crop_px.min(faces[r].crop_px);
sum += cal.probability(cos, min_crop, 0.0) as f64;
count += 1.0;
}
if count == 0.0 {
continue;
}
let p = (sum / count) as f32;
// Strictly better wins; on a tie the lowest group index, which is
// the engine's own tiebreak.
if p >= min_probability && best.is_none_or(|(bp, _)| p > bp) {
best = Some((p, g));
}
}
let Some((_, g)) = best else { continue };
let own = group_of[probe];
clusters[g].members.push(probe);
clusters[g].members.sort_unstable();
clusters[g].person = clusters[g].person.or(face.confirmed_person);
images[g].insert(face.image);
group_of[probe] = g;
moved.push(own);
}
if moved.is_empty() {
return clusters;
}
// The singletons the probes left behind, then the order `Engine::finish`
// promises: largest first, lowest member first among equals.
let mut vacated = vec![false; clusters.len()];
for g in moved {
vacated[g] = true;
}
let mut out: Vec<Cluster> = clusters
.into_iter()
.zip(vacated)
.filter(|(_, gone)| !gone)
.map(|(c, _)| c)
.collect();
out.sort_by(|x, y| {
y.members
.len()
.cmp(&x.members.len())
.then(x.members[0].cmp(&y.members[0]))
});
out
}
/// Split one person's faces into the groups a raised threshold separates them
@@ -726,10 +906,19 @@ mod tests {
image,
embedding: at_cosine(identity, cosine),
crop_px: 150.0,
quality: None,
confirmed_person: None,
}
}
/// A face too short to be a reference: compared, never compared against.
fn probe(face: u64, image: u64, identity: usize, cosine: f32) -> Candidate {
Candidate {
quality: Some(crate::embedding::MIN_GALLERY_QUALITY - 5.0),
..candidate(face, image, identity, cosine)
}
}
/// A calibration steep enough that the test's cosines are unambiguous:
/// 0.6 is near-certain, 0.1 is near-impossible.
fn cal() -> Calibration {
@@ -1099,6 +1288,7 @@ mod tests {
image,
embedding: at_cosine(p, cosine),
crop_px: 60.0 + ((out.len() % 11) as f32) * 25.0,
quality: None,
confirmed_person: None,
});
image += 1;
@@ -1182,6 +1372,7 @@ mod tests {
image: 5_000,
embedding: at_cosine(200, 1.0),
crop_px: 150.0,
quality: None,
confirmed_person: None,
});
let out = cluster(&faces, &cal(), DEFAULT_MERGE_PROBABILITY);
@@ -1191,4 +1382,117 @@ mod tests {
"the outlier was absorbed"
);
}
// ── the gallery ───────────────────────────────────────────────────────
/// A short vector is still somebody: it joins the group it matches.
#[test]
fn a_probe_joins_the_group_it_matches() {
let faces = vec![
candidate(1, 10, 0, 1.0),
candidate(2, 11, 0, 0.95),
probe(3, 12, 0, 0.92),
];
let out = cluster(&faces, &cal(), DEFAULT_MERGE_PROBABILITY);
assert_eq!(out.len(), 1);
assert_eq!(out[0].members, vec![0, 1, 2]);
}
/// Two short vectors that resemble each other are noise agreeing with
/// noise, and there is nothing in the gallery for either to be measured
/// against.
#[test]
fn two_probes_are_never_grouped_with_each_other() {
let faces = vec![probe(1, 10, 0, 1.0), probe(2, 11, 0, 0.98)];
let out = cluster(&faces, &cal(), DEFAULT_MERGE_PROBABILITY);
assert_eq!(out.len(), 2, "two probes were grouped: {out:?}");
}
/// The point of measuring against the gallery only: a probe that has been
/// placed is not a stepping stone for the next one.
#[test]
fn a_placed_probe_is_not_what_the_next_probe_is_measured_against() {
let mut first = probe(2, 11, 0, 0.6);
// 0.6 along identity 0 and 0.8 along its perpendicular: near enough to
// the reference to join it, and much nearer to the face below.
first.embedding = at_cosine(0, 0.6);
let mut second = probe(3, 12, 0, 0.0);
second.embedding = at_cosine(0, 0.0);
let faces = vec![candidate(1, 10, 0, 1.0), first, second];
let out = cluster(&faces, &cal(), DEFAULT_MERGE_PROBABILITY);
let group = out.iter().find(|c| c.members.contains(&0)).unwrap();
assert_eq!(
group.members,
vec![0, 1],
"the first probe should have joined"
);
assert!(
out.iter().any(|c| c.members == vec![2]),
"the second probe reached the group through the first: {out:?}"
);
}
/// A confirmation on a probe is still the user's word: the group it joins
/// becomes that person, and a group already someone else's is closed to it.
#[test]
fn a_probe_carries_its_confirmation_and_respects_others() {
let mut anchored = probe(3, 12, 0, 0.92);
anchored.confirmed_person = Some(7);
let faces = vec![
candidate(1, 10, 0, 1.0),
candidate(2, 11, 0, 0.95),
anchored,
];
let out = cluster(&faces, &cal(), DEFAULT_MERGE_PROBABILITY);
assert_eq!(out.len(), 1);
assert_eq!(out[0].person, Some(7));
let mut theirs = candidate(1, 10, 0, 1.0);
theirs.confirmed_person = Some(8);
let faces = vec![theirs, candidate(2, 11, 0, 0.95), {
let mut a = probe(3, 12, 0, 0.92);
a.confirmed_person = Some(7);
a
}];
let out = cluster(&faces, &cal(), DEFAULT_MERGE_PROBABILITY);
assert!(
out.iter()
.any(|c| c.members == vec![2] && c.person == Some(7)),
"a probe confirmed as one person joined another's group: {out:?}"
);
}
/// The co-occurrence rule follows a probe in: once it has joined, its
/// photograph is the group's.
#[test]
fn a_probe_cannot_join_a_group_holding_a_face_from_its_own_photograph() {
let faces = vec![
candidate(1, 10, 0, 1.0),
candidate(2, 11, 0, 0.95),
probe(3, 10, 0, 0.92),
];
let out = cluster(&faces, &cal(), DEFAULT_MERGE_PROBABILITY);
assert!(out.iter().any(|c| c.members == vec![2]), "{out:?}");
}
/// A probe's placement is scored like anyone else's, from the references
/// it matched — and the references' own scores do not hear from it.
#[test]
fn a_probe_is_scored_but_is_not_evidence() {
let gallery_only = vec![candidate(1, 10, 0, 1.0), candidate(2, 11, 0, 0.95)];
let without = cluster_scored(&gallery_only, &cal(), DEFAULT_MERGE_PROBABILITY);
let mut with_probe = gallery_only.clone();
with_probe.push(probe(3, 12, 0, 0.99));
let with = cluster_scored(&with_probe, &cal(), DEFAULT_MERGE_PROBABILITY);
assert_eq!(with.clusters[0].members, vec![0, 1, 2]);
assert!(with.confidence[2] > 0.9, "{}", with.confidence[2]);
assert_eq!(
&with.confidence[..2],
&without.confidence[..],
"a probe changed what the references were sure of"
);
}
}
core/dr-face/src/embed.rs
+43 -5
View File
@@ -16,6 +16,41 @@ use crate::align::{Aligned112, ALIGNED_EDGE};
use crate::embedding::{normalise, Embedding, ModelId, EMBEDDING_DIM};
use crate::{install_backend, FaceError};
/// What one pass of the embedder produces: the direction, and the length.
///
/// Two fields rather than a `quality` on [`Embedding`], because every other
/// holder of an `Embedding` relies on it being unit length and compares by
/// dot product; the length is a separate fact about the same face, and it is
/// stored separately too.
#[derive(Debug, Clone, PartialEq)]
pub struct Embedded {
pub embedding: Embedding,
/// L2 norm of the raw model output.
///
/// The model's own opinion of how recognisable the crop was — see
/// [`crate::embedding::MIN_GALLERY_QUALITY`] for what it means and where
/// it is used.
pub quality: f32,
}
impl Embedded {
/// Storage form: the **raw** vector, `512 × f16`.
///
/// Not the unit vector. The length is the quality, and a store that held
/// only the direction would have thrown it away at the one moment it could
/// be known — which is what this crate used to do. Readers re-normalise
/// ([`Embedding::from_f16_bytes`]), so every comparison is still a dot
/// product, and [`crate::embedding::read_f16_bytes`] gives the length back
/// to a reader that wants it.
///
/// f16 costs nothing extra at this scale: its precision is relative, so a
/// component of a vector of length 20 is kept to the same three figures as
/// the same component scaled to length 1.
pub fn to_f16_bytes(&self) -> Vec<u8> {
self.embedding.to_f16_bytes_scaled(self.quality)
}
}
/// A loaded ArcFace graph.
pub struct Embedder {
session: ort::session::Session,
@@ -63,7 +98,7 @@ impl Embedder {
}
/// Embed one aligned face.
pub fn embed(&mut self, face: &Aligned112) -> Result<Embedding, FaceError> {
pub fn embed(&mut self, face: &Aligned112) -> Result<Embedded, FaceError> {
// `(x·255 − 127.5) / 128` — see the `/128` note in `detect::Letterbox`.
let px = face.pixels();
let mut input = Array4::<f32>::zeros((1, 3, ALIGNED_EDGE, ALIGNED_EDGE));
@@ -95,11 +130,14 @@ impl Embedder {
let mut v = Box::new([0.0_f32; EMBEDDING_DIM]);
v.copy_from_slice(&data[..EMBEDDING_DIM]);
normalise(&mut v);
let quality = normalise(&mut v);
Ok(Embedding {
model: self.model.clone(),
v,
Ok(Embedded {
embedding: Embedding {
model: self.model.clone(),
v,
},
quality,
})
}
}
core/dr-face/src/embedding.rs
+119 -18
View File
@@ -12,6 +12,43 @@
/// Embedding dimensionality. Fixed by the model family, not a parameter.
pub const EMBEDDING_DIM: usize = 512;
/// The shortest raw embedding a face may be *compared against*.
///
/// # What the length of the vector says
///
/// ArcFace is trained on the direction of its output and nothing else, and
/// the length it leaves behind turns out to be a free quality signal: the
/// magnitude grows with how recognisable the crop was to the model, and a
/// blurred, occluded, badly lit or hard-profile face comes out short. MagFace
/// (Meng et al., CVPR 2021) made that the training objective; the plain
/// ArcFace heads this crate runs already show it, weaker but usable, which is
/// why it is worth keeping the number the normalisation discards.
///
/// # Why it gates the gallery and not the face
///
/// A short vector is a bad *reference*: it sits nearer the centre of the
/// sphere than a real identity does and matches a little of everyone, which
/// is exactly the face that welds two people together in a clustering pass.
/// It is not a bad *probe* — the face is still real, still somebody, and
/// comparing it against good references is the only way it will ever be named.
/// So a face below this floor is compared against the gallery and never
/// becomes part of it: see `cluster::Candidate::in_gallery`.
///
/// 14 is the operating point for `w600k_mbf`, whose norms on the reference
/// library run from about 8 on a blur to the high 20s on a clean portrait. A
/// face whose quality was never recorded — indexed before the number was kept
/// — is not gated, because a rule that cannot be checked should admit, not
/// exclude.
pub const MIN_GALLERY_QUALITY: f32 = 14.0;
/// Whether an embedding of this quality may serve as a reference.
///
/// `None` is "not measured", and is admitted: the rule is about a number that
/// was read and found short, not about a number that is missing.
pub fn in_gallery(quality: Option<f32>) -> bool {
quality.is_none_or(|q| q >= MIN_GALLERY_QUALITY)
}
/// Which model produced an embedding.
///
/// Embeddings from different models are not comparable, and this is the one
@@ -58,10 +95,19 @@ impl Embedding {
}
/// Storage form: `512 × f16`, 1 KB per face (catalog.md §10.1).
///
/// This writes the unit vector. What the catalog stores is the raw one —
/// `embed::Embedded::to_f16_bytes` — because the length is the quality
/// and a unit vector has none left to read.
pub fn to_f16_bytes(&self) -> Vec<u8> {
self.to_f16_bytes_scaled(1.0)
}
/// The unit vector scaled by `length`, as `512 × f16`.
pub(crate) fn to_f16_bytes_scaled(&self, length: f32) -> Vec<u8> {
let mut out = Vec::with_capacity(EMBEDDING_DIM * 2);
for &x in self.v.iter() {
out.extend_from_slice(&f32_to_f16_bits(x).to_le_bytes());
out.extend_from_slice(&f32_to_f16_bits(x * length).to_le_bytes());
}
out
}
@@ -71,25 +117,42 @@ impl Embedding {
/// The f16 round-trip perturbs a unit vector by ~1e-3 in cosine — three
/// orders below the separation between a match and a non-match — but the
/// drift is free to remove and invisible if left, so it is removed here
/// rather than remembered at every call site.
/// rather than remembered at every call site. The same pass is what turns
/// a stored raw vector back into the unit one every comparison expects.
pub fn from_f16_bytes(model: ModelId, bytes: &[u8]) -> Option<Self> {
if bytes.len() != EMBEDDING_DIM * 2 {
return None;
}
let mut v = Box::new([0.0_f32; EMBEDDING_DIM]);
for (i, chunk) in bytes.chunks_exact(2).enumerate() {
v[i] = f16_bits_to_f32(u16::from_le_bytes([chunk[0], chunk[1]]));
}
normalise(&mut v);
Some(Self { model, v })
read_f16_bytes(model, bytes).map(|(e, _)| e)
}
}
/// Read a stored vector back, with the length it was stored at.
///
/// The length is the quality where the blob is a raw one, and ~1 where it is
/// a unit vector from before raw vectors were stored — which is why the
/// catalog keeps the quality beside the blob rather than deriving it from
/// this: a unit vector reads as a quality of 1, not as "unmeasured".
pub fn read_f16_bytes(model: ModelId, bytes: &[u8]) -> Option<(Embedding, f32)> {
if bytes.len() != EMBEDDING_DIM * 2 {
return None;
}
let mut v = Box::new([0.0_f32; EMBEDDING_DIM]);
for (i, chunk) in bytes.chunks_exact(2).enumerate() {
v[i] = f16_bits_to_f32(u16::from_le_bytes([chunk[0], chunk[1]]));
}
let length = normalise(&mut v);
Some((Embedding { model, v }, length))
}
fn dot(a: &[f32; EMBEDDING_DIM], b: &[f32; EMBEDDING_DIM]) -> f32 {
a.iter().zip(b.iter()).map(|(x, y)| x * y).sum()
}
pub(crate) fn normalise(v: &mut [f32; EMBEDDING_DIM]) {
/// Scale `v` to unit length, and return the length it had.
///
/// The length is the one thing about the raw output that survives being
/// thrown away by everything downstream, and it is a quality signal
/// ([`MIN_GALLERY_QUALITY`]) — so it comes back out rather than being lost
/// here.
pub(crate) fn normalise(v: &mut [f32; EMBEDDING_DIM]) -> f32 {
// Clamped rather than checked: a zero-norm embedding is a broken model,
// not a runtime condition worth an error path, and dividing by 1e-6 keeps
// the NaN out of the catalog.
@@ -97,6 +160,7 @@ pub(crate) fn normalise(v: &mut [f32; EMBEDDING_DIM]) {
for x in v.iter_mut() {
*x /= norm;
}
norm
}
// ── f16 ───────────────────────────────────────────────────────────────────
@@ -113,9 +177,10 @@ fn f32_to_f16_bits(x: f32) -> u16 {
let mant = bits & 0x007f_ffff;
if exp >= 0x1f {
// Overflow, inf, or NaN. Embeddings are unit-norm so this is the
// broken-model path; infinity is the honest answer, not a clamp that
// hides it.
// Overflow, inf, or NaN. No component of an embedding exceeds its
// length, and the lengths this model produces are in the tens, so
// this is the broken-model path; infinity is the honest answer, not a
// clamp that hides it.
return sign
| 0x7c00
| if mant != 0 && exp == 0x1f + 112 {
@@ -125,9 +190,9 @@ fn f32_to_f16_bits(x: f32) -> u16 {
};
}
if exp <= 0 {
// Subnormal or underflow. A component of a unit 512-vector is ~0.04,
// nowhere near here, so this branch exists for correctness rather than
// for traffic.
// Subnormal or underflow. A component of a unit 512-vector is ~0.04
// and a stored one is that times the length, nowhere near here, so
// this branch exists for correctness rather than for traffic.
if exp < -10 {
return sign;
}
@@ -221,6 +286,42 @@ mod tests {
}
}
#[test]
fn normalising_reports_the_length_it_removed() {
let mut v = Box::new([0.0_f32; EMBEDDING_DIM]);
v[0] = 3.0;
v[1] = 4.0;
let norm = normalise(&mut v);
assert!((norm - 5.0).abs() < 1e-6, "norm {norm}");
assert!((v[0] - 0.6).abs() < 1e-6 && (v[1] - 0.8).abs() < 1e-6);
}
/// The gate admits what it cannot measure: a face from before the number
/// was kept is not a face that was found wanting.
#[test]
fn an_unmeasured_quality_is_admitted_to_the_gallery() {
assert!(in_gallery(None));
assert!(in_gallery(Some(MIN_GALLERY_QUALITY)));
assert!(in_gallery(Some(27.5)));
assert!(!in_gallery(Some(MIN_GALLERY_QUALITY - 0.01)));
assert!(!in_gallery(Some(8.0)));
}
/// The storage form carries the length, and the length comes back out —
/// without touching the direction every comparison is made on.
#[test]
fn a_raw_vector_round_trips_with_its_length() {
let e = unit(3);
let raw = e.to_f16_bytes_scaled(21.5);
let (back, length) = read_f16_bytes(e.model.clone(), &raw).unwrap();
assert!((length - 21.5).abs() < 0.05, "length {length}");
assert!(e.cosine(&back).unwrap() > 0.9999);
// A unit vector from an older store reads as length 1, not as an
// error — see `read_f16_bytes` on why that is not "unmeasured".
let (_, one) = read_f16_bytes(e.model.clone(), &e.to_f16_bytes()).unwrap();
assert!((one - 1.0).abs() < 1e-2, "length {one}");
}
#[test]
fn f16_round_trip_rejects_a_wrong_length_blob() {
assert!(Embedding::from_f16_bytes(ModelId::new("m"), &[0u8; 100]).is_none());
core/dr-face/src/lib.rs
+4 -2
View File
@@ -75,8 +75,10 @@ pub use cluster::{
#[cfg(feature = "inference")]
pub use detect::{DetectOptions, Detection, Detector};
#[cfg(feature = "inference")]
pub use embed::Embedder;
pub use embedding::{Embedding, ModelId, EMBEDDING_DIM};
pub use embed::{Embedded, Embedder};
pub use embedding::{
in_gallery, read_f16_bytes, Embedding, ModelId, EMBEDDING_DIM, MIN_GALLERY_QUALITY,
};
pub use naming::{name_for_instance, name_instances, NamedFace};
/// What can go wrong between an image and a face.
core/dr-face/src/neighbours.rs
+47 -2
View File
@@ -117,6 +117,17 @@ pub struct Faces<'a> {
/// Which photograph each face came from. Two faces in one frame are not
/// the same person, so those pairs are never returned (docs/faces.md §9).
pub images: &'a [u64],
/// Which faces may be compared *against* — the gallery
/// ([`crate::embedding::MIN_GALLERY_QUALITY`]).
///
/// A pair needs at least one gallery side: a probe measured against a
/// reference is a comparison, two short vectors measured against each
/// other is noise agreeing with noise, and those pairs are never returned.
/// Filtered here rather than by the caller for the same reason
/// co-occurrence is: what this module leaves out of the list stays out of
/// the graph, the components and the merge order, so nothing downstream
/// has to remember the rule.
pub gallery: &'a [bool],
}
impl Faces<'_> {
@@ -272,8 +283,9 @@ fn scan_rows(scan: &Scan, from: usize, to: usize, out: &mut Vec<Pair>) {
let a = faces.row(i);
let crop_a = faces.crop_px[i];
let image_a = faces.images[i];
let gallery_a = faces.gallery[i];
for j in start..tile_end {
if image_a == faces.images[j] {
if image_a == faces.images[j] || !(gallery_a || faces.gallery[j]) {
continue;
}
let cos = dot(a, faces.row(j));
@@ -552,6 +564,7 @@ mod tests {
embeddings: Vec<f32>,
crop_px: Vec<f32>,
images: Vec<u64>,
gallery: Vec<bool>,
}
impl Set {
@@ -561,6 +574,7 @@ mod tests {
dim: DIM,
crop_px: &self.crop_px,
images: &self.images,
gallery: &self.gallery,
}
}
@@ -588,10 +602,12 @@ mod tests {
}
}
let crop_px = vec![150.0; embeddings.len()];
let gallery = vec![true; embeddings.len()];
Set {
embeddings: embeddings.concat(),
crop_px,
images,
gallery,
}
}
@@ -601,7 +617,7 @@ mod tests {
let mut out = Vec::new();
for i in 0..n {
for j in i + 1..n {
if faces.images[i] == faces.images[j] {
if faces.images[i] == faces.images[j] || !(faces.gallery[i] || faces.gallery[j]) {
continue;
}
let cos: f32 = faces
@@ -750,6 +766,35 @@ mod tests {
assert!(above_threshold(&s.faces(), &cal(), 0.9).is_empty());
}
/// A probe against a reference is a comparison; two probes against each
/// other is not. The rule lives here so that nothing downstream sees the
/// pair at all.
#[test]
fn two_faces_outside_the_gallery_are_never_paired() {
let mut s = population(1, 3, 1.0);
s.gallery = vec![false, false, true];
let pairs = above_threshold(&s.faces(), &cal(), 0.9);
assert!(
!pairs.iter().any(|p| p.i == 0 && p.j == 1),
"two probes were paired with each other"
);
// Each probe is still measured against the one reference.
assert!(pairs.iter().any(|p| p.i == 0 && p.j == 2));
assert!(pairs.iter().any(|p| p.i == 1 && p.j == 2));
}
#[test]
fn the_gallery_rule_matches_the_reference_at_scale() {
let mut s = population(60, 8, 0.97);
for (i, g) in s.gallery.iter_mut().enumerate() {
*g = i % 3 != 0;
}
let f = s.faces();
let got = above_threshold(&f, &cal(), 0.9);
let want = reference(&f, &cal(), 0.9);
assert!(same_pairs(&got, &want), "{} vs {}", got.len(), want.len());
}
#[test]
fn pairs_come_back_in_index_order() {
let s = population(300, 8, 0.97);