Keep each face's quality, and never compare against a poor one
The embedder's raw output has a length, and the length is a reading of how recognisable the crop was: a blur, an occlusion or a hard profile comes out short. Normalising threw it away. A short vector sits near the middle of the sphere and matches a little of everyone, which is how one bad crop bridges two people in a grouping pass. So the length is kept — the store now holds the raw vector, re-normalised on load, with the length beside it as `faces.quality` — and a face under MIN_GALLERY_QUALITY (14) is a probe: measured against the gallery and placed where it fits, but never what another face is measured against. Two probes are never paired, and a probe is nobody's evidence for a confidence. The People screen shows the number as "Quality 17.3", dimmed below the floor. Faces indexed before this stored unit vectors and have no reading; they are admitted to the gallery, and schema V14 forgets the run marker of every image holding one so the next indexing pass measures them. A peer's unmeasured shard faces are not adopted, or a sync would write that marker back.
This commit is contained in:
@@ -34,6 +34,10 @@ struct Known {
|
||||
crop_px: f32,
|
||||
}
|
||||
|
||||
/// What the catalog holds per face, decoded: photograph, vector, size,
|
||||
/// quality.
|
||||
type Decoded = (u64, Vec<f32>, f32, Option<f32>);
|
||||
|
||||
fn main() {
|
||||
let args: Vec<String> = std::env::args().skip(1).collect();
|
||||
let Some(path) = args.first() else {
|
||||
@@ -59,10 +63,10 @@ fn main() {
|
||||
|
||||
let model = dr_face::ModelId::new(MODEL_ID.to_string());
|
||||
let stored = faces::embeddings(conn, MODEL_ID).expect("embeddings");
|
||||
let mut embedding_of = HashMap::new();
|
||||
for (id, image, blob, crop_px) in stored {
|
||||
if let Some(e) = dr_face::Embedding::from_f16_bytes(model.clone(), &blob) {
|
||||
embedding_of.insert(id, (image.0, e.v.to_vec(), crop_px));
|
||||
let mut embedding_of: HashMap<faces::FaceId, Decoded> = HashMap::new();
|
||||
for f in stored {
|
||||
if let Some(e) = dr_face::Embedding::from_f16_bytes(model.clone(), &f.embedding) {
|
||||
embedding_of.insert(f.face, (f.image.0, e.v.to_vec(), f.crop_px, f.quality));
|
||||
}
|
||||
}
|
||||
println!("faces with embeddings: {}", embedding_of.len());
|
||||
@@ -82,7 +86,7 @@ fn main() {
|
||||
if !f.confirmed {
|
||||
continue;
|
||||
}
|
||||
if let Some((image, embedding, crop_px)) = embedding_of.get(&f.id) {
|
||||
if let Some((image, embedding, crop_px, _)) = embedding_of.get(&f.id) {
|
||||
mine.push(Known {
|
||||
image: *image,
|
||||
person: p.id,
|
||||
@@ -288,19 +292,22 @@ fn band(label: &str, v: &[f32]) {
|
||||
/// The whole library through the real clusterer, for the numbers it would
|
||||
/// actually write.
|
||||
fn full_library(
|
||||
embedding_of: &HashMap<faces::FaceId, (u64, Vec<f32>, f32)>,
|
||||
embedding_of: &HashMap<faces::FaceId, Decoded>,
|
||||
confirmed: &HashMap<faces::FaceId, u64>,
|
||||
cal: &dr_face::Calibration,
|
||||
) {
|
||||
let mut candidates: Vec<dr_face::Candidate> = embedding_of
|
||||
.iter()
|
||||
.map(|(id, (image, embedding, crop_px))| dr_face::Candidate {
|
||||
face: id.0,
|
||||
image: *image,
|
||||
embedding: embedding.clone(),
|
||||
crop_px: *crop_px,
|
||||
confirmed_person: confirmed.get(id).copied(),
|
||||
})
|
||||
.map(
|
||||
|(id, (image, embedding, crop_px, quality))| dr_face::Candidate {
|
||||
face: id.0,
|
||||
image: *image,
|
||||
embedding: embedding.clone(),
|
||||
crop_px: *crop_px,
|
||||
quality: *quality,
|
||||
confirmed_person: confirmed.get(id).copied(),
|
||||
},
|
||||
)
|
||||
.collect();
|
||||
candidates.sort_by_key(|c| c.face);
|
||||
|
||||
@@ -317,11 +324,13 @@ fn full_library(
|
||||
.collect();
|
||||
let crop_px: Vec<f32> = candidates.iter().map(|c| c.crop_px).collect();
|
||||
let images: Vec<u64> = candidates.iter().map(|c| c.image).collect();
|
||||
let gallery: Vec<bool> = candidates.iter().map(|c| c.in_gallery()).collect();
|
||||
let view = dr_face::neighbours::Faces {
|
||||
embeddings: &flat,
|
||||
dim,
|
||||
crop_px: &crop_px,
|
||||
images: &images,
|
||||
gallery: &gallery,
|
||||
};
|
||||
|
||||
let t = std::time::Instant::now();
|
||||
@@ -335,8 +344,7 @@ fn full_library(
|
||||
let agglomerate = t.elapsed().as_secs_f64() - scan;
|
||||
|
||||
let t = std::time::Instant::now();
|
||||
let _ =
|
||||
dr_face::identity_shares(candidates.len(), &clusters, &evidence, dr_face::TOP_MATCHES);
|
||||
let _ = dr_face::identity_shares(&gallery, &clusters, &evidence, dr_face::TOP_MATCHES);
|
||||
println!(
|
||||
" scan {scan:.2}s ({} evidence pairs) · agglomerate {agglomerate:.2}s · score {:.2}s",
|
||||
evidence.len(),
|
||||
|
||||
@@ -76,6 +76,9 @@ pub struct SharedFace {
|
||||
pub confidence: f32,
|
||||
pub embedding: Vec<u8>,
|
||||
pub crop_px: f32,
|
||||
/// See `faces::DetectedFace::quality`. `None` from a shard written before
|
||||
/// the number was kept.
|
||||
pub quality: Option<f32>,
|
||||
/// The face cut out and encoded, or empty where none was kept.
|
||||
///
|
||||
/// Travels with the face rather than in the catalog snapshot, which is the
|
||||
@@ -224,8 +227,8 @@ impl FaceShardStore {
|
||||
tx.execute(
|
||||
"INSERT INTO faces
|
||||
(file_id, model_id, x, y, w, h, landmarks, confidence,
|
||||
embedding, crop_px, crop)
|
||||
VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, ?11)",
|
||||
embedding, crop_px, crop, quality)
|
||||
VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, ?11, ?12)",
|
||||
rusqlite::params![
|
||||
f.file_id as i64,
|
||||
f.model_id,
|
||||
@@ -238,6 +241,7 @@ impl FaceShardStore {
|
||||
f.embedding,
|
||||
f.crop_px as f64,
|
||||
(!f.crop.is_empty()).then_some(f.crop.as_slice()),
|
||||
f.quality.map(f64::from),
|
||||
],
|
||||
)?;
|
||||
}
|
||||
@@ -442,9 +446,10 @@ impl FaceShardStore {
|
||||
}
|
||||
let mut fq = src.prepare(&format!(
|
||||
"SELECT f.file_id, f.model_id, f.x, f.y, f.w, f.h, f.landmarks,
|
||||
f.confidence, f.embedding, f.crop_px, {}
|
||||
f.confidence, f.embedding, f.crop_px, {}, {}
|
||||
FROM faces f WHERE f.file_id = ?1 AND f.model_id = ?2",
|
||||
crop_column(&src)
|
||||
column_or_null(&src, "crop"),
|
||||
column_or_null(&src, "quality"),
|
||||
))?;
|
||||
let faces: Vec<SharedFace> = fq
|
||||
.query_map(rusqlite::params![file_id, &model_id], read_shared_face)?
|
||||
@@ -484,7 +489,8 @@ impl FaceShardStore {
|
||||
let Some(edge) = edge else { return Ok(None) };
|
||||
|
||||
let mut q = conn.prepare(
|
||||
"SELECT file_id, model_id, x, y, w, h, landmarks, confidence, embedding, crop_px, crop
|
||||
"SELECT file_id, model_id, x, y, w, h, landmarks, confidence, embedding, crop_px,
|
||||
crop, quality
|
||||
FROM faces WHERE file_id = ?1 AND model_id = ?2",
|
||||
)?;
|
||||
let faces: Vec<SharedFace> = q
|
||||
@@ -541,6 +547,7 @@ fn upgrade_shard(conn: &Connection) -> Result<(), CatalogError> {
|
||||
for (table, column, decl) in [
|
||||
("faces", "crop", "BLOB"),
|
||||
("indexed", "indexed_at", "INTEGER"),
|
||||
("faces", "quality", "REAL"),
|
||||
] {
|
||||
if !has_column(conn, table, column)? {
|
||||
conn.execute_batch(&format!("ALTER TABLE {table} ADD COLUMN {column} {decl}"))?;
|
||||
@@ -555,16 +562,19 @@ fn has_column(conn: &Connection, table: &str, column: &str) -> Result<bool, Cata
|
||||
Ok(stmt.exists(rusqlite::params![table, column])?)
|
||||
}
|
||||
|
||||
/// `f.crop`, or a `NULL` standing in for it.
|
||||
/// `f.<column>`, or a `NULL` standing in for it.
|
||||
///
|
||||
/// A shard downloaded from a peer is opened **read-only** and cannot be
|
||||
/// upgraded, so one written before crops existed has to be read as it is rather
|
||||
/// than repaired. Selecting a literal keeps the column count the same, which is
|
||||
/// what lets [`read_shared_face`] stay a single function.
|
||||
fn crop_column(conn: &Connection) -> &'static str {
|
||||
match has_column(conn, "faces", "crop") {
|
||||
Ok(true) => "f.crop",
|
||||
_ => "NULL",
|
||||
/// upgraded, so one written before a column existed has to be read as it is
|
||||
/// rather than repaired. Selecting a literal keeps the column count the same,
|
||||
/// which is what lets [`read_shared_face`] stay a single function.
|
||||
///
|
||||
/// `column` is one of this module's own names, never anything read from
|
||||
/// outside, which is what makes formatting it into SQL acceptable.
|
||||
fn column_or_null(conn: &Connection, column: &'static str) -> String {
|
||||
match has_column(conn, "faces", column) {
|
||||
Ok(true) => format!("f.{column}"),
|
||||
_ => "NULL".to_string(),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -637,7 +647,8 @@ pub fn export_to_shards_reporting(
|
||||
continue;
|
||||
}
|
||||
let mut fq = conn.prepare(
|
||||
"SELECT x, y, w, h, landmarks, detector_confidence, embedding, crop_px, crop
|
||||
"SELECT x, y, w, h, landmarks, detector_confidence, embedding, crop_px, crop,
|
||||
quality
|
||||
FROM faces WHERE image_id = ?1 AND model_id = ?2",
|
||||
)?;
|
||||
let faces: Vec<SharedFace> = fq
|
||||
@@ -654,6 +665,7 @@ pub fn export_to_shards_reporting(
|
||||
embedding: r.get(6)?,
|
||||
crop_px: r.get::<_, f64>(7)? as f32,
|
||||
crop: r.get::<_, Option<Vec<u8>>>(8)?.unwrap_or_default(),
|
||||
quality: r.get::<_, Option<f64>>(9)?.map(|q| q as f32),
|
||||
})
|
||||
})?
|
||||
.collect::<Result<_, _>>()?;
|
||||
@@ -710,6 +722,14 @@ pub fn import_from_shards(
|
||||
let Some((faces, edge)) = store.get_image(file_id as u64, model_id)? else {
|
||||
continue;
|
||||
};
|
||||
// A peer that embedded before the quality was kept has done work this
|
||||
// device cannot finish: the number exists only at embedding time, and
|
||||
// adopting the faces would write the run marker that keeps them from
|
||||
// ever being measured (schema V14). Left for this device's own pass —
|
||||
// or for the peer's, whose re-export replaces these.
|
||||
if faces.iter().any(|f| f.quality.is_none()) {
|
||||
continue;
|
||||
}
|
||||
let local: Vec<crate::faces::DetectedFace> = faces
|
||||
.into_iter()
|
||||
.map(|f| crate::faces::DetectedFace {
|
||||
@@ -721,6 +741,7 @@ pub fn import_from_shards(
|
||||
confidence: f.confidence,
|
||||
embedding: f.embedding,
|
||||
crop_px: f.crop_px,
|
||||
quality: f.quality,
|
||||
model_id: f.model_id,
|
||||
// A peer that indexed before crops existed sends none, and the
|
||||
// reader falls back to the proxy exactly as it does for a face
|
||||
@@ -766,6 +787,7 @@ fn read_shared_face(r: &rusqlite::Row<'_>) -> rusqlite::Result<SharedFace> {
|
||||
embedding: r.get(8)?,
|
||||
crop_px: r.get::<_, f64>(9)? as f32,
|
||||
crop: r.get::<_, Option<Vec<u8>>>(10)?.unwrap_or_default(),
|
||||
quality: r.get::<_, Option<f64>>(11)?.map(|q| q as f32),
|
||||
})
|
||||
}
|
||||
|
||||
@@ -849,7 +871,11 @@ CREATE TABLE IF NOT EXISTS faces (
|
||||
crop_px REAL NOT NULL,
|
||||
-- The face, cut out. NULL where the face was found before crops were kept,
|
||||
-- or adopted from a peer that did not have one.
|
||||
crop BLOB
|
||||
crop BLOB,
|
||||
-- Length of the raw embedding (`faces::DetectedFace::quality`). NULL from
|
||||
-- a build that did not keep it, and a face the receiving device will not
|
||||
-- adopt -- see `import_from_shards`.
|
||||
quality REAL
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS faces_file ON faces(file_id, model_id);
|
||||
|
||||
@@ -908,6 +934,7 @@ mod tests {
|
||||
confidence: 0.87,
|
||||
embedding: vec![seed; 1024],
|
||||
crop_px: 180.0,
|
||||
quality: Some(17.5),
|
||||
crop: vec![seed; 64],
|
||||
}
|
||||
}
|
||||
@@ -925,6 +952,7 @@ mod tests {
|
||||
assert_eq!(edge, 1024);
|
||||
assert_eq!(faces[0].embedding.len(), 1024);
|
||||
assert!((faces[0].crop_px - 180.0).abs() < 1e-3);
|
||||
assert_eq!(faces[0].quality, Some(17.5));
|
||||
}
|
||||
|
||||
/// The case the run marker exists for, carried across the wire: an image
|
||||
@@ -1115,6 +1143,7 @@ mod catalog_round_trip {
|
||||
confidence: 0.9,
|
||||
embedding: vec![seed; 1024],
|
||||
crop_px: 180.0,
|
||||
quality: Some(20.0),
|
||||
model_id: "w600k_mbf".into(),
|
||||
crop: vec![seed; 64],
|
||||
}
|
||||
@@ -1162,9 +1191,47 @@ mod catalog_round_trip {
|
||||
let got = faces::for_image(&b, dr_types::ImageId(90)).unwrap();
|
||||
assert_eq!(got.len(), 1);
|
||||
assert!((got[0].crop_px - 180.0).abs() < 1e-3);
|
||||
assert_eq!(got[0].quality, Some(20.0));
|
||||
assert!((got[0].landmarks[2].0 - 0.15).abs() < 1e-5);
|
||||
let emb = faces::embeddings(&b, "w600k_mbf").unwrap();
|
||||
assert!(emb.iter().any(|(_, _, blob, _)| blob[0] == 1));
|
||||
assert!(emb.iter().any(|e| e.embedding[0] == 1));
|
||||
}
|
||||
|
||||
/// A face a peer embedded without measuring it is work this device
|
||||
/// cannot finish, and adopting it would write the marker that stops it
|
||||
/// ever being measured. The image stays outstanding instead.
|
||||
#[test]
|
||||
fn a_peers_unmeasured_faces_are_left_for_this_device_to_index() {
|
||||
let b = device(&[(90, 5001), (91, 5002)]);
|
||||
let mut store = FaceShardStore::open(&tempdir("unmeasured")).unwrap();
|
||||
let shared = |file_id: u64, quality: Option<f32>| SharedFace {
|
||||
file_id,
|
||||
model_id: "w600k_mbf".into(),
|
||||
x: 0.1,
|
||||
y: 0.2,
|
||||
w: 0.15,
|
||||
h: 0.2,
|
||||
landmarks: vec![1; 40],
|
||||
confidence: 0.87,
|
||||
embedding: vec![1; 1024],
|
||||
crop_px: 180.0,
|
||||
quality,
|
||||
crop: Vec::new(),
|
||||
};
|
||||
store
|
||||
.put_image(5001, "w600k_mbf", 2560, &[shared(5001, None)])
|
||||
.unwrap();
|
||||
store
|
||||
.put_image(5002, "w600k_mbf", 2560, &[shared(5002, Some(19.0))])
|
||||
.unwrap();
|
||||
|
||||
assert_eq!(import_from_shards(&b, &store, "w600k_mbf").unwrap(), 1);
|
||||
let cov = faces::coverage(&b, "w600k_mbf").unwrap();
|
||||
assert_eq!(cov.indexed, 1);
|
||||
assert_eq!(cov.outstanding(), 1, "the unmeasured image was adopted");
|
||||
assert!(faces::for_image(&b, dr_types::ImageId(90))
|
||||
.unwrap()
|
||||
.is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -1187,7 +1254,7 @@ mod catalog_round_trip {
|
||||
"a peer's copy replaced work this device had already done"
|
||||
);
|
||||
let emb = faces::embeddings(&b, "w600k_mbf").unwrap();
|
||||
assert_eq!(emb[0].2[0], 9, "B's own embedding was overwritten");
|
||||
assert_eq!(emb[0].embedding[0], 9, "B's own embedding was overwritten");
|
||||
}
|
||||
|
||||
/// A device holding a subset of the library takes only its own part.
|
||||
@@ -1281,6 +1348,7 @@ mod catalog_round_trip {
|
||||
confidence: 0.87,
|
||||
embedding: vec![seed; 1024],
|
||||
crop_px: 180.0,
|
||||
quality: None,
|
||||
crop: vec![seed; 64],
|
||||
}
|
||||
}
|
||||
@@ -1430,6 +1498,10 @@ mod catalog_round_trip {
|
||||
let (faces, _) = store.get_image(77, "w600k_mbf").unwrap().unwrap();
|
||||
assert_eq!(faces.len(), 1);
|
||||
assert!(faces[0].crop.is_empty(), "a crop was invented from nowhere");
|
||||
assert_eq!(
|
||||
faces[0].quality, None,
|
||||
"a quality was invented from nowhere"
|
||||
);
|
||||
let _ = std::fs::remove_dir_all(&dir);
|
||||
}
|
||||
|
||||
|
||||
@@ -61,10 +61,22 @@ pub struct DetectedFace {
|
||||
/// Five `(x, y)` pairs, normalised the same way.
|
||||
pub landmarks: [(f32, f32); 5],
|
||||
pub confidence: f32,
|
||||
/// 512 × f16, L2-normalised — `dr_face::Embedding::to_f16_bytes`.
|
||||
/// 512 × f16, the raw model output — `dr_face::Embedded::to_f16_bytes`.
|
||||
///
|
||||
/// Raw rather than unit length, so the length ([`Self::quality`]) is in
|
||||
/// the blob and not only beside it. Readers re-normalise on load.
|
||||
pub embedding: Vec<u8>,
|
||||
/// Source pixels across the aligned crop (docs/faces.md §7).
|
||||
pub crop_px: f32,
|
||||
/// Length of the raw embedding before normalisation — the model's own
|
||||
/// reading of how recognisable the crop was, and the gate on whether
|
||||
/// this face may be compared *against* (`dr_face::MIN_GALLERY_QUALITY`).
|
||||
///
|
||||
/// `None` where it was never measured: a face indexed, here or by a peer,
|
||||
/// before raw vectors were stored. The unit vector those builds kept has
|
||||
/// no length left to read, so the only way to measure one is to embed it
|
||||
/// again (schema V14).
|
||||
pub quality: Option<f32>,
|
||||
/// Which model produced the embedding. Comparing across models is the one
|
||||
/// mistake that yields plausible garbage rather than an error.
|
||||
pub model_id: String,
|
||||
@@ -94,6 +106,9 @@ pub struct Face {
|
||||
pub landmarks: [(f32, f32); 5],
|
||||
pub confidence: f32,
|
||||
pub crop_px: f32,
|
||||
/// See [`DetectedFace::quality`]. `None` for a face indexed before it was
|
||||
/// recorded.
|
||||
pub quality: Option<f32>,
|
||||
pub model_id: String,
|
||||
/// `None` when the face belongs to no one yet.
|
||||
pub person: Option<PersonId>,
|
||||
@@ -184,8 +199,8 @@ pub fn record_detections(
|
||||
tx.execute(
|
||||
"INSERT INTO faces
|
||||
(image_id, x, y, w, h, landmarks, detector_confidence,
|
||||
embedding, crop_px, model_id, detected_at, crop)
|
||||
VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, ?11, ?12)",
|
||||
embedding, crop_px, model_id, detected_at, crop, quality)
|
||||
VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, ?11, ?12, ?13)",
|
||||
rusqlite::params![
|
||||
image_id.0 as i64,
|
||||
f.x as f64,
|
||||
@@ -202,6 +217,7 @@ pub fn record_detections(
|
||||
// the database rather than two the readers each have to know
|
||||
// about.
|
||||
(!f.crop.is_empty()).then_some(f.crop.as_slice()),
|
||||
f.quality.map(f64::from),
|
||||
],
|
||||
)?;
|
||||
let id = FaceId(tx.last_insert_rowid() as u64);
|
||||
@@ -366,7 +382,7 @@ pub fn for_image(conn: &Connection, image_id: ImageId) -> Result<Vec<Face>, Cata
|
||||
let mut q = conn.prepare(
|
||||
"SELECT f.id, f.image_id, f.x, f.y, f.w, f.h, f.landmarks,
|
||||
f.detector_confidence, f.crop_px, f.model_id,
|
||||
fp.person_id, fp.probability, fp.confirmed
|
||||
fp.person_id, fp.probability, fp.confirmed, f.quality
|
||||
FROM faces f
|
||||
LEFT JOIN face_person fp ON fp.face_id = f.id
|
||||
WHERE f.image_id = ?1
|
||||
@@ -393,9 +409,18 @@ pub fn unassigned(conn: &Connection, model_id: &str) -> Result<Vec<FaceId>, Cata
|
||||
|
||||
/// One face's stored embedding, as the clustering pass consumes it.
|
||||
///
|
||||
/// A named type rather than a tuple because it crosses a crate boundary and
|
||||
/// "the third element" is not a thing anyone should have to remember.
|
||||
pub type StoredEmbedding = (FaceId, ImageId, Vec<u8>, f32);
|
||||
/// A struct rather than a tuple because it crosses a crate boundary and "the
|
||||
/// fourth element" is not a thing anyone should have to remember.
|
||||
#[derive(Debug, Clone, PartialEq)]
|
||||
pub struct StoredEmbedding {
|
||||
pub face: FaceId,
|
||||
pub image: ImageId,
|
||||
/// 512 × f16 — `dr_face::Embedding::from_f16_bytes` reads it.
|
||||
pub embedding: Vec<u8>,
|
||||
pub crop_px: f32,
|
||||
/// See [`DetectedFace::quality`].
|
||||
pub quality: Option<f32>,
|
||||
}
|
||||
|
||||
/// Embeddings for clustering, oldest first so the pass is deterministic.
|
||||
///
|
||||
@@ -404,16 +429,17 @@ pub type StoredEmbedding = (FaceId, ImageId, Vec<u8>, f32);
|
||||
/// would double the memory of the one operation that holds them all at once.
|
||||
pub fn embeddings(conn: &Connection, model_id: &str) -> Result<Vec<StoredEmbedding>, CatalogError> {
|
||||
let mut q = conn.prepare(
|
||||
"SELECT id, image_id, embedding, crop_px FROM faces
|
||||
"SELECT id, image_id, embedding, crop_px, quality FROM faces
|
||||
WHERE model_id = ?1 ORDER BY id",
|
||||
)?;
|
||||
let rows = q.query_map([model_id], |r| {
|
||||
Ok((
|
||||
FaceId(r.get::<_, i64>(0)? as u64),
|
||||
ImageId(r.get::<_, i64>(1)? as u64),
|
||||
r.get::<_, Vec<u8>>(2)?,
|
||||
r.get::<_, f64>(3)? as f32,
|
||||
))
|
||||
Ok(StoredEmbedding {
|
||||
face: FaceId(r.get::<_, i64>(0)? as u64),
|
||||
image: ImageId(r.get::<_, i64>(1)? as u64),
|
||||
embedding: r.get::<_, Vec<u8>>(2)?,
|
||||
crop_px: r.get::<_, f64>(3)? as f32,
|
||||
quality: r.get::<_, Option<f64>>(4)?.map(|q| q as f32),
|
||||
})
|
||||
})?;
|
||||
rows.collect::<Result<_, _>>().map_err(Into::into)
|
||||
}
|
||||
@@ -643,7 +669,7 @@ pub fn for_person(
|
||||
let mut q = conn.prepare(
|
||||
"SELECT f.id, f.image_id, f.x, f.y, f.w, f.h, f.landmarks,
|
||||
f.detector_confidence, f.crop_px, f.model_id,
|
||||
fp.person_id, fp.probability, fp.confirmed
|
||||
fp.person_id, fp.probability, fp.confirmed, f.quality
|
||||
FROM faces f
|
||||
JOIN face_person fp ON fp.face_id = f.id
|
||||
WHERE fp.person_id = ?1 AND (?2 OR fp.confirmed = 1)
|
||||
@@ -894,6 +920,7 @@ fn read_face(r: &rusqlite::Row<'_>) -> rusqlite::Result<Face> {
|
||||
landmarks: blob_to_landmarks(&r.get::<_, Vec<u8>>(6)?),
|
||||
confidence: r.get::<_, f64>(7)? as f32,
|
||||
crop_px: r.get::<_, f64>(8)? as f32,
|
||||
quality: r.get::<_, Option<f64>>(13)?.map(|q| q as f32),
|
||||
model_id: r.get(9)?,
|
||||
person: person.map(|p| PersonId(p as u64)),
|
||||
probability: r.get::<_, Option<f64>>(11)?.unwrap_or(0.0) as f32,
|
||||
@@ -1016,6 +1043,7 @@ mod tests {
|
||||
confidence: 0.9,
|
||||
embedding: vec![seed; 1024],
|
||||
crop_px: 180.0,
|
||||
quality: Some(10.0 + f32::from(seed)),
|
||||
model_id: "w600k_mbf".into(),
|
||||
crop: Vec::new(),
|
||||
}
|
||||
@@ -1041,6 +1069,15 @@ mod tests {
|
||||
assert!((got[0].crop_px - 180.0).abs() < 1e-3);
|
||||
assert!((got[0].landmarks[2].1 - 0.2).abs() < 1e-5);
|
||||
assert!(got[0].person.is_none());
|
||||
// Both readers carry the quality, and the one for the grouping pass
|
||||
// carries it as the option it is.
|
||||
let mut qualities: Vec<Option<f32>> = got.iter().map(|f| f.quality).collect();
|
||||
qualities.sort_by(|a, b| a.partial_cmp(b).unwrap());
|
||||
assert_eq!(qualities, vec![Some(11.0), Some(12.0)]);
|
||||
let stored = embeddings(&c, "w600k_mbf").unwrap();
|
||||
assert_eq!(stored.len(), 2);
|
||||
assert_eq!(stored[0].quality, Some(11.0), "oldest first");
|
||||
assert_eq!(stored[1].quality, Some(12.0));
|
||||
}
|
||||
|
||||
/// Re-detection is coalesced per image, so it must replace rather than
|
||||
@@ -1450,10 +1487,11 @@ mod tests {
|
||||
record_detections(&c, img, "w600k_mbf", 1024, &[face(7)]).unwrap();
|
||||
let e = embeddings(&c, "w600k_mbf").unwrap();
|
||||
assert_eq!(e.len(), 1);
|
||||
assert_eq!(e[0].1, img);
|
||||
assert_eq!(e[0].2.len(), 1024);
|
||||
assert_eq!(e[0].2[0], 7);
|
||||
assert!((e[0].3 - 180.0).abs() < 1e-3);
|
||||
assert_eq!(e[0].image, img);
|
||||
assert_eq!(e[0].embedding.len(), 1024);
|
||||
assert_eq!(e[0].embedding[0], 7);
|
||||
assert!((e[0].crop_px - 180.0).abs() < 1e-3);
|
||||
assert_eq!(e[0].quality, Some(17.0));
|
||||
}
|
||||
|
||||
// ── stored crops ──────────────────────────────────────────────────────
|
||||
|
||||
@@ -15,7 +15,7 @@ use rusqlite::Connection;
|
||||
use crate::error::CatalogError;
|
||||
|
||||
/// Schema version this build writes and understands.
|
||||
pub const SCHEMA_VERSION: i64 = 13;
|
||||
pub const SCHEMA_VERSION: i64 = 14;
|
||||
|
||||
/// Apply migrations up to [`SCHEMA_VERSION`].
|
||||
///
|
||||
@@ -119,6 +119,23 @@ pub fn migrate(conn: &Connection) -> Result<i64, CatalogError> {
|
||||
tx.commit()?;
|
||||
}
|
||||
|
||||
if from < 14 {
|
||||
let tx = conn.unchecked_transaction()?;
|
||||
// `ALTER TABLE ... ADD COLUMN` has no `IF NOT EXISTS`, and NFR-R5
|
||||
// wants this re-enterable: a catalog whose `user_version` was rewound
|
||||
// by a rollback already has the column, and would otherwise fail its
|
||||
// next open on it.
|
||||
let has_quality: bool = tx
|
||||
.prepare("SELECT 1 FROM pragma_table_info('faces') WHERE name = 'quality'")?
|
||||
.exists([])?;
|
||||
if !has_quality {
|
||||
tx.execute_batch("ALTER TABLE faces ADD COLUMN quality REAL;")?;
|
||||
}
|
||||
tx.execute_batch(V14)?;
|
||||
tx.pragma_update(None, "user_version", 14)?;
|
||||
tx.commit()?;
|
||||
}
|
||||
|
||||
Ok(from)
|
||||
}
|
||||
|
||||
@@ -251,7 +268,8 @@ pub fn for_attached(schema_name: &str) -> String {
|
||||
format!(
|
||||
"{}\n{}\n{}\n\
|
||||
ALTER TABLE {schema_name}.people ADD COLUMN ignored INTEGER NOT NULL DEFAULT 0;\n\
|
||||
ALTER TABLE {schema_name}.faces ADD COLUMN crop BLOB;",
|
||||
ALTER TABLE {schema_name}.faces ADD COLUMN crop BLOB;\n\
|
||||
ALTER TABLE {schema_name}.faces ADD COLUMN quality REAL;",
|
||||
rewrite_for_attached(V1, schema_name),
|
||||
rewrite_for_attached(V6, schema_name),
|
||||
rewrite_for_attached(V8, schema_name),
|
||||
@@ -569,6 +587,55 @@ CREATE TABLE IF NOT EXISTS sidecars (
|
||||
);
|
||||
"#;
|
||||
|
||||
const V14: &str = r#"
|
||||
-- TRACES: FR-CULL-9 | FR-CULL-10
|
||||
-- How recognisable the model found each face, and a second look at the faces
|
||||
-- it was never asked about.
|
||||
--
|
||||
-- The embedder's raw output has a length, and the length is a quality
|
||||
-- reading: it grows with how much of a face the model could make out, and a
|
||||
-- blur, an occlusion or a hard profile comes out short (dr_face::embedding,
|
||||
-- `MIN_GALLERY_QUALITY`). Normalising threw it away. A short vector sits
|
||||
-- near the middle of the sphere and matches a little of everyone, which is
|
||||
-- how one bad crop bridges two people in a grouping pass -- so a face below
|
||||
-- the floor is compared against the others and never compared *against*.
|
||||
--
|
||||
-- Nullable, and NULL means "never measured": every face indexed before this
|
||||
-- version stored the unit vector, whose length is one whatever the crop was.
|
||||
-- A face with no reading is admitted to the gallery, because a rule that
|
||||
-- cannot be checked should admit rather than exclude -- but it is also a
|
||||
-- face this rule is not yet protecting anyone from, and the only way to
|
||||
-- measure it is to embed it again.
|
||||
--
|
||||
-- So the run markers of every image that holds a face are forgotten, exactly
|
||||
-- as V12 forgot the runs made against too small a proxy, and for the same
|
||||
-- reason: the work list is "images with no marker", and an image examined
|
||||
-- without this number is otherwise indistinguishable from one examined with
|
||||
-- it. The faces stay where they are and keep drawing the People screen until
|
||||
-- the next pass replaces them, `record_detections` carries the user's
|
||||
-- confirmations across by box overlap, and an image that was examined and
|
||||
-- found empty keeps its marker -- there is nothing on it to re-measure.
|
||||
--
|
||||
-- The cost is a re-fetch of every image with a face on it, on the next pass
|
||||
-- the user starts. That is a whole-library transfer (FR-NC-6), and it starts
|
||||
-- when they say so, not here.
|
||||
--
|
||||
-- From this version the `embedding` blob is the **raw** model output rather
|
||||
-- than the unit vector V8 describes -- the length is the quality, and a store
|
||||
-- that kept only the direction had thrown it away. Readers re-normalise on
|
||||
-- load, so a unit blob from before and a raw blob from now compare alike;
|
||||
-- `quality` is that length kept beside the blob for the readers that never
|
||||
-- load the vector, and NULL rather than 1.0 for the old rows, because a unit
|
||||
-- vector reads as a length of one and one is not "unmeasured".
|
||||
--
|
||||
-- The column itself is added in `migrate`, guarded, because ALTER has no
|
||||
-- IF NOT EXISTS and this step has to be re-enterable (NFR-R5).
|
||||
DELETE FROM face_index
|
||||
WHERE EXISTS (SELECT 1 FROM faces f
|
||||
WHERE f.image_id = face_index.image_id
|
||||
AND f.model_id = face_index.model_id);
|
||||
"#;
|
||||
|
||||
const V9: &str = r#"
|
||||
-- TRACES: FR-CULL-8
|
||||
-- A record that face detection has *run* on an image, distinct from what it
|
||||
@@ -648,7 +715,7 @@ CREATE TABLE faces (
|
||||
x REAL NOT NULL, y REAL NOT NULL, w REAL NOT NULL, h REAL NOT NULL,
|
||||
landmarks BLOB NOT NULL, -- 5 x (x, y) f32, normalised likewise
|
||||
detector_confidence REAL NOT NULL,
|
||||
embedding BLOB NOT NULL, -- 512 x f16, L2-normalised
|
||||
embedding BLOB NOT NULL, -- 512 x f16; unit length until V14, raw since
|
||||
-- Source pixels across the aligned 112x112 crop (docs/faces.md §7).
|
||||
--
|
||||
-- Not cosmetic: it is the honest quality signal for the UI, a feature in
|
||||
@@ -1405,6 +1472,64 @@ mod tests {
|
||||
assert_eq!(kept, vec![3, 4]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn v14_forgets_runs_that_found_faces_but_never_measured_them() {
|
||||
let c = mem();
|
||||
c.pragma_update(None, "user_version", 0).unwrap();
|
||||
migrate(&c).unwrap();
|
||||
c.execute(
|
||||
"INSERT INTO roots(id, kind, label) VALUES (1, 'local', 'test')",
|
||||
[],
|
||||
)
|
||||
.unwrap();
|
||||
c.execute(
|
||||
"INSERT INTO images(id, root_id, source_ref, added_at)
|
||||
VALUES (1,1,'a',0),(2,1,'b',0),(3,1,'c',0)",
|
||||
[],
|
||||
)
|
||||
.unwrap();
|
||||
// Image 1 was examined and holds a face; 2 was examined and found
|
||||
// empty; 3 holds a face found by a different model.
|
||||
for (image, model) in [(1, "m"), (2, "m"), (3, "m")] {
|
||||
c.execute(
|
||||
"INSERT INTO face_index(image_id, model_id, indexed_at, faces_found, source_edge)
|
||||
VALUES (?1, ?2, 0, 0, 2560)",
|
||||
rusqlite::params![image, model],
|
||||
)
|
||||
.unwrap();
|
||||
}
|
||||
for (image, model) in [(1, "m"), (3, "other")] {
|
||||
c.execute(
|
||||
"INSERT INTO faces
|
||||
(image_id, x, y, w, h, landmarks, detector_confidence, embedding,
|
||||
crop_px, model_id, detected_at)
|
||||
VALUES (?1, 0.1, 0.1, 0.2, 0.2, X'00', 0.9, X'00', 180.0, ?2, 0)",
|
||||
rusqlite::params![image, model],
|
||||
)
|
||||
.unwrap();
|
||||
}
|
||||
c.pragma_update(None, "user_version", 13).unwrap();
|
||||
|
||||
migrate(&c).unwrap();
|
||||
|
||||
let kept: Vec<i64> = c
|
||||
.prepare("SELECT image_id FROM face_index ORDER BY image_id")
|
||||
.unwrap()
|
||||
.query_map([], |r| r.get(0))
|
||||
.unwrap()
|
||||
.map(Result::unwrap)
|
||||
.collect();
|
||||
// 1 goes: it has a face with no quality. 2 stays: nothing on it to
|
||||
// measure. 3 stays: its face belongs to a run this marker does not
|
||||
// describe.
|
||||
assert_eq!(kept, vec![2, 3]);
|
||||
// And the faces themselves are untouched.
|
||||
let faces: i64 = c
|
||||
.query_row("SELECT count(*) FROM faces", [], |r| r.get(0))
|
||||
.unwrap();
|
||||
assert_eq!(faces, 2);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn job_uniqueness_coalesces_rather_than_duplicating() {
|
||||
let c = mem();
|
||||
|
||||
@@ -63,15 +63,16 @@ fn main() {
|
||||
let embed_ms = t.elapsed().as_secs_f64() * 1e3;
|
||||
|
||||
println!(
|
||||
" [{i}] conf {:.3} box {:.0},{:.0} {:.0}×{:.0} crop_px {:.0} embed {embed_ms:.0} ms",
|
||||
" [{i}] conf {:.3} box {:.0},{:.0} {:.0}×{:.0} crop_px {:.0} quality {:.1} embed {embed_ms:.0} ms",
|
||||
d.confidence,
|
||||
d.bbox.0,
|
||||
d.bbox.1,
|
||||
d.width(),
|
||||
d.height(),
|
||||
aligned.source_px(),
|
||||
emb.quality,
|
||||
);
|
||||
all.push((path.clone(), i, emb));
|
||||
all.push((path.clone(), i, emb.embedding));
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -46,11 +46,13 @@ fn main() {
|
||||
);
|
||||
for n in sizes {
|
||||
let (embeddings, crop_px, images) = population(n);
|
||||
let gallery = vec![true; n];
|
||||
let faces = Faces {
|
||||
embeddings: &embeddings,
|
||||
dim: EMBEDDING_DIM,
|
||||
crop_px: &crop_px,
|
||||
images: &images,
|
||||
gallery: &gallery,
|
||||
};
|
||||
|
||||
let start = std::time::Instant::now();
|
||||
|
||||
+48
-11
@@ -136,11 +136,24 @@ pub const RIVAL_FLOOR: f32 = 0.5;
|
||||
///
|
||||
/// A face in no group, or one with no evidence for anybody, scores 0.
|
||||
///
|
||||
/// `gallery` is one flag per face — which faces may be evidence at all
|
||||
/// ([`crate::embedding::MIN_GALLERY_QUALITY`]). Its length is the face count.
|
||||
/// A pair is evidence *about* either face but only *from* a gallery one: a
|
||||
/// probe learns from the references it matched, and a reference learns nothing
|
||||
/// from a probe that happened to match it, however well. Without that, the one
|
||||
/// short vector in a group would be the strongest match every face in it had.
|
||||
///
|
||||
/// `pairs` must be the *evidence* list — scanned at [`RIVAL_FLOOR`], not at the
|
||||
/// merge threshold. Passing the merge list still works but silently removes
|
||||
/// every rival weaker than a merge, which is most of them, and every uniqueness
|
||||
/// collapses to 1.
|
||||
pub fn identity_shares(faces: usize, clusters: &[Cluster], pairs: &[Pair], top: usize) -> Vec<f32> {
|
||||
pub fn identity_shares(
|
||||
gallery: &[bool],
|
||||
clusters: &[Cluster],
|
||||
pairs: &[Pair],
|
||||
top: usize,
|
||||
) -> Vec<f32> {
|
||||
let faces = gallery.len();
|
||||
// An identity is a *person*, not a group. One person routinely holds
|
||||
// several anchored groups — the same reason they hold several unnamed ones
|
||||
// — and keying this by group had Catherine competing with Catherine, which
|
||||
@@ -174,15 +187,16 @@ pub fn identity_shares(faces: usize, clusters: &[Cluster], pairs: &[Pair], top:
|
||||
}
|
||||
// A pair is evidence in both directions: j's identity hears about i,
|
||||
// and i's identity hears about j. The pair list holds each unordered
|
||||
// pair once, so both have to be recorded here.
|
||||
// pair once, so both have to be recorded here — each only where the
|
||||
// face doing the telling is in the gallery.
|
||||
let (gi, gj) = (group_of[p.i], group_of[p.j]);
|
||||
if gj != usize::MAX {
|
||||
if gj != usize::MAX && gallery[p.j] {
|
||||
evidence[p.i]
|
||||
.entry(key_of[gj])
|
||||
.or_default()
|
||||
.push(p.probability);
|
||||
}
|
||||
if gi != usize::MAX {
|
||||
if gi != usize::MAX && gallery[p.i] {
|
||||
evidence[p.j]
|
||||
.entry(key_of[gi])
|
||||
.or_default()
|
||||
@@ -255,6 +269,11 @@ mod tests {
|
||||
Pair { i, j, probability }
|
||||
}
|
||||
|
||||
/// `n` faces, every one of them fit to be compared against.
|
||||
fn all(n: usize) -> Vec<bool> {
|
||||
vec![true; n]
|
||||
}
|
||||
|
||||
/// The failure the module exists to fix: face 0 matches its own group's
|
||||
/// three members strongly, and the group has forty more it is unrelated to.
|
||||
/// The old within-group mean reported ~0.07 for this.
|
||||
@@ -264,7 +283,7 @@ mod tests {
|
||||
let clusters = vec![cluster(&members)];
|
||||
let pairs = vec![pair(0, 1, 0.99), pair(0, 2, 0.97), pair(0, 3, 0.95)];
|
||||
|
||||
let shares = identity_shares(44, &clusters, &pairs, TOP_MATCHES);
|
||||
let shares = identity_shares(&all(44), &clusters, &pairs, TOP_MATCHES);
|
||||
assert!(
|
||||
(shares[0] - 0.97).abs() < 1e-6,
|
||||
"the mean of its three real matches, undiluted: {}",
|
||||
@@ -284,7 +303,7 @@ mod tests {
|
||||
pair(0, 4, 0.90),
|
||||
];
|
||||
|
||||
let shares = identity_shares(5, &clusters, &pairs, TOP_MATCHES);
|
||||
let shares = identity_shares(&all(5), &clusters, &pairs, TOP_MATCHES);
|
||||
// Coherent at 0.90, and only half of the evidence is its own.
|
||||
assert!(
|
||||
(shares[0] - 0.45).abs() < 1e-6,
|
||||
@@ -298,9 +317,9 @@ mod tests {
|
||||
#[test]
|
||||
fn a_rival_too_weak_to_merge_still_lowers_the_confidence() {
|
||||
let clusters = vec![named(&[0, 1], 1), named(&[2, 3], 2)];
|
||||
let sure = identity_shares(4, &clusters, &[pair(0, 1, 0.95)], TOP_MATCHES);
|
||||
let sure = identity_shares(&all(4), &clusters, &[pair(0, 1, 0.95)], TOP_MATCHES);
|
||||
let contested = identity_shares(
|
||||
4,
|
||||
&all(4),
|
||||
&clusters,
|
||||
&[pair(0, 1, 0.95), pair(0, 2, 0.60)],
|
||||
TOP_MATCHES,
|
||||
@@ -321,7 +340,7 @@ mod tests {
|
||||
fn an_unnamed_group_is_not_treated_as_competition() {
|
||||
let clusters = vec![cluster(&[0, 1]), cluster(&[2, 3])];
|
||||
let shares = identity_shares(
|
||||
4,
|
||||
&all(4),
|
||||
&clusters,
|
||||
&[pair(0, 1, 0.95), pair(0, 2, 0.90)],
|
||||
TOP_MATCHES,
|
||||
@@ -343,7 +362,7 @@ mod tests {
|
||||
let mut pairs: Vec<Pair> = (1..11).map(|j| pair(0, j, 0.90)).collect();
|
||||
pairs.extend((11..62).map(|j| pair(0, j, 0.55)));
|
||||
|
||||
let shares = identity_shares(62, &clusters, &pairs, TOP_MATCHES);
|
||||
let shares = identity_shares(&all(62), &clusters, &pairs, TOP_MATCHES);
|
||||
// Ten at 0.90 against ten at 0.55 — not fifty-one at 0.55.
|
||||
assert!(
|
||||
(shares[0] - 0.90 * (9.0 / 14.5)).abs() < 1e-5,
|
||||
@@ -352,11 +371,29 @@ mod tests {
|
||||
);
|
||||
}
|
||||
|
||||
/// A probe learns from the references it matched; a reference learns
|
||||
/// nothing from a probe. The pair is the same pair — what differs is who
|
||||
/// is doing the telling.
|
||||
#[test]
|
||||
fn a_face_outside_the_gallery_is_nobody_s_evidence() {
|
||||
let clusters = vec![named(&[0, 1, 2], 1)];
|
||||
let gallery = vec![true, true, false];
|
||||
let pairs = vec![pair(0, 1, 0.80), pair(0, 2, 0.99), pair(1, 2, 0.99)];
|
||||
|
||||
let shares = identity_shares(&gallery, &clusters, &pairs, TOP_MATCHES);
|
||||
// Faces 0 and 1 hear only from each other: the 0.99 the probe offered
|
||||
// them is not counted.
|
||||
assert!((shares[0] - 0.80).abs() < 1e-6, "{}", shares[0]);
|
||||
assert!((shares[1] - 0.80).abs() < 1e-6, "{}", shares[1]);
|
||||
// The probe hears from both references.
|
||||
assert!((shares[2] - 0.99).abs() < 1e-6, "{}", shares[2]);
|
||||
}
|
||||
|
||||
/// A face nothing has any evidence about claims nothing.
|
||||
#[test]
|
||||
fn a_face_with_no_evidence_reports_no_confidence() {
|
||||
let clusters = vec![cluster(&[0, 1])];
|
||||
let shares = identity_shares(2, &clusters, &[], TOP_MATCHES);
|
||||
let shares = identity_shares(&all(2), &clusters, &[], TOP_MATCHES);
|
||||
assert_eq!(shares, vec![0.0, 0.0]);
|
||||
}
|
||||
}
|
||||
|
||||
+307
-3
@@ -19,6 +19,23 @@
|
||||
//! and clustering never moves it. Two groups holding confirmations of
|
||||
//! *different* people cannot merge, whatever their similarity says.
|
||||
//!
|
||||
//! # The gallery, and the faces that are only ever compared against it
|
||||
//!
|
||||
//! A third defence, and the cheapest of all: **a short embedding is never a
|
||||
//! reference.** The length of the raw vector is the model's own reading of
|
||||
//! how recognisable the crop was ([`crate::embedding::MIN_GALLERY_QUALITY`]),
|
||||
//! and a short one sits near the centre of the sphere, matching a little of
|
||||
//! everybody. One of those in a group is a bridge to the next group over.
|
||||
//!
|
||||
//! So the population is split. Faces at or above the floor are the
|
||||
//! **gallery**, and they cluster exactly as described below. Faces under it
|
||||
//! are **probes**: each is measured against the finished groups and joins the
|
||||
//! one it fits, by the same average-link rule and under the same constraints
|
||||
//! — but it is measured against the gallery members only, never against
|
||||
//! another probe, and once placed it is never part of what the next face is
|
||||
//! measured against. A blurred photograph of a known person is still named;
|
||||
//! it just cannot vouch for anyone else.
|
||||
//!
|
||||
//! # Average link, not single link
|
||||
//!
|
||||
//! Single-link chains: one bad edge welds two identities together, and it is
|
||||
@@ -117,6 +134,11 @@ pub struct Candidate {
|
||||
pub embedding: Vec<f32>,
|
||||
/// Source pixels across the aligned crop, for the calibration's size term.
|
||||
pub crop_px: f32,
|
||||
/// Length of the raw embedding, where it was recorded
|
||||
/// ([`crate::embedding::MIN_GALLERY_QUALITY`]). `None` for a face indexed
|
||||
/// before it was kept, which is admitted to the gallery — see
|
||||
/// [`Candidate::in_gallery`].
|
||||
pub quality: Option<f32>,
|
||||
/// The person this face is *confirmed* to be, if any.
|
||||
///
|
||||
/// Suggestions are deliberately not passed here. They are this function's
|
||||
@@ -125,6 +147,13 @@ pub struct Candidate {
|
||||
pub confirmed_person: Option<u64>,
|
||||
}
|
||||
|
||||
impl Candidate {
|
||||
/// Whether this face may be compared *against*, as well as compared.
|
||||
pub fn in_gallery(&self) -> bool {
|
||||
crate::embedding::in_gallery(self.quality)
|
||||
}
|
||||
}
|
||||
|
||||
/// One group of faces the clusterer believes are one person.
|
||||
#[derive(Debug, Clone, PartialEq)]
|
||||
pub struct Cluster {
|
||||
@@ -202,7 +231,7 @@ pub fn cluster_scored(faces: &[Candidate], cal: &Calibration, min_probability: f
|
||||
|
||||
let clusters = build(faces, cal, min_probability, &merges);
|
||||
let confidence = crate::assign::identity_shares(
|
||||
faces.len(),
|
||||
&columns.gallery,
|
||||
&clusters,
|
||||
&evidence,
|
||||
crate::assign::TOP_MATCHES,
|
||||
@@ -225,6 +254,7 @@ struct Columns {
|
||||
dim: usize,
|
||||
crop_px: Vec<f32>,
|
||||
images: Vec<u64>,
|
||||
gallery: Vec<bool>,
|
||||
}
|
||||
|
||||
impl Columns {
|
||||
@@ -245,6 +275,7 @@ impl Columns {
|
||||
dim,
|
||||
crop_px: faces.iter().map(|f| f.crop_px).collect(),
|
||||
images: faces.iter().map(|f| f.image).collect(),
|
||||
gallery: faces.iter().map(Candidate::in_gallery).collect(),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -254,22 +285,171 @@ impl Columns {
|
||||
dim: self.dim,
|
||||
crop_px: &self.crop_px,
|
||||
images: &self.images,
|
||||
gallery: &self.gallery,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Agglomerate the gallery over its pairs, then place the probes.
|
||||
///
|
||||
/// `pairs` is what [`neighbours::above_threshold`] returned: every pair has a
|
||||
/// gallery side, but a pair with a probe on the other side is not a merge —
|
||||
/// it is the evidence [`place_probes`] works from. Only the gallery-to-gallery
|
||||
/// pairs reach the engine, so a probe enters it as a singleton with no edges
|
||||
/// and comes out exactly as it went in.
|
||||
fn build(
|
||||
faces: &[Candidate],
|
||||
cal: &Calibration,
|
||||
min_probability: f32,
|
||||
pairs: &[neighbours::Pair],
|
||||
) -> Vec<Cluster> {
|
||||
let gallery: Vec<bool> = faces.iter().map(Candidate::in_gallery).collect();
|
||||
let (merges, probe_pairs): (Vec<_>, Vec<_>) = pairs
|
||||
.iter()
|
||||
.copied()
|
||||
.partition(|p| gallery[p.i] && gallery[p.j]);
|
||||
|
||||
let mut engine = Engine::new(faces, cal, min_probability);
|
||||
let parts = components(faces.len(), pairs);
|
||||
let parts = components(faces.len(), &merges);
|
||||
for (component, edges) in parts.members.iter().zip(&parts.edges) {
|
||||
engine.agglomerate(component, edges);
|
||||
}
|
||||
engine.finish()
|
||||
let dot = engine.dot;
|
||||
let clusters = engine.finish();
|
||||
if probe_pairs.is_empty() {
|
||||
return clusters;
|
||||
}
|
||||
place_probes(
|
||||
faces,
|
||||
cal,
|
||||
min_probability,
|
||||
dot,
|
||||
&gallery,
|
||||
clusters,
|
||||
&probe_pairs,
|
||||
)
|
||||
}
|
||||
|
||||
/// Put each probe into the finished group it fits, or leave it alone.
|
||||
///
|
||||
/// The same decision the engine makes for a singleton — average link over the
|
||||
/// group, at or above `min_probability`, subject to [`Engine::can_link`]'s two
|
||||
/// constraints — with one difference that is the whole point: the average is
|
||||
/// over the group's **gallery** members. A probe already placed is not part of
|
||||
/// what the next one is measured against, so a run of short vectors cannot
|
||||
/// pull each other in one after another.
|
||||
///
|
||||
/// Probes are placed in index order and each placement is final, which is
|
||||
/// what keeps this deterministic. The group a probe joins gains its
|
||||
/// photograph, so a second face from the same frame cannot follow it — the
|
||||
/// co-occurrence rule, applied exactly as the engine applies it.
|
||||
fn place_probes(
|
||||
faces: &[Candidate],
|
||||
cal: &Calibration,
|
||||
min_probability: f32,
|
||||
dot: neighbours::DotFn,
|
||||
gallery: &[bool],
|
||||
mut clusters: Vec<Cluster>,
|
||||
probe_pairs: &[neighbours::Pair],
|
||||
) -> Vec<Cluster> {
|
||||
// Where each face sits, and what each group's photographs and gallery
|
||||
// members are. The probe's own singleton is here too, and is dropped once
|
||||
// it has moved.
|
||||
let mut group_of = vec![usize::MAX; faces.len()];
|
||||
for (g, c) in clusters.iter().enumerate() {
|
||||
for &m in &c.members {
|
||||
group_of[m] = g;
|
||||
}
|
||||
}
|
||||
let mut images: Vec<HashSet<u64>> = clusters
|
||||
.iter()
|
||||
.map(|c| c.members.iter().map(|&m| faces[m].image).collect())
|
||||
.collect();
|
||||
let references: Vec<Vec<usize>> = clusters
|
||||
.iter()
|
||||
.map(|c| c.members.iter().copied().filter(|&m| gallery[m]).collect())
|
||||
.collect();
|
||||
|
||||
// Which groups each probe has any above-threshold pair into. Only those
|
||||
// can average above the threshold — the argument the module note makes
|
||||
// for the engine holds here unchanged.
|
||||
let mut candidates: Vec<Vec<usize>> = vec![Vec::new(); faces.len()];
|
||||
for p in probe_pairs {
|
||||
let (probe, reference) = if gallery[p.i] { (p.j, p.i) } else { (p.i, p.j) };
|
||||
candidates[probe].push(group_of[reference]);
|
||||
}
|
||||
|
||||
let mut moved: Vec<usize> = Vec::new();
|
||||
for probe in 0..faces.len() {
|
||||
if gallery[probe] || candidates[probe].is_empty() {
|
||||
continue;
|
||||
}
|
||||
let mut groups = std::mem::take(&mut candidates[probe]);
|
||||
groups.sort_unstable();
|
||||
groups.dedup();
|
||||
|
||||
let face = &faces[probe];
|
||||
let mut best: Option<(f32, usize)> = None;
|
||||
for g in groups {
|
||||
let target = &clusters[g];
|
||||
if let (Some(mine), Some(theirs)) = (face.confirmed_person, target.person) {
|
||||
if mine != theirs {
|
||||
continue;
|
||||
}
|
||||
}
|
||||
if images[g].contains(&face.image) {
|
||||
continue;
|
||||
}
|
||||
let (mut sum, mut count) = (0.0_f64, 0.0_f64);
|
||||
for &r in &references[g] {
|
||||
let cos = dot(&face.embedding, &faces[r].embedding);
|
||||
let min_crop = face.crop_px.min(faces[r].crop_px);
|
||||
sum += cal.probability(cos, min_crop, 0.0) as f64;
|
||||
count += 1.0;
|
||||
}
|
||||
if count == 0.0 {
|
||||
continue;
|
||||
}
|
||||
let p = (sum / count) as f32;
|
||||
// Strictly better wins; on a tie the lowest group index, which is
|
||||
// the engine's own tiebreak.
|
||||
if p >= min_probability && best.is_none_or(|(bp, _)| p > bp) {
|
||||
best = Some((p, g));
|
||||
}
|
||||
}
|
||||
|
||||
let Some((_, g)) = best else { continue };
|
||||
let own = group_of[probe];
|
||||
clusters[g].members.push(probe);
|
||||
clusters[g].members.sort_unstable();
|
||||
clusters[g].person = clusters[g].person.or(face.confirmed_person);
|
||||
images[g].insert(face.image);
|
||||
group_of[probe] = g;
|
||||
moved.push(own);
|
||||
}
|
||||
|
||||
if moved.is_empty() {
|
||||
return clusters;
|
||||
}
|
||||
// The singletons the probes left behind, then the order `Engine::finish`
|
||||
// promises: largest first, lowest member first among equals.
|
||||
let mut vacated = vec![false; clusters.len()];
|
||||
for g in moved {
|
||||
vacated[g] = true;
|
||||
}
|
||||
let mut out: Vec<Cluster> = clusters
|
||||
.into_iter()
|
||||
.zip(vacated)
|
||||
.filter(|(_, gone)| !gone)
|
||||
.map(|(c, _)| c)
|
||||
.collect();
|
||||
out.sort_by(|x, y| {
|
||||
y.members
|
||||
.len()
|
||||
.cmp(&x.members.len())
|
||||
.then(x.members[0].cmp(&y.members[0]))
|
||||
});
|
||||
out
|
||||
}
|
||||
|
||||
/// Split one person's faces into the groups a raised threshold separates them
|
||||
@@ -726,10 +906,19 @@ mod tests {
|
||||
image,
|
||||
embedding: at_cosine(identity, cosine),
|
||||
crop_px: 150.0,
|
||||
quality: None,
|
||||
confirmed_person: None,
|
||||
}
|
||||
}
|
||||
|
||||
/// A face too short to be a reference: compared, never compared against.
|
||||
fn probe(face: u64, image: u64, identity: usize, cosine: f32) -> Candidate {
|
||||
Candidate {
|
||||
quality: Some(crate::embedding::MIN_GALLERY_QUALITY - 5.0),
|
||||
..candidate(face, image, identity, cosine)
|
||||
}
|
||||
}
|
||||
|
||||
/// A calibration steep enough that the test's cosines are unambiguous:
|
||||
/// 0.6 is near-certain, 0.1 is near-impossible.
|
||||
fn cal() -> Calibration {
|
||||
@@ -1099,6 +1288,7 @@ mod tests {
|
||||
image,
|
||||
embedding: at_cosine(p, cosine),
|
||||
crop_px: 60.0 + ((out.len() % 11) as f32) * 25.0,
|
||||
quality: None,
|
||||
confirmed_person: None,
|
||||
});
|
||||
image += 1;
|
||||
@@ -1182,6 +1372,7 @@ mod tests {
|
||||
image: 5_000,
|
||||
embedding: at_cosine(200, 1.0),
|
||||
crop_px: 150.0,
|
||||
quality: None,
|
||||
confirmed_person: None,
|
||||
});
|
||||
let out = cluster(&faces, &cal(), DEFAULT_MERGE_PROBABILITY);
|
||||
@@ -1191,4 +1382,117 @@ mod tests {
|
||||
"the outlier was absorbed"
|
||||
);
|
||||
}
|
||||
|
||||
// ── the gallery ───────────────────────────────────────────────────────
|
||||
|
||||
/// A short vector is still somebody: it joins the group it matches.
|
||||
#[test]
|
||||
fn a_probe_joins_the_group_it_matches() {
|
||||
let faces = vec![
|
||||
candidate(1, 10, 0, 1.0),
|
||||
candidate(2, 11, 0, 0.95),
|
||||
probe(3, 12, 0, 0.92),
|
||||
];
|
||||
let out = cluster(&faces, &cal(), DEFAULT_MERGE_PROBABILITY);
|
||||
assert_eq!(out.len(), 1);
|
||||
assert_eq!(out[0].members, vec![0, 1, 2]);
|
||||
}
|
||||
|
||||
/// Two short vectors that resemble each other are noise agreeing with
|
||||
/// noise, and there is nothing in the gallery for either to be measured
|
||||
/// against.
|
||||
#[test]
|
||||
fn two_probes_are_never_grouped_with_each_other() {
|
||||
let faces = vec![probe(1, 10, 0, 1.0), probe(2, 11, 0, 0.98)];
|
||||
let out = cluster(&faces, &cal(), DEFAULT_MERGE_PROBABILITY);
|
||||
assert_eq!(out.len(), 2, "two probes were grouped: {out:?}");
|
||||
}
|
||||
|
||||
/// The point of measuring against the gallery only: a probe that has been
|
||||
/// placed is not a stepping stone for the next one.
|
||||
#[test]
|
||||
fn a_placed_probe_is_not_what_the_next_probe_is_measured_against() {
|
||||
let mut first = probe(2, 11, 0, 0.6);
|
||||
// 0.6 along identity 0 and 0.8 along its perpendicular: near enough to
|
||||
// the reference to join it, and much nearer to the face below.
|
||||
first.embedding = at_cosine(0, 0.6);
|
||||
let mut second = probe(3, 12, 0, 0.0);
|
||||
second.embedding = at_cosine(0, 0.0);
|
||||
let faces = vec![candidate(1, 10, 0, 1.0), first, second];
|
||||
|
||||
let out = cluster(&faces, &cal(), DEFAULT_MERGE_PROBABILITY);
|
||||
let group = out.iter().find(|c| c.members.contains(&0)).unwrap();
|
||||
assert_eq!(
|
||||
group.members,
|
||||
vec![0, 1],
|
||||
"the first probe should have joined"
|
||||
);
|
||||
assert!(
|
||||
out.iter().any(|c| c.members == vec![2]),
|
||||
"the second probe reached the group through the first: {out:?}"
|
||||
);
|
||||
}
|
||||
|
||||
/// A confirmation on a probe is still the user's word: the group it joins
|
||||
/// becomes that person, and a group already someone else's is closed to it.
|
||||
#[test]
|
||||
fn a_probe_carries_its_confirmation_and_respects_others() {
|
||||
let mut anchored = probe(3, 12, 0, 0.92);
|
||||
anchored.confirmed_person = Some(7);
|
||||
let faces = vec![
|
||||
candidate(1, 10, 0, 1.0),
|
||||
candidate(2, 11, 0, 0.95),
|
||||
anchored,
|
||||
];
|
||||
let out = cluster(&faces, &cal(), DEFAULT_MERGE_PROBABILITY);
|
||||
assert_eq!(out.len(), 1);
|
||||
assert_eq!(out[0].person, Some(7));
|
||||
|
||||
let mut theirs = candidate(1, 10, 0, 1.0);
|
||||
theirs.confirmed_person = Some(8);
|
||||
let faces = vec![theirs, candidate(2, 11, 0, 0.95), {
|
||||
let mut a = probe(3, 12, 0, 0.92);
|
||||
a.confirmed_person = Some(7);
|
||||
a
|
||||
}];
|
||||
let out = cluster(&faces, &cal(), DEFAULT_MERGE_PROBABILITY);
|
||||
assert!(
|
||||
out.iter()
|
||||
.any(|c| c.members == vec![2] && c.person == Some(7)),
|
||||
"a probe confirmed as one person joined another's group: {out:?}"
|
||||
);
|
||||
}
|
||||
|
||||
/// The co-occurrence rule follows a probe in: once it has joined, its
|
||||
/// photograph is the group's.
|
||||
#[test]
|
||||
fn a_probe_cannot_join_a_group_holding_a_face_from_its_own_photograph() {
|
||||
let faces = vec![
|
||||
candidate(1, 10, 0, 1.0),
|
||||
candidate(2, 11, 0, 0.95),
|
||||
probe(3, 10, 0, 0.92),
|
||||
];
|
||||
let out = cluster(&faces, &cal(), DEFAULT_MERGE_PROBABILITY);
|
||||
assert!(out.iter().any(|c| c.members == vec![2]), "{out:?}");
|
||||
}
|
||||
|
||||
/// A probe's placement is scored like anyone else's, from the references
|
||||
/// it matched — and the references' own scores do not hear from it.
|
||||
#[test]
|
||||
fn a_probe_is_scored_but_is_not_evidence() {
|
||||
let gallery_only = vec![candidate(1, 10, 0, 1.0), candidate(2, 11, 0, 0.95)];
|
||||
let without = cluster_scored(&gallery_only, &cal(), DEFAULT_MERGE_PROBABILITY);
|
||||
|
||||
let mut with_probe = gallery_only.clone();
|
||||
with_probe.push(probe(3, 12, 0, 0.99));
|
||||
let with = cluster_scored(&with_probe, &cal(), DEFAULT_MERGE_PROBABILITY);
|
||||
|
||||
assert_eq!(with.clusters[0].members, vec![0, 1, 2]);
|
||||
assert!(with.confidence[2] > 0.9, "{}", with.confidence[2]);
|
||||
assert_eq!(
|
||||
&with.confidence[..2],
|
||||
&without.confidence[..],
|
||||
"a probe changed what the references were sure of"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -16,6 +16,41 @@ use crate::align::{Aligned112, ALIGNED_EDGE};
|
||||
use crate::embedding::{normalise, Embedding, ModelId, EMBEDDING_DIM};
|
||||
use crate::{install_backend, FaceError};
|
||||
|
||||
/// What one pass of the embedder produces: the direction, and the length.
|
||||
///
|
||||
/// Two fields rather than a `quality` on [`Embedding`], because every other
|
||||
/// holder of an `Embedding` relies on it being unit length and compares by
|
||||
/// dot product; the length is a separate fact about the same face, and it is
|
||||
/// stored separately too.
|
||||
#[derive(Debug, Clone, PartialEq)]
|
||||
pub struct Embedded {
|
||||
pub embedding: Embedding,
|
||||
/// L2 norm of the raw model output.
|
||||
///
|
||||
/// The model's own opinion of how recognisable the crop was — see
|
||||
/// [`crate::embedding::MIN_GALLERY_QUALITY`] for what it means and where
|
||||
/// it is used.
|
||||
pub quality: f32,
|
||||
}
|
||||
|
||||
impl Embedded {
|
||||
/// Storage form: the **raw** vector, `512 × f16`.
|
||||
///
|
||||
/// Not the unit vector. The length is the quality, and a store that held
|
||||
/// only the direction would have thrown it away at the one moment it could
|
||||
/// be known — which is what this crate used to do. Readers re-normalise
|
||||
/// ([`Embedding::from_f16_bytes`]), so every comparison is still a dot
|
||||
/// product, and [`crate::embedding::read_f16_bytes`] gives the length back
|
||||
/// to a reader that wants it.
|
||||
///
|
||||
/// f16 costs nothing extra at this scale: its precision is relative, so a
|
||||
/// component of a vector of length 20 is kept to the same three figures as
|
||||
/// the same component scaled to length 1.
|
||||
pub fn to_f16_bytes(&self) -> Vec<u8> {
|
||||
self.embedding.to_f16_bytes_scaled(self.quality)
|
||||
}
|
||||
}
|
||||
|
||||
/// A loaded ArcFace graph.
|
||||
pub struct Embedder {
|
||||
session: ort::session::Session,
|
||||
@@ -63,7 +98,7 @@ impl Embedder {
|
||||
}
|
||||
|
||||
/// Embed one aligned face.
|
||||
pub fn embed(&mut self, face: &Aligned112) -> Result<Embedding, FaceError> {
|
||||
pub fn embed(&mut self, face: &Aligned112) -> Result<Embedded, FaceError> {
|
||||
// `(x·255 − 127.5) / 128` — see the `/128` note in `detect::Letterbox`.
|
||||
let px = face.pixels();
|
||||
let mut input = Array4::<f32>::zeros((1, 3, ALIGNED_EDGE, ALIGNED_EDGE));
|
||||
@@ -95,11 +130,14 @@ impl Embedder {
|
||||
|
||||
let mut v = Box::new([0.0_f32; EMBEDDING_DIM]);
|
||||
v.copy_from_slice(&data[..EMBEDDING_DIM]);
|
||||
normalise(&mut v);
|
||||
let quality = normalise(&mut v);
|
||||
|
||||
Ok(Embedding {
|
||||
model: self.model.clone(),
|
||||
v,
|
||||
Ok(Embedded {
|
||||
embedding: Embedding {
|
||||
model: self.model.clone(),
|
||||
v,
|
||||
},
|
||||
quality,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
+119
-18
@@ -12,6 +12,43 @@
|
||||
/// Embedding dimensionality. Fixed by the model family, not a parameter.
|
||||
pub const EMBEDDING_DIM: usize = 512;
|
||||
|
||||
/// The shortest raw embedding a face may be *compared against*.
|
||||
///
|
||||
/// # What the length of the vector says
|
||||
///
|
||||
/// ArcFace is trained on the direction of its output and nothing else, and
|
||||
/// the length it leaves behind turns out to be a free quality signal: the
|
||||
/// magnitude grows with how recognisable the crop was to the model, and a
|
||||
/// blurred, occluded, badly lit or hard-profile face comes out short. MagFace
|
||||
/// (Meng et al., CVPR 2021) made that the training objective; the plain
|
||||
/// ArcFace heads this crate runs already show it, weaker but usable, which is
|
||||
/// why it is worth keeping the number the normalisation discards.
|
||||
///
|
||||
/// # Why it gates the gallery and not the face
|
||||
///
|
||||
/// A short vector is a bad *reference*: it sits nearer the centre of the
|
||||
/// sphere than a real identity does and matches a little of everyone, which
|
||||
/// is exactly the face that welds two people together in a clustering pass.
|
||||
/// It is not a bad *probe* — the face is still real, still somebody, and
|
||||
/// comparing it against good references is the only way it will ever be named.
|
||||
/// So a face below this floor is compared against the gallery and never
|
||||
/// becomes part of it: see `cluster::Candidate::in_gallery`.
|
||||
///
|
||||
/// 14 is the operating point for `w600k_mbf`, whose norms on the reference
|
||||
/// library run from about 8 on a blur to the high 20s on a clean portrait. A
|
||||
/// face whose quality was never recorded — indexed before the number was kept
|
||||
/// — is not gated, because a rule that cannot be checked should admit, not
|
||||
/// exclude.
|
||||
pub const MIN_GALLERY_QUALITY: f32 = 14.0;
|
||||
|
||||
/// Whether an embedding of this quality may serve as a reference.
|
||||
///
|
||||
/// `None` is "not measured", and is admitted: the rule is about a number that
|
||||
/// was read and found short, not about a number that is missing.
|
||||
pub fn in_gallery(quality: Option<f32>) -> bool {
|
||||
quality.is_none_or(|q| q >= MIN_GALLERY_QUALITY)
|
||||
}
|
||||
|
||||
/// Which model produced an embedding.
|
||||
///
|
||||
/// Embeddings from different models are not comparable, and this is the one
|
||||
@@ -58,10 +95,19 @@ impl Embedding {
|
||||
}
|
||||
|
||||
/// Storage form: `512 × f16`, 1 KB per face (catalog.md §10.1).
|
||||
///
|
||||
/// This writes the unit vector. What the catalog stores is the raw one —
|
||||
/// `embed::Embedded::to_f16_bytes` — because the length is the quality
|
||||
/// and a unit vector has none left to read.
|
||||
pub fn to_f16_bytes(&self) -> Vec<u8> {
|
||||
self.to_f16_bytes_scaled(1.0)
|
||||
}
|
||||
|
||||
/// The unit vector scaled by `length`, as `512 × f16`.
|
||||
pub(crate) fn to_f16_bytes_scaled(&self, length: f32) -> Vec<u8> {
|
||||
let mut out = Vec::with_capacity(EMBEDDING_DIM * 2);
|
||||
for &x in self.v.iter() {
|
||||
out.extend_from_slice(&f32_to_f16_bits(x).to_le_bytes());
|
||||
out.extend_from_slice(&f32_to_f16_bits(x * length).to_le_bytes());
|
||||
}
|
||||
out
|
||||
}
|
||||
@@ -71,25 +117,42 @@ impl Embedding {
|
||||
/// The f16 round-trip perturbs a unit vector by ~1e-3 in cosine — three
|
||||
/// orders below the separation between a match and a non-match — but the
|
||||
/// drift is free to remove and invisible if left, so it is removed here
|
||||
/// rather than remembered at every call site.
|
||||
/// rather than remembered at every call site. The same pass is what turns
|
||||
/// a stored raw vector back into the unit one every comparison expects.
|
||||
pub fn from_f16_bytes(model: ModelId, bytes: &[u8]) -> Option<Self> {
|
||||
if bytes.len() != EMBEDDING_DIM * 2 {
|
||||
return None;
|
||||
}
|
||||
let mut v = Box::new([0.0_f32; EMBEDDING_DIM]);
|
||||
for (i, chunk) in bytes.chunks_exact(2).enumerate() {
|
||||
v[i] = f16_bits_to_f32(u16::from_le_bytes([chunk[0], chunk[1]]));
|
||||
}
|
||||
normalise(&mut v);
|
||||
Some(Self { model, v })
|
||||
read_f16_bytes(model, bytes).map(|(e, _)| e)
|
||||
}
|
||||
}
|
||||
|
||||
/// Read a stored vector back, with the length it was stored at.
|
||||
///
|
||||
/// The length is the quality where the blob is a raw one, and ~1 where it is
|
||||
/// a unit vector from before raw vectors were stored — which is why the
|
||||
/// catalog keeps the quality beside the blob rather than deriving it from
|
||||
/// this: a unit vector reads as a quality of 1, not as "unmeasured".
|
||||
pub fn read_f16_bytes(model: ModelId, bytes: &[u8]) -> Option<(Embedding, f32)> {
|
||||
if bytes.len() != EMBEDDING_DIM * 2 {
|
||||
return None;
|
||||
}
|
||||
let mut v = Box::new([0.0_f32; EMBEDDING_DIM]);
|
||||
for (i, chunk) in bytes.chunks_exact(2).enumerate() {
|
||||
v[i] = f16_bits_to_f32(u16::from_le_bytes([chunk[0], chunk[1]]));
|
||||
}
|
||||
let length = normalise(&mut v);
|
||||
Some((Embedding { model, v }, length))
|
||||
}
|
||||
|
||||
fn dot(a: &[f32; EMBEDDING_DIM], b: &[f32; EMBEDDING_DIM]) -> f32 {
|
||||
a.iter().zip(b.iter()).map(|(x, y)| x * y).sum()
|
||||
}
|
||||
|
||||
pub(crate) fn normalise(v: &mut [f32; EMBEDDING_DIM]) {
|
||||
/// Scale `v` to unit length, and return the length it had.
|
||||
///
|
||||
/// The length is the one thing about the raw output that survives being
|
||||
/// thrown away by everything downstream, and it is a quality signal
|
||||
/// ([`MIN_GALLERY_QUALITY`]) — so it comes back out rather than being lost
|
||||
/// here.
|
||||
pub(crate) fn normalise(v: &mut [f32; EMBEDDING_DIM]) -> f32 {
|
||||
// Clamped rather than checked: a zero-norm embedding is a broken model,
|
||||
// not a runtime condition worth an error path, and dividing by 1e-6 keeps
|
||||
// the NaN out of the catalog.
|
||||
@@ -97,6 +160,7 @@ pub(crate) fn normalise(v: &mut [f32; EMBEDDING_DIM]) {
|
||||
for x in v.iter_mut() {
|
||||
*x /= norm;
|
||||
}
|
||||
norm
|
||||
}
|
||||
|
||||
// ── f16 ───────────────────────────────────────────────────────────────────
|
||||
@@ -113,9 +177,10 @@ fn f32_to_f16_bits(x: f32) -> u16 {
|
||||
let mant = bits & 0x007f_ffff;
|
||||
|
||||
if exp >= 0x1f {
|
||||
// Overflow, inf, or NaN. Embeddings are unit-norm so this is the
|
||||
// broken-model path; infinity is the honest answer, not a clamp that
|
||||
// hides it.
|
||||
// Overflow, inf, or NaN. No component of an embedding exceeds its
|
||||
// length, and the lengths this model produces are in the tens, so
|
||||
// this is the broken-model path; infinity is the honest answer, not a
|
||||
// clamp that hides it.
|
||||
return sign
|
||||
| 0x7c00
|
||||
| if mant != 0 && exp == 0x1f + 112 {
|
||||
@@ -125,9 +190,9 @@ fn f32_to_f16_bits(x: f32) -> u16 {
|
||||
};
|
||||
}
|
||||
if exp <= 0 {
|
||||
// Subnormal or underflow. A component of a unit 512-vector is ~0.04,
|
||||
// nowhere near here, so this branch exists for correctness rather than
|
||||
// for traffic.
|
||||
// Subnormal or underflow. A component of a unit 512-vector is ~0.04
|
||||
// and a stored one is that times the length, nowhere near here, so
|
||||
// this branch exists for correctness rather than for traffic.
|
||||
if exp < -10 {
|
||||
return sign;
|
||||
}
|
||||
@@ -221,6 +286,42 @@ mod tests {
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn normalising_reports_the_length_it_removed() {
|
||||
let mut v = Box::new([0.0_f32; EMBEDDING_DIM]);
|
||||
v[0] = 3.0;
|
||||
v[1] = 4.0;
|
||||
let norm = normalise(&mut v);
|
||||
assert!((norm - 5.0).abs() < 1e-6, "norm {norm}");
|
||||
assert!((v[0] - 0.6).abs() < 1e-6 && (v[1] - 0.8).abs() < 1e-6);
|
||||
}
|
||||
|
||||
/// The gate admits what it cannot measure: a face from before the number
|
||||
/// was kept is not a face that was found wanting.
|
||||
#[test]
|
||||
fn an_unmeasured_quality_is_admitted_to_the_gallery() {
|
||||
assert!(in_gallery(None));
|
||||
assert!(in_gallery(Some(MIN_GALLERY_QUALITY)));
|
||||
assert!(in_gallery(Some(27.5)));
|
||||
assert!(!in_gallery(Some(MIN_GALLERY_QUALITY - 0.01)));
|
||||
assert!(!in_gallery(Some(8.0)));
|
||||
}
|
||||
|
||||
/// The storage form carries the length, and the length comes back out —
|
||||
/// without touching the direction every comparison is made on.
|
||||
#[test]
|
||||
fn a_raw_vector_round_trips_with_its_length() {
|
||||
let e = unit(3);
|
||||
let raw = e.to_f16_bytes_scaled(21.5);
|
||||
let (back, length) = read_f16_bytes(e.model.clone(), &raw).unwrap();
|
||||
assert!((length - 21.5).abs() < 0.05, "length {length}");
|
||||
assert!(e.cosine(&back).unwrap() > 0.9999);
|
||||
// A unit vector from an older store reads as length 1, not as an
|
||||
// error — see `read_f16_bytes` on why that is not "unmeasured".
|
||||
let (_, one) = read_f16_bytes(e.model.clone(), &e.to_f16_bytes()).unwrap();
|
||||
assert!((one - 1.0).abs() < 1e-2, "length {one}");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn f16_round_trip_rejects_a_wrong_length_blob() {
|
||||
assert!(Embedding::from_f16_bytes(ModelId::new("m"), &[0u8; 100]).is_none());
|
||||
|
||||
@@ -75,8 +75,10 @@ pub use cluster::{
|
||||
#[cfg(feature = "inference")]
|
||||
pub use detect::{DetectOptions, Detection, Detector};
|
||||
#[cfg(feature = "inference")]
|
||||
pub use embed::Embedder;
|
||||
pub use embedding::{Embedding, ModelId, EMBEDDING_DIM};
|
||||
pub use embed::{Embedded, Embedder};
|
||||
pub use embedding::{
|
||||
in_gallery, read_f16_bytes, Embedding, ModelId, EMBEDDING_DIM, MIN_GALLERY_QUALITY,
|
||||
};
|
||||
pub use naming::{name_for_instance, name_instances, NamedFace};
|
||||
|
||||
/// What can go wrong between an image and a face.
|
||||
|
||||
@@ -117,6 +117,17 @@ pub struct Faces<'a> {
|
||||
/// Which photograph each face came from. Two faces in one frame are not
|
||||
/// the same person, so those pairs are never returned (docs/faces.md §9).
|
||||
pub images: &'a [u64],
|
||||
/// Which faces may be compared *against* — the gallery
|
||||
/// ([`crate::embedding::MIN_GALLERY_QUALITY`]).
|
||||
///
|
||||
/// A pair needs at least one gallery side: a probe measured against a
|
||||
/// reference is a comparison, two short vectors measured against each
|
||||
/// other is noise agreeing with noise, and those pairs are never returned.
|
||||
/// Filtered here rather than by the caller for the same reason
|
||||
/// co-occurrence is: what this module leaves out of the list stays out of
|
||||
/// the graph, the components and the merge order, so nothing downstream
|
||||
/// has to remember the rule.
|
||||
pub gallery: &'a [bool],
|
||||
}
|
||||
|
||||
impl Faces<'_> {
|
||||
@@ -272,8 +283,9 @@ fn scan_rows(scan: &Scan, from: usize, to: usize, out: &mut Vec<Pair>) {
|
||||
let a = faces.row(i);
|
||||
let crop_a = faces.crop_px[i];
|
||||
let image_a = faces.images[i];
|
||||
let gallery_a = faces.gallery[i];
|
||||
for j in start..tile_end {
|
||||
if image_a == faces.images[j] {
|
||||
if image_a == faces.images[j] || !(gallery_a || faces.gallery[j]) {
|
||||
continue;
|
||||
}
|
||||
let cos = dot(a, faces.row(j));
|
||||
@@ -552,6 +564,7 @@ mod tests {
|
||||
embeddings: Vec<f32>,
|
||||
crop_px: Vec<f32>,
|
||||
images: Vec<u64>,
|
||||
gallery: Vec<bool>,
|
||||
}
|
||||
|
||||
impl Set {
|
||||
@@ -561,6 +574,7 @@ mod tests {
|
||||
dim: DIM,
|
||||
crop_px: &self.crop_px,
|
||||
images: &self.images,
|
||||
gallery: &self.gallery,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -588,10 +602,12 @@ mod tests {
|
||||
}
|
||||
}
|
||||
let crop_px = vec![150.0; embeddings.len()];
|
||||
let gallery = vec![true; embeddings.len()];
|
||||
Set {
|
||||
embeddings: embeddings.concat(),
|
||||
crop_px,
|
||||
images,
|
||||
gallery,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -601,7 +617,7 @@ mod tests {
|
||||
let mut out = Vec::new();
|
||||
for i in 0..n {
|
||||
for j in i + 1..n {
|
||||
if faces.images[i] == faces.images[j] {
|
||||
if faces.images[i] == faces.images[j] || !(faces.gallery[i] || faces.gallery[j]) {
|
||||
continue;
|
||||
}
|
||||
let cos: f32 = faces
|
||||
@@ -750,6 +766,35 @@ mod tests {
|
||||
assert!(above_threshold(&s.faces(), &cal(), 0.9).is_empty());
|
||||
}
|
||||
|
||||
/// A probe against a reference is a comparison; two probes against each
|
||||
/// other is not. The rule lives here so that nothing downstream sees the
|
||||
/// pair at all.
|
||||
#[test]
|
||||
fn two_faces_outside_the_gallery_are_never_paired() {
|
||||
let mut s = population(1, 3, 1.0);
|
||||
s.gallery = vec![false, false, true];
|
||||
let pairs = above_threshold(&s.faces(), &cal(), 0.9);
|
||||
assert!(
|
||||
!pairs.iter().any(|p| p.i == 0 && p.j == 1),
|
||||
"two probes were paired with each other"
|
||||
);
|
||||
// Each probe is still measured against the one reference.
|
||||
assert!(pairs.iter().any(|p| p.i == 0 && p.j == 2));
|
||||
assert!(pairs.iter().any(|p| p.i == 1 && p.j == 2));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn the_gallery_rule_matches_the_reference_at_scale() {
|
||||
let mut s = population(60, 8, 0.97);
|
||||
for (i, g) in s.gallery.iter_mut().enumerate() {
|
||||
*g = i % 3 != 0;
|
||||
}
|
||||
let f = s.faces();
|
||||
let got = above_threshold(&f, &cal(), 0.9);
|
||||
let want = reference(&f, &cal(), 0.9);
|
||||
assert!(same_pairs(&got, &want), "{} vs {}", got.len(), want.len());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn pairs_come_back_in_index_order() {
|
||||
let s = population(300, 8, 0.97);
|
||||
|
||||
+6
-1
@@ -742,7 +742,12 @@ CREATE TABLE faces (
|
||||
x REAL NOT NULL, y REAL NOT NULL, w REAL NOT NULL, h REAL NOT NULL,
|
||||
landmarks BLOB, -- 5 × (x, y) f32, the alignment input
|
||||
detector_confidence REAL NOT NULL,
|
||||
embedding BLOB NOT NULL, -- 512 × f16, L2-normalised
|
||||
embedding BLOB NOT NULL, -- 512 × f16, the raw model output; re-normalised on load
|
||||
-- Length of that vector: the model's own reading of how recognisable the
|
||||
-- crop was, and the gate on whether this face may be compared *against*
|
||||
-- (faces.md §6, §9). NULL for a face stored as a unit vector before it
|
||||
-- was kept.
|
||||
quality REAL,
|
||||
-- Which model produced this. An embedding is only comparable to others
|
||||
-- from the same model; mixing them silently yields nonsense similarities.
|
||||
model_id TEXT NOT NULL,
|
||||
|
||||
+37
-4
@@ -447,9 +447,28 @@ every measured number in §1's table was produced with `/128`. The difference is
|
||||
almost certainly immaterial, but "almost certainly" is not a reason to pick silently — write `/128` to
|
||||
match the numbers we have, and settle it with one back-to-back run in §12.
|
||||
|
||||
Output is 512 floats; **L2-normalise before storing**, so every downstream comparison is a dot product
|
||||
and no code path has to remember to normalise. The reference clamps the norm at 1e-6 before dividing,
|
||||
which costs nothing and removes a NaN path.
|
||||
Output is 512 floats. Every downstream comparison is a dot product over the **unit** vector, and the
|
||||
reference L2-normalises before storing so that no code path has to remember to. The reference clamps
|
||||
the norm at 1e-6 before dividing, which costs nothing and removes a NaN path.
|
||||
|
||||
**Keep the length.** The norm the normalisation divides out is not noise. ArcFace trains the
|
||||
direction of its output and nothing else, and the magnitude it leaves behind grows with how much of a
|
||||
face the model could make out — MagFace (Meng et al., CVPR 2021) made that the training objective,
|
||||
and the plain ArcFace heads this crate runs already show it, weaker but usable. A blur, an occlusion,
|
||||
a hard profile or a badly lit crop comes out short. On the reference library `w600k_mbf`'s norms run
|
||||
from about 8 on a blur to the high 20s on a clean portrait.
|
||||
|
||||
So the store holds the **raw** vector, not the unit one — `dr_face::Embedded::to_f16_bytes` — and
|
||||
readers re-normalise on load, which they had to do anyway (below). f16 keeps the same three figures
|
||||
of a component whatever the vector's length, so this costs nothing in precision. The length is also
|
||||
kept beside the blob as `faces.quality`, for the readers that never load the vector (the People
|
||||
screen), and it is `NULL` for a face stored as a unit vector before this — a unit vector reads as a
|
||||
length of one, and one is not "unmeasured".
|
||||
|
||||
What the number does is in §9: a face whose quality is under **`MIN_GALLERY_QUALITY` = 14** is still
|
||||
placed, but is never what another face is compared *against*. The screen shows it as "Quality 17.3",
|
||||
dimmed below the floor, so a user asking why a group did not gather the rest of a person can see that
|
||||
none of its members can vouch for anyone.
|
||||
|
||||
Some exports of these graphs are fp16 in, fp16 out. The reference detects this from the graph's
|
||||
declared element type rather than from the filename; worth porting, because the alternative failure is
|
||||
@@ -461,7 +480,8 @@ between a match and a non-match, and the halving matters because these rows are
|
||||
contemplates optionally syncing.
|
||||
|
||||
Re-normalise on load after the f16 widen. It is one pass over 512 floats and it removes a class of
|
||||
drift that is otherwise invisible.
|
||||
drift that is otherwise invisible — and, since the blob is raw, it is what turns the stored vector
|
||||
back into the unit one every comparison expects.
|
||||
|
||||
---
|
||||
|
||||
@@ -785,6 +805,19 @@ here, and it is the phone and tablet story that should decide whether it gets bu
|
||||
- **Confirmed faces are anchors.** A confirmation is user data (FR-CULL-12) and clustering never
|
||||
moves it. Two clusters each containing confirmations of *different* people cannot merge; a cluster
|
||||
containing confirmations of one person absorbs suggestions but never reassigns the confirmed.
|
||||
- **A short embedding is never a reference.** The length of the raw vector is the model's own
|
||||
reading of the crop (§6), and a short one sits near the middle of the sphere, matching a little of
|
||||
everybody — one of those in a group is a bridge to the next group over. So the population is
|
||||
split: faces at or above `MIN_GALLERY_QUALITY` are the **gallery** and cluster as described below;
|
||||
faces under it are **probes**, each measured against the finished groups and placed in the one it
|
||||
fits by the same average-link rule under the same two constraints — but measured against gallery
|
||||
members only, never against another probe, and once placed never part of what the next face is
|
||||
measured against. Two probes are never paired at all, and `neighbours` drops those pairs before
|
||||
anything downstream sees them. A probe's confidence (§9.1) is computed from the references it
|
||||
matched; a reference's confidence hears nothing from a probe. A face whose quality was never
|
||||
recorded is admitted to the gallery — a rule that cannot be checked admits rather than excludes —
|
||||
and schema V14 forgets the run marker of every image holding one, so the next indexing pass
|
||||
measures it.
|
||||
|
||||
**The algorithm.** Constrained average-link agglomeration over the probability graph, merging while
|
||||
the average pairwise probability exceeds **0.9** and no cannot-link is violated. Average-link rather
|
||||
|
||||
+4
-4
@@ -197,7 +197,7 @@ Double-click is what a file manager and a Lightroom panel use for the same thing
|
||||
|
||||
Grouping over-merges on siblings, on parents and children, and on the same person a decade apart, so splitting is as prominent as merging. A tool that can only merge makes its own errors permanent.
|
||||
|
||||
<sub>`ui/dr-ui/ui/identity.slint:130`</sub>
|
||||
<sub>`ui/dr-ui/ui/identity.slint:161`</sub>
|
||||
|
||||
### Rule on a suggested face
|
||||
|
||||
@@ -206,7 +206,7 @@ Grouping over-merges on siblings, on parents and children, and on the same perso
|
||||
|
||||
A face is either the system's guess or the user's judgement, and the two are never conflated. A rejection is remembered, so the face is not suggested for that person again. The gesture note above is the whole label: a tick and a cross are only "confirm" and "reject" to someone who can see the suggestion they sit beside, and `IconButton`'s fallback would announce them as "check" and "cross" — two icon names that say nothing about which person is being ruled on.
|
||||
|
||||
<sub>`ui/dr-ui/ui/identity.slint:150`</sub>
|
||||
<sub>`ui/dr-ui/ui/identity.slint:181`</sub>
|
||||
|
||||
### See a person's photographs
|
||||
|
||||
@@ -215,7 +215,7 @@ A face is either the system's guess or the user's judgement, and the two are nev
|
||||
|
||||
This is the point of having identified anybody. Without it the screen is a filing cabinet with no drawer handles.
|
||||
|
||||
<sub>`ui/dr-ui/ui/identity.slint:571`</sub>
|
||||
<sub>`ui/dr-ui/ui/identity.slint:602`</sub>
|
||||
|
||||
### Change how faces are grouped
|
||||
|
||||
@@ -224,7 +224,7 @@ This is the point of having identified anybody. Without it the screen is a filin
|
||||
|
||||
The right match confidence is a property of your library, not of the model. "What would this do?" answers for this library without writing anything; names, confirmations and the groups you have set aside are kept whatever the dials say.
|
||||
|
||||
<sub>`ui/dr-ui/ui/identity.slint:608`</sub>
|
||||
<sub>`ui/dr-ui/ui/identity.slint:639`</sub>
|
||||
|
||||
## Library grid
|
||||
|
||||
|
||||
+21
-21
File diff suppressed because one or more lines are too long
@@ -286,15 +286,16 @@ fn tune_thresholds(catalog: &Catalog) {
|
||||
|
||||
let model = dr_face::ModelId::new(MODEL_ID.to_string());
|
||||
let mut candidates = Vec::with_capacity(stored.len());
|
||||
for (face_id, image_id, blob, crop_px) in stored {
|
||||
let Some(emb) = dr_face::Embedding::from_f16_bytes(model.clone(), &blob) else {
|
||||
for f in stored {
|
||||
let Some(emb) = dr_face::Embedding::from_f16_bytes(model.clone(), &f.embedding) else {
|
||||
continue;
|
||||
};
|
||||
candidates.push(dr_face::Candidate {
|
||||
face: face_id.0,
|
||||
image: image_id.0,
|
||||
face: f.face.0,
|
||||
image: f.image.0,
|
||||
embedding: emb.v.to_vec(),
|
||||
crop_px,
|
||||
crop_px: f.crop_px,
|
||||
quality: f.quality,
|
||||
confirmed_person: None,
|
||||
});
|
||||
}
|
||||
|
||||
+17
-12
@@ -306,7 +306,7 @@ pub fn index_proxy(
|
||||
continue;
|
||||
}
|
||||
|
||||
let embedding = embedder.embed(&aligned)?;
|
||||
let embedded = embedder.embed(&aligned)?;
|
||||
|
||||
out.push(DetectedFace {
|
||||
x: d.bbox.0 / long_edge,
|
||||
@@ -315,8 +315,9 @@ pub fn index_proxy(
|
||||
h: d.height() / long_edge,
|
||||
landmarks: normalise_landmarks(&d.landmarks, long_edge),
|
||||
confidence: d.confidence,
|
||||
embedding: embedding.to_f16_bytes(),
|
||||
embedding: embedded.to_f16_bytes(),
|
||||
crop_px: aligned.source_px(),
|
||||
quality: Some(embedded.quality),
|
||||
model_id: embedder.model().as_str().to_string(),
|
||||
// Cut here, while the buffer is still in hand. This is the only
|
||||
// moment in the whole pipeline where the pixels are free.
|
||||
@@ -419,7 +420,7 @@ pub fn index_native(
|
||||
continue;
|
||||
}
|
||||
|
||||
let embedding = embedder.embed(&aligned)?;
|
||||
let embedded = embedder.embed(&aligned)?;
|
||||
let (bx, by) = (d.bbox.0 * sx, d.bbox.1 * sy);
|
||||
let (bw, bh) = (d.width() * sx, d.height() * sy);
|
||||
|
||||
@@ -430,8 +431,9 @@ pub fn index_native(
|
||||
h: bh / long_edge,
|
||||
landmarks: normalise_landmarks(&landmarks, long_edge),
|
||||
confidence: d.confidence,
|
||||
embedding: embedding.to_f16_bytes(),
|
||||
embedding: embedded.to_f16_bytes(),
|
||||
crop_px: aligned.source_px(),
|
||||
quality: Some(embedded.quality),
|
||||
model_id: embedder.model().as_str().to_string(),
|
||||
crop: cut_crop_native(native, width, height, (bx, by, bw, bh)).unwrap_or_default(),
|
||||
});
|
||||
@@ -837,19 +839,20 @@ impl Population {
|
||||
let model = ModelId::new(model_id.to_string());
|
||||
let mut candidates = Vec::with_capacity(stored.len());
|
||||
let mut ids = Vec::with_capacity(stored.len());
|
||||
for (face_id, image_id, blob, crop_px) in stored {
|
||||
let Some(emb) = dr_face::Embedding::from_f16_bytes(model.clone(), &blob) else {
|
||||
log::warn!("face {face_id:?} has a malformed embedding, skipped");
|
||||
for f in stored {
|
||||
let Some(emb) = dr_face::Embedding::from_f16_bytes(model.clone(), &f.embedding) else {
|
||||
log::warn!("face {:?} has a malformed embedding, skipped", f.face);
|
||||
continue;
|
||||
};
|
||||
candidates.push(dr_face::Candidate {
|
||||
face: face_id.0,
|
||||
image: image_id.0,
|
||||
face: f.face.0,
|
||||
image: f.image.0,
|
||||
embedding: emb.v.to_vec(),
|
||||
crop_px,
|
||||
confirmed_person: anchors.get(&face_id).map(|p| p.0),
|
||||
crop_px: f.crop_px,
|
||||
quality: f.quality,
|
||||
confirmed_person: anchors.get(&f.face).map(|p| p.0),
|
||||
});
|
||||
ids.push(face_id);
|
||||
ids.push(f.face);
|
||||
}
|
||||
|
||||
Ok(Self {
|
||||
@@ -1516,6 +1519,7 @@ mod tests {
|
||||
landmarks: [(0.0, 0.0); 5],
|
||||
confidence: 0.9,
|
||||
crop_px: 120.0,
|
||||
quality: None,
|
||||
model_id: "w600k_mbf".into(),
|
||||
person: None,
|
||||
probability: 0.0,
|
||||
@@ -1633,6 +1637,7 @@ mod tests {
|
||||
confidence: 0.9,
|
||||
embedding: embedding(identity, cosine),
|
||||
crop_px: 150.0,
|
||||
quality: None,
|
||||
model_id: TEST_MODEL.to_string(),
|
||||
crop: Vec::new(),
|
||||
};
|
||||
|
||||
@@ -97,6 +97,10 @@ pub struct FaceCell {
|
||||
/// Source pixels across the aligned crop. Small faces embed worse, and the
|
||||
/// user deserves to know which of a bad suggestion's causes is in play.
|
||||
pub crop_px: f32,
|
||||
/// The model's own reading of how recognisable the crop was — the length
|
||||
/// of its raw embedding (`dr_face::MIN_GALLERY_QUALITY`). `None` for a
|
||||
/// face indexed before it was kept.
|
||||
pub quality: Option<f32>,
|
||||
}
|
||||
|
||||
impl FaceCell {
|
||||
@@ -115,6 +119,31 @@ impl FaceCell {
|
||||
format!("{:.0}% likely", self.probability * 100.0)
|
||||
}
|
||||
}
|
||||
|
||||
/// The quality as text — "Quality 17.3", or "Quality —" where it was never
|
||||
/// measured.
|
||||
///
|
||||
/// Called *quality* and not *norm* on the screen, because that is what the
|
||||
/// number is used as and what the user can act on: a low one is the model
|
||||
/// saying it could not make the face out, and the fix is a better
|
||||
/// photograph. One decimal, because the gate sits at a whole number and
|
||||
/// the faces worth a second look are the ones just either side of it.
|
||||
pub fn quality_label(&self) -> String {
|
||||
match self.quality {
|
||||
Some(q) => format!("Quality {q:.1}"),
|
||||
None => "Quality —".into(),
|
||||
}
|
||||
}
|
||||
|
||||
/// Whether this face is good enough to be compared *against*.
|
||||
///
|
||||
/// What the screen dims the quality for: a face below the floor is still
|
||||
/// somebody and still placed, but it vouches for no one else, and a user
|
||||
/// wondering why a person's group did not gather the rest of them should
|
||||
/// be able to see that none of its members can.
|
||||
pub fn in_gallery(&self) -> bool {
|
||||
dr_face::in_gallery(self.quality)
|
||||
}
|
||||
}
|
||||
|
||||
/// Put a grouping preview into words.
|
||||
@@ -254,6 +283,7 @@ pub fn load_faces(
|
||||
confirmed: f.confirmed,
|
||||
probability: f.probability,
|
||||
crop_px: f.crop_px,
|
||||
quality: f.quality,
|
||||
});
|
||||
}
|
||||
|
||||
@@ -615,23 +645,24 @@ pub fn preview_split(
|
||||
let model = dr_face::ModelId::new(model_id.to_string());
|
||||
let stored: std::collections::HashMap<_, _> = faces::embeddings(conn, model_id)?
|
||||
.into_iter()
|
||||
.map(|(id, image, blob, crop_px)| (id, (image, blob, crop_px)))
|
||||
.map(|e| (e.face, e))
|
||||
.collect();
|
||||
|
||||
let mut candidates = Vec::with_capacity(cells.len());
|
||||
let mut order = Vec::with_capacity(cells.len());
|
||||
for c in &cells {
|
||||
let Some((image, blob, crop_px)) = stored.get(&c.face) else {
|
||||
let Some(e) = stored.get(&c.face) else {
|
||||
continue;
|
||||
};
|
||||
let Some(emb) = dr_face::Embedding::from_f16_bytes(model.clone(), blob) else {
|
||||
let Some(emb) = dr_face::Embedding::from_f16_bytes(model.clone(), &e.embedding) else {
|
||||
continue;
|
||||
};
|
||||
candidates.push(dr_face::Candidate {
|
||||
face: c.face.0,
|
||||
image: image.0,
|
||||
image: e.image.0,
|
||||
embedding: emb.v.to_vec(),
|
||||
crop_px: *crop_px,
|
||||
crop_px: e.crop_px,
|
||||
quality: e.quality,
|
||||
confirmed_person: None,
|
||||
});
|
||||
order.push(c.clone());
|
||||
@@ -755,6 +786,7 @@ mod tests {
|
||||
confidence: 0.9,
|
||||
embedding: vec![seed; 1024],
|
||||
crop_px: 180.0,
|
||||
quality: Some(f32::from(seed) + 10.0),
|
||||
model_id: "w600k_mbf".into(),
|
||||
crop: Vec::new(),
|
||||
}
|
||||
@@ -802,6 +834,7 @@ mod tests {
|
||||
confirmed: false,
|
||||
probability: 0.87,
|
||||
crop_px: 120.0,
|
||||
quality: Some(17.26),
|
||||
};
|
||||
assert_eq!(cell.confidence_label(), "87% likely");
|
||||
|
||||
@@ -812,6 +845,41 @@ mod tests {
|
||||
assert_eq!(confirmed.confidence_label(), "Confirmed");
|
||||
}
|
||||
|
||||
/// The number is the embedding's length, and the screen calls it what it
|
||||
/// is used as. A face from before it was kept says so rather than showing
|
||||
/// a zero that would read as the worst face in the library.
|
||||
#[test]
|
||||
fn the_quality_is_labelled_as_such_and_dimmed_below_the_floor() {
|
||||
let cell = FaceCell {
|
||||
face: FaceId(1),
|
||||
image: ImageId(1),
|
||||
crop: None,
|
||||
confirmed: false,
|
||||
probability: 0.5,
|
||||
crop_px: 120.0,
|
||||
quality: Some(17.26),
|
||||
};
|
||||
assert_eq!(cell.quality_label(), "Quality 17.3");
|
||||
assert!(cell.in_gallery());
|
||||
|
||||
let poor = FaceCell {
|
||||
quality: Some(9.4),
|
||||
..cell.clone()
|
||||
};
|
||||
assert_eq!(poor.quality_label(), "Quality 9.4");
|
||||
assert!(!poor.in_gallery());
|
||||
|
||||
let unmeasured = FaceCell {
|
||||
quality: None,
|
||||
..cell
|
||||
};
|
||||
assert_eq!(unmeasured.quality_label(), "Quality —");
|
||||
assert!(
|
||||
unmeasured.in_gallery(),
|
||||
"an unmeasured face is not a poor one"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn the_people_rail_reports_the_unassigned_pool() {
|
||||
let c = catalog();
|
||||
|
||||
@@ -478,6 +478,8 @@ fn push_faces(window: &AppWindow, ctl: &IdentityController, cells: &[FaceCell])
|
||||
confirmed: c.confirmed,
|
||||
confidence: c.confidence_label().into(),
|
||||
crop_px: c.crop_px as i32,
|
||||
quality: c.quality_label().into(),
|
||||
in_gallery: c.in_gallery(),
|
||||
picked: picked.contains(&c.face),
|
||||
})
|
||||
.collect();
|
||||
|
||||
@@ -5937,6 +5937,7 @@ mod tests {
|
||||
confidence: 0.9,
|
||||
embedding: vec![0u8; 1024],
|
||||
crop_px: 120.0,
|
||||
quality: None,
|
||||
crop: Vec::new(),
|
||||
model_id: "w600k_mbf".into(),
|
||||
};
|
||||
@@ -5986,6 +5987,7 @@ mod tests {
|
||||
confidence: 0.9,
|
||||
embedding: vec![0u8; 1024],
|
||||
crop_px: 120.0,
|
||||
quality: None,
|
||||
crop: Vec::new(),
|
||||
model_id: "w600k_mbf".into(),
|
||||
};
|
||||
@@ -7012,6 +7014,7 @@ mod tests {
|
||||
confidence: 0.9,
|
||||
embedding: vec![0u8; 1024],
|
||||
crop_px: 120.0,
|
||||
quality: None,
|
||||
crop: Vec::new(),
|
||||
model_id: "w600k_mbf".into(),
|
||||
};
|
||||
|
||||
@@ -51,6 +51,15 @@ export struct IdentityFace {
|
||||
// with has causes, and "the face was 41 pixels across" is one the user can
|
||||
// act on by finding a better photograph.
|
||||
crop-px: int,
|
||||
// "Quality 17.3" — the model's own reading of how recognisable the crop
|
||||
// was, composed in Rust like `confidence` and for the same reason. The
|
||||
// other cause a bad suggestion can have, and the one the grouping pass
|
||||
// acts on: a face below the floor is placed but never compared against.
|
||||
quality: string,
|
||||
// Whether the face clears that floor. Drawn, not just known, because a
|
||||
// group that has gathered none of a person's other photographs has an
|
||||
// explanation the user can only see if every member's label says it.
|
||||
in-gallery: bool,
|
||||
// Part of the current multi-select — what a split would carry.
|
||||
picked: bool,
|
||||
}
|
||||
@@ -110,6 +119,28 @@ component FaceCell inherits Rectangle {
|
||||
vertical-alignment: center;
|
||||
}
|
||||
|
||||
// The quality, over the foot of the crop. In the picture rather
|
||||
// than the caption because the caption is the verdict controls'
|
||||
// and at the compact width there is no room beside them; and
|
||||
// dimmed below the floor, which is the one state of it that
|
||||
// changes what the grouping does.
|
||||
Rectangle {
|
||||
x: 0;
|
||||
y: parent.height - self.height;
|
||||
width: parent.width;
|
||||
height: 16px;
|
||||
background: Theme.ground.with-alpha(0.55);
|
||||
Text {
|
||||
text: face.quality;
|
||||
color: face.in-gallery ? Theme.ink-dim : Theme.warn-ink;
|
||||
font-size: Theme.text-sm;
|
||||
horizontal-alignment: center;
|
||||
vertical-alignment: center;
|
||||
width: parent.width;
|
||||
height: parent.height;
|
||||
}
|
||||
}
|
||||
|
||||
// A confirmed face carries a quiet marker rather than a badge: the
|
||||
// grid is mostly confirmed once the user has worked through it, and
|
||||
// a loud mark on the common case is just noise.
|
||||
|
||||
Reference in New Issue
Block a user