diff --git a/core/dr-catalog/src/merge.rs b/core/dr-catalog/src/merge.rs index af216c7..efa4876 100644 --- a/core/dr-catalog/src/merge.rs +++ b/core/dr-catalog/src/merge.rs @@ -113,6 +113,9 @@ pub struct MergeReport { pub faces_kept_local: usize, /// "Not this person" judgements taken from the remote. pub faces_rejected: usize, + /// Remote faces placed on a local one by their embedding, where the + /// boxes disagreed or were ambiguous (see `match_faces`). + pub faces_matched_by_embedding: usize, /// Redundant identities for one word, retired by /// [`crate::keywords::fuse_duplicates`]. @@ -973,15 +976,17 @@ fn attached_has_table(conn: &Connection, schema: &str, table: &str) -> Result Result<(), CatalogError> { // A remote written before faces existed has none of these tables, and one // written before V10 has no `ignored`. Both are ordinary — `remote_is_ @@ -1076,7 +1081,9 @@ fn merge_people_within(tx: &Connection, report: &mut MergeReport) -> Result<(), } // ---- match the remote's faces onto this device's ---------------------- - let face_map = match_faces(tx)?; + let matched = match_faces(tx)?; + report.faces_matched_by_embedding += matched.by_embedding; + let face_map = matched.map; if face_map.is_empty() { return Ok(()); } @@ -1225,7 +1232,45 @@ fn remote_has_column(tx: &Connection, table: &str, column: &str) -> Result, + /// Every local face on a synced photograph, by the photograph's + /// cross-device id -- what "another face in the same photograph" means + /// to the merge. + on_file: std::collections::HashMap>, + /// The inverse of `on_file`. + file_of: std::collections::HashMap, + /// Pairs the boxes could not settle and the embeddings did. + by_embedding: usize, +} + +/// Remote face row id to local face row id, by photograph, box and vector. /// /// See [`merge_people_within`] for why a face has no shared identity and this /// has to be derived. Faces are compared within an *embedder* @@ -1235,25 +1280,42 @@ fn remote_has_column(tx: &Connection, table: &str, column: &str) -> Result Result, CatalogError> { +/// +/// # Two passes, and the second is rare +/// +/// **By box.** A remote face and a local one on the same photograph are the +/// same face when their boxes overlap by at least 0.5 IoU and neither has +/// another such candidate. That settles 18,348 of the reference library's +/// 19,052 remote faces, reads no vector, and is the whole of a steady-state +/// pass. +/// +/// **By embedding**, only on the photographs where a remote face is left +/// over -- no box overlapped it, or two did. Their vectors are read (a few +/// hundred photographs, not the library's 19 MB of them) and a remote face is +/// paired with the local face it resembles most when the cosine is at least +/// [`SAME_FACE_ACROSS_DEVICES`], the pair is each other's best, each leads +/// its runner-up by [`DECISIVE_MARGIN`], and the local face was not already +/// claimed by a box. That is the face whose box one device drew somewhere +/// else -- twenty on the reference library, boxes at IoU 0 with cosines of +/// 0.72 to 0.96 -- and the face between two overlapping boxes. Anything less +/// decisive stays unmatched, which is what a new face is: a name that fails +/// to cross can be given again, a name put on the wrong face is a false +/// merge the user has to find. +fn match_faces(tx: &Connection) -> Result { + use std::collections::HashMap; + /// Loose on purpose — "the same face in the frame", not "the same /// rectangle". The figure `record_detections` uses for the same job. const MIN_IOU: f32 = 0.5; type Boxed = (i64, f32, f32, f32, f32); + type Key = (i64, String); ensure_face_box_index(tx); - // Local faces, grouped by the photograph's cross-device id. - let mut local: std::collections::HashMap<(i64, String), Vec> = - std::collections::HashMap::new(); - { - let mut stmt = tx.prepare( - "SELECT f.id, r.file_id, f.model_id, f.x, f.y, f.w, f.h - FROM main.faces f - JOIN main.remote r ON r.image_id = f.image_id - WHERE r.file_id IS NOT NULL", - )?; + let read_boxes = |sql: &str| -> Result>, CatalogError> { + let mut out: HashMap> = HashMap::new(); + let mut stmt = tx.prepare(sql)?; let rows = stmt.query_map([], |r| { Ok(( r.get::<_, i64>(1)?, @@ -1270,50 +1332,187 @@ fn match_faces(tx: &Connection) -> Result, C for row in rows { let (file_id, model, boxed) = row?; let embedder = crate::faces::embedder_of(&model).to_string(); - local.entry((file_id, embedder)).or_default().push(boxed); + out.entry((file_id, embedder)).or_default().push(boxed); + } + Ok(out) + }; + + // Local faces, grouped by the photograph's cross-device id. + let local = read_boxes( + "SELECT f.id, r.file_id, f.model_id, f.x, f.y, f.w, f.h + FROM main.faces f + JOIN main.remote r ON r.image_id = f.image_id + WHERE r.file_id IS NOT NULL", + )?; + if local.is_empty() { + return Ok(FaceMatch::default()); + } + let mut out = FaceMatch::default(); + for ((file_id, _), faces) in &local { + let on = out.on_file.entry(*file_id).or_default(); + for &(id, ..) in faces { + on.push(id); + out.file_of.insert(id, *file_id); } } - if local.is_empty() { - return Ok(Default::default()); - } - let mut map = std::collections::HashMap::new(); - let mut stmt = tx.prepare( + let remote = read_boxes( "SELECT f.id, r.file_id, f.model_id, f.x, f.y, f.w, f.h FROM remote_cat.faces f JOIN remote_cat.remote r ON r.image_id = f.image_id WHERE r.file_id IS NOT NULL", )?; - let rows = stmt.query_map([], |r| { - Ok(( - r.get::<_, i64>(0)?, - r.get::<_, i64>(1)?, - r.get::<_, String>(2)?, - ( - r.get::<_, f64>(3)? as f32, - r.get::<_, f64>(4)? as f32, - r.get::<_, f64>(5)? as f32, - r.get::<_, f64>(6)? as f32, - ), - )) - })?; - for row in rows { - let (remote_id, file_id, model, rbox) = row?; - let embedder = crate::faces::embedder_of(&model).to_string(); - let Some(candidates) = local.get(&(file_id, embedder)) else { + // ---- by box ----------------------------------------------------------- + // Per group, which local face (by index) each remote face took. + let mut left_over: Vec<(&Key, Vec>)> = Vec::new(); + for (key, theirs) in &remote { + let Some(ours) = local.get(key) else { continue; }; - let best = candidates + let overlaps: Vec> = theirs .iter() - .map(|&(id, x, y, w, h)| (id, iou(rbox, (x, y, w, h)))) - .filter(|&(_, score)| score >= MIN_IOU) - .max_by(|a, b| a.1.total_cmp(&b.1)); - if let Some((local_id, _)) = best { - map.insert(remote_id, local_id); + .map(|&(_, x, y, w, h)| { + ours.iter() + .map(|&(_, lx, ly, lw, lh)| iou((x, y, w, h), (lx, ly, lw, lh)) >= MIN_IOU) + .collect() + }) + .collect(); + let mut taken: Vec> = vec![None; theirs.len()]; + for (i, row) in overlaps.iter().enumerate() { + let mut hits = row.iter().enumerate().filter(|(_, &hit)| hit); + let (Some((j, _)), None) = (hits.next(), hits.next()) else { + continue; + }; + if overlaps.iter().filter(|other| other[j]).count() == 1 { + taken[i] = Some(j); + out.map.insert(theirs[i].0, ours[j].0); + } + } + // Worth reading vectors for only where a remote face is still + // unplaced and a local face is still free to be its counterpart. + let free = ours.len() > taken.iter().flatten().count(); + if free && taken.iter().any(Option::is_none) { + left_over.push((key, taken)); } } - Ok(map) + if left_over.is_empty() { + return Ok(out); + } + + // ---- by embedding, for what the boxes left -------------------------- + let wanted = |side: &HashMap>| -> String { + let ids: Vec = left_over + .iter() + .flat_map(|(key, _)| side[*key].iter().map(|b| b.0.to_string())) + .collect(); + format!("[{}]", ids.join(",")) + }; + // One statement per side, keyed by row id, for the faces of those + // photographs only. + let read_vectors = |schema: &str, ids: String| -> Result>, CatalogError> { + let mut stmt = tx.prepare(&format!( + "SELECT f.id, f.embedding + FROM json_each(?1) j + JOIN {schema}.faces f ON f.id = j.value" + ))?; + let rows = stmt.query_map([ids], |r| Ok((r.get(0)?, r.get(1)?)))?; + Ok(rows.collect::>()?) + }; + let our_vectors = read_vectors("main", wanted(&local))?; + let their_vectors = read_vectors("remote_cat", wanted(&remote))?; + + for (key, taken) in left_over { + let model = dr_face::ModelId::new(key.1.as_str()); + let decode = + |vectors: &HashMap>, faces: &[Boxed]| -> Vec> { + faces + .iter() + .map(|b| { + let blob = vectors.get(&b.0)?; + dr_face::Embedding::from_f16_bytes(model.clone(), blob) + }) + .collect() + }; + let (theirs, ours) = (&remote[key], &local[key]); + let pairs = pair_by_embedding( + &decode(&their_vectors, theirs), + &decode(&our_vectors, ours), + &taken, + ); + for (i, j) in pairs { + out.map.insert(theirs[i].0, ours[j].0); + out.by_embedding += 1; + } + } + Ok(out) +} + +/// The pairs the embeddings decide, as `(remote index, local index)`, for the +/// remote faces the boxes left unplaced (`taken[i] == None`). +/// +/// Both sides are compared in full -- a local face a box already claimed can +/// still be a remote face's best resemblance, and then that remote face is +/// not placed elsewhere, because its best counterpart is spoken for and its +/// second best is not decisive. Vectors that are missing or of another +/// embedder compare as nothing ([`dr_face::Embedding::cosine`]). +fn pair_by_embedding( + theirs: &[Option], + ours: &[Option], + taken: &[Option], +) -> Vec<(usize, usize)> { + let cos: Vec> = theirs + .iter() + .map(|t| { + ours.iter() + .map(|o| match (t, o) { + (Some(t), Some(o)) => t.cosine(o).unwrap_or(f32::NEG_INFINITY), + _ => f32::NEG_INFINITY, + }) + .collect() + }) + .collect(); + + /// The index of the largest value, and by how much it leads the next. + fn best(values: impl Iterator) -> Option<(usize, f32, f32)> { + let mut first: Option<(usize, f32)> = None; + let mut second = f32::NEG_INFINITY; + for (at, v) in values.enumerate() { + match first { + Some((_, top)) if v <= top => second = second.max(v), + _ => { + if let Some((_, top)) = first { + second = top; + } + first = Some((at, v)); + } + } + } + first.map(|(at, top)| (at, top, second)) + } + let decisive = + |top: f32, second: f32| top >= SAME_FACE_ACROSS_DEVICES && top - second >= DECISIVE_MARGIN; + + let claimed: std::collections::HashSet = taken.iter().flatten().copied().collect(); + let mut pairs = Vec::new(); + for (i, row) in cos.iter().enumerate() { + if taken[i].is_some() { + continue; + } + let Some((j, top, second)) = best(row.iter().copied()) else { + continue; + }; + if claimed.contains(&j) || !decisive(top, second) { + continue; + } + let Some((back, top, second)) = best(cos.iter().map(|row| row[j])) else { + continue; + }; + if back == i && decisive(top, second) { + pairs.push((i, j)); + } + } + pairs } /// The index the local half of [`match_faces`] is read from: every column @@ -2541,4 +2740,181 @@ mod tests { .unwrap(); assert_eq!(people, 1); } + + // ── faces the boxes cannot place, and their vectors ─────────────────── + + /// A unit vector in the embedder's space, the same for the same seed. + /// Two seeds are near-orthogonal, as two strangers' faces are. + fn vector(seed: u32) -> Vec { + let mut s = seed.wrapping_mul(2_654_435_761).wrapping_add(1); + let mut v: Vec = (0..dr_face::EMBEDDING_DIM) + .map(|_| { + s = s.wrapping_mul(1_664_525).wrapping_add(1_013_904_223); + (s >> 8) as f32 / (1u32 << 23) as f32 - 0.5 + }) + .collect(); + let norm = v.iter().map(|x| x * x).sum::().sqrt(); + v.iter_mut().for_each(|x| *x /= norm); + v + } + + /// Store `v` as `face`'s embedding, as the embedder would. + fn embed(c: &Connection, db: &str, face: i64, v: &[f32]) { + let e = dr_face::Embedding { + model: dr_face::ModelId::new("w600k_mbf"), + v: Box::new(v.try_into().unwrap()), + }; + c.execute( + &format!("UPDATE {db}.faces SET embedding = ?2 WHERE id = ?1"), + rusqlite::params![face, e.to_f16_bytes()], + ) + .unwrap(); + } + + /// Anna confirmed on the remote's face 42. + fn anna_on(c: &Connection, remote: i64) { + add_person(c, "remote_cat", 3, "u-anna", "Anna", false); + assign(c, "remote_cat", remote, 3, true); + } + + /// The case #77 was opened for: one device drew the box somewhere else — + /// on the reference library, whole photographs whose boxes sit at IoU 0 + /// with cosines above 0.9 — and the name stayed behind. The vector says + /// it is the same face. + #[test] + fn a_shifted_box_with_the_same_embedding_matches() { + let c = two_catalogs(); + for db in ["main", "remote_cat"] { + add_synced_image(&c, db, 1, 5000); + } + let local = add_face(&c, "main", 7, 1, 0.10); + let remote = add_face(&c, "remote_cat", 42, 1, 0.60); + embed(&c, "main", local, &vector(1)); + embed(&c, "remote_cat", remote, &vector(1)); + anna_on(&c, remote); + + let report = merge_all(&c).unwrap(); + assert_eq!(report.faces_matched_by_embedding, 1); + assert_eq!(person_of(&c, local), Some(("Anna".to_string(), true))); + } + + /// Two faces close enough that both boxes overlap the remote's by more + /// than half: the box cannot say which, and must not guess. The vector + /// can. + #[test] + fn two_overlapping_faces_are_told_apart_by_embedding() { + let c = two_catalogs(); + for db in ["main", "remote_cat"] { + add_synced_image(&c, db, 1, 5000); + } + let front = add_face(&c, "main", 7, 1, 0.30); + let behind = add_face(&c, "main", 8, 1, 0.36); + let remote = add_face(&c, "remote_cat", 42, 1, 0.33); + embed(&c, "main", front, &vector(1)); + embed(&c, "main", behind, &vector(2)); + embed(&c, "remote_cat", remote, &vector(2)); + anna_on(&c, remote); + + merge_all(&c).unwrap(); + assert_eq!(person_of(&c, behind), Some(("Anna".to_string(), true))); + assert_eq!(person_of(&c, front), None); + } + + /// The same two overlapping boxes, and vectors that do not decide: the + /// face stays unmatched rather than going to the larger overlap. + #[test] + fn an_ambiguous_box_with_no_decisive_vector_stays_unmatched() { + let c = two_catalogs(); + for db in ["main", "remote_cat"] { + add_synced_image(&c, db, 1, 5000); + } + let front = add_face(&c, "main", 7, 1, 0.30); + let behind = add_face(&c, "main", 8, 1, 0.35); + let remote = add_face(&c, "remote_cat", 42, 1, 0.33); + embed(&c, "main", front, &vector(1)); + embed(&c, "main", behind, &vector(2)); + // Equally like both: 0.71 each, no margin. + let between: Vec = vector(1) + .iter() + .zip(vector(2)) + .map(|(a, b)| (a + b) / 2f32.sqrt()) + .collect(); + embed(&c, "remote_cat", remote, &between); + anna_on(&c, remote); + + merge_all(&c).unwrap(); + assert_eq!(person_of(&c, front), None); + assert_eq!(person_of(&c, behind), None); + } + + /// The same person in another photograph has the same vector — the + /// worst lookalike there is — and is not the same face. Nor is a face + /// in the right photograph that neither box nor vector ties to it: that + /// is a face this device found and the other did not, and it stays new. + #[test] + fn a_similar_embedding_in_a_different_photograph_never_matches() { + let c = two_catalogs(); + for db in ["main", "remote_cat"] { + add_synced_image(&c, db, 1, 5000); + add_synced_image(&c, db, 2, 6000); + } + let elsewhere = add_face(&c, "main", 7, 2, 0.10); + let stranger = add_face(&c, "main", 8, 1, 0.10); + let remote = add_face(&c, "remote_cat", 42, 1, 0.60); + embed(&c, "main", elsewhere, &vector(1)); + embed(&c, "main", stranger, &vector(2)); + embed(&c, "remote_cat", remote, &vector(1)); + anna_on(&c, remote); + + let report = merge_all(&c).unwrap(); + assert_eq!(report.faces_matched_by_embedding, 0); + assert_eq!(person_of(&c, elsewhere), None, "matched across photographs"); + assert_eq!(person_of(&c, stranger), None, "a new face was matched"); + } + + /// Vectors from two embedders live in two spaces; a cosine between + /// them is a number that means nothing. + #[test] + fn different_embedders_never_compare() { + let c = two_catalogs(); + for db in ["main", "remote_cat"] { + add_synced_image(&c, db, 1, 5000); + } + let local = add_face(&c, "main", 7, 1, 0.10); + c.execute( + "UPDATE main.faces SET model_id = 'scrfd_10g+other_embedder' WHERE id = ?1", + [local], + ) + .unwrap(); + let remote = add_face(&c, "remote_cat", 42, 1, 0.60); + embed(&c, "main", local, &vector(1)); + embed(&c, "remote_cat", remote, &vector(1)); + anna_on(&c, remote); + + merge_all(&c).unwrap(); + assert_eq!(person_of(&c, local), None, "matched across embedders"); + } + + /// A local face its box already placed is not handed to a second remote + /// face because that one resembles it: one face, one counterpart. + #[test] + fn a_face_the_box_placed_is_not_taken_again_by_a_vector() { + let c = two_catalogs(); + for db in ["main", "remote_cat"] { + add_synced_image(&c, db, 1, 5000); + } + let placed = add_face(&c, "main", 7, 1, 0.10); + let free = add_face(&c, "main", 8, 1, 0.70); + let by_box = add_face(&c, "remote_cat", 41, 1, 0.10); + let remote = add_face(&c, "remote_cat", 42, 1, 0.40); + embed(&c, "main", placed, &vector(1)); + embed(&c, "main", free, &vector(3)); + embed(&c, "remote_cat", by_box, &vector(2)); + embed(&c, "remote_cat", remote, &vector(1)); + anna_on(&c, remote); + + merge_all(&c).unwrap(); + assert_eq!(person_of(&c, placed), None); + assert_eq!(person_of(&c, free), None); + } }