//! TRACES: FR-CULL-8 | FR-CULL-9 | FR-CULL-10 | NFR-ARCH-2 | NFR-SEC-5 //! Face indexing and clustering, as background passes over the library. //! //! `dr-face` knows how to find a face in a buffer and `dr-catalog` knows how to //! store one. This is the pass that connects them: read the proxy the grid //! already built, detect, align, embed, write, and — once detection has gone //! quiet — group what was found into people. //! //! # Detection runs on proxies, never on originals //! //! FR-CULL-8 pins this to the FR-CULL-2 ladder, and the consequence is the //! thing that makes the feature affordable: a library that has been browsed has //! already paid for its proxies, so face indexing adds **no RAW decodes that //! were not already happening**. The tier is `ThumbSize::Large` — 1024 on the //! long edge — and docs/faces.md §7 has the table of what the embedder actually //! receives at that resolution. //! //! # Why clustering is a separate pass and not a job //! //! Detection is per image and parallel, so it is a job. Clustering is a //! *whole-library* operation over the embeddings detection produced: it has no //! natural `subject_id`, and running it per photograph would rebuild the world //! on every one. It therefore runs debounced, when detection has been idle and //! the face count has moved materially (catalog.md §10.2). use std::path::PathBuf; use std::sync::mpsc::{Receiver, Sender}; use dr_catalog::faces::{self, DetectedFace}; use dr_catalog::Catalog; use dr_face::{align, Calibration, DetectOptions, Detector, Embedder, ModelId}; use dr_thumbs::{ThumbSize, ThumbStore}; use dr_types::ImageId; /// The tier faces are found on. See the module note. pub const FACE_TIER: ThumbSize = ThumbSize::Large; /// Progress from an indexing sweep. #[derive(Debug, Clone, PartialEq)] pub enum FaceSweepMessage { /// How many images will be visited. Sent once, before any work. Total(usize), /// One image finished, with the faces found in it. Indexed { image: ImageId, faces: usize }, /// The pass ended. Finished { images: usize, faces: usize, failed: usize, }, } /// An image waiting to be indexed. #[derive(Debug, Clone, PartialEq)] pub struct FaceRequest { pub image_id: ImageId, /// The thumbnail store's key — `oc:fileid`, stable across a server-side /// move and the same id every other client sees (FR-NC-5). pub file_id: u64, } /// Images that have a usable proxy and have not been through this model. /// /// Asks `face_index` — the *run* marker — rather than asking whether the image /// has any faces. Those are different questions, and confusing them is the /// difference between a pass that converges and one that does not: a /// photograph with no face in it would otherwise look identical to one never /// examined, so every landscape in the library would be re-detected on every /// run, for ever. See the V9 migration. /// /// Keyed on the model, so a model upgrade re-indexes rather than leaving the /// library half-described by weights that are no longer comparable. pub fn faces_outstanding( catalog: &Catalog, store: &ThumbStore, model_id: &str, ) -> Result, dr_catalog::CatalogError> { let mut stmt = catalog.connection().prepare( "SELECT i.id, r.file_id FROM images i JOIN remote r ON r.image_id = i.id WHERE r.file_id IS NOT NULL AND i.trashed_at IS NULL AND NOT EXISTS ( SELECT 1 FROM face_index fi WHERE fi.image_id = i.id AND fi.model_id = ?1 ) ORDER BY i.id", )?; let rows = stmt .query_map([model_id], |r| { Ok(FaceRequest { image_id: ImageId(r.get::<_, i64>(0)? as u64), file_id: r.get::<_, i64>(1)? as u64, }) })? .filter_map(Result::ok) // A proxy that is not in the store yet is not this pass's problem: the // thumbnail sweep builds it, and the next face pass picks the image up. // Requesting one here would put face indexing on the network path, // which FR-CULL-8 explicitly keeps it off. .filter(|req| store.contains(req.file_id, FACE_TIER)) .collect(); Ok(rows) } /// What a coverage check found. /// /// The catalog can say how many images have been through the model; only this /// layer can say *why* the rest have not, because the reason usually lives in /// the thumbnail store rather than the catalog. #[derive(Debug, Clone, Copy, PartialEq, Eq, Default)] pub struct IndexAudit { pub coverage: faces::Coverage, /// Outstanding and ready: the proxy exists, so a sweep would do these now. pub ready: u64, /// Outstanding and blocked: no proxy yet, so the thumbnail sweep has to run /// first. Reported separately because it is not a face-indexing problem and /// telling the user to "run indexing again" would not fix it. pub awaiting_proxy: u64, } impl IndexAudit { /// One line, for a log or a status strip. pub fn summary(&self) -> String { let c = &self.coverage; if c.images == 0 { return "no images in the library".into(); } // Whole numbers read fine at 40% and lie at 0.47%, which rounds to // "0%" beside a count of 110 — a figure that says the feature is // broken when it is merely early. One decimal below ten percent, and // a floor so real progress never displays as none. let pct = c.fraction() * 100.0; let shown = if c.indexed > 0 && pct < 0.1 { "<0.1%".to_string() } else if pct < 10.0 { format!("{pct:.1}%") } else { format!("{pct:.0}%") }; let mut s = format!( "{}/{} images indexed ({shown}), {} face(s), {} image(s) with none", c.indexed, c.images, c.faces, c.without_faces, ); if self.ready > 0 { s.push_str(&format!("; {} ready to index", self.ready)); } if self.awaiting_proxy > 0 { s.push_str(&format!("; {} awaiting a proxy", self.awaiting_proxy)); } s } } /// Check every library image for a face-detection run marker. /// /// The batch pass that answers "has face recognition been over all of this", /// and the one to run before deciding whether to start a sweep. Cheap: two /// counts and one indexed scan, no decoding and no inference. pub fn audit( catalog: &Catalog, store: &ThumbStore, model_id: &str, ) -> Result { let conn = catalog.connection(); let coverage = faces::coverage(conn, model_id)?; // Split the outstanding set by whether a proxy exists. This is the query // `faces_outstanding` runs without the store filter, so the two cannot // disagree about what is outstanding. let mut stmt = conn.prepare( "SELECT r.file_id FROM images i JOIN remote r ON r.image_id = i.id WHERE r.file_id IS NOT NULL AND i.trashed_at IS NULL AND NOT EXISTS ( SELECT 1 FROM face_index fi WHERE fi.image_id = i.id AND fi.model_id = ?1 )", )?; let (mut ready, mut awaiting) = (0u64, 0u64); for file_id in stmt .query_map([model_id], |r| r.get::<_, i64>(0))? .filter_map(Result::ok) { if store.contains(file_id as u64, FACE_TIER) { ready += 1; } else { awaiting += 1; } } Ok(IndexAudit { coverage, ready, awaiting_proxy: awaiting, }) } /// Detect and embed every face in one decoded proxy. /// /// Coordinates come back **normalised to the long edge**, which is what the /// catalog stores: a face must survive the proxy it was found on being evicted /// and regenerated at a different size. /// /// A face whose landmarks are degenerate is dropped rather than stored with a /// junk embedding. That happens — a detector firing on a motion-blurred profile /// can put all five landmarks on a line — and one junk embedding in the /// clustering graph can bridge two real people. pub fn index_proxy( detector: &mut Detector, embedder: &mut Embedder, rgb: &[f32], width: usize, height: usize, options: &DetectOptions, ) -> Result, dr_face::FaceError> { let long_edge = width.max(height) as f32; if long_edge <= 0.0 { return Ok(Vec::new()); } let dets = detector.detect(rgb, width, height, options)?; let mut out = Vec::with_capacity(dets.len()); for d in &dets { let Some(aligned) = align::warp(rgb, width, height, &d.landmarks) else { log::debug!("face with degenerate landmarks skipped"); continue; }; let embedding = embedder.embed(&aligned)?; out.push(DetectedFace { x: d.bbox.0 / long_edge, y: d.bbox.1 / long_edge, w: d.width() / long_edge, h: d.height() / long_edge, landmarks: normalise_landmarks(&d.landmarks, long_edge), confidence: d.confidence, embedding: embedding.to_f16_bytes(), crop_px: aligned.source_px(), model_id: embedder.model().as_str().to_string(), }); } Ok(out) } fn normalise_landmarks(lm: &[(f32, f32); 5], long_edge: f32) -> [(f32, f32); 5] { let mut out = [(0.0_f32, 0.0_f32); 5]; for (o, &(x, y)) in out.iter_mut().zip(lm.iter()) { *o = (x / long_edge, y / long_edge); } out } /// Index every image that needs it, in the background. /// /// Strictly background work: it competes with thumbnailing, not with rendering /// (NFR-ARCH-2), and it is interruptible simply by dropping the receiver — the /// next run resumes from what is already in the catalog, because /// [`faces_outstanding`] asks the catalog what is missing rather than keeping a /// cursor. That is what makes it survive process death (FR-PLAT-AND-3) with no /// repeated work beyond the in-flight image. #[allow(clippy::too_many_arguments)] pub fn spawn_face_sweep( catalog_path: PathBuf, store_dir: PathBuf, detector_model: PathBuf, embedder_model: PathBuf, model_id: String, options: DetectOptions, ) -> Receiver { let (tx, rx) = std::sync::mpsc::channel(); std::thread::spawn(move || { let finish_empty = |tx: &Sender| { let _ = tx.send(FaceSweepMessage::Finished { images: 0, faces: 0, failed: 0, }); }; let catalog = match Catalog::open(&catalog_path) { Ok(c) => c, Err(e) => { log::warn!("face sweep: cannot open catalog: {e}"); finish_empty(&tx); return; } }; let store = match ThumbStore::open(&store_dir) { Ok(s) => s, Err(e) => { log::warn!("face sweep: cannot open the thumbnail store: {e}"); finish_empty(&tx); return; } }; // Models first: they are the expensive failure, and there is no point // listing ten thousand images before discovering the weights are // missing. This is also the path a library with face indexing enabled // but no model downloaded takes (docs/faces.md §2.2), so it must be a // quiet return rather than an error. let mut detector = match Detector::from_path(&detector_model) { Ok(d) => d, Err(e) => { log::warn!("face sweep: cannot load the detector: {e}"); finish_empty(&tx); return; } }; let mut embedder = match Embedder::from_path(&embedder_model, ModelId::new(model_id.clone())) { Ok(e) => e, Err(e) => { log::warn!("face sweep: cannot load the embedder: {e}"); finish_empty(&tx); return; } }; let wanted = match faces_outstanding(&catalog, &store, &model_id) { Ok(w) => w, Err(e) => { log::warn!("face sweep: {e}"); finish_empty(&tx); return; } }; let total = wanted.len(); if total == 0 { log::info!("face sweep: every image with a proxy is already indexed"); finish_empty(&tx); return; } log::info!("face sweep: {total} image(s) to index"); if tx.send(FaceSweepMessage::Total(total)).is_err() { return; } let (mut images, mut found, mut failed) = (0usize, 0usize, 0usize); for req in wanted { let thumb = match store.get(req.file_id, FACE_TIER) { Ok(Some(t)) => t, // Evicted between the listing and now. Not a failure: the next // pass will find it, or the thumbnail sweep will rebuild it. Ok(None) => continue, Err(e) => { log::debug!("face sweep: reading proxy for {:?}: {e}", req.image_id); failed += 1; continue; } }; let (w, h, rgba) = match dr_thumbs::codec::decode_rgba(&thumb.bytes) { Ok(v) => v, Err(e) => { log::debug!("face sweep: decoding proxy for {:?}: {e}", req.image_id); failed += 1; continue; } }; let rgb = rgba_to_rgb_f32(&rgba); let faces = match index_proxy( &mut detector, &mut embedder, &rgb, w as usize, h as usize, &options, ) { Ok(f) => f, Err(e) => { log::debug!("face sweep: indexing {:?}: {e}", req.image_id); failed += 1; continue; } }; if let Err(e) = faces::record_detections( catalog.connection(), req.image_id, &model_id, w.max(h), &faces, ) { log::warn!("face sweep: storing faces for {:?}: {e}", req.image_id); failed += 1; continue; } images += 1; found += faces.len(); if tx .send(FaceSweepMessage::Indexed { image: req.image_id, faces: faces.len(), }) .is_err() { // Receiver dropped: the window closed, or the user turned face // indexing off. Stop, leaving everything written so far. log::info!("face sweep: cancelled after {images} image(s)"); return; } } log::info!("face sweep: {found} face(s) across {images} image(s), {failed} failed"); let _ = tx.send(FaceSweepMessage::Finished { images, faces: found, failed, }); }); rx } /// Group the library's faces into people, writing suggestions. /// /// Confirmations are never touched: they enter the clusterer as anchors and /// come back out unchanged, which is the invariant FR-CULL-10 turns on. What /// this writes is the *suggested* half, and it may be re-run at any time. /// /// Returns how many suggestions were written and how many new unnamed people /// were created. pub fn recluster( catalog: &Catalog, model_id: &str, min_probability: f32, ) -> Result<(usize, usize), dr_catalog::CatalogError> { let conn = catalog.connection(); // No valid calibration is not a reason to refuse to cluster — it is a // reason not to *display* a confidence (FR-CULL-9). `Calibration::default` // is the reference implementation's fitted curve with `valid` false, which // is a documented operating point rather than an invented one. let cal = faces::calibration(conn, model_id)? .map(|(c, _)| c) .unwrap_or_else(Calibration::default); let stored = faces::embeddings(conn, model_id)?; if stored.is_empty() { return Ok((0, 0)); } // Which faces the user has already ruled on, so they enter as anchors. let mut confirmed = std::collections::HashMap::new(); for p in faces::people(conn)? { for f in faces::for_person(conn, p.id, false)? { confirmed.insert(f.id, p.id); } } let model = ModelId::new(model_id.to_string()); let mut candidates = Vec::with_capacity(stored.len()); let mut ids = Vec::with_capacity(stored.len()); for (face_id, image_id, blob, crop_px) in stored { let Some(emb) = dr_face::Embedding::from_f16_bytes(model.clone(), &blob) else { log::warn!("face {face_id:?} has a malformed embedding, skipped"); continue; }; candidates.push(dr_face::Candidate { face: face_id.0, image: image_id.0, embedding: emb.v.to_vec(), crop_px, confirmed_person: confirmed.get(&face_id).map(|p| p.0), }); ids.push(face_id); } let clusters = dr_face::cluster(&candidates, &cal, min_probability); let mut suggested = 0usize; let mut created = 0usize; for c in &clusters { // A group of one is not a person. Naming every stray face would fill // the People view with noise the user then has to dismiss. if c.members.len() < 2 && c.person.is_none() { continue; } let person = match c.person { Some(p) => faces::PersonId(p), None => { created += 1; // Unnamed: FR-CULL-10 has the user name a group, and a group // the system named would be a guess wearing a fact's clothes. faces::create_person(conn, "")? } }; for &m in &c.members { let face = ids[m]; if confirmed.contains_key(&face) { continue; } // The probability the user is shown is the group's own coherence, // not the single best edge into it — a face admitted by one strong // match to an outlier should not present as certain. let p = group_probability(&candidates, c, m, &cal); if faces::suggest(conn, face, person, p)? { suggested += 1; } } } log::info!( "reclustered {} face(s) into {} group(s): {suggested} suggestion(s), {created} new", candidates.len(), clusters.len() ); Ok((suggested, created)) } /// Mean calibrated probability between one member and the rest of its group. fn group_probability( candidates: &[dr_face::Candidate], cluster: &dr_face::Cluster, member: usize, cal: &Calibration, ) -> f32 { let me = &candidates[member]; let mut sum = 0.0; let mut n = 0.0; for &other in &cluster.members { if other == member { continue; } let them = &candidates[other]; let cos: f32 = me .embedding .iter() .zip(&them.embedding) .map(|(a, b)| a * b) .sum(); sum += cal.probability(cos, me.crop_px.min(them.crop_px), 0.0); n += 1.0; } if n == 0.0 { 1.0 } else { sum / n } } /// A face cut out of its photograph, ready to draw. #[derive(Debug, Clone, PartialEq)] pub struct FaceCrop { pub width: u32, pub height: u32, /// Tightly packed RGBA. pub rgba: Vec, } /// How much of the surrounding frame a face crop keeps, per side. /// /// A face cut exactly to its detection box reads as a mugshot: no hair, no /// chin, no context, and a row of them is genuinely hard to tell apart — which /// matters, because telling them apart is the entire task the People screen /// asks of the user. A third on each side gives back the head. const CROP_MARGIN: f32 = 0.35; /// Cut one face out of its proxy. /// /// The box is normalised to the long edge (the catalog's convention), so this /// works whatever size the proxy happens to be now — the property that made /// normalising worth the trouble. The thumbnail cache is entitled to evict a /// proxy and regenerate it at another resolution, and a face stored in pixels /// would then point at the wrong part of the picture. pub fn crop_face( rgba: &[u8], width: u32, height: u32, face: &faces::Face, out_edge: u32, ) -> Option { if width == 0 || height == 0 || out_edge == 0 { return None; } let long_edge = width.max(height) as f32; // Square, centred on the face: the grid draws square cells, and cropping to // a square here rather than letterboxing there means the face fills the // cell instead of floating in it. let cx = (face.x + face.w * 0.5) * long_edge; let cy = (face.y + face.h * 0.5) * long_edge; let half = (face.w.max(face.h) * long_edge * 0.5) * (1.0 + CROP_MARGIN); if !(half.is_finite() && half > 0.5) { return None; } let mut out = vec![0u8; (out_edge * out_edge * 4) as usize]; let step = (half * 2.0) / out_edge as f32; for oy in 0..out_edge { let sy = cy - half + (oy as f32 + 0.5) * step; for ox in 0..out_edge { let sx = cx - half + (ox as f32 + 0.5) * step; let o = ((oy * out_edge + ox) * 4) as usize; // Nearest neighbour: this is a downscale of an already-small proxy // shown at ~96 px, and a bilinear tap would cost four reads per // pixel for a difference nobody can see at that size. Outside the // frame stays transparent, so a face at the very edge of the // picture is drawn short rather than smeared. if sx < 0.0 || sy < 0.0 || sx >= width as f32 || sy >= height as f32 { continue; } let i = ((sy as u32 * width + sx as u32) * 4) as usize; if i + 4 <= rgba.len() { out[o..o + 4].copy_from_slice(&rgba[i..i + 4]); } } } Some(FaceCrop { width: out_edge, height: out_edge, rgba: out, }) } /// `dr-thumbs` decodes to RGBA; `dr-face` reads packed `f32` RGB. fn rgba_to_rgb_f32(rgba: &[u8]) -> Vec { let mut out = Vec::with_capacity(rgba.len() / 4 * 3); for px in rgba.chunks_exact(4) { out.push(px[0] as f32 / 255.0); out.push(px[1] as f32 / 255.0); out.push(px[2] as f32 / 255.0); } out } #[cfg(test)] mod tests { use super::*; #[test] fn an_audit_summary_names_both_kinds_of_outstanding() { let a = IndexAudit { coverage: faces::Coverage { images: 100, indexed: 60, without_faces: 40, faces: 35, }, ready: 30, awaiting_proxy: 10, }; let s = a.summary(); assert!(s.contains("60/100"), "{s}"); assert!(s.contains("60%"), "{s}"); assert!(!s.contains("60.0%"), "whole numbers above ten percent: {s}"); assert!(s.contains("30 ready"), "{s}"); assert!(s.contains("10 awaiting"), "{s}"); } #[test] fn a_complete_audit_mentions_neither() { let a = IndexAudit { coverage: faces::Coverage { images: 10, indexed: 10, without_faces: 7, faces: 4, }, ready: 0, awaiting_proxy: 0, }; let s = a.summary(); assert!(!s.contains("ready"), "{s}"); assert!(!s.contains("awaiting"), "{s}"); assert!(s.contains("10/10"), "{s}"); } /// The figure the real library actually produced: 110 of 23,528 rounds to /// "0%" at whole-number precision, which reads as nothing having happened. #[test] fn early_progress_does_not_display_as_zero() { let a = IndexAudit { coverage: faces::Coverage { images: 23_528, indexed: 110, without_faces: 64, faces: 125, }, ready: 69, awaiting_proxy: 23_349, }; let s = a.summary(); assert!(s.contains("0.5%"), "{s}"); assert!(!s.contains("(0%)"), "{s}"); } #[test] fn a_single_image_in_a_huge_library_still_shows_something() { let a = IndexAudit { coverage: faces::Coverage { images: 100_000, indexed: 1, without_faces: 1, faces: 0, }, ready: 99_999, awaiting_proxy: 0, }; assert!(a.summary().contains("<0.1%"), "{}", a.summary()); } #[test] fn an_empty_library_says_so_rather_than_reporting_zero_of_zero() { assert_eq!(IndexAudit::default().summary(), "no images in the library"); } #[test] fn landmarks_normalise_against_the_long_edge() { let lm = [ (512.0, 256.0), (0.0, 0.0), (1024.0, 512.0), (10.0, 20.0), (5.0, 5.0), ]; let n = normalise_landmarks(&lm, 1024.0); assert!((n[0].0 - 0.5).abs() < 1e-6); assert!((n[0].1 - 0.25).abs() < 1e-6); assert!((n[2].0 - 1.0).abs() < 1e-6); } fn gradient(width: u32, height: u32) -> Vec { let mut v = vec![0u8; (width * height * 4) as usize]; for (i, px) in v.chunks_exact_mut(4).enumerate() { // A gradient, so a mis-placed crop shows up as the wrong value // rather than as more of the same colour. px[0] = (i % 251) as u8; px[1] = 40; px[2] = 90; px[3] = 255; } v } fn stored_face(x: f32, y: f32, w: f32, h: f32) -> faces::Face { faces::Face { id: faces::FaceId(1), image_id: ImageId(1), x, y, w, h, landmarks: [(0.0, 0.0); 5], confidence: 0.9, crop_px: 120.0, model_id: "w600k_mbf".into(), person: None, probability: 0.0, confirmed: false, } } #[test] fn a_face_crop_is_square_and_the_size_asked_for() { let px = gradient(1024, 683); let c = crop_face(&px, 1024, 683, &stored_face(0.3, 0.2, 0.1, 0.15), 96).unwrap(); assert_eq!((c.width, c.height), (96, 96)); assert_eq!(c.rgba.len(), 96 * 96 * 4); } #[test] fn a_degenerate_box_yields_no_crop_rather_than_a_panic() { let px = gradient(64, 64); assert!(crop_face(&px, 64, 64, &stored_face(0.5, 0.5, 0.0, 0.0), 96).is_none()); assert!(crop_face(&px, 0, 0, &stored_face(0.1, 0.1, 0.2, 0.2), 96).is_none()); assert!(crop_face(&px, 64, 64, &stored_face(0.1, 0.1, 0.2, 0.2), 0).is_none()); } /// A face at the very edge of the frame is drawn short, not smeared: the /// out-of-frame margin stays transparent. #[test] fn a_face_at_the_edge_keeps_a_transparent_margin() { let px = gradient(200, 200); let c = crop_face(&px, 200, 200, &stored_face(0.0, 0.0, 0.1, 0.1), 32).unwrap(); assert_eq!(c.rgba[3], 0, "outside the frame should be transparent"); let centre = ((16 * 32 + 16) * 4 + 3) as usize; assert_eq!(c.rgba[centre], 255, "the face itself should be opaque"); } /// The normalised box means a proxy regenerated at another resolution still /// crops the same part of the picture — what the catalog's normalisation is /// for. #[test] fn the_same_face_crops_the_same_region_at_two_proxy_sizes() { let face = stored_face(0.25, 0.25, 0.2, 0.2); let small = crop_face(&gradient(400, 400), 400, 400, &face, 16).unwrap(); let large = crop_face(&gradient(800, 800), 800, 800, &face, 16).unwrap(); assert!(small.rgba.chunks_exact(4).all(|p| p[3] == 255)); assert!(large.rgba.chunks_exact(4).all(|p| p[3] == 255)); } #[test] fn the_crop_keeps_margin_around_the_detection_box() { // A 0.1-wide face in a 1000px frame is 100px; with the margin the crop // spans 100 * 1.35 = 135px of source. let face = stored_face(0.4, 0.4, 0.1, 0.1); let c = crop_face(&gradient(1000, 1000), 1000, 1000, &face, 135).unwrap(); assert_eq!(c.width, 135); // Fully inside the frame, so nothing is transparent. assert!(c.rgba.chunks_exact(4).all(|p| p[3] == 255)); } #[test] fn rgba_becomes_packed_rgb_dropping_alpha() { let rgba = [255u8, 128, 0, 255, 0, 0, 0, 128]; let rgb = rgba_to_rgb_f32(&rgba); assert_eq!(rgb.len(), 6); assert!((rgb[0] - 1.0).abs() < 1e-6); assert!((rgb[1] - 128.0 / 255.0).abs() < 1e-6); assert!((rgb[2] - 0.0).abs() < 1e-6); assert!(rgb[3..6].iter().all(|&v| v == 0.0)); } }