Index the library's faces, and group them into people
Wires dr-face to dr-catalog: a background sweep that reads the proxy the grid already built, detects, aligns, embeds and stores, then a clustering pass that turns those embeddings into suggested people. Detection runs on the Large thumbnail tier and nowhere else. That is what makes the feature affordable -- a browsed library has already paid for its proxies, so face indexing adds no RAW decode that was not already happening -- and it is why an image whose proxy is missing is skipped rather than fetched: requesting one here would put face indexing on the network path FR-CULL-8 keeps it off. The sweep keeps no cursor. It asks the catalog what is missing, so it resumes after process death with no repeated work beyond the in-flight image, and cancelling is dropping the receiver. recluster writes only the suggested half. Confirmed faces go in as anchors and come back untouched, and a cluster of one stays nameless -- naming every stray face would fill the People view with noise the user then has to dismiss. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Claude Opus 5
@@ -31,6 +31,9 @@ dr-ingest.workspace = true
|
||||
dr-film.workspace = true
|
||||
dr-pipeline.workspace = true
|
||||
dr-catalog.workspace = true
|
||||
# The face pipeline, with the ONNX runtime: this is the layer that actually
|
||||
# runs the models over the library (docs/faces.md).
|
||||
dr-face = { workspace = true, features = ["inference"] }
|
||||
dr-thumbs.workspace = true
|
||||
# The library module writes scan results straight into the catalog, so it
|
||||
# needs the same SQLite types dr-catalog exposes.
|
||||
|
||||
@@ -0,0 +1,483 @@
|
||||
//! TRACES: FR-CULL-8 | FR-CULL-9 | FR-CULL-10 | NFR-ARCH-2 | NFR-SEC-5
|
||||
//! Face indexing and clustering, as background passes over the library.
|
||||
//!
|
||||
//! `dr-face` knows how to find a face in a buffer and `dr-catalog` knows how to
|
||||
//! store one. This is the pass that connects them: read the proxy the grid
|
||||
//! already built, detect, align, embed, write, and — once detection has gone
|
||||
//! quiet — group what was found into people.
|
||||
//!
|
||||
//! # Detection runs on proxies, never on originals
|
||||
//!
|
||||
//! FR-CULL-8 pins this to the FR-CULL-2 ladder, and the consequence is the
|
||||
//! thing that makes the feature affordable: a library that has been browsed has
|
||||
//! already paid for its proxies, so face indexing adds **no RAW decodes that
|
||||
//! were not already happening**. The tier is `ThumbSize::Large` — 1024 on the
|
||||
//! long edge — and docs/faces.md §7 has the table of what the embedder actually
|
||||
//! receives at that resolution.
|
||||
//!
|
||||
//! # Why clustering is a separate pass and not a job
|
||||
//!
|
||||
//! Detection is per image and parallel, so it is a job. Clustering is a
|
||||
//! *whole-library* operation over the embeddings detection produced: it has no
|
||||
//! natural `subject_id`, and running it per photograph would rebuild the world
|
||||
//! on every one. It therefore runs debounced, when detection has been idle and
|
||||
//! the face count has moved materially (catalog.md §10.2).
|
||||
|
||||
use std::path::PathBuf;
|
||||
use std::sync::mpsc::{Receiver, Sender};
|
||||
|
||||
use dr_catalog::faces::{self, DetectedFace};
|
||||
use dr_catalog::Catalog;
|
||||
use dr_face::{align, Calibration, DetectOptions, Detector, Embedder, ModelId};
|
||||
use dr_thumbs::{ThumbSize, ThumbStore};
|
||||
use dr_types::ImageId;
|
||||
|
||||
/// The tier faces are found on. See the module note.
|
||||
pub const FACE_TIER: ThumbSize = ThumbSize::Large;
|
||||
|
||||
/// Progress from an indexing sweep.
|
||||
#[derive(Debug, Clone, PartialEq)]
|
||||
pub enum FaceSweepMessage {
|
||||
/// How many images will be visited. Sent once, before any work.
|
||||
Total(usize),
|
||||
/// One image finished, with the faces found in it.
|
||||
Indexed { image: ImageId, faces: usize },
|
||||
/// The pass ended.
|
||||
Finished {
|
||||
images: usize,
|
||||
faces: usize,
|
||||
failed: usize,
|
||||
},
|
||||
}
|
||||
|
||||
/// An image waiting to be indexed.
|
||||
#[derive(Debug, Clone, PartialEq)]
|
||||
pub struct FaceRequest {
|
||||
pub image_id: ImageId,
|
||||
/// The thumbnail store's key — `oc:fileid`, stable across a server-side
|
||||
/// move and the same id every other client sees (FR-NC-5).
|
||||
pub file_id: u64,
|
||||
}
|
||||
|
||||
/// Images that have a usable proxy and no faces recorded for this model.
|
||||
///
|
||||
/// Keyed on the model, so a model upgrade re-indexes rather than leaving the
|
||||
/// library half-described by weights that are no longer comparable — the
|
||||
/// failure `faces.model_id` exists to make detectable.
|
||||
pub fn faces_outstanding(
|
||||
catalog: &Catalog,
|
||||
store: &ThumbStore,
|
||||
model_id: &str,
|
||||
) -> Result<Vec<FaceRequest>, dr_catalog::CatalogError> {
|
||||
let mut stmt = catalog.connection().prepare(
|
||||
"SELECT i.id, r.file_id
|
||||
FROM images i
|
||||
JOIN remote r ON r.image_id = i.id
|
||||
WHERE r.file_id IS NOT NULL
|
||||
AND i.trashed_at IS NULL
|
||||
AND NOT EXISTS (
|
||||
SELECT 1 FROM faces f
|
||||
WHERE f.image_id = i.id AND f.model_id = ?1
|
||||
)
|
||||
ORDER BY i.id",
|
||||
)?;
|
||||
let rows = stmt
|
||||
.query_map([model_id], |r| {
|
||||
Ok(FaceRequest {
|
||||
image_id: ImageId(r.get::<_, i64>(0)? as u64),
|
||||
file_id: r.get::<_, i64>(1)? as u64,
|
||||
})
|
||||
})?
|
||||
.filter_map(Result::ok)
|
||||
// A proxy that is not in the store yet is not this pass's problem: the
|
||||
// thumbnail sweep builds it, and the next face pass picks the image up.
|
||||
// Requesting one here would put face indexing on the network path,
|
||||
// which FR-CULL-8 explicitly keeps it off.
|
||||
.filter(|req| store.contains(req.file_id, FACE_TIER))
|
||||
.collect();
|
||||
Ok(rows)
|
||||
}
|
||||
|
||||
/// Detect and embed every face in one decoded proxy.
|
||||
///
|
||||
/// Coordinates come back **normalised to the long edge**, which is what the
|
||||
/// catalog stores: a face must survive the proxy it was found on being evicted
|
||||
/// and regenerated at a different size.
|
||||
///
|
||||
/// A face whose landmarks are degenerate is dropped rather than stored with a
|
||||
/// junk embedding. That happens — a detector firing on a motion-blurred profile
|
||||
/// can put all five landmarks on a line — and one junk embedding in the
|
||||
/// clustering graph can bridge two real people.
|
||||
pub fn index_proxy(
|
||||
detector: &mut Detector,
|
||||
embedder: &mut Embedder,
|
||||
rgb: &[f32],
|
||||
width: usize,
|
||||
height: usize,
|
||||
options: &DetectOptions,
|
||||
) -> Result<Vec<DetectedFace>, dr_face::FaceError> {
|
||||
let long_edge = width.max(height) as f32;
|
||||
if long_edge <= 0.0 {
|
||||
return Ok(Vec::new());
|
||||
}
|
||||
|
||||
let dets = detector.detect(rgb, width, height, options)?;
|
||||
let mut out = Vec::with_capacity(dets.len());
|
||||
|
||||
for d in &dets {
|
||||
let Some(aligned) = align::warp(rgb, width, height, &d.landmarks) else {
|
||||
log::debug!("face with degenerate landmarks skipped");
|
||||
continue;
|
||||
};
|
||||
let embedding = embedder.embed(&aligned)?;
|
||||
|
||||
out.push(DetectedFace {
|
||||
x: d.bbox.0 / long_edge,
|
||||
y: d.bbox.1 / long_edge,
|
||||
w: d.width() / long_edge,
|
||||
h: d.height() / long_edge,
|
||||
landmarks: normalise_landmarks(&d.landmarks, long_edge),
|
||||
confidence: d.confidence,
|
||||
embedding: embedding.to_f16_bytes(),
|
||||
crop_px: aligned.source_px(),
|
||||
model_id: embedder.model().as_str().to_string(),
|
||||
});
|
||||
}
|
||||
|
||||
Ok(out)
|
||||
}
|
||||
|
||||
fn normalise_landmarks(lm: &[(f32, f32); 5], long_edge: f32) -> [(f32, f32); 5] {
|
||||
let mut out = [(0.0_f32, 0.0_f32); 5];
|
||||
for (o, &(x, y)) in out.iter_mut().zip(lm.iter()) {
|
||||
*o = (x / long_edge, y / long_edge);
|
||||
}
|
||||
out
|
||||
}
|
||||
|
||||
/// Index every image that needs it, in the background.
|
||||
///
|
||||
/// Strictly background work: it competes with thumbnailing, not with rendering
|
||||
/// (NFR-ARCH-2), and it is interruptible simply by dropping the receiver — the
|
||||
/// next run resumes from what is already in the catalog, because
|
||||
/// [`faces_outstanding`] asks the catalog what is missing rather than keeping a
|
||||
/// cursor. That is what makes it survive process death (FR-PLAT-AND-3) with no
|
||||
/// repeated work beyond the in-flight image.
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
pub fn spawn_face_sweep(
|
||||
catalog_path: PathBuf,
|
||||
store_dir: PathBuf,
|
||||
detector_model: PathBuf,
|
||||
embedder_model: PathBuf,
|
||||
model_id: String,
|
||||
options: DetectOptions,
|
||||
) -> Receiver<FaceSweepMessage> {
|
||||
let (tx, rx) = std::sync::mpsc::channel();
|
||||
|
||||
std::thread::spawn(move || {
|
||||
let finish_empty = |tx: &Sender<FaceSweepMessage>| {
|
||||
let _ = tx.send(FaceSweepMessage::Finished {
|
||||
images: 0,
|
||||
faces: 0,
|
||||
failed: 0,
|
||||
});
|
||||
};
|
||||
|
||||
let catalog = match Catalog::open(&catalog_path) {
|
||||
Ok(c) => c,
|
||||
Err(e) => {
|
||||
log::warn!("face sweep: cannot open catalog: {e}");
|
||||
finish_empty(&tx);
|
||||
return;
|
||||
}
|
||||
};
|
||||
let store = match ThumbStore::open(&store_dir) {
|
||||
Ok(s) => s,
|
||||
Err(e) => {
|
||||
log::warn!("face sweep: cannot open the thumbnail store: {e}");
|
||||
finish_empty(&tx);
|
||||
return;
|
||||
}
|
||||
};
|
||||
|
||||
// Models first: they are the expensive failure, and there is no point
|
||||
// listing ten thousand images before discovering the weights are
|
||||
// missing. This is also the path a library with face indexing enabled
|
||||
// but no model downloaded takes (docs/faces.md §2.2), so it must be a
|
||||
// quiet return rather than an error.
|
||||
let mut detector = match Detector::from_path(&detector_model) {
|
||||
Ok(d) => d,
|
||||
Err(e) => {
|
||||
log::warn!("face sweep: cannot load the detector: {e}");
|
||||
finish_empty(&tx);
|
||||
return;
|
||||
}
|
||||
};
|
||||
let mut embedder = match Embedder::from_path(&embedder_model, ModelId::new(model_id.clone()))
|
||||
{
|
||||
Ok(e) => e,
|
||||
Err(e) => {
|
||||
log::warn!("face sweep: cannot load the embedder: {e}");
|
||||
finish_empty(&tx);
|
||||
return;
|
||||
}
|
||||
};
|
||||
|
||||
let wanted = match faces_outstanding(&catalog, &store, &model_id) {
|
||||
Ok(w) => w,
|
||||
Err(e) => {
|
||||
log::warn!("face sweep: {e}");
|
||||
finish_empty(&tx);
|
||||
return;
|
||||
}
|
||||
};
|
||||
|
||||
let total = wanted.len();
|
||||
if total == 0 {
|
||||
log::info!("face sweep: every image with a proxy is already indexed");
|
||||
finish_empty(&tx);
|
||||
return;
|
||||
}
|
||||
log::info!("face sweep: {total} image(s) to index");
|
||||
if tx.send(FaceSweepMessage::Total(total)).is_err() {
|
||||
return;
|
||||
}
|
||||
|
||||
let (mut images, mut found, mut failed) = (0usize, 0usize, 0usize);
|
||||
|
||||
for req in wanted {
|
||||
let thumb = match store.get(req.file_id, FACE_TIER) {
|
||||
Ok(Some(t)) => t,
|
||||
// Evicted between the listing and now. Not a failure: the next
|
||||
// pass will find it, or the thumbnail sweep will rebuild it.
|
||||
Ok(None) => continue,
|
||||
Err(e) => {
|
||||
log::debug!("face sweep: reading proxy for {:?}: {e}", req.image_id);
|
||||
failed += 1;
|
||||
continue;
|
||||
}
|
||||
};
|
||||
|
||||
let (w, h, rgba) = match dr_thumbs::codec::decode_rgba(&thumb.bytes) {
|
||||
Ok(v) => v,
|
||||
Err(e) => {
|
||||
log::debug!("face sweep: decoding proxy for {:?}: {e}", req.image_id);
|
||||
failed += 1;
|
||||
continue;
|
||||
}
|
||||
};
|
||||
let rgb = rgba_to_rgb_f32(&rgba);
|
||||
|
||||
let faces = match index_proxy(
|
||||
&mut detector,
|
||||
&mut embedder,
|
||||
&rgb,
|
||||
w as usize,
|
||||
h as usize,
|
||||
&options,
|
||||
) {
|
||||
Ok(f) => f,
|
||||
Err(e) => {
|
||||
log::debug!("face sweep: indexing {:?}: {e}", req.image_id);
|
||||
failed += 1;
|
||||
continue;
|
||||
}
|
||||
};
|
||||
|
||||
if let Err(e) = faces::record_detections(catalog.connection(), req.image_id, &faces) {
|
||||
log::warn!("face sweep: storing faces for {:?}: {e}", req.image_id);
|
||||
failed += 1;
|
||||
continue;
|
||||
}
|
||||
|
||||
images += 1;
|
||||
found += faces.len();
|
||||
if tx
|
||||
.send(FaceSweepMessage::Indexed {
|
||||
image: req.image_id,
|
||||
faces: faces.len(),
|
||||
})
|
||||
.is_err()
|
||||
{
|
||||
// Receiver dropped: the window closed, or the user turned face
|
||||
// indexing off. Stop, leaving everything written so far.
|
||||
log::info!("face sweep: cancelled after {images} image(s)");
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
log::info!("face sweep: {found} face(s) across {images} image(s), {failed} failed");
|
||||
let _ = tx.send(FaceSweepMessage::Finished {
|
||||
images,
|
||||
faces: found,
|
||||
failed,
|
||||
});
|
||||
});
|
||||
|
||||
rx
|
||||
}
|
||||
|
||||
/// Group the library's faces into people, writing suggestions.
|
||||
///
|
||||
/// Confirmations are never touched: they enter the clusterer as anchors and
|
||||
/// come back out unchanged, which is the invariant FR-CULL-10 turns on. What
|
||||
/// this writes is the *suggested* half, and it may be re-run at any time.
|
||||
///
|
||||
/// Returns how many suggestions were written and how many new unnamed people
|
||||
/// were created.
|
||||
pub fn recluster(
|
||||
catalog: &Catalog,
|
||||
model_id: &str,
|
||||
min_probability: f32,
|
||||
) -> Result<(usize, usize), dr_catalog::CatalogError> {
|
||||
let conn = catalog.connection();
|
||||
|
||||
// No valid calibration is not a reason to refuse to cluster — it is a
|
||||
// reason not to *display* a confidence (FR-CULL-9). `Calibration::default`
|
||||
// is the reference implementation's fitted curve with `valid` false, which
|
||||
// is a documented operating point rather than an invented one.
|
||||
let cal = faces::calibration(conn, model_id)?
|
||||
.map(|(c, _)| c)
|
||||
.unwrap_or_else(Calibration::default);
|
||||
|
||||
let stored = faces::embeddings(conn, model_id)?;
|
||||
if stored.is_empty() {
|
||||
return Ok((0, 0));
|
||||
}
|
||||
|
||||
// Which faces the user has already ruled on, so they enter as anchors.
|
||||
let mut confirmed = std::collections::HashMap::new();
|
||||
for p in faces::people(conn)? {
|
||||
for f in faces::for_person(conn, p.id, false)? {
|
||||
confirmed.insert(f.id, p.id);
|
||||
}
|
||||
}
|
||||
|
||||
let model = ModelId::new(model_id.to_string());
|
||||
let mut candidates = Vec::with_capacity(stored.len());
|
||||
let mut ids = Vec::with_capacity(stored.len());
|
||||
for (face_id, image_id, blob, crop_px) in stored {
|
||||
let Some(emb) = dr_face::Embedding::from_f16_bytes(model.clone(), &blob) else {
|
||||
log::warn!("face {face_id:?} has a malformed embedding, skipped");
|
||||
continue;
|
||||
};
|
||||
candidates.push(dr_face::Candidate {
|
||||
face: face_id.0,
|
||||
image: image_id.0,
|
||||
embedding: emb.v.to_vec(),
|
||||
crop_px,
|
||||
confirmed_person: confirmed.get(&face_id).map(|p| p.0),
|
||||
});
|
||||
ids.push(face_id);
|
||||
}
|
||||
|
||||
let clusters = dr_face::cluster(&candidates, &cal, min_probability);
|
||||
|
||||
let mut suggested = 0usize;
|
||||
let mut created = 0usize;
|
||||
for c in &clusters {
|
||||
// A group of one is not a person. Naming every stray face would fill
|
||||
// the People view with noise the user then has to dismiss.
|
||||
if c.members.len() < 2 && c.person.is_none() {
|
||||
continue;
|
||||
}
|
||||
|
||||
let person = match c.person {
|
||||
Some(p) => faces::PersonId(p),
|
||||
None => {
|
||||
created += 1;
|
||||
// Unnamed: FR-CULL-10 has the user name a group, and a group
|
||||
// the system named would be a guess wearing a fact's clothes.
|
||||
faces::create_person(conn, "")?
|
||||
}
|
||||
};
|
||||
|
||||
for &m in &c.members {
|
||||
let face = ids[m];
|
||||
if confirmed.contains_key(&face) {
|
||||
continue;
|
||||
}
|
||||
// The probability the user is shown is the group's own coherence,
|
||||
// not the single best edge into it — a face admitted by one strong
|
||||
// match to an outlier should not present as certain.
|
||||
let p = group_probability(&candidates, c, m, &cal);
|
||||
if faces::suggest(conn, face, person, p)? {
|
||||
suggested += 1;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
log::info!(
|
||||
"reclustered {} face(s) into {} group(s): {suggested} suggestion(s), {created} new",
|
||||
candidates.len(),
|
||||
clusters.len()
|
||||
);
|
||||
Ok((suggested, created))
|
||||
}
|
||||
|
||||
/// Mean calibrated probability between one member and the rest of its group.
|
||||
fn group_probability(
|
||||
candidates: &[dr_face::Candidate],
|
||||
cluster: &dr_face::Cluster,
|
||||
member: usize,
|
||||
cal: &Calibration,
|
||||
) -> f32 {
|
||||
let me = &candidates[member];
|
||||
let mut sum = 0.0;
|
||||
let mut n = 0.0;
|
||||
for &other in &cluster.members {
|
||||
if other == member {
|
||||
continue;
|
||||
}
|
||||
let them = &candidates[other];
|
||||
let cos: f32 = me
|
||||
.embedding
|
||||
.iter()
|
||||
.zip(&them.embedding)
|
||||
.map(|(a, b)| a * b)
|
||||
.sum();
|
||||
sum += cal.probability(cos, me.crop_px.min(them.crop_px), 0.0);
|
||||
n += 1.0;
|
||||
}
|
||||
if n == 0.0 {
|
||||
1.0
|
||||
} else {
|
||||
sum / n
|
||||
}
|
||||
}
|
||||
|
||||
/// `dr-thumbs` decodes to RGBA; `dr-face` reads packed `f32` RGB.
|
||||
fn rgba_to_rgb_f32(rgba: &[u8]) -> Vec<f32> {
|
||||
let mut out = Vec::with_capacity(rgba.len() / 4 * 3);
|
||||
for px in rgba.chunks_exact(4) {
|
||||
out.push(px[0] as f32 / 255.0);
|
||||
out.push(px[1] as f32 / 255.0);
|
||||
out.push(px[2] as f32 / 255.0);
|
||||
}
|
||||
out
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn landmarks_normalise_against_the_long_edge() {
|
||||
let lm = [(512.0, 256.0), (0.0, 0.0), (1024.0, 512.0), (10.0, 20.0), (5.0, 5.0)];
|
||||
let n = normalise_landmarks(&lm, 1024.0);
|
||||
assert!((n[0].0 - 0.5).abs() < 1e-6);
|
||||
assert!((n[0].1 - 0.25).abs() < 1e-6);
|
||||
assert!((n[2].0 - 1.0).abs() < 1e-6);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn rgba_becomes_packed_rgb_dropping_alpha() {
|
||||
let rgba = [255u8, 128, 0, 255, 0, 0, 0, 128];
|
||||
let rgb = rgba_to_rgb_f32(&rgba);
|
||||
assert_eq!(rgb.len(), 6);
|
||||
assert!((rgb[0] - 1.0).abs() < 1e-6);
|
||||
assert!((rgb[1] - 128.0 / 255.0).abs() < 1e-6);
|
||||
assert!((rgb[2] - 0.0).abs() < 1e-6);
|
||||
assert!(rgb[3..6].iter().all(|&v| v == 0.0));
|
||||
}
|
||||
}
|
||||
@@ -22,6 +22,7 @@
|
||||
mod activity;
|
||||
mod collections_ui;
|
||||
mod derived_sync;
|
||||
pub mod faces;
|
||||
mod develop;
|
||||
mod export;
|
||||
mod gradient;
|
||||
|
||||
Reference in New Issue
Block a user