Sync face data as sealed shards, so a second device does not re-index
Indexing 23,500 images is about two hours of CPU, and the result is byte-identical on every device: the same model over the same proxy produces the same embedding. Paying for it once per account rather than once per device is the point. Shards rather than the catalog snapshot, because the snapshot goes up whole on every sync and a fully indexed library carries roughly 30 MB of embeddings. That is exactly the cost the thumbnail store's 25 MB cap exists to bound, so face shards use the same cap -- imported from dr_thumbs rather than restated, since the number is a statement about sync cost and the two must not drift apart. The split follows the one already there: bulk immutable data in sealed shards, small mutable data in the catalog snapshot. Faces, landmarks, embeddings and run markers shard; people, names and assignments ride the catalog and merge by uuid. Keyed on oc:fileid throughout, never on image_id, because a row id means nothing on another device. The run marker travels with the faces it describes. Without it a receiving device cannot tell an image with no faces from one never examined, and would re-detect every landscape it had just adopted. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
@@ -4,6 +4,7 @@
|
||||
//! indexes whatever is missing one.
|
||||
//!
|
||||
//! cargo run -p dr-ui --example face_index -- CATALOG.db THUMBS_DIR [--run DET.onnx EMB.onnx]
|
||||
//! cargo run -p dr-ui --example face_index -- CATALOG.db THUMBS_DIR --cluster
|
||||
//!
|
||||
//! # Why this exists beside the button in the Identity screen
|
||||
//!
|
||||
@@ -81,6 +82,23 @@ fn main() {
|
||||
println!(" ready to index {}", audit.ready);
|
||||
println!(" awaiting proxy {}", audit.awaiting_proxy);
|
||||
|
||||
// Grouping is a separate step from indexing on purpose: it is a
|
||||
// whole-library operation over the embeddings detection produced, and it is
|
||||
// worth running *after* a sweep rather than during one (catalog.md §10.2).
|
||||
if args.iter().any(|a| a == "--cluster") {
|
||||
match dr_ui::faces::recluster(&catalog, MODEL_ID, dr_face::DEFAULT_MERGE_PROBABILITY) {
|
||||
Ok((suggested, created)) => {
|
||||
println!("\nclustering: {suggested} suggestion(s), {created} new group(s)");
|
||||
report_people(&catalog);
|
||||
}
|
||||
Err(e) => {
|
||||
eprintln!("clustering failed: {e}");
|
||||
std::process::exit(1);
|
||||
}
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
let run = args.iter().position(|a| a == "--run");
|
||||
let Some(i) = run else {
|
||||
if audit.coverage.is_complete() {
|
||||
@@ -129,7 +147,7 @@ fn main() {
|
||||
seen += 1;
|
||||
// One line per image would be thousands of lines; one per
|
||||
// twenty-five is enough to see it moving and to estimate.
|
||||
if seen % 25 == 0 || faces > 0 {
|
||||
if seen.is_multiple_of(25) || faces > 0 {
|
||||
let rate = seen as f64 / start.elapsed().as_secs_f64().max(1e-6);
|
||||
println!(
|
||||
" {seen}/{total} {:.2} img/s ~{:.0} min left",
|
||||
@@ -154,4 +172,31 @@ fn main() {
|
||||
if let Ok(a) = faces::audit(&catalog, &store, MODEL_ID) {
|
||||
println!("{}", a.summary());
|
||||
}
|
||||
println!("\nrun again with --cluster to group these faces into people.");
|
||||
}
|
||||
|
||||
/// What the clustering proposed, largest group first.
|
||||
fn report_people(catalog: &Catalog) {
|
||||
let Ok(people) = dr_catalog::faces::people(catalog.connection()) else {
|
||||
return;
|
||||
};
|
||||
if people.is_empty() {
|
||||
println!("no groups — too few faces, or none similar enough to group.");
|
||||
return;
|
||||
}
|
||||
println!("\n{} group(s):", people.len());
|
||||
for p in people.iter().take(30) {
|
||||
let name = if p.name.is_empty() {
|
||||
"(unnamed)".to_string()
|
||||
} else {
|
||||
p.name.clone()
|
||||
};
|
||||
println!(
|
||||
" {name:<24} {} confirmed, {} suggested",
|
||||
p.confirmed_faces, p.suggested_faces
|
||||
);
|
||||
}
|
||||
if people.len() > 30 {
|
||||
println!(" … and {} more", people.len() - 30);
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user