Add a dedup_people example to measure the job on a catalog copy
`dedup_people COPY.sqlite` prints the listed people and faces before and after, what the first run merged and kept apart and why, and the time of three runs. The second and third runs are the cost the job adds to every sync. `--peer PEER_COPY.sqlite` then plays two sync round trips. The peer merges with the previous release's code path and no job, this side merges back through sync::merge_remote, and the named people each side lists are compared after every step. On the reference pair: the desktop merges Claudine, Jessie x2, Mathias and Noemi (80 -> 75 named). The tablet also merges its empty second Ian (80 -> 74). The first run takes 1.2 s on the desktop, which is building faces_box; the merge builds it first in practice. Later runs take 8-15 ms.
This commit is contained in:
@@ -0,0 +1,141 @@
|
|||||||
|
//! Run the people and face deduplication (#78) on a copy of a real catalog.
|
||||||
|
//!
|
||||||
|
//! cargo run --release -p dr-catalog --example dedup_people -- COPY.sqlite [--peer PEER_COPY.sqlite]
|
||||||
|
//!
|
||||||
|
//! It writes: run it against a *copy* (`sqlite3 catalog.sqlite ".backup
|
||||||
|
//! copy.sqlite"`), never the library's own file. Prints the live people and
|
||||||
|
//! faces before and after, what the first run merged and kept apart, and
|
||||||
|
//! how long the first and a second run took -- the second is the cost the
|
||||||
|
//! job adds to every sync once a catalog is clean.
|
||||||
|
//!
|
||||||
|
//! `--peer` then plays a sync round trip with another device's catalog (a
|
||||||
|
//! copy of the server snapshot, which it also writes): the peer merges this
|
||||||
|
//! one as the previous release would, with no job after it, then this one
|
||||||
|
//! merges the peer back through `sync::merge_remote`, twice. The named
|
||||||
|
//! people each side lists are printed after each step; they should agree.
|
||||||
|
|
||||||
|
use std::path::PathBuf;
|
||||||
|
use std::time::Instant;
|
||||||
|
|
||||||
|
use dr_catalog::{dedup_people, merge, schema, sync};
|
||||||
|
use rusqlite::Connection;
|
||||||
|
|
||||||
|
fn open(path: &std::path::Path) -> Connection {
|
||||||
|
let conn = Connection::open(path).expect("open the catalog copy");
|
||||||
|
schema::configure(&conn).expect("configure");
|
||||||
|
schema::migrate(&conn).expect("migrate");
|
||||||
|
conn
|
||||||
|
}
|
||||||
|
|
||||||
|
/// The named people a device lists, as `name (uuid prefix)`, sorted.
|
||||||
|
fn named(conn: &Connection) -> Vec<String> {
|
||||||
|
let mut v: Vec<String> = conn
|
||||||
|
.prepare(
|
||||||
|
"SELECT name, substr(uuid, 1, 8) FROM people
|
||||||
|
WHERE merged_into IS NULL AND trim(name) <> ''",
|
||||||
|
)
|
||||||
|
.unwrap()
|
||||||
|
.query_map([], |r| {
|
||||||
|
Ok(format!(
|
||||||
|
"{} ({})",
|
||||||
|
r.get::<_, String>(0)?,
|
||||||
|
r.get::<_, String>(1)?
|
||||||
|
))
|
||||||
|
})
|
||||||
|
.unwrap()
|
||||||
|
.collect::<Result<_, _>>()
|
||||||
|
.unwrap();
|
||||||
|
v.sort();
|
||||||
|
v
|
||||||
|
}
|
||||||
|
|
||||||
|
fn main() {
|
||||||
|
let mut args: Vec<String> = std::env::args().skip(1).collect();
|
||||||
|
let peer = args.iter().position(|a| a == "--peer").map(|at| {
|
||||||
|
let p = PathBuf::from(&args[at + 1]);
|
||||||
|
args.drain(at..at + 2);
|
||||||
|
p
|
||||||
|
});
|
||||||
|
let Some(path) = args.first().map(PathBuf::from) else {
|
||||||
|
eprintln!("usage: dedup_people COPY.sqlite [--peer PEER_COPY.sqlite]");
|
||||||
|
std::process::exit(2);
|
||||||
|
};
|
||||||
|
let conn = open(&path);
|
||||||
|
|
||||||
|
let counts = |label: &str| {
|
||||||
|
let q = |sql: &str| -> i64 { conn.query_row(sql, [], |r| r.get(0)).unwrap() };
|
||||||
|
println!(
|
||||||
|
"{label}: {} people listed ({} named), {} redirects, {} faces, {} confirmed",
|
||||||
|
q("SELECT COUNT(*) FROM people WHERE merged_into IS NULL"),
|
||||||
|
q("SELECT COUNT(*) FROM people WHERE merged_into IS NULL AND trim(name) <> ''"),
|
||||||
|
q("SELECT COUNT(*) FROM people WHERE merged_into IS NOT NULL"),
|
||||||
|
q("SELECT COUNT(*) FROM faces"),
|
||||||
|
q("SELECT COUNT(*) FROM face_person WHERE confirmed = 1"),
|
||||||
|
);
|
||||||
|
};
|
||||||
|
|
||||||
|
counts("before");
|
||||||
|
for pass in ["first", "second", "third"] {
|
||||||
|
let started = Instant::now();
|
||||||
|
let report = dedup_people::run(&conn).expect("dedup");
|
||||||
|
let took = started.elapsed();
|
||||||
|
println!("{pass} run: {took:?}, changed: {}", report.changed());
|
||||||
|
if pass == "first" {
|
||||||
|
println!(" merged: {:?}", report.merged);
|
||||||
|
for k in &report.kept_apart {
|
||||||
|
println!(
|
||||||
|
" kept apart: {:?} ({}) from {}: {:?}",
|
||||||
|
k.name, k.uuid, k.survivor, k.why
|
||||||
|
);
|
||||||
|
}
|
||||||
|
println!(
|
||||||
|
" redirects followed {}, cycles broken {}, faces fused {}, faces confirmed apart {}",
|
||||||
|
report.redirects_followed,
|
||||||
|
report.cycles_broken,
|
||||||
|
report.faces_fused,
|
||||||
|
report.faces_confirmed_apart
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
counts("after");
|
||||||
|
|
||||||
|
let Some(peer_path) = peer else { return };
|
||||||
|
let peer = open(&peer_path);
|
||||||
|
let show = |step: &str| {
|
||||||
|
let (ours, theirs) = (named(&conn), named(&peer));
|
||||||
|
println!(
|
||||||
|
"{step}: this device lists {} named, the peer {}; {}",
|
||||||
|
ours.len(),
|
||||||
|
theirs.len(),
|
||||||
|
if ours == theirs {
|
||||||
|
"the same".to_string()
|
||||||
|
} else {
|
||||||
|
format!("differ:\n here {ours:?}\n peer {theirs:?}")
|
||||||
|
}
|
||||||
|
);
|
||||||
|
};
|
||||||
|
show("before the round trip");
|
||||||
|
for round in 1..=2 {
|
||||||
|
peer.execute(
|
||||||
|
"ATTACH DATABASE ?1 AS remote_cat",
|
||||||
|
[path.to_string_lossy().as_ref()],
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
|
let theirs = merge::merge_all(&peer).expect("the peer's merge");
|
||||||
|
peer.execute("DETACH DATABASE remote_cat", []).unwrap();
|
||||||
|
println!(
|
||||||
|
"round {round}: the peer took {} people updated, {} inserted",
|
||||||
|
theirs.people_updated, theirs.people_inserted
|
||||||
|
);
|
||||||
|
show(&format!("round {round}, after the peer's merge"));
|
||||||
|
let started = Instant::now();
|
||||||
|
let ours = sync::merge_remote(&conn, &peer_path).expect("our merge");
|
||||||
|
println!(
|
||||||
|
"round {round}: merge_remote with the job took {:?}; {} people updated, {} inserted",
|
||||||
|
started.elapsed(),
|
||||||
|
ours.people_updated,
|
||||||
|
ours.people_inserted
|
||||||
|
);
|
||||||
|
show(&format!("round {round}, after ours"));
|
||||||
|
}
|
||||||
|
}
|
||||||
Reference in New Issue
Block a user