diff --git a/core/dr-catalog/src/duplicates.rs b/core/dr-catalog/src/duplicates.rs new file mode 100644 index 0000000..0881068 --- /dev/null +++ b/core/dr-catalog/src/duplicates.rs @@ -0,0 +1,1100 @@ +//! TRACES: FR-CAT-11a +//! The same original, catalogued more than once — found, proved, and folded +//! onto one copy. +//! +//! [`crate::dedup`] stops a card being imported twice. This is the other +//! half of FR-CAT-11: a library that already holds the same RAW in several +//! folders — a dated folder, a `bck/` beside it, a renamed export tree — +//! because it was assembled by hand before DarkRoom saw it. On the reference +//! library that is 1,836 moments held two to five times over. +//! +//! # Three steps, and only the last one writes +//! +//! **Candidates** ([`candidates`]) are one grouped query: same root, same +//! camera, same capture instant, same byte count. Nothing is trashed on that +//! evidence alone — two bodies of one model firing in the same second with +//! files of one size is unlikely, not impossible. +//! +//! **Proof** is the caller's, because it reads bytes: a stored content hash +//! where every copy has one, otherwise a digest of the first and last +//! [`PROBE_WINDOW`] of each file, kept here ([`record_probes`]) keyed on the +//! size and mtime it was taken at, so a second review costs nothing. +//! +//! **Consolidation** ([`consolidate`]) folds each copy's judgements onto the +//! survivor and records the copies as trashed, in one transaction per +//! group: a group is either wholly consolidated or untouched. The files are +//! moved by the caller *before* this commits, as every trash is +//! ([`crate::trash::record_trashed`] says why), and moved back if it fails. +//! +//! # What merges +//! +//! Collections are unioned, the survivor keeping its own position and +//! taking a copy's where it had none. Keywords are unioned. The rating is +//! the highest. A flag or a label is taken where the copies that carry one +//! agree, and left as the survivor had it where they do not — reported as a +//! conflict rather than decided silently. Faces follow +//! [`crate::faces::carry_onto_copy`]. Develop edits live in sidecars, which +//! this crate does not read; the caller compares them and does not consolidate +//! a group whose edits differ (virtual copies, FR-CAT-12, are where two edits +//! of one frame belong). + +use std::collections::HashMap; + +use rusqlite::{Connection, OptionalExtension}; + +use dr_types::ImageId; + +use crate::error::CatalogError; +use crate::faces::FaceCarry; + +/// How much of each end of a file the sameness probe reads. +/// +/// A megabyte from the front holds the EXIF, the maker notes and the +/// embedded previews; a megabyte from the back is sensor data. Two files +/// already equal in size, camera and capture instant that also agree there +/// are the same file for any purpose a library has — and proving it costs +/// two megabytes rather than the whole 25 MB RAW. +pub const PROBE_WINDOW: u64 = 1 << 20; + +/// Folder names that say "this is a spare": a survivor is never taken from +/// under one while a copy elsewhere is not. +/// +/// Matched against the *words* of each folder name, case-insensitively, so +/// `bck`, `Backup 2019`, `old_raws` and `Copy of Alps` all count and +/// `Bold` and `Oldenburg` do not. +pub const BACKUP_WORDS: &[&str] = &[ + "bck", + "bak", + "backup", + "backups", + "copy", + "copies", + "old", + "dup", + "dups", + "duplicate", + "duplicates", + "archive", + "spare", +]; + +/// One catalogued copy in a duplicate group. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct Copy { + pub image: ImageId, + pub source_ref: String, + pub file_size: u64, + pub file_mtime: Option, + pub added_at: i64, + /// A full digest of the bytes, where an import or an earlier check + /// computed one. + pub content_hash: Option, + /// The sameness probe stored for this copy, if it is still valid — taken + /// at the size and mtime the catalog now records, over the current + /// [`PROBE_WINDOW`]. + pub probe: Option, + /// `oc:fileid`, the thumbnail store's key. + pub file_id: Option, +} + +impl Copy { + /// The file's own name, the last component of its key. + pub fn file_name(&self) -> &str { + file_name(&self.source_ref) + } +} + +/// Copies that look like one original: same root, camera, instant and size. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct Group { + pub root_id: i64, + pub camera: String, + pub captured_at: i64, + pub file_size: u64, + /// By image id, so the order is stable between two reads. + pub copies: Vec, +} + +/// Which rows can be duplicates at all: dated, identified, sized, and +/// present in the grid — not trashed, and not a JPEG a RAW beside it +/// already stands for. +const CANDIDATE: &str = "captured_at IS NOT NULL AND camera IS NOT NULL AND file_size IS NOT NULL \ + AND trashed_at IS NULL AND shadowed_by IS NULL"; + +/// How many duplicate groups the library holds, and how many copies +/// consolidating them all would move to the trash. +/// +/// Counted in SQL, for the settings line and the sidebar: 40 ms on the +/// 24k-image reference library, and it builds no list. +pub fn count(conn: &Connection) -> Result<(usize, usize), CatalogError> { + let (groups, extra): (i64, i64) = conn.query_row( + &format!( + "SELECT COUNT(*), coalesce(SUM(n - 1), 0) FROM ( + SELECT COUNT(*) AS n FROM images + WHERE {CANDIDATE} + GROUP BY root_id, camera, captured_at, file_size + HAVING COUNT(*) > 1)" + ), + [], + |r| Ok((r.get(0)?, r.get(1)?)), + )?; + Ok((groups as usize, extra as usize)) +} + +/// Every duplicate group, with its copies, in one query. +/// +/// Grouped in SQL and joined back for the rows, so the planner walks +/// `images_captured` once rather than once per group. Each copy carries the +/// probe stored for it, where one is still valid. +pub fn candidates(conn: &Connection) -> Result, CatalogError> { + ensure_probe_table(conn)?; + let mut stmt = conn.prepare(&format!( + "WITH g AS ( + SELECT root_id, camera, captured_at, file_size FROM images + WHERE {CANDIDATE} + GROUP BY root_id, camera, captured_at, file_size + HAVING COUNT(*) > 1) + SELECT i.id, i.root_id, i.camera, i.captured_at, i.file_size, i.source_ref, + i.file_mtime, i.added_at, i.content_hash, r.file_id, p.digest + FROM g + JOIN images i ON i.root_id = g.root_id AND i.camera = g.camera + AND i.captured_at = g.captured_at AND i.file_size = g.file_size + LEFT JOIN remote r ON r.image_id = i.id + LEFT JOIN dedup_probes p ON p.image_id = i.id AND p.file_size = i.file_size + AND p.file_mtime IS i.file_mtime AND p.window = ?1 + WHERE i.trashed_at IS NULL AND i.shadowed_by IS NULL + ORDER BY i.captured_at, i.camera, i.file_size, i.root_id, i.id" + ))?; + let mut groups: Vec = Vec::new(); + let mut rows = stmt.query([PROBE_WINDOW as i64])?; + while let Some(r) = rows.next()? { + let root_id: i64 = r.get(1)?; + let camera: String = r.get(2)?; + let captured_at: i64 = r.get(3)?; + let file_size = r.get::<_, i64>(4)? as u64; + let copy = Copy { + image: ImageId(r.get::<_, i64>(0)? as u64), + source_ref: r.get(5)?, + file_size, + file_mtime: r.get(6)?, + added_at: r.get(7)?, + content_hash: r.get(8)?, + file_id: r.get::<_, Option>(9)?.map(|v| v as u64), + probe: r.get(10)?, + }; + match groups.last_mut() { + Some(g) + if g.root_id == root_id + && g.camera == camera + && g.captured_at == captured_at + && g.file_size == file_size => + { + g.copies.push(copy) + } + _ => groups.push(Group { + root_id, + camera, + captured_at, + file_size, + copies: vec![copy], + }), + } + } + Ok(groups) +} + +/// The last component of a stored source reference — see +/// [`crate::dedup`] for why both separators. +fn file_name(source_ref: &str) -> &str { + source_ref.rsplit(['/', ':']).next().unwrap_or(source_ref) +} + +/// Whether any folder on the way to this file is named like a spare copy. +/// +/// Folders only: the file's own name is the camera-naming rule's business. +pub fn is_backup_path(source_ref: &str) -> bool { + let mut parts: Vec<&str> = source_ref.split(['/', ':']).collect(); + parts.pop(); + parts.iter().any(|folder| { + folder + .split(|c: char| !c.is_ascii_alphanumeric()) + .any(|word| BACKUP_WORDS.iter().any(|b| word.eq_ignore_ascii_case(b))) + }) +} + +/// Whether a file still carries the name its camera gave it. +/// +/// Every maker spells it the same way: a short prefix of capitals or +/// underscores, then a four- or five-digit counter — `IMG_4623`, +/// `_MG_4623`, `DSC_0042`, `_DSC0042`, `DSCF1234`, `DSC01234`, `IMGP0001`, +/// `DJI_0001` — and Olympus and Panasonic a letter and seven digits, +/// `P1010001`. A renamed copy (`20230628_0642`, `alps-12`, `IMG_4623-2`) +/// does not fit. +pub fn camera_named(name: &str) -> bool { + let stem = match name.rsplit_once('.') { + Some((stem, _)) => stem, + None => name, + }; + let bytes = stem.as_bytes(); + if !(7..=10).contains(&bytes.len()) { + return false; + } + // Olympus / Panasonic: P, then the month or a model digit, then six. + if bytes[0] == b'P' && bytes[1..].iter().all(|b| b.is_ascii_alphanumeric()) { + let rest = &bytes[1..]; + if rest.len() == 7 && rest[1..].iter().all(u8::is_ascii_digit) { + return true; + } + } + let digits = bytes + .iter() + .rev() + .take_while(|b| b.is_ascii_digit()) + .count(); + let prefix = &bytes[..bytes.len() - digits]; + (4..=5).contains(&digits) + && (3..=5).contains(&prefix.len()) + && !prefix[0].is_ascii_digit() + && prefix + .iter() + .all(|b| b.is_ascii_uppercase() || b.is_ascii_digit() || *b == b'_') +} + +/// Which copy of a group stays in the library, by index into `copies`. +/// +/// In order: not under a backup-looking folder; then still named the way +/// the camera named it; then catalogued first; then the lowest id, so the +/// answer never depends on anything but the rows. +pub fn survivor(copies: &[Copy]) -> usize { + copies + .iter() + .enumerate() + .min_by_key(|(_, c)| { + ( + is_backup_path(&c.source_ref), + !camera_named(c.file_name()), + c.added_at, + c.image.0, + ) + }) + .map(|(i, _)| i) + .unwrap_or(0) +} + +// ── probes ─────────────────────────────────────────────────────────────── + +/// Where sameness probes are kept. +/// +/// Created on first use rather than by a migration, deliberately. A new +/// schema version makes every older build refuse this catalog's snapshot at +/// sync (`crate::sync::remote_is_mergeable`), so the tablet would stop +/// merging until it was updated — a high price for a cache. An older build +/// that meets this table ignores it, and a rebuilt catalog simply probes +/// again. +pub fn ensure_probe_table(conn: &Connection) -> Result<(), CatalogError> { + conn.execute_batch( + "CREATE TABLE IF NOT EXISTS dedup_probes ( + image_id INTEGER PRIMARY KEY REFERENCES images(id) ON DELETE CASCADE, + -- What the file was when it was read. A probe of a file that has + -- since changed is not evidence about the file it is now. + file_size INTEGER NOT NULL, + file_mtime INTEGER, + -- The bytes read from each end, so a change to PROBE_WINDOW + -- invalidates rather than compares unlike with unlike. + window INTEGER NOT NULL, + digest TEXT NOT NULL, + probed_at INTEGER NOT NULL + );", + )?; + Ok(()) +} + +/// One sameness probe, as the checking job produced it. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct Probe { + pub image: ImageId, + pub file_size: u64, + pub file_mtime: Option, + pub digest: String, +} + +/// Keep probes for the next review, in one transaction. +pub fn record_probes(conn: &Connection, probes: &[Probe], now: i64) -> Result<(), CatalogError> { + if probes.is_empty() { + return Ok(()); + } + ensure_probe_table(conn)?; + let tx = conn.unchecked_transaction()?; + { + let mut stmt = tx.prepare( + "INSERT INTO dedup_probes (image_id, file_size, file_mtime, window, digest, probed_at) + VALUES (?1, ?2, ?3, ?4, ?5, ?6) + ON CONFLICT(image_id) DO UPDATE SET + file_size = excluded.file_size, file_mtime = excluded.file_mtime, + window = excluded.window, digest = excluded.digest, + probed_at = excluded.probed_at", + )?; + for p in probes { + stmt.execute(rusqlite::params![ + p.image.0 as i64, + p.file_size as i64, + p.file_mtime, + PROBE_WINDOW as i64, + p.digest, + now + ])?; + } + } + tx.commit()?; + Ok(()) +} + +// ── consolidation ──────────────────────────────────────────────────────── + +/// What consolidating one group did, or would do. +#[derive(Debug, Clone, PartialEq, Eq, Default)] +pub struct Outcome { + /// The survivor's rating before and after. + pub rating: (u8, u8), + /// The survivor's flag before and after (catalog codes). + pub flag: (i64, i64), + /// The survivor's label before and after (catalog codes, `None` for none). + pub label: (Option, Option), + /// Copies carried different flags; the survivor's was kept. + pub flag_conflict: bool, + /// Copies carried different labels; the survivor's was kept. + pub label_conflict: bool, + pub keywords_added: usize, + /// Collections the survivor joined, having been in none of them. + pub collections_joined: usize, + pub faces: FaceCarry, + /// Copies recorded as trashed. Zero for a preview. + pub trashed: usize, +} + +impl Outcome { + /// Whether anything besides the trash would change on the survivor. + pub fn merges_anything(&self) -> bool { + self.rating.0 != self.rating.1 + || self.flag.0 != self.flag.1 + || self.label.0 != self.label.1 + || self.keywords_added > 0 + || self.collections_joined > 0 + || self.faces.moved + self.faces.named + self.faces.rejections > 0 + } +} + +/// Fold `copies` onto `survivor` and record them as trashed at the paths +/// they were moved to, in one transaction. +/// +/// Call **after** the files are in the trash, like +/// [`crate::trash::record_trashed`]; if this fails, move them back. Every +/// copy is re-checked against the survivor first — still present, still +/// untrashed, still the same camera, instant and size — and a group that no +/// longer holds is refused with [`CatalogError::StaleDuplicate`] and nothing +/// written. +pub fn consolidate( + conn: &Connection, + survivor: ImageId, + copies: &[(ImageId, String)], + now: i64, +) -> Result { + let tx = conn.unchecked_transaction()?; + let ids: Vec = copies.iter().map(|(id, _)| *id).collect(); + let mut out = merge_within(&tx, survivor, &ids)?; + out.trashed = crate::trash::record_trashed_within(&tx, copies, now)?; + if out.trashed != copies.len() { + return Err(CatalogError::StaleDuplicate(format!( + "{} of {} copies could be recorded as trashed", + out.trashed, + copies.len() + ))); + } + tx.commit()?; + Ok(out) +} + +/// What [`consolidate`] would do, without doing it. +/// +/// The same code in a transaction that is rolled back, so the review can +/// never describe a merge other than the one the button performs. +pub fn preview( + conn: &Connection, + survivor: ImageId, + copies: &[ImageId], +) -> Result { + let tx = conn.unchecked_transaction()?; + let out = merge_within(&tx, survivor, copies)?; + tx.rollback()?; + Ok(out) +} + +/// The judgement a default version carries: `(version id, rating, flag, label)`. +type Judged = (i64, i64, i64, Option); + +fn merge_within( + tx: &Connection, + survivor: ImageId, + copies: &[ImageId], +) -> Result { + check_still_duplicates(tx, survivor, copies)?; + + // Versions first: every judgement hangs off one. The survivor gets one + // if it has none; a copy without one simply has nothing to give. + let sv = crate::rating::default_version_id(tx, survivor)?; + let judged = |version: i64| -> Result { + Ok(tx.query_row( + "SELECT id, rating, flag, label FROM versions WHERE id = ?1", + [version], + |r| Ok((r.get(0)?, r.get(1)?, r.get(2)?, r.get(3)?)), + )?) + }; + let ours = judged(sv)?; + let mut theirs: Vec = Vec::new(); + for copy in copies { + let v: Option = tx + .query_row( + "SELECT id FROM versions WHERE image_id = ?1 + ORDER BY is_default DESC, id ASC LIMIT 1", + [copy.0 as i64], + |r| r.get(0), + ) + .optional()?; + if let Some(v) = v { + theirs.push(judged(v)?); + } + } + + let mut out = Outcome { + rating: (ours.1 as u8, ours.1 as u8), + flag: (ours.2, ours.2), + label: (ours.3, ours.3), + ..Default::default() + }; + + // The rating: the highest anyone gave it. + let best = theirs + .iter() + .map(|t| t.1) + .chain([ours.1]) + .max() + .unwrap_or(0); + out.rating.1 = best as u8; + + // Flag and label: the value the copies that carry one agree on. Where + // they disagree, the survivor keeps its own and the review says so. + let flags: std::collections::BTreeSet = theirs + .iter() + .map(|t| t.2) + .chain([ours.2]) + .filter(|f| *f != 0) + .collect(); + match flags.len() { + 0 => {} + 1 => out.flag.1 = *flags.iter().next().unwrap_or(&ours.2), + _ => out.flag_conflict = true, + } + let labels: std::collections::BTreeSet = + theirs.iter().filter_map(|t| t.3).chain(ours.3).collect(); + match labels.len() { + 0 => {} + 1 => out.label.1 = labels.iter().next().copied(), + _ => out.label_conflict = true, + } + tx.execute( + "UPDATE versions SET rating = ?2, flag = ?3, label = ?4 WHERE id = ?1", + rusqlite::params![sv, out.rating.1 as i64, out.flag.1, out.label.1], + )?; + + // Keywords: the union, onto the survivor's default version. + { + let mut add = tx.prepare( + "INSERT OR IGNORE INTO keywords (version_id, keyword) + SELECT ?1, keyword FROM keywords WHERE version_id = ?2", + )?; + for t in &theirs { + out.keywords_added += add.execute(rusqlite::params![sv, t.0])?; + } + } + + // Collections: the survivor keeps its own place in each; where it was + // in none, it takes the first copy's place, and the collection's + // revision moves so the change syncs like any other edit. + { + let mut memberships = tx.prepare( + "SELECT collection_id, position, added FROM collection_members + WHERE image_id = ?1 ORDER BY collection_id", + )?; + let mut join = tx.prepare( + "INSERT INTO collection_members (collection_id, image_id, position, added) + VALUES (?1, ?2, ?3, ?4) + ON CONFLICT(collection_id, image_id) DO NOTHING", + )?; + let mut touch = tx.prepare( + "UPDATE collections SET revision = revision + 1, modified = ?2 WHERE id = ?1", + )?; + let now = crate::faces::now_secs(); + for copy in copies { + let rows: Vec<(i64, Option, i64)> = memberships + .query_map([copy.0 as i64], |r| Ok((r.get(0)?, r.get(1)?, r.get(2)?)))? + .collect::>()?; + for (collection, position, added) in rows { + if join.execute(rusqlite::params![ + collection, + survivor.0 as i64, + position, + added + ])? > 0 + { + touch.execute(rusqlite::params![collection, now])?; + out.collections_joined += 1; + } + } + } + } + + // Faces, and the names on them. + for copy in copies { + let carried = crate::faces::carry_onto_copy(tx, *copy, survivor)?; + out.faces.moved += carried.moved; + out.faces.named += carried.named; + out.faces.rejections += carried.rejections; + out.faces.conflicts += carried.conflicts; + out.faces.unmatched_named += carried.unmatched_named; + } + + Ok(out) +} + +/// Refuse a plan the catalog no longer agrees with. +fn check_still_duplicates( + tx: &Connection, + survivor: ImageId, + copies: &[ImageId], +) -> Result<(), CatalogError> { + type Key = (i64, Option, Option, Option); + let key = |image: ImageId| -> Result, CatalogError> { + Ok(tx + .query_row( + "SELECT root_id, camera, captured_at, file_size, trashed_at, shadowed_by + FROM images WHERE id = ?1", + [image.0 as i64], + |r| { + let trashed: Option = r.get(4)?; + let shadowed: Option = r.get(5)?; + Ok(( + (r.get(0)?, r.get(1)?, r.get(2)?, r.get(3)?), + trashed.is_none() && shadowed.is_none(), + )) + }, + ) + .optional()?) + }; + let Some((ours, true)) = key(survivor)? else { + return Err(CatalogError::StaleDuplicate(format!( + "image {} is gone or already in the trash", + survivor.0 + ))); + }; + if ours.1.is_none() || ours.2.is_none() || ours.3.is_none() { + return Err(CatalogError::StaleDuplicate(format!( + "image {} has no camera, capture time or size", + survivor.0 + ))); + } + let mut seen = HashMap::new(); + for copy in copies { + if *copy == survivor || seen.insert(copy.0, ()).is_some() { + return Err(CatalogError::StaleDuplicate(format!( + "image {} is named twice", + copy.0 + ))); + } + match key(*copy)? { + Some((theirs, true)) if theirs == ours => {} + _ => { + return Err(CatalogError::StaleDuplicate(format!( + "image {} no longer matches image {}", + copy.0, survivor.0 + ))) + } + } + } + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::Catalog; + + const SHOT: i64 = 1_687_602_063; + + /// The reference example: one Canon frame in four folders, two of them + /// renamed, and an unrelated frame beside it. + fn library() -> Catalog { + let cat = Catalog::in_memory().unwrap(); + let c = cat.connection(); + c.execute( + "INSERT INTO roots(id, kind, label) VALUES (1, 'remote', 'PhotosRaw')", + [], + ) + .unwrap(); + let rows: &[(i64, &str, i64, i64)] = &[ + (1, "PhotosRaw/2023/bck/_MG_4623.CR2", SHOT, 100), + (2, "PhotosRaw/2023/2023-06-24/_MG_4623.CR2", SHOT, 100), + ( + 3, + "PhotosRaw/Darktable/20230628_no_name/20230628_0642.CR2", + SHOT, + 100, + ), + (4, "PhotosRaw/alps trip/Raw/20230628_0642.CR2", SHOT, 101), + // Same instant, different size: a different file. + (5, "PhotosRaw/2023/2023-06-24/_MG_4624.CR2", SHOT, 100), + ]; + for (id, path, at, added) in rows { + let size = if *id == 5 { 27_000_000 } else { 27_741_184 }; + c.execute( + "INSERT INTO images(id, root_id, source_ref, captured_at, camera, file_size, + file_mtime, added_at) + VALUES (?1, 1, ?2, ?3, 'Canon EOS 6D', ?4, 7, ?5)", + rusqlite::params![id, path, at, size, added], + ) + .unwrap(); + c.execute( + "INSERT INTO remote(image_id, file_id) VALUES (?1, ?2)", + rusqlite::params![id, 1000 + id], + ) + .unwrap(); + } + crate::rating::ensure_default_versions(c).unwrap(); + cat + } + + fn trash_all(cat: &Catalog, survivor: u64, copies: &[u64]) -> Outcome { + let moved: Vec<(ImageId, String)> = copies + .iter() + .map(|id| { + ( + ImageId(*id), + crate::trash::trash_path("PhotosRaw", ImageId(*id), "x.CR2"), + ) + }) + .collect(); + consolidate(cat.connection(), ImageId(survivor), &moved, 5_000).unwrap() + } + + #[test] + fn the_copies_of_one_frame_are_one_group_and_the_other_frame_is_not() { + let cat = library(); + let groups = candidates(cat.connection()).unwrap(); + assert_eq!(groups.len(), 1); + let ids: Vec = groups[0].copies.iter().map(|c| c.image.0).collect(); + assert_eq!(ids, vec![1, 2, 3, 4]); + assert_eq!(groups[0].copies[0].file_id, Some(1001)); + assert_eq!(count(cat.connection()).unwrap(), (1, 3)); + } + + #[test] + fn trashed_shadowed_and_undated_rows_are_never_candidates() { + let cat = library(); + let c = cat.connection(); + c.execute("UPDATE images SET trashed_at = 1 WHERE id = 1", []) + .unwrap(); + c.execute("UPDATE images SET shadowed_by = 5 WHERE id = 2", []) + .unwrap(); + c.execute("UPDATE images SET captured_at = NULL WHERE id = 3", []) + .unwrap(); + // One copy left: not a group. + assert!(candidates(c).unwrap().is_empty()); + assert_eq!(count(c).unwrap(), (0, 0)); + } + + #[test] + fn another_camera_at_the_same_instant_is_not_a_copy() { + let cat = library(); + let c = cat.connection(); + c.execute("UPDATE images SET camera = 'Canon EOS R5' WHERE id = 4", []) + .unwrap(); + let groups = candidates(c).unwrap(); + assert_eq!(groups[0].copies.len(), 3); + } + + #[test] + fn backup_folders_are_recognised_by_their_words() { + assert!(is_backup_path("PhotosRaw/2023/bck/_MG_4623.CR2")); + assert!(is_backup_path("Backup 2019/IMG_0001.CR2")); + assert!(is_backup_path("a/old_raws/IMG_0001.CR2")); + assert!(is_backup_path("Copy of Alps/IMG_0001.CR2")); + assert!(!is_backup_path("PhotosRaw/2023/2023-06-24/_MG_4623.CR2")); + assert!(!is_backup_path("Oldenburg/Bold/IMG_0001.CR2")); + // The file's own name is not a folder. + assert!(!is_backup_path("2023/copy.CR2")); + } + + #[test] + fn camera_names_are_told_from_renamed_files() { + for name in [ + "_MG_4623.CR2", + "IMG_0001.CR3", + "DSC_0042.NEF", + "_DSC0042.ARW", + "DSCF1234.RAF", + "DSC01234.ARW", + "IMGP0001.PEF", + "P1010001.ORF", + "DJI_0001.DNG", + ] { + assert!(camera_named(name), "{name}"); + } + for name in [ + "20230628_0642.CR2", + "IMG_4623-2.CR2", + "alps-12.CR2", + "holiday.CR2", + "img_0001.cr2", + ] { + assert!(!camera_named(name), "{name}"); + } + } + + #[test] + fn the_survivor_is_the_camera_named_copy_outside_a_backup_folder() { + let cat = library(); + let groups = candidates(cat.connection()).unwrap(); + let at = survivor(&groups[0].copies); + assert_eq!( + groups[0].copies[at].source_ref, + "PhotosRaw/2023/2023-06-24/_MG_4623.CR2" + ); + } + + #[test] + fn with_names_equal_the_oldest_then_the_lowest_id_survives() { + let cat = library(); + let c = cat.connection(); + let groups = candidates(c).unwrap(); + // Only the two renamed copies: neither is camera-named nor backed up. + let renamed: Vec = groups[0].copies[2..].to_vec(); + assert_eq!(renamed[survivor(&renamed)].image, ImageId(3), "oldest"); + let mut tied = renamed.clone(); + tied[1].added_at = tied[0].added_at; + assert_eq!(tied[survivor(&tied)].image, ImageId(3), "lowest id"); + } + + #[test] + fn a_backup_folder_loses_even_to_a_renamed_copy() { + let cat = library(); + let groups = candidates(cat.connection()).unwrap(); + let pair = vec![groups[0].copies[0].clone(), groups[0].copies[3].clone()]; + assert_eq!(pair[survivor(&pair)].image, ImageId(4)); + } + + #[test] + fn probes_are_kept_and_forgotten_when_the_file_changes() { + let cat = library(); + let c = cat.connection(); + record_probes( + c, + &[Probe { + image: ImageId(1), + file_size: 27_741_184, + file_mtime: Some(7), + digest: "abc".into(), + }], + 1, + ) + .unwrap(); + let groups = candidates(c).unwrap(); + assert_eq!(groups[0].copies[0].probe.as_deref(), Some("abc")); + assert_eq!(groups[0].copies[1].probe, None); + + // Touched since: the probe describes a file that no longer exists. + c.execute("UPDATE images SET file_mtime = 8 WHERE id = 1", []) + .unwrap(); + assert_eq!(candidates(c).unwrap()[0].copies[0].probe, None); + } + + #[test] + fn the_highest_rating_and_every_keyword_reach_the_survivor() { + let cat = library(); + let c = cat.connection(); + c.execute("UPDATE versions SET rating = 2 WHERE image_id = 2", []) + .unwrap(); + c.execute("UPDATE versions SET rating = 4 WHERE image_id = 1", []) + .unwrap(); + crate::keywords::assign(c, &[ImageId(1)], "Alps").unwrap(); + crate::keywords::assign(c, &[ImageId(2)], "Summer").unwrap(); + crate::keywords::assign(c, &[ImageId(3)], "Alps").unwrap(); + + let out = trash_all(&cat, 2, &[1, 3, 4]); + assert_eq!(out.rating, (2, 4)); + assert_eq!(out.keywords_added, 1); + assert_eq!( + crate::keywords::for_image(c, ImageId(2)).unwrap(), + vec!["Alps".to_string(), "Summer".to_string()] + ); + assert_eq!(crate::rating::judgement(c, ImageId(2)).unwrap().rating, 4); + } + + #[test] + fn a_flag_the_copies_agree_on_is_taken_and_a_disagreement_is_reported() { + let cat = library(); + let c = cat.connection(); + // Only one copy is flagged: nothing to disagree with. + c.execute("UPDATE versions SET flag = 1 WHERE image_id = 3", []) + .unwrap(); + let out = preview(c, ImageId(2), &[ImageId(1), ImageId(3)]).unwrap(); + assert_eq!(out.flag, (0, 1)); + assert!(!out.flag_conflict); + + // A pick and a reject: the survivor keeps its own, and says so. + c.execute("UPDATE versions SET flag = 2 WHERE image_id = 1", []) + .unwrap(); + let out = trash_all(&cat, 2, &[1, 3]); + assert!(out.flag_conflict); + assert_eq!(out.flag, (0, 0)); + + // Labels: two copies agreeing on red is agreement. + let cat = library(); + let c = cat.connection(); + c.execute("UPDATE versions SET label = 1 WHERE image_id IN (1, 3)", []) + .unwrap(); + let out = trash_all(&cat, 2, &[1, 3]); + assert!(!out.label_conflict); + assert_eq!(out.label, (None, Some(1))); + } + + #[test] + fn a_preview_writes_nothing() { + let cat = library(); + let c = cat.connection(); + c.execute("UPDATE versions SET rating = 5 WHERE image_id = 1", []) + .unwrap(); + let out = preview(c, ImageId(2), &[ImageId(1)]).unwrap(); + assert_eq!(out.rating, (0, 5)); + assert!(out.merges_anything()); + assert_eq!(out.trashed, 0); + assert_eq!(crate::rating::judgement(c, ImageId(2)).unwrap().rating, 0); + let trashed: i64 = c + .query_row( + "SELECT COUNT(*) FROM images WHERE trashed_at IS NOT NULL", + [], + |r| r.get(0), + ) + .unwrap(); + assert_eq!(trashed, 0); + } + + #[test] + fn collections_are_unioned_and_the_survivor_keeps_its_place() { + let cat = library(); + let c = cat.connection(); + let alps = + crate::collections::create(c, "Alps", None, crate::CollectionKind::Manual).unwrap(); + let best = + crate::collections::create(c, "Best", None, crate::CollectionKind::Manual).unwrap(); + // Alps: survivor at 0, copy 1 at 1. Best: only copy 3, third in line. + crate::collections::add_images(c, alps, &[ImageId(2), ImageId(1)]).unwrap(); + crate::collections::add_images(c, best, &[ImageId(5), ImageId(4), ImageId(3)]).unwrap(); + let rev = |id: dr_types::CollectionId| -> i64 { + c.query_row( + "SELECT revision FROM collections WHERE id = ?1", + [id.0 as i64], + |r| r.get(0), + ) + .unwrap() + }; + let (alps_rev, best_rev) = (rev(alps), rev(best)); + + let out = trash_all(&cat, 2, &[1, 3, 4]); + assert_eq!(out.collections_joined, 1, "Best only; Alps already had it"); + + let position = |coll: dr_types::CollectionId| -> Option { + c.query_row( + "SELECT position FROM collection_members + WHERE collection_id = ?1 AND image_id = 2", + [coll.0 as i64], + |r| r.get(0), + ) + .unwrap() + }; + assert_eq!(position(alps), Some(0), "its own place kept"); + // The first copy that was in Best is 3 (ids in order), at position 2. + assert_eq!(position(best), Some(2), "the copy's place taken"); + assert_eq!(rev(alps), alps_rev, "no change, no revision"); + assert_eq!(rev(best), best_rev + 1); + } + + #[test] + fn the_copies_land_in_the_trash_with_their_origins() { + let cat = library(); + let c = cat.connection(); + let out = trash_all(&cat, 2, &[1, 3, 4]); + assert_eq!(out.trashed, 3); + let listed = crate::trash::list(c, 10).unwrap(); + let mut from: Vec = listed.iter().map(|t| t.trashed_from.clone()).collect(); + from.sort(); + assert_eq!( + from, + vec![ + "PhotosRaw/2023/bck/_MG_4623.CR2".to_string(), + "PhotosRaw/Darktable/20230628_no_name/20230628_0642.CR2".to_string(), + "PhotosRaw/alps trip/Raw/20230628_0642.CR2".to_string(), + ] + ); + assert!(candidates(c).unwrap().is_empty()); + } + + #[test] + fn a_failure_part_way_leaves_the_group_untouched() { + let cat = library(); + let c = cat.connection(); + c.execute("UPDATE versions SET rating = 5 WHERE image_id = 1", []) + .unwrap(); + crate::keywords::assign(c, &[ImageId(1)], "Alps").unwrap(); + // The rating and keyword are merged from copy 1 first, then the + // trash write for copy 4 fails, as a full disk would fail it. + c.execute_batch( + "CREATE TRIGGER fail_on_4 BEFORE UPDATE OF source_ref ON images + WHEN NEW.id = 4 BEGIN SELECT RAISE(ABORT, 'disk full'); END;", + ) + .unwrap(); + let moved = vec![ + (ImageId(1), "PhotosRaw/.darkroom-trash/1-a.CR2".to_string()), + (ImageId(4), "PhotosRaw/.darkroom-trash/4-a.CR2".to_string()), + ]; + assert!(consolidate(c, ImageId(2), &moved, 5_000).is_err()); + + assert_eq!(crate::rating::judgement(c, ImageId(2)).unwrap().rating, 0); + assert!(crate::keywords::for_image(c, ImageId(2)) + .unwrap() + .is_empty()); + let trashed: i64 = c + .query_row( + "SELECT COUNT(*) FROM images WHERE trashed_at IS NOT NULL", + [], + |r| r.get(0), + ) + .unwrap(); + assert_eq!(trashed, 0, "copy 1 is not half-trashed"); + } + + #[test] + fn a_stale_plan_is_refused() { + let cat = library(); + let c = cat.connection(); + // 5 is a different size: not a copy of 2. + let moved = vec![(ImageId(5), "t".to_string())]; + assert!(matches!( + consolidate(c, ImageId(2), &moved, 1), + Err(CatalogError::StaleDuplicate(_)) + )); + // Already trashed since the review. + c.execute("UPDATE images SET trashed_at = 1 WHERE id = 1", []) + .unwrap(); + let moved = vec![(ImageId(1), "t".to_string())]; + assert!(consolidate(c, ImageId(2), &moved, 1).is_err()); + // The survivor named as its own copy. + let moved = vec![(ImageId(2), "t".to_string())]; + assert!(consolidate(c, ImageId(2), &moved, 1).is_err()); + } + + fn face(c: &Connection, image: i64, x: f64) -> i64 { + c.execute( + "INSERT INTO faces (image_id, x, y, w, h, landmarks, detector_confidence, + embedding, crop_px, model_id, detected_at) + VALUES (?1, ?2, 0.1, 0.2, 0.3, x'00', 0.9, x'00', 100, 'w600k_mbf', 0)", + rusqlite::params![image, x], + ) + .unwrap(); + c.last_insert_rowid() + } + + fn marker(c: &Connection, image: i64, found: i64) { + c.execute( + "INSERT INTO face_index (image_id, model_id, indexed_at, faces_found, source_edge) + VALUES (?1, 'w600k_mbf', 0, ?2, 1024)", + rusqlite::params![image, found], + ) + .unwrap(); + } + + #[test] + fn faces_move_onto_a_survivor_that_has_none() { + let cat = library(); + let c = cat.connection(); + let f = face(c, 1, 0.1); + marker(c, 1, 1); + let anna = crate::faces::create_person(c, "Anna").unwrap(); + crate::faces::confirm(c, crate::FaceId(f as u64), anna).unwrap(); + + let out = trash_all(&cat, 2, &[1]); + assert_eq!(out.faces.moved, 1); + let on: i64 = c + .query_row("SELECT image_id FROM faces WHERE id = ?1", [f], |r| { + r.get(0) + }) + .unwrap(); + assert_eq!(on, 2); + assert_eq!( + crate::faces::images_for_person(c, anna, false).unwrap(), + vec![ImageId(2)] + ); + let markers: i64 = c + .query_row( + "SELECT COUNT(*) FROM face_index WHERE image_id = 2", + [], + |r| r.get(0), + ) + .unwrap(); + assert_eq!(markers, 1, "the run marker follows the faces"); + } + + #[test] + fn a_name_is_carried_onto_the_survivors_own_face_without_duplicating_it() { + let cat = library(); + let c = cat.connection(); + let theirs = face(c, 1, 0.1); + let elsewhere = face(c, 1, 0.6); + let ours = face(c, 2, 0.1); + let named_ours = face(c, 2, 0.6); + let anna = crate::faces::create_person(c, "Anna").unwrap(); + let ben = crate::faces::create_person(c, "Ben").unwrap(); + crate::faces::confirm(c, crate::FaceId(theirs as u64), anna).unwrap(); + crate::faces::reject(c, crate::FaceId(theirs as u64), ben).unwrap(); + // The second face: named differently on each copy. + crate::faces::confirm(c, crate::FaceId(elsewhere as u64), anna).unwrap(); + crate::faces::confirm(c, crate::FaceId(named_ours as u64), ben).unwrap(); + + let out = trash_all(&cat, 2, &[1]); + assert_eq!(out.faces.moved, 0); + assert_eq!(out.faces.named, 1); + assert_eq!(out.faces.rejections, 1); + assert_eq!(out.faces.conflicts, 1); + + let faces_on_2: i64 = c + .query_row("SELECT COUNT(*) FROM faces WHERE image_id = 2", [], |r| { + r.get(0) + }) + .unwrap(); + assert_eq!(faces_on_2, 2, "no duplicate boxes"); + let who: i64 = c + .query_row( + "SELECT person_id FROM face_person WHERE face_id = ?1", + [ours], + |r| r.get(0), + ) + .unwrap(); + assert_eq!(who, anna.0 as i64); + let who: i64 = c + .query_row( + "SELECT person_id FROM face_person WHERE face_id = ?1", + [named_ours], + |r| r.get(0), + ) + .unwrap(); + assert_eq!(who, ben.0 as i64, "the survivor's own name wins"); + } +} diff --git a/core/dr-catalog/src/error.rs b/core/dr-catalog/src/error.rs index e88ec7f..4ea09f4 100644 --- a/core/dr-catalog/src/error.rs +++ b/core/dr-catalog/src/error.rs @@ -96,6 +96,13 @@ pub enum CatalogError { #[error("io: {0}")] Io(String), + + /// TRACES: FR-CAT-11a + /// A duplicate group planned earlier no longer holds: a copy was trashed, + /// rescanned or changed since the review was drawn. The group is left + /// untouched rather than consolidated on a stale plan. + #[error("no longer a duplicate: {0}")] + StaleDuplicate(String), } impl From for CatalogError { diff --git a/core/dr-catalog/src/faces.rs b/core/dr-catalog/src/faces.rs index b22dc52..df107e6 100644 --- a/core/dr-catalog/src/faces.rs +++ b/core/dr-catalog/src/faces.rs @@ -1599,6 +1599,170 @@ fn iou(a: (f32, f32, f32, f32), b: (f32, f32, f32, f32)) -> f32 { } } +/// TRACES: FR-CAT-11a | FR-CULL-10 +/// What [`carry_onto_copy`] did with one byte-identical copy's faces. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)] +pub struct FaceCarry { + /// Faces moved onto the survivor outright, because it had none. + pub moved: usize, + /// Names and suggestions put on a survivor's face that lacked one. + pub named: usize, + /// Rejections added to a survivor's face. + pub rejections: usize, + /// Faces named one person on the copy and another on the survivor. The + /// survivor's name is kept; the copy keeps its own, in the trash. + pub conflicts: usize, + /// Named faces on the copy that match nothing on the survivor. Left with + /// the copy rather than mixed into another pipeline's faces. + pub unmatched_named: usize, +} + +/// TRACES: FR-CAT-11a | FR-CULL-10 +/// Bring what one copy of a photograph knows about its faces onto another +/// copy of the same bytes, inside the caller's transaction. +/// +/// Faces are per image, so two copies of one file indexed separately hold +/// two sets of the same boxes, and a name confirmed on one is invisible on +/// the other. The rule is the one [`record_detections_within`] keeps: an +/// image holds one pipeline's faces at a time, and the user's judgements +/// are what must survive. +/// +/// - **The survivor has no faces at all.** The copy's faces and its run +/// markers move over wholesale — nothing is duplicated, and the survivor +/// is spared a detection pass it would only repeat. Its own markers go +/// first, because a marker saying "examined, nothing found" over an image +/// that now holds faces is the V12 state. +/// - **The survivor has faces.** Each of the copy's faces is paired with the +/// survivor face its box overlaps most (IoU above one half — the bytes are +/// the same, so the boxes coincide). A name or suggestion is carried onto +/// a survivor face that has none, a confirmation outranks a suggestion, +/// and two different confirmed names are a conflict the survivor wins. +/// Rejections are unioned. The copy's faces stay where they are, with the +/// copy. +pub fn carry_onto_copy( + tx: &Connection, + from: ImageId, + to: ImageId, +) -> Result { + let mut out = FaceCarry::default(); + let boxes = |image: ImageId| -> Result, CatalogError> { + let mut q = tx.prepare( + "SELECT f.id, f.x, f.y, f.w, f.h, fp.person_id, fp.probability, fp.confirmed + FROM faces f + LEFT JOIN face_person fp ON fp.face_id = f.id + WHERE f.image_id = ?1 + ORDER BY f.id", + )?; + let rows = q.query_map([image.0 as i64], |r| { + let person: Option = r.get(5)?; + Ok(CopyFace { + id: r.get(0)?, + rect: ( + r.get::<_, f64>(1)? as f32, + r.get::<_, f64>(2)? as f32, + r.get::<_, f64>(3)? as f32, + r.get::<_, f64>(4)? as f32, + ), + assignment: match person { + Some(p) => Some((p, r.get::<_, f64>(6)?, r.get::<_, i64>(7)? != 0)), + None => None, + }, + }) + })?; + Ok(rows.collect::, _>>()?) + }; + + let theirs = boxes(from)?; + if theirs.is_empty() { + return Ok(out); + } + let ours = boxes(to)?; + + if ours.is_empty() { + tx.execute("DELETE FROM face_index WHERE image_id = ?1", [to.0 as i64])?; + tx.execute( + "UPDATE face_index SET image_id = ?2 WHERE image_id = ?1", + rusqlite::params![from.0 as i64, to.0 as i64], + )?; + out.moved = tx.execute( + "UPDATE faces SET image_id = ?2 WHERE image_id = ?1", + rusqlite::params![from.0 as i64, to.0 as i64], + )?; + return Ok(out); + } + + let mut taken = vec![false; ours.len()]; + for face in &theirs { + let best = ours + .iter() + .enumerate() + .filter(|(i, _)| !taken[*i]) + .map(|(i, o)| (i, iou(face.rect, o.rect))) + .filter(|(_, overlap)| *overlap > 0.5) + .max_by(|a, b| a.1.total_cmp(&b.1)); + let Some((at, _)) = best else { + if face.assignment.is_some_and(|(_, _, confirmed)| confirmed) { + out.unmatched_named += 1; + } + continue; + }; + taken[at] = true; + let target = &ours[at]; + + out.rejections += tx.execute( + "INSERT OR IGNORE INTO face_person_rejected (face_id, person_id) + SELECT ?2, person_id FROM face_person_rejected WHERE face_id = ?1", + rusqlite::params![face.id, target.id], + )?; + + let Some((person, probability, confirmed)) = face.assignment else { + continue; + }; + let carry = match target.assignment { + None => true, + // The same person: only a confirmation upgrades a suggestion. + Some((p, _, theirs_confirmed)) if p == person => confirmed && !theirs_confirmed, + // Another person, only suggested there: the user's word wins. + Some((_, _, false)) => confirmed, + // Another person, confirmed there: the survivor keeps its name. + Some((_, _, true)) => { + if confirmed { + out.conflicts += 1; + } + false + } + }; + if carry { + tx.execute( + "INSERT INTO face_person (face_id, person_id, probability, confirmed) + VALUES (?1, ?2, ?3, ?4) + ON CONFLICT(face_id) DO UPDATE SET + person_id = excluded.person_id, + probability = excluded.probability, + confirmed = excluded.confirmed", + rusqlite::params![target.id, person, probability, confirmed], + )?; + // A name the user gave outranks a rejection of the same pair made + // on the survivor — the same order `confirm` applies. + if confirmed { + tx.execute( + "DELETE FROM face_person_rejected WHERE face_id = ?1 AND person_id = ?2", + rusqlite::params![target.id, person], + )?; + } + out.named += 1; + } + } + Ok(out) +} + +/// One face as [`carry_onto_copy`] pairs it. +struct CopyFace { + id: i64, + rect: (f32, f32, f32, f32), + assignment: Option<(i64, f64, bool)>, +} + pub(crate) fn now_secs() -> i64 { std::time::SystemTime::now() .duration_since(std::time::UNIX_EPOCH) diff --git a/core/dr-catalog/src/lib.rs b/core/dr-catalog/src/lib.rs index fd6f93a..cd939a0 100644 --- a/core/dr-catalog/src/lib.rs +++ b/core/dr-catalog/src/lib.rs @@ -40,6 +40,7 @@ pub mod bursts; pub mod cache; pub mod collections; pub mod dedup; +pub mod duplicates; pub mod error; pub mod face_shard; pub mod faces; diff --git a/core/dr-catalog/src/trash.rs b/core/dr-catalog/src/trash.rs index 63c5733..114e540 100644 --- a/core/dr-catalog/src/trash.rs +++ b/core/dr-catalog/src/trash.rs @@ -129,28 +129,40 @@ pub fn record_trashed( return Ok(0); } let tx = conn.unchecked_transaction()?; - let mut n = 0; - - { - let mut stmt = tx.prepare( - "UPDATE images - SET trashed_from = CASE - WHEN trashed_at IS NULL THEN source_ref - ELSE trashed_from - END, - source_ref = ?2, - trashed_at = coalesce(trashed_at, ?3) - WHERE id = ?1", - )?; - for (image, path) in moved { - n += stmt.execute(rusqlite::params![image.0 as i64, path, now])?; - } - } - + let n = record_trashed_within(&tx, moved, now)?; tx.commit()?; Ok(n) } +/// [`record_trashed`] inside a transaction the caller owns. +/// +/// For a caller whose trash is one half of a larger write that must land +/// whole or not at all — consolidating duplicates (`crate::duplicates`) +/// merges a copy's judgements onto the survivor and trashes the copy in one +/// commit. `unchecked_transaction` cannot nest, so this is offered here +/// rather than wrapped from above. +pub fn record_trashed_within( + tx: &Connection, + moved: &[(ImageId, String)], + now: i64, +) -> Result { + let mut n = 0; + let mut stmt = tx.prepare( + "UPDATE images + SET trashed_from = CASE + WHEN trashed_at IS NULL THEN source_ref + ELSE trashed_from + END, + source_ref = ?2, + trashed_at = coalesce(trashed_at, ?3) + WHERE id = ?1", + )?; + for (image, path) in moved { + n += stmt.execute(rusqlite::params![image.0 as i64, path, now])?; + } + Ok(n) +} + /// Record that images have been moved back out of the trash. /// /// Call after the move succeeds, for the same reason as [`record_trashed`].