//! TRACES: FR-CAT-11a //! The same original, catalogued more than once — found, proved, and folded //! onto one copy. //! //! [`crate::dedup`] stops a card being imported twice. This is the other //! half of FR-CAT-11: a library that already holds the same RAW in several //! folders — a dated folder, a `bck/` beside it, a renamed export tree — //! because it was assembled by hand before DarkRoom saw it. On the reference //! library that is 1,836 moments held two to five times over. //! //! # Three steps, and only the last one writes //! //! **Candidates** ([`candidates`]) are one grouped query: same root, same //! camera, same capture instant, same byte count. Nothing is trashed on that //! evidence alone — two bodies of one model firing in the same second with //! files of one size is unlikely, not impossible. //! //! **Proof** is the caller's, because it reads bytes: a stored content hash //! where every copy has one, otherwise a digest of the first and last //! [`PROBE_WINDOW`] of each file, kept here ([`record_probes`]) keyed on the //! size and mtime it was taken at, so a second review costs nothing. //! //! **Consolidation** ([`consolidate`]) folds each copy's judgements onto the //! survivor and records the copies as trashed, in one transaction per //! group: a group is either wholly consolidated or untouched. The files are //! moved by the caller *before* this commits, as every trash is //! ([`crate::trash::record_trashed`] says why), and moved back if it fails. //! //! # What merges //! //! Collections are unioned, the survivor keeping its own position and //! taking a copy's where it had none. Keywords are unioned. The rating is //! the highest. A flag or a label is taken where the copies that carry one //! agree, and left as the survivor had it where they do not — reported as a //! conflict rather than decided silently. Faces follow //! [`crate::faces::carry_onto_copy`]. Develop edits live in sidecars, which //! this crate does not read; the caller compares them and does not consolidate //! a group whose edits differ (virtual copies, FR-CAT-12, are where two edits //! of one frame belong). use std::collections::HashMap; use rusqlite::{Connection, OptionalExtension}; use dr_types::ImageId; use crate::error::CatalogError; use crate::faces::FaceCarry; /// How much of each end of a file the sameness probe reads. /// /// A megabyte from the front holds the EXIF, the maker notes and the /// embedded previews; a megabyte from the back is sensor data. Two files /// already equal in size, camera and capture instant that also agree there /// are the same file for any purpose a library has — and proving it costs /// two megabytes rather than the whole 25 MB RAW. pub const PROBE_WINDOW: u64 = 1 << 20; /// Folder names that say "this is a spare": a survivor is never taken from /// under one while a copy elsewhere is not. /// /// Matched against the *words* of each folder name, case-insensitively, so /// `bck`, `Backup 2019`, `old_raws` and `Copy of Alps` all count and /// `Bold` and `Oldenburg` do not. pub const BACKUP_WORDS: &[&str] = &[ "bck", "bak", "backup", "backups", "copy", "copies", "old", "dup", "dups", "duplicate", "duplicates", "archive", "spare", ]; /// One catalogued copy in a duplicate group. #[derive(Debug, Clone, PartialEq, Eq)] pub struct Copy { pub image: ImageId, pub source_ref: String, pub file_size: u64, pub file_mtime: Option, pub added_at: i64, /// A full digest of the bytes, where an import or an earlier check /// computed one. pub content_hash: Option, /// The sameness probe stored for this copy, if it is still valid — taken /// at the size and mtime the catalog now records, over the current /// [`PROBE_WINDOW`]. pub probe: Option, /// `oc:fileid`, the thumbnail store's key. pub file_id: Option, } impl Copy { /// The file's own name, the last component of its key. pub fn file_name(&self) -> &str { file_name(&self.source_ref) } } /// Copies that look like one original: same root, camera, instant and size. #[derive(Debug, Clone, PartialEq, Eq)] pub struct Group { pub root_id: i64, pub camera: String, pub captured_at: i64, pub file_size: u64, /// By image id, so the order is stable between two reads. pub copies: Vec, } /// Which rows can be duplicates at all: dated, identified, sized, and /// present in the grid — not trashed, and not a JPEG a RAW beside it /// already stands for. const CANDIDATE: &str = "captured_at IS NOT NULL AND camera IS NOT NULL AND file_size IS NOT NULL \ AND trashed_at IS NULL AND shadowed_by IS NULL"; /// How many duplicate groups the library holds, and how many copies /// consolidating them all would move to the trash. /// /// Counted in SQL, for the settings line and the sidebar: 40 ms on the /// 24k-image reference library, and it builds no list. pub fn count(conn: &Connection) -> Result<(usize, usize), CatalogError> { let (groups, extra): (i64, i64) = conn.query_row( &format!( "SELECT COUNT(*), coalesce(SUM(n - 1), 0) FROM ( SELECT COUNT(*) AS n FROM images WHERE {CANDIDATE} GROUP BY root_id, camera, captured_at, file_size HAVING COUNT(*) > 1)" ), [], |r| Ok((r.get(0)?, r.get(1)?)), )?; Ok((groups as usize, extra as usize)) } /// Every duplicate group, with its copies, in one query. /// /// Grouped in SQL and joined back for the rows, so the planner walks /// `images_captured` once rather than once per group. Each copy carries the /// probe stored for it, where one is still valid. pub fn candidates(conn: &Connection) -> Result, CatalogError> { ensure_probe_table(conn)?; let mut stmt = conn.prepare(&format!( "WITH g AS ( SELECT root_id, camera, captured_at, file_size FROM images WHERE {CANDIDATE} GROUP BY root_id, camera, captured_at, file_size HAVING COUNT(*) > 1) SELECT i.id, i.root_id, i.camera, i.captured_at, i.file_size, i.source_ref, i.file_mtime, i.added_at, i.content_hash, r.file_id, p.digest FROM g JOIN images i ON i.root_id = g.root_id AND i.camera = g.camera AND i.captured_at = g.captured_at AND i.file_size = g.file_size LEFT JOIN remote r ON r.image_id = i.id LEFT JOIN dedup_probes p ON p.image_id = i.id AND p.file_size = i.file_size AND p.file_mtime IS i.file_mtime AND p.window = ?1 WHERE i.trashed_at IS NULL AND i.shadowed_by IS NULL ORDER BY i.captured_at, i.camera, i.file_size, i.root_id, i.id" ))?; let mut groups: Vec = Vec::new(); let mut rows = stmt.query([PROBE_WINDOW as i64])?; while let Some(r) = rows.next()? { let root_id: i64 = r.get(1)?; let camera: String = r.get(2)?; let captured_at: i64 = r.get(3)?; let file_size = r.get::<_, i64>(4)? as u64; let copy = Copy { image: ImageId(r.get::<_, i64>(0)? as u64), source_ref: r.get(5)?, file_size, file_mtime: r.get(6)?, added_at: r.get(7)?, content_hash: r.get(8)?, file_id: r.get::<_, Option>(9)?.map(|v| v as u64), probe: r.get(10)?, }; match groups.last_mut() { Some(g) if g.root_id == root_id && g.camera == camera && g.captured_at == captured_at && g.file_size == file_size => { g.copies.push(copy) } _ => groups.push(Group { root_id, camera, captured_at, file_size, copies: vec![copy], }), } } Ok(groups) } /// The last component of a stored source reference — see /// [`crate::dedup`] for why both separators. fn file_name(source_ref: &str) -> &str { source_ref.rsplit(['/', ':']).next().unwrap_or(source_ref) } /// Whether any folder on the way to this file is named like a spare copy. /// /// Folders only: the file's own name is the camera-naming rule's business. pub fn is_backup_path(source_ref: &str) -> bool { let mut parts: Vec<&str> = source_ref.split(['/', ':']).collect(); parts.pop(); parts.iter().any(|folder| { folder .split(|c: char| !c.is_ascii_alphanumeric()) .any(|word| BACKUP_WORDS.iter().any(|b| word.eq_ignore_ascii_case(b))) }) } /// Whether a file still carries the name its camera gave it. /// /// Every maker spells it the same way: a short prefix of capitals or /// underscores, then a four- or five-digit counter — `IMG_4623`, /// `_MG_4623`, `DSC_0042`, `_DSC0042`, `DSCF1234`, `DSC01234`, `IMGP0001`, /// `DJI_0001` — and Olympus and Panasonic a letter and seven digits, /// `P1010001`. A renamed copy (`20230628_0642`, `alps-12`, `IMG_4623-2`) /// does not fit. pub fn camera_named(name: &str) -> bool { let stem = match name.rsplit_once('.') { Some((stem, _)) => stem, None => name, }; let bytes = stem.as_bytes(); if !(7..=10).contains(&bytes.len()) { return false; } // Olympus / Panasonic: P, then the month or a model digit, then six. if bytes[0] == b'P' && bytes[1..].iter().all(|b| b.is_ascii_alphanumeric()) { let rest = &bytes[1..]; if rest.len() == 7 && rest[1..].iter().all(u8::is_ascii_digit) { return true; } } let digits = bytes .iter() .rev() .take_while(|b| b.is_ascii_digit()) .count(); let prefix = &bytes[..bytes.len() - digits]; (4..=5).contains(&digits) && (3..=5).contains(&prefix.len()) && !prefix[0].is_ascii_digit() && prefix .iter() .all(|b| b.is_ascii_uppercase() || b.is_ascii_digit() || *b == b'_') } /// Which copy of a group stays in the library, by index into `copies`. /// /// In order: not under a backup-looking folder; then still named the way /// the camera named it; then catalogued first; then the lowest id, so the /// answer never depends on anything but the rows. pub fn survivor(copies: &[Copy]) -> usize { copies .iter() .enumerate() .min_by_key(|(_, c)| { ( is_backup_path(&c.source_ref), !camera_named(c.file_name()), c.added_at, c.image.0, ) }) .map(|(i, _)| i) .unwrap_or(0) } // ── probes ─────────────────────────────────────────────────────────────── /// Where sameness probes are kept. /// /// Created on first use rather than by a migration, deliberately. A new /// schema version makes every older build refuse this catalog's snapshot at /// sync (`crate::sync::remote_is_mergeable`), so the tablet would stop /// merging until it was updated — a high price for a cache. An older build /// that meets this table ignores it, and a rebuilt catalog simply probes /// again. pub fn ensure_probe_table(conn: &Connection) -> Result<(), CatalogError> { conn.execute_batch( "CREATE TABLE IF NOT EXISTS dedup_probes ( image_id INTEGER PRIMARY KEY REFERENCES images(id) ON DELETE CASCADE, -- What the file was when it was read. A probe of a file that has -- since changed is not evidence about the file it is now. file_size INTEGER NOT NULL, file_mtime INTEGER, -- The bytes read from each end, so a change to PROBE_WINDOW -- invalidates rather than compares unlike with unlike. window INTEGER NOT NULL, digest TEXT NOT NULL, probed_at INTEGER NOT NULL );", )?; Ok(()) } /// One sameness probe, as the checking job produced it. #[derive(Debug, Clone, PartialEq, Eq)] pub struct Probe { pub image: ImageId, pub file_size: u64, pub file_mtime: Option, pub digest: String, } /// Keep probes for the next review, in one transaction. pub fn record_probes(conn: &Connection, probes: &[Probe], now: i64) -> Result<(), CatalogError> { if probes.is_empty() { return Ok(()); } ensure_probe_table(conn)?; let tx = conn.unchecked_transaction()?; { let mut stmt = tx.prepare( "INSERT INTO dedup_probes (image_id, file_size, file_mtime, window, digest, probed_at) VALUES (?1, ?2, ?3, ?4, ?5, ?6) ON CONFLICT(image_id) DO UPDATE SET file_size = excluded.file_size, file_mtime = excluded.file_mtime, window = excluded.window, digest = excluded.digest, probed_at = excluded.probed_at", )?; for p in probes { stmt.execute(rusqlite::params![ p.image.0 as i64, p.file_size as i64, p.file_mtime, PROBE_WINDOW as i64, p.digest, now ])?; } } tx.commit()?; Ok(()) } // ── consolidation ──────────────────────────────────────────────────────── /// What consolidating one group did, or would do. #[derive(Debug, Clone, PartialEq, Eq, Default)] pub struct Outcome { /// The survivor's rating before and after. pub rating: (u8, u8), /// The survivor's flag before and after (catalog codes). pub flag: (i64, i64), /// The survivor's label before and after (catalog codes, `None` for none). pub label: (Option, Option), /// Copies carried different flags; the survivor's was kept. pub flag_conflict: bool, /// Copies carried different labels; the survivor's was kept. pub label_conflict: bool, pub keywords_added: usize, /// Collections the survivor joined, having been in none of them. pub collections_joined: usize, pub faces: FaceCarry, /// Copies recorded as trashed. Zero for a preview. pub trashed: usize, } impl Outcome { /// Whether anything besides the trash would change on the survivor. pub fn merges_anything(&self) -> bool { self.rating.0 != self.rating.1 || self.flag.0 != self.flag.1 || self.label.0 != self.label.1 || self.keywords_added > 0 || self.collections_joined > 0 || self.faces.moved + self.faces.named + self.faces.rejections > 0 } } /// Fold `copies` onto `survivor` and record them as trashed at the paths /// they were moved to, in one transaction. /// /// Call **after** the files are in the trash, like /// [`crate::trash::record_trashed`]; if this fails, move them back. Every /// copy is re-checked against the survivor first — still present, still /// untrashed, still the same camera, instant and size — and a group that no /// longer holds is refused with [`CatalogError::StaleDuplicate`] and nothing /// written. pub fn consolidate( conn: &Connection, survivor: ImageId, copies: &[(ImageId, String)], now: i64, ) -> Result { let tx = conn.unchecked_transaction()?; let ids: Vec = copies.iter().map(|(id, _)| *id).collect(); let mut out = merge_within(&tx, survivor, &ids)?; out.trashed = crate::trash::record_trashed_within(&tx, copies, now)?; if out.trashed != copies.len() { return Err(CatalogError::StaleDuplicate(format!( "{} of {} copies could be recorded as trashed", out.trashed, copies.len() ))); } tx.commit()?; Ok(out) } /// What [`consolidate`] would do, without doing it. /// /// The same code in a transaction that is rolled back, so the review can /// never describe a merge other than the one the button performs. pub fn preview( conn: &Connection, survivor: ImageId, copies: &[ImageId], ) -> Result { let tx = conn.unchecked_transaction()?; let out = merge_within(&tx, survivor, copies)?; tx.rollback()?; Ok(out) } /// The judgement a default version carries: `(version id, rating, flag, label)`. type Judged = (i64, i64, i64, Option); fn merge_within( tx: &Connection, survivor: ImageId, copies: &[ImageId], ) -> Result { check_still_duplicates(tx, survivor, copies)?; // Versions first: every judgement hangs off one. The survivor gets one // if it has none; a copy without one simply has nothing to give. let sv = crate::rating::default_version_id(tx, survivor)?; let judged = |version: i64| -> Result { Ok(tx.query_row( "SELECT id, rating, flag, label FROM versions WHERE id = ?1", [version], |r| Ok((r.get(0)?, r.get(1)?, r.get(2)?, r.get(3)?)), )?) }; let ours = judged(sv)?; let mut theirs: Vec = Vec::new(); for copy in copies { let v: Option = tx .query_row( "SELECT id FROM versions WHERE image_id = ?1 ORDER BY is_default DESC, id ASC LIMIT 1", [copy.0 as i64], |r| r.get(0), ) .optional()?; if let Some(v) = v { theirs.push(judged(v)?); } } let mut out = Outcome { rating: (ours.1 as u8, ours.1 as u8), flag: (ours.2, ours.2), label: (ours.3, ours.3), ..Default::default() }; // The rating: the highest anyone gave it. let best = theirs .iter() .map(|t| t.1) .chain([ours.1]) .max() .unwrap_or(0); out.rating.1 = best as u8; // Flag and label: the value the copies that carry one agree on. Where // they disagree, the survivor keeps its own and the review says so. let flags: std::collections::BTreeSet = theirs .iter() .map(|t| t.2) .chain([ours.2]) .filter(|f| *f != 0) .collect(); match flags.len() { 0 => {} 1 => out.flag.1 = *flags.iter().next().unwrap_or(&ours.2), _ => out.flag_conflict = true, } let labels: std::collections::BTreeSet = theirs.iter().filter_map(|t| t.3).chain(ours.3).collect(); match labels.len() { 0 => {} 1 => out.label.1 = labels.iter().next().copied(), _ => out.label_conflict = true, } tx.execute( "UPDATE versions SET rating = ?2, flag = ?3, label = ?4 WHERE id = ?1", rusqlite::params![sv, out.rating.1 as i64, out.flag.1, out.label.1], )?; // Keywords: the union, onto the survivor's default version. { let mut add = tx.prepare( "INSERT OR IGNORE INTO keywords (version_id, keyword) SELECT ?1, keyword FROM keywords WHERE version_id = ?2", )?; for t in &theirs { out.keywords_added += add.execute(rusqlite::params![sv, t.0])?; } } // Collections: the survivor keeps its own place in each; where it was // in none, it takes the first copy's place, and the collection's // revision moves so the change syncs like any other edit. { let mut memberships = tx.prepare( "SELECT collection_id, position, added FROM collection_members WHERE image_id = ?1 ORDER BY collection_id", )?; let mut join = tx.prepare( "INSERT INTO collection_members (collection_id, image_id, position, added) VALUES (?1, ?2, ?3, ?4) ON CONFLICT(collection_id, image_id) DO NOTHING", )?; let mut touch = tx.prepare( "UPDATE collections SET revision = revision + 1, modified = ?2 WHERE id = ?1", )?; let now = crate::faces::now_secs(); for copy in copies { let rows: Vec<(i64, Option, i64)> = memberships .query_map([copy.0 as i64], |r| Ok((r.get(0)?, r.get(1)?, r.get(2)?)))? .collect::>()?; for (collection, position, added) in rows { if join.execute(rusqlite::params![ collection, survivor.0 as i64, position, added ])? > 0 { touch.execute(rusqlite::params![collection, now])?; out.collections_joined += 1; } } } } // Faces, and the names on them. for copy in copies { let carried = crate::faces::carry_onto_copy(tx, *copy, survivor)?; out.faces.moved += carried.moved; out.faces.named += carried.named; out.faces.rejections += carried.rejections; out.faces.conflicts += carried.conflicts; out.faces.unmatched_named += carried.unmatched_named; } Ok(out) } /// Refuse a plan the catalog no longer agrees with. fn check_still_duplicates( tx: &Connection, survivor: ImageId, copies: &[ImageId], ) -> Result<(), CatalogError> { type Key = (i64, Option, Option, Option); let key = |image: ImageId| -> Result, CatalogError> { Ok(tx .query_row( "SELECT root_id, camera, captured_at, file_size, trashed_at, shadowed_by FROM images WHERE id = ?1", [image.0 as i64], |r| { let trashed: Option = r.get(4)?; let shadowed: Option = r.get(5)?; Ok(( (r.get(0)?, r.get(1)?, r.get(2)?, r.get(3)?), trashed.is_none() && shadowed.is_none(), )) }, ) .optional()?) }; let Some((ours, true)) = key(survivor)? else { return Err(CatalogError::StaleDuplicate(format!( "image {} is gone or already in the trash", survivor.0 ))); }; if ours.1.is_none() || ours.2.is_none() || ours.3.is_none() { return Err(CatalogError::StaleDuplicate(format!( "image {} has no camera, capture time or size", survivor.0 ))); } let mut seen = HashMap::new(); for copy in copies { if *copy == survivor || seen.insert(copy.0, ()).is_some() { return Err(CatalogError::StaleDuplicate(format!( "image {} is named twice", copy.0 ))); } match key(*copy)? { Some((theirs, true)) if theirs == ours => {} _ => { return Err(CatalogError::StaleDuplicate(format!( "image {} no longer matches image {}", copy.0, survivor.0 ))) } } } Ok(()) } #[cfg(test)] mod tests { use super::*; use crate::Catalog; const SHOT: i64 = 1_687_602_063; /// The reference example: one Canon frame in four folders, two of them /// renamed, and an unrelated frame beside it. fn library() -> Catalog { let cat = Catalog::in_memory().unwrap(); let c = cat.connection(); c.execute( "INSERT INTO roots(id, kind, label) VALUES (1, 'remote', 'PhotosRaw')", [], ) .unwrap(); let rows: &[(i64, &str, i64, i64)] = &[ (1, "PhotosRaw/2023/bck/_MG_4623.CR2", SHOT, 100), (2, "PhotosRaw/2023/2023-06-24/_MG_4623.CR2", SHOT, 100), ( 3, "PhotosRaw/Darktable/20230628_no_name/20230628_0642.CR2", SHOT, 100, ), (4, "PhotosRaw/alps trip/Raw/20230628_0642.CR2", SHOT, 101), // Same instant, different size: a different file. (5, "PhotosRaw/2023/2023-06-24/_MG_4624.CR2", SHOT, 100), ]; for (id, path, at, added) in rows { let size = if *id == 5 { 27_000_000 } else { 27_741_184 }; c.execute( "INSERT INTO images(id, root_id, source_ref, captured_at, camera, file_size, file_mtime, added_at) VALUES (?1, 1, ?2, ?3, 'Canon EOS 6D', ?4, 7, ?5)", rusqlite::params![id, path, at, size, added], ) .unwrap(); c.execute( "INSERT INTO remote(image_id, file_id) VALUES (?1, ?2)", rusqlite::params![id, 1000 + id], ) .unwrap(); } crate::rating::ensure_default_versions(c).unwrap(); cat } fn trash_all(cat: &Catalog, survivor: u64, copies: &[u64]) -> Outcome { let moved: Vec<(ImageId, String)> = copies .iter() .map(|id| { ( ImageId(*id), crate::trash::trash_path("PhotosRaw", ImageId(*id), "x.CR2"), ) }) .collect(); consolidate(cat.connection(), ImageId(survivor), &moved, 5_000).unwrap() } #[test] fn the_copies_of_one_frame_are_one_group_and_the_other_frame_is_not() { let cat = library(); let groups = candidates(cat.connection()).unwrap(); assert_eq!(groups.len(), 1); let ids: Vec = groups[0].copies.iter().map(|c| c.image.0).collect(); assert_eq!(ids, vec![1, 2, 3, 4]); assert_eq!(groups[0].copies[0].file_id, Some(1001)); assert_eq!(count(cat.connection()).unwrap(), (1, 3)); } #[test] fn trashed_shadowed_and_undated_rows_are_never_candidates() { let cat = library(); let c = cat.connection(); c.execute("UPDATE images SET trashed_at = 1 WHERE id = 1", []) .unwrap(); c.execute("UPDATE images SET shadowed_by = 5 WHERE id = 2", []) .unwrap(); c.execute("UPDATE images SET captured_at = NULL WHERE id = 3", []) .unwrap(); // One copy left: not a group. assert!(candidates(c).unwrap().is_empty()); assert_eq!(count(c).unwrap(), (0, 0)); } #[test] fn another_camera_at_the_same_instant_is_not_a_copy() { let cat = library(); let c = cat.connection(); c.execute("UPDATE images SET camera = 'Canon EOS R5' WHERE id = 4", []) .unwrap(); let groups = candidates(c).unwrap(); assert_eq!(groups[0].copies.len(), 3); } #[test] fn backup_folders_are_recognised_by_their_words() { assert!(is_backup_path("PhotosRaw/2023/bck/_MG_4623.CR2")); assert!(is_backup_path("Backup 2019/IMG_0001.CR2")); assert!(is_backup_path("a/old_raws/IMG_0001.CR2")); assert!(is_backup_path("Copy of Alps/IMG_0001.CR2")); assert!(!is_backup_path("PhotosRaw/2023/2023-06-24/_MG_4623.CR2")); assert!(!is_backup_path("Oldenburg/Bold/IMG_0001.CR2")); // The file's own name is not a folder. assert!(!is_backup_path("2023/copy.CR2")); } #[test] fn camera_names_are_told_from_renamed_files() { for name in [ "_MG_4623.CR2", "IMG_0001.CR3", "DSC_0042.NEF", "_DSC0042.ARW", "DSCF1234.RAF", "DSC01234.ARW", "IMGP0001.PEF", "P1010001.ORF", "DJI_0001.DNG", ] { assert!(camera_named(name), "{name}"); } for name in [ "20230628_0642.CR2", "IMG_4623-2.CR2", "alps-12.CR2", "holiday.CR2", "img_0001.cr2", ] { assert!(!camera_named(name), "{name}"); } } #[test] fn the_survivor_is_the_camera_named_copy_outside_a_backup_folder() { let cat = library(); let groups = candidates(cat.connection()).unwrap(); let at = survivor(&groups[0].copies); assert_eq!( groups[0].copies[at].source_ref, "PhotosRaw/2023/2023-06-24/_MG_4623.CR2" ); } #[test] fn with_names_equal_the_oldest_then_the_lowest_id_survives() { let cat = library(); let c = cat.connection(); let groups = candidates(c).unwrap(); // Only the two renamed copies: neither is camera-named nor backed up. let renamed: Vec = groups[0].copies[2..].to_vec(); assert_eq!(renamed[survivor(&renamed)].image, ImageId(3), "oldest"); let mut tied = renamed.clone(); tied[1].added_at = tied[0].added_at; assert_eq!(tied[survivor(&tied)].image, ImageId(3), "lowest id"); } #[test] fn a_backup_folder_loses_even_to_a_renamed_copy() { let cat = library(); let groups = candidates(cat.connection()).unwrap(); let pair = vec![groups[0].copies[0].clone(), groups[0].copies[3].clone()]; assert_eq!(pair[survivor(&pair)].image, ImageId(4)); } #[test] fn probes_are_kept_and_forgotten_when_the_file_changes() { let cat = library(); let c = cat.connection(); record_probes( c, &[Probe { image: ImageId(1), file_size: 27_741_184, file_mtime: Some(7), digest: "abc".into(), }], 1, ) .unwrap(); let groups = candidates(c).unwrap(); assert_eq!(groups[0].copies[0].probe.as_deref(), Some("abc")); assert_eq!(groups[0].copies[1].probe, None); // Touched since: the probe describes a file that no longer exists. c.execute("UPDATE images SET file_mtime = 8 WHERE id = 1", []) .unwrap(); assert_eq!(candidates(c).unwrap()[0].copies[0].probe, None); } #[test] fn the_highest_rating_and_every_keyword_reach_the_survivor() { let cat = library(); let c = cat.connection(); c.execute("UPDATE versions SET rating = 2 WHERE image_id = 2", []) .unwrap(); c.execute("UPDATE versions SET rating = 4 WHERE image_id = 1", []) .unwrap(); crate::keywords::assign(c, &[ImageId(1)], "Alps").unwrap(); crate::keywords::assign(c, &[ImageId(2)], "Summer").unwrap(); crate::keywords::assign(c, &[ImageId(3)], "Alps").unwrap(); let out = trash_all(&cat, 2, &[1, 3, 4]); assert_eq!(out.rating, (2, 4)); assert_eq!(out.keywords_added, 1); assert_eq!( crate::keywords::for_image(c, ImageId(2)).unwrap(), vec!["Alps".to_string(), "Summer".to_string()] ); assert_eq!(crate::rating::judgement(c, ImageId(2)).unwrap().rating, 4); } #[test] fn a_flag_the_copies_agree_on_is_taken_and_a_disagreement_is_reported() { let cat = library(); let c = cat.connection(); // Only one copy is flagged: nothing to disagree with. c.execute("UPDATE versions SET flag = 1 WHERE image_id = 3", []) .unwrap(); let out = preview(c, ImageId(2), &[ImageId(1), ImageId(3)]).unwrap(); assert_eq!(out.flag, (0, 1)); assert!(!out.flag_conflict); // A pick and a reject: the survivor keeps its own, and says so. c.execute("UPDATE versions SET flag = 2 WHERE image_id = 1", []) .unwrap(); let out = trash_all(&cat, 2, &[1, 3]); assert!(out.flag_conflict); assert_eq!(out.flag, (0, 0)); // Labels: two copies agreeing on red is agreement. let cat = library(); let c = cat.connection(); c.execute("UPDATE versions SET label = 1 WHERE image_id IN (1, 3)", []) .unwrap(); let out = trash_all(&cat, 2, &[1, 3]); assert!(!out.label_conflict); assert_eq!(out.label, (None, Some(1))); } #[test] fn a_preview_writes_nothing() { let cat = library(); let c = cat.connection(); c.execute("UPDATE versions SET rating = 5 WHERE image_id = 1", []) .unwrap(); let out = preview(c, ImageId(2), &[ImageId(1)]).unwrap(); assert_eq!(out.rating, (0, 5)); assert!(out.merges_anything()); assert_eq!(out.trashed, 0); assert_eq!(crate::rating::judgement(c, ImageId(2)).unwrap().rating, 0); let trashed: i64 = c .query_row( "SELECT COUNT(*) FROM images WHERE trashed_at IS NOT NULL", [], |r| r.get(0), ) .unwrap(); assert_eq!(trashed, 0); } #[test] fn collections_are_unioned_and_the_survivor_keeps_its_place() { let cat = library(); let c = cat.connection(); let alps = crate::collections::create(c, "Alps", None, crate::CollectionKind::Manual).unwrap(); let best = crate::collections::create(c, "Best", None, crate::CollectionKind::Manual).unwrap(); // Alps: survivor at 0, copy 1 at 1. Best: only copy 3, third in line. crate::collections::add_images(c, alps, &[ImageId(2), ImageId(1)]).unwrap(); crate::collections::add_images(c, best, &[ImageId(5), ImageId(4), ImageId(3)]).unwrap(); let rev = |id: dr_types::CollectionId| -> i64 { c.query_row( "SELECT revision FROM collections WHERE id = ?1", [id.0 as i64], |r| r.get(0), ) .unwrap() }; let (alps_rev, best_rev) = (rev(alps), rev(best)); let out = trash_all(&cat, 2, &[1, 3, 4]); assert_eq!(out.collections_joined, 1, "Best only; Alps already had it"); let position = |coll: dr_types::CollectionId| -> Option { c.query_row( "SELECT position FROM collection_members WHERE collection_id = ?1 AND image_id = 2", [coll.0 as i64], |r| r.get(0), ) .unwrap() }; assert_eq!(position(alps), Some(0), "its own place kept"); // The first copy that was in Best is 3 (ids in order), at position 2. assert_eq!(position(best), Some(2), "the copy's place taken"); assert_eq!(rev(alps), alps_rev, "no change, no revision"); assert_eq!(rev(best), best_rev + 1); } #[test] fn the_copies_land_in_the_trash_with_their_origins() { let cat = library(); let c = cat.connection(); let out = trash_all(&cat, 2, &[1, 3, 4]); assert_eq!(out.trashed, 3); let listed = crate::trash::list(c, 10).unwrap(); let mut from: Vec = listed.iter().map(|t| t.trashed_from.clone()).collect(); from.sort(); assert_eq!( from, vec![ "PhotosRaw/2023/bck/_MG_4623.CR2".to_string(), "PhotosRaw/Darktable/20230628_no_name/20230628_0642.CR2".to_string(), "PhotosRaw/alps trip/Raw/20230628_0642.CR2".to_string(), ] ); assert!(candidates(c).unwrap().is_empty()); } #[test] fn a_failure_part_way_leaves_the_group_untouched() { let cat = library(); let c = cat.connection(); c.execute("UPDATE versions SET rating = 5 WHERE image_id = 1", []) .unwrap(); crate::keywords::assign(c, &[ImageId(1)], "Alps").unwrap(); // The rating and keyword are merged from copy 1 first, then the // trash write for copy 4 fails, as a full disk would fail it. c.execute_batch( "CREATE TRIGGER fail_on_4 BEFORE UPDATE OF source_ref ON images WHEN NEW.id = 4 BEGIN SELECT RAISE(ABORT, 'disk full'); END;", ) .unwrap(); let moved = vec![ (ImageId(1), "PhotosRaw/.darkroom-trash/1-a.CR2".to_string()), (ImageId(4), "PhotosRaw/.darkroom-trash/4-a.CR2".to_string()), ]; assert!(consolidate(c, ImageId(2), &moved, 5_000).is_err()); assert_eq!(crate::rating::judgement(c, ImageId(2)).unwrap().rating, 0); assert!(crate::keywords::for_image(c, ImageId(2)) .unwrap() .is_empty()); let trashed: i64 = c .query_row( "SELECT COUNT(*) FROM images WHERE trashed_at IS NOT NULL", [], |r| r.get(0), ) .unwrap(); assert_eq!(trashed, 0, "copy 1 is not half-trashed"); } #[test] fn a_stale_plan_is_refused() { let cat = library(); let c = cat.connection(); // 5 is a different size: not a copy of 2. let moved = vec![(ImageId(5), "t".to_string())]; assert!(matches!( consolidate(c, ImageId(2), &moved, 1), Err(CatalogError::StaleDuplicate(_)) )); // Already trashed since the review. c.execute("UPDATE images SET trashed_at = 1 WHERE id = 1", []) .unwrap(); let moved = vec![(ImageId(1), "t".to_string())]; assert!(consolidate(c, ImageId(2), &moved, 1).is_err()); // The survivor named as its own copy. let moved = vec![(ImageId(2), "t".to_string())]; assert!(consolidate(c, ImageId(2), &moved, 1).is_err()); } fn face(c: &Connection, image: i64, x: f64) -> i64 { c.execute( "INSERT INTO faces (image_id, x, y, w, h, landmarks, detector_confidence, embedding, crop_px, model_id, detected_at) VALUES (?1, ?2, 0.1, 0.2, 0.3, x'00', 0.9, x'00', 100, 'w600k_mbf', 0)", rusqlite::params![image, x], ) .unwrap(); c.last_insert_rowid() } fn marker(c: &Connection, image: i64, found: i64) { c.execute( "INSERT INTO face_index (image_id, model_id, indexed_at, faces_found, source_edge) VALUES (?1, 'w600k_mbf', 0, ?2, 1024)", rusqlite::params![image, found], ) .unwrap(); } #[test] fn faces_move_onto_a_survivor_that_has_none() { let cat = library(); let c = cat.connection(); let f = face(c, 1, 0.1); marker(c, 1, 1); let anna = crate::faces::create_person(c, "Anna").unwrap(); crate::faces::confirm(c, crate::FaceId(f as u64), anna).unwrap(); let out = trash_all(&cat, 2, &[1]); assert_eq!(out.faces.moved, 1); let on: i64 = c .query_row("SELECT image_id FROM faces WHERE id = ?1", [f], |r| { r.get(0) }) .unwrap(); assert_eq!(on, 2); assert_eq!( crate::faces::images_for_person(c, anna, false).unwrap(), vec![ImageId(2)] ); let markers: i64 = c .query_row( "SELECT COUNT(*) FROM face_index WHERE image_id = 2", [], |r| r.get(0), ) .unwrap(); assert_eq!(markers, 1, "the run marker follows the faces"); } #[test] fn a_name_is_carried_onto_the_survivors_own_face_without_duplicating_it() { let cat = library(); let c = cat.connection(); let theirs = face(c, 1, 0.1); let elsewhere = face(c, 1, 0.6); let ours = face(c, 2, 0.1); let named_ours = face(c, 2, 0.6); let anna = crate::faces::create_person(c, "Anna").unwrap(); let ben = crate::faces::create_person(c, "Ben").unwrap(); crate::faces::confirm(c, crate::FaceId(theirs as u64), anna).unwrap(); crate::faces::reject(c, crate::FaceId(theirs as u64), ben).unwrap(); // The second face: named differently on each copy. crate::faces::confirm(c, crate::FaceId(elsewhere as u64), anna).unwrap(); crate::faces::confirm(c, crate::FaceId(named_ours as u64), ben).unwrap(); let out = trash_all(&cat, 2, &[1]); assert_eq!(out.faces.moved, 0); assert_eq!(out.faces.named, 1); assert_eq!(out.faces.rejections, 1); assert_eq!(out.faces.conflicts, 1); let faces_on_2: i64 = c .query_row("SELECT COUNT(*) FROM faces WHERE image_id = 2", [], |r| { r.get(0) }) .unwrap(); assert_eq!(faces_on_2, 2, "no duplicate boxes"); let who: i64 = c .query_row( "SELECT person_id FROM face_person WHERE face_id = ?1", [ours], |r| r.get(0), ) .unwrap(); assert_eq!(who, anna.0 as i64); let who: i64 = c .query_row( "SELECT person_id FROM face_person WHERE face_id = ?1", [named_ours], |r| r.get(0), ) .unwrap(); assert_eq!(who, ben.0 as i64, "the survivor's own name wins"); } }