diff --git a/Cargo.lock b/Cargo.lock index 5b5b749..cf0f2d7 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1578,6 +1578,7 @@ dependencies = [ "dr-decode", "dr-export", "dr-gpu", + "dr-ingest", "dr-pipeline", "dr-plat", "dr-segment", diff --git a/core/dr-catalog/src/dedup.rs b/core/dr-catalog/src/dedup.rs new file mode 100644 index 0000000..bfab905 --- /dev/null +++ b/core/dr-catalog/src/dedup.rs @@ -0,0 +1,308 @@ +//! TRACES: FR-CAT-11 +//! Has this photograph been imported before? +//! +//! Two tiers, because neither alone is enough and they cost very different +//! amounts. The metadata tier — capture time, camera, size, the name the +//! camera gave it — is answerable from the catalog before a byte leaves the +//! card, which is what makes re-inserting an already-imported card cost a +//! metadata read per file rather than a full transfer. The content tier +//! catches what the first misses: the same frame arriving under a different +//! name, from a second card, or after somebody renamed it. +//! +//! # Why the filename is compared here rather than in SQL +//! +//! `images.source_ref` holds the whole opaque key — a relative path on Linux, +//! a document id on SAF — and the camera's filename is only its last +//! component. Matching that in SQL means `LIKE '%/IMG_0001.CR3'`, which cannot +//! use an index, scans the whole table, and is wrong on SAF where the +//! separator is not `/`. So the query narrows on the indexed columns and the +//! handful of rows that survive are compared in Rust, the same way the grid +//! already derives a display name. +//! +//! # Filename alone is never sufficient +//! +//! Camera filenames wrap at `IMG_9999` and start again, so a library of any +//! age holds several unrelated `IMG_0001.CR3`. That is why the cheap tier +//! carries capture time and camera as well, and why the expensive tier exists +//! at all. + +use rusqlite::Connection; + +use crate::CatalogError; + +/// The last component of a stored source reference. +/// +/// Splits on both separators for the same reason `Catalog::window` does: the +/// key's shape belongs to the storage that produced it, and a SAF document id +/// is delimited with `:`. +fn file_name(source_ref: &str) -> &str { + source_ref.rsplit(['/', ':']).next().unwrap_or(source_ref) +} + +/// Whether the catalog already holds this photograph, on metadata alone. +/// +/// `camera` is the joined make-and-model string the scan stores, not the raw +/// EXIF pair — the caller composes it the same way, or the comparison is +/// always false. +/// +/// A `captured_at` of `None` makes this answer `false` rather than matching +/// every undated image in the library: without a capture time the key is +/// filename plus size, which two frames from the same body collide on +/// routinely. An undated file falls through to the content tier, which is +/// slower and right. +pub fn seen_by_metadata( + conn: &Connection, + captured_at: Option, + camera: Option<&str>, + size: u64, + original_name: &str, +) -> Result { + let Some(captured_at) = captured_at else { + return Ok(false); + }; + + // `images_captured` indexes the capture time, so this reads a few rows + // even in a library of fifty thousand: one instant to the second holds + // one frame, or a handful on a body shooting a burst. + let mut stmt = conn.prepare( + "SELECT source_ref FROM images + WHERE captured_at = ?1 + AND (?2 IS NULL OR camera IS ?2) + AND (file_size IS NULL OR file_size = ?3)", + )?; + let mut rows = stmt.query(rusqlite::params![captured_at, camera, size as i64])?; + while let Some(row) = rows.next()? { + let source_ref: String = row.get(0)?; + if file_name(&source_ref).eq_ignore_ascii_case(original_name) { + return Ok(true); + } + } + Ok(false) +} + +/// Whether these exact bytes are already in the library. +/// +/// The tier that costs a read of the file. Cheap here — `images_hash` is a +/// partial index over the rows that have one — and expensive for the caller, +/// which had to hash something to ask. +pub fn seen_by_content(conn: &Connection, digest: &str) -> Result { + let n: i64 = conn.query_row( + "SELECT COUNT(*) FROM images WHERE content_hash = ?1", + [digest], + |r| r.get(0), + )?; + Ok(n > 0) +} + +/// Record the digest of a file the import computed. +/// +/// An import reads every byte anyway, so the hash is free at that moment and +/// costs a full read of an 80 MB file at any other. Storing it is what lets +/// the *next* import answer [`seen_by_content`] without reading anything. +/// +/// Matched on `source_ref` within a root, which is how the scan that just +/// catalogued the imported file identifies it. Returns how many rows were +/// updated: zero means the scan has not reached the file yet, which is a +/// normal race and not an error. +pub fn set_content_hash( + conn: &Connection, + root_id: u64, + source_ref: &str, + digest: &str, +) -> Result { + Ok(conn.execute( + "UPDATE images SET content_hash = ?3 + WHERE root_id = ?1 AND source_ref = ?2", + rusqlite::params![root_id as i64, source_ref, digest], + )?) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::Catalog; + + /// A catalog holding one photograph, as a scan plus a metadata pass would + /// leave it. + fn with_one() -> Catalog { + let cat = Catalog::in_memory().unwrap(); + let c = cat.connection(); + c.execute( + "INSERT INTO roots(id, kind, label) VALUES (1, 'local', 'lib')", + [], + ) + .unwrap(); + c.execute( + "INSERT INTO images(root_id, source_ref, captured_at, camera, file_size, + content_hash, added_at) + VALUES (1, '2026/2026-08-22/IMG_0001.CR3', 1787407200, 'Canon EOS R5', + 9, 'deadbeef', 0)", + [], + ) + .unwrap(); + cat + } + + #[test] + fn re_inserting_the_same_card_is_recognised_before_a_transfer() { + let cat = with_one(); + assert!(seen_by_metadata( + cat.connection(), + Some(1_787_407_200), + Some("Canon EOS R5"), + 9, + "IMG_0001.CR3" + ) + .unwrap()); + } + + #[test] + fn a_different_frame_at_the_same_instant_is_not_a_duplicate() { + // Two bodies firing together, or a burst. The name separates them. + let cat = with_one(); + assert!(!seen_by_metadata( + cat.connection(), + Some(1_787_407_200), + Some("Canon EOS R5"), + 9, + "IMG_0002.CR3" + ) + .unwrap()); + } + + #[test] + fn the_same_name_from_a_different_camera_is_not_a_duplicate() { + // IMG_0001.CR3 exists on every card ever formatted. + let cat = with_one(); + assert!(!seen_by_metadata( + cat.connection(), + Some(1_787_407_200), + Some("NIKON Z 9"), + 9, + "IMG_0001.CR3" + ) + .unwrap()); + } + + #[test] + fn the_same_name_at_a_different_time_is_not_a_duplicate() { + // The IMG_9999 wrap: the library holds an unrelated IMG_0001.CR3 from + // four years ago, and matching on name alone would refuse to import + // today's. + let cat = with_one(); + assert!(!seen_by_metadata( + cat.connection(), + Some(1_600_000_000), + Some("Canon EOS R5"), + 9, + "IMG_0001.CR3" + ) + .unwrap()); + } + + #[test] + fn an_undated_file_falls_through_to_the_content_tier() { + // Not "matches everything undated" — that would silently refuse to + // import a whole card of scanned film. + let cat = with_one(); + assert!(!seen_by_metadata(cat.connection(), None, None, 9, "IMG_0001.CR3").unwrap()); + } + + #[test] + fn a_file_that_grew_is_not_the_one_already_held() { + // A truncated earlier import, or a different rendition of the same + // frame. Same instant, same camera, same name, different bytes. + let cat = with_one(); + assert!(!seen_by_metadata( + cat.connection(), + Some(1_787_407_200), + Some("Canon EOS R5"), + 1234, + "IMG_0001.CR3" + ) + .unwrap()); + } + + #[test] + fn a_row_with_no_recorded_size_still_matches() { + // The scan stores a size, but a row merged from another device may + // not have one, and refusing to match it would re-import the library. + let cat = with_one(); + cat.connection() + .execute("UPDATE images SET file_size = NULL", []) + .unwrap(); + assert!(seen_by_metadata( + cat.connection(), + Some(1_787_407_200), + Some("Canon EOS R5"), + 9, + "IMG_0001.CR3" + ) + .unwrap()); + } + + #[test] + fn the_same_frame_renamed_is_caught_by_its_bytes() { + let cat = with_one(); + // The metadata tier misses it... + assert!(!seen_by_metadata( + cat.connection(), + Some(1_787_407_200), + Some("Canon EOS R5"), + 9, + "holiday-42.CR3" + ) + .unwrap()); + // ...and the content tier does not. + assert!(seen_by_content(cat.connection(), "deadbeef").unwrap()); + assert!(!seen_by_content(cat.connection(), "cafe").unwrap()); + } + + #[test] + fn a_digest_recorded_now_answers_the_next_import() { + let cat = Catalog::in_memory().unwrap(); + let c = cat.connection(); + c.execute( + "INSERT INTO roots(id, kind, label) VALUES (1, 'local', 'lib')", + [], + ) + .unwrap(); + c.execute( + "INSERT INTO images(root_id, source_ref, added_at) + VALUES (1, '2026/2026-08-22/IMG_0001.CR3', 0)", + [], + ) + .unwrap(); + + assert!(!seen_by_content(c, "abc123").unwrap()); + let n = set_content_hash(c, 1, "2026/2026-08-22/IMG_0001.CR3", "abc123").unwrap(); + assert_eq!(n, 1); + assert!(seen_by_content(c, "abc123").unwrap()); + } + + #[test] + fn recording_a_digest_before_the_scan_arrives_is_not_an_error() { + // The import writes the file and the scan catalogues it; between those + // two moments there is no row to update, and that is a race rather + // than a failure. + let cat = Catalog::in_memory().unwrap(); + let c = cat.connection(); + c.execute( + "INSERT INTO roots(id, kind, label) VALUES (1, 'local', 'lib')", + [], + ) + .unwrap(); + assert_eq!( + set_content_hash(c, 1, "not/scanned/yet.CR3", "abc").unwrap(), + 0 + ); + } + + #[test] + fn a_name_is_the_last_component_of_either_kind_of_key() { + assert_eq!(file_name("2026/2026-08-22/IMG_0001.CR3"), "IMG_0001.CR3"); + // A SAF document id delimits with a colon. + assert_eq!(file_name("primary:DCIM/Camera/IMG_1.CR3"), "IMG_1.CR3"); + assert_eq!(file_name("IMG_0001.CR3"), "IMG_0001.CR3"); + } +} diff --git a/core/dr-catalog/src/lib.rs b/core/dr-catalog/src/lib.rs index aca35d4..208ac42 100644 --- a/core/dr-catalog/src/lib.rs +++ b/core/dr-catalog/src/lib.rs @@ -33,6 +33,7 @@ use rusqlite::Connection; pub mod cache; pub mod collections; +pub mod dedup; pub mod error; pub mod jobs; pub mod merge; @@ -46,6 +47,7 @@ pub mod walk; pub use cache::{Budget, Cache, DEFAULT_BUDGET_BYTES}; pub use collections::{Collection, CollectionKind, TreeRow}; +pub use dedup::{seen_by_content, seen_by_metadata, set_content_hash}; pub use error::CatalogError; pub use jobs::{Job, JobKind, Priority}; pub use merge::MergeReport; diff --git a/core/dr-ingest/src/lib.rs b/core/dr-ingest/src/lib.rs index 2ad1a7f..7b25506 100644 --- a/core/dr-ingest/src/lib.rs +++ b/core/dr-ingest/src/lib.rs @@ -131,7 +131,7 @@ impl Default for Options { } /// One file offered for import. -#[derive(Debug, Clone)] +#[derive(Debug, Clone, PartialEq, Eq)] pub struct Candidate { /// Where it is now — on the card. pub source: SourceRef, diff --git a/platform/dr-plat/src/lib.rs b/platform/dr-plat/src/lib.rs index 5225c40..7869397 100644 --- a/platform/dr-plat/src/lib.rs +++ b/platform/dr-plat/src/lib.rs @@ -6,6 +6,7 @@ pub mod secrets; pub mod storage; +pub mod volumes; pub use secrets::{ EphemeralSecretStore, PlatformSecretStore, SecretError, SecretKind, SecretRef, SecretStore, @@ -14,3 +15,4 @@ pub use storage::{ DirRef, Entry, LocalStorage, NewFile, Node, SeekableRead, Storage, StorageError, WritableStorage, }; +pub use volumes::{volumes, Volume}; diff --git a/platform/dr-plat/src/volumes.rs b/platform/dr-plat/src/volumes.rs new file mode 100644 index 0000000..15998ac --- /dev/null +++ b/platform/dr-plat/src/volumes.rs @@ -0,0 +1,331 @@ +//! TRACES: FR-CAT-10 | NFR-PORT-1 +//! Finding the card. +//! +//! FR-CAT-10 asks for removable-volume insertion to be detected "where the +//! platform permits", which is a careful phrase and this module is why. There +//! is no portable answer: Linux has a mount table and a sysfs flag, Android +//! has neither and hands out a document tree the user picked (ARCH §6.9). +//! So this reports what it can and returns an empty list where it cannot, +//! and every caller must still offer the user a way to say where the card is. +//! +//! # This is the one place besides `grant` that names a path +//! +//! `storage` explains why nothing above it takes a `Path`: a library location +//! is a [`crate::storage::LocalStorage::grant`] away from being a `RootId`, +//! and Android has no path to give. Volume discovery sits on the same side of +//! that line — it produces exactly what `grant` consumes, the folder that is +//! about to become a root. It is discovery of a path rather than use of one. +//! +//! # Why a heuristic rather than a device manager +//! +//! Talking to udisks2 over D-Bus would be authoritative and would add a D-Bus +//! dependency, a service that may not be running, and a permission dialog on +//! some desktops — for a question the filesystem can answer directly. The +//! mount table plus the sysfs `removable` flag covers USB card readers, +//! phones mounted as storage, and external drives. What it does not cover is +//! an internal SD reader that reports itself fixed, which is why a `DCIM` +//! directory is reported alongside and weighted more heavily than the flag: +//! a volume with `DCIM` on it is a camera card whatever sysfs believes. + +use std::path::{Path, PathBuf}; + +/// Somewhere a card might be. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct Volume { + /// What to call it in a list. The mount point's last component, which is + /// what desktops name after the filesystem label. + pub label: String, + /// Where it is mounted. Hand this to + /// [`LocalStorage::grant`](crate::storage::LocalStorage::grant). + pub path: PathBuf, + /// Whether the kernel calls the underlying device removable. + pub removable: bool, + /// Whether it holds a `DCIM` directory — the strongest signal there is + /// that this is a camera card rather than a backup drive. + pub has_dcim: bool, +} + +impl Volume { + /// Whether to show this one first. + /// + /// `DCIM` outranks the kernel flag: an internal SD reader often reports + /// its device as fixed, and the user with a card in it does not care what + /// sysfs thinks. + pub fn is_likely_card(&self) -> bool { + self.has_dcim || self.removable + } +} + +/// Every mounted volume that might hold photographs. +/// +/// Ordered with the likely cards first and, within each group, by label, so +/// the list does not reorder itself between two calls a second apart. +/// Returns empty where the platform does not permit the question — which is +/// not an error and must not be presented as one. +pub fn volumes() -> Vec { + let mut found = platform_volumes(); + found.sort_by(|a, b| { + b.is_likely_card() + .cmp(&a.is_likely_card()) + .then_with(|| a.label.cmp(&b.label)) + }); + found +} + +#[cfg(target_os = "linux")] +fn platform_volumes() -> Vec { + let table = match std::fs::read_to_string("/proc/mounts") { + Ok(t) => t, + Err(e) => { + log::debug!("no mount table: {e}"); + return Vec::new(); + } + }; + parse_mounts(&table) + .into_iter() + .filter(|m| is_candidate(m)) + .map(|m| { + let removable = device_is_removable(&m.device).unwrap_or(false) + // A desktop that automounts under /run/media or /media has + // already decided this is removable, and it had udisks2 to ask. + || under_media_dir(&m.mount_point); + Volume { + label: label_for(&m.mount_point), + has_dcim: m.mount_point.join("DCIM").is_dir(), + removable, + path: m.mount_point, + } + }) + .collect() +} + +/// Everywhere else: no answer, and saying so is the honest result. +/// +/// On Android the question is not merely unanswerable but wrong — storage is +/// reached through a tree the user granted, and a card appears there or not at +/// all (ARCH §6.9, FR-PLAT-AND-1). +#[cfg(not(target_os = "linux"))] +fn platform_volumes() -> Vec { + Vec::new() +} + +/// One line of the mount table. +#[derive(Debug, Clone, PartialEq, Eq)] +struct Mount { + device: String, + mount_point: PathBuf, + fs_type: String, +} + +/// Parse `/proc/mounts`. +/// +/// Split out from the reading so it can be tested against real tables without +/// a card plugged in — which is the only way this file is testable at all. +fn parse_mounts(table: &str) -> Vec { + table + .lines() + .filter_map(|line| { + let mut fields = line.split_whitespace(); + let device = fields.next()?; + let mount_point = fields.next()?; + let fs_type = fields.next()?; + Some(Mount { + device: unescape(device), + mount_point: PathBuf::from(unescape(mount_point)), + fs_type: fs_type.to_string(), + }) + }) + .collect() +} + +/// Undo the octal escaping the kernel applies to spaces and tabs. +/// +/// A card is very often labelled with a space in it — `EOS DIGITAL` is what +/// Canon writes — and mounted at `/run/media/duncan/EOS\040DIGITAL`. Without +/// this the path does not exist and the card is invisible. +fn unescape(field: &str) -> String { + let mut out = String::with_capacity(field.len()); + let bytes = field.as_bytes(); + let mut i = 0; + while i < bytes.len() { + if bytes[i] == b'\\' && i + 3 < bytes.len() { + let digits = &field[i + 1..i + 4]; + if let Ok(c) = u8::from_str_radix(digits, 8) { + out.push(c as char); + i += 4; + continue; + } + } + out.push(bytes[i] as char); + i += 1; + } + out +} + +/// Whether a mount could hold a photograph library at all. +/// +/// Excludes the pseudo-filesystems, which is most of the table, and anything +/// not backed by a block device. A card reader always is. +fn is_candidate(m: &Mount) -> bool { + const FILESYSTEMS: &[&str] = &[ + // What cameras format cards as, in practice: FAT32 below 32 GB, + // exFAT above it, which is why an SDXC card is always exfat. + "vfat", "exfat", "ntfs", "ntfs3", + "fuseblk", // And what an external drive carrying an archive is. + "ext2", "ext3", "ext4", "btrfs", "xfs", "f2fs", "hfsplus", "apfs", + ]; + m.device.starts_with("/dev/") && FILESYSTEMS.contains(&m.fs_type.as_str()) +} + +/// Whether the desktop automounted this, which means it already decided. +fn under_media_dir(mount_point: &Path) -> bool { + let s = mount_point.to_string_lossy(); + s.starts_with("/run/media/") || s.starts_with("/media/") || s.starts_with("/mnt/media/") +} + +/// Ask sysfs whether the device behind a partition is removable. +/// +/// `/dev/sdb1` is a partition; the flag lives on its whole-disk parent, +/// `/sys/block/sdb/removable`. Trailing digits are stripped to get there, +/// except on `mmcblk0p1` and `nvme0n1p1`, whose parents keep their digits and +/// lose only the `pN` suffix — strip naively and `mmcblk0p1` becomes +/// `mmcblk`, which does not exist, and every SD card in an internal reader +/// reports itself fixed. +fn device_is_removable(device: &str) -> Option { + let name = device.strip_prefix("/dev/")?; + let disk = whole_disk(name); + let flag = std::fs::read_to_string(format!("/sys/block/{disk}/removable")).ok()?; + Some(flag.trim() == "1") +} + +/// The whole-disk name a partition belongs to. +fn whole_disk(partition: &str) -> String { + // `mmcblk0p1` -> `mmcblk0`, `nvme0n1p3` -> `nvme0n1`: these spell the + // partition with a `p`, and the disk name legitimately ends in a digit. + if partition.starts_with("mmcblk") || partition.starts_with("nvme") { + if let Some((disk, tail)) = partition.rsplit_once('p') { + if !tail.is_empty() && tail.chars().all(|c| c.is_ascii_digit()) { + return disk.to_string(); + } + } + return partition.to_string(); + } + // `sdb1` -> `sdb`. A SCSI-style disk name never ends in a digit. + partition + .trim_end_matches(|c: char| c.is_ascii_digit()) + .to_string() +} + +/// What to call a volume in a list. +fn label_for(mount_point: &Path) -> String { + mount_point + .file_name() + .map(|n| n.to_string_lossy().to_string()) + .unwrap_or_else(|| mount_point.to_string_lossy().to_string()) +} + +#[cfg(test)] +mod tests { + use super::*; + + const TABLE: &str = "\ +proc /proc proc rw,nosuid 0 0 +sys /sys sysfs rw,nosuid 0 0 +/dev/nvme0n1p2 / btrfs rw,noatime 0 0 +/dev/nvme0n1p1 /boot vfat rw,relatime 0 0 +/dev/sdb1 /run/media/duncan/EOS\\040DIGITAL exfat rw,nosuid 0 0 +tmpfs /run/user/1000 tmpfs rw,nosuid 0 0 +"; + + #[test] + fn a_mount_table_yields_its_block_devices() { + let mounts = parse_mounts(TABLE); + assert_eq!(mounts.len(), 6); + let candidates: Vec<_> = mounts.iter().filter(|m| is_candidate(m)).collect(); + // Three block-backed filesystems; the pseudo ones are gone. + assert_eq!(candidates.len(), 3); + assert!(candidates.iter().all(|m| m.device.starts_with("/dev/"))); + } + + #[test] + fn a_card_labelled_with_a_space_is_not_lost() { + // Canon writes `EOS DIGITAL`, the kernel escapes the space, and a path + // that keeps the escape does not exist. + let mounts = parse_mounts(TABLE); + let card = mounts + .iter() + .find(|m| m.device == "/dev/sdb1") + .expect("the card"); + assert_eq!( + card.mount_point, + PathBuf::from("/run/media/duncan/EOS DIGITAL") + ); + assert_eq!(label_for(&card.mount_point), "EOS DIGITAL"); + } + + #[test] + fn an_automounted_volume_is_treated_as_removable() { + // The desktop had udisks2 to ask and already decided. + assert!(under_media_dir(Path::new("/run/media/duncan/EOS DIGITAL"))); + assert!(under_media_dir(Path::new("/media/usb0"))); + assert!(!under_media_dir(Path::new("/home/duncan/Photos"))); + // Not a prefix match on the word: `/media-server` is not automounted. + assert!(!under_media_dir(Path::new("/mediaserver/x"))); + } + + #[test] + fn a_partition_resolves_to_the_disk_that_carries_the_flag() { + assert_eq!(whole_disk("sdb1"), "sdb"); + assert_eq!(whole_disk("sda12"), "sda"); + // The naive strip turns these into `mmcblk` and `nvme`, neither of + // which exists — and every SD card in an internal reader then reports + // itself fixed. + assert_eq!(whole_disk("mmcblk0p1"), "mmcblk0"); + assert_eq!(whole_disk("nvme0n1p3"), "nvme0n1"); + // A whole disk named directly is already the answer. + assert_eq!(whole_disk("mmcblk0"), "mmcblk0"); + assert_eq!(whole_disk("sdb"), "sdb"); + } + + #[test] + fn a_camera_card_outranks_a_backup_drive() { + let card = Volume { + label: "EOS DIGITAL".into(), + path: "/run/media/duncan/EOS DIGITAL".into(), + removable: false, + has_dcim: true, + }; + let drive = Volume { + label: "archive".into(), + path: "/run/media/duncan/archive".into(), + removable: true, + has_dcim: false, + }; + let fixed = Volume { + label: "boot".into(), + path: "/boot".into(), + removable: false, + has_dcim: false, + }; + // An internal reader reports its device fixed, and the user with a + // card in it does not care what sysfs thinks. + assert!(card.is_likely_card()); + assert!(drive.is_likely_card()); + assert!(!fixed.is_likely_card()); + } + + #[test] + fn asking_where_there_is_no_answer_is_not_an_error() { + // The call must work on a machine with no card, in CI, and on a + // platform that cannot answer at all — an empty list, never a panic. + let _ = volumes(); + } + + #[test] + fn an_octal_escape_that_is_not_one_is_left_alone() { + // A backslash near the end of a field must not read past it. + assert_eq!(unescape("/mnt/odd\\"), "/mnt/odd\\"); + assert_eq!(unescape("/mnt/a\\04"), "/mnt/a\\04"); + assert_eq!(unescape("/mnt/a\\040b"), "/mnt/a b"); + } +} diff --git a/ui/dr-ui/Cargo.toml b/ui/dr-ui/Cargo.toml index 8acb901..4a58c5f 100644 --- a/ui/dr-ui/Cargo.toml +++ b/ui/dr-ui/Cargo.toml @@ -27,6 +27,7 @@ dr-plat.workspace = true dr-sync.workspace = true dr-sync-nextcloud.workspace = true dr-export.workspace = true +dr-ingest.workspace = true dr-pipeline.workspace = true dr-catalog.workspace = true dr-thumbs.workspace = true diff --git a/ui/dr-ui/src/import.rs b/ui/dr-ui/src/import.rs new file mode 100644 index 0000000..fd1d807 --- /dev/null +++ b/ui/dr-ui/src/import.rs @@ -0,0 +1,600 @@ +//! TRACES: FR-CAT-10 | FR-CAT-11 | FR-NC-7a +//! Bringing a card into the library: the work, off the interface thread. +//! +//! [`dr_ingest`] does the copying and knows nothing about cards, catalogs or +//! windows — it is handed a list of files, a probe for their metadata and a +//! question about duplicates. This supplies all three, and runs the result on +//! a worker. +//! +//! The shape is the one every background operation in this crate has: a +//! thread with an `mpsc` channel, drained by a `slint::Timer` on the UI +//! thread, reporting into [`crate::activity`]. See `import_ui` for the +//! draining half. +//! +//! # Two phases, one worker +//! +//! Surveying a card is itself slow — a full card is two thousand files across +//! a directory tree on a bus that manages 40 MB/s on a good day — so it runs +//! on the worker too and reports what it found before any copying starts. The +//! user sees "1,847 photographs, 61.2 GB" and then a progress bar, rather than +//! a frozen window and then a progress bar. +//! +//! # The import does not catalogue what it wrote +//! +//! It writes files into the library and then asks for a scan, rather than +//! inserting rows itself. FR-CAT-1 already turns files on disk into catalog +//! rows, and a second path into the `images` table would be a second set of +//! bugs about folders, formats and metadata state. What the import *does* +//! record afterwards is each file's digest (`dr_catalog::dedup`), which the +//! scan cannot know because it never reads a whole file. + +use std::path::{Path, PathBuf}; +use std::sync::atomic::{AtomicBool, Ordering}; +use std::sync::mpsc::{Receiver, Sender}; +use std::sync::Arc; + +use dr_ingest::{Candidate, DupKey, Imported, Ingest, Options, Report, Shot}; +use dr_plat::{DirRef, LocalStorage, Storage, WritableStorage}; +use dr_types::{FormatFilter, RootId}; + +/// Which root the card is granted as, and which the library is. +/// +/// Two distinct ids in one `LocalStorage` would also work; separate storages +/// keep the read-only source and the writable destination separate all the +/// way down, which is the distinction `WritableStorage` exists to make. +const CARD: RootId = RootId(9001); +const LIBRARY: RootId = RootId(9002); +const BACKUP: RootId = RootId(9003); + +/// A cancel flag shared with the worker. +pub type Cancel = Arc; + +/// Everything an import needs to run. +#[derive(Debug, Clone)] +pub struct Request { + /// Where the card is mounted. + pub card: PathBuf, + /// The library root the template's folders are created under. + pub library: PathBuf, + /// The optional second destination (FR-CAT-10). + pub backup: Option, + /// The catalog, opened by the worker for its duplicate queries. + /// + /// A path rather than a connection: `rusqlite::Connection` is not `Sync`, + /// and every other worker in this crate opens its own for the same reason. + pub catalog: PathBuf, + /// The library's root id *in the catalog*, which is what + /// [`dr_catalog::set_content_hash`] matches on. Unrelated to [`LIBRARY`], + /// which is this module's handle on the same folder. + pub catalog_root: u64, + /// Which file types to take off the card. + pub filter: FormatFilter, + pub options: Options, +} + +/// What the worker sends back. +#[derive(Debug, Clone)] +pub enum Message { + /// The card has been walked. Sent once, before any copying. + Surveyed { files: usize, bytes: u64 }, + /// One file finished — imported, skipped or failed. + Progress { + done: usize, + total: usize, + bytes: u64, + name: String, + }, + /// The run ended. Always the last message. + Finished(Outcome), + /// The run could not start at all. + Failed(String), +} + +/// What an import did, in the form the interface reports it. +#[derive(Debug, Clone, Default, PartialEq, Eq)] +pub struct Outcome { + pub imported: usize, + pub duplicates: usize, + pub failed: usize, + /// Files dated by modification time rather than by the shutter + /// (FR-NC-7a). Named rather than counted: the user needs to know *which* + /// photographs are filed under a date that is not when they were taken. + pub undated: Vec, + pub bytes: u64, + pub cancelled: bool, + /// The folders that were written into, for the message at the end and for + /// the upload that follows (FR-NC-7a). + pub folders: Vec, + /// Card files that may now be deleted, for a move-import (FR-NC-7b). + /// + /// Carried out rather than acted on here: this crate knows the local write + /// succeeded, and not whether the upload did. + pub retirable: usize, +} + +/// Start an import. +/// +/// Returns immediately; the work happens on a thread and reports through the +/// channel. A dropped receiver does not stop the worker — the cancel flag +/// does, and the caller holds it. +pub fn spawn(request: Request, cancel: Cancel) -> Receiver { + let (tx, rx) = std::sync::mpsc::channel(); + std::thread::Builder::new() + .name("import".into()) + .spawn(move || { + if let Err(e) = run(request, &cancel, &tx) { + let _ = tx.send(Message::Failed(e)); + } + }) + .expect("spawning the import worker"); + rx +} + +fn run(request: Request, cancel: &Cancel, tx: &Sender) -> Result<(), String> { + let card = LocalStorage::with_root(CARD, request.card.clone()); + let library = LocalStorage::with_root(LIBRARY, request.library.clone()); + let backup = request + .backup + .as_ref() + .map(|p| LocalStorage::with_root(BACKUP, p.clone())); + + let candidates = survey(&card, CARD, &request.filter, &|| { + cancel.load(Ordering::Relaxed) + }) + .map_err(|e| format!("could not read the card: {e}"))?; + + let bytes: u64 = candidates.iter().map(|c| c.size).sum(); + let _ = tx.send(Message::Surveyed { + files: candidates.len(), + bytes, + }); + + // Opened after the survey rather than before: a card that turns out to be + // empty should not also report a catalog it never needed. + let catalog = dr_catalog::Catalog::open(&request.catalog) + .map_err(|e| format!("could not open the catalog: {e}"))?; + + let library_root = DirRef::root(LIBRARY); + let backup_root = DirRef::root(BACKUP); + let ingest = Ingest { + source: &card, + dest: &library, + dest_root: &library_root, + backup: backup.as_ref().map(|b| (b as &dyn WritableStorage, &backup_root)), + options: request.options.clone(), + }; + + let report = ingest.run( + &candidates, + &|c| probe(&card, c), + &|key| is_duplicate(catalog.connection(), key), + &|| cancel.load(Ordering::Relaxed), + &mut |p| { + let _ = tx.send(Message::Progress { + done: p.done, + total: p.total, + bytes: p.bytes, + name: p.name.clone(), + }); + }, + ); + + // The digests, so the *next* import can answer the content tier without + // reading anything. Best-effort and after the fact: the rows do not exist + // until a scan has caught up, and a hash that misses its row costs one + // wasted transfer next time rather than a failure now. + record_digests(catalog.connection(), request.catalog_root, &report.imported); + + let _ = tx.send(Message::Finished(summarise(&report))); + Ok(()) +} + +/// Walk a card for files worth importing. +/// +/// Recursive rather than `DCIM`-only: a card carries `DCIM/100CANON`, a phone +/// carries `DCIM/Camera`, and a card that has been through a computer carries +/// whatever somebody put on it. Walking the whole volume finds all three, and +/// the format filter is what keeps the result to photographs. +pub fn survey( + storage: &dyn Storage, + root: RootId, + filter: &FormatFilter, + cancel: &dyn Fn() -> bool, +) -> Result, dr_plat::StorageError> { + let mut out = Vec::new(); + let mut queue = vec![storage.root_dir(root)?]; + while let Some(dir) = queue.pop() { + if cancel() { + break; + } + // A directory that cannot be read is skipped rather than fatal: cards + // carry vendor folders with odd permissions, and one of them must not + // cost the user the other nine hundred photographs (FR-CAT-9). + let entries = match storage.list(&dir) { + Ok(e) => e, + Err(e) => { + log::warn!("skipping {dir}: {e}"); + continue; + } + }; + for entry in entries { + if entry.meta.is_dir { + if !dr_catalog::walk::is_excluded(&entry.meta.name) { + if let dr_plat::Node::Dir(d) = entry.node { + queue.push(d); + } + } + continue; + } + if !filter.allows_name(&entry.meta.name) { + continue; + } + if let dr_plat::Node::File(source) = entry.node { + out.push(Candidate { + source, + name: entry.meta.name, + size: entry.meta.size, + // The listing's reading, which is the fallback the folder + // template uses when a file carries no capture time. + modified_at: entry.meta.mtime / 1000, + }); + } + } + } + // A card walks in whatever order the filesystem hands back, which for FAT + // is creation order per directory and arbitrary across them. Sorted so an + // import reports its progress through a card in the order a photographer + // would expect, and so two runs of the same card behave the same way. + out.sort_by(|a, b| a.name.cmp(&b.name)); + Ok(out) +} + +/// Read one candidate's capture metadata. +/// +/// A header read rather than the whole file: `dr_decode::HEADER_BYTES` is +/// enough for EXIF on every format in the tree, and reading 80 MB per file to +/// learn a date would take longer than the import itself. +fn probe(card: &dyn Storage, candidate: &Candidate) -> Option { + let header = card + .read_range(&candidate.source, 0..dr_decode::HEADER_BYTES) + .ok()?; + let md = dr_decode::metadata(&header).ok()?; + Some(Shot { + captured_at: md.captured_at, + captured_offset: md.captured_offset, + make: md.make, + model: md.model, + modified_at: candidate.modified_at, + }) +} + +/// FR-CAT-11's two tiers, against the catalog. +/// +/// The camera string is composed the way the scan composes it +/// ([`crate::library::camera_label`]) — the comparison is against what a scan +/// wrote, so a second spelling here would silently disable the cheap tier. +fn is_duplicate(conn: &rusqlite::Connection, key: &DupKey) -> bool { + if let Some(digest) = &key.digest { + return dr_catalog::seen_by_content(conn, digest).unwrap_or(false); + } + let camera = crate::library::camera_label(key.make.as_deref(), key.model.as_deref()); + dr_catalog::seen_by_metadata( + conn, + key.captured_at, + camera.as_deref(), + key.size, + &key.original_name, + ) + .unwrap_or(false) +} + +/// Store what the import learned that a scan cannot. +fn record_digests(conn: &rusqlite::Connection, root: u64, imported: &[Imported]) { + for image in imported { + if let Err(e) = dr_catalog::set_content_hash(conn, root, image.written.key(), &image.digest) + { + log::warn!("recording the digest of {}: {e}", image.name); + } + } +} + +/// Reduce a report to what the interface says about it. +pub fn summarise(report: &Report) -> Outcome { + let mut folders: Vec = report + .imported + .iter() + .map(|i| i.folders.join("/")) + .collect(); + folders.sort(); + folders.dedup(); + + Outcome { + imported: report.imported.len(), + duplicates: report.duplicates.len(), + failed: report.failed.len(), + undated: report.undated.clone(), + bytes: report.bytes, + cancelled: report.cancelled, + folders, + retirable: report.retirable.len(), + } +} + +/// The sentence shown when an import ends. +/// +/// Says what happened to every file, because the counts that are zero are the +/// ones worth not mentioning and the ones that are not are the whole point. +/// A run that imported nothing says so rather than reporting success in the +/// abstract. +pub fn describe(outcome: &Outcome) -> String { + let mut parts = Vec::new(); + if outcome.imported > 0 { + parts.push(format!( + "{} imported ({})", + outcome.imported, + crate::activity::describe_bytes(outcome.bytes) + )); + } + if outcome.duplicates > 0 { + parts.push(format!("{} already in the library", outcome.duplicates)); + } + if outcome.failed > 0 { + parts.push(format!("{} failed", outcome.failed)); + } + + let head = if parts.is_empty() { + "Nothing to import".to_string() + } else { + parts.join(", ") + }; + + let mut out = if outcome.cancelled { + format!("Stopped — {head}") + } else { + head + }; + + // Where they went, which is the question the folder template raises. + match outcome.folders.len() { + 0 => {} + 1 => out.push_str(&format!(" into {}", outcome.folders[0])), + n => out.push_str(&format!(" into {n} folders")), + } + + // FR-NC-7a: the mtime fallback is reported rather than silent. + if !outcome.undated.is_empty() { + out.push_str(&format!( + ". {} had no capture time and {} filed by modification date", + outcome.undated.len(), + if outcome.undated.len() == 1 { + "was" + } else { + "were" + } + )); + } + out +} + +/// Whether a folder looks like somewhere a card would be. +/// +/// Used to warn before an import writes into the wrong place: a library root +/// and a card mount point are both just directories, and the two are very +/// easy to swap in a dialog. +pub fn looks_like_a_card(path: &Path) -> bool { + path.join("DCIM").is_dir() +} + +#[cfg(test)] +mod tests { + use super::*; + use dr_ingest::DateSource; + + struct Tree(PathBuf); + + impl Tree { + fn new(name: &str) -> Self { + let dir = std::env::temp_dir().join(format!( + "dr-ui-import-{name}-{}-{:?}", + std::process::id(), + std::thread::current().id() + )); + let _ = std::fs::remove_dir_all(&dir); + std::fs::create_dir_all(&dir).unwrap(); + Tree(dir) + } + fn file(&self, rel: &str, bytes: &[u8]) -> &Self { + let p = self.0.join(rel); + std::fs::create_dir_all(p.parent().unwrap()).unwrap(); + std::fs::write(p, bytes).unwrap(); + self + } + } + + impl Drop for Tree { + fn drop(&mut self) { + let _ = std::fs::remove_dir_all(&self.0); + } + } + + fn survey_of(t: &Tree) -> Vec { + let storage = LocalStorage::with_root(CARD, t.0.clone()); + survey(&storage, CARD, &FormatFilter::all(), &|| false).unwrap() + } + + #[test] + fn a_survey_finds_photographs_wherever_the_camera_put_them() { + let t = Tree::new("survey"); + // Three real layouts at once: a Canon card, a phone, and files + // somebody dropped at the top level. + t.file("DCIM/100CANON/IMG_0001.CR3", b"a") + .file("DCIM/Camera/PXL_0002.dng", b"bb") + .file("loose.NEF", b"ccc"); + + let found = survey_of(&t); + assert_eq!(found.len(), 3); + assert_eq!( + found.iter().map(|c| c.name.as_str()).collect::>(), + ["IMG_0001.CR3", "PXL_0002.dng", "loose.NEF"] + ); + } + + #[test] + fn a_survey_leaves_what_is_not_a_photograph() { + let t = Tree::new("survey-filter"); + t.file("DCIM/100CANON/IMG_0001.CR3", b"a") + // Every card carries these, and none of them is an import. + .file("DCIM/100CANON/IMG_0001.THM", b"x") + .file("MISC/settings.dat", b"x") + .file("readme.txt", b"x"); + + let found = survey_of(&t); + assert_eq!(found.len(), 1); + assert_eq!(found[0].name, "IMG_0001.CR3"); + } + + #[test] + fn a_survey_is_ordered_the_same_way_twice() { + // FAT hands directories back in creation order and volumes in none at + // all, so an unsorted survey reports its progress through a card in an + // order that changes between runs. + let t = Tree::new("survey-order"); + t.file("DCIM/100CANON/IMG_0003.CR3", b"c") + .file("DCIM/100CANON/IMG_0001.CR3", b"a") + .file("DCIM/101CANON/IMG_0002.CR3", b"b"); + assert_eq!(survey_of(&t), survey_of(&t)); + } + + #[test] + fn a_survey_carries_what_the_folder_template_needs() { + let t = Tree::new("survey-meta"); + t.file("DCIM/100CANON/IMG_0001.CR3", b"12345"); + let found = survey_of(&t); + assert_eq!(found[0].size, 5); + // Seconds, not milliseconds — a template handed a millisecond reading + // files everything in the year 56000. + let year = dr_types::civil_from_unix(found[0].modified_at).year; + assert!((2000..2200).contains(&year), "mtime looked like {year}"); + } + + #[test] + fn a_survey_can_be_stopped() { + let t = Tree::new("survey-cancel"); + t.file("DCIM/100CANON/IMG_0001.CR3", b"a"); + let storage = LocalStorage::with_root(CARD, t.0.clone()); + let found = survey(&storage, CARD, &FormatFilter::all(), &|| true).unwrap(); + assert!(found.is_empty()); + } + + #[test] + fn a_card_is_recognised_by_its_dcim_folder() { + let t = Tree::new("looks-like"); + t.file("DCIM/100CANON/IMG_0001.CR3", b"a"); + assert!(looks_like_a_card(&t.0)); + assert!(!looks_like_a_card(&t.0.join("DCIM/100CANON"))); + } + + // ---- what the interface says ----------------------------------------- + + fn outcome() -> Outcome { + Outcome { + imported: 3, + bytes: 3 * 1024 * 1024, + folders: vec!["2026/2026-08-22".into()], + ..Default::default() + } + } + + #[test] + fn a_finished_import_says_what_it_did_and_where() { + let text = describe(&outcome()); + assert!(text.starts_with("3 imported ("), "{text}"); + assert!(text.ends_with(" into 2026/2026-08-22"), "{text}"); + } + + #[test] + fn a_card_that_was_already_imported_says_so_rather_than_nothing() { + let o = Outcome { + imported: 0, + duplicates: 12, + bytes: 0, + folders: vec![], + ..Default::default() + }; + // The failure mode this guards against is a dialog that closes with a + // cheerful "done" after transferring nothing. + assert_eq!(describe(&o), "12 already in the library"); + } + + #[test] + fn an_empty_card_is_not_reported_as_a_success() { + assert_eq!(describe(&Outcome::default()), "Nothing to import"); + } + + #[test] + fn a_cancelled_run_leads_with_the_fact_that_it_stopped() { + let o = Outcome { + cancelled: true, + ..outcome() + }; + assert!(describe(&o).starts_with("Stopped — 3 imported"), "{}", describe(&o)); + } + + #[test] + fn undated_files_are_named_in_the_result() { + // FR-NC-7a: a file dated by mtime is filed under when it was last + // written, which for a card through a reader is often today. + let o = Outcome { + undated: vec!["SCAN_01.TIF".into()], + ..outcome() + }; + let text = describe(&o); + assert!(text.contains("1 had no capture time"), "{text}"); + assert!(text.contains("was filed by modification date"), "{text}"); + + let many = Outcome { + undated: vec!["a".into(), "b".into()], + ..outcome() + }; + assert!(describe(&many).contains("2 had no capture time")); + assert!(describe(&many).contains("were filed")); + } + + #[test] + fn several_days_on_one_card_are_counted_rather_than_listed() { + let o = Outcome { + folders: vec!["2026/2026-08-21".into(), "2026/2026-08-22".into()], + ..outcome() + }; + assert!(describe(&o).ends_with("into 2 folders"), "{}", describe(&o)); + } + + #[test] + fn a_summary_collapses_a_days_worth_of_files_into_one_folder() { + let report = Report { + imported: (0..3) + .map(|i| Imported { + source: dr_types::SourceRef::Local { + root: CARD, + relative: format!("IMG_{i}.CR3"), + }, + written: dr_types::SourceRef::Local { + root: LIBRARY, + relative: format!("2026/2026-08-22/IMG_{i}.CR3"), + }, + folders: vec!["2026".into(), "2026-08-22".into()], + name: format!("IMG_{i}.CR3"), + digest: "x".into(), + size: 10, + dated_from: DateSource::Capture, + backup: None, + }) + .collect(), + bytes: 30, + ..Default::default() + }; + let o = summarise(&report); + assert_eq!(o.imported, 3); + assert_eq!(o.folders, ["2026/2026-08-22"]); + } +} diff --git a/ui/dr-ui/src/lib.rs b/ui/dr-ui/src/lib.rs index acc62aa..606401b 100644 --- a/ui/dr-ui/src/lib.rs +++ b/ui/dr-ui/src/lib.rs @@ -26,6 +26,7 @@ mod develop; mod export; mod gradient; mod histogram; +mod import; mod labels; mod library; mod library_ui; diff --git a/ui/dr-ui/src/library.rs b/ui/dr-ui/src/library.rs index ba89f69..a25a035 100644 --- a/ui/dr-ui/src/library.rs +++ b/ui/dr-ui/src/library.rs @@ -1981,21 +1981,31 @@ fn collect_metadata(header: &[u8], req: &ThumbnailRequest, out: &mut Vec { - Some(model.trim().to_string()) - } - (Some(make), Some(model)) => Some(format!("{} {}", make.trim(), model.trim())), - (None, Some(model)) => Some(model.trim().to_string()), - _ => None, - }, + camera: camera_label(md.make.as_deref(), md.model.as_deref()), lens: md.lens.map(|l| l.trim().to_string()), iso: md.iso, }); } +/// TRACES: FR-CAT-11 +/// The camera string the catalog stores, from an EXIF make and model. +/// +/// One definition rather than one per caller, because an import's duplicate +/// check compares against what a scan wrote (`dr_catalog::dedup`). Two +/// spellings of the same body would not fail loudly — they would silently +/// disable the cheap tier, and every re-inserted card would transfer in full +/// before the digest caught it. +pub fn camera_label(make: Option<&str>, model: Option<&str>) -> Option { + match (make, model) { + // Bodies repeat the make inside the model ("Canon EOS 6D"), so + // joining unconditionally yields "Canon Canon EOS 6D". + (Some(make), Some(model)) if model.starts_with(make) => Some(model.trim().to_string()), + (Some(make), Some(model)) => Some(format!("{} {}", make.trim(), model.trim())), + (None, Some(model)) => Some(model.trim().to_string()), + _ => None, + } +} + /// Write a batch of dates and tell the UI, draining `found`. /// /// Separate from the loop so the same path serves both the periodic flush and