//! TRACES: FR-CAT-10 | FR-CAT-11 //! Bringing photographs in from a card. //! //! Distinct from `dr_catalog::scan`, and the distinction is the whole reason //! this crate exists: a scan catalogues files where they already are, where an //! import *moves them* from a card into the library. A scan that failed //! halfway has read nothing; an import that fails halfway has written //! something. //! //! # What this crate will not do //! //! **It never deletes from the card.** A move-import is expressed as a copy //! plus a later retirement the caller performs once it is satisfied — because //! on a library that syncs, "satisfied" means the upload was confirmed, not //! that the local write returned (FR-NC-7b). A card erased against an //! in-flight upload is the one failure in this application with no undo, so //! the deletion is deliberately not reachable from the code path that is in a //! hurry. See [`retire`]. //! //! **It does not decode.** Capture metadata arrives through [`Probe`], so this //! crate does not depend on `dr-decode` and a card of unreadable files costs no //! demosaic. The same shape `dr_export::name::resolve_name` uses for its //! collision test, for the same reason: the answer differs by caller and none //! of the variants belong here. //! //! **It does not know what a duplicate is.** That question is a catalog query //! (FR-CAT-11), so it arrives through [`DuplicateCheck`]. use std::collections::HashMap; use std::io::Read; use dr_plat::{DirRef, NewFile, SeekableRead, Storage, StorageError, WritableStorage}; use dr_types::SourceRef; use sha2::{Digest, Sha256}; pub mod layout; pub use layout::{expand, DateSource, Destination, Shot, DEFAULT_TEMPLATE}; /// How many bytes move between the card and the destination at a time. /// /// A RAW is 20–100 MB and there is no reason to hold one in memory: the read, /// the hash and the write are all streaming, so peak usage is this buffer /// whether the card holds one file or two thousand. 1 MiB rather than /// something smaller because a card reader's throughput collapses on small /// reads, and rather than something larger because the second copy /// (FR-CAT-10's backup destination) doubles it. const CHUNK: usize = 1024 * 1024; /// Something went wrong importing one file. /// /// Per-file rather than per-run: a card with one unreadable file must import /// the other nine hundred, so this is recorded against the item in /// [`Report::failed`] and the run continues. #[derive(Debug, thiserror::Error)] pub enum IngestError { #[error("reading the card: {0}")] Source(StorageError), #[error("writing the destination: {0}")] Destination(StorageError), #[error("writing the backup copy: {0}")] Backup(StorageError), /// The bytes that arrived are not the bytes that left. /// /// A dying card, a full destination that reported success, a cable on its /// way out. The written file is removed before this is returned — a /// corrupt import that stayed on disk would be found by the next scan and /// catalogued as though it were fine. #[error("verification failed: wrote {written} bytes, {reason}")] Verify { written: u64, reason: String }, #[error("io error: {0}")] Io(String), /// The user asked to stop. Not a failure of the file it is recorded /// against — that file simply never started. #[error("cancelled")] Cancelled, } /// What to do about a file the catalog has seen before. #[derive(Debug, Clone, Copy, PartialEq, Eq, Default)] pub enum DuplicatePolicy { /// Transfer nothing. Re-inserting a card that was already imported is the /// common case and should cost one metadata read per file. #[default] Skip, /// Import anyway, under a name that does not collide. ImportAsNew, } /// Whether the card copy may be deleted once the import is confirmed. /// /// Recorded rather than acted on — see the crate docs on why the deletion does /// not live here. #[derive(Debug, Clone, Copy, PartialEq, Eq, Default)] pub enum TransferMode { #[default] Copy, /// The user asked for a move. [`Report::retirable`] carries the sources /// whose card copies are now redundant. Move, } /// How an import is configured. #[derive(Debug, Clone)] pub struct Options { /// The folder template (FR-NC-7a). See [`layout::DEFAULT_TEMPLATE`]. pub folder_template: String, pub mode: TransferMode, pub on_duplicate: DuplicatePolicy, /// Re-read each written file and compare digests (FR-CAT-10). /// /// On by default and worth the second read: the failure it catches is a /// silently corrupt original, which is unrecoverable once the card is /// reused and invisible until the photograph is opened months later. pub verify: bool, } impl Default for Options { fn default() -> Self { Self { folder_template: DEFAULT_TEMPLATE.to_string(), mode: TransferMode::Copy, on_duplicate: DuplicatePolicy::Skip, verify: true, } } } /// What a destination folder already holds: folder key -> (name -> size). /// /// Built lazily as the template sends files to new folders, and kept for the /// run so that re-importing a card costs one listing per day rather than one /// per photograph. type Listings = HashMap>; /// One file offered for import. #[derive(Debug, Clone, PartialEq, Eq)] pub struct Candidate { /// Where it is now — on the card. pub source: SourceRef, /// Its name on the card, extension included. pub name: String, /// Size in bytes, for progress. Zero where unknown. pub size: u64, /// The file's modification time, Unix seconds. The date fallback. pub modified_at: i64, } /// What identifies a photograph independently of where it is stored. /// /// FR-CAT-11's two tiers in one type. The metadata tier — capture time, camera, /// original filename — is answerable before a byte is transferred, which is /// what makes re-inserting an already-imported card cheap. The `digest` tier is /// only knowable after reading the file, and catches the case the first misses: /// the same frame arriving under a different name. /// /// Camera *serial* would be the stronger second key and is not here, because /// nothing in the tree reads it yet — `dr_decode::Metadata` carries make and /// model only. Make and model plus capture time plus filename is what is /// available; the field is worth adding when the decoder grows it, and the /// digest tier covers the gap meanwhile. #[derive(Debug, Clone, PartialEq, Eq)] pub struct DupKey { pub captured_at: Option, pub make: Option, pub model: Option, /// The name the camera gave it. Camera filenames wrap at `IMG_9999`, so /// this is never sufficient alone. pub original_name: String, pub size: u64, /// SHA-256 of the file's bytes, once they have been read. `None` on the /// pre-transfer question. pub digest: Option, } /// Reads capture metadata for one candidate. /// /// Returning `None` is not an error: a card holds files no decoder in this /// application understands, and they are still imported — dated by mtime, and /// reported as such (FR-NC-7a). pub type Probe<'a> = dyn Fn(&Candidate) -> Option + 'a; /// Answers whether the catalog already holds this photograph (FR-CAT-11). /// /// Called twice per file: once before any transfer with `digest: None`, and /// once after the bytes have been read with the digest filled in. An /// implementation that can only answer the first should return `false` for the /// second rather than guessing. pub type DuplicateCheck<'a> = dyn Fn(&DupKey) -> bool + 'a; /// Progress, reported per file as it completes. pub type OnProgress<'a> = dyn FnMut(&Progress) + 'a; /// Where an import has got to. #[derive(Debug, Clone)] pub struct Progress { /// Files finished, successfully or not. pub done: usize, pub total: usize, /// Bytes transferred so far, backup copies not counted twice. pub bytes: u64, /// The file just finished. pub name: String, } /// One file that made it in. #[derive(Debug, Clone)] pub struct Imported { /// Where it came from. pub source: SourceRef, /// Where it now is, as the destination storage named it — **not** as the /// caller asked for it. A SAF provider renames on collision by itself, so /// reading this back rather than assuming is not defensive style, it is /// the only way to be right on Android (ARCH §6.9). pub written: SourceRef, /// The folders it was filed under, outermost first (FR-NC-7a). The upload /// path expands the same template to the same segments, so this is also /// where it belongs on the server. pub folders: Vec, /// The name it was written under, which differs from the card's when the /// name was taken. pub name: String, /// SHA-256 of its bytes. Stored so the next import can answer FR-CAT-11's /// second tier without re-reading anything. pub digest: String, pub size: u64, /// Whether the folder date came from the shutter or from mtime. pub dated_from: DateSource, /// Where the second copy went, if one was asked for. pub backup: Option, } /// What an import did. #[derive(Debug, Default)] pub struct Report { pub imported: Vec, /// Files the catalog already held (FR-CAT-11). Nothing was transferred. pub duplicates: Vec, pub failed: Vec<(Candidate, IngestError)>, /// Sources whose card copies are now redundant, for a move-import. /// /// **Populated, never acted on.** The caller deletes these once it is /// satisfied the import is safe, which on a syncing library means after the /// upload is confirmed (FR-NC-7b). Empty for a copy-import. pub retirable: Vec, /// Files dated by modification time because they carried no capture time. /// /// FR-NC-7a requires this to be visible rather than silent: they are filed /// under when they were last written, which is often the day the card was /// read rather than the day the photograph was taken. pub undated: Vec, pub bytes: u64, /// Whether the run stopped early because the caller asked it to. pub cancelled: bool, } impl Report { /// Whether anything at all was written. pub fn is_empty(&self) -> bool { self.imported.is_empty() } } /// An import in progress. /// /// A struct rather than a function with nine parameters, and the borrows are /// the point: the source is `&dyn Storage` and the destination is /// `&dyn WritableStorage`, which is exactly the asymmetry of copying off a /// card. A card mounted read-only cannot be passed where a destination is /// wanted, and the compiler says so rather than the filesystem saying so /// halfway through. pub struct Ingest<'a> { pub source: &'a dyn Storage, pub dest: &'a dyn WritableStorage, /// The library root, or a folder inside it. Template folders are created /// beneath this. pub dest_root: &'a DirRef, /// The optional simultaneous second destination (FR-CAT-10). /// /// Written from the same read as the primary, not copied from it /// afterwards: a backup made by re-reading the primary would inherit a bad /// write rather than catch it, and re-reading the *card* doubles the wear /// on the one copy that still exists. pub backup: Option<(&'a dyn WritableStorage, &'a DirRef)>, pub options: Options, } impl Ingest<'_> { /// Run the import. /// /// `cancel` is polled between files rather than within one: a half-written /// RAW is worse than a few more seconds of waiting, and at 1 MiB chunks the /// longest a cancel waits is one file. pub fn run( &self, candidates: &[Candidate], probe: &Probe<'_>, is_duplicate: &DuplicateCheck<'_>, cancel: &dyn Fn() -> bool, on_progress: &mut OnProgress<'_>, ) -> Report { let mut report = Report::default(); let total = candidates.len(); // What each destination folder already holds, listed once per folder // rather than probed per file: a card is two thousand files landing in // a handful of days, so this is a handful of listings against two // thousand round trips through the storage layer. let mut listings: Listings = HashMap::new(); for candidate in candidates { if cancel() { report.cancelled = true; break; } // The probe reads *content*, so the modification time is not its // to report — it comes from the listing that produced the // candidate. Overwritten rather than defaulted, so a probe that // leaves the field at zero cannot file a whole card under 1970. let mut shot = probe(candidate).unwrap_or_default(); shot.modified_at = candidate.modified_at; // Tier one, before a byte moves: the card that was already // imported costs one metadata read per file and no transfer. let key = DupKey { captured_at: shot.captured_at, make: shot.make.clone(), model: shot.model.clone(), original_name: candidate.name.clone(), size: candidate.size, digest: None, }; if self.options.on_duplicate == DuplicatePolicy::Skip && is_duplicate(&key) { report.duplicates.push(candidate.clone()); report.done(&mut *on_progress, total, candidate); continue; } match self.one(candidate, &shot, &key, is_duplicate, &mut listings) { Ok(Some(imported)) => { report.bytes += imported.size; if imported.dated_from == DateSource::Modified { report.undated.push(imported.name.clone()); } if self.options.mode == TransferMode::Move { report.retirable.push(imported.source.clone()); } report.imported.push(imported); } // The digest tier caught it after reading. Nothing was kept. Ok(None) => report.duplicates.push(candidate.clone()), Err(e) => { log::warn!("importing {}: {e}", candidate.name); report.failed.push((candidate.clone(), e)); } } report.done(&mut *on_progress, total, candidate); } report } /// Import one file. `Ok(None)` means the digest tier found a duplicate. fn one( &self, candidate: &Candidate, shot: &Shot, key: &DupKey, is_duplicate: &DuplicateCheck<'_>, listings: &mut Listings, ) -> Result, IngestError> { let destination = layout::expand(&self.options.folder_template, shot); let dir = self.make_dirs(self.dest, self.dest_root, &destination.segments)?; let backup_dir = match self.backup { Some((storage, root)) => Some(( storage, self.make_dirs(storage, root, &destination.segments) .map_err(|e| match e { IngestError::Destination(e) => IngestError::Backup(e), other => other, })?, )), None => None, }; // TRACES: FR-CAT-11 // The card that has already been imported into this very folder. // // Distinct from the catalog's answer and needed alongside it: on a // library whose catalog describes a *server*, a file sitting in the // local destination has no row to be found by, so the question "have I // already copied this" is only answerable by looking. Without this, // re-importing a card fills the folder with `-1` copies of everything // — the rename below is for a genuinely different photograph that // happens to share a name, and it cannot tell the two cases apart on // its own. // // Name plus size, not a digest: this runs before any transfer, and // hashing to answer it would read the whole card to save reading the // whole card. A file of the same name and the same length in the folder // this import would have written it to is the same photograph in every // case that is not deliberately constructed; the digest tier below // still catches the same frame under a *different* name. let existing = self.folder_listing(listings, &dir)?; if self.options.on_duplicate == DuplicatePolicy::Skip && existing.get(&candidate.name) == Some(&candidate.size) { return Ok(None); } let name = self.free_name(self.dest, &dir, &candidate.name)?; let mut reader = self .source .open(&candidate.source) .map_err(IngestError::Source)?; let primary = self .dest .create_file(&dir, &name) .map_err(IngestError::Destination)?; let secondary = match &backup_dir { Some((storage, bdir)) => { let bname = self.free_name(*storage, bdir, &candidate.name)?; Some( storage .create_file(bdir, &bname) .map_err(IngestError::Backup)?, ) } None => None, }; let written = self.stream(&mut reader, primary, secondary); // Any failure past the point of creating the file leaves bytes on // disk that no catalog knows about. Take them back before returning, // or the next scan catalogues a truncated RAW as though it were fine. let (digest, size, target, backup_target) = match written { Ok(v) => v, Err((e, targets)) => { self.discard(targets); return Err(e); } }; if self.options.verify { if let Err(e) = self.verify(&target, &digest, size) { self.discard( vec![(self.dest, target)] .into_iter() .chain(backup_target.map(|t| (self.backup.unwrap().0, t))) .collect(), ); return Err(e); } } // Tier two (FR-CAT-11): the same frame under a different name. Only // answerable now, and cheap to act on — the file is removed rather // than catalogued twice. if self.options.on_duplicate == DuplicatePolicy::Skip { let with_digest = DupKey { digest: Some(digest.clone()), ..key.clone() }; if is_duplicate(&with_digest) { self.discard( vec![(self.dest, target)] .into_iter() .chain(backup_target.map(|t| (self.backup.unwrap().0, t))) .collect(), ); return Ok(None); } } Ok(Some(Imported { source: candidate.source.clone(), written: target, folders: destination.segments, name, digest, size, dated_from: destination.dated_from, backup: backup_target, })) } /// Create each level of the template, keeping what the provider returned. /// /// Never `create_dir_all` on a joined path: the segments are separate /// because on SAF the child's reference can only come from the call that /// created it (ARCH §6.9). fn make_dirs( &self, storage: &dyn WritableStorage, root: &DirRef, segments: &[String], ) -> Result { let mut dir = root.clone(); for segment in segments { dir = storage .create_dir(&dir, segment) .map_err(IngestError::Destination)?; } Ok(dir) } /// What a destination folder holds, listed at most once per run. /// /// A folder that cannot be listed yields an empty map rather than an /// error: the consequence is a re-import that copies what was already /// there, which the digest tier then removes — where failing the import /// would cost the user the photographs that are not yet anywhere. fn folder_listing<'a>( &self, listings: &'a mut Listings, dir: &DirRef, ) -> Result<&'a HashMap, IngestError> { let key = format!("{}:{}", dir.root_id().0, dir.key()); Ok(listings .entry(key) .or_insert_with(|| match self.dest.list(dir) { Ok(entries) => entries .into_iter() .filter(|e| !e.meta.is_dir) .map(|e| (e.meta.name, e.meta.size)) .collect(), Err(e) => { log::warn!("could not list {dir}: {e}"); HashMap::new() } })) } /// A name not already taken in the destination. /// /// Two cards can hold `IMG_0001.CR3` and they are different photographs. /// The suffix goes before the extension so the file still opens. fn free_name( &self, storage: &dyn WritableStorage, dir: &DirRef, name: &str, ) -> Result { if !storage .exists(dir, name) .map_err(IngestError::Destination)? { return Ok(name.to_string()); } let (stem, ext) = match name.rsplit_once('.') { Some((s, e)) if !s.is_empty() => (s, Some(e)), _ => (name, None), }; for n in 1..1000 { let candidate = match ext { Some(ext) => format!("{stem}-{n}.{ext}"), None => format!("{stem}-{n}"), }; if !storage .exists(dir, &candidate) .map_err(IngestError::Destination)? { return Ok(candidate); } } Err(IngestError::Destination(StorageError::AlreadyExists( name.to_string(), ))) } /// Read once, write twice, hash on the way past. /// /// The backup is written from the same read rather than copied from the /// primary afterwards — see [`Ingest::backup`]. On error the created files /// come back with it so the caller can take them away again. #[allow(clippy::type_complexity)] fn stream( &self, reader: &mut Box, primary: NewFile, secondary: Option, ) -> Result< (String, u64, SourceRef, Option), (IngestError, Vec<(&dyn WritableStorage, SourceRef)>), > { let NewFile { source: target, sink: mut out, } = primary; let (backup_target, mut backup_out) = match secondary { Some(NewFile { source, sink }) => (Some(source), Some(sink)), None => (None, None), }; let created = |extra: Option<&SourceRef>| -> Vec<(&dyn WritableStorage, SourceRef)> { let mut v: Vec<(&dyn WritableStorage, SourceRef)> = vec![(self.dest, target.clone())]; if let (Some(t), Some((storage, _))) = (extra, self.backup) { v.push((storage, t.clone())); } v }; let mut hasher = Sha256::new(); let mut buf = vec![0u8; CHUNK]; let mut size = 0u64; loop { let n = match reader.read(&mut buf) { Ok(0) => break, Ok(n) => n, Err(e) => { return Err(( IngestError::Source(StorageError::Io(e.to_string())), created(backup_target.as_ref()), )) } }; let chunk = &buf[..n]; hasher.update(chunk); size += n as u64; if let Err(e) = out.write_all(chunk) { return Err(( IngestError::Destination(StorageError::Io(e.to_string())), created(backup_target.as_ref()), )); } if let Some(b) = backup_out.as_mut() { if let Err(e) = b.write_all(chunk) { return Err(( IngestError::Backup(StorageError::Io(e.to_string())), created(backup_target.as_ref()), )); } } } // Flush explicitly rather than on drop. A buffered writer swallows the // error when it is dropped, and "the disk filled on the last chunk" is // exactly the failure this whole path exists to catch. if let Err(e) = out.flush() { return Err(( IngestError::Destination(StorageError::Io(e.to_string())), created(backup_target.as_ref()), )); } if let Some(b) = backup_out.as_mut() { if let Err(e) = b.flush() { return Err(( IngestError::Backup(StorageError::Io(e.to_string())), created(backup_target.as_ref()), )); } } drop(out); drop(backup_out); Ok((hex(&hasher.finalize()), size, target, backup_target)) } /// Re-read what was written and confirm it is what was sent. /// /// Reads the *destination*, not the buffer that was just hashed. Hashing /// what is still in memory would confirm only that the process can add — it /// would pass on a full disk, a dying card and a truncated write alike. fn verify(&self, target: &SourceRef, expected: &str, size: u64) -> Result<(), IngestError> { let mut reader = self.dest.open(target).map_err(|e| IngestError::Verify { written: size, reason: format!("cannot re-read it: {e}"), })?; let mut hasher = Sha256::new(); let mut buf = vec![0u8; CHUNK]; let mut seen = 0u64; loop { let n = reader.read(&mut buf).map_err(|e| IngestError::Verify { written: size, reason: format!("cannot re-read it: {e}"), })?; if n == 0 { break; } hasher.update(&buf[..n]); seen += n as u64; } let got = hex(&hasher.finalize()); if seen != size { return Err(IngestError::Verify { written: size, reason: format!("{seen} bytes came back"), }); } if got != expected { return Err(IngestError::Verify { written: size, reason: format!("digest {got} does not match {expected}"), }); } Ok(()) } /// Take back files this import created but is not keeping. /// /// Failures here are logged and not propagated: this runs on a path that /// already has an error to report, and replacing it with "and the cleanup /// also failed" loses the one the user needs. fn discard(&self, targets: Vec<(&dyn WritableStorage, SourceRef)>) { for (storage, target) in targets { if let Err(e) = storage.remove_file(&target) { log::warn!("could not remove a partial import: {e}"); } } } } impl Report { fn done(&self, on_progress: &mut OnProgress<'_>, total: usize, candidate: &Candidate) { on_progress(&Progress { done: self.duplicates.len() + self.imported.len() + self.failed.len(), total, bytes: self.bytes, name: candidate.name.clone(), }); } } /// TRACES: FR-CAT-10 | FR-NC-7b /// Delete the card copies of files that are safely in the library. /// /// Separate from [`Ingest::run`] and deliberately awkward to reach. Pass only /// [`Report::retirable`], and only once whatever "safe" means for this library /// is true — for one that syncs, that is the upload confirmed, not the local /// write returned (FR-NC-7b). There is no undo behind this function: the card /// is where the only other copy was. /// /// Returns the sources it could not delete, which is a card pulled mid-run or /// a write-protect switch, and neither is worth stopping for. pub fn retire(card: &dyn WritableStorage, sources: &[SourceRef]) -> Vec<(SourceRef, StorageError)> { let mut failed = Vec::new(); for src in sources { if let Err(e) = card.remove_file(src) { log::warn!("could not retire {src:?} from the card: {e}"); failed.push((src.clone(), e)); } } failed } /// A digest as lower-case hex, the form `sha256sum` prints. fn hex(bytes: &[u8]) -> String { let mut s = String::with_capacity(bytes.len() * 2); for b in bytes { s.push_str(&format!("{b:02x}")); } s } #[cfg(test)] mod tests { use super::*; use std::cell::RefCell; use std::path::PathBuf; use dr_plat::{Entry, LocalStorage}; use dr_types::{ByteRange, DirState, RootId}; pub(super) const CARD: RootId = RootId(1); pub(super) const LIB: RootId = RootId(2); /// 2026-08-22T14:00:00Z. const AUG_22: i64 = 1_787_407_200; /// A throwaway directory, removed when the test ends. pub(super) struct Tree(PathBuf); impl Tree { fn new(name: &str) -> Self { let dir = std::env::temp_dir().join(format!( "dr-ingest-{name}-{}-{:?}", std::process::id(), std::thread::current().id() )); let _ = std::fs::remove_dir_all(&dir); std::fs::create_dir_all(&dir).expect("temp dir"); Tree(dir) } fn sub(&self, rel: &str) -> PathBuf { let p = self.0.join(rel); std::fs::create_dir_all(&p).expect("mkdir"); p } } impl Drop for Tree { fn drop(&mut self) { let _ = std::fs::remove_dir_all(&self.0); } } /// A card with files on it, and the library it imports into. pub(super) struct Fixture { _tree: Tree, pub(super) card: LocalStorage, card_path: PathBuf, pub(super) lib: LocalStorage, pub(super) lib_path: PathBuf, } pub(super) fn fixture(name: &str) -> Fixture { let tree = Tree::new(name); let card_path = tree.sub("card/DCIM/100CANON"); let lib_path = tree.sub("library"); Fixture { card: LocalStorage::with_root(CARD, tree.0.join("card")), lib: LocalStorage::with_root(LIB, lib_path.clone()), card_path, lib_path, _tree: tree, } } impl Fixture { /// Put a file on the card and return the candidate for it. pub(super) fn shoot(&self, name: &str, bytes: &[u8]) -> Candidate { std::fs::write(self.card_path.join(name), bytes).expect("write"); Candidate { source: SourceRef::Local { root: CARD, relative: format!("DCIM/100CANON/{name}"), }, name: name.to_string(), size: bytes.len() as u64, modified_at: AUG_22, } } pub(super) fn lib_root(&self) -> DirRef { DirRef::root(LIB) } pub(super) fn read(&self, rel: &str) -> Vec { std::fs::read(self.lib_path.join(rel)).expect("read imported file") } } /// A `Shot` that dates to the 22nd of August 2026. fn on_the_22nd() -> Shot { Shot { captured_at: Some(AUG_22), make: Some("Canon".into()), model: Some("EOS R5".into()), ..Default::default() } } pub(super) fn never(_: &DupKey) -> bool { false } fn run( f: &Fixture, candidates: &[Candidate], options: Options, probe: &Probe<'_>, is_duplicate: &DuplicateCheck<'_>, ) -> Report { let root = f.lib_root(); let ingest = Ingest { source: &f.card, dest: &f.lib, dest_root: &root, backup: None, options, }; ingest.run(candidates, probe, is_duplicate, &|| false, &mut |_| {}) } #[test] fn a_card_is_filed_by_year_and_day() { let f = fixture("by-date"); let c = f.shoot("IMG_0001.CR3", b"raw bytes"); let report = run( &f, &[c], Options::default(), &|_| Some(on_the_22nd()), &never, ); assert_eq!(report.imported.len(), 1); let imported = &report.imported[0]; assert_eq!(imported.folders, ["2026", "2026-08-22"]); assert_eq!(f.read("2026/2026-08-22/IMG_0001.CR3"), b"raw bytes"); // The digest is stored so the next import can answer FR-CAT-11's // second tier without reading the file again. assert_eq!(imported.digest.len(), 64); assert_eq!(report.bytes, 9); } #[test] fn the_digest_is_the_one_sha256sum_would_print() { let f = fixture("digest"); let c = f.shoot("IMG_0001.CR3", b"abc"); let report = run( &f, &[c], Options::default(), &|_| Some(on_the_22nd()), &never, ); // Stored and compared across devices and across years, so it has to be // a value someone can recompute at a shell prompt. assert_eq!( report.imported[0].digest, "ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad" ); } #[test] fn a_card_that_was_already_imported_transfers_nothing() { let f = fixture("dupe-cheap"); let c = f.shoot("IMG_0001.CR3", b"raw bytes"); // Tier one: answered from metadata, before a byte moves. let asked: RefCell> = RefCell::new(Vec::new()); let report = run( &f, &[c], Options::default(), &|_| Some(on_the_22nd()), &|k| { asked.borrow_mut().push(k.clone()); true }, ); assert_eq!(report.duplicates.len(), 1); assert!(report.imported.is_empty()); assert!(!f.lib_path.join("2026").exists(), "nothing was written"); // Asked exactly once, without a digest — the whole point is that the // file was never read. assert_eq!(asked.borrow().len(), 1); assert_eq!(asked.borrow()[0].digest, None); assert_eq!(asked.borrow()[0].original_name, "IMG_0001.CR3"); } #[test] fn the_same_frame_under_a_different_name_is_caught_by_its_digest() { let f = fixture("dupe-digest"); let c = f.shoot("DSC_4242.NEF", b"the same photograph"); // Tier two: the metadata key misses (the name differs), so the file is // read — and then must not be kept. let report = run( &f, &[c], Options::default(), &|_| Some(on_the_22nd()), &|k| k.digest.is_some(), ); assert_eq!(report.duplicates.len(), 1); assert!(report.imported.is_empty()); assert!( !f.lib_path.join("2026/2026-08-22/DSC_4242.NEF").exists(), "the copy made to compute the digest was taken back" ); } #[test] fn importing_as_new_keeps_a_known_duplicate() { let f = fixture("dupe-keep"); let c = f.shoot("IMG_0001.CR3", b"raw bytes"); let report = run( &f, &[c], Options { on_duplicate: DuplicatePolicy::ImportAsNew, ..Default::default() }, &|_| Some(on_the_22nd()), &|_| true, ); assert_eq!(report.imported.len(), 1); assert!(report.duplicates.is_empty()); } #[test] fn two_cards_holding_the_same_filename_do_not_overwrite() { let f = fixture("collision"); // Camera filenames wrap at IMG_9999, so this is ordinary rather than // exotic: two different photographs, one name. let first = f.shoot("IMG_0001.CR3", b"the first frame"); let report = run( &f, &[first], Options::default(), &|_| Some(on_the_22nd()), &never, ); assert_eq!(report.imported[0].name, "IMG_0001.CR3"); let second = f.shoot("IMG_0001.CR3", b"a different frame"); let report = run( &f, &[second], Options::default(), &|_| Some(on_the_22nd()), &never, ); assert_eq!(report.imported[0].name, "IMG_0001-1.CR3"); // The suffix goes before the extension, so the file still opens. assert_eq!(f.read("2026/2026-08-22/IMG_0001.CR3"), b"the first frame"); assert_eq!( f.read("2026/2026-08-22/IMG_0001-1.CR3"), b"a different frame" ); } #[test] fn a_file_with_no_capture_time_is_reported_not_hidden() { let f = fixture("undated"); let c = f.shoot("SCAN_01.TIF", b"scanned film"); // No decoder understands it, so it is dated by mtime — and the user is // told, because mtime is often the day the card was read (FR-NC-7a). let report = run(&f, &[c], Options::default(), &|_| None, &never); assert_eq!(report.imported.len(), 1); assert_eq!(report.imported[0].dated_from, DateSource::Modified); assert_eq!(report.undated, ["SCAN_01.TIF"]); assert_eq!(f.read("2026/2026-08-22/SCAN_01.TIF"), b"scanned film"); } #[test] fn a_probe_that_forgets_the_modification_time_does_not_file_a_card_under_1970() { let f = fixture("probe-zero"); let c = f.shoot("IMG_0001.CR3", b"x"); // A probe returns content metadata and nothing else. If its zeroed // mtime were trusted, a card of undated files would land in 1970. let report = run( &f, &[c], Options::default(), &|_| Some(Shot::default()), &never, ); assert_eq!(report.imported[0].folders, ["2026", "2026-08-22"]); } #[test] fn a_backup_copy_is_written_from_the_same_read() { let f = fixture("backup"); let backup_path = f._tree.sub("backup"); let backup = LocalStorage::with_root(RootId(3), backup_path.clone()); let c = f.shoot("IMG_0001.CR3", b"raw bytes"); let root = f.lib_root(); let backup_root = DirRef::root(RootId(3)); let ingest = Ingest { source: &f.card, dest: &f.lib, dest_root: &root, backup: Some((&backup, &backup_root)), options: Options::default(), }; let report = ingest.run( &[c], &|_| Some(on_the_22nd()), &never, &|| false, &mut |_| {}, ); assert_eq!(report.imported.len(), 1); assert!(report.imported[0].backup.is_some()); // The same folder template on both sides, so the backup is a library // rather than a heap. assert_eq!( std::fs::read(backup_path.join("2026/2026-08-22/IMG_0001.CR3")).unwrap(), b"raw bytes" ); // One read of the card produced both, so the byte count is not doubled. assert_eq!(report.bytes, 9); } #[test] fn an_unreadable_file_does_not_stop_the_card() { let f = fixture("partial-failure"); let good = f.shoot("IMG_0001.CR3", b"fine"); // A file the listing saw and the reader cannot open — a card on its // way out. The other nine hundred images still have to import. let missing = Candidate { source: SourceRef::Local { root: CARD, relative: "DCIM/100CANON/GONE.CR3".into(), }, name: "GONE.CR3".into(), size: 4, modified_at: AUG_22, }; let also_good = f.shoot("IMG_0002.CR3", b"also fine"); let report = run( &f, &[good, missing, also_good], Options::default(), &|_| Some(on_the_22nd()), &never, ); assert_eq!(report.imported.len(), 2); assert_eq!(report.failed.len(), 1); assert_eq!(report.failed[0].0.name, "GONE.CR3"); // And nothing was left behind for the next scan to catalogue. assert!(!f.lib_path.join("2026/2026-08-22/GONE.CR3").exists()); } #[test] fn cancelling_stops_between_files_rather_than_inside_one() { let f = fixture("cancel"); let a = f.shoot("IMG_0001.CR3", b"first"); let b = f.shoot("IMG_0002.CR3", b"second"); let c = f.shoot("IMG_0003.CR3", b"third"); let seen = RefCell::new(0usize); let root = f.lib_root(); let ingest = Ingest { source: &f.card, dest: &f.lib, dest_root: &root, backup: None, options: Options::default(), }; let report = ingest.run( &[a, b, c], &|_| Some(on_the_22nd()), &never, &|| { let mut n = seen.borrow_mut(); *n += 1; // Cancel is asked before each file: allow two, refuse the third. *n > 2 }, &mut |_| {}, ); assert!(report.cancelled); assert_eq!(report.imported.len(), 2); // Whole files or nothing — a half-written RAW is worse than waiting. assert_eq!(f.read("2026/2026-08-22/IMG_0002.CR3"), b"second"); assert!(!f.lib_path.join("2026/2026-08-22/IMG_0003.CR3").exists()); } #[test] fn progress_counts_every_file_including_the_ones_it_skipped() { let f = fixture("progress"); let a = f.shoot("IMG_0001.CR3", b"first"); let b = f.shoot("IMG_0002.CR3", b"second"); let ticks: RefCell> = RefCell::new(Vec::new()); let root = f.lib_root(); let ingest = Ingest { source: &f.card, dest: &f.lib, dest_root: &root, backup: None, options: Options::default(), }; ingest.run( &[a, b], &|_| Some(on_the_22nd()), &never, &|| false, &mut |p| ticks.borrow_mut().push((p.done, p.total)), ); assert_eq!(*ticks.borrow(), [(1, 2), (2, 2)]); } #[test] fn a_move_import_records_what_may_be_deleted_but_deletes_nothing() { let f = fixture("move"); let c = f.shoot("IMG_0001.CR3", b"raw bytes"); let on_card = f.card_path.join("IMG_0001.CR3"); let report = run( &f, &[c], Options { mode: TransferMode::Move, ..Default::default() }, &|_| Some(on_the_22nd()), &never, ); assert_eq!(report.retirable.len(), 1); // The card is still intact. On a syncing library the deletion waits for // the upload to be confirmed (FR-NC-7b), and this crate cannot know // that it was. assert!(on_card.exists()); // Only when the caller says so. let failed = retire(&f.card, &report.retirable); assert!(failed.is_empty()); assert!(!on_card.exists()); } #[test] fn a_copy_import_marks_nothing_for_deletion() { let f = fixture("copy-keeps"); let c = f.shoot("IMG_0001.CR3", b"raw bytes"); let report = run( &f, &[c], Options::default(), &|_| Some(on_the_22nd()), &never, ); assert!(report.retirable.is_empty()); } // ---- verification (FR-CAT-10) --------------------------------------- /// A destination that writes fewer bytes than it was given. /// /// What a card on its way out, or a filesystem that reported a successful /// write into a full disk, looks like from here. Verification exists for /// exactly this, so it needs a way to happen on purpose. struct Truncating(LocalStorage); struct ShortSink(Box); impl std::io::Write for ShortSink { fn write(&mut self, buf: &[u8]) -> std::io::Result { // Drops the last byte of every chunk and claims it wrote them all, // which is precisely the failure a digest catches and a byte count // alone would not. let keep = buf.len().saturating_sub(1); self.0.write_all(&buf[..keep])?; Ok(buf.len()) } fn flush(&mut self) -> std::io::Result<()> { self.0.flush() } } impl Storage for Truncating { fn roots(&self) -> Vec { self.0.roots() } fn root_dir(&self, root: RootId) -> Result { self.0.root_dir(root) } fn dir_state(&self, dir: &DirRef) -> Result { self.0.dir_state(dir) } fn list(&self, dir: &DirRef) -> Result, StorageError> { self.0.list(dir) } fn open(&self, src: &SourceRef) -> Result, StorageError> { self.0.open(src) } fn read_range(&self, src: &SourceRef, range: ByteRange) -> Result, StorageError> { self.0.read_range(src, range) } } impl WritableStorage for Truncating { fn create_dir(&self, parent: &DirRef, name: &str) -> Result { self.0.create_dir(parent, name) } fn create_file(&self, parent: &DirRef, name: &str) -> Result { let nf = self.0.create_file(parent, name)?; Ok(NewFile { source: nf.source, sink: Box::new(ShortSink(nf.sink)), }) } fn exists(&self, parent: &DirRef, name: &str) -> Result { self.0.exists(parent, name) } fn remove_file(&self, src: &SourceRef) -> Result<(), StorageError> { self.0.remove_file(src) } } #[test] fn a_corrupt_write_is_caught_and_taken_back() { let f = fixture("verify-fails"); let c = f.shoot("IMG_0001.CR3", b"raw bytes"); let lib = Truncating(LocalStorage::with_root(LIB, f.lib_path.clone())); let root = DirRef::root(LIB); let ingest = Ingest { source: &f.card, dest: &lib, dest_root: &root, backup: None, options: Options::default(), }; let report = ingest.run( &[c], &|_| Some(on_the_22nd()), &never, &|| false, &mut |_| {}, ); assert!(report.imported.is_empty()); assert_eq!(report.failed.len(), 1); assert!( matches!(report.failed[0].1, IngestError::Verify { .. }), "{:?}", report.failed[0].1 ); // Removed rather than left: a truncated RAW on disk would be found by // the next scan and catalogued as though it were fine. assert!(!f.lib_path.join("2026/2026-08-22/IMG_0001.CR3").exists()); } #[test] fn verification_can_be_turned_off_and_then_nothing_catches_it() { let f = fixture("verify-off"); let c = f.shoot("IMG_0001.CR3", b"raw bytes"); let lib = Truncating(LocalStorage::with_root(LIB, f.lib_path.clone())); let root = DirRef::root(LIB); let ingest = Ingest { source: &f.card, dest: &lib, dest_root: &root, backup: None, options: Options { verify: false, ..Default::default() }, }; let report = ingest.run( &[c], &|_| Some(on_the_22nd()), &never, &|| false, &mut |_| {}, ); // Documents the cost of the option rather than endorsing it: the // import "succeeds" and the file on disk is short. assert_eq!(report.imported.len(), 1); assert_eq!(f.read("2026/2026-08-22/IMG_0001.CR3"), b"raw byte"); } #[test] fn a_file_larger_than_one_chunk_survives_the_round_trip() { let f = fixture("large"); // Streaming is the whole design — a RAW is 20–100 MB and never sits in // memory. Anything that only works below the buffer size is a bug this // catches. let bytes: Vec = (0..(CHUNK * 2 + 1234)).map(|i| (i % 251) as u8).collect(); let c = f.shoot("IMG_0001.CR3", &bytes); let report = run( &f, &[c], Options::default(), &|_| Some(on_the_22nd()), &never, ); assert_eq!(report.imported.len(), 1); assert_eq!(report.imported[0].size, bytes.len() as u64); assert_eq!(f.read("2026/2026-08-22/IMG_0001.CR3"), bytes); } // ---- shared with `already_there_tests` ------------------------------- /// A probe that dates everything to the 22nd of August 2026. pub(super) fn probe_22nd(_: &Candidate) -> Option { Some(on_the_22nd()) } /// Import a card with the defaults, against a catalog that knows nothing. pub(super) fn run_once(f: &Fixture, candidates: &[Candidate]) -> Report { let root = f.lib_root(); let ingest = Ingest { source: &f.card, dest: &f.lib, dest_root: &root, backup: None, options: Options::default(), }; ingest.run(candidates, &probe_22nd, &never, &|| false, &mut |_| {}) } /// A destination that counts how many times it is listed. /// /// The listing is the whole cost of knowing what a folder already holds, /// so "once per folder" is a property worth a test rather than a comment. pub(super) struct CountingList { inner: LocalStorage, listings: std::cell::Cell, } impl CountingList { pub(super) fn new(inner: LocalStorage) -> Self { Self { inner, listings: std::cell::Cell::new(0), } } pub(super) fn listings(&self) -> usize { self.listings.get() } } // `Cell` is not `Sync`, and `Storage` requires it. Sound here because the // test drives this from one thread; spelled out rather than left implicit. unsafe impl Sync for CountingList {} impl Storage for CountingList { fn roots(&self) -> Vec { self.inner.roots() } fn root_dir(&self, root: RootId) -> Result { self.inner.root_dir(root) } fn dir_state(&self, dir: &DirRef) -> Result { self.inner.dir_state(dir) } fn list(&self, dir: &DirRef) -> Result, StorageError> { self.listings.set(self.listings.get() + 1); self.inner.list(dir) } fn open(&self, src: &SourceRef) -> Result, StorageError> { self.inner.open(src) } fn read_range(&self, src: &SourceRef, range: ByteRange) -> Result, StorageError> { self.inner.read_range(src, range) } } impl WritableStorage for CountingList { fn create_dir(&self, parent: &DirRef, name: &str) -> Result { self.inner.create_dir(parent, name) } fn create_file(&self, parent: &DirRef, name: &str) -> Result { self.inner.create_file(parent, name) } fn exists(&self, parent: &DirRef, name: &str) -> Result { self.inner.exists(parent, name) } fn remove_file(&self, src: &SourceRef) -> Result<(), StorageError> { self.inner.remove_file(src) } } } #[cfg(test)] mod already_there_tests { use super::tests::*; use super::*; use dr_plat::LocalStorage; #[test] fn re_importing_the_same_card_copies_nothing() { let f = fixture("already-there"); let c = f.shoot("IMG_0001.CR3", b"raw bytes"); // First import: it lands. let first = run_once(&f, std::slice::from_ref(&c)); assert_eq!(first.imported.len(), 1); // Second import of the same card, with a catalog that knows nothing — // which is the state on a library whose catalog describes a server. // Without the folder listing this produced IMG_0001-1.CR3. let second = run_once(&f, &[c]); assert_eq!(second.duplicates.len(), 1); assert!(second.imported.is_empty()); assert!( !f.lib_path.join("2026/2026-08-22/IMG_0001-1.CR3").exists(), "a second copy was written under a new name" ); } #[test] fn a_different_photograph_of_the_same_name_still_lands() { let f = fixture("same-name-different-frame"); let first = f.shoot("IMG_0001.CR3", b"the first frame"); assert_eq!(run_once(&f, &[first]).imported.len(), 1); // Camera filenames wrap at IMG_9999, so this is ordinary. Different // length, so the size check separates it from the one already there. let second = f.shoot("IMG_0001.CR3", b"a different and longer frame"); let report = run_once(&f, &[second]); assert_eq!(report.imported.len(), 1); assert_eq!(report.imported[0].name, "IMG_0001-1.CR3"); } #[test] fn importing_again_deliberately_still_works() { let f = fixture("already-there-forced"); let c = f.shoot("IMG_0001.CR3", b"raw bytes"); assert_eq!(run_once(&f, std::slice::from_ref(&c)).imported.len(), 1); // The user asked for it, so the folder listing must not veto them. let root = f.lib_root(); let ingest = Ingest { source: &f.card, dest: &f.lib, dest_root: &root, backup: None, options: Options { on_duplicate: DuplicatePolicy::ImportAsNew, ..Default::default() }, }; let report = ingest.run(&[c], &probe_22nd, &never, &|| false, &mut |_| {}); assert_eq!(report.imported.len(), 1); assert_eq!(report.imported[0].name, "IMG_0001-1.CR3"); } #[test] fn a_folder_is_listed_once_however_many_files_land_in_it() { // The property that keeps this affordable: a card of two thousand // frames from one day is one listing, not two thousand. let f = fixture("one-listing"); let candidates: Vec = (0..12) .map(|i| f.shoot(&format!("IMG_{i:04}.CR3"), format!("frame {i}").as_bytes())) .collect(); let counting = CountingList::new(LocalStorage::with_root(LIB, f.lib_path.clone())); let root = DirRef::root(LIB); let ingest = Ingest { source: &f.card, dest: &counting, dest_root: &root, backup: None, options: Options::default(), }; let report = ingest.run(&candidates, &probe_22nd, &never, &|| false, &mut |_| {}); assert_eq!(report.imported.len(), 12); // One for the day folder. `create_dir` and `exists` do not list. assert_eq!(counting.listings(), 1); } }