diff --git a/Cargo.lock b/Cargo.lock index a897932..5b5b749 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1461,6 +1461,18 @@ dependencies = [ "wgpu", ] +[[package]] +name = "dr-ingest" +version = "0.1.0" +dependencies = [ + "dr-plat", + "dr-types", + "env_logger", + "log", + "sha2", + "thiserror 2.0.20", +] + [[package]] name = "dr-lens" version = "0.1.0" diff --git a/Cargo.toml b/Cargo.toml index b257625..d2bc93f 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -6,6 +6,7 @@ members = [ "core/dr-thumbs", "core/dr-decode", "core/dr-export", + "core/dr-ingest", "core/dr-gpu", "core/dr-lens", "core/dr-pipeline", @@ -33,6 +34,7 @@ dr-catalog = { path = "core/dr-catalog" } dr-thumbs = { path = "core/dr-thumbs" } dr-decode = { path = "core/dr-decode" } dr-export = { path = "core/dr-export" } +dr-ingest = { path = "core/dr-ingest" } dr-gpu = { path = "core/dr-gpu" } dr-lens = { path = "core/dr-lens" } dr-pipeline = { path = "core/dr-pipeline" } diff --git a/core/dr-ingest/Cargo.toml b/core/dr-ingest/Cargo.toml new file mode 100644 index 0000000..8f42f99 --- /dev/null +++ b/core/dr-ingest/Cargo.toml @@ -0,0 +1,30 @@ +[package] +name = "dr-ingest" +version.workspace = true +edition.workspace = true +rust-version.workspace = true +license.workspace = true + +[dependencies] +dr-types.workspace = true +# The `Storage` and `WritableStorage` traits. An import reads a real card and +# writes a real destination, and this is how `core/` reaches the platform +# without a `#[cfg(target_os)]` of its own (ARCH §4.1) — the same dependency +# `dr-catalog` takes, for the same reason. +dr-plat.workspace = true +thiserror.workspace = true +log.workspace = true + +# Content hashing: copy verification (FR-CAT-10) and the second duplicate tier +# (FR-CAT-11). Pure Rust, no C, so it cross-compiles under the NDK like every +# other choice in this tree — and it is already in the lock file as a +# transitive dependency, so it costs no new build. +# +# SHA-256 rather than a fast non-cryptographic hash, which would be sufficient +# for verifying a copy: the digest is *stored* and compared across devices and +# across years, so it wants to be a named standard someone can recompute with +# `sha256sum` rather than whatever we happened to implement. +sha2 = "0.10" + +[dev-dependencies] +env_logger.workspace = true diff --git a/core/dr-ingest/src/layout.rs b/core/dr-ingest/src/layout.rs new file mode 100644 index 0000000..7035f46 --- /dev/null +++ b/core/dr-ingest/src/layout.rs @@ -0,0 +1,343 @@ +//! TRACES: FR-CAT-10 | FR-NC-7a +//! Where an imported photograph is filed. +//! +//! One template, two destinations. A card import expands it into directories +//! under a library root (FR-CAT-10); an upload expands the *same* template +//! into folders on the server (FR-NC-7a). They must agree, because a library +//! that is `2026/2026-08-22/` locally and something else remotely is two +//! libraries that happen to hold the same files. +//! +//! # Why this returns segments rather than a path +//! +//! Neither caller can use a path. The local side creates each level through +//! [`dr_plat::WritableStorage`], which takes a parent reference plus one name +//! because a SAF document id is not composable (ARCH §6.9). The remote side +//! joins onto a `RemotePath`, which normalises separators its own way. A +//! `String` with slashes in it would be taken apart again by both, and the +//! taking-apart is where a `..` or an empty segment would slip through. +//! +//! So expansion hands back `Vec`, each element already checked to be +//! one ordinary directory name. + +use dr_types::civil_from_unix_at; + +/// What a folder template can refer to. +/// +/// Deliberately not the decoder's `Metadata`: this crate does not depend on +/// `dr-decode` (see the crate docs), and a template needs six fields out of +/// twenty. The caller fills it from whatever it read. +#[derive(Debug, Clone, Default, PartialEq, Eq)] +pub struct Shot { + /// When the shutter fired, Unix seconds, as EXIF recorded it. + pub captured_at: Option, + /// Minutes east of UTC where the camera recorded a zone. + pub captured_offset: Option, + pub make: Option, + pub model: Option, + /// The file's modification time, Unix seconds — the fallback when there is + /// no capture time, and the reason [`Shot`] always dates to *something*. + pub modified_at: i64, +} + +/// Which reading the folder name came from. +/// +/// FR-NC-7a requires the fallback to be visible in the import report rather +/// than silent. An image filed by mtime is filed by when it was last *copied*, +/// which for a card that has been through a card reader is often today — so a +/// user seeing a folder full of unrelated dates needs to be told why. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum DateSource { + /// EXIF capture time, in the zone the camera recorded. + Capture, + /// File modification time. Not when the photograph was taken. + Modified, +} + +/// The default layout: one directory per year, one per capture day beneath it. +/// +/// `2026/2026-08-22`. The day folder repeats the year rather than being bare +/// `08-22`, because a folder is often seen out of its parent's context — in a +/// file manager's recent list, in a Nextcloud share link, in a backup listing +/// — and `08-22` alone does not say which year it belongs to. +pub const DEFAULT_TEMPLATE: &str = "{yyyy}/{yyyy}-{mm}-{dd}"; + +/// An expanded destination. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct Destination { + /// Directory names, outermost first. Each is one ordinary name: never + /// empty, never `.` or `..`, never containing a separator. + pub segments: Vec, + /// Which timestamp produced the date tokens. + pub dated_from: DateSource, +} + +impl Destination { + /// The layout as a display string — for a confirmation dialog and for logs. + pub fn display(&self) -> String { + self.segments.join("/") + } +} + +/// Expand a folder template for one photograph. +/// +/// Unknown tokens are left verbatim, matching `dr_export::name::expand`: a +/// user who typed `{yyy}` should see it in the folder name and understand what +/// happened, where a silently dropped token produces a directory whose name +/// says nothing about why it is called that. +/// +/// The result is **a pure function of the template and the shot**, which +/// FR-NC-7a requires: two devices uploading the same frame must compute the +/// same destination, so nothing here may consult local library layout, the +/// current time, or a counter. +pub fn expand(template: &str, shot: &Shot) -> Destination { + let (when, dated_from) = match shot.captured_at { + Some(t) => (t, DateSource::Capture), + None => (shot.modified_at, DateSource::Modified), + }; + // The offset applies only to a capture reading. An mtime is already an + // absolute instant with no camera zone attached to it, and adding the + // camera's offset to it would be arithmetic on two unrelated clocks. + let civil = match dated_from { + DateSource::Capture => civil_from_unix_at(when, shot.captured_offset), + DateSource::Modified => civil_from_unix_at(when, None), + }; + + let year = civil.year.to_string(); + let short_year = format!("{:02}", civil.year.rem_euclid(100)); + let month = format!("{:02}", civil.month); + let day = format!("{:02}", civil.day); + + let mut out = String::with_capacity(template.len() + 16); + let mut rest = template; + while let Some(open) = rest.find('{') { + out.push_str(&rest[..open]); + let Some(close) = rest[open..].find('}') else { + // An unclosed brace is literal text. Consumed here rather than + // left to the tail append, which has already had everything + // before the brace taken from it. + out.push_str(&rest[open..]); + rest = ""; + break; + }; + let token = &rest[open + 1..open + close]; + match token { + "yyyy" => out.push_str(&year), + "yy" => out.push_str(&short_year), + "mm" => out.push_str(&month), + "dd" => out.push_str(&day), + "date" => out.push_str(&format!("{year}-{month}-{day}")), + "make" => out.push_str(shot.make.as_deref().unwrap_or_default()), + "model" => out.push_str(shot.model.as_deref().unwrap_or_default()), + _ => out.push_str(&rest[open..open + close + 1]), + } + rest = &rest[open + close + 1..]; + } + out.push_str(rest); + + let segments: Vec = out + .split('/') + .map(sanitise_segment) + // Empty segments are dropped rather than kept: a template of + // `{make}/{yyyy}` on a camera whose make is unknown must not produce + // a nameless directory level, and `//` from a typo must not either. + .filter(|s| !s.is_empty()) + .collect(); + + // A template that expanded to nothing at all — `{make}` alone, on a file + // with no make. Falling back to the date is the one answer always + // available, and it is what the user asked for in every other case. + let segments = if segments.is_empty() { + vec![year.clone(), format!("{year}-{month}-{day}")] + } else { + segments + }; + + Destination { + segments, + dated_from, + } +} + +/// Reduce one path segment to something every destination will accept. +/// +/// The same intersection of rules as `dr_export::name::sanitise` — Linux, +/// SAF and WebDAV, plus Windows for the sync client downstream — applied to a +/// directory name rather than a filename stem. Kept apart from the exporter's +/// because a segment has one rule a stem does not: `.` and `..` are legal +/// filenames and are *not* legal directory names here, since both would move +/// the destination rather than name it. +fn sanitise_segment(seg: &str) -> String { + let mut out: String = seg + .chars() + .map(|c| match c { + '/' | '\\' | ':' | '*' | '?' | '"' | '<' | '>' | '|' => '-', + c if (c as u32) < 0x20 => '-', + c => c, + }) + .collect(); + // Trailing dots and spaces are legal on Linux and rejected by Windows, and + // a name ending in one is almost always a token that expanded to nothing. + while out.ends_with('.') || out.ends_with(' ') { + out.pop(); + } + let out = out.trim_start().to_string(); + // What is left of `.` or `..` after that pop is an empty string, which the + // caller drops. Spelled out rather than relied upon, because a segment + // that climbed out of the destination would be the one bug in this file + // that mattered. + if out == "." || out == ".." { + return String::new(); + } + out +} + +#[cfg(test)] +mod tests { + use super::*; + + /// 2026-08-22T14:00:00Z. + const AUG_22: i64 = 1_787_407_200; + + fn shot(captured_at: Option) -> Shot { + Shot { + captured_at, + modified_at: AUG_22, + ..Default::default() + } + } + + #[test] + fn the_default_template_is_a_year_then_a_day() { + let d = expand(DEFAULT_TEMPLATE, &shot(Some(AUG_22))); + assert_eq!(d.segments, ["2026", "2026-08-22"]); + assert_eq!(d.dated_from, DateSource::Capture); + } + + #[test] + fn the_day_folder_is_zero_padded_so_a_year_sorts() { + // 2026-01-05T00:00:00Z. Unpadded, `2026-1-5` sorts after `2026-10-1` + // in every file manager there is. + let d = expand(DEFAULT_TEMPLATE, &shot(Some(1_767_571_200))); + assert_eq!(d.segments, ["2026", "2026-01-05"]); + } + + #[test] + fn a_shot_is_filed_in_the_cameras_own_day_not_utcs() { + // 22:30 UTC on the 21st is already the 22nd in Tokyo. A night shoot + // must not be split across two folders at whatever hour UTC rolls. + let s = Shot { + captured_at: Some(1_787_351_400), + captured_offset: Some(540), + ..shot(None) + }; + assert_eq!( + expand(DEFAULT_TEMPLATE, &s).segments, + ["2026", "2026-08-22"] + ); + } + + #[test] + fn a_file_with_no_capture_time_falls_back_visibly() { + let d = expand(DEFAULT_TEMPLATE, &shot(None)); + assert_eq!(d.segments, ["2026", "2026-08-22"]); + // The requirement is not that it lands somewhere — it is that the + // report can say it was dated by mtime rather than by the shutter. + assert_eq!(d.dated_from, DateSource::Modified); + } + + #[test] + fn the_cameras_zone_is_not_applied_to_a_modification_time() { + // A camera zone belongs to the shutter reading. An mtime carrying a + // +540 shift would be filed a day out for no reason at all. + let s = Shot { + captured_at: None, + captured_offset: Some(540), + modified_at: 1_787_351_400, // 21st, 22:30 UTC + ..Default::default() + }; + assert_eq!( + expand(DEFAULT_TEMPLATE, &s).segments, + ["2026", "2026-08-21"] + ); + } + + #[test] + fn every_token_expands() { + let s = Shot { + captured_at: Some(AUG_22), + make: Some("Canon".into()), + model: Some("EOS R5".into()), + ..shot(None) + }; + let d = expand("{make}/{model}/{yy}/{date}/{yyyy}-{mm}-{dd}", &s); + assert_eq!( + d.segments, + ["Canon", "EOS R5", "26", "2026-08-22", "2026-08-22"] + ); + } + + #[test] + fn an_unknown_token_is_left_where_the_user_can_see_it() { + let d = expand("{yyy}", &shot(Some(AUG_22))); + assert_eq!(d.segments, ["{yyy}"]); + } + + #[test] + fn a_segment_cannot_climb_out_of_the_destination() { + // The one bug in this file that would matter: a template, or a camera + // model, that walks up out of the library root. + let d = expand("../../etc/{yyyy}", &shot(Some(AUG_22))); + assert_eq!(d.segments, ["etc", "2026"]); + + let s = Shot { + model: Some("../..".into()), + ..shot(Some(AUG_22)) + }; + assert_eq!(expand("{model}/{yyyy}", &s).segments, ["2026"]); + } + + #[test] + fn a_leading_slash_does_not_make_it_absolute() { + let d = expand("/{yyyy}", &shot(Some(AUG_22))); + assert_eq!(d.segments, ["2026"]); + } + + #[test] + fn an_empty_token_does_not_leave_a_nameless_level() { + // No make recorded. `{make}/{yyyy}` must be one level, not two with + // an unnamed parent. + let d = expand("{make}/{yyyy}", &shot(Some(AUG_22))); + assert_eq!(d.segments, ["2026"]); + } + + #[test] + fn a_template_that_expands_to_nothing_falls_back_to_the_date() { + let d = expand("{make}", &shot(Some(AUG_22))); + assert_eq!(d.segments, ["2026", "2026-08-22"]); + } + + #[test] + fn characters_no_destination_would_take_are_replaced() { + let s = Shot { + // Nikon writes its model with a colon on some bodies, and a colon + // is a stream separator on Windows and illegal on SAF. + model: Some("NIKON:Z 9".into()), + ..shot(Some(AUG_22)) + }; + assert_eq!(expand("{model}", &s).segments, ["NIKON-Z 9"]); + } + + #[test] + fn expansion_does_not_depend_on_where_it_runs() { + // FR-NC-7a: two devices must compute the same remote destination for + // the same frame, so the same input twice is the same answer twice. + let s = shot(Some(AUG_22)); + assert_eq!(expand(DEFAULT_TEMPLATE, &s), expand(DEFAULT_TEMPLATE, &s)); + } + + #[test] + fn a_display_string_is_the_segments_joined() { + let d = expand(DEFAULT_TEMPLATE, &shot(Some(AUG_22))); + assert_eq!(d.display(), "2026/2026-08-22"); + } +} diff --git a/core/dr-ingest/src/lib.rs b/core/dr-ingest/src/lib.rs new file mode 100644 index 0000000..2ad1a7f --- /dev/null +++ b/core/dr-ingest/src/lib.rs @@ -0,0 +1,1319 @@ +//! TRACES: FR-CAT-10 | FR-CAT-11 +//! Bringing photographs in from a card. +//! +//! Distinct from `dr_catalog::scan`, and the distinction is the whole reason +//! this crate exists: a scan catalogues files where they already are, where an +//! import *moves them* from a card into the library. A scan that failed +//! halfway has read nothing; an import that fails halfway has written +//! something. +//! +//! # What this crate will not do +//! +//! **It never deletes from the card.** A move-import is expressed as a copy +//! plus a later retirement the caller performs once it is satisfied — because +//! on a library that syncs, "satisfied" means the upload was confirmed, not +//! that the local write returned (FR-NC-7b). A card erased against an +//! in-flight upload is the one failure in this application with no undo, so +//! the deletion is deliberately not reachable from the code path that is in a +//! hurry. See [`retire`]. +//! +//! **It does not decode.** Capture metadata arrives through [`Probe`], so this +//! crate does not depend on `dr-decode` and a card of unreadable files costs no +//! demosaic. The same shape `dr_export::name::resolve_name` uses for its +//! collision test, for the same reason: the answer differs by caller and none +//! of the variants belong here. +//! +//! **It does not know what a duplicate is.** That question is a catalog query +//! (FR-CAT-11), so it arrives through [`DuplicateCheck`]. + +use std::io::Read; + +use dr_plat::{DirRef, NewFile, SeekableRead, Storage, StorageError, WritableStorage}; +use dr_types::SourceRef; +use sha2::{Digest, Sha256}; + +pub mod layout; + +pub use layout::{expand, DateSource, Destination, Shot, DEFAULT_TEMPLATE}; + +/// How many bytes move between the card and the destination at a time. +/// +/// A RAW is 20–100 MB and there is no reason to hold one in memory: the read, +/// the hash and the write are all streaming, so peak usage is this buffer +/// whether the card holds one file or two thousand. 1 MiB rather than +/// something smaller because a card reader's throughput collapses on small +/// reads, and rather than something larger because the second copy +/// (FR-CAT-10's backup destination) doubles it. +const CHUNK: usize = 1024 * 1024; + +/// Something went wrong importing one file. +/// +/// Per-file rather than per-run: a card with one unreadable file must import +/// the other nine hundred, so this is recorded against the item in +/// [`Report::failed`] and the run continues. +#[derive(Debug, thiserror::Error)] +pub enum IngestError { + #[error("reading the card: {0}")] + Source(StorageError), + + #[error("writing the destination: {0}")] + Destination(StorageError), + + #[error("writing the backup copy: {0}")] + Backup(StorageError), + + /// The bytes that arrived are not the bytes that left. + /// + /// A dying card, a full destination that reported success, a cable on its + /// way out. The written file is removed before this is returned — a + /// corrupt import that stayed on disk would be found by the next scan and + /// catalogued as though it were fine. + #[error("verification failed: wrote {written} bytes, {reason}")] + Verify { written: u64, reason: String }, + + #[error("io error: {0}")] + Io(String), + + /// The user asked to stop. Not a failure of the file it is recorded + /// against — that file simply never started. + #[error("cancelled")] + Cancelled, +} + +/// What to do about a file the catalog has seen before. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)] +pub enum DuplicatePolicy { + /// Transfer nothing. Re-inserting a card that was already imported is the + /// common case and should cost one metadata read per file. + #[default] + Skip, + /// Import anyway, under a name that does not collide. + ImportAsNew, +} + +/// Whether the card copy may be deleted once the import is confirmed. +/// +/// Recorded rather than acted on — see the crate docs on why the deletion does +/// not live here. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)] +pub enum TransferMode { + #[default] + Copy, + /// The user asked for a move. [`Report::retirable`] carries the sources + /// whose card copies are now redundant. + Move, +} + +/// How an import is configured. +#[derive(Debug, Clone)] +pub struct Options { + /// The folder template (FR-NC-7a). See [`layout::DEFAULT_TEMPLATE`]. + pub folder_template: String, + pub mode: TransferMode, + pub on_duplicate: DuplicatePolicy, + /// Re-read each written file and compare digests (FR-CAT-10). + /// + /// On by default and worth the second read: the failure it catches is a + /// silently corrupt original, which is unrecoverable once the card is + /// reused and invisible until the photograph is opened months later. + pub verify: bool, +} + +impl Default for Options { + fn default() -> Self { + Self { + folder_template: DEFAULT_TEMPLATE.to_string(), + mode: TransferMode::Copy, + on_duplicate: DuplicatePolicy::Skip, + verify: true, + } + } +} + +/// One file offered for import. +#[derive(Debug, Clone)] +pub struct Candidate { + /// Where it is now — on the card. + pub source: SourceRef, + /// Its name on the card, extension included. + pub name: String, + /// Size in bytes, for progress. Zero where unknown. + pub size: u64, + /// The file's modification time, Unix seconds. The date fallback. + pub modified_at: i64, +} + +/// What identifies a photograph independently of where it is stored. +/// +/// FR-CAT-11's two tiers in one type. The metadata tier — capture time, camera, +/// original filename — is answerable before a byte is transferred, which is +/// what makes re-inserting an already-imported card cheap. The `digest` tier is +/// only knowable after reading the file, and catches the case the first misses: +/// the same frame arriving under a different name. +/// +/// Camera *serial* would be the stronger second key and is not here, because +/// nothing in the tree reads it yet — `dr_decode::Metadata` carries make and +/// model only. Make and model plus capture time plus filename is what is +/// available; the field is worth adding when the decoder grows it, and the +/// digest tier covers the gap meanwhile. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct DupKey { + pub captured_at: Option, + pub make: Option, + pub model: Option, + /// The name the camera gave it. Camera filenames wrap at `IMG_9999`, so + /// this is never sufficient alone. + pub original_name: String, + pub size: u64, + /// SHA-256 of the file's bytes, once they have been read. `None` on the + /// pre-transfer question. + pub digest: Option, +} + +/// Reads capture metadata for one candidate. +/// +/// Returning `None` is not an error: a card holds files no decoder in this +/// application understands, and they are still imported — dated by mtime, and +/// reported as such (FR-NC-7a). +pub type Probe<'a> = dyn Fn(&Candidate) -> Option + 'a; + +/// Answers whether the catalog already holds this photograph (FR-CAT-11). +/// +/// Called twice per file: once before any transfer with `digest: None`, and +/// once after the bytes have been read with the digest filled in. An +/// implementation that can only answer the first should return `false` for the +/// second rather than guessing. +pub type DuplicateCheck<'a> = dyn Fn(&DupKey) -> bool + 'a; + +/// Progress, reported per file as it completes. +pub type OnProgress<'a> = dyn FnMut(&Progress) + 'a; + +/// Where an import has got to. +#[derive(Debug, Clone)] +pub struct Progress { + /// Files finished, successfully or not. + pub done: usize, + pub total: usize, + /// Bytes transferred so far, backup copies not counted twice. + pub bytes: u64, + /// The file just finished. + pub name: String, +} + +/// One file that made it in. +#[derive(Debug, Clone)] +pub struct Imported { + /// Where it came from. + pub source: SourceRef, + /// Where it now is, as the destination storage named it — **not** as the + /// caller asked for it. A SAF provider renames on collision by itself, so + /// reading this back rather than assuming is not defensive style, it is + /// the only way to be right on Android (ARCH §6.9). + pub written: SourceRef, + /// The folders it was filed under, outermost first (FR-NC-7a). The upload + /// path expands the same template to the same segments, so this is also + /// where it belongs on the server. + pub folders: Vec, + /// The name it was written under, which differs from the card's when the + /// name was taken. + pub name: String, + /// SHA-256 of its bytes. Stored so the next import can answer FR-CAT-11's + /// second tier without re-reading anything. + pub digest: String, + pub size: u64, + /// Whether the folder date came from the shutter or from mtime. + pub dated_from: DateSource, + /// Where the second copy went, if one was asked for. + pub backup: Option, +} + +/// What an import did. +#[derive(Debug, Default)] +pub struct Report { + pub imported: Vec, + /// Files the catalog already held (FR-CAT-11). Nothing was transferred. + pub duplicates: Vec, + pub failed: Vec<(Candidate, IngestError)>, + /// Sources whose card copies are now redundant, for a move-import. + /// + /// **Populated, never acted on.** The caller deletes these once it is + /// satisfied the import is safe, which on a syncing library means after the + /// upload is confirmed (FR-NC-7b). Empty for a copy-import. + pub retirable: Vec, + /// Files dated by modification time because they carried no capture time. + /// + /// FR-NC-7a requires this to be visible rather than silent: they are filed + /// under when they were last written, which is often the day the card was + /// read rather than the day the photograph was taken. + pub undated: Vec, + pub bytes: u64, + /// Whether the run stopped early because the caller asked it to. + pub cancelled: bool, +} + +impl Report { + /// Whether anything at all was written. + pub fn is_empty(&self) -> bool { + self.imported.is_empty() + } +} + +/// An import in progress. +/// +/// A struct rather than a function with nine parameters, and the borrows are +/// the point: the source is `&dyn Storage` and the destination is +/// `&dyn WritableStorage`, which is exactly the asymmetry of copying off a +/// card. A card mounted read-only cannot be passed where a destination is +/// wanted, and the compiler says so rather than the filesystem saying so +/// halfway through. +pub struct Ingest<'a> { + pub source: &'a dyn Storage, + pub dest: &'a dyn WritableStorage, + /// The library root, or a folder inside it. Template folders are created + /// beneath this. + pub dest_root: &'a DirRef, + /// The optional simultaneous second destination (FR-CAT-10). + /// + /// Written from the same read as the primary, not copied from it + /// afterwards: a backup made by re-reading the primary would inherit a bad + /// write rather than catch it, and re-reading the *card* doubles the wear + /// on the one copy that still exists. + pub backup: Option<(&'a dyn WritableStorage, &'a DirRef)>, + pub options: Options, +} + +impl Ingest<'_> { + /// Run the import. + /// + /// `cancel` is polled between files rather than within one: a half-written + /// RAW is worse than a few more seconds of waiting, and at 1 MiB chunks the + /// longest a cancel waits is one file. + pub fn run( + &self, + candidates: &[Candidate], + probe: &Probe<'_>, + is_duplicate: &DuplicateCheck<'_>, + cancel: &dyn Fn() -> bool, + on_progress: &mut OnProgress<'_>, + ) -> Report { + let mut report = Report::default(); + let total = candidates.len(); + + for candidate in candidates { + if cancel() { + report.cancelled = true; + break; + } + + // The probe reads *content*, so the modification time is not its + // to report — it comes from the listing that produced the + // candidate. Overwritten rather than defaulted, so a probe that + // leaves the field at zero cannot file a whole card under 1970. + let mut shot = probe(candidate).unwrap_or_default(); + shot.modified_at = candidate.modified_at; + + // Tier one, before a byte moves: the card that was already + // imported costs one metadata read per file and no transfer. + let key = DupKey { + captured_at: shot.captured_at, + make: shot.make.clone(), + model: shot.model.clone(), + original_name: candidate.name.clone(), + size: candidate.size, + digest: None, + }; + if self.options.on_duplicate == DuplicatePolicy::Skip && is_duplicate(&key) { + report.duplicates.push(candidate.clone()); + report.done(&mut *on_progress, total, candidate); + continue; + } + + match self.one(candidate, &shot, &key, is_duplicate) { + Ok(Some(imported)) => { + report.bytes += imported.size; + if imported.dated_from == DateSource::Modified { + report.undated.push(imported.name.clone()); + } + if self.options.mode == TransferMode::Move { + report.retirable.push(imported.source.clone()); + } + report.imported.push(imported); + } + // The digest tier caught it after reading. Nothing was kept. + Ok(None) => report.duplicates.push(candidate.clone()), + Err(e) => { + log::warn!("importing {}: {e}", candidate.name); + report.failed.push((candidate.clone(), e)); + } + } + report.done(&mut *on_progress, total, candidate); + } + + report + } + + /// Import one file. `Ok(None)` means the digest tier found a duplicate. + fn one( + &self, + candidate: &Candidate, + shot: &Shot, + key: &DupKey, + is_duplicate: &DuplicateCheck<'_>, + ) -> Result, IngestError> { + let destination = layout::expand(&self.options.folder_template, shot); + + let dir = self.make_dirs(self.dest, self.dest_root, &destination.segments)?; + let backup_dir = match self.backup { + Some((storage, root)) => Some(( + storage, + self.make_dirs(storage, root, &destination.segments) + .map_err(|e| match e { + IngestError::Destination(e) => IngestError::Backup(e), + other => other, + })?, + )), + None => None, + }; + + let name = self.free_name(self.dest, &dir, &candidate.name)?; + + let mut reader = self + .source + .open(&candidate.source) + .map_err(IngestError::Source)?; + + let primary = self + .dest + .create_file(&dir, &name) + .map_err(IngestError::Destination)?; + let secondary = match &backup_dir { + Some((storage, bdir)) => { + let bname = self.free_name(*storage, bdir, &candidate.name)?; + Some( + storage + .create_file(bdir, &bname) + .map_err(IngestError::Backup)?, + ) + } + None => None, + }; + + let written = self.stream(&mut reader, primary, secondary); + + // Any failure past the point of creating the file leaves bytes on + // disk that no catalog knows about. Take them back before returning, + // or the next scan catalogues a truncated RAW as though it were fine. + let (digest, size, target, backup_target) = match written { + Ok(v) => v, + Err((e, targets)) => { + self.discard(targets); + return Err(e); + } + }; + + if self.options.verify { + if let Err(e) = self.verify(&target, &digest, size) { + self.discard( + vec![(self.dest, target)] + .into_iter() + .chain(backup_target.map(|t| (self.backup.unwrap().0, t))) + .collect(), + ); + return Err(e); + } + } + + // Tier two (FR-CAT-11): the same frame under a different name. Only + // answerable now, and cheap to act on — the file is removed rather + // than catalogued twice. + if self.options.on_duplicate == DuplicatePolicy::Skip { + let with_digest = DupKey { + digest: Some(digest.clone()), + ..key.clone() + }; + if is_duplicate(&with_digest) { + self.discard( + vec![(self.dest, target)] + .into_iter() + .chain(backup_target.map(|t| (self.backup.unwrap().0, t))) + .collect(), + ); + return Ok(None); + } + } + + Ok(Some(Imported { + source: candidate.source.clone(), + written: target, + folders: destination.segments, + name, + digest, + size, + dated_from: destination.dated_from, + backup: backup_target, + })) + } + + /// Create each level of the template, keeping what the provider returned. + /// + /// Never `create_dir_all` on a joined path: the segments are separate + /// because on SAF the child's reference can only come from the call that + /// created it (ARCH §6.9). + fn make_dirs( + &self, + storage: &dyn WritableStorage, + root: &DirRef, + segments: &[String], + ) -> Result { + let mut dir = root.clone(); + for segment in segments { + dir = storage + .create_dir(&dir, segment) + .map_err(IngestError::Destination)?; + } + Ok(dir) + } + + /// A name not already taken in the destination. + /// + /// Two cards can hold `IMG_0001.CR3` and they are different photographs. + /// The suffix goes before the extension so the file still opens. + fn free_name( + &self, + storage: &dyn WritableStorage, + dir: &DirRef, + name: &str, + ) -> Result { + if !storage + .exists(dir, name) + .map_err(IngestError::Destination)? + { + return Ok(name.to_string()); + } + let (stem, ext) = match name.rsplit_once('.') { + Some((s, e)) if !s.is_empty() => (s, Some(e)), + _ => (name, None), + }; + for n in 1..1000 { + let candidate = match ext { + Some(ext) => format!("{stem}-{n}.{ext}"), + None => format!("{stem}-{n}"), + }; + if !storage + .exists(dir, &candidate) + .map_err(IngestError::Destination)? + { + return Ok(candidate); + } + } + Err(IngestError::Destination(StorageError::AlreadyExists( + name.to_string(), + ))) + } + + /// Read once, write twice, hash on the way past. + /// + /// The backup is written from the same read rather than copied from the + /// primary afterwards — see [`Ingest::backup`]. On error the created files + /// come back with it so the caller can take them away again. + #[allow(clippy::type_complexity)] + fn stream( + &self, + reader: &mut Box, + primary: NewFile, + secondary: Option, + ) -> Result< + (String, u64, SourceRef, Option), + (IngestError, Vec<(&dyn WritableStorage, SourceRef)>), + > { + let NewFile { + source: target, + sink: mut out, + } = primary; + let (backup_target, mut backup_out) = match secondary { + Some(NewFile { source, sink }) => (Some(source), Some(sink)), + None => (None, None), + }; + + let created = |extra: Option<&SourceRef>| -> Vec<(&dyn WritableStorage, SourceRef)> { + let mut v: Vec<(&dyn WritableStorage, SourceRef)> = vec![(self.dest, target.clone())]; + if let (Some(t), Some((storage, _))) = (extra, self.backup) { + v.push((storage, t.clone())); + } + v + }; + + let mut hasher = Sha256::new(); + let mut buf = vec![0u8; CHUNK]; + let mut size = 0u64; + loop { + let n = match reader.read(&mut buf) { + Ok(0) => break, + Ok(n) => n, + Err(e) => { + return Err(( + IngestError::Source(StorageError::Io(e.to_string())), + created(backup_target.as_ref()), + )) + } + }; + let chunk = &buf[..n]; + hasher.update(chunk); + size += n as u64; + if let Err(e) = out.write_all(chunk) { + return Err(( + IngestError::Destination(StorageError::Io(e.to_string())), + created(backup_target.as_ref()), + )); + } + if let Some(b) = backup_out.as_mut() { + if let Err(e) = b.write_all(chunk) { + return Err(( + IngestError::Backup(StorageError::Io(e.to_string())), + created(backup_target.as_ref()), + )); + } + } + } + + // Flush explicitly rather than on drop. A buffered writer swallows the + // error when it is dropped, and "the disk filled on the last chunk" is + // exactly the failure this whole path exists to catch. + if let Err(e) = out.flush() { + return Err(( + IngestError::Destination(StorageError::Io(e.to_string())), + created(backup_target.as_ref()), + )); + } + if let Some(b) = backup_out.as_mut() { + if let Err(e) = b.flush() { + return Err(( + IngestError::Backup(StorageError::Io(e.to_string())), + created(backup_target.as_ref()), + )); + } + } + drop(out); + drop(backup_out); + + Ok((hex(&hasher.finalize()), size, target, backup_target)) + } + + /// Re-read what was written and confirm it is what was sent. + /// + /// Reads the *destination*, not the buffer that was just hashed. Hashing + /// what is still in memory would confirm only that the process can add — it + /// would pass on a full disk, a dying card and a truncated write alike. + fn verify(&self, target: &SourceRef, expected: &str, size: u64) -> Result<(), IngestError> { + let mut reader = self.dest.open(target).map_err(|e| IngestError::Verify { + written: size, + reason: format!("cannot re-read it: {e}"), + })?; + let mut hasher = Sha256::new(); + let mut buf = vec![0u8; CHUNK]; + let mut seen = 0u64; + loop { + let n = reader.read(&mut buf).map_err(|e| IngestError::Verify { + written: size, + reason: format!("cannot re-read it: {e}"), + })?; + if n == 0 { + break; + } + hasher.update(&buf[..n]); + seen += n as u64; + } + let got = hex(&hasher.finalize()); + if seen != size { + return Err(IngestError::Verify { + written: size, + reason: format!("{seen} bytes came back"), + }); + } + if got != expected { + return Err(IngestError::Verify { + written: size, + reason: format!("digest {got} does not match {expected}"), + }); + } + Ok(()) + } + + /// Take back files this import created but is not keeping. + /// + /// Failures here are logged and not propagated: this runs on a path that + /// already has an error to report, and replacing it with "and the cleanup + /// also failed" loses the one the user needs. + fn discard(&self, targets: Vec<(&dyn WritableStorage, SourceRef)>) { + for (storage, target) in targets { + if let Err(e) = storage.remove_file(&target) { + log::warn!("could not remove a partial import: {e}"); + } + } + } +} + +impl Report { + fn done(&self, on_progress: &mut OnProgress<'_>, total: usize, candidate: &Candidate) { + on_progress(&Progress { + done: self.duplicates.len() + self.imported.len() + self.failed.len(), + total, + bytes: self.bytes, + name: candidate.name.clone(), + }); + } +} + +/// TRACES: FR-CAT-10 | FR-NC-7b +/// Delete the card copies of files that are safely in the library. +/// +/// Separate from [`Ingest::run`] and deliberately awkward to reach. Pass only +/// [`Report::retirable`], and only once whatever "safe" means for this library +/// is true — for one that syncs, that is the upload confirmed, not the local +/// write returned (FR-NC-7b). There is no undo behind this function: the card +/// is where the only other copy was. +/// +/// Returns the sources it could not delete, which is a card pulled mid-run or +/// a write-protect switch, and neither is worth stopping for. +pub fn retire(card: &dyn WritableStorage, sources: &[SourceRef]) -> Vec<(SourceRef, StorageError)> { + let mut failed = Vec::new(); + for src in sources { + if let Err(e) = card.remove_file(src) { + log::warn!("could not retire {src:?} from the card: {e}"); + failed.push((src.clone(), e)); + } + } + failed +} + +/// A digest as lower-case hex, the form `sha256sum` prints. +fn hex(bytes: &[u8]) -> String { + let mut s = String::with_capacity(bytes.len() * 2); + for b in bytes { + s.push_str(&format!("{b:02x}")); + } + s +} + +#[cfg(test)] +mod tests { + use super::*; + use std::cell::RefCell; + use std::path::PathBuf; + + use dr_plat::{Entry, LocalStorage}; + use dr_types::{ByteRange, DirState, RootId}; + + const CARD: RootId = RootId(1); + const LIB: RootId = RootId(2); + /// 2026-08-22T14:00:00Z. + const AUG_22: i64 = 1_787_407_200; + + /// A throwaway directory, removed when the test ends. + struct Tree(PathBuf); + + impl Tree { + fn new(name: &str) -> Self { + let dir = std::env::temp_dir().join(format!( + "dr-ingest-{name}-{}-{:?}", + std::process::id(), + std::thread::current().id() + )); + let _ = std::fs::remove_dir_all(&dir); + std::fs::create_dir_all(&dir).expect("temp dir"); + Tree(dir) + } + + fn sub(&self, rel: &str) -> PathBuf { + let p = self.0.join(rel); + std::fs::create_dir_all(&p).expect("mkdir"); + p + } + } + + impl Drop for Tree { + fn drop(&mut self) { + let _ = std::fs::remove_dir_all(&self.0); + } + } + + /// A card with files on it, and the library it imports into. + struct Fixture { + _tree: Tree, + card: LocalStorage, + card_path: PathBuf, + lib: LocalStorage, + lib_path: PathBuf, + } + + fn fixture(name: &str) -> Fixture { + let tree = Tree::new(name); + let card_path = tree.sub("card/DCIM/100CANON"); + let lib_path = tree.sub("library"); + Fixture { + card: LocalStorage::with_root(CARD, tree.0.join("card")), + lib: LocalStorage::with_root(LIB, lib_path.clone()), + card_path, + lib_path, + _tree: tree, + } + } + + impl Fixture { + /// Put a file on the card and return the candidate for it. + fn shoot(&self, name: &str, bytes: &[u8]) -> Candidate { + std::fs::write(self.card_path.join(name), bytes).expect("write"); + Candidate { + source: SourceRef::Local { + root: CARD, + relative: format!("DCIM/100CANON/{name}"), + }, + name: name.to_string(), + size: bytes.len() as u64, + modified_at: AUG_22, + } + } + + fn lib_root(&self) -> DirRef { + DirRef::root(LIB) + } + + fn read(&self, rel: &str) -> Vec { + std::fs::read(self.lib_path.join(rel)).expect("read imported file") + } + } + + /// A `Shot` that dates to the 22nd of August 2026. + fn on_the_22nd() -> Shot { + Shot { + captured_at: Some(AUG_22), + make: Some("Canon".into()), + model: Some("EOS R5".into()), + ..Default::default() + } + } + + fn never(_: &DupKey) -> bool { + false + } + + fn run( + f: &Fixture, + candidates: &[Candidate], + options: Options, + probe: &Probe<'_>, + is_duplicate: &DuplicateCheck<'_>, + ) -> Report { + let root = f.lib_root(); + let ingest = Ingest { + source: &f.card, + dest: &f.lib, + dest_root: &root, + backup: None, + options, + }; + ingest.run(candidates, probe, is_duplicate, &|| false, &mut |_| {}) + } + + #[test] + fn a_card_is_filed_by_year_and_day() { + let f = fixture("by-date"); + let c = f.shoot("IMG_0001.CR3", b"raw bytes"); + + let report = run( + &f, + &[c], + Options::default(), + &|_| Some(on_the_22nd()), + &never, + ); + + assert_eq!(report.imported.len(), 1); + let imported = &report.imported[0]; + assert_eq!(imported.folders, ["2026", "2026-08-22"]); + assert_eq!(f.read("2026/2026-08-22/IMG_0001.CR3"), b"raw bytes"); + // The digest is stored so the next import can answer FR-CAT-11's + // second tier without reading the file again. + assert_eq!(imported.digest.len(), 64); + assert_eq!(report.bytes, 9); + } + + #[test] + fn the_digest_is_the_one_sha256sum_would_print() { + let f = fixture("digest"); + let c = f.shoot("IMG_0001.CR3", b"abc"); + let report = run( + &f, + &[c], + Options::default(), + &|_| Some(on_the_22nd()), + &never, + ); + // Stored and compared across devices and across years, so it has to be + // a value someone can recompute at a shell prompt. + assert_eq!( + report.imported[0].digest, + "ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad" + ); + } + + #[test] + fn a_card_that_was_already_imported_transfers_nothing() { + let f = fixture("dupe-cheap"); + let c = f.shoot("IMG_0001.CR3", b"raw bytes"); + + // Tier one: answered from metadata, before a byte moves. + let asked: RefCell> = RefCell::new(Vec::new()); + let report = run( + &f, + &[c], + Options::default(), + &|_| Some(on_the_22nd()), + &|k| { + asked.borrow_mut().push(k.clone()); + true + }, + ); + + assert_eq!(report.duplicates.len(), 1); + assert!(report.imported.is_empty()); + assert!(!f.lib_path.join("2026").exists(), "nothing was written"); + // Asked exactly once, without a digest — the whole point is that the + // file was never read. + assert_eq!(asked.borrow().len(), 1); + assert_eq!(asked.borrow()[0].digest, None); + assert_eq!(asked.borrow()[0].original_name, "IMG_0001.CR3"); + } + + #[test] + fn the_same_frame_under_a_different_name_is_caught_by_its_digest() { + let f = fixture("dupe-digest"); + let c = f.shoot("DSC_4242.NEF", b"the same photograph"); + + // Tier two: the metadata key misses (the name differs), so the file is + // read — and then must not be kept. + let report = run( + &f, + &[c], + Options::default(), + &|_| Some(on_the_22nd()), + &|k| k.digest.is_some(), + ); + + assert_eq!(report.duplicates.len(), 1); + assert!(report.imported.is_empty()); + assert!( + !f.lib_path.join("2026/2026-08-22/DSC_4242.NEF").exists(), + "the copy made to compute the digest was taken back" + ); + } + + #[test] + fn importing_as_new_keeps_a_known_duplicate() { + let f = fixture("dupe-keep"); + let c = f.shoot("IMG_0001.CR3", b"raw bytes"); + + let report = run( + &f, + &[c], + Options { + on_duplicate: DuplicatePolicy::ImportAsNew, + ..Default::default() + }, + &|_| Some(on_the_22nd()), + &|_| true, + ); + + assert_eq!(report.imported.len(), 1); + assert!(report.duplicates.is_empty()); + } + + #[test] + fn two_cards_holding_the_same_filename_do_not_overwrite() { + let f = fixture("collision"); + // Camera filenames wrap at IMG_9999, so this is ordinary rather than + // exotic: two different photographs, one name. + let first = f.shoot("IMG_0001.CR3", b"the first frame"); + let report = run( + &f, + &[first], + Options::default(), + &|_| Some(on_the_22nd()), + &never, + ); + assert_eq!(report.imported[0].name, "IMG_0001.CR3"); + + let second = f.shoot("IMG_0001.CR3", b"a different frame"); + let report = run( + &f, + &[second], + Options::default(), + &|_| Some(on_the_22nd()), + &never, + ); + + assert_eq!(report.imported[0].name, "IMG_0001-1.CR3"); + // The suffix goes before the extension, so the file still opens. + assert_eq!(f.read("2026/2026-08-22/IMG_0001.CR3"), b"the first frame"); + assert_eq!( + f.read("2026/2026-08-22/IMG_0001-1.CR3"), + b"a different frame" + ); + } + + #[test] + fn a_file_with_no_capture_time_is_reported_not_hidden() { + let f = fixture("undated"); + let c = f.shoot("SCAN_01.TIF", b"scanned film"); + + // No decoder understands it, so it is dated by mtime — and the user is + // told, because mtime is often the day the card was read (FR-NC-7a). + let report = run(&f, &[c], Options::default(), &|_| None, &never); + + assert_eq!(report.imported.len(), 1); + assert_eq!(report.imported[0].dated_from, DateSource::Modified); + assert_eq!(report.undated, ["SCAN_01.TIF"]); + assert_eq!(f.read("2026/2026-08-22/SCAN_01.TIF"), b"scanned film"); + } + + #[test] + fn a_probe_that_forgets_the_modification_time_does_not_file_a_card_under_1970() { + let f = fixture("probe-zero"); + let c = f.shoot("IMG_0001.CR3", b"x"); + // A probe returns content metadata and nothing else. If its zeroed + // mtime were trusted, a card of undated files would land in 1970. + let report = run( + &f, + &[c], + Options::default(), + &|_| Some(Shot::default()), + &never, + ); + assert_eq!(report.imported[0].folders, ["2026", "2026-08-22"]); + } + + #[test] + fn a_backup_copy_is_written_from_the_same_read() { + let f = fixture("backup"); + let backup_path = f._tree.sub("backup"); + let backup = LocalStorage::with_root(RootId(3), backup_path.clone()); + let c = f.shoot("IMG_0001.CR3", b"raw bytes"); + + let root = f.lib_root(); + let backup_root = DirRef::root(RootId(3)); + let ingest = Ingest { + source: &f.card, + dest: &f.lib, + dest_root: &root, + backup: Some((&backup, &backup_root)), + options: Options::default(), + }; + let report = ingest.run( + &[c], + &|_| Some(on_the_22nd()), + &never, + &|| false, + &mut |_| {}, + ); + + assert_eq!(report.imported.len(), 1); + assert!(report.imported[0].backup.is_some()); + // The same folder template on both sides, so the backup is a library + // rather than a heap. + assert_eq!( + std::fs::read(backup_path.join("2026/2026-08-22/IMG_0001.CR3")).unwrap(), + b"raw bytes" + ); + // One read of the card produced both, so the byte count is not doubled. + assert_eq!(report.bytes, 9); + } + + #[test] + fn an_unreadable_file_does_not_stop_the_card() { + let f = fixture("partial-failure"); + let good = f.shoot("IMG_0001.CR3", b"fine"); + // A file the listing saw and the reader cannot open — a card on its + // way out. The other nine hundred images still have to import. + let missing = Candidate { + source: SourceRef::Local { + root: CARD, + relative: "DCIM/100CANON/GONE.CR3".into(), + }, + name: "GONE.CR3".into(), + size: 4, + modified_at: AUG_22, + }; + let also_good = f.shoot("IMG_0002.CR3", b"also fine"); + + let report = run( + &f, + &[good, missing, also_good], + Options::default(), + &|_| Some(on_the_22nd()), + &never, + ); + + assert_eq!(report.imported.len(), 2); + assert_eq!(report.failed.len(), 1); + assert_eq!(report.failed[0].0.name, "GONE.CR3"); + // And nothing was left behind for the next scan to catalogue. + assert!(!f.lib_path.join("2026/2026-08-22/GONE.CR3").exists()); + } + + #[test] + fn cancelling_stops_between_files_rather_than_inside_one() { + let f = fixture("cancel"); + let a = f.shoot("IMG_0001.CR3", b"first"); + let b = f.shoot("IMG_0002.CR3", b"second"); + let c = f.shoot("IMG_0003.CR3", b"third"); + + let seen = RefCell::new(0usize); + let root = f.lib_root(); + let ingest = Ingest { + source: &f.card, + dest: &f.lib, + dest_root: &root, + backup: None, + options: Options::default(), + }; + let report = ingest.run( + &[a, b, c], + &|_| Some(on_the_22nd()), + &never, + &|| { + let mut n = seen.borrow_mut(); + *n += 1; + // Cancel is asked before each file: allow two, refuse the third. + *n > 2 + }, + &mut |_| {}, + ); + + assert!(report.cancelled); + assert_eq!(report.imported.len(), 2); + // Whole files or nothing — a half-written RAW is worse than waiting. + assert_eq!(f.read("2026/2026-08-22/IMG_0002.CR3"), b"second"); + assert!(!f.lib_path.join("2026/2026-08-22/IMG_0003.CR3").exists()); + } + + #[test] + fn progress_counts_every_file_including_the_ones_it_skipped() { + let f = fixture("progress"); + let a = f.shoot("IMG_0001.CR3", b"first"); + let b = f.shoot("IMG_0002.CR3", b"second"); + + let ticks: RefCell> = RefCell::new(Vec::new()); + let root = f.lib_root(); + let ingest = Ingest { + source: &f.card, + dest: &f.lib, + dest_root: &root, + backup: None, + options: Options::default(), + }; + ingest.run( + &[a, b], + &|_| Some(on_the_22nd()), + &never, + &|| false, + &mut |p| ticks.borrow_mut().push((p.done, p.total)), + ); + + assert_eq!(*ticks.borrow(), [(1, 2), (2, 2)]); + } + + #[test] + fn a_move_import_records_what_may_be_deleted_but_deletes_nothing() { + let f = fixture("move"); + let c = f.shoot("IMG_0001.CR3", b"raw bytes"); + let on_card = f.card_path.join("IMG_0001.CR3"); + + let report = run( + &f, + &[c], + Options { + mode: TransferMode::Move, + ..Default::default() + }, + &|_| Some(on_the_22nd()), + &never, + ); + + assert_eq!(report.retirable.len(), 1); + // The card is still intact. On a syncing library the deletion waits for + // the upload to be confirmed (FR-NC-7b), and this crate cannot know + // that it was. + assert!(on_card.exists()); + + // Only when the caller says so. + let failed = retire(&f.card, &report.retirable); + assert!(failed.is_empty()); + assert!(!on_card.exists()); + } + + #[test] + fn a_copy_import_marks_nothing_for_deletion() { + let f = fixture("copy-keeps"); + let c = f.shoot("IMG_0001.CR3", b"raw bytes"); + let report = run( + &f, + &[c], + Options::default(), + &|_| Some(on_the_22nd()), + &never, + ); + assert!(report.retirable.is_empty()); + } + + // ---- verification (FR-CAT-10) --------------------------------------- + + /// A destination that writes fewer bytes than it was given. + /// + /// What a card on its way out, or a filesystem that reported a successful + /// write into a full disk, looks like from here. Verification exists for + /// exactly this, so it needs a way to happen on purpose. + struct Truncating(LocalStorage); + + struct ShortSink(Box); + + impl std::io::Write for ShortSink { + fn write(&mut self, buf: &[u8]) -> std::io::Result { + // Drops the last byte of every chunk and claims it wrote them all, + // which is precisely the failure a digest catches and a byte count + // alone would not. + let keep = buf.len().saturating_sub(1); + self.0.write_all(&buf[..keep])?; + Ok(buf.len()) + } + fn flush(&mut self) -> std::io::Result<()> { + self.0.flush() + } + } + + impl Storage for Truncating { + fn roots(&self) -> Vec { + self.0.roots() + } + fn root_dir(&self, root: RootId) -> Result { + self.0.root_dir(root) + } + fn dir_state(&self, dir: &DirRef) -> Result { + self.0.dir_state(dir) + } + fn list(&self, dir: &DirRef) -> Result, StorageError> { + self.0.list(dir) + } + fn open(&self, src: &SourceRef) -> Result, StorageError> { + self.0.open(src) + } + fn read_range(&self, src: &SourceRef, range: ByteRange) -> Result, StorageError> { + self.0.read_range(src, range) + } + } + + impl WritableStorage for Truncating { + fn create_dir(&self, parent: &DirRef, name: &str) -> Result { + self.0.create_dir(parent, name) + } + fn create_file(&self, parent: &DirRef, name: &str) -> Result { + let nf = self.0.create_file(parent, name)?; + Ok(NewFile { + source: nf.source, + sink: Box::new(ShortSink(nf.sink)), + }) + } + fn exists(&self, parent: &DirRef, name: &str) -> Result { + self.0.exists(parent, name) + } + fn remove_file(&self, src: &SourceRef) -> Result<(), StorageError> { + self.0.remove_file(src) + } + } + + #[test] + fn a_corrupt_write_is_caught_and_taken_back() { + let f = fixture("verify-fails"); + let c = f.shoot("IMG_0001.CR3", b"raw bytes"); + let lib = Truncating(LocalStorage::with_root(LIB, f.lib_path.clone())); + let root = DirRef::root(LIB); + + let ingest = Ingest { + source: &f.card, + dest: &lib, + dest_root: &root, + backup: None, + options: Options::default(), + }; + let report = ingest.run( + &[c], + &|_| Some(on_the_22nd()), + &never, + &|| false, + &mut |_| {}, + ); + + assert!(report.imported.is_empty()); + assert_eq!(report.failed.len(), 1); + assert!( + matches!(report.failed[0].1, IngestError::Verify { .. }), + "{:?}", + report.failed[0].1 + ); + // Removed rather than left: a truncated RAW on disk would be found by + // the next scan and catalogued as though it were fine. + assert!(!f.lib_path.join("2026/2026-08-22/IMG_0001.CR3").exists()); + } + + #[test] + fn verification_can_be_turned_off_and_then_nothing_catches_it() { + let f = fixture("verify-off"); + let c = f.shoot("IMG_0001.CR3", b"raw bytes"); + let lib = Truncating(LocalStorage::with_root(LIB, f.lib_path.clone())); + let root = DirRef::root(LIB); + + let ingest = Ingest { + source: &f.card, + dest: &lib, + dest_root: &root, + backup: None, + options: Options { + verify: false, + ..Default::default() + }, + }; + let report = ingest.run( + &[c], + &|_| Some(on_the_22nd()), + &never, + &|| false, + &mut |_| {}, + ); + + // Documents the cost of the option rather than endorsing it: the + // import "succeeds" and the file on disk is short. + assert_eq!(report.imported.len(), 1); + assert_eq!(f.read("2026/2026-08-22/IMG_0001.CR3"), b"raw byte"); + } + + #[test] + fn a_file_larger_than_one_chunk_survives_the_round_trip() { + let f = fixture("large"); + // Streaming is the whole design — a RAW is 20–100 MB and never sits in + // memory. Anything that only works below the buffer size is a bug this + // catches. + let bytes: Vec = (0..(CHUNK * 2 + 1234)).map(|i| (i % 251) as u8).collect(); + let c = f.shoot("IMG_0001.CR3", &bytes); + + let report = run( + &f, + &[c], + Options::default(), + &|_| Some(on_the_22nd()), + &never, + ); + + assert_eq!(report.imported.len(), 1); + assert_eq!(report.imported[0].size, bytes.len() as u64); + assert_eq!(f.read("2026/2026-08-22/IMG_0001.CR3"), bytes); + } +}