From 7d1e6f724e02399df50c12c9c8bfa789dc21ef0e Mon Sep 17 00:00:00 2001 From: Duncan Tourolle Date: Sat, 22 Aug 2026 14:00:47 +0200 Subject: [PATCH] Import photographs from a card MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Distinct from a scan, and the distinction is the whole reason the crate exists: a scan catalogues files where they already are, where an import moves them from a card into the library. A scan that fails halfway has read nothing; an import that fails halfway has written something. So the failure paths are the design. Bytes stream at 1 MiB and are hashed on the way past, so an 80 MB RAW never sits in memory. The second destination (FR-CAT-10's backup copy) is written from the same read rather than copied from the primary afterwards — a backup made by re-reading the primary would inherit a bad write rather than catch it, and re-reading the card doubles the wear on the one copy that still exists. Verification re-reads the destination, because hashing what is still in memory would pass on a full disk, a dying card and a truncated write alike. Anything that fails past the point of creating the file takes the file back, or the next scan catalogues a truncated RAW as though it were fine. Three things the crate refuses to know. It never deletes from the card: a move-import records what is now redundant and a separate retire() does the deleting, because on a syncing library "safe" means the upload was confirmed (FR-NC-7b). It does not decode, so capture metadata arrives through a probe and a card of unreadable files costs no demosaic. And it does not know what a duplicate is, since that is a catalog query — FR-CAT-11's two tiers arrive as one closure asked twice, once before any transfer and once with the digest. Camera serial would be the stronger metadata key and is absent, because nothing in the tree reads it yet; make and model plus capture time and the original filename is what is available, and the digest tier covers the gap. Co-Authored-By: Claude Opus 5 (1M context) --- Cargo.lock | 12 + Cargo.toml | 2 + core/dr-ingest/Cargo.toml | 30 + core/dr-ingest/src/layout.rs | 343 +++++++++ core/dr-ingest/src/lib.rs | 1319 ++++++++++++++++++++++++++++++++++ 5 files changed, 1706 insertions(+) create mode 100644 core/dr-ingest/Cargo.toml create mode 100644 core/dr-ingest/src/layout.rs create mode 100644 core/dr-ingest/src/lib.rs diff --git a/Cargo.lock b/Cargo.lock index a897932..5b5b749 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1461,6 +1461,18 @@ dependencies = [ "wgpu", ] +[[package]] +name = "dr-ingest" +version = "0.1.0" +dependencies = [ + "dr-plat", + "dr-types", + "env_logger", + "log", + "sha2", + "thiserror 2.0.20", +] + [[package]] name = "dr-lens" version = "0.1.0" diff --git a/Cargo.toml b/Cargo.toml index b257625..d2bc93f 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -6,6 +6,7 @@ members = [ "core/dr-thumbs", "core/dr-decode", "core/dr-export", + "core/dr-ingest", "core/dr-gpu", "core/dr-lens", "core/dr-pipeline", @@ -33,6 +34,7 @@ dr-catalog = { path = "core/dr-catalog" } dr-thumbs = { path = "core/dr-thumbs" } dr-decode = { path = "core/dr-decode" } dr-export = { path = "core/dr-export" } +dr-ingest = { path = "core/dr-ingest" } dr-gpu = { path = "core/dr-gpu" } dr-lens = { path = "core/dr-lens" } dr-pipeline = { path = "core/dr-pipeline" } diff --git a/core/dr-ingest/Cargo.toml b/core/dr-ingest/Cargo.toml new file mode 100644 index 0000000..8f42f99 --- /dev/null +++ b/core/dr-ingest/Cargo.toml @@ -0,0 +1,30 @@ +[package] +name = "dr-ingest" +version.workspace = true +edition.workspace = true +rust-version.workspace = true +license.workspace = true + +[dependencies] +dr-types.workspace = true +# The `Storage` and `WritableStorage` traits. An import reads a real card and +# writes a real destination, and this is how `core/` reaches the platform +# without a `#[cfg(target_os)]` of its own (ARCH §4.1) — the same dependency +# `dr-catalog` takes, for the same reason. +dr-plat.workspace = true +thiserror.workspace = true +log.workspace = true + +# Content hashing: copy verification (FR-CAT-10) and the second duplicate tier +# (FR-CAT-11). Pure Rust, no C, so it cross-compiles under the NDK like every +# other choice in this tree — and it is already in the lock file as a +# transitive dependency, so it costs no new build. +# +# SHA-256 rather than a fast non-cryptographic hash, which would be sufficient +# for verifying a copy: the digest is *stored* and compared across devices and +# across years, so it wants to be a named standard someone can recompute with +# `sha256sum` rather than whatever we happened to implement. +sha2 = "0.10" + +[dev-dependencies] +env_logger.workspace = true diff --git a/core/dr-ingest/src/layout.rs b/core/dr-ingest/src/layout.rs new file mode 100644 index 0000000..7035f46 --- /dev/null +++ b/core/dr-ingest/src/layout.rs @@ -0,0 +1,343 @@ +//! TRACES: FR-CAT-10 | FR-NC-7a +//! Where an imported photograph is filed. +//! +//! One template, two destinations. A card import expands it into directories +//! under a library root (FR-CAT-10); an upload expands the *same* template +//! into folders on the server (FR-NC-7a). They must agree, because a library +//! that is `2026/2026-08-22/` locally and something else remotely is two +//! libraries that happen to hold the same files. +//! +//! # Why this returns segments rather than a path +//! +//! Neither caller can use a path. The local side creates each level through +//! [`dr_plat::WritableStorage`], which takes a parent reference plus one name +//! because a SAF document id is not composable (ARCH §6.9). The remote side +//! joins onto a `RemotePath`, which normalises separators its own way. A +//! `String` with slashes in it would be taken apart again by both, and the +//! taking-apart is where a `..` or an empty segment would slip through. +//! +//! So expansion hands back `Vec`, each element already checked to be +//! one ordinary directory name. + +use dr_types::civil_from_unix_at; + +/// What a folder template can refer to. +/// +/// Deliberately not the decoder's `Metadata`: this crate does not depend on +/// `dr-decode` (see the crate docs), and a template needs six fields out of +/// twenty. The caller fills it from whatever it read. +#[derive(Debug, Clone, Default, PartialEq, Eq)] +pub struct Shot { + /// When the shutter fired, Unix seconds, as EXIF recorded it. + pub captured_at: Option, + /// Minutes east of UTC where the camera recorded a zone. + pub captured_offset: Option, + pub make: Option, + pub model: Option, + /// The file's modification time, Unix seconds — the fallback when there is + /// no capture time, and the reason [`Shot`] always dates to *something*. + pub modified_at: i64, +} + +/// Which reading the folder name came from. +/// +/// FR-NC-7a requires the fallback to be visible in the import report rather +/// than silent. An image filed by mtime is filed by when it was last *copied*, +/// which for a card that has been through a card reader is often today — so a +/// user seeing a folder full of unrelated dates needs to be told why. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum DateSource { + /// EXIF capture time, in the zone the camera recorded. + Capture, + /// File modification time. Not when the photograph was taken. + Modified, +} + +/// The default layout: one directory per year, one per capture day beneath it. +/// +/// `2026/2026-08-22`. The day folder repeats the year rather than being bare +/// `08-22`, because a folder is often seen out of its parent's context — in a +/// file manager's recent list, in a Nextcloud share link, in a backup listing +/// — and `08-22` alone does not say which year it belongs to. +pub const DEFAULT_TEMPLATE: &str = "{yyyy}/{yyyy}-{mm}-{dd}"; + +/// An expanded destination. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct Destination { + /// Directory names, outermost first. Each is one ordinary name: never + /// empty, never `.` or `..`, never containing a separator. + pub segments: Vec, + /// Which timestamp produced the date tokens. + pub dated_from: DateSource, +} + +impl Destination { + /// The layout as a display string — for a confirmation dialog and for logs. + pub fn display(&self) -> String { + self.segments.join("/") + } +} + +/// Expand a folder template for one photograph. +/// +/// Unknown tokens are left verbatim, matching `dr_export::name::expand`: a +/// user who typed `{yyy}` should see it in the folder name and understand what +/// happened, where a silently dropped token produces a directory whose name +/// says nothing about why it is called that. +/// +/// The result is **a pure function of the template and the shot**, which +/// FR-NC-7a requires: two devices uploading the same frame must compute the +/// same destination, so nothing here may consult local library layout, the +/// current time, or a counter. +pub fn expand(template: &str, shot: &Shot) -> Destination { + let (when, dated_from) = match shot.captured_at { + Some(t) => (t, DateSource::Capture), + None => (shot.modified_at, DateSource::Modified), + }; + // The offset applies only to a capture reading. An mtime is already an + // absolute instant with no camera zone attached to it, and adding the + // camera's offset to it would be arithmetic on two unrelated clocks. + let civil = match dated_from { + DateSource::Capture => civil_from_unix_at(when, shot.captured_offset), + DateSource::Modified => civil_from_unix_at(when, None), + }; + + let year = civil.year.to_string(); + let short_year = format!("{:02}", civil.year.rem_euclid(100)); + let month = format!("{:02}", civil.month); + let day = format!("{:02}", civil.day); + + let mut out = String::with_capacity(template.len() + 16); + let mut rest = template; + while let Some(open) = rest.find('{') { + out.push_str(&rest[..open]); + let Some(close) = rest[open..].find('}') else { + // An unclosed brace is literal text. Consumed here rather than + // left to the tail append, which has already had everything + // before the brace taken from it. + out.push_str(&rest[open..]); + rest = ""; + break; + }; + let token = &rest[open + 1..open + close]; + match token { + "yyyy" => out.push_str(&year), + "yy" => out.push_str(&short_year), + "mm" => out.push_str(&month), + "dd" => out.push_str(&day), + "date" => out.push_str(&format!("{year}-{month}-{day}")), + "make" => out.push_str(shot.make.as_deref().unwrap_or_default()), + "model" => out.push_str(shot.model.as_deref().unwrap_or_default()), + _ => out.push_str(&rest[open..open + close + 1]), + } + rest = &rest[open + close + 1..]; + } + out.push_str(rest); + + let segments: Vec = out + .split('/') + .map(sanitise_segment) + // Empty segments are dropped rather than kept: a template of + // `{make}/{yyyy}` on a camera whose make is unknown must not produce + // a nameless directory level, and `//` from a typo must not either. + .filter(|s| !s.is_empty()) + .collect(); + + // A template that expanded to nothing at all — `{make}` alone, on a file + // with no make. Falling back to the date is the one answer always + // available, and it is what the user asked for in every other case. + let segments = if segments.is_empty() { + vec![year.clone(), format!("{year}-{month}-{day}")] + } else { + segments + }; + + Destination { + segments, + dated_from, + } +} + +/// Reduce one path segment to something every destination will accept. +/// +/// The same intersection of rules as `dr_export::name::sanitise` — Linux, +/// SAF and WebDAV, plus Windows for the sync client downstream — applied to a +/// directory name rather than a filename stem. Kept apart from the exporter's +/// because a segment has one rule a stem does not: `.` and `..` are legal +/// filenames and are *not* legal directory names here, since both would move +/// the destination rather than name it. +fn sanitise_segment(seg: &str) -> String { + let mut out: String = seg + .chars() + .map(|c| match c { + '/' | '\\' | ':' | '*' | '?' | '"' | '<' | '>' | '|' => '-', + c if (c as u32) < 0x20 => '-', + c => c, + }) + .collect(); + // Trailing dots and spaces are legal on Linux and rejected by Windows, and + // a name ending in one is almost always a token that expanded to nothing. + while out.ends_with('.') || out.ends_with(' ') { + out.pop(); + } + let out = out.trim_start().to_string(); + // What is left of `.` or `..` after that pop is an empty string, which the + // caller drops. Spelled out rather than relied upon, because a segment + // that climbed out of the destination would be the one bug in this file + // that mattered. + if out == "." || out == ".." { + return String::new(); + } + out +} + +#[cfg(test)] +mod tests { + use super::*; + + /// 2026-08-22T14:00:00Z. + const AUG_22: i64 = 1_787_407_200; + + fn shot(captured_at: Option) -> Shot { + Shot { + captured_at, + modified_at: AUG_22, + ..Default::default() + } + } + + #[test] + fn the_default_template_is_a_year_then_a_day() { + let d = expand(DEFAULT_TEMPLATE, &shot(Some(AUG_22))); + assert_eq!(d.segments, ["2026", "2026-08-22"]); + assert_eq!(d.dated_from, DateSource::Capture); + } + + #[test] + fn the_day_folder_is_zero_padded_so_a_year_sorts() { + // 2026-01-05T00:00:00Z. Unpadded, `2026-1-5` sorts after `2026-10-1` + // in every file manager there is. + let d = expand(DEFAULT_TEMPLATE, &shot(Some(1_767_571_200))); + assert_eq!(d.segments, ["2026", "2026-01-05"]); + } + + #[test] + fn a_shot_is_filed_in_the_cameras_own_day_not_utcs() { + // 22:30 UTC on the 21st is already the 22nd in Tokyo. A night shoot + // must not be split across two folders at whatever hour UTC rolls. + let s = Shot { + captured_at: Some(1_787_351_400), + captured_offset: Some(540), + ..shot(None) + }; + assert_eq!( + expand(DEFAULT_TEMPLATE, &s).segments, + ["2026", "2026-08-22"] + ); + } + + #[test] + fn a_file_with_no_capture_time_falls_back_visibly() { + let d = expand(DEFAULT_TEMPLATE, &shot(None)); + assert_eq!(d.segments, ["2026", "2026-08-22"]); + // The requirement is not that it lands somewhere — it is that the + // report can say it was dated by mtime rather than by the shutter. + assert_eq!(d.dated_from, DateSource::Modified); + } + + #[test] + fn the_cameras_zone_is_not_applied_to_a_modification_time() { + // A camera zone belongs to the shutter reading. An mtime carrying a + // +540 shift would be filed a day out for no reason at all. + let s = Shot { + captured_at: None, + captured_offset: Some(540), + modified_at: 1_787_351_400, // 21st, 22:30 UTC + ..Default::default() + }; + assert_eq!( + expand(DEFAULT_TEMPLATE, &s).segments, + ["2026", "2026-08-21"] + ); + } + + #[test] + fn every_token_expands() { + let s = Shot { + captured_at: Some(AUG_22), + make: Some("Canon".into()), + model: Some("EOS R5".into()), + ..shot(None) + }; + let d = expand("{make}/{model}/{yy}/{date}/{yyyy}-{mm}-{dd}", &s); + assert_eq!( + d.segments, + ["Canon", "EOS R5", "26", "2026-08-22", "2026-08-22"] + ); + } + + #[test] + fn an_unknown_token_is_left_where_the_user_can_see_it() { + let d = expand("{yyy}", &shot(Some(AUG_22))); + assert_eq!(d.segments, ["{yyy}"]); + } + + #[test] + fn a_segment_cannot_climb_out_of_the_destination() { + // The one bug in this file that would matter: a template, or a camera + // model, that walks up out of the library root. + let d = expand("../../etc/{yyyy}", &shot(Some(AUG_22))); + assert_eq!(d.segments, ["etc", "2026"]); + + let s = Shot { + model: Some("../..".into()), + ..shot(Some(AUG_22)) + }; + assert_eq!(expand("{model}/{yyyy}", &s).segments, ["2026"]); + } + + #[test] + fn a_leading_slash_does_not_make_it_absolute() { + let d = expand("/{yyyy}", &shot(Some(AUG_22))); + assert_eq!(d.segments, ["2026"]); + } + + #[test] + fn an_empty_token_does_not_leave_a_nameless_level() { + // No make recorded. `{make}/{yyyy}` must be one level, not two with + // an unnamed parent. + let d = expand("{make}/{yyyy}", &shot(Some(AUG_22))); + assert_eq!(d.segments, ["2026"]); + } + + #[test] + fn a_template_that_expands_to_nothing_falls_back_to_the_date() { + let d = expand("{make}", &shot(Some(AUG_22))); + assert_eq!(d.segments, ["2026", "2026-08-22"]); + } + + #[test] + fn characters_no_destination_would_take_are_replaced() { + let s = Shot { + // Nikon writes its model with a colon on some bodies, and a colon + // is a stream separator on Windows and illegal on SAF. + model: Some("NIKON:Z 9".into()), + ..shot(Some(AUG_22)) + }; + assert_eq!(expand("{model}", &s).segments, ["NIKON-Z 9"]); + } + + #[test] + fn expansion_does_not_depend_on_where_it_runs() { + // FR-NC-7a: two devices must compute the same remote destination for + // the same frame, so the same input twice is the same answer twice. + let s = shot(Some(AUG_22)); + assert_eq!(expand(DEFAULT_TEMPLATE, &s), expand(DEFAULT_TEMPLATE, &s)); + } + + #[test] + fn a_display_string_is_the_segments_joined() { + let d = expand(DEFAULT_TEMPLATE, &shot(Some(AUG_22))); + assert_eq!(d.display(), "2026/2026-08-22"); + } +} diff --git a/core/dr-ingest/src/lib.rs b/core/dr-ingest/src/lib.rs new file mode 100644 index 0000000..2ad1a7f --- /dev/null +++ b/core/dr-ingest/src/lib.rs @@ -0,0 +1,1319 @@ +//! TRACES: FR-CAT-10 | FR-CAT-11 +//! Bringing photographs in from a card. +//! +//! Distinct from `dr_catalog::scan`, and the distinction is the whole reason +//! this crate exists: a scan catalogues files where they already are, where an +//! import *moves them* from a card into the library. A scan that failed +//! halfway has read nothing; an import that fails halfway has written +//! something. +//! +//! # What this crate will not do +//! +//! **It never deletes from the card.** A move-import is expressed as a copy +//! plus a later retirement the caller performs once it is satisfied — because +//! on a library that syncs, "satisfied" means the upload was confirmed, not +//! that the local write returned (FR-NC-7b). A card erased against an +//! in-flight upload is the one failure in this application with no undo, so +//! the deletion is deliberately not reachable from the code path that is in a +//! hurry. See [`retire`]. +//! +//! **It does not decode.** Capture metadata arrives through [`Probe`], so this +//! crate does not depend on `dr-decode` and a card of unreadable files costs no +//! demosaic. The same shape `dr_export::name::resolve_name` uses for its +//! collision test, for the same reason: the answer differs by caller and none +//! of the variants belong here. +//! +//! **It does not know what a duplicate is.** That question is a catalog query +//! (FR-CAT-11), so it arrives through [`DuplicateCheck`]. + +use std::io::Read; + +use dr_plat::{DirRef, NewFile, SeekableRead, Storage, StorageError, WritableStorage}; +use dr_types::SourceRef; +use sha2::{Digest, Sha256}; + +pub mod layout; + +pub use layout::{expand, DateSource, Destination, Shot, DEFAULT_TEMPLATE}; + +/// How many bytes move between the card and the destination at a time. +/// +/// A RAW is 20–100 MB and there is no reason to hold one in memory: the read, +/// the hash and the write are all streaming, so peak usage is this buffer +/// whether the card holds one file or two thousand. 1 MiB rather than +/// something smaller because a card reader's throughput collapses on small +/// reads, and rather than something larger because the second copy +/// (FR-CAT-10's backup destination) doubles it. +const CHUNK: usize = 1024 * 1024; + +/// Something went wrong importing one file. +/// +/// Per-file rather than per-run: a card with one unreadable file must import +/// the other nine hundred, so this is recorded against the item in +/// [`Report::failed`] and the run continues. +#[derive(Debug, thiserror::Error)] +pub enum IngestError { + #[error("reading the card: {0}")] + Source(StorageError), + + #[error("writing the destination: {0}")] + Destination(StorageError), + + #[error("writing the backup copy: {0}")] + Backup(StorageError), + + /// The bytes that arrived are not the bytes that left. + /// + /// A dying card, a full destination that reported success, a cable on its + /// way out. The written file is removed before this is returned — a + /// corrupt import that stayed on disk would be found by the next scan and + /// catalogued as though it were fine. + #[error("verification failed: wrote {written} bytes, {reason}")] + Verify { written: u64, reason: String }, + + #[error("io error: {0}")] + Io(String), + + /// The user asked to stop. Not a failure of the file it is recorded + /// against — that file simply never started. + #[error("cancelled")] + Cancelled, +} + +/// What to do about a file the catalog has seen before. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)] +pub enum DuplicatePolicy { + /// Transfer nothing. Re-inserting a card that was already imported is the + /// common case and should cost one metadata read per file. + #[default] + Skip, + /// Import anyway, under a name that does not collide. + ImportAsNew, +} + +/// Whether the card copy may be deleted once the import is confirmed. +/// +/// Recorded rather than acted on — see the crate docs on why the deletion does +/// not live here. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)] +pub enum TransferMode { + #[default] + Copy, + /// The user asked for a move. [`Report::retirable`] carries the sources + /// whose card copies are now redundant. + Move, +} + +/// How an import is configured. +#[derive(Debug, Clone)] +pub struct Options { + /// The folder template (FR-NC-7a). See [`layout::DEFAULT_TEMPLATE`]. + pub folder_template: String, + pub mode: TransferMode, + pub on_duplicate: DuplicatePolicy, + /// Re-read each written file and compare digests (FR-CAT-10). + /// + /// On by default and worth the second read: the failure it catches is a + /// silently corrupt original, which is unrecoverable once the card is + /// reused and invisible until the photograph is opened months later. + pub verify: bool, +} + +impl Default for Options { + fn default() -> Self { + Self { + folder_template: DEFAULT_TEMPLATE.to_string(), + mode: TransferMode::Copy, + on_duplicate: DuplicatePolicy::Skip, + verify: true, + } + } +} + +/// One file offered for import. +#[derive(Debug, Clone)] +pub struct Candidate { + /// Where it is now — on the card. + pub source: SourceRef, + /// Its name on the card, extension included. + pub name: String, + /// Size in bytes, for progress. Zero where unknown. + pub size: u64, + /// The file's modification time, Unix seconds. The date fallback. + pub modified_at: i64, +} + +/// What identifies a photograph independently of where it is stored. +/// +/// FR-CAT-11's two tiers in one type. The metadata tier — capture time, camera, +/// original filename — is answerable before a byte is transferred, which is +/// what makes re-inserting an already-imported card cheap. The `digest` tier is +/// only knowable after reading the file, and catches the case the first misses: +/// the same frame arriving under a different name. +/// +/// Camera *serial* would be the stronger second key and is not here, because +/// nothing in the tree reads it yet — `dr_decode::Metadata` carries make and +/// model only. Make and model plus capture time plus filename is what is +/// available; the field is worth adding when the decoder grows it, and the +/// digest tier covers the gap meanwhile. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct DupKey { + pub captured_at: Option, + pub make: Option, + pub model: Option, + /// The name the camera gave it. Camera filenames wrap at `IMG_9999`, so + /// this is never sufficient alone. + pub original_name: String, + pub size: u64, + /// SHA-256 of the file's bytes, once they have been read. `None` on the + /// pre-transfer question. + pub digest: Option, +} + +/// Reads capture metadata for one candidate. +/// +/// Returning `None` is not an error: a card holds files no decoder in this +/// application understands, and they are still imported — dated by mtime, and +/// reported as such (FR-NC-7a). +pub type Probe<'a> = dyn Fn(&Candidate) -> Option + 'a; + +/// Answers whether the catalog already holds this photograph (FR-CAT-11). +/// +/// Called twice per file: once before any transfer with `digest: None`, and +/// once after the bytes have been read with the digest filled in. An +/// implementation that can only answer the first should return `false` for the +/// second rather than guessing. +pub type DuplicateCheck<'a> = dyn Fn(&DupKey) -> bool + 'a; + +/// Progress, reported per file as it completes. +pub type OnProgress<'a> = dyn FnMut(&Progress) + 'a; + +/// Where an import has got to. +#[derive(Debug, Clone)] +pub struct Progress { + /// Files finished, successfully or not. + pub done: usize, + pub total: usize, + /// Bytes transferred so far, backup copies not counted twice. + pub bytes: u64, + /// The file just finished. + pub name: String, +} + +/// One file that made it in. +#[derive(Debug, Clone)] +pub struct Imported { + /// Where it came from. + pub source: SourceRef, + /// Where it now is, as the destination storage named it — **not** as the + /// caller asked for it. A SAF provider renames on collision by itself, so + /// reading this back rather than assuming is not defensive style, it is + /// the only way to be right on Android (ARCH §6.9). + pub written: SourceRef, + /// The folders it was filed under, outermost first (FR-NC-7a). The upload + /// path expands the same template to the same segments, so this is also + /// where it belongs on the server. + pub folders: Vec, + /// The name it was written under, which differs from the card's when the + /// name was taken. + pub name: String, + /// SHA-256 of its bytes. Stored so the next import can answer FR-CAT-11's + /// second tier without re-reading anything. + pub digest: String, + pub size: u64, + /// Whether the folder date came from the shutter or from mtime. + pub dated_from: DateSource, + /// Where the second copy went, if one was asked for. + pub backup: Option, +} + +/// What an import did. +#[derive(Debug, Default)] +pub struct Report { + pub imported: Vec, + /// Files the catalog already held (FR-CAT-11). Nothing was transferred. + pub duplicates: Vec, + pub failed: Vec<(Candidate, IngestError)>, + /// Sources whose card copies are now redundant, for a move-import. + /// + /// **Populated, never acted on.** The caller deletes these once it is + /// satisfied the import is safe, which on a syncing library means after the + /// upload is confirmed (FR-NC-7b). Empty for a copy-import. + pub retirable: Vec, + /// Files dated by modification time because they carried no capture time. + /// + /// FR-NC-7a requires this to be visible rather than silent: they are filed + /// under when they were last written, which is often the day the card was + /// read rather than the day the photograph was taken. + pub undated: Vec, + pub bytes: u64, + /// Whether the run stopped early because the caller asked it to. + pub cancelled: bool, +} + +impl Report { + /// Whether anything at all was written. + pub fn is_empty(&self) -> bool { + self.imported.is_empty() + } +} + +/// An import in progress. +/// +/// A struct rather than a function with nine parameters, and the borrows are +/// the point: the source is `&dyn Storage` and the destination is +/// `&dyn WritableStorage`, which is exactly the asymmetry of copying off a +/// card. A card mounted read-only cannot be passed where a destination is +/// wanted, and the compiler says so rather than the filesystem saying so +/// halfway through. +pub struct Ingest<'a> { + pub source: &'a dyn Storage, + pub dest: &'a dyn WritableStorage, + /// The library root, or a folder inside it. Template folders are created + /// beneath this. + pub dest_root: &'a DirRef, + /// The optional simultaneous second destination (FR-CAT-10). + /// + /// Written from the same read as the primary, not copied from it + /// afterwards: a backup made by re-reading the primary would inherit a bad + /// write rather than catch it, and re-reading the *card* doubles the wear + /// on the one copy that still exists. + pub backup: Option<(&'a dyn WritableStorage, &'a DirRef)>, + pub options: Options, +} + +impl Ingest<'_> { + /// Run the import. + /// + /// `cancel` is polled between files rather than within one: a half-written + /// RAW is worse than a few more seconds of waiting, and at 1 MiB chunks the + /// longest a cancel waits is one file. + pub fn run( + &self, + candidates: &[Candidate], + probe: &Probe<'_>, + is_duplicate: &DuplicateCheck<'_>, + cancel: &dyn Fn() -> bool, + on_progress: &mut OnProgress<'_>, + ) -> Report { + let mut report = Report::default(); + let total = candidates.len(); + + for candidate in candidates { + if cancel() { + report.cancelled = true; + break; + } + + // The probe reads *content*, so the modification time is not its + // to report — it comes from the listing that produced the + // candidate. Overwritten rather than defaulted, so a probe that + // leaves the field at zero cannot file a whole card under 1970. + let mut shot = probe(candidate).unwrap_or_default(); + shot.modified_at = candidate.modified_at; + + // Tier one, before a byte moves: the card that was already + // imported costs one metadata read per file and no transfer. + let key = DupKey { + captured_at: shot.captured_at, + make: shot.make.clone(), + model: shot.model.clone(), + original_name: candidate.name.clone(), + size: candidate.size, + digest: None, + }; + if self.options.on_duplicate == DuplicatePolicy::Skip && is_duplicate(&key) { + report.duplicates.push(candidate.clone()); + report.done(&mut *on_progress, total, candidate); + continue; + } + + match self.one(candidate, &shot, &key, is_duplicate) { + Ok(Some(imported)) => { + report.bytes += imported.size; + if imported.dated_from == DateSource::Modified { + report.undated.push(imported.name.clone()); + } + if self.options.mode == TransferMode::Move { + report.retirable.push(imported.source.clone()); + } + report.imported.push(imported); + } + // The digest tier caught it after reading. Nothing was kept. + Ok(None) => report.duplicates.push(candidate.clone()), + Err(e) => { + log::warn!("importing {}: {e}", candidate.name); + report.failed.push((candidate.clone(), e)); + } + } + report.done(&mut *on_progress, total, candidate); + } + + report + } + + /// Import one file. `Ok(None)` means the digest tier found a duplicate. + fn one( + &self, + candidate: &Candidate, + shot: &Shot, + key: &DupKey, + is_duplicate: &DuplicateCheck<'_>, + ) -> Result, IngestError> { + let destination = layout::expand(&self.options.folder_template, shot); + + let dir = self.make_dirs(self.dest, self.dest_root, &destination.segments)?; + let backup_dir = match self.backup { + Some((storage, root)) => Some(( + storage, + self.make_dirs(storage, root, &destination.segments) + .map_err(|e| match e { + IngestError::Destination(e) => IngestError::Backup(e), + other => other, + })?, + )), + None => None, + }; + + let name = self.free_name(self.dest, &dir, &candidate.name)?; + + let mut reader = self + .source + .open(&candidate.source) + .map_err(IngestError::Source)?; + + let primary = self + .dest + .create_file(&dir, &name) + .map_err(IngestError::Destination)?; + let secondary = match &backup_dir { + Some((storage, bdir)) => { + let bname = self.free_name(*storage, bdir, &candidate.name)?; + Some( + storage + .create_file(bdir, &bname) + .map_err(IngestError::Backup)?, + ) + } + None => None, + }; + + let written = self.stream(&mut reader, primary, secondary); + + // Any failure past the point of creating the file leaves bytes on + // disk that no catalog knows about. Take them back before returning, + // or the next scan catalogues a truncated RAW as though it were fine. + let (digest, size, target, backup_target) = match written { + Ok(v) => v, + Err((e, targets)) => { + self.discard(targets); + return Err(e); + } + }; + + if self.options.verify { + if let Err(e) = self.verify(&target, &digest, size) { + self.discard( + vec![(self.dest, target)] + .into_iter() + .chain(backup_target.map(|t| (self.backup.unwrap().0, t))) + .collect(), + ); + return Err(e); + } + } + + // Tier two (FR-CAT-11): the same frame under a different name. Only + // answerable now, and cheap to act on — the file is removed rather + // than catalogued twice. + if self.options.on_duplicate == DuplicatePolicy::Skip { + let with_digest = DupKey { + digest: Some(digest.clone()), + ..key.clone() + }; + if is_duplicate(&with_digest) { + self.discard( + vec![(self.dest, target)] + .into_iter() + .chain(backup_target.map(|t| (self.backup.unwrap().0, t))) + .collect(), + ); + return Ok(None); + } + } + + Ok(Some(Imported { + source: candidate.source.clone(), + written: target, + folders: destination.segments, + name, + digest, + size, + dated_from: destination.dated_from, + backup: backup_target, + })) + } + + /// Create each level of the template, keeping what the provider returned. + /// + /// Never `create_dir_all` on a joined path: the segments are separate + /// because on SAF the child's reference can only come from the call that + /// created it (ARCH §6.9). + fn make_dirs( + &self, + storage: &dyn WritableStorage, + root: &DirRef, + segments: &[String], + ) -> Result { + let mut dir = root.clone(); + for segment in segments { + dir = storage + .create_dir(&dir, segment) + .map_err(IngestError::Destination)?; + } + Ok(dir) + } + + /// A name not already taken in the destination. + /// + /// Two cards can hold `IMG_0001.CR3` and they are different photographs. + /// The suffix goes before the extension so the file still opens. + fn free_name( + &self, + storage: &dyn WritableStorage, + dir: &DirRef, + name: &str, + ) -> Result { + if !storage + .exists(dir, name) + .map_err(IngestError::Destination)? + { + return Ok(name.to_string()); + } + let (stem, ext) = match name.rsplit_once('.') { + Some((s, e)) if !s.is_empty() => (s, Some(e)), + _ => (name, None), + }; + for n in 1..1000 { + let candidate = match ext { + Some(ext) => format!("{stem}-{n}.{ext}"), + None => format!("{stem}-{n}"), + }; + if !storage + .exists(dir, &candidate) + .map_err(IngestError::Destination)? + { + return Ok(candidate); + } + } + Err(IngestError::Destination(StorageError::AlreadyExists( + name.to_string(), + ))) + } + + /// Read once, write twice, hash on the way past. + /// + /// The backup is written from the same read rather than copied from the + /// primary afterwards — see [`Ingest::backup`]. On error the created files + /// come back with it so the caller can take them away again. + #[allow(clippy::type_complexity)] + fn stream( + &self, + reader: &mut Box, + primary: NewFile, + secondary: Option, + ) -> Result< + (String, u64, SourceRef, Option), + (IngestError, Vec<(&dyn WritableStorage, SourceRef)>), + > { + let NewFile { + source: target, + sink: mut out, + } = primary; + let (backup_target, mut backup_out) = match secondary { + Some(NewFile { source, sink }) => (Some(source), Some(sink)), + None => (None, None), + }; + + let created = |extra: Option<&SourceRef>| -> Vec<(&dyn WritableStorage, SourceRef)> { + let mut v: Vec<(&dyn WritableStorage, SourceRef)> = vec![(self.dest, target.clone())]; + if let (Some(t), Some((storage, _))) = (extra, self.backup) { + v.push((storage, t.clone())); + } + v + }; + + let mut hasher = Sha256::new(); + let mut buf = vec![0u8; CHUNK]; + let mut size = 0u64; + loop { + let n = match reader.read(&mut buf) { + Ok(0) => break, + Ok(n) => n, + Err(e) => { + return Err(( + IngestError::Source(StorageError::Io(e.to_string())), + created(backup_target.as_ref()), + )) + } + }; + let chunk = &buf[..n]; + hasher.update(chunk); + size += n as u64; + if let Err(e) = out.write_all(chunk) { + return Err(( + IngestError::Destination(StorageError::Io(e.to_string())), + created(backup_target.as_ref()), + )); + } + if let Some(b) = backup_out.as_mut() { + if let Err(e) = b.write_all(chunk) { + return Err(( + IngestError::Backup(StorageError::Io(e.to_string())), + created(backup_target.as_ref()), + )); + } + } + } + + // Flush explicitly rather than on drop. A buffered writer swallows the + // error when it is dropped, and "the disk filled on the last chunk" is + // exactly the failure this whole path exists to catch. + if let Err(e) = out.flush() { + return Err(( + IngestError::Destination(StorageError::Io(e.to_string())), + created(backup_target.as_ref()), + )); + } + if let Some(b) = backup_out.as_mut() { + if let Err(e) = b.flush() { + return Err(( + IngestError::Backup(StorageError::Io(e.to_string())), + created(backup_target.as_ref()), + )); + } + } + drop(out); + drop(backup_out); + + Ok((hex(&hasher.finalize()), size, target, backup_target)) + } + + /// Re-read what was written and confirm it is what was sent. + /// + /// Reads the *destination*, not the buffer that was just hashed. Hashing + /// what is still in memory would confirm only that the process can add — it + /// would pass on a full disk, a dying card and a truncated write alike. + fn verify(&self, target: &SourceRef, expected: &str, size: u64) -> Result<(), IngestError> { + let mut reader = self.dest.open(target).map_err(|e| IngestError::Verify { + written: size, + reason: format!("cannot re-read it: {e}"), + })?; + let mut hasher = Sha256::new(); + let mut buf = vec![0u8; CHUNK]; + let mut seen = 0u64; + loop { + let n = reader.read(&mut buf).map_err(|e| IngestError::Verify { + written: size, + reason: format!("cannot re-read it: {e}"), + })?; + if n == 0 { + break; + } + hasher.update(&buf[..n]); + seen += n as u64; + } + let got = hex(&hasher.finalize()); + if seen != size { + return Err(IngestError::Verify { + written: size, + reason: format!("{seen} bytes came back"), + }); + } + if got != expected { + return Err(IngestError::Verify { + written: size, + reason: format!("digest {got} does not match {expected}"), + }); + } + Ok(()) + } + + /// Take back files this import created but is not keeping. + /// + /// Failures here are logged and not propagated: this runs on a path that + /// already has an error to report, and replacing it with "and the cleanup + /// also failed" loses the one the user needs. + fn discard(&self, targets: Vec<(&dyn WritableStorage, SourceRef)>) { + for (storage, target) in targets { + if let Err(e) = storage.remove_file(&target) { + log::warn!("could not remove a partial import: {e}"); + } + } + } +} + +impl Report { + fn done(&self, on_progress: &mut OnProgress<'_>, total: usize, candidate: &Candidate) { + on_progress(&Progress { + done: self.duplicates.len() + self.imported.len() + self.failed.len(), + total, + bytes: self.bytes, + name: candidate.name.clone(), + }); + } +} + +/// TRACES: FR-CAT-10 | FR-NC-7b +/// Delete the card copies of files that are safely in the library. +/// +/// Separate from [`Ingest::run`] and deliberately awkward to reach. Pass only +/// [`Report::retirable`], and only once whatever "safe" means for this library +/// is true — for one that syncs, that is the upload confirmed, not the local +/// write returned (FR-NC-7b). There is no undo behind this function: the card +/// is where the only other copy was. +/// +/// Returns the sources it could not delete, which is a card pulled mid-run or +/// a write-protect switch, and neither is worth stopping for. +pub fn retire(card: &dyn WritableStorage, sources: &[SourceRef]) -> Vec<(SourceRef, StorageError)> { + let mut failed = Vec::new(); + for src in sources { + if let Err(e) = card.remove_file(src) { + log::warn!("could not retire {src:?} from the card: {e}"); + failed.push((src.clone(), e)); + } + } + failed +} + +/// A digest as lower-case hex, the form `sha256sum` prints. +fn hex(bytes: &[u8]) -> String { + let mut s = String::with_capacity(bytes.len() * 2); + for b in bytes { + s.push_str(&format!("{b:02x}")); + } + s +} + +#[cfg(test)] +mod tests { + use super::*; + use std::cell::RefCell; + use std::path::PathBuf; + + use dr_plat::{Entry, LocalStorage}; + use dr_types::{ByteRange, DirState, RootId}; + + const CARD: RootId = RootId(1); + const LIB: RootId = RootId(2); + /// 2026-08-22T14:00:00Z. + const AUG_22: i64 = 1_787_407_200; + + /// A throwaway directory, removed when the test ends. + struct Tree(PathBuf); + + impl Tree { + fn new(name: &str) -> Self { + let dir = std::env::temp_dir().join(format!( + "dr-ingest-{name}-{}-{:?}", + std::process::id(), + std::thread::current().id() + )); + let _ = std::fs::remove_dir_all(&dir); + std::fs::create_dir_all(&dir).expect("temp dir"); + Tree(dir) + } + + fn sub(&self, rel: &str) -> PathBuf { + let p = self.0.join(rel); + std::fs::create_dir_all(&p).expect("mkdir"); + p + } + } + + impl Drop for Tree { + fn drop(&mut self) { + let _ = std::fs::remove_dir_all(&self.0); + } + } + + /// A card with files on it, and the library it imports into. + struct Fixture { + _tree: Tree, + card: LocalStorage, + card_path: PathBuf, + lib: LocalStorage, + lib_path: PathBuf, + } + + fn fixture(name: &str) -> Fixture { + let tree = Tree::new(name); + let card_path = tree.sub("card/DCIM/100CANON"); + let lib_path = tree.sub("library"); + Fixture { + card: LocalStorage::with_root(CARD, tree.0.join("card")), + lib: LocalStorage::with_root(LIB, lib_path.clone()), + card_path, + lib_path, + _tree: tree, + } + } + + impl Fixture { + /// Put a file on the card and return the candidate for it. + fn shoot(&self, name: &str, bytes: &[u8]) -> Candidate { + std::fs::write(self.card_path.join(name), bytes).expect("write"); + Candidate { + source: SourceRef::Local { + root: CARD, + relative: format!("DCIM/100CANON/{name}"), + }, + name: name.to_string(), + size: bytes.len() as u64, + modified_at: AUG_22, + } + } + + fn lib_root(&self) -> DirRef { + DirRef::root(LIB) + } + + fn read(&self, rel: &str) -> Vec { + std::fs::read(self.lib_path.join(rel)).expect("read imported file") + } + } + + /// A `Shot` that dates to the 22nd of August 2026. + fn on_the_22nd() -> Shot { + Shot { + captured_at: Some(AUG_22), + make: Some("Canon".into()), + model: Some("EOS R5".into()), + ..Default::default() + } + } + + fn never(_: &DupKey) -> bool { + false + } + + fn run( + f: &Fixture, + candidates: &[Candidate], + options: Options, + probe: &Probe<'_>, + is_duplicate: &DuplicateCheck<'_>, + ) -> Report { + let root = f.lib_root(); + let ingest = Ingest { + source: &f.card, + dest: &f.lib, + dest_root: &root, + backup: None, + options, + }; + ingest.run(candidates, probe, is_duplicate, &|| false, &mut |_| {}) + } + + #[test] + fn a_card_is_filed_by_year_and_day() { + let f = fixture("by-date"); + let c = f.shoot("IMG_0001.CR3", b"raw bytes"); + + let report = run( + &f, + &[c], + Options::default(), + &|_| Some(on_the_22nd()), + &never, + ); + + assert_eq!(report.imported.len(), 1); + let imported = &report.imported[0]; + assert_eq!(imported.folders, ["2026", "2026-08-22"]); + assert_eq!(f.read("2026/2026-08-22/IMG_0001.CR3"), b"raw bytes"); + // The digest is stored so the next import can answer FR-CAT-11's + // second tier without reading the file again. + assert_eq!(imported.digest.len(), 64); + assert_eq!(report.bytes, 9); + } + + #[test] + fn the_digest_is_the_one_sha256sum_would_print() { + let f = fixture("digest"); + let c = f.shoot("IMG_0001.CR3", b"abc"); + let report = run( + &f, + &[c], + Options::default(), + &|_| Some(on_the_22nd()), + &never, + ); + // Stored and compared across devices and across years, so it has to be + // a value someone can recompute at a shell prompt. + assert_eq!( + report.imported[0].digest, + "ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad" + ); + } + + #[test] + fn a_card_that_was_already_imported_transfers_nothing() { + let f = fixture("dupe-cheap"); + let c = f.shoot("IMG_0001.CR3", b"raw bytes"); + + // Tier one: answered from metadata, before a byte moves. + let asked: RefCell> = RefCell::new(Vec::new()); + let report = run( + &f, + &[c], + Options::default(), + &|_| Some(on_the_22nd()), + &|k| { + asked.borrow_mut().push(k.clone()); + true + }, + ); + + assert_eq!(report.duplicates.len(), 1); + assert!(report.imported.is_empty()); + assert!(!f.lib_path.join("2026").exists(), "nothing was written"); + // Asked exactly once, without a digest — the whole point is that the + // file was never read. + assert_eq!(asked.borrow().len(), 1); + assert_eq!(asked.borrow()[0].digest, None); + assert_eq!(asked.borrow()[0].original_name, "IMG_0001.CR3"); + } + + #[test] + fn the_same_frame_under_a_different_name_is_caught_by_its_digest() { + let f = fixture("dupe-digest"); + let c = f.shoot("DSC_4242.NEF", b"the same photograph"); + + // Tier two: the metadata key misses (the name differs), so the file is + // read — and then must not be kept. + let report = run( + &f, + &[c], + Options::default(), + &|_| Some(on_the_22nd()), + &|k| k.digest.is_some(), + ); + + assert_eq!(report.duplicates.len(), 1); + assert!(report.imported.is_empty()); + assert!( + !f.lib_path.join("2026/2026-08-22/DSC_4242.NEF").exists(), + "the copy made to compute the digest was taken back" + ); + } + + #[test] + fn importing_as_new_keeps_a_known_duplicate() { + let f = fixture("dupe-keep"); + let c = f.shoot("IMG_0001.CR3", b"raw bytes"); + + let report = run( + &f, + &[c], + Options { + on_duplicate: DuplicatePolicy::ImportAsNew, + ..Default::default() + }, + &|_| Some(on_the_22nd()), + &|_| true, + ); + + assert_eq!(report.imported.len(), 1); + assert!(report.duplicates.is_empty()); + } + + #[test] + fn two_cards_holding_the_same_filename_do_not_overwrite() { + let f = fixture("collision"); + // Camera filenames wrap at IMG_9999, so this is ordinary rather than + // exotic: two different photographs, one name. + let first = f.shoot("IMG_0001.CR3", b"the first frame"); + let report = run( + &f, + &[first], + Options::default(), + &|_| Some(on_the_22nd()), + &never, + ); + assert_eq!(report.imported[0].name, "IMG_0001.CR3"); + + let second = f.shoot("IMG_0001.CR3", b"a different frame"); + let report = run( + &f, + &[second], + Options::default(), + &|_| Some(on_the_22nd()), + &never, + ); + + assert_eq!(report.imported[0].name, "IMG_0001-1.CR3"); + // The suffix goes before the extension, so the file still opens. + assert_eq!(f.read("2026/2026-08-22/IMG_0001.CR3"), b"the first frame"); + assert_eq!( + f.read("2026/2026-08-22/IMG_0001-1.CR3"), + b"a different frame" + ); + } + + #[test] + fn a_file_with_no_capture_time_is_reported_not_hidden() { + let f = fixture("undated"); + let c = f.shoot("SCAN_01.TIF", b"scanned film"); + + // No decoder understands it, so it is dated by mtime — and the user is + // told, because mtime is often the day the card was read (FR-NC-7a). + let report = run(&f, &[c], Options::default(), &|_| None, &never); + + assert_eq!(report.imported.len(), 1); + assert_eq!(report.imported[0].dated_from, DateSource::Modified); + assert_eq!(report.undated, ["SCAN_01.TIF"]); + assert_eq!(f.read("2026/2026-08-22/SCAN_01.TIF"), b"scanned film"); + } + + #[test] + fn a_probe_that_forgets_the_modification_time_does_not_file_a_card_under_1970() { + let f = fixture("probe-zero"); + let c = f.shoot("IMG_0001.CR3", b"x"); + // A probe returns content metadata and nothing else. If its zeroed + // mtime were trusted, a card of undated files would land in 1970. + let report = run( + &f, + &[c], + Options::default(), + &|_| Some(Shot::default()), + &never, + ); + assert_eq!(report.imported[0].folders, ["2026", "2026-08-22"]); + } + + #[test] + fn a_backup_copy_is_written_from_the_same_read() { + let f = fixture("backup"); + let backup_path = f._tree.sub("backup"); + let backup = LocalStorage::with_root(RootId(3), backup_path.clone()); + let c = f.shoot("IMG_0001.CR3", b"raw bytes"); + + let root = f.lib_root(); + let backup_root = DirRef::root(RootId(3)); + let ingest = Ingest { + source: &f.card, + dest: &f.lib, + dest_root: &root, + backup: Some((&backup, &backup_root)), + options: Options::default(), + }; + let report = ingest.run( + &[c], + &|_| Some(on_the_22nd()), + &never, + &|| false, + &mut |_| {}, + ); + + assert_eq!(report.imported.len(), 1); + assert!(report.imported[0].backup.is_some()); + // The same folder template on both sides, so the backup is a library + // rather than a heap. + assert_eq!( + std::fs::read(backup_path.join("2026/2026-08-22/IMG_0001.CR3")).unwrap(), + b"raw bytes" + ); + // One read of the card produced both, so the byte count is not doubled. + assert_eq!(report.bytes, 9); + } + + #[test] + fn an_unreadable_file_does_not_stop_the_card() { + let f = fixture("partial-failure"); + let good = f.shoot("IMG_0001.CR3", b"fine"); + // A file the listing saw and the reader cannot open — a card on its + // way out. The other nine hundred images still have to import. + let missing = Candidate { + source: SourceRef::Local { + root: CARD, + relative: "DCIM/100CANON/GONE.CR3".into(), + }, + name: "GONE.CR3".into(), + size: 4, + modified_at: AUG_22, + }; + let also_good = f.shoot("IMG_0002.CR3", b"also fine"); + + let report = run( + &f, + &[good, missing, also_good], + Options::default(), + &|_| Some(on_the_22nd()), + &never, + ); + + assert_eq!(report.imported.len(), 2); + assert_eq!(report.failed.len(), 1); + assert_eq!(report.failed[0].0.name, "GONE.CR3"); + // And nothing was left behind for the next scan to catalogue. + assert!(!f.lib_path.join("2026/2026-08-22/GONE.CR3").exists()); + } + + #[test] + fn cancelling_stops_between_files_rather_than_inside_one() { + let f = fixture("cancel"); + let a = f.shoot("IMG_0001.CR3", b"first"); + let b = f.shoot("IMG_0002.CR3", b"second"); + let c = f.shoot("IMG_0003.CR3", b"third"); + + let seen = RefCell::new(0usize); + let root = f.lib_root(); + let ingest = Ingest { + source: &f.card, + dest: &f.lib, + dest_root: &root, + backup: None, + options: Options::default(), + }; + let report = ingest.run( + &[a, b, c], + &|_| Some(on_the_22nd()), + &never, + &|| { + let mut n = seen.borrow_mut(); + *n += 1; + // Cancel is asked before each file: allow two, refuse the third. + *n > 2 + }, + &mut |_| {}, + ); + + assert!(report.cancelled); + assert_eq!(report.imported.len(), 2); + // Whole files or nothing — a half-written RAW is worse than waiting. + assert_eq!(f.read("2026/2026-08-22/IMG_0002.CR3"), b"second"); + assert!(!f.lib_path.join("2026/2026-08-22/IMG_0003.CR3").exists()); + } + + #[test] + fn progress_counts_every_file_including_the_ones_it_skipped() { + let f = fixture("progress"); + let a = f.shoot("IMG_0001.CR3", b"first"); + let b = f.shoot("IMG_0002.CR3", b"second"); + + let ticks: RefCell> = RefCell::new(Vec::new()); + let root = f.lib_root(); + let ingest = Ingest { + source: &f.card, + dest: &f.lib, + dest_root: &root, + backup: None, + options: Options::default(), + }; + ingest.run( + &[a, b], + &|_| Some(on_the_22nd()), + &never, + &|| false, + &mut |p| ticks.borrow_mut().push((p.done, p.total)), + ); + + assert_eq!(*ticks.borrow(), [(1, 2), (2, 2)]); + } + + #[test] + fn a_move_import_records_what_may_be_deleted_but_deletes_nothing() { + let f = fixture("move"); + let c = f.shoot("IMG_0001.CR3", b"raw bytes"); + let on_card = f.card_path.join("IMG_0001.CR3"); + + let report = run( + &f, + &[c], + Options { + mode: TransferMode::Move, + ..Default::default() + }, + &|_| Some(on_the_22nd()), + &never, + ); + + assert_eq!(report.retirable.len(), 1); + // The card is still intact. On a syncing library the deletion waits for + // the upload to be confirmed (FR-NC-7b), and this crate cannot know + // that it was. + assert!(on_card.exists()); + + // Only when the caller says so. + let failed = retire(&f.card, &report.retirable); + assert!(failed.is_empty()); + assert!(!on_card.exists()); + } + + #[test] + fn a_copy_import_marks_nothing_for_deletion() { + let f = fixture("copy-keeps"); + let c = f.shoot("IMG_0001.CR3", b"raw bytes"); + let report = run( + &f, + &[c], + Options::default(), + &|_| Some(on_the_22nd()), + &never, + ); + assert!(report.retirable.is_empty()); + } + + // ---- verification (FR-CAT-10) --------------------------------------- + + /// A destination that writes fewer bytes than it was given. + /// + /// What a card on its way out, or a filesystem that reported a successful + /// write into a full disk, looks like from here. Verification exists for + /// exactly this, so it needs a way to happen on purpose. + struct Truncating(LocalStorage); + + struct ShortSink(Box); + + impl std::io::Write for ShortSink { + fn write(&mut self, buf: &[u8]) -> std::io::Result { + // Drops the last byte of every chunk and claims it wrote them all, + // which is precisely the failure a digest catches and a byte count + // alone would not. + let keep = buf.len().saturating_sub(1); + self.0.write_all(&buf[..keep])?; + Ok(buf.len()) + } + fn flush(&mut self) -> std::io::Result<()> { + self.0.flush() + } + } + + impl Storage for Truncating { + fn roots(&self) -> Vec { + self.0.roots() + } + fn root_dir(&self, root: RootId) -> Result { + self.0.root_dir(root) + } + fn dir_state(&self, dir: &DirRef) -> Result { + self.0.dir_state(dir) + } + fn list(&self, dir: &DirRef) -> Result, StorageError> { + self.0.list(dir) + } + fn open(&self, src: &SourceRef) -> Result, StorageError> { + self.0.open(src) + } + fn read_range(&self, src: &SourceRef, range: ByteRange) -> Result, StorageError> { + self.0.read_range(src, range) + } + } + + impl WritableStorage for Truncating { + fn create_dir(&self, parent: &DirRef, name: &str) -> Result { + self.0.create_dir(parent, name) + } + fn create_file(&self, parent: &DirRef, name: &str) -> Result { + let nf = self.0.create_file(parent, name)?; + Ok(NewFile { + source: nf.source, + sink: Box::new(ShortSink(nf.sink)), + }) + } + fn exists(&self, parent: &DirRef, name: &str) -> Result { + self.0.exists(parent, name) + } + fn remove_file(&self, src: &SourceRef) -> Result<(), StorageError> { + self.0.remove_file(src) + } + } + + #[test] + fn a_corrupt_write_is_caught_and_taken_back() { + let f = fixture("verify-fails"); + let c = f.shoot("IMG_0001.CR3", b"raw bytes"); + let lib = Truncating(LocalStorage::with_root(LIB, f.lib_path.clone())); + let root = DirRef::root(LIB); + + let ingest = Ingest { + source: &f.card, + dest: &lib, + dest_root: &root, + backup: None, + options: Options::default(), + }; + let report = ingest.run( + &[c], + &|_| Some(on_the_22nd()), + &never, + &|| false, + &mut |_| {}, + ); + + assert!(report.imported.is_empty()); + assert_eq!(report.failed.len(), 1); + assert!( + matches!(report.failed[0].1, IngestError::Verify { .. }), + "{:?}", + report.failed[0].1 + ); + // Removed rather than left: a truncated RAW on disk would be found by + // the next scan and catalogued as though it were fine. + assert!(!f.lib_path.join("2026/2026-08-22/IMG_0001.CR3").exists()); + } + + #[test] + fn verification_can_be_turned_off_and_then_nothing_catches_it() { + let f = fixture("verify-off"); + let c = f.shoot("IMG_0001.CR3", b"raw bytes"); + let lib = Truncating(LocalStorage::with_root(LIB, f.lib_path.clone())); + let root = DirRef::root(LIB); + + let ingest = Ingest { + source: &f.card, + dest: &lib, + dest_root: &root, + backup: None, + options: Options { + verify: false, + ..Default::default() + }, + }; + let report = ingest.run( + &[c], + &|_| Some(on_the_22nd()), + &never, + &|| false, + &mut |_| {}, + ); + + // Documents the cost of the option rather than endorsing it: the + // import "succeeds" and the file on disk is short. + assert_eq!(report.imported.len(), 1); + assert_eq!(f.read("2026/2026-08-22/IMG_0001.CR3"), b"raw byte"); + } + + #[test] + fn a_file_larger_than_one_chunk_survives_the_round_trip() { + let f = fixture("large"); + // Streaming is the whole design — a RAW is 20–100 MB and never sits in + // memory. Anything that only works below the buffer size is a bug this + // catches. + let bytes: Vec = (0..(CHUNK * 2 + 1234)).map(|i| (i % 251) as u8).collect(); + let c = f.shoot("IMG_0001.CR3", &bytes); + + let report = run( + &f, + &[c], + Options::default(), + &|_| Some(on_the_22nd()), + &never, + ); + + assert_eq!(report.imported.len(), 1); + assert_eq!(report.imported[0].size, bytes.len() as u64); + assert_eq!(f.read("2026/2026-08-22/IMG_0001.CR3"), bytes); + } +}