diff --git a/Cargo.lock b/Cargo.lock index f654272..b2ea7c3 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1330,7 +1330,9 @@ checksum = "d8b14ccef22fc6f5a8f4d7d768562a182c04ce9a3b3157b91390b52ddfdf1a76" name = "dr-catalog" version = "0.1.0" dependencies = [ + "dr-plat", "dr-types", + "env_logger", "log", "rusqlite", "serde_json", diff --git a/core/dr-catalog/Cargo.toml b/core/dr-catalog/Cargo.toml index fe2e1b3..6be0996 100644 --- a/core/dr-catalog/Cargo.toml +++ b/core/dr-catalog/Cargo.toml @@ -7,9 +7,19 @@ license.workspace = true [dependencies] dr-types.workspace = true +# The `Storage` trait, and nothing else from it. A scan has to read a real +# directory, and this is how `core/` reaches the platform without a +# `#[cfg(target_os)]` of its own (ARCH §4.1: calls go downward). +dr-plat.workspace = true rusqlite.workspace = true thiserror.workspace = true log.workspace = true # `collections.selector_json` — the stored form of a smart collection's # selector. The column predates this dependency; nothing else here is JSON. serde_json.workspace = true + +# For the `scan_local` example only, which is a diagnostic tool: what it is +# diagnosing is often a folder the scan warned about and skipped, and those +# warnings go to `log`. +[dev-dependencies] +env_logger.workspace = true diff --git a/core/dr-catalog/examples/scan_local.rs b/core/dr-catalog/examples/scan_local.rs new file mode 100644 index 0000000..a75694f --- /dev/null +++ b/core/dr-catalog/examples/scan_local.rs @@ -0,0 +1,111 @@ +//! Scan a real folder on this machine into a catalog, and say what it cost. +//! +//! cargo run -p dr-catalog --example scan_local -- ~/Pictures [catalog.sqlite] +//! +//! **Run it twice.** The first run is a full walk; the second is the one worth +//! watching, because on an unchanged library it should list no directories at +//! all and take a fraction of the time. That difference is NFR-P1, and a +//! synthetic test cannot show it at the scale a real library does — 121,785 +//! files in a synced folder is a different question from twenty in a temporary +//! directory. +//! +//! Writes only to the catalog file, which defaults to a fixed path in the +//! system temporary directory so a second run has something to compare +//! against. Nothing in the scanned folder is touched. + +use std::path::PathBuf; + +use dr_catalog::walk::{ensure_root, scan_root, RootKind}; +use dr_catalog::Catalog; +use dr_plat::LocalStorage; +use dr_types::FormatFilter; + +fn main() { + env_logger::init(); + + let mut args = std::env::args().skip(1); + let Some(dir) = args.next().map(PathBuf::from) else { + eprintln!("usage: scan_local [catalog.sqlite]"); + std::process::exit(2); + }; + let catalog_path = args + .next() + .map(PathBuf::from) + .unwrap_or_else(|| std::env::temp_dir().join("darkroom-scan-local.sqlite")); + + let catalog = match Catalog::open(&catalog_path) { + Ok(c) => c, + Err(e) => { + eprintln!("cannot open {}: {e}", catalog_path.display()); + std::process::exit(1); + } + }; + println!("catalog: {}", catalog_path.display()); + + // The label is how the grant is spelled, and the only place a path is + // written down. Everything after this line addresses files by `RootId`. + let label = dir.display().to_string(); + let root = match ensure_root(catalog.connection(), RootKind::Local, &label) { + Ok(r) => r, + Err(e) => { + eprintln!("cannot record the root: {e}"); + std::process::exit(1); + } + }; + let storage = LocalStorage::with_root(root, &dir); + + let now = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .map(|d| d.as_secs() as i64) + .unwrap_or(0); + + let started = std::time::Instant::now(); + let report = match scan_root( + catalog.connection(), + &storage, + root, + &FormatFilter::all(), + now, + || false, + |p| { + // One line per hundred directories: enough to show it is alive on a + // large library, not enough to be the thing that slows it down. + let visited = p.directories_listed + p.directories_pruned; + if visited % 100 == 0 { + println!( + " … {visited} directories ({} pruned), {} images", + p.directories_pruned, p.images_found + ); + } + }, + ) { + Ok(r) => r, + Err(e) => { + eprintln!("scan failed: {e}"); + std::process::exit(1); + } + }; + let elapsed = started.elapsed(); + + let total: i64 = catalog + .connection() + .query_row("SELECT count(*) FROM images", [], |r| r.get(0)) + .unwrap_or(-1); + + println!("\noutcome: {:?}", report.outcome); + println!( + "directories: {} listed, {} pruned", + report.progress.directories_listed, report.progress.directories_pruned + ); + println!( + "images: {} new, {} changed, {} unchanged, {} removed", + report.inserted, report.updated, report.unchanged, report.images_removed + ); + println!("folders: {} removed", report.folders_removed); + println!("catalogued: {total} in total"); + println!("took: {:.2?}", elapsed); + + if report.progress.directories_listed == 0 && report.progress.directories_pruned > 0 { + println!("\nnothing had changed: every folder was proven unchanged by one probe"); + } +} diff --git a/core/dr-catalog/src/error.rs b/core/dr-catalog/src/error.rs index 56391f7..b6d6f08 100644 --- a/core/dr-catalog/src/error.rs +++ b/core/dr-catalog/src/error.rs @@ -25,6 +25,15 @@ pub enum CatalogError { #[error("root {0} is unreachable; scan aborted without pruning")] RootUnreachable(u64), + /// A scan was asked for a root the catalog has no row for. + /// + /// A caller's mistake rather than a user's: the row is created when the + /// grant is obtained, because the label — the path, the tree URI — is known + /// only there. Inventing one here would file the library under a name + /// nothing else would look it up by. + #[error("no such root: {0}")] + NoSuchRoot(u64), + /// A smart collection whose selector references itself, directly or via /// another collection. #[error("collection {0} would form a cycle")] diff --git a/core/dr-catalog/src/lib.rs b/core/dr-catalog/src/lib.rs index db77b74..23ca044 100644 --- a/core/dr-catalog/src/lib.rs +++ b/core/dr-catalog/src/lib.rs @@ -11,6 +11,7 @@ //! //! - [`schema`] — tables and forward-only migrations //! - [`scan`] — incremental discovery that prunes unchanged directories +//! - [`walk`] — those decisions driven against real storage, local or SAF //! - [`query`] — selectors compiled to indexed SQL, windowed for the grid //! - [`collections`] — the collection tree and membership the UI edits //! - [`jobs`] — the durable background work queue @@ -41,6 +42,7 @@ pub mod scan; pub mod schema; pub mod sync; pub mod trash; +pub mod walk; pub use cache::{Budget, Cache, DEFAULT_BUDGET_BYTES}; pub use collections::{Collection, CollectionKind, TreeRow}; @@ -51,6 +53,7 @@ pub use query::{Query, Sort}; pub use rating::{Judgement, MAX_RATING}; pub use scan::{DirAction, DirState, EntryAction, ScanOutcome}; pub use trash::{TrashedImage, TRASH_DIR}; +pub use walk::{ensure_root, scan_root, RootKind, ScanProgress, ScanReport}; /// One row of the library grid. /// diff --git a/core/dr-catalog/src/query.rs b/core/dr-catalog/src/query.rs index 574489a..25b2bac 100644 --- a/core/dr-catalog/src/query.rs +++ b/core/dr-catalog/src/query.rs @@ -318,7 +318,10 @@ fn flag_code(f: FlagState) -> i64 { } } -fn availability_code(a: Availability) -> i64 { +/// The stored form of an availability. Shared with [`crate::walk`], which +/// writes the column this reads — two spellings of the same mapping would +/// filter for a state nothing ever writes. +pub(crate) fn availability_code(a: Availability) -> i64 { match a { Availability::MetadataOnly => 0, Availability::Preview => 1, diff --git a/core/dr-catalog/src/scan.rs b/core/dr-catalog/src/scan.rs index e1414ce..baf1cd1 100644 --- a/core/dr-catalog/src/scan.rs +++ b/core/dr-catalog/src/scan.rs @@ -14,35 +14,13 @@ //! //! This module holds the decision logic and the deletion-sweep rules; walking //! an actual directory belongs to the platform layer, which supplies -//! [`DirState`] and [`DirEntry`]. +//! [`DirState`] and [`DirEntry`]. [`crate::walk`] is what puts the two +//! together. + +pub use dr_types::{DirEntry, DirState}; use dr_types::FormatFilter; -/// What a directory looked like when last scanned, and what it looks like now. -/// -/// Both fields are cheap to obtain: one `stat` locally, one -/// `DocumentsContract` metadata query on SAF. -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub struct DirState { - pub mtime: i64, - /// Direct children, files and directories alike. - /// - /// mtime alone misses a delete-and-create inside one timestamp tick, and - /// coarse-granularity providers widen that window. The count does not - /// close the hole — a paired add and remove moves neither — but a bare add - /// or remove moves the count, and those are far commoner. - pub entry_count: u32, -} - -/// One entry from a directory listing. -#[derive(Debug, Clone, PartialEq, Eq)] -pub struct DirEntry { - pub name: String, - pub is_dir: bool, - pub size: u64, - pub mtime: i64, -} - /// What the scanner should do with a directory, before listing it. #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub enum DirAction { @@ -100,12 +78,17 @@ pub fn classify_entry( } /// Outcome of a scan, which decides whether pruning may run. -#[derive(Debug, Clone, Copy, PartialEq, Eq)] +/// +/// `Cancelled` is the default because a scan that has not run has proven +/// nothing absent, and every default in this area must fail towards keeping +/// photographs. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)] pub enum ScanOutcome { /// Every reachable folder was visited. Complete, /// The user cancelled. Partial state is valid — jobs are resumable — but /// unvisited folders must not be read as deleted. + #[default] Cancelled, /// The root itself could not be opened: drive unplugged, SAF grant /// revoked, share unmounted. diff --git a/core/dr-catalog/src/walk.rs b/core/dr-catalog/src/walk.rs new file mode 100644 index 0000000..c4172e7 --- /dev/null +++ b/core/dr-catalog/src/walk.rs @@ -0,0 +1,1367 @@ +//! TRACES: FR-CAT-1 | FR-CAT-1a | FR-CAT-9 | NFR-PORT-1 | NFR-P1 +//! Turning the decisions in [`scan`](crate::scan) into a catalogued library. +//! +//! [`scan`](crate::scan) knows what a changed directory *means* and holds no +//! I/O; [`dr_plat::Storage`] knows how to read a directory and holds no +//! catalog. This module is the only place the two meet, which is what keeps +//! both of them testable on their own — and what lets Android arrive as a +//! second `Storage` implementation with nothing here to change (ARCH §6.9). +//! +//! # What it costs on a library that has not changed +//! +//! One probe per directory and nothing else: no listing, no `stat` per file, no +//! row written. The saving is not incidental — 50k images in 2k folders is 2k +//! probes against 50k stats, and NFR-P1 is written against the former. +//! +//! # What pruning cannot see +//! +//! Writing to an existing file moves neither its directory's mtime nor its +//! entry count, so a folder full of rewritten photographs looks untouched and +//! is skipped. This is the price of pruning at the directory level, and it is +//! worth being plain about rather than discovering later. +//! +//! It bites less than it sounds. Almost nothing rewrites a RAW in place: an +//! export, a backup restore, `mv`, and every editor that saves safely write a +//! new file beside the old one and rename over it, which does move both the +//! mtime and — briefly — the count. Those are caught. What is missed is a +//! genuine in-place write, which for a photograph library is close to nothing; +//! and it is missed only until the folder is listed for some other reason. +//! +//! # The deletion sweep, which is the dangerous part +//! +//! Rows are deleted in two places, and both are guarded by the same principle: +//! **absence is only evidence of deletion where absence was actually +//! observed.** A folder that was listed proves its missing images are gone. A +//! folder that was pruned proves nothing about its contents, and a scan that +//! was cancelled or that failed part-way proves nothing about the folders it +//! never reached — hence [`ScanOutcome::may_prune`], and hence the folder sweep +//! running once at the end rather than as it goes. +//! +//! Get this wrong and an unplugged drive deletes the library. Every image would +//! be absent, every folder unreached, and the sweep would take all of them +//! along with their ratings and their edits. + +use std::collections::HashMap; + +use dr_plat::{DirRef, Node, Storage}; +use dr_types::{Availability, FormatFilter, RootId, SourceRef}; +use rusqlite::{Connection, OptionalExtension}; + +use crate::error::CatalogError; +use crate::jobs::{self, JobKind, Priority}; +use crate::query::availability_code; +use crate::scan::{ + classify_dir, classify_entry, DirAction, DirState, EntryAction, KnownFile, ScanOutcome, +}; +use crate::trash::TRASH_DIR; + +/// How deep to recurse before giving up. +/// +/// A symlink loop would otherwise walk forever — the filesystem implementation +/// follows symlinks deliberately, because a photographer who symlinks last +/// year's drive into the library means it. Real libraries are nowhere near this +/// deep. The same figure `dr_sync::scan` uses, for the same reason. +pub const MAX_DEPTH: usize = 32; + +/// TRACES: FR-CAT-3 +/// Directory holding derived state — thumbnail shards and catalog snapshots. +/// +/// Quoted rather than imported, exactly as [`TRASH_DIR`] is quoted in +/// [`crate::trash`]: `dr-catalog` does not depend on `dr-sync`, and adding that +/// dependency for one string would invert the layering. A test asserts the two +/// agree. +pub const DERIVED_DIR: &str = ".darkroom-derived"; + +/// Whether a directory is one the scan must stay out of. +/// +/// **The trash exclusion is half of the soft delete.** Trashed images are real +/// files in a real folder under the library root (FR-CAT-15); a scan that +/// walked it would re-index them as ordinary photographs and the delete would +/// come undone on the next refresh. +/// +/// Matched on the folder's own name at any depth, so a nested library moved in +/// wholesale carries its exclusions with it. +pub fn is_excluded(name: &str) -> bool { + matches!(name, TRASH_DIR | DERIVED_DIR) +} + +/// Which grant a root represents — the `roots.kind` column. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum RootKind { + /// A directory on a filesystem. + Local, + /// An Android persisted document tree. + Saf, + /// A folder on a remote server, reached through the sync layer. + Remote, +} + +impl RootKind { + pub fn as_str(self) -> &'static str { + match self { + RootKind::Local => "local", + RootKind::Saf => "saf", + RootKind::Remote => "remote", + } + } +} + +/// TRACES: FR-CAT-1 +/// Find or create the row for a granted library location. +/// +/// `label` is how the grant is spelled for this kind of root — the directory +/// path on Linux, the tree URI on Android, the remote folder on a server. It is +/// display text and identity, never something `core/` resolves: reaching the +/// files is [`Storage`]'s business and takes a [`RootId`], which is the whole +/// point of FR-CAT-1a. +/// +/// Idempotent, because it has to be: a second row for the same folder would +/// fragment the library across two roots, splitting the images and comparing +/// each half against the wrong scan generation. The `UNIQUE(kind, label)` in +/// the schema is what enforces it; this reuses the row rather than failing. +pub fn ensure_root(conn: &Connection, kind: RootKind, label: &str) -> Result { + conn.execute( + "INSERT INTO roots(kind, label) VALUES (?1, ?2) ON CONFLICT DO NOTHING", + rusqlite::params![kind.as_str(), label], + )?; + let id: i64 = conn.query_row( + "SELECT id FROM roots WHERE kind = ?1 AND label = ?2", + rusqlite::params![kind.as_str(), label], + |r| r.get(0), + )?; + Ok(RootId(id as u64)) +} + +/// Progress during a scan, so a large library reports rather than appears hung. +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] +pub struct ScanProgress { + pub directories_listed: usize, + /// Directories proven unchanged and skipped. The value of pruning, made + /// visible — on a healthy rescan this is nearly the whole library. + pub directories_pruned: usize, + pub images_found: usize, +} + +/// What a scan did. +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] +pub struct ScanReport { + pub progress: ScanProgress, + pub inserted: usize, + /// Files whose size or mtime moved: metadata re-read, thumbnail rebuilt, + /// content hash dropped. + pub updated: usize, + /// Files already catalogued and untouched. The overwhelming majority, and + /// each one costs nothing but a hash lookup. + pub unchanged: usize, + /// Rows removed because the file was *observed* to be gone. + pub images_removed: usize, + pub folders_removed: usize, + pub outcome: ScanOutcome, +} + +/// TRACES: FR-CAT-1 | FR-CAT-9 | NFR-P1 +/// Scan one granted root into the catalog. +/// +/// `cancel` is polled once per directory — FR-CAT-1 requires a scan the user +/// can stop, and a cancelled scan leaves valid partial state: everything +/// listed is committed, nothing is pruned. +/// +/// `now` is passed rather than read, so a test can assert on `added_at` and so +/// two rows written by one scan carry one timestamp. +/// +/// Returns a report rather than failing on an unreachable root. That is not +/// leniency: [`ScanOutcome`] is *how* the caller is told, it is what +/// [`ScanOutcome::may_prune`] consumes, and an `Err` here would discard the +/// counts of everything the scan did manage to do. A `CatalogError` means the +/// database itself is in trouble. +pub fn scan_root( + conn: &Connection, + storage: &dyn Storage, + root: RootId, + formats: &FormatFilter, + now: i64, + cancel: impl Fn() -> bool, + mut on_progress: impl FnMut(ScanProgress), +) -> Result { + let mut report = ScanReport::default(); + + // A fresh generation per *run*, taken and stored before anything is + // walked — not per completed scan. A cancelled run marks the folders it + // reached, and if the next run reused the number those marks would look + // current: a folder deleted in between would carry the generation the + // sweep is about to compare against, and would survive. + let generation = bump_generation(conn, root, now)?; + + let start = match storage.root_dir(root) { + Ok(d) => d, + // Not merely "no images": the grant is gone or the drive is out. Every + // image under it is unreachable, not deleted (FR-CAT-9). + Err(e) => { + log::warn!("scan: root {} unreachable: {e}", root.0); + mark_root_offline(conn, root)?; + report.outcome = ScanOutcome::RootUnreachable; + return Ok(report); + } + }; + + // Explicit stack rather than recursion, as in `dr_sync::scan`: a + // pathological tree must not overflow. Each item carries the catalog id of + // its parent folder, which is what threads the folder tree together. + let mut stack: Vec<(DirRef, usize, Option)> = vec![(start, 0, None)]; + let mut cancelled = false; + let mut partial = false; + + while let Some((dir, depth, parent)) = stack.pop() { + if cancel() { + log::info!("scan: cancelled at {dir}"); + cancelled = true; + break; + } + + // Excluded by name, before it is probed or listed. The root itself is + // exempt: a user who granted `.darkroom-trash` as their library meant + // it, and refusing to scan the folder they chose would look like the + // app doing nothing. + if !dir.is_root() && is_excluded(dir.name()) { + continue; + } + + if depth > MAX_DEPTH { + // Deeper folders exist and were not seen, so this scan cannot + // prove anything absent. + log::warn!("scan: depth limit at {dir}, not descending further"); + partial = true; + continue; + } + + let state = match storage.dir_state(&dir) { + Ok(s) => s, + Err(e) if dir.is_root() => { + log::warn!("scan: root {} unreachable: {e}", root.0); + mark_root_offline(conn, root)?; + report.outcome = ScanOutcome::RootUnreachable; + return Ok(report); + } + // One folder failed while the library as a whole is fine — a + // permission on a subdirectory, most likely. Its images stay in + // the catalog, and the sweep is disarmed for the whole run. + Err(e) => { + log::warn!("scan: {dir} could not be probed: {e}"); + partial = true; + continue; + } + }; + + // One transaction per directory. A single transaction for the whole + // scan would make a 50k-image first run all-or-nothing and unresumable; + // one per row would fsync 50k times. + let tx = conn.unchecked_transaction()?; + let (folder_id, stored) = touch_folder(&tx, root, &dir, parent, generation)?; + + match classify_dir(stored, state) { + // Unchanged. Its own contents need not be read, but a change in a + // *grandchild* moves no timestamp here — a filesystem propagates + // nothing upward — so the known children are still walked. + DirAction::RecurseOnly => { + report.progress.directories_pruned += 1; + for child in known_children(&tx, root, folder_id)? { + stack.push((child, depth + 1, Some(folder_id))); + } + tx.commit()?; + } + + DirAction::ListAndRecurse => { + let entries = match storage.list(&dir) { + Ok(e) => e, + Err(e) if dir.is_root() => { + drop(tx); + log::warn!("scan: root {} unreachable: {e}", root.0); + mark_root_offline(conn, root)?; + report.outcome = ScanOutcome::RootUnreachable; + return Ok(report); + } + Err(e) => { + // Rolled back rather than committed: the folder row's + // new generation would otherwise claim it was reached + // successfully. + drop(tx); + log::warn!("scan: {dir} could not be listed: {e}"); + partial = true; + continue; + } + }; + + report.progress.directories_listed += 1; + + // Emptied as the listing is walked, so whatever is left at the + // end is precisely what the folder no longer holds. Removing + // rather than marking keeps the sweep O(1) per file instead of + // a scan of the folder's rows per file. + let mut known = catalogued_images(&tx, folder_id)?; + + for entry in &entries { + match &entry.node { + Node::Dir(child) => stack.push((child.clone(), depth + 1, Some(folder_id))), + Node::File(src) => { + // Taken out of `known` whatever is decided next: + // this file was seen, and the sweep at the end is + // about the ones that were not. + let existing = known.remove(src.key()); + let action = + classify_entry(&entry.meta, existing.map(|e| e.as_known()), formats); + + // A format the filter no longer asks for, left + // exactly as it is — row and all. Unticking JPEG + // means stop looking for new ones, not delete the + // ones already found: the file is sitting right + // there, and absence is what justifies a delete. + if action == EntryAction::Ignored { + continue; + } + report.progress.images_found += 1; + + match action { + EntryAction::Unchanged => { + report.unchanged += 1; + // Unchanged in size and mtime, but perhaps + // not in reachability: a file that was + // marked offline while the drive was out is + // back, and nothing else would ever notice, + // because nothing about it has moved. + if let Some(row) = &existing { + restore_availability(&tx, row, src)?; + } + } + EntryAction::Insert => { + let id = insert_image( + &tx, + root, + folder_id, + src, + entry.meta.size, + entry.meta.mtime, + now, + )?; + queue_reading_it(&tx, id)?; + report.inserted += 1; + } + EntryAction::Changed => { + let id = update_image( + &tx, + root, + folder_id, + src, + entry.meta.size, + entry.meta.mtime, + )?; + queue_reading_it(&tx, id)?; + report.updated += 1; + } + EntryAction::Ignored => unreachable!("returned above"), + } + } + } + } + + // The file-level sweep, and the only place absence was + // observed: this folder was read, and these rows were not in + // it. Trashed images are exempt — their file has been moved + // into the trash on purpose, so being absent from the folder is + // the expected state, and deleting the row would lose the way + // back (FR-CAT-15). + for row in known.values() { + if row.trashed { + continue; + } + tx.execute("DELETE FROM images WHERE id = ?1", [row.id])?; + report.images_removed += 1; + } + + // Only now, with the listing read and reconciled, is the + // directory's state safe to record. Storing it before would let + // the next scan prune a folder whose contents were never + // actually seen, hiding every image beneath it permanently — + // the same trap `dr_sync::scan` documents for ETags. + tx.execute( + "UPDATE folders SET mtime = ?1, entry_count = ?2 WHERE id = ?3", + rusqlite::params![state.mtime, state.entry_count, folder_id], + )?; + tx.commit()?; + } + } + + on_progress(report.progress); + } + + report.outcome = if cancelled { + ScanOutcome::Cancelled + } else if partial { + ScanOutcome::PartialFailure + } else { + ScanOutcome::Complete + }; + + if report.outcome.may_prune() { + let (folders, images) = prune_unreached(conn, root, generation)?; + report.folders_removed = folders; + report.images_removed += images; + } + + Ok(report) +} + +/// Take the next scan generation and record it against the root. +fn bump_generation(conn: &Connection, root: RootId, now: i64) -> Result { + let current: Option = conn + .query_row( + "SELECT scan_generation FROM roots WHERE id = ?1", + [root.0 as i64], + |r| r.get(0), + ) + .optional()?; + let current = current.ok_or(CatalogError::NoSuchRoot(root.0))?; + + let next = current + 1; + conn.execute( + "UPDATE roots SET scan_generation = ?1, last_seen = ?2 WHERE id = ?3", + rusqlite::params![next, now, root.0 as i64], + )?; + Ok(next) +} + +/// TRACES: FR-CAT-9 +/// Mark every image under a root as unreachable. +/// +/// The other half of FR-CAT-9's distinction: a source *proven absent* may leave +/// the catalog, a source merely *unreachable* is marked offline and keeps its +/// ratings and its edits. The grid then says "offline" rather than showing a +/// library that has silently lost half its photographs. +/// +/// The folders forget what they last looked like, which costs a full re-listing +/// when the drive comes back. That is the price of the marking: every row under +/// the root now claims something about the files that is no longer known to be +/// true, and only reading the directories again can settle it. Pruning would +/// skip them all and leave a plugged-in library showing as offline forever. +fn mark_root_offline(conn: &Connection, root: RootId) -> Result<(), CatalogError> { + let root_id = root.0 as i64; + conn.execute( + "UPDATE images SET availability = ?1 WHERE root_id = ?2 AND availability != ?1", + rusqlite::params![availability_code(Availability::Offline), root_id], + )?; + conn.execute( + "UPDATE folders SET mtime = NULL, entry_count = NULL WHERE root_id = ?1", + [root_id], + )?; + Ok(()) +} + +/// Record that a folder was reached in this generation, and report what the +/// last scan saw there. +/// +/// The generation is written on arrival, before the folder is read: it means +/// "reached", which is what the end-of-scan sweep asks about. The `mtime` and +/// `entry_count` that mean "read" are written separately, and only afterwards. +fn touch_folder( + conn: &Connection, + root: RootId, + dir: &DirRef, + parent: Option, + generation: i64, +) -> Result<(i64, Option), CatalogError> { + conn.execute( + "INSERT INTO folders(root_id, parent_id, path, scanned_generation) + VALUES (?1, ?2, ?3, ?4) + ON CONFLICT(root_id, path) DO UPDATE SET + scanned_generation = excluded.scanned_generation, + parent_id = excluded.parent_id", + rusqlite::params![root.0 as i64, parent, dir.key(), generation], + )?; + + let (id, mtime, count): (i64, Option, Option) = conn.query_row( + "SELECT id, mtime, entry_count FROM folders WHERE root_id = ?1 AND path = ?2", + rusqlite::params![root.0 as i64, dir.key()], + |r| Ok((r.get(0)?, r.get(1)?, r.get(2)?)), + )?; + + // Both or neither: a folder recorded by some other path (a remote scan + // storing an ETag, say) has no local state, and half of one must not be + // read as "unchanged since 1970". + let stored = match (mtime, count) { + (Some(mtime), Some(count)) => Some(DirState { + mtime, + entry_count: count as u32, + }), + _ => None, + }; + Ok((id, stored)) +} + +/// The subfolders the catalog already knows about, as references to walk. +/// +/// This is what makes pruning safe on a filesystem. A directory's mtime moves +/// when its own entries change and not when a grandchild's do, so an unchanged +/// folder says nothing about the tree below it; the stored keys are how the +/// walk carries on downward without listing. +fn known_children( + conn: &Connection, + root: RootId, + folder_id: i64, +) -> Result, CatalogError> { + let mut stmt = conn.prepare("SELECT path FROM folders WHERE parent_id = ?1")?; + let rows = stmt.query_map([folder_id], |r| r.get::<_, String>(0))?; + Ok(rows + .flatten() + .map(|path| DirRef::from_parts(root, path)) + .collect()) +} + +/// What the catalog holds for one image, as the scan needs it. +#[derive(Clone, Copy)] +struct Catalogued { + id: i64, + size: Option, + mtime: Option, + availability: i64, + trashed: bool, +} + +impl Catalogued { + /// What [`classify_entry`] compares against. + /// + /// A row with no recorded size or mtime — inserted by a remote scan, or by + /// an older build — is given values no real file can match, so it is + /// classified `Changed` and re-read. The alternative, treating it as + /// unknown, would insert a duplicate. + fn as_known(&self) -> KnownFile { + KnownFile { + size: self.size.unwrap_or(u64::MAX), + mtime: self.mtime.unwrap_or(i64::MIN), + } + } +} + +/// Every image the catalog has in one folder, keyed by its stored reference. +/// +/// Read once per folder rather than queried per file: a folder of 2,000 frames +/// is one statement and one hash lookup each, not 2,000 indexed selects. +fn catalogued_images( + conn: &Connection, + folder_id: i64, +) -> Result, CatalogError> { + let mut stmt = conn.prepare( + "SELECT id, source_ref, file_size, file_mtime, availability, trashed_at + FROM images WHERE folder_id = ?1", + )?; + let rows = stmt.query_map([folder_id], |r| { + Ok(( + r.get::<_, String>(1)?, + Catalogued { + id: r.get(0)?, + size: r.get::<_, Option>(2)?.map(|v| v as u64), + mtime: r.get(3)?, + availability: r.get(4)?, + trashed: r.get::<_, Option>(5)?.is_some(), + }, + )) + })?; + Ok(rows.flatten().collect()) +} + +/// Put an unchanged file's availability back where the evidence says it should +/// be, and write nothing if it is already there. +/// +/// The write has to be conditional: an unchanged file is the overwhelmingly +/// common case, and an `UPDATE` per image per scan would make a no-op rescan +/// write the whole library. +fn restore_availability( + conn: &Connection, + row: &Catalogued, + src: &SourceRef, +) -> Result<(), CatalogError> { + let should_be = availability_code(availability_of(src)); + if row.availability == should_be { + return Ok(()); + } + conn.execute( + "UPDATE images SET availability = ?1 WHERE id = ?2", + rusqlite::params![should_be, row.id], + )?; + Ok(()) +} + +/// How much of this image is on hand. +/// +/// A local file is the original — unless it is a Nextcloud VFS placeholder, a +/// one-byte stub standing in for a dehydrated file whose bytes are on the +/// server (ARCH §9.0). Reading such a stub does not trigger a fetch on Linux, +/// so calling it `Original` would promise pixels that are not there. Catalogued +/// as the image it stands for, marked offline: honest, and FR-NC-6c's whole +/// point. +fn availability_of(src: &SourceRef) -> Availability { + if src.is_placeholder() { + Availability::Offline + } else { + Availability::Original + } +} + +fn insert_image( + conn: &Connection, + root: RootId, + folder_id: i64, + src: &SourceRef, + size: u64, + mtime: i64, + now: i64, +) -> Result { + // `metadata_state = 1`: the scan knows the name, the size and the mtime, + // and has read no EXIF. Claiming otherwise would make a date filter + // silently wrong on a freshly scanned library. + conn.execute( + "INSERT INTO images(root_id, folder_id, source_ref, format, file_size, file_mtime, + availability, metadata_state, added_at) + VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, 1, ?8) + ON CONFLICT(root_id, source_ref) DO UPDATE SET + folder_id = excluded.folder_id, + file_size = excluded.file_size, + file_mtime = excluded.file_mtime, + availability = excluded.availability", + rusqlite::params![ + root.0 as i64, + folder_id, + src.key(), + src.extension(), + size as i64, + mtime, + availability_code(availability_of(src)), + now, + ], + )?; + image_id(conn, root, src) +} + +fn update_image( + conn: &Connection, + root: RootId, + folder_id: i64, + src: &SourceRef, + size: u64, + mtime: i64, +) -> Result { + // The content hash is dropped, not recomputed: it described bytes that no + // longer exist, and leaving it would let reconnect-by-hash match this image + // to a file it is no longer a copy of. `metadata_state` goes back to 1 for + // the same reason — the EXIF on record was read from the old file. + conn.execute( + "UPDATE images SET folder_id = ?1, file_size = ?2, file_mtime = ?3, + content_hash = NULL, metadata_state = 1, availability = ?4 + WHERE root_id = ?5 AND source_ref = ?6", + rusqlite::params![ + folder_id, + size as i64, + mtime, + availability_code(availability_of(src)), + root.0 as i64, + src.key(), + ], + )?; + image_id(conn, root, src) +} + +fn image_id(conn: &Connection, root: RootId, src: &SourceRef) -> Result { + Ok(conn.query_row( + "SELECT id FROM images WHERE root_id = ?1 AND source_ref = ?2", + rusqlite::params![root.0 as i64, src.key()], + |r| r.get(0), + )?) +} + +/// Queue the work that turns a stat-only row into a usable grid cell. +/// +/// Enqueued inside the scan's transaction, so a folder's rows and the jobs that +/// finish them land together — a crash between the two would otherwise leave +/// images no worker was ever told about. +fn queue_reading_it(conn: &Connection, image_id: i64) -> Result<(), CatalogError> { + jobs::enqueue( + conn, + JobKind::ExtractMetadata, + Some(image_id), + Priority::Background, + None, + )?; + jobs::enqueue( + conn, + JobKind::Thumbnail, + Some(image_id), + Priority::Background, + None, + ) +} + +/// TRACES: FR-CAT-9 +/// Delete the folders this scan did not reach, and the images inside them. +/// +/// Runs only after a [`ScanOutcome::Complete`] scan, which is the guard that +/// stops an unplugged drive from taking the library with it. +/// +/// Trashed images are detached first. Their folder may well be one of the ones +/// vanishing — the user deleted the whole directory — and the cascade would +/// take the row with it, orphaning a file sitting in the trash with no way back +/// (FR-CAT-15). +fn prune_unreached( + conn: &Connection, + root: RootId, + generation: i64, +) -> Result<(usize, usize), CatalogError> { + const STALE: &str = "SELECT id FROM folders WHERE root_id = ?1 AND scanned_generation < ?2"; + + let tx = conn.unchecked_transaction()?; + let root_id = root.0 as i64; + let params = rusqlite::params![root_id, generation]; + + tx.execute( + &format!( + "UPDATE images SET folder_id = NULL + WHERE trashed_at IS NOT NULL AND folder_id IN ({STALE})" + ), + params, + )?; + + let images: i64 = tx.query_row( + &format!("SELECT count(*) FROM images WHERE folder_id IN ({STALE})"), + params, + |r| r.get(0), + )?; + let folders = tx.execute( + "DELETE FROM folders WHERE root_id = ?1 AND scanned_generation < ?2", + params, + )?; + tx.commit()?; + + Ok((folders, images as usize)) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::Catalog; + use dr_plat::LocalStorage; + use dr_types::Format; + use std::cell::Cell; + use std::fs; + use std::path::PathBuf; + + /// A throwaway library on disk, removed when the test ends. + /// + /// The real [`LocalStorage`] rather than a fake, because what most needs + /// protecting here is the seam between the two crates: a fake would agree + /// with whatever this module expects, which is exactly the mistake a scan + /// against a real directory cannot make. + struct Library { + dir: PathBuf, + catalog: Catalog, + root: RootId, + } + + impl Library { + fn new(name: &str) -> Self { + let dir = std::env::temp_dir().join(format!( + "dr-walk-{name}-{}-{:?}", + std::process::id(), + std::thread::current().id() + )); + let _ = fs::remove_dir_all(&dir); + fs::create_dir_all(&dir).expect("temp dir"); + + let catalog = Catalog::in_memory().expect("catalog"); + let root = ensure_root( + catalog.connection(), + RootKind::Local, + &dir.display().to_string(), + ) + .expect("root"); + + Library { dir, catalog, root } + } + + fn file(&self, rel: &str, bytes: &[u8]) -> &Self { + let p = self.dir.join(rel); + if let Some(parent) = p.parent() { + fs::create_dir_all(parent).expect("mkdir"); + } + fs::write(p, bytes).expect("write"); + self + } + + /// Overwrite a file the way a program that cares does: write beside it, + /// then rename over the top. + /// + /// Not incidental to the test — it is *why* the change is visible. A + /// plain write moves nothing about the directory, and the scan would + /// never look (see the module docs). + fn resave(&self, rel: &str, bytes: &[u8]) -> &Self { + let target = self.dir.join(rel); + let tmp = target.with_extension("tmp"); + fs::write(&tmp, bytes).expect("write"); + fs::rename(&tmp, &target).expect("rename"); + self + } + + fn remove(&self, rel: &str) -> &Self { + let p = self.dir.join(rel); + if p.is_dir() { + fs::remove_dir_all(p).expect("rmdir"); + } else { + fs::remove_file(p).expect("rm"); + } + self + } + + fn storage(&self) -> LocalStorage { + LocalStorage::with_root(self.root, self.dir.clone()) + } + + fn conn(&self) -> &Connection { + self.catalog.connection() + } + + fn scan(&self) -> ScanReport { + self.scan_with(&FormatFilter::all(), &self.storage(), || false) + } + + fn scan_with( + &self, + formats: &FormatFilter, + storage: &dyn Storage, + cancel: impl Fn() -> bool, + ) -> ScanReport { + scan_root( + self.conn(), + storage, + self.root, + formats, + 1_000, + cancel, + |_| {}, + ) + .expect("scan") + } + + fn names(&self) -> Vec { + let mut stmt = self + .conn() + .prepare("SELECT source_ref FROM images ORDER BY source_ref") + .unwrap(); + let rows = stmt.query_map([], |r| r.get::<_, String>(0)).unwrap(); + rows.flatten().collect() + } + + fn count(&self, sql: &str) -> i64 { + self.conn().query_row(sql, [], |r| r.get(0)).unwrap() + } + } + + impl Drop for Library { + fn drop(&mut self) { + let _ = fs::remove_dir_all(&self.dir); + } + } + + #[test] + fn a_folder_of_photographs_becomes_a_catalogued_library() { + // The whole point of the exercise: before this existed, no image + // reached the catalog without a Nextcloud account. + let lib = Library::new("first-scan"); + lib.file("2026/08/IMG_0001.CR3", b"raw") + .file("2026/08/IMG_0002.CR3", b"raw") + .file("2025/IMG_0003.NEF", b"raw") + .file("2025/notes.txt", b"not a photograph"); + + let r = lib.scan(); + + assert_eq!(r.outcome, ScanOutcome::Complete); + assert_eq!(r.inserted, 3); + assert_eq!( + lib.names(), + vec![ + "2025/IMG_0003.NEF", + "2026/08/IMG_0001.CR3", + "2026/08/IMG_0002.CR3" + ] + ); + } + + #[test] + fn a_scanned_image_is_addressed_by_reference_not_by_path() { + // FR-CAT-1a. The stored reference is relative to the root, so the + // library can be moved, or reached through SAF where no path exists at + // all, without rewriting a single row. + let lib = Library::new("relative"); + lib.file("2026/IMG.CR3", b"raw"); + lib.scan(); + + let stored = &lib.names()[0]; + assert_eq!(stored, "2026/IMG.CR3"); + assert!( + !stored.contains(&lib.dir.display().to_string()), + "an absolute path leaked into the catalog: {stored}" + ); + } + + #[test] + fn a_rescan_of_an_unchanged_library_reads_no_directory() { + // NFR-P1 in one assertion. If this fails, every rescan is a full walk + // and a 50k-image library takes minutes instead of seconds. + let lib = Library::new("prune"); + lib.file("2026/08/IMG_0001.CR3", b"raw") + .file("2025/IMG_0003.NEF", b"raw"); + + let first = lib.scan(); + assert_eq!(first.progress.directories_listed, 4); // root, 2026, 2026/08, 2025 + + let second = lib.scan(); + assert_eq!(second.progress.directories_listed, 0); + assert_eq!(second.progress.directories_pruned, 4); + assert_eq!(second.inserted, 0); + assert_eq!(second.images_removed, 0); + assert_eq!(second.outcome, ScanOutcome::Complete); + } + + #[test] + fn a_change_under_an_unchanged_parent_is_still_found() { + // A filesystem propagates nothing upward: adding a file to `2026/08` + // leaves `2026` and the root untouched. A scan that pruned at the first + // unchanged folder would never see the new photograph — which is why + // the walk carries on into known children rather than stopping. + let lib = Library::new("grandchild"); + lib.file("2026/08/IMG_0001.CR3", b"raw"); + lib.scan(); + + lib.file("2026/08/IMG_0002.CR3", b"raw"); + let r = lib.scan(); + + assert_eq!(r.inserted, 1); + assert_eq!(r.progress.directories_pruned, 2, "root and 2026"); + assert_eq!(r.progress.directories_listed, 1, "2026/08"); + } + + #[test] + fn a_deleted_file_leaves_the_catalog() { + let lib = Library::new("deleted"); + lib.file("a.CR3", b"raw").file("b.CR3", b"raw"); + lib.scan(); + + lib.remove("b.CR3"); + let r = lib.scan(); + + assert_eq!(r.images_removed, 1); + assert_eq!(lib.names(), vec!["a.CR3"]); + } + + #[test] + fn a_deleted_folder_takes_its_images_with_it() { + let lib = Library::new("deleted-folder"); + lib.file("2025/a.CR3", b"raw").file("2026/b.CR3", b"raw"); + lib.scan(); + + lib.remove("2025"); + let r = lib.scan(); + + assert_eq!(r.folders_removed, 1); + assert_eq!(lib.names(), vec!["2026/b.CR3"]); + } + + #[test] + fn an_unreachable_root_marks_images_offline_and_deletes_nothing() { + // The most expensive mistake this code could make. Every folder looks + // unreached when a drive is unplugged, so a sweep would take the entire + // library — ratings, flags and edits included (FR-CAT-9). + let lib = Library::new("unplugged"); + lib.file("2026/IMG.CR3", b"raw"); + lib.scan(); + + let unplugged = LocalStorage::with_root(lib.root, lib.dir.join("gone")); + let r = lib.scan_with(&FormatFilter::all(), &unplugged, || false); + + assert_eq!(r.outcome, ScanOutcome::RootUnreachable); + assert!(!r.outcome.may_prune()); + assert_eq!(lib.names().len(), 1, "the library was deleted"); + assert_eq!( + lib.count("SELECT availability FROM images"), + availability_code(Availability::Offline) + ); + } + + #[test] + fn a_root_that_comes_back_is_available_again() { + // The other half: a drive plugged back in must return the library to + // usable, not leave it permanently marked offline. + let lib = Library::new("replugged"); + lib.file("IMG.CR3", b"raw"); + lib.scan(); + + let unplugged = LocalStorage::with_root(lib.root, lib.dir.join("gone")); + lib.scan_with(&FormatFilter::all(), &unplugged, || false); + lib.scan(); + + assert_eq!( + lib.count("SELECT availability FROM images"), + availability_code(Availability::Original) + ); + } + + #[test] + fn a_cancelled_scan_deletes_nothing() { + // Partial state is valid — jobs are resumable — but the folders that + // were never visited must not be read as absent. + let lib = Library::new("cancelled"); + lib.file("2025/a.CR3", b"raw") + .file("2026/b.CR3", b"raw") + .file("2027/c.CR3", b"raw"); + lib.scan(); + + // Stop after the first directory. + let seen = Cell::new(0); + let r = lib.scan_with(&FormatFilter::all(), &lib.storage(), || { + let n = seen.get(); + seen.set(n + 1); + n > 0 + }); + + assert_eq!(r.outcome, ScanOutcome::Cancelled); + assert_eq!(r.images_removed, 0); + assert_eq!(r.folders_removed, 0); + assert_eq!(lib.names().len(), 3); + } + + #[test] + fn a_resaved_file_is_queued_for_rereading_and_loses_its_stale_hash() { + let lib = Library::new("resaved"); + lib.file("IMG.CR3", b"raw"); + lib.scan(); + + lib.conn() + .execute( + "UPDATE images SET content_hash = 'abc', metadata_state = 2", + [], + ) + .unwrap(); + + // Same name, different bytes: an export written over the original, or + // a file restored from a backup. Both arrive by rename, which is what + // makes them visible to an incremental scan. + lib.resave("IMG.CR3", b"different bytes entirely"); + let r = lib.scan(); + + assert_eq!(r.updated, 1); + assert_eq!( + lib.count("SELECT count(*) FROM images WHERE content_hash IS NULL"), + 1, + "a hash of bytes that no longer exist would match this image to the \ + wrong file on reconnect" + ); + assert_eq!(lib.count("SELECT metadata_state FROM images"), 1); + assert_eq!( + lib.count("SELECT count(*) FROM jobs WHERE kind = 1"), + 1, + "EXIF must be re-read" + ); + } + + #[test] + fn a_file_that_has_not_moved_is_not_requeued_when_its_folder_is_reread() { + // Adding one photograph to a folder of two thousand must cost one + // thumbnail, not two thousand. Without this, every import rebuilds + // everything around it. + let lib = Library::new("no-requeue"); + lib.file("a.CR3", b"raw"); + lib.scan(); + lib.conn().execute("DELETE FROM jobs", []).unwrap(); + + lib.file("b.CR3", b"raw"); + let r = lib.scan(); + + assert_eq!(r.inserted, 1); + assert_eq!(r.unchanged, 1); + assert_eq!( + lib.count("SELECT count(*) FROM jobs"), + 2, + "EXIF and a thumbnail for the new image, and nothing for the old one" + ); + } + + #[test] + fn a_file_rewritten_in_place_is_not_noticed_until_its_folder_changes() { + // The documented limit of directory-level pruning, held by a test so it + // stays a known trade and does not become a surprise. An in-place write + // moves neither the directory's mtime nor its entry count, so nothing + // says to look — and a scan that looked anyway would be a full walk of + // the library every time. + let lib = Library::new("in-place"); + lib.file("IMG.CR3", b"raw"); + lib.scan(); + + fs::write(lib.dir.join("IMG.CR3"), b"rewritten in place, same folder").unwrap(); + assert_eq!(lib.scan().updated, 0); + + // Anything that touches the folder brings it back into view. + lib.file("other.CR3", b"raw"); + assert_eq!(lib.scan().updated, 1); + } + + #[test] + fn a_new_image_is_queued_for_a_thumbnail_and_for_exif() { + let lib = Library::new("queued"); + lib.file("IMG.CR3", b"raw"); + lib.scan(); + + assert_eq!( + lib.count("SELECT count(*) FROM jobs WHERE kind = 2"), + 1, + "no thumbnail job means an empty grid cell forever" + ); + assert_eq!(lib.count("SELECT count(*) FROM jobs WHERE kind = 1"), 1); + assert_eq!( + lib.count("SELECT metadata_state FROM images"), + 1, + "the scan read no EXIF and must not claim to have" + ); + } + + #[test] + fn the_format_filter_decides_what_is_catalogued() { + let lib = Library::new("formats"); + lib.file("IMG.CR3", b"raw") + .file("IMG.NEF", b"raw") + .file("IMG.JPG", b"jpeg"); + + let r = lib.scan_with( + &FormatFilter::from_formats([Format::Cr3]), + &lib.storage(), + || false, + ); + assert_eq!(r.inserted, 1); + assert_eq!(lib.names(), vec!["IMG.CR3"]); + } + + #[test] + fn narrowing_the_filter_stops_finding_files_it_does_not_delete_them() { + // Unticking JPEG is a statement about what to look for, not an + // instruction to discard work. The files are sitting in the folder, + // and absence is the only thing that justifies deleting a row — so a + // tick-box must not quietly take a hundred rated photographs with it. + let lib = Library::new("filter-narrowed"); + lib.file("IMG.CR3", b"raw").file("IMG.JPG", b"jpeg"); + lib.scan(); + assert_eq!(lib.names().len(), 2); + + // Something has to change for the folder to be listed at all. + lib.file("NEW.CR3", b"raw"); + lib.scan_with(&FormatFilter::raw_only(), &lib.storage(), || false); + + assert!( + lib.names().contains(&"IMG.JPG".to_string()), + "a file that is still there was deleted by a filter change" + ); + } + + #[test] + fn a_dehydrated_placeholder_is_catalogued_as_the_image_it_stands_for() { + // A Nextcloud-synced folder scanned as a local library is the common + // case, and 121,785 of these exist in a real one (ARCH §9.0). Each is a + // one-byte stub; reading it fetches nothing on Linux. Catalogued as a + // CR2 and marked offline, so the grid says so rather than showing a + // decode failure. + let lib = Library::new("placeholder"); + lib.file("_MG_4130.CR2.nextcloud", b"\0") + .file("_MG_4131.CR2", b"real bytes"); + lib.scan(); + + assert_eq!( + lib.count("SELECT availability FROM images WHERE source_ref LIKE '%.nextcloud'"), + availability_code(Availability::Offline) + ); + assert_eq!( + lib.count("SELECT count(*) FROM images WHERE format = 'cr2'"), + 2, + "the stub is a CR2, not an unknown '.nextcloud' type" + ); + assert_eq!( + lib.count("SELECT availability FROM images WHERE source_ref = '_MG_4131.CR2'"), + availability_code(Availability::Original) + ); + } + + #[test] + fn the_trash_folder_is_never_scanned() { + // The other half of the soft delete. A scan that walked the trash would + // re-index every trashed photograph as an ordinary one, undoing the + // delete on the next refresh (FR-CAT-15). + let lib = Library::new("trash"); + lib.file("a.CR3", b"raw") + .file(&format!("{TRASH_DIR}/1-b.CR3"), b"raw") + .file(&format!("{DERIVED_DIR}/thumbs.sqlite"), b"x"); + lib.scan(); + + assert_eq!(lib.names(), vec!["a.CR3"]); + assert_eq!( + lib.count(&format!( + "SELECT count(*) FROM folders WHERE path LIKE '{TRASH_DIR}%'" + )), + 0, + "the trash must cost nothing, not merely be filtered out" + ); + } + + #[test] + fn a_trashed_image_survives_a_rescan_of_the_folder_it_left() { + // Its file has been moved into the trash on purpose, so being absent + // from its old folder is the expected state. Deleting the row would + // lose `trashed_from` and with it the only way back (FR-CAT-15). + let lib = Library::new("trashed-row"); + lib.file("2026/a.CR3", b"raw").file("2026/b.CR3", b"raw"); + lib.scan(); + + // Trash `b` the way the UI does: move the file, then record. + fs::create_dir_all(lib.dir.join(TRASH_DIR)).unwrap(); + fs::rename( + lib.dir.join("2026/b.CR3"), + lib.dir.join(format!("{TRASH_DIR}/2-b.CR3")), + ) + .unwrap(); + lib.conn() + .execute( + "UPDATE images SET trashed_at = 100, trashed_from = source_ref, + source_ref = ?1 + WHERE source_ref = '2026/b.CR3'", + [format!("{TRASH_DIR}/2-b.CR3")], + ) + .unwrap(); + + lib.scan(); + + assert_eq!( + lib.count("SELECT count(*) FROM images WHERE trashed_at IS NOT NULL"), + 1, + "the trashed image was swept away by a rescan" + ); + } + + #[test] + fn a_trashed_image_outlives_the_folder_it_came_from() { + // Same rule, reached through the folder sweep instead: the user deleted + // the whole directory, and the cascade would have taken a row whose + // file is sitting safely in the trash. + let lib = Library::new("trashed-orphan"); + lib.file("2026/a.CR3", b"raw"); + lib.scan(); + + fs::create_dir_all(lib.dir.join(TRASH_DIR)).unwrap(); + fs::rename( + lib.dir.join("2026/a.CR3"), + lib.dir.join(format!("{TRASH_DIR}/1-a.CR3")), + ) + .unwrap(); + lib.conn() + .execute( + "UPDATE images SET trashed_at = 100, trashed_from = source_ref, + source_ref = ?1 + WHERE source_ref = '2026/a.CR3'", + [format!("{TRASH_DIR}/1-a.CR3")], + ) + .unwrap(); + lib.remove("2026"); + + lib.scan(); + + assert_eq!( + lib.count("SELECT count(*) FROM images WHERE trashed_at IS NOT NULL"), + 1 + ); + } + + #[test] + fn scanning_twice_does_not_fragment_the_library_across_two_roots() { + // `ensure_root` is idempotent because a second row would split the + // images between two roots and compare each half against the wrong scan + // generation. + let lib = Library::new("one-root"); + let again = + ensure_root(lib.conn(), RootKind::Local, &lib.dir.display().to_string()).unwrap(); + + assert_eq!(again, lib.root); + assert_eq!(lib.count("SELECT count(*) FROM roots"), 1); + } + + #[test] + fn every_run_takes_a_fresh_generation() { + // Not "once per completed scan": a cancelled run marks the folders it + // reached, and reusing the number would let those marks look current to + // the next sweep — so a folder deleted in between would survive it. + let lib = Library::new("generations"); + lib.file("a.CR3", b"raw"); + + lib.scan_with(&FormatFilter::all(), &lib.storage(), || true); + let after_cancel = lib.count("SELECT scan_generation FROM roots"); + lib.scan(); + assert!(lib.count("SELECT scan_generation FROM roots") > after_cancel); + } + + #[test] + fn a_scan_reports_progress_as_it_goes() { + // A first scan of a large library is minutes long; without this the UI + // has nothing to say for all of it. + let lib = Library::new("progress"); + lib.file("2025/a.CR3", b"raw").file("2026/b.CR3", b"raw"); + + let mut updates = Vec::new(); + scan_root( + lib.conn(), + &lib.storage(), + lib.root, + &FormatFilter::all(), + 1_000, + || false, + |p| updates.push(p), + ) + .unwrap(); + + assert_eq!(updates.len(), 3, "one per directory visited"); + assert_eq!(updates.last().unwrap().images_found, 2); + } + + #[test] + fn scanning_a_root_with_no_catalog_row_is_a_typed_error() { + // Rather than inventing a root: the label is the grant, and only the + // caller that obtained the grant knows how to spell it. + let lib = Library::new("no-root"); + let err = scan_root( + lib.conn(), + &lib.storage(), + RootId(999), + &FormatFilter::all(), + 0, + || false, + |_| {}, + ); + assert!(matches!(err, Err(CatalogError::NoSuchRoot(999)))); + } + + #[test] + fn the_excluded_directories_match_the_ones_the_remote_scanner_excludes() { + // Two pairs of constants in two crates that must agree, or a soft + // delete comes undone on whichever side disagrees. + assert!(is_excluded(TRASH_DIR)); + assert!(is_excluded(DERIVED_DIR)); + assert_eq!(DERIVED_DIR, dr_sync_derived_dir()); + assert!(!is_excluded("2026")); + assert!(!is_excluded(".darkroom-trash-old")); + } + + /// The scanner's constant, quoted rather than imported — `dr-catalog` does + /// not depend on `dr-sync`, and adding that dependency for one string would + /// invert the layering. + fn dr_sync_derived_dir() -> &'static str { + ".darkroom-derived" + } +} diff --git a/core/dr-types/src/lib.rs b/core/dr-types/src/lib.rs index 3fbeea9..328c43e 100644 --- a/core/dr-types/src/lib.rs +++ b/core/dr-types/src/lib.rs @@ -63,16 +63,25 @@ pub enum SourceRef { impl SourceRef { /// A stable, display-friendly name — the last path component. pub fn display_name(&self) -> &str { - let full = match self { - SourceRef::Local { relative, .. } => relative.as_str(), - SourceRef::Document { document_id, .. } => document_id.as_str(), - SourceRef::Remote { path, .. } => path.as_str(), - }; + let full = self.key(); // SAF document ids use ':' as a separator; paths use '/'. Split on // whichever appears last so both yield a sensible name. full.rsplit(['/', ':']).next().unwrap_or(full) } + /// The stored form — what the catalog holds in `images.source_ref`. + /// + /// Which of the three variants a key belongs to is not recorded beside it, + /// because the root already says: `roots.kind` is `'local' | 'saf' | + /// 'remote'`, and every image names its root. + pub fn key(&self) -> &str { + match self { + SourceRef::Local { relative, .. } => relative.as_str(), + SourceRef::Document { document_id, .. } => document_id.as_str(), + SourceRef::Remote { path, .. } => path.as_str(), + } + } + /// Lowercase file extension, if any. /// /// Looks through a Nextcloud VFS placeholder suffix, so a dehydrated @@ -251,6 +260,55 @@ impl FormatFilter { } } +/// TRACES: FR-CAT-1 | NFR-PORT-1 +/// What a directory looked like when last scanned, and what it looks like now. +/// +/// Both fields are cheap to obtain: one `stat` plus a name-only directory read +/// locally, one `DocumentsContract` query on SAF. Neither costs a `stat` per +/// child, which is the whole point — see `dr_catalog::scan`. +/// +/// Lives here rather than in either crate that uses it because it is the +/// sentence the platform layer says to the catalog: `dr-plat` produces it by +/// probing, `dr-catalog` stores it and compares. A copy on each side would be +/// two types that must agree by convention. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct DirState { + /// Milliseconds since the epoch. **Not seconds**, and the difference is not + /// pedantry: change detection asks whether a timestamp *moved*, so the + /// granularity of the unit is the width of the window in which a change is + /// invisible. A second is long enough to copy a card and start a scan, and + /// anything happening inside one tick of the stored value looks exactly + /// like nothing happening — the directory is pruned and the photographs + /// never appear. + /// + /// Both platforms can supply it: a filesystem records nanoseconds, and + /// SAF's `COLUMN_LAST_MODIFIED` is already in milliseconds. + pub mtime: i64, + /// Direct children, files and directories alike. + /// + /// mtime alone misses a delete-and-create inside one timestamp tick, and + /// coarse-granularity providers widen that window. The count does not + /// close the hole — a paired add and remove moves neither — but a bare add + /// or remove moves the count, and those are far commoner. + pub entry_count: u32, +} + +/// TRACES: FR-CAT-1 | NFR-PORT-1 +/// One entry from a directory listing, as the scanner classifies it. +/// +/// Deliberately carries no handle to the entry itself: this is what the +/// *decision* logic reads, and it must be constructible in a test without a +/// filesystem. The platform pairs it with the reference needed to reach the +/// entry (`dr_plat::Entry`). +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct DirEntry { + pub name: String, + pub is_dir: bool, + pub size: u64, + /// Milliseconds since the epoch, as in [`DirState::mtime`]. + pub mtime: i64, +} + /// How much of an image is available locally (FR-NC-6c). /// /// Surfaced in the UI so a user always knows what they have — the failure diff --git a/platform/dr-plat/src/lib.rs b/platform/dr-plat/src/lib.rs index f0151cf..a889172 100644 --- a/platform/dr-plat/src/lib.rs +++ b/platform/dr-plat/src/lib.rs @@ -5,7 +5,9 @@ //! ARCH §10). pub mod secrets; +pub mod storage; pub use secrets::{ EphemeralSecretStore, PlatformSecretStore, SecretError, SecretKind, SecretRef, SecretStore, }; +pub use storage::{DirRef, Entry, LocalStorage, Node, SeekableRead, Storage, StorageError}; diff --git a/platform/dr-plat/src/storage.rs b/platform/dr-plat/src/storage.rs new file mode 100644 index 0000000..9817646 --- /dev/null +++ b/platform/dr-plat/src/storage.rs @@ -0,0 +1,861 @@ +//! TRACES: FR-CAT-1 | FR-CAT-1a | NFR-PORT-1 | NFR-PORT-3 +//! Reaching stored bytes without naming a path (ARCH §3.1, §10). +//! +//! Android's Storage Access Framework hands out no filesystem path (ARCH §6.9), +//! so **nothing above this module may take one**. A library location is a +//! [`RootId`] the user granted; everything inside it is a [`DirRef`] or a +//! [`SourceRef`], both of which are opaque `(root, key)` pairs whose keys only +//! the implementation that produced them knows how to read. +//! +//! A `Path` therefore appears exactly once in the whole application: at +//! [`LocalStorage::grant`], where the folder the user picked is handed in. From +//! there on it is a `RootId`. +//! +//! # Adding Android SAF later +//! +//! It is a second implementation of [`Storage`] and no change at any call site. +//! Two properties of this API are what buy that, and both look like ceremony +//! until SAF is the thing being written: +//! +//! - **A listing hands back references, never names for the caller to join.** +//! A SAF document id is not composable — `parent_id + "/" + name` is not the +//! child's id, and the only way to learn a child's id is the children query +//! that produced the listing. So [`Entry`] carries the [`DirRef`] or +//! [`SourceRef`] the provider itself returned, and no caller ever builds one +//! by concatenation. [`LocalStorage`] could perfectly well have exposed a +//! "join a name onto a directory" helper; that helper is the one thing a SAF +//! implementation could not have provided. +//! - **A reference is a key that survives a restart.** The catalog stores the +//! key and rebuilds the reference with [`DirRef::from_parts`] on the next +//! run. On Linux the key is a relative path; on SAF it is a document id under +//! a persisted tree grant, which is re-resolvable for exactly the same +//! reason. +//! +//! What SAF will need in addition is the grant itself — the persisted tree URI, +//! which the catalog's `roots.grant_blob` column already has a home for, and +//! which is handed to the implementation at construction just as a path is +//! here. + +use std::collections::BTreeMap; +use std::fmt; +use std::io::{Read, Seek}; +use std::path::{Component, Path, PathBuf}; + +use dr_types::{ByteRange, DirEntry, DirState, RootId, SourceRef}; + +/// TRACES: FR-CAT-1a +/// An opaque, re-resolvable reference to a *directory* under a granted root. +/// +/// The counterpart of [`SourceRef`], which addresses a file. One type rather +/// than a mirrored three-variant enum because a directory is never resolved by +/// anything except the storage that owns its root: the root's kind already +/// determines how the key is read, so a discriminator on each reference would +/// only repeat it. +/// +/// The `key` is **opaque to callers** and stable across restarts. Its meaning +/// belongs to the implementation — a relative path under the root on a +/// filesystem, a `DocumentsContract` document id on SAF — and the catalog +/// stores it verbatim in `folders.path` so a later run can rebuild the +/// reference with [`DirRef::from_parts`]. +#[derive(Debug, Clone, PartialEq, Eq, Hash, PartialOrd, Ord)] +pub struct DirRef { + root: RootId, + key: String, +} + +impl DirRef { + /// The granted root itself, which every walk starts from. + pub fn root(root: RootId) -> Self { + Self { + root, + key: String::new(), + } + } + + /// Rebuild a reference from a key a previous scan stored. + /// + /// The re-resolution FR-CAT-1a requires: after a restart the catalog holds + /// keys and nothing else, and a scan that could not resume from them would + /// have to walk the whole library to find the folder it left off in. + pub fn from_parts(root: RootId, key: impl Into) -> Self { + Self { + root, + key: key.into(), + } + } + + pub fn root_id(&self) -> RootId { + self.root + } + + /// The stored form. Meaningful only to the storage that produced it. + pub fn key(&self) -> &str { + &self.key + } + + /// Whether this is the granted root rather than something inside it. + /// + /// The walk needs it: a failure at the root is the whole library being + /// unreachable, and the deletion sweep must not run; a failure below it is + /// one folder (FR-CAT-9). + pub fn is_root(&self) -> bool { + self.key.is_empty() + } + + /// The last component, for exclusion checks and display. + pub fn name(&self) -> &str { + self.key.rsplit(['/', ':']).next().unwrap_or(&self.key) + } +} + +impl fmt::Display for DirRef { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + if self.key.is_empty() { + write!(f, "root {}", self.root.0) + } else { + write!(f, "{}", self.key) + } + } +} + +/// How to reach a listed entry. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum Node { + Dir(DirRef), + File(SourceRef), +} + +/// One entry from a listing: what the scanner classifies on, and how to reach +/// it. +/// +/// `meta.is_dir` and the [`Node`] variant always agree — the implementation +/// sets both from one observation. They are separate because they serve +/// different readers: `meta` goes to `dr_catalog::scan`, which decides, and +/// `node` goes to whatever acts on the decision. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct Entry { + pub meta: DirEntry, + pub node: Node, +} + +/// A stream that can be read and seeked — what a decoder wants. +/// +/// Blanket-implemented, so a `File`, a `Cursor>` in a test, and a +/// future SAF `ParcelFileDescriptor` wrapper all qualify without ceremony. +pub trait SeekableRead: Read + Seek + Send {} +impl SeekableRead for T {} + +/// TRACES: NFR-ARCH-4 +/// Something went wrong reaching storage. +/// +/// Typed, and never a panic: a library is on removable media, on a network +/// mount, or behind a permission the user can revoke while the app is running, +/// so every one of these is a normal Tuesday rather than a bug. +/// +/// Distinct from [`dr_types::SourceError`], which is what the *decode* path +/// sees. This one also speaks about roots and directories, which a decoder has +/// no concept of. +#[derive(Debug, thiserror::Error)] +pub enum StorageError { + /// A reference naming a root this storage was never granted. + /// + /// Almost always a stale catalog row: the library was removed and its rows + /// outlived it. + #[error("root {0} was not granted to this storage")] + UnknownRoot(u64), + + /// A reference of the wrong shape — a SAF document id handed to the + /// filesystem implementation, or the reverse. + /// + /// Refused rather than guessed at: the two address spaces have no overlap, + /// and a guess would read the wrong file rather than fail. + #[error("{0}")] + Unsupported(&'static str), + + /// A stored key that would leave its root. + /// + /// The catalog is a file on disk that other programs can edit, so a key + /// containing `..` is possible however it got there. Refused, because a + /// grant to one folder must not become a read of the whole filesystem + /// (NFR-SEC-1). + #[error("key {0:?} escapes its root")] + EscapesRoot(String), + + #[error("not found: {0}")] + NotFound(String), + + #[error("permission denied: {0}")] + PermissionDenied(String), + + #[error("not a directory: {0}")] + NotADirectory(String), + + #[error("range reads unsupported by this storage")] + RangeUnsupported, + + #[error("io error: {0}")] + Io(String), +} + +/// TRACES: FR-CAT-1 | FR-CAT-1a | NFR-PORT-1 +/// Enumerate and read the contents of granted library roots. +/// +/// Implemented per platform and injected at construction, so `core/` contains +/// no `#[cfg(target_os)]` (ARCH §10). +pub trait Storage: Send + Sync { + /// Every root this storage can currently reach. + fn roots(&self) -> Vec; + + /// The granted root as a directory, which is where a walk begins. + /// + /// Fails on a root that was never granted, so a stale catalog row is a + /// typed error rather than an empty library. + fn root_dir(&self, root: RootId) -> Result; + + /// Probe a directory without reading its contents. + /// + /// The cheap half of the pair, and the reason incremental scanning is + /// affordable: this costs one `stat` plus a name-only directory read, + /// where [`list`](Self::list) costs a `stat` per child. On a library of 2k + /// folders and 50k images that is 2k probes against 50k, which is the + /// difference between meeting and missing NFR-P1. + fn dir_state(&self, dir: &DirRef) -> Result; + + /// List a directory's direct children, with the metadata to classify them. + /// + /// Returns the whole listing rather than streaming it through a callback, + /// because the caller needs the complete set at once: an image in the + /// catalog that this listing does *not* contain has been deleted, and that + /// conclusion cannot be drawn one entry at a time. + fn list(&self, dir: &DirRef) -> Result, StorageError>; + + /// Open a seekable stream over a source. + fn open(&self, src: &SourceRef) -> Result, StorageError>; + + /// Read a byte range without opening the whole source. + /// + /// First-class rather than a convenience over [`open`](Self::open), because + /// for the case that matters it is a different operation and not a smaller + /// one: extracting an embedded JPEG preview from an 80 MB RAW over the + /// network transfers 1–3 MB (ARCH §3.1, FR-CULL-2, FR-NC-3). + /// + /// A range reaching past the end yields the bytes that exist — a short read + /// is the honest answer, and a caller needing exactly `n` bytes must say so + /// by checking the length. + fn read_range(&self, src: &SourceRef, range: ByteRange) -> Result, StorageError>; +} + +/// TRACES: FR-PLAT-LIN-1 | NFR-PORT-1 +/// The filesystem implementation: a root is a directory the user picked. +/// +/// Also the right implementation for a Flatpak, where the portal returns a real +/// path the sandbox can see (FR-PLAT-LIN-3). +/// +/// Present on Android too, and harmless there: the type compiles wherever +/// `std::fs` does, so `ui/` can name it unconditionally. It is not how an +/// Android user's library is reached — that is SAF, and ARCH §6.9 explains +/// why nothing else is on offer. +#[derive(Debug, Default)] +pub struct LocalStorage { + /// Ordered so [`roots`](Storage::roots) is stable, which keeps a scan of + /// several roots reproducible. + roots: BTreeMap, +} + +impl LocalStorage { + pub fn new() -> Self { + Self::default() + } + + /// One granted root, the common case. + pub fn with_root(root: RootId, dir: impl Into) -> Self { + let mut s = Self::new(); + s.grant(root, dir); + s + } + + /// Record that the user granted `dir` as `root`. + /// + /// **The only place a `Path` enters the application.** Above this line a + /// library location is a `RootId`, which is what lets the same catalog and + /// the same scanner run against SAF, where no path exists at all. + pub fn grant(&mut self, root: RootId, dir: impl Into) { + self.roots.insert(root, dir.into()); + } + + /// Where a root sits, for the app that granted it — to show the user, or to + /// store so the grant survives a restart. + pub fn root_path(&self, root: RootId) -> Option<&Path> { + self.roots.get(&root).map(|p| p.as_path()) + } + + fn base(&self, root: RootId) -> Result<&Path, StorageError> { + self.roots + .get(&root) + .map(|p| p.as_path()) + .ok_or(StorageError::UnknownRoot(root.0)) + } + + /// Turn a `(root, key)` pair back into a path, refusing anything that + /// leaves the root. + /// + /// Every component must be an ordinary name: `..` would climb out, and an + /// absolute key would discard the root entirely — `Path::join` silently + /// replaces rather than appends when handed one, which is how a "relative" + /// path of `/etc` becomes a read of `/etc`. + fn resolve(&self, root: RootId, key: &str) -> Result { + let base = self.base(root)?; + if key.is_empty() { + return Ok(base.to_path_buf()); + } + let rel = Path::new(key); + if !rel.components().all(|c| matches!(c, Component::Normal(_))) { + return Err(StorageError::EscapesRoot(key.to_string())); + } + Ok(base.join(rel)) + } + + fn file_path(&self, src: &SourceRef) -> Result { + match src { + SourceRef::Local { root, relative } => self.resolve(*root, relative), + SourceRef::Document { .. } => Err(StorageError::Unsupported( + "a SAF document reference cannot be read from the filesystem", + )), + SourceRef::Remote { .. } => Err(StorageError::Unsupported( + "a remote reference is resolved by the sync layer, not by storage", + )), + } + } +} + +/// A child's key, built the one place that is allowed to build one. +/// +/// Concatenation is safe here and only here: this implementation *chose* to +/// make its keys relative paths, so it is the only code entitled to know that +/// they compose. A SAF implementation has no equivalent (see the module docs), +/// which is why this is a private helper and not a method on [`DirRef`]. +fn child_key(parent: &str, name: &str) -> String { + if parent.is_empty() { + name.to_string() + } else { + format!("{parent}/{name}") + } +} + +impl Storage for LocalStorage { + fn roots(&self) -> Vec { + self.roots.keys().copied().collect() + } + + fn root_dir(&self, root: RootId) -> Result { + self.base(root)?; + Ok(DirRef::root(root)) + } + + fn dir_state(&self, dir: &DirRef) -> Result { + let path = self.resolve(dir.root_id(), dir.key())?; + let meta = std::fs::metadata(&path).map_err(|e| map_io(&path, e))?; + if !meta.is_dir() { + return Err(StorageError::NotADirectory(path.display().to_string())); + } + + // Names only — `read_dir` yields entries from `getdents` without a + // `stat` per child, so the count costs one pass and no per-file I/O. + // That is what makes this the cheap probe `list` is not. + let entry_count = std::fs::read_dir(&path) + .map_err(|e| map_io(&path, e))? + .count(); + + Ok(DirState { + mtime: modified_millis(&meta), + // Saturating rather than wrapping: a directory of four billion + // entries would otherwise wrap to a small number and could compare + // equal after a change. It cannot happen, and being wrong about it + // would be silent. + entry_count: u32::try_from(entry_count).unwrap_or(u32::MAX), + }) + } + + fn list(&self, dir: &DirRef) -> Result, StorageError> { + let path = self.resolve(dir.root_id(), dir.key())?; + let read = std::fs::read_dir(&path).map_err(|e| map_io(&path, e))?; + + let mut out = Vec::new(); + for entry in read { + let entry = match entry { + Ok(e) => e, + // One unreadable entry is not an unreadable directory. Skipping + // it loses one file; failing the listing would make the folder + // look empty, and an empty folder is a *deletion* to the sweep. + Err(e) => { + log::debug!("list {}: skipping unreadable entry: {e}", path.display()); + continue; + } + }; + + // A name that is not UTF-8 cannot become a key, and a lossy + // conversion would produce a key that resolves to nothing — an + // image catalogued and then permanently unreadable. Skipped, and + // said out loud, because the user's file is real and we are + // choosing not to see it. + let Some(name) = entry.file_name().to_str().map(str::to_owned) else { + log::warn!( + "list {}: skipping {:?}, whose name is not valid UTF-8", + path.display(), + entry.file_name() + ); + continue; + }; + + // Follows symlinks, unlike `DirEntry::metadata`. A photographer who + // symlinks last year's drive into the library means it as part of + // the library. The walk's depth limit is what stops a loop. + let child = path.join(&name); + let meta = match std::fs::metadata(&child) { + Ok(m) => m, + // A broken symlink, or a file deleted between the listing and + // this stat. Neither is an error worth failing a folder for. + Err(e) => { + log::debug!("list {}: skipping {name}: {e}", path.display()); + continue; + } + }; + + let key = child_key(dir.key(), &name); + let is_dir = meta.is_dir(); + out.push(Entry { + meta: DirEntry { + name, + is_dir, + size: meta.len(), + mtime: modified_millis(&meta), + }, + node: if is_dir { + Node::Dir(DirRef::from_parts(dir.root_id(), key)) + } else { + Node::File(SourceRef::Local { + root: dir.root_id(), + relative: key, + }) + }, + }); + } + + // Directory order is filesystem order, which is arbitrary and differs + // between runs. Sorting makes a scan reproducible and a test able to + // assert on what it found. + out.sort_by(|a, b| a.meta.name.cmp(&b.meta.name)); + Ok(out) + } + + fn open(&self, src: &SourceRef) -> Result, StorageError> { + let path = self.file_path(src)?; + let file = std::fs::File::open(&path).map_err(|e| map_io(&path, e))?; + Ok(Box::new(file)) + } + + fn read_range(&self, src: &SourceRef, range: ByteRange) -> Result, StorageError> { + use std::io::SeekFrom; + + if range.end <= range.start { + return Ok(Vec::new()); + } + let path = self.file_path(src)?; + let mut file = std::fs::File::open(&path).map_err(|e| map_io(&path, e))?; + file.seek(SeekFrom::Start(range.start)) + .map_err(|e| map_io(&path, e))?; + + // `take` and grow, rather than a buffer sized to the request: the range + // comes from a header the file itself declared, and a corrupt one + // asking for four gigabytes must not be allocated before it is known + // that four gigabytes exist (NFR-SEC-1). + let mut buf = Vec::new(); + file.take(range.end - range.start) + .read_to_end(&mut buf) + .map_err(|e| map_io(&path, e))?; + Ok(buf) + } +} + +/// Milliseconds since the epoch, or 0 where the platform will not say. +/// +/// Milliseconds because that is the unit change detection is expressed in, and +/// the unit's granularity is the width of the window in which a change is +/// invisible — see [`dr_types::DirState::mtime`]. Seconds would hide a card +/// imported and scanned within the same tick. +/// +/// A file whose mtime is unreadable compares equal to itself forever and so is +/// never re-read. That is the better failure: the alternative, a value that +/// changes each time it is asked for, would re-process the file on every scan. +fn modified_millis(meta: &std::fs::Metadata) -> i64 { + let Ok(t) = meta.modified() else { + return 0; + }; + match t.duration_since(std::time::UNIX_EPOCH) { + Ok(d) => i64::try_from(d.as_millis()).unwrap_or(i64::MAX), + // Before 1970. Rare, but an archive of digitised film can carry one, + // and it must not become a huge positive number. + Err(e) => i64::try_from(e.duration().as_millis()) + .unwrap_or(i64::MAX) + .saturating_neg(), + } +} + +fn map_io(path: &Path, e: std::io::Error) -> StorageError { + let what = path.display().to_string(); + match e.kind() { + std::io::ErrorKind::NotFound => StorageError::NotFound(what), + std::io::ErrorKind::PermissionDenied => StorageError::PermissionDenied(what), + _ => StorageError::Io(format!("{what}: {e}")), + } +} + +#[cfg(test)] +mod tests { + use super::*; + use std::fs; + + /// A throwaway directory tree, removed when the test ends. + struct Tree(PathBuf); + + impl Tree { + fn new(name: &str) -> Self { + let dir = std::env::temp_dir().join(format!( + "dr-plat-{name}-{}-{:?}", + std::process::id(), + std::thread::current().id() + )); + let _ = fs::remove_dir_all(&dir); + fs::create_dir_all(&dir).expect("temp dir"); + Tree(dir) + } + + fn dir(&self, rel: &str) -> &Self { + fs::create_dir_all(self.0.join(rel)).expect("mkdir"); + self + } + + fn file(&self, rel: &str, bytes: &[u8]) -> &Self { + let p = self.0.join(rel); + if let Some(parent) = p.parent() { + fs::create_dir_all(parent).expect("mkdir"); + } + fs::write(p, bytes).expect("write"); + self + } + + fn storage(&self) -> LocalStorage { + LocalStorage::with_root(RootId(1), self.0.clone()) + } + } + + impl Drop for Tree { + fn drop(&mut self) { + let _ = fs::remove_dir_all(&self.0); + } + } + + const ROOT: RootId = RootId(1); + + #[test] + fn a_listing_names_files_and_directories_apart() { + let t = Tree::new("listing"); + t.file("IMG_0001.CR3", b"raw").dir("2026"); + let s = t.storage(); + + let entries = s.list(&s.root_dir(ROOT).unwrap()).unwrap(); + assert_eq!(entries.len(), 2); + assert_eq!(entries[0].meta.name, "2026"); + assert!(entries[0].meta.is_dir); + assert!(matches!(entries[0].node, Node::Dir(_))); + assert_eq!(entries[1].meta.name, "IMG_0001.CR3"); + assert_eq!(entries[1].meta.size, 3); + assert!(matches!(entries[1].node, Node::File(_))); + } + + #[test] + fn a_listing_hands_back_references_the_caller_never_composes() { + // The property that makes a SAF implementation a drop-in: the child's + // reference comes from the listing, because on SAF it is the only place + // it can come from. If a test ever has to build one by joining strings, + // the abstraction has already leaked. + let t = Tree::new("refs"); + t.file("2026/08/IMG_0042.CR3", b"raw"); + let s = t.storage(); + + let year = match &s.list(&s.root_dir(ROOT).unwrap()).unwrap()[0].node { + Node::Dir(d) => d.clone(), + other => panic!("expected a directory, got {other:?}"), + }; + let month = match &s.list(&year).unwrap()[0].node { + Node::Dir(d) => d.clone(), + other => panic!("expected a directory, got {other:?}"), + }; + let file = match &s.list(&month).unwrap()[0].node { + Node::File(f) => f.clone(), + other => panic!("expected a file, got {other:?}"), + }; + + assert_eq!( + file, + SourceRef::Local { + root: ROOT, + relative: "2026/08/IMG_0042.CR3".into() + } + ); + assert!(s.open(&file).is_ok()); + } + + #[test] + fn a_stored_key_reopens_the_same_directory_after_a_restart() { + // What FR-CAT-1a's "re-resolvable" means in practice: the catalog keeps + // keys, not handles, and a scan resuming tomorrow rebuilds the + // reference from one. Without this a restart is a full rewalk. + let t = Tree::new("reresolve"); + t.file("2026/IMG.CR3", b"raw"); + let s = t.storage(); + + let key = match &s.list(&s.root_dir(ROOT).unwrap()).unwrap()[0].node { + Node::Dir(d) => d.key().to_string(), + other => panic!("expected a directory, got {other:?}"), + }; + + let rebuilt = DirRef::from_parts(ROOT, key); + assert_eq!(s.list(&rebuilt).unwrap()[0].meta.name, "IMG.CR3"); + } + + #[test] + fn a_key_that_climbs_out_of_the_root_is_refused() { + // A grant is to one folder. The catalog is an ordinary file that other + // programs can edit, so a `..` in a key is reachable however it got + // there, and honouring it would turn a grant to ~/Photos into a read of + // the whole filesystem (NFR-SEC-1). + let t = Tree::new("escape"); + let s = t.storage(); + + for key in ["../etc", "a/../../etc", "/etc"] { + let dir = DirRef::from_parts(ROOT, key); + assert!( + matches!(s.dir_state(&dir), Err(StorageError::EscapesRoot(_))), + "{key} was not refused" + ); + } + } + + #[test] + fn an_absolute_key_does_not_silently_replace_the_root() { + // `Path::join` replaces rather than appends when given an absolute + // path, so this one is not merely an escape — it is an escape that + // looks like ordinary joining and would never be noticed in review. + let t = Tree::new("absolute"); + let s = t.storage(); + let src = SourceRef::Local { + root: ROOT, + relative: "/etc/passwd".into(), + }; + assert!(matches!( + s.read_range(&src, 0..16), + Err(StorageError::EscapesRoot(_)) + )); + } + + #[test] + fn probing_a_directory_reports_what_change_detection_needs() { + let t = Tree::new("probe"); + t.file("a.CR3", b"1").file("b.CR3", b"2"); + let s = t.storage(); + + let before = s.dir_state(&s.root_dir(ROOT).unwrap()).unwrap(); + assert_eq!(before.entry_count, 2); + + t.file("c.CR3", b"3"); + let after = s.dir_state(&s.root_dir(ROOT).unwrap()).unwrap(); + assert_ne!( + before, after, + "an added file must move the state, or the folder is pruned and the \ + image never enters the catalog" + ); + assert_eq!(after.entry_count, 3); + } + + #[test] + fn the_probe_counts_directories_as_well_as_files() { + // A new subfolder full of images changes nothing about the parent's + // files. If the count ignored directories, the parent would look + // unchanged and the whole subtree would go unseen. + let t = Tree::new("probe-dirs"); + t.file("a.CR3", b"1"); + let s = t.storage(); + let before = s.dir_state(&s.root_dir(ROOT).unwrap()).unwrap(); + + t.dir("2026"); + let after = s.dir_state(&s.root_dir(ROOT).unwrap()).unwrap(); + assert_eq!(after.entry_count, before.entry_count + 1); + } + + #[test] + fn a_range_read_returns_only_the_bytes_asked_for() { + let t = Tree::new("range"); + t.file("IMG.CR3", b"0123456789"); + let s = t.storage(); + let src = SourceRef::Local { + root: ROOT, + relative: "IMG.CR3".into(), + }; + + assert_eq!(s.read_range(&src, 2..6).unwrap(), b"2345"); + assert_eq!(s.read_range(&src, 0..0).unwrap(), b""); + } + + #[test] + fn a_range_past_the_end_is_short_rather_than_an_error() { + // Preview offsets come out of the file's own header. A truncated or + // mis-parsed one must yield "here is what exists", not a failed decode + // and not a four-gigabyte allocation (NFR-SEC-1). + let t = Tree::new("range-eof"); + t.file("IMG.CR3", b"0123456789"); + let s = t.storage(); + let src = SourceRef::Local { + root: ROOT, + relative: "IMG.CR3".into(), + }; + + assert_eq!(s.read_range(&src, 8..u64::MAX / 2).unwrap(), b"89"); + assert_eq!(s.read_range(&src, 999..1_000).unwrap(), b""); + } + + #[test] + fn an_open_stream_can_seek() { + // The decoders need it: a RAW's preview lives at an offset the header + // names, and a forward-only stream would mean reading 80 MB to get 2. + let t = Tree::new("seek"); + t.file("IMG.CR3", b"0123456789"); + let s = t.storage(); + let mut r = s + .open(&SourceRef::Local { + root: ROOT, + relative: "IMG.CR3".into(), + }) + .unwrap(); + + r.seek(std::io::SeekFrom::Start(5)).unwrap(); + let mut buf = [0u8; 2]; + r.read_exact(&mut buf).unwrap(); + assert_eq!(&buf, b"56"); + } + + #[test] + fn an_ungranted_root_is_a_typed_error_not_an_empty_library() { + // A removed library leaves catalog rows behind. Reporting them as + // "nothing here" would let the deletion sweep take the lot (FR-CAT-9). + let s = LocalStorage::new(); + assert!(matches!( + s.root_dir(RootId(7)), + Err(StorageError::UnknownRoot(7)) + )); + assert!(matches!( + s.dir_state(&DirRef::root(RootId(7))), + Err(StorageError::UnknownRoot(7)) + )); + assert!(s.roots().is_empty()); + } + + #[test] + fn a_missing_directory_is_not_reported_as_empty() { + // The same failure from the other direction: an unplugged drive must + // error, because an empty listing means every image under it was + // deleted. + let t = Tree::new("missing"); + let s = t.storage(); + let gone = DirRef::from_parts(ROOT, "nowhere"); + assert!(matches!(s.dir_state(&gone), Err(StorageError::NotFound(_)))); + assert!(matches!(s.list(&gone), Err(StorageError::NotFound(_)))); + } + + #[test] + fn a_reference_of_the_wrong_kind_is_refused_rather_than_guessed_at() { + // A SAF document id is not a path. Treating it as one would resolve to + // some other file, which is worse than failing. + let t = Tree::new("wrong-kind"); + let s = t.storage(); + let saf = SourceRef::Document { + tree: ROOT, + document_id: "primary:DCIM/IMG.CR3".into(), + }; + assert!(matches!(s.open(&saf), Err(StorageError::Unsupported(_)))); + let remote = SourceRef::Remote { + file_id: 1, + path: "Photos/IMG.CR3".into(), + }; + assert!(matches!(s.open(&remote), Err(StorageError::Unsupported(_)))); + } + + #[test] + fn several_roots_coexist() { + // Two libraries on two drives is FR-CAT-1's "one or more roots", and + // each reference carries which one it belongs to. + let a = Tree::new("multi-a"); + a.file("a.CR3", b"1"); + let b = Tree::new("multi-b"); + b.file("b.CR3", b"2"); + + let mut s = LocalStorage::new(); + s.grant(RootId(1), a.0.clone()); + s.grant(RootId(2), b.0.clone()); + + assert_eq!(s.roots(), vec![RootId(1), RootId(2)]); + assert_eq!( + s.list(&s.root_dir(RootId(2)).unwrap()).unwrap()[0] + .meta + .name, + "b.CR3" + ); + } + + #[test] + fn a_listing_is_ordered_the_same_way_twice() { + // Filesystem order is arbitrary and differs between runs; a scan that + // depended on it would produce a different catalog each time. + let t = Tree::new("order"); + for n in ["c.CR3", "a.CR3", "b.CR3"] { + t.file(n, b"x"); + } + let s = t.storage(); + let names: Vec = s + .list(&s.root_dir(ROOT).unwrap()) + .unwrap() + .into_iter() + .map(|e| e.meta.name) + .collect(); + assert_eq!(names, vec!["a.CR3", "b.CR3", "c.CR3"]); + } + + #[cfg(unix)] + #[test] + fn a_name_that_is_not_utf8_is_skipped_rather_than_mangled() { + // A lossy conversion would produce a key that resolves to nothing: the + // image would be catalogued and then permanently unopenable. Better to + // not see the file than to promise it and fail later. + use std::os::unix::ffi::OsStrExt; + + let t = Tree::new("non-utf8"); + t.file("good.CR3", b"1"); + let bad = t.0.join(std::ffi::OsStr::from_bytes(b"bad\xff.CR3")); + fs::write(&bad, b"2").expect("write"); + + let s = t.storage(); + let entries = s.list(&s.root_dir(ROOT).unwrap()).unwrap(); + assert_eq!(entries.len(), 1); + assert_eq!(entries[0].meta.name, "good.CR3"); + } + + #[test] + fn the_root_is_distinguishable_from_what_is_inside_it() { + // The walk keys the difference between "the library is unreachable" and + // "one folder failed" on this, and those have opposite consequences for + // the deletion sweep (FR-CAT-9). + assert!(DirRef::root(ROOT).is_root()); + assert!(!DirRef::from_parts(ROOT, "2026").is_root()); + assert_eq!(DirRef::from_parts(ROOT, "2026/08").name(), "08"); + } +}