From bb71f141e7a8710a80ef33ab36604593f1753a6d Mon Sep 17 00:00:00 2001 From: Duncan Tourolle Date: Mon, 17 Aug 2026 08:59:17 +0200 Subject: [PATCH] Let a folder on this machine be scanned into the catalog MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `dr_catalog::scan` has known since it was written what a changed directory means — when to prune, when to list, and the one question that decides whether a deletion sweep is safe. It was fully tested and nothing called it, because walking a real directory "belongs to the platform layer" and the platform layer was eleven lines re-exporting `secrets`. So every photograph in DarkRoom arrived over WebDAV, and a user without a Nextcloud account saw nothing at all. This is the missing half: a `Storage` trait, a filesystem implementation of it, and the driver that pours one into the other. The trait is shaped by the platform it does *not* yet support. Android's SAF gives no filesystem path, which is why `SourceRef` exists; less obviously, it gives no way to *compose* one either — a document id is opaque, and the only way to learn a child's id is the children query that returned it. So a listing hands back the reference to each entry rather than a name for the caller to join onto a parent, and there is deliberately no "path + name" helper anywhere above `LocalStorage`. That single restriction is what makes SAF a second implementation rather than a second set of call sites. A reference is otherwise an opaque `(RootId, key)` pair the catalog stores verbatim and rebuilds later, which a persisted tree grant supports exactly as a relative path does. A `Path` now appears in one place: `LocalStorage::grant`, where the folder the user picked is handed in. Everything above it addresses a `RootId`. `dr_catalog::walk` is the seam. It probes a directory, asks `scan` what that means, lists only when told to, and reconciles what it found against the rows it holds. Two things it does are worth saying out loud, because both are ways to lose a library: Absence only counts where absence was observed. A listed folder proves its missing images are gone; a pruned one proves nothing about its contents, and a scan that was cancelled or that failed part-way proves nothing about folders it never reached. So the file sweep runs per listed folder, the folder sweep runs once at the end and only after a complete scan, and a root that cannot be reached at all marks its images offline and deletes nothing — FR-CAT-9's line between proven-absent and merely-unreachable, which is the difference between unplugging a drive and losing everything on it. A trashed image is absent from its folder on purpose. It is exempt from both sweeps, and detached from a folder about to be deleted rather than cascaded away with it, or a soft delete would come undone the first time the folder it came from was rescanned. Two things the tests taught, both changes to what was there before: Modification times are now milliseconds, not seconds. Change detection asks whether a timestamp moved, so the unit's granularity is the width of the window in which a change is invisible — and a second is long enough to copy a card and start a scan. The test that caught it looked like a test bug; it was not. SAF reports milliseconds natively, so this is also the unit that needs no conversion on the platform with the coarser clock. And an in-place rewrite of an existing file is invisible to directory-level pruning, because writing to a file moves neither its directory's mtime nor its entry count. That is a real limit, now documented and held by a test rather than left to be discovered. It bites less than it reads: an export, a restore, `mv`, and every editor that saves safely write beside the file and rename over it, which does move both. Narrowing the format filter no longer deletes what it stops matching, which fell out of the same principle: unticking JPEG says stop looking for new ones, not discard the hundred already rated. The files are sitting right there. `DirState` and `DirEntry` move to `dr-types`. They are the sentence the platform says to the catalog and both crates need the same one; `scan` re-exports them so nothing that used them has changed. Not done: the UI. The launch screen's "Open library" flow is account-shaped from the first field to the thumbnail worker, and giving it a local branch is its own piece of work rather than a button. `cargo run -p dr-catalog --example scan_local -- ~/Pictures` scans a real folder and reports what it cost; run it twice to see the second run list nothing. Co-Authored-By: Claude Opus 5 --- Cargo.lock | 2 + core/dr-catalog/Cargo.toml | 10 + core/dr-catalog/examples/scan_local.rs | 111 ++ core/dr-catalog/src/error.rs | 9 + core/dr-catalog/src/lib.rs | 3 + core/dr-catalog/src/query.rs | 5 +- core/dr-catalog/src/scan.rs | 37 +- core/dr-catalog/src/walk.rs | 1367 ++++++++++++++++++++++++ core/dr-types/src/lib.rs | 68 +- platform/dr-plat/src/lib.rs | 2 + platform/dr-plat/src/storage.rs | 861 +++++++++++++++ 11 files changed, 2442 insertions(+), 33 deletions(-) create mode 100644 core/dr-catalog/examples/scan_local.rs create mode 100644 core/dr-catalog/src/walk.rs create mode 100644 platform/dr-plat/src/storage.rs diff --git a/Cargo.lock b/Cargo.lock index f654272..b2ea7c3 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1330,7 +1330,9 @@ checksum = "d8b14ccef22fc6f5a8f4d7d768562a182c04ce9a3b3157b91390b52ddfdf1a76" name = "dr-catalog" version = "0.1.0" dependencies = [ + "dr-plat", "dr-types", + "env_logger", "log", "rusqlite", "serde_json", diff --git a/core/dr-catalog/Cargo.toml b/core/dr-catalog/Cargo.toml index fe2e1b3..6be0996 100644 --- a/core/dr-catalog/Cargo.toml +++ b/core/dr-catalog/Cargo.toml @@ -7,9 +7,19 @@ license.workspace = true [dependencies] dr-types.workspace = true +# The `Storage` trait, and nothing else from it. A scan has to read a real +# directory, and this is how `core/` reaches the platform without a +# `#[cfg(target_os)]` of its own (ARCH §4.1: calls go downward). +dr-plat.workspace = true rusqlite.workspace = true thiserror.workspace = true log.workspace = true # `collections.selector_json` — the stored form of a smart collection's # selector. The column predates this dependency; nothing else here is JSON. serde_json.workspace = true + +# For the `scan_local` example only, which is a diagnostic tool: what it is +# diagnosing is often a folder the scan warned about and skipped, and those +# warnings go to `log`. +[dev-dependencies] +env_logger.workspace = true diff --git a/core/dr-catalog/examples/scan_local.rs b/core/dr-catalog/examples/scan_local.rs new file mode 100644 index 0000000..a75694f --- /dev/null +++ b/core/dr-catalog/examples/scan_local.rs @@ -0,0 +1,111 @@ +//! Scan a real folder on this machine into a catalog, and say what it cost. +//! +//! cargo run -p dr-catalog --example scan_local -- ~/Pictures [catalog.sqlite] +//! +//! **Run it twice.** The first run is a full walk; the second is the one worth +//! watching, because on an unchanged library it should list no directories at +//! all and take a fraction of the time. That difference is NFR-P1, and a +//! synthetic test cannot show it at the scale a real library does — 121,785 +//! files in a synced folder is a different question from twenty in a temporary +//! directory. +//! +//! Writes only to the catalog file, which defaults to a fixed path in the +//! system temporary directory so a second run has something to compare +//! against. Nothing in the scanned folder is touched. + +use std::path::PathBuf; + +use dr_catalog::walk::{ensure_root, scan_root, RootKind}; +use dr_catalog::Catalog; +use dr_plat::LocalStorage; +use dr_types::FormatFilter; + +fn main() { + env_logger::init(); + + let mut args = std::env::args().skip(1); + let Some(dir) = args.next().map(PathBuf::from) else { + eprintln!("usage: scan_local [catalog.sqlite]"); + std::process::exit(2); + }; + let catalog_path = args + .next() + .map(PathBuf::from) + .unwrap_or_else(|| std::env::temp_dir().join("darkroom-scan-local.sqlite")); + + let catalog = match Catalog::open(&catalog_path) { + Ok(c) => c, + Err(e) => { + eprintln!("cannot open {}: {e}", catalog_path.display()); + std::process::exit(1); + } + }; + println!("catalog: {}", catalog_path.display()); + + // The label is how the grant is spelled, and the only place a path is + // written down. Everything after this line addresses files by `RootId`. + let label = dir.display().to_string(); + let root = match ensure_root(catalog.connection(), RootKind::Local, &label) { + Ok(r) => r, + Err(e) => { + eprintln!("cannot record the root: {e}"); + std::process::exit(1); + } + }; + let storage = LocalStorage::with_root(root, &dir); + + let now = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .map(|d| d.as_secs() as i64) + .unwrap_or(0); + + let started = std::time::Instant::now(); + let report = match scan_root( + catalog.connection(), + &storage, + root, + &FormatFilter::all(), + now, + || false, + |p| { + // One line per hundred directories: enough to show it is alive on a + // large library, not enough to be the thing that slows it down. + let visited = p.directories_listed + p.directories_pruned; + if visited % 100 == 0 { + println!( + " … {visited} directories ({} pruned), {} images", + p.directories_pruned, p.images_found + ); + } + }, + ) { + Ok(r) => r, + Err(e) => { + eprintln!("scan failed: {e}"); + std::process::exit(1); + } + }; + let elapsed = started.elapsed(); + + let total: i64 = catalog + .connection() + .query_row("SELECT count(*) FROM images", [], |r| r.get(0)) + .unwrap_or(-1); + + println!("\noutcome: {:?}", report.outcome); + println!( + "directories: {} listed, {} pruned", + report.progress.directories_listed, report.progress.directories_pruned + ); + println!( + "images: {} new, {} changed, {} unchanged, {} removed", + report.inserted, report.updated, report.unchanged, report.images_removed + ); + println!("folders: {} removed", report.folders_removed); + println!("catalogued: {total} in total"); + println!("took: {:.2?}", elapsed); + + if report.progress.directories_listed == 0 && report.progress.directories_pruned > 0 { + println!("\nnothing had changed: every folder was proven unchanged by one probe"); + } +} diff --git a/core/dr-catalog/src/error.rs b/core/dr-catalog/src/error.rs index 56391f7..b6d6f08 100644 --- a/core/dr-catalog/src/error.rs +++ b/core/dr-catalog/src/error.rs @@ -25,6 +25,15 @@ pub enum CatalogError { #[error("root {0} is unreachable; scan aborted without pruning")] RootUnreachable(u64), + /// A scan was asked for a root the catalog has no row for. + /// + /// A caller's mistake rather than a user's: the row is created when the + /// grant is obtained, because the label — the path, the tree URI — is known + /// only there. Inventing one here would file the library under a name + /// nothing else would look it up by. + #[error("no such root: {0}")] + NoSuchRoot(u64), + /// A smart collection whose selector references itself, directly or via /// another collection. #[error("collection {0} would form a cycle")] diff --git a/core/dr-catalog/src/lib.rs b/core/dr-catalog/src/lib.rs index db77b74..23ca044 100644 --- a/core/dr-catalog/src/lib.rs +++ b/core/dr-catalog/src/lib.rs @@ -11,6 +11,7 @@ //! //! - [`schema`] — tables and forward-only migrations //! - [`scan`] — incremental discovery that prunes unchanged directories +//! - [`walk`] — those decisions driven against real storage, local or SAF //! - [`query`] — selectors compiled to indexed SQL, windowed for the grid //! - [`collections`] — the collection tree and membership the UI edits //! - [`jobs`] — the durable background work queue @@ -41,6 +42,7 @@ pub mod scan; pub mod schema; pub mod sync; pub mod trash; +pub mod walk; pub use cache::{Budget, Cache, DEFAULT_BUDGET_BYTES}; pub use collections::{Collection, CollectionKind, TreeRow}; @@ -51,6 +53,7 @@ pub use query::{Query, Sort}; pub use rating::{Judgement, MAX_RATING}; pub use scan::{DirAction, DirState, EntryAction, ScanOutcome}; pub use trash::{TrashedImage, TRASH_DIR}; +pub use walk::{ensure_root, scan_root, RootKind, ScanProgress, ScanReport}; /// One row of the library grid. /// diff --git a/core/dr-catalog/src/query.rs b/core/dr-catalog/src/query.rs index 574489a..25b2bac 100644 --- a/core/dr-catalog/src/query.rs +++ b/core/dr-catalog/src/query.rs @@ -318,7 +318,10 @@ fn flag_code(f: FlagState) -> i64 { } } -fn availability_code(a: Availability) -> i64 { +/// The stored form of an availability. Shared with [`crate::walk`], which +/// writes the column this reads — two spellings of the same mapping would +/// filter for a state nothing ever writes. +pub(crate) fn availability_code(a: Availability) -> i64 { match a { Availability::MetadataOnly => 0, Availability::Preview => 1, diff --git a/core/dr-catalog/src/scan.rs b/core/dr-catalog/src/scan.rs index e1414ce..baf1cd1 100644 --- a/core/dr-catalog/src/scan.rs +++ b/core/dr-catalog/src/scan.rs @@ -14,35 +14,13 @@ //! //! This module holds the decision logic and the deletion-sweep rules; walking //! an actual directory belongs to the platform layer, which supplies -//! [`DirState`] and [`DirEntry`]. +//! [`DirState`] and [`DirEntry`]. [`crate::walk`] is what puts the two +//! together. + +pub use dr_types::{DirEntry, DirState}; use dr_types::FormatFilter; -/// What a directory looked like when last scanned, and what it looks like now. -/// -/// Both fields are cheap to obtain: one `stat` locally, one -/// `DocumentsContract` metadata query on SAF. -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub struct DirState { - pub mtime: i64, - /// Direct children, files and directories alike. - /// - /// mtime alone misses a delete-and-create inside one timestamp tick, and - /// coarse-granularity providers widen that window. The count does not - /// close the hole — a paired add and remove moves neither — but a bare add - /// or remove moves the count, and those are far commoner. - pub entry_count: u32, -} - -/// One entry from a directory listing. -#[derive(Debug, Clone, PartialEq, Eq)] -pub struct DirEntry { - pub name: String, - pub is_dir: bool, - pub size: u64, - pub mtime: i64, -} - /// What the scanner should do with a directory, before listing it. #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub enum DirAction { @@ -100,12 +78,17 @@ pub fn classify_entry( } /// Outcome of a scan, which decides whether pruning may run. -#[derive(Debug, Clone, Copy, PartialEq, Eq)] +/// +/// `Cancelled` is the default because a scan that has not run has proven +/// nothing absent, and every default in this area must fail towards keeping +/// photographs. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)] pub enum ScanOutcome { /// Every reachable folder was visited. Complete, /// The user cancelled. Partial state is valid — jobs are resumable — but /// unvisited folders must not be read as deleted. + #[default] Cancelled, /// The root itself could not be opened: drive unplugged, SAF grant /// revoked, share unmounted. diff --git a/core/dr-catalog/src/walk.rs b/core/dr-catalog/src/walk.rs new file mode 100644 index 0000000..c4172e7 --- /dev/null +++ b/core/dr-catalog/src/walk.rs @@ -0,0 +1,1367 @@ +//! TRACES: FR-CAT-1 | FR-CAT-1a | FR-CAT-9 | NFR-PORT-1 | NFR-P1 +//! Turning the decisions in [`scan`](crate::scan) into a catalogued library. +//! +//! [`scan`](crate::scan) knows what a changed directory *means* and holds no +//! I/O; [`dr_plat::Storage`] knows how to read a directory and holds no +//! catalog. This module is the only place the two meet, which is what keeps +//! both of them testable on their own — and what lets Android arrive as a +//! second `Storage` implementation with nothing here to change (ARCH §6.9). +//! +//! # What it costs on a library that has not changed +//! +//! One probe per directory and nothing else: no listing, no `stat` per file, no +//! row written. The saving is not incidental — 50k images in 2k folders is 2k +//! probes against 50k stats, and NFR-P1 is written against the former. +//! +//! # What pruning cannot see +//! +//! Writing to an existing file moves neither its directory's mtime nor its +//! entry count, so a folder full of rewritten photographs looks untouched and +//! is skipped. This is the price of pruning at the directory level, and it is +//! worth being plain about rather than discovering later. +//! +//! It bites less than it sounds. Almost nothing rewrites a RAW in place: an +//! export, a backup restore, `mv`, and every editor that saves safely write a +//! new file beside the old one and rename over it, which does move both the +//! mtime and — briefly — the count. Those are caught. What is missed is a +//! genuine in-place write, which for a photograph library is close to nothing; +//! and it is missed only until the folder is listed for some other reason. +//! +//! # The deletion sweep, which is the dangerous part +//! +//! Rows are deleted in two places, and both are guarded by the same principle: +//! **absence is only evidence of deletion where absence was actually +//! observed.** A folder that was listed proves its missing images are gone. A +//! folder that was pruned proves nothing about its contents, and a scan that +//! was cancelled or that failed part-way proves nothing about the folders it +//! never reached — hence [`ScanOutcome::may_prune`], and hence the folder sweep +//! running once at the end rather than as it goes. +//! +//! Get this wrong and an unplugged drive deletes the library. Every image would +//! be absent, every folder unreached, and the sweep would take all of them +//! along with their ratings and their edits. + +use std::collections::HashMap; + +use dr_plat::{DirRef, Node, Storage}; +use dr_types::{Availability, FormatFilter, RootId, SourceRef}; +use rusqlite::{Connection, OptionalExtension}; + +use crate::error::CatalogError; +use crate::jobs::{self, JobKind, Priority}; +use crate::query::availability_code; +use crate::scan::{ + classify_dir, classify_entry, DirAction, DirState, EntryAction, KnownFile, ScanOutcome, +}; +use crate::trash::TRASH_DIR; + +/// How deep to recurse before giving up. +/// +/// A symlink loop would otherwise walk forever — the filesystem implementation +/// follows symlinks deliberately, because a photographer who symlinks last +/// year's drive into the library means it. Real libraries are nowhere near this +/// deep. The same figure `dr_sync::scan` uses, for the same reason. +pub const MAX_DEPTH: usize = 32; + +/// TRACES: FR-CAT-3 +/// Directory holding derived state — thumbnail shards and catalog snapshots. +/// +/// Quoted rather than imported, exactly as [`TRASH_DIR`] is quoted in +/// [`crate::trash`]: `dr-catalog` does not depend on `dr-sync`, and adding that +/// dependency for one string would invert the layering. A test asserts the two +/// agree. +pub const DERIVED_DIR: &str = ".darkroom-derived"; + +/// Whether a directory is one the scan must stay out of. +/// +/// **The trash exclusion is half of the soft delete.** Trashed images are real +/// files in a real folder under the library root (FR-CAT-15); a scan that +/// walked it would re-index them as ordinary photographs and the delete would +/// come undone on the next refresh. +/// +/// Matched on the folder's own name at any depth, so a nested library moved in +/// wholesale carries its exclusions with it. +pub fn is_excluded(name: &str) -> bool { + matches!(name, TRASH_DIR | DERIVED_DIR) +} + +/// Which grant a root represents — the `roots.kind` column. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum RootKind { + /// A directory on a filesystem. + Local, + /// An Android persisted document tree. + Saf, + /// A folder on a remote server, reached through the sync layer. + Remote, +} + +impl RootKind { + pub fn as_str(self) -> &'static str { + match self { + RootKind::Local => "local", + RootKind::Saf => "saf", + RootKind::Remote => "remote", + } + } +} + +/// TRACES: FR-CAT-1 +/// Find or create the row for a granted library location. +/// +/// `label` is how the grant is spelled for this kind of root — the directory +/// path on Linux, the tree URI on Android, the remote folder on a server. It is +/// display text and identity, never something `core/` resolves: reaching the +/// files is [`Storage`]'s business and takes a [`RootId`], which is the whole +/// point of FR-CAT-1a. +/// +/// Idempotent, because it has to be: a second row for the same folder would +/// fragment the library across two roots, splitting the images and comparing +/// each half against the wrong scan generation. The `UNIQUE(kind, label)` in +/// the schema is what enforces it; this reuses the row rather than failing. +pub fn ensure_root(conn: &Connection, kind: RootKind, label: &str) -> Result { + conn.execute( + "INSERT INTO roots(kind, label) VALUES (?1, ?2) ON CONFLICT DO NOTHING", + rusqlite::params![kind.as_str(), label], + )?; + let id: i64 = conn.query_row( + "SELECT id FROM roots WHERE kind = ?1 AND label = ?2", + rusqlite::params![kind.as_str(), label], + |r| r.get(0), + )?; + Ok(RootId(id as u64)) +} + +/// Progress during a scan, so a large library reports rather than appears hung. +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] +pub struct ScanProgress { + pub directories_listed: usize, + /// Directories proven unchanged and skipped. The value of pruning, made + /// visible — on a healthy rescan this is nearly the whole library. + pub directories_pruned: usize, + pub images_found: usize, +} + +/// What a scan did. +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] +pub struct ScanReport { + pub progress: ScanProgress, + pub inserted: usize, + /// Files whose size or mtime moved: metadata re-read, thumbnail rebuilt, + /// content hash dropped. + pub updated: usize, + /// Files already catalogued and untouched. The overwhelming majority, and + /// each one costs nothing but a hash lookup. + pub unchanged: usize, + /// Rows removed because the file was *observed* to be gone. + pub images_removed: usize, + pub folders_removed: usize, + pub outcome: ScanOutcome, +} + +/// TRACES: FR-CAT-1 | FR-CAT-9 | NFR-P1 +/// Scan one granted root into the catalog. +/// +/// `cancel` is polled once per directory — FR-CAT-1 requires a scan the user +/// can stop, and a cancelled scan leaves valid partial state: everything +/// listed is committed, nothing is pruned. +/// +/// `now` is passed rather than read, so a test can assert on `added_at` and so +/// two rows written by one scan carry one timestamp. +/// +/// Returns a report rather than failing on an unreachable root. That is not +/// leniency: [`ScanOutcome`] is *how* the caller is told, it is what +/// [`ScanOutcome::may_prune`] consumes, and an `Err` here would discard the +/// counts of everything the scan did manage to do. A `CatalogError` means the +/// database itself is in trouble. +pub fn scan_root( + conn: &Connection, + storage: &dyn Storage, + root: RootId, + formats: &FormatFilter, + now: i64, + cancel: impl Fn() -> bool, + mut on_progress: impl FnMut(ScanProgress), +) -> Result { + let mut report = ScanReport::default(); + + // A fresh generation per *run*, taken and stored before anything is + // walked — not per completed scan. A cancelled run marks the folders it + // reached, and if the next run reused the number those marks would look + // current: a folder deleted in between would carry the generation the + // sweep is about to compare against, and would survive. + let generation = bump_generation(conn, root, now)?; + + let start = match storage.root_dir(root) { + Ok(d) => d, + // Not merely "no images": the grant is gone or the drive is out. Every + // image under it is unreachable, not deleted (FR-CAT-9). + Err(e) => { + log::warn!("scan: root {} unreachable: {e}", root.0); + mark_root_offline(conn, root)?; + report.outcome = ScanOutcome::RootUnreachable; + return Ok(report); + } + }; + + // Explicit stack rather than recursion, as in `dr_sync::scan`: a + // pathological tree must not overflow. Each item carries the catalog id of + // its parent folder, which is what threads the folder tree together. + let mut stack: Vec<(DirRef, usize, Option)> = vec![(start, 0, None)]; + let mut cancelled = false; + let mut partial = false; + + while let Some((dir, depth, parent)) = stack.pop() { + if cancel() { + log::info!("scan: cancelled at {dir}"); + cancelled = true; + break; + } + + // Excluded by name, before it is probed or listed. The root itself is + // exempt: a user who granted `.darkroom-trash` as their library meant + // it, and refusing to scan the folder they chose would look like the + // app doing nothing. + if !dir.is_root() && is_excluded(dir.name()) { + continue; + } + + if depth > MAX_DEPTH { + // Deeper folders exist and were not seen, so this scan cannot + // prove anything absent. + log::warn!("scan: depth limit at {dir}, not descending further"); + partial = true; + continue; + } + + let state = match storage.dir_state(&dir) { + Ok(s) => s, + Err(e) if dir.is_root() => { + log::warn!("scan: root {} unreachable: {e}", root.0); + mark_root_offline(conn, root)?; + report.outcome = ScanOutcome::RootUnreachable; + return Ok(report); + } + // One folder failed while the library as a whole is fine — a + // permission on a subdirectory, most likely. Its images stay in + // the catalog, and the sweep is disarmed for the whole run. + Err(e) => { + log::warn!("scan: {dir} could not be probed: {e}"); + partial = true; + continue; + } + }; + + // One transaction per directory. A single transaction for the whole + // scan would make a 50k-image first run all-or-nothing and unresumable; + // one per row would fsync 50k times. + let tx = conn.unchecked_transaction()?; + let (folder_id, stored) = touch_folder(&tx, root, &dir, parent, generation)?; + + match classify_dir(stored, state) { + // Unchanged. Its own contents need not be read, but a change in a + // *grandchild* moves no timestamp here — a filesystem propagates + // nothing upward — so the known children are still walked. + DirAction::RecurseOnly => { + report.progress.directories_pruned += 1; + for child in known_children(&tx, root, folder_id)? { + stack.push((child, depth + 1, Some(folder_id))); + } + tx.commit()?; + } + + DirAction::ListAndRecurse => { + let entries = match storage.list(&dir) { + Ok(e) => e, + Err(e) if dir.is_root() => { + drop(tx); + log::warn!("scan: root {} unreachable: {e}", root.0); + mark_root_offline(conn, root)?; + report.outcome = ScanOutcome::RootUnreachable; + return Ok(report); + } + Err(e) => { + // Rolled back rather than committed: the folder row's + // new generation would otherwise claim it was reached + // successfully. + drop(tx); + log::warn!("scan: {dir} could not be listed: {e}"); + partial = true; + continue; + } + }; + + report.progress.directories_listed += 1; + + // Emptied as the listing is walked, so whatever is left at the + // end is precisely what the folder no longer holds. Removing + // rather than marking keeps the sweep O(1) per file instead of + // a scan of the folder's rows per file. + let mut known = catalogued_images(&tx, folder_id)?; + + for entry in &entries { + match &entry.node { + Node::Dir(child) => stack.push((child.clone(), depth + 1, Some(folder_id))), + Node::File(src) => { + // Taken out of `known` whatever is decided next: + // this file was seen, and the sweep at the end is + // about the ones that were not. + let existing = known.remove(src.key()); + let action = + classify_entry(&entry.meta, existing.map(|e| e.as_known()), formats); + + // A format the filter no longer asks for, left + // exactly as it is — row and all. Unticking JPEG + // means stop looking for new ones, not delete the + // ones already found: the file is sitting right + // there, and absence is what justifies a delete. + if action == EntryAction::Ignored { + continue; + } + report.progress.images_found += 1; + + match action { + EntryAction::Unchanged => { + report.unchanged += 1; + // Unchanged in size and mtime, but perhaps + // not in reachability: a file that was + // marked offline while the drive was out is + // back, and nothing else would ever notice, + // because nothing about it has moved. + if let Some(row) = &existing { + restore_availability(&tx, row, src)?; + } + } + EntryAction::Insert => { + let id = insert_image( + &tx, + root, + folder_id, + src, + entry.meta.size, + entry.meta.mtime, + now, + )?; + queue_reading_it(&tx, id)?; + report.inserted += 1; + } + EntryAction::Changed => { + let id = update_image( + &tx, + root, + folder_id, + src, + entry.meta.size, + entry.meta.mtime, + )?; + queue_reading_it(&tx, id)?; + report.updated += 1; + } + EntryAction::Ignored => unreachable!("returned above"), + } + } + } + } + + // The file-level sweep, and the only place absence was + // observed: this folder was read, and these rows were not in + // it. Trashed images are exempt — their file has been moved + // into the trash on purpose, so being absent from the folder is + // the expected state, and deleting the row would lose the way + // back (FR-CAT-15). + for row in known.values() { + if row.trashed { + continue; + } + tx.execute("DELETE FROM images WHERE id = ?1", [row.id])?; + report.images_removed += 1; + } + + // Only now, with the listing read and reconciled, is the + // directory's state safe to record. Storing it before would let + // the next scan prune a folder whose contents were never + // actually seen, hiding every image beneath it permanently — + // the same trap `dr_sync::scan` documents for ETags. + tx.execute( + "UPDATE folders SET mtime = ?1, entry_count = ?2 WHERE id = ?3", + rusqlite::params![state.mtime, state.entry_count, folder_id], + )?; + tx.commit()?; + } + } + + on_progress(report.progress); + } + + report.outcome = if cancelled { + ScanOutcome::Cancelled + } else if partial { + ScanOutcome::PartialFailure + } else { + ScanOutcome::Complete + }; + + if report.outcome.may_prune() { + let (folders, images) = prune_unreached(conn, root, generation)?; + report.folders_removed = folders; + report.images_removed += images; + } + + Ok(report) +} + +/// Take the next scan generation and record it against the root. +fn bump_generation(conn: &Connection, root: RootId, now: i64) -> Result { + let current: Option = conn + .query_row( + "SELECT scan_generation FROM roots WHERE id = ?1", + [root.0 as i64], + |r| r.get(0), + ) + .optional()?; + let current = current.ok_or(CatalogError::NoSuchRoot(root.0))?; + + let next = current + 1; + conn.execute( + "UPDATE roots SET scan_generation = ?1, last_seen = ?2 WHERE id = ?3", + rusqlite::params![next, now, root.0 as i64], + )?; + Ok(next) +} + +/// TRACES: FR-CAT-9 +/// Mark every image under a root as unreachable. +/// +/// The other half of FR-CAT-9's distinction: a source *proven absent* may leave +/// the catalog, a source merely *unreachable* is marked offline and keeps its +/// ratings and its edits. The grid then says "offline" rather than showing a +/// library that has silently lost half its photographs. +/// +/// The folders forget what they last looked like, which costs a full re-listing +/// when the drive comes back. That is the price of the marking: every row under +/// the root now claims something about the files that is no longer known to be +/// true, and only reading the directories again can settle it. Pruning would +/// skip them all and leave a plugged-in library showing as offline forever. +fn mark_root_offline(conn: &Connection, root: RootId) -> Result<(), CatalogError> { + let root_id = root.0 as i64; + conn.execute( + "UPDATE images SET availability = ?1 WHERE root_id = ?2 AND availability != ?1", + rusqlite::params![availability_code(Availability::Offline), root_id], + )?; + conn.execute( + "UPDATE folders SET mtime = NULL, entry_count = NULL WHERE root_id = ?1", + [root_id], + )?; + Ok(()) +} + +/// Record that a folder was reached in this generation, and report what the +/// last scan saw there. +/// +/// The generation is written on arrival, before the folder is read: it means +/// "reached", which is what the end-of-scan sweep asks about. The `mtime` and +/// `entry_count` that mean "read" are written separately, and only afterwards. +fn touch_folder( + conn: &Connection, + root: RootId, + dir: &DirRef, + parent: Option, + generation: i64, +) -> Result<(i64, Option), CatalogError> { + conn.execute( + "INSERT INTO folders(root_id, parent_id, path, scanned_generation) + VALUES (?1, ?2, ?3, ?4) + ON CONFLICT(root_id, path) DO UPDATE SET + scanned_generation = excluded.scanned_generation, + parent_id = excluded.parent_id", + rusqlite::params![root.0 as i64, parent, dir.key(), generation], + )?; + + let (id, mtime, count): (i64, Option, Option) = conn.query_row( + "SELECT id, mtime, entry_count FROM folders WHERE root_id = ?1 AND path = ?2", + rusqlite::params![root.0 as i64, dir.key()], + |r| Ok((r.get(0)?, r.get(1)?, r.get(2)?)), + )?; + + // Both or neither: a folder recorded by some other path (a remote scan + // storing an ETag, say) has no local state, and half of one must not be + // read as "unchanged since 1970". + let stored = match (mtime, count) { + (Some(mtime), Some(count)) => Some(DirState { + mtime, + entry_count: count as u32, + }), + _ => None, + }; + Ok((id, stored)) +} + +/// The subfolders the catalog already knows about, as references to walk. +/// +/// This is what makes pruning safe on a filesystem. A directory's mtime moves +/// when its own entries change and not when a grandchild's do, so an unchanged +/// folder says nothing about the tree below it; the stored keys are how the +/// walk carries on downward without listing. +fn known_children( + conn: &Connection, + root: RootId, + folder_id: i64, +) -> Result, CatalogError> { + let mut stmt = conn.prepare("SELECT path FROM folders WHERE parent_id = ?1")?; + let rows = stmt.query_map([folder_id], |r| r.get::<_, String>(0))?; + Ok(rows + .flatten() + .map(|path| DirRef::from_parts(root, path)) + .collect()) +} + +/// What the catalog holds for one image, as the scan needs it. +#[derive(Clone, Copy)] +struct Catalogued { + id: i64, + size: Option, + mtime: Option, + availability: i64, + trashed: bool, +} + +impl Catalogued { + /// What [`classify_entry`] compares against. + /// + /// A row with no recorded size or mtime — inserted by a remote scan, or by + /// an older build — is given values no real file can match, so it is + /// classified `Changed` and re-read. The alternative, treating it as + /// unknown, would insert a duplicate. + fn as_known(&self) -> KnownFile { + KnownFile { + size: self.size.unwrap_or(u64::MAX), + mtime: self.mtime.unwrap_or(i64::MIN), + } + } +} + +/// Every image the catalog has in one folder, keyed by its stored reference. +/// +/// Read once per folder rather than queried per file: a folder of 2,000 frames +/// is one statement and one hash lookup each, not 2,000 indexed selects. +fn catalogued_images( + conn: &Connection, + folder_id: i64, +) -> Result, CatalogError> { + let mut stmt = conn.prepare( + "SELECT id, source_ref, file_size, file_mtime, availability, trashed_at + FROM images WHERE folder_id = ?1", + )?; + let rows = stmt.query_map([folder_id], |r| { + Ok(( + r.get::<_, String>(1)?, + Catalogued { + id: r.get(0)?, + size: r.get::<_, Option>(2)?.map(|v| v as u64), + mtime: r.get(3)?, + availability: r.get(4)?, + trashed: r.get::<_, Option>(5)?.is_some(), + }, + )) + })?; + Ok(rows.flatten().collect()) +} + +/// Put an unchanged file's availability back where the evidence says it should +/// be, and write nothing if it is already there. +/// +/// The write has to be conditional: an unchanged file is the overwhelmingly +/// common case, and an `UPDATE` per image per scan would make a no-op rescan +/// write the whole library. +fn restore_availability( + conn: &Connection, + row: &Catalogued, + src: &SourceRef, +) -> Result<(), CatalogError> { + let should_be = availability_code(availability_of(src)); + if row.availability == should_be { + return Ok(()); + } + conn.execute( + "UPDATE images SET availability = ?1 WHERE id = ?2", + rusqlite::params![should_be, row.id], + )?; + Ok(()) +} + +/// How much of this image is on hand. +/// +/// A local file is the original — unless it is a Nextcloud VFS placeholder, a +/// one-byte stub standing in for a dehydrated file whose bytes are on the +/// server (ARCH §9.0). Reading such a stub does not trigger a fetch on Linux, +/// so calling it `Original` would promise pixels that are not there. Catalogued +/// as the image it stands for, marked offline: honest, and FR-NC-6c's whole +/// point. +fn availability_of(src: &SourceRef) -> Availability { + if src.is_placeholder() { + Availability::Offline + } else { + Availability::Original + } +} + +fn insert_image( + conn: &Connection, + root: RootId, + folder_id: i64, + src: &SourceRef, + size: u64, + mtime: i64, + now: i64, +) -> Result { + // `metadata_state = 1`: the scan knows the name, the size and the mtime, + // and has read no EXIF. Claiming otherwise would make a date filter + // silently wrong on a freshly scanned library. + conn.execute( + "INSERT INTO images(root_id, folder_id, source_ref, format, file_size, file_mtime, + availability, metadata_state, added_at) + VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, 1, ?8) + ON CONFLICT(root_id, source_ref) DO UPDATE SET + folder_id = excluded.folder_id, + file_size = excluded.file_size, + file_mtime = excluded.file_mtime, + availability = excluded.availability", + rusqlite::params![ + root.0 as i64, + folder_id, + src.key(), + src.extension(), + size as i64, + mtime, + availability_code(availability_of(src)), + now, + ], + )?; + image_id(conn, root, src) +} + +fn update_image( + conn: &Connection, + root: RootId, + folder_id: i64, + src: &SourceRef, + size: u64, + mtime: i64, +) -> Result { + // The content hash is dropped, not recomputed: it described bytes that no + // longer exist, and leaving it would let reconnect-by-hash match this image + // to a file it is no longer a copy of. `metadata_state` goes back to 1 for + // the same reason — the EXIF on record was read from the old file. + conn.execute( + "UPDATE images SET folder_id = ?1, file_size = ?2, file_mtime = ?3, + content_hash = NULL, metadata_state = 1, availability = ?4 + WHERE root_id = ?5 AND source_ref = ?6", + rusqlite::params![ + folder_id, + size as i64, + mtime, + availability_code(availability_of(src)), + root.0 as i64, + src.key(), + ], + )?; + image_id(conn, root, src) +} + +fn image_id(conn: &Connection, root: RootId, src: &SourceRef) -> Result { + Ok(conn.query_row( + "SELECT id FROM images WHERE root_id = ?1 AND source_ref = ?2", + rusqlite::params![root.0 as i64, src.key()], + |r| r.get(0), + )?) +} + +/// Queue the work that turns a stat-only row into a usable grid cell. +/// +/// Enqueued inside the scan's transaction, so a folder's rows and the jobs that +/// finish them land together — a crash between the two would otherwise leave +/// images no worker was ever told about. +fn queue_reading_it(conn: &Connection, image_id: i64) -> Result<(), CatalogError> { + jobs::enqueue( + conn, + JobKind::ExtractMetadata, + Some(image_id), + Priority::Background, + None, + )?; + jobs::enqueue( + conn, + JobKind::Thumbnail, + Some(image_id), + Priority::Background, + None, + ) +} + +/// TRACES: FR-CAT-9 +/// Delete the folders this scan did not reach, and the images inside them. +/// +/// Runs only after a [`ScanOutcome::Complete`] scan, which is the guard that +/// stops an unplugged drive from taking the library with it. +/// +/// Trashed images are detached first. Their folder may well be one of the ones +/// vanishing — the user deleted the whole directory — and the cascade would +/// take the row with it, orphaning a file sitting in the trash with no way back +/// (FR-CAT-15). +fn prune_unreached( + conn: &Connection, + root: RootId, + generation: i64, +) -> Result<(usize, usize), CatalogError> { + const STALE: &str = "SELECT id FROM folders WHERE root_id = ?1 AND scanned_generation < ?2"; + + let tx = conn.unchecked_transaction()?; + let root_id = root.0 as i64; + let params = rusqlite::params![root_id, generation]; + + tx.execute( + &format!( + "UPDATE images SET folder_id = NULL + WHERE trashed_at IS NOT NULL AND folder_id IN ({STALE})" + ), + params, + )?; + + let images: i64 = tx.query_row( + &format!("SELECT count(*) FROM images WHERE folder_id IN ({STALE})"), + params, + |r| r.get(0), + )?; + let folders = tx.execute( + "DELETE FROM folders WHERE root_id = ?1 AND scanned_generation < ?2", + params, + )?; + tx.commit()?; + + Ok((folders, images as usize)) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::Catalog; + use dr_plat::LocalStorage; + use dr_types::Format; + use std::cell::Cell; + use std::fs; + use std::path::PathBuf; + + /// A throwaway library on disk, removed when the test ends. + /// + /// The real [`LocalStorage`] rather than a fake, because what most needs + /// protecting here is the seam between the two crates: a fake would agree + /// with whatever this module expects, which is exactly the mistake a scan + /// against a real directory cannot make. + struct Library { + dir: PathBuf, + catalog: Catalog, + root: RootId, + } + + impl Library { + fn new(name: &str) -> Self { + let dir = std::env::temp_dir().join(format!( + "dr-walk-{name}-{}-{:?}", + std::process::id(), + std::thread::current().id() + )); + let _ = fs::remove_dir_all(&dir); + fs::create_dir_all(&dir).expect("temp dir"); + + let catalog = Catalog::in_memory().expect("catalog"); + let root = ensure_root( + catalog.connection(), + RootKind::Local, + &dir.display().to_string(), + ) + .expect("root"); + + Library { dir, catalog, root } + } + + fn file(&self, rel: &str, bytes: &[u8]) -> &Self { + let p = self.dir.join(rel); + if let Some(parent) = p.parent() { + fs::create_dir_all(parent).expect("mkdir"); + } + fs::write(p, bytes).expect("write"); + self + } + + /// Overwrite a file the way a program that cares does: write beside it, + /// then rename over the top. + /// + /// Not incidental to the test — it is *why* the change is visible. A + /// plain write moves nothing about the directory, and the scan would + /// never look (see the module docs). + fn resave(&self, rel: &str, bytes: &[u8]) -> &Self { + let target = self.dir.join(rel); + let tmp = target.with_extension("tmp"); + fs::write(&tmp, bytes).expect("write"); + fs::rename(&tmp, &target).expect("rename"); + self + } + + fn remove(&self, rel: &str) -> &Self { + let p = self.dir.join(rel); + if p.is_dir() { + fs::remove_dir_all(p).expect("rmdir"); + } else { + fs::remove_file(p).expect("rm"); + } + self + } + + fn storage(&self) -> LocalStorage { + LocalStorage::with_root(self.root, self.dir.clone()) + } + + fn conn(&self) -> &Connection { + self.catalog.connection() + } + + fn scan(&self) -> ScanReport { + self.scan_with(&FormatFilter::all(), &self.storage(), || false) + } + + fn scan_with( + &self, + formats: &FormatFilter, + storage: &dyn Storage, + cancel: impl Fn() -> bool, + ) -> ScanReport { + scan_root( + self.conn(), + storage, + self.root, + formats, + 1_000, + cancel, + |_| {}, + ) + .expect("scan") + } + + fn names(&self) -> Vec { + let mut stmt = self + .conn() + .prepare("SELECT source_ref FROM images ORDER BY source_ref") + .unwrap(); + let rows = stmt.query_map([], |r| r.get::<_, String>(0)).unwrap(); + rows.flatten().collect() + } + + fn count(&self, sql: &str) -> i64 { + self.conn().query_row(sql, [], |r| r.get(0)).unwrap() + } + } + + impl Drop for Library { + fn drop(&mut self) { + let _ = fs::remove_dir_all(&self.dir); + } + } + + #[test] + fn a_folder_of_photographs_becomes_a_catalogued_library() { + // The whole point of the exercise: before this existed, no image + // reached the catalog without a Nextcloud account. + let lib = Library::new("first-scan"); + lib.file("2026/08/IMG_0001.CR3", b"raw") + .file("2026/08/IMG_0002.CR3", b"raw") + .file("2025/IMG_0003.NEF", b"raw") + .file("2025/notes.txt", b"not a photograph"); + + let r = lib.scan(); + + assert_eq!(r.outcome, ScanOutcome::Complete); + assert_eq!(r.inserted, 3); + assert_eq!( + lib.names(), + vec![ + "2025/IMG_0003.NEF", + "2026/08/IMG_0001.CR3", + "2026/08/IMG_0002.CR3" + ] + ); + } + + #[test] + fn a_scanned_image_is_addressed_by_reference_not_by_path() { + // FR-CAT-1a. The stored reference is relative to the root, so the + // library can be moved, or reached through SAF where no path exists at + // all, without rewriting a single row. + let lib = Library::new("relative"); + lib.file("2026/IMG.CR3", b"raw"); + lib.scan(); + + let stored = &lib.names()[0]; + assert_eq!(stored, "2026/IMG.CR3"); + assert!( + !stored.contains(&lib.dir.display().to_string()), + "an absolute path leaked into the catalog: {stored}" + ); + } + + #[test] + fn a_rescan_of_an_unchanged_library_reads_no_directory() { + // NFR-P1 in one assertion. If this fails, every rescan is a full walk + // and a 50k-image library takes minutes instead of seconds. + let lib = Library::new("prune"); + lib.file("2026/08/IMG_0001.CR3", b"raw") + .file("2025/IMG_0003.NEF", b"raw"); + + let first = lib.scan(); + assert_eq!(first.progress.directories_listed, 4); // root, 2026, 2026/08, 2025 + + let second = lib.scan(); + assert_eq!(second.progress.directories_listed, 0); + assert_eq!(second.progress.directories_pruned, 4); + assert_eq!(second.inserted, 0); + assert_eq!(second.images_removed, 0); + assert_eq!(second.outcome, ScanOutcome::Complete); + } + + #[test] + fn a_change_under_an_unchanged_parent_is_still_found() { + // A filesystem propagates nothing upward: adding a file to `2026/08` + // leaves `2026` and the root untouched. A scan that pruned at the first + // unchanged folder would never see the new photograph — which is why + // the walk carries on into known children rather than stopping. + let lib = Library::new("grandchild"); + lib.file("2026/08/IMG_0001.CR3", b"raw"); + lib.scan(); + + lib.file("2026/08/IMG_0002.CR3", b"raw"); + let r = lib.scan(); + + assert_eq!(r.inserted, 1); + assert_eq!(r.progress.directories_pruned, 2, "root and 2026"); + assert_eq!(r.progress.directories_listed, 1, "2026/08"); + } + + #[test] + fn a_deleted_file_leaves_the_catalog() { + let lib = Library::new("deleted"); + lib.file("a.CR3", b"raw").file("b.CR3", b"raw"); + lib.scan(); + + lib.remove("b.CR3"); + let r = lib.scan(); + + assert_eq!(r.images_removed, 1); + assert_eq!(lib.names(), vec!["a.CR3"]); + } + + #[test] + fn a_deleted_folder_takes_its_images_with_it() { + let lib = Library::new("deleted-folder"); + lib.file("2025/a.CR3", b"raw").file("2026/b.CR3", b"raw"); + lib.scan(); + + lib.remove("2025"); + let r = lib.scan(); + + assert_eq!(r.folders_removed, 1); + assert_eq!(lib.names(), vec!["2026/b.CR3"]); + } + + #[test] + fn an_unreachable_root_marks_images_offline_and_deletes_nothing() { + // The most expensive mistake this code could make. Every folder looks + // unreached when a drive is unplugged, so a sweep would take the entire + // library — ratings, flags and edits included (FR-CAT-9). + let lib = Library::new("unplugged"); + lib.file("2026/IMG.CR3", b"raw"); + lib.scan(); + + let unplugged = LocalStorage::with_root(lib.root, lib.dir.join("gone")); + let r = lib.scan_with(&FormatFilter::all(), &unplugged, || false); + + assert_eq!(r.outcome, ScanOutcome::RootUnreachable); + assert!(!r.outcome.may_prune()); + assert_eq!(lib.names().len(), 1, "the library was deleted"); + assert_eq!( + lib.count("SELECT availability FROM images"), + availability_code(Availability::Offline) + ); + } + + #[test] + fn a_root_that_comes_back_is_available_again() { + // The other half: a drive plugged back in must return the library to + // usable, not leave it permanently marked offline. + let lib = Library::new("replugged"); + lib.file("IMG.CR3", b"raw"); + lib.scan(); + + let unplugged = LocalStorage::with_root(lib.root, lib.dir.join("gone")); + lib.scan_with(&FormatFilter::all(), &unplugged, || false); + lib.scan(); + + assert_eq!( + lib.count("SELECT availability FROM images"), + availability_code(Availability::Original) + ); + } + + #[test] + fn a_cancelled_scan_deletes_nothing() { + // Partial state is valid — jobs are resumable — but the folders that + // were never visited must not be read as absent. + let lib = Library::new("cancelled"); + lib.file("2025/a.CR3", b"raw") + .file("2026/b.CR3", b"raw") + .file("2027/c.CR3", b"raw"); + lib.scan(); + + // Stop after the first directory. + let seen = Cell::new(0); + let r = lib.scan_with(&FormatFilter::all(), &lib.storage(), || { + let n = seen.get(); + seen.set(n + 1); + n > 0 + }); + + assert_eq!(r.outcome, ScanOutcome::Cancelled); + assert_eq!(r.images_removed, 0); + assert_eq!(r.folders_removed, 0); + assert_eq!(lib.names().len(), 3); + } + + #[test] + fn a_resaved_file_is_queued_for_rereading_and_loses_its_stale_hash() { + let lib = Library::new("resaved"); + lib.file("IMG.CR3", b"raw"); + lib.scan(); + + lib.conn() + .execute( + "UPDATE images SET content_hash = 'abc', metadata_state = 2", + [], + ) + .unwrap(); + + // Same name, different bytes: an export written over the original, or + // a file restored from a backup. Both arrive by rename, which is what + // makes them visible to an incremental scan. + lib.resave("IMG.CR3", b"different bytes entirely"); + let r = lib.scan(); + + assert_eq!(r.updated, 1); + assert_eq!( + lib.count("SELECT count(*) FROM images WHERE content_hash IS NULL"), + 1, + "a hash of bytes that no longer exist would match this image to the \ + wrong file on reconnect" + ); + assert_eq!(lib.count("SELECT metadata_state FROM images"), 1); + assert_eq!( + lib.count("SELECT count(*) FROM jobs WHERE kind = 1"), + 1, + "EXIF must be re-read" + ); + } + + #[test] + fn a_file_that_has_not_moved_is_not_requeued_when_its_folder_is_reread() { + // Adding one photograph to a folder of two thousand must cost one + // thumbnail, not two thousand. Without this, every import rebuilds + // everything around it. + let lib = Library::new("no-requeue"); + lib.file("a.CR3", b"raw"); + lib.scan(); + lib.conn().execute("DELETE FROM jobs", []).unwrap(); + + lib.file("b.CR3", b"raw"); + let r = lib.scan(); + + assert_eq!(r.inserted, 1); + assert_eq!(r.unchanged, 1); + assert_eq!( + lib.count("SELECT count(*) FROM jobs"), + 2, + "EXIF and a thumbnail for the new image, and nothing for the old one" + ); + } + + #[test] + fn a_file_rewritten_in_place_is_not_noticed_until_its_folder_changes() { + // The documented limit of directory-level pruning, held by a test so it + // stays a known trade and does not become a surprise. An in-place write + // moves neither the directory's mtime nor its entry count, so nothing + // says to look — and a scan that looked anyway would be a full walk of + // the library every time. + let lib = Library::new("in-place"); + lib.file("IMG.CR3", b"raw"); + lib.scan(); + + fs::write(lib.dir.join("IMG.CR3"), b"rewritten in place, same folder").unwrap(); + assert_eq!(lib.scan().updated, 0); + + // Anything that touches the folder brings it back into view. + lib.file("other.CR3", b"raw"); + assert_eq!(lib.scan().updated, 1); + } + + #[test] + fn a_new_image_is_queued_for_a_thumbnail_and_for_exif() { + let lib = Library::new("queued"); + lib.file("IMG.CR3", b"raw"); + lib.scan(); + + assert_eq!( + lib.count("SELECT count(*) FROM jobs WHERE kind = 2"), + 1, + "no thumbnail job means an empty grid cell forever" + ); + assert_eq!(lib.count("SELECT count(*) FROM jobs WHERE kind = 1"), 1); + assert_eq!( + lib.count("SELECT metadata_state FROM images"), + 1, + "the scan read no EXIF and must not claim to have" + ); + } + + #[test] + fn the_format_filter_decides_what_is_catalogued() { + let lib = Library::new("formats"); + lib.file("IMG.CR3", b"raw") + .file("IMG.NEF", b"raw") + .file("IMG.JPG", b"jpeg"); + + let r = lib.scan_with( + &FormatFilter::from_formats([Format::Cr3]), + &lib.storage(), + || false, + ); + assert_eq!(r.inserted, 1); + assert_eq!(lib.names(), vec!["IMG.CR3"]); + } + + #[test] + fn narrowing_the_filter_stops_finding_files_it_does_not_delete_them() { + // Unticking JPEG is a statement about what to look for, not an + // instruction to discard work. The files are sitting in the folder, + // and absence is the only thing that justifies deleting a row — so a + // tick-box must not quietly take a hundred rated photographs with it. + let lib = Library::new("filter-narrowed"); + lib.file("IMG.CR3", b"raw").file("IMG.JPG", b"jpeg"); + lib.scan(); + assert_eq!(lib.names().len(), 2); + + // Something has to change for the folder to be listed at all. + lib.file("NEW.CR3", b"raw"); + lib.scan_with(&FormatFilter::raw_only(), &lib.storage(), || false); + + assert!( + lib.names().contains(&"IMG.JPG".to_string()), + "a file that is still there was deleted by a filter change" + ); + } + + #[test] + fn a_dehydrated_placeholder_is_catalogued_as_the_image_it_stands_for() { + // A Nextcloud-synced folder scanned as a local library is the common + // case, and 121,785 of these exist in a real one (ARCH §9.0). Each is a + // one-byte stub; reading it fetches nothing on Linux. Catalogued as a + // CR2 and marked offline, so the grid says so rather than showing a + // decode failure. + let lib = Library::new("placeholder"); + lib.file("_MG_4130.CR2.nextcloud", b"\0") + .file("_MG_4131.CR2", b"real bytes"); + lib.scan(); + + assert_eq!( + lib.count("SELECT availability FROM images WHERE source_ref LIKE '%.nextcloud'"), + availability_code(Availability::Offline) + ); + assert_eq!( + lib.count("SELECT count(*) FROM images WHERE format = 'cr2'"), + 2, + "the stub is a CR2, not an unknown '.nextcloud' type" + ); + assert_eq!( + lib.count("SELECT availability FROM images WHERE source_ref = '_MG_4131.CR2'"), + availability_code(Availability::Original) + ); + } + + #[test] + fn the_trash_folder_is_never_scanned() { + // The other half of the soft delete. A scan that walked the trash would + // re-index every trashed photograph as an ordinary one, undoing the + // delete on the next refresh (FR-CAT-15). + let lib = Library::new("trash"); + lib.file("a.CR3", b"raw") + .file(&format!("{TRASH_DIR}/1-b.CR3"), b"raw") + .file(&format!("{DERIVED_DIR}/thumbs.sqlite"), b"x"); + lib.scan(); + + assert_eq!(lib.names(), vec!["a.CR3"]); + assert_eq!( + lib.count(&format!( + "SELECT count(*) FROM folders WHERE path LIKE '{TRASH_DIR}%'" + )), + 0, + "the trash must cost nothing, not merely be filtered out" + ); + } + + #[test] + fn a_trashed_image_survives_a_rescan_of_the_folder_it_left() { + // Its file has been moved into the trash on purpose, so being absent + // from its old folder is the expected state. Deleting the row would + // lose `trashed_from` and with it the only way back (FR-CAT-15). + let lib = Library::new("trashed-row"); + lib.file("2026/a.CR3", b"raw").file("2026/b.CR3", b"raw"); + lib.scan(); + + // Trash `b` the way the UI does: move the file, then record. + fs::create_dir_all(lib.dir.join(TRASH_DIR)).unwrap(); + fs::rename( + lib.dir.join("2026/b.CR3"), + lib.dir.join(format!("{TRASH_DIR}/2-b.CR3")), + ) + .unwrap(); + lib.conn() + .execute( + "UPDATE images SET trashed_at = 100, trashed_from = source_ref, + source_ref = ?1 + WHERE source_ref = '2026/b.CR3'", + [format!("{TRASH_DIR}/2-b.CR3")], + ) + .unwrap(); + + lib.scan(); + + assert_eq!( + lib.count("SELECT count(*) FROM images WHERE trashed_at IS NOT NULL"), + 1, + "the trashed image was swept away by a rescan" + ); + } + + #[test] + fn a_trashed_image_outlives_the_folder_it_came_from() { + // Same rule, reached through the folder sweep instead: the user deleted + // the whole directory, and the cascade would have taken a row whose + // file is sitting safely in the trash. + let lib = Library::new("trashed-orphan"); + lib.file("2026/a.CR3", b"raw"); + lib.scan(); + + fs::create_dir_all(lib.dir.join(TRASH_DIR)).unwrap(); + fs::rename( + lib.dir.join("2026/a.CR3"), + lib.dir.join(format!("{TRASH_DIR}/1-a.CR3")), + ) + .unwrap(); + lib.conn() + .execute( + "UPDATE images SET trashed_at = 100, trashed_from = source_ref, + source_ref = ?1 + WHERE source_ref = '2026/a.CR3'", + [format!("{TRASH_DIR}/1-a.CR3")], + ) + .unwrap(); + lib.remove("2026"); + + lib.scan(); + + assert_eq!( + lib.count("SELECT count(*) FROM images WHERE trashed_at IS NOT NULL"), + 1 + ); + } + + #[test] + fn scanning_twice_does_not_fragment_the_library_across_two_roots() { + // `ensure_root` is idempotent because a second row would split the + // images between two roots and compare each half against the wrong scan + // generation. + let lib = Library::new("one-root"); + let again = + ensure_root(lib.conn(), RootKind::Local, &lib.dir.display().to_string()).unwrap(); + + assert_eq!(again, lib.root); + assert_eq!(lib.count("SELECT count(*) FROM roots"), 1); + } + + #[test] + fn every_run_takes_a_fresh_generation() { + // Not "once per completed scan": a cancelled run marks the folders it + // reached, and reusing the number would let those marks look current to + // the next sweep — so a folder deleted in between would survive it. + let lib = Library::new("generations"); + lib.file("a.CR3", b"raw"); + + lib.scan_with(&FormatFilter::all(), &lib.storage(), || true); + let after_cancel = lib.count("SELECT scan_generation FROM roots"); + lib.scan(); + assert!(lib.count("SELECT scan_generation FROM roots") > after_cancel); + } + + #[test] + fn a_scan_reports_progress_as_it_goes() { + // A first scan of a large library is minutes long; without this the UI + // has nothing to say for all of it. + let lib = Library::new("progress"); + lib.file("2025/a.CR3", b"raw").file("2026/b.CR3", b"raw"); + + let mut updates = Vec::new(); + scan_root( + lib.conn(), + &lib.storage(), + lib.root, + &FormatFilter::all(), + 1_000, + || false, + |p| updates.push(p), + ) + .unwrap(); + + assert_eq!(updates.len(), 3, "one per directory visited"); + assert_eq!(updates.last().unwrap().images_found, 2); + } + + #[test] + fn scanning_a_root_with_no_catalog_row_is_a_typed_error() { + // Rather than inventing a root: the label is the grant, and only the + // caller that obtained the grant knows how to spell it. + let lib = Library::new("no-root"); + let err = scan_root( + lib.conn(), + &lib.storage(), + RootId(999), + &FormatFilter::all(), + 0, + || false, + |_| {}, + ); + assert!(matches!(err, Err(CatalogError::NoSuchRoot(999)))); + } + + #[test] + fn the_excluded_directories_match_the_ones_the_remote_scanner_excludes() { + // Two pairs of constants in two crates that must agree, or a soft + // delete comes undone on whichever side disagrees. + assert!(is_excluded(TRASH_DIR)); + assert!(is_excluded(DERIVED_DIR)); + assert_eq!(DERIVED_DIR, dr_sync_derived_dir()); + assert!(!is_excluded("2026")); + assert!(!is_excluded(".darkroom-trash-old")); + } + + /// The scanner's constant, quoted rather than imported — `dr-catalog` does + /// not depend on `dr-sync`, and adding that dependency for one string would + /// invert the layering. + fn dr_sync_derived_dir() -> &'static str { + ".darkroom-derived" + } +} diff --git a/core/dr-types/src/lib.rs b/core/dr-types/src/lib.rs index 3fbeea9..328c43e 100644 --- a/core/dr-types/src/lib.rs +++ b/core/dr-types/src/lib.rs @@ -63,16 +63,25 @@ pub enum SourceRef { impl SourceRef { /// A stable, display-friendly name — the last path component. pub fn display_name(&self) -> &str { - let full = match self { - SourceRef::Local { relative, .. } => relative.as_str(), - SourceRef::Document { document_id, .. } => document_id.as_str(), - SourceRef::Remote { path, .. } => path.as_str(), - }; + let full = self.key(); // SAF document ids use ':' as a separator; paths use '/'. Split on // whichever appears last so both yield a sensible name. full.rsplit(['/', ':']).next().unwrap_or(full) } + /// The stored form — what the catalog holds in `images.source_ref`. + /// + /// Which of the three variants a key belongs to is not recorded beside it, + /// because the root already says: `roots.kind` is `'local' | 'saf' | + /// 'remote'`, and every image names its root. + pub fn key(&self) -> &str { + match self { + SourceRef::Local { relative, .. } => relative.as_str(), + SourceRef::Document { document_id, .. } => document_id.as_str(), + SourceRef::Remote { path, .. } => path.as_str(), + } + } + /// Lowercase file extension, if any. /// /// Looks through a Nextcloud VFS placeholder suffix, so a dehydrated @@ -251,6 +260,55 @@ impl FormatFilter { } } +/// TRACES: FR-CAT-1 | NFR-PORT-1 +/// What a directory looked like when last scanned, and what it looks like now. +/// +/// Both fields are cheap to obtain: one `stat` plus a name-only directory read +/// locally, one `DocumentsContract` query on SAF. Neither costs a `stat` per +/// child, which is the whole point — see `dr_catalog::scan`. +/// +/// Lives here rather than in either crate that uses it because it is the +/// sentence the platform layer says to the catalog: `dr-plat` produces it by +/// probing, `dr-catalog` stores it and compares. A copy on each side would be +/// two types that must agree by convention. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct DirState { + /// Milliseconds since the epoch. **Not seconds**, and the difference is not + /// pedantry: change detection asks whether a timestamp *moved*, so the + /// granularity of the unit is the width of the window in which a change is + /// invisible. A second is long enough to copy a card and start a scan, and + /// anything happening inside one tick of the stored value looks exactly + /// like nothing happening — the directory is pruned and the photographs + /// never appear. + /// + /// Both platforms can supply it: a filesystem records nanoseconds, and + /// SAF's `COLUMN_LAST_MODIFIED` is already in milliseconds. + pub mtime: i64, + /// Direct children, files and directories alike. + /// + /// mtime alone misses a delete-and-create inside one timestamp tick, and + /// coarse-granularity providers widen that window. The count does not + /// close the hole — a paired add and remove moves neither — but a bare add + /// or remove moves the count, and those are far commoner. + pub entry_count: u32, +} + +/// TRACES: FR-CAT-1 | NFR-PORT-1 +/// One entry from a directory listing, as the scanner classifies it. +/// +/// Deliberately carries no handle to the entry itself: this is what the +/// *decision* logic reads, and it must be constructible in a test without a +/// filesystem. The platform pairs it with the reference needed to reach the +/// entry (`dr_plat::Entry`). +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct DirEntry { + pub name: String, + pub is_dir: bool, + pub size: u64, + /// Milliseconds since the epoch, as in [`DirState::mtime`]. + pub mtime: i64, +} + /// How much of an image is available locally (FR-NC-6c). /// /// Surfaced in the UI so a user always knows what they have — the failure diff --git a/platform/dr-plat/src/lib.rs b/platform/dr-plat/src/lib.rs index f0151cf..a889172 100644 --- a/platform/dr-plat/src/lib.rs +++ b/platform/dr-plat/src/lib.rs @@ -5,7 +5,9 @@ //! ARCH §10). pub mod secrets; +pub mod storage; pub use secrets::{ EphemeralSecretStore, PlatformSecretStore, SecretError, SecretKind, SecretRef, SecretStore, }; +pub use storage::{DirRef, Entry, LocalStorage, Node, SeekableRead, Storage, StorageError}; diff --git a/platform/dr-plat/src/storage.rs b/platform/dr-plat/src/storage.rs new file mode 100644 index 0000000..9817646 --- /dev/null +++ b/platform/dr-plat/src/storage.rs @@ -0,0 +1,861 @@ +//! TRACES: FR-CAT-1 | FR-CAT-1a | NFR-PORT-1 | NFR-PORT-3 +//! Reaching stored bytes without naming a path (ARCH §3.1, §10). +//! +//! Android's Storage Access Framework hands out no filesystem path (ARCH §6.9), +//! so **nothing above this module may take one**. A library location is a +//! [`RootId`] the user granted; everything inside it is a [`DirRef`] or a +//! [`SourceRef`], both of which are opaque `(root, key)` pairs whose keys only +//! the implementation that produced them knows how to read. +//! +//! A `Path` therefore appears exactly once in the whole application: at +//! [`LocalStorage::grant`], where the folder the user picked is handed in. From +//! there on it is a `RootId`. +//! +//! # Adding Android SAF later +//! +//! It is a second implementation of [`Storage`] and no change at any call site. +//! Two properties of this API are what buy that, and both look like ceremony +//! until SAF is the thing being written: +//! +//! - **A listing hands back references, never names for the caller to join.** +//! A SAF document id is not composable — `parent_id + "/" + name` is not the +//! child's id, and the only way to learn a child's id is the children query +//! that produced the listing. So [`Entry`] carries the [`DirRef`] or +//! [`SourceRef`] the provider itself returned, and no caller ever builds one +//! by concatenation. [`LocalStorage`] could perfectly well have exposed a +//! "join a name onto a directory" helper; that helper is the one thing a SAF +//! implementation could not have provided. +//! - **A reference is a key that survives a restart.** The catalog stores the +//! key and rebuilds the reference with [`DirRef::from_parts`] on the next +//! run. On Linux the key is a relative path; on SAF it is a document id under +//! a persisted tree grant, which is re-resolvable for exactly the same +//! reason. +//! +//! What SAF will need in addition is the grant itself — the persisted tree URI, +//! which the catalog's `roots.grant_blob` column already has a home for, and +//! which is handed to the implementation at construction just as a path is +//! here. + +use std::collections::BTreeMap; +use std::fmt; +use std::io::{Read, Seek}; +use std::path::{Component, Path, PathBuf}; + +use dr_types::{ByteRange, DirEntry, DirState, RootId, SourceRef}; + +/// TRACES: FR-CAT-1a +/// An opaque, re-resolvable reference to a *directory* under a granted root. +/// +/// The counterpart of [`SourceRef`], which addresses a file. One type rather +/// than a mirrored three-variant enum because a directory is never resolved by +/// anything except the storage that owns its root: the root's kind already +/// determines how the key is read, so a discriminator on each reference would +/// only repeat it. +/// +/// The `key` is **opaque to callers** and stable across restarts. Its meaning +/// belongs to the implementation — a relative path under the root on a +/// filesystem, a `DocumentsContract` document id on SAF — and the catalog +/// stores it verbatim in `folders.path` so a later run can rebuild the +/// reference with [`DirRef::from_parts`]. +#[derive(Debug, Clone, PartialEq, Eq, Hash, PartialOrd, Ord)] +pub struct DirRef { + root: RootId, + key: String, +} + +impl DirRef { + /// The granted root itself, which every walk starts from. + pub fn root(root: RootId) -> Self { + Self { + root, + key: String::new(), + } + } + + /// Rebuild a reference from a key a previous scan stored. + /// + /// The re-resolution FR-CAT-1a requires: after a restart the catalog holds + /// keys and nothing else, and a scan that could not resume from them would + /// have to walk the whole library to find the folder it left off in. + pub fn from_parts(root: RootId, key: impl Into) -> Self { + Self { + root, + key: key.into(), + } + } + + pub fn root_id(&self) -> RootId { + self.root + } + + /// The stored form. Meaningful only to the storage that produced it. + pub fn key(&self) -> &str { + &self.key + } + + /// Whether this is the granted root rather than something inside it. + /// + /// The walk needs it: a failure at the root is the whole library being + /// unreachable, and the deletion sweep must not run; a failure below it is + /// one folder (FR-CAT-9). + pub fn is_root(&self) -> bool { + self.key.is_empty() + } + + /// The last component, for exclusion checks and display. + pub fn name(&self) -> &str { + self.key.rsplit(['/', ':']).next().unwrap_or(&self.key) + } +} + +impl fmt::Display for DirRef { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + if self.key.is_empty() { + write!(f, "root {}", self.root.0) + } else { + write!(f, "{}", self.key) + } + } +} + +/// How to reach a listed entry. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum Node { + Dir(DirRef), + File(SourceRef), +} + +/// One entry from a listing: what the scanner classifies on, and how to reach +/// it. +/// +/// `meta.is_dir` and the [`Node`] variant always agree — the implementation +/// sets both from one observation. They are separate because they serve +/// different readers: `meta` goes to `dr_catalog::scan`, which decides, and +/// `node` goes to whatever acts on the decision. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct Entry { + pub meta: DirEntry, + pub node: Node, +} + +/// A stream that can be read and seeked — what a decoder wants. +/// +/// Blanket-implemented, so a `File`, a `Cursor>` in a test, and a +/// future SAF `ParcelFileDescriptor` wrapper all qualify without ceremony. +pub trait SeekableRead: Read + Seek + Send {} +impl SeekableRead for T {} + +/// TRACES: NFR-ARCH-4 +/// Something went wrong reaching storage. +/// +/// Typed, and never a panic: a library is on removable media, on a network +/// mount, or behind a permission the user can revoke while the app is running, +/// so every one of these is a normal Tuesday rather than a bug. +/// +/// Distinct from [`dr_types::SourceError`], which is what the *decode* path +/// sees. This one also speaks about roots and directories, which a decoder has +/// no concept of. +#[derive(Debug, thiserror::Error)] +pub enum StorageError { + /// A reference naming a root this storage was never granted. + /// + /// Almost always a stale catalog row: the library was removed and its rows + /// outlived it. + #[error("root {0} was not granted to this storage")] + UnknownRoot(u64), + + /// A reference of the wrong shape — a SAF document id handed to the + /// filesystem implementation, or the reverse. + /// + /// Refused rather than guessed at: the two address spaces have no overlap, + /// and a guess would read the wrong file rather than fail. + #[error("{0}")] + Unsupported(&'static str), + + /// A stored key that would leave its root. + /// + /// The catalog is a file on disk that other programs can edit, so a key + /// containing `..` is possible however it got there. Refused, because a + /// grant to one folder must not become a read of the whole filesystem + /// (NFR-SEC-1). + #[error("key {0:?} escapes its root")] + EscapesRoot(String), + + #[error("not found: {0}")] + NotFound(String), + + #[error("permission denied: {0}")] + PermissionDenied(String), + + #[error("not a directory: {0}")] + NotADirectory(String), + + #[error("range reads unsupported by this storage")] + RangeUnsupported, + + #[error("io error: {0}")] + Io(String), +} + +/// TRACES: FR-CAT-1 | FR-CAT-1a | NFR-PORT-1 +/// Enumerate and read the contents of granted library roots. +/// +/// Implemented per platform and injected at construction, so `core/` contains +/// no `#[cfg(target_os)]` (ARCH §10). +pub trait Storage: Send + Sync { + /// Every root this storage can currently reach. + fn roots(&self) -> Vec; + + /// The granted root as a directory, which is where a walk begins. + /// + /// Fails on a root that was never granted, so a stale catalog row is a + /// typed error rather than an empty library. + fn root_dir(&self, root: RootId) -> Result; + + /// Probe a directory without reading its contents. + /// + /// The cheap half of the pair, and the reason incremental scanning is + /// affordable: this costs one `stat` plus a name-only directory read, + /// where [`list`](Self::list) costs a `stat` per child. On a library of 2k + /// folders and 50k images that is 2k probes against 50k, which is the + /// difference between meeting and missing NFR-P1. + fn dir_state(&self, dir: &DirRef) -> Result; + + /// List a directory's direct children, with the metadata to classify them. + /// + /// Returns the whole listing rather than streaming it through a callback, + /// because the caller needs the complete set at once: an image in the + /// catalog that this listing does *not* contain has been deleted, and that + /// conclusion cannot be drawn one entry at a time. + fn list(&self, dir: &DirRef) -> Result, StorageError>; + + /// Open a seekable stream over a source. + fn open(&self, src: &SourceRef) -> Result, StorageError>; + + /// Read a byte range without opening the whole source. + /// + /// First-class rather than a convenience over [`open`](Self::open), because + /// for the case that matters it is a different operation and not a smaller + /// one: extracting an embedded JPEG preview from an 80 MB RAW over the + /// network transfers 1–3 MB (ARCH §3.1, FR-CULL-2, FR-NC-3). + /// + /// A range reaching past the end yields the bytes that exist — a short read + /// is the honest answer, and a caller needing exactly `n` bytes must say so + /// by checking the length. + fn read_range(&self, src: &SourceRef, range: ByteRange) -> Result, StorageError>; +} + +/// TRACES: FR-PLAT-LIN-1 | NFR-PORT-1 +/// The filesystem implementation: a root is a directory the user picked. +/// +/// Also the right implementation for a Flatpak, where the portal returns a real +/// path the sandbox can see (FR-PLAT-LIN-3). +/// +/// Present on Android too, and harmless there: the type compiles wherever +/// `std::fs` does, so `ui/` can name it unconditionally. It is not how an +/// Android user's library is reached — that is SAF, and ARCH §6.9 explains +/// why nothing else is on offer. +#[derive(Debug, Default)] +pub struct LocalStorage { + /// Ordered so [`roots`](Storage::roots) is stable, which keeps a scan of + /// several roots reproducible. + roots: BTreeMap, +} + +impl LocalStorage { + pub fn new() -> Self { + Self::default() + } + + /// One granted root, the common case. + pub fn with_root(root: RootId, dir: impl Into) -> Self { + let mut s = Self::new(); + s.grant(root, dir); + s + } + + /// Record that the user granted `dir` as `root`. + /// + /// **The only place a `Path` enters the application.** Above this line a + /// library location is a `RootId`, which is what lets the same catalog and + /// the same scanner run against SAF, where no path exists at all. + pub fn grant(&mut self, root: RootId, dir: impl Into) { + self.roots.insert(root, dir.into()); + } + + /// Where a root sits, for the app that granted it — to show the user, or to + /// store so the grant survives a restart. + pub fn root_path(&self, root: RootId) -> Option<&Path> { + self.roots.get(&root).map(|p| p.as_path()) + } + + fn base(&self, root: RootId) -> Result<&Path, StorageError> { + self.roots + .get(&root) + .map(|p| p.as_path()) + .ok_or(StorageError::UnknownRoot(root.0)) + } + + /// Turn a `(root, key)` pair back into a path, refusing anything that + /// leaves the root. + /// + /// Every component must be an ordinary name: `..` would climb out, and an + /// absolute key would discard the root entirely — `Path::join` silently + /// replaces rather than appends when handed one, which is how a "relative" + /// path of `/etc` becomes a read of `/etc`. + fn resolve(&self, root: RootId, key: &str) -> Result { + let base = self.base(root)?; + if key.is_empty() { + return Ok(base.to_path_buf()); + } + let rel = Path::new(key); + if !rel.components().all(|c| matches!(c, Component::Normal(_))) { + return Err(StorageError::EscapesRoot(key.to_string())); + } + Ok(base.join(rel)) + } + + fn file_path(&self, src: &SourceRef) -> Result { + match src { + SourceRef::Local { root, relative } => self.resolve(*root, relative), + SourceRef::Document { .. } => Err(StorageError::Unsupported( + "a SAF document reference cannot be read from the filesystem", + )), + SourceRef::Remote { .. } => Err(StorageError::Unsupported( + "a remote reference is resolved by the sync layer, not by storage", + )), + } + } +} + +/// A child's key, built the one place that is allowed to build one. +/// +/// Concatenation is safe here and only here: this implementation *chose* to +/// make its keys relative paths, so it is the only code entitled to know that +/// they compose. A SAF implementation has no equivalent (see the module docs), +/// which is why this is a private helper and not a method on [`DirRef`]. +fn child_key(parent: &str, name: &str) -> String { + if parent.is_empty() { + name.to_string() + } else { + format!("{parent}/{name}") + } +} + +impl Storage for LocalStorage { + fn roots(&self) -> Vec { + self.roots.keys().copied().collect() + } + + fn root_dir(&self, root: RootId) -> Result { + self.base(root)?; + Ok(DirRef::root(root)) + } + + fn dir_state(&self, dir: &DirRef) -> Result { + let path = self.resolve(dir.root_id(), dir.key())?; + let meta = std::fs::metadata(&path).map_err(|e| map_io(&path, e))?; + if !meta.is_dir() { + return Err(StorageError::NotADirectory(path.display().to_string())); + } + + // Names only — `read_dir` yields entries from `getdents` without a + // `stat` per child, so the count costs one pass and no per-file I/O. + // That is what makes this the cheap probe `list` is not. + let entry_count = std::fs::read_dir(&path) + .map_err(|e| map_io(&path, e))? + .count(); + + Ok(DirState { + mtime: modified_millis(&meta), + // Saturating rather than wrapping: a directory of four billion + // entries would otherwise wrap to a small number and could compare + // equal after a change. It cannot happen, and being wrong about it + // would be silent. + entry_count: u32::try_from(entry_count).unwrap_or(u32::MAX), + }) + } + + fn list(&self, dir: &DirRef) -> Result, StorageError> { + let path = self.resolve(dir.root_id(), dir.key())?; + let read = std::fs::read_dir(&path).map_err(|e| map_io(&path, e))?; + + let mut out = Vec::new(); + for entry in read { + let entry = match entry { + Ok(e) => e, + // One unreadable entry is not an unreadable directory. Skipping + // it loses one file; failing the listing would make the folder + // look empty, and an empty folder is a *deletion* to the sweep. + Err(e) => { + log::debug!("list {}: skipping unreadable entry: {e}", path.display()); + continue; + } + }; + + // A name that is not UTF-8 cannot become a key, and a lossy + // conversion would produce a key that resolves to nothing — an + // image catalogued and then permanently unreadable. Skipped, and + // said out loud, because the user's file is real and we are + // choosing not to see it. + let Some(name) = entry.file_name().to_str().map(str::to_owned) else { + log::warn!( + "list {}: skipping {:?}, whose name is not valid UTF-8", + path.display(), + entry.file_name() + ); + continue; + }; + + // Follows symlinks, unlike `DirEntry::metadata`. A photographer who + // symlinks last year's drive into the library means it as part of + // the library. The walk's depth limit is what stops a loop. + let child = path.join(&name); + let meta = match std::fs::metadata(&child) { + Ok(m) => m, + // A broken symlink, or a file deleted between the listing and + // this stat. Neither is an error worth failing a folder for. + Err(e) => { + log::debug!("list {}: skipping {name}: {e}", path.display()); + continue; + } + }; + + let key = child_key(dir.key(), &name); + let is_dir = meta.is_dir(); + out.push(Entry { + meta: DirEntry { + name, + is_dir, + size: meta.len(), + mtime: modified_millis(&meta), + }, + node: if is_dir { + Node::Dir(DirRef::from_parts(dir.root_id(), key)) + } else { + Node::File(SourceRef::Local { + root: dir.root_id(), + relative: key, + }) + }, + }); + } + + // Directory order is filesystem order, which is arbitrary and differs + // between runs. Sorting makes a scan reproducible and a test able to + // assert on what it found. + out.sort_by(|a, b| a.meta.name.cmp(&b.meta.name)); + Ok(out) + } + + fn open(&self, src: &SourceRef) -> Result, StorageError> { + let path = self.file_path(src)?; + let file = std::fs::File::open(&path).map_err(|e| map_io(&path, e))?; + Ok(Box::new(file)) + } + + fn read_range(&self, src: &SourceRef, range: ByteRange) -> Result, StorageError> { + use std::io::SeekFrom; + + if range.end <= range.start { + return Ok(Vec::new()); + } + let path = self.file_path(src)?; + let mut file = std::fs::File::open(&path).map_err(|e| map_io(&path, e))?; + file.seek(SeekFrom::Start(range.start)) + .map_err(|e| map_io(&path, e))?; + + // `take` and grow, rather than a buffer sized to the request: the range + // comes from a header the file itself declared, and a corrupt one + // asking for four gigabytes must not be allocated before it is known + // that four gigabytes exist (NFR-SEC-1). + let mut buf = Vec::new(); + file.take(range.end - range.start) + .read_to_end(&mut buf) + .map_err(|e| map_io(&path, e))?; + Ok(buf) + } +} + +/// Milliseconds since the epoch, or 0 where the platform will not say. +/// +/// Milliseconds because that is the unit change detection is expressed in, and +/// the unit's granularity is the width of the window in which a change is +/// invisible — see [`dr_types::DirState::mtime`]. Seconds would hide a card +/// imported and scanned within the same tick. +/// +/// A file whose mtime is unreadable compares equal to itself forever and so is +/// never re-read. That is the better failure: the alternative, a value that +/// changes each time it is asked for, would re-process the file on every scan. +fn modified_millis(meta: &std::fs::Metadata) -> i64 { + let Ok(t) = meta.modified() else { + return 0; + }; + match t.duration_since(std::time::UNIX_EPOCH) { + Ok(d) => i64::try_from(d.as_millis()).unwrap_or(i64::MAX), + // Before 1970. Rare, but an archive of digitised film can carry one, + // and it must not become a huge positive number. + Err(e) => i64::try_from(e.duration().as_millis()) + .unwrap_or(i64::MAX) + .saturating_neg(), + } +} + +fn map_io(path: &Path, e: std::io::Error) -> StorageError { + let what = path.display().to_string(); + match e.kind() { + std::io::ErrorKind::NotFound => StorageError::NotFound(what), + std::io::ErrorKind::PermissionDenied => StorageError::PermissionDenied(what), + _ => StorageError::Io(format!("{what}: {e}")), + } +} + +#[cfg(test)] +mod tests { + use super::*; + use std::fs; + + /// A throwaway directory tree, removed when the test ends. + struct Tree(PathBuf); + + impl Tree { + fn new(name: &str) -> Self { + let dir = std::env::temp_dir().join(format!( + "dr-plat-{name}-{}-{:?}", + std::process::id(), + std::thread::current().id() + )); + let _ = fs::remove_dir_all(&dir); + fs::create_dir_all(&dir).expect("temp dir"); + Tree(dir) + } + + fn dir(&self, rel: &str) -> &Self { + fs::create_dir_all(self.0.join(rel)).expect("mkdir"); + self + } + + fn file(&self, rel: &str, bytes: &[u8]) -> &Self { + let p = self.0.join(rel); + if let Some(parent) = p.parent() { + fs::create_dir_all(parent).expect("mkdir"); + } + fs::write(p, bytes).expect("write"); + self + } + + fn storage(&self) -> LocalStorage { + LocalStorage::with_root(RootId(1), self.0.clone()) + } + } + + impl Drop for Tree { + fn drop(&mut self) { + let _ = fs::remove_dir_all(&self.0); + } + } + + const ROOT: RootId = RootId(1); + + #[test] + fn a_listing_names_files_and_directories_apart() { + let t = Tree::new("listing"); + t.file("IMG_0001.CR3", b"raw").dir("2026"); + let s = t.storage(); + + let entries = s.list(&s.root_dir(ROOT).unwrap()).unwrap(); + assert_eq!(entries.len(), 2); + assert_eq!(entries[0].meta.name, "2026"); + assert!(entries[0].meta.is_dir); + assert!(matches!(entries[0].node, Node::Dir(_))); + assert_eq!(entries[1].meta.name, "IMG_0001.CR3"); + assert_eq!(entries[1].meta.size, 3); + assert!(matches!(entries[1].node, Node::File(_))); + } + + #[test] + fn a_listing_hands_back_references_the_caller_never_composes() { + // The property that makes a SAF implementation a drop-in: the child's + // reference comes from the listing, because on SAF it is the only place + // it can come from. If a test ever has to build one by joining strings, + // the abstraction has already leaked. + let t = Tree::new("refs"); + t.file("2026/08/IMG_0042.CR3", b"raw"); + let s = t.storage(); + + let year = match &s.list(&s.root_dir(ROOT).unwrap()).unwrap()[0].node { + Node::Dir(d) => d.clone(), + other => panic!("expected a directory, got {other:?}"), + }; + let month = match &s.list(&year).unwrap()[0].node { + Node::Dir(d) => d.clone(), + other => panic!("expected a directory, got {other:?}"), + }; + let file = match &s.list(&month).unwrap()[0].node { + Node::File(f) => f.clone(), + other => panic!("expected a file, got {other:?}"), + }; + + assert_eq!( + file, + SourceRef::Local { + root: ROOT, + relative: "2026/08/IMG_0042.CR3".into() + } + ); + assert!(s.open(&file).is_ok()); + } + + #[test] + fn a_stored_key_reopens_the_same_directory_after_a_restart() { + // What FR-CAT-1a's "re-resolvable" means in practice: the catalog keeps + // keys, not handles, and a scan resuming tomorrow rebuilds the + // reference from one. Without this a restart is a full rewalk. + let t = Tree::new("reresolve"); + t.file("2026/IMG.CR3", b"raw"); + let s = t.storage(); + + let key = match &s.list(&s.root_dir(ROOT).unwrap()).unwrap()[0].node { + Node::Dir(d) => d.key().to_string(), + other => panic!("expected a directory, got {other:?}"), + }; + + let rebuilt = DirRef::from_parts(ROOT, key); + assert_eq!(s.list(&rebuilt).unwrap()[0].meta.name, "IMG.CR3"); + } + + #[test] + fn a_key_that_climbs_out_of_the_root_is_refused() { + // A grant is to one folder. The catalog is an ordinary file that other + // programs can edit, so a `..` in a key is reachable however it got + // there, and honouring it would turn a grant to ~/Photos into a read of + // the whole filesystem (NFR-SEC-1). + let t = Tree::new("escape"); + let s = t.storage(); + + for key in ["../etc", "a/../../etc", "/etc"] { + let dir = DirRef::from_parts(ROOT, key); + assert!( + matches!(s.dir_state(&dir), Err(StorageError::EscapesRoot(_))), + "{key} was not refused" + ); + } + } + + #[test] + fn an_absolute_key_does_not_silently_replace_the_root() { + // `Path::join` replaces rather than appends when given an absolute + // path, so this one is not merely an escape — it is an escape that + // looks like ordinary joining and would never be noticed in review. + let t = Tree::new("absolute"); + let s = t.storage(); + let src = SourceRef::Local { + root: ROOT, + relative: "/etc/passwd".into(), + }; + assert!(matches!( + s.read_range(&src, 0..16), + Err(StorageError::EscapesRoot(_)) + )); + } + + #[test] + fn probing_a_directory_reports_what_change_detection_needs() { + let t = Tree::new("probe"); + t.file("a.CR3", b"1").file("b.CR3", b"2"); + let s = t.storage(); + + let before = s.dir_state(&s.root_dir(ROOT).unwrap()).unwrap(); + assert_eq!(before.entry_count, 2); + + t.file("c.CR3", b"3"); + let after = s.dir_state(&s.root_dir(ROOT).unwrap()).unwrap(); + assert_ne!( + before, after, + "an added file must move the state, or the folder is pruned and the \ + image never enters the catalog" + ); + assert_eq!(after.entry_count, 3); + } + + #[test] + fn the_probe_counts_directories_as_well_as_files() { + // A new subfolder full of images changes nothing about the parent's + // files. If the count ignored directories, the parent would look + // unchanged and the whole subtree would go unseen. + let t = Tree::new("probe-dirs"); + t.file("a.CR3", b"1"); + let s = t.storage(); + let before = s.dir_state(&s.root_dir(ROOT).unwrap()).unwrap(); + + t.dir("2026"); + let after = s.dir_state(&s.root_dir(ROOT).unwrap()).unwrap(); + assert_eq!(after.entry_count, before.entry_count + 1); + } + + #[test] + fn a_range_read_returns_only_the_bytes_asked_for() { + let t = Tree::new("range"); + t.file("IMG.CR3", b"0123456789"); + let s = t.storage(); + let src = SourceRef::Local { + root: ROOT, + relative: "IMG.CR3".into(), + }; + + assert_eq!(s.read_range(&src, 2..6).unwrap(), b"2345"); + assert_eq!(s.read_range(&src, 0..0).unwrap(), b""); + } + + #[test] + fn a_range_past_the_end_is_short_rather_than_an_error() { + // Preview offsets come out of the file's own header. A truncated or + // mis-parsed one must yield "here is what exists", not a failed decode + // and not a four-gigabyte allocation (NFR-SEC-1). + let t = Tree::new("range-eof"); + t.file("IMG.CR3", b"0123456789"); + let s = t.storage(); + let src = SourceRef::Local { + root: ROOT, + relative: "IMG.CR3".into(), + }; + + assert_eq!(s.read_range(&src, 8..u64::MAX / 2).unwrap(), b"89"); + assert_eq!(s.read_range(&src, 999..1_000).unwrap(), b""); + } + + #[test] + fn an_open_stream_can_seek() { + // The decoders need it: a RAW's preview lives at an offset the header + // names, and a forward-only stream would mean reading 80 MB to get 2. + let t = Tree::new("seek"); + t.file("IMG.CR3", b"0123456789"); + let s = t.storage(); + let mut r = s + .open(&SourceRef::Local { + root: ROOT, + relative: "IMG.CR3".into(), + }) + .unwrap(); + + r.seek(std::io::SeekFrom::Start(5)).unwrap(); + let mut buf = [0u8; 2]; + r.read_exact(&mut buf).unwrap(); + assert_eq!(&buf, b"56"); + } + + #[test] + fn an_ungranted_root_is_a_typed_error_not_an_empty_library() { + // A removed library leaves catalog rows behind. Reporting them as + // "nothing here" would let the deletion sweep take the lot (FR-CAT-9). + let s = LocalStorage::new(); + assert!(matches!( + s.root_dir(RootId(7)), + Err(StorageError::UnknownRoot(7)) + )); + assert!(matches!( + s.dir_state(&DirRef::root(RootId(7))), + Err(StorageError::UnknownRoot(7)) + )); + assert!(s.roots().is_empty()); + } + + #[test] + fn a_missing_directory_is_not_reported_as_empty() { + // The same failure from the other direction: an unplugged drive must + // error, because an empty listing means every image under it was + // deleted. + let t = Tree::new("missing"); + let s = t.storage(); + let gone = DirRef::from_parts(ROOT, "nowhere"); + assert!(matches!(s.dir_state(&gone), Err(StorageError::NotFound(_)))); + assert!(matches!(s.list(&gone), Err(StorageError::NotFound(_)))); + } + + #[test] + fn a_reference_of_the_wrong_kind_is_refused_rather_than_guessed_at() { + // A SAF document id is not a path. Treating it as one would resolve to + // some other file, which is worse than failing. + let t = Tree::new("wrong-kind"); + let s = t.storage(); + let saf = SourceRef::Document { + tree: ROOT, + document_id: "primary:DCIM/IMG.CR3".into(), + }; + assert!(matches!(s.open(&saf), Err(StorageError::Unsupported(_)))); + let remote = SourceRef::Remote { + file_id: 1, + path: "Photos/IMG.CR3".into(), + }; + assert!(matches!(s.open(&remote), Err(StorageError::Unsupported(_)))); + } + + #[test] + fn several_roots_coexist() { + // Two libraries on two drives is FR-CAT-1's "one or more roots", and + // each reference carries which one it belongs to. + let a = Tree::new("multi-a"); + a.file("a.CR3", b"1"); + let b = Tree::new("multi-b"); + b.file("b.CR3", b"2"); + + let mut s = LocalStorage::new(); + s.grant(RootId(1), a.0.clone()); + s.grant(RootId(2), b.0.clone()); + + assert_eq!(s.roots(), vec![RootId(1), RootId(2)]); + assert_eq!( + s.list(&s.root_dir(RootId(2)).unwrap()).unwrap()[0] + .meta + .name, + "b.CR3" + ); + } + + #[test] + fn a_listing_is_ordered_the_same_way_twice() { + // Filesystem order is arbitrary and differs between runs; a scan that + // depended on it would produce a different catalog each time. + let t = Tree::new("order"); + for n in ["c.CR3", "a.CR3", "b.CR3"] { + t.file(n, b"x"); + } + let s = t.storage(); + let names: Vec = s + .list(&s.root_dir(ROOT).unwrap()) + .unwrap() + .into_iter() + .map(|e| e.meta.name) + .collect(); + assert_eq!(names, vec!["a.CR3", "b.CR3", "c.CR3"]); + } + + #[cfg(unix)] + #[test] + fn a_name_that_is_not_utf8_is_skipped_rather_than_mangled() { + // A lossy conversion would produce a key that resolves to nothing: the + // image would be catalogued and then permanently unopenable. Better to + // not see the file than to promise it and fail later. + use std::os::unix::ffi::OsStrExt; + + let t = Tree::new("non-utf8"); + t.file("good.CR3", b"1"); + let bad = t.0.join(std::ffi::OsStr::from_bytes(b"bad\xff.CR3")); + fs::write(&bad, b"2").expect("write"); + + let s = t.storage(); + let entries = s.list(&s.root_dir(ROOT).unwrap()).unwrap(); + assert_eq!(entries.len(), 1); + assert_eq!(entries[0].meta.name, "good.CR3"); + } + + #[test] + fn the_root_is_distinguishable_from_what_is_inside_it() { + // The walk keys the difference between "the library is unreachable" and + // "one folder failed" on this, and those have opposite consequences for + // the deletion sweep (FR-CAT-9). + assert!(DirRef::root(ROOT).is_root()); + assert!(!DirRef::from_parts(ROOT, "2026").is_root()); + assert_eq!(DirRef::from_parts(ROOT, "2026/08").name(), "08"); + } +}