Merge branch 'local-libraries'

core/dr-catalog/Cargo.toml

@@ -7,9 +7,19 @@ license.workspace = true
 
 [dependencies]
 dr-types.workspace = true
+# The `Storage` trait, and nothing else from it. A scan has to read a real
+# directory, and this is how `core/` reaches the platform without a
+# `#[cfg(target_os)]` of its own (ARCH §4.1: calls go downward).
+dr-plat.workspace = true
 rusqlite.workspace = true
 thiserror.workspace = true
 log.workspace = true
 # `collections.selector_json` — the stored form of a smart collection's
 # selector. The column predates this dependency; nothing else here is JSON.
 serde_json.workspace = true
+
+# For the `scan_local` example only, which is a diagnostic tool: what it is
+# diagnosing is often a folder the scan warned about and skipped, and those
+# warnings go to `log`.
+[dev-dependencies]
+env_logger.workspace = true

core/dr-catalog/examples/scan_local.rs

@@ -0,0 +1,111 @@
+//! Scan a real folder on this machine into a catalog, and say what it cost.
+//!
+//!   cargo run -p dr-catalog --example scan_local -- ~/Pictures [catalog.sqlite]
+//!
+//! **Run it twice.** The first run is a full walk; the second is the one worth
+//! watching, because on an unchanged library it should list no directories at
+//! all and take a fraction of the time. That difference is NFR-P1, and a
+//! synthetic test cannot show it at the scale a real library does — 121,785
+//! files in a synced folder is a different question from twenty in a temporary
+//! directory.
+//!
+//! Writes only to the catalog file, which defaults to a fixed path in the
+//! system temporary directory so a second run has something to compare
+//! against. Nothing in the scanned folder is touched.
+
+use std::path::PathBuf;
+
+use dr_catalog::walk::{ensure_root, scan_root, RootKind};
+use dr_catalog::Catalog;
+use dr_plat::LocalStorage;
+use dr_types::FormatFilter;
+
+fn main() {
+    env_logger::init();
+
+    let mut args = std::env::args().skip(1);
+    let Some(dir) = args.next().map(PathBuf::from) else {
+        eprintln!("usage: scan_local <directory> [catalog.sqlite]");
+        std::process::exit(2);
+    };
+    let catalog_path = args
+        .next()
+        .map(PathBuf::from)
+        .unwrap_or_else(|| std::env::temp_dir().join("darkroom-scan-local.sqlite"));
+
+    let catalog = match Catalog::open(&catalog_path) {
+        Ok(c) => c,
+        Err(e) => {
+            eprintln!("cannot open {}: {e}", catalog_path.display());
+            std::process::exit(1);
+        }
+    };
+    println!("catalog: {}", catalog_path.display());
+
+    // The label is how the grant is spelled, and the only place a path is
+    // written down. Everything after this line addresses files by `RootId`.
+    let label = dir.display().to_string();
+    let root = match ensure_root(catalog.connection(), RootKind::Local, &label) {
+        Ok(r) => r,
+        Err(e) => {
+            eprintln!("cannot record the root: {e}");
+            std::process::exit(1);
+        }
+    };
+    let storage = LocalStorage::with_root(root, &dir);
+
+    let now = std::time::SystemTime::now()
+        .duration_since(std::time::UNIX_EPOCH)
+        .map(|d| d.as_secs() as i64)
+        .unwrap_or(0);
+
+    let started = std::time::Instant::now();
+    let report = match scan_root(
+        catalog.connection(),
+        &storage,
+        root,
+        &FormatFilter::all(),
+        now,
+        || false,
+        |p| {
+            // One line per hundred directories: enough to show it is alive on a
+            // large library, not enough to be the thing that slows it down.
+            let visited = p.directories_listed + p.directories_pruned;
+            if visited % 100 == 0 {
+                println!(
+                    "  … {visited} directories ({} pruned), {} images",
+                    p.directories_pruned, p.images_found
+                );
+            }
+        },
+    ) {
+        Ok(r) => r,
+        Err(e) => {
+            eprintln!("scan failed: {e}");
+            std::process::exit(1);
+        }
+    };
+    let elapsed = started.elapsed();
+
+    let total: i64 = catalog
+        .connection()
+        .query_row("SELECT count(*) FROM images", [], |r| r.get(0))
+        .unwrap_or(-1);
+
+    println!("\noutcome:     {:?}", report.outcome);
+    println!(
+        "directories: {} listed, {} pruned",
+        report.progress.directories_listed, report.progress.directories_pruned
+    );
+    println!(
+        "images:      {} new, {} changed, {} unchanged, {} removed",
+        report.inserted, report.updated, report.unchanged, report.images_removed
+    );
+    println!("folders:     {} removed", report.folders_removed);
+    println!("catalogued:  {total} in total");
+    println!("took:        {:.2?}", elapsed);
+
+    if report.progress.directories_listed == 0 && report.progress.directories_pruned > 0 {
+        println!("\nnothing had changed: every folder was proven unchanged by one probe");
+    }
+}

core/dr-catalog/src/error.rs

@@ -25,6 +25,15 @@ pub enum CatalogError {
     #[error("root {0} is unreachable; scan aborted without pruning")]
     RootUnreachable(u64),
 
+    /// A scan was asked for a root the catalog has no row for.
+    ///
+    /// A caller's mistake rather than a user's: the row is created when the
+    /// grant is obtained, because the label — the path, the tree URI — is known
+    /// only there. Inventing one here would file the library under a name
+    /// nothing else would look it up by.
+    #[error("no such root: {0}")]
+    NoSuchRoot(u64),
+
     /// A smart collection whose selector references itself, directly or via
     /// another collection.
     #[error("collection {0} would form a cycle")]

core/dr-catalog/src/lib.rs

@@ -11,6 +11,7 @@
 //!
 //! - [`schema`] — tables and forward-only migrations
 //! - [`scan`] — incremental discovery that prunes unchanged directories
+//! - [`walk`] — those decisions driven against real storage, local or SAF
 //! - [`query`] — selectors compiled to indexed SQL, windowed for the grid
 //! - [`collections`] — the collection tree and membership the UI edits
 //! - [`jobs`] — the durable background work queue
@@ -41,6 +42,7 @@ pub mod scan;
 pub mod schema;
 pub mod sync;
 pub mod trash;
+pub mod walk;
 
 pub use cache::{Budget, Cache, DEFAULT_BUDGET_BYTES};
 pub use collections::{Collection, CollectionKind, TreeRow};
@@ -51,6 +53,7 @@ pub use query::{Query, Sort};
 pub use rating::{Judgement, MAX_RATING};
 pub use scan::{DirAction, DirState, EntryAction, ScanOutcome};
 pub use trash::{TrashedImage, TRASH_DIR};
+pub use walk::{ensure_root, scan_root, RootKind, ScanProgress, ScanReport};
 
 /// One row of the library grid.
 ///

core/dr-catalog/src/query.rs

@@ -318,7 +318,10 @@ fn flag_code(f: FlagState) -> i64 {
     }
 }
 
-fn availability_code(a: Availability) -> i64 {
+/// The stored form of an availability. Shared with [`crate::walk`], which
+/// writes the column this reads — two spellings of the same mapping would
+/// filter for a state nothing ever writes.
+pub(crate) fn availability_code(a: Availability) -> i64 {
     match a {
         Availability::MetadataOnly => 0,
         Availability::Preview => 1,

core/dr-catalog/src/scan.rs

@@ -14,35 +14,13 @@
 //!
 //! This module holds the decision logic and the deletion-sweep rules; walking
 //! an actual directory belongs to the platform layer, which supplies
-//! [`DirState`] and [`DirEntry`].
+//! [`DirState`] and [`DirEntry`]. [`crate::walk`] is what puts the two
+//! together.
+
+pub use dr_types::{DirEntry, DirState};
 
 use dr_types::FormatFilter;
 
-/// What a directory looked like when last scanned, and what it looks like now.
-///
-/// Both fields are cheap to obtain: one `stat` locally, one
-/// `DocumentsContract` metadata query on SAF.
-#[derive(Debug, Clone, Copy, PartialEq, Eq)]
-pub struct DirState {
-    pub mtime: i64,
-    /// Direct children, files and directories alike.
-    ///
-    /// mtime alone misses a delete-and-create inside one timestamp tick, and
-    /// coarse-granularity providers widen that window. The count does not
-    /// close the hole — a paired add and remove moves neither — but a bare add
-    /// or remove moves the count, and those are far commoner.
-    pub entry_count: u32,
-}
-
-/// One entry from a directory listing.
-#[derive(Debug, Clone, PartialEq, Eq)]
-pub struct DirEntry {
-    pub name: String,
-    pub is_dir: bool,
-    pub size: u64,
-    pub mtime: i64,
-}
-
 /// What the scanner should do with a directory, before listing it.
 #[derive(Debug, Clone, Copy, PartialEq, Eq)]
 pub enum DirAction {
@@ -100,12 +78,17 @@ pub fn classify_entry(
 }
 
 /// Outcome of a scan, which decides whether pruning may run.
-#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+///
+/// `Cancelled` is the default because a scan that has not run has proven
+/// nothing absent, and every default in this area must fail towards keeping
+/// photographs.
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)]
 pub enum ScanOutcome {
     /// Every reachable folder was visited.
     Complete,
     /// The user cancelled. Partial state is valid — jobs are resumable — but
     /// unvisited folders must not be read as deleted.
+    #[default]
     Cancelled,
     /// The root itself could not be opened: drive unplugged, SAF grant
     /// revoked, share unmounted.

core/dr-types/src/lib.rs

@@ -63,16 +63,25 @@ pub enum SourceRef {
 impl SourceRef {
     /// A stable, display-friendly name — the last path component.
     pub fn display_name(&self) -> &str {
-        let full = match self {
-            SourceRef::Local { relative, .. } => relative.as_str(),
-            SourceRef::Document { document_id, .. } => document_id.as_str(),
-            SourceRef::Remote { path, .. } => path.as_str(),
-        };
+        let full = self.key();
         // SAF document ids use ':' as a separator; paths use '/'. Split on
         // whichever appears last so both yield a sensible name.
         full.rsplit(['/', ':']).next().unwrap_or(full)
     }
 
+    /// The stored form — what the catalog holds in `images.source_ref`.
+    ///
+    /// Which of the three variants a key belongs to is not recorded beside it,
+    /// because the root already says: `roots.kind` is `'local' | 'saf' |
+    /// 'remote'`, and every image names its root.
+    pub fn key(&self) -> &str {
+        match self {
+            SourceRef::Local { relative, .. } => relative.as_str(),
+            SourceRef::Document { document_id, .. } => document_id.as_str(),
+            SourceRef::Remote { path, .. } => path.as_str(),
+        }
+    }
+
     /// Lowercase file extension, if any.
     ///
     /// Looks through a Nextcloud VFS placeholder suffix, so a dehydrated
@@ -251,6 +260,55 @@ impl FormatFilter {
     }
 }
 
+/// TRACES: FR-CAT-1 | NFR-PORT-1
+/// What a directory looked like when last scanned, and what it looks like now.
+///
+/// Both fields are cheap to obtain: one `stat` plus a name-only directory read
+/// locally, one `DocumentsContract` query on SAF. Neither costs a `stat` per
+/// child, which is the whole point — see `dr_catalog::scan`.
+///
+/// Lives here rather than in either crate that uses it because it is the
+/// sentence the platform layer says to the catalog: `dr-plat` produces it by
+/// probing, `dr-catalog` stores it and compares. A copy on each side would be
+/// two types that must agree by convention.
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub struct DirState {
+    /// Milliseconds since the epoch. **Not seconds**, and the difference is not
+    /// pedantry: change detection asks whether a timestamp *moved*, so the
+    /// granularity of the unit is the width of the window in which a change is
+    /// invisible. A second is long enough to copy a card and start a scan, and
+    /// anything happening inside one tick of the stored value looks exactly
+    /// like nothing happening — the directory is pruned and the photographs
+    /// never appear.
+    ///
+    /// Both platforms can supply it: a filesystem records nanoseconds, and
+    /// SAF's `COLUMN_LAST_MODIFIED` is already in milliseconds.
+    pub mtime: i64,
+    /// Direct children, files and directories alike.
+    ///
+    /// mtime alone misses a delete-and-create inside one timestamp tick, and
+    /// coarse-granularity providers widen that window. The count does not
+    /// close the hole — a paired add and remove moves neither — but a bare add
+    /// or remove moves the count, and those are far commoner.
+    pub entry_count: u32,
+}
+
+/// TRACES: FR-CAT-1 | NFR-PORT-1
+/// One entry from a directory listing, as the scanner classifies it.
+///
+/// Deliberately carries no handle to the entry itself: this is what the
+/// *decision* logic reads, and it must be constructible in a test without a
+/// filesystem. The platform pairs it with the reference needed to reach the
+/// entry (`dr_plat::Entry`).
+#[derive(Debug, Clone, PartialEq, Eq)]
+pub struct DirEntry {
+    pub name: String,
+    pub is_dir: bool,
+    pub size: u64,
+    /// Milliseconds since the epoch, as in [`DirState::mtime`].
+    pub mtime: i64,
+}
+
 /// How much of an image is available locally (FR-NC-6c).
 ///
 /// Surfaced in the UI so a user always knows what they have — the failure