Add folder scan with format selection; validate A3 on a real library

Library setup as the user described it: pick a folder, choose which RAW
types to look for, scan recursively.

  dr-types::FormatFilter  the tick-box selection, seeing through VFS
                          placeholder suffixes so a dehydrated CR2 still
                          matches as a CR2
  dr-sync::scan           recursive walk, Depth:1 per directory, pruning
                          unchanged subtrees where the backend propagates
                          directory ETags

Verified against nextcloud.tourolle.paris (34.0.2) on a real library:

  browse root      32 entries, 98ms
  scan PhotosRaw   17,185 RAW files in 334 directories, 34.1s
                   (7,836 CR2 + 9,349 DNG)
  range read       262KB of a 21.5MB DNG in 119ms — 1.22% of the file,
                   and enough to read "Canon EOS 6D | ISO 100"

That last line is assumption A3 validated on real data. Cataloguing this
library by whole-file fetch would move roughly 370GB; the range path
moves a few MB.

Pruning is capability-gated rather than assumed: with per-entry ETags a
probe costs a request and proves nothing about children, so it is skipped
entirely. A test asserts zero probes in that case.

Still unresolved: /core/preview returns 400 for every parameter
combination tried, including on a JPEG the server reports as having a
preview. Not a request-shape bug — it fails identically bare. Recorded
rather than worked around; ARCH §6.7 already treats server previews as
opportunistic, so nothing depends on it.

core/dr-sync/Cargo.toml

@@ -10,3 +10,6 @@ dr-types.workspace = true
 async-trait.workspace = true
 thiserror.workspace = true
 log.workspace = true
+
+[dev-dependencies]
+tokio = { workspace = true }

core/dr-sync/src/lib.rs

@@ -22,10 +22,12 @@ use async_trait::async_trait;
 
 pub mod capability;
 pub mod error;
+pub mod scan;
 pub mod types;
 
 pub use capability::{Capabilities, ChangeDetection, ChunkConstraints, ServerPreviews};
 pub use error::RemoteError;
+pub use scan::{scan, ScanProgress, ScanResult};
 pub use types::{
     Cursor, EntryKind, Identity, Precondition, RemoteChange, RemoteEntry, RemoteId, RemotePath,
     Validator,

core/dr-sync/src/scan.rs

@@ -0,0 +1,464 @@
+//! Recursive discovery of images under a chosen remote folder.
+//!
+//! The library-setup path: the user picks a folder, ticks the formats they
+//! shoot, and this walks the tree finding matching files (FR-CAT-1, M-5).
+//!
+//! Depth:1 per directory, never `Depth: infinity` — the latter is frequently
+//! disabled and prohibitively expensive where it is not (ARCH §8.4). Where the
+//! backend propagates directory ETags, an unchanged subtree is skipped whole,
+//! which is what keeps a re-scan proportional to what changed rather than to
+//! library size.
+
+use std::collections::HashMap;
+
+use dr_types::FormatFilter;
+
+use crate::{
+    Capabilities, ChangeDetection, EntryKind, RemoteBackend, RemoteEntry, RemoteError, RemotePath,
+    Validator,
+};
+
+/// Progress during a scan, so the UI can show something on a large library.
+#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
+pub struct ScanProgress {
+    pub directories_listed: usize,
+    /// Directories skipped because their ETag was unchanged. The value of
+    /// pruning, made visible.
+    pub directories_pruned: usize,
+    pub images_found: usize,
+}
+
+/// The result of a scan.
+#[derive(Debug, Clone, Default)]
+pub struct ScanResult {
+    /// Files matching the format filter.
+    pub images: Vec<RemoteEntry>,
+    /// Every directory visited, with its ETag, so the next scan can prune.
+    ///
+    /// **Must be persisted.** Without stored folder ETags there is nothing to
+    /// compare against and every scan is a full walk (ARCH §6.6).
+    pub directories: Vec<(RemotePath, Validator)>,
+    pub progress: ScanProgress,
+}
+
+/// How deep to recurse before giving up.
+///
+/// A symlink loop or a pathological tree would otherwise walk forever. Real
+/// photo libraries are nowhere near this deep.
+const MAX_DEPTH: usize = 32;
+
+/// TRACES: FR-CAT-1 | FR-NC-4 | M-5 | M-7
+/// Walk `root` recursively, collecting files the filter accepts.
+///
+/// `known` maps previously seen directories to their ETags. Pass an empty map
+/// for a first scan; pass the stored ETags to prune unchanged subtrees.
+///
+/// `on_progress` is called after each directory so a long scan can report
+/// rather than appear hung.
+pub async fn scan<B, F>(
+    backend: &B,
+    root: &RemotePath,
+    filter: &FormatFilter,
+    known: &HashMap<RemotePath, Validator>,
+    mut on_progress: F,
+) -> Result<ScanResult, RemoteError>
+where
+    B: RemoteBackend + ?Sized,
+    F: FnMut(ScanProgress),
+{
+    let prunable = supports_pruning(backend.capabilities());
+    let mut result = ScanResult::default();
+
+    // Explicit stack rather than recursion: an async recursive fn needs
+    // boxing, and a deep tree could overflow.
+    let mut stack = vec![(root.clone(), 0usize)];
+
+    while let Some((dir, depth)) = stack.pop() {
+        if depth > MAX_DEPTH {
+            log::warn!("scan: depth limit at {dir}, not descending further");
+            continue;
+        }
+
+        // Prune: if the directory's ETag is unchanged, nothing anywhere
+        // beneath it changed either, because Nextcloud propagates upward.
+        if prunable {
+            if let Some(previous) = known.get(&dir) {
+                match backend.dir_validator(&dir).await {
+                    Ok(current) if &current == previous => {
+                        result.progress.directories_pruned += 1;
+                        on_progress(result.progress);
+                        continue;
+                    }
+                    Ok(_) => {}
+                    // A probe failure is not fatal — fall through to listing,
+                    // which is correct, just not free.
+                    Err(e) => log::debug!("scan: validator probe failed for {dir}: {e}"),
+                }
+            }
+        }
+
+        let entries = match backend.list(&dir, None).await {
+            Ok(e) => e,
+            Err(RemoteError::NotFound(_)) => {
+                // Deleted between listing its parent and reaching it.
+                log::debug!("scan: {dir} vanished during the walk");
+                continue;
+            }
+            Err(e) => return Err(e),
+        };
+
+        result.progress.directories_listed += 1;
+
+        for entry in entries {
+            match entry.kind {
+                EntryKind::Directory => {
+                    result
+                        .directories
+                        .push((entry.path.clone(), entry.validator.clone()));
+                    stack.push((entry.path, depth + 1));
+                }
+                EntryKind::File => {
+                    if filter.allows_name(entry.path.name()) {
+                        result.images.push(entry);
+                        result.progress.images_found += 1;
+                    }
+                }
+            }
+        }
+
+        on_progress(result.progress);
+    }
+
+    // Sort so a scan is reproducible and the grid has a stable order.
+    result.images.sort_by(|a, b| a.path.cmp(&b.path));
+    result.directories.sort_by(|a, b| a.0.cmp(&b.0));
+    Ok(result)
+}
+
+/// Whether pruning is worth attempting against this backend.
+///
+/// Only propagating ETags make an unchanged parent prove an unchanged
+/// subtree. With per-entry ETags the probe costs a request and proves
+/// nothing about children, so it is pure overhead.
+fn supports_pruning(caps: &Capabilities) -> bool {
+    matches!(caps.change_detection, ChangeDetection::PropagatingEtags)
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use crate::{RemoteId, ServerPreviews};
+    use async_trait::async_trait;
+    use std::cell::RefCell;
+    use std::ops::Range;
+
+    /// A backend over an in-memory tree, counting requests so tests can
+    /// assert that pruning actually avoids work.
+    struct FakeBackend {
+        tree: HashMap<String, Vec<RemoteEntry>>,
+        etags: HashMap<String, &'static str>,
+        caps: Capabilities,
+        lists: RefCell<usize>,
+        probes: RefCell<usize>,
+    }
+
+    // The fake is single-threaded; tests never share it across threads.
+    unsafe impl Sync for FakeBackend {}
+
+    fn dir(path: &str, etag: &str) -> RemoteEntry {
+        RemoteEntry {
+            id: RemoteId::Path(RemotePath::new(path)),
+            path: RemotePath::new(path),
+            kind: EntryKind::Directory,
+            validator: Validator::new(etag),
+            size: 0,
+            modified: None,
+            has_preview: false,
+        }
+    }
+
+    fn file(path: &str) -> RemoteEntry {
+        RemoteEntry {
+            id: RemoteId::Path(RemotePath::new(path)),
+            path: RemotePath::new(path),
+            kind: EntryKind::File,
+            validator: Validator::new("f"),
+            size: 1000,
+            modified: None,
+            has_preview: false,
+        }
+    }
+
+    impl FakeBackend {
+        /// Photos/{2025/{a.CR2,b.jpg}, 2026/{c.NEF,notes.txt}}
+        fn sample(change_detection: ChangeDetection) -> Self {
+            let mut tree = HashMap::new();
+            tree.insert(
+                "Photos".into(),
+                vec![dir("Photos/2025", "e2025"), dir("Photos/2026", "e2026")],
+            );
+            tree.insert(
+                "Photos/2025".into(),
+                vec![file("Photos/2025/a.CR2"), file("Photos/2025/b.jpg")],
+            );
+            tree.insert(
+                "Photos/2026".into(),
+                vec![file("Photos/2026/c.NEF"), file("Photos/2026/notes.txt")],
+            );
+
+            let mut etags = HashMap::new();
+            etags.insert("Photos".to_string(), "root");
+            etags.insert("Photos/2025".to_string(), "e2025");
+            etags.insert("Photos/2026".to_string(), "e2026");
+
+            Self {
+                tree,
+                etags,
+                caps: Capabilities {
+                    change_detection,
+                    stable_ids: true,
+                    range_reads: true,
+                    chunked_upload: None,
+                    bulk_upload: false,
+                    conditional_write: true,
+                    server_previews: ServerPreviews::None,
+                },
+                lists: RefCell::new(0),
+                probes: RefCell::new(0),
+            }
+        }
+    }
+
+    #[async_trait]
+    impl RemoteBackend for FakeBackend {
+        fn capabilities(&self) -> &Capabilities {
+            &self.caps
+        }
+        fn name(&self) -> &str {
+            "fake"
+        }
+        async fn list(
+            &self,
+            dir: &RemotePath,
+            _since: Option<&Validator>,
+        ) -> Result<Vec<RemoteEntry>, RemoteError> {
+            *self.lists.borrow_mut() += 1;
+            Ok(self.tree.get(dir.as_str()).cloned().unwrap_or_default())
+        }
+        async fn dir_validator(&self, dir: &RemotePath) -> Result<Validator, RemoteError> {
+            *self.probes.borrow_mut() += 1;
+            self.etags
+                .get(dir.as_str())
+                .map(|e| Validator::new(*e))
+                .ok_or_else(|| RemoteError::NotFound(dir.to_string()))
+        }
+        async fn delta(
+            &self,
+            _c: &crate::Cursor,
+        ) -> Result<(Vec<crate::RemoteChange>, crate::Cursor), RemoteError> {
+            Err(RemoteError::Unsupported("fake"))
+        }
+        async fn get(
+            &self,
+            _id: &RemoteId,
+            _r: Option<Range<u64>>,
+        ) -> Result<Vec<u8>, RemoteError> {
+            Ok(Vec::new())
+        }
+        async fn put(
+            &self,
+            _p: &RemotePath,
+            _b: Vec<u8>,
+            _pc: Option<crate::Precondition>,
+        ) -> Result<Validator, RemoteError> {
+            Err(RemoteError::Unsupported("fake"))
+        }
+        async fn delete(
+            &self,
+            _id: &RemoteId,
+            _pc: Option<crate::Precondition>,
+        ) -> Result<(), RemoteError> {
+            Err(RemoteError::Unsupported("fake"))
+        }
+    }
+
+    #[tokio::test]
+    async fn finds_images_recursively() {
+        let b = FakeBackend::sample(ChangeDetection::PropagatingEtags);
+        let r = scan(
+            &b,
+            &RemotePath::new("Photos"),
+            &FormatFilter::all(),
+            &HashMap::new(),
+            |_| {},
+        )
+        .await
+        .unwrap();
+
+        // notes.txt is not an image; the other three are.
+        assert_eq!(r.images.len(), 3);
+        assert_eq!(r.progress.images_found, 3);
+        assert_eq!(r.progress.directories_listed, 3);
+    }
+
+    #[tokio::test]
+    async fn the_format_filter_is_applied() {
+        let b = FakeBackend::sample(ChangeDetection::PropagatingEtags);
+        let r = scan(
+            &b,
+            &RemotePath::new("Photos"),
+            &FormatFilter::from_formats([dr_types::Format::Cr2]),
+            &HashMap::new(),
+            |_| {},
+        )
+        .await
+        .unwrap();
+
+        assert_eq!(r.images.len(), 1);
+        assert_eq!(r.images[0].path.name(), "a.CR2");
+    }
+
+    #[tokio::test]
+    async fn unchanged_subtrees_are_pruned() {
+        let b = FakeBackend::sample(ChangeDetection::PropagatingEtags);
+        let mut known = HashMap::new();
+        known.insert(RemotePath::new("Photos/2025"), Validator::new("e2025"));
+
+        let r = scan(
+            &b,
+            &RemotePath::new("Photos"),
+            &FormatFilter::all(),
+            &known,
+            |_| {},
+        )
+        .await
+        .unwrap();
+
+        // 2025 was proven unchanged by a single probe, so it was never listed
+        // and its files were not re-enumerated.
+        assert_eq!(r.progress.directories_pruned, 1);
+        assert_eq!(r.progress.directories_listed, 2);
+        assert!(r.images.iter().all(|i| !i.path.as_str().contains("2025")));
+    }
+
+    #[tokio::test]
+    async fn a_changed_etag_defeats_pruning() {
+        let b = FakeBackend::sample(ChangeDetection::PropagatingEtags);
+        let mut known = HashMap::new();
+        known.insert(RemotePath::new("Photos/2025"), Validator::new("stale"));
+
+        let r = scan(
+            &b,
+            &RemotePath::new("Photos"),
+            &FormatFilter::all(),
+            &known,
+            |_| {},
+        )
+        .await
+        .unwrap();
+
+        assert_eq!(r.progress.directories_pruned, 0);
+        assert_eq!(r.images.len(), 3);
+    }
+
+    #[tokio::test]
+    async fn pruning_is_not_attempted_without_propagating_etags() {
+        // Per-entry ETags say nothing about children, so probing would cost a
+        // request and prove nothing.
+        let b = FakeBackend::sample(ChangeDetection::LocalEtags);
+        let mut known = HashMap::new();
+        known.insert(RemotePath::new("Photos/2025"), Validator::new("e2025"));
+
+        let r = scan(
+            &b,
+            &RemotePath::new("Photos"),
+            &FormatFilter::all(),
+            &known,
+            |_| {},
+        )
+        .await
+        .unwrap();
+
+        assert_eq!(*b.probes.borrow(), 0, "must not probe");
+        assert_eq!(r.progress.directories_pruned, 0);
+        assert_eq!(r.images.len(), 3);
+    }
+
+    #[tokio::test]
+    async fn directory_etags_are_returned_for_persistence() {
+        // Without these the next scan has nothing to compare and prunes
+        // nothing (ARCH §6.6).
+        let b = FakeBackend::sample(ChangeDetection::PropagatingEtags);
+        let r = scan(
+            &b,
+            &RemotePath::new("Photos"),
+            &FormatFilter::all(),
+            &HashMap::new(),
+            |_| {},
+        )
+        .await
+        .unwrap();
+
+        assert_eq!(r.directories.len(), 2);
+        assert!(r
+            .directories
+            .iter()
+            .any(|(p, v)| p.as_str() == "Photos/2025" && v.as_str() == "e2025"));
+    }
+
+    #[tokio::test]
+    async fn results_are_ordered_reproducibly() {
+        let b = FakeBackend::sample(ChangeDetection::PropagatingEtags);
+        let r = scan(
+            &b,
+            &RemotePath::new("Photos"),
+            &FormatFilter::all(),
+            &HashMap::new(),
+            |_| {},
+        )
+        .await
+        .unwrap();
+
+        let paths: Vec<&str> = r.images.iter().map(|i| i.path.as_str()).collect();
+        let mut sorted = paths.clone();
+        sorted.sort();
+        assert_eq!(paths, sorted);
+    }
+
+    #[tokio::test]
+    async fn progress_is_reported_per_directory() {
+        let b = FakeBackend::sample(ChangeDetection::PropagatingEtags);
+        let mut updates = Vec::new();
+        scan(
+            &b,
+            &RemotePath::new("Photos"),
+            &FormatFilter::all(),
+            &HashMap::new(),
+            |p| updates.push(p),
+        )
+        .await
+        .unwrap();
+
+        // One per directory visited, so a long scan never looks hung.
+        assert_eq!(updates.len(), 3);
+        assert_eq!(updates.last().unwrap().images_found, 3);
+    }
+
+    #[tokio::test]
+    async fn an_empty_filter_finds_nothing_but_still_walks() {
+        let b = FakeBackend::sample(ChangeDetection::PropagatingEtags);
+        let r = scan(
+            &b,
+            &RemotePath::new("Photos"),
+            &FormatFilter::from_formats([]),
+            &HashMap::new(),
+            |_| {},
+        )
+        .await
+        .unwrap();
+
+        assert!(r.images.is_empty());
+        // The walk still happened, so directory ETags are still collected.
+        assert_eq!(r.directories.len(), 2);
+    }
+}