DarkRoom/core/dr-sync/src/scan.rs

//! Recursive discovery of images under a chosen remote folder.
//!
//! The library-setup path: the user picks a folder, ticks the formats they
//! shoot, and this walks the tree finding matching files (FR-CAT-1, M-5).
//!
//! Depth:1 per directory, never `Depth: infinity` — the latter is frequently
//! disabled and prohibitively expensive where it is not (ARCH §8.4). Where the
//! backend propagates directory ETags, an unchanged subtree is skipped whole,
//! which is what keeps a re-scan proportional to what changed rather than to
//! library size.

use std::collections::HashMap;

use dr_types::FormatFilter;

use crate::{
    Capabilities, ChangeDetection, EntryKind, RemoteBackend, RemoteEntry, RemoteError, RemotePath,
    Validator,
};

/// Progress during a scan, so the UI can show something on a large library.
#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
pub struct ScanProgress {
    pub directories_listed: usize,
    /// Directories skipped because their ETag was unchanged. The value of
    /// pruning, made visible.
    pub directories_pruned: usize,
    pub images_found: usize,
}

/// TRACES: FR-CAT-8 | FR-NC-9
/// Extension of a DarkRoom sidecar, as it appears in a listing.
///
/// Mirrors `dr_pipeline::sidecar::EXTENSION`. Duplicated rather than shared for
/// the reason [`TRASH_DIR`] is duplicated in `dr_catalog`: this crate depends
/// on nothing above it, and one `const` is a smaller price than a dependency
/// on the whole edit format to recognise four characters in a filename.
pub const SIDECAR_EXTENSION: &str = "drsc";

/// TRACES: FR-CAT-13
/// Extension of a standard XMP sidecar — Lightroom's `IMG_0001.xmp`,
/// darktable's `IMG_0001.CR3.xmp`, and every other editor's.
///
/// Collected alongside DarkRoom's own for the same reason and at the same
/// cost: it is in the listing already, and it is the file the ratings and
/// keywords of a library edited elsewhere are in. Which images a given
/// `.xmp` describes is the catalog's question, since the two naming
/// conventions resolve differently and only the catalog knows the images.
pub const XMP_EXTENSION: &str = "xmp";

/// The result of a scan.
#[derive(Debug, Clone, Default)]
pub struct ScanResult {
    /// Files matching the format filter.
    pub images: Vec<RemoteEntry>,
    /// TRACES: FR-CAT-8 | FR-NC-9
    /// Sidecars seen in the same listings, with the ETag they had.
    ///
    /// **Collected here because it is free.** A sidecar is a file in the same
    /// directory as the photograph it describes, so every one of them is
    /// already in a `PROPFIND` response this walk has paid for. Discovering
    /// them any other way — a probe per image — would be one request per
    /// photograph over a link that may be mobile data, which is why the pull
    /// did not exist at all before this.
    ///
    /// The validator is what makes the pull incremental: a sidecar whose ETag
    /// is unchanged since the last scan holds nothing this device has not
    /// already read, and is not fetched. ETag pruning means an untouched
    /// subtree is never even listed, so a library nobody has edited costs
    /// nothing.
    pub sidecars: Vec<RemoteEntry>,
    /// Directories whose contents were **actually listed**, with the ETag
    /// observed at that moment.
    ///
    /// **Must be persisted**, or every scan is a full walk (ARCH §6.6).
    ///
    /// Only listed directories appear here, and that distinction is
    /// load-bearing. A directory discovered as a child of another is *known*
    /// but not yet *read*: storing its ETag then would let the next scan prune
    /// a subtree whose contents were never seen, hiding every file beneath it
    /// permanently. Storing it only after listing means an interrupted scan
    /// re-reads that folder next time — slower, and correct.
    pub directories: Vec<(RemotePath, Validator)>,
    pub progress: ScanProgress,
}

/// How deep to recurse before giving up.
///
/// A symlink loop or a pathological tree would otherwise walk forever. Real
/// photo libraries are nowhere near this deep.
const MAX_DEPTH: usize = 32;

/// TRACES: FR-CAT-15
/// Directory name holding soft-deleted images.
///
/// Inside the library root rather than beside it: the root is the only place the
/// user granted access to, and on Nextcloud a `MOVE` out of it may cross a share
/// boundary the account cannot write to.
///
/// Leading dot so other tools treat it as hidden, and a name specific enough not
/// to collide with a photographer's own folder — "Trash" alone is a plausible
/// album title.
pub const TRASH_DIR: &str = ".darkroom-trash";

/// TRACES: FR-CAT-3
/// Directory name holding derived state pushed to the server — thumbnail
/// shards and the catalog snapshot.
///
/// Excluded from the scan for the same reason as the trash, though for a
/// different failure: its contents are `.sqlite` files that no format filter
/// would match, so nothing would be *indexed*, but the walk would still pay a
/// listing for it on every sync of every device.
pub const DERIVED_DIR: &str = ".darkroom-derived";

/// Whether a directory should be skipped by the scan.
///
/// **The trash must be excluded or the soft delete does not hold.** Trashed
/// images live in a real folder under the library root, so a scan that walked it
/// would re-index them as ordinary photographs and they would reappear in the
/// grid — the delete undone by the next refresh.
///
/// Matched on the final path component, not a prefix: a photograph in
/// `2019/.darkroom-trash/` (a nested library moved in wholesale) is excluded on
/// the same grounds as one at the root.
pub fn is_excluded(dir: &RemotePath) -> bool {
    matches!(dir.name(), TRASH_DIR | DERIVED_DIR)
}

/// TRACES: FR-CAT-1 | FR-NC-4 | M-5 | M-7
/// Walk `root` recursively, collecting files the filter accepts.
///
/// `known` maps previously seen directories to their ETags. Pass an empty map
/// for a first scan; pass the stored ETags to prune unchanged subtrees.
///
/// `on_progress` is called after each directory so a long scan can report
/// rather than appear hung.
pub async fn scan<B, F>(
    backend: &B,
    root: &RemotePath,
    filter: &FormatFilter,
    known: &HashMap<RemotePath, Validator>,
    mut on_progress: F,
) -> Result<ScanResult, RemoteError>
where
    B: RemoteBackend + ?Sized,
    F: FnMut(ScanProgress),
{
    let prunable = supports_pruning(backend.capabilities());
    let mut result = ScanResult::default();

    // Explicit stack rather than recursion: an async recursive fn needs
    // boxing, and a deep tree could overflow.
    //
    // Each item carries the validator its *parent* reported, so it can be
    // recorded once the directory has actually been listed. The root has none
    // until it is probed.
    let mut stack: Vec<(RemotePath, usize, Option<Validator>)> = vec![(root.clone(), 0, None)];

    while let Some((dir, depth, seen_validator)) = stack.pop() {
        if depth > MAX_DEPTH {
            log::warn!("scan: depth limit at {dir}, not descending further");
            continue;
        }

        // The trash, skipped before it is listed or even probed. Trashed images
        // are real files in a real folder under the root (FR-CAT-15), so
        // walking it would re-index them and undo the delete on the next scan.
        //
        // Deliberately *not* recorded in `result.directories`: an excluded
        // directory has no ETag worth storing, and storing one would let a
        // later scan believe it had been read.
        if is_excluded(&dir) {
            log::debug!("scan: skipping {dir}");
            continue;
        }

        // Prune: if the directory's ETag is unchanged, nothing anywhere
        // beneath it changed either, because Nextcloud propagates upward.
        let mut current_validator = seen_validator;
        if prunable {
            let known_validator = known.get(&dir);
            // Probe when there is something to compare against, and also when
            // this directory arrived without a validator — the root, which no
            // parent listing described. Without that second case the root's
            // ETag is never recorded, and the one-request no-op sync that
            // ETag pruning exists for can never fire.
            if known_validator.is_some() || current_validator.is_none() {
                match backend.dir_validator(&dir).await {
                    Ok(current) => {
                        if known_validator == Some(&current) {
                            result.progress.directories_pruned += 1;
                            // Re-record it: the entry must survive this scan,
                            // or the next one has nothing to compare against
                            // and walks the whole subtree again.
                            result.directories.push((dir.clone(), current));
                            on_progress(result.progress);
                            continue;
                        }
                        current_validator = Some(current);
                    }
                    // A probe failure is not fatal — fall through to listing,
                    // which is correct, just not free.
                    Err(e) => log::debug!("scan: validator probe failed for {dir}: {e}"),
                }
            }
        }

        let entries = match backend.list(&dir, None).await {
            Ok(e) => e,

            // TRACES: FR-PLAT-AND-2 | FR-CAT-9
            // The root is the one directory the walk may not step over, and
            // `depth == 0` is the only place it can be — nothing is ever
            // pushed at that depth but the root itself.
            //
            // Below, a directory that has gone is a directory that went away
            // between its parent being listed and it being reached, and
            // continuing is right. At the root the identical error means the
            // *library* is gone, and continuing is catastrophic in a way that
            // is completely silent: the walk ends, the scan succeeds having
            // found nothing, and the app reports a healthy library with no new
            // images while every path in the catalog now points nowhere.
            //
            // Refused rather than reclassified. Only these two causes are —
            // a `Network` failure at the root is still a network failure, and
            // must stay one or an unplugged network cable would present itself
            // as a revoked permission and offline mode would never engage.
            Err(RemoteError::NotFound(_)) if depth == 0 => {
                return Err(RemoteError::RootUnavailable(format!(
                    "{root} is no longer there"
                )));
            }
            Err(RemoteError::PermissionDenied) if depth == 0 => {
                // Deliberately not the variant's own message, which describes
                // a Nextcloud share that refuses to *update* a sidecar. At the
                // root nothing has been read at all.
                return Err(RemoteError::RootUnavailable(format!(
                    "{root} can no longer be read"
                )));
            }
            Err(RemoteError::NotFound(_)) => {
                // Deleted between listing its parent and reaching it.
                log::debug!("scan: {dir} vanished during the walk");
                continue;
            }
            Err(e) => return Err(e),
        };

        result.progress.directories_listed += 1;

        // Only now that the contents have been read is it safe to record this
        // directory's ETag. Recording it at discovery time would let a later
        // scan prune a subtree that was never actually listed, hiding every
        // file beneath it.
        if let Some(v) = current_validator {
            result.directories.push((dir.clone(), v));
        }

        for entry in entries {
            match entry.kind {
                EntryKind::Directory => {
                    // The validator travels with it, to be recorded when the
                    // child is listed — not here.
                    stack.push((entry.path, depth + 1, Some(entry.validator)));
                }
                EntryKind::File => {
                    if filter.allows_name(entry.path.name()) {
                        result.images.push(entry);
                        result.progress.images_found += 1;
                    } else if is_sidecar(entry.path.name()) {
                        // Not counted in `images_found`: the progress figure is
                        // what the user is shown, and it means photographs.
                        result.sidecars.push(entry);
                    }
                }
            }
        }

        on_progress(result.progress);
    }

    // Sort so a scan is reproducible and the grid has a stable order.
    result.images.sort_by(|a, b| a.path.cmp(&b.path));
    result.sidecars.sort_by(|a, b| a.path.cmp(&b.path));
    result.directories.sort_by(|a, b| a.0.cmp(&b.0));
    Ok(result)
}

/// Whether a filename is a sidecar — DarkRoom's own, or a standard XMP one.
///
/// Case-insensitive on the extension alone. A server that upper-cased the
/// suffix — or a file copied through a filesystem that did — still describes a
/// photograph, and failing to recognise it would silently lose the edit rather
/// than fail visibly.
fn is_sidecar(name: &str) -> bool {
    name.rsplit_once('.').is_some_and(|(stem, ext)| {
        !stem.is_empty()
            && (ext.eq_ignore_ascii_case(SIDECAR_EXTENSION)
                || ext.eq_ignore_ascii_case(XMP_EXTENSION))
    })
}

/// Whether pruning is worth attempting against this backend.
///
/// Only propagating ETags make an unchanged parent prove an unchanged
/// subtree. With per-entry ETags the probe costs a request and proves
/// nothing about children, so it is pure overhead.
fn supports_pruning(caps: &Capabilities) -> bool {
    matches!(caps.change_detection, ChangeDetection::PropagatingEtags)
}

#[cfg(test)]
mod tests {
    use super::*;
    use crate::{RemoteId, ServerPreviews};
    use async_trait::async_trait;
    use std::cell::RefCell;
    use std::ops::Range;

    /// A backend over an in-memory tree, counting requests so tests can
    /// assert that pruning actually avoids work.
    struct FakeBackend {
        tree: HashMap<String, Vec<RemoteEntry>>,
        etags: HashMap<String, &'static str>,
        caps: Capabilities,
        lists: RefCell<usize>,
        probes: RefCell<usize>,
        /// Directories whose listing fails, and how.
        ///
        /// An absent directory is not enough to model this: the fake answers
        /// an unknown path with an empty listing, which is exactly the shape
        /// the walk must *not* confuse with a library that has gone away.
        deny: HashMap<String, Deny>,
    }

    /// The two ways a real backend refuses a directory that is still named in
    /// the catalog: it is not there, or it may not be read.
    #[derive(Clone, Copy)]
    enum Deny {
        Missing,
        Forbidden,
    }

    // The fake is single-threaded; tests never share it across threads.
    unsafe impl Sync for FakeBackend {}

    fn dir(path: &str, etag: &str) -> RemoteEntry {
        RemoteEntry {
            id: RemoteId::Path(RemotePath::new(path)),
            path: RemotePath::new(path),
            kind: EntryKind::Directory,
            validator: Validator::new(etag),
            size: 0,
            modified: None,
            has_preview: false,
            materialised: true,
        }
    }

    fn file(path: &str) -> RemoteEntry {
        RemoteEntry {
            id: RemoteId::Path(RemotePath::new(path)),
            path: RemotePath::new(path),
            kind: EntryKind::File,
            validator: Validator::new("f"),
            size: 1000,
            modified: None,
            has_preview: false,
            materialised: true,
        }
    }

    impl FakeBackend {
        /// Photos/{2025/{a.CR2,b.jpg}, 2026/{c.NEF,notes.txt}}
        fn sample(change_detection: ChangeDetection) -> Self {
            let mut tree = HashMap::new();
            tree.insert(
                "Photos".into(),
                vec![dir("Photos/2025", "e2025"), dir("Photos/2026", "e2026")],
            );
            tree.insert(
                "Photos/2025".into(),
                vec![file("Photos/2025/a.CR2"), file("Photos/2025/b.jpg")],
            );
            tree.insert(
                "Photos/2026".into(),
                vec![file("Photos/2026/c.NEF"), file("Photos/2026/notes.txt")],
            );

            let mut etags = HashMap::new();
            etags.insert("Photos".to_string(), "root");
            etags.insert("Photos/2025".to_string(), "e2025");
            etags.insert("Photos/2026".to_string(), "e2026");

            Self {
                tree,
                etags,
                caps: Capabilities {
                    change_detection,
                    stable_ids: true,
                    range_reads: true,
                    chunked_upload: None,
                    bulk_upload: false,
                    conditional_write: true,
                    server_previews: ServerPreviews::None,
                    materialisation: crate::Materialisation::Always,
                },
                lists: RefCell::new(0),
                probes: RefCell::new(0),
                deny: HashMap::new(),
            }
        }

        /// Make one directory refuse to be listed.
        fn denying(mut self, path: &str, how: Deny) -> Self {
            self.deny.insert(path.to_string(), how);
            self
        }
    }

    #[async_trait]
    impl RemoteBackend for FakeBackend {
        fn capabilities(&self) -> &Capabilities {
            &self.caps
        }
        fn name(&self) -> &str {
            "fake"
        }
        async fn list(
            &self,
            dir: &RemotePath,
            _since: Option<&Validator>,
        ) -> Result<Vec<RemoteEntry>, RemoteError> {
            *self.lists.borrow_mut() += 1;
            match self.deny.get(dir.as_str()) {
                Some(Deny::Missing) => return Err(RemoteError::NotFound(dir.to_string())),
                Some(Deny::Forbidden) => return Err(RemoteError::PermissionDenied),
                None => {}
            }
            Ok(self.tree.get(dir.as_str()).cloned().unwrap_or_default())
        }
        async fn dir_validator(&self, dir: &RemotePath) -> Result<Validator, RemoteError> {
            *self.probes.borrow_mut() += 1;
            self.etags
                .get(dir.as_str())
                .map(|e| Validator::new(*e))
                .ok_or_else(|| RemoteError::NotFound(dir.to_string()))
        }
        async fn delta(
            &self,
            _c: &crate::Cursor,
        ) -> Result<(Vec<crate::RemoteChange>, crate::Cursor), RemoteError> {
            Err(RemoteError::Unsupported("fake"))
        }
        async fn get(
            &self,
            _id: &RemoteId,
            _r: Option<Range<u64>>,
        ) -> Result<Vec<u8>, RemoteError> {
            Ok(Vec::new())
        }
        async fn put(
            &self,
            _p: &RemotePath,
            _b: Vec<u8>,
            _pc: Option<crate::Precondition>,
        ) -> Result<Validator, RemoteError> {
            Err(RemoteError::Unsupported("fake"))
        }
        async fn delete(
            &self,
            _id: &RemoteId,
            _pc: Option<crate::Precondition>,
        ) -> Result<(), RemoteError> {
            Err(RemoteError::Unsupported("fake"))
        }

        async fn move_to(&self, _from: &RemoteId, _to: &RemotePath) -> Result<(), RemoteError> {
            Err(RemoteError::Unsupported("fake"))
        }

        async fn create_dir(&self, _path: &RemotePath) -> Result<(), RemoteError> {
            Err(RemoteError::Unsupported("fake"))
        }
    }

    #[test]
    fn the_derived_folder_is_excluded_like_the_trash() {
        // Its contents are .sqlite files no format filter would match, so
        // nothing would be *indexed* — but the walk would still pay a listing
        // for it on every sync of every device.
        assert!(is_excluded(&RemotePath::new("Photos/.darkroom-derived")));
        assert!(is_excluded(&RemotePath::new("Photos/.darkroom-trash")));
        assert!(!is_excluded(&RemotePath::new("Photos/2026")));
        // Matched on the final component, so a nested library moved in
        // wholesale is excluded on the same grounds.
        assert!(is_excluded(&RemotePath::new("a/b/.darkroom-derived")));
    }

    #[tokio::test]
    async fn finds_images_recursively() {
        let b = FakeBackend::sample(ChangeDetection::PropagatingEtags);
        let r = scan(
            &b,
            &RemotePath::new("Photos"),
            &FormatFilter::all(),
            &HashMap::new(),
            |_| {},
        )
        .await
        .unwrap();

        // notes.txt is not an image; the other three are.
        assert_eq!(r.images.len(), 3);
        assert_eq!(r.progress.images_found, 3);
        assert_eq!(r.progress.directories_listed, 3);
    }

    /// TRACES: FR-PLAT-AND-2 | FR-CAT-9
    #[tokio::test]
    async fn a_root_that_is_gone_is_a_failure_and_not_an_empty_library() {
        // The silent one. A vanished directory below the root is stepped over,
        // and before this the root was stepped over on the same terms — which
        // ended the walk immediately, returned `Ok` with nothing in it, and
        // let the app report a successful scan of a library that no longer
        // exists. Nothing in that path is ever told the library went away, so
        // nothing marks it offline and nothing tells the user.
        let b =
            FakeBackend::sample(ChangeDetection::PropagatingEtags).denying("Photos", Deny::Missing);
        let e = scan(
            &b,
            &RemotePath::new("Photos"),
            &FormatFilter::all(),
            &HashMap::new(),
            |_| {},
        )
        .await
        .expect_err("a library that is not there is not a library with no photographs");

        assert!(e.indicates_lost_root(), "got {e:?}");
        assert!(!e.indicates_offline(), "waiting will not bring this back");
        assert!(e.to_string().contains("Photos"), "names the library: {e}");
    }

    /// TRACES: FR-PLAT-AND-2
    #[tokio::test]
    async fn a_root_that_may_not_be_read_reports_the_root_and_not_the_share_advice() {
        // A Nextcloud share withdrawn, a directory the process may no longer
        // read — and the shape a revoked Android tree grant will have when one
        // can be held at all. Reported as plain `PermissionDenied` it would
        // have carried that variant's message, which is several lines about a
        // Nextcloud share refusing to *update* an existing sidecar: advice for
        // a case where reads work, offered to a user whose reads have stopped
        // entirely.
        let b = FakeBackend::sample(ChangeDetection::PropagatingEtags)
            .denying("Photos", Deny::Forbidden);
        let e = scan(
            &b,
            &RemotePath::new("Photos"),
            &FormatFilter::all(),
            &HashMap::new(),
            |_| {},
        )
        .await
        .expect_err("a root that cannot be read is a failed scan");

        assert!(e.indicates_lost_root(), "got {e:?}");
        assert!(
            !e.to_string().contains("sidecar"),
            "the share-permission advice does not belong here: {e}"
        );
    }

    /// TRACES: FR-PLAT-AND-2
    #[tokio::test]
    async fn a_folder_that_goes_away_below_the_root_is_still_stepped_over() {
        // The other side of the split, and the reason the root is keyed on
        // depth rather than on the error. A subfolder deleted between its
        // parent being listed and it being reached is ordinary, and failing
        // the scan over it would abandon every photograph beside it.
        let b = FakeBackend::sample(ChangeDetection::PropagatingEtags)
            .denying("Photos/2025", Deny::Missing);
        let r = scan(
            &b,
            &RemotePath::new("Photos"),
            &FormatFilter::all(),
            &HashMap::new(),
            |_| {},
        )
        .await
        .expect("one folder going away is not the library going away");

        assert_eq!(r.images.len(), 1, "2026 was still walked");
    }

    /// A library with a trash folder holding a soft-deleted image.
    fn with_trash() -> FakeBackend {
        let mut b = FakeBackend::sample(ChangeDetection::PropagatingEtags);
        b.tree.insert(
            "Photos".into(),
            vec![
                dir("Photos/2025", "e2025"),
                dir("Photos/2026", "e2026"),
                dir("Photos/.darkroom-trash", "etrash"),
            ],
        );
        b.tree.insert(
            "Photos/.darkroom-trash".into(),
            vec![file("Photos/.darkroom-trash/9-deleted.CR2")],
        );
        b.etags
            .insert("Photos/.darkroom-trash".to_string(), "etrash");
        b
    }

    #[tokio::test]
    async fn the_trash_folder_is_never_scanned() {
        // TRACES: FR-CAT-15
        // The other half of the soft delete: a scan that walked the trash would
        // re-index every trashed photograph and undo the delete on the next
        // refresh.
        let b = with_trash();
        let result = scan(
            &b,
            &RemotePath::new("Photos"),
            &FormatFilter::default(),
            &HashMap::new(),
            |_| {},
        )
        .await
        .unwrap();

        assert!(
            !result
                .images
                .iter()
                .any(|i| i.path.as_str().contains(".darkroom-trash")),
            "a trashed image must not come back as an ordinary one"
        );
        // The real photographs are still found.
        assert!(result
            .images
            .iter()
            .any(|i| i.path.as_str().ends_with("a.CR2")));
    }

    #[tokio::test]
    async fn the_trash_folder_costs_no_requests() {
        // Not merely filtered out of the results — never listed and never
        // probed. A folder whose contents are deliberately invisible must not
        // cost a round trip on every scan.
        let b = with_trash();
        scan(
            &b,
            &RemotePath::new("Photos"),
            &FormatFilter::default(),
            &HashMap::new(),
            |_| {},
        )
        .await
        .unwrap();

        // Photos, Photos/2025, Photos/2026 — and not the trash, which would
        // make four.
        assert_eq!(*b.lists.borrow(), 3, "the trash was listed");
    }

    #[tokio::test]
    async fn the_trash_folder_gets_no_stored_etag() {
        // Storing one would let a later scan believe the folder had been read,
        // which is the failure the `directories` doc comment warns about.
        let b = with_trash();
        let result = scan(
            &b,
            &RemotePath::new("Photos"),
            &FormatFilter::default(),
            &HashMap::new(),
            |_| {},
        )
        .await
        .unwrap();

        assert!(!result
            .directories
            .iter()
            .any(|(p, _)| p.as_str().contains(".darkroom-trash")));
    }

    #[test]
    fn exclusion_matches_the_folder_name_at_any_depth() {
        // A nested library moved in wholesale carries its own trash.
        assert!(is_excluded(&RemotePath::new("Photos/.darkroom-trash")));
        assert!(is_excluded(&RemotePath::new("a/b/c/.darkroom-trash")));
        assert!(is_excluded(&RemotePath::new(".darkroom-trash")));
        // And nothing else is swept up by it.
        assert!(!is_excluded(&RemotePath::new("Photos/2025")));
        assert!(!is_excluded(&RemotePath::new("Photos/trash")));
        assert!(!is_excluded(&RemotePath::new("Photos/.darkroom-trash-old")));
    }

    #[tokio::test]
    async fn the_format_filter_is_applied() {
        let b = FakeBackend::sample(ChangeDetection::PropagatingEtags);
        let r = scan(
            &b,
            &RemotePath::new("Photos"),
            &FormatFilter::from_formats([dr_types::Format::Cr2]),
            &HashMap::new(),
            |_| {},
        )
        .await
        .unwrap();

        assert_eq!(r.images.len(), 1);
        assert_eq!(r.images[0].path.name(), "a.CR2");
    }

    #[tokio::test]
    async fn unchanged_subtrees_are_pruned() {
        let b = FakeBackend::sample(ChangeDetection::PropagatingEtags);
        let mut known = HashMap::new();
        known.insert(RemotePath::new("Photos/2025"), Validator::new("e2025"));

        let r = scan(
            &b,
            &RemotePath::new("Photos"),
            &FormatFilter::all(),
            &known,
            |_| {},
        )
        .await
        .unwrap();

        // 2025 was proven unchanged by a single probe, so it was never listed
        // and its files were not re-enumerated.
        assert_eq!(r.progress.directories_pruned, 1);
        assert_eq!(r.progress.directories_listed, 2);
        assert!(r.images.iter().all(|i| !i.path.as_str().contains("2025")));
    }

    #[tokio::test]
    async fn a_changed_etag_defeats_pruning() {
        let b = FakeBackend::sample(ChangeDetection::PropagatingEtags);
        let mut known = HashMap::new();
        known.insert(RemotePath::new("Photos/2025"), Validator::new("stale"));

        let r = scan(
            &b,
            &RemotePath::new("Photos"),
            &FormatFilter::all(),
            &known,
            |_| {},
        )
        .await
        .unwrap();

        assert_eq!(r.progress.directories_pruned, 0);
        assert_eq!(r.images.len(), 3);
    }

    #[tokio::test]
    async fn pruning_is_not_attempted_without_propagating_etags() {
        // Per-entry ETags say nothing about children, so probing would cost a
        // request and prove nothing.
        let b = FakeBackend::sample(ChangeDetection::LocalEtags);
        let mut known = HashMap::new();
        known.insert(RemotePath::new("Photos/2025"), Validator::new("e2025"));

        let r = scan(
            &b,
            &RemotePath::new("Photos"),
            &FormatFilter::all(),
            &known,
            |_| {},
        )
        .await
        .unwrap();

        assert_eq!(*b.probes.borrow(), 0, "must not probe");
        assert_eq!(r.progress.directories_pruned, 0);
        assert_eq!(r.images.len(), 3);
    }

    #[tokio::test]
    async fn directory_etags_are_returned_for_persistence() {
        // Without these the next scan has nothing to compare and prunes
        // nothing (ARCH §6.6).
        let b = FakeBackend::sample(ChangeDetection::PropagatingEtags);
        let r = scan(
            &b,
            &RemotePath::new("Photos"),
            &FormatFilter::all(),
            &HashMap::new(),
            |_| {},
        )
        .await
        .unwrap();

        // Three: the root and its two children. The root matters most — its
        // ETag is what turns the next no-op sync into a single request.
        assert_eq!(r.directories.len(), 3);
        assert!(r
            .directories
            .iter()
            .any(|(p, v)| p.as_str() == "Photos/2025" && v.as_str() == "e2025"));
        assert!(
            r.directories.iter().any(|(p, _)| p.as_str() == "Photos"),
            "the scan root must be recorded, or pruning can never start there"
        );
    }

    #[tokio::test]
    async fn only_listed_directories_are_recorded() {
        // The bug this guards against permanently hid files: recording a
        // directory's ETag when it was *discovered* rather than when it was
        // *listed* meant a scan that stopped early still stored ETags for
        // subtrees it never read. The next scan then probed those ETags,
        // found them unchanged, and pruned folders whose contents had never
        // been seen — so their images never entered the catalog at all.
        let b = FakeBackend::sample(ChangeDetection::PropagatingEtags);
        let r = scan(
            &b,
            &RemotePath::new("Photos"),
            &FormatFilter::all(),
            &HashMap::new(),
            |_| {},
        )
        .await
        .unwrap();

        // Every recorded directory must be one that was actually listed.
        for (path, _) in &r.directories {
            assert!(
                b.tree.contains_key(path.as_str()),
                "{path} was recorded without being listed"
            );
        }
        assert_eq!(r.progress.directories_listed, r.directories.len());
    }

    #[tokio::test]
    async fn a_pruned_directory_keeps_its_recorded_etag() {
        // Pruning must not drop the entry: without it the next scan has
        // nothing to compare and re-walks the whole subtree, turning every
        // sync back into a full walk.
        let b = FakeBackend::sample(ChangeDetection::PropagatingEtags);
        let mut known = HashMap::new();
        known.insert(RemotePath::new("Photos/2025"), Validator::new("e2025"));

        let r = scan(
            &b,
            &RemotePath::new("Photos"),
            &FormatFilter::all(),
            &known,
            |_| {},
        )
        .await
        .unwrap();

        assert_eq!(r.progress.directories_pruned, 1);
        assert!(
            r.directories
                .iter()
                .any(|(p, v)| p.as_str() == "Photos/2025" && v.as_str() == "e2025"),
            "a pruned directory must still be recorded for the next scan"
        );
    }

    #[tokio::test]
    async fn results_are_ordered_reproducibly() {
        let b = FakeBackend::sample(ChangeDetection::PropagatingEtags);
        let r = scan(
            &b,
            &RemotePath::new("Photos"),
            &FormatFilter::all(),
            &HashMap::new(),
            |_| {},
        )
        .await
        .unwrap();

        let paths: Vec<&str> = r.images.iter().map(|i| i.path.as_str()).collect();
        let mut sorted = paths.clone();
        sorted.sort();
        assert_eq!(paths, sorted);
    }

    #[tokio::test]
    async fn progress_is_reported_per_directory() {
        let b = FakeBackend::sample(ChangeDetection::PropagatingEtags);
        let mut updates = Vec::new();
        scan(
            &b,
            &RemotePath::new("Photos"),
            &FormatFilter::all(),
            &HashMap::new(),
            |p| updates.push(p),
        )
        .await
        .unwrap();

        // One per directory visited, so a long scan never looks hung.
        assert_eq!(updates.len(), 3);
        assert_eq!(updates.last().unwrap().images_found, 3);
    }

    #[tokio::test]
    async fn an_empty_filter_finds_nothing_but_still_walks() {
        let b = FakeBackend::sample(ChangeDetection::PropagatingEtags);
        let r = scan(
            &b,
            &RemotePath::new("Photos"),
            &FormatFilter::from_formats([]),
            &HashMap::new(),
            |_| {},
        )
        .await
        .unwrap();

        assert!(r.images.is_empty());
        // The walk still happened, so directory ETags are still collected —
        // the root plus its two children.
        assert_eq!(r.directories.len(), 3);
    }

    /// TRACES: FR-CAT-8 | FR-NC-9
    /// Sidecars are reported, so a judgement made elsewhere can be read back.
    ///
    /// They are already in every listing the walk pays for; collecting them
    /// costs no request. Discovering them any other way would be a probe per
    /// photograph, which is why nothing read them at all before this.
    #[tokio::test]
    async fn sidecars_are_collected_from_the_listings_the_walk_already_makes() {
        let mut b = FakeBackend::sample(ChangeDetection::PropagatingEtags);
        b.tree.insert(
            "Photos/2025".into(),
            vec![
                file("Photos/2025/a.CR2"),
                file("Photos/2025/a.drsc"),
                file("Photos/2025/b.jpg"),
                // Upper-cased by a filesystem somewhere along the way; still a
                // sidecar, and losing it would lose the edit silently.
                file("Photos/2025/b.DRSC"),
                // TRACES: FR-CAT-13
                // And the standard kind, in both of its spellings.
                file("Photos/2025/a.xmp"),
                file("Photos/2025/b.jpg.xmp"),
            ],
        );
        let before = *b.lists.borrow();

        let r = scan(
            &b,
            &RemotePath::new("Photos"),
            &FormatFilter::all(),
            &HashMap::new(),
            |_| {},
        )
        .await
        .unwrap();

        assert_eq!(
            r.sidecars
                .iter()
                .map(|e| e.path.as_str().to_string())
                .collect::<Vec<_>>(),
            vec![
                "Photos/2025/a.drsc",
                "Photos/2025/a.xmp",
                "Photos/2025/b.DRSC",
                "Photos/2025/b.jpg.xmp",
            ]
        );
        assert_eq!(*b.lists.borrow() - before, 3, "no extra requests");

        // And they are not photographs: the count the user is shown must not
        // double because a library has been edited.
        assert_eq!(r.progress.images_found, 3);
        assert!(r
            .images
            .iter()
            .all(|e| !e.path.name().contains("drsc") && !e.path.name().contains("xmp")));
    }

    /// A file whose *name* is only an extension is not a sidecar for anything.
    #[test]
    fn a_bare_extension_is_not_a_sidecar() {
        assert!(is_sidecar("a.drsc"));
        assert!(is_sidecar("a.DRSC"));
        assert!(is_sidecar("a.b.drsc"));
        assert!(!is_sidecar(".drsc"));
        assert!(!is_sidecar("drsc"));
        assert!(!is_sidecar("a.drsc.tmp"));
        assert!(!is_sidecar("a.CR2"));
    }
}