Add folder scan with format selection; validate A3 on a real library
Library setup as the user described it: pick a folder, choose which RAW
types to look for, scan recursively.
dr-types::FormatFilter the tick-box selection, seeing through VFS
placeholder suffixes so a dehydrated CR2 still
matches as a CR2
dr-sync::scan recursive walk, Depth:1 per directory, pruning
unchanged subtrees where the backend propagates
directory ETags
Verified against nextcloud.tourolle.paris (34.0.2) on a real library:
browse root 32 entries, 98ms
scan PhotosRaw 17,185 RAW files in 334 directories, 34.1s
(7,836 CR2 + 9,349 DNG)
range read 262KB of a 21.5MB DNG in 119ms — 1.22% of the file,
and enough to read "Canon EOS 6D | ISO 100"
That last line is assumption A3 validated on real data. Cataloguing this
library by whole-file fetch would move roughly 370GB; the range path
moves a few MB.
Pruning is capability-gated rather than assumed: with per-entry ETags a
probe costs a request and proves nothing about children, so it is skipped
entirely. A test asserts zero probes in that case.
Still unresolved: /core/preview returns 400 for every parameter
combination tried, including on a JPEG the server reports as having a
preview. Not a request-shape bug — it fails identically bare. Recorded
rather than worked around; ARCH §6.7 already treats server previews as
opportunistic, so nothing depends on it.
This commit is contained in:
@@ -0,0 +1,254 @@
|
||||
//! TRACES: FR-CAT-1 | FR-CAT-9 | NFR-P1
|
||||
//! Incremental scan: the local analogue of ETag pruning.
|
||||
//!
|
||||
//! Nextcloud propagates ETags up the tree, so one request proves a whole
|
||||
//! library unchanged (ARCH §8.4). A filesystem offers no such guarantee — a
|
||||
//! directory's mtime moves when its *direct* entries change and not when a
|
||||
//! grandchild does, so there is no cheap "did anything below here change"
|
||||
//! probe.
|
||||
//!
|
||||
//! Local scan therefore prunes at each level rather than at the root: one
|
||||
//! metadata probe per directory when nothing changed, instead of one per file.
|
||||
//! A 50k-image library in ~2k folders costs 2k probes, which is the difference
|
||||
//! between meeting and missing NFR-P1 on SAF.
|
||||
//!
|
||||
//! This module holds the decision logic and the deletion-sweep rules; walking
|
||||
//! an actual directory belongs to the platform layer, which supplies
|
||||
//! [`DirState`] and [`DirEntry`].
|
||||
|
||||
use dr_types::FormatFilter;
|
||||
|
||||
/// What a directory looked like when last scanned, and what it looks like now.
|
||||
///
|
||||
/// Both fields are cheap to obtain: one `stat` locally, one
|
||||
/// `DocumentsContract` metadata query on SAF.
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub struct DirState {
|
||||
pub mtime: i64,
|
||||
/// Direct children, files and directories alike.
|
||||
///
|
||||
/// mtime alone misses a delete-and-create inside one timestamp tick, and
|
||||
/// coarse-granularity providers widen that window. The count does not
|
||||
/// close the hole — a paired add and remove moves neither — but a bare add
|
||||
/// or remove moves the count, and those are far commoner.
|
||||
pub entry_count: u32,
|
||||
}
|
||||
|
||||
/// One entry from a directory listing.
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct DirEntry {
|
||||
pub name: String,
|
||||
pub is_dir: bool,
|
||||
pub size: u64,
|
||||
pub mtime: i64,
|
||||
}
|
||||
|
||||
/// What the scanner should do with a directory, before listing it.
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub enum DirAction {
|
||||
/// Contents unchanged. Skip the listing, but still recurse into known
|
||||
/// children — without upward propagation, a deep change is invisible from
|
||||
/// here.
|
||||
RecurseOnly,
|
||||
/// List and reconcile, then recurse.
|
||||
ListAndRecurse,
|
||||
}
|
||||
|
||||
/// Decide whether a directory needs listing.
|
||||
pub fn classify_dir(stored: Option<DirState>, current: DirState) -> DirAction {
|
||||
match stored {
|
||||
Some(s) if s == current => DirAction::RecurseOnly,
|
||||
_ => DirAction::ListAndRecurse,
|
||||
}
|
||||
}
|
||||
|
||||
/// What reconciling one listed entry against the catalog implies.
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub enum EntryAction {
|
||||
/// Not catalogued. Insert at `metadata_state = 1` and queue EXIF.
|
||||
Insert,
|
||||
/// Catalogued and unchanged. The common case, and it must cost nothing.
|
||||
Unchanged,
|
||||
/// Size or mtime moved: re-read metadata, rebuild the thumbnail, and drop
|
||||
/// the content hash, which is no longer valid.
|
||||
Changed,
|
||||
/// Recognised but not a format the user asked to scan for.
|
||||
Ignored,
|
||||
}
|
||||
|
||||
/// What the catalog already holds for a source.
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub struct KnownFile {
|
||||
pub size: u64,
|
||||
pub mtime: i64,
|
||||
}
|
||||
|
||||
/// Classify one listed file.
|
||||
pub fn classify_entry(
|
||||
entry: &DirEntry,
|
||||
known: Option<KnownFile>,
|
||||
formats: &FormatFilter,
|
||||
) -> EntryAction {
|
||||
if !formats.allows_name(&entry.name) {
|
||||
return EntryAction::Ignored;
|
||||
}
|
||||
match known {
|
||||
None => EntryAction::Insert,
|
||||
Some(k) if k.size == entry.size && k.mtime == entry.mtime => EntryAction::Unchanged,
|
||||
Some(_) => EntryAction::Changed,
|
||||
}
|
||||
}
|
||||
|
||||
/// Outcome of a scan, which decides whether pruning may run.
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub enum ScanOutcome {
|
||||
/// Every reachable folder was visited.
|
||||
Complete,
|
||||
/// The user cancelled. Partial state is valid — jobs are resumable — but
|
||||
/// unvisited folders must not be read as deleted.
|
||||
Cancelled,
|
||||
/// The root itself could not be opened: drive unplugged, SAF grant
|
||||
/// revoked, share unmounted.
|
||||
RootUnreachable,
|
||||
/// Some subtree failed while the root was fine.
|
||||
PartialFailure,
|
||||
}
|
||||
|
||||
impl ScanOutcome {
|
||||
/// Whether the deletion sweep may run.
|
||||
///
|
||||
/// **The most dangerous decision in the catalog.** The sweep deletes every
|
||||
/// folder not reached by this scan's generation. After an incomplete scan
|
||||
/// that is most of the library, so it runs only on `Complete`.
|
||||
///
|
||||
/// FR-CAT-9 draws exactly this line: a source *proven absent* may leave
|
||||
/// the catalog; a source merely *unreachable* is marked offline and kept,
|
||||
/// with its ratings and edits intact.
|
||||
pub fn may_prune(self) -> bool {
|
||||
matches!(self, ScanOutcome::Complete)
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use dr_types::Format;
|
||||
|
||||
const A: DirState = DirState {
|
||||
mtime: 100,
|
||||
entry_count: 5,
|
||||
};
|
||||
|
||||
#[test]
|
||||
fn unchanged_directory_is_not_listed() {
|
||||
assert_eq!(classify_dir(Some(A), A), DirAction::RecurseOnly);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_never_seen_directory_is_listed() {
|
||||
assert_eq!(classify_dir(None, A), DirAction::ListAndRecurse);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn changed_mtime_forces_a_listing() {
|
||||
let now = DirState { mtime: 101, ..A };
|
||||
assert_eq!(classify_dir(Some(A), now), DirAction::ListAndRecurse);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn entry_count_catches_what_mtime_misses() {
|
||||
// A file added within the same timestamp tick: mtime is unchanged, so
|
||||
// mtime alone would skip this directory and lose the new image.
|
||||
let now = DirState {
|
||||
mtime: 100,
|
||||
entry_count: 6,
|
||||
};
|
||||
assert_eq!(classify_dir(Some(A), now), DirAction::ListAndRecurse);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn unchanged_file_costs_nothing() {
|
||||
let e = DirEntry {
|
||||
name: "IMG_0001.CR3".into(),
|
||||
is_dir: false,
|
||||
size: 30_000_000,
|
||||
mtime: 500,
|
||||
};
|
||||
let known = KnownFile {
|
||||
size: 30_000_000,
|
||||
mtime: 500,
|
||||
};
|
||||
assert_eq!(
|
||||
classify_entry(&e, Some(known), &FormatFilter::all()),
|
||||
EntryAction::Unchanged
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_resaved_file_is_reprocessed() {
|
||||
let e = DirEntry {
|
||||
name: "IMG_0001.CR3".into(),
|
||||
is_dir: false,
|
||||
size: 30_000_001,
|
||||
mtime: 900,
|
||||
};
|
||||
let known = KnownFile {
|
||||
size: 30_000_000,
|
||||
mtime: 500,
|
||||
};
|
||||
assert_eq!(
|
||||
classify_entry(&e, Some(known), &FormatFilter::all()),
|
||||
EntryAction::Changed
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn format_filter_excludes_unwanted_types() {
|
||||
let jpeg = DirEntry {
|
||||
name: "IMG_0001.JPG".into(),
|
||||
is_dir: false,
|
||||
size: 1,
|
||||
mtime: 1,
|
||||
};
|
||||
assert_eq!(
|
||||
classify_entry(&jpeg, None, &FormatFilter::raw_only()),
|
||||
EntryAction::Ignored
|
||||
);
|
||||
assert_eq!(
|
||||
classify_entry(&jpeg, None, &FormatFilter::all()),
|
||||
EntryAction::Insert
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_placeholder_is_catalogued_as_the_image_it_stands_for() {
|
||||
// 121,785 of these in a real synced folder (ARCH §9.0). Each must
|
||||
// enter the catalog as a CR2 marked offline, not be skipped as an
|
||||
// unknown ".nextcloud" type.
|
||||
let stub = DirEntry {
|
||||
name: "_MG_4130.CR2.nextcloud".into(),
|
||||
is_dir: false,
|
||||
size: 1,
|
||||
mtime: 1,
|
||||
};
|
||||
assert_eq!(
|
||||
classify_entry(&stub, None, &FormatFilter::from_formats([Format::Cr2])),
|
||||
EntryAction::Insert
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn pruning_requires_a_complete_scan() {
|
||||
assert!(ScanOutcome::Complete.may_prune());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn an_unreachable_root_never_prunes() {
|
||||
// The guard that stops an unplugged drive from deleting the library:
|
||||
// every folder would look unreached, so the sweep would take all of
|
||||
// them (FR-CAT-9).
|
||||
assert!(!ScanOutcome::RootUnreachable.may_prune());
|
||||
assert!(!ScanOutcome::Cancelled.may_prune());
|
||||
assert!(!ScanOutcome::PartialFailure.may_prune());
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user