Merge branch 'local-libraries'

This commit is contained in:
2026-08-17 09:26:03 +02:00
11 changed files with 2442 additions and 33 deletions
Generated
+2
View File
@@ -1330,7 +1330,9 @@ checksum = "d8b14ccef22fc6f5a8f4d7d768562a182c04ce9a3b3157b91390b52ddfdf1a76"
name = "dr-catalog"
version = "0.1.0"
dependencies = [
"dr-plat",
"dr-types",
"env_logger",
"log",
"rusqlite",
"serde_json",
+10
View File
@@ -7,9 +7,19 @@ license.workspace = true
[dependencies]
dr-types.workspace = true
# The `Storage` trait, and nothing else from it. A scan has to read a real
# directory, and this is how `core/` reaches the platform without a
# `#[cfg(target_os)]` of its own (ARCH §4.1: calls go downward).
dr-plat.workspace = true
rusqlite.workspace = true
thiserror.workspace = true
log.workspace = true
# `collections.selector_json` — the stored form of a smart collection's
# selector. The column predates this dependency; nothing else here is JSON.
serde_json.workspace = true
# For the `scan_local` example only, which is a diagnostic tool: what it is
# diagnosing is often a folder the scan warned about and skipped, and those
# warnings go to `log`.
[dev-dependencies]
env_logger.workspace = true
+111
View File
@@ -0,0 +1,111 @@
//! Scan a real folder on this machine into a catalog, and say what it cost.
//!
//! cargo run -p dr-catalog --example scan_local -- ~/Pictures [catalog.sqlite]
//!
//! **Run it twice.** The first run is a full walk; the second is the one worth
//! watching, because on an unchanged library it should list no directories at
//! all and take a fraction of the time. That difference is NFR-P1, and a
//! synthetic test cannot show it at the scale a real library does — 121,785
//! files in a synced folder is a different question from twenty in a temporary
//! directory.
//!
//! Writes only to the catalog file, which defaults to a fixed path in the
//! system temporary directory so a second run has something to compare
//! against. Nothing in the scanned folder is touched.
use std::path::PathBuf;
use dr_catalog::walk::{ensure_root, scan_root, RootKind};
use dr_catalog::Catalog;
use dr_plat::LocalStorage;
use dr_types::FormatFilter;
fn main() {
env_logger::init();
let mut args = std::env::args().skip(1);
let Some(dir) = args.next().map(PathBuf::from) else {
eprintln!("usage: scan_local <directory> [catalog.sqlite]");
std::process::exit(2);
};
let catalog_path = args
.next()
.map(PathBuf::from)
.unwrap_or_else(|| std::env::temp_dir().join("darkroom-scan-local.sqlite"));
let catalog = match Catalog::open(&catalog_path) {
Ok(c) => c,
Err(e) => {
eprintln!("cannot open {}: {e}", catalog_path.display());
std::process::exit(1);
}
};
println!("catalog: {}", catalog_path.display());
// The label is how the grant is spelled, and the only place a path is
// written down. Everything after this line addresses files by `RootId`.
let label = dir.display().to_string();
let root = match ensure_root(catalog.connection(), RootKind::Local, &label) {
Ok(r) => r,
Err(e) => {
eprintln!("cannot record the root: {e}");
std::process::exit(1);
}
};
let storage = LocalStorage::with_root(root, &dir);
let now = std::time::SystemTime::now()
.duration_since(std::time::UNIX_EPOCH)
.map(|d| d.as_secs() as i64)
.unwrap_or(0);
let started = std::time::Instant::now();
let report = match scan_root(
catalog.connection(),
&storage,
root,
&FormatFilter::all(),
now,
|| false,
|p| {
// One line per hundred directories: enough to show it is alive on a
// large library, not enough to be the thing that slows it down.
let visited = p.directories_listed + p.directories_pruned;
if visited % 100 == 0 {
println!(
" … {visited} directories ({} pruned), {} images",
p.directories_pruned, p.images_found
);
}
},
) {
Ok(r) => r,
Err(e) => {
eprintln!("scan failed: {e}");
std::process::exit(1);
}
};
let elapsed = started.elapsed();
let total: i64 = catalog
.connection()
.query_row("SELECT count(*) FROM images", [], |r| r.get(0))
.unwrap_or(-1);
println!("\noutcome: {:?}", report.outcome);
println!(
"directories: {} listed, {} pruned",
report.progress.directories_listed, report.progress.directories_pruned
);
println!(
"images: {} new, {} changed, {} unchanged, {} removed",
report.inserted, report.updated, report.unchanged, report.images_removed
);
println!("folders: {} removed", report.folders_removed);
println!("catalogued: {total} in total");
println!("took: {:.2?}", elapsed);
if report.progress.directories_listed == 0 && report.progress.directories_pruned > 0 {
println!("\nnothing had changed: every folder was proven unchanged by one probe");
}
}
+9
View File
@@ -25,6 +25,15 @@ pub enum CatalogError {
#[error("root {0} is unreachable; scan aborted without pruning")]
RootUnreachable(u64),
/// A scan was asked for a root the catalog has no row for.
///
/// A caller's mistake rather than a user's: the row is created when the
/// grant is obtained, because the label — the path, the tree URI — is known
/// only there. Inventing one here would file the library under a name
/// nothing else would look it up by.
#[error("no such root: {0}")]
NoSuchRoot(u64),
/// A smart collection whose selector references itself, directly or via
/// another collection.
#[error("collection {0} would form a cycle")]
+3
View File
@@ -11,6 +11,7 @@
//!
//! - [`schema`] — tables and forward-only migrations
//! - [`scan`] — incremental discovery that prunes unchanged directories
//! - [`walk`] — those decisions driven against real storage, local or SAF
//! - [`query`] — selectors compiled to indexed SQL, windowed for the grid
//! - [`collections`] — the collection tree and membership the UI edits
//! - [`jobs`] — the durable background work queue
@@ -41,6 +42,7 @@ pub mod scan;
pub mod schema;
pub mod sync;
pub mod trash;
pub mod walk;
pub use cache::{Budget, Cache, DEFAULT_BUDGET_BYTES};
pub use collections::{Collection, CollectionKind, TreeRow};
@@ -51,6 +53,7 @@ pub use query::{Query, Sort};
pub use rating::{Judgement, MAX_RATING};
pub use scan::{DirAction, DirState, EntryAction, ScanOutcome};
pub use trash::{TrashedImage, TRASH_DIR};
pub use walk::{ensure_root, scan_root, RootKind, ScanProgress, ScanReport};
/// One row of the library grid.
///
+4 -1
View File
@@ -318,7 +318,10 @@ fn flag_code(f: FlagState) -> i64 {
}
}
fn availability_code(a: Availability) -> i64 {
/// The stored form of an availability. Shared with [`crate::walk`], which
/// writes the column this reads — two spellings of the same mapping would
/// filter for a state nothing ever writes.
pub(crate) fn availability_code(a: Availability) -> i64 {
match a {
Availability::MetadataOnly => 0,
Availability::Preview => 1,
+10 -27
View File
@@ -14,35 +14,13 @@
//!
//! This module holds the decision logic and the deletion-sweep rules; walking
//! an actual directory belongs to the platform layer, which supplies
//! [`DirState`] and [`DirEntry`].
//! [`DirState`] and [`DirEntry`]. [`crate::walk`] is what puts the two
//! together.
pub use dr_types::{DirEntry, DirState};
use dr_types::FormatFilter;
/// What a directory looked like when last scanned, and what it looks like now.
///
/// Both fields are cheap to obtain: one `stat` locally, one
/// `DocumentsContract` metadata query on SAF.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub struct DirState {
pub mtime: i64,
/// Direct children, files and directories alike.
///
/// mtime alone misses a delete-and-create inside one timestamp tick, and
/// coarse-granularity providers widen that window. The count does not
/// close the hole — a paired add and remove moves neither — but a bare add
/// or remove moves the count, and those are far commoner.
pub entry_count: u32,
}
/// One entry from a directory listing.
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct DirEntry {
pub name: String,
pub is_dir: bool,
pub size: u64,
pub mtime: i64,
}
/// What the scanner should do with a directory, before listing it.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum DirAction {
@@ -100,12 +78,17 @@ pub fn classify_entry(
}
/// Outcome of a scan, which decides whether pruning may run.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
///
/// `Cancelled` is the default because a scan that has not run has proven
/// nothing absent, and every default in this area must fail towards keeping
/// photographs.
#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)]
pub enum ScanOutcome {
/// Every reachable folder was visited.
Complete,
/// The user cancelled. Partial state is valid — jobs are resumable — but
/// unvisited folders must not be read as deleted.
#[default]
Cancelled,
/// The root itself could not be opened: drive unplugged, SAF grant
/// revoked, share unmounted.
File diff suppressed because it is too large Load Diff
+63 -5
View File
@@ -63,16 +63,25 @@ pub enum SourceRef {
impl SourceRef {
/// A stable, display-friendly name — the last path component.
pub fn display_name(&self) -> &str {
let full = match self {
SourceRef::Local { relative, .. } => relative.as_str(),
SourceRef::Document { document_id, .. } => document_id.as_str(),
SourceRef::Remote { path, .. } => path.as_str(),
};
let full = self.key();
// SAF document ids use ':' as a separator; paths use '/'. Split on
// whichever appears last so both yield a sensible name.
full.rsplit(['/', ':']).next().unwrap_or(full)
}
/// The stored form — what the catalog holds in `images.source_ref`.
///
/// Which of the three variants a key belongs to is not recorded beside it,
/// because the root already says: `roots.kind` is `'local' | 'saf' |
/// 'remote'`, and every image names its root.
pub fn key(&self) -> &str {
match self {
SourceRef::Local { relative, .. } => relative.as_str(),
SourceRef::Document { document_id, .. } => document_id.as_str(),
SourceRef::Remote { path, .. } => path.as_str(),
}
}
/// Lowercase file extension, if any.
///
/// Looks through a Nextcloud VFS placeholder suffix, so a dehydrated
@@ -251,6 +260,55 @@ impl FormatFilter {
}
}
/// TRACES: FR-CAT-1 | NFR-PORT-1
/// What a directory looked like when last scanned, and what it looks like now.
///
/// Both fields are cheap to obtain: one `stat` plus a name-only directory read
/// locally, one `DocumentsContract` query on SAF. Neither costs a `stat` per
/// child, which is the whole point — see `dr_catalog::scan`.
///
/// Lives here rather than in either crate that uses it because it is the
/// sentence the platform layer says to the catalog: `dr-plat` produces it by
/// probing, `dr-catalog` stores it and compares. A copy on each side would be
/// two types that must agree by convention.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub struct DirState {
/// Milliseconds since the epoch. **Not seconds**, and the difference is not
/// pedantry: change detection asks whether a timestamp *moved*, so the
/// granularity of the unit is the width of the window in which a change is
/// invisible. A second is long enough to copy a card and start a scan, and
/// anything happening inside one tick of the stored value looks exactly
/// like nothing happening — the directory is pruned and the photographs
/// never appear.
///
/// Both platforms can supply it: a filesystem records nanoseconds, and
/// SAF's `COLUMN_LAST_MODIFIED` is already in milliseconds.
pub mtime: i64,
/// Direct children, files and directories alike.
///
/// mtime alone misses a delete-and-create inside one timestamp tick, and
/// coarse-granularity providers widen that window. The count does not
/// close the hole — a paired add and remove moves neither — but a bare add
/// or remove moves the count, and those are far commoner.
pub entry_count: u32,
}
/// TRACES: FR-CAT-1 | NFR-PORT-1
/// One entry from a directory listing, as the scanner classifies it.
///
/// Deliberately carries no handle to the entry itself: this is what the
/// *decision* logic reads, and it must be constructible in a test without a
/// filesystem. The platform pairs it with the reference needed to reach the
/// entry (`dr_plat::Entry`).
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct DirEntry {
pub name: String,
pub is_dir: bool,
pub size: u64,
/// Milliseconds since the epoch, as in [`DirState::mtime`].
pub mtime: i64,
}
/// How much of an image is available locally (FR-NC-6c).
///
/// Surfaced in the UI so a user always knows what they have — the failure
+2
View File
@@ -5,7 +5,9 @@
//! ARCH §10).
pub mod secrets;
pub mod storage;
pub use secrets::{
EphemeralSecretStore, PlatformSecretStore, SecretError, SecretKind, SecretRef, SecretStore,
};
pub use storage::{DirRef, Entry, LocalStorage, Node, SeekableRead, Storage, StorageError};
+861
View File
@@ -0,0 +1,861 @@
//! TRACES: FR-CAT-1 | FR-CAT-1a | NFR-PORT-1 | NFR-PORT-3
//! Reaching stored bytes without naming a path (ARCH §3.1, §10).
//!
//! Android's Storage Access Framework hands out no filesystem path (ARCH §6.9),
//! so **nothing above this module may take one**. A library location is a
//! [`RootId`] the user granted; everything inside it is a [`DirRef`] or a
//! [`SourceRef`], both of which are opaque `(root, key)` pairs whose keys only
//! the implementation that produced them knows how to read.
//!
//! A `Path` therefore appears exactly once in the whole application: at
//! [`LocalStorage::grant`], where the folder the user picked is handed in. From
//! there on it is a `RootId`.
//!
//! # Adding Android SAF later
//!
//! It is a second implementation of [`Storage`] and no change at any call site.
//! Two properties of this API are what buy that, and both look like ceremony
//! until SAF is the thing being written:
//!
//! - **A listing hands back references, never names for the caller to join.**
//! A SAF document id is not composable — `parent_id + "/" + name` is not the
//! child's id, and the only way to learn a child's id is the children query
//! that produced the listing. So [`Entry`] carries the [`DirRef`] or
//! [`SourceRef`] the provider itself returned, and no caller ever builds one
//! by concatenation. [`LocalStorage`] could perfectly well have exposed a
//! "join a name onto a directory" helper; that helper is the one thing a SAF
//! implementation could not have provided.
//! - **A reference is a key that survives a restart.** The catalog stores the
//! key and rebuilds the reference with [`DirRef::from_parts`] on the next
//! run. On Linux the key is a relative path; on SAF it is a document id under
//! a persisted tree grant, which is re-resolvable for exactly the same
//! reason.
//!
//! What SAF will need in addition is the grant itself — the persisted tree URI,
//! which the catalog's `roots.grant_blob` column already has a home for, and
//! which is handed to the implementation at construction just as a path is
//! here.
use std::collections::BTreeMap;
use std::fmt;
use std::io::{Read, Seek};
use std::path::{Component, Path, PathBuf};
use dr_types::{ByteRange, DirEntry, DirState, RootId, SourceRef};
/// TRACES: FR-CAT-1a
/// An opaque, re-resolvable reference to a *directory* under a granted root.
///
/// The counterpart of [`SourceRef`], which addresses a file. One type rather
/// than a mirrored three-variant enum because a directory is never resolved by
/// anything except the storage that owns its root: the root's kind already
/// determines how the key is read, so a discriminator on each reference would
/// only repeat it.
///
/// The `key` is **opaque to callers** and stable across restarts. Its meaning
/// belongs to the implementation — a relative path under the root on a
/// filesystem, a `DocumentsContract` document id on SAF — and the catalog
/// stores it verbatim in `folders.path` so a later run can rebuild the
/// reference with [`DirRef::from_parts`].
#[derive(Debug, Clone, PartialEq, Eq, Hash, PartialOrd, Ord)]
pub struct DirRef {
root: RootId,
key: String,
}
impl DirRef {
/// The granted root itself, which every walk starts from.
pub fn root(root: RootId) -> Self {
Self {
root,
key: String::new(),
}
}
/// Rebuild a reference from a key a previous scan stored.
///
/// The re-resolution FR-CAT-1a requires: after a restart the catalog holds
/// keys and nothing else, and a scan that could not resume from them would
/// have to walk the whole library to find the folder it left off in.
pub fn from_parts(root: RootId, key: impl Into<String>) -> Self {
Self {
root,
key: key.into(),
}
}
pub fn root_id(&self) -> RootId {
self.root
}
/// The stored form. Meaningful only to the storage that produced it.
pub fn key(&self) -> &str {
&self.key
}
/// Whether this is the granted root rather than something inside it.
///
/// The walk needs it: a failure at the root is the whole library being
/// unreachable, and the deletion sweep must not run; a failure below it is
/// one folder (FR-CAT-9).
pub fn is_root(&self) -> bool {
self.key.is_empty()
}
/// The last component, for exclusion checks and display.
pub fn name(&self) -> &str {
self.key.rsplit(['/', ':']).next().unwrap_or(&self.key)
}
}
impl fmt::Display for DirRef {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
if self.key.is_empty() {
write!(f, "root {}", self.root.0)
} else {
write!(f, "{}", self.key)
}
}
}
/// How to reach a listed entry.
#[derive(Debug, Clone, PartialEq, Eq)]
pub enum Node {
Dir(DirRef),
File(SourceRef),
}
/// One entry from a listing: what the scanner classifies on, and how to reach
/// it.
///
/// `meta.is_dir` and the [`Node`] variant always agree — the implementation
/// sets both from one observation. They are separate because they serve
/// different readers: `meta` goes to `dr_catalog::scan`, which decides, and
/// `node` goes to whatever acts on the decision.
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct Entry {
pub meta: DirEntry,
pub node: Node,
}
/// A stream that can be read and seeked — what a decoder wants.
///
/// Blanket-implemented, so a `File`, a `Cursor<Vec<u8>>` in a test, and a
/// future SAF `ParcelFileDescriptor` wrapper all qualify without ceremony.
pub trait SeekableRead: Read + Seek + Send {}
impl<T: Read + Seek + Send> SeekableRead for T {}
/// TRACES: NFR-ARCH-4
/// Something went wrong reaching storage.
///
/// Typed, and never a panic: a library is on removable media, on a network
/// mount, or behind a permission the user can revoke while the app is running,
/// so every one of these is a normal Tuesday rather than a bug.
///
/// Distinct from [`dr_types::SourceError`], which is what the *decode* path
/// sees. This one also speaks about roots and directories, which a decoder has
/// no concept of.
#[derive(Debug, thiserror::Error)]
pub enum StorageError {
/// A reference naming a root this storage was never granted.
///
/// Almost always a stale catalog row: the library was removed and its rows
/// outlived it.
#[error("root {0} was not granted to this storage")]
UnknownRoot(u64),
/// A reference of the wrong shape — a SAF document id handed to the
/// filesystem implementation, or the reverse.
///
/// Refused rather than guessed at: the two address spaces have no overlap,
/// and a guess would read the wrong file rather than fail.
#[error("{0}")]
Unsupported(&'static str),
/// A stored key that would leave its root.
///
/// The catalog is a file on disk that other programs can edit, so a key
/// containing `..` is possible however it got there. Refused, because a
/// grant to one folder must not become a read of the whole filesystem
/// (NFR-SEC-1).
#[error("key {0:?} escapes its root")]
EscapesRoot(String),
#[error("not found: {0}")]
NotFound(String),
#[error("permission denied: {0}")]
PermissionDenied(String),
#[error("not a directory: {0}")]
NotADirectory(String),
#[error("range reads unsupported by this storage")]
RangeUnsupported,
#[error("io error: {0}")]
Io(String),
}
/// TRACES: FR-CAT-1 | FR-CAT-1a | NFR-PORT-1
/// Enumerate and read the contents of granted library roots.
///
/// Implemented per platform and injected at construction, so `core/` contains
/// no `#[cfg(target_os)]` (ARCH §10).
pub trait Storage: Send + Sync {
/// Every root this storage can currently reach.
fn roots(&self) -> Vec<RootId>;
/// The granted root as a directory, which is where a walk begins.
///
/// Fails on a root that was never granted, so a stale catalog row is a
/// typed error rather than an empty library.
fn root_dir(&self, root: RootId) -> Result<DirRef, StorageError>;
/// Probe a directory without reading its contents.
///
/// The cheap half of the pair, and the reason incremental scanning is
/// affordable: this costs one `stat` plus a name-only directory read,
/// where [`list`](Self::list) costs a `stat` per child. On a library of 2k
/// folders and 50k images that is 2k probes against 50k, which is the
/// difference between meeting and missing NFR-P1.
fn dir_state(&self, dir: &DirRef) -> Result<DirState, StorageError>;
/// List a directory's direct children, with the metadata to classify them.
///
/// Returns the whole listing rather than streaming it through a callback,
/// because the caller needs the complete set at once: an image in the
/// catalog that this listing does *not* contain has been deleted, and that
/// conclusion cannot be drawn one entry at a time.
fn list(&self, dir: &DirRef) -> Result<Vec<Entry>, StorageError>;
/// Open a seekable stream over a source.
fn open(&self, src: &SourceRef) -> Result<Box<dyn SeekableRead>, StorageError>;
/// Read a byte range without opening the whole source.
///
/// First-class rather than a convenience over [`open`](Self::open), because
/// for the case that matters it is a different operation and not a smaller
/// one: extracting an embedded JPEG preview from an 80 MB RAW over the
/// network transfers 1–3 MB (ARCH §3.1, FR-CULL-2, FR-NC-3).
///
/// A range reaching past the end yields the bytes that exist — a short read
/// is the honest answer, and a caller needing exactly `n` bytes must say so
/// by checking the length.
fn read_range(&self, src: &SourceRef, range: ByteRange) -> Result<Vec<u8>, StorageError>;
}
/// TRACES: FR-PLAT-LIN-1 | NFR-PORT-1
/// The filesystem implementation: a root is a directory the user picked.
///
/// Also the right implementation for a Flatpak, where the portal returns a real
/// path the sandbox can see (FR-PLAT-LIN-3).
///
/// Present on Android too, and harmless there: the type compiles wherever
/// `std::fs` does, so `ui/` can name it unconditionally. It is not how an
/// Android user's library is reached — that is SAF, and ARCH §6.9 explains
/// why nothing else is on offer.
#[derive(Debug, Default)]
pub struct LocalStorage {
/// Ordered so [`roots`](Storage::roots) is stable, which keeps a scan of
/// several roots reproducible.
roots: BTreeMap<RootId, PathBuf>,
}
impl LocalStorage {
pub fn new() -> Self {
Self::default()
}
/// One granted root, the common case.
pub fn with_root(root: RootId, dir: impl Into<PathBuf>) -> Self {
let mut s = Self::new();
s.grant(root, dir);
s
}
/// Record that the user granted `dir` as `root`.
///
/// **The only place a `Path` enters the application.** Above this line a
/// library location is a `RootId`, which is what lets the same catalog and
/// the same scanner run against SAF, where no path exists at all.
pub fn grant(&mut self, root: RootId, dir: impl Into<PathBuf>) {
self.roots.insert(root, dir.into());
}
/// Where a root sits, for the app that granted it — to show the user, or to
/// store so the grant survives a restart.
pub fn root_path(&self, root: RootId) -> Option<&Path> {
self.roots.get(&root).map(|p| p.as_path())
}
fn base(&self, root: RootId) -> Result<&Path, StorageError> {
self.roots
.get(&root)
.map(|p| p.as_path())
.ok_or(StorageError::UnknownRoot(root.0))
}
/// Turn a `(root, key)` pair back into a path, refusing anything that
/// leaves the root.
///
/// Every component must be an ordinary name: `..` would climb out, and an
/// absolute key would discard the root entirely — `Path::join` silently
/// replaces rather than appends when handed one, which is how a "relative"
/// path of `/etc` becomes a read of `/etc`.
fn resolve(&self, root: RootId, key: &str) -> Result<PathBuf, StorageError> {
let base = self.base(root)?;
if key.is_empty() {
return Ok(base.to_path_buf());
}
let rel = Path::new(key);
if !rel.components().all(|c| matches!(c, Component::Normal(_))) {
return Err(StorageError::EscapesRoot(key.to_string()));
}
Ok(base.join(rel))
}
fn file_path(&self, src: &SourceRef) -> Result<PathBuf, StorageError> {
match src {
SourceRef::Local { root, relative } => self.resolve(*root, relative),
SourceRef::Document { .. } => Err(StorageError::Unsupported(
"a SAF document reference cannot be read from the filesystem",
)),
SourceRef::Remote { .. } => Err(StorageError::Unsupported(
"a remote reference is resolved by the sync layer, not by storage",
)),
}
}
}
/// A child's key, built the one place that is allowed to build one.
///
/// Concatenation is safe here and only here: this implementation *chose* to
/// make its keys relative paths, so it is the only code entitled to know that
/// they compose. A SAF implementation has no equivalent (see the module docs),
/// which is why this is a private helper and not a method on [`DirRef`].
fn child_key(parent: &str, name: &str) -> String {
if parent.is_empty() {
name.to_string()
} else {
format!("{parent}/{name}")
}
}
impl Storage for LocalStorage {
fn roots(&self) -> Vec<RootId> {
self.roots.keys().copied().collect()
}
fn root_dir(&self, root: RootId) -> Result<DirRef, StorageError> {
self.base(root)?;
Ok(DirRef::root(root))
}
fn dir_state(&self, dir: &DirRef) -> Result<DirState, StorageError> {
let path = self.resolve(dir.root_id(), dir.key())?;
let meta = std::fs::metadata(&path).map_err(|e| map_io(&path, e))?;
if !meta.is_dir() {
return Err(StorageError::NotADirectory(path.display().to_string()));
}
// Names only — `read_dir` yields entries from `getdents` without a
// `stat` per child, so the count costs one pass and no per-file I/O.
// That is what makes this the cheap probe `list` is not.
let entry_count = std::fs::read_dir(&path)
.map_err(|e| map_io(&path, e))?
.count();
Ok(DirState {
mtime: modified_millis(&meta),
// Saturating rather than wrapping: a directory of four billion
// entries would otherwise wrap to a small number and could compare
// equal after a change. It cannot happen, and being wrong about it
// would be silent.
entry_count: u32::try_from(entry_count).unwrap_or(u32::MAX),
})
}
fn list(&self, dir: &DirRef) -> Result<Vec<Entry>, StorageError> {
let path = self.resolve(dir.root_id(), dir.key())?;
let read = std::fs::read_dir(&path).map_err(|e| map_io(&path, e))?;
let mut out = Vec::new();
for entry in read {
let entry = match entry {
Ok(e) => e,
// One unreadable entry is not an unreadable directory. Skipping
// it loses one file; failing the listing would make the folder
// look empty, and an empty folder is a *deletion* to the sweep.
Err(e) => {
log::debug!("list {}: skipping unreadable entry: {e}", path.display());
continue;
}
};
// A name that is not UTF-8 cannot become a key, and a lossy
// conversion would produce a key that resolves to nothing — an
// image catalogued and then permanently unreadable. Skipped, and
// said out loud, because the user's file is real and we are
// choosing not to see it.
let Some(name) = entry.file_name().to_str().map(str::to_owned) else {
log::warn!(
"list {}: skipping {:?}, whose name is not valid UTF-8",
path.display(),
entry.file_name()
);
continue;
};
// Follows symlinks, unlike `DirEntry::metadata`. A photographer who
// symlinks last year's drive into the library means it as part of
// the library. The walk's depth limit is what stops a loop.
let child = path.join(&name);
let meta = match std::fs::metadata(&child) {
Ok(m) => m,
// A broken symlink, or a file deleted between the listing and
// this stat. Neither is an error worth failing a folder for.
Err(e) => {
log::debug!("list {}: skipping {name}: {e}", path.display());
continue;
}
};
let key = child_key(dir.key(), &name);
let is_dir = meta.is_dir();
out.push(Entry {
meta: DirEntry {
name,
is_dir,
size: meta.len(),
mtime: modified_millis(&meta),
},
node: if is_dir {
Node::Dir(DirRef::from_parts(dir.root_id(), key))
} else {
Node::File(SourceRef::Local {
root: dir.root_id(),
relative: key,
})
},
});
}
// Directory order is filesystem order, which is arbitrary and differs
// between runs. Sorting makes a scan reproducible and a test able to
// assert on what it found.
out.sort_by(|a, b| a.meta.name.cmp(&b.meta.name));
Ok(out)
}
fn open(&self, src: &SourceRef) -> Result<Box<dyn SeekableRead>, StorageError> {
let path = self.file_path(src)?;
let file = std::fs::File::open(&path).map_err(|e| map_io(&path, e))?;
Ok(Box::new(file))
}
fn read_range(&self, src: &SourceRef, range: ByteRange) -> Result<Vec<u8>, StorageError> {
use std::io::SeekFrom;
if range.end <= range.start {
return Ok(Vec::new());
}
let path = self.file_path(src)?;
let mut file = std::fs::File::open(&path).map_err(|e| map_io(&path, e))?;
file.seek(SeekFrom::Start(range.start))
.map_err(|e| map_io(&path, e))?;
// `take` and grow, rather than a buffer sized to the request: the range
// comes from a header the file itself declared, and a corrupt one
// asking for four gigabytes must not be allocated before it is known
// that four gigabytes exist (NFR-SEC-1).
let mut buf = Vec::new();
file.take(range.end - range.start)
.read_to_end(&mut buf)
.map_err(|e| map_io(&path, e))?;
Ok(buf)
}
}
/// Milliseconds since the epoch, or 0 where the platform will not say.
///
/// Milliseconds because that is the unit change detection is expressed in, and
/// the unit's granularity is the width of the window in which a change is
/// invisible — see [`dr_types::DirState::mtime`]. Seconds would hide a card
/// imported and scanned within the same tick.
///
/// A file whose mtime is unreadable compares equal to itself forever and so is
/// never re-read. That is the better failure: the alternative, a value that
/// changes each time it is asked for, would re-process the file on every scan.
fn modified_millis(meta: &std::fs::Metadata) -> i64 {
let Ok(t) = meta.modified() else {
return 0;
};
match t.duration_since(std::time::UNIX_EPOCH) {
Ok(d) => i64::try_from(d.as_millis()).unwrap_or(i64::MAX),
// Before 1970. Rare, but an archive of digitised film can carry one,
// and it must not become a huge positive number.
Err(e) => i64::try_from(e.duration().as_millis())
.unwrap_or(i64::MAX)
.saturating_neg(),
}
}
fn map_io(path: &Path, e: std::io::Error) -> StorageError {
let what = path.display().to_string();
match e.kind() {
std::io::ErrorKind::NotFound => StorageError::NotFound(what),
std::io::ErrorKind::PermissionDenied => StorageError::PermissionDenied(what),
_ => StorageError::Io(format!("{what}: {e}")),
}
}
#[cfg(test)]
mod tests {
use super::*;
use std::fs;
/// A throwaway directory tree, removed when the test ends.
struct Tree(PathBuf);
impl Tree {
fn new(name: &str) -> Self {
let dir = std::env::temp_dir().join(format!(
"dr-plat-{name}-{}-{:?}",
std::process::id(),
std::thread::current().id()
));
let _ = fs::remove_dir_all(&dir);
fs::create_dir_all(&dir).expect("temp dir");
Tree(dir)
}
fn dir(&self, rel: &str) -> &Self {
fs::create_dir_all(self.0.join(rel)).expect("mkdir");
self
}
fn file(&self, rel: &str, bytes: &[u8]) -> &Self {
let p = self.0.join(rel);
if let Some(parent) = p.parent() {
fs::create_dir_all(parent).expect("mkdir");
}
fs::write(p, bytes).expect("write");
self
}
fn storage(&self) -> LocalStorage {
LocalStorage::with_root(RootId(1), self.0.clone())
}
}
impl Drop for Tree {
fn drop(&mut self) {
let _ = fs::remove_dir_all(&self.0);
}
}
const ROOT: RootId = RootId(1);
#[test]
fn a_listing_names_files_and_directories_apart() {
let t = Tree::new("listing");
t.file("IMG_0001.CR3", b"raw").dir("2026");
let s = t.storage();
let entries = s.list(&s.root_dir(ROOT).unwrap()).unwrap();
assert_eq!(entries.len(), 2);
assert_eq!(entries[0].meta.name, "2026");
assert!(entries[0].meta.is_dir);
assert!(matches!(entries[0].node, Node::Dir(_)));
assert_eq!(entries[1].meta.name, "IMG_0001.CR3");
assert_eq!(entries[1].meta.size, 3);
assert!(matches!(entries[1].node, Node::File(_)));
}
#[test]
fn a_listing_hands_back_references_the_caller_never_composes() {
// The property that makes a SAF implementation a drop-in: the child's
// reference comes from the listing, because on SAF it is the only place
// it can come from. If a test ever has to build one by joining strings,
// the abstraction has already leaked.
let t = Tree::new("refs");
t.file("2026/08/IMG_0042.CR3", b"raw");
let s = t.storage();
let year = match &s.list(&s.root_dir(ROOT).unwrap()).unwrap()[0].node {
Node::Dir(d) => d.clone(),
other => panic!("expected a directory, got {other:?}"),
};
let month = match &s.list(&year).unwrap()[0].node {
Node::Dir(d) => d.clone(),
other => panic!("expected a directory, got {other:?}"),
};
let file = match &s.list(&month).unwrap()[0].node {
Node::File(f) => f.clone(),
other => panic!("expected a file, got {other:?}"),
};
assert_eq!(
file,
SourceRef::Local {
root: ROOT,
relative: "2026/08/IMG_0042.CR3".into()
}
);
assert!(s.open(&file).is_ok());
}
#[test]
fn a_stored_key_reopens_the_same_directory_after_a_restart() {
// What FR-CAT-1a's "re-resolvable" means in practice: the catalog keeps
// keys, not handles, and a scan resuming tomorrow rebuilds the
// reference from one. Without this a restart is a full rewalk.
let t = Tree::new("reresolve");
t.file("2026/IMG.CR3", b"raw");
let s = t.storage();
let key = match &s.list(&s.root_dir(ROOT).unwrap()).unwrap()[0].node {
Node::Dir(d) => d.key().to_string(),
other => panic!("expected a directory, got {other:?}"),
};
let rebuilt = DirRef::from_parts(ROOT, key);
assert_eq!(s.list(&rebuilt).unwrap()[0].meta.name, "IMG.CR3");
}
#[test]
fn a_key_that_climbs_out_of_the_root_is_refused() {
// A grant is to one folder. The catalog is an ordinary file that other
// programs can edit, so a `..` in a key is reachable however it got
// there, and honouring it would turn a grant to ~/Photos into a read of
// the whole filesystem (NFR-SEC-1).
let t = Tree::new("escape");
let s = t.storage();
for key in ["../etc", "a/../../etc", "/etc"] {
let dir = DirRef::from_parts(ROOT, key);
assert!(
matches!(s.dir_state(&dir), Err(StorageError::EscapesRoot(_))),
"{key} was not refused"
);
}
}
#[test]
fn an_absolute_key_does_not_silently_replace_the_root() {
// `Path::join` replaces rather than appends when given an absolute
// path, so this one is not merely an escape — it is an escape that
// looks like ordinary joining and would never be noticed in review.
let t = Tree::new("absolute");
let s = t.storage();
let src = SourceRef::Local {
root: ROOT,
relative: "/etc/passwd".into(),
};
assert!(matches!(
s.read_range(&src, 0..16),
Err(StorageError::EscapesRoot(_))
));
}
#[test]
fn probing_a_directory_reports_what_change_detection_needs() {
let t = Tree::new("probe");
t.file("a.CR3", b"1").file("b.CR3", b"2");
let s = t.storage();
let before = s.dir_state(&s.root_dir(ROOT).unwrap()).unwrap();
assert_eq!(before.entry_count, 2);
t.file("c.CR3", b"3");
let after = s.dir_state(&s.root_dir(ROOT).unwrap()).unwrap();
assert_ne!(
before, after,
"an added file must move the state, or the folder is pruned and the \
image never enters the catalog"
);
assert_eq!(after.entry_count, 3);
}
#[test]
fn the_probe_counts_directories_as_well_as_files() {
// A new subfolder full of images changes nothing about the parent's
// files. If the count ignored directories, the parent would look
// unchanged and the whole subtree would go unseen.
let t = Tree::new("probe-dirs");
t.file("a.CR3", b"1");
let s = t.storage();
let before = s.dir_state(&s.root_dir(ROOT).unwrap()).unwrap();
t.dir("2026");
let after = s.dir_state(&s.root_dir(ROOT).unwrap()).unwrap();
assert_eq!(after.entry_count, before.entry_count + 1);
}
#[test]
fn a_range_read_returns_only_the_bytes_asked_for() {
let t = Tree::new("range");
t.file("IMG.CR3", b"0123456789");
let s = t.storage();
let src = SourceRef::Local {
root: ROOT,
relative: "IMG.CR3".into(),
};
assert_eq!(s.read_range(&src, 2..6).unwrap(), b"2345");
assert_eq!(s.read_range(&src, 0..0).unwrap(), b"");
}
#[test]
fn a_range_past_the_end_is_short_rather_than_an_error() {
// Preview offsets come out of the file's own header. A truncated or
// mis-parsed one must yield "here is what exists", not a failed decode
// and not a four-gigabyte allocation (NFR-SEC-1).
let t = Tree::new("range-eof");
t.file("IMG.CR3", b"0123456789");
let s = t.storage();
let src = SourceRef::Local {
root: ROOT,
relative: "IMG.CR3".into(),
};
assert_eq!(s.read_range(&src, 8..u64::MAX / 2).unwrap(), b"89");
assert_eq!(s.read_range(&src, 999..1_000).unwrap(), b"");
}
#[test]
fn an_open_stream_can_seek() {
// The decoders need it: a RAW's preview lives at an offset the header
// names, and a forward-only stream would mean reading 80 MB to get 2.
let t = Tree::new("seek");
t.file("IMG.CR3", b"0123456789");
let s = t.storage();
let mut r = s
.open(&SourceRef::Local {
root: ROOT,
relative: "IMG.CR3".into(),
})
.unwrap();
r.seek(std::io::SeekFrom::Start(5)).unwrap();
let mut buf = [0u8; 2];
r.read_exact(&mut buf).unwrap();
assert_eq!(&buf, b"56");
}
#[test]
fn an_ungranted_root_is_a_typed_error_not_an_empty_library() {
// A removed library leaves catalog rows behind. Reporting them as
// "nothing here" would let the deletion sweep take the lot (FR-CAT-9).
let s = LocalStorage::new();
assert!(matches!(
s.root_dir(RootId(7)),
Err(StorageError::UnknownRoot(7))
));
assert!(matches!(
s.dir_state(&DirRef::root(RootId(7))),
Err(StorageError::UnknownRoot(7))
));
assert!(s.roots().is_empty());
}
#[test]
fn a_missing_directory_is_not_reported_as_empty() {
// The same failure from the other direction: an unplugged drive must
// error, because an empty listing means every image under it was
// deleted.
let t = Tree::new("missing");
let s = t.storage();
let gone = DirRef::from_parts(ROOT, "nowhere");
assert!(matches!(s.dir_state(&gone), Err(StorageError::NotFound(_))));
assert!(matches!(s.list(&gone), Err(StorageError::NotFound(_))));
}
#[test]
fn a_reference_of_the_wrong_kind_is_refused_rather_than_guessed_at() {
// A SAF document id is not a path. Treating it as one would resolve to
// some other file, which is worse than failing.
let t = Tree::new("wrong-kind");
let s = t.storage();
let saf = SourceRef::Document {
tree: ROOT,
document_id: "primary:DCIM/IMG.CR3".into(),
};
assert!(matches!(s.open(&saf), Err(StorageError::Unsupported(_))));
let remote = SourceRef::Remote {
file_id: 1,
path: "Photos/IMG.CR3".into(),
};
assert!(matches!(s.open(&remote), Err(StorageError::Unsupported(_))));
}
#[test]
fn several_roots_coexist() {
// Two libraries on two drives is FR-CAT-1's "one or more roots", and
// each reference carries which one it belongs to.
let a = Tree::new("multi-a");
a.file("a.CR3", b"1");
let b = Tree::new("multi-b");
b.file("b.CR3", b"2");
let mut s = LocalStorage::new();
s.grant(RootId(1), a.0.clone());
s.grant(RootId(2), b.0.clone());
assert_eq!(s.roots(), vec![RootId(1), RootId(2)]);
assert_eq!(
s.list(&s.root_dir(RootId(2)).unwrap()).unwrap()[0]
.meta
.name,
"b.CR3"
);
}
#[test]
fn a_listing_is_ordered_the_same_way_twice() {
// Filesystem order is arbitrary and differs between runs; a scan that
// depended on it would produce a different catalog each time.
let t = Tree::new("order");
for n in ["c.CR3", "a.CR3", "b.CR3"] {
t.file(n, b"x");
}
let s = t.storage();
let names: Vec<String> = s
.list(&s.root_dir(ROOT).unwrap())
.unwrap()
.into_iter()
.map(|e| e.meta.name)
.collect();
assert_eq!(names, vec!["a.CR3", "b.CR3", "c.CR3"]);
}
#[cfg(unix)]
#[test]
fn a_name_that_is_not_utf8_is_skipped_rather_than_mangled() {
// A lossy conversion would produce a key that resolves to nothing: the
// image would be catalogued and then permanently unopenable. Better to
// not see the file than to promise it and fail later.
use std::os::unix::ffi::OsStrExt;
let t = Tree::new("non-utf8");
t.file("good.CR3", b"1");
let bad = t.0.join(std::ffi::OsStr::from_bytes(b"bad\xff.CR3"));
fs::write(&bad, b"2").expect("write");
let s = t.storage();
let entries = s.list(&s.root_dir(ROOT).unwrap()).unwrap();
assert_eq!(entries.len(), 1);
assert_eq!(entries[0].meta.name, "good.CR3");
}
#[test]
fn the_root_is_distinguishable_from_what_is_inside_it() {
// The walk keys the difference between "the library is unreachable" and
// "one folder failed" on this, and those have opposite consequences for
// the deletion sweep (FR-CAT-9).
assert!(DirRef::root(ROOT).is_root());
assert!(!DirRef::from_parts(ROOT, "2026").is_root());
assert_eq!(DirRef::from_parts(ROOT, "2026/08").name(), "08");
}
}