Merge branch 'local-libraries'
This commit is contained in:
Generated
+2
@@ -1330,7 +1330,9 @@ checksum = "d8b14ccef22fc6f5a8f4d7d768562a182c04ce9a3b3157b91390b52ddfdf1a76"
|
|||||||
name = "dr-catalog"
|
name = "dr-catalog"
|
||||||
version = "0.1.0"
|
version = "0.1.0"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
|
"dr-plat",
|
||||||
"dr-types",
|
"dr-types",
|
||||||
|
"env_logger",
|
||||||
"log",
|
"log",
|
||||||
"rusqlite",
|
"rusqlite",
|
||||||
"serde_json",
|
"serde_json",
|
||||||
|
|||||||
@@ -7,9 +7,19 @@ license.workspace = true
|
|||||||
|
|
||||||
[dependencies]
|
[dependencies]
|
||||||
dr-types.workspace = true
|
dr-types.workspace = true
|
||||||
|
# The `Storage` trait, and nothing else from it. A scan has to read a real
|
||||||
|
# directory, and this is how `core/` reaches the platform without a
|
||||||
|
# `#[cfg(target_os)]` of its own (ARCH §4.1: calls go downward).
|
||||||
|
dr-plat.workspace = true
|
||||||
rusqlite.workspace = true
|
rusqlite.workspace = true
|
||||||
thiserror.workspace = true
|
thiserror.workspace = true
|
||||||
log.workspace = true
|
log.workspace = true
|
||||||
# `collections.selector_json` — the stored form of a smart collection's
|
# `collections.selector_json` — the stored form of a smart collection's
|
||||||
# selector. The column predates this dependency; nothing else here is JSON.
|
# selector. The column predates this dependency; nothing else here is JSON.
|
||||||
serde_json.workspace = true
|
serde_json.workspace = true
|
||||||
|
|
||||||
|
# For the `scan_local` example only, which is a diagnostic tool: what it is
|
||||||
|
# diagnosing is often a folder the scan warned about and skipped, and those
|
||||||
|
# warnings go to `log`.
|
||||||
|
[dev-dependencies]
|
||||||
|
env_logger.workspace = true
|
||||||
|
|||||||
@@ -0,0 +1,111 @@
|
|||||||
|
//! Scan a real folder on this machine into a catalog, and say what it cost.
|
||||||
|
//!
|
||||||
|
//! cargo run -p dr-catalog --example scan_local -- ~/Pictures [catalog.sqlite]
|
||||||
|
//!
|
||||||
|
//! **Run it twice.** The first run is a full walk; the second is the one worth
|
||||||
|
//! watching, because on an unchanged library it should list no directories at
|
||||||
|
//! all and take a fraction of the time. That difference is NFR-P1, and a
|
||||||
|
//! synthetic test cannot show it at the scale a real library does — 121,785
|
||||||
|
//! files in a synced folder is a different question from twenty in a temporary
|
||||||
|
//! directory.
|
||||||
|
//!
|
||||||
|
//! Writes only to the catalog file, which defaults to a fixed path in the
|
||||||
|
//! system temporary directory so a second run has something to compare
|
||||||
|
//! against. Nothing in the scanned folder is touched.
|
||||||
|
|
||||||
|
use std::path::PathBuf;
|
||||||
|
|
||||||
|
use dr_catalog::walk::{ensure_root, scan_root, RootKind};
|
||||||
|
use dr_catalog::Catalog;
|
||||||
|
use dr_plat::LocalStorage;
|
||||||
|
use dr_types::FormatFilter;
|
||||||
|
|
||||||
|
fn main() {
|
||||||
|
env_logger::init();
|
||||||
|
|
||||||
|
let mut args = std::env::args().skip(1);
|
||||||
|
let Some(dir) = args.next().map(PathBuf::from) else {
|
||||||
|
eprintln!("usage: scan_local <directory> [catalog.sqlite]");
|
||||||
|
std::process::exit(2);
|
||||||
|
};
|
||||||
|
let catalog_path = args
|
||||||
|
.next()
|
||||||
|
.map(PathBuf::from)
|
||||||
|
.unwrap_or_else(|| std::env::temp_dir().join("darkroom-scan-local.sqlite"));
|
||||||
|
|
||||||
|
let catalog = match Catalog::open(&catalog_path) {
|
||||||
|
Ok(c) => c,
|
||||||
|
Err(e) => {
|
||||||
|
eprintln!("cannot open {}: {e}", catalog_path.display());
|
||||||
|
std::process::exit(1);
|
||||||
|
}
|
||||||
|
};
|
||||||
|
println!("catalog: {}", catalog_path.display());
|
||||||
|
|
||||||
|
// The label is how the grant is spelled, and the only place a path is
|
||||||
|
// written down. Everything after this line addresses files by `RootId`.
|
||||||
|
let label = dir.display().to_string();
|
||||||
|
let root = match ensure_root(catalog.connection(), RootKind::Local, &label) {
|
||||||
|
Ok(r) => r,
|
||||||
|
Err(e) => {
|
||||||
|
eprintln!("cannot record the root: {e}");
|
||||||
|
std::process::exit(1);
|
||||||
|
}
|
||||||
|
};
|
||||||
|
let storage = LocalStorage::with_root(root, &dir);
|
||||||
|
|
||||||
|
let now = std::time::SystemTime::now()
|
||||||
|
.duration_since(std::time::UNIX_EPOCH)
|
||||||
|
.map(|d| d.as_secs() as i64)
|
||||||
|
.unwrap_or(0);
|
||||||
|
|
||||||
|
let started = std::time::Instant::now();
|
||||||
|
let report = match scan_root(
|
||||||
|
catalog.connection(),
|
||||||
|
&storage,
|
||||||
|
root,
|
||||||
|
&FormatFilter::all(),
|
||||||
|
now,
|
||||||
|
|| false,
|
||||||
|
|p| {
|
||||||
|
// One line per hundred directories: enough to show it is alive on a
|
||||||
|
// large library, not enough to be the thing that slows it down.
|
||||||
|
let visited = p.directories_listed + p.directories_pruned;
|
||||||
|
if visited % 100 == 0 {
|
||||||
|
println!(
|
||||||
|
" … {visited} directories ({} pruned), {} images",
|
||||||
|
p.directories_pruned, p.images_found
|
||||||
|
);
|
||||||
|
}
|
||||||
|
},
|
||||||
|
) {
|
||||||
|
Ok(r) => r,
|
||||||
|
Err(e) => {
|
||||||
|
eprintln!("scan failed: {e}");
|
||||||
|
std::process::exit(1);
|
||||||
|
}
|
||||||
|
};
|
||||||
|
let elapsed = started.elapsed();
|
||||||
|
|
||||||
|
let total: i64 = catalog
|
||||||
|
.connection()
|
||||||
|
.query_row("SELECT count(*) FROM images", [], |r| r.get(0))
|
||||||
|
.unwrap_or(-1);
|
||||||
|
|
||||||
|
println!("\noutcome: {:?}", report.outcome);
|
||||||
|
println!(
|
||||||
|
"directories: {} listed, {} pruned",
|
||||||
|
report.progress.directories_listed, report.progress.directories_pruned
|
||||||
|
);
|
||||||
|
println!(
|
||||||
|
"images: {} new, {} changed, {} unchanged, {} removed",
|
||||||
|
report.inserted, report.updated, report.unchanged, report.images_removed
|
||||||
|
);
|
||||||
|
println!("folders: {} removed", report.folders_removed);
|
||||||
|
println!("catalogued: {total} in total");
|
||||||
|
println!("took: {:.2?}", elapsed);
|
||||||
|
|
||||||
|
if report.progress.directories_listed == 0 && report.progress.directories_pruned > 0 {
|
||||||
|
println!("\nnothing had changed: every folder was proven unchanged by one probe");
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -25,6 +25,15 @@ pub enum CatalogError {
|
|||||||
#[error("root {0} is unreachable; scan aborted without pruning")]
|
#[error("root {0} is unreachable; scan aborted without pruning")]
|
||||||
RootUnreachable(u64),
|
RootUnreachable(u64),
|
||||||
|
|
||||||
|
/// A scan was asked for a root the catalog has no row for.
|
||||||
|
///
|
||||||
|
/// A caller's mistake rather than a user's: the row is created when the
|
||||||
|
/// grant is obtained, because the label — the path, the tree URI — is known
|
||||||
|
/// only there. Inventing one here would file the library under a name
|
||||||
|
/// nothing else would look it up by.
|
||||||
|
#[error("no such root: {0}")]
|
||||||
|
NoSuchRoot(u64),
|
||||||
|
|
||||||
/// A smart collection whose selector references itself, directly or via
|
/// A smart collection whose selector references itself, directly or via
|
||||||
/// another collection.
|
/// another collection.
|
||||||
#[error("collection {0} would form a cycle")]
|
#[error("collection {0} would form a cycle")]
|
||||||
|
|||||||
@@ -11,6 +11,7 @@
|
|||||||
//!
|
//!
|
||||||
//! - [`schema`] — tables and forward-only migrations
|
//! - [`schema`] — tables and forward-only migrations
|
||||||
//! - [`scan`] — incremental discovery that prunes unchanged directories
|
//! - [`scan`] — incremental discovery that prunes unchanged directories
|
||||||
|
//! - [`walk`] — those decisions driven against real storage, local or SAF
|
||||||
//! - [`query`] — selectors compiled to indexed SQL, windowed for the grid
|
//! - [`query`] — selectors compiled to indexed SQL, windowed for the grid
|
||||||
//! - [`collections`] — the collection tree and membership the UI edits
|
//! - [`collections`] — the collection tree and membership the UI edits
|
||||||
//! - [`jobs`] — the durable background work queue
|
//! - [`jobs`] — the durable background work queue
|
||||||
@@ -41,6 +42,7 @@ pub mod scan;
|
|||||||
pub mod schema;
|
pub mod schema;
|
||||||
pub mod sync;
|
pub mod sync;
|
||||||
pub mod trash;
|
pub mod trash;
|
||||||
|
pub mod walk;
|
||||||
|
|
||||||
pub use cache::{Budget, Cache, DEFAULT_BUDGET_BYTES};
|
pub use cache::{Budget, Cache, DEFAULT_BUDGET_BYTES};
|
||||||
pub use collections::{Collection, CollectionKind, TreeRow};
|
pub use collections::{Collection, CollectionKind, TreeRow};
|
||||||
@@ -51,6 +53,7 @@ pub use query::{Query, Sort};
|
|||||||
pub use rating::{Judgement, MAX_RATING};
|
pub use rating::{Judgement, MAX_RATING};
|
||||||
pub use scan::{DirAction, DirState, EntryAction, ScanOutcome};
|
pub use scan::{DirAction, DirState, EntryAction, ScanOutcome};
|
||||||
pub use trash::{TrashedImage, TRASH_DIR};
|
pub use trash::{TrashedImage, TRASH_DIR};
|
||||||
|
pub use walk::{ensure_root, scan_root, RootKind, ScanProgress, ScanReport};
|
||||||
|
|
||||||
/// One row of the library grid.
|
/// One row of the library grid.
|
||||||
///
|
///
|
||||||
|
|||||||
@@ -318,7 +318,10 @@ fn flag_code(f: FlagState) -> i64 {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
fn availability_code(a: Availability) -> i64 {
|
/// The stored form of an availability. Shared with [`crate::walk`], which
|
||||||
|
/// writes the column this reads — two spellings of the same mapping would
|
||||||
|
/// filter for a state nothing ever writes.
|
||||||
|
pub(crate) fn availability_code(a: Availability) -> i64 {
|
||||||
match a {
|
match a {
|
||||||
Availability::MetadataOnly => 0,
|
Availability::MetadataOnly => 0,
|
||||||
Availability::Preview => 1,
|
Availability::Preview => 1,
|
||||||
|
|||||||
+10
-27
@@ -14,35 +14,13 @@
|
|||||||
//!
|
//!
|
||||||
//! This module holds the decision logic and the deletion-sweep rules; walking
|
//! This module holds the decision logic and the deletion-sweep rules; walking
|
||||||
//! an actual directory belongs to the platform layer, which supplies
|
//! an actual directory belongs to the platform layer, which supplies
|
||||||
//! [`DirState`] and [`DirEntry`].
|
//! [`DirState`] and [`DirEntry`]. [`crate::walk`] is what puts the two
|
||||||
|
//! together.
|
||||||
|
|
||||||
|
pub use dr_types::{DirEntry, DirState};
|
||||||
|
|
||||||
use dr_types::FormatFilter;
|
use dr_types::FormatFilter;
|
||||||
|
|
||||||
/// What a directory looked like when last scanned, and what it looks like now.
|
|
||||||
///
|
|
||||||
/// Both fields are cheap to obtain: one `stat` locally, one
|
|
||||||
/// `DocumentsContract` metadata query on SAF.
|
|
||||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
|
||||||
pub struct DirState {
|
|
||||||
pub mtime: i64,
|
|
||||||
/// Direct children, files and directories alike.
|
|
||||||
///
|
|
||||||
/// mtime alone misses a delete-and-create inside one timestamp tick, and
|
|
||||||
/// coarse-granularity providers widen that window. The count does not
|
|
||||||
/// close the hole — a paired add and remove moves neither — but a bare add
|
|
||||||
/// or remove moves the count, and those are far commoner.
|
|
||||||
pub entry_count: u32,
|
|
||||||
}
|
|
||||||
|
|
||||||
/// One entry from a directory listing.
|
|
||||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
|
||||||
pub struct DirEntry {
|
|
||||||
pub name: String,
|
|
||||||
pub is_dir: bool,
|
|
||||||
pub size: u64,
|
|
||||||
pub mtime: i64,
|
|
||||||
}
|
|
||||||
|
|
||||||
/// What the scanner should do with a directory, before listing it.
|
/// What the scanner should do with a directory, before listing it.
|
||||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||||
pub enum DirAction {
|
pub enum DirAction {
|
||||||
@@ -100,12 +78,17 @@ pub fn classify_entry(
|
|||||||
}
|
}
|
||||||
|
|
||||||
/// Outcome of a scan, which decides whether pruning may run.
|
/// Outcome of a scan, which decides whether pruning may run.
|
||||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
///
|
||||||
|
/// `Cancelled` is the default because a scan that has not run has proven
|
||||||
|
/// nothing absent, and every default in this area must fail towards keeping
|
||||||
|
/// photographs.
|
||||||
|
#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)]
|
||||||
pub enum ScanOutcome {
|
pub enum ScanOutcome {
|
||||||
/// Every reachable folder was visited.
|
/// Every reachable folder was visited.
|
||||||
Complete,
|
Complete,
|
||||||
/// The user cancelled. Partial state is valid — jobs are resumable — but
|
/// The user cancelled. Partial state is valid — jobs are resumable — but
|
||||||
/// unvisited folders must not be read as deleted.
|
/// unvisited folders must not be read as deleted.
|
||||||
|
#[default]
|
||||||
Cancelled,
|
Cancelled,
|
||||||
/// The root itself could not be opened: drive unplugged, SAF grant
|
/// The root itself could not be opened: drive unplugged, SAF grant
|
||||||
/// revoked, share unmounted.
|
/// revoked, share unmounted.
|
||||||
|
|||||||
File diff suppressed because it is too large
Load Diff
@@ -63,16 +63,25 @@ pub enum SourceRef {
|
|||||||
impl SourceRef {
|
impl SourceRef {
|
||||||
/// A stable, display-friendly name — the last path component.
|
/// A stable, display-friendly name — the last path component.
|
||||||
pub fn display_name(&self) -> &str {
|
pub fn display_name(&self) -> &str {
|
||||||
let full = match self {
|
let full = self.key();
|
||||||
SourceRef::Local { relative, .. } => relative.as_str(),
|
|
||||||
SourceRef::Document { document_id, .. } => document_id.as_str(),
|
|
||||||
SourceRef::Remote { path, .. } => path.as_str(),
|
|
||||||
};
|
|
||||||
// SAF document ids use ':' as a separator; paths use '/'. Split on
|
// SAF document ids use ':' as a separator; paths use '/'. Split on
|
||||||
// whichever appears last so both yield a sensible name.
|
// whichever appears last so both yield a sensible name.
|
||||||
full.rsplit(['/', ':']).next().unwrap_or(full)
|
full.rsplit(['/', ':']).next().unwrap_or(full)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// The stored form — what the catalog holds in `images.source_ref`.
|
||||||
|
///
|
||||||
|
/// Which of the three variants a key belongs to is not recorded beside it,
|
||||||
|
/// because the root already says: `roots.kind` is `'local' | 'saf' |
|
||||||
|
/// 'remote'`, and every image names its root.
|
||||||
|
pub fn key(&self) -> &str {
|
||||||
|
match self {
|
||||||
|
SourceRef::Local { relative, .. } => relative.as_str(),
|
||||||
|
SourceRef::Document { document_id, .. } => document_id.as_str(),
|
||||||
|
SourceRef::Remote { path, .. } => path.as_str(),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
/// Lowercase file extension, if any.
|
/// Lowercase file extension, if any.
|
||||||
///
|
///
|
||||||
/// Looks through a Nextcloud VFS placeholder suffix, so a dehydrated
|
/// Looks through a Nextcloud VFS placeholder suffix, so a dehydrated
|
||||||
@@ -251,6 +260,55 @@ impl FormatFilter {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// TRACES: FR-CAT-1 | NFR-PORT-1
|
||||||
|
/// What a directory looked like when last scanned, and what it looks like now.
|
||||||
|
///
|
||||||
|
/// Both fields are cheap to obtain: one `stat` plus a name-only directory read
|
||||||
|
/// locally, one `DocumentsContract` query on SAF. Neither costs a `stat` per
|
||||||
|
/// child, which is the whole point — see `dr_catalog::scan`.
|
||||||
|
///
|
||||||
|
/// Lives here rather than in either crate that uses it because it is the
|
||||||
|
/// sentence the platform layer says to the catalog: `dr-plat` produces it by
|
||||||
|
/// probing, `dr-catalog` stores it and compares. A copy on each side would be
|
||||||
|
/// two types that must agree by convention.
|
||||||
|
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||||
|
pub struct DirState {
|
||||||
|
/// Milliseconds since the epoch. **Not seconds**, and the difference is not
|
||||||
|
/// pedantry: change detection asks whether a timestamp *moved*, so the
|
||||||
|
/// granularity of the unit is the width of the window in which a change is
|
||||||
|
/// invisible. A second is long enough to copy a card and start a scan, and
|
||||||
|
/// anything happening inside one tick of the stored value looks exactly
|
||||||
|
/// like nothing happening — the directory is pruned and the photographs
|
||||||
|
/// never appear.
|
||||||
|
///
|
||||||
|
/// Both platforms can supply it: a filesystem records nanoseconds, and
|
||||||
|
/// SAF's `COLUMN_LAST_MODIFIED` is already in milliseconds.
|
||||||
|
pub mtime: i64,
|
||||||
|
/// Direct children, files and directories alike.
|
||||||
|
///
|
||||||
|
/// mtime alone misses a delete-and-create inside one timestamp tick, and
|
||||||
|
/// coarse-granularity providers widen that window. The count does not
|
||||||
|
/// close the hole — a paired add and remove moves neither — but a bare add
|
||||||
|
/// or remove moves the count, and those are far commoner.
|
||||||
|
pub entry_count: u32,
|
||||||
|
}
|
||||||
|
|
||||||
|
/// TRACES: FR-CAT-1 | NFR-PORT-1
|
||||||
|
/// One entry from a directory listing, as the scanner classifies it.
|
||||||
|
///
|
||||||
|
/// Deliberately carries no handle to the entry itself: this is what the
|
||||||
|
/// *decision* logic reads, and it must be constructible in a test without a
|
||||||
|
/// filesystem. The platform pairs it with the reference needed to reach the
|
||||||
|
/// entry (`dr_plat::Entry`).
|
||||||
|
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||||
|
pub struct DirEntry {
|
||||||
|
pub name: String,
|
||||||
|
pub is_dir: bool,
|
||||||
|
pub size: u64,
|
||||||
|
/// Milliseconds since the epoch, as in [`DirState::mtime`].
|
||||||
|
pub mtime: i64,
|
||||||
|
}
|
||||||
|
|
||||||
/// How much of an image is available locally (FR-NC-6c).
|
/// How much of an image is available locally (FR-NC-6c).
|
||||||
///
|
///
|
||||||
/// Surfaced in the UI so a user always knows what they have — the failure
|
/// Surfaced in the UI so a user always knows what they have — the failure
|
||||||
|
|||||||
@@ -5,7 +5,9 @@
|
|||||||
//! ARCH §10).
|
//! ARCH §10).
|
||||||
|
|
||||||
pub mod secrets;
|
pub mod secrets;
|
||||||
|
pub mod storage;
|
||||||
|
|
||||||
pub use secrets::{
|
pub use secrets::{
|
||||||
EphemeralSecretStore, PlatformSecretStore, SecretError, SecretKind, SecretRef, SecretStore,
|
EphemeralSecretStore, PlatformSecretStore, SecretError, SecretKind, SecretRef, SecretStore,
|
||||||
};
|
};
|
||||||
|
pub use storage::{DirRef, Entry, LocalStorage, Node, SeekableRead, Storage, StorageError};
|
||||||
|
|||||||
@@ -0,0 +1,861 @@
|
|||||||
|
//! TRACES: FR-CAT-1 | FR-CAT-1a | NFR-PORT-1 | NFR-PORT-3
|
||||||
|
//! Reaching stored bytes without naming a path (ARCH §3.1, §10).
|
||||||
|
//!
|
||||||
|
//! Android's Storage Access Framework hands out no filesystem path (ARCH §6.9),
|
||||||
|
//! so **nothing above this module may take one**. A library location is a
|
||||||
|
//! [`RootId`] the user granted; everything inside it is a [`DirRef`] or a
|
||||||
|
//! [`SourceRef`], both of which are opaque `(root, key)` pairs whose keys only
|
||||||
|
//! the implementation that produced them knows how to read.
|
||||||
|
//!
|
||||||
|
//! A `Path` therefore appears exactly once in the whole application: at
|
||||||
|
//! [`LocalStorage::grant`], where the folder the user picked is handed in. From
|
||||||
|
//! there on it is a `RootId`.
|
||||||
|
//!
|
||||||
|
//! # Adding Android SAF later
|
||||||
|
//!
|
||||||
|
//! It is a second implementation of [`Storage`] and no change at any call site.
|
||||||
|
//! Two properties of this API are what buy that, and both look like ceremony
|
||||||
|
//! until SAF is the thing being written:
|
||||||
|
//!
|
||||||
|
//! - **A listing hands back references, never names for the caller to join.**
|
||||||
|
//! A SAF document id is not composable — `parent_id + "/" + name` is not the
|
||||||
|
//! child's id, and the only way to learn a child's id is the children query
|
||||||
|
//! that produced the listing. So [`Entry`] carries the [`DirRef`] or
|
||||||
|
//! [`SourceRef`] the provider itself returned, and no caller ever builds one
|
||||||
|
//! by concatenation. [`LocalStorage`] could perfectly well have exposed a
|
||||||
|
//! "join a name onto a directory" helper; that helper is the one thing a SAF
|
||||||
|
//! implementation could not have provided.
|
||||||
|
//! - **A reference is a key that survives a restart.** The catalog stores the
|
||||||
|
//! key and rebuilds the reference with [`DirRef::from_parts`] on the next
|
||||||
|
//! run. On Linux the key is a relative path; on SAF it is a document id under
|
||||||
|
//! a persisted tree grant, which is re-resolvable for exactly the same
|
||||||
|
//! reason.
|
||||||
|
//!
|
||||||
|
//! What SAF will need in addition is the grant itself — the persisted tree URI,
|
||||||
|
//! which the catalog's `roots.grant_blob` column already has a home for, and
|
||||||
|
//! which is handed to the implementation at construction just as a path is
|
||||||
|
//! here.
|
||||||
|
|
||||||
|
use std::collections::BTreeMap;
|
||||||
|
use std::fmt;
|
||||||
|
use std::io::{Read, Seek};
|
||||||
|
use std::path::{Component, Path, PathBuf};
|
||||||
|
|
||||||
|
use dr_types::{ByteRange, DirEntry, DirState, RootId, SourceRef};
|
||||||
|
|
||||||
|
/// TRACES: FR-CAT-1a
|
||||||
|
/// An opaque, re-resolvable reference to a *directory* under a granted root.
|
||||||
|
///
|
||||||
|
/// The counterpart of [`SourceRef`], which addresses a file. One type rather
|
||||||
|
/// than a mirrored three-variant enum because a directory is never resolved by
|
||||||
|
/// anything except the storage that owns its root: the root's kind already
|
||||||
|
/// determines how the key is read, so a discriminator on each reference would
|
||||||
|
/// only repeat it.
|
||||||
|
///
|
||||||
|
/// The `key` is **opaque to callers** and stable across restarts. Its meaning
|
||||||
|
/// belongs to the implementation — a relative path under the root on a
|
||||||
|
/// filesystem, a `DocumentsContract` document id on SAF — and the catalog
|
||||||
|
/// stores it verbatim in `folders.path` so a later run can rebuild the
|
||||||
|
/// reference with [`DirRef::from_parts`].
|
||||||
|
#[derive(Debug, Clone, PartialEq, Eq, Hash, PartialOrd, Ord)]
|
||||||
|
pub struct DirRef {
|
||||||
|
root: RootId,
|
||||||
|
key: String,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl DirRef {
|
||||||
|
/// The granted root itself, which every walk starts from.
|
||||||
|
pub fn root(root: RootId) -> Self {
|
||||||
|
Self {
|
||||||
|
root,
|
||||||
|
key: String::new(),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Rebuild a reference from a key a previous scan stored.
|
||||||
|
///
|
||||||
|
/// The re-resolution FR-CAT-1a requires: after a restart the catalog holds
|
||||||
|
/// keys and nothing else, and a scan that could not resume from them would
|
||||||
|
/// have to walk the whole library to find the folder it left off in.
|
||||||
|
pub fn from_parts(root: RootId, key: impl Into<String>) -> Self {
|
||||||
|
Self {
|
||||||
|
root,
|
||||||
|
key: key.into(),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn root_id(&self) -> RootId {
|
||||||
|
self.root
|
||||||
|
}
|
||||||
|
|
||||||
|
/// The stored form. Meaningful only to the storage that produced it.
|
||||||
|
pub fn key(&self) -> &str {
|
||||||
|
&self.key
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Whether this is the granted root rather than something inside it.
|
||||||
|
///
|
||||||
|
/// The walk needs it: a failure at the root is the whole library being
|
||||||
|
/// unreachable, and the deletion sweep must not run; a failure below it is
|
||||||
|
/// one folder (FR-CAT-9).
|
||||||
|
pub fn is_root(&self) -> bool {
|
||||||
|
self.key.is_empty()
|
||||||
|
}
|
||||||
|
|
||||||
|
/// The last component, for exclusion checks and display.
|
||||||
|
pub fn name(&self) -> &str {
|
||||||
|
self.key.rsplit(['/', ':']).next().unwrap_or(&self.key)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
impl fmt::Display for DirRef {
|
||||||
|
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
|
||||||
|
if self.key.is_empty() {
|
||||||
|
write!(f, "root {}", self.root.0)
|
||||||
|
} else {
|
||||||
|
write!(f, "{}", self.key)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// How to reach a listed entry.
|
||||||
|
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||||
|
pub enum Node {
|
||||||
|
Dir(DirRef),
|
||||||
|
File(SourceRef),
|
||||||
|
}
|
||||||
|
|
||||||
|
/// One entry from a listing: what the scanner classifies on, and how to reach
|
||||||
|
/// it.
|
||||||
|
///
|
||||||
|
/// `meta.is_dir` and the [`Node`] variant always agree — the implementation
|
||||||
|
/// sets both from one observation. They are separate because they serve
|
||||||
|
/// different readers: `meta` goes to `dr_catalog::scan`, which decides, and
|
||||||
|
/// `node` goes to whatever acts on the decision.
|
||||||
|
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||||
|
pub struct Entry {
|
||||||
|
pub meta: DirEntry,
|
||||||
|
pub node: Node,
|
||||||
|
}
|
||||||
|
|
||||||
|
/// A stream that can be read and seeked — what a decoder wants.
|
||||||
|
///
|
||||||
|
/// Blanket-implemented, so a `File`, a `Cursor<Vec<u8>>` in a test, and a
|
||||||
|
/// future SAF `ParcelFileDescriptor` wrapper all qualify without ceremony.
|
||||||
|
pub trait SeekableRead: Read + Seek + Send {}
|
||||||
|
impl<T: Read + Seek + Send> SeekableRead for T {}
|
||||||
|
|
||||||
|
/// TRACES: NFR-ARCH-4
|
||||||
|
/// Something went wrong reaching storage.
|
||||||
|
///
|
||||||
|
/// Typed, and never a panic: a library is on removable media, on a network
|
||||||
|
/// mount, or behind a permission the user can revoke while the app is running,
|
||||||
|
/// so every one of these is a normal Tuesday rather than a bug.
|
||||||
|
///
|
||||||
|
/// Distinct from [`dr_types::SourceError`], which is what the *decode* path
|
||||||
|
/// sees. This one also speaks about roots and directories, which a decoder has
|
||||||
|
/// no concept of.
|
||||||
|
#[derive(Debug, thiserror::Error)]
|
||||||
|
pub enum StorageError {
|
||||||
|
/// A reference naming a root this storage was never granted.
|
||||||
|
///
|
||||||
|
/// Almost always a stale catalog row: the library was removed and its rows
|
||||||
|
/// outlived it.
|
||||||
|
#[error("root {0} was not granted to this storage")]
|
||||||
|
UnknownRoot(u64),
|
||||||
|
|
||||||
|
/// A reference of the wrong shape — a SAF document id handed to the
|
||||||
|
/// filesystem implementation, or the reverse.
|
||||||
|
///
|
||||||
|
/// Refused rather than guessed at: the two address spaces have no overlap,
|
||||||
|
/// and a guess would read the wrong file rather than fail.
|
||||||
|
#[error("{0}")]
|
||||||
|
Unsupported(&'static str),
|
||||||
|
|
||||||
|
/// A stored key that would leave its root.
|
||||||
|
///
|
||||||
|
/// The catalog is a file on disk that other programs can edit, so a key
|
||||||
|
/// containing `..` is possible however it got there. Refused, because a
|
||||||
|
/// grant to one folder must not become a read of the whole filesystem
|
||||||
|
/// (NFR-SEC-1).
|
||||||
|
#[error("key {0:?} escapes its root")]
|
||||||
|
EscapesRoot(String),
|
||||||
|
|
||||||
|
#[error("not found: {0}")]
|
||||||
|
NotFound(String),
|
||||||
|
|
||||||
|
#[error("permission denied: {0}")]
|
||||||
|
PermissionDenied(String),
|
||||||
|
|
||||||
|
#[error("not a directory: {0}")]
|
||||||
|
NotADirectory(String),
|
||||||
|
|
||||||
|
#[error("range reads unsupported by this storage")]
|
||||||
|
RangeUnsupported,
|
||||||
|
|
||||||
|
#[error("io error: {0}")]
|
||||||
|
Io(String),
|
||||||
|
}
|
||||||
|
|
||||||
|
/// TRACES: FR-CAT-1 | FR-CAT-1a | NFR-PORT-1
|
||||||
|
/// Enumerate and read the contents of granted library roots.
|
||||||
|
///
|
||||||
|
/// Implemented per platform and injected at construction, so `core/` contains
|
||||||
|
/// no `#[cfg(target_os)]` (ARCH §10).
|
||||||
|
pub trait Storage: Send + Sync {
|
||||||
|
/// Every root this storage can currently reach.
|
||||||
|
fn roots(&self) -> Vec<RootId>;
|
||||||
|
|
||||||
|
/// The granted root as a directory, which is where a walk begins.
|
||||||
|
///
|
||||||
|
/// Fails on a root that was never granted, so a stale catalog row is a
|
||||||
|
/// typed error rather than an empty library.
|
||||||
|
fn root_dir(&self, root: RootId) -> Result<DirRef, StorageError>;
|
||||||
|
|
||||||
|
/// Probe a directory without reading its contents.
|
||||||
|
///
|
||||||
|
/// The cheap half of the pair, and the reason incremental scanning is
|
||||||
|
/// affordable: this costs one `stat` plus a name-only directory read,
|
||||||
|
/// where [`list`](Self::list) costs a `stat` per child. On a library of 2k
|
||||||
|
/// folders and 50k images that is 2k probes against 50k, which is the
|
||||||
|
/// difference between meeting and missing NFR-P1.
|
||||||
|
fn dir_state(&self, dir: &DirRef) -> Result<DirState, StorageError>;
|
||||||
|
|
||||||
|
/// List a directory's direct children, with the metadata to classify them.
|
||||||
|
///
|
||||||
|
/// Returns the whole listing rather than streaming it through a callback,
|
||||||
|
/// because the caller needs the complete set at once: an image in the
|
||||||
|
/// catalog that this listing does *not* contain has been deleted, and that
|
||||||
|
/// conclusion cannot be drawn one entry at a time.
|
||||||
|
fn list(&self, dir: &DirRef) -> Result<Vec<Entry>, StorageError>;
|
||||||
|
|
||||||
|
/// Open a seekable stream over a source.
|
||||||
|
fn open(&self, src: &SourceRef) -> Result<Box<dyn SeekableRead>, StorageError>;
|
||||||
|
|
||||||
|
/// Read a byte range without opening the whole source.
|
||||||
|
///
|
||||||
|
/// First-class rather than a convenience over [`open`](Self::open), because
|
||||||
|
/// for the case that matters it is a different operation and not a smaller
|
||||||
|
/// one: extracting an embedded JPEG preview from an 80 MB RAW over the
|
||||||
|
/// network transfers 1–3 MB (ARCH §3.1, FR-CULL-2, FR-NC-3).
|
||||||
|
///
|
||||||
|
/// A range reaching past the end yields the bytes that exist — a short read
|
||||||
|
/// is the honest answer, and a caller needing exactly `n` bytes must say so
|
||||||
|
/// by checking the length.
|
||||||
|
fn read_range(&self, src: &SourceRef, range: ByteRange) -> Result<Vec<u8>, StorageError>;
|
||||||
|
}
|
||||||
|
|
||||||
|
/// TRACES: FR-PLAT-LIN-1 | NFR-PORT-1
|
||||||
|
/// The filesystem implementation: a root is a directory the user picked.
|
||||||
|
///
|
||||||
|
/// Also the right implementation for a Flatpak, where the portal returns a real
|
||||||
|
/// path the sandbox can see (FR-PLAT-LIN-3).
|
||||||
|
///
|
||||||
|
/// Present on Android too, and harmless there: the type compiles wherever
|
||||||
|
/// `std::fs` does, so `ui/` can name it unconditionally. It is not how an
|
||||||
|
/// Android user's library is reached — that is SAF, and ARCH §6.9 explains
|
||||||
|
/// why nothing else is on offer.
|
||||||
|
#[derive(Debug, Default)]
|
||||||
|
pub struct LocalStorage {
|
||||||
|
/// Ordered so [`roots`](Storage::roots) is stable, which keeps a scan of
|
||||||
|
/// several roots reproducible.
|
||||||
|
roots: BTreeMap<RootId, PathBuf>,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl LocalStorage {
|
||||||
|
pub fn new() -> Self {
|
||||||
|
Self::default()
|
||||||
|
}
|
||||||
|
|
||||||
|
/// One granted root, the common case.
|
||||||
|
pub fn with_root(root: RootId, dir: impl Into<PathBuf>) -> Self {
|
||||||
|
let mut s = Self::new();
|
||||||
|
s.grant(root, dir);
|
||||||
|
s
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Record that the user granted `dir` as `root`.
|
||||||
|
///
|
||||||
|
/// **The only place a `Path` enters the application.** Above this line a
|
||||||
|
/// library location is a `RootId`, which is what lets the same catalog and
|
||||||
|
/// the same scanner run against SAF, where no path exists at all.
|
||||||
|
pub fn grant(&mut self, root: RootId, dir: impl Into<PathBuf>) {
|
||||||
|
self.roots.insert(root, dir.into());
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Where a root sits, for the app that granted it — to show the user, or to
|
||||||
|
/// store so the grant survives a restart.
|
||||||
|
pub fn root_path(&self, root: RootId) -> Option<&Path> {
|
||||||
|
self.roots.get(&root).map(|p| p.as_path())
|
||||||
|
}
|
||||||
|
|
||||||
|
fn base(&self, root: RootId) -> Result<&Path, StorageError> {
|
||||||
|
self.roots
|
||||||
|
.get(&root)
|
||||||
|
.map(|p| p.as_path())
|
||||||
|
.ok_or(StorageError::UnknownRoot(root.0))
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Turn a `(root, key)` pair back into a path, refusing anything that
|
||||||
|
/// leaves the root.
|
||||||
|
///
|
||||||
|
/// Every component must be an ordinary name: `..` would climb out, and an
|
||||||
|
/// absolute key would discard the root entirely — `Path::join` silently
|
||||||
|
/// replaces rather than appends when handed one, which is how a "relative"
|
||||||
|
/// path of `/etc` becomes a read of `/etc`.
|
||||||
|
fn resolve(&self, root: RootId, key: &str) -> Result<PathBuf, StorageError> {
|
||||||
|
let base = self.base(root)?;
|
||||||
|
if key.is_empty() {
|
||||||
|
return Ok(base.to_path_buf());
|
||||||
|
}
|
||||||
|
let rel = Path::new(key);
|
||||||
|
if !rel.components().all(|c| matches!(c, Component::Normal(_))) {
|
||||||
|
return Err(StorageError::EscapesRoot(key.to_string()));
|
||||||
|
}
|
||||||
|
Ok(base.join(rel))
|
||||||
|
}
|
||||||
|
|
||||||
|
fn file_path(&self, src: &SourceRef) -> Result<PathBuf, StorageError> {
|
||||||
|
match src {
|
||||||
|
SourceRef::Local { root, relative } => self.resolve(*root, relative),
|
||||||
|
SourceRef::Document { .. } => Err(StorageError::Unsupported(
|
||||||
|
"a SAF document reference cannot be read from the filesystem",
|
||||||
|
)),
|
||||||
|
SourceRef::Remote { .. } => Err(StorageError::Unsupported(
|
||||||
|
"a remote reference is resolved by the sync layer, not by storage",
|
||||||
|
)),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// A child's key, built the one place that is allowed to build one.
|
||||||
|
///
|
||||||
|
/// Concatenation is safe here and only here: this implementation *chose* to
|
||||||
|
/// make its keys relative paths, so it is the only code entitled to know that
|
||||||
|
/// they compose. A SAF implementation has no equivalent (see the module docs),
|
||||||
|
/// which is why this is a private helper and not a method on [`DirRef`].
|
||||||
|
fn child_key(parent: &str, name: &str) -> String {
|
||||||
|
if parent.is_empty() {
|
||||||
|
name.to_string()
|
||||||
|
} else {
|
||||||
|
format!("{parent}/{name}")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
impl Storage for LocalStorage {
|
||||||
|
fn roots(&self) -> Vec<RootId> {
|
||||||
|
self.roots.keys().copied().collect()
|
||||||
|
}
|
||||||
|
|
||||||
|
fn root_dir(&self, root: RootId) -> Result<DirRef, StorageError> {
|
||||||
|
self.base(root)?;
|
||||||
|
Ok(DirRef::root(root))
|
||||||
|
}
|
||||||
|
|
||||||
|
fn dir_state(&self, dir: &DirRef) -> Result<DirState, StorageError> {
|
||||||
|
let path = self.resolve(dir.root_id(), dir.key())?;
|
||||||
|
let meta = std::fs::metadata(&path).map_err(|e| map_io(&path, e))?;
|
||||||
|
if !meta.is_dir() {
|
||||||
|
return Err(StorageError::NotADirectory(path.display().to_string()));
|
||||||
|
}
|
||||||
|
|
||||||
|
// Names only — `read_dir` yields entries from `getdents` without a
|
||||||
|
// `stat` per child, so the count costs one pass and no per-file I/O.
|
||||||
|
// That is what makes this the cheap probe `list` is not.
|
||||||
|
let entry_count = std::fs::read_dir(&path)
|
||||||
|
.map_err(|e| map_io(&path, e))?
|
||||||
|
.count();
|
||||||
|
|
||||||
|
Ok(DirState {
|
||||||
|
mtime: modified_millis(&meta),
|
||||||
|
// Saturating rather than wrapping: a directory of four billion
|
||||||
|
// entries would otherwise wrap to a small number and could compare
|
||||||
|
// equal after a change. It cannot happen, and being wrong about it
|
||||||
|
// would be silent.
|
||||||
|
entry_count: u32::try_from(entry_count).unwrap_or(u32::MAX),
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
fn list(&self, dir: &DirRef) -> Result<Vec<Entry>, StorageError> {
|
||||||
|
let path = self.resolve(dir.root_id(), dir.key())?;
|
||||||
|
let read = std::fs::read_dir(&path).map_err(|e| map_io(&path, e))?;
|
||||||
|
|
||||||
|
let mut out = Vec::new();
|
||||||
|
for entry in read {
|
||||||
|
let entry = match entry {
|
||||||
|
Ok(e) => e,
|
||||||
|
// One unreadable entry is not an unreadable directory. Skipping
|
||||||
|
// it loses one file; failing the listing would make the folder
|
||||||
|
// look empty, and an empty folder is a *deletion* to the sweep.
|
||||||
|
Err(e) => {
|
||||||
|
log::debug!("list {}: skipping unreadable entry: {e}", path.display());
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
// A name that is not UTF-8 cannot become a key, and a lossy
|
||||||
|
// conversion would produce a key that resolves to nothing — an
|
||||||
|
// image catalogued and then permanently unreadable. Skipped, and
|
||||||
|
// said out loud, because the user's file is real and we are
|
||||||
|
// choosing not to see it.
|
||||||
|
let Some(name) = entry.file_name().to_str().map(str::to_owned) else {
|
||||||
|
log::warn!(
|
||||||
|
"list {}: skipping {:?}, whose name is not valid UTF-8",
|
||||||
|
path.display(),
|
||||||
|
entry.file_name()
|
||||||
|
);
|
||||||
|
continue;
|
||||||
|
};
|
||||||
|
|
||||||
|
// Follows symlinks, unlike `DirEntry::metadata`. A photographer who
|
||||||
|
// symlinks last year's drive into the library means it as part of
|
||||||
|
// the library. The walk's depth limit is what stops a loop.
|
||||||
|
let child = path.join(&name);
|
||||||
|
let meta = match std::fs::metadata(&child) {
|
||||||
|
Ok(m) => m,
|
||||||
|
// A broken symlink, or a file deleted between the listing and
|
||||||
|
// this stat. Neither is an error worth failing a folder for.
|
||||||
|
Err(e) => {
|
||||||
|
log::debug!("list {}: skipping {name}: {e}", path.display());
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
let key = child_key(dir.key(), &name);
|
||||||
|
let is_dir = meta.is_dir();
|
||||||
|
out.push(Entry {
|
||||||
|
meta: DirEntry {
|
||||||
|
name,
|
||||||
|
is_dir,
|
||||||
|
size: meta.len(),
|
||||||
|
mtime: modified_millis(&meta),
|
||||||
|
},
|
||||||
|
node: if is_dir {
|
||||||
|
Node::Dir(DirRef::from_parts(dir.root_id(), key))
|
||||||
|
} else {
|
||||||
|
Node::File(SourceRef::Local {
|
||||||
|
root: dir.root_id(),
|
||||||
|
relative: key,
|
||||||
|
})
|
||||||
|
},
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
// Directory order is filesystem order, which is arbitrary and differs
|
||||||
|
// between runs. Sorting makes a scan reproducible and a test able to
|
||||||
|
// assert on what it found.
|
||||||
|
out.sort_by(|a, b| a.meta.name.cmp(&b.meta.name));
|
||||||
|
Ok(out)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn open(&self, src: &SourceRef) -> Result<Box<dyn SeekableRead>, StorageError> {
|
||||||
|
let path = self.file_path(src)?;
|
||||||
|
let file = std::fs::File::open(&path).map_err(|e| map_io(&path, e))?;
|
||||||
|
Ok(Box::new(file))
|
||||||
|
}
|
||||||
|
|
||||||
|
fn read_range(&self, src: &SourceRef, range: ByteRange) -> Result<Vec<u8>, StorageError> {
|
||||||
|
use std::io::SeekFrom;
|
||||||
|
|
||||||
|
if range.end <= range.start {
|
||||||
|
return Ok(Vec::new());
|
||||||
|
}
|
||||||
|
let path = self.file_path(src)?;
|
||||||
|
let mut file = std::fs::File::open(&path).map_err(|e| map_io(&path, e))?;
|
||||||
|
file.seek(SeekFrom::Start(range.start))
|
||||||
|
.map_err(|e| map_io(&path, e))?;
|
||||||
|
|
||||||
|
// `take` and grow, rather than a buffer sized to the request: the range
|
||||||
|
// comes from a header the file itself declared, and a corrupt one
|
||||||
|
// asking for four gigabytes must not be allocated before it is known
|
||||||
|
// that four gigabytes exist (NFR-SEC-1).
|
||||||
|
let mut buf = Vec::new();
|
||||||
|
file.take(range.end - range.start)
|
||||||
|
.read_to_end(&mut buf)
|
||||||
|
.map_err(|e| map_io(&path, e))?;
|
||||||
|
Ok(buf)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Milliseconds since the epoch, or 0 where the platform will not say.
|
||||||
|
///
|
||||||
|
/// Milliseconds because that is the unit change detection is expressed in, and
|
||||||
|
/// the unit's granularity is the width of the window in which a change is
|
||||||
|
/// invisible — see [`dr_types::DirState::mtime`]. Seconds would hide a card
|
||||||
|
/// imported and scanned within the same tick.
|
||||||
|
///
|
||||||
|
/// A file whose mtime is unreadable compares equal to itself forever and so is
|
||||||
|
/// never re-read. That is the better failure: the alternative, a value that
|
||||||
|
/// changes each time it is asked for, would re-process the file on every scan.
|
||||||
|
fn modified_millis(meta: &std::fs::Metadata) -> i64 {
|
||||||
|
let Ok(t) = meta.modified() else {
|
||||||
|
return 0;
|
||||||
|
};
|
||||||
|
match t.duration_since(std::time::UNIX_EPOCH) {
|
||||||
|
Ok(d) => i64::try_from(d.as_millis()).unwrap_or(i64::MAX),
|
||||||
|
// Before 1970. Rare, but an archive of digitised film can carry one,
|
||||||
|
// and it must not become a huge positive number.
|
||||||
|
Err(e) => i64::try_from(e.duration().as_millis())
|
||||||
|
.unwrap_or(i64::MAX)
|
||||||
|
.saturating_neg(),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn map_io(path: &Path, e: std::io::Error) -> StorageError {
|
||||||
|
let what = path.display().to_string();
|
||||||
|
match e.kind() {
|
||||||
|
std::io::ErrorKind::NotFound => StorageError::NotFound(what),
|
||||||
|
std::io::ErrorKind::PermissionDenied => StorageError::PermissionDenied(what),
|
||||||
|
_ => StorageError::Io(format!("{what}: {e}")),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
mod tests {
|
||||||
|
use super::*;
|
||||||
|
use std::fs;
|
||||||
|
|
||||||
|
/// A throwaway directory tree, removed when the test ends.
|
||||||
|
struct Tree(PathBuf);
|
||||||
|
|
||||||
|
impl Tree {
|
||||||
|
fn new(name: &str) -> Self {
|
||||||
|
let dir = std::env::temp_dir().join(format!(
|
||||||
|
"dr-plat-{name}-{}-{:?}",
|
||||||
|
std::process::id(),
|
||||||
|
std::thread::current().id()
|
||||||
|
));
|
||||||
|
let _ = fs::remove_dir_all(&dir);
|
||||||
|
fs::create_dir_all(&dir).expect("temp dir");
|
||||||
|
Tree(dir)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn dir(&self, rel: &str) -> &Self {
|
||||||
|
fs::create_dir_all(self.0.join(rel)).expect("mkdir");
|
||||||
|
self
|
||||||
|
}
|
||||||
|
|
||||||
|
fn file(&self, rel: &str, bytes: &[u8]) -> &Self {
|
||||||
|
let p = self.0.join(rel);
|
||||||
|
if let Some(parent) = p.parent() {
|
||||||
|
fs::create_dir_all(parent).expect("mkdir");
|
||||||
|
}
|
||||||
|
fs::write(p, bytes).expect("write");
|
||||||
|
self
|
||||||
|
}
|
||||||
|
|
||||||
|
fn storage(&self) -> LocalStorage {
|
||||||
|
LocalStorage::with_root(RootId(1), self.0.clone())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
impl Drop for Tree {
|
||||||
|
fn drop(&mut self) {
|
||||||
|
let _ = fs::remove_dir_all(&self.0);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
const ROOT: RootId = RootId(1);
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn a_listing_names_files_and_directories_apart() {
|
||||||
|
let t = Tree::new("listing");
|
||||||
|
t.file("IMG_0001.CR3", b"raw").dir("2026");
|
||||||
|
let s = t.storage();
|
||||||
|
|
||||||
|
let entries = s.list(&s.root_dir(ROOT).unwrap()).unwrap();
|
||||||
|
assert_eq!(entries.len(), 2);
|
||||||
|
assert_eq!(entries[0].meta.name, "2026");
|
||||||
|
assert!(entries[0].meta.is_dir);
|
||||||
|
assert!(matches!(entries[0].node, Node::Dir(_)));
|
||||||
|
assert_eq!(entries[1].meta.name, "IMG_0001.CR3");
|
||||||
|
assert_eq!(entries[1].meta.size, 3);
|
||||||
|
assert!(matches!(entries[1].node, Node::File(_)));
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn a_listing_hands_back_references_the_caller_never_composes() {
|
||||||
|
// The property that makes a SAF implementation a drop-in: the child's
|
||||||
|
// reference comes from the listing, because on SAF it is the only place
|
||||||
|
// it can come from. If a test ever has to build one by joining strings,
|
||||||
|
// the abstraction has already leaked.
|
||||||
|
let t = Tree::new("refs");
|
||||||
|
t.file("2026/08/IMG_0042.CR3", b"raw");
|
||||||
|
let s = t.storage();
|
||||||
|
|
||||||
|
let year = match &s.list(&s.root_dir(ROOT).unwrap()).unwrap()[0].node {
|
||||||
|
Node::Dir(d) => d.clone(),
|
||||||
|
other => panic!("expected a directory, got {other:?}"),
|
||||||
|
};
|
||||||
|
let month = match &s.list(&year).unwrap()[0].node {
|
||||||
|
Node::Dir(d) => d.clone(),
|
||||||
|
other => panic!("expected a directory, got {other:?}"),
|
||||||
|
};
|
||||||
|
let file = match &s.list(&month).unwrap()[0].node {
|
||||||
|
Node::File(f) => f.clone(),
|
||||||
|
other => panic!("expected a file, got {other:?}"),
|
||||||
|
};
|
||||||
|
|
||||||
|
assert_eq!(
|
||||||
|
file,
|
||||||
|
SourceRef::Local {
|
||||||
|
root: ROOT,
|
||||||
|
relative: "2026/08/IMG_0042.CR3".into()
|
||||||
|
}
|
||||||
|
);
|
||||||
|
assert!(s.open(&file).is_ok());
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn a_stored_key_reopens_the_same_directory_after_a_restart() {
|
||||||
|
// What FR-CAT-1a's "re-resolvable" means in practice: the catalog keeps
|
||||||
|
// keys, not handles, and a scan resuming tomorrow rebuilds the
|
||||||
|
// reference from one. Without this a restart is a full rewalk.
|
||||||
|
let t = Tree::new("reresolve");
|
||||||
|
t.file("2026/IMG.CR3", b"raw");
|
||||||
|
let s = t.storage();
|
||||||
|
|
||||||
|
let key = match &s.list(&s.root_dir(ROOT).unwrap()).unwrap()[0].node {
|
||||||
|
Node::Dir(d) => d.key().to_string(),
|
||||||
|
other => panic!("expected a directory, got {other:?}"),
|
||||||
|
};
|
||||||
|
|
||||||
|
let rebuilt = DirRef::from_parts(ROOT, key);
|
||||||
|
assert_eq!(s.list(&rebuilt).unwrap()[0].meta.name, "IMG.CR3");
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn a_key_that_climbs_out_of_the_root_is_refused() {
|
||||||
|
// A grant is to one folder. The catalog is an ordinary file that other
|
||||||
|
// programs can edit, so a `..` in a key is reachable however it got
|
||||||
|
// there, and honouring it would turn a grant to ~/Photos into a read of
|
||||||
|
// the whole filesystem (NFR-SEC-1).
|
||||||
|
let t = Tree::new("escape");
|
||||||
|
let s = t.storage();
|
||||||
|
|
||||||
|
for key in ["../etc", "a/../../etc", "/etc"] {
|
||||||
|
let dir = DirRef::from_parts(ROOT, key);
|
||||||
|
assert!(
|
||||||
|
matches!(s.dir_state(&dir), Err(StorageError::EscapesRoot(_))),
|
||||||
|
"{key} was not refused"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn an_absolute_key_does_not_silently_replace_the_root() {
|
||||||
|
// `Path::join` replaces rather than appends when given an absolute
|
||||||
|
// path, so this one is not merely an escape — it is an escape that
|
||||||
|
// looks like ordinary joining and would never be noticed in review.
|
||||||
|
let t = Tree::new("absolute");
|
||||||
|
let s = t.storage();
|
||||||
|
let src = SourceRef::Local {
|
||||||
|
root: ROOT,
|
||||||
|
relative: "/etc/passwd".into(),
|
||||||
|
};
|
||||||
|
assert!(matches!(
|
||||||
|
s.read_range(&src, 0..16),
|
||||||
|
Err(StorageError::EscapesRoot(_))
|
||||||
|
));
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn probing_a_directory_reports_what_change_detection_needs() {
|
||||||
|
let t = Tree::new("probe");
|
||||||
|
t.file("a.CR3", b"1").file("b.CR3", b"2");
|
||||||
|
let s = t.storage();
|
||||||
|
|
||||||
|
let before = s.dir_state(&s.root_dir(ROOT).unwrap()).unwrap();
|
||||||
|
assert_eq!(before.entry_count, 2);
|
||||||
|
|
||||||
|
t.file("c.CR3", b"3");
|
||||||
|
let after = s.dir_state(&s.root_dir(ROOT).unwrap()).unwrap();
|
||||||
|
assert_ne!(
|
||||||
|
before, after,
|
||||||
|
"an added file must move the state, or the folder is pruned and the \
|
||||||
|
image never enters the catalog"
|
||||||
|
);
|
||||||
|
assert_eq!(after.entry_count, 3);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn the_probe_counts_directories_as_well_as_files() {
|
||||||
|
// A new subfolder full of images changes nothing about the parent's
|
||||||
|
// files. If the count ignored directories, the parent would look
|
||||||
|
// unchanged and the whole subtree would go unseen.
|
||||||
|
let t = Tree::new("probe-dirs");
|
||||||
|
t.file("a.CR3", b"1");
|
||||||
|
let s = t.storage();
|
||||||
|
let before = s.dir_state(&s.root_dir(ROOT).unwrap()).unwrap();
|
||||||
|
|
||||||
|
t.dir("2026");
|
||||||
|
let after = s.dir_state(&s.root_dir(ROOT).unwrap()).unwrap();
|
||||||
|
assert_eq!(after.entry_count, before.entry_count + 1);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn a_range_read_returns_only_the_bytes_asked_for() {
|
||||||
|
let t = Tree::new("range");
|
||||||
|
t.file("IMG.CR3", b"0123456789");
|
||||||
|
let s = t.storage();
|
||||||
|
let src = SourceRef::Local {
|
||||||
|
root: ROOT,
|
||||||
|
relative: "IMG.CR3".into(),
|
||||||
|
};
|
||||||
|
|
||||||
|
assert_eq!(s.read_range(&src, 2..6).unwrap(), b"2345");
|
||||||
|
assert_eq!(s.read_range(&src, 0..0).unwrap(), b"");
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn a_range_past_the_end_is_short_rather_than_an_error() {
|
||||||
|
// Preview offsets come out of the file's own header. A truncated or
|
||||||
|
// mis-parsed one must yield "here is what exists", not a failed decode
|
||||||
|
// and not a four-gigabyte allocation (NFR-SEC-1).
|
||||||
|
let t = Tree::new("range-eof");
|
||||||
|
t.file("IMG.CR3", b"0123456789");
|
||||||
|
let s = t.storage();
|
||||||
|
let src = SourceRef::Local {
|
||||||
|
root: ROOT,
|
||||||
|
relative: "IMG.CR3".into(),
|
||||||
|
};
|
||||||
|
|
||||||
|
assert_eq!(s.read_range(&src, 8..u64::MAX / 2).unwrap(), b"89");
|
||||||
|
assert_eq!(s.read_range(&src, 999..1_000).unwrap(), b"");
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn an_open_stream_can_seek() {
|
||||||
|
// The decoders need it: a RAW's preview lives at an offset the header
|
||||||
|
// names, and a forward-only stream would mean reading 80 MB to get 2.
|
||||||
|
let t = Tree::new("seek");
|
||||||
|
t.file("IMG.CR3", b"0123456789");
|
||||||
|
let s = t.storage();
|
||||||
|
let mut r = s
|
||||||
|
.open(&SourceRef::Local {
|
||||||
|
root: ROOT,
|
||||||
|
relative: "IMG.CR3".into(),
|
||||||
|
})
|
||||||
|
.unwrap();
|
||||||
|
|
||||||
|
r.seek(std::io::SeekFrom::Start(5)).unwrap();
|
||||||
|
let mut buf = [0u8; 2];
|
||||||
|
r.read_exact(&mut buf).unwrap();
|
||||||
|
assert_eq!(&buf, b"56");
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn an_ungranted_root_is_a_typed_error_not_an_empty_library() {
|
||||||
|
// A removed library leaves catalog rows behind. Reporting them as
|
||||||
|
// "nothing here" would let the deletion sweep take the lot (FR-CAT-9).
|
||||||
|
let s = LocalStorage::new();
|
||||||
|
assert!(matches!(
|
||||||
|
s.root_dir(RootId(7)),
|
||||||
|
Err(StorageError::UnknownRoot(7))
|
||||||
|
));
|
||||||
|
assert!(matches!(
|
||||||
|
s.dir_state(&DirRef::root(RootId(7))),
|
||||||
|
Err(StorageError::UnknownRoot(7))
|
||||||
|
));
|
||||||
|
assert!(s.roots().is_empty());
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn a_missing_directory_is_not_reported_as_empty() {
|
||||||
|
// The same failure from the other direction: an unplugged drive must
|
||||||
|
// error, because an empty listing means every image under it was
|
||||||
|
// deleted.
|
||||||
|
let t = Tree::new("missing");
|
||||||
|
let s = t.storage();
|
||||||
|
let gone = DirRef::from_parts(ROOT, "nowhere");
|
||||||
|
assert!(matches!(s.dir_state(&gone), Err(StorageError::NotFound(_))));
|
||||||
|
assert!(matches!(s.list(&gone), Err(StorageError::NotFound(_))));
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn a_reference_of_the_wrong_kind_is_refused_rather_than_guessed_at() {
|
||||||
|
// A SAF document id is not a path. Treating it as one would resolve to
|
||||||
|
// some other file, which is worse than failing.
|
||||||
|
let t = Tree::new("wrong-kind");
|
||||||
|
let s = t.storage();
|
||||||
|
let saf = SourceRef::Document {
|
||||||
|
tree: ROOT,
|
||||||
|
document_id: "primary:DCIM/IMG.CR3".into(),
|
||||||
|
};
|
||||||
|
assert!(matches!(s.open(&saf), Err(StorageError::Unsupported(_))));
|
||||||
|
let remote = SourceRef::Remote {
|
||||||
|
file_id: 1,
|
||||||
|
path: "Photos/IMG.CR3".into(),
|
||||||
|
};
|
||||||
|
assert!(matches!(s.open(&remote), Err(StorageError::Unsupported(_))));
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn several_roots_coexist() {
|
||||||
|
// Two libraries on two drives is FR-CAT-1's "one or more roots", and
|
||||||
|
// each reference carries which one it belongs to.
|
||||||
|
let a = Tree::new("multi-a");
|
||||||
|
a.file("a.CR3", b"1");
|
||||||
|
let b = Tree::new("multi-b");
|
||||||
|
b.file("b.CR3", b"2");
|
||||||
|
|
||||||
|
let mut s = LocalStorage::new();
|
||||||
|
s.grant(RootId(1), a.0.clone());
|
||||||
|
s.grant(RootId(2), b.0.clone());
|
||||||
|
|
||||||
|
assert_eq!(s.roots(), vec![RootId(1), RootId(2)]);
|
||||||
|
assert_eq!(
|
||||||
|
s.list(&s.root_dir(RootId(2)).unwrap()).unwrap()[0]
|
||||||
|
.meta
|
||||||
|
.name,
|
||||||
|
"b.CR3"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn a_listing_is_ordered_the_same_way_twice() {
|
||||||
|
// Filesystem order is arbitrary and differs between runs; a scan that
|
||||||
|
// depended on it would produce a different catalog each time.
|
||||||
|
let t = Tree::new("order");
|
||||||
|
for n in ["c.CR3", "a.CR3", "b.CR3"] {
|
||||||
|
t.file(n, b"x");
|
||||||
|
}
|
||||||
|
let s = t.storage();
|
||||||
|
let names: Vec<String> = s
|
||||||
|
.list(&s.root_dir(ROOT).unwrap())
|
||||||
|
.unwrap()
|
||||||
|
.into_iter()
|
||||||
|
.map(|e| e.meta.name)
|
||||||
|
.collect();
|
||||||
|
assert_eq!(names, vec!["a.CR3", "b.CR3", "c.CR3"]);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(unix)]
|
||||||
|
#[test]
|
||||||
|
fn a_name_that_is_not_utf8_is_skipped_rather_than_mangled() {
|
||||||
|
// A lossy conversion would produce a key that resolves to nothing: the
|
||||||
|
// image would be catalogued and then permanently unopenable. Better to
|
||||||
|
// not see the file than to promise it and fail later.
|
||||||
|
use std::os::unix::ffi::OsStrExt;
|
||||||
|
|
||||||
|
let t = Tree::new("non-utf8");
|
||||||
|
t.file("good.CR3", b"1");
|
||||||
|
let bad = t.0.join(std::ffi::OsStr::from_bytes(b"bad\xff.CR3"));
|
||||||
|
fs::write(&bad, b"2").expect("write");
|
||||||
|
|
||||||
|
let s = t.storage();
|
||||||
|
let entries = s.list(&s.root_dir(ROOT).unwrap()).unwrap();
|
||||||
|
assert_eq!(entries.len(), 1);
|
||||||
|
assert_eq!(entries[0].meta.name, "good.CR3");
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn the_root_is_distinguishable_from_what_is_inside_it() {
|
||||||
|
// The walk keys the difference between "the library is unreachable" and
|
||||||
|
// "one folder failed" on this, and those have opposite consequences for
|
||||||
|
// the deletion sweep (FR-CAT-9).
|
||||||
|
assert!(DirRef::root(ROOT).is_root());
|
||||||
|
assert!(!DirRef::from_parts(ROOT, "2026").is_root());
|
||||||
|
assert_eq!(DirRef::from_parts(ROOT, "2026/08").name(), "08");
|
||||||
|
}
|
||||||
|
}
|
||||||
Reference in New Issue
Block a user