dtourolle/DarkRoom
1
0
Fork 0
Code Issues 39 Pull Requests Actions 8 Packages Projects Releases 20 Wiki Activity

Merge master: pluggable storage, and a name that anchors

Browse Source 
Conflicts were docs/traceability.md alone, and it is generated — so it
was regenerated rather than hand-merged. dr-face was untouched on the
other side; ui/dr-ui/src/faces.rs and identity_ui.rs auto-merged, the
first around recluster's anchoring and the second around load_faces.

Worth recording because the two branches met on the same problem from
different ends. Master's "Let a name hold a group together" is the fix
for the sixteen Catherines — fourteen of them empty — that this branch
found while measuring the library and reported without fixing.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
dtourolle
2026-08-29 12:30:01 +02:00
co-authored by Claude Opus 5
parent b2250cc460 21d599b420
commit da20d42d33
54 changed files with 6173 additions and 986 deletions
Download Patch File Download Diff File Expand all files Collapse all files
core/dr-catalog/src/cache.rs
+50
View File
@@ -222,6 +222,56 @@ impl Cache {
Ok(())
}
/// TRACES: FR-NC-6c | FR-NC-6a
/// Record an original this cache does **not** own the bytes of.
///
/// The virtual-filesystem case. On a library kept by a sync client the
/// original is materialised *in the library folder itself*, so copying it
/// under `originals/` would hold two copies of every pinned photograph —
/// and the copy would be the one the budget could evict while the real
/// disk cost stayed.
///
/// So the bytes are left where they are and only the bookkeeping is kept.
/// `path` is deliberately `NULL`, which is what makes this safe:
/// [`release`](Self::release) deletes the file a row names, and a row that
/// names none deletes nothing. **That matters more than it sounds.**
/// Deleting a materialised file inside a synced folder does not free a
/// cache — it deletes the photograph, and the client propagates that to
/// the server and to every other device. Handing the disk back is the
/// backend's job (`RemoteBackend::dematerialise`), not this one's.
///
/// `bytes` is what the original occupies where it lies, for the budget and
/// for reporting; pass 0 where it is not known.
pub fn record_in_place(
&self,
conn: &Connection,
image: ImageId,
bytes: u64,
pinned: bool,
now: i64,
) -> Result<(), CatalogError> {
conn.execute(
"INSERT INTO image_cache
(image_id, tier_actual, tier_desired, bytes, last_used, pinned, path)
VALUES (?1, ?2, ?2, ?3, ?4, ?5, NULL)
ON CONFLICT(image_id) DO UPDATE SET
tier_actual = ?2,
tier_desired = max(tier_desired, ?2),
bytes = ?3,
last_used = ?4,
pinned = max(pinned, ?5),
path = NULL",
rusqlite::params![
image.0 as i64,
Tier::Original.stored(),
bytes as i64,
now,
i64::from(pinned),
],
)?;
Ok(())
}
/// Read a cached original back, if it is here.
///
/// Touches `last_used`, which is what makes the eviction order reflect
core/dr-sync-folder/Cargo.toml
+19
View File
@@ -0,0 +1,19 @@
[package]
name = "dr-sync-folder"
version.workspace = true
edition.workspace = true
rust-version.workspace = true
license.workspace = true
[dependencies]
dr-types.workspace = true
dr-sync.workspace = true
async-trait.workspace = true
thiserror.workspace = true
log.workspace = true
# Filesystem work runs on the blocking pool rather than on the async worker
# that called it — see the module docs.
tokio = { workspace = true }
[dev-dependencies]
tokio = { workspace = true }
core/dr-sync-folder/examples/scan.rs
+71
View File
@@ -0,0 +1,71 @@
//! Scan a real folder through the engine, and read a preview out of it.
//!
//! ```text
//! cargo run -p dr-sync-folder --example scan -- /path/to/photos
//! ```
//!
//! Exercises the same code the application runs: `dr_sync::scan` driving the
//! folder connector, then a ranged `get` of the kind the thumbnail worker
//! makes. Reads only — it never writes into the folder it is pointed at.
use std::collections::HashMap;
use dr_sync::{RemoteBackend, RemoteId, RemotePath};
use dr_sync_folder::FolderBackend;
use dr_types::FormatFilter;
#[tokio::main(flavor = "current_thread")]
async fn main() {
let Some(root) = std::env::args().nth(1) else {
eprintln!("usage: scan <folder>");
std::process::exit(2);
};
let backend = match FolderBackend::new(&root) {
Ok(b) => b,
Err(e) => {
eprintln!("{e}");
std::process::exit(1);
}
};
let caps = backend.capabilities();
println!("{} at {root}", backend.name());
println!(
" strategy: {}",
dr_sync::SyncStrategy::for_capabilities(caps).describe()
);
let started = std::time::Instant::now();
let result = dr_sync::scan(
&backend,
&RemotePath::root(),
&FormatFilter::all(),
&HashMap::new(),
|_| {},
)
.await
.expect("scan");
println!(
" {} image(s) in {} director(ies), {:?}",
result.images.len(),
result.progress.directories_listed,
started.elapsed()
);
let Some(first) = result.images.first() else {
return;
};
println!(
" first: {} ({} bytes) id {:?}",
first.path, first.size, first.id
);
// The shape of request the thumbnail worker makes: a header window, not
// the whole file.
let head = backend
.get(&RemoteId::Path(first.path.clone()), Some(0..65536))
.await
.expect("ranged read");
println!(" read {} header bytes", head.len());
}
core/dr-sync-folder/examples/vfs_cycle.rs
+142
View File
@@ -0,0 +1,142 @@
//! A placeholder library, borrowed and given back.
//!
//! ```text
//! cargo run -p dr-sync-folder --example vfs_cycle
//! ```
//!
//! Builds a tree in the system temp directory shaped like a suffix-mode VFS
//! folder, runs the real engine over it, and reports what the borrow cost.
//! Touches nothing outside its own scratch directory.
use std::borrow::Cow;
use std::collections::HashMap;
use std::path::{Path, PathBuf};
use std::sync::Arc;
use dr_sync::{RemoteBackend, RemoteError, RemoteId, RemotePath};
use dr_sync_folder::{BorrowPool, FolderBackend, Vfs};
use dr_types::FormatFilter;
/// Stands in for the sync client, renaming exactly as suffix mode does.
struct Client;
impl Vfs for Client {
fn name(&self) -> &'static str {
"demo"
}
fn is_placeholder(&self, on_disk: &str) -> bool {
on_disk.ends_with(".stub")
}
fn real_name<'a>(&self, on_disk: &'a str) -> &'a str {
on_disk.strip_suffix(".stub").unwrap_or(on_disk)
}
fn placeholder_name(&self, name: &str) -> Cow<'_, str> {
Cow::Owned(format!("{name}.stub"))
}
fn can_materialise(&self) -> bool {
true
}
fn materialise(&self, local: &Path) -> Result<(), RemoteError> {
let real = PathBuf::from(local.to_string_lossy().strip_suffix(".stub").unwrap());
std::fs::write(&real, vec![7u8; 25 * 1024 * 1024]).unwrap();
std::fs::remove_file(local).unwrap();
Ok(())
}
fn dematerialise(&self, local: &Path) -> Result<(), RemoteError> {
std::fs::write(format!("{}.stub", local.display()), [0u8]).unwrap();
std::fs::remove_file(local).unwrap();
Ok(())
}
}
fn disk_used(root: &Path) -> u64 {
fn walk(p: &Path, total: &mut u64) {
if let Ok(entries) = std::fs::read_dir(p) {
for e in entries.flatten() {
let Ok(m) = e.metadata() else { continue };
if m.is_dir() {
walk(&e.path(), total);
} else {
*total += m.len();
}
}
}
}
let mut t = 0;
walk(root, &mut t);
t
}
#[tokio::main(flavor = "current_thread")]
async fn main() {
let root = std::env::temp_dir().join("dr-vfs-cycle");
let _ = std::fs::remove_dir_all(&root);
std::fs::create_dir_all(root.join("2026/03")).unwrap();
// Ninety dehydrated photographs, and ten the user already keeps.
for i in 0..90 {
std::fs::write(root.join(format!("2026/03/IMG_{i:04}.CR2.stub")), [0u8]).unwrap();
}
for i in 90..100 {
std::fs::write(
root.join(format!("2026/03/IMG_{i:04}.CR2")),
vec![1u8; 25 * 1024 * 1024],
)
.unwrap();
}
let b = FolderBackend::with_vfs(&root, Arc::new(Client)).unwrap();
println!("materialisation: {:?}", b.capabilities().materialisation);
println!("on disk at rest: {} MB", disk_used(&root) / 1_048_576);
let scan = dr_sync::scan(
&b,
&RemotePath::root(),
&FormatFilter::all(),
&HashMap::new(),
|_| {},
)
.await
.unwrap();
let absent = scan.images.iter().filter(|e| !e.materialised).count();
println!(
"scanned {} photograph(s), {absent} not downloaded",
scan.images.len()
);
// Names, not stubs — this is what the catalog records.
println!("first: {}", scan.images[0].path);
// A pass over the library, one photograph at a time.
let pool = BorrowPool::new();
let mut peak = 0u64;
let mut fetched = 0usize;
for entry in &scan.images {
let held = pool.borrow(&b, &entry.path).await.unwrap();
if held.hydrated() {
fetched += 1;
}
// Read it, as a thumbnail pass would.
let n = b
.get(&RemoteId::Path(entry.path.clone()), Some(0..65536))
.await
.unwrap()
.len();
assert_eq!(n, 65536);
peak = peak.max(disk_used(&root));
drop(held);
// Release as we go, which is what keeps the peak flat.
pool.release_all(&b).await;
}
println!("fetched {fetched} of {}", scan.images.len());
println!("peak on disk: {} MB", peak / 1_048_576);
println!("after the pass: {} MB", disk_used(&root) / 1_048_576);
println!(
"the ten the user already had: {} still here",
(90..100)
.filter(|i| root.join(format!("2026/03/IMG_{i:04}.CR2")).is_file())
.count()
);
let _ = std::fs::remove_dir_all(&root);
}
core/dr-sync-folder/src/borrow.rs
+207
View File
@@ -0,0 +1,207 @@
// TRACES: FR-NC-6c | FR-NC-6a
//! Hydrating a file for as long as it is needed, and no longer.
//!
//! A pass over a library — thumbnails, face indexing — needs each photograph's
//! bytes for a moment and never again. On a virtual-filesystem folder those
//! bytes may not be here, and fetching them is whole-file: hydrating a 17,000
//! image library to index it would land the entire library on a disk the user
//! deliberately keeps most of it off (ARCH §9.0).
//!
//! So hydration is a **borrow**. Ask for a file, use it, give it back. Peak
//! disk becomes the working set rather than the library, and the transfer is
//! paid once for a thumbnail that is then kept for ever — and pushed to the
//! server for other devices, which never pay it at all.
//!
//! # The rule that makes it safe
//!
//! **A file is returned to the state it was found in.** If it was already
//! downloaded — the user pinned it, opened it yesterday, or never uses VFS —
//! the borrow leaves it downloaded. Only what this pass hydrated is released.
//! Anything else silently undoes a choice the user made, and "my pinned trip
//! evaporated after an indexing run" is the kind of failure that makes people
//! stop trusting the feature.
//!
//! # Why it is reference counted
//!
//! Lanes run concurrently and two of them meet on the same file: the
//! thumbnail pass and the face pass want the same RAW. Without counting, the
//! first to finish dehydrates the file the second is reading. With it, the
//! transfer is paid once and the release happens when the last borrower is
//! done.
use std::collections::HashMap;
use std::sync::{Arc, Mutex};
use dr_sync::{RemoteBackend, RemoteError, RemoteId, RemotePath};
/// What a borrow is holding, per path.
#[derive(Debug, Default)]
struct Held {
/// How many borrowers are using it now.
borrowers: usize,
/// Whether *we* brought it here. False means it was already downloaded
/// and must be left that way.
ours: bool,
}
/// Tracks what has been hydrated and by whom.
///
/// Cheap to clone — every worker holds one and they share the same state.
#[derive(Clone, Default, Debug)]
pub struct BorrowPool {
held: Arc<Mutex<HashMap<RemotePath, Held>>>,
}
/// What a completed borrow did, for reporting a pass's real cost.
#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
pub struct BorrowStats {
/// Files that were already here. These cost nothing.
pub already_local: usize,
/// Files this pass downloaded.
pub hydrated: usize,
/// Files released again afterwards.
pub released: usize,
/// Files left downloaded because they were already so.
pub kept: usize,
}
impl BorrowPool {
pub fn new() -> Self {
Self::default()
}
/// Borrow a file's content for the life of the returned guard.
///
/// Downloads it if it is a placeholder; does nothing if it is already
/// here. The guard releases it on drop, but only if this pool hydrated it
/// and nothing else still holds it.
///
/// A backend without [`Materialisation::OnDemand`] short-circuits: the
/// borrow succeeds and does nothing, so a caller written for a VFS library
/// runs unchanged against a server or a plain folder.
///
/// [`Materialisation::OnDemand`]: dr_sync::Materialisation::OnDemand
pub async fn borrow<'p>(
&'p self,
backend: &dyn RemoteBackend,
path: &RemotePath,
) -> Result<Borrowed<'p>, RemoteError> {
if !backend.capabilities().materialisation.can_materialise() {
return Ok(Borrowed {
pool: None,
path: path.clone(),
hydrated: false,
});
}
// Another borrower already has it: join them rather than asking the
// client a second time.
{
let mut held = self.lock();
if let Some(entry) = held.get_mut(path) {
entry.borrowers += 1;
return Ok(Borrowed {
pool: Some(self),
path: path.clone(),
hydrated: false,
});
}
}
// The backend answers whether *it* fetched the content, because it had
// to look before deciding. Determining that here instead would cost a
// directory listing per file, and getting it wrong in the wrong
// direction releases a file the user pinned.
let ours = backend.materialise(&RemoteId::Path(path.clone())).await?;
self.lock()
.insert(path.clone(), Held { borrowers: 1, ours });
Ok(Borrowed {
pool: Some(self),
path: path.clone(),
hydrated: ours,
})
}
/// Release everything this pool still holds that it hydrated.
///
/// The end-of-pass sweep. A guard dropped on a panicking worker cannot run
/// its async release, so the pool is drained deliberately at the end
/// rather than trusted to unwind cleanly.
pub async fn release_all(&self, backend: &dyn RemoteBackend) -> BorrowStats {
let ours: Vec<RemotePath> = {
let held = self.lock();
held.iter()
.filter(|(_, h)| h.ours)
.map(|(p, _)| p.clone())
.collect()
};
let mut stats = BorrowStats::default();
for path in ours {
match backend.dematerialise(&RemoteId::Path(path.clone())).await {
Ok(()) => stats.released += 1,
// Not fatal, and not worth failing a completed pass over: the
// content stays, which costs disk and loses nothing.
Err(e) => log::debug!("releasing {path}: {e}"),
}
}
self.lock().clear();
stats
}
/// How many paths are currently held.
pub fn held(&self) -> usize {
self.lock().len()
}
fn lock(&self) -> std::sync::MutexGuard<'_, HashMap<RemotePath, Held>> {
// A poisoned lock means a worker panicked while holding it. The map is
// bookkeeping, not a resource — carrying on with it is better than
// taking the whole pass down.
self.held.lock().unwrap_or_else(|e| e.into_inner())
}
}
/// A file held local for as long as this lives.
///
/// Dropping it marks the borrow finished. The actual release happens in
/// [`BorrowPool::release_all`], because dropping cannot await.
#[derive(Debug)]
pub struct Borrowed<'p> {
pool: Option<&'p BorrowPool>,
path: RemotePath,
/// Whether this borrow was the one that downloaded it.
hydrated: bool,
}
impl Borrowed<'_> {
/// Whether this borrow paid for a download.
pub fn hydrated(&self) -> bool {
self.hydrated
}
pub fn path(&self) -> &RemotePath {
&self.path
}
}
impl Drop for Borrowed<'_> {
fn drop(&mut self) {
let Some(pool) = self.pool else { return };
let mut held = pool.lock();
if let Some(entry) = held.get_mut(&self.path) {
entry.borrowers = entry.borrowers.saturating_sub(1);
// Left in the map even at zero borrowers: `release_all` needs to
// know it was ours, and a file wanted again a moment later should
// not be downloaded twice.
if entry.borrowers == 0 && !entry.ours {
held.remove(&self.path);
}
}
}
}
#[cfg(test)]
mod tests;
core/dr-sync-folder/src/borrow/tests.rs
+266
View File
@@ -0,0 +1,266 @@
//! The borrow contract, against a filesystem and a fake client.
//!
//! The fake stands in for the sync client's socket, not for the filesystem:
//! it renames stubs exactly as suffix-mode VFS does, so everything under test
//! is the real path resolution and the real state tracking.
use super::*;
use crate::{FolderBackend, Vfs};
use std::path::{Path, PathBuf};
use std::sync::atomic::{AtomicUsize, Ordering};
/// A stand-in for a sync client, counting what it was asked to do.
struct FakeClient {
suffix: &'static str,
hydrations: AtomicUsize,
dehydrations: AtomicUsize,
/// When true, refuse to hydrate — the client is running but the server is
/// not reachable.
broken: bool,
}
impl FakeClient {
fn new() -> Arc<Self> {
Arc::new(Self {
suffix: ".nextcloud",
hydrations: AtomicUsize::new(0),
dehydrations: AtomicUsize::new(0),
broken: false,
})
}
fn broken() -> Arc<Self> {
Arc::new(Self {
suffix: ".nextcloud",
hydrations: AtomicUsize::new(0),
dehydrations: AtomicUsize::new(0),
broken: true,
})
}
}
impl Vfs for FakeClient {
fn name(&self) -> &'static str {
"fake"
}
fn is_placeholder(&self, on_disk: &str) -> bool {
on_disk.ends_with(self.suffix)
}
fn real_name<'a>(&self, on_disk: &'a str) -> &'a str {
on_disk.strip_suffix(self.suffix).unwrap_or(on_disk)
}
fn placeholder_name(&self, name: &str) -> std::borrow::Cow<'_, str> {
std::borrow::Cow::Owned(format!("{name}{}", self.suffix))
}
fn can_materialise(&self) -> bool {
true
}
fn materialise(&self, local: &Path) -> Result<(), RemoteError> {
self.hydrations.fetch_add(1, Ordering::SeqCst);
if self.broken {
return Err(RemoteError::Network("no server".into()));
}
// Suffix mode renames rather than filling in place, and writes the
// real content.
let real = PathBuf::from(local.to_string_lossy().strip_suffix(self.suffix).unwrap());
std::fs::write(&real, vec![9u8; 4096]).unwrap();
std::fs::remove_file(local).unwrap();
Ok(())
}
fn dematerialise(&self, local: &Path) -> Result<(), RemoteError> {
self.dehydrations.fetch_add(1, Ordering::SeqCst);
let stub = format!("{}{}", local.display(), self.suffix);
std::fs::write(&stub, [0u8]).unwrap();
std::fs::remove_file(local).unwrap();
Ok(())
}
}
struct Tmp(PathBuf);
impl Tmp {
fn new(name: &str) -> Self {
let d = std::env::temp_dir().join(format!("dr-borrow-{name}"));
let _ = std::fs::remove_dir_all(&d);
std::fs::create_dir_all(&d).unwrap();
Tmp(d)
}
/// A dehydrated photograph.
fn stub(&self, rel: &str) -> &Self {
std::fs::write(self.0.join(format!("{rel}.nextcloud")), [0u8]).unwrap();
self
}
/// One the user already has.
fn real(&self, rel: &str) -> &Self {
std::fs::write(self.0.join(rel), vec![1u8; 2048]).unwrap();
self
}
fn has(&self, rel: &str) -> bool {
self.0.join(rel).is_file()
}
fn backend(&self, vfs: Arc<dyn Vfs>) -> FolderBackend {
FolderBackend::with_vfs(&self.0, vfs).unwrap()
}
}
impl Drop for Tmp {
fn drop(&mut self) {
let _ = std::fs::remove_dir_all(&self.0);
}
}
#[tokio::test]
async fn a_borrowed_placeholder_is_downloaded_and_given_back() {
let t = Tmp::new("cycle");
t.stub("a.CR2");
let client = FakeClient::new();
let b = t.backend(client.clone());
let pool = BorrowPool::new();
let path = RemotePath::new("a.CR2");
{
let held = pool.borrow(&b, &path).await.unwrap();
assert!(held.hydrated(), "this borrow paid for it");
assert!(t.has("a.CR2"), "content is here while borrowed");
assert_eq!(
b.get(&RemoteId::Path(path.clone()), None)
.await
.unwrap()
.len(),
4096
);
}
let stats = pool.release_all(&b).await;
assert_eq!(stats.released, 1);
assert!(!t.has("a.CR2"), "given back");
assert!(t.has("a.CR2.nextcloud"), "a placeholder is left behind");
assert_eq!(client.dehydrations.load(Ordering::SeqCst), 1);
}
#[tokio::test]
async fn a_file_the_user_already_had_is_never_taken_away() {
// The rule the whole design rests on. Silently undoing a pin — or just a
// file someone opened yesterday — after an indexing run is the failure
// that would make people stop trusting this.
let t = Tmp::new("keep");
t.real("pinned.CR2");
let client = FakeClient::new();
let b = t.backend(client.clone());
let pool = BorrowPool::new();
{
let held = pool
.borrow(&b, &RemotePath::new("pinned.CR2"))
.await
.unwrap();
assert!(!held.hydrated(), "nothing was downloaded");
}
let stats = pool.release_all(&b).await;
assert_eq!(stats.released, 0);
assert!(t.has("pinned.CR2"), "still here");
assert_eq!(client.hydrations.load(Ordering::SeqCst), 0);
assert_eq!(client.dehydrations.load(Ordering::SeqCst), 0);
}
#[tokio::test]
async fn two_lanes_wanting_one_file_download_it_once() {
// The thumbnail pass and the face pass meet on the same RAW. Without
// counting, the first to finish dehydrates the file the second is reading.
let t = Tmp::new("shared");
t.stub("a.CR2");
let client = FakeClient::new();
let b = t.backend(client.clone());
let pool = BorrowPool::new();
let path = RemotePath::new("a.CR2");
let first = pool.borrow(&b, &path).await.unwrap();
let second = pool.borrow(&b, &path).await.unwrap();
assert_eq!(client.hydrations.load(Ordering::SeqCst), 1, "paid once");
drop(first);
assert!(t.has("a.CR2"), "still held by the second borrower");
drop(second);
pool.release_all(&b).await;
assert!(!t.has("a.CR2"));
}
#[tokio::test]
async fn a_failed_download_does_not_leave_a_phantom_borrow() {
// The client is up but the server is not. The pass must see the failure
// and the pool must not believe it holds anything.
let t = Tmp::new("failed");
t.stub("a.CR2");
let b = t.backend(FakeClient::broken());
let pool = BorrowPool::new();
let e = pool
.borrow(&b, &RemotePath::new("a.CR2"))
.await
.unwrap_err();
assert!(matches!(e, RemoteError::Network(_)), "{e:?}");
assert_eq!(pool.held(), 0);
assert!(t.has("a.CR2.nextcloud"), "left as it was found");
}
#[tokio::test]
async fn borrowing_against_a_plain_folder_does_nothing_at_all() {
// A caller written for a VFS library must run unchanged elsewhere, or
// every sweep grows two code paths.
let t = Tmp::new("plain");
t.real("a.CR2");
let b = FolderBackend::new(&t.0).unwrap();
let pool = BorrowPool::new();
let held = pool.borrow(&b, &RemotePath::new("a.CR2")).await.unwrap();
assert!(!held.hydrated());
drop(held);
assert_eq!(pool.release_all(&b).await.released, 0);
assert!(t.has("a.CR2"));
}
#[tokio::test]
async fn the_backend_is_what_decides_whether_a_file_was_ours() {
// Not the pool, and not the caller. The backend had to look before
// deciding whether to ask, so it can answer for the cost of that same
// `stat`; a borrower working it out separately would pay a directory
// listing per file and could get it wrong in the direction that releases
// a file the user pinned.
let t = Tmp::new("who-decides");
t.real("had.CR2").stub("wanted.CR2");
let b = t.backend(FakeClient::new());
assert!(
!b.materialise(&RemoteId::Path(RemotePath::new("had.CR2")))
.await
.unwrap(),
"already here, so not ours to release"
);
assert!(
b.materialise(&RemoteId::Path(RemotePath::new("wanted.CR2")))
.await
.unwrap(),
"this call fetched it"
);
}
#[tokio::test]
async fn a_file_borrowed_twice_in_one_pass_is_fetched_once_and_released_once() {
// Thumbnailing and face indexing visit the same photograph. Fetching it
// per stage doubles the transfer over the whole library.
let t = Tmp::new("sequential");
t.stub("a.CR2");
let client = FakeClient::new();
let b = t.backend(client.clone());
let pool = BorrowPool::new();
let path = RemotePath::new("a.CR2");
// Sequential borrows, as two passes over one work list would make.
drop(pool.borrow(&b, &path).await.unwrap());
drop(pool.borrow(&b, &path).await.unwrap());
assert_eq!(client.hydrations.load(Ordering::SeqCst), 1, "paid once");
let stats = pool.release_all(&b).await;
assert_eq!(stats.released, 1, "given back once");
}
core/dr-sync-folder/src/lib.rs
+869
View File
@@ -0,0 +1,869 @@
// TRACES: FR-NC-13 | FR-NC-12
//! A library that is just a directory.
//!
//! The second [`RemoteBackend`], and the one that exists to prove the first
//! was an abstraction rather than a description. It serves a plain folder: a
//! local disk, an NFS or SMB mount, a Nextcloud desktop client's synced copy,
//! an external drive. No server, no account, no credential.
//!
//! # What it is honestly worse at, and why that is fine
//!
//! Nextcloud's fast path rests on directory ETags propagating up the tree, so
//! one request against the root proves a 50k-image library unchanged. A POSIX
//! directory's mtime says only that its own entry list changed — not that a
//! grandchild's *contents* did — so there is nothing here to propagate and
//! [`ChangeDetection::LocalEtags`] is the truthful answer. The engine reads
//! that and walks the tree every scan instead of pruning it.
//!
//! Which costs almost nothing, because the walk that was expensive was
//! expensive for a reason this backend does not have. Fifty thousand
//! `stat` calls against a local filesystem take well under a second; fifty
//! thousand `PROPFIND`s do not. The capability model is what lets both be
//! driven by the same engine at the speed each one actually runs at.
//!
//! # Identity
//!
//! [`RemoteId::Stable`] here is a hash of the path relative to the library
//! root. That gives the catalog what it needs — a `u64` that names a
//! photograph, is the same on every device looking at the same folder, and
//! does not change when the file is edited — which is what keys the thumbnail
//! shards and the face index (`catalog.md` §10.1).
//!
//! It does **not** survive a rename, and [`Capabilities::stable_ids`] says so.
//! A moved photograph is seen as a delete and an add, and its thumbnail is
//! derived again. That is the documented degradation for a backend without
//! server-assigned ids, and it is the right trade here: the alternative,
//! keying on the inode, is stable across a rename but *differs between
//! devices* and is reused by the filesystem after a delete — so two machines
//! would disagree about which photograph a thumbnail belonged to, and a
//! recycled inode would silently attach an old thumbnail to a new image.
//! Re-deriving a thumbnail is a cost; showing the wrong one is a bug.
//!
//! # Blocking
//!
//! Every filesystem call goes through the blocking pool. On a local disk that
//! is overkill; on the NFS mount this backend is most useful over, a stalled
//! server would otherwise wedge the async worker that made the call and every
//! other request sharing it.
use std::io::{Read, Seek, SeekFrom, Write};
use std::ops::Range;
use std::path::{Component, Path, PathBuf};
use std::sync::Arc;
use async_trait::async_trait;
use dr_sync::{
Account, BackendProvider, Capabilities, ChangeDetection, Connection, Cursor, EntryKind,
Materialisation, Precondition, RemoteBackend, RemoteChange, RemoteEntry, RemoteError, RemoteId,
RemotePath, ServerPreviews, SignIn, Validator,
};
pub mod borrow;
pub mod vfs;
pub use borrow::{BorrowPool, BorrowStats, Borrowed};
pub use vfs::{NoVfs, Vfs};
/// The id written to [`Account::backend`] for a folder library.
///
/// On-disk configuration: changing it orphans every folder account.
pub const BACKEND_ID: &str = "folder";
/// TRACES: FR-NC-13 | FR-NC-6c
/// Registers the folder connector.
///
/// See [`dr_sync::provider`] for what each method is for.
///
/// # The detector
///
/// This crate knows how to read a directory and nothing about sync clients,
/// so the placeholder convention arrives from outside: whoever registers the
/// provider supplies a function that recognises a synced folder and returns
/// the [`Vfs`] for it. That keeps `dr-sync-folder` free of any client's
/// protocol, and it is what lets one connector serve a plain disk, a Nextcloud
/// tree, and whatever comes next.
///
/// Detection runs per connection because the answer changes: the same
/// directory offers hydration while the client is up and not while it is down.
/// Recognises a placeholder convention in a directory, if any applies.
///
/// Runs per connection rather than once, because the answer changes: the same
/// folder offers hydration while the sync client is up and not while it is
/// down.
pub type VfsDetector = dyn Fn(&Path) -> Option<Arc<dyn Vfs>> + Send + Sync;
#[derive(Default)]
pub struct FolderProvider {
detect_vfs: Option<Box<VfsDetector>>,
}
impl FolderProvider {
/// A folder connector that treats every directory as ordinary.
pub fn new() -> Self {
Self::default()
}
/// A folder connector that recognises placeholder conventions.
pub fn with_vfs_detector(
detect: impl Fn(&Path) -> Option<Arc<dyn Vfs>> + Send + Sync + 'static,
) -> Self {
Self {
detect_vfs: Some(Box::new(detect)),
}
}
fn vfs_for(&self, root: &Path) -> Arc<dyn Vfs> {
self.detect_vfs
.as_ref()
.and_then(|d| d(root))
.unwrap_or_else(|| Arc::new(NoVfs))
}
}
impl BackendProvider for FolderProvider {
fn id(&self) -> &'static str {
BACKEND_ID
}
fn display_name(&self) -> &'static str {
"Folder"
}
fn endpoint_label(&self) -> &'static str {
"Folder"
}
fn endpoint_placeholder(&self) -> &'static str {
"/home/you/Pictures"
}
fn sign_in(&self) -> SignIn {
SignIn::EndpointOnly
}
/// Check the directory before an account is written for it.
///
/// A typo here would otherwise be stored, skip the launch screen on the
/// next start, and surface as a scan that finds nothing — which reads as
/// a broken library rather than a wrong path. The messages say what to fix.
fn normalise_endpoint(&self, input: &str) -> Result<String, String> {
let trimmed = input.trim();
if trimmed.is_empty() {
return Err("Choose the folder your photographs are in.".into());
}
// `~` is what a person types and what a shell would have expanded;
// nothing expands it here, so a stored `~/Pictures` becomes a
// directory literally named `~`.
let expanded = match trimmed.strip_prefix("~/") {
Some(rest) => match std::env::var_os("HOME") {
Some(home) => PathBuf::from(home).join(rest),
None => return Err("No home directory to expand ~ against.".into()),
},
None => PathBuf::from(trimmed),
};
if !expanded.is_absolute() {
return Err("Give the full path to the folder, starting at /.".into());
}
if !expanded.exists() {
return Err(format!("No folder at {}.", expanded.display()));
}
if !expanded.is_dir() {
return Err(format!("{} is a file, not a folder.", expanded.display()));
}
// Resolved so a library reached through a symlink or a `..` is stored
// under one name. Two spellings of one folder would otherwise be two
// accounts with two catalogs indexing the same photographs.
let canonical = expanded
.canonicalize()
.map_err(|e| format!("Cannot read {}: {e}", expanded.display()))?;
Ok(canonical.to_string_lossy().into_owned())
}
fn account_for(&self, endpoint: &str) -> Result<Account, RemoteError> {
Ok(Account::new(BACKEND_ID, endpoint))
}
fn connect(&self, conn: &Connection) -> Result<Box<dyn RemoteBackend>, RemoteError> {
let root = Path::new(&conn.account.endpoint);
Ok(Box::new(FolderBackend::with_vfs(root, self.vfs_for(root))?))
}
}
/// TRACES: FR-NC-13 | FR-NC-4 | FR-NC-6c
/// A library rooted at a directory.
#[derive(Clone)]
pub struct FolderBackend {
root: PathBuf,
/// The placeholder convention in force, [`NoVfs`] for an ordinary folder.
vfs: Arc<dyn Vfs>,
caps: Capabilities,
}
impl std::fmt::Debug for FolderBackend {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.debug_struct("FolderBackend")
.field("root", &self.root)
.field("vfs", &self.vfs.name())
.finish_non_exhaustive()
}
}
impl FolderBackend {
/// Open the folder at `root`.
///
/// The directory must exist now. It may stop existing later — a drive
/// unplugged, a mount dropped — and that surfaces per-operation as
/// [`RemoteError::Network`], which is what puts the app into offline mode
/// and leaves the catalog readable, exactly as a dead server does.
pub fn new(root: impl Into<PathBuf>) -> Result<Self, RemoteError> {
Self::with_vfs(root, Arc::new(NoVfs))
}
/// Open the folder at `root` under a placeholder convention.
///
/// The convention is chosen by the caller rather than sniffed here: the
/// connector that knows how to talk to a given sync client is the one that
/// knows whether it is running (see `dr_sync_nextcloud`).
pub fn with_vfs(root: impl Into<PathBuf>, vfs: Arc<dyn Vfs>) -> Result<Self, RemoteError> {
let root = root.into();
if !root.is_dir() {
return Err(RemoteError::Configuration(format!(
"{} is not a folder",
root.display()
)));
}
// Reported per connection, not per backend: the same folder offers
// hydration while the client is up and not while it is down, so this
// cannot be a constant of the type (see `vfs`).
let materialisation = if vfs.can_materialise() {
Materialisation::OnDemand
} else if vfs.name() == NoVfs.name() {
Materialisation::Always
} else {
Materialisation::Placeholders
};
Ok(Self {
root,
vfs,
caps: Capabilities {
// A directory's mtime describes its own entry list and nothing
// below it, so there is no propagation to exploit; the engine
// walks and compares per entry.
change_detection: ChangeDetection::LocalEtags,
// A path hash does not survive a rename. See the module docs
// for why the inode is not used instead.
stable_ids: false,
range_reads: true,
// Not a protocol with a message size limit; a write is a write.
chunked_upload: None,
bulk_upload: false,
conditional_write: true,
server_previews: ServerPreviews::None,
materialisation,
},
})
}
pub fn root(&self) -> &Path {
&self.root
}
/// The local path for a remote path, refusing anything that escapes.
///
/// The guard is not theoretical. A `RemotePath` is built from strings that
/// reach us from a catalog written by another device and from filenames on
/// the remote itself, and this backend resolves them against a real
/// filesystem with the user's own permissions. `../../.ssh/id_ed25519` is
/// a legal path segment; without this it would be a legal *read*.
fn resolve(&self, path: &RemotePath) -> Result<PathBuf, RemoteError> {
let rel = Path::new(path.as_str());
for component in rel.components() {
match component {
Component::Normal(_) => {}
Component::CurDir => {}
Component::ParentDir | Component::RootDir | Component::Prefix(_) => {
return Err(RemoteError::Configuration(format!(
"{path} leaves the library folder"
)));
}
}
}
Ok(self.root.join(rel))
}
/// Where a photograph's bytes are on disk, and whether they are really
/// there.
///
/// A placeholder lives under a *different* name — suffix-mode VFS renames
/// on hydration rather than filling in place — so every read and write has
/// to look for both. The materialised name is tried first: it is the
/// common case, and the second `stat` is paid only when it misses.
///
/// Returns the path to use and whether it holds real content.
fn locate(&self, path: &RemotePath) -> Result<(PathBuf, bool), RemoteError> {
let direct = self.resolve(path)?;
if self.vfs.name() == NoVfs.name() || direct.exists() {
return Ok((direct, true));
}
let stub = self.resolve(&RemotePath::new(
self.vfs.placeholder_name(path.as_str()).into_owned(),
))?;
if stub.exists() {
return Ok((stub, false));
}
// Neither: genuinely missing. Report the name the caller asked for.
Ok((direct, true))
}
/// The local path a [`RemoteId`] names.
///
/// A stable id here is a hash and nothing can be resolved from it, exactly
/// as a Nextcloud `oc:fileid` names no WebDAV endpoint. Callers hold the
/// path alongside it in the catalog and pass that.
fn resolve_id(&self, id: &RemoteId) -> Result<PathBuf, RemoteError> {
match id {
RemoteId::Path(p) => self.resolve(p),
RemoteId::Stable(_) => Err(RemoteError::Unsupported(
"a folder cannot be addressed by id; use RemoteId::Path",
)),
}
}
/// [`locate`](Self::locate) for an id.
fn locate_id(&self, id: &RemoteId) -> Result<(PathBuf, bool), RemoteError> {
match id {
RemoteId::Path(p) => self.locate(p),
RemoteId::Stable(_) => Err(RemoteError::Unsupported(
"a folder cannot be addressed by id; use RemoteId::Path",
)),
}
}
}
/// Run a filesystem operation off the async worker that asked for it.
///
/// See the module docs: a stalled network mount must not take the caller's
/// runtime with it.
async fn blocking<T, F>(f: F) -> Result<T, RemoteError>
where
F: FnOnce() -> Result<T, RemoteError> + Send + 'static,
T: Send + 'static,
{
match tokio::task::spawn_blocking(f).await {
Ok(r) => r,
// The only way a blocking task fails to produce a result is a panic
// inside it, which is a bug here rather than a condition the caller
// can act on — but crashing the worker over it would lose a whole
// scan, so it is reported like any other failure.
Err(e) => Err(RemoteError::Protocol(format!("folder task failed: {e}"))),
}
}
/// Map an IO failure to the error the engine already knows how to handle.
///
/// The classification is the point. [`RemoteError::indicates_offline`] drives
/// offline mode, so a vanished mount must reach it as `Network` — that is
/// precisely the "the library is unreachable, keep working from the catalog"
/// case — while a permissions problem must not, because going offline over one
/// forbidden file would hide a fixable problem behind a network banner.
fn map_io(e: std::io::Error, what: &str) -> RemoteError {
use std::io::ErrorKind as K;
match e.kind() {
K::NotFound => RemoteError::NotFound(what.to_string()),
K::PermissionDenied => RemoteError::PermissionDenied,
K::AlreadyExists => RemoteError::PreconditionFailed,
// ENOSPC and friends. Quota is what the engine calls "no room".
K::StorageFull | K::QuotaExceeded | K::FileTooLarge => RemoteError::QuotaExceeded,
// A dropped mount answers ESTALE/EIO/ENOTCONN, and the honest reading
// is the same as a dead server: the library cannot be reached now, and
// may be again shortly.
K::HostUnreachable
| K::NetworkUnreachable
| K::NetworkDown
| K::ConnectionAborted
| K::ConnectionReset
| K::NotConnected
| K::BrokenPipe
| K::TimedOut => RemoteError::Network(format!("{what}: {e}")),
_ => RemoteError::Protocol(format!("{what}: {e}")),
}
}
/// How long to wait for a requested download to land.
///
/// Generous, because the file may be tens of megabytes over a domestic
/// connection, and bounded, because a client that has stopped transferring
/// must not wedge a whole pass.
const MATERIALISE_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(300);
/// How often to look for the materialised file while waiting.
const POLL: std::time::Duration = std::time::Duration::from_millis(200);
/// The identity of a file, from its path relative to the library root.
///
/// FNV-1a rather than `DefaultHasher`, whose output is explicitly unstable
/// between Rust releases: this value is written into the catalog and into the
/// thumbnail index, and must mean the same thing after a toolchain upgrade as
/// it did before one.
fn identity(path: &RemotePath) -> u64 {
let mut h: u64 = 0xcbf2_9ce4_8422_2325;
for b in path.as_str().as_bytes() {
h ^= *b as u64;
h = h.wrapping_mul(0x0000_0100_0000_01b3);
}
h
}
/// A file's validator: its size and modification time.
///
/// The pair, not either alone. An mtime with one-second granularity — which is
/// what some filesystems and most network mounts report — cannot distinguish
/// two writes in the same second, and a size alone cannot see an edit that
/// preserved it. Together they miss only a same-second write of identical
/// length, which for a photograph is a rewrite of the same frame.
fn validator_of(meta: &std::fs::Metadata) -> Validator {
let (secs, nanos) = meta
.modified()
.ok()
.and_then(|t| t.duration_since(std::time::UNIX_EPOCH).ok())
.map(|d| (d.as_secs(), d.subsec_nanos()))
.unwrap_or((0, 0));
Validator::new(format!("{:x}-{:x}.{:x}", meta.len(), secs, nanos))
}
fn modified_secs(meta: &std::fs::Metadata) -> Option<i64> {
meta.modified()
.ok()
.and_then(|t| t.duration_since(std::time::UNIX_EPOCH).ok())
.map(|d| d.as_secs() as i64)
}
#[async_trait]
impl RemoteBackend for FolderBackend {
fn capabilities(&self) -> &Capabilities {
&self.caps
}
fn name(&self) -> &str {
"Folder"
}
async fn list(
&self,
dir: &RemotePath,
_since: Option<&Validator>,
) -> Result<Vec<RemoteEntry>, RemoteError> {
let local = self.resolve(dir)?;
let dir = dir.clone();
let vfs = self.vfs.clone();
blocking(move || {
let read =
std::fs::read_dir(&local).map_err(|e| map_io(e, &local.display().to_string()))?;
let mut out = Vec::new();
for entry in read {
let entry = match entry {
Ok(e) => e,
// One unreadable entry must not fail the listing: a
// scan of a real library meets a broken symlink or a
// file being written, and abandoning the whole
// directory over it loses every photograph beside it.
Err(e) => {
log::debug!("skipping an entry in {}: {e}", local.display());
continue;
}
};
let name = entry.file_name();
let Some(name) = name.to_str() else {
// A name that is not UTF-8 cannot round-trip through a
// `RemotePath`, and quietly mangling it would produce a
// path that addresses a different file — or none.
log::warn!("skipping a non-UTF-8 name in {}", local.display());
continue;
};
// `metadata`, not `symlink_metadata`: a symlinked shoot
// folder is a normal way to assemble a library, and the
// scan's depth limit is what stops a loop.
let meta = match entry.metadata() {
Ok(m) => m,
Err(e) => {
log::debug!("skipping {name}: {e}");
continue;
}
};
// The photograph's own name, never the stub's. Identity is
// derived from it, so downloading a file must not look like a
// delete and an add — and `source_ref` must match what every
// other device calls the same photograph.
let stub = vfs.is_placeholder(name);
let path = dir.join(vfs.real_name(name));
out.push(RemoteEntry {
id: RemoteId::Stable(identity(&path)),
kind: if meta.is_dir() {
EntryKind::Directory
} else {
EntryKind::File
},
validator: validator_of(&meta),
// A stub is one byte and says nothing about what it stands
// for. Reporting that byte count would put a 1-byte
// `file_size` in the catalog for most of the library.
size: if stub { 0 } else { meta.len() },
modified: modified_secs(&meta),
// No renderer behind a folder; previews are extracted
// locally from the file itself.
has_preview: false,
materialised: !stub,
path,
});
}
Ok(out)
})
.await
}
/// Not offered.
///
/// A directory's mtime changes when its own entries are added or removed
/// and at no other time, so it cannot answer the question this method
/// exists for — "did anything below here change?". Returning it anyway
/// would let a future caller prune a subtree whose contents had been
/// edited, and hide those edits for as long as the folder list held still.
async fn dir_validator(&self, _dir: &RemotePath) -> Result<Validator, RemoteError> {
Err(RemoteError::Unsupported(
"a folder's mtime does not propagate; use per-entry validators",
))
}
async fn delta(&self, _cursor: &Cursor) -> Result<(Vec<RemoteChange>, Cursor), RemoteError> {
Err(RemoteError::Unsupported("a folder keeps no change feed"))
}
async fn get(&self, id: &RemoteId, range: Option<Range<u64>>) -> Result<Vec<u8>, RemoteError> {
let (local, materialised) = self.locate_id(id)?;
if !materialised {
// The one byte in the stub is not the file. Returning it produced
// a sidecar that parsed as empty and a thumbnail that never
// decoded; reporting `NotFound` made the sidecar writer treat an
// existing document as absent and overwrite it.
return Err(RemoteError::NotMaterialised(local.display().to_string()));
}
blocking(move || {
let what = local.display().to_string();
let mut file = std::fs::File::open(&local).map_err(|e| map_io(e, &what))?;
let Some(r) = range else {
let mut buf = Vec::new();
file.read_to_end(&mut buf).map_err(|e| map_io(e, &what))?;
return Ok(buf);
};
// A short read at the end of the file is not an error: the header
// extractor asks for a fixed window and the file may be smaller
// than it, which is the ordinary case for a small JPEG.
file.seek(SeekFrom::Start(r.start))
.map_err(|e| map_io(e, &what))?;
let want = r.end.saturating_sub(r.start);
let mut buf = Vec::new();
file.take(want)
.read_to_end(&mut buf)
.map_err(|e| map_io(e, &what))?;
Ok(buf)
})
.await
}
async fn put(
&self,
path: &RemotePath,
body: Vec<u8>,
precond: Option<Precondition>,
) -> Result<Validator, RemoteError> {
let (found, materialised) = self.locate(path)?;
// Where the content belongs, which is not where a placeholder for it
// sits — suffix-mode VFS gives the two different names.
let local = self.resolve(path)?;
// A stub is still this file, so what to do about it depends entirely
// on what the caller is promising.
let replaces = if materialised {
None
} else {
match &precond {
// Nothing here can satisfy it: the validator on a placeholder
// describes the placeholder. The caller fetches the content
// and tries again, which is what the typed error asks for.
Some(Precondition::IfMatch(_)) => {
return Err(RemoteError::NotMaterialised(found.display().to_string()))
}
// Something *is* there — the file exists, only its content is
// elsewhere — so a create-if-absent must fail.
Some(Precondition::IfAbsent) => return Err(RemoteError::PreconditionFailed),
// An unconditional write replaces the whole file, so there is
// nothing in the stub worth reading and no reason to download
// it first. Refusing here instead was a mistake: derived state
// lives in the library folder and the client dehydrates it
// like anything else, so a refusal meant sync could never
// write to a folder it had been away from.
None => Some(found),
}
};
blocking(move || {
let what = local.display().to_string();
if let Some(parent) = local.parent() {
std::fs::create_dir_all(parent)
.map_err(|e| map_io(e, &parent.display().to_string()))?;
}
match &precond {
// Genuinely atomic: `O_CREAT | O_EXCL` is one syscall, so two
// devices racing to create a sidecar cannot both win.
Some(Precondition::IfAbsent) => {
let mut f = std::fs::OpenOptions::new()
.write(true)
.create_new(true)
.open(&local)
.map_err(|e| map_io(e, &what))?;
f.write_all(&body).map_err(|e| map_io(e, &what))?;
f.sync_all().map_err(|e| map_io(e, &what))?;
let meta = f.metadata().map_err(|e| map_io(e, &what))?;
return Ok(validator_of(&meta));
}
// Compare, then swap. A POSIX filesystem has no compare-and-
// swap, so this narrows the window to the microseconds between
// the `stat` and the `rename` rather than closing it. That is
// still far tighter than the fallback the engine uses when a
// backend declares no conditional write at all — comparing
// revision counters *inside* the sidecar, which spans a whole
// read-modify-write — which is why the capability is declared
// rather than refused.
Some(Precondition::IfMatch(expected)) => {
let meta = std::fs::metadata(&local).map_err(|e| map_io(e, &what))?;
if &validator_of(&meta) != expected {
return Err(RemoteError::PreconditionFailed);
}
}
None => {}
}
// Write beside the destination and rename over it, so a reader
// never sees a half-written sidecar and an interrupted write
// cannot destroy the file it was replacing. Beside, not in
// `/tmp`: a rename across filesystems is not atomic, and on
// Android `/tmp` is a different one.
let tmp = local.with_extension(format!(
"{}.darkroom-tmp",
local.extension().and_then(|e| e.to_str()).unwrap_or("")
));
let write = (|| -> Result<(), RemoteError> {
let mut f = std::fs::File::create(&tmp).map_err(|e| map_io(e, &what))?;
f.write_all(&body).map_err(|e| map_io(e, &what))?;
f.sync_all().map_err(|e| map_io(e, &what))
})();
if let Err(e) = write {
let _ = std::fs::remove_file(&tmp);
return Err(e);
}
if let Err(e) = std::fs::rename(&tmp, &local) {
let _ = std::fs::remove_file(&tmp);
return Err(map_io(e, &what));
}
// The stub goes only once the content is safely in place. The
// other order risks leaving neither, and in a synced tree an
// absence is a deletion the client would propagate.
if let Some(stub) = replaces {
if let Err(e) = std::fs::remove_file(&stub) {
// The content landed, so the write succeeded; a leftover
// placeholder beside it is untidy rather than harmful, and
// the client reconciles the pair on its next pass.
log::warn!("removing placeholder {}: {e}", stub.display());
}
}
let meta = std::fs::metadata(&local).map_err(|e| map_io(e, &what))?;
Ok(validator_of(&meta))
})
.await
}
/// Delete a file, or an empty directory.
///
/// **Not recursive, unlike WebDAV's `DELETE` on a collection.** The
/// divergence is deliberate: a folder library is the user's own
/// photographs on their own disk, with no server-side trash behind it, so
/// a caller that passed the wrong path would have no way back. Nothing in
/// the engine deletes a directory — the soft delete is a
/// [`move_to`](RemoteBackend::move_to) into the trash folder — so refusing
/// costs nothing and the guard is free.
async fn delete(
&self,
id: &RemoteId,
precond: Option<Precondition>,
) -> Result<(), RemoteError> {
// Deliberately by whichever name is on disk: deleting a photograph
// means deleting it whether or not its content happens to be here, and
// a stub left behind would be re-listed by the next scan.
let (local, _) = self.locate_id(id)?;
blocking(move || {
let what = local.display().to_string();
let meta = std::fs::symlink_metadata(&local).map_err(|e| map_io(e, &what))?;
match &precond {
Some(Precondition::IfMatch(expected)) => {
if &validator_of(&meta) != expected {
return Err(RemoteError::PreconditionFailed);
}
}
// "Delete only if nothing is there" is not a thing to ask of a
// delete; something is there or the `stat` above already
// failed.
Some(Precondition::IfAbsent) => {
return Err(RemoteError::Unsupported(
"IfAbsent is not meaningful on a delete",
))
}
None => {}
}
if meta.is_dir() {
std::fs::remove_dir(&local).map_err(|e| {
if e.kind() == std::io::ErrorKind::DirectoryNotEmpty {
RemoteError::Configuration(format!(
"{what} is not empty; a folder library will not delete a tree"
))
} else {
map_io(e, &what)
}
})
} else {
std::fs::remove_file(&local).map_err(|e| map_io(e, &what))
}
})
.await
}
async fn move_to(&self, from: &RemoteId, to: &RemotePath) -> Result<(), RemoteError> {
// Move whichever name exists. Trashing a photograph that is not
// downloaded is a perfectly ordinary thing to do, and it must move the
// stub — renaming a placeholder keeps it a placeholder.
let (src, materialised) = self.locate_id(from)?;
let dst = if materialised {
self.resolve(to)?
} else {
// The destination keeps the placeholder suffix, or the client
// would see a one-byte file appear where a photograph should be.
self.resolve(&RemotePath::new(
self.vfs.placeholder_name(to.as_str()).into_owned(),
))?
};
blocking(move || {
let what = dst.display().to_string();
// Parents first: the trash folder does not exist until the first
// photograph is trashed, and the trait promises this creates it.
if let Some(parent) = dst.parent() {
std::fs::create_dir_all(parent)
.map_err(|e| map_io(e, &parent.display().to_string()))?;
}
match std::fs::rename(&src, &dst) {
Ok(()) => Ok(()),
// EXDEV. Both paths are inside one library root, so this
// needs a root that spans a mount point — a shoot folder
// that is its own mount, which is an ordinary way to attach
// an archive drive. Copy and unlink rather than refusing:
// the identity a rename would have preserved is a path hash
// here, and it changes either way.
Err(e) if e.raw_os_error() == Some(18) => {
std::fs::copy(&src, &dst).map_err(|e| map_io(e, &what))?;
std::fs::remove_file(&src).map_err(|e| {
// The copy landed. Leaving the original is a
// duplicate, which the next scan will show; losing
// the copy would be worse.
let _ = std::fs::remove_file(&dst);
map_io(e, &src.display().to_string())
})
}
Err(e) => Err(map_io(e, &what)),
}
})
.await
}
/// TRACES: FR-NC-6c
/// Ask the sync client to download a placeholder, and wait for it.
///
/// Suffix-mode VFS *renames* on hydration, so completion is the
/// materialised path appearing — not the stub changing size. Polling the
/// original would wait forever.
async fn materialise(&self, id: &RemoteId) -> Result<bool, RemoteError> {
let (local, materialised) = self.locate_id(id)?;
if materialised {
// Already here. Not an error, and not a reason to ask again — and
// `false` is what tells a borrower to leave it alone afterwards.
return Ok(false);
}
let vfs = self.vfs.clone();
let target = self.resolve_id(id)?;
blocking(move || {
vfs.materialise(&local)?;
// The client acknowledges the command, not the transfer, so this
// waits for the file to appear. A bounded wait: a hydration that
// has not landed in this long is one the caller should be told
// about rather than blocked on for ever — the pass can come back
// to it.
let deadline = std::time::Instant::now() + MATERIALISE_TIMEOUT;
while std::time::Instant::now() < deadline {
if target.is_file() {
return Ok(true);
}
std::thread::sleep(POLL);
}
Err(RemoteError::Network(format!(
"{} did not download within {}s",
target.display(),
MATERIALISE_TIMEOUT.as_secs()
)))
})
.await
}
/// TRACES: FR-NC-6c
/// Hand the content back, leaving a placeholder.
///
/// **Never a delete.** In a synced tree removing the file propagates the
/// removal to the server; the client is asked to dehydrate, and if it
/// cannot the content simply stays.
async fn dematerialise(&self, id: &RemoteId) -> Result<(), RemoteError> {
let (local, materialised) = self.locate_id(id)?;
if !materialised {
return Ok(());
}
let vfs = self.vfs.clone();
blocking(move || vfs.dematerialise(&local)).await
}
async fn create_dir(&self, path: &RemotePath) -> Result<(), RemoteError> {
let local = self.resolve(path)?;
blocking(move || {
// `create_dir_all` makes parents and succeeds on one that already
// exists, which is exactly the contract.
std::fs::create_dir_all(&local).map_err(|e| map_io(e, &local.display().to_string()))
})
.await
}
}
#[cfg(test)]
mod tests;
core/dr-sync-folder/src/tests.rs
+741
View File
@@ -0,0 +1,741 @@
//! Behaviour of the folder connector, against real directories.
//!
//! No mocks: the whole point of this backend is what a filesystem actually
//! does, and a double would only assert what this file assumes.
use super::*;
use dr_sync::{scan, RemoteBackend};
use dr_types::FormatFilter;
use std::collections::HashMap;
/// A throwaway library root.
///
/// Under the system temp directory, named for the test, and cleared first so a
/// crashed run cannot leave state that makes the next one pass.
struct Tmp(PathBuf);
impl Tmp {
fn new(name: &str) -> Self {
let d = std::env::temp_dir().join(format!("dr-folder-test-{name}"));
let _ = std::fs::remove_dir_all(&d);
std::fs::create_dir_all(&d).unwrap();
Tmp(d)
}
fn file(&self, rel: &str, body: &[u8]) -> &Self {
let p = self.0.join(rel);
std::fs::create_dir_all(p.parent().unwrap()).unwrap();
std::fs::write(p, body).unwrap();
self
}
fn backend(&self) -> FolderBackend {
FolderBackend::new(&self.0).unwrap()
}
}
impl Drop for Tmp {
fn drop(&mut self) {
let _ = std::fs::remove_dir_all(&self.0);
}
}
fn names(entries: &[RemoteEntry]) -> Vec<String> {
let mut v: Vec<String> = entries.iter().map(|e| e.path.name().to_string()).collect();
v.sort();
v
}
// --- opening --------------------------------------------------------------
#[test]
fn a_missing_folder_is_a_configuration_error_not_a_network_one() {
// It must not put the app into offline mode: nothing was unreachable, the
// account names somewhere that is not a folder.
let err = FolderBackend::new("/definitely/not/here").unwrap_err();
assert!(matches!(err, RemoteError::Configuration(_)), "{err:?}");
assert!(!err.indicates_offline());
}
// --- listing --------------------------------------------------------------
#[tokio::test]
async fn listing_reports_files_and_directories() {
let t = Tmp::new("list");
t.file("a.CR2", b"raw").file("sub/b.CR2", b"raw");
let b = t.backend();
let root = b.list(&RemotePath::root(), None).await.unwrap();
assert_eq!(names(&root), vec!["a.CR2", "sub"]);
let kinds: HashMap<_, _> = root
.iter()
.map(|e| (e.path.name().to_string(), e.kind))
.collect();
assert_eq!(kinds["a.CR2"], EntryKind::File);
assert_eq!(kinds["sub"], EntryKind::Directory);
let sub = b.list(&RemotePath::new("sub"), None).await.unwrap();
assert_eq!(names(&sub), vec!["b.CR2"]);
// Paths are rooted at the library, not at the filesystem.
assert_eq!(sub[0].path.as_str(), "sub/b.CR2");
}
#[tokio::test]
async fn a_listing_carries_the_size_a_scan_needs() {
let t = Tmp::new("size");
t.file("a.CR2", &[7u8; 1234]);
let e = &t.backend().list(&RemotePath::root(), None).await.unwrap()[0];
assert_eq!(e.size, 1234);
assert!(e.modified.is_some());
// Nothing behind a folder renders anything.
assert!(!e.has_preview);
}
#[tokio::test]
async fn listing_a_missing_directory_is_not_found() {
let t = Tmp::new("missing");
let e = t
.backend()
.list(&RemotePath::new("nope"), None)
.await
.unwrap_err();
assert!(matches!(e, RemoteError::NotFound(_)), "{e:?}");
}
// --- identity and validators ---------------------------------------------
#[tokio::test]
async fn identity_is_stable_across_an_edit_but_not_across_a_rename() {
// The catalog keys thumbnails and faces on this id, so editing a file must
// not orphan its thumbnail. A rename is a different photograph as far as
// this backend can tell, which `Capabilities::stable_ids` reports.
let t = Tmp::new("identity");
t.file("a.CR2", b"one");
let b = t.backend();
let before = b.list(&RemotePath::root(), None).await.unwrap()[0]
.id
.clone();
t.file("a.CR2", b"two-different-length");
let after = b.list(&RemotePath::root(), None).await.unwrap()[0]
.id
.clone();
assert_eq!(before, after, "an edit is not a new photograph");
std::fs::rename(t.0.join("a.CR2"), t.0.join("b.CR2")).unwrap();
let renamed = b.list(&RemotePath::root(), None).await.unwrap()[0]
.id
.clone();
assert_ne!(before, renamed);
assert!(!b.capabilities().stable_ids, "and the capability says so");
}
#[tokio::test]
async fn two_libraries_agree_on_the_identity_of_the_same_photograph() {
// Two devices mounting one share must key the thumbnail index the same
// way, or each re-derives what the other already stored. This is why the
// id is a path hash and not an inode.
let a = Tmp::new("id-a");
let b = Tmp::new("id-b");
a.file("2026/x.CR2", b"one");
b.file("2026/x.CR2", b"quite different bytes");
let ida = a
.backend()
.list(&RemotePath::new("2026"), None)
.await
.unwrap()[0]
.id
.clone();
let idb = b
.backend()
.list(&RemotePath::new("2026"), None)
.await
.unwrap()[0]
.id
.clone();
assert_eq!(ida, idb);
}
#[tokio::test]
async fn a_validator_changes_when_the_content_does() {
let t = Tmp::new("validator");
t.file("a.CR2", b"one");
let b = t.backend();
let before = b.list(&RemotePath::root(), None).await.unwrap()[0]
.validator
.clone();
// A different length, so this holds on a filesystem with one-second mtime
// granularity as well as on one with nanoseconds.
t.file("a.CR2", b"a rather longer body");
let after = b.list(&RemotePath::root(), None).await.unwrap()[0]
.validator
.clone();
assert_ne!(before, after);
}
#[tokio::test]
async fn a_folder_does_not_pretend_to_prune() {
// Answering with the directory's own mtime would let a caller skip a
// subtree whose files had been edited, hiding those edits indefinitely.
let t = Tmp::new("prune");
let b = t.backend();
assert!(matches!(
b.dir_validator(&RemotePath::root()).await,
Err(RemoteError::Unsupported(_))
));
assert_eq!(
b.capabilities().change_detection,
ChangeDetection::LocalEtags
);
}
// --- reading --------------------------------------------------------------
#[tokio::test]
async fn a_whole_file_and_a_range_both_read() {
let t = Tmp::new("get");
t.file("a.CR2", b"0123456789");
let b = t.backend();
let id = RemoteId::Path(RemotePath::new("a.CR2"));
assert_eq!(b.get(&id, None).await.unwrap(), b"0123456789");
assert_eq!(b.get(&id, Some(2..5)).await.unwrap(), b"234");
}
#[tokio::test]
async fn a_range_past_the_end_returns_what_is_there() {
// The header extractor asks for a fixed window; a small JPEG is shorter
// than it, and failing would make every small file undatable.
let t = Tmp::new("shortrange");
t.file("a.JPG", b"abc");
let got = t
.backend()
.get(&RemoteId::Path(RemotePath::new("a.JPG")), Some(0..65536))
.await
.unwrap();
assert_eq!(got, b"abc");
}
#[tokio::test]
async fn a_bare_identity_cannot_address_a_file() {
// Same contract as the Nextcloud connector: the id says *which*
// photograph, the path says *where*. Callers hold both.
let t = Tmp::new("byid");
t.file("a.CR2", b"x");
let e = t
.backend()
.get(&RemoteId::Stable(1), None)
.await
.unwrap_err();
assert!(matches!(e, RemoteError::Unsupported(_)), "{e:?}");
}
#[tokio::test]
async fn nothing_reachable_from_a_remote_path_escapes_the_library() {
// A `RemotePath` is built from names on the remote and from a catalog
// another device wrote. Resolving one against a real filesystem with the
// user's own permissions makes `..` a read of anything they own.
let t = Tmp::new("escape");
let b = t.backend();
for attempt in ["../../../etc/passwd", "sub/../../outside"] {
let e = b
.get(&RemoteId::Path(RemotePath::new(attempt)), None)
.await
.unwrap_err();
assert!(
matches!(e, RemoteError::Configuration(_)),
"{attempt} was not refused: {e:?}"
);
}
}
// --- writing --------------------------------------------------------------
#[tokio::test]
async fn a_write_creates_the_folders_it_needs() {
let t = Tmp::new("put");
let b = t.backend();
b.put(&RemotePath::new("2026/03/a.xmp"), b"<x/>".to_vec(), None)
.await
.unwrap();
assert_eq!(std::fs::read(t.0.join("2026/03/a.xmp")).unwrap(), b"<x/>");
}
#[tokio::test]
async fn a_write_leaves_no_temporary_behind() {
// The rename-into-place is invisible from outside, and must stay that way:
// a stray `.darkroom-tmp` in a shoot folder would be listed by the scan.
let t = Tmp::new("puttmp");
let b = t.backend();
b.put(&RemotePath::new("a.xmp"), b"x".to_vec(), None)
.await
.unwrap();
assert_eq!(
names(&b.list(&RemotePath::root(), None).await.unwrap()),
vec!["a.xmp"]
);
}
#[tokio::test]
async fn an_overwrite_replaces_rather_than_appends() {
let t = Tmp::new("overwrite");
t.file("a.xmp", b"the older and much longer body");
let b = t.backend();
b.put(&RemotePath::new("a.xmp"), b"new".to_vec(), None)
.await
.unwrap();
assert_eq!(std::fs::read(t.0.join("a.xmp")).unwrap(), b"new");
}
#[tokio::test]
async fn if_absent_creates_once_and_refuses_after() {
let t = Tmp::new("ifabsent");
let b = t.backend();
let p = RemotePath::new("a.xmp");
b.put(&p, b"first".to_vec(), Some(Precondition::IfAbsent))
.await
.unwrap();
let e = b
.put(&p, b"second".to_vec(), Some(Precondition::IfAbsent))
.await
.unwrap_err();
assert!(matches!(e, RemoteError::PreconditionFailed), "{e:?}");
assert_eq!(std::fs::read(t.0.join("a.xmp")).unwrap(), b"first");
}
#[tokio::test]
async fn if_match_writes_on_the_expected_version_and_refuses_a_stale_one() {
// The sidecar conflict path (ARCH §8.5): a failure here means another
// device wrote first, and triggers a merge rather than an overwrite.
let t = Tmp::new("ifmatch");
t.file("a.xmp", b"one");
let b = t.backend();
let p = RemotePath::new("a.xmp");
let current = b.list(&RemotePath::root(), None).await.unwrap()[0]
.validator
.clone();
let after = b
.put(
&p,
b"two".to_vec(),
Some(Precondition::IfMatch(current.clone())),
)
.await
.unwrap();
assert_ne!(after, current);
let e = b
.put(&p, b"three".to_vec(), Some(Precondition::IfMatch(current)))
.await
.unwrap_err();
assert!(matches!(e, RemoteError::PreconditionFailed), "{e:?}");
assert_eq!(std::fs::read(t.0.join("a.xmp")).unwrap(), b"two");
}
#[tokio::test]
async fn the_validator_a_write_returns_is_the_one_a_listing_reports() {
// Otherwise the next conditional write fails against a file nobody else
// touched, and every sidecar update becomes a spurious conflict.
let t = Tmp::new("putvalidator");
let b = t.backend();
let p = RemotePath::new("a.xmp");
let written = b.put(&p, b"body".to_vec(), None).await.unwrap();
let listed = b.list(&RemotePath::root(), None).await.unwrap()[0]
.validator
.clone();
assert_eq!(written, listed);
}
// --- moving and deleting --------------------------------------------------
#[tokio::test]
async fn a_move_creates_the_trash_folder_it_needs() {
// The soft delete (FR-CAT-15): the trash does not exist until the first
// photograph goes into it, and the trait promises the move makes it.
let t = Tmp::new("move");
t.file("a.CR2", b"raw");
let b = t.backend();
b.move_to(
&RemoteId::Path(RemotePath::new("a.CR2")),
&RemotePath::new(".darkroom-trash/a.CR2"),
)
.await
.unwrap();
assert!(!t.0.join("a.CR2").exists());
assert_eq!(
std::fs::read(t.0.join(".darkroom-trash/a.CR2")).unwrap(),
b"raw"
);
}
#[tokio::test]
async fn deleting_a_file_removes_it() {
let t = Tmp::new("delete");
t.file("a.CR2", b"raw");
let b = t.backend();
b.delete(&RemoteId::Path(RemotePath::new("a.CR2")), None)
.await
.unwrap();
assert!(!t.0.join("a.CR2").exists());
}
#[tokio::test]
async fn deleting_refuses_to_take_a_tree_with_it() {
// Deliberately unlike WebDAV. There is no server-side trash behind a local
// folder, so a caller with a wrong path would have no way back.
let t = Tmp::new("deletetree");
t.file("shoot/a.CR2", b"raw");
let e = t
.backend()
.delete(&RemoteId::Path(RemotePath::new("shoot")), None)
.await
.unwrap_err();
assert!(matches!(e, RemoteError::Configuration(_)), "{e:?}");
assert!(t.0.join("shoot/a.CR2").exists());
}
#[tokio::test]
async fn a_conditional_delete_refuses_a_file_that_changed() {
let t = Tmp::new("deletecond");
t.file("a.CR2", b"raw");
let b = t.backend();
let stale = Validator::new("0-0.0");
let e = b
.delete(
&RemoteId::Path(RemotePath::new("a.CR2")),
Some(Precondition::IfMatch(stale)),
)
.await
.unwrap_err();
assert!(matches!(e, RemoteError::PreconditionFailed), "{e:?}");
assert!(t.0.join("a.CR2").exists());
}
#[tokio::test]
async fn creating_a_directory_twice_succeeds() {
// Callers use this to guarantee a destination, not to claim they made it.
let t = Tmp::new("mkdir");
let b = t.backend();
let p = RemotePath::new("2026/03");
b.create_dir(&p).await.unwrap();
b.create_dir(&p).await.unwrap();
assert!(t.0.join("2026/03").is_dir());
}
// --- driven by the engine -------------------------------------------------
#[tokio::test]
async fn the_scan_engine_walks_a_folder_library() {
// The claim this whole crate makes: the engine written for one backend
// drives another with no change. Nothing below is folder-specific.
let t = Tmp::new("scan");
t.file("2026/03/a.CR2", b"raw")
.file("2026/03/b.JPG", b"jpeg")
.file("2026/04/c.CR2", b"raw")
.file("2026/notes.txt", b"text")
.file(".darkroom-trash/deleted.CR2", b"raw");
let result = scan(
&t.backend(),
&RemotePath::root(),
&FormatFilter::from_formats([dr_types::Format::Cr2]),
&HashMap::new(),
|_| {},
)
.await
.unwrap();
let found: Vec<&str> = result.images.iter().map(|e| e.path.as_str()).collect();
// The filter picked the RAWs; the trash was skipped, or the soft delete
// would undo itself on the next scan.
assert_eq!(found, vec!["2026/03/a.CR2", "2026/04/c.CR2"]);
assert_eq!(result.progress.directories_pruned, 0, "nothing to prune");
}
#[tokio::test]
async fn an_upload_lands_where_the_engine_places_it() {
let t = Tmp::new("upload");
let b = t.backend();
let placed = dr_sync::upload_original(
&b,
&RemotePath::root(),
&["2026".to_string(), "03".to_string()],
"a.CR2",
b"raw".to_vec(),
)
.await
.unwrap();
assert_eq!(placed.path().as_str(), "2026/03/a.CR2");
assert_eq!(std::fs::read(t.0.join("2026/03/a.CR2")).unwrap(), b"raw");
}
// --- the provider ---------------------------------------------------------
#[test]
fn an_endpoint_is_checked_before_an_account_is_written_for_it() {
let t = Tmp::new("provider");
let p = FolderProvider::new();
assert!(p.normalise_endpoint(" ").is_err(), "empty");
assert!(p.normalise_endpoint("Pictures").is_err(), "relative");
assert!(p.normalise_endpoint("/no/such/place").is_err(), "missing");
t.file("a.CR2", b"x");
assert!(
p.normalise_endpoint(&t.0.join("a.CR2").to_string_lossy())
.is_err(),
"a file is not a library"
);
let ok = p.normalise_endpoint(&t.0.to_string_lossy()).unwrap();
assert_eq!(PathBuf::from(&ok), t.0.canonicalize().unwrap());
}
#[test]
fn two_spellings_of_one_folder_become_one_account() {
// Otherwise the same photographs are indexed twice, into two catalogs.
let t = Tmp::new("canonical");
t.file("sub/a.CR2", b"x");
let p = FolderProvider::new();
let direct = p
.normalise_endpoint(&t.0.join("sub").to_string_lossy())
.unwrap();
let roundabout = p
.normalise_endpoint(&t.0.join("sub/../sub").to_string_lossy())
.unwrap();
assert_eq!(direct, roundabout);
}
#[test]
fn a_folder_account_needs_no_credential() {
let p = FolderProvider::new();
assert_eq!(p.sign_in(), SignIn::EndpointOnly);
assert!(!p.sign_in().needs_secret());
let account = p.account_for("/mnt/photos").unwrap();
assert_eq!(account.backend, BACKEND_ID);
assert_eq!(account.endpoint, "/mnt/photos");
assert!(account.login.is_empty());
}
#[test]
fn the_registry_opens_a_folder_account() {
// End to end through the abstraction: an account, a registry, a backend —
// with nothing in between naming this crate.
let t = Tmp::new("registry");
let mut registry = dr_sync::BackendRegistry::new();
registry.register(std::sync::Arc::new(FolderProvider::new()));
let account = Account::new(BACKEND_ID, t.0.to_string_lossy());
let backend = registry.connect(&Connection::new(account, None)).unwrap();
assert_eq!(backend.name(), "Folder");
}
// --- virtual filesystems --------------------------------------------------
//
// A suffix-mode convention, matching the only one Linux supports. The
// behaviour under test is what the *backend* does with it; the borrow cycle
// has its own tests beside the pool.
struct SuffixVfs;
impl Vfs for SuffixVfs {
fn name(&self) -> &'static str {
"suffix"
}
fn is_placeholder(&self, on_disk: &str) -> bool {
on_disk.ends_with(".stub")
}
fn real_name<'a>(&self, on_disk: &'a str) -> &'a str {
on_disk.strip_suffix(".stub").unwrap_or(on_disk)
}
fn placeholder_name(&self, name: &str) -> std::borrow::Cow<'_, str> {
std::borrow::Cow::Owned(format!("{name}.stub"))
}
}
fn with_stubs(t: &Tmp) -> FolderBackend {
FolderBackend::with_vfs(&t.0, std::sync::Arc::new(SuffixVfs)).unwrap()
}
#[tokio::test]
async fn a_placeholder_is_listed_under_the_photographs_own_name() {
// The catalog records this as `source_ref`, and identity is derived from
// it. Reporting the stub's name gives the same photograph two identities
// and a name no other device recognises.
let t = Tmp::new("vfs-name");
t.file("shoot/IMG_0001.CR2.stub", &[0u8]);
let b = with_stubs(&t);
let entries = b.list(&RemotePath::new("shoot"), None).await.unwrap();
assert_eq!(entries[0].path.as_str(), "shoot/IMG_0001.CR2");
assert!(!entries[0].materialised, "the content is not here");
// One byte is not the photograph's size, and putting it in the catalog
// would claim a 30 MB RAW is a single byte.
assert_eq!(entries[0].size, 0, "unknown, not one");
}
#[tokio::test]
async fn identity_survives_a_download() {
// The failure this prevents: downloading a photograph looked like a
// delete and an add, which orphaned its thumbnail and its face rows.
let t = Tmp::new("vfs-identity");
t.file("a.CR2.stub", &[0u8]);
let b = with_stubs(&t);
let before = b.list(&RemotePath::root(), None).await.unwrap()[0]
.id
.clone();
std::fs::remove_file(t.0.join("a.CR2.stub")).unwrap();
std::fs::write(t.0.join("a.CR2"), vec![3u8; 4096]).unwrap();
let after = b.list(&RemotePath::root(), None).await.unwrap()[0]
.id
.clone();
assert_eq!(before, after, "the same photograph throughout");
}
#[tokio::test]
async fn reading_a_placeholder_is_distinguishable_from_a_missing_file() {
// The distinction the sidecar writer depends on: "not here" is fetchable
// and "not found" means create a new one. Conflating them overwrites an
// existing sidecar with a fresh document.
let t = Tmp::new("vfs-read");
t.file("a.drsc.stub", &[0u8]);
let b = with_stubs(&t);
let stub = b
.get(&RemoteId::Path(RemotePath::new("a.drsc")), None)
.await
.unwrap_err();
assert!(matches!(stub, RemoteError::NotMaterialised(_)), "{stub:?}");
let absent = b
.get(&RemoteId::Path(RemotePath::new("nothing.drsc")), None)
.await
.unwrap_err();
assert!(matches!(absent, RemoteError::NotFound(_)), "{absent:?}");
// And emphatically not the stub's one byte, which is what made a
// dehydrated sidecar parse as an empty document.
assert!(!matches!(stub, RemoteError::NotFound(_)));
}
#[tokio::test]
async fn an_unconditional_write_replaces_a_placeholder() {
// Derived state — shards, the catalog snapshot — lives in the library
// folder, so the client dehydrates it like anything else. Refusing here
// meant sync could never write to a folder it had been away from. The
// whole file is being replaced, so there is nothing in the stub to keep.
let t = Tmp::new("vfs-write");
t.file("a.drsc.stub", &[0u8]);
let b = with_stubs(&t);
b.put(&RemotePath::new("a.drsc"), b"<new/>".to_vec(), None)
.await
.unwrap();
assert_eq!(std::fs::read(t.0.join("a.drsc")).unwrap(), b"<new/>");
// And exactly one file for one document: a leftover stub beside it is a
// conflict the client would resolve in favour of whichever it saw last.
assert!(!t.0.join("a.drsc.stub").exists(), "placeholder left behind");
assert_eq!(
names(&b.list(&RemotePath::root(), None).await.unwrap()),
vec!["a.drsc"]
);
}
#[tokio::test]
async fn a_conditional_write_over_a_placeholder_asks_for_the_content_first() {
// `IfMatch` guards a read-modify-write. A stub's validator describes the
// placeholder, not the document, so nothing here can satisfy it — and
// quietly writing anyway is how the other device's edits are lost.
let t = Tmp::new("vfs-write-cond");
t.file("a.drsc.stub", &[0u8]);
let b = with_stubs(&t);
let e = b
.put(
&RemotePath::new("a.drsc"),
b"<new/>".to_vec(),
Some(Precondition::IfMatch(Validator::new("whatever"))),
)
.await
.unwrap_err();
assert!(matches!(e, RemoteError::NotMaterialised(_)), "{e:?}");
assert!(!t.0.join("a.drsc").exists(), "nothing written");
// And a create-if-absent fails, because the file *is* there — only its
// content is elsewhere.
let e = b
.put(
&RemotePath::new("a.drsc"),
b"<new/>".to_vec(),
Some(Precondition::IfAbsent),
)
.await
.unwrap_err();
assert!(matches!(e, RemoteError::PreconditionFailed), "{e:?}");
}
#[tokio::test]
async fn trashing_a_photograph_that_is_not_downloaded_moves_the_placeholder() {
// Culling without downloading is the ordinary way to use a VFS library.
// The stub has to move, and has to stay a stub — leaving it behind means
// the next scan re-lists the image and undoes the delete.
let t = Tmp::new("vfs-trash");
t.file("a.CR2.stub", &[0u8]);
let b = with_stubs(&t);
b.move_to(
&RemoteId::Path(RemotePath::new("a.CR2")),
&RemotePath::new(".darkroom-trash/a.CR2"),
)
.await
.unwrap();
assert!(!t.0.join("a.CR2.stub").exists());
assert!(
t.0.join(".darkroom-trash/a.CR2.stub").is_file(),
"still a stub"
);
}
#[tokio::test]
async fn a_folder_without_a_client_still_lists_and_reads_what_is_there() {
// No hydration available is a degraded mode, not a broken one: the
// materialised half of the library works completely.
let t = Tmp::new("vfs-degraded");
t.file("here.CR2", b"real").file("gone.CR2.stub", &[0u8]);
let b = with_stubs(&t);
assert_eq!(
b.capabilities().materialisation,
dr_sync::Materialisation::Placeholders,
"stubs exist and nothing can fetch them"
);
assert!(!b.capabilities().materialisation.can_materialise());
let got = b
.get(&RemoteId::Path(RemotePath::new("here.CR2")), None)
.await
.unwrap();
assert_eq!(got, b"real");
}
#[test]
fn a_plain_folder_reports_that_everything_it_lists_is_readable() {
let t = Tmp::new("vfs-plain");
assert_eq!(
t.backend().capabilities().materialisation,
dr_sync::Materialisation::Always
);
}
core/dr-sync-folder/src/vfs.rs
+131
View File
@@ -0,0 +1,131 @@
// TRACES: FR-NC-6c
//! Virtual-filesystem conventions layered over a directory.
//!
//! A sync client in virtual-files mode leaves a *placeholder* where a file is
//! catalogued but not downloaded. The folder is otherwise ordinary, so all of
//! [`FolderBackend`](crate::FolderBackend) applies — only three questions
//! differ, and they are the whole of this trait: what is a placeholder, what
//! is the photograph really called, and can the content be summoned.
//!
//! # Why this is not a separate backend
//!
//! It varies nothing about listing, reading, writing, moving or deleting — a
//! second connector would duplicate every one of those to change a name test.
//! More decisively, **the interesting capability is not a property of the
//! backend at all**: the same folder can materialise on demand while the sync
//! client is running and cannot when it is not, so it has to be computed per
//! connection either way. Registering a `folder-vfs` provider beside `folder`
//! would ask the user to choose between two things that differ by whether a
//! background process happens to be up.
//!
//! # Why the plain case is a `Vfs` too
//!
//! [`NoVfs`] answers "nothing is a placeholder" and refuses to materialise.
//! That keeps one code path through the backend rather than an `Option` tested
//! at every call site, and it is the shape a third convention — Dropbox,
//! OneDrive, macOS FileProvider — slots into.
//!
//! # What is *not* abstracted here
//!
//! Windows and macOS express placeholders in filesystem metadata rather than
//! in the name: a reparse point, or `st_blocks == 0` against a non-zero
//! `st_size`. That form needs a `Metadata` to answer, not a name, and the one
//! convention this project has met needs only a name. Widening the trait for a
//! platform nobody has run this on would be guessing at the shape.
use std::borrow::Cow;
use std::path::Path;
use dr_sync::RemoteError;
/// A placeholder convention, and what can be done about it.
///
/// Implementations are held behind an `Arc` and used from every worker
/// thread.
pub trait Vfs: Send + Sync {
/// A name for logs and the interface. "none", "Nextcloud".
fn name(&self) -> &'static str;
/// Whether a name **on disk** stands for content that is not here.
fn is_placeholder(&self, on_disk: &str) -> bool;
/// The photograph's own name, given whatever is on disk.
///
/// This is what the catalog records and what identity is derived from, so
/// a file keeps one name and one id across being downloaded and released.
/// Reporting the on-disk name instead makes hydration look like a delete
/// and an add.
fn real_name<'a>(&self, on_disk: &'a str) -> &'a str;
/// What a placeholder for `name` would be called on disk.
fn placeholder_name(&self, name: &str) -> Cow<'_, str>;
/// Whether content can actually be summoned right now.
///
/// False where the mechanism is absent — the client is not running, the
/// platform has no socket — which is an ordinary state and not an error.
/// The backend reports [`Materialisation::Placeholders`] rather than
/// [`OnDemand`] when this is false.
///
/// [`Materialisation::Placeholders`]: dr_sync::Materialisation::Placeholders
/// [`OnDemand`]: dr_sync::Materialisation::OnDemand
fn can_materialise(&self) -> bool {
false
}
/// Ask for a placeholder's content. Whole-file and slow.
fn materialise(&self, _local: &Path) -> Result<(), RemoteError> {
Err(RemoteError::Unsupported("this folder has no VFS client"))
}
/// Give the content back, leaving a placeholder.
///
/// **Must not delete.** In a synced tree a deletion propagates to the
/// server and removes the photograph from every device. An implementation
/// that cannot dehydrate returns `Unsupported`.
fn dematerialise(&self, _local: &Path) -> Result<(), RemoteError> {
Err(RemoteError::Unsupported("this folder has no VFS client"))
}
}
/// An ordinary directory: every file is what it appears to be.
#[derive(Debug, Clone, Copy, Default)]
pub struct NoVfs;
impl Vfs for NoVfs {
fn name(&self) -> &'static str {
"none"
}
fn is_placeholder(&self, _on_disk: &str) -> bool {
false
}
fn real_name<'a>(&self, on_disk: &'a str) -> &'a str {
on_disk
}
fn placeholder_name(&self, name: &str) -> Cow<'_, str> {
// Nothing is ever a placeholder here, so the only honest answer is
// the name itself — the backend will look for it, not find a second
// candidate, and report the file missing.
Cow::Owned(name.to_string())
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn a_plain_folder_has_no_placeholders_and_cannot_summon_anything() {
let v = NoVfs;
assert!(
!v.is_placeholder("IMG.CR2.nextcloud"),
"not this folder's convention"
);
assert_eq!(v.real_name("IMG.CR2"), "IMG.CR2");
assert!(!v.can_materialise());
assert!(v.materialise(Path::new("/x")).is_err());
// And it must refuse rather than approximate: deleting a file to
// "dehydrate" it would remove the photograph.
assert!(v.dematerialise(Path::new("/x")).is_err());
}
}
core/dr-sync-nextcloud/Cargo.toml
+3
View File
@@ -8,6 +8,9 @@ license.workspace = true
[dependencies]
dr-types.workspace = true
dr-sync.workspace = true
# For the VFS convention: the folder connector does the filesystem work, and
# this crate supplies the placeholder rules and the client socket.
dr-sync-folder.workspace = true
dr-plat.workspace = true
reqwest.workspace = true
rustls.workspace = true
core/dr-sync-nextcloud/examples/connect.rs
+11 -7
View File
@@ -23,8 +23,9 @@ use std::collections::HashMap;
use std::time::Instant;
use dr_plat::PlatformSecretStore;
use dr_sync::{Account, AccountStore, Secret};
use dr_sync::{RemoteBackend, RemoteId, RemotePath, SyncStrategy};
use dr_sync_nextcloud::{auth, AppCredentials, NextcloudBackend, Session, SessionStore};
use dr_sync_nextcloud::{auth, AppCredentials, NextcloudBackend, NextcloudProvider};
#[tokio::main]
async fn main() {
@@ -57,17 +58,20 @@ async fn main() {
// Sessions persist across runs: credentials in the platform keyring
// (FR-NC-2), everything else as ordinary config.
let sessions = SessionStore::open(Box::new(PlatformSecretStore::new()));
let sessions = AccountStore::open(Box::new(PlatformSecretStore::new()));
if !sessions.can_remember() {
println!("note: no secrets daemon — sign-in will not persist this session");
}
let existing = sessions
.current()
.filter(|s| s.server == server.trim_end_matches('/'));
.filter(|s| s.endpoint == server.trim_end_matches('/'));
let (session, creds) = match existing {
Some(s) => match sessions.credentials(&s) {
Some(s) => match sessions
.connection(&s, true)
.and_then(|c| Ok(NextcloudProvider::credentials(&c)?))
{
Ok(c) => {
println!("signed in: {}", s.describe());
(s, c)
@@ -235,7 +239,7 @@ fn describe_filter(f: &dr_types::FormatFilter) -> String {
}
/// Run Login Flow v2 and persist the result.
async fn sign_in(server: &str, sessions: &SessionStore) -> (Session, AppCredentials) {
async fn sign_in(server: &str, sessions: &AccountStore) -> (Account, AppCredentials) {
let client = match dr_sync_nextcloud::http_client("DarkRoom") {
Ok(c) => c,
Err(e) => {
@@ -270,8 +274,8 @@ async fn sign_in(server: &str, sessions: &SessionStore) -> (Session, AppCredenti
creds.login_name.clone()
});
let session = Session::new(&creds, user_id);
match sessions.save(&session, &creds) {
let session = NextcloudProvider::account_from(&creds, user_id);
match sessions.save(&session, Some(&Secret::new(&creds.app_password))) {
Ok(()) => println!(" session saved to {}", sessions.config_path().display()),
Err(e) => eprintln!(" could not persist session: {e}"),
}
core/dr-sync-nextcloud/examples/put_probe.rs
+10 -5
View File
@@ -18,7 +18,8 @@
//! and deleted again, which tests creation and costs nothing.
use dr_plat::PlatformSecretStore;
use dr_sync_nextcloud::session::SessionStore;
use dr_sync::AccountStore;
use dr_sync_nextcloud::NextcloudProvider;
#[tokio::main(flavor = "current_thread")]
async fn main() {
@@ -30,15 +31,19 @@ async fn main() {
std::process::exit(2);
};
let sessions = SessionStore::open(Box::new(PlatformSecretStore::new()));
let sessions = AccountStore::open(Box::new(PlatformSecretStore::new()));
let Some(session) = sessions
.current()
.filter(|s| s.server == server.trim_end_matches('/'))
.filter(|s| s.endpoint == server.trim_end_matches('/'))
else {
eprintln!("no stored session for {server}");
std::process::exit(1);
};
let creds = match sessions.credentials(&session) {
let creds = match sessions
.connection(&session, true)
.map_err(|e| e.to_string())
.and_then(|c| NextcloudProvider::credentials(&c).map_err(|e| e.to_string()))
{
Ok(c) => c,
Err(e) => {
eprintln!("credentials: {e}");
@@ -48,7 +53,7 @@ async fn main() {
let url = format!(
"{}/remote.php/dav/files/{}/{}",
session.server.trim_end_matches('/'),
session.endpoint.trim_end_matches('/'),
session.user_id,
path
);
core/dr-sync-nextcloud/examples/writetest.rs
+8 -4
View File
@@ -1,18 +1,22 @@
//! One-shot write probe: PUT a tiny file, report the status, DELETE it.
use dr_plat::PlatformSecretStore;
use dr_sync::{RemoteBackend, RemoteId, RemotePath};
use dr_sync_nextcloud::{NextcloudBackend, SessionStore};
use dr_sync::{AccountStore, RemoteBackend, RemoteId, RemotePath};
use dr_sync_nextcloud::{NextcloudBackend, NextcloudProvider};
#[tokio::main(flavor = "current_thread")]
async fn main() {
env_logger::Builder::from_env(env_logger::Env::default().default_filter_or("info")).init();
let store = SessionStore::open(Box::new(PlatformSecretStore::new()));
let store = AccountStore::open(Box::new(PlatformSecretStore::new()));
let Some(session) = store.current() else {
println!("no stored session");
return;
};
let creds = match store.credentials(&session) {
let creds = match store
.connection(&session, true)
.map_err(|e| e.to_string())
.and_then(|c| NextcloudProvider::credentials(&c).map_err(|e| e.to_string()))
{
Ok(c) => c,
Err(e) => {
println!("credentials: {e}");
core/dr-sync-nextcloud/src/desktop_client.rs
+135
View File
@@ -163,3 +163,138 @@ mod tests {
let _ = DesktopClient::detect();
}
}
/// TRACES: FR-NC-6c
/// The desktop client's placeholder convention, as a
/// [`Vfs`](dr_sync_folder::Vfs).
///
/// This is what turns a folder the client syncs into a library DarkRoom can
/// open: the folder connector handles every filesystem operation, and this
/// answers the three questions it cannot — what is a stub, what is the
/// photograph called, and can the content be fetched and given back.
///
/// **Linux suffix mode only**, which is the only mode Linux supports
/// (ARCH §9.0). A dehydrated `IMG.CR2` exists solely as `IMG.CR2.nextcloud`
/// holding one byte.
pub struct NextcloudVfs {
/// `None` where no client is running. The folder still lists and reads
/// correctly; it simply cannot fetch what is not there, which the backend
/// reports as `Materialisation::Placeholders`.
client: Option<DesktopClient>,
}
impl NextcloudVfs {
/// Attach to a running client, if there is one.
///
/// Absence is the ordinary state — Android always, desktop whenever the
/// client is not running — and never an error.
pub fn detect() -> Self {
Self {
client: DesktopClient::detect(),
}
}
/// Whether a directory looks like one this client syncs.
///
/// Used to decide whether to apply this convention at all. Deliberately
/// cheap and deliberately not authoritative: the client's own database
/// would answer properly, but it is a private schema, and being wrong here
/// costs one extra `stat` per read rather than anything correctness
/// depends on.
pub fn looks_synced(root: &Path) -> bool {
std::fs::read_dir(root)
.map(|entries| {
entries.flatten().any(|e| {
let name = e.file_name();
let name = name.to_string_lossy();
// The client's per-folder journal sits at the sync root,
// and a stub anywhere beneath it is equally conclusive.
name.starts_with("._sync_") && name.ends_with(".db")
|| name.ends_with(dr_types::PLACEHOLDER_SUFFIX)
})
})
.unwrap_or(false)
}
}
impl dr_sync_folder::Vfs for NextcloudVfs {
fn name(&self) -> &'static str {
"Nextcloud"
}
fn is_placeholder(&self, on_disk: &str) -> bool {
on_disk.ends_with(dr_types::PLACEHOLDER_SUFFIX)
}
fn real_name<'a>(&self, on_disk: &'a str) -> &'a str {
on_disk
.strip_suffix(dr_types::PLACEHOLDER_SUFFIX)
.unwrap_or(on_disk)
}
fn placeholder_name(&self, name: &str) -> std::borrow::Cow<'_, str> {
std::borrow::Cow::Owned(format!("{name}{}", dr_types::PLACEHOLDER_SUFFIX))
}
fn can_materialise(&self) -> bool {
self.client.is_some()
}
fn materialise(&self, local: &Path) -> Result<(), RemoteError> {
self.client
.as_ref()
.ok_or(RemoteError::Unsupported(
"no Nextcloud desktop client is running to fetch this",
))?
.make_available_locally(local)
}
fn dematerialise(&self, local: &Path) -> Result<(), RemoteError> {
self.client
.as_ref()
.ok_or(RemoteError::Unsupported(
"no Nextcloud desktop client is running to release this",
))?
.make_online_only(local)
}
}
#[cfg(test)]
mod vfs_tests {
use super::*;
use dr_sync_folder::Vfs as _;
#[test]
fn a_stub_is_recognised_and_reports_the_photographs_name() {
let v = NextcloudVfs { client: None };
assert!(v.is_placeholder("IMG_4130.CR2.nextcloud"));
assert!(!v.is_placeholder("IMG_4130.CR2"));
// The name the catalog records, so identity survives a download.
assert_eq!(v.real_name("IMG_4130.CR2.nextcloud"), "IMG_4130.CR2");
assert_eq!(v.real_name("IMG_4130.CR2"), "IMG_4130.CR2");
assert_eq!(v.placeholder_name("IMG_4130.CR2"), "IMG_4130.CR2.nextcloud");
}
#[test]
fn without_a_client_it_refuses_rather_than_pretending() {
// The folder still works; it just cannot fetch. Silently doing nothing
// would make a borrow think it had the content.
let v = NextcloudVfs { client: None };
assert!(!v.can_materialise());
assert!(v.materialise(Path::new("/x/a.CR2.nextcloud")).is_err());
assert!(v.dematerialise(Path::new("/x/a.CR2")).is_err());
}
#[test]
fn an_ordinary_folder_is_not_mistaken_for_a_synced_one() {
let d = std::env::temp_dir().join("dr-vfs-detect");
let _ = std::fs::remove_dir_all(&d);
std::fs::create_dir_all(&d).unwrap();
std::fs::write(d.join("a.CR2"), b"raw").unwrap();
assert!(!NextcloudVfs::looks_synced(&d));
std::fs::write(d.join("b.CR2.nextcloud"), [0u8]).unwrap();
assert!(NextcloudVfs::looks_synced(&d));
let _ = std::fs::remove_dir_all(&d);
}
}
core/dr-sync-nextcloud/src/lib.rs
+13 -4
View File
@@ -1,4 +1,9 @@
//! Nextcloud connector — the only [`RemoteBackend`] implementation.
//! Nextcloud connector.
//!
//! One of two [`RemoteBackend`] implementations, registered through
//! [`NextcloudProvider`]. What an *account* is no longer lives here — that is
//! [`dr_sync::Account`], which has no server in it — so this crate is the
//! protocol and nothing else.
//!
//! Hand-rolled over `reqwest` rather than built on a WebDAV crate (D7). No
//! mature Nextcloud crate exists, and the operations that matter here are
@@ -16,11 +21,11 @@ use dr_sync::{
pub mod auth;
pub mod desktop_client;
mod propfind;
pub mod session;
pub mod provider;
pub use auth::{AppCredentials, LoginFlow};
pub use desktop_client::DesktopClient;
pub use session::{Session, SessionError, SessionStore};
pub use desktop_client::{DesktopClient, NextcloudVfs};
pub use provider::NextcloudProvider;
/// Chunk sizes Nextcloud's chunked upload v2 accepts.
const CHUNKS: ChunkConstraints = ChunkConstraints {
@@ -68,6 +73,10 @@ impl NextcloudBackend {
// Stock Nextcloud ships no RAW preview provider (ARCH §6.7).
// Probed per-account at setup and upgraded where present.
server_previews: ServerPreviews::CommonFormatsOnly,
// The server answers for everything it lists. Placeholders
// belong to a locally *synced folder*, which is the folder
// connector's business (`desktop_client::NextcloudVfs`).
materialisation: dr_sync::Materialisation::Always,
},
})
}
core/dr-sync-nextcloud/src/propfind.rs
+3
View File
@@ -92,6 +92,9 @@ pub fn parse_multistatus(xml: &str, base: &str) -> Result<Vec<RemoteEntry>, Remo
size: r.content_length.unwrap_or(0),
modified: r.last_modified.as_deref().and_then(parse_http_date),
has_preview: r.has_preview,
// Everything WebDAV lists can be fetched; placeholders are a
// property of a locally synced folder, not of the server.
materialised: true,
});
}
Ok(out)
core/dr-sync-nextcloud/src/provider.rs
+174
View File
@@ -0,0 +1,174 @@
// TRACES: FR-NC-12 | FR-NC-1
//! Registering Nextcloud as a storage backend.
//!
//! The account model this connector used to own now lives in
//! [`dr_sync::account`], where it has no server in it. What is left here is
//! the part that genuinely is Nextcloud: an endpoint is an HTTPS URL, an
//! account is established through Login Flow v2, and the credential is an app
//! password.
//!
//! Nothing above `dr_ui::remote` refers to this type.
use dr_sync::{
Account, BackendProvider, Connection, RemoteBackend, RemoteError, SignIn, LEGACY_BACKEND,
};
use crate::{AppCredentials, NextcloudBackend};
/// The id written to [`Account::backend`] for a Nextcloud account.
///
/// The same string [`dr_sync::LEGACY_BACKEND`] freezes, because every account
/// configured before there was a choice is one of these and must keep the
/// catalog directory it already has.
pub const BACKEND_ID: &str = LEGACY_BACKEND;
/// Registers the Nextcloud connector.
pub struct NextcloudProvider;
impl NextcloudProvider {
/// The account a completed login flow describes.
///
/// `user_id` is the DAV path segment, which is not always the login name:
/// a login can be an email address while the user id is something else,
/// and building `/remote.php/dav/files/<login>/` from the wrong one 404s
/// every request.
pub fn account_from(creds: &AppCredentials, user_id: impl Into<String>) -> Account {
Account::new(BACKEND_ID, creds.server.trim_end_matches('/'))
.with_login(creds.login_name.clone(), user_id)
}
/// The credentials a stored account plus its secret amount to.
///
/// [`AppCredentials`] stays the connector's own type rather than becoming
/// something general: an app password, an OAuth token and a bucket key
/// pair have no useful common shape, and inventing one would produce a
/// wrong answer confidently. The general form is [`Connection`]; this is
/// the translation into what one protocol needs.
pub fn credentials(conn: &Connection) -> Result<AppCredentials, RemoteError> {
Ok(AppCredentials {
server: conn.account.endpoint.clone(),
login_name: conn.account.login.clone(),
app_password: conn.require_secret()?.expose().to_string(),
})
}
}
impl BackendProvider for NextcloudProvider {
fn id(&self) -> &'static str {
BACKEND_ID
}
fn display_name(&self) -> &'static str {
"Nextcloud"
}
fn endpoint_label(&self) -> &'static str {
"Server"
}
fn endpoint_placeholder(&self) -> &'static str {
"https://cloud.example.com"
}
fn sign_in(&self) -> SignIn {
SignIn::Browser
}
/// Normalise a server address typed by hand.
///
/// Users type `cloud.example.com`, not a URL. Assume HTTPS rather than
/// failing, and never silently accept plain HTTP — NFR-SEC-3 requires TLS,
/// and an unencrypted default would be a security decision made on the
/// user's behalf without telling them.
fn normalise_endpoint(&self, input: &str) -> Result<String, String> {
let s = input.trim().trim_end_matches('/');
if s.is_empty() {
return Err("Enter the address of your Nextcloud server.".into());
}
if s.starts_with("https://") {
Ok(s.to_string())
} else if let Some(rest) = s.strip_prefix("http://") {
// Upgrade rather than accept. If the server genuinely has no TLS
// the connection fails loudly, which is the correct outcome.
Ok(format!("https://{rest}"))
} else {
Ok(format!("https://{s}"))
}
}
fn connect(&self, conn: &Connection) -> Result<Box<dyn RemoteBackend>, RemoteError> {
let creds = Self::credentials(conn)?;
Ok(Box::new(NextcloudBackend::new(
&creds,
&conn.account.user_id,
)?))
}
}
#[cfg(test)]
mod tests {
use super::*;
fn creds() -> AppCredentials {
AppCredentials {
server: "https://cloud.example/".into(),
login_name: "duncan@example.com".into(),
app_password: "token".into(),
}
}
#[test]
fn an_address_typed_by_hand_becomes_an_https_url() {
let p = NextcloudProvider;
assert_eq!(
p.normalise_endpoint("cloud.example.com/").unwrap(),
"https://cloud.example.com"
);
// Upgraded, never accepted: NFR-SEC-3.
assert_eq!(
p.normalise_endpoint("http://cloud.example.com").unwrap(),
"https://cloud.example.com"
);
assert!(p.normalise_endpoint(" ").is_err());
}
#[test]
fn the_account_keeps_the_dav_user_id_apart_from_the_login() {
// A login can be an email address while the user id is something
// else; building the DAV path from the wrong one 404s everything.
let a = NextcloudProvider::account_from(&creds(), "duncan");
assert_eq!(a.login, "duncan@example.com");
assert_eq!(a.user_id, "duncan");
assert_eq!(a.endpoint, "https://cloud.example");
}
#[test]
fn a_nextcloud_account_keeps_its_historical_catalog_directory() {
// Frozen: this names the directory holding the catalog, the thumbnail
// shards and un-uploaded sidecars.
let a = NextcloudProvider::account_from(&creds(), "duncan");
assert_eq!(a.namespace(), "cloud-example-duncan");
}
#[test]
fn connecting_without_a_credential_is_unauthenticated_not_a_crash() {
// A cleared keyring or a revoked app password arrives here as an
// account with no secret. The caller re-runs the login flow.
let account = NextcloudProvider::account_from(&creds(), "duncan");
match NextcloudProvider.connect(&Connection::new(account, None)) {
Err(RemoteError::Unauthenticated) => {}
Err(e) => panic!("wrong error: {e:?}"),
Ok(b) => panic!("connected without a credential as {}", b.name()),
}
}
#[test]
fn a_stored_account_and_its_secret_rebuild_the_credentials() {
let account = NextcloudProvider::account_from(&creds(), "duncan");
let conn = Connection::new(account, Some(dr_sync::Secret::new("token")));
let rebuilt = NextcloudProvider::credentials(&conn).unwrap();
assert_eq!(rebuilt.server, "https://cloud.example");
assert_eq!(rebuilt.login_name, "duncan@example.com");
assert_eq!(rebuilt.app_password, "token");
}
}
core/dr-sync-nextcloud/src/session.rs
-451
View File
@@ -1,451 +0,0 @@
//! Account sessions — logging in once and staying logged in.
//!
//! Splits deliberately in two:
//!
//! - **Credentials** go to platform secure storage (FR-NC-2). Never the
//! catalog, never a file, never a log line.
//! - **Everything else** — server, login, chosen root, format filter — is
//! ordinary configuration, safe to write as plain JSON.
//!
//! That split is what lets the app show "signed in as duncan, watching
//! /PhotosRaw" before it has touched the keyring, and re-authenticate cleanly
//! if the credential has been revoked server-side.
use std::path::{Path, PathBuf};
use dr_plat::{SecretError, SecretRef, SecretStore};
use dr_sync::RemoteError;
use dr_types::{Format, FormatFilter};
use serde::{Deserialize, Serialize};
use crate::AppCredentials;
/// Where configuration is written, when the platform has told us.
///
/// Android has no `$HOME` and no XDG directories, so the guess below resolves
/// to a path the app cannot write. Nothing failed loudly: the session list went
/// to a doomed path, so credentials survived only as long as the process did and
/// backgrounding the app lost the account (ARCH §6.9 — no core API may assume a
/// filesystem path on Android).
///
/// The platform layer sets this once at startup, before any store is opened.
static DATA_DIR: std::sync::OnceLock<PathBuf> = std::sync::OnceLock::new();
/// TRACES: FR-NC-2
/// Declare the per-app directory configuration belongs in.
///
/// Call before opening any store; later calls are ignored rather than racing.
/// On Android this is `AndroidApp::internal_data_path`, which is private to the
/// app and survives being backgrounded. Desktop needs no call — the XDG
/// fallback is correct there.
pub fn set_data_dir(dir: PathBuf) {
let _ = DATA_DIR.set(dir);
}
/// The directory configuration lives in.
fn config_dir() -> PathBuf {
if let Some(d) = DATA_DIR.get() {
return d.clone();
}
std::env::var_os("XDG_CONFIG_HOME")
.map(PathBuf::from)
.unwrap_or_else(|| PathBuf::from(std::env::var("HOME").unwrap_or_default()).join(".config"))
.join("darkroom")
}
/// A configured account, minus its credential.
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
pub struct Session {
pub server: String,
pub login: String,
/// The DAV path segment, which may differ from `login` — a login can be
/// an email address while the user id is something else.
pub user_id: String,
/// The folder chosen as the library root. Empty means the account root.
#[serde(default)]
pub root: String,
/// Which formats the scan looks for (the tick-boxes).
#[serde(default)]
pub formats: Vec<String>,
/// Unix seconds of the last completed scan, for display.
#[serde(default)]
pub last_scan: Option<i64>,
}
impl Session {
pub fn new(creds: &AppCredentials, user_id: impl Into<String>) -> Self {
Self {
server: creds.server.trim_end_matches('/').to_string(),
login: creds.login_name.clone(),
user_id: user_id.into(),
root: String::new(),
formats: Vec::new(),
last_scan: None,
}
}
/// The stored format selection, defaulting to every supported format.
///
/// An unconfigured session must find everything rather than nothing.
pub fn format_filter(&self) -> FormatFilter {
if self.formats.is_empty() {
FormatFilter::all()
} else {
FormatFilter::from_formats(
self.formats
.iter()
.filter_map(|s| Format::from_extension(&s.to_ascii_lowercase())),
)
}
}
pub fn set_format_filter(&mut self, filter: &FormatFilter) {
self.formats = filter
.iter()
.map(|f| format!("{f:?}").to_lowercase())
.collect();
}
/// Where this session's credential lives.
pub fn secret_ref(&self) -> SecretRef {
SecretRef::app_password(&self.server, &self.login)
}
/// A short description for the UI.
pub fn describe(&self) -> String {
let host = self
.server
.trim_start_matches("https://")
.trim_start_matches("http://");
if self.root.is_empty() {
format!("{} on {host}", self.login)
} else {
format!("{} on {host}/{}", self.login, self.root)
}
}
}
/// TRACES: FR-NC-1 | FR-NC-2 | M-1 | M-2
/// Loads and saves sessions, keeping credentials in secure storage.
pub struct SessionStore {
config_path: PathBuf,
secrets: Box<dyn SecretStore>,
}
/// What is written to disk. Versioned so a format change is a migration
/// rather than a parse failure.
#[derive(Debug, Default, Serialize, Deserialize)]
struct ConfigFile {
#[serde(default = "one")]
version: u32,
#[serde(default)]
sessions: Vec<Session>,
}
fn one() -> u32 {
1
}
impl SessionStore {
/// Open the store at the platform config location.
///
/// Linux: `$XDG_CONFIG_HOME/darkroom/sessions.json`, falling back to
/// `~/.config` (FR-PLAT-LIN-1).
pub fn open(secrets: Box<dyn SecretStore>) -> Self {
Self::open_at(config_dir().join("sessions.json"), secrets)
}
/// Open at an explicit path — used by tests, and by anything wanting a
/// non-default config location.
/// Where configuration lives, for callers that need to sit files beside it.
pub fn data_dir() -> PathBuf {
config_dir()
}
pub fn open_at(config_path: PathBuf, secrets: Box<dyn SecretStore>) -> Self {
Self {
config_path,
secrets,
}
}
pub fn config_path(&self) -> &Path {
&self.config_path
}
/// Whether credentials can be remembered at all.
///
/// Where false the UI should say sign-in will not persist, rather than
/// letting the user discover it next launch.
pub fn can_remember(&self) -> bool {
self.secrets.is_available()
}
/// Every configured session. Missing or unreadable config yields an empty
/// list rather than an error — a first run is not a failure.
pub fn list(&self) -> Vec<Session> {
self.read_config().sessions
}
/// The most recently configured session, if any.
pub fn current(&self) -> Option<Session> {
self.read_config().sessions.into_iter().next_back()
}
/// Persist a session and its credential.
///
/// The credential goes to secure storage first: if that fails there is no
/// point recording a session that cannot authenticate.
pub fn save(&self, session: &Session, creds: &AppCredentials) -> Result<(), SessionError> {
self.secrets
.store(&session.secret_ref(), &creds.app_password)?;
let mut config = self.read_config();
config
.sessions
.retain(|s| !(s.server == session.server && s.login == session.login));
config.sessions.push(session.clone());
self.write_config(&config)
}
/// Update a session's settings, leaving its credential untouched.
pub fn update(&self, session: &Session) -> Result<(), SessionError> {
let mut config = self.read_config();
match config
.sessions
.iter_mut()
.find(|s| s.server == session.server && s.login == session.login)
{
Some(existing) => *existing = session.clone(),
None => config.sessions.push(session.clone()),
}
self.write_config(&config)
}
/// Rebuild credentials for a session from secure storage.
///
/// [`SecretError::NotFound`] means the credential was revoked or the
/// keyring was cleared — the caller re-runs the login flow.
pub fn credentials(&self, session: &Session) -> Result<AppCredentials, SessionError> {
let password = self.secrets.retrieve(&session.secret_ref())?;
Ok(AppCredentials {
server: session.server.clone(),
login_name: session.login.clone(),
app_password: password,
})
}
/// Forget a session and delete its credential.
///
/// The credential is removed even if the config write fails, so a logout
/// never leaves a usable secret behind.
pub fn forget(&self, session: &Session) -> Result<(), SessionError> {
let deleted = self.secrets.delete(&session.secret_ref());
let mut config = self.read_config();
config
.sessions
.retain(|s| !(s.server == session.server && s.login == session.login));
let written = self.write_config(&config);
deleted?;
written
}
fn read_config(&self) -> ConfigFile {
std::fs::read_to_string(&self.config_path)
.ok()
.and_then(|t| serde_json::from_str(&t).ok())
.unwrap_or_default()
}
fn write_config(&self, config: &ConfigFile) -> Result<(), SessionError> {
if let Some(parent) = self.config_path.parent() {
std::fs::create_dir_all(parent)?;
}
let json = serde_json::to_string_pretty(config)?;
// Write and rename, so an interrupted save cannot truncate an
// existing config.
let tmp = self.config_path.with_extension("tmp");
std::fs::write(&tmp, json)?;
std::fs::rename(&tmp, &self.config_path)?;
Ok(())
}
}
#[derive(Debug, thiserror::Error)]
pub enum SessionError {
#[error("secure storage: {0}")]
Secret(#[from] SecretError),
#[error("config io: {0}")]
Io(#[from] std::io::Error),
#[error("config format: {0}")]
Serde(#[from] serde_json::Error),
#[error(transparent)]
Remote(#[from] RemoteError),
}
#[cfg(test)]
mod tests {
use super::*;
use dr_plat::EphemeralSecretStore;
fn creds() -> AppCredentials {
AppCredentials {
server: "https://cloud.example/".into(),
login_name: "duncan".into(),
app_password: "secret-token".into(),
}
}
fn store_in(dir: &Path) -> SessionStore {
SessionStore::open_at(
dir.join("sessions.json"),
Box::new(EphemeralSecretStore::new()),
)
}
fn tmpdir(name: &str) -> PathBuf {
let d = std::env::temp_dir().join(format!("darkroom-test-{name}"));
let _ = std::fs::remove_dir_all(&d);
std::fs::create_dir_all(&d).unwrap();
d
}
#[test]
fn a_saved_session_survives_reopening() {
let dir = tmpdir("survives");
let secrets = Box::new(EphemeralSecretStore::new());
// Same secret store instance, as a real process would have.
let store = SessionStore::open_at(dir.join("sessions.json"), secrets);
let mut s = Session::new(&creds(), "duncan");
s.root = "PhotosRaw".into();
store.save(&s, &creds()).unwrap();
let reloaded = store.current().expect("session persisted");
assert_eq!(reloaded.login, "duncan");
assert_eq!(reloaded.root, "PhotosRaw");
// Trailing slash normalised, so URLs built from it are consistent.
assert_eq!(reloaded.server, "https://cloud.example");
}
#[test]
fn the_credential_never_reaches_the_config_file() {
// NFR-SEC-2: the whole point of the split.
let dir = tmpdir("nocreds");
let store = store_in(&dir);
let s = Session::new(&creds(), "duncan");
store.save(&s, &creds()).unwrap();
let text = std::fs::read_to_string(dir.join("sessions.json")).unwrap();
assert!(!text.contains("secret-token"), "credential leaked to disk");
assert!(text.contains("duncan"), "session metadata should be there");
}
#[test]
fn credentials_round_trip_through_secure_storage() {
let dir = tmpdir("roundtrip");
let store = store_in(&dir);
let s = Session::new(&creds(), "duncan");
store.save(&s, &creds()).unwrap();
let got = store.credentials(&s).unwrap();
assert_eq!(got.app_password, "secret-token");
assert_eq!(got.login_name, "duncan");
}
#[test]
fn forgetting_removes_both_halves() {
let dir = tmpdir("forget");
let store = store_in(&dir);
let s = Session::new(&creds(), "duncan");
store.save(&s, &creds()).unwrap();
store.forget(&s).unwrap();
assert!(store.current().is_none());
assert!(matches!(
store.credentials(&s),
Err(SessionError::Secret(SecretError::NotFound))
));
}
#[test]
fn saving_the_same_account_twice_does_not_duplicate_it() {
let dir = tmpdir("dedupe");
let store = store_in(&dir);
let mut s = Session::new(&creds(), "duncan");
store.save(&s, &creds()).unwrap();
s.root = "Photos".into();
store.save(&s, &creds()).unwrap();
assert_eq!(store.list().len(), 1);
assert_eq!(store.current().unwrap().root, "Photos");
}
#[test]
fn a_missing_config_is_a_first_run_not_an_error() {
let dir = tmpdir("firstrun");
let store = store_in(&dir);
assert!(store.list().is_empty());
assert!(store.current().is_none());
}
#[test]
fn a_corrupt_config_does_not_prevent_starting() {
// Better to present a first-run state than to refuse to launch.
let dir = tmpdir("corrupt");
std::fs::write(dir.join("sessions.json"), "{ not json").unwrap();
let store = store_in(&dir);
assert!(store.list().is_empty());
}
#[test]
fn format_selection_round_trips() {
let dir = tmpdir("formats");
let store = store_in(&dir);
let mut s = Session::new(&creds(), "duncan");
s.set_format_filter(&FormatFilter::from_formats([Format::Cr2, Format::Dng]));
store.save(&s, &creds()).unwrap();
let f = store.current().unwrap().format_filter();
assert!(f.allows(Format::Cr2));
assert!(f.allows(Format::Dng));
assert!(!f.allows(Format::Nef));
}
#[test]
fn an_unset_filter_means_every_format() {
// Never "no formats", which would silently find nothing.
let s = Session::new(&creds(), "duncan");
let f = s.format_filter();
assert!(f.allows(Format::Cr2));
assert!(f.allows(Format::Jpeg));
}
#[test]
fn describe_is_readable_and_hides_the_scheme() {
let mut s = Session::new(&creds(), "duncan");
assert_eq!(s.describe(), "duncan on cloud.example");
s.root = "PhotosRaw".into();
assert_eq!(s.describe(), "duncan on cloud.example/PhotosRaw");
}
#[test]
fn updating_settings_leaves_the_credential_alone() {
let dir = tmpdir("update");
let store = store_in(&dir);
let mut s = Session::new(&creds(), "duncan");
store.save(&s, &creds()).unwrap();
s.root = "Elsewhere".into();
store.update(&s).unwrap();
assert_eq!(store.current().unwrap().root, "Elsewhere");
assert_eq!(store.credentials(&s).unwrap().app_password, "secret-token");
}
}
core/dr-sync/Cargo.toml
+5
View File
@@ -7,7 +7,12 @@ license.workspace = true
[dependencies]
dr-types.workspace = true
# Accounts keep their credential in platform secure storage, never in the
# config file they are otherwise written to (NFR-SEC-2).
dr-plat.workspace = true
async-trait.workspace = true
serde.workspace = true
serde_json.workspace = true
thiserror.workspace = true
log.workspace = true
core/dr-sync/src/account.rs
+804
View File
@@ -0,0 +1,804 @@
// TRACES: FR-NC-12 | FR-NC-2
//! What a configured library *is*, with no connector in it.
//!
//! Before this existed, "an account" meant a Nextcloud server URL, a login
//! name and a DAV user id, and that shape reached every layer above:
//! `dr-ui` stored it, keyed its caches off it, threaded it through a dozen
//! worker threads and handed it to a constructor named after one product.
//! [`RemoteBackend`](crate::RemoteBackend) was abstract; everything that
//! *reached* a backend was not, so a second connector had nowhere to live.
//!
//! An [`Account`] is what remains once the product is taken out: somewhere a
//! library lives ([`endpoint`](Account::endpoint)), a folder inside it
//! ([`root`](Account::root)), and the settings the scan needs. What an
//! endpoint means is the connector's business — a URL for Nextcloud, a
//! directory for a plain folder, a bucket for whatever comes next.
//!
//! # The split that has to survive
//!
//! Credentials go to platform secure storage (FR-NC-2). Never the catalog,
//! never a file, never a log line. Everything else is ordinary configuration
//! written as plain JSON. That split is what lets the app show "signed in as
//! duncan, watching /PhotosRaw" before it has touched the keyring — and it is
//! why a [`Connection`] carries the two halves separately rather than as one
//! blob.
use std::path::{Path, PathBuf};
use dr_plat::{SecretError, SecretRef, SecretStore};
use dr_types::{Format, FormatFilter};
use serde::{Deserialize, Serialize};
use crate::RemoteError;
/// The connector every account had before there was a choice.
///
/// Named here, in connector-neutral code, for exactly one reason:
/// [`Account::namespace`] must keep producing the same string for these
/// accounts as the hard-coded Nextcloud version did. That string is a
/// directory name holding a catalog, thumbnail shards, un-uploaded sidecars
/// and an export outbox. Changing it does not lose that data, it *abandons*
/// it — silently, as an upgrade — and costs a full rescan of the library on
/// top.
///
/// Nothing else in this crate branches on a connector's identity, and nothing
/// else should.
pub const LEGACY_BACKEND: &str = "nextcloud";
/// Where configuration is written, when the platform has told us.
///
/// Android has no `$HOME` and no XDG directories, so the guess below resolves
/// to a path the app cannot write. Nothing failed loudly: the account list went
/// to a doomed path, so credentials survived only as long as the process did and
/// backgrounding the app lost the account (ARCH §6.9 — no core API may assume a
/// filesystem path on Android).
///
/// The platform layer sets this once at startup, before any store is opened.
static DATA_DIR: std::sync::OnceLock<PathBuf> = std::sync::OnceLock::new();
/// TRACES: FR-NC-2
/// Declare the per-app directory configuration belongs in.
///
/// Call before opening any store; later calls are ignored rather than racing.
/// On Android this is `AndroidApp::internal_data_path`, which is private to the
/// app and survives being backgrounded. Desktop needs no call — the XDG
/// fallback is correct there.
pub fn set_data_dir(dir: PathBuf) {
let _ = DATA_DIR.set(dir);
}
/// The directory configuration lives in.
pub fn config_dir() -> PathBuf {
if let Some(d) = DATA_DIR.get() {
return d.clone();
}
std::env::var_os("XDG_CONFIG_HOME")
.map(PathBuf::from)
.unwrap_or_else(|| PathBuf::from(std::env::var("HOME").unwrap_or_default()).join(".config"))
.join("darkroom")
}
/// TRACES: FR-NC-12
/// A configured library, minus its credential.
///
/// Every field but [`backend`](Self::backend) is interpreted by the connector
/// that owns it. Code above this layer reads them for display and for cache
/// keys and never for meaning.
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
pub struct Account {
/// Which connector serves this library, as
/// [`BackendProvider::id`](crate::BackendProvider::id).
///
/// Defaulted rather than required, because every account written before
/// there was a choice omits it and every one of them is a Nextcloud
/// account. A missing field here must load, not fail — a config the app
/// refuses to parse is an account the user has to set up again.
#[serde(default = "legacy_backend")]
pub backend: String,
/// Where the library lives, in whatever form the connector addresses:
/// `https://cloud.example` for Nextcloud, `/mnt/photos` for a folder.
///
/// Stored under its historical name so existing configuration loads
/// unchanged.
#[serde(rename = "server")]
pub endpoint: String,
/// Who we are, where that means anything. Empty for connectors with no
/// notion of a user — it is shown, and used to key the credential.
#[serde(default)]
pub login: String,
/// A connector-defined sub-address. Nextcloud's DAV path segment, which
/// may differ from `login` because a login can be an email address while
/// the user id is something else. Empty where the connector has no use
/// for one.
#[serde(default)]
pub user_id: String,
/// The folder chosen as the library root, relative to the endpoint. Empty
/// means the endpoint itself.
#[serde(default)]
pub root: String,
/// Which formats the scan looks for (the tick-boxes).
#[serde(default)]
pub formats: Vec<String>,
/// Unix seconds of the last completed scan, for display.
#[serde(default)]
pub last_scan: Option<i64>,
}
fn legacy_backend() -> String {
LEGACY_BACKEND.to_string()
}
impl Account {
/// A bare account for `backend` at `endpoint`, with nothing chosen yet.
pub fn new(backend: impl Into<String>, endpoint: impl Into<String>) -> Self {
Self {
backend: backend.into(),
endpoint: endpoint.into(),
login: String::new(),
user_id: String::new(),
root: String::new(),
formats: Vec::new(),
last_scan: None,
}
}
pub fn with_login(mut self, login: impl Into<String>, user_id: impl Into<String>) -> Self {
self.login = login.into();
self.user_id = user_id.into();
self
}
/// Whether two records name the same account.
///
/// The identity the store deduplicates on. Endpoint and login together,
/// because one server can hold two accounts and one machine can hold two
/// folders — but the *same* pair twice is the same library reconfigured,
/// not a second one.
pub fn is_same_as(&self, other: &Account) -> bool {
self.backend == other.backend
&& self.endpoint == other.endpoint
&& self.login == other.login
}
/// The stored format selection, defaulting to every supported format.
///
/// An unconfigured account must find everything rather than nothing.
pub fn format_filter(&self) -> FormatFilter {
if self.formats.is_empty() {
FormatFilter::all()
} else {
FormatFilter::from_formats(
self.formats
.iter()
.filter_map(|s| Format::from_extension(&s.to_ascii_lowercase())),
)
}
}
pub fn set_format_filter(&mut self, filter: &FormatFilter) {
self.formats = filter
.iter()
.map(|f| format!("{f:?}").to_lowercase())
.collect();
}
/// Where this account's credential lives, for connectors that need one.
pub fn secret_ref(&self) -> SecretRef {
SecretRef::app_password(&self.endpoint, &self.login)
}
/// A short description for the UI.
///
/// Reads for both shapes without asking the connector: "duncan on
/// cloud.example/PhotosRaw" where there is a login, and just the location
/// where there is not — a folder library has no user to name, and
/// inventing one ("(local) on /mnt/photos") would be worse than saying
/// where it is.
pub fn describe(&self) -> String {
let place = self
.endpoint
.trim_start_matches("https://")
.trim_start_matches("http://");
let place = if self.root.is_empty() {
place.to_string()
} else {
format!("{}/{}", place.trim_end_matches('/'), self.root)
};
if self.login.is_empty() {
place
} else {
format!("{} on {place}", self.login)
}
}
/// TRACES: FR-NC-10 | NFR-R1
/// The directory name this account's local data hangs off.
///
/// Not a display string and not stable across a change of endpoint: it is
/// the key for the catalog, the thumbnail shards, the sidecar spool and
/// the export outbox. Two accounts must never collide here — one would
/// index the other's library — and one account must produce the same
/// answer on every launch, forever, or its data is abandoned in place.
///
/// The Nextcloud form is reproduced byte for byte from what
/// `catalog_path` computed before accounts were multi-backend
/// ([`LEGACY_BACKEND`]). Everything else is prefixed by its connector, so
/// a folder library at `/srv/photos` and a hypothetical S3 bucket of the
/// same name cannot land in one directory.
pub fn namespace(&self) -> String {
let slug = slugify(
self.endpoint
.trim_start_matches("https://")
.trim_start_matches("http://"),
);
if self.backend == LEGACY_BACKEND {
// Frozen. See LEGACY_BACKEND.
return format!("{slug}-{}", self.user_id);
}
let tail = if self.user_id.is_empty() {
String::new()
} else {
format!("-{}", slugify(&self.user_id))
};
let name = format!("{}-{slug}{tail}", slugify(&self.backend));
shorten(&name)
}
}
/// Everything that is not `[A-Za-z0-9]`, flattened to `-`.
///
/// Not an escape and not reversible: the result names a directory, and the
/// only property it needs is that it is a legal filename on every platform
/// the app runs on.
fn slugify(s: &str) -> String {
s.chars()
.map(|c| if c.is_ascii_alphanumeric() { c } else { '-' })
.collect()
}
/// Cap a namespace at a length every filesystem accepts.
///
/// A folder endpoint is an absolute path and can be far longer than a server
/// URL — deep enough to exceed the 255-byte component limit on ext4 and APFS
/// alike, at which point creating the catalog directory fails and the library
/// cannot be opened at all. Truncating alone would make two deep paths under
/// one parent collide, so the discarded tail is replaced by a hash of the
/// whole.
fn shorten(name: &str) -> String {
const MAX: usize = 96;
if name.len() <= MAX {
return name.to_string();
}
let head: String = name.chars().take(MAX - 17).collect();
format!("{head}-{:016x}", fnv1a64(name.as_bytes()))
}
/// FNV-1a, 64-bit.
///
/// Written out rather than taken from `DefaultHasher`, whose output is
/// explicitly not stable between Rust releases. This one keys a directory that
/// must be found again after a toolchain upgrade.
fn fnv1a64(bytes: &[u8]) -> u64 {
let mut h: u64 = 0xcbf2_9ce4_8422_2325;
for b in bytes {
h ^= *b as u64;
h = h.wrapping_mul(0x0000_0100_0000_01b3);
}
h
}
/// TRACES: FR-NC-2 | NFR-SEC-2
/// A credential, kept out of logs by construction.
///
/// The inner string is reachable only through [`expose`](Secret::expose), so
/// the ways a secret leaks — a `{:?}` on a struct that happens to contain one,
/// a `Display` in an error message — do not compile into a leak. NFR-SEC-2 is
/// the requirement; this is the part of it that a reviewer cannot forget to
/// apply.
#[derive(Clone, PartialEq, Eq)]
pub struct Secret(String);
impl Secret {
pub fn new(value: impl Into<String>) -> Self {
Secret(value.into())
}
/// The credential itself. Every call site is a place to check.
pub fn expose(&self) -> &str {
&self.0
}
}
impl std::fmt::Debug for Secret {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.write_str("Secret(***)")
}
}
/// Everything needed to open a backend, in one movable value.
///
/// Workers run on their own threads and each one needs its own way in, so this
/// is `Clone` and owns what it holds. It replaced a pair of arguments —
/// credentials and a user id — that had to be threaded together through
/// fifteen functions and could be passed in the wrong order.
#[derive(Debug, Clone)]
pub struct Connection {
pub account: Account,
/// `None` where the connector needs no credential, which is the ordinary
/// state of a folder library rather than a failure to load one.
pub secret: Option<Secret>,
}
impl Connection {
pub fn new(account: Account, secret: Option<Secret>) -> Self {
Self { account, secret }
}
/// The credential, or [`RemoteError::Unauthenticated`].
///
/// For connectors that require one: turning the absence into the error the
/// caller already handles saves every implementation writing the same
/// `ok_or`.
pub fn require_secret(&self) -> Result<&Secret, RemoteError> {
self.secret.as_ref().ok_or(RemoteError::Unauthenticated)
}
}
/// TRACES: FR-NC-1 | FR-NC-2 | M-1 | M-2
/// Loads and saves accounts, keeping credentials in secure storage.
pub struct AccountStore {
config_path: PathBuf,
secrets: Box<dyn SecretStore>,
}
/// What is written to disk. Versioned so a format change is a migration
/// rather than a parse failure.
#[derive(Debug, Default, Serialize, Deserialize)]
struct ConfigFile {
#[serde(default = "one")]
version: u32,
/// Named `sessions` on disk because that is what it has always been
/// called there, and renaming the key would orphan every existing config.
#[serde(default)]
sessions: Vec<Account>,
}
fn one() -> u32 {
1
}
impl AccountStore {
/// Open the store at the platform config location.
///
/// Linux: `$XDG_CONFIG_HOME/darkroom/sessions.json`, falling back to
/// `~/.config` (FR-PLAT-LIN-1).
pub fn open(secrets: Box<dyn SecretStore>) -> Self {
Self::open_at(config_dir().join("sessions.json"), secrets)
}
/// Where configuration lives, for callers that need to sit files beside it.
pub fn data_dir() -> PathBuf {
config_dir()
}
/// Open at an explicit path — used by tests, and by anything wanting a
/// non-default config location.
pub fn open_at(config_path: PathBuf, secrets: Box<dyn SecretStore>) -> Self {
Self {
config_path,
secrets,
}
}
pub fn config_path(&self) -> &Path {
&self.config_path
}
/// Whether credentials can be remembered at all.
///
/// Where false the UI should say sign-in will not persist, rather than
/// letting the user discover it next launch.
pub fn can_remember(&self) -> bool {
self.secrets.is_available()
}
/// Every configured account. Missing or unreadable config yields an empty
/// list rather than an error — a first run is not a failure.
pub fn list(&self) -> Vec<Account> {
self.read_config().sessions
}
/// The most recently configured account, if any.
pub fn current(&self) -> Option<Account> {
self.read_config().sessions.into_iter().next_back()
}
/// Persist an account and its credential.
///
/// The credential goes to secure storage first: if that fails there is no
/// point recording an account that cannot authenticate. `None` is the
/// ordinary case for a connector that needs no credential, and stores
/// nothing rather than an empty secret.
pub fn save(&self, account: &Account, secret: Option<&Secret>) -> Result<(), AccountError> {
if let Some(s) = secret {
self.secrets.store(&account.secret_ref(), s.expose())?;
}
let mut config = self.read_config();
config.sessions.retain(|a| !a.is_same_as(account));
config.sessions.push(account.clone());
self.write_config(&config)
}
/// Update an account's settings, leaving its credential untouched.
pub fn update(&self, account: &Account) -> Result<(), AccountError> {
let mut config = self.read_config();
match config.sessions.iter_mut().find(|a| a.is_same_as(account)) {
Some(existing) => *existing = account.clone(),
None => config.sessions.push(account.clone()),
}
self.write_config(&config)
}
/// Rebuild a connection for an account, fetching its credential.
///
/// `needs_secret` is the connector's answer, passed in rather than
/// inferred: an account with an empty login might be a folder library or
/// might be a broken Nextcloud record, and guessing turns the second into
/// a silent unauthenticated connection instead of an error the user can
/// act on.
///
/// [`SecretError::NotFound`] means the credential was revoked or the
/// keyring was cleared — the caller re-runs the sign-in.
pub fn connection(
&self,
account: &Account,
needs_secret: bool,
) -> Result<Connection, AccountError> {
let secret = if needs_secret {
Some(Secret::new(self.secrets.retrieve(&account.secret_ref())?))
} else {
None
};
Ok(Connection::new(account.clone(), secret))
}
/// Forget an account and delete its credential.
///
/// The credential is removed even if the config write fails, so a logout
/// never leaves a usable secret behind. A connector that stores none
/// reports [`SecretError::NotFound`], which is not a failure to forget.
pub fn forget(&self, account: &Account) -> Result<(), AccountError> {
let deleted = match self.secrets.delete(&account.secret_ref()) {
Err(SecretError::NotFound) => Ok(()),
other => other,
};
let mut config = self.read_config();
config.sessions.retain(|a| !a.is_same_as(account));
let written = self.write_config(&config);
deleted?;
written
}
fn read_config(&self) -> ConfigFile {
std::fs::read_to_string(&self.config_path)
.ok()
.and_then(|t| serde_json::from_str(&t).ok())
.unwrap_or_default()
}
fn write_config(&self, config: &ConfigFile) -> Result<(), AccountError> {
if let Some(parent) = self.config_path.parent() {
std::fs::create_dir_all(parent)?;
}
let json = serde_json::to_string_pretty(config)?;
// Write and rename, so an interrupted save cannot truncate an
// existing config.
let tmp = self.config_path.with_extension("tmp");
std::fs::write(&tmp, json)?;
std::fs::rename(&tmp, &self.config_path)?;
Ok(())
}
}
#[derive(Debug, thiserror::Error)]
pub enum AccountError {
#[error("secure storage: {0}")]
Secret(#[from] SecretError),
#[error("config io: {0}")]
Io(#[from] std::io::Error),
#[error("config format: {0}")]
Serde(#[from] serde_json::Error),
#[error(transparent)]
Remote(#[from] RemoteError),
}
#[cfg(test)]
mod tests {
use super::*;
use dr_plat::EphemeralSecretStore;
fn nextcloud() -> Account {
Account::new(LEGACY_BACKEND, "https://cloud.example").with_login("duncan", "duncan")
}
fn folder() -> Account {
Account::new("folder", "/mnt/photos")
}
fn store_in(dir: &Path) -> AccountStore {
AccountStore::open_at(
dir.join("sessions.json"),
Box::new(EphemeralSecretStore::new()),
)
}
fn tmpdir(name: &str) -> PathBuf {
let d = std::env::temp_dir().join(format!("darkroom-account-test-{name}"));
let _ = std::fs::remove_dir_all(&d);
std::fs::create_dir_all(&d).unwrap();
d
}
#[test]
fn a_saved_account_survives_reopening() {
let dir = tmpdir("survives");
let store = store_in(&dir);
let mut a = nextcloud();
a.root = "PhotosRaw".into();
store.save(&a, Some(&Secret::new("token"))).unwrap();
let reloaded = store.current().expect("account persisted");
assert_eq!(reloaded.login, "duncan");
assert_eq!(reloaded.root, "PhotosRaw");
}
#[test]
fn the_credential_never_reaches_the_config_file() {
// NFR-SEC-2: the whole point of the split.
let dir = tmpdir("nocreds");
let store = store_in(&dir);
store
.save(&nextcloud(), Some(&Secret::new("secret-token")))
.unwrap();
let text = std::fs::read_to_string(dir.join("sessions.json")).unwrap();
assert!(!text.contains("secret-token"), "credential leaked to disk");
assert!(text.contains("duncan"), "account metadata should be there");
}
#[test]
fn a_secret_does_not_print_itself() {
// The leak this closes is indirect: a `{:?}` on any struct holding a
// connection used to print the app password.
let c = Connection::new(nextcloud(), Some(Secret::new("hunter2")));
let printed = format!("{c:?}");
assert!(!printed.contains("hunter2"), "credential leaked to a log");
}
#[test]
fn credentials_round_trip_through_secure_storage() {
let dir = tmpdir("roundtrip");
let store = store_in(&dir);
let a = nextcloud();
store.save(&a, Some(&Secret::new("secret-token"))).unwrap();
let conn = store.connection(&a, true).unwrap();
assert_eq!(conn.require_secret().unwrap().expose(), "secret-token");
}
#[test]
fn a_credentialless_account_connects_without_touching_the_keyring() {
// A folder library must open on a machine with no secrets daemon at
// all — asking for a credential it does not have would fail the one
// backend that needs nothing.
let dir = tmpdir("nosecret");
let store = store_in(&dir);
let a = folder();
store.save(&a, None).unwrap();
let conn = store.connection(&a, false).unwrap();
assert!(conn.secret.is_none());
assert!(matches!(
conn.require_secret(),
Err(RemoteError::Unauthenticated)
));
}
#[test]
fn forgetting_removes_both_halves() {
let dir = tmpdir("forget");
let store = store_in(&dir);
let a = nextcloud();
store.save(&a, Some(&Secret::new("token"))).unwrap();
store.forget(&a).unwrap();
assert!(store.current().is_none());
assert!(matches!(
store.connection(&a, true),
Err(AccountError::Secret(SecretError::NotFound))
));
}
#[test]
fn forgetting_a_credentialless_account_is_not_an_error() {
// There is no secret to delete, and reporting the absence as a failure
// would leave a folder library that cannot be signed out of.
let dir = tmpdir("forget-folder");
let store = store_in(&dir);
let a = folder();
store.save(&a, None).unwrap();
store.forget(&a).unwrap();
assert!(store.current().is_none());
}
#[test]
fn two_backends_at_the_same_endpoint_are_two_accounts() {
let dir = tmpdir("twobackends");
let store = store_in(&dir);
store.save(&Account::new("folder", "/mnt/p"), None).unwrap();
store.save(&Account::new("webdav", "/mnt/p"), None).unwrap();
assert_eq!(store.list().len(), 2);
}
#[test]
fn saving_the_same_account_twice_does_not_duplicate_it() {
let dir = tmpdir("dedupe");
let store = store_in(&dir);
let mut a = nextcloud();
store.save(&a, Some(&Secret::new("token"))).unwrap();
a.root = "Photos".into();
store.save(&a, Some(&Secret::new("token"))).unwrap();
assert_eq!(store.list().len(), 1);
assert_eq!(store.current().unwrap().root, "Photos");
}
#[test]
fn a_missing_config_is_a_first_run_not_an_error() {
let dir = tmpdir("firstrun");
let store = store_in(&dir);
assert!(store.list().is_empty());
assert!(store.current().is_none());
}
#[test]
fn a_corrupt_config_does_not_prevent_starting() {
// Better to present a first-run state than to refuse to launch.
let dir = tmpdir("corrupt");
std::fs::write(dir.join("sessions.json"), "{ not json").unwrap();
let store = store_in(&dir);
assert!(store.list().is_empty());
}
#[test]
fn a_config_written_before_backends_existed_still_loads() {
// The upgrade path. Every account written by an earlier version omits
// `backend`, and refusing to parse one would make an upgrade look
// like a signed-out app with a library that has to be set up again.
let dir = tmpdir("legacy");
std::fs::write(
dir.join("sessions.json"),
r#"{"version":1,"sessions":[{"server":"https://cloud.example",
"login":"duncan","user_id":"duncan","root":"PhotosRaw",
"formats":[],"last_scan":null}]}"#,
)
.unwrap();
let a = store_in(&dir).current().expect("legacy account loads");
assert_eq!(a.backend, LEGACY_BACKEND);
assert_eq!(a.endpoint, "https://cloud.example");
assert_eq!(a.root, "PhotosRaw");
}
#[test]
fn a_legacy_account_keeps_the_directory_its_data_is_already_in() {
// Frozen deliberately: this string names the directory holding the
// catalog, the thumbnail shards and un-uploaded sidecars. A change
// here abandons all three and forces a full rescan.
let a = Account::new(LEGACY_BACKEND, "https://cloud.example.com").with_login("d", "duncan");
assert_eq!(a.namespace(), "cloud-example-com-duncan");
}
#[test]
fn a_new_backend_cannot_collide_with_a_legacy_one() {
let ns = Account::new("folder", "/mnt/photos").namespace();
assert!(ns.starts_with("folder-"), "{ns}");
assert_ne!(ns, Account::new(LEGACY_BACKEND, "/mnt/photos").namespace());
}
#[test]
fn two_folders_never_share_a_directory() {
// Two libraries in one catalog would index each other's images.
assert_ne!(
Account::new("folder", "/mnt/photos/2025").namespace(),
Account::new("folder", "/mnt/photos/2026").namespace()
);
}
#[test]
fn a_very_deep_folder_still_yields_a_legal_directory_name() {
// Past 255 bytes the catalog directory cannot be created at all, and
// the library simply fails to open.
let deep = format!("/{}", vec!["a-rather-long-folder-name"; 40].join("/"));
let a = Account::new("folder", &deep);
let ns = a.namespace();
assert!(ns.len() <= 96, "{} chars", ns.len());
// Truncation alone would make these two the same directory.
let b = Account::new("folder", format!("{deep}/second"));
assert_ne!(ns, b.namespace());
}
#[test]
fn describe_reads_for_an_account_with_no_user() {
// A folder library has nobody to name; "(none) on /mnt/photos" would
// be worse than saying where it is.
let mut a = folder();
assert_eq!(a.describe(), "/mnt/photos");
a.root = "2026".into();
assert_eq!(a.describe(), "/mnt/photos/2026");
}
#[test]
fn describe_is_readable_and_hides_the_scheme() {
let mut a = nextcloud();
assert_eq!(a.describe(), "duncan on cloud.example");
a.root = "PhotosRaw".into();
assert_eq!(a.describe(), "duncan on cloud.example/PhotosRaw");
}
#[test]
fn format_selection_round_trips() {
let mut a = nextcloud();
a.set_format_filter(&FormatFilter::from_formats([Format::Cr2, Format::Dng]));
let f = a.format_filter();
assert!(f.allows(Format::Cr2));
assert!(f.allows(Format::Dng));
assert!(!f.allows(Format::Nef));
}
#[test]
fn an_unset_filter_means_every_format() {
// Never "no formats", which would silently find nothing.
let f = nextcloud().format_filter();
assert!(f.allows(Format::Cr2));
assert!(f.allows(Format::Jpeg));
}
#[test]
fn updating_settings_leaves_the_credential_alone() {
let dir = tmpdir("update");
let store = store_in(&dir);
let mut a = nextcloud();
store.save(&a, Some(&Secret::new("secret-token"))).unwrap();
a.root = "Elsewhere".into();
store.update(&a).unwrap();
assert_eq!(store.current().unwrap().root, "Elsewhere");
assert_eq!(
store
.connection(&a, true)
.unwrap()
.require_secret()
.unwrap()
.expose(),
"secret-token"
);
}
}
core/dr-sync/src/capability.rs
+49
View File
@@ -38,6 +38,50 @@ pub struct ChunkConstraints {
pub max_chunks: u32,
}
/// TRACES: FR-NC-6c
/// Whether every listed object's content is actually reachable.
///
/// Every backend but a virtual-filesystem folder answers [`Always`](Self::Always).
/// A VFS folder is the case this exists for: the sync client leaves a
/// placeholder where a file is catalogued but not downloaded, so the name is
/// listable and the bytes are not (ARCH §9.0).
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum Materialisation {
/// Listing an object means its content can be read. Every server backend,
/// and a plain directory.
Always,
/// Some objects are placeholders, and nothing this process can do will
/// change that — the sync client is not running, or the platform offers no
/// way to ask. Such an object reads as
/// [`RemoteError::NotMaterialised`](crate::RemoteError::NotMaterialised)
/// and is shown as offline rather than broken.
Placeholders,
/// Some objects are placeholders, and this backend can ask for their
/// content — and give it back.
///
/// **Whole-file, and that is the whole difficulty.** Hydration has two
/// states, one byte or all bytes, so using it to fill a grid transfers the
/// entire library to produce thumbnails (ARCH §9.0). It belongs to the
/// originals tier — an image opened in develop, exported, or deliberately
/// pinned — and to passes the user has asked for and been quoted a price
/// on. Never to browsing.
OnDemand,
}
impl Materialisation {
/// Whether content can be fetched on request.
pub fn can_materialise(self) -> bool {
matches!(self, Materialisation::OnDemand)
}
/// Whether some objects may have no content locally.
pub fn has_placeholders(self) -> bool {
!matches!(self, Materialisation::Always)
}
}
/// TRACES: FR-NC-3
/// Whether the server can render thumbnails, and for what.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
@@ -69,6 +113,8 @@ pub struct Capabilities {
/// Conditional write (If-Match), for conflict-safe sidecar updates.
pub conditional_write: bool,
pub server_previews: ServerPreviews,
/// Whether a listed object's content is necessarily present.
pub materialisation: Materialisation,
}
impl Capabilities {
@@ -83,6 +129,9 @@ impl Capabilities {
bulk_upload: false,
conditional_write: false,
server_previews: ServerPreviews::None,
// The weakest backend still answers for everything it lists;
// placeholders are a property a backend opts into.
materialisation: Materialisation::Always,
}
}
core/dr-sync/src/error.rs
+31
View File
@@ -36,11 +36,42 @@ pub enum RemoteError {
#[error("not found: {0}")]
NotFound(String),
/// TRACES: FR-NC-6c
/// The object exists, but its content is not on this device.
///
/// A virtual-filesystem placeholder: the sync client holds the name and a
/// stub, and the bytes are still on the server (ARCH §9.0).
///
/// **Emphatically not [`NotFound`](Self::NotFound), and the distinction is
/// what stops a silent data loss.** The sidecar writer reads before it
/// writes, and treats a miss as "there is no sidecar yet, create one" — so
/// a dehydrated sidecar reported as absent makes it write a fresh document
/// over an existing one, discarding every edit another device had put
/// there. It is also the difference between an error a user can act on
/// (fetch it) and one they cannot (it is gone).
#[error("not on this device: {0}")]
NotMaterialised(String),
/// The backend does not support this operation. Expected, not a bug —
/// callers check capabilities and adapt.
#[error("operation unsupported by this backend: {0}")]
Unsupported(&'static str),
/// The account is configured wrongly, or for a backend this build has no
/// connector for.
///
/// **Not a network failure and not an auth failure**, which is why it is
/// its own variant. A folder library whose directory has been unmounted,
/// or an account naming a backend a cut-down build was not compiled with,
/// produces a request that never leaves the process — reporting either as
/// `Network` would put the app into offline mode and tell the user their
/// connection is down, and reporting them as `AuthFailed` would send them
/// to re-enter a credential that is fine. The message names what is wrong
/// with the configuration, because that is the only thing that will fix
/// it.
#[error("account misconfigured: {0}")]
Configuration(String),
/// A conditional write failed: the remote changed underneath us. Triggers
/// the sidecar merge path (ARCH §8.5).
#[error("precondition failed — remote was modified")]
core/dr-sync/src/lib.rs
+65 -4
View File
@@ -1,9 +1,14 @@
//! Pluggable remote storage for DarkRoom.
//!
//! Defines the [`RemoteBackend`] trait and the capability model the sync
//! engine adapts to. Only the Nextcloud connector is implemented
//! (`dr-sync-nextcloud`), but the boundary is designed so other backends can
//! be added without touching the engine.
//! engine adapts to, plus the pieces that let the application hold a backend
//! without naming one: an [`Account`] that is configuration rather than a
//! server, and a [`BackendProvider`] registry that turns one into a live
//! connection.
//!
//! Two connectors ship: `dr-sync-nextcloud` and `dr-sync-folder`. Adding a
//! third is implementing those two traits and registering the result — see
//! [`provider`] for the whole contract.
//!
//! # Why capabilities rather than a common denominator
//!
@@ -20,15 +25,21 @@ use std::ops::Range;
use async_trait::async_trait;
pub mod account;
pub mod capability;
pub mod error;
pub mod provider;
pub mod reachability;
pub mod scan;
pub mod types;
pub mod upload;
pub use capability::{Capabilities, ChangeDetection, ChunkConstraints, ServerPreviews};
pub use account::{Account, AccountError, AccountStore, Connection, Secret, LEGACY_BACKEND};
pub use capability::{
Capabilities, ChangeDetection, ChunkConstraints, Materialisation, ServerPreviews,
};
pub use error::RemoteError;
pub use provider::{BackendProvider, BackendRegistry, SignIn};
pub use reachability::{Connectivity, Reachability};
pub use scan::{scan, ScanProgress, ScanResult};
pub use types::{
@@ -142,6 +153,56 @@ pub trait RemoteBackend: Send + Sync {
/// destination, not to claim they created it.
async fn create_dir(&self, path: &RemotePath) -> Result<(), RemoteError>;
// ---- materialisation --------------------------------------------------
/// TRACES: FR-NC-6c
/// Ask for a placeholder's content to be brought to this device.
///
/// Only meaningful where [`Capabilities::materialisation`] is
/// [`Materialisation::OnDemand`]; others return
/// [`RemoteError::Unsupported`].
///
/// **Whole-file, and slow.** There is no partial hydration: a placeholder
/// becomes one byte or all of them, so this transfers a 27 MB RAW to
/// answer a question a 256 KB range read would have answered (ARCH §9.0
/// finding 3). It is for the originals tier — develop, export, a pin the
/// user asked for — and for passes the user has been quoted a price on and
/// agreed to. **Never for filling a grid**: doing so downloads the entire
/// library to produce thumbnails.
///
/// Returns once the content is readable, and **whether this call is what
/// brought it here** — `false` meaning it was already local.
///
/// That boolean is the whole basis of borrowing. A caller releasing what
/// it fetched must not release what the user already had, and after the
/// fact the two are indistinguishable; the backend knows because it had to
/// look before deciding whether to ask. Answering it here costs the `stat`
/// the implementation performs anyway, where a caller determining it
/// separately would pay a directory listing per file.
async fn materialise(&self, _id: &RemoteId) -> Result<bool, RemoteError> {
Err(RemoteError::Unsupported(
"this backend has no placeholders to materialise",
))
}
/// Give a placeholder's content back, freeing the disk it held.
///
/// The counterpart that makes hydration a *borrow* rather than an
/// acquisition: a pass that hydrates a library to index it can return each
/// file as it finishes, so peak disk is the working set rather than the
/// library.
///
/// **Never destructive.** On a synced folder this asks the client to
/// dehydrate; it must not delete, because a deletion in a synced tree
/// propagates to the server and removes the photograph everywhere. An
/// implementation that cannot dehydrate must return
/// [`RemoteError::Unsupported`] rather than approximating it.
async fn dematerialise(&self, _id: &RemoteId) -> Result<(), RemoteError> {
Err(RemoteError::Unsupported(
"this backend has no placeholders to release",
))
}
// ---- optional ---------------------------------------------------------
/// Server-rendered thumbnail, where available.
core/dr-sync/src/provider.rs
+278
View File
@@ -0,0 +1,278 @@
// TRACES: FR-NC-12
//! How a connector announces itself.
//!
//! [`RemoteBackend`] says what a backend can *do* once it is open.
//! [`BackendProvider`] says everything the application needs before that: what
//! to call it, what a library location looks like, whether signing in involves
//! a browser, and how to turn a stored [`Account`] into a live backend.
//!
//! Together they are the whole contract. Adding a storage layer is:
//!
//! 1. implement [`RemoteBackend`] over your protocol,
//! 2. implement [`BackendProvider`] beside it,
//! 3. register it in `dr_ui::remote`.
//!
//! Nothing above that module names a connector, so nothing above it changes.
//!
//! # Why sign-in is a shape rather than a method
//!
//! It would be tidier for a provider to expose `async fn sign_in()` and let
//! the launch screen await it. It would also be wrong: Nextcloud's Login Flow
//! v2 is a browser handshake the user completes elsewhere while the app polls,
//! so it is not one call, it does not finish on our schedule, and the screen
//! has to render a URL and a waiting state in the middle of it. A folder needs
//! none of that. [`SignIn`] names which of those two shapes the screen must
//! draw, and the flow itself stays where its protocol is.
use std::sync::Arc;
use crate::{Account, Connection, RemoteBackend, RemoteError};
/// What establishing an account involves.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum SignIn {
/// A handshake the user completes outside the app, yielding a credential
/// the app then stores. Nextcloud's Login Flow v2.
///
/// The connector drives it; the launch screen only shows the waiting
/// state, because what happens in the middle is protocol-specific.
Browser,
/// The endpoint is the whole account. Nothing to authenticate, nothing to
/// store in the keyring, no waiting state to draw — a local folder.
EndpointOnly,
}
impl SignIn {
/// Whether an account of this shape has a credential in secure storage.
pub fn needs_secret(self) -> bool {
matches!(self, SignIn::Browser)
}
}
/// TRACES: FR-NC-12
/// A storage connector, described well enough to configure without naming it.
///
/// Implementations are held in an [`Arc`] inside a [`BackendRegistry`] and
/// must be usable from any thread: the launch screen reads them on the UI
/// thread and workers open connections from them on their own.
pub trait BackendProvider: Send + Sync {
/// The stable identifier written to [`Account::backend`].
///
/// **It is on-disk configuration.** Changing it after anyone has an
/// account orphans that account, so pick it once.
fn id(&self) -> &'static str;
/// What to call this in the interface. "Nextcloud", "Folder".
fn display_name(&self) -> &'static str;
/// What to label the endpoint field: "Server address", "Folder".
fn endpoint_label(&self) -> &'static str;
/// An example endpoint, for the empty field.
fn endpoint_placeholder(&self) -> &'static str;
/// How an account of this kind is established.
fn sign_in(&self) -> SignIn;
/// Turn what the user typed into the form that gets stored.
///
/// Two jobs, and the second is the important one: this is where a bad
/// endpoint is *rejected*, before an account is written for a library that
/// does not exist. The error is shown to the user, so it says what is
/// wrong rather than naming a type.
fn normalise_endpoint(&self, input: &str) -> Result<String, String>;
/// Build an account from a normalised endpoint alone.
///
/// Only meaningful for [`SignIn::EndpointOnly`]; a browser flow produces
/// its account from what the handshake returned, so the default here
/// refuses rather than inventing one.
fn account_for(&self, endpoint: &str) -> Result<Account, RemoteError> {
let _ = endpoint;
Err(RemoteError::Unsupported(
"this backend establishes an account through its sign-in flow",
))
}
/// Open a live backend.
///
/// Cheap and synchronous: it validates configuration and constructs a
/// client, and does not talk to the remote. Workers call it per task, so
/// anything expensive here is paid over and over.
fn connect(&self, conn: &Connection) -> Result<Box<dyn RemoteBackend>, RemoteError>;
}
/// TRACES: FR-NC-12 | FR-NC-13
/// The connectors this build has.
///
/// One instance is built at startup and consulted by everything that needs a
/// backend. The registry is the *only* thing that knows connectors exist,
/// which is what keeps the layers above free of them.
#[derive(Clone, Default)]
pub struct BackendRegistry {
providers: Vec<Arc<dyn BackendProvider>>,
}
impl BackendRegistry {
pub fn new() -> Self {
Self::default()
}
/// Add a connector.
///
/// Later registrations of an id replace earlier ones, so a build can
/// substitute a connector — a test double for a real server — without the
/// registry needing to know it happened.
pub fn register(&mut self, provider: Arc<dyn BackendProvider>) -> &mut Self {
let id = provider.id();
self.providers.retain(|p| p.id() != id);
self.providers.push(provider);
self
}
/// The connector for an id.
pub fn get(&self, id: &str) -> Option<&Arc<dyn BackendProvider>> {
self.providers.iter().find(|p| p.id() == id)
}
/// The connector an account names, or a message naming the account's.
///
/// The error case is real rather than defensive: a configuration file can
/// outlive the build that wrote it, and a user moving between a full
/// desktop build and a cut-down one will have accounts this binary cannot
/// serve. Saying which backend is missing is the difference between that
/// and "could not open library".
pub fn for_account(&self, account: &Account) -> Result<&Arc<dyn BackendProvider>, RemoteError> {
self.get(&account.backend).ok_or_else(|| {
RemoteError::Configuration(format!(
"no storage backend named {:?} in this build",
account.backend
))
})
}
/// Open the backend an account is configured for.
pub fn connect(&self, conn: &Connection) -> Result<Box<dyn RemoteBackend>, RemoteError> {
self.for_account(&conn.account)?.connect(conn)
}
/// Every connector, in registration order. What the launch screen offers.
pub fn providers(&self) -> &[Arc<dyn BackendProvider>] {
&self.providers
}
}
impl std::fmt::Debug for BackendRegistry {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.debug_struct("BackendRegistry")
.field(
"providers",
&self.providers.iter().map(|p| p.id()).collect::<Vec<_>>(),
)
.finish()
}
}
#[cfg(test)]
mod tests {
use super::*;
struct Stub(&'static str);
impl BackendProvider for Stub {
fn id(&self) -> &'static str {
self.0
}
fn display_name(&self) -> &'static str {
"Stub"
}
fn endpoint_label(&self) -> &'static str {
"Where"
}
fn endpoint_placeholder(&self) -> &'static str {
"somewhere"
}
fn sign_in(&self) -> SignIn {
SignIn::EndpointOnly
}
fn normalise_endpoint(&self, input: &str) -> Result<String, String> {
if input.trim().is_empty() {
Err("say where the library is".into())
} else {
Ok(input.trim().to_string())
}
}
fn connect(&self, _conn: &Connection) -> Result<Box<dyn RemoteBackend>, RemoteError> {
Err(RemoteError::Unsupported("stub"))
}
}
fn registry() -> BackendRegistry {
let mut r = BackendRegistry::new();
r.register(Arc::new(Stub("alpha")));
r.register(Arc::new(Stub("beta")));
r
}
#[test]
fn a_registered_backend_is_found_by_id() {
assert_eq!(registry().get("beta").map(|p| p.id()), Some("beta"));
}
#[test]
fn registering_an_id_twice_replaces_rather_than_shadows() {
let mut r = registry();
r.register(Arc::new(Stub("alpha")));
assert_eq!(r.providers().len(), 2, "{r:?}");
}
#[test]
fn an_account_for_a_missing_backend_says_which_one() {
// A config can outlive the build that wrote it. "could not open
// library" would send the user to check their server.
let account = Account::new("s3", "bucket");
let err = match registry().for_account(&account) {
Err(e) => e.to_string(),
Ok(p) => panic!("a backend this build has no connector for: {}", p.id()),
};
assert!(err.contains("s3"), "{err}");
}
#[test]
fn an_endpoint_only_backend_needs_no_credential() {
assert!(!SignIn::EndpointOnly.needs_secret());
assert!(SignIn::Browser.needs_secret());
}
#[test]
fn a_browser_backend_refuses_to_invent_an_account() {
// Building one from an endpoint would skip the handshake and store an
// account with no credential, which fails later and further away.
struct Interactive;
impl BackendProvider for Interactive {
fn id(&self) -> &'static str {
"i"
}
fn display_name(&self) -> &'static str {
"I"
}
fn endpoint_label(&self) -> &'static str {
"Server"
}
fn endpoint_placeholder(&self) -> &'static str {
""
}
fn sign_in(&self) -> SignIn {
SignIn::Browser
}
fn normalise_endpoint(&self, i: &str) -> Result<String, String> {
Ok(i.into())
}
fn connect(&self, _: &Connection) -> Result<Box<dyn RemoteBackend>, RemoteError> {
Err(RemoteError::Unsupported("stub"))
}
}
assert!(Interactive.account_for("https://x").is_err());
}
}
core/dr-sync/src/scan.rs
+3
View File
@@ -253,6 +253,7 @@ mod tests {
size: 0,
modified: None,
has_preview: false,
materialised: true,
}
}
@@ -265,6 +266,7 @@ mod tests {
size: 1000,
modified: None,
has_preview: false,
materialised: true,
}
}
@@ -301,6 +303,7 @@ mod tests {
bulk_upload: false,
conditional_write: true,
server_previews: ServerPreviews::None,
materialisation: crate::Materialisation::Always,
},
lists: RefCell::new(0),
probes: RefCell::new(0),
core/dr-sync/src/types.rs
+17
View File
@@ -97,6 +97,23 @@ pub struct RemoteEntry {
/// Whether the server claims a renderable preview exists. Advisory: stock
/// Nextcloud reports none for RAW (ARCH §6.7).
pub has_preview: bool,
/// TRACES: FR-NC-6c
/// Whether [`get`](crate::RemoteBackend::get) can produce this object's
/// content right now.
///
/// True for everything a server backend lists — the bytes are remote, but
/// they are reachable. False only for a virtual-filesystem placeholder,
/// where the name is on this device and the content is not (ARCH §9.0).
///
/// The catalog maps this to [`Availability::Offline`](dr_types::Availability),
/// which is the difference between a photograph shown as *not downloaded*
/// and one shown as broken.
///
/// **`size` is not meaningful when this is false.** A Linux suffix-mode
/// stub is one byte and carries no record of what it stands for, so there
/// is nothing to report but zero.
pub materialised: bool,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
core/dr-sync/src/upload.rs
+1
View File
@@ -237,6 +237,7 @@ mod tests {
size: *size,
modified: None,
has_preview: false,
materialised: true,
})
.collect())
}
core/dr-types/src/settings.rs
+11 -6
View File
@@ -407,10 +407,9 @@ impl Default for ExportSettings {
/// [`create_dir`](../../dr_sync/trait.RemoteBackend.html) and behaves
/// identically on both platforms.
///
/// It is also where the photographs already are. A library that lives on
/// Nextcloud and exports to a phone's local storage has put the output
/// somewhere the user's other devices cannot see, which is rarely what was
/// meant.
/// It is also where the photographs already are. A library that lives on a
/// server and exports to a phone's local storage has put the output somewhere
/// the user's other devices cannot see, which is rarely what was meant.
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
#[serde(rename_all = "snake_case")]
pub enum ExportTarget {
@@ -425,7 +424,11 @@ pub enum ExportTarget {
/// destination was filled in. It is excluded from [`Self::available`]
/// rather than offered and then failing.
Device,
/// A folder on the connected account, created if absent.
/// A folder in the connected library, created if absent.
///
/// Named for where it goes rather than for what is behind it: the library
/// may be a Nextcloud account or a folder on a mount, and the export
/// behaves identically either way.
Remote,
}
@@ -466,7 +469,9 @@ impl ExportTarget {
pub fn label(self) -> &'static str {
match self {
Self::Device => "This device",
Self::Remote => "Nextcloud",
// Not the connector's name: the library may be a server or a
// folder, and the setting means the same thing for both.
Self::Remote => "The library",
}
}