diff --git a/core/dr-sync-folder/examples/vfs_cycle.rs b/core/dr-sync-folder/examples/vfs_cycle.rs new file mode 100644 index 0000000..29c1e31 --- /dev/null +++ b/core/dr-sync-folder/examples/vfs_cycle.rs @@ -0,0 +1,142 @@ +//! A placeholder library, borrowed and given back. +//! +//! ```text +//! cargo run -p dr-sync-folder --example vfs_cycle +//! ``` +//! +//! Builds a tree in the system temp directory shaped like a suffix-mode VFS +//! folder, runs the real engine over it, and reports what the borrow cost. +//! Touches nothing outside its own scratch directory. +use std::borrow::Cow; +use std::collections::HashMap; +use std::path::{Path, PathBuf}; +use std::sync::Arc; + +use dr_sync::{RemoteBackend, RemoteError, RemoteId, RemotePath}; +use dr_sync_folder::{BorrowPool, FolderBackend, Vfs}; +use dr_types::FormatFilter; + +/// Stands in for the sync client, renaming exactly as suffix mode does. +struct Client; + +impl Vfs for Client { + fn name(&self) -> &'static str { + "demo" + } + fn is_placeholder(&self, on_disk: &str) -> bool { + on_disk.ends_with(".stub") + } + fn real_name<'a>(&self, on_disk: &'a str) -> &'a str { + on_disk.strip_suffix(".stub").unwrap_or(on_disk) + } + fn placeholder_name(&self, name: &str) -> Cow<'_, str> { + Cow::Owned(format!("{name}.stub")) + } + fn can_materialise(&self) -> bool { + true + } + fn materialise(&self, local: &Path) -> Result<(), RemoteError> { + let real = PathBuf::from(local.to_string_lossy().strip_suffix(".stub").unwrap()); + std::fs::write(&real, vec![7u8; 25 * 1024 * 1024]).unwrap(); + std::fs::remove_file(local).unwrap(); + Ok(()) + } + fn dematerialise(&self, local: &Path) -> Result<(), RemoteError> { + std::fs::write(format!("{}.stub", local.display()), [0u8]).unwrap(); + std::fs::remove_file(local).unwrap(); + Ok(()) + } +} + +fn disk_used(root: &Path) -> u64 { + fn walk(p: &Path, total: &mut u64) { + if let Ok(entries) = std::fs::read_dir(p) { + for e in entries.flatten() { + let Ok(m) = e.metadata() else { continue }; + if m.is_dir() { + walk(&e.path(), total); + } else { + *total += m.len(); + } + } + } + } + let mut t = 0; + walk(root, &mut t); + t +} + +#[tokio::main(flavor = "current_thread")] +async fn main() { + let root = std::env::temp_dir().join("dr-vfs-cycle"); + let _ = std::fs::remove_dir_all(&root); + std::fs::create_dir_all(root.join("2026/03")).unwrap(); + + // Ninety dehydrated photographs, and ten the user already keeps. + for i in 0..90 { + std::fs::write(root.join(format!("2026/03/IMG_{i:04}.CR2.stub")), [0u8]).unwrap(); + } + for i in 90..100 { + std::fs::write( + root.join(format!("2026/03/IMG_{i:04}.CR2")), + vec![1u8; 25 * 1024 * 1024], + ) + .unwrap(); + } + + let b = FolderBackend::with_vfs(&root, Arc::new(Client)).unwrap(); + println!("materialisation: {:?}", b.capabilities().materialisation); + println!("on disk at rest: {} MB", disk_used(&root) / 1_048_576); + + let scan = dr_sync::scan( + &b, + &RemotePath::root(), + &FormatFilter::all(), + &HashMap::new(), + |_| {}, + ) + .await + .unwrap(); + + let absent = scan.images.iter().filter(|e| !e.materialised).count(); + println!( + "scanned {} photograph(s), {absent} not downloaded", + scan.images.len() + ); + // Names, not stubs — this is what the catalog records. + println!("first: {}", scan.images[0].path); + + // A pass over the library, one photograph at a time. + let pool = BorrowPool::new(); + let mut peak = 0u64; + let mut fetched = 0usize; + for entry in &scan.images { + let held = pool.borrow(&b, &entry.path).await.unwrap(); + if held.hydrated() { + fetched += 1; + } + // Read it, as a thumbnail pass would. + let n = b + .get(&RemoteId::Path(entry.path.clone()), Some(0..65536)) + .await + .unwrap() + .len(); + assert_eq!(n, 65536); + peak = peak.max(disk_used(&root)); + drop(held); + // Release as we go, which is what keeps the peak flat. + pool.release_all(&b).await; + } + + println!("fetched {fetched} of {}", scan.images.len()); + println!("peak on disk: {} MB", peak / 1_048_576); + println!("after the pass: {} MB", disk_used(&root) / 1_048_576); + println!( + "the ten the user already had: {} still here", + (90..100) + .filter(|i| root.join(format!("2026/03/IMG_{i:04}.CR2")).is_file()) + .count() + ); + + let _ = std::fs::remove_dir_all(&root); +} diff --git a/docs/architecture.md b/docs/architecture.md index c189f1a..2b3192d 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -892,7 +892,9 @@ unamended for the direct connector, which must still never hydrate to browse. 1. **Hydration is a borrow, not an acquisition.** A file is returned to the state it was found in — what the pass downloaded is released, what the user already had is left alone. Peak disk is the - working set, not the library. + working set, not the library. **Measured 2026-08-29** over 100 photographs of 25 MB, 90 of them + dehydrated: peak 275 MB against 2,500 MB unborrowed, back to 250 MB afterwards, and all ten the + user already held still there. 2. **It is paid once.** Thumbnails are kept, and `derived_sync` pushes the shards to the server, so a second device downloads 200 MB of shards instead of hydrating 340 GB of RAWs. 3. **It is quoted and consented to.** Never automatic, never on the browsing path, always diff --git a/docs/storage.md b/docs/storage.md index 6d867a6..a3e8342 100644 --- a/docs/storage.md +++ b/docs/storage.md @@ -465,6 +465,21 @@ A borrow against a plain folder or a server backend short-circuits and does nothing, so a pass written for a VFS library runs unchanged everywhere rather than growing two code paths. +**Measured 2026-08-29**, `cargo run -p dr-sync-folder --example vfs_cycle`: a +library of 100 photographs at 25 MB each, 90 of them dehydrated and 10 the user +keeps. A pass over all 100, borrowing and releasing as it goes: + +| | | +|---|---| +| on disk at rest | 250 MB | +| **peak during the pass** | **275 MB** — the resting set plus one photograph | +| without borrowing | 2,500 MB | +| on disk afterwards | 250 MB | +| of the 10 the user already had | 10 still there | + +The peak is the working set, not the library, and the release is selective. + + ### 6.4 Release means dehydrate, never delete The single most dangerous thing in this feature. A synced folder is not a diff --git a/docs/traceability.md b/docs/traceability.md index 1016208..b61aef8 100644 --- a/docs/traceability.md +++ b/docs/traceability.md @@ -9,7 +9,7 @@ Denominators are parsed from [`requirements.md`](requirements.md) at run time, n | Metric | Value | |---|---| -| Source files scanned | 286 | +| Source files scanned | 287 | | TRACES tags found | 840 | | Requirements defined | 179 | | Requirements covered | 107 |