Measure the borrow, rather than asserting it bounds the disk
The claim that hydration-as-a-borrow makes peak disk the working set rather than the library was so far an argument. `--example vfs_cycle` runs it: 100 photographs of 25 MB, 90 dehydrated, a pass over all of them through the real engine. Peak 275 MB — the resting set plus one photograph — against 2,500 MB had the pass simply fetched everything. Back to 250 MB afterwards, and all ten files the user already kept still there, which is the half of the contract that matters more. Recorded in docs/storage.md §6.3 and ARCH §9.0a, because a bound argued from a number nobody measured is one that gets quietly lost.
This commit is contained in:
@@ -0,0 +1,142 @@
|
||||
//! A placeholder library, borrowed and given back.
|
||||
//!
|
||||
//! ```text
|
||||
//! cargo run -p dr-sync-folder --example vfs_cycle
|
||||
//! ```
|
||||
//!
|
||||
//! Builds a tree in the system temp directory shaped like a suffix-mode VFS
|
||||
//! folder, runs the real engine over it, and reports what the borrow cost.
|
||||
//! Touches nothing outside its own scratch directory.
|
||||
use std::borrow::Cow;
|
||||
use std::collections::HashMap;
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::sync::Arc;
|
||||
|
||||
use dr_sync::{RemoteBackend, RemoteError, RemoteId, RemotePath};
|
||||
use dr_sync_folder::{BorrowPool, FolderBackend, Vfs};
|
||||
use dr_types::FormatFilter;
|
||||
|
||||
/// Stands in for the sync client, renaming exactly as suffix mode does.
|
||||
struct Client;
|
||||
|
||||
impl Vfs for Client {
|
||||
fn name(&self) -> &'static str {
|
||||
"demo"
|
||||
}
|
||||
fn is_placeholder(&self, on_disk: &str) -> bool {
|
||||
on_disk.ends_with(".stub")
|
||||
}
|
||||
fn real_name<'a>(&self, on_disk: &'a str) -> &'a str {
|
||||
on_disk.strip_suffix(".stub").unwrap_or(on_disk)
|
||||
}
|
||||
fn placeholder_name(&self, name: &str) -> Cow<'_, str> {
|
||||
Cow::Owned(format!("{name}.stub"))
|
||||
}
|
||||
fn can_materialise(&self) -> bool {
|
||||
true
|
||||
}
|
||||
fn materialise(&self, local: &Path) -> Result<(), RemoteError> {
|
||||
let real = PathBuf::from(local.to_string_lossy().strip_suffix(".stub").unwrap());
|
||||
std::fs::write(&real, vec![7u8; 25 * 1024 * 1024]).unwrap();
|
||||
std::fs::remove_file(local).unwrap();
|
||||
Ok(())
|
||||
}
|
||||
fn dematerialise(&self, local: &Path) -> Result<(), RemoteError> {
|
||||
std::fs::write(format!("{}.stub", local.display()), [0u8]).unwrap();
|
||||
std::fs::remove_file(local).unwrap();
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
fn disk_used(root: &Path) -> u64 {
|
||||
fn walk(p: &Path, total: &mut u64) {
|
||||
if let Ok(entries) = std::fs::read_dir(p) {
|
||||
for e in entries.flatten() {
|
||||
let Ok(m) = e.metadata() else { continue };
|
||||
if m.is_dir() {
|
||||
walk(&e.path(), total);
|
||||
} else {
|
||||
*total += m.len();
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
let mut t = 0;
|
||||
walk(root, &mut t);
|
||||
t
|
||||
}
|
||||
|
||||
#[tokio::main(flavor = "current_thread")]
|
||||
async fn main() {
|
||||
let root = std::env::temp_dir().join("dr-vfs-cycle");
|
||||
let _ = std::fs::remove_dir_all(&root);
|
||||
std::fs::create_dir_all(root.join("2026/03")).unwrap();
|
||||
|
||||
// Ninety dehydrated photographs, and ten the user already keeps.
|
||||
for i in 0..90 {
|
||||
std::fs::write(root.join(format!("2026/03/IMG_{i:04}.CR2.stub")), [0u8]).unwrap();
|
||||
}
|
||||
for i in 90..100 {
|
||||
std::fs::write(
|
||||
root.join(format!("2026/03/IMG_{i:04}.CR2")),
|
||||
vec![1u8; 25 * 1024 * 1024],
|
||||
)
|
||||
.unwrap();
|
||||
}
|
||||
|
||||
let b = FolderBackend::with_vfs(&root, Arc::new(Client)).unwrap();
|
||||
println!("materialisation: {:?}", b.capabilities().materialisation);
|
||||
println!("on disk at rest: {} MB", disk_used(&root) / 1_048_576);
|
||||
|
||||
let scan = dr_sync::scan(
|
||||
&b,
|
||||
&RemotePath::root(),
|
||||
&FormatFilter::all(),
|
||||
&HashMap::new(),
|
||||
|_| {},
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
let absent = scan.images.iter().filter(|e| !e.materialised).count();
|
||||
println!(
|
||||
"scanned {} photograph(s), {absent} not downloaded",
|
||||
scan.images.len()
|
||||
);
|
||||
// Names, not stubs — this is what the catalog records.
|
||||
println!("first: {}", scan.images[0].path);
|
||||
|
||||
// A pass over the library, one photograph at a time.
|
||||
let pool = BorrowPool::new();
|
||||
let mut peak = 0u64;
|
||||
let mut fetched = 0usize;
|
||||
for entry in &scan.images {
|
||||
let held = pool.borrow(&b, &entry.path).await.unwrap();
|
||||
if held.hydrated() {
|
||||
fetched += 1;
|
||||
}
|
||||
// Read it, as a thumbnail pass would.
|
||||
let n = b
|
||||
.get(&RemoteId::Path(entry.path.clone()), Some(0..65536))
|
||||
.await
|
||||
.unwrap()
|
||||
.len();
|
||||
assert_eq!(n, 65536);
|
||||
peak = peak.max(disk_used(&root));
|
||||
drop(held);
|
||||
// Release as we go, which is what keeps the peak flat.
|
||||
pool.release_all(&b).await;
|
||||
}
|
||||
|
||||
println!("fetched {fetched} of {}", scan.images.len());
|
||||
println!("peak on disk: {} MB", peak / 1_048_576);
|
||||
println!("after the pass: {} MB", disk_used(&root) / 1_048_576);
|
||||
println!(
|
||||
"the ten the user already had: {} still here",
|
||||
(90..100)
|
||||
.filter(|i| root.join(format!("2026/03/IMG_{i:04}.CR2")).is_file())
|
||||
.count()
|
||||
);
|
||||
|
||||
let _ = std::fs::remove_dir_all(&root);
|
||||
}
|
||||
@@ -892,7 +892,9 @@ unamended for the direct connector, which must still never hydrate to browse.
|
||||
|
||||
1. **Hydration is a borrow, not an acquisition.** A file is returned to the state it was found in —
|
||||
what the pass downloaded is released, what the user already had is left alone. Peak disk is the
|
||||
working set, not the library.
|
||||
working set, not the library. **Measured 2026-08-29** over 100 photographs of 25 MB, 90 of them
|
||||
dehydrated: peak 275 MB against 2,500 MB unborrowed, back to 250 MB afterwards, and all ten the
|
||||
user already held still there.
|
||||
2. **It is paid once.** Thumbnails are kept, and `derived_sync` pushes the shards to the server, so
|
||||
a second device downloads 200 MB of shards instead of hydrating 340 GB of RAWs.
|
||||
3. **It is quoted and consented to.** Never automatic, never on the browsing path, always
|
||||
|
||||
@@ -465,6 +465,21 @@ A borrow against a plain folder or a server backend short-circuits and does
|
||||
nothing, so a pass written for a VFS library runs unchanged everywhere rather
|
||||
than growing two code paths.
|
||||
|
||||
**Measured 2026-08-29**, `cargo run -p dr-sync-folder --example vfs_cycle`: a
|
||||
library of 100 photographs at 25 MB each, 90 of them dehydrated and 10 the user
|
||||
keeps. A pass over all 100, borrowing and releasing as it goes:
|
||||
|
||||
| | |
|
||||
|---|---|
|
||||
| on disk at rest | 250 MB |
|
||||
| **peak during the pass** | **275 MB** — the resting set plus one photograph |
|
||||
| without borrowing | 2,500 MB |
|
||||
| on disk afterwards | 250 MB |
|
||||
| of the 10 the user already had | 10 still there |
|
||||
|
||||
The peak is the working set, not the library, and the release is selective.
|
||||
|
||||
|
||||
### 6.4 Release means dehydrate, never delete
|
||||
|
||||
The single most dangerous thing in this feature. A synced folder is not a
|
||||
|
||||
@@ -9,7 +9,7 @@ Denominators are parsed from [`requirements.md`](requirements.md) at run time, n
|
||||
|
||||
| Metric | Value |
|
||||
|---|---|
|
||||
| Source files scanned | 286 |
|
||||
| Source files scanned | 287 |
|
||||
| TRACES tags found | 840 |
|
||||
| Requirements defined | 179 |
|
||||
| Requirements covered | 107 |
|
||||
|
||||
Reference in New Issue
Block a user