Add folder scan with format selection; validate A3 on a real library
Library setup as the user described it: pick a folder, choose which RAW
types to look for, scan recursively.
dr-types::FormatFilter the tick-box selection, seeing through VFS
placeholder suffixes so a dehydrated CR2 still
matches as a CR2
dr-sync::scan recursive walk, Depth:1 per directory, pruning
unchanged subtrees where the backend propagates
directory ETags
Verified against nextcloud.tourolle.paris (34.0.2) on a real library:
browse root 32 entries, 98ms
scan PhotosRaw 17,185 RAW files in 334 directories, 34.1s
(7,836 CR2 + 9,349 DNG)
range read 262KB of a 21.5MB DNG in 119ms — 1.22% of the file,
and enough to read "Canon EOS 6D | ISO 100"
That last line is assumption A3 validated on real data. Cataloguing this
library by whole-file fetch would move roughly 370GB; the range path
moves a few MB.
Pruning is capability-gated rather than assumed: with per-entry ETags a
probe costs a request and proves nothing about children, so it is skipped
entirely. A test asserts zero probes in that case.
Still unresolved: /core/preview returns 400 for every parameter
combination tried, including on a JPEG the server reports as having a
preview. Not a request-shape bug — it fails identically bare. Recorded
rather than worked around; ARCH §6.7 already treats server previews as
opportunistic, so nothing depends on it.
This commit is contained in:
@@ -1,9 +1,13 @@
|
||||
//! Exercise the connector against a real Nextcloud server.
|
||||
//!
|
||||
//! ```text
|
||||
//! cargo run -p dr-sync-nextcloud --example connect -- https://cloud.example [remote/path]
|
||||
//! cargo run -p dr-sync-nextcloud --example connect -- <server> [path] [--raw|--formats cr2,nef]
|
||||
//! ```
|
||||
//!
|
||||
//! With no path it lists the account root so a folder can be chosen. Given a
|
||||
//! path it scans recursively for images matching the format filter, which is
|
||||
//! the library-setup flow (FR-CAT-1).
|
||||
//!
|
||||
//! **Strictly read-only.** PROPFIND, ETag probes, range GETs and preview
|
||||
//! requests only — no PUT, MOVE or DELETE — so it cannot alter a live library.
|
||||
//!
|
||||
@@ -15,6 +19,7 @@
|
||||
//! not re-authenticate. That location is for *testing convenience* only;
|
||||
//! FR-NC-2 requires the real app to use platform secure storage.
|
||||
|
||||
use std::collections::HashMap;
|
||||
use std::path::PathBuf;
|
||||
use std::time::Instant;
|
||||
|
||||
@@ -30,7 +35,25 @@ async fn main() {
|
||||
eprintln!("usage: connect <server-url> [remote/path]");
|
||||
std::process::exit(2);
|
||||
};
|
||||
let start_path = args.next().unwrap_or_default();
|
||||
let rest: Vec<String> = args.collect();
|
||||
let start_path = rest
|
||||
.iter()
|
||||
.find(|a| !a.starts_with("--"))
|
||||
.cloned()
|
||||
.unwrap_or_default();
|
||||
|
||||
// Format selection — the tick-boxes, as a CLI flag.
|
||||
let filter = if let Some(i) = rest.iter().position(|a| a == "--formats") {
|
||||
let list = rest.get(i + 1).cloned().unwrap_or_default();
|
||||
dr_types::FormatFilter::from_formats(
|
||||
list.split(',')
|
||||
.filter_map(|s| dr_types::Format::from_extension(s.trim())),
|
||||
)
|
||||
} else if rest.iter().any(|a| a == "--raw") {
|
||||
dr_types::FormatFilter::raw_only()
|
||||
} else {
|
||||
dr_types::FormatFilter::all()
|
||||
};
|
||||
|
||||
let creds = match load_cached(&server) {
|
||||
Some(c) => {
|
||||
@@ -63,126 +86,134 @@ async fn main() {
|
||||
println!("cheap no-op sync: {}", strategy.has_cheap_noop());
|
||||
|
||||
let root = RemotePath::new(&start_path);
|
||||
let mut failures = 0;
|
||||
|
||||
// ---- 1. listing ----------------------------------------------------
|
||||
println!("\n[1] PROPFIND Depth:1 on /{start_path}");
|
||||
let t = Instant::now();
|
||||
let entries = match backend.list(&root, None).await {
|
||||
Ok(e) => {
|
||||
println!(
|
||||
" {} entries in {:.0}ms",
|
||||
e.len(),
|
||||
t.elapsed().as_secs_f64() * 1000.0
|
||||
);
|
||||
e
|
||||
// No path given: list this level so the user can pick a folder.
|
||||
if start_path.is_empty() {
|
||||
println!("\n[browse] PROPFIND Depth:1 on the account root");
|
||||
let t = Instant::now();
|
||||
match backend.list(&root, None).await {
|
||||
Ok(entries) => {
|
||||
println!(
|
||||
" {} entries in {:.0}ms\n",
|
||||
entries.len(),
|
||||
t.elapsed().as_secs_f64() * 1000.0
|
||||
);
|
||||
let mut dirs: Vec<_> = entries
|
||||
.iter()
|
||||
.filter(|e| e.kind == dr_sync::EntryKind::Directory)
|
||||
.collect();
|
||||
dirs.sort_by_key(|e| e.path.name().to_ascii_lowercase());
|
||||
for d in &dirs {
|
||||
println!(" {}/", d.path.name());
|
||||
}
|
||||
println!(
|
||||
"\n Re-run with a folder to scan it, e.g.:\n … {} \"{}\" --raw",
|
||||
server,
|
||||
dirs.first().map(|d| d.path.name()).unwrap_or("Photos")
|
||||
);
|
||||
}
|
||||
Err(e) => {
|
||||
eprintln!(" FAILED: {e}");
|
||||
std::process::exit(1);
|
||||
}
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
// A path was given: scan it recursively for the selected formats.
|
||||
println!("\n[scan] {} under /{start_path}", describe_filter(&filter));
|
||||
let t = Instant::now();
|
||||
let result = match dr_sync::scan(&backend, &root, &filter, &HashMap::new(), |p| {
|
||||
if p.directories_listed % 25 == 0 && p.directories_listed > 0 {
|
||||
print!(
|
||||
"\r {} dirs, {} images…",
|
||||
p.directories_listed, p.images_found
|
||||
);
|
||||
let _ = std::io::Write::flush(&mut std::io::stdout());
|
||||
}
|
||||
})
|
||||
.await
|
||||
{
|
||||
Ok(r) => r,
|
||||
Err(e) => {
|
||||
println!(" FAILED: {e}");
|
||||
failures += 1;
|
||||
Vec::new()
|
||||
eprintln!("\n FAILED: {e}");
|
||||
std::process::exit(1);
|
||||
}
|
||||
};
|
||||
let scan_ms = t.elapsed().as_secs_f64() * 1000.0;
|
||||
|
||||
for e in entries.iter().take(5) {
|
||||
println!(
|
||||
"\r {} images in {} directories, {:.1}s",
|
||||
result.images.len(),
|
||||
result.progress.directories_listed,
|
||||
scan_ms / 1000.0
|
||||
);
|
||||
|
||||
let mut by_ext: std::collections::BTreeMap<String, usize> = Default::default();
|
||||
for i in &result.images {
|
||||
let ext = i
|
||||
.path
|
||||
.name()
|
||||
.rsplit_once('.')
|
||||
.map(|(_, e)| e.to_ascii_uppercase())
|
||||
.unwrap_or_default();
|
||||
*by_ext.entry(ext).or_default() += 1;
|
||||
}
|
||||
for (ext, n) in &by_ext {
|
||||
println!(" {ext:<6} {n}");
|
||||
}
|
||||
|
||||
// Prove the fast path on a real RAW: metadata from a header range alone.
|
||||
let raw = result.images.iter().find(|i| {
|
||||
i.path
|
||||
.name()
|
||||
.rsplit_once('.')
|
||||
.and_then(|(_, e)| dr_types::Format::from_extension(&e.to_ascii_lowercase()))
|
||||
.is_some_and(|f| f.is_raw())
|
||||
&& i.size > 300_000
|
||||
});
|
||||
|
||||
if let Some(f) = raw {
|
||||
println!(
|
||||
" {:?} {:<40} {:>10} id={:?}{}",
|
||||
e.kind,
|
||||
truncate(e.path.name(), 40),
|
||||
human(e.size),
|
||||
e.id,
|
||||
if e.has_preview { " preview" } else { "" }
|
||||
);
|
||||
}
|
||||
if entries.len() > 5 {
|
||||
println!(" … {} more", entries.len() - 5);
|
||||
}
|
||||
|
||||
// ---- 2. the pruning probe ------------------------------------------
|
||||
println!("\n[2] PROPFIND Depth:0 — the ETag pruning probe (FR-NC-4)");
|
||||
let t = Instant::now();
|
||||
match backend.dir_validator(&root).await {
|
||||
Ok(v) => println!(
|
||||
" etag {} in {:.0}ms — one request proves the tree unchanged",
|
||||
v.as_str(),
|
||||
t.elapsed().as_secs_f64() * 1000.0
|
||||
),
|
||||
Err(e) => {
|
||||
println!(" FAILED: {e}");
|
||||
failures += 1;
|
||||
}
|
||||
}
|
||||
|
||||
// ---- 3. range read -------------------------------------------------
|
||||
// The mechanism the whole mobile story rests on (FR-NC-3).
|
||||
let file = entries
|
||||
.iter()
|
||||
.find(|e| e.kind == dr_sync::EntryKind::File && e.size > 300_000);
|
||||
|
||||
if let Some(f) = file {
|
||||
println!(
|
||||
"\n[3] Range GET — first 256KB of {} ({})",
|
||||
"\n[range] first 256KB of {} ({})",
|
||||
f.path.name(),
|
||||
human(f.size)
|
||||
);
|
||||
let id = RemoteId::Path(f.path.clone());
|
||||
|
||||
let t = Instant::now();
|
||||
match backend.get(&id, Some(0..262_144)).await {
|
||||
Ok(bytes) => {
|
||||
let ms = t.elapsed().as_secs_f64() * 1000.0;
|
||||
let pct = (bytes.len() as f64 / f.size as f64) * 100.0;
|
||||
println!(
|
||||
" got {} in {ms:.0}ms ({pct:.1}% of the file)",
|
||||
" {} in {ms:.0}ms — {pct:.2}% of the file",
|
||||
human(bytes.len() as u64)
|
||||
);
|
||||
|
||||
if bytes.len() as u64 >= f.size {
|
||||
println!(" WARNING: whole file returned — server ignored Range");
|
||||
failures += 1;
|
||||
} else {
|
||||
println!(" range requests work — remote browsing is viable");
|
||||
}
|
||||
|
||||
if let Some(fmt) = dr_decode::probe(&bytes) {
|
||||
println!(" probe: {fmt:?}");
|
||||
}
|
||||
match dr_decode::metadata(&bytes) {
|
||||
Ok(m) => println!(
|
||||
" metadata from the range alone: {} {}",
|
||||
m.make.unwrap_or_default(),
|
||||
m.model.unwrap_or_default()
|
||||
" metadata from that range alone: {} {} | {}",
|
||||
m.make.unwrap_or_default().trim(),
|
||||
m.model.unwrap_or_default().trim(),
|
||||
m.iso.map(|i| format!("ISO {i}")).unwrap_or_default()
|
||||
),
|
||||
Err(e) => println!(" metadata: {e}"),
|
||||
}
|
||||
}
|
||||
Err(e) => {
|
||||
println!(" FAILED: {e}");
|
||||
failures += 1;
|
||||
}
|
||||
}
|
||||
|
||||
// ---- 4. server preview ------------------------------------------
|
||||
println!("\n[4] Server preview (ARCH §6.7 expects none for RAW)");
|
||||
match backend.thumbnail(&f.id, 256).await {
|
||||
Ok(Some(b)) => println!(" {} returned", human(b.len() as u64)),
|
||||
Ok(None) => println!(" none — as expected; local extraction is the path"),
|
||||
Err(e) => println!(" error: {e}"),
|
||||
Err(e) => println!(" FAILED: {e}"),
|
||||
}
|
||||
} else {
|
||||
println!("\n[3] skipped — no file over 300KB at this path");
|
||||
println!("\n[range] skipped — no RAW over 300KB found");
|
||||
}
|
||||
|
||||
println!(
|
||||
"\n{}",
|
||||
if failures == 0 {
|
||||
"all checks passed"
|
||||
} else {
|
||||
"FAILURES"
|
||||
}
|
||||
);
|
||||
if failures > 0 {
|
||||
std::process::exit(1);
|
||||
println!("\nscan complete");
|
||||
}
|
||||
|
||||
fn describe_filter(f: &dr_types::FormatFilter) -> String {
|
||||
let names: Vec<&str> = f.iter().map(|x| x.label()).collect();
|
||||
if names.len() >= 9 {
|
||||
"all supported formats".into()
|
||||
} else {
|
||||
names.join(", ")
|
||||
}
|
||||
}
|
||||
|
||||
@@ -271,11 +302,3 @@ fn human(bytes: u64) -> String {
|
||||
b => format!("{b}B"),
|
||||
}
|
||||
}
|
||||
|
||||
fn truncate(s: &str, n: usize) -> String {
|
||||
if s.chars().count() <= n {
|
||||
s.to_string()
|
||||
} else {
|
||||
format!("{}…", s.chars().take(n - 1).collect::<String>())
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user