Add Nextcloud connector; reject VFS as a transfer mechanism
Investigated using the Nextcloud desktop client's Virtual Files as a
cache instead of talking to the server directly. Measured on this
machine (client 4.0.7): the configured folder holds 121,785 placeholders
against 10,267 materialised files, including 7,037 CR2 and 9,411 DNG.
Three findings, each independently disqualifying:
- Linux VFS is *suffix* mode. A dehydrated IMG.CR2 exists only as
IMG.CR2.nextcloud holding one byte; the real name is absent.
- Reading a placeholder does not hydrate it. dd of the first 256KB
returned 1 byte, the stub was unchanged, and the real name never
appeared. There is no FUSE layer — the stub is an inert marker.
- Even with hydration the granularity is wrong: VFS has two states,
1 byte or all bytes, and the preview tier needs a ~256KB prefix of
a 27MB file. That is ~100x what FR-NC-3 requires.
Recorded as ARCH §9.0. Coexistence is still supported: dr-types now
recognises *.nextcloud stubs, and the viewer lists them as "not
downloaded" rather than as corrupt files or not at all.
So the connector talks to the server directly, as D7 specified.
Implemented: Login Flow v2, PROPFIND with oc:fileid and nc:has-preview,
ETag pruning via a Depth:0 probe, range GET with local slicing when the
server ignores the header, conditional PUT, and /core/preview with
forceIcon=false. delta() returns Unsupported and says why.
Chunked upload v2 is not implemented yet — put() rejects bodies over
5MB explicitly rather than silently truncating.
Two bugs found by testing: my hand-computed epoch in a date test was a
day out (the parser was right), and quick-xml reaches EOF on truncated
input without erroring, so unbalanced elements needed an explicit check
— a half-parsed multistatus must not look like an empty directory.
83 tests passing.
This commit is contained in:
@@ -0,0 +1,439 @@
|
||||
//! Nextcloud connector — the only [`RemoteBackend`] implementation.
|
||||
//!
|
||||
//! Hand-rolled over `reqwest` rather than built on a WebDAV crate (D7). No
|
||||
//! mature Nextcloud crate exists, and the operations that matter here are
|
||||
//! Nextcloud extensions: `oc:fileid`, propagating directory ETags, chunked
|
||||
//! upload v2, and Login Flow v2. A general WebDAV client exposes none of them.
|
||||
|
||||
use std::ops::Range;
|
||||
|
||||
use async_trait::async_trait;
|
||||
use dr_sync::{
|
||||
Capabilities, ChangeDetection, ChunkConstraints, Cursor, Precondition, RemoteBackend,
|
||||
RemoteChange, RemoteEntry, RemoteError, RemoteId, RemotePath, ServerPreviews, Validator,
|
||||
};
|
||||
|
||||
pub mod auth;
|
||||
mod propfind;
|
||||
|
||||
pub use auth::{AppCredentials, LoginFlow};
|
||||
|
||||
/// Chunk sizes Nextcloud's chunked upload v2 accepts.
|
||||
const CHUNKS: ChunkConstraints = ChunkConstraints {
|
||||
min_chunk: 5 * 1024 * 1024,
|
||||
max_chunk: 5 * 1024 * 1024 * 1024,
|
||||
// Below the 5 MB floor a single PUT is simpler and no slower.
|
||||
single_shot_below: 5 * 1024 * 1024,
|
||||
max_chunks: 10_000,
|
||||
};
|
||||
|
||||
/// TRACES: FR-NC-12 | M-5
|
||||
/// A connected Nextcloud account.
|
||||
pub struct NextcloudBackend {
|
||||
client: reqwest::Client,
|
||||
server: String,
|
||||
login: String,
|
||||
password: String,
|
||||
/// `/remote.php/dav/files/<user>/` — the prefix stripped from hrefs.
|
||||
dav_base: String,
|
||||
caps: Capabilities,
|
||||
}
|
||||
|
||||
impl NextcloudBackend {
|
||||
/// Build a backend from credentials obtained via [`auth`].
|
||||
pub fn new(creds: &AppCredentials, user_id: &str) -> Result<Self, RemoteError> {
|
||||
let client = reqwest::Client::builder()
|
||||
.user_agent("DarkRoom")
|
||||
.build()
|
||||
.map_err(|e| RemoteError::Network(e.to_string()))?;
|
||||
|
||||
let server = creds.server.trim_end_matches('/').to_string();
|
||||
let dav_base = format!("/remote.php/dav/files/{user_id}/");
|
||||
|
||||
Ok(Self {
|
||||
client,
|
||||
server,
|
||||
login: creds.login_name.clone(),
|
||||
password: creds.app_password.clone(),
|
||||
dav_base,
|
||||
caps: Capabilities {
|
||||
// The property that makes a no-op sync one request (ARCH §8.1).
|
||||
change_detection: ChangeDetection::PropagatingEtags,
|
||||
stable_ids: true,
|
||||
range_reads: true,
|
||||
chunked_upload: Some(CHUNKS),
|
||||
bulk_upload: true,
|
||||
conditional_write: true,
|
||||
// Stock Nextcloud ships no RAW preview provider (ARCH §6.7).
|
||||
// Probed per-account at setup and upgraded where present.
|
||||
server_previews: ServerPreviews::CommonFormatsOnly,
|
||||
},
|
||||
})
|
||||
}
|
||||
|
||||
fn url_for(&self, path: &RemotePath) -> String {
|
||||
let p = path.as_str();
|
||||
if p.is_empty() {
|
||||
format!("{}{}", self.server, self.dav_base)
|
||||
} else {
|
||||
format!("{}{}{}", self.server, self.dav_base, encode_path(p))
|
||||
}
|
||||
}
|
||||
|
||||
fn url_for_id(&self, id: &RemoteId) -> Result<String, RemoteError> {
|
||||
match id {
|
||||
RemoteId::Path(p) => Ok(self.url_for(p)),
|
||||
// A stable id alone is not addressable over WebDAV; the caller
|
||||
// holds the path alongside it in the catalog.
|
||||
RemoteId::Stable(_) => Err(RemoteError::Unsupported(
|
||||
"fetch by fileid requires a path; use RemoteId::Path",
|
||||
)),
|
||||
}
|
||||
}
|
||||
|
||||
async fn propfind(
|
||||
&self,
|
||||
path: &RemotePath,
|
||||
depth: &str,
|
||||
body: &'static str,
|
||||
) -> Result<String, RemoteError> {
|
||||
let resp = self
|
||||
.client
|
||||
.request(
|
||||
reqwest::Method::from_bytes(b"PROPFIND").expect("valid method"),
|
||||
self.url_for(path),
|
||||
)
|
||||
.basic_auth(&self.login, Some(&self.password))
|
||||
.header("Depth", depth)
|
||||
.header(reqwest::header::CONTENT_TYPE, "application/xml")
|
||||
.body(body)
|
||||
.send()
|
||||
.await
|
||||
.map_err(|e| RemoteError::Network(e.to_string()))?;
|
||||
|
||||
map_status(resp.status(), path.as_str())?;
|
||||
resp.text()
|
||||
.await
|
||||
.map_err(|e| RemoteError::Network(e.to_string()))
|
||||
}
|
||||
}
|
||||
|
||||
#[async_trait]
|
||||
impl RemoteBackend for NextcloudBackend {
|
||||
fn capabilities(&self) -> &Capabilities {
|
||||
&self.caps
|
||||
}
|
||||
|
||||
fn name(&self) -> &str {
|
||||
"Nextcloud"
|
||||
}
|
||||
|
||||
async fn list(
|
||||
&self,
|
||||
dir: &RemotePath,
|
||||
_since: Option<&Validator>,
|
||||
) -> Result<Vec<RemoteEntry>, RemoteError> {
|
||||
// Depth 1 only. Depth: infinity is frequently disabled and
|
||||
// prohibitively expensive where it is not (ARCH §8.4).
|
||||
let xml = self.propfind(dir, "1", propfind::PROPFIND_BODY).await?;
|
||||
propfind::parse_multistatus(&xml, &self.dav_base)
|
||||
}
|
||||
|
||||
async fn dir_validator(&self, dir: &RemotePath) -> Result<Validator, RemoteError> {
|
||||
// The pruning probe: Depth 0, ETag only. Against the root this is the
|
||||
// single request that proves a 50k-image library unchanged.
|
||||
let xml = self
|
||||
.propfind(dir, "0", propfind::PROPFIND_ETAG_ONLY)
|
||||
.await?;
|
||||
propfind::parse_self_etag(&xml)
|
||||
}
|
||||
|
||||
async fn delta(&self, _cursor: &Cursor) -> Result<(Vec<RemoteChange>, Cursor), RemoteError> {
|
||||
// Verified absent: Nextcloud's Directory.php does not implement
|
||||
// ISyncCollection, and sync tokens exist only for CalDAV/CardDAV
|
||||
// (ARCH §6.6). The engine falls back to ETag pruning.
|
||||
Err(RemoteError::Unsupported(
|
||||
"Nextcloud has no RFC 6578 sync-collection for files",
|
||||
))
|
||||
}
|
||||
|
||||
async fn get(&self, id: &RemoteId, range: Option<Range<u64>>) -> Result<Vec<u8>, RemoteError> {
|
||||
let url = self.url_for_id(id)?;
|
||||
let mut req = self
|
||||
.client
|
||||
.get(&url)
|
||||
.basic_auth(&self.login, Some(&self.password));
|
||||
|
||||
let wanted = range.clone();
|
||||
if let Some(r) = &range {
|
||||
req = req.header(
|
||||
reqwest::header::RANGE,
|
||||
format!("bytes={}-{}", r.start, r.end.saturating_sub(1)),
|
||||
);
|
||||
}
|
||||
|
||||
let resp = req
|
||||
.send()
|
||||
.await
|
||||
.map_err(|e| RemoteError::Network(e.to_string()))?;
|
||||
let status = resp.status();
|
||||
map_status(status, &url)?;
|
||||
|
||||
let body = resp
|
||||
.bytes()
|
||||
.await
|
||||
.map_err(|e| RemoteError::Network(e.to_string()))?
|
||||
.to_vec();
|
||||
|
||||
// Nextcloud does not advertise Accept-Ranges, so support is detected
|
||||
// by the response code rather than by probing with HEAD (ARCH §6.7).
|
||||
// A 200 to a ranged request means the server ignored it and sent
|
||||
// everything; slice locally so callers still get what they asked for.
|
||||
if let Some(r) = wanted {
|
||||
if status.as_u16() == 200 && body.len() as u64 > r.end {
|
||||
let start = r.start.min(body.len() as u64) as usize;
|
||||
let end = r.end.min(body.len() as u64) as usize;
|
||||
return Ok(body[start..end].to_vec());
|
||||
}
|
||||
}
|
||||
|
||||
Ok(body)
|
||||
}
|
||||
|
||||
async fn put(
|
||||
&self,
|
||||
path: &RemotePath,
|
||||
body: Vec<u8>,
|
||||
precond: Option<Precondition>,
|
||||
) -> Result<Validator, RemoteError> {
|
||||
// Chunked upload is an implementation detail of put, chosen by size —
|
||||
// exposing it on the trait would leak this protocol (ARCH §8.3).
|
||||
if body.len() as u64 >= CHUNKS.single_shot_below {
|
||||
return Err(RemoteError::Unsupported(
|
||||
"chunked upload v2 not implemented yet",
|
||||
));
|
||||
}
|
||||
|
||||
let mut req = self
|
||||
.client
|
||||
.put(self.url_for(path))
|
||||
.basic_auth(&self.login, Some(&self.password));
|
||||
|
||||
match &precond {
|
||||
Some(Precondition::IfMatch(v)) => {
|
||||
req = req.header(reqwest::header::IF_MATCH, format!("\"{}\"", v.as_str()));
|
||||
}
|
||||
Some(Precondition::IfAbsent) => {
|
||||
req = req.header(reqwest::header::IF_NONE_MATCH, "*");
|
||||
}
|
||||
None => {}
|
||||
}
|
||||
|
||||
let resp = req
|
||||
.body(body)
|
||||
.send()
|
||||
.await
|
||||
.map_err(|e| RemoteError::Network(e.to_string()))?;
|
||||
map_status(resp.status(), path.as_str())?;
|
||||
|
||||
resp.headers()
|
||||
.get(reqwest::header::ETAG)
|
||||
.and_then(|v| v.to_str().ok())
|
||||
.map(Validator::new)
|
||||
.ok_or_else(|| RemoteError::Protocol("no ETag on PUT response".into()))
|
||||
}
|
||||
|
||||
async fn delete(
|
||||
&self,
|
||||
id: &RemoteId,
|
||||
precond: Option<Precondition>,
|
||||
) -> Result<(), RemoteError> {
|
||||
let url = self.url_for_id(id)?;
|
||||
let mut req = self
|
||||
.client
|
||||
.delete(&url)
|
||||
.basic_auth(&self.login, Some(&self.password));
|
||||
|
||||
if let Some(Precondition::IfMatch(v)) = &precond {
|
||||
req = req.header(reqwest::header::IF_MATCH, format!("\"{}\"", v.as_str()));
|
||||
}
|
||||
|
||||
let resp = req
|
||||
.send()
|
||||
.await
|
||||
.map_err(|e| RemoteError::Network(e.to_string()))?;
|
||||
map_status(resp.status(), &url)
|
||||
}
|
||||
|
||||
async fn thumbnail(&self, id: &RemoteId, size: u32) -> Result<Option<Vec<u8>>, RemoteError> {
|
||||
let RemoteId::Stable(file_id) = id else {
|
||||
return Ok(None);
|
||||
};
|
||||
|
||||
// forceIcon=false is mandatory: the default returns a generic
|
||||
// mimetype icon for files the server cannot render, which would
|
||||
// otherwise be cached as though it were a thumbnail (ARCH §6.7).
|
||||
let url = format!(
|
||||
"{}/core/preview?fileId={file_id}&x={size}&y={size}&a=true&forceIcon=false",
|
||||
self.server
|
||||
);
|
||||
|
||||
let resp = self
|
||||
.client
|
||||
.get(&url)
|
||||
.basic_auth(&self.login, Some(&self.password))
|
||||
.send()
|
||||
.await
|
||||
.map_err(|e| RemoteError::Network(e.to_string()))?;
|
||||
|
||||
// 404 means no preview for this file — expected for RAW, and a
|
||||
// fall-through rather than an error.
|
||||
if resp.status().as_u16() == 404 {
|
||||
return Ok(None);
|
||||
}
|
||||
map_status(resp.status(), &url)?;
|
||||
|
||||
Ok(Some(
|
||||
resp.bytes()
|
||||
.await
|
||||
.map_err(|e| RemoteError::Network(e.to_string()))?
|
||||
.to_vec(),
|
||||
))
|
||||
}
|
||||
}
|
||||
|
||||
/// Translate an HTTP status into a typed error.
|
||||
fn map_status(status: reqwest::StatusCode, what: &str) -> Result<(), RemoteError> {
|
||||
match status.as_u16() {
|
||||
200..=299 => Ok(()),
|
||||
401 | 403 => Err(RemoteError::AuthFailed),
|
||||
404 => Err(RemoteError::NotFound(what.to_string())),
|
||||
// Drives the sidecar merge path rather than an overwrite (ARCH §8.5).
|
||||
412 => Err(RemoteError::PreconditionFailed),
|
||||
507 => Err(RemoteError::QuotaExceeded),
|
||||
s => Err(RemoteError::Server {
|
||||
status: s,
|
||||
detail: what.to_string(),
|
||||
}),
|
||||
}
|
||||
}
|
||||
|
||||
/// Percent-encode each path segment, leaving separators intact.
|
||||
fn encode_path(path: &str) -> String {
|
||||
path.split('/')
|
||||
.map(|seg| {
|
||||
seg.bytes()
|
||||
.map(|b| match b {
|
||||
b'A'..=b'Z' | b'a'..=b'z' | b'0'..=b'9' | b'-' | b'_' | b'.' | b'~' => {
|
||||
(b as char).to_string()
|
||||
}
|
||||
_ => format!("%{b:02X}"),
|
||||
})
|
||||
.collect::<String>()
|
||||
})
|
||||
.collect::<Vec<_>>()
|
||||
.join("/")
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
fn backend() -> NextcloudBackend {
|
||||
NextcloudBackend::new(
|
||||
&AppCredentials {
|
||||
server: "https://cloud.example/".into(),
|
||||
login_name: "duncan".into(),
|
||||
app_password: "token".into(),
|
||||
},
|
||||
"duncan",
|
||||
)
|
||||
.unwrap()
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn capabilities_declare_the_nextcloud_fast_path() {
|
||||
let b = backend();
|
||||
let c = b.capabilities();
|
||||
// Propagating ETags are what make a no-op sync one request.
|
||||
assert_eq!(c.change_detection, ChangeDetection::PropagatingEtags);
|
||||
assert!(c.stable_ids);
|
||||
assert!(c.range_reads);
|
||||
assert!(c.conditional_write);
|
||||
// Stock Nextcloud cannot render RAW previews.
|
||||
assert_eq!(c.server_previews, ServerPreviews::CommonFormatsOnly);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn strategy_selects_etag_pruning() {
|
||||
use dr_sync::SyncStrategy;
|
||||
let s = SyncStrategy::for_capabilities(backend().capabilities());
|
||||
assert_eq!(s, SyncStrategy::EtagPruning);
|
||||
assert!(s.has_cheap_noop());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn urls_include_the_dav_prefix() {
|
||||
let b = backend();
|
||||
assert_eq!(
|
||||
b.url_for(&RemotePath::new("Photos/2026")),
|
||||
"https://cloud.example/remote.php/dav/files/duncan/Photos/2026"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn root_url_has_no_trailing_segment() {
|
||||
let b = backend();
|
||||
assert_eq!(
|
||||
b.url_for(&RemotePath::root()),
|
||||
"https://cloud.example/remote.php/dav/files/duncan/"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn path_segments_are_encoded_but_separators_are_not() {
|
||||
assert_eq!(encode_path("My Trip/a b.CR2"), "My%20Trip/a%20b.CR2");
|
||||
assert_eq!(encode_path("Photos/2026"), "Photos/2026");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn fetching_by_bare_fileid_is_rejected_clearly() {
|
||||
// Callers hold the path alongside the id; failing loudly beats
|
||||
// constructing a URL that cannot work.
|
||||
let b = backend();
|
||||
assert!(matches!(
|
||||
b.url_for_id(&RemoteId::Stable(42)),
|
||||
Err(RemoteError::Unsupported(_))
|
||||
));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn statuses_map_to_actionable_errors() {
|
||||
use reqwest::StatusCode;
|
||||
assert!(map_status(StatusCode::OK, "x").is_ok());
|
||||
assert!(matches!(
|
||||
map_status(StatusCode::UNAUTHORIZED, "x"),
|
||||
Err(RemoteError::AuthFailed)
|
||||
));
|
||||
// 412 must be distinguishable — it drives the sidecar merge.
|
||||
assert!(matches!(
|
||||
map_status(StatusCode::PRECONDITION_FAILED, "x"),
|
||||
Err(RemoteError::PreconditionFailed)
|
||||
));
|
||||
assert!(matches!(
|
||||
map_status(StatusCode::INSUFFICIENT_STORAGE, "x"),
|
||||
Err(RemoteError::QuotaExceeded)
|
||||
));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn delta_is_unsupported_and_says_why() {
|
||||
// Verified absent in the server; the engine must fall back rather
|
||||
// than treat this as a failure.
|
||||
let b = backend();
|
||||
assert!(matches!(
|
||||
b.delta(&Cursor::new("x")).await,
|
||||
Err(RemoteError::Unsupported(_))
|
||||
));
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user