Put an uploaded original in its dated folder
The transport was built and had nowhere to aim: put_chunked has existed since the connector landed, and the only things pushed through it are thumbnail shards and the catalog snapshot. The folder segments arrive already expanded, from dr_ingest::layout, rather than being re-derived here. That split is what makes FR-NC-7a's first consequence true — if this module worked out the folders from whatever the local library happened to look like, two machines with differently organised libraries would file the same photograph in two different places on one server. A name is resolved against one listing rather than a probe per candidate: a day folder is one PROPFIND and the answer covers every collision in it, where probing costs a round trip per attempt over a link that may be mobile data. Two cameras both produce IMG_0001.CR3, and the second must not overwrite the first. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Claude Opus 5
@@ -25,6 +25,7 @@ pub mod error;
|
||||
pub mod reachability;
|
||||
pub mod scan;
|
||||
pub mod types;
|
||||
pub mod upload;
|
||||
|
||||
pub use capability::{Capabilities, ChangeDetection, ChunkConstraints, ServerPreviews};
|
||||
pub use error::RemoteError;
|
||||
@@ -34,6 +35,7 @@ pub use types::{
|
||||
Cursor, EntryKind, Identity, Precondition, RemoteChange, RemoteEntry, RemoteId, RemotePath,
|
||||
Validator,
|
||||
};
|
||||
pub use upload::{destination, upload_original};
|
||||
|
||||
/// TRACES: FR-NC-12
|
||||
/// A remote storage backend.
|
||||
|
||||
@@ -0,0 +1,266 @@
|
||||
//! TRACES: FR-NC-7a | FR-NC-7b
|
||||
//! Where an uploaded original lands on the server.
|
||||
//!
|
||||
//! FR-NC-7 says how the bytes travel — chunked upload v2, `MKCOL`, `MOVE` of
|
||||
//! the `.file` pseudo-entry. This says *where they go*, which the transport
|
||||
//! deliberately has no opinion about.
|
||||
//!
|
||||
//! # The template is expanded on the ingest side, not here
|
||||
//!
|
||||
//! An upload receives folder *segments*, already expanded and already checked
|
||||
//! to be ordinary names (`dr_ingest::layout`). That split is what makes
|
||||
//! FR-NC-7a's first consequence true: the destination is a pure function of
|
||||
//! capture metadata and the template, so two devices uploading the same frame
|
||||
//! compute the same path. If this module re-derived the folders from whatever
|
||||
//! the local library happened to look like, two machines with differently
|
||||
//! organised libraries would file the same photograph in two different places
|
||||
//! on one server.
|
||||
//!
|
||||
//! # What this deliberately does not do
|
||||
//!
|
||||
//! It does not move anything already on the server. FR-NC-7a's second
|
||||
//! consequence: the template governs placement *on upload only*, because the
|
||||
//! remote library is reachable by other clients and restructuring it under
|
||||
//! them is not ours to do.
|
||||
|
||||
use crate::{EntryKind, RemoteBackend, RemoteError, RemotePath, Validator};
|
||||
|
||||
/// The folder an original belongs in, under a library root.
|
||||
///
|
||||
/// Segments are joined one at a time rather than formatted into a string,
|
||||
/// because [`RemotePath::join`] is what normalises separators — a segment
|
||||
/// arriving with a stray slash must not silently become two levels.
|
||||
pub fn destination(library: &RemotePath, folders: &[String]) -> RemotePath {
|
||||
folders
|
||||
.iter()
|
||||
.filter(|s| !s.is_empty())
|
||||
.fold(library.clone(), |acc, seg| acc.join(seg))
|
||||
}
|
||||
|
||||
/// Upload one original into its dated folder.
|
||||
///
|
||||
/// Creates the folders if they are missing — `create_dir` makes parents and
|
||||
/// succeeds on one that already exists, so the second import of a day costs
|
||||
/// one request rather than a conflict.
|
||||
///
|
||||
/// Returns where it actually went, which may not be the name that was asked
|
||||
/// for: two cameras produce `IMG_0001.CR3` and the second must not overwrite
|
||||
/// the first. The caller records the returned path, never the one it passed.
|
||||
pub async fn upload_original(
|
||||
backend: &dyn RemoteBackend,
|
||||
library: &RemotePath,
|
||||
folders: &[String],
|
||||
name: &str,
|
||||
body: Vec<u8>,
|
||||
) -> Result<(RemotePath, Validator), RemoteError> {
|
||||
let dir = destination(library, folders);
|
||||
backend.create_dir(&dir).await?;
|
||||
|
||||
let name = free_name(backend, &dir, name).await?;
|
||||
let path = dir.join(&name);
|
||||
let validator = backend.put(&path, body, None).await?;
|
||||
Ok((path, validator))
|
||||
}
|
||||
|
||||
/// A name not already taken in a remote folder.
|
||||
///
|
||||
/// One listing rather than a probe per candidate: a day folder is listed in a
|
||||
/// single `PROPFIND` and the answer covers every collision in it, where
|
||||
/// probing would be a round trip per attempt over a link that may be mobile
|
||||
/// data.
|
||||
///
|
||||
/// A folder that cannot be listed yields the original name rather than an
|
||||
/// error. That is the safe direction on a server that has just been asked to
|
||||
/// create the folder: the alternative is failing an upload because a listing
|
||||
/// raced the `MKCOL`, and a genuine collision still fails at the `PUT` if the
|
||||
/// caller asked for a precondition.
|
||||
async fn free_name(
|
||||
backend: &dyn RemoteBackend,
|
||||
dir: &RemotePath,
|
||||
name: &str,
|
||||
) -> Result<String, RemoteError> {
|
||||
let taken: std::collections::HashSet<String> = match backend.list(dir, None).await {
|
||||
Ok(entries) => entries
|
||||
.into_iter()
|
||||
.filter(|e| e.kind == EntryKind::File)
|
||||
.map(|e| e.path.name().to_string())
|
||||
.collect(),
|
||||
Err(RemoteError::NotFound(_)) => Default::default(),
|
||||
Err(e) => return Err(e),
|
||||
};
|
||||
|
||||
if !taken.contains(name) {
|
||||
return Ok(name.to_string());
|
||||
}
|
||||
|
||||
// The same suffix convention the local import uses, so a photograph that
|
||||
// collided on both sides carries the same name in both places.
|
||||
let (stem, ext) = match name.rsplit_once('.') {
|
||||
Some((s, e)) if !s.is_empty() => (s, Some(e)),
|
||||
_ => (name, None),
|
||||
};
|
||||
for n in 1..1000 {
|
||||
let candidate = match ext {
|
||||
Some(ext) => format!("{stem}-{n}.{ext}"),
|
||||
None => format!("{stem}-{n}"),
|
||||
};
|
||||
if !taken.contains(&candidate) {
|
||||
return Ok(candidate);
|
||||
}
|
||||
}
|
||||
Err(RemoteError::Unsupported(
|
||||
"a thousand files of the same name in one folder",
|
||||
))
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use crate::types::{RemoteEntry, RemoteId};
|
||||
|
||||
fn segs(v: &[&str]) -> Vec<String> {
|
||||
v.iter().map(|s| s.to_string()).collect()
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn an_original_lands_under_year_then_day() {
|
||||
let dest = destination(
|
||||
&RemotePath::new("PhotosRaw"),
|
||||
&segs(&["2026", "2026-08-22"]),
|
||||
);
|
||||
assert_eq!(dest.as_str(), "PhotosRaw/2026/2026-08-22");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn an_empty_segment_does_not_leave_a_nameless_folder() {
|
||||
let dest = destination(&RemotePath::new("PhotosRaw"), &segs(&["2026", "", "x"]));
|
||||
assert_eq!(dest.as_str(), "PhotosRaw/2026/x");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_library_at_the_root_still_works() {
|
||||
let dest = destination(&RemotePath::root(), &segs(&["2026", "2026-08-22"]));
|
||||
assert_eq!(dest.as_str(), "2026/2026-08-22");
|
||||
}
|
||||
|
||||
// The upload path itself needs a backend, which is exercised against a
|
||||
// fake in `dr-sync-nextcloud`'s tests and against a real server in S8.
|
||||
// What is worth pinning here is the collision rule, since it decides
|
||||
// whether a second camera's `IMG_0001.CR3` overwrites the first.
|
||||
struct Listing(Vec<&'static str>);
|
||||
|
||||
#[async_trait::async_trait]
|
||||
impl RemoteBackend for Listing {
|
||||
fn capabilities(&self) -> &crate::Capabilities {
|
||||
unimplemented!("not reached by the collision tests")
|
||||
}
|
||||
fn name(&self) -> &str {
|
||||
"listing"
|
||||
}
|
||||
async fn list(
|
||||
&self,
|
||||
dir: &RemotePath,
|
||||
_since: Option<&Validator>,
|
||||
) -> Result<Vec<RemoteEntry>, RemoteError> {
|
||||
Ok(self
|
||||
.0
|
||||
.iter()
|
||||
.map(|n| RemoteEntry {
|
||||
id: RemoteId::Path(dir.join(n)),
|
||||
path: dir.join(n),
|
||||
kind: EntryKind::File,
|
||||
validator: Validator::new("v"),
|
||||
size: 0,
|
||||
modified: None,
|
||||
has_preview: false,
|
||||
})
|
||||
.collect())
|
||||
}
|
||||
async fn dir_validator(&self, _dir: &RemotePath) -> Result<Validator, RemoteError> {
|
||||
Err(RemoteError::Unsupported("test backend"))
|
||||
}
|
||||
async fn delta(
|
||||
&self,
|
||||
_cursor: &crate::Cursor,
|
||||
) -> Result<(Vec<crate::RemoteChange>, crate::Cursor), RemoteError> {
|
||||
Err(RemoteError::Unsupported("test backend"))
|
||||
}
|
||||
async fn get(
|
||||
&self,
|
||||
_id: &RemoteId,
|
||||
_range: Option<std::ops::Range<u64>>,
|
||||
) -> Result<Vec<u8>, RemoteError> {
|
||||
Err(RemoteError::Unsupported("test backend"))
|
||||
}
|
||||
async fn put(
|
||||
&self,
|
||||
_path: &RemotePath,
|
||||
_body: Vec<u8>,
|
||||
_precond: Option<crate::Precondition>,
|
||||
) -> Result<Validator, RemoteError> {
|
||||
Ok(Validator::new("v"))
|
||||
}
|
||||
async fn delete(
|
||||
&self,
|
||||
_id: &RemoteId,
|
||||
_precond: Option<crate::Precondition>,
|
||||
) -> Result<(), RemoteError> {
|
||||
Err(RemoteError::Unsupported("test backend"))
|
||||
}
|
||||
async fn move_to(&self, _from: &RemoteId, _to: &RemotePath) -> Result<(), RemoteError> {
|
||||
Err(RemoteError::Unsupported("test backend"))
|
||||
}
|
||||
async fn create_dir(&self, _path: &RemotePath) -> Result<(), RemoteError> {
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
/// Poll a future to completion on this thread.
|
||||
///
|
||||
/// A hand-rolled poll rather than a runtime dependency for two tests: the
|
||||
/// fake backend answers immediately, so a future that came back pending
|
||||
/// would mean the fake had grown a suspension the tests do not model.
|
||||
fn block_on<F: std::future::Future>(f: F) -> F::Output {
|
||||
use std::task::{Context, Poll, RawWaker, RawWakerVTable, Waker};
|
||||
fn noop(_: *const ()) {}
|
||||
fn clone(p: *const ()) -> RawWaker {
|
||||
RawWaker::new(p, &VTABLE)
|
||||
}
|
||||
static VTABLE: RawWakerVTable = RawWakerVTable::new(clone, noop, noop, noop);
|
||||
let waker = unsafe { Waker::from_raw(RawWaker::new(std::ptr::null(), &VTABLE)) };
|
||||
let mut cx = Context::from_waker(&waker);
|
||||
let mut f = std::pin::pin!(f);
|
||||
match f.as_mut().poll(&mut cx) {
|
||||
Poll::Ready(v) => v,
|
||||
Poll::Pending => panic!("the fake backend never yields"),
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_second_cameras_img_0001_does_not_overwrite_the_first() {
|
||||
let backend = Listing(vec!["IMG_0001.CR3"]);
|
||||
let (path, _) = block_on(upload_original(
|
||||
&backend,
|
||||
&RemotePath::new("PhotosRaw"),
|
||||
&segs(&["2026", "2026-08-22"]),
|
||||
"IMG_0001.CR3",
|
||||
vec![1, 2, 3],
|
||||
))
|
||||
.unwrap();
|
||||
assert_eq!(path.as_str(), "PhotosRaw/2026/2026-08-22/IMG_0001-1.CR3");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_free_name_is_used_as_it_is() {
|
||||
let backend = Listing(vec!["IMG_0002.CR3"]);
|
||||
let (path, _) = block_on(upload_original(
|
||||
&backend,
|
||||
&RemotePath::new("PhotosRaw"),
|
||||
&segs(&["2026", "2026-08-22"]),
|
||||
"IMG_0001.CR3",
|
||||
vec![1, 2, 3],
|
||||
))
|
||||
.unwrap();
|
||||
assert_eq!(path.as_str(), "PhotosRaw/2026/2026-08-22/IMG_0001.CR3");
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user