//! TRACES: FR-EXP-6 | FR-EXP-7 | FR-NC-10 //! Placing an exported file, and the cache that makes offline unremarkable. //! //! [`dr_export`] turns a frame into bytes and a name and stops there, because //! where those bytes go differs by more than a path. This is the other half: //! it decides the destination and gets them there. //! //! # Everything is staged first //! //! An export to the server is written to a local **outbox** before any upload //! is attempted, and the upload drains that outbox afterwards. Not a fallback //! for the offline case — the *only* path, with offline merely meaning the //! drain finds nothing to do. //! //! Doing it the other way, uploading directly and staging only on failure, //! looks simpler and has two bad properties. The failure path is then the one //! that is rarely exercised and always broken, and the moment the network //! drops mid-batch some exports exist and some do not with nothing recording //! which. Staging first means an export is *finished* the instant it is //! written; the upload is a separate promise the app keeps later. //! //! # Why the outbox is not in the cache //! //! It lives beside the catalog, with the thumbnail shards, rather than under //! the evictable cache. `dr_catalog::cache` draws the line already: passively //! cached originals are a convenience and go under LRU, pinned ones are a //! promise and never do. An export waiting to upload is a promise — the user //! was told the export succeeded — and sweeping it away to reclaim disk would //! destroy work that no longer exists anywhere else. use std::path::{Path, PathBuf}; use dr_export::Encoded; use dr_sync::{RemoteBackend, RemotePath}; use dr_sync_nextcloud::{AppCredentials, NextcloudBackend}; use dr_types::ExportTarget; /// Where exports wait for a server that is not there yet. /// /// Beside the catalog, for the reason in the module docs. Per account, /// because the destination folder is a path on one particular server and an /// entry queued for one account is meaningless to another. pub fn outbox_dir(server: &str, user_id: &str) -> PathBuf { crate::library::catalog_path(server, user_id) .parent() .map(|p| p.join("outbox")) .unwrap_or_else(|| std::env::temp_dir().join("darkroom-outbox")) } /// One export waiting to go up. /// /// The record sits beside the bytes as `.dest`, holding the remote /// folder it belongs in. A single flat file rather than a database: the queue /// is small, the entries are independent, and the recovery story for a /// half-written text file is to ignore it — which is exactly what parsing it /// does. #[derive(Debug, Clone, PartialEq, Eq)] pub struct Pending { /// The staged bytes on this device. pub local: PathBuf, /// Remote folder, relative to the library root. Empty means the root. pub remote_dir: String, /// The filename to give it there. pub name: String, } impl Pending { /// Full remote path for this entry, under `root`. fn remote_path(&self, root: &str) -> RemotePath { let mut parts: Vec<&str> = Vec::new(); for segment in [root, self.remote_dir.as_str()] { for part in segment.split('/') { if !part.is_empty() { parts.push(part); } } } parts.push(&self.name); RemotePath::new(parts.join("/")) } /// The folder this entry's file belongs in, as a remote path. fn remote_folder(&self, root: &str) -> RemotePath { let mut parts: Vec<&str> = Vec::new(); for segment in [root, self.remote_dir.as_str()] { for part in segment.split('/') { if !part.is_empty() { parts.push(part); } } } RemotePath::new(parts.join("/")) } } /// Where an export was put, for the interface to report. #[derive(Debug, Clone, PartialEq, Eq)] pub enum Placed { /// Written straight to a folder on this device. Device(PathBuf), /// Staged locally, awaiting upload to the named remote folder. Queued { local: PathBuf, remote_dir: String }, } impl Placed { /// A sentence for the status line. pub fn describe(&self) -> String { match self { Placed::Device(path) => format!( "Exported {}", path.file_name() .map(|n| n.to_string_lossy().into_owned()) .unwrap_or_default() ), // Named as queued rather than exported: the file is real and // finished, but it is not yet where the user asked for it, and // saying "exported to Nextcloud" before it has uploaded would be // a claim the app cannot keep if the disk is pulled. Placed::Queued { remote_dir, .. } => { let dir = if remote_dir.is_empty() { "the library root".to_string() } else { remote_dir.clone() }; format!("Queued for {dir}") } } } } /// Write an encoded export to wherever the settings say it goes. /// /// `destination` is a filesystem path for [`ExportTarget::Device`] and a /// remote folder for [`ExportTarget::Remote`] — the widening `ExportSettings` /// documents, resolved here because this is the layer that knows what a path /// means on this platform. pub fn place( encoded: &Encoded, target: ExportTarget, destination: &str, outbox: &Path, ) -> Result { match target { ExportTarget::Device => { if destination.trim().is_empty() { return Err("No export folder is set. Choose one in Settings.".into()); } let dir = PathBuf::from(destination); std::fs::create_dir_all(&dir).map_err(|e| format!("{}: {e}", dir.display()))?; let path = dir.join(&encoded.name); std::fs::write(&path, &encoded.bytes) .map_err(|e| format!("{}: {e}", path.display()))?; Ok(Placed::Device(path)) } ExportTarget::Remote => { let local = stage(encoded, destination, outbox)?; Ok(Placed::Queued { local, remote_dir: destination.to_string(), }) } } } /// Write bytes and their destination record into the outbox. fn stage(encoded: &Encoded, remote_dir: &str, outbox: &Path) -> Result { std::fs::create_dir_all(outbox).map_err(|e| format!("{}: {e}", outbox.display()))?; // The staged name is the export's name, deduplicated against the outbox // rather than against the server: two exports queued before either has // uploaded would otherwise overwrite each other here, and the second one // would silently replace the first before anybody saw it. let mut candidate = outbox.join(&encoded.name); let mut n = 1; while candidate.exists() { let stem = Path::new(&encoded.name) .file_stem() .map(|s| s.to_string_lossy().into_owned()) .unwrap_or_else(|| "export".into()); let ext = Path::new(&encoded.name) .extension() .map(|s| s.to_string_lossy().into_owned()) .unwrap_or_default(); candidate = outbox.join(format!("{stem}-{n}.{ext}")); n += 1; if n > 10_000 { return Err("the outbox is full of files by this name".into()); } } // Bytes first, then the record. The order matters on a process that may // be killed between the two: an orphan payload with no record is ignored // by the drain and swept later, where a record naming bytes that were // never written would be a permanent failure retried forever. std::fs::write(&candidate, &encoded.bytes) .map_err(|e| format!("{}: {e}", candidate.display()))?; let record = candidate.with_extension(format!( "{}.dest", candidate .extension() .map(|s| s.to_string_lossy().into_owned()) .unwrap_or_default() )); // The remote folder and the intended name, one per line. Not JSON: two // strings do not need a parser, and a format a human can repair by hand // is worth something for a queue holding the only copy of someone's work. std::fs::write(&record, format!("{remote_dir}\n{}\n", encoded.name)) .map_err(|e| format!("{}: {e}", record.display()))?; Ok(candidate) } /// Everything currently waiting in the outbox. /// /// A payload with no record is skipped rather than guessed at — see the write /// order in [`stage`]. pub fn pending(outbox: &Path) -> Vec { let Ok(entries) = std::fs::read_dir(outbox) else { return Vec::new(); }; let mut out = Vec::new(); for entry in entries.flatten() { let path = entry.path(); if path.extension().and_then(|e| e.to_str()) != Some("dest") { continue; } // `photo.jpg.dest` describes `photo.jpg`. let local = path.with_extension(""); if !local.exists() { continue; } let Ok(text) = std::fs::read_to_string(&path) else { continue; }; let mut lines = text.lines(); let remote_dir = lines.next().unwrap_or("").to_string(); let name = lines.next().unwrap_or("").to_string(); if name.is_empty() { continue; } out.push(Pending { local, remote_dir, name, }); } // Stable order so a drain is reproducible and a stuck entry is obvious // rather than appearing to move around the queue. out.sort_by(|a, b| a.local.cmp(&b.local)); out } /// How many exports are waiting. For the interface to show, and cheap enough /// to call on a redraw. pub fn pending_count(outbox: &Path) -> usize { pending(outbox).len() } /// Remove an entry and its record, once it is safely on the server. fn clear(entry: &Pending) { let record = PathBuf::from(format!("{}.dest", entry.local.display())); let _ = std::fs::remove_file(&entry.local); let _ = std::fs::remove_file(&record); } /// Progress from the upload worker. #[derive(Debug)] pub enum UploadMessage { Status(String), /// Uploaded, still pending, and the first error if there was one. Finished { uploaded: usize, remaining: usize, error: Option, }, } /// Drain the outbox to the server. /// /// Its own thread with its own runtime, like every other network path here — /// the Slint loop must never block (NFR-P9). /// /// A failure leaves the entry in place and stops the run. Continuing past a /// network error would burn the whole queue against a server that is not /// answering, and the next pass costs nothing. pub fn spawn_upload( creds: AppCredentials, user_id: String, root: String, outbox: PathBuf, ) -> std::sync::mpsc::Receiver { let (tx, rx) = std::sync::mpsc::channel(); std::thread::spawn(move || { let rt = match crate::net_runtime::build() { Ok(rt) => rt, Err(e) => { let _ = tx.send(UploadMessage::Finished { uploaded: 0, remaining: pending_count(&outbox), error: Some(e.to_string()), }); return; } }; rt.block_on(async { let backend = match NextcloudBackend::new(&creds, &user_id) { Ok(b) => b, Err(e) => { let _ = tx.send(UploadMessage::Finished { uploaded: 0, remaining: pending_count(&outbox), error: Some(e.to_string()), }); return; } }; let queue = pending(&outbox); let total = queue.len(); let mut uploaded = 0; let mut error = None; for (i, entry) in queue.iter().enumerate() { let _ = tx.send(UploadMessage::Status(format!( "uploading {} ({}/{total})", entry.name, i + 1 ))); let Ok(bytes) = std::fs::read(&entry.local) else { // The payload vanished under us. Drop the record too; // retrying forever against a file that is gone helps // nobody. clear(entry); continue; }; // The folder may not exist — this is the first export into it // — and `create_dir` treats "already there" as success, so it // is unconditional rather than guarded by a check that would // cost a request every time. if let Err(e) = backend.create_dir(&entry.remote_folder(&root)).await { error = Some(e.to_string()); break; } match backend.put(&entry.remote_path(&root), bytes, None).await { Ok(_) => { clear(entry); uploaded += 1; } Err(e) => { error = Some(e.to_string()); break; } } } let _ = tx.send(UploadMessage::Finished { uploaded, remaining: pending_count(&outbox), error, }); }); }); rx } #[cfg(test)] mod tests { use super::*; fn encoded(name: &str, bytes: &[u8]) -> Encoded { Encoded { name: name.to_string(), bytes: bytes.to_vec(), width: 4, height: 4, } } fn tmp() -> PathBuf { let dir = std::env::temp_dir().join(format!( "dr-outbox-test-{}-{:?}", std::process::id(), std::thread::current().id() )); let _ = std::fs::remove_dir_all(&dir); std::fs::create_dir_all(&dir).unwrap(); dir } #[test] fn a_device_export_writes_the_file() { let dir = tmp(); let target = dir.join("exports"); let placed = place( &encoded("a.jpg", b"hello"), ExportTarget::Device, target.to_str().unwrap(), &dir.join("outbox"), ) .unwrap(); assert_eq!(placed, Placed::Device(target.join("a.jpg"))); assert_eq!(std::fs::read(target.join("a.jpg")).unwrap(), b"hello"); } #[test] fn a_device_export_creates_a_folder_that_is_not_there() { // Exporting into a folder the user typed but has not made is the // common case, not an error. let dir = tmp(); let target = dir.join("deep/nested/exports"); assert!(place( &encoded("a.jpg", b"x"), ExportTarget::Device, target.to_str().unwrap(), &dir, ) .is_ok()); assert!(target.join("a.jpg").exists()); } #[test] fn a_device_export_with_no_folder_says_so() { // Rather than writing to the process's working directory, which is // wherever the app happened to be launched from. let dir = tmp(); let err = place(&encoded("a.jpg", b"x"), ExportTarget::Device, " ", &dir).unwrap_err(); assert!(err.contains("Settings"), "unhelpful message: {err}"); } #[test] fn a_remote_export_is_staged_rather_than_sent() { // The property the offline story rests on: the export is complete on // disk before any network call is attempted. let dir = tmp(); let outbox = dir.join("outbox"); let placed = place( &encoded("a.jpg", b"hello"), ExportTarget::Remote, "Exports/2026", &outbox, ) .unwrap(); match placed { Placed::Queued { local, remote_dir } => { assert_eq!(std::fs::read(&local).unwrap(), b"hello"); assert_eq!(remote_dir, "Exports/2026"); } other => panic!("expected a queued export, got {other:?}"), } } #[test] fn a_staged_export_is_found_again_with_its_destination() { // What survives a process death: the drain has to be able to // reconstruct where a file was going from the disk alone. let dir = tmp(); let outbox = dir.join("outbox"); place( &encoded("a.jpg", b"one"), ExportTarget::Remote, "Exports", &outbox, ) .unwrap(); let queue = pending(&outbox); assert_eq!(queue.len(), 1); assert_eq!(queue[0].remote_dir, "Exports"); assert_eq!(queue[0].name, "a.jpg"); } #[test] fn two_exports_of_the_same_name_both_survive_the_outbox() { // Both were asked for and neither has uploaded, so the second must // not overwrite the first while it waits. let dir = tmp(); let outbox = dir.join("outbox"); place( &encoded("a.jpg", b"one"), ExportTarget::Remote, "E", &outbox, ) .unwrap(); place( &encoded("a.jpg", b"two"), ExportTarget::Remote, "E", &outbox, ) .unwrap(); let queue = pending(&outbox); assert_eq!(queue.len(), 2); // Both still claim the name they should arrive under; only the local // staging name differs. assert!(queue.iter().all(|p| p.name == "a.jpg")); assert_ne!(queue[0].local, queue[1].local); } #[test] fn a_payload_with_no_record_is_ignored() { // The window a kill between the two writes leaves behind. It must not // become an upload to nowhere. let dir = tmp(); let outbox = dir.join("outbox"); std::fs::create_dir_all(&outbox).unwrap(); std::fs::write(outbox.join("orphan.jpg"), b"x").unwrap(); assert!(pending(&outbox).is_empty()); } #[test] fn a_record_with_no_payload_is_ignored() { let dir = tmp(); let outbox = dir.join("outbox"); std::fs::create_dir_all(&outbox).unwrap(); std::fs::write(outbox.join("ghost.jpg.dest"), "E\nghost.jpg\n").unwrap(); assert!(pending(&outbox).is_empty()); } #[test] fn an_empty_outbox_is_not_an_error() { // Called on every sync pass, including before anything is exported // and on a device where the directory has never been created. assert!(pending(Path::new("/nonexistent/darkroom/outbox")).is_empty()); assert_eq!(pending_count(Path::new("/nonexistent/darkroom/outbox")), 0); } #[test] fn the_remote_path_joins_root_folder_and_name() { let entry = Pending { local: PathBuf::from("/tmp/a.jpg"), remote_dir: "Exports/2026".into(), name: "a.jpg".into(), }; assert_eq!( entry.remote_path("Photos").as_str(), "Photos/Exports/2026/a.jpg" ); assert_eq!( entry.remote_folder("Photos").as_str(), "Photos/Exports/2026" ); } #[test] fn an_empty_folder_exports_to_the_library_root() { // "Ask each time" is not set here — an empty remote folder means the // root, and it must not produce a double slash the server rejects. let entry = Pending { local: PathBuf::from("/tmp/a.jpg"), remote_dir: String::new(), name: "a.jpg".into(), }; assert_eq!(entry.remote_path("Photos").as_str(), "Photos/a.jpg"); } #[test] fn stray_slashes_do_not_produce_an_unusable_path() { // The folder comes from a picker or a text field, and either can hand // over a leading or trailing slash. let entry = Pending { local: PathBuf::from("/tmp/a.jpg"), remote_dir: "/Exports/".into(), name: "a.jpg".into(), }; assert_eq!( entry.remote_path("/Photos/").as_str(), "Photos/Exports/a.jpg" ); } #[test] fn the_status_line_never_claims_an_upload_that_has_not_happened() { // A queued export is real and finished, but it is not on the server, // and saying so before it is would be a promise the app cannot keep. let queued = Placed::Queued { local: PathBuf::from("/tmp/a.jpg"), remote_dir: "Exports".into(), }; let text = queued.describe(); assert!(text.contains("Queued"), "{text}"); assert!(!text.contains("Exported"), "{text}"); assert!(Placed::Device(PathBuf::from("/tmp/a.jpg")) .describe() .contains("Exported")); } }