//! The background passes that reach beyond the loaded window: the day's //! catalog backup, the metadata sweep that dates every image rather than //! only the ones scrolled past, the whole-library thumbnail pass, and the //! exchange with the server that follows each of them. //! //! Split from `window` because these run on their own timers over the whole //! library rather than in response to what the grid is showing. //! See `docs/dev/code-health.md` CH-1. use std::rc::Rc; use slint::ComponentHandle; use crate::library; use crate::{AppWindow, Library}; use super::controller::{stop, LibraryController}; use super::open::schedule_reload; use super::timeline::refresh_timeline; use super::window::load_window; /// TRACES: NFR-R2 /// Take the day's catalog backup if one is due, off the UI thread. /// /// Decided cheaply first — [`dr_catalog::recovery::backup_due`] reads a /// directory listing, not the catalog — so the ordinary case of "backed up /// this morning already" costs no thread and no connection. fn spawn_scheduled_backup(ctl: &Rc) { let Some((conn, _)) = ctl.session.borrow().clone() else { return; }; let catalog_path = library::catalog_path(&conn.account); if !dr_catalog::recovery::backup_due(&catalog_path) { return; } std::thread::spawn(move || { let catalog = match dr_catalog::Catalog::open(&catalog_path) { Ok(c) => c, Err(e) => { log::warn!("scheduled backup: opening the catalog: {e}"); return; } }; if let Err(e) = dr_catalog::recovery::backup_if_due(catalog.connection(), &catalog_path) { log::warn!("scheduled backup: {e}"); } }); } /// Push shards and the catalog to the server, and take what it has. /// /// Fired when the scan completes, so a fresh device inherits its peers' /// work before deriving any of its own; after the sweep completes, when /// there is a finished index worth sharing; and from the Sync button for an /// explicit exchange. A pass still running when the next trigger fires is /// left to finish — the guard below. pub(super) fn start_derived_sync(window: &AppWindow, ctl: &Rc) { // An escape hatch for running the app against a *copied* library without // touching the account's real server. // // Added after doing exactly that by accident: a local test launch // completed its thumbnail sweep, which fires this, which pushed a test // catalog over the live one. Redirecting `XDG_DATA_HOME` isolates the // catalog and the thumbnails but not the server, and nothing said so. // // The guard belongs here rather than at the call sites: the sweep firing a // sync is correct — the point of building thumbnails is sharing them — and // a flag checked in three places would eventually be missed in a fourth. if std::env::var_os("DARKROOM_NO_SYNC").is_some() { log::info!("derived sync suppressed by DARKROOM_NO_SYNC"); return; } let Some((conn, _)) = ctl.session.borrow().clone() else { return; }; // Already running: a second pass would race the first over the same // scratch files. if ctl.sync_timer.borrow().is_some() && window.global::().get_library_syncing() { return; } // TRACES: FR-CAT-9 // Nothing to push to and nothing to take. Attempting it would upload // shards into a timeout and light the "Syncing…" indicator over work that // cannot start; the shards are unchanged on disk and go out on the next // sync once the server is back. if ctl.is_offline() { log::debug!("offline: skipping derived sync"); return; } let catalog_path = library::catalog_path(&conn.account); let scratch = catalog_path .parent() .map(|p| p.join("scratch")) .unwrap_or_else(std::env::temp_dir); let _ = std::fs::create_dir_all(&scratch); // TRACES: FR-EXP-7 | FR-NC-10 // Drain the export outbox on the same pass, and before the shards. An // export the user was told had succeeded is waiting here, and it is the // one thing in this directory that exists nowhere else — a thumbnail // shard can be rebuilt from the originals, and the catalog is an index. // // Fire-and-forget rather than reported: it runs on its own thread and // clears entries as they land, so a partial run leaves the rest queued // for next time and nothing is lost by not watching it. It reports // through the log until an export has a place in the activity list. { let outbox = crate::export::outbox_dir(&conn.account); if crate::export::pending_count(&outbox) > 0 { let rx = crate::export::spawn_upload(conn.clone(), conn.account.root.clone(), outbox); std::thread::spawn(move || { while let Ok(msg) = rx.recv() { match msg { crate::export::UploadMessage::Status(s) => log::info!("export: {s}"), crate::export::UploadMessage::Finished { uploaded, remaining, error, } => { log::info!("export: {uploaded} uploaded, {remaining} still queued"); if let Some(e) = error { log::warn!("export upload stopped: {e}"); } } } } }); } } window.global::().set_library_syncing(true); let rx = crate::derived_sync::spawn_sync( conn.clone(), conn.account.root.clone(), library::thumbs_dir(&conn.account), catalog_path, library::place_path(&conn.account), scratch, ctl.face_model_id.borrow().clone(), ); let timer = slint::Timer::default(); let weak = window.as_weak(); let ctl_cb = ctl.clone(); // The sync reports stages rather than counts, so it stays indeterminate and // says what stage it is in. let job = ctl .activity .begin(crate::activity::Kind::Sync, "Syncing with the server"); timer.start( slint::TimerMode::Repeated, std::time::Duration::from_millis(300), move || { let Some(w) = weak.upgrade() else { return }; loop { let msg = match rx.try_recv() { Ok(m) => m, Err(std::sync::mpsc::TryRecvError::Empty) => return, Err(std::sync::mpsc::TryRecvError::Disconnected) => { w.global::().set_library_syncing(false); job.fail("stopped without finishing"); stop(&ctl_cb.sync_timer); return; } }; match msg { crate::derived_sync::SyncMessage::Status(s) => { job.detail(s.clone()); w.global::().set_library_status(s.into()); } crate::derived_sync::SyncMessage::Finished(report) => { log::info!( "sync: {} shard(s) up, {} down ({} thumbnails), \ catalog {}{}{}", report.shards_uploaded, report.shards_downloaded, report.thumbnails_adopted, // Replacing a damaged copy is said apart from an // ordinary push: it is the one push that discarded // something, and a log line that called it "pushed" // would hide the only moment worth going back to. match (report.catalog_uploaded, report.catalog_replaced) { (true, true) => "pushed over a damaged copy", (true, false) => "pushed", _ => "not pushed", }, if report.collections_gained > 0 { format!(", {} collection(s) gained", report.collections_gained) } else { String::new() }, if report.dates_gained > 0 { format!(", {} date(s) gained", report.dates_gained) } else { String::new() } ); w.global::().set_library_syncing(false); let summary = format!( "{} shard(s) up, {} down", report.shards_uploaded, report.shards_downloaded ); job.finish(if report.did_anything() { summary.clone() } else { "nothing to exchange".to_string() }); if report.did_anything() { w.global::() .set_library_status(format!("synced · {summary}").into()); } // Adopted thumbnails and merged collections both change // what the grid should show; adopted dates change its // order, and the timeline beside it. if report.thumbnails_adopted > 0 || report.collections_gained > 0 || report.members_gained > 0 || report.dates_gained > 0 { load_window(&w, &ctl_cb); } if report.dates_gained > 0 { let borrow = ctl_cb.catalog(); let borrow = borrow.borrow(); if let Some(cat) = borrow.as_ref() { refresh_timeline(&w, cat, &ctl_cb); } } // TRACES: FR-CAT-7 // And the sidebar, which the grid reload does not // touch. Membership counts as a change: a sync that // files 127 photographs into a collection both devices // already had gains no *collection*, so keying this on // `collections_gained` alone left the tree showing no // number beside a collection that had just been filled. // // The scan already does this after it finishes; the // sync merges the same tables and did not. if report.collections_gained > 0 || report.members_gained > 0 { let coll = ctl_cb.coll_ctl.borrow().as_ref().and_then(|w| w.upgrade()); if let Some(coll) = coll { let borrow = ctl_cb.catalog(); let borrow = borrow.borrow(); if let Some(cat) = borrow.as_ref() { crate::collections_ui::refresh_tree(&w, &coll, cat); } } } stop(&ctl_cb.sync_timer); return; } crate::derived_sync::SyncMessage::Failed(e) => { log::warn!("sync failed: {e}"); job.fail(e.to_string()); w.global::().set_library_syncing(false); // Not an error banner: a failed sync costs nothing — // everything is still local and the next pass retries. w.global::() .set_library_status(format!("sync failed: {e}").into()); stop(&ctl_cb.sync_timer); return; } } } }, ); *ctl.sync_timer.borrow_mut() = Some(timer); } /// Start the whole-library sweep and report its progress. /// /// The grid only ever fetches what is on screen, so without this the timeline /// describes the fraction of the library that happened to be scrolled past. /// This covers the rest. pub(super) fn start_sweep(window: &AppWindow, ctl: &Rc) { let Some((conn, _)) = ctl.session.borrow().clone() else { return; }; // TRACES: FR-CAT-9 // The sweep is nothing but network reads — one header fetch per undated // image, across the whole library. Offline it would spend a timeout on // every one of them, running for hours to learn nothing, while the // progress bar implied work was happening. It resumes on reconnect, and // the images it has already dated stay dated. if ctl.is_offline() { log::debug!("offline: not starting the metadata sweep"); return; } let rx = library::spawn_sweep(conn.clone(), library::catalog_path(&conn.account)); let timer = slint::Timer::default(); let weak = window.as_weak(); let ctl_cb = ctl.clone(); // Hours on a large library, and entirely invisible outside the grid until // now: the register is where a user who has gone to develop can still see // that indexing is running and how far it has got. let job = ctl .activity .begin(crate::activity::Kind::Index, "Indexing capture times"); timer.start( slint::TimerMode::Repeated, // Slower than the thumbnail drain: this runs for tens of minutes and // its progress does not need per-frame accuracy. std::time::Duration::from_millis(400), move || { let Some(w) = weak.upgrade() else { return }; loop { let msg = match rx.try_recv() { Ok(m) => m, Err(std::sync::mpsc::TryRecvError::Empty) => return, Err(std::sync::mpsc::TryRecvError::Disconnected) => { w.global::().set_library_sweep_total(0); job.fail("stopped without finishing"); stop(&ctl_cb.sweep_timer); return; } }; match msg { library::SweepMessage::Total(n) => { w.global::().set_library_sweep_total(n as i32); w.global::().set_library_sweep_done(0); job.total(n); } library::SweepMessage::Progress { done, dated } => { w.global::().set_library_sweep_done(done as i32); job.progress( done, w.global::().get_library_sweep_total() as usize, ); // Rebuild as it goes: the histogram growing while the // sweep runs is the visible sign it is working. if dated > 0 { let borrow = ctl_cb.catalog.borrow(); if let Some(catalog) = borrow.as_ref() { refresh_timeline(&w, catalog, &ctl_cb); } } } library::SweepMessage::Finished { dated } => { log::info!("sweep finished: {dated} dated"); job.finish(format!("{dated} dated")); w.global::().set_library_sweep_total(0); { let borrow = ctl_cb.catalog.borrow(); if let Some(catalog) = borrow.as_ref() { refresh_timeline(&w, catalog, &ctl_cb); } } // TRACES: NFR-R2 // The scheduled backup, at the moment the catalog is // quietest and a day's edits have just been folded in. // On its own thread, with its own connection: a copy // of a 130 MB file is a second or two the Slint loop // must not spend, and it runs beside the sync below, // which is only a reader of the same file. spawn_scheduled_backup(&ctl_cb); // Now that indexing is complete, hand the result to the // server so a second device inherits it rather than // repeating hours of range fetches. start_derived_sync(&w, &ctl_cb); stop(&ctl_cb.sweep_timer); return; } } } }, ); *ctl.sweep_timer.borrow_mut() = Some(timer); } /// TRACES: FR-CAT-3 | FR-NC-3 | FR-NC-7 /// Thumbnail every photograph in the library, then hand the shards over. /// /// # Why it is asked for rather than assumed /// /// The grid's own fetching is demand-driven on purpose: a remote library is /// browsed over a connection that must not be saturated to show one screen /// (FR-NC-3). But that leaves the thumbnail store holding only what has been /// looked at, and the store is the one derived thing worth syncing — a second /// device that downloads the shards gets a full grid without touching a RAW. /// So the complete set is worth an hour of transfers *once*, on a machine /// plugged in, at a moment the user chose. That is this. /// /// The sync at the end is not a separate courtesy: a filled store that never /// leaves this device is most of the cost for none of the point. pub(super) fn start_thumbnail_sweep(window: &AppWindow, ctl: &Rc) { let Some((conn, _)) = ctl.session.borrow().clone() else { return; }; // Already running. A second pass would re-fetch everything the first is // part way through — the work list is built from what the store lacks, and // the first pass has not filled it yet. if ctl.thumb_sweep_timer.borrow().is_some() && window.global::().get_library_thumbnailing() { return; } // TRACES: FR-CAT-9 // Every image here is a range fetch. Offline the pass would spend a // timeout per photograph and store nothing, so it is refused rather than // started — and said so, because this one was pressed deliberately and a // button that silently does nothing is worse than one that declines. if ctl.is_offline() { log::debug!("offline: not starting the thumbnail pass"); window .global::() .set_library_status("Offline — thumbnailing needs the server.".into()); return; } window.global::().set_library_thumbnailing(true); let rx = library::spawn_thumbnail_sweep( conn.clone(), library::catalog_path(&conn.account), library::thumbs_dir(&conn.account), ); let timer = slint::Timer::default(); let weak = window.as_weak(); let ctl_cb = ctl.clone(); let job = ctl.activity.begin( crate::activity::Kind::Thumbnails, "Thumbnailing the library", ); // The total arrives in the first message rather than up front — counting // it means asking the store about every image — and the progress messages // after it carry only cumulative counts, so it is kept here rather than // re-derived. let total = std::cell::Cell::new(0usize); timer.start( slint::TimerMode::Repeated, // As slow as the metadata sweep's drain, and for the same reason: this // runs for tens of minutes and a chunk lands every few seconds. std::time::Duration::from_millis(400), move || { let Some(w) = weak.upgrade() else { return }; loop { let msg = match rx.try_recv() { Ok(m) => m, Err(std::sync::mpsc::TryRecvError::Empty) => return, Err(std::sync::mpsc::TryRecvError::Disconnected) => { w.global::().set_library_thumbnailing(false); job.fail("stopped without finishing"); stop(&ctl_cb.thumb_sweep_timer); return; } }; match msg { library::ThumbSweepMessage::Total(n) => { total.set(n); job.total(n); w.global::() .set_library_status(format!("Thumbnailing {n} photograph(s)…").into()); } library::ThumbSweepMessage::Progress { done, stored } => { job.progress(done, total.get()); // Dates arrive on the same headers, so the histogram // grows as this runs — the visible sign it is working // on a page that is not the grid. if stored > 0 { let borrow = ctl_cb.catalog.borrow(); if let Some(catalog) = borrow.as_ref() { refresh_timeline(&w, catalog, &ctl_cb); } } } library::ThumbSweepMessage::Finished { stored, failed, offline, } => { log::info!( "thumbnail pass finished: {stored} stored, {failed} without a preview" ); w.global::().set_library_thumbnailing(false); stop(&ctl_cb.thumb_sweep_timer); let summary = if offline { format!("{stored} stored · stopped, server unreachable") } else if failed > 0 { format!("{stored} stored · {failed} with no usable preview") } else { format!("{stored} stored") }; w.global::() .set_library_status(summary.clone().into()); // Finished rather than failed even when the server // dropped: the pass is resumable and what it did build // is stored, and a red row would invite the user to // redo work that is already done. job.finish(summary); { let borrow = ctl_cb.catalog.borrow(); if let Some(catalog) = borrow.as_ref() { refresh_timeline(&w, catalog, &ctl_cb); } } // Cells that showed a placeholder have pixels now, and // the grid asks the store once per photograph — so // without this the library the pass just thumbnailed // stays blank until something else reloads the window. if w.get_library_visible() { ctl_cb.requested.borrow_mut().clear(); schedule_reload(&w, &ctl_cb); } // The point of the pass: the shards go to the server so // every other device inherits them. Skipped when the // connection is already gone — the sync would only // discover the same thing more slowly. if !offline { start_derived_sync(&w, &ctl_cb); } // And now every photograph in the library has a // thumbnail, which is the only moment all of its burst // signatures can be computed. See `bursts::start_pass`, // which owns the pass and everything it needs to drain // itself; what it wants from here is the paths and a // way to say the grid has something new to draw. if let Some((conn, _)) = ctl_cb.session.borrow().clone() { let weak_after = w.as_weak(); let ctl_after = ctl_cb.clone(); crate::bursts::start_pass( library::catalog_path(&conn.account), library::thumbs_dir(&conn.account), move |bursts| { let Some(w) = weak_after.upgrade() else { return; }; if bursts > 0 && w.get_library_visible() { schedule_reload(&w, &ctl_after); } }, ); } return; } } } }, ); *ctl.thumb_sweep_timer.borrow_mut() = Some(timer); }