567 lines
26 KiB
Rust
567 lines
26 KiB
Rust
//! The background passes that reach beyond the loaded window: the day's
|
|
//! catalog backup, the metadata sweep that dates every image rather than
|
|
//! only the ones scrolled past, the whole-library thumbnail pass, and the
|
|
//! exchange with the server that follows each of them.
|
|
//!
|
|
//! Split from `window` because these run on their own timers over the whole
|
|
//! library rather than in response to what the grid is showing.
|
|
//! See `docs/dev/code-health.md` CH-1.
|
|
|
|
use std::rc::Rc;
|
|
|
|
use slint::ComponentHandle;
|
|
|
|
use crate::library;
|
|
use crate::{AppWindow, Library};
|
|
|
|
use super::controller::{stop, LibraryController};
|
|
use super::open::schedule_reload;
|
|
use super::timeline::refresh_timeline;
|
|
use super::window::load_window;
|
|
|
|
/// TRACES: NFR-R2
|
|
/// Take the day's catalog backup if one is due, off the UI thread.
|
|
///
|
|
/// Decided cheaply first — [`dr_catalog::recovery::backup_due`] reads a
|
|
/// directory listing, not the catalog — so the ordinary case of "backed up
|
|
/// this morning already" costs no thread and no connection.
|
|
fn spawn_scheduled_backup(ctl: &Rc<LibraryController>) {
|
|
let Some((conn, _)) = ctl.session.borrow().clone() else {
|
|
return;
|
|
};
|
|
let catalog_path = library::catalog_path(&conn.account);
|
|
if !dr_catalog::recovery::backup_due(&catalog_path) {
|
|
return;
|
|
}
|
|
std::thread::spawn(move || {
|
|
let catalog = match dr_catalog::Catalog::open(&catalog_path) {
|
|
Ok(c) => c,
|
|
Err(e) => {
|
|
log::warn!("scheduled backup: opening the catalog: {e}");
|
|
return;
|
|
}
|
|
};
|
|
if let Err(e) = dr_catalog::recovery::backup_if_due(catalog.connection(), &catalog_path) {
|
|
log::warn!("scheduled backup: {e}");
|
|
}
|
|
});
|
|
}
|
|
|
|
/// Push shards and the catalog to the server, and take what it has.
|
|
///
|
|
/// Fired when the scan completes, so a fresh device inherits its peers'
|
|
/// work before deriving any of its own; after the sweep completes, when
|
|
/// there is a finished index worth sharing; and from the Sync button for an
|
|
/// explicit exchange. A pass still running when the next trigger fires is
|
|
/// left to finish — the guard below.
|
|
pub(super) fn start_derived_sync(window: &AppWindow, ctl: &Rc<LibraryController>) {
|
|
// An escape hatch for running the app against a *copied* library without
|
|
// touching the account's real server.
|
|
//
|
|
// Added after doing exactly that by accident: a local test launch
|
|
// completed its thumbnail sweep, which fires this, which pushed a test
|
|
// catalog over the live one. Redirecting `XDG_DATA_HOME` isolates the
|
|
// catalog and the thumbnails but not the server, and nothing said so.
|
|
//
|
|
// The guard belongs here rather than at the call sites: the sweep firing a
|
|
// sync is correct — the point of building thumbnails is sharing them — and
|
|
// a flag checked in three places would eventually be missed in a fourth.
|
|
if std::env::var_os("DARKROOM_NO_SYNC").is_some() {
|
|
log::info!("derived sync suppressed by DARKROOM_NO_SYNC");
|
|
return;
|
|
}
|
|
|
|
let Some((conn, _)) = ctl.session.borrow().clone() else {
|
|
return;
|
|
};
|
|
// Already running: a second pass would race the first over the same
|
|
// scratch files.
|
|
if ctl.sync_timer.borrow().is_some() && window.global::<Library>().get_library_syncing() {
|
|
return;
|
|
}
|
|
|
|
// TRACES: FR-CAT-9
|
|
// Nothing to push to and nothing to take. Attempting it would upload
|
|
// shards into a timeout and light the "Syncing…" indicator over work that
|
|
// cannot start; the shards are unchanged on disk and go out on the next
|
|
// sync once the server is back.
|
|
if ctl.is_offline() {
|
|
log::debug!("offline: skipping derived sync");
|
|
return;
|
|
}
|
|
|
|
let catalog_path = library::catalog_path(&conn.account);
|
|
let scratch = catalog_path
|
|
.parent()
|
|
.map(|p| p.join("scratch"))
|
|
.unwrap_or_else(std::env::temp_dir);
|
|
let _ = std::fs::create_dir_all(&scratch);
|
|
|
|
// TRACES: FR-EXP-7 | FR-NC-10
|
|
// Drain the export outbox on the same pass, and before the shards. An
|
|
// export the user was told had succeeded is waiting here, and it is the
|
|
// one thing in this directory that exists nowhere else — a thumbnail
|
|
// shard can be rebuilt from the originals, and the catalog is an index.
|
|
//
|
|
// Fire-and-forget rather than reported: it runs on its own thread and
|
|
// clears entries as they land, so a partial run leaves the rest queued
|
|
// for next time and nothing is lost by not watching it. It reports
|
|
// through the log until an export has a place in the activity list.
|
|
{
|
|
let outbox = crate::export::outbox_dir(&conn.account);
|
|
if crate::export::pending_count(&outbox) > 0 {
|
|
let rx = crate::export::spawn_upload(conn.clone(), conn.account.root.clone(), outbox);
|
|
std::thread::spawn(move || {
|
|
while let Ok(msg) = rx.recv() {
|
|
match msg {
|
|
crate::export::UploadMessage::Status(s) => log::info!("export: {s}"),
|
|
crate::export::UploadMessage::Finished {
|
|
uploaded,
|
|
remaining,
|
|
error,
|
|
} => {
|
|
log::info!("export: {uploaded} uploaded, {remaining} still queued");
|
|
if let Some(e) = error {
|
|
log::warn!("export upload stopped: {e}");
|
|
}
|
|
}
|
|
}
|
|
}
|
|
});
|
|
}
|
|
}
|
|
|
|
window.global::<Library>().set_library_syncing(true);
|
|
let rx = crate::derived_sync::spawn_sync(
|
|
conn.clone(),
|
|
conn.account.root.clone(),
|
|
library::thumbs_dir(&conn.account),
|
|
catalog_path,
|
|
library::place_path(&conn.account),
|
|
scratch,
|
|
ctl.face_model_id.borrow().clone(),
|
|
);
|
|
|
|
let timer = slint::Timer::default();
|
|
let weak = window.as_weak();
|
|
let ctl_cb = ctl.clone();
|
|
|
|
// The sync reports stages rather than counts, so it stays indeterminate and
|
|
// says what stage it is in.
|
|
let job = ctl
|
|
.activity
|
|
.begin(crate::activity::Kind::Sync, "Syncing with the server");
|
|
|
|
timer.start(
|
|
slint::TimerMode::Repeated,
|
|
std::time::Duration::from_millis(300),
|
|
move || {
|
|
let Some(w) = weak.upgrade() else { return };
|
|
loop {
|
|
let msg = match rx.try_recv() {
|
|
Ok(m) => m,
|
|
Err(std::sync::mpsc::TryRecvError::Empty) => return,
|
|
Err(std::sync::mpsc::TryRecvError::Disconnected) => {
|
|
w.global::<Library>().set_library_syncing(false);
|
|
job.fail("stopped without finishing");
|
|
stop(&ctl_cb.sync_timer);
|
|
return;
|
|
}
|
|
};
|
|
|
|
match msg {
|
|
crate::derived_sync::SyncMessage::Status(s) => {
|
|
job.detail(s.clone());
|
|
w.global::<Library>().set_library_status(s.into());
|
|
}
|
|
crate::derived_sync::SyncMessage::Finished(report) => {
|
|
log::info!(
|
|
"sync: {} shard(s) up, {} down ({} thumbnails), \
|
|
catalog {}{}{}",
|
|
report.shards_uploaded,
|
|
report.shards_downloaded,
|
|
report.thumbnails_adopted,
|
|
// Replacing a damaged copy is said apart from an
|
|
// ordinary push: it is the one push that discarded
|
|
// something, and a log line that called it "pushed"
|
|
// would hide the only moment worth going back to.
|
|
match (report.catalog_uploaded, report.catalog_replaced) {
|
|
(true, true) => "pushed over a damaged copy",
|
|
(true, false) => "pushed",
|
|
_ => "not pushed",
|
|
},
|
|
if report.collections_gained > 0 {
|
|
format!(", {} collection(s) gained", report.collections_gained)
|
|
} else {
|
|
String::new()
|
|
},
|
|
if report.dates_gained > 0 {
|
|
format!(", {} date(s) gained", report.dates_gained)
|
|
} else {
|
|
String::new()
|
|
}
|
|
);
|
|
w.global::<Library>().set_library_syncing(false);
|
|
let summary = format!(
|
|
"{} shard(s) up, {} down",
|
|
report.shards_uploaded, report.shards_downloaded
|
|
);
|
|
job.finish(if report.did_anything() {
|
|
summary.clone()
|
|
} else {
|
|
"nothing to exchange".to_string()
|
|
});
|
|
if report.did_anything() {
|
|
w.global::<Library>()
|
|
.set_library_status(format!("synced · {summary}").into());
|
|
}
|
|
// Adopted thumbnails and merged collections both change
|
|
// what the grid should show; adopted dates change its
|
|
// order, and the timeline beside it.
|
|
if report.thumbnails_adopted > 0
|
|
|| report.collections_gained > 0
|
|
|| report.members_gained > 0
|
|
|| report.dates_gained > 0
|
|
{
|
|
load_window(&w, &ctl_cb);
|
|
}
|
|
if report.dates_gained > 0 {
|
|
let borrow = ctl_cb.catalog();
|
|
let borrow = borrow.borrow();
|
|
if let Some(cat) = borrow.as_ref() {
|
|
refresh_timeline(&w, cat, &ctl_cb);
|
|
}
|
|
}
|
|
// TRACES: FR-CAT-7
|
|
// And the sidebar, which the grid reload does not
|
|
// touch. Membership counts as a change: a sync that
|
|
// files 127 photographs into a collection both devices
|
|
// already had gains no *collection*, so keying this on
|
|
// `collections_gained` alone left the tree showing no
|
|
// number beside a collection that had just been filled.
|
|
//
|
|
// The scan already does this after it finishes; the
|
|
// sync merges the same tables and did not.
|
|
if report.collections_gained > 0 || report.members_gained > 0 {
|
|
let coll = ctl_cb.coll_ctl.borrow().as_ref().and_then(|w| w.upgrade());
|
|
if let Some(coll) = coll {
|
|
let borrow = ctl_cb.catalog();
|
|
let borrow = borrow.borrow();
|
|
if let Some(cat) = borrow.as_ref() {
|
|
crate::collections_ui::refresh_tree(&w, &coll, cat);
|
|
}
|
|
}
|
|
}
|
|
stop(&ctl_cb.sync_timer);
|
|
return;
|
|
}
|
|
crate::derived_sync::SyncMessage::Failed(e) => {
|
|
log::warn!("sync failed: {e}");
|
|
job.fail(e.to_string());
|
|
w.global::<Library>().set_library_syncing(false);
|
|
// Not an error banner: a failed sync costs nothing —
|
|
// everything is still local and the next pass retries.
|
|
w.global::<Library>()
|
|
.set_library_status(format!("sync failed: {e}").into());
|
|
stop(&ctl_cb.sync_timer);
|
|
return;
|
|
}
|
|
}
|
|
}
|
|
},
|
|
);
|
|
|
|
*ctl.sync_timer.borrow_mut() = Some(timer);
|
|
}
|
|
|
|
/// Start the whole-library sweep and report its progress.
|
|
///
|
|
/// The grid only ever fetches what is on screen, so without this the timeline
|
|
/// describes the fraction of the library that happened to be scrolled past.
|
|
/// This covers the rest.
|
|
pub(super) fn start_sweep(window: &AppWindow, ctl: &Rc<LibraryController>) {
|
|
let Some((conn, _)) = ctl.session.borrow().clone() else {
|
|
return;
|
|
};
|
|
|
|
// TRACES: FR-CAT-9
|
|
// The sweep is nothing but network reads — one header fetch per undated
|
|
// image, across the whole library. Offline it would spend a timeout on
|
|
// every one of them, running for hours to learn nothing, while the
|
|
// progress bar implied work was happening. It resumes on reconnect, and
|
|
// the images it has already dated stay dated.
|
|
if ctl.is_offline() {
|
|
log::debug!("offline: not starting the metadata sweep");
|
|
return;
|
|
}
|
|
|
|
let rx = library::spawn_sweep(conn.clone(), library::catalog_path(&conn.account));
|
|
|
|
let timer = slint::Timer::default();
|
|
let weak = window.as_weak();
|
|
let ctl_cb = ctl.clone();
|
|
|
|
// Hours on a large library, and entirely invisible outside the grid until
|
|
// now: the register is where a user who has gone to develop can still see
|
|
// that indexing is running and how far it has got.
|
|
let job = ctl
|
|
.activity
|
|
.begin(crate::activity::Kind::Index, "Indexing capture times");
|
|
|
|
timer.start(
|
|
slint::TimerMode::Repeated,
|
|
// Slower than the thumbnail drain: this runs for tens of minutes and
|
|
// its progress does not need per-frame accuracy.
|
|
std::time::Duration::from_millis(400),
|
|
move || {
|
|
let Some(w) = weak.upgrade() else { return };
|
|
|
|
loop {
|
|
let msg = match rx.try_recv() {
|
|
Ok(m) => m,
|
|
Err(std::sync::mpsc::TryRecvError::Empty) => return,
|
|
Err(std::sync::mpsc::TryRecvError::Disconnected) => {
|
|
w.global::<Library>().set_library_sweep_total(0);
|
|
job.fail("stopped without finishing");
|
|
stop(&ctl_cb.sweep_timer);
|
|
return;
|
|
}
|
|
};
|
|
|
|
match msg {
|
|
library::SweepMessage::Total(n) => {
|
|
w.global::<Library>().set_library_sweep_total(n as i32);
|
|
w.global::<Library>().set_library_sweep_done(0);
|
|
job.total(n);
|
|
}
|
|
library::SweepMessage::Progress { done, dated } => {
|
|
w.global::<Library>().set_library_sweep_done(done as i32);
|
|
job.progress(
|
|
done,
|
|
w.global::<Library>().get_library_sweep_total() as usize,
|
|
);
|
|
// Rebuild as it goes: the histogram growing while the
|
|
// sweep runs is the visible sign it is working.
|
|
if dated > 0 {
|
|
let borrow = ctl_cb.catalog.borrow();
|
|
if let Some(catalog) = borrow.as_ref() {
|
|
refresh_timeline(&w, catalog, &ctl_cb);
|
|
}
|
|
}
|
|
}
|
|
library::SweepMessage::Finished { dated } => {
|
|
log::info!("sweep finished: {dated} dated");
|
|
job.finish(format!("{dated} dated"));
|
|
w.global::<Library>().set_library_sweep_total(0);
|
|
{
|
|
let borrow = ctl_cb.catalog.borrow();
|
|
if let Some(catalog) = borrow.as_ref() {
|
|
refresh_timeline(&w, catalog, &ctl_cb);
|
|
}
|
|
}
|
|
// TRACES: NFR-R2
|
|
// The scheduled backup, at the moment the catalog is
|
|
// quietest and a day's edits have just been folded in.
|
|
// On its own thread, with its own connection: a copy
|
|
// of a 130 MB file is a second or two the Slint loop
|
|
// must not spend, and it runs beside the sync below,
|
|
// which is only a reader of the same file.
|
|
spawn_scheduled_backup(&ctl_cb);
|
|
// Now that indexing is complete, hand the result to the
|
|
// server so a second device inherits it rather than
|
|
// repeating hours of range fetches.
|
|
start_derived_sync(&w, &ctl_cb);
|
|
stop(&ctl_cb.sweep_timer);
|
|
return;
|
|
}
|
|
}
|
|
}
|
|
},
|
|
);
|
|
|
|
*ctl.sweep_timer.borrow_mut() = Some(timer);
|
|
}
|
|
|
|
/// TRACES: FR-CAT-3 | FR-NC-3 | FR-NC-7
|
|
/// Thumbnail every photograph in the library, then hand the shards over.
|
|
///
|
|
/// # Why it is asked for rather than assumed
|
|
///
|
|
/// The grid's own fetching is demand-driven on purpose: a remote library is
|
|
/// browsed over a connection that must not be saturated to show one screen
|
|
/// (FR-NC-3). But that leaves the thumbnail store holding only what has been
|
|
/// looked at, and the store is the one derived thing worth syncing — a second
|
|
/// device that downloads the shards gets a full grid without touching a RAW.
|
|
/// So the complete set is worth an hour of transfers *once*, on a machine
|
|
/// plugged in, at a moment the user chose. That is this.
|
|
///
|
|
/// The sync at the end is not a separate courtesy: a filled store that never
|
|
/// leaves this device is most of the cost for none of the point.
|
|
pub(super) fn start_thumbnail_sweep(window: &AppWindow, ctl: &Rc<LibraryController>) {
|
|
let Some((conn, _)) = ctl.session.borrow().clone() else {
|
|
return;
|
|
};
|
|
|
|
// Already running. A second pass would re-fetch everything the first is
|
|
// part way through — the work list is built from what the store lacks, and
|
|
// the first pass has not filled it yet.
|
|
if ctl.thumb_sweep_timer.borrow().is_some()
|
|
&& window.global::<Library>().get_library_thumbnailing()
|
|
{
|
|
return;
|
|
}
|
|
|
|
// TRACES: FR-CAT-9
|
|
// Every image here is a range fetch. Offline the pass would spend a
|
|
// timeout per photograph and store nothing, so it is refused rather than
|
|
// started — and said so, because this one was pressed deliberately and a
|
|
// button that silently does nothing is worse than one that declines.
|
|
if ctl.is_offline() {
|
|
log::debug!("offline: not starting the thumbnail pass");
|
|
window
|
|
.global::<Library>()
|
|
.set_library_status("Offline — thumbnailing needs the server.".into());
|
|
return;
|
|
}
|
|
|
|
window.global::<Library>().set_library_thumbnailing(true);
|
|
let rx = library::spawn_thumbnail_sweep(
|
|
conn.clone(),
|
|
library::catalog_path(&conn.account),
|
|
library::thumbs_dir(&conn.account),
|
|
);
|
|
|
|
let timer = slint::Timer::default();
|
|
let weak = window.as_weak();
|
|
let ctl_cb = ctl.clone();
|
|
|
|
let job = ctl.activity.begin(
|
|
crate::activity::Kind::Thumbnails,
|
|
"Thumbnailing the library",
|
|
);
|
|
|
|
// The total arrives in the first message rather than up front — counting
|
|
// it means asking the store about every image — and the progress messages
|
|
// after it carry only cumulative counts, so it is kept here rather than
|
|
// re-derived.
|
|
let total = std::cell::Cell::new(0usize);
|
|
|
|
timer.start(
|
|
slint::TimerMode::Repeated,
|
|
// As slow as the metadata sweep's drain, and for the same reason: this
|
|
// runs for tens of minutes and a chunk lands every few seconds.
|
|
std::time::Duration::from_millis(400),
|
|
move || {
|
|
let Some(w) = weak.upgrade() else { return };
|
|
|
|
loop {
|
|
let msg = match rx.try_recv() {
|
|
Ok(m) => m,
|
|
Err(std::sync::mpsc::TryRecvError::Empty) => return,
|
|
Err(std::sync::mpsc::TryRecvError::Disconnected) => {
|
|
w.global::<Library>().set_library_thumbnailing(false);
|
|
job.fail("stopped without finishing");
|
|
stop(&ctl_cb.thumb_sweep_timer);
|
|
return;
|
|
}
|
|
};
|
|
|
|
match msg {
|
|
library::ThumbSweepMessage::Total(n) => {
|
|
total.set(n);
|
|
job.total(n);
|
|
w.global::<Library>()
|
|
.set_library_status(format!("Thumbnailing {n} photograph(s)…").into());
|
|
}
|
|
library::ThumbSweepMessage::Progress { done, stored } => {
|
|
job.progress(done, total.get());
|
|
// Dates arrive on the same headers, so the histogram
|
|
// grows as this runs — the visible sign it is working
|
|
// on a page that is not the grid.
|
|
if stored > 0 {
|
|
let borrow = ctl_cb.catalog.borrow();
|
|
if let Some(catalog) = borrow.as_ref() {
|
|
refresh_timeline(&w, catalog, &ctl_cb);
|
|
}
|
|
}
|
|
}
|
|
library::ThumbSweepMessage::Finished {
|
|
stored,
|
|
failed,
|
|
offline,
|
|
} => {
|
|
log::info!(
|
|
"thumbnail pass finished: {stored} stored, {failed} without a preview"
|
|
);
|
|
w.global::<Library>().set_library_thumbnailing(false);
|
|
stop(&ctl_cb.thumb_sweep_timer);
|
|
|
|
let summary = if offline {
|
|
format!("{stored} stored · stopped, server unreachable")
|
|
} else if failed > 0 {
|
|
format!("{stored} stored · {failed} with no usable preview")
|
|
} else {
|
|
format!("{stored} stored")
|
|
};
|
|
w.global::<Library>()
|
|
.set_library_status(summary.clone().into());
|
|
// Finished rather than failed even when the server
|
|
// dropped: the pass is resumable and what it did build
|
|
// is stored, and a red row would invite the user to
|
|
// redo work that is already done.
|
|
job.finish(summary);
|
|
|
|
{
|
|
let borrow = ctl_cb.catalog.borrow();
|
|
if let Some(catalog) = borrow.as_ref() {
|
|
refresh_timeline(&w, catalog, &ctl_cb);
|
|
}
|
|
}
|
|
// Cells that showed a placeholder have pixels now, and
|
|
// the grid asks the store once per photograph — so
|
|
// without this the library the pass just thumbnailed
|
|
// stays blank until something else reloads the window.
|
|
if w.get_library_visible() {
|
|
ctl_cb.requested.borrow_mut().clear();
|
|
schedule_reload(&w, &ctl_cb);
|
|
}
|
|
|
|
// The point of the pass: the shards go to the server so
|
|
// every other device inherits them. Skipped when the
|
|
// connection is already gone — the sync would only
|
|
// discover the same thing more slowly.
|
|
if !offline {
|
|
start_derived_sync(&w, &ctl_cb);
|
|
}
|
|
// And now every photograph in the library has a
|
|
// thumbnail, which is the only moment all of its burst
|
|
// signatures can be computed. See `bursts::start_pass`,
|
|
// which owns the pass and everything it needs to drain
|
|
// itself; what it wants from here is the paths and a
|
|
// way to say the grid has something new to draw.
|
|
if let Some((conn, _)) = ctl_cb.session.borrow().clone() {
|
|
let weak_after = w.as_weak();
|
|
let ctl_after = ctl_cb.clone();
|
|
crate::bursts::start_pass(
|
|
library::catalog_path(&conn.account),
|
|
library::thumbs_dir(&conn.account),
|
|
move |bursts| {
|
|
let Some(w) = weak_after.upgrade() else {
|
|
return;
|
|
};
|
|
if bursts > 0 && w.get_library_visible() {
|
|
schedule_reload(&w, &ctl_after);
|
|
}
|
|
},
|
|
);
|
|
}
|
|
return;
|
|
}
|
|
}
|
|
}
|
|
},
|
|
);
|
|
|
|
*ctl.thumb_sweep_timer.borrow_mut() = Some(timer);
|
|
}
|