DarkRoom/ui/dr-ui/src/library_ui/sync.rs

//! The background passes that reach beyond the loaded window: the day's
//! catalog backup, the metadata sweep that dates every image rather than
//! only the ones scrolled past, the whole-library thumbnail pass, and the
//! exchange with the server that follows each of them.
//!
//! Split from `window` because these run on their own timers over the whole
//! library rather than in response to what the grid is showing.
//! See `docs/dev/code-health.md` CH-1.

use crate::executors::{self, Executor};
use std::rc::Rc;

use slint::ComponentHandle;

use crate::library;
use crate::{AppWindow, Library};

use super::controller::{stop, LibraryController};
use super::open::schedule_reload;
use super::timeline::refresh_timeline;
use super::window::load_window;

/// TRACES: NFR-R2
/// Take the day's catalog backup if one is due, off the UI thread.
///
/// Decided cheaply first — [`dr_catalog::recovery::backup_due`] reads a
/// directory listing, not the catalog — so the ordinary case of "backed up
/// this morning already" costs no thread and no connection.
fn spawn_scheduled_backup(ctl: &Rc<LibraryController>) {
    let Some((conn, _)) = ctl.session.borrow().clone() else {
        return;
    };
    let catalog_path = library::catalog_path(&conn.account);
    if !dr_catalog::recovery::backup_due(&catalog_path) {
        return;
    }
    executors::spawn(Executor::Io, "backup", move || {
        let catalog = match dr_catalog::Catalog::open(&catalog_path) {
            Ok(c) => c,
            Err(e) => {
                log::warn!("scheduled backup: opening the catalog: {e}");
                return;
            }
        };
        if let Err(e) = dr_catalog::recovery::backup_if_due(catalog.connection(), &catalog_path) {
            log::warn!("scheduled backup: {e}");
        }
    });
}

/// Push shards and the catalog to the server, and take what it has.
///
/// Fired when the scan completes, so a fresh device inherits its peers'
/// work before deriving any of its own; after the sweep completes, when
/// there is a finished index worth sharing; and from the Sync button for an
/// explicit exchange. A pass still running when the next trigger fires is
/// left to finish — the guard below.
pub(super) fn start_derived_sync(window: &AppWindow, ctl: &Rc<LibraryController>) {
    // An escape hatch for running the app against a *copied* library without
    // touching the account's real server.
    //
    // Added after doing exactly that by accident: a local test launch
    // completed its thumbnail sweep, which fires this, which pushed a test
    // catalog over the live one. Redirecting `XDG_DATA_HOME` isolates the
    // catalog and the thumbnails but not the server, and nothing said so.
    //
    // The guard belongs here rather than at the call sites: the sweep firing a
    // sync is correct — the point of building thumbnails is sharing them — and
    // a flag checked in three places would eventually be missed in a fourth.
    if std::env::var_os("DARKROOM_NO_SYNC").is_some() {
        log::info!("derived sync suppressed by DARKROOM_NO_SYNC");
        return;
    }

    let Some((conn, _)) = ctl.session.borrow().clone() else {
        return;
    };
    // Already running: a second pass would race the first over the same
    // scratch files.
    if ctl.sync_timer.borrow().is_some() && window.global::<Library>().get_library_syncing() {
        return;
    }

    // TRACES: FR-CAT-9
    // Nothing to push to and nothing to take. Attempting it would upload
    // shards into a timeout and light the "Syncing…" indicator over work that
    // cannot start; the shards are unchanged on disk and go out on the next
    // sync once the server is back.
    if ctl.is_offline() {
        log::debug!("offline: skipping derived sync");
        return;
    }

    let catalog_path = library::catalog_path(&conn.account);
    let scratch = catalog_path
        .parent()
        .map(|p| p.join("scratch"))
        .unwrap_or_else(std::env::temp_dir);
    let _ = std::fs::create_dir_all(&scratch);

    // TRACES: FR-EXP-7 | FR-NC-10
    // Drain the export outbox on the same pass, and before the shards. An
    // export the user was told had succeeded is waiting here, and it is the
    // one thing in this directory that exists nowhere else — a thumbnail
    // shard can be rebuilt from the originals, and the catalog is an index.
    //
    // Fire-and-forget rather than reported: it runs on its own thread and
    // clears entries as they land, so a partial run leaves the rest queued
    // for next time and nothing is lost by not watching it. It reports
    // through the log until an export has a place in the activity list.
    {
        let outbox = crate::export::outbox_dir(&conn.account);
        if crate::export::pending_count(&outbox) > 0 {
            let rx = crate::export::spawn_upload(conn.clone(), conn.account.root.clone(), outbox);
            crate::export::watch_upload(rx, window.as_weak());
        }
    }

    window.global::<Library>().set_library_syncing(true);
    let rx = crate::derived_sync::spawn_sync(
        conn.clone(),
        conn.account.root.clone(),
        library::thumbs_dir(&conn.account),
        catalog_path,
        library::place_path(&conn.account),
        crate::derived_sync::PresetFiles {
            library: crate::preset_store::PresetStore::open()
                .path()
                .to_path_buf(),
            base: library::presets_base_path(&conn.account),
        },
        scratch,
        ctl.face_model_id.borrow().clone(),
    );

    let timer = slint::Timer::default();
    let weak = window.as_weak();
    let ctl_cb = ctl.clone();

    // The sync reports stages rather than counts, so it stays indeterminate and
    // says what stage it is in.
    let job = ctl
        .activity
        .begin(crate::activity::Kind::Sync, "Syncing with the server");

    timer.start(
        slint::TimerMode::Repeated,
        std::time::Duration::from_millis(300),
        move || {
            let Some(w) = weak.upgrade() else { return };
            loop {
                let msg = match rx.try_recv() {
                    Ok(m) => m,
                    Err(std::sync::mpsc::TryRecvError::Empty) => return,
                    Err(std::sync::mpsc::TryRecvError::Disconnected) => {
                        w.global::<Library>().set_library_syncing(false);
                        job.fail("stopped without finishing");
                        stop(&ctl_cb.sync_timer);
                        return;
                    }
                };

                match msg {
                    crate::derived_sync::SyncMessage::Status(s) => {
                        job.detail(s.clone());
                        w.global::<Library>().set_library_status(s.into());
                    }
                    crate::derived_sync::SyncMessage::Finished(report) => {
                        log::info!(
                            "sync: {} shard(s) up, {} down ({} thumbnails), \
                             catalog {}{}{}",
                            report.shards_uploaded,
                            report.shards_downloaded,
                            report.thumbnails_adopted,
                            // Replacing a damaged copy is said apart from an
                            // ordinary push: it is the one push that discarded
                            // something, and a log line that called it "pushed"
                            // would hide the only moment worth going back to.
                            match (report.catalog_uploaded, report.catalog_replaced) {
                                (true, true) => "pushed over a damaged copy",
                                (true, false) => "pushed",
                                _ => "not pushed",
                            },
                            if report.collections_gained > 0 {
                                format!(", {} collection(s) gained", report.collections_gained)
                            } else {
                                String::new()
                            },
                            if report.dates_gained > 0 {
                                format!(", {} date(s) gained", report.dates_gained)
                            } else {
                                String::new()
                            }
                        );
                        w.global::<Library>().set_library_syncing(false);
                        let summary = format!(
                            "{} shard(s) up, {} down",
                            report.shards_uploaded, report.shards_downloaded
                        );
                        job.finish(if report.did_anything() {
                            summary.clone()
                        } else {
                            "nothing to exchange".to_string()
                        });
                        if report.did_anything() {
                            w.global::<Library>()
                                .set_library_status(format!("synced · {summary}").into());
                        }
                        // Adopted thumbnails and merged collections both change
                        // what the grid should show; adopted dates change its
                        // order, and the timeline beside it.
                        if report.thumbnails_adopted > 0
                            || report.collections_gained > 0
                            || report.members_gained > 0
                            || report.dates_gained > 0
                        {
                            load_window(&w, &ctl_cb);
                        }
                        if report.dates_gained > 0 {
                            let borrow = ctl_cb.catalog();
                            let borrow = borrow.borrow();
                            if let Some(cat) = borrow.as_ref() {
                                refresh_timeline(&w, cat, &ctl_cb);
                            }
                        }
                        // TRACES: FR-CAT-7
                        // And the sidebar, which the grid reload does not
                        // touch. Membership counts as a change: a sync that
                        // files 127 photographs into a collection both devices
                        // already had gains no *collection*, so keying this on
                        // `collections_gained` alone left the tree showing no
                        // number beside a collection that had just been filled.
                        //
                        // The scan already does this after it finishes; the
                        // sync merges the same tables and did not.
                        if report.collections_gained > 0 || report.members_gained > 0 {
                            let coll = ctl_cb.coll_ctl.borrow().as_ref().and_then(|w| w.upgrade());
                            if let Some(coll) = coll {
                                let borrow = ctl_cb.catalog();
                                let borrow = borrow.borrow();
                                if let Some(cat) = borrow.as_ref() {
                                    crate::collections_ui::refresh_tree(&w, &coll, cat);
                                }
                            }
                        }
                        // TRACES: FR-DEV-6
                        // Presets saved on another device, into the list the
                        // develop view is drawing.
                        if report.presets_adopted {
                            let named = ctl_cb
                                .named_presets
                                .borrow()
                                .as_ref()
                                .and_then(|n| n.upgrade());
                            if let Some(named) = named {
                                named.reload();
                                crate::presets::render_named(&w, &named);
                            }
                        }
                        stop(&ctl_cb.sync_timer);
                        return;
                    }
                    crate::derived_sync::SyncMessage::Failed(e) => {
                        log::warn!("sync failed: {e}");
                        job.fail(e.to_string());
                        w.global::<Library>().set_library_syncing(false);
                        // Not an error banner: a failed sync costs nothing —
                        // everything is still local and the next pass retries.
                        w.global::<Library>()
                            .set_library_status(format!("sync failed: {e}").into());
                        stop(&ctl_cb.sync_timer);
                        return;
                    }
                }
            }
        },
    );

    *ctl.sync_timer.borrow_mut() = Some(timer);
}

/// Start the whole-library sweep and report its progress.
///
/// The grid only ever fetches what is on screen, so without this the timeline
/// describes the fraction of the library that happened to be scrolled past.
/// This covers the rest.
pub(super) fn start_sweep(window: &AppWindow, ctl: &Rc<LibraryController>) {
    let Some((conn, _)) = ctl.session.borrow().clone() else {
        return;
    };

    // TRACES: FR-CAT-9
    // The sweep is nothing but network reads — one header fetch per undated
    // image, across the whole library. Offline it would spend a timeout on
    // every one of them, running for hours to learn nothing, while the
    // progress bar implied work was happening. It resumes on reconnect, and
    // the images it has already dated stay dated.
    if ctl.is_offline() {
        log::debug!("offline: not starting the metadata sweep");
        return;
    }

    let rx = library::spawn_sweep(conn.clone(), library::catalog_path(&conn.account));

    let timer = slint::Timer::default();
    let weak = window.as_weak();
    let ctl_cb = ctl.clone();

    // Hours on a large library, and entirely invisible outside the grid until
    // now: the register is where a user who has gone to develop can still see
    // that indexing is running and how far it has got.
    let job = ctl
        .activity
        .begin(crate::activity::Kind::Index, "Indexing capture times");

    timer.start(
        slint::TimerMode::Repeated,
        // Slower than the thumbnail drain: this runs for tens of minutes and
        // its progress does not need per-frame accuracy.
        std::time::Duration::from_millis(400),
        move || {
            let Some(w) = weak.upgrade() else { return };

            loop {
                let msg = match rx.try_recv() {
                    Ok(m) => m,
                    Err(std::sync::mpsc::TryRecvError::Empty) => return,
                    Err(std::sync::mpsc::TryRecvError::Disconnected) => {
                        w.global::<Library>().set_library_sweep_total(0);
                        job.fail("stopped without finishing");
                        stop(&ctl_cb.sweep_timer);
                        return;
                    }
                };

                match msg {
                    library::SweepMessage::Total(n) => {
                        w.global::<Library>().set_library_sweep_total(n as i32);
                        w.global::<Library>().set_library_sweep_done(0);
                        job.total(n);
                    }
                    library::SweepMessage::Progress { done, dated } => {
                        w.global::<Library>().set_library_sweep_done(done as i32);
                        job.progress(
                            done,
                            w.global::<Library>().get_library_sweep_total() as usize,
                        );
                        // Rebuild as it goes: the histogram growing while the
                        // sweep runs is the visible sign it is working.
                        if dated > 0 {
                            let borrow = ctl_cb.catalog.borrow();
                            if let Some(catalog) = borrow.as_ref() {
                                refresh_timeline(&w, catalog, &ctl_cb);
                            }
                        }
                    }
                    library::SweepMessage::Finished { dated } => {
                        log::info!("sweep finished: {dated} dated");
                        job.finish(format!("{dated} dated"));
                        w.global::<Library>().set_library_sweep_total(0);
                        {
                            let borrow = ctl_cb.catalog.borrow();
                            if let Some(catalog) = borrow.as_ref() {
                                refresh_timeline(&w, catalog, &ctl_cb);
                                // A capture time is half of what makes two
                                // files one photograph, and the sweep is
                                // what reads it: a copy is a duplicate only
                                // once it is dated.
                                if dated > 0 {
                                    crate::duplicates_ui::refresh_count(&w, catalog);
                                }
                            }
                        }
                        // TRACES: NFR-R2
                        // The scheduled backup, at the moment the catalog is
                        // quietest and a day's edits have just been folded in.
                        // On its own thread, with its own connection: a copy
                        // of a 130 MB file is a second or two the Slint loop
                        // must not spend, and it runs beside the sync below,
                        // which is only a reader of the same file.
                        spawn_scheduled_backup(&ctl_cb);
                        // Now that indexing is complete, hand the result to the
                        // server so a second device inherits it rather than
                        // repeating hours of range fetches.
                        start_derived_sync(&w, &ctl_cb);
                        stop(&ctl_cb.sweep_timer);
                        return;
                    }
                }
            }
        },
    );

    *ctl.sweep_timer.borrow_mut() = Some(timer);
}

/// TRACES: FR-CAT-3 | FR-NC-3 | FR-NC-7
/// Thumbnail every photograph in the library, then hand the shards over.
///
/// # Why it is asked for rather than assumed
///
/// The grid's own fetching is demand-driven on purpose: a remote library is
/// browsed over a connection that must not be saturated to show one screen
/// (FR-NC-3). But that leaves the thumbnail store holding only what has been
/// looked at, and the store is the one derived thing worth syncing — a second
/// device that downloads the shards gets a full grid without touching a RAW.
/// So the complete set is worth an hour of transfers *once*, on a machine
/// plugged in, at a moment the user chose. That is this.
///
/// The sync at the end is not a separate courtesy: a filled store that never
/// leaves this device is most of the cost for none of the point.
pub(super) fn start_thumbnail_sweep(window: &AppWindow, ctl: &Rc<LibraryController>) {
    let Some((conn, _)) = ctl.session.borrow().clone() else {
        return;
    };

    // Already running. A second pass would re-fetch everything the first is
    // part way through — the work list is built from what the store lacks, and
    // the first pass has not filled it yet.
    if ctl.thumb_sweep_timer.borrow().is_some()
        && window.global::<Library>().get_library_thumbnailing()
    {
        return;
    }

    // TRACES: FR-CAT-9
    // Every image here is a range fetch. Offline the pass would spend a
    // timeout per photograph and store nothing, so it is refused rather than
    // started — and said so, because this one was pressed deliberately and a
    // button that silently does nothing is worse than one that declines.
    if ctl.is_offline() {
        log::debug!("offline: not starting the thumbnail pass");
        window
            .global::<Library>()
            .set_library_status("Offline — thumbnailing needs the server.".into());
        return;
    }

    window.global::<Library>().set_library_thumbnailing(true);
    let rx = library::spawn_thumbnail_sweep(
        conn.clone(),
        library::catalog_path(&conn.account),
        library::thumbs_dir(&conn.account),
    );

    let timer = slint::Timer::default();
    let weak = window.as_weak();
    let ctl_cb = ctl.clone();

    let job = ctl.activity.begin(
        crate::activity::Kind::Thumbnails,
        "Thumbnailing the library",
    );

    // The total arrives in the first message rather than up front — counting
    // it means asking the store about every image — and the progress messages
    // after it carry only cumulative counts, so it is kept here rather than
    // re-derived.
    let total = std::cell::Cell::new(0usize);

    timer.start(
        slint::TimerMode::Repeated,
        // As slow as the metadata sweep's drain, and for the same reason: this
        // runs for tens of minutes and a chunk lands every few seconds.
        std::time::Duration::from_millis(400),
        move || {
            let Some(w) = weak.upgrade() else { return };

            loop {
                let msg = match rx.try_recv() {
                    Ok(m) => m,
                    Err(std::sync::mpsc::TryRecvError::Empty) => return,
                    Err(std::sync::mpsc::TryRecvError::Disconnected) => {
                        w.global::<Library>().set_library_thumbnailing(false);
                        job.fail("stopped without finishing");
                        stop(&ctl_cb.thumb_sweep_timer);
                        return;
                    }
                };

                match msg {
                    library::ThumbSweepMessage::Total(n) => {
                        total.set(n);
                        job.total(n);
                        w.global::<Library>()
                            .set_library_status(format!("Thumbnailing {n} photograph(s)…").into());
                    }
                    library::ThumbSweepMessage::Progress { done, stored } => {
                        job.progress(done, total.get());
                        // Dates arrive on the same headers, so the histogram
                        // grows as this runs — the visible sign it is working
                        // on a page that is not the grid.
                        if stored > 0 {
                            let borrow = ctl_cb.catalog.borrow();
                            if let Some(catalog) = borrow.as_ref() {
                                refresh_timeline(&w, catalog, &ctl_cb);
                            }
                        }
                    }
                    library::ThumbSweepMessage::Finished {
                        stored,
                        failed,
                        offline,
                    } => {
                        log::info!(
                            "thumbnail pass finished: {stored} stored, {failed} without a preview"
                        );
                        w.global::<Library>().set_library_thumbnailing(false);
                        stop(&ctl_cb.thumb_sweep_timer);

                        let summary = if offline {
                            format!("{stored} stored · stopped, server unreachable")
                        } else if failed > 0 {
                            format!("{stored} stored · {failed} with no usable preview")
                        } else {
                            format!("{stored} stored")
                        };
                        w.global::<Library>()
                            .set_library_status(summary.clone().into());
                        // Finished rather than failed even when the server
                        // dropped: the pass is resumable and what it did build
                        // is stored, and a red row would invite the user to
                        // redo work that is already done.
                        job.finish(summary);

                        {
                            let borrow = ctl_cb.catalog.borrow();
                            if let Some(catalog) = borrow.as_ref() {
                                refresh_timeline(&w, catalog, &ctl_cb);
                            }
                        }
                        // Cells that showed a placeholder have pixels now, and
                        // the grid asks the store once per photograph — so
                        // without this the library the pass just thumbnailed
                        // stays blank until something else reloads the window.
                        if w.get_library_visible() {
                            ctl_cb.requested.borrow_mut().clear();
                            schedule_reload(&w, &ctl_cb);
                        }

                        // The point of the pass: the shards go to the server so
                        // every other device inherits them. Skipped when the
                        // connection is already gone — the sync would only
                        // discover the same thing more slowly.
                        if !offline {
                            start_derived_sync(&w, &ctl_cb);
                        }
                        // And now every photograph in the library has a
                        // thumbnail, which is the only moment all of its burst
                        // signatures can be computed. See `bursts::start_pass`,
                        // which owns the pass and everything it needs to drain
                        // itself; what it wants from here is the paths and a
                        // way to say the grid has something new to draw.
                        if let Some((conn, _)) = ctl_cb.session.borrow().clone() {
                            let weak_after = w.as_weak();
                            let ctl_after = ctl_cb.clone();
                            crate::bursts::start_pass(
                                library::catalog_path(&conn.account),
                                library::thumbs_dir(&conn.account),
                                move |bursts| {
                                    let Some(w) = weak_after.upgrade() else {
                                        return;
                                    };
                                    if bursts > 0 && w.get_library_visible() {
                                        schedule_reload(&w, &ctl_after);
                                    }
                                },
                            );
                        }
                        return;
                    }
                }
            }
        },
    );

    *ctl.thumb_sweep_timer.borrow_mut() = Some(timer);
}