//! TRACES: FR-CAT-11a | FR-CAT-15 | FR-UI-4 | NFR-P9 //! The duplicate originals review: which groups there are, which copy stays, //! what the check found, and the button that consolidates them. //! //! The page is a view of [`Reviewed`] groups held here. The catalog is read //! on the UI thread only for what one query answers — the groups themselves, //! and a merge preview for the thirty rows on the page — and every read of a //! file happens on a worker (`crate::duplicates`), which reports per group. use std::cell::{Cell, RefCell}; use std::collections::HashMap; use std::rc::Rc; use std::sync::atomic::{AtomicBool, Ordering}; use std::sync::mpsc::Receiver; use std::sync::Arc; use dr_catalog::duplicates::Outcome; use dr_catalog::Catalog; use dr_types::ImageId; use slint::{ComponentHandle, ModelRc, VecModel}; use crate::activity::{Activity, ActivityLog}; use crate::duplicates::{self as dups, Check, DupMessage, Reviewed}; use crate::library_ui::LibraryController; use crate::{AppWindow, DuplicateCopy, DuplicateRow, Duplicates, Library, Page}; /// Groups on one page of the review. A page is what gets a thumbnail and a /// merge preview, so this bounds the work a redraw does whatever the size /// of the library. pub const PAGE: usize = 30; /// What runs when a consolidation ends: the grid, the sidebar and the trash /// brought up to date. pub type After = Rc; /// How a group ended in this session. #[derive(Debug, Clone)] enum Ended { Done(String), Failed(String), } /// What the review holds between redraws. pub struct DuplicatesController { review: RefCell>, page: Cell, ended: RefCell>, /// Pictures for the page on screen, by `oc:fileid`. thumbs: RefCell>, running: Cell, consolidating: Cell, stop: RefCell>, timer: RefCell>, activity: Rc, /// Survivors consolidated by the running job, whose sidecars are written /// when it ends. survivors: RefCell>, counts: Cell<(usize, usize, usize)>, } impl DuplicatesController { pub fn new(activity: Rc) -> Rc { Rc::new(Self { review: RefCell::default(), page: Cell::new(0), ended: RefCell::default(), thumbs: RefCell::default(), running: Cell::new(false), consolidating: Cell::new(false), stop: RefCell::default(), timer: RefCell::default(), activity, survivors: RefCell::default(), counts: Cell::new((0, 0, 0)), }) } } /// TRACES: FR-CAT-11a /// Put the number of duplicate groups where the sidebar and settings show /// it. One `COUNT` query; called when the catalog is opened, after a scan, /// and after anything moves files in or out of the trash — the events that /// can change the answer. pub fn refresh_count(window: &AppWindow, catalog: &Catalog) { let (groups, _) = dr_catalog::duplicates::count(catalog.connection()).unwrap_or((0, 0)); window.global::().set_group_count(groups as i32); } /// Wire the page. `after` runs when a consolidation ends, to bring the grid, /// the sidebar and the trash up to date. pub fn wire( window: &AppWindow, ctl: Rc, library: Rc, after: After, ) { let g = window.global::(); { let (weak, ctl, library) = (window.as_weak(), ctl.clone(), library.clone()); g.on_open(move || { let Some(w) = weak.upgrade() else { return }; open(&w, &ctl, &library); }); } { let (weak, ctl, library) = (window.as_weak(), ctl.clone(), library.clone()); g.on_close(move || { let Some(w) = weak.upgrade() else { return }; // A running job carries on and reports to the register; the // page can be opened again to watch it. w.set_active_page(Page::None); if !ctl.running.get() { ctl.thumbs.borrow_mut().clear(); if let Some(cat) = library.catalog().borrow().as_ref() { refresh_count(&w, cat); } } }); } { let (weak, ctl, library) = (window.as_weak(), ctl.clone(), library.clone()); g.on_check(move || { let Some(w) = weak.upgrade() else { return }; start_check(&w, &ctl, &library); }); } { let ctl = ctl.clone(); g.on_stop(move || { ctl.stop.borrow().store(true, Ordering::Relaxed); }); } { let (weak, ctl, library, after) = ( window.as_weak(), ctl.clone(), library.clone(), after.clone(), ); g.on_confirm(move || { let Some(w) = weak.upgrade() else { return }; start_consolidate(&w, &ctl, &library, after.clone()); }); } { let (weak, ctl, library) = (window.as_weak(), ctl.clone(), library.clone()); g.on_prev_page(move || { let Some(w) = weak.upgrade() else { return }; ctl.page.set(ctl.page.get().saturating_sub(1)); ctl.thumbs.borrow_mut().clear(); show(&w, &ctl, &library); }); } { let (weak, ctl, library) = (window.as_weak(), ctl.clone(), library.clone()); g.on_next_page(move || { let Some(w) = weak.upgrade() else { return }; let last = ctl.review.borrow().len().saturating_sub(1) / PAGE; ctl.page.set((ctl.page.get() + 1).min(last)); ctl.thumbs.borrow_mut().clear(); show(&w, &ctl, &library); }); } { let (weak, ctl, library) = (window.as_weak(), ctl.clone(), library.clone()); g.on_pick_survivor(move |row, copy| { let Some(w) = weak.upgrade() else { return }; if ctl.running.get() { return; } let index = ctl.page.get() * PAGE + row.max(0) as usize; { let mut review = ctl.review.borrow_mut(); let Some(r) = review.get_mut(index) else { return; }; if (copy as usize) < r.group.copies.len() { r.survivor = copy as usize; } } show(&w, &ctl, &library); }); } { let (weak, ctl, library) = (window.as_weak(), ctl.clone(), library.clone()); g.on_toggle(move |row, on| { let Some(w) = weak.upgrade() else { return }; if ctl.running.get() { return; } let index = ctl.page.get() * PAGE + row.max(0) as usize; if let Some(r) = ctl.review.borrow_mut().get_mut(index) { r.include = on; } show(&w, &ctl, &library); }); } } /// Open the page on a fresh read of the groups. /// /// What was decided about a group that is still the same group — its check, /// the copy chosen to stay, whether it is in — is carried over, so leaving /// the page and coming back loses nothing. fn open(window: &AppWindow, ctl: &Rc, library: &Rc) { window.set_active_page(Page::Duplicates); if ctl.running.get() { show(window, ctl, library); return; } let groups = { let catalog = library.catalog(); let borrow = catalog.borrow(); let Some(cat) = borrow.as_ref() else { window .global::() .set_error("Open a library first.".into()); return; }; refresh_count(window, cat); match dr_catalog::duplicates::candidates(cat.connection()) { Ok(g) => g, Err(e) => { window .global::() .set_error(format!("Reading duplicates: {e}").into()); return; } } }; let before: HashMap, Reviewed> = ctl .review .borrow_mut() .drain(..) .map(|r| (ids(&r), r)) .collect(); let review: Vec = groups .into_iter() .map(|g| { let fresh = Reviewed::new(g); match before.get(&ids(&fresh)) { Some(old) => Reviewed { group: fresh.group, survivor: old.survivor, include: old.include, check: old.check.clone(), }, None => fresh, } }) .collect(); *ctl.review.borrow_mut() = review; ctl.ended.borrow_mut().clear(); ctl.thumbs.borrow_mut().clear(); ctl.page.set(0); window.global::().set_error("".into()); show(window, ctl, library); } fn ids(r: &Reviewed) -> Vec { r.group.copies.iter().map(|c| c.image).collect() } /// Megabytes or gigabytes, as a transfer is quoted. fn quote_bytes(bytes: u64) -> String { let mb = bytes as f64 / (1024.0 * 1024.0); if mb >= 1024.0 { format!("{:.1} GB", mb / 1024.0) } else { format!("{mb:.0} MB") } } fn plural(n: usize, one: &str, many: &str) -> String { format!("{n} {}", if n == 1 { one } else { many }) } /// Redraw the page from what the controller holds. fn show(window: &AppWindow, ctl: &Rc, library: &Rc) { let g = window.global::(); let review = ctl.review.borrow(); let summary = dups::summarise(&review); let total = review.len(); // The dry run, and what the check would cost. let text = if total == 0 { "No duplicate originals: every file is in the library once.".to_string() } else if summary.unchecked == total { format!( "{} of the same file in more than one folder, {} spare. \ Check them before anything moves.", plural(total, "group", "groups"), plural( review.iter().map(|r| r.group.copies.len() - 1).sum(), "copy", "copies" ), ) } else { let mut t = format!( "{}, {} to trash, {} skipped", plural(summary.groups, "group", "groups"), plural(summary.files, "file", "files"), summary.skipped ); if summary.excluded > 0 { t.push_str(&format!(", {} left out", summary.excluded)); } if summary.unchecked > 0 { t.push_str(&format!(", {} not checked yet", summary.unchecked)); } t.push('.'); t }; g.set_summary(text.into()); let unread: Vec<&Reviewed> = review .iter() .filter(|r| r.check == Check::Unchecked) .collect(); let to_read: usize = unread .iter() .filter(|r| !r.group.copies.iter().all(|c| c.content_hash.is_some())) .flat_map(|r| r.group.copies.iter()) .filter(|c| c.probe.is_none()) .map(|c| c.file_size.min(2 * dr_catalog::duplicates::PROBE_WINDOW) as usize) .sum(); g.set_can_check(!unread.is_empty() && !ctl.running.get()); g.set_check_label(format!("Check {}", plural(unread.len(), "group", "groups")).into()); g.set_check_cost(if unread.is_empty() { "".into() } else if to_read == 0 { "The check reads each copy's sidecar; the files were read before.".into() } else { format!( "The check reads the first and last megabyte of each copy not read \ before — about {} — and each copy's sidecar. What it reads is kept, \ so a second check costs nothing.", quote_bytes(to_read as u64) ) .into() }); g.set_can_confirm(summary.files > 0 && !ctl.running.get()); g.set_confirm_label( if summary.files == 1 { "Move 1 copy to trash".to_string() } else { format!("Move {} copies to trash", summary.files) } .into(), ); // The page. let pages = total.div_ceil(PAGE).max(1); let page = ctl.page.get().min(pages - 1); let first = page * PAGE; let last = (first + PAGE).min(total); g.set_has_prev(page > 0); g.set_has_next(page + 1 < pages); g.set_page_label(if total == 0 { "".into() } else { format!("Groups {}–{} of {}", first + 1, last, total).into() }); let catalog = library.catalog(); let borrow = catalog.borrow(); let store = library .session() .and_then(|c| dr_thumbs::ThumbStore::open(&crate::library::thumbs_dir(&c.account)).ok()); let ended = ctl.ended.borrow(); let mut rows = Vec::with_capacity(last - first); for (index, r) in review.iter().enumerate().take(last).skip(first) { let (thumb, has_thumb) = thumbnail(ctl, store.as_ref(), r); let done = matches!(ended.get(&index), Some(Ended::Done(_))); let (status, warn) = match (ended.get(&index), &r.check) { (Some(Ended::Done(s)), _) => (s.clone(), false), (Some(Ended::Failed(s)), _) => (format!("Left as it was: {s}"), true), (None, Check::Unchecked) => ("Not checked yet".to_string(), false), (None, Check::Skip(why)) => (format!("Skipped: {why}"), true), (None, Check::Same(_)) if !r.include => ( "The same file · left out: nothing will move".to_string(), false, ), (None, Check::Same(edits)) => ( if edits.edited.iter().any(|e| *e) { "The same file, with one edit between them".to_string() } else { "The same file".to_string() }, false, ), }; let detail = match (&r.check, borrow.as_ref()) { (Check::Same(edits), Some(cat)) if r.include && !done => { let survivor = r.group.copies[r.survivor].image; let others: Vec = r .group .copies .iter() .enumerate() .filter(|(i, _)| *i != r.survivor) .map(|(_, c)| c.image) .collect(); let carried = edits .source_for(r.survivor) .map(|i| r.group.copies[i].file_name().to_string()); match dr_catalog::duplicates::preview(cat.connection(), survivor, &others) { Ok(o) => describe(&o, carried.as_deref()), Err(e) => format!("Cannot preview the merge: {e}"), } } _ => String::new(), }; let title = format!( "{} · {} · {:.1} MB", r.group.camera, dr_types::format_date(r.group.captured_at), r.group.file_size as f64 / (1024.0 * 1024.0) ); rows.push(DuplicateRow { title: title.into(), thumb, has_thumb, copies: ModelRc::new(VecModel::from( r.group .copies .iter() .enumerate() .map(|(i, c)| DuplicateCopy { path: c.source_ref.clone().into(), survivor: i == r.survivor, }) .collect::>(), )), status: status.into(), detail: detail.into(), warn, include: r.include, can_include: matches!(r.check, Check::Same(_) | Check::Unchecked), done, }); } g.set_rows(ModelRc::new(VecModel::from(rows))); } /// One picture for a group, from the first copy the thumbnail store holds. fn thumbnail( ctl: &Rc, store: Option<&dr_thumbs::ThumbStore>, r: &Reviewed, ) -> (slint::Image, bool) { let mut thumbs = ctl.thumbs.borrow_mut(); for id in r.group.copies.iter().filter_map(|c| c.file_id) { if let Some(img) = thumbs.get(&id) { return (img.clone(), true); } } let Some(store) = store else { return (slint::Image::default(), false); }; for id in r.group.copies.iter().filter_map(|c| c.file_id) { let Ok(Some(t)) = store.get(id, dr_thumbs::ThumbSize::Grid) else { continue; }; let Ok((w, h, rgba)) = dr_thumbs::codec::decode_rgba(&t.bytes) else { continue; }; let mut buf = slint::SharedPixelBuffer::::new(w, h); let n = buf.make_mut_bytes().len().min(rgba.len()); buf.make_mut_bytes()[..n].copy_from_slice(&rgba[..n]); let img = slint::Image::from_rgba8(buf); thumbs.insert(id, img.clone()); return (img, true); } (slint::Image::default(), false) } /// What a consolidation brings to the survivor, in words. pub fn describe(o: &Outcome, carried_edit: Option<&str>) -> String { let mut parts: Vec = Vec::new(); if o.rating.1 != o.rating.0 { parts.push(format!("rating {}★ (was {}★)", o.rating.1, o.rating.0)); } if o.flag.1 != o.flag.0 { parts.push( match o.flag.1 { 1 => "the pick flag", 2 => "the reject flag", _ => "no flag", } .to_string(), ); } if o.label.1 != o.label.0 { parts.push("a colour label".into()); } if o.keywords_added > 0 { parts.push(plural(o.keywords_added, "keyword", "keywords")); } if o.collections_joined > 0 { parts.push(plural(o.collections_joined, "collection", "collections")); } if o.faces.moved > 0 { parts.push(plural(o.faces.moved, "face", "faces")); } if o.faces.named > 0 { parts.push(plural(o.faces.named, "name", "names")); } if let Some(from) = carried_edit { parts.push(format!("the edit from {from}")); } let mut out = if parts.is_empty() { "Nothing to merge: the copy that stays already has it all.".to_string() } else { format!("The copy that stays gains {}.", parts.join(", ")) }; let mut conflicts = Vec::new(); if o.flag_conflict { conflicts.push("flags"); } if o.label_conflict { conflicts.push("colour labels"); } if !conflicts.is_empty() { out.push_str(&format!( " The copies disagree on {}; the one that stays keeps its own.", conflicts.join(" and ") )); } if o.faces.conflicts > 0 { out.push_str(&format!( " {} named differently; the one that stays keeps its names.", plural(o.faces.conflicts, "face is", "faces are") )); } if o.faces.unmatched_named > 0 { out.push_str(&format!( " {} stay with the copies in the trash.", plural(o.faces.unmatched_named, "named face", "named faces") )); } out } fn start_check( window: &AppWindow, ctl: &Rc, library: &Rc, ) { if ctl.running.get() { return; } let Some(conn) = library.session() else { return; }; let Some(cache_dir) = library.sidecar_cache_dir() else { return; }; let groups: Vec<(usize, dr_catalog::duplicates::Group)> = ctl .review .borrow() .iter() .enumerate() .filter(|(_, r)| r.check == Check::Unchecked) .map(|(i, r)| (i, r.group.clone())) .collect(); if groups.is_empty() { return; } let stop = Arc::new(AtomicBool::new(false)); *ctl.stop.borrow_mut() = stop.clone(); let count = groups.len(); let rx = dups::spawn_check( conn.clone(), crate::library::catalog_path(&conn.account), cache_dir, groups, stop, ); let job = ctl.activity.begin( crate::activity::Kind::Download, format!( "Checking {}", plural(count, "duplicate group", "duplicate groups") ), ); ctl.consolidating.set(false); ctl.counts.set((0, 0, 0)); drain(window, ctl, library, rx, job, None); } fn start_consolidate( window: &AppWindow, ctl: &Rc, library: &Rc, after: After, ) { if ctl.running.get() { return; } let Some(conn) = library.session() else { return; }; let Some(cache_dir) = library.sidecar_cache_dir() else { return; }; let plans = { let catalog = library.catalog(); let borrow = catalog.borrow(); let Some(cat) = borrow.as_ref() else { return }; match dups::plan(cat, &conn.account.root, &ctl.review.borrow()) { Ok(p) => p, Err(e) => { window .global::() .set_error(format!("Planning: {e}").into()); return; } } }; if plans.is_empty() { return; } let files: usize = plans.iter().map(|p| p.copies.len()).sum(); log::info!( "duplicates: consolidating {} group(s), {files} copies to the trash", plans.len() ); let stop = Arc::new(AtomicBool::new(false)); *ctl.stop.borrow_mut() = stop.clone(); let rx = dups::spawn_consolidate( conn.clone(), crate::library::catalog_path(&conn.account), cache_dir, plans, stop, ); let job = ctl.activity.begin( crate::activity::Kind::Trash, format!( "Moving {} to the trash", plural(files, "duplicate copy", "duplicate copies") ), ); ctl.consolidating.set(true); ctl.survivors.borrow_mut().clear(); ctl.counts.set((0, 0, 0)); window .global::() .set_library_status(format!("Moving {files} duplicate copies to the trash…").into()); drain(window, ctl, library, rx, job, Some(after)); } /// Take a worker's reports on the UI thread, as every job here does: a /// channel polled by a timer, so nothing waits on a file. fn drain( window: &AppWindow, ctl: &Rc, library: &Rc, rx: Receiver, job: Activity, after: Option, ) { ctl.running.set(true); let g = window.global::(); g.set_running(true); g.set_fraction(0.0); g.set_progress_label( if ctl.consolidating.get() { "Consolidating…" } else { "Checking…" } .into(), ); show(window, ctl, library); let (weak, ctl_cb, library) = (window.as_weak(), ctl.clone(), library.clone()); let timer = slint::Timer::default(); timer.start( slint::TimerMode::Repeated, std::time::Duration::from_millis(150), move || { let Some(w) = weak.upgrade() else { return }; let ctl = &ctl_cb; let mut changed = false; loop { let msg = match rx.try_recv() { Ok(m) => m, Err(std::sync::mpsc::TryRecvError::Empty) => break, Err(std::sync::mpsc::TryRecvError::Disconnected) => { finish(&w, ctl, &library, &job, Some("stopped without finishing".into()), after.as_ref()); return; } }; changed = true; match msg { DupMessage::Checked { index, check } => { if let Some(r) = ctl.review.borrow_mut().get_mut(index) { r.check = check; } } DupMessage::Consolidated { index, survivor, outcome, } => { ctl.survivors.borrow_mut().push(survivor); let (ok, files, bad) = ctl.counts.get(); ctl.counts.set((ok + 1, files + outcome.trashed, bad)); ctl.ended.borrow_mut().insert( index, Ended::Done(format!( "Consolidated: {} in the trash", plural(outcome.trashed, "copy", "copies") )), ); } DupMessage::Failed { index, reason } => { let (ok, files, bad) = ctl.counts.get(); ctl.counts.set((ok, files, bad + 1)); ctl.ended.borrow_mut().insert(index, Ended::Failed(reason)); } DupMessage::Progress { done, total } => { job.progress(done, total); let g = w.global::(); g.set_fraction(done as f32 / total.max(1) as f32); g.set_progress_label( format!( "{} {done} of {total} groups", if ctl.consolidating.get() { "Consolidated" } else { "Checked" } ) .into(), ); if ctl.consolidating.get() { let (_, files, _) = ctl.counts.get(); w.global::().set_library_status( format!("Duplicates: {done} of {total} groups · {files} copies in the trash") .into(), ); } } DupMessage::Finished { stopped } => { finish(&w, ctl, &library, &job, stopped, after.as_ref()); return; } } } // One redraw per tick, and only of the page on screen. if changed && w.get_active_page() == Page::Duplicates { show(&w, ctl, &library); } }, ); *ctl.timer.borrow_mut() = Some(timer); } fn finish( w: &AppWindow, ctl: &Rc, library: &Rc, job: &Activity, stopped: Option, after: Option<&After>, ) { ctl.running.set(false); let g = w.global::(); g.set_running(false); if ctl.consolidating.get() { let (ok, files, bad) = ctl.counts.get(); let skipped = dups::summarise(&ctl.review.borrow()).skipped; let mut status = format!( "Duplicates: {} consolidated, {} in the trash, {} skipped", plural(ok, "group", "groups"), plural(files, "copy", "copies"), skipped + bad ); if let Some(why) = &stopped { status.push_str(&format!(" · {why}")); } if bad > 0 || stopped.is_some() { job.fail(status.clone()); } else { job.finish(status.clone()); } w.global::().set_library_status(status.into()); // The consolidated groups leave the review: they are no longer // duplicates, and their rows would only offer to do it again. A // failed group stays, with its reason, under its new index. { let old_ended = std::mem::take(&mut *ctl.ended.borrow_mut()); let old = std::mem::take(&mut *ctl.review.borrow_mut()); let mut review = Vec::with_capacity(old.len()); let mut ended = HashMap::new(); for (i, r) in old.into_iter().enumerate() { match old_ended.get(&i) { Some(Ended::Done(_)) => continue, Some(e) => { ended.insert(review.len(), e.clone()); } None => {} } review.push(r); } *ctl.review.borrow_mut() = review; *ctl.ended.borrow_mut() = ended; } let survivors = std::mem::take(&mut *ctl.survivors.borrow_mut()); crate::library_ui::save_judgements(w, library, &survivors); if let Some(after) = after { after(w); } if let Some(cat) = library.catalog().borrow().as_ref() { refresh_count(w, cat); } } else { let s = dups::summarise(&ctl.review.borrow()); let note = format!( "{} the same, {} skipped", plural(s.groups + s.excluded, "group", "groups"), s.skipped ); match &stopped { Some(why) => { job.fail(format!("{note} · {why}")); g.set_error(why.as_str().into()); } None => job.finish(note), } } if let Some(t) = ctl.timer.borrow().as_ref() { t.stop(); } ctl.thumbs.borrow_mut().clear(); show(w, ctl, library); } #[cfg(test)] mod tests { use super::*; #[test] fn a_merge_is_described_in_words_with_its_conflicts() { let mut o = Outcome { rating: (2, 4), keywords_added: 1, collections_joined: 2, flag_conflict: true, ..Default::default() }; o.faces.named = 1; let s = describe(&o, Some("_MG_4623.CR2")); assert!(s.contains("rating 4★ (was 2★)"), "{s}"); assert!(s.contains("1 keyword,"), "{s}"); assert!(s.contains("2 collections"), "{s}"); assert!(s.contains("1 name"), "{s}"); assert!(s.contains("the edit from _MG_4623.CR2"), "{s}"); assert!(s.contains("disagree on flags"), "{s}"); } #[test] fn nothing_to_merge_says_so() { let s = describe(&Outcome::default(), None); assert!(s.starts_with("Nothing to merge"), "{s}"); } #[test] fn transfers_are_quoted_in_the_unit_a_person_reads() { assert_eq!(quote_bytes(10 * 1024 * 1024 * 1024), "10.0 GB"); assert_eq!(quote_bytes(6 * 1024 * 1024), "6 MB"); } }