//! TRACES: FR-CAT-13 | FR-NC-9 | NFR-R4 //! Standard XMP sidecars, read into the catalog and written back out of it. //! //! `dr-xmp` reads and writes the file. This is the other half the requirement //! asks for and the half `docs/outstanding.md` called "the larger": which //! images a given `.xmp` describes, what the catalog holds about them, how //! the two are reconciled, where a disagreement goes, and the write in the //! other direction. Nothing here parses XML and nothing in `dr-xmp` knows a //! catalog exists. //! //! # Two conventions for one file //! //! Lightroom writes `IMG_0001.xmp` beside `IMG_0001.CR3`; darktable writes //! `IMG_0001.CR3.xmp`. Both are answered by [`images_for`]: the path with //! `.xmp` taken off is tried as an image path first, exactly, and failing that //! as a stem shared with the images beside it — the rule DarkRoom's own //! sidecar already follows, under which a RAW and the JPEG the camera wrote //! beside it are one photograph (FR-CAT-11) and share the document. //! //! # Precedence, and where a disagreement goes //! //! A standard XMP carries no revision and no device, so nothing in it can say //! whether its rating is newer than the catalog's. The automatic pull //! therefore runs [`dr_xmp::reconcile`] with the catalog winning: keywords //! union, and every whole-valued field is taken only where the catalog holds //! none. A genuine disagreement — both sides hold a value, and different ones — //! is written to `xmp_conflicts` rather than resolved, and the requirement's //! "a metadata reload offered" is that table with a button in front of it. //! [`reload`] is the button: the same reconciliation with the sidecar winning, //! asked for by a person. //! //! # Detection //! //! "External modification of an XMP sidecar shall be detected." The scan //! already records the ETag of every sidecar it has taken in, and fetches //! only those whose ETag has moved. An `.xmp` edited in another application //! is precisely a file whose ETag has moved, so the detection is the pull's //! ordinary incrementality — nothing watches a directory, and nothing needs //! to. What is new is what happens after: the re-read reconciles again, and //! the second reading is where a conflict first appears. //! //! # Writing, and why it is off //! //! NFR-R4 makes source-adjacent writes opt-in. A library shared with another //! editor is one where a file DarkRoom wrote can be read by something else, //! and that is exactly what [`Xmp::rewrite`] is built for — it rewrites only //! the properties DarkRoom owns and copies everything else through byte for //! byte. But the option to write beside somebody's originals is theirs to //! switch on, and until they do the catalog and DarkRoom's own sidecar are the //! only things a judgement reaches. use dr_catalog::Catalog; use dr_types::{FlagState, ImageId}; use dr_xmp::{Precedence, Rating, Xmp}; use rusqlite::Connection; /// The two places an image's XMP sidecar may be, in the order they are /// tried when writing: the darktable spelling first, because it names the /// image unambiguously, then Lightroom's. /// /// When reading, whichever the scan found is the one read; this is for the /// write, which has to choose. An existing file of either spelling is /// rewritten in place, and a photograph with neither gets Lightroom's, since /// it is the spelling more applications look for. pub fn candidate_paths(image_path: &str) -> [String; 2] { let stem = match image_path.rsplit_once('.') { Some((stem, ext)) if !ext.contains('/') => stem, _ => image_path, }; [format!("{image_path}.xmp"), format!("{stem}.xmp")] } /// Whether a listing entry is a standard XMP sidecar rather than DarkRoom's. pub fn is_xmp(path: &str) -> bool { path.rsplit_once('.') .is_some_and(|(_, ext)| ext.eq_ignore_ascii_case(dr_sync::scan::XMP_EXTENSION)) } /// The images an `.xmp` at `path` describes: their ids and default versions. /// /// Empty when the sidecar sits beside nothing this catalog knows, which is /// the ordinary case for a file that arrived before its photograph was /// scanned — the next pull reads it again, because no ETag is recorded for a /// sidecar that reached nothing. pub fn images_for(conn: &Connection, root_id: i64, path: &str) -> Vec<(ImageId, i64)> { let Some(named) = path .strip_suffix(".xmp") .or_else(|| path.strip_suffix(".XMP")) else { return Vec::new(); }; let ids = |sql: &str, arg: &str| -> Vec { let Ok(mut stmt) = conn.prepare(sql) else { return Vec::new(); }; stmt.query_map(rusqlite::params![root_id, arg], |r| r.get::<_, i64>(0)) .map(|rows| rows.flatten().map(|i| ImageId(i as u64)).collect()) .unwrap_or_default() }; // darktable's spelling: the name before `.xmp` is the image itself. let mut images = ids( "SELECT id FROM images WHERE root_id = ?1 AND source_ref = ?2", named, ); if images.is_empty() { // Lightroom's: a stem shared with the photograph, and with the JPEG // beside it. The escape is what makes `%` and `_` in a folder name // literal, and the Rust-side check is what actually decides — a LIKE // is a filter, not an answer. let prefix = named .replace('\\', "\\\\") .replace('%', "\\%") .replace('_', "\\_"); let candidates: Vec<(ImageId, String)> = { let Ok(mut stmt) = conn.prepare( "SELECT id, source_ref FROM images WHERE root_id = ?1 AND source_ref LIKE ?2 ESCAPE '\\'", ) else { return Vec::new(); }; stmt.query_map(rusqlite::params![root_id, format!("{prefix}.%")], |r| { Ok((ImageId(r.get::<_, i64>(0)? as u64), r.get::<_, String>(1)?)) }) .map(|rows| rows.flatten().collect()) .unwrap_or_default() }; images = candidates .into_iter() .filter(|(_, source)| candidate_paths(source)[1].eq_ignore_ascii_case(path)) .map(|(id, _)| id) .collect(); } images .into_iter() .filter_map(|image| { let version = dr_catalog::rating::default_version_id(conn, image).ok()?; Some((image, version)) }) .collect() } /// What the catalog holds about one image, as the sidecar would carry it. /// /// Rating and flag are two axes here and one field there: a rejection is /// written as Adobe's `-1` and a rating as its stars, and a frame that is /// both rejected and starred loses the stars in the file — the loss the /// format has and `dr_xmp::Rating` records. Reading goes the other way in /// [`apply`]. pub fn record_of(conn: &Connection, image: ImageId, version: i64) -> Xmp { let mut xmp = Xmp::default(); let row = conn .query_row( "SELECT rating, flag, label FROM versions WHERE id = ?1", [version], |r| { Ok(( r.get::<_, i64>(0)?, r.get::<_, i64>(1)?, r.get::<_, Option>(2)?, )) }, ) .ok(); if let Some((rating, flag, label)) = row { xmp.rating = if flag == flag_code(FlagState::Reject) { Some(Rating::Rejected) } else if rating > 0 { Some(Rating::Stars(rating.clamp(0, 5) as u8)) } else { None }; xmp.set_colour(dr_catalog::rating::label_from_code(label)); } xmp.keywords = dr_catalog::keywords::for_image(conn, image).unwrap_or_default(); xmp } /// Write a reconciled record onto one image. /// /// Only what the record holds: a `None` rating is *unrated* and is not /// written over a star, for the reason `dr_pipeline::sidecar::merge_judgement` /// gives — a file that says nothing cannot erase an afternoon's culling. A /// rejection sets the flag and leaves the stars alone; stars set the rating /// and, if the frame was rejected, lift the rejection, since the file said /// it was worth a number. Keywords are added and never removed here, which /// is what a union means. `label` is set where the file names one of the /// five colours. /// /// Returns whether any column moved. pub fn apply( conn: &Connection, image: ImageId, version: i64, record: &Xmp, ) -> Result { let mut changed = false; match record.rating { Some(Rating::Rejected) => { changed |= conn .execute( "UPDATE versions SET flag = ?2 WHERE id = ?1 AND flag <> ?2", rusqlite::params![version, flag_code(FlagState::Reject)], ) .map_err(|e| e.to_string())? > 0; } Some(Rating::Stars(n)) if n > 0 => { changed |= conn .execute( "UPDATE versions SET rating = ?2, flag = CASE WHEN flag = ?3 THEN 0 ELSE flag END WHERE id = ?1 AND (rating <> ?2 OR flag = ?3)", rusqlite::params![version, n.min(5) as i64, flag_code(FlagState::Reject)], ) .map_err(|e| e.to_string())? > 0; } _ => {} } if let Some(colour) = record.colour() { changed |= conn .execute( "UPDATE versions SET label = ?2 WHERE id = ?1 AND label IS NOT ?2", rusqlite::params![version, dr_catalog::rating::label_code(colour)], ) .map_err(|e| e.to_string())? > 0; } if !record.keywords.is_empty() { let have = dr_catalog::keywords::for_image(conn, image).unwrap_or_default(); for word in &record.keywords { if have.iter().any(|h| h.eq_ignore_ascii_case(word)) { continue; } match dr_catalog::keywords::assign(conn, &[image], word) { Ok(n) => changed |= n > 0, // A word the vocabulary refuses — empty once trimmed, or a // separator on its own — costs that word and not the file. Err(e) => log::debug!("xmp keyword {word:?} on {}: {e}", image.0), } } } Ok(changed) } /// TRACES: FR-CAT-13 /// Before rewriting an existing sidecar, keep what the catalog cannot hold. /// /// `Xmp::rewrite` replaces the owned properties wholesale — that is what /// makes "no rating" mean the rating goes — and the catalog has columns for /// three of them: rating and flag, label, keywords. A record built from the /// catalog therefore says nothing about a title, a caption, a copyright /// line or a hierarchical subject, and writing it as it stands would delete /// all four from a sidecar Lightroom wrote, on every judgement. So those /// come through from the file, and so does a label that is not one of the /// five colours — the catalog kept no text for it, and the file's is the /// only copy. pub fn carry_through(record: &mut Xmp, existing: &Xmp) { record.hierarchical_subjects = existing.hierarchical_subjects.clone(); record.title = existing.title.clone(); record.description = existing.description.clone(); record.creators = existing.creators.clone(); record.copyright = existing.copyright.clone(); record.credit = existing.credit.clone(); record.usage_terms = existing.usage_terms.clone(); if record.label.is_none() && existing.colour().is_none() { record.label = existing.label.clone(); } } /// What one sidecar did when taken in. #[derive(Debug, Default, Clone, PartialEq, Eq)] pub struct TakenIn { /// Images whose rows moved. pub changed: usize, /// Images the sidecar described at all. Zero means it sits beside nothing /// this catalog has yet, and its ETag must not be recorded. pub described: usize, /// The fields both sides held and disagreed on, across every image. pub conflicts: Vec, } /// TRACES: FR-CAT-13 | FR-NC-9 /// Take a standard sidecar into the catalog, with the catalog winning. /// /// The automatic path. Every image the file describes is reconciled against /// what the catalog holds for it; the merged record is applied; and a /// disagreement is recorded in `xmp_conflicts` for a person to settle, never /// resolved here. pub fn take_in( conn: &Connection, root_id: i64, path: &str, text: &str, now: i64, ) -> Result { let sidecar = Xmp::parse(text).map_err(|e| e.to_string())?; let mut out = TakenIn::default(); for (image, version) in images_for(conn, root_id, path) { out.described += 1; let mine = record_of(conn, image, version); let reconciled = dr_xmp::reconcile(&mine, &sidecar, Precedence::Catalog); if apply(conn, image, version, &reconciled.merged)? { out.changed += 1; } for field in reconciled.conflicts { if !out.conflicts.contains(&field) { out.conflicts.push(field); } } } if out.conflicts.is_empty() { clear_conflict(conn, root_id, path); } else if out.described > 0 { record_conflict(conn, root_id, path, &out.conflicts, now); } Ok(out) } /// TRACES: FR-CAT-13 /// The offered reload: the same reconciliation with the sidecar winning. /// /// Asked for by a person, per file, which is the consent the module /// documentation says this needs. Clears the conflict whatever the outcome — /// the person has now seen it, and if the file still disagrees the next pull /// that sees it change will say so again. pub fn reload(conn: &Connection, root_id: i64, path: &str, text: &str) -> Result { let sidecar = Xmp::parse(text).map_err(|e| e.to_string())?; let mut out = TakenIn::default(); for (image, version) in images_for(conn, root_id, path) { out.described += 1; let mine = record_of(conn, image, version); let reconciled = dr_xmp::reconcile(&mine, &sidecar, Precedence::Sidecar); // The sidecar wins outright on the whole-valued fields, and that // includes a rating it holds and the catalog's it replaces — which // `apply` does. What `apply` will not do is *clear* a star the file // does not mention, and a reload does not ask it to: the file said // nothing, and nothing is not a judgement. if apply(conn, image, version, &reconciled.merged)? { out.changed += 1; } } clear_conflict(conn, root_id, path); Ok(out) } /// One outstanding disagreement, for the settings page. #[derive(Debug, Clone, PartialEq, Eq)] pub struct Conflict { pub path: String, pub fields: String, } /// Every sidecar in this library that disagrees with the catalog. pub fn conflicts(catalog: &Catalog, root_id: i64) -> Vec { let Ok(mut stmt) = catalog .connection() .prepare("SELECT path, fields FROM xmp_conflicts WHERE root_id = ?1 ORDER BY path") else { return Vec::new(); }; stmt.query_map([root_id], |r| { Ok(Conflict { path: r.get(0)?, fields: r.get(1)?, }) }) .map(|rows| rows.flatten().collect()) .unwrap_or_default() } fn record_conflict( conn: &Connection, root_id: i64, path: &str, fields: &[dr_xmp::Field], now: i64, ) { let fields = fields .iter() .map(|f| format!("{f:?}")) .collect::>() .join(" "); let done = conn.execute( "INSERT INTO xmp_conflicts(root_id, path, fields, seen_at) VALUES (?1, ?2, ?3, ?4) ON CONFLICT(root_id, path) DO UPDATE SET fields = excluded.fields, seen_at = excluded.seen_at", rusqlite::params![root_id, path, fields, now], ); if let Err(e) = done { log::debug!("recording xmp conflict for {path}: {e}"); } } fn clear_conflict(conn: &Connection, root_id: i64, path: &str) { let _ = conn.execute( "DELETE FROM xmp_conflicts WHERE root_id = ?1 AND path = ?2", rusqlite::params![root_id, path], ); } fn flag_code(flag: FlagState) -> i64 { match flag { FlagState::Unflagged => 0, FlagState::Pick => 1, FlagState::Reject => 2, } } #[cfg(test)] mod tests { use super::*; fn catalog() -> Catalog { let c = Catalog::in_memory().unwrap(); c.connection() .execute_batch( "INSERT INTO roots(id, kind, label) VALUES (1, 'remote', 'lib'); INSERT INTO images(id, root_id, source_ref, added_at) VALUES (1, 1, 'Photos/IMG_0001.CR3', 0), (2, 1, 'Photos/IMG_0001.JPG', 0), (3, 1, 'Photos/IMG_0002.CR3', 0);", ) .unwrap(); dr_catalog::rating::ensure_default_versions(c.connection()).unwrap(); c } const LIGHTROOM: &str = r#" puffiniceland "#; /// Both spellings reach the photograph, and Lightroom's reaches the JPEG /// beside it too. #[test] fn both_namings_find_their_images() { let c = catalog(); let conn = c.connection(); let mut lr: Vec = images_for(conn, 1, "Photos/IMG_0001.xmp") .into_iter() .map(|(i, _)| i.0) .collect(); lr.sort(); assert_eq!(lr, [1, 2], "the RAW and the JPEG are one photograph"); let dt: Vec = images_for(conn, 1, "Photos/IMG_0001.CR3.xmp") .into_iter() .map(|(i, _)| i.0) .collect(); assert_eq!(dt, [1], "darktable's names the file exactly"); assert!(images_for(conn, 1, "Photos/IMG_9999.xmp").is_empty()); } /// The ordinary pull: an empty catalog takes everything the file says. #[test] fn a_sidecar_fills_what_the_catalog_lacks() { let c = catalog(); let conn = c.connection(); let taken = take_in(conn, 1, "Photos/IMG_0002.xmp", LIGHTROOM, 1).unwrap(); assert_eq!(taken.described, 1); assert_eq!(taken.changed, 1); assert!(taken.conflicts.is_empty()); let record = record_of(conn, ImageId(3), 3); assert_eq!(record.rating, Some(Rating::Stars(4))); assert_eq!(record.colour(), Some(dr_types::ColourLabel::Red)); assert_eq!(record.keywords, ["iceland", "puffin"]); assert!(conflicts(&c, 1).is_empty()); } /// A rating the catalog already holds is not overwritten by the file, /// the disagreement is recorded, and the reload — a person asking — /// takes the file's. #[test] fn a_disagreement_is_recorded_and_a_reload_settles_it() { let c = catalog(); let conn = c.connection(); dr_catalog::rating::set_rating(conn, ImageId(3), 2).unwrap(); let taken = take_in(conn, 1, "Photos/IMG_0002.xmp", LIGHTROOM, 1).unwrap(); assert_eq!(taken.conflicts, [dr_xmp::Field::Rating]); assert_eq!( record_of(conn, ImageId(3), 3).rating, Some(Rating::Stars(2)), "the catalog's own rating stands" ); assert_eq!( record_of(conn, ImageId(3), 3).keywords, ["iceland", "puffin"], "while the keywords still union" ); let open = conflicts(&c, 1); assert_eq!(open.len(), 1); assert_eq!(open[0].path, "Photos/IMG_0002.xmp"); assert_eq!(open[0].fields, "Rating"); reload(conn, 1, "Photos/IMG_0002.xmp", LIGHTROOM).unwrap(); assert_eq!( record_of(conn, ImageId(3), 3).rating, Some(Rating::Stars(4)) ); assert!(conflicts(&c, 1).is_empty(), "settled"); } /// A rejection and a star count are two axes here and one field there. #[test] fn a_rejection_crosses_as_adobes_minus_one_and_back() { let c = catalog(); let conn = c.connection(); dr_catalog::rating::set_flag(conn, ImageId(3), FlagState::Reject).unwrap(); assert_eq!( record_of(conn, ImageId(3), 3).rating, Some(Rating::Rejected) ); let back = Xmp { rating: Some(Rating::Stars(3)), ..Default::default() }; apply(conn, ImageId(3), 3, &back).unwrap(); let after = record_of(conn, ImageId(3), 3); assert_eq!( after.rating, Some(Rating::Stars(3)), "the stars lift the rejection" ); } /// A judgement written over a Lightroom sidecar must not cost the caption /// Lightroom wrote, since the catalog never held it. #[test] fn a_rewrite_keeps_what_the_catalog_has_no_column_for() { let theirs = Xmp { title: Some("Puffins at dusk".into()), copyright: Some("© Someone".into()), hierarchical_subjects: vec!["Places|Iceland".into()], label: Some("Second choice".into()), rating: Some(Rating::Stars(1)), ..Default::default() }; let mut ours = Xmp { rating: Some(Rating::Stars(4)), keywords: vec!["puffin".into()], ..Default::default() }; carry_through(&mut ours, &theirs); assert_eq!(ours.rating, Some(Rating::Stars(4)), "the judgement is ours"); assert_eq!(ours.title.as_deref(), Some("Puffins at dusk")); assert_eq!(ours.copyright.as_deref(), Some("© Someone")); assert_eq!(ours.hierarchical_subjects, ["Places|Iceland"]); assert_eq!( ours.label.as_deref(), Some("Second choice"), "a label that is not one of the five colours has no other copy" ); let mut red = Xmp::default(); red.set_colour(Some(dr_types::ColourLabel::Red)); carry_through(&mut red, &theirs); assert_eq!( red.label.as_deref(), Some("Red"), "but a colour of ours wins" ); } /// The two spellings a write chooses between. #[test] fn the_write_tries_darktables_spelling_then_lightrooms() { assert_eq!( candidate_paths("Photos/IMG_0001.CR3"), ["Photos/IMG_0001.CR3.xmp", "Photos/IMG_0001.xmp"] ); assert_eq!( candidate_paths("a.b/IMG"), ["a.b/IMG.xmp", "a.b/IMG.xmp"], "a dot in a folder is not an extension" ); } }