DarkRoom/ui/dr-ui/src/xmp_sync.rs

//! TRACES: FR-CAT-13 | FR-NC-9 | NFR-R4
//! Standard XMP sidecars, read into the catalog and written back out of it.
//!
//! `dr-xmp` reads and writes the file. This is the other half the requirement
//! asks for and the half `docs/dev/outstanding.md` called "the larger": which
//! images a given `.xmp` describes, what the catalog holds about them, how
//! the two are reconciled, where a disagreement goes, and the write in the
//! other direction. Nothing here parses XML and nothing in `dr-xmp` knows a
//! catalog exists.
//!
//! # Two conventions for one file
//!
//! Lightroom writes `IMG_0001.xmp` beside `IMG_0001.CR3`; darktable writes
//! `IMG_0001.CR3.xmp`. Both are answered by [`images_for`]: the path with
//! `.xmp` taken off is tried as an image path first, exactly, and failing that
//! as a stem shared with the images beside it — the rule DarkRoom's own
//! sidecar already follows, under which a RAW and the JPEG the camera wrote
//! beside it are one photograph (FR-CAT-11) and share the document.
//!
//! # Precedence, and where a disagreement goes
//!
//! A standard XMP carries no revision and no device, so nothing in it can say
//! whether its rating is newer than the catalog's. The automatic pull
//! therefore runs [`dr_xmp::reconcile`] with the catalog winning: keywords
//! union, and every whole-valued field is taken only where the catalog holds
//! none. A genuine disagreement — both sides hold a value, and different ones —
//! is written to `xmp_conflicts` rather than resolved, and the requirement's
//! "a metadata reload offered" is that table with a button in front of it.
//! [`reload`] is the button: the same reconciliation with the sidecar winning,
//! asked for by a person.
//!
//! # Detection
//!
//! "External modification of an XMP sidecar shall be detected." The scan
//! already records the ETag of every sidecar it has taken in, and fetches
//! only those whose ETag has moved. An `.xmp` edited in another application
//! is precisely a file whose ETag has moved, so the detection is the pull's
//! ordinary incrementality — nothing watches a directory, and nothing needs
//! to. What is new is what happens after: the re-read reconciles again, and
//! the second reading is where a conflict first appears.
//!
//! # Writing, and why it is off
//!
//! NFR-R4 makes source-adjacent writes opt-in. A library shared with another
//! editor is one where a file DarkRoom wrote can be read by something else,
//! and that is exactly what [`Xmp::rewrite`] is built for — it rewrites only
//! the properties DarkRoom owns and copies everything else through byte for
//! byte. But the option to write beside somebody's originals is theirs to
//! switch on, and until they do the catalog and DarkRoom's own sidecar are the
//! only things a judgement reaches.

use dr_catalog::Catalog;
use dr_types::{FlagState, ImageId};
use dr_xmp::{Precedence, Rating, Xmp};
use rusqlite::Connection;

/// The two places an image's XMP sidecar may be, in the order they are
/// tried when writing: the darktable spelling first, because it names the
/// image unambiguously, then Lightroom's.
///
/// When reading, whichever the scan found is the one read; this is for the
/// write, which has to choose. An existing file of either spelling is
/// rewritten in place, and a photograph with neither gets Lightroom's, since
/// it is the spelling more applications look for.
pub fn candidate_paths(image_path: &str) -> [String; 2] {
    let stem = match image_path.rsplit_once('.') {
        Some((stem, ext)) if !ext.contains('/') => stem,
        _ => image_path,
    };
    [format!("{image_path}.xmp"), format!("{stem}.xmp")]
}

/// Whether a listing entry is a standard XMP sidecar rather than DarkRoom's.
pub fn is_xmp(path: &str) -> bool {
    path.rsplit_once('.')
        .is_some_and(|(_, ext)| ext.eq_ignore_ascii_case(dr_sync::scan::XMP_EXTENSION))
}

/// The images an `.xmp` at `path` describes: their ids and default versions.
///
/// Empty when the sidecar sits beside nothing this catalog knows, which is
/// the ordinary case for a file that arrived before its photograph was
/// scanned — the next pull reads it again, because no ETag is recorded for a
/// sidecar that reached nothing.
pub fn images_for(conn: &Connection, root_id: i64, path: &str) -> Vec<(ImageId, i64)> {
    let Some(named) = path
        .strip_suffix(".xmp")
        .or_else(|| path.strip_suffix(".XMP"))
    else {
        return Vec::new();
    };

    let ids = |sql: &str, arg: &str| -> Vec<ImageId> {
        let Ok(mut stmt) = conn.prepare(sql) else {
            return Vec::new();
        };
        stmt.query_map(rusqlite::params![root_id, arg], |r| r.get::<_, i64>(0))
            .map(|rows| rows.flatten().map(|i| ImageId(i as u64)).collect())
            .unwrap_or_default()
    };

    // darktable's spelling: the name before `.xmp` is the image itself.
    let mut images = ids(
        "SELECT id FROM images WHERE root_id = ?1 AND source_ref = ?2",
        named,
    );
    if images.is_empty() {
        // Lightroom's: a stem shared with the photograph, and with the JPEG
        // beside it. The escape is what makes `%` and `_` in a folder name
        // literal, and the Rust-side check is what actually decides — a LIKE
        // is a filter, not an answer.
        let prefix = named
            .replace('\\', "\\\\")
            .replace('%', "\\%")
            .replace('_', "\\_");
        let candidates: Vec<(ImageId, String)> = {
            let Ok(mut stmt) = conn.prepare(
                "SELECT id, source_ref FROM images
                  WHERE root_id = ?1 AND source_ref LIKE ?2 ESCAPE '\\'",
            ) else {
                return Vec::new();
            };
            stmt.query_map(rusqlite::params![root_id, format!("{prefix}.%")], |r| {
                Ok((ImageId(r.get::<_, i64>(0)? as u64), r.get::<_, String>(1)?))
            })
            .map(|rows| rows.flatten().collect())
            .unwrap_or_default()
        };
        images = candidates
            .into_iter()
            .filter(|(_, source)| candidate_paths(source)[1].eq_ignore_ascii_case(path))
            .map(|(id, _)| id)
            .collect();
    }

    images
        .into_iter()
        .filter_map(|image| {
            let version = dr_catalog::rating::default_version_id(conn, image).ok()?;
            Some((image, version))
        })
        .collect()
}

/// What the catalog holds about one image, as the sidecar would carry it.
///
/// Rating and flag are two axes here and one field there: a rejection is
/// written as Adobe's `-1` and a rating as its stars, and a frame that is
/// both rejected and starred loses the stars in the file — the loss the
/// format has and `dr_xmp::Rating` records. Reading goes the other way in
/// [`apply`].
pub fn record_of(conn: &Connection, image: ImageId, version: i64) -> Xmp {
    let mut xmp = Xmp::default();
    let row = conn
        .query_row(
            "SELECT rating, flag, label FROM versions WHERE id = ?1",
            [version],
            |r| {
                Ok((
                    r.get::<_, i64>(0)?,
                    r.get::<_, i64>(1)?,
                    r.get::<_, Option<i64>>(2)?,
                ))
            },
        )
        .ok();
    if let Some((rating, flag, label)) = row {
        xmp.rating = if flag == flag_code(FlagState::Reject) {
            Some(Rating::Rejected)
        } else if rating > 0 {
            Some(Rating::Stars(rating.clamp(0, 5) as u8))
        } else {
            None
        };
        xmp.set_colour(dr_catalog::rating::label_from_code(label));
    }
    xmp.keywords = dr_catalog::keywords::for_image(conn, image).unwrap_or_default();
    xmp
}

/// Write a reconciled record onto one image.
///
/// Only what the record holds: a `None` rating is *unrated* and is not
/// written over a star, for the reason `dr_pipeline::sidecar::merge_judgement`
/// gives — a file that says nothing cannot erase an afternoon's culling. A
/// rejection sets the flag and leaves the stars alone; stars set the rating
/// and, if the frame was rejected, lift the rejection, since the file said
/// it was worth a number. Keywords are added and never removed here, which
/// is what a union means. `label` is set where the file names one of the
/// five colours.
///
/// Returns whether any column moved.
pub fn apply(
    conn: &Connection,
    image: ImageId,
    version: i64,
    record: &Xmp,
) -> Result<bool, String> {
    let mut changed = false;
    match record.rating {
        Some(Rating::Rejected) => {
            changed |= conn
                .execute(
                    "UPDATE versions SET flag = ?2 WHERE id = ?1 AND flag <> ?2",
                    rusqlite::params![version, flag_code(FlagState::Reject)],
                )
                .map_err(|e| e.to_string())?
                > 0;
        }
        Some(Rating::Stars(n)) if n > 0 => {
            changed |= conn
                .execute(
                    "UPDATE versions
                        SET rating = ?2,
                            flag = CASE WHEN flag = ?3 THEN 0 ELSE flag END
                      WHERE id = ?1 AND (rating <> ?2 OR flag = ?3)",
                    rusqlite::params![version, n.min(5) as i64, flag_code(FlagState::Reject)],
                )
                .map_err(|e| e.to_string())?
                > 0;
        }
        _ => {}
    }
    if let Some(colour) = record.colour() {
        changed |= conn
            .execute(
                "UPDATE versions SET label = ?2 WHERE id = ?1 AND label IS NOT ?2",
                rusqlite::params![version, dr_catalog::rating::label_code(colour)],
            )
            .map_err(|e| e.to_string())?
            > 0;
    }
    if !record.keywords.is_empty() {
        let have = dr_catalog::keywords::for_image(conn, image).unwrap_or_default();
        for word in &record.keywords {
            if have.iter().any(|h| h.eq_ignore_ascii_case(word)) {
                continue;
            }
            match dr_catalog::keywords::assign(conn, &[image], word) {
                Ok(n) => changed |= n > 0,
                // A word the vocabulary refuses — empty once trimmed, or a
                // separator on its own — costs that word and not the file.
                Err(e) => log::debug!("xmp keyword {word:?} on {}: {e}", image.0),
            }
        }
    }
    Ok(changed)
}

/// TRACES: FR-CAT-13
/// Before rewriting an existing sidecar, keep what the catalog cannot hold.
///
/// `Xmp::rewrite` replaces the owned properties wholesale — that is what
/// makes "no rating" mean the rating goes — and the catalog has columns for
/// three of them: rating and flag, label, keywords. A record built from the
/// catalog therefore says nothing about a title, a caption, a copyright
/// line or a hierarchical subject, and writing it as it stands would delete
/// all four from a sidecar Lightroom wrote, on every judgement. So those
/// come through from the file, and so does a label that is not one of the
/// five colours — the catalog kept no text for it, and the file's is the
/// only copy.
pub fn carry_through(record: &mut Xmp, existing: &Xmp) {
    record.hierarchical_subjects = existing.hierarchical_subjects.clone();
    record.title = existing.title.clone();
    record.description = existing.description.clone();
    record.creators = existing.creators.clone();
    record.copyright = existing.copyright.clone();
    record.credit = existing.credit.clone();
    record.usage_terms = existing.usage_terms.clone();
    if record.label.is_none() && existing.colour().is_none() {
        record.label = existing.label.clone();
    }
}

/// What one sidecar did when taken in.
#[derive(Debug, Default, Clone, PartialEq, Eq)]
pub struct TakenIn {
    /// Images whose rows moved.
    pub changed: usize,
    /// Images the sidecar described at all. Zero means it sits beside nothing
    /// this catalog has yet, and its ETag must not be recorded.
    pub described: usize,
    /// The fields both sides held and disagreed on, across every image.
    pub conflicts: Vec<dr_xmp::Field>,
}

/// TRACES: FR-CAT-13 | FR-NC-9
/// Take a standard sidecar into the catalog, with the catalog winning.
///
/// The automatic path. Every image the file describes is reconciled against
/// what the catalog holds for it; the merged record is applied; and a
/// disagreement is recorded in `xmp_conflicts` for a person to settle, never
/// resolved here.
pub fn take_in(
    conn: &Connection,
    root_id: i64,
    path: &str,
    text: &str,
    now: i64,
) -> Result<TakenIn, String> {
    let sidecar = Xmp::parse(text).map_err(|e| e.to_string())?;
    let mut out = TakenIn::default();
    for (image, version) in images_for(conn, root_id, path) {
        out.described += 1;
        let mine = record_of(conn, image, version);
        let reconciled = dr_xmp::reconcile(&mine, &sidecar, Precedence::Catalog);
        if apply(conn, image, version, &reconciled.merged)? {
            out.changed += 1;
        }
        for field in reconciled.conflicts {
            if !out.conflicts.contains(&field) {
                out.conflicts.push(field);
            }
        }
    }
    if out.conflicts.is_empty() {
        clear_conflict(conn, root_id, path);
    } else if out.described > 0 {
        record_conflict(conn, root_id, path, &out.conflicts, now);
    }
    Ok(out)
}

/// TRACES: FR-CAT-13
/// The offered reload: the same reconciliation with the sidecar winning.
///
/// Asked for by a person, per file, which is the consent the module
/// documentation says this needs. Clears the conflict whatever the outcome —
/// the person has now seen it, and if the file still disagrees the next pull
/// that sees it change will say so again.
pub fn reload(conn: &Connection, root_id: i64, path: &str, text: &str) -> Result<TakenIn, String> {
    let sidecar = Xmp::parse(text).map_err(|e| e.to_string())?;
    let mut out = TakenIn::default();
    for (image, version) in images_for(conn, root_id, path) {
        out.described += 1;
        let mine = record_of(conn, image, version);
        let reconciled = dr_xmp::reconcile(&mine, &sidecar, Precedence::Sidecar);
        // The sidecar wins outright on the whole-valued fields, and that
        // includes a rating it holds and the catalog's it replaces — which
        // `apply` does. What `apply` will not do is *clear* a star the file
        // does not mention, and a reload does not ask it to: the file said
        // nothing, and nothing is not a judgement.
        if apply(conn, image, version, &reconciled.merged)? {
            out.changed += 1;
        }
    }
    clear_conflict(conn, root_id, path);
    Ok(out)
}

/// One outstanding disagreement, for the settings page.
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct Conflict {
    pub path: String,
    pub fields: String,
}

/// Every sidecar in this library that disagrees with the catalog.
pub fn conflicts(catalog: &Catalog, root_id: i64) -> Vec<Conflict> {
    let Ok(mut stmt) = catalog
        .connection()
        .prepare("SELECT path, fields FROM xmp_conflicts WHERE root_id = ?1 ORDER BY path")
    else {
        return Vec::new();
    };
    stmt.query_map([root_id], |r| {
        Ok(Conflict {
            path: r.get(0)?,
            fields: r.get(1)?,
        })
    })
    .map(|rows| rows.flatten().collect())
    .unwrap_or_default()
}

fn record_conflict(
    conn: &Connection,
    root_id: i64,
    path: &str,
    fields: &[dr_xmp::Field],
    now: i64,
) {
    let fields = fields
        .iter()
        .map(|f| format!("{f:?}"))
        .collect::<Vec<_>>()
        .join(" ");
    let done = conn.execute(
        "INSERT INTO xmp_conflicts(root_id, path, fields, seen_at) VALUES (?1, ?2, ?3, ?4)
         ON CONFLICT(root_id, path) DO UPDATE SET
             fields = excluded.fields, seen_at = excluded.seen_at",
        rusqlite::params![root_id, path, fields, now],
    );
    if let Err(e) = done {
        log::debug!("recording xmp conflict for {path}: {e}");
    }
}

fn clear_conflict(conn: &Connection, root_id: i64, path: &str) {
    let _ = conn.execute(
        "DELETE FROM xmp_conflicts WHERE root_id = ?1 AND path = ?2",
        rusqlite::params![root_id, path],
    );
}

fn flag_code(flag: FlagState) -> i64 {
    match flag {
        FlagState::Unflagged => 0,
        FlagState::Pick => 1,
        FlagState::Reject => 2,
    }
}

#[cfg(test)]
mod tests {
    use super::*;

    fn catalog() -> Catalog {
        let c = Catalog::in_memory().unwrap();
        c.connection()
            .execute_batch(
                "INSERT INTO roots(id, kind, label) VALUES (1, 'remote', 'lib');
                 INSERT INTO images(id, root_id, source_ref, added_at)
                      VALUES (1, 1, 'Photos/IMG_0001.CR3', 0),
                             (2, 1, 'Photos/IMG_0001.JPG', 0),
                             (3, 1, 'Photos/IMG_0002.CR3', 0);",
            )
            .unwrap();
        dr_catalog::rating::ensure_default_versions(c.connection()).unwrap();
        c
    }

    const LIGHTROOM: &str = r#"<?xpacket begin="" id="W5M0MpCehiHzreSzNTczkc9d"?>
<x:xmpmeta xmlns:x="adobe:ns:meta/">
 <rdf:RDF xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#">
  <rdf:Description rdf:about=""
    xmlns:xmp="http://ns.adobe.com/xap/1.0/"
    xmlns:dc="http://purl.org/dc/elements/1.1/"
    xmp:Rating="4"
    xmp:Label="Red">
   <dc:subject><rdf:Bag><rdf:li>puffin</rdf:li><rdf:li>iceland</rdf:li></rdf:Bag></dc:subject>
  </rdf:Description>
 </rdf:RDF>
</x:xmpmeta>
<?xpacket end="w"?>"#;

    /// Both spellings reach the photograph, and Lightroom's reaches the JPEG
    /// beside it too.
    #[test]
    fn both_namings_find_their_images() {
        let c = catalog();
        let conn = c.connection();
        let mut lr: Vec<u64> = images_for(conn, 1, "Photos/IMG_0001.xmp")
            .into_iter()
            .map(|(i, _)| i.0)
            .collect();
        lr.sort();
        assert_eq!(lr, [1, 2], "the RAW and the JPEG are one photograph");

        let dt: Vec<u64> = images_for(conn, 1, "Photos/IMG_0001.CR3.xmp")
            .into_iter()
            .map(|(i, _)| i.0)
            .collect();
        assert_eq!(dt, [1], "darktable's names the file exactly");

        assert!(images_for(conn, 1, "Photos/IMG_9999.xmp").is_empty());
    }

    /// The ordinary pull: an empty catalog takes everything the file says.
    #[test]
    fn a_sidecar_fills_what_the_catalog_lacks() {
        let c = catalog();
        let conn = c.connection();
        let taken = take_in(conn, 1, "Photos/IMG_0002.xmp", LIGHTROOM, 1).unwrap();
        assert_eq!(taken.described, 1);
        assert_eq!(taken.changed, 1);
        assert!(taken.conflicts.is_empty());

        let record = record_of(conn, ImageId(3), 3);
        assert_eq!(record.rating, Some(Rating::Stars(4)));
        assert_eq!(record.colour(), Some(dr_types::ColourLabel::Red));
        assert_eq!(record.keywords, ["iceland", "puffin"]);
        assert!(conflicts(&c, 1).is_empty());
    }

    /// A rating the catalog already holds is not overwritten by the file,
    /// the disagreement is recorded, and the reload — a person asking —
    /// takes the file's.
    #[test]
    fn a_disagreement_is_recorded_and_a_reload_settles_it() {
        let c = catalog();
        let conn = c.connection();
        dr_catalog::rating::set_rating(conn, ImageId(3), 2).unwrap();

        let taken = take_in(conn, 1, "Photos/IMG_0002.xmp", LIGHTROOM, 1).unwrap();
        assert_eq!(taken.conflicts, [dr_xmp::Field::Rating]);
        assert_eq!(
            record_of(conn, ImageId(3), 3).rating,
            Some(Rating::Stars(2)),
            "the catalog's own rating stands"
        );
        assert_eq!(
            record_of(conn, ImageId(3), 3).keywords,
            ["iceland", "puffin"],
            "while the keywords still union"
        );
        let open = conflicts(&c, 1);
        assert_eq!(open.len(), 1);
        assert_eq!(open[0].path, "Photos/IMG_0002.xmp");
        assert_eq!(open[0].fields, "Rating");

        reload(conn, 1, "Photos/IMG_0002.xmp", LIGHTROOM).unwrap();
        assert_eq!(
            record_of(conn, ImageId(3), 3).rating,
            Some(Rating::Stars(4))
        );
        assert!(conflicts(&c, 1).is_empty(), "settled");
    }

    /// A rejection and a star count are two axes here and one field there.
    #[test]
    fn a_rejection_crosses_as_adobes_minus_one_and_back() {
        let c = catalog();
        let conn = c.connection();
        dr_catalog::rating::set_flag(conn, ImageId(3), FlagState::Reject).unwrap();
        assert_eq!(
            record_of(conn, ImageId(3), 3).rating,
            Some(Rating::Rejected)
        );

        let back = Xmp {
            rating: Some(Rating::Stars(3)),
            ..Default::default()
        };
        apply(conn, ImageId(3), 3, &back).unwrap();
        let after = record_of(conn, ImageId(3), 3);
        assert_eq!(
            after.rating,
            Some(Rating::Stars(3)),
            "the stars lift the rejection"
        );
    }

    /// A judgement written over a Lightroom sidecar must not cost the caption
    /// Lightroom wrote, since the catalog never held it.
    #[test]
    fn a_rewrite_keeps_what_the_catalog_has_no_column_for() {
        let theirs = Xmp {
            title: Some("Puffins at dusk".into()),
            copyright: Some("© Someone".into()),
            hierarchical_subjects: vec!["Places|Iceland".into()],
            label: Some("Second choice".into()),
            rating: Some(Rating::Stars(1)),
            ..Default::default()
        };

        let mut ours = Xmp {
            rating: Some(Rating::Stars(4)),
            keywords: vec!["puffin".into()],
            ..Default::default()
        };
        carry_through(&mut ours, &theirs);

        assert_eq!(ours.rating, Some(Rating::Stars(4)), "the judgement is ours");
        assert_eq!(ours.title.as_deref(), Some("Puffins at dusk"));
        assert_eq!(ours.copyright.as_deref(), Some("© Someone"));
        assert_eq!(ours.hierarchical_subjects, ["Places|Iceland"]);
        assert_eq!(
            ours.label.as_deref(),
            Some("Second choice"),
            "a label that is not one of the five colours has no other copy"
        );

        let mut red = Xmp::default();
        red.set_colour(Some(dr_types::ColourLabel::Red));
        carry_through(&mut red, &theirs);
        assert_eq!(
            red.label.as_deref(),
            Some("Red"),
            "but a colour of ours wins"
        );
    }

    /// The two spellings a write chooses between.
    #[test]
    fn the_write_tries_darktables_spelling_then_lightrooms() {
        assert_eq!(
            candidate_paths("Photos/IMG_0001.CR3"),
            ["Photos/IMG_0001.CR3.xmp", "Photos/IMG_0001.xmp"]
        );
        assert_eq!(
            candidate_paths("a.b/IMG"),
            ["a.b/IMG.xmp", "a.b/IMG.xmp"],
            "a dot in a folder is not an extension"
        );
    }
}