dtourolle/DarkRoom
1
0
Fork 0
Code Issues 39 Pull Requests Actions Packages Projects Releases 20 Wiki Activity

Pair RAW and JPEG from the unpaired JPEGs, not from every RAW

Browse Source 
`Catalog::open` runs the backfill every time, and every worker thread
opens its own catalog: the develop view does it to fetch each original and
again for each neighbour it prefetches, and the sync, sweep, burst and
thumbnail workers each do it too. On the reference library (24k images)
an open cost 26 ms of CPU, and most of it was `pair_raw_and_jpeg` reading
all 17,000 RAWs into a map of lowercased stems to find partners for the
1,900 JPEGs that have none -- the same 1,900 on every open.

It now starts from the small side. The unpaired JPEGs are read first, and
it stops there if there are none; otherwise it reads the RAWs in the
folders those JPEGs sit in (plus the unfiled ones when an unfiled JPEG is
waiting), which is 142 on the reference library. A pair is same-folder by
definition, so no pairing is lost; the RAWs are read in id order, so where
two share a stem the later one still wins as it did in the table scan; and
a pass with nothing to pair no longer opens and commits an empty write
transaction.

catalog_bench, best of 20, CPU: `Catalog::open` 26 ms -> 12 ms together
with the next commit (the backfill 24 ms -> 11 ms; this step is ~10 ms of
that). A test covers pairs found among other folders and unfiled images.
This commit is contained in:
dtourolle
2026-09-25 22:06:58 -04:00
parent 2fd1b0b8ce
commit 1f26e1e627
1 changed files with 78 additions and 21 deletions
Download Patch File Download Diff File Expand all files Collapse all files
core/dr-catalog/src/schema.rs
+78 -21
View File
@@ -494,15 +494,51 @@ fn rewrite_for_attached(sql: &str, schema_name: &str) -> String {
fn pair_raw_and_jpeg(conn: &Connection) -> Result<usize, CatalogError> {
use std::collections::HashMap;
// (folder, lowercase stem) -> RAW id, built in one pass over the RAWs.
// The small side first: the JPEGs not yet paired. On a settled library
// these are the ones with no RAW beside them -- 1,900 of 24,000 on the
// reference library -- and this runs on every open, including the ones
// the develop view makes for each photograph it fetches. Reading every
// RAW to find the handful that share a folder with one of them was most
// of what opening the catalog cost.
let jpegs: Vec<(i64, Option<i64>, String)> = {
let mut stmt = conn.prepare(
"SELECT id, folder_id, source_ref FROM images
WHERE lower(format) IN ('jpg','jpeg') AND shadowed_by IS NULL",
)?;
let rows = stmt.query_map([], |r| {
Ok((
r.get::<_, i64>(0)?,
r.get::<_, Option<i64>>(1)?,
r.get::<_, String>(2)?,
))
})?;
rows.filter_map(Result::ok).collect()
};
if jpegs.is_empty() {
return Ok(0);
}
// (folder, lowercase stem) -> RAW id, over the folders those JPEGs are in
// and no others: a pair is same-folder by definition. Ordered by id so
// that where two RAWs share a stem the later one wins, as it did when this
// read every RAW in table order.
let mut folders: Vec<i64> = jpegs.iter().filter_map(|(_, f, _)| *f).collect();
folders.sort_unstable();
folders.dedup();
let unfiled = jpegs.iter().any(|(_, f, _)| f.is_none());
let folders = serde_json::to_string(&folders).unwrap_or_else(|_| "[]".to_string());
let mut raws: HashMap<(Option<i64>, String), i64> = HashMap::new();
{
let mut stmt = conn.prepare(
"SELECT id, folder_id, source_ref FROM images
WHERE lower(format) IN
('cr2','cr3','nef','arw','raf','rw2','orf','dng')",
('cr2','cr3','nef','arw','raf','rw2','orf','dng')
AND (folder_id IN (SELECT value FROM json_each(?1))
OR (?2 AND folder_id IS NULL))
ORDER BY id",
)?;
let rows = stmt.query_map([], |r| {
let rows = stmt.query_map(rusqlite::params![folders, unfiled], |r| {
Ok((
r.get::<_, i64>(0)?,
r.get::<_, Option<i64>>(1)?,
@@ -518,25 +554,16 @@ fn pair_raw_and_jpeg(conn: &Connection) -> Result<usize, CatalogError> {
return Ok(0);
}
let pairs: Vec<(i64, i64)> = {
let mut stmt = conn.prepare(
"SELECT id, folder_id, source_ref FROM images
WHERE lower(format) IN ('jpg','jpeg') AND shadowed_by IS NULL",
)?;
let rows = stmt.query_map([], |r| {
Ok((
r.get::<_, i64>(0)?,
r.get::<_, Option<i64>>(1)?,
r.get::<_, String>(2)?,
))
})?;
rows.filter_map(|row| {
let (id, folder, path) = row.ok()?;
let raw = raws.get(&(folder, stem_of(&path).to_ascii_lowercase()))?;
Some((id, *raw))
let pairs: Vec<(i64, i64)> = jpegs
.iter()
.filter_map(|(id, folder, path)| {
let raw = raws.get(&(*folder, stem_of(path).to_ascii_lowercase()))?;
Some((*id, *raw))
})
.collect()
};
.collect();
if pairs.is_empty() {
return Ok(0);
}
let tx = conn.unchecked_transaction()?;
for (jpeg, raw) in &pairs {
@@ -1582,6 +1609,36 @@ mod tests {
assert_eq!(backfilled(&c, "shadowed_by"), 0);
}
#[test]
fn a_pair_is_found_among_other_folders_and_unfiled_images() {
// The RAWs are read only from the folders an unpaired JPEG is in,
// plus the unfiled ones when an unfiled JPEG is waiting: each JPEG
// must still find its own sibling, and only its own.
let c = with_root();
let raw_a = image(&c, Some(1), "a/IMG_7.CR2", "cr2");
image(&c, Some(2), "b/IMG_7.CR2", "cr2");
image(&c, Some(2), "b/IMG_8.CR2", "cr2");
let raw_unfiled = image(&c, None, "IMG_9.DNG", "dng");
let jpeg_a = image(&c, Some(1), "a/IMG_7.JPG", "jpg");
let jpeg_unfiled = image(&c, None, "IMG_9.jpg", "jpg");
image(&c, Some(1), "a/IMG_9.jpg", "jpg");
assert_eq!(backfilled(&c, "shadowed_by"), 2);
let of = |id: i64| -> Option<i64> {
c.query_row("SELECT shadowed_by FROM images WHERE id = ?1", [id], |r| {
r.get(0)
})
.unwrap()
};
assert_eq!(of(jpeg_a), Some(raw_a));
assert_eq!(of(jpeg_unfiled), Some(raw_unfiled));
assert_eq!(
backfilled(&c, "shadowed_by"),
0,
"settled on the second pass"
);
}
#[test]
fn a_raw_is_never_shadowed_by_a_jpeg() {
// The relationship is one-way: the RAW is the photograph.