diff --git a/core/dr-catalog/src/jobs.rs b/core/dr-catalog/src/jobs.rs index 2bede77..5b7f3cb 100644 --- a/core/dr-catalog/src/jobs.rs +++ b/core/dr-catalog/src/jobs.rs @@ -185,7 +185,8 @@ pub fn enqueue( priority: Priority, payload: Option<&str>, ) -> Result<(), CatalogError> { - conn.execute( + // Cached: a scan enqueues one per photograph it lists. + conn.prepare_cached( "INSERT INTO jobs(kind, subject_id, priority, state, payload) VALUES (?1, ?2, ?3, 0, ?4) ON CONFLICT(kind, subject_id) DO UPDATE SET @@ -195,8 +196,13 @@ pub fn enqueue( state = CASE WHEN jobs.state = 2 THEN 0 ELSE jobs.state END, attempts = CASE WHEN jobs.state = 2 THEN 0 ELSE jobs.attempts END, not_before = CASE WHEN jobs.state = 2 THEN 0 ELSE jobs.not_before END", - rusqlite::params![kind as i64, subject_id, priority as i64, payload], - )?; + )? + .execute(rusqlite::params![ + kind as i64, + subject_id, + priority as i64, + payload + ])?; Ok(()) } diff --git a/ui/dr-ui/src/library/scan.rs b/ui/dr-ui/src/library/scan.rs index 7f52514..55b1855 100644 --- a/ui/dr-ui/src/library/scan.rs +++ b/ui/dr-ui/src/library/scan.rs @@ -603,103 +603,120 @@ pub(super) fn persist( // Folder ETags first โ€” without these persisted, the next scan prunes // nothing and walks the whole tree again (ARCH ยง6.6). - for (path, validator) in &result.directories { - tx.execute( + { + let mut folder = tx.prepare_cached( "INSERT INTO folders(root_id, path, etag) VALUES (?1, ?2, ?3) ON CONFLICT(root_id, path) DO UPDATE SET etag = excluded.etag", - rusqlite::params![root_id, path.as_str(), validator.as_str()], )?; + for (path, validator) in &result.directories { + folder.execute(rusqlite::params![ + root_id, + path.as_str(), + validator.as_str() + ])?; + } } + // Every statement below runs once per photograph listed -- 1,600 for one + // folder whose sidecar changed, 24,000 for a first scan -- so each is + // prepared once, and a folder's id is looked up once per folder rather + // than once per photograph in it. + let mut folder_of = + tx.prepare_cached("SELECT id FROM folders WHERE root_id = ?1 AND path = ?2")?; + let mut folder_ids: std::collections::HashMap> = + std::collections::HashMap::new(); + // TRACES: FR-PLAT-AND-2 | FR-CAT-9 + // The `availability` arm is what ends an offline library, and it does + // it one photograph at a time. 3 is `Availability::Offline` and 0 is + // `MetadataOnly`, the same code this statement inserts new rows with โ€” + // so a row that was marked offline when the root became unreachable is + // returned to exactly the state a fresh scan would have given it, and + // a row that was never marked is not touched at all. + // + // Conditional rather than a blanket reset for the same reason + // `dr_catalog::walk` restores per file rather than per root: the only + // thing that may clear "I could not reach this" is having reached it, + // and this statement runs precisely once per file the scan listed. + let mut upsert = tx.prepare_cached( + "INSERT INTO images(root_id, folder_id, source_ref, format, file_size, + availability, metadata_state, added_at) + VALUES (?1, ?2, ?3, ?4, ?5, 0, 1, ?6) + ON CONFLICT(root_id, source_ref) DO UPDATE SET + file_size = excluded.file_size, + folder_id = excluded.folder_id, + availability = CASE WHEN images.availability = 3 + THEN 0 ELSE images.availability END", + )?; + let mut image_of = + tx.prepare_cached("SELECT id FROM images WHERE root_id = ?1 AND source_ref = ?2")?; + let mut remote = tx.prepare_cached( + "INSERT INTO remote(image_id, file_id, etag, remote_path) + VALUES (?1, ?2, ?3, ?4) + ON CONFLICT(image_id) DO UPDATE SET + etag = excluded.etag, remote_path = excluded.remote_path", + )?; + for entry in &result.images { - let folder_id: Option = entry.path.parent().and_then(|p| { - tx.query_row( - "SELECT id FROM folders WHERE root_id = ?1 AND path = ?2", - rusqlite::params![root_id, p.as_str()], - |r| r.get(0), - ) - .ok() - }); + let folder_id: Option = match entry.path.parent() { + None => None, + Some(p) => match folder_ids.get(p.as_str()) { + Some(id) => *id, + None => { + let id = folder_of + .query_row(rusqlite::params![root_id, p.as_str()], |r| r.get(0)) + .ok(); + folder_ids.insert(p.as_str().to_string(), id); + id + } + }, + }; - // TRACES: FR-PLAT-AND-2 | FR-CAT-9 - // The `availability` arm is what ends an offline library, and it does - // it one photograph at a time. 3 is `Availability::Offline` and 0 is - // `MetadataOnly`, the same code this statement inserts new rows with โ€” - // so a row that was marked offline when the root became unreachable is - // returned to exactly the state a fresh scan would have given it, and - // a row that was never marked is not touched at all. - // - // Conditional rather than a blanket reset for the same reason - // `dr_catalog::walk` restores per file rather than per root: the only - // thing that may clear "I could not reach this" is having reached it, - // and this statement runs precisely once per file the scan listed. - tx.execute( - "INSERT INTO images(root_id, folder_id, source_ref, format, file_size, - availability, metadata_state, added_at) - VALUES (?1, ?2, ?3, ?4, ?5, 0, 1, ?6) - ON CONFLICT(root_id, source_ref) DO UPDATE SET - file_size = excluded.file_size, - folder_id = excluded.folder_id, - availability = CASE WHEN images.availability = 3 - THEN 0 ELSE images.availability END", - rusqlite::params![ - root_id, - folder_id, - entry.path.as_str(), - entry - .path - .name() - .rsplit_once('.') - .map(|(_, e)| e.to_ascii_lowercase()), - entry.size as i64, - now_secs(), - ], - )?; + upsert.execute(rusqlite::params![ + root_id, + folder_id, + entry.path.as_str(), + entry + .path + .name() + .rsplit_once('.') + .map(|(_, e)| e.to_ascii_lowercase()), + entry.size as i64, + now_secs(), + ])?; - let image_id: i64 = tx.query_row( - "SELECT id FROM images WHERE root_id = ?1 AND source_ref = ?2", - rusqlite::params![root_id, entry.path.as_str()], - |r| r.get(0), - )?; + let image_id: i64 = image_of + .query_row(rusqlite::params![root_id, entry.path.as_str()], |r| { + r.get(0) + })?; // Remote identity, keyed on oc:fileid so a server-side move is a move // rather than a re-download (FR-NC-5). if let RemoteId::Stable(file_id) = entry.id { - tx.execute( - "INSERT INTO remote(image_id, file_id, etag, remote_path) - VALUES (?1, ?2, ?3, ?4) - ON CONFLICT(image_id) DO UPDATE SET - etag = excluded.etag, remote_path = excluded.remote_path", - rusqlite::params![ - image_id, - file_id as i64, - entry.validator.as_str(), - entry.path.as_str() - ], - )?; + remote.execute(rusqlite::params![ + image_id, + file_id as i64, + entry.validator.as_str(), + entry.path.as_str() + ])?; } + + // Its thumbnail job, in the same transaction as the row it points at, + // so a failure mid-insert cannot leave one pointing at a row that + // never landed. It used to follow the commit, one autocommitting + // statement per photograph, each finding its image again by path: a + // relisted folder of 1,600 paid 1,600 commits for rows that are + // almost all already queued. + let _ = dr_catalog::jobs::enqueue( + &tx, + JobKind::Thumbnail, + Some(image_id), + Priority::Background, + None, + ); } + drop((folder_of, upsert, image_of, remote)); tx.commit()?; - - // Thumbnail jobs after the commit, so a failure mid-insert does not leave - // jobs pointing at rows that never landed. - for entry in &result.images { - if let Ok(image_id) = conn.query_row( - "SELECT id FROM images WHERE root_id = ?1 AND source_ref = ?2", - rusqlite::params![root_id, entry.path.as_str()], - |r| r.get::<_, i64>(0), - ) { - let _ = dr_catalog::jobs::enqueue( - conn, - JobKind::Thumbnail, - Some(image_id), - Priority::Background, - None, - ); - } - } - Ok(()) }