dtourolle/DarkRoom
1
0
Fork 0
Code Issues 39 Pull Requests Actions Packages Projects Releases 20 Wiki Activity

Send a re-indexed image's faces to the other devices

Browse Source 
`export_to_shards` asked `store.contains(file_id)` and skipped anything the
shard store had already heard of. So an image was exported exactly once, and
re-indexing it updated the catalog and nothing else — every other device kept
the first answer for ever.

That is not hypothetical. This library was re-indexed after the detection floors
changed and crops were added, going from 1,807 faces to 15,194; the shard store
still held the original 1,807, written before any of it. Nothing the re-index
produced could reach another device.

The shard's `indexed` table now carries the catalog's own `indexed_at`, and the
export compares against it. A re-indexed image goes again; an unchanged one
still costs nothing. Copied from the catalog rather than stamped when the shard
is written, because a shard-local write time advances even when nothing changed
and could not answer the question.

The column is nullable so a shard written before it still reads: absent means
"cannot vouch for it", which forces one re-export and then settles.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
dtourolle
2026-08-28 09:27:16 +02:00
co-authored by Claude Opus 5
parent b60f9d10d9
commit cf614efa61
2 changed files with 101 additions and 26 deletions
Download Patch File Download Diff File Expand all files Collapse all files
core/dr-catalog/src/face_shard.rs
+86 -11
View File
@@ -113,6 +113,34 @@ impl FaceShardStore {
&self.client
}
/// When this store last recorded an export of `file_id`.
///
/// `None` for an image it does not hold, or one held from before the column
/// existed. Both mean "cannot vouch for it", and the export treats them the
/// same: send it again.
pub fn indexed_at(&self, file_id: u64, model_id: &str) -> Option<i64> {
let shard: i64 = self
.index
.query_row(
"SELECT shard FROM entries WHERE file_id = ?1 AND model_id = ?2",
rusqlite::params![file_id as i64, model_id],
|r| r.get(0),
)
.optional()
.ok()
.flatten()?;
let conn = self.open_shard(shard as u32, false).ok()?;
conn.query_row(
"SELECT indexed_at FROM indexed WHERE file_id = ?1 AND model_id = ?2",
rusqlite::params![file_id as i64, model_id],
|r| r.get::<_, Option<i64>>(0),
)
.optional()
.ok()
.flatten()
.flatten()
}
pub fn contains(&self, file_id: u64, model_id: &str) -> bool {
self.index
.query_row(
@@ -151,6 +179,22 @@ impl FaceShardStore {
model_id: &str,
source_edge: u32,
faces: &[SharedFace],
) -> Result<u32, CatalogError> {
self.put_image_at(file_id, model_id, source_edge, faces, None)
}
/// [`Self::put_image`], carrying the catalog's own index time.
///
/// Separate entry point rather than a sixth positional argument on the
/// common one: every caller that has a timestamp is the export path, and
/// every caller that does not is a test or an adoption.
pub fn put_image_at(
&mut self,
file_id: u64,
model_id: &str,
source_edge: u32,
faces: &[SharedFace],
indexed_at: Option<i64>,
) -> Result<u32, CatalogError> {
let incoming = faces
.iter()
@@ -189,16 +233,18 @@ impl FaceShardStore {
)?;
}
tx.execute(
"INSERT INTO indexed (file_id, model_id, faces_found, source_edge)
VALUES (?1, ?2, ?3, ?4)
"INSERT INTO indexed (file_id, model_id, faces_found, source_edge, indexed_at)
VALUES (?1, ?2, ?3, ?4, ?5)
ON CONFLICT(file_id, model_id) DO UPDATE SET
faces_found = excluded.faces_found,
source_edge = excluded.source_edge",
source_edge = excluded.source_edge,
indexed_at = excluded.indexed_at",
rusqlite::params![
file_id as i64,
model_id,
faces.len() as i64,
source_edge as i64
source_edge as i64,
indexed_at
],
)?;
tx.commit()?;
@@ -404,19 +450,33 @@ pub fn export_to_shards(
model_id: &str,
) -> Result<usize, CatalogError> {
let mut q = conn.prepare(
"SELECT r.file_id, fi.image_id, fi.source_edge
"SELECT r.file_id, fi.image_id, fi.source_edge, fi.indexed_at
FROM face_index fi
JOIN remote r ON r.image_id = fi.image_id
WHERE fi.model_id = ?1
ORDER BY fi.image_id",
)?;
let rows: Vec<(i64, i64, i64)> = q
.query_map([model_id], |r| Ok((r.get(0)?, r.get(1)?, r.get(2)?)))?
let rows: Vec<(i64, i64, i64, i64)> = q
.query_map([model_id], |r| {
Ok((r.get(0)?, r.get(1)?, r.get(2)?, r.get(3)?))
})?
.collect::<Result<_, _>>()?;
let mut exported = 0;
for (file_id, image_id, edge) in rows {
if store.contains(file_id as u64, model_id) {
for (file_id, image_id, edge, indexed_at) in rows {
// **Not `contains`.** Asking only whether the shard has heard of this
// image means it is exported once and never again — so re-indexing it
// updates the catalog and nothing else, and every other device keeps
// the first answer for ever. That is exactly what happened when the
// detection floors changed and crops were added: a library re-indexed
// from 1,800 faces to 15,000 went on syncing the original 1,800.
//
// The comparison is against when the *catalog* indexed it, so a
// re-index is visible and an unchanged image still costs nothing.
if store
.indexed_at(file_id as u64, model_id)
.is_some_and(|was| was >= indexed_at)
{
continue;
}
let mut fq = conn.prepare(
@@ -441,7 +501,13 @@ pub fn export_to_shards(
})?
.collect::<Result<_, _>>()?;
store.put_image(file_id as u64, model_id, edge as u32, &faces)?;
store.put_image_at(
file_id as u64,
model_id,
edge as u32,
&faces,
Some(indexed_at),
)?;
exported += 1;
}
Ok(exported)
@@ -628,8 +694,17 @@ CREATE TABLE IF NOT EXISTS indexed (
model_id TEXT NOT NULL,
faces_found INTEGER NOT NULL,
source_edge INTEGER NOT NULL,
PRIMARY KEY(file_id, model_id)
-- When the *catalog* indexed this image, copied across rather than the
-- time the shard was written. It is what lets a later pass notice that an
-- image has been re-indexed since it was exported; a shard-local write
-- time could not, because it advances even when nothing changed.
--
-- Nullable so a shard written before this column can still be read: an
-- absent value reads as "unknown", which forces one re-export and then
-- settles.
indexed_at INTEGER
);
CREATE UNIQUE INDEX IF NOT EXISTS indexed_key ON indexed(file_id, model_id);
"#;
#[cfg(test)]