dtourolle/DarkRoom
1
0
Fork 0
Code Issues 39 Pull Requests Actions Packages Projects Releases 20 Wiki Activity

Keep the dense landmarks behind each eye reading, packed

Browse Source 
The 106 points the eye boxes were cut from, stored beside the reading as
16-bit fixed point over the frame: 424 bytes a face, a seventh of a pixel
on a 6000-pixel frame, where f16 at the same size would have been six.
Derived data like the embedding, kept for the same reason — it cost a
fetch and a model run, and the next per-face pass should run from the
catalog. Shards carry it; a peer's shard from before it is still read.
This commit is contained in:
dtourolle
2026-09-19 14:24:15 +02:00
parent 85cc2b1dcc
commit facb44cb55
10 changed files with 264 additions and 99 deletions
Download Patch File Download Diff File Expand all files Collapse all files
core/dr-catalog/src/face_shard.rs
+25 -8
View File
@@ -48,9 +48,10 @@ pub const SHARD_MAX_BYTES: u64 = dr_thumbs::SHARD_MAX_BYTES;
/// Bytes one stored face occupies, near enough to bound a shard by.
///
/// Counted rather than measured: the embedding is fixed at 512 × f16, the
/// landmarks at 5 × 2 × f32, and the rest is a handful of numbers. Measuring
/// landmarks at 5 × 2 × f32 and the dense ones at 106 × 2 × u16, and the
/// rest is a handful of numbers. Measuring
/// the file after each insert would mean a `VACUUM` to get an honest answer.
const BYTES_PER_FACE: u64 = 1024 + 40 + 64;
const BYTES_PER_FACE: u64 = 1024 + 40 + 424 + 64;
/// Bytes a stored crop occupies, near enough to bound a shard by.
///
@@ -82,6 +83,8 @@ pub struct SharedFace {
/// See `faces::DetectedFace::eyes`. `None` from a peer without the eye
/// models, or a shard written before they existed.
pub eyes: Option<dr_face::EyeReading>,
/// See `faces::DetectedFace::landmarks_dense`; empty where none.
pub landmarks_dense: Vec<u8>,
/// The face cut out and encoded, or empty where none was kept.
///
/// Travels with the face rather than in the catalog snapshot, which is the
@@ -255,9 +258,10 @@ impl FaceShardStore {
(file_id, model_id, x, y, w, h, landmarks, confidence,
embedding, crop_px, crop, quality,
eye_right, eye_right_px, eye_right_sharp,
eye_left, eye_left_px, eye_left_sharp, sunglasses)
eye_left, eye_left_px, eye_left_sharp, sunglasses,
landmarks_dense)
VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, ?11, ?12,
?13, ?14, ?15, ?16, ?17, ?18, ?19)",
?13, ?14, ?15, ?16, ?17, ?18, ?19, ?20)",
rusqlite::params![
f.file_id as i64,
f.model_id,
@@ -278,6 +282,7 @@ impl FaceShardStore {
f.eyes.map(|e| f64::from(e.left.px)),
f.eyes.map(|e| f64::from(e.left.sharpness)),
f.eyes.map(|e| f64::from(e.sunglasses)),
(!f.landmarks_dense.is_empty()).then_some(f.landmarks_dense.as_slice()),
],
)?;
}
@@ -482,7 +487,7 @@ impl FaceShardStore {
}
let mut fq = src.prepare(&format!(
"SELECT f.file_id, f.model_id, f.x, f.y, f.w, f.h, f.landmarks,
f.confidence, f.embedding, f.crop_px, {}, {}, {}
f.confidence, f.embedding, f.crop_px, {}, {}, {}, {}
FROM faces f WHERE f.file_id = ?1 AND f.model_id = ?2",
column_or_null(&src, "crop"),
column_or_null(&src, "quality"),
@@ -491,6 +496,7 @@ impl FaceShardStore {
.map(|c| column_or_null(&src, c))
.collect::<Vec<_>>()
.join(", "),
column_or_null(&src, "landmarks_dense"),
))?;
let faces: Vec<SharedFace> = fq
.query_map(rusqlite::params![file_id, &model_id], read_shared_face)?
@@ -532,7 +538,7 @@ impl FaceShardStore {
let mut q = conn.prepare(
"SELECT file_id, model_id, x, y, w, h, landmarks, confidence, embedding, crop_px,
crop, quality, eye_right, eye_right_px, eye_right_sharp,
eye_left, eye_left_px, eye_left_sharp, sunglasses
eye_left, eye_left_px, eye_left_sharp, sunglasses, landmarks_dense
FROM faces WHERE file_id = ?1 AND model_id = ?2",
)?;
let faces: Vec<SharedFace> = q
@@ -597,6 +603,7 @@ fn upgrade_shard(conn: &Connection) -> Result<(), CatalogError> {
("faces", "eye_left_px", "REAL"),
("faces", "eye_left_sharp", "REAL"),
("faces", "sunglasses", "REAL"),
("faces", "landmarks_dense", "BLOB"),
] {
if !has_column(conn, table, column)? {
conn.execute_batch(&format!("ALTER TABLE {table} ADD COLUMN {column} {decl}"))?;
@@ -704,7 +711,7 @@ pub fn export_to_shards_reporting(
let mut fq = conn.prepare(
"SELECT x, y, w, h, landmarks, detector_confidence, embedding, crop_px, crop,
quality, eye_right, eye_right_px, eye_right_sharp,
eye_left, eye_left_px, eye_left_sharp, sunglasses
eye_left, eye_left_px, eye_left_sharp, sunglasses, landmarks_dense
FROM faces WHERE image_id = ?1 AND model_id = ?2",
)?;
let faces: Vec<SharedFace> = fq
@@ -723,6 +730,7 @@ pub fn export_to_shards_reporting(
crop: r.get::<_, Option<Vec<u8>>>(8)?.unwrap_or_default(),
quality: r.get::<_, Option<f64>>(9)?.map(|q| q as f32),
eyes: crate::faces::read_eyes(r, 10)?,
landmarks_dense: r.get::<_, Option<Vec<u8>>>(17)?.unwrap_or_default(),
})
})?
.collect::<Result<_, _>>()?;
@@ -839,6 +847,7 @@ pub fn import_from_shards(
crop_px: f.crop_px,
quality: f.quality,
eyes: f.eyes,
landmarks_dense: f.landmarks_dense,
model_id: f.model_id,
// A peer that indexed before crops existed sends none, and the
// reader falls back to the proxy exactly as it does for a face
@@ -886,6 +895,7 @@ fn read_shared_face(r: &rusqlite::Row<'_>) -> rusqlite::Result<SharedFace> {
crop: r.get::<_, Option<Vec<u8>>>(10)?.unwrap_or_default(),
quality: r.get::<_, Option<f64>>(11)?.map(|q| q as f32),
eyes: crate::faces::read_eyes(r, 12)?,
landmarks_dense: r.get::<_, Option<Vec<u8>>>(19)?.unwrap_or_default(),
})
}
@@ -983,7 +993,10 @@ CREATE TABLE IF NOT EXISTS faces (
eye_left REAL,
eye_left_px REAL,
eye_left_sharp REAL,
sunglasses REAL
sunglasses REAL,
-- The dense landmarks behind the reading (`faces::DetectedFace::
-- landmarks_dense`), 424 bytes packed; NULL where none.
landmarks_dense BLOB
);
CREATE INDEX IF NOT EXISTS faces_file ON faces(file_id, model_id);
@@ -1044,6 +1057,7 @@ mod tests {
crop_px: 180.0,
quality: Some(17.5),
eyes: None,
landmarks_dense: Vec::new(),
crop: vec![seed; 64],
}
}
@@ -1254,6 +1268,7 @@ mod catalog_round_trip {
crop_px: 180.0,
quality: Some(20.0),
eyes: None,
landmarks_dense: Vec::new(),
model_id: "w600k_mbf".into(),
crop: vec![seed; 64],
}
@@ -1370,6 +1385,7 @@ mod catalog_round_trip {
crop_px: 180.0,
quality,
eyes: None,
landmarks_dense: Vec::new(),
crop: Vec::new(),
};
store
@@ -1557,6 +1573,7 @@ mod catalog_round_trip {
crop_px: 180.0,
quality: None,
eyes: None,
landmarks_dense: Vec::new(),
crop: vec![seed; 64],
}
}
core/dr-catalog/src/faces.rs
+33 -5
View File
@@ -127,6 +127,11 @@ pub struct DetectedFace {
/// models were not present at indexing, or the face came from a peer
/// that had none; the measuring pass fills it in (schema V16).
pub eyes: Option<EyeReading>,
/// The 106 dense landmarks the eyes were read from, as
/// `dr_face::Landmarks::to_packed_bytes` — 424 bytes, or empty where
/// the face was never read (schema V18). Kept so a later per-face pass
/// need not fetch the original again.
pub landmarks_dense: Vec<u8>,
/// Which model produced the embedding. Comparing across models is the one
/// mistake that yields plausible garbage rather than an error.
pub model_id: String,
@@ -161,6 +166,8 @@ pub struct Face {
pub quality: Option<f32>,
/// See [`DetectedFace::eyes`]. `None` for a face never read.
pub eyes: Option<EyeReading>,
/// See [`DetectedFace::landmarks_dense`]. Empty for a face never read.
pub landmarks_dense: Vec<u8>,
pub model_id: String,
/// `None` when the face belongs to no one yet.
pub person: Option<PersonId>,
@@ -267,9 +274,9 @@ pub fn record_detections(
(image_id, x, y, w, h, landmarks, detector_confidence,
embedding, crop_px, model_id, detected_at, crop, quality,
eye_right, eye_right_px, eye_right_sharp,
eye_left, eye_left_px, eye_left_sharp, sunglasses)
eye_left, eye_left_px, eye_left_sharp, sunglasses, landmarks_dense)
VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, ?11, ?12, ?13,
?14, ?15, ?16, ?17, ?18, ?19, ?20)",
?14, ?15, ?16, ?17, ?18, ?19, ?20, ?21)",
rusqlite::params![
image_id.0 as i64,
f.x as f64,
@@ -294,6 +301,7 @@ pub fn record_detections(
f.eyes.map(|e| f64::from(e.left.px)),
f.eyes.map(|e| f64::from(e.left.sharpness)),
f.eyes.map(|e| f64::from(e.sunglasses)),
(!f.landmarks_dense.is_empty()).then_some(f.landmarks_dense.as_slice()),
],
)?;
let id = FaceId(tx.last_insert_rowid() as u64);
@@ -354,6 +362,8 @@ pub struct Measurement {
/// See [`DetectedFace::eyes`]. `None` where the measuring device has no
/// eye models, in which case the stored reading is left as it was.
pub eyes: Option<EyeReading>,
/// See [`DetectedFace::landmarks_dense`]; written with `eyes`.
pub landmarks_dense: Vec<u8>,
}
/// Write fresh embeddings over faces that were found before their quality was
@@ -397,7 +407,7 @@ pub fn record_measurements(
"UPDATE faces
SET eye_right = ?2, eye_right_px = ?3, eye_right_sharp = ?4,
eye_left = ?5, eye_left_px = ?6, eye_left_sharp = ?7,
sunglasses = ?8
sunglasses = ?8, landmarks_dense = ?9
WHERE id = ?1",
rusqlite::params![
m.face.0 as i64,
@@ -408,6 +418,7 @@ pub fn record_measurements(
f64::from(e.left.px),
f64::from(e.left.sharpness),
f64::from(e.sunglasses),
(!m.landmarks_dense.is_empty()).then_some(m.landmarks_dense.as_slice()),
],
)?;
}
@@ -619,7 +630,8 @@ pub fn for_image(conn: &Connection, image_id: ImageId) -> Result<Vec<Face>, Cata
f.detector_confidence, f.crop_px, f.model_id,
fp.person_id, fp.probability, fp.confirmed, f.quality,
f.eye_right, f.eye_right_px, f.eye_right_sharp,
f.eye_left, f.eye_left_px, f.eye_left_sharp, f.sunglasses
f.eye_left, f.eye_left_px, f.eye_left_sharp, f.sunglasses,
f.landmarks_dense
FROM faces f
LEFT JOIN face_person fp ON fp.face_id = f.id
WHERE f.image_id = ?1
@@ -914,7 +926,8 @@ pub fn for_person(
f.detector_confidence, f.crop_px, f.model_id,
fp.person_id, fp.probability, fp.confirmed, f.quality,
f.eye_right, f.eye_right_px, f.eye_right_sharp,
f.eye_left, f.eye_left_px, f.eye_left_sharp, f.sunglasses
f.eye_left, f.eye_left_px, f.eye_left_sharp, f.sunglasses,
f.landmarks_dense
FROM faces f
JOIN face_person fp ON fp.face_id = f.id
WHERE fp.person_id = ?1 AND (?2 OR fp.confirmed = 1)
@@ -1175,6 +1188,7 @@ fn read_face(r: &rusqlite::Row<'_>) -> rusqlite::Result<Face> {
crop_px: r.get::<_, f64>(8)? as f32,
quality: r.get::<_, Option<f64>>(13)?.map(|q| q as f32),
eyes: read_eyes(r, 14)?,
landmarks_dense: r.get::<_, Option<Vec<u8>>>(21)?.unwrap_or_default(),
model_id: r.get(9)?,
person: person.map(|p| PersonId(p as u64)),
probability: r.get::<_, Option<f64>>(11)?.unwrap_or(0.0) as f32,
@@ -1361,6 +1375,7 @@ mod tests {
},
sunglasses: 0.1,
}),
landmarks_dense: vec![7; 424],
model_id: "w600k_mbf".into(),
crop: Vec::new(),
}
@@ -1398,6 +1413,11 @@ mod tests {
// And the eyes, as one reading.
assert_eq!(got[0].eyes.map(|e| e.left.open), Some(0.8));
assert_eq!(got[0].eyes.map(|e| e.right.sharpness), Some(0.2));
assert_eq!(
got[0].landmarks_dense.len(),
424,
"the dense landmarks ride along"
);
assert_eq!(
got[0].eyes.map(|e| e.state()),
Some(dr_face::EyeState::Open)
@@ -1442,6 +1462,7 @@ mod tests {
embedding: vec![9; 1024],
quality: 21.5,
eyes: None,
landmarks_dense: Vec::new(),
}],
&[],
)
@@ -1471,11 +1492,17 @@ mod tests {
},
sunglasses: 0.0,
}),
landmarks_dense: vec![9; 424],
}],
&[],
)
.unwrap();
assert_eq!(faces_unmeasured(&c, "w600k_mbf", true).unwrap(), 0);
assert_eq!(
for_image(&c, img).unwrap()[0].landmarks_dense.len(),
424,
"written with the reading"
);
let got = for_image(&c, img).unwrap();
let read = got.iter().find(|f| f.id == ids[0]).unwrap();
assert_eq!(
@@ -1527,6 +1554,7 @@ mod tests {
embedding: vec![9; 1024],
quality: 21.5,
eyes: None,
landmarks_dense: Vec::new(),
}],
&[ids[1]],
)
core/dr-catalog/src/schema.rs
+30 -2
View File
@@ -15,7 +15,7 @@ use rusqlite::Connection;
use crate::error::CatalogError;
/// Schema version this build writes and understands.
pub const SCHEMA_VERSION: i64 = 17;
pub const SCHEMA_VERSION: i64 = 18;
/// Apply migrations up to [`SCHEMA_VERSION`].
///
@@ -167,6 +167,20 @@ pub fn migrate(conn: &Connection) -> Result<i64, CatalogError> {
tx.commit()?;
}
if from < 18 {
let tx = conn.unchecked_transaction()?;
// Guarded like V14's and V16's columns: ALTER has no IF NOT EXISTS
// and the step must be re-enterable (NFR-R5).
let present: bool = tx
.prepare("SELECT 1 FROM pragma_table_info('faces') WHERE name = 'landmarks_dense'")?
.exists([])?;
if !present {
tx.execute_batch("ALTER TABLE faces ADD COLUMN landmarks_dense BLOB;")?;
}
tx.pragma_update(None, "user_version", 18)?;
tx.commit()?;
}
Ok(from)
}
@@ -348,7 +362,8 @@ pub fn for_attached(schema_name: &str) -> String {
ALTER TABLE {schema_name}.faces ADD COLUMN eye_left REAL;\n\
ALTER TABLE {schema_name}.faces ADD COLUMN eye_left_px REAL;\n\
ALTER TABLE {schema_name}.faces ADD COLUMN eye_left_sharp REAL;\n\
ALTER TABLE {schema_name}.faces ADD COLUMN sunglasses REAL;",
ALTER TABLE {schema_name}.faces ADD COLUMN sunglasses REAL;\n\
ALTER TABLE {schema_name}.faces ADD COLUMN landmarks_dense BLOB;",
rewrite_for_attached(V1, schema_name),
rewrite_for_attached(V6, schema_name),
rewrite_for_attached(V8, schema_name),
@@ -751,6 +766,19 @@ CREATE TABLE IF NOT EXISTS xmp_conflicts (
);
"#;
// V18 -- TRACES: FR-CULL-8a | FR-CULL-12
//
// The 106 dense landmarks the eye pass reads its eye boxes from, kept beside
// the reading as `dr_face::Landmarks::to_packed_bytes`: 106 x (x, y) as
// 16-bit fixed point over the frame, 424 bytes a face, a seventh of a
// pixel on a 6000-pixel frame. Derived data under FR-CULL-12 -- rebuilt by
// re-reading, never in a sidecar -- and stored for the same reason the
// embedding is: it cost a fetch of the original and a model run, and the
// next per-face pass (head pose, expression) should not have to pay either
// again. NULL where the face was never read.
//
// Added in `migrate`, guarded, like every ALTER here (NFR-R5).
const V17: &str = r#"
-- TRACES: FR-CULL-8a | FR-CULL-13 | NFR-P9
-- The eyes-open filter's index, and a lesson about where a column lands.