Store seven eye numbers per face rather than three

Per eye P(open), the pixels across its box and the sharpness of the
patch; and P(sunglasses). The verdict — open, closed, sunglasses,
unclear — stays a rule in dr_face::eyes so the floors can move without
re-measuring twenty thousand faces. Shards carry the same seven, and a
peer's shard from before any of them is still read.

core/dr-catalog/src/face_shard.rs

@@ -254,9 +254,10 @@ impl FaceShardStore {
                 "INSERT INTO faces
                      (file_id, model_id, x, y, w, h, landmarks, confidence,
                       embedding, crop_px, crop, quality,
-                      eye_right, eye_left, sunglasses)
+                      eye_right, eye_right_px, eye_right_sharp,
+                      eye_left, eye_left_px, eye_left_sharp, sunglasses)
                  VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, ?11, ?12,
-                         ?13, ?14, ?15)",
+                         ?13, ?14, ?15, ?16, ?17, ?18, ?19)",
                 rusqlite::params![
                     f.file_id as i64,
                     f.model_id,
@@ -270,8 +271,12 @@ impl FaceShardStore {
                     f.crop_px as f64,
                     (!f.crop.is_empty()).then_some(f.crop.as_slice()),
                     f.quality.map(f64::from),
-                    f.eyes.map(|e| f64::from(e.right_open)),
+                    f.eyes.map(|e| f64::from(e.right.open)),
-                    f.eyes.map(|e| f64::from(e.left_open)),
+                    f.eyes.map(|e| f64::from(e.right.px)),
+                    f.eyes.map(|e| f64::from(e.right.sharpness)),
+                    f.eyes.map(|e| f64::from(e.left.open)),
+                    f.eyes.map(|e| f64::from(e.left.px)),
+                    f.eyes.map(|e| f64::from(e.left.sharpness)),
                     f.eyes.map(|e| f64::from(e.sunglasses)),
                 ],
             )?;
@@ -477,13 +482,15 @@ impl FaceShardStore {
             }
             let mut fq = src.prepare(&format!(
                 "SELECT f.file_id, f.model_id, f.x, f.y, f.w, f.h, f.landmarks,
-                        f.confidence, f.embedding, f.crop_px, {}, {}, {}, {}, {}
+                        f.confidence, f.embedding, f.crop_px, {}, {}, {}
                    FROM faces f WHERE f.file_id = ?1 AND f.model_id = ?2",
                 column_or_null(&src, "crop"),
                 column_or_null(&src, "quality"),
-                column_or_null(&src, "eye_right"),
+                crate::schema::EYE_COLUMNS
-                column_or_null(&src, "eye_left"),
+                    .iter()
-                column_or_null(&src, "sunglasses"),
+                    .map(|c| column_or_null(&src, c))
+                    .collect::<Vec<_>>()
+                    .join(", "),
             ))?;
             let faces: Vec<SharedFace> = fq
                 .query_map(rusqlite::params![file_id, &model_id], read_shared_face)?
@@ -524,7 +531,8 @@ impl FaceShardStore {
 
         let mut q = conn.prepare(
             "SELECT file_id, model_id, x, y, w, h, landmarks, confidence, embedding, crop_px,
-                    crop, quality, eye_right, eye_left, sunglasses
+                    crop, quality, eye_right, eye_right_px, eye_right_sharp,
+                    eye_left, eye_left_px, eye_left_sharp, sunglasses
                FROM faces WHERE file_id = ?1 AND model_id = ?2",
         )?;
         let faces: Vec<SharedFace> = q
@@ -583,7 +591,11 @@ fn upgrade_shard(conn: &Connection) -> Result<(), CatalogError> {
         ("indexed", "indexed_at", "INTEGER"),
         ("faces", "quality", "REAL"),
         ("faces", "eye_right", "REAL"),
+        ("faces", "eye_right_px", "REAL"),
+        ("faces", "eye_right_sharp", "REAL"),
         ("faces", "eye_left", "REAL"),
+        ("faces", "eye_left_px", "REAL"),
+        ("faces", "eye_left_sharp", "REAL"),
         ("faces", "sunglasses", "REAL"),
     ] {
         if !has_column(conn, table, column)? {
@@ -608,7 +620,7 @@ fn has_column(conn: &Connection, table: &str, column: &str) -> Result<bool, Cata
 ///
 /// `column` is one of this module's own names, never anything read from
 /// outside, which is what makes formatting it into SQL acceptable.
-fn column_or_null(conn: &Connection, column: &'static str) -> String {
+fn column_or_null(conn: &Connection, column: &str) -> String {
     match has_column(conn, "faces", column) {
         Ok(true) => format!("f.{column}"),
         _ => "NULL".to_string(),
@@ -691,7 +703,8 @@ pub fn export_to_shards_reporting(
         }
         let mut fq = conn.prepare(
             "SELECT x, y, w, h, landmarks, detector_confidence, embedding, crop_px, crop,
-                    quality, eye_right, eye_left, sunglasses
+                    quality, eye_right, eye_right_px, eye_right_sharp,
+                    eye_left, eye_left_px, eye_left_sharp, sunglasses
                FROM faces WHERE image_id = ?1 AND model_id = ?2",
         )?;
         let faces: Vec<SharedFace> = fq
@@ -961,12 +974,16 @@ CREATE TABLE IF NOT EXISTS faces (
     -- a build that did not keep it, and a face the receiving device will not
     -- adopt -- see `import_from_shards`.
     quality    REAL,
-    -- The eye reading (`faces::DetectedFace::eyes`), all three or none. NULL
+    -- The eye reading (`faces::DetectedFace::eyes`), all seven or none. NULL
     -- from a peer without the eye models; adopted anyway, and read by the
     -- receiving device's own measuring pass if it has them.
-    eye_right  REAL,
+    eye_right       REAL,
-    eye_left   REAL,
+    eye_right_px    REAL,
-    sunglasses REAL
+    eye_right_sharp REAL,
+    eye_left        REAL,
+    eye_left_px     REAL,
+    eye_left_sharp  REAL,
+    sunglasses      REAL
 );
 CREATE INDEX IF NOT EXISTS faces_file ON faces(file_id, model_id);
 

core/dr-catalog/src/faces.rs

@@ -58,7 +58,7 @@ use rusqlite::{Connection, OptionalExtension};
 use dr_types::ImageId;
 
 use crate::error::CatalogError;
-use dr_face::EyeReading;
+use dr_face::{Eye, EyeReading};
 
 /// The embedder half of a model id: what makes two faces comparable.
 ///
@@ -266,9 +266,10 @@ pub fn record_detections(
             "INSERT INTO faces
                  (image_id, x, y, w, h, landmarks, detector_confidence,
                   embedding, crop_px, model_id, detected_at, crop, quality,
-                  eye_right, eye_left, sunglasses)
+                  eye_right, eye_right_px, eye_right_sharp,
+                  eye_left, eye_left_px, eye_left_sharp, sunglasses)
              VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, ?11, ?12, ?13,
-                     ?14, ?15, ?16)",
+                     ?14, ?15, ?16, ?17, ?18, ?19, ?20)",
             rusqlite::params![
                 image_id.0 as i64,
                 f.x as f64,
@@ -286,8 +287,12 @@ pub fn record_detections(
                 // about.
                 (!f.crop.is_empty()).then_some(f.crop.as_slice()),
                 f.quality.map(f64::from),
-                f.eyes.map(|e| f64::from(e.right_open)),
+                f.eyes.map(|e| f64::from(e.right.open)),
-                f.eyes.map(|e| f64::from(e.left_open)),
+                f.eyes.map(|e| f64::from(e.right.px)),
+                f.eyes.map(|e| f64::from(e.right.sharpness)),
+                f.eyes.map(|e| f64::from(e.left.open)),
+                f.eyes.map(|e| f64::from(e.left.px)),
+                f.eyes.map(|e| f64::from(e.left.sharpness)),
                 f.eyes.map(|e| f64::from(e.sunglasses)),
             ],
         )?;
@@ -389,12 +394,20 @@ pub fn record_measurements(
         // a face a peer had already read must not blank the reading.
         if let Some(e) = m.eyes {
             tx.execute(
-                "UPDATE faces SET eye_right = ?2, eye_left = ?3, sunglasses = ?4 WHERE id = ?1",
+                "UPDATE faces
+                    SET eye_right = ?2, eye_right_px = ?3, eye_right_sharp = ?4,
+                        eye_left = ?5, eye_left_px = ?6, eye_left_sharp = ?7,
+                        sunglasses = ?8
+                  WHERE id = ?1",
                 rusqlite::params![
                     m.face.0 as i64,
-                    f64::from(e.right_open),
+                    f64::from(e.right.open),
-                    f64::from(e.left_open),
+                    f64::from(e.right.px),
-                    f64::from(e.sunglasses)
+                    f64::from(e.right.sharpness),
+                    f64::from(e.left.open),
+                    f64::from(e.left.px),
+                    f64::from(e.left.sharpness),
+                    f64::from(e.sunglasses),
                 ],
             )?;
         }
@@ -605,7 +618,8 @@ pub fn for_image(conn: &Connection, image_id: ImageId) -> Result<Vec<Face>, Cata
         "SELECT f.id, f.image_id, f.x, f.y, f.w, f.h, f.landmarks,
                 f.detector_confidence, f.crop_px, f.model_id,
                 fp.person_id, fp.probability, fp.confirmed, f.quality,
-                f.eye_right, f.eye_left, f.sunglasses
+                f.eye_right, f.eye_right_px, f.eye_right_sharp,
+                f.eye_left, f.eye_left_px, f.eye_left_sharp, f.sunglasses
            FROM faces f
            LEFT JOIN face_person fp ON fp.face_id = f.id
           WHERE f.image_id = ?1
@@ -899,7 +913,8 @@ pub fn for_person(
         "SELECT f.id, f.image_id, f.x, f.y, f.w, f.h, f.landmarks,
                 f.detector_confidence, f.crop_px, f.model_id,
                 fp.person_id, fp.probability, fp.confirmed, f.quality,
-                f.eye_right, f.eye_left, f.sunglasses
+                f.eye_right, f.eye_right_px, f.eye_right_sharp,
+                f.eye_left, f.eye_left_px, f.eye_left_sharp, f.sunglasses
            FROM faces f
            JOIN face_person fp ON fp.face_id = f.id
           WHERE fp.person_id = ?1 AND (?2 OR fp.confirmed = 1)
@@ -1167,26 +1182,38 @@ fn read_face(r: &rusqlite::Row<'_>) -> rusqlite::Result<Face> {
     })
 }
 
-/// The three eye columns at `first`, `first + 1`, `first + 2`, as one reading.
+/// The seven eye columns at `first` .. `first + 6`, in [`EYE_COLUMNS`]'s
+/// order, as one reading.
 ///
-/// All three or none: they are written together, and a row with one of them
+/// All seven or none: they are written together, and a row with one of
-/// NULL is a row nothing in this crate produced. Read as absent rather than
+/// them NULL is a row nothing in this crate produced. Read as absent rather
-/// invented, which is what a reader of a half-written row deserves.
+/// than invented, which is what a reader of a half-written row deserves.
+///
+/// [`EYE_COLUMNS`]: crate::schema::EYE_COLUMNS
 pub(crate) fn read_eyes(
     r: &rusqlite::Row<'_>,
     first: usize,
 ) -> rusqlite::Result<Option<EyeReading>> {
-    let right: Option<f64> = r.get(first)?;
+    let mut v = [0.0_f32; 7];
-    let left: Option<f64> = r.get(first + 1)?;
+    for (i, slot) in v.iter_mut().enumerate() {
-    let sunglasses: Option<f64> = r.get(first + 2)?;
+        match r.get::<_, Option<f64>>(first + i)? {
-    Ok(match (right, left, sunglasses) {
+            Some(x) => *slot = x as f32,
-        (Some(right), Some(left), Some(sunglasses)) => Some(EyeReading {
+            None => return Ok(None),
-            right_open: right as f32,
+        }
-            left_open: left as f32,
+    }
-            sunglasses: sunglasses as f32,
+    Ok(Some(EyeReading {
-        }),
+        right: Eye {
-        _ => None,
+            open: v[0],
-    })
+            px: v[1],
+            sharpness: v[2],
+        },
+        left: Eye {
+            open: v[3],
+            px: v[4],
+            sharpness: v[5],
+        },
+        sunglasses: v[6],
+    }))
 }
 
 /// The predicate "this face still needs measuring", over `faces` aliased as
@@ -1322,8 +1349,16 @@ mod tests {
             crop_px: 180.0,
             quality: Some(10.0 + f32::from(seed)),
             eyes: Some(EyeReading {
-                right_open: 0.9,
+                right: Eye {
-                left_open: 0.8,
+                    open: 0.9,
+                    px: 40.0,
+                    sharpness: 0.2,
+                },
+                left: Eye {
+                    open: 0.8,
+                    px: 38.0,
+                    sharpness: 0.3,
+                },
                 sunglasses: 0.1,
             }),
             model_id: "w600k_mbf".into(),
@@ -1361,14 +1396,15 @@ mod tests {
         assert_eq!(stored[0].quality, Some(11.0), "oldest first");
         assert_eq!(stored[1].quality, Some(12.0));
         // And the eyes, as one reading.
-        assert_eq!(got[0].eyes.map(|e| e.left_open), Some(0.8));
+        assert_eq!(got[0].eyes.map(|e| e.left.open), Some(0.8));
+        assert_eq!(got[0].eyes.map(|e| e.right.sharpness), Some(0.2));
         assert_eq!(
             got[0].eyes.map(|e| e.state()),
             Some(dr_face::EyeState::Open)
         );
     }
 
-    /// The three eye columns are one fact: a face with none of them reads as
+    /// The seven eye columns are one fact: a face with none of them reads as
     /// unread, and the measuring pass is what fills them.
     #[test]
     fn eyes_are_measured_only_where_the_device_can_read_them() {
@@ -1423,8 +1459,16 @@ mod tests {
                 embedding: vec![9; 1024],
                 quality: 21.5,
                 eyes: Some(EyeReading {
-                    right_open: 0.2,
+                    right: Eye {
-                    left_open: 0.9,
+                        open: 0.2,
+                        px: 40.0,
+                        sharpness: 0.2,
+                    },
+                    left: Eye {
+                        open: 0.9,
+                        px: 40.0,
+                        sharpness: 0.2,
+                    },
                     sunglasses: 0.0,
                 }),
             }],

core/dr-catalog/src/schema.rs

@@ -163,11 +163,19 @@ pub fn migrate(conn: &Connection) -> Result<i64, CatalogError> {
     Ok(from)
 }
 
-/// The three columns V16 adds to `faces`, in the order the readers name them.
+/// The seven columns V16 adds to `faces`, in the order the readers name them.
 ///
 /// Named once because three places have to agree on them: this migration,
 /// [`for_attached`], and the face shard's own catch-up (`face_shard`).
-pub const EYE_COLUMNS: [&str; 3] = ["eye_right", "eye_left", "sunglasses"];
+pub const EYE_COLUMNS: [&str; 7] = [
+    "eye_right",
+    "eye_right_px",
+    "eye_right_sharp",
+    "eye_left",
+    "eye_left_px",
+    "eye_left_sharp",
+    "sunglasses",
+];
 
 /// Recompute columns a migration added, for rows that predate it.
 ///
@@ -328,7 +336,11 @@ pub fn for_attached(schema_name: &str) -> String {
          ALTER TABLE {schema_name}.faces ADD COLUMN crop BLOB;\n\
          ALTER TABLE {schema_name}.faces ADD COLUMN quality REAL;\n\
          ALTER TABLE {schema_name}.faces ADD COLUMN eye_right REAL;\n\
+         ALTER TABLE {schema_name}.faces ADD COLUMN eye_right_px REAL;\n\
+         ALTER TABLE {schema_name}.faces ADD COLUMN eye_right_sharp REAL;\n\
          ALTER TABLE {schema_name}.faces ADD COLUMN eye_left REAL;\n\
+         ALTER TABLE {schema_name}.faces ADD COLUMN eye_left_px REAL;\n\
+         ALTER TABLE {schema_name}.faces ADD COLUMN eye_left_sharp REAL;\n\
          ALTER TABLE {schema_name}.faces ADD COLUMN sunglasses REAL;",
         rewrite_for_attached(V1, schema_name),
         rewrite_for_attached(V6, schema_name),
@@ -734,16 +746,19 @@ CREATE TABLE IF NOT EXISTS xmp_conflicts (
 
 // V16 -- TRACES: FR-CULL-13
 //
-// What each face's eyes are doing: P(open) for the subject's right and left
+// What each face's eyes are doing: for each eye P(open), the source pixels
-// eye, and P(sunglasses) for the head, each a sigmoid from a classifier run
+// across its box and the sharpness of the patch the classifier saw; and
-// over the aligned crop at indexing time (dr_face::classify). Three numbers
+// P(sunglasses) for the head. Seven numbers rather than a verdict, because
-// rather than a verdict, because the verdict is a rule with thresholds in it
+// the verdict is a rule with thresholds in it (dr_face::eyes::EyeReading::
-// (dr_face::eyes::EyeReading::state) and a rule belongs in code that can be
+// state) and a rule belongs in code that can be changed, not in rows that
-// changed, not in rows that would have to be re-measured.
+// would have to be re-measured.
 //
-// Sunglasses are a column of their own and not folded into the eye numbers,
+// The pixels and the sharpness are what stop a smear reading as a blink: an
-// because the eye classifier answers confidently over dark glass and its
+// eye too small or too soft to read is not asked, and a face with no
-// answer means nothing there. A filter for "eyes open" reads all three.
+// readable eye is "unclear", which no filter drops. Sunglasses are a column
+// of their own for the same kind of reason — the eye classifier answers
+// confidently over dark glass, and its answer means nothing there. A filter
+// for "eyes open" reads all seven.
 //
 // NULL means "never measured" -- a face indexed before this version, or on a
 // device without the eye models -- and a NULL is left alone by every filter