//! Naming a segmented person from the face inside it. //! //! `dr-segment` recognises *a person*; this crate recognises *which* person. //! Putting the two together costs one containment test and turns "person" in //! the mask list into "Anna" — which is the difference between a vocabulary of //! eighty COCO classes and a vocabulary that includes the user's family. //! //! Pure geometry: no model, no catalog, no Slint. The caller supplies boxes and //! names from wherever it keeps them. //! //! # Why containment and not overlap //! //! A face is a small part of the person it belongs to, and it is *inside* them. //! Intersection-over-union would be near zero for a correct match — the face is //! perhaps a twentieth of the person's area — so IoU is the wrong measure //! entirely here and would reject every true pairing. /// A face box with a name attached. #[derive(Debug, Clone, PartialEq)] pub struct NamedFace<'a> { /// `(x0, y0, x1, y1)`, in the same space as the instance boxes. pub bbox: (f32, f32, f32, f32), pub name: &'a str, } /// How much of a face must lie inside an instance to belong to it. /// /// Not 1.0: the detector's face box and the segmenter's person box come from /// different models and disagree at the edges, most visibly around hair and /// chin. A face 85% inside a person is that person's face. const MIN_CONTAINMENT: f32 = 0.7; /// The name to show for one segmented instance, if a face identifies it. /// /// `None` leaves the instance labelled as the model found it. That is the right /// default in every uncertain case: a mask list saying "person" is merely /// unhelpful, where one saying "Anna" about her brother is wrong, and the user /// has no way to tell which they are looking at. /// /// Where several named faces sit inside one instance — two people the segmenter /// merged into one blob — the largest face wins, on the grounds that it is the /// nearer subject and the one the box is mostly about. If two are within a /// whisker of each other the instance stays unnamed, because at that point the /// box genuinely covers two people and picking either is a coin toss. pub fn name_for_instance<'a>( instance: (f32, f32, f32, f32), faces: &[NamedFace<'a>], ) -> Option<&'a str> { let instance_area = area(instance); if instance_area <= 0.0 { return None; } let mut candidates: Vec<(f32, &'a str)> = faces .iter() .filter_map(|f| { let fa = area(f.bbox); if fa <= 0.0 { return None; } let inside = intersection(instance, f.bbox); if inside / fa < MIN_CONTAINMENT { return None; } Some((fa, f.name)) }) .collect(); if candidates.is_empty() { return None; } candidates.sort_by(|a, b| b.0.total_cmp(&a.0)); // Two comparably sized faces in one box: the segmenter has merged two // people and there is no honest way to pick. Distinct names only — the // same person detected twice (a mirror, a reflection) is not ambiguous. if let [(first, a), (second, b), ..] = candidates.as_slice() { if a != b && *second > *first * 0.8 { return None; } } Some(candidates[0].1) } /// Relabel a list of instances, in place, from the faces found in the image. /// /// `is_person` decides which classes are eligible. Only person-like classes /// should be: a face inside a `tv` or a `laptop` is a photograph of someone on /// a screen, and renaming the television to "Anna" would be worse than leaving /// it alone. /// /// Returns how many instances gained a name. pub fn name_instances( instances: &mut [T], faces: &[NamedFace<'_>], bbox_of: impl Fn(&T) -> (f32, f32, f32, f32), is_person: impl Fn(&T) -> bool, set_name: impl Fn(&mut T, &str), ) -> usize { let mut named = 0; for inst in instances.iter_mut() { if !is_person(inst) { continue; } if let Some(name) = name_for_instance(bbox_of(inst), faces) { let name = name.to_string(); set_name(inst, &name); named += 1; } } named } fn area(b: (f32, f32, f32, f32)) -> f32 { ((b.2 - b.0).max(0.0)) * ((b.3 - b.1).max(0.0)) } fn intersection(a: (f32, f32, f32, f32), b: (f32, f32, f32, f32)) -> f32 { let w = (a.2.min(b.2) - a.0.max(b.0)).max(0.0); let h = (a.3.min(b.3) - a.1.max(b.1)).max(0.0); w * h } #[cfg(test)] mod tests { use super::*; /// A person filling most of a portrait, with their face near the top. const PERSON: (f32, f32, f32, f32) = (100.0, 50.0, 400.0, 900.0); const FACE: (f32, f32, f32, f32) = (200.0, 80.0, 300.0, 220.0); fn named(bbox: (f32, f32, f32, f32), name: &str) -> NamedFace<'_> { NamedFace { bbox, name } } #[test] fn a_face_inside_a_person_names_them() { let faces = [named(FACE, "Anna")]; assert_eq!(name_for_instance(PERSON, &faces), Some("Anna")); } #[test] fn a_face_elsewhere_in_the_frame_names_nothing() { let faces = [named((800.0, 80.0, 900.0, 220.0), "Anna")]; assert_eq!(name_for_instance(PERSON, &faces), None); } /// The measure has to be containment. A correct pairing has an IoU near /// zero, so anything IoU-based would reject every true match. #[test] fn a_tiny_face_in_a_large_person_still_matches() { let tall = (0.0, 0.0, 500.0, 2000.0); let small = (240.0, 40.0, 280.0, 100.0); assert_eq!( name_for_instance(tall, &[named(small, "Anna")]), Some("Anna") ); } #[test] fn a_face_mostly_outside_the_person_is_not_theirs() { // Overlapping the person's edge, but only just. let straddling = (60.0, 80.0, 140.0, 220.0); assert_eq!( name_for_instance(PERSON, &[named(straddling, "Anna")]), None ); } /// A tight portrait, where the segmenter's "person" is head and shoulders /// and the face is most of it. /// /// This **is** named, and an earlier version of this module wrongly /// refused to on the grounds that a face filling its instance meant the /// two models disagreed. It does not: it means the photograph is a /// close-up, which is the case where naming the region is most useful and /// most certain. Left as a test because the reasoning is easy to get /// backwards a second time. #[test] fn a_tight_portrait_is_named_rather_than_treated_as_suspicious() { let head = (100.0, 50.0, 400.0, 400.0); let face = (110.0, 60.0, 390.0, 390.0); assert_eq!( name_for_instance(head, &[named(face, "Anna")]), Some("Anna") ); } /// A face box *larger* than the instance is a genuine disagreement, and /// containment rejects it without needing a size rule: most of the face /// lies outside the box it is supposed to belong to. #[test] fn a_face_larger_than_the_instance_does_not_name_it() { let small_instance = (200.0, 200.0, 260.0, 260.0); let huge_face = (100.0, 100.0, 500.0, 500.0); assert_eq!( name_for_instance(small_instance, &[named(huge_face, "Anna")]), None ); } /// Two people merged into one blob: naming either would be a coin toss, /// and a mask list saying "Anna" about her brother is worse than one /// saying "person". #[test] fn two_comparable_faces_in_one_instance_leave_it_unnamed() { let wide = (0.0, 0.0, 800.0, 900.0); let faces = [ named((100.0, 80.0, 200.0, 220.0), "Anna"), named((500.0, 85.0, 605.0, 230.0), "Bob"), ]; assert_eq!(name_for_instance(wide, &faces), None); } /// But a clearly nearer subject wins: the box is mostly about them. #[test] fn a_much_larger_face_wins_over_someone_in_the_background() { let wide = (0.0, 0.0, 800.0, 900.0); let faces = [ named((100.0, 80.0, 300.0, 360.0), "Anna"), named((600.0, 85.0, 640.0, 140.0), "distant"), ]; assert_eq!(name_for_instance(wide, &faces), Some("Anna")); } /// The same person found twice — a mirror, a reflection — is not ambiguous /// even though the two faces are comparable. #[test] fn the_same_name_twice_is_not_an_ambiguity() { let wide = (0.0, 0.0, 800.0, 900.0); let faces = [ named((100.0, 80.0, 200.0, 220.0), "Anna"), named((500.0, 85.0, 605.0, 230.0), "Anna"), ]; assert_eq!(name_for_instance(wide, &faces), Some("Anna")); } #[test] fn degenerate_boxes_name_nothing_rather_than_panicking() { assert_eq!( name_for_instance((0.0, 0.0, 0.0, 0.0), &[named(FACE, "A")]), None ); assert_eq!( name_for_instance(PERSON, &[named((5.0, 5.0, 5.0, 5.0), "A")]), None ); assert_eq!(name_for_instance(PERSON, &[]), None); } #[derive(Debug, PartialEq)] struct Inst { class: String, bbox: (f32, f32, f32, f32), } #[test] fn only_person_instances_are_renamed() { let mut instances = vec![ Inst { class: "person".into(), bbox: PERSON, }, // A face on a screen must not rename the television. Inst { class: "tv".into(), bbox: PERSON, }, ]; let faces = [named(FACE, "Anna")]; let n = name_instances( &mut instances, &faces, |i| i.bbox, |i| i.class == "person", |i, name| i.class = name.to_string(), ); assert_eq!(n, 1); assert_eq!(instances[0].class, "Anna"); assert_eq!(instances[1].class, "tv"); } #[test] fn an_unrecognised_person_keeps_the_models_own_label() { let mut instances = vec![Inst { class: "person".into(), bbox: PERSON, }]; let n = name_instances( &mut instances, &[], |i| i.bbox, |i| i.class == "person", |i, name| i.class = name.to_string(), ); assert_eq!(n, 0); assert_eq!(instances[0].class, "person"); } }