DarkRoom/core/dr-face/src/naming.rs

//! Naming a segmented person from the face inside it.
//!
//! `dr-segment` recognises *a person*; this crate recognises *which* person.
//! Putting the two together costs one containment test and turns "person" in
//! the mask list into "Anna" — which is the difference between a vocabulary of
//! eighty COCO classes and a vocabulary that includes the user's family.
//!
//! Pure geometry: no model, no catalog, no Slint. The caller supplies boxes and
//! names from wherever it keeps them.
//!
//! # Why containment and not overlap
//!
//! A face is a small part of the person it belongs to, and it is *inside* them.
//! Intersection-over-union would be near zero for a correct match — the face is
//! perhaps a twentieth of the person's area — so IoU is the wrong measure
//! entirely here and would reject every true pairing.

/// A face box with a name attached.
#[derive(Debug, Clone, PartialEq)]
pub struct NamedFace<'a> {
    /// `(x0, y0, x1, y1)`, in the same space as the instance boxes.
    pub bbox: (f32, f32, f32, f32),
    pub name: &'a str,
}

/// How much of a face must lie inside an instance to belong to it.
///
/// Not 1.0: the detector's face box and the segmenter's person box come from
/// different models and disagree at the edges, most visibly around hair and
/// chin. A face 85% inside a person is that person's face.
const MIN_CONTAINMENT: f32 = 0.7;

/// The name to show for one segmented instance, if a face identifies it.
///
/// `None` leaves the instance labelled as the model found it. That is the right
/// default in every uncertain case: a mask list saying "person" is merely
/// unhelpful, where one saying "Anna" about her brother is wrong, and the user
/// has no way to tell which they are looking at.
///
/// Where several named faces sit inside one instance — two people the segmenter
/// merged into one blob — the largest face wins, on the grounds that it is the
/// nearer subject and the one the box is mostly about. If two are within a
/// whisker of each other the instance stays unnamed, because at that point the
/// box genuinely covers two people and picking either is a coin toss.
pub fn name_for_instance<'a>(
    instance: (f32, f32, f32, f32),
    faces: &[NamedFace<'a>],
) -> Option<&'a str> {
    let instance_area = area(instance);
    if instance_area <= 0.0 {
        return None;
    }

    let mut candidates: Vec<(f32, &'a str)> = faces
        .iter()
        .filter_map(|f| {
            let fa = area(f.bbox);
            if fa <= 0.0 {
                return None;
            }
            let inside = intersection(instance, f.bbox);
            if inside / fa < MIN_CONTAINMENT {
                return None;
            }
            Some((fa, f.name))
        })
        .collect();

    if candidates.is_empty() {
        return None;
    }
    candidates.sort_by(|a, b| b.0.total_cmp(&a.0));

    // Two comparably sized faces in one box: the segmenter has merged two
    // people and there is no honest way to pick. Distinct names only — the
    // same person detected twice (a mirror, a reflection) is not ambiguous.
    if let [(first, a), (second, b), ..] = candidates.as_slice() {
        if a != b && *second > *first * 0.8 {
            return None;
        }
    }

    Some(candidates[0].1)
}

/// Relabel a list of instances, in place, from the faces found in the image.
///
/// `is_person` decides which classes are eligible. Only person-like classes
/// should be: a face inside a `tv` or a `laptop` is a photograph of someone on
/// a screen, and renaming the television to "Anna" would be worse than leaving
/// it alone.
///
/// Returns how many instances gained a name.
pub fn name_instances<T>(
    instances: &mut [T],
    faces: &[NamedFace<'_>],
    bbox_of: impl Fn(&T) -> (f32, f32, f32, f32),
    is_person: impl Fn(&T) -> bool,
    set_name: impl Fn(&mut T, &str),
) -> usize {
    let mut named = 0;
    for inst in instances.iter_mut() {
        if !is_person(inst) {
            continue;
        }
        if let Some(name) = name_for_instance(bbox_of(inst), faces) {
            let name = name.to_string();
            set_name(inst, &name);
            named += 1;
        }
    }
    named
}

fn area(b: (f32, f32, f32, f32)) -> f32 {
    ((b.2 - b.0).max(0.0)) * ((b.3 - b.1).max(0.0))
}

fn intersection(a: (f32, f32, f32, f32), b: (f32, f32, f32, f32)) -> f32 {
    let w = (a.2.min(b.2) - a.0.max(b.0)).max(0.0);
    let h = (a.3.min(b.3) - a.1.max(b.1)).max(0.0);
    w * h
}

#[cfg(test)]
mod tests {
    use super::*;

    /// A person filling most of a portrait, with their face near the top.
    const PERSON: (f32, f32, f32, f32) = (100.0, 50.0, 400.0, 900.0);
    const FACE: (f32, f32, f32, f32) = (200.0, 80.0, 300.0, 220.0);

    fn named(bbox: (f32, f32, f32, f32), name: &str) -> NamedFace<'_> {
        NamedFace { bbox, name }
    }

    #[test]
    fn a_face_inside_a_person_names_them() {
        let faces = [named(FACE, "Anna")];
        assert_eq!(name_for_instance(PERSON, &faces), Some("Anna"));
    }

    #[test]
    fn a_face_elsewhere_in_the_frame_names_nothing() {
        let faces = [named((800.0, 80.0, 900.0, 220.0), "Anna")];
        assert_eq!(name_for_instance(PERSON, &faces), None);
    }

    /// The measure has to be containment. A correct pairing has an IoU near
    /// zero, so anything IoU-based would reject every true match.
    #[test]
    fn a_tiny_face_in_a_large_person_still_matches() {
        let tall = (0.0, 0.0, 500.0, 2000.0);
        let small = (240.0, 40.0, 280.0, 100.0);
        assert_eq!(
            name_for_instance(tall, &[named(small, "Anna")]),
            Some("Anna")
        );
    }

    #[test]
    fn a_face_mostly_outside_the_person_is_not_theirs() {
        // Overlapping the person's edge, but only just.
        let straddling = (60.0, 80.0, 140.0, 220.0);
        assert_eq!(
            name_for_instance(PERSON, &[named(straddling, "Anna")]),
            None
        );
    }

    /// A tight portrait, where the segmenter's "person" is head and shoulders
    /// and the face is most of it.
    ///
    /// This **is** named, and an earlier version of this module wrongly
    /// refused to on the grounds that a face filling its instance meant the
    /// two models disagreed. It does not: it means the photograph is a
    /// close-up, which is the case where naming the region is most useful and
    /// most certain. Left as a test because the reasoning is easy to get
    /// backwards a second time.
    #[test]
    fn a_tight_portrait_is_named_rather_than_treated_as_suspicious() {
        let head = (100.0, 50.0, 400.0, 400.0);
        let face = (110.0, 60.0, 390.0, 390.0);
        assert_eq!(
            name_for_instance(head, &[named(face, "Anna")]),
            Some("Anna")
        );
    }

    /// A face box *larger* than the instance is a genuine disagreement, and
    /// containment rejects it without needing a size rule: most of the face
    /// lies outside the box it is supposed to belong to.
    #[test]
    fn a_face_larger_than_the_instance_does_not_name_it() {
        let small_instance = (200.0, 200.0, 260.0, 260.0);
        let huge_face = (100.0, 100.0, 500.0, 500.0);
        assert_eq!(
            name_for_instance(small_instance, &[named(huge_face, "Anna")]),
            None
        );
    }

    /// Two people merged into one blob: naming either would be a coin toss,
    /// and a mask list saying "Anna" about her brother is worse than one
    /// saying "person".
    #[test]
    fn two_comparable_faces_in_one_instance_leave_it_unnamed() {
        let wide = (0.0, 0.0, 800.0, 900.0);
        let faces = [
            named((100.0, 80.0, 200.0, 220.0), "Anna"),
            named((500.0, 85.0, 605.0, 230.0), "Bob"),
        ];
        assert_eq!(name_for_instance(wide, &faces), None);
    }

    /// But a clearly nearer subject wins: the box is mostly about them.
    #[test]
    fn a_much_larger_face_wins_over_someone_in_the_background() {
        let wide = (0.0, 0.0, 800.0, 900.0);
        let faces = [
            named((100.0, 80.0, 300.0, 360.0), "Anna"),
            named((600.0, 85.0, 640.0, 140.0), "distant"),
        ];
        assert_eq!(name_for_instance(wide, &faces), Some("Anna"));
    }

    /// The same person found twice — a mirror, a reflection — is not ambiguous
    /// even though the two faces are comparable.
    #[test]
    fn the_same_name_twice_is_not_an_ambiguity() {
        let wide = (0.0, 0.0, 800.0, 900.0);
        let faces = [
            named((100.0, 80.0, 200.0, 220.0), "Anna"),
            named((500.0, 85.0, 605.0, 230.0), "Anna"),
        ];
        assert_eq!(name_for_instance(wide, &faces), Some("Anna"));
    }

    #[test]
    fn degenerate_boxes_name_nothing_rather_than_panicking() {
        assert_eq!(
            name_for_instance((0.0, 0.0, 0.0, 0.0), &[named(FACE, "A")]),
            None
        );
        assert_eq!(
            name_for_instance(PERSON, &[named((5.0, 5.0, 5.0, 5.0), "A")]),
            None
        );
        assert_eq!(name_for_instance(PERSON, &[]), None);
    }

    #[derive(Debug, PartialEq)]
    struct Inst {
        class: String,
        bbox: (f32, f32, f32, f32),
    }

    #[test]
    fn only_person_instances_are_renamed() {
        let mut instances = vec![
            Inst {
                class: "person".into(),
                bbox: PERSON,
            },
            // A face on a screen must not rename the television.
            Inst {
                class: "tv".into(),
                bbox: PERSON,
            },
        ];
        let faces = [named(FACE, "Anna")];

        let n = name_instances(
            &mut instances,
            &faces,
            |i| i.bbox,
            |i| i.class == "person",
            |i, name| i.class = name.to_string(),
        );

        assert_eq!(n, 1);
        assert_eq!(instances[0].class, "Anna");
        assert_eq!(instances[1].class, "tv");
    }

    #[test]
    fn an_unrecognised_person_keeps_the_models_own_label() {
        let mut instances = vec![Inst {
            class: "person".into(),
            bbox: PERSON,
        }];
        let n = name_instances(
            &mut instances,
            &[],
            |i| i.bbox,
            |i| i.class == "person",
            |i, name| i.class = name.to_string(),
        );
        assert_eq!(n, 0);
        assert_eq!(instances[0].class, "person");
    }
}