diff --git a/core/dr-segment/src/semantic.rs b/core/dr-segment/src/semantic.rs index 885b0b0..0a48af5 100644 --- a/core/dr-segment/src/semantic.rs +++ b/core/dr-segment/src/semantic.rs @@ -145,6 +145,39 @@ impl Instance { } } + /// Take in another view of the same object, from an overlapping tile. + /// + /// Pointwise maximum over the coverage, union of the boxes, higher of the + /// scores — see `merge_into` for why each is right for its own quantity. + /// Both instances describe the whole frame in the same coordinates, so no + /// resampling is involved and the masks are already aligned pixel for + /// pixel. + fn absorb(&mut self, other: Self) { + debug_assert_eq!( + (self.width, self.height), + (other.width, other.height), + "instances from one detection run share the frame they are defined over" + ); + for (mine, theirs) in self.mask.iter_mut().zip(&other.mask) { + if *theirs > *mine { + *mine = *theirs; + } + } + self.bbox = ( + self.bbox.0.min(other.bbox.0), + self.bbox.1.min(other.bbox.1), + self.bbox.2.max(other.bbox.2), + self.bbox.3.max(other.bbox.3), + ); + if other.score > self.score { + self.score = other.score; + // The name travels with the score: they are one judgement, and a + // mask labelled by the less confident of two detections would be + // labelled by the one we just decided to trust less. + self.class_name = other.class_name; + } + } + fn iou(&self, other: &Self, threshold: f32) -> f32 { let mut inter = 0usize; let mut union = 0usize; @@ -570,8 +603,33 @@ fn assemble_mask( /// /// Only needed for [`Tiling::Grid`]: an object straddling a seam is seen by /// both tiles, and without this it would appear twice in the list a person -/// chooses from. Keeps the higher-scoring copy, which is generally the tile -/// that saw more of the object. +/// chooses from. +/// +/// # Why the two are unioned rather than one of them chosen +/// +/// This kept the higher-scoring copy and discarded the other. That is the +/// wrong answer for the case tiling exists to serve, and it quietly threw away +/// what tiling had just paid for. +/// +/// A tile sees the part of an object that falls inside it and nothing of the +/// rest, so an object on a seam produces *two partial masks*, neither of them +/// the object. Keeping the better one keeps the larger fragment — a bird with +/// its tail cut off at the tile edge, described by whichever tile held more of +/// the bird. Both halves exist; only one survived. +/// +/// Unioning is sound precisely because the tiles overlap. With a 25% overlap +/// every pixel is seen by at least one tile at full resolution and pixels near +/// a seam by two, so the pointwise maximum of the two coverages is the better +/// estimate everywhere rather than a compromise: where only one tile saw a +/// pixel it is the only opinion available, and where both did, the higher +/// value comes from the tile that had more context around it. Taking a maximum +/// of soft coverage also keeps the mask soft, which is what `prior.rs` weights +/// merges by and what a mask layer's edge treatment needs. +/// +/// The score is the higher of the two rather than a blend. It is shown to a +/// photographer beside the class name and means "how sure the model is this is +/// a bird"; averaging in the tile that saw only a wingtip would make a +/// confident detection look doubtful for straddling a seam. fn merge_into(found: &mut Vec, batch: Vec, options: &SemanticOptions) { for candidate in batch { let duplicate = found.iter_mut().find(|existing| { @@ -580,8 +638,7 @@ fn merge_into(found: &mut Vec, batch: Vec, options: &Semanti }); match duplicate { - Some(existing) if existing.score < candidate.score => *existing = candidate, - Some(_) => {} + Some(existing) => existing.absorb(candidate), None => found.push(candidate), } } @@ -671,6 +728,79 @@ mod tests { assert!(lb.pad_y > 100.0, "wide image should pad in y: {}", lb.pad_y); } + /// A subject on a seam, seen in part by each of two tiles. + /// + /// The failure this pins down is not a crash and not a duplicate: it is a + /// mask that looks plausible and is missing the half of the subject that + /// fell in the other tile. Keeping the higher-scoring detection produced + /// exactly that, and it is invisible unless you already know what the + /// whole subject should have been. + #[test] + fn two_tiles_seeing_one_subject_produce_the_whole_subject() { + // The left tile sees the left half strongly and nothing of the right; + // the right tile sees the right half. Together they are one bar. + let left = instance_with( + &[0.9, 0.9, 0.9, 0.8, 0.0, 0.0, 0.0, 0.0], + 0.80, + (0.0, 0.0, 4.0, 1.0), + ); + let right = instance_with( + &[0.0, 0.0, 0.3, 0.7, 0.9, 0.9, 0.9, 0.0], + 0.60, + (2.0, 0.0, 7.0, 1.0), + ); + + let options = SemanticOptions { + mask_threshold: 0.5, + merge_iou: 0.1, + ..SemanticOptions::default() + }; + + let mut found = vec![left]; + merge_into(&mut found, vec![right], &options); + + assert_eq!(found.len(), 1, "one subject, not two"); + let m = &found[0]; + + // Every pixel either tile was sure about survives. Under the old + // keep-the-better-one rule, pixels 4..=6 were lost entirely. + for i in [0, 1, 2, 3, 4, 5, 6] { + assert!( + m.mask[i] >= 0.5, + "pixel {i} was seen by a tile and must survive the merge: {:?}", + m.mask + ); + } + assert!(m.mask[7] < 0.5, "a pixel neither tile saw must stay out"); + + // Pointwise maximum, not an average: pixel 2 is 0.9 in one tile and + // 0.3 in the other, and averaging would report 0.6 — a softer edge + // than either tile actually saw. + assert!( + (m.mask[2] - 0.9).abs() < 1e-6, + "expected the max, got {}", + m.mask[2] + ); + + assert!( + (m.score - 0.80).abs() < 1e-6, + "the confident detection's score survives" + ); + assert_eq!(m.bbox, (0.0, 0.0, 7.0, 1.0), "the box covers both halves"); + } + + fn instance_with(mask: &[f32], score: f32, bbox: (f32, f32, f32, f32)) -> Instance { + Instance { + class_id: 14, + class_name: "bird".into(), + score, + bbox, + mask: mask.to_vec(), + width: mask.len(), + height: 1, + } + } + /// The seam case tiling exists for, and the one it must not double-count. #[test] fn grid_tiling_covers_the_frame_with_overlap() {