diff --git a/ui/dr-ui/src/develop.rs b/ui/dr-ui/src/develop.rs index a78b7e7..b19f490 100644 --- a/ui/dr-ui/src/develop.rs +++ b/ui/dr-ui/src/develop.rs @@ -218,6 +218,263 @@ impl SegmentationJob { } } +/// Longest edge the refine crop is rendered at. +/// +/// Matches [`dr_segment::semantic::INPUT_EDGE`] rather than exceeding it: the +/// model's own input is still fixed at 640x640, so rendering the crop larger +/// only gets downsampled again inside the model's letterbox. The resolution +/// win is entirely from *what fills the window* — a padded crop around one +/// subject rather than the whole frame — not from feeding the model more +/// pixels than it has ever read. +const REFINE_EDGE: u32 = 640; + +/// How much of the box's own size is added on each side before cropping. +/// +/// Context for the model to place the subject's edge against, and slack for +/// a box that under-ran the subject slightly on the first pass. Not so much +/// that a second subject standing nearby gets pulled into the same window. +const REFINE_PADDING: f32 = 0.25; + +/// TRACES: FR-DEV-3 +/// A refine pass lifted out of the session that asked for it. +/// +/// The same split as [`SegmentationJob`], for the same reason — see its +/// docs. This one answers for a single already-detected instance rather than +/// the whole frame: it re-runs the model over a padded crop of just that +/// subject's box, so the subject reaches the model at its own size instead +/// of squeezed into the model's fixed window alongside everything else in +/// the photograph. +pub struct RefineJob { + ctx: GpuContext, + source: Arc, + session: SessionId, + abandon: Abandon, + /// Which instance this answers for. A snapshot of what it needs from the + /// segmentation, taken when the job was built — the same reason + /// `SegmentationJob` carries a proxy render rather than the session. + index: usize, + class_name: Arc, + bbox: (f32, f32, f32, f32), + proxy: (usize, usize), +} + +impl RefineJob { + pub fn session(&self) -> SessionId { + self.session + } + + pub fn abandon(&self) -> Abandon { + self.abandon.clone() + } + + /// TRACES: FR-DEV-3 + /// Re-detect this one subject at higher effective resolution. **Blocking.** + /// + /// `Ok(None)` covers both "abandoned" and "the model found nothing of the + /// same class in the crop" — a photographer pressing refine on a subject + /// that the padded window no longer contains is a possible outcome, not + /// a bug, and the caller treats it as "kept what was there" either way. + pub fn run(&self) -> Result, String> { + if self.abandon.asked() { + return Ok(None); + } + + let (px0, py0, px1, py1) = self.bbox; + let (pw, ph) = (self.proxy.0 as f32, self.proxy.1 as f32); + if pw <= 0.0 || ph <= 0.0 { + return Ok(None); + } + + let (bw, bh) = ((px1 - px0).max(1.0), (py1 - py0).max(1.0)); + let (padx, pady) = (bw * REFINE_PADDING, bh * REFINE_PADDING); + let x0 = (px0 - padx).max(0.0); + let y0 = (py0 - pady).max(0.0); + let x1 = (px1 + padx).min(pw); + let y1 = (py1 + pady).min(ph); + if x1 <= x0 || y1 <= y0 { + return Ok(None); + } + let (cw_px, ch_px) = (x1 - x0, y1 - y0); + + let view = CropRect { + x: x0 / pw, + y: y0 / ph, + width: cw_px / pw, + height: ch_px / ph, + }; + + // Aspect-matched render target, capped against blowing a tiny box up + // absurdly far past what the source ever had to offer. + let scale = (REFINE_EDGE as f32 / cw_px.max(ch_px)).min(4.0); + let (tw, th) = ( + (cw_px * scale).round().max(1.0) as u32, + (ch_px * scale).round().max(1.0) as u32, + ); + + if self.abandon.asked() { + return Ok(None); + } + let (rgb, rw, rh) = self.render_view(view, tw, th)?; + if self.abandon.asked() { + return Ok(None); + } + + let mut model = dr_segment::SemanticModel::embedded().map_err(|e| e.to_string())?; + let options = dr_segment::SemanticOptions { + tiling: dr_segment::Tiling::Whole, + ..dr_segment::SemanticOptions::default() + }; + let found = model + .detect(&rgb, rw, rh, &options) + .map_err(|e| e.to_string())?; + + // The crop was built around one subject, so the right answer among + // whatever the model found in it is the same class closest to the + // window's centre — not merely the highest score, which a second, + // unrelated instance caught in the padding could win. + let (cx, cy) = (rw as f32 * 0.5, rh as f32 * 0.5); + let best = found + .into_iter() + .filter(|i| i.class_name.as_ref() == self.class_name.as_ref()) + .min_by(|a, b| centre_distance(a, cx, cy).total_cmp(¢re_distance(b, cx, cy))); + + let Some(instance) = best else { + return Ok(None); + }; + + // Downsampled back onto the shared proxy grid like every other + // instance's mask is, but built from a sharper source than the + // whole-frame pass ever saw for this subject. + let mask = paste_into_proxy(&instance.mask, rw, rh, self.proxy, (x0, y0), (cw_px, ch_px)); + let bbox = ( + x0 + instance.bbox.0 / scale, + y0 + instance.bbox.1 / scale, + x0 + instance.bbox.2 / scale, + y0 + instance.bbox.3 / scale, + ); + + Ok(Some(RefinedInstance { + index: self.index, + summary: segmentation::InstanceSummary { + class_name: instance.class_name, + score: instance.score, + mask, + bbox, + }, + })) + } + + /// Render the *unedited* image, showing only `view`, at `(width, height)`. + /// + /// The same neutral, as-captured render [`SegmentationJob::neutral_proxy`] + /// uses — a refine pass must read the same kind of pixels the first pass + /// did, or a subject would gain or lose an edge depending on which pass + /// found it. `view` is framing's ephemeral viewport (`Framing::set_view`), + /// the mechanism the on-screen zoom already uses to render a region at + /// more than proxy resolution — not the crop tool's own persisted + /// rectangle, and nothing here touches that. + fn render_view( + &self, + view: CropRect, + width: u32, + height: u32, + ) -> Result<(Vec, usize, usize), String> { + let mut neutral = EditGraph::default_chain(); + neutral.framing_mut().set_view(view); + + let mut pass = AdjustPass::new(&self.ctx); + pass.render(&self.source, &neutral.compose(), width, height) + .map_err(|e| format!("could not render the refine crop: {e}"))?; + let (rgba, pw, ph) = pass + .export_pixels() + .map_err(|e| format!("could not read the refine crop: {e}"))?; + + let rgb = rgba + .chunks_exact(4) + .flat_map(|p| { + [ + p[0] as f32 / 255.0, + p[1] as f32 / 255.0, + p[2] as f32 / 255.0, + ] + }) + .collect(); + Ok((rgb, pw as usize, ph as usize)) + } +} + +/// What a finished [`RefineJob`] hands back: a replacement for one instance +/// in the segmentation it was run against. +pub struct RefinedInstance { + index: usize, + summary: segmentation::InstanceSummary, +} + +fn centre_distance(instance: &dr_segment::Instance, cx: f32, cy: f32) -> f32 { + let (x0, y0, x1, y1) = instance.bbox; + let (ix, iy) = ((x0 + x1) * 0.5, (y0 + y1) * 0.5); + ((ix - cx).powi(2) + (iy - cy).powi(2)).sqrt() +} + +/// Sample a crop-local mask back onto its footprint in the shared proxy grid. +/// +/// Bilinear, the same as every other resampling in this mask pipeline +/// (`dr_segment::semantic`'s own prototype sampling, the letterbox that feeds +/// it) — correct whether the crop was rendered denser than the proxy (the +/// common case this feature exists for) or coarser than it (a subject large +/// enough that refining it buys little, which still renders a sensible if +/// unremarkable answer rather than a distorted one). +fn paste_into_proxy( + crop_mask: &[f32], + crop_w: usize, + crop_h: usize, + proxy: (usize, usize), + origin_px: (f32, f32), + extent_px: (f32, f32), +) -> Vec { + let (pw, ph) = proxy; + let mut out = vec![0u8; pw * ph]; + let (ew, eh) = extent_px; + if crop_w == 0 || crop_h == 0 || pw == 0 || ph == 0 || ew <= 0.0 || eh <= 0.0 { + return out; + } + + let (ox, oy) = origin_px; + let px0 = ox.floor().max(0.0) as usize; + let py0 = oy.floor().max(0.0) as usize; + let px1 = ((ox + ew).ceil() as usize).min(pw); + let py1 = ((oy + eh).ceil() as usize).min(ph); + + for py in py0..py1 { + let gy = (py as f32 + 0.5 - oy) / eh * crop_h as f32; + if gy < 0.0 || gy >= crop_h as f32 { + continue; + } + for px in px0..px1 { + let gx = (px as f32 + 0.5 - ox) / ew * crop_w as f32; + if gx < 0.0 || gx >= crop_w as f32 { + continue; + } + let v = bilinear_sample(crop_mask, crop_w, crop_h, gx, gy); + out[py * pw + px] = (v.clamp(0.0, 1.0) * 255.0).round() as u8; + } + } + out +} + +fn bilinear_sample(mask: &[f32], w: usize, h: usize, x: f32, y: f32) -> f32 { + let (fx0, fy0) = (x.floor(), y.floor()); + let (fx, fy) = (x - fx0, y - fy0); + let x0 = (fx0 as isize).clamp(0, w as isize - 1) as usize; + let y0 = (fy0 as isize).clamp(0, h as isize - 1) as usize; + let x1 = (x0 + 1).min(w - 1); + let y1 = (y0 + 1).min(h - 1); + let at = |x: usize, y: usize| mask[y * w + x]; + let top = at(x0, y0) * (1.0 - fx) + at(x1, y0) * fx; + let bot = at(x0, y1) * (1.0 - fx) + at(x1, y1) * fx; + top * (1.0 - fy) + bot * fy +} + pub struct DevelopSession { /// This session's name, for work that outlives the frame it started on. id: SessionId, @@ -265,12 +522,17 @@ pub struct DevelopSession { /// has, so this deliberately does *not* include them: dragging those /// sliders must not rebuild anything. subject_key: u64, - /// Which layer the develop panel is editing, if any. + /// Which layers the develop panel is editing, if any. /// - /// This is what lets one panel serve both scopes: with a layer selected, - /// the sliders read and write *its* chain, and the photographer is - /// adjusting a region rather than the frame. - active_mask: Option, + /// This is what lets one panel serve both scopes: with layers selected, + /// the sliders read and write *their* chains, and the photographer is + /// adjusting one or more regions rather than the frame. + /// + /// A plain click replaces this outright; a modifier-click toggles one id + /// in or out, so several layers can be shaped by the same slider drag — + /// "make these three subjects a stop darker" is one gesture rather than + /// three. Order is insertion order and nothing reads it, only membership. + active_masks: Vec, /// Whether to draw the false-coloured region overlay. show_overlay: bool, /// Which attribute the panel is filtered to, or all of them. @@ -356,7 +618,7 @@ impl DevelopSession { masks: None, subjects: None, subject_key: 0, - active_mask: None, + active_masks: Vec::new(), show_overlay: false, active_tab: None, curve_channel: 0, @@ -436,14 +698,27 @@ impl DevelopSession { .and_then(|i| self.tabs().get(i).map(|(a, _)| *a)); } + /// The selection's representative layer, for anything that can only show + /// one answer — which tab is open, what value a slider currently reads. + /// + /// The first id selected, not "the" active layer: with more than one + /// selected there is no single truth to show, and the panel has to pick + /// something. Whichever layer this is, [`Self::set_param`] and + /// [`Self::reset_op`] still write to every selected layer, not just this + /// one — a slider shows one number and applies it everywhere selected. fn active_layer(&self) -> Option<&MaskLayer> { - let id = self.active_mask.as_ref()?; + let id = self.active_masks.first()?; self.graph.masks().get(id) } - fn active_layer_mut(&mut self) -> Option<&mut MaskLayer> { - let id = self.active_mask.clone()?; - self.graph.masks_mut().get_mut(&id) + /// Every selected layer, mutably — what a batched slider or reset walks. + fn active_layers_mut(&mut self) -> impl Iterator { + let selected = self.active_masks.clone(); + self.graph + .masks_mut() + .layers_mut() + .iter_mut() + .filter(move |l| selected.contains(&l.id)) } } @@ -968,11 +1243,11 @@ impl DevelopSession { let Some(cap) = usize::try_from(op_index).ok().and_then(|i| caps.get(i)) else { return; }; - if self.active_mask.is_some() { + if !self.active_masks.is_empty() { let params: Vec<_> = cap.params.iter().map(|p| (p.id, p.default)).collect(); let id = cap.id.0; - if let Some(layer) = self.active_layer_mut() { - for (param, default) in params { + for layer in self.active_layers_mut() { + for &(param, default) in ¶ms { layer.set_param(id, param, default); } } @@ -1005,12 +1280,17 @@ impl DevelopSession { log::warn!("control at ({op_index}, {param_index}) has no parameter"); return; }; - if self.active_mask.is_some() { - if let Some(layer) = self.active_layer_mut() { + if !self.active_masks.is_empty() { + // Every selected layer is set to the same absolute value the + // slider now shows, not offset by however far each one already + // was from it — the slider has one position, and "apply this + // reading to all of them" is the reading a photographer gets + // from watching it move. + for layer in self.active_layers_mut() { layer.set_param(op.0, param, value); } // Coalesced the same way a global drag is: a slider dragged across - // a masked layer is still one gesture and must undo as one. + // masked layers is still one gesture and must undo as one. let edit = Edit::for_param(&self.graph, op, param); self.history.record(&self.graph, edit); return; @@ -1338,6 +1618,55 @@ impl DevelopSession { self.subject_key = 0; } + /// TRACES: FR-DEV-3 + /// Everything a refine pass needs for one already-detected instance, so + /// it can be run somewhere else — see [`Self::segmentation_job`] for why + /// the split exists. + /// + /// `None` where there is nothing to refine: no segmentation yet, or an + /// index the panel offered a button for a moment ago but the segmentation + /// underneath has since changed. + pub fn refine_job(&self, index: usize) -> Option { + let seg = self.segmentation.as_ref()?; + let instance = seg.instances().get(index)?; + Some(RefineJob { + ctx: self.ctx.clone(), + source: self.demosaiced.clone(), + session: self.id, + abandon: Abandon::default(), + index, + class_name: instance.class_name.clone(), + bbox: instance.bbox, + proxy: seg.proxy_size(), + }) + } + + /// Take on a refined instance found elsewhere. + /// + /// Same caution as [`Self::adopt_segmentation`]: the caller checks the + /// job's session before handing back its answer, not this. Every mask + /// layer pointing at this instance's index reads it fresh next redraw — + /// `subject_key` is reset because the pixels changed under an index + /// `subject_signature` has no way to know changed, unlike a morphology + /// slider it does track. + pub fn adopt_refined(&mut self, refined: RefinedInstance) { + if let Some(seg) = self.segmentation.as_mut() { + seg.replace_instance(refined.index, refined.summary); + } + self.subjects = None; + self.subject_key = 0; + } + + /// The mask layer's original detection index, for the refine button — + /// `None` for a layer that is not a subject at all (a gradient has no + /// instance to re-detect). + pub fn subject_instance_index(&self, id: &str) -> Option { + match &self.graph.masks().get(id)?.source { + MaskSource::Subject { index, .. } => Some(*index as usize), + _ => None, + } + } + /// Find the subjects and take them on, blocking until both are done. /// /// Test-only, and deliberately: a session-shaped blocking call is exactly @@ -1456,7 +1785,7 @@ impl DevelopSession { l.id.clone(), l.display_name().to_string(), l.enabled, - Some(&l.id) == self.active_mask.as_ref(), + self.active_masks.iter().any(|a| a == &l.id), ) }) .collect() @@ -1488,15 +1817,25 @@ impl DevelopSession { .is_some_and(|l| l.active_ops().next().is_some()) } + /// The panel's representative selection — see [`Self::active_layer`] for + /// what "representative" means once more than one layer is selected. pub fn active_mask(&self) -> Option<&str> { - self.active_mask.as_deref() + self.active_masks.first().map(String::as_str) + } + + /// Every selected layer's id, in selection order. + pub fn active_masks(&self) -> &[String] { + &self.active_masks } /// TRACES: FR-DEV-3 | FR-UI-3 /// The selected gradient's handles, in fractions of the shown image. /// - /// Empty unless a gradient layer is selected, which is what makes this the - /// panel's whole test for "is there anything to draw on the canvas". + /// Empty unless **exactly one** gradient layer is selected. Dragging a + /// shared handle for several gradients at once has no single geometry to + /// move — each one's centre, angle and extent differ — so multi-select + /// simply offers no handles rather than moving one layer's shape while + /// silently leaving the others behind. /// /// Recomputed on every redraw rather than cached, because the answer /// changes with the *view* and not only with the mask: a pan moves every @@ -1504,6 +1843,9 @@ impl DevelopSession { /// not work worth caching, and a cache keyed on the wrong thing is how a /// handle comes to sit where the mask used to be. pub fn gradient_handles(&self) -> Vec { + if self.active_masks.len() != 1 { + return Vec::new(); + } let Some(layer) = self.active_layer() else { return Vec::new(); }; @@ -1525,9 +1867,12 @@ impl DevelopSession { press: (f32, f32), now: (f32, f32), ) -> Option { + if self.active_masks.len() != 1 { + return None; + } let (sw, sh) = self.demosaiced.size(); let framing = *self.graph.framing(); - let id = self.active_mask.clone()?; + let id = self.active_masks.first()?.clone(); let start = match origin { Some(s) => s.clone(), None => self.graph.masks().get(&id)?.source.clone(), @@ -1557,12 +1902,33 @@ impl DevelopSession { self.active_layer().map(|l| l.source.clone()) } - /// Select a layer for editing, or `None` to return the panel to the - /// global chain. + /// Select exactly one layer for editing, or `None` to return the panel to + /// the global chain. Replaces whatever was selected before, including a + /// multi-selection — the ordinary, unmodified click. pub fn set_active_mask(&mut self, id: Option<&str>) { - self.active_mask = id + self.active_masks = id .filter(|id| self.graph.masks().get(id).is_some()) - .map(|id| id.to_string()); + .map(|id| vec![id.to_string()]) + .unwrap_or_default(); + } + + /// Add or remove one layer from the selection, keeping the rest — the + /// modifier-click that builds a multi-selection. + /// + /// A layer id the graph no longer has is dropped rather than toggled in: + /// the row that offered it is stale by the time the click lands, and + /// selecting a ghost would make every subsequent batched edit silently + /// skip it (`active_layers_mut` filters by membership, not existence). + pub fn toggle_active_mask(&mut self, id: &str) { + if self.graph.masks().get(id).is_none() { + return; + } + match self.active_masks.iter().position(|a| a == id) { + Some(i) => { + self.active_masks.remove(i); + } + None => self.active_masks.push(id.to_string()), + } } /// Mask the object under a normalised image point. @@ -1606,7 +1972,7 @@ impl DevelopSession { if !self.graph.masks_mut().push(layer) { return None; } - self.active_mask = Some(id.clone()); + self.active_masks = vec![id.clone()]; self.history.record(&self.graph, Edit::Discrete); Some(id) } @@ -1635,16 +2001,14 @@ impl DevelopSession { { return None; } - self.active_mask = Some(id.clone()); + self.active_masks = vec![id.clone()]; self.history.record(&self.graph, Edit::Discrete); Some(id) } pub fn remove_mask(&mut self, id: &str) { if self.graph.masks_mut().remove(id).is_some() { - if self.active_mask.as_deref() == Some(id) { - self.active_mask = None; - } + self.active_masks.retain(|a| a != id); self.history.record(&self.graph, Edit::Discrete); } } @@ -2671,6 +3035,95 @@ mod tests { ); } + /// TRACES: FR-DEV-3 + /// Multi-select: one slider, applied to every selected layer. + /// + /// `toggle_active_mask` builds the selection a control-click makes, and + /// `set_param`/`reset_op` are what a drag and a reset call — this pins + /// down that both fan out to every layer in it rather than only the + /// first, which is the whole point of selecting more than one. + #[test] + fn a_slider_moved_with_two_layers_selected_moves_both() { + let Some(ctx) = headless() else { return }; + let rgba: Vec = (0..8 * 8).flat_map(|_| [128u8, 128, 128, 255]).collect(); + let mut session = DevelopSession::open_rgb(&ctx, &rgba, 8, 8, dr_types::Orientation::NORMAL) + .expect("session"); + + let a = session.add_gradient_mask(true).expect("first gradient"); + let b = session.add_gradient_mask(false).expect("second gradient"); + // Adding `b` selected it alone — build the multi-selection a + // control-click would, starting from that single-layer state. + session.toggle_active_mask(&a); + assert_eq!(session.active_masks(), [b.clone(), a.clone()].as_slice()); + + assert!( + !session.mask_is_adjusted(&a) && !session.mask_is_adjusted(&b), + "neither layer has been touched yet" + ); + + let row = session.rows()[0].clone(); + session.set_param(row.op_index, row.param_index, row.maximum); + + assert!( + session.mask_is_adjusted(&a) && session.mask_is_adjusted(&b), + "one slider, both layers selected, both layers must show the edit" + ); + + // And a reset walks the same set. + session.reset_op(row.op_index); + assert!( + !session.mask_is_adjusted(&a) && !session.mask_is_adjusted(&b), + "resetting with both selected must clear both, not just the one \ + the panel happens to read values from" + ); + } + + /// TRACES: FR-DEV-3 + /// A refine crop's mask, pasted back onto the proxy grid it replaces, + /// must land exactly where the crop was — not shifted by the origin, not + /// scaled onto the wrong footprint. + #[test] + fn a_refined_mask_pastes_back_at_the_crops_own_position() { + // A crop entirely on (1.0), covering proxy pixels 2..6 in x and + // 2..6 in y of an 8x8 proxy — everywhere inside that box must read + // back as fully covered, everywhere outside as untouched. + let crop = vec![1.0f32; 4 * 4]; + let mask = paste_into_proxy(&crop, 4, 4, (8, 8), (2.0, 2.0), (4.0, 4.0)); + + assert_eq!(mask[2 * 8 + 2], 255, "top-left corner of the box"); + assert_eq!(mask[5 * 8 + 5], 255, "bottom-right corner of the box"); + assert_eq!(mask[0], 0, "outside the box, untouched"); + assert_eq!(mask[7 * 8 + 7], 0, "outside the box, untouched"); + } + + #[test] + fn bilinear_sample_averages_its_four_neighbours() { + // Two rows, black then white: the exact midpoint reads as grey. + let mask = [0.0f32, 0.0, 1.0, 1.0]; + let v = bilinear_sample(&mask, 2, 2, 0.5, 0.5); + assert!((v - 0.5).abs() < 1e-6, "expected the midpoint grey, got {v}"); + } + + /// A control-click twice — once to add, once to remove — is a no-op on + /// the selection, which is the sanity check for `toggle_active_mask` + /// itself before trusting anything built on it. + #[test] + fn toggling_a_layer_twice_returns_to_the_starting_selection() { + let Some(ctx) = headless() else { return }; + let rgba: Vec = (0..8 * 8).flat_map(|_| [128u8, 128, 128, 255]).collect(); + let mut session = DevelopSession::open_rgb(&ctx, &rgba, 8, 8, dr_types::Orientation::NORMAL) + .expect("session"); + + let a = session.add_gradient_mask(true).expect("gradient"); + assert_eq!(session.active_masks(), [a.clone()].as_slice()); + + session.toggle_active_mask(&a); + assert!(session.active_masks().is_empty(), "removed by the toggle"); + + session.toggle_active_mask(&a); + assert_eq!(session.active_masks(), [a.clone()].as_slice(), "added back"); + } + /// TRACES: FR-DEV-3 /// The mask array and the segmentation proxy are the same size on purpose. /// diff --git a/ui/dr-ui/src/masks_ui.rs b/ui/dr-ui/src/masks_ui.rs index d3ed093..2553d05 100644 --- a/ui/dr-ui/src/masks_ui.rs +++ b/ui/dr-ui/src/masks_ui.rs @@ -28,7 +28,7 @@ use std::rc::Rc; use slint::{ComponentHandle as _, Model as _, ModelRc, VecModel}; -use crate::develop::{Abandon, DevelopSession, Segmented, SessionId}; +use crate::develop::{Abandon, DevelopSession, RefinedInstance, Segmented, SessionId}; use crate::segmentation; use crate::{sync_rows, AppWindow, GradientHandle, MaskRow, ParamRow, SubjectRow}; @@ -48,17 +48,18 @@ pub(crate) const GLOBAL_SCOPE: &str = "ADJUST"; /// string the row in the stack above shows — one name for one thing, so the /// selected row and the panel it scopes cannot appear to disagree. pub(crate) fn scope_label(session: &DevelopSession) -> String { - let Some(id) = session.active_mask() else { - return GLOBAL_SCOPE.to_string(); - }; - session - .mask_layers() - .into_iter() - .find(|(layer_id, ..)| layer_id == id) - .map_or_else( - || GLOBAL_SCOPE.to_string(), - |(_, label, ..)| label.to_uppercase(), - ) + match session.active_masks() { + [] => GLOBAL_SCOPE.to_string(), + [id] => session + .mask_layers() + .into_iter() + .find(|(layer_id, ..)| layer_id == id) + .map_or_else( + || GLOBAL_SCOPE.to_string(), + |(_, label, ..)| label.to_uppercase(), + ), + many => format!("{} LAYERS", many.len()), + } } /// How often the window looks to see whether the model has finished. @@ -347,6 +348,49 @@ pub(crate) fn wire( }); } + // --- refining one subject's mask --------------------------------------- + // + // Its own `Running` slot rather than sharing the segmentation one: the + // two answer different questions (the whole frame's subjects versus one + // already-found instance) and there is no reason a refine in flight + // should block a fresh "Find subjects", or the reverse. + let refining: Rc> = Rc::default(); + { + let weak = window.as_weak(); + let session = session.clone(); + let redraw = redraw.clone(); + let rows = rows.clone(); + let running = refining.clone(); + window.on_mask_refined(move |id| { + let Some(w) = weak.upgrade() else { return }; + let Some(index) = session + .borrow() + .as_ref() + .and_then(|s| s.subject_instance_index(&id)) + else { + return; + }; + let Some(job) = session.borrow().as_ref().and_then(|s| s.refine_job(index)) else { + return; + }; + let session_id = job.session(); + + let claimed = running.borrow_mut().start(session_id, job.abandon()); + if !claimed { + return; + } + + w.set_refining(true); + + let (tx, rx) = std::sync::mpsc::channel(); + std::thread::spawn(move || { + let _ = tx.send(job.run()); + }); + + watch_refine(&w, session_id, rx, &session, &rows, &redraw, &running); + }); + } + // --- dragging a gradient on the photograph ---------------------------- // // The geometry the gesture started from, held for its duration. @@ -435,15 +479,23 @@ pub(crate) fn wire( let weak = window.as_weak(); let session = session.clone(); let rows = rows.clone(); - window.on_mask_selected(move |id| { + window.on_mask_selected(move |id, extend| { let Some(w) = weak.upgrade() else { return }; { let mut slot = session.borrow_mut(); let Some(s) = slot.as_mut() else { return }; - // Clicking the selected layer again deselects it, which is how - // the panel gets back to the whole photograph without a - // separate "edit globally" control. - if s.active_mask() == Some(id.as_str()) { + if extend { + // Control/command-click: add or remove this one layer, + // keeping whatever else was already selected. + s.toggle_active_mask(&id); + } else if s.active_masks() == [id.to_string()] { + // Clicking the sole selected layer again deselects it, + // which is how the panel gets back to the whole + // photograph without a separate "edit globally" control. + // Only when it is the *only* one selected — a plain click + // on one row of a multi-selection collapses to just that + // row rather than clearing everything, which is the more + // useful reading of "I clicked a specific layer". s.set_active_mask(None); } else { s.set_active_mask(Some(&id)); @@ -672,6 +724,63 @@ fn watch( running.borrow_mut().poll = Some(timer); } +/// [`watch`]'s counterpart for a refine pass — same shape, same reason: the +/// window polls rather than being called back from the worker, and a +/// photograph the user has left discards its answer instead of applying it. +fn watch_refine( + window: &AppWindow, + id: SessionId, + rx: std::sync::mpsc::Receiver, String>>, + session: &Rc>>, + rows: &Rc>, + redraw: &Rc, + running: &Rc>, +) { + let weak = window.as_weak(); + let session = session.clone(); + let rows = rows.clone(); + let redraw = redraw.clone(); + let slot = running.clone(); + + let timer = Rc::new(slint::Timer::default()); + let stop = Rc::downgrade(&timer); + timer.start(slint::TimerMode::Repeated, POLL, move || { + let done = || { + if let Some(t) = stop.upgrade() { + t.stop(); + } + }; + + if delivery(id, open_session(&session)) == Delivery::Discard { + slot.borrow_mut().finish(id); + done(); + return; + } + + let Ok(answer) = rx.try_recv() else { return }; + slot.borrow_mut().finish(id); + done(); + + let Some(w) = weak.upgrade() else { return }; + match answer { + Ok(Some(refined)) => { + if let Some(s) = session.borrow_mut().as_mut() { + s.adopt_refined(refined); + } + } + // Abandoned, or the model found nothing of the same class in the + // padded crop — both leave the instance exactly as it was. + Ok(None) => {} + Err(e) => log::warn!("refine failed: {e}"), + } + w.set_refining(false); + sync(&w, &session); + sync_rows(&w, &rows, &session); + redraw(&w); + }); + running.borrow_mut().poll = Some(timer); +} + /// Clear the panel when the open image changes. /// /// Its own function rather than a call to [`sync`] with an empty session, @@ -683,6 +792,7 @@ fn watch( pub(crate) fn reset(window: &AppWindow) { window.set_overlay_on(false); window.set_segmenting(false); + window.set_refining(false); window.set_segmented(false); window.set_mask_rows(ModelRc::new(VecModel::::default())); window.set_subject_rows(ModelRc::new(VecModel::::default())); diff --git a/ui/dr-ui/src/segmentation.rs b/ui/dr-ui/src/segmentation.rs index 4a667f8..18d68e8 100644 --- a/ui/dr-ui/src/segmentation.rs +++ b/ui/dr-ui/src/segmentation.rs @@ -49,6 +49,12 @@ pub struct InstanceSummary { /// This is the *source* a distance field is built from, not the mask /// itself — `dr_segment::Shaped` turns it into one. pub mask: Vec, + /// `(x0, y0, x1, y1)` in [`Segmentation::proxy_size`] pixels. + /// + /// Carried through from `dr_segment::Instance` rather than re-derived + /// from the mask, so a refine pass knows what region to crop without + /// scanning a proxy-sized buffer for its own extent. + pub bbox: (f32, f32, f32, f32), } /// One image's recognised objects, ready to mask. @@ -83,6 +89,18 @@ impl Segmentation { self.instances.get(index).map(|i| i.mask.as_slice()) } + /// Replace one instance in place, keeping every other index and the + /// signature unchanged. + /// + /// What a refine pass calls once it has a sharper mask for the subject at + /// `index`: the layers pointing at this run by index still mean what they + /// meant, they just read better pixels now. + pub fn replace_instance(&mut self, index: usize, instance: InstanceSummary) { + if let Some(slot) = self.instances.get_mut(index) { + *slot = instance; + } + } + /// The strongest instance covering a point in normalised image /// coordinates. /// @@ -245,6 +263,7 @@ pub fn compute( class_name: i.class_name.clone(), score: i.score, mask: quantise(&i.mask), + bbox: i.bbox, }) .collect(); @@ -338,11 +357,13 @@ mod tests { class_name: "bus".into(), score: 0.5, mask: big, + bbox: (0.0, 0.0, 6.0, h as f32), }, InstanceSummary { class_name: "person".into(), score: 0.9, mask: small, + bbox: (4.0, 0.0, 8.0, h as f32), }, ], signature: 1, diff --git a/ui/dr-ui/ui/app.slint b/ui/dr-ui/ui/app.slint index 2b0a9f8..c3ea7db 100644 --- a/ui/dr-ui/ui/app.slint +++ b/ui/dr-ui/ui/app.slint @@ -924,6 +924,7 @@ export component AppWindow inherits Window { in property <[SubjectRow]> subject-rows; in property segmented: false; in property segmenting: false; + in property refining: false; /// The selected layer's name, already resolved, or "ADJUST" when the /// controls are global. Names the adjust panel's heading. in property adjust-scope: "ADJUST"; @@ -947,8 +948,9 @@ export component AppWindow inherits Window { /// A click on the photograph, in fractions of the shown image, plus /// whether it should extend the selection rather than replace it. callback region-picked(float, float, bool); - callback mask-selected(string); + callback mask-selected(string, bool); callback mask-removed(string); + callback mask-refined(string); callback mask-toggled(string, bool); callback mask-invert-toggled(string, bool); callback mask-opacity-changed(string, float); @@ -2268,14 +2270,16 @@ in property panel-visible: true; subjects: root.subject-rows; segmented: root.segmented; segmenting: root.segmenting; + refining: root.refining; // Two-way: the panel is where it is toggled and // the canvas is what obeys it, so neither owns // the value alone. overlay-hidden <=> root.overlay-hidden; segment(fine) => { root.segment-image(fine); } - mask-selected(id) => { root.mask-selected(id); } + mask-selected(id, extend) => { root.mask-selected(id, extend); } mask-removed(id) => { root.mask-removed(id); } + mask-refined(id) => { root.mask-refined(id); } mask-toggled(id, on) => { root.mask-toggled(id, on); } mask-invert-toggled(id, on) => { root.mask-invert-toggled(id, on); diff --git a/ui/dr-ui/ui/masks.slint b/ui/dr-ui/ui/masks.slint index 55039b3..3eab228 100644 --- a/ui/dr-ui/ui/masks.slint +++ b/ui/dr-ui/ui/masks.slint @@ -73,8 +73,13 @@ export struct SubjectRow { component MaskEntry inherits Rectangle { in property data; in property enabled: true; + /// Whether a refine pass is running (for any row — only one runs at a + /// time), so this row's button can say so and refuse a second press. + in property refining: false; - callback selected(); + /// `true` when the click should add this row to the selection rather + /// than replace it with just this row. + callback selected(bool); callback removed(); callback toggled(bool); callback invert-toggled(bool); @@ -83,6 +88,7 @@ component MaskEntry inherits Rectangle { callback falloff-picked(int); callback morphology-picked(int); callback morph-radius-changed(float); + callback refined(); height: layout.preferred-height; background: root.data.selected ? Theme.surface-raised : transparent; @@ -91,7 +97,15 @@ component MaskEntry inherits Rectangle { touch := TouchArea { mouse-cursor: pointer; enabled: root.enabled; - clicked => { root.selected(); } + // `clicked` carries no modifiers, so a plain click and a + // multi-select click need the pointer event itself: released, over + // the row, with control or command held is "add to the selection" + // rather than "replace it". + pointer-event(event) => { + if event.kind == PointerEventKind.up && event.button == PointerEventButton.left { + root.selected(event.modifiers.control || event.modifiers.meta); + } + } } layout := VerticalLayout { @@ -190,6 +204,15 @@ component MaskEntry inherits Rectangle { } } + // Only a subject has a detection behind it worth re-running. A + // gradient's edge is geometry the photographer placed, and there + // is nothing for the model to find twice. + if root.data.kind == "subject": Button { + text: root.refining ? "Refining…" : "Refine mask"; + enabled: !root.refining; + clicked => { root.refined(); } + } + // The edge. All of it reads one signed distance field, so these // are free to move — none of them recomputes anything except a // compound morphology. @@ -242,6 +265,8 @@ export component MaskPanel inherits Rectangle { in property segmented: false; /// One is being computed now. in property segmenting: false; + /// A refine pass is running for one subject layer. + in property refining: false; /// Whether the photographer has hidden the overlay to see the photograph. /// Two-way bound to the window's own, so the canvas and this control never @@ -252,8 +277,11 @@ export component MaskPanel inherits Rectangle { /// `true` asks for the slower, tiled pass. callback segment(bool); - callback mask-selected(string); + /// `true` when the click should add the layer to the selection rather + /// than replace it. + callback mask-selected(string, bool); callback mask-removed(string); + callback mask-refined(string); callback mask-toggled(string, bool); callback mask-invert-toggled(string, bool); callback mask-opacity-changed(string, float); @@ -428,8 +456,10 @@ export component MaskPanel inherits Rectangle { for mask in root.masks: MaskEntry { data: mask; enabled: root.enabled; - selected => { root.mask-selected(mask.id); } + refining: root.refining; + selected(extend) => { root.mask-selected(mask.id, extend); } removed => { root.mask-removed(mask.id); } + refined => { root.mask-refined(mask.id); } toggled(on) => { root.mask-toggled(mask.id, on); } invert-toggled(on) => { root.mask-invert-toggled(mask.id, on); } opacity-changed(v) => { root.mask-opacity-changed(mask.id, v); }