diff --git a/apps/darkroom-desktop/Cargo.toml b/apps/darkroom-desktop/Cargo.toml index 48f4a7f..dc62547 100644 --- a/apps/darkroom-desktop/Cargo.toml +++ b/apps/darkroom-desktop/Cargo.toml @@ -6,7 +6,7 @@ rust-version.workspace = true license.workspace = true [dependencies] -dr-ui.workspace = true +dr-ui = { workspace = true, features = ["scene-model"] } # For the panic hook and the log sink, directly rather than through dr-ui: # both have to be installed before `dr_ui::run`, because a panic during startup # is exactly the one they exist to catch and record (NFR-OPS-1, NFR-OPS-2). diff --git a/core/dr-gpu/src/mask.rs b/core/dr-gpu/src/mask.rs index 3c4aab0..faadecb 100644 --- a/core/dr-gpu/src/mask.rs +++ b/core/dr-gpu/src/mask.rs @@ -579,13 +579,21 @@ impl MaskPass { // fields are built per layer, in this same order, because two // layers over one subject can carry different morphology. let subject = match &layer.source { - MaskSource::Subject { .. } => match subjects.filter(|s| slot < s.len()) { - Some(s) => (s, slot), - None => { - log::warn!("mask layer {} has no distance field; skipping", layer.id); - continue; + // Category alongside Subject: both are model coverage turned + // into a distance field, both are built per layer in this same + // order, and leaving a category out of here is precisely the + // failure the comment above warns about — it binds the 1x1 + // placeholder, so the mask covers everything and the + // adjustment silently goes global. + MaskSource::Subject { .. } | MaskSource::Category { .. } => { + match subjects.filter(|s| slot < s.len()) { + Some(s) => (s, slot), + None => { + log::warn!("mask layer {} has no distance field; skipping", layer.id); + continue; + } } - }, + } _ => (&self.empty_subject, 0), }; @@ -654,7 +662,13 @@ impl MaskPass { // edge; the field is in proxy pixels. Converting here keeps the // stored edit resolution-independent while the shader works in the // units its texture is actually measured in. - MaskSource::Subject { .. } => { + // A category shares the subject's mode, and that is not a + // shortcut: both arrive as a soft coverage buffer at proxy + // resolution and both are turned into a distance field before they + // reach here. The shader has no way to tell them apart and no + // reason to want one — what differs is only which model produced + // the coverage. + MaskSource::Subject { .. } | MaskSource::Category { .. } => { let short = field_short_edge(width, height); MaskParams { mode: MODE_SUBJECT, diff --git a/core/dr-pipeline/src/mask.rs b/core/dr-pipeline/src/mask.rs index 5ed23da..0f9b561 100644 --- a/core/dr-pipeline/src/mask.rs +++ b/core/dr-pipeline/src/mask.rs @@ -532,6 +532,38 @@ pub enum MaskSource { score: f32, }, + /// Every pixel of one photographic category, from the scene model. + /// + /// The counterpart to [`Self::Subject`], and the difference is the whole + /// reason both exist. A subject is *one* instance — this dog, not that one + /// — found by a COCO-trained instance model. A category is *all* the sky, + /// or all the foliage, from an ADE20K-trained semantic model that has no + /// notion of instances at all (docs/segmentation.md §16). + /// + /// So this is what a global grade attaches to: lift the sky, desaturate + /// the vegetation, warm the architecture. Asking it for "that person + /// rather than the other two" is a category error — the model merged them + /// before the mask ever existed, and [`Self::Subject`] is the source for + /// that question. + /// + /// Stored as identity like a subject, and for the same reason: the + /// coverage is megabytes and is reproducible from the same model over the + /// same image. + Category { + /// Which segmentation run produced it, so a layer can tell whether + /// the name below still refers to something that was computed. + signature: u64, + /// The category name from `models/scene/categories.txt` — "sky", + /// "vegetation". + /// + /// A name rather than an index because the descriptor is editable: a + /// category added to it would silently renumber every layer stored + /// against an index, and the failure would be a mask quietly grading + /// the wrong thing. A name that no longer exists is simply not found, + /// and the layer reads as stale. + name: String, + }, + /// A linear gradient — the graduated-filter mask. /// /// Geometry is in **normalised source coordinates**, so it survives a crop, @@ -596,6 +628,7 @@ impl MaskSource { match self { Self::Regions { .. } => "regions", Self::Subject { .. } => "subject", + Self::Category { .. } => "category", Self::Linear { .. } => "linear", Self::Radial { .. } => "radial", Self::Brush { .. } => "brush", @@ -825,9 +858,9 @@ impl MaskLayer { /// should offer to recompute rather than render it. pub fn is_stale(&self, current: u64) -> bool { match self.source { - MaskSource::Regions { signature, .. } | MaskSource::Subject { signature, .. } => { - signature != current - } + MaskSource::Regions { signature, .. } + | MaskSource::Subject { signature, .. } + | MaskSource::Category { signature, .. } => signature != current, // A gradient is geometry in normalised coordinates. It means the // same thing whatever was or was not detected, so nothing about a // new run can invalidate it. Painted strokes are the same: they are diff --git a/core/dr-pipeline/src/sidecar.rs b/core/dr-pipeline/src/sidecar.rs index 6f83616..d0857da 100644 --- a/core/dr-pipeline/src/sidecar.rs +++ b/core/dr-pipeline/src/sidecar.rs @@ -971,6 +971,10 @@ fn write_mask(out: &mut String, version: &str, layer: &MaskLayer) { let _ = writeln!(out, "class = {class}"); let _ = writeln!(out, "score = {}", format_value(*score)); } + MaskSource::Category { signature, name } => { + let _ = writeln!(out, "signature = {signature}"); + let _ = writeln!(out, "category = {name}"); + } MaskSource::Linear { centre, angle, @@ -1124,6 +1128,7 @@ struct PartialMask { ids: Vec, index: u32, class: String, + category: String, score: f32, centre: (f32, f32), radii: (f32, f32), @@ -1154,6 +1159,7 @@ impl PartialMask { level: 0, ids: Vec::new(), index: 0, + category: String::new(), class: String::new(), score: 0.0, centre: (0.5, 0.5), @@ -1191,6 +1197,11 @@ impl PartialMask { self.ids.sort_unstable(); self.ids.dedup(); } + // A category's own key rather than reusing `class`: both name a + // thing the mask covers, but one is a COCO instance's label and + // the other an entry in the scene descriptor, and a file that + // conflated them would round-trip a subject into a category. + "category" => self.category = value.to_string(), "index" => self.index = value.parse().unwrap_or(0), "class" => self.class = value.to_string(), "score" => self.score = value.parse().unwrap_or(0.0), @@ -1257,6 +1268,10 @@ impl PartialMask { class: self.class, score: self.score, }, + "category" => MaskSource::Category { + signature: self.signature, + name: self.category, + }, "linear" => MaskSource::Linear { centre: self.centre, angle: self.angle, diff --git a/core/dr-pipeline/tests/mask_sidecar.rs b/core/dr-pipeline/tests/mask_sidecar.rs index 80596a8..c995bc9 100644 --- a/core/dr-pipeline/tests/mask_sidecar.rs +++ b/core/dr-pipeline/tests/mask_sidecar.rs @@ -616,3 +616,64 @@ fn show_a_sidecar() { println!("\n{}", sidecar.to_text()); } + +/// A category mask is a name plus a signature, and both have to survive. +/// +/// The name especially. `MaskSource::Category` stores one rather than an index +/// precisely so that editing `models/scene/categories.txt` cannot repoint a +/// stored layer — and that reasoning is worth nothing if the name is what the +/// sidecar drops. +#[test] +fn category_masks_keep_their_name() { + let mut graph = EditGraph::default_chain(); + + let mut sky = MaskLayer::new( + "m1", + MaskSource::Category { + signature: 0x5EED_1234, + name: "sky".into(), + }, + ); + sky.name = "sky".into(); + sky.feather = 0.03; + sky.set_param("exposure", ParamId("exposure"), 0.5); + graph.masks_mut().push(sky); + + // A multi-word name too: the writer emits `category = swimming pool` on one + // line, and a reader that split on whitespace would silently truncate it to + // a category no model has. + let mut pool = MaskLayer::new( + "m2", + MaskSource::Category { + signature: 0x5EED_1234, + name: "swimming pool".into(), + }, + ); + pool.set_param("saturation", ParamId("saturation"), -30.0); + graph.masks_mut().push(pool); + + let restored = round_trip(&graph); + let layers = restored.masks().layers(); + assert_eq!(layers.len(), 2); + assert_eq!(layers[0].source, graph.masks().layers()[0].source); + assert_eq!(layers[1].source, graph.masks().layers()[1].source); + assert!((layers[0].feather - 0.03).abs() < 1e-6); +} + +/// A category is stale when its run is, exactly as a subject is. +/// +/// The coverage buffer lives in the session, not the sidecar, so a layer whose +/// signature no longer matches is pointing at pixels that were never computed. +/// Rendering it anyway would mask nothing and read as a broken adjustment. +#[test] +fn a_category_from_another_run_is_stale() { + let layer = MaskLayer::new( + "m1", + MaskSource::Category { + signature: 1, + name: "sky".into(), + }, + ); + assert!(layer.is_stale(2), "a different run must invalidate it"); + assert!(!layer.is_stale(1), "the run it was built against must not"); +} diff --git a/core/dr-segment/src/scene.rs b/core/dr-segment/src/scene.rs index b5ae215..1dcb5f1 100644 --- a/core/dr-segment/src/scene.rs +++ b/core/dr-segment/src/scene.rs @@ -57,6 +57,8 @@ use std::sync::Arc; use ndarray::ArrayView3; +#[cfg(test)] +use crate::semantic::INPUT_EDGE; use crate::semantic::{install_backend, Letterbox, Window}; use crate::SegmentError; @@ -353,6 +355,34 @@ impl Scene { } } +impl Scene { + /// Build a scene from known weights, for tests. + /// + /// The geometry in [`Scene::rasterise`] is a letterbox inverse, and a + /// letterbox inverse is exactly the kind of code that produces confident, + /// plausible, wrong answers. Testing it needs weights whose correct + /// destination is known in advance, which a real inference can never give. + #[cfg(test)] + fn from_weights(names: Vec>, weight: Vec, gw: usize, gh: usize) -> Self { + // A square source, so the letterbox is the identity and any offset + // this finds is the mapping's own rather than the padding's. + let window = Window { + x: 0.0, + y: 0.0, + w: INPUT_EDGE as f32, + h: INPUT_EDGE as f32, + }; + Self { + names, + weight, + grid_width: gw, + grid_height: gh, + letterbox: Letterbox::fit(window.w, window.h), + window, + } + } +} + /// Read `models/scene/categories.txt`, resolving class names to indices. /// /// Hand-written rather than generated, unlike the `.classes.json` beside it, @@ -477,6 +507,98 @@ mod tests { assert!(cats.iter().any(|c| &*c.name == "vegetation")); } + /// Weight put in the top half of the grid must come back in the top half + /// of the image. + /// + /// The one property that makes a category mask worth anything: if the + /// model says "sky up here" and `rasterise` puts it down there, every + /// grade lands on the wrong half of the photograph and nothing about the + /// numbers looks wrong. A vertical split is the cheapest arrangement that + /// catches a flipped axis, and a flipped axis is the mistake this code is + /// actually prone to. + #[test] + fn rasterise_keeps_weight_on_the_side_it_came_from() { + let (gw, gh) = (8usize, 8usize); + let mut weight = vec![0.0f32; gw * gh]; + for y in 0..gh / 2 { + for x in 0..gw { + weight[y * gw + x] = 1.0; + } + } + let scene = Scene::from_weights(vec![Arc::from("sky")], weight, gw, gh); + + let (w, h) = (64usize, 64usize); + let mask = scene.rasterise(0, w, h).expect("category 0 exists"); + + let mean = |y0: usize, y1: usize| { + let band: f32 = (y0..y1) + .flat_map(|y| (0..w).map(move |x| (y, x))) + .map(|(y, x)| mask[y * w + x]) + .sum(); + band / ((y1 - y0) * w) as f32 + }; + let top = mean(0, h / 4); + let bottom = mean(3 * h / 4, h); + assert!( + top > 0.9, + "the half that had the weight should keep it: {top}" + ); + assert!( + bottom < 0.1, + "the half that had none should stay empty: {bottom}" + ); + } + + /// A horizontal split too, because a transpose passes the vertical test. + /// + /// Swapping x and y maps a top band onto a left band, and the check above + /// would still see the top band full. Two axes is what makes the pair + /// meaningful; either alone is not. + #[test] + fn rasterise_does_not_transpose() { + let (gw, gh) = (8usize, 8usize); + let mut weight = vec![0.0f32; gw * gh]; + for y in 0..gh { + for x in 0..gw / 2 { + weight[y * gw + x] = 1.0; + } + } + let scene = Scene::from_weights(vec![Arc::from("left")], weight, gw, gh); + + let (w, h) = (64usize, 64usize); + let mask = scene.rasterise(0, w, h).expect("category 0 exists"); + let mean = |x0: usize, x1: usize| { + let band: f32 = (0..h) + .flat_map(|y| (x0..x1).map(move |x| (y, x))) + .map(|(y, x)| mask[y * w + x]) + .sum(); + band / (h * (x1 - x0)) as f32 + }; + assert!(mean(0, w / 4) > 0.9, "left stays left"); + assert!(mean(3 * w / 4, w) < 0.1, "right stays empty"); + } + + /// Coverage is the fraction of the frame, not a count or a sum. + /// + /// The scene tab hides a category below half a percent, so a coverage that + /// is off by a factor of the grid size would either hide everything or + /// hide nothing, and both look like the model failing rather than the + /// arithmetic. + #[test] + fn coverage_is_a_fraction_of_the_frame() { + let (gw, gh) = (10usize, 10usize); + let mut weight = vec![0.0f32; gw * gh]; + for cell in weight.iter_mut().take(25) { + *cell = 1.0; + } + let scene = Scene::from_weights(vec![Arc::from("quarter")], weight, gw, gh); + let c = scene.coverage(0); + assert!( + (c - 0.25).abs() < 1e-6, + "a quarter of the cells is 0.25, got {c}" + ); + } + /// Softmax then group: the reported weights must never exceed one, and /// must equal one exactly when the categories name every class. #[test] diff --git a/ui/dr-ui/Cargo.toml b/ui/dr-ui/Cargo.toml index 03db558..8d2aded 100644 --- a/ui/dr-ui/Cargo.toml +++ b/ui/dr-ui/Cargo.toml @@ -114,6 +114,15 @@ slint-build.workspace = true serde_norway.workspace = true [features] +# Compile the scene model into the binary. +# +# On by default for the desktop app and *off* for Android, which unpacks the +# same graph from APK assets instead — 24 MB of constant is worth avoiding in a +# mobile install and not worth the plumbing to avoid on a desktop one. Without +# it `segmentation::scene_categories` falls back to the installed-file lookup, +# which is what a packaged desktop build uses too. +scene-model = ["dr-segment/embedded-scene-model"] + default = [] # Debug convenience: re-read style.yaml at startup so a palette can be tuned # without rebuilding. Costs the constant-folding of every token, so it stays diff --git a/ui/dr-ui/src/develop.rs b/ui/dr-ui/src/develop.rs index cd4bf25..7ba76ae 100644 --- a/ui/dr-ui/src/develop.rs +++ b/ui/dr-ui/src/develop.rs @@ -1795,6 +1795,23 @@ impl DevelopSession { MaskSource::Subject { index, .. } => { mix(1); mix(*index as u64); + if layer.morphology.is_compound() { + mix(match layer.morphology { + Morphology::Close => 2, + Morphology::Open => 3, + _ => 0, + }); + mix(layer.morph_radius.to_bits() as u64); + } + } + // Hashed by *name*, and `4` rather than `1` so a category + // named the same as an instance index could never collide with + // it. The field has to be rebuilt when either changes. + MaskSource::Category { name, .. } => { + mix(4); + for b in name.as_bytes() { + mix(*b as u64); + } // Only the compound operations change the field itself. if layer.morphology.is_compound() { mix(match layer.morphology { @@ -1830,6 +1847,25 @@ impl DevelopSession { let mut fields: Vec> = Vec::new(); for layer in self.graph.masks().active() { let field = match &layer.source { + MaskSource::Category { name, .. } => seg + .category_mask(name) + .map(|coverage| { + dr_segment::Shaped::build( + coverage, + pw, + ph, + 128, + morphology_for(layer.morphology), + layer.morph_radius * pw.min(ph) as f32, + ) + .distance + }) + // Full size, never `unwrap_or_default`: an empty vec is a + // wrong-sized field, `SubjectMasks::upload` rejects the + // whole batch on one, and every other layer in the stack + // then loses its mask too. One stale name should cost one + // layer, not all of them. + .unwrap_or_else(|| vec![-1.0; pw * ph]), MaskSource::Subject { index, .. } => seg .instance_mask(*index as usize) .map(|coverage| { @@ -1845,7 +1881,7 @@ impl DevelopSession { ) .distance }) - .unwrap_or_default(), + .unwrap_or_else(|| vec![-1.0; pw * ph]), // A placeholder of the right size, so the slot indices line up // with `active()` whatever mix of sources the stack holds. _ => vec![-1.0; pw * ph], @@ -2594,6 +2630,54 @@ impl DevelopSession { self.add_subject_mask(index) } + /// Add a layer covering one photographic category. + /// + /// The counterpart to [`Self::add_subject_mask`], and it takes a *name* + /// rather than an index for the reason `MaskSource::Category` stores one: + /// the descriptor grouping ADE20K's classes is editable, so an index would + /// silently repoint every stored layer the first time a category was + /// added to it. + /// + /// The layer is named after the category, because "sky" is a better name + /// for a layer than "Mask 3" and the user can rename it anyway. + pub fn add_category_mask(&mut self, name: &str) -> Option { + use dr_pipeline::mask::MaskSource; + + let seg = self.segmentation.as_ref()?; + let signature = seg.signature(); + // Refuse a category this run did not produce rather than creating a + // layer that renders empty: an empty mask looks like a broken + // adjustment, where a button that does nothing at least says so. + if seg.category_mask(name).is_none() { + return None; + } + + let id = self.graph.masks().next_id(); + let mut layer = MaskLayer::new( + id.clone(), + MaskSource::Category { + signature, + name: name.to_string(), + }, + ); + layer.name = name.to_string(); + if !self.graph.masks_mut().push(layer) { + return None; + } + self.active_masks = vec![id.clone()]; + self.history + .record(&self.graph, Edit::Action(labels::step::MASK_ADDED)); + Some(id) + } + + /// What the scene model found in this frame, largest category first. + /// + /// Empty when no scene model was available, which is an ordinary state — + /// see `segmentation::scene_categories`. + pub fn categories(&self) -> &[segmentation::CategorySummary] { + self.segmentation.as_ref().map_or(&[], |s| s.categories()) + } + /// Add a layer selecting one detected subject. /// /// The instance's own coverage is the mask, rather than the watershed @@ -2768,7 +2852,9 @@ impl DevelopSession { self.graph.masks().get(id).is_some_and(|l| { matches!( l.source, - MaskSource::Subject { .. } | MaskSource::Regions { .. } + MaskSource::Subject { .. } + | MaskSource::Category { .. } + | MaskSource::Regions { .. } ) }) } diff --git a/ui/dr-ui/src/masks_ui.rs b/ui/dr-ui/src/masks_ui.rs index 2553d05..152b822 100644 --- a/ui/dr-ui/src/masks_ui.rs +++ b/ui/dr-ui/src/masks_ui.rs @@ -30,7 +30,7 @@ use slint::{ComponentHandle as _, Model as _, ModelRc, VecModel}; use crate::develop::{Abandon, DevelopSession, RefinedInstance, Segmented, SessionId}; use crate::segmentation; -use crate::{sync_rows, AppWindow, GradientHandle, MaskRow, ParamRow, SubjectRow}; +use crate::{sync_rows, AppWindow, CategoryRow, GradientHandle, MaskRow, ParamRow, SubjectRow}; /// What the adjust panel's heading says when the controls are global. /// @@ -146,12 +146,28 @@ impl Running { } } +/// A descriptor name as a list label. +/// +/// Only the first letter, because the names in `models/scene/categories.txt` +/// are already the words a photographer would use — "sky", "vegetation" — and +/// a lookup table would be a second place to edit every time one is added. +/// Title case on a multi-word name would be wrong anyway: "Swimming pool", not +/// "Swimming Pool". +fn category_label(name: &str) -> String { + let mut chars = name.chars(); + match chars.next() { + Some(first) => first.to_uppercase().collect::() + chars.as_str(), + None => String::new(), + } +} + /// Push every mask-related property from the session into the window. pub(crate) fn sync(window: &AppWindow, session: &Rc>>) { let slot = session.borrow(); let Some(s) = slot.as_ref() else { window.set_mask_rows(ModelRc::new(VecModel::::default())); window.set_subject_rows(ModelRc::new(VecModel::::default())); + window.set_category_rows(ModelRc::new(VecModel::::default())); window.set_segmented(false); window.set_adjust_scope(GLOBAL_SCOPE.into()); clear_handles(window); @@ -193,6 +209,19 @@ pub(crate) fn sync(window: &AppWindow, session: &Rc = s + .categories() + .iter() + .map(|c| CategoryRow { + name: c.name.as_ref().into(), + label: category_label(&c.name).into(), + coverage: c.coverage, + }) + .collect(); + window.set_category_rows(ModelRc::new(VecModel::from(categories))); + window.set_segmented(s.has_segmentation()); window.set_adjust_scope(scope_label(s).into()); sync_handles(window, s); @@ -636,6 +665,22 @@ pub(crate) fn wire( redraw(&w); }); } + { + let weak = window.as_weak(); + let session = session.clone(); + let redraw = redraw.clone(); + let rows = rows.clone(); + window.on_add_category_mask(move |name| { + let Some(w) = weak.upgrade() else { return }; + if let Some(s) = session.borrow_mut().as_mut() { + s.add_category_mask(&name); + } + sync(&w, &session); + sync_rows(&w, &rows, &session); + redraw(&w); + }); + } + { let weak = window.as_weak(); let session = session.clone(); @@ -796,6 +841,7 @@ pub(crate) fn reset(window: &AppWindow) { window.set_segmented(false); window.set_mask_rows(ModelRc::new(VecModel::::default())); window.set_subject_rows(ModelRc::new(VecModel::::default())); + window.set_category_rows(ModelRc::new(VecModel::::default())); window.set_adjust_scope(GLOBAL_SCOPE.into()); clear_handles(window); } diff --git a/ui/dr-ui/src/segmentation.rs b/ui/dr-ui/src/segmentation.rs index cb19c1d..013238d 100644 --- a/ui/dr-ui/src/segmentation.rs +++ b/ui/dr-ui/src/segmentation.rs @@ -36,6 +36,23 @@ use dr_gpu::GpuContext; use dr_pipeline::mask::segmentation_signature; use dr_types::Orientation; +/// One photographic category, over the whole frame. +/// +/// The counterpart to [`InstanceSummary`] and deliberately thinner: a category +/// has no box, because it is not one object in one place — sky is wherever the +/// sky is, in as many disconnected pieces as the frame has windows. +#[derive(Debug, Clone)] +pub struct CategorySummary { + pub name: std::sync::Arc, + /// Fraction of the frame this category covers, for ordering the list and + /// for hiding a category that would give the user a control that does + /// nothing. + pub coverage: f32, + /// Coverage at proxy resolution, quantised to a byte — the same + /// representation, and for the same reasons, as `InstanceSummary::mask`. + pub mask: Vec, +} + /// One recognised object. #[derive(Debug, Clone)] pub struct InstanceSummary { @@ -61,6 +78,14 @@ pub struct InstanceSummary { /// One image's recognised objects, ready to mask. pub struct Segmentation { instances: Vec, + /// What the scene model made of the same frame, empty when no scene model + /// could be found. + /// + /// Empty is an ordinary state, not a failure: a build without the weights + /// compiled in and without them installed simply offers no categories, the + /// same way a missing face model turns face indexing off rather than + /// stopping the app. + categories: Vec, /// Identifies this run, so a stored layer can tell whether the index it /// holds still means what it meant. signature: u64, @@ -90,6 +115,21 @@ impl Segmentation { self.instances.get(index).map(|i| i.mask.as_slice()) } + pub fn categories(&self) -> &[CategorySummary] { + &self.categories + } + + /// One category's coverage, at [`Self::proxy_size`]. + /// + /// By name, matching `MaskSource::Category`. A linear scan because there + /// are eight of them and a map would be more machinery than lookup. + pub fn category_mask(&self, name: &str) -> Option<&[u8]> { + self.categories + .iter() + .find(|c| &*c.name == name) + .map(|c| c.mask.as_slice()) + } + /// Replace one instance in place, keeping every other index and the /// signature unchanged. /// @@ -309,8 +349,21 @@ pub fn compute( ^ orientation_key(orientation), ); + // The scene pass, on the same upright frame and laid back down the same + // way. Failures here are logged and dropped rather than propagated: no + // scene model is an ordinary state, and a photograph that can be masked by + // subject should not become unopenable because the categories are absent. + let categories = match scene_categories(&stood_up, uw, uh, orientation) { + Ok(c) => c, + Err(e) => { + log::info!("no scene categories for this frame: {e}"); + Vec::new() + } + }; + Ok(Segmentation { instances, + categories, signature, // **Sensor space, not the model's.** `lay_down` put every mask back, // so the grid a stored layer indexes into is the one it always was — @@ -475,6 +528,86 @@ fn detect( .map_err(|e| e.to_string()) } +/// Weigh the photographic categories, if this build can find a scene model. +/// +/// Separate from [`detect`] rather than folded into it because the two are +/// independent: a build with no scene model still segments subjects, and a +/// frame with no recognisable subject still has sky. Neither failure should +/// take the other down. +fn scene_categories( + rgb: &[f32], + width: usize, + height: usize, + orientation: Orientation, +) -> Result, String> { + let mut model = load_scene_model()?; + let scene = model + .analyse(rgb, width, height) + .map_err(|e| e.to_string())?; + + let mut out = Vec::new(); + for (index, name) in scene.categories().iter().enumerate() { + let coverage = scene.coverage(index); + // A category the model barely saw is not worth a mask buffer the size + // of the proxy, and offering it in the list would be offering a + // control that does nothing when moved. The threshold is the one the + // example prints against. + if coverage < 0.005 { + continue; + } + let Some(weights) = scene.rasterise(index, width, height) else { + continue; + }; + // Back into sensor space, exactly as an instance mask is: the grid a + // stored layer indexes into has to be the sensor's whatever the model + // was shown. `lay_down` wants a box too, so it gets the whole frame — + // a category has no meaningful extent. + let (mask, _) = lay_down( + &weights, + (0.0, 0.0, width as f32, height as f32), + width, + height, + orientation, + ); + out.push(CategorySummary { + name: name.clone(), + coverage, + mask: quantise(&mask), + }); + } + + // Largest first, which is the order the list is worth reading in. + out.sort_by(|a, b| b.coverage.total_cmp(&a.coverage)); + Ok(out) +} + +/// Find a scene model: compiled in if this build has one, installed otherwise. +/// +/// The order matters. A build with the weights compiled in should not be +/// silently overridden by a stale file in a data directory, and a build +/// without them has nothing to fall back *from* — so "embedded, then +/// installed" is the only ordering that is not surprising either way. +fn load_scene_model() -> Result { + #[cfg(feature = "scene-model")] + { + return dr_segment::SceneModel::embedded().map_err(|e| e.to_string()); + } + #[cfg(not(feature = "scene-model"))] + { + // Account-independent, like `shared_face_models_dir` and for the same + // reason: this runs on a worker with no session in hand. Android + // unpacks the APK's copy to exactly this directory before any store + // opens. + let dir = crate::library::shared_face_models_dir(); + dr_segment::SceneModel::from_path( + dir.join("yolo26s-sem-ade20k.onnx"), + dir.join("yolo26s-sem-ade20k.classes.json"), + dir.join("categories.txt"), + ) + .map_err(|e| e.to_string()) + } +} + /// The model's soft coverage, to a byte per pixel. /// /// Rounded rather than truncated, so a coverage of exactly 0.5 lands on the @@ -519,6 +652,7 @@ mod tests { bbox: (4.0, 0.0, 8.0, h as f32), }, ], + categories: Vec::new(), signature: 1, proxy: (w, h), } @@ -555,6 +689,7 @@ mod tests { fn a_click_on_nothing_selects_nothing() { let seg = Segmentation { instances: Vec::new(), + categories: Vec::new(), signature: 1, proxy: (4, 4), }; @@ -582,6 +717,7 @@ mod tests { fn the_overlay_is_transparent_where_nothing_was_found() { let seg = Segmentation { instances: Vec::new(), + categories: Vec::new(), signature: 1, proxy: (4, 4), }; diff --git a/ui/dr-ui/ui/app.slint b/ui/dr-ui/ui/app.slint index 8cdd4bf..11da7ec 100644 --- a/ui/dr-ui/ui/app.slint +++ b/ui/dr-ui/ui/app.slint @@ -1,6 +1,6 @@ import { Theme } from "theme.slint"; import { AdjustPanel, GeometryPanel, GroupStrip, ParamRow, TransferPanel, ViewMode } from "adjust.slint"; -import { GradientHandle, GradientHandles, HandleRole, MaskPanel, MaskRow, SubjectRow } from "masks.slint"; +import { CategoryRow, GradientHandle, GradientHandles, HandleRole, MaskPanel, MaskRow, SubjectRow } from "masks.slint"; import { SpotHandle, SpotHandles, SpotPanel, SpotRole } from "spots.slint"; import { CropOverlay } from "crop.slint"; import { HistoryPanel, HistoryRow } from "history.slint"; @@ -985,6 +985,7 @@ export component AppWindow inherits Window { in property <[MaskRow]> mask-rows; in property <[SubjectRow]> subject-rows; + in property <[CategoryRow]> category-rows; in property segmented: false; in property segmenting: false; in property refining: false; @@ -1068,6 +1069,7 @@ export component AppWindow inherits Window { callback mask-morph-radius-changed(string, float); callback add-gradient-mask(bool); callback add-subject-mask(int); + callback add-category-mask(string); /// Whether the develop column — image info, geometry, adjust — is shown. /// @@ -2510,6 +2512,7 @@ in property panel-visible: true; enabled: root.adjust-enabled; masks: root.mask-rows; subjects: root.subject-rows; + categories: root.category-rows; segmented: root.segmented; segmenting: root.segmenting; refining: root.refining; @@ -2543,6 +2546,7 @@ in property panel-visible: true; } add-gradient(radial) => { root.add-gradient-mask(radial); } add-subject(i) => { root.add-subject-mask(i); } + add-category(n) => { root.add-category-mask(n); } } if root.local-mode: Rectangle { diff --git a/ui/dr-ui/ui/masks.slint b/ui/dr-ui/ui/masks.slint index f3bf94f..6c97449 100644 --- a/ui/dr-ui/ui/masks.slint +++ b/ui/dr-ui/ui/masks.slint @@ -70,6 +70,18 @@ export struct SubjectRow { score: float, } +/// One photographic category the scene model weighed. +export struct CategoryRow { + /// The descriptor's name — "sky", "vegetation". Carried rather than an + /// index because the descriptor is editable and an index would repoint. + name: string, + label: string, + /// 0..1 of the frame. Shown for the same reason a subject's score is: it + /// tells the photographer whether the category is worth reaching for + /// before they spend a click finding out. + coverage: float, +} + component MaskEntry inherits Rectangle { in property data; in property enabled: true; @@ -270,6 +282,7 @@ export component MaskPanel inherits Rectangle { in property enabled: true; in property <[MaskRow]> masks; in property <[SubjectRow]> subjects; + in property <[CategoryRow]> categories; /// A region map has been computed for this image. in property segmented: false; @@ -302,6 +315,7 @@ export component MaskPanel inherits Rectangle { callback add-gradient(bool); callback add-subject(int); + callback add-category(string); background: Theme.surface; @@ -450,6 +464,55 @@ export component MaskPanel inherits Rectangle { } } + // --- whole categories --------------------------------------------- + // + // Below the subjects rather than above, and the order is the argument: + // clicking the photograph is how a local adjustment usually starts, so + // the things that were *found* come first. Categories are the move you + // reach for deliberately — grade the sky, not this one bird. + if root.enabled && root.segmented && root.categories.length > 0: Caption { + text: "Or grade a whole category. One adjustment covers every pixel of it."; + wrap: word-wrap; + } + + if root.enabled && root.segmented: VerticalLayout { + spacing: 0px; + for category in root.categories: category-row := TouchArea { + height: Theme.touch-target; + mouse-cursor: pointer; + clicked => { root.add-category(category.name); } + + HorizontalLayout { + spacing: Theme.gap-sm; + + Label { + text: category.label; + emphasised: category-row.has-hover; + horizontal-stretch: 1; + overflow: elide; + // Same layout-time reason as the subject row above: + // a Text asks for its whole string's width whether or + // not it elides, and the develop column takes the + // widest minimum any panel declares. + max-width: 160px; + vertical-alignment: center; + } + + Label { + // Coverage as a percentage, which is the unit the + // number means something in. A category under half a + // percent never reaches this list at all. + // A plain Label is already `ink-dim`; only + // `emphasised` lifts it, and a coverage readout is + // exactly the thing that should not compete with the + // name beside it. + text: round(category.coverage * 100) + "%"; + vertical-alignment: center; + } + } + } + } + if root.enabled && root.segmented && root.subjects.length == 0: Caption { text: "Nothing recognised. The model knows people, animals and vehicles — " + "a landscape has no subject for it to find. Add a gradient instead.";