Merge: mask a whole category, not just one instance

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>

# Conflicts:
#	apps/darkroom-desktop/Cargo.toml
#	docs/traceability.md
This commit is contained in:
2026-08-30 16:37:31 +02:00
12 changed files with 604 additions and 15 deletions
+9
View File
@@ -114,6 +114,15 @@ slint-build.workspace = true
serde_norway.workspace = true
[features]
# Compile the scene model into the binary.
#
# On by default for the desktop app and *off* for Android, which unpacks the
# same graph from APK assets instead — 24 MB of constant is worth avoiding in a
# mobile install and not worth the plumbing to avoid on a desktop one. Without
# it `segmentation::scene_categories` falls back to the installed-file lookup,
# which is what a packaged desktop build uses too.
scene-model = ["dr-segment/embedded-scene-model"]
default = []
# Debug convenience: re-read style.yaml at startup so a palette can be tuned
# without rebuilding. Costs the constant-folding of every token, so it stays
+88 -2
View File
@@ -1795,6 +1795,23 @@ impl DevelopSession {
MaskSource::Subject { index, .. } => {
mix(1);
mix(*index as u64);
if layer.morphology.is_compound() {
mix(match layer.morphology {
Morphology::Close => 2,
Morphology::Open => 3,
_ => 0,
});
mix(layer.morph_radius.to_bits() as u64);
}
}
// Hashed by *name*, and `4` rather than `1` so a category
// named the same as an instance index could never collide with
// it. The field has to be rebuilt when either changes.
MaskSource::Category { name, .. } => {
mix(4);
for b in name.as_bytes() {
mix(*b as u64);
}
// Only the compound operations change the field itself.
if layer.morphology.is_compound() {
mix(match layer.morphology {
@@ -1830,6 +1847,25 @@ impl DevelopSession {
let mut fields: Vec<Vec<f32>> = Vec::new();
for layer in self.graph.masks().active() {
let field = match &layer.source {
MaskSource::Category { name, .. } => seg
.category_mask(name)
.map(|coverage| {
dr_segment::Shaped::build(
coverage,
pw,
ph,
128,
morphology_for(layer.morphology),
layer.morph_radius * pw.min(ph) as f32,
)
.distance
})
// Full size, never `unwrap_or_default`: an empty vec is a
// wrong-sized field, `SubjectMasks::upload` rejects the
// whole batch on one, and every other layer in the stack
// then loses its mask too. One stale name should cost one
// layer, not all of them.
.unwrap_or_else(|| vec![-1.0; pw * ph]),
MaskSource::Subject { index, .. } => seg
.instance_mask(*index as usize)
.map(|coverage| {
@@ -1845,7 +1881,7 @@ impl DevelopSession {
)
.distance
})
.unwrap_or_default(),
.unwrap_or_else(|| vec![-1.0; pw * ph]),
// A placeholder of the right size, so the slot indices line up
// with `active()` whatever mix of sources the stack holds.
_ => vec![-1.0; pw * ph],
@@ -2594,6 +2630,54 @@ impl DevelopSession {
self.add_subject_mask(index)
}
/// Add a layer covering one photographic category.
///
/// The counterpart to [`Self::add_subject_mask`], and it takes a *name*
/// rather than an index for the reason `MaskSource::Category` stores one:
/// the descriptor grouping ADE20K's classes is editable, so an index would
/// silently repoint every stored layer the first time a category was
/// added to it.
///
/// The layer is named after the category, because "sky" is a better name
/// for a layer than "Mask 3" and the user can rename it anyway.
pub fn add_category_mask(&mut self, name: &str) -> Option<String> {
use dr_pipeline::mask::MaskSource;
let seg = self.segmentation.as_ref()?;
let signature = seg.signature();
// Refuse a category this run did not produce rather than creating a
// layer that renders empty: an empty mask looks like a broken
// adjustment, where a button that does nothing at least says so.
if seg.category_mask(name).is_none() {
return None;
}
let id = self.graph.masks().next_id();
let mut layer = MaskLayer::new(
id.clone(),
MaskSource::Category {
signature,
name: name.to_string(),
},
);
layer.name = name.to_string();
if !self.graph.masks_mut().push(layer) {
return None;
}
self.active_masks = vec![id.clone()];
self.history
.record(&self.graph, Edit::Action(labels::step::MASK_ADDED));
Some(id)
}
/// What the scene model found in this frame, largest category first.
///
/// Empty when no scene model was available, which is an ordinary state —
/// see `segmentation::scene_categories`.
pub fn categories(&self) -> &[segmentation::CategorySummary] {
self.segmentation.as_ref().map_or(&[], |s| s.categories())
}
/// Add a layer selecting one detected subject.
///
/// The instance's own coverage is the mask, rather than the watershed
@@ -2768,7 +2852,9 @@ impl DevelopSession {
self.graph.masks().get(id).is_some_and(|l| {
matches!(
l.source,
MaskSource::Subject { .. } | MaskSource::Regions { .. }
MaskSource::Subject { .. }
| MaskSource::Category { .. }
| MaskSource::Regions { .. }
)
})
}
+47 -1
View File
@@ -30,7 +30,7 @@ use slint::{ComponentHandle as _, Model as _, ModelRc, VecModel};
use crate::develop::{Abandon, DevelopSession, RefinedInstance, Segmented, SessionId};
use crate::segmentation;
use crate::{sync_rows, AppWindow, GradientHandle, MaskRow, ParamRow, SubjectRow};
use crate::{sync_rows, AppWindow, CategoryRow, GradientHandle, MaskRow, ParamRow, SubjectRow};
/// What the adjust panel's heading says when the controls are global.
///
@@ -146,12 +146,28 @@ impl Running {
}
}
/// A descriptor name as a list label.
///
/// Only the first letter, because the names in `models/scene/categories.txt`
/// are already the words a photographer would use — "sky", "vegetation" — and
/// a lookup table would be a second place to edit every time one is added.
/// Title case on a multi-word name would be wrong anyway: "Swimming pool", not
/// "Swimming Pool".
fn category_label(name: &str) -> String {
let mut chars = name.chars();
match chars.next() {
Some(first) => first.to_uppercase().collect::<String>() + chars.as_str(),
None => String::new(),
}
}
/// Push every mask-related property from the session into the window.
pub(crate) fn sync(window: &AppWindow, session: &Rc<RefCell<Option<DevelopSession>>>) {
let slot = session.borrow();
let Some(s) = slot.as_ref() else {
window.set_mask_rows(ModelRc::new(VecModel::<MaskRow>::default()));
window.set_subject_rows(ModelRc::new(VecModel::<SubjectRow>::default()));
window.set_category_rows(ModelRc::new(VecModel::<CategoryRow>::default()));
window.set_segmented(false);
window.set_adjust_scope(GLOBAL_SCOPE.into());
clear_handles(window);
@@ -193,6 +209,19 @@ pub(crate) fn sync(window: &AppWindow, session: &Rc<RefCell<Option<DevelopSessio
.collect();
window.set_subject_rows(ModelRc::new(VecModel::from(subjects)));
// Already largest-first from the precompute, and already filtered to the
// ones with enough coverage to be worth a control.
let categories: Vec<CategoryRow> = s
.categories()
.iter()
.map(|c| CategoryRow {
name: c.name.as_ref().into(),
label: category_label(&c.name).into(),
coverage: c.coverage,
})
.collect();
window.set_category_rows(ModelRc::new(VecModel::from(categories)));
window.set_segmented(s.has_segmentation());
window.set_adjust_scope(scope_label(s).into());
sync_handles(window, s);
@@ -636,6 +665,22 @@ pub(crate) fn wire(
redraw(&w);
});
}
{
let weak = window.as_weak();
let session = session.clone();
let redraw = redraw.clone();
let rows = rows.clone();
window.on_add_category_mask(move |name| {
let Some(w) = weak.upgrade() else { return };
if let Some(s) = session.borrow_mut().as_mut() {
s.add_category_mask(&name);
}
sync(&w, &session);
sync_rows(&w, &rows, &session);
redraw(&w);
});
}
{
let weak = window.as_weak();
let session = session.clone();
@@ -796,6 +841,7 @@ pub(crate) fn reset(window: &AppWindow) {
window.set_segmented(false);
window.set_mask_rows(ModelRc::new(VecModel::<MaskRow>::default()));
window.set_subject_rows(ModelRc::new(VecModel::<SubjectRow>::default()));
window.set_category_rows(ModelRc::new(VecModel::<CategoryRow>::default()));
window.set_adjust_scope(GLOBAL_SCOPE.into());
clear_handles(window);
}
+136
View File
@@ -36,6 +36,23 @@ use dr_gpu::GpuContext;
use dr_pipeline::mask::segmentation_signature;
use dr_types::Orientation;
/// One photographic category, over the whole frame.
///
/// The counterpart to [`InstanceSummary`] and deliberately thinner: a category
/// has no box, because it is not one object in one place — sky is wherever the
/// sky is, in as many disconnected pieces as the frame has windows.
#[derive(Debug, Clone)]
pub struct CategorySummary {
pub name: std::sync::Arc<str>,
/// Fraction of the frame this category covers, for ordering the list and
/// for hiding a category that would give the user a control that does
/// nothing.
pub coverage: f32,
/// Coverage at proxy resolution, quantised to a byte — the same
/// representation, and for the same reasons, as `InstanceSummary::mask`.
pub mask: Vec<u8>,
}
/// One recognised object.
#[derive(Debug, Clone)]
pub struct InstanceSummary {
@@ -61,6 +78,14 @@ pub struct InstanceSummary {
/// One image's recognised objects, ready to mask.
pub struct Segmentation {
instances: Vec<InstanceSummary>,
/// What the scene model made of the same frame, empty when no scene model
/// could be found.
///
/// Empty is an ordinary state, not a failure: a build without the weights
/// compiled in and without them installed simply offers no categories, the
/// same way a missing face model turns face indexing off rather than
/// stopping the app.
categories: Vec<CategorySummary>,
/// Identifies this run, so a stored layer can tell whether the index it
/// holds still means what it meant.
signature: u64,
@@ -90,6 +115,21 @@ impl Segmentation {
self.instances.get(index).map(|i| i.mask.as_slice())
}
pub fn categories(&self) -> &[CategorySummary] {
&self.categories
}
/// One category's coverage, at [`Self::proxy_size`].
///
/// By name, matching `MaskSource::Category`. A linear scan because there
/// are eight of them and a map would be more machinery than lookup.
pub fn category_mask(&self, name: &str) -> Option<&[u8]> {
self.categories
.iter()
.find(|c| &*c.name == name)
.map(|c| c.mask.as_slice())
}
/// Replace one instance in place, keeping every other index and the
/// signature unchanged.
///
@@ -309,8 +349,21 @@ pub fn compute(
^ orientation_key(orientation),
);
// The scene pass, on the same upright frame and laid back down the same
// way. Failures here are logged and dropped rather than propagated: no
// scene model is an ordinary state, and a photograph that can be masked by
// subject should not become unopenable because the categories are absent.
let categories = match scene_categories(&stood_up, uw, uh, orientation) {
Ok(c) => c,
Err(e) => {
log::info!("no scene categories for this frame: {e}");
Vec::new()
}
};
Ok(Segmentation {
instances,
categories,
signature,
// **Sensor space, not the model's.** `lay_down` put every mask back,
// so the grid a stored layer indexes into is the one it always was —
@@ -475,6 +528,86 @@ fn detect(
.map_err(|e| e.to_string())
}
/// Weigh the photographic categories, if this build can find a scene model.
///
/// Separate from [`detect`] rather than folded into it because the two are
/// independent: a build with no scene model still segments subjects, and a
/// frame with no recognisable subject still has sky. Neither failure should
/// take the other down.
fn scene_categories(
rgb: &[f32],
width: usize,
height: usize,
orientation: Orientation,
) -> Result<Vec<CategorySummary>, String> {
let mut model = load_scene_model()?;
let scene = model
.analyse(rgb, width, height)
.map_err(|e| e.to_string())?;
let mut out = Vec::new();
for (index, name) in scene.categories().iter().enumerate() {
let coverage = scene.coverage(index);
// A category the model barely saw is not worth a mask buffer the size
// of the proxy, and offering it in the list would be offering a
// control that does nothing when moved. The threshold is the one the
// example prints against.
if coverage < 0.005 {
continue;
}
let Some(weights) = scene.rasterise(index, width, height) else {
continue;
};
// Back into sensor space, exactly as an instance mask is: the grid a
// stored layer indexes into has to be the sensor's whatever the model
// was shown. `lay_down` wants a box too, so it gets the whole frame —
// a category has no meaningful extent.
let (mask, _) = lay_down(
&weights,
(0.0, 0.0, width as f32, height as f32),
width,
height,
orientation,
);
out.push(CategorySummary {
name: name.clone(),
coverage,
mask: quantise(&mask),
});
}
// Largest first, which is the order the list is worth reading in.
out.sort_by(|a, b| b.coverage.total_cmp(&a.coverage));
Ok(out)
}
/// Find a scene model: compiled in if this build has one, installed otherwise.
///
/// The order matters. A build with the weights compiled in should not be
/// silently overridden by a stale file in a data directory, and a build
/// without them has nothing to fall back *from* — so "embedded, then
/// installed" is the only ordering that is not surprising either way.
fn load_scene_model() -> Result<dr_segment::SceneModel, String> {
#[cfg(feature = "scene-model")]
{
return dr_segment::SceneModel::embedded().map_err(|e| e.to_string());
}
#[cfg(not(feature = "scene-model"))]
{
// Account-independent, like `shared_face_models_dir` and for the same
// reason: this runs on a worker with no session in hand. Android
// unpacks the APK's copy to exactly this directory before any store
// opens.
let dir = crate::library::shared_face_models_dir();
dr_segment::SceneModel::from_path(
dir.join("yolo26s-sem-ade20k.onnx"),
dir.join("yolo26s-sem-ade20k.classes.json"),
dir.join("categories.txt"),
)
.map_err(|e| e.to_string())
}
}
/// The model's soft coverage, to a byte per pixel.
///
/// Rounded rather than truncated, so a coverage of exactly 0.5 lands on the
@@ -519,6 +652,7 @@ mod tests {
bbox: (4.0, 0.0, 8.0, h as f32),
},
],
categories: Vec::new(),
signature: 1,
proxy: (w, h),
}
@@ -555,6 +689,7 @@ mod tests {
fn a_click_on_nothing_selects_nothing() {
let seg = Segmentation {
instances: Vec::new(),
categories: Vec::new(),
signature: 1,
proxy: (4, 4),
};
@@ -582,6 +717,7 @@ mod tests {
fn the_overlay_is_transparent_where_nothing_was_found() {
let seg = Segmentation {
instances: Vec::new(),
categories: Vec::new(),
signature: 1,
proxy: (4, 4),
};
+5 -1
View File
@@ -1,6 +1,6 @@
import { Theme } from "theme.slint";
import { AdjustPanel, GeometryPanel, GroupStrip, ParamRow, TransferPanel, ViewMode } from "adjust.slint";
import { GradientHandle, GradientHandles, HandleRole, MaskPanel, MaskRow, SubjectRow } from "masks.slint";
import { CategoryRow, GradientHandle, GradientHandles, HandleRole, MaskPanel, MaskRow, SubjectRow } from "masks.slint";
import { SpotHandle, SpotHandles, SpotPanel, SpotRole } from "spots.slint";
import { CropOverlay } from "crop.slint";
import { HistoryPanel, HistoryRow } from "history.slint";
@@ -985,6 +985,7 @@ export component AppWindow inherits Window {
in property <[MaskRow]> mask-rows;
in property <[SubjectRow]> subject-rows;
in property <[CategoryRow]> category-rows;
in property <bool> segmented: false;
in property <bool> segmenting: false;
in property <bool> refining: false;
@@ -1068,6 +1069,7 @@ export component AppWindow inherits Window {
callback mask-morph-radius-changed(string, float);
callback add-gradient-mask(bool);
callback add-subject-mask(int);
callback add-category-mask(string);
/// Whether the develop column — image info, geometry, adjust — is shown.
///
@@ -2510,6 +2512,7 @@ in property <bool> panel-visible: true;
enabled: root.adjust-enabled;
masks: root.mask-rows;
subjects: root.subject-rows;
categories: root.category-rows;
segmented: root.segmented;
segmenting: root.segmenting;
refining: root.refining;
@@ -2543,6 +2546,7 @@ in property <bool> panel-visible: true;
}
add-gradient(radial) => { root.add-gradient-mask(radial); }
add-subject(i) => { root.add-subject-mask(i); }
add-category(n) => { root.add-category-mask(n); }
}
if root.local-mode: Rectangle {
+63
View File
@@ -70,6 +70,18 @@ export struct SubjectRow {
score: float,
}
/// One photographic category the scene model weighed.
export struct CategoryRow {
/// The descriptor's name — "sky", "vegetation". Carried rather than an
/// index because the descriptor is editable and an index would repoint.
name: string,
label: string,
/// 0..1 of the frame. Shown for the same reason a subject's score is: it
/// tells the photographer whether the category is worth reaching for
/// before they spend a click finding out.
coverage: float,
}
component MaskEntry inherits Rectangle {
in property <MaskRow> data;
in property <bool> enabled: true;
@@ -270,6 +282,7 @@ export component MaskPanel inherits Rectangle {
in property <bool> enabled: true;
in property <[MaskRow]> masks;
in property <[SubjectRow]> subjects;
in property <[CategoryRow]> categories;
/// A region map has been computed for this image.
in property <bool> segmented: false;
@@ -302,6 +315,7 @@ export component MaskPanel inherits Rectangle {
callback add-gradient(bool);
callback add-subject(int);
callback add-category(string);
background: Theme.surface;
@@ -450,6 +464,55 @@ export component MaskPanel inherits Rectangle {
}
}
// --- whole categories ---------------------------------------------
//
// Below the subjects rather than above, and the order is the argument:
// clicking the photograph is how a local adjustment usually starts, so
// the things that were *found* come first. Categories are the move you
// reach for deliberately — grade the sky, not this one bird.
if root.enabled && root.segmented && root.categories.length > 0: Caption {
text: "Or grade a whole category. One adjustment covers every pixel of it.";
wrap: word-wrap;
}
if root.enabled && root.segmented: VerticalLayout {
spacing: 0px;
for category in root.categories: category-row := TouchArea {
height: Theme.touch-target;
mouse-cursor: pointer;
clicked => { root.add-category(category.name); }
HorizontalLayout {
spacing: Theme.gap-sm;
Label {
text: category.label;
emphasised: category-row.has-hover;
horizontal-stretch: 1;
overflow: elide;
// Same layout-time reason as the subject row above:
// a Text asks for its whole string's width whether or
// not it elides, and the develop column takes the
// widest minimum any panel declares.
max-width: 160px;
vertical-alignment: center;
}
Label {
// Coverage as a percentage, which is the unit the
// number means something in. A category under half a
// percent never reaches this list at all.
// A plain Label is already `ink-dim`; only
// `emphasised` lifts it, and a coverage readout is
// exactly the thing that should not compete with the
// name beside it.
text: round(category.coverage * 100) + "%";
vertical-alignment: center;
}
}
}
}
if root.enabled && root.segmented && root.subjects.length == 0: Caption {
text: "Nothing recognised. The model knows people, animals and vehicles — "
+ "a landscape has no subject for it to find. Add a gradient instead.";