dtourolle/DarkRoom
1
0
Fork 0
Code Issues 39 Pull Requests Actions Packages Projects Releases 20 Wiki Activity

Merge: mask a whole category, not just one instance

Browse Source 
Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>

# Conflicts:
#	apps/darkroom-desktop/Cargo.toml
#	docs/traceability.md
This commit is contained in:
dtourolle
2026-08-30 16:37:31 +02:00
parent 327b8c7839 df06240b2d
commit 328fda6f7c
12 changed files with 604 additions and 15 deletions
Download Patch File Download Diff File Expand all files Collapse all files
apps/darkroom-desktop/Cargo.toml
+1 -1
View File
@@ -6,7 +6,7 @@ rust-version.workspace = true
license.workspace = true
[dependencies]
dr-ui.workspace = true
dr-ui = { workspace = true, features = ["scene-model"] }
# For the panic hook and the log sink, directly rather than through dr-ui:
# both have to be installed before `dr_ui::run`, because a panic during startup
# is exactly the one they exist to catch and record (NFR-OPS-1, NFR-OPS-2).
core/dr-gpu/src/mask.rs
+21 -7
View File
@@ -579,13 +579,21 @@ impl MaskPass {
// fields are built per layer, in this same order, because two
// layers over one subject can carry different morphology.
let subject = match &layer.source {
MaskSource::Subject { .. } => match subjects.filter(|s| slot < s.len()) {
Some(s) => (s, slot),
None => {
log::warn!("mask layer {} has no distance field; skipping", layer.id);
continue;
// Category alongside Subject: both are model coverage turned
// into a distance field, both are built per layer in this same
// order, and leaving a category out of here is precisely the
// failure the comment above warns about — it binds the 1x1
// placeholder, so the mask covers everything and the
// adjustment silently goes global.
MaskSource::Subject { .. } | MaskSource::Category { .. } => {
match subjects.filter(|s| slot < s.len()) {
Some(s) => (s, slot),
None => {
log::warn!("mask layer {} has no distance field; skipping", layer.id);
continue;
}
}
},
}
_ => (&self.empty_subject, 0),
};
@@ -654,7 +662,13 @@ impl MaskPass {
// edge; the field is in proxy pixels. Converting here keeps the
// stored edit resolution-independent while the shader works in the
// units its texture is actually measured in.
MaskSource::Subject { .. } => {
// A category shares the subject's mode, and that is not a
// shortcut: both arrive as a soft coverage buffer at proxy
// resolution and both are turned into a distance field before they
// reach here. The shader has no way to tell them apart and no
// reason to want one — what differs is only which model produced
// the coverage.
MaskSource::Subject { .. } | MaskSource::Category { .. } => {
let short = field_short_edge(width, height);
MaskParams {
mode: MODE_SUBJECT,
core/dr-pipeline/src/mask.rs
+36 -3
View File
@@ -532,6 +532,38 @@ pub enum MaskSource {
score: f32,
},
/// Every pixel of one photographic category, from the scene model.
///
/// The counterpart to [`Self::Subject`], and the difference is the whole
/// reason both exist. A subject is *one* instance — this dog, not that one
/// — found by a COCO-trained instance model. A category is *all* the sky,
/// or all the foliage, from an ADE20K-trained semantic model that has no
/// notion of instances at all (docs/segmentation.md §16).
///
/// So this is what a global grade attaches to: lift the sky, desaturate
/// the vegetation, warm the architecture. Asking it for "that person
/// rather than the other two" is a category error — the model merged them
/// before the mask ever existed, and [`Self::Subject`] is the source for
/// that question.
///
/// Stored as identity like a subject, and for the same reason: the
/// coverage is megabytes and is reproducible from the same model over the
/// same image.
Category {
/// Which segmentation run produced it, so a layer can tell whether
/// the name below still refers to something that was computed.
signature: u64,
/// The category name from `models/scene/categories.txt` — "sky",
/// "vegetation".
///
/// A name rather than an index because the descriptor is editable: a
/// category added to it would silently renumber every layer stored
/// against an index, and the failure would be a mask quietly grading
/// the wrong thing. A name that no longer exists is simply not found,
/// and the layer reads as stale.
name: String,
},
/// A linear gradient — the graduated-filter mask.
///
/// Geometry is in **normalised source coordinates**, so it survives a crop,
@@ -596,6 +628,7 @@ impl MaskSource {
match self {
Self::Regions { .. } => "regions",
Self::Subject { .. } => "subject",
Self::Category { .. } => "category",
Self::Linear { .. } => "linear",
Self::Radial { .. } => "radial",
Self::Brush { .. } => "brush",
@@ -825,9 +858,9 @@ impl MaskLayer {
/// should offer to recompute rather than render it.
pub fn is_stale(&self, current: u64) -> bool {
match self.source {
MaskSource::Regions { signature, .. } | MaskSource::Subject { signature, .. } => {
signature != current
}
MaskSource::Regions { signature, .. }
| MaskSource::Subject { signature, .. }
| MaskSource::Category { signature, .. } => signature != current,
// A gradient is geometry in normalised coordinates. It means the
// same thing whatever was or was not detected, so nothing about a
// new run can invalidate it. Painted strokes are the same: they are
core/dr-pipeline/src/sidecar.rs
+15
View File
@@ -971,6 +971,10 @@ fn write_mask(out: &mut String, version: &str, layer: &MaskLayer) {
let _ = writeln!(out, "class = {class}");
let _ = writeln!(out, "score = {}", format_value(*score));
}
MaskSource::Category { signature, name } => {
let _ = writeln!(out, "signature = {signature}");
let _ = writeln!(out, "category = {name}");
}
MaskSource::Linear {
centre,
angle,
@@ -1124,6 +1128,7 @@ struct PartialMask {
ids: Vec<u32>,
index: u32,
class: String,
category: String,
score: f32,
centre: (f32, f32),
radii: (f32, f32),
@@ -1154,6 +1159,7 @@ impl PartialMask {
level: 0,
ids: Vec::new(),
index: 0,
category: String::new(),
class: String::new(),
score: 0.0,
centre: (0.5, 0.5),
@@ -1191,6 +1197,11 @@ impl PartialMask {
self.ids.sort_unstable();
self.ids.dedup();
}
// A category's own key rather than reusing `class`: both name a
// thing the mask covers, but one is a COCO instance's label and
// the other an entry in the scene descriptor, and a file that
// conflated them would round-trip a subject into a category.
"category" => self.category = value.to_string(),
"index" => self.index = value.parse().unwrap_or(0),
"class" => self.class = value.to_string(),
"score" => self.score = value.parse().unwrap_or(0.0),
@@ -1257,6 +1268,10 @@ impl PartialMask {
class: self.class,
score: self.score,
},
"category" => MaskSource::Category {
signature: self.signature,
name: self.category,
},
"linear" => MaskSource::Linear {
centre: self.centre,
angle: self.angle,
core/dr-pipeline/tests/mask_sidecar.rs
+61
View File
@@ -616,3 +616,64 @@ fn show_a_sidecar() {
println!("\n{}", sidecar.to_text());
}
/// A category mask is a name plus a signature, and both have to survive.
///
/// The name especially. `MaskSource::Category` stores one rather than an index
/// precisely so that editing `models/scene/categories.txt` cannot repoint a
/// stored layer — and that reasoning is worth nothing if the name is what the
/// sidecar drops.
#[test]
fn category_masks_keep_their_name() {
let mut graph = EditGraph::default_chain();
let mut sky = MaskLayer::new(
"m1",
MaskSource::Category {
signature: 0x5EED_1234,
name: "sky".into(),
},
);
sky.name = "sky".into();
sky.feather = 0.03;
sky.set_param("exposure", ParamId("exposure"), 0.5);
graph.masks_mut().push(sky);
// A multi-word name too: the writer emits `category = swimming pool` on one
// line, and a reader that split on whitespace would silently truncate it to
// a category no model has.
let mut pool = MaskLayer::new(
"m2",
MaskSource::Category {
signature: 0x5EED_1234,
name: "swimming pool".into(),
},
);
pool.set_param("saturation", ParamId("saturation"), -30.0);
graph.masks_mut().push(pool);
let restored = round_trip(&graph);
let layers = restored.masks().layers();
assert_eq!(layers.len(), 2);
assert_eq!(layers[0].source, graph.masks().layers()[0].source);
assert_eq!(layers[1].source, graph.masks().layers()[1].source);
assert!((layers[0].feather - 0.03).abs() < 1e-6);
}
/// A category is stale when its run is, exactly as a subject is.
///
/// The coverage buffer lives in the session, not the sidecar, so a layer whose
/// signature no longer matches is pointing at pixels that were never computed.
/// Rendering it anyway would mask nothing and read as a broken adjustment.
#[test]
fn a_category_from_another_run_is_stale() {
let layer = MaskLayer::new(
"m1",
MaskSource::Category {
signature: 1,
name: "sky".into(),
},
);
assert!(layer.is_stale(2), "a different run must invalidate it");
assert!(!layer.is_stale(1), "the run it was built against must not");
}
core/dr-segment/src/scene.rs
+122
View File
@@ -57,6 +57,8 @@ use std::sync::Arc;
use ndarray::ArrayView3;
#[cfg(test)]
use crate::semantic::INPUT_EDGE;
use crate::semantic::{install_backend, Letterbox, Window};
use crate::SegmentError;
@@ -353,6 +355,34 @@ impl Scene {
}
}
impl Scene {
/// Build a scene from known weights, for tests.
///
/// The geometry in [`Scene::rasterise`] is a letterbox inverse, and a
/// letterbox inverse is exactly the kind of code that produces confident,
/// plausible, wrong answers. Testing it needs weights whose correct
/// destination is known in advance, which a real inference can never give.
#[cfg(test)]
fn from_weights(names: Vec<Arc<str>>, weight: Vec<f32>, gw: usize, gh: usize) -> Self {
// A square source, so the letterbox is the identity and any offset
// this finds is the mapping's own rather than the padding's.
let window = Window {
x: 0.0,
y: 0.0,
w: INPUT_EDGE as f32,
h: INPUT_EDGE as f32,
};
Self {
names,
weight,
grid_width: gw,
grid_height: gh,
letterbox: Letterbox::fit(window.w, window.h),
window,
}
}
}
/// Read `models/scene/categories.txt`, resolving class names to indices.
///
/// Hand-written rather than generated, unlike the `.classes.json` beside it,
@@ -477,6 +507,98 @@ mod tests {
assert!(cats.iter().any(|c| &*c.name == "vegetation"));
}
/// Weight put in the top half of the grid must come back in the top half
/// of the image.
///
/// The one property that makes a category mask worth anything: if the
/// model says "sky up here" and `rasterise` puts it down there, every
/// grade lands on the wrong half of the photograph and nothing about the
/// numbers looks wrong. A vertical split is the cheapest arrangement that
/// catches a flipped axis, and a flipped axis is the mistake this code is
/// actually prone to.
#[test]
fn rasterise_keeps_weight_on_the_side_it_came_from() {
let (gw, gh) = (8usize, 8usize);
let mut weight = vec![0.0f32; gw * gh];
for y in 0..gh / 2 {
for x in 0..gw {
weight[y * gw + x] = 1.0;
}
}
let scene = Scene::from_weights(vec![Arc::from("sky")], weight, gw, gh);
let (w, h) = (64usize, 64usize);
let mask = scene.rasterise(0, w, h).expect("category 0 exists");
let mean = |y0: usize, y1: usize| {
let band: f32 = (y0..y1)
.flat_map(|y| (0..w).map(move |x| (y, x)))
.map(|(y, x)| mask[y * w + x])
.sum();
band / ((y1 - y0) * w) as f32
};
let top = mean(0, h / 4);
let bottom = mean(3 * h / 4, h);
assert!(
top > 0.9,
"the half that had the weight should keep it: {top}"
);
assert!(
bottom < 0.1,
"the half that had none should stay empty: {bottom}"
);
}
/// A horizontal split too, because a transpose passes the vertical test.
///
/// Swapping x and y maps a top band onto a left band, and the check above
/// would still see the top band full. Two axes is what makes the pair
/// meaningful; either alone is not.
#[test]
fn rasterise_does_not_transpose() {
let (gw, gh) = (8usize, 8usize);
let mut weight = vec![0.0f32; gw * gh];
for y in 0..gh {
for x in 0..gw / 2 {
weight[y * gw + x] = 1.0;
}
}
let scene = Scene::from_weights(vec![Arc::from("left")], weight, gw, gh);
let (w, h) = (64usize, 64usize);
let mask = scene.rasterise(0, w, h).expect("category 0 exists");
let mean = |x0: usize, x1: usize| {
let band: f32 = (0..h)
.flat_map(|y| (x0..x1).map(move |x| (y, x)))
.map(|(y, x)| mask[y * w + x])
.sum();
band / (h * (x1 - x0)) as f32
};
assert!(mean(0, w / 4) > 0.9, "left stays left");
assert!(mean(3 * w / 4, w) < 0.1, "right stays empty");
}
/// Coverage is the fraction of the frame, not a count or a sum.
///
/// The scene tab hides a category below half a percent, so a coverage that
/// is off by a factor of the grid size would either hide everything or
/// hide nothing, and both look like the model failing rather than the
/// arithmetic.
#[test]
fn coverage_is_a_fraction_of_the_frame() {
let (gw, gh) = (10usize, 10usize);
let mut weight = vec![0.0f32; gw * gh];
for cell in weight.iter_mut().take(25) {
*cell = 1.0;
}
let scene = Scene::from_weights(vec![Arc::from("quarter")], weight, gw, gh);
let c = scene.coverage(0);
assert!(
(c - 0.25).abs() < 1e-6,
"a quarter of the cells is 0.25, got {c}"
);
}
/// Softmax then group: the reported weights must never exceed one, and
/// must equal one exactly when the categories name every class.
#[test]
ui/dr-ui/Cargo.toml
+9
View File
@@ -114,6 +114,15 @@ slint-build.workspace = true
serde_norway.workspace = true
[features]
# Compile the scene model into the binary.
#
# On by default for the desktop app and *off* for Android, which unpacks the
# same graph from APK assets instead — 24 MB of constant is worth avoiding in a
# mobile install and not worth the plumbing to avoid on a desktop one. Without
# it `segmentation::scene_categories` falls back to the installed-file lookup,
# which is what a packaged desktop build uses too.
scene-model = ["dr-segment/embedded-scene-model"]
default = []
# Debug convenience: re-read style.yaml at startup so a palette can be tuned
# without rebuilding. Costs the constant-folding of every token, so it stays
ui/dr-ui/src/develop.rs
+88 -2
View File
@@ -1795,6 +1795,23 @@ impl DevelopSession {
MaskSource::Subject { index, .. } => {
mix(1);
mix(*index as u64);
if layer.morphology.is_compound() {
mix(match layer.morphology {
Morphology::Close => 2,
Morphology::Open => 3,
_ => 0,
});
mix(layer.morph_radius.to_bits() as u64);
}
}
// Hashed by *name*, and `4` rather than `1` so a category
// named the same as an instance index could never collide with
// it. The field has to be rebuilt when either changes.
MaskSource::Category { name, .. } => {
mix(4);
for b in name.as_bytes() {
mix(*b as u64);
}
// Only the compound operations change the field itself.
if layer.morphology.is_compound() {
mix(match layer.morphology {
@@ -1830,6 +1847,25 @@ impl DevelopSession {
let mut fields: Vec<Vec<f32>> = Vec::new();
for layer in self.graph.masks().active() {
let field = match &layer.source {
MaskSource::Category { name, .. } => seg
.category_mask(name)
.map(|coverage| {
dr_segment::Shaped::build(
coverage,
pw,
ph,
128,
morphology_for(layer.morphology),
layer.morph_radius * pw.min(ph) as f32,
)
.distance
})
// Full size, never `unwrap_or_default`: an empty vec is a
// wrong-sized field, `SubjectMasks::upload` rejects the
// whole batch on one, and every other layer in the stack
// then loses its mask too. One stale name should cost one
// layer, not all of them.
.unwrap_or_else(|| vec![-1.0; pw * ph]),
MaskSource::Subject { index, .. } => seg
.instance_mask(*index as usize)
.map(|coverage| {
@@ -1845,7 +1881,7 @@ impl DevelopSession {
)
.distance
})
.unwrap_or_default(),
.unwrap_or_else(|| vec![-1.0; pw * ph]),
// A placeholder of the right size, so the slot indices line up
// with `active()` whatever mix of sources the stack holds.
_ => vec![-1.0; pw * ph],
@@ -2594,6 +2630,54 @@ impl DevelopSession {
self.add_subject_mask(index)
}
/// Add a layer covering one photographic category.
///
/// The counterpart to [`Self::add_subject_mask`], and it takes a *name*
/// rather than an index for the reason `MaskSource::Category` stores one:
/// the descriptor grouping ADE20K's classes is editable, so an index would
/// silently repoint every stored layer the first time a category was
/// added to it.
///
/// The layer is named after the category, because "sky" is a better name
/// for a layer than "Mask 3" and the user can rename it anyway.
pub fn add_category_mask(&mut self, name: &str) -> Option<String> {
use dr_pipeline::mask::MaskSource;
let seg = self.segmentation.as_ref()?;
let signature = seg.signature();
// Refuse a category this run did not produce rather than creating a
// layer that renders empty: an empty mask looks like a broken
// adjustment, where a button that does nothing at least says so.
if seg.category_mask(name).is_none() {
return None;
}
let id = self.graph.masks().next_id();
let mut layer = MaskLayer::new(
id.clone(),
MaskSource::Category {
signature,
name: name.to_string(),
},
);
layer.name = name.to_string();
if !self.graph.masks_mut().push(layer) {
return None;
}
self.active_masks = vec![id.clone()];
self.history
.record(&self.graph, Edit::Action(labels::step::MASK_ADDED));
Some(id)
}
/// What the scene model found in this frame, largest category first.
///
/// Empty when no scene model was available, which is an ordinary state —
/// see `segmentation::scene_categories`.
pub fn categories(&self) -> &[segmentation::CategorySummary] {
self.segmentation.as_ref().map_or(&[], |s| s.categories())
}
/// Add a layer selecting one detected subject.
///
/// The instance's own coverage is the mask, rather than the watershed
@@ -2768,7 +2852,9 @@ impl DevelopSession {
self.graph.masks().get(id).is_some_and(|l| {
matches!(
l.source,
MaskSource::Subject { .. } | MaskSource::Regions { .. }
MaskSource::Subject { .. }
| MaskSource::Category { .. }
| MaskSource::Regions { .. }
)
})
}
ui/dr-ui/src/masks_ui.rs
+47 -1
View File
@@ -30,7 +30,7 @@ use slint::{ComponentHandle as _, Model as _, ModelRc, VecModel};
use crate::develop::{Abandon, DevelopSession, RefinedInstance, Segmented, SessionId};
use crate::segmentation;
use crate::{sync_rows, AppWindow, GradientHandle, MaskRow, ParamRow, SubjectRow};
use crate::{sync_rows, AppWindow, CategoryRow, GradientHandle, MaskRow, ParamRow, SubjectRow};
/// What the adjust panel's heading says when the controls are global.
///
@@ -146,12 +146,28 @@ impl Running {
}
}
/// A descriptor name as a list label.
///
/// Only the first letter, because the names in `models/scene/categories.txt`
/// are already the words a photographer would use — "sky", "vegetation" — and
/// a lookup table would be a second place to edit every time one is added.
/// Title case on a multi-word name would be wrong anyway: "Swimming pool", not
/// "Swimming Pool".
fn category_label(name: &str) -> String {
let mut chars = name.chars();
match chars.next() {
Some(first) => first.to_uppercase().collect::<String>() + chars.as_str(),
None => String::new(),
}
}
/// Push every mask-related property from the session into the window.
pub(crate) fn sync(window: &AppWindow, session: &Rc<RefCell<Option<DevelopSession>>>) {
let slot = session.borrow();
let Some(s) = slot.as_ref() else {
window.set_mask_rows(ModelRc::new(VecModel::<MaskRow>::default()));
window.set_subject_rows(ModelRc::new(VecModel::<SubjectRow>::default()));
window.set_category_rows(ModelRc::new(VecModel::<CategoryRow>::default()));
window.set_segmented(false);
window.set_adjust_scope(GLOBAL_SCOPE.into());
clear_handles(window);
@@ -193,6 +209,19 @@ pub(crate) fn sync(window: &AppWindow, session: &Rc<RefCell<Option<DevelopSessio
.collect();
window.set_subject_rows(ModelRc::new(VecModel::from(subjects)));
// Already largest-first from the precompute, and already filtered to the
// ones with enough coverage to be worth a control.
let categories: Vec<CategoryRow> = s
.categories()
.iter()
.map(|c| CategoryRow {
name: c.name.as_ref().into(),
label: category_label(&c.name).into(),
coverage: c.coverage,
})
.collect();
window.set_category_rows(ModelRc::new(VecModel::from(categories)));
window.set_segmented(s.has_segmentation());
window.set_adjust_scope(scope_label(s).into());
sync_handles(window, s);
@@ -636,6 +665,22 @@ pub(crate) fn wire(
redraw(&w);
});
}
{
let weak = window.as_weak();
let session = session.clone();
let redraw = redraw.clone();
let rows = rows.clone();
window.on_add_category_mask(move |name| {
let Some(w) = weak.upgrade() else { return };
if let Some(s) = session.borrow_mut().as_mut() {
s.add_category_mask(&name);
}
sync(&w, &session);
sync_rows(&w, &rows, &session);
redraw(&w);
});
}
{
let weak = window.as_weak();
let session = session.clone();
@@ -796,6 +841,7 @@ pub(crate) fn reset(window: &AppWindow) {
window.set_segmented(false);
window.set_mask_rows(ModelRc::new(VecModel::<MaskRow>::default()));
window.set_subject_rows(ModelRc::new(VecModel::<SubjectRow>::default()));
window.set_category_rows(ModelRc::new(VecModel::<CategoryRow>::default()));
window.set_adjust_scope(GLOBAL_SCOPE.into());
clear_handles(window);
}
ui/dr-ui/src/segmentation.rs
+136
View File
@@ -36,6 +36,23 @@ use dr_gpu::GpuContext;
use dr_pipeline::mask::segmentation_signature;
use dr_types::Orientation;
/// One photographic category, over the whole frame.
///
/// The counterpart to [`InstanceSummary`] and deliberately thinner: a category
/// has no box, because it is not one object in one place — sky is wherever the
/// sky is, in as many disconnected pieces as the frame has windows.
#[derive(Debug, Clone)]
pub struct CategorySummary {
pub name: std::sync::Arc<str>,
/// Fraction of the frame this category covers, for ordering the list and
/// for hiding a category that would give the user a control that does
/// nothing.
pub coverage: f32,
/// Coverage at proxy resolution, quantised to a byte — the same
/// representation, and for the same reasons, as `InstanceSummary::mask`.
pub mask: Vec<u8>,
}
/// One recognised object.
#[derive(Debug, Clone)]
pub struct InstanceSummary {
@@ -61,6 +78,14 @@ pub struct InstanceSummary {
/// One image's recognised objects, ready to mask.
pub struct Segmentation {
instances: Vec<InstanceSummary>,
/// What the scene model made of the same frame, empty when no scene model
/// could be found.
///
/// Empty is an ordinary state, not a failure: a build without the weights
/// compiled in and without them installed simply offers no categories, the
/// same way a missing face model turns face indexing off rather than
/// stopping the app.
categories: Vec<CategorySummary>,
/// Identifies this run, so a stored layer can tell whether the index it
/// holds still means what it meant.
signature: u64,
@@ -90,6 +115,21 @@ impl Segmentation {
self.instances.get(index).map(|i| i.mask.as_slice())
}
pub fn categories(&self) -> &[CategorySummary] {
&self.categories
}
/// One category's coverage, at [`Self::proxy_size`].
///
/// By name, matching `MaskSource::Category`. A linear scan because there
/// are eight of them and a map would be more machinery than lookup.
pub fn category_mask(&self, name: &str) -> Option<&[u8]> {
self.categories
.iter()
.find(|c| &*c.name == name)
.map(|c| c.mask.as_slice())
}
/// Replace one instance in place, keeping every other index and the
/// signature unchanged.
///
@@ -309,8 +349,21 @@ pub fn compute(
^ orientation_key(orientation),
);
// The scene pass, on the same upright frame and laid back down the same
// way. Failures here are logged and dropped rather than propagated: no
// scene model is an ordinary state, and a photograph that can be masked by
// subject should not become unopenable because the categories are absent.
let categories = match scene_categories(&stood_up, uw, uh, orientation) {
Ok(c) => c,
Err(e) => {
log::info!("no scene categories for this frame: {e}");
Vec::new()
}
};
Ok(Segmentation {
instances,
categories,
signature,
// **Sensor space, not the model's.** `lay_down` put every mask back,
// so the grid a stored layer indexes into is the one it always was —
@@ -475,6 +528,86 @@ fn detect(
.map_err(|e| e.to_string())
}
/// Weigh the photographic categories, if this build can find a scene model.
///
/// Separate from [`detect`] rather than folded into it because the two are
/// independent: a build with no scene model still segments subjects, and a
/// frame with no recognisable subject still has sky. Neither failure should
/// take the other down.
fn scene_categories(
rgb: &[f32],
width: usize,
height: usize,
orientation: Orientation,
) -> Result<Vec<CategorySummary>, String> {
let mut model = load_scene_model()?;
let scene = model
.analyse(rgb, width, height)
.map_err(|e| e.to_string())?;
let mut out = Vec::new();
for (index, name) in scene.categories().iter().enumerate() {
let coverage = scene.coverage(index);
// A category the model barely saw is not worth a mask buffer the size
// of the proxy, and offering it in the list would be offering a
// control that does nothing when moved. The threshold is the one the
// example prints against.
if coverage < 0.005 {
continue;
}
let Some(weights) = scene.rasterise(index, width, height) else {
continue;
};
// Back into sensor space, exactly as an instance mask is: the grid a
// stored layer indexes into has to be the sensor's whatever the model
// was shown. `lay_down` wants a box too, so it gets the whole frame —
// a category has no meaningful extent.
let (mask, _) = lay_down(
&weights,
(0.0, 0.0, width as f32, height as f32),
width,
height,
orientation,
);
out.push(CategorySummary {
name: name.clone(),
coverage,
mask: quantise(&mask),
});
}
// Largest first, which is the order the list is worth reading in.
out.sort_by(|a, b| b.coverage.total_cmp(&a.coverage));
Ok(out)
}
/// Find a scene model: compiled in if this build has one, installed otherwise.
///
/// The order matters. A build with the weights compiled in should not be
/// silently overridden by a stale file in a data directory, and a build
/// without them has nothing to fall back *from* — so "embedded, then
/// installed" is the only ordering that is not surprising either way.
fn load_scene_model() -> Result<dr_segment::SceneModel, String> {
#[cfg(feature = "scene-model")]
{
return dr_segment::SceneModel::embedded().map_err(|e| e.to_string());
}
#[cfg(not(feature = "scene-model"))]
{
// Account-independent, like `shared_face_models_dir` and for the same
// reason: this runs on a worker with no session in hand. Android
// unpacks the APK's copy to exactly this directory before any store
// opens.
let dir = crate::library::shared_face_models_dir();
dr_segment::SceneModel::from_path(
dir.join("yolo26s-sem-ade20k.onnx"),
dir.join("yolo26s-sem-ade20k.classes.json"),
dir.join("categories.txt"),
)
.map_err(|e| e.to_string())
}
}
/// The model's soft coverage, to a byte per pixel.
///
/// Rounded rather than truncated, so a coverage of exactly 0.5 lands on the
@@ -519,6 +652,7 @@ mod tests {
bbox: (4.0, 0.0, 8.0, h as f32),
},
],
categories: Vec::new(),
signature: 1,
proxy: (w, h),
}
@@ -555,6 +689,7 @@ mod tests {
fn a_click_on_nothing_selects_nothing() {
let seg = Segmentation {
instances: Vec::new(),
categories: Vec::new(),
signature: 1,
proxy: (4, 4),
};
@@ -582,6 +717,7 @@ mod tests {
fn the_overlay_is_transparent_where_nothing_was_found() {
let seg = Segmentation {
instances: Vec::new(),
categories: Vec::new(),
signature: 1,
proxy: (4, 4),
};
ui/dr-ui/ui/app.slint
+5 -1
View File
@@ -1,6 +1,6 @@
import { Theme } from "theme.slint";
import { AdjustPanel, GeometryPanel, GroupStrip, ParamRow, TransferPanel, ViewMode } from "adjust.slint";
import { GradientHandle, GradientHandles, HandleRole, MaskPanel, MaskRow, SubjectRow } from "masks.slint";
import { CategoryRow, GradientHandle, GradientHandles, HandleRole, MaskPanel, MaskRow, SubjectRow } from "masks.slint";
import { SpotHandle, SpotHandles, SpotPanel, SpotRole } from "spots.slint";
import { CropOverlay } from "crop.slint";
import { HistoryPanel, HistoryRow } from "history.slint";
@@ -985,6 +985,7 @@ export component AppWindow inherits Window {
in property <[MaskRow]> mask-rows;
in property <[SubjectRow]> subject-rows;
in property <[CategoryRow]> category-rows;
in property <bool> segmented: false;
in property <bool> segmenting: false;
in property <bool> refining: false;
@@ -1068,6 +1069,7 @@ export component AppWindow inherits Window {
callback mask-morph-radius-changed(string, float);
callback add-gradient-mask(bool);
callback add-subject-mask(int);
callback add-category-mask(string);
/// Whether the develop column — image info, geometry, adjust — is shown.
///
@@ -2510,6 +2512,7 @@ in property <bool> panel-visible: true;
enabled: root.adjust-enabled;
masks: root.mask-rows;
subjects: root.subject-rows;
categories: root.category-rows;
segmented: root.segmented;
segmenting: root.segmenting;
refining: root.refining;
@@ -2543,6 +2546,7 @@ in property <bool> panel-visible: true;
}
add-gradient(radial) => { root.add-gradient-mask(radial); }
add-subject(i) => { root.add-subject-mask(i); }
add-category(n) => { root.add-category-mask(n); }
}
if root.local-mode: Rectangle {
ui/dr-ui/ui/masks.slint
+63
View File
@@ -70,6 +70,18 @@ export struct SubjectRow {
score: float,
}
/// One photographic category the scene model weighed.
export struct CategoryRow {
/// The descriptor's name — "sky", "vegetation". Carried rather than an
/// index because the descriptor is editable and an index would repoint.
name: string,
label: string,
/// 0..1 of the frame. Shown for the same reason a subject's score is: it
/// tells the photographer whether the category is worth reaching for
/// before they spend a click finding out.
coverage: float,
}
component MaskEntry inherits Rectangle {
in property <MaskRow> data;
in property <bool> enabled: true;
@@ -270,6 +282,7 @@ export component MaskPanel inherits Rectangle {
in property <bool> enabled: true;
in property <[MaskRow]> masks;
in property <[SubjectRow]> subjects;
in property <[CategoryRow]> categories;
/// A region map has been computed for this image.
in property <bool> segmented: false;
@@ -302,6 +315,7 @@ export component MaskPanel inherits Rectangle {
callback add-gradient(bool);
callback add-subject(int);
callback add-category(string);
background: Theme.surface;
@@ -450,6 +464,55 @@ export component MaskPanel inherits Rectangle {
}
}
// --- whole categories ---------------------------------------------
//
// Below the subjects rather than above, and the order is the argument:
// clicking the photograph is how a local adjustment usually starts, so
// the things that were *found* come first. Categories are the move you
// reach for deliberately — grade the sky, not this one bird.
if root.enabled && root.segmented && root.categories.length > 0: Caption {
text: "Or grade a whole category. One adjustment covers every pixel of it.";
wrap: word-wrap;
}
if root.enabled && root.segmented: VerticalLayout {
spacing: 0px;
for category in root.categories: category-row := TouchArea {
height: Theme.touch-target;
mouse-cursor: pointer;
clicked => { root.add-category(category.name); }
HorizontalLayout {
spacing: Theme.gap-sm;
Label {
text: category.label;
emphasised: category-row.has-hover;
horizontal-stretch: 1;
overflow: elide;
// Same layout-time reason as the subject row above:
// a Text asks for its whole string's width whether or
// not it elides, and the develop column takes the
// widest minimum any panel declares.
max-width: 160px;
vertical-alignment: center;
}
Label {
// Coverage as a percentage, which is the unit the
// number means something in. A category under half a
// percent never reaches this list at all.
// A plain Label is already `ink-dim`; only
// `emphasised` lifts it, and a coverage readout is
// exactly the thing that should not compete with the
// name beside it.
text: round(category.coverage * 100) + "%";
vertical-alignment: center;
}
}
}
}
if root.enabled && root.segmented && root.subjects.length == 0: Caption {
text: "Nothing recognised. The model knows people, animals and vehicles — "
+ "a landscape has no subject for it to find. Add a gradient instead.";