diff --git a/core/dr-gpu/src/merge.rs b/core/dr-gpu/src/merge.rs index 4325068..f0980a9 100644 --- a/core/dr-gpu/src/merge.rs +++ b/core/dr-gpu/src/merge.rs @@ -30,18 +30,27 @@ //! are the caller's to provide and cache — `source` is asked for frame `k` //! as it is needed, and a caller short of memory may demosaic on demand. //! +//! # The blend +//! +//! With a seam map (`dr_pano::seam`), a frame's weight at a pixel is its +//! share of the map about that pixel — whole on its own side of a seam, +//! nothing on the other, and a ramp across a window `seam_blend` pixels +//! wide that follows the seam. Without one, or where the map has nothing +//! to say, the weight is the distance to the frame's edge over `feather`, +//! which hides exposure steps and does not hide parallax: the average draws +//! anything the frames disagree on twice. +//! //! # What is not here yet //! -//! A feathered blend, not seams and a Laplacian pyramid: the weight is the -//! distance to the frame's edge, which hides exposure steps and small -//! misalignments and does not hide parallax. Gain is a scalar per frame -//! the caller supplies. Both are panorama.md §10's step 5, after the path -//! writes a file end to end. +//! A Laplacian pyramid, which would let the seam's blend be narrow for +//! detail and wide for exposure at once. Gain is a scalar per frame the +//! caller supplies. use std::sync::Arc; use dr_pano::bundle::Cameras; use dr_pano::projection::{Bounds, Projection}; +use dr_pano::seam::SeamMap; use wgpu::util::DeviceExt; use crate::readback::await_mapping; @@ -58,7 +67,7 @@ pub struct MergeFrame { } /// The output the merge produces. -#[derive(Debug, Clone, Copy, PartialEq)] +#[derive(Debug, Clone, PartialEq)] pub struct MergeOutput { pub projection: Projection, /// The projection's scale in output pixels: the cylinder's radius, the @@ -69,6 +78,11 @@ pub struct MergeOutput { pub bounds: Bounds, /// Pixels over which a frame's weight ramps up from its edge. pub feather: f32, + /// Which frame each part of the output is taken from, laid out at the + /// proxies' scale; `None` for the feathered average everywhere. + pub seams: Option>, + /// The width, in output pixels, of the blend across a seam. + pub seam_blend: f32, /// Chunk size: the unit of GPU work and of memory. pub chunk: (u32, u32), /// Multiplies a normalised sample (1.0 = white) to the sensor's scale. @@ -115,6 +129,12 @@ struct WarpParams { feather: f32, clip_onset: f32, balance: [f32; 4], + seam_origin: [f32; 2], + seam_size: [u32; 2], + seam_px: f32, + seam_radius: f32, + frame_index: u32, + seam_on: u32, } #[repr(C)] @@ -182,6 +202,16 @@ impl MergePass { count: None, }, storage(2, false), + wgpu::BindGroupLayoutEntry { + binding: 3, + visibility: wgpu::ShaderStages::COMPUTE, + ty: wgpu::BindingType::Texture { + sample_type: wgpu::TextureSampleType::Uint, + view_dimension: wgpu::TextureViewDimension::D2, + multisampled: false, + }, + count: None, + }, ], }); let resolve_layout = @@ -279,6 +309,32 @@ impl MergePass { let mut band_cov = vec![false; (out_w * ch) as usize]; let mut chunk_px: Vec = Vec::new(); + // The seam map, once for the whole output, and where it sits in + // this output's coordinates. A one-texel stand-in when there is + // none, because the binding is not optional. + let (seam_tex, seam_origin, seam_px, seam_radius, seam_size) = match &output.seams { + Some(m) => { + let ((ou, ov), px) = m.at_scale(output.scale); + let radius = m.blend_radius(output.scale, f64::from(output.seam_blend)); + ( + self.label_texture(m.width as u32, m.height as u32, &m.labels), + [ou as f32, ov as f32], + px as f32, + radius as f32, + [m.width as u32, m.height as u32], + ) + } + None => ( + self.label_texture(1, 1, &[dr_pano::seam::NONE]), + [0.0; 2], + 1.0, + 1.0, + [1, 1], + ), + }; + let seam_view = seam_tex.create_view(&Default::default()); + let seam_on = u32::from(output.seams.is_some()); + let mut y = 0u32; while y < out_h { let rows = ch.min(out_h - y); @@ -346,8 +402,14 @@ impl MergePass { output.balance[2].max(1e-3), 0.0, ], + seam_origin, + seam_size, + seam_px, + seam_radius, + frame_index: k as u32, + seam_on, }; - self.accumulate(¶ms, tile); + self.accumulate(¶ms, tile, &seam_view); } self.resolve_chunk((cols, rows), output.sample_scale, &mut chunk_px)?; @@ -385,7 +447,42 @@ impl MergePass { self.ctx.queue.submit(Some(enc.finish())); } - fn accumulate(&mut self, params: &WarpParams, tile: &wgpu::Texture) { + /// The seam map's labels as an `r8uint` texture. + fn label_texture(&self, width: u32, height: u32, labels: &[u8]) -> wgpu::Texture { + let size = wgpu::Extent3d { + width, + height, + depth_or_array_layers: 1, + }; + let tex = self.ctx.device.create_texture(&wgpu::TextureDescriptor { + label: Some("merge-seams"), + size, + mip_level_count: 1, + sample_count: 1, + dimension: wgpu::TextureDimension::D2, + format: wgpu::TextureFormat::R8Uint, + usage: wgpu::TextureUsages::TEXTURE_BINDING | wgpu::TextureUsages::COPY_DST, + view_formats: &[], + }); + self.ctx.queue.write_texture( + wgpu::TexelCopyTextureInfo { + texture: &tex, + mip_level: 0, + origin: wgpu::Origin3d::ZERO, + aspect: wgpu::TextureAspect::All, + }, + labels, + wgpu::TexelCopyBufferLayout { + offset: 0, + bytes_per_row: Some(width), + rows_per_image: Some(height), + }, + size, + ); + tex + } + + fn accumulate(&mut self, params: &WarpParams, tile: &wgpu::Texture, seams: &wgpu::TextureView) { let chunk = (params.chunk_size[0], params.chunk_size[1]); let uniforms = self .ctx @@ -417,6 +514,10 @@ impl MergePass { binding: 2, resource: acc.as_entire_binding(), }, + wgpu::BindGroupEntry { + binding: 3, + resource: wgpu::BindingResource::TextureView(seams), + }, ], }); let mut enc = self.ctx.device.create_command_encoder(&Default::default()); diff --git a/core/dr-gpu/src/shaders/merge.wgsl b/core/dr-gpu/src/shaders/merge.wgsl index 332eaab..5043a79 100644 --- a/core/dr-gpu/src/shaders/merge.wgsl +++ b/core/dr-gpu/src/shaders/merge.wgsl @@ -5,8 +5,9 @@ // pixel it asks which direction that pixel looks along, turns the // direction into the frame's camera, projects it to a source pixel, and // if that pixel is inside the tile that was rendered for this chunk, -// samples it and adds it — weighted by its distance from the frame's edge -// — into the accumulator. `resolve` runs once per chunk after every frame +// samples it and adds it — weighted by the frame's share of the seam map +// there, or by its distance from the frame's edge where there is no map — +// into the accumulator. `resolve` runs once per chunk after every frame // has been added: divides the sums by the weights and packs the result as // sixteen-bit samples at the sensor's scale (FR-MRG-3). // @@ -50,12 +51,83 @@ struct Params { // white balance the composite will be developed with. clip_onset: f32, balance: vec4, + // The seam map (`dr_pano::seam`): where its texel (0, 0)'s corner sits + // in this output's centred coordinates, its size, output pixels per + // texel, the blend's radius in texels, which frame this dispatch is, + // and whether there is a map at all. + seam_origin: vec2, + seam_size: vec2, + seam_px: f32, + seam_radius: f32, + frame_index: u32, + seam_on: u32, }; @group(0) @binding(0) var p: Params; @group(0) @binding(1) var tile: texture_2d; // rgb·w summed, then w: four floats per chunk pixel. @group(0) @binding(2) var acc: array>; +// One frame index per texel, 255 for none. +@group(0) @binding(3) var seams: texture_2d; + +const NO_FRAME: u32 = 255u; + +fn label(i: i32, j: i32) -> u32 { + if (i < 0 || j < 0 || i >= i32(p.seam_size.x) || j >= i32(p.seam_size.y)) { + return NO_FRAME; + } + return textureLoad(seams, vec2(i, j), 0).r; +} + +// This frame's share of the seam map about output point (u, v): the +// tent-weighted fraction of the texels within the radius that it owns, and +// the weight of the texels owned by anyone (zero where the map has nothing +// to say). `SeamMap::share` verbatim. +fn seam_share(u: f32, v: f32) -> vec2 { + let x = (u - p.seam_origin.x) / p.seam_px - 0.5; + let y = (v - p.seam_origin.y) / p.seam_px - 0.5; + let r = max(p.seam_radius, 1.0); + let x0 = i32(ceil(x - r)); + let x1 = i32(floor(x + r)); + let y0 = i32(ceil(y - r)); + let y1 = i32(floor(y + r)); + // Most pixels are nowhere near a seam: if the window's corners, edge + // midpoints and centre agree, so does the window. A seam crossing it + // has to cross its border, between two of those. + let xm = i32(round(x)); + let ym = i32(round(y)); + let c = label(xm, ym); + if (label(x0, y0) == c && label(x1, y0) == c && label(x0, y1) == c && label(x1, y1) == c + && label(xm, y0) == c && label(xm, y1) == c && label(x0, ym) == c && label(x1, ym) == c) { + if (c == NO_FRAME) { + return vec2(0.0, 0.0); + } + return vec2(select(0.0, 1.0, c == p.frame_index), 1.0); + } + var mine = 0.0; + var owned = 0.0; + for (var j = y0; j <= y1; j = j + 1) { + let wy = 1.0 - abs(y - f32(j)) / r; + if (wy <= 0.0) { + continue; + } + for (var i = x0; i <= x1; i = i + 1) { + let wx = 1.0 - abs(x - f32(i)) / r; + let l = label(i, j); + if (wx <= 0.0 || l == NO_FRAME) { + continue; + } + owned = owned + wx * wy; + if (l == p.frame_index) { + mine = mine + wx * wy; + } + } + } + if (owned <= 0.0) { + return vec2(0.0, 0.0); + } + return vec2(mine / owned, 1.0); +} fn to_direction(u: f32, v: f32) -> vec3 { let s = p.proj_scale; @@ -98,7 +170,17 @@ fn warp(@builtin(global_invocation_id) gid: vec3) { if (edge <= 0.0) { return; } - let w = clamp(edge / max(p.feather, 1.0), 0.0, 1.0); + var w = clamp(edge / max(p.feather, 1.0), 0.0, 1.0); + // With seams, the share of the map scales it. The small floor keeps + // the feather underneath as the answer wherever no frame that reaches + // this pixel owns it — the map is coarser than the output, so at the + // frames' outer edges it can name a frame that falls just short. + if (p.seam_on != 0u) { + let s = seam_share(u, v); + if (s.y > 0.0) { + w = w * (s.x + 1e-4); + } + } // Into the tile. let tx = sx - p.tile_origin.x; let ty = sy - p.tile_origin.y; diff --git a/core/dr-pano/src/lib.rs b/core/dr-pano/src/lib.rs index 2871948..afbb63f 100644 --- a/core/dr-pano/src/lib.rs +++ b/core/dr-pano/src/lib.rs @@ -22,6 +22,7 @@ //! - [`align`] — the whole thing, from features to cameras, honest about //! what it could not place. //! - [`projection`] — perspective, cylindrical, spherical. +//! - [`seam`] — which frame each output pixel is taken from. //! - [`linalg`] — the small dense algebra all of it uses. //! //! # What it depends on @@ -43,6 +44,7 @@ pub mod matching; #[cfg(feature = "xfeat")] pub mod migan; pub mod projection; +pub mod seam; #[cfg(feature = "xfeat")] pub mod xfeat; @@ -52,6 +54,7 @@ pub use features::{Features, Keypoint}; pub use fill::{fill_border, Inpainter, Observer, Params as FillParams}; pub use image::Gray; pub use projection::Projection; +pub use seam::{SeamMap, SeamOptions}; #[derive(Debug, thiserror::Error)] pub enum PanoError { diff --git a/core/dr-pano/src/seam.rs b/core/dr-pano/src/seam.rs new file mode 100644 index 0000000..73c3fe6 --- /dev/null +++ b/core/dr-pano/src/seam.rs @@ -0,0 +1,691 @@ +//! TRACES: FR-MRG-10 +//! Where each frame gives way to the next. +//! +//! The first merges averaged every overlap: each frame weighted by its +//! distance from its own edge, so that across two hundred pixels one frame +//! faded into the other. That hides an exposure step and does not hide +//! anything that differs between the frames — parallax on a near slope, a +//! walker, a branch in the wind — which the average draws twice, half as +//! bright, a soft double edge at 1:1. +//! +//! A seam answers it the way every stitcher does: in an overlap, each output +//! pixel is taken from *one* frame, and the line where the choice changes is +//! put where the frames agree and the picture is smooth — through sky, +//! along a shadow, round the walker rather than through him — and away from +//! either frame's edge, where vignetting and the lens correction's fringe +//! live. The blend is then narrow and only across that line. +//! +//! # How +//! +//! At proxy resolution, on the output surface, which fits (panorama.md §5: +//! "it is a mask, not an image"): +//! +//! 1. Frames are laid down one at a time, each next to one already placed. +//! The composite so far is a label per texel and the value its owner saw. +//! 2. Where a new frame overlaps the composite, a cost per texel: the +//! difference between the two (after the gains), how much detail either +//! has there, and how near either frame's edge it is — smoothed over a +//! few texels, because "agree" means locally, not at one pixel. +//! 3. The cut is a path across the overlap, perpendicular to the line from +//! the composite's frames to the new one, found by dynamic programming +//! one row at a time: the per-column seam panorama.md §4 chose over a +//! graph cut because it is the GPU-friendly shape. Texels on the new +//! frame's side of the path become its own. +//! +//! What the merge reads is [`SeamMap::share`]: the fraction of a small +//! window about a point that is labelled with a frame, tent-weighted, which +//! is a narrow blend that follows the seam. `merge.wgsl` computes the same +//! thing on the GPU from the same labels. + +use crate::bundle::Cameras; +use crate::image::Gray; +use crate::projection::{self, Projection}; + +/// No frame owns this texel. +pub const NONE: u8 = 255; + +/// The most frames a map can label: one less than [`NONE`]. +pub const MAX_FRAMES: usize = NONE as usize; + +/// Which frame each texel of the output takes its pixels from. +#[derive(Debug, Clone, PartialEq)] +pub struct SeamMap { + pub width: usize, + pub height: usize, + /// The projection scale the map was laid out at: the proxies' focal + /// length. Output coordinates at any other scale are this times the + /// ratio of the scales. + pub scale: f64, + /// Centred output coordinates, at `scale`, of texel (0, 0)'s top-left + /// corner. + pub origin: (f64, f64), + /// Output units per texel, at `scale`. + pub px: f64, + /// Row-major, one per texel: the frame's index, or [`NONE`]. + pub labels: Vec, +} + +#[derive(Debug, Clone, Copy, PartialEq)] +pub struct SeamOptions { + /// The widest the map is laid out, in texels. Wider than the proxies' + /// own resolution buys nothing. + pub max_width: usize, + /// How much detail costs against disagreement: a seam through texture + /// shows even where the frames agree, because the blend across it + /// softens it. + pub detail: f32, + /// How much a frame's edge costs, and how far in from it the cost + /// reaches, in proxy pixels. Frame edges are where vignetting is + /// darkest and the lens correction ran out of sensor. + pub edge: f32, + pub edge_margin: f32, + /// The radius, in texels, a texel's cost looks about it for the worst + /// of its neighbours: at least the radius the merge blends across. + pub smoothing: usize, +} + +impl Default for SeamOptions { + fn default() -> Self { + SeamOptions { + max_width: 2048, + detail: 0.5, + edge: 0.5, + edge_margin: 24.0, + smoothing: 4, + } + } +} + +/// The most texels a blend reaches either side of a seam. The merge's +/// shader loads the square of twice this per pixel per frame near a seam. +pub const MAX_BLEND_RADIUS: f64 = 4.0; + +/// Cost of a texel outside the overlap: high enough that the path keeps to +/// the overlap wherever there is one, finite so that a row with a gap in it +/// still has an answer. +const OUTSIDE: f32 = 1.0e3; + +impl SeamMap { + /// The map's origin and texel size in the coordinates of an output + /// laid out at `scale` (the full-resolution focal length, or a fraction + /// of it). + pub fn at_scale(&self, scale: f64) -> ((f64, f64), f64) { + let r = scale / self.scale; + ((self.origin.0 * r, self.origin.1 * r), self.px * r) + } + + /// The radius, in texels, of a blend `blend_px` output pixels wide in an + /// output laid out at `scale`: what [`Self::share`] and the shader are + /// given, so that the preview and the merge blend alike. + pub fn blend_radius(&self, scale: f64, blend_px: f64) -> f64 { + let (_, px) = self.at_scale(scale); + (blend_px / 2.0 / px).clamp(1.0, MAX_BLEND_RADIUS) + } + + /// The share frame `k` has of output point `(u, v)` given at `scale`: + /// the tent-weighted fraction of the texels within `radius` (in texels) + /// that it owns. `None` where no texel in reach is owned at all — the + /// map has nothing to say there, and the caller falls back to its + /// feather. + /// + /// This is the function `merge.wgsl`'s `seam_share` repeats; the two + /// must agree. + pub fn share(&self, k: usize, u: f64, v: f64, scale: f64, radius: f64) -> Option { + let ((ou, ov), px) = self.at_scale(scale); + let x = (u - ou) / px - 0.5; + let y = (v - ov) / px - 0.5; + let r = radius.max(1.0); + let (x0, x1) = ((x - r).ceil() as i64, (x + r).floor() as i64); + let (y0, y1) = ((y - r).ceil() as i64, (y + r).floor() as i64); + let (mut mine, mut all) = (0.0f64, 0.0f64); + for j in y0.max(0)..=y1.min(self.height as i64 - 1) { + let wy = 1.0 - (y - j as f64).abs() / r; + if wy <= 0.0 { + continue; + } + for i in x0.max(0)..=x1.min(self.width as i64 - 1) { + let wx = 1.0 - (x - i as f64).abs() / r; + if wx <= 0.0 { + continue; + } + let l = self.labels[j as usize * self.width + i as usize]; + if l == NONE { + continue; + } + all += wx * wy; + if usize::from(l) == k { + mine += wx * wy; + } + } + } + (all > 0.0).then(|| (mine / all) as f32) + } +} + +/// One frame warped onto the map: its gain-corrected value and its distance +/// from its own edge (in proxy pixels) per texel, NaN where it does not +/// reach. +struct Warped { + value: Vec, + edge: Vec, +} + +/// Lay seams across the overlaps of `proxies`, aligned by `cameras` (at the +/// proxies' scale), with `gains` the linear multipliers the merge will +/// apply. `None` if the frames project nowhere or there are more than +/// [`MAX_FRAMES`]. +pub fn find( + proxies: &[&Gray], + cameras: &Cameras, + gains: &[f32], + projection: Projection, + opts: &SeamOptions, +) -> Option { + let n = proxies.len(); + if n == 0 || n > MAX_FRAMES || cameras.rotations.len() != n || gains.len() != n { + return None; + } + let (fw, fh) = (proxies[0].width as f64, proxies[0].height as f64); + let scale = cameras.focal; + let bounds = projection::bounds(projection, scale, cameras, (fw, fh))?; + let width = opts.max_width.min(bounds.width().ceil() as usize).max(1); + let px = bounds.width() / width as f64; + let height = ((bounds.height() / px).ceil() as usize).max(1); + let mut map = SeamMap { + width, + height, + scale, + origin: (bounds.min_u, bounds.min_v), + px, + labels: vec![NONE; width * height], + }; + + // Where each frame's centre lands, in texels: what orders the frames + // and orients each cut. + let centres: Vec<(f64, f64)> = (0..n) + .map(|k| { + let d = cameras.bearing(k, (0.0, 0.0)); + projection + .from_direction(scale, d) + .map(|(u, v)| ((u - bounds.min_u) / px, (v - bounds.min_v) / px)) + .unwrap_or((width as f64 / 2.0, height as f64 / 2.0)) + }) + .collect(); + + // The composite so far: what its owner saw, and how far from the + // owner's edge. + let mut value = vec![f32::NAN; width * height]; + let mut edge = vec![f32::NAN; width * height]; + + for k in order(¢res, (width as f64 / 2.0, height as f64 / 2.0)) { + let w = warp(&map, proxies[k], cameras, k, gains[k], projection); + let overlap: Vec = (0..width * height) + .filter(|&i| map.labels[i] != NONE && !w.value[i].is_nan()) + .collect(); + // Texels nobody owns yet are the new frame's without a cut. + let mut take: Vec = map + .labels + .iter() + .zip(&w.value) + .map(|(&l, v)| l == NONE && !v.is_nan()) + .collect(); + if !overlap.is_empty() { + cut( + &map, &value, &edge, &w, &overlap, ¢res, k, opts, &mut take, + ); + } + for i in 0..width * height { + if take[i] { + map.labels[i] = k as u8; + value[i] = w.value[i]; + edge[i] = w.edge[i]; + } + } + } + Some(map) +} + +/// The order frames are laid down in: the one nearest the middle first, +/// then always the unplaced frame nearest any placed one, so that each new +/// frame meets the composite along an overlap rather than across a gap. +fn order(centres: &[(f64, f64)], middle: (f64, f64)) -> Vec { + let d2 = |a: (f64, f64), b: (f64, f64)| (a.0 - b.0).powi(2) + (a.1 - b.1).powi(2); + let n = centres.len(); + let mut placed = vec![false; n]; + let mut out = Vec::with_capacity(n); + let first = (0..n) + .min_by(|&a, &b| d2(centres[a], middle).total_cmp(&d2(centres[b], middle))) + .expect("at least one frame"); + placed[first] = true; + out.push(first); + while out.len() < n { + let next = (0..n) + .filter(|&k| !placed[k]) + .min_by(|&a, &b| { + let near = |k: usize| { + out.iter() + .map(|&p| d2(centres[k], centres[p])) + .fold(f64::MAX, f64::min) + }; + near(a).total_cmp(&near(b)) + }) + .expect("an unplaced frame"); + placed[next] = true; + out.push(next); + } + out +} + +/// Frame `k` sampled at every texel's centre, bilinearly. The proxy is +/// gamma-encoded grey, so the gain (linear) becomes `gain^(1/2.2)` on it. +fn warp( + map: &SeamMap, + g: &Gray, + cameras: &Cameras, + k: usize, + gain: f32, + projection: Projection, +) -> Warped { + let (fw, fh) = (g.width as f64, g.height as f64); + let gain = gain.max(1e-6).powf(1.0 / 2.2); + let mut value = vec![f32::NAN; map.width * map.height]; + let mut edge = vec![f32::NAN; map.width * map.height]; + for ty in 0..map.height { + let v = map.origin.1 + (ty as f64 + 0.5) * map.px; + for tx in 0..map.width { + let u = map.origin.0 + (tx as f64 + 0.5) * map.px; + let d = projection.to_direction(map.scale, u, v); + let Some((x, y)) = cameras.project(k, d) else { + continue; + }; + let (x, y) = (x + fw / 2.0 - 0.5, y + fh / 2.0 - 0.5); + let e = x.min(fw - 1.0 - x).min(y).min(fh - 1.0 - y); + if e < 0.0 { + continue; + } + let (x0, y0) = (x.floor() as usize, y.floor() as usize); + let (x1, y1) = ((x0 + 1).min(g.width - 1), (y0 + 1).min(g.height - 1)); + let (ax, ay) = ((x - x0 as f64) as f32, (y - y0 as f64) as f32); + let at = |xx: usize, yy: usize| g.data[yy * g.width + xx]; + let top = at(x0, y0) * (1.0 - ax) + at(x1, y0) * ax; + let bot = at(x0, y1) * (1.0 - ax) + at(x1, y1) * ax; + let i = ty * map.width + tx; + value[i] = (top * (1.0 - ay) + bot * ay) * gain; + edge[i] = e as f32; + } + } + Warped { value, edge } +} + +/// Central-difference gradient magnitude of `plane` at texel `i`, from the +/// neighbours that exist. +fn detail(plane: &[f32], width: usize, height: usize, i: usize) -> f32 { + let (x, y) = (i % width, i / width); + let c = plane[i]; + let mut g = 0.0f32; + let mut diff = |j: usize| { + let n = plane[j]; + if !n.is_nan() { + g = g.max((n - c).abs()); + } + }; + if x > 0 { + diff(i - 1); + } + if x + 1 < width { + diff(i + 1); + } + if y > 0 { + diff(i - width); + } + if y + 1 < height { + diff(i + width); + } + g +} + +/// Cut the overlap between the composite and frame `k`, marking in `take` +/// the overlap texels that go to `k`. +#[allow(clippy::too_many_arguments)] +fn cut( + map: &SeamMap, + value: &[f32], + edge: &[f32], + new: &Warped, + overlap: &[usize], + centres: &[(f64, f64)], + k: usize, + opts: &SeamOptions, + take: &mut [bool], +) { + let (w, h) = (map.width, map.height); + + // The raw cost per overlap texel. + let mut raw = vec![f32::NAN; w * h]; + let margin = opts.edge_margin.max(1.0); + for &i in overlap { + let differ = (value[i] - new.value[i]).abs(); + let detail = detail(value, w, h, i).max(detail(&new.value, w, h, i)); + let near = (1.0 - edge[i].min(new.edge[i]) / margin).max(0.0); + raw[i] = differ + opts.detail * detail + opts.edge * near * near + 1e-3; + } + // The worst over a small window: a texel is only cheap if its whole + // neighbourhood agrees, so the path keeps at least the blend's radius + // clear of a difference rather than threading the one lucky texel + // beside it — the blend straddles the path by that much and would + // otherwise reach the difference anyway. + let r = opts.smoothing as isize; + let mut cost = vec![OUTSIDE; w * h]; + for &i in overlap { + let (x, y) = ((i % w) as isize, (i / w) as isize); + let mut worst = 0.0f32; + for dy in -r..=r { + for dx in -r..=r { + let (xx, yy) = (x + dx, y + dy); + if xx < 0 || yy < 0 || xx >= w as isize || yy >= h as isize { + continue; + } + let c = raw[yy as usize * w + xx as usize]; + if !c.is_nan() { + worst = worst.max(c); + } + } + } + cost[i] = worst; + } + + // The axis the cut crosses: from the composite's frames, weighted by how + // much of the overlap each owns, to the new frame. + let mut from = (0.0f64, 0.0f64); + for &i in overlap { + let c = centres[usize::from(map.labels[i])]; + from = (from.0 + c.0, from.1 + c.1); + } + let m = overlap.len() as f64; + from = (from.0 / m, from.1 / m); + let to = centres[k]; + let (mut ax, mut ay) = (to.0 - from.0, to.1 - from.1); + let len = (ax * ax + ay * ay).sqrt(); + if len < 1e-6 { + (ax, ay) = (1.0, 0.0); + } else { + (ax, ay) = (ax / len, ay / len); + } + // Along the cut: perpendicular to the axis. + let (bx, by) = (-ay, ax); + + // The overlap's extent in (s along the cut, t across it). + let st = |i: usize| { + let (x, y) = ((i % w) as f64 + 0.5, (i / w) as f64 + 0.5); + (x * bx + y * by, x * ax + y * ay) + }; + let (mut s0, mut s1, mut t0, mut t1) = (f64::MAX, f64::MIN, f64::MAX, f64::MIN); + for &i in overlap { + let (s, t) = st(i); + s0 = s0.min(s); + s1 = s1.max(s); + t0 = t0.min(t); + t1 = t1.max(t); + } + let rows = (s1 - s0).round() as usize + 1; + let cols = (t1 - t0).round() as usize + 1; + + // The grid in (s, t), each cell sampled from the texel it falls in, so + // that a rotated overlap has no holes. + let mut grid = vec![OUTSIDE; rows * cols]; + let mut any = vec![false; rows]; + for si in 0..rows { + for ti in 0..cols { + let (s, t) = (s0 + si as f64, t0 + ti as f64); + let x = s * bx + t * ax; + let y = s * by + t * ay; + if x < 0.0 || y < 0.0 { + continue; + } + let (x, y) = (x as usize, y as usize); + if x >= w || y >= h { + continue; + } + let c = cost[y * w + x]; + if c < OUTSIDE { + grid[si * cols + ti] = c; + any[si] = true; + } + } + } + + // Dynamic programming down the rows: the path moves at most one column + // per row, and starts afresh after a row with no overlap in it. + let mut acc = grid.clone(); + let mut from_col = vec![0u32; rows * cols]; + for si in 1..rows { + if !any[si] { + continue; + } + let prev = &acc[(si - 1) * cols..si * cols].to_vec(); + if !any[si - 1] { + continue; + } + for ti in 0..cols { + let mut best = (prev[ti], ti); + if ti > 0 && prev[ti - 1] < best.0 { + best = (prev[ti - 1], ti - 1); + } + if ti + 1 < cols && prev[ti + 1] < best.0 { + best = (prev[ti + 1], ti + 1); + } + acc[si * cols + ti] += best.0; + from_col[si * cols + ti] = best.1 as u32; + } + } + // Back up from the end of each run of rows with overlap. + let mut seam = vec![usize::MAX; rows]; + let mut si = rows; + while si > 0 { + si -= 1; + if !any[si] { + continue; + } + let row = &acc[si * cols..(si + 1) * cols]; + let mut t = (0..cols) + .min_by(|&a, &b| row[a].total_cmp(&row[b])) + .unwrap_or(0); + loop { + seam[si] = t; + if si == 0 || !any[si - 1] { + break; + } + t = from_col[si * cols + t] as usize; + si -= 1; + } + } + + // The new frame takes the side of the path its centre is on. + for &i in overlap { + let (s, t) = st(i); + let si = ((s - s0).round() as usize).min(rows - 1); + let ti = (t - t0).round(); + if seam[si] != usize::MAX && ti >= seam[si] as f64 { + take[i] = true; + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::linalg::{Mat3, Vec3}; + + /// A scene as a function of direction, and frames of it rendered by the + /// same cameras the seam reads. + fn render( + cameras: &Cameras, + k: usize, + size: (usize, usize), + scene: impl Fn(Vec3) -> f32, + ) -> Gray { + let (w, h) = size; + let mut data = vec![0.0; w * h]; + for y in 0..h { + for x in 0..w { + let p = ( + x as f64 + 0.5 - w as f64 / 2.0, + y as f64 + 0.5 - h as f64 / 2.0, + ); + data[y * w + x] = scene(cameras.bearing(k, p)); + } + } + Gray { + width: w, + height: h, + data, + } + } + + fn yaw(a: f64) -> Mat3 { + let (s, c) = a.sin_cos(); + Mat3([[c, 0.0, s], [0.0, 1.0, 0.0], [-s, 0.0, c]]) + } + + /// Smooth, with a little texture: what a sky over a slope looks like to + /// the cost. + fn landscape(d: Vec3) -> f32 { + let (x, y) = (d.x() / d.z(), d.y() / d.z()); + let texture = if y > 0.1 { 0.1 * (y * 40.0).sin() } else { 0.0 }; + (0.5 + 0.2 * (x * 3.0).sin() + texture).clamp(0.0, 1.0) as f32 + } + + fn pair() -> Cameras { + Cameras { + rotations: vec![Mat3::IDENTITY, yaw(0.35)], + focal: 300.0, + } + } + + #[test] + fn one_frame_owns_everything_it_reaches() { + let cameras = Cameras { + rotations: vec![Mat3::IDENTITY], + focal: 300.0, + }; + let g = render(&cameras, 0, (320, 240), landscape); + let map = find( + &[&g], + &cameras, + &[1.0], + Projection::Perspective, + &Default::default(), + ) + .unwrap(); + let owned = map.labels.iter().filter(|&&l| l == 0).count(); + assert!(owned as f64 > 0.95 * (map.width * map.height) as f64); + } + + #[test] + fn each_frame_keeps_its_own_side() { + let cameras = pair(); + let frames: Vec = (0..2) + .map(|k| render(&cameras, k, (320, 240), landscape)) + .collect(); + let refs: Vec<&Gray> = frames.iter().collect(); + let map = find( + &refs, + &cameras, + &[1.0, 1.0], + Projection::Cylindrical, + &Default::default(), + ) + .unwrap(); + let mid = map.height / 2 * map.width; + assert_eq!(map.labels[mid + 2], 0, "the left edge is frame 0's alone"); + assert_eq!( + map.labels[mid + map.width - 3], + 1, + "the right edge is frame 1's" + ); + // One change of owner along every row that both frames cross. + for y in 0..map.height { + let row = &map.labels[y * map.width..(y + 1) * map.width]; + let owned: Vec = row.iter().copied().filter(|&l| l != NONE).collect(); + let changes = owned.windows(2).filter(|p| p[0] != p[1]).count(); + assert!(changes <= 1, "row {y} changes owner {changes} times"); + } + } + + #[test] + fn the_seam_goes_round_what_only_one_frame_saw() { + // Frame 1 saw something frame 0 did not — a figure that walked into + // the overlap — in the middle of where the two meet. + let cameras = pair(); + let figure = Vec3::new(0.175f64.sin(), 0.0, 0.175f64.cos()); + let walker = |d: Vec3| { + let near = (d.x() - figure.x()).abs() < 0.04 && (d.y() - figure.y()).abs() < 0.15; + if near { + 0.95 + } else { + landscape(d) + } + }; + let frames = [ + render(&cameras, 0, (320, 240), landscape), + render(&cameras, 1, (320, 240), walker), + ]; + let refs: Vec<&Gray> = frames.iter().collect(); + let map = find( + &refs, + &cameras, + &[1.0, 1.0], + Projection::Cylindrical, + &Default::default(), + ) + .unwrap(); + // Every texel of the figure is taken from the same frame, with a + // blend radius of room to spare, so it is either all there or not at + // all — never half. + let (u, v) = Projection::Cylindrical + .from_direction(map.scale, figure) + .unwrap(); + let mut owners = std::collections::HashSet::new(); + // The figure's extent on the surface, plus the blend's radius. + let radius = 3.0; + let reach = |half: f64| half * map.scale + radius * map.px; + let (ru, rv) = (reach(0.04), reach(0.15)); + let mut dv = -rv; + while dv <= rv { + let mut du = -ru; + while du <= ru { + let s = map.share(1, u + du, v + dv, map.scale, radius); + owners.insert((s.unwrap() * 100.0).round() as i32); + du += map.px; + } + dv += map.px; + } + assert_eq!(owners.len(), 1, "the figure is split: shares {owners:?}"); + } + + #[test] + fn share_is_a_blend_across_the_seam_and_whole_away_from_it() { + let map = SeamMap { + width: 8, + height: 1, + scale: 1.0, + origin: (0.0, 0.0), + px: 1.0, + labels: vec![0, 0, 0, 0, 1, 1, 1, 1], + }; + assert_eq!(map.share(0, 1.5, 0.5, 1.0, 2.0), Some(1.0)); + assert_eq!(map.share(1, 6.5, 0.5, 1.0, 2.0), Some(1.0)); + let at_seam = map.share(0, 4.0, 0.5, 1.0, 2.0).unwrap(); + assert!((at_seam - 0.5).abs() < 1e-6, "{at_seam}"); + // And at twice the scale, the same point is twice as far out. + assert_eq!( + map.share(0, 8.0, 1.0, 2.0, 2.0), + map.share(0, 4.0, 0.5, 1.0, 2.0) + ); + let empty = SeamMap { + labels: vec![NONE; 8], + ..map + }; + assert_eq!(empty.share(0, 4.0, 0.5, 1.0, 2.0), None); + } +} diff --git a/docs/dev/panorama.md b/docs/dev/panorama.md index f5f9ecf..025b162 100644 --- a/docs/dev/panorama.md +++ b/docs/dev/panorama.md @@ -367,7 +367,7 @@ Built, on branch `merge/panorama`, in the order §10 gave: | The camera-space tap | `OutputMode::CameraLinear`, `AdjustPass::render_camera_linear` | Done, `rgba32float`, tiles by view rect | | Linear DNG writer, streamed | `dr_export::write_linear_dng` | Done; rawler reads it back | | A three-sample `RawImage` re-entering the pipeline | `dr-decode`, `DemosaicedImage::from_linear_rgb16` | Done | -| Warp, accumulate, resolve, chunk by chunk | `dr_gpu::MergePass`, `merge.wgsl` | Done; feathered blend, scalar gain | +| Warp, accumulate, resolve, chunk by chunk | `dr_gpu::MergePass`, `merge.wgsl` | Done; seams (§11.1) over a feather, scalar gain | | The job: load, proxies, align, gains, confirm, merge, provenance | `dr_ui::merge` | Done; `examples/merge.rs` drives it headless | | The page: table, preview, projection, Merge/Stop/Back; the grid's button | `merge.slint`, `merge_ui.rs` | Done; `DARKROOM_START_MERGE=a.CR2,b.CR2` lands on it | | Placement beside the sources through the outbox, rescan | `merge_ui.rs` | Done, untested against a server | @@ -385,9 +385,11 @@ half. the file carries the black border. The largest inscribed rectangle over the coverage, then the DNG's `DefaultCropOrigin`/`DefaultCropSize`, so nothing is thrown away and the develop view opens on the picture. -2. **Seams and the pyramid** (§10 step 5). The feather hides exposure and - small misalignment; parallax on the near slope will show as a soft - double edge at 1:1. +2. **The pyramid** (§10 step 5). Seams landed 2026-09-30 (§11.1); the + blend across them is one width for every frequency, so an exposure step + the gains leave is narrowed to the seam's 64 px rather than hidden over + the old 200. A Laplacian pyramid would blend low frequencies wide and + detail narrow. 3. **Vignetting in the tap.** The lens profile's distortion is applied before the fetch; its vignetting is an operation and is not. Frame edges are darker than their centres by the lens's falloff, and the feather @@ -402,6 +404,35 @@ half. catalog's `content_hash` is null for most images most of the time. The hash can join it when the catalog has one. +### 11.1 Seams — 2026-09-30 + +The feather averaged every overlap over 200 px, so anything the frames +disagreed on — parallax on the near slope, a walker, wind in a branch — came +out twice at half strength: a soft double edge at 1:1, reported as a glitch. + +`dr_pano::seam` now chooses, per output texel at proxy resolution, which +frame it is taken from. Frames are laid down nearest-first; where a new one +overlaps the composite, each texel costs the gain-corrected difference +between the two, plus the detail either has there, plus nearness to either +frame's edge (vignetting, the lens correction's fringe), taken as the +**worst** over a 4-texel window so the path stays a blend radius clear of a +difference rather than grazing it. The cut is a dynamic-programming path +across the overlap, perpendicular to the line from the composite's frames to +the new one: §4's per-column seam, not a graph cut. The map is computed per +projection, for the page's preview and again for the merge. + +`merge.wgsl` weights a frame by its tent-filtered share of the label map +about each pixel (`SeamMap::share`, repeated verbatim), over a window +`seam_blend_px` wide (64, capped at 4 texels either side). The edge feather +remains underneath as a factor and, with a 1e-4 floor, as the answer where +the map names no frame that reaches the pixel. `--feather-only` on +`examples/merge.rs` merges the old way, for comparison. + +Known limits: one axis per new frame, so in a multi-row set a frame +overlapping its left neighbour and the row above is cut along a compromise +direction; the cost reads grey proxies, so a difference in hue alone is +invisible to it. + ## 12. Filling the border instead of cropping it — MI-GAN, read and measured 2026-09-19 Raised after the first merges: the ragged border a cylinder leaves could be diff --git a/docs/dev/traceability.md b/docs/dev/traceability.md index 8ed808c..ea7dd7b 100644 --- a/docs/dev/traceability.md +++ b/docs/dev/traceability.md @@ -9,8 +9,8 @@ Denominators are parsed from [`requirements.md`](requirements.md) at run time, n | Metric | Value | |---|---| -| Source files scanned | 479 | -| TRACES tags found | 1983 | +| Source files scanned | 480 | +| TRACES tags found | 1984 | | Requirements defined | 193 | | Requirements deferred (post-v1) | 24 | | Requirements covered | 165 | @@ -109,13 +109,13 @@ _None._ | FR-EXP-9 | [`core/dr-decode/src/lib.rs:537`](../../core/dr-decode/src/lib.rs#L537), [`core/dr-export/src/lib.rs:132`](../../core/dr-export/src/lib.rs#L132), [`core/dr-export/src/lib.rs:1`](../../core/dr-export/src/lib.rs#L1), [`core/dr-gpu/src/adjust.rs:1491`](../../core/dr-gpu/src/adjust.rs#L1491), [`ui/dr-ui/examples/face_native.rs:1`](../../ui/dr-ui/examples/face_native.rs#L1), [`ui/dr-ui/src/decoder_seam.rs:255`](../../ui/dr-ui/src/decoder_seam.rs#L255), [`ui/dr-ui/src/develop/mask_ops.rs:1130`](../../ui/dr-ui/src/develop/mask_ops.rs#L1130), [`ui/dr-ui/src/develop/render.rs:671`](../../ui/dr-ui/src/develop/render.rs#L671), [`ui/dr-ui/src/develop/render.rs:709`](../../ui/dr-ui/src/develop/render.rs#L709), [`ui/dr-ui/src/lib.rs:654`](../../ui/dr-ui/src/lib.rs#L654), [`ui/dr-ui/src/library/sweep.rs:538`](../../ui/dr-ui/src/library/sweep.rs#L538), [`ui/dr-ui/tests/export_ignores_the_viewport.rs:1`](../../ui/dr-ui/tests/export_ignores_the_viewport.rs#L1) | | FR-INF-1 | [`ui/dr-ui/src/lib.rs:1450`](../../ui/dr-ui/src/lib.rs#L1450), [`ui/dr-ui/ui/settings.slint:824`](../../ui/dr-ui/ui/settings.slint#L824) | | FR-MRG-1 | [`core/dr-pano/src/align.rs:1`](../../core/dr-pano/src/align.rs#L1), [`core/dr-pano/src/lib.rs:1`](../../core/dr-pano/src/lib.rs#L1), [`ui/dr-ui/src/lib.rs:1876`](../../ui/dr-ui/src/lib.rs#L1876), [`ui/dr-ui/src/merge.rs:1`](../../ui/dr-ui/src/merge.rs#L1), [`ui/dr-ui/src/merge_ui.rs:1`](../../ui/dr-ui/src/merge_ui.rs#L1), [`ui/dr-ui/ui/app.slint:1533`](../../ui/dr-ui/ui/app.slint#L1533), [`ui/dr-ui/ui/app.slint:592`](../../ui/dr-ui/ui/app.slint#L592), [`ui/dr-ui/ui/library.slint:2049`](../../ui/dr-ui/ui/library.slint#L2049), [`ui/dr-ui/ui/library.slint:5197`](../../ui/dr-ui/ui/library.slint#L5197), [`ui/dr-ui/ui/merge.slint:1`](../../ui/dr-ui/ui/merge.slint#L1) | -| FR-MRG-10 | [`core/dr-gpu/src/merge.rs:1`](../../core/dr-gpu/src/merge.rs#L1), [`core/dr-gpu/src/shaders/merge.wgsl:1`](../../core/dr-gpu/src/shaders/merge.wgsl#L1), [`core/dr-pano/src/lib.rs:1`](../../core/dr-pano/src/lib.rs#L1) | +| FR-MRG-10 | [`core/dr-gpu/src/merge.rs:1`](../../core/dr-gpu/src/merge.rs#L1), [`core/dr-gpu/src/shaders/merge.wgsl:1`](../../core/dr-gpu/src/shaders/merge.wgsl#L1), [`core/dr-pano/src/lib.rs:1`](../../core/dr-pano/src/lib.rs#L1), [`core/dr-pano/src/seam.rs:1`](../../core/dr-pano/src/seam.rs#L1) | | FR-MRG-11 | [`core/dr-gpu/src/merge.rs:1`](../../core/dr-gpu/src/merge.rs#L1), [`core/dr-gpu/src/shaders/merge.wgsl:1`](../../core/dr-gpu/src/shaders/merge.wgsl#L1) | | FR-MRG-2 | [`core/dr-gpu/src/adjust.rs:1514`](../../core/dr-gpu/src/adjust.rs#L1514), [`core/dr-gpu/src/adjust.rs:1641`](../../core/dr-gpu/src/adjust.rs#L1641), [`core/dr-gpu/src/adjust.rs:528`](../../core/dr-gpu/src/adjust.rs#L528), [`core/dr-gpu/src/adjust.rs:94`](../../core/dr-gpu/src/adjust.rs#L94), [`core/dr-pipeline/src/graph.rs:916`](../../core/dr-pipeline/src/graph.rs#L916), [`core/dr-pipeline/src/operation.rs:1150`](../../core/dr-pipeline/src/operation.rs#L1150), [`core/dr-pipeline/src/operation.rs:2529`](../../core/dr-pipeline/src/operation.rs#L2529), [`core/dr-pipeline/src/operation.rs:2569`](../../core/dr-pipeline/src/operation.rs#L2569), [`core/dr-pipeline/src/operation.rs:516`](../../core/dr-pipeline/src/operation.rs#L516), [`core/dr-pipeline/src/operation.rs:749`](../../core/dr-pipeline/src/operation.rs#L749), [`ui/dr-ui/src/merge.rs:1`](../../ui/dr-ui/src/merge.rs#L1) | | FR-MRG-3 | [`core/dr-decode/examples/linear_dng.rs:1`](../../core/dr-decode/examples/linear_dng.rs#L1), [`core/dr-decode/src/lib.rs:128`](../../core/dr-decode/src/lib.rs#L128), [`core/dr-decode/src/lib.rs:135`](../../core/dr-decode/src/lib.rs#L135), [`core/dr-decode/src/lib.rs:580`](../../core/dr-decode/src/lib.rs#L580), [`core/dr-decode/src/profile.rs:397`](../../core/dr-decode/src/profile.rs#L397), [`core/dr-export/src/dng.rs:1`](../../core/dr-export/src/dng.rs#L1), [`core/dr-gpu/src/demosaic.rs:337`](../../core/dr-gpu/src/demosaic.rs#L337), [`ui/dr-ui/src/merge.rs:1`](../../ui/dr-ui/src/merge.rs#L1) | -| FR-MRG-4 | [`core/dr-export/src/inscribed.rs:1`](../../core/dr-export/src/inscribed.rs#L1), [`core/dr-pano/src/fill.rs:1`](../../core/dr-pano/src/fill.rs#L1), [`core/dr-pano/src/migan.rs:1`](../../core/dr-pano/src/migan.rs#L1), [`core/dr-pano/src/projection.rs:1`](../../core/dr-pano/src/projection.rs#L1), [`ui/dr-ui/src/library/paths.rs:365`](../../ui/dr-ui/src/library/paths.rs#L365), [`ui/dr-ui/src/library/paths.rs:369`](../../ui/dr-ui/src/library/paths.rs#L369), [`ui/dr-ui/src/merge.rs:166`](../../ui/dr-ui/src/merge.rs#L166), [`ui/dr-ui/src/merge_ui.rs:1`](../../ui/dr-ui/src/merge_ui.rs#L1), [`ui/dr-ui/ui/merge.slint:1`](../../ui/dr-ui/ui/merge.slint#L1), [`ui/dr-ui/ui/merge.slint:50`](../../ui/dr-ui/ui/merge.slint#L50) | +| FR-MRG-4 | [`core/dr-export/src/inscribed.rs:1`](../../core/dr-export/src/inscribed.rs#L1), [`core/dr-pano/src/fill.rs:1`](../../core/dr-pano/src/fill.rs#L1), [`core/dr-pano/src/migan.rs:1`](../../core/dr-pano/src/migan.rs#L1), [`core/dr-pano/src/projection.rs:1`](../../core/dr-pano/src/projection.rs#L1), [`ui/dr-ui/src/library/paths.rs:365`](../../ui/dr-ui/src/library/paths.rs#L365), [`ui/dr-ui/src/library/paths.rs:369`](../../ui/dr-ui/src/library/paths.rs#L369), [`ui/dr-ui/src/merge.rs:174`](../../ui/dr-ui/src/merge.rs#L174), [`ui/dr-ui/src/merge_ui.rs:1`](../../ui/dr-ui/src/merge_ui.rs#L1), [`ui/dr-ui/ui/merge.slint:1`](../../ui/dr-ui/ui/merge.slint#L1), [`ui/dr-ui/ui/merge.slint:50`](../../ui/dr-ui/ui/merge.slint#L50) | | FR-MRG-5 | [`core/dr-pano/src/align.rs:1`](../../core/dr-pano/src/align.rs#L1), [`ui/dr-ui/src/merge.rs:1`](../../ui/dr-ui/src/merge.rs#L1), [`ui/dr-ui/src/merge_ui.rs:1`](../../ui/dr-ui/src/merge_ui.rs#L1), [`ui/dr-ui/ui/merge.slint:1`](../../ui/dr-ui/ui/merge.slint#L1) | -| FR-MRG-6 | [`core/dr-pipeline/src/sidecar.rs:1055`](../../core/dr-pipeline/src/sidecar.rs#L1055), [`core/dr-pipeline/src/sidecar.rs:116`](../../core/dr-pipeline/src/sidecar.rs#L116), [`core/dr-pipeline/src/sidecar.rs:126`](../../core/dr-pipeline/src/sidecar.rs#L126), [`core/dr-pipeline/src/sidecar.rs:2284`](../../core/dr-pipeline/src/sidecar.rs#L2284), [`core/dr-pipeline/src/sidecar.rs:860`](../../core/dr-pipeline/src/sidecar.rs#L860), [`core/dr-thumbs/src/lib.rs:75`](../../core/dr-thumbs/src/lib.rs#L75), [`ui/dr-ui/src/export.rs:149`](../../ui/dr-ui/src/export.rs#L149), [`ui/dr-ui/src/export.rs:349`](../../ui/dr-ui/src/export.rs#L349), [`ui/dr-ui/src/export.rs:579`](../../ui/dr-ui/src/export.rs#L579), [`ui/dr-ui/src/lib.rs:1905`](../../ui/dr-ui/src/lib.rs#L1905), [`ui/dr-ui/src/library/cells.rs:353`](../../ui/dr-ui/src/library/cells.rs#L353), [`ui/dr-ui/src/library/cells.rs:374`](../../ui/dr-ui/src/library/cells.rs#L374), [`ui/dr-ui/src/library/cells.rs:56`](../../ui/dr-ui/src/library/cells.rs#L56), [`ui/dr-ui/src/library/composite.rs:1`](../../ui/dr-ui/src/library/composite.rs#L1), [`ui/dr-ui/src/library/sweep.rs:31`](../../ui/dr-ui/src/library/sweep.rs#L31), [`ui/dr-ui/src/library_ui/controller.rs:116`](../../ui/dr-ui/src/library_ui/controller.rs#L116), [`ui/dr-ui/src/library_ui/controller.rs:126`](../../ui/dr-ui/src/library_ui/controller.rs#L126), [`ui/dr-ui/src/library_ui/grid.rs:422`](../../ui/dr-ui/src/library_ui/grid.rs#L422), [`ui/dr-ui/src/library_ui/layout.rs:1`](../../ui/dr-ui/src/library_ui/layout.rs#L1), [`ui/dr-ui/src/library_ui/layout.rs:51`](../../ui/dr-ui/src/library_ui/layout.rs#L51), [`ui/dr-ui/src/library_ui/open.rs:44`](../../ui/dr-ui/src/library_ui/open.rs#L44), [`ui/dr-ui/src/library_ui/window.rs:233`](../../ui/dr-ui/src/library_ui/window.rs#L233), [`ui/dr-ui/src/library_ui/window.rs:476`](../../ui/dr-ui/src/library_ui/window.rs#L476), [`ui/dr-ui/src/library_ui/window.rs:664`](../../ui/dr-ui/src/library_ui/window.rs#L664), [`ui/dr-ui/src/merge.rs:1030`](../../ui/dr-ui/src/merge.rs#L1030), [`ui/dr-ui/src/merge.rs:250`](../../ui/dr-ui/src/merge.rs#L250), [`ui/dr-ui/src/merge.rs:987`](../../ui/dr-ui/src/merge.rs#L987), [`ui/dr-ui/src/merge_thumbs.rs:1`](../../ui/dr-ui/src/merge_thumbs.rs#L1), [`ui/dr-ui/src/merge_ui.rs:52`](../../ui/dr-ui/src/merge_ui.rs#L52), [`ui/dr-ui/ui/library.slint:1486`](../../ui/dr-ui/ui/library.slint#L1486), [`ui/dr-ui/ui/library.slint:486`](../../ui/dr-ui/ui/library.slint#L486), [`ui/dr-ui/ui/library.slint:549`](../../ui/dr-ui/ui/library.slint#L549), [`ui/dr-ui/ui/library.slint:637`](../../ui/dr-ui/ui/library.slint#L637) | +| FR-MRG-6 | [`core/dr-pipeline/src/sidecar.rs:1055`](../../core/dr-pipeline/src/sidecar.rs#L1055), [`core/dr-pipeline/src/sidecar.rs:116`](../../core/dr-pipeline/src/sidecar.rs#L116), [`core/dr-pipeline/src/sidecar.rs:126`](../../core/dr-pipeline/src/sidecar.rs#L126), [`core/dr-pipeline/src/sidecar.rs:2284`](../../core/dr-pipeline/src/sidecar.rs#L2284), [`core/dr-pipeline/src/sidecar.rs:860`](../../core/dr-pipeline/src/sidecar.rs#L860), [`core/dr-thumbs/src/lib.rs:75`](../../core/dr-thumbs/src/lib.rs#L75), [`ui/dr-ui/src/export.rs:149`](../../ui/dr-ui/src/export.rs#L149), [`ui/dr-ui/src/export.rs:349`](../../ui/dr-ui/src/export.rs#L349), [`ui/dr-ui/src/export.rs:579`](../../ui/dr-ui/src/export.rs#L579), [`ui/dr-ui/src/lib.rs:1905`](../../ui/dr-ui/src/lib.rs#L1905), [`ui/dr-ui/src/library/cells.rs:353`](../../ui/dr-ui/src/library/cells.rs#L353), [`ui/dr-ui/src/library/cells.rs:374`](../../ui/dr-ui/src/library/cells.rs#L374), [`ui/dr-ui/src/library/cells.rs:56`](../../ui/dr-ui/src/library/cells.rs#L56), [`ui/dr-ui/src/library/composite.rs:1`](../../ui/dr-ui/src/library/composite.rs#L1), [`ui/dr-ui/src/library/sweep.rs:31`](../../ui/dr-ui/src/library/sweep.rs#L31), [`ui/dr-ui/src/library_ui/controller.rs:116`](../../ui/dr-ui/src/library_ui/controller.rs#L116), [`ui/dr-ui/src/library_ui/controller.rs:126`](../../ui/dr-ui/src/library_ui/controller.rs#L126), [`ui/dr-ui/src/library_ui/grid.rs:422`](../../ui/dr-ui/src/library_ui/grid.rs#L422), [`ui/dr-ui/src/library_ui/layout.rs:1`](../../ui/dr-ui/src/library_ui/layout.rs#L1), [`ui/dr-ui/src/library_ui/layout.rs:51`](../../ui/dr-ui/src/library_ui/layout.rs#L51), [`ui/dr-ui/src/library_ui/open.rs:44`](../../ui/dr-ui/src/library_ui/open.rs#L44), [`ui/dr-ui/src/library_ui/window.rs:233`](../../ui/dr-ui/src/library_ui/window.rs#L233), [`ui/dr-ui/src/library_ui/window.rs:476`](../../ui/dr-ui/src/library_ui/window.rs#L476), [`ui/dr-ui/src/library_ui/window.rs:664`](../../ui/dr-ui/src/library_ui/window.rs#L664), [`ui/dr-ui/src/merge.rs:1015`](../../ui/dr-ui/src/merge.rs#L1015), [`ui/dr-ui/src/merge.rs:1058`](../../ui/dr-ui/src/merge.rs#L1058), [`ui/dr-ui/src/merge.rs:258`](../../ui/dr-ui/src/merge.rs#L258), [`ui/dr-ui/src/merge_thumbs.rs:1`](../../ui/dr-ui/src/merge_thumbs.rs#L1), [`ui/dr-ui/src/merge_ui.rs:52`](../../ui/dr-ui/src/merge_ui.rs#L52), [`ui/dr-ui/ui/library.slint:1486`](../../ui/dr-ui/ui/library.slint#L1486), [`ui/dr-ui/ui/library.slint:486`](../../ui/dr-ui/ui/library.slint#L486), [`ui/dr-ui/ui/library.slint:549`](../../ui/dr-ui/ui/library.slint#L549), [`ui/dr-ui/ui/library.slint:637`](../../ui/dr-ui/ui/library.slint#L637) | | FR-MRG-7 | [`ui/dr-ui/src/merge.rs:1`](../../ui/dr-ui/src/merge.rs#L1), [`ui/dr-ui/src/merge_ui.rs:1`](../../ui/dr-ui/src/merge_ui.rs#L1), [`ui/dr-ui/ui/app.slint:592`](../../ui/dr-ui/ui/app.slint#L592), [`ui/dr-ui/ui/merge.slint:1`](../../ui/dr-ui/ui/merge.slint#L1) | | FR-MRG-8 | [`core/dr-pano/src/xfeat.rs:1`](../../core/dr-pano/src/xfeat.rs#L1), [`core/dr-segment/examples/onnx_probe.rs:1`](../../core/dr-segment/examples/onnx_probe.rs#L1) | | FR-NC-1 | [`core/dr-sync-nextcloud/src/auth.rs:206`](../../core/dr-sync-nextcloud/src/auth.rs#L206), [`core/dr-sync-nextcloud/src/auth.rs:49`](../../core/dr-sync-nextcloud/src/auth.rs#L49), [`core/dr-sync-nextcloud/src/provider.rs:1`](../../core/dr-sync-nextcloud/src/provider.rs#L1), [`core/dr-sync/src/account.rs:375`](../../core/dr-sync/src/account.rs#L375), [`ui/dr-ui/src/launch.rs:316`](../../ui/dr-ui/src/launch.rs#L316), [`ui/dr-ui/src/launch.rs:61`](../../ui/dr-ui/src/launch.rs#L61), [`ui/dr-ui/src/launch_ui.rs:521`](../../ui/dr-ui/src/launch_ui.rs#L521) | @@ -147,7 +147,7 @@ _None._ | FR-PLAT-WIN-2 | [`apps/darkroom-desktop/build.rs:1`](../../apps/darkroom-desktop/build.rs#L1), [`apps/darkroom-desktop/src/main.rs:6`](../../apps/darkroom-desktop/src/main.rs#L6), [`ui/dr-ui/src/launch_ui.rs:937`](../../ui/dr-ui/src/launch_ui.rs#L937) | | FR-PLAT-WIN-3 | [`apps/darkroom-desktop/src/main.rs:19`](../../apps/darkroom-desktop/src/main.rs#L19) | | FR-RAW-1 | [`core/dr-decode/src/lib.rs:250`](../../core/dr-decode/src/lib.rs#L250), [`core/dr-types/src/lib.rs:132`](../../core/dr-types/src/lib.rs#L132), [`core/dr-types/src/lib.rs:203`](../../core/dr-types/src/lib.rs#L203) | -| FR-RAW-2 | [`core/dr-decode/src/decoder.rs:1`](../../core/dr-decode/src/decoder.rs#L1), [`core/dr-decode/src/decoder.rs:26`](../../core/dr-decode/src/decoder.rs#L26), [`core/dr-decode/src/decoder.rs:59`](../../core/dr-decode/src/decoder.rs#L59), [`core/dr-decode/src/decoder.rs:95`](../../core/dr-decode/src/decoder.rs#L95), [`ui/dr-ui/src/decoder_seam.rs:183`](../../ui/dr-ui/src/decoder_seam.rs#L183), [`ui/dr-ui/src/decoder_seam.rs:1`](../../ui/dr-ui/src/decoder_seam.rs#L1), [`ui/dr-ui/src/decoder_seam.rs:214`](../../ui/dr-ui/src/decoder_seam.rs#L214), [`ui/dr-ui/src/decoder_seam.rs:255`](../../ui/dr-ui/src/decoder_seam.rs#L255), [`ui/dr-ui/src/export.rs:1096`](../../ui/dr-ui/src/export.rs#L1096), [`ui/dr-ui/src/export.rs:850`](../../ui/dr-ui/src/export.rs#L850), [`ui/dr-ui/src/import.rs:96`](../../ui/dr-ui/src/import.rs#L96), [`ui/dr-ui/src/library/sweep.rs:301`](../../ui/dr-ui/src/library/sweep.rs#L301), [`ui/dr-ui/src/library/sweep.rs:629`](../../ui/dr-ui/src/library/sweep.rs#L629), [`ui/dr-ui/src/library/thumbnails_gen.rs:101`](../../ui/dr-ui/src/library/thumbnails_gen.rs#L101), [`ui/dr-ui/src/merge.rs:113`](../../ui/dr-ui/src/merge.rs#L113), [`ui/dr-ui/src/repairs.rs:243`](../../ui/dr-ui/src/repairs.rs#L243) | +| FR-RAW-2 | [`core/dr-decode/src/decoder.rs:1`](../../core/dr-decode/src/decoder.rs#L1), [`core/dr-decode/src/decoder.rs:26`](../../core/dr-decode/src/decoder.rs#L26), [`core/dr-decode/src/decoder.rs:59`](../../core/dr-decode/src/decoder.rs#L59), [`core/dr-decode/src/decoder.rs:95`](../../core/dr-decode/src/decoder.rs#L95), [`ui/dr-ui/src/decoder_seam.rs:183`](../../ui/dr-ui/src/decoder_seam.rs#L183), [`ui/dr-ui/src/decoder_seam.rs:1`](../../ui/dr-ui/src/decoder_seam.rs#L1), [`ui/dr-ui/src/decoder_seam.rs:214`](../../ui/dr-ui/src/decoder_seam.rs#L214), [`ui/dr-ui/src/decoder_seam.rs:255`](../../ui/dr-ui/src/decoder_seam.rs#L255), [`ui/dr-ui/src/export.rs:1096`](../../ui/dr-ui/src/export.rs#L1096), [`ui/dr-ui/src/export.rs:850`](../../ui/dr-ui/src/export.rs#L850), [`ui/dr-ui/src/import.rs:96`](../../ui/dr-ui/src/import.rs#L96), [`ui/dr-ui/src/library/sweep.rs:301`](../../ui/dr-ui/src/library/sweep.rs#L301), [`ui/dr-ui/src/library/sweep.rs:629`](../../ui/dr-ui/src/library/sweep.rs#L629), [`ui/dr-ui/src/library/thumbnails_gen.rs:101`](../../ui/dr-ui/src/library/thumbnails_gen.rs#L101), [`ui/dr-ui/src/merge.rs:119`](../../ui/dr-ui/src/merge.rs#L119), [`ui/dr-ui/src/repairs.rs:243`](../../ui/dr-ui/src/repairs.rs#L243) | | FR-RAW-3 | [`core/dr-decode/src/lib.rs:146`](../../core/dr-decode/src/lib.rs#L146), [`core/dr-decode/src/lib.rs:537`](../../core/dr-decode/src/lib.rs#L537), [`core/dr-decode/src/locate.rs:1435`](../../core/dr-decode/src/locate.rs#L1435), [`core/dr-gpu/src/demosaic.rs:1182`](../../core/dr-gpu/src/demosaic.rs#L1182), [`core/dr-gpu/src/demosaic.rs:749`](../../core/dr-gpu/src/demosaic.rs#L749), [`core/dr-gpu/tests/hot_pixels.rs:1`](../../core/dr-gpu/tests/hot_pixels.rs#L1) | | FR-RAW-4 | [`core/dr-decode/src/error.rs:1`](../../core/dr-decode/src/error.rs#L1), [`core/dr-decode/src/error.rs:30`](../../core/dr-decode/src/error.rs#L30), [`ui/dr-ui/src/lib.rs:295`](../../ui/dr-ui/src/lib.rs#L295) | | FR-RAW-5 | [`core/dr-decode/src/lib.rs:174`](../../core/dr-decode/src/lib.rs#L174), [`core/dr-gpu/src/demosaic.rs:1056`](../../core/dr-gpu/src/demosaic.rs#L1056), [`core/dr-gpu/src/demosaic.rs:1180`](../../core/dr-gpu/src/demosaic.rs#L1180), [`core/dr-gpu/src/demosaic.rs:34`](../../core/dr-gpu/src/demosaic.rs#L34), [`core/dr-gpu/src/demosaic.rs:977`](../../core/dr-gpu/src/demosaic.rs#L977) | diff --git a/ui/dr-ui/examples/merge.rs b/ui/dr-ui/examples/merge.rs index aaffc38..416146e 100644 --- a/ui/dr-ui/examples/merge.rs +++ b/ui/dr-ui/examples/merge.rs @@ -6,6 +6,9 @@ //! cargo run -p dr-ui --example merge --release -- --leave-out 3 fixtures/pano/2025-08-05/*.CR2 //! ``` //! +//! `--feather-only` merges without seams — every overlap the feathered +//! average — for comparing the two on the same set. +//! //! `--leave-out N` unticks frame `N` once the first alignment is in, as the //! page's box would, so the job aligns again without it. //! @@ -25,6 +28,7 @@ fn main() { let mut fill = false; let mut wait_engines = false; let mut leave_out: Option = None; + let mut seams = true; loop { match args.first().map(String::as_str) { Some("--out") => { @@ -39,6 +43,10 @@ fn main() { args.remove(0); leave_out = args.remove(0).parse().ok(); } + Some("--feather-only") => { + args.remove(0); + seams = false; + } Some("--wait-engines") => { args.remove(0); wait_engines = true; @@ -101,6 +109,7 @@ fn main() { dr_ui::merge::MergeRequest::new(frames, dr_ui::merge::MergeDestination::Local(dir)); // The filler from the shared models directory, as the app finds it. request.inpaint_model = dr_ui::merge::inpaint_model_path(); + request.seams = seams; let (tx, rx) = std::sync::mpsc::channel(); let (decide, decision) = std::sync::mpsc::channel(); let cancel = dr_ui::merge::Cancel::default(); diff --git a/ui/dr-ui/src/merge.rs b/ui/dr-ui/src/merge.rs index da7952f..85fbe46 100644 --- a/ui/dr-ui/src/merge.rs +++ b/ui/dr-ui/src/merge.rs @@ -42,7 +42,7 @@ use dr_gpu::{ }; use dr_pano::bundle::Cameras; use dr_pano::projection::{self, Projection}; -use dr_pano::{Alignment, Gray}; +use dr_pano::{Alignment, Gray, SeamMap}; use dr_pipeline::EditGraph; pub use crate::export::Cancel; @@ -103,8 +103,14 @@ pub struct MergeRequest { pub destination: MergeDestination, /// `None` for the projection the field of view suggests. pub projection: Option, - /// Pixels over which a frame's weight ramps up from its edge. + /// Pixels over which a frame's weight ramps up from its edge, where no + /// seam says which frame a pixel is taken from. pub feather_px: f32, + /// Whether overlaps are cut along seams (`dr_pano::seam`) or averaged + /// across `feather_px` everywhere. + pub seams: bool, + /// The width, in output pixels, of the blend across a seam. + pub seam_blend_px: f32, /// GPU work unit; also the DNG strip height. pub chunk: (u32, u32), /// The border filler's weights, if the device has them (FR-MRG-4). @@ -122,6 +128,8 @@ impl MergeRequest { destination, projection: None, feather_px: 200.0, + seams: true, + seam_blend_px: 64.0, chunk: (2048, 512), inpaint_model: None, decoder: dr_decode::default(), @@ -478,11 +486,20 @@ fn run_inner( let preview = match s.alignment.is_complete() { false => None, true => { + let map = request + .seams + .then(|| seams(&proxies, &s.alignment, &s.gains, projection)) + .flatten(); + let blend = map.as_deref().map(|m| { + let full = m.scale * s.focal_full / s.alignment.focal; + (m, m.blend_radius(full, f64::from(request.seam_blend_px))) + }); let (w, h, mut rgb, mut known) = preview_planes( &colour, &proxies, &s.alignment, &s.gains, + blend, projection, first_raw, 1600, @@ -605,6 +622,13 @@ fn run_inner( } }; + // The seams, on the surface chosen, over the frames kept. + let seam_map = request.seams.then(|| { + let kept: Vec<&Gray> = solved.keep.iter().map(|&k| &proxies[k]).collect(); + seams(&kept, &solved.alignment, &solved.gains, projection) + }); + let seam_map = seam_map.flatten(); + // From here on only the kept frames exist, indexed as the alignment // indexes them. let Solved { @@ -704,6 +728,8 @@ fn run_inner( scale: focal_full, bounds, feather: request.feather_px, + seams: seam_map.clone(), + seam_blend: request.seam_blend_px, chunk: request.chunk, sample_scale: white_level as f32, balance, @@ -746,6 +772,8 @@ fn run_inner( max_v: bounds.max_v / q, }, feather: request.feather_px, + seams: seam_map.clone(), + seam_blend: request.seam_blend_px / q as f32, chunk: request.chunk, sample_scale: white_level as f32, balance, @@ -1227,21 +1255,51 @@ fn solve( }) } +/// Which frame each part of the composite comes from (`dr_pano::seam`), on +/// `projection` at the proxies' scale. `None` when the alignment is not +/// complete — the merge then falls back to the feathered average. +fn seams( + proxies: &[&Gray], + alignment: &Alignment, + gains: &[f32], + projection: Projection, +) -> Option> { + if !alignment.is_complete() { + return None; + } + let t = Instant::now(); + let map = dr_pano::seam::find( + proxies, + &alignment.cameras(), + gains, + projection, + &dr_pano::SeamOptions::default(), + ); + log::info!( + "merge: seams over {} frames in {:?}", + proxies.len(), + t.elapsed() + ); + map.map(Arc::new) +} + /// The aligned set on its surface, in colour, for the page. /// /// A quick look, not the pipeline: the first frame's white balance and -/// matrix, a gamma, and the frames averaged where they overlap with their -/// gains applied. Ghosting here is the alignment's error and banding is -/// the gains', which is exactly what the photographer is being asked to -/// look at. Fitted to 1600 px across. +/// matrix, a gamma, and the frames joined along the same seams the merge +/// will use, with their gains applied. Ghosting here is the alignment's +/// error and banding is the gains', which is exactly what the photographer +/// is being asked to look at. Fitted to 1600 px across. /// The aligned set on its surface, in colour, as planes: `(w, h, rgb 0..1, /// known)`. Display-ish space — the first frame's white balance and matrix, /// a gamma — which is also what the border filler was trained on. +#[allow(clippy::too_many_arguments)] fn preview_planes( colour: &[&[f32]], proxies: &[&Gray], alignment: &Alignment, gains: &[f32], + seams: Option<(&SeamMap, f64)>, projection: Projection, first: &RawImage, max_width: usize, @@ -1264,7 +1322,7 @@ fn preview_planes( let v = bounds.min_v + (oy as f64 + 0.5) * px; let d = projection.to_direction(scale, u, v); let mut sum = [0.0f32; 3]; - let mut n = 0u32; + let mut n = 0.0f32; for (k, g) in proxies.iter().enumerate() { let Some((x, y)) = cameras.project(k, d) else { continue; @@ -1276,16 +1334,21 @@ fn preview_planes( let i = (y as usize * g.width + x as usize) * 3; let cam = grey_if_blown([colour[k][i], colour[k][i + 1], colour[k][i + 2]], look.wb); + // The seam's share, as the merge weighs it, with the same + // floor under it; an even average where there is no map. + let w = seams + .and_then(|(m, radius)| m.share(k, u, v, scale, radius)) + .map_or(1.0, |s| s + 1e-4); for c in 0..3 { - sum[c] += cam[c] * gains[k]; + sum[c] += cam[c] * gains[k] * w; } - n += 1; + n += w; } - if n == 0 { + if n <= 0.0 { continue; } let o = oy * out_w + ox; - let d = look.to_display([sum[0] / n as f32, sum[1] / n as f32, sum[2] / n as f32]); + let d = look.to_display([sum[0] / n, sum[1] / n, sum[2] / n]); rgb[o * 3..o * 3 + 3].copy_from_slice(&d); known[o] = true; }