diff --git a/ui/dr-ui/src/inference.rs b/ui/dr-ui/src/inference.rs index 08055ff..aa2a06e 100644 --- a/ui/dr-ui/src/inference.rs +++ b/ui/dr-ui/src/inference.rs @@ -32,6 +32,9 @@ pub fn init(runtime_dirs: Vec) { Role::EyeClassifier, dir.join(crate::library::SUNGLASSES_MODEL), )); + if let Some(p) = crate::library::inpaint_model() { + models.push((Role::Inpainter, p)); + } models.retain(|(_, p)| p.is_file()); dr_inference_engine::init(dr_inference_engine::Config { diff --git a/ui/dr-ui/src/lib.rs b/ui/dr-ui/src/lib.rs index 7681a86..b6474ec 100644 --- a/ui/dr-ui/src/lib.rs +++ b/ui/dr-ui/src/lib.rs @@ -79,6 +79,7 @@ pub use develop::DevelopSession; /// Public for the Android entry point, which is the only caller that knows the /// APK may carry a bundled copy and has to write it out before any store is /// opened — see `library::shared_face_models_dir`. +pub use library::inpaint_model; pub use library::shared_face_models_dir; pub use library::FaceModelPaths; diff --git a/ui/dr-ui/src/library.rs b/ui/dr-ui/src/library.rs index 40b42ff..5fe1f26 100644 --- a/ui/dr-ui/src/library.rs +++ b/ui/dr-ui/src/library.rs @@ -4425,6 +4425,22 @@ fn system_face_models_dirs() -> Vec { .collect() } +/// TRACES: FR-MRG-4 +/// The panorama border filler, as shipped in `models/inpaint/`. +pub const INPAINT_MODEL: &str = "migan-512.onnx"; + +/// TRACES: FR-MRG-4 +/// Where the border filler is, if it is anywhere: the shared user +/// directory, then the system ones — the same search as the faces', minus +/// the per-account step, because a fill is not identity-bearing and no +/// library has a reason to pin its own. +pub fn inpaint_model() -> Option { + std::iter::once(shared_face_models_dir()) + .chain(system_face_models_dirs()) + .map(|d| d.join(INPAINT_MODEL)) + .find(|p| p.is_file()) +} + /// One grid cell's data, read from the catalog. #[derive(Debug, Clone, PartialEq, Eq)] pub struct LibraryCell { diff --git a/ui/dr-ui/src/merge.rs b/ui/dr-ui/src/merge.rs index c675f83..0dd713a 100644 --- a/ui/dr-ui/src/merge.rs +++ b/ui/dr-ui/src/merge.rs @@ -44,6 +44,12 @@ use dr_pipeline::EditGraph; pub use crate::export::Cancel; +/// The border filler's weights, where the app looks for them; for the +/// headless example, which has no library open. +pub fn inpaint_model_path() -> Option { + crate::library::inpaint_model() +} + /// One frame as the job receives it: the file's bytes, already fetched, /// and a name for messages and for the composite's own name. #[derive(Debug, Clone)] @@ -93,6 +99,9 @@ pub struct MergeRequest { pub feather_px: f32, /// GPU work unit; also the DNG strip height. pub chunk: (u32, u32), + /// The border filler's weights, if the device has them (FR-MRG-4). + /// `None` and the fill is not offered. + pub inpaint_model: Option, } impl MergeRequest { @@ -103,6 +112,7 @@ impl MergeRequest { projection: None, feather_px: 200.0, chunk: (2048, 512), + inpaint_model: None, } } } @@ -113,16 +123,76 @@ impl MergeRequest { pub enum Decision { Merge { projection: Option, + /// Fill the border with invented pixels rather than crop to the + /// picture (FR-MRG-4), with these knobs. Never the default; the + /// page asks. + fill: Option, }, /// Draw the alignment again on another surface, and report it again. /// The page's projection chips: what a choice looks like before it is - /// confirmed, at proxy cost rather than the merge's. + /// confirmed, at proxy cost rather than the merge's — and the fill, at + /// preview size, for the same reason: invented pixels are seen before + /// they are written. Preview { projection: Option, + fill: Option, }, Abandon, } +/// TRACES: FR-MRG-4 +/// The border fill's knobs, every one on the page while the fill is +/// experimental: the fill's own (`dr_pano::FillParams`, in pixels of the +/// working image) and the two around it. +#[derive(Debug, Clone, Copy, PartialEq)] +pub struct FillSettings { + /// The working image is the composite at `1/scale`: 2 is half, the + /// desktop's default; 4 a quarter, for the tablet (FR-MRG-9). + pub scale: u32, + /// How many working pixels of the coverage edge count as fringe and + /// are regenerated too. + pub erosion: usize, + pub params: dr_pano::FillParams, +} + +impl Default for FillSettings { + fn default() -> Self { + FillSettings { + scale: 2, + erosion: 4, + params: dr_pano::FillParams { + feather: 48, + ..Default::default() + }, + } + } +} + +impl FillSettings { + /// The same knobs for a picture at `ratio` of the composite's size + /// (the page's preview): the working-pixel counts rescaled so the fill + /// looks the same there as it will in the merge. + pub fn at(&self, ratio: f64) -> FillSettings { + let k = ratio * self.scale as f64; + let px = |v: usize| ((v as f64) * k).ceil().max(1.0) as usize; + FillSettings { + scale: 1, + erosion: px(self.erosion), + params: dr_pano::FillParams { + coarse: self.params.coarse, + band: px(self.params.band).max(8), + mirror_depth: px(self.params.mirror_depth), + feather: if self.params.feather == 0 { + 0 + } else { + px(self.params.feather) + }, + stride: self.params.stride, + }, + } + } +} + /// What the job says while it runs. #[derive(Debug, Clone)] pub enum MergeEvent { @@ -164,6 +234,10 @@ pub struct AlignmentReport { /// The aligned set drawn on the suggested surface at proxy resolution, /// `(width, height, rgba)` — what the photographer confirms. pub preview: Option<(u32, u32, Vec)>, + /// Whether the preview's border is filled (FR-MRG-4). + pub filled: bool, + /// The filler: where it runs, or why it is not available. + pub filler: Result, } /// Run the whole job on the calling thread, reporting on `events`. @@ -301,32 +375,78 @@ fn run_inner( } else { vec![1.0; frames.len()] }; - let report = |projection: Projection| -> Result { - let bounds = projection::bounds(projection, focal_full, &cameras, frame_size) - .ok_or("the frames project nowhere")?; - let preview = alignment.is_complete().then(|| { - preview( - &colour, - &proxies, - &alignment, - &gains, - projection, - &frames[0].raw, - ) - }); - Ok(AlignmentReport { - focal_mm: focal_full * 36.0 / full_long, - rms_px: alignment.rms_px, - projection, - width: bounds.width().ceil() as u32, - height: bounds.height().ceil() as u32, - frames: describe(&alignment), - links: alignment.links.len(), - preview, - }) + // The filler, loaded once if the device has it. Its absence is a + // report, not a failure: the crop is always there. + let mut filler: Option = None; + let filler_status: Result = match request.inpaint_model.as_deref() { + None => Err("no border-fill model on this device".into()), + Some(path) => match dr_pano::migan::MiGan::from_path(path) { + Ok(m) => { + let rung = m.rung().map(|r| r.label().to_string()).unwrap_or_default(); + filler = Some(m); + Ok(rung) + } + Err(e) => Err(format!("the border-fill model could not be loaded: {e}")), + }, }; + let first_raw = &frames[0].raw; + let mut report = + |projection: Projection, fill: Option| -> Result { + let bounds = projection::bounds(projection, focal_full, &cameras, frame_size) + .ok_or("the frames project nowhere")?; + let mut filled = false; + let preview = match alignment.is_complete() { + false => None, + true => { + let (w, h, mut rgb, mut known) = preview_planes( + &colour, &proxies, &alignment, &gains, projection, first_raw, 1600, + ); + if let Some(settings) = fill { + if let Some(model) = filler.as_mut() { + progress("Filling the preview", 0, 1); + // The knobs at the preview's scale. + let here = settings.at(w as f64 / bounds.width()); + dr_pano::fill::erode(&mut known, w, h, here.erosion); + match dr_pano::fill_border( + &mut rgb, + w, + h, + &known, + model, + here.params, + &mut |_, _| {}, + ) { + Ok(_) => filled = true, + Err(e) => log::warn!("preview fill: {e}"), + } + } + } + Some(preview_rgba( + &rgb, + w, + h, + &alignment, + projection, + &bounds_of(&proxies, &alignment, projection), + &proxies, + )) + } + }; + Ok(AlignmentReport { + focal_mm: focal_full * 36.0 / full_long, + rms_px: alignment.rms_px, + projection, + width: bounds.width().ceil() as u32, + height: bounds.height().ceil() as u32, + frames: describe(&alignment), + links: alignment.links.len(), + preview, + filled, + filler: filler_status.clone(), + }) + }; progress("Drawing the preview", 0, 1); - let _ = events.send(MergeEvent::Aligned(report(suggested)?)); + let _ = events.send(MergeEvent::Aligned(report(suggested, None)?)); if !alignment.is_complete() { let names: Vec = alignment .unaligned @@ -342,14 +462,20 @@ fn run_inner( // Never automatic (FR-MRG-1): nothing is written until the alignment // has been seen and confirmed. Polled, so a cancel while waiting is // seen within a moment. - let projection = loop { + let (projection, fill) = loop { if cancel.is_cancelled() { return Ok(None); } match decision.recv_timeout(std::time::Duration::from_millis(100)) { - Ok(Decision::Merge { projection: p }) => break p.unwrap_or(suggested), - Ok(Decision::Preview { projection: p }) => { - let _ = events.send(MergeEvent::Aligned(report(p.unwrap_or(suggested))?)); + Ok(Decision::Merge { + projection: p, + fill, + }) => break (p.unwrap_or(suggested), fill.filter(|_| filler.is_some())), + Ok(Decision::Preview { + projection: p, + fill, + }) => { + let _ = events.send(MergeEvent::Aligned(report(p.unwrap_or(suggested), fill)?)); } Ok(Decision::Abandon) => return Ok(None), Err(std::sync::mpsc::RecvTimeoutError::Timeout) => continue, @@ -411,6 +537,134 @@ fn run_inner( }) .collect(); + let t = Instant::now(); + let mut pass = MergePass::new(ctx).map_err(|e| e.to_string())?; + let mut resident: VecDeque<(usize, Arc)> = VecDeque::new(); + + // 5a. The border fill (FR-MRG-4), when asked for: the composite at a + // quarter of its resolution, its border filled by the model in display + // space, and the result taken back to camera space for the full merge + // to sample wherever nothing was covered. A quarter because the fill is + // sky and slope at the edge of a picture, where a quarter is more than + // the eye resolves, and because it puts the fixture's border at thirty + // tiles rather than a hundred. + let fill_cam: Option<(usize, usize, Vec, Vec)> = if let Some(settings) = fill { + let model = filler.as_mut().expect("fill was only chosen with a filler"); + progress("Filling the border", 0, 1); + // Half the composite's resolution by default: a quarter left a + // visible gap in sharpness against the real pixels beside the + // fill. Memory is the working image in f32, 400 MB for a 137 MP + // composite; the tablet will want a quarter (FR-MRG-9). + let q = settings.scale.max(1) as f64; + let qout = MergeOutput { + projection, + scale: focal_full / q, + bounds: projection::Bounds { + min_u: bounds.min_u / q, + min_v: bounds.min_v / q, + max_u: bounds.max_u / q, + max_v: bounds.max_v / q, + }, + feather: request.feather_px, + chunk: request.chunk, + sample_scale: white_level as f32, + }; + let (qw, qh) = (qout.width() as usize, qout.height() as usize); + let mut qrgb = vec![0.0f32; qw * qh * 3]; + let mut qknown = vec![false; qw * qh]; + pass.merge( + &mut adjust, + &merge_frames, + &cameras, + (frames[0].size.0, frames[0].size.1), + &qout, + |k| { + if let Some((_, img)) = resident.iter().find(|(i, _)| *i == k) { + return Ok(img.clone()); + } + let img = Arc::new(demosaicer.run(&frames[k].raw)?); + resident.push_back((k, img.clone())); + while resident.len() > 2 { + resident.pop_front(); + } + Ok(img) + }, + |band| { + let first = band.first_row as usize; + for r in 0..band.rows as usize { + for x in 0..qw { + let src = r * qw + x; + let dst = (first + r) * qw + x; + for c in 0..3 { + qrgb[dst * 3 + c] = + f32::from(band.rgb[src * 3 + c]) / white_level as f32; + } + qknown[dst] = band.covered[src]; + } + } + Ok(()) + }, + || cancel.is_cancelled(), + ) + .map_err(|e| e.to_string())?; + if cancel.is_cancelled() { + cleanup(&out_path); + return Ok(None); + } + // Camera space to the display-ish space the model was trained on. + let look = Look::of(first_raw); + let mut disp = qrgb.clone(); + for p in disp.chunks_exact_mut(3) { + let d = look.to_display([p[0], p[1], p[2]]); + p.copy_from_slice(&d); + } + dr_pano::fill::erode(&mut qknown, qw, qh, settings.erosion); + // `DR_FILL_DUMP=dir` writes what the filler was given, so a fill + // can be re-run and looked at stage by stage without the merge + // (`cargo run -p dr-pano --example fill`). + if let Some(dir) = std::env::var_os("DR_FILL_DUMP") { + let dir = PathBuf::from(dir); + if let Err(e) = dump_fill_input(&dir, &disp, &qknown, qw, qh) { + log::warn!("fill dump: {e}"); + } + } + let total_tiles = std::cell::Cell::new(0usize); + let feather = settings.params.feather; + dr_pano::fill_border( + &mut disp, + qw, + qh, + &qknown, + model, + settings.params, + &mut |n, total| { + total_tiles.set(total); + progress("Filling the border", n, total.max(1)); + }, + ) + .map_err(|e| e.to_string())?; + // And back: the filled pixels — the hole and the feathered margin — + // to camera space; the rest as the merge produced them. + let mut margin = qknown.clone(); + dr_pano::fill::erode(&mut margin, qw, qh, feather); + for (i, p) in qrgb.chunks_exact_mut(3).enumerate() { + if !margin[i] { + let d = &disp[i * 3..i * 3 + 3]; + let cam = look.to_camera([d[0], d[1], d[2]]); + p.copy_from_slice(&cam); + } + } + let fill_mask: Vec = margin; + log::info!( + "merge: border filled at {qw}×{qh}, {} tiles, in {:?}", + total_tiles.get(), + t.elapsed() + ); + Some((qw, qh, qrgb, fill_mask)) + } else { + None + }; + // The writer pulls strips; the merge pushes bands. A channel between // them, and the writer on its own thread, so neither waits on the // other's pace more than one band. @@ -420,6 +674,7 @@ fn run_inner( // once the last strip is in, by which time every band has been seen. let inscribed = Arc::new(std::sync::Mutex::new(dr_export::Inscribed::new(out_w))); let inscribed_for_writer = inscribed.clone(); + let fill_for_writer = fill_cam.is_some(); let file = std::fs::File::create(&out_path).map_err(|e| format!("{}: {e}", out_path.display()))?; let writer = std::thread::spawn(move || -> Result<(), String> { @@ -438,14 +693,19 @@ fn run_inner( buf.extend_from_slice(&band); Ok(()) }, - || inscribed_for_writer.lock().ok().map(|i| i.best()), + // A filled composite is the whole picture; a cropped one opens + // on the rectangle the frames covered. + move || { + if fill_for_writer { + None + } else { + inscribed_for_writer.lock().ok().map(|i| i.best()) + } + }, ) .map_err(|e| e.to_string()) }); - let t = Instant::now(); - let mut pass = MergePass::new(ctx).map_err(|e| e.to_string())?; - let mut resident: VecDeque<(usize, Arc)> = VecDeque::new(); let total_bands = out_h.div_ceil(rows_per_strip) as usize; let mut bands_done = 0usize; progress("Merging", 0, total_bands); @@ -472,8 +732,38 @@ fn run_inner( if let Ok(mut i) = inscribed.lock() { i.push_rows(band.covered, band.rows); } + let mut rgb = band.rgb.to_vec(); + // Where no frame reached, the fill — sampled bilinearly from + // the quarter-resolution result, at the sensor's scale. + if let Some((qw, qh, qrgb, fill_mask)) = &fill_cam { + let sx = *qw as f64 / out_w as f64; + let sy = *qh as f64 / out_h as f64; + for r in 0..band.rows as usize { + for x in 0..out_w as usize { + let i = r * out_w as usize + x; + let fx = ((x as f64 + 0.5) * sx - 0.5).clamp(0.0, (*qw - 1) as f64); + let fy = (((band.first_row as usize + r) as f64 + 0.5) * sy - 0.5) + .clamp(0.0, (*qh - 1) as f64); + // Uncovered, or inside the feathered margin: the fill. + let (mx, my) = (fx.round() as usize, fy.round() as usize); + if band.covered[i] && fill_mask[my.min(qh - 1) * qw + mx.min(qw - 1)] { + continue; + } + let (x0, y0) = (fx as usize, fy as usize); + let (x1, y1) = ((x0 + 1).min(qw - 1), (y0 + 1).min(qh - 1)); + let (tx, ty) = ((fx - x0 as f64) as f32, (fy - y0 as f64) as f32); + for c in 0..3 { + let at = |xx: usize, yy: usize| qrgb[(yy * qw + xx) * 3 + c]; + let v = (at(x0, y0) * (1.0 - tx) + at(x1, y0) * tx) * (1.0 - ty) + + (at(x0, y1) * (1.0 - tx) + at(x1, y1) * tx) * ty; + rgb[i * 3 + c] = + (v * white_level as f32).round().clamp(0.0, 65535.0) as u16; + } + } + } + } band_tx - .send(band.rgb.to_vec()) + .send(rgb) .map_err(|_| dr_gpu::GpuError::Readback("the writer stopped".into())) }, || cancel.is_cancelled(), @@ -514,14 +804,23 @@ fn run_inner( let mut sidecar = dr_pipeline::Sidecar::new(); sidecar.derived_from = request.frames.iter().map(|f| f.name.clone()).collect(); sidecar.merge = Some(format!( - "panorama {} {:.1}mm {} frames", + "panorama {} {:.1}mm {} frames{}", match projection { Projection::Perspective => "perspective", Projection::Cylindrical => "cylindrical", Projection::Spherical => "spherical", }, focal_full * 36.0 / full_long, - frames.len() + frames.len(), + // Invented pixels declare themselves (FR-MRG-6, D17's rule), with + // the knobs that made them while the fill is experimental. + match fill { + Some(f) => format!( + " border filled (1/{} scale, erosion {}, coarse {}, band {}, mirror {}, feather {})", + f.scale, f.erosion, f.params.coarse, f.params.band, f.params.mirror_depth, f.params.feather + ), + None => String::new(), + } )); let sidecar_path = out_path.with_extension(dr_pipeline::sidecar::EXTENSION); if let Err(e) = std::fs::write(&sidecar_path, sidecar.to_text()) { @@ -597,28 +896,30 @@ fn camera_proxy( /// gains applied. Ghosting here is the alignment's error and banding is /// the gains', which is exactly what the photographer is being asked to /// look at. Fitted to 1600 px across. -fn preview( +/// The aligned set on its surface, in colour, as planes: `(w, h, rgb 0..1, +/// known)`. Display-ish space — the first frame's white balance and matrix, +/// a gamma — which is also what the border filler was trained on. +fn preview_planes( colour: &[Vec], proxies: &[Gray], alignment: &Alignment, gains: &[f32], projection: Projection, first: &RawImage, -) -> (u32, u32, Vec) { + max_width: usize, +) -> (usize, usize, Vec, Vec) { let cameras = alignment.cameras(); let (fw, fh) = (proxies[0].width as f64, proxies[0].height as f64); let scale = alignment.focal; let Some(bounds) = projection::bounds(projection, scale, &cameras, (fw, fh)) else { - return (0, 0, Vec::new()); + return (0, 0, Vec::new(), Vec::new()); }; - let out_w = 1600usize.min(bounds.width().ceil() as usize).max(1); + let out_w = max_width.min(bounds.width().ceil() as usize).max(1); let px = bounds.width() / out_w as f64; let out_h = ((bounds.height() / px).ceil() as usize).max(1); - let wb = first.wb_coeffs; - let m = first - .color_matrix - .unwrap_or([1.0, 0.0, 0.0, 0.0, 1.0, 0.0, 0.0, 0.0, 1.0]); - let mut out = vec![0u8; out_w * out_h * 4]; + let look = Look::of(first); + let mut rgb = vec![0.0f32; out_w * out_h * 3]; + let mut known = vec![false; out_w * out_h]; for oy in 0..out_h { for ox in 0..out_w { let u = bounds.min_u + (ox as f64 + 0.5) * px; @@ -640,30 +941,60 @@ fn preview( } n += 1; } - let o = (oy * out_w + ox) * 4; if n == 0 { - out[o + 3] = 255; continue; } - let cam = [ - sum[0] / n as f32 * wb[0], - sum[1] / n as f32 * wb[1], - sum[2] / n as f32 * wb[2], - ]; - for c in 0..3 { - let lin = m[c * 3] * cam[0] + m[c * 3 + 1] * cam[1] + m[c * 3 + 2] * cam[2]; - out[o + c] = (lin.clamp(0.0, 1.0).powf(1.0 / 2.2) * 255.0) as u8; - } - out[o + 3] = 255; + let o = oy * out_w + ox; + let d = look.to_display([sum[0] / n as f32, sum[1] / n as f32, sum[2] / n as f32]); + rgb[o * 3..o * 3 + 3].copy_from_slice(&d); + known[o] = true; } } + (out_w, out_h, rgb, known) +} - // Each frame's outline, so the page shows *frames* and not one picture: - // a sweep of frames overlapping by more than half reads as a single - // photograph otherwise, which is what a photographer looking for a - // misplaced frame needs to see through. Walked along each border and - // drawn where it lands, two pixels wide, in a colour the scene does not - // have. +fn bounds_of( + proxies: &[Gray], + alignment: &Alignment, + projection: Projection, +) -> projection::Bounds { + let (fw, fh) = (proxies[0].width as f64, proxies[0].height as f64); + projection::bounds(projection, alignment.focal, &alignment.cameras(), (fw, fh)).unwrap_or( + projection::Bounds { + min_u: 0.0, + min_v: 0.0, + max_u: 1.0, + max_v: 1.0, + }, + ) +} + +/// The planes to RGBA for the page, with each frame outlined. +/// +/// The outlines, so the page shows *frames* and not one picture: a sweep +/// of frames overlapping by more than half reads as a single photograph +/// otherwise, which is what a photographer looking for a misplaced frame +/// needs to see through. Walked along each border and drawn where it +/// lands, two pixels wide, in a colour the scene does not have. +fn preview_rgba( + rgb: &[f32], + out_w: usize, + out_h: usize, + alignment: &Alignment, + projection: Projection, + bounds: &projection::Bounds, + proxies: &[Gray], +) -> (u32, u32, Vec) { + let cameras = alignment.cameras(); + let (fw, fh) = (proxies[0].width as f64, proxies[0].height as f64); + let scale = alignment.focal; + let px = bounds.width() / out_w.max(1) as f64; + let mut out = vec![255u8; out_w * out_h * 4]; + for i in 0..out_w * out_h { + for c in 0..3 { + out[i * 4 + c] = (rgb[i * 3 + c].clamp(0.0, 1.0) * 255.0) as u8; + } + } let steps = 600; for k in 0..proxies.len() { for s in 0..steps { @@ -697,6 +1028,80 @@ fn preview( (out_w as u32, out_h as u32, out) } +/// The quick look from camera space to something like sRGB and back: the +/// first frame's white balance and matrix, a 2.2 gamma. Not the pipeline — +/// the page's preview and the filler's input, both of which want a +/// plausible picture rather than a calibrated one — and invertible, so the +/// filler's answer comes back to camera space where the DNG lives. +struct Look { + wb: [f32; 3], + m: [f32; 9], + inv: [f32; 9], +} + +impl Look { + fn of(raw: &RawImage) -> Self { + let wb = [raw.wb_coeffs[0], raw.wb_coeffs[1], raw.wb_coeffs[2]]; + let m = raw + .color_matrix + .unwrap_or([1.0, 0.0, 0.0, 0.0, 1.0, 0.0, 0.0, 0.0, 1.0]); + Look { + wb, + m, + inv: invert3(&m).unwrap_or([1.0, 0.0, 0.0, 0.0, 1.0, 0.0, 0.0, 0.0, 1.0]), + } + } + + fn to_display(&self, cam: [f32; 3]) -> [f32; 3] { + let b = [ + cam[0] * self.wb[0], + cam[1] * self.wb[1], + cam[2] * self.wb[2], + ]; + let row = + |c: usize| self.m[c * 3] * b[0] + self.m[c * 3 + 1] * b[1] + self.m[c * 3 + 2] * b[2]; + let encode = |lin: f32| lin.clamp(0.0, 1.0).powf(1.0 / 2.2); + [encode(row(0)), encode(row(1)), encode(row(2))] + } + + fn to_camera(&self, disp: [f32; 3]) -> [f32; 3] { + let lin = [ + disp[0].max(0.0).powf(2.2), + disp[1].max(0.0).powf(2.2), + disp[2].max(0.0).powf(2.2), + ]; + let row = |c: usize| { + self.inv[c * 3] * lin[0] + self.inv[c * 3 + 1] * lin[1] + self.inv[c * 3 + 2] * lin[2] + }; + let b = [row(0), row(1), row(2)]; + [ + (b[0] / self.wb[0].max(1e-4)).max(0.0), + (b[1] / self.wb[1].max(1e-4)).max(0.0), + (b[2] / self.wb[2].max(1e-4)).max(0.0), + ] + } +} + +fn invert3(m: &[f32; 9]) -> Option<[f32; 9]> { + let (a, b, c, d, e, f, g, h, i) = (m[0], m[1], m[2], m[3], m[4], m[5], m[6], m[7], m[8]); + let det = a * (e * i - f * h) - b * (d * i - f * g) + c * (d * h - e * g); + if det.abs() < 1e-12 { + return None; + } + let s = 1.0 / det; + Some([ + (e * i - f * h) * s, + (c * h - b * i) * s, + (b * f - c * e) * s, + (f * g - d * i) * s, + (a * i - c * g) * s, + (c * d - a * f) * s, + (d * h - e * g) * s, + (b * g - a * h) * s, + (a * e - b * d) * s, + ]) +} + /// The horizontal and vertical field of view the cameras span, in radians: /// the angle between the extreme frame centres plus one frame's own field. fn field_of_view(cameras: &Cameras, frame: (f64, f64)) -> (f64, f64) { @@ -865,3 +1270,24 @@ pub fn drain(rx: &Receiver) -> Vec { } out } + +/// The filler's input as the debugging example reads it: `input.ppm`, the +/// display-space picture, and `known.pgm`, 255 where a frame reached. +fn dump_fill_input( + dir: &Path, + rgb: &[f32], + known: &[bool], + w: usize, + h: usize, +) -> std::io::Result<()> { + std::fs::create_dir_all(dir)?; + let mut ppm = format!("P6\n{w} {h}\n255\n").into_bytes(); + ppm.extend( + rgb.iter() + .map(|v| (v.clamp(0.0, 1.0) * 255.0).round() as u8), + ); + std::fs::write(dir.join("input.ppm"), ppm)?; + let mut pgm = format!("P5\n{w} {h}\n255\n").into_bytes(); + pgm.extend(known.iter().map(|&k| if k { 255u8 } else { 0 })); + std::fs::write(dir.join("known.pgm"), pgm) +} diff --git a/ui/dr-ui/src/merge_ui.rs b/ui/dr-ui/src/merge_ui.rs index cff05cb..60452b3 100644 --- a/ui/dr-ui/src/merge_ui.rs +++ b/ui/dr-ui/src/merge_ui.rs @@ -19,7 +19,8 @@ use slint::ComponentHandle; use crate::activity::{Activity, ActivityLog, Kind}; use crate::merge::{ - self, AlignmentReport, Cancel, Decision, MergeDestination, MergeEvent, MergeInput, MergeRequest, + self, AlignmentReport, Cancel, Decision, FillSettings, MergeDestination, MergeEvent, + MergeInput, MergeRequest, }; use crate::{AppWindow, MergeFrameRow}; @@ -41,9 +42,21 @@ pub struct MergeController { timer: RefCell>, /// The projection chip: 0 auto, then the three in `Projection`'s order. projection: Cell, + /// The border chip: 0 crop, 1 fill (FR-MRG-4). + border: Cell, + /// The fill's knobs, as the page's sliders have them. + fill: Cell, report: RefCell>, } +impl MergeController { + /// The fill to ask for: the knobs if the border chip says fill, else + /// none. + fn fill_choice(&self) -> Option { + (self.border.get() == 1).then(|| self.fill.get()) + } +} + impl MergeController { pub fn new(activity: Rc) -> Rc { Rc::new(MergeController { @@ -51,6 +64,8 @@ impl MergeController { job: RefCell::new(None), timer: RefCell::new(None), projection: Cell::new(0), + border: Cell::new(0), + fill: Cell::new(FillSettings::default()), report: RefCell::new(None), }) } @@ -235,6 +250,19 @@ pub fn wire( fetch, &on_done, ); + // `DARKROOM_START_MERGE_FILL=1` opens it with the fill chosen, + // to look at the knobs; the preview fills once the alignment + // is in, as a chip pick would have it. + if std::env::var_os("DARKROOM_START_MERGE_FILL").is_some() { + ctl.border.set(1); + window.set_merge_border_selected(1); + if let Some(job) = ctl.job.borrow().as_ref() { + let _ = job.decide.send(Decision::Preview { + projection: None, + fill: ctl.fill_choice(), + }); + } + } } } @@ -245,8 +273,9 @@ pub fn wire( window.on_merge_confirm(move || { let Some(w) = weak.upgrade() else { return }; let projection = chip_projection(ctl.projection.get()); + let fill = ctl.fill_choice(); if let Some(job) = ctl.job.borrow().as_ref() { - let _ = job.decide.send(Decision::Merge { projection }); + let _ = job.decide.send(Decision::Merge { projection, fill }); job.activity.detail("merging"); } w.set_merge_running(true); @@ -266,10 +295,71 @@ pub fn wire( if let Some(job) = ctl.job.borrow().as_ref() { let _ = job.decide.send(Decision::Preview { projection: chip_projection(i), + fill: ctl.fill_choice(), }); } }); } + { + let weak = window.as_weak(); + let ctl = ctl.clone(); + window.on_merge_border_picked(move |i| { + let Some(w) = weak.upgrade() else { return }; + ctl.border.set(i); + w.set_merge_border_selected(i); + // The fill is shown before it is chosen for good: the job + // redraws the preview with the border filled, or not. + if let Some(job) = ctl.job.borrow().as_ref() { + let _ = job.decide.send(Decision::Preview { + projection: chip_projection(ctl.projection.get()), + fill: ctl.fill_choice(), + }); + } + w.set_merge_running(true); + w.set_merge_stage( + if i == 1 { + "Filling the preview" + } else { + "Drawing the preview" + } + .into(), + ); + }); + } + + { + let weak = window.as_weak(); + let ctl = ctl.clone(); + window.on_merge_fill_knob(move |name, value| { + let Some(w) = weak.upgrade() else { return }; + let mut f = ctl.fill.get(); + let v = value.round().max(0.0) as usize; + match name.as_str() { + "scale" => f.scale = (v as u32).clamp(1, 8), + "erosion" => f.erosion = v, + "coarse" => f.params.coarse = v.max(1), + "band" => f.params.band = v.max(8), + "mirror" => f.params.mirror_depth = v.max(1), + "feather" => f.params.feather = v, + _ => return, + } + ctl.fill.set(f); + show_fill_knobs(&w, &f); + // A knob turned while the fill is chosen redraws the preview + // with it; turned under the crop it waits for the chip. + if ctl.border.get() != 1 { + return; + } + if let Some(job) = ctl.job.borrow().as_ref() { + let _ = job.decide.send(Decision::Preview { + projection: chip_projection(ctl.projection.get()), + fill: Some(f), + }); + } + w.set_merge_running(true); + w.set_merge_stage("Filling the preview".into()); + }); + } // --- stopping and leaving -------------------------------------------- { @@ -332,7 +422,8 @@ fn start( let Some(frames) = fetch(&tx, &cancel) else { return; }; - let request = MergeRequest::new(frames, destination); + let mut request = MergeRequest::new(frames, destination); + request.inpaint_model = crate::library::inpaint_model(); merge::run(gpu, request, tx, decision, cancel); }); } @@ -346,6 +437,9 @@ fn start( }); *ctl.report.borrow_mut() = None; ctl.projection.set(0); + ctl.border.set(0); + ctl.fill.set(FillSettings::default()); + show_fill_knobs(window, &FillSettings::default()); // The page, from a clean slate. window.set_merge_stage("Fetching".into()); @@ -361,6 +455,10 @@ fn start( >::new()))); window.set_merge_preview(slint::Image::default()); window.set_merge_projection_selected(0); + window.set_merge_border_selected(0); + window.set_merge_fill_available(false); + window.set_merge_fill_note("".into()); + window.set_merge_preview_filled(false); window.set_merge_done(false); window.set_show_merge(true); @@ -530,4 +628,25 @@ fn show_report(window: &AppWindow, report: &AlignmentReport, names: &[String]) { let buffer = slint::SharedPixelBuffer::::clone_from_slice(rgba, *w, *h); window.set_merge_preview(slint::Image::from_rgba8(buffer)); } + window.set_merge_preview_filled(report.filled); + match &report.filler { + Ok(rung) => { + window.set_merge_fill_available(true); + window.set_merge_fill_note(format!("runs on {rung}").into()); + } + Err(why) => { + window.set_merge_fill_available(false); + window.set_merge_fill_note(format!("The border can only be cropped: {why}.").into()); + } + } +} + +/// The sliders, from the settings. +fn show_fill_knobs(window: &AppWindow, f: &FillSettings) { + window.set_merge_fill_scale(f.scale as f32); + window.set_merge_fill_erosion(f.erosion as f32); + window.set_merge_fill_coarse(f.params.coarse as f32); + window.set_merge_fill_band(f.params.band as f32); + window.set_merge_fill_mirror(f.params.mirror_depth as f32); + window.set_merge_fill_feather(f.params.feather as f32); } diff --git a/ui/dr-ui/ui/app.slint b/ui/dr-ui/ui/app.slint index 8267239..ac9ec4b 100644 --- a/ui/dr-ui/ui/app.slint +++ b/ui/dr-ui/ui/app.slint @@ -840,6 +840,17 @@ export component AppWindow inherits Window { in property <[MergeFrameRow]> merge-frames; in property merge-preview; in property merge-projection-selected: 0; + in property merge-border-selected: 0; + in property merge-fill-available: false; + in property merge-fill-note: ""; + in property merge-preview-filled: false; + // The fill's knobs (experimental; FR-MRG-4), in working pixels. + in property merge-fill-scale: 2; + in property merge-fill-erosion: 4; + in property merge-fill-coarse: 4; + in property merge-fill-band: 96; + in property merge-fill-mirror: 48; + in property merge-fill-feather: 48; in property merge-done: false; /// Merge the grid's selection: opens the page and starts the alignment. callback library-merge-selection(); @@ -847,6 +858,9 @@ export component AppWindow inherits Window { callback merge-confirm(); callback merge-abandon(); callback merge-projection-picked(int); + callback merge-border-picked(int); + /// A fill knob turned: its name and the new value. + callback merge-fill-knob(string, float); callback merge-close(); callback import-open(); @@ -1312,10 +1326,22 @@ in property panel-visible: true; frames: root.merge-frames; preview: root.merge-preview; projection-selected: root.merge-projection-selected; + border-selected: root.merge-border-selected; + fill-available: root.merge-fill-available; + fill-note: root.merge-fill-note; + preview-filled: root.merge-preview-filled; + fill-scale: root.merge-fill-scale; + fill-erosion: root.merge-fill-erosion; + fill-coarse: root.merge-fill-coarse; + fill-band: root.merge-fill-band; + fill-mirror: root.merge-fill-mirror; + fill-feather: root.merge-fill-feather; done: root.merge-done; merge() => { root.merge-confirm(); } abandon() => { root.merge-abandon(); } projection-picked(i) => { root.merge-projection-picked(i); } + border-picked(i) => { root.merge-border-picked(i); } + fill-knob(name, value) => { root.merge-fill-knob(name, value); } close() => { root.merge-close(); } } diff --git a/ui/dr-ui/ui/merge.slint b/ui/dr-ui/ui/merge.slint index 158d54e..89c530c 100644 --- a/ui/dr-ui/ui/merge.slint +++ b/ui/dr-ui/ui/merge.slint @@ -11,7 +11,7 @@ import { Theme } from "theme.slint"; import { Button, PanelHeading, Label, Value, Caption, Panel, ProgressBar } from "widgets.slint"; -import { Segmented } from "controls.slint"; +import { Segmented, SliderRow } from "controls.slint"; /// One source frame's line in the alignment table. export struct MergeFrameRow { @@ -43,6 +43,31 @@ export component MergePage inherits Rectangle { in property <[string]> projection-labels: ["Auto", "Perspective", "Cylindrical", "Spherical"]; in property projection-selected: 0; + // TRACES: FR-MRG-4 + // The border: cropped to the picture, or filled with invented pixels. + // Crop is the default and the honest one; the fill is offered only + // where the device has the model, and what it invents is on the + // preview before it is confirmed. + in property <[string]> border-labels: ["Crop to the picture", "Fill the border"]; + in property border-selected: 0; + /// Whether the fill can be offered at all, and where it would run — + /// "TensorRT", "CPU" — or why not. + in property fill-available: false; + in property fill-note; + /// Whether the preview on screen has its border filled. + in property preview-filled: false; + /// The fill's knobs — experimental, so every one is on the page: the + /// working scale (the composite's resolution over this), the coverage + /// fringe eroded, the coarse pass's reduction, the fine band width, + /// how deep the mirrored context reaches, and the feather across the + /// seam. Working pixels, at the merge's working scale. + in property fill-scale: 2; + in property fill-erosion: 4; + in property fill-coarse: 4; + in property fill-band: 96; + in property fill-mirror: 48; + in property fill-feather: 48; + /// The composite, once written: its name for the status line, and /// whether Open can show it. in property done: false; @@ -50,6 +75,10 @@ export component MergePage inherits Rectangle { callback merge(); callback abandon(); callback projection-picked(int); + callback border-picked(int); + /// A knob turned: its name ("scale", "erosion", "coarse", "band", + /// "mirror", "feather") and the new value. + callback fill-knob(string, float); callback close(); background: Theme.ground; @@ -168,6 +197,79 @@ export component MergePage inherits Rectangle { enabled: !root.running && !root.done; picked(i) => { root.projection-picked(i); } } + + Segmented { + label: "Border"; + options: root.border-labels; + selected: root.border-selected; + enabled: !root.running && !root.done && root.fill-available; + picked(i) => { root.border-picked(i); } + } + // Said plainly, because these are pixels the camera never + // saw: what the fill is, where it runs, and that the + // composite will say it was filled. + Caption { + text: root.fill-available + ? (root.preview-filled + ? "The border above is filled with pixels the model invented from the picture beside them (" + root.fill-note + "). The composite records that it was filled." + : "Filling the border invents pixels from the picture beside them (" + root.fill-note + "). Choose it to see what it would do before merging.") + : root.fill-note; + wrap: word-wrap; + } + // The knobs, while the fill is experimental: turned + // under the fill, the preview redraws with them. + if root.fill-available && root.border-selected == 1 : VerticalLayout { + spacing: 4px; + Caption { text: "Experimental — the fill's knobs. Each redraws the preview."; wrap: word-wrap; } + SliderRow { + label: "Working scale"; + hint: "the composite's resolution over this: 2 is half, 4 a quarter"; + value: root.fill-scale; default-value: 2; minimum: 1; maximum: 8; + enabled: !root.running && !root.done; + changed(v) => { root.fill-knob("scale", v); } + reset => { root.fill-knob("scale", 2); } + } + SliderRow { + label: "Edge erosion"; + hint: "working pixels of the coverage edge regenerated as fringe"; + value: root.fill-erosion; default-value: 4; minimum: 0; maximum: 64; + enabled: !root.running && !root.done; + changed(v) => { root.fill-knob("erosion", v); } + reset => { root.fill-knob("erosion", 4); } + } + SliderRow { + label: "Coarse pass"; + hint: "reduction of the structure pass; 1 skips it"; + value: root.fill-coarse; default-value: 4; minimum: 1; maximum: 8; + enabled: !root.running && !root.done; + changed(v) => { root.fill-knob("coarse", v); } + reset => { root.fill-knob("coarse", 4); } + } + SliderRow { + label: "Band width"; + hint: "the fine passes' bands from the edge outward, working pixels"; + value: root.fill-band; default-value: 96; minimum: 16; maximum: 384; + enabled: !root.running && !root.done; + changed(v) => { root.fill-knob("band", v); } + reset => { root.fill-knob("band", 96); } + } + SliderRow { + label: "Mirror depth"; + hint: "how far into the picture the mirrored context reaches"; + value: root.fill-mirror; default-value: 48; minimum: 4; maximum: 512; + enabled: !root.running && !root.done; + changed(v) => { root.fill-knob("mirror", v); } + reset => { root.fill-knob("mirror", 48); } + } + SliderRow { + label: "Seam feather"; + hint: "working pixels inside the real edge blended with the fill; 0 is a hard cut"; + value: root.fill-feather; default-value: 48; minimum: 0; maximum: 256; + enabled: !root.running && !root.done; + changed(v) => { root.fill-knob("feather", v); } + reset => { root.fill-knob("feather", 48); } + } + } } // --- the frames ----------------------------------------