//! TRACES: FR-MRG-1 | FR-MRG-2 | FR-MRG-3 | FR-MRG-5 | FR-MRG-7 //! The merge job: from a set of files to a linear DNG beside them. //! //! The orchestration of a panorama, with no interface types in it: it runs //! on a thread of its own and reports through a channel, on the pattern //! [`crate::export`]'s batch established (FR-MRG-7). The interface drains //! the channel from a timer; a headless example drains it from a loop. //! //! # Stages //! //! 1. **Load.** Each file is read and decoded to sensor data, and its edit //! graph built the way a session would — orientation, lens profile — //! because those are the two things the camera-space tap uses //! (FR-MRG-2). //! 2. **Proxies.** Each frame is demosaiced and rendered through the tap //! at proxy size, upright; the keypoint detector reads that. //! 3. **Alignment.** `dr_pano::match_pairs` once, then `dr_pano::solve` //! over the frames kept — again whenever the page leaves one out. A //! frame it could not place is named (FR-MRG-5), and the merge waits //! until it is left out. //! 4. **Gain.** One scalar per frame from the proxies' overlaps, so the //! stop of exposure drift a hand-held sweep collects does not band. //! 5. **Merge.** `dr_gpu::MergePass`, chunk by chunk, into a DNG written //! strip by strip beside the first source (FR-MRG-3). //! //! The demosaiced frames are the memory: 160 MB each at f16 for a 20 MP //! body, so at most two are resident and the rest are demosaiced again //! when a band needs them. That is the trade FR-MRG-11 asks for — the //! sensor data stays on the CPU at a fifth of the size — and it is what //! makes a twelve-frame set fit. use crate::executors::{self, Executor}; use std::collections::VecDeque; use std::path::{Path, PathBuf}; use std::sync::mpsc::{Receiver, Sender}; use std::sync::Arc; use std::time::Instant; use dr_decode::RawImage; use dr_gpu::{ AdjustPass, DemosaicedImage, Demosaicer, GpuContext, MergeFrame, MergeOutput, MergePass, }; use dr_pano::bundle::Cameras; use dr_pano::projection::{self, Projection}; use dr_pano::{Alignment, Gray, SeamMap}; use dr_pipeline::EditGraph; pub use crate::export::Cancel; /// The border filler's weights, where the app looks for them; for the /// headless example, which has no library open. pub fn inpaint_model_path() -> Option { crate::library::inpaint_model() } /// One frame as the job receives it: the file's bytes, already fetched, /// and a name for messages and for the composite's own name. #[derive(Debug, Clone)] pub struct MergeInput { /// The file's name, `_MG_8320.CR2`. pub name: String, pub bytes: Arc>, } impl MergeInput { pub fn read(path: &Path) -> Result { Ok(MergeInput { name: path .file_name() .map(|n| n.to_string_lossy().into_owned()) .unwrap_or_else(|| "frame".into()), bytes: Arc::new(std::fs::read(path).map_err(|e| format!("{}: {e}", path.display()))?), }) } } /// Where the composite goes (FR-MRG-3). #[derive(Debug, Clone)] pub enum MergeDestination { /// A folder on this device: written there directly. Local(PathBuf), /// Beside its sources in the library, through the export outbox: the /// file is staged with a destination record and the drain uploads it — /// the one path that works for a folder library and a server alike, and /// the one FR-MRG-3 names for a folder the job cannot write itself. Outbox { outbox: PathBuf, /// The sources' folder, relative to the library root. remote_dir: String, /// The names the library already holds in that folder, as the /// catalog has them. The upload replaces whatever is at its name, so /// a second merge of the same frames must not be called what the /// first one was. taken: std::collections::HashSet, }, } /// What the job is told. #[derive(Debug, Clone)] pub struct MergeRequest { /// The frames, in capture order. pub frames: Vec, pub destination: MergeDestination, /// `None` for the projection the field of view suggests. pub projection: Option, /// Pixels over which a frame's weight ramps up from its edge, where no /// seam says which frame a pixel is taken from. pub feather_px: f32, /// Whether overlaps are cut along seams (`dr_pano::seam`) or averaged /// across `feather_px` everywhere. pub seams: bool, /// The width, in output pixels, of the blend across a seam. pub seam_blend_px: f32, /// GPU work unit; also the DNG strip height. pub chunk: (u32, u32), /// The border filler's weights, if the device has them (FR-MRG-4). /// `None` and the fill is not offered. pub inpaint_model: Option, /// TRACES: FR-RAW-2 /// What reads and decodes each frame. pub decoder: &'static dyn dr_decode::Decoder, } impl MergeRequest { pub fn new(frames: Vec, destination: MergeDestination) -> Self { MergeRequest { frames, destination, projection: None, feather_px: 200.0, seams: true, seam_blend_px: 64.0, chunk: (2048, 512), inpaint_model: None, decoder: dr_decode::default(), } } } /// What the photographer decides once the alignment is shown (FR-MRG-1: /// never automatic). #[derive(Debug, Clone, PartialEq)] pub enum Decision { Merge { projection: Option, /// Fill the border with invented pixels rather than crop to the /// picture (FR-MRG-4), with these knobs. Never the default; the /// page asks. fill: Option, }, /// Draw the alignment again on another surface, and report it again. /// The page's projection chips: what a choice looks like before it is /// confirmed, at proxy cost rather than the merge's — and the fill, at /// preview size, for the same reason: invented pixels are seen before /// they are written. Preview { projection: Option, fill: Option, }, /// Align again over these frames — one flag per input frame, `true` to /// keep it — and report, drawn as `projection` and `fill` say. The /// page's checkboxes: a frame that does not fit is left out without /// leaving the page, and the frames are not read, demosaiced or /// searched for keypoints again; only the matching, the solve and the /// gains run over the ones kept. Frames { keep: Vec, projection: Option, fill: Option, }, Abandon, } /// TRACES: FR-MRG-4 /// The border fill's knobs, every one on the page while the fill is /// experimental: the fill's own (`dr_pano::FillParams`, in pixels of the /// working image) and the two around it. #[derive(Debug, Clone, Copy, PartialEq)] pub struct FillSettings { /// The working image is the composite at `1/scale`: 2 is half, the /// desktop's default; 4 a quarter, for the tablet (FR-MRG-9). pub scale: u32, /// How many working pixels of the coverage edge count as fringe and /// are regenerated too. pub erosion: usize, pub params: dr_pano::FillParams, } impl Default for FillSettings { fn default() -> Self { FillSettings { scale: 2, erosion: 4, params: dr_pano::FillParams { feather: 48, ..Default::default() }, } } } impl FillSettings { /// The same knobs for a picture at `ratio` of the composite's size /// (the page's preview): the working-pixel counts rescaled so the fill /// looks the same there as it will in the merge. pub fn at(&self, ratio: f64) -> FillSettings { let k = ratio * self.scale as f64; let px = |v: usize| ((v as f64) * k).ceil().max(1.0) as usize; FillSettings { scale: 1, erosion: px(self.erosion), params: dr_pano::FillParams { coarse: self.params.coarse, band: px(self.params.band).max(8), mirror_depth: if self.params.mirror_depth == 0 { 0 } else { px(self.params.mirror_depth) }, feather: if self.params.feather == 0 { 0 } else { px(self.params.feather) }, stride: self.params.stride, }, } } } /// What the job says while it runs. #[derive(Debug, Clone)] pub enum MergeEvent { /// A stage, with progress within it. Progress { stage: &'static str, done: usize, total: usize, }, /// The alignment, before any pixel is written: what the interface shows /// for the photographer to confirm, and what names a failure. Aligned(AlignmentReport), /// The composite is written: on the device at `path`, or staged in the /// outbox for the drain to upload, in which case `staged` is true. /// `composite` is what the library needs to show it before the upload /// has finished and a scan has found it (FR-MRG-6). Done { path: PathBuf, staged: bool, width: u32, height: u32, composite: Box, }, Failed(String), Cancelled, } /// TRACES: FR-MRG-6 /// A finished composite, described for the catalog: everything a scan and a /// header read would have found, known here without either. #[derive(Debug, Clone, PartialEq)] pub struct Composite { /// Its name where it goes — in the library folder for a staged merge. pub name: String, /// The picture it opens on: the crop where the border was cropped, the /// whole composite where it was filled. pub width: u32, pub height: u32, /// The middle of the sweep, as written into the DNG; `None` where no /// frame carried a time. pub captured_at: Option, pub captured_offset: Option, pub camera: Option, pub lens: Option, pub iso: Option, pub file_size: u64, /// Made from the final image as develop would first show it /// ([`crate::merge_thumbs`]); empty if that failed, and the grid then /// thumbnails it the ordinary way. pub thumbnails: Vec<(dr_thumbs::ThumbSize, dr_thumbs::Thumbnail)>, } impl MergeEvent { /// Whether this is the job's last word: after one of these the worker /// has nothing more to say, so its channel closing is expected. pub fn is_final(&self) -> bool { matches!( self, MergeEvent::Done { .. } | MergeEvent::Failed(_) | MergeEvent::Cancelled ) } } /// The alignment, described for a panel. #[derive(Debug, Clone)] pub struct AlignmentReport { /// Equivalent focal length in millimetres on full frame, from the fit. pub focal_mm: f64, pub rms_px: f64, pub projection: Projection, pub width: u32, pub height: u32, /// Per frame, in input order: yaw and pitch in degrees if aligned, or /// why not — "left out" for a frame the photographer unticked. pub frames: Vec>, /// Per frame, in input order: whether it is in this alignment. pub included: Vec, pub links: usize, /// The aligned set drawn on the suggested surface at proxy resolution, /// `(width, height, rgba)` — what the photographer confirms. pub preview: Option<(u32, u32, Vec)>, /// Whether the preview's border is filled (FR-MRG-4). pub filled: bool, /// The filler: where it runs, or why it is not available. pub filler: Result, } impl AlignmentReport { /// Whether every frame kept was placed, and there are two of them: the /// only alignment the job will merge. pub fn is_complete(&self) -> bool { self.included.iter().filter(|k| **k).count() >= 2 && self .frames .iter() .zip(&self.included) .all(|(f, &kept)| !kept || f.is_ok()) } } /// Run the whole job on the calling thread, reporting on `events`. /// /// After `Aligned` the job waits on `decision` — the photographer's /// confirmation, or the headless caller's immediate `Merge` — and returns /// when the file is written, the job failed, or `cancel` was seen. pub fn run( ctx: GpuContext, request: MergeRequest, events: Sender, decision: Receiver, cancel: Cancel, ) { let send = |e: MergeEvent| { let _ = events.send(e); }; // Caught rather than allowed to unwind the thread: wgpu reports a device // that has run out of memory by panicking, and a twelve-frame merge on a // GPU another process is using is exactly where that happens. Uncaught, // the thread died, the sender went with it, and the page sat on "Stop" // with every control disabled and nothing to say why — the crash record // on disk was the only sign. The panic hook still writes that record; // this is what puts the reason on the page. let result = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { run_inner(&ctx, &request, &events, &decision, &cancel) })); match result { Ok(Ok(Some(done))) => send(done), Ok(Ok(None)) => send(MergeEvent::Cancelled), Ok(Err(e)) => send(MergeEvent::Failed(e)), Err(panic) => { let detail = panic .downcast_ref::<&str>() .map(|s| (*s).to_string()) .or_else(|| panic.downcast_ref::().cloned()) .unwrap_or_else(|| "panicked with a non-string payload".to_string()); log::error!("merge worker panicked: {detail}"); send(MergeEvent::Failed(format!("internal error: {detail}"))); } } } /// The frames as loaded: sensor data on the CPU and the graph the tap uses. struct Loaded { raw: RawImage, meta: dr_decode::Metadata, graph: Arc, /// The upright size, which is what the cameras are in. size: (u32, u32), } fn run_inner( ctx: &GpuContext, request: &MergeRequest, events: &Sender, decision: &Receiver, cancel: &Cancel, ) -> Result, String> { if request.frames.len() < 2 { return Err("a panorama needs at least two frames".into()); } let progress = |stage: &'static str, done: usize, total: usize| { let _ = events.send(MergeEvent::Progress { stage, done, total }); }; // 1. Load. let t = Instant::now(); let mut frames: Vec = Vec::with_capacity(request.frames.len()); for (i, input) in request.frames.iter().enumerate() { if cancel.is_cancelled() { return Ok(None); } progress("Reading", i, request.frames.len()); let bytes = &input.bytes; let meta = request.decoder.metadata(bytes).unwrap_or_default(); let raw = request .decoder .decode(bytes) .map_err(|e| format!("{}: {e}", input.name))?; let orientation = meta.orientation.unwrap_or_default(); let mut graph = EditGraph::default_chain(); graph.set_orientation(orientation); graph.set_lens_profile(crate::develop::DevelopSession::profile_for(&meta)); let (w, h) = (raw.crop.width, raw.crop.height); let size = if orientation.quarter_turns % 2 == 1 { (h, w) } else { (w, h) }; frames.push(Loaded { raw, meta, graph: Arc::new(graph), size, }); } log::info!("merge: {} frames read in {:?}", frames.len(), t.elapsed()); // 2. Proxies and keypoints. let t = Instant::now(); let demosaicer = Demosaicer::new(ctx).map_err(|e| e.to_string())?; let mut adjust = AdjustPass::new(ctx); let mut detector = dr_pano::xfeat::XFeat::embedded().map_err(|e| e.to_string())?; let mut features = Vec::with_capacity(frames.len()); let mut proxies: Vec = Vec::with_capacity(frames.len()); let mut colour: Vec> = Vec::with_capacity(frames.len()); for (i, f) in frames.iter().enumerate() { if cancel.is_cancelled() { return Ok(None); } progress("Finding features", i, frames.len()); let image = demosaicer.run(&f.raw).map_err(|e| e.to_string())?; let (proxy, rgb) = camera_proxy(&mut adjust, &image, &f.graph, f.size)?; features.push(detector.detect(&proxy).map_err(|e| e.to_string())?); proxies.push(proxy); colour.push(rgb); } log::info!("merge: features in {:?}", t.elapsed()); // 3–4. Alignment and gains, over the frames the photographer keeps. // Everything up to the pairs — each frame read, demosaiced, searched for // keypoints, and every pair matched — is done once; unticking a frame // on the page solves again from the pairs already measured. progress("Aligning", 0, 1); let t = Instant::now(); let pairs = dr_pano::match_pairs(&features, &dr_pano::AlignOptions::default()) .map_err(|e| e.to_string())?; log::info!("merge: pairs matched in {:?}", t.elapsed()); let mut included = vec![true; frames.len()]; let mut solved = solve(&frames, &pairs, &proxies, &included, request.projection)?; // The filler, loaded once if the device has it. Its absence is a // report, not a failure: the crop is always there. let mut filler: Option = None; let filler_status: Result = match request.inpaint_model.as_deref() { None => Err("no border-fill model on this device".into()), Some(path) => match dr_pano::migan::MiGan::from_path(path) { Ok(m) => { let rung = m.rung().map(|r| r.label().to_string()).unwrap_or_default(); filler = Some(m); Ok(rung) } Err(e) => Err(format!("the border-fill model could not be loaded: {e}")), }, }; let has_filler = filler.is_some(); let mut report = |s: &Solved, included: &[bool], projection: Projection, fill: Option| -> Result { let bounds = projection::bounds(projection, s.focal_full, &s.cameras, s.frame_size) .ok_or("the frames project nowhere")?; let proxies: Vec<&Gray> = s.keep.iter().map(|&k| &proxies[k]).collect(); let colour: Vec<&[f32]> = s.keep.iter().map(|&k| colour[k].as_slice()).collect(); let first_raw = &frames[s.keep[0]].raw; let mut filled = false; let preview = match s.alignment.is_complete() { false => None, true => { let map = request .seams .then(|| seams(&proxies, &s.alignment, &s.gains, projection)) .flatten(); let blend = map.as_deref().map(|m| { let full = m.scale * s.focal_full / s.alignment.focal; (m, m.blend_radius(full, f64::from(request.seam_blend_px))) }); let (w, h, mut rgb, mut known) = preview_planes( &colour, &proxies, &s.alignment, &s.gains, blend, projection, first_raw, 1600, ); if let Some(settings) = fill { if let Some(model) = filler.as_mut() { progress("Filling the preview", 0, 1); // The knobs at the preview's scale. let here = settings.at(w as f64 / bounds.width()); dr_pano::fill::erode(&mut known, w, h, here.erosion); match dr_pano::fill_border( &mut rgb, w, h, &known, model, here.params, &mut |_, _| {}, ) { Ok(_) => filled = true, Err(e) => log::warn!("preview fill: {e}"), } } } Some(preview_rgba( &rgb, w, h, &s.alignment, projection, &bounds_of(&proxies, &s.alignment, projection), &proxies, )) } }; // The solve's frames back in input order, the unticked ones named. let mut rows: Vec> = vec![Err("left out".to_string()); included.len()]; for (&k, row) in s.keep.iter().zip(describe(&s.alignment)) { rows[k] = row; } Ok(AlignmentReport { focal_mm: s.focal_full * 36.0 / s.full_long, rms_px: s.alignment.rms_px, projection, width: bounds.width().ceil() as u32, height: bounds.height().ceil() as u32, frames: rows, included: included.to_vec(), links: s.alignment.links.len(), preview, filled, filler: filler_status.clone(), }) }; progress("Drawing the preview", 0, 1); let _ = events.send(MergeEvent::Aligned(report( &solved, &included, solved.suggested, None, )?)); // Never automatic (FR-MRG-1): nothing is written until the alignment // has been seen and confirmed. Polled, so a cancel while waiting is // seen within a moment. A frame that could not be placed no longer // ends the job (FR-MRG-5 names it on its row): the page leaves it out // and the job aligns again without it. let (projection, fill) = loop { if cancel.is_cancelled() { return Ok(None); } match decision.recv_timeout(std::time::Duration::from_millis(100)) { Ok(Decision::Merge { projection: p, fill, }) => { // The page's button is off until the alignment is // complete; a press that raced a re-solve is dropped. if !solved.alignment.is_complete() { continue; } break (p.unwrap_or(solved.suggested), fill.filter(|_| has_filler)); } Ok(Decision::Preview { projection: p, fill, }) => { let _ = events.send(MergeEvent::Aligned(report( &solved, &included, p.unwrap_or(solved.suggested), fill, )?)); } Ok(Decision::Frames { keep, projection: p, fill, }) => { // Two frames or it is not a panorama; the page does not // offer fewer, and a malformed list is not acted on. if keep.len() != frames.len() || keep.iter().filter(|k| **k).count() < 2 { continue; } progress("Aligning", 0, 1); included = keep; solved = solve(&frames, &pairs, &proxies, &included, request.projection)?; progress("Drawing the preview", 0, 1); let _ = events.send(MergeEvent::Aligned(report( &solved, &included, p.unwrap_or(solved.suggested), fill, )?)); } Ok(Decision::Abandon) => return Ok(None), Err(std::sync::mpsc::RecvTimeoutError::Timeout) => continue, Err(std::sync::mpsc::RecvTimeoutError::Disconnected) => return Ok(None), } }; // The seams, on the surface chosen, over the frames kept. let seam_map = request.seams.then(|| { let kept: Vec<&Gray> = solved.keep.iter().map(|&k| &proxies[k]).collect(); seams(&kept, &solved.alignment, &solved.gains, projection) }); let seam_map = seam_map.flatten(); // From here on only the kept frames exist, indexed as the alignment // indexes them. let Solved { keep, cameras, focal_full, full_long, frame_size, gains, .. } = solved; let first_name = request.frames[keep[0]].name.clone(); let frames: Vec = frames .into_iter() .enumerate() .filter(|(k, _)| included[*k]) .map(|(_, f)| f) .collect(); let first_raw = &frames[0].raw; let bounds = projection::bounds(projection, focal_full, &cameras, frame_size) .ok_or("the frames project nowhere")?; // 5. Merge, into a DNG beside the first frame. let name = format!("{}-pano.dng", stem(Path::new(&first_name))); let (out_path, staged) = match &request.destination { MergeDestination::Local(dir) => (unused_name(dir, &name, &Default::default()), false), MergeDestination::Outbox { outbox, remote_dir, taken, } => { std::fs::create_dir_all(outbox).map_err(|e| format!("{}: {e}", outbox.display()))?; let path = unused_name(outbox, &name, taken); // The record first here, unlike an export: the payload is // written over minutes and a record naming a half-written file // is worse than a payload with no record, so the record is // removed again if the merge fails. The drain skips payloads // without records. let record = crate::export::destination_record(&path); let final_name = path .file_name() .map(|n| n.to_string_lossy().into_owned()) .unwrap_or(name.clone()); std::fs::write(&record, format!("{remote_dir}\n{final_name}\n")) .map_err(|e| format!("{}: {e}", record.display()))?; (path, true) } }; // Written under a name of its own and renamed into place once the last // strip is in. The record above names `out_path`, and a drain that runs // while the merge is still writing — a sync pass fires one whenever the // sweep finishes — takes whatever is at that name: it uploaded the first // few hundred megabytes of a composite, cleared the record, and the file // on the server stayed truncated for good. `pending` skips a record whose // payload does not exist yet, so the rename is what releases it. let part_path = part_name(&out_path); let cleanup = |path: &Path| { let _ = std::fs::remove_file(path); let _ = std::fs::remove_file(part_name(path)); if staged { let _ = std::fs::remove_file(crate::export::destination_record(path)); let _ = std::fs::remove_file(crate::export::thumbnails_record(path)); } }; let (out_w, out_h) = (report_size(&bounds).0, report_size(&bounds).1); let first = &frames[0]; let black = first.raw.black_level[0]; let white_level = u32::from(first.raw.white_level.saturating_sub(black)).max(1); let profile = dng_profile(first, white_level); let balance = profile.as_shot_neutral.map(|n| 1.0 / n.max(1e-3)); let (_, mut carried) = crate::export::header_for_file(&first.meta); // The thumbnails' copy of the profile; the writer takes the original. let thumb_profile = profile.clone(); // The composite is dated at the middle of its sweep — the mean of the // frames' capture times — so it sorts among the frames it was made // from, not at the first of them and not at the moment of the merge. // The zone is the first frame's; a sweep does not cross one. let stamps: Vec = frames.iter().filter_map(|f| f.meta.captured_at).collect(); if !stamps.is_empty() { let sum: i128 = stamps.iter().map(|&t| t as i128).sum(); carried.captured_at = Some((sum / stamps.len() as i128) as i64); } // What the catalog is told about the file (FR-MRG-6), taken before the // header moves into the writer. let thumb_header = carried.clone(); let described = ( carried.captured_at, carried.captured_offset, crate::library::camera_label(carried.make.as_deref(), carried.model.as_deref()), carried.lens.as_ref().map(|l| l.trim().to_string()), carried.iso, ); let output = MergeOutput { projection, scale: focal_full, bounds, feather: request.feather_px, seams: seam_map.clone(), seam_blend: request.seam_blend_px, chunk: request.chunk, sample_scale: white_level as f32, balance, }; let merge_frames: Vec = frames .iter() .zip(&gains) .map(|(f, &gain)| MergeFrame { graph: f.graph.clone(), gain, }) .collect(); let t = Instant::now(); let mut pass = MergePass::new(ctx).map_err(|e| e.to_string())?; let mut resident: VecDeque<(usize, Arc)> = VecDeque::new(); // 5a. The border fill (FR-MRG-4), when asked for: the composite at a // quarter of its resolution, its border filled by the model in display // space, and the result taken back to camera space for the full merge // to sample wherever nothing was covered. A quarter because the fill is // sky and slope at the edge of a picture, where a quarter is more than // the eye resolves, and because it puts the fixture's border at thirty // tiles rather than a hundred. let fill_cam: Option<(usize, usize, Vec, Vec)> = if let Some(settings) = fill { let model = filler.as_mut().expect("fill was only chosen with a filler"); progress("Filling the border", 0, 1); // Half the composite's resolution by default: a quarter left a // visible gap in sharpness against the real pixels beside the // fill. Memory is the working image in f32, 400 MB for a 137 MP // composite; the tablet will want a quarter (FR-MRG-9). let q = settings.scale.max(1) as f64; let qout = MergeOutput { projection, scale: focal_full / q, bounds: projection::Bounds { min_u: bounds.min_u / q, min_v: bounds.min_v / q, max_u: bounds.max_u / q, max_v: bounds.max_v / q, }, feather: request.feather_px, seams: seam_map.clone(), seam_blend: request.seam_blend_px / q as f32, chunk: request.chunk, sample_scale: white_level as f32, balance, }; let (qw, qh) = (qout.width() as usize, qout.height() as usize); let mut qrgb = vec![0.0f32; qw * qh * 3]; let mut qknown = vec![false; qw * qh]; pass.merge( &mut adjust, &merge_frames, &cameras, (frames[0].size.0, frames[0].size.1), &qout, |k| { if let Some((_, img)) = resident.iter().find(|(i, _)| *i == k) { return Ok(img.clone()); } let img = Arc::new(demosaicer.run(&frames[k].raw)?); resident.push_back((k, img.clone())); while resident.len() > 2 { resident.pop_front(); } Ok(img) }, |band| { let first = band.first_row as usize; for r in 0..band.rows as usize { for x in 0..qw { let src = r * qw + x; let dst = (first + r) * qw + x; for c in 0..3 { qrgb[dst * 3 + c] = f32::from(band.rgb[src * 3 + c]) / white_level as f32; } qknown[dst] = band.covered[src]; } } Ok(()) }, || cancel.is_cancelled(), ) .map_err(|e| e.to_string())?; if cancel.is_cancelled() { cleanup(&out_path); return Ok(None); } // Camera space to the display-ish space the model was trained on. let look = Look::of(first_raw); let mut disp = qrgb.clone(); for p in disp.chunks_exact_mut(3) { let d = look.to_display([p[0], p[1], p[2]]); p.copy_from_slice(&d); } dr_pano::fill::erode(&mut qknown, qw, qh, settings.erosion); // `DR_FILL_DUMP=dir` writes what the filler was given, so a fill // can be re-run and looked at stage by stage without the merge // (`cargo run -p dr-pano --example fill`). if let Some(dir) = std::env::var_os("DR_FILL_DUMP") { let dir = PathBuf::from(dir); if let Err(e) = dump_fill_input(&dir, &disp, &qknown, qw, qh) { log::warn!("fill dump: {e}"); } } let total_tiles = std::cell::Cell::new(0usize); let feather = settings.params.feather; dr_pano::fill_border( &mut disp, qw, qh, &qknown, model, settings.params, &mut |n, total| { total_tiles.set(total); progress("Filling the border", n, total.max(1)); }, ) .map_err(|e| e.to_string())?; // And back: the filled pixels — the hole and the feathered margin — // to camera space; the rest as the merge produced them. let mut margin = qknown.clone(); dr_pano::fill::erode(&mut margin, qw, qh, feather); for (i, p) in qrgb.chunks_exact_mut(3).enumerate() { if !margin[i] { let d = &disp[i * 3..i * 3 + 3]; let cam = look.to_camera([d[0], d[1], d[2]]); p.copy_from_slice(&cam); } } let fill_mask: Vec = margin; log::info!( "merge: border filled at {qw}×{qh}, {} tiles, in {:?}", total_tiles.get(), t.elapsed() ); Some((qw, qh, qrgb, fill_mask)) } else { None }; // The writer pulls strips; the merge pushes bands. A channel between // them, and the writer on its own thread, so neither waits on the // other's pace more than one band. let (band_tx, band_rx) = std::sync::mpsc::sync_channel::>(1); let rows_per_strip = request.chunk.1.max(1); // The crop (FR-MRG-4), found as the bands go by; the writer reads it // once the last strip is in, by which time every band has been seen. let inscribed = Arc::new(std::sync::Mutex::new(dr_export::Inscribed::new(out_w))); let inscribed_for_writer = inscribed.clone(); let fill_for_writer = fill_cam.is_some(); let file = std::fs::File::create(&part_path).map_err(|e| format!("{}: {e}", part_path.display()))?; let writer = executors::spawn(Executor::Io, "dng-write", move || -> Result<(), String> { let mut file = std::io::BufWriter::new(file); dr_export::write_linear_dng( &mut file, out_w, out_h, rows_per_strip, &profile, Some(&carried), |_, buf| { let band = band_rx.recv().map_err(|_| { dr_export::ExportError::Encode("the merge stopped early".into()) })?; buf.extend_from_slice(&band); Ok(()) }, // A filled composite is the whole picture; a cropped one opens // on the rectangle the frames covered. move || { if fill_for_writer { None } else { inscribed_for_writer.lock().ok().map(|i| i.best()) } }, ) .map_err(|e| e.to_string()) }); let total_bands = out_h.div_ceil(rows_per_strip) as usize; let mut bands_done = 0usize; // The final image, reduced as it goes by, for the thumbnails (FR-MRG-6). let mut reduced = crate::merge_thumbs::Reduced::new(out_w, out_h, crate::merge_thumbs::SOURCE_EDGE); progress("Merging", 0, total_bands); let merged = pass.merge( &mut adjust, &merge_frames, &cameras, (frames[0].size.0, frames[0].size.1), &output, |k| { if let Some((_, img)) = resident.iter().find(|(i, _)| *i == k) { return Ok(img.clone()); } let img = Arc::new(demosaicer.run(&frames[k].raw)?); resident.push_back((k, img.clone())); while resident.len() > 2 { resident.pop_front(); } Ok(img) }, |band| { bands_done += 1; progress("Merging", bands_done, total_bands); if let Ok(mut i) = inscribed.lock() { i.push_rows(band.covered, band.rows); } let mut rgb = band.rgb.to_vec(); // Where no frame reached, the fill — sampled bilinearly from // the quarter-resolution result, at the sensor's scale. if let Some((qw, qh, qrgb, fill_mask)) = &fill_cam { let sx = *qw as f64 / out_w as f64; let sy = *qh as f64 / out_h as f64; for r in 0..band.rows as usize { for x in 0..out_w as usize { let i = r * out_w as usize + x; let fx = ((x as f64 + 0.5) * sx - 0.5).clamp(0.0, (*qw - 1) as f64); let fy = (((band.first_row as usize + r) as f64 + 0.5) * sy - 0.5) .clamp(0.0, (*qh - 1) as f64); // Uncovered, or inside the feathered margin: the fill. let (mx, my) = (fx.round() as usize, fy.round() as usize); if band.covered[i] && fill_mask[my.min(qh - 1) * qw + mx.min(qw - 1)] { continue; } let (x0, y0) = (fx as usize, fy as usize); let (x1, y1) = ((x0 + 1).min(qw - 1), (y0 + 1).min(qh - 1)); let (tx, ty) = ((fx - x0 as f64) as f32, (fy - y0 as f64) as f32); for c in 0..3 { let at = |xx: usize, yy: usize| qrgb[(yy * qw + xx) * 3 + c]; let v = (at(x0, y0) * (1.0 - tx) + at(x1, y0) * tx) * (1.0 - ty) + (at(x0, y1) * (1.0 - tx) + at(x1, y1) * tx) * ty; rgb[i * 3 + c] = (v * white_level as f32).round().clamp(0.0, 65535.0) as u16; } } } } reduced.push(band.first_row, band.rows, &rgb); band_tx .send(rgb) .map_err(|_| dr_gpu::GpuError::Readback("the writer stopped".into())) }, || cancel.is_cancelled(), ); drop(band_tx); let written = writer .join() .unwrap_or_else(|_| Err("the writer panicked".into())); match merged { Ok(()) => {} Err(e) if cancel.is_cancelled() => { cleanup(&out_path); log::info!("merge cancelled: {e}"); return Ok(None); } Err(e) => { cleanup(&out_path); return Err(e.to_string()); } } if let Err(e) = written { cleanup(&out_path); return Err(e); } // The rectangle the file opens on, as the writer recorded it. let picture = if fill_cam.is_some() { None } else { inscribed .lock() .ok() .map(|i| clamp_crop(i.best(), out_w, out_h)) .filter(|r| r.width > 0 && r.height > 0) }; // TRACES: FR-MRG-6 // The thumbnails, from the final image, developed as develop will first // show the file. Before the rename, and staged beside the payload before // it: the rename is what lets a drain take the file, and the drain is // what puts these in the store once the server has named it. // The merge's GPU memory back before develop asks for its own. drop(resident); drop(pass); progress("Making thumbnails", 0, 1); let (pw, ph) = picture.map_or((out_w, out_h), |r| (r.width, r.height)); let thumbnails = match crate::merge_thumbs::render( ctx, reduced, &thumb_profile, &thumb_header, picture, request.decoder, &crate::merge_thumbs::classes(pw, ph), ) { Ok(t) => t, Err(e) => { log::warn!("merge: the thumbnails could not be made: {e}"); Vec::new() } }; if staged && !thumbnails.is_empty() { if let Err(e) = crate::export::write_thumbnails(&out_path, &thumbnails) { log::warn!("merge: the thumbnails could not be staged: {e}"); } } if let Err(e) = std::fs::rename(&part_path, &out_path) { cleanup(&out_path); return Err(format!("{}: {e}", out_path.display())); } log::info!( "merge: {}×{} written to {} in {:?}", out_w, out_h, out_path.display(), t.elapsed() ); // TRACES: FR-MRG-6 // Provenance, beside the composite: what it was merged from, in order, // and how. Its own file and its own outbox record, so the drain places // it as it places the composite; a sidecar that failed to write is // reported but does not un-write the composite. let mut sidecar = dr_pipeline::Sidecar::new(); sidecar.derived_from = request.frames.iter().map(|f| f.name.clone()).collect(); sidecar.merge = Some(format!( "panorama {} {:.1}mm {} frames{}", match projection { Projection::Perspective => "perspective", Projection::Cylindrical => "cylindrical", Projection::Spherical => "spherical", }, focal_full * 36.0 / full_long, frames.len(), // Invented pixels declare themselves (FR-MRG-6, D17's rule), with // the knobs that made them while the fill is experimental. match fill { Some(f) => format!( " border filled (1/{} scale, erosion {}, coarse {}, band {}, mirror {}, feather {})", f.scale, f.erosion, f.params.coarse, f.params.band, f.params.mirror_depth, f.params.feather ), None => String::new(), } )); let sidecar_path = out_path.with_extension(dr_pipeline::sidecar::EXTENSION); if let Err(e) = std::fs::write(&sidecar_path, sidecar.to_text()) { log::warn!("merge: the sidecar could not be written: {e}"); } else if let MergeDestination::Outbox { remote_dir, .. } = &request.destination { let record = crate::export::destination_record(&sidecar_path); let name = sidecar_path .file_name() .map(|n| n.to_string_lossy().into_owned()) .unwrap_or_default(); if let Err(e) = std::fs::write(&record, format!("{remote_dir}\n{name}\n")) { log::warn!("merge: the sidecar's record could not be written: {e}"); } } let (captured_at, captured_offset, camera, lens, iso) = described; let composite = Composite { name: out_path .file_name() .map(|n| n.to_string_lossy().into_owned()) .unwrap_or_default(), width: pw, height: ph, captured_at, captured_offset, camera, lens, iso, file_size: std::fs::metadata(&out_path).map(|m| m.len()).unwrap_or(0), thumbnails, }; Ok(Some(MergeEvent::Done { path: out_path, staged, width: out_w, height: out_h, composite: Box::new(composite), })) } /// The crop as `write_linear_dng` writes it: inside the frame. fn clamp_crop(r: dr_export::Rect, width: u32, height: u32) -> dr_export::Rect { let x = r.x.min(width.saturating_sub(1)); let y = r.y.min(height.saturating_sub(1)); dr_export::Rect { x, y, width: r.width.min(width - x), height: r.height.min(height - y), } } /// The proxy the detector reads: the frame through the camera-space tap at /// proxy size, upright, as gamma-encoded grey. /// /// Through the tap rather than the embedded preview so that the keypoints /// are in the *undistorted* frame the tiles will be rendered in — a lens /// profile moves the corners by tens of pixels at full resolution, and an /// alignment measured on the distorted preview would be wrong by that much /// at the seams. fn camera_proxy( adjust: &mut AdjustPass, image: &DemosaicedImage, graph: &EditGraph, upright: (u32, u32), ) -> Result<(Gray, Vec), String> { let long = dr_pano::xfeat::INPUT_LONG_EDGE as f64; let scale = (long / f64::from(upright.0.max(upright.1))).min(1.0); let w = ((f64::from(upright.0) * scale).round() as u32).max(1); let h = ((f64::from(upright.1) * scale).round() as u32).max(1); let shader = graph.compose_camera_linear(dr_pipeline::CropRect::default()); adjust .render_camera_linear(image, &shader, w, h) .map_err(|e| e.to_string())?; let (rgba, rw, rh) = adjust.read_camera_linear().map_err(|e| e.to_string())?; // Camera RGB is unbalanced — green-heavy on every Bayer body — and // linear. Luma weights would over-count green further; an equal mean is // as good a grey as any for corners and edges, and the gamma is what // gives the detector the contrast range it was trained on. let data: Vec = rgba .chunks_exact(4) .map(|p| ((p[0] + p[1] + p[2]) / 3.0).clamp(0.0, 1.0).powf(1.0 / 2.2)) .collect(); let rgb: Vec = rgba .chunks_exact(4) .flat_map(|p| [p[0], p[1], p[2]]) .collect(); Ok(( Gray { width: rw as usize, height: rh as usize, data, }, rgb, )) } /// An alignment over some of the frames, and what follows from it. struct Solved { /// The input frames solved, in input order: the alignment's frame `i` /// is input frame `keep[i]`. keep: Vec, alignment: Alignment, /// At full resolution. cameras: Cameras, focal_full: f64, /// The first kept frame's long edge and upright size, full resolution. full_long: f64, frame_size: (f64, f64), /// The projection the field of view suggests, or the request's. suggested: Projection, /// One per kept frame. gains: Vec, } /// Align the frames `included` keeps, from the pairs already measured, /// and take the geometry to full resolution and the gains from it. fn solve( frames: &[Loaded], pairs: &dr_pano::Pairs, proxies: &[Gray], included: &[bool], projection: Option, ) -> Result { let keep: Vec = (0..frames.len()).filter(|&k| included[k]).collect(); let t = Instant::now(); let alignment = dr_pano::solve(pairs, included, &dr_pano::AlignOptions::default()) .map_err(|e| e.to_string())?; log::info!( "merge: {} frames aligned in {:?}, focal {:.1} px, rms {:.2} px, {} links", keep.len(), t.elapsed(), alignment.focal, alignment.rms_px, alignment.links.len() ); // The geometry at full resolution: the proxy's long edge against the // frame's. let first = &frames[keep[0]]; let proxy_long = proxies[keep[0]].width.max(proxies[keep[0]].height) as f64; let full_long = first.size.0.max(first.size.1) as f64; let focal_full = alignment.focal * full_long / proxy_long; let cameras = Cameras { rotations: alignment .rotations .iter() .map(|r| r.unwrap_or(dr_pano::linalg::Mat3::IDENTITY)) .collect(), focal: focal_full, }; let frame_size = (first.size.0 as f64, first.size.1 as f64); let (hfov, vfov) = field_of_view(&cameras, frame_size); let suggested = projection.unwrap_or_else(|| Projection::suggest(hfov, vfov)); // Gains, before the report so the preview shows them. let gains = if alignment.is_complete() { let kept: Vec<&Gray> = keep.iter().map(|&k| &proxies[k]).collect(); gains(&kept, &alignment) } else { vec![1.0; keep.len()] }; Ok(Solved { keep, alignment, cameras, focal_full, full_long, frame_size, suggested, gains, }) } /// Which frame each part of the composite comes from (`dr_pano::seam`), on /// `projection` at the proxies' scale. `None` when the alignment is not /// complete — the merge then falls back to the feathered average. fn seams( proxies: &[&Gray], alignment: &Alignment, gains: &[f32], projection: Projection, ) -> Option> { if !alignment.is_complete() { return None; } let t = Instant::now(); let map = dr_pano::seam::find( proxies, &alignment.cameras(), gains, projection, &dr_pano::SeamOptions::default(), ); log::info!( "merge: seams over {} frames in {:?}", proxies.len(), t.elapsed() ); map.map(Arc::new) } /// The aligned set on its surface, in colour, for the page. /// /// A quick look, not the pipeline: the first frame's white balance and /// matrix, a gamma, and the frames joined along the same seams the merge /// will use, with their gains applied. Ghosting here is the alignment's /// error and banding is the gains', which is exactly what the photographer /// is being asked to look at. Fitted to 1600 px across. /// The aligned set on its surface, in colour, as planes: `(w, h, rgb 0..1, /// known)`. Display-ish space — the first frame's white balance and matrix, /// a gamma — which is also what the border filler was trained on. #[allow(clippy::too_many_arguments)] fn preview_planes( colour: &[&[f32]], proxies: &[&Gray], alignment: &Alignment, gains: &[f32], seams: Option<(&SeamMap, f64)>, projection: Projection, first: &RawImage, max_width: usize, ) -> (usize, usize, Vec, Vec) { let cameras = alignment.cameras(); let (fw, fh) = (proxies[0].width as f64, proxies[0].height as f64); let scale = alignment.focal; let Some(bounds) = projection::bounds(projection, scale, &cameras, (fw, fh)) else { return (0, 0, Vec::new(), Vec::new()); }; let out_w = max_width.min(bounds.width().ceil() as usize).max(1); let px = bounds.width() / out_w as f64; let out_h = ((bounds.height() / px).ceil() as usize).max(1); let look = Look::of(first); let mut rgb = vec![0.0f32; out_w * out_h * 3]; let mut known = vec![false; out_w * out_h]; for oy in 0..out_h { for ox in 0..out_w { let u = bounds.min_u + (ox as f64 + 0.5) * px; let v = bounds.min_v + (oy as f64 + 0.5) * px; let d = projection.to_direction(scale, u, v); let mut sum = [0.0f32; 3]; let mut n = 0.0f32; for (k, g) in proxies.iter().enumerate() { let Some((x, y)) = cameras.project(k, d) else { continue; }; let (x, y) = (x + fw / 2.0, y + fh / 2.0); if x < 0.0 || y < 0.0 || x >= fw - 1.0 || y >= fh - 1.0 { continue; } let i = (y as usize * g.width + x as usize) * 3; let cam = grey_if_blown([colour[k][i], colour[k][i + 1], colour[k][i + 2]], look.wb); // The seam's share, as the merge weighs it, with the same // floor under it; an even average where there is no map. let w = seams .and_then(|(m, radius)| m.share(k, u, v, scale, radius)) .map_or(1.0, |s| s + 1e-4); for c in 0..3 { sum[c] += cam[c] * gains[k] * w; } n += w; } if n <= 0.0 { continue; } let o = oy * out_w + ox; let d = look.to_display([sum[0] / n, sum[1] / n, sum[2] / n]); rgb[o * 3..o * 3 + 3].copy_from_slice(&d); known[o] = true; } } (out_w, out_h, rgb, known) } fn bounds_of( proxies: &[&Gray], alignment: &Alignment, projection: Projection, ) -> projection::Bounds { let (fw, fh) = (proxies[0].width as f64, proxies[0].height as f64); projection::bounds(projection, alignment.focal, &alignment.cameras(), (fw, fh)).unwrap_or( projection::Bounds { min_u: 0.0, min_v: 0.0, max_u: 1.0, max_v: 1.0, }, ) } /// The planes to RGBA for the page, with each frame outlined. /// /// The outlines, so the page shows *frames* and not one picture: a sweep /// of frames overlapping by more than half reads as a single photograph /// otherwise, which is what a photographer looking for a misplaced frame /// needs to see through. Walked along each border and drawn where it /// lands, two pixels wide, in a colour the scene does not have. fn preview_rgba( rgb: &[f32], out_w: usize, out_h: usize, alignment: &Alignment, projection: Projection, bounds: &projection::Bounds, proxies: &[&Gray], ) -> (u32, u32, Vec) { let cameras = alignment.cameras(); let (fw, fh) = (proxies[0].width as f64, proxies[0].height as f64); let scale = alignment.focal; let px = bounds.width() / out_w.max(1) as f64; let mut out = vec![255u8; out_w * out_h * 4]; for i in 0..out_w * out_h { for c in 0..3 { out[i * 4 + c] = (rgb[i * 3 + c].clamp(0.0, 1.0) * 255.0) as u8; } } let steps = 600; for k in 0..proxies.len() { for s in 0..steps { let t = s as f64 / steps as f64; for p in [ (-fw / 2.0 + fw * t, -fh / 2.0), (-fw / 2.0 + fw * t, fh / 2.0 - 1.0), (-fw / 2.0, -fh / 2.0 + fh * t), (fw / 2.0 - 1.0, -fh / 2.0 + fh * t), ] { let d = cameras.bearing(k, p); let Some((u, v)) = projection.from_direction(scale, d) else { continue; }; let x = ((u - bounds.min_u) / px) as i64; let y = ((v - bounds.min_v) / px) as i64; for (dx, dy) in [(0, 0), (1, 0), (0, 1), (1, 1)] { let (xx, yy) = (x + dx, y + dy); if xx < 0 || yy < 0 || xx >= out_w as i64 || yy >= out_h as i64 { continue; } let o = (yy as usize * out_w + xx as usize) * 4; out[o] = 255; out[o + 1] = 176; out[o + 2] = 0; out[o + 3] = 255; } } } } (out_w as u32, out_h as u32, out) } /// The quick look from camera space to something like sRGB and back: the /// first frame's white balance and matrix, a 2.2 gamma. Not the pipeline — /// the page's preview and the filler's input, both of which want a /// plausible picture rather than a calibrated one — and invertible, so the /// filler's answer comes back to camera space where the DNG lives. struct Look { wb: [f32; 3], m: [f32; 9], inv: [f32; 9], } impl Look { fn of(raw: &RawImage) -> Self { let wb = [raw.wb_coeffs[0], raw.wb_coeffs[1], raw.wb_coeffs[2]]; let m = raw .color_matrix .unwrap_or([1.0, 0.0, 0.0, 0.0, 1.0, 0.0, 0.0, 0.0, 1.0]); Look { wb, m, inv: invert3(&m).unwrap_or([1.0, 0.0, 0.0, 0.0, 1.0, 0.0, 0.0, 0.0, 1.0]), } } fn to_display(&self, cam: [f32; 3]) -> [f32; 3] { let b = [ cam[0] * self.wb[0], cam[1] * self.wb[1], cam[2] * self.wb[2], ]; let row = |c: usize| self.m[c * 3] * b[0] + self.m[c * 3 + 1] * b[1] + self.m[c * 3 + 2] * b[2]; let encode = |lin: f32| lin.clamp(0.0, 1.0).powf(1.0 / 2.2); [encode(row(0)), encode(row(1)), encode(row(2))] } fn to_camera(&self, disp: [f32; 3]) -> [f32; 3] { let lin = [ disp[0].max(0.0).powf(2.2), disp[1].max(0.0).powf(2.2), disp[2].max(0.0).powf(2.2), ]; let row = |c: usize| { self.inv[c * 3] * lin[0] + self.inv[c * 3 + 1] * lin[1] + self.inv[c * 3 + 2] * lin[2] }; let b = [row(0), row(1), row(2)]; [ (b[0] / self.wb[0].max(1e-4)).max(0.0), (b[1] / self.wb[1].max(1e-4)).max(0.0), (b[2] / self.wb[2].max(1e-4)).max(0.0), ] } } /// A blown camera sample as the grey `wb` balances it to; the merge /// shader's rule, for the preview. /// /// A clipped photosite is (1, 1, 1), which balances to magenta. The develop /// pipeline pulls it back to the brightest balanced channel as it nears /// white (`CLIP_ONSET`); this does the same in camera space, before any /// gain moves it off the white level or an overlap averages it with a /// neighbour's real sky — after either, nothing downstream can tell it was /// blown and the clouds come out pink. fn grey_if_blown(cam: [f32; 3], wb: [f32; 3]) -> [f32; 3] { let peak = cam[0].max(cam[1]).max(cam[2]); let t = ((peak - dr_pipeline::CLIP_ONSET) / (1.0 - dr_pipeline::CLIP_ONSET)).clamp(0.0, 1.0); let clipped = t * t * (3.0 - 2.0 * t); if clipped <= 0.0 { return cam; } let wb = wb.map(|w| w.max(1e-3)); let grey = (cam[0] * wb[0]).max(cam[1] * wb[1]).max(cam[2] * wb[2]); [0, 1, 2].map(|c| cam[c] + (grey / wb[c] - cam[c]) * clipped) } fn invert3(m: &[f32; 9]) -> Option<[f32; 9]> { let (a, b, c, d, e, f, g, h, i) = (m[0], m[1], m[2], m[3], m[4], m[5], m[6], m[7], m[8]); let det = a * (e * i - f * h) - b * (d * i - f * g) + c * (d * h - e * g); if det.abs() < 1e-12 { return None; } let s = 1.0 / det; Some([ (e * i - f * h) * s, (c * h - b * i) * s, (b * f - c * e) * s, (f * g - d * i) * s, (a * i - c * g) * s, (c * d - a * f) * s, (d * h - e * g) * s, (b * g - a * h) * s, (a * e - b * d) * s, ]) } /// The horizontal and vertical field of view the cameras span, in radians: /// the angle between the extreme frame centres plus one frame's own field. fn field_of_view(cameras: &Cameras, frame: (f64, f64)) -> (f64, f64) { let f = cameras.focal; let own_h = 2.0 * (frame.0 / (2.0 * f)).atan(); let own_v = 2.0 * (frame.1 / (2.0 * f)).atan(); let (mut min_yaw, mut max_yaw, mut min_pitch, mut max_pitch) = (0.0f64, 0.0f64, 0.0f64, 0.0f64); for r in &cameras.rotations { let d = *r * dr_pano::linalg::Vec3::new(0.0, 0.0, 1.0); let yaw = d.x().atan2(d.z()); let pitch = d.y().asin(); min_yaw = min_yaw.min(yaw); max_yaw = max_yaw.max(yaw); min_pitch = min_pitch.min(pitch); max_pitch = max_pitch.max(pitch); } (max_yaw - min_yaw + own_h, max_pitch - min_pitch + own_v) } fn report_size(b: &projection::Bounds) -> (u32, u32) { ( b.width().ceil().max(1.0) as u32, b.height().ceil().max(1.0) as u32, ) } fn describe(a: &Alignment) -> Vec> { a.rotations .iter() .enumerate() .map(|(k, r)| match r { Some(r) => { let yaw = r.0[0][2].atan2(r.0[2][2]).to_degrees(); let pitch = (-r.0[1][2]).asin().to_degrees(); Ok((yaw, pitch)) } None => Err(a .unaligned .iter() .find(|(i, _)| *i == k) .map(|(_, why)| why.to_string()) .unwrap_or_else(|| "not aligned".into())), }) .collect() } /// One gain per frame, from the proxies' overlaps. /// /// For each link, the mean grey of each frame over the region both see is /// compared; the log-gains that best reconcile every link are solved in /// least squares with the reference frame held at 1. Grey here is the /// proxy's gamma-encoded value raised back to linear, so the gain is a /// linear multiplier as the shader applies it. fn gains(proxies: &[&Gray], a: &Alignment) -> Vec { let n = proxies.len(); let cameras = a.cameras(); let (w, h) = (proxies[0].width as f64, proxies[0].height as f64); let linear = |g: &Gray, x: usize, y: usize| g.data[y * g.width + x].powf(2.2) as f64; // Ratios per link. let mut ratios: Vec<(usize, usize, f64)> = Vec::new(); for l in &a.links { let (mut sum_i, mut sum_j, mut count) = (0.0, 0.0, 0usize); let step = 8; for y in (0..proxies[l.i].height).step_by(step) { for x in (0..proxies[l.i].width).step_by(step) { let p = (x as f64 - w / 2.0, y as f64 - h / 2.0); let d = cameras.bearing(l.i, p); let Some((qx, qy)) = cameras.project(l.j, d) else { continue; }; let (qx, qy) = (qx + w / 2.0, qy + h / 2.0); if qx < 0.0 || qy < 0.0 || qx >= w - 1.0 || qy >= h - 1.0 { continue; } sum_i += linear(proxies[l.i], x, y); sum_j += linear(proxies[l.j], qx as usize, qy as usize); count += 1; } } if count >= 50 && sum_i > 0.0 && sum_j > 0.0 { ratios.push((l.i, l.j, (sum_i / sum_j).ln())); } } if ratios.is_empty() { return vec![1.0; n]; } // Least squares on log gains: g_j - g_i = ln(mean_i / mean_j), with // the reference frame's gain fixed at 0 by a strong prior. let root = a .rotations .iter() .position(|r| *r == Some(dr_pano::linalg::Mat3::IDENTITY)) .unwrap_or(0); let mut ata = dr_pano::linalg::DMat::zeros(n); let mut atb = vec![0.0f64; n]; for &(i, j, r) in &ratios { // Row: +1 at j, −1 at i, rhs r. ata[(j, j)] += 1.0; ata[(i, i)] += 1.0; ata[(i, j)] -= 1.0; ata[(j, i)] -= 1.0; atb[j] += r; atb[i] -= r; } ata[(root, root)] += 1e6; // A tiny ridge keeps an unlinked frame (there are none if the alignment // is complete) from making the system singular. for k in 0..n { ata[(k, k)] += 1e-9; } match ata.solve_spd(&atb) { Some(g) => g.iter().map(|v| v.exp() as f32).collect(), None => vec![1.0; n], } } fn dng_profile(first: &Loaded, white_level: u32) -> dr_export::DngProfile { let wb = first.raw.wb_coeffs; let neutral_from_wb = [ 1.0 / wb[0].max(1e-3), 1.0 / wb[1].max(1e-3), 1.0 / wb[2].max(1e-3), ]; let (calibrations, as_shot_neutral) = match &first.raw.profile { Some(p) => (p.dng_calibrations(), p.neutral().unwrap_or(neutral_from_wb)), None => (Vec::new(), neutral_from_wb), }; let unique_model = match (first.raw.make.trim(), first.raw.model.trim()) { ("", "") => "DarkRoom panorama".to_string(), (make, model) if model.starts_with(make) => model.to_string(), (make, model) => format!("{make} {model}"), }; dr_export::DngProfile { unique_model, calibrations, as_shot_neutral, white_level, } } fn stem(p: &Path) -> String { p.file_stem() .map(|s| s.to_string_lossy().into_owned()) .unwrap_or_else(|| p.display().to_string()) } /// Where the composite is written until it is whole: `x.dng.part` for /// `x.dng`. fn part_name(path: &Path) -> PathBuf { let mut name = path.as_os_str().to_owned(); name.push(".part"); PathBuf::from(name) } /// `name` in `dir`, numbered if that name is taken: a merge never /// overwrites (FR-MRG-3). `name` in `dir`, or `name-2`, `name-3`… — the first that neither `dir` /// nor `taken` (the names already at the destination) holds. fn unused_name(dir: &Path, name: &str, taken: &std::collections::HashSet) -> PathBuf { let mut candidate = name.to_string(); let base = stem(Path::new(name)); let mut n = 2; while dir.join(&candidate).exists() || part_name(&dir.join(&candidate)).exists() || taken.contains(&candidate) { candidate = format!("{base}-{n}.dng"); n += 1; } dir.join(candidate) } /// Drain everything a job has said so far. /// Everything the worker has said since the last call, and whether it has /// hung up — a closed channel after nothing [`MergeEvent::is_final`] is a /// worker that died mid-job. pub fn drain(rx: &Receiver) -> (Vec, bool) { let mut out = Vec::new(); loop { match rx.try_recv() { Ok(e) => out.push(e), Err(std::sync::mpsc::TryRecvError::Empty) => return (out, false), Err(std::sync::mpsc::TryRecvError::Disconnected) => return (out, true), } } } /// The filler's input as the debugging example reads it: `input.ppm`, the /// display-space picture, and `known.pgm`, 255 where a frame reached. fn dump_fill_input( dir: &Path, rgb: &[f32], known: &[bool], w: usize, h: usize, ) -> std::io::Result<()> { std::fs::create_dir_all(dir)?; let mut ppm = format!("P6\n{w} {h}\n255\n").into_bytes(); ppm.extend( rgb.iter() .map(|v| (v.clamp(0.0, 1.0) * 255.0).round() as u8), ); std::fs::write(dir.join("input.ppm"), ppm)?; let mut pgm = format!("P5\n{w} {h}\n255\n").into_bytes(); pgm.extend(known.iter().map(|&k| if k { 255u8 } else { 0 })); std::fs::write(dir.join("known.pgm"), pgm) } #[cfg(test)] mod tests { use super::*; #[test] fn a_blown_sample_stays_grey_under_a_gain_below_one() { // The 6D's as-shot balance: a clipped (1, 1, 1) balances to // (1.93, 1, 1.68), magenta. After the rule, balanced, every channel // is equal — and still equal once a gain of 0.9 has scaled it. let wb = [1.93, 1.0, 1.68]; let cam = grey_if_blown([1.0, 1.0, 1.0], wb).map(|v| v * 0.9); let b = [cam[0] * wb[0], cam[1] * wb[1], cam[2] * wb[2]]; assert!( (b[0] - b[1]).abs() < 1e-5 && (b[1] - b[2]).abs() < 1e-5, "{b:?}" ); // And a sample well below white is left alone. assert_eq!(grey_if_blown([0.5, 0.8, 0.4], wb), [0.5, 0.8, 0.4]); } #[test] fn a_frame_left_out_does_not_hold_the_merge_back() { let report = |frames: Vec>, included: Vec| AlignmentReport { focal_mm: 50.0, rms_px: 1.0, projection: Projection::Cylindrical, width: 1, height: 1, frames, included, links: 1, preview: None, filled: false, filler: Err(String::new()), }; let placed = || Ok((0.0, 0.0)); let lost = || Err("no consistent overlap with any other frame".to_string()); // A frame that could not be placed holds the merge back while it // is kept, and not once it is left out. assert!(!report(vec![placed(), placed(), lost()], vec![true; 3]).is_complete()); assert!(report(vec![placed(), placed(), lost()], vec![true, true, false]).is_complete()); // Two frames or it is not a panorama. assert!( !report(vec![placed(), placed(), placed()], vec![true, false, false]).is_complete() ); } #[test] fn a_second_merge_is_not_named_over_the_first_in_the_library() { // The outbox is empty once the first has uploaded, so only the // catalog knows the name is taken — and the upload replaces // whatever is at its name. let dir = std::env::temp_dir().join(format!("dr-merge-names-{}", std::process::id())); let _ = std::fs::remove_dir_all(&dir); std::fs::create_dir_all(&dir).unwrap(); let none = std::collections::HashSet::new(); assert_eq!( unused_name(&dir, "a-pano.dng", &none), dir.join("a-pano.dng") ); let taken: std::collections::HashSet = ["a-pano.dng".to_string(), "a-pano-2.dng".to_string()].into(); assert_eq!( unused_name(&dir, "a-pano.dng", &taken), dir.join("a-pano-3.dng") ); // And one still being written in the outbox holds its name too. std::fs::write(part_name(&dir.join("a-pano.dng")), b"").unwrap(); assert_eq!( unused_name(&dir, "a-pano.dng", &none), dir.join("a-pano-2.dng") ); let _ = std::fs::remove_dir_all(&dir); } #[test] fn a_worker_that_hangs_up_mid_job_is_reported_as_gone() { // The failure the page could not see: a panic drops the sender with // no final event, and `try_recv`'s Disconnected used to be folded // into "nothing new". let (tx, rx) = std::sync::mpsc::channel(); tx.send(MergeEvent::Progress { stage: "Reading", done: 1, total: 12, }) .unwrap(); let (events, gone) = drain(&rx); assert_eq!(events.len(), 1); assert!(!gone, "the sender is still alive"); drop(tx); let (events, gone) = drain(&rx); assert!(events.is_empty()); assert!(gone, "and now it is not"); } #[test] fn a_job_that_said_its_last_word_is_not_a_dead_worker() { let (tx, rx) = std::sync::mpsc::channel(); tx.send(MergeEvent::Failed("out of memory".into())).unwrap(); drop(tx); let (events, gone) = drain(&rx); assert!(gone); assert!(events.iter().any(MergeEvent::is_final), "Failed is final"); assert!(MergeEvent::Cancelled.is_final()); assert!(!MergeEvent::Progress { stage: "x", done: 0, total: 0 } .is_final()); } }