dtourolle
414094bd38
With the trait in place the claim still meant nothing while every caller named dr_decode's free functions: a second decoder would have had to be threaded through the scan, the thumbnail ladder, import, the viewer, export, merge and repairs at the moment it arrived. Each of those now takes a &dyn Decoder and reads headers, previews, orientation and sensor data through it, including the header budget a remote fetch asks for (header_bytes) and where it finds the embedded preview (locate_preview). Only the places that start a job name dr_decode::default(): the thumbnail, sweep and thumbnail-sweep threads, the viewer's open handlers, and the request structs a job is handed (BatchRequest, MergeRequest, the import Request, the repairs Toolkit), so a caller can be given another decoder by changing what it is handed. The default is rawler through the same free functions as before, so nothing a user sees changes. The trait gains Debug as a supertrait so request structs that derive Debug can carry one.
1394 lines
52 KiB
Rust
1394 lines
52 KiB
Rust
//! TRACES: FR-MRG-1 | FR-MRG-2 | FR-MRG-3 | FR-MRG-5 | FR-MRG-7
|
||
//! The merge job: from a set of files to a linear DNG beside them.
|
||
//!
|
||
//! The orchestration of a panorama, with no interface types in it: it runs
|
||
//! on a thread of its own and reports through a channel, on the pattern
|
||
//! [`crate::export`]'s batch established (FR-MRG-7). The interface drains
|
||
//! the channel from a timer; a headless example drains it from a loop.
|
||
//!
|
||
//! # Stages
|
||
//!
|
||
//! 1. **Load.** Each file is read and decoded to sensor data, and its edit
|
||
//! graph built the way a session would — orientation, lens profile —
|
||
//! because those are the two things the camera-space tap uses
|
||
//! (FR-MRG-2).
|
||
//! 2. **Proxies.** Each frame is demosaiced and rendered through the tap
|
||
//! at proxy size, upright; the keypoint detector reads that.
|
||
//! 3. **Alignment.** `dr_pano::align`. A frame it could not place stops
|
||
//! the job with the frame named (FR-MRG-5).
|
||
//! 4. **Gain.** One scalar per frame from the proxies' overlaps, so the
|
||
//! stop of exposure drift a hand-held sweep collects does not band.
|
||
//! 5. **Merge.** `dr_gpu::MergePass`, chunk by chunk, into a DNG written
|
||
//! strip by strip beside the first source (FR-MRG-3).
|
||
//!
|
||
//! The demosaiced frames are the memory: 160 MB each at f16 for a 20 MP
|
||
//! body, so at most two are resident and the rest are demosaiced again
|
||
//! when a band needs them. That is the trade FR-MRG-11 asks for — the
|
||
//! sensor data stays on the CPU at a fifth of the size — and it is what
|
||
//! makes a twelve-frame set fit.
|
||
|
||
use std::collections::VecDeque;
|
||
use std::path::{Path, PathBuf};
|
||
use std::sync::mpsc::{Receiver, Sender};
|
||
use std::sync::Arc;
|
||
use std::time::Instant;
|
||
|
||
use dr_decode::RawImage;
|
||
use dr_gpu::{
|
||
AdjustPass, DemosaicedImage, Demosaicer, GpuContext, MergeFrame, MergeOutput, MergePass,
|
||
};
|
||
use dr_pano::bundle::Cameras;
|
||
use dr_pano::projection::{self, Projection};
|
||
use dr_pano::{Alignment, Gray};
|
||
use dr_pipeline::EditGraph;
|
||
|
||
pub use crate::export::Cancel;
|
||
|
||
/// The border filler's weights, where the app looks for them; for the
|
||
/// headless example, which has no library open.
|
||
pub fn inpaint_model_path() -> Option<PathBuf> {
|
||
crate::library::inpaint_model()
|
||
}
|
||
|
||
/// One frame as the job receives it: the file's bytes, already fetched,
|
||
/// and a name for messages and for the composite's own name.
|
||
#[derive(Debug, Clone)]
|
||
pub struct MergeInput {
|
||
/// The file's name, `_MG_8320.CR2`.
|
||
pub name: String,
|
||
pub bytes: Arc<Vec<u8>>,
|
||
}
|
||
|
||
impl MergeInput {
|
||
pub fn read(path: &Path) -> Result<Self, String> {
|
||
Ok(MergeInput {
|
||
name: path
|
||
.file_name()
|
||
.map(|n| n.to_string_lossy().into_owned())
|
||
.unwrap_or_else(|| "frame".into()),
|
||
bytes: Arc::new(std::fs::read(path).map_err(|e| format!("{}: {e}", path.display()))?),
|
||
})
|
||
}
|
||
}
|
||
|
||
/// Where the composite goes (FR-MRG-3).
|
||
#[derive(Debug, Clone)]
|
||
pub enum MergeDestination {
|
||
/// A folder on this device: written there directly.
|
||
Local(PathBuf),
|
||
/// Beside its sources in the library, through the export outbox: the
|
||
/// file is staged with a destination record and the drain uploads it —
|
||
/// the one path that works for a folder library and a server alike, and
|
||
/// the one FR-MRG-3 names for a folder the job cannot write itself.
|
||
Outbox {
|
||
outbox: PathBuf,
|
||
/// The sources' folder, relative to the library root.
|
||
remote_dir: String,
|
||
},
|
||
}
|
||
|
||
/// What the job is told.
|
||
#[derive(Debug, Clone)]
|
||
pub struct MergeRequest {
|
||
/// The frames, in capture order.
|
||
pub frames: Vec<MergeInput>,
|
||
pub destination: MergeDestination,
|
||
/// `None` for the projection the field of view suggests.
|
||
pub projection: Option<Projection>,
|
||
/// Pixels over which a frame's weight ramps up from its edge.
|
||
pub feather_px: f32,
|
||
/// GPU work unit; also the DNG strip height.
|
||
pub chunk: (u32, u32),
|
||
/// The border filler's weights, if the device has them (FR-MRG-4).
|
||
/// `None` and the fill is not offered.
|
||
pub inpaint_model: Option<PathBuf>,
|
||
/// TRACES: FR-RAW-2
|
||
/// What reads and decodes each frame.
|
||
pub decoder: &'static dyn dr_decode::Decoder,
|
||
}
|
||
|
||
impl MergeRequest {
|
||
pub fn new(frames: Vec<MergeInput>, destination: MergeDestination) -> Self {
|
||
MergeRequest {
|
||
frames,
|
||
destination,
|
||
projection: None,
|
||
feather_px: 200.0,
|
||
chunk: (2048, 512),
|
||
inpaint_model: None,
|
||
decoder: dr_decode::default(),
|
||
}
|
||
}
|
||
}
|
||
|
||
/// What the photographer decides once the alignment is shown (FR-MRG-1:
|
||
/// never automatic).
|
||
#[derive(Debug, Clone, Copy, PartialEq)]
|
||
pub enum Decision {
|
||
Merge {
|
||
projection: Option<Projection>,
|
||
/// Fill the border with invented pixels rather than crop to the
|
||
/// picture (FR-MRG-4), with these knobs. Never the default; the
|
||
/// page asks.
|
||
fill: Option<FillSettings>,
|
||
},
|
||
/// Draw the alignment again on another surface, and report it again.
|
||
/// The page's projection chips: what a choice looks like before it is
|
||
/// confirmed, at proxy cost rather than the merge's — and the fill, at
|
||
/// preview size, for the same reason: invented pixels are seen before
|
||
/// they are written.
|
||
Preview {
|
||
projection: Option<Projection>,
|
||
fill: Option<FillSettings>,
|
||
},
|
||
Abandon,
|
||
}
|
||
|
||
/// TRACES: FR-MRG-4
|
||
/// The border fill's knobs, every one on the page while the fill is
|
||
/// experimental: the fill's own (`dr_pano::FillParams`, in pixels of the
|
||
/// working image) and the two around it.
|
||
#[derive(Debug, Clone, Copy, PartialEq)]
|
||
pub struct FillSettings {
|
||
/// The working image is the composite at `1/scale`: 2 is half, the
|
||
/// desktop's default; 4 a quarter, for the tablet (FR-MRG-9).
|
||
pub scale: u32,
|
||
/// How many working pixels of the coverage edge count as fringe and
|
||
/// are regenerated too.
|
||
pub erosion: usize,
|
||
pub params: dr_pano::FillParams,
|
||
}
|
||
|
||
impl Default for FillSettings {
|
||
fn default() -> Self {
|
||
FillSettings {
|
||
scale: 2,
|
||
erosion: 4,
|
||
params: dr_pano::FillParams {
|
||
feather: 48,
|
||
..Default::default()
|
||
},
|
||
}
|
||
}
|
||
}
|
||
|
||
impl FillSettings {
|
||
/// The same knobs for a picture at `ratio` of the composite's size
|
||
/// (the page's preview): the working-pixel counts rescaled so the fill
|
||
/// looks the same there as it will in the merge.
|
||
pub fn at(&self, ratio: f64) -> FillSettings {
|
||
let k = ratio * self.scale as f64;
|
||
let px = |v: usize| ((v as f64) * k).ceil().max(1.0) as usize;
|
||
FillSettings {
|
||
scale: 1,
|
||
erosion: px(self.erosion),
|
||
params: dr_pano::FillParams {
|
||
coarse: self.params.coarse,
|
||
band: px(self.params.band).max(8),
|
||
mirror_depth: if self.params.mirror_depth == 0 {
|
||
0
|
||
} else {
|
||
px(self.params.mirror_depth)
|
||
},
|
||
feather: if self.params.feather == 0 {
|
||
0
|
||
} else {
|
||
px(self.params.feather)
|
||
},
|
||
stride: self.params.stride,
|
||
},
|
||
}
|
||
}
|
||
}
|
||
|
||
/// What the job says while it runs.
|
||
#[derive(Debug, Clone)]
|
||
pub enum MergeEvent {
|
||
/// A stage, with progress within it.
|
||
Progress {
|
||
stage: &'static str,
|
||
done: usize,
|
||
total: usize,
|
||
},
|
||
/// The alignment, before any pixel is written: what the interface shows
|
||
/// for the photographer to confirm, and what names a failure.
|
||
Aligned(AlignmentReport),
|
||
/// The composite is written: on the device at `path`, or staged in the
|
||
/// outbox for the drain to upload, in which case `staged` is true and
|
||
/// the library learns of it when the folder is next scanned.
|
||
Done {
|
||
path: PathBuf,
|
||
staged: bool,
|
||
width: u32,
|
||
height: u32,
|
||
},
|
||
Failed(String),
|
||
Cancelled,
|
||
}
|
||
|
||
impl MergeEvent {
|
||
/// Whether this is the job's last word: after one of these the worker
|
||
/// has nothing more to say, so its channel closing is expected.
|
||
pub fn is_final(&self) -> bool {
|
||
matches!(
|
||
self,
|
||
MergeEvent::Done { .. } | MergeEvent::Failed(_) | MergeEvent::Cancelled
|
||
)
|
||
}
|
||
}
|
||
|
||
/// The alignment, described for a panel.
|
||
#[derive(Debug, Clone)]
|
||
pub struct AlignmentReport {
|
||
/// Equivalent focal length in millimetres on full frame, from the fit.
|
||
pub focal_mm: f64,
|
||
pub rms_px: f64,
|
||
pub projection: Projection,
|
||
pub width: u32,
|
||
pub height: u32,
|
||
/// Per frame, in input order: yaw and pitch in degrees if aligned, or
|
||
/// why not.
|
||
pub frames: Vec<Result<(f64, f64), String>>,
|
||
pub links: usize,
|
||
/// The aligned set drawn on the suggested surface at proxy resolution,
|
||
/// `(width, height, rgba)` — what the photographer confirms.
|
||
pub preview: Option<(u32, u32, Vec<u8>)>,
|
||
/// Whether the preview's border is filled (FR-MRG-4).
|
||
pub filled: bool,
|
||
/// The filler: where it runs, or why it is not available.
|
||
pub filler: Result<String, String>,
|
||
}
|
||
|
||
/// Run the whole job on the calling thread, reporting on `events`.
|
||
///
|
||
/// After `Aligned` the job waits on `decision` — the photographer's
|
||
/// confirmation, or the headless caller's immediate `Merge` — and returns
|
||
/// when the file is written, the job failed, or `cancel` was seen.
|
||
pub fn run(
|
||
ctx: GpuContext,
|
||
request: MergeRequest,
|
||
events: Sender<MergeEvent>,
|
||
decision: Receiver<Decision>,
|
||
cancel: Cancel,
|
||
) {
|
||
let send = |e: MergeEvent| {
|
||
let _ = events.send(e);
|
||
};
|
||
// Caught rather than allowed to unwind the thread: wgpu reports a device
|
||
// that has run out of memory by panicking, and a twelve-frame merge on a
|
||
// GPU another process is using is exactly where that happens. Uncaught,
|
||
// the thread died, the sender went with it, and the page sat on "Stop"
|
||
// with every control disabled and nothing to say why — the crash record
|
||
// on disk was the only sign. The panic hook still writes that record;
|
||
// this is what puts the reason on the page.
|
||
let result = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| {
|
||
run_inner(&ctx, &request, &events, &decision, &cancel)
|
||
}));
|
||
match result {
|
||
Ok(Ok(Some(done))) => send(done),
|
||
Ok(Ok(None)) => send(MergeEvent::Cancelled),
|
||
Ok(Err(e)) => send(MergeEvent::Failed(e)),
|
||
Err(panic) => {
|
||
let detail = panic
|
||
.downcast_ref::<&str>()
|
||
.map(|s| (*s).to_string())
|
||
.or_else(|| panic.downcast_ref::<String>().cloned())
|
||
.unwrap_or_else(|| "panicked with a non-string payload".to_string());
|
||
log::error!("merge worker panicked: {detail}");
|
||
send(MergeEvent::Failed(format!("internal error: {detail}")));
|
||
}
|
||
}
|
||
}
|
||
|
||
/// The frames as loaded: sensor data on the CPU and the graph the tap uses.
|
||
struct Loaded {
|
||
raw: RawImage,
|
||
meta: dr_decode::Metadata,
|
||
graph: Arc<EditGraph>,
|
||
/// The upright size, which is what the cameras are in.
|
||
size: (u32, u32),
|
||
}
|
||
|
||
fn run_inner(
|
||
ctx: &GpuContext,
|
||
request: &MergeRequest,
|
||
events: &Sender<MergeEvent>,
|
||
decision: &Receiver<Decision>,
|
||
cancel: &Cancel,
|
||
) -> Result<Option<MergeEvent>, String> {
|
||
if request.frames.len() < 2 {
|
||
return Err("a panorama needs at least two frames".into());
|
||
}
|
||
let progress = |stage: &'static str, done: usize, total: usize| {
|
||
let _ = events.send(MergeEvent::Progress { stage, done, total });
|
||
};
|
||
|
||
// 1. Load.
|
||
let t = Instant::now();
|
||
let mut frames: Vec<Loaded> = Vec::with_capacity(request.frames.len());
|
||
for (i, input) in request.frames.iter().enumerate() {
|
||
if cancel.is_cancelled() {
|
||
return Ok(None);
|
||
}
|
||
progress("Reading", i, request.frames.len());
|
||
let bytes = &input.bytes;
|
||
let meta = request.decoder.metadata(bytes).unwrap_or_default();
|
||
let raw = request
|
||
.decoder
|
||
.decode(bytes)
|
||
.map_err(|e| format!("{}: {e}", input.name))?;
|
||
let orientation = meta.orientation.unwrap_or_default();
|
||
let mut graph = EditGraph::default_chain();
|
||
graph.set_orientation(orientation);
|
||
graph.set_lens_profile(crate::develop::DevelopSession::profile_for(&meta));
|
||
let (w, h) = (raw.crop.width, raw.crop.height);
|
||
let size = if orientation.quarter_turns % 2 == 1 {
|
||
(h, w)
|
||
} else {
|
||
(w, h)
|
||
};
|
||
frames.push(Loaded {
|
||
raw,
|
||
meta,
|
||
graph: Arc::new(graph),
|
||
size,
|
||
});
|
||
}
|
||
log::info!("merge: {} frames read in {:?}", frames.len(), t.elapsed());
|
||
|
||
// 2. Proxies and keypoints.
|
||
let t = Instant::now();
|
||
let demosaicer = Demosaicer::new(ctx).map_err(|e| e.to_string())?;
|
||
let mut adjust = AdjustPass::new(ctx);
|
||
let mut detector = dr_pano::xfeat::XFeat::embedded().map_err(|e| e.to_string())?;
|
||
let mut features = Vec::with_capacity(frames.len());
|
||
let mut proxies: Vec<Gray> = Vec::with_capacity(frames.len());
|
||
let mut colour: Vec<Vec<f32>> = Vec::with_capacity(frames.len());
|
||
for (i, f) in frames.iter().enumerate() {
|
||
if cancel.is_cancelled() {
|
||
return Ok(None);
|
||
}
|
||
progress("Finding features", i, frames.len());
|
||
let image = demosaicer.run(&f.raw).map_err(|e| e.to_string())?;
|
||
let (proxy, rgb) = camera_proxy(&mut adjust, &image, &f.graph, f.size)?;
|
||
features.push(detector.detect(&proxy).map_err(|e| e.to_string())?);
|
||
proxies.push(proxy);
|
||
colour.push(rgb);
|
||
}
|
||
log::info!("merge: features in {:?}", t.elapsed());
|
||
|
||
// 3. Alignment.
|
||
let t = Instant::now();
|
||
progress("Aligning", 0, 1);
|
||
let alignment =
|
||
dr_pano::align(&features, &dr_pano::AlignOptions::default()).map_err(|e| e.to_string())?;
|
||
log::info!(
|
||
"merge: aligned in {:?}, focal {:.1} px, rms {:.2} px, {} links",
|
||
t.elapsed(),
|
||
alignment.focal,
|
||
alignment.rms_px,
|
||
alignment.links.len()
|
||
);
|
||
|
||
// The geometry at full resolution: the proxy's long edge against the
|
||
// frame's.
|
||
let proxy_long = proxies[0].width.max(proxies[0].height) as f64;
|
||
let full_long = frames[0].size.0.max(frames[0].size.1) as f64;
|
||
let focal_full = alignment.focal * full_long / proxy_long;
|
||
let cameras = Cameras {
|
||
rotations: alignment
|
||
.rotations
|
||
.iter()
|
||
.map(|r| r.unwrap_or(dr_pano::linalg::Mat3::IDENTITY))
|
||
.collect(),
|
||
focal: focal_full,
|
||
};
|
||
let frame_size = (frames[0].size.0 as f64, frames[0].size.1 as f64);
|
||
let (hfov, vfov) = field_of_view(&cameras, frame_size);
|
||
let projection = request
|
||
.projection
|
||
.unwrap_or_else(|| Projection::suggest(hfov, vfov));
|
||
let suggested = projection;
|
||
|
||
// 4. Gains, before the report so the preview shows them.
|
||
let gains = if alignment.is_complete() {
|
||
gains(&proxies, &alignment)
|
||
} else {
|
||
vec![1.0; frames.len()]
|
||
};
|
||
// The filler, loaded once if the device has it. Its absence is a
|
||
// report, not a failure: the crop is always there.
|
||
let mut filler: Option<dr_pano::migan::MiGan> = None;
|
||
let filler_status: Result<String, String> = match request.inpaint_model.as_deref() {
|
||
None => Err("no border-fill model on this device".into()),
|
||
Some(path) => match dr_pano::migan::MiGan::from_path(path) {
|
||
Ok(m) => {
|
||
let rung = m.rung().map(|r| r.label().to_string()).unwrap_or_default();
|
||
filler = Some(m);
|
||
Ok(rung)
|
||
}
|
||
Err(e) => Err(format!("the border-fill model could not be loaded: {e}")),
|
||
},
|
||
};
|
||
let first_raw = &frames[0].raw;
|
||
let mut report =
|
||
|projection: Projection, fill: Option<FillSettings>| -> Result<AlignmentReport, String> {
|
||
let bounds = projection::bounds(projection, focal_full, &cameras, frame_size)
|
||
.ok_or("the frames project nowhere")?;
|
||
let mut filled = false;
|
||
let preview = match alignment.is_complete() {
|
||
false => None,
|
||
true => {
|
||
let (w, h, mut rgb, mut known) = preview_planes(
|
||
&colour, &proxies, &alignment, &gains, projection, first_raw, 1600,
|
||
);
|
||
if let Some(settings) = fill {
|
||
if let Some(model) = filler.as_mut() {
|
||
progress("Filling the preview", 0, 1);
|
||
// The knobs at the preview's scale.
|
||
let here = settings.at(w as f64 / bounds.width());
|
||
dr_pano::fill::erode(&mut known, w, h, here.erosion);
|
||
match dr_pano::fill_border(
|
||
&mut rgb,
|
||
w,
|
||
h,
|
||
&known,
|
||
model,
|
||
here.params,
|
||
&mut |_, _| {},
|
||
) {
|
||
Ok(_) => filled = true,
|
||
Err(e) => log::warn!("preview fill: {e}"),
|
||
}
|
||
}
|
||
}
|
||
Some(preview_rgba(
|
||
&rgb,
|
||
w,
|
||
h,
|
||
&alignment,
|
||
projection,
|
||
&bounds_of(&proxies, &alignment, projection),
|
||
&proxies,
|
||
))
|
||
}
|
||
};
|
||
Ok(AlignmentReport {
|
||
focal_mm: focal_full * 36.0 / full_long,
|
||
rms_px: alignment.rms_px,
|
||
projection,
|
||
width: bounds.width().ceil() as u32,
|
||
height: bounds.height().ceil() as u32,
|
||
frames: describe(&alignment),
|
||
links: alignment.links.len(),
|
||
preview,
|
||
filled,
|
||
filler: filler_status.clone(),
|
||
})
|
||
};
|
||
progress("Drawing the preview", 0, 1);
|
||
let _ = events.send(MergeEvent::Aligned(report(suggested, None)?));
|
||
if !alignment.is_complete() {
|
||
let names: Vec<String> = alignment
|
||
.unaligned
|
||
.iter()
|
||
.map(|(k, why)| format!("{}: {why}", request.frames[*k].name))
|
||
.collect();
|
||
return Err(format!(
|
||
"not every frame could be placed — {}",
|
||
names.join("; ")
|
||
));
|
||
}
|
||
|
||
// Never automatic (FR-MRG-1): nothing is written until the alignment
|
||
// has been seen and confirmed. Polled, so a cancel while waiting is
|
||
// seen within a moment.
|
||
let (projection, fill) = loop {
|
||
if cancel.is_cancelled() {
|
||
return Ok(None);
|
||
}
|
||
match decision.recv_timeout(std::time::Duration::from_millis(100)) {
|
||
Ok(Decision::Merge {
|
||
projection: p,
|
||
fill,
|
||
}) => break (p.unwrap_or(suggested), fill.filter(|_| filler.is_some())),
|
||
Ok(Decision::Preview {
|
||
projection: p,
|
||
fill,
|
||
}) => {
|
||
let _ = events.send(MergeEvent::Aligned(report(p.unwrap_or(suggested), fill)?));
|
||
}
|
||
Ok(Decision::Abandon) => return Ok(None),
|
||
Err(std::sync::mpsc::RecvTimeoutError::Timeout) => continue,
|
||
Err(std::sync::mpsc::RecvTimeoutError::Disconnected) => return Ok(None),
|
||
}
|
||
};
|
||
let bounds = projection::bounds(projection, focal_full, &cameras, frame_size)
|
||
.ok_or("the frames project nowhere")?;
|
||
|
||
// 5. Merge, into a DNG beside the first frame.
|
||
let name = format!("{}-pano.dng", stem(Path::new(&request.frames[0].name)));
|
||
let (out_path, staged) = match &request.destination {
|
||
MergeDestination::Local(dir) => (unused_name(dir, &name), false),
|
||
MergeDestination::Outbox { outbox, remote_dir } => {
|
||
std::fs::create_dir_all(outbox).map_err(|e| format!("{}: {e}", outbox.display()))?;
|
||
let path = unused_name(outbox, &name);
|
||
// The record first here, unlike an export: the payload is
|
||
// written over minutes and a record naming a half-written file
|
||
// is worse than a payload with no record, so the record is
|
||
// removed again if the merge fails. The drain skips payloads
|
||
// without records.
|
||
let record = crate::export::destination_record(&path);
|
||
let final_name = path
|
||
.file_name()
|
||
.map(|n| n.to_string_lossy().into_owned())
|
||
.unwrap_or(name.clone());
|
||
std::fs::write(&record, format!("{remote_dir}\n{final_name}\n"))
|
||
.map_err(|e| format!("{}: {e}", record.display()))?;
|
||
(path, true)
|
||
}
|
||
};
|
||
let cleanup = |path: &Path| {
|
||
let _ = std::fs::remove_file(path);
|
||
if staged {
|
||
let _ = std::fs::remove_file(crate::export::destination_record(path));
|
||
}
|
||
};
|
||
let (out_w, out_h) = (report_size(&bounds).0, report_size(&bounds).1);
|
||
let first = &frames[0];
|
||
let black = first.raw.black_level[0];
|
||
let white_level = u32::from(first.raw.white_level.saturating_sub(black)).max(1);
|
||
let profile = dng_profile(first, white_level);
|
||
let (_, mut carried) = crate::export::header_for_file(&first.meta);
|
||
// The composite is dated at the middle of its sweep — the mean of the
|
||
// frames' capture times — so it sorts among the frames it was made
|
||
// from, not at the first of them and not at the moment of the merge.
|
||
// The zone is the first frame's; a sweep does not cross one.
|
||
let stamps: Vec<i64> = frames.iter().filter_map(|f| f.meta.captured_at).collect();
|
||
if !stamps.is_empty() {
|
||
let sum: i128 = stamps.iter().map(|&t| t as i128).sum();
|
||
carried.captured_at = Some((sum / stamps.len() as i128) as i64);
|
||
}
|
||
|
||
let output = MergeOutput {
|
||
projection,
|
||
scale: focal_full,
|
||
bounds,
|
||
feather: request.feather_px,
|
||
chunk: request.chunk,
|
||
sample_scale: white_level as f32,
|
||
};
|
||
let merge_frames: Vec<MergeFrame> = frames
|
||
.iter()
|
||
.zip(&gains)
|
||
.map(|(f, &gain)| MergeFrame {
|
||
graph: f.graph.clone(),
|
||
gain,
|
||
})
|
||
.collect();
|
||
|
||
let t = Instant::now();
|
||
let mut pass = MergePass::new(ctx).map_err(|e| e.to_string())?;
|
||
let mut resident: VecDeque<(usize, Arc<DemosaicedImage>)> = VecDeque::new();
|
||
|
||
// 5a. The border fill (FR-MRG-4), when asked for: the composite at a
|
||
// quarter of its resolution, its border filled by the model in display
|
||
// space, and the result taken back to camera space for the full merge
|
||
// to sample wherever nothing was covered. A quarter because the fill is
|
||
// sky and slope at the edge of a picture, where a quarter is more than
|
||
// the eye resolves, and because it puts the fixture's border at thirty
|
||
// tiles rather than a hundred.
|
||
let fill_cam: Option<(usize, usize, Vec<f32>, Vec<bool>)> = if let Some(settings) = fill {
|
||
let model = filler.as_mut().expect("fill was only chosen with a filler");
|
||
progress("Filling the border", 0, 1);
|
||
// Half the composite's resolution by default: a quarter left a
|
||
// visible gap in sharpness against the real pixels beside the
|
||
// fill. Memory is the working image in f32, 400 MB for a 137 MP
|
||
// composite; the tablet will want a quarter (FR-MRG-9).
|
||
let q = settings.scale.max(1) as f64;
|
||
let qout = MergeOutput {
|
||
projection,
|
||
scale: focal_full / q,
|
||
bounds: projection::Bounds {
|
||
min_u: bounds.min_u / q,
|
||
min_v: bounds.min_v / q,
|
||
max_u: bounds.max_u / q,
|
||
max_v: bounds.max_v / q,
|
||
},
|
||
feather: request.feather_px,
|
||
chunk: request.chunk,
|
||
sample_scale: white_level as f32,
|
||
};
|
||
let (qw, qh) = (qout.width() as usize, qout.height() as usize);
|
||
let mut qrgb = vec![0.0f32; qw * qh * 3];
|
||
let mut qknown = vec![false; qw * qh];
|
||
pass.merge(
|
||
&mut adjust,
|
||
&merge_frames,
|
||
&cameras,
|
||
(frames[0].size.0, frames[0].size.1),
|
||
&qout,
|
||
|k| {
|
||
if let Some((_, img)) = resident.iter().find(|(i, _)| *i == k) {
|
||
return Ok(img.clone());
|
||
}
|
||
let img = Arc::new(demosaicer.run(&frames[k].raw)?);
|
||
resident.push_back((k, img.clone()));
|
||
while resident.len() > 2 {
|
||
resident.pop_front();
|
||
}
|
||
Ok(img)
|
||
},
|
||
|band| {
|
||
let first = band.first_row as usize;
|
||
for r in 0..band.rows as usize {
|
||
for x in 0..qw {
|
||
let src = r * qw + x;
|
||
let dst = (first + r) * qw + x;
|
||
for c in 0..3 {
|
||
qrgb[dst * 3 + c] =
|
||
f32::from(band.rgb[src * 3 + c]) / white_level as f32;
|
||
}
|
||
qknown[dst] = band.covered[src];
|
||
}
|
||
}
|
||
Ok(())
|
||
},
|
||
|| cancel.is_cancelled(),
|
||
)
|
||
.map_err(|e| e.to_string())?;
|
||
if cancel.is_cancelled() {
|
||
cleanup(&out_path);
|
||
return Ok(None);
|
||
}
|
||
// Camera space to the display-ish space the model was trained on.
|
||
let look = Look::of(first_raw);
|
||
let mut disp = qrgb.clone();
|
||
for p in disp.chunks_exact_mut(3) {
|
||
let d = look.to_display([p[0], p[1], p[2]]);
|
||
p.copy_from_slice(&d);
|
||
}
|
||
dr_pano::fill::erode(&mut qknown, qw, qh, settings.erosion);
|
||
// `DR_FILL_DUMP=dir` writes what the filler was given, so a fill
|
||
// can be re-run and looked at stage by stage without the merge
|
||
// (`cargo run -p dr-pano --example fill`).
|
||
if let Some(dir) = std::env::var_os("DR_FILL_DUMP") {
|
||
let dir = PathBuf::from(dir);
|
||
if let Err(e) = dump_fill_input(&dir, &disp, &qknown, qw, qh) {
|
||
log::warn!("fill dump: {e}");
|
||
}
|
||
}
|
||
let total_tiles = std::cell::Cell::new(0usize);
|
||
let feather = settings.params.feather;
|
||
dr_pano::fill_border(
|
||
&mut disp,
|
||
qw,
|
||
qh,
|
||
&qknown,
|
||
model,
|
||
settings.params,
|
||
&mut |n, total| {
|
||
total_tiles.set(total);
|
||
progress("Filling the border", n, total.max(1));
|
||
},
|
||
)
|
||
.map_err(|e| e.to_string())?;
|
||
// And back: the filled pixels — the hole and the feathered margin —
|
||
// to camera space; the rest as the merge produced them.
|
||
let mut margin = qknown.clone();
|
||
dr_pano::fill::erode(&mut margin, qw, qh, feather);
|
||
for (i, p) in qrgb.chunks_exact_mut(3).enumerate() {
|
||
if !margin[i] {
|
||
let d = &disp[i * 3..i * 3 + 3];
|
||
let cam = look.to_camera([d[0], d[1], d[2]]);
|
||
p.copy_from_slice(&cam);
|
||
}
|
||
}
|
||
let fill_mask: Vec<bool> = margin;
|
||
log::info!(
|
||
"merge: border filled at {qw}×{qh}, {} tiles, in {:?}",
|
||
total_tiles.get(),
|
||
t.elapsed()
|
||
);
|
||
Some((qw, qh, qrgb, fill_mask))
|
||
} else {
|
||
None
|
||
};
|
||
|
||
// The writer pulls strips; the merge pushes bands. A channel between
|
||
// them, and the writer on its own thread, so neither waits on the
|
||
// other's pace more than one band.
|
||
let (band_tx, band_rx) = std::sync::mpsc::sync_channel::<Vec<u16>>(1);
|
||
let rows_per_strip = request.chunk.1.max(1);
|
||
// The crop (FR-MRG-4), found as the bands go by; the writer reads it
|
||
// once the last strip is in, by which time every band has been seen.
|
||
let inscribed = Arc::new(std::sync::Mutex::new(dr_export::Inscribed::new(out_w)));
|
||
let inscribed_for_writer = inscribed.clone();
|
||
let fill_for_writer = fill_cam.is_some();
|
||
let file =
|
||
std::fs::File::create(&out_path).map_err(|e| format!("{}: {e}", out_path.display()))?;
|
||
let writer = std::thread::spawn(move || -> Result<(), String> {
|
||
let mut file = std::io::BufWriter::new(file);
|
||
dr_export::write_linear_dng(
|
||
&mut file,
|
||
out_w,
|
||
out_h,
|
||
rows_per_strip,
|
||
&profile,
|
||
Some(&carried),
|
||
|_, buf| {
|
||
let band = band_rx.recv().map_err(|_| {
|
||
dr_export::ExportError::Encode("the merge stopped early".into())
|
||
})?;
|
||
buf.extend_from_slice(&band);
|
||
Ok(())
|
||
},
|
||
// A filled composite is the whole picture; a cropped one opens
|
||
// on the rectangle the frames covered.
|
||
move || {
|
||
if fill_for_writer {
|
||
None
|
||
} else {
|
||
inscribed_for_writer.lock().ok().map(|i| i.best())
|
||
}
|
||
},
|
||
)
|
||
.map_err(|e| e.to_string())
|
||
});
|
||
|
||
let total_bands = out_h.div_ceil(rows_per_strip) as usize;
|
||
let mut bands_done = 0usize;
|
||
progress("Merging", 0, total_bands);
|
||
let merged = pass.merge(
|
||
&mut adjust,
|
||
&merge_frames,
|
||
&cameras,
|
||
(frames[0].size.0, frames[0].size.1),
|
||
&output,
|
||
|k| {
|
||
if let Some((_, img)) = resident.iter().find(|(i, _)| *i == k) {
|
||
return Ok(img.clone());
|
||
}
|
||
let img = Arc::new(demosaicer.run(&frames[k].raw)?);
|
||
resident.push_back((k, img.clone()));
|
||
while resident.len() > 2 {
|
||
resident.pop_front();
|
||
}
|
||
Ok(img)
|
||
},
|
||
|band| {
|
||
bands_done += 1;
|
||
progress("Merging", bands_done, total_bands);
|
||
if let Ok(mut i) = inscribed.lock() {
|
||
i.push_rows(band.covered, band.rows);
|
||
}
|
||
let mut rgb = band.rgb.to_vec();
|
||
// Where no frame reached, the fill — sampled bilinearly from
|
||
// the quarter-resolution result, at the sensor's scale.
|
||
if let Some((qw, qh, qrgb, fill_mask)) = &fill_cam {
|
||
let sx = *qw as f64 / out_w as f64;
|
||
let sy = *qh as f64 / out_h as f64;
|
||
for r in 0..band.rows as usize {
|
||
for x in 0..out_w as usize {
|
||
let i = r * out_w as usize + x;
|
||
let fx = ((x as f64 + 0.5) * sx - 0.5).clamp(0.0, (*qw - 1) as f64);
|
||
let fy = (((band.first_row as usize + r) as f64 + 0.5) * sy - 0.5)
|
||
.clamp(0.0, (*qh - 1) as f64);
|
||
// Uncovered, or inside the feathered margin: the fill.
|
||
let (mx, my) = (fx.round() as usize, fy.round() as usize);
|
||
if band.covered[i] && fill_mask[my.min(qh - 1) * qw + mx.min(qw - 1)] {
|
||
continue;
|
||
}
|
||
let (x0, y0) = (fx as usize, fy as usize);
|
||
let (x1, y1) = ((x0 + 1).min(qw - 1), (y0 + 1).min(qh - 1));
|
||
let (tx, ty) = ((fx - x0 as f64) as f32, (fy - y0 as f64) as f32);
|
||
for c in 0..3 {
|
||
let at = |xx: usize, yy: usize| qrgb[(yy * qw + xx) * 3 + c];
|
||
let v = (at(x0, y0) * (1.0 - tx) + at(x1, y0) * tx) * (1.0 - ty)
|
||
+ (at(x0, y1) * (1.0 - tx) + at(x1, y1) * tx) * ty;
|
||
rgb[i * 3 + c] =
|
||
(v * white_level as f32).round().clamp(0.0, 65535.0) as u16;
|
||
}
|
||
}
|
||
}
|
||
}
|
||
band_tx
|
||
.send(rgb)
|
||
.map_err(|_| dr_gpu::GpuError::Readback("the writer stopped".into()))
|
||
},
|
||
|| cancel.is_cancelled(),
|
||
);
|
||
drop(band_tx);
|
||
let written = writer
|
||
.join()
|
||
.unwrap_or_else(|_| Err("the writer panicked".into()));
|
||
match merged {
|
||
Ok(()) => {}
|
||
Err(e) if cancel.is_cancelled() => {
|
||
cleanup(&out_path);
|
||
log::info!("merge cancelled: {e}");
|
||
return Ok(None);
|
||
}
|
||
Err(e) => {
|
||
cleanup(&out_path);
|
||
return Err(e.to_string());
|
||
}
|
||
}
|
||
if let Err(e) = written {
|
||
cleanup(&out_path);
|
||
return Err(e);
|
||
}
|
||
log::info!(
|
||
"merge: {}×{} written to {} in {:?}",
|
||
out_w,
|
||
out_h,
|
||
out_path.display(),
|
||
t.elapsed()
|
||
);
|
||
|
||
// TRACES: FR-MRG-6
|
||
// Provenance, beside the composite: what it was merged from, in order,
|
||
// and how. Its own file and its own outbox record, so the drain places
|
||
// it as it places the composite; a sidecar that failed to write is
|
||
// reported but does not un-write the composite.
|
||
let mut sidecar = dr_pipeline::Sidecar::new();
|
||
sidecar.derived_from = request.frames.iter().map(|f| f.name.clone()).collect();
|
||
sidecar.merge = Some(format!(
|
||
"panorama {} {:.1}mm {} frames{}",
|
||
match projection {
|
||
Projection::Perspective => "perspective",
|
||
Projection::Cylindrical => "cylindrical",
|
||
Projection::Spherical => "spherical",
|
||
},
|
||
focal_full * 36.0 / full_long,
|
||
frames.len(),
|
||
// Invented pixels declare themselves (FR-MRG-6, D17's rule), with
|
||
// the knobs that made them while the fill is experimental.
|
||
match fill {
|
||
Some(f) => format!(
|
||
" border filled (1/{} scale, erosion {}, coarse {}, band {}, mirror {}, feather {})",
|
||
f.scale, f.erosion, f.params.coarse, f.params.band, f.params.mirror_depth, f.params.feather
|
||
),
|
||
None => String::new(),
|
||
}
|
||
));
|
||
let sidecar_path = out_path.with_extension(dr_pipeline::sidecar::EXTENSION);
|
||
if let Err(e) = std::fs::write(&sidecar_path, sidecar.to_text()) {
|
||
log::warn!("merge: the sidecar could not be written: {e}");
|
||
} else if let MergeDestination::Outbox { remote_dir, .. } = &request.destination {
|
||
let record = crate::export::destination_record(&sidecar_path);
|
||
let name = sidecar_path
|
||
.file_name()
|
||
.map(|n| n.to_string_lossy().into_owned())
|
||
.unwrap_or_default();
|
||
if let Err(e) = std::fs::write(&record, format!("{remote_dir}\n{name}\n")) {
|
||
log::warn!("merge: the sidecar's record could not be written: {e}");
|
||
}
|
||
}
|
||
|
||
Ok(Some(MergeEvent::Done {
|
||
path: out_path,
|
||
staged,
|
||
width: out_w,
|
||
height: out_h,
|
||
}))
|
||
}
|
||
|
||
/// The proxy the detector reads: the frame through the camera-space tap at
|
||
/// proxy size, upright, as gamma-encoded grey.
|
||
///
|
||
/// Through the tap rather than the embedded preview so that the keypoints
|
||
/// are in the *undistorted* frame the tiles will be rendered in — a lens
|
||
/// profile moves the corners by tens of pixels at full resolution, and an
|
||
/// alignment measured on the distorted preview would be wrong by that much
|
||
/// at the seams.
|
||
fn camera_proxy(
|
||
adjust: &mut AdjustPass,
|
||
image: &DemosaicedImage,
|
||
graph: &EditGraph,
|
||
upright: (u32, u32),
|
||
) -> Result<(Gray, Vec<f32>), String> {
|
||
let long = dr_pano::xfeat::INPUT_LONG_EDGE as f64;
|
||
let scale = (long / f64::from(upright.0.max(upright.1))).min(1.0);
|
||
let w = ((f64::from(upright.0) * scale).round() as u32).max(1);
|
||
let h = ((f64::from(upright.1) * scale).round() as u32).max(1);
|
||
let shader = graph.compose_camera_linear(dr_pipeline::CropRect::default());
|
||
adjust
|
||
.render_camera_linear(image, &shader, w, h)
|
||
.map_err(|e| e.to_string())?;
|
||
let (rgba, rw, rh) = adjust.read_camera_linear().map_err(|e| e.to_string())?;
|
||
// Camera RGB is unbalanced — green-heavy on every Bayer body — and
|
||
// linear. Luma weights would over-count green further; an equal mean is
|
||
// as good a grey as any for corners and edges, and the gamma is what
|
||
// gives the detector the contrast range it was trained on.
|
||
let data: Vec<f32> = rgba
|
||
.chunks_exact(4)
|
||
.map(|p| ((p[0] + p[1] + p[2]) / 3.0).clamp(0.0, 1.0).powf(1.0 / 2.2))
|
||
.collect();
|
||
let rgb: Vec<f32> = rgba
|
||
.chunks_exact(4)
|
||
.flat_map(|p| [p[0], p[1], p[2]])
|
||
.collect();
|
||
Ok((
|
||
Gray {
|
||
width: rw as usize,
|
||
height: rh as usize,
|
||
data,
|
||
},
|
||
rgb,
|
||
))
|
||
}
|
||
|
||
/// The aligned set on its surface, in colour, for the page.
|
||
///
|
||
/// A quick look, not the pipeline: the first frame's white balance and
|
||
/// matrix, a gamma, and the frames averaged where they overlap with their
|
||
/// gains applied. Ghosting here is the alignment's error and banding is
|
||
/// the gains', which is exactly what the photographer is being asked to
|
||
/// look at. Fitted to 1600 px across.
|
||
/// The aligned set on its surface, in colour, as planes: `(w, h, rgb 0..1,
|
||
/// known)`. Display-ish space — the first frame's white balance and matrix,
|
||
/// a gamma — which is also what the border filler was trained on.
|
||
fn preview_planes(
|
||
colour: &[Vec<f32>],
|
||
proxies: &[Gray],
|
||
alignment: &Alignment,
|
||
gains: &[f32],
|
||
projection: Projection,
|
||
first: &RawImage,
|
||
max_width: usize,
|
||
) -> (usize, usize, Vec<f32>, Vec<bool>) {
|
||
let cameras = alignment.cameras();
|
||
let (fw, fh) = (proxies[0].width as f64, proxies[0].height as f64);
|
||
let scale = alignment.focal;
|
||
let Some(bounds) = projection::bounds(projection, scale, &cameras, (fw, fh)) else {
|
||
return (0, 0, Vec::new(), Vec::new());
|
||
};
|
||
let out_w = max_width.min(bounds.width().ceil() as usize).max(1);
|
||
let px = bounds.width() / out_w as f64;
|
||
let out_h = ((bounds.height() / px).ceil() as usize).max(1);
|
||
let look = Look::of(first);
|
||
let mut rgb = vec![0.0f32; out_w * out_h * 3];
|
||
let mut known = vec![false; out_w * out_h];
|
||
for oy in 0..out_h {
|
||
for ox in 0..out_w {
|
||
let u = bounds.min_u + (ox as f64 + 0.5) * px;
|
||
let v = bounds.min_v + (oy as f64 + 0.5) * px;
|
||
let d = projection.to_direction(scale, u, v);
|
||
let mut sum = [0.0f32; 3];
|
||
let mut n = 0u32;
|
||
for (k, g) in proxies.iter().enumerate() {
|
||
let Some((x, y)) = cameras.project(k, d) else {
|
||
continue;
|
||
};
|
||
let (x, y) = (x + fw / 2.0, y + fh / 2.0);
|
||
if x < 0.0 || y < 0.0 || x >= fw - 1.0 || y >= fh - 1.0 {
|
||
continue;
|
||
}
|
||
let i = (y as usize * g.width + x as usize) * 3;
|
||
for c in 0..3 {
|
||
sum[c] += colour[k][i + c] * gains[k];
|
||
}
|
||
n += 1;
|
||
}
|
||
if n == 0 {
|
||
continue;
|
||
}
|
||
let o = oy * out_w + ox;
|
||
let d = look.to_display([sum[0] / n as f32, sum[1] / n as f32, sum[2] / n as f32]);
|
||
rgb[o * 3..o * 3 + 3].copy_from_slice(&d);
|
||
known[o] = true;
|
||
}
|
||
}
|
||
(out_w, out_h, rgb, known)
|
||
}
|
||
|
||
fn bounds_of(
|
||
proxies: &[Gray],
|
||
alignment: &Alignment,
|
||
projection: Projection,
|
||
) -> projection::Bounds {
|
||
let (fw, fh) = (proxies[0].width as f64, proxies[0].height as f64);
|
||
projection::bounds(projection, alignment.focal, &alignment.cameras(), (fw, fh)).unwrap_or(
|
||
projection::Bounds {
|
||
min_u: 0.0,
|
||
min_v: 0.0,
|
||
max_u: 1.0,
|
||
max_v: 1.0,
|
||
},
|
||
)
|
||
}
|
||
|
||
/// The planes to RGBA for the page, with each frame outlined.
|
||
///
|
||
/// The outlines, so the page shows *frames* and not one picture: a sweep
|
||
/// of frames overlapping by more than half reads as a single photograph
|
||
/// otherwise, which is what a photographer looking for a misplaced frame
|
||
/// needs to see through. Walked along each border and drawn where it
|
||
/// lands, two pixels wide, in a colour the scene does not have.
|
||
fn preview_rgba(
|
||
rgb: &[f32],
|
||
out_w: usize,
|
||
out_h: usize,
|
||
alignment: &Alignment,
|
||
projection: Projection,
|
||
bounds: &projection::Bounds,
|
||
proxies: &[Gray],
|
||
) -> (u32, u32, Vec<u8>) {
|
||
let cameras = alignment.cameras();
|
||
let (fw, fh) = (proxies[0].width as f64, proxies[0].height as f64);
|
||
let scale = alignment.focal;
|
||
let px = bounds.width() / out_w.max(1) as f64;
|
||
let mut out = vec![255u8; out_w * out_h * 4];
|
||
for i in 0..out_w * out_h {
|
||
for c in 0..3 {
|
||
out[i * 4 + c] = (rgb[i * 3 + c].clamp(0.0, 1.0) * 255.0) as u8;
|
||
}
|
||
}
|
||
let steps = 600;
|
||
for k in 0..proxies.len() {
|
||
for s in 0..steps {
|
||
let t = s as f64 / steps as f64;
|
||
for p in [
|
||
(-fw / 2.0 + fw * t, -fh / 2.0),
|
||
(-fw / 2.0 + fw * t, fh / 2.0 - 1.0),
|
||
(-fw / 2.0, -fh / 2.0 + fh * t),
|
||
(fw / 2.0 - 1.0, -fh / 2.0 + fh * t),
|
||
] {
|
||
let d = cameras.bearing(k, p);
|
||
let Some((u, v)) = projection.from_direction(scale, d) else {
|
||
continue;
|
||
};
|
||
let x = ((u - bounds.min_u) / px) as i64;
|
||
let y = ((v - bounds.min_v) / px) as i64;
|
||
for (dx, dy) in [(0, 0), (1, 0), (0, 1), (1, 1)] {
|
||
let (xx, yy) = (x + dx, y + dy);
|
||
if xx < 0 || yy < 0 || xx >= out_w as i64 || yy >= out_h as i64 {
|
||
continue;
|
||
}
|
||
let o = (yy as usize * out_w + xx as usize) * 4;
|
||
out[o] = 255;
|
||
out[o + 1] = 176;
|
||
out[o + 2] = 0;
|
||
out[o + 3] = 255;
|
||
}
|
||
}
|
||
}
|
||
}
|
||
(out_w as u32, out_h as u32, out)
|
||
}
|
||
|
||
/// The quick look from camera space to something like sRGB and back: the
|
||
/// first frame's white balance and matrix, a 2.2 gamma. Not the pipeline —
|
||
/// the page's preview and the filler's input, both of which want a
|
||
/// plausible picture rather than a calibrated one — and invertible, so the
|
||
/// filler's answer comes back to camera space where the DNG lives.
|
||
struct Look {
|
||
wb: [f32; 3],
|
||
m: [f32; 9],
|
||
inv: [f32; 9],
|
||
}
|
||
|
||
impl Look {
|
||
fn of(raw: &RawImage) -> Self {
|
||
let wb = [raw.wb_coeffs[0], raw.wb_coeffs[1], raw.wb_coeffs[2]];
|
||
let m = raw
|
||
.color_matrix
|
||
.unwrap_or([1.0, 0.0, 0.0, 0.0, 1.0, 0.0, 0.0, 0.0, 1.0]);
|
||
Look {
|
||
wb,
|
||
m,
|
||
inv: invert3(&m).unwrap_or([1.0, 0.0, 0.0, 0.0, 1.0, 0.0, 0.0, 0.0, 1.0]),
|
||
}
|
||
}
|
||
|
||
fn to_display(&self, cam: [f32; 3]) -> [f32; 3] {
|
||
let b = [
|
||
cam[0] * self.wb[0],
|
||
cam[1] * self.wb[1],
|
||
cam[2] * self.wb[2],
|
||
];
|
||
let row =
|
||
|c: usize| self.m[c * 3] * b[0] + self.m[c * 3 + 1] * b[1] + self.m[c * 3 + 2] * b[2];
|
||
let encode = |lin: f32| lin.clamp(0.0, 1.0).powf(1.0 / 2.2);
|
||
[encode(row(0)), encode(row(1)), encode(row(2))]
|
||
}
|
||
|
||
fn to_camera(&self, disp: [f32; 3]) -> [f32; 3] {
|
||
let lin = [
|
||
disp[0].max(0.0).powf(2.2),
|
||
disp[1].max(0.0).powf(2.2),
|
||
disp[2].max(0.0).powf(2.2),
|
||
];
|
||
let row = |c: usize| {
|
||
self.inv[c * 3] * lin[0] + self.inv[c * 3 + 1] * lin[1] + self.inv[c * 3 + 2] * lin[2]
|
||
};
|
||
let b = [row(0), row(1), row(2)];
|
||
[
|
||
(b[0] / self.wb[0].max(1e-4)).max(0.0),
|
||
(b[1] / self.wb[1].max(1e-4)).max(0.0),
|
||
(b[2] / self.wb[2].max(1e-4)).max(0.0),
|
||
]
|
||
}
|
||
}
|
||
|
||
fn invert3(m: &[f32; 9]) -> Option<[f32; 9]> {
|
||
let (a, b, c, d, e, f, g, h, i) = (m[0], m[1], m[2], m[3], m[4], m[5], m[6], m[7], m[8]);
|
||
let det = a * (e * i - f * h) - b * (d * i - f * g) + c * (d * h - e * g);
|
||
if det.abs() < 1e-12 {
|
||
return None;
|
||
}
|
||
let s = 1.0 / det;
|
||
Some([
|
||
(e * i - f * h) * s,
|
||
(c * h - b * i) * s,
|
||
(b * f - c * e) * s,
|
||
(f * g - d * i) * s,
|
||
(a * i - c * g) * s,
|
||
(c * d - a * f) * s,
|
||
(d * h - e * g) * s,
|
||
(b * g - a * h) * s,
|
||
(a * e - b * d) * s,
|
||
])
|
||
}
|
||
|
||
/// The horizontal and vertical field of view the cameras span, in radians:
|
||
/// the angle between the extreme frame centres plus one frame's own field.
|
||
fn field_of_view(cameras: &Cameras, frame: (f64, f64)) -> (f64, f64) {
|
||
let f = cameras.focal;
|
||
let own_h = 2.0 * (frame.0 / (2.0 * f)).atan();
|
||
let own_v = 2.0 * (frame.1 / (2.0 * f)).atan();
|
||
let (mut min_yaw, mut max_yaw, mut min_pitch, mut max_pitch) = (0.0f64, 0.0f64, 0.0f64, 0.0f64);
|
||
for r in &cameras.rotations {
|
||
let d = *r * dr_pano::linalg::Vec3::new(0.0, 0.0, 1.0);
|
||
let yaw = d.x().atan2(d.z());
|
||
let pitch = d.y().asin();
|
||
min_yaw = min_yaw.min(yaw);
|
||
max_yaw = max_yaw.max(yaw);
|
||
min_pitch = min_pitch.min(pitch);
|
||
max_pitch = max_pitch.max(pitch);
|
||
}
|
||
(max_yaw - min_yaw + own_h, max_pitch - min_pitch + own_v)
|
||
}
|
||
|
||
fn report_size(b: &projection::Bounds) -> (u32, u32) {
|
||
(
|
||
b.width().ceil().max(1.0) as u32,
|
||
b.height().ceil().max(1.0) as u32,
|
||
)
|
||
}
|
||
|
||
fn describe(a: &Alignment) -> Vec<Result<(f64, f64), String>> {
|
||
a.rotations
|
||
.iter()
|
||
.enumerate()
|
||
.map(|(k, r)| match r {
|
||
Some(r) => {
|
||
let yaw = r.0[0][2].atan2(r.0[2][2]).to_degrees();
|
||
let pitch = (-r.0[1][2]).asin().to_degrees();
|
||
Ok((yaw, pitch))
|
||
}
|
||
None => Err(a
|
||
.unaligned
|
||
.iter()
|
||
.find(|(i, _)| *i == k)
|
||
.map(|(_, why)| why.to_string())
|
||
.unwrap_or_else(|| "not aligned".into())),
|
||
})
|
||
.collect()
|
||
}
|
||
|
||
/// One gain per frame, from the proxies' overlaps.
|
||
///
|
||
/// For each link, the mean grey of each frame over the region both see is
|
||
/// compared; the log-gains that best reconcile every link are solved in
|
||
/// least squares with the reference frame held at 1. Grey here is the
|
||
/// proxy's gamma-encoded value raised back to linear, so the gain is a
|
||
/// linear multiplier as the shader applies it.
|
||
fn gains(proxies: &[Gray], a: &Alignment) -> Vec<f32> {
|
||
let n = proxies.len();
|
||
let cameras = a.cameras();
|
||
let (w, h) = (proxies[0].width as f64, proxies[0].height as f64);
|
||
let linear = |g: &Gray, x: usize, y: usize| g.data[y * g.width + x].powf(2.2) as f64;
|
||
|
||
// Ratios per link.
|
||
let mut ratios: Vec<(usize, usize, f64)> = Vec::new();
|
||
for l in &a.links {
|
||
let (mut sum_i, mut sum_j, mut count) = (0.0, 0.0, 0usize);
|
||
let step = 8;
|
||
for y in (0..proxies[l.i].height).step_by(step) {
|
||
for x in (0..proxies[l.i].width).step_by(step) {
|
||
let p = (x as f64 - w / 2.0, y as f64 - h / 2.0);
|
||
let d = cameras.bearing(l.i, p);
|
||
let Some((qx, qy)) = cameras.project(l.j, d) else {
|
||
continue;
|
||
};
|
||
let (qx, qy) = (qx + w / 2.0, qy + h / 2.0);
|
||
if qx < 0.0 || qy < 0.0 || qx >= w - 1.0 || qy >= h - 1.0 {
|
||
continue;
|
||
}
|
||
sum_i += linear(&proxies[l.i], x, y);
|
||
sum_j += linear(&proxies[l.j], qx as usize, qy as usize);
|
||
count += 1;
|
||
}
|
||
}
|
||
if count >= 50 && sum_i > 0.0 && sum_j > 0.0 {
|
||
ratios.push((l.i, l.j, (sum_i / sum_j).ln()));
|
||
}
|
||
}
|
||
if ratios.is_empty() {
|
||
return vec![1.0; n];
|
||
}
|
||
// Least squares on log gains: g_j - g_i = ln(mean_i / mean_j), with
|
||
// the reference frame's gain fixed at 0 by a strong prior.
|
||
let root = a
|
||
.rotations
|
||
.iter()
|
||
.position(|r| *r == Some(dr_pano::linalg::Mat3::IDENTITY))
|
||
.unwrap_or(0);
|
||
let mut ata = dr_pano::linalg::DMat::zeros(n);
|
||
let mut atb = vec![0.0f64; n];
|
||
for &(i, j, r) in &ratios {
|
||
// Row: +1 at j, −1 at i, rhs r.
|
||
ata[(j, j)] += 1.0;
|
||
ata[(i, i)] += 1.0;
|
||
ata[(i, j)] -= 1.0;
|
||
ata[(j, i)] -= 1.0;
|
||
atb[j] += r;
|
||
atb[i] -= r;
|
||
}
|
||
ata[(root, root)] += 1e6;
|
||
// A tiny ridge keeps an unlinked frame (there are none if the alignment
|
||
// is complete) from making the system singular.
|
||
for k in 0..n {
|
||
ata[(k, k)] += 1e-9;
|
||
}
|
||
match ata.solve_spd(&atb) {
|
||
Some(g) => g.iter().map(|v| v.exp() as f32).collect(),
|
||
None => vec![1.0; n],
|
||
}
|
||
}
|
||
|
||
fn dng_profile(first: &Loaded, white_level: u32) -> dr_export::DngProfile {
|
||
let wb = first.raw.wb_coeffs;
|
||
let neutral_from_wb = [
|
||
1.0 / wb[0].max(1e-3),
|
||
1.0 / wb[1].max(1e-3),
|
||
1.0 / wb[2].max(1e-3),
|
||
];
|
||
let (calibrations, as_shot_neutral) = match &first.raw.profile {
|
||
Some(p) => (p.dng_calibrations(), p.neutral().unwrap_or(neutral_from_wb)),
|
||
None => (Vec::new(), neutral_from_wb),
|
||
};
|
||
let unique_model = match (first.raw.make.trim(), first.raw.model.trim()) {
|
||
("", "") => "DarkRoom panorama".to_string(),
|
||
(make, model) if model.starts_with(make) => model.to_string(),
|
||
(make, model) => format!("{make} {model}"),
|
||
};
|
||
dr_export::DngProfile {
|
||
unique_model,
|
||
calibrations,
|
||
as_shot_neutral,
|
||
white_level,
|
||
}
|
||
}
|
||
|
||
fn stem(p: &Path) -> String {
|
||
p.file_stem()
|
||
.map(|s| s.to_string_lossy().into_owned())
|
||
.unwrap_or_else(|| p.display().to_string())
|
||
}
|
||
|
||
/// `name` in `dir`, numbered if that name is taken: a merge never
|
||
/// overwrites (FR-MRG-3).
|
||
fn unused_name(dir: &Path, name: &str) -> PathBuf {
|
||
let mut candidate = dir.join(name);
|
||
let base = stem(Path::new(name));
|
||
let mut n = 2;
|
||
while candidate.exists() {
|
||
candidate = dir.join(format!("{base}-{n}.dng"));
|
||
n += 1;
|
||
}
|
||
candidate
|
||
}
|
||
|
||
/// Drain everything a job has said so far.
|
||
/// Everything the worker has said since the last call, and whether it has
|
||
/// hung up — a closed channel after nothing [`MergeEvent::is_final`] is a
|
||
/// worker that died mid-job.
|
||
pub fn drain(rx: &Receiver<MergeEvent>) -> (Vec<MergeEvent>, bool) {
|
||
let mut out = Vec::new();
|
||
loop {
|
||
match rx.try_recv() {
|
||
Ok(e) => out.push(e),
|
||
Err(std::sync::mpsc::TryRecvError::Empty) => return (out, false),
|
||
Err(std::sync::mpsc::TryRecvError::Disconnected) => return (out, true),
|
||
}
|
||
}
|
||
}
|
||
|
||
/// The filler's input as the debugging example reads it: `input.ppm`, the
|
||
/// display-space picture, and `known.pgm`, 255 where a frame reached.
|
||
fn dump_fill_input(
|
||
dir: &Path,
|
||
rgb: &[f32],
|
||
known: &[bool],
|
||
w: usize,
|
||
h: usize,
|
||
) -> std::io::Result<()> {
|
||
std::fs::create_dir_all(dir)?;
|
||
let mut ppm = format!("P6\n{w} {h}\n255\n").into_bytes();
|
||
ppm.extend(
|
||
rgb.iter()
|
||
.map(|v| (v.clamp(0.0, 1.0) * 255.0).round() as u8),
|
||
);
|
||
std::fs::write(dir.join("input.ppm"), ppm)?;
|
||
let mut pgm = format!("P5\n{w} {h}\n255\n").into_bytes();
|
||
pgm.extend(known.iter().map(|&k| if k { 255u8 } else { 0 }));
|
||
std::fs::write(dir.join("known.pgm"), pgm)
|
||
}
|
||
|
||
#[cfg(test)]
|
||
mod tests {
|
||
use super::*;
|
||
|
||
#[test]
|
||
fn a_worker_that_hangs_up_mid_job_is_reported_as_gone() {
|
||
// The failure the page could not see: a panic drops the sender with
|
||
// no final event, and `try_recv`'s Disconnected used to be folded
|
||
// into "nothing new".
|
||
let (tx, rx) = std::sync::mpsc::channel();
|
||
tx.send(MergeEvent::Progress {
|
||
stage: "Reading",
|
||
done: 1,
|
||
total: 12,
|
||
})
|
||
.unwrap();
|
||
let (events, gone) = drain(&rx);
|
||
assert_eq!(events.len(), 1);
|
||
assert!(!gone, "the sender is still alive");
|
||
|
||
drop(tx);
|
||
let (events, gone) = drain(&rx);
|
||
assert!(events.is_empty());
|
||
assert!(gone, "and now it is not");
|
||
}
|
||
|
||
#[test]
|
||
fn a_job_that_said_its_last_word_is_not_a_dead_worker() {
|
||
let (tx, rx) = std::sync::mpsc::channel();
|
||
tx.send(MergeEvent::Failed("out of memory".into())).unwrap();
|
||
drop(tx);
|
||
let (events, gone) = drain(&rx);
|
||
assert!(gone);
|
||
assert!(events.iter().any(MergeEvent::is_final), "Failed is final");
|
||
assert!(MergeEvent::Cancelled.is_final());
|
||
assert!(!MergeEvent::Progress {
|
||
stage: "x",
|
||
done: 0,
|
||
total: 0
|
||
}
|
||
.is_final());
|
||
}
|
||
}
|