dtourolle
b1d1c47261
Thirty-nine spawn sites in dr-ui, and one in the Android entry point, called std::thread::spawn or a Builder of their own, and most of the threads they started were <unnamed> in a panic message or a profiler. Each now calls executors::spawn with its executor and a role, so the thread is named <executor>:<role> — net:sync, decode:thumbs, io:catalog-open — and knows which executor it is on. The three that already set a name (automation, import, prefetch) keep their name as the role. Behaviour is unchanged: each job still gets a thread of its own when it starts, and spawn panics where std::thread::spawn did. The module's documentation now says how a job is assigned: by what it spends its time on, so a sweep that fetches bytes and then decodes them is Decode, and a sidecar write that touches the catalog is Network. Left as they were: the segmentation and refine workers in masks_ui.rs, which another change is reworking, and test-only threads.
1395 lines
52 KiB
Rust
1395 lines
52 KiB
Rust
//! TRACES: FR-MRG-1 | FR-MRG-2 | FR-MRG-3 | FR-MRG-5 | FR-MRG-7
|
||
//! The merge job: from a set of files to a linear DNG beside them.
|
||
//!
|
||
//! The orchestration of a panorama, with no interface types in it: it runs
|
||
//! on a thread of its own and reports through a channel, on the pattern
|
||
//! [`crate::export`]'s batch established (FR-MRG-7). The interface drains
|
||
//! the channel from a timer; a headless example drains it from a loop.
|
||
//!
|
||
//! # Stages
|
||
//!
|
||
//! 1. **Load.** Each file is read and decoded to sensor data, and its edit
|
||
//! graph built the way a session would — orientation, lens profile —
|
||
//! because those are the two things the camera-space tap uses
|
||
//! (FR-MRG-2).
|
||
//! 2. **Proxies.** Each frame is demosaiced and rendered through the tap
|
||
//! at proxy size, upright; the keypoint detector reads that.
|
||
//! 3. **Alignment.** `dr_pano::align`. A frame it could not place stops
|
||
//! the job with the frame named (FR-MRG-5).
|
||
//! 4. **Gain.** One scalar per frame from the proxies' overlaps, so the
|
||
//! stop of exposure drift a hand-held sweep collects does not band.
|
||
//! 5. **Merge.** `dr_gpu::MergePass`, chunk by chunk, into a DNG written
|
||
//! strip by strip beside the first source (FR-MRG-3).
|
||
//!
|
||
//! The demosaiced frames are the memory: 160 MB each at f16 for a 20 MP
|
||
//! body, so at most two are resident and the rest are demosaiced again
|
||
//! when a band needs them. That is the trade FR-MRG-11 asks for — the
|
||
//! sensor data stays on the CPU at a fifth of the size — and it is what
|
||
//! makes a twelve-frame set fit.
|
||
|
||
use crate::executors::{self, Executor};
|
||
use std::collections::VecDeque;
|
||
use std::path::{Path, PathBuf};
|
||
use std::sync::mpsc::{Receiver, Sender};
|
||
use std::sync::Arc;
|
||
use std::time::Instant;
|
||
|
||
use dr_decode::RawImage;
|
||
use dr_gpu::{
|
||
AdjustPass, DemosaicedImage, Demosaicer, GpuContext, MergeFrame, MergeOutput, MergePass,
|
||
};
|
||
use dr_pano::bundle::Cameras;
|
||
use dr_pano::projection::{self, Projection};
|
||
use dr_pano::{Alignment, Gray};
|
||
use dr_pipeline::EditGraph;
|
||
|
||
pub use crate::export::Cancel;
|
||
|
||
/// The border filler's weights, where the app looks for them; for the
|
||
/// headless example, which has no library open.
|
||
pub fn inpaint_model_path() -> Option<PathBuf> {
|
||
crate::library::inpaint_model()
|
||
}
|
||
|
||
/// One frame as the job receives it: the file's bytes, already fetched,
|
||
/// and a name for messages and for the composite's own name.
|
||
#[derive(Debug, Clone)]
|
||
pub struct MergeInput {
|
||
/// The file's name, `_MG_8320.CR2`.
|
||
pub name: String,
|
||
pub bytes: Arc<Vec<u8>>,
|
||
}
|
||
|
||
impl MergeInput {
|
||
pub fn read(path: &Path) -> Result<Self, String> {
|
||
Ok(MergeInput {
|
||
name: path
|
||
.file_name()
|
||
.map(|n| n.to_string_lossy().into_owned())
|
||
.unwrap_or_else(|| "frame".into()),
|
||
bytes: Arc::new(std::fs::read(path).map_err(|e| format!("{}: {e}", path.display()))?),
|
||
})
|
||
}
|
||
}
|
||
|
||
/// Where the composite goes (FR-MRG-3).
|
||
#[derive(Debug, Clone)]
|
||
pub enum MergeDestination {
|
||
/// A folder on this device: written there directly.
|
||
Local(PathBuf),
|
||
/// Beside its sources in the library, through the export outbox: the
|
||
/// file is staged with a destination record and the drain uploads it —
|
||
/// the one path that works for a folder library and a server alike, and
|
||
/// the one FR-MRG-3 names for a folder the job cannot write itself.
|
||
Outbox {
|
||
outbox: PathBuf,
|
||
/// The sources' folder, relative to the library root.
|
||
remote_dir: String,
|
||
},
|
||
}
|
||
|
||
/// What the job is told.
|
||
#[derive(Debug, Clone)]
|
||
pub struct MergeRequest {
|
||
/// The frames, in capture order.
|
||
pub frames: Vec<MergeInput>,
|
||
pub destination: MergeDestination,
|
||
/// `None` for the projection the field of view suggests.
|
||
pub projection: Option<Projection>,
|
||
/// Pixels over which a frame's weight ramps up from its edge.
|
||
pub feather_px: f32,
|
||
/// GPU work unit; also the DNG strip height.
|
||
pub chunk: (u32, u32),
|
||
/// The border filler's weights, if the device has them (FR-MRG-4).
|
||
/// `None` and the fill is not offered.
|
||
pub inpaint_model: Option<PathBuf>,
|
||
/// TRACES: FR-RAW-2
|
||
/// What reads and decodes each frame.
|
||
pub decoder: &'static dyn dr_decode::Decoder,
|
||
}
|
||
|
||
impl MergeRequest {
|
||
pub fn new(frames: Vec<MergeInput>, destination: MergeDestination) -> Self {
|
||
MergeRequest {
|
||
frames,
|
||
destination,
|
||
projection: None,
|
||
feather_px: 200.0,
|
||
chunk: (2048, 512),
|
||
inpaint_model: None,
|
||
decoder: dr_decode::default(),
|
||
}
|
||
}
|
||
}
|
||
|
||
/// What the photographer decides once the alignment is shown (FR-MRG-1:
|
||
/// never automatic).
|
||
#[derive(Debug, Clone, Copy, PartialEq)]
|
||
pub enum Decision {
|
||
Merge {
|
||
projection: Option<Projection>,
|
||
/// Fill the border with invented pixels rather than crop to the
|
||
/// picture (FR-MRG-4), with these knobs. Never the default; the
|
||
/// page asks.
|
||
fill: Option<FillSettings>,
|
||
},
|
||
/// Draw the alignment again on another surface, and report it again.
|
||
/// The page's projection chips: what a choice looks like before it is
|
||
/// confirmed, at proxy cost rather than the merge's — and the fill, at
|
||
/// preview size, for the same reason: invented pixels are seen before
|
||
/// they are written.
|
||
Preview {
|
||
projection: Option<Projection>,
|
||
fill: Option<FillSettings>,
|
||
},
|
||
Abandon,
|
||
}
|
||
|
||
/// TRACES: FR-MRG-4
|
||
/// The border fill's knobs, every one on the page while the fill is
|
||
/// experimental: the fill's own (`dr_pano::FillParams`, in pixels of the
|
||
/// working image) and the two around it.
|
||
#[derive(Debug, Clone, Copy, PartialEq)]
|
||
pub struct FillSettings {
|
||
/// The working image is the composite at `1/scale`: 2 is half, the
|
||
/// desktop's default; 4 a quarter, for the tablet (FR-MRG-9).
|
||
pub scale: u32,
|
||
/// How many working pixels of the coverage edge count as fringe and
|
||
/// are regenerated too.
|
||
pub erosion: usize,
|
||
pub params: dr_pano::FillParams,
|
||
}
|
||
|
||
impl Default for FillSettings {
|
||
fn default() -> Self {
|
||
FillSettings {
|
||
scale: 2,
|
||
erosion: 4,
|
||
params: dr_pano::FillParams {
|
||
feather: 48,
|
||
..Default::default()
|
||
},
|
||
}
|
||
}
|
||
}
|
||
|
||
impl FillSettings {
|
||
/// The same knobs for a picture at `ratio` of the composite's size
|
||
/// (the page's preview): the working-pixel counts rescaled so the fill
|
||
/// looks the same there as it will in the merge.
|
||
pub fn at(&self, ratio: f64) -> FillSettings {
|
||
let k = ratio * self.scale as f64;
|
||
let px = |v: usize| ((v as f64) * k).ceil().max(1.0) as usize;
|
||
FillSettings {
|
||
scale: 1,
|
||
erosion: px(self.erosion),
|
||
params: dr_pano::FillParams {
|
||
coarse: self.params.coarse,
|
||
band: px(self.params.band).max(8),
|
||
mirror_depth: if self.params.mirror_depth == 0 {
|
||
0
|
||
} else {
|
||
px(self.params.mirror_depth)
|
||
},
|
||
feather: if self.params.feather == 0 {
|
||
0
|
||
} else {
|
||
px(self.params.feather)
|
||
},
|
||
stride: self.params.stride,
|
||
},
|
||
}
|
||
}
|
||
}
|
||
|
||
/// What the job says while it runs.
|
||
#[derive(Debug, Clone)]
|
||
pub enum MergeEvent {
|
||
/// A stage, with progress within it.
|
||
Progress {
|
||
stage: &'static str,
|
||
done: usize,
|
||
total: usize,
|
||
},
|
||
/// The alignment, before any pixel is written: what the interface shows
|
||
/// for the photographer to confirm, and what names a failure.
|
||
Aligned(AlignmentReport),
|
||
/// The composite is written: on the device at `path`, or staged in the
|
||
/// outbox for the drain to upload, in which case `staged` is true and
|
||
/// the library learns of it when the folder is next scanned.
|
||
Done {
|
||
path: PathBuf,
|
||
staged: bool,
|
||
width: u32,
|
||
height: u32,
|
||
},
|
||
Failed(String),
|
||
Cancelled,
|
||
}
|
||
|
||
impl MergeEvent {
|
||
/// Whether this is the job's last word: after one of these the worker
|
||
/// has nothing more to say, so its channel closing is expected.
|
||
pub fn is_final(&self) -> bool {
|
||
matches!(
|
||
self,
|
||
MergeEvent::Done { .. } | MergeEvent::Failed(_) | MergeEvent::Cancelled
|
||
)
|
||
}
|
||
}
|
||
|
||
/// The alignment, described for a panel.
|
||
#[derive(Debug, Clone)]
|
||
pub struct AlignmentReport {
|
||
/// Equivalent focal length in millimetres on full frame, from the fit.
|
||
pub focal_mm: f64,
|
||
pub rms_px: f64,
|
||
pub projection: Projection,
|
||
pub width: u32,
|
||
pub height: u32,
|
||
/// Per frame, in input order: yaw and pitch in degrees if aligned, or
|
||
/// why not.
|
||
pub frames: Vec<Result<(f64, f64), String>>,
|
||
pub links: usize,
|
||
/// The aligned set drawn on the suggested surface at proxy resolution,
|
||
/// `(width, height, rgba)` — what the photographer confirms.
|
||
pub preview: Option<(u32, u32, Vec<u8>)>,
|
||
/// Whether the preview's border is filled (FR-MRG-4).
|
||
pub filled: bool,
|
||
/// The filler: where it runs, or why it is not available.
|
||
pub filler: Result<String, String>,
|
||
}
|
||
|
||
/// Run the whole job on the calling thread, reporting on `events`.
|
||
///
|
||
/// After `Aligned` the job waits on `decision` — the photographer's
|
||
/// confirmation, or the headless caller's immediate `Merge` — and returns
|
||
/// when the file is written, the job failed, or `cancel` was seen.
|
||
pub fn run(
|
||
ctx: GpuContext,
|
||
request: MergeRequest,
|
||
events: Sender<MergeEvent>,
|
||
decision: Receiver<Decision>,
|
||
cancel: Cancel,
|
||
) {
|
||
let send = |e: MergeEvent| {
|
||
let _ = events.send(e);
|
||
};
|
||
// Caught rather than allowed to unwind the thread: wgpu reports a device
|
||
// that has run out of memory by panicking, and a twelve-frame merge on a
|
||
// GPU another process is using is exactly where that happens. Uncaught,
|
||
// the thread died, the sender went with it, and the page sat on "Stop"
|
||
// with every control disabled and nothing to say why — the crash record
|
||
// on disk was the only sign. The panic hook still writes that record;
|
||
// this is what puts the reason on the page.
|
||
let result = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| {
|
||
run_inner(&ctx, &request, &events, &decision, &cancel)
|
||
}));
|
||
match result {
|
||
Ok(Ok(Some(done))) => send(done),
|
||
Ok(Ok(None)) => send(MergeEvent::Cancelled),
|
||
Ok(Err(e)) => send(MergeEvent::Failed(e)),
|
||
Err(panic) => {
|
||
let detail = panic
|
||
.downcast_ref::<&str>()
|
||
.map(|s| (*s).to_string())
|
||
.or_else(|| panic.downcast_ref::<String>().cloned())
|
||
.unwrap_or_else(|| "panicked with a non-string payload".to_string());
|
||
log::error!("merge worker panicked: {detail}");
|
||
send(MergeEvent::Failed(format!("internal error: {detail}")));
|
||
}
|
||
}
|
||
}
|
||
|
||
/// The frames as loaded: sensor data on the CPU and the graph the tap uses.
|
||
struct Loaded {
|
||
raw: RawImage,
|
||
meta: dr_decode::Metadata,
|
||
graph: Arc<EditGraph>,
|
||
/// The upright size, which is what the cameras are in.
|
||
size: (u32, u32),
|
||
}
|
||
|
||
fn run_inner(
|
||
ctx: &GpuContext,
|
||
request: &MergeRequest,
|
||
events: &Sender<MergeEvent>,
|
||
decision: &Receiver<Decision>,
|
||
cancel: &Cancel,
|
||
) -> Result<Option<MergeEvent>, String> {
|
||
if request.frames.len() < 2 {
|
||
return Err("a panorama needs at least two frames".into());
|
||
}
|
||
let progress = |stage: &'static str, done: usize, total: usize| {
|
||
let _ = events.send(MergeEvent::Progress { stage, done, total });
|
||
};
|
||
|
||
// 1. Load.
|
||
let t = Instant::now();
|
||
let mut frames: Vec<Loaded> = Vec::with_capacity(request.frames.len());
|
||
for (i, input) in request.frames.iter().enumerate() {
|
||
if cancel.is_cancelled() {
|
||
return Ok(None);
|
||
}
|
||
progress("Reading", i, request.frames.len());
|
||
let bytes = &input.bytes;
|
||
let meta = request.decoder.metadata(bytes).unwrap_or_default();
|
||
let raw = request
|
||
.decoder
|
||
.decode(bytes)
|
||
.map_err(|e| format!("{}: {e}", input.name))?;
|
||
let orientation = meta.orientation.unwrap_or_default();
|
||
let mut graph = EditGraph::default_chain();
|
||
graph.set_orientation(orientation);
|
||
graph.set_lens_profile(crate::develop::DevelopSession::profile_for(&meta));
|
||
let (w, h) = (raw.crop.width, raw.crop.height);
|
||
let size = if orientation.quarter_turns % 2 == 1 {
|
||
(h, w)
|
||
} else {
|
||
(w, h)
|
||
};
|
||
frames.push(Loaded {
|
||
raw,
|
||
meta,
|
||
graph: Arc::new(graph),
|
||
size,
|
||
});
|
||
}
|
||
log::info!("merge: {} frames read in {:?}", frames.len(), t.elapsed());
|
||
|
||
// 2. Proxies and keypoints.
|
||
let t = Instant::now();
|
||
let demosaicer = Demosaicer::new(ctx).map_err(|e| e.to_string())?;
|
||
let mut adjust = AdjustPass::new(ctx);
|
||
let mut detector = dr_pano::xfeat::XFeat::embedded().map_err(|e| e.to_string())?;
|
||
let mut features = Vec::with_capacity(frames.len());
|
||
let mut proxies: Vec<Gray> = Vec::with_capacity(frames.len());
|
||
let mut colour: Vec<Vec<f32>> = Vec::with_capacity(frames.len());
|
||
for (i, f) in frames.iter().enumerate() {
|
||
if cancel.is_cancelled() {
|
||
return Ok(None);
|
||
}
|
||
progress("Finding features", i, frames.len());
|
||
let image = demosaicer.run(&f.raw).map_err(|e| e.to_string())?;
|
||
let (proxy, rgb) = camera_proxy(&mut adjust, &image, &f.graph, f.size)?;
|
||
features.push(detector.detect(&proxy).map_err(|e| e.to_string())?);
|
||
proxies.push(proxy);
|
||
colour.push(rgb);
|
||
}
|
||
log::info!("merge: features in {:?}", t.elapsed());
|
||
|
||
// 3. Alignment.
|
||
let t = Instant::now();
|
||
progress("Aligning", 0, 1);
|
||
let alignment =
|
||
dr_pano::align(&features, &dr_pano::AlignOptions::default()).map_err(|e| e.to_string())?;
|
||
log::info!(
|
||
"merge: aligned in {:?}, focal {:.1} px, rms {:.2} px, {} links",
|
||
t.elapsed(),
|
||
alignment.focal,
|
||
alignment.rms_px,
|
||
alignment.links.len()
|
||
);
|
||
|
||
// The geometry at full resolution: the proxy's long edge against the
|
||
// frame's.
|
||
let proxy_long = proxies[0].width.max(proxies[0].height) as f64;
|
||
let full_long = frames[0].size.0.max(frames[0].size.1) as f64;
|
||
let focal_full = alignment.focal * full_long / proxy_long;
|
||
let cameras = Cameras {
|
||
rotations: alignment
|
||
.rotations
|
||
.iter()
|
||
.map(|r| r.unwrap_or(dr_pano::linalg::Mat3::IDENTITY))
|
||
.collect(),
|
||
focal: focal_full,
|
||
};
|
||
let frame_size = (frames[0].size.0 as f64, frames[0].size.1 as f64);
|
||
let (hfov, vfov) = field_of_view(&cameras, frame_size);
|
||
let projection = request
|
||
.projection
|
||
.unwrap_or_else(|| Projection::suggest(hfov, vfov));
|
||
let suggested = projection;
|
||
|
||
// 4. Gains, before the report so the preview shows them.
|
||
let gains = if alignment.is_complete() {
|
||
gains(&proxies, &alignment)
|
||
} else {
|
||
vec![1.0; frames.len()]
|
||
};
|
||
// The filler, loaded once if the device has it. Its absence is a
|
||
// report, not a failure: the crop is always there.
|
||
let mut filler: Option<dr_pano::migan::MiGan> = None;
|
||
let filler_status: Result<String, String> = match request.inpaint_model.as_deref() {
|
||
None => Err("no border-fill model on this device".into()),
|
||
Some(path) => match dr_pano::migan::MiGan::from_path(path) {
|
||
Ok(m) => {
|
||
let rung = m.rung().map(|r| r.label().to_string()).unwrap_or_default();
|
||
filler = Some(m);
|
||
Ok(rung)
|
||
}
|
||
Err(e) => Err(format!("the border-fill model could not be loaded: {e}")),
|
||
},
|
||
};
|
||
let first_raw = &frames[0].raw;
|
||
let mut report =
|
||
|projection: Projection, fill: Option<FillSettings>| -> Result<AlignmentReport, String> {
|
||
let bounds = projection::bounds(projection, focal_full, &cameras, frame_size)
|
||
.ok_or("the frames project nowhere")?;
|
||
let mut filled = false;
|
||
let preview = match alignment.is_complete() {
|
||
false => None,
|
||
true => {
|
||
let (w, h, mut rgb, mut known) = preview_planes(
|
||
&colour, &proxies, &alignment, &gains, projection, first_raw, 1600,
|
||
);
|
||
if let Some(settings) = fill {
|
||
if let Some(model) = filler.as_mut() {
|
||
progress("Filling the preview", 0, 1);
|
||
// The knobs at the preview's scale.
|
||
let here = settings.at(w as f64 / bounds.width());
|
||
dr_pano::fill::erode(&mut known, w, h, here.erosion);
|
||
match dr_pano::fill_border(
|
||
&mut rgb,
|
||
w,
|
||
h,
|
||
&known,
|
||
model,
|
||
here.params,
|
||
&mut |_, _| {},
|
||
) {
|
||
Ok(_) => filled = true,
|
||
Err(e) => log::warn!("preview fill: {e}"),
|
||
}
|
||
}
|
||
}
|
||
Some(preview_rgba(
|
||
&rgb,
|
||
w,
|
||
h,
|
||
&alignment,
|
||
projection,
|
||
&bounds_of(&proxies, &alignment, projection),
|
||
&proxies,
|
||
))
|
||
}
|
||
};
|
||
Ok(AlignmentReport {
|
||
focal_mm: focal_full * 36.0 / full_long,
|
||
rms_px: alignment.rms_px,
|
||
projection,
|
||
width: bounds.width().ceil() as u32,
|
||
height: bounds.height().ceil() as u32,
|
||
frames: describe(&alignment),
|
||
links: alignment.links.len(),
|
||
preview,
|
||
filled,
|
||
filler: filler_status.clone(),
|
||
})
|
||
};
|
||
progress("Drawing the preview", 0, 1);
|
||
let _ = events.send(MergeEvent::Aligned(report(suggested, None)?));
|
||
if !alignment.is_complete() {
|
||
let names: Vec<String> = alignment
|
||
.unaligned
|
||
.iter()
|
||
.map(|(k, why)| format!("{}: {why}", request.frames[*k].name))
|
||
.collect();
|
||
return Err(format!(
|
||
"not every frame could be placed — {}",
|
||
names.join("; ")
|
||
));
|
||
}
|
||
|
||
// Never automatic (FR-MRG-1): nothing is written until the alignment
|
||
// has been seen and confirmed. Polled, so a cancel while waiting is
|
||
// seen within a moment.
|
||
let (projection, fill) = loop {
|
||
if cancel.is_cancelled() {
|
||
return Ok(None);
|
||
}
|
||
match decision.recv_timeout(std::time::Duration::from_millis(100)) {
|
||
Ok(Decision::Merge {
|
||
projection: p,
|
||
fill,
|
||
}) => break (p.unwrap_or(suggested), fill.filter(|_| filler.is_some())),
|
||
Ok(Decision::Preview {
|
||
projection: p,
|
||
fill,
|
||
}) => {
|
||
let _ = events.send(MergeEvent::Aligned(report(p.unwrap_or(suggested), fill)?));
|
||
}
|
||
Ok(Decision::Abandon) => return Ok(None),
|
||
Err(std::sync::mpsc::RecvTimeoutError::Timeout) => continue,
|
||
Err(std::sync::mpsc::RecvTimeoutError::Disconnected) => return Ok(None),
|
||
}
|
||
};
|
||
let bounds = projection::bounds(projection, focal_full, &cameras, frame_size)
|
||
.ok_or("the frames project nowhere")?;
|
||
|
||
// 5. Merge, into a DNG beside the first frame.
|
||
let name = format!("{}-pano.dng", stem(Path::new(&request.frames[0].name)));
|
||
let (out_path, staged) = match &request.destination {
|
||
MergeDestination::Local(dir) => (unused_name(dir, &name), false),
|
||
MergeDestination::Outbox { outbox, remote_dir } => {
|
||
std::fs::create_dir_all(outbox).map_err(|e| format!("{}: {e}", outbox.display()))?;
|
||
let path = unused_name(outbox, &name);
|
||
// The record first here, unlike an export: the payload is
|
||
// written over minutes and a record naming a half-written file
|
||
// is worse than a payload with no record, so the record is
|
||
// removed again if the merge fails. The drain skips payloads
|
||
// without records.
|
||
let record = crate::export::destination_record(&path);
|
||
let final_name = path
|
||
.file_name()
|
||
.map(|n| n.to_string_lossy().into_owned())
|
||
.unwrap_or(name.clone());
|
||
std::fs::write(&record, format!("{remote_dir}\n{final_name}\n"))
|
||
.map_err(|e| format!("{}: {e}", record.display()))?;
|
||
(path, true)
|
||
}
|
||
};
|
||
let cleanup = |path: &Path| {
|
||
let _ = std::fs::remove_file(path);
|
||
if staged {
|
||
let _ = std::fs::remove_file(crate::export::destination_record(path));
|
||
}
|
||
};
|
||
let (out_w, out_h) = (report_size(&bounds).0, report_size(&bounds).1);
|
||
let first = &frames[0];
|
||
let black = first.raw.black_level[0];
|
||
let white_level = u32::from(first.raw.white_level.saturating_sub(black)).max(1);
|
||
let profile = dng_profile(first, white_level);
|
||
let (_, mut carried) = crate::export::header_for_file(&first.meta);
|
||
// The composite is dated at the middle of its sweep — the mean of the
|
||
// frames' capture times — so it sorts among the frames it was made
|
||
// from, not at the first of them and not at the moment of the merge.
|
||
// The zone is the first frame's; a sweep does not cross one.
|
||
let stamps: Vec<i64> = frames.iter().filter_map(|f| f.meta.captured_at).collect();
|
||
if !stamps.is_empty() {
|
||
let sum: i128 = stamps.iter().map(|&t| t as i128).sum();
|
||
carried.captured_at = Some((sum / stamps.len() as i128) as i64);
|
||
}
|
||
|
||
let output = MergeOutput {
|
||
projection,
|
||
scale: focal_full,
|
||
bounds,
|
||
feather: request.feather_px,
|
||
chunk: request.chunk,
|
||
sample_scale: white_level as f32,
|
||
};
|
||
let merge_frames: Vec<MergeFrame> = frames
|
||
.iter()
|
||
.zip(&gains)
|
||
.map(|(f, &gain)| MergeFrame {
|
||
graph: f.graph.clone(),
|
||
gain,
|
||
})
|
||
.collect();
|
||
|
||
let t = Instant::now();
|
||
let mut pass = MergePass::new(ctx).map_err(|e| e.to_string())?;
|
||
let mut resident: VecDeque<(usize, Arc<DemosaicedImage>)> = VecDeque::new();
|
||
|
||
// 5a. The border fill (FR-MRG-4), when asked for: the composite at a
|
||
// quarter of its resolution, its border filled by the model in display
|
||
// space, and the result taken back to camera space for the full merge
|
||
// to sample wherever nothing was covered. A quarter because the fill is
|
||
// sky and slope at the edge of a picture, where a quarter is more than
|
||
// the eye resolves, and because it puts the fixture's border at thirty
|
||
// tiles rather than a hundred.
|
||
let fill_cam: Option<(usize, usize, Vec<f32>, Vec<bool>)> = if let Some(settings) = fill {
|
||
let model = filler.as_mut().expect("fill was only chosen with a filler");
|
||
progress("Filling the border", 0, 1);
|
||
// Half the composite's resolution by default: a quarter left a
|
||
// visible gap in sharpness against the real pixels beside the
|
||
// fill. Memory is the working image in f32, 400 MB for a 137 MP
|
||
// composite; the tablet will want a quarter (FR-MRG-9).
|
||
let q = settings.scale.max(1) as f64;
|
||
let qout = MergeOutput {
|
||
projection,
|
||
scale: focal_full / q,
|
||
bounds: projection::Bounds {
|
||
min_u: bounds.min_u / q,
|
||
min_v: bounds.min_v / q,
|
||
max_u: bounds.max_u / q,
|
||
max_v: bounds.max_v / q,
|
||
},
|
||
feather: request.feather_px,
|
||
chunk: request.chunk,
|
||
sample_scale: white_level as f32,
|
||
};
|
||
let (qw, qh) = (qout.width() as usize, qout.height() as usize);
|
||
let mut qrgb = vec![0.0f32; qw * qh * 3];
|
||
let mut qknown = vec![false; qw * qh];
|
||
pass.merge(
|
||
&mut adjust,
|
||
&merge_frames,
|
||
&cameras,
|
||
(frames[0].size.0, frames[0].size.1),
|
||
&qout,
|
||
|k| {
|
||
if let Some((_, img)) = resident.iter().find(|(i, _)| *i == k) {
|
||
return Ok(img.clone());
|
||
}
|
||
let img = Arc::new(demosaicer.run(&frames[k].raw)?);
|
||
resident.push_back((k, img.clone()));
|
||
while resident.len() > 2 {
|
||
resident.pop_front();
|
||
}
|
||
Ok(img)
|
||
},
|
||
|band| {
|
||
let first = band.first_row as usize;
|
||
for r in 0..band.rows as usize {
|
||
for x in 0..qw {
|
||
let src = r * qw + x;
|
||
let dst = (first + r) * qw + x;
|
||
for c in 0..3 {
|
||
qrgb[dst * 3 + c] =
|
||
f32::from(band.rgb[src * 3 + c]) / white_level as f32;
|
||
}
|
||
qknown[dst] = band.covered[src];
|
||
}
|
||
}
|
||
Ok(())
|
||
},
|
||
|| cancel.is_cancelled(),
|
||
)
|
||
.map_err(|e| e.to_string())?;
|
||
if cancel.is_cancelled() {
|
||
cleanup(&out_path);
|
||
return Ok(None);
|
||
}
|
||
// Camera space to the display-ish space the model was trained on.
|
||
let look = Look::of(first_raw);
|
||
let mut disp = qrgb.clone();
|
||
for p in disp.chunks_exact_mut(3) {
|
||
let d = look.to_display([p[0], p[1], p[2]]);
|
||
p.copy_from_slice(&d);
|
||
}
|
||
dr_pano::fill::erode(&mut qknown, qw, qh, settings.erosion);
|
||
// `DR_FILL_DUMP=dir` writes what the filler was given, so a fill
|
||
// can be re-run and looked at stage by stage without the merge
|
||
// (`cargo run -p dr-pano --example fill`).
|
||
if let Some(dir) = std::env::var_os("DR_FILL_DUMP") {
|
||
let dir = PathBuf::from(dir);
|
||
if let Err(e) = dump_fill_input(&dir, &disp, &qknown, qw, qh) {
|
||
log::warn!("fill dump: {e}");
|
||
}
|
||
}
|
||
let total_tiles = std::cell::Cell::new(0usize);
|
||
let feather = settings.params.feather;
|
||
dr_pano::fill_border(
|
||
&mut disp,
|
||
qw,
|
||
qh,
|
||
&qknown,
|
||
model,
|
||
settings.params,
|
||
&mut |n, total| {
|
||
total_tiles.set(total);
|
||
progress("Filling the border", n, total.max(1));
|
||
},
|
||
)
|
||
.map_err(|e| e.to_string())?;
|
||
// And back: the filled pixels — the hole and the feathered margin —
|
||
// to camera space; the rest as the merge produced them.
|
||
let mut margin = qknown.clone();
|
||
dr_pano::fill::erode(&mut margin, qw, qh, feather);
|
||
for (i, p) in qrgb.chunks_exact_mut(3).enumerate() {
|
||
if !margin[i] {
|
||
let d = &disp[i * 3..i * 3 + 3];
|
||
let cam = look.to_camera([d[0], d[1], d[2]]);
|
||
p.copy_from_slice(&cam);
|
||
}
|
||
}
|
||
let fill_mask: Vec<bool> = margin;
|
||
log::info!(
|
||
"merge: border filled at {qw}×{qh}, {} tiles, in {:?}",
|
||
total_tiles.get(),
|
||
t.elapsed()
|
||
);
|
||
Some((qw, qh, qrgb, fill_mask))
|
||
} else {
|
||
None
|
||
};
|
||
|
||
// The writer pulls strips; the merge pushes bands. A channel between
|
||
// them, and the writer on its own thread, so neither waits on the
|
||
// other's pace more than one band.
|
||
let (band_tx, band_rx) = std::sync::mpsc::sync_channel::<Vec<u16>>(1);
|
||
let rows_per_strip = request.chunk.1.max(1);
|
||
// The crop (FR-MRG-4), found as the bands go by; the writer reads it
|
||
// once the last strip is in, by which time every band has been seen.
|
||
let inscribed = Arc::new(std::sync::Mutex::new(dr_export::Inscribed::new(out_w)));
|
||
let inscribed_for_writer = inscribed.clone();
|
||
let fill_for_writer = fill_cam.is_some();
|
||
let file =
|
||
std::fs::File::create(&out_path).map_err(|e| format!("{}: {e}", out_path.display()))?;
|
||
let writer = executors::spawn(Executor::Io, "dng-write", move || -> Result<(), String> {
|
||
let mut file = std::io::BufWriter::new(file);
|
||
dr_export::write_linear_dng(
|
||
&mut file,
|
||
out_w,
|
||
out_h,
|
||
rows_per_strip,
|
||
&profile,
|
||
Some(&carried),
|
||
|_, buf| {
|
||
let band = band_rx.recv().map_err(|_| {
|
||
dr_export::ExportError::Encode("the merge stopped early".into())
|
||
})?;
|
||
buf.extend_from_slice(&band);
|
||
Ok(())
|
||
},
|
||
// A filled composite is the whole picture; a cropped one opens
|
||
// on the rectangle the frames covered.
|
||
move || {
|
||
if fill_for_writer {
|
||
None
|
||
} else {
|
||
inscribed_for_writer.lock().ok().map(|i| i.best())
|
||
}
|
||
},
|
||
)
|
||
.map_err(|e| e.to_string())
|
||
});
|
||
|
||
let total_bands = out_h.div_ceil(rows_per_strip) as usize;
|
||
let mut bands_done = 0usize;
|
||
progress("Merging", 0, total_bands);
|
||
let merged = pass.merge(
|
||
&mut adjust,
|
||
&merge_frames,
|
||
&cameras,
|
||
(frames[0].size.0, frames[0].size.1),
|
||
&output,
|
||
|k| {
|
||
if let Some((_, img)) = resident.iter().find(|(i, _)| *i == k) {
|
||
return Ok(img.clone());
|
||
}
|
||
let img = Arc::new(demosaicer.run(&frames[k].raw)?);
|
||
resident.push_back((k, img.clone()));
|
||
while resident.len() > 2 {
|
||
resident.pop_front();
|
||
}
|
||
Ok(img)
|
||
},
|
||
|band| {
|
||
bands_done += 1;
|
||
progress("Merging", bands_done, total_bands);
|
||
if let Ok(mut i) = inscribed.lock() {
|
||
i.push_rows(band.covered, band.rows);
|
||
}
|
||
let mut rgb = band.rgb.to_vec();
|
||
// Where no frame reached, the fill — sampled bilinearly from
|
||
// the quarter-resolution result, at the sensor's scale.
|
||
if let Some((qw, qh, qrgb, fill_mask)) = &fill_cam {
|
||
let sx = *qw as f64 / out_w as f64;
|
||
let sy = *qh as f64 / out_h as f64;
|
||
for r in 0..band.rows as usize {
|
||
for x in 0..out_w as usize {
|
||
let i = r * out_w as usize + x;
|
||
let fx = ((x as f64 + 0.5) * sx - 0.5).clamp(0.0, (*qw - 1) as f64);
|
||
let fy = (((band.first_row as usize + r) as f64 + 0.5) * sy - 0.5)
|
||
.clamp(0.0, (*qh - 1) as f64);
|
||
// Uncovered, or inside the feathered margin: the fill.
|
||
let (mx, my) = (fx.round() as usize, fy.round() as usize);
|
||
if band.covered[i] && fill_mask[my.min(qh - 1) * qw + mx.min(qw - 1)] {
|
||
continue;
|
||
}
|
||
let (x0, y0) = (fx as usize, fy as usize);
|
||
let (x1, y1) = ((x0 + 1).min(qw - 1), (y0 + 1).min(qh - 1));
|
||
let (tx, ty) = ((fx - x0 as f64) as f32, (fy - y0 as f64) as f32);
|
||
for c in 0..3 {
|
||
let at = |xx: usize, yy: usize| qrgb[(yy * qw + xx) * 3 + c];
|
||
let v = (at(x0, y0) * (1.0 - tx) + at(x1, y0) * tx) * (1.0 - ty)
|
||
+ (at(x0, y1) * (1.0 - tx) + at(x1, y1) * tx) * ty;
|
||
rgb[i * 3 + c] =
|
||
(v * white_level as f32).round().clamp(0.0, 65535.0) as u16;
|
||
}
|
||
}
|
||
}
|
||
}
|
||
band_tx
|
||
.send(rgb)
|
||
.map_err(|_| dr_gpu::GpuError::Readback("the writer stopped".into()))
|
||
},
|
||
|| cancel.is_cancelled(),
|
||
);
|
||
drop(band_tx);
|
||
let written = writer
|
||
.join()
|
||
.unwrap_or_else(|_| Err("the writer panicked".into()));
|
||
match merged {
|
||
Ok(()) => {}
|
||
Err(e) if cancel.is_cancelled() => {
|
||
cleanup(&out_path);
|
||
log::info!("merge cancelled: {e}");
|
||
return Ok(None);
|
||
}
|
||
Err(e) => {
|
||
cleanup(&out_path);
|
||
return Err(e.to_string());
|
||
}
|
||
}
|
||
if let Err(e) = written {
|
||
cleanup(&out_path);
|
||
return Err(e);
|
||
}
|
||
log::info!(
|
||
"merge: {}×{} written to {} in {:?}",
|
||
out_w,
|
||
out_h,
|
||
out_path.display(),
|
||
t.elapsed()
|
||
);
|
||
|
||
// TRACES: FR-MRG-6
|
||
// Provenance, beside the composite: what it was merged from, in order,
|
||
// and how. Its own file and its own outbox record, so the drain places
|
||
// it as it places the composite; a sidecar that failed to write is
|
||
// reported but does not un-write the composite.
|
||
let mut sidecar = dr_pipeline::Sidecar::new();
|
||
sidecar.derived_from = request.frames.iter().map(|f| f.name.clone()).collect();
|
||
sidecar.merge = Some(format!(
|
||
"panorama {} {:.1}mm {} frames{}",
|
||
match projection {
|
||
Projection::Perspective => "perspective",
|
||
Projection::Cylindrical => "cylindrical",
|
||
Projection::Spherical => "spherical",
|
||
},
|
||
focal_full * 36.0 / full_long,
|
||
frames.len(),
|
||
// Invented pixels declare themselves (FR-MRG-6, D17's rule), with
|
||
// the knobs that made them while the fill is experimental.
|
||
match fill {
|
||
Some(f) => format!(
|
||
" border filled (1/{} scale, erosion {}, coarse {}, band {}, mirror {}, feather {})",
|
||
f.scale, f.erosion, f.params.coarse, f.params.band, f.params.mirror_depth, f.params.feather
|
||
),
|
||
None => String::new(),
|
||
}
|
||
));
|
||
let sidecar_path = out_path.with_extension(dr_pipeline::sidecar::EXTENSION);
|
||
if let Err(e) = std::fs::write(&sidecar_path, sidecar.to_text()) {
|
||
log::warn!("merge: the sidecar could not be written: {e}");
|
||
} else if let MergeDestination::Outbox { remote_dir, .. } = &request.destination {
|
||
let record = crate::export::destination_record(&sidecar_path);
|
||
let name = sidecar_path
|
||
.file_name()
|
||
.map(|n| n.to_string_lossy().into_owned())
|
||
.unwrap_or_default();
|
||
if let Err(e) = std::fs::write(&record, format!("{remote_dir}\n{name}\n")) {
|
||
log::warn!("merge: the sidecar's record could not be written: {e}");
|
||
}
|
||
}
|
||
|
||
Ok(Some(MergeEvent::Done {
|
||
path: out_path,
|
||
staged,
|
||
width: out_w,
|
||
height: out_h,
|
||
}))
|
||
}
|
||
|
||
/// The proxy the detector reads: the frame through the camera-space tap at
|
||
/// proxy size, upright, as gamma-encoded grey.
|
||
///
|
||
/// Through the tap rather than the embedded preview so that the keypoints
|
||
/// are in the *undistorted* frame the tiles will be rendered in — a lens
|
||
/// profile moves the corners by tens of pixels at full resolution, and an
|
||
/// alignment measured on the distorted preview would be wrong by that much
|
||
/// at the seams.
|
||
fn camera_proxy(
|
||
adjust: &mut AdjustPass,
|
||
image: &DemosaicedImage,
|
||
graph: &EditGraph,
|
||
upright: (u32, u32),
|
||
) -> Result<(Gray, Vec<f32>), String> {
|
||
let long = dr_pano::xfeat::INPUT_LONG_EDGE as f64;
|
||
let scale = (long / f64::from(upright.0.max(upright.1))).min(1.0);
|
||
let w = ((f64::from(upright.0) * scale).round() as u32).max(1);
|
||
let h = ((f64::from(upright.1) * scale).round() as u32).max(1);
|
||
let shader = graph.compose_camera_linear(dr_pipeline::CropRect::default());
|
||
adjust
|
||
.render_camera_linear(image, &shader, w, h)
|
||
.map_err(|e| e.to_string())?;
|
||
let (rgba, rw, rh) = adjust.read_camera_linear().map_err(|e| e.to_string())?;
|
||
// Camera RGB is unbalanced — green-heavy on every Bayer body — and
|
||
// linear. Luma weights would over-count green further; an equal mean is
|
||
// as good a grey as any for corners and edges, and the gamma is what
|
||
// gives the detector the contrast range it was trained on.
|
||
let data: Vec<f32> = rgba
|
||
.chunks_exact(4)
|
||
.map(|p| ((p[0] + p[1] + p[2]) / 3.0).clamp(0.0, 1.0).powf(1.0 / 2.2))
|
||
.collect();
|
||
let rgb: Vec<f32> = rgba
|
||
.chunks_exact(4)
|
||
.flat_map(|p| [p[0], p[1], p[2]])
|
||
.collect();
|
||
Ok((
|
||
Gray {
|
||
width: rw as usize,
|
||
height: rh as usize,
|
||
data,
|
||
},
|
||
rgb,
|
||
))
|
||
}
|
||
|
||
/// The aligned set on its surface, in colour, for the page.
|
||
///
|
||
/// A quick look, not the pipeline: the first frame's white balance and
|
||
/// matrix, a gamma, and the frames averaged where they overlap with their
|
||
/// gains applied. Ghosting here is the alignment's error and banding is
|
||
/// the gains', which is exactly what the photographer is being asked to
|
||
/// look at. Fitted to 1600 px across.
|
||
/// The aligned set on its surface, in colour, as planes: `(w, h, rgb 0..1,
|
||
/// known)`. Display-ish space — the first frame's white balance and matrix,
|
||
/// a gamma — which is also what the border filler was trained on.
|
||
fn preview_planes(
|
||
colour: &[Vec<f32>],
|
||
proxies: &[Gray],
|
||
alignment: &Alignment,
|
||
gains: &[f32],
|
||
projection: Projection,
|
||
first: &RawImage,
|
||
max_width: usize,
|
||
) -> (usize, usize, Vec<f32>, Vec<bool>) {
|
||
let cameras = alignment.cameras();
|
||
let (fw, fh) = (proxies[0].width as f64, proxies[0].height as f64);
|
||
let scale = alignment.focal;
|
||
let Some(bounds) = projection::bounds(projection, scale, &cameras, (fw, fh)) else {
|
||
return (0, 0, Vec::new(), Vec::new());
|
||
};
|
||
let out_w = max_width.min(bounds.width().ceil() as usize).max(1);
|
||
let px = bounds.width() / out_w as f64;
|
||
let out_h = ((bounds.height() / px).ceil() as usize).max(1);
|
||
let look = Look::of(first);
|
||
let mut rgb = vec![0.0f32; out_w * out_h * 3];
|
||
let mut known = vec![false; out_w * out_h];
|
||
for oy in 0..out_h {
|
||
for ox in 0..out_w {
|
||
let u = bounds.min_u + (ox as f64 + 0.5) * px;
|
||
let v = bounds.min_v + (oy as f64 + 0.5) * px;
|
||
let d = projection.to_direction(scale, u, v);
|
||
let mut sum = [0.0f32; 3];
|
||
let mut n = 0u32;
|
||
for (k, g) in proxies.iter().enumerate() {
|
||
let Some((x, y)) = cameras.project(k, d) else {
|
||
continue;
|
||
};
|
||
let (x, y) = (x + fw / 2.0, y + fh / 2.0);
|
||
if x < 0.0 || y < 0.0 || x >= fw - 1.0 || y >= fh - 1.0 {
|
||
continue;
|
||
}
|
||
let i = (y as usize * g.width + x as usize) * 3;
|
||
for c in 0..3 {
|
||
sum[c] += colour[k][i + c] * gains[k];
|
||
}
|
||
n += 1;
|
||
}
|
||
if n == 0 {
|
||
continue;
|
||
}
|
||
let o = oy * out_w + ox;
|
||
let d = look.to_display([sum[0] / n as f32, sum[1] / n as f32, sum[2] / n as f32]);
|
||
rgb[o * 3..o * 3 + 3].copy_from_slice(&d);
|
||
known[o] = true;
|
||
}
|
||
}
|
||
(out_w, out_h, rgb, known)
|
||
}
|
||
|
||
fn bounds_of(
|
||
proxies: &[Gray],
|
||
alignment: &Alignment,
|
||
projection: Projection,
|
||
) -> projection::Bounds {
|
||
let (fw, fh) = (proxies[0].width as f64, proxies[0].height as f64);
|
||
projection::bounds(projection, alignment.focal, &alignment.cameras(), (fw, fh)).unwrap_or(
|
||
projection::Bounds {
|
||
min_u: 0.0,
|
||
min_v: 0.0,
|
||
max_u: 1.0,
|
||
max_v: 1.0,
|
||
},
|
||
)
|
||
}
|
||
|
||
/// The planes to RGBA for the page, with each frame outlined.
|
||
///
|
||
/// The outlines, so the page shows *frames* and not one picture: a sweep
|
||
/// of frames overlapping by more than half reads as a single photograph
|
||
/// otherwise, which is what a photographer looking for a misplaced frame
|
||
/// needs to see through. Walked along each border and drawn where it
|
||
/// lands, two pixels wide, in a colour the scene does not have.
|
||
fn preview_rgba(
|
||
rgb: &[f32],
|
||
out_w: usize,
|
||
out_h: usize,
|
||
alignment: &Alignment,
|
||
projection: Projection,
|
||
bounds: &projection::Bounds,
|
||
proxies: &[Gray],
|
||
) -> (u32, u32, Vec<u8>) {
|
||
let cameras = alignment.cameras();
|
||
let (fw, fh) = (proxies[0].width as f64, proxies[0].height as f64);
|
||
let scale = alignment.focal;
|
||
let px = bounds.width() / out_w.max(1) as f64;
|
||
let mut out = vec![255u8; out_w * out_h * 4];
|
||
for i in 0..out_w * out_h {
|
||
for c in 0..3 {
|
||
out[i * 4 + c] = (rgb[i * 3 + c].clamp(0.0, 1.0) * 255.0) as u8;
|
||
}
|
||
}
|
||
let steps = 600;
|
||
for k in 0..proxies.len() {
|
||
for s in 0..steps {
|
||
let t = s as f64 / steps as f64;
|
||
for p in [
|
||
(-fw / 2.0 + fw * t, -fh / 2.0),
|
||
(-fw / 2.0 + fw * t, fh / 2.0 - 1.0),
|
||
(-fw / 2.0, -fh / 2.0 + fh * t),
|
||
(fw / 2.0 - 1.0, -fh / 2.0 + fh * t),
|
||
] {
|
||
let d = cameras.bearing(k, p);
|
||
let Some((u, v)) = projection.from_direction(scale, d) else {
|
||
continue;
|
||
};
|
||
let x = ((u - bounds.min_u) / px) as i64;
|
||
let y = ((v - bounds.min_v) / px) as i64;
|
||
for (dx, dy) in [(0, 0), (1, 0), (0, 1), (1, 1)] {
|
||
let (xx, yy) = (x + dx, y + dy);
|
||
if xx < 0 || yy < 0 || xx >= out_w as i64 || yy >= out_h as i64 {
|
||
continue;
|
||
}
|
||
let o = (yy as usize * out_w + xx as usize) * 4;
|
||
out[o] = 255;
|
||
out[o + 1] = 176;
|
||
out[o + 2] = 0;
|
||
out[o + 3] = 255;
|
||
}
|
||
}
|
||
}
|
||
}
|
||
(out_w as u32, out_h as u32, out)
|
||
}
|
||
|
||
/// The quick look from camera space to something like sRGB and back: the
|
||
/// first frame's white balance and matrix, a 2.2 gamma. Not the pipeline —
|
||
/// the page's preview and the filler's input, both of which want a
|
||
/// plausible picture rather than a calibrated one — and invertible, so the
|
||
/// filler's answer comes back to camera space where the DNG lives.
|
||
struct Look {
|
||
wb: [f32; 3],
|
||
m: [f32; 9],
|
||
inv: [f32; 9],
|
||
}
|
||
|
||
impl Look {
|
||
fn of(raw: &RawImage) -> Self {
|
||
let wb = [raw.wb_coeffs[0], raw.wb_coeffs[1], raw.wb_coeffs[2]];
|
||
let m = raw
|
||
.color_matrix
|
||
.unwrap_or([1.0, 0.0, 0.0, 0.0, 1.0, 0.0, 0.0, 0.0, 1.0]);
|
||
Look {
|
||
wb,
|
||
m,
|
||
inv: invert3(&m).unwrap_or([1.0, 0.0, 0.0, 0.0, 1.0, 0.0, 0.0, 0.0, 1.0]),
|
||
}
|
||
}
|
||
|
||
fn to_display(&self, cam: [f32; 3]) -> [f32; 3] {
|
||
let b = [
|
||
cam[0] * self.wb[0],
|
||
cam[1] * self.wb[1],
|
||
cam[2] * self.wb[2],
|
||
];
|
||
let row =
|
||
|c: usize| self.m[c * 3] * b[0] + self.m[c * 3 + 1] * b[1] + self.m[c * 3 + 2] * b[2];
|
||
let encode = |lin: f32| lin.clamp(0.0, 1.0).powf(1.0 / 2.2);
|
||
[encode(row(0)), encode(row(1)), encode(row(2))]
|
||
}
|
||
|
||
fn to_camera(&self, disp: [f32; 3]) -> [f32; 3] {
|
||
let lin = [
|
||
disp[0].max(0.0).powf(2.2),
|
||
disp[1].max(0.0).powf(2.2),
|
||
disp[2].max(0.0).powf(2.2),
|
||
];
|
||
let row = |c: usize| {
|
||
self.inv[c * 3] * lin[0] + self.inv[c * 3 + 1] * lin[1] + self.inv[c * 3 + 2] * lin[2]
|
||
};
|
||
let b = [row(0), row(1), row(2)];
|
||
[
|
||
(b[0] / self.wb[0].max(1e-4)).max(0.0),
|
||
(b[1] / self.wb[1].max(1e-4)).max(0.0),
|
||
(b[2] / self.wb[2].max(1e-4)).max(0.0),
|
||
]
|
||
}
|
||
}
|
||
|
||
fn invert3(m: &[f32; 9]) -> Option<[f32; 9]> {
|
||
let (a, b, c, d, e, f, g, h, i) = (m[0], m[1], m[2], m[3], m[4], m[5], m[6], m[7], m[8]);
|
||
let det = a * (e * i - f * h) - b * (d * i - f * g) + c * (d * h - e * g);
|
||
if det.abs() < 1e-12 {
|
||
return None;
|
||
}
|
||
let s = 1.0 / det;
|
||
Some([
|
||
(e * i - f * h) * s,
|
||
(c * h - b * i) * s,
|
||
(b * f - c * e) * s,
|
||
(f * g - d * i) * s,
|
||
(a * i - c * g) * s,
|
||
(c * d - a * f) * s,
|
||
(d * h - e * g) * s,
|
||
(b * g - a * h) * s,
|
||
(a * e - b * d) * s,
|
||
])
|
||
}
|
||
|
||
/// The horizontal and vertical field of view the cameras span, in radians:
|
||
/// the angle between the extreme frame centres plus one frame's own field.
|
||
fn field_of_view(cameras: &Cameras, frame: (f64, f64)) -> (f64, f64) {
|
||
let f = cameras.focal;
|
||
let own_h = 2.0 * (frame.0 / (2.0 * f)).atan();
|
||
let own_v = 2.0 * (frame.1 / (2.0 * f)).atan();
|
||
let (mut min_yaw, mut max_yaw, mut min_pitch, mut max_pitch) = (0.0f64, 0.0f64, 0.0f64, 0.0f64);
|
||
for r in &cameras.rotations {
|
||
let d = *r * dr_pano::linalg::Vec3::new(0.0, 0.0, 1.0);
|
||
let yaw = d.x().atan2(d.z());
|
||
let pitch = d.y().asin();
|
||
min_yaw = min_yaw.min(yaw);
|
||
max_yaw = max_yaw.max(yaw);
|
||
min_pitch = min_pitch.min(pitch);
|
||
max_pitch = max_pitch.max(pitch);
|
||
}
|
||
(max_yaw - min_yaw + own_h, max_pitch - min_pitch + own_v)
|
||
}
|
||
|
||
fn report_size(b: &projection::Bounds) -> (u32, u32) {
|
||
(
|
||
b.width().ceil().max(1.0) as u32,
|
||
b.height().ceil().max(1.0) as u32,
|
||
)
|
||
}
|
||
|
||
fn describe(a: &Alignment) -> Vec<Result<(f64, f64), String>> {
|
||
a.rotations
|
||
.iter()
|
||
.enumerate()
|
||
.map(|(k, r)| match r {
|
||
Some(r) => {
|
||
let yaw = r.0[0][2].atan2(r.0[2][2]).to_degrees();
|
||
let pitch = (-r.0[1][2]).asin().to_degrees();
|
||
Ok((yaw, pitch))
|
||
}
|
||
None => Err(a
|
||
.unaligned
|
||
.iter()
|
||
.find(|(i, _)| *i == k)
|
||
.map(|(_, why)| why.to_string())
|
||
.unwrap_or_else(|| "not aligned".into())),
|
||
})
|
||
.collect()
|
||
}
|
||
|
||
/// One gain per frame, from the proxies' overlaps.
|
||
///
|
||
/// For each link, the mean grey of each frame over the region both see is
|
||
/// compared; the log-gains that best reconcile every link are solved in
|
||
/// least squares with the reference frame held at 1. Grey here is the
|
||
/// proxy's gamma-encoded value raised back to linear, so the gain is a
|
||
/// linear multiplier as the shader applies it.
|
||
fn gains(proxies: &[Gray], a: &Alignment) -> Vec<f32> {
|
||
let n = proxies.len();
|
||
let cameras = a.cameras();
|
||
let (w, h) = (proxies[0].width as f64, proxies[0].height as f64);
|
||
let linear = |g: &Gray, x: usize, y: usize| g.data[y * g.width + x].powf(2.2) as f64;
|
||
|
||
// Ratios per link.
|
||
let mut ratios: Vec<(usize, usize, f64)> = Vec::new();
|
||
for l in &a.links {
|
||
let (mut sum_i, mut sum_j, mut count) = (0.0, 0.0, 0usize);
|
||
let step = 8;
|
||
for y in (0..proxies[l.i].height).step_by(step) {
|
||
for x in (0..proxies[l.i].width).step_by(step) {
|
||
let p = (x as f64 - w / 2.0, y as f64 - h / 2.0);
|
||
let d = cameras.bearing(l.i, p);
|
||
let Some((qx, qy)) = cameras.project(l.j, d) else {
|
||
continue;
|
||
};
|
||
let (qx, qy) = (qx + w / 2.0, qy + h / 2.0);
|
||
if qx < 0.0 || qy < 0.0 || qx >= w - 1.0 || qy >= h - 1.0 {
|
||
continue;
|
||
}
|
||
sum_i += linear(&proxies[l.i], x, y);
|
||
sum_j += linear(&proxies[l.j], qx as usize, qy as usize);
|
||
count += 1;
|
||
}
|
||
}
|
||
if count >= 50 && sum_i > 0.0 && sum_j > 0.0 {
|
||
ratios.push((l.i, l.j, (sum_i / sum_j).ln()));
|
||
}
|
||
}
|
||
if ratios.is_empty() {
|
||
return vec![1.0; n];
|
||
}
|
||
// Least squares on log gains: g_j - g_i = ln(mean_i / mean_j), with
|
||
// the reference frame's gain fixed at 0 by a strong prior.
|
||
let root = a
|
||
.rotations
|
||
.iter()
|
||
.position(|r| *r == Some(dr_pano::linalg::Mat3::IDENTITY))
|
||
.unwrap_or(0);
|
||
let mut ata = dr_pano::linalg::DMat::zeros(n);
|
||
let mut atb = vec![0.0f64; n];
|
||
for &(i, j, r) in &ratios {
|
||
// Row: +1 at j, −1 at i, rhs r.
|
||
ata[(j, j)] += 1.0;
|
||
ata[(i, i)] += 1.0;
|
||
ata[(i, j)] -= 1.0;
|
||
ata[(j, i)] -= 1.0;
|
||
atb[j] += r;
|
||
atb[i] -= r;
|
||
}
|
||
ata[(root, root)] += 1e6;
|
||
// A tiny ridge keeps an unlinked frame (there are none if the alignment
|
||
// is complete) from making the system singular.
|
||
for k in 0..n {
|
||
ata[(k, k)] += 1e-9;
|
||
}
|
||
match ata.solve_spd(&atb) {
|
||
Some(g) => g.iter().map(|v| v.exp() as f32).collect(),
|
||
None => vec![1.0; n],
|
||
}
|
||
}
|
||
|
||
fn dng_profile(first: &Loaded, white_level: u32) -> dr_export::DngProfile {
|
||
let wb = first.raw.wb_coeffs;
|
||
let neutral_from_wb = [
|
||
1.0 / wb[0].max(1e-3),
|
||
1.0 / wb[1].max(1e-3),
|
||
1.0 / wb[2].max(1e-3),
|
||
];
|
||
let (calibrations, as_shot_neutral) = match &first.raw.profile {
|
||
Some(p) => (p.dng_calibrations(), p.neutral().unwrap_or(neutral_from_wb)),
|
||
None => (Vec::new(), neutral_from_wb),
|
||
};
|
||
let unique_model = match (first.raw.make.trim(), first.raw.model.trim()) {
|
||
("", "") => "DarkRoom panorama".to_string(),
|
||
(make, model) if model.starts_with(make) => model.to_string(),
|
||
(make, model) => format!("{make} {model}"),
|
||
};
|
||
dr_export::DngProfile {
|
||
unique_model,
|
||
calibrations,
|
||
as_shot_neutral,
|
||
white_level,
|
||
}
|
||
}
|
||
|
||
fn stem(p: &Path) -> String {
|
||
p.file_stem()
|
||
.map(|s| s.to_string_lossy().into_owned())
|
||
.unwrap_or_else(|| p.display().to_string())
|
||
}
|
||
|
||
/// `name` in `dir`, numbered if that name is taken: a merge never
|
||
/// overwrites (FR-MRG-3).
|
||
fn unused_name(dir: &Path, name: &str) -> PathBuf {
|
||
let mut candidate = dir.join(name);
|
||
let base = stem(Path::new(name));
|
||
let mut n = 2;
|
||
while candidate.exists() {
|
||
candidate = dir.join(format!("{base}-{n}.dng"));
|
||
n += 1;
|
||
}
|
||
candidate
|
||
}
|
||
|
||
/// Drain everything a job has said so far.
|
||
/// Everything the worker has said since the last call, and whether it has
|
||
/// hung up — a closed channel after nothing [`MergeEvent::is_final`] is a
|
||
/// worker that died mid-job.
|
||
pub fn drain(rx: &Receiver<MergeEvent>) -> (Vec<MergeEvent>, bool) {
|
||
let mut out = Vec::new();
|
||
loop {
|
||
match rx.try_recv() {
|
||
Ok(e) => out.push(e),
|
||
Err(std::sync::mpsc::TryRecvError::Empty) => return (out, false),
|
||
Err(std::sync::mpsc::TryRecvError::Disconnected) => return (out, true),
|
||
}
|
||
}
|
||
}
|
||
|
||
/// The filler's input as the debugging example reads it: `input.ppm`, the
|
||
/// display-space picture, and `known.pgm`, 255 where a frame reached.
|
||
fn dump_fill_input(
|
||
dir: &Path,
|
||
rgb: &[f32],
|
||
known: &[bool],
|
||
w: usize,
|
||
h: usize,
|
||
) -> std::io::Result<()> {
|
||
std::fs::create_dir_all(dir)?;
|
||
let mut ppm = format!("P6\n{w} {h}\n255\n").into_bytes();
|
||
ppm.extend(
|
||
rgb.iter()
|
||
.map(|v| (v.clamp(0.0, 1.0) * 255.0).round() as u8),
|
||
);
|
||
std::fs::write(dir.join("input.ppm"), ppm)?;
|
||
let mut pgm = format!("P5\n{w} {h}\n255\n").into_bytes();
|
||
pgm.extend(known.iter().map(|&k| if k { 255u8 } else { 0 }));
|
||
std::fs::write(dir.join("known.pgm"), pgm)
|
||
}
|
||
|
||
#[cfg(test)]
|
||
mod tests {
|
||
use super::*;
|
||
|
||
#[test]
|
||
fn a_worker_that_hangs_up_mid_job_is_reported_as_gone() {
|
||
// The failure the page could not see: a panic drops the sender with
|
||
// no final event, and `try_recv`'s Disconnected used to be folded
|
||
// into "nothing new".
|
||
let (tx, rx) = std::sync::mpsc::channel();
|
||
tx.send(MergeEvent::Progress {
|
||
stage: "Reading",
|
||
done: 1,
|
||
total: 12,
|
||
})
|
||
.unwrap();
|
||
let (events, gone) = drain(&rx);
|
||
assert_eq!(events.len(), 1);
|
||
assert!(!gone, "the sender is still alive");
|
||
|
||
drop(tx);
|
||
let (events, gone) = drain(&rx);
|
||
assert!(events.is_empty());
|
||
assert!(gone, "and now it is not");
|
||
}
|
||
|
||
#[test]
|
||
fn a_job_that_said_its_last_word_is_not_a_dead_worker() {
|
||
let (tx, rx) = std::sync::mpsc::channel();
|
||
tx.send(MergeEvent::Failed("out of memory".into())).unwrap();
|
||
drop(tx);
|
||
let (events, gone) = drain(&rx);
|
||
assert!(gone);
|
||
assert!(events.iter().any(MergeEvent::is_final), "Failed is final");
|
||
assert!(MergeEvent::Cancelled.is_final());
|
||
assert!(!MergeEvent::Progress {
|
||
stage: "x",
|
||
done: 0,
|
||
total: 0
|
||
}
|
||
.is_final());
|
||
}
|
||
}
|