Dump RAW mosaics for training the learned denoise

denoise.md §4.4 requires the training repo to read photosites through
dr-decode, not LibRaw, so black and white levels, the active area and the
CFA phase match what the app will feed the network. mosaic_dump reads
`input<TAB>prefix` lines and writes the whole readout as .npy plus a JSON
of what decode and metadata report. The masked border is kept: its
optically black photosites are a free dark frame for the noise profile.
This commit is contained in:
2026-10-03 10:25:27 -04:00
parent 2fad846cd1
commit 1f266a4478
2 changed files with 100 additions and 1 deletions
+99
View File
@@ -0,0 +1,99 @@
//! Dump RAW files' mosaics for training the learned denoise (FR-DEV-3g).
//!
//! The training repo must read photosites the way the app reads them —
//! same black and white levels, same active area, same CFA phase — or a
//! network trained on one phase runs on another and paints moiré everywhere
//! (denoise.md §4.4). So it reads this, not LibRaw.
//!
//! Reads `input<TAB>output-prefix` lines on stdin and writes, per line,
//! `prefix.npy` (the whole readout, masked border included, `u16`, row-major)
//! and `prefix.json` (what `decode` and `metadata` say about it). The border
//! is kept because its optically black photosites are a dark frame for free:
//! read noise and row noise at that ISO.
//!
//! ```sh
//! printf 'IMG_0001.CR2\tout/IMG_0001\n' |
//! cargo run --release -p dr-decode --example mosaic_dump
//! ```
use std::io::{BufRead, Write};
fn main() {
let mut failed = 0;
for line in std::io::stdin().lock().lines() {
let line = line.expect("stdin");
let Some((input, prefix)) = line.split_once('\t') else {
continue;
};
match dump(input, prefix) {
Ok(()) => println!("ok\t{input}"),
Err(e) => {
failed += 1;
println!("fail\t{input}\t{e}");
}
}
std::io::stdout().flush().ok();
}
std::process::exit(if failed > 0 { 1 } else { 0 });
}
fn dump(input: &str, prefix: &str) -> Result<(), String> {
let bytes = std::fs::read(input).map_err(|e| e.to_string())?;
let raw = dr_decode::decode(&bytes).map_err(|e| e.to_string())?;
if raw.samples_per_pixel != 1 {
return Err("linear DNG: no photosites".into());
}
let meta = dr_decode::metadata(&bytes).map_err(|e| e.to_string())?;
let mut npy = Vec::with_capacity(raw.data.len() * 2 + 128);
let mut header = format!(
"{{'descr': '<u2', 'fortran_order': False, 'shape': ({}, {}), }}",
raw.height, raw.width
);
// The header, its magic and length are padded to a multiple of 64.
while (10 + header.len() + 1) % 64 != 0 {
header.push(' ');
}
header.push('\n');
npy.extend_from_slice(b"\x93NUMPY\x01\x00");
npy.extend_from_slice(&(header.len() as u16).to_le_bytes());
npy.extend_from_slice(header.as_bytes());
for v in &raw.data {
npy.extend_from_slice(&v.to_le_bytes());
}
std::fs::write(format!("{prefix}.npy"), npy).map_err(|e| e.to_string())?;
let opt = |v: Option<f32>| v.map_or("null".to_string(), |v| v.to_string());
let matrix = raw
.color_matrix
.map_or("null".to_string(), |m| format!("{m:?}"));
let json = format!(
concat!(
"{{\"source\": {:?}, \"make\": {:?}, \"model\": {:?}, ",
"\"width\": {}, \"height\": {}, ",
"\"crop\": [{}, {}, {}, {}], \"cfa\": {:?}, ",
"\"black\": {:?}, \"white\": {}, \"wb\": {:?}, \"cam_to_srgb\": {}, ",
"\"iso\": {}, \"shutter\": {}, \"aperture\": {}, \"captured_at\": {}}}\n"
),
input,
raw.make,
raw.model,
raw.width,
raw.height,
raw.crop.x,
raw.crop.y,
raw.crop.width,
raw.crop.height,
format!("{:?}", raw.cfa_pattern),
raw.black_level,
raw.white_level,
raw.wb_coeffs,
matrix,
meta.iso.map_or("null".to_string(), |v| v.to_string()),
opt(meta.shutter),
opt(meta.aperture),
meta.captured_at
.map_or("null".to_string(), |v| v.to_string()),
);
std::fs::write(format!("{prefix}.json"), json).map_err(|e| e.to_string())
}
+1 -1
View File
@@ -9,7 +9,7 @@ Denominators are parsed from [`requirements.md`](requirements.md) at run time, n
| Metric | Value |
|---|---|
| Source files scanned | 486 |
| Source files scanned | 487 |
| TRACES tags found | 2033 |
| Requirements defined | 193 |
| Requirements deferred (post-v1) | 24 |