From b1c99b57967120770ffe05c76138be6a4b1c100f Mon Sep 17 00:00:00 2001 From: Duncan Tourolle Date: Sat, 19 Sep 2026 23:04:24 +0200 Subject: [PATCH] Let the fill show the model an open void: mirror depth 0 means no ring and nothing known beyond the band --- core/dr-pano/src/fill.rs | 100 +++++++++++++++++++++++++++++++++------ docs/panorama.md | 37 +++++++++++++++ docs/traceability.md | 2 +- models/LICENCE.md | 13 +++-- tools/export-migan.sh | 6 +++ ui/dr-ui/src/merge.rs | 6 ++- ui/dr-ui/src/merge_ui.rs | 2 +- ui/dr-ui/ui/merge.slint | 20 ++++---- 8 files changed, 155 insertions(+), 31 deletions(-) diff --git a/core/dr-pano/src/fill.rs b/core/dr-pano/src/fill.rs index cbe1219..bd82468 100644 --- a/core/dr-pano/src/fill.rs +++ b/core/dr-pano/src/fill.rs @@ -89,12 +89,19 @@ pub struct Params { pub coarse: usize, /// The fine passes' band width. pub band: usize, - /// How deep into the picture the mirrored context reaches. A plain - /// reflection of a deep hole pulls in whatever is that far from the - /// edge — a ridge, a peak — and the model, told that is what lies - /// beyond, paints it upside down. Folding the reflection within this - /// band keeps the ring looking like the edge it continues (sky beside - /// sky, grass beside grass) and nothing further away. + /// How deep into the picture the mirrored context reaches, or **zero + /// for no mirrored context at all**: the void is then shown to the + /// model as it is — reaching the picture's edge with nothing beyond, + /// and, beyond the band being filled, still unknown. That is what the + /// shipped model was trained on (a fine-tune of MI-GAN on voids cut + /// from photographs the way a cylindrical merge cuts them, see + /// `docs/panorama.md` §14); a ring would give it a fold to continue. + /// + /// Non-zero is the stock model's crutch: a plain reflection of a deep + /// hole pulls in whatever is that far from the edge — a ridge, a peak — + /// and the model, told that is what lies beyond, paints it upside down. + /// Folding the reflection within this band keeps the ring looking like + /// the edge it continues and nothing further away. pub mirror_depth: usize, /// How far inside the real edge the fill also regenerates, the two /// blended by distance. A hard cut between real pixels and invented @@ -108,9 +115,9 @@ pub struct Params { impl Default for Params { fn default() -> Self { Params { - coarse: 4, - band: 96, - mirror_depth: 48, + coarse: 1, + band: 192, + mirror_depth: 0, feather: 24, stride: 384, } @@ -141,7 +148,10 @@ pub fn fill_border( } = params; let q = q.max(1); let band = band.max(8); - let mirror_depth = mirror_depth.max(1); + // No ring: the void beyond the band stays unknown, as in the model's + // training; with a ring the far side is the coarse fill, presented as + // known, which the stock model needed to see something there. + let open = mirror_depth == 0; if width == 0 || height == 0 || rgb.len() != width * height * 3 || known.len() != width * height { return Err(PanoError::Input("fill: buffer sizes disagree".into())); @@ -227,7 +237,11 @@ pub fn fill_border( let mut any = false; for i in 0..width * height { let in_band = !known[i] && dist[i] > lo && dist[i] <= hi; - band_known[i] = !in_band; + band_known[i] = if open { + known[i] || dist[i] <= lo + } else { + !in_band + }; any |= in_band; } if !any { @@ -369,7 +383,7 @@ fn fill_once( if known[i] { continue; } - let p = (yy + RING) * pw + (xx + RING); + let p = (yy + ctx.ring) * pw + (xx + ctx.ring); if wsum[p] > 0.0 { for ch in 0..3 { rgb[i * 3 + ch] = (acc[p * 3 + ch] / wsum[p]).clamp(0.0, 1.0); @@ -478,12 +492,25 @@ fn fold(d: usize, depth: usize) -> usize { struct MirroredContext { width: usize, height: usize, + /// The padding on every side: `RING` with mirrored context, 0 without. + ring: usize, rgb: Vec, hole: Vec, } impl MirroredContext { fn build(rgb: &[f32], width: usize, height: usize, known: &[bool], depth: usize) -> Self { + if depth == 0 { + // Open: the picture as it is, the hole as it is. What the hole + // holds does not matter — the model masks it out. + return MirroredContext { + width, + height, + ring: 0, + rgb: rgb.to_vec(), + hole: known.iter().map(|&k| !k).collect(), + }; + } let fold = |d: usize| fold(d, depth); let (pw, ph) = (width + 2 * RING, height + 2 * RING); let mut canvas = vec![0.0f32; pw * ph * 3]; @@ -534,6 +561,7 @@ impl MirroredContext { MirroredContext { width: pw, height: ph, + ring: RING, rgb: canvas, hole, } @@ -634,7 +662,10 @@ mod tests { 200, &known, &mut model, - test_params(0), + Params { + mirror_depth: 48, + ..test_params(0) + }, &mut |_, _| {}, ) .unwrap(); @@ -648,6 +679,42 @@ mod tests { } } + #[test] + fn an_open_void_reaches_the_tile_edge_and_stays_unknown_beyond_the_band() { + // A 150-tall hole above and below; bands of 96. With no ring the + // first band's tiles sit at the picture's edge, so a tile's top + // row is unknown, and the rows deeper than the band are unknown + // too — not "known" coarse fill — exactly as the model was trained. + let (mut rgb, known) = picture(200, 500, 150); + let mut model = Flat { + tile: 64, + seen: Vec::new(), + }; + fill_border( + &mut rgb, + 200, + 500, + &known, + &mut model, + Params { + band: 96, + ..test_params(0) + }, + &mut |_, _| {}, + ) + .unwrap(); + // The first pass's tile at the picture's top edge is unknown + // through and through: the hole is 150 deep, the tile 64, and + // nothing beyond the band was presented as known. With a ring, or + // with the far side shown as coarse fill, no tile is ever all hole. + assert!(model.seen.iter().any(|(_, k)| k.iter().all(|&v| !v))); + for i in 0..200 * 500 { + if !known[i] { + assert!((rgb[i * 3] - 0.5).abs() < 1e-4, "pixel {i}"); + } + } + } + #[test] fn the_fine_passes_run_in_bands_after_the_coarse_one() { // A 150-tall hole above and below a picture: the coarse pass sees @@ -663,7 +730,12 @@ mod tests { 500, &known, &mut model, - test_params(0), + Params { + coarse: 4, + band: 96, + mirror_depth: 48, + ..test_params(0) + }, &mut |_, _| {}, ) .unwrap(); diff --git a/docs/panorama.md b/docs/panorama.md index be2d80c..4ee555c 100644 --- a/docs/panorama.md +++ b/docs/panorama.md @@ -486,3 +486,40 @@ rectangle. `models/LICENCE.md`), installed by the PKGBUILD and unpacked by the APK beside the face and scene models; `tools/export-migan.sh` regenerates it from the upstream checkpoint. + +## 14. The fill, trained — 2026-09-20 + +§13.5 named the remaining fault: in a deep corner the stock model puts its +Places2 prior — clouds, peaks, a water line — into a hole, because it was +trained on holes *inside* pictures and a panorama's border is a hole with +the picture on one side and nothing on the other. Every ring (mirror, +replicate, detrend) treated the symptom. The fix is a model that has seen +the real thing: **MI-GAN's 512 generator fine-tuned on border-shaped voids +cut from the user's own photographs**, in a separate repository +(`darkroom-infill`, beside this one), so the truth beyond the void is known +and the model learns one-sided extrapolation. + +**What it was trained on.** Voids made the way this merge makes them: +frames with a yaw, a common pitch and per-frame roll, projected onto the +cylinder and rasterised, the canvas their union's bounding box, the void +the canvas outside the union — arcs where straight edges bent, cusps where +frames meet, the bow-tie wedge at a corner (a third of tiles are cut at a +canvas corner). Voids to 256 px deep at a 512 tile. Half the deep tiles +train the *second pass*: a no-grad first pass fills the tile, its nearest +band (64–256 px) is marked known, and the remaining void is the example — +so the model continues its own output without drift, which is how `fill` +runs deep voids. Data: ~7 400 pictures — the 1024-px proxy tier of the +library and ~2 000 raws sampled evenly across every year, developed at +half size. Loss: hole-weighted L1, VGG16 perceptual, a hinge PatchGAN. +One night on the reference desktop's RTX 3050. + +**What changed here.** `FillParams::mirror_depth` **0** is now "no ring": +the void reaches the tile's edge with nothing beyond, and beyond the band +being filled the void stays *unknown* rather than presented as known coarse +fill — the two conditions the model was trained under. Defaults: mirror 0, +coarse 1 (the coarse pass seeds nothing the model is allowed to see), band +192. The ring remains on the page for the stock model's sake, at any depth +above zero. The model file is a drop-in (`models/inpaint/migan-512.onnx`, +same six operators, same tensors) and the engine loads it unchanged. + +**Measured.** PLACEHOLDER diff --git a/docs/traceability.md b/docs/traceability.md index 920091f..609fbff 100644 --- a/docs/traceability.md +++ b/docs/traceability.md @@ -107,7 +107,7 @@ A tag naming an ID `requirements.md` does not define — what renumbering produc | FR-MRG-3 | [`core/dr-decode/examples/linear_dng.rs:1`](../core/dr-decode/examples/linear_dng.rs#L1), [`core/dr-decode/src/lib.rs:137`](../core/dr-decode/src/lib.rs#L137), [`core/dr-decode/src/lib.rs:144`](../core/dr-decode/src/lib.rs#L144), [`core/dr-decode/src/lib.rs:598`](../core/dr-decode/src/lib.rs#L598), [`core/dr-decode/src/profile.rs:395`](../core/dr-decode/src/profile.rs#L395), [`core/dr-export/src/dng.rs:1`](../core/dr-export/src/dng.rs#L1), [`core/dr-gpu/src/demosaic.rs:254`](../core/dr-gpu/src/demosaic.rs#L254), [`ui/dr-ui/src/merge.rs:1`](../ui/dr-ui/src/merge.rs#L1) | | FR-MRG-4 | [`core/dr-export/src/inscribed.rs:1`](../core/dr-export/src/inscribed.rs#L1), [`core/dr-pano/src/fill.rs:1`](../core/dr-pano/src/fill.rs#L1), [`core/dr-pano/src/migan.rs:1`](../core/dr-pano/src/migan.rs#L1), [`core/dr-pano/src/projection.rs:1`](../core/dr-pano/src/projection.rs#L1), [`ui/dr-ui/src/library.rs:4457`](../ui/dr-ui/src/library.rs#L4457), [`ui/dr-ui/src/library.rs:4461`](../ui/dr-ui/src/library.rs#L4461), [`ui/dr-ui/src/merge.rs:143`](../ui/dr-ui/src/merge.rs#L143), [`ui/dr-ui/src/merge_ui.rs:1`](../ui/dr-ui/src/merge_ui.rs#L1), [`ui/dr-ui/ui/merge.slint:1`](../ui/dr-ui/ui/merge.slint#L1), [`ui/dr-ui/ui/merge.slint:46`](../ui/dr-ui/ui/merge.slint#L46) | | FR-MRG-5 | [`core/dr-pano/src/align.rs:1`](../core/dr-pano/src/align.rs#L1), [`ui/dr-ui/src/merge.rs:1`](../ui/dr-ui/src/merge.rs#L1), [`ui/dr-ui/src/merge_ui.rs:1`](../ui/dr-ui/src/merge_ui.rs#L1), [`ui/dr-ui/ui/merge.slint:1`](../ui/dr-ui/ui/merge.slint#L1) | -| FR-MRG-6 | [`core/dr-pipeline/src/sidecar.rs:1033`](../core/dr-pipeline/src/sidecar.rs#L1033), [`core/dr-pipeline/src/sidecar.rs:113`](../core/dr-pipeline/src/sidecar.rs#L113), [`core/dr-pipeline/src/sidecar.rs:123`](../core/dr-pipeline/src/sidecar.rs#L123), [`core/dr-pipeline/src/sidecar.rs:2241`](../core/dr-pipeline/src/sidecar.rs#L2241), [`core/dr-pipeline/src/sidecar.rs:841`](../core/dr-pipeline/src/sidecar.rs#L841), [`ui/dr-ui/src/merge.rs:838`](../ui/dr-ui/src/merge.rs#L838) | +| FR-MRG-6 | [`core/dr-pipeline/src/sidecar.rs:1033`](../core/dr-pipeline/src/sidecar.rs#L1033), [`core/dr-pipeline/src/sidecar.rs:113`](../core/dr-pipeline/src/sidecar.rs#L113), [`core/dr-pipeline/src/sidecar.rs:123`](../core/dr-pipeline/src/sidecar.rs#L123), [`core/dr-pipeline/src/sidecar.rs:2241`](../core/dr-pipeline/src/sidecar.rs#L2241), [`core/dr-pipeline/src/sidecar.rs:841`](../core/dr-pipeline/src/sidecar.rs#L841), [`ui/dr-ui/src/merge.rs:842`](../ui/dr-ui/src/merge.rs#L842) | | FR-MRG-7 | [`ui/dr-ui/src/merge.rs:1`](../ui/dr-ui/src/merge.rs#L1), [`ui/dr-ui/src/merge_ui.rs:1`](../ui/dr-ui/src/merge_ui.rs#L1), [`ui/dr-ui/ui/app.slint:829`](../ui/dr-ui/ui/app.slint#L829), [`ui/dr-ui/ui/merge.slint:1`](../ui/dr-ui/ui/merge.slint#L1) | | FR-MRG-8 | [`core/dr-pano/src/xfeat.rs:1`](../core/dr-pano/src/xfeat.rs#L1), [`core/dr-segment/examples/onnx_probe.rs:1`](../core/dr-segment/examples/onnx_probe.rs#L1) | | FR-NC-1 | [`core/dr-sync-nextcloud/src/auth.rs:132`](../core/dr-sync-nextcloud/src/auth.rs#L132), [`core/dr-sync-nextcloud/src/auth.rs:44`](../core/dr-sync-nextcloud/src/auth.rs#L44), [`core/dr-sync-nextcloud/src/provider.rs:1`](../core/dr-sync-nextcloud/src/provider.rs#L1), [`core/dr-sync/src/account.rs:375`](../core/dr-sync/src/account.rs#L375), [`ui/dr-ui/src/launch.rs:316`](../ui/dr-ui/src/launch.rs#L316), [`ui/dr-ui/src/launch.rs:61`](../ui/dr-ui/src/launch.rs#L61), [`ui/dr-ui/src/launch_ui.rs:418`](../ui/dr-ui/src/launch_ui.rs#L418) | diff --git a/models/LICENCE.md b/models/LICENCE.md index 350e0a5..2039c0e 100644 --- a/models/LICENCE.md +++ b/models/LICENCE.md @@ -104,11 +104,16 @@ the dataset licence restricts models trained on it by name. | File | Source | Trained on | Used by | |---|---|---|---| -| `inpaint/migan-512.onnx` | `migan_512_places2.pt` from `https://github.com/Picsart-AI-Research/MI-GAN` (Sargsyan et al., ICCV 2023) | Places2, by the authors | the panorama border fill (FR-MRG-4) | +| `inpaint/migan-512.onnx` | `migan_512_places2.pt` from `https://github.com/Picsart-AI-Research/MI-GAN` (Sargsyan et al., ICCV 2023), **fine-tuned** in the `darkroom-infill` repository (2026-09-20) | Places2 by the authors, then ~7 400 of the maintainer's own photographs with border-shaped voids | the panorama border fill (FR-MRG-4) | -Exported by `tools/export-migan.sh`: the bare 512 generator at a fixed -`1×4×512×512`, six operator types. The tiling, the context and the blend are -Rust (`dr_pano::fill`). +The bare 512 generator at a fixed `1×4×512×512`, six operator types; the +tiling, the context and the blend are Rust (`dr_pano::fill`). Since +2026-09-20 the shipped file is the fine-tune (`docs/panorama.md` §14), +exported by `python -m infill.export` in `darkroom-infill`; +`tools/export-migan.sh` still produces the stock generator from the upstream +checkpoint, which the fine-tune starts from. The fine-tuned weights are a +derivative of the MIT weights trained on photographs the maintainer owns, +and carry the same MIT grant. **MIT, code and weights alike** — `LICENSE` and `LICENSE-WEIGHTS` in the repository, both read on 2026-09-19, both the plain MIT text with no further diff --git a/tools/export-migan.sh b/tools/export-migan.sh index b81f82b..9c96630 100755 --- a/tools/export-migan.sh +++ b/tools/export-migan.sh @@ -8,6 +8,12 @@ # # ./tools/export-migan.sh # -> models/inpaint/migan-512.onnx # +# Since 2026-09-20 the file that ships is not this export but a fine-tune of +# it on panorama-border voids (docs/panorama.md §14), made in the +# `darkroom-infill` repository with `python -m infill.export`. This script +# still yields the stock generator — the fine-tune's starting point, and the +# model the page's "mirror depth" knob above zero was built around. +# # Requires `uv`. Everything else is fetched into a throwaway venv, including # a CPU-only torch and `gdown` for the checkpoint, which the authors keep on # Google Drive (models/LICENCE.md has the licence; it is MIT). diff --git a/ui/dr-ui/src/merge.rs b/ui/dr-ui/src/merge.rs index a660c4c..322ab76 100644 --- a/ui/dr-ui/src/merge.rs +++ b/ui/dr-ui/src/merge.rs @@ -181,7 +181,11 @@ impl FillSettings { params: dr_pano::FillParams { coarse: self.params.coarse, band: px(self.params.band).max(8), - mirror_depth: px(self.params.mirror_depth), + mirror_depth: if self.params.mirror_depth == 0 { + 0 + } else { + px(self.params.mirror_depth) + }, feather: if self.params.feather == 0 { 0 } else { diff --git a/ui/dr-ui/src/merge_ui.rs b/ui/dr-ui/src/merge_ui.rs index e5b4e5e..9968e80 100644 --- a/ui/dr-ui/src/merge_ui.rs +++ b/ui/dr-ui/src/merge_ui.rs @@ -339,7 +339,7 @@ pub fn wire( "erosion" => f.erosion = v, "coarse" => f.params.coarse = v.max(1), "band" => f.params.band = v.max(8), - "mirror" => f.params.mirror_depth = v.max(1), + "mirror" => f.params.mirror_depth = v, "feather" => f.params.feather = v, _ => return, } diff --git a/ui/dr-ui/ui/merge.slint b/ui/dr-ui/ui/merge.slint index 89c530c..ac77150 100644 --- a/ui/dr-ui/ui/merge.slint +++ b/ui/dr-ui/ui/merge.slint @@ -63,9 +63,9 @@ export component MergePage inherits Rectangle { /// seam. Working pixels, at the merge's working scale. in property fill-scale: 2; in property fill-erosion: 4; - in property fill-coarse: 4; - in property fill-band: 96; - in property fill-mirror: 48; + in property fill-coarse: 1; + in property fill-band: 192; + in property fill-mirror: 0; in property fill-feather: 48; /// The composite, once written: its name for the status line, and @@ -240,26 +240,26 @@ export component MergePage inherits Rectangle { SliderRow { label: "Coarse pass"; hint: "reduction of the structure pass; 1 skips it"; - value: root.fill-coarse; default-value: 4; minimum: 1; maximum: 8; + value: root.fill-coarse; default-value: 1; minimum: 1; maximum: 8; enabled: !root.running && !root.done; changed(v) => { root.fill-knob("coarse", v); } - reset => { root.fill-knob("coarse", 4); } + reset => { root.fill-knob("coarse", 1); } } SliderRow { label: "Band width"; hint: "the fine passes' bands from the edge outward, working pixels"; - value: root.fill-band; default-value: 96; minimum: 16; maximum: 384; + value: root.fill-band; default-value: 192; minimum: 16; maximum: 384; enabled: !root.running && !root.done; changed(v) => { root.fill-knob("band", v); } - reset => { root.fill-knob("band", 96); } + reset => { root.fill-knob("band", 192); } } SliderRow { label: "Mirror depth"; - hint: "how far into the picture the mirrored context reaches"; - value: root.fill-mirror; default-value: 48; minimum: 4; maximum: 512; + hint: "0: the void as it is, what the model was trained on; otherwise how far into the picture a mirrored context reaches"; + value: root.fill-mirror; default-value: 0; minimum: 0; maximum: 512; enabled: !root.running && !root.done; changed(v) => { root.fill-knob("mirror", v); } - reset => { root.fill-knob("mirror", 48); } + reset => { root.fill-knob("mirror", 0); } } SliderRow { label: "Seam feather";