Let the fill show the model an open void: mirror depth 0 means no ring and nothing known beyond the band
This commit is contained in:
+86
-14
@@ -89,12 +89,19 @@ pub struct Params {
|
||||
pub coarse: usize,
|
||||
/// The fine passes' band width.
|
||||
pub band: usize,
|
||||
/// How deep into the picture the mirrored context reaches. A plain
|
||||
/// reflection of a deep hole pulls in whatever is that far from the
|
||||
/// edge — a ridge, a peak — and the model, told that is what lies
|
||||
/// beyond, paints it upside down. Folding the reflection within this
|
||||
/// band keeps the ring looking like the edge it continues (sky beside
|
||||
/// sky, grass beside grass) and nothing further away.
|
||||
/// How deep into the picture the mirrored context reaches, or **zero
|
||||
/// for no mirrored context at all**: the void is then shown to the
|
||||
/// model as it is — reaching the picture's edge with nothing beyond,
|
||||
/// and, beyond the band being filled, still unknown. That is what the
|
||||
/// shipped model was trained on (a fine-tune of MI-GAN on voids cut
|
||||
/// from photographs the way a cylindrical merge cuts them, see
|
||||
/// `docs/panorama.md` §14); a ring would give it a fold to continue.
|
||||
///
|
||||
/// Non-zero is the stock model's crutch: a plain reflection of a deep
|
||||
/// hole pulls in whatever is that far from the edge — a ridge, a peak —
|
||||
/// and the model, told that is what lies beyond, paints it upside down.
|
||||
/// Folding the reflection within this band keeps the ring looking like
|
||||
/// the edge it continues and nothing further away.
|
||||
pub mirror_depth: usize,
|
||||
/// How far inside the real edge the fill also regenerates, the two
|
||||
/// blended by distance. A hard cut between real pixels and invented
|
||||
@@ -108,9 +115,9 @@ pub struct Params {
|
||||
impl Default for Params {
|
||||
fn default() -> Self {
|
||||
Params {
|
||||
coarse: 4,
|
||||
band: 96,
|
||||
mirror_depth: 48,
|
||||
coarse: 1,
|
||||
band: 192,
|
||||
mirror_depth: 0,
|
||||
feather: 24,
|
||||
stride: 384,
|
||||
}
|
||||
@@ -141,7 +148,10 @@ pub fn fill_border(
|
||||
} = params;
|
||||
let q = q.max(1);
|
||||
let band = band.max(8);
|
||||
let mirror_depth = mirror_depth.max(1);
|
||||
// No ring: the void beyond the band stays unknown, as in the model's
|
||||
// training; with a ring the far side is the coarse fill, presented as
|
||||
// known, which the stock model needed to see something there.
|
||||
let open = mirror_depth == 0;
|
||||
if width == 0 || height == 0 || rgb.len() != width * height * 3 || known.len() != width * height
|
||||
{
|
||||
return Err(PanoError::Input("fill: buffer sizes disagree".into()));
|
||||
@@ -227,7 +237,11 @@ pub fn fill_border(
|
||||
let mut any = false;
|
||||
for i in 0..width * height {
|
||||
let in_band = !known[i] && dist[i] > lo && dist[i] <= hi;
|
||||
band_known[i] = !in_band;
|
||||
band_known[i] = if open {
|
||||
known[i] || dist[i] <= lo
|
||||
} else {
|
||||
!in_band
|
||||
};
|
||||
any |= in_band;
|
||||
}
|
||||
if !any {
|
||||
@@ -369,7 +383,7 @@ fn fill_once(
|
||||
if known[i] {
|
||||
continue;
|
||||
}
|
||||
let p = (yy + RING) * pw + (xx + RING);
|
||||
let p = (yy + ctx.ring) * pw + (xx + ctx.ring);
|
||||
if wsum[p] > 0.0 {
|
||||
for ch in 0..3 {
|
||||
rgb[i * 3 + ch] = (acc[p * 3 + ch] / wsum[p]).clamp(0.0, 1.0);
|
||||
@@ -478,12 +492,25 @@ fn fold(d: usize, depth: usize) -> usize {
|
||||
struct MirroredContext {
|
||||
width: usize,
|
||||
height: usize,
|
||||
/// The padding on every side: `RING` with mirrored context, 0 without.
|
||||
ring: usize,
|
||||
rgb: Vec<f32>,
|
||||
hole: Vec<bool>,
|
||||
}
|
||||
|
||||
impl MirroredContext {
|
||||
fn build(rgb: &[f32], width: usize, height: usize, known: &[bool], depth: usize) -> Self {
|
||||
if depth == 0 {
|
||||
// Open: the picture as it is, the hole as it is. What the hole
|
||||
// holds does not matter — the model masks it out.
|
||||
return MirroredContext {
|
||||
width,
|
||||
height,
|
||||
ring: 0,
|
||||
rgb: rgb.to_vec(),
|
||||
hole: known.iter().map(|&k| !k).collect(),
|
||||
};
|
||||
}
|
||||
let fold = |d: usize| fold(d, depth);
|
||||
let (pw, ph) = (width + 2 * RING, height + 2 * RING);
|
||||
let mut canvas = vec![0.0f32; pw * ph * 3];
|
||||
@@ -534,6 +561,7 @@ impl MirroredContext {
|
||||
MirroredContext {
|
||||
width: pw,
|
||||
height: ph,
|
||||
ring: RING,
|
||||
rgb: canvas,
|
||||
hole,
|
||||
}
|
||||
@@ -634,7 +662,10 @@ mod tests {
|
||||
200,
|
||||
&known,
|
||||
&mut model,
|
||||
test_params(0),
|
||||
Params {
|
||||
mirror_depth: 48,
|
||||
..test_params(0)
|
||||
},
|
||||
&mut |_, _| {},
|
||||
)
|
||||
.unwrap();
|
||||
@@ -648,6 +679,42 @@ mod tests {
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn an_open_void_reaches_the_tile_edge_and_stays_unknown_beyond_the_band() {
|
||||
// A 150-tall hole above and below; bands of 96. With no ring the
|
||||
// first band's tiles sit at the picture's edge, so a tile's top
|
||||
// row is unknown, and the rows deeper than the band are unknown
|
||||
// too — not "known" coarse fill — exactly as the model was trained.
|
||||
let (mut rgb, known) = picture(200, 500, 150);
|
||||
let mut model = Flat {
|
||||
tile: 64,
|
||||
seen: Vec::new(),
|
||||
};
|
||||
fill_border(
|
||||
&mut rgb,
|
||||
200,
|
||||
500,
|
||||
&known,
|
||||
&mut model,
|
||||
Params {
|
||||
band: 96,
|
||||
..test_params(0)
|
||||
},
|
||||
&mut |_, _| {},
|
||||
)
|
||||
.unwrap();
|
||||
// The first pass's tile at the picture's top edge is unknown
|
||||
// through and through: the hole is 150 deep, the tile 64, and
|
||||
// nothing beyond the band was presented as known. With a ring, or
|
||||
// with the far side shown as coarse fill, no tile is ever all hole.
|
||||
assert!(model.seen.iter().any(|(_, k)| k.iter().all(|&v| !v)));
|
||||
for i in 0..200 * 500 {
|
||||
if !known[i] {
|
||||
assert!((rgb[i * 3] - 0.5).abs() < 1e-4, "pixel {i}");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn the_fine_passes_run_in_bands_after_the_coarse_one() {
|
||||
// A 150-tall hole above and below a picture: the coarse pass sees
|
||||
@@ -663,7 +730,12 @@ mod tests {
|
||||
500,
|
||||
&known,
|
||||
&mut model,
|
||||
test_params(0),
|
||||
Params {
|
||||
coarse: 4,
|
||||
band: 96,
|
||||
mirror_depth: 48,
|
||||
..test_params(0)
|
||||
},
|
||||
&mut |_, _| {},
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
@@ -486,3 +486,40 @@ rectangle.
|
||||
`models/LICENCE.md`), installed by the PKGBUILD and unpacked by the APK
|
||||
beside the face and scene models; `tools/export-migan.sh` regenerates it
|
||||
from the upstream checkpoint.
|
||||
|
||||
## 14. The fill, trained — 2026-09-20
|
||||
|
||||
§13.5 named the remaining fault: in a deep corner the stock model puts its
|
||||
Places2 prior — clouds, peaks, a water line — into a hole, because it was
|
||||
trained on holes *inside* pictures and a panorama's border is a hole with
|
||||
the picture on one side and nothing on the other. Every ring (mirror,
|
||||
replicate, detrend) treated the symptom. The fix is a model that has seen
|
||||
the real thing: **MI-GAN's 512 generator fine-tuned on border-shaped voids
|
||||
cut from the user's own photographs**, in a separate repository
|
||||
(`darkroom-infill`, beside this one), so the truth beyond the void is known
|
||||
and the model learns one-sided extrapolation.
|
||||
|
||||
**What it was trained on.** Voids made the way this merge makes them:
|
||||
frames with a yaw, a common pitch and per-frame roll, projected onto the
|
||||
cylinder and rasterised, the canvas their union's bounding box, the void
|
||||
the canvas outside the union — arcs where straight edges bent, cusps where
|
||||
frames meet, the bow-tie wedge at a corner (a third of tiles are cut at a
|
||||
canvas corner). Voids to 256 px deep at a 512 tile. Half the deep tiles
|
||||
train the *second pass*: a no-grad first pass fills the tile, its nearest
|
||||
band (64–256 px) is marked known, and the remaining void is the example —
|
||||
so the model continues its own output without drift, which is how `fill`
|
||||
runs deep voids. Data: ~7 400 pictures — the 1024-px proxy tier of the
|
||||
library and ~2 000 raws sampled evenly across every year, developed at
|
||||
half size. Loss: hole-weighted L1, VGG16 perceptual, a hinge PatchGAN.
|
||||
One night on the reference desktop's RTX 3050.
|
||||
|
||||
**What changed here.** `FillParams::mirror_depth` **0** is now "no ring":
|
||||
the void reaches the tile's edge with nothing beyond, and beyond the band
|
||||
being filled the void stays *unknown* rather than presented as known coarse
|
||||
fill — the two conditions the model was trained under. Defaults: mirror 0,
|
||||
coarse 1 (the coarse pass seeds nothing the model is allowed to see), band
|
||||
192. The ring remains on the page for the stock model's sake, at any depth
|
||||
above zero. The model file is a drop-in (`models/inpaint/migan-512.onnx`,
|
||||
same six operators, same tensors) and the engine loads it unchanged.
|
||||
|
||||
**Measured.** PLACEHOLDER
|
||||
|
||||
@@ -107,7 +107,7 @@ A tag naming an ID `requirements.md` does not define — what renumbering produc
|
||||
| FR-MRG-3 | [`core/dr-decode/examples/linear_dng.rs:1`](../core/dr-decode/examples/linear_dng.rs#L1), [`core/dr-decode/src/lib.rs:137`](../core/dr-decode/src/lib.rs#L137), [`core/dr-decode/src/lib.rs:144`](../core/dr-decode/src/lib.rs#L144), [`core/dr-decode/src/lib.rs:598`](../core/dr-decode/src/lib.rs#L598), [`core/dr-decode/src/profile.rs:395`](../core/dr-decode/src/profile.rs#L395), [`core/dr-export/src/dng.rs:1`](../core/dr-export/src/dng.rs#L1), [`core/dr-gpu/src/demosaic.rs:254`](../core/dr-gpu/src/demosaic.rs#L254), [`ui/dr-ui/src/merge.rs:1`](../ui/dr-ui/src/merge.rs#L1) |
|
||||
| FR-MRG-4 | [`core/dr-export/src/inscribed.rs:1`](../core/dr-export/src/inscribed.rs#L1), [`core/dr-pano/src/fill.rs:1`](../core/dr-pano/src/fill.rs#L1), [`core/dr-pano/src/migan.rs:1`](../core/dr-pano/src/migan.rs#L1), [`core/dr-pano/src/projection.rs:1`](../core/dr-pano/src/projection.rs#L1), [`ui/dr-ui/src/library.rs:4457`](../ui/dr-ui/src/library.rs#L4457), [`ui/dr-ui/src/library.rs:4461`](../ui/dr-ui/src/library.rs#L4461), [`ui/dr-ui/src/merge.rs:143`](../ui/dr-ui/src/merge.rs#L143), [`ui/dr-ui/src/merge_ui.rs:1`](../ui/dr-ui/src/merge_ui.rs#L1), [`ui/dr-ui/ui/merge.slint:1`](../ui/dr-ui/ui/merge.slint#L1), [`ui/dr-ui/ui/merge.slint:46`](../ui/dr-ui/ui/merge.slint#L46) |
|
||||
| FR-MRG-5 | [`core/dr-pano/src/align.rs:1`](../core/dr-pano/src/align.rs#L1), [`ui/dr-ui/src/merge.rs:1`](../ui/dr-ui/src/merge.rs#L1), [`ui/dr-ui/src/merge_ui.rs:1`](../ui/dr-ui/src/merge_ui.rs#L1), [`ui/dr-ui/ui/merge.slint:1`](../ui/dr-ui/ui/merge.slint#L1) |
|
||||
| FR-MRG-6 | [`core/dr-pipeline/src/sidecar.rs:1033`](../core/dr-pipeline/src/sidecar.rs#L1033), [`core/dr-pipeline/src/sidecar.rs:113`](../core/dr-pipeline/src/sidecar.rs#L113), [`core/dr-pipeline/src/sidecar.rs:123`](../core/dr-pipeline/src/sidecar.rs#L123), [`core/dr-pipeline/src/sidecar.rs:2241`](../core/dr-pipeline/src/sidecar.rs#L2241), [`core/dr-pipeline/src/sidecar.rs:841`](../core/dr-pipeline/src/sidecar.rs#L841), [`ui/dr-ui/src/merge.rs:838`](../ui/dr-ui/src/merge.rs#L838) |
|
||||
| FR-MRG-6 | [`core/dr-pipeline/src/sidecar.rs:1033`](../core/dr-pipeline/src/sidecar.rs#L1033), [`core/dr-pipeline/src/sidecar.rs:113`](../core/dr-pipeline/src/sidecar.rs#L113), [`core/dr-pipeline/src/sidecar.rs:123`](../core/dr-pipeline/src/sidecar.rs#L123), [`core/dr-pipeline/src/sidecar.rs:2241`](../core/dr-pipeline/src/sidecar.rs#L2241), [`core/dr-pipeline/src/sidecar.rs:841`](../core/dr-pipeline/src/sidecar.rs#L841), [`ui/dr-ui/src/merge.rs:842`](../ui/dr-ui/src/merge.rs#L842) |
|
||||
| FR-MRG-7 | [`ui/dr-ui/src/merge.rs:1`](../ui/dr-ui/src/merge.rs#L1), [`ui/dr-ui/src/merge_ui.rs:1`](../ui/dr-ui/src/merge_ui.rs#L1), [`ui/dr-ui/ui/app.slint:829`](../ui/dr-ui/ui/app.slint#L829), [`ui/dr-ui/ui/merge.slint:1`](../ui/dr-ui/ui/merge.slint#L1) |
|
||||
| FR-MRG-8 | [`core/dr-pano/src/xfeat.rs:1`](../core/dr-pano/src/xfeat.rs#L1), [`core/dr-segment/examples/onnx_probe.rs:1`](../core/dr-segment/examples/onnx_probe.rs#L1) |
|
||||
| FR-NC-1 | [`core/dr-sync-nextcloud/src/auth.rs:132`](../core/dr-sync-nextcloud/src/auth.rs#L132), [`core/dr-sync-nextcloud/src/auth.rs:44`](../core/dr-sync-nextcloud/src/auth.rs#L44), [`core/dr-sync-nextcloud/src/provider.rs:1`](../core/dr-sync-nextcloud/src/provider.rs#L1), [`core/dr-sync/src/account.rs:375`](../core/dr-sync/src/account.rs#L375), [`ui/dr-ui/src/launch.rs:316`](../ui/dr-ui/src/launch.rs#L316), [`ui/dr-ui/src/launch.rs:61`](../ui/dr-ui/src/launch.rs#L61), [`ui/dr-ui/src/launch_ui.rs:418`](../ui/dr-ui/src/launch_ui.rs#L418) |
|
||||
|
||||
+9
-4
@@ -104,11 +104,16 @@ the dataset licence restricts models trained on it by name.
|
||||
|
||||
| File | Source | Trained on | Used by |
|
||||
|---|---|---|---|
|
||||
| `inpaint/migan-512.onnx` | `migan_512_places2.pt` from `https://github.com/Picsart-AI-Research/MI-GAN` (Sargsyan et al., ICCV 2023) | Places2, by the authors | the panorama border fill (FR-MRG-4) |
|
||||
| `inpaint/migan-512.onnx` | `migan_512_places2.pt` from `https://github.com/Picsart-AI-Research/MI-GAN` (Sargsyan et al., ICCV 2023), **fine-tuned** in the `darkroom-infill` repository (2026-09-20) | Places2 by the authors, then ~7 400 of the maintainer's own photographs with border-shaped voids | the panorama border fill (FR-MRG-4) |
|
||||
|
||||
Exported by `tools/export-migan.sh`: the bare 512 generator at a fixed
|
||||
`1×4×512×512`, six operator types. The tiling, the context and the blend are
|
||||
Rust (`dr_pano::fill`).
|
||||
The bare 512 generator at a fixed `1×4×512×512`, six operator types; the
|
||||
tiling, the context and the blend are Rust (`dr_pano::fill`). Since
|
||||
2026-09-20 the shipped file is the fine-tune (`docs/panorama.md` §14),
|
||||
exported by `python -m infill.export` in `darkroom-infill`;
|
||||
`tools/export-migan.sh` still produces the stock generator from the upstream
|
||||
checkpoint, which the fine-tune starts from. The fine-tuned weights are a
|
||||
derivative of the MIT weights trained on photographs the maintainer owns,
|
||||
and carry the same MIT grant.
|
||||
|
||||
**MIT, code and weights alike** — `LICENSE` and `LICENSE-WEIGHTS` in the
|
||||
repository, both read on 2026-09-19, both the plain MIT text with no further
|
||||
|
||||
@@ -8,6 +8,12 @@
|
||||
#
|
||||
# ./tools/export-migan.sh # -> models/inpaint/migan-512.onnx
|
||||
#
|
||||
# Since 2026-09-20 the file that ships is not this export but a fine-tune of
|
||||
# it on panorama-border voids (docs/panorama.md §14), made in the
|
||||
# `darkroom-infill` repository with `python -m infill.export`. This script
|
||||
# still yields the stock generator — the fine-tune's starting point, and the
|
||||
# model the page's "mirror depth" knob above zero was built around.
|
||||
#
|
||||
# Requires `uv`. Everything else is fetched into a throwaway venv, including
|
||||
# a CPU-only torch and `gdown` for the checkpoint, which the authors keep on
|
||||
# Google Drive (models/LICENCE.md has the licence; it is MIT).
|
||||
|
||||
@@ -181,7 +181,11 @@ impl FillSettings {
|
||||
params: dr_pano::FillParams {
|
||||
coarse: self.params.coarse,
|
||||
band: px(self.params.band).max(8),
|
||||
mirror_depth: px(self.params.mirror_depth),
|
||||
mirror_depth: if self.params.mirror_depth == 0 {
|
||||
0
|
||||
} else {
|
||||
px(self.params.mirror_depth)
|
||||
},
|
||||
feather: if self.params.feather == 0 {
|
||||
0
|
||||
} else {
|
||||
|
||||
@@ -339,7 +339,7 @@ pub fn wire<S, C, F>(
|
||||
"erosion" => f.erosion = v,
|
||||
"coarse" => f.params.coarse = v.max(1),
|
||||
"band" => f.params.band = v.max(8),
|
||||
"mirror" => f.params.mirror_depth = v.max(1),
|
||||
"mirror" => f.params.mirror_depth = v,
|
||||
"feather" => f.params.feather = v,
|
||||
_ => return,
|
||||
}
|
||||
|
||||
+10
-10
@@ -63,9 +63,9 @@ export component MergePage inherits Rectangle {
|
||||
/// seam. Working pixels, at the merge's working scale.
|
||||
in property <float> fill-scale: 2;
|
||||
in property <float> fill-erosion: 4;
|
||||
in property <float> fill-coarse: 4;
|
||||
in property <float> fill-band: 96;
|
||||
in property <float> fill-mirror: 48;
|
||||
in property <float> fill-coarse: 1;
|
||||
in property <float> fill-band: 192;
|
||||
in property <float> fill-mirror: 0;
|
||||
in property <float> fill-feather: 48;
|
||||
|
||||
/// The composite, once written: its name for the status line, and
|
||||
@@ -240,26 +240,26 @@ export component MergePage inherits Rectangle {
|
||||
SliderRow {
|
||||
label: "Coarse pass";
|
||||
hint: "reduction of the structure pass; 1 skips it";
|
||||
value: root.fill-coarse; default-value: 4; minimum: 1; maximum: 8;
|
||||
value: root.fill-coarse; default-value: 1; minimum: 1; maximum: 8;
|
||||
enabled: !root.running && !root.done;
|
||||
changed(v) => { root.fill-knob("coarse", v); }
|
||||
reset => { root.fill-knob("coarse", 4); }
|
||||
reset => { root.fill-knob("coarse", 1); }
|
||||
}
|
||||
SliderRow {
|
||||
label: "Band width";
|
||||
hint: "the fine passes' bands from the edge outward, working pixels";
|
||||
value: root.fill-band; default-value: 96; minimum: 16; maximum: 384;
|
||||
value: root.fill-band; default-value: 192; minimum: 16; maximum: 384;
|
||||
enabled: !root.running && !root.done;
|
||||
changed(v) => { root.fill-knob("band", v); }
|
||||
reset => { root.fill-knob("band", 96); }
|
||||
reset => { root.fill-knob("band", 192); }
|
||||
}
|
||||
SliderRow {
|
||||
label: "Mirror depth";
|
||||
hint: "how far into the picture the mirrored context reaches";
|
||||
value: root.fill-mirror; default-value: 48; minimum: 4; maximum: 512;
|
||||
hint: "0: the void as it is, what the model was trained on; otherwise how far into the picture a mirrored context reaches";
|
||||
value: root.fill-mirror; default-value: 0; minimum: 0; maximum: 512;
|
||||
enabled: !root.running && !root.done;
|
||||
changed(v) => { root.fill-knob("mirror", v); }
|
||||
reset => { root.fill-knob("mirror", 48); }
|
||||
reset => { root.fill-knob("mirror", 0); }
|
||||
}
|
||||
SliderRow {
|
||||
label: "Seam feather";
|
||||
|
||||
Reference in New Issue
Block a user