diff --git a/apps/darkroom-android/src/lib.rs b/apps/darkroom-android/src/lib.rs index f649e21..2bd4bbf 100644 --- a/apps/darkroom-android/src/lib.rs +++ b/apps/darkroom-android/src/lib.rs @@ -356,6 +356,8 @@ fn unpack_bundled_models(app: &slint::android::AndroidApp) { "yolo26s-sem-ade20k.classes.json", ), (c"models/categories.txt", "categories.txt"), + // The panorama border filler (FR-MRG-4); MIT, 28 MB. + (c"models/migan-512.onnx", "migan-512.onnx"), ]; let dir = dr_ui::shared_face_models_dir(); diff --git a/docker/android/assemble-apk.sh b/docker/android/assemble-apk.sh index adfe128..33730ed 100755 --- a/docker/android/assemble-apk.sh +++ b/docker/android/assemble-apk.sh @@ -299,7 +299,7 @@ fi rm -rf "${OUT}/staging/assets/models" mkdir -p "${OUT}/staging/assets/models" _bundled="" -for _dir in face scene; do +for _dir in face scene inpaint; do ASSETS="${REPO}/models/${_dir}" compgen -G "${ASSETS}/*.onnx" >/dev/null || continue # An LFS pointer is ~130 bytes and looks exactly like a model to `cp`. Left diff --git a/models/LICENCE.md b/models/LICENCE.md index 265ec1b..350e0a5 100644 --- a/models/LICENCE.md +++ b/models/LICENCE.md @@ -11,8 +11,8 @@ same script: Both come from `https://huggingface.co/Ultralytics/YOLO26`. The face weights in `face/` are a separate matter with a separate grant — see `face/README.md`. -The keypoint weights in `keypoints/` are a third, and the easiest — see the -last section. +The keypoint weights in `keypoints/` and the border filler in `inpaint/` are +the other two, and the easiest — see the last two sections. ## The grant @@ -99,3 +99,21 @@ XFeat trains on MegaDepth, which is itself a research dataset, but the weights are released under the repository's licence without a data-derived restriction — unlike the gaze models §7 of the requirements declined, where the dataset licence restricts models trained on it by name. + +## `inpaint/` — MI-GAN, MIT + +| File | Source | Trained on | Used by | +|---|---|---|---| +| `inpaint/migan-512.onnx` | `migan_512_places2.pt` from `https://github.com/Picsart-AI-Research/MI-GAN` (Sargsyan et al., ICCV 2023) | Places2, by the authors | the panorama border fill (FR-MRG-4) | + +Exported by `tools/export-migan.sh`: the bare 512 generator at a fixed +`1×4×512×512`, six operator types. The tiling, the context and the blend are +Rust (`dr_pano::fill`). + +**MIT, code and weights alike** — `LICENSE` and `LICENSE-WEIGHTS` in the +repository, both read on 2026-09-19, both the plain MIT text with no further +grant. GPL-compatible, store-compatible, nothing to read around: the cleanest +position of any model here. The training set is Places2, a research dataset, +but the weights are released under the repository's licence without a +data-derived restriction (contrast the gaze models §7 of the requirements +declined, and the InsightFace grant of D13). diff --git a/models/inpaint/migan-512.onnx b/models/inpaint/migan-512.onnx new file mode 100644 index 0000000..2331b79 --- /dev/null +++ b/models/inpaint/migan-512.onnx @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:42dede5a5fa86891443cacf6c976cfa81641f5870bd69b71ca5d2f907c1c73d8 +size 28009057 diff --git a/packaging/PKGBUILD b/packaging/PKGBUILD index 6c755f4..d5a86fb 100644 --- a/packaging/PKGBUILD +++ b/packaging/PKGBUILD @@ -94,4 +94,13 @@ package() { for _m in yolo26s-sem-ade20k.onnx yolo26s-sem-ade20k.classes.json categories.txt; do install -Dm644 "models/scene/${_m}" "${pkgdir}/usr/share/darkroom/models/${_m}" done + + # The panorama border filler (MI-GAN, MIT — models/LICENCE.md). Same + # pointer check as the scene model, same directory, same reason. + _src="models/inpaint/migan-512.onnx" + if [[ "$(stat -c%s "${_src}")" -lt 100000 ]]; then + echo "error: the inpainting model is an LFS pointer — run: git lfs pull" >&2 + return 1 + fi + install -Dm644 "${_src}" "${pkgdir}/usr/share/darkroom/models/migan-512.onnx" } diff --git a/tools/export-migan.sh b/tools/export-migan.sh new file mode 100755 index 0000000..b81f82b --- /dev/null +++ b/tools/export-migan.sh @@ -0,0 +1,65 @@ +#!/usr/bin/env bash +# Re-export the border-fill model that ships in models/inpaint/. +# +# The .onnx is committed (D14, FR-MRG-4), so this is not part of any build — +# it exists so the committed artefact is reproducible rather than a binary +# someone once produced and nobody can regenerate. Run it when bumping the +# model. +# +# ./tools/export-migan.sh # -> models/inpaint/migan-512.onnx +# +# Requires `uv`. Everything else is fetched into a throwaway venv, including +# a CPU-only torch and `gdown` for the checkpoint, which the authors keep on +# Google Drive (models/LICENCE.md has the licence; it is MIT). +# +# ## What is exported, and what is not +# +# The *bare* 512 generator, at a fixed 1×4×512×512: channel 0 is the mask +# minus a half (1 where the picture is known), channels 1–3 the RGB in −1..1 +# with the unknown pixels zeroed; out come three planes in −1..1. The +# authors' "pipeline" ONNX — crop around the mask, resize, blend, all in the +# graph with dynamic shapes — is what tract cannot load, and every one of +# those steps is done in Rust (`dr_pano::fill`), where a tile of a panorama +# border needs different context from a brush stroke anyway. +set -euo pipefail + +HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +REPO="$(cd "${HERE}/.." && pwd)" +OUT="${REPO}/models/inpaint" + +# Not `mktemp -d` under /tmp: a tmpfs, and torch is a gigabyte. +WORK="$(mktemp -d -p "${TMPDIR:-/var/tmp}")" +trap 'rm -rf "${WORK}"' EXIT + +echo "==> exporting MI-GAN in ${WORK}" +cd "${WORK}" +git clone -q --depth 1 https://github.com/Picsart-AI-Research/MI-GAN.git migan +uv venv -q --python 3.12 venv +VIRTUAL_ENV="${WORK}/venv" uv pip install -q --index-url https://download.pytorch.org/whl/cpu torch +VIRTUAL_ENV="${WORK}/venv" uv pip install -q onnx onnxslim gdown + +# The 512 Places2 checkpoint, from the authors' Drive folder. +"${WORK}/venv/bin/gdown" --quiet "https://drive.google.com/uc?id=1D_YCuCgo20S2256sqpedsmENNm2WMtVY" -O "${WORK}/migan_512_places2.pt" + +VIRTUAL_ENV="${WORK}/venv" "${WORK}/venv/bin/python" - "${WORK}/migan" "${WORK}/migan_512_places2.pt" "${WORK}/migan-512.onnx" <<'PY' +import sys, torch, onnx, onnxslim +sys.path.insert(0, sys.argv[1]) +from lib.model_zoo.migan_inference import Generator as MIGAN + +repo, ckpt, out = sys.argv[1], sys.argv[2], sys.argv[3] +model = MIGAN(resolution=512).eval() +model.load_state_dict(torch.load(ckpt, map_location="cpu")) +torch.onnx.export(model, torch.zeros(1, 4, 512, 512), out, opset_version=17, dynamo=False, + input_names=["input"], output_names=["output"], + dynamic_axes=None, do_constant_folding=True) +m = onnxslim.slim(onnx.load(out)) +onnx.checker.check_model(m) +onnx.save(m, out) +print("ops:", sorted({n.op_type for n in m.graph.node})) +for o in m.graph.output: + print("out", o.name, [d.dim_value for d in o.type.tensor_type.shape.dim]) +PY + +mkdir -p "${OUT}" +cp "${WORK}/migan-512.onnx" "${OUT}/migan-512.onnx" +echo "==> ${OUT}/migan-512.onnx"