dr-pano: a second XFeat shape for portrait frames, and a matcher that takes seconds

Twelve real frames from the fixture set now align in 4.5 s — 4.4 s of
matching, 118 ms of bundle adjustment — where the first run took 51 s and
left the first two frames out.

The matcher computes each pair's similarity matrix once, across the
cores, with a dot product written to vectorise; both nearest-neighbour
directions read it. The frames that failed were portrait: fitted into the
landscape input they used 512 of 1024 px, and their thin overlap did not
survive at half resolution. The same weights are now exported at 768×1024
as well and the detector picks the shape by aspect. The example aligns
from embedded previews and draws the set on a cylinder; on the fixture the
sweep is 152° at a fitted 47.9 mm against the EXIF's 50, RMS 1.5 px, and
the overlaps show no ghosting.
This commit is contained in:
2026-09-19 15:24:12 +02:00
parent 231b4a54ab
commit 54290b9540
9 changed files with 358 additions and 103 deletions
+16 -5
View File
@@ -8,20 +8,31 @@
use std::path::Path; use std::path::Path;
const MODEL: &str = "../../models/keypoints/xfeat-1024.onnx"; const MODELS: &[&str] = &[
"../../models/keypoints/xfeat-1024.onnx",
"../../models/keypoints/xfeat-768.onnx",
];
fn main() { fn main() {
println!("cargo:rerun-if-changed={MODEL}"); for m in MODELS {
println!("cargo:rerun-if-changed={m}");
}
println!("cargo:rerun-if-changed=build.rs"); println!("cargo:rerun-if-changed=build.rs");
if std::env::var_os("CARGO_FEATURE_EMBEDDED_MODEL").is_none() { if std::env::var_os("CARGO_FEATURE_EMBEDDED_MODEL").is_none() {
return; return;
} }
let path = Path::new(MODEL); for model in MODELS.iter().copied() {
check(model);
}
}
fn check(model: &str) {
let path = Path::new(model);
let Ok(bytes) = std::fs::read(path) else { let Ok(bytes) = std::fs::read(path) else {
panic!( panic!(
"\n\n{MODEL} is missing.\n\ "\n\n{model} is missing.\n\
It ships in Git LFS. Run `git lfs install && git lfs pull`, or build \ It ships in Git LFS. Run `git lfs install && git lfs pull`, or build \
with `--no-default-features` for a geometry-only build.\n" with `--no-default-features` for a geometry-only build.\n"
); );
@@ -29,7 +40,7 @@ fn main() {
if bytes.starts_with(b"version https://git-lfs.github.com/spec/") { if bytes.starts_with(b"version https://git-lfs.github.com/spec/") {
panic!( panic!(
"\n\n{MODEL} is a Git LFS pointer, not the model.\n\ "\n\n{model} is a Git LFS pointer, not the model.\n\
Run `git lfs install && git lfs pull`, or build with \ Run `git lfs install && git lfs pull`, or build with \
`--no-default-features` for a geometry-only build.\n" `--no-default-features` for a geometry-only build.\n"
); );
+172
View File
@@ -0,0 +1,172 @@
//! Align real frames from their embedded previews and draw the result.
//!
//! ```sh
//! cargo run -p dr-pano --example align --release -- fixtures/pano/2025-08-05/*.CR2
//! cargo run -p dr-pano --example align --release -- out-prefix frame1.CR2 frame2.CR2 …
//! ```
//!
//! The point of looking rather than asserting: a rotation solve that is
//! numerically converged and geometrically wrong — a mirrored axis, a
//! transposed homography, an orientation applied the wrong way — produces
//! perfectly plausible residuals and a picture that is obviously broken.
//! This writes `<prefix>-cyl.ppm`: every frame's preview warped onto a
//! cylinder and averaged where they overlap, at a size that fits on a
//! screen. Ghosting in the overlaps is the alignment error, made visible.
//!
//! Previews, not RAW: the alignment runs on proxies in the application too
//! (FR-MRG-7), and a camera's embedded JPEG is a proxy the decoder already
//! extracts in milliseconds. What is different from the real path is only
//! that the pixels are the camera's rendering rather than ours, which the
//! geometry does not care about.
use std::path::PathBuf;
use std::time::Instant;
use dr_pano::bundle::Cameras;
use dr_pano::{align, xfeat::XFeat, AlignOptions, Gray, Projection};
fn main() {
env_logger::init();
let mut args: Vec<String> = std::env::args().skip(1).collect();
if args.is_empty() {
eprintln!("usage: align [out-prefix] <frame>...");
std::process::exit(2);
}
let prefix = if args[0].ends_with(".CR2") || args[0].ends_with(".dng") || args[0].ends_with(".jpg") {
"align".to_string()
} else {
args.remove(0)
};
let paths: Vec<PathBuf> = args.iter().map(PathBuf::from).collect();
// Previews, oriented, at proxy size.
let t = Instant::now();
let mut proxies: Vec<Gray> = Vec::new();
for p in &paths {
let bytes = std::fs::read(p).expect("read");
let preview = dr_decode::extract_preview(&bytes, dr_decode::PreviewSize::Full)
.expect("embedded preview");
let orientation = dr_decode::orientation(&bytes[..bytes.len().min(dr_decode::HEADER_BYTES as usize)])
.unwrap_or(dr_types::Orientation::NORMAL);
let tag = match orientation.quarter_turns {
1 => 6,
2 => 3,
3 => 8,
_ => 1,
};
let gray = Gray::from_rgba8(&preview.rgba, preview.width as usize, preview.height as usize)
.oriented(tag);
let (fitted, _) = gray.fitted(dr_pano::xfeat::INPUT_LONG_EDGE, dr_pano::xfeat::INPUT_LONG_EDGE);
println!(
"{:<14} preview {}×{} orientation {} → proxy {}×{}",
p.file_name().unwrap().to_string_lossy(),
preview.width,
preview.height,
tag,
fitted.width,
fitted.height
);
proxies.push(fitted);
}
println!("previews in {:?}", t.elapsed());
// Keypoints.
let t = Instant::now();
let mut detector = XFeat::embedded().expect("model");
let features: Vec<_> = proxies
.iter()
.map(|g| detector.detect(g).expect("detect"))
.collect();
for (i, f) in features.iter().enumerate() {
println!("frame {i}: {} keypoints", f.len());
}
println!("detection in {:?} ({:?} per frame)", t.elapsed(), t.elapsed() / proxies.len() as u32);
// Alignment.
let t = Instant::now();
let opts = AlignOptions::default();
let alignment = align(&features, &opts).expect("align");
println!("alignment in {:?}", t.elapsed());
println!(
"focal {:.1} px, long edge {} px ({:.1} mm on full frame), rms {:.3} px",
alignment.focal,
proxies[0].width.max(proxies[0].height),
alignment.focal * 36.0 / proxies[0].width.max(proxies[0].height) as f64,
alignment.rms_px
);
for l in &alignment.links {
println!(" link {}–{}: {} inliers of {} matches", l.i, l.j, l.inliers, l.matches);
}
for (k, why) in &alignment.unaligned {
println!(" UNALIGNED frame {k}: {why}");
}
let root = alignment
.rotations
.iter()
.position(|r| *r == Some(dr_pano::linalg::Mat3::IDENTITY))
.unwrap_or(0);
for (k, r) in alignment.rotations.iter().enumerate() {
if let Some(r) = r {
// Yaw about y, pitch about x, roll about z, from the matrix's
// columns — enough to read a sweep by eye.
let yaw = r.0[0][2].atan2(r.0[2][2]).to_degrees();
let pitch = (-r.0[1][2]).asin().to_degrees();
let roll = r.0[1][0].atan2(r.0[1][1]).to_degrees();
println!(
" frame {k}: yaw {yaw:7.2}° pitch {pitch:6.2}° roll {roll:6.2}°{}",
if k == root { " (reference)" } else { "" }
);
}
}
if !alignment.is_complete() {
eprintln!("not drawing: the set is not fully aligned");
std::process::exit(1);
}
// Draw: a cylinder, averaged where frames overlap.
let t = Instant::now();
let cameras: Cameras = alignment.cameras();
let (fw, fh) = (proxies[0].width as f64, proxies[0].height as f64);
let scale = alignment.focal;
let bounds = dr_pano::projection::bounds(Projection::Cylindrical, scale, &cameras, (fw, fh))
.expect("bounds");
// Fit to 3000 px wide.
let out_w = 3000usize;
let px = bounds.width() / out_w as f64;
let out_h = (bounds.height() / px).ceil() as usize;
let mut sum = vec![0.0f32; out_w * out_h];
let mut count = vec![0u16; out_w * out_h];
for oy in 0..out_h {
for ox in 0..out_w {
let u = bounds.min_u + (ox as f64 + 0.5) * px;
let v = bounds.min_v + (oy as f64 + 0.5) * px;
let d = Projection::Cylindrical.to_direction(scale, u, v);
for (k, g) in proxies.iter().enumerate() {
let Some((x, y)) = cameras.project(k, d) else { continue };
let (x, y) = (x + g.width as f64 / 2.0, y + g.height as f64 / 2.0);
if x < 0.0 || y < 0.0 || x >= g.width as f64 - 1.0 || y >= g.height as f64 - 1.0 {
continue;
}
let (x0, y0) = (x as usize, y as usize);
let (tx, ty) = ((x - x0 as f64) as f32, (y - y0 as f64) as f32);
let p = |xx: usize, yy: usize| g.data[yy * g.width + xx];
let val = (p(x0, y0) * (1.0 - tx) + p(x0 + 1, y0) * tx) * (1.0 - ty)
+ (p(x0, y0 + 1) * (1.0 - tx) + p(x0 + 1, y0 + 1) * tx) * ty;
sum[oy * out_w + ox] += val;
count[oy * out_w + ox] += 1;
}
}
}
let mut ppm = format!("P5\n{out_w} {out_h}\n255\n").into_bytes();
ppm.extend(sum.iter().zip(&count).map(|(s, c)| {
if *c == 0 {
0u8
} else {
((s / f32::from(*c)).clamp(0.0, 1.0) * 255.0) as u8
}
}));
let out = format!("{prefix}-cyl.pgm");
std::fs::write(&out, ppm).expect("write");
println!("wrote {out} ({out_w}×{out_h}) in {:?}", t.elapsed());
}
+15
View File
@@ -151,9 +151,11 @@ pub fn align(frames: &[Features], opts: &AlignOptions) -> Result<Alignment, Pano
let mut links = Vec::new(); let mut links = Vec::new();
let mut observations: Vec<Observation> = Vec::new(); let mut observations: Vec<Observation> = Vec::new();
let mut matched_any = vec![false; n]; let mut matched_any = vec![false; n];
let t_match = std::time::Instant::now();
for i in 0..n { for i in 0..n {
for j in i + 1..n { for j in i + 1..n {
let matches: Vec<Match> = match_features(&frames[i], &frames[j], opts.min_similarity); let matches: Vec<Match> = match_features(&frames[i], &frames[j], opts.min_similarity);
log::debug!("pair {i}-{j}: {} matches", matches.len());
if matches.len() < 4 { if matches.len() < 4 {
continue; continue;
} }
@@ -175,6 +177,10 @@ pub fn align(frames: &[Features], opts: &AlignOptions) -> Result<Alignment, Pano
continue; continue;
}; };
let needed = (8.0 + 0.3 * matches.len() as f64).ceil() as usize; let needed = (8.0 + 0.3 * matches.len() as f64).ceil() as usize;
log::debug!(
"pair {i}-{j}: {} inliers, {needed} needed",
inliers.len()
);
if inliers.len() <= needed || inliers.len() < opts.min_inliers { if inliers.len() <= needed || inliers.len() < opts.min_inliers {
continue; continue;
} }
@@ -204,6 +210,8 @@ pub fn align(frames: &[Features], opts: &AlignOptions) -> Result<Alignment, Pano
} }
} }
log::debug!("matching and pairwise geometry in {:?}", t_match.elapsed());
// 3: the focal length. // 3: the focal length.
let mut estimates: Vec<f64> = links let mut estimates: Vec<f64> = links
.iter() .iter()
@@ -304,7 +312,14 @@ pub fn align(frames: &[Features], opts: &AlignOptions) -> Result<Alignment, Pano
}) })
}) })
.collect(); .collect();
let t_adjust = std::time::Instant::now();
let adjusted = bundle::adjust(start, &obs, &opts.adjust)?; let adjusted = bundle::adjust(start, &obs, &opts.adjust)?;
log::debug!(
"bundle adjustment: {} observations, {} iterations in {:?}",
obs.len(),
adjusted.iterations,
t_adjust.elapsed()
);
for (slot, &k) in aligned.iter().enumerate() { for (slot, &k) in aligned.iter().enumerate() {
rotations[k] = Some(adjusted.cameras.rotations[slot]); rotations[k] = Some(adjusted.cameras.rotations[slot]);
} }
+62 -32
View File
@@ -9,12 +9,15 @@
//! cleverer matcher. A learned matcher (LightGlue) is the step after this //! cleverer matcher. A learned matcher (LightGlue) is the step after this
//! one fails on a real set, and it has not (panorama.md §6). //! one fails on a real set, and it has not (panorama.md §6).
//! //!
//! Brute force. `4096 × 4096 × 64` multiply-adds is a billion, which is //! Brute force. `4096 × 4096 × 64` multiply-adds is a billion per pair,
//! tens of milliseconds a pair on one core, and there are at most a few //! and a twelve-frame set has sixty-six pairs: a minute single-threaded
//! dozen pairs. Not worth an index. //! and scalar (measured 2026-09-19: 51 s), a few seconds vectorised across
//! the cores. Not worth an index, but worth doing properly.
use crate::features::{Features, DESCRIPTOR_LEN}; use crate::features::{Features, DESCRIPTOR_LEN};
const _: () = assert!(DESCRIPTOR_LEN % 8 == 0);
/// A correspondence: keypoint `a` in the first image matches keypoint `b` /// A correspondence: keypoint `a` in the first image matches keypoint `b`
/// in the second, with the cosine similarity of their descriptors. /// in the second, with the cosine similarity of their descriptors.
#[derive(Debug, Clone, Copy, PartialEq)] #[derive(Debug, Clone, Copy, PartialEq)]
@@ -32,48 +35,75 @@ pub fn match_features(a: &Features, b: &Features, min_similarity: f32) -> Vec<Ma
if a.is_empty() || b.is_empty() { if a.is_empty() || b.is_empty() {
return Vec::new(); return Vec::new();
} }
let best_ab = nearest(a, b); let (na, nb) = (a.len(), b.len());
let best_ba = nearest(b, a);
// The whole similarity matrix, once. Both nearest-neighbour directions
// read it, which halves the multiply-adds against computing each
// direction on its own; 4096 × 4096 × f32 is 64 MB, transient.
let mut sim = vec![0.0f32; na * nb];
let threads = std::thread::available_parallelism()
.map(usize::from)
.unwrap_or(1)
.clamp(1, 16);
let rows_per = na.div_ceil(threads);
std::thread::scope(|scope| {
for (t, chunk) in sim.chunks_mut(rows_per * nb).enumerate() {
scope.spawn(move || {
let first = t * rows_per;
for (r, row) in chunk.chunks_mut(nb).enumerate() {
let da = a.descriptor(first + r);
for (j, cell) in row.iter_mut().enumerate() {
*cell = dot(da, b.descriptor(j));
}
}
});
}
});
// Best in `b` for each `a`, and best in `a` for each `b`.
let best_ab: Vec<(usize, f32)> = sim
.chunks_exact(nb)
.map(|row| {
row.iter()
.enumerate()
.fold((0usize, f32::MIN), |acc, (j, &s)| if s > acc.1 { (j, s) } else { acc })
})
.collect();
let mut best_ba = vec![(0usize, f32::MIN); nb];
for (i, row) in sim.chunks_exact(nb).enumerate() {
for (j, &s) in row.iter().enumerate() {
if s > best_ba[j].1 {
best_ba[j] = (i, s);
}
}
}
best_ab best_ab
.iter() .iter()
.enumerate() .enumerate()
.filter_map(|(ia, &(ib, sim))| { .filter_map(|(ia, &(ib, s))| {
(best_ba[ib].0 == ia && sim >= min_similarity).then_some(Match { (best_ba[ib].0 == ia && s >= min_similarity).then_some(Match {
a: ia, a: ia,
b: ib, b: ib,
similarity: sim, similarity: s,
}) })
}) })
.collect() .collect()
} }
/// For each descriptor in `from`, the index of its nearest in `to` and the
/// similarity.
fn nearest(from: &Features, to: &Features) -> Vec<(usize, f32)> {
(0..from.len())
.map(|i| {
let d = from.descriptor(i);
let mut best = (0usize, f32::MIN);
for j in 0..to.len() {
let s = dot(d, to.descriptor(j));
if s > best.1 {
best = (j, s);
}
}
best
})
.collect()
}
#[inline] #[inline]
fn dot(a: &[f32], b: &[f32]) -> f32 { fn dot(a: &[f32], b: &[f32]) -> f32 {
// Written as a plain loop over a fixed length so the compiler // Eight independent accumulators over exact 8-lane chunks: the shape
// vectorises it; the length is a constant and the slices are exact. // the compiler turns into one vector multiply-add per chunk, and no
let mut s = 0.0f32; // bounds checks inside the loop. `DESCRIPTOR_LEN` is a multiple of 8.
for k in 0..DESCRIPTOR_LEN { let (a, b) = (&a[..DESCRIPTOR_LEN], &b[..DESCRIPTOR_LEN]);
s += a[k] * b[k]; let mut acc = [0.0f32; 8];
for (ca, cb) in a.chunks_exact(8).zip(b.chunks_exact(8)) {
for k in 0..8 {
acc[k] += ca[k] * cb[k];
} }
s }
acc.iter().sum()
} }
#[cfg(test)] #[cfg(test)]
+45 -29
View File
@@ -11,17 +11,27 @@ use crate::features::{decode_xfeat, DecodeOptions, Features, XFeatMaps, DESCRIPT
use crate::image::Gray; use crate::image::Gray;
use crate::PanoError; use crate::PanoError;
/// The input shape the shipped export was made for. A different size is a /// The two input shapes the shipped exports were made for: one landscape,
/// different file (`tools/export-xfeat.sh`). /// one portrait, the same weights. A frame is fitted into whichever
pub const INPUT_WIDTH: usize = 1024; /// matches its aspect, so a portrait set does not spend half the
pub const INPUT_HEIGHT: usize = 768; /// detector's width on padding — which is what the 6D fixture did before
/// the second export existed (512 × 768 of a 1024 × 768 input). A
/// different size is a different file (`tools/export-xfeat.sh`).
pub const INPUT_LANDSCAPE: (usize, usize) = (1024, 768);
pub const INPUT_PORTRAIT: (usize, usize) = (768, 1024);
/// The long edge of the detector's input, for callers sizing a proxy.
pub const INPUT_LONG_EDGE: usize = 1024;
#[cfg(feature = "embedded-model")] #[cfg(feature = "embedded-model")]
const EMBEDDED_MODEL: &[u8] = include_bytes!("../../../models/keypoints/xfeat-1024.onnx"); const EMBEDDED_LANDSCAPE: &[u8] = include_bytes!("../../../models/keypoints/xfeat-1024.onnx");
#[cfg(feature = "embedded-model")]
const EMBEDDED_PORTRAIT: &[u8] = include_bytes!("../../../models/keypoints/xfeat-768.onnx");
/// A loaded detector. /// A loaded detector: the network at both shapes.
pub struct XFeat { pub struct XFeat {
session: ort::session::Session, landscape: ort::session::Session,
portrait: ort::session::Session,
pub options: DecodeOptions, pub options: DecodeOptions,
} }
@@ -29,49 +39,55 @@ impl XFeat {
/// The weights compiled into the binary. /// The weights compiled into the binary.
#[cfg(feature = "embedded-model")] #[cfg(feature = "embedded-model")]
pub fn embedded() -> Result<Self, PanoError> { pub fn embedded() -> Result<Self, PanoError> {
Self::from_bytes(EMBEDDED_MODEL) Self::from_bytes(EMBEDDED_LANDSCAPE, EMBEDDED_PORTRAIT)
} }
pub fn from_path(path: &std::path::Path) -> Result<Self, PanoError> { /// From the two exports on disk.
let bytes = std::fs::read(path).map_err(PanoError::ModelRead)?; pub fn from_paths(landscape: &std::path::Path, portrait: &std::path::Path) -> Result<Self, PanoError> {
Self::from_bytes(&bytes) let l = std::fs::read(landscape).map_err(PanoError::ModelRead)?;
let p = std::fs::read(portrait).map_err(PanoError::ModelRead)?;
Self::from_bytes(&l, &p)
} }
pub fn from_bytes(bytes: &[u8]) -> Result<Self, PanoError> { pub fn from_bytes(landscape: &[u8], portrait: &[u8]) -> Result<Self, PanoError> {
install_backend(); install_backend();
let session = ort::session::Session::builder() let session = |bytes: &[u8]| {
ort::session::Session::builder()
.map_err(PanoError::Inference)? .map_err(PanoError::Inference)?
.commit_from_memory(bytes) .commit_from_memory(bytes)
.map_err(PanoError::Inference)?; .map_err(PanoError::Inference)
};
Ok(XFeat { Ok(XFeat {
session, landscape: session(landscape)?,
portrait: session(portrait)?,
options: DecodeOptions::default(), options: DecodeOptions::default(),
}) })
} }
/// Detect keypoints in an upright grayscale image. /// Detect keypoints in an upright grayscale image.
/// ///
/// The image is fitted into the network's fixed input — scaled down if /// The image is fitted into the network's input of matching aspect —
/// larger, never up, and padded to the right and bottom — and the /// scaled down if larger, never up, and padded to the right and bottom
/// keypoints come back in the coordinates of `image` itself, so a /// — and the keypoints come back in the coordinates of `image` itself,
/// caller that already scaled a frame to a proxy maps them on with the /// so a caller that already scaled a frame to a proxy maps them on with
/// scale it used and nothing else. /// the scale it used and nothing else.
pub fn detect(&mut self, image: &Gray) -> Result<Features, PanoError> { pub fn detect(&mut self, image: &Gray) -> Result<Features, PanoError> {
let (fitted, scale) = image.fitted(INPUT_WIDTH, INPUT_HEIGHT); let ((in_w, in_h), session) = if image.height > image.width {
let padded = fitted.padded(INPUT_WIDTH, INPUT_HEIGHT); (INPUT_PORTRAIT, &mut self.portrait)
} else {
(INPUT_LANDSCAPE, &mut self.landscape)
};
let (fitted, scale) = image.fitted(in_w, in_h);
let padded = fitted.padded(in_w, in_h);
let input = ndarray::Array::from_shape_vec( let input = ndarray::Array::from_shape_vec(ndarray::IxDyn(&[1, 1, in_h, in_w]), padded.data)
ndarray::IxDyn(&[1, 1, INPUT_HEIGHT, INPUT_WIDTH]),
padded.data,
)
.expect("shape matches the buffer by construction"); .expect("shape matches the buffer by construction");
let tensor = ort::value::Tensor::from_array(input).map_err(PanoError::Inference)?; let tensor = ort::value::Tensor::from_array(input).map_err(PanoError::Inference)?;
let outputs = self let outputs = session
.session
.run(ort::inputs![tensor]) .run(ort::inputs![tensor])
.map_err(PanoError::Inference)?; .map_err(PanoError::Inference)?;
let (w8, h8) = (INPUT_WIDTH / 8, INPUT_HEIGHT / 8); let (w8, h8) = (in_w / 8, in_h / 8);
let expect = |i: usize, channels: usize| -> Result<Vec<f32>, PanoError> { let expect = |i: usize, channels: usize| -> Result<Vec<f32>, PanoError> {
let (shape, data) = outputs[i] let (shape, data) = outputs[i]
.try_extract_tensor::<f32>() .try_extract_tensor::<f32>()
+20 -20
View File
File diff suppressed because one or more lines are too long
+6 -3
View File
@@ -76,10 +76,13 @@ rather than a code change — as this document predicted it would be.
| File | Source | Trained on | Used by | | File | Source | Trained on | Used by |
|---|---|---|---| |---|---|---|---|
| `keypoints/xfeat-1024.onnx` | `weights/xfeat.pt` from `https://github.com/verlab/accelerated_features` | MegaDepth + synthetic warps, by the authors | panorama alignment (FR-MRG-8) | | `keypoints/xfeat-1024.onnx` | `weights/xfeat.pt` from `https://github.com/verlab/accelerated_features` | MegaDepth + synthetic warps, by the authors | panorama alignment (FR-MRG-8), landscape frames |
| `keypoints/xfeat-768.onnx` | the same weights | — | the same, portrait frames |
Exported by `tools/export-xfeat.sh` at a fixed 768×1024 grayscale input. Exported by `tools/export-xfeat.sh` at fixed grayscale inputs of 1024×768
Only the convolutional network is in the file; the keypoint decoding is Rust. and 768×1024 — the same weights twice, because tract needs a static shape
and a portrait frame in a landscape input wastes half of it. Only the
convolutional network is in each file; the keypoint decoding is Rust.
**The repository and its weights are Apache-2.0**, read on 2026-09-19 from the **The repository and its weights are Apache-2.0**, read on 2026-09-19 from the
`LICENSE` at its root, with no separate grant on the checkpoint and no `LICENSE` at its root, with no separate grant on the checkpoint and no
Binary file not shown.
+16 -11
View File
@@ -6,8 +6,12 @@
# someone once produced and nobody can regenerate. Run it when bumping the # someone once produced and nobody can regenerate. Run it when bumping the
# model or changing its input size. # model or changing its input size.
# #
# ./tools/export-xfeat.sh # 768×1024 -> models/keypoints/xfeat-1024.onnx # ./tools/export-xfeat.sh # both shapes -> models/keypoints/xfeat-{1024,768}.onnx
# ./tools/export-xfeat.sh 576 768 # another fixed size #
# Two files from one set of weights: 1024 wide × 768 tall for landscape
# frames and 768 × 1024 for portrait, chosen by the frame's aspect at run
# time. One landscape file would fit a portrait frame at half the width and
# waste half the detector on padding, which is what the 6D fixture did.
# #
# Requires `uv`. Everything else is fetched into a throwaway venv, including # Requires `uv`. Everything else is fetched into a throwaway venv, including
# a CPU-only torch — the export needs no GPU and the CUDA wheels are 2 GB. # a CPU-only torch — the export needs no GPU and the CUDA wheels are 2 GB.
@@ -37,23 +41,25 @@ set -euo pipefail
HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
REPO="$(cd "${HERE}/.." && pwd)" REPO="$(cd "${HERE}/.." && pwd)"
H="${1:-768}"
W="${2:-1024}"
OUT="${REPO}/models/keypoints" OUT="${REPO}/models/keypoints"
NAME="xfeat-${W}"
# Not `mktemp -d` under /tmp: a tmpfs, and torch is a gigabyte. # Not `mktemp -d` under /tmp: a tmpfs, and torch is a gigabyte.
WORK="$(mktemp -d -p "${TMPDIR:-/var/tmp}")" WORK="$(mktemp -d -p "${TMPDIR:-/var/tmp}")"
trap 'rm -rf "${WORK}"' EXIT trap 'rm -rf "${WORK}"' EXIT
echo "==> exporting XFeat at ${H}×${W} in ${WORK}" echo "==> exporting XFeat in ${WORK}"
cd "${WORK}" cd "${WORK}"
git clone -q --depth 1 https://github.com/verlab/accelerated_features.git xfeat git clone -q --depth 1 https://github.com/verlab/accelerated_features.git xfeat
uv venv -q --python 3.12 venv uv venv -q --python 3.12 venv
VIRTUAL_ENV="${WORK}/venv" uv pip install -q --index-url https://download.pytorch.org/whl/cpu torch VIRTUAL_ENV="${WORK}/venv" uv pip install -q --index-url https://download.pytorch.org/whl/cpu torch
VIRTUAL_ENV="${WORK}/venv" uv pip install -q onnx onnxslim VIRTUAL_ENV="${WORK}/venv" uv pip install -q onnx onnxslim
VIRTUAL_ENV="${WORK}/venv" "${WORK}/venv/bin/python" - "${WORK}/xfeat" "${WORK}/${NAME}.onnx" "${H}" "${W}" <<'PY' mkdir -p "${OUT}"
for shape in "768 1024" "1024 768"; do
set -- $shape
H="$1"; W="$2"
NAME="xfeat-${W}"
VIRTUAL_ENV="${WORK}/venv" "${WORK}/venv/bin/python" - "${WORK}/xfeat" "${WORK}/${NAME}.onnx" "${H}" "${W}" <<'PY'
import sys, torch, onnx, onnxslim import sys, torch, onnx, onnxslim
sys.path.insert(0, sys.argv[1]) sys.path.insert(0, sys.argv[1])
from modules.model import XFeatModel from modules.model import XFeatModel
@@ -72,7 +78,6 @@ print("ops:", sorted({n.op_type for n in m.graph.node}))
for o in m.graph.output: for o in m.graph.output:
print("out", o.name, [d.dim_value for d in o.type.tensor_type.shape.dim]) print("out", o.name, [d.dim_value for d in o.type.tensor_type.shape.dim])
PY PY
cp "${WORK}/${NAME}.onnx" "${OUT}/${NAME}.onnx"
mkdir -p "${OUT}" echo "==> ${OUT}/${NAME}.onnx"
cp "${WORK}/${NAME}.onnx" "${OUT}/${NAME}.onnx" done
echo "==> ${OUT}/${NAME}.onnx"