dr-pano: a second XFeat shape for portrait frames, and a matcher that takes seconds
Twelve real frames from the fixture set now align in 4.5 s — 4.4 s of matching, 118 ms of bundle adjustment — where the first run took 51 s and left the first two frames out. The matcher computes each pair's similarity matrix once, across the cores, with a dot product written to vectorise; both nearest-neighbour directions read it. The frames that failed were portrait: fitted into the landscape input they used 512 of 1024 px, and their thin overlap did not survive at half resolution. The same weights are now exported at 768×1024 as well and the detector picks the shape by aspect. The example aligns from embedded previews and draws the set on a cylinder; on the fixture the sweep is 152° at a fitted 47.9 mm against the EXIF's 50, RMS 1.5 px, and the overlaps show no ghosting.
This commit is contained in:
+16
-5
@@ -8,20 +8,31 @@
|
|||||||
|
|
||||||
use std::path::Path;
|
use std::path::Path;
|
||||||
|
|
||||||
const MODEL: &str = "../../models/keypoints/xfeat-1024.onnx";
|
const MODELS: &[&str] = &[
|
||||||
|
"../../models/keypoints/xfeat-1024.onnx",
|
||||||
|
"../../models/keypoints/xfeat-768.onnx",
|
||||||
|
];
|
||||||
|
|
||||||
fn main() {
|
fn main() {
|
||||||
println!("cargo:rerun-if-changed={MODEL}");
|
for m in MODELS {
|
||||||
|
println!("cargo:rerun-if-changed={m}");
|
||||||
|
}
|
||||||
println!("cargo:rerun-if-changed=build.rs");
|
println!("cargo:rerun-if-changed=build.rs");
|
||||||
|
|
||||||
if std::env::var_os("CARGO_FEATURE_EMBEDDED_MODEL").is_none() {
|
if std::env::var_os("CARGO_FEATURE_EMBEDDED_MODEL").is_none() {
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
let path = Path::new(MODEL);
|
for model in MODELS.iter().copied() {
|
||||||
|
check(model);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn check(model: &str) {
|
||||||
|
let path = Path::new(model);
|
||||||
let Ok(bytes) = std::fs::read(path) else {
|
let Ok(bytes) = std::fs::read(path) else {
|
||||||
panic!(
|
panic!(
|
||||||
"\n\n{MODEL} is missing.\n\
|
"\n\n{model} is missing.\n\
|
||||||
It ships in Git LFS. Run `git lfs install && git lfs pull`, or build \
|
It ships in Git LFS. Run `git lfs install && git lfs pull`, or build \
|
||||||
with `--no-default-features` for a geometry-only build.\n"
|
with `--no-default-features` for a geometry-only build.\n"
|
||||||
);
|
);
|
||||||
@@ -29,7 +40,7 @@ fn main() {
|
|||||||
|
|
||||||
if bytes.starts_with(b"version https://git-lfs.github.com/spec/") {
|
if bytes.starts_with(b"version https://git-lfs.github.com/spec/") {
|
||||||
panic!(
|
panic!(
|
||||||
"\n\n{MODEL} is a Git LFS pointer, not the model.\n\
|
"\n\n{model} is a Git LFS pointer, not the model.\n\
|
||||||
Run `git lfs install && git lfs pull`, or build with \
|
Run `git lfs install && git lfs pull`, or build with \
|
||||||
`--no-default-features` for a geometry-only build.\n"
|
`--no-default-features` for a geometry-only build.\n"
|
||||||
);
|
);
|
||||||
|
|||||||
@@ -0,0 +1,172 @@
|
|||||||
|
//! Align real frames from their embedded previews and draw the result.
|
||||||
|
//!
|
||||||
|
//! ```sh
|
||||||
|
//! cargo run -p dr-pano --example align --release -- fixtures/pano/2025-08-05/*.CR2
|
||||||
|
//! cargo run -p dr-pano --example align --release -- out-prefix frame1.CR2 frame2.CR2 …
|
||||||
|
//! ```
|
||||||
|
//!
|
||||||
|
//! The point of looking rather than asserting: a rotation solve that is
|
||||||
|
//! numerically converged and geometrically wrong — a mirrored axis, a
|
||||||
|
//! transposed homography, an orientation applied the wrong way — produces
|
||||||
|
//! perfectly plausible residuals and a picture that is obviously broken.
|
||||||
|
//! This writes `<prefix>-cyl.ppm`: every frame's preview warped onto a
|
||||||
|
//! cylinder and averaged where they overlap, at a size that fits on a
|
||||||
|
//! screen. Ghosting in the overlaps is the alignment error, made visible.
|
||||||
|
//!
|
||||||
|
//! Previews, not RAW: the alignment runs on proxies in the application too
|
||||||
|
//! (FR-MRG-7), and a camera's embedded JPEG is a proxy the decoder already
|
||||||
|
//! extracts in milliseconds. What is different from the real path is only
|
||||||
|
//! that the pixels are the camera's rendering rather than ours, which the
|
||||||
|
//! geometry does not care about.
|
||||||
|
|
||||||
|
use std::path::PathBuf;
|
||||||
|
use std::time::Instant;
|
||||||
|
|
||||||
|
use dr_pano::bundle::Cameras;
|
||||||
|
use dr_pano::{align, xfeat::XFeat, AlignOptions, Gray, Projection};
|
||||||
|
|
||||||
|
fn main() {
|
||||||
|
env_logger::init();
|
||||||
|
let mut args: Vec<String> = std::env::args().skip(1).collect();
|
||||||
|
if args.is_empty() {
|
||||||
|
eprintln!("usage: align [out-prefix] <frame>...");
|
||||||
|
std::process::exit(2);
|
||||||
|
}
|
||||||
|
let prefix = if args[0].ends_with(".CR2") || args[0].ends_with(".dng") || args[0].ends_with(".jpg") {
|
||||||
|
"align".to_string()
|
||||||
|
} else {
|
||||||
|
args.remove(0)
|
||||||
|
};
|
||||||
|
let paths: Vec<PathBuf> = args.iter().map(PathBuf::from).collect();
|
||||||
|
|
||||||
|
// Previews, oriented, at proxy size.
|
||||||
|
let t = Instant::now();
|
||||||
|
let mut proxies: Vec<Gray> = Vec::new();
|
||||||
|
for p in &paths {
|
||||||
|
let bytes = std::fs::read(p).expect("read");
|
||||||
|
let preview = dr_decode::extract_preview(&bytes, dr_decode::PreviewSize::Full)
|
||||||
|
.expect("embedded preview");
|
||||||
|
let orientation = dr_decode::orientation(&bytes[..bytes.len().min(dr_decode::HEADER_BYTES as usize)])
|
||||||
|
.unwrap_or(dr_types::Orientation::NORMAL);
|
||||||
|
let tag = match orientation.quarter_turns {
|
||||||
|
1 => 6,
|
||||||
|
2 => 3,
|
||||||
|
3 => 8,
|
||||||
|
_ => 1,
|
||||||
|
};
|
||||||
|
let gray = Gray::from_rgba8(&preview.rgba, preview.width as usize, preview.height as usize)
|
||||||
|
.oriented(tag);
|
||||||
|
let (fitted, _) = gray.fitted(dr_pano::xfeat::INPUT_LONG_EDGE, dr_pano::xfeat::INPUT_LONG_EDGE);
|
||||||
|
println!(
|
||||||
|
"{:<14} preview {}×{} orientation {} → proxy {}×{}",
|
||||||
|
p.file_name().unwrap().to_string_lossy(),
|
||||||
|
preview.width,
|
||||||
|
preview.height,
|
||||||
|
tag,
|
||||||
|
fitted.width,
|
||||||
|
fitted.height
|
||||||
|
);
|
||||||
|
proxies.push(fitted);
|
||||||
|
}
|
||||||
|
println!("previews in {:?}", t.elapsed());
|
||||||
|
|
||||||
|
// Keypoints.
|
||||||
|
let t = Instant::now();
|
||||||
|
let mut detector = XFeat::embedded().expect("model");
|
||||||
|
let features: Vec<_> = proxies
|
||||||
|
.iter()
|
||||||
|
.map(|g| detector.detect(g).expect("detect"))
|
||||||
|
.collect();
|
||||||
|
for (i, f) in features.iter().enumerate() {
|
||||||
|
println!("frame {i}: {} keypoints", f.len());
|
||||||
|
}
|
||||||
|
println!("detection in {:?} ({:?} per frame)", t.elapsed(), t.elapsed() / proxies.len() as u32);
|
||||||
|
|
||||||
|
// Alignment.
|
||||||
|
let t = Instant::now();
|
||||||
|
let opts = AlignOptions::default();
|
||||||
|
let alignment = align(&features, &opts).expect("align");
|
||||||
|
println!("alignment in {:?}", t.elapsed());
|
||||||
|
println!(
|
||||||
|
"focal {:.1} px, long edge {} px ({:.1} mm on full frame), rms {:.3} px",
|
||||||
|
alignment.focal,
|
||||||
|
proxies[0].width.max(proxies[0].height),
|
||||||
|
alignment.focal * 36.0 / proxies[0].width.max(proxies[0].height) as f64,
|
||||||
|
alignment.rms_px
|
||||||
|
);
|
||||||
|
for l in &alignment.links {
|
||||||
|
println!(" link {}–{}: {} inliers of {} matches", l.i, l.j, l.inliers, l.matches);
|
||||||
|
}
|
||||||
|
for (k, why) in &alignment.unaligned {
|
||||||
|
println!(" UNALIGNED frame {k}: {why}");
|
||||||
|
}
|
||||||
|
let root = alignment
|
||||||
|
.rotations
|
||||||
|
.iter()
|
||||||
|
.position(|r| *r == Some(dr_pano::linalg::Mat3::IDENTITY))
|
||||||
|
.unwrap_or(0);
|
||||||
|
for (k, r) in alignment.rotations.iter().enumerate() {
|
||||||
|
if let Some(r) = r {
|
||||||
|
// Yaw about y, pitch about x, roll about z, from the matrix's
|
||||||
|
// columns — enough to read a sweep by eye.
|
||||||
|
let yaw = r.0[0][2].atan2(r.0[2][2]).to_degrees();
|
||||||
|
let pitch = (-r.0[1][2]).asin().to_degrees();
|
||||||
|
let roll = r.0[1][0].atan2(r.0[1][1]).to_degrees();
|
||||||
|
println!(
|
||||||
|
" frame {k}: yaw {yaw:7.2}° pitch {pitch:6.2}° roll {roll:6.2}°{}",
|
||||||
|
if k == root { " (reference)" } else { "" }
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if !alignment.is_complete() {
|
||||||
|
eprintln!("not drawing: the set is not fully aligned");
|
||||||
|
std::process::exit(1);
|
||||||
|
}
|
||||||
|
|
||||||
|
// Draw: a cylinder, averaged where frames overlap.
|
||||||
|
let t = Instant::now();
|
||||||
|
let cameras: Cameras = alignment.cameras();
|
||||||
|
let (fw, fh) = (proxies[0].width as f64, proxies[0].height as f64);
|
||||||
|
let scale = alignment.focal;
|
||||||
|
let bounds = dr_pano::projection::bounds(Projection::Cylindrical, scale, &cameras, (fw, fh))
|
||||||
|
.expect("bounds");
|
||||||
|
// Fit to 3000 px wide.
|
||||||
|
let out_w = 3000usize;
|
||||||
|
let px = bounds.width() / out_w as f64;
|
||||||
|
let out_h = (bounds.height() / px).ceil() as usize;
|
||||||
|
let mut sum = vec![0.0f32; out_w * out_h];
|
||||||
|
let mut count = vec![0u16; out_w * out_h];
|
||||||
|
for oy in 0..out_h {
|
||||||
|
for ox in 0..out_w {
|
||||||
|
let u = bounds.min_u + (ox as f64 + 0.5) * px;
|
||||||
|
let v = bounds.min_v + (oy as f64 + 0.5) * px;
|
||||||
|
let d = Projection::Cylindrical.to_direction(scale, u, v);
|
||||||
|
for (k, g) in proxies.iter().enumerate() {
|
||||||
|
let Some((x, y)) = cameras.project(k, d) else { continue };
|
||||||
|
let (x, y) = (x + g.width as f64 / 2.0, y + g.height as f64 / 2.0);
|
||||||
|
if x < 0.0 || y < 0.0 || x >= g.width as f64 - 1.0 || y >= g.height as f64 - 1.0 {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
let (x0, y0) = (x as usize, y as usize);
|
||||||
|
let (tx, ty) = ((x - x0 as f64) as f32, (y - y0 as f64) as f32);
|
||||||
|
let p = |xx: usize, yy: usize| g.data[yy * g.width + xx];
|
||||||
|
let val = (p(x0, y0) * (1.0 - tx) + p(x0 + 1, y0) * tx) * (1.0 - ty)
|
||||||
|
+ (p(x0, y0 + 1) * (1.0 - tx) + p(x0 + 1, y0 + 1) * tx) * ty;
|
||||||
|
sum[oy * out_w + ox] += val;
|
||||||
|
count[oy * out_w + ox] += 1;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
let mut ppm = format!("P5\n{out_w} {out_h}\n255\n").into_bytes();
|
||||||
|
ppm.extend(sum.iter().zip(&count).map(|(s, c)| {
|
||||||
|
if *c == 0 {
|
||||||
|
0u8
|
||||||
|
} else {
|
||||||
|
((s / f32::from(*c)).clamp(0.0, 1.0) * 255.0) as u8
|
||||||
|
}
|
||||||
|
}));
|
||||||
|
let out = format!("{prefix}-cyl.pgm");
|
||||||
|
std::fs::write(&out, ppm).expect("write");
|
||||||
|
println!("wrote {out} ({out_w}×{out_h}) in {:?}", t.elapsed());
|
||||||
|
}
|
||||||
@@ -151,9 +151,11 @@ pub fn align(frames: &[Features], opts: &AlignOptions) -> Result<Alignment, Pano
|
|||||||
let mut links = Vec::new();
|
let mut links = Vec::new();
|
||||||
let mut observations: Vec<Observation> = Vec::new();
|
let mut observations: Vec<Observation> = Vec::new();
|
||||||
let mut matched_any = vec![false; n];
|
let mut matched_any = vec![false; n];
|
||||||
|
let t_match = std::time::Instant::now();
|
||||||
for i in 0..n {
|
for i in 0..n {
|
||||||
for j in i + 1..n {
|
for j in i + 1..n {
|
||||||
let matches: Vec<Match> = match_features(&frames[i], &frames[j], opts.min_similarity);
|
let matches: Vec<Match> = match_features(&frames[i], &frames[j], opts.min_similarity);
|
||||||
|
log::debug!("pair {i}-{j}: {} matches", matches.len());
|
||||||
if matches.len() < 4 {
|
if matches.len() < 4 {
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
@@ -175,6 +177,10 @@ pub fn align(frames: &[Features], opts: &AlignOptions) -> Result<Alignment, Pano
|
|||||||
continue;
|
continue;
|
||||||
};
|
};
|
||||||
let needed = (8.0 + 0.3 * matches.len() as f64).ceil() as usize;
|
let needed = (8.0 + 0.3 * matches.len() as f64).ceil() as usize;
|
||||||
|
log::debug!(
|
||||||
|
"pair {i}-{j}: {} inliers, {needed} needed",
|
||||||
|
inliers.len()
|
||||||
|
);
|
||||||
if inliers.len() <= needed || inliers.len() < opts.min_inliers {
|
if inliers.len() <= needed || inliers.len() < opts.min_inliers {
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
@@ -204,6 +210,8 @@ pub fn align(frames: &[Features], opts: &AlignOptions) -> Result<Alignment, Pano
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
log::debug!("matching and pairwise geometry in {:?}", t_match.elapsed());
|
||||||
|
|
||||||
// 3: the focal length.
|
// 3: the focal length.
|
||||||
let mut estimates: Vec<f64> = links
|
let mut estimates: Vec<f64> = links
|
||||||
.iter()
|
.iter()
|
||||||
@@ -304,7 +312,14 @@ pub fn align(frames: &[Features], opts: &AlignOptions) -> Result<Alignment, Pano
|
|||||||
})
|
})
|
||||||
})
|
})
|
||||||
.collect();
|
.collect();
|
||||||
|
let t_adjust = std::time::Instant::now();
|
||||||
let adjusted = bundle::adjust(start, &obs, &opts.adjust)?;
|
let adjusted = bundle::adjust(start, &obs, &opts.adjust)?;
|
||||||
|
log::debug!(
|
||||||
|
"bundle adjustment: {} observations, {} iterations in {:?}",
|
||||||
|
obs.len(),
|
||||||
|
adjusted.iterations,
|
||||||
|
t_adjust.elapsed()
|
||||||
|
);
|
||||||
for (slot, &k) in aligned.iter().enumerate() {
|
for (slot, &k) in aligned.iter().enumerate() {
|
||||||
rotations[k] = Some(adjusted.cameras.rotations[slot]);
|
rotations[k] = Some(adjusted.cameras.rotations[slot]);
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -9,12 +9,15 @@
|
|||||||
//! cleverer matcher. A learned matcher (LightGlue) is the step after this
|
//! cleverer matcher. A learned matcher (LightGlue) is the step after this
|
||||||
//! one fails on a real set, and it has not (panorama.md §6).
|
//! one fails on a real set, and it has not (panorama.md §6).
|
||||||
//!
|
//!
|
||||||
//! Brute force. `4096 × 4096 × 64` multiply-adds is a billion, which is
|
//! Brute force. `4096 × 4096 × 64` multiply-adds is a billion per pair,
|
||||||
//! tens of milliseconds a pair on one core, and there are at most a few
|
//! and a twelve-frame set has sixty-six pairs: a minute single-threaded
|
||||||
//! dozen pairs. Not worth an index.
|
//! and scalar (measured 2026-09-19: 51 s), a few seconds vectorised across
|
||||||
|
//! the cores. Not worth an index, but worth doing properly.
|
||||||
|
|
||||||
use crate::features::{Features, DESCRIPTOR_LEN};
|
use crate::features::{Features, DESCRIPTOR_LEN};
|
||||||
|
|
||||||
|
const _: () = assert!(DESCRIPTOR_LEN % 8 == 0);
|
||||||
|
|
||||||
/// A correspondence: keypoint `a` in the first image matches keypoint `b`
|
/// A correspondence: keypoint `a` in the first image matches keypoint `b`
|
||||||
/// in the second, with the cosine similarity of their descriptors.
|
/// in the second, with the cosine similarity of their descriptors.
|
||||||
#[derive(Debug, Clone, Copy, PartialEq)]
|
#[derive(Debug, Clone, Copy, PartialEq)]
|
||||||
@@ -32,48 +35,75 @@ pub fn match_features(a: &Features, b: &Features, min_similarity: f32) -> Vec<Ma
|
|||||||
if a.is_empty() || b.is_empty() {
|
if a.is_empty() || b.is_empty() {
|
||||||
return Vec::new();
|
return Vec::new();
|
||||||
}
|
}
|
||||||
let best_ab = nearest(a, b);
|
let (na, nb) = (a.len(), b.len());
|
||||||
let best_ba = nearest(b, a);
|
|
||||||
|
// The whole similarity matrix, once. Both nearest-neighbour directions
|
||||||
|
// read it, which halves the multiply-adds against computing each
|
||||||
|
// direction on its own; 4096 × 4096 × f32 is 64 MB, transient.
|
||||||
|
let mut sim = vec![0.0f32; na * nb];
|
||||||
|
let threads = std::thread::available_parallelism()
|
||||||
|
.map(usize::from)
|
||||||
|
.unwrap_or(1)
|
||||||
|
.clamp(1, 16);
|
||||||
|
let rows_per = na.div_ceil(threads);
|
||||||
|
std::thread::scope(|scope| {
|
||||||
|
for (t, chunk) in sim.chunks_mut(rows_per * nb).enumerate() {
|
||||||
|
scope.spawn(move || {
|
||||||
|
let first = t * rows_per;
|
||||||
|
for (r, row) in chunk.chunks_mut(nb).enumerate() {
|
||||||
|
let da = a.descriptor(first + r);
|
||||||
|
for (j, cell) in row.iter_mut().enumerate() {
|
||||||
|
*cell = dot(da, b.descriptor(j));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
});
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
// Best in `b` for each `a`, and best in `a` for each `b`.
|
||||||
|
let best_ab: Vec<(usize, f32)> = sim
|
||||||
|
.chunks_exact(nb)
|
||||||
|
.map(|row| {
|
||||||
|
row.iter()
|
||||||
|
.enumerate()
|
||||||
|
.fold((0usize, f32::MIN), |acc, (j, &s)| if s > acc.1 { (j, s) } else { acc })
|
||||||
|
})
|
||||||
|
.collect();
|
||||||
|
let mut best_ba = vec![(0usize, f32::MIN); nb];
|
||||||
|
for (i, row) in sim.chunks_exact(nb).enumerate() {
|
||||||
|
for (j, &s) in row.iter().enumerate() {
|
||||||
|
if s > best_ba[j].1 {
|
||||||
|
best_ba[j] = (i, s);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
best_ab
|
best_ab
|
||||||
.iter()
|
.iter()
|
||||||
.enumerate()
|
.enumerate()
|
||||||
.filter_map(|(ia, &(ib, sim))| {
|
.filter_map(|(ia, &(ib, s))| {
|
||||||
(best_ba[ib].0 == ia && sim >= min_similarity).then_some(Match {
|
(best_ba[ib].0 == ia && s >= min_similarity).then_some(Match {
|
||||||
a: ia,
|
a: ia,
|
||||||
b: ib,
|
b: ib,
|
||||||
similarity: sim,
|
similarity: s,
|
||||||
})
|
})
|
||||||
})
|
})
|
||||||
.collect()
|
.collect()
|
||||||
}
|
}
|
||||||
|
|
||||||
/// For each descriptor in `from`, the index of its nearest in `to` and the
|
|
||||||
/// similarity.
|
|
||||||
fn nearest(from: &Features, to: &Features) -> Vec<(usize, f32)> {
|
|
||||||
(0..from.len())
|
|
||||||
.map(|i| {
|
|
||||||
let d = from.descriptor(i);
|
|
||||||
let mut best = (0usize, f32::MIN);
|
|
||||||
for j in 0..to.len() {
|
|
||||||
let s = dot(d, to.descriptor(j));
|
|
||||||
if s > best.1 {
|
|
||||||
best = (j, s);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
best
|
|
||||||
})
|
|
||||||
.collect()
|
|
||||||
}
|
|
||||||
|
|
||||||
#[inline]
|
#[inline]
|
||||||
fn dot(a: &[f32], b: &[f32]) -> f32 {
|
fn dot(a: &[f32], b: &[f32]) -> f32 {
|
||||||
// Written as a plain loop over a fixed length so the compiler
|
// Eight independent accumulators over exact 8-lane chunks: the shape
|
||||||
// vectorises it; the length is a constant and the slices are exact.
|
// the compiler turns into one vector multiply-add per chunk, and no
|
||||||
let mut s = 0.0f32;
|
// bounds checks inside the loop. `DESCRIPTOR_LEN` is a multiple of 8.
|
||||||
for k in 0..DESCRIPTOR_LEN {
|
let (a, b) = (&a[..DESCRIPTOR_LEN], &b[..DESCRIPTOR_LEN]);
|
||||||
s += a[k] * b[k];
|
let mut acc = [0.0f32; 8];
|
||||||
|
for (ca, cb) in a.chunks_exact(8).zip(b.chunks_exact(8)) {
|
||||||
|
for k in 0..8 {
|
||||||
|
acc[k] += ca[k] * cb[k];
|
||||||
}
|
}
|
||||||
s
|
}
|
||||||
|
acc.iter().sum()
|
||||||
}
|
}
|
||||||
|
|
||||||
#[cfg(test)]
|
#[cfg(test)]
|
||||||
|
|||||||
+45
-29
@@ -11,17 +11,27 @@ use crate::features::{decode_xfeat, DecodeOptions, Features, XFeatMaps, DESCRIPT
|
|||||||
use crate::image::Gray;
|
use crate::image::Gray;
|
||||||
use crate::PanoError;
|
use crate::PanoError;
|
||||||
|
|
||||||
/// The input shape the shipped export was made for. A different size is a
|
/// The two input shapes the shipped exports were made for: one landscape,
|
||||||
/// different file (`tools/export-xfeat.sh`).
|
/// one portrait, the same weights. A frame is fitted into whichever
|
||||||
pub const INPUT_WIDTH: usize = 1024;
|
/// matches its aspect, so a portrait set does not spend half the
|
||||||
pub const INPUT_HEIGHT: usize = 768;
|
/// detector's width on padding — which is what the 6D fixture did before
|
||||||
|
/// the second export existed (512 × 768 of a 1024 × 768 input). A
|
||||||
|
/// different size is a different file (`tools/export-xfeat.sh`).
|
||||||
|
pub const INPUT_LANDSCAPE: (usize, usize) = (1024, 768);
|
||||||
|
pub const INPUT_PORTRAIT: (usize, usize) = (768, 1024);
|
||||||
|
|
||||||
|
/// The long edge of the detector's input, for callers sizing a proxy.
|
||||||
|
pub const INPUT_LONG_EDGE: usize = 1024;
|
||||||
|
|
||||||
#[cfg(feature = "embedded-model")]
|
#[cfg(feature = "embedded-model")]
|
||||||
const EMBEDDED_MODEL: &[u8] = include_bytes!("../../../models/keypoints/xfeat-1024.onnx");
|
const EMBEDDED_LANDSCAPE: &[u8] = include_bytes!("../../../models/keypoints/xfeat-1024.onnx");
|
||||||
|
#[cfg(feature = "embedded-model")]
|
||||||
|
const EMBEDDED_PORTRAIT: &[u8] = include_bytes!("../../../models/keypoints/xfeat-768.onnx");
|
||||||
|
|
||||||
/// A loaded detector.
|
/// A loaded detector: the network at both shapes.
|
||||||
pub struct XFeat {
|
pub struct XFeat {
|
||||||
session: ort::session::Session,
|
landscape: ort::session::Session,
|
||||||
|
portrait: ort::session::Session,
|
||||||
pub options: DecodeOptions,
|
pub options: DecodeOptions,
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -29,49 +39,55 @@ impl XFeat {
|
|||||||
/// The weights compiled into the binary.
|
/// The weights compiled into the binary.
|
||||||
#[cfg(feature = "embedded-model")]
|
#[cfg(feature = "embedded-model")]
|
||||||
pub fn embedded() -> Result<Self, PanoError> {
|
pub fn embedded() -> Result<Self, PanoError> {
|
||||||
Self::from_bytes(EMBEDDED_MODEL)
|
Self::from_bytes(EMBEDDED_LANDSCAPE, EMBEDDED_PORTRAIT)
|
||||||
}
|
}
|
||||||
|
|
||||||
pub fn from_path(path: &std::path::Path) -> Result<Self, PanoError> {
|
/// From the two exports on disk.
|
||||||
let bytes = std::fs::read(path).map_err(PanoError::ModelRead)?;
|
pub fn from_paths(landscape: &std::path::Path, portrait: &std::path::Path) -> Result<Self, PanoError> {
|
||||||
Self::from_bytes(&bytes)
|
let l = std::fs::read(landscape).map_err(PanoError::ModelRead)?;
|
||||||
|
let p = std::fs::read(portrait).map_err(PanoError::ModelRead)?;
|
||||||
|
Self::from_bytes(&l, &p)
|
||||||
}
|
}
|
||||||
|
|
||||||
pub fn from_bytes(bytes: &[u8]) -> Result<Self, PanoError> {
|
pub fn from_bytes(landscape: &[u8], portrait: &[u8]) -> Result<Self, PanoError> {
|
||||||
install_backend();
|
install_backend();
|
||||||
let session = ort::session::Session::builder()
|
let session = |bytes: &[u8]| {
|
||||||
|
ort::session::Session::builder()
|
||||||
.map_err(PanoError::Inference)?
|
.map_err(PanoError::Inference)?
|
||||||
.commit_from_memory(bytes)
|
.commit_from_memory(bytes)
|
||||||
.map_err(PanoError::Inference)?;
|
.map_err(PanoError::Inference)
|
||||||
|
};
|
||||||
Ok(XFeat {
|
Ok(XFeat {
|
||||||
session,
|
landscape: session(landscape)?,
|
||||||
|
portrait: session(portrait)?,
|
||||||
options: DecodeOptions::default(),
|
options: DecodeOptions::default(),
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Detect keypoints in an upright grayscale image.
|
/// Detect keypoints in an upright grayscale image.
|
||||||
///
|
///
|
||||||
/// The image is fitted into the network's fixed input — scaled down if
|
/// The image is fitted into the network's input of matching aspect —
|
||||||
/// larger, never up, and padded to the right and bottom — and the
|
/// scaled down if larger, never up, and padded to the right and bottom
|
||||||
/// keypoints come back in the coordinates of `image` itself, so a
|
/// — and the keypoints come back in the coordinates of `image` itself,
|
||||||
/// caller that already scaled a frame to a proxy maps them on with the
|
/// so a caller that already scaled a frame to a proxy maps them on with
|
||||||
/// scale it used and nothing else.
|
/// the scale it used and nothing else.
|
||||||
pub fn detect(&mut self, image: &Gray) -> Result<Features, PanoError> {
|
pub fn detect(&mut self, image: &Gray) -> Result<Features, PanoError> {
|
||||||
let (fitted, scale) = image.fitted(INPUT_WIDTH, INPUT_HEIGHT);
|
let ((in_w, in_h), session) = if image.height > image.width {
|
||||||
let padded = fitted.padded(INPUT_WIDTH, INPUT_HEIGHT);
|
(INPUT_PORTRAIT, &mut self.portrait)
|
||||||
|
} else {
|
||||||
|
(INPUT_LANDSCAPE, &mut self.landscape)
|
||||||
|
};
|
||||||
|
let (fitted, scale) = image.fitted(in_w, in_h);
|
||||||
|
let padded = fitted.padded(in_w, in_h);
|
||||||
|
|
||||||
let input = ndarray::Array::from_shape_vec(
|
let input = ndarray::Array::from_shape_vec(ndarray::IxDyn(&[1, 1, in_h, in_w]), padded.data)
|
||||||
ndarray::IxDyn(&[1, 1, INPUT_HEIGHT, INPUT_WIDTH]),
|
|
||||||
padded.data,
|
|
||||||
)
|
|
||||||
.expect("shape matches the buffer by construction");
|
.expect("shape matches the buffer by construction");
|
||||||
let tensor = ort::value::Tensor::from_array(input).map_err(PanoError::Inference)?;
|
let tensor = ort::value::Tensor::from_array(input).map_err(PanoError::Inference)?;
|
||||||
let outputs = self
|
let outputs = session
|
||||||
.session
|
|
||||||
.run(ort::inputs![tensor])
|
.run(ort::inputs![tensor])
|
||||||
.map_err(PanoError::Inference)?;
|
.map_err(PanoError::Inference)?;
|
||||||
|
|
||||||
let (w8, h8) = (INPUT_WIDTH / 8, INPUT_HEIGHT / 8);
|
let (w8, h8) = (in_w / 8, in_h / 8);
|
||||||
let expect = |i: usize, channels: usize| -> Result<Vec<f32>, PanoError> {
|
let expect = |i: usize, channels: usize| -> Result<Vec<f32>, PanoError> {
|
||||||
let (shape, data) = outputs[i]
|
let (shape, data) = outputs[i]
|
||||||
.try_extract_tensor::<f32>()
|
.try_extract_tensor::<f32>()
|
||||||
|
|||||||
+20
-20
File diff suppressed because one or more lines are too long
+6
-3
@@ -76,10 +76,13 @@ rather than a code change — as this document predicted it would be.
|
|||||||
|
|
||||||
| File | Source | Trained on | Used by |
|
| File | Source | Trained on | Used by |
|
||||||
|---|---|---|---|
|
|---|---|---|---|
|
||||||
| `keypoints/xfeat-1024.onnx` | `weights/xfeat.pt` from `https://github.com/verlab/accelerated_features` | MegaDepth + synthetic warps, by the authors | panorama alignment (FR-MRG-8) |
|
| `keypoints/xfeat-1024.onnx` | `weights/xfeat.pt` from `https://github.com/verlab/accelerated_features` | MegaDepth + synthetic warps, by the authors | panorama alignment (FR-MRG-8), landscape frames |
|
||||||
|
| `keypoints/xfeat-768.onnx` | the same weights | — | the same, portrait frames |
|
||||||
|
|
||||||
Exported by `tools/export-xfeat.sh` at a fixed 768×1024 grayscale input.
|
Exported by `tools/export-xfeat.sh` at fixed grayscale inputs of 1024×768
|
||||||
Only the convolutional network is in the file; the keypoint decoding is Rust.
|
and 768×1024 — the same weights twice, because tract needs a static shape
|
||||||
|
and a portrait frame in a landscape input wastes half of it. Only the
|
||||||
|
convolutional network is in each file; the keypoint decoding is Rust.
|
||||||
|
|
||||||
**The repository and its weights are Apache-2.0**, read on 2026-09-19 from the
|
**The repository and its weights are Apache-2.0**, read on 2026-09-19 from the
|
||||||
`LICENSE` at its root, with no separate grant on the checkpoint and no
|
`LICENSE` at its root, with no separate grant on the checkpoint and no
|
||||||
|
|||||||
Binary file not shown.
+13
-8
@@ -6,8 +6,12 @@
|
|||||||
# someone once produced and nobody can regenerate. Run it when bumping the
|
# someone once produced and nobody can regenerate. Run it when bumping the
|
||||||
# model or changing its input size.
|
# model or changing its input size.
|
||||||
#
|
#
|
||||||
# ./tools/export-xfeat.sh # 768×1024 -> models/keypoints/xfeat-1024.onnx
|
# ./tools/export-xfeat.sh # both shapes -> models/keypoints/xfeat-{1024,768}.onnx
|
||||||
# ./tools/export-xfeat.sh 576 768 # another fixed size
|
#
|
||||||
|
# Two files from one set of weights: 1024 wide × 768 tall for landscape
|
||||||
|
# frames and 768 × 1024 for portrait, chosen by the frame's aspect at run
|
||||||
|
# time. One landscape file would fit a portrait frame at half the width and
|
||||||
|
# waste half the detector on padding, which is what the 6D fixture did.
|
||||||
#
|
#
|
||||||
# Requires `uv`. Everything else is fetched into a throwaway venv, including
|
# Requires `uv`. Everything else is fetched into a throwaway venv, including
|
||||||
# a CPU-only torch — the export needs no GPU and the CUDA wheels are 2 GB.
|
# a CPU-only torch — the export needs no GPU and the CUDA wheels are 2 GB.
|
||||||
@@ -37,22 +41,24 @@ set -euo pipefail
|
|||||||
HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||||
REPO="$(cd "${HERE}/.." && pwd)"
|
REPO="$(cd "${HERE}/.." && pwd)"
|
||||||
|
|
||||||
H="${1:-768}"
|
|
||||||
W="${2:-1024}"
|
|
||||||
OUT="${REPO}/models/keypoints"
|
OUT="${REPO}/models/keypoints"
|
||||||
NAME="xfeat-${W}"
|
|
||||||
|
|
||||||
# Not `mktemp -d` under /tmp: a tmpfs, and torch is a gigabyte.
|
# Not `mktemp -d` under /tmp: a tmpfs, and torch is a gigabyte.
|
||||||
WORK="$(mktemp -d -p "${TMPDIR:-/var/tmp}")"
|
WORK="$(mktemp -d -p "${TMPDIR:-/var/tmp}")"
|
||||||
trap 'rm -rf "${WORK}"' EXIT
|
trap 'rm -rf "${WORK}"' EXIT
|
||||||
|
|
||||||
echo "==> exporting XFeat at ${H}×${W} in ${WORK}"
|
echo "==> exporting XFeat in ${WORK}"
|
||||||
cd "${WORK}"
|
cd "${WORK}"
|
||||||
git clone -q --depth 1 https://github.com/verlab/accelerated_features.git xfeat
|
git clone -q --depth 1 https://github.com/verlab/accelerated_features.git xfeat
|
||||||
uv venv -q --python 3.12 venv
|
uv venv -q --python 3.12 venv
|
||||||
VIRTUAL_ENV="${WORK}/venv" uv pip install -q --index-url https://download.pytorch.org/whl/cpu torch
|
VIRTUAL_ENV="${WORK}/venv" uv pip install -q --index-url https://download.pytorch.org/whl/cpu torch
|
||||||
VIRTUAL_ENV="${WORK}/venv" uv pip install -q onnx onnxslim
|
VIRTUAL_ENV="${WORK}/venv" uv pip install -q onnx onnxslim
|
||||||
|
|
||||||
|
mkdir -p "${OUT}"
|
||||||
|
for shape in "768 1024" "1024 768"; do
|
||||||
|
set -- $shape
|
||||||
|
H="$1"; W="$2"
|
||||||
|
NAME="xfeat-${W}"
|
||||||
VIRTUAL_ENV="${WORK}/venv" "${WORK}/venv/bin/python" - "${WORK}/xfeat" "${WORK}/${NAME}.onnx" "${H}" "${W}" <<'PY'
|
VIRTUAL_ENV="${WORK}/venv" "${WORK}/venv/bin/python" - "${WORK}/xfeat" "${WORK}/${NAME}.onnx" "${H}" "${W}" <<'PY'
|
||||||
import sys, torch, onnx, onnxslim
|
import sys, torch, onnx, onnxslim
|
||||||
sys.path.insert(0, sys.argv[1])
|
sys.path.insert(0, sys.argv[1])
|
||||||
@@ -72,7 +78,6 @@ print("ops:", sorted({n.op_type for n in m.graph.node}))
|
|||||||
for o in m.graph.output:
|
for o in m.graph.output:
|
||||||
print("out", o.name, [d.dim_value for d in o.type.tensor_type.shape.dim])
|
print("out", o.name, [d.dim_value for d in o.type.tensor_type.shape.dim])
|
||||||
PY
|
PY
|
||||||
|
|
||||||
mkdir -p "${OUT}"
|
|
||||||
cp "${WORK}/${NAME}.onnx" "${OUT}/${NAME}.onnx"
|
cp "${WORK}/${NAME}.onnx" "${OUT}/${NAME}.onnx"
|
||||||
echo "==> ${OUT}/${NAME}.onnx"
|
echo "==> ${OUT}/${NAME}.onnx"
|
||||||
|
done
|
||||||
|
|||||||
Reference in New Issue
Block a user