dr-pano: a second XFeat shape for portrait frames, and a matcher that takes seconds
Twelve real frames from the fixture set now align in 4.5 s — 4.4 s of matching, 118 ms of bundle adjustment — where the first run took 51 s and left the first two frames out. The matcher computes each pair's similarity matrix once, across the cores, with a dot product written to vectorise; both nearest-neighbour directions read it. The frames that failed were portrait: fitted into the landscape input they used 512 of 1024 px, and their thin overlap did not survive at half resolution. The same weights are now exported at 768×1024 as well and the detector picks the shape by aspect. The example aligns from embedded previews and draws the set on a cylinder; on the fixture the sweep is 152° at a fitted 47.9 mm against the EXIF's 50, RMS 1.5 px, and the overlaps show no ghosting.
This commit is contained in:
+16
-5
@@ -8,20 +8,31 @@
|
||||
|
||||
use std::path::Path;
|
||||
|
||||
const MODEL: &str = "../../models/keypoints/xfeat-1024.onnx";
|
||||
const MODELS: &[&str] = &[
|
||||
"../../models/keypoints/xfeat-1024.onnx",
|
||||
"../../models/keypoints/xfeat-768.onnx",
|
||||
];
|
||||
|
||||
fn main() {
|
||||
println!("cargo:rerun-if-changed={MODEL}");
|
||||
for m in MODELS {
|
||||
println!("cargo:rerun-if-changed={m}");
|
||||
}
|
||||
println!("cargo:rerun-if-changed=build.rs");
|
||||
|
||||
if std::env::var_os("CARGO_FEATURE_EMBEDDED_MODEL").is_none() {
|
||||
return;
|
||||
}
|
||||
|
||||
let path = Path::new(MODEL);
|
||||
for model in MODELS.iter().copied() {
|
||||
check(model);
|
||||
}
|
||||
}
|
||||
|
||||
fn check(model: &str) {
|
||||
let path = Path::new(model);
|
||||
let Ok(bytes) = std::fs::read(path) else {
|
||||
panic!(
|
||||
"\n\n{MODEL} is missing.\n\
|
||||
"\n\n{model} is missing.\n\
|
||||
It ships in Git LFS. Run `git lfs install && git lfs pull`, or build \
|
||||
with `--no-default-features` for a geometry-only build.\n"
|
||||
);
|
||||
@@ -29,7 +40,7 @@ fn main() {
|
||||
|
||||
if bytes.starts_with(b"version https://git-lfs.github.com/spec/") {
|
||||
panic!(
|
||||
"\n\n{MODEL} is a Git LFS pointer, not the model.\n\
|
||||
"\n\n{model} is a Git LFS pointer, not the model.\n\
|
||||
Run `git lfs install && git lfs pull`, or build with \
|
||||
`--no-default-features` for a geometry-only build.\n"
|
||||
);
|
||||
|
||||
@@ -0,0 +1,172 @@
|
||||
//! Align real frames from their embedded previews and draw the result.
|
||||
//!
|
||||
//! ```sh
|
||||
//! cargo run -p dr-pano --example align --release -- fixtures/pano/2025-08-05/*.CR2
|
||||
//! cargo run -p dr-pano --example align --release -- out-prefix frame1.CR2 frame2.CR2 …
|
||||
//! ```
|
||||
//!
|
||||
//! The point of looking rather than asserting: a rotation solve that is
|
||||
//! numerically converged and geometrically wrong — a mirrored axis, a
|
||||
//! transposed homography, an orientation applied the wrong way — produces
|
||||
//! perfectly plausible residuals and a picture that is obviously broken.
|
||||
//! This writes `<prefix>-cyl.ppm`: every frame's preview warped onto a
|
||||
//! cylinder and averaged where they overlap, at a size that fits on a
|
||||
//! screen. Ghosting in the overlaps is the alignment error, made visible.
|
||||
//!
|
||||
//! Previews, not RAW: the alignment runs on proxies in the application too
|
||||
//! (FR-MRG-7), and a camera's embedded JPEG is a proxy the decoder already
|
||||
//! extracts in milliseconds. What is different from the real path is only
|
||||
//! that the pixels are the camera's rendering rather than ours, which the
|
||||
//! geometry does not care about.
|
||||
|
||||
use std::path::PathBuf;
|
||||
use std::time::Instant;
|
||||
|
||||
use dr_pano::bundle::Cameras;
|
||||
use dr_pano::{align, xfeat::XFeat, AlignOptions, Gray, Projection};
|
||||
|
||||
fn main() {
|
||||
env_logger::init();
|
||||
let mut args: Vec<String> = std::env::args().skip(1).collect();
|
||||
if args.is_empty() {
|
||||
eprintln!("usage: align [out-prefix] <frame>...");
|
||||
std::process::exit(2);
|
||||
}
|
||||
let prefix = if args[0].ends_with(".CR2") || args[0].ends_with(".dng") || args[0].ends_with(".jpg") {
|
||||
"align".to_string()
|
||||
} else {
|
||||
args.remove(0)
|
||||
};
|
||||
let paths: Vec<PathBuf> = args.iter().map(PathBuf::from).collect();
|
||||
|
||||
// Previews, oriented, at proxy size.
|
||||
let t = Instant::now();
|
||||
let mut proxies: Vec<Gray> = Vec::new();
|
||||
for p in &paths {
|
||||
let bytes = std::fs::read(p).expect("read");
|
||||
let preview = dr_decode::extract_preview(&bytes, dr_decode::PreviewSize::Full)
|
||||
.expect("embedded preview");
|
||||
let orientation = dr_decode::orientation(&bytes[..bytes.len().min(dr_decode::HEADER_BYTES as usize)])
|
||||
.unwrap_or(dr_types::Orientation::NORMAL);
|
||||
let tag = match orientation.quarter_turns {
|
||||
1 => 6,
|
||||
2 => 3,
|
||||
3 => 8,
|
||||
_ => 1,
|
||||
};
|
||||
let gray = Gray::from_rgba8(&preview.rgba, preview.width as usize, preview.height as usize)
|
||||
.oriented(tag);
|
||||
let (fitted, _) = gray.fitted(dr_pano::xfeat::INPUT_LONG_EDGE, dr_pano::xfeat::INPUT_LONG_EDGE);
|
||||
println!(
|
||||
"{:<14} preview {}×{} orientation {} → proxy {}×{}",
|
||||
p.file_name().unwrap().to_string_lossy(),
|
||||
preview.width,
|
||||
preview.height,
|
||||
tag,
|
||||
fitted.width,
|
||||
fitted.height
|
||||
);
|
||||
proxies.push(fitted);
|
||||
}
|
||||
println!("previews in {:?}", t.elapsed());
|
||||
|
||||
// Keypoints.
|
||||
let t = Instant::now();
|
||||
let mut detector = XFeat::embedded().expect("model");
|
||||
let features: Vec<_> = proxies
|
||||
.iter()
|
||||
.map(|g| detector.detect(g).expect("detect"))
|
||||
.collect();
|
||||
for (i, f) in features.iter().enumerate() {
|
||||
println!("frame {i}: {} keypoints", f.len());
|
||||
}
|
||||
println!("detection in {:?} ({:?} per frame)", t.elapsed(), t.elapsed() / proxies.len() as u32);
|
||||
|
||||
// Alignment.
|
||||
let t = Instant::now();
|
||||
let opts = AlignOptions::default();
|
||||
let alignment = align(&features, &opts).expect("align");
|
||||
println!("alignment in {:?}", t.elapsed());
|
||||
println!(
|
||||
"focal {:.1} px, long edge {} px ({:.1} mm on full frame), rms {:.3} px",
|
||||
alignment.focal,
|
||||
proxies[0].width.max(proxies[0].height),
|
||||
alignment.focal * 36.0 / proxies[0].width.max(proxies[0].height) as f64,
|
||||
alignment.rms_px
|
||||
);
|
||||
for l in &alignment.links {
|
||||
println!(" link {}–{}: {} inliers of {} matches", l.i, l.j, l.inliers, l.matches);
|
||||
}
|
||||
for (k, why) in &alignment.unaligned {
|
||||
println!(" UNALIGNED frame {k}: {why}");
|
||||
}
|
||||
let root = alignment
|
||||
.rotations
|
||||
.iter()
|
||||
.position(|r| *r == Some(dr_pano::linalg::Mat3::IDENTITY))
|
||||
.unwrap_or(0);
|
||||
for (k, r) in alignment.rotations.iter().enumerate() {
|
||||
if let Some(r) = r {
|
||||
// Yaw about y, pitch about x, roll about z, from the matrix's
|
||||
// columns — enough to read a sweep by eye.
|
||||
let yaw = r.0[0][2].atan2(r.0[2][2]).to_degrees();
|
||||
let pitch = (-r.0[1][2]).asin().to_degrees();
|
||||
let roll = r.0[1][0].atan2(r.0[1][1]).to_degrees();
|
||||
println!(
|
||||
" frame {k}: yaw {yaw:7.2}° pitch {pitch:6.2}° roll {roll:6.2}°{}",
|
||||
if k == root { " (reference)" } else { "" }
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
if !alignment.is_complete() {
|
||||
eprintln!("not drawing: the set is not fully aligned");
|
||||
std::process::exit(1);
|
||||
}
|
||||
|
||||
// Draw: a cylinder, averaged where frames overlap.
|
||||
let t = Instant::now();
|
||||
let cameras: Cameras = alignment.cameras();
|
||||
let (fw, fh) = (proxies[0].width as f64, proxies[0].height as f64);
|
||||
let scale = alignment.focal;
|
||||
let bounds = dr_pano::projection::bounds(Projection::Cylindrical, scale, &cameras, (fw, fh))
|
||||
.expect("bounds");
|
||||
// Fit to 3000 px wide.
|
||||
let out_w = 3000usize;
|
||||
let px = bounds.width() / out_w as f64;
|
||||
let out_h = (bounds.height() / px).ceil() as usize;
|
||||
let mut sum = vec![0.0f32; out_w * out_h];
|
||||
let mut count = vec![0u16; out_w * out_h];
|
||||
for oy in 0..out_h {
|
||||
for ox in 0..out_w {
|
||||
let u = bounds.min_u + (ox as f64 + 0.5) * px;
|
||||
let v = bounds.min_v + (oy as f64 + 0.5) * px;
|
||||
let d = Projection::Cylindrical.to_direction(scale, u, v);
|
||||
for (k, g) in proxies.iter().enumerate() {
|
||||
let Some((x, y)) = cameras.project(k, d) else { continue };
|
||||
let (x, y) = (x + g.width as f64 / 2.0, y + g.height as f64 / 2.0);
|
||||
if x < 0.0 || y < 0.0 || x >= g.width as f64 - 1.0 || y >= g.height as f64 - 1.0 {
|
||||
continue;
|
||||
}
|
||||
let (x0, y0) = (x as usize, y as usize);
|
||||
let (tx, ty) = ((x - x0 as f64) as f32, (y - y0 as f64) as f32);
|
||||
let p = |xx: usize, yy: usize| g.data[yy * g.width + xx];
|
||||
let val = (p(x0, y0) * (1.0 - tx) + p(x0 + 1, y0) * tx) * (1.0 - ty)
|
||||
+ (p(x0, y0 + 1) * (1.0 - tx) + p(x0 + 1, y0 + 1) * tx) * ty;
|
||||
sum[oy * out_w + ox] += val;
|
||||
count[oy * out_w + ox] += 1;
|
||||
}
|
||||
}
|
||||
}
|
||||
let mut ppm = format!("P5\n{out_w} {out_h}\n255\n").into_bytes();
|
||||
ppm.extend(sum.iter().zip(&count).map(|(s, c)| {
|
||||
if *c == 0 {
|
||||
0u8
|
||||
} else {
|
||||
((s / f32::from(*c)).clamp(0.0, 1.0) * 255.0) as u8
|
||||
}
|
||||
}));
|
||||
let out = format!("{prefix}-cyl.pgm");
|
||||
std::fs::write(&out, ppm).expect("write");
|
||||
println!("wrote {out} ({out_w}×{out_h}) in {:?}", t.elapsed());
|
||||
}
|
||||
@@ -151,9 +151,11 @@ pub fn align(frames: &[Features], opts: &AlignOptions) -> Result<Alignment, Pano
|
||||
let mut links = Vec::new();
|
||||
let mut observations: Vec<Observation> = Vec::new();
|
||||
let mut matched_any = vec![false; n];
|
||||
let t_match = std::time::Instant::now();
|
||||
for i in 0..n {
|
||||
for j in i + 1..n {
|
||||
let matches: Vec<Match> = match_features(&frames[i], &frames[j], opts.min_similarity);
|
||||
log::debug!("pair {i}-{j}: {} matches", matches.len());
|
||||
if matches.len() < 4 {
|
||||
continue;
|
||||
}
|
||||
@@ -175,6 +177,10 @@ pub fn align(frames: &[Features], opts: &AlignOptions) -> Result<Alignment, Pano
|
||||
continue;
|
||||
};
|
||||
let needed = (8.0 + 0.3 * matches.len() as f64).ceil() as usize;
|
||||
log::debug!(
|
||||
"pair {i}-{j}: {} inliers, {needed} needed",
|
||||
inliers.len()
|
||||
);
|
||||
if inliers.len() <= needed || inliers.len() < opts.min_inliers {
|
||||
continue;
|
||||
}
|
||||
@@ -204,6 +210,8 @@ pub fn align(frames: &[Features], opts: &AlignOptions) -> Result<Alignment, Pano
|
||||
}
|
||||
}
|
||||
|
||||
log::debug!("matching and pairwise geometry in {:?}", t_match.elapsed());
|
||||
|
||||
// 3: the focal length.
|
||||
let mut estimates: Vec<f64> = links
|
||||
.iter()
|
||||
@@ -304,7 +312,14 @@ pub fn align(frames: &[Features], opts: &AlignOptions) -> Result<Alignment, Pano
|
||||
})
|
||||
})
|
||||
.collect();
|
||||
let t_adjust = std::time::Instant::now();
|
||||
let adjusted = bundle::adjust(start, &obs, &opts.adjust)?;
|
||||
log::debug!(
|
||||
"bundle adjustment: {} observations, {} iterations in {:?}",
|
||||
obs.len(),
|
||||
adjusted.iterations,
|
||||
t_adjust.elapsed()
|
||||
);
|
||||
for (slot, &k) in aligned.iter().enumerate() {
|
||||
rotations[k] = Some(adjusted.cameras.rotations[slot]);
|
||||
}
|
||||
|
||||
@@ -9,12 +9,15 @@
|
||||
//! cleverer matcher. A learned matcher (LightGlue) is the step after this
|
||||
//! one fails on a real set, and it has not (panorama.md §6).
|
||||
//!
|
||||
//! Brute force. `4096 × 4096 × 64` multiply-adds is a billion, which is
|
||||
//! tens of milliseconds a pair on one core, and there are at most a few
|
||||
//! dozen pairs. Not worth an index.
|
||||
//! Brute force. `4096 × 4096 × 64` multiply-adds is a billion per pair,
|
||||
//! and a twelve-frame set has sixty-six pairs: a minute single-threaded
|
||||
//! and scalar (measured 2026-09-19: 51 s), a few seconds vectorised across
|
||||
//! the cores. Not worth an index, but worth doing properly.
|
||||
|
||||
use crate::features::{Features, DESCRIPTOR_LEN};
|
||||
|
||||
const _: () = assert!(DESCRIPTOR_LEN % 8 == 0);
|
||||
|
||||
/// A correspondence: keypoint `a` in the first image matches keypoint `b`
|
||||
/// in the second, with the cosine similarity of their descriptors.
|
||||
#[derive(Debug, Clone, Copy, PartialEq)]
|
||||
@@ -32,48 +35,75 @@ pub fn match_features(a: &Features, b: &Features, min_similarity: f32) -> Vec<Ma
|
||||
if a.is_empty() || b.is_empty() {
|
||||
return Vec::new();
|
||||
}
|
||||
let best_ab = nearest(a, b);
|
||||
let best_ba = nearest(b, a);
|
||||
let (na, nb) = (a.len(), b.len());
|
||||
|
||||
// The whole similarity matrix, once. Both nearest-neighbour directions
|
||||
// read it, which halves the multiply-adds against computing each
|
||||
// direction on its own; 4096 × 4096 × f32 is 64 MB, transient.
|
||||
let mut sim = vec![0.0f32; na * nb];
|
||||
let threads = std::thread::available_parallelism()
|
||||
.map(usize::from)
|
||||
.unwrap_or(1)
|
||||
.clamp(1, 16);
|
||||
let rows_per = na.div_ceil(threads);
|
||||
std::thread::scope(|scope| {
|
||||
for (t, chunk) in sim.chunks_mut(rows_per * nb).enumerate() {
|
||||
scope.spawn(move || {
|
||||
let first = t * rows_per;
|
||||
for (r, row) in chunk.chunks_mut(nb).enumerate() {
|
||||
let da = a.descriptor(first + r);
|
||||
for (j, cell) in row.iter_mut().enumerate() {
|
||||
*cell = dot(da, b.descriptor(j));
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
});
|
||||
|
||||
// Best in `b` for each `a`, and best in `a` for each `b`.
|
||||
let best_ab: Vec<(usize, f32)> = sim
|
||||
.chunks_exact(nb)
|
||||
.map(|row| {
|
||||
row.iter()
|
||||
.enumerate()
|
||||
.fold((0usize, f32::MIN), |acc, (j, &s)| if s > acc.1 { (j, s) } else { acc })
|
||||
})
|
||||
.collect();
|
||||
let mut best_ba = vec![(0usize, f32::MIN); nb];
|
||||
for (i, row) in sim.chunks_exact(nb).enumerate() {
|
||||
for (j, &s) in row.iter().enumerate() {
|
||||
if s > best_ba[j].1 {
|
||||
best_ba[j] = (i, s);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
best_ab
|
||||
.iter()
|
||||
.enumerate()
|
||||
.filter_map(|(ia, &(ib, sim))| {
|
||||
(best_ba[ib].0 == ia && sim >= min_similarity).then_some(Match {
|
||||
.filter_map(|(ia, &(ib, s))| {
|
||||
(best_ba[ib].0 == ia && s >= min_similarity).then_some(Match {
|
||||
a: ia,
|
||||
b: ib,
|
||||
similarity: sim,
|
||||
similarity: s,
|
||||
})
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
/// For each descriptor in `from`, the index of its nearest in `to` and the
|
||||
/// similarity.
|
||||
fn nearest(from: &Features, to: &Features) -> Vec<(usize, f32)> {
|
||||
(0..from.len())
|
||||
.map(|i| {
|
||||
let d = from.descriptor(i);
|
||||
let mut best = (0usize, f32::MIN);
|
||||
for j in 0..to.len() {
|
||||
let s = dot(d, to.descriptor(j));
|
||||
if s > best.1 {
|
||||
best = (j, s);
|
||||
}
|
||||
}
|
||||
best
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
#[inline]
|
||||
fn dot(a: &[f32], b: &[f32]) -> f32 {
|
||||
// Written as a plain loop over a fixed length so the compiler
|
||||
// vectorises it; the length is a constant and the slices are exact.
|
||||
let mut s = 0.0f32;
|
||||
for k in 0..DESCRIPTOR_LEN {
|
||||
s += a[k] * b[k];
|
||||
// Eight independent accumulators over exact 8-lane chunks: the shape
|
||||
// the compiler turns into one vector multiply-add per chunk, and no
|
||||
// bounds checks inside the loop. `DESCRIPTOR_LEN` is a multiple of 8.
|
||||
let (a, b) = (&a[..DESCRIPTOR_LEN], &b[..DESCRIPTOR_LEN]);
|
||||
let mut acc = [0.0f32; 8];
|
||||
for (ca, cb) in a.chunks_exact(8).zip(b.chunks_exact(8)) {
|
||||
for k in 0..8 {
|
||||
acc[k] += ca[k] * cb[k];
|
||||
}
|
||||
}
|
||||
s
|
||||
acc.iter().sum()
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
|
||||
+48
-32
@@ -11,17 +11,27 @@ use crate::features::{decode_xfeat, DecodeOptions, Features, XFeatMaps, DESCRIPT
|
||||
use crate::image::Gray;
|
||||
use crate::PanoError;
|
||||
|
||||
/// The input shape the shipped export was made for. A different size is a
|
||||
/// different file (`tools/export-xfeat.sh`).
|
||||
pub const INPUT_WIDTH: usize = 1024;
|
||||
pub const INPUT_HEIGHT: usize = 768;
|
||||
/// The two input shapes the shipped exports were made for: one landscape,
|
||||
/// one portrait, the same weights. A frame is fitted into whichever
|
||||
/// matches its aspect, so a portrait set does not spend half the
|
||||
/// detector's width on padding — which is what the 6D fixture did before
|
||||
/// the second export existed (512 × 768 of a 1024 × 768 input). A
|
||||
/// different size is a different file (`tools/export-xfeat.sh`).
|
||||
pub const INPUT_LANDSCAPE: (usize, usize) = (1024, 768);
|
||||
pub const INPUT_PORTRAIT: (usize, usize) = (768, 1024);
|
||||
|
||||
/// The long edge of the detector's input, for callers sizing a proxy.
|
||||
pub const INPUT_LONG_EDGE: usize = 1024;
|
||||
|
||||
#[cfg(feature = "embedded-model")]
|
||||
const EMBEDDED_MODEL: &[u8] = include_bytes!("../../../models/keypoints/xfeat-1024.onnx");
|
||||
const EMBEDDED_LANDSCAPE: &[u8] = include_bytes!("../../../models/keypoints/xfeat-1024.onnx");
|
||||
#[cfg(feature = "embedded-model")]
|
||||
const EMBEDDED_PORTRAIT: &[u8] = include_bytes!("../../../models/keypoints/xfeat-768.onnx");
|
||||
|
||||
/// A loaded detector.
|
||||
/// A loaded detector: the network at both shapes.
|
||||
pub struct XFeat {
|
||||
session: ort::session::Session,
|
||||
landscape: ort::session::Session,
|
||||
portrait: ort::session::Session,
|
||||
pub options: DecodeOptions,
|
||||
}
|
||||
|
||||
@@ -29,49 +39,55 @@ impl XFeat {
|
||||
/// The weights compiled into the binary.
|
||||
#[cfg(feature = "embedded-model")]
|
||||
pub fn embedded() -> Result<Self, PanoError> {
|
||||
Self::from_bytes(EMBEDDED_MODEL)
|
||||
Self::from_bytes(EMBEDDED_LANDSCAPE, EMBEDDED_PORTRAIT)
|
||||
}
|
||||
|
||||
pub fn from_path(path: &std::path::Path) -> Result<Self, PanoError> {
|
||||
let bytes = std::fs::read(path).map_err(PanoError::ModelRead)?;
|
||||
Self::from_bytes(&bytes)
|
||||
/// From the two exports on disk.
|
||||
pub fn from_paths(landscape: &std::path::Path, portrait: &std::path::Path) -> Result<Self, PanoError> {
|
||||
let l = std::fs::read(landscape).map_err(PanoError::ModelRead)?;
|
||||
let p = std::fs::read(portrait).map_err(PanoError::ModelRead)?;
|
||||
Self::from_bytes(&l, &p)
|
||||
}
|
||||
|
||||
pub fn from_bytes(bytes: &[u8]) -> Result<Self, PanoError> {
|
||||
pub fn from_bytes(landscape: &[u8], portrait: &[u8]) -> Result<Self, PanoError> {
|
||||
install_backend();
|
||||
let session = ort::session::Session::builder()
|
||||
.map_err(PanoError::Inference)?
|
||||
.commit_from_memory(bytes)
|
||||
.map_err(PanoError::Inference)?;
|
||||
let session = |bytes: &[u8]| {
|
||||
ort::session::Session::builder()
|
||||
.map_err(PanoError::Inference)?
|
||||
.commit_from_memory(bytes)
|
||||
.map_err(PanoError::Inference)
|
||||
};
|
||||
Ok(XFeat {
|
||||
session,
|
||||
landscape: session(landscape)?,
|
||||
portrait: session(portrait)?,
|
||||
options: DecodeOptions::default(),
|
||||
})
|
||||
}
|
||||
|
||||
/// Detect keypoints in an upright grayscale image.
|
||||
///
|
||||
/// The image is fitted into the network's fixed input — scaled down if
|
||||
/// larger, never up, and padded to the right and bottom — and the
|
||||
/// keypoints come back in the coordinates of `image` itself, so a
|
||||
/// caller that already scaled a frame to a proxy maps them on with the
|
||||
/// scale it used and nothing else.
|
||||
/// The image is fitted into the network's input of matching aspect —
|
||||
/// scaled down if larger, never up, and padded to the right and bottom
|
||||
/// — and the keypoints come back in the coordinates of `image` itself,
|
||||
/// so a caller that already scaled a frame to a proxy maps them on with
|
||||
/// the scale it used and nothing else.
|
||||
pub fn detect(&mut self, image: &Gray) -> Result<Features, PanoError> {
|
||||
let (fitted, scale) = image.fitted(INPUT_WIDTH, INPUT_HEIGHT);
|
||||
let padded = fitted.padded(INPUT_WIDTH, INPUT_HEIGHT);
|
||||
let ((in_w, in_h), session) = if image.height > image.width {
|
||||
(INPUT_PORTRAIT, &mut self.portrait)
|
||||
} else {
|
||||
(INPUT_LANDSCAPE, &mut self.landscape)
|
||||
};
|
||||
let (fitted, scale) = image.fitted(in_w, in_h);
|
||||
let padded = fitted.padded(in_w, in_h);
|
||||
|
||||
let input = ndarray::Array::from_shape_vec(
|
||||
ndarray::IxDyn(&[1, 1, INPUT_HEIGHT, INPUT_WIDTH]),
|
||||
padded.data,
|
||||
)
|
||||
.expect("shape matches the buffer by construction");
|
||||
let input = ndarray::Array::from_shape_vec(ndarray::IxDyn(&[1, 1, in_h, in_w]), padded.data)
|
||||
.expect("shape matches the buffer by construction");
|
||||
let tensor = ort::value::Tensor::from_array(input).map_err(PanoError::Inference)?;
|
||||
let outputs = self
|
||||
.session
|
||||
let outputs = session
|
||||
.run(ort::inputs![tensor])
|
||||
.map_err(PanoError::Inference)?;
|
||||
|
||||
let (w8, h8) = (INPUT_WIDTH / 8, INPUT_HEIGHT / 8);
|
||||
let (w8, h8) = (in_w / 8, in_h / 8);
|
||||
let expect = |i: usize, channels: usize| -> Result<Vec<f32>, PanoError> {
|
||||
let (shape, data) = outputs[i]
|
||||
.try_extract_tensor::<f32>()
|
||||
|
||||
+20
-20
File diff suppressed because one or more lines are too long
+6
-3
@@ -76,10 +76,13 @@ rather than a code change — as this document predicted it would be.
|
||||
|
||||
| File | Source | Trained on | Used by |
|
||||
|---|---|---|---|
|
||||
| `keypoints/xfeat-1024.onnx` | `weights/xfeat.pt` from `https://github.com/verlab/accelerated_features` | MegaDepth + synthetic warps, by the authors | panorama alignment (FR-MRG-8) |
|
||||
| `keypoints/xfeat-1024.onnx` | `weights/xfeat.pt` from `https://github.com/verlab/accelerated_features` | MegaDepth + synthetic warps, by the authors | panorama alignment (FR-MRG-8), landscape frames |
|
||||
| `keypoints/xfeat-768.onnx` | the same weights | — | the same, portrait frames |
|
||||
|
||||
Exported by `tools/export-xfeat.sh` at a fixed 768×1024 grayscale input.
|
||||
Only the convolutional network is in the file; the keypoint decoding is Rust.
|
||||
Exported by `tools/export-xfeat.sh` at fixed grayscale inputs of 1024×768
|
||||
and 768×1024 — the same weights twice, because tract needs a static shape
|
||||
and a portrait frame in a landscape input wastes half of it. Only the
|
||||
convolutional network is in each file; the keypoint decoding is Rust.
|
||||
|
||||
**The repository and its weights are Apache-2.0**, read on 2026-09-19 from the
|
||||
`LICENSE` at its root, with no separate grant on the checkpoint and no
|
||||
|
||||
Binary file not shown.
+16
-11
@@ -6,8 +6,12 @@
|
||||
# someone once produced and nobody can regenerate. Run it when bumping the
|
||||
# model or changing its input size.
|
||||
#
|
||||
# ./tools/export-xfeat.sh # 768×1024 -> models/keypoints/xfeat-1024.onnx
|
||||
# ./tools/export-xfeat.sh 576 768 # another fixed size
|
||||
# ./tools/export-xfeat.sh # both shapes -> models/keypoints/xfeat-{1024,768}.onnx
|
||||
#
|
||||
# Two files from one set of weights: 1024 wide × 768 tall for landscape
|
||||
# frames and 768 × 1024 for portrait, chosen by the frame's aspect at run
|
||||
# time. One landscape file would fit a portrait frame at half the width and
|
||||
# waste half the detector on padding, which is what the 6D fixture did.
|
||||
#
|
||||
# Requires `uv`. Everything else is fetched into a throwaway venv, including
|
||||
# a CPU-only torch — the export needs no GPU and the CUDA wheels are 2 GB.
|
||||
@@ -37,23 +41,25 @@ set -euo pipefail
|
||||
HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
REPO="$(cd "${HERE}/.." && pwd)"
|
||||
|
||||
H="${1:-768}"
|
||||
W="${2:-1024}"
|
||||
OUT="${REPO}/models/keypoints"
|
||||
NAME="xfeat-${W}"
|
||||
|
||||
# Not `mktemp -d` under /tmp: a tmpfs, and torch is a gigabyte.
|
||||
WORK="$(mktemp -d -p "${TMPDIR:-/var/tmp}")"
|
||||
trap 'rm -rf "${WORK}"' EXIT
|
||||
|
||||
echo "==> exporting XFeat at ${H}×${W} in ${WORK}"
|
||||
echo "==> exporting XFeat in ${WORK}"
|
||||
cd "${WORK}"
|
||||
git clone -q --depth 1 https://github.com/verlab/accelerated_features.git xfeat
|
||||
uv venv -q --python 3.12 venv
|
||||
VIRTUAL_ENV="${WORK}/venv" uv pip install -q --index-url https://download.pytorch.org/whl/cpu torch
|
||||
VIRTUAL_ENV="${WORK}/venv" uv pip install -q onnx onnxslim
|
||||
|
||||
VIRTUAL_ENV="${WORK}/venv" "${WORK}/venv/bin/python" - "${WORK}/xfeat" "${WORK}/${NAME}.onnx" "${H}" "${W}" <<'PY'
|
||||
mkdir -p "${OUT}"
|
||||
for shape in "768 1024" "1024 768"; do
|
||||
set -- $shape
|
||||
H="$1"; W="$2"
|
||||
NAME="xfeat-${W}"
|
||||
VIRTUAL_ENV="${WORK}/venv" "${WORK}/venv/bin/python" - "${WORK}/xfeat" "${WORK}/${NAME}.onnx" "${H}" "${W}" <<'PY'
|
||||
import sys, torch, onnx, onnxslim
|
||||
sys.path.insert(0, sys.argv[1])
|
||||
from modules.model import XFeatModel
|
||||
@@ -72,7 +78,6 @@ print("ops:", sorted({n.op_type for n in m.graph.node}))
|
||||
for o in m.graph.output:
|
||||
print("out", o.name, [d.dim_value for d in o.type.tensor_type.shape.dim])
|
||||
PY
|
||||
|
||||
mkdir -p "${OUT}"
|
||||
cp "${WORK}/${NAME}.onnx" "${OUT}/${NAME}.onnx"
|
||||
echo "==> ${OUT}/${NAME}.onnx"
|
||||
cp "${WORK}/${NAME}.onnx" "${OUT}/${NAME}.onnx"
|
||||
echo "==> ${OUT}/${NAME}.onnx"
|
||||
done
|
||||
|
||||
Reference in New Issue
Block a user