dr-pano: a second XFeat shape for portrait frames, and a matcher that takes seconds
Twelve real frames from the fixture set now align in 4.5 s — 4.4 s of matching, 118 ms of bundle adjustment — where the first run took 51 s and left the first two frames out. The matcher computes each pair's similarity matrix once, across the cores, with a dot product written to vectorise; both nearest-neighbour directions read it. The frames that failed were portrait: fitted into the landscape input they used 512 of 1024 px, and their thin overlap did not survive at half resolution. The same weights are now exported at 768×1024 as well and the detector picks the shape by aspect. The example aligns from embedded previews and draws the set on a cylinder; on the fixture the sweep is 152° at a fitted 47.9 mm against the EXIF's 50, RMS 1.5 px, and the overlaps show no ghosting.
This commit is contained in:
@@ -151,9 +151,11 @@ pub fn align(frames: &[Features], opts: &AlignOptions) -> Result<Alignment, Pano
|
||||
let mut links = Vec::new();
|
||||
let mut observations: Vec<Observation> = Vec::new();
|
||||
let mut matched_any = vec![false; n];
|
||||
let t_match = std::time::Instant::now();
|
||||
for i in 0..n {
|
||||
for j in i + 1..n {
|
||||
let matches: Vec<Match> = match_features(&frames[i], &frames[j], opts.min_similarity);
|
||||
log::debug!("pair {i}-{j}: {} matches", matches.len());
|
||||
if matches.len() < 4 {
|
||||
continue;
|
||||
}
|
||||
@@ -175,6 +177,10 @@ pub fn align(frames: &[Features], opts: &AlignOptions) -> Result<Alignment, Pano
|
||||
continue;
|
||||
};
|
||||
let needed = (8.0 + 0.3 * matches.len() as f64).ceil() as usize;
|
||||
log::debug!(
|
||||
"pair {i}-{j}: {} inliers, {needed} needed",
|
||||
inliers.len()
|
||||
);
|
||||
if inliers.len() <= needed || inliers.len() < opts.min_inliers {
|
||||
continue;
|
||||
}
|
||||
@@ -204,6 +210,8 @@ pub fn align(frames: &[Features], opts: &AlignOptions) -> Result<Alignment, Pano
|
||||
}
|
||||
}
|
||||
|
||||
log::debug!("matching and pairwise geometry in {:?}", t_match.elapsed());
|
||||
|
||||
// 3: the focal length.
|
||||
let mut estimates: Vec<f64> = links
|
||||
.iter()
|
||||
@@ -304,7 +312,14 @@ pub fn align(frames: &[Features], opts: &AlignOptions) -> Result<Alignment, Pano
|
||||
})
|
||||
})
|
||||
.collect();
|
||||
let t_adjust = std::time::Instant::now();
|
||||
let adjusted = bundle::adjust(start, &obs, &opts.adjust)?;
|
||||
log::debug!(
|
||||
"bundle adjustment: {} observations, {} iterations in {:?}",
|
||||
obs.len(),
|
||||
adjusted.iterations,
|
||||
t_adjust.elapsed()
|
||||
);
|
||||
for (slot, &k) in aligned.iter().enumerate() {
|
||||
rotations[k] = Some(adjusted.cameras.rotations[slot]);
|
||||
}
|
||||
|
||||
@@ -9,12 +9,15 @@
|
||||
//! cleverer matcher. A learned matcher (LightGlue) is the step after this
|
||||
//! one fails on a real set, and it has not (panorama.md §6).
|
||||
//!
|
||||
//! Brute force. `4096 × 4096 × 64` multiply-adds is a billion, which is
|
||||
//! tens of milliseconds a pair on one core, and there are at most a few
|
||||
//! dozen pairs. Not worth an index.
|
||||
//! Brute force. `4096 × 4096 × 64` multiply-adds is a billion per pair,
|
||||
//! and a twelve-frame set has sixty-six pairs: a minute single-threaded
|
||||
//! and scalar (measured 2026-09-19: 51 s), a few seconds vectorised across
|
||||
//! the cores. Not worth an index, but worth doing properly.
|
||||
|
||||
use crate::features::{Features, DESCRIPTOR_LEN};
|
||||
|
||||
const _: () = assert!(DESCRIPTOR_LEN % 8 == 0);
|
||||
|
||||
/// A correspondence: keypoint `a` in the first image matches keypoint `b`
|
||||
/// in the second, with the cosine similarity of their descriptors.
|
||||
#[derive(Debug, Clone, Copy, PartialEq)]
|
||||
@@ -32,48 +35,75 @@ pub fn match_features(a: &Features, b: &Features, min_similarity: f32) -> Vec<Ma
|
||||
if a.is_empty() || b.is_empty() {
|
||||
return Vec::new();
|
||||
}
|
||||
let best_ab = nearest(a, b);
|
||||
let best_ba = nearest(b, a);
|
||||
let (na, nb) = (a.len(), b.len());
|
||||
|
||||
// The whole similarity matrix, once. Both nearest-neighbour directions
|
||||
// read it, which halves the multiply-adds against computing each
|
||||
// direction on its own; 4096 × 4096 × f32 is 64 MB, transient.
|
||||
let mut sim = vec![0.0f32; na * nb];
|
||||
let threads = std::thread::available_parallelism()
|
||||
.map(usize::from)
|
||||
.unwrap_or(1)
|
||||
.clamp(1, 16);
|
||||
let rows_per = na.div_ceil(threads);
|
||||
std::thread::scope(|scope| {
|
||||
for (t, chunk) in sim.chunks_mut(rows_per * nb).enumerate() {
|
||||
scope.spawn(move || {
|
||||
let first = t * rows_per;
|
||||
for (r, row) in chunk.chunks_mut(nb).enumerate() {
|
||||
let da = a.descriptor(first + r);
|
||||
for (j, cell) in row.iter_mut().enumerate() {
|
||||
*cell = dot(da, b.descriptor(j));
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
});
|
||||
|
||||
// Best in `b` for each `a`, and best in `a` for each `b`.
|
||||
let best_ab: Vec<(usize, f32)> = sim
|
||||
.chunks_exact(nb)
|
||||
.map(|row| {
|
||||
row.iter()
|
||||
.enumerate()
|
||||
.fold((0usize, f32::MIN), |acc, (j, &s)| if s > acc.1 { (j, s) } else { acc })
|
||||
})
|
||||
.collect();
|
||||
let mut best_ba = vec![(0usize, f32::MIN); nb];
|
||||
for (i, row) in sim.chunks_exact(nb).enumerate() {
|
||||
for (j, &s) in row.iter().enumerate() {
|
||||
if s > best_ba[j].1 {
|
||||
best_ba[j] = (i, s);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
best_ab
|
||||
.iter()
|
||||
.enumerate()
|
||||
.filter_map(|(ia, &(ib, sim))| {
|
||||
(best_ba[ib].0 == ia && sim >= min_similarity).then_some(Match {
|
||||
.filter_map(|(ia, &(ib, s))| {
|
||||
(best_ba[ib].0 == ia && s >= min_similarity).then_some(Match {
|
||||
a: ia,
|
||||
b: ib,
|
||||
similarity: sim,
|
||||
similarity: s,
|
||||
})
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
/// For each descriptor in `from`, the index of its nearest in `to` and the
|
||||
/// similarity.
|
||||
fn nearest(from: &Features, to: &Features) -> Vec<(usize, f32)> {
|
||||
(0..from.len())
|
||||
.map(|i| {
|
||||
let d = from.descriptor(i);
|
||||
let mut best = (0usize, f32::MIN);
|
||||
for j in 0..to.len() {
|
||||
let s = dot(d, to.descriptor(j));
|
||||
if s > best.1 {
|
||||
best = (j, s);
|
||||
}
|
||||
}
|
||||
best
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
#[inline]
|
||||
fn dot(a: &[f32], b: &[f32]) -> f32 {
|
||||
// Written as a plain loop over a fixed length so the compiler
|
||||
// vectorises it; the length is a constant and the slices are exact.
|
||||
let mut s = 0.0f32;
|
||||
for k in 0..DESCRIPTOR_LEN {
|
||||
s += a[k] * b[k];
|
||||
// Eight independent accumulators over exact 8-lane chunks: the shape
|
||||
// the compiler turns into one vector multiply-add per chunk, and no
|
||||
// bounds checks inside the loop. `DESCRIPTOR_LEN` is a multiple of 8.
|
||||
let (a, b) = (&a[..DESCRIPTOR_LEN], &b[..DESCRIPTOR_LEN]);
|
||||
let mut acc = [0.0f32; 8];
|
||||
for (ca, cb) in a.chunks_exact(8).zip(b.chunks_exact(8)) {
|
||||
for k in 0..8 {
|
||||
acc[k] += ca[k] * cb[k];
|
||||
}
|
||||
}
|
||||
s
|
||||
acc.iter().sum()
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
|
||||
+48
-32
@@ -11,17 +11,27 @@ use crate::features::{decode_xfeat, DecodeOptions, Features, XFeatMaps, DESCRIPT
|
||||
use crate::image::Gray;
|
||||
use crate::PanoError;
|
||||
|
||||
/// The input shape the shipped export was made for. A different size is a
|
||||
/// different file (`tools/export-xfeat.sh`).
|
||||
pub const INPUT_WIDTH: usize = 1024;
|
||||
pub const INPUT_HEIGHT: usize = 768;
|
||||
/// The two input shapes the shipped exports were made for: one landscape,
|
||||
/// one portrait, the same weights. A frame is fitted into whichever
|
||||
/// matches its aspect, so a portrait set does not spend half the
|
||||
/// detector's width on padding — which is what the 6D fixture did before
|
||||
/// the second export existed (512 × 768 of a 1024 × 768 input). A
|
||||
/// different size is a different file (`tools/export-xfeat.sh`).
|
||||
pub const INPUT_LANDSCAPE: (usize, usize) = (1024, 768);
|
||||
pub const INPUT_PORTRAIT: (usize, usize) = (768, 1024);
|
||||
|
||||
/// The long edge of the detector's input, for callers sizing a proxy.
|
||||
pub const INPUT_LONG_EDGE: usize = 1024;
|
||||
|
||||
#[cfg(feature = "embedded-model")]
|
||||
const EMBEDDED_MODEL: &[u8] = include_bytes!("../../../models/keypoints/xfeat-1024.onnx");
|
||||
const EMBEDDED_LANDSCAPE: &[u8] = include_bytes!("../../../models/keypoints/xfeat-1024.onnx");
|
||||
#[cfg(feature = "embedded-model")]
|
||||
const EMBEDDED_PORTRAIT: &[u8] = include_bytes!("../../../models/keypoints/xfeat-768.onnx");
|
||||
|
||||
/// A loaded detector.
|
||||
/// A loaded detector: the network at both shapes.
|
||||
pub struct XFeat {
|
||||
session: ort::session::Session,
|
||||
landscape: ort::session::Session,
|
||||
portrait: ort::session::Session,
|
||||
pub options: DecodeOptions,
|
||||
}
|
||||
|
||||
@@ -29,49 +39,55 @@ impl XFeat {
|
||||
/// The weights compiled into the binary.
|
||||
#[cfg(feature = "embedded-model")]
|
||||
pub fn embedded() -> Result<Self, PanoError> {
|
||||
Self::from_bytes(EMBEDDED_MODEL)
|
||||
Self::from_bytes(EMBEDDED_LANDSCAPE, EMBEDDED_PORTRAIT)
|
||||
}
|
||||
|
||||
pub fn from_path(path: &std::path::Path) -> Result<Self, PanoError> {
|
||||
let bytes = std::fs::read(path).map_err(PanoError::ModelRead)?;
|
||||
Self::from_bytes(&bytes)
|
||||
/// From the two exports on disk.
|
||||
pub fn from_paths(landscape: &std::path::Path, portrait: &std::path::Path) -> Result<Self, PanoError> {
|
||||
let l = std::fs::read(landscape).map_err(PanoError::ModelRead)?;
|
||||
let p = std::fs::read(portrait).map_err(PanoError::ModelRead)?;
|
||||
Self::from_bytes(&l, &p)
|
||||
}
|
||||
|
||||
pub fn from_bytes(bytes: &[u8]) -> Result<Self, PanoError> {
|
||||
pub fn from_bytes(landscape: &[u8], portrait: &[u8]) -> Result<Self, PanoError> {
|
||||
install_backend();
|
||||
let session = ort::session::Session::builder()
|
||||
.map_err(PanoError::Inference)?
|
||||
.commit_from_memory(bytes)
|
||||
.map_err(PanoError::Inference)?;
|
||||
let session = |bytes: &[u8]| {
|
||||
ort::session::Session::builder()
|
||||
.map_err(PanoError::Inference)?
|
||||
.commit_from_memory(bytes)
|
||||
.map_err(PanoError::Inference)
|
||||
};
|
||||
Ok(XFeat {
|
||||
session,
|
||||
landscape: session(landscape)?,
|
||||
portrait: session(portrait)?,
|
||||
options: DecodeOptions::default(),
|
||||
})
|
||||
}
|
||||
|
||||
/// Detect keypoints in an upright grayscale image.
|
||||
///
|
||||
/// The image is fitted into the network's fixed input — scaled down if
|
||||
/// larger, never up, and padded to the right and bottom — and the
|
||||
/// keypoints come back in the coordinates of `image` itself, so a
|
||||
/// caller that already scaled a frame to a proxy maps them on with the
|
||||
/// scale it used and nothing else.
|
||||
/// The image is fitted into the network's input of matching aspect —
|
||||
/// scaled down if larger, never up, and padded to the right and bottom
|
||||
/// — and the keypoints come back in the coordinates of `image` itself,
|
||||
/// so a caller that already scaled a frame to a proxy maps them on with
|
||||
/// the scale it used and nothing else.
|
||||
pub fn detect(&mut self, image: &Gray) -> Result<Features, PanoError> {
|
||||
let (fitted, scale) = image.fitted(INPUT_WIDTH, INPUT_HEIGHT);
|
||||
let padded = fitted.padded(INPUT_WIDTH, INPUT_HEIGHT);
|
||||
let ((in_w, in_h), session) = if image.height > image.width {
|
||||
(INPUT_PORTRAIT, &mut self.portrait)
|
||||
} else {
|
||||
(INPUT_LANDSCAPE, &mut self.landscape)
|
||||
};
|
||||
let (fitted, scale) = image.fitted(in_w, in_h);
|
||||
let padded = fitted.padded(in_w, in_h);
|
||||
|
||||
let input = ndarray::Array::from_shape_vec(
|
||||
ndarray::IxDyn(&[1, 1, INPUT_HEIGHT, INPUT_WIDTH]),
|
||||
padded.data,
|
||||
)
|
||||
.expect("shape matches the buffer by construction");
|
||||
let input = ndarray::Array::from_shape_vec(ndarray::IxDyn(&[1, 1, in_h, in_w]), padded.data)
|
||||
.expect("shape matches the buffer by construction");
|
||||
let tensor = ort::value::Tensor::from_array(input).map_err(PanoError::Inference)?;
|
||||
let outputs = self
|
||||
.session
|
||||
let outputs = session
|
||||
.run(ort::inputs![tensor])
|
||||
.map_err(PanoError::Inference)?;
|
||||
|
||||
let (w8, h8) = (INPUT_WIDTH / 8, INPUT_HEIGHT / 8);
|
||||
let (w8, h8) = (in_w / 8, in_h / 8);
|
||||
let expect = |i: usize, channels: usize| -> Result<Vec<f32>, PanoError> {
|
||||
let (shape, data) = outputs[i]
|
||||
.try_extract_tensor::<f32>()
|
||||
|
||||
Reference in New Issue
Block a user