Move the panorama keypoint detector onto the engine, and probe with a detector
XFeat's two exports are a Keypoints role now; the crate no longer names tract, and the app compiles TensorRT engines for both ahead of the first merge. The probe picks the smallest *detector* rather than the smallest file: the tablet's first run chose the 112 KB eye classifier, which has no int8 form, and reported the Hexagon as failed for want of one.
This commit is contained in:
Generated
+1
-1
@@ -1550,12 +1550,12 @@ name = "dr-pano"
|
||||
version = "0.12.2"
|
||||
dependencies = [
|
||||
"dr-decode",
|
||||
"dr-inference-engine",
|
||||
"dr-types",
|
||||
"env_logger",
|
||||
"log",
|
||||
"ndarray",
|
||||
"ort",
|
||||
"ort-tract",
|
||||
"thiserror 2.0.20",
|
||||
]
|
||||
|
||||
|
||||
@@ -335,9 +335,15 @@ fn unpack_bundled_models(app: &slint::android::AndroidApp) {
|
||||
// chose that rung and ignores it otherwise.
|
||||
const BUNDLED: [(&std::ffi::CStr, &str); 13] = [
|
||||
(c"models/scrfd_500m_640.onnx", "scrfd_500m_640.onnx"),
|
||||
(c"models/scrfd_500m_640.int8.onnx", "scrfd_500m_640.int8.onnx"),
|
||||
(
|
||||
c"models/scrfd_500m_640.int8.onnx",
|
||||
"scrfd_500m_640.int8.onnx",
|
||||
),
|
||||
(c"models/scrfd_2.5g_640.onnx", "scrfd_2.5g_640.onnx"),
|
||||
(c"models/scrfd_2.5g_640.int8.onnx", "scrfd_2.5g_640.int8.onnx"),
|
||||
(
|
||||
c"models/scrfd_2.5g_640.int8.onnx",
|
||||
"scrfd_2.5g_640.int8.onnx",
|
||||
),
|
||||
(c"models/scrfd_10g_640.onnx", "scrfd_10g_640.onnx"),
|
||||
(c"models/scrfd_10g_640.int8.onnx", "scrfd_10g_640.int8.onnx"),
|
||||
(c"models/arcface_mbf_b1.onnx", "arcface_mbf_b1.onnx"),
|
||||
|
||||
@@ -39,6 +39,8 @@ pub enum Role {
|
||||
Landmarks,
|
||||
/// The eye-state and sunglasses classifiers, a few hundred kilobytes.
|
||||
EyeClassifier,
|
||||
/// XFeat, the panorama keypoint detector (docs/panorama.md).
|
||||
Keypoints,
|
||||
}
|
||||
|
||||
/// Which numeric form of a model a session was built from.
|
||||
|
||||
@@ -115,17 +115,24 @@ fn finish(cache: Cache) {
|
||||
s.probing = false;
|
||||
}
|
||||
|
||||
/// The smallest configured model: the detector on every device shipped
|
||||
/// today, and a ~2 MB graph is the cheapest real test of a provider.
|
||||
/// The smallest detector, or the smallest model of any role if there is
|
||||
/// none. A ~2 MB detector is the cheapest real test of a provider, and the
|
||||
/// detector is the role the int8 forms exist for — the eye classifiers are
|
||||
/// smaller still, and a Hexagon probed with one would fail for want of a
|
||||
/// form nobody ships.
|
||||
fn probe_model(cfg: &Config) -> Option<(Role, PathBuf)> {
|
||||
cfg.models
|
||||
.iter()
|
||||
.filter_map(|(role, path)| {
|
||||
let size = std::fs::metadata(path).ok()?.len();
|
||||
Some((size, *role, path.clone()))
|
||||
})
|
||||
.min_by_key(|(size, _, _)| *size)
|
||||
.map(|(_, role, path)| (role, path))
|
||||
let smallest = |want: Option<Role>| {
|
||||
cfg.models
|
||||
.iter()
|
||||
.filter(|(role, _)| want.is_none_or(|w| *role == w))
|
||||
.filter_map(|(role, path)| {
|
||||
let size = std::fs::metadata(path).ok()?.len();
|
||||
Some((size, *role, path.clone()))
|
||||
})
|
||||
.min_by_key(|(size, _, _)| *size)
|
||||
.map(|(_, role, path)| (role, path))
|
||||
};
|
||||
smallest(Some(Role::Detector)).or_else(|| smallest(None))
|
||||
}
|
||||
|
||||
/// Build strictly, run once for the engine, then time three runs; the
|
||||
|
||||
@@ -12,11 +12,12 @@ thiserror.workspace = true
|
||||
log.workspace = true
|
||||
|
||||
# Inference for the learned keypoint detector, on the same footing as
|
||||
# `dr-segment`: `ort` is the API, tract is the engine, and both are optional
|
||||
# so that the geometry — matching, the rotation solve, the projections — is a
|
||||
# dependency-free crate that tests without a model.
|
||||
# `dr-segment`: `ort` is the API, `dr-inference-engine` decides what runs
|
||||
# it (docs/inference.md), and both are optional so that the geometry —
|
||||
# matching, the rotation solve, the projections — is a dependency-free crate
|
||||
# that tests without a model.
|
||||
ort = { workspace = true, optional = true }
|
||||
ort-tract = { workspace = true, optional = true }
|
||||
dr-inference-engine = { workspace = true, optional = true }
|
||||
ndarray = { workspace = true, optional = true }
|
||||
|
||||
[dev-dependencies]
|
||||
@@ -30,7 +31,7 @@ default = ["xfeat", "embedded-model"]
|
||||
|
||||
# The XFeat detector (FR-MRG-8). Off, the crate has no model and no runtime,
|
||||
# and `Detector` has no implementation — a build that only wants the geometry.
|
||||
xfeat = ["dep:ort", "dep:ort-tract", "dep:ndarray"]
|
||||
xfeat = ["dep:ort", "dep:dr-inference-engine", "dep:ndarray"]
|
||||
|
||||
# Compile the weights into the binary, for the same reason `dr-segment` does:
|
||||
# Android hands the app no path to read a model from (ARCH §6.9).
|
||||
|
||||
@@ -63,3 +63,13 @@ pub enum PanoError {
|
||||
#[error("inference: {0}")]
|
||||
Inference(#[source] ort::Error),
|
||||
}
|
||||
|
||||
#[cfg(feature = "xfeat")]
|
||||
impl From<dr_inference_engine::Error> for PanoError {
|
||||
fn from(e: dr_inference_engine::Error) -> Self {
|
||||
match e {
|
||||
dr_inference_engine::Error::Inference(e) => PanoError::Inference(e),
|
||||
dr_inference_engine::Error::Io(e) => PanoError::ModelRead(e),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
+22
-28
@@ -2,10 +2,11 @@
|
||||
//! The XFeat detector — the network under tract, and the decoder after it.
|
||||
//!
|
||||
//! Apache-2.0 weights (`models/LICENCE.md`), exported at a fixed shape by
|
||||
//! `tools/export-xfeat.sh` and loaded through the same `ort`-over-tract
|
||||
//! backend `dr-segment` and `dr-face` use, so this adds no runtime and no C
|
||||
//! to the tree. ~300 ms per frame on the reference desktop, ~400 ms on the
|
||||
//! tablet (S15.2, S15.4).
|
||||
//! `tools/export-xfeat.sh` and loaded through the same `dr-inference-engine`
|
||||
//! `dr-segment` and `dr-face` use, so this adds no runtime and no C to the
|
||||
//! tree; what runs it is the device's business (docs/inference.md). ~300 ms
|
||||
//! per frame on tract on the reference desktop, ~400 ms on the tablet
|
||||
//! (S15.2, S15.4).
|
||||
|
||||
use crate::features::{decode_xfeat, DecodeOptions, Features, XFeatMaps, DESCRIPTOR_LEN};
|
||||
use crate::image::Gray;
|
||||
@@ -30,11 +31,18 @@ const EMBEDDED_PORTRAIT: &[u8] = include_bytes!("../../../models/keypoints/xfeat
|
||||
|
||||
/// A loaded detector: the network at both shapes.
|
||||
pub struct XFeat {
|
||||
landscape: ort::session::Session,
|
||||
portrait: ort::session::Session,
|
||||
landscape: dr_inference_engine::Model,
|
||||
portrait: dr_inference_engine::Model,
|
||||
pub options: DecodeOptions,
|
||||
}
|
||||
|
||||
/// The bytes of both exports compiled into the binary, for whoever compiles
|
||||
/// engines ahead of the first request (docs/inference.md §6).
|
||||
#[cfg(feature = "embedded-model")]
|
||||
pub fn embedded_model_bytes() -> [&'static [u8]; 2] {
|
||||
[EMBEDDED_LANDSCAPE, EMBEDDED_PORTRAIT]
|
||||
}
|
||||
|
||||
impl XFeat {
|
||||
/// The weights compiled into the binary.
|
||||
#[cfg(feature = "embedded-model")]
|
||||
@@ -53,16 +61,10 @@ impl XFeat {
|
||||
}
|
||||
|
||||
pub fn from_bytes(landscape: &[u8], portrait: &[u8]) -> Result<Self, PanoError> {
|
||||
install_backend();
|
||||
let session = |bytes: &[u8]| {
|
||||
ort::session::Session::builder()
|
||||
.map_err(PanoError::Inference)?
|
||||
.commit_from_memory(bytes)
|
||||
.map_err(PanoError::Inference)
|
||||
};
|
||||
use dr_inference_engine::{Form, Role};
|
||||
Ok(XFeat {
|
||||
landscape: session(landscape)?,
|
||||
portrait: session(portrait)?,
|
||||
landscape: dr_inference_engine::open(Role::Keypoints, Form::F32, landscape)?,
|
||||
portrait: dr_inference_engine::open(Role::Keypoints, Form::F32, portrait)?,
|
||||
options: DecodeOptions::default(),
|
||||
})
|
||||
}
|
||||
@@ -75,11 +77,13 @@ impl XFeat {
|
||||
/// so a caller that already scaled a frame to a proxy maps them on with
|
||||
/// the scale it used and nothing else.
|
||||
pub fn detect(&mut self, image: &Gray) -> Result<Features, PanoError> {
|
||||
let ((in_w, in_h), session) = if image.height > image.width {
|
||||
(INPUT_PORTRAIT, &mut self.portrait)
|
||||
let ((in_w, in_h), model) = if image.height > image.width {
|
||||
(INPUT_PORTRAIT, &self.portrait)
|
||||
} else {
|
||||
(INPUT_LANDSCAPE, &mut self.landscape)
|
||||
(INPUT_LANDSCAPE, &self.landscape)
|
||||
};
|
||||
let acquired = model.acquire()?;
|
||||
let mut session = acquired.lock();
|
||||
let (fitted, scale) = image.fitted(in_w, in_h);
|
||||
let padded = fitted.padded(in_w, in_h);
|
||||
|
||||
@@ -145,13 +149,3 @@ impl XFeat {
|
||||
Ok(features)
|
||||
}
|
||||
}
|
||||
|
||||
fn install_backend() {
|
||||
use std::sync::Once;
|
||||
static ONCE: Once = Once::new();
|
||||
ONCE.call_once(|| {
|
||||
// False if another crate installed it first, which is fine: there is
|
||||
// one backend compiled in for it to have chosen.
|
||||
let _ = ort::set_api(ort_tract::api());
|
||||
});
|
||||
}
|
||||
|
||||
+11
-11
@@ -25,7 +25,7 @@ Sampling a neutral is the first move of the tonal pass — every colour judgemen
|
||||
|
||||
Anchored on the fingers' midpoint, and on the pointer, so the gesture reads as magnifying the picture rather than sliding it about. Double-tap is the way to an exact 1:1; this is the way to everything in between.
|
||||
|
||||
<sub>`ui/dr-ui/ui/app.slint:2036`</sub>
|
||||
<sub>`ui/dr-ui/ui/app.slint:2040`</sub>
|
||||
|
||||
### Move a magnified photograph about
|
||||
|
||||
@@ -34,7 +34,7 @@ Anchored on the fingers' midpoint, and on the pointer, so the gesture reads as m
|
||||
|
||||
Only once there is something outside the viewport to reach, which is why the cursor becomes a hand exactly then. The view is clamped to the frame: panning past the edge would show undefined area beside the photograph, and that reads as a rendering fault rather than as the end of the picture.
|
||||
|
||||
<sub>`ui/dr-ui/ui/app.slint:2127`</sub>
|
||||
<sub>`ui/dr-ui/ui/app.slint:2131`</sub>
|
||||
|
||||
### Paint a mask by hand
|
||||
|
||||
@@ -43,7 +43,7 @@ Only once there is something outside the viewport to reach, which is why the cur
|
||||
|
||||
A model's mask stops inside a shoulder and leaks into the hair, and no single edge control fixes two errors that go opposite ways. The whole stroke is one step in the history, so taking a mark back costs one press however long it took to make.
|
||||
|
||||
<sub>`ui/dr-ui/ui/app.slint:2214`</sub>
|
||||
<sub>`ui/dr-ui/ui/app.slint:2218`</sub>
|
||||
|
||||
### Take back the last change
|
||||
|
||||
@@ -53,7 +53,7 @@ A model's mask stops inside a shoulder and leaks into the hair, and no single ed
|
||||
|
||||
A whole drag is one step, so undo takes back a decision rather than a frame of a gesture. The list is there because arriving six steps back costs what arriving from one does.
|
||||
|
||||
<sub>`ui/dr-ui/ui/app.slint:2435`</sub>
|
||||
<sub>`ui/dr-ui/ui/app.slint:2439`</sub>
|
||||
|
||||
### Do it again after taking it back
|
||||
|
||||
@@ -61,7 +61,7 @@ A whole drag is one step, so undo takes back a decision rather than a frame of a
|
||||
- **Pointer** — Click it, or press Redo in the History header
|
||||
- **Keyboard** — Ctrl+Shift+Z
|
||||
|
||||
<sub>`ui/dr-ui/ui/app.slint:2448`</sub>
|
||||
<sub>`ui/dr-ui/ui/app.slint:2452`</sub>
|
||||
|
||||
### Copy the settings from this photograph
|
||||
|
||||
@@ -71,7 +71,7 @@ A whole drag is one step, so undo takes back a decision rather than a frame of a
|
||||
|
||||
The panel is the copy that has to work: a tablet has no modifier key to hold and no menu bar to hang the action from. The shortcut is an accelerator for a control that is on screen either way.
|
||||
|
||||
<sub>`ui/dr-ui/ui/app.slint:2481`</sub>
|
||||
<sub>`ui/dr-ui/ui/app.slint:2485`</sub>
|
||||
|
||||
### Paste the settings onto this photograph
|
||||
|
||||
@@ -81,7 +81,7 @@ The panel is the copy that has to work: a tablet has no modifier key to hold and
|
||||
|
||||
The button names what would be pasted — "3 adjustments", and whether the crop is coming with it — which the shortcut cannot say. Both paste the same scope.
|
||||
|
||||
<sub>`ui/dr-ui/ui/app.slint:2493`</sub>
|
||||
<sub>`ui/dr-ui/ui/app.slint:2497`</sub>
|
||||
|
||||
### Change which group of adjustments is on screen
|
||||
|
||||
@@ -91,7 +91,7 @@ The button names what would be pasted — "3 adjustments", and whether the crop
|
||||
|
||||
The groups are whatever the operation set declares itself to be about, so there are as many as the pipeline has and no key can be assigned to one of them by name. Stepping is the binding that survives a node being added.
|
||||
|
||||
<sub>`ui/dr-ui/ui/app.slint:2521`</sub>
|
||||
<sub>`ui/dr-ui/ui/app.slint:2525`</sub>
|
||||
|
||||
### Look at the photograph at 1:1
|
||||
|
||||
@@ -101,7 +101,7 @@ The groups are whatever the operation set declares itself to be about, so there
|
||||
|
||||
Noise reduction and capture sharpening are judgements about single pixels, and a fitted view averages several of the file's into each one on screen — so the frame looks softer than it is and the correction goes too far. The point and the magnification survive opening the next photograph, which is what makes checking the same eye across forty portraits forty keystrokes rather than forty pans.
|
||||
|
||||
<sub>`ui/dr-ui/ui/app.slint:2556`</sub>
|
||||
<sub>`ui/dr-ui/ui/app.slint:2560`</sub>
|
||||
|
||||
### Move to the next or previous photograph
|
||||
|
||||
@@ -111,7 +111,7 @@ Noise reduction and capture sharpening are judgements about single pixels, and a
|
||||
|
||||
The edit on screen is saved on the way out, so stepping through a folder is as much a departure as going back to the grid and loses nothing.
|
||||
|
||||
<sub>`ui/dr-ui/ui/app.slint:2608`</sub>
|
||||
<sub>`ui/dr-ui/ui/app.slint:2612`</sub>
|
||||
|
||||
### See the photograph before you edited it
|
||||
|
||||
@@ -121,7 +121,7 @@ The edit on screen is saved on the way out, so stepping through a folder is as m
|
||||
|
||||
Held rather than toggled, and no split screen: a split halves the working image on the tablet the column was sized for, and the comparison photographers describe making is a flick back and forth. It takes no history step, so checking whether a frame is overcooked costs nothing to undo afterwards.
|
||||
|
||||
<sub>`ui/dr-ui/ui/app.slint:2732`</sub>
|
||||
<sub>`ui/dr-ui/ui/app.slint:2736`</sub>
|
||||
|
||||
### Put one control back to its default
|
||||
|
||||
|
||||
+72
-72
File diff suppressed because one or more lines are too long
@@ -38,7 +38,14 @@ pub fn init(runtime_dirs: Vec<PathBuf>) {
|
||||
runtime_dirs,
|
||||
cache_dir: crate::library::inference_cache_dir(),
|
||||
models,
|
||||
embedded: vec![(Role::Segmenter, dr_segment::embedded_model_bytes())],
|
||||
embedded: {
|
||||
let [landscape, portrait] = dr_pano::xfeat::embedded_model_bytes();
|
||||
vec![
|
||||
(Role::Segmenter, dr_segment::embedded_model_bytes()),
|
||||
(Role::Keypoints, landscape),
|
||||
(Role::Keypoints, portrait),
|
||||
]
|
||||
},
|
||||
ceiling: None,
|
||||
threads: 0,
|
||||
decay: std::time::Duration::ZERO,
|
||||
|
||||
Reference in New Issue
Block a user