Add dr-inference-engine and route every model session through it

One crate names the runtime, the providers and the devices; dr-face and
dr-segment ask it for a session by role. It hands ort an API table once
per process — from a libonnxruntime it dlopens when the app names a
directory holding one, otherwise from tract — so the Rust build stays
free of C on every target and a package can install the runtime as a
file (docs/inference.md §3).

Sessions live in a registry behind a Model handle that holds the bytes,
not the session: every use refreshes a timestamp and a reaper unloads
whatever sat idle past the decay. A scan that runs the detector on each
image never lets it go idle; a click in the develop view lets the
segmenter go after thirty seconds; a handle used after that reloads,
and reloads on a higher rung if a compiled engine has landed meanwhile.

The probe walks the platform's ladder by building strict sessions and
timing them against the CPU provider, caches the choice against a
fingerprint of the runtime, driver, hardware and models, and compiles
engines for the selected rung in the background, smallest model first.
Nothing in this commit turns the native path on: the apps still run on
tract until they call init with a runtime directory.

core/dr-segment/Cargo.toml

@@ -11,10 +11,11 @@ build = "build.rs"
 thiserror.workspace = true
 log.workspace = true
 
-# Inference. `ort` is the API; **tract is the engine** — see the workspace
-# manifest for why the C++ ONNX Runtime is not linked here.
+# Inference. `ort` is the API; **what runs it is `dr-inference-engine`'s
+# business** — tract, or an ONNX Runtime the app found on disk, on whichever
+# provider the device has (docs/inference.md). This crate never names either.
 ort = { workspace = true, optional = true }
-ort-tract = { workspace = true, optional = true }
+dr-inference-engine = { workspace = true, optional = true }
 ndarray = { workspace = true, optional = true }
 
 [dev-dependencies]
@@ -35,7 +36,7 @@ default = ["semantic", "embedded-model"]
 # Separable because the watershed half is genuinely independent of it: with
 # this off, `dr-segment` is a pure-CPU graph algorithm crate with no model to
 # carry, which is what the headless hierarchy tests want.
-semantic = ["dep:ort", "dep:ort-tract", "dep:ndarray"]
+semantic = ["dep:ort", "dep:dr-inference-engine", "dep:ndarray"]
 
 # Compile the weights into the binary.
 #

core/dr-segment/src/lib.rs

@@ -98,3 +98,13 @@ pub enum SegmentError {
     #[error("category descriptor: {0}")]
     CategoryDescriptor(String),
 }
+
+#[cfg(feature = "semantic")]
+impl From<dr_inference_engine::Error> for SegmentError {
+    fn from(e: dr_inference_engine::Error) -> Self {
+        match e {
+            dr_inference_engine::Error::Inference(e) => SegmentError::Inference(e),
+            dr_inference_engine::Error::Io(e) => SegmentError::ModelRead(e),
+        }
+    }
+}

core/dr-segment/src/scene.rs

@@ -59,7 +59,7 @@ use ndarray::ArrayView3;
 
 #[cfg(test)]
 use crate::semantic::INPUT_EDGE;
-use crate::semantic::{install_backend, Letterbox, Window};
+use crate::semantic::{Letterbox, Window};
 use crate::SegmentError;
 
 /// Classes in the ADE20K vocabulary the scene model was trained on.
@@ -84,7 +84,7 @@ pub struct Category {
 
 /// The scene model, and the categories it has been told to report.
 pub struct SceneModel {
-    session: ort::session::Session,
+    session: dr_inference_engine::Model,
     categories: Vec<Category>,
 }
 
@@ -132,12 +132,12 @@ impl SceneModel {
     }
 
     pub fn from_bytes(bytes: &[u8], categories: Vec<Category>) -> Result<Self, SegmentError> {
-        install_backend();
-
-        let session = ort::session::Session::builder()
-            .map_err(SegmentError::Inference)?
-            .commit_from_memory(bytes)
-            .map_err(SegmentError::Inference)?;
+        // f32, as for `SemanticModel`; see there.
+        let session = dr_inference_engine::open(
+            dr_inference_engine::Role::Scene,
+            dr_inference_engine::Form::F32,
+            bytes,
+        )?;
 
         Ok(Self {
             session,
@@ -170,13 +170,9 @@ impl SceneModel {
             });
         }
 
-        // Split the borrow: `run` needs the session mutably while
-        // `marginalise` needs the categories, and going through `self` for
-        // both at once is what the borrow checker objects to.
-        let Self {
-            session,
-            categories,
-        } = self;
+        let categories = &self.categories;
+        let acquired = self.session.acquire()?;
+        let mut session = acquired.lock();
 
         let window = Window {
             x: 0.0,

core/dr-segment/src/semantic.rs

@@ -194,7 +194,7 @@ impl Instance {
 /// Holds an `ort` session, so it is neither `Clone` nor cheap to build —
 /// construct once and keep it. Loading is ~50 ms.
 pub struct SemanticModel {
-    session: ort::session::Session,
+    session: dr_inference_engine::Model,
     classes: Vec<Arc<str>>,
 }
 
@@ -229,15 +229,14 @@ impl SemanticModel {
     }
 
     pub fn from_bytes(bytes: &[u8], classes: Vec<Arc<str>>) -> Result<Self, SegmentError> {
-        // Idempotent, and it must happen before any other `ort` call: with
-        // `alternative-backend` there is no linked runtime to fall back on, so
-        // an un-set API is a panic rather than a slow path.
-        install_backend();
-
-        let session = ort::session::Session::builder()
-            .map_err(SegmentError::Inference)?
-            .commit_from_memory(bytes)
-            .map_err(SegmentError::Inference)?;
+        // The f32 graph on whatever the device's backend is. An int8 form
+        // for the Hexagon waits on docs/inference.md §10 M7 — the mask
+        // boundary has to be measured before it moves.
+        let session = dr_inference_engine::open(
+            dr_inference_engine::Role::Segmenter,
+            dr_inference_engine::Form::F32,
+            bytes,
+        )?;
 
         Ok(Self { session, classes })
     }
@@ -332,8 +331,9 @@ impl SemanticModel {
         let letterbox = Letterbox::fit(window.w, window.h);
         let input = letterbox.sample(rgb, width, height, window);
 
-        let outputs = self
-            .session
+        let acquired = self.session.acquire()?;
+        let mut session = acquired.lock();
+        let outputs = session
             .run(ort::inputs![
                 ort::value::Tensor::from_array(input).map_err(SegmentError::Inference)?
             ])
@@ -674,18 +674,6 @@ fn steps(extent: f32, edge: f32, stride: f32) -> usize {
     }
 }
 
-/// Point `ort` at tract, exactly once per process.
-pub(crate) fn install_backend() {
-    use std::sync::Once;
-    static ONCE: Once = Once::new();
-    ONCE.call_once(|| {
-        // Returns false if an API was already installed, which is not an error
-        // — it means something else got here first, and there is only one
-        // backend compiled in for it to have chosen.
-        let _ = ort::set_api(ort_tract::api());
-    });
-}
-
 /// Read the class list written beside the model by `tools/export-seg-model.sh`.
 ///
 /// A deliberately small hand-rolled reader for a flat array of strings, rather