From b562d7b1afcb8f6f4b43fe7286d3180e0ecdf9c1 Mon Sep 17 00:00:00 2001 From: Duncan Tourolle Date: Tue, 29 Sep 2026 21:32:24 -0400 Subject: [PATCH] Stop retrying a provider that took the app down MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The probe runs in the app's process, and a provider can fail by aborting rather than by returning an error — XNNPACK did on SCRFD. A rung that does that once would do it on every launch, before the first photograph is on screen. Every session build above the CPU, the probe's and each background compile's, now writes what it is attempting to `attempt` in the cache directory first and removes it after. After two launches in a row that died inside the same attempt it is refused and recorded — a rung in `failed`, an engine in the new `refused` — until the fingerprint changes. Two, not one, because quitting during a TensorRT compile leaves the same file. --- core/dr-inference-engine/src/engines.rs | 21 +++++- core/dr-inference-engine/src/lib.rs | 6 ++ core/dr-inference-engine/src/probe.rs | 98 ++++++++++++++++++++++++- docs/dev/inference.md | 8 ++ 4 files changed, 129 insertions(+), 4 deletions(-) diff --git a/core/dr-inference-engine/src/engines.rs b/core/dr-inference-engine/src/engines.rs index 5af4887..3a8670a 100644 --- a/core/dr-inference-engine/src/engines.rs +++ b/core/dr-inference-engine/src/engines.rs @@ -90,12 +90,27 @@ pub fn run() { Source::Bytes(b) => (b.to_vec(), format!("embedded {role:?}")), }; let key = key(rung, &bytes); - if state().lock().unwrap().cache.compiled.contains(&key) { - continue; + { + let s = state().lock().unwrap(); + if s.cache.compiled.contains(&key) || s.cache.refused.contains(&key) { + continue; + } } log::info!("inference: compiling {name} for {}", rung.label()); let started = std::time::Instant::now(); - match crate::session::build(rung, role, &bytes, &cfg) { + let built = match crate::probe::attempt(&cfg, &key, || { + crate::session::build(rung, role, &bytes, &cfg) + }) { + Ok(built) => built, + Err(_) => { + // Refused: the process died inside this compile before. + let mut s = state().lock().unwrap(); + s.cache.refused.insert(key); + crate::probe::write_cache(&s.config, &s.cache); + continue; + } + }; + match built { Ok(session) => { drop(session); let mut s = state().lock().unwrap(); diff --git a/core/dr-inference-engine/src/lib.rs b/core/dr-inference-engine/src/lib.rs index 6846f6e..7b96c47 100644 --- a/core/dr-inference-engine/src/lib.rs +++ b/core/dr-inference-engine/src/lib.rs @@ -355,6 +355,12 @@ struct Cache { /// the fingerprint changes: a wedged driver must not cost every launch /// thirty seconds. failed: Vec<(Rung, String)>, + /// Engine keys whose compile the process died inside, launch after + /// launch (`probe::attempt`). Left on the fallback until the + /// fingerprint changes. Defaulted, so a cache from before this field + /// still reads. + #[serde(default)] + refused: BTreeSet, } struct State { diff --git a/core/dr-inference-engine/src/probe.rs b/core/dr-inference-engine/src/probe.rs index 7282c46..0e0de85 100644 --- a/core/dr-inference-engine/src/probe.rs +++ b/core/dr-inference-engine/src/probe.rs @@ -81,7 +81,11 @@ pub fn run(runtime: Runtime) { log::info!("inference: floor {floor:.1} ms on the CPU provider"); for rung in ladder(cfg.ceiling) { - match time_rung(rung, role, &canonical, &cfg) { + let timed = attempt(&cfg, &format!("probe {}", rung.label()), || { + time_rung(rung, role, &canonical, &cfg) + }) + .and_then(|timed| timed); + match timed { Ok((ms, key)) if ms < floor => { cache.rung = Some(rung); cache.reason = format!("{ms:.1} ms against {floor:.1} ms on the CPU"); @@ -118,6 +122,52 @@ fn finish(cache: Cache) { s.probing = false; } +/// How many launches in a row may die inside one attempt before it is +/// refused. Two, not one: quitting the app while TensorRT spends forty +/// seconds on an engine leaves the same trace as a provider that aborted. +const STRIKES: u32 = 2; + +/// Run `f` — a session build on a provider — with `what` written down +/// first, so that if the provider takes the process with it the next launch +/// knows what to stop trying. +/// +/// A provider can fail by aborting rather than by returning an error: +/// XNNPACK did on SCRFD (§2), and a C++ exception or a panic across the C +/// API is an abort. The probe runs in the app's own process, so a rung that +/// does this once would do it on every launch, before the first photograph +/// is on screen. The file (`attempt` in the cache directory) holds the +/// attempt and how many launches have started it without finishing; +/// finishing, by success or by error, removes it. After [`STRIKES`] the +/// attempt is refused, and the caller records the refusal in the cache, +/// where it lasts until the fingerprint changes like any other failure. +pub fn attempt(cfg: &Config, what: &str, f: impl FnOnce() -> T) -> Result { + if cfg.cache_dir.as_os_str().is_empty() { + return Ok(f()); + } + let path = cfg.cache_dir.join("attempt"); + let died = std::fs::read_to_string(&path) + .ok() + .and_then(|s| { + let (w, n) = s.split_once('\t')?; + (w == what).then(|| n.trim().parse::().ok())? + }) + .unwrap_or(0); + if died >= STRIKES { + log::error!("inference: the app died during `{what}` on the last {died} launches; not trying it again"); + return Err(format!( + "the app died while trying this on {died} launches in a row" + )); + } + if died > 0 { + log::warn!("inference: the last launch died during `{what}`; trying it once more"); + } + let _ = std::fs::create_dir_all(&cfg.cache_dir); + let _ = std::fs::write(&path, format!("{what}\t{}", died + 1)); + let out = f(); + let _ = std::fs::remove_file(&path); + Ok(out) +} + /// The smallest detector, or the smallest model of any role if there is /// none. A ~2 MB detector is the cheapest real test of a provider, and the /// detector is the role the int8 forms exist for — the eye classifiers are @@ -350,3 +400,49 @@ pub fn write_cache(cfg: &Config, cache: &Cache) { } } } + +#[cfg(test)] +mod tests { + use super::*; + + fn a_cache_dir(name: &str) -> Config { + let dir = std::env::temp_dir().join(format!("dr-attempt-{}-{name}", std::process::id())); + let _ = std::fs::remove_dir_all(&dir); + Config { + cache_dir: dir, + ..Config::default() + } + } + + /// What a launch that died inside `what` leaves behind. + fn died_inside(cfg: &Config, what: &str, launches: u32) { + std::fs::create_dir_all(&cfg.cache_dir).unwrap(); + std::fs::write(cfg.cache_dir.join("attempt"), format!("{what}\t{launches}")).unwrap(); + } + + #[test] + fn a_finished_attempt_leaves_no_trace() { + let cfg = a_cache_dir("finished"); + assert_eq!(attempt(&cfg, "probe CoreML", || 7), Ok(7)); + assert!(!cfg.cache_dir.join("attempt").exists()); + } + + #[test] + fn one_death_is_forgiven_and_two_are_not() { + let cfg = a_cache_dir("strikes"); + died_inside(&cfg, "probe CoreML", 1); + assert_eq!(attempt(&cfg, "probe CoreML", || 7), Ok(7)); + + died_inside(&cfg, "probe CoreML", 2); + let mut ran = false; + assert!(attempt(&cfg, "probe CoreML", || ran = true).is_err()); + assert!(!ran, "a refused attempt must not run"); + } + + #[test] + fn another_attempts_deaths_do_not_count() { + let cfg = a_cache_dir("other"); + died_inside(&cfg, "probe TensorRT", 2); + assert_eq!(attempt(&cfg, "probe CUDA", || 7), Ok(7)); + } +} diff --git a/docs/dev/inference.md b/docs/dev/inference.md index e821b7d..8e61106 100644 --- a/docs/dev/inference.md +++ b/docs/dev/inference.md @@ -270,6 +270,14 @@ What the probe may not do: - **Retry a rung that failed within a session.** A failed probe is cached as a failure with the same inputs; the rung is tried again when an input changes. Otherwise a wedged driver means a thirty-second stall on every launch. +- **Crash the app twice for the same reason.** The probe runs in the app's process, and a provider + can fail by aborting rather than by returning an error (XNNPACK on SCRFD, §2). Every session build + on a rung above the CPU — the probe's, and each background compile of §6 — writes what it is + attempting to `attempt` in the cache directory first and removes it after. A launch that finds the + file knows the last one died inside that attempt; after two such launches in a row the attempt is + refused and recorded like any other failure (a rung in `failed`, an engine in `refused`), until + the fingerprint changes. Two, not one, because quitting during a forty-second TensorRT compile + leaves the same file. - **Choose for the user without saying so.** Settings gains one row, *Inference backend*, showing what was chosen and why in one line ("Hexagon NPU · int8 · QNN 2.42"; "CPU · ONNX Runtime 1.30 · TensorRT probe failed: cuDNN 8 required"), with an override to force any lower rung. The about