diff --git a/core/dr-inference-engine/src/engines.rs b/core/dr-inference-engine/src/engines.rs index 5af4887..3a8670a 100644 --- a/core/dr-inference-engine/src/engines.rs +++ b/core/dr-inference-engine/src/engines.rs @@ -90,12 +90,27 @@ pub fn run() { Source::Bytes(b) => (b.to_vec(), format!("embedded {role:?}")), }; let key = key(rung, &bytes); - if state().lock().unwrap().cache.compiled.contains(&key) { - continue; + { + let s = state().lock().unwrap(); + if s.cache.compiled.contains(&key) || s.cache.refused.contains(&key) { + continue; + } } log::info!("inference: compiling {name} for {}", rung.label()); let started = std::time::Instant::now(); - match crate::session::build(rung, role, &bytes, &cfg) { + let built = match crate::probe::attempt(&cfg, &key, || { + crate::session::build(rung, role, &bytes, &cfg) + }) { + Ok(built) => built, + Err(_) => { + // Refused: the process died inside this compile before. + let mut s = state().lock().unwrap(); + s.cache.refused.insert(key); + crate::probe::write_cache(&s.config, &s.cache); + continue; + } + }; + match built { Ok(session) => { drop(session); let mut s = state().lock().unwrap(); diff --git a/core/dr-inference-engine/src/lib.rs b/core/dr-inference-engine/src/lib.rs index 6846f6e..7b96c47 100644 --- a/core/dr-inference-engine/src/lib.rs +++ b/core/dr-inference-engine/src/lib.rs @@ -355,6 +355,12 @@ struct Cache { /// the fingerprint changes: a wedged driver must not cost every launch /// thirty seconds. failed: Vec<(Rung, String)>, + /// Engine keys whose compile the process died inside, launch after + /// launch (`probe::attempt`). Left on the fallback until the + /// fingerprint changes. Defaulted, so a cache from before this field + /// still reads. + #[serde(default)] + refused: BTreeSet, } struct State { diff --git a/core/dr-inference-engine/src/probe.rs b/core/dr-inference-engine/src/probe.rs index 7282c46..0e0de85 100644 --- a/core/dr-inference-engine/src/probe.rs +++ b/core/dr-inference-engine/src/probe.rs @@ -81,7 +81,11 @@ pub fn run(runtime: Runtime) { log::info!("inference: floor {floor:.1} ms on the CPU provider"); for rung in ladder(cfg.ceiling) { - match time_rung(rung, role, &canonical, &cfg) { + let timed = attempt(&cfg, &format!("probe {}", rung.label()), || { + time_rung(rung, role, &canonical, &cfg) + }) + .and_then(|timed| timed); + match timed { Ok((ms, key)) if ms < floor => { cache.rung = Some(rung); cache.reason = format!("{ms:.1} ms against {floor:.1} ms on the CPU"); @@ -118,6 +122,52 @@ fn finish(cache: Cache) { s.probing = false; } +/// How many launches in a row may die inside one attempt before it is +/// refused. Two, not one: quitting the app while TensorRT spends forty +/// seconds on an engine leaves the same trace as a provider that aborted. +const STRIKES: u32 = 2; + +/// Run `f` — a session build on a provider — with `what` written down +/// first, so that if the provider takes the process with it the next launch +/// knows what to stop trying. +/// +/// A provider can fail by aborting rather than by returning an error: +/// XNNPACK did on SCRFD (§2), and a C++ exception or a panic across the C +/// API is an abort. The probe runs in the app's own process, so a rung that +/// does this once would do it on every launch, before the first photograph +/// is on screen. The file (`attempt` in the cache directory) holds the +/// attempt and how many launches have started it without finishing; +/// finishing, by success or by error, removes it. After [`STRIKES`] the +/// attempt is refused, and the caller records the refusal in the cache, +/// where it lasts until the fingerprint changes like any other failure. +pub fn attempt(cfg: &Config, what: &str, f: impl FnOnce() -> T) -> Result { + if cfg.cache_dir.as_os_str().is_empty() { + return Ok(f()); + } + let path = cfg.cache_dir.join("attempt"); + let died = std::fs::read_to_string(&path) + .ok() + .and_then(|s| { + let (w, n) = s.split_once('\t')?; + (w == what).then(|| n.trim().parse::().ok())? + }) + .unwrap_or(0); + if died >= STRIKES { + log::error!("inference: the app died during `{what}` on the last {died} launches; not trying it again"); + return Err(format!( + "the app died while trying this on {died} launches in a row" + )); + } + if died > 0 { + log::warn!("inference: the last launch died during `{what}`; trying it once more"); + } + let _ = std::fs::create_dir_all(&cfg.cache_dir); + let _ = std::fs::write(&path, format!("{what}\t{}", died + 1)); + let out = f(); + let _ = std::fs::remove_file(&path); + Ok(out) +} + /// The smallest detector, or the smallest model of any role if there is /// none. A ~2 MB detector is the cheapest real test of a provider, and the /// detector is the role the int8 forms exist for — the eye classifiers are @@ -350,3 +400,49 @@ pub fn write_cache(cfg: &Config, cache: &Cache) { } } } + +#[cfg(test)] +mod tests { + use super::*; + + fn a_cache_dir(name: &str) -> Config { + let dir = std::env::temp_dir().join(format!("dr-attempt-{}-{name}", std::process::id())); + let _ = std::fs::remove_dir_all(&dir); + Config { + cache_dir: dir, + ..Config::default() + } + } + + /// What a launch that died inside `what` leaves behind. + fn died_inside(cfg: &Config, what: &str, launches: u32) { + std::fs::create_dir_all(&cfg.cache_dir).unwrap(); + std::fs::write(cfg.cache_dir.join("attempt"), format!("{what}\t{launches}")).unwrap(); + } + + #[test] + fn a_finished_attempt_leaves_no_trace() { + let cfg = a_cache_dir("finished"); + assert_eq!(attempt(&cfg, "probe CoreML", || 7), Ok(7)); + assert!(!cfg.cache_dir.join("attempt").exists()); + } + + #[test] + fn one_death_is_forgiven_and_two_are_not() { + let cfg = a_cache_dir("strikes"); + died_inside(&cfg, "probe CoreML", 1); + assert_eq!(attempt(&cfg, "probe CoreML", || 7), Ok(7)); + + died_inside(&cfg, "probe CoreML", 2); + let mut ran = false; + assert!(attempt(&cfg, "probe CoreML", || ran = true).is_err()); + assert!(!ran, "a refused attempt must not run"); + } + + #[test] + fn another_attempts_deaths_do_not_count() { + let cfg = a_cache_dir("other"); + died_inside(&cfg, "probe TensorRT", 2); + assert_eq!(attempt(&cfg, "probe CUDA", || 7), Ok(7)); + } +} diff --git a/docs/dev/inference.md b/docs/dev/inference.md index e821b7d..8e61106 100644 --- a/docs/dev/inference.md +++ b/docs/dev/inference.md @@ -270,6 +270,14 @@ What the probe may not do: - **Retry a rung that failed within a session.** A failed probe is cached as a failure with the same inputs; the rung is tried again when an input changes. Otherwise a wedged driver means a thirty-second stall on every launch. +- **Crash the app twice for the same reason.** The probe runs in the app's process, and a provider + can fail by aborting rather than by returning an error (XNNPACK on SCRFD, §2). Every session build + on a rung above the CPU — the probe's, and each background compile of §6 — writes what it is + attempting to `attempt` in the cache directory first and removes it after. A launch that finds the + file knows the last one died inside that attempt; after two such launches in a row the attempt is + refused and recorded like any other failure (a rung in `failed`, an engine in `refused`), until + the fingerprint changes. Two, not one, because quitting during a forty-second TensorRT compile + leaves the same file. - **Choose for the user without saying so.** Settings gains one row, *Inference backend*, showing what was chosen and why in one line ("Hexagon NPU · int8 · QNN 2.42"; "CPU · ONNX Runtime 1.30 · TensorRT probe failed: cuDNN 8 required"), with an override to force any lower rung. The about