Stop retrying a provider that took the app down

The probe runs in the app's process, and a provider can fail by aborting
rather than by returning an error — XNNPACK did on SCRFD. A rung that does
that once would do it on every launch, before the first photograph is on
screen.

Every session build above the CPU, the probe's and each background
compile's, now writes what it is attempting to `attempt` in the cache
directory first and removes it after. After two launches in a row that
died inside the same attempt it is refused and recorded — a rung in
`failed`, an engine in the new `refused` — until the fingerprint changes.
Two, not one, because quitting during a TensorRT compile leaves the same
file.

core/dr-inference-engine/src/engines.rs

@@ -90,12 +90,27 @@ pub fn run() {
             Source::Bytes(b) => (b.to_vec(), format!("embedded {role:?}")),
         };
         let key = key(rung, &bytes);
-        if state().lock().unwrap().cache.compiled.contains(&key) {
-            continue;
+        {
+            let s = state().lock().unwrap();
+            if s.cache.compiled.contains(&key) || s.cache.refused.contains(&key) {
+                continue;
+            }
         }
         log::info!("inference: compiling {name} for {}", rung.label());
         let started = std::time::Instant::now();
-        match crate::session::build(rung, role, &bytes, &cfg) {
+        let built = match crate::probe::attempt(&cfg, &key, || {
+            crate::session::build(rung, role, &bytes, &cfg)
+        }) {
+            Ok(built) => built,
+            Err(_) => {
+                // Refused: the process died inside this compile before.
+                let mut s = state().lock().unwrap();
+                s.cache.refused.insert(key);
+                crate::probe::write_cache(&s.config, &s.cache);
+                continue;
+            }
+        };
+        match built {
             Ok(session) => {
                 drop(session);
                 let mut s = state().lock().unwrap();

core/dr-inference-engine/src/lib.rs

@@ -348,6 +348,12 @@ struct Cache {
     /// the fingerprint changes: a wedged driver must not cost every launch
     /// thirty seconds.
     failed: Vec<(Rung, String)>,
+    /// Engine keys whose compile the process died inside, launch after
+    /// launch (`probe::attempt`). Left on the fallback until the
+    /// fingerprint changes. Defaulted, so a cache from before this field
+    /// still reads.
+    #[serde(default)]
+    refused: BTreeSet<String>,
 }
 
 struct State {

core/dr-inference-engine/src/probe.rs

@@ -81,7 +81,11 @@ pub fn run(runtime: Runtime) {
     log::info!("inference: floor {floor:.1} ms on the CPU provider");
 
     for rung in ladder(cfg.ceiling) {
-        match time_rung(rung, role, &canonical, &cfg) {
+        let timed = attempt(&cfg, &format!("probe {}", rung.label()), || {
+            time_rung(rung, role, &canonical, &cfg)
+        })
+        .and_then(|timed| timed);
+        match timed {
             Ok((ms, key)) if ms < floor => {
                 cache.rung = Some(rung);
                 cache.reason = format!("{ms:.1} ms against {floor:.1} ms on the CPU");
@@ -118,6 +122,52 @@ fn finish(cache: Cache) {
     s.probing = false;
 }
 
+/// How many launches in a row may die inside one attempt before it is
+/// refused. Two, not one: quitting the app while TensorRT spends forty
+/// seconds on an engine leaves the same trace as a provider that aborted.
+const STRIKES: u32 = 2;
+
+/// Run `f` — a session build on a provider — with `what` written down
+/// first, so that if the provider takes the process with it the next launch
+/// knows what to stop trying.
+///
+/// A provider can fail by aborting rather than by returning an error:
+/// XNNPACK did on SCRFD (§2), and a C++ exception or a panic across the C
+/// API is an abort. The probe runs in the app's own process, so a rung that
+/// does this once would do it on every launch, before the first photograph
+/// is on screen. The file (`attempt` in the cache directory) holds the
+/// attempt and how many launches have started it without finishing;
+/// finishing, by success or by error, removes it. After [`STRIKES`] the
+/// attempt is refused, and the caller records the refusal in the cache,
+/// where it lasts until the fingerprint changes like any other failure.
+pub fn attempt<T>(cfg: &Config, what: &str, f: impl FnOnce() -> T) -> Result<T, String> {
+    if cfg.cache_dir.as_os_str().is_empty() {
+        return Ok(f());
+    }
+    let path = cfg.cache_dir.join("attempt");
+    let died = std::fs::read_to_string(&path)
+        .ok()
+        .and_then(|s| {
+            let (w, n) = s.split_once('\t')?;
+            (w == what).then(|| n.trim().parse::<u32>().ok())?
+        })
+        .unwrap_or(0);
+    if died >= STRIKES {
+        log::error!("inference: the app died during `{what}` on the last {died} launches; not trying it again");
+        return Err(format!(
+            "the app died while trying this on {died} launches in a row"
+        ));
+    }
+    if died > 0 {
+        log::warn!("inference: the last launch died during `{what}`; trying it once more");
+    }
+    let _ = std::fs::create_dir_all(&cfg.cache_dir);
+    let _ = std::fs::write(&path, format!("{what}\t{}", died + 1));
+    let out = f();
+    let _ = std::fs::remove_file(&path);
+    Ok(out)
+}
+
 /// The smallest detector, or the smallest model of any role if there is
 /// none. A ~2 MB detector is the cheapest real test of a provider, and the
 /// detector is the role the int8 forms exist for — the eye classifiers are
@@ -338,3 +388,49 @@ pub fn write_cache(cfg: &Config, cache: &Cache) {
         }
     }
 }
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    fn a_cache_dir(name: &str) -> Config {
+        let dir = std::env::temp_dir().join(format!("dr-attempt-{}-{name}", std::process::id()));
+        let _ = std::fs::remove_dir_all(&dir);
+        Config {
+            cache_dir: dir,
+            ..Config::default()
+        }
+    }
+
+    /// What a launch that died inside `what` leaves behind.
+    fn died_inside(cfg: &Config, what: &str, launches: u32) {
+        std::fs::create_dir_all(&cfg.cache_dir).unwrap();
+        std::fs::write(cfg.cache_dir.join("attempt"), format!("{what}\t{launches}")).unwrap();
+    }
+
+    #[test]
+    fn a_finished_attempt_leaves_no_trace() {
+        let cfg = a_cache_dir("finished");
+        assert_eq!(attempt(&cfg, "probe CoreML", || 7), Ok(7));
+        assert!(!cfg.cache_dir.join("attempt").exists());
+    }
+
+    #[test]
+    fn one_death_is_forgiven_and_two_are_not() {
+        let cfg = a_cache_dir("strikes");
+        died_inside(&cfg, "probe CoreML", 1);
+        assert_eq!(attempt(&cfg, "probe CoreML", || 7), Ok(7));
+
+        died_inside(&cfg, "probe CoreML", 2);
+        let mut ran = false;
+        assert!(attempt(&cfg, "probe CoreML", || ran = true).is_err());
+        assert!(!ran, "a refused attempt must not run");
+    }
+
+    #[test]
+    fn another_attempts_deaths_do_not_count() {
+        let cfg = a_cache_dir("other");
+        died_inside(&cfg, "probe TensorRT", 2);
+        assert_eq!(attempt(&cfg, "probe CUDA", || 7), Ok(7));
+    }
+}