diff --git a/core/dr-inference-engine/src/lib.rs b/core/dr-inference-engine/src/lib.rs index 9a91c71..721bb28 100644 --- a/core/dr-inference-engine/src/lib.rs +++ b/core/dr-inference-engine/src/lib.rs @@ -412,6 +412,12 @@ struct Cache { /// still reads. #[serde(default)] refused: BTreeSet, + /// Probes run under this fingerprint (`probe::run`): a fall-back to the + /// CPU is re-probed until there have been `RETRIES`. Defaulted, so a + /// cache from 0.22.0 or before — which may hold exactly such a verdict — + /// probes again. + #[serde(default)] + attempts: u32, } struct State { diff --git a/core/dr-inference-engine/src/probe.rs b/core/dr-inference-engine/src/probe.rs index 167a8a8..be8f895 100644 --- a/core/dr-inference-engine/src/probe.rs +++ b/core/dr-inference-engine/src/probe.rs @@ -31,6 +31,38 @@ fn ladder(ceiling: Option) -> Vec { .collect() } +/// Probes under one fingerprint that may end on the CPU after an +/// accelerator failed or lost, before that answer is kept. +const RETRIES: u32 = 3; + +/// What a cached probe result is good for. +#[derive(Debug, PartialEq)] +enum Reuse { + /// Use it as it is. + Keep, + /// Probe again: it fell back to the CPU after this many probes. + Again(u32), + /// Another device, runtime or model set: probe from the start. + Fresh, +} + +/// The CPU because an accelerator failed or lost is asked again on the next +/// launches, a few times: a failure can be a moment's (QNN could not create +/// its device on 0.22.0's first launch after the update), and keeping it for +/// good left the tablet's every model on the CPU. Bounded, so a wedged +/// driver costs a few launches, not all. +fn reuse(cached: &Cache, fingerprint: &str) -> Reuse { + if cached.fingerprint != fingerprint || cached.rung.is_none() { + return Reuse::Fresh; + } + let fell_back = cached.rung == Some(Rung::Cpu) && !cached.failed.is_empty(); + if fell_back && cached.attempts < RETRIES { + Reuse::Again(cached.attempts) + } else { + Reuse::Keep + } +} + /// The probe body. Sets the cache and clears `probing` when done; never /// panics out, because a failed probe is a result (the floor) and not an /// error. @@ -38,20 +70,33 @@ pub fn run(runtime: Runtime) { let cfg = state().lock().unwrap().config.clone(); let fingerprint = fingerprint(&runtime, &cfg); + let mut attempts = 0; if let Some(cached) = read_cache(&cfg) { - if cached.fingerprint == fingerprint && cached.rung.is_some() { - log::info!( - "inference: cached selection {} ({})", - cached.rung.unwrap().label(), - cached.reason - ); - finish(cached); - return; + match reuse(&cached, &fingerprint) { + Reuse::Keep => { + log::info!( + "inference: cached selection {} ({})", + cached.rung.map_or("?", |r| r.label()), + cached.reason + ); + finish(cached); + return; + } + Reuse::Again(n) => { + attempts = n; + log::info!( + "inference: probing again after falling back to the CPU ({}), attempt {} of {RETRIES}", + cached.reason, + n + 1 + ); + } + Reuse::Fresh => {} } } let mut cache = Cache { fingerprint, + attempts: attempts + 1, ..Cache::default() }; @@ -212,8 +257,8 @@ fn time_rung( } let bytes = std::fs::read(&path).map_err(|e| e.to_string())?; let started = Instant::now(); - let mut session = - crate::session::build(rung, role, &bytes, cfg).map_err(|e| first_line(&e.to_string()))?; + let mut session = crate::session::build_probe(rung, role, &bytes, cfg) + .map_err(|e| first_line(&e.to_string()))?; log::info!( "inference: {} session built in {:.1} s", rung.label(), @@ -494,4 +539,34 @@ mod tests { died_inside(&cfg, "probe TensorRT", 2); assert_eq!(attempt(&cfg, "probe CUDA", || 7), Ok(7)); } + + /// The tablet's cache after 0.22.0's first launch, as 0.22.0 wrote it: + /// no `attempts`, the Hexagon "rejected", the CPU selected. + const TABLET: &str = r#"{"fingerprint":"f","rung":"Cpu","reason":"Hexagon NPU 28.5 ms, slower than the CPU's 19.4 ms","compiled":[],"failed":[["Hexagon","28.5 ms, slower than the CPU's 19.4 ms"]]}"#; + + #[test] + fn a_fall_back_to_the_cpu_is_probed_again_a_few_times() { + let mut cache: Cache = serde_json::from_str(TABLET).unwrap(); + assert_eq!(cache.attempts, 0, "a 0.22.0 cache reads as never retried"); + assert_eq!(reuse(&cache, "f"), Reuse::Again(0)); + cache.attempts = RETRIES - 1; + assert_eq!(reuse(&cache, "f"), Reuse::Again(RETRIES - 1)); + cache.attempts = RETRIES; + assert_eq!(reuse(&cache, "f"), Reuse::Keep, "then it is kept"); + } + + #[test] + fn an_accelerator_chosen_or_a_cpu_only_device_is_kept() { + let mut cache: Cache = serde_json::from_str(TABLET).unwrap(); + cache.rung = Some(Rung::Hexagon); + assert_eq!(reuse(&cache, "f"), Reuse::Keep); + cache.rung = Some(Rung::Cpu); + cache.failed.clear(); + assert_eq!( + reuse(&cache, "f"), + Reuse::Keep, + "nothing failed: the only rung" + ); + assert_eq!(reuse(&cache, "other"), Reuse::Fresh); + } } diff --git a/core/dr-inference-engine/src/session.rs b/core/dr-inference-engine/src/session.rs index fdc4ed2..b60dbef 100644 --- a/core/dr-inference-engine/src/session.rs +++ b/core/dr-inference-engine/src/session.rs @@ -13,6 +13,30 @@ use crate::{Config, Role, Rung}; /// its clock instead (§4): a provider that hands real work to the CPU is /// slower than the CPU floor and rejected by the same measurement. pub fn build(rung: Rung, role: Role, bytes: &[u8], cfg: &Config) -> ort::Result { + build_with(rung, role, bytes, cfg, false) +} + +/// [`build`] for the probe: on the Hexagon, a session that cannot put the +/// whole graph on the NPU fails instead of running the rest on the CPU. +/// +/// The probe times a rung by its session, and a QNN provider that could not +/// create its device still builds one — with every node on the CPU behind +/// it. 0.22.0's first launch on the tablet timed that (28.5 ms against the +/// CPU's own 19.4) and put every model on the CPU. Only the probe is strict: +/// some shipped graphs keep a few nodes on the CPU on purpose +/// (`tools/quantise-models.py`, `float_nodes`), and the probe's detector is +/// not one of them. +pub fn build_probe(rung: Rung, role: Role, bytes: &[u8], cfg: &Config) -> ort::Result { + build_with(rung, role, bytes, cfg, rung == Rung::Hexagon) +} + +fn build_with( + rung: Rung, + role: Role, + bytes: &[u8], + cfg: &Config, + strict: bool, +) -> ort::Result { // No optimisation level named. ONNX Runtime's default is already its // fullest, and on tract any level but "disabled" means `into_optimized`, // whose optimiser divides by zero inside yolo26n-seg (tract-data @@ -22,6 +46,9 @@ pub fn build(rung: Rung, role: Role, bytes: &[u8], cfg: &Config) -> ort::Result< if crate::api::runtime().is_native() { b = with_runtime_log(b)?; } + if strict { + b = b.with_config_entry("session.disable_cpu_ep_fallback", "1")?; + } // A Hexagon session loads the compiled context when there is one and // compiles it from the model when there is not; the engine thread is // what makes the second case rare (§6).