From ef71bb328954302ab5f3385d8295364d72472886 Mon Sep 17 00:00:00 2001 From: Duncan Tourolle Date: Sun, 4 Oct 2026 19:07:39 -0400 Subject: [PATCH] Time OpenVINO and WebGPU in ep_probe MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The Intel and vendor-neutral rungs need a measurement before they join the ladder (docs/dev/inference.md §2). Both register through the generic key/value entry point with the option names ONNX Runtime reads at the wheel's version: OpenVINO 1.24 (`openvino_provider_factory.cc`), WebGPU 1.27 (`webgpu_provider_options.h`, prefixed by the runtime). DARKROOM_EPS narrows the list to the families a runtime carries. --- core/dr-inference-engine/examples/ep_probe.rs | 134 +++++++++++++++++- 1 file changed, 127 insertions(+), 7 deletions(-) diff --git a/core/dr-inference-engine/examples/ep_probe.rs b/core/dr-inference-engine/examples/ep_probe.rs index bb1868e..bdb0e62 100644 --- a/core/dr-inference-engine/examples/ep_probe.rs +++ b/core/dr-inference-engine/examples/ep_probe.rs @@ -24,6 +24,14 @@ enum Ep { Cpu, MiGraphX, MiGraphXFp16, + OpenVinoCpu, + OpenVinoGpu, + OpenVinoGpuFp16, + OpenVinoNpu, + /// Dawn's low-power adapter: the integrated GPU on a hybrid machine. + WebGpuLow, + /// Dawn's high-performance adapter: the discrete one, if there is one. + WebGpuHigh, } impl Ep { @@ -32,20 +40,110 @@ impl Ep { Ep::Cpu => "CPU", Ep::MiGraphX => "MIGraphX f32", Ep::MiGraphXFp16 => "MIGraphX fp16", + Ep::OpenVinoCpu => "OpenVINO CPU", + Ep::OpenVinoGpu => "OpenVINO GPU", + Ep::OpenVinoGpuFp16 => "OpenVINO GPU16", + Ep::OpenVinoNpu => "OpenVINO NPU", + Ep::WebGpuLow => "WebGPU low", + Ep::WebGpuHigh => "WebGPU high", } } + + /// Whether a second build reads what the first one compiled. + fn caches(self) -> bool { + matches!( + self, + Ep::MiGraphX + | Ep::MiGraphXFp16 + | Ep::OpenVinoGpu + | Ep::OpenVinoGpuFp16 + | Ep::OpenVinoNpu + ) + } } fn build(ep: Ep, bytes: &[u8], threads: usize, cache: &Path) -> ort::Result { let mut b = ort::session::Session::builder()?.with_intra_threads(threads)?; + let dir = |sub: &str| { + let d = cache.join(sub); + let _ = std::fs::create_dir_all(&d); + d.to_string_lossy().into_owned() + }; match ep { Ep::Cpu => {} Ep::MiGraphX => migraphx(&mut b, false, &cache.join("f32"))?, Ep::MiGraphXFp16 => migraphx(&mut b, true, &cache.join("fp16"))?, + // Option names as `openvino_provider_factory.cc` reads them at 1.24. + Ep::OpenVinoCpu => append(&mut b, c"OpenVINO", &[("device_type", "CPU".into())])?, + Ep::OpenVinoGpu => append( + &mut b, + c"OpenVINO", + &[ + ("device_type", "GPU".into()), + ("precision", "FP32".into()), + ("cache_dir", dir("ov-gpu-f32")), + ], + )?, + Ep::OpenVinoGpuFp16 => append( + &mut b, + c"OpenVINO", + &[ + ("device_type", "GPU".into()), + ("precision", "FP16".into()), + ("cache_dir", dir("ov-gpu-fp16")), + ], + )?, + Ep::OpenVinoNpu => append( + &mut b, + c"OpenVINO", + &[("device_type", "NPU".into()), ("cache_dir", dir("ov-npu"))], + )?, + // `webgpu_provider_options.h` at 1.27; the runtime prefixes the key. + Ep::WebGpuLow => append( + &mut b, + c"WebGPU", + &[("powerPreference", "low-power".into())], + )?, + Ep::WebGpuHigh => append( + &mut b, + c"WebGPU", + &[("powerPreference", "high-performance".into())], + )?, } b.commit_from_memory(bytes) } +/// Any provider through the generic key/value entry point. +fn append( + b: &mut ort::session::builder::SessionBuilder, + name: &std::ffi::CStr, + options: &[(&str, String)], +) -> ort::Result<()> { + use ort::AsPointer; + use std::ffi::CString; + let keys: Vec = options + .iter() + .map(|(k, _)| CString::new(*k).unwrap()) + .collect(); + let values: Vec = options + .iter() + .map(|(_, v)| CString::new(v.as_bytes()).unwrap()) + .collect(); + let key_ptrs: Vec<_> = keys.iter().map(|k| k.as_ptr()).collect(); + let value_ptrs: Vec<_> = values.iter().map(|v| v.as_ptr()).collect(); + // SAFETY: as `migraphx` below. + unsafe { + let status = (ort::api().SessionOptionsAppendExecutionProvider)( + b.ptr_mut(), + name.as_ptr(), + key_ptrs.as_ptr(), + value_ptrs.as_ptr(), + keys.len(), + ); + ort::Error::result_from_status(status) + } +} + /// Register MIGraphX through the generic key/value API. `ort`'s own /// builder fills the legacy `OrtMIGraphXProviderOptions`, which 1.29 reads /// for its precision flags and nothing else: the model cache directory — @@ -155,13 +253,35 @@ fn main() { // A compiling provider is built twice: the second build reads the // program the first wrote, and its time is what a launch after the // first costs. - let plan = [ - (Ep::Cpu, false), - (Ep::MiGraphX, false), - (Ep::MiGraphX, true), - (Ep::MiGraphXFp16, false), - (Ep::MiGraphXFp16, true), - ]; + // DARKROOM_EPS narrows the list (`cpu,openvino,webgpu,migraphx`); + // a runtime without a provider fails its build in a millisecond + // anyway, so the default is all of them. + let wanted = std::env::var("DARKROOM_EPS").unwrap_or_default(); + let on = |family: &str| wanted.is_empty() || wanted.split(',').any(|w| w == family); + let mut plan = Vec::new(); + for (family, eps) in [ + ("cpu", &[Ep::Cpu][..]), + ("migraphx", &[Ep::MiGraphX, Ep::MiGraphXFp16][..]), + ( + "openvino", + &[ + Ep::OpenVinoCpu, + Ep::OpenVinoGpu, + Ep::OpenVinoGpuFp16, + Ep::OpenVinoNpu, + ][..], + ), + ("webgpu", &[Ep::WebGpuLow, Ep::WebGpuHigh][..]), + ] { + if on(family) { + for &ep in eps { + plan.push((ep, false)); + if ep.caches() { + plan.push((ep, true)); + } + } + } + } for (ep, cached) in plan { let started = Instant::now(); match build(ep, &bytes, threads, &cache) {