Time OpenVINO and WebGPU in ep_probe
The Intel and vendor-neutral rungs need a measurement before they join the ladder (docs/dev/inference.md §2). Both register through the generic key/value entry point with the option names ONNX Runtime reads at the wheel's version: OpenVINO 1.24 (`openvino_provider_factory.cc`), WebGPU 1.27 (`webgpu_provider_options.h`, prefixed by the runtime). DARKROOM_EPS narrows the list to the families a runtime carries.
This commit is contained in:
@@ -24,6 +24,14 @@ enum Ep {
|
||||
Cpu,
|
||||
MiGraphX,
|
||||
MiGraphXFp16,
|
||||
OpenVinoCpu,
|
||||
OpenVinoGpu,
|
||||
OpenVinoGpuFp16,
|
||||
OpenVinoNpu,
|
||||
/// Dawn's low-power adapter: the integrated GPU on a hybrid machine.
|
||||
WebGpuLow,
|
||||
/// Dawn's high-performance adapter: the discrete one, if there is one.
|
||||
WebGpuHigh,
|
||||
}
|
||||
|
||||
impl Ep {
|
||||
@@ -32,20 +40,110 @@ impl Ep {
|
||||
Ep::Cpu => "CPU",
|
||||
Ep::MiGraphX => "MIGraphX f32",
|
||||
Ep::MiGraphXFp16 => "MIGraphX fp16",
|
||||
Ep::OpenVinoCpu => "OpenVINO CPU",
|
||||
Ep::OpenVinoGpu => "OpenVINO GPU",
|
||||
Ep::OpenVinoGpuFp16 => "OpenVINO GPU16",
|
||||
Ep::OpenVinoNpu => "OpenVINO NPU",
|
||||
Ep::WebGpuLow => "WebGPU low",
|
||||
Ep::WebGpuHigh => "WebGPU high",
|
||||
}
|
||||
}
|
||||
|
||||
/// Whether a second build reads what the first one compiled.
|
||||
fn caches(self) -> bool {
|
||||
matches!(
|
||||
self,
|
||||
Ep::MiGraphX
|
||||
| Ep::MiGraphXFp16
|
||||
| Ep::OpenVinoGpu
|
||||
| Ep::OpenVinoGpuFp16
|
||||
| Ep::OpenVinoNpu
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
fn build(ep: Ep, bytes: &[u8], threads: usize, cache: &Path) -> ort::Result<ort::session::Session> {
|
||||
let mut b = ort::session::Session::builder()?.with_intra_threads(threads)?;
|
||||
let dir = |sub: &str| {
|
||||
let d = cache.join(sub);
|
||||
let _ = std::fs::create_dir_all(&d);
|
||||
d.to_string_lossy().into_owned()
|
||||
};
|
||||
match ep {
|
||||
Ep::Cpu => {}
|
||||
Ep::MiGraphX => migraphx(&mut b, false, &cache.join("f32"))?,
|
||||
Ep::MiGraphXFp16 => migraphx(&mut b, true, &cache.join("fp16"))?,
|
||||
// Option names as `openvino_provider_factory.cc` reads them at 1.24.
|
||||
Ep::OpenVinoCpu => append(&mut b, c"OpenVINO", &[("device_type", "CPU".into())])?,
|
||||
Ep::OpenVinoGpu => append(
|
||||
&mut b,
|
||||
c"OpenVINO",
|
||||
&[
|
||||
("device_type", "GPU".into()),
|
||||
("precision", "FP32".into()),
|
||||
("cache_dir", dir("ov-gpu-f32")),
|
||||
],
|
||||
)?,
|
||||
Ep::OpenVinoGpuFp16 => append(
|
||||
&mut b,
|
||||
c"OpenVINO",
|
||||
&[
|
||||
("device_type", "GPU".into()),
|
||||
("precision", "FP16".into()),
|
||||
("cache_dir", dir("ov-gpu-fp16")),
|
||||
],
|
||||
)?,
|
||||
Ep::OpenVinoNpu => append(
|
||||
&mut b,
|
||||
c"OpenVINO",
|
||||
&[("device_type", "NPU".into()), ("cache_dir", dir("ov-npu"))],
|
||||
)?,
|
||||
// `webgpu_provider_options.h` at 1.27; the runtime prefixes the key.
|
||||
Ep::WebGpuLow => append(
|
||||
&mut b,
|
||||
c"WebGPU",
|
||||
&[("powerPreference", "low-power".into())],
|
||||
)?,
|
||||
Ep::WebGpuHigh => append(
|
||||
&mut b,
|
||||
c"WebGPU",
|
||||
&[("powerPreference", "high-performance".into())],
|
||||
)?,
|
||||
}
|
||||
b.commit_from_memory(bytes)
|
||||
}
|
||||
|
||||
/// Any provider through the generic key/value entry point.
|
||||
fn append(
|
||||
b: &mut ort::session::builder::SessionBuilder,
|
||||
name: &std::ffi::CStr,
|
||||
options: &[(&str, String)],
|
||||
) -> ort::Result<()> {
|
||||
use ort::AsPointer;
|
||||
use std::ffi::CString;
|
||||
let keys: Vec<CString> = options
|
||||
.iter()
|
||||
.map(|(k, _)| CString::new(*k).unwrap())
|
||||
.collect();
|
||||
let values: Vec<CString> = options
|
||||
.iter()
|
||||
.map(|(_, v)| CString::new(v.as_bytes()).unwrap())
|
||||
.collect();
|
||||
let key_ptrs: Vec<_> = keys.iter().map(|k| k.as_ptr()).collect();
|
||||
let value_ptrs: Vec<_> = values.iter().map(|v| v.as_ptr()).collect();
|
||||
// SAFETY: as `migraphx` below.
|
||||
unsafe {
|
||||
let status = (ort::api().SessionOptionsAppendExecutionProvider)(
|
||||
b.ptr_mut(),
|
||||
name.as_ptr(),
|
||||
key_ptrs.as_ptr(),
|
||||
value_ptrs.as_ptr(),
|
||||
keys.len(),
|
||||
);
|
||||
ort::Error::result_from_status(status)
|
||||
}
|
||||
}
|
||||
|
||||
/// Register MIGraphX through the generic key/value API. `ort`'s own
|
||||
/// builder fills the legacy `OrtMIGraphXProviderOptions`, which 1.29 reads
|
||||
/// for its precision flags and nothing else: the model cache directory —
|
||||
@@ -155,13 +253,35 @@ fn main() {
|
||||
// A compiling provider is built twice: the second build reads the
|
||||
// program the first wrote, and its time is what a launch after the
|
||||
// first costs.
|
||||
let plan = [
|
||||
(Ep::Cpu, false),
|
||||
(Ep::MiGraphX, false),
|
||||
(Ep::MiGraphX, true),
|
||||
(Ep::MiGraphXFp16, false),
|
||||
(Ep::MiGraphXFp16, true),
|
||||
];
|
||||
// DARKROOM_EPS narrows the list (`cpu,openvino,webgpu,migraphx`);
|
||||
// a runtime without a provider fails its build in a millisecond
|
||||
// anyway, so the default is all of them.
|
||||
let wanted = std::env::var("DARKROOM_EPS").unwrap_or_default();
|
||||
let on = |family: &str| wanted.is_empty() || wanted.split(',').any(|w| w == family);
|
||||
let mut plan = Vec::new();
|
||||
for (family, eps) in [
|
||||
("cpu", &[Ep::Cpu][..]),
|
||||
("migraphx", &[Ep::MiGraphX, Ep::MiGraphXFp16][..]),
|
||||
(
|
||||
"openvino",
|
||||
&[
|
||||
Ep::OpenVinoCpu,
|
||||
Ep::OpenVinoGpu,
|
||||
Ep::OpenVinoGpuFp16,
|
||||
Ep::OpenVinoNpu,
|
||||
][..],
|
||||
),
|
||||
("webgpu", &[Ep::WebGpuLow, Ep::WebGpuHigh][..]),
|
||||
] {
|
||||
if on(family) {
|
||||
for &ep in eps {
|
||||
plan.push((ep, false));
|
||||
if ep.caches() {
|
||||
plan.push((ep, true));
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
for (ep, cached) in plan {
|
||||
let started = Instant::now();
|
||||
match build(ep, &bytes, threads, &cache) {
|
||||
|
||||
Reference in New Issue
Block a user