//! Where a control is, asked by its name — for the scripts that record the //! manual, and for nobody else. //! //! `tools/manual/drive.py` moves a real pointer over the window with xdotool, //! and until this existed it aimed at pixel coordinates written into //! `scenes.py` by hand. A panel that grew a row moved every slider below it, //! and the recording went on clicking where the slider used to be: no error, //! just a picture of the wrong thing. This answers "where is *Exposure*?" from //! the element tree itself, by the same accessible label a screen reader reads //! (NFR-A11Y-2), so a script that names controls survives a layout change the //! way a screen-reader user does. //! //! **Absent from every normal build.** The module exists only under the //! `automation` cargo feature, which no default build enables and //! `tools/manual/record.sh` does; and even in a build that has it, nothing //! listens unless `DR_AUTOMATION` names a socket path. It only reads the tree //! and answers: the input itself still arrives as ordinary pointer and key //! events from outside, so a scene exercises exactly what a person's hand //! would, gesture recognisers and all. //! //! ## The protocol //! //! One request per line on a Unix socket, one JSON line back: //! //! - `ping` — `"ok"`. //! - `locate LABEL` — every element whose accessible label is exactly //! `LABEL`: `[{"label","id","type","role","x","y","w","h","opacity", //! "enabled","checked","value","min","max"}, …]`, in physical window //! pixels, tree order. //! - `labels [SUBSTRING]` — every labelled element, or those whose label //! contains `SUBSTRING`; the same objects. For finding what a scene can //! name. //! - `locate-id ID` — the same, for elements by their id in the markup //! (`canvas-image`, or qualified, `CropOverlay::move-area`) or by //! component type (`Timeline`). For the few things a scene has to aim at //! that are not controls — the photograph, the crop rectangle — and so //! have no accessible name to give. //! - `ids [SUBSTRING]` — every element with an id, for finding those. //! - `window` — `{"w","h","scale"}`. //! - `canvas` — `{"w","h"}` of the develop canvas's current image, so a //! script can place a point on the photograph within `canvas-image`. //! //! Only elements Slint considers visible are answered — not hidden, not //! clipped away by a scrolled list — so "on screen" is the default. //! //! Waiting is the client's job (poll `locate`), which keeps this stateless. use crate::executors::{self, Executor}; use std::io::{BufRead, BufReader, Write}; use std::os::unix::net::{UnixListener, UnixStream}; use std::sync::mpsc; use std::time::Duration; use i_slint_backend_testing::{ElementHandle, ElementRoot as _}; use serde_json::{json, Value}; use slint::ComponentHandle as _; use crate::AppWindow; /// Listen on `DR_AUTOMATION` if it is set. Called once, before the event loop. pub(crate) fn attach(window: &AppWindow) { let Some(path) = std::env::var_os("DR_AUTOMATION") else { return; }; let _ = std::fs::remove_file(&path); let listener = match UnixListener::bind(&path) { Ok(l) => l, Err(e) => { log::warn!("automation: cannot listen on {path:?}: {e}"); return; } }; log::info!("automation: listening on {path:?}"); let weak = window.as_weak(); executors::try_spawn(Executor::Io, "automation", move || { for stream in listener.incoming().flatten() { let weak = weak.clone(); executors::spawn(Executor::Io, "auto-conn", move || serve(stream, weak)); } }) .ok(); } fn serve(stream: UnixStream, weak: slint::Weak) { let Ok(mut out) = stream.try_clone() else { return; }; for line in BufReader::new(stream).lines() { let Ok(line) = line else { return }; let (tx, rx) = mpsc::channel(); let request = line.clone(); let posted = weak.upgrade_in_event_loop(move |w| { let _ = tx.send(answer(&w, &request)); }); let reply = match posted { Ok(()) => rx .recv_timeout(Duration::from_secs(10)) .unwrap_or_else(|_| json!({"error": "the event loop did not answer"})), Err(e) => json!({"error": e.to_string()}), }; if writeln!(out, "{reply}").is_err() { return; } } } fn answer(window: &AppWindow, request: &str) -> Value { let (verb, rest) = request.split_once(' ').unwrap_or((request, "")); let scale = window.window().scale_factor(); match verb { "ping" => json!("ok"), "window" => { let size = window.window().size(); json!({"w": size.width, "h": size.height, "scale": scale}) } "locate" => Value::Array( ElementHandle::find_by_accessible_label(window, rest) .map(|e| describe(&e, scale)) .collect(), ), "locate-id" => { let (needle, qualified) = (rest.to_string(), format!("::{rest}")); query(window, scale, move |e| { e.id() .is_some_and(|id| id == needle.as_str() || id.ends_with(&qualified)) || e.type_name().is_some_and(|t| t == needle.as_str()) }) } "ids" => { let needle = rest.to_string(); query(window, scale, move |e| { e.id() .is_some_and(|id| !id.is_empty() && id.contains(needle.as_str())) }) } "labels" => { let needle = rest.to_string(); query(window, scale, move |e| { e.accessible_label() .is_some_and(|l| !l.is_empty() && l.contains(needle.as_str())) }) } "canvas" => { let size = window.get_canvas().size(); json!({"w": size.width, "h": size.height}) } _ => json!({"error": format!("unknown request {verb:?}")}), } } /// Every drawn element the predicate accepts, described, in tree order. fn query( window: &AppWindow, scale: f32, predicate: impl Fn(&ElementHandle) -> bool + 'static, ) -> Value { window .root_element() .query_descendants() .match_predicate(predicate) .find_all() .iter() .map(|e| describe(e, scale)) .collect() } fn describe(e: &ElementHandle, scale: f32) -> Value { let at = e.absolute_position(); let size = e.size(); json!({ "label": e.accessible_label().map(|s| s.to_string()), "id": e.id().map(|s| s.to_string()), "type": e.type_name().map(|s| s.to_string()), "role": e.accessible_role().map(|r| format!("{r:?}")), "x": (at.x * scale).round(), "y": (at.y * scale).round(), "w": (size.width * scale).round(), "h": (size.height * scale).round(), "opacity": e.computed_opacity(), "enabled": e.accessible_enabled(), "checked": e.accessible_checked(), "value": e.accessible_value().map(|s| s.to_string()), "min": e.accessible_value_minimum(), "max": e.accessible_value_maximum(), }) }