DarkRoom/ui/dr-ui/src/automation.rs

//! Where a control is, asked by its name — for the scripts that record the
//! manual, and for nobody else.
//!
//! `tools/manual/drive.py` moves a real pointer over the window with xdotool,
//! and until this existed it aimed at pixel coordinates written into
//! `scenes.py` by hand. A panel that grew a row moved every slider below it,
//! and the recording went on clicking where the slider used to be: no error,
//! just a picture of the wrong thing. This answers "where is *Exposure*?" from
//! the element tree itself, by the same accessible label a screen reader reads
//! (NFR-A11Y-2), so a script that names controls survives a layout change the
//! way a screen-reader user does.
//!
//! **Absent from every normal build.** The module exists only under the
//! `automation` cargo feature, which no default build enables and
//! `tools/manual/record.sh` does; and even in a build that has it, nothing
//! listens unless `DR_AUTOMATION` names a socket path. It only reads the tree
//! and answers: the input itself still arrives as ordinary pointer and key
//! events from outside, so a scene exercises exactly what a person's hand
//! would, gesture recognisers and all.
//!
//! ## The protocol
//!
//! One request per line on a Unix socket, one JSON line back:
//!
//! - `ping` — `"ok"`.
//! - `locate LABEL` — every element whose accessible label is exactly
//!   `LABEL`: `[{"label","id","type","role","x","y","w","h","opacity",
//!   "enabled","checked","value","min","max"}, …]`, in physical window
//!   pixels, tree order.
//! - `labels [SUBSTRING]` — every labelled element, or those whose label
//!   contains `SUBSTRING`; the same objects. For finding what a scene can
//!   name.
//! - `locate-id ID` — the same, for elements by their id in the markup
//!   (`canvas-image`, or qualified, `CropOverlay::move-area`) or by
//!   component type (`Timeline`). For the few things a scene has to aim at
//!   that are not controls — the photograph, the crop rectangle — and so
//!   have no accessible name to give.
//! - `ids [SUBSTRING]` — every element with an id, for finding those.
//! - `window` — `{"w","h","scale"}`.
//! - `canvas` — `{"w","h"}` of the develop canvas's current image, so a
//!   script can place a point on the photograph within `canvas-image`.
//!
//! Only elements Slint considers visible are answered — not hidden, not
//! clipped away by a scrolled list — so "on screen" is the default.
//!
//! Waiting is the client's job (poll `locate`), which keeps this stateless.

use crate::executors::{self, Executor};
use std::io::{BufRead, BufReader, Write};
use std::os::unix::net::{UnixListener, UnixStream};
use std::sync::mpsc;
use std::time::Duration;

use i_slint_backend_testing::{ElementHandle, ElementRoot as _};
use serde_json::{json, Value};
use slint::ComponentHandle as _;

use crate::AppWindow;

/// Listen on `DR_AUTOMATION` if it is set. Called once, before the event loop.
pub(crate) fn attach(window: &AppWindow) {
    let Some(path) = std::env::var_os("DR_AUTOMATION") else {
        return;
    };
    let _ = std::fs::remove_file(&path);
    let listener = match UnixListener::bind(&path) {
        Ok(l) => l,
        Err(e) => {
            log::warn!("automation: cannot listen on {path:?}: {e}");
            return;
        }
    };
    log::info!("automation: listening on {path:?}");
    let weak = window.as_weak();
    executors::try_spawn(Executor::Io, "automation", move || {
        for stream in listener.incoming().flatten() {
            let weak = weak.clone();
            executors::spawn(Executor::Io, "auto-conn", move || serve(stream, weak));
        }
    })
    .ok();
}

fn serve(stream: UnixStream, weak: slint::Weak<AppWindow>) {
    let Ok(mut out) = stream.try_clone() else {
        return;
    };
    for line in BufReader::new(stream).lines() {
        let Ok(line) = line else { return };
        let (tx, rx) = mpsc::channel();
        let request = line.clone();
        let posted = weak.upgrade_in_event_loop(move |w| {
            let _ = tx.send(answer(&w, &request));
        });
        let reply = match posted {
            Ok(()) => rx
                .recv_timeout(Duration::from_secs(10))
                .unwrap_or_else(|_| json!({"error": "the event loop did not answer"})),
            Err(e) => json!({"error": e.to_string()}),
        };
        if writeln!(out, "{reply}").is_err() {
            return;
        }
    }
}

fn answer(window: &AppWindow, request: &str) -> Value {
    let (verb, rest) = request.split_once(' ').unwrap_or((request, ""));
    let scale = window.window().scale_factor();
    match verb {
        "ping" => json!("ok"),
        "window" => {
            let size = window.window().size();
            json!({"w": size.width, "h": size.height, "scale": scale})
        }
        "locate" => Value::Array(
            ElementHandle::find_by_accessible_label(window, rest)
                .map(|e| describe(&e, scale))
                .collect(),
        ),
        "locate-id" => {
            let (needle, qualified) = (rest.to_string(), format!("::{rest}"));
            query(window, scale, move |e| {
                e.id()
                    .is_some_and(|id| id == needle.as_str() || id.ends_with(&qualified))
                    || e.type_name().is_some_and(|t| t == needle.as_str())
            })
        }
        "ids" => {
            let needle = rest.to_string();
            query(window, scale, move |e| {
                e.id()
                    .is_some_and(|id| !id.is_empty() && id.contains(needle.as_str()))
            })
        }
        "labels" => {
            let needle = rest.to_string();
            query(window, scale, move |e| {
                e.accessible_label()
                    .is_some_and(|l| !l.is_empty() && l.contains(needle.as_str()))
            })
        }
        "canvas" => {
            let size = window.get_canvas().size();
            json!({"w": size.width, "h": size.height})
        }
        _ => json!({"error": format!("unknown request {verb:?}")}),
    }
}

/// Every drawn element the predicate accepts, described, in tree order.
fn query(
    window: &AppWindow,
    scale: f32,
    predicate: impl Fn(&ElementHandle) -> bool + 'static,
) -> Value {
    window
        .root_element()
        .query_descendants()
        .match_predicate(predicate)
        .find_all()
        .iter()
        .map(|e| describe(e, scale))
        .collect()
}

fn describe(e: &ElementHandle, scale: f32) -> Value {
    let at = e.absolute_position();
    let size = e.size();
    json!({
        "label": e.accessible_label().map(|s| s.to_string()),
        "id": e.id().map(|s| s.to_string()),
        "type": e.type_name().map(|s| s.to_string()),
        "role": e.accessible_role().map(|r| format!("{r:?}")),
        "x": (at.x * scale).round(),
        "y": (at.y * scale).round(),
        "w": (size.width * scale).round(),
        "h": (size.height * scale).round(),
        "opacity": e.computed_opacity(),
        "enabled": e.accessible_enabled(),
        "checked": e.accessible_checked(),
        "value": e.accessible_value().map(|s| s.to_string()),
        "min": e.accessible_value_minimum(),
        "max": e.accessible_value_maximum(),
    })
}