From 0eae81ec5968ed3d9613b2e1480a41ba0958b286 Mon Sep 17 00:00:00 2001
From: Duncan Tourolle <duncan@tourolle.paris>
Date: Sun, 28 Jun 2026 20:38:58 +0200
Subject: [PATCH] Add JRay support

---
 src-tauri/src/commands/repository.rs         | 17 +++++
 src-tauri/src/lib.rs                         |  5 +-
 src-tauri/src/repository/hybrid.rs           |  6 ++
 src-tauri/src/repository/mod.rs              |  2 +-
 src-tauri/src/repository/online.rs           | 68 +++++++++++++++++
 src/lib/api/bindings.ts                      | 16 ++++
 src/lib/api/repository-client.ts             | 10 +++
 src/lib/components/player/VideoPlayer.svelte | 77 ++++++++++++++++++++
 8 files changed, 198 insertions(+), 3 deletions(-)

diff --git a/src-tauri/src/commands/repository.rs b/src-tauri/src/commands/repository.rs
index 5b0e6e7..fd1bbf2 100644
--- a/src-tauri/src/commands/repository.rs
+++ b/src-tauri/src/commands/repository.rs
@@ -179,6 +179,23 @@ pub async fn repository_get_item(
         .map_err(|e| format!("{:?}", e))
 }
 
+/// Query the optional JRay plugin for the actors on screen at time `t`
+/// (seconds) in an item. Returns an empty list when JRay isn't installed or
+/// has no data for the item, so the caller can render nothing without error.
+#[tauri::command]
+#[specta::specta]
+pub async fn repository_jray_actors_at(
+    manager: State<'_, RepositoryManagerWrapper>,
+    handle: String,
+    item_id: String,
+    t: f64,
+) -> Result<Vec<crate::repository::JRayActor>, String> {
+    let repo = manager.0.get(&handle).ok_or("Repository not found")?;
+    repo.as_ref().get_jray_actors(&item_id, t)
+        .await
+        .map_err(|e| format!("{:?}", e))
+}
+
 /// Get latest items in a library
 #[tauri::command]
 #[specta::specta]
diff --git a/src-tauri/src/lib.rs b/src-tauri/src/lib.rs
index 6ac846b..5abfe82 100644
--- a/src-tauri/src/lib.rs
+++ b/src-tauri/src/lib.rs
@@ -96,7 +96,7 @@ use commands::{
     storage_save_series_audio_preference, storage_get_series_audio_preference,
     // Repository commands
     repository_create, repository_destroy, repository_get_libraries, repository_get_items,
-    repository_get_item, repository_get_latest_items, repository_get_resume_items,
+    repository_get_item, repository_jray_actors_at, repository_get_latest_items, repository_get_resume_items,
     repository_get_next_up_episodes, repository_get_recently_played_audio, repository_get_resume_movies,
     repository_get_rediscover_albums,
     repository_get_genres, repository_search, repository_get_playback_info,
@@ -634,6 +634,7 @@ fn specta_builder() -> Builder<tauri::Wry> {
             repository_get_libraries,
             repository_get_items,
             repository_get_item,
+            repository_jray_actors_at,
             repository_get_latest_items,
             repository_get_resume_items,
             repository_get_next_up_episodes,
@@ -676,7 +677,6 @@ fn specta_builder() -> Builder<tauri::Wry> {
         ])
 }
 
-#[cfg_attr(mobile, tauri::mobile_entry_point)]
 /// Configure GStreamer (the media backend behind WebKitGTK's HTML5 `<video>`
 /// element on Linux) to prefer hardware-accelerated VAAPI decoding when the
 /// host provides it, falling back to software decoding otherwise.
@@ -752,6 +752,7 @@ fn set_env_if_unset(key: &str, value: &str) {
     }
 }
 
+#[cfg_attr(mobile, tauri::mobile_entry_point)]
 pub fn run() {
     // Initialize logger
     env_logger::Builder::from_default_env()
diff --git a/src-tauri/src/repository/hybrid.rs b/src-tauri/src/repository/hybrid.rs
index 4c40177..3037f3b 100644
--- a/src-tauri/src/repository/hybrid.rs
+++ b/src-tauri/src/repository/hybrid.rs
@@ -45,6 +45,12 @@ impl HybridRepository {
         self.online.download_bytes(url).await
     }
 
+    /// Query the JRay plugin for actors on screen at time `t`. Online-only
+    /// (the plugin lives on the Jellyfin server); empty when JRay isn't present.
+    pub async fn get_jray_actors(&self, item_id: &str, t: f64) -> Result<Vec<super::JRayActor>, RepoError> {
+        self.online.get_jray_actors(item_id, t).await
+    }
+
     /// Get video stream URL with optional seeking support.
     /// This method is online-only since offline playback uses local file paths.
     pub async fn get_video_stream_url(
diff --git a/src-tauri/src/repository/mod.rs b/src-tauri/src/repository/mod.rs
index 1edbbc9..c9a2db4 100644
--- a/src-tauri/src/repository/mod.rs
+++ b/src-tauri/src/repository/mod.rs
@@ -4,7 +4,7 @@ pub mod offline;
 pub mod hybrid;
 
 pub use types::*;
-pub use online::OnlineRepository;
+pub use online::{OnlineRepository, JRayActor};
 pub use offline::OfflineRepository;
 pub use hybrid::HybridRepository;
 
diff --git a/src-tauri/src/repository/online.rs b/src-tauri/src/repository/online.rs
index 1652a09..4a71886 100644
--- a/src-tauri/src/repository/online.rs
+++ b/src-tauri/src/repository/online.rs
@@ -11,6 +11,30 @@ use crate::connectivity::ConnectivityReporter;
 use crate::jellyfin::HttpClient;
 use super::{MediaRepository, types::*};
 
+/// A single actor returned by the JRay plugin's "context at time t" endpoint.
+///
+/// Mirrors the `actors[]` objects from `GET /Plugins/JRay/Items/{id}/jray?t=`.
+/// `jellyfin_id` (a Jellyfin Person item GUID) is preferred for navigation;
+/// the IMDb/TMDb ids are informational fallbacks. Unknown ids are `""`.
+#[derive(specta::Type, Debug, Clone, Serialize, Deserialize)]
+pub struct JRayActor {
+    pub name: String,
+    #[serde(default)]
+    pub imdb_id: String,
+    #[serde(default)]
+    pub tmdb_id: String,
+    #[serde(default)]
+    pub jellyfin_id: String,
+}
+
+/// Envelope returned by the JRay `jray?t=` endpoint. Extra keys (future
+/// `locations`, `trivia`, …) are ignored so the client tolerates schema growth.
+#[derive(Debug, Clone, Deserialize)]
+struct JRayContext {
+    #[serde(default)]
+    actors: Vec<JRayActor>,
+}
+
 /// Online repository - fetches data from Jellyfin server via HTTP
 pub struct OnlineRepository {
     http_client: Arc<HttpClient>,
@@ -104,6 +128,20 @@ impl OnlineRepository {
             .map_err(|e| format!("Failed to read bytes: {}", e))
     }
 
+    /// Query the JRay plugin for the actors on screen at time `t` (seconds) in
+    /// the given item. Returns an empty list when the plugin isn't installed or
+    /// has no truth data for the item (HTTP 404), so callers can treat "no JRay"
+    /// and "nobody on screen" identically. Other failures propagate.
+    pub async fn get_jray_actors(&self, item_id: &str, t: f64) -> Result<Vec<JRayActor>, RepoError> {
+        let endpoint = format!("/Plugins/JRay/Items/{}/jray?t={}", item_id, t);
+        match self.get_json::<JRayContext>(&endpoint).await {
+            Ok(context) => Ok(context.actors),
+            // No plugin / no truth data for this item — not an error to the user.
+            Err(RepoError::NotFound { .. }) => Ok(Vec::new()),
+            Err(e) => Err(e),
+        }
+    }
+
     /// Make authenticated GET request
     async fn get_json<T: for<'de> Deserialize<'de>>(&self, endpoint: &str) -> Result<T, RepoError> {
         let result = self.get_json_inner(endpoint).await;
@@ -1958,4 +1996,34 @@ mod tests {
         assert_eq!(urlencoding::encode("Star Wars"), "Star%20Wars");
         assert_eq!(urlencoding::encode("Tom & Jerry"), "Tom%20%26%20Jerry");
     }
+
+    #[test]
+    fn test_jray_context_deserializes_actors() {
+        // The jray?t= envelope as documented in the JRay truth file format.
+        let json = r#"{
+            "actors": [
+                { "name": "Tom Hanks", "imdb_id": "nm0000158", "tmdb_id": "31", "jellyfin_id": "abc123-guid" }
+            ]
+        }"#;
+        let ctx: JRayContext = serde_json::from_str(json).expect("should parse");
+        assert_eq!(ctx.actors.len(), 1);
+        assert_eq!(ctx.actors[0].name, "Tom Hanks");
+        assert_eq!(ctx.actors[0].jellyfin_id, "abc123-guid");
+    }
+
+    #[test]
+    fn test_jray_context_ignores_unknown_keys_and_missing_ids() {
+        // Future fields (locations/trivia) must be ignored, and absent id keys
+        // must default to "" rather than failing to parse.
+        let json = r#"{
+            "actors": [ { "name": "Extra" } ],
+            "locations": ["Beach"],
+            "trivia": "filmed in 1994"
+        }"#;
+        let ctx: JRayContext = serde_json::from_str(json).expect("should tolerate extra keys");
+        assert_eq!(ctx.actors.len(), 1);
+        assert_eq!(ctx.actors[0].name, "Extra");
+        assert_eq!(ctx.actors[0].imdb_id, "");
+        assert_eq!(ctx.actors[0].jellyfin_id, "");
+    }
 }
diff --git a/src/lib/api/bindings.ts b/src/lib/api/bindings.ts
index f831a0a..0b6fe47 100644
--- a/src/lib/api/bindings.ts
+++ b/src/lib/api/bindings.ts
@@ -1068,6 +1068,14 @@ async repositoryGetItems(handle: string, parentId: string, options: GetItemsOpti
 async repositoryGetItem(handle: string, itemId: string) : Promise<MediaItem> {
     return await TAURI_INVOKE("repository_get_item", { handle, itemId });
 },
+/**
+ * Query the optional JRay plugin for the actors on screen at time `t`
+ * (seconds) in an item. Returns an empty list when JRay isn't installed or
+ * has no data for the item, so the caller can render nothing without error.
+ */
+async repositoryJrayActorsAt(handle: string, itemId: string, t: number) : Promise<JRayActor[]> {
+    return await TAURI_INVOKE("repository_jray_actors_at", { handle, itemId, t });
+},
 /**
  * Get latest items in a library
  */
@@ -1553,6 +1561,14 @@ export type ImageOptions = { maxWidth?: number | null; maxHeight?: number | null
  * Image type
  */
 export type ImageType = "Primary" | "Backdrop" | "Banner" | "Thumb" | "Logo"
+/**
+ * A single actor returned by the JRay plugin's "context at time t" endpoint.
+ * 
+ * Mirrors the `actors[]` objects from `GET /Plugins/JRay/Items/{id}/jray?t=`.
+ * `jellyfin_id` (a Jellyfin Person item GUID) is preferred for navigation;
+ * the IMDb/TMDb ids are informational fallbacks. Unknown ids are `""`.
+ */
+export type JRayActor = { name: string; imdb_id?: string; tmdb_id?: string; jellyfin_id?: string }
 /**
  * Library (media collection)
  */
diff --git a/src/lib/api/repository-client.ts b/src/lib/api/repository-client.ts
index 59a3581..557ec61 100644
--- a/src/lib/api/repository-client.ts
+++ b/src/lib/api/repository-client.ts
@@ -3,6 +3,7 @@
 // NO direct HTTP calls - everything routes through Rust backend
 
 import { commands } from "./bindings";
+import type { JRayActor } from "./bindings";
 import type { QualityPreset } from "./quality-presets";
 import type {
   Library,
@@ -90,6 +91,15 @@ export class RepositoryClient {
     return commands.repositoryGetItem(this.ensureHandle(), itemId);
   }
 
+  /**
+   * Query the optional JRay plugin for the actors on screen at time `t`
+   * (seconds) in an item. Resolves to an empty array when JRay isn't installed
+   * or has no data for the item.
+   */
+  async jrayActorsAt(itemId: string, t: number): Promise<JRayActor[]> {
+    return commands.repositoryJrayActorsAt(this.ensureHandle(), itemId, t);
+  }
+
   async getLatestItems(parentId: string, limit?: number): Promise<MediaItem[]> {
     return commands.repositoryGetLatestItems(this.ensureHandle(), parentId, limit ?? null);
   }
diff --git a/src/lib/components/player/VideoPlayer.svelte b/src/lib/components/player/VideoPlayer.svelte
index a8185ca..585637b 100644
--- a/src/lib/components/player/VideoPlayer.svelte
+++ b/src/lib/components/player/VideoPlayer.svelte
@@ -1,7 +1,9 @@
 <!-- TRACES: UR-003, UR-005, UR-020, UR-021, UR-026 | DR-010, DR-023, DR-024 -->
 <script lang="ts">
   import { onMount, onDestroy, untrack } from "svelte";
+  import { goto } from "$app/navigation";
   import { commands } from "$lib/api/bindings";
+  import type { JRayActor } from "$lib/api/bindings";
   import { listen } from "@tauri-apps/api/event";
   import Hls from "hls.js";
   import type { MediaItem } from "$lib/api/types";
@@ -750,6 +752,58 @@
     }, 5000);
   }
 
+  // ===== JRay "who's on screen" pause overlay (optional plugin) =====
+  // On pause, ask the JRay Jellyfin plugin which actors are on screen at the
+  // current timestamp and show them as a tappable overlay. If the plugin isn't
+  // installed (or has no data for this item), the call resolves to [] and the
+  // overlay simply doesn't render. Tapping an actor with a resolved Jellyfin
+  // Person id navigates to that person's library page.
+  let jrayActors = $state<JRayActor[]>([]);
+  // Monotonic token so a slow in-flight request can't overwrite a newer pause
+  // (or a resume that cleared the list).
+  let jrayRequestId = 0;
+
+  async function fetchJrayActors() {
+    const itemId = media?.id;
+    if (!itemId) return;
+    const token = ++jrayRequestId;
+    const t = currentTime;
+    try {
+      const actors = await auth.getRepository().jrayActorsAt(itemId, t);
+      // Discard if a newer pause/resume happened while we were waiting.
+      if (token === jrayRequestId) {
+        jrayActors = actors;
+      }
+    } catch (err) {
+      console.warn("[VideoPlayer] JRay lookup failed:", err);
+      if (token === jrayRequestId) jrayActors = [];
+    }
+  }
+
+  function clearJrayActors() {
+    jrayRequestId++; // invalidate any in-flight request
+    jrayActors = [];
+  }
+
+  function openJrayActor(actor: JRayActor) {
+    if (actor.jellyfin_id) {
+      goto(`/library/${actor.jellyfin_id}`);
+    }
+  }
+
+  // Drive the JRay overlay off the single `isPlaying` flag so it works for both
+  // the HTML5 <video> (desktop) and the native ExoPlayer path (Android, which
+  // updates isPlaying via the player://state-changed event). Fetch when we go
+  // paused, clear when we resume. untrack() keeps this from re-running on every
+  // currentTime tick — only isPlaying transitions matter.
+  $effect(() => {
+    if (isPlaying) {
+      untrack(clearJrayActors);
+    } else if (isMediaReady) {
+      untrack(fetchJrayActors);
+    }
+  });
+
   function handlePlay() {
     isPlaying = true;
     startTimeUpdates(); // Start RAF loop for smooth time updates
@@ -1404,6 +1458,29 @@
         </svg>
       </button>
     {/if}
+
+    <!-- JRay "who's on screen" overlay: shown while paused when the JRay plugin
+         returned actors for the current timestamp. Tapping an actor with a
+         resolved Jellyfin Person id opens their library page. -->
+    {#if !isPlaying && !isSeeking && jrayActors.length > 0}
+      <div class="absolute top-4 right-4 max-w-xs bg-black/70 rounded-lg p-3 backdrop-blur-sm pointer-events-auto">
+        <div class="text-white/60 text-xs font-medium uppercase tracking-wide mb-2">On screen</div>
+        <div class="flex flex-col gap-1">
+          {#each jrayActors as actor (actor.name + actor.jellyfin_id)}
+            {#if actor.jellyfin_id}
+              <button
+                class="text-left text-white text-sm hover:text-blue-300 transition-colors"
+                onclick={() => openJrayActor(actor)}
+              >
+                {actor.name}
+              </button>
+            {:else}
+              <span class="text-white/80 text-sm">{actor.name}</span>
+            {/if}
+          {/each}
+        </div>
+      </div>
+    {/if}
   </div>
 
   <!-- Controls -->