From c04d5a3dccd9f9c81e4ad7ae782196a72afc6733 Mon Sep 17 00:00:00 2001 From: Duncan Tourolle Date: Fri, 31 Jul 2026 11:29:46 +0200 Subject: [PATCH] JR-004, JR-005, JR-006: scene-scoped read path Presence was decided by a LINQ predicate inline in the controller, so the semantics SR-002 sets were nowhere stated in code -- the read path complied by accident rather than by requirement. PresenceLookup is now the unit that decides, tagged, with the reasoning next to it. JR-004: windows are served exactly as given. UT-021 pins that [0,10] and [10,20] are not merged despite looking mergeable -- two windows mean a genuine departure and return, and collapsing them answers a different question from the one the truth file asked. UT-022 pins a byte-identical round trip. JR-005: bounds inclusive at both ends, zero-length windows are real sightings rather than degenerate ones to discard, overlaps resolve. The wording was the larger half of JR-005. The overlay rendered a bare list: it asserted nothing, but told the viewer nothing either, and the default reading of a paused frame is "these people are on screen" -- exactly what SR-002 forbids. It now carries an "In this scene" heading. ActorAtTime became ActorInScene, and README no longer contains "on screen" anywhere; it stated the forbidden reading outright in seven places, including the opening sentence. JR-006: measured rather than assumed. UT-023 builds 50 actors x 1000 windows and asserts the response is bounded by actor count, never window count. The lookup is a full scan on purpose -- an early exit on `start > t` would exploit the sortedness the format requires, but would silently under-report the moment one producer emitted windows out of order. UT-020 pins that unsorted input still resolves; WindowsAreSorted is a diagnostic, not a correctness dependency. Third mutation check: making the end bound exclusive fails UT-016 and UT-018 and nothing else. One character turns an inclusive window into a half-open one, dropping an actor at exactly the moment a scene ends. TRACES: UT-016, UT-017, UT-018, UT-019, UT-020, UT-021, UT-022, UT-023 TRACES: JR-004, JR-005, JR-006 | SR-002 Co-Authored-By: Claude Opus 5 --- .../PresenceLookupTests.cs | 150 ++++++++++++++++++ .../Controllers/ActorsController.cs | 10 +- .../{ActorAtTime.cs => ActorInScene.cs} | 12 +- Jellyfin.Plugin.JRay/Models/JRayContext.cs | 4 +- Jellyfin.Plugin.JRay/Models/TruthActor.cs | 2 +- Jellyfin.Plugin.JRay/Models/TruthFile.cs | 2 +- .../Services/PresenceLookup.cs | 119 ++++++++++++++ Jellyfin.Plugin.JRay/Web/jray-overlay.js | 16 ++ README.md | 33 ++-- SPEC.md | 44 +++-- docs/requirements.md | 33 ++-- 11 files changed, 378 insertions(+), 47 deletions(-) create mode 100644 Jellyfin.Plugin.JRay.Tests/PresenceLookupTests.cs rename Jellyfin.Plugin.JRay/Models/{ActorAtTime.cs => ActorInScene.cs} (66%) create mode 100644 Jellyfin.Plugin.JRay/Services/PresenceLookup.cs diff --git a/Jellyfin.Plugin.JRay.Tests/PresenceLookupTests.cs b/Jellyfin.Plugin.JRay.Tests/PresenceLookupTests.cs new file mode 100644 index 0000000..4d2b427 --- /dev/null +++ b/Jellyfin.Plugin.JRay.Tests/PresenceLookupTests.cs @@ -0,0 +1,150 @@ +using System.Diagnostics; +using System.Linq; +using System.Text.Json; +using Jellyfin.Plugin.JRay.Models; +using Jellyfin.Plugin.JRay.Services; +using Xunit; + +namespace Jellyfin.Plugin.JRay.Tests; + +/// +/// JR-004 (windows are scene-membership claims, served verbatim), JR-005 (query +/// semantics and inclusive bounds) and JR-006 (numerous windows). +/// +/// These pin the semantics SR-002 sets. The failure they exist to prevent is a +/// well-meaning "tidy-up" — merging adjacent windows, trimming a zero-length +/// one, or collapsing overlaps — each of which silently answers a different +/// question from the one the truth file asked. +/// +/// TRACES: UT-016, UT-017, UT-018, UT-019, UT-020, UT-021, UT-022, UT-023 | JR-004, JR-005, JR-006 +/// +public class PresenceLookupTests +{ + private static TruthActor Actor(params double[][] windows) + { + var actor = new TruthActor { Name = "Steve Buscemi", TmdbId = "884" }; + foreach (var w in windows) + { + actor.Scenes.Add(w); + } + + return actor; + } + + // UT-016 + [Theory] + [InlineData(12.0)] // exactly the start + [InlineData(30.0)] // inside + [InlineData(45.0)] // exactly the end + public void IsPresentAt_WithinInclusiveBounds_IsPresent(double t) + { + // Both ends inclusive: a window is [start, end], not [start, end). + Assert.True(PresenceLookup.IsPresentAt(Actor([12.0, 45.0]), t)); + } + + // UT-017 + [Theory] + [InlineData(11.999)] + [InlineData(45.001)] + public void IsPresentAt_OutsideBounds_IsAbsent(double t) + { + Assert.False(PresenceLookup.IsPresentAt(Actor([12.0, 45.0]), t)); + } + + // UT-018 + [Fact] + public void IsPresentAt_ZeroLengthWindow_IsPresentAtThatInstant() + { + // A single sighting is a legitimate window. Discarding it as degenerate + // would drop the actor from a scene they are demonstrably in. + Assert.True(PresenceLookup.IsPresentAt(Actor([30.0, 30.0]), 30.0)); + } + + // UT-019 + [Fact] + public void IsPresentAt_OverlappingWindows_IsPresentInsideTheEnclosingOne() + { + // [0,100] encloses [50,60]. A lookup that assumed non-overlapping, + // sorted windows and stopped at the first start > t would miss t = 80. + Assert.True(PresenceLookup.IsPresentAt(Actor([0.0, 100.0], [50.0, 60.0]), 80.0)); + } + + // UT-020 + [Fact] + public void IsPresentAt_UnsortedWindows_StillFindsPresence() + { + // Sortedness is a producer guarantee, not something correctness may + // depend on. A file that violates it must still be read correctly. + var actor = Actor([100.0, 110.0], [10.0, 20.0]); + + Assert.True(PresenceLookup.IsPresentAt(actor, 15.0)); + Assert.False(PresenceLookup.WindowsAreSorted(actor)); + } + + // UT-021 + [Fact] + public void ActorsPresentAt_AdjacentWindowsAreNeverMerged() + { + // [0,10] and [10,20] look mergeable. They must not be merged: two + // windows mean a genuine departure and return, and the plugin does not + // reinterpret that claim. The actor is reported once, from two windows. + var truth = new TruthFile(); + truth.Actors.Add(Actor([0.0, 10.0], [10.0, 20.0])); + + Assert.Single(PresenceLookup.ActorsPresentAt(truth, 10.0)); + Assert.Equal(2, truth.Actors[0].Scenes.Count); + } + + // UT-022 + [Fact] + public void TruthFile_RoundTrips_WithWindowsByteIdentical() + { + // JR-004: served exactly as given. A round trip through the serializer + // is where a silent normalisation would show up. + const string Json = """ + {"schema_version":1,"movie":"/m.mkv","sample_fps":1,"anneal_sec":2, + "actors":[{"name":"A","imdb_id":"","tmdb_id":"884","jellyfin_id":"", + "scenes":[[0.0,10.0],[10.0,20.0],[30.0,30.0]]}]} + """; + + var parsed = JsonSerializer.Deserialize(Json, new JsonSerializerOptions(JsonSerializerDefaults.Web))!; + var windows = parsed.Actors[0].Scenes; + + Assert.Equal(3, windows.Count); + Assert.Equal([0.0, 10.0], windows[0]); + Assert.Equal([10.0, 20.0], windows[1]); + Assert.Equal([30.0, 30.0], windows[2]); + } + + // UT-023 + [Fact] + public void ActorsPresentAt_WithManyWindows_StaysCheapAndBoundsTheResponse() + { + // SR-002: windows may be numerous; consumers must not assume a handful + // of long ones. Track-extent presence with a short re-acquisition + // timeout produces many short windows per actor. + var truth = new TruthFile(); + for (var a = 0; a < 50; a++) + { + var actor = Actor(); + for (var w = 0; w < 1000; w++) + { + actor.Scenes.Add([w * 10.0, (w * 10.0) + 4.0]); + } + + truth.Actors.Add(actor); + } + + var sw = Stopwatch.StartNew(); + var present = PresenceLookup.ActorsPresentAt(truth, 5002.0).ToList(); + sw.Stop(); + + // The response is bounded by actor count, never by window count — which + // is what keeps `jray?t=` small however finely presence is sliced. + Assert.Equal(50, present.Count); + + // 50 000 windows scanned. Generous bound: this asserts the read path is + // not accidentally quadratic, not a precise budget on a shared runner. + Assert.True(sw.ElapsedMilliseconds < 250, $"lookup took {sw.ElapsedMilliseconds} ms"); + } +} diff --git a/Jellyfin.Plugin.JRay/Controllers/ActorsController.cs b/Jellyfin.Plugin.JRay/Controllers/ActorsController.cs index 9bd6936..520abd2 100644 --- a/Jellyfin.Plugin.JRay/Controllers/ActorsController.cs +++ b/Jellyfin.Plugin.JRay/Controllers/ActorsController.cs @@ -1,8 +1,8 @@ using System; -using System.Linq; using System.Threading; using System.Threading.Tasks; using Jellyfin.Plugin.JRay.Models; +using Jellyfin.Plugin.JRay.Services; using Jellyfin.Plugin.JRay.Services.Interfaces; using Microsoft.AspNetCore.Authorization; using Microsoft.AspNetCore.Http; @@ -38,7 +38,7 @@ public class ActorsController : ControllerBase } /// - /// Gets the full actor timeline (every actor with their on-screen scene windows) for a movie. + /// Gets the full actor timeline (every actor with their scene-presence windows) for a movie. /// /// The Jellyfin item id. /// Cancellation token. @@ -58,7 +58,7 @@ public class ActorsController : ControllerBase } /// - /// Gets the JRay context (currently: on-screen actors) at a given timestamp. + /// Gets the JRay context (currently: the actors in the scene) at a given timestamp. /// This is an extensible envelope — future fields (locations, trivia, etc.) /// will be added here without changing the route. /// @@ -78,9 +78,9 @@ public class ActorsController : ControllerBase } var context = new JRayContext(); - foreach (var actor in truth.Actors.Where(actor => actor.Scenes.Any(scene => scene.Length == 2 && scene[0] <= t && t <= scene[1]))) + foreach (var actor in PresenceLookup.ActorsPresentAt(truth, t)) { - context.Actors.Add(new ActorAtTime + context.Actors.Add(new ActorInScene { Name = actor.Name, ImdbId = actor.ImdbId, diff --git a/Jellyfin.Plugin.JRay/Models/ActorAtTime.cs b/Jellyfin.Plugin.JRay/Models/ActorInScene.cs similarity index 66% rename from Jellyfin.Plugin.JRay/Models/ActorAtTime.cs rename to Jellyfin.Plugin.JRay/Models/ActorInScene.cs index c017442..7dc1f73 100644 --- a/Jellyfin.Plugin.JRay/Models/ActorAtTime.cs +++ b/Jellyfin.Plugin.JRay/Models/ActorInScene.cs @@ -3,9 +3,17 @@ using System.Text.Json.Serialization; namespace Jellyfin.Plugin.JRay.Models; /// -/// An actor visible on screen at a queried timestamp. +/// An actor present in the scene at a queried timestamp. /// -public class ActorAtTime +/// +/// "Present in the scene", not "visible on screen". The truth file makes a claim +/// about scene membership, so an actor who has turned away or is off-camera +/// during a reverse shot is still present. The type was named +/// ActorAtTime, which invited exactly the instantaneous reading SR-002 +/// forbids. +/// +// TRACES: JR-005 | SR-002 +public class ActorInScene { /// /// Gets or sets the actor's display name. diff --git a/Jellyfin.Plugin.JRay/Models/JRayContext.cs b/Jellyfin.Plugin.JRay/Models/JRayContext.cs index ff198f1..08935e7 100644 --- a/Jellyfin.Plugin.JRay/Models/JRayContext.cs +++ b/Jellyfin.Plugin.JRay/Models/JRayContext.cs @@ -11,8 +11,8 @@ namespace Jellyfin.Plugin.JRay.Models; public class JRayContext { /// - /// Gets the list of actors visible on screen at the queried timestamp. + /// Gets the list of actors visible in the scene at the queried timestamp. /// [JsonPropertyName("actors")] - public Collection Actors { get; } = new(); + public Collection Actors { get; } = new(); } diff --git a/Jellyfin.Plugin.JRay/Models/TruthActor.cs b/Jellyfin.Plugin.JRay/Models/TruthActor.cs index 7ba9620..67e3955 100644 --- a/Jellyfin.Plugin.JRay/Models/TruthActor.cs +++ b/Jellyfin.Plugin.JRay/Models/TruthActor.cs @@ -45,7 +45,7 @@ public class TruthActor public string JellyfinId { get; set; } = string.Empty; /// - /// Gets the list of [start_sec, end_sec] windows during which the actor is on screen. + /// Gets the list of [start_sec, end_sec] windows during which the actor is in the scene. /// [JsonPropertyName("scenes")] [JsonObjectCreationHandling(JsonObjectCreationHandling.Populate)] diff --git a/Jellyfin.Plugin.JRay/Models/TruthFile.cs b/Jellyfin.Plugin.JRay/Models/TruthFile.cs index b020866..1e7511d 100644 --- a/Jellyfin.Plugin.JRay/Models/TruthFile.cs +++ b/Jellyfin.Plugin.JRay/Models/TruthFile.cs @@ -45,7 +45,7 @@ public class TruthFile public double AnnealSec { get; set; } /// - /// Gets the list of actors detected in the film, each with their on-screen scene windows. + /// Gets the list of actors in the film, each with their scene-presence windows. /// [JsonPropertyName("actors")] [JsonObjectCreationHandling(JsonObjectCreationHandling.Populate)] diff --git a/Jellyfin.Plugin.JRay/Services/PresenceLookup.cs b/Jellyfin.Plugin.JRay/Services/PresenceLookup.cs new file mode 100644 index 0000000..3a9ac44 --- /dev/null +++ b/Jellyfin.Plugin.JRay/Services/PresenceLookup.cs @@ -0,0 +1,119 @@ +using System.Collections.Generic; +using Jellyfin.Plugin.JRay.Models; + +namespace Jellyfin.Plugin.JRay.Services; + +/// +/// Answers "which actors are in the scene at time t" from a truth file. +/// +/// +/// This is the unit that decides presence, so the scene-scoped semantics live +/// here rather than being spread through the controller. +/// +/// +/// A window is a claim about scene membership, not a recognition event. +/// An actor who turns away, is occluded, or is off-camera while the shot cuts to +/// whoever they are speaking to is still present. Two windows mean a genuine +/// departure and return, not a break in detection — so this code reads windows +/// exactly as given and never merges, splits, trims, or reorders them. +/// +/// +/// Bounds are inclusive at both ends, matching the format's definition. That +/// makes adjacent windows such as [0,10] and [10,20] both contain +/// t = 10; reporting the actor present once is correct, and is not a +/// reason to merge the windows. +/// +/// +// TRACES: JR-004, JR-005, JR-006 | SR-002 +public static class PresenceLookup +{ + /// + /// Determines whether an actor is present in the scene at . + /// + /// The actor entry from a truth file. + /// The timestamp, in seconds. + /// true when any window contains . + public static bool IsPresentAt(TruthActor actor, double t) + { + if (actor is null) + { + return false; + } + + // A full scan, deliberately: windows may be numerous, but correctness + // must not depend on the producer having honoured the sortedness + // guarantee. An early exit on `start > t` would be faster and would + // silently under-report the moment one file arrived out of order — + // trading a correctness risk for a saving that does not matter at this + // scale (see JR-006). + foreach (var window in actor.Scenes) + { + if (window.Length == 2 && window[0] <= t && t <= window[1]) + { + return true; + } + } + + return false; + } + + /// + /// Lists the actors present in the scene at , in the + /// order the truth file lists them. + /// + /// The truth file. + /// The timestamp, in seconds. + /// The actors whose windows contain . + public static IEnumerable ActorsPresentAt(TruthFile truth, double t) + { + if (truth is null) + { + yield break; + } + + foreach (var actor in truth.Actors) + { + if (IsPresentAt(actor, t)) + { + yield return actor; + } + } + } + + /// + /// Determines whether an actor's windows are sorted by start time, as the + /// truth-file format requires of producers. + /// + /// + /// Presence lookup does not depend on this — it is a diagnostic. A file that + /// fails it is still read correctly, but it signals a producer bug worth + /// surfacing rather than absorbing silently. + /// + /// The actor entry from a truth file. + /// true when every window starts at or after its predecessor. + public static bool WindowsAreSorted(TruthActor actor) + { + if (actor is null) + { + return true; + } + + double previousStart = double.NegativeInfinity; + foreach (var window in actor.Scenes) + { + if (window.Length != 2) + { + continue; + } + + if (window[0] < previousStart) + { + return false; + } + + previousStart = window[0]; + } + + return true; + } +} diff --git a/Jellyfin.Plugin.JRay/Web/jray-overlay.js b/Jellyfin.Plugin.JRay/Web/jray-overlay.js index bce1d11..a234d43 100644 --- a/Jellyfin.Plugin.JRay/Web/jray-overlay.js +++ b/Jellyfin.Plugin.JRay/Web/jray-overlay.js @@ -218,6 +218,22 @@ overlayEl.style.gap = '12px'; overlayEl.style.pointerEvents = 'none'; + // "In this scene", not "on screen now". Presence is scene-scoped, so + // this list includes people the camera is not currently pointing at — + // without the heading a viewer reads a paused frame and concludes the + // overlay is wrong whenever someone is off-camera mid-conversation. + var heading = document.createElement('div'); + heading.className = 'jrayOverlayHeading'; + heading.textContent = 'In this scene'; + heading.style.width = '100%'; + heading.style.color = '#fff'; + heading.style.opacity = '0.75'; + heading.style.fontSize = '13px'; + heading.style.textTransform = 'uppercase'; + heading.style.letterSpacing = '0.08em'; + heading.style.textShadow = '0 1px 3px rgba(0,0,0,0.9)'; + overlayEl.appendChild(heading); + actors.forEach(function (actor) { var card = document.createElement('div'); card.className = 'jrayActorCard'; diff --git a/README.md b/README.md index 9692683..421fbea 100644 --- a/README.md +++ b/README.md @@ -1,11 +1,11 @@ # JRay -A Jellyfin plugin that brings an actor-overlay (think Amazon "X-Ray") feature to your media: pause a movie and JRay shows you which actors are on screen at that exact moment. +A Jellyfin plugin that brings an actor-overlay (think Amazon "X-Ray") feature to your media: pause a movie and JRay shows you which actors are in the scene you paused in. JRay reads "truth" files produced offline by the [scene-actor-extraction](https://github.com/dtourolle/scene-actor-extraction) pipeline (face detection + recognition) and exposes an API to query which actors -are visible at a given timestamp. A small overlay, injected into the Jellyfin web +are present in the scene at a given timestamp. A small overlay, injected into the Jellyfin web client, displays the result when you pause playback. ## Status @@ -47,7 +47,7 @@ patch on startup, so there is nothing to clean up by hand. See ## Features - **Pause overlay** — pause a movie or episode in the web client and see the - actors currently on screen, without leaving the player. + actors in the current scene, without leaving the player. - **Sidecar truth files** — drop a `Movie.jray.json` next to `Movie.mkv` and JRay picks it up automatically (suffix configurable). - **Remote truth push** — for servers that can't run the extraction pipeline @@ -82,12 +82,12 @@ patch on startup, so there is nothing to clean up by hand. See ``` 1. The extraction pipeline analyses a film offline and emits a **truth file** - listing each detected actor and the time windows they're on screen. + listing each actor and the time windows they are present in the film. 2. JRay loads that truth file either from a **sidecar** next to the media (`Movie.jray.json`) or from a **managed store** populated via the push API. 3. On startup JRay injects a small `