Background-audio handoff for video + repository/player refactor

Hand video playback off to a native audio-only stream when the app is
backgrounded or locked, with no on-device video decode (UR-040). Adds
player_enter/exit_background_audio commands, an audio-only stream URL
for video items across the repository layer, and the frontend handoff
state machine wired into VideoPlayer. Includes accompanying
repository/offline/player refactoring and regenerates the traceability
matrix.
This commit is contained in:
2026-07-22 21:52:07 +02:00
parent 4e6ab017d4
commit 3fbf6afdbc
72 changed files with 6728 additions and 2338 deletions
+49 -1
View File
@@ -19,6 +19,40 @@ export const commands = {
async playerPlayItem(item: PlayItemRequest) : Promise<PlayerStatus> {
return await TAURI_INVOKE("player_play_item", { item });
},
/**
* Enter background-audio mode: hand playback of the currently-watched video off
* to the native ExoPlayer *audio* path so the audio keeps playing while the app
* is backgrounded/locked, with no client-side video decode (UR-040).
*
* `stream_url` MUST be an audio-only URL (see
* `get_audio_only_stream_url_for_video`). The item is created as
* `MediaType::Audio` so it starts an audio session and loads into the native
* backend with `mediaType="audio"` — the WebView `<video>` is torn down on the
* frontend side, so exactly one audio source is ever active.
*
* This deliberately goes through the queue-based `play_item` path (NOT a
* side-channel) so end-of-track lands in `on_playback_ended`, which already
* honors the sleep timer (Time/Episodes/EndOfTrack) and drives autoplay-next.
* The sleep-timer state is intentionally left untouched by the handoff.
*
* TRACES: UR-040 | DR-052 | UT-061, IT-013
*/
async playerEnterBackgroundAudio(item: PlayItemRequest, positionSeconds: number) : Promise<PlayerStatus> {
return await TAURI_INVOKE("player_enter_background_audio", { item, positionSeconds });
},
/**
* Exit background-audio mode: stop the native audio player and return its final
* position so the frontend can reload the WebView `<video>` there (UR-040).
*
* Returns the position in seconds. The sleep timer is intentionally left
* untouched — if it fired while backgrounded, playback is already stopped and
* this simply reports the last position.
*
* TRACES: UR-040 | DR-052 | UT-061, IT-013
*/
async playerExitBackgroundAudio() : Promise<number> {
return await TAURI_INVOKE("player_exit_background_audio");
},
/**
* Play a queue of media items
*
@@ -1206,6 +1240,14 @@ async repositoryGetVideoStreamUrl(handle: string, itemId: string, mediaSourceId:
async repositoryGetAudioStreamUrl(handle: string, itemId: string) : Promise<string> {
return await TAURI_INVOKE("repository_get_audio_stream_url", { handle, itemId });
},
/**
* Get an audio-only stream URL for a *video* item (background-audio handoff).
*
* TRACES: UR-040 | JA-032 | UT-061
*/
async repositoryGetAudioOnlyStreamUrlForVideo(handle: string, itemId: string, mediaSourceId: string | null, startTimeSeconds: number | null, audioStreamIndex: number | null) : Promise<string> {
return await TAURI_INVOKE("repository_get_audio_only_stream_url_for_video", { handle, itemId, mediaSourceId, startTimeSeconds, audioStreamIndex });
},
/**
* Get Live TV channels (broadcast / IPTV) for browsing
*/
@@ -1760,7 +1802,13 @@ videoCodec: string;
/**
* Whether the video requires server-side transcoding
*/
needsTranscoding: boolean }
needsTranscoding: boolean;
/**
* Optional now-playing metadata. Used by the background-audio handoff so the
* lockscreen/miniplayer show the item (title/subtitle/artwork). Defaulted so
* existing video-only callers need not send them.
*/
artist?: string | null; primaryImageTag?: string | null; serverId?: string | null }
/**
* Queue context for remote transfer - what type of queue is this?
*/
+32
View File
@@ -390,6 +390,38 @@ describe("RepositoryClient", () => {
});
});
it("should get audio-only stream URL for a video item (camelCase params)", async () => {
// TRACES: UR-040 | JA-032 | UT-061
const mockUrl = "https://server.com/Audio/item123/universal?AudioStreamIndex=2";
(invoke as any).mockResolvedValueOnce(mockUrl);
const url = await client.getAudioOnlyStreamUrlForVideo("item123", "source456", 193, 2);
expect(url).toBe(mockUrl);
expect(invoke).toHaveBeenCalledWith("repository_get_audio_only_stream_url_for_video", {
handle: "test-handle-123",
itemId: "item123",
mediaSourceId: "source456",
startTimeSeconds: 193,
audioStreamIndex: 2,
});
});
it("should default optional params to null for audio-only stream URL", async () => {
// TRACES: UR-040 | JA-032 | UT-061
(invoke as any).mockResolvedValueOnce("https://server.com/Audio/item123/universal");
await client.getAudioOnlyStreamUrlForVideo("item123");
expect(invoke).toHaveBeenCalledWith("repository_get_audio_only_stream_url_for_video", {
handle: "test-handle-123",
itemId: "item123",
mediaSourceId: null,
startTimeSeconds: null,
audioStreamIndex: null,
});
});
it("should report playback progress", async () => {
(invoke as any).mockResolvedValueOnce(undefined);
+20
View File
@@ -172,6 +172,26 @@ export class RepositoryClient {
);
}
/**
* Audio-only stream URL for a video item, for the background-audio handoff.
* The server extracts just the audio track — no video is decoded on-device.
* TRACES: UR-040 | JA-032
*/
async getAudioOnlyStreamUrlForVideo(
itemId: string,
mediaSourceId?: string,
startTimeSeconds?: number,
audioStreamIndex?: number
): Promise<string> {
return commands.repositoryGetAudioOnlyStreamUrlForVideo(
this.ensureHandle(),
itemId,
mediaSourceId ?? null,
startTimeSeconds ?? null,
audioStreamIndex ?? null
);
}
// ===== Live TV / Channels =====
/** Browse Live TV channels (broadcast / IPTV). */
+164 -3
View File
@@ -1,4 +1,4 @@
<!-- TRACES: UR-003, UR-005, UR-020, UR-021, UR-026 | DR-010, DR-023, DR-024 -->
<!-- TRACES: UR-003, UR-005, UR-020, UR-021, UR-026, UR-040 | DR-010, DR-023, DR-024, DR-051, DR-052 -->
<script lang="ts">
import { onMount, onDestroy, untrack } from "svelte";
import { goto } from "$app/navigation";
@@ -18,7 +18,20 @@
import { playerController } from "$lib/player";
import { Html5PlayerAdapter, type Html5ElementBridge } from "$lib/player/adapters";
import { createRustReportHost } from "$lib/player/adapters/rustReportHost";
import { isPipSupported, enterPip } from "$lib/utils/pictureInPicture";
import { isPipSupported, enterPip, setAutoEnterEnabled } from "$lib/utils/pictureInPicture";
import {
isBackgroundAudioSupported,
setBackgroundAudioEnabled,
subscribeAppBackgrounded,
subscribeAppForegrounded,
} from "$lib/utils/backgroundAudio";
import {
computeHandoffPosition,
initialHandoffState,
shouldEnterBackgroundAudio,
shouldExitBackgroundAudio,
type BackgroundAudioState,
} from "./backgroundAudioHandoff";
interface Props {
media: MediaItem | null;
@@ -489,6 +502,15 @@
// Set up progress reporting interval
onMount(async () => {
// Background-audio lifecycle listeners MUST be registered synchronously —
// before any await below — per the native-mode pitfall (an await here can
// flip the component into HTML5 mode). Unsubscribers go into nativeUnlisteners
// so onDestroy tears them down.
if (backgroundAudioSupported) {
nativeUnlisteners.push(subscribeAppBackgrounded(enterBackgroundAudioHandoff));
nativeUnlisteners.push(subscribeAppForegrounded(exitBackgroundAudioHandoff));
}
// Initialize player via Rust - Rust will decide which backend to use based on platform
if (media && currentStreamUrl) {
try {
@@ -681,12 +703,19 @@
clearInterval(debugLogInterval);
}
// Remove native backend event listeners
// Remove native backend event listeners (incl. background-audio lifecycle subs)
for (const unlisten of nativeUnlisteners) {
unlisten();
}
nativeUnlisteners = [];
// Re-assert defaults so this player's background-audio choice can't leak into
// the next one: disarm background audio and restore auto-PiP.
if (backgroundAudioSupported) {
setBackgroundAudioEnabled(false);
setAutoEnterEnabled(true);
}
// Clean up HLS.js instance - prevent dual audio on unmount
if (hls) {
console.log("[VideoPlayer] Destroying HLS.js instance on unmount");
@@ -815,6 +844,25 @@
console.log("[VideoPlayer] Video unmuted on canplay, volume: 1.0");
}
// Returning from background audio: resume the <video> at the position native
// audio reached, restoring the prior play/pause state. Takes precedence over
// the resume-point seek below (which is for a fresh load, not a handoff).
if (pendingForegroundSeek !== null && videoElement) {
const seekTo = pendingForegroundSeek;
const shouldPlay = pendingForegroundPlay;
pendingForegroundSeek = null;
pendingForegroundPlay = false;
hasPerformedInitialSeek = true;
try {
videoElement.currentTime = seekTo;
currentTime = seekTo;
if (shouldPlay) await videoElement.play();
} catch (err) {
console.error("[VideoPlayer] Failed to resume after background audio:", err);
}
return;
}
// Seek to initial position if resuming playback
if (initialPosition && initialPosition > 0 && !hasPerformedInitialSeek && videoElement) {
console.log("[VideoPlayer] Seeking to initial position:", initialPosition);
@@ -1092,6 +1140,104 @@
enterPip();
}
// ===== Background audio (UR-040, Android) =====
// Keep the video's audio playing when the app is backgrounded/locked by handing
// playback off to the native ExoPlayer audio service; the WebView <video> is
// torn down so no video is decoded. Mutually exclusive with auto-PiP.
//
// Resolved synchronously (no await) for the same native-mode reason as PiP.
const backgroundAudioSupported = isBackgroundAudioSupported();
let backgroundAudioOn = $state(false); // v1: default OFF each session
let handoffState: BackgroundAudioState = { ...initialHandoffState };
function toggleBackgroundAudio() {
backgroundAudioOn = !backgroundAudioOn;
// Arm/disarm native background-audio mode AND flip auto-PiP the other way,
// so exactly one background behavior is active.
setBackgroundAudioEnabled(backgroundAudioOn);
setAutoEnterEnabled(!backgroundAudioOn);
}
// App went to background/locked while background-audio is armed: hand off to
// native audio and stop the WebView video decode.
async function enterBackgroundAudioHandoff() {
if (!shouldEnterBackgroundAudio(backgroundAudioOn, handoffState)) return;
// `currentTime` is the component's authoritative ABSOLUTE position (the RAF
// loop keeps it at seekOffset + element.currentTime, and it survives HLS
// transcode segment resets). Reading videoElement.currentTime directly is
// wrong for transcoded streams (it's the in-segment offset) and can read 0
// if the element is mid-teardown — which shipped audio starting from 0:00.
const pos = computeHandoffPosition(currentTime, 0);
const wasPlaying = isPlaying;
console.log("[VideoPlayer] Background-audio handoff at position:", pos.toFixed(1));
handoffState = { active: true, wasPlaying };
try {
if (!media) return;
// Ask the server for an audio-only stream of this video item (no video
// decode), carrying the selected audio track and resume position.
const audioUrl = await auth.getRepository().getAudioOnlyStreamUrlForVideo(
media.id,
mediaSourceId ?? undefined,
pos,
selectedAudioTrackIndex ?? undefined,
);
await commands.playerEnterBackgroundAudio(
{
id: media.id,
title: media.name,
streamUrl: audioUrl,
videoCodec: "aac",
needsTranscoding: false,
// Now-playing metadata so the lockscreen/miniplayer show the item.
artist: media.seriesName ?? null,
primaryImageTag: media.primaryImageTag ?? null,
serverId: media.serverId ?? null,
},
pos,
);
// Tear down the WebView <video>/HLS decode AFTER native audio has started,
// so there is never a gap — and exactly one audio source is ever live.
tearDownHls();
if (videoElement) {
videoElement.pause();
videoElement.removeAttribute("src");
videoElement.load();
}
} catch (err) {
console.error("[VideoPlayer] Background-audio handoff failed:", err);
handoffState = { ...initialHandoffState };
}
}
// App returned to foreground: stop native audio, reload the WebView <video> at
// the position native reached, and restore play/pause.
async function exitBackgroundAudioHandoff() {
if (!shouldExitBackgroundAudio(handoffState)) return;
const wasPlaying = handoffState.wasPlaying;
handoffState = { ...initialHandoffState };
try {
const pos = await commands.playerExitBackgroundAudio();
// Reload the video at the returned position. Resetting these re-runs the
// HLS init $effect and reveals/seeks the element as on a fresh load.
hasPerformedInitialSeek = false;
lastAppliedInitialPosition = undefined;
seekOffset = 0;
isMediaReady = false;
// Re-point the element at the (unchanged) video stream URL; assigning a new
// reference restarts the HLS effect even if the string is identical.
currentStreamUrl = streamUrl;
// Seek to where native audio left off once the element is ready again.
pendingForegroundSeek = pos;
pendingForegroundPlay = wasPlaying;
} catch (err) {
console.error("[VideoPlayer] Background-audio return failed:", err);
}
}
// Consumed by handleCanPlay after the <video> reloads on foreground.
let pendingForegroundSeek: number | null = null;
let pendingForegroundPlay = false;
function toggleFullscreen() {
if (!document.fullscreenElement) {
document.documentElement.requestFullscreen();
@@ -1745,6 +1891,21 @@
</button>
{/if}
<!-- Background audio (Android only) — keep audio playing when the app is
backgrounded/locked; video decode stops. Suppresses auto-PiP while on. -->
{#if backgroundAudioSupported}
<button
onclick={toggleBackgroundAudio}
class={backgroundAudioOn ? "text-blue-400 hover:text-blue-300" : "text-white hover:text-gray-300"}
aria-label="Background audio"
aria-pressed={backgroundAudioOn}
>
<svg class="w-6 h-6" fill="currentColor" viewBox="0 0 24 24">
<path d="M12 1c-4.97 0-9 4.03-9 9v7c0 1.66 1.34 3 3 3h3v-8H5v-2c0-3.87 3.13-7 7-7s7 3.13 7 7v2h-4v8h3c1.66 0 3-1.34 3-3v-7c0-4.97-4.03-9-9-9z" />
</svg>
</button>
{/if}
<!-- Fullscreen -->
<button onclick={toggleFullscreen} class="text-white hover:text-gray-300" aria-label="Toggle fullscreen">
<svg class="w-6 h-6" fill="currentColor" viewBox="0 0 24 24">
@@ -0,0 +1,61 @@
import { describe, it, expect } from "vitest";
import {
computeHandoffPosition,
initialHandoffState,
shouldEnterBackgroundAudio,
shouldExitBackgroundAudio,
type BackgroundAudioState,
} from "./backgroundAudioHandoff";
// TRACES: UR-040 | DR-052 | UT-060
describe("backgroundAudioHandoff", () => {
describe("computeHandoffPosition", () => {
it("sums element time and transcode seekOffset (absolute position)", () => {
// Transcoded HLS resets element time to 0 after a reload; seekOffset carries
// the cumulative offset. The audio stream must resume at the absolute pos.
expect(computeHandoffPosition(12, 180)).toBe(192);
});
it("handles a direct stream with no offset", () => {
expect(computeHandoffPosition(45, 0)).toBe(45);
});
it("never returns a negative position", () => {
expect(computeHandoffPosition(-5, 0)).toBe(0);
});
});
describe("shouldEnterBackgroundAudio", () => {
it("enters when toggle is on and not already handed off", () => {
expect(shouldEnterBackgroundAudio(true, initialHandoffState)).toBe(true);
});
it("does not enter when the toggle is off", () => {
expect(shouldEnterBackgroundAudio(false, initialHandoffState)).toBe(false);
});
it("does not double-enter when already active", () => {
const active: BackgroundAudioState = { active: true, wasPlaying: true };
expect(shouldEnterBackgroundAudio(true, active)).toBe(false);
});
});
describe("shouldExitBackgroundAudio", () => {
it("exits when a handoff is active", () => {
const active: BackgroundAudioState = { active: true, wasPlaying: false };
expect(shouldExitBackgroundAudio(active)).toBe(true);
});
it("does not exit when no handoff happened", () => {
expect(shouldExitBackgroundAudio(initialHandoffState)).toBe(false);
});
it("exits even if the toggle was turned off while backgrounded", () => {
// shouldExit ignores the toggle by design, so turning it off mid-background
// still returns cleanly to video on foreground.
const active: BackgroundAudioState = { active: true, wasPlaying: true };
expect(shouldExitBackgroundAudio(active)).toBe(true);
});
});
});
@@ -0,0 +1,57 @@
/**
* Pure helpers for the video background-audio handoff (UR-040).
*
* TRACES: UR-040 | DR-052 | UT-060
*
* Kept free of Svelte/DOM so the handoff arithmetic and state transitions are
* unit-testable without mounting the player. The component
* (VideoPlayer.svelte) owns the actual `<video>` teardown and IPC calls.
*/
/**
* Absolute playback position to resume the audio stream at.
*
* Transcoded HLS playback tracks time as `videoElement.currentTime + seekOffset`
* (the element resets to 0 after each transcode reload; `seekOffset` carries the
* cumulative offset). Background audio must resume at that ABSOLUTE position, so
* both terms are summed here mirroring the `effectiveTime` used elsewhere in
* the player.
*/
export function computeHandoffPosition(elementCurrentTime: number, seekOffset: number): number {
const pos = elementCurrentTime + seekOffset;
return pos > 0 ? pos : 0;
}
/**
* The handoff state. `wasPlaying` is captured on the way out so play/pause is
* restored when the app returns to the foreground.
*/
export interface BackgroundAudioState {
active: boolean;
wasPlaying: boolean;
}
export const initialHandoffState: BackgroundAudioState = {
active: false,
wasPlaying: false,
};
/**
* Whether a background signal should trigger the audio handoff right now.
* Only when the toggle is on and we're not already handed off.
*/
export function shouldEnterBackgroundAudio(
toggleOn: boolean,
state: BackgroundAudioState
): boolean {
return toggleOn && !state.active;
}
/**
* Whether a foreground signal should trigger the return to WebView video.
* Only when we actually handed off (regardless of the current toggle value, so
* turning the toggle off while backgrounded still returns cleanly).
*/
export function shouldExitBackgroundAudio(state: BackgroundAudioState): boolean {
return state.active;
}
@@ -0,0 +1,43 @@
import { describe, it, expect, vi, beforeEach } from "vitest";
import { invoke } from "@tauri-apps/api/core";
import { commands } from "$lib/api/bindings";
vi.mock("@tauri-apps/api/core");
// TRACES: UR-040 | DR-052 | UT-061
//
// Guards the Tauri v2 camelCase param rule for the background-audio commands:
// the command NAME stays snake_case; params are camelCase.
describe("background-audio player commands (param naming)", () => {
beforeEach(() => {
vi.clearAllMocks();
});
it("player_enter_background_audio sends item + positionSeconds (camelCase)", async () => {
(invoke as any).mockResolvedValueOnce({});
const item = {
id: "vid-1",
title: "Episode 1",
streamUrl: "https://server/Audio/vid-1/universal?AudioStreamIndex=1",
videoCodec: "aac",
needsTranscoding: false,
};
await commands.playerEnterBackgroundAudio(item, 193);
expect(invoke).toHaveBeenCalledWith("player_enter_background_audio", {
item,
positionSeconds: 193,
});
});
it("player_exit_background_audio takes no params and returns a position", async () => {
(invoke as any).mockResolvedValueOnce(193.5);
const pos = await commands.playerExitBackgroundAudio();
expect(pos).toBe(193.5);
expect(invoke).toHaveBeenCalledWith("player_exit_background_audio");
});
});
+75
View File
@@ -0,0 +1,75 @@
/**
* Background-audio support, Android only.
*
* TRACES: UR-040 | IR-025, DR-051
*
* Keeps a video's *audio* playing when the app is backgrounded or the screen is
* locked, while video decode stops. This is a HANDOFF: the WebView `<video>`
* element (which decodes video) is torn down and the same item is played back
* audio-only through the native ExoPlayer foreground service. It is NOT the
* WebView staying alive an Android WebView `<video>` does not keep audio
* playing once the app is backgrounded.
*
* The `AndroidBackgroundAudio` @JavascriptInterface (installed by MainActivity)
* carries the toggle state to native; native signals background/foreground back
* to the frontend as DOM CustomEvents (`jellytau-background` /
* `jellytau-foreground`) see subscribeAppBackgrounded/Foregrounded below.
*
* Unsupported (no-op) on every non-Android platform.
*/
interface AndroidBackgroundAudioBridge {
setEnabled(enabled: boolean): void;
isSupported(): boolean;
}
declare global {
interface Window {
AndroidBackgroundAudio?: AndroidBackgroundAudioBridge;
}
}
function bridge(): AndroidBackgroundAudioBridge | undefined {
if (typeof window === "undefined") return undefined;
return window.AndroidBackgroundAudio;
}
/** Whether background audio is available — used to decide if the toggle renders. */
export function isBackgroundAudioSupported(): boolean {
try {
return bridge()?.isSupported() ?? false;
} catch (err) {
console.warn("[BgAudio] isSupported check failed:", err);
return false;
}
}
/**
* Arm/disarm background-audio mode for the current video. When armed, the native
* side runs the audio handoff on background instead of entering PiP.
*/
export function setBackgroundAudioEnabled(enabled: boolean): void {
try {
bridge()?.setEnabled(enabled);
} catch (err) {
console.warn("[BgAudio] Failed to set enabled:", err);
}
}
/**
* Subscribe to the native "app backgrounded" signal (Home/app-switch/lock).
* Returns an unsubscribe function. No-op where unsupported (the event never
* fires on non-Android platforms).
*/
export function subscribeAppBackgrounded(handler: () => void): () => void {
if (typeof window === "undefined") return () => {};
window.addEventListener("jellytau-background", handler);
return () => window.removeEventListener("jellytau-background", handler);
}
/** Subscribe to the native "app foregrounded" signal. Returns an unsubscribe fn. */
export function subscribeAppForegrounded(handler: () => void): () => void {
if (typeof window === "undefined") return () => {};
window.addEventListener("jellytau-foreground", handler);
return () => window.removeEventListener("jellytau-foreground", handler);
}
+8 -2
View File
@@ -1,6 +1,8 @@
/**
* Picture-in-picture support, Android only.
*
* TRACES: UR-041 | IR-026 | DR-053
*
* Video on Android renders into a native ExoPlayer SurfaceView behind the
* WebView, so PiP is driven by the Activity (which shrinks into a floating
* window) rather than the HTML5 `requestPictureInPicture()` API. The bridge is
@@ -68,8 +70,12 @@ export function enterPip(): void {
/**
* Enable/disable auto-entering PiP when the user backgrounds the app.
*
* Disabled while casting: playback is happening on another device, so a PiP
* window here would render an empty black box.
* This is a coarse frontend override; the authoritative gate is the native
* `canEnterPip` guard, which already refuses PiP unless a local video surface
* is actively rendering (so audio playback, menu/library browsing, and
* remote/cast sessions never enter PiP regardless of this flag). The only
* caller today is the background-audio toggle, which disarms auto-PiP so the
* two background behaviours stay mutually exclusive.
*/
export function setAutoEnterEnabled(enabled: boolean): void {
try {