android-picture-in-picture #12

Merged
dtourolle merged 5 commits from android-picture-in-picture into master 2026-07-22 20:29:05 +00:00
8 changed files with 569 additions and 41 deletions
Showing only changes of commit 1fa5aa46f9 - Show all commits
+22
View File
@@ -41,6 +41,19 @@ if [ -f "$APP_GRADLE_SRC" ]; then
echo " Copied: app/build.gradle.kts"
fi
# AndroidManifest.xml. `tauri android init` regenerates gen/android from
# tauri.conf.json and would drop our hand-maintained entries (media playback
# service + permissions, hardware acceleration, picture-in-picture attributes
# on MainActivity), so this tracked copy is the source of truth and must be
# restored after a regen. Gradle reads ONLY the gen/ copy - there is no
# manifest-merger hook here, so this must be the complete manifest.
MANIFEST_SRC="$PROJECT_ROOT/src-tauri/android/src/main/AndroidManifest.xml"
MANIFEST_DST="$PROJECT_ROOT/src-tauri/gen/android/app/src/main/AndroidManifest.xml"
if [ -f "$MANIFEST_SRC" ]; then
cp "$MANIFEST_SRC" "$MANIFEST_DST"
echo " Copied: app/src/main/AndroidManifest.xml"
fi
# Custom ProGuard/R8 keep rules. Required for minified release builds:
# the player/ and security/ Kotlin classes are loaded by name via JNI from
# Rust, so R8 can't see the references and would strip them without this.
@@ -65,6 +78,15 @@ if [ -d "$RES_SRC" ]; then
cp "$dir"/* "$RES_DST/$name/"
echo " Copied res: $name"
done
# values/ (themes.xml): status-bar styling that `tauri android init` does
# not generate. Previously this directory was tracked but never copied, so
# the theme customizations below never reached a build.
if [ -d "$RES_SRC/values" ]; then
mkdir -p "$RES_DST/values"
cp "$RES_SRC"/values/*.xml "$RES_DST/values/"
echo " Copied res: values"
fi
# We ship only the color adaptive icon (background + foreground). Drop any
# monochrome layer Tauri may generate: the themed-icon monochrome doesn't
# render well, and our adaptive-icon xml no longer references it, so a stray
@@ -9,6 +9,16 @@
-keep class com.dtourolle.jellytau.player.** { *; }
-keep class com.dtourolle.jellytau.security.** { *; }
# Picture-in-picture is driven from the WebView through an
# @JavascriptInterface bridge, so the only references to these methods live
# in JavaScript. R8 sees them as unused and would strip them, silently
# breaking the PiP button in release builds only.
-keep class com.dtourolle.jellytau.PictureInPictureManager { *; }
-keep class com.dtourolle.jellytau.VideoOverlayManager { *; }
-keepclassmembers class * {
@android.webkit.JavascriptInterface <methods>;
}
# Media3 / ExoPlayer is accessed reflectively in places; keep it intact.
-keep class androidx.media3.** { *; }
-dontwarn androidx.media3.**
-39
View File
@@ -1,39 +0,0 @@
plugins {
id("com.android.library")
id("org.jetbrains.kotlin.android")
}
android {
namespace = "com.dtourolle.jellytau.player"
compileSdk = 36
defaultConfig {
minSdk = 24
}
buildTypes {
getByName("debug") {
}
getByName("release") {
isMinifyEnabled = false
}
}
compileOptions {
sourceCompatibility = JavaVersion.VERSION_1_8
targetCompatibility = JavaVersion.VERSION_1_8
}
kotlinOptions {
jvmTarget = "1.8"
}
}
dependencies {
implementation("androidx.media3:media3-exoplayer:1.5.1")
implementation("androidx.media3:media3-exoplayer-hls:1.5.1")
implementation("androidx.media3:media3-common:1.5.1")
implementation("androidx.media3:media3-session:1.5.1")
implementation("androidx.media:media:1.7.0") // For MediaSessionCompat and VolumeProviderCompat
implementation("org.jetbrains.kotlinx:kotlinx-coroutines-android:1.8.1")
}
+64 -2
View File
@@ -1,5 +1,67 @@
<?xml version="1.0" encoding="utf-8"?>
<!--
Authoritative AndroidManifest for JellyTau.
NOTE: this is NOT a manifest-merger fragment. Gradle only ever reads
gen/android/app/src/main/AndroidManifest.xml, and `tauri android init`
regenerates that file from tauri.conf.json - dropping everything below.
scripts/sync-android-sources.sh copies this file over the generated one,
so this is the full manifest and the single source of truth.
(An earlier version of this file was a partial <application> fragment on the
assumption that Tauri merged it. It did not: the hardwareAccelerated flag it
declared never reached any built APK. It is folded in properly below.)
-->
<manifest xmlns:android="http://schemas.android.com/apk/res/android">
<!-- Enable hardware acceleration for video playback performance -->
<application android:hardwareAccelerated="true" />
<uses-permission android:name="android.permission.INTERNET" />
<uses-permission android:name="android.permission.WAKE_LOCK" />
<uses-permission android:name="android.permission.FOREGROUND_SERVICE" />
<uses-permission android:name="android.permission.FOREGROUND_SERVICE_MEDIA_PLAYBACK" />
<uses-permission android:name="android.permission.POST_NOTIFICATIONS" />
<!-- AndroidTV support -->
<uses-feature android:name="android.software.leanback" android:required="false" />
<application
android:icon="@mipmap/ic_launcher"
android:label="@string/app_name"
android:theme="@style/Theme.jellytau"
android:hardwareAccelerated="true"
android:usesCleartextTraffic="${usesCleartextTraffic}">
<activity
android:configChanges="orientation|keyboardHidden|keyboard|screenSize|locale|smallestScreenSize|screenLayout|uiMode|density"
android:launchMode="singleTask"
android:label="@string/main_activity_title"
android:name=".MainActivity"
android:exported="true"
android:supportsPictureInPicture="true"
android:resizeableActivity="true">
<intent-filter>
<action android:name="android.intent.action.MAIN" />
<category android:name="android.intent.category.LAUNCHER" />
<!-- AndroidTV support -->
<category android:name="android.intent.category.LEANBACK_LAUNCHER" />
</intent-filter>
</activity>
<provider
android:name="androidx.core.content.FileProvider"
android:authorities="${applicationId}.fileprovider"
android:exported="false"
android:grantUriPermissions="true">
<meta-data
android:name="android.support.FILE_PROVIDER_PATHS"
android:resource="@xml/file_paths" />
</provider>
<!-- Media playback service for lockscreen controls -->
<service
android:name="com.dtourolle.jellytau.player.JellyTauPlaybackService"
android:foregroundServiceType="mediaPlayback"
android:exported="true">
<intent-filter>
<action android:name="androidx.media3.session.MediaSessionService" />
</intent-filter>
</service>
</application>
</manifest>
@@ -22,6 +22,14 @@ class MainActivity : TauriActivity() {
private var audioFocusRequest: AudioFocusRequest? = null
private val audioManager by lazy { getSystemService(Context.AUDIO_SERVICE) as AudioManager }
/**
* Whether backgrounding the app during video should auto-enter PiP.
* The frontend clears this when video isn't the active local surface
* (e.g. remote/cast playback) via AndroidPictureInPicture.setAutoEnterEnabled.
*/
@Volatile
private var autoEnterPipEnabled = true
override fun onCreate(savedInstanceState: Bundle?) {
enableEdgeToEdge()
super.onCreate(savedInstanceState)
@@ -37,6 +45,28 @@ class MainActivity : TauriActivity() {
configureWebViewForMedia()
}
/**
* Called when the user leaves the app via Home or the gesture equivalent
* (but NOT via Back). This is the standard hook for auto-entering PiP so
* video keeps playing in a floating window instead of being backgrounded.
*/
override fun onUserLeaveHint() {
super.onUserLeaveHint()
if (autoEnterPipEnabled && PictureInPictureManager.canEnterPip(this)) {
android.util.Log.d("MainActivity", "User leaving with video active - entering PiP")
PictureInPictureManager.enterPip(this)
}
}
override fun onPictureInPictureModeChanged(
isInPictureInPictureMode: Boolean,
newConfig: android.content.res.Configuration
) {
super.onPictureInPictureModeChanged(isInPictureInPictureMode, newConfig)
android.util.Log.d("MainActivity", "PiP mode changed: $isInPictureInPictureMode")
PictureInPictureManager.onPipModeChanged(this, isInPictureInPictureMode)
}
private fun configureWebViewForMedia() {
try {
val webView = findWebView(window.decorView)
@@ -70,6 +100,35 @@ class MainActivity : TauriActivity() {
}, "AndroidAudioFocus")
android.util.Log.d("MainActivity", "JavaScript interface 'AndroidAudioFocus' added")
// Add JavaScript interface for picture-in-picture control.
// enterPip/canEnterPip must run on the main thread; @JavascriptInterface
// methods are invoked on a WebView binder thread.
webView.addJavascriptInterface(object : Any() {
@JavascriptInterface
fun enterPip() {
handler.post { PictureInPictureManager.enterPip(this@MainActivity) }
}
/** Whether the PiP button should be offered in the player UI at all. */
@JavascriptInterface
fun isSupported(): Boolean {
return PictureInPictureManager.isPipSupported(this@MainActivity)
}
/** Whether entering PiP would work right now (local video playing). */
@JavascriptInterface
fun canEnterPip(): Boolean {
return PictureInPictureManager.canEnterPip(this@MainActivity)
}
/** Let the frontend opt out of auto-PiP (e.g. while casting). */
@JavascriptInterface
fun setAutoEnterEnabled(enabled: Boolean) {
autoEnterPipEnabled = enabled
}
}, "AndroidPictureInPicture")
android.util.Log.d("MainActivity", "JavaScript interface 'AndroidPictureInPicture' added")
// Set WebChromeClient to handle video playback and audio focus
webView.webChromeClient = object : WebChromeClient() {
override fun onShowCustomView(view: View?, callback: CustomViewCallback?) {
@@ -0,0 +1,310 @@
package com.dtourolle.jellytau
import android.app.Activity
import android.app.PictureInPictureParams
import android.app.RemoteAction
import android.content.BroadcastReceiver
import android.content.Context
import android.content.Intent
import android.content.IntentFilter
import android.graphics.drawable.Icon
import android.os.Build
import android.util.Rational
import android.view.ViewGroup
import android.webkit.WebView
import androidx.annotation.RequiresApi
import com.dtourolle.jellytau.player.JellyTauPlayer
/**
* Drives Android picture-in-picture for native (ExoPlayer) video playback.
*
* PiP shrinks the whole Activity into a floating window, so the only thing that
* should remain visible is the video SurfaceView that [VideoOverlayManager]
* attached at the bottom of the z-order. The WebView carrying the Svelte UI is
* hidden for the duration - it is opaque and sits *above* the surface, so
* leaving it visible would occlude the video entirely.
*
* Playback itself is untouched: ExoPlayer keeps rendering into the same surface
* across the transition, so entering and leaving PiP never interrupts the video.
*/
object PictureInPictureManager {
private const val TAG = "PictureInPictureManager"
/** Action for the play/pause RemoteAction shown inside the PiP window. */
private const val ACTION_MEDIA_CONTROL = "com.dtourolle.jellytau.PIP_MEDIA_CONTROL"
private const val EXTRA_CONTROL_TYPE = "control_type"
private const val CONTROL_PLAY = 1
private const val CONTROL_PAUSE = 2
/** Request codes must differ per action or the PendingIntents collapse into one. */
private const val REQUEST_PLAY = 101
private const val REQUEST_PAUSE = 102
private var receiver: BroadcastReceiver? = null
private var hiddenWebView: WebView? = null
/**
* Whether this device/OS can do PiP at all. Android 8.0 introduced the API,
* and the user (or device manufacturer) can disable the feature per-app.
*/
fun isPipSupported(activity: Activity): Boolean {
if (Build.VERSION.SDK_INT < Build.VERSION_CODES.O) return false
return activity.packageManager.hasSystemFeature(
android.content.pm.PackageManager.FEATURE_PICTURE_IN_PICTURE
)
}
/**
* Whether entering PiP right now makes sense: a native video must actually
* be playing locally. Audio-only playback and remote/cast sessions render
* nothing on this device, so a PiP window would be an empty black box.
*/
fun canEnterPip(activity: Activity): Boolean {
if (!isPipSupported(activity)) return false
return try {
val player = JellyTauPlayer.getInstance()
player.isPlayingVideo() &&
player.getSurfaceView() != null &&
VideoOverlayManager.isVideoSurfaceAttached()
} catch (e: Exception) {
android.util.Log.w(TAG, "canEnterPip check failed", e)
false
}
}
/**
* Enter picture-in-picture, sizing the window to the video's aspect ratio.
*
* @return true if the system accepted the transition.
*/
fun enterPip(activity: Activity): Boolean {
if (!canEnterPip(activity)) {
android.util.Log.d(TAG, "Not entering PiP: no local video playing")
return false
}
if (Build.VERSION.SDK_INT < Build.VERSION_CODES.O) return false
return try {
val params = buildParams(activity)
val entered = activity.enterPictureInPictureMode(params)
android.util.Log.d(TAG, "enterPictureInPictureMode returned $entered")
entered
} catch (e: Exception) {
// IllegalStateException here means PiP is disallowed (e.g. the user
// turned it off in system settings). Never crash over it.
android.util.Log.e(TAG, "Failed to enter PiP", e)
false
}
}
/**
* Build PiP params: aspect ratio from the current video, plus a play/pause
* RemoteAction reflecting the live playback state.
*/
@RequiresApi(Build.VERSION_CODES.O)
private fun buildParams(activity: Activity): PictureInPictureParams {
val builder = PictureInPictureParams.Builder()
aspectRatioFor()?.let { builder.setAspectRatio(it) }
builder.setActions(listOf(buildPlayPauseAction(activity)))
return builder.build()
}
/**
* The video's aspect ratio, clamped to the range Android accepts.
*
* The platform rejects ratios outside roughly 1:2.39 - 2.39:1 with an
* IllegalArgumentException, which would otherwise take down the Activity on
* unusually tall or wide content.
*/
private fun aspectRatioFor(): Rational? {
val player = try {
JellyTauPlayer.getInstance()
} catch (e: Exception) {
return null
}
val surface = player.getSurfaceView() ?: return null
// The surface has already been letterboxed to the video's aspect ratio
// by fitSurfaceToScreen(), so its measured bounds are the video shape.
val width = surface.width
val height = surface.height
if (width <= 0 || height <= 0) return null
val ratio = width.toDouble() / height.toDouble()
val minRatio = 1.0 / 2.39
val maxRatio = 2.39
val clamped = ratio.coerceIn(minRatio, maxRatio)
// Scale to integers; Rational(width, height) directly can overflow for
// large surfaces, and the clamped value may not match the raw pixels.
return Rational((clamped * 1000).toInt(), 1000)
}
@RequiresApi(Build.VERSION_CODES.O)
private fun buildPlayPauseAction(activity: Activity): RemoteAction {
val isPlaying = try {
JellyTauPlayer.getInstance().getExoPlayer().isPlaying
} catch (e: Exception) {
false
}
val (iconRes, title, controlType, requestCode) = if (isPlaying) {
Quad(
android.R.drawable.ic_media_pause,
"Pause",
CONTROL_PAUSE,
REQUEST_PAUSE
)
} else {
Quad(
android.R.drawable.ic_media_play,
"Play",
CONTROL_PLAY,
REQUEST_PLAY
)
}
val intent = Intent(ACTION_MEDIA_CONTROL)
.putExtra(EXTRA_CONTROL_TYPE, controlType)
// Explicit package keeps the broadcast internal to the app.
.setPackage(activity.packageName)
val flags = android.app.PendingIntent.FLAG_UPDATE_CURRENT or
android.app.PendingIntent.FLAG_IMMUTABLE
val pendingIntent = android.app.PendingIntent.getBroadcast(
activity,
requestCode,
intent,
flags
)
return RemoteAction(
Icon.createWithResource(activity, iconRes),
title,
title,
pendingIntent
)
}
private data class Quad<A, B, C, D>(
val first: A,
val second: B,
val third: C,
val fourth: D
)
/**
* Refresh the PiP window's action button so it tracks play/pause state
* while the window is open. Safe to call when not in PiP (no-op).
*/
fun updatePipActions(activity: Activity) {
if (Build.VERSION.SDK_INT < Build.VERSION_CODES.O) return
if (!activity.isInPictureInPictureMode) return
try {
activity.setPictureInPictureParams(buildParams(activity))
} catch (e: Exception) {
android.util.Log.w(TAG, "Failed to update PiP actions", e)
}
}
/**
* Called from MainActivity.onPictureInPictureModeChanged.
*
* Entering: hide the WebView so only the video surface shows, and register
* the receiver backing the PiP play/pause button.
* Leaving: restore the WebView and unregister.
*/
fun onPipModeChanged(activity: Activity, isInPipMode: Boolean) {
if (isInPipMode) {
hideWebView(activity)
registerReceiver(activity)
} else {
unregisterReceiver(activity)
showWebView()
// The surface was laid out against the tiny PiP bounds; re-fit it to
// the restored full-screen bounds or the video stays postage-stamp sized.
try {
JellyTauPlayer.getInstance().fitSurfaceToScreen()
} catch (e: Exception) {
android.util.Log.w(TAG, "Failed to re-fit surface after PiP", e)
}
}
}
private fun hideWebView(activity: Activity) {
val webView = findWebView(activity.window.decorView)
if (webView == null) {
android.util.Log.w(TAG, "No WebView found to hide for PiP")
return
}
// GONE rather than INVISIBLE: the WebView is opaque, and GONE also stops
// it from consuming layout space in the shrunken window.
webView.visibility = android.view.View.GONE
hiddenWebView = webView
android.util.Log.d(TAG, "WebView hidden for PiP")
}
private fun showWebView() {
hiddenWebView?.let {
it.visibility = android.view.View.VISIBLE
android.util.Log.d(TAG, "WebView restored after PiP")
}
hiddenWebView = null
}
private fun registerReceiver(activity: Activity) {
if (receiver != null) return
val r = object : BroadcastReceiver() {
override fun onReceive(context: Context?, intent: Intent?) {
if (intent?.action != ACTION_MEDIA_CONTROL) return
val player = try {
JellyTauPlayer.getInstance()
} catch (e: Exception) {
return
}
when (intent.getIntExtra(EXTRA_CONTROL_TYPE, 0)) {
CONTROL_PLAY -> player.play()
CONTROL_PAUSE -> player.pause()
}
// Swap the button to reflect the new state.
updatePipActions(activity)
}
}
val filter = IntentFilter(ACTION_MEDIA_CONTROL)
if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.TIRAMISU) {
activity.registerReceiver(r, filter, Context.RECEIVER_NOT_EXPORTED)
} else {
@Suppress("UnspecifiedRegisterReceiverFlag")
activity.registerReceiver(r, filter)
}
receiver = r
android.util.Log.d(TAG, "PiP media control receiver registered")
}
private fun unregisterReceiver(activity: Activity) {
receiver?.let {
try {
activity.unregisterReceiver(it)
} catch (e: IllegalArgumentException) {
// Already unregistered - harmless.
}
}
receiver = null
}
private fun findWebView(view: android.view.View): WebView? {
if (view is WebView) return view
if (view is ViewGroup) {
for (i in 0 until view.childCount) {
findWebView(view.getChildAt(i))?.let { return it }
}
}
return null
}
}
@@ -18,6 +18,7 @@
import { playerController } from "$lib/player";
import { Html5PlayerAdapter, type Html5ElementBridge } from "$lib/player/adapters";
import { createRustReportHost } from "$lib/player/adapters/rustReportHost";
import { isPipSupported, enterPip } from "$lib/utils/pictureInPicture";
interface Props {
media: MediaItem | null;
@@ -1081,6 +1082,16 @@
}
}
// Resolved once at component setup: the PiP bridge is installed by
// MainActivity before the page loads and never changes for the session.
// Synchronous by design - no await in onMount (see VideoPlayer native-mode
// pitfalls: awaiting there flips the component into HTML5 mode).
const pipSupported = isPipSupported();
function handlePictureInPicture() {
enterPip();
}
function toggleFullscreen() {
if (!document.fullscreenElement) {
document.documentElement.requestFullscreen();
@@ -1721,6 +1732,19 @@
<!-- Volume Control -->
<VolumeControl size="md" />
<!-- Picture-in-picture (Android only) -->
{#if pipSupported}
<button
onclick={handlePictureInPicture}
class="text-white hover:text-gray-300"
aria-label="Picture in picture"
>
<svg class="w-6 h-6" fill="currentColor" viewBox="0 0 24 24">
<path d="M19 11h-8v6h8v-6zm4 8V4.98C23 3.88 22.1 3 21 3H3c-1.1 0-2 .88-2 1.98V19c0 1.1.9 2 2 2h18c1.1 0 2-.9 2-2zm-2 .02H3V4.97h18v14.05z" />
</svg>
</button>
{/if}
<!-- Fullscreen -->
<button onclick={toggleFullscreen} class="text-white hover:text-gray-300" aria-label="Toggle fullscreen">
<svg class="w-6 h-6" fill="currentColor" viewBox="0 0 24 24">
+80
View File
@@ -0,0 +1,80 @@
/**
* Picture-in-picture support, Android only.
*
* Video on Android renders into a native ExoPlayer SurfaceView behind the
* WebView, so PiP is driven by the Activity (which shrinks into a floating
* window) rather than the HTML5 `requestPictureInPicture()` API. The bridge is
* the `AndroidPictureInPicture` @JavascriptInterface installed by MainActivity.
*
* On every other platform this module reports unsupported. Notably WebKitGTK
* (the Linux webview) does not implement the Picture-in-Picture Web API at all,
* so there is no HTML5 fallback to reach for.
*/
interface AndroidPictureInPictureBridge {
enterPip(): void;
isSupported(): boolean;
canEnterPip(): boolean;
setAutoEnterEnabled(enabled: boolean): void;
}
declare global {
interface Window {
AndroidPictureInPicture?: AndroidPictureInPictureBridge;
}
}
function bridge(): AndroidPictureInPictureBridge | undefined {
if (typeof window === "undefined") return undefined;
return window.AndroidPictureInPicture;
}
/**
* Whether the device can do PiP at all - used to decide if the button should
* be rendered. False on desktop, and on Android devices where the user has
* disabled the feature.
*/
export function isPipSupported(): boolean {
try {
return bridge()?.isSupported() ?? false;
} catch (err) {
console.warn("[PiP] isSupported check failed:", err);
return false;
}
}
/**
* Whether entering PiP would succeed right now: a native video must be playing
* locally. False during audio playback and while casting to a remote session.
*/
export function canEnterPip(): boolean {
try {
return bridge()?.canEnterPip() ?? false;
} catch (err) {
console.warn("[PiP] canEnterPip check failed:", err);
return false;
}
}
/** Enter picture-in-picture. No-op where unsupported. */
export function enterPip(): void {
try {
bridge()?.enterPip();
} catch (err) {
console.error("[PiP] Failed to enter picture-in-picture:", err);
}
}
/**
* Enable/disable auto-entering PiP when the user backgrounds the app.
*
* Disabled while casting: playback is happening on another device, so a PiP
* window here would render an empty black box.
*/
export function setAutoEnterEnabled(enabled: boolean): void {
try {
bridge()?.setAutoEnterEnabled(enabled);
} catch (err) {
console.warn("[PiP] Failed to set auto-enter:", err);
}
}