From 1fa5aa46f98a27068970a0c3d8c9e35b1f38c9d8 Mon Sep 17 00:00:00 2001 From: Duncan Tourolle Date: Sat, 18 Jul 2026 12:59:39 +0200 Subject: [PATCH 1/5] Android picture-in-picture, and fix three dead Android config files Add PiP for native (ExoPlayer) video on Android. Video renders into a SurfaceView behind the WebView, so PiP is driven by the Activity shrinking into a floating window rather than the HTML5 PiP API (which WebKitGTK does not implement, hence Android-only). - PictureInPictureManager.kt: enter PiP with the video's aspect ratio (clamped to the 1:2.39-2.39:1 range Android accepts, outside which it throws), plus a play/pause RemoteAction. Hides the WebView while in PiP - it is opaque and sits above the surface, so it would otherwise occlude the video entirely - and re-fits the surface on exit. - MainActivity.kt: onUserLeaveHint auto-PiP, onPictureInPictureModeChanged, and an AndroidPictureInPicture JS interface following the existing AndroidAudioFocus pattern. - pictureInPicture.ts + VideoPlayer.svelte: PiP button, rendered only when the native bridge reports support. - proguard: keep rules for @JavascriptInterface methods, which are only referenced from JS and would be stripped in minified release builds. Casting needs no special handling: canEnterPip() checks natively that a local video surface is attached and playing, which a remote session lacks. While wiring the manifest, found that three tracked files under src-tauri/android/ were never reaching any build. Gradle reads only gen/android/app/src/main/, and sync-android-sources.sh did not copy them: - src/main/AndroidManifest.xml was a partial fragment written as if Tauri merged it. It does not - there is no manifest-merger hook here, so its hardwareAccelerated flag never reached an APK. Promoted to the complete authoritative manifest (folding in that flag) and synced. - src/main/res/values/themes.xml (transparent status bar, fitsSystemWindows) was never copied; the sync only globbed mipmap-*. Now synced. - build.gradle.kts was a leftover com.android.library module config with stale media3 1.5.1 deps. The live deps are in app/build.gradle.kts at 1.5.0. Deleted. Verified: merged manifest now carries hardwareAccelerated, supportsPictureInPicture, resizeableActivity and the density configChange; themes.xml compiles into merged resources; Kotlin builds warning-free; svelte-check clean; 537 frontend tests pass. Not verified: PiP behaviour on a device, and the release keep rules against a minified build. assembleUniversalDebug cannot complete in this environment - the Rust step wants a dev-server addr file that only exists under `tauri android dev`. Co-Authored-By: Claude Opus 4.8 --- scripts/sync-android-sources.sh | 22 ++ src-tauri/android/app/proguard-jellytau.pro | 10 + src-tauri/android/build.gradle.kts | 39 --- .../android/src/main/AndroidManifest.xml | 66 +++- .../com/dtourolle/jellytau/MainActivity.kt | 59 ++++ .../jellytau/PictureInPictureManager.kt | 310 ++++++++++++++++++ src/lib/components/player/VideoPlayer.svelte | 24 ++ src/lib/utils/pictureInPicture.ts | 80 +++++ 8 files changed, 569 insertions(+), 41 deletions(-) delete mode 100644 src-tauri/android/build.gradle.kts create mode 100644 src-tauri/android/src/main/java/com/dtourolle/jellytau/PictureInPictureManager.kt create mode 100644 src/lib/utils/pictureInPicture.ts diff --git a/scripts/sync-android-sources.sh b/scripts/sync-android-sources.sh index 9a32be27..269c896d 100755 --- a/scripts/sync-android-sources.sh +++ b/scripts/sync-android-sources.sh @@ -41,6 +41,19 @@ if [ -f "$APP_GRADLE_SRC" ]; then echo " Copied: app/build.gradle.kts" fi +# AndroidManifest.xml. `tauri android init` regenerates gen/android from +# tauri.conf.json and would drop our hand-maintained entries (media playback +# service + permissions, hardware acceleration, picture-in-picture attributes +# on MainActivity), so this tracked copy is the source of truth and must be +# restored after a regen. Gradle reads ONLY the gen/ copy - there is no +# manifest-merger hook here, so this must be the complete manifest. +MANIFEST_SRC="$PROJECT_ROOT/src-tauri/android/src/main/AndroidManifest.xml" +MANIFEST_DST="$PROJECT_ROOT/src-tauri/gen/android/app/src/main/AndroidManifest.xml" +if [ -f "$MANIFEST_SRC" ]; then + cp "$MANIFEST_SRC" "$MANIFEST_DST" + echo " Copied: app/src/main/AndroidManifest.xml" +fi + # Custom ProGuard/R8 keep rules. Required for minified release builds: # the player/ and security/ Kotlin classes are loaded by name via JNI from # Rust, so R8 can't see the references and would strip them without this. @@ -65,6 +78,15 @@ if [ -d "$RES_SRC" ]; then cp "$dir"/* "$RES_DST/$name/" echo " Copied res: $name" done + + # values/ (themes.xml): status-bar styling that `tauri android init` does + # not generate. Previously this directory was tracked but never copied, so + # the theme customizations below never reached a build. + if [ -d "$RES_SRC/values" ]; then + mkdir -p "$RES_DST/values" + cp "$RES_SRC"/values/*.xml "$RES_DST/values/" + echo " Copied res: values" + fi # We ship only the color adaptive icon (background + foreground). Drop any # monochrome layer Tauri may generate: the themed-icon monochrome doesn't # render well, and our adaptive-icon xml no longer references it, so a stray diff --git a/src-tauri/android/app/proguard-jellytau.pro b/src-tauri/android/app/proguard-jellytau.pro index 210db214..e23757b8 100644 --- a/src-tauri/android/app/proguard-jellytau.pro +++ b/src-tauri/android/app/proguard-jellytau.pro @@ -9,6 +9,16 @@ -keep class com.dtourolle.jellytau.player.** { *; } -keep class com.dtourolle.jellytau.security.** { *; } +# Picture-in-picture is driven from the WebView through an +# @JavascriptInterface bridge, so the only references to these methods live +# in JavaScript. R8 sees them as unused and would strip them, silently +# breaking the PiP button in release builds only. +-keep class com.dtourolle.jellytau.PictureInPictureManager { *; } +-keep class com.dtourolle.jellytau.VideoOverlayManager { *; } +-keepclassmembers class * { + @android.webkit.JavascriptInterface ; +} + # Media3 / ExoPlayer is accessed reflectively in places; keep it intact. -keep class androidx.media3.** { *; } -dontwarn androidx.media3.** diff --git a/src-tauri/android/build.gradle.kts b/src-tauri/android/build.gradle.kts deleted file mode 100644 index 3ddbaaf7..00000000 --- a/src-tauri/android/build.gradle.kts +++ /dev/null @@ -1,39 +0,0 @@ -plugins { - id("com.android.library") - id("org.jetbrains.kotlin.android") -} - -android { - namespace = "com.dtourolle.jellytau.player" - compileSdk = 36 - - defaultConfig { - minSdk = 24 - } - - buildTypes { - getByName("debug") { - } - getByName("release") { - isMinifyEnabled = false - } - } - - compileOptions { - sourceCompatibility = JavaVersion.VERSION_1_8 - targetCompatibility = JavaVersion.VERSION_1_8 - } - - kotlinOptions { - jvmTarget = "1.8" - } -} - -dependencies { - implementation("androidx.media3:media3-exoplayer:1.5.1") - implementation("androidx.media3:media3-exoplayer-hls:1.5.1") - implementation("androidx.media3:media3-common:1.5.1") - implementation("androidx.media3:media3-session:1.5.1") - implementation("androidx.media:media:1.7.0") // For MediaSessionCompat and VolumeProviderCompat - implementation("org.jetbrains.kotlinx:kotlinx-coroutines-android:1.8.1") -} diff --git a/src-tauri/android/src/main/AndroidManifest.xml b/src-tauri/android/src/main/AndroidManifest.xml index 643a70d3..bfb25b50 100644 --- a/src-tauri/android/src/main/AndroidManifest.xml +++ b/src-tauri/android/src/main/AndroidManifest.xml @@ -1,5 +1,67 @@ + - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/src-tauri/android/src/main/java/com/dtourolle/jellytau/MainActivity.kt b/src-tauri/android/src/main/java/com/dtourolle/jellytau/MainActivity.kt index 25a6834a..f263ff83 100644 --- a/src-tauri/android/src/main/java/com/dtourolle/jellytau/MainActivity.kt +++ b/src-tauri/android/src/main/java/com/dtourolle/jellytau/MainActivity.kt @@ -22,6 +22,14 @@ class MainActivity : TauriActivity() { private var audioFocusRequest: AudioFocusRequest? = null private val audioManager by lazy { getSystemService(Context.AUDIO_SERVICE) as AudioManager } + /** + * Whether backgrounding the app during video should auto-enter PiP. + * The frontend clears this when video isn't the active local surface + * (e.g. remote/cast playback) via AndroidPictureInPicture.setAutoEnterEnabled. + */ + @Volatile + private var autoEnterPipEnabled = true + override fun onCreate(savedInstanceState: Bundle?) { enableEdgeToEdge() super.onCreate(savedInstanceState) @@ -37,6 +45,28 @@ class MainActivity : TauriActivity() { configureWebViewForMedia() } + /** + * Called when the user leaves the app via Home or the gesture equivalent + * (but NOT via Back). This is the standard hook for auto-entering PiP so + * video keeps playing in a floating window instead of being backgrounded. + */ + override fun onUserLeaveHint() { + super.onUserLeaveHint() + if (autoEnterPipEnabled && PictureInPictureManager.canEnterPip(this)) { + android.util.Log.d("MainActivity", "User leaving with video active - entering PiP") + PictureInPictureManager.enterPip(this) + } + } + + override fun onPictureInPictureModeChanged( + isInPictureInPictureMode: Boolean, + newConfig: android.content.res.Configuration + ) { + super.onPictureInPictureModeChanged(isInPictureInPictureMode, newConfig) + android.util.Log.d("MainActivity", "PiP mode changed: $isInPictureInPictureMode") + PictureInPictureManager.onPipModeChanged(this, isInPictureInPictureMode) + } + private fun configureWebViewForMedia() { try { val webView = findWebView(window.decorView) @@ -70,6 +100,35 @@ class MainActivity : TauriActivity() { }, "AndroidAudioFocus") android.util.Log.d("MainActivity", "JavaScript interface 'AndroidAudioFocus' added") + // Add JavaScript interface for picture-in-picture control. + // enterPip/canEnterPip must run on the main thread; @JavascriptInterface + // methods are invoked on a WebView binder thread. + webView.addJavascriptInterface(object : Any() { + @JavascriptInterface + fun enterPip() { + handler.post { PictureInPictureManager.enterPip(this@MainActivity) } + } + + /** Whether the PiP button should be offered in the player UI at all. */ + @JavascriptInterface + fun isSupported(): Boolean { + return PictureInPictureManager.isPipSupported(this@MainActivity) + } + + /** Whether entering PiP would work right now (local video playing). */ + @JavascriptInterface + fun canEnterPip(): Boolean { + return PictureInPictureManager.canEnterPip(this@MainActivity) + } + + /** Let the frontend opt out of auto-PiP (e.g. while casting). */ + @JavascriptInterface + fun setAutoEnterEnabled(enabled: Boolean) { + autoEnterPipEnabled = enabled + } + }, "AndroidPictureInPicture") + android.util.Log.d("MainActivity", "JavaScript interface 'AndroidPictureInPicture' added") + // Set WebChromeClient to handle video playback and audio focus webView.webChromeClient = object : WebChromeClient() { override fun onShowCustomView(view: View?, callback: CustomViewCallback?) { diff --git a/src-tauri/android/src/main/java/com/dtourolle/jellytau/PictureInPictureManager.kt b/src-tauri/android/src/main/java/com/dtourolle/jellytau/PictureInPictureManager.kt new file mode 100644 index 00000000..731a4b2c --- /dev/null +++ b/src-tauri/android/src/main/java/com/dtourolle/jellytau/PictureInPictureManager.kt @@ -0,0 +1,310 @@ +package com.dtourolle.jellytau + +import android.app.Activity +import android.app.PictureInPictureParams +import android.app.RemoteAction +import android.content.BroadcastReceiver +import android.content.Context +import android.content.Intent +import android.content.IntentFilter +import android.graphics.drawable.Icon +import android.os.Build +import android.util.Rational +import android.view.ViewGroup +import android.webkit.WebView +import androidx.annotation.RequiresApi +import com.dtourolle.jellytau.player.JellyTauPlayer + +/** + * Drives Android picture-in-picture for native (ExoPlayer) video playback. + * + * PiP shrinks the whole Activity into a floating window, so the only thing that + * should remain visible is the video SurfaceView that [VideoOverlayManager] + * attached at the bottom of the z-order. The WebView carrying the Svelte UI is + * hidden for the duration - it is opaque and sits *above* the surface, so + * leaving it visible would occlude the video entirely. + * + * Playback itself is untouched: ExoPlayer keeps rendering into the same surface + * across the transition, so entering and leaving PiP never interrupts the video. + */ +object PictureInPictureManager { + + private const val TAG = "PictureInPictureManager" + + /** Action for the play/pause RemoteAction shown inside the PiP window. */ + private const val ACTION_MEDIA_CONTROL = "com.dtourolle.jellytau.PIP_MEDIA_CONTROL" + private const val EXTRA_CONTROL_TYPE = "control_type" + private const val CONTROL_PLAY = 1 + private const val CONTROL_PAUSE = 2 + + /** Request codes must differ per action or the PendingIntents collapse into one. */ + private const val REQUEST_PLAY = 101 + private const val REQUEST_PAUSE = 102 + + private var receiver: BroadcastReceiver? = null + private var hiddenWebView: WebView? = null + + /** + * Whether this device/OS can do PiP at all. Android 8.0 introduced the API, + * and the user (or device manufacturer) can disable the feature per-app. + */ + fun isPipSupported(activity: Activity): Boolean { + if (Build.VERSION.SDK_INT < Build.VERSION_CODES.O) return false + return activity.packageManager.hasSystemFeature( + android.content.pm.PackageManager.FEATURE_PICTURE_IN_PICTURE + ) + } + + /** + * Whether entering PiP right now makes sense: a native video must actually + * be playing locally. Audio-only playback and remote/cast sessions render + * nothing on this device, so a PiP window would be an empty black box. + */ + fun canEnterPip(activity: Activity): Boolean { + if (!isPipSupported(activity)) return false + return try { + val player = JellyTauPlayer.getInstance() + player.isPlayingVideo() && + player.getSurfaceView() != null && + VideoOverlayManager.isVideoSurfaceAttached() + } catch (e: Exception) { + android.util.Log.w(TAG, "canEnterPip check failed", e) + false + } + } + + /** + * Enter picture-in-picture, sizing the window to the video's aspect ratio. + * + * @return true if the system accepted the transition. + */ + fun enterPip(activity: Activity): Boolean { + if (!canEnterPip(activity)) { + android.util.Log.d(TAG, "Not entering PiP: no local video playing") + return false + } + if (Build.VERSION.SDK_INT < Build.VERSION_CODES.O) return false + + return try { + val params = buildParams(activity) + val entered = activity.enterPictureInPictureMode(params) + android.util.Log.d(TAG, "enterPictureInPictureMode returned $entered") + entered + } catch (e: Exception) { + // IllegalStateException here means PiP is disallowed (e.g. the user + // turned it off in system settings). Never crash over it. + android.util.Log.e(TAG, "Failed to enter PiP", e) + false + } + } + + /** + * Build PiP params: aspect ratio from the current video, plus a play/pause + * RemoteAction reflecting the live playback state. + */ + @RequiresApi(Build.VERSION_CODES.O) + private fun buildParams(activity: Activity): PictureInPictureParams { + val builder = PictureInPictureParams.Builder() + + aspectRatioFor()?.let { builder.setAspectRatio(it) } + builder.setActions(listOf(buildPlayPauseAction(activity))) + + return builder.build() + } + + /** + * The video's aspect ratio, clamped to the range Android accepts. + * + * The platform rejects ratios outside roughly 1:2.39 - 2.39:1 with an + * IllegalArgumentException, which would otherwise take down the Activity on + * unusually tall or wide content. + */ + private fun aspectRatioFor(): Rational? { + val player = try { + JellyTauPlayer.getInstance() + } catch (e: Exception) { + return null + } + + val surface = player.getSurfaceView() ?: return null + // The surface has already been letterboxed to the video's aspect ratio + // by fitSurfaceToScreen(), so its measured bounds are the video shape. + val width = surface.width + val height = surface.height + if (width <= 0 || height <= 0) return null + + val ratio = width.toDouble() / height.toDouble() + val minRatio = 1.0 / 2.39 + val maxRatio = 2.39 + val clamped = ratio.coerceIn(minRatio, maxRatio) + + // Scale to integers; Rational(width, height) directly can overflow for + // large surfaces, and the clamped value may not match the raw pixels. + return Rational((clamped * 1000).toInt(), 1000) + } + + @RequiresApi(Build.VERSION_CODES.O) + private fun buildPlayPauseAction(activity: Activity): RemoteAction { + val isPlaying = try { + JellyTauPlayer.getInstance().getExoPlayer().isPlaying + } catch (e: Exception) { + false + } + + val (iconRes, title, controlType, requestCode) = if (isPlaying) { + Quad( + android.R.drawable.ic_media_pause, + "Pause", + CONTROL_PAUSE, + REQUEST_PAUSE + ) + } else { + Quad( + android.R.drawable.ic_media_play, + "Play", + CONTROL_PLAY, + REQUEST_PLAY + ) + } + + val intent = Intent(ACTION_MEDIA_CONTROL) + .putExtra(EXTRA_CONTROL_TYPE, controlType) + // Explicit package keeps the broadcast internal to the app. + .setPackage(activity.packageName) + + val flags = android.app.PendingIntent.FLAG_UPDATE_CURRENT or + android.app.PendingIntent.FLAG_IMMUTABLE + + val pendingIntent = android.app.PendingIntent.getBroadcast( + activity, + requestCode, + intent, + flags + ) + + return RemoteAction( + Icon.createWithResource(activity, iconRes), + title, + title, + pendingIntent + ) + } + + private data class Quad( + val first: A, + val second: B, + val third: C, + val fourth: D + ) + + /** + * Refresh the PiP window's action button so it tracks play/pause state + * while the window is open. Safe to call when not in PiP (no-op). + */ + fun updatePipActions(activity: Activity) { + if (Build.VERSION.SDK_INT < Build.VERSION_CODES.O) return + if (!activity.isInPictureInPictureMode) return + try { + activity.setPictureInPictureParams(buildParams(activity)) + } catch (e: Exception) { + android.util.Log.w(TAG, "Failed to update PiP actions", e) + } + } + + /** + * Called from MainActivity.onPictureInPictureModeChanged. + * + * Entering: hide the WebView so only the video surface shows, and register + * the receiver backing the PiP play/pause button. + * Leaving: restore the WebView and unregister. + */ + fun onPipModeChanged(activity: Activity, isInPipMode: Boolean) { + if (isInPipMode) { + hideWebView(activity) + registerReceiver(activity) + } else { + unregisterReceiver(activity) + showWebView() + // The surface was laid out against the tiny PiP bounds; re-fit it to + // the restored full-screen bounds or the video stays postage-stamp sized. + try { + JellyTauPlayer.getInstance().fitSurfaceToScreen() + } catch (e: Exception) { + android.util.Log.w(TAG, "Failed to re-fit surface after PiP", e) + } + } + } + + private fun hideWebView(activity: Activity) { + val webView = findWebView(activity.window.decorView) + if (webView == null) { + android.util.Log.w(TAG, "No WebView found to hide for PiP") + return + } + // GONE rather than INVISIBLE: the WebView is opaque, and GONE also stops + // it from consuming layout space in the shrunken window. + webView.visibility = android.view.View.GONE + hiddenWebView = webView + android.util.Log.d(TAG, "WebView hidden for PiP") + } + + private fun showWebView() { + hiddenWebView?.let { + it.visibility = android.view.View.VISIBLE + android.util.Log.d(TAG, "WebView restored after PiP") + } + hiddenWebView = null + } + + private fun registerReceiver(activity: Activity) { + if (receiver != null) return + + val r = object : BroadcastReceiver() { + override fun onReceive(context: Context?, intent: Intent?) { + if (intent?.action != ACTION_MEDIA_CONTROL) return + val player = try { + JellyTauPlayer.getInstance() + } catch (e: Exception) { + return + } + when (intent.getIntExtra(EXTRA_CONTROL_TYPE, 0)) { + CONTROL_PLAY -> player.play() + CONTROL_PAUSE -> player.pause() + } + // Swap the button to reflect the new state. + updatePipActions(activity) + } + } + + val filter = IntentFilter(ACTION_MEDIA_CONTROL) + if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.TIRAMISU) { + activity.registerReceiver(r, filter, Context.RECEIVER_NOT_EXPORTED) + } else { + @Suppress("UnspecifiedRegisterReceiverFlag") + activity.registerReceiver(r, filter) + } + receiver = r + android.util.Log.d(TAG, "PiP media control receiver registered") + } + + private fun unregisterReceiver(activity: Activity) { + receiver?.let { + try { + activity.unregisterReceiver(it) + } catch (e: IllegalArgumentException) { + // Already unregistered - harmless. + } + } + receiver = null + } + + private fun findWebView(view: android.view.View): WebView? { + if (view is WebView) return view + if (view is ViewGroup) { + for (i in 0 until view.childCount) { + findWebView(view.getChildAt(i))?.let { return it } + } + } + return null + } +} diff --git a/src/lib/components/player/VideoPlayer.svelte b/src/lib/components/player/VideoPlayer.svelte index c1bfb710..9e763168 100644 --- a/src/lib/components/player/VideoPlayer.svelte +++ b/src/lib/components/player/VideoPlayer.svelte @@ -18,6 +18,7 @@ import { playerController } from "$lib/player"; import { Html5PlayerAdapter, type Html5ElementBridge } from "$lib/player/adapters"; import { createRustReportHost } from "$lib/player/adapters/rustReportHost"; + import { isPipSupported, enterPip } from "$lib/utils/pictureInPicture"; interface Props { media: MediaItem | null; @@ -1081,6 +1082,16 @@ } } + // Resolved once at component setup: the PiP bridge is installed by + // MainActivity before the page loads and never changes for the session. + // Synchronous by design - no await in onMount (see VideoPlayer native-mode + // pitfalls: awaiting there flips the component into HTML5 mode). + const pipSupported = isPipSupported(); + + function handlePictureInPicture() { + enterPip(); + } + function toggleFullscreen() { if (!document.fullscreenElement) { document.documentElement.requestFullscreen(); @@ -1721,6 +1732,19 @@ + + {#if pipSupported} + + {/if} +