using System; using System.Buffers.Binary; using System.Diagnostics; using System.Globalization; using System.IO; using System.Threading; using System.Threading.Tasks; using MediaBrowser.Controller.MediaEncoding; using Microsoft.Extensions.Logging; namespace Jellyfin.Plugin.JRay.Services; /// /// Computes the v1 audio signature for a media file, decoding its centre window /// with the FFmpeg binary Jellyfin already ships. /// /// /// No new dependency. FFmpeg performs decode, downmix and resample; the /// plugin adds only the fixed FFT and bin-peak extraction in /// . The binary is reached through /// , so an installation that can /// transcode can compute signatures, with nothing further to install and no /// second copy of FFmpeg to keep in step. /// /// The pipeline computes the same signature for files it processes locally /// (extraction IR-004); this exists for the files it never sees. Both /// producers must therefore agree exactly, including on the decode: the command /// below is the CLI spelling of what the pipeline asks libswresample for — best /// audio stream, mono, 11025 Hz, 32-bit float — and the golden fixture pins the /// decoded PCM as well as the signature, so a codec-level divergence is /// distinguishable from a DSP-level one (JR-043). /// /// /// Every failure degrades to no signature rather than to an error. A /// signature is an enhancement to cut matching; a missing one costs a tier, and /// must never be able to break a fetch. /// /// // TRACES: JR-042 | SR-003 public class AudioSignatureService { private readonly IMediaEncoder _encoder; private readonly ILogger _logger; /// /// Initializes a new instance of the class. /// /// Supplies the path of the FFmpeg binary Jellyfin ships. /// Logger. public AudioSignatureService(IMediaEncoder encoder, ILogger logger) { _encoder = encoder; _logger = logger; } /// /// Computes the signature of the 120 s window centred on the media's /// midpoint. /// /// /// The centre is used because the head and tail are the least /// content-specific parts of a release: logos and cold opens at one end, /// credits at the other. /// /// Media shorter than yields /// null: the window underflows, so there is no signature — the /// identical rule the extraction producer applies, since diverging here /// would break exactly the short items most likely to be misidentified /// (JR-044, whose remaining half — applying no sync offset — belongs with /// signature matching). /// /// /// Path of the media file. /// The item's runtime, as Jellyfin knows it. /// Cancellation token. /// /// The v1:-prefixed signature, or null for short media, media /// with no usable audio, and any decode failure. /// public Task ComputeAsync( string path, double runtimeSeconds, CancellationToken cancellationToken) => ComputeWithEncoderAsync(_encoder?.EncoderPath, path, runtimeSeconds, _logger, cancellationToken); /// /// with the FFmpeg binary named explicitly. /// /// /// Internal so the golden-fixture test can drive the real decode path with /// whatever FFmpeg the machine has, rather than standing up a fake /// — a stub of a thirty-member interface would /// be the larger risk of the two, and it is the decode that is under test. /// /// Path of the FFmpeg binary to run. /// Path of the media file. /// The item's runtime, as Jellyfin knows it. /// Logger. /// Cancellation token. /// The v1:-prefixed signature, or null. internal static async Task ComputeWithEncoderAsync( string? encoderPath, string path, double runtimeSeconds, ILogger logger, CancellationToken cancellationToken) { if (string.IsNullOrEmpty(path) || runtimeSeconds < AudioSignature.WindowSec) { return null; } if (string.IsNullOrEmpty(encoderPath)) { logger.LogDebug("No FFmpeg binary available; skipping the audio signature for {Path}", path); return null; } try { var samples = await DecodeCentreWindowAsync(encoderPath, path, runtimeSeconds, logger, cancellationToken) .ConfigureAwait(false); if (samples is null) { return null; } return AudioSignature.FromMonoSamples(samples); } catch (OperationCanceledException) { throw; } catch (Exception ex) { logger.LogDebug(ex, "Audio signature failed for {Path}; continuing without one", path); return null; } } /// /// Decodes the centre window as mono 32-bit float PCM at 11025 Hz. /// /// /// The stream is truncated to exactly /// samples, so the frame count is /// the same for every input rather than wobbling with seek granularity or a /// resampler tail. /// /// No -map is given: FFmpeg's default audio selection is the same /// "best stream" choice the pipeline makes with /// av_find_best_stream, and naming 0:a:0 instead would pick a /// different track from the pipeline's on any file whose first audio stream /// is not its main one — a commentary track, say. /// /// private static async Task DecodeCentreWindowAsync( string encoderPath, string path, double runtimeSeconds, ILogger logger, CancellationToken cancellationToken) { var start = (runtimeSeconds / 2.0) - (AudioSignature.WindowSec / 2.0); if (start < 0.0) { start = 0.0; } var startArgument = start.ToString("0.000", CultureInfo.InvariantCulture); var windowArgument = AudioSignature.WindowSec.ToString("0.000", CultureInfo.InvariantCulture); var rateArgument = AudioSignature.SampleRate.ToString(CultureInfo.InvariantCulture); var startInfo = new ProcessStartInfo { FileName = encoderPath, RedirectStandardOutput = true, RedirectStandardError = true, RedirectStandardInput = false, UseShellExecute = false, CreateNoWindow = true, }; startInfo.ArgumentList.Add("-nostdin"); startInfo.ArgumentList.Add("-v"); startInfo.ArgumentList.Add("error"); // Input seeking, so FFmpeg does not decode the whole file to reach the // middle of it. Accurate by default: it seeks to the preceding keyframe // and discards the excess, which is what the pipeline does by hand. startInfo.ArgumentList.Add("-ss"); startInfo.ArgumentList.Add(startArgument); startInfo.ArgumentList.Add("-i"); startInfo.ArgumentList.Add(path); startInfo.ArgumentList.Add("-t"); startInfo.ArgumentList.Add(windowArgument); startInfo.ArgumentList.Add("-vn"); startInfo.ArgumentList.Add("-sn"); startInfo.ArgumentList.Add("-dn"); startInfo.ArgumentList.Add("-ac"); startInfo.ArgumentList.Add("1"); startInfo.ArgumentList.Add("-ar"); startInfo.ArgumentList.Add(rateArgument); startInfo.ArgumentList.Add("-f"); startInfo.ArgumentList.Add("f32le"); startInfo.ArgumentList.Add("-"); using var process = new Process { StartInfo = startInfo }; if (!process.Start()) { return null; } try { // Read stderr concurrently: it is redirected, so leaving it unread // would deadlock the moment FFmpeg filled the pipe. var errorTask = process.StandardError.ReadToEndAsync(cancellationToken); var bytes = await ReadWindowAsync(process.StandardOutput.BaseStream, cancellationToken) .ConfigureAwait(false); var error = await errorTask.ConfigureAwait(false); if (bytes.Length < AudioSignature.FrameSize * sizeof(float)) { // No audio stream, an unreadable file, or a runtime Jellyfin // knows but the container does not support seeking into. logger.LogDebug( "FFmpeg returned {Bytes} bytes of audio for {Path}: {Error}", bytes.Length, path, error); return null; } var samples = new float[bytes.Length / sizeof(float)]; for (var i = 0; i < samples.Length; i++) { samples[i] = BinaryPrimitives.ReadSingleLittleEndian(bytes.AsSpan(i * sizeof(float))); } return samples; } finally { if (!process.HasExited) { try { process.Kill(entireProcessTree: true); } catch (InvalidOperationException) { // Exited between the check and the kill. } } } } private static async Task ReadWindowAsync(Stream stream, CancellationToken cancellationToken) { var wanted = AudioSignature.WindowSamples * sizeof(float); var buffer = new byte[wanted]; var filled = 0; while (filled < wanted) { var read = await stream.ReadAsync(buffer.AsMemory(filled, wanted - filled), cancellationToken) .ConfigureAwait(false); if (read == 0) { break; } filled += read; } if (filled == wanted) { return buffer; } var truncated = new byte[filled]; Array.Copy(buffer, truncated, filled); return truncated; } }