diff --git a/Jellyfin.Plugin.JRay.Tests/AudioSignatureMatcherTests.cs b/Jellyfin.Plugin.JRay.Tests/AudioSignatureMatcherTests.cs new file mode 100644 index 0000000..919a7d9 --- /dev/null +++ b/Jellyfin.Plugin.JRay.Tests/AudioSignatureMatcherTests.cs @@ -0,0 +1,270 @@ +using System; +using System.IO; +using System.Text.Json; +using System.Threading; +using System.Threading.Tasks; +using Jellyfin.Plugin.JRay.Configuration; +using Jellyfin.Plugin.JRay.Services; +using Microsoft.Extensions.Logging.Abstractions; +using Xunit; + +namespace Jellyfin.Plugin.JRay.Tests; + +/// +/// JR-044 (media shorter than 120 s emits no signature and takes no sync offset) +/// and JR-045 (the `v1:` prefix is emitted *and honoured*). +/// +/// Both requirements have a producer half and a consumer half. The producer +/// halves live with and were closed by +/// UT-038 … UT-044; these are the consumer halves, which needed a reader — +/// — before they could be closed at all. +/// +/// The distinction that matters throughout: **refusing is a correct answer, and +/// a silently wrong one is not.** An unreadable or out-of-version signature must +/// drop the item to the runtime tier, never score as if it were understood. +/// +/// TRACES: UT-045, UT-046, UT-047, UT-048, UT-049, UT-050, UT-051, UT-052 | JR-044, JR-045 +/// +public class AudioSignatureMatcherTests +{ + private static readonly string FixtureDir = + Path.Combine(AppContext.BaseDirectory, "fixtures", "audio"); + + private static readonly Lazy GoldenSignature = new(() => + JsonDocument.Parse(File.ReadAllText(Path.Combine(FixtureDir, "jray_audio_v1_golden.json"))) + .RootElement.GetProperty("signature").GetString()!); + + private static readonly Lazy GoldenFrames = new(() => + AudioSignatureMatcher.TryParseFrames(GoldenSignature.Value)!); + + // A feature-length runtime, so the window-anchor term is exercised at a + // realistic scale rather than at the 120 s boundary. + private const double FeatureRuntime = 7200.0; + + // UT-045 — JR-044, the producer boundary. + [Fact] + public async Task ShortMedia_YieldsNoSignature_AndExactly120sDoes() + { + // Decisive because it is the *same file* either side of the boundary: + // only the runtime differs, so a null cannot be blamed on the decode. + // The fixture is exactly 120.000 s, which is the boundary itself. + var fixture = Path.Combine(FixtureDir, "jray_audio_v1_tone.flac"); + + // Below the window: refused before the encoder is ever consulted, which + // is what the deliberately invalid path proves. + Assert.Null(await AudioSignatureService.ComputeWithEncoderAsync( + "/nonexistent/ffmpeg", + fixture, + AudioSignature.WindowSec - 0.001, + NullLogger.Instance, + CancellationToken.None).ConfigureAwait(true)); + + var ffmpeg = FindFfmpeg(); + if (ffmpeg is null) + { + return; + } + + Assert.Null(await AudioSignatureService.ComputeWithEncoderAsync( + ffmpeg, + fixture, + AudioSignature.WindowSec - 0.001, + NullLogger.Instance, + CancellationToken.None).ConfigureAwait(true)); + + // Exactly at the boundary the window fits, so a signature is emitted. + // Both producers must agree here or they diverge on precisely the short + // items most likely to be misidentified. + Assert.Equal( + GoldenSignature.Value, + await AudioSignatureService.ComputeWithEncoderAsync( + ffmpeg, + fixture, + AudioSignature.WindowSec, + NullLogger.Instance, + CancellationToken.None).ConfigureAwait(true)); + } + + // UT-046 — JR-044, the consumer half. + [Fact] + public void ShortMedia_TakesNoOffset_EvenWithTwoValidSignatures() + { + var signature = GoldenSignature.Value; + + // Two identical, perfectly valid signatures — the strongest possible + // match — still yield nothing when either side is under the window. The + // rule is checked on the runtime, not inferred from a missing string, + // because the runtime is what both producers test. + Assert.Null(AudioSignatureMatcher.Compare( + signature, signature, AudioSignature.WindowSec - 0.001, FeatureRuntime)); + Assert.Null(AudioSignatureMatcher.Compare( + signature, signature, FeatureRuntime, AudioSignature.WindowSec - 0.001)); + + // At exactly the boundary it matches, so the refusal above is the + // threshold and not a blanket refusal. + Assert.NotNull(AudioSignatureMatcher.Compare( + signature, signature, AudioSignature.WindowSec, AudioSignature.WindowSec)); + } + + // UT-047 — JR-045, the requirement's whole point. + [Fact] + public void UnknownVersionPrefix_IsRefused_NotParsed() + { + // A v2 signature from a future producer, whose payload is byte-identical + // to a valid v1 one. Parsing it as v1 would yield a confident, plausible, + // wrong score; refusing drops the item to the runtime tier, which is + // correct. This is the entire reason the prefix is separate from + // schema_version. + var payload = GoldenSignature.Value[AudioSignature.VersionPrefix.Length..]; + + Assert.Null(AudioSignatureMatcher.TryParseFrames("v2:" + payload)); + Assert.Null(AudioSignatureMatcher.TryParseFrames("v10:" + payload)); + Assert.Null(AudioSignatureMatcher.TryParseFrames(payload)); // no prefix at all + Assert.Null(AudioSignatureMatcher.TryParseFrames("V1:" + payload)); // case is not cosmetic + + // And it must be refused by the matcher too, not merely by the parser — + // a v2 signature produces no match and therefore no offset. + Assert.Null(AudioSignatureMatcher.Compare( + GoldenSignature.Value, "v2:" + payload, FeatureRuntime, FeatureRuntime)); + Assert.Null(AudioSignatureMatcher.Compare( + "v2:" + payload, GoldenSignature.Value, FeatureRuntime, FeatureRuntime)); + } + + // UT-048 — JR-045, the accepting side. + [Fact] + public void V1Signature_ParsesToExactlyTheProducedFrames() + { + // The reader is the inverse of the producer, checked against the golden + // vector rather than against the producer's own output, so the two are + // pinned to the fixture and not merely to each other. + var frames = AudioSignatureMatcher.TryParseFrames(GoldenSignature.Value); + + Assert.NotNull(frames); + Assert.Equal(AudioSignature.ExpectedFrames, frames!.Length); + Assert.Equal(GoldenSignature.Value, AudioSignature.VersionPrefix + Convert.ToBase64String(frames)); + } + + // UT-049 — JR-045, structural refusal. + [Fact] + public void MalformedSignatures_AreRefused_WithoutThrowing() + { + Assert.Null(AudioSignatureMatcher.TryParseFrames(null)); + Assert.Null(AudioSignatureMatcher.TryParseFrames(string.Empty)); + Assert.Null(AudioSignatureMatcher.TryParseFrames("v1:")); // empty payload + Assert.Null(AudioSignatureMatcher.TryParseFrames("v1:not!base64!")); + + // Bit 7 is reserved by the packing — a byte with it set is not a frame. + // The server refuses this on upload; the client must not accept what the + // server would have rejected. + Assert.Null(AudioSignatureMatcher.TryParseFrames( + AudioSignature.VersionPrefix + Convert.ToBase64String(new byte[] { 0x04, 0x80, 0x08 }))); + } + + // UT-050 — the aligned case. + [Fact] + public void IdenticalSignatures_ScorePerfectly_AtZeroOffset() + { + var match = AudioSignatureMatcher.Compare( + GoldenSignature.Value, GoldenSignature.Value, FeatureRuntime, FeatureRuntime); + + Assert.NotNull(match); + Assert.Equal(1.0, match!.Value.Score); + Assert.Equal(0, match.Value.OffsetFrames); + Assert.Equal(0.0, match.Value.OffsetSec); + Assert.Equal(MatchTier.Audio, match.Value.Tier); + } + + // UT-051 — the case the feature exists for. + [Fact] + public void AShiftedRelease_RecoversTheOffset_RatherThanFailingToMatch() + { + // A release trimmed differently from the one the manifest was built on: + // the same cut, sampled at a different point. Before offset recovery this + // failed the runtime tier outright; the recovered shift is what makes one + // stored manifest serve every trim. + const int Shift = 100; + const int Span = 1000; + const int RemoteStart = 144; + + var source = GoldenFrames.Value; + var remote = Signature(source, RemoteStart, Span); + var local = Signature(source, RemoteStart - Shift, Span); + + var match = AudioSignatureMatcher.Compare(local, remote, FeatureRuntime, FeatureRuntime); + + Assert.NotNull(match); + Assert.Equal(1.0, match!.Value.Score); + Assert.Equal(Shift, match.Value.OffsetFrames); + Assert.Equal(MatchTier.Audio, match.Value.Tier); + + // Equal runtimes, so the window-anchor term vanishes and the offset is + // purely the recovered slide. + Assert.Equal(Shift * AudioSignatureMatcher.FrameSeconds, match.Value.OffsetSec, 9); + } + + // UT-052 — the rejecting case, and the anchor term. + [Fact] + public void UnrelatedContent_DoesNotMatch_AndRuntimeSkewShiftsTheOffset() + { + // Two independent band sequences agree about 1 frame in 32, far below the + // loose floor. A matcher that returned its argmax regardless would hand + // back a confident alignment for unrelated films. + Assert.Null(AudioSignatureMatcher.Compare( + GoldenSignature.Value, PseudoRandomSignature(1288, seed: 12345), FeatureRuntime, FeatureRuntime)); + + // Both windows are centred on their own file's midpoint, so when the + // runtimes differ the windows start at different absolute times and that + // difference is part of the offset. Without this term the offset would be + // wrong by half the runtime difference on every shifted release. + var skew = 40.0; + var match = AudioSignatureMatcher.Compare( + GoldenSignature.Value, + GoldenSignature.Value, + FeatureRuntime + skew, + FeatureRuntime); + + Assert.NotNull(match); + Assert.Equal(0, match!.Value.OffsetFrames); + Assert.Equal(skew / 2.0, match.Value.OffsetSec, 9); + } + + private static string Signature(byte[] source, int start, int count) + => AudioSignature.VersionPrefix + Convert.ToBase64String(source, start, count); + + private static string PseudoRandomSignature(int frames, int seed) + { + var bytes = new byte[frames]; + var state = (uint)seed; + for (var i = 0; i < frames; i++) + { + // Deterministic LCG — a fixed sequence, so a failure here is + // reproducible rather than flaky. + state = (state * 1664525u) + 1013904223u; + bytes[i] = (byte)((((state >> 16) % AudioSignature.NumBands) << 2) | ((state >> 8) & 0x03)); + } + + return AudioSignature.VersionPrefix + Convert.ToBase64String(bytes); + } + + private static string? FindFfmpeg() + { + var configured = Environment.GetEnvironmentVariable("JRAY_TEST_FFMPEG"); + if (!string.IsNullOrEmpty(configured)) + { + return File.Exists(configured) ? configured : null; + } + + var name = OperatingSystem.IsWindows() ? "ffmpeg.exe" : "ffmpeg"; + foreach (var dir in (Environment.GetEnvironmentVariable("PATH") ?? string.Empty) + .Split(Path.PathSeparator, StringSplitOptions.RemoveEmptyEntries)) + { + var candidate = Path.Combine(dir, name); + if (File.Exists(candidate)) + { + return candidate; + } + } + + return null; + } +} diff --git a/Jellyfin.Plugin.JRay/Services/AudioSignatureMatch.cs b/Jellyfin.Plugin.JRay/Services/AudioSignatureMatch.cs new file mode 100644 index 0000000..400725e --- /dev/null +++ b/Jellyfin.Plugin.JRay/Services/AudioSignatureMatch.cs @@ -0,0 +1,70 @@ +using System; +using Jellyfin.Plugin.JRay.Configuration; + +namespace Jellyfin.Plugin.JRay.Services; + +/// +/// The outcome of comparing a local audio signature against a remote one. +/// +/// +/// Produced only when the two signatures actually align; a comparison that +/// reaches no tier yields no result at all rather than a zero-scored one, so a +/// caller cannot mistake "did not match" for "matched at the bottom". +/// +public readonly struct AudioSignatureMatch : IEquatable +{ + /// + /// Gets the fraction of overlapping frames whose peak band agreed, in [0, 1]. + /// + public double Score { get; init; } + + /// + /// Gets the best-scoring alignment, in frames, between the two analysis + /// windows. + /// + /// + /// Positive means the local window lags the remote one. This is a + /// window-relative quantity and is not the offset to apply to + /// timings — see , which additionally accounts for + /// the two windows being anchored at different points in their files. + /// + public int OffsetFrames { get; init; } + + /// + /// Gets the seconds to add to every remote window to bring it into the local + /// file's timebase. + /// + /// + /// This is the quantity JR-030 applies at store time, and the one + /// takes. + /// + public double OffsetSec { get; init; } + + /// Gets the tier this score earns. + public MatchTier Tier { get; init; } + + /// Compares two matches for equality. + /// Left operand. + /// Right operand. + /// true when the two are equal. + public static bool operator ==(AudioSignatureMatch left, AudioSignatureMatch right) => left.Equals(right); + + /// Compares two matches for inequality. + /// Left operand. + /// Right operand. + /// true when the two differ. + public static bool operator !=(AudioSignatureMatch left, AudioSignatureMatch right) => !left.Equals(right); + + /// + public bool Equals(AudioSignatureMatch other) + => Score.Equals(other.Score) + && OffsetFrames == other.OffsetFrames + && OffsetSec.Equals(other.OffsetSec) + && Tier == other.Tier; + + /// + public override bool Equals(object? obj) => obj is AudioSignatureMatch other && Equals(other); + + /// + public override int GetHashCode() => HashCode.Combine(Score, OffsetFrames, OffsetSec, Tier); +} diff --git a/Jellyfin.Plugin.JRay/Services/AudioSignatureMatcher.cs b/Jellyfin.Plugin.JRay/Services/AudioSignatureMatcher.cs new file mode 100644 index 0000000..34d37fb --- /dev/null +++ b/Jellyfin.Plugin.JRay/Services/AudioSignatureMatcher.cs @@ -0,0 +1,207 @@ +using System; +using Jellyfin.Plugin.JRay.Configuration; + +namespace Jellyfin.Plugin.JRay.Services; + +/// +/// Reads v1 audio signatures and aligns two of them, recovering the time offset +/// between differently trimmed releases of one cut. +/// +/// +/// The consumer half of the signature feature: +/// produces, this reads. Matching belongs to the plugin rather than the server +/// because the offset is applied client-side (JR-030) and manifests are never +/// rewritten — one stored manifest serves every trim of the same cut. +/// +/// The construction is public server specification §3 "Matching and offset +/// recovery": slide one signature against the other over ±600 frames, score the +/// fraction of overlapping frames whose peak band agrees, and take the +/// argmax. The energy class is deliberately not scored — it is the coarser and +/// less re-encoding-stable of the two fields, and the specification's rule names +/// the peak bin alone. +/// +/// +// TRACES: JR-044, JR-045 | SR-003 +public static class AudioSignatureMatcher +{ + /// Widest alignment searched, in frames (±56 s). + /// + /// Covers realistic trim differences. A release that differs in *speed* + /// (a PAL 4% speed-up) is not a constant offset and is correctly rejected by + /// the score threshold rather than mis-aligned by this search. + /// + public const int MaxOffsetFrames = 600; + + /// Score at or above which two signatures are the same cut. + public const double AudioThreshold = 0.85; + + /// Score at or above which two signatures are possibly the same cut. + public const double LooseThreshold = 0.60; + + /// + /// Fewest overlapping frames an alignment must have before its score counts. + /// + /// + /// Not from the specification — a guard this implementation adds. The + /// slide is defined over overlapping frames, and without a floor the extreme + /// offsets compare a handful of frames, where a chance agreement scores 1.0 + /// and beats the true alignment. 64 frames is ~6 s. It only ever excludes + /// alignments near the ±600 limit: two full-length signatures overlap by 688 + /// frames even at the widest offset, so this never binds on the case the + /// feature exists for. + /// + public const int MinOverlapFrames = 64; + + /// Gets the duration one STFT frame advances, in seconds. + public static double FrameSeconds => (double)AudioSignature.HopSize / AudioSignature.SampleRate; + + /// + /// Parses a signature string into its per-frame bytes, refusing anything that + /// is not a well-formed v1: signature. + /// + /// + /// JR-045. An unknown prefix is refused, not parsed: a v2: + /// signature from a future producer describes a DSP chain this build does not + /// implement, so scoring it as v1 would silently produce a wrong answer where + /// declining produces a correct one — the item simply falls to the runtime + /// tier. That is the entire purpose of the prefix being separate from + /// schema_version. + /// + /// Structure is checked too, matching what the server validates on upload: + /// every byte is a 5-bit band index and a 2-bit energy class, so bit 7 is + /// always clear. + /// + /// + /// A signature string, or null. + /// The frame bytes, or null if this is not a v1 signature. + public static byte[]? TryParseFrames(string? signature) + { + if (string.IsNullOrEmpty(signature) + || !signature.StartsWith(AudioSignature.VersionPrefix, StringComparison.Ordinal)) + { + return null; + } + + var payload = signature[AudioSignature.VersionPrefix.Length..]; + Span decoded = new byte[((payload.Length / 4) + 1) * 3]; + if (!Convert.TryFromBase64String(payload, decoded, out var written) || written == 0) + { + return null; + } + + var frames = decoded[..written].ToArray(); + foreach (var frame in frames) + { + if ((frame & 0x80) != 0) + { + return null; + } + } + + return frames; + } + + /// + /// Aligns a local signature against a remote one and reports the tier and + /// offset it earns. + /// + /// + /// Returns null — no match, and therefore no offset — when + /// either signature is absent or is not a v1 signature (JR-045), when either + /// item is shorter than the analysis window (JR-044), or when the best + /// alignment scores below . + /// + /// The short-media rule is checked on the runtimes rather than inferred from + /// a missing signature, because the two producers must apply the identical + /// rule and the runtime is what both of them test. An item under 120 s falls + /// back to the runtime tier, which is adequate: a sub-two-minute item is + /// rarely the ambiguous-providence case the signature exists to solve. + /// + /// + /// Signature computed from the local file. + /// Signature carried by the fetched manifest. + /// Runtime of the local file, in seconds. + /// Runtime the manifest records, in seconds. + /// The match, or null when the two do not align. + public static AudioSignatureMatch? Compare( + string? localSignature, + string? remoteSignature, + double localRuntimeSec, + double remoteRuntimeSec) + { + // JR-044 — below the window there is no signature to trust and no offset + // to apply, whatever the strings happen to contain. + if (localRuntimeSec < AudioSignature.WindowSec || remoteRuntimeSec < AudioSignature.WindowSec) + { + return null; + } + + var local = TryParseFrames(localSignature); + var remote = TryParseFrames(remoteSignature); + if (local is null || remote is null) + { + return null; + } + + var bestScore = -1.0; + var bestOffset = 0; + var found = false; + + for (var d = -MaxOffsetFrames; d <= MaxOffsetFrames; d++) + { + // i indexes the remote signature; the local frame it is compared + // against is i + d, which must fall inside the local signature. + var first = Math.Max(0, -d); + var last = Math.Min(remote.Length, local.Length - d); + var overlap = last - first; + if (overlap < MinOverlapFrames) + { + continue; + } + + var agreed = 0; + for (var i = first; i < last; i++) + { + if (((local[i + d] >> 2) & 0x1F) == ((remote[i] >> 2) & 0x1F)) + { + agreed++; + } + } + + var score = (double)agreed / overlap; + + // Ties go to the alignment closest to zero: when a signature is + // degenerate enough that several offsets score alike, "not shifted" + // is the reading that does least damage, and the choice must be + // deterministic rather than an artefact of iteration order. + if (!found || score > bestScore || (score == bestScore && Math.Abs(d) < Math.Abs(bestOffset))) + { + bestScore = score; + bestOffset = d; + found = true; + } + } + + if (!found || bestScore < LooseThreshold) + { + return null; + } + + // Both windows are centred on their own file's midpoint, so an offset + // between the two windows is only part of the answer: the windows start + // at different absolute times whenever the runtimes differ, and that + // difference is the rest of it. Runtimes are >= WindowSec here, so + // neither start clamps at zero — the same condition under which + // AudioSignatureService does not clamp either. + var localStart = (localRuntimeSec / 2.0) - (AudioSignature.WindowSec / 2.0); + var remoteStart = (remoteRuntimeSec / 2.0) - (AudioSignature.WindowSec / 2.0); + + return new AudioSignatureMatch + { + Score = bestScore, + OffsetFrames = bestOffset, + OffsetSec = (localStart - remoteStart) + (bestOffset * FrameSeconds), + Tier = bestScore >= AudioThreshold ? MatchTier.Audio : MatchTier.Loose, + }; + } +}