feat(audio): signature reader, offset recovery, and version refusal

Closes the consumer halves of JR-044 and JR-045, which were blocked on
there being no reader at all. `AudioSignatureMatcher` implements the
specification's slide — ±600 frames, scoring the fraction of overlapping
frames whose peak band agrees — and returns the tier and offset.

JR-045: `TryParseFrames` refuses any prefix but `v1:`. A `v2:` signature
from a future producer describes a DSP chain this build does not
implement, so scoring it as v1 would be a confident wrong answer where
declining is a correct one — the item drops to the runtime tier, which is
the entire reason the prefix is separate from `schema_version`.

JR-044: a runtime under 120 s yields no match and therefore no offset,
read off the runtime rather than inferred from a missing string, because
the runtime is what both producers test. The boundary is asserted on one
file at 119.999 s and 120.000 s, so a null cannot be blamed on the decode.

The offset has two terms, which is easy to miss: the recovered slide, and
the difference between where the two windows are anchored, since both are
centred on their own file's midpoint. A release carrying 40 s of extra
head material recovers 20 s from each.

One parameter is not from the specification and is marked as such in the
code: an alignment must overlap by at least 64 frames before its score
counts, or the extreme offsets compare a handful of frames where a chance
agreement scores 1.0 and beats the true alignment.

TRACES: JR-044, JR-045 | SR-003
This commit is contained in:
2026-07-31 16:24:23 +02:00
parent 17be9bcc4e
commit c863fe85f5
3 changed files with 547 additions and 0 deletions
@@ -0,0 +1,270 @@
using System;
using System.IO;
using System.Text.Json;
using System.Threading;
using System.Threading.Tasks;
using Jellyfin.Plugin.JRay.Configuration;
using Jellyfin.Plugin.JRay.Services;
using Microsoft.Extensions.Logging.Abstractions;
using Xunit;
namespace Jellyfin.Plugin.JRay.Tests;
/// <summary>
/// JR-044 (media shorter than 120 s emits no signature and takes no sync offset)
/// and JR-045 (the `v1:` prefix is emitted *and honoured*).
///
/// Both requirements have a producer half and a consumer half. The producer
/// halves live with <see cref="AudioSignature"/> and were closed by
/// UT-038 … UT-044; these are the consumer halves, which needed a reader —
/// <see cref="AudioSignatureMatcher"/> — before they could be closed at all.
///
/// The distinction that matters throughout: **refusing is a correct answer, and
/// a silently wrong one is not.** An unreadable or out-of-version signature must
/// drop the item to the runtime tier, never score as if it were understood.
///
/// TRACES: UT-045, UT-046, UT-047, UT-048, UT-049, UT-050, UT-051, UT-052 | JR-044, JR-045
/// </summary>
public class AudioSignatureMatcherTests
{
private static readonly string FixtureDir =
Path.Combine(AppContext.BaseDirectory, "fixtures", "audio");
private static readonly Lazy<string> GoldenSignature = new(() =>
JsonDocument.Parse(File.ReadAllText(Path.Combine(FixtureDir, "jray_audio_v1_golden.json")))
.RootElement.GetProperty("signature").GetString()!);
private static readonly Lazy<byte[]> GoldenFrames = new(() =>
AudioSignatureMatcher.TryParseFrames(GoldenSignature.Value)!);
// A feature-length runtime, so the window-anchor term is exercised at a
// realistic scale rather than at the 120 s boundary.
private const double FeatureRuntime = 7200.0;
// UT-045 — JR-044, the producer boundary.
[Fact]
public async Task ShortMedia_YieldsNoSignature_AndExactly120sDoes()
{
// Decisive because it is the *same file* either side of the boundary:
// only the runtime differs, so a null cannot be blamed on the decode.
// The fixture is exactly 120.000 s, which is the boundary itself.
var fixture = Path.Combine(FixtureDir, "jray_audio_v1_tone.flac");
// Below the window: refused before the encoder is ever consulted, which
// is what the deliberately invalid path proves.
Assert.Null(await AudioSignatureService.ComputeWithEncoderAsync(
"/nonexistent/ffmpeg",
fixture,
AudioSignature.WindowSec - 0.001,
NullLogger.Instance,
CancellationToken.None).ConfigureAwait(true));
var ffmpeg = FindFfmpeg();
if (ffmpeg is null)
{
return;
}
Assert.Null(await AudioSignatureService.ComputeWithEncoderAsync(
ffmpeg,
fixture,
AudioSignature.WindowSec - 0.001,
NullLogger.Instance,
CancellationToken.None).ConfigureAwait(true));
// Exactly at the boundary the window fits, so a signature is emitted.
// Both producers must agree here or they diverge on precisely the short
// items most likely to be misidentified.
Assert.Equal(
GoldenSignature.Value,
await AudioSignatureService.ComputeWithEncoderAsync(
ffmpeg,
fixture,
AudioSignature.WindowSec,
NullLogger.Instance,
CancellationToken.None).ConfigureAwait(true));
}
// UT-046 — JR-044, the consumer half.
[Fact]
public void ShortMedia_TakesNoOffset_EvenWithTwoValidSignatures()
{
var signature = GoldenSignature.Value;
// Two identical, perfectly valid signatures — the strongest possible
// match — still yield nothing when either side is under the window. The
// rule is checked on the runtime, not inferred from a missing string,
// because the runtime is what both producers test.
Assert.Null(AudioSignatureMatcher.Compare(
signature, signature, AudioSignature.WindowSec - 0.001, FeatureRuntime));
Assert.Null(AudioSignatureMatcher.Compare(
signature, signature, FeatureRuntime, AudioSignature.WindowSec - 0.001));
// At exactly the boundary it matches, so the refusal above is the
// threshold and not a blanket refusal.
Assert.NotNull(AudioSignatureMatcher.Compare(
signature, signature, AudioSignature.WindowSec, AudioSignature.WindowSec));
}
// UT-047 — JR-045, the requirement's whole point.
[Fact]
public void UnknownVersionPrefix_IsRefused_NotParsed()
{
// A v2 signature from a future producer, whose payload is byte-identical
// to a valid v1 one. Parsing it as v1 would yield a confident, plausible,
// wrong score; refusing drops the item to the runtime tier, which is
// correct. This is the entire reason the prefix is separate from
// schema_version.
var payload = GoldenSignature.Value[AudioSignature.VersionPrefix.Length..];
Assert.Null(AudioSignatureMatcher.TryParseFrames("v2:" + payload));
Assert.Null(AudioSignatureMatcher.TryParseFrames("v10:" + payload));
Assert.Null(AudioSignatureMatcher.TryParseFrames(payload)); // no prefix at all
Assert.Null(AudioSignatureMatcher.TryParseFrames("V1:" + payload)); // case is not cosmetic
// And it must be refused by the matcher too, not merely by the parser —
// a v2 signature produces no match and therefore no offset.
Assert.Null(AudioSignatureMatcher.Compare(
GoldenSignature.Value, "v2:" + payload, FeatureRuntime, FeatureRuntime));
Assert.Null(AudioSignatureMatcher.Compare(
"v2:" + payload, GoldenSignature.Value, FeatureRuntime, FeatureRuntime));
}
// UT-048 — JR-045, the accepting side.
[Fact]
public void V1Signature_ParsesToExactlyTheProducedFrames()
{
// The reader is the inverse of the producer, checked against the golden
// vector rather than against the producer's own output, so the two are
// pinned to the fixture and not merely to each other.
var frames = AudioSignatureMatcher.TryParseFrames(GoldenSignature.Value);
Assert.NotNull(frames);
Assert.Equal(AudioSignature.ExpectedFrames, frames!.Length);
Assert.Equal(GoldenSignature.Value, AudioSignature.VersionPrefix + Convert.ToBase64String(frames));
}
// UT-049 — JR-045, structural refusal.
[Fact]
public void MalformedSignatures_AreRefused_WithoutThrowing()
{
Assert.Null(AudioSignatureMatcher.TryParseFrames(null));
Assert.Null(AudioSignatureMatcher.TryParseFrames(string.Empty));
Assert.Null(AudioSignatureMatcher.TryParseFrames("v1:")); // empty payload
Assert.Null(AudioSignatureMatcher.TryParseFrames("v1:not!base64!"));
// Bit 7 is reserved by the packing — a byte with it set is not a frame.
// The server refuses this on upload; the client must not accept what the
// server would have rejected.
Assert.Null(AudioSignatureMatcher.TryParseFrames(
AudioSignature.VersionPrefix + Convert.ToBase64String(new byte[] { 0x04, 0x80, 0x08 })));
}
// UT-050 — the aligned case.
[Fact]
public void IdenticalSignatures_ScorePerfectly_AtZeroOffset()
{
var match = AudioSignatureMatcher.Compare(
GoldenSignature.Value, GoldenSignature.Value, FeatureRuntime, FeatureRuntime);
Assert.NotNull(match);
Assert.Equal(1.0, match!.Value.Score);
Assert.Equal(0, match.Value.OffsetFrames);
Assert.Equal(0.0, match.Value.OffsetSec);
Assert.Equal(MatchTier.Audio, match.Value.Tier);
}
// UT-051 — the case the feature exists for.
[Fact]
public void AShiftedRelease_RecoversTheOffset_RatherThanFailingToMatch()
{
// A release trimmed differently from the one the manifest was built on:
// the same cut, sampled at a different point. Before offset recovery this
// failed the runtime tier outright; the recovered shift is what makes one
// stored manifest serve every trim.
const int Shift = 100;
const int Span = 1000;
const int RemoteStart = 144;
var source = GoldenFrames.Value;
var remote = Signature(source, RemoteStart, Span);
var local = Signature(source, RemoteStart - Shift, Span);
var match = AudioSignatureMatcher.Compare(local, remote, FeatureRuntime, FeatureRuntime);
Assert.NotNull(match);
Assert.Equal(1.0, match!.Value.Score);
Assert.Equal(Shift, match.Value.OffsetFrames);
Assert.Equal(MatchTier.Audio, match.Value.Tier);
// Equal runtimes, so the window-anchor term vanishes and the offset is
// purely the recovered slide.
Assert.Equal(Shift * AudioSignatureMatcher.FrameSeconds, match.Value.OffsetSec, 9);
}
// UT-052 — the rejecting case, and the anchor term.
[Fact]
public void UnrelatedContent_DoesNotMatch_AndRuntimeSkewShiftsTheOffset()
{
// Two independent band sequences agree about 1 frame in 32, far below the
// loose floor. A matcher that returned its argmax regardless would hand
// back a confident alignment for unrelated films.
Assert.Null(AudioSignatureMatcher.Compare(
GoldenSignature.Value, PseudoRandomSignature(1288, seed: 12345), FeatureRuntime, FeatureRuntime));
// Both windows are centred on their own file's midpoint, so when the
// runtimes differ the windows start at different absolute times and that
// difference is part of the offset. Without this term the offset would be
// wrong by half the runtime difference on every shifted release.
var skew = 40.0;
var match = AudioSignatureMatcher.Compare(
GoldenSignature.Value,
GoldenSignature.Value,
FeatureRuntime + skew,
FeatureRuntime);
Assert.NotNull(match);
Assert.Equal(0, match!.Value.OffsetFrames);
Assert.Equal(skew / 2.0, match.Value.OffsetSec, 9);
}
private static string Signature(byte[] source, int start, int count)
=> AudioSignature.VersionPrefix + Convert.ToBase64String(source, start, count);
private static string PseudoRandomSignature(int frames, int seed)
{
var bytes = new byte[frames];
var state = (uint)seed;
for (var i = 0; i < frames; i++)
{
// Deterministic LCG — a fixed sequence, so a failure here is
// reproducible rather than flaky.
state = (state * 1664525u) + 1013904223u;
bytes[i] = (byte)((((state >> 16) % AudioSignature.NumBands) << 2) | ((state >> 8) & 0x03));
}
return AudioSignature.VersionPrefix + Convert.ToBase64String(bytes);
}
private static string? FindFfmpeg()
{
var configured = Environment.GetEnvironmentVariable("JRAY_TEST_FFMPEG");
if (!string.IsNullOrEmpty(configured))
{
return File.Exists(configured) ? configured : null;
}
var name = OperatingSystem.IsWindows() ? "ffmpeg.exe" : "ffmpeg";
foreach (var dir in (Environment.GetEnvironmentVariable("PATH") ?? string.Empty)
.Split(Path.PathSeparator, StringSplitOptions.RemoveEmptyEntries))
{
var candidate = Path.Combine(dir, name);
if (File.Exists(candidate))
{
return candidate;
}
}
return null;
}
}
@@ -0,0 +1,70 @@
using System;
using Jellyfin.Plugin.JRay.Configuration;
namespace Jellyfin.Plugin.JRay.Services;
/// <summary>
/// The outcome of comparing a local audio signature against a remote one.
/// </summary>
/// <remarks>
/// Produced only when the two signatures actually align; a comparison that
/// reaches no tier yields no result at all rather than a zero-scored one, so a
/// caller cannot mistake "did not match" for "matched at the bottom".
/// </remarks>
public readonly struct AudioSignatureMatch : IEquatable<AudioSignatureMatch>
{
/// <summary>
/// Gets the fraction of overlapping frames whose peak band agreed, in [0, 1].
/// </summary>
public double Score { get; init; }
/// <summary>
/// Gets the best-scoring alignment, in frames, between the two analysis
/// windows.
/// </summary>
/// <remarks>
/// Positive means the local window lags the remote one. This is a
/// window-relative quantity and is <b>not</b> the offset to apply to
/// timings — see <see cref="OffsetSec"/>, which additionally accounts for
/// the two windows being anchored at different points in their files.
/// </remarks>
public int OffsetFrames { get; init; }
/// <summary>
/// Gets the seconds to add to every remote window to bring it into the local
/// file's timebase.
/// </summary>
/// <remarks>
/// This is the quantity JR-030 applies at store time, and the one
/// <see cref="ManifestConverter.ToTruthFile"/> takes.
/// </remarks>
public double OffsetSec { get; init; }
/// <summary>Gets the tier this score earns.</summary>
public MatchTier Tier { get; init; }
/// <summary>Compares two matches for equality.</summary>
/// <param name="left">Left operand.</param>
/// <param name="right">Right operand.</param>
/// <returns><c>true</c> when the two are equal.</returns>
public static bool operator ==(AudioSignatureMatch left, AudioSignatureMatch right) => left.Equals(right);
/// <summary>Compares two matches for inequality.</summary>
/// <param name="left">Left operand.</param>
/// <param name="right">Right operand.</param>
/// <returns><c>true</c> when the two differ.</returns>
public static bool operator !=(AudioSignatureMatch left, AudioSignatureMatch right) => !left.Equals(right);
/// <inheritdoc/>
public bool Equals(AudioSignatureMatch other)
=> Score.Equals(other.Score)
&& OffsetFrames == other.OffsetFrames
&& OffsetSec.Equals(other.OffsetSec)
&& Tier == other.Tier;
/// <inheritdoc/>
public override bool Equals(object? obj) => obj is AudioSignatureMatch other && Equals(other);
/// <inheritdoc/>
public override int GetHashCode() => HashCode.Combine(Score, OffsetFrames, OffsetSec, Tier);
}
@@ -0,0 +1,207 @@
using System;
using Jellyfin.Plugin.JRay.Configuration;
namespace Jellyfin.Plugin.JRay.Services;
/// <summary>
/// Reads v1 audio signatures and aligns two of them, recovering the time offset
/// between differently trimmed releases of one cut.
/// </summary>
/// <remarks>
/// The <b>consumer</b> half of the signature feature: <see cref="AudioSignature"/>
/// produces, this reads. Matching belongs to the plugin rather than the server
/// because the offset is applied client-side (JR-030) and manifests are never
/// rewritten — one stored manifest serves every trim of the same cut.
/// <para>
/// The construction is public server specification §3 "Matching and offset
/// recovery": slide one signature against the other over ±600 frames, score the
/// fraction of overlapping frames whose <b>peak band</b> agrees, and take the
/// argmax. The energy class is deliberately not scored — it is the coarser and
/// less re-encoding-stable of the two fields, and the specification's rule names
/// the peak bin alone.
/// </para>
/// </remarks>
// TRACES: JR-044, JR-045 | SR-003
public static class AudioSignatureMatcher
{
/// <summary>Widest alignment searched, in frames (±56 s).</summary>
/// <remarks>
/// Covers realistic trim differences. A release that differs in *speed*
/// (a PAL 4% speed-up) is not a constant offset and is correctly rejected by
/// the score threshold rather than mis-aligned by this search.
/// </remarks>
public const int MaxOffsetFrames = 600;
/// <summary>Score at or above which two signatures are the same cut.</summary>
public const double AudioThreshold = 0.85;
/// <summary>Score at or above which two signatures are possibly the same cut.</summary>
public const double LooseThreshold = 0.60;
/// <summary>
/// Fewest overlapping frames an alignment must have before its score counts.
/// </summary>
/// <remarks>
/// <b>Not from the specification</b> — a guard this implementation adds. The
/// slide is defined over overlapping frames, and without a floor the extreme
/// offsets compare a handful of frames, where a chance agreement scores 1.0
/// and beats the true alignment. 64 frames is ~6 s. It only ever excludes
/// alignments near the ±600 limit: two full-length signatures overlap by 688
/// frames even at the widest offset, so this never binds on the case the
/// feature exists for.
/// </remarks>
public const int MinOverlapFrames = 64;
/// <summary>Gets the duration one STFT frame advances, in seconds.</summary>
public static double FrameSeconds => (double)AudioSignature.HopSize / AudioSignature.SampleRate;
/// <summary>
/// Parses a signature string into its per-frame bytes, refusing anything that
/// is not a well-formed <c>v1:</c> signature.
/// </summary>
/// <remarks>
/// JR-045. An unknown prefix is <b>refused, not parsed</b>: a <c>v2:</c>
/// signature from a future producer describes a DSP chain this build does not
/// implement, so scoring it as v1 would silently produce a wrong answer where
/// declining produces a correct one — the item simply falls to the runtime
/// tier. That is the entire purpose of the prefix being separate from
/// <c>schema_version</c>.
/// <para>
/// Structure is checked too, matching what the server validates on upload:
/// every byte is a 5-bit band index and a 2-bit energy class, so bit 7 is
/// always clear.
/// </para>
/// </remarks>
/// <param name="signature">A signature string, or <c>null</c>.</param>
/// <returns>The frame bytes, or <c>null</c> if this is not a v1 signature.</returns>
public static byte[]? TryParseFrames(string? signature)
{
if (string.IsNullOrEmpty(signature)
|| !signature.StartsWith(AudioSignature.VersionPrefix, StringComparison.Ordinal))
{
return null;
}
var payload = signature[AudioSignature.VersionPrefix.Length..];
Span<byte> decoded = new byte[((payload.Length / 4) + 1) * 3];
if (!Convert.TryFromBase64String(payload, decoded, out var written) || written == 0)
{
return null;
}
var frames = decoded[..written].ToArray();
foreach (var frame in frames)
{
if ((frame & 0x80) != 0)
{
return null;
}
}
return frames;
}
/// <summary>
/// Aligns a local signature against a remote one and reports the tier and
/// offset it earns.
/// </summary>
/// <remarks>
/// Returns <c>null</c> — no match, and therefore <b>no offset</b> — when
/// either signature is absent or is not a v1 signature (JR-045), when either
/// item is shorter than the analysis window (JR-044), or when the best
/// alignment scores below <see cref="LooseThreshold"/>.
/// <para>
/// The short-media rule is checked on the runtimes rather than inferred from
/// a missing signature, because the two producers must apply the identical
/// rule and the runtime is what both of them test. An item under 120 s falls
/// back to the runtime tier, which is adequate: a sub-two-minute item is
/// rarely the ambiguous-providence case the signature exists to solve.
/// </para>
/// </remarks>
/// <param name="localSignature">Signature computed from the local file.</param>
/// <param name="remoteSignature">Signature carried by the fetched manifest.</param>
/// <param name="localRuntimeSec">Runtime of the local file, in seconds.</param>
/// <param name="remoteRuntimeSec">Runtime the manifest records, in seconds.</param>
/// <returns>The match, or <c>null</c> when the two do not align.</returns>
public static AudioSignatureMatch? Compare(
string? localSignature,
string? remoteSignature,
double localRuntimeSec,
double remoteRuntimeSec)
{
// JR-044 — below the window there is no signature to trust and no offset
// to apply, whatever the strings happen to contain.
if (localRuntimeSec < AudioSignature.WindowSec || remoteRuntimeSec < AudioSignature.WindowSec)
{
return null;
}
var local = TryParseFrames(localSignature);
var remote = TryParseFrames(remoteSignature);
if (local is null || remote is null)
{
return null;
}
var bestScore = -1.0;
var bestOffset = 0;
var found = false;
for (var d = -MaxOffsetFrames; d <= MaxOffsetFrames; d++)
{
// i indexes the remote signature; the local frame it is compared
// against is i + d, which must fall inside the local signature.
var first = Math.Max(0, -d);
var last = Math.Min(remote.Length, local.Length - d);
var overlap = last - first;
if (overlap < MinOverlapFrames)
{
continue;
}
var agreed = 0;
for (var i = first; i < last; i++)
{
if (((local[i + d] >> 2) & 0x1F) == ((remote[i] >> 2) & 0x1F))
{
agreed++;
}
}
var score = (double)agreed / overlap;
// Ties go to the alignment closest to zero: when a signature is
// degenerate enough that several offsets score alike, "not shifted"
// is the reading that does least damage, and the choice must be
// deterministic rather than an artefact of iteration order.
if (!found || score > bestScore || (score == bestScore && Math.Abs(d) < Math.Abs(bestOffset)))
{
bestScore = score;
bestOffset = d;
found = true;
}
}
if (!found || bestScore < LooseThreshold)
{
return null;
}
// Both windows are centred on their own file's midpoint, so an offset
// between the two windows is only part of the answer: the windows start
// at different absolute times whenever the runtimes differ, and that
// difference is the rest of it. Runtimes are >= WindowSec here, so
// neither start clamps at zero — the same condition under which
// AudioSignatureService does not clamp either.
var localStart = (localRuntimeSec / 2.0) - (AudioSignature.WindowSec / 2.0);
var remoteStart = (remoteRuntimeSec / 2.0) - (AudioSignature.WindowSec / 2.0);
return new AudioSignatureMatch
{
Score = bestScore,
OffsetFrames = bestOffset,
OffsetSec = (localStart - remoteStart) + (bestOffset * FrameSeconds),
Tier = bestScore >= AudioThreshold ? MatchTier.Audio : MatchTier.Loose,
};
}
}