Compare commits
3
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
55b4633324 | ||
|
|
8c5fb5950d | ||
|
|
1e247c4c7d |
@@ -0,0 +1,214 @@
|
||||
using System;
|
||||
using System.IO;
|
||||
using System.Text.Json;
|
||||
using Jellyfin.Plugin.JRay.Configuration;
|
||||
using Jellyfin.Plugin.JRay.Models;
|
||||
using Jellyfin.Plugin.JRay.Services;
|
||||
using Xunit;
|
||||
|
||||
namespace Jellyfin.Plugin.JRay.Tests;
|
||||
|
||||
/// <summary>
|
||||
/// JR-047 — a fetched manifest is aligned against the local file before its
|
||||
/// windows are stored, and how that was decided is recorded.
|
||||
///
|
||||
/// The claim under test is that **the local alignment supersedes the server's
|
||||
/// offset**. The server has never seen this file; its offset can only be a
|
||||
/// runtime-difference inference, while a local alignment compares the manifest's
|
||||
/// own signature against the media the windows will be drawn over.
|
||||
///
|
||||
/// The counterweight is that a signature must never break a fetch. Every way the
|
||||
/// local path can fail — switched off, no manifest signature, short media, a
|
||||
/// failed decode, or two signatures that do not match — has to fall back to the
|
||||
/// server's offset rather than refusing. Both halves are asserted here.
|
||||
///
|
||||
/// `Resolve` is the decision, split out from the decode so it can be driven
|
||||
/// without FFmpeg or a media file.
|
||||
///
|
||||
/// TRACES: UT-053, UT-054, UT-055, UT-056, UT-057 | JR-047
|
||||
/// </summary>
|
||||
public class ManifestAlignerTests
|
||||
{
|
||||
private const double FeatureRuntime = 7200.0;
|
||||
private const double ServerOffset = 3.5;
|
||||
|
||||
private static readonly string FixtureDir =
|
||||
Path.Combine(AppContext.BaseDirectory, "fixtures", "audio");
|
||||
|
||||
private static readonly Lazy<string> GoldenSignature = new(() =>
|
||||
JsonDocument.Parse(File.ReadAllText(Path.Combine(FixtureDir, "jray_audio_v1_golden.json")))
|
||||
.RootElement.GetProperty("signature").GetString()!);
|
||||
|
||||
private static readonly Lazy<byte[]> GoldenFrames = new(() =>
|
||||
AudioSignatureMatcher.TryParseFrames(GoldenSignature.Value)!);
|
||||
|
||||
// UT-053
|
||||
[Fact]
|
||||
public void ALocalAlignment_SupersedesTheServersOffset()
|
||||
{
|
||||
// Same cut, sampled 100 frames apart — a differently trimmed release.
|
||||
// The server offered 3.5 s from a runtime comparison; the local audio
|
||||
// says otherwise, and the local answer is the one that gets applied.
|
||||
const int Shift = 100;
|
||||
var source = GoldenFrames.Value;
|
||||
|
||||
var alignment = ManifestAligner.Resolve(
|
||||
Signature(source, 44, 1000),
|
||||
Signature(source, 144, 1000),
|
||||
FeatureRuntime,
|
||||
FeatureRuntime,
|
||||
MatchTier.Runtime,
|
||||
ServerOffset);
|
||||
|
||||
Assert.Equal(AlignmentSource.Local, alignment.Source);
|
||||
Assert.Equal(MatchTier.Audio, alignment.Tier);
|
||||
Assert.Equal(Shift, alignment.OffsetFrames);
|
||||
Assert.Equal(1.0, alignment.Score!.Value);
|
||||
Assert.Equal(Shift * AudioSignatureMatcher.FrameSeconds, alignment.OffsetSec, 9);
|
||||
|
||||
// The server's claim is kept rather than overwritten: the applied offset
|
||||
// is otherwise unrecoverable once the windows are shifted, and the two
|
||||
// disagreeing is exactly what someone debugging would need to see.
|
||||
Assert.Equal(ServerOffset, alignment.ServerOffsetSec);
|
||||
Assert.Equal(MatchTier.Runtime, alignment.ServerTier);
|
||||
}
|
||||
|
||||
// UT-054
|
||||
[Fact]
|
||||
public void TheLocalSignature_IsRecorded_SoALaterFetchNeedNotDecodeAgain()
|
||||
{
|
||||
// The decode is the expensive half and the reason signatures are opt-in.
|
||||
// Keeping the local one beside the truth file is what lets a second
|
||||
// manifest be aligned for free.
|
||||
var alignment = ManifestAligner.Resolve(
|
||||
GoldenSignature.Value,
|
||||
GoldenSignature.Value,
|
||||
FeatureRuntime,
|
||||
FeatureRuntime,
|
||||
MatchTier.Runtime,
|
||||
ServerOffset);
|
||||
|
||||
Assert.Equal(GoldenSignature.Value, alignment.LocalSignature);
|
||||
Assert.Equal(AlignmentSource.Local, alignment.Source);
|
||||
Assert.Equal(0.0, alignment.OffsetSec);
|
||||
}
|
||||
|
||||
// UT-055
|
||||
[Fact]
|
||||
public void EveryUnavailableLocalPath_FallsBackToTheServer_RatherThanRefusing()
|
||||
{
|
||||
// A signature is an enhancement to cut matching. A missing one costs a
|
||||
// tier; it must never be able to break a fetch, so each of these stores
|
||||
// the manifest on the server's terms.
|
||||
var cases = new (string? Local, string? Manifest, string Why)[]
|
||||
{
|
||||
(null, GoldenSignature.Value, "signatures off, or the decode failed"),
|
||||
(GoldenSignature.Value, null, "the manifest carried no signature"),
|
||||
(null, null, "neither side has one"),
|
||||
};
|
||||
|
||||
foreach (var (local, manifest, why) in cases)
|
||||
{
|
||||
var alignment = ManifestAligner.Resolve(
|
||||
local, manifest, FeatureRuntime, FeatureRuntime, MatchTier.Runtime, ServerOffset);
|
||||
|
||||
Assert.Equal(AlignmentSource.Server, alignment.Source);
|
||||
Assert.Equal(ServerOffset, alignment.OffsetSec);
|
||||
Assert.Equal(MatchTier.Runtime, alignment.Tier);
|
||||
Assert.Null(alignment.Score);
|
||||
Assert.Null(alignment.OffsetFrames);
|
||||
}
|
||||
|
||||
// Short media is the same fallback, reached through JR-044 rather than
|
||||
// through a missing string: both sides have a perfectly valid signature
|
||||
// and it is still the runtime that decides.
|
||||
var shortMedia = ManifestAligner.Resolve(
|
||||
GoldenSignature.Value,
|
||||
GoldenSignature.Value,
|
||||
119.0,
|
||||
FeatureRuntime,
|
||||
MatchTier.Runtime,
|
||||
ServerOffset);
|
||||
|
||||
Assert.Equal(AlignmentSource.Server, shortMedia.Source);
|
||||
Assert.Equal(ServerOffset, shortMedia.OffsetSec);
|
||||
|
||||
// A `v2:` signature from a future producer is *un-comparable*, not a
|
||||
// mismatch (JR-045). Reporting it as one would tell the user their audio
|
||||
// disagrees with the manifest when all that happened is the producer
|
||||
// moved ahead of this build.
|
||||
var futureProducer = ManifestAligner.Resolve(
|
||||
GoldenSignature.Value,
|
||||
"v2:" + GoldenSignature.Value[AudioSignature.VersionPrefix.Length..],
|
||||
FeatureRuntime,
|
||||
FeatureRuntime,
|
||||
MatchTier.Runtime,
|
||||
ServerOffset);
|
||||
|
||||
Assert.Equal(AlignmentSource.Server, futureProducer.Source);
|
||||
Assert.Equal(ServerOffset, futureProducer.OffsetSec);
|
||||
}
|
||||
|
||||
// UT-056
|
||||
[Fact]
|
||||
public void TwoSignaturesThatDoNotMatch_AreRecorded_ButStillDoNotBreakTheFetch()
|
||||
{
|
||||
// The strongest available hint that a manifest describes different
|
||||
// content. It is not treated as a failure — the audio may legitimately
|
||||
// differ, a different language track being the obvious case — but it is
|
||||
// not discarded either.
|
||||
var alignment = ManifestAligner.Resolve(
|
||||
GoldenSignature.Value,
|
||||
PseudoRandomSignature(1288, seed: 4242),
|
||||
FeatureRuntime,
|
||||
FeatureRuntime,
|
||||
MatchTier.Runtime,
|
||||
ServerOffset);
|
||||
|
||||
Assert.Equal(AlignmentSource.LocalMismatch, alignment.Source);
|
||||
|
||||
// The manifest is still stored, on the server's terms.
|
||||
Assert.Equal(ServerOffset, alignment.OffsetSec);
|
||||
Assert.Equal(MatchTier.Runtime, alignment.Tier);
|
||||
}
|
||||
|
||||
// UT-057
|
||||
[Fact]
|
||||
public void AMismatchOutranksTheTier_InTheCaveatShownToTheUser()
|
||||
{
|
||||
// A `runtime`-tier match normally needs no caveat at all, so without
|
||||
// this the strongest warning available would be the one never shown.
|
||||
var mismatch = new TruthAlignment { Source = AlignmentSource.LocalMismatch };
|
||||
|
||||
var caveat = ManifestConverter.DescribeCaveat(MatchTier.Runtime, 0.0, mismatch);
|
||||
Assert.NotNull(caveat);
|
||||
Assert.Contains("does not match", caveat, StringComparison.Ordinal);
|
||||
|
||||
// It outranks `loose` too, which would otherwise have claimed the slot
|
||||
// with the weaker statement — that the runtimes differ, not the audio.
|
||||
var looseCaveat = ManifestConverter.DescribeCaveat(MatchTier.Loose, 0.0, mismatch);
|
||||
Assert.Contains("does not match", looseCaveat!, StringComparison.Ordinal);
|
||||
|
||||
// A clean local alignment keeps the existing behaviour: a shift is
|
||||
// explained, an aligned match needs nothing.
|
||||
var local = new TruthAlignment { Source = AlignmentSource.Local };
|
||||
Assert.Contains("shifted by", ManifestConverter.DescribeCaveat(MatchTier.Audio, 9.29, local)!, StringComparison.Ordinal);
|
||||
Assert.Null(ManifestConverter.DescribeCaveat(MatchTier.Audio, 0.0, local));
|
||||
}
|
||||
|
||||
private static string Signature(byte[] source, int start, int count)
|
||||
=> AudioSignature.VersionPrefix + Convert.ToBase64String(source, start, count);
|
||||
|
||||
private static string PseudoRandomSignature(int frames, int seed)
|
||||
{
|
||||
var bytes = new byte[frames];
|
||||
var state = (uint)seed;
|
||||
for (var i = 0; i < frames; i++)
|
||||
{
|
||||
state = (state * 1664525u) + 1013904223u;
|
||||
bytes[i] = (byte)((((state >> 16) % AudioSignature.NumBands) << 2) | ((state >> 8) & 0x03));
|
||||
}
|
||||
|
||||
return AudioSignature.VersionPrefix + Convert.ToBase64String(bytes);
|
||||
}
|
||||
}
|
||||
@@ -31,6 +31,7 @@ public class ManifestController : ControllerBase
|
||||
private readonly ILibraryManager _libraryManager;
|
||||
private readonly IManifestExchangeClient _exchange;
|
||||
private readonly IManagedTruthStore _truthStore;
|
||||
private readonly ManifestAligner _aligner;
|
||||
private readonly ILogger<ManifestController> _logger;
|
||||
|
||||
/// <summary>
|
||||
@@ -39,16 +40,19 @@ public class ManifestController : ControllerBase
|
||||
/// <param name="libraryManager">Library manager.</param>
|
||||
/// <param name="exchange">Manifest exchange client.</param>
|
||||
/// <param name="truthStore">Managed truth store.</param>
|
||||
/// <param name="aligner">Aligns a fetched manifest to the local file.</param>
|
||||
/// <param name="logger">Logger.</param>
|
||||
public ManifestController(
|
||||
ILibraryManager libraryManager,
|
||||
IManifestExchangeClient exchange,
|
||||
IManagedTruthStore truthStore,
|
||||
ManifestAligner aligner,
|
||||
ILogger<ManifestController> logger)
|
||||
{
|
||||
_libraryManager = libraryManager;
|
||||
_exchange = exchange;
|
||||
_truthStore = truthStore;
|
||||
_aligner = aligner;
|
||||
_logger = logger;
|
||||
}
|
||||
|
||||
@@ -102,35 +106,53 @@ public class ManifestController : ControllerBase
|
||||
return NotFound(new { error = "no configured server had an acceptable manifest" });
|
||||
}
|
||||
|
||||
// Align before storing. The server has never seen this file, so where a
|
||||
// local audio alignment is possible it supersedes the offset the server
|
||||
// sent — and it needs no round trip, so no signature leaves the
|
||||
// instance (JR-047).
|
||||
var alignment = await _aligner.AlignAsync(
|
||||
item.Path ?? string.Empty,
|
||||
item.RunTimeTicks is { } ticks ? TimeSpan.FromTicks(ticks).TotalSeconds : 0.0,
|
||||
outcome.Manifest,
|
||||
outcome.Tier,
|
||||
outcome.OffsetSec,
|
||||
config.ComputeAudioSignatures,
|
||||
cancellationToken)
|
||||
.ConfigureAwait(false);
|
||||
|
||||
var caveat = ManifestConverter.DescribeCaveat(alignment.Tier, alignment.OffsetSec, alignment);
|
||||
|
||||
// The offset is applied here, once, so the stored truth is always in the
|
||||
// local file's own timebase and no reader needs offset awareness.
|
||||
var truth = ManifestConverter.ToTruthFile(outcome.Manifest, outcome.OffsetSec, item.Path ?? string.Empty);
|
||||
var truth = ManifestConverter.ToTruthFile(outcome.Manifest, alignment.OffsetSec, item.Path ?? string.Empty);
|
||||
var provenance = new TruthProvenance
|
||||
{
|
||||
Source = TruthSource.Fetched,
|
||||
ServerUrl = outcome.ServerUrl,
|
||||
MatchTier = outcome.Tier,
|
||||
OffsetSec = outcome.OffsetSec,
|
||||
Caveat = ManifestConverter.DescribeCaveat(outcome.Tier, outcome.OffsetSec),
|
||||
MatchTier = alignment.Tier,
|
||||
OffsetSec = alignment.OffsetSec,
|
||||
Alignment = alignment,
|
||||
Caveat = caveat,
|
||||
RecordedAt = DateTime.UtcNow,
|
||||
};
|
||||
|
||||
await _truthStore.SaveAsync(itemId, truth, provenance, cancellationToken).ConfigureAwait(false);
|
||||
|
||||
_logger.LogInformation(
|
||||
"Stored manifest for {ItemId} from {Server} at tier {Tier} (offset {Offset}s)",
|
||||
"Stored manifest for {ItemId} from {Server} at tier {Tier} (offset {Offset}s, aligned by {Source})",
|
||||
itemId,
|
||||
outcome.ServerUrl,
|
||||
outcome.Tier,
|
||||
outcome.OffsetSec);
|
||||
alignment.Tier,
|
||||
alignment.OffsetSec,
|
||||
alignment.Source);
|
||||
|
||||
return Ok(new ManifestFetchResult
|
||||
{
|
||||
ServerUrl = outcome.ServerUrl,
|
||||
Match = outcome.Tier.ToString().ToLowerInvariant(),
|
||||
OffsetSec = outcome.OffsetSec,
|
||||
Match = alignment.Tier.ToString().ToLowerInvariant(),
|
||||
OffsetSec = alignment.OffsetSec,
|
||||
ActorCount = truth.Actors.Count,
|
||||
Caveat = ManifestConverter.DescribeCaveat(outcome.Tier, outcome.OffsetSec),
|
||||
Caveat = caveat,
|
||||
});
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,42 @@
|
||||
namespace Jellyfin.Plugin.JRay.Models;
|
||||
|
||||
/// <summary>
|
||||
/// Which comparison produced the offset that was applied to a fetched manifest.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// Recorded because the two are not equally strong evidence. The server has
|
||||
/// never seen the local file, so its offset is at best a runtime-difference
|
||||
/// inference; a local alignment compares the manifest's own audio signature
|
||||
/// against the file the windows will actually be drawn over.
|
||||
/// </remarks>
|
||||
public enum AlignmentSource
|
||||
{
|
||||
/// <summary>
|
||||
/// The server's offset was applied — no local alignment was possible.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// Either signatures are switched off, the item is under the 120 s window,
|
||||
/// the manifest carried no signature, or the decode failed. A signature is
|
||||
/// an enhancement, so every one of those degrades to this rather than
|
||||
/// failing the fetch.
|
||||
/// </remarks>
|
||||
Server = 0,
|
||||
|
||||
/// <summary>
|
||||
/// A local audio alignment was computed and its offset was applied.
|
||||
/// </summary>
|
||||
Local = 1,
|
||||
|
||||
/// <summary>
|
||||
/// A local alignment was attempted and the two signatures did not match at
|
||||
/// any tier; the server's offset was applied and the disagreement recorded.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// Deliberately not a failure. The audio may legitimately differ — a
|
||||
/// different language track, a heavy re-encode — and a signature must never
|
||||
/// be able to break a fetch. But it is the strongest available hint that a
|
||||
/// manifest describes different content, so it is surfaced as a caveat
|
||||
/// rather than discarded.
|
||||
/// </remarks>
|
||||
LocalMismatch = 2,
|
||||
}
|
||||
@@ -0,0 +1,74 @@
|
||||
using System.Text.Json.Serialization;
|
||||
using Jellyfin.Plugin.JRay.Configuration;
|
||||
|
||||
namespace Jellyfin.Plugin.JRay.Models;
|
||||
|
||||
/// <summary>
|
||||
/// How a fetched manifest was aligned to the local file, recorded alongside the
|
||||
/// truth data.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// The offset is applied once, at store time (JR-030), after which the stored
|
||||
/// windows look native and nothing would say they had been shifted. This block
|
||||
/// is what makes that reconstructable: which comparison produced the offset, how
|
||||
/// strong it was, and the local file's own signature, so a later fetch can
|
||||
/// re-align without decoding the media again.
|
||||
/// </remarks>
|
||||
// TRACES: JR-047 | SR-003
|
||||
public class TruthAlignment
|
||||
{
|
||||
/// <summary>Gets or sets which comparison produced the applied offset.</summary>
|
||||
[JsonPropertyName("source")]
|
||||
public AlignmentSource Source { get; set; }
|
||||
|
||||
/// <summary>Gets or sets the tier that was applied.</summary>
|
||||
[JsonPropertyName("tier")]
|
||||
public MatchTier Tier { get; set; }
|
||||
|
||||
/// <summary>Gets or sets the offset applied to every window, in seconds.</summary>
|
||||
[JsonPropertyName("offset_sec")]
|
||||
public double OffsetSec { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// Gets or sets the local alignment's score, or <c>null</c> when none was
|
||||
/// computed.
|
||||
/// </summary>
|
||||
[JsonPropertyName("score")]
|
||||
public double? Score { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// Gets or sets the recovered slide between the two analysis windows, in
|
||||
/// frames, or <c>null</c> when no local alignment was computed.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// Only part of <see cref="OffsetSec"/>: the rest comes from the two windows
|
||||
/// being anchored at different points when the runtimes differ.
|
||||
/// </remarks>
|
||||
[JsonPropertyName("offset_frames")]
|
||||
public int? OffsetFrames { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// Gets or sets the local file's own <c>v1:</c> audio signature, or
|
||||
/// <c>null</c> when none was computed.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// Kept so a later fetch can align against a new manifest without running
|
||||
/// FFmpeg over the media again — the expensive half of the operation, and
|
||||
/// the reason signatures are opt-in. It never leaves the instance: it is
|
||||
/// stored beside the truth file, and contribution strips provenance
|
||||
/// entirely (JR-034).
|
||||
/// </remarks>
|
||||
[JsonPropertyName("local_signature")]
|
||||
public string? LocalSignature { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// Gets or sets the offset the server claimed, retained even when a local
|
||||
/// alignment superseded it.
|
||||
/// </summary>
|
||||
[JsonPropertyName("server_offset_sec")]
|
||||
public double ServerOffsetSec { get; set; }
|
||||
|
||||
/// <summary>Gets or sets the tier the server claimed.</summary>
|
||||
[JsonPropertyName("server_tier")]
|
||||
public MatchTier ServerTier { get; set; }
|
||||
}
|
||||
@@ -74,6 +74,18 @@ public class TruthProvenance
|
||||
[JsonPropertyName("offset_sec")]
|
||||
public double OffsetSec { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// Gets or sets how the applied offset was arrived at, or null for local
|
||||
/// sources and for fetches made before alignment was recorded.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// <see cref="OffsetSec"/> says what was applied; this says why, and keeps
|
||||
/// the local file's own signature so a later fetch can re-align without
|
||||
/// decoding the media again (JR-047).
|
||||
/// </remarks>
|
||||
[JsonPropertyName("alignment")]
|
||||
public TruthAlignment? Alignment { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// Gets or sets a human-readable caveat to surface with the overlay, or
|
||||
/// null when the claim needs none.
|
||||
|
||||
@@ -23,9 +23,12 @@ public class ServiceRegistrator : IPluginServiceRegistrator
|
||||
// retried once per item (JR-037).
|
||||
serviceCollection.AddSingleton<IManifestExchangeClient, ManifestExchangeClient>();
|
||||
|
||||
// Registered concretely: the signature is computed, not yet consumed, so
|
||||
// there is no second implementation for an interface to abstract over
|
||||
// and nothing to gain from inventing one (JR-042).
|
||||
// Registered concretely: there is no second implementation for an
|
||||
// interface to abstract over and nothing to gain from inventing one
|
||||
// (JR-042). The aligner consumes the signature on the fetch path, so a
|
||||
// manifest is checked against the local file before its windows are
|
||||
// stored (JR-047).
|
||||
serviceCollection.AddSingleton<AudioSignatureService>();
|
||||
serviceCollection.AddSingleton<ManifestAligner>();
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,205 @@
|
||||
using System;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using Jellyfin.Plugin.JRay.Configuration;
|
||||
using Jellyfin.Plugin.JRay.Models;
|
||||
using Microsoft.Extensions.Logging;
|
||||
|
||||
namespace Jellyfin.Plugin.JRay.Services;
|
||||
|
||||
/// <summary>
|
||||
/// Aligns a freshly fetched manifest to the local file before its windows are
|
||||
/// stored.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// <b>Why the client re-derives an offset the server already sent.</b> The
|
||||
/// server has never seen the local file. Its offset is a claim about a runtime
|
||||
/// it was told, so it can only ever be a runtime-difference inference. A local
|
||||
/// alignment compares the manifest's own audio signature against the file the
|
||||
/// windows will actually be drawn over, which is the authoritative comparison —
|
||||
/// and it needs no round trip, so no signature leaves the instance. Where the
|
||||
/// two disagree, the local one wins.
|
||||
/// <para>
|
||||
/// This is what jRay's specification means by matching being "a consumer
|
||||
/// concern": the server never rewrites a manifest, so one stored manifest serves
|
||||
/// every trim of the same cut, and each client shifts it to its own timebase
|
||||
/// (JR-030).
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// <b>Degradation, not failure.</b> Signatures switched off, an item under the
|
||||
/// 120 s window, a manifest with no signature, a missing FFmpeg binary, a decode
|
||||
/// error, or two signatures that simply do not match — every one of these falls
|
||||
/// back to the server's offset. A signature is an enhancement to cut matching; a
|
||||
/// missing one costs a tier and must never be able to break a fetch.
|
||||
/// </para>
|
||||
/// </remarks>
|
||||
// TRACES: JR-047 | SR-003
|
||||
public class ManifestAligner
|
||||
{
|
||||
private readonly AudioSignatureService _signatures;
|
||||
private readonly ILogger<ManifestAligner> _logger;
|
||||
|
||||
/// <summary>
|
||||
/// Initializes a new instance of the <see cref="ManifestAligner"/> class.
|
||||
/// </summary>
|
||||
/// <param name="signatures">Computes the local file's audio signature.</param>
|
||||
/// <param name="logger">Logger.</param>
|
||||
public ManifestAligner(AudioSignatureService signatures, ILogger<ManifestAligner> logger)
|
||||
{
|
||||
_signatures = signatures;
|
||||
_logger = logger;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Decides which offset to apply, given a local signature that has already
|
||||
/// been computed.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// Split from <see cref="AlignAsync"/> so the decision is testable without
|
||||
/// an FFmpeg binary or a media file: everything interesting happens here,
|
||||
/// and the caller only supplies the two strings.
|
||||
/// </remarks>
|
||||
/// <param name="localSignature">The local file's signature, or <c>null</c>.</param>
|
||||
/// <param name="manifestSignature">The manifest's signature, or <c>null</c>.</param>
|
||||
/// <param name="localRuntimeSec">Local file runtime, in seconds.</param>
|
||||
/// <param name="manifestRuntimeSec">Runtime the manifest records, in seconds.</param>
|
||||
/// <param name="serverTier">The tier the server reported.</param>
|
||||
/// <param name="serverOffsetSec">The offset the server reported.</param>
|
||||
/// <returns>What to apply, and how it was decided.</returns>
|
||||
public static TruthAlignment Resolve(
|
||||
string? localSignature,
|
||||
string? manifestSignature,
|
||||
double localRuntimeSec,
|
||||
double manifestRuntimeSec,
|
||||
MatchTier serverTier,
|
||||
double serverOffsetSec)
|
||||
{
|
||||
var alignment = new TruthAlignment
|
||||
{
|
||||
Source = AlignmentSource.Server,
|
||||
Tier = serverTier,
|
||||
OffsetSec = serverOffsetSec,
|
||||
ServerTier = serverTier,
|
||||
ServerOffsetSec = serverOffsetSec,
|
||||
LocalSignature = localSignature,
|
||||
};
|
||||
|
||||
// "No alignment was possible" and "the audio does not match" are very
|
||||
// different things to tell someone, and `Compare` returns null for both.
|
||||
// A comparison is only possible when both items clear the 120 s window
|
||||
// (JR-044) and both signatures parse as v1 (JR-045) — a `v2:` signature
|
||||
// from a future producer is un-comparable, not a mismatch. Separating
|
||||
// them here is what keeps a 90-second extra from being reported as
|
||||
// content that disagrees with its own manifest.
|
||||
var comparable = localSignature is not null
|
||||
&& manifestSignature is not null
|
||||
&& localRuntimeSec >= AudioSignature.WindowSec
|
||||
&& manifestRuntimeSec >= AudioSignature.WindowSec
|
||||
&& AudioSignatureMatcher.TryParseFrames(localSignature) is not null
|
||||
&& AudioSignatureMatcher.TryParseFrames(manifestSignature) is not null;
|
||||
|
||||
if (!comparable)
|
||||
{
|
||||
return alignment;
|
||||
}
|
||||
|
||||
var match = AudioSignatureMatcher.Compare(
|
||||
localSignature, manifestSignature, localRuntimeSec, manifestRuntimeSec);
|
||||
|
||||
if (match is null)
|
||||
{
|
||||
// Both sides had a signature and they did not align at any tier. The
|
||||
// server's offset still applies — the audio may legitimately differ,
|
||||
// and a signature must not break a fetch — but this is the strongest
|
||||
// available hint that the manifest describes different content, so
|
||||
// it is recorded rather than silently discarded.
|
||||
alignment.Source = AlignmentSource.LocalMismatch;
|
||||
return alignment;
|
||||
}
|
||||
|
||||
alignment.Source = AlignmentSource.Local;
|
||||
alignment.Tier = match.Value.Tier;
|
||||
alignment.OffsetSec = match.Value.OffsetSec;
|
||||
alignment.Score = match.Value.Score;
|
||||
alignment.OffsetFrames = match.Value.OffsetFrames;
|
||||
return alignment;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Computes the local file's signature if it can, then resolves the
|
||||
/// alignment.
|
||||
/// </summary>
|
||||
/// <param name="mediaPath">Path of the local media file.</param>
|
||||
/// <param name="localRuntimeSec">Local file runtime, in seconds.</param>
|
||||
/// <param name="manifest">The fetched manifest.</param>
|
||||
/// <param name="serverTier">The tier the server reported.</param>
|
||||
/// <param name="serverOffsetSec">The offset the server reported.</param>
|
||||
/// <param name="computeSignatures">Whether signatures are enabled in configuration.</param>
|
||||
/// <param name="cancellationToken">Cancellation token.</param>
|
||||
/// <returns>What to apply, and how it was decided.</returns>
|
||||
public async Task<TruthAlignment> AlignAsync(
|
||||
string mediaPath,
|
||||
double localRuntimeSec,
|
||||
Jmanifest manifest,
|
||||
MatchTier serverTier,
|
||||
double serverOffsetSec,
|
||||
bool computeSignatures,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(manifest);
|
||||
|
||||
var manifestSignature = manifest.Cut?.AudioSignature;
|
||||
string? localSignature = null;
|
||||
|
||||
// The decode is the expensive half, so it is skipped outright when it
|
||||
// could not change the answer: no manifest signature to compare against
|
||||
// means no local alignment is possible.
|
||||
if (computeSignatures && manifestSignature is not null && !string.IsNullOrEmpty(mediaPath))
|
||||
{
|
||||
localSignature = await _signatures
|
||||
.ComputeAsync(mediaPath, localRuntimeSec, cancellationToken)
|
||||
.ConfigureAwait(false);
|
||||
}
|
||||
|
||||
var alignment = Resolve(
|
||||
localSignature,
|
||||
manifestSignature,
|
||||
localRuntimeSec,
|
||||
manifest.Cut?.RuntimeSec ?? 0.0,
|
||||
serverTier,
|
||||
serverOffsetSec);
|
||||
|
||||
switch (alignment.Source)
|
||||
{
|
||||
case AlignmentSource.Local:
|
||||
_logger.LogInformation(
|
||||
"Local audio alignment for {Path}: score {Score:F3}, {Frames} frames, offset {Offset:F3}s at tier {Tier} (server said {ServerOffset:F3}s at {ServerTier})",
|
||||
mediaPath,
|
||||
alignment.Score,
|
||||
alignment.OffsetFrames,
|
||||
alignment.OffsetSec,
|
||||
alignment.Tier,
|
||||
alignment.ServerOffsetSec,
|
||||
alignment.ServerTier);
|
||||
break;
|
||||
|
||||
case AlignmentSource.LocalMismatch:
|
||||
_logger.LogWarning(
|
||||
"Audio signatures for {Path} did not align with the fetched manifest; applying the server's {Offset:F3}s at {Tier}. This may be a different cut, a different language track, or a heavy re-encode.",
|
||||
mediaPath,
|
||||
alignment.OffsetSec,
|
||||
alignment.Tier);
|
||||
break;
|
||||
|
||||
default:
|
||||
_logger.LogDebug(
|
||||
"No local alignment for {Path}; applying the server's {Offset:F3}s at {Tier}",
|
||||
mediaPath,
|
||||
alignment.OffsetSec,
|
||||
alignment.Tier);
|
||||
break;
|
||||
}
|
||||
|
||||
return alignment;
|
||||
}
|
||||
}
|
||||
@@ -118,9 +118,19 @@ public static class ManifestConverter
|
||||
/// </remarks>
|
||||
/// <param name="tier">The tier achieved.</param>
|
||||
/// <param name="offsetSec">The offset applied.</param>
|
||||
/// <param name="alignment">How the offset was arrived at, when known.</param>
|
||||
/// <returns>A caveat string, or null when the match needs no explanation.</returns>
|
||||
public static string? DescribeCaveat(MatchTier tier, double offsetSec)
|
||||
public static string? DescribeCaveat(MatchTier tier, double offsetSec, TruthAlignment? alignment = null)
|
||||
{
|
||||
// Ranked before the tier, because it is the stronger statement: the
|
||||
// server's tier says the runtimes are compatible, while a signature
|
||||
// mismatch says the audio itself is not. The second outranks the first.
|
||||
if (alignment?.Source == AlignmentSource.LocalMismatch)
|
||||
{
|
||||
return "This file's audio does not match the fetched manifest — it may be a different cut, "
|
||||
+ "a different language track, or a heavy re-encode. Timings may be wrong.";
|
||||
}
|
||||
|
||||
if (tier == MatchTier.Loose)
|
||||
{
|
||||
return "Matched loosely — the runtime differs from this server's copy, so timings may drift.";
|
||||
|
||||
@@ -816,10 +816,39 @@ offsets compare a handful of frames, where a chance agreement scores 1.0 and
|
||||
beats the true alignment. It never binds on the real case — two full-length
|
||||
signatures still overlap by 688 frames at the widest offset.
|
||||
|
||||
**Gap:** the signature is computed and can now be read, but nothing yet *calls*
|
||||
either: `ComputeAudioSignatures` still gates nothing, no fetch path attaches a
|
||||
signature or consults the matcher, and no contribution carries one. That wiring
|
||||
belongs to the fetch and contribute paths (JR-031, JR-034), not here.
|
||||
JR-047 connects it. `ManifestAligner` runs on the fetch path, before anything is
|
||||
stored: it computes the local file's signature, compares it against the one the
|
||||
manifest carries, and **applies the local offset in preference to the server's**.
|
||||
The server has never seen this file — its offset is a runtime-difference
|
||||
inference at best — and the comparison needs no round trip, so no signature
|
||||
leaves the instance. `ComputeAudioSignatures` now gates that decode.
|
||||
|
||||
Everything that can go wrong degrades to the server's offset rather than
|
||||
refusing: signatures off, no manifest signature, media under the window, a `v2:`
|
||||
producer, a missing binary, a decode error. A signature is an enhancement to cut
|
||||
matching and must never be able to break a fetch. A *genuine* disagreement —
|
||||
both signatures valid, both items long enough, best alignment still under 0.60 —
|
||||
is recorded and surfaced as a caveat that outranks the tier's own, since "the
|
||||
audio does not match" is a stronger statement than "the runtimes differ"; the
|
||||
manifest is still stored, because the audio may legitimately differ.
|
||||
|
||||
"Un-comparable" and "does not match" are kept distinct. A 90-second extra is not
|
||||
content disagreeing with its manifest, and reporting it as such would be worse
|
||||
than saying nothing.
|
||||
|
||||
The applied offset, the score, the recovered slide, and the local file's own
|
||||
signature are written beside the truth file (`TruthAlignment`) — the offset is
|
||||
otherwise unrecoverable once the windows are shifted, and the stored signature
|
||||
lets a later fetch align without decoding again.
|
||||
|
||||
See [`docs/audio-alignment.md`](docs/audio-alignment.md) for the mechanism end to
|
||||
end.
|
||||
|
||||
**Gap:** contribution does not yet attach a signature (JR-034), and the stored
|
||||
`local_signature` is written but not read back, so today every fetch decodes.
|
||||
Server-side catalogue matching — `POST /manifests/search` and the `audio` tier —
|
||||
is the server's UR-009 and is deliberately sequenced after signature coverage
|
||||
accumulates; nothing here depends on it.
|
||||
|
||||
---
|
||||
|
||||
|
||||
@@ -0,0 +1,288 @@
|
||||
# Audio signature and alignment — how it works
|
||||
|
||||
How a manifest fetched from a public server is checked against, and shifted onto,
|
||||
your own copy of a film.
|
||||
|
||||
Requirements: `JR-042` … `JR-045`, `JR-047`. Construction is owned by
|
||||
[`JRay-public-server/SPEC.md` §3](../../JRay-public-server/SPEC.md); this document
|
||||
is the mechanism end to end.
|
||||
|
||||
---
|
||||
|
||||
## The problem
|
||||
|
||||
A manifest says "Peter Capaldi is on screen from 42:10 to 42:38". Your file may
|
||||
not agree, because releases of the same film are trimmed differently — a
|
||||
distributor logo here, a longer certificate card there. Thirty seconds of extra
|
||||
head material makes every window in the manifest wrong by thirty seconds.
|
||||
|
||||
The old defence was the runtime tier: accept a manifest only if the runtimes
|
||||
agree within ±2 s. That rejects exactly the releases it should be fixing, and
|
||||
accepts anything that happens to be the same length.
|
||||
|
||||
A **content-derived signature** answers the question the exchange actually needs
|
||||
— *do these timings apply to this media?* — and, when the answer is "yes but
|
||||
shifted", says by how much.
|
||||
|
||||
Deliberately, it does not answer *what file is this?* There is no file hash
|
||||
anywhere in the system; the tier was withdrawn on legal grounds. The signature
|
||||
identifies a **cut**, so two different encodes of the same edit agree.
|
||||
|
||||
---
|
||||
|
||||
## The signature
|
||||
|
||||
120 seconds from the **centre** of the media — the head and tail are the least
|
||||
content-specific parts of a release, being logos and credits.
|
||||
|
||||
```
|
||||
decode centre window runtime/2 ± 60 s
|
||||
downmix mono
|
||||
resample 11025 Hz
|
||||
STFT 4096-sample frame, 1024-sample hop (~93 ms), Hann
|
||||
band 300–3000 Hz, 32 logarithmically spaced bands
|
||||
per frame peak band index (5 bits) + energy class (2 bits)
|
||||
-> one byte, bit 7 always clear
|
||||
result 1288 bytes, base64, prefixed "v1:"
|
||||
```
|
||||
|
||||
Peak bins are used because they survive lossy re-encoding, loudness
|
||||
normalisation and channel-layout differences, where absolute magnitudes do not.
|
||||
Same principle as Chromaprint/AcoustID, but self-contained: no external service
|
||||
is queried, so no lookup leaks which titles an instance holds.
|
||||
|
||||
Bit 7 being always clear is not decoration. It is what makes an arbitrary byte
|
||||
string *not* a valid signature, which is what the server validates on upload, and
|
||||
what stops the field being usable as a payload channel. The length is fixed by
|
||||
the construction rather than merely bounded, for the same reason — a caller
|
||||
cannot choose it, so it cannot become a variable-size container.
|
||||
|
||||
### Why the parameters are written down twice
|
||||
|
||||
The specification's prose does not determine a byte stream. Two people
|
||||
implementing "32 log-spaced bands, take the peak" will disagree on at least:
|
||||
|
||||
- `float` or `double` — the fixture has frames whose two strongest bands are
|
||||
within 1.3% of each other, so `float` is not sufficient;
|
||||
- periodic Hann (`/N`) or symmetric (`/(N-1)`);
|
||||
- whether a band's value is the **mean** or the **sum** of its magnitudes (sum
|
||||
favours wide high bands over narrow low ones);
|
||||
- which way an `argmax` breaks ties;
|
||||
- whether the frame count is `1 + (n - 4096)/1024` or something that wobbles
|
||||
with the resampler tail.
|
||||
|
||||
Every one of those is pinned at the top of
|
||||
[`AudioSignature.cs`](../Jellyfin.Plugin.JRay/Services/AudioSignature.cs) and
|
||||
matched in the C++ producer. A signature that differs in any parameter simply
|
||||
does not match, which defeats the entire point of having one.
|
||||
|
||||
### Two implementations, proven equal
|
||||
|
||||
The extraction pipeline (C++, `IR-004`) computes this for files it processes
|
||||
locally. The plugin (C#, `JR-042`) computes it for the files the pipeline never
|
||||
sees. Both must agree exactly.
|
||||
|
||||
That is a *checked* claim, not an aspiration. `fixtures/audio/` holds three files
|
||||
byte-identical to the extraction repo's copies:
|
||||
|
||||
| File | What it pins |
|
||||
|---|---|
|
||||
| `jray_audio_v1_tone.flac` | 120 s of tones stepping through all 32 bands, amplitudes walking a golden-ratio sequence so all four energy classes appear |
|
||||
| `jray_audio_v1_golden.json` | The expected signature, the band→FFT-bin table, and checksums of the decoded PCM |
|
||||
| `make_fixture.py` | Regenerates the media from plain arithmetic — no numpy, ports to any language in ~20 lines |
|
||||
|
||||
The binding check regenerates the fixture PCM from `make_fixture.py`'s arithmetic
|
||||
and verifies it against the recorded `s16le`/`f32le` checksums **before** making
|
||||
any DSP claim. So it runs on a CI host with no codec at all, and a decode
|
||||
divergence stays distinguishable from a DSP one. The two tests that drive real
|
||||
FFmpeg self-skip without a binary — a check that skips is not a check, so it is
|
||||
never the only cover for a claim.
|
||||
|
||||
---
|
||||
|
||||
## Matching, and the offset
|
||||
|
||||
Two signatures are compared by sliding one against the other:
|
||||
|
||||
```
|
||||
for offset in -600 .. +600 frames: # ±56 s
|
||||
score(offset) = fraction of overlapping frames whose peak band matches
|
||||
best = argmax score
|
||||
```
|
||||
|
||||
| Score | Meaning |
|
||||
|---|---|
|
||||
| `≥ 0.85` | Same cut. `audio` tier. The offset applies |
|
||||
| `0.60 – 0.85` | Possibly the same cut, degraded audio. `loose` tier, surfaced as a caveat |
|
||||
| `< 0.60` | Different content. No match |
|
||||
|
||||
Only the **peak band** is scored. The energy class is the coarser and less
|
||||
re-encoding-stable of the two fields, and the specification's rule names the peak
|
||||
bin alone.
|
||||
|
||||
Speed-differing releases (a PAL 4% speed-up) are not a constant offset and are
|
||||
correctly rejected by the score threshold rather than mis-aligned.
|
||||
|
||||
### The offset has two terms
|
||||
|
||||
This is the part that is easy to get wrong. Both windows are centred on **their
|
||||
own file's** midpoint, so when the runtimes differ the two windows do not start
|
||||
at the same point in the content:
|
||||
|
||||
```
|
||||
offset = (local_window_start - manifest_window_start) + slide × 1024/11025
|
||||
└────────── anchor difference ──────────┘ └──── recovered ────┘
|
||||
```
|
||||
|
||||
A release carrying 40 s of extra head material recovers **20 s from each term**.
|
||||
Using the slide alone would be wrong by half the runtime difference on every
|
||||
shifted release. The specification's pseudocode describes only the slide, because
|
||||
it is written from the server's position, where both signatures are being
|
||||
compared against one stored manifest.
|
||||
|
||||
### One parameter that is not from the specification
|
||||
|
||||
An alignment must overlap by at least **64 frames** (~6 s) before its score
|
||||
counts. Without a floor the extreme offsets compare a handful of frames, where a
|
||||
chance agreement scores 1.0 and beats the true alignment. It never binds on the
|
||||
real case: two full-length signatures still overlap by 688 frames at the widest
|
||||
offset. It is marked as a local addition in the code.
|
||||
|
||||
---
|
||||
|
||||
## What happens when a manifest arrives
|
||||
|
||||
`ManifestController.FetchItem` → `ManifestAligner.AlignAsync`, before anything is
|
||||
stored:
|
||||
|
||||
```
|
||||
1. Server returns a manifest, a tier, and an offset.
|
||||
2. Does the manifest carry cut.audio_signature? no -> use the server's offset
|
||||
Are signatures enabled in configuration? no -> use the server's offset
|
||||
3. Decode this file's centre window, compute its signature.
|
||||
4. Compare the two.
|
||||
match -> apply the LOCAL offset and tier
|
||||
no match -> apply the server's offset, record the disagreement
|
||||
not comparable -> apply the server's offset
|
||||
5. Apply the offset to every window, once, at store time.
|
||||
6. Write the truth file, and the alignment beside it.
|
||||
```
|
||||
|
||||
### Why the local answer wins
|
||||
|
||||
**The server has never seen your file.** Its offset is a claim about a runtime it
|
||||
was told — at best a runtime-difference inference. A local alignment compares the
|
||||
manifest's own signature against the media the windows will actually be drawn
|
||||
over, which is the authoritative comparison. It also needs no round trip, so no
|
||||
signature ever leaves the instance.
|
||||
|
||||
This is what the specification means by matching being "a consumer concern": the
|
||||
server never rewrites a manifest, so **one stored manifest serves every trim of
|
||||
the same cut**, and each client shifts it onto its own timebase.
|
||||
|
||||
### Degradation, never failure
|
||||
|
||||
A signature is an enhancement to cut matching. A missing one costs a tier and
|
||||
**must never be able to break a fetch**. Every one of these stores the manifest
|
||||
on the server's terms:
|
||||
|
||||
- signatures switched off in configuration (they are opt-in — the decode costs a
|
||||
second or two of I/O per item);
|
||||
- the item is under 120 s, so the window underflows and there is no signature to
|
||||
compute (`JR-044`);
|
||||
- the manifest carried no signature;
|
||||
- the manifest's signature is `v2:` from a future producer — **refused, not
|
||||
parsed** (`JR-045`), because scoring an unknown DSP chain as v1 would be a
|
||||
confident wrong answer where declining is a correct one;
|
||||
- no FFmpeg binary, no audio stream, or a decode error.
|
||||
|
||||
### "Un-comparable" and "does not match" are different
|
||||
|
||||
The distinction matters more than it looks. A 90-second extra is not content that
|
||||
disagrees with its manifest — it is content that could not be compared. Reporting
|
||||
the first as the second would show a user a scary warning about the wrong thing.
|
||||
|
||||
A genuine mismatch — both signatures present, both valid, both items long enough,
|
||||
and the best alignment still below 0.60 — is the strongest available hint that a
|
||||
manifest describes different content. It is **still not a failure**: the audio may
|
||||
legitimately differ, a different language track being the obvious case. So the
|
||||
manifest is stored on the server's terms and the disagreement is surfaced as a
|
||||
caveat, which outranks the tier's own caveat because it is the stronger
|
||||
statement.
|
||||
|
||||
---
|
||||
|
||||
## What gets stored
|
||||
|
||||
The offset is applied **once**, at store time, so the stored windows are always
|
||||
in your file's own timebase and no read path needs offset awareness (`JR-030`).
|
||||
That makes the offset unrecoverable afterwards — the windows look native — which
|
||||
is why the alignment is recorded beside the truth file:
|
||||
|
||||
```json
|
||||
"alignment": {
|
||||
"source": "Local",
|
||||
"tier": "Audio",
|
||||
"offset_sec": 40.0,
|
||||
"score": 0.97,
|
||||
"offset_frames": 215,
|
||||
"local_signature": "v1:AAAAA…",
|
||||
"server_offset_sec": 0.0,
|
||||
"server_tier": "Runtime"
|
||||
}
|
||||
```
|
||||
|
||||
`local_signature` is kept so a **later fetch aligns for free** — the decode is
|
||||
the expensive half and the reason signatures are opt-in. It never leaves the
|
||||
instance: provenance is stored beside the truth file, not inside it, and
|
||||
contribution strips provenance entirely (`JR-034`).
|
||||
|
||||
The truth file itself is untouched by any of this. Injecting fields would mean
|
||||
the bytes served back are not the bytes the producer wrote, which is the property
|
||||
`JR-004` turns on.
|
||||
|
||||
---
|
||||
|
||||
## Where the code is
|
||||
|
||||
| Concern | File |
|
||||
|---|---|
|
||||
| The DSP | [`Services/AudioSignature.cs`](../Jellyfin.Plugin.JRay/Services/AudioSignature.cs) |
|
||||
| The decode | [`Services/AudioSignatureService.cs`](../Jellyfin.Plugin.JRay/Services/AudioSignatureService.cs) |
|
||||
| Reading and matching | [`Services/AudioSignatureMatcher.cs`](../Jellyfin.Plugin.JRay/Services/AudioSignatureMatcher.cs) |
|
||||
| The fetch-path decision | [`Services/ManifestAligner.cs`](../Jellyfin.Plugin.JRay/Services/ManifestAligner.cs) |
|
||||
| What is recorded | [`Models/TruthAlignment.cs`](../Jellyfin.Plugin.JRay/Models/TruthAlignment.cs) |
|
||||
| Server-side validation | `JRay-public-server/src/validate.rs`, `validate_audio_signature` |
|
||||
| C++ producer | `scene-actor-extraction/src/audio_signature.{hpp,cpp}` |
|
||||
|
||||
Tests: `AudioSignatureTests` (UT-038 … UT-044), `AudioSignatureMatcherTests`
|
||||
(UT-045 … UT-052), `ManifestAlignerTests` (UT-053 … UT-057).
|
||||
|
||||
## Verifying the cross-repo claim by hand
|
||||
|
||||
```sh
|
||||
# C# side
|
||||
cd jRay && dotnet test Jellyfin.Plugin.JRay.Tests/Jellyfin.Plugin.JRay.Tests.csproj
|
||||
|
||||
# C++ side, against the same fixture
|
||||
cd scene-actor-extraction/build && python3 -c "
|
||||
import json, sae_audio
|
||||
g = json.load(open('../tests/fixtures/audio/jray_audio_v1_golden.json'))
|
||||
print(sae_audio.compute_signature('../tests/fixtures/audio/jray_audio_v1_tone.flac') == g['signature'])
|
||||
"
|
||||
|
||||
# and that the fixtures really are the same bytes
|
||||
md5sum jRay/Jellyfin.Plugin.JRay.Tests/fixtures/audio/jray_audio_v1_golden.json \
|
||||
scene-actor-extraction/tests/fixtures/audio/jray_audio_v1_golden.json
|
||||
```
|
||||
|
||||
## Not built yet
|
||||
|
||||
- **Matching against a server's catalogue.** `POST /manifests/search` and the
|
||||
`audio` tier server-side are `UR-009`, still in progress. The sequencing is
|
||||
deliberate — accumulate signatures first, enable matching once coverage is
|
||||
useful. Nothing here depends on it: alignment works from the signature the
|
||||
manifest already carries.
|
||||
- **Contribution.** Uploads do not yet attach a signature (`JR-034`).
|
||||
- **Re-using the stored signature.** It is written but not yet read back on a
|
||||
second fetch, so today every fetch decodes.
|
||||
+29
-1
@@ -72,6 +72,11 @@ Tag code with `// TRACES: JR-012 | SR-002`.
|
||||
| UT-050 | Identical signatures score 1.0 at offset 0 and reach the `audio` tier | JR-044 | **Passing** |
|
||||
| UT-051 | **A shifted release recovers its offset** rather than failing to match — the case the feature exists for | JR-044 | **Passing** |
|
||||
| UT-052 | Unrelated content yields **no match at all**, and runtime skew contributes its window-anchor term to the offset | JR-044 | **Passing** |
|
||||
| UT-053 | **A local alignment supersedes the server's offset**, and the server's claim is retained rather than overwritten | JR-047 | **Passing** |
|
||||
| UT-054 | The local signature is recorded, so a later fetch aligns without decoding the media again | JR-047 | **Passing** |
|
||||
| UT-055 | Every unavailable local path — off, no manifest signature, neither, short media, **and a `v2:` producer** — falls back to the server rather than refusing | JR-047 | **Passing** |
|
||||
| UT-056 | Two signatures that genuinely disagree are recorded as a mismatch and **still do not break the fetch** | JR-047 | **Passing** |
|
||||
| UT-057 | A mismatch **outranks the tier** in the caveat shown to the user — a `runtime` match would otherwise show nothing at all | JR-047 | **Passing** |
|
||||
|
||||
All execute and pass. UT-043 and UT-044 are the two that need an FFmpeg binary,
|
||||
which the plugin gets from Jellyfin at run time and a bare CI container may not
|
||||
@@ -216,18 +221,40 @@ preserved.
|
||||
| JR-040 | The config page states plainly that **each configured server multiplies the exposure** | **PR-005** | Medium | Planned |
|
||||
| JR-041 | The plugin never fetches, stores, or transmits gallery data — reference faces or embeddings. It has no gallery code path at all | **SR-005** | High | Done |
|
||||
|
||||
## Audio signature (JR-042 … JR-045)
|
||||
## Audio signature (JR-042 … JR-045, JR-047)
|
||||
|
||||
Mirror-image of extraction `IR-004`/`IR-005`/`IR-007`/`IR-008`. Both producers
|
||||
must agree **bit-for-bit**, so each obligation is stated on both sides rather
|
||||
than assumed to be inherited.
|
||||
|
||||
JR-042 … JR-045 are the signature itself, produced and read. **JR-047 is what
|
||||
uses it**: without a consumer on the fetch path the other four are a fingerprint
|
||||
nothing ever fingerprints. See [`audio-alignment.md`](audio-alignment.md) for the
|
||||
end-to-end mechanism.
|
||||
|
||||
| ID | Requirement | Traces to | Priority | Status |
|
||||
|---|---|---|---|---|
|
||||
| JR-042 | Compute the signature **exactly** per server spec §3, using the FFmpeg binary Jellyfin already ships via `IMediaEncoder.EncoderPath` — no new dependency | SR-003 | Medium | **Done** (UT-039…044) — `AudioSignature` is the DSP, `AudioSignatureService` the decode; FFmpeg is invoked as a child process for decode, downmix and resample, and nothing was added to the project's dependencies |
|
||||
| JR-043 | Golden-vector fixture **shared with the extraction repo**, proving the two implementations are bit-exact | SR-003 | High | **Done** (UT-038, UT-039) — `fixtures/audio/` holds the extraction repo's three files byte-identically; the C# signature equals the recorded vector exactly |
|
||||
| JR-044 | Media shorter than 120 s: emit no signature and apply no sync offset — identical rule in both producers | SR-003 | Low | **Done** (UT-045, UT-046, UT-050…052) — the producer half was already in `AudioSignatureService`; the consumer half needed a reader, so `AudioSignatureMatcher` implements the §3 slide and declines an offset outright below the window. The boundary is asserted on one file at 119.999 s and 120.000 s |
|
||||
| JR-045 | Emit and honour the signature's own `v1:` prefix, so a DSP change is detectable rather than silently non-matching | SR-003 | Low | **Done** (UT-047…049) — `AudioSignatureMatcher.TryParseFrames` refuses any prefix but `v1:`, and refuses malformed or structurally invalid payloads, so a future producer's `v2:` drops the item to the runtime tier instead of scoring as if it were understood |
|
||||
| JR-047 | **A fetched manifest is aligned against the local file before its windows are stored**, and the alignment is recorded beside the truth data | SR-003 | High | **Done** (UT-053…057) — `ManifestAligner` runs on the fetch path. A local alignment supersedes the server's offset, since the server has never seen this file; every unavailable path degrades to the server's offset rather than refusing, and a genuine signature disagreement is recorded and surfaced as a caveat without failing the fetch |
|
||||
|
||||
> **Outstanding: §3's score gained ±1 frame of tolerance and this matcher has
|
||||
> not.** `AudioSignatureMatcher` implements the exact-frame rule §3 carried
|
||||
> until `JRay-public-server` UR-009 landed. The change was
|
||||
> `scene-actor-extraction` VR-014's measurement: over 40 correctly recovered
|
||||
> offsets on real film audio the exact rule scored 27 of them below 0.85 and
|
||||
> demoted them to `loose`, because the two windows are cut on their own file's
|
||||
> frame grid and those grids do not coincide. With ±1 frame all 40 reach
|
||||
> `audio` and the strongest false match is unmoved at 0.16.
|
||||
>
|
||||
> **Nothing is misaligned by the divergence** — the offset the matcher recovers
|
||||
> is unaffected, and JR-047 makes the local answer supersede the server's — but
|
||||
> the plugin will label as `loose` alignments the server calls `audio`, which is
|
||||
> a caveat shown to a user for a match that is not in doubt. JR-044's matcher
|
||||
> and [`audio-alignment.md`](audio-alignment.md)'s scoring table both need the
|
||||
> revised rule. Until then this repo implements a superseded version of §3.
|
||||
|
||||
## Human-in-the-loop association (JR-046)
|
||||
|
||||
@@ -355,6 +382,7 @@ framework reference and leans on `RollForward` to reach the 10.0 runtime.
|
||||
| JR-043 | **T1** | Signature matches the shared golden vector **bit-for-bit** | The fixture PCM is regenerated from `make_fixture.py` and checked against the recorded decode checksums first, so the check binds on a host with no codec and a decode divergence is distinguishable from a DSP one |
|
||||
| JR-044 | T1 | Media < 120 s yields no signature and no offset | Exactly 120 s — the boundary both repos must agree on |
|
||||
| JR-045 | T1 | `v1:` emitted; an unknown prefix is refused, not parsed | `v2:` signature from a future producer |
|
||||
| JR-047 | **T1** | A fetched manifest is aligned locally before storage, and the alignment is recorded | **Every way the local path can be unavailable must degrade to the server's offset, never refuse** — signatures off, no manifest signature, media under the window, a `v2:` producer. Distinguish those from a genuine mismatch: a 90 s extra is not content that disagrees with its manifest, and telling a user it is would be worse than saying nothing |
|
||||
| JR-046 | T2 + **T4** | *Assertions deferred* — recording an association and persisting it is T2; the review UI itself is T4 | Cannot be written until the truth-file interface for unidentified presence is settled (system open question 2) and AR-021/AR-022 land |
|
||||
|
||||
Three are worth singling out. **JR-021** and **JR-041** are static checks because
|
||||
|
||||
+51
-10
@@ -3,7 +3,7 @@
|
||||
<!-- GENERATED FILE - do not edit by hand. -->
|
||||
<!-- Regenerate: scripts/traceability/traceability-gate.sh -->
|
||||
|
||||
**Generated:** 2026-07-31T14:19:46+00:00
|
||||
**Generated:** 2026-07-31T15:05:02+00:00
|
||||
|
||||
Denominators are read from [`requirements.md`](requirements.md) at run time, never hardcoded. Coverage counts a requirement only when it is tagged in source **and** has a verification tier this repo's CI host can execute (`T1, T2, static`).
|
||||
|
||||
@@ -11,13 +11,13 @@ Denominators are read from [`requirements.md`](requirements.md) at run time, nev
|
||||
|
||||
| Metric | Value |
|
||||
|---|---|
|
||||
| Source files scanned | 74 |
|
||||
| TRACES tags found | 45 |
|
||||
| Source files scanned | 78 |
|
||||
| TRACES tags found | 48 |
|
||||
| EXCEPTION tags found | 0 |
|
||||
| Requirements defined | 46 |
|
||||
| Requirements covered | 37 |
|
||||
| **Coverage** | **80.4%** (37/46) |
|
||||
| Coverage of CI-executable scope | 84.1% (37/44) |
|
||||
| Requirements defined | 47 |
|
||||
| Requirements covered | 38 |
|
||||
| **Coverage** | **80.9%** (38/47) |
|
||||
| Coverage of CI-executable scope | 84.4% (38/45) |
|
||||
| Tagged but unexecuted in CI | 1 |
|
||||
| Orphan tags | 0 |
|
||||
|
||||
@@ -25,9 +25,9 @@ Denominators are read from [`requirements.md`](requirements.md) at run time, nev
|
||||
|
||||
| Type | Covered | Tagged but unexecuted | Defined |
|
||||
|---|---|---|---|
|
||||
| JR | 37 | 1 | 46 |
|
||||
| JR | 38 | 1 | 47 |
|
||||
|
||||
- **UT** tags present (separate taxonomy, not counted in coverage): UT-001, UT-002, UT-003, UT-004, UT-005, UT-007, UT-008, UT-009, UT-010, UT-011, UT-012, UT-013, UT-014, UT-015, UT-016, UT-017, UT-018, UT-019, UT-020, UT-021, UT-022, UT-023, UT-024, UT-025, UT-026, UT-027, UT-028, UT-029, UT-030, UT-031, UT-032, UT-033, UT-034, UT-035, UT-036, UT-037, UT-038, UT-039, UT-040, UT-041, UT-042, UT-043, UT-044, UT-045, UT-046, UT-047, UT-048, UT-049, UT-050, UT-051, UT-052
|
||||
- **UT** tags present (separate taxonomy, not counted in coverage): UT-001, UT-002, UT-003, UT-004, UT-005, UT-007, UT-008, UT-009, UT-010, UT-011, UT-012, UT-013, UT-014, UT-015, UT-016, UT-017, UT-018, UT-019, UT-020, UT-021, UT-022, UT-023, UT-024, UT-025, UT-026, UT-027, UT-028, UT-029, UT-030, UT-031, UT-032, UT-033, UT-034, UT-035, UT-036, UT-037, UT-038, UT-039, UT-040, UT-041, UT-042, UT-043, UT-044, UT-045, UT-046, UT-047, UT-048, UT-049, UT-050, UT-051, UT-052, UT-053, UT-054, UT-055, UT-056, UT-057
|
||||
- **PR** tags present (separate taxonomy, not counted in coverage): PR-001, PR-003, PR-004, PR-005, PR-006
|
||||
- **SR** tags present (separate taxonomy, not counted in coverage): SR-001, SR-002, SR-003, SR-004, SR-005
|
||||
|
||||
@@ -110,6 +110,7 @@ _None._
|
||||
| JR-044 | **Done** (UT-045, U… | T1 | SR-003 | covered | `Jellyfin.Plugin.JRay.Tests/AudioSignatureMatcherTests.cs`, `Jellyfin.Plugin.JRay/Services/AudioSignatureMatcher.cs` | Media shorter than 120 s: emit no signature and apply no sync offset … |
|
||||
| JR-045 | **Done** (UT-047…04… | T1 | SR-003 | covered | `Jellyfin.Plugin.JRay.Tests/AudioSignatureMatcherTests.cs`, `Jellyfin.Plugin.JRay/Services/AudioSignatureMatcher.cs` | Emit and honour the signature's own `v1:` prefix, so a DSP change is … |
|
||||
| JR-046 | **TBD** | T2, T4 | [system §4](../scripts/vendor/jray-proj… | untagged | - | Review UI for unidentified track clusters: show context crops, pick f… |
|
||||
| JR-047 | **Done** (UT-053…05… | T1 | SR-003 | covered | `Jellyfin.Plugin.JRay.Tests/ManifestAlignerTests.cs`, `Jellyfin.Plugin.JRay/Models/TruthAlignment.cs`, `Jellyfin.Plugin.JRay/Services/ManifestAligner.cs` | **A fetched manifest is aligned against the local file before its win… |
|
||||
|
||||
## Detailed mapping
|
||||
|
||||
@@ -384,6 +385,14 @@ _None._
|
||||
- [`Jellyfin.Plugin.JRay/Services/AudioSignatureMatcher.cs:24`](../Jellyfin.Plugin.JRay/Services/AudioSignatureMatcher.cs#L24) — `public static class AudioSignatureMatcher`
|
||||
- [`Jellyfin.Plugin.JRay.Tests/AudioSignatureMatcherTests.cs:26`](../Jellyfin.Plugin.JRay.Tests/AudioSignatureMatcherTests.cs#L26) — `public class AudioSignatureMatcherTests`
|
||||
|
||||
### JR-047
|
||||
|
||||
**Locations:** 3
|
||||
|
||||
- [`Jellyfin.Plugin.JRay/Models/TruthAlignment.cs:17`](../Jellyfin.Plugin.JRay/Models/TruthAlignment.cs#L17) — `public class TruthAlignment`
|
||||
- [`Jellyfin.Plugin.JRay/Services/ManifestAligner.cs:36`](../Jellyfin.Plugin.JRay/Services/ManifestAligner.cs#L36) — `public class ManifestAligner`
|
||||
- [`Jellyfin.Plugin.JRay.Tests/ManifestAlignerTests.cs:28`](../Jellyfin.Plugin.JRay.Tests/ManifestAlignerTests.cs#L28) — `public class ManifestAlignerTests`
|
||||
|
||||
### PR-001
|
||||
|
||||
**Locations:** 3
|
||||
@@ -449,10 +458,11 @@ _None._
|
||||
|
||||
### SR-003
|
||||
|
||||
**Locations:** 11
|
||||
**Locations:** 13
|
||||
|
||||
- [`Jellyfin.Plugin.JRay/Controllers/TruthController.cs:28`](../Jellyfin.Plugin.JRay/Controllers/TruthController.cs#L28) — `public class TruthController : ControllerBase`
|
||||
- [`Jellyfin.Plugin.JRay/Models/Jmanifest.cs:21`](../Jellyfin.Plugin.JRay/Models/Jmanifest.cs#L21) — `public class Jmanifest`
|
||||
- [`Jellyfin.Plugin.JRay/Models/TruthAlignment.cs:17`](../Jellyfin.Plugin.JRay/Models/TruthAlignment.cs#L17) — `public class TruthAlignment`
|
||||
- [`Jellyfin.Plugin.JRay/Models/TruthCut.cs:15`](../Jellyfin.Plugin.JRay/Models/TruthCut.cs#L15) — `public class TruthCut`
|
||||
- [`Jellyfin.Plugin.JRay/Models/TruthExtraction.cs:21`](../Jellyfin.Plugin.JRay/Models/TruthExtraction.cs#L21) — `public class TruthExtraction`
|
||||
- [`Jellyfin.Plugin.JRay/Models/TruthFile.cs:26`](../Jellyfin.Plugin.JRay/Models/TruthFile.cs#L26) — `public class TruthFile`
|
||||
@@ -460,6 +470,7 @@ _None._
|
||||
- [`Jellyfin.Plugin.JRay/Services/AudioSignature.cs:49`](../Jellyfin.Plugin.JRay/Services/AudioSignature.cs#L49) — `public static class AudioSignature`
|
||||
- [`Jellyfin.Plugin.JRay/Services/AudioSignatureMatcher.cs:24`](../Jellyfin.Plugin.JRay/Services/AudioSignatureMatcher.cs#L24) — `public static class AudioSignatureMatcher`
|
||||
- [`Jellyfin.Plugin.JRay/Services/AudioSignatureService.cs:39`](../Jellyfin.Plugin.JRay/Services/AudioSignatureService.cs#L39) — `public class AudioSignatureService`
|
||||
- [`Jellyfin.Plugin.JRay/Services/ManifestAligner.cs:36`](../Jellyfin.Plugin.JRay/Services/ManifestAligner.cs#L36) — `public class ManifestAligner`
|
||||
- [`Jellyfin.Plugin.JRay/Services/ManifestConverter.cs:11`](../Jellyfin.Plugin.JRay/Services/ManifestConverter.cs#L11) — `public static class ManifestConverter`
|
||||
- [`Jellyfin.Plugin.JRay/Services/TruthSchema.cs:35`](../Jellyfin.Plugin.JRay/Services/TruthSchema.cs#L35) — `public static class TruthSchema`
|
||||
|
||||
@@ -783,3 +794,33 @@ _None._
|
||||
|
||||
- [`Jellyfin.Plugin.JRay.Tests/AudioSignatureMatcherTests.cs:26`](../Jellyfin.Plugin.JRay.Tests/AudioSignatureMatcherTests.cs#L26) — `public class AudioSignatureMatcherTests`
|
||||
|
||||
### UT-053
|
||||
|
||||
**Locations:** 1
|
||||
|
||||
- [`Jellyfin.Plugin.JRay.Tests/ManifestAlignerTests.cs:28`](../Jellyfin.Plugin.JRay.Tests/ManifestAlignerTests.cs#L28) — `public class ManifestAlignerTests`
|
||||
|
||||
### UT-054
|
||||
|
||||
**Locations:** 1
|
||||
|
||||
- [`Jellyfin.Plugin.JRay.Tests/ManifestAlignerTests.cs:28`](../Jellyfin.Plugin.JRay.Tests/ManifestAlignerTests.cs#L28) — `public class ManifestAlignerTests`
|
||||
|
||||
### UT-055
|
||||
|
||||
**Locations:** 1
|
||||
|
||||
- [`Jellyfin.Plugin.JRay.Tests/ManifestAlignerTests.cs:28`](../Jellyfin.Plugin.JRay.Tests/ManifestAlignerTests.cs#L28) — `public class ManifestAlignerTests`
|
||||
|
||||
### UT-056
|
||||
|
||||
**Locations:** 1
|
||||
|
||||
- [`Jellyfin.Plugin.JRay.Tests/ManifestAlignerTests.cs:28`](../Jellyfin.Plugin.JRay.Tests/ManifestAlignerTests.cs#L28) — `public class ManifestAlignerTests`
|
||||
|
||||
### UT-057
|
||||
|
||||
**Locations:** 1
|
||||
|
||||
- [`Jellyfin.Plugin.JRay.Tests/ManifestAlignerTests.cs:28`](../Jellyfin.Plugin.JRay.Tests/ManifestAlignerTests.cs#L28) — `public class ManifestAlignerTests`
|
||||
|
||||
|
||||
Reference in New Issue
Block a user