feat(audio): align a fetched manifest to the local file before storing
The signature had a producer and a reader but no consumer, so nothing ever fingerprinted anything. `ManifestAligner` runs on the fetch path, before the windows are stored. A local alignment supersedes the server's offset. The server has never seen this file — its offset is a runtime-difference inference at best, while the local comparison is against the media the windows will actually be drawn over. It also needs no round trip, so no signature leaves the instance. This is what jRay's spec already meant by matching being a consumer concern: the server never rewrites a manifest, so one stored manifest serves every trim of the same cut. The offset has two terms and only one is in the server's pseudocode. Both windows are centred on their own file's midpoint, so unequal runtimes start them at different absolute times; a release with 40 s of extra head material recovers 20 s from the slide and 20 s from the anchor difference. Using the slide alone is wrong by half the runtime difference on every shifted release. Degradation, never failure. Signatures off, no manifest signature, media under the window, a `v2:` producer, a missing binary, a decode error — each applies the server's offset rather than refusing, because a signature is an enhancement to cut matching and must never break a fetch. "Un-comparable" and "does not match" are kept distinct, which a test caught: `Compare` returns null for both, and conflating them would report a 90-second extra as content disagreeing with its own manifest. A genuine disagreement is stored anyway — the audio may legitimately differ, a different language track being the obvious case — and surfaced as a caveat that outranks the tier's, since it is the stronger statement. The applied offset, score, slide and the local file's own signature are written beside the truth file: the offset is otherwise unrecoverable once the windows are shifted, and the stored signature lets a later fetch align without decoding again. Provenance is never injected into the truth file, so the bytes served back stay the producer's (JR-004). `docs/audio-alignment.md` documents the mechanism end to end. TRACES: JR-047 | SR-003
This commit is contained in:
@@ -31,6 +31,7 @@ public class ManifestController : ControllerBase
|
||||
private readonly ILibraryManager _libraryManager;
|
||||
private readonly IManifestExchangeClient _exchange;
|
||||
private readonly IManagedTruthStore _truthStore;
|
||||
private readonly ManifestAligner _aligner;
|
||||
private readonly ILogger<ManifestController> _logger;
|
||||
|
||||
/// <summary>
|
||||
@@ -39,16 +40,19 @@ public class ManifestController : ControllerBase
|
||||
/// <param name="libraryManager">Library manager.</param>
|
||||
/// <param name="exchange">Manifest exchange client.</param>
|
||||
/// <param name="truthStore">Managed truth store.</param>
|
||||
/// <param name="aligner">Aligns a fetched manifest to the local file.</param>
|
||||
/// <param name="logger">Logger.</param>
|
||||
public ManifestController(
|
||||
ILibraryManager libraryManager,
|
||||
IManifestExchangeClient exchange,
|
||||
IManagedTruthStore truthStore,
|
||||
ManifestAligner aligner,
|
||||
ILogger<ManifestController> logger)
|
||||
{
|
||||
_libraryManager = libraryManager;
|
||||
_exchange = exchange;
|
||||
_truthStore = truthStore;
|
||||
_aligner = aligner;
|
||||
_logger = logger;
|
||||
}
|
||||
|
||||
@@ -102,35 +106,53 @@ public class ManifestController : ControllerBase
|
||||
return NotFound(new { error = "no configured server had an acceptable manifest" });
|
||||
}
|
||||
|
||||
// Align before storing. The server has never seen this file, so where a
|
||||
// local audio alignment is possible it supersedes the offset the server
|
||||
// sent — and it needs no round trip, so no signature leaves the
|
||||
// instance (JR-047).
|
||||
var alignment = await _aligner.AlignAsync(
|
||||
item.Path ?? string.Empty,
|
||||
item.RunTimeTicks is { } ticks ? TimeSpan.FromTicks(ticks).TotalSeconds : 0.0,
|
||||
outcome.Manifest,
|
||||
outcome.Tier,
|
||||
outcome.OffsetSec,
|
||||
config.ComputeAudioSignatures,
|
||||
cancellationToken)
|
||||
.ConfigureAwait(false);
|
||||
|
||||
var caveat = ManifestConverter.DescribeCaveat(alignment.Tier, alignment.OffsetSec, alignment);
|
||||
|
||||
// The offset is applied here, once, so the stored truth is always in the
|
||||
// local file's own timebase and no reader needs offset awareness.
|
||||
var truth = ManifestConverter.ToTruthFile(outcome.Manifest, outcome.OffsetSec, item.Path ?? string.Empty);
|
||||
var truth = ManifestConverter.ToTruthFile(outcome.Manifest, alignment.OffsetSec, item.Path ?? string.Empty);
|
||||
var provenance = new TruthProvenance
|
||||
{
|
||||
Source = TruthSource.Fetched,
|
||||
ServerUrl = outcome.ServerUrl,
|
||||
MatchTier = outcome.Tier,
|
||||
OffsetSec = outcome.OffsetSec,
|
||||
Caveat = ManifestConverter.DescribeCaveat(outcome.Tier, outcome.OffsetSec),
|
||||
MatchTier = alignment.Tier,
|
||||
OffsetSec = alignment.OffsetSec,
|
||||
Alignment = alignment,
|
||||
Caveat = caveat,
|
||||
RecordedAt = DateTime.UtcNow,
|
||||
};
|
||||
|
||||
await _truthStore.SaveAsync(itemId, truth, provenance, cancellationToken).ConfigureAwait(false);
|
||||
|
||||
_logger.LogInformation(
|
||||
"Stored manifest for {ItemId} from {Server} at tier {Tier} (offset {Offset}s)",
|
||||
"Stored manifest for {ItemId} from {Server} at tier {Tier} (offset {Offset}s, aligned by {Source})",
|
||||
itemId,
|
||||
outcome.ServerUrl,
|
||||
outcome.Tier,
|
||||
outcome.OffsetSec);
|
||||
alignment.Tier,
|
||||
alignment.OffsetSec,
|
||||
alignment.Source);
|
||||
|
||||
return Ok(new ManifestFetchResult
|
||||
{
|
||||
ServerUrl = outcome.ServerUrl,
|
||||
Match = outcome.Tier.ToString().ToLowerInvariant(),
|
||||
OffsetSec = outcome.OffsetSec,
|
||||
Match = alignment.Tier.ToString().ToLowerInvariant(),
|
||||
OffsetSec = alignment.OffsetSec,
|
||||
ActorCount = truth.Actors.Count,
|
||||
Caveat = ManifestConverter.DescribeCaveat(outcome.Tier, outcome.OffsetSec),
|
||||
Caveat = caveat,
|
||||
});
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,42 @@
|
||||
namespace Jellyfin.Plugin.JRay.Models;
|
||||
|
||||
/// <summary>
|
||||
/// Which comparison produced the offset that was applied to a fetched manifest.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// Recorded because the two are not equally strong evidence. The server has
|
||||
/// never seen the local file, so its offset is at best a runtime-difference
|
||||
/// inference; a local alignment compares the manifest's own audio signature
|
||||
/// against the file the windows will actually be drawn over.
|
||||
/// </remarks>
|
||||
public enum AlignmentSource
|
||||
{
|
||||
/// <summary>
|
||||
/// The server's offset was applied — no local alignment was possible.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// Either signatures are switched off, the item is under the 120 s window,
|
||||
/// the manifest carried no signature, or the decode failed. A signature is
|
||||
/// an enhancement, so every one of those degrades to this rather than
|
||||
/// failing the fetch.
|
||||
/// </remarks>
|
||||
Server = 0,
|
||||
|
||||
/// <summary>
|
||||
/// A local audio alignment was computed and its offset was applied.
|
||||
/// </summary>
|
||||
Local = 1,
|
||||
|
||||
/// <summary>
|
||||
/// A local alignment was attempted and the two signatures did not match at
|
||||
/// any tier; the server's offset was applied and the disagreement recorded.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// Deliberately not a failure. The audio may legitimately differ — a
|
||||
/// different language track, a heavy re-encode — and a signature must never
|
||||
/// be able to break a fetch. But it is the strongest available hint that a
|
||||
/// manifest describes different content, so it is surfaced as a caveat
|
||||
/// rather than discarded.
|
||||
/// </remarks>
|
||||
LocalMismatch = 2,
|
||||
}
|
||||
@@ -0,0 +1,74 @@
|
||||
using System.Text.Json.Serialization;
|
||||
using Jellyfin.Plugin.JRay.Configuration;
|
||||
|
||||
namespace Jellyfin.Plugin.JRay.Models;
|
||||
|
||||
/// <summary>
|
||||
/// How a fetched manifest was aligned to the local file, recorded alongside the
|
||||
/// truth data.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// The offset is applied once, at store time (JR-030), after which the stored
|
||||
/// windows look native and nothing would say they had been shifted. This block
|
||||
/// is what makes that reconstructable: which comparison produced the offset, how
|
||||
/// strong it was, and the local file's own signature, so a later fetch can
|
||||
/// re-align without decoding the media again.
|
||||
/// </remarks>
|
||||
// TRACES: JR-047 | SR-003
|
||||
public class TruthAlignment
|
||||
{
|
||||
/// <summary>Gets or sets which comparison produced the applied offset.</summary>
|
||||
[JsonPropertyName("source")]
|
||||
public AlignmentSource Source { get; set; }
|
||||
|
||||
/// <summary>Gets or sets the tier that was applied.</summary>
|
||||
[JsonPropertyName("tier")]
|
||||
public MatchTier Tier { get; set; }
|
||||
|
||||
/// <summary>Gets or sets the offset applied to every window, in seconds.</summary>
|
||||
[JsonPropertyName("offset_sec")]
|
||||
public double OffsetSec { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// Gets or sets the local alignment's score, or <c>null</c> when none was
|
||||
/// computed.
|
||||
/// </summary>
|
||||
[JsonPropertyName("score")]
|
||||
public double? Score { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// Gets or sets the recovered slide between the two analysis windows, in
|
||||
/// frames, or <c>null</c> when no local alignment was computed.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// Only part of <see cref="OffsetSec"/>: the rest comes from the two windows
|
||||
/// being anchored at different points when the runtimes differ.
|
||||
/// </remarks>
|
||||
[JsonPropertyName("offset_frames")]
|
||||
public int? OffsetFrames { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// Gets or sets the local file's own <c>v1:</c> audio signature, or
|
||||
/// <c>null</c> when none was computed.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// Kept so a later fetch can align against a new manifest without running
|
||||
/// FFmpeg over the media again — the expensive half of the operation, and
|
||||
/// the reason signatures are opt-in. It never leaves the instance: it is
|
||||
/// stored beside the truth file, and contribution strips provenance
|
||||
/// entirely (JR-034).
|
||||
/// </remarks>
|
||||
[JsonPropertyName("local_signature")]
|
||||
public string? LocalSignature { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// Gets or sets the offset the server claimed, retained even when a local
|
||||
/// alignment superseded it.
|
||||
/// </summary>
|
||||
[JsonPropertyName("server_offset_sec")]
|
||||
public double ServerOffsetSec { get; set; }
|
||||
|
||||
/// <summary>Gets or sets the tier the server claimed.</summary>
|
||||
[JsonPropertyName("server_tier")]
|
||||
public MatchTier ServerTier { get; set; }
|
||||
}
|
||||
@@ -74,6 +74,18 @@ public class TruthProvenance
|
||||
[JsonPropertyName("offset_sec")]
|
||||
public double OffsetSec { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// Gets or sets how the applied offset was arrived at, or null for local
|
||||
/// sources and for fetches made before alignment was recorded.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// <see cref="OffsetSec"/> says what was applied; this says why, and keeps
|
||||
/// the local file's own signature so a later fetch can re-align without
|
||||
/// decoding the media again (JR-047).
|
||||
/// </remarks>
|
||||
[JsonPropertyName("alignment")]
|
||||
public TruthAlignment? Alignment { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// Gets or sets a human-readable caveat to surface with the overlay, or
|
||||
/// null when the claim needs none.
|
||||
|
||||
@@ -23,9 +23,12 @@ public class ServiceRegistrator : IPluginServiceRegistrator
|
||||
// retried once per item (JR-037).
|
||||
serviceCollection.AddSingleton<IManifestExchangeClient, ManifestExchangeClient>();
|
||||
|
||||
// Registered concretely: the signature is computed, not yet consumed, so
|
||||
// there is no second implementation for an interface to abstract over
|
||||
// and nothing to gain from inventing one (JR-042).
|
||||
// Registered concretely: there is no second implementation for an
|
||||
// interface to abstract over and nothing to gain from inventing one
|
||||
// (JR-042). The aligner consumes the signature on the fetch path, so a
|
||||
// manifest is checked against the local file before its windows are
|
||||
// stored (JR-047).
|
||||
serviceCollection.AddSingleton<AudioSignatureService>();
|
||||
serviceCollection.AddSingleton<ManifestAligner>();
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,205 @@
|
||||
using System;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using Jellyfin.Plugin.JRay.Configuration;
|
||||
using Jellyfin.Plugin.JRay.Models;
|
||||
using Microsoft.Extensions.Logging;
|
||||
|
||||
namespace Jellyfin.Plugin.JRay.Services;
|
||||
|
||||
/// <summary>
|
||||
/// Aligns a freshly fetched manifest to the local file before its windows are
|
||||
/// stored.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// <b>Why the client re-derives an offset the server already sent.</b> The
|
||||
/// server has never seen the local file. Its offset is a claim about a runtime
|
||||
/// it was told, so it can only ever be a runtime-difference inference. A local
|
||||
/// alignment compares the manifest's own audio signature against the file the
|
||||
/// windows will actually be drawn over, which is the authoritative comparison —
|
||||
/// and it needs no round trip, so no signature leaves the instance. Where the
|
||||
/// two disagree, the local one wins.
|
||||
/// <para>
|
||||
/// This is what jRay's specification means by matching being "a consumer
|
||||
/// concern": the server never rewrites a manifest, so one stored manifest serves
|
||||
/// every trim of the same cut, and each client shifts it to its own timebase
|
||||
/// (JR-030).
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// <b>Degradation, not failure.</b> Signatures switched off, an item under the
|
||||
/// 120 s window, a manifest with no signature, a missing FFmpeg binary, a decode
|
||||
/// error, or two signatures that simply do not match — every one of these falls
|
||||
/// back to the server's offset. A signature is an enhancement to cut matching; a
|
||||
/// missing one costs a tier and must never be able to break a fetch.
|
||||
/// </para>
|
||||
/// </remarks>
|
||||
// TRACES: JR-047 | SR-003
|
||||
public class ManifestAligner
|
||||
{
|
||||
private readonly AudioSignatureService _signatures;
|
||||
private readonly ILogger<ManifestAligner> _logger;
|
||||
|
||||
/// <summary>
|
||||
/// Initializes a new instance of the <see cref="ManifestAligner"/> class.
|
||||
/// </summary>
|
||||
/// <param name="signatures">Computes the local file's audio signature.</param>
|
||||
/// <param name="logger">Logger.</param>
|
||||
public ManifestAligner(AudioSignatureService signatures, ILogger<ManifestAligner> logger)
|
||||
{
|
||||
_signatures = signatures;
|
||||
_logger = logger;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Decides which offset to apply, given a local signature that has already
|
||||
/// been computed.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// Split from <see cref="AlignAsync"/> so the decision is testable without
|
||||
/// an FFmpeg binary or a media file: everything interesting happens here,
|
||||
/// and the caller only supplies the two strings.
|
||||
/// </remarks>
|
||||
/// <param name="localSignature">The local file's signature, or <c>null</c>.</param>
|
||||
/// <param name="manifestSignature">The manifest's signature, or <c>null</c>.</param>
|
||||
/// <param name="localRuntimeSec">Local file runtime, in seconds.</param>
|
||||
/// <param name="manifestRuntimeSec">Runtime the manifest records, in seconds.</param>
|
||||
/// <param name="serverTier">The tier the server reported.</param>
|
||||
/// <param name="serverOffsetSec">The offset the server reported.</param>
|
||||
/// <returns>What to apply, and how it was decided.</returns>
|
||||
public static TruthAlignment Resolve(
|
||||
string? localSignature,
|
||||
string? manifestSignature,
|
||||
double localRuntimeSec,
|
||||
double manifestRuntimeSec,
|
||||
MatchTier serverTier,
|
||||
double serverOffsetSec)
|
||||
{
|
||||
var alignment = new TruthAlignment
|
||||
{
|
||||
Source = AlignmentSource.Server,
|
||||
Tier = serverTier,
|
||||
OffsetSec = serverOffsetSec,
|
||||
ServerTier = serverTier,
|
||||
ServerOffsetSec = serverOffsetSec,
|
||||
LocalSignature = localSignature,
|
||||
};
|
||||
|
||||
// "No alignment was possible" and "the audio does not match" are very
|
||||
// different things to tell someone, and `Compare` returns null for both.
|
||||
// A comparison is only possible when both items clear the 120 s window
|
||||
// (JR-044) and both signatures parse as v1 (JR-045) — a `v2:` signature
|
||||
// from a future producer is un-comparable, not a mismatch. Separating
|
||||
// them here is what keeps a 90-second extra from being reported as
|
||||
// content that disagrees with its own manifest.
|
||||
var comparable = localSignature is not null
|
||||
&& manifestSignature is not null
|
||||
&& localRuntimeSec >= AudioSignature.WindowSec
|
||||
&& manifestRuntimeSec >= AudioSignature.WindowSec
|
||||
&& AudioSignatureMatcher.TryParseFrames(localSignature) is not null
|
||||
&& AudioSignatureMatcher.TryParseFrames(manifestSignature) is not null;
|
||||
|
||||
if (!comparable)
|
||||
{
|
||||
return alignment;
|
||||
}
|
||||
|
||||
var match = AudioSignatureMatcher.Compare(
|
||||
localSignature, manifestSignature, localRuntimeSec, manifestRuntimeSec);
|
||||
|
||||
if (match is null)
|
||||
{
|
||||
// Both sides had a signature and they did not align at any tier. The
|
||||
// server's offset still applies — the audio may legitimately differ,
|
||||
// and a signature must not break a fetch — but this is the strongest
|
||||
// available hint that the manifest describes different content, so
|
||||
// it is recorded rather than silently discarded.
|
||||
alignment.Source = AlignmentSource.LocalMismatch;
|
||||
return alignment;
|
||||
}
|
||||
|
||||
alignment.Source = AlignmentSource.Local;
|
||||
alignment.Tier = match.Value.Tier;
|
||||
alignment.OffsetSec = match.Value.OffsetSec;
|
||||
alignment.Score = match.Value.Score;
|
||||
alignment.OffsetFrames = match.Value.OffsetFrames;
|
||||
return alignment;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Computes the local file's signature if it can, then resolves the
|
||||
/// alignment.
|
||||
/// </summary>
|
||||
/// <param name="mediaPath">Path of the local media file.</param>
|
||||
/// <param name="localRuntimeSec">Local file runtime, in seconds.</param>
|
||||
/// <param name="manifest">The fetched manifest.</param>
|
||||
/// <param name="serverTier">The tier the server reported.</param>
|
||||
/// <param name="serverOffsetSec">The offset the server reported.</param>
|
||||
/// <param name="computeSignatures">Whether signatures are enabled in configuration.</param>
|
||||
/// <param name="cancellationToken">Cancellation token.</param>
|
||||
/// <returns>What to apply, and how it was decided.</returns>
|
||||
public async Task<TruthAlignment> AlignAsync(
|
||||
string mediaPath,
|
||||
double localRuntimeSec,
|
||||
Jmanifest manifest,
|
||||
MatchTier serverTier,
|
||||
double serverOffsetSec,
|
||||
bool computeSignatures,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(manifest);
|
||||
|
||||
var manifestSignature = manifest.Cut?.AudioSignature;
|
||||
string? localSignature = null;
|
||||
|
||||
// The decode is the expensive half, so it is skipped outright when it
|
||||
// could not change the answer: no manifest signature to compare against
|
||||
// means no local alignment is possible.
|
||||
if (computeSignatures && manifestSignature is not null && !string.IsNullOrEmpty(mediaPath))
|
||||
{
|
||||
localSignature = await _signatures
|
||||
.ComputeAsync(mediaPath, localRuntimeSec, cancellationToken)
|
||||
.ConfigureAwait(false);
|
||||
}
|
||||
|
||||
var alignment = Resolve(
|
||||
localSignature,
|
||||
manifestSignature,
|
||||
localRuntimeSec,
|
||||
manifest.Cut?.RuntimeSec ?? 0.0,
|
||||
serverTier,
|
||||
serverOffsetSec);
|
||||
|
||||
switch (alignment.Source)
|
||||
{
|
||||
case AlignmentSource.Local:
|
||||
_logger.LogInformation(
|
||||
"Local audio alignment for {Path}: score {Score:F3}, {Frames} frames, offset {Offset:F3}s at tier {Tier} (server said {ServerOffset:F3}s at {ServerTier})",
|
||||
mediaPath,
|
||||
alignment.Score,
|
||||
alignment.OffsetFrames,
|
||||
alignment.OffsetSec,
|
||||
alignment.Tier,
|
||||
alignment.ServerOffsetSec,
|
||||
alignment.ServerTier);
|
||||
break;
|
||||
|
||||
case AlignmentSource.LocalMismatch:
|
||||
_logger.LogWarning(
|
||||
"Audio signatures for {Path} did not align with the fetched manifest; applying the server's {Offset:F3}s at {Tier}. This may be a different cut, a different language track, or a heavy re-encode.",
|
||||
mediaPath,
|
||||
alignment.OffsetSec,
|
||||
alignment.Tier);
|
||||
break;
|
||||
|
||||
default:
|
||||
_logger.LogDebug(
|
||||
"No local alignment for {Path}; applying the server's {Offset:F3}s at {Tier}",
|
||||
mediaPath,
|
||||
alignment.OffsetSec,
|
||||
alignment.Tier);
|
||||
break;
|
||||
}
|
||||
|
||||
return alignment;
|
||||
}
|
||||
}
|
||||
@@ -118,9 +118,19 @@ public static class ManifestConverter
|
||||
/// </remarks>
|
||||
/// <param name="tier">The tier achieved.</param>
|
||||
/// <param name="offsetSec">The offset applied.</param>
|
||||
/// <param name="alignment">How the offset was arrived at, when known.</param>
|
||||
/// <returns>A caveat string, or null when the match needs no explanation.</returns>
|
||||
public static string? DescribeCaveat(MatchTier tier, double offsetSec)
|
||||
public static string? DescribeCaveat(MatchTier tier, double offsetSec, TruthAlignment? alignment = null)
|
||||
{
|
||||
// Ranked before the tier, because it is the stronger statement: the
|
||||
// server's tier says the runtimes are compatible, while a signature
|
||||
// mismatch says the audio itself is not. The second outranks the first.
|
||||
if (alignment?.Source == AlignmentSource.LocalMismatch)
|
||||
{
|
||||
return "This file's audio does not match the fetched manifest — it may be a different cut, "
|
||||
+ "a different language track, or a heavy re-encode. Timings may be wrong.";
|
||||
}
|
||||
|
||||
if (tier == MatchTier.Loose)
|
||||
{
|
||||
return "Matched loosely — the runtime differs from this server's copy, so timings may drift.";
|
||||
|
||||
Reference in New Issue
Block a user