Initial implementation: core vertical slice

Implements the core of SPEC.md — the manifest exchange, less audio-tier matching (§3) and federation (§9a), both of which the spec sequences as later work. - §2 Jmanifest format and series bundles - §3 cut matching: exact / runtime / loose tiers - §4 API, less POST /manifests/search - §5 rate limiting; §5a trust model, anonymous bearer tokens - §6 upload validation, all four stages - §7 relational storage, no JSON blob on the write path - §8 Rust + Axum + SQLite, single serialized writer, in-process job queue - §9a content addressing, computed on upload Reconciled against the system spec: - anneal_sec removed, withdrawn upstream by AR-012/AR-013. Presence follows track extent, so a track survives its own gaps and there is nothing to anneal. Its successor extinction_sec and the new gallery_scope are accepted and stored; scope enters the §7 ranking. A manifest still carrying anneal_sec is a hard 400, not silently ignored — it came from a pipeline whose window semantics differ from what this server assumes. - Audio signature: media under 120 s now emits no signature at all, matching scene-actor-extraction IR-007. The earlier §3 draft allowed a shortened window under 150 s, which was the weaker rule — a caller-varying length is the property SR-004 forbids. - UR IDs regularised to UR-nnn; docs/requirements.md registers 32 requirements, each tracing to an SR-nnn or PR-nnn. 189 tests: unit, end-to-end through the real router, and an injection suite covering SQL, JSON, header and Unicode payloads. Writing that suite found two real gaps, both fixed here: compatibility homoglyphs passed the §5a character class, and a one-frame audio signature was accepted on a feature-length item. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
2026-07-30 18:14:02 +02:00
commit a848750a65
38 changed files with 13014 additions and 0 deletions
@@ -0,0 +1,403 @@
+//! Manifest ingestion: the shared path behind `POST /manifests` and
+//! `POST /manifests/bundle`, and the path a federation pull will reuse (§9a
+//! "re-derive, don't inherit").
+//!
+//! Stages 0 and 1 are layers; stage 2 is parse + [`crate::validate`]. What
+//! happens here is persistence plus enqueueing the stage 3 check: the upload is
+//! accepted with `202` and the manifest is held **unlisted** until the cast check
+//! completes — it is not served to anyone in the meantime (§6).
+
+use anyhow::Context;
+
+use crate::content_id::{self, CanonicalActor, CanonicalCut, CanonicalIdentity};
+use crate::db::repo::{self, NewManifest};
+use crate::model::IdentityType;
+use crate::validate::ValidManifest;
+
+/// Outcome of persisting one manifest.
+#[derive(Debug, Clone)]
+pub enum IngestOutcome {
+    /// Held unlisted pending the §6 stage 3 cast check.
+    Pending { manifest_id: String },
+    /// §4 `409` — identical `(identity, cut)` from this contributor.
+    DuplicateFromContributor { manifest_id: String },
+    /// §9a — the exact same content is already held, from any source. Skipped
+    /// without re-validation, which is the deduplication content addressing buys.
+    DuplicateContent { manifest_id: String },
+}
+
+impl IngestOutcome {
+    pub fn manifest_id(&self) -> &str {
+        match self {
+            IngestOutcome::Pending { manifest_id }
+            | IngestOutcome::DuplicateFromContributor { manifest_id }
+            | IngestOutcome::DuplicateContent { manifest_id } => manifest_id,
+        }
+    }
+}
+
+/// Job payload for the §6 stage 3 check.
+#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)]
+pub struct CastCheckJob {
+    pub manifest_id: String,
+}
+
+pub const JOB_CAST_CHECK: &str = "cast_check";
+
+/// Persists a validated manifest and enqueues its cast check, all in one
+/// transaction — so a manifest is never left listed-but-unchecked, and its scene
+/// rows go in as a single transaction rather than one per row (§8).
+pub fn persist(
+    tx: &rusqlite::Transaction<'_>,
+    valid: &ValidManifest,
+    contributor_id: Option<&str>,
+    origin: &str,
+    ingested_from: Option<&str>,
+    now: &str,
+) -> anyhow::Result<IngestOutcome> {
+    let m = &valid.manifest;
+    let kind = m.identity.kind;
+    let tmdb_id = m.identity.effective_tmdb_id();
+    let imdb_id = m.identity.effective_imdb_id();
+
+    let title_id = repo::upsert_title(
+        tx,
+        kind,
+        tmdb_id,
+        imdb_id,
+        m.identity.title.as_deref(),
+        m.identity.year,
+        now,
+    )
+    .context("resolving title")?;
+
+    let (season, episode) = match kind {
+        IdentityType::Movie => (None, None),
+        IdentityType::Episode => (m.identity.season, m.identity.episode),
+    };
+
+    // Content addressing over the *submitted* actor ids. Recomputed after the
+    // cast check drops unmatched actors, since dropping changes the content.
+    let cid = compute_content_id(valid);
+
+    if let Some(existing) = repo::manifest_by_content_id(tx, &cid)? {
+        return Ok(IngestOutcome::DuplicateContent { manifest_id: existing });
+    }
+
+    if let Some(c) = contributor_id {
+        if let Some(existing) = repo::duplicate_from_contributor(
+            tx,
+            &title_id,
+            season,
+            episode,
+            m.cut.runtime_sec,
+            m.cut.video_hash.as_deref(),
+            c,
+        )? {
+            return Ok(IngestOutcome::DuplicateFromContributor { manifest_id: existing });
+        }
+    }
+
+    let manifest_id = ulid::Ulid::new().to_string();
+    let extraction = m.extraction.as_ref();
+
+    repo::insert_manifest(
+        tx,
+        &NewManifest {
+            id: &manifest_id,
+            title_id: &title_id,
+            season,
+            episode,
+            runtime_sec: m.cut.runtime_sec,
+            video_hash: m.cut.video_hash.as_deref(),
+            // Stored as an attribute, not part of identity (§9a).
+            audio_signature: None,
+            audio_sig_coarse: None,
+            sample_fps: extraction.and_then(|e| e.sample_fps),
+            extinction_sec: extraction.and_then(|e| e.extinction_sec),
+            pipeline_version: extraction.and_then(|e| e.pipeline_version.as_deref()),
+            gallery_scope: extraction.and_then(|e| e.gallery_scope).map(|g| g.as_str()),
+            contributor_id,
+            // Held unlisted until stage 3 completes (§6).
+            status: "pending",
+            content_id: Some(&cid),
+            origin,
+            ingested_from,
+            created_at: now,
+        },
+    )?;
+
+    // Actors are recorded by TMDB person id only. Those without one cannot be
+    // stored at all — there is no name column to put them in (§5a, §7) — so they
+    // are carried into the cast check via the submitted payload instead.
+    for actor in &valid.actor_scenes_cs {
+        if let Some(person_id) = actor.tmdb_id {
+            repo::insert_actor_scenes(tx, &manifest_id, person_id, &actor.scenes_cs)?;
+        }
+    }
+
+    let payload = serde_json::to_string(&CastCheckJob { manifest_id: manifest_id.clone() })?;
+    repo::enqueue_job(tx, JOB_CAST_CHECK, &payload, now)?;
+
+    Ok(IngestOutcome::Pending { manifest_id })
+}
+
+/// Computes the §9a `content_id` for a validated manifest.
+pub fn compute_content_id(valid: &ValidManifest) -> String {
+    let m = &valid.manifest;
+    let identity = CanonicalIdentity {
+        kind: match m.identity.kind {
+            IdentityType::Movie => "movie",
+            IdentityType::Episode => "episode",
+        },
+        tmdb_id: m.identity.effective_tmdb_id().map(str::to_string),
+        imdb_id: m.identity.effective_imdb_id().map(str::to_string),
+        season: m.identity.season,
+        episode: m.identity.episode,
+    };
+    let cut = CanonicalCut {
+        runtime_cs: crate::validate::to_centiseconds(m.cut.runtime_sec),
+        video_hash: m.cut.video_hash.clone(),
+    };
+    let actors: Vec<CanonicalActor> = valid
+        .actor_scenes_cs
+        .iter()
+        .filter_map(|a| {
+            a.tmdb_id
+                .map(|id| CanonicalActor { tmdb_person_id: id, scenes_cs: a.scenes_cs.clone() })
+        })
+        .collect();
+
+    content_id::content_id(&identity, &cut, &actors)
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use crate::db::Db;
+    use crate::model::Jmanifest;
+    use crate::validate::validate_manifest;
+
+    const NOW: &str = "2026-07-30T12:00:00Z";
+
+    fn valid_from(json: &str) -> ValidManifest {
+        let m: Jmanifest = serde_json::from_str(json).unwrap();
+        validate_manifest(m).unwrap()
+    }
+
+    fn movie_json(tmdb: &str, runtime: f64) -> String {
+        format!(
+            r#"{{"jmanifest_version":1,
+                "identity":{{"type":"movie","tmdb_id":"{tmdb}","title":"A Film"}},
+                "cut":{{"runtime_sec":{runtime}}},
+                "extraction":{{"sample_fps":5,"pipeline_version":"test 0.1"}},
+                "actors":[{{"name":"Steve Buscemi","tmdb_id":"884","scenes":[[10.0,20.0]]}},
+                          {{"name":"Michael Palin","tmdb_id":"11007","scenes":[[30.0,40.0]]}}]}}"#
+        )
+    }
+
+    #[tokio::test]
+    async fn persists_as_pending_and_enqueues_a_check() {
+        let db = Db::open(":memory:").unwrap();
+        let valid = valid_from(&movie_json("504172", 6420.5));
+
+        let (outcome, status, jobs) = db
+            .write(move |tx| {
+                let c = repo::insert_contributor(tx, "h", NOW)?;
+                let outcome = persist(tx, &valid, Some(&c), "local", None, NOW)?;
+                let status = repo::manifest_status(tx, outcome.manifest_id())?;
+                let jobs = repo::lease_jobs(tx, NOW, 10)?;
+                Ok((outcome, status, jobs))
+            })
+            .await
+            .unwrap();
+
+        assert!(matches!(outcome, IngestOutcome::Pending { .. }));
+        // §6: held unlisted, not served to anyone, until stage 3 completes.
+        assert_eq!(status.unwrap().0, "pending");
+        assert_eq!(jobs.len(), 1);
+        assert_eq!(jobs[0].kind, JOB_CAST_CHECK);
+    }
+
+    #[tokio::test]
+    async fn a_pending_manifest_is_not_served() {
+        let db = Db::open(":memory:").unwrap();
+        let valid = valid_from(&movie_json("504172", 6420.5));
+        let candidates = db
+            .write(move |tx| {
+                let c = repo::insert_contributor(tx, "h", NOW)?;
+                persist(tx, &valid, Some(&c), "local", None, NOW)?;
+                let title =
+                    repo::find_title(tx, IdentityType::Movie, Some("504172"), None)?.unwrap();
+                repo::candidates_for_title(tx, &title.id, None, None)
+            })
+            .await
+            .unwrap();
+        assert!(candidates.is_empty());
+    }
+
+    #[tokio::test]
+    async fn identical_content_deduplicates() {
+        // §9a: a manifest whose `content_id` is already present is skipped
+        // without re-validation.
+        let db = Db::open(":memory:").unwrap();
+        let a = valid_from(&movie_json("504172", 6420.5));
+        let b = valid_from(&movie_json("504172", 6420.5));
+
+        let (first, second) = db
+            .write(move |tx| {
+                let c1 = repo::insert_contributor(tx, "h1", NOW)?;
+                let c2 = repo::insert_contributor(tx, "h2", NOW)?;
+                let first = persist(tx, &a, Some(&c1), "local", None, NOW)?;
+                // A *different* contributor, so this is content dedup, not the
+                // per-contributor 409.
+                let second = persist(tx, &b, Some(&c2), "local", None, NOW)?;
+                Ok((first, second))
+            })
+            .await
+            .unwrap();
+
+        assert!(matches!(first, IngestOutcome::Pending { .. }));
+        assert!(matches!(second, IngestOutcome::DuplicateContent { .. }));
+        assert_eq!(first.manifest_id(), second.manifest_id());
+    }
+
+    #[tokio::test]
+    async fn same_contributor_resubmitting_the_same_cut_is_a_duplicate() {
+        let db = Db::open(":memory:").unwrap();
+        // Same identity and cut, different actor timings => different content_id,
+        // so this exercises the per-contributor 409 path specifically.
+        let a = valid_from(&movie_json("504172", 6420.5));
+        let b = valid_from(
+            r#"{"jmanifest_version":1,
+                "identity":{"type":"movie","tmdb_id":"504172","title":"A Film"},
+                "cut":{"runtime_sec":6420.5},
+                "actors":[{"name":"Steve Buscemi","tmdb_id":"884","scenes":[[11.0,21.0]]}]}"#,
+        );
+
+        let second = db
+            .write(move |tx| {
+                let c = repo::insert_contributor(tx, "h", NOW)?;
+                persist(tx, &a, Some(&c), "local", None, NOW)?;
+                persist(tx, &b, Some(&c), "local", None, NOW)
+            })
+            .await
+            .unwrap();
+
+        assert!(matches!(second, IngestOutcome::DuplicateFromContributor { .. }));
+    }
+
+    #[tokio::test]
+    async fn different_cuts_of_one_title_coexist() {
+        // §7: multiple manifests may coexist for the same title with different
+        // cuts — that is the point.
+        let db = Db::open(":memory:").unwrap();
+        let a = valid_from(&movie_json("504172", 6420.5));
+        let b = valid_from(&movie_json("504172", 7000.0));
+
+        let (x, y) = db
+            .write(move |tx| {
+                let c = repo::insert_contributor(tx, "h", NOW)?;
+                let x = persist(tx, &a, Some(&c), "local", None, NOW)?;
+                let y = persist(tx, &b, Some(&c), "local", None, NOW)?;
+                Ok((x, y))
+            })
+            .await
+            .unwrap();
+
+        assert!(matches!(x, IngestOutcome::Pending { .. }));
+        assert!(matches!(y, IngestOutcome::Pending { .. }));
+        assert_ne!(x.manifest_id(), y.manifest_id());
+    }
+
+    #[tokio::test]
+    async fn episode_manifests_carry_their_coordinates() {
+        let db = Db::open(":memory:").unwrap();
+        let valid = valid_from(
+            r#"{"jmanifest_version":1,
+                "identity":{"type":"episode","series_tmdb_id":"1396","title":"Breaking Bad",
+                            "season":2,"episode":5},
+                "cut":{"runtime_sec":2820.0},
+                "actors":[{"name":"Bryan Cranston","tmdb_id":"17419","scenes":[[10.0,20.0]]}]}"#,
+        );
+        let row = db
+            .write(move |tx| {
+                let c = repo::insert_contributor(tx, "h", NOW)?;
+                let o = persist(tx, &valid, Some(&c), "local", None, NOW)?;
+                Ok(repo::manifest_by_id(tx, o.manifest_id())?.unwrap())
+            })
+            .await
+            .unwrap();
+        assert_eq!((row.season, row.episode), (Some(2), Some(5)));
+    }
+
+    #[tokio::test]
+    async fn upload_metadata_is_not_echoed_back_as_actor_names() {
+        // §5a/§7: only integers reach the database. The submitted name is used
+        // for matching and never persisted, so before the cast check populates
+        // `people` there is no name to serve.
+        let db = Db::open(":memory:").unwrap();
+        let valid = valid_from(&movie_json("504172", 6420.5));
+        let actors = db
+            .write(move |tx| {
+                let c = repo::insert_contributor(tx, "h", NOW)?;
+                let o = persist(tx, &valid, Some(&c), "local", None, NOW)?;
+                repo::actors_for_manifest(tx, o.manifest_id())
+            })
+            .await
+            .unwrap();
+        assert_eq!(actors.len(), 2);
+        assert!(actors.iter().all(|a| a.name.is_none()));
+    }
+
+    #[test]
+    fn content_id_excludes_extraction_metadata() {
+        // §9a: `extraction` metadata and local state are excluded, so two
+        // servers validating the same upload agree.
+        let a = valid_from(&movie_json("504172", 6420.5));
+        let b = valid_from(
+            r#"{"jmanifest_version":1,
+                "identity":{"type":"movie","tmdb_id":"504172","title":"A Film"},
+                "cut":{"runtime_sec":6420.5},
+                "extraction":{"sample_fps":1,"extinction_sec":9,"pipeline_version":"other 9.9",
+                              "gallery_size":5},
+                "actors":[{"name":"Steve Buscemi","tmdb_id":"884","scenes":[[10.0,20.0]]},
+                          {"name":"Michael Palin","tmdb_id":"11007","scenes":[[30.0,40.0]]}]}"#,
+        );
+        assert_eq!(compute_content_id(&a), compute_content_id(&b));
+    }
+
+    #[test]
+    fn content_id_excludes_the_audio_signature() {
+        // §9a is explicit: including it would produce different content_ids for
+        // identical content and silently break federation deduplication.
+        let a = valid_from(&movie_json("504172", 6420.5));
+        let sig = format!("v1:{}", "A".repeat(1720));
+        let with_sig = format!(
+            r#"{{"jmanifest_version":1,
+                "identity":{{"type":"movie","tmdb_id":"504172","title":"A Film"}},
+                "cut":{{"runtime_sec":6420.5,"audio_signature":"{sig}"}},
+                "extraction":{{"sample_fps":5,"pipeline_version":"test 0.1"}},
+                "actors":[{{"name":"Steve Buscemi","tmdb_id":"884","scenes":[[10.0,20.0]]}},
+                          {{"name":"Michael Palin","tmdb_id":"11007","scenes":[[30.0,40.0]]}}]}}"#
+        );
+        let b = valid_from(&with_sig);
+        assert_eq!(compute_content_id(&a), compute_content_id(&b));
+    }
+
+    #[test]
+    fn content_id_excludes_submitted_names() {
+        // Names are not persisted, so they must not be part of identity either —
+        // otherwise a renamed resubmission would evade deduplication.
+        let a = valid_from(&movie_json("504172", 6420.5));
+        let b = valid_from(
+            r#"{"jmanifest_version":1,
+                "identity":{"type":"movie","tmdb_id":"504172","title":"A Film"},
+                "cut":{"runtime_sec":6420.5},
+                "extraction":{"sample_fps":5,"pipeline_version":"test 0.1"},
+                "actors":[{"name":"Someone Else","tmdb_id":"884","scenes":[[10.0,20.0]]},
+                          {"name":"Another Person","tmdb_id":"11007","scenes":[[30.0,40.0]]}]}"#,
+        );
+        assert_eq!(compute_content_id(&a), compute_content_id(&b));
+    }
+}