JRay-public-server/src/ingest.rs

//! Manifest ingestion: the shared path behind `POST /manifests` and
//! `POST /manifests/bundle`, and the path a federation pull will reuse (§9a
//! "re-derive, don't inherit").
//!
//! Stages 0 and 1 are layers; stage 2 is parse + [`crate::validate`]. What
//! happens here is persistence plus enqueueing the stage 3 check: the upload is
//! accepted with `202` and the manifest is held **unlisted** until the cast check
//! completes — it is not served to anyone in the meantime (§6).

use anyhow::Context;

use crate::content_id::{self, CanonicalActor, CanonicalCut, CanonicalIdentity};
use crate::db::repo::{self, NewManifest};
use crate::model::IdentityType;
use crate::validate::ValidManifest;

/// Outcome of persisting one manifest.
#[derive(Debug, Clone)]
pub enum IngestOutcome {
    /// Held unlisted pending the §6 stage 3 cast check.
    Pending { manifest_id: String },
    /// §4 `409` — identical `(identity, cut)` from this contributor.
    DuplicateFromContributor { manifest_id: String },
    /// §9a — the exact same content is already held, from any source. Skipped
    /// without re-validation, which is the deduplication content addressing buys.
    DuplicateContent { manifest_id: String },
}

impl IngestOutcome {
    pub fn manifest_id(&self) -> &str {
        match self {
            IngestOutcome::Pending { manifest_id }
            | IngestOutcome::DuplicateFromContributor { manifest_id }
            | IngestOutcome::DuplicateContent { manifest_id } => manifest_id,
        }
    }
}

/// Job payload for the §6 stage 3 check.
#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)]
pub struct CastCheckJob {
    pub manifest_id: String,
}

pub const JOB_CAST_CHECK: &str = "cast_check";

/// Persists a validated manifest and enqueues its cast check, all in one
/// transaction — so a manifest is never left listed-but-unchecked, and its scene
/// rows go in as a single transaction rather than one per row (§8).
/// TRACES: UR-002, UR-012 | SR-005 | PR-006
pub fn persist(
    tx: &rusqlite::Transaction<'_>,
    valid: &ValidManifest,
    contributor_id: Option<&str>,
    origin: &str,
    ingested_from: Option<&str>,
    now: &str,
) -> anyhow::Result<IngestOutcome> {
    let m = &valid.manifest;
    let kind = m.identity.kind;
    let tmdb_id = m.identity.effective_tmdb_id();
    let imdb_id = m.identity.effective_imdb_id();

    let title_id = repo::upsert_title(
        tx,
        kind,
        tmdb_id,
        imdb_id,
        m.identity.title.as_deref(),
        m.identity.year,
        now,
    )
    .context("resolving title")?;

    let (season, episode) = match kind {
        IdentityType::Movie => (None, None),
        IdentityType::Episode => (m.identity.season, m.identity.episode),
    };

    // Content addressing over the *submitted* actor ids. Recomputed after the
    // cast check drops unmatched actors, since dropping changes the content.
    let cid = compute_content_id(valid);

    if let Some(existing) = repo::manifest_by_content_id(tx, &cid)? {
        return Ok(IngestOutcome::DuplicateContent { manifest_id: existing });
    }

    if let Some(c) = contributor_id {
        if let Some(existing) = repo::duplicate_from_contributor(
            tx,
            &title_id,
            season,
            episode,
            m.cut.runtime_sec,
            m.cut.video_hash.as_deref(),
            c,
        )? {
            return Ok(IngestOutcome::DuplicateFromContributor { manifest_id: existing });
        }
    }

    let manifest_id = ulid::Ulid::new().to_string();
    let extraction = m.extraction.as_ref();

    repo::insert_manifest(
        tx,
        &NewManifest {
            id: &manifest_id,
            title_id: &title_id,
            season,
            episode,
            runtime_sec: m.cut.runtime_sec,
            video_hash: m.cut.video_hash.as_deref(),
            // Stored as an attribute, not part of identity (§9a).
            audio_signature: None,
            audio_sig_coarse: None,
            sample_fps: extraction.and_then(|e| e.sample_fps),
            extinction_sec: extraction.and_then(|e| e.extinction_sec),
            pipeline_version: extraction.and_then(|e| e.pipeline_version.as_deref()),
            gallery_scope: extraction.and_then(|e| e.gallery_scope).map(|g| g.as_str()),
            contributor_id,
            // Held unlisted until stage 3 completes (§6).
            status: "pending",
            content_id: Some(&cid),
            origin,
            ingested_from,
            created_at: now,
        },
    )?;

    // Actors are recorded by TMDB person id only. Those without one cannot be
    // stored at all — there is no name column to put them in (§5a, §7) — so they
    // are carried into the cast check via the submitted payload instead.
    for actor in &valid.actor_scenes_cs {
        if let Some(person_id) = actor.tmdb_id {
            repo::insert_actor_scenes(tx, &manifest_id, person_id, &actor.scenes_cs)?;
        }
    }

    let payload = serde_json::to_string(&CastCheckJob { manifest_id: manifest_id.clone() })?;
    repo::enqueue_job(tx, JOB_CAST_CHECK, &payload, now)?;

    Ok(IngestOutcome::Pending { manifest_id })
}

/// Computes the §9a `content_id` for a validated manifest.
pub fn compute_content_id(valid: &ValidManifest) -> String {
    let m = &valid.manifest;
    let identity = CanonicalIdentity {
        kind: match m.identity.kind {
            IdentityType::Movie => "movie",
            IdentityType::Episode => "episode",
        },
        tmdb_id: m.identity.effective_tmdb_id().map(str::to_string),
        imdb_id: m.identity.effective_imdb_id().map(str::to_string),
        season: m.identity.season,
        episode: m.identity.episode,
    };
    let cut = CanonicalCut {
        runtime_cs: crate::validate::to_centiseconds(m.cut.runtime_sec),
        video_hash: m.cut.video_hash.clone(),
    };
    let actors: Vec<CanonicalActor> = valid
        .actor_scenes_cs
        .iter()
        .filter_map(|a| {
            a.tmdb_id.map(|id| CanonicalActor {
                tmdb_person_id: id,
                // Timings only. §9a excludes belief and route from identity:
                // they are producer-side estimates that may differ between
                // pipeline versions for identical content, so hashing them
                // would give two servers different ids for the same
                // manifest — the failure mode centisecond quantisation
                // exists to remove. They replicate as attributes instead.
                scenes_cs: a.scenes_cs.iter().map(|s| (s.start_cs, s.end_cs)).collect(),
            })
        })
        .collect();

    content_id::content_id(&identity, &cut, &actors)
}

#[cfg(test)]
mod tests {
    use super::*;
    use crate::db::Db;
    use crate::model::Jmanifest;
    use crate::validate::validate_manifest;

    const NOW: &str = "2026-07-30T12:00:00Z";

    fn valid_from(json: &str) -> ValidManifest {
        let m: Jmanifest = serde_json::from_str(json).unwrap();
        validate_manifest(m).unwrap()
    }

    fn movie_json(tmdb: &str, runtime: f64) -> String {
        format!(
            r#"{{"jmanifest_version":2,
                "identity":{{"type":"movie","tmdb_id":"{tmdb}","title":"A Film"}},
                "cut":{{"runtime_sec":{runtime}}},
                "extraction":{{"sample_fps":5,"pipeline_version":"test 0.1"}},
                "actors":[{{"name":"Steve Buscemi","tmdb_id":"884","scenes":[{{"start":10.0,"end":20.0}}]}},
                          {{"name":"Michael Palin","tmdb_id":"11007","scenes":[{{"start":30.0,"end":40.0}}]}}]}}"#
        )
    }

    #[tokio::test]
    async fn persists_as_pending_and_enqueues_a_check() {
        let db = Db::open(":memory:").unwrap();
        let valid = valid_from(&movie_json("504172", 6420.5));

        let (outcome, status, jobs) = db
            .write(move |tx| {
                let c = repo::insert_contributor(tx, "h", NOW)?;
                let outcome = persist(tx, &valid, Some(&c), "local", None, NOW)?;
                let status = repo::manifest_status(tx, outcome.manifest_id())?;
                let jobs = repo::lease_jobs(tx, NOW, 10)?;
                Ok((outcome, status, jobs))
            })
            .await
            .unwrap();

        assert!(matches!(outcome, IngestOutcome::Pending { .. }));
        // §6: held unlisted, not served to anyone, until stage 3 completes.
        assert_eq!(status.unwrap().0, "pending");
        assert_eq!(jobs.len(), 1);
        assert_eq!(jobs[0].kind, JOB_CAST_CHECK);
    }

    #[tokio::test]
    async fn a_pending_manifest_is_not_served() {
        let db = Db::open(":memory:").unwrap();
        let valid = valid_from(&movie_json("504172", 6420.5));
        let candidates = db
            .write(move |tx| {
                let c = repo::insert_contributor(tx, "h", NOW)?;
                persist(tx, &valid, Some(&c), "local", None, NOW)?;
                let title =
                    repo::find_title(tx, IdentityType::Movie, Some("504172"), None)?.unwrap();
                repo::candidates_for_title(tx, &title.id, None, None)
            })
            .await
            .unwrap();
        assert!(candidates.is_empty());
    }

    #[tokio::test]
    async fn identical_content_deduplicates() {
        // §9a: a manifest whose `content_id` is already present is skipped
        // without re-validation.
        let db = Db::open(":memory:").unwrap();
        let a = valid_from(&movie_json("504172", 6420.5));
        let b = valid_from(&movie_json("504172", 6420.5));

        let (first, second) = db
            .write(move |tx| {
                let c1 = repo::insert_contributor(tx, "h1", NOW)?;
                let c2 = repo::insert_contributor(tx, "h2", NOW)?;
                let first = persist(tx, &a, Some(&c1), "local", None, NOW)?;
                // A *different* contributor, so this is content dedup, not the
                // per-contributor 409.
                let second = persist(tx, &b, Some(&c2), "local", None, NOW)?;
                Ok((first, second))
            })
            .await
            .unwrap();

        assert!(matches!(first, IngestOutcome::Pending { .. }));
        assert!(matches!(second, IngestOutcome::DuplicateContent { .. }));
        assert_eq!(first.manifest_id(), second.manifest_id());
    }

    #[tokio::test]
    async fn same_contributor_resubmitting_the_same_cut_is_a_duplicate() {
        let db = Db::open(":memory:").unwrap();
        // Same identity and cut, different actor timings => different content_id,
        // so this exercises the per-contributor 409 path specifically.
        let a = valid_from(&movie_json("504172", 6420.5));
        let b = valid_from(
            r#"{"jmanifest_version":2,
                "identity":{"type":"movie","tmdb_id":"504172","title":"A Film"},
                "cut":{"runtime_sec":6420.5},
                "actors":[{"name":"Steve Buscemi","tmdb_id":"884","scenes":[{"start":11.0,"end":21.0}]}]}"#,
        );

        let second = db
            .write(move |tx| {
                let c = repo::insert_contributor(tx, "h", NOW)?;
                persist(tx, &a, Some(&c), "local", None, NOW)?;
                persist(tx, &b, Some(&c), "local", None, NOW)
            })
            .await
            .unwrap();

        assert!(matches!(second, IngestOutcome::DuplicateFromContributor { .. }));
    }

    #[tokio::test]
    async fn different_cuts_of_one_title_coexist() {
        // §7: multiple manifests may coexist for the same title with different
        // cuts — that is the point.
        let db = Db::open(":memory:").unwrap();
        let a = valid_from(&movie_json("504172", 6420.5));
        let b = valid_from(&movie_json("504172", 7000.0));

        let (x, y) = db
            .write(move |tx| {
                let c = repo::insert_contributor(tx, "h", NOW)?;
                let x = persist(tx, &a, Some(&c), "local", None, NOW)?;
                let y = persist(tx, &b, Some(&c), "local", None, NOW)?;
                Ok((x, y))
            })
            .await
            .unwrap();

        assert!(matches!(x, IngestOutcome::Pending { .. }));
        assert!(matches!(y, IngestOutcome::Pending { .. }));
        assert_ne!(x.manifest_id(), y.manifest_id());
    }

    #[tokio::test]
    async fn episode_manifests_carry_their_coordinates() {
        let db = Db::open(":memory:").unwrap();
        let valid = valid_from(
            r#"{"jmanifest_version":2,
                "identity":{"type":"episode","series_tmdb_id":"1396","title":"Breaking Bad",
                            "season":2,"episode":5},
                "cut":{"runtime_sec":2820.0},
                "actors":[{"name":"Bryan Cranston","tmdb_id":"17419","scenes":[{"start":10.0,"end":20.0}]}]}"#,
        );
        let row = db
            .write(move |tx| {
                let c = repo::insert_contributor(tx, "h", NOW)?;
                let o = persist(tx, &valid, Some(&c), "local", None, NOW)?;
                Ok(repo::manifest_by_id(tx, o.manifest_id())?.unwrap())
            })
            .await
            .unwrap();
        assert_eq!((row.season, row.episode), (Some(2), Some(5)));
    }

    #[tokio::test]
    async fn upload_metadata_is_not_echoed_back_as_actor_names() {
        // §5a/§7: only integers reach the database. The submitted name is used
        // for matching and never persisted, so before the cast check populates
        // `people` there is no name to serve.
        let db = Db::open(":memory:").unwrap();
        let valid = valid_from(&movie_json("504172", 6420.5));
        let actors = db
            .write(move |tx| {
                let c = repo::insert_contributor(tx, "h", NOW)?;
                let o = persist(tx, &valid, Some(&c), "local", None, NOW)?;
                repo::actors_for_manifest(tx, o.manifest_id())
            })
            .await
            .unwrap();
        assert_eq!(actors.len(), 2);
        assert!(actors.iter().all(|a| a.name.is_none()));
    }

    #[test]
    fn content_id_excludes_extraction_metadata() {
        // §9a: `extraction` metadata and local state are excluded, so two
        // servers validating the same upload agree.
        let a = valid_from(&movie_json("504172", 6420.5));
        let b = valid_from(
            r#"{"jmanifest_version":2,
                "identity":{"type":"movie","tmdb_id":"504172","title":"A Film"},
                "cut":{"runtime_sec":6420.5},
                "extraction":{"sample_fps":1,"extinction_sec":9,"pipeline_version":"other 9.9",
                              "gallery_size":5},
                "actors":[{"name":"Steve Buscemi","tmdb_id":"884","scenes":[{"start":10.0,"end":20.0}]},
                          {"name":"Michael Palin","tmdb_id":"11007","scenes":[{"start":30.0,"end":40.0}]}]}"#,
        );
        assert_eq!(compute_content_id(&a), compute_content_id(&b));
    }

    #[test]
    fn content_id_excludes_the_audio_signature() {
        // §9a is explicit: including it would produce different content_ids for
        // identical content and silently break federation deduplication.
        let a = valid_from(&movie_json("504172", 6420.5));
        let sig = format!("v1:{}", "A".repeat(1720));
        let with_sig = format!(
            r#"{{"jmanifest_version":2,
                "identity":{{"type":"movie","tmdb_id":"504172","title":"A Film"}},
                "cut":{{"runtime_sec":6420.5,"audio_signature":"{sig}"}},
                "extraction":{{"sample_fps":5,"pipeline_version":"test 0.1"}},
                "actors":[{{"name":"Steve Buscemi","tmdb_id":"884","scenes":[{{"start":10.0,"end":20.0}}]}},
                          {{"name":"Michael Palin","tmdb_id":"11007","scenes":[{{"start":30.0,"end":40.0}}]}}]}}"#
        );
        let b = valid_from(&with_sig);
        assert_eq!(compute_content_id(&a), compute_content_id(&b));
    }

    #[test]
    fn content_id_excludes_belief_and_route() {
        // The trap in the SR-003 bump (UR-018). Belief is a producer-side
        // estimate that may legitimately differ between pipeline versions for
        // identical timings, so hashing it would give two servers different ids
        // for the same manifest — the exact failure mode §9a quantises
        // centiseconds to avoid, reintroduced one field along.
        //
        // Two manifests, same windows, wildly different confidence and routes.
        let bare = r#"{"jmanifest_version":2,
            "identity":{"type":"movie","tmdb_id":"504172","title":"A Film"},
            "cut":{"runtime_sec":6420.5},
            "actors":[{"name":"Steve Buscemi","tmdb_id":"884",
                       "scenes":[{"start":10.0,"end":20.0}]},
                      {"name":"Michael Palin","tmdb_id":"11007",
                       "scenes":[{"start":30.0,"end":40.0}]}]}"#;
        let believed = r#"{"jmanifest_version":2,
            "identity":{"type":"movie","tmdb_id":"504172","title":"A Film"},
            "cut":{"runtime_sec":6420.5},
            "actors":[{"name":"Steve Buscemi","tmdb_id":"884",
                       "scenes":[{"start":10.0,"end":20.0,"belief":0.98,"route":"live"}]},
                      {"name":"Michael Palin","tmdb_id":"11007",
                       "scenes":[{"start":30.0,"end":40.0,"belief":0.31,"route":"deferred"}]}]}"#;

        assert_eq!(
            compute_content_id(&valid_from(bare)),
            compute_content_id(&valid_from(believed)),
            "belief and route must not enter identity"
        );

        // And the same content at a *different* belief still deduplicates.
        let other_belief = believed.replace("0.98", "0.42").replace("live", "pooled");
        assert_eq!(
            compute_content_id(&valid_from(believed)),
            compute_content_id(&valid_from(&other_belief)),
        );

        // Sanity: a genuine timing change *does* alter the id, so the test above
        // is not passing because the hash ignores everything.
        let shifted = bare.replace("\"end\":20.0", "\"end\":21.0");
        assert_ne!(
            compute_content_id(&valid_from(bare)),
            compute_content_id(&valid_from(&shifted))
        );
    }

    #[test]
    fn content_id_excludes_submitted_names() {
        // Names are not persisted, so they must not be part of identity either —
        // otherwise a renamed resubmission would evade deduplication.
        let a = valid_from(&movie_json("504172", 6420.5));
        let b = valid_from(
            r#"{"jmanifest_version":2,
                "identity":{"type":"movie","tmdb_id":"504172","title":"A Film"},
                "cut":{"runtime_sec":6420.5},
                "extraction":{"sample_fps":5,"pipeline_version":"test 0.1"},
                "actors":[{"name":"Someone Else","tmdb_id":"884","scenes":[{"start":10.0,"end":20.0}]},
                          {"name":"Another Person","tmdb_id":"11007","scenes":[{"start":30.0,"end":40.0}]}]}"#,
        );
        assert_eq!(compute_content_id(&a), compute_content_id(&b));
    }
}