scene-actor-extraction/src/types.hpp

#pragma once
#include <array>
#include <cstdint>
#include <string>
#include <vector>

#include <opencv2/core.hpp>

#include "gallery/embedder_stamp.hpp"

// ── Embedding ─────────────────────────────────────────────────────────────────
// 512-dim L2-normalised ArcFace embedding
using Embedding = std::array<float, 512>;

inline float cosine_similarity(const Embedding& a, const Embedding& b) {
    float dot = 0.f;
    for (int i = 0; i < 512; ++i) dot += a[i] * b[i];
    return dot;
}

// ── Frame ─────────────────────────────────────────────────────────────────────
// Raw sampled frame from the movie.  eof=true is the pipeline shutdown sentinel:
// every node must forward it immediately without processing.
struct Frame {
    cv::Mat  image;
    double   timestamp_sec{0.0};
    int64_t  frame_idx{-1};
    bool     eof{false};
    bool     is_cut{false};   // histogram: intra-scene camera-angle change (tracker reset)
    bool     is_scene_boundary{false};  // TransNetV2: true shot/scene boundary (opt-in)
    float    cut_score{0.f};  // histogram cut score = 1 - hist_corr (0=identical, ~1=cut); HUD/debug
    float    bbox_upscale{1.f};  // multiply detector bboxes/landmarks by this to map back to
                                 // original video resolution (>1 when dense_scale downscaled the frame)
};

// ── CutEvent ──────────────────────────────────────────────────────────────────
// Emitted by SceneDetectorFunc when TransNetV2 localises a shot boundary, keyed
// by the boundary frame's timestamp. eof=true is the shutdown sentinel.
struct CutEvent {
    double   timestamp_sec{0.0};
    float    probability{0.f};   // sigmoid boundary score at the peak
    bool     eof{false};
};

// ── ArcFace alignment ─────────────────────────────────────────────────────────
// Canonical 5-point target positions for a 112×112 ArcFace crop.
// Landmark order: right-eye, left-eye, nose, right-mouth, left-mouth
// (matches SCRFD output order — no reordering needed).
inline constexpr float kArcFaceRef[5][2] = {
    {38.2946f, 51.6963f},
    {73.5318f, 51.5014f},
    {56.0252f, 71.7366f},
    {41.5493f, 92.3655f},
    {70.7299f, 92.2041f},
};

// ── DetectedFace ──────────────────────────────────────────────────────────────
// One face found by SCRFD in a Frame.
// Landmark order matches ArcFace convention (same as SCRFD output order):
//   [0] right-eye-centre  [1] left-eye-centre  [2] nose
//   [3] right-mouth       [4] left-mouth
/// TRACES: AR-028 | SR-002
struct DetectedFace {
    cv::Rect2f                   bbox;
    std::array<cv::Point2f, 5>   landmarks;
    float                        confidence{0.f};

    // ── AR-028 quality vector ────────────────────────────────────────────────
    // Three axes, kept separate and never collapsed into one scalar: they fail
    // for different reasons, have different remedies, and do not earn the same
    // response. Carried, not consumed — the vector travels with the face into
    // the VR-001 dump so a threshold can be re-litigated against recorded data
    // rather than by re-running video.
    //
    // **Size is the third axis and is deliberately not a field here.** It is
    // `bbox`, which every consumer already has, scaled by the frame's
    // `bbox_upscale` to reach the original resolution AR-002 thresholds in.
    // Copying it into a second field would put the same quantity in two
    // coordinate spaces inside one struct — the trap SCHEMA.md records for
    // `bbox_upscale` — and the copy would be the one that drifts.
    //
    // Both fields below are -1 until the aligner runs, so *unscored* is
    // distinguishable from *scored badly*. Nothing downstream may read a
    // negative value as a quality.

    // AR-029 sharpness: normalised Laplacian variance over the aligned crop,
    // dimensionless. Falls with motion blur and soft focus; invariant to
    // contrast, and taken on the fixed 112×112 canvas so it cannot re-measure
    // face size. See crop_sharpness() for the construction and its one hazard.
    float                        sharpness{-1.f};

    // AR-030 visibility: RMS landmark misfit, in canonical 112×112 pixels, left
    // over after the best similarity fit to the ArcFace template. Rises with
    // out-of-plane pose and with occlusion; blind to in-plane roll and to face
    // size, both of which the fit absorbs. Set by the aligner, which is where
    // the transform is computed; -1 until then.
    float                        alignment_residual{-1.f};
};

// ── Pipeline messages ─────────────────────────────────────────────────────────

struct SceneFrame {
    Frame                     source;
    std::vector<DetectedFace> faces;   // empty when no faces detected (or eof)
};

struct AlignedSceneFrame {
    Frame                     source;
    std::vector<DetectedFace> faces;
    std::vector<cv::Mat>      crops;   // 112×112 BGR, ArcFace-ready; parallel to faces
};

struct EmbeddedSceneFrame {
    Frame                     source;
    std::vector<DetectedFace> faces;
    std::vector<cv::Mat>      crops;   // forwarded for debug rendering downstream
    std::vector<Embedding>    embeddings;
};

// ── Face tracking ─────────────────────────────────────────────────────────────
// Output of FaceTrackerFunc — EmbeddedSceneFrame augmented with per-detection
// track context.

struct TrackedSceneFrame {
    Frame                     source;
    std::vector<DetectedFace> faces;
    std::vector<cv::Mat>      crops;
    std::vector<int>          track_ids;          // -1 = brand-new track this frame
    std::vector<Embedding>    embeddings;          // per-frame raw (from embedder)
};

// ── Identity matching ─────────────────────────────────────────────────────────

struct IdentifiedActor {
    int         actor_idx{-1};   // index into ActorGallery::actors; -1 = unknown
    int         track_id{-1};    // face track ID from FaceTrackerFunc
    std::string name;
    std::string imdb_id;
    std::string tmdb_id;
    std::string jellyfin_id;     // Jellyfin Person item GUID, if gallery was built from Jellyfin
    float       similarity{0.f}; // calibrated P(match) or cosine similarity; 0 for unknowns
    cv::Rect2f  bbox;
    cv::Mat     crop;            // 112×112 aligned crop (stored as shared_ptr by KPN)
};

struct MatchedSceneFrame {
    Frame                        source;
    std::vector<IdentifiedActor> actors;  // includes unknowns (actor_idx == -1)
};

// ── Scene annotation ──────────────────────────────────────────────────────────
// Output of the scene tracker: one per sampled frame.
// visible_actors contains all actors still within their extinction window.
struct SceneAnnotation {
    double                       timestamp_sec{0.0};
    std::vector<IdentifiedActor> visible_actors;
    bool                         eof{false};
    // Carried through from Frame so the sink can collect boundaries for flood-fill
    // presence (PresenceMode::flood). is_cut is the always-on histogram cut
    // (camera_position_change_detector) — the boundary flood-fill uses by default.
    // is_scene_boundary is the opt-in TransNetV2 shot boundary (0 unless scene
    // detection ran); kept for a future out-of-process scene detector.
    bool                         is_cut{false};
    bool                         is_scene_boundary{false};
};

// ── Actor gallery ─────────────────────────────────────────────────────────────
// Loaded once at startup; baked into the identity matcher.
struct ActorGallery {
    struct Actor {
        std::string              imdb_id;
        std::string              tmdb_id;
        std::string              jellyfin_id;  // Jellyfin Person item GUID, if known
        std::string              name;
        std::vector<Embedding>   embeddings;  // one per reference image
        std::vector<std::string> source_images;
    };
    std::vector<Actor> actors;

    /// TRACES: GR-004 | SR-001
    // Which embedder produced every embedding above. Empty == the file predates
    // model binding; see gallery/embedder_stamp.hpp for what is checked and why.
    EmbedderStamp embedder;

    // Cached Platt-sigmoid calibration (see gallery/gallery_calibration.hpp),
    // stored alongside the gallery in HDF5 so it never needs recomputing
    // unless the reference embeddings actually change. calib_valid=false and
    // calib_hash=0 means "not present in this file, compute it."
    float    calib_a{10.f};
    float    calib_b{-5.f};
    bool     calib_valid{false};
    uint64_t calib_hash{0};
};

// ── Channel byte accounting ───────────────────────────────────────────────────
/// TRACES: AR-004 | SR-002
///
/// KPN measures a channel's occupancy in *items* and its bandwidth in bytes,
/// and gets the byte figure from `kpn::ChannelDataSize<T>`. That primary
/// template returns `sizeof(T)` — right for a POD, badly wrong for every type
/// below, each of which is a handful of vectors and a `cv::Mat` header owning
/// megabytes on the heap.
///
/// Unspecialised, the diagnostics reported roughly 200 bytes for a message
/// carrying a full decoded frame — off by four orders of magnitude at 1080p.
/// That is not merely a cosmetic stat: it is the one instrument for choosing
/// channel capacities against a memory ceiling, which is the open half of
/// AR-004, and it was reading fiction.
///
/// **What the number means.** `cv::Mat` is reference-counted, so one decoded
/// frame referenced from several messages is counted once per reference. The
/// sum is therefore an upper bound on distinct bytes, and the right bound for
/// the question being asked: how much would this channel keep alive if nothing
/// else held it.
///
/// Declared against a forward declaration rather than including
/// `<kpn/channel.hpp>` here, so the message definitions keep no dependency on
/// the framework that carries them — and so any translation unit that can see
/// these types also sees their sizes, which is what stops one channel being
/// instantiated with the default and another with the specialisation.

namespace kpn { template<typename T> struct ChannelDataSize; }

namespace sae::bytes {

inline std::size_t of(const cv::Mat& m) {
    return m.empty() ? 0u : m.total() * m.elemSize();
}
inline std::size_t of(const std::vector<cv::Mat>& v) {
    std::size_t n = 0;
    for (const auto& m : v) n += of(m);
    return n;
}
inline std::size_t of(const Frame& f) { return sizeof(Frame) + of(f.image); }

inline std::size_t of(const std::vector<IdentifiedActor>& v) {
    std::size_t n = v.size() * sizeof(IdentifiedActor);
    for (const auto& a : v) {
        n += of(a.crop);
        // The id strings are short but there is one set per actor per frame,
        // and a crowd frame carries dozens.
        n += a.name.capacity() + a.imdb_id.capacity()
           + a.tmdb_id.capacity() + a.jellyfin_id.capacity();
    }
    return n;
}

}  // namespace sae::bytes

template<> struct kpn::ChannelDataSize<Frame> {
    static std::size_t bytes(const Frame& f) { return sae::bytes::of(f); }
};

template<> struct kpn::ChannelDataSize<SceneFrame> {
    static std::size_t bytes(const SceneFrame& v) {
        return sizeof(SceneFrame) + sae::bytes::of(v.source)
             + v.faces.size() * sizeof(DetectedFace);
    }
};

template<> struct kpn::ChannelDataSize<AlignedSceneFrame> {
    static std::size_t bytes(const AlignedSceneFrame& v) {
        return sizeof(AlignedSceneFrame) + sae::bytes::of(v.source)
             + v.faces.size() * sizeof(DetectedFace)
             + sae::bytes::of(v.crops);
    }
};

template<> struct kpn::ChannelDataSize<EmbeddedSceneFrame> {
    static std::size_t bytes(const EmbeddedSceneFrame& v) {
        return sizeof(EmbeddedSceneFrame) + sae::bytes::of(v.source)
             + v.faces.size() * sizeof(DetectedFace)
             + sae::bytes::of(v.crops)
             + v.embeddings.size() * sizeof(Embedding);
    }
};

template<> struct kpn::ChannelDataSize<TrackedSceneFrame> {
    static std::size_t bytes(const TrackedSceneFrame& v) {
        return sizeof(TrackedSceneFrame) + sae::bytes::of(v.source)
             + v.faces.size() * sizeof(DetectedFace)
             + sae::bytes::of(v.crops)
             + v.track_ids.size() * sizeof(int)
             + v.embeddings.size() * sizeof(Embedding);
    }
};

template<> struct kpn::ChannelDataSize<MatchedSceneFrame> {
    static std::size_t bytes(const MatchedSceneFrame& v) {
        return sizeof(MatchedSceneFrame) + sae::bytes::of(v.source)
             + sae::bytes::of(v.actors);
    }
};

template<> struct kpn::ChannelDataSize<SceneAnnotation> {
    static std::size_t bytes(const SceneAnnotation& v) {
        return sizeof(SceneAnnotation) + sae::bytes::of(v.visible_actors);
    }
};

// CutEvent owns nothing on the heap, so the default sizeof(T) is already right.