#pragma once #include #include #include #include #include #include "gallery/embedder_stamp.hpp" // ── Embedding ───────────────────────────────────────────────────────────────── // 512-dim L2-normalised ArcFace embedding using Embedding = std::array; inline float cosine_similarity(const Embedding& a, const Embedding& b) { float dot = 0.f; for (int i = 0; i < 512; ++i) dot += a[i] * b[i]; return dot; } // ── Frame ───────────────────────────────────────────────────────────────────── // Raw sampled frame from the movie. eof=true is the pipeline shutdown sentinel: // every node must forward it immediately without processing. struct Frame { cv::Mat image; double timestamp_sec{0.0}; int64_t frame_idx{-1}; bool eof{false}; bool is_cut{false}; // histogram: intra-scene camera-angle change (tracker reset) bool is_scene_boundary{false}; // TransNetV2: true shot/scene boundary (opt-in) float cut_score{0.f}; // histogram cut score = 1 - hist_corr (0=identical, ~1=cut); HUD/debug float bbox_upscale{1.f}; // multiply detector bboxes/landmarks by this to map back to // original video resolution (>1 when dense_scale downscaled the frame) }; // ── CutEvent ────────────────────────────────────────────────────────────────── // Emitted by SceneDetectorFunc when TransNetV2 localises a shot boundary, keyed // by the boundary frame's timestamp. eof=true is the shutdown sentinel. struct CutEvent { double timestamp_sec{0.0}; float probability{0.f}; // sigmoid boundary score at the peak bool eof{false}; }; // ── ArcFace alignment ───────────────────────────────────────────────────────── // Canonical 5-point target positions for a 112×112 ArcFace crop. // Landmark order: right-eye, left-eye, nose, right-mouth, left-mouth // (matches SCRFD output order — no reordering needed). inline constexpr float kArcFaceRef[5][2] = { {38.2946f, 51.6963f}, {73.5318f, 51.5014f}, {56.0252f, 71.7366f}, {41.5493f, 92.3655f}, {70.7299f, 92.2041f}, }; // ── DetectedFace ────────────────────────────────────────────────────────────── // One face found by SCRFD in a Frame. // Landmark order matches ArcFace convention (same as SCRFD output order): // [0] right-eye-centre [1] left-eye-centre [2] nose // [3] right-mouth [4] left-mouth /// TRACES: AR-028 | SR-002 struct DetectedFace { cv::Rect2f bbox; std::array landmarks; float confidence{0.f}; // ── AR-028 quality vector ──────────────────────────────────────────────── // Three axes, kept separate and never collapsed into one scalar: they fail // for different reasons, have different remedies, and do not earn the same // response. Carried, not consumed — the vector travels with the face into // the VR-001 dump so a threshold can be re-litigated against recorded data // rather than by re-running video. // // **Size is the third axis and is deliberately not a field here.** It is // `bbox`, which every consumer already has, scaled by the frame's // `bbox_upscale` to reach the original resolution AR-002 thresholds in. // Copying it into a second field would put the same quantity in two // coordinate spaces inside one struct — the trap SCHEMA.md records for // `bbox_upscale` — and the copy would be the one that drifts. // // Both fields below are -1 until the aligner runs, so *unscored* is // distinguishable from *scored badly*. Nothing downstream may read a // negative value as a quality. // AR-029 sharpness: normalised Laplacian variance over the aligned crop, // dimensionless. Falls with motion blur and soft focus; invariant to // contrast, and taken on the fixed 112×112 canvas so it cannot re-measure // face size. See crop_sharpness() for the construction and its one hazard. float sharpness{-1.f}; // AR-030 visibility: RMS landmark misfit, in canonical 112×112 pixels, left // over after the best similarity fit to the ArcFace template. Rises with // out-of-plane pose and with occlusion; blind to in-plane roll and to face // size, both of which the fit absorbs. Set by the aligner, which is where // the transform is computed; -1 until then. float alignment_residual{-1.f}; }; // ── Pipeline messages ───────────────────────────────────────────────────────── struct SceneFrame { Frame source; std::vector faces; // empty when no faces detected (or eof) }; struct AlignedSceneFrame { Frame source; std::vector faces; std::vector crops; // 112×112 BGR, ArcFace-ready; parallel to faces }; struct EmbeddedSceneFrame { Frame source; std::vector faces; std::vector crops; // forwarded for debug rendering downstream std::vector embeddings; }; // ── Face tracking ───────────────────────────────────────────────────────────── // Output of FaceTrackerFunc — EmbeddedSceneFrame augmented with per-detection // track context. struct TrackedSceneFrame { Frame source; std::vector faces; std::vector crops; std::vector track_ids; // -1 = brand-new track this frame std::vector embeddings; // per-frame raw (from embedder) }; // ── Identity matching ───────────────────────────────────────────────────────── struct IdentifiedActor { int actor_idx{-1}; // index into ActorGallery::actors; -1 = unknown int track_id{-1}; // face track ID from FaceTrackerFunc std::string name; std::string imdb_id; std::string tmdb_id; std::string jellyfin_id; // Jellyfin Person item GUID, if gallery was built from Jellyfin float similarity{0.f}; // calibrated P(match) or cosine similarity; 0 for unknowns cv::Rect2f bbox; cv::Mat crop; // 112×112 aligned crop (stored as shared_ptr by KPN) }; struct MatchedSceneFrame { Frame source; std::vector actors; // includes unknowns (actor_idx == -1) }; // ── Scene annotation ────────────────────────────────────────────────────────── // Output of the scene tracker: one per sampled frame. // visible_actors contains all actors still within their extinction window. struct SceneAnnotation { double timestamp_sec{0.0}; std::vector visible_actors; bool eof{false}; }; // ── Actor gallery ───────────────────────────────────────────────────────────── // Loaded once at startup; baked into the identity matcher. struct ActorGallery { struct Actor { std::string imdb_id; std::string tmdb_id; std::string jellyfin_id; // Jellyfin Person item GUID, if known std::string name; std::vector embeddings; // one per reference image std::vector source_images; }; std::vector actors; /// TRACES: GR-004 | SR-001 // Which embedder produced every embedding above. Empty == the file predates // model binding; see gallery/embedder_stamp.hpp for what is checked and why. EmbedderStamp embedder; // Cached Platt-sigmoid calibration (see gallery/gallery_calibration.hpp), // stored alongside the gallery in HDF5 so it never needs recomputing // unless the reference embeddings actually change. calib_valid=false and // calib_hash=0 means "not present in this file, compute it." float calib_a{10.f}; float calib_b{-5.f}; bool calib_valid{false}; uint64_t calib_hash{0}; }; // ── Channel byte accounting ─────────────────────────────────────────────────── /// TRACES: AR-004 | SR-002 /// /// KPN measures a channel's occupancy in *items* and its bandwidth in bytes, /// and gets the byte figure from `kpn::ChannelDataSize`. That primary /// template returns `sizeof(T)` — right for a POD, badly wrong for every type /// below, each of which is a handful of vectors and a `cv::Mat` header owning /// megabytes on the heap. /// /// Unspecialised, the diagnostics reported roughly 200 bytes for a message /// carrying a full decoded frame — off by four orders of magnitude at 1080p. /// That is not merely a cosmetic stat: it is the one instrument for choosing /// channel capacities against a memory ceiling, which is the open half of /// AR-004, and it was reading fiction. /// /// **What the number means.** `cv::Mat` is reference-counted, so one decoded /// frame referenced from several messages is counted once per reference. The /// sum is therefore an upper bound on distinct bytes, and the right bound for /// the question being asked: how much would this channel keep alive if nothing /// else held it. /// /// Declared against a forward declaration rather than including /// `` here, so the message definitions keep no dependency on /// the framework that carries them — and so any translation unit that can see /// these types also sees their sizes, which is what stops one channel being /// instantiated with the default and another with the specialisation. namespace kpn { template struct ChannelDataSize; } namespace sae::bytes { inline std::size_t of(const cv::Mat& m) { return m.empty() ? 0u : m.total() * m.elemSize(); } inline std::size_t of(const std::vector& v) { std::size_t n = 0; for (const auto& m : v) n += of(m); return n; } inline std::size_t of(const Frame& f) { return sizeof(Frame) + of(f.image); } inline std::size_t of(const std::vector& v) { std::size_t n = v.size() * sizeof(IdentifiedActor); for (const auto& a : v) { n += of(a.crop); // The id strings are short but there is one set per actor per frame, // and a crowd frame carries dozens. n += a.name.capacity() + a.imdb_id.capacity() + a.tmdb_id.capacity() + a.jellyfin_id.capacity(); } return n; } } // namespace sae::bytes template<> struct kpn::ChannelDataSize { static std::size_t bytes(const Frame& f) { return sae::bytes::of(f); } }; template<> struct kpn::ChannelDataSize { static std::size_t bytes(const SceneFrame& v) { return sizeof(SceneFrame) + sae::bytes::of(v.source) + v.faces.size() * sizeof(DetectedFace); } }; template<> struct kpn::ChannelDataSize { static std::size_t bytes(const AlignedSceneFrame& v) { return sizeof(AlignedSceneFrame) + sae::bytes::of(v.source) + v.faces.size() * sizeof(DetectedFace) + sae::bytes::of(v.crops); } }; template<> struct kpn::ChannelDataSize { static std::size_t bytes(const EmbeddedSceneFrame& v) { return sizeof(EmbeddedSceneFrame) + sae::bytes::of(v.source) + v.faces.size() * sizeof(DetectedFace) + sae::bytes::of(v.crops) + v.embeddings.size() * sizeof(Embedding); } }; template<> struct kpn::ChannelDataSize { static std::size_t bytes(const TrackedSceneFrame& v) { return sizeof(TrackedSceneFrame) + sae::bytes::of(v.source) + v.faces.size() * sizeof(DetectedFace) + sae::bytes::of(v.crops) + v.track_ids.size() * sizeof(int) + v.embeddings.size() * sizeof(Embedding); } }; template<> struct kpn::ChannelDataSize { static std::size_t bytes(const MatchedSceneFrame& v) { return sizeof(MatchedSceneFrame) + sae::bytes::of(v.source) + sae::bytes::of(v.actors); } }; template<> struct kpn::ChannelDataSize { static std::size_t bytes(const SceneAnnotation& v) { return sizeof(SceneAnnotation) + sae::bytes::of(v.visible_actors); } }; // CutEvent owns nothing on the heap, so the default sizeof(T) is already right.