diff --git a/src/audio_bindings.cpp b/src/audio_bindings.cpp new file mode 100644 index 0000000..f09e501 --- /dev/null +++ b/src/audio_bindings.cpp @@ -0,0 +1,125 @@ +// sae_audio — Python module wrapping the v1 audio signature (audio_signature.*). +// +/// TRACES: IR-004, IR-005 | SR-003 +// +// Exists so a study or a test can drive the **shipped** signature code from +// Python instead of porting the DSP to numpy. A numpy port would be a third +// implementation of a fingerprint that only works if every implementation +// agrees byte for byte, and it would be the one nobody checks against the +// golden vector — so the offset-recovery validation (VR-014) calls this. +// +// Bound with nanobind, as `sae_embed` and `sae_kpn` are. Not pybind11: a second +// binding framework in one build is a second set of ABI and lifetime rules to +// get right, for a module that needs nothing nanobind lacks. +// +// The module deliberately stops at the producer's edge. Matching — sliding one +// signature against another and scoring the overlap — is the *consumer's* +// algorithm (server SPEC §3, and the jRay plugin implements it), so it is not +// bound here and a caller writing a slide in numpy is not re-implementing +// anything this repo owns. + +#include "audio_signature.hpp" + +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include + +namespace nb = nanobind; +using namespace nb::literals; +using namespace sae::audio; + +namespace { + +using MonoArray = nb::ndarray, nb::c_contig, nb::device::cpu>; + +// Hand the vector's buffer to Python without copying 1.3 M samples, and let a +// capsule own it: the array outlives this call, so the storage has to as well. +nb::object own_as_ndarray(std::vector&& samples) { + auto* held = new std::vector(std::move(samples)); + nb::capsule owner(held, [](void* p) noexcept { + delete static_cast*>(p); + }); + const std::size_t n = held->size(); + return nb::cast(nb::ndarray>(held->data(), {n}, owner)); +} + +std::vector to_vector(const MonoArray& a) { + return std::vector(a.data(), a.data() + a.shape(0)); +} + +} // namespace + +NB_MODULE(sae_audio, m) { + m.doc() = + "JRay v1 audio signature (JRay-public-server SPEC.md section 3), as the " + "extraction pipeline computes it. The constants below are the contract: " + "changing any of them is a v1 -> v2 change."; + + m.attr("sample_rate") = kSampleRate; + m.attr("frame_size") = kFrameSize; + m.attr("hop_size") = kHopSize; + m.attr("num_bands") = kNumBands; + m.attr("band_lo_hz") = kBandLoHz; + m.attr("band_hi_hz") = kBandHiHz; + m.attr("window_sec") = kWindowSec; + m.attr("window_samples") = kWindowSamples; + m.attr("expected_frames") = kExpectedFrames; + m.attr("version_prefix") = std::string(kVersionPrefix); + + m.def( + "compute_signature", + [](const std::string& path) { return compute_signature(path); }, + "path"_a, + "Signature of the 120 s window centred on the media's midpoint, or None " + "for media shorter than the window (IR-007), media with no audio " + "stream, and any decode failure — degradation, never an exception."); + + m.def( + "decode_centre_window", + [](const std::string& path) -> nb::object { + std::optional> mono = decode_centre_window(path); + if (!mono) { + return nb::none(); + } + + return own_as_ndarray(std::move(*mono)); + }, + "path"_a, + "The decoded centre window as float32 mono at 11025 Hz, or None. Exposed " + "so a caller can slice or perturb real audio and re-sign it without " + "going back through a container."); + + m.def( + "signature_from_mono", + [](const MonoArray& mono) { return signature_from_mono(to_vector(mono)); }, + "mono"_a, + "Signature of mono float32 samples already at 11025 Hz, in [-1, 1). None " + "when fewer than one whole frame is given."); + + m.def( + "pack_frames", + [](const MonoArray& mono) { + std::vector packed = pack_frames(to_vector(mono)); + return nb::bytes(reinterpret_cast(packed.data()), packed.size()); + }, + "mono"_a, + "One packed byte per whole STFT frame: (band << 2) | energy_class. This " + "is the payload the signature base64-encodes."); + + m.def( + "band_fft_bins", + [] { + const auto& table = band_fft_bins(); + return std::vector>(table.begin(), table.end()); + }, + "The half-open FFT bin range owned by each of the 32 log-spaced bands."); +} diff --git a/src/evidence_discount.hpp b/src/evidence_discount.hpp index 10e7901..0fdc91f 100644 --- a/src/evidence_discount.hpp +++ b/src/evidence_discount.hpp @@ -41,7 +41,26 @@ public: struct Config { int max_views{8}; ///< distinct views remembered per track float admit_below{0.6f}; ///< P(same view) under this ⇒ a new view - float floor{0.0f}; ///< minimum weight for a redundant observation + + /// Ceiling on the correlation between two observations of one track. + /// + /// This is what bounds the accumulation. `n_eff = n / (1 + (n-1)·rho)` + /// tends to `1/rho` as `n` grows, so `rho_max` sets how much a single + /// repeated view can ever be worth: 0.5 caps it at two observations, + /// no matter how long the shot runs. + /// + /// 0.5 caps a repeated view at two independent observations' worth, + /// which is what lets a track the matcher accepts on frame after frame + /// actually become owned. Higher values starve ownership; the sweep + /// (VR-007) decides where it belongs. + /// + /// It is capped below 1 deliberately. P(same view) near 1 says the two + /// crops look alike; it does not say the second carries no information. + /// A fresh frame is a fresh detection, a fresh alignment and a fresh + /// noise realisation, so a little independent evidence survives even a + /// perfectly held pose. Setting this to 1 recovers the original bug — + /// belief frozen after the first frame. + float rho_max{0.5f}; }; // Two constructors rather than a defaulted argument: `Config{}` as a default @@ -53,12 +72,36 @@ public: EvidenceDiscounter(Calibrate cal, Config cfg) : cal_(std::move(cal)), cfg_(cfg) {} - /// Weight in [0,1] for one observation, updating `views` when the - /// observation is novel enough to count as a distinct look at the subject. + /// The marginal evidence one observation adds, in units of independent + /// observations. /// - /// The first observation on a track always counts in full: there is nothing - /// for it to be redundant with. - float weight(std::vector& views, const Embedding& e) const { + /// Each frame is a Bayesian update, so confidence must keep growing — but + /// correlated observations must grow it less, and must not grow it without + /// bound. The standard treatment is **effective sample size**: + /// + /// n_eff(n) = n / (1 + (n-1)·rho) + /// + /// and this returns `n_eff(n) - n_eff(n-1)`, the gain from *this* frame. + /// The shape is right at both ends: with rho = 0 every frame counts fully + /// and the belief accumulates linearly, while as rho rises the series + /// converges on `1/rho` and a held pose stops adding no matter how long it + /// is held. + /// + /// The two failure modes it sits between are both real and both were hit: + /// a weight of 0 for repeats froze the belief after one frame, so a track + /// recognised on 318 frames was owned on none; a constant floor grew it + /// linearly forever, so a long shot could out-argue genuinely varied + /// evidence purely by lasting longer. + /// + /// `rho` is estimated from P(same view) against the closest stored view, + /// capped by `rho_max`. The first observation has nothing to be redundant + /// with and counts in full. + /// `n_seen` is the count of observations already folded into THIS track. + /// It is a parameter rather than discounter state because one discounter + /// serves every track: holding the count internally would pool unrelated + /// tracks into one effective sample, so a busy film would silently discount + /// each track by how many others happened to be on screen. + float weight(std::vector& views, int n_seen, const Embedding& e) const { if (views.empty()) { views.push_back(e); return 1.0f; @@ -68,9 +111,12 @@ public: for (const auto& v : views) p_same = std::max(p_same, cal_(cosine_similarity(v, e))); - // Weight is the probability this is *not* a repeat of something already - // counted. A near-duplicate contributes ~0; an unseen pose ~1. - const float w = std::max(cfg_.floor, 1.0f - p_same); + const float rho = std::min(cfg_.rho_max, std::max(0.0f, p_same)); + + const float n_prev = static_cast(std::max(1, n_seen)); + const float n_now = n_prev + 1.0f; + auto n_eff = [rho](float n) { return n / (1.0f + (n - 1.0f) * rho); }; + const float w = std::max(0.0f, n_eff(n_now) - n_eff(n_prev)); if (p_same < cfg_.admit_below && static_cast(views.size()) < cfg_.max_views) { diff --git a/src/face_embedder_engine.hpp b/src/face_embedder_engine.hpp index 28298ff..0ef2996 100644 --- a/src/face_embedder_engine.hpp +++ b/src/face_embedder_engine.hpp @@ -123,6 +123,14 @@ public: Embedding embed_crop(const cv::Mat& crop) { return embedder_->embed_one(crop); } + // Batched form. A study embedding thousands of crops one at a time pays the + // per-call overhead thousands of times over; the backend already batches. + std::vector embed_crops(const std::vector& crops) { + return embedder_->embed(crops); + } + + int max_batch() const { return embedder_->max_batch(); } + private: std::unique_ptr detector_; std::unique_ptr embedder_; diff --git a/src/gallery/gallery_store.hpp b/src/gallery/gallery_store.hpp index 4566811..c46adf5 100644 --- a/src/gallery/gallery_store.hpp +++ b/src/gallery/gallery_store.hpp @@ -6,6 +6,14 @@ // legacy gallery.json files are still readable for backward compatibility but // save_gallery always writes HDF5 regardless of the requested extension. // +// GR-005 is preserved here by absence: this is the only path that serialises a +// gallery, and it reads and writes the local filesystem only. There is no +// upload, no client, and no encoder that could put an embedding on a wire — the +// public server refuses to carry one (SR-004/UR-012), and the prohibition holds +// on this side by there being nothing that would try. +// +/// TRACES: GR-005 | SR-005 +// // HDF5 layout: // /embeddings float32 [N, 512] all actors' refs concatenated, row-major // /offset int64 [A] first row of actor a in /embeddings diff --git a/src/main.cpp b/src/main.cpp index 603c4fd..925d082 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -227,7 +227,7 @@ int main(int argc, char** argv) { SceneTrackerFunc tracker_fn {cfg}; ResultSinkFunc sink_fn {cfg, done}; - /// TRACES: AR-012, AR-016, IR-002, IR-003 | SR-002 + /// TRACES: AR-012, AR-016 | IR-002, IR-003 | SR-002 // A reaped track goes straight to the aggregator, so the registry holds only // live tracks and its size is bounded by concurrent on-screen faces rather // than growing with the film. diff --git a/src/nodes/camera_position_change_detector_node.hpp b/src/nodes/camera_position_change_detector_node.hpp index b8b6087..7f3704d 100644 --- a/src/nodes/camera_position_change_detector_node.hpp +++ b/src/nodes/camera_position_change_detector_node.hpp @@ -26,7 +26,8 @@ // The node is a pure pass-through: it forwards the Frame unchanged except for // is_cut, so it slots between frame_source and face_detector without altering the // downstream contract. eof frames are forwarded immediately without processing. - +// +/// TRACES: AR-009 | SR-002 struct CameraPositionChangeDetectorFunc { static constexpr std::string_view label() { return "camera_position_change_detector"; } diff --git a/src/nodes/embedder_node.hpp b/src/nodes/embedder_node.hpp index 2ef1b87..e623c57 100644 --- a/src/nodes/embedder_node.hpp +++ b/src/nodes/embedder_node.hpp @@ -17,7 +17,8 @@ // All crops in one frame are batched into a single forward pass (capped at // embed_batch_size). The backend serialises itself; we only call it from the // single embedder thread. - +// +/// TRACES: AR-006 | SR-002 struct EmbedderFunc { static constexpr std::string_view label() { return "embedder"; } diff --git a/src/nodes/result_sink_node.hpp b/src/nodes/result_sink_node.hpp index 419e3ff..10f2026 100644 --- a/src/nodes/result_sink_node.hpp +++ b/src/nodes/result_sink_node.hpp @@ -46,7 +46,7 @@ using json = nlohmann::json; struct ResultSinkFunc { static constexpr std::string_view label() { return "result_sink"; } - /// TRACES: AR-012, AR-017, IR-002 | SR-002, SR-003 + /// TRACES: AR-012, AR-017 | IR-002 | SR-002, SR-003 /// A finished presence claim from the registry. Called from inside the /// registry's reap while it holds its own lock, so this must stay a cheap /// push and must never re-enter the registry. @@ -158,7 +158,7 @@ private: struct ActorMeta { std::string name, imdb_id, tmdb_id, jellyfin_id; }; // Core logic: merge per-frame detections into annealed [start, end] windows. - /// TRACES: AR-012, IR-002 | SR-002 + /// TRACES: AR-012 | IR-002 | SR-002 /// A claim already IS a window — `[first_seen, last_seen]` of a track the /// actor owned. There is no annealing pass: `anneal_sec` existed to bridge /// gaps between isolated accepted frames, and a track that survives its own diff --git a/src/python_bindings.cpp b/src/python_bindings.cpp index 327b597..c3d9532 100644 --- a/src/python_bindings.cpp +++ b/src/python_bindings.cpp @@ -133,7 +133,30 @@ NB_MODULE(sae_embed, m) { return vec_to_numpy(std::vector(emb.begin(), emb.end())); }, "crop"_a, "Embed a caller-supplied 112x112 aligned BGR crop. The stage-level " - "entry point for studies that degrade or re-align a crop themselves."); + "entry point for studies that degrade or re-align a crop themselves.") + .def("embed_crops", [](FaceEmbedderEngine& e, + nb::ndarray, nb::c_contig, + nb::device::cpu> crops) { + if (crops.shape(1) != 112 || crops.shape(2) != 112 || crops.shape(3) != 3) + throw std::invalid_argument("embed_crops expects (N,112,112,3) uint8 BGR"); + const size_t n = crops.shape(0); + std::vector mats; + mats.reserve(n); + for (size_t i = 0; i < n; ++i) + mats.emplace_back(112, 112, CV_8UC3, + const_cast(crops.data()) + i * 112 * 112 * 3); + std::vector out = e.embed_crops(mats); + auto* buf = new float[n * 512]; + for (size_t i = 0; i < n; ++i) + std::memcpy(buf + i * 512, out[i].data(), 512 * sizeof(float)); + nb::capsule owner(buf, [](void* p) noexcept { delete[] static_cast(p); }); + size_t shape[2] = {n, 512}; + return nb::ndarray(buf, 2, shape, owner); + }, "crops"_a, + "Batched embed_crop: (N,112,112,3) uint8 BGR in, (N,512) float32 out. " + "The backend batches internally, so this avoids paying per-call " + "overhead once per crop across a large study.") + .def_prop_ro("max_batch", [](FaceEmbedderEngine& e) { return e.max_batch(); }); m.def("align_face", [](ImageArray img, nb::ndarray, nb::c_contig, diff --git a/src/track_registry.hpp b/src/track_registry.hpp index f6e78cd..eb23dc9 100644 --- a/src/track_registry.hpp +++ b/src/track_registry.hpp @@ -60,7 +60,13 @@ struct Track { double first_seen{0.0}; std::optional last_seen; ///< unset ⇒ on screen std::optional actor; ///< set once a posterior crosses - std::map belief; ///< actor_idx → accumulated log-odds + /// actor_idx → accumulated log(1 − P). Lazy-OR (noisy-OR) accumulation: + /// each frame is new evidence that this track is that actor, and the + /// combined belief is the probability that *at least one* sighting was + /// right. Stored as log(1−P) because that makes the update additive and + /// keeps precision where it matters — as P approaches 1, (1−P) is the + /// quantity with the significant digits. + std::map belief; Embedding mean{}; ///< running directional mean std::vector views; ///< distinct looks, for AR-025 discounting float discounted_weight{0.f}; ///< sum of applied weights @@ -146,14 +152,20 @@ public: if (it == tracks_.end()) { ++dropped_votes_; return; } Track& t = it->second; - const float w = discounter_.weight(t.views, e); - t.belief[actor_idx] += w * logit(posterior); + const float w = discounter_.weight(t.views, t.n_obs, e); + + // Weighted lazy-OR: P_new = 1 − (1 − P_old)·(1 − p)^w, which in log + // space is a plain sum. w is the discounted evidence (AR-025), so a + // repeated view still advances the belief but by a fraction of what a + // genuinely new look would. + const float p = std::min(1.f - 1e-6f, std::max(1e-6f, posterior)); + t.belief[actor_idx] += w * std::log(1.f - p); t.discounted_weight += w; ++t.n_obs; - const int best = argmax_belief(t); - const float best_lo = t.belief[best]; - if (best_lo < cfg_.ownership_logodds) return; + const int best = argmax_belief(t); + const float best_p = 1.f - std::exp(t.belief[best]); + if (best_p < own_threshold()) return; if (!t.actor.has_value()) { claim_locked(t, best); @@ -285,20 +297,28 @@ private: d.effective_obs = t.discounted_weight; if (t.actor) { d.actor_idx = *t.actor; - d.belief = logistic(t.belief[*t.actor]); + d.belief = 1.f - std::exp(t.belief[*t.actor]); auto oi = owner_index_.find(*t.actor); if (oi != owner_index_.end() && oi->second == t.id) owner_index_.erase(oi); } on_dead_(d); } + /// Most-believed actor. belief holds log(1 − P), so the strongest claim is + /// the *most negative* entry, not the largest. static int argmax_belief(const Track& t) { int best = -1; - float hi = -1e30f; - for (const auto& [a, lo] : t.belief) if (lo > hi) { hi = lo; best = a; } + float lo = 1e30f; + for (const auto& [a, v] : t.belief) if (v < lo) { lo = v; best = a; } return best; } + /// Ownership expressed as a probability. Config still carries log-odds so + /// the knob keeps its meaning across this change. + float own_threshold() const { + return 1.f / (1.f + std::exp(-cfg_.ownership_logodds)); + } + static void update_mean(Track& t, const Embedding& e) { // Directional mean: accumulate then re-normalise to the unit sphere, so // cosine against it stays a plain dot product. diff --git a/tests/fixtures/audio/bali_offset_200s.flac b/tests/fixtures/audio/bali_offset_200s.flac new file mode 100644 index 0000000..d1f6b77 Binary files /dev/null and b/tests/fixtures/audio/bali_offset_200s.flac differ diff --git a/tests/fixtures/audio/make_offset_fixture.sh b/tests/fixtures/audio/make_offset_fixture.sh new file mode 100755 index 0000000..ae19614 --- /dev/null +++ b/tests/fixtures/audio/make_offset_fixture.sh @@ -0,0 +1,60 @@ +#!/bin/sh +# +# Regenerate bali_offset_200s.flac — the real-audio fixture behind VR-014, the +# audio-signature offset-recovery validation. +# +# sh make_offset_fixture.sh /path/to/clips +# +# Why real audio and not a second synthetic tone: jray_audio_v1_tone.flac pins +# the *arithmetic* (IR-005) and is deliberately built so every band and every +# energy class appears. It cannot answer the question VR-014 asks — whether the +# peak-bin sequence of ordinary film audio is distinctive enough that sliding +# one signature against another finds the true alignment and only the true +# alignment. Tones are pathologically easy for that; dialogue and score are not. +# +# Source: five scene clips from "Road to Bali" (1952), the public-domain corpus +# this repo already uses for the replay fixtures — tests/fixtures/dumps/bali_*.h5 +# are dumps of these same clips. Each is under the 120 s window on its own +# (29-77 s), so they are concatenated in scene order to make a source long +# enough that a 120 s window can slide inside it. +# +# 200 s is chosen, not arbitrary: the window is 120 s and the match search is +# capped at +/-600 frames (~55.7 s), so a source of 120 + 56 s is the shortest +# one that can place two windows at the edge of the cap. The 200 s here leaves +# room to go past it as well, which is what lets the test check that an +# out-of-range offset is declined rather than guessed. +# +# Encoded mono at 11025 Hz, 16-bit, which is exactly what the signature decodes +# to anyway. That keeps a 200 s fixture at ~2.4 MB instead of ~20 MB, and makes +# every trim below sample-exact — the test measures offset recovery, not the +# resampler, which tests/test_audio_signature.cpp already covers (UT-103). +# +# FLAC because it is lossless: the decoded PCM is the same on every machine, so +# a signature computed from this file is reproducible. A lossy fixture would +# make the measurement depend on the decoder version. +# +# sha256 of the committed file: +# 4a952e46a090a9acd9eae56996250ec03e08e0d04ee139ac0a42f1690a536c83 +# A regenerated file that hashes differently means the source clips or the +# encoder changed, and VR-014's recorded numbers should be re-measured — the +# offsets will still be exact, but the scores are this audio's. + +set -eu + +CLIPS="${1:-../../../../bali}" +OUT="$(dirname "$0")/bali_offset_200s.flac" +LIST="$(mktemp)" +trap 'rm -f "$LIST"' EXIT + +for scene in 13 27 28 31 46; do + clip="$CLIPS/Road_To_Bali-$scene.webm" + [ -f "$clip" ] || { echo "missing clip: $clip" >&2; exit 1; } + echo "file '$(cd "$(dirname "$clip")" && pwd)/$(basename "$clip")'" >> "$LIST" +done + +ffmpeg -nostdin -v error -y -f concat -safe 0 -i "$LIST" \ + -vn -t 200 -ac 1 -ar 11025 -sample_fmt s16 \ + -c:a flac -compression_level 12 "$OUT" + +echo "wrote $OUT" +sha256sum "$OUT" 2>/dev/null || shasum -a 256 "$OUT" diff --git a/tests/test_replay_fixtures.cpp b/tests/test_replay_fixtures.cpp index 77704fd..325d85c 100644 --- a/tests/test_replay_fixtures.cpp +++ b/tests/test_replay_fixtures.cpp @@ -1,6 +1,6 @@ // Replay tests — the real tracker and registry driven from committed fixtures. // -// TRACES: AR-012, AR-013, AR-004, VR-001, VR-002 | IT-001 +// TRACES: AR-004, AR-012, AR-013 | VR-001, VR-002 | IT-001 // // Tier T2: composition, not units. The registry tests construct awkward states // directly; these check that the pieces behave when wired together and fed real diff --git a/tests/test_track_registry.cpp b/tests/test_track_registry.cpp index 20d54d0..980749a 100644 --- a/tests/test_track_registry.cpp +++ b/tests/test_track_registry.cpp @@ -330,3 +330,43 @@ TEST_CASE("the registry takes a probability, not a cosine", "[registry][AR-024]" REQUIRE(sink.claims.size() == 1); CHECK(sink.claims[0].actor_idx == -1); // never owned } + +// ── AR-025 — repeated evidence must GROW confidence, not cap it ────────────── +TEST_CASE("confidence grows across frames of the same face", "[registry][AR-025]") { + // Found on a real clip: 318 frame-level identifications across 385 frames + // produced ZERO owned tracks. The truth file named nobody while the matcher + // was accepting on most frames. + // + // Cause: the correlation discount was an annihilator rather than an + // attenuator. Weight = 1 - P(same view), so once a track had one stored + // view every later frame of that same face scored ~0.01 and belief stopped + // moving. A single observation just over the accept threshold is + // logit(0.78) ~ 1.27, under the ownership bar — recognised every frame, + // owned on none. + // + // Correlated evidence should accumulate SLOWER than independent evidence, + // never stop accumulating. Each frame is a Bayesian update. + TrackRegistry reg(cfg(), disc()); + Sink sink; sink.attach(reg); + + int id; + { auto f = reg.begin_frame(0.0); id = f.create(0.0, axis(0)); } + + // A face held on screen: the same person, the same pose, frame after frame. + for (int i = 0; i < 50; ++i) { + // The frame scope must close before observe(): it holds the registry + // lock for its lifetime and the mutex is not recursive, so observing + // inside the scope self-deadlocks. In the pipeline these are separate + // nodes, so the ordering falls out naturally — but the API allows the + // mistake, and it hangs rather than failing. + { auto f = reg.begin_frame(i * 0.2); f.mark_seen(id, i * 0.2, axis(0)); } + reg.observe(id, 5, 0.78f, axis(0)); + } + reg.flush(20.0); + + REQUIRE(sink.claims.size() == 1); + CHECK(sink.claims[0].actor_idx == 5); + + // ...but it must still be worth far less than 50 independent looks would be. + CHECK(sink.claims[0].effective_obs < 25.0f); +}