diff --git a/docs/requirements.md b/docs/requirements.md index 44d3efd..ea2037d 100644 --- a/docs/requirements.md +++ b/docs/requirements.md @@ -101,7 +101,7 @@ Status: `Done` · `In Progress` · `Planned` · `TBD` · `Withdrawn` | ID | Requirement | Traces to | Priority | Status | |---|---|---|---|---| | VR-001 | HDF5 post-inference dump at the embedded-frame boundary | PR-002 | High | Done | -| VR-002 | Replay drives the **real** KPN nodes, not a reimplementation | PR-002 | High | Done | +| VR-002 | Replay drives the **real** KPN nodes, not a reimplementation | PR-002 | High | **Done** — replay driven from committed fixtures in `tests/test_replay_fixtures.cpp`; determinism asserted | | VR-003 | Scoring: micro-F1 against X-Ray, precision/recall logged at every evaluation | PR-002 | High | Done | | VR-004 | Reproducible validation corpus with ground truth | PR-002 | High | Done | | VR-005 | Minimum face size study — TPI/FPI vs probe size, gallery held at native res | PR-002 | Medium | **Done** — knee at 24–32 px; 32 px gives 98.1% TPI, 0.0 FPI at every size | diff --git a/docs/traceability.md b/docs/traceability.md index 1259108..b75c5f4 100644 --- a/docs/traceability.md +++ b/docs/traceability.md @@ -3,7 +3,7 @@ -**Generated:** 2026-07-31T08:10:51+00:00 +**Generated:** 2026-07-31T08:35:29+00:00 Denominators are read from [`requirements.md`](requirements.md) at run time, never hardcoded. Coverage counts a requirement only when it is tagged in source **and** has a verification tier this repo's CI host can execute (`T1, T2, T3, static`). @@ -78,9 +78,9 @@ _None._ | ID | Status | Tier | Traces to | Trace state | Tagged in | Requirement | |---|---|---|---|---|---|---| | AR-001 | Done | T3 | SR-002 | covered | `src/nodes/face_detector_node.hpp` | Detect faces in sampled frames; emit bbox, confidence, 5-point landma… | -| AR-002 | Planned | T2 | SR-002 | untagged | - | Minimum face size 66×66 px, expressed in **original** resolution (dec… | +| AR-002 | Planned | unset | SR-002 | untagged | - | Minimum face size **32×32 px** (VR-005 measured), expressed in **orig… | | AR-003 | Planned | T1, T2, T4 | SR-002 | untagged | - | No fixed per-frame face cap — crowd scenes must not lose background c… | -| AR-004 | Planned | T1, T4 | SR-002 | untagged | - | Backpressure: unbounded faces/frame absorbed by slowing, never by dro… | +| AR-004 | **Done** — KPN node… | T1, T4 | SR-002 | untagged | - | Backpressure: unbounded faces/frame absorbed by slowing, never by dro… | | AR-005 | Done | T1, T3 | SR-002 | covered | `src/face_utils.hpp` | Align to 112×112 via ArcFace 5-point similarity transform | | AR-006 | Done | T3 | SR-002 | untagged | - | 512-d L2-normalised embeddings, batched | | AR-007 | **Done** — `track_a… | T2 | SR-002 | covered | `src/config.hpp`, `src/main.cpp`, `src/nodes/face_tracker_node.hpp` | Associate detections by IoU + embedding, with **frame-dependent** wei… | @@ -133,7 +133,7 @@ _None._ | VR-002 | Done | out-of-ci | PR-002 | tagged, unexecuted | `scripts/optimizer/replay.py` | Replay drives the **real** KPN nodes, not a reimplementation | | VR-003 | Done | out-of-ci | PR-002 | tagged, unexecuted | `scripts/optimizer/second_score.py` | Scoring: micro-F1 against X-Ray, precision/recall logged at every eva… | | VR-004 | Done | out-of-ci | PR-002 | untagged | - | Reproducible validation corpus with ground truth | -| VR-005 | Planned | out-of-ci | PR-002 | untagged | - | Minimum face size study — TPI/FPI vs probe size, gallery held at nati… | +| VR-005 | **Done** — knee at … | out-of-ci | PR-002 | untagged | - | Minimum face size study — TPI/FPI vs probe size, gallery held at nati… | | VR-006 | Planned | out-of-ci | PR-002 | untagged | - | Re-tune `scene_threshold` once native-rate decode lands | | VR-007 | Planned | out-of-ci | PR-002 | untagged | - | Expansion band, clustering threshold, and deferred-pass ablation | | VR-008 | Planned | out-of-ci | PR-002 | untagged | - | Gallery scaling benchmark — throughput vs gallery size | diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 1099c79..807dac8 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -22,6 +22,7 @@ add_executable(sae_tests test_track_gallery.cpp test_face_tracker.cpp test_track_registry.cpp + test_replay_fixtures.cpp test_audio_signature.cpp ${CMAKE_SOURCE_DIR}/src/backends/gemm_backend.cpp ${CMAKE_SOURCE_DIR}/src/gallery/gallery_store.cpp diff --git a/tests/test_replay_fixtures.cpp b/tests/test_replay_fixtures.cpp new file mode 100644 index 0000000..77704fd --- /dev/null +++ b/tests/test_replay_fixtures.cpp @@ -0,0 +1,258 @@ +// Replay tests — the real tracker and registry driven from committed fixtures. +// +// TRACES: AR-012, AR-013, AR-004, VR-001, VR-002 | IT-001 +// +// Tier T2: composition, not units. The registry tests construct awkward states +// directly; these check that the pieces behave when wired together and fed real +// footage — 480x360 public-domain clips at 5 fps, with the cuts, gaps and +// crowded frames that actual film produces and synthetic input does not. +// +// No GPU and no model: the fixtures are HDF5 dumps taken after embedding, so +// everything here is CPU maths. That is what lets this run on the CI host at +// all (see docs/requirements.md, "CI never calls a model"). +// +// Driving the node functors directly rather than through a KPN network is +// deliberate: functors are plain objects, so there are no threads, no channels +// and no scheduling — the same input gives the same output every time, which is +// exactly what a fixture-based test needs. +#include + +#include "config.hpp" +#include "evidence_discount.hpp" +#include "nodes/face_tracker_node.hpp" +#include "track_registry.hpp" +#include "types.hpp" + +#include + +#include +#include +#include +#include +#include + +namespace { + +// ── Fixture reader ─────────────────────────────────────────────────────────── +// The flat/ragged layout of scripts/optimizer/SCHEMA.md: per-face arrays +// concatenated, with a per-frame index table pointing into them. +struct Dump { + std::vector ts; + std::vector is_cut; + std::vector face_offset; + std::vector face_count; + std::vector emb; + std::vector bbox; // 4 per face + std::string embedder; + + std::size_t frames() const { return ts.size(); } + std::size_t faces() const { return emb.size(); } +}; + +template +std::vector read1d(H5::Group& g, const char* name, const H5::DataType& dt) { + H5::DataSet ds = g.openDataSet(name); + hsize_t n = 0; + ds.getSpace().getSimpleExtentDims(&n, nullptr); + std::vector out(n); + if (n) ds.read(out.data(), dt); + return out; +} + +Dump load(const std::string& path) { + H5::H5File f(path, H5F_ACC_RDONLY); + H5::Group frames = f.openGroup("frames"); + H5::Group faces = f.openGroup("faces"); + + Dump d; + d.ts = read1d(frames, "timestamp_sec", H5::PredType::NATIVE_DOUBLE); + d.is_cut = read1d(frames, "is_cut", H5::PredType::NATIVE_UINT8); + d.face_offset = read1d(frames, "face_offset", H5::PredType::NATIVE_INT64); + d.face_count = read1d(frames, "face_count", H5::PredType::NATIVE_INT32); + + H5::DataSet e = faces.openDataSet("embedding"); + hsize_t dims[2]{0, 0}; + e.getSpace().getSimpleExtentDims(dims, nullptr); + std::vector flat(dims[0] * dims[1]); + if (!flat.empty()) e.read(flat.data(), H5::PredType::NATIVE_FLOAT); + d.emb.resize(dims[0]); + for (hsize_t i = 0; i < dims[0]; ++i) + std::copy_n(flat.begin() + i * dims[1], 512, d.emb[i].begin()); + + // bbox is 2-D [N,4]; reading it with the 1-D helper would size the buffer + // from the first extent only and then read four times that many floats. + { + H5::DataSet bs = faces.openDataSet("bbox"); + hsize_t bd[2]{0, 0}; + bs.getSpace().getSimpleExtentDims(bd, nullptr); + d.bbox.resize(bd[0] * bd[1]); + if (!d.bbox.empty()) bs.read(d.bbox.data(), H5::PredType::NATIVE_FLOAT); + } + + // GR-004: the dump records which embedder produced it, so a replay cannot + // be silently scored against a gallery from a different model. + if (f.attrExists("embedder_model")) { + // Written as a variable-length string (embedding_dump_node.hpp:99), so + // the read must name the same type explicitly. + H5::StrType vlen(H5::PredType::C_S1, H5T_VARIABLE); + f.openAttribute("embedder_model").read(vlen, d.embedder); + } + return d; +} + +std::string fixture(const char* name) { + return std::string(SAE_TEST_FIXTURES_DIR) + "/dumps/" + name; +} + +// ── Harness ────────────────────────────────────────────────────────────────── +struct Replay { + std::vector claims; + std::vector track_ids; // per face, in fixture order + std::size_t faces_seen{0}; +}; + +Replay run(const Dump& d, double extinction = 10.0) { + Replay r; + TrackRegistry::Config rc; + rc.extinction_sec = extinction; + + auto cal = [](float cos) { return std::max(0.f, cos); }; + auto reg = std::make_shared(rc, EvidenceDiscounter(cal)); + reg->on_track_dead([&r](const DeadTrack& t) { r.claims.push_back(t); }); + + Config cfg; + cfg.track_assoc_min_prob = 0.5f; + FaceTrackerFunc ft(cfg, reg, cal); + + for (std::size_t i = 0; i < d.frames(); ++i) { + EmbeddedSceneFrame ef; + ef.source.timestamp_sec = d.ts[i]; + ef.source.is_cut = d.is_cut[i] != 0; + + const int64_t off = d.face_offset[i]; + const int32_t n = d.face_count[i]; + for (int32_t k = 0; k < n; ++k) { + DetectedFace face; + const float* b = &d.bbox[(off + k) * 4]; + face.bbox = cv::Rect2f(b[0], b[1], b[2], b[3]); + face.confidence = 1.0f; + ef.faces.push_back(face); + ef.crops.push_back(cv::Mat()); + ef.embeddings.push_back(d.emb[off + k]); + } + r.faces_seen += static_cast(n); + + auto out = ft(std::move(ef)); + for (int id : out.track_ids) r.track_ids.push_back(id); + } + + reg->flush(d.ts.empty() ? 0.0 : d.ts.back()); + return r; +} + +} // namespace + +// ── AR-004 / VR-001 — the fixtures are intact and self-describing ──────────── +TEST_CASE("fixtures are complete and carry their embedder identity", + "[replay][AR-004][VR-001]") { + // Frame counts are exact rather than approximate. Before node outputs + // blocked on a full channel, generation lost most of a clip and what it + // lost depended on timing — these numbers could not have been asserted. + struct Expect { const char* file; std::size_t frames, faces; }; + const Expect all[] = { + {"bali_13.h5", 385, 693}, + {"bali_27.h5", 335, 335}, + {"bali_28.h5", 345, 368}, + {"bali_31.h5", 145, 203}, + {"bali_46.h5", 385, 140}, + }; + + for (const auto& x : all) { + INFO(x.file); + Dump d = load(fixture(x.file)); + CHECK(d.frames() == x.frames); + CHECK(d.faces() == x.faces); + CHECK(d.embedder == "LVFace-B_Glint360K.onnx"); + + // face_offset must be contiguous: a gap means faces went missing + // between frames, which no consumer could detect. + int64_t running = 0; + for (std::size_t i = 0; i < d.frames(); ++i) { + REQUIRE(d.face_offset[i] == running); + running += d.face_count[i]; + } + CHECK(static_cast(running) == d.faces()); + } +} + +// ── VR-002 — replay is deterministic ───────────────────────────────────────── +TEST_CASE("replaying a fixture twice gives identical tracks", "[replay][VR-002]") { + // The property the whole fixture strategy rests on. If this fails, every + // golden output derived from a fixture is unreliable and the CI replay + // tier is worthless. + Dump d = load(fixture("bali_28.h5")); + Replay a = run(d); + Replay b = run(d); + + REQUIRE(a.track_ids.size() == b.track_ids.size()); + CHECK(a.track_ids == b.track_ids); + REQUIRE(a.claims.size() == b.claims.size()); + for (std::size_t i = 0; i < a.claims.size(); ++i) { + CHECK(a.claims[i].first_seen == b.claims[i].first_seen); + CHECK(a.claims[i].last_seen == b.claims[i].last_seen); + } +} + +// ── AR-012 / AR-013 — window invariants on real footage ────────────────────── +TEST_CASE("every face is assigned a track and every track closes", + "[replay][AR-012]") { + Dump d = load(fixture("bali_13.h5")); + Replay r = run(d); + + CHECK(r.track_ids.size() == r.faces_seen); + for (int id : r.track_ids) CHECK(id >= 0); // nothing silently unassigned + + // flush() must leave nothing behind: a track still open at EOF would be a + // window that never reaches the output. + CHECK(r.claims.size() > 0); +} + +TEST_CASE("windows are well-formed and inside the clip", "[replay][AR-013]") { + for (const char* f : {"bali_13.h5", "bali_27.h5", "bali_28.h5", + "bali_31.h5", "bali_46.h5"}) { + INFO(f); + Dump d = load(fixture(f)); + Replay r = run(d); + const double t0 = d.ts.front(), t1 = d.ts.back(); + + for (const auto& c : r.claims) { + // A window ends at the last sighting, never after it — so it can + // never extend past the footage that produced it. + CHECK(c.first_seen <= c.last_seen); + CHECK(c.first_seen >= t0); + CHECK(c.last_seen <= t1); + } + } +} + +TEST_CASE("a longer extinction window yields fewer, longer tracks", + "[replay][AR-013]") { + // The timeout decides whether a gap is absorbed into one window or splits + // it in two, so lengthening it must merge tracks rather than multiply them. + // On sparse footage this is the difference the constant actually makes. + Dump d = load(fixture("bali_46.h5")); // 140 faces over 385 frames + Replay tight = run(d, /*extinction=*/1.0); + Replay loose = run(d, /*extinction=*/30.0); + + CHECK(loose.claims.size() <= tight.claims.size()); +} + +// ── AR-007 — cuts are exercised by the corpus, not just by construction ────── +TEST_CASE("the cut-heavy fixture actually contains cuts", "[replay][AR-007]") { + // Guards the corpus rather than the code: if a regeneration produced a + // fixture with no cuts, the association tests above would still pass while + // silently testing nothing about viewpoint changes. + Dump d = load(fixture("bali_28.h5")); + const int cuts = std::count(d.is_cut.begin(), d.is_cut.end(), uint8_t{1}); + CHECK(cuts >= 5); +}