Compare commits
2
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
 dtourolle
|
faaa71fa09 | ||
|
dtourolle
|
054c4c8b8f |
@@ -53,7 +53,7 @@ jobs:
|
||||
# before it does. JellyTau's gate was believed for months while it was
|
||||
# dividing by frozen literals; untested gate logic is how that happens.
|
||||
- name: Test the extractor
|
||||
run: python3 scripts/vendor/jray-project/scripts/traceability/test_extract_traces.py
|
||||
run: python3 scripts/traceability/test_extract_traces.py
|
||||
|
||||
# Threshold policy and every other repo-specific setting live in
|
||||
# traceability.toml, not here, so local runs and CI runs cannot disagree
|
||||
@@ -63,7 +63,7 @@ jobs:
|
||||
# A misconfigured run (zero requirements parsed, zero files scanned) is a
|
||||
# hard failure rather than a plausible-looking 0%.
|
||||
- name: Traceability gate
|
||||
run: sh scripts/vendor/jray-project/scripts/traceability/traceability-gate.sh
|
||||
run: sh scripts/traceability/traceability-gate.sh
|
||||
|
||||
- name: Check modified files for traces
|
||||
if: github.event_name == 'pull_request'
|
||||
|
||||
@@ -19,10 +19,6 @@ compile_commands.json
|
||||
# coverage). Regenerate with scripts/docs/run_holdout_all_models.py and
|
||||
# scripts/docs/gallery_coverage_per_film.py.
|
||||
!docs_data/*.json
|
||||
# Exception: test fixtures are inputs, not build output. The audio golden
|
||||
# vector (IR-005) is shared verbatim with the jRay plugin repo, so it has to be
|
||||
# tracked. Regenerate the media with tests/fixtures/audio/make_fixture.py.
|
||||
!tests/fixtures/**
|
||||
# Video files
|
||||
*.mp4
|
||||
*.mkv
|
||||
|
||||
@@ -2,6 +2,3 @@
|
||||
path = external/KPN
|
||||
url = https://gitea.tourolle.paris/dtourolle/KPN.git
|
||||
branch = master
|
||||
[submodule "jray-project"]
|
||||
path = scripts/vendor/jray-project
|
||||
url = git@gitea.tourolle.paris:dtourolle/jray-project.git
|
||||
|
||||
+8
-46
@@ -152,24 +152,6 @@ if(SAE_GEMM_BACKEND STREQUAL "CPU")
|
||||
set_target_properties(gemm_backend PROPERTIES POSITION_INDEPENDENT_CODE ON)
|
||||
target_include_directories(gemm_backend PRIVATE src)
|
||||
target_compile_definitions(gemm_backend PRIVATE SAE_GEMM_CPU)
|
||||
|
||||
# AR-026/AR-027: back the CPU path with OpenBLAS when present. Optional, so
|
||||
# the build gains no hard dependency — but without it the fallback is a
|
||||
# scalar loop, which does not hold up against a library-scale gallery, and
|
||||
# the CPU path is exactly what CI (no GPU) and the cpu builder image use.
|
||||
find_package(PkgConfig QUIET)
|
||||
if(PkgConfig_FOUND)
|
||||
pkg_check_modules(OPENBLAS QUIET openblas)
|
||||
endif()
|
||||
if(OPENBLAS_FOUND)
|
||||
message(STATUS "GEMM backend: CPU + OpenBLAS ${OPENBLAS_VERSION}")
|
||||
target_compile_definitions(gemm_backend PRIVATE SAE_GEMM_CBLAS)
|
||||
target_include_directories(gemm_backend PRIVATE ${OPENBLAS_INCLUDE_DIRS})
|
||||
target_link_libraries(gemm_backend PRIVATE ${OPENBLAS_LINK_LIBRARIES})
|
||||
else()
|
||||
message(WARNING "GEMM backend: CPU scalar fallback — OpenBLAS not found. "
|
||||
"Correct, but slow on a large gallery (AR-027).")
|
||||
endif()
|
||||
elseif(SAE_GEMM_BACKEND STREQUAL "CUDA")
|
||||
find_library(CUBLAS_LIB cublas
|
||||
HINTS /opt/cuda/targets/x86_64-linux/lib /opt/cuda/lib64
|
||||
@@ -214,31 +196,23 @@ endif()
|
||||
# FFmpeg (hwaccel video decode: CUDA/VAAPI, runtime-detected + swscale colour
|
||||
# conversion). Hwaccel support is built into libavcodec/libavutil; no extra
|
||||
# libraries are needed here.
|
||||
# libswresample is the audio side of the same dependency — downmix + resample
|
||||
# for the audio signature (IR-004, src/audio_signature.cpp). Not a new project
|
||||
# dependency: it ships with the libav* set already required above.
|
||||
find_package(PkgConfig REQUIRED)
|
||||
pkg_check_modules(AVFORMAT REQUIRED libavformat)
|
||||
pkg_check_modules(AVCODEC REQUIRED libavcodec)
|
||||
pkg_check_modules(AVUTIL REQUIRED libavutil)
|
||||
pkg_check_modules(SWSCALE REQUIRED libswscale)
|
||||
pkg_check_modules(SWRESAMPLE REQUIRED libswresample)
|
||||
pkg_check_modules(AVFORMAT REQUIRED libavformat)
|
||||
pkg_check_modules(AVCODEC REQUIRED libavcodec)
|
||||
pkg_check_modules(AVUTIL REQUIRED libavutil)
|
||||
pkg_check_modules(SWSCALE REQUIRED libswscale)
|
||||
|
||||
add_library(ffmpeg_libs INTERFACE)
|
||||
target_compile_options(ffmpeg_libs INTERFACE
|
||||
${AVFORMAT_CFLAGS_OTHER} ${AVCODEC_CFLAGS_OTHER}
|
||||
${AVUTIL_CFLAGS_OTHER} ${SWSCALE_CFLAGS_OTHER}
|
||||
${SWRESAMPLE_CFLAGS_OTHER})
|
||||
${AVUTIL_CFLAGS_OTHER} ${SWSCALE_CFLAGS_OTHER})
|
||||
target_include_directories(ffmpeg_libs INTERFACE
|
||||
${AVFORMAT_INCLUDE_DIRS} ${AVCODEC_INCLUDE_DIRS}
|
||||
${AVUTIL_INCLUDE_DIRS} ${SWSCALE_INCLUDE_DIRS}
|
||||
${SWRESAMPLE_INCLUDE_DIRS})
|
||||
${AVUTIL_INCLUDE_DIRS} ${SWSCALE_INCLUDE_DIRS})
|
||||
target_link_libraries(ffmpeg_libs INTERFACE
|
||||
${AVFORMAT_LIBRARIES} ${AVCODEC_LIBRARIES}
|
||||
${AVUTIL_LIBRARIES} ${SWSCALE_LIBRARIES}
|
||||
${SWRESAMPLE_LIBRARIES})
|
||||
message(STATUS "FFmpeg: avformat=${AVFORMAT_VERSION} avcodec=${AVCODEC_VERSION} "
|
||||
"swresample=${SWRESAMPLE_VERSION}")
|
||||
${AVUTIL_LIBRARIES} ${SWSCALE_LIBRARIES})
|
||||
message(STATUS "FFmpeg: avformat=${AVFORMAT_VERSION} avcodec=${AVCODEC_VERSION}")
|
||||
|
||||
# nlohmann/json (gallery + output serialisation)
|
||||
include(FetchContent)
|
||||
@@ -275,8 +249,6 @@ find_package(HDF5 REQUIRED COMPONENTS CXX)
|
||||
add_library(sae_gallery STATIC
|
||||
src/gallery/gallery_store.cpp
|
||||
src/gallery/gallery_builder.cpp
|
||||
src/audio_signature.cpp # IR-004 — content-derived audio signature
|
||||
src/gallery/embedder_stamp.cpp # GR-004 — gallery/embedder binding
|
||||
)
|
||||
set_target_properties(sae_gallery PROPERTIES POSITION_INDEPENDENT_CODE ON)
|
||||
target_include_directories(sae_gallery PUBLIC src ${HDF5_INCLUDE_DIRS})
|
||||
@@ -308,16 +280,6 @@ target_link_libraries(sae_embed PRIVATE sae_gallery)
|
||||
nanobind_add_module(sae_kpn src/kpn_bindings.cpp)
|
||||
target_link_libraries(sae_kpn PRIVATE sae_gallery)
|
||||
|
||||
# ── sae_audio — Python module: the v1 audio signature (IR-004) ────────────────
|
||||
# Compiles audio_signature.cpp directly and links only FFmpeg, rather than
|
||||
# linking sae_gallery: the signature needs no model, no OpenCV and no HDF5, and
|
||||
# a module that dragged all three in would make `import sae_audio` depend on a
|
||||
# GPU-capable build of a repo whose audio path is pure CPU DSP. tests/ compiles
|
||||
# the same source the same way, for the same reason.
|
||||
nanobind_add_module(sae_audio src/audio_bindings.cpp src/audio_signature.cpp)
|
||||
target_include_directories(sae_audio PRIVATE src)
|
||||
target_link_libraries(sae_audio PRIVATE ffmpeg_libs)
|
||||
|
||||
# HDF5 already found above (before sae_gallery); vars HDF5_CXX_LIBRARIES / _INCLUDE_DIRS
|
||||
# are reused by scene_analyze / dump_embeddings below.
|
||||
|
||||
|
||||
+28
-558
@@ -40,28 +40,12 @@ Detect faces in sampled video frames.
|
||||
presence (SR-002) a lower rate still answers the question, but it lengthens the
|
||||
interval between samples and so weakens IoU-based association; sweep the two
|
||||
together (VR-002).
|
||||
- **Minimum face size is 40×40 px**, expressed in **original video resolution**,
|
||||
- **Minimum face size is 66×66 px**, expressed in **original video resolution**,
|
||||
not decoded-frame pixels. Stating it in original space decouples it from
|
||||
`dense_scale`: otherwise a 0.5 downscale silently doubles the effective
|
||||
threshold, and dense mode is exactly what scene detection uses.
|
||||
|
||||
40 is **measured, not estimated** — it replaces an earlier 66 px guess. Two
|
||||
studies bracket it, and the difference between them is the whole reason the
|
||||
number is 40 rather than 32:
|
||||
|
||||
- **VR-005** degrades an already-aligned 112×112 crop and matches it against
|
||||
a native-resolution gallery. Alignment is held perfect, so it isolates the
|
||||
*embedder*: the knee sits at 24–32 px, and 32 px still returns 98.1% TPI.
|
||||
- **VR-013** downscales the **whole frame before the detector**, so detection
|
||||
and landmark regression degrade along with it. End to end, holding 90% of
|
||||
the plateau needs roughly **50 px**, against VR-005's ~22 px.
|
||||
|
||||
The gap is detection and landmark error, which VR-005 excludes by construction
|
||||
— so VR-005 is an **upper bound on quality**, not a threshold, and reading a
|
||||
floor off it would admit faces in the falling region. **AR-002 therefore takes
|
||||
VR-013's number.** 40 sits below the 50 px plateau deliberately: FPI is 0.0% at
|
||||
every scale in both studies, so resolution loss costs recall and never
|
||||
precision, and an over-tight floor discards presence that SR-002 requires.
|
||||
66 is a working estimate of where ArcFace embeddings stop being reliable, not a
|
||||
measured value — it should be replaced by the result of VR-005.
|
||||
- Emits bounding box, detector confidence, and 5-point landmarks.
|
||||
- Bounding boxes must be reported in **original video pixel space**. When
|
||||
`dense_scale < 1` downscales the decoded frame, coordinates are rescaled by
|
||||
@@ -75,9 +59,7 @@ Detect faces in sampled video frames.
|
||||
**Current:** SCRFD-500MF via `face_detector_node.hpp`, thresholds in `config.hpp`
|
||||
(`detector_conf` 0.5, `detector_nms` 0.4), `min_face_px` 40, `max_faces` 10.
|
||||
|
||||
**Gap:** `min_face_px` re-expressed in original resolution — the value 40 is
|
||||
already correct after VR-013, so what remains is the space it is measured in, not
|
||||
the number; `max_faces`
|
||||
**Gap:** `min_face_px` → 66 and re-expressed in original resolution; `max_faces`
|
||||
removed, gated on backpressure (AR-004).
|
||||
|
||||
## AR-004 — Backpressure
|
||||
@@ -98,49 +80,8 @@ by dropping work or growing without limit.
|
||||
- Memory is the real limit: faces carry 112×112 crops plus 512-float embeddings.
|
||||
Backpressure must engage on bytes in flight, not just item counts.
|
||||
|
||||
### The fix is not in this repo
|
||||
|
||||
**Every node output in KPN uses the dropping `push()`** (`pool_node.hpp:404`,
|
||||
`:710`; also `branch.hpp`, `fanout.hpp`, `interrupt_node.hpp`). A lossless
|
||||
`push_blocking()` — "wait for the consumer to drain instead of dropping; the
|
||||
producer just runs slower" — already exists on both `Channel`
|
||||
(`channel.hpp:144`) and `OutputPort` (`variant_node.hpp:81`), **and nothing
|
||||
calls it.**
|
||||
|
||||
So AR-004 is a change to the KPN repository, not to this one. It needs either a
|
||||
per-channel lossless policy or a network-wide default, and this pipeline should
|
||||
select lossless: a dropped frame here does not degrade a result, it silently
|
||||
changes one.
|
||||
|
||||
**Measured, not inferred.** One 77 s clip at 5 fps should yield ~385 sampled
|
||||
frames. On CPU it produced 49, ending at 51 s, with 285 frames dropped at
|
||||
`camera_pos` and 51 at `face_aligner`. Rebuilt with CUDA the same clip ran in
|
||||
29 s and reached EOF correctly — and still dropped **320** frames at
|
||||
`camera_pos`, yielding 65. Faster hardware moves where the queue backs up; it
|
||||
does not change what happens when it does.
|
||||
|
||||
Two consequences worth stating:
|
||||
|
||||
- **Raising channel capacity is a stopgap, not a fix.** It lowers the
|
||||
probability of overflow without changing the behaviour on overflow, and the
|
||||
failure it hides is silent corruption of the output.
|
||||
- **Fixture generation is blocked on this** (VR-001), because what gets dropped
|
||||
depends on timing. The same command run twice can produce different dumps, and
|
||||
a golden fixture cannot be built on that.
|
||||
|
||||
**Current:** fixed in KPN — node data outputs use `push_blocking`, sentinels
|
||||
remain out-of-band so EOF can always overtake a stalled data path. Verified on
|
||||
the same clip: 385 of 385 sampled frames written, zero drops, and two
|
||||
consecutive runs byte-identical where previously they were not.
|
||||
|
||||
It also ran *faster* (29 s → 17 s). A dropped frame has already cost its decode,
|
||||
and the overflow exception cost more — so the lossy path was paying for work it
|
||||
then discarded.
|
||||
|
||||
**Gap:** the remaining half — bounding by **bytes in flight** rather than item
|
||||
count. Channel capacity is still a count of items, and a face carries a 112×112
|
||||
crop plus a 512-float embedding, so a crowded frame occupies far more memory per
|
||||
slot than a sparse one. That matters once `max_faces` is removed (AR-003).
|
||||
**Gap:** entire requirement. This is a prerequisite for removing `max_faces`, not
|
||||
a follow-up to it.
|
||||
|
||||
## AR-005 — Face alignment and crop
|
||||
|
||||
@@ -151,71 +92,9 @@ Produce the exact input ArcFace expects.
|
||||
nose, left mouth, right mouth).
|
||||
- Alignment is the *only* geometric normalisation; no additional augmentation at
|
||||
inference.
|
||||
- **The transform is fitted by Umeyama's closed-form least squares over all five
|
||||
points**, which is what InsightFace uses (skimage's `SimilarityTransform` *is*
|
||||
`_umeyama`) and therefore what produced the crops ArcFace and LVFace were
|
||||
trained on. The canonical warp is part of the input distribution, not an
|
||||
implementation detail (AR-011).
|
||||
- **Not a robust estimator.** A RANSAC fit buys a small residual by discarding
|
||||
the landmarks that disagree with the model, and on a turned face those are the
|
||||
foreshortened ones — the signal AR-030 reads. With five points and a two-point
|
||||
minimal sample it also cannot separate a mis-detected landmark from honest
|
||||
out-of-plane rotation, so the robustness is nominal while the cost to AR-030 is
|
||||
total. It is RNG-driven besides, which made replay determinism a property of
|
||||
thread scheduling.
|
||||
|
||||
**Current:** `align_face()` in `src/face_utils.hpp`, Umeyama fit via
|
||||
`umeyama_similarity()`, `cv::warpAffine` to `{112, 112}`. **Gap:** none.
|
||||
|
||||
> **Migration note — this was a defect, not a refinement.** Until this landed the
|
||||
> fit was `cv::estimateAffinePartial2D(…, cv::RANSAC, 3.0)`. The expectation was
|
||||
> that the two agree wherever RANSAC keeps all five points, leaving a small
|
||||
> divergence on non-frontal faces. **Measured, that is wrong.** On 400 random
|
||||
> gallery headshots, one model held fixed and only the estimator varied:
|
||||
>
|
||||
> | | median | p90 | max |
|
||||
> |---|---|---|---|
|
||||
> | Crop disagreement (source px, over the crop corners) | 16.97 | 75.91 | 223.31 |
|
||||
> | `cos(umeyama, ransac)` for the resulting embedding | 0.791 | — | — |
|
||||
>
|
||||
> 83.5 % of crops embed to a cosine below 0.99 of their Umeyama counterpart —
|
||||
> they are not the same face crop. The mechanism is that a 4-DoF similarity is
|
||||
> exactly determined by **two** points, so every minimal RANSAC sample fits its
|
||||
> own pair perfectly and is then scored on the other three. Real landmarks sit a
|
||||
> median 2.74 canonical px from any similarity fit to the template (see AR-030
|
||||
> below), so images with a landmark outside the 3 px band are the common case,
|
||||
> not the exception; RANSAC then keeps two or three inliers and returns a wildly
|
||||
> under-determined transform.
|
||||
>
|
||||
> **Every gallery baked before this change must be rebuilt** — GR-004's embedder
|
||||
> stamp catches a model change, not an aligner change, so nothing else would say
|
||||
> so.
|
||||
>
|
||||
> **How much this cost in accuracy is a separate question, and the answer appears
|
||||
> to be: less than the crop numbers suggest.** Rebuilding the full gallery
|
||||
> (2456 actors) moved the intra/inter separation the AR-023 calibration is fitted
|
||||
> from only slightly:
|
||||
>
|
||||
> | | intra-actor | inter-actor | separation |
|
||||
> |---|---|---|---|
|
||||
> | RANSAC | 0.6234 | 0.0407 | 0.5827 |
|
||||
> | Umeyama | 0.6340 | 0.0440 | 0.5900 |
|
||||
>
|
||||
> The reconciliation is that the old warp was *wrong but self-consistent*: it
|
||||
> produced a differently-framed face rather than a scrambled one, gallery and
|
||||
> probe went through the same estimator, and the embedder tolerates framing
|
||||
> variation. So the figures in `model-bakeoff.md`, `best-model.md` and
|
||||
> `pose-expansion.md` were all produced through the broken warp on both sides and
|
||||
> should be re-run, but there is no measured basis for expecting them to move far.
|
||||
>
|
||||
> The sharper evidence of the old instability is duplicate detection: rebuilding
|
||||
> with an unchanged `dedup_tol` dropped **1614** near-duplicate images, where the
|
||||
> original build dropped on the order of a hundred. Near-identical source images
|
||||
> used to embed to visibly different vectors — RANSAC fitting two-point subsets is
|
||||
> unstable under small landmark perturbations, and being RNG-driven it was not
|
||||
> reproducible either. That instability is what a tracker accumulating evidence
|
||||
> across frames pays for, and it is the strongest reason the fix is worth having
|
||||
> independently of any accuracy delta.
|
||||
**Current:** `align_face()` in `src/face_utils.hpp:9-22`, `cv::warpAffine` to
|
||||
`{112, 112}`. **Gap:** none.
|
||||
|
||||
## AR-006 — Embedding
|
||||
|
||||
@@ -231,121 +110,6 @@ Generate a 512-d embedding per aligned crop.
|
||||
**Current:** `embedder_node.hpp` + `face_embedder_engine.hpp`; default
|
||||
LVFace-B_Glint360K. **Gap:** none.
|
||||
|
||||
## AR-028 … AR-030 — Embedding input quality
|
||||
|
||||
An embedder handed a face it cannot represent does not fail. It returns a
|
||||
confident, plausible, wrong vector, and that vector then competes on equal terms
|
||||
with every good one in the gallery — the same failure mode AR-011 names for
|
||||
whole models, occurring here at the level of a single region. Quality assessment
|
||||
is how that is caught **at inference**, rather than inferred afterwards from a
|
||||
study of why a film scored badly.
|
||||
|
||||
Three axes, assessed on every face before its embedding is used as identity
|
||||
evidence. They are kept separate and **not collapsed into one scalar**: they fail
|
||||
for different reasons, have different remedies, and — as below — do not even earn
|
||||
the same response.
|
||||
|
||||
- **Size** — already AR-002, floor at 40×40 px in original resolution, measured
|
||||
end to end by VR-013. It is the precedent for the other two: the
|
||||
threshold was *located*, not chosen.
|
||||
- **Sharpness** — motion blur and soft focus destroy the high-frequency detail
|
||||
the embedder keys on, and unlike size they leave the bounding box looking
|
||||
perfectly healthy. Measured on the **112×112 aligned crop**, not the raw box:
|
||||
the crop is already scale-normalised, so a measure taken there cannot silently
|
||||
re-measure face size and double-count it against AR-002.
|
||||
- **Visibility** — extreme pose or occlusion means the face presents fewer of the
|
||||
features the embedding assumes are present. The measure is the **residual of
|
||||
the AR-005 alignment fit**: the RMS landmark error, in canonical 112×112
|
||||
pixels, left over after the best similarity transform onto the ArcFace
|
||||
template. It costs nothing — the transform is computed for the warp regardless,
|
||||
and the residual is what that fit could not explain.
|
||||
|
||||
Two properties earn it the job over an explicit yaw estimate:
|
||||
|
||||
- A similarity absorbs rotation, uniform scale and translation **exactly**,
|
||||
so the residual is by construction the non-similarity part of the
|
||||
deformation: out-of-plane rotation and foreshortening. In-plane roll
|
||||
contributes nothing, so "a tilted head reads as a turned one" is excluded
|
||||
structurally rather than by tuning. The destination frame is fixed, so face
|
||||
size cannot leak in either — that is AR-002's axis, and double-counting it
|
||||
would make a small frontal face look occluded.
|
||||
- It responds to **occlusion** and to plainly broken landmark sets, which an
|
||||
angle regressor by construction does not: a hand across the face is not a
|
||||
rotation, but it does displace landmarks.
|
||||
|
||||
Indicative magnitudes from a synthetic foreshortening sweep (`k ≈ cos yaw`):
|
||||
`k=1.0 → 0.00`, `0.9 → 1.18`, `0.75 → 3.11`, `0.5 → 6.72`, `0.3 → 9.85`
|
||||
canonical px. Smooth and monotone with a usable range; the mapping onto real
|
||||
faces is VR-012's to establish, and no threshold is set from these numbers.
|
||||
|
||||
**The synthetic ladder is noise-free and therefore optimistic about the low
|
||||
end.** Measured on 400 real TMDB/Jellyfin headshots — the most frontal, most
|
||||
cooperative population the pipeline ever sees — the residual runs p5 1.11,
|
||||
median 2.74, p90 4.82, max 6.35 canonical px. So landmark noise alone occupies
|
||||
roughly the first 3 px, and the synthetic sweep's "26° yaw ≈ 1.2 px" sits
|
||||
*below* the noise floor on real data. VR-012 must set any threshold against
|
||||
this measured distribution, and a discount curve has to treat the first few
|
||||
pixels as uninformative rather than as mild pose.
|
||||
|
||||
Neither a dedicated landmark model (`models/2d106det.onnx` is present but
|
||||
referenced nowhere — and it emits points, not pose) nor a direct pose CNN is
|
||||
adopted unless VR-012 shows the residual insufficient. If one is needed the
|
||||
candidate is **6DRepNet** (MIT, RepVGG-B1g2, 3.47° MAE on AFLW2000) rather than
|
||||
Hopenet, which it dominates on accuracy, licence, recency and export
|
||||
friendliness. Two caveats to record before that happens: both are trained on
|
||||
**300W-LP**, which inherits research-only terms from 300W's constituent sets,
|
||||
and both want their own loosely-framed ROI rather than the ArcFace crop — a
|
||||
second warp and a second image in flight, which lands on AR-004's byte-based
|
||||
backpressure gap. It would also have to run **per track** — over the bounded
|
||||
view set AR-019's diversity buffer already keeps — not per face per frame,
|
||||
which is the cost rule applied as written: fewer regions, never a degraded
|
||||
input.
|
||||
|
||||
**Failing an axis discounts the observation; it does not delete the detection.**
|
||||
Only size drops the face outright, and only because VR-005 measured a knee below
|
||||
which the embedding carries no signal to discount. Blur and pose are different:
|
||||
|
||||
- A blurred or turned face is still evidence of **presence**, which is what
|
||||
SR-002 actually asks about.
|
||||
- The tracker admits a link on position *or* identity precisely so that a face
|
||||
"whose embedding degraded (blur, profile turn)" stays linkable. Remove the
|
||||
detection and the track fragments, costing the window extent AR-012/AR-013
|
||||
exist to protect.
|
||||
- AR-019 harvests non-frontal views *because* TMDB headshots are frontal.
|
||||
Discarding turned faces starves the mechanism built to fix the pose problem of
|
||||
its raw material, and AR-020 then has nothing to resolve at EOF.
|
||||
|
||||
The natural home for the discount is `EvidenceDiscounter` (AR-025), which already
|
||||
weights how far one observation may move a track's belief. Note that its present
|
||||
weight is pure *novelty*, so a profile view — maximally distant from everything
|
||||
counted so far — currently scores near 1.0 and moves the belief hardest, when
|
||||
against a frontal gallery it deserves the least trust. Novelty and reliability
|
||||
are orthogonal and multiply; quality supplies the second term.
|
||||
|
||||
**Quality is carried, not consumed.** The vector travels with the face and is
|
||||
written to the VR-001 dump alongside the embedding, so a threshold can be
|
||||
re-litigated against recorded data instead of by re-running video, and so
|
||||
VR-010's provenance records what the run actually admitted.
|
||||
|
||||
**No quality threshold is hand-set.** Each axis either has a measured knee
|
||||
(VR-012, as VR-005 did for size) or it discounts rather than drops — a
|
||||
hand-chosen cutoff on an uncalibrated measure is the same unfalsifiable magic
|
||||
number AR-024 retired for similarity, and it would fail the same way: meaning
|
||||
something different for every detector, every embedder and every film.
|
||||
|
||||
**Current:** visibility is measured and carried — `estimate_alignment()` in
|
||||
`src/face_utils.hpp` returns the residual alongside the transform, and
|
||||
`FaceAlignerFunc` writes it to `DetectedFace::alignment_residual`. Size is
|
||||
`min_face_px` (40, decoded-frame space — AR-002 still open). Sharpness is
|
||||
unmeasured. Nothing yet *consumes* any of it: no discount is applied, and
|
||||
`align_face()` still drops the degenerate-fit case without counting it.
|
||||
|
||||
**Gap:** AR-029 entirely. For AR-030, the measure exists but the discount does
|
||||
not — it must reach `EvidenceDiscounter` as the reliability term. For AR-028, the
|
||||
residual does not yet reach the VR-001 dump, which is what VR-012 needs to run
|
||||
from fixtures; that is the next step, since it unblocks the study that sets
|
||||
every remaining behaviour.
|
||||
|
||||
## AR-007, AR-008 — Tracking
|
||||
|
||||
Link detections across frames into tracks representing one physical person.
|
||||
@@ -390,50 +154,6 @@ Two distinct signals, deliberately kept separate:
|
||||
- **`is_scene_boundary`** — opt-in (`--scene-detect`). TransNetV2 over a densely
|
||||
decoded, downscaled stream flags a *true shot/scene boundary*.
|
||||
|
||||
> **`is_scene_boundary` currently has no producer.** `grep -rn is_scene_boundary
|
||||
> src/` finds no assignment anywhere: `SceneDetectorFunc` is a *terminal sink*
|
||||
> (`main.cpp:298-300`, `kpn::out<>`) that writes `scenes.json` and never
|
||||
> annotates the `Frame` flowing to the face pipeline. The field is therefore
|
||||
> always `false`, and the dump column (`embedding_dump_node.hpp:38`) is a
|
||||
> constant 0. Compounding it, `main.cpp:280` returns from the
|
||||
> `--dump-embeddings` branch *before* the `scene_detect` branch at `:296`, so no
|
||||
> dump-producing path even instantiates the detector.
|
||||
>
|
||||
>
|
||||
> **It cannot be fixed by making the node a pass-through.** TransNetV2 buffers
|
||||
> `kWindow` = 100 dense frames before it can score any of them, runs inference
|
||||
> every `scene_stride` (50) frames, and trusts only each window's centre. So a
|
||||
> boundary at time *T* is not known until roughly 100 dense frames after *T* —
|
||||
> about **3.3 s at 30 fps**. The face pipeline runs on a parallel branch and has
|
||||
> long since passed *T* by then. An association hint that arrives after the
|
||||
> association is worthless.
|
||||
>
|
||||
> Three ways out, none free:
|
||||
>
|
||||
> 1. **Two-pass.** Run scene detection to completion, then analyse faces with
|
||||
> boundaries already known. Simple and correct; costs a second decode of the
|
||||
> whole file, and dense decode is already the pipeline's dominant cost.
|
||||
> 2. **Delay the face branch** by the detector's window latency. Keeps one pass;
|
||||
> adds a buffering stage and couples the two branches' timing, which is the
|
||||
> kind of coupling that produces heisenbugs under backpressure.
|
||||
> 3. **Leave it unwired.** Accept that `is_cut` is the only association hint.
|
||||
>
|
||||
> **Option 3 costs less than it appears**, which is why this is a decision rather
|
||||
> than a bug. Since the redesign made cuts and boundaries do the *same thing* —
|
||||
> both say "spatial continuity is broken, associate on embedding" — TransNetV2
|
||||
> adds nothing over the histogram except on transitions the histogram misses:
|
||||
> slow dissolves and fades, where there is no frame-to-frame discontinuity to
|
||||
> detect. That is a real but narrow gap.
|
||||
>
|
||||
> The value TransNetV2 retains is in **AR-019**, whose promotion gate requires a
|
||||
> span with no cut *and* no boundary. There a late answer is still usable,
|
||||
> because promotion happens when a track is confirmed rather than per frame.
|
||||
> Wiring it there — offline, against the collected boundary list — is cheaper
|
||||
> than any of the three options above and does not touch the hot path.
|
||||
>
|
||||
> **Recommendation: option 3 plus the AR-019 wiring**, and revisit if dissolve-
|
||||
> heavy material shows association failures the histogram misses.
|
||||
|
||||
Both feed AR-007 as **association hints**: they tell the tracker that spatial
|
||||
continuity is broken and that association should weight embedding over IoU.
|
||||
Neither ends a presence window (AR-012).
|
||||
@@ -666,30 +386,12 @@ An embedding is admitted only if its similarity to one already in the store fall
|
||||
admitting it risks poisoning the store.
|
||||
|
||||
A starting band of roughly **0.90–0.95** is the working estimate, to be tuned
|
||||
(VR-007). Note this is deliberately conservative compared to the retired
|
||||
`expand_novelty_sim` (0.55), which promoted embeddings *far* from the gallery —
|
||||
(VR-007). Note this is deliberately conservative compared to the current
|
||||
`expand_novelty_sim` (0.55), which promotes embeddings *far* from the gallery —
|
||||
much more aggressive, and much more exposed to admitting the wrong person.
|
||||
|
||||
Both bounds must be expressed as calibrated probabilities, not raw cosines (AR-024).
|
||||
|
||||
The lower bound is asked twice. `admit` compares a newcomer against its
|
||||
*closest* existing member, which a gradually drifting track can chain past: every
|
||||
step inside the band while the endpoints are strangers — the shape a track-ID
|
||||
collision takes over a slow pan. So the same bound is re-applied across **every
|
||||
pair** in the store before promotion. One bound, two enforcement points; not a
|
||||
second constant.
|
||||
|
||||
Novelty is deliberately **not** a threshold. The store's eviction policy orders
|
||||
its members by similarity to the actor's existing references and drops the
|
||||
best-recognised one, so novelty-seeking is a ranking with nothing to tune, and
|
||||
the band's upper bound already refuses the redundant views at the door.
|
||||
|
||||
**Current:** implemented in `gallery/track_gallery.hpp` — `admit` at the door,
|
||||
`store_coherence` at promotion, both bounds from `Config::expand_band_lo/hi`.
|
||||
Refusals are counted (`band_rejected`).
|
||||
|
||||
**Gap:** the bounds themselves are unswept working values (VR-007).
|
||||
|
||||
### AR-019 — Expansion of known actors
|
||||
|
||||
When a track is owned (AR-012), its store is promoted into a **per-film, in-memory
|
||||
@@ -790,14 +492,14 @@ natural unit for anonymous presence, should that be adopted (AR-012, TBD).
|
||||
open (VR-007).
|
||||
|
||||
**Current:** `src/gallery/track_gallery.hpp` implements a per-track diversity
|
||||
buffer with eviction biased to gallery-far poses, admission and promotion both
|
||||
gated on the AR-018 band in probability space, cleared on `is_cut`. Wired at
|
||||
`identity_matcher_node.hpp:227`, cleared at `:126`; the calibration is handed
|
||||
over at `:114`.
|
||||
buffer with eviction biased to gallery-far poses, promotion gated on
|
||||
`expand_novelty_sim` / `expand_track_spread_max`, cleared on `is_cut`. Wired at
|
||||
`identity_matcher_node.hpp:227`, cleared at `:126`.
|
||||
|
||||
**Gap:** all three quiet-signal conditions rather than only `is_cut`; and the
|
||||
whole of AR-020 — the TBI queue, the deferred pass, and deferring output until it
|
||||
completes.
|
||||
**Gap:** the band rule of AR-018 replacing the current novelty/spread gates; all
|
||||
three quiet-signal conditions rather than only `is_cut`; probability space
|
||||
throughout (AR-024); and the whole of AR-020 — the TBI queue, the deferred pass, and
|
||||
deferring output until it completes.
|
||||
|
||||
## AR-022 — Unidentified-track capture
|
||||
|
||||
@@ -1055,16 +757,9 @@ prerequisite.
|
||||
|
||||
## DP-005 — Installation and provisioning
|
||||
|
||||
- Native install, **no Docker at runtime** — GPU passthrough is the most fragile
|
||||
part of a containerised setup and exists only because of the container.
|
||||
Natively the GPU works with the host drivers and media paths need no
|
||||
re-mounting. This constrains how the software *runs*, not how it is *built*:
|
||||
DP-008 uses containers as build environments precisely because that side has
|
||||
none of these problems.
|
||||
- The installer may **fetch a prebuilt binary** (DP-008) instead of compiling.
|
||||
Compiling stays supported, but should not be the only path — it is the slowest
|
||||
and most fragile step of a first install. TRT engines are still built locally
|
||||
either way (DP-008).
|
||||
- Native install, **no Docker** — GPU passthrough is the most fragile part of a
|
||||
containerised setup and exists only because of the container. Natively the GPU
|
||||
works with the host drivers and media paths need no re-mounting.
|
||||
- An installer (`scripts/build_install.py`) consuming one `install.yaml`:
|
||||
platform (nvidia/amd/cpu), embedder model, gallery scan cadence, install
|
||||
prefix; runtime secrets written to a `.env`, editable without recompiling.
|
||||
@@ -1075,130 +770,6 @@ prerequisite.
|
||||
|
||||
**Gap:** installer unbuilt.
|
||||
|
||||
## DP-007 — CI build image
|
||||
|
||||
CI runs on an Intel N100 with no discrete GPU, so the test build must configure
|
||||
**CPU-only** and must not require CUDA, TensorRT or ROCm:
|
||||
|
||||
```
|
||||
-DSAE_INFERENCE_BACKEND=ORT -DSAE_GEMM_BACKEND=CPU
|
||||
```
|
||||
|
||||
A prebuilt container image supplies the toolchain, published to the **Gitea
|
||||
container registry** and pinned by tag — matching the `jellytau-builder`
|
||||
precedent. Building dependencies per CI run is untenable on an N100, and OpenCV 5
|
||||
from source would dominate every run.
|
||||
|
||||
The same registry stores corpus dump fixtures as generic packages (see the
|
||||
fixtures table in `requirements.md`). Rebuild the image when its dependency set
|
||||
changes, not per run, and pin CI to a tag rather than `latest` so a rebuild
|
||||
cannot silently change what a green build meant.
|
||||
|
||||
**Required in the image:**
|
||||
|
||||
| Dependency | Why |
|
||||
|---|---|
|
||||
| CMake, C++ toolchain, pkg-config | Build |
|
||||
| **OpenCV 5** | `CMakeLists.txt:25` prefers 5, falls back to 4. The branch targets 5, so the image should carry it — it is not yet in most distro repos and building it per-run is prohibitive |
|
||||
| HDF5 (C++) | Galleries are HDF5-native; also the dump format |
|
||||
| FFmpeg dev libs — `libavformat`, `libavcodec`, `libavutil`, `libswscale`, **`libswresample`** | Decode. See the note below on swresample |
|
||||
| Python 3 + numpy, h5py, scipy | Python-side tests, replay, traceability tooling |
|
||||
| **OpenBLAS** | Backs the CPU similarity GEMM. Without it the fallback is a scalar loop, and the CPU path is exactly what this host runs — see below |
|
||||
| Catch2, nlohmann/json | **Vendored into the image, not fetched.** Both are `FetchContent`-ed today (`CMakeLists.txt:220`, `tests/CMakeLists.txt:8`), which makes every CI run depend on GitHub reachability |
|
||||
|
||||
**OpenBLAS is not optional here, despite being optional in the build.** CI has no
|
||||
GPU, so `SAE_GEMM_BACKEND=CPU` is the only path it exercises — and since AR-003
|
||||
removed the per-frame face cap, a crowded frame scores many faces against a
|
||||
library-scale gallery. The scalar fallback is correct but scales badly, which
|
||||
would make the CPU path the bottleneck in the one place it cannot be avoided
|
||||
(AR-027). The build warns when it is missing rather than failing, so a developer
|
||||
without it still gets a working tree; the image must not be that case.
|
||||
|
||||
The test target links it too. Otherwise the suite compiles the scalar fallback
|
||||
while the image ships CBLAS, and CI would verify a kernel that is not the one
|
||||
running in production.
|
||||
|
||||
**Deliberately excluded:** CUDA, TensorRT, ROCm — no GPU to use them. Also the
|
||||
ONNX Runtime *GPU* providers; only the CPU provider is relevant, and only for T3
|
||||
smoke tests.
|
||||
|
||||
**Models are not baked into the image.** The seven ONNX files total ~725 MB and
|
||||
live in Git LFS. T1/T2 tests are model-free by design
|
||||
(`tests/CMakeLists.txt:1-4`), so the default image needs none. T3 smoke tests
|
||||
require a model and should pull it via LFS in a separate job rather than
|
||||
inflating the image tenfold for a minority of tests.
|
||||
|
||||
**`libswresample` is a real gap, not a formality.** The current
|
||||
`pkg_check_modules` list (`CMakeLists.txt:200-203`) covers avformat, avcodec,
|
||||
avutil and swscale but **not** swresample — which IR-004 needs to downmix to mono
|
||||
and resample to 11025 Hz. It must be added alongside the audio-signature work.
|
||||
|
||||
**Gap:** entire requirement. The image does not exist, and no CI config is
|
||||
present in this repo.
|
||||
|
||||
## DP-008 — Builder images and release binaries
|
||||
|
||||
Produce prebuilt binaries per backend so deployment does not require every user
|
||||
to compile the project.
|
||||
|
||||
**This does not contradict DP-005.** That requirement rejects Docker as a
|
||||
*runtime* — GPU passthrough is the most fragile part of a containerised setup and
|
||||
exists only because of the container. Using Docker as a *build* environment is
|
||||
the opposite case: hermetic, reproducible, and it lets one machine produce
|
||||
binaries for backends it cannot itself run. Build in a container; run natively.
|
||||
|
||||
### Image matrix
|
||||
|
||||
The build has two independent axes (`CMakeLists.txt:48-49`), so the useful
|
||||
combinations are:
|
||||
|
||||
| Image | `SAE_INFERENCE_BACKEND` | `SAE_GEMM_BACKEND` | Target |
|
||||
|---|---|---|---|
|
||||
| `sae-builder-cpu` | ORT | CPU | CI (DP-007), and the smoke-test fallback |
|
||||
| `sae-builder-cuda` | TRT | CUDA | NVIDIA |
|
||||
| `sae-builder-rocm` | ORT | ROCM | AMD |
|
||||
|
||||
All three carry the DP-007 dependency set (OpenCV 5, HDF5, FFmpeg incl.
|
||||
swresample, vendored Catch2/nlohmann) and differ only in the accelerator stack.
|
||||
The CPU image is the CI image — one artifact, two uses.
|
||||
|
||||
Published to the Gitea container registry, pinned by tag, rebuilt when the
|
||||
dependency set changes rather than per run.
|
||||
|
||||
### What ships, and what cannot
|
||||
|
||||
**Ships:** the `scene_analyze` binary and its companions, per backend.
|
||||
|
||||
**Cannot ship: TensorRT engines.** `.engine` files are specific to the GPU
|
||||
architecture and TRT version they were built on — `scripts/build_trt_engines.sh`
|
||||
must still run on the target machine. A prebuilt binary shortens the install; it
|
||||
does not remove the local engine-build step, and the installer must not imply
|
||||
otherwise.
|
||||
|
||||
**Cannot ship: models.** ~725 MB in LFS, and orthogonal to the binary.
|
||||
|
||||
### The constraint that decides the base image
|
||||
|
||||
**A binary built in a container runs against the host's glibc.** Build on a
|
||||
newer base than the oldest supported host and it fails at load with
|
||||
`GLIBC_2.xx not found` — the classic and entirely avoidable trap when shipping
|
||||
binaries out of containers.
|
||||
|
||||
So the base is chosen for the *oldest* glibc to be supported, not for
|
||||
convenience or recency. Accelerator libraries have the same shape of problem:
|
||||
the binary links against a driver-provided runtime, so each image must document
|
||||
the CUDA/ROCm version range its output is compatible with, and the installer
|
||||
must check it rather than discovering a mismatch at first inference.
|
||||
|
||||
### Jobs
|
||||
|
||||
A release job per backend, producing a tagged artifact in the registry. These are
|
||||
**not** the CI gate — the gate runs the CPU image on every push (DP-007);
|
||||
release builds run on tag. Their outputs are what DP-005's installer fetches
|
||||
when the user does not want to compile.
|
||||
|
||||
**Gap:** entire requirement. No images, no release jobs.
|
||||
|
||||
## DP-006 — Gallery maintenance as a background concern
|
||||
|
||||
- Incremental gallery refresh runs on a timer (`gallery_scan_interval`, default
|
||||
@@ -1288,10 +859,8 @@ rewritten.
|
||||
|
||||
**Media shorter than 120 s.** The window `runtime/2 ± 60 s` underflows, so no
|
||||
signature is emitted and **no sync offset is applied**. Such items fall back to
|
||||
the runtime tier, which is adequate: a 90-second extra or trailer is not the
|
||||
content whose cut alignment matters. (There is no `exact` tier: the file-hash
|
||||
tier was withdrawn on legal grounds — it fingerprinted an individual copy rather
|
||||
than the cut the timings describe. See the server spec §3.) Both producers must apply the identical
|
||||
the runtime/exact tiers, which is adequate: a 90-second extra or trailer is not
|
||||
the content whose cut alignment matters. Both producers must apply the identical
|
||||
rule, or they diverge on exactly the short items most likely to be
|
||||
mis-identified.
|
||||
|
||||
@@ -1313,47 +882,7 @@ plugin. Consequences to carry through:
|
||||
- Files never processed by this pipeline still get a signature from the plugin;
|
||||
the two paths coexist deliberately.
|
||||
|
||||
**Current:** `src/audio_signature.*` implements the construction, and
|
||||
`tests/fixtures/audio/` holds the golden vector shared verbatim with the plugin
|
||||
repo, which now matches it byte for byte from C# (jRay `JR-042`/`JR-043`).
|
||||
`sae_audio` (nanobind, as `sae_embed` and `sae_kpn` are) exposes the same C++ to
|
||||
Python so a study drives the shipped code rather than a numpy port.
|
||||
|
||||
**VR-014 measures what the golden vector cannot** — that the signature actually
|
||||
aligns a differently trimmed release, on real film audio rather than a synthetic
|
||||
tone. It does, with an order of magnitude to spare.
|
||||
|
||||
**The accuracy question is settled and is not close.** What the offset is *for*
|
||||
is shifting scene windows, which are seconds long, so half a second of error is
|
||||
invisible; the budget is 500 ms. Over 40 random offsets inside the ±600-frame cap
|
||||
the recovered offset was the nearest frame every time — **worst error 46 ms**.
|
||||
That figure is the quantisation floor rather than a measurement of quality: the
|
||||
offset is expressed in whole 92.88 ms frames, so no correct answer can ever be
|
||||
worse than half a frame. The `runtime/2` anchor behaves as specified through real
|
||||
head-trimmed files (cutting `delta` from the head moves the window by
|
||||
`delta/2`), and both an out-of-cap offset and unrelated content are declined
|
||||
outright (0.10 and 0.07).
|
||||
|
||||
**Where it is soft is tier labelling, not alignment.** The *score* at the correct
|
||||
offset falls with sub-frame misalignment — 0.94–0.99 when the true offset lands
|
||||
within 0.1 of a frame boundary, 0.69–0.73 at half a frame — because the two
|
||||
windows' frame grids no longer coincide. The offset stays right, but only 13 of
|
||||
40 cleared the server's 0.85 `audio` threshold and the other 27 were demoted to
|
||||
`loose`, a tier that means "possibly the same cut, degraded audio". The threshold
|
||||
was calibrated on a re-encode at *zero* offset, where the score is 1.00.
|
||||
|
||||
The remedy is measured, not proposed (UT-108): counting a frame as agreeing if
|
||||
its peak bin matches **within ±1 frame** returns all 40 to `audio` (worst 0.906)
|
||||
while unrelated content and out-of-cap offsets stay at 0.12 and 0.16 — the gap
|
||||
that makes the threshold mean anything is untouched. It costs 81 ms of offset
|
||||
accuracy, of a 500 ms budget, because the flattened peak lets the argmax pick an
|
||||
adjacent frame. ±2 frames buys nothing further. Adopting it is a
|
||||
[server spec](../../JRay-public-server/SPEC.md) §3 change — the score is
|
||||
normative and shared by three repos — so this repo measures it and leaves the
|
||||
decision there.
|
||||
|
||||
**Gap:** the signature is computed but **not yet emitted** into the truth file —
|
||||
that is the `IR-002` field and the coordinated `schema_version` bump.
|
||||
**Gap:** entire requirement — no audio path exists in the pipeline today.
|
||||
|
||||
## IR-006 — Jellyfin round-trip
|
||||
|
||||
@@ -1416,57 +945,8 @@ surfaced as a build report.
|
||||
different models are meaningless but *look* plausible — this fails silently and
|
||||
expensively otherwise.
|
||||
|
||||
### The stamp
|
||||
|
||||
Two fields, written together: the model file's **basename** and the **SHA-256 of
|
||||
its bytes** (plus `embed_dim` as a cheap extra guard). Stored as the `/embedder`
|
||||
group in the gallery HDF5, and as an optional top-level `"embedder"` object in
|
||||
the legacy JSON format.
|
||||
|
||||
The hash *decides*; the name is what a human *reads*. Neither alone is enough. A
|
||||
name is a promise rather than a fact — models get re-exported, re-quantised and
|
||||
overwritten in place under an unchanged filename, which is exactly the case where
|
||||
the weights differ and nothing else does, so a name-only stamp is blind to the
|
||||
failure it exists to catch. A hash alone is correct but unactionable: *"expected
|
||||
3f2a…, got 9c1b…"* tells an operator nothing about what to do next. SHA-256 over
|
||||
the file is derived from the artefact rather than asserted about it, needs no
|
||||
registry kept up to date, and costs ~0.1 s for a 250 MB ONNX once per process.
|
||||
|
||||
### Verdicts
|
||||
|
||||
| Verdict | When | Default | Under strict mode |
|
||||
|---|---|---|---|
|
||||
| `match` | hashes agree | proceed | proceed |
|
||||
| `weak_match` | names agree, one side unhashable | **warn** | **error** |
|
||||
| `unstamped` | gallery predates GR-004 | **warn** | **error** |
|
||||
| `unknown_embedder` | gallery stamped, embedder unidentifiable | **warn** | **error** |
|
||||
| `mismatch` | proven different models | **error** | **error** |
|
||||
|
||||
**A mismatch is fatal in every mode, with no bypass**, and the message names both
|
||||
sides — what the gallery was built with and what is loaded.
|
||||
|
||||
The three "cannot prove it" verdicts warn loudly instead, because they describe an
|
||||
*unknown* state rather than a *known-bad* one, and because every gallery built
|
||||
before this requirement is unstamped. Hard-failing all of them would make the
|
||||
check something people route around rather than trust. Strict mode
|
||||
(`--require-gallery-stamp`, or `SAE_REQUIRE_GALLERY_STAMP=1`, which propagates to
|
||||
subprocesses) promotes them to errors — that is the mode measurement work runs in.
|
||||
`scripts/stamp_gallery.py` re-binds an existing gallery without re-embedding, so
|
||||
migration costs one command; that is what makes "warn" a temporary state rather
|
||||
than a permanent one.
|
||||
|
||||
### Scope of the check
|
||||
|
||||
Embedding **dumps** carry the same stamp (`embedder_model` / `embedder_sha256`
|
||||
root attributes, `scripts/optimizer/SCHEMA.md`): a replay has no live embedder, so
|
||||
the dump *is* the embedder as far as the gallery is concerned. Derived galleries
|
||||
(filter, cast-restrict) inherit their source's stamp; `--merge` and the JSON
|
||||
gallery merge check *before* writing, since a merged file holding two embedding
|
||||
spaces cannot be untangled afterwards by any later check.
|
||||
|
||||
**Gap:** none. Stamped in `gallery_builder.cpp` and the Python builders; verified
|
||||
in `scene_analyze`, `scene_preview`, the `sae_kpn` matcher binding, `replay.py`,
|
||||
`optimize.py`, `movienet_eval.py` and the merge paths.
|
||||
**Gap:** named as step 4 of the `service-conversion.md` implementation plan;
|
||||
unbuilt. This is the highest-value small fix in the document.
|
||||
|
||||
## GR-006 … GR-009 — Provenance tiers and poisoning guard
|
||||
|
||||
@@ -1539,11 +1019,9 @@ Persist pipeline state at the point where the expensive work ends.
|
||||
variable-length HDF5 types and reads straight into numpy.
|
||||
- Stores per frame: `timestamp_sec`, `frame_idx`, `is_cut`, `is_scene_boundary`.
|
||||
Per face: `embedding` [N,512], `bbox` [N,4], `landmarks` [N,10], `confidence`.
|
||||
- Invariants: embeddings unit-norm; `face_offset` contiguous; bboxes and
|
||||
landmarks in **decoded-frame** pixels with `bbox_upscale` recorded alongside
|
||||
(the dump is a faithful tap, so it does not transform what the tracker saw —
|
||||
see VR-010); frames with no faces still get a row so timestamps stay dense;
|
||||
EOF sentinels not written.
|
||||
- Invariants: embeddings unit-norm; `face_offset` contiguous; bboxes already in
|
||||
original resolution; frames with no faces still get a row so timestamps stay
|
||||
dense; EOF sentinels not written.
|
||||
- Enabled by `--dump-embeddings out.h5`; teeing must not perturb the live result.
|
||||
|
||||
Schema owned by [`scripts/optimizer/SCHEMA.md`](../scripts/optimizer/SCHEMA.md).
|
||||
@@ -1602,14 +1080,6 @@ Verify `landmarks`/`bbox`/`is_cut` suffice, and bump `schema_version` if not.
|
||||
Quantify where ArcFace degrades, replacing the 66×66 estimate in A1 with a
|
||||
measurement.
|
||||
|
||||
> **Result, and its limit.** Knee at 24–32 px; 32 px returns 98.1% TPI at 0.0
|
||||
> FPI. But the probe is an already-aligned 112×112 crop, so alignment is held
|
||||
> perfect and this measures the **embedder alone** — an upper bound, not a
|
||||
> threshold. **VR-013** re-asks the question end to end, downscaling the whole
|
||||
> frame before the detector, and lands near 50 px. AR-002's floor of 40 px comes
|
||||
> from VR-013; this study is what shows how much of the gap is detection and
|
||||
> landmark error rather than embedding.
|
||||
|
||||
**Method.**
|
||||
|
||||
1. Select ~100 gallery actors having more than one mugshot.
|
||||
|
||||
+8
-79
@@ -243,9 +243,7 @@ Context crops opt-in behind `--dump-unidentified-crops`.
|
||||
|
||||
**Depends on:** nothing (AR-002, AR-011); AR-004 blocks AR-003.
|
||||
|
||||
- **AR-002** — `min_face_px` stays **40** (VR-013 measured it end to end) but must
|
||||
be expressed in original resolution rather than decoded-frame space. The value
|
||||
is already right in `config.hpp`; the change is the coordinate space.
|
||||
- **AR-002** — `min_face_px` → 66, expressed in original resolution.
|
||||
- **AR-011** — feed TransNetV2 at native rate; derive the dedup window from
|
||||
source fps rather than the hardcoded `0.04 s`.
|
||||
- **AR-004** — backpressure. `kMaxFaces` (`identity_matcher_node.hpp:133`)
|
||||
@@ -262,23 +260,13 @@ Context crops opt-in behind `--dump-unidentified-crops`.
|
||||
|
||||
# Gallery
|
||||
|
||||
## GR-004 — Model binding — **DONE**
|
||||
## GR-004 — Model binding
|
||||
|
||||
**Depended on:** nothing. Landed before any measurement work, as intended.
|
||||
**Depends on:** nothing. **Startable immediately, highest value per line.**
|
||||
|
||||
Stamp = model basename + SHA-256 of the ONNX, written as the `/embedder` group at
|
||||
build time (`gallery_builder.cpp`, `sae_gallery.save_gallery_hdf5`) and verified
|
||||
at load in `scene_analyze`, `scene_preview`, the `sae_kpn` matcher binding,
|
||||
`replay.py`, `optimize.py` and `movienet_eval.py`. Mismatch is a hard error naming
|
||||
both sides, with no bypass. Embedding dumps carry the same stamp, since a replay
|
||||
has no live embedder to check against.
|
||||
|
||||
Unstamped legacy galleries **warn loudly and proceed** rather than failing:
|
||||
unknown is not known-bad, and hard-failing every pre-existing gallery would turn
|
||||
the check into something people disable. `--require-gallery-stamp` /
|
||||
`SAE_REQUIRE_GALLERY_STAMP=1` promotes that to a hard error — measurement runs
|
||||
should set it. `scripts/stamp_gallery.py` re-binds an existing gallery without
|
||||
re-embedding, so the warning state is cheap to leave.
|
||||
Stamp embedder identity into the gallery at build; verify at load in
|
||||
`scene_analyze`, `replay.py` and the optimizer. Mismatch is a hard error naming
|
||||
both sides.
|
||||
|
||||
Cross-model similarities are meaningless but *look* plausible — this fails
|
||||
silently and expensively, and it would corrupt every measurement taken during the
|
||||
@@ -326,67 +314,8 @@ Windows carry belief and route; `extraction.*` gains `extinction_sec` and
|
||||
|
||||
## VR-005 — Minimum face size study
|
||||
|
||||
**Depends on:** nothing. Standalone Python, no C++ contact. **Done** — knee at
|
||||
24–32 px. It measures the embedder with alignment held perfect, so it bounds the
|
||||
answer from below rather than setting it; AR-002's floor comes from **VR-013**,
|
||||
which sweeps input resolution end to end and lands at 40 px.
|
||||
|
||||
## VR-013 — Cross-source identification probe
|
||||
|
||||
**Depends on:** `sae_embed` exposing `detect()`, `align_face()`, `embed_crop()`
|
||||
and the gallery calibration — it drives the shipped C++ rather than reimplementing
|
||||
it, which is what VR-005 could not do.
|
||||
|
||||
Gallery from one recording, probes from another, sweeping the probe's **input
|
||||
resolution before the detector**, so detection and landmark regression degrade
|
||||
with the frame. `experiments/xsource/`.
|
||||
|
||||
**Findings.** Holding 90% of the plateau needs ~50 px end to end against VR-005's
|
||||
~22 px; `min_face_px` 40 is right and 32 would admit faces in the falling region.
|
||||
FPI is 0.0% at every scale — resolution loss goes entirely to TBI, never to a
|
||||
wrong name. The ceiling is **cross-view, not resolution**: everyone matches
|
||||
themselves within a recording (0.55–0.85) and collapses across two (0.14–0.45),
|
||||
and only the subject with frontal *gallery* references identified reliably — so
|
||||
the lever is gallery pose coverage (`docs/pose-expansion.md`), not a better
|
||||
landmark model. Averaging SCRFD's NMS-discarded landmark estimates lifts
|
||||
cross-clip TPI 41% → 49% for one forward pass.
|
||||
|
||||
**Open.** Four identities and one shoot, so the shape is the result and the
|
||||
absolute rates are not. Both clips hold all four people, so there is no
|
||||
out-of-gallery class and the 10×-weighted out-of-cast misID is untested — holding
|
||||
one identity out of the gallery would fix that.
|
||||
|
||||
## VR-014 — Audio-signature offset recovery
|
||||
|
||||
**Depends on:** `sae_audio` exposing `compute_signature()` and
|
||||
`signature_from_mono()` — it drives the shipped C++, as VR-013 does, so the
|
||||
thing measured is the thing that ships.
|
||||
|
||||
`scripts/validation/test_audio_offset.py` over
|
||||
`tests/fixtures/audio/superhero_offset_200s.flac`: 200 s of public-domain film audio
|
||||
(the same SuperHero clips the replay fixtures use), long enough for a 120 s
|
||||
window to slide past the ±600-frame search cap. The slide itself is numpy here
|
||||
on purpose — matching belongs to the consumer, so writing it out keeps this a
|
||||
test of the signature rather than of somebody's matcher.
|
||||
|
||||
**Findings.** Alignment is a solved problem here: the offset is the nearest frame
|
||||
in every in-cap trial, worst error **46 ms against a 500 ms budget**, and 46 ms is
|
||||
the quantisation floor — offsets are whole 92.88 ms frames, so no correct answer
|
||||
can be worse. The `runtime/2` anchor's factor of two holds through real trimmed
|
||||
files, and out-of-cap offsets and unrelated content are both declined.
|
||||
|
||||
**The score is where the slack is, and it costs a tier rather than accuracy.** It
|
||||
tracks sub-frame misalignment — 0.94–0.99 near a frame boundary, 0.69–0.73 at
|
||||
half a frame — so two thirds of correct alignments miss the server's 0.85 `audio`
|
||||
threshold and land in `loose`. UT-108 measures the fix rather than proposing one:
|
||||
±1 frame of slack in the score returns all 40 to `audio` (min 0.906) with false
|
||||
matches unmoved at 0.12–0.16, costing 81 ms of the budget. See
|
||||
[`SPEC.md`](SPEC.md) IR-004 — the score is normative in the server spec, so the
|
||||
change is theirs to make.
|
||||
|
||||
**Open.** One source, one language, one era of recording. The shape (offset exact,
|
||||
score set by sub-frame phase) should hold generally, but the absolute scores are
|
||||
this fixture's.
|
||||
**Depends on:** nothing. Standalone Python, no C++ contact. Produces the measured
|
||||
value replacing AR-002's 66 px estimate.
|
||||
|
||||
## VR-001 — Dump audit
|
||||
|
||||
|
||||
+48
-164
@@ -29,35 +29,32 @@ Status: `Done` · `In Progress` · `Planned` · `TBD` · `Withdrawn`
|
||||
| ID | Requirement | Traces to | Priority | Status |
|
||||
|---|---|---|---|---|
|
||||
| AR-001 | Detect faces in sampled frames; emit bbox, confidence, 5-point landmarks in original pixel space | SR-002 | High | Done |
|
||||
| AR-002 | Minimum face size **40×40 px** (VR-013 measured end to end; VR-005's 32 px is an embedder-only upper bound), expressed in **original** resolution (decoupled from `dense_scale`) | SR-002 | High | Planned |
|
||||
| AR-003 | No fixed per-frame face cap — crowd scenes must not lose background cast | SR-002 | Medium | **Done** — `max_faces` defaults to 0 (no cap); the matcher batches through its GEMM buffer instead of throwing |
|
||||
| AR-004 | Backpressure: unbounded faces/frame absorbed by slowing, never by dropping or throwing | SR-002 | High | **Mostly** — node outputs *park* on a full channel: the value is held, the worker released, and a channel space-callback resumes the node. Replaces `push_blocking`, which parked a scheduler worker inside the push and, with one thread per node, stopped that node draining its own input. Verified: 385/385 frames, 0 drops. **Gap:** a rare hang survives, ~1 run in 20 at a 300 s timeout (was: every run). `FanoutNode` still drops on overflow (`fanout.hpp:129`) rather than parking, so the AR-010 scene join sheds frames exactly when the dense branch falls behind |
|
||||
| AR-005 | Align to 112×112 via ArcFace 5-point similarity transform, fitted by **Umeyama least squares over all five points** (as InsightFace does) — never a robust fit, which would discard the landmarks AR-030 reads | SR-002 | High | **Done** — `umeyama_similarity()`. The RANSAC fit it replaces disagreed by a median 17 source px on 400 headshots, 83.5% of crops embedding below cos 0.99, and was unstable and RNG-driven: rebuilding caught 1614 near-duplicates against the original build's ~100. **All galleries rebuilt** (2456 actors, 10254 embeddings); measured separation gain is small (0.583 → 0.590), so recorded accuracy figures should be re-run but are not expected to move far |
|
||||
| AR-002 | Minimum face size 66×66 px, expressed in **original** resolution (decoupled from `dense_scale`) | SR-002 | High | Planned |
|
||||
| AR-003 | No fixed per-frame face cap — crowd scenes must not lose background cast | SR-002 | Medium | Planned |
|
||||
| AR-004 | Backpressure: unbounded faces/frame absorbed by slowing, never by dropping or throwing | SR-002 | High | Planned |
|
||||
| AR-005 | Align to 112×112 via ArcFace 5-point similarity transform | SR-002 | High | Done |
|
||||
| AR-006 | 512-d L2-normalised embeddings, batched | SR-002 | High | Done |
|
||||
| AR-007 | Associate detections by IoU + embedding, with **frame-dependent** weighting | SR-002 | High | **Done** — `track_alpha` is the base for ordinary frames; drops to embedding-only on cut/boundary and for dormant tracks |
|
||||
| AR-008 | One track pool keyed on `last_seen`; no separate revival path | SR-002 | High | **Done** — one pool keyed on `last_seen`; park/revive branch deleted |
|
||||
| AR-007 | Associate detections by IoU + embedding, with **frame-dependent** weighting | SR-002 | High | In Progress |
|
||||
| AR-008 | One track pool keyed on `last_seen`; no separate revival path | SR-002 | High | Planned |
|
||||
| AR-009 | Camera-cut detection (histogram) as an association hint | SR-002 | High | Done |
|
||||
| AR-010 | Scene-boundary detection (TransNetV2) as an association hint | SR-002 | Medium | **Done** — decode butterfly joined via `SceneBoundaries`; the sampled branch waits for the detector's watermark. Frames past its last scored window are counted as unverified, never assumed boundary-free |
|
||||
| AR-010 | Scene-boundary detection (TransNetV2) as an association hint | SR-002 | Medium | In Progress |
|
||||
| AR-011 | **Every model is fed the input it was trained for** — cost reduced by running less often, never by degrading one inference | SR-002 | High | Planned |
|
||||
| AR-012 | Presence follows **track extent**, not per-frame recognition | **SR-002** | High | **Done** — `src/track_registry.hpp`; window is `[first_seen, last_seen]` of an owned track |
|
||||
| AR-013 | `last_seen` optional state machine; window ends at last sighting, never after | SR-002 | High | **Done** — `last_seen` optional is the whole state machine; interior gaps absorbed, trailing cool-down never claimed |
|
||||
| AR-014 | Belief swap A→B terminates the track and starts a new one | SR-002 | Medium | **Done** — swap closes at `last_seen` and opens a successor at the swap frame; counted |
|
||||
| AR-015 | Two live tracks owned by one actor ⇒ treat as a detected cut, re-associate | SR-002 | Medium | **Done** — reverse index detects it on the causing update; counted |
|
||||
| AR-016 | All tracks closed at EOF — a film ends with faces on screen | SR-002 | High | **Done** — `flush()`, idempotent, closes at last sighting or final tick |
|
||||
| AR-017 | Every presence claim carries its belief and identification route | SR-002 | High | **Done** — `DeadTrack` carries belief and observation count |
|
||||
| AR-018 | Per-subject embedding store with banded admission (novel enough, safe enough) | SR-005 | Medium | **Done** — banded admission in probability space, bounds from `expand_band_lo/hi`; the lower bound re-asked pairwise at promotion, since `admit` compares only against the nearest member and a drifting track can chain past it. Retires `expand_novelty_sim` and `expand_track_spread_max` — novelty is now the eviction ordering, not a threshold. Rejections counted. Bounds unswept (VR-007) |
|
||||
| AR-019 | Per-film gallery annex from owned tracks; acquires the non-frontal views TMDB lacks | SR-005 | Medium | **Done** — all three discontinuity signals clear the buffers; ownership comes from the registry, not a second local tally |
|
||||
| AR-012 | Presence follows **track extent**, not per-frame recognition | **SR-002** | High | Planned |
|
||||
| AR-013 | `last_seen` optional state machine; window ends at last sighting, never after | SR-002 | High | Planned |
|
||||
| AR-014 | Belief swap A→B terminates the track and starts a new one | SR-002 | Medium | Planned |
|
||||
| AR-015 | Two live tracks owned by one actor ⇒ treat as a detected cut, re-associate | SR-002 | Medium | Planned |
|
||||
| AR-016 | All tracks closed at EOF — a film ends with faces on screen | SR-002 | High | Planned |
|
||||
| AR-017 | Every presence claim carries its belief and identification route | SR-002 | High | Planned |
|
||||
| AR-018 | Per-subject embedding store with banded admission (novel enough, safe enough) | SR-005 | Medium | Planned |
|
||||
| AR-019 | Per-film gallery annex from owned tracks; acquires the non-frontal views TMDB lacks | SR-005 | Medium | In Progress |
|
||||
| AR-020 | Deferred re-identification of unknown tracks against the final expanded gallery | SR-005 | High | Planned |
|
||||
| AR-021 | Cluster unknown tracks into one entity per person, under temporal cannot-link constraints | SR-005 | Medium | Planned |
|
||||
| AR-022 | Capture still-unidentified tracks: embeddings, metadata, **context crops** for human review | §4 | Medium | Planned |
|
||||
| AR-023 | Fit sigmoid calibration from intra/inter similarity distributions | SR-002 | High | Done |
|
||||
| AR-024 | **Always the calibrated probability, never a raw cosine** — exceptions recorded | SR-002 | High | **Done** — association, accumulation and expansion all in probability space; `track_max_embed_dist`, `cut_revive_sim`, `expand_novelty_sim`, `expand_track_spread_max` retired |
|
||||
| AR-025 | Per-track Bayesian accumulation in log-odds, with correlated-observation discounting | SR-002 | High | **Done** — log-odds accumulation with correlation discounting owned by the registry, `src/evidence_discount.hpp` |
|
||||
| AR-024 | **Always the calibrated probability, never a raw cosine** — exceptions recorded | SR-002 | High | Planned |
|
||||
| AR-025 | Per-track Bayesian accumulation in log-odds, with correlated-observation discounting | SR-002 | High | Planned |
|
||||
| AR-026 | All similarity computed as GEMM, including annex and deferred pass | SR-001 | High | In Progress |
|
||||
| AR-027 | Throughput acceptable for **arbitrary** gallery size | SR-001 | High | Planned |
|
||||
| AR-028 | **Embedding input quality assessed and carried** — every face scored on size, sharpness and visibility before its embedding is used as identity evidence; the vector travels with the face and reaches the VR-001 dump | SR-002 | High | Planned |
|
||||
| AR-029 | Sharpness measure on the **aligned crop** (scale-normalised, so it cannot re-measure size) | SR-002 | Medium | Planned |
|
||||
| AR-030 | Visibility measure from the AR-001 5-point landmarks — extreme pose or occlusion **discounts the observation, never deletes the detection** | SR-002 | Medium | **In Progress** — measure is the AR-005 alignment residual (`estimate_alignment()`), carried on `DetectedFace`; roll/scale invariance and monotonicity under foreshortening asserted. Nothing consumes it as a discount yet |
|
||||
|
||||
## Deployment (DP)
|
||||
|
||||
@@ -69,20 +66,18 @@ Status: `Done` · `In Progress` · `Planned` · `TBD` · `Withdrawn`
|
||||
| DP-004 | Opportunistic/idle mode: external trigger, hard stop, implicit re-queue | PR-004 | Medium | Planned |
|
||||
| DP-005 | Native installer, no Docker; Fedora + Arch | PR-004 | Medium | Planned |
|
||||
| DP-006 | Background incremental gallery refresh on a timer | PR-003 | Medium | Planned |
|
||||
| DP-007 | CI builder image, CPU-only, pinned by tag in the Gitea container registry | PR-004 | High | Planned |
|
||||
| DP-008 | Builder images + release jobs per backend (cpu / cuda / rocm); ship binaries, not engines | PR-004 | Medium | Planned |
|
||||
|
||||
## Integration (IR)
|
||||
|
||||
| ID | Requirement | Traces to | Priority | Status |
|
||||
|---|---|---|---|---|
|
||||
| IR-001 | Emit the JRay truth format as sibling `.jray.json` | SR-003 | High | Done |
|
||||
| IR-002 | Windows carry belief + route; `extraction.*` carries `extinction_sec`, `gallery_scope` | SR-003 | High | **Done** — `schema_version: 2`; windows are objects with `belief` + `route`; `extraction.*` carries `extinction_sec` and `gallery_scope`; `anneal_sec` removed |
|
||||
| IR-003 | Output written **after** the deferred pass, not at EOF | SR-003 | High | **In Progress** — sink builds windows from registry claims and flushes at EOF; the deferred pass (AR-020) does not exist yet, so output is still final at EOF |
|
||||
| IR-004 | Compute the audio signature exactly per server spec §3 | SR-003 | Medium | **Done** — `src/audio_signature.*`; not yet emitted into the truth file (IR-002) |
|
||||
| IR-005 | Golden-vector fixture shared with the plugin repo to prove bit-exactness | SR-003 | High | **Done** — `tests/fixtures/audio/`; v1 parameters now normative in server spec §3 |
|
||||
| IR-007 | Media < 120 s: emit no signature, apply no sync offset — identical rule in both producers | SR-003 | Low | **Done** |
|
||||
| IR-008 | Emit and honour the signature's own `v1:` version prefix | SR-003 | Low | **Done** |
|
||||
| IR-002 | Windows carry belief + route; `extraction.*` carries `extinction_sec`, `gallery_scope` | SR-003 | High | Planned |
|
||||
| IR-003 | Output written **after** the deferred pass, not at EOF | SR-003 | High | Planned |
|
||||
| IR-004 | Compute the audio signature exactly per server spec §3 | SR-003 | Medium | Planned |
|
||||
| IR-005 | Golden-vector fixture shared with the plugin repo to prove bit-exactness | SR-003 | High | Planned |
|
||||
| IR-007 | Media < 120 s: emit no signature, apply no sync offset — identical rule in both producers | SR-003 | Low | Planned |
|
||||
| IR-008 | Emit and honour the signature's own `v1:` version prefix | SR-003 | Low | Planned |
|
||||
| IR-006 | Jellyfin round-trip: pull pending queue, push complete results only | SR-001 | High | Done |
|
||||
|
||||
## Gallery (GR)
|
||||
@@ -91,8 +86,8 @@ Status: `Done` · `In Progress` · `Planned` · `TBD` · `Withdrawn`
|
||||
|---|---|---|---|---|
|
||||
| GR-001 | Build gallery from Jellyfin library cast, TMDB profile fallback | SR-001, SR-005 | High | Done |
|
||||
| GR-002 | Incremental `--merge` refresh without re-embedding known actors | PR-003 | High | Done |
|
||||
| GR-003 | Report coverage: zero-image actors, under-referenced actors, dedup, calibration PDFs | SR-001 | Medium | **Done** — `gallery/gallery_report.hpp`, written next to the gallery by `build_gallery`. Zero-usable-image actors come from the build audit, which a stored gallery cannot reconstruct; also `distinct_references`, `duplicates_removed`, and the intra/inter distributions the calibration fits and would otherwise discard |
|
||||
| GR-004 | Stamp embedder identity into the gallery; **hard startup error** on mismatch | SR-001 | High | **Done** — basename + SHA-256 + `embed_dim`; mismatch fatal with no bypass, unstamped warns unless `--require-gallery-stamp`; `scripts/stamp_gallery.py` migrates in place |
|
||||
| GR-003 | Report coverage: zero-image actors, under-referenced actors, dedup, calibration PDFs | SR-001 | Medium | Planned |
|
||||
| GR-004 | Stamp embedder identity into the gallery; **hard startup error** on mismatch | SR-001 | High | Planned |
|
||||
| GR-005 | Gallery data never leaves the instance | **SR-005** | High | Done |
|
||||
| GR-006 | Provenance tiers: baked / harvested / confirmed, distinguishable per embedding | SR-005 | High | Planned |
|
||||
| GR-007 | Persist harvested embeddings **flagged and reviewable**, never silently equal to baked | SR-005 | Medium | Planned |
|
||||
@@ -104,19 +99,14 @@ Status: `Done` · `In Progress` · `Planned` · `TBD` · `Withdrawn`
|
||||
| ID | Requirement | Traces to | Priority | Status |
|
||||
|---|---|---|---|---|
|
||||
| VR-001 | HDF5 post-inference dump at the embedded-frame boundary | PR-002 | High | Done |
|
||||
| VR-002 | Replay drives the **real** KPN nodes, not a reimplementation | PR-002 | High | **Done** — replay driven from committed fixtures in `tests/test_replay_fixtures.cpp`; determinism asserted |
|
||||
| VR-002 | Replay drives the **real** KPN nodes, not a reimplementation | PR-002 | High | Done |
|
||||
| VR-003 | Scoring: micro-F1 against X-Ray, precision/recall logged at every evaluation | PR-002 | High | Done |
|
||||
| VR-004 | Reproducible validation corpus with ground truth | PR-002 | High | Done |
|
||||
| VR-005 | Minimum face size study — TPI/FPI vs probe size, gallery held at native res | PR-002 | Medium | **Done** — knee at 24–32 px; 32 px gives 98.1% TPI, 0.0 FPI at every size. Degrades an already-aligned 112×112 crop, so it isolates the embedder and is an **upper bound**; VR-013 measures the same question end to end and AR-002 takes its number, not this one |
|
||||
| VR-005 | Minimum face size study — TPI/FPI vs probe size, gallery held at native res | PR-002 | Medium | Planned |
|
||||
| VR-006 | Re-tune `scene_threshold` once native-rate decode lands | PR-002 | Low | Planned |
|
||||
| VR-007 | Expansion band, clustering threshold, and deferred-pass ablation | PR-002 | Medium | Planned |
|
||||
| VR-008 | Gallery scaling benchmark — throughput vs gallery size | PR-002 | Medium | Planned |
|
||||
| VR-009 | Verify accumulated posteriors are calibrated against held-out tracks | PR-002 | High | Planned |
|
||||
| VR-010 | Dump provenance attributes — embedder model, detector settings, `dense_scale`, `scene_detect`, sample rate | PR-002 | **High** | Planned |
|
||||
| VR-011 | Rewrite the replay harness for the post-AR-012 output contract | PR-002 | High | Planned |
|
||||
| VR-012 | Quality-knee study — TPI/FPI vs sharpness and vs pose, as VR-005 did for size; also settles whether the 5-point pose proxy needs a dedicated landmark model | PR-002 | Medium | Planned |
|
||||
| VR-014 | Audio-signature **offset recovery on real content** — a known trim recovered from film audio, not from the synthetic golden tone | PR-002 | Medium | **Done** — 40 random in-cap offsets, every one recovered to the nearest frame: **worst error 46 ms against a 500 ms budget**, and 46 ms is the floor rather than a result, since the offset is quantised to whole 92.88 ms frames. The `runtime/2` anchor confirmed through real head-trimmed files (a `delta` trim moves the window by `delta/2`). The one soft spot is **tier labelling, not accuracy**: the score falls with sub-frame misalignment (0.94–0.99 near a frame boundary, 0.69–0.73 at half a frame), so 27/40 correct alignments were demoted to `loose`. ±1 frame of slack in the *score* fixes it — measured, all 40 back to `audio` (min 0.906), false matches unmoved at 0.12–0.16, costing 81 ms of the budget |
|
||||
| VR-013 | Cross-source identification probe — gallery from one recording, probes from another, swept over input resolution end to end | PR-002 | Medium | **In Progress** — holding 90% of the plateau needs ~50 px end to end against VR-005's ~22 px, the gap being detection and landmark error; **`min_face_px` 40, since 32 admits faces in the falling region** (AR-002). FPI 0.0% at every scale. Ceiling is cross-view, not resolution |
|
||||
|
||||
---
|
||||
|
||||
@@ -130,62 +120,21 @@ Four tiers, in decreasing order of preference:
|
||||
|
||||
| Tier | Runs in CI | What it covers |
|
||||
|---|---|---|
|
||||
| **T1 — Functor unit** | Yes | A KPN node's `operator()` driven directly with hand-built inputs |
|
||||
| **T2 — Replay** | Yes | The composed pipeline driven from an HDF5 fixture — no GPU, no video |
|
||||
| **T1 — CPU unit** | Yes | Pure logic: registry state machine, belief accumulation, clustering, band admission, calibration maths |
|
||||
| **T2 — Replay** | Yes | Real pipeline nodes driven from an HDF5 fixture — no GPU, no video |
|
||||
| **T3 — CPU inference** | Yes, slowly | ORT CPU provider over a handful of frames; smoke tests only |
|
||||
| **T4 — GPU** | **No** | Throughput, TRT engines, large-gallery GEMM |
|
||||
|
||||
### T1 is the primary tier, and KPN is why
|
||||
**T2 is the reason this is workable.** The HDF5 dump (VR-001) captures state
|
||||
after decode → detect → align → embed and before tracking and matching, so
|
||||
everything downstream — which is where nearly all of the new design lives — is
|
||||
cheap CPU maths replayable from a fixture. Tracking, presence windows, belief
|
||||
accumulation, expansion, deferred re-identification and clustering are all
|
||||
verifiable on an N100 at full fidelity, not in miniature.
|
||||
|
||||
**Node functors are plain callable structs, constructed independently of the
|
||||
network that wraps them** (`main.cpp:186-207` builds them as stack objects;
|
||||
`ObjectNode` merely adapts them). So a node is testable by constructing it and
|
||||
calling `operator()` — no channels, no threads, no network, no fixture.
|
||||
|
||||
This is already the established pattern, not a proposal:
|
||||
`tests/test_face_tracker.cpp` "drives the node's `operator()` with hand-built
|
||||
`EmbeddedSceneFrame`s and inspects the emitted `track_ids`", and does so
|
||||
"pure, GPU-free, model-free".
|
||||
|
||||
The consequence is that most of the redesign is verifiable **without any
|
||||
fixture at all**: construct exactly the awkward state — a belief swap, two live
|
||||
tracks converging on one actor, a film ending mid-track, a gap one frame under
|
||||
the timeout — rather than hunting for a clip that happens to exhibit it.
|
||||
|
||||
Four hazards this removes outright:
|
||||
|
||||
- **No fixture-provenance risk** for these tests — the inputs are synthetic and
|
||||
explicit.
|
||||
- **No "fixture must be replayed from frame 0"** concern — state is constructed
|
||||
directly.
|
||||
- **No cross-test state leakage** (e.g. a tracker's `next_id_` persisting) — each
|
||||
test constructs a fresh functor.
|
||||
- **No replay-harness nondeterminism** — no channels, so no EOF-tail heuristics
|
||||
or silent drops.
|
||||
|
||||
It also means **a dead upstream producer does not block testing a downstream
|
||||
consumer.** `is_scene_boundary` currently has no producer (see AR-010), which
|
||||
would make a *replay* test of the frame-dependent `track_alpha` pass vacuously —
|
||||
but a T1 test simply constructs a frame with `is_scene_boundary = true` and
|
||||
asserts the weighting changes. The producer gap is a pipeline defect to fix, not
|
||||
a verification blocker.
|
||||
|
||||
### T2 covers what T1 cannot
|
||||
|
||||
Replay remains necessary for **composition** — that the nodes wired together
|
||||
behave as the sum of their parts — and for realistic data at scale, which
|
||||
synthetic inputs cannot honestly imitate. It is the tier that would catch a
|
||||
wiring error, a channel-capacity problem, or an ordering assumption that only
|
||||
appears under concurrency.
|
||||
|
||||
The HDF5 dump (VR-001) captures state after decode → detect → align → embed, so
|
||||
replay needs no GPU and no video. That was built for the optimizer; it doubles as
|
||||
CI, which is a strong argument for keeping the schema honest and for replay
|
||||
driving the *real* nodes rather than a reimplementation (VR-002).
|
||||
|
||||
**Fixtures and studies are generated locally**, on the development machine where
|
||||
the models, galleries and media already exist. CI consumes them; it never
|
||||
produces them.
|
||||
That was already true for the optimizer. It now doubles as the CI strategy, which
|
||||
is a strong argument for keeping the dump schema honest (VR-001) and for the
|
||||
replay driving the *real* nodes rather than a reimplementation (VR-002).
|
||||
|
||||
**Small committed fixtures are required.** A few HDF5 dumps covering the awkward
|
||||
cases — a cut, a belief swap, two live tracks converging, a film ending
|
||||
@@ -213,54 +162,12 @@ as such rather than counted as covered.
|
||||
| GR-001 … GR-005 | T1 + T3 | Gallery assembly is I/O and bookkeeping; embedding is T3 smoke |
|
||||
| GR-006 … GR-008 | T1 | Tiering and outlier detection operate on stored embeddings |
|
||||
| VR-* | Out of CI | Studies are run deliberately and their results committed as documents |
|
||||
| VR-014 | **T2** | The exception, and the reason the blanket row above is not the whole story: its fixture is committed and its signature is CPU-only DSP, so the study *is* a test a CI host can run — not a measurement someone has to remember to repeat |
|
||||
|
||||
**One consequence worth stating:** AR-027 (arbitrary gallery scale) is
|
||||
structurally unverifiable on the CI host. It needs a GPU host and a synthetic
|
||||
large gallery, so it is the requirement most likely to silently regress. Its
|
||||
benchmark (VR-008) should run on a schedule rather than on demand.
|
||||
|
||||
### CI never calls a model
|
||||
|
||||
**Not "should not" — cannot.** The N100 has no GPU, and even the ONNX Runtime CPU
|
||||
provider is impractical: a measured run of the embedder on this hardware sits at
|
||||
~930 ms per frame, so a 77 s clip at 5 fps would take roughly six minutes of
|
||||
inference alone. Every model invocation therefore happens **locally, ahead of
|
||||
time**, and CI consumes the result as data.
|
||||
|
||||
This is what makes the T1/T2 split load-bearing rather than a preference: T1 and
|
||||
T2 are the only tiers that can exist in CI at all.
|
||||
|
||||
### Fixture corpus — `hero/`
|
||||
|
||||
Five clips of **SuperHero (1952)**, ~77 s each, 480×360, 30 fps, 42 MB total.
|
||||
|
||||
Public domain, and that is the reason to use it rather than a convenience:
|
||||
**derived fixtures — dumps, crops, golden outputs — can be committed without the
|
||||
rights question that rules out sharing gallery data (SR-005).** A fixture cut
|
||||
from a copyrighted title could not live in the repository at all.
|
||||
|
||||
Two properties to design around rather than discover:
|
||||
|
||||
- **480×360 means small faces.** At this resolution a face is often 40–80 px, so
|
||||
the AR-002 minimum of 40 px (original resolution) sits at the very bottom of
|
||||
that range: the filter is close to binding, and anything shot wider is lost.
|
||||
Fixture generation must set `--min-face-px` explicitly and record it, or the
|
||||
dumps will be sparse for reasons unrelated to what is being tested.
|
||||
- **77 s is short.** At 1 fps that is 77 frames — too thin to exercise an
|
||||
extinction window measured in tens of seconds. Generate at 5 fps (≈385 frames,
|
||||
~1 MB) and record the rate in provenance, since the behaviour under test
|
||||
changes with it.
|
||||
|
||||
> **AR-004 blocks reproducible fixture generation.** A trial run of one clip
|
||||
> produced 49 frames of an expected ~385, ending at 51 s of 77 s, with the
|
||||
> diagnostics reporting 285 frames dropped at `camera_pos` and 51 at
|
||||
> `face_aligner`. Channels overflow and **drop** rather than blocking, and what
|
||||
> gets dropped depends on timing — so the same command run twice can produce
|
||||
> different dumps. Golden fixtures cannot be built on that. AR-004 is therefore
|
||||
> a prerequisite for VR-001 fixtures, not merely a throughput concern for crowd
|
||||
> scenes.
|
||||
|
||||
### Fixtures — precomputed inference, pulled by CI
|
||||
|
||||
The N100 cannot run inference at any useful rate, so **inference output is
|
||||
@@ -269,29 +176,16 @@ what looks like GPU work into pure CPU replay.
|
||||
|
||||
| Fixture | Contents | Size | Storage |
|
||||
|---|---|---|---|
|
||||
| **Edge-case dumps** | ~6 short clips (30–60 s), one per awkward behaviour | ~0.1–1 MB each | **Committed in-repo** |
|
||||
| **Corpus dumps** | Full-length titles from the validation corpus | ~21–38 MB each | **Gitea package registry**, pinned by version + checksum |
|
||||
| **Edge-case dumps** | ~6 short clips (30–60 s), one per awkward behaviour | ~1 MB each | **Committed in-repo** |
|
||||
| **Corpus dumps** | Full-length titles from the validation corpus | ~30 MB each | Pinned artifact, fetched by checksum |
|
||||
| **Synthetic gallery** | Random unit-norm embeddings, fixed seed | small | Generated at test time |
|
||||
| **Golden truth files** | Expected output for each edge-case dump | KB | Committed |
|
||||
| **Audio golden vectors** | FLAC + expected signature + parameter contract | ~600 KB | Committed, **shared with the plugin repo** |
|
||||
| **Audio golden vectors** | Short WAV + expected signature | KB | Committed, **shared with the plugin repo** |
|
||||
|
||||
Edge-case dumps are small enough to commit, and being in-repo means they version
|
||||
with the code that reads them.
|
||||
|
||||
**Corpus dumps go to the Gitea package registry, not Git LFS.** Both are
|
||||
available — the models already use LFS — but their fetch semantics differ in a
|
||||
way that matters here. LFS objects are pulled on clone unless a developer
|
||||
explicitly skips them, so ~38 MB per title behind LFS taxes everyone who clones,
|
||||
forever, for data that only CI and the optimizer ever read. Registry artifacts
|
||||
are fetched on demand by the job that needs them.
|
||||
|
||||
Rule of thumb: **LFS for what the build needs; the package registry for what a
|
||||
particular job needs.** Models are the former; corpus dumps and the CI image
|
||||
(DP-007) are the latter.
|
||||
|
||||
Pin by version and verify by checksum on fetch. A fixture that changes silently
|
||||
under CI is worse than a missing one, because the failure presents as a code
|
||||
regression.
|
||||
with the code that reads them. Corpus dumps are pulled by pinned checksum from
|
||||
the artifact store rather than committed, since they are large and change only
|
||||
when the dump schema does.
|
||||
|
||||
**Generation must be reproducible and versioned.** A script, run on a GPU host,
|
||||
regenerates every fixture from source clips; it is re-run when the VR-001 schema
|
||||
@@ -310,10 +204,10 @@ because it will be trusted.
|
||||
| ID | Tier | Test asserts | Edge cases to cover |
|
||||
|---|---|---|---|
|
||||
| AR-001 | T3 | Detector returns plausible boxes on a known frame | — smoke only |
|
||||
| AR-002 | T2 | Faces below 40 px (original res) are dropped | Exactly at threshold; with `dense_scale` 0.5 — the interaction that motivated the requirement |
|
||||
| AR-002 | T2 | Faces below 66 px (original res) are dropped | Exactly at threshold; with `dense_scale` 0.5 — the interaction that motivated the requirement |
|
||||
| AR-003 | T2 | No cap applied; a 40-face frame yields 40 | Crowd frame |
|
||||
| AR-004 | T1 | Saturated input blocks rather than drops or throws | Bounded queue at capacity; **byte-based** limit with large crops; SIGTERM mid-block |
|
||||
| AR-005 | T1 | Known landmarks → expected 112×112 warp; the fit never mirrors | Landmarks near frame edge; degenerate/collinear points; a mirrored set — SVD returns a reflection unless the determinant guard rejects it |
|
||||
| AR-005 | T1 | Known landmarks → expected 112×112 warp | Landmarks near frame edge; degenerate/collinear points |
|
||||
| AR-006 | T3 | Embeddings are unit-norm | Batch smaller than, equal to, larger than `embed_batch_size` |
|
||||
| AR-007 | T2 | Association picks the right track | Two faces crossing paths; one leaving frame as another enters |
|
||||
| AR-008 | T2 | One pool; dormant tracks match on embedding, not IoU | Dormant track whose old bbox overlaps a *different* new face — must not match on position |
|
||||
@@ -335,20 +229,10 @@ because it will be trusted.
|
||||
| AR-025 | T1 | Log-odds accumulate; correlated frames discounted | 30 identical frames must **not** reach the certainty of 30 diverse ones |
|
||||
| AR-026 | T1 + T4 | GEMM path produces same result as reference loop | Equivalence on small input in CI; throughput on GPU host |
|
||||
| AR-027 | **T4** | Throughput at 10²…10⁵ actors | Scheduled, not on-demand |
|
||||
| AR-028 | **T2** | No embedding reaches the matcher unscored; the vector survives into the dump | Face failing exactly one axis; all three healthy; a face whose landmarks are degenerate — scored, not silently vanished |
|
||||
| AR-029 | T1 | Synthetic blur ladder → monotonically falling sharpness | Gaussian vs motion blur; **small sharp face vs large soft one** — size must not leak into this axis |
|
||||
| AR-030 | T1 | Alignment residual rises monotonically with foreshortening | **In-plane roll, scale and translation must leave it at zero** — the property that makes it a pose measure rather than a pose-and-everything-else measure; face size must not shift it; degenerate landmarks report not-ok rather than a number |
|
||||
| VR-012 | **T4** | Knee located per axis on held-out films | Report each candidate threshold's cost in **lost true presence**, not only its gain in precision — a gate that improves misID by discarding half the cast has not helped |
|
||||
| VR-013 | **T4** | Identification holds across two recordings of the same people, and degrades to TBI rather than to a wrong name as input resolution falls | Gallery and probes must come from *different* recordings — a hold-one-out over one recording measures a much easier problem and will not surface the cross-view failure. Ground truth is hand-sorted; labels propagated by embedding similarity would keep only the faces the embedder already gets right |
|
||||
| IR-001/002 | T1 | Serialised output matches golden file | Zero-length window; actor with many windows |
|
||||
| IR-003 | T1 | Output written after deferred pass | Not at EOF |
|
||||
| IR-004/005 | **T1** | Signature matches golden vector bit-for-bit | Identical result in both producer repos |
|
||||
| VR-014 | **T2** | A known trim offset is recovered from **real film audio**, to the nearest frame | An offset past the ±600-frame cap and unrelated content must both be *declined*, never given a best-effort alignment. Fixture and signature are both CPU-only, so unlike the other VR rows this one is CI-*executable* — though the repo's only workflow today is the traceability gate, so nothing runs it there yet. The signature comes from the shipped C++ through `sae_audio`; a numpy port would be a third implementation nobody checks against the golden vector |
|
||||
| IR-006 | T1 + manual | Queue pull and result push against a stubbed Jellyfin API | Partial result never pushed; push only after the deferred pass |
|
||||
| IR-007 | **T1** | Media < 120 s emits no signature at all | Exactly 120 s; just under; zero-length audio. Must match the plugin's cutoff exactly — a caller-varying window length is what SR-004 forbids |
|
||||
| IR-008 | T1 | `v1:` prefix emitted and honoured on read | Unknown prefix rejected, not guessed |
|
||||
| GR-009 | T1 | Human-confirmed associations persist and are tier-tagged | Survives a gallery rebuild; distinguishable from baked and harvested |
|
||||
| GR-004 | T1 | Mismatched embedder → hard startup error | Error names both sides; **unstamped warns, and errors under `SAE_REQUIRE_GALLERY_STAMP`**; same filename + different SHA-256 must still be a mismatch |
|
||||
| IR-004/005 | **T1** | Signature matches golden vector bit-for-bit | **Media < 120 s → no signature**; identical result in both repos |
|
||||
| GR-004 | T1 | Mismatched embedder → hard startup error | Error names both sides |
|
||||
| GR-008 | T1 | Outlier flagged among an actor's references | Injected poisoned embedding detected |
|
||||
| VR-009 | T1 | Posterior calibration holds | A 0.99 posterior is wrong ~1% of the time on held-out tracks |
|
||||
|
||||
|
||||
+64
-757
@@ -3,7 +3,7 @@
|
||||
<!-- GENERATED FILE - do not edit by hand. -->
|
||||
<!-- Regenerate: scripts/traceability/traceability-gate.sh -->
|
||||
|
||||
**Generated:** 2026-07-31T20:39:35+00:00
|
||||
**Generated:** 2026-07-30T16:35:36+00:00
|
||||
|
||||
Denominators are read from [`requirements.md`](requirements.md) at run time, never hardcoded. Coverage counts a requirement only when it is tagged in source **and** has a verification tier this repo's CI host can execute (`T1, T2, T3, static`).
|
||||
|
||||
@@ -11,30 +11,26 @@ Denominators are read from [`requirements.md`](requirements.md) at run time, nev
|
||||
|
||||
| Metric | Value |
|
||||
|---|---|
|
||||
| Source files scanned | 112 |
|
||||
| TRACES tags found | 148 |
|
||||
| Source files scanned | 86 |
|
||||
| TRACES tags found | 0 |
|
||||
| EXCEPTION tags found | 0 |
|
||||
| Requirements defined | 69 |
|
||||
| Requirements covered | 38 |
|
||||
| **Coverage** | **55.1%** (38/69) |
|
||||
| Coverage of CI-executable scope | 67.9% (38/56) |
|
||||
| Tagged but unexecuted in CI | 8 |
|
||||
| Requirements defined | 59 |
|
||||
| Requirements covered | 0 |
|
||||
| **Coverage** | **0.0%** (0/59) |
|
||||
| Coverage of CI-executable scope | 0.0% (0/50) |
|
||||
| Tagged but unexecuted in CI | 0 |
|
||||
| Orphan tags | 0 |
|
||||
|
||||
### By type
|
||||
|
||||
| Type | Covered | Tagged but unexecuted | Defined |
|
||||
|---|---|---|---|
|
||||
| AR | 22 | 1 | 30 |
|
||||
| DP | 2 | 0 | 8 |
|
||||
| IR | 8 | 0 | 8 |
|
||||
| GR | 5 | 0 | 9 |
|
||||
| VR | 1 | 7 | 14 |
|
||||
| AR | 0 | 0 | 27 |
|
||||
| DP | 0 | 0 | 6 |
|
||||
| IR | 0 | 0 | 8 |
|
||||
| GR | 0 | 0 | 9 |
|
||||
| VR | 0 | 0 | 9 |
|
||||
|
||||
- **UT** tags present (separate taxonomy, not counted in coverage): UT-001, UT-101, UT-102, UT-103, UT-104, UT-105, UT-106, UT-107, UT-108
|
||||
- **IT** tags present (separate taxonomy, not counted in coverage): IT-001
|
||||
- **PR** tags present (separate taxonomy, not counted in coverage): PR-002, PR-004
|
||||
- **SR** tags present (separate taxonomy, not counted in coverage): SR-001, SR-002, SR-003, SR-005
|
||||
|
||||
## Not executable in CI
|
||||
|
||||
@@ -42,21 +38,15 @@ These requirements have no verification tier this repo's CI host can run, so a t
|
||||
|
||||
| ID | Tiers | Tagged in source | Requirement |
|
||||
|---|---|---|---|
|
||||
| AR-027 | T4 | yes | Throughput acceptable for **arbitrary** gallery size |
|
||||
| VR-001 | out-of-ci | yes | HDF5 post-inference dump at the embedded-frame boundary |
|
||||
| VR-002 | out-of-ci | yes | Replay drives the **real** KPN nodes, not a reimplementation |
|
||||
| VR-003 | out-of-ci | yes | Scoring: micro-F1 against X-Ray, precision/recall logged at every eva… |
|
||||
| VR-004 | out-of-ci | yes | Reproducible validation corpus with ground truth |
|
||||
| VR-005 | out-of-ci | yes | Minimum face size study — TPI/FPI vs probe size, gallery held at nati… |
|
||||
| AR-027 | T4 | no | Throughput acceptable for **arbitrary** gallery size |
|
||||
| VR-001 | out-of-ci | no | HDF5 post-inference dump at the embedded-frame boundary |
|
||||
| VR-002 | out-of-ci | no | Replay drives the **real** KPN nodes, not a reimplementation |
|
||||
| VR-003 | out-of-ci | no | Scoring: micro-F1 against X-Ray, precision/recall logged at every eva… |
|
||||
| VR-004 | out-of-ci | no | Reproducible validation corpus with ground truth |
|
||||
| VR-005 | out-of-ci | no | Minimum face size study — TPI/FPI vs probe size, gallery held at nati… |
|
||||
| VR-006 | out-of-ci | no | Re-tune `scene_threshold` once native-rate decode lands |
|
||||
| VR-007 | out-of-ci | no | Expansion band, clustering threshold, and deferred-pass ablation |
|
||||
| VR-008 | out-of-ci | no | Gallery scaling benchmark — throughput vs gallery size |
|
||||
| VR-010 | out-of-ci | yes | Dump provenance attributes — embedder model, detector settings, `dens… |
|
||||
| VR-011 | out-of-ci | no | Rewrite the replay harness for the post-AR-012 output contract |
|
||||
| VR-012 | T4, out-of-ci | no | Quality-knee study — TPI/FPI vs sharpness and vs pose, as VR-005 did … |
|
||||
| VR-013 | T4, out-of-ci | yes | Cross-source identification probe — gallery from one recording, probe… |
|
||||
|
||||
**Tagged but unexecuted:** AR-027, VR-001, VR-002, VR-003, VR-004, VR-005, VR-010, VR-013 — a test exists and is tagged, but this CI host cannot run it. Report those runs separately.
|
||||
|
||||
## Orphan tags
|
||||
|
||||
@@ -80,750 +70,67 @@ _None._
|
||||
|
||||
| ID | Status | Tier | Traces to | Trace state | Tagged in | Requirement |
|
||||
|---|---|---|---|---|---|---|
|
||||
| AR-001 | Done | T3 | SR-002 | covered | `src/nodes/face_detector_node.hpp` | Detect faces in sampled frames; emit bbox, confidence, 5-point landma… |
|
||||
| AR-002 | Planned | T2 | SR-002 | untagged | - | Minimum face size **40×40 px** (VR-013 measured end to end; VR-005's … |
|
||||
| AR-003 | **Done** — `max_fac… | T1, T2, T4 | SR-002 | covered | `src/config.hpp`, `src/nodes/face_detector_node.hpp`, `src/nodes/identity_matcher_node.hpp` | No fixed per-frame face cap — crowd scenes must not lose background c… |
|
||||
| AR-004 | **Done** — KPN node… | T1, T4 | SR-002 | covered | `src/main.cpp`, `src/nodes/identity_matcher_node.hpp`, `tests/test_replay_fixtures.cpp` | Backpressure: unbounded faces/frame absorbed by slowing, never by dro… |
|
||||
| AR-005 | **Done** — `umeyama… | T1, T3 | SR-002 | covered | `src/face_utils.hpp`, `src/nodes/face_aligner_node.hpp`, `tests/test_face_utils.cpp` | Align to 112×112 via ArcFace 5-point similarity transform, fitted by … |
|
||||
| AR-006 | Done | T3 | SR-002 | covered | `src/nodes/embedder_node.hpp` | 512-d L2-normalised embeddings, batched |
|
||||
| AR-007 | **Done** — `track_a… | T2 | SR-002 | covered | `src/config.hpp`, `src/main.cpp`, `src/nodes/face_tracker_node.hpp`, `tests/test_face_tracker.cpp` | Associate detections by IoU + embedding, with **frame-dependent** wei… |
|
||||
| AR-008 | **Done** — one pool… | T2 | SR-002 | covered | `src/config.hpp`, `src/main.cpp`, `src/nodes/face_tracker_node.hpp`, `tests/test_face_tracker.cpp` | One track pool keyed on `last_seen`; no separate revival path |
|
||||
| AR-009 | Done | T2 | SR-002 | covered | `src/nodes/camera_position_change_detector_node.hpp` | Camera-cut detection (histogram) as an association hint |
|
||||
| AR-010 | **Done** — decode b… | T2 | SR-002 | covered | `src/main.cpp`, `src/nodes/scene_boundary_annotator_node.hpp`, `src/nodes/scene_detector_node.hpp`, `src/scene_boundaries.hpp` | Scene-boundary detection (TransNetV2) as an association hint |
|
||||
| AR-001 | Done | T3 | SR-002 | untagged | - | Detect faces in sampled frames; emit bbox, confidence, 5-point landma… |
|
||||
| AR-002 | Planned | T2 | SR-002 | untagged | - | Minimum face size 66×66 px, expressed in **original** resolution (dec… |
|
||||
| AR-003 | Planned | T1, T2, T4 | SR-002 | untagged | - | No fixed per-frame face cap — crowd scenes must not lose background c… |
|
||||
| AR-004 | Planned | T1, T4 | SR-002 | untagged | - | Backpressure: unbounded faces/frame absorbed by slowing, never by dro… |
|
||||
| AR-005 | Done | T1, T3 | SR-002 | untagged | - | Align to 112×112 via ArcFace 5-point similarity transform |
|
||||
| AR-006 | Done | T3 | SR-002 | untagged | - | 512-d L2-normalised embeddings, batched |
|
||||
| AR-007 | In Progress | T2 | SR-002 | untagged | - | Associate detections by IoU + embedding, with **frame-dependent** wei… |
|
||||
| AR-008 | Planned | T2 | SR-002 | untagged | - | One track pool keyed on `last_seen`; no separate revival path |
|
||||
| AR-009 | Done | T2 | SR-002 | untagged | - | Camera-cut detection (histogram) as an association hint |
|
||||
| AR-010 | In Progress | T2 | SR-002 | untagged | - | Scene-boundary detection (TransNetV2) as an association hint |
|
||||
| AR-011 | Planned | T1, T2 | SR-002 | untagged | - | **Every model is fed the input it was trained for** — cost reduced by… |
|
||||
| AR-012 | **Done** — `src/tra… | T2 | **SR-002** | covered | `src/main.cpp`, `src/nodes/identity_matcher_node.hpp`, `src/nodes/result_sink_node.hpp`, `src/track_registry.hpp`, `tests/test_replay_fixtures.cpp`, `tests/test_track_registry.cpp` | Presence follows **track extent**, not per-frame recognition |
|
||||
| AR-013 | **Done** — `last_se… | T2 | SR-002 | covered | `src/track_registry.hpp`, `tests/test_replay_fixtures.cpp`, `tests/test_track_registry.cpp` | `last_seen` optional state machine; window ends at last sighting, nev… |
|
||||
| AR-014 | **Done** — swap clo… | T2 | SR-002 | covered | `src/track_registry.hpp`, `tests/test_track_registry.cpp` | Belief swap A→B terminates the track and starts a new one |
|
||||
| AR-015 | **Done** — reverse … | T2 | SR-002 | covered | `src/track_registry.hpp`, `tests/test_track_registry.cpp` | Two live tracks owned by one actor ⇒ treat as a detected cut, re-asso… |
|
||||
| AR-016 | **Done** — `flush()… | T2 | SR-002 | covered | `src/main.cpp`, `src/nodes/result_sink_node.hpp`, `src/track_registry.hpp`, `tests/test_track_registry.cpp` | All tracks closed at EOF — a film ends with faces on screen |
|
||||
| AR-017 | **Done** — `DeadTra… | T1, T2 | SR-002 | covered | `src/nodes/result_sink_node.hpp`, `src/track_registry.hpp`, `tests/test_track_registry.cpp` | Every presence claim carries its belief and identification route |
|
||||
| AR-018 | **Done** — banded a… | T1, T2 | SR-005 | covered | `src/config.hpp`, `src/gallery/track_gallery.hpp`, `src/nodes/identity_matcher_node.hpp`, `tests/test_track_gallery.cpp` | Per-subject embedding store with banded admission (novel enough, safe… |
|
||||
| AR-019 | **Done** — all thre… | T2 | SR-005 | covered | `src/gallery/track_gallery.hpp`, `src/nodes/identity_matcher_node.hpp`, `tests/test_track_gallery.cpp` | Per-film gallery annex from owned tracks; acquires the non-frontal vi… |
|
||||
| AR-012 | Planned | T2 | **SR-002** | untagged | - | Presence follows **track extent**, not per-frame recognition |
|
||||
| AR-013 | Planned | T2 | SR-002 | untagged | - | `last_seen` optional state machine; window ends at last sighting, nev… |
|
||||
| AR-014 | Planned | T2 | SR-002 | untagged | - | Belief swap A→B terminates the track and starts a new one |
|
||||
| AR-015 | Planned | T2 | SR-002 | untagged | - | Two live tracks owned by one actor ⇒ treat as a detected cut, re-asso… |
|
||||
| AR-016 | Planned | T2 | SR-002 | untagged | - | All tracks closed at EOF — a film ends with faces on screen |
|
||||
| AR-017 | Planned | T1, T2 | SR-002 | untagged | - | Every presence claim carries its belief and identification route |
|
||||
| AR-018 | Planned | T1, T2 | SR-005 | untagged | - | Per-subject embedding store with banded admission (novel enough, safe… |
|
||||
| AR-019 | In Progress | T2 | SR-005 | untagged | - | Per-film gallery annex from owned tracks; acquires the non-frontal vi… |
|
||||
| AR-020 | Planned | T2 | SR-005 | untagged | - | Deferred re-identification of unknown tracks against the final expand… |
|
||||
| AR-021 | Planned | T2 | SR-005 | untagged | - | Cluster unknown tracks into one entity per person, under temporal can… |
|
||||
| AR-022 | Planned | T1, T2 | §4 | untagged | - | Capture still-unidentified tracks: embeddings, metadata, **context cr… |
|
||||
| AR-023 | Done | T1 | SR-002 | covered | `src/gallery/gallery_calibration.hpp`, `src/nodes/identity_matcher_node.hpp`, `tests/test_calibration.cpp` | Fit sigmoid calibration from intra/inter similarity distributions |
|
||||
| AR-024 | **Done** — associat… | T1, static | SR-002 | covered | `src/config.hpp`, `src/evidence_discount.hpp`, `src/gallery/gallery_calibration.hpp`, `src/gallery/track_gallery.hpp`, `src/main.cpp`, `src/nodes/face_tracker_node.hpp`, `src/nodes/identity_matcher_node.hpp`, `tests/test_track_gallery.cpp` | **Always the calibrated probability, never a raw cosine** — exception… |
|
||||
| AR-025 | **Done** — log-odds… | T1 | SR-002 | covered | `src/evidence_discount.hpp`, `src/nodes/identity_matcher_node.hpp` | Per-track Bayesian accumulation in log-odds, with correlated-observat… |
|
||||
| AR-026 | In Progress | T1, T4 | SR-001 | covered | `src/backends/gemm_backend.cpp`, `tests/test_similarity.cpp` | All similarity computed as GEMM, including annex and deferred pass |
|
||||
| AR-027 | Planned | T4 | SR-001 | tagged, unexecuted | `src/backends/gemm_backend.cpp` | Throughput acceptable for **arbitrary** gallery size |
|
||||
| AR-028 | Planned | T2 | SR-002 | untagged | - | **Embedding input quality assessed and carried** — every face scored … |
|
||||
| AR-029 | Planned | T1 | SR-002 | untagged | - | Sharpness measure on the **aligned crop** (scale-normalised, so it ca… |
|
||||
| AR-030 | **In Progress** — m… | T1 | SR-002 | covered | `src/face_utils.hpp`, `src/nodes/face_aligner_node.hpp`, `tests/test_face_utils.cpp` | Visibility measure from the AR-001 5-point landmarks — extreme pose o… |
|
||||
| DP-001 | Done | T1, manual | PR-004 | covered | `src/main.cpp` | One analysis core; modes are front-ends and must not fork pipeline lo… |
|
||||
| DP-002 | Done | T1, manual | PR-004 | covered | `src/main.cpp` | Batch CLI over one title |
|
||||
| AR-023 | Done | T1 | SR-002 | untagged | - | Fit sigmoid calibration from intra/inter similarity distributions |
|
||||
| AR-024 | Planned | T1, static | SR-002 | untagged | - | **Always the calibrated probability, never a raw cosine** — exception… |
|
||||
| AR-025 | Planned | T1 | SR-002 | untagged | - | Per-track Bayesian accumulation in log-odds, with correlated-observat… |
|
||||
| AR-026 | In Progress | T1, T4 | SR-001 | untagged | - | All similarity computed as GEMM, including annex and deferred pass |
|
||||
| AR-027 | Planned | T4 | SR-001 | untagged | - | Throughput acceptable for **arbitrary** gallery size |
|
||||
| DP-001 | Done | T1, manual | PR-004 | untagged | - | One analysis core; modes are front-ends and must not fork pipeline lo… |
|
||||
| DP-002 | Done | T1, manual | PR-004 | untagged | - | Batch CLI over one title |
|
||||
| DP-003 | Planned | T1, manual | PR-004 | untagged | - | On-demand resident service with bounded, observable queue |
|
||||
| DP-004 | Planned | T1, manual | PR-004 | untagged | - | Opportunistic/idle mode: external trigger, hard stop, implicit re-que… |
|
||||
| DP-005 | Planned | T1, manual | PR-004 | untagged | - | Native installer, no Docker; Fedora + Arch |
|
||||
| DP-006 | Planned | T1, manual | PR-003 | untagged | - | Background incremental gallery refresh on a timer |
|
||||
| DP-007 | Planned | T1, manual | PR-004 | untagged | - | CI builder image, CPU-only, pinned by tag in the Gitea container regi… |
|
||||
| DP-008 | Planned | T1, manual | PR-004 | untagged | - | Builder images + release jobs per backend (cpu / cuda / rocm); ship b… |
|
||||
| IR-001 | Done | T1 | SR-003 | covered | `src/nodes/result_sink_node.hpp` | Emit the JRay truth format as sibling `.jray.json` |
|
||||
| IR-002 | **Done** — `schema_… | T1 | SR-003 | covered | `src/config.hpp`, `src/main.cpp`, `src/nodes/result_sink_node.hpp` | Windows carry belief + route; `extraction.*` carries `extinction_sec`… |
|
||||
| IR-003 | **In Progress** — s… | T1 | SR-003 | covered | `src/main.cpp` | Output written **after** the deferred pass, not at EOF |
|
||||
| IR-004 | **Done** — `src/aud… | T1 | SR-003 | covered | `scripts/validation/test_audio_offset.py`, `src/audio_bindings.cpp`, `src/audio_signature.cpp`, `src/audio_signature.hpp`, `tests/test_audio_signature.cpp` | Compute the audio signature exactly per server spec §3 |
|
||||
| IR-005 | **Done** — `tests/f… | T1 | SR-003 | covered | `src/audio_bindings.cpp`, `src/audio_signature.cpp`, `src/audio_signature.hpp`, `tests/test_audio_signature.cpp` | Golden-vector fixture shared with the plugin repo to prove bit-exactn… |
|
||||
| IR-006 | Done | T1, manual | SR-001 | covered | `scripts/run_from_jellyfin.py` | Jellyfin round-trip: pull pending queue, push complete results only |
|
||||
| IR-007 | **Done** | T1 | SR-003 | covered | `src/audio_signature.cpp`, `src/audio_signature.hpp`, `tests/test_audio_signature.cpp` | Media < 120 s: emit no signature, apply no sync offset — identical ru… |
|
||||
| IR-008 | **Done** | T1 | SR-003 | covered | `src/audio_signature.cpp`, `src/audio_signature.hpp`, `tests/test_audio_signature.cpp` | Emit and honour the signature's own `v1:` version prefix |
|
||||
| GR-001 | Done | T1, T3 | SR-001, SR-005 | covered | `scripts/make_jellyfin_gallery.py` | Build gallery from Jellyfin library cast, TMDB profile fallback |
|
||||
| GR-002 | Done | T1, T3 | PR-003 | covered | `scripts/make_jellyfin_gallery.py` | Incremental `--merge` refresh without re-embedding known actors |
|
||||
| GR-003 | **Done** — `gallery… | T1, T3 | SR-001 | covered | `src/build_gallery.cpp`, `src/gallery/gallery_calibration.hpp`, `src/gallery/gallery_report.hpp`, `tests/test_calibration.cpp` | Report coverage: zero-image actors, under-referenced actors, dedup, c… |
|
||||
| GR-004 | **Done** — basename… | T1, T3 | SR-001 | covered | `scripts/filter_gallery.py`, `scripts/make_gallery.py`, `scripts/make_jellyfin_gallery.py`, `scripts/movienet_eval.py`, `scripts/optimizer/fetch_missing_actors.py`, `scripts/optimizer/optimize.py`, `scripts/optimizer/reembed_gallery.py`, `scripts/optimizer/replay.py`, `scripts/sae_embed_loader.py`, `scripts/sae_gallery.py`, `scripts/sae_stamp.py`, `scripts/stamp_gallery.py`, `src/config.hpp`, `src/gallery/embedder_stamp.cpp`, `src/gallery/embedder_stamp.hpp`, `src/gallery/gallery_builder.cpp`, `src/gallery/gallery_store.cpp`, `src/kpn_bindings.cpp`, `src/main.cpp`, `src/nodes/embedding_dump_node.hpp`, `src/scene_preview.cpp`, `src/types.hpp`, `tests/test_gallery_store.cpp` | Stamp embedder identity into the gallery; **hard startup error** on m… |
|
||||
| GR-005 | Done | T1, T3 | **SR-005** | covered | `src/gallery/gallery_store.hpp` | Gallery data never leaves the instance |
|
||||
| IR-001 | Done | T1 | SR-003 | untagged | - | Emit the JRay truth format as sibling `.jray.json` |
|
||||
| IR-002 | Planned | T1 | SR-003 | untagged | - | Windows carry belief + route; `extraction.*` carries `extinction_sec`… |
|
||||
| IR-003 | Planned | T1 | SR-003 | untagged | - | Output written **after** the deferred pass, not at EOF |
|
||||
| IR-004 | Planned | T1 | SR-003 | untagged | - | Compute the audio signature exactly per server spec §3 |
|
||||
| IR-005 | Planned | T1 | SR-003 | untagged | - | Golden-vector fixture shared with the plugin repo to prove bit-exactn… |
|
||||
| IR-006 | Done | unset | SR-001 | untagged | - | Jellyfin round-trip: pull pending queue, push complete results only |
|
||||
| IR-007 | Planned | unset | SR-003 | untagged | - | Media < 120 s: emit no signature, apply no sync offset — identical ru… |
|
||||
| IR-008 | Planned | unset | SR-003 | untagged | - | Emit and honour the signature's own `v1:` version prefix |
|
||||
| GR-001 | Done | T1, T3 | SR-001, SR-005 | untagged | - | Build gallery from Jellyfin library cast, TMDB profile fallback |
|
||||
| GR-002 | Done | T1, T3 | PR-003 | untagged | - | Incremental `--merge` refresh without re-embedding known actors |
|
||||
| GR-003 | Planned | T1, T3 | SR-001 | untagged | - | Report coverage: zero-image actors, under-referenced actors, dedup, c… |
|
||||
| GR-004 | Planned | T1, T3 | SR-001 | untagged | - | Stamp embedder identity into the gallery; **hard startup error** on m… |
|
||||
| GR-005 | Done | T1, T3 | **SR-005** | untagged | - | Gallery data never leaves the instance |
|
||||
| GR-006 | Planned | T1 | SR-005 | untagged | - | Provenance tiers: baked / harvested / confirmed, distinguishable per … |
|
||||
| GR-007 | Planned | T1 | SR-005 | untagged | - | Persist harvested embeddings **flagged and reviewable**, never silent… |
|
||||
| GR-008 | Planned | T1 | SR-005 | untagged | - | Flag distributional outliers among an actor's references (poisoning g… |
|
||||
| GR-009 | TBD | T1 | §4 | untagged | - | Human-confirmed associations persist and improve future extractions |
|
||||
| VR-001 | Done | out-of-ci | PR-002 | tagged, unexecuted | `src/nodes/embedding_dump_node.hpp`, `tests/test_replay_fixtures.cpp` | HDF5 post-inference dump at the embedded-frame boundary |
|
||||
| VR-002 | **Done** — replay d… | out-of-ci | PR-002 | tagged, unexecuted | `scripts/optimizer/replay.py`, `tests/test_replay_fixtures.cpp` | Replay drives the **real** KPN nodes, not a reimplementation |
|
||||
| VR-003 | Done | out-of-ci | PR-002 | tagged, unexecuted | `scripts/optimizer/second_score.py` | Scoring: micro-F1 against X-Ray, precision/recall logged at every eva… |
|
||||
| VR-004 | Done | out-of-ci | PR-002 | tagged, unexecuted | `scripts/validation/ground_truth.py` | Reproducible validation corpus with ground truth |
|
||||
| VR-005 | **Done** — knee at … | out-of-ci | PR-002 | tagged, unexecuted | `scripts/validation/min_face_size.py` | Minimum face size study — TPI/FPI vs probe size, gallery held at nati… |
|
||||
| GR-009 | TBD | unset | §4 | untagged | - | Human-confirmed associations persist and improve future extractions |
|
||||
| VR-001 | Done | out-of-ci | PR-002 | untagged | - | HDF5 post-inference dump at the embedded-frame boundary |
|
||||
| VR-002 | Done | out-of-ci | PR-002 | untagged | - | Replay drives the **real** KPN nodes, not a reimplementation |
|
||||
| VR-003 | Done | out-of-ci | PR-002 | untagged | - | Scoring: micro-F1 against X-Ray, precision/recall logged at every eva… |
|
||||
| VR-004 | Done | out-of-ci | PR-002 | untagged | - | Reproducible validation corpus with ground truth |
|
||||
| VR-005 | Planned | out-of-ci | PR-002 | untagged | - | Minimum face size study — TPI/FPI vs probe size, gallery held at nati… |
|
||||
| VR-006 | Planned | out-of-ci | PR-002 | untagged | - | Re-tune `scene_threshold` once native-rate decode lands |
|
||||
| VR-007 | Planned | out-of-ci | PR-002 | untagged | - | Expansion band, clustering threshold, and deferred-pass ablation |
|
||||
| VR-008 | Planned | out-of-ci | PR-002 | untagged | - | Gallery scaling benchmark — throughput vs gallery size |
|
||||
| VR-009 | Planned | T1, out-of-ci | PR-002 | untagged | - | Verify accumulated posteriors are calibrated against held-out tracks |
|
||||
| VR-010 | Planned | out-of-ci | PR-002 | tagged, unexecuted | `src/nodes/embedding_dump_node.hpp` | Dump provenance attributes — embedder model, detector settings, `dens… |
|
||||
| VR-011 | Planned | out-of-ci | PR-002 | untagged | - | Rewrite the replay harness for the post-AR-012 output contract |
|
||||
| VR-012 | Planned | T4, out-of-ci | PR-002 | untagged | - | Quality-knee study — TPI/FPI vs sharpness and vs pose, as VR-005 did … |
|
||||
| VR-013 | **In Progress** — h… | T4, out-of-ci | PR-002 | tagged, unexecuted | `experiments/xsource/resolution_sweep.py`, `experiments/xsource/verify_labels.py` | Cross-source identification probe — gallery from one recording, probe… |
|
||||
| VR-014 | **Done** — 40 rando… | T2, out-of-ci | PR-002 | covered | `scripts/validation/test_audio_offset.py` | Audio-signature **offset recovery on real content** — a known trim re… |
|
||||
|
||||
## Detailed mapping
|
||||
|
||||
### AR-001
|
||||
|
||||
**Locations:** 1
|
||||
|
||||
- [`src/nodes/face_detector_node.hpp:2`](../src/nodes/face_detector_node.hpp#L2) — `Unknown`
|
||||
|
||||
### AR-003
|
||||
|
||||
**Locations:** 3
|
||||
|
||||
- [`src/config.hpp:44`](../src/config.hpp#L44) — `Unknown`
|
||||
- [`src/nodes/face_detector_node.hpp:47`](../src/nodes/face_detector_node.hpp#L47) — `private:`
|
||||
- [`src/nodes/identity_matcher_node.hpp:166`](../src/nodes/identity_matcher_node.hpp#L166) — `std::vector<float> host_query(static_cast<size_t>(kMaxFaces) * 512);`
|
||||
|
||||
### AR-004
|
||||
|
||||
**Locations:** 4
|
||||
|
||||
- [`src/main.cpp:86`](../src/main.cpp#L86) — `static constexpr std::size_t kSceneJoinDepth = 256;`
|
||||
- [`src/main.cpp:319`](../src/main.cpp#L319) — `Unknown`
|
||||
- [`src/nodes/identity_matcher_node.hpp:166`](../src/nodes/identity_matcher_node.hpp#L166) — `std::vector<float> host_query(static_cast<size_t>(kMaxFaces) * 512);`
|
||||
- [`tests/test_replay_fixtures.cpp:3`](../tests/test_replay_fixtures.cpp#L3) — `Unknown`
|
||||
|
||||
### AR-005
|
||||
|
||||
**Locations:** 3
|
||||
|
||||
- [`src/face_utils.hpp:2`](../src/face_utils.hpp#L2) — `Unknown`
|
||||
- [`src/nodes/face_aligner_node.hpp:7`](../src/nodes/face_aligner_node.hpp#L7) — `struct FaceAlignerFunc`
|
||||
- [`tests/test_face_utils.cpp:1`](../tests/test_face_utils.cpp#L1) — `Unknown`
|
||||
|
||||
### AR-006
|
||||
|
||||
**Locations:** 1
|
||||
|
||||
- [`src/nodes/embedder_node.hpp:21`](../src/nodes/embedder_node.hpp#L21) — `struct EmbedderFunc`
|
||||
|
||||
### AR-007
|
||||
|
||||
**Locations:** 4
|
||||
|
||||
- [`src/config.hpp:108`](../src/config.hpp#L108) — `Unknown`
|
||||
- [`src/main.cpp:213`](../src/main.cpp#L213) — `reg_cfg, EvidenceDiscounter(same_person));`
|
||||
- [`src/nodes/face_tracker_node.hpp:2`](../src/nodes/face_tracker_node.hpp#L2) — `Unknown`
|
||||
- [`tests/test_face_tracker.cpp:1`](../tests/test_face_tracker.cpp#L1) — `Unknown`
|
||||
|
||||
### AR-008
|
||||
|
||||
**Locations:** 4
|
||||
|
||||
- [`src/config.hpp:108`](../src/config.hpp#L108) — `Unknown`
|
||||
- [`src/main.cpp:213`](../src/main.cpp#L213) — `reg_cfg, EvidenceDiscounter(same_person));`
|
||||
- [`src/nodes/face_tracker_node.hpp:2`](../src/nodes/face_tracker_node.hpp#L2) — `Unknown`
|
||||
- [`tests/test_face_tracker.cpp:1`](../tests/test_face_tracker.cpp#L1) — `Unknown`
|
||||
|
||||
### AR-009
|
||||
|
||||
**Locations:** 1
|
||||
|
||||
- [`src/nodes/camera_position_change_detector_node.hpp:30`](../src/nodes/camera_position_change_detector_node.hpp#L30) — `struct CameraPositionChangeDetectorFunc`
|
||||
|
||||
### AR-010
|
||||
|
||||
**Locations:** 9
|
||||
|
||||
- [`src/main.cpp:86`](../src/main.cpp#L86) — `static constexpr std::size_t kSceneJoinDepth = 256;`
|
||||
- [`src/main.cpp:329`](../src/main.cpp#L329) — `Unknown`
|
||||
- [`src/main.cpp:399`](../src/main.cpp#L399) — `return run_net(std::move(net));`
|
||||
- [`src/main.cpp:431`](../src/main.cpp#L431) — `Unknown`
|
||||
- [`src/nodes/scene_boundary_annotator_node.hpp:2`](../src/nodes/scene_boundary_annotator_node.hpp#L2) — `Unknown`
|
||||
- [`src/nodes/scene_detector_node.hpp:36`](../src/nodes/scene_detector_node.hpp#L36) — `static constexpr std::string_view label() { return "scene_detector"; }`
|
||||
- [`src/nodes/scene_detector_node.hpp:110`](../src/nodes/scene_detector_node.hpp#L110) — `void flush_remaining()`
|
||||
- [`src/nodes/scene_detector_node.hpp:144`](../src/nodes/scene_detector_node.hpp#L144) — `void write_output()`
|
||||
- [`src/scene_boundaries.hpp:2`](../src/scene_boundaries.hpp#L2) — `Unknown`
|
||||
|
||||
### AR-012
|
||||
|
||||
**Locations:** 9
|
||||
|
||||
- [`src/main.cpp:213`](../src/main.cpp#L213) — `reg_cfg, EvidenceDiscounter(same_person));`
|
||||
- [`src/main.cpp:230`](../src/main.cpp#L230) — `reg_cfg, EvidenceDiscounter(same_person));`
|
||||
- [`src/nodes/identity_matcher_node.hpp:125`](../src/nodes/identity_matcher_node.hpp#L125) — `const GalleryCalibration& calibration() const { return cal_; }`
|
||||
- [`src/nodes/identity_matcher_node.hpp:272`](../src/nodes/identity_matcher_node.hpp#L272) — `Unknown`
|
||||
- [`src/nodes/result_sink_node.hpp:49`](../src/nodes/result_sink_node.hpp#L49) — `static constexpr std::string_view label() { return "result_sink"; }`
|
||||
- [`src/nodes/result_sink_node.hpp:161`](../src/nodes/result_sink_node.hpp#L161) — `struct ActorMeta { std::string name, imdb_id, tmdb_id, jellyfin_id; };`
|
||||
- [`src/track_registry.hpp:2`](../src/track_registry.hpp#L2) — `Unknown`
|
||||
- [`tests/test_replay_fixtures.cpp:3`](../tests/test_replay_fixtures.cpp#L3) — `Unknown`
|
||||
- [`tests/test_track_registry.cpp:3`](../tests/test_track_registry.cpp#L3) — `Unknown`
|
||||
|
||||
### AR-013
|
||||
|
||||
**Locations:** 3
|
||||
|
||||
- [`src/track_registry.hpp:2`](../src/track_registry.hpp#L2) — `Unknown`
|
||||
- [`tests/test_replay_fixtures.cpp:3`](../tests/test_replay_fixtures.cpp#L3) — `Unknown`
|
||||
- [`tests/test_track_registry.cpp:3`](../tests/test_track_registry.cpp#L3) — `Unknown`
|
||||
|
||||
### AR-014
|
||||
|
||||
**Locations:** 2
|
||||
|
||||
- [`src/track_registry.hpp:2`](../src/track_registry.hpp#L2) — `Unknown`
|
||||
- [`tests/test_track_registry.cpp:3`](../tests/test_track_registry.cpp#L3) — `Unknown`
|
||||
|
||||
### AR-015
|
||||
|
||||
**Locations:** 2
|
||||
|
||||
- [`src/track_registry.hpp:2`](../src/track_registry.hpp#L2) — `Unknown`
|
||||
- [`tests/test_track_registry.cpp:3`](../tests/test_track_registry.cpp#L3) — `Unknown`
|
||||
|
||||
### AR-016
|
||||
|
||||
**Locations:** 4
|
||||
|
||||
- [`src/main.cpp:230`](../src/main.cpp#L230) — `reg_cfg, EvidenceDiscounter(same_person));`
|
||||
- [`src/nodes/result_sink_node.hpp:63`](../src/nodes/result_sink_node.hpp#L63) — `void set_pre_write_hook(std::function<void(double)> fn) { pre_write_ = std::move(fn); }`
|
||||
- [`src/track_registry.hpp:2`](../src/track_registry.hpp#L2) — `Unknown`
|
||||
- [`tests/test_track_registry.cpp:3`](../tests/test_track_registry.cpp#L3) — `Unknown`
|
||||
|
||||
### AR-017
|
||||
|
||||
**Locations:** 3
|
||||
|
||||
- [`src/nodes/result_sink_node.hpp:49`](../src/nodes/result_sink_node.hpp#L49) — `static constexpr std::string_view label() { return "result_sink"; }`
|
||||
- [`src/track_registry.hpp:2`](../src/track_registry.hpp#L2) — `Unknown`
|
||||
- [`tests/test_track_registry.cpp:3`](../tests/test_track_registry.cpp#L3) — `Unknown`
|
||||
|
||||
### AR-018
|
||||
|
||||
**Locations:** 5
|
||||
|
||||
- [`src/config.hpp:152`](../src/config.hpp#L152) — `Unknown`
|
||||
- [`src/gallery/track_gallery.hpp:164`](../src/gallery/track_gallery.hpp#L164) — `struct TrackState`
|
||||
- [`src/gallery/track_gallery.hpp:274`](../src/gallery/track_gallery.hpp#L274) — `static int plurality_actor(const TrackState& ts)`
|
||||
- [`src/nodes/identity_matcher_node.hpp:110`](../src/nodes/identity_matcher_node.hpp#L110) — `std::vector<float> host_gallery(static_cast<size_t>(n_gallery_) * 512);`
|
||||
- [`tests/test_track_gallery.cpp:1`](../tests/test_track_gallery.cpp#L1) — `Unknown`
|
||||
|
||||
### AR-019
|
||||
|
||||
**Locations:** 4
|
||||
|
||||
- [`src/gallery/track_gallery.hpp:123`](../src/gallery/track_gallery.hpp#L123) — `void forget(int track_id) { tracks_.erase(track_id); }`
|
||||
- [`src/nodes/identity_matcher_node.hpp:147`](../src/nodes/identity_matcher_node.hpp#L147) — `MatchedSceneFrame operator()(TrackedSceneFrame tf)`
|
||||
- [`src/nodes/identity_matcher_node.hpp:285`](../src/nodes/identity_matcher_node.hpp#L285) — `Unknown`
|
||||
- [`tests/test_track_gallery.cpp:1`](../tests/test_track_gallery.cpp#L1) — `Unknown`
|
||||
|
||||
### AR-023
|
||||
|
||||
**Locations:** 4
|
||||
|
||||
- [`src/gallery/gallery_calibration.hpp:2`](../src/gallery/gallery_calibration.hpp#L2) — `Unknown`
|
||||
- [`src/gallery/gallery_calibration.hpp:53`](../src/gallery/gallery_calibration.hpp#L53) — `float boundary_at(float p = 0.5f, float log_prior_odds = 0.f) const`
|
||||
- [`src/nodes/identity_matcher_node.hpp:117`](../src/nodes/identity_matcher_node.hpp#L117) — `const GalleryCalibration& calibration() const { return cal_; }`
|
||||
- [`tests/test_calibration.cpp:1`](../tests/test_calibration.cpp#L1) — `Unknown`
|
||||
|
||||
### AR-024
|
||||
|
||||
**Locations:** 12
|
||||
|
||||
- [`src/config.hpp:108`](../src/config.hpp#L108) — `Unknown`
|
||||
- [`src/config.hpp:152`](../src/config.hpp#L152) — `Unknown`
|
||||
- [`src/evidence_discount.hpp:2`](../src/evidence_discount.hpp#L2) — `Unknown`
|
||||
- [`src/gallery/gallery_calibration.hpp:53`](../src/gallery/gallery_calibration.hpp#L53) — `float boundary_at(float p = 0.5f, float log_prior_odds = 0.f) const`
|
||||
- [`src/gallery/track_gallery.hpp:133`](../src/gallery/track_gallery.hpp#L133) — `void set_calibration(std::function<float(float)> c) { calibrate_ = std::move(c); }`
|
||||
- [`src/gallery/track_gallery.hpp:164`](../src/gallery/track_gallery.hpp#L164) — `struct TrackState`
|
||||
- [`src/gallery/track_gallery.hpp:274`](../src/gallery/track_gallery.hpp#L274) — `static int plurality_actor(const TrackState& ts)`
|
||||
- [`src/main.cpp:213`](../src/main.cpp#L213) — `reg_cfg, EvidenceDiscounter(same_person));`
|
||||
- [`src/nodes/face_tracker_node.hpp:2`](../src/nodes/face_tracker_node.hpp#L2) — `Unknown`
|
||||
- [`src/nodes/identity_matcher_node.hpp:110`](../src/nodes/identity_matcher_node.hpp#L110) — `std::vector<float> host_gallery(static_cast<size_t>(n_gallery_) * 512);`
|
||||
- [`src/nodes/identity_matcher_node.hpp:117`](../src/nodes/identity_matcher_node.hpp#L117) — `const GalleryCalibration& calibration() const { return cal_; }`
|
||||
- [`tests/test_track_gallery.cpp:1`](../tests/test_track_gallery.cpp#L1) — `Unknown`
|
||||
|
||||
### AR-025
|
||||
|
||||
**Locations:** 3
|
||||
|
||||
- [`src/evidence_discount.hpp:2`](../src/evidence_discount.hpp#L2) — `Unknown`
|
||||
- [`src/nodes/identity_matcher_node.hpp:125`](../src/nodes/identity_matcher_node.hpp#L125) — `const GalleryCalibration& calibration() const { return cal_; }`
|
||||
- [`src/nodes/identity_matcher_node.hpp:272`](../src/nodes/identity_matcher_node.hpp#L272) — `Unknown`
|
||||
|
||||
### AR-026
|
||||
|
||||
**Locations:** 2
|
||||
|
||||
- [`src/backends/gemm_backend.cpp:44`](../src/backends/gemm_backend.cpp#L44) — `constexpr int kDim = 512;`
|
||||
- [`tests/test_similarity.cpp:1`](../tests/test_similarity.cpp#L1) — `Unknown`
|
||||
|
||||
### AR-027
|
||||
|
||||
**Locations:** 1
|
||||
|
||||
- [`src/backends/gemm_backend.cpp:44`](../src/backends/gemm_backend.cpp#L44) — `constexpr int kDim = 512;`
|
||||
|
||||
### AR-030
|
||||
|
||||
**Locations:** 3
|
||||
|
||||
- [`src/face_utils.hpp:2`](../src/face_utils.hpp#L2) — `Unknown`
|
||||
- [`src/nodes/face_aligner_node.hpp:7`](../src/nodes/face_aligner_node.hpp#L7) — `struct FaceAlignerFunc`
|
||||
- [`tests/test_face_utils.cpp:1`](../tests/test_face_utils.cpp#L1) — `Unknown`
|
||||
|
||||
### DP-001
|
||||
|
||||
**Locations:** 1
|
||||
|
||||
- [`src/main.cpp:3`](../src/main.cpp#L3) — `Unknown`
|
||||
|
||||
### DP-002
|
||||
|
||||
**Locations:** 1
|
||||
|
||||
- [`src/main.cpp:3`](../src/main.cpp#L3) — `Unknown`
|
||||
|
||||
### GR-001
|
||||
|
||||
**Locations:** 1
|
||||
|
||||
- [`scripts/make_jellyfin_gallery.py:4`](../scripts/make_jellyfin_gallery.py#L4) — `Unknown`
|
||||
|
||||
### GR-002
|
||||
|
||||
**Locations:** 1
|
||||
|
||||
- [`scripts/make_jellyfin_gallery.py:4`](../scripts/make_jellyfin_gallery.py#L4) — `Unknown`
|
||||
|
||||
### GR-003
|
||||
|
||||
**Locations:** 16
|
||||
|
||||
- [`src/build_gallery.cpp:83`](../src/build_gallery.cpp#L83) — `Unknown`
|
||||
- [`src/gallery/gallery_calibration.hpp:80`](../src/gallery/gallery_calibration.hpp#L80) — `struct GalleryCalibrationStats`
|
||||
- [`src/gallery/gallery_calibration.hpp:145`](../src/gallery/gallery_calibration.hpp#L145) — `std::vector<bool> actor_eligible(n_actors, false);`
|
||||
- [`src/gallery/gallery_calibration.hpp:294`](../src/gallery/gallery_calibration.hpp#L294) — `Unknown`
|
||||
- [`src/gallery/gallery_report.hpp:2`](../src/gallery/gallery_report.hpp#L2) — `Unknown`
|
||||
- [`src/gallery/gallery_report.hpp:52`](../src/gallery/gallery_report.hpp#L52) — `struct GalleryBuildAudit`
|
||||
- [`src/gallery/gallery_report.hpp:73`](../src/gallery/gallery_report.hpp#L73) — `struct GalleryReport`
|
||||
- [`src/gallery/gallery_report.hpp:154`](../src/gallery/gallery_report.hpp#L154) — `inline GalleryReport build_gallery_report(const ActorGallery& gallery,`
|
||||
- [`src/gallery/gallery_report.hpp:295`](../src/gallery/gallery_report.hpp#L295) — `inline nlohmann::json gallery_report_to_json(const GalleryReport& r)`
|
||||
- [`src/gallery/gallery_report.hpp:361`](../src/gallery/gallery_report.hpp#L361) — `inline GalleryReport gallery_report_from_json(const nlohmann::json& j)`
|
||||
- [`src/gallery/gallery_report.hpp:446`](../src/gallery/gallery_report.hpp#L446) — `inline void save_gallery_report(const std::string& path, const GalleryReport& r)`
|
||||
- [`src/gallery/gallery_report.hpp:454`](../src/gallery/gallery_report.hpp#L454) — `inline GalleryReport load_gallery_report(const std::string& path)`
|
||||
- [`src/gallery/gallery_report.hpp:464`](../src/gallery/gallery_report.hpp#L464) — `return gallery_report_from_json(j);`
|
||||
- [`tests/test_calibration.cpp:192`](../tests/test_calibration.cpp#L192) — `Embedding unit_axis(int slot)`
|
||||
- [`tests/test_calibration.cpp:222`](../tests/test_calibration.cpp#L222) — `Unknown`
|
||||
- [`tests/test_calibration.cpp:248`](../tests/test_calibration.cpp#L248) — `Unknown`
|
||||
|
||||
### GR-004
|
||||
|
||||
**Locations:** 44
|
||||
|
||||
- [`src/config.hpp:54`](../src/config.hpp#L54) — `Unknown`
|
||||
- [`src/gallery/embedder_stamp.cpp:1`](../src/gallery/embedder_stamp.cpp#L1) — `Unknown`
|
||||
- [`src/gallery/embedder_stamp.hpp:2`](../src/gallery/embedder_stamp.hpp#L2) — `Unknown`
|
||||
- [`src/gallery/gallery_builder.cpp:45`](../src/gallery/gallery_builder.cpp#L45) — `ActorGallery build_gallery(const BuildConfig& cfg)`
|
||||
- [`src/gallery/gallery_store.cpp:82`](../src/gallery/gallery_store.cpp#L82) — `H5::StrType str(H5::PredType::C_S1, H5T_VARIABLE);`
|
||||
- [`src/gallery/gallery_store.cpp:167`](../src/gallery/gallery_store.cpp#L167) — `H5::DataSpace scalar(H5S_SCALAR);`
|
||||
- [`src/gallery/gallery_store.cpp:219`](../src/gallery/gallery_store.cpp#L219) — `Unknown`
|
||||
- [`src/kpn_bindings.cpp:167`](../src/kpn_bindings.cpp#L167) — `Unknown`
|
||||
- [`src/kpn_bindings.cpp:217`](../src/kpn_bindings.cpp#L217) — `Unknown`
|
||||
- [`src/main.cpp:188`](../src/main.cpp#L188) — `Unknown`
|
||||
- [`src/nodes/embedding_dump_node.hpp:127`](../src/nodes/embedding_dump_node.hpp#L127) — `static constexpr std::string_view label() { return "embedding_dump"; }`
|
||||
- [`src/nodes/embedding_dump_node.hpp:238`](../src/nodes/embedding_dump_node.hpp#L238) — `H5::H5File file(path_, H5F_ACC_TRUNC);`
|
||||
- [`src/scene_preview.cpp:133`](../src/scene_preview.cpp#L133) — `int main(int argc, char** argv)`
|
||||
- [`src/types.hpp:148`](../src/types.hpp#L148) — `struct Actor`
|
||||
- [`tests/test_gallery_store.cpp:182`](../tests/test_gallery_store.cpp#L182) — `TempFile tf("gallery_stamped.h5");`
|
||||
- [`tests/test_gallery_store.cpp:201`](../tests/test_gallery_store.cpp#L201) — `TempFile tf("gallery_stamped.h5");`
|
||||
- [`tests/test_gallery_store.cpp:219`](../tests/test_gallery_store.cpp#L219) — `TempFile tf("gallery_unstamped.h5");`
|
||||
- [`tests/test_gallery_store.cpp:240`](../tests/test_gallery_store.cpp#L240) — `TempFile tf("gallery_json_stamp.json");`
|
||||
- [`tests/test_gallery_store.cpp:252`](../tests/test_gallery_store.cpp#L252) — `Unknown`
|
||||
- [`tests/test_gallery_store.cpp:268`](../tests/test_gallery_store.cpp#L268) — `Unknown`
|
||||
- [`tests/test_gallery_store.cpp:300`](../tests/test_gallery_store.cpp#L300) — `Unknown`
|
||||
- [`tests/test_gallery_store.cpp:309`](../tests/test_gallery_store.cpp#L309) — `Unknown`
|
||||
- [`tests/test_gallery_store.cpp:333`](../tests/test_gallery_store.cpp#L333) — `Unknown`
|
||||
- [`tests/test_gallery_store.cpp:348`](../tests/test_gallery_store.cpp#L348) — `Unknown`
|
||||
- [`tests/test_gallery_store.cpp:367`](../tests/test_gallery_store.cpp#L367) — `Unknown`
|
||||
- [`tests/test_gallery_store.cpp:384`](../tests/test_gallery_store.cpp#L384) — `TempFile tf("fake_model.onnx");`
|
||||
- [`scripts/filter_gallery.py:80`](../scripts/filter_gallery.py#L80) — `if actor_jellyfin_id(a) in cast_ids]`
|
||||
- [`scripts/make_gallery.py:181`](../scripts/make_gallery.py#L181) — `Unknown`
|
||||
- [`scripts/make_jellyfin_gallery.py:448`](../scripts/make_jellyfin_gallery.py#L448) — `Unknown`
|
||||
- [`scripts/make_jellyfin_gallery.py:456`](../scripts/make_jellyfin_gallery.py#L456) — `Unknown`
|
||||
- [`scripts/movienet_eval.py:65`](../scripts/movienet_eval.py#L65) — `with open(args.gt) as f:`
|
||||
- [`scripts/optimizer/fetch_missing_actors.py:62`](../scripts/optimizer/fetch_missing_actors.py#L62) — `def fetch(missing_path, out_path, token, build_dir, models_dir, arcface,`
|
||||
- [`scripts/optimizer/fetch_missing_actors.py:109`](../scripts/optimizer/fetch_missing_actors.py#L109) — `def merge(base_path, add_path, out_path):`
|
||||
- [`scripts/optimizer/fetch_missing_actors.py:124`](../scripts/optimizer/fetch_missing_actors.py#L124) — `def merge(base_path, add_path, out_path):`
|
||||
- [`scripts/optimizer/optimize.py:186`](../scripts/optimizer/optimize.py#L186) — `Unknown`
|
||||
- [`scripts/optimizer/optimize.py:202`](../scripts/optimizer/optimize.py#L202) — `if not Path(f["dump"]).exists():`
|
||||
- [`scripts/optimizer/reembed_gallery.py:62`](../scripts/optimizer/reembed_gallery.py#L62) — `for i, a in enumerate(ref["actors"], 1):`
|
||||
- [`scripts/optimizer/replay.py:113`](../scripts/optimizer/replay.py#L113) — `Unknown`
|
||||
- [`scripts/optimizer/replay.py:253`](../scripts/optimizer/replay.py#L253) — `Unknown`
|
||||
- [`scripts/sae_embed_loader.py:23`](../scripts/sae_embed_loader.py#L23) — `def resolve_arcface(models_dir: str, arcface: str \| None = None) -> str:`
|
||||
- [`scripts/sae_gallery.py:171`](../scripts/sae_gallery.py#L171) — `if not _stamp_empty(embedder):`
|
||||
- [`scripts/sae_gallery.py:200`](../scripts/sae_gallery.py#L200) — `for a in range(len(offset)):`
|
||||
- [`scripts/sae_stamp.py:3`](../scripts/sae_stamp.py#L3) — `Unknown`
|
||||
- [`scripts/stamp_gallery.py:4`](../scripts/stamp_gallery.py#L4) — `Unknown`
|
||||
|
||||
### GR-005
|
||||
|
||||
**Locations:** 1
|
||||
|
||||
- [`src/gallery/gallery_store.hpp:15`](../src/gallery/gallery_store.hpp#L15) — `Unknown`
|
||||
|
||||
### IR-001
|
||||
|
||||
**Locations:** 1
|
||||
|
||||
- [`src/nodes/result_sink_node.hpp:2`](../src/nodes/result_sink_node.hpp#L2) — `Unknown`
|
||||
|
||||
### IR-002
|
||||
|
||||
**Locations:** 5
|
||||
|
||||
- [`src/config.hpp:20`](../src/config.hpp#L20) — `struct Config`
|
||||
- [`src/main.cpp:230`](../src/main.cpp#L230) — `reg_cfg, EvidenceDiscounter(same_person));`
|
||||
- [`src/nodes/result_sink_node.hpp:49`](../src/nodes/result_sink_node.hpp#L49) — `static constexpr std::string_view label() { return "result_sink"; }`
|
||||
- [`src/nodes/result_sink_node.hpp:122`](../src/nodes/result_sink_node.hpp#L122) — `void write_output()`
|
||||
- [`src/nodes/result_sink_node.hpp:161`](../src/nodes/result_sink_node.hpp#L161) — `struct ActorMeta { std::string name, imdb_id, tmdb_id, jellyfin_id; };`
|
||||
|
||||
### IR-003
|
||||
|
||||
**Locations:** 1
|
||||
|
||||
- [`src/main.cpp:230`](../src/main.cpp#L230) — `reg_cfg, EvidenceDiscounter(same_person));`
|
||||
|
||||
### IR-004
|
||||
|
||||
**Locations:** 18
|
||||
|
||||
- [`src/audio_bindings.cpp:3`](../src/audio_bindings.cpp#L3) — `Unknown`
|
||||
- [`src/audio_signature.cpp:3`](../src/audio_signature.cpp#L3) — `Unknown`
|
||||
- [`src/audio_signature.cpp:265`](../src/audio_signature.cpp#L265) — `std::vector<std::uint8_t> pack_frames(const std::vector<float>& mono)`
|
||||
- [`src/audio_signature.cpp:320`](../src/audio_signature.cpp#L320) — `std::optional<std::string> signature_from_mono(const std::vector<float>& mono)`
|
||||
- [`src/audio_signature.cpp:327`](../src/audio_signature.cpp#L327) — `std::optional<std::vector<float>> decode_centre_window(const std::string& path)`
|
||||
- [`src/audio_signature.cpp:421`](../src/audio_signature.cpp#L421) — `std::optional<std::string> compute_signature(const std::string& path)`
|
||||
- [`src/audio_signature.hpp:4`](../src/audio_signature.hpp#L4) — `Unknown`
|
||||
- [`tests/test_audio_signature.cpp:3`](../tests/test_audio_signature.cpp#L3) — `Unknown`
|
||||
- [`tests/test_audio_signature.cpp:129`](../tests/test_audio_signature.cpp#L129) — `Unknown`
|
||||
- [`tests/test_audio_signature.cpp:151`](../tests/test_audio_signature.cpp#L151) — `Unknown`
|
||||
- [`tests/test_audio_signature.cpp:167`](../tests/test_audio_signature.cpp#L167) — `Unknown`
|
||||
- [`tests/test_audio_signature.cpp:236`](../tests/test_audio_signature.cpp#L236) — `Unknown`
|
||||
- [`tests/test_audio_signature.cpp:263`](../tests/test_audio_signature.cpp#L263) — `TempWav w("centred300");`
|
||||
- [`tests/test_audio_signature.cpp:301`](../tests/test_audio_signature.cpp#L301) — `Unknown`
|
||||
- [`tests/test_audio_signature.cpp:314`](../tests/test_audio_signature.cpp#L314) — `kNumBands * std::log10(1000.0 / kBandLoHz) / std::log10(kBandHiHz / kBandLoHz)));`
|
||||
- [`tests/test_audio_signature.cpp:328`](../tests/test_audio_signature.cpp#L328) — `std::vector<float> a(kWindowSamples / 50);`
|
||||
- [`tests/test_audio_signature.cpp:344`](../tests/test_audio_signature.cpp#L344) — `return base64_encode(reinterpret_cast<const std::uint8_t*>(s.data()), s.size());`
|
||||
- [`scripts/validation/test_audio_offset.py:5`](../scripts/validation/test_audio_offset.py#L5) — `The golden vector (IR-005) proves the *arithmetic* is identical in both`
|
||||
|
||||
### IR-005
|
||||
|
||||
**Locations:** 6
|
||||
|
||||
- [`src/audio_bindings.cpp:3`](../src/audio_bindings.cpp#L3) — `Unknown`
|
||||
- [`src/audio_signature.cpp:421`](../src/audio_signature.cpp#L421) — `std::optional<std::string> compute_signature(const std::string& path)`
|
||||
- [`src/audio_signature.hpp:4`](../src/audio_signature.hpp#L4) — `Unknown`
|
||||
- [`tests/test_audio_signature.cpp:3`](../tests/test_audio_signature.cpp#L3) — `Unknown`
|
||||
- [`tests/test_audio_signature.cpp:129`](../tests/test_audio_signature.cpp#L129) — `Unknown`
|
||||
- [`tests/test_audio_signature.cpp:138`](../tests/test_audio_signature.cpp#L138) — `Unknown`
|
||||
|
||||
### IR-006
|
||||
|
||||
**Locations:** 1
|
||||
|
||||
- [`scripts/run_from_jellyfin.py:4`](../scripts/run_from_jellyfin.py#L4) — `Unknown`
|
||||
|
||||
### IR-007
|
||||
|
||||
**Locations:** 8
|
||||
|
||||
- [`src/audio_signature.cpp:3`](../src/audio_signature.cpp#L3) — `Unknown`
|
||||
- [`src/audio_signature.cpp:327`](../src/audio_signature.cpp#L327) — `std::optional<std::vector<float>> decode_centre_window(const std::string& path)`
|
||||
- [`src/audio_signature.cpp:421`](../src/audio_signature.cpp#L421) — `std::optional<std::string> compute_signature(const std::string& path)`
|
||||
- [`src/audio_signature.hpp:4`](../src/audio_signature.hpp#L4) — `Unknown`
|
||||
- [`tests/test_audio_signature.cpp:3`](../tests/test_audio_signature.cpp#L3) — `Unknown`
|
||||
- [`tests/test_audio_signature.cpp:201`](../tests/test_audio_signature.cpp#L201) — `TempWav w("short30");`
|
||||
- [`tests/test_audio_signature.cpp:219`](../tests/test_audio_signature.cpp#L219) — `TempWav w("exact120");`
|
||||
- [`tests/test_audio_signature.cpp:229`](../tests/test_audio_signature.cpp#L229) — `TempWav w("exact120");`
|
||||
|
||||
### IR-008
|
||||
|
||||
**Locations:** 7
|
||||
|
||||
- [`src/audio_signature.cpp:3`](../src/audio_signature.cpp#L3) — `Unknown`
|
||||
- [`src/audio_signature.cpp:320`](../src/audio_signature.cpp#L320) — `std::optional<std::string> signature_from_mono(const std::vector<float>& mono)`
|
||||
- [`src/audio_signature.cpp:421`](../src/audio_signature.cpp#L421) — `std::optional<std::string> compute_signature(const std::string& path)`
|
||||
- [`src/audio_signature.hpp:4`](../src/audio_signature.hpp#L4) — `Unknown`
|
||||
- [`tests/test_audio_signature.cpp:3`](../tests/test_audio_signature.cpp#L3) — `Unknown`
|
||||
- [`tests/test_audio_signature.cpp:129`](../tests/test_audio_signature.cpp#L129) — `Unknown`
|
||||
- [`tests/test_audio_signature.cpp:167`](../tests/test_audio_signature.cpp#L167) — `Unknown`
|
||||
|
||||
### IT-001
|
||||
|
||||
**Locations:** 1
|
||||
|
||||
- [`tests/test_replay_fixtures.cpp:3`](../tests/test_replay_fixtures.cpp#L3) — `Unknown`
|
||||
|
||||
### PR-002
|
||||
|
||||
**Locations:** 10
|
||||
|
||||
- [`src/nodes/embedding_dump_node.hpp:2`](../src/nodes/embedding_dump_node.hpp#L2) — `Unknown`
|
||||
- [`src/nodes/embedding_dump_node.hpp:18`](../src/nodes/embedding_dump_node.hpp#L18) — `Unknown`
|
||||
- [`src/nodes/embedding_dump_node.hpp:133`](../src/nodes/embedding_dump_node.hpp#L133) — `static constexpr std::string_view label() { return "embedding_dump"; }`
|
||||
- [`src/nodes/embedding_dump_node.hpp:159`](../src/nodes/embedding_dump_node.hpp#L159) — `void operator()(EmbeddedSceneFrame ef)`
|
||||
- [`src/nodes/embedding_dump_node.hpp:242`](../src/nodes/embedding_dump_node.hpp#L242) — `H5::H5File file(path_, H5F_ACC_TRUNC);`
|
||||
- [`scripts/optimizer/replay.py:5`](../scripts/optimizer/replay.py#L5) — `Reads an embedding dump (scripts/optimizer/SCHEMA.md), feeds each frame as an`
|
||||
- [`scripts/optimizer/second_score.py:5`](../scripts/optimizer/second_score.py#L5) — `Unknown`
|
||||
- [`scripts/validation/ground_truth.py:24`](../scripts/validation/ground_truth.py#L24) — `Unknown`
|
||||
- [`experiments/xsource/resolution_sweep.py:4`](../experiments/xsource/resolution_sweep.py#L4) — `Unknown`
|
||||
- [`experiments/xsource/verify_labels.py:4`](../experiments/xsource/verify_labels.py#L4) — `Unknown`
|
||||
|
||||
### PR-004
|
||||
|
||||
**Locations:** 1
|
||||
|
||||
- [`src/main.cpp:3`](../src/main.cpp#L3) — `Unknown`
|
||||
|
||||
### SR-001
|
||||
|
||||
**Locations:** 64
|
||||
|
||||
- [`src/backends/gemm_backend.cpp:44`](../src/backends/gemm_backend.cpp#L44) — `constexpr int kDim = 512;`
|
||||
- [`src/build_gallery.cpp:83`](../src/build_gallery.cpp#L83) — `Unknown`
|
||||
- [`src/config.hpp:54`](../src/config.hpp#L54) — `Unknown`
|
||||
- [`src/gallery/embedder_stamp.cpp:1`](../src/gallery/embedder_stamp.cpp#L1) — `Unknown`
|
||||
- [`src/gallery/embedder_stamp.hpp:2`](../src/gallery/embedder_stamp.hpp#L2) — `Unknown`
|
||||
- [`src/gallery/gallery_builder.cpp:45`](../src/gallery/gallery_builder.cpp#L45) — `ActorGallery build_gallery(const BuildConfig& cfg)`
|
||||
- [`src/gallery/gallery_calibration.hpp:80`](../src/gallery/gallery_calibration.hpp#L80) — `struct GalleryCalibrationStats`
|
||||
- [`src/gallery/gallery_calibration.hpp:145`](../src/gallery/gallery_calibration.hpp#L145) — `std::vector<bool> actor_eligible(n_actors, false);`
|
||||
- [`src/gallery/gallery_calibration.hpp:294`](../src/gallery/gallery_calibration.hpp#L294) — `Unknown`
|
||||
- [`src/gallery/gallery_report.hpp:2`](../src/gallery/gallery_report.hpp#L2) — `Unknown`
|
||||
- [`src/gallery/gallery_report.hpp:52`](../src/gallery/gallery_report.hpp#L52) — `struct GalleryBuildAudit`
|
||||
- [`src/gallery/gallery_report.hpp:73`](../src/gallery/gallery_report.hpp#L73) — `struct GalleryReport`
|
||||
- [`src/gallery/gallery_report.hpp:154`](../src/gallery/gallery_report.hpp#L154) — `inline GalleryReport build_gallery_report(const ActorGallery& gallery,`
|
||||
- [`src/gallery/gallery_report.hpp:295`](../src/gallery/gallery_report.hpp#L295) — `inline nlohmann::json gallery_report_to_json(const GalleryReport& r)`
|
||||
- [`src/gallery/gallery_report.hpp:361`](../src/gallery/gallery_report.hpp#L361) — `inline GalleryReport gallery_report_from_json(const nlohmann::json& j)`
|
||||
- [`src/gallery/gallery_report.hpp:446`](../src/gallery/gallery_report.hpp#L446) — `inline void save_gallery_report(const std::string& path, const GalleryReport& r)`
|
||||
- [`src/gallery/gallery_report.hpp:454`](../src/gallery/gallery_report.hpp#L454) — `inline GalleryReport load_gallery_report(const std::string& path)`
|
||||
- [`src/gallery/gallery_report.hpp:464`](../src/gallery/gallery_report.hpp#L464) — `return gallery_report_from_json(j);`
|
||||
- [`src/gallery/gallery_store.cpp:82`](../src/gallery/gallery_store.cpp#L82) — `H5::StrType str(H5::PredType::C_S1, H5T_VARIABLE);`
|
||||
- [`src/gallery/gallery_store.cpp:167`](../src/gallery/gallery_store.cpp#L167) — `H5::DataSpace scalar(H5S_SCALAR);`
|
||||
- [`src/gallery/gallery_store.cpp:219`](../src/gallery/gallery_store.cpp#L219) — `Unknown`
|
||||
- [`src/kpn_bindings.cpp:167`](../src/kpn_bindings.cpp#L167) — `Unknown`
|
||||
- [`src/kpn_bindings.cpp:217`](../src/kpn_bindings.cpp#L217) — `Unknown`
|
||||
- [`src/main.cpp:188`](../src/main.cpp#L188) — `Unknown`
|
||||
- [`src/nodes/embedding_dump_node.hpp:127`](../src/nodes/embedding_dump_node.hpp#L127) — `static constexpr std::string_view label() { return "embedding_dump"; }`
|
||||
- [`src/nodes/embedding_dump_node.hpp:238`](../src/nodes/embedding_dump_node.hpp#L238) — `H5::H5File file(path_, H5F_ACC_TRUNC);`
|
||||
- [`src/scene_preview.cpp:133`](../src/scene_preview.cpp#L133) — `int main(int argc, char** argv)`
|
||||
- [`src/types.hpp:148`](../src/types.hpp#L148) — `struct Actor`
|
||||
- [`tests/test_calibration.cpp:192`](../tests/test_calibration.cpp#L192) — `Embedding unit_axis(int slot)`
|
||||
- [`tests/test_calibration.cpp:222`](../tests/test_calibration.cpp#L222) — `Unknown`
|
||||
- [`tests/test_calibration.cpp:248`](../tests/test_calibration.cpp#L248) — `Unknown`
|
||||
- [`tests/test_gallery_store.cpp:182`](../tests/test_gallery_store.cpp#L182) — `TempFile tf("gallery_stamped.h5");`
|
||||
- [`tests/test_gallery_store.cpp:201`](../tests/test_gallery_store.cpp#L201) — `TempFile tf("gallery_stamped.h5");`
|
||||
- [`tests/test_gallery_store.cpp:219`](../tests/test_gallery_store.cpp#L219) — `TempFile tf("gallery_unstamped.h5");`
|
||||
- [`tests/test_gallery_store.cpp:240`](../tests/test_gallery_store.cpp#L240) — `TempFile tf("gallery_json_stamp.json");`
|
||||
- [`tests/test_gallery_store.cpp:252`](../tests/test_gallery_store.cpp#L252) — `Unknown`
|
||||
- [`tests/test_gallery_store.cpp:268`](../tests/test_gallery_store.cpp#L268) — `Unknown`
|
||||
- [`tests/test_gallery_store.cpp:300`](../tests/test_gallery_store.cpp#L300) — `Unknown`
|
||||
- [`tests/test_gallery_store.cpp:309`](../tests/test_gallery_store.cpp#L309) — `Unknown`
|
||||
- [`tests/test_gallery_store.cpp:333`](../tests/test_gallery_store.cpp#L333) — `Unknown`
|
||||
- [`tests/test_gallery_store.cpp:348`](../tests/test_gallery_store.cpp#L348) — `Unknown`
|
||||
- [`tests/test_gallery_store.cpp:367`](../tests/test_gallery_store.cpp#L367) — `Unknown`
|
||||
- [`tests/test_gallery_store.cpp:384`](../tests/test_gallery_store.cpp#L384) — `TempFile tf("fake_model.onnx");`
|
||||
- [`tests/test_similarity.cpp:1`](../tests/test_similarity.cpp#L1) — `Unknown`
|
||||
- [`scripts/filter_gallery.py:80`](../scripts/filter_gallery.py#L80) — `if actor_jellyfin_id(a) in cast_ids]`
|
||||
- [`scripts/make_gallery.py:181`](../scripts/make_gallery.py#L181) — `Unknown`
|
||||
- [`scripts/make_jellyfin_gallery.py:4`](../scripts/make_jellyfin_gallery.py#L4) — `Unknown`
|
||||
- [`scripts/make_jellyfin_gallery.py:448`](../scripts/make_jellyfin_gallery.py#L448) — `Unknown`
|
||||
- [`scripts/make_jellyfin_gallery.py:456`](../scripts/make_jellyfin_gallery.py#L456) — `Unknown`
|
||||
- [`scripts/movienet_eval.py:65`](../scripts/movienet_eval.py#L65) — `with open(args.gt) as f:`
|
||||
- [`scripts/optimizer/fetch_missing_actors.py:62`](../scripts/optimizer/fetch_missing_actors.py#L62) — `def fetch(missing_path, out_path, token, build_dir, models_dir, arcface,`
|
||||
- [`scripts/optimizer/fetch_missing_actors.py:109`](../scripts/optimizer/fetch_missing_actors.py#L109) — `def merge(base_path, add_path, out_path):`
|
||||
- [`scripts/optimizer/fetch_missing_actors.py:124`](../scripts/optimizer/fetch_missing_actors.py#L124) — `def merge(base_path, add_path, out_path):`
|
||||
- [`scripts/optimizer/optimize.py:186`](../scripts/optimizer/optimize.py#L186) — `Unknown`
|
||||
- [`scripts/optimizer/optimize.py:202`](../scripts/optimizer/optimize.py#L202) — `if not Path(f["dump"]).exists():`
|
||||
- [`scripts/optimizer/reembed_gallery.py:62`](../scripts/optimizer/reembed_gallery.py#L62) — `for i, a in enumerate(ref["actors"], 1):`
|
||||
- [`scripts/optimizer/replay.py:113`](../scripts/optimizer/replay.py#L113) — `Unknown`
|
||||
- [`scripts/optimizer/replay.py:253`](../scripts/optimizer/replay.py#L253) — `Unknown`
|
||||
- [`scripts/run_from_jellyfin.py:4`](../scripts/run_from_jellyfin.py#L4) — `Unknown`
|
||||
- [`scripts/sae_embed_loader.py:23`](../scripts/sae_embed_loader.py#L23) — `def resolve_arcface(models_dir: str, arcface: str \| None = None) -> str:`
|
||||
- [`scripts/sae_gallery.py:171`](../scripts/sae_gallery.py#L171) — `if not _stamp_empty(embedder):`
|
||||
- [`scripts/sae_gallery.py:200`](../scripts/sae_gallery.py#L200) — `for a in range(len(offset)):`
|
||||
- [`scripts/sae_stamp.py:3`](../scripts/sae_stamp.py#L3) — `Unknown`
|
||||
- [`scripts/stamp_gallery.py:4`](../scripts/stamp_gallery.py#L4) — `Unknown`
|
||||
|
||||
### SR-002
|
||||
|
||||
**Locations:** 35
|
||||
|
||||
- [`src/config.hpp:44`](../src/config.hpp#L44) — `Unknown`
|
||||
- [`src/config.hpp:108`](../src/config.hpp#L108) — `Unknown`
|
||||
- [`src/evidence_discount.hpp:2`](../src/evidence_discount.hpp#L2) — `Unknown`
|
||||
- [`src/face_utils.hpp:2`](../src/face_utils.hpp#L2) — `Unknown`
|
||||
- [`src/gallery/gallery_calibration.hpp:2`](../src/gallery/gallery_calibration.hpp#L2) — `Unknown`
|
||||
- [`src/gallery/gallery_calibration.hpp:53`](../src/gallery/gallery_calibration.hpp#L53) — `float boundary_at(float p = 0.5f, float log_prior_odds = 0.f) const`
|
||||
- [`src/main.cpp:86`](../src/main.cpp#L86) — `static constexpr std::size_t kSceneJoinDepth = 256;`
|
||||
- [`src/main.cpp:213`](../src/main.cpp#L213) — `reg_cfg, EvidenceDiscounter(same_person));`
|
||||
- [`src/main.cpp:230`](../src/main.cpp#L230) — `reg_cfg, EvidenceDiscounter(same_person));`
|
||||
- [`src/main.cpp:319`](../src/main.cpp#L319) — `Unknown`
|
||||
- [`src/main.cpp:329`](../src/main.cpp#L329) — `Unknown`
|
||||
- [`src/main.cpp:399`](../src/main.cpp#L399) — `return run_net(std::move(net));`
|
||||
- [`src/main.cpp:431`](../src/main.cpp#L431) — `Unknown`
|
||||
- [`src/nodes/camera_position_change_detector_node.hpp:30`](../src/nodes/camera_position_change_detector_node.hpp#L30) — `struct CameraPositionChangeDetectorFunc`
|
||||
- [`src/nodes/embedder_node.hpp:21`](../src/nodes/embedder_node.hpp#L21) — `struct EmbedderFunc`
|
||||
- [`src/nodes/face_aligner_node.hpp:7`](../src/nodes/face_aligner_node.hpp#L7) — `struct FaceAlignerFunc`
|
||||
- [`src/nodes/face_detector_node.hpp:2`](../src/nodes/face_detector_node.hpp#L2) — `Unknown`
|
||||
- [`src/nodes/face_detector_node.hpp:47`](../src/nodes/face_detector_node.hpp#L47) — `private:`
|
||||
- [`src/nodes/face_tracker_node.hpp:2`](../src/nodes/face_tracker_node.hpp#L2) — `Unknown`
|
||||
- [`src/nodes/identity_matcher_node.hpp:117`](../src/nodes/identity_matcher_node.hpp#L117) — `const GalleryCalibration& calibration() const { return cal_; }`
|
||||
- [`src/nodes/identity_matcher_node.hpp:125`](../src/nodes/identity_matcher_node.hpp#L125) — `const GalleryCalibration& calibration() const { return cal_; }`
|
||||
- [`src/nodes/identity_matcher_node.hpp:166`](../src/nodes/identity_matcher_node.hpp#L166) — `std::vector<float> host_query(static_cast<size_t>(kMaxFaces) * 512);`
|
||||
- [`src/nodes/identity_matcher_node.hpp:272`](../src/nodes/identity_matcher_node.hpp#L272) — `Unknown`
|
||||
- [`src/nodes/result_sink_node.hpp:49`](../src/nodes/result_sink_node.hpp#L49) — `static constexpr std::string_view label() { return "result_sink"; }`
|
||||
- [`src/nodes/result_sink_node.hpp:63`](../src/nodes/result_sink_node.hpp#L63) — `void set_pre_write_hook(std::function<void(double)> fn) { pre_write_ = std::move(fn); }`
|
||||
- [`src/nodes/result_sink_node.hpp:161`](../src/nodes/result_sink_node.hpp#L161) — `struct ActorMeta { std::string name, imdb_id, tmdb_id, jellyfin_id; };`
|
||||
- [`src/nodes/scene_boundary_annotator_node.hpp:2`](../src/nodes/scene_boundary_annotator_node.hpp#L2) — `Unknown`
|
||||
- [`src/nodes/scene_detector_node.hpp:36`](../src/nodes/scene_detector_node.hpp#L36) — `static constexpr std::string_view label() { return "scene_detector"; }`
|
||||
- [`src/nodes/scene_detector_node.hpp:110`](../src/nodes/scene_detector_node.hpp#L110) — `void flush_remaining()`
|
||||
- [`src/nodes/scene_detector_node.hpp:144`](../src/nodes/scene_detector_node.hpp#L144) — `void write_output()`
|
||||
- [`src/scene_boundaries.hpp:2`](../src/scene_boundaries.hpp#L2) — `Unknown`
|
||||
- [`src/track_registry.hpp:2`](../src/track_registry.hpp#L2) — `Unknown`
|
||||
- [`tests/test_calibration.cpp:1`](../tests/test_calibration.cpp#L1) — `Unknown`
|
||||
- [`tests/test_face_tracker.cpp:1`](../tests/test_face_tracker.cpp#L1) — `Unknown`
|
||||
- [`tests/test_face_utils.cpp:1`](../tests/test_face_utils.cpp#L1) — `Unknown`
|
||||
|
||||
### SR-003
|
||||
|
||||
**Locations:** 7
|
||||
|
||||
- [`src/audio_bindings.cpp:3`](../src/audio_bindings.cpp#L3) — `Unknown`
|
||||
- [`src/audio_signature.cpp:3`](../src/audio_signature.cpp#L3) — `Unknown`
|
||||
- [`src/audio_signature.hpp:4`](../src/audio_signature.hpp#L4) — `Unknown`
|
||||
- [`src/config.hpp:20`](../src/config.hpp#L20) — `struct Config`
|
||||
- [`src/nodes/result_sink_node.hpp:2`](../src/nodes/result_sink_node.hpp#L2) — `Unknown`
|
||||
- [`src/nodes/result_sink_node.hpp:49`](../src/nodes/result_sink_node.hpp#L49) — `static constexpr std::string_view label() { return "result_sink"; }`
|
||||
- [`src/nodes/result_sink_node.hpp:122`](../src/nodes/result_sink_node.hpp#L122) — `void write_output()`
|
||||
|
||||
### SR-005
|
||||
|
||||
**Locations:** 11
|
||||
|
||||
- [`src/config.hpp:152`](../src/config.hpp#L152) — `Unknown`
|
||||
- [`src/gallery/gallery_store.hpp:15`](../src/gallery/gallery_store.hpp#L15) — `Unknown`
|
||||
- [`src/gallery/track_gallery.hpp:123`](../src/gallery/track_gallery.hpp#L123) — `void forget(int track_id) { tracks_.erase(track_id); }`
|
||||
- [`src/gallery/track_gallery.hpp:133`](../src/gallery/track_gallery.hpp#L133) — `void set_calibration(std::function<float(float)> c) { calibrate_ = std::move(c); }`
|
||||
- [`src/gallery/track_gallery.hpp:164`](../src/gallery/track_gallery.hpp#L164) — `struct TrackState`
|
||||
- [`src/gallery/track_gallery.hpp:274`](../src/gallery/track_gallery.hpp#L274) — `static int plurality_actor(const TrackState& ts)`
|
||||
- [`src/nodes/identity_matcher_node.hpp:110`](../src/nodes/identity_matcher_node.hpp#L110) — `std::vector<float> host_gallery(static_cast<size_t>(n_gallery_) * 512);`
|
||||
- [`src/nodes/identity_matcher_node.hpp:147`](../src/nodes/identity_matcher_node.hpp#L147) — `MatchedSceneFrame operator()(TrackedSceneFrame tf)`
|
||||
- [`src/nodes/identity_matcher_node.hpp:285`](../src/nodes/identity_matcher_node.hpp#L285) — `Unknown`
|
||||
- [`tests/test_track_gallery.cpp:1`](../tests/test_track_gallery.cpp#L1) — `Unknown`
|
||||
- [`scripts/make_jellyfin_gallery.py:4`](../scripts/make_jellyfin_gallery.py#L4) — `Unknown`
|
||||
|
||||
### UT-001
|
||||
|
||||
**Locations:** 1
|
||||
|
||||
- [`tests/test_track_registry.cpp:3`](../tests/test_track_registry.cpp#L3) — `Unknown`
|
||||
|
||||
### UT-101
|
||||
|
||||
**Locations:** 5
|
||||
|
||||
- [`tests/test_audio_signature.cpp:3`](../tests/test_audio_signature.cpp#L3) — `Unknown`
|
||||
- [`tests/test_audio_signature.cpp:129`](../tests/test_audio_signature.cpp#L129) — `Unknown`
|
||||
- [`tests/test_audio_signature.cpp:138`](../tests/test_audio_signature.cpp#L138) — `Unknown`
|
||||
- [`tests/test_audio_signature.cpp:151`](../tests/test_audio_signature.cpp#L151) — `Unknown`
|
||||
- [`tests/test_audio_signature.cpp:167`](../tests/test_audio_signature.cpp#L167) — `Unknown`
|
||||
|
||||
### UT-102
|
||||
|
||||
**Locations:** 5
|
||||
|
||||
- [`tests/test_audio_signature.cpp:3`](../tests/test_audio_signature.cpp#L3) — `Unknown`
|
||||
- [`tests/test_audio_signature.cpp:201`](../tests/test_audio_signature.cpp#L201) — `TempWav w("short30");`
|
||||
- [`tests/test_audio_signature.cpp:219`](../tests/test_audio_signature.cpp#L219) — `TempWav w("exact120");`
|
||||
- [`tests/test_audio_signature.cpp:229`](../tests/test_audio_signature.cpp#L229) — `TempWav w("exact120");`
|
||||
- [`tests/test_audio_signature.cpp:236`](../tests/test_audio_signature.cpp#L236) — `Unknown`
|
||||
|
||||
### UT-103
|
||||
|
||||
**Locations:** 2
|
||||
|
||||
- [`tests/test_audio_signature.cpp:3`](../tests/test_audio_signature.cpp#L3) — `Unknown`
|
||||
- [`tests/test_audio_signature.cpp:263`](../tests/test_audio_signature.cpp#L263) — `TempWav w("centred300");`
|
||||
|
||||
### UT-104
|
||||
|
||||
**Locations:** 5
|
||||
|
||||
- [`tests/test_audio_signature.cpp:3`](../tests/test_audio_signature.cpp#L3) — `Unknown`
|
||||
- [`tests/test_audio_signature.cpp:301`](../tests/test_audio_signature.cpp#L301) — `Unknown`
|
||||
- [`tests/test_audio_signature.cpp:314`](../tests/test_audio_signature.cpp#L314) — `kNumBands * std::log10(1000.0 / kBandLoHz) / std::log10(kBandHiHz / kBandLoHz)));`
|
||||
- [`tests/test_audio_signature.cpp:328`](../tests/test_audio_signature.cpp#L328) — `std::vector<float> a(kWindowSamples / 50);`
|
||||
- [`tests/test_audio_signature.cpp:344`](../tests/test_audio_signature.cpp#L344) — `return base64_encode(reinterpret_cast<const std::uint8_t*>(s.data()), s.size());`
|
||||
|
||||
### UT-105
|
||||
|
||||
**Locations:** 1
|
||||
|
||||
- [`scripts/validation/test_audio_offset.py:5`](../scripts/validation/test_audio_offset.py#L5) — `The golden vector (IR-005) proves the *arithmetic* is identical in both`
|
||||
|
||||
### UT-106
|
||||
|
||||
**Locations:** 1
|
||||
|
||||
- [`scripts/validation/test_audio_offset.py:5`](../scripts/validation/test_audio_offset.py#L5) — `The golden vector (IR-005) proves the *arithmetic* is identical in both`
|
||||
|
||||
### UT-107
|
||||
|
||||
**Locations:** 1
|
||||
|
||||
- [`scripts/validation/test_audio_offset.py:5`](../scripts/validation/test_audio_offset.py#L5) — `The golden vector (IR-005) proves the *arithmetic* is identical in both`
|
||||
|
||||
### UT-108
|
||||
|
||||
**Locations:** 1
|
||||
|
||||
- [`scripts/validation/test_audio_offset.py:5`](../scripts/validation/test_audio_offset.py#L5) — `The golden vector (IR-005) proves the *arithmetic* is identical in both`
|
||||
|
||||
### VR-001
|
||||
|
||||
**Locations:** 2
|
||||
|
||||
- [`src/nodes/embedding_dump_node.hpp:2`](../src/nodes/embedding_dump_node.hpp#L2) — `Unknown`
|
||||
- [`tests/test_replay_fixtures.cpp:3`](../tests/test_replay_fixtures.cpp#L3) — `Unknown`
|
||||
|
||||
### VR-002
|
||||
|
||||
**Locations:** 2
|
||||
|
||||
- [`tests/test_replay_fixtures.cpp:3`](../tests/test_replay_fixtures.cpp#L3) — `Unknown`
|
||||
- [`scripts/optimizer/replay.py:5`](../scripts/optimizer/replay.py#L5) — `Reads an embedding dump (scripts/optimizer/SCHEMA.md), feeds each frame as an`
|
||||
|
||||
### VR-003
|
||||
|
||||
**Locations:** 1
|
||||
|
||||
- [`scripts/optimizer/second_score.py:5`](../scripts/optimizer/second_score.py#L5) — `Unknown`
|
||||
|
||||
### VR-004
|
||||
|
||||
**Locations:** 1
|
||||
|
||||
- [`scripts/validation/ground_truth.py:24`](../scripts/validation/ground_truth.py#L24) — `Unknown`
|
||||
|
||||
### VR-005
|
||||
|
||||
**Locations:** 1
|
||||
|
||||
- [`scripts/validation/min_face_size.py:5`](../scripts/validation/min_face_size.py#L5) — `Unknown`
|
||||
|
||||
### VR-010
|
||||
|
||||
**Locations:** 5
|
||||
|
||||
- [`src/nodes/embedding_dump_node.hpp:2`](../src/nodes/embedding_dump_node.hpp#L2) — `Unknown`
|
||||
- [`src/nodes/embedding_dump_node.hpp:18`](../src/nodes/embedding_dump_node.hpp#L18) — `Unknown`
|
||||
- [`src/nodes/embedding_dump_node.hpp:133`](../src/nodes/embedding_dump_node.hpp#L133) — `static constexpr std::string_view label() { return "embedding_dump"; }`
|
||||
- [`src/nodes/embedding_dump_node.hpp:159`](../src/nodes/embedding_dump_node.hpp#L159) — `void operator()(EmbeddedSceneFrame ef)`
|
||||
- [`src/nodes/embedding_dump_node.hpp:242`](../src/nodes/embedding_dump_node.hpp#L242) — `H5::H5File file(path_, H5F_ACC_TRUNC);`
|
||||
|
||||
### VR-013
|
||||
|
||||
**Locations:** 2
|
||||
|
||||
- [`experiments/xsource/resolution_sweep.py:4`](../experiments/xsource/resolution_sweep.py#L4) — `Unknown`
|
||||
- [`experiments/xsource/verify_labels.py:4`](../experiments/xsource/verify_labels.py#L4) — `Unknown`
|
||||
|
||||
### VR-014
|
||||
|
||||
**Locations:** 1
|
||||
|
||||
- [`scripts/validation/test_audio_offset.py:5`](../scripts/validation/test_audio_offset.py#L5) — `The golden vector (IR-005) proves the *arithmetic* is identical in both`
|
||||
_No TRACES tags found yet. Tags are added as code is written; an empty matrix on a new tree is the correct reading, not a failure._
|
||||
|
||||
|
||||
@@ -11,17 +11,6 @@ manifests/
|
||||
trajectories/
|
||||
results/
|
||||
|
||||
# Cross-source identification study: source clips and the hand-sorted face
|
||||
# crops. The sorting is human ground truth and expensive to redo, so it goes to
|
||||
# the artifact registry rather than being regenerated — push it once sorted.
|
||||
xsource/clips/
|
||||
xsource/labelling/
|
||||
xsource/frames/
|
||||
xsource/cache/
|
||||
xsource/results_*.json
|
||||
xsource/failure_analysis.json
|
||||
xsource/*.jpg
|
||||
|
||||
# Raw run logs and scratch scripts (regenerated by every run).
|
||||
_scratch/
|
||||
|
||||
|
||||
@@ -1,92 +0,0 @@
|
||||
# xsource — cross-source identification probe (VR-013)
|
||||
|
||||
Gallery from **one** recording, probes from **another**, swept over the probe's
|
||||
input resolution. Complements VR-005, which asked the same question over gallery
|
||||
mugshots: that one degrades an already-aligned 112×112 crop, holding alignment
|
||||
perfect, so it isolates the embedder. This one downscales the **whole frame**
|
||||
before the detector, so detection and landmark regression degrade with it.
|
||||
|
||||
Corpus: two Pexels clips of one shoot (4096×2160, 25 fps), four people, all four
|
||||
present in both.
|
||||
|
||||
## Getting the data
|
||||
|
||||
Clips, frames and hand-sorted crops are gitignored; they live in the artifact
|
||||
registry.
|
||||
|
||||
scripts/artifacts/pull_artifacts.sh xsource # clips + labelling, frames regenerated
|
||||
scripts/artifacts/push_artifacts.sh xsource # after correcting labels
|
||||
|
||||
Pulling fetches the two clips and the hand-sorted crops, then regenerates the
|
||||
frames with ffmpeg — ~320 MB of PNG that is deterministic from the clips, so it
|
||||
is not worth shipping. Extraction settings are pinned in the pull script because
|
||||
the manifests key on frame filenames *and* on detection order within each frame;
|
||||
`verify_labels.py` runs at the end and will fail loudly if they drift.
|
||||
|
||||
Pull never overwrites an existing `labelling/`. That directory is human ground
|
||||
truth — somebody looked at all 167 crops and put each one in a folder — and it
|
||||
is the expensive part of this study, so push it once corrected.
|
||||
|
||||
Clips are Pexels-licensed: free to use, no attribution required, but not
|
||||
CC or MIT. Fine as a frozen CI artifact on private infrastructure; do not
|
||||
redistribute them as stock content.
|
||||
|
||||
## Scripts
|
||||
|
||||
| script | does |
|
||||
|---|---|
|
||||
| `dump_faces.py` | detect every face, write a context crop per detection + a manifest |
|
||||
| `redraw_boxes.py` | redraw those crops with the detection boxed, in place |
|
||||
| `propose_labels.py` | propose labels for one clip from another clip's hand-sorted folders |
|
||||
| `make_review_site.py` | local `review.html` — current label, crop, better match, correct and export |
|
||||
| `apply_corrections.py` | apply the exported `corrections.json` |
|
||||
| `verify_labels.py` | integrity gate: index consistency, duplicates, separation. Exits non-zero on failure |
|
||||
| `resolution_sweep.py` | the VR-013 measurement |
|
||||
| `failure_analysis.py` | what explains the misses — pose, size, blur, detector confidence |
|
||||
| `landmark_voting.py` | average SCRFD's overlapping detections instead of discarding them |
|
||||
| `pose_label.py` | mesh-estimated head pose, for hand correction (feeds VR-012) |
|
||||
|
||||
Everything drives the shipped C++ through `sae_embed`; nothing reimplements
|
||||
detection, alignment, the embedder or the calibration. Scoring goes through the
|
||||
production gallery sigmoid — never a raw cosine (AR-024).
|
||||
|
||||
LD_PRELOAD=/usr/lib/libcudnn_cnn.so.9 python3 resolution_sweep.py
|
||||
|
||||
The preload is needed while ORT's CUDA provider looks for
|
||||
`cudnnGetConvolutionBackwardDataAlgorithm_v7`, which cuDNN 9 moved into
|
||||
`libcudnn_cnn.so.9` behind a dispatch stub. Without it everything silently falls
|
||||
back to CPU.
|
||||
|
||||
## What it found
|
||||
|
||||
**Resolution is not the binding constraint here.** TPI holds ~41–47% from 4096×2160
|
||||
down to ~45 px faces, then falls: 23 px → 26%, 18 px → 12%, 14 px → 1.5%. Holding
|
||||
90% of the plateau needs roughly 50 px end to end, against VR-005's ~22 px — the
|
||||
gap is detection and landmark error, which VR-005 excludes by construction.
|
||||
|
||||
**FPI is 0.0% at every scale.** Resolution loss goes entirely to TBI: the pipeline
|
||||
stops naming people rather than naming the wrong one.
|
||||
|
||||
**The ceiling is cross-view, not resolution.** Every person matches themselves
|
||||
strongly *within* a recording (sim 0.55–0.85) and collapses *across* the two
|
||||
(0.14–0.45, threshold 0.335). Only the person with frontal **gallery** references
|
||||
identified reliably, whatever their probe pose — so the lever is gallery pose
|
||||
coverage (`docs/pose-expansion.md`), not a better landmark model.
|
||||
|
||||
**Landmark voting helps.** SCRFD predicts each face from several anchors and NMS
|
||||
discards all but one, throwing away a median of 3 landmark estimates per face.
|
||||
Averaging them, weighted by confidence, lifts cross-clip TPI 41% → 49% for one
|
||||
forward pass and no extra model. A MediaPipe mesh as landmark source went the
|
||||
other way (41% → 16%): more stable within a recording, but a ring centroid is not
|
||||
the annotated landmark ArcFace was trained on, and the embedder punishes the
|
||||
off-distribution crop.
|
||||
|
||||
## Reading these numbers
|
||||
|
||||
Four identities, 70 probes, one shoot. The ~47% plateau is pose, not resolution —
|
||||
half these faces are turned away and never clear threshold at any scale, so the
|
||||
absolute rates say little and the *shape* is the result. Both clips contain all
|
||||
four people, so there is no out-of-gallery class and the 10×-weighted out-of-cast
|
||||
misID is **untested** here; holding one identity out of the gallery would fix
|
||||
that. And the resolution curve is dominated by the single subject whose gallery
|
||||
references are frontal.
|
||||
@@ -1,44 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Apply corrections.json exported from review.html.
|
||||
|
||||
python3 apply_corrections.py ~/Downloads/corrections.json [--dry-run]
|
||||
|
||||
Moves each crop to the folder you chose. "discard" goes to labelling/<clip>/discard/,
|
||||
which the sweep ignores — nothing is deleted, so a misclick is recoverable.
|
||||
|
||||
Refuses to move a file it cannot find exactly once, rather than guessing: a
|
||||
half-applied correction set would put a crop in two folders and quietly
|
||||
duplicate a label.
|
||||
"""
|
||||
import sys, json, glob, os, shutil
|
||||
|
||||
if len(sys.argv) < 2:
|
||||
sys.exit(__doc__)
|
||||
path = sys.argv[1]
|
||||
DRY = "--dry-run" in sys.argv
|
||||
corr = json.load(open(path))
|
||||
if not corr:
|
||||
sys.exit("no corrections in that file")
|
||||
|
||||
moved = skipped = 0
|
||||
for fname, c in corr.items():
|
||||
clip, to = c["clip"], c["to"]
|
||||
hits = glob.glob(f"labelling/{clip}/**/{fname}", recursive=True)
|
||||
if len(hits) != 1:
|
||||
print(f"[skip] {fname}: found {len(hits)} copies, expected 1")
|
||||
skipped += 1
|
||||
continue
|
||||
src = hits[0]
|
||||
dst_dir = f"labelling/{clip}/{to}"
|
||||
dst = f"{dst_dir}/{fname}"
|
||||
if os.path.abspath(src) == os.path.abspath(dst):
|
||||
continue
|
||||
print(f"{'would move' if DRY else 'move'} {c['from']} -> {to}: {fname}")
|
||||
if not DRY:
|
||||
os.makedirs(dst_dir, exist_ok=True)
|
||||
shutil.move(src, dst)
|
||||
moved += 1
|
||||
|
||||
print(f"\n{moved} moved, {skipped} skipped{' (dry run)' if DRY else ''}")
|
||||
if not DRY and moved:
|
||||
print("re-run verify_labels.py to confirm the set is still consistent")
|
||||
@@ -1,90 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Dump face crops from both clips for hand-labelling.
|
||||
|
||||
Writes labelling/<clip>/unsorted/<name>.jpg — a context crop around each
|
||||
detection, big enough to recognise a person by eye. Move them into
|
||||
labelling/<clip>/person_A/, person_B/, ... and the sweep reads those folders as
|
||||
ground truth.
|
||||
|
||||
Filenames carry a cNN_ cluster-hint prefix so visually similar faces sort next
|
||||
to each other in a file manager. The hint is only an ordering convenience —
|
||||
the folder you drop a file into is what counts, and the sweep never reads the
|
||||
prefix.
|
||||
|
||||
Detection and alignment run through the shipped C++ (sae_embed). Every crop
|
||||
keeps its clip, frame and native-resolution bbox in manifest.json, so probe
|
||||
detections at reduced scale can be tied back to a labelled face geometrically,
|
||||
by position, rather than by embedding similarity — which would be circular.
|
||||
"""
|
||||
import sys, glob, json, os, shutil
|
||||
import numpy as np
|
||||
import cv2
|
||||
|
||||
sys.path.insert(0, "/home/dtourolle/Development/Jray-project/scene-actor-extraction/build-ort")
|
||||
import sae_embed
|
||||
|
||||
M = "/home/dtourolle/Development/Jray-project/scene-actor-extraction/models/"
|
||||
CLIPS = ["5157339", "5157344"]
|
||||
MIN_PX = 60
|
||||
CTX = 256 # context-crop side, for human recognisability
|
||||
|
||||
eng = sae_embed.FaceEmbedder(detector_model=M + "scrfd_500m_bnkps.onnx",
|
||||
arcface_model=M + "arcface_w600k_r50.onnx",
|
||||
conf=0.5, nms=0.4, max_side=0)
|
||||
|
||||
for clip in CLIPS:
|
||||
out_dir = f"labelling/{clip}/unsorted"
|
||||
if os.path.isdir(f"labelling/{clip}"):
|
||||
print(f"[skip] labelling/{clip} exists — not overwriting your sorting",
|
||||
file=sys.stderr)
|
||||
continue
|
||||
os.makedirs(out_dir, exist_ok=True)
|
||||
|
||||
entries = []
|
||||
for p in sorted(glob.glob(f"pex/d{clip}_*.png")):
|
||||
frame = p.rsplit("_", 1)[-1].split(".")[0]
|
||||
img = cv2.imread(p)
|
||||
for i, d in enumerate(eng.detect(img)):
|
||||
x, y, w, h = d.bbox
|
||||
if min(w, h) < MIN_PX:
|
||||
continue
|
||||
lm = np.array(d.landmarks, dtype=np.float32).reshape(5, 2)
|
||||
crop = sae_embed.align_face(img, lm)
|
||||
if crop is None:
|
||||
continue
|
||||
emb = np.asarray(eng.embed_crop(crop), dtype=np.float32)
|
||||
|
||||
pad = int(0.5 * max(w, h))
|
||||
x0, y0 = max(0, int(x) - pad), max(0, int(y) - pad)
|
||||
x1, y1 = min(img.shape[1], int(x + w) + pad), min(img.shape[0], int(y + h) + pad)
|
||||
ctx = cv2.resize(img[y0:y1, x0:x1], (CTX, CTX))
|
||||
|
||||
entries.append({"clip": clip, "frame": frame, "idx": i,
|
||||
"bbox": [float(x), float(y), float(w), float(h)],
|
||||
"px": float(min(w, h)), "conf": float(d.confidence),
|
||||
"emb": emb, "ctx": ctx})
|
||||
|
||||
# cluster hint only — greedy, purely to group similar faces in the file list
|
||||
E = np.stack([e["emb"] for e in entries])
|
||||
hint = -np.ones(len(entries), int)
|
||||
k = 0
|
||||
for i in range(len(entries)):
|
||||
if hint[i] >= 0:
|
||||
continue
|
||||
hint[i] = k
|
||||
for j in range(i + 1, len(entries)):
|
||||
if hint[j] < 0 and float(E[i] @ E[j]) > 0.5:
|
||||
hint[j] = k
|
||||
k += 1
|
||||
|
||||
manifest = []
|
||||
for e, h in zip(entries, hint):
|
||||
name = f"c{h:02d}_{e['clip']}_f{e['frame']}_i{e['idx']}_{int(e['px'])}px.jpg"
|
||||
cv2.imwrite(f"{out_dir}/{name}", e["ctx"])
|
||||
manifest.append({k: v for k, v in e.items() if k not in ("emb", "ctx")}
|
||||
| {"file": name, "cluster_hint": int(h)})
|
||||
|
||||
json.dump(manifest, open(f"labelling/{clip}/manifest.json", "w"), indent=1)
|
||||
print(f"[{clip}] {len(manifest)} crops in {out_dir}, {k} cluster hints, "
|
||||
f"face px {min(m['px'] for m in manifest):.0f}–{max(m['px'] for m in manifest):.0f}",
|
||||
file=sys.stderr)
|
||||
@@ -1,174 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
"""What explains the misses? Head pose, face size, blur, detector confidence.
|
||||
|
||||
For every hand-labelled probe face, computes the calibrated probability against
|
||||
its OWN gallery entry — so a low value is a false negative, not a mistake about
|
||||
who it is — and pairs it with covariates that might explain the failure.
|
||||
|
||||
Head pose comes from solvePnP of the 5 landmarks against a canonical 3D face,
|
||||
giving yaw/pitch/roll in degrees.
|
||||
|
||||
CAVEAT, and it matters: the pose estimate is derived from the same 5
|
||||
landmarks the alignment uses. Where those landmarks are unreliable the pose
|
||||
estimate is unreliable too, and both degrade for the same reason. So this
|
||||
can show that failures concentrate at high yaw; it cannot cleanly separate
|
||||
"the head was turned" from "the landmarks were wrong because the head was
|
||||
turned". Those are the same physical cause, but not the same fix — the
|
||||
first argues for gallery pose coverage, the second for a better landmark
|
||||
source.
|
||||
|
||||
A sanity check is printed first: pose is estimated per person, and if it does
|
||||
not recover what is visible in the review sheets (one subject frontal, another
|
||||
in profile, another looking down) then the estimate is not worth reading.
|
||||
|
||||
Similarities go through the production gallery sigmoid, never compared raw.
|
||||
"""
|
||||
import sys, glob, json, os
|
||||
import numpy as np
|
||||
import cv2
|
||||
|
||||
sys.path.insert(0, "/home/dtourolle/Development/Jray-project/scene-actor-extraction/build-ort")
|
||||
import sae_embed
|
||||
|
||||
ROOT = "/home/dtourolle/Development/Jray-project/scene-actor-extraction/"
|
||||
M = ROOT + "models/"
|
||||
GALLERY_CLIP, PROBE_CLIP = "5157344", "5157339"
|
||||
PROB_THRESHOLD = 0.754
|
||||
|
||||
# Canonical 3D face, ordered as types.hpp:60 —
|
||||
# [0] right-eye [1] left-eye [2] nose [3] right-mouth [4] left-mouth.
|
||||
# The subject's right eye sits to the LEFT in image space, hence the negative X.
|
||||
FACE_3D = np.array([
|
||||
(-34.0, 35.0, -28.0),
|
||||
( 34.0, 35.0, -28.0),
|
||||
( 0.0, 0.0, 0.0),
|
||||
(-26.0, -32.0, -25.0),
|
||||
( 26.0, -32.0, -25.0),
|
||||
], dtype=np.float64)
|
||||
|
||||
eng = sae_embed.FaceEmbedder(detector_model=M + "scrfd_500m_bnkps.onnx",
|
||||
arcface_model=M + "LVFace-B_Glint360K.onnx",
|
||||
conf=0.5, nms=0.4, max_side=0)
|
||||
cal = sae_embed.gallery_calibration(ROOT + "gallery_lvface.h5")
|
||||
|
||||
|
||||
def head_pose(lm, w, h):
|
||||
"""yaw, pitch, roll in degrees. Focal length assumed = image width."""
|
||||
cam = np.array([[w, 0, w / 2], [0, w, h / 2], [0, 0, 1]], dtype=np.float64)
|
||||
ok, rvec, _ = cv2.solvePnP(FACE_3D, lm.astype(np.float64), cam, None,
|
||||
flags=cv2.SOLVEPNP_EPNP)
|
||||
if not ok:
|
||||
return None
|
||||
R, _ = cv2.Rodrigues(rvec)
|
||||
sy = np.sqrt(R[0, 0] ** 2 + R[1, 0] ** 2)
|
||||
if sy > 1e-6:
|
||||
pitch = np.degrees(np.arctan2(-R[2, 0], sy))
|
||||
yaw = np.degrees(np.arctan2(R[1, 0], R[0, 0]))
|
||||
roll = np.degrees(np.arctan2(R[2, 1], R[2, 2]))
|
||||
else:
|
||||
pitch = np.degrees(np.arctan2(-R[2, 0], sy)); yaw = 0.0
|
||||
roll = np.degrees(np.arctan2(-R[1, 2], R[1, 1]))
|
||||
# solvePnP's yaw wraps near +/-180 for a face pointing at the camera;
|
||||
# fold it to a "degrees away from frontal" magnitude.
|
||||
yaw = ((yaw + 180) % 360) - 180
|
||||
if abs(yaw) > 90:
|
||||
yaw = np.sign(yaw) * (180 - abs(yaw))
|
||||
return yaw, pitch, roll
|
||||
|
||||
|
||||
def collect(clip):
|
||||
lab = {os.path.basename(p): os.path.basename(os.path.dirname(p))
|
||||
for p in glob.glob(f"labelling/{clip}/*/*.jpg")
|
||||
if os.path.basename(os.path.dirname(p)) not in ("discard", "unsorted")}
|
||||
man = {m["file"]: m for m in json.load(open(f"labelling/{clip}/manifest.json"))}
|
||||
rows = []
|
||||
for frame in sorted({man[f]["frame"] for f in lab}):
|
||||
img = cv2.imread(f"frames/d{clip}_{frame}.png")
|
||||
dets = eng.detect(img)
|
||||
H, W = img.shape[:2]
|
||||
for f, person in lab.items():
|
||||
m = man[f]
|
||||
if m["frame"] != frame or m["idx"] >= len(dets):
|
||||
continue
|
||||
d = dets[m["idx"]]
|
||||
lm = np.array(d.landmarks, dtype=np.float32).reshape(5, 2)
|
||||
crop = sae_embed.align_face(img, lm)
|
||||
if crop is None:
|
||||
continue
|
||||
pose = head_pose(lm, W, H)
|
||||
x, y, w, h = d.bbox
|
||||
g = cv2.cvtColor(np.asarray(crop), cv2.COLOR_BGR2GRAY)
|
||||
rows.append({
|
||||
"person": person, "px": float(min(w, h)), "conf": float(d.confidence),
|
||||
"yaw": pose[0] if pose else np.nan, "pitch": pose[1] if pose else np.nan,
|
||||
"roll": pose[2] if pose else np.nan,
|
||||
"blur": float(cv2.Laplacian(g, cv2.CV_64F).var()),
|
||||
"emb": np.asarray(eng.embed_crop(crop), dtype=np.float32)})
|
||||
return rows
|
||||
|
||||
|
||||
gal_rows = collect(GALLERY_CLIP)
|
||||
prb_rows = collect(PROBE_CLIP)
|
||||
gal = {}
|
||||
for r in gal_rows:
|
||||
gal.setdefault(r["person"], []).append(r["emb"])
|
||||
gal = {p: np.stack(v) for p, v in gal.items()}
|
||||
|
||||
for r in prb_rows:
|
||||
if r["person"] in gal:
|
||||
s = float((gal[r["person"]] @ r["emb"]).max()) # best-of-N, own actor
|
||||
r["p"] = cal.probability(s)
|
||||
r["sim"] = s
|
||||
else:
|
||||
r["p"] = np.nan
|
||||
rows = [r for r in prb_rows if not np.isnan(r.get("p", np.nan))]
|
||||
print(f"[data] {len(rows)} labelled probe faces with a gallery entry\n", file=sys.stderr)
|
||||
|
||||
# ── sanity check: does the pose estimate recover what the sheets show? ───────
|
||||
print("pose by person (does this match the review sheets?)")
|
||||
print(f"{'person':>7}{'n':>5}{'|yaw| med':>11}{'pitch med':>11}{'P med':>8}{'hit rate':>10}")
|
||||
for p in sorted({r['person'] for r in rows}):
|
||||
sub = [r for r in rows if r["person"] == p]
|
||||
print(f"{p:>7}{len(sub):>5}"
|
||||
f"{np.median([abs(r['yaw']) for r in sub]):>11.1f}"
|
||||
f"{np.median([r['pitch'] for r in sub]):>11.1f}"
|
||||
f"{np.median([r['p'] for r in sub]):>8.3f}"
|
||||
f"{100*np.mean([r['p'] > PROB_THRESHOLD for r in sub]):>9.0f}%")
|
||||
|
||||
# ── P binned by each covariate ───────────────────────────────────────────────
|
||||
def binned(name, key, edges, fmt="{:.0f}"):
|
||||
print(f"\nP(match) by {name}")
|
||||
print(f"{'bin':>16}{'n':>5}{'P med':>9}{'hit rate':>10}{'sim med':>9}")
|
||||
vals = np.array([r[key] for r in rows])
|
||||
for lo, hi in zip(edges[:-1], edges[1:]):
|
||||
sub = [r for r, v in zip(rows, vals) if lo <= v < hi]
|
||||
if not sub:
|
||||
continue
|
||||
lbl = f"{fmt.format(lo)}–{fmt.format(hi)}"
|
||||
print(f"{lbl:>16}{len(sub):>5}"
|
||||
f"{np.median([r['p'] for r in sub]):>9.3f}"
|
||||
f"{100*np.mean([r['p'] > PROB_THRESHOLD for r in sub]):>9.0f}%"
|
||||
f"{np.median([r['sim'] for r in sub]):>9.3f}")
|
||||
|
||||
for r in rows:
|
||||
r["absyaw"] = abs(r["yaw"])
|
||||
r["abspitch"] = abs(r["pitch"])
|
||||
binned("|yaw| (deg from frontal)", "absyaw", [0, 10, 20, 30, 45, 60, 91])
|
||||
binned("|pitch| (deg)", "abspitch", [0, 10, 20, 30, 45, 91])
|
||||
binned("face size (px)", "px", [0, 130, 150, 175, 200, 400])
|
||||
binned("blur (laplacian var)", "blur", [0, 50, 150, 400, 1000, 1e9])
|
||||
binned("detector confidence", "conf", [0.5, 0.6, 0.7, 0.8, 0.9, 1.01], "{:.2f}")
|
||||
|
||||
# ── how much does each covariate actually explain? ───────────────────────────
|
||||
print("\nSpearman rank correlation with P(match):")
|
||||
def spearman(a, b):
|
||||
ra = np.argsort(np.argsort(a)); rb = np.argsort(np.argsort(b))
|
||||
return float(np.corrcoef(ra, rb)[0, 1])
|
||||
P = np.array([r["p"] for r in rows])
|
||||
for key, label in [("absyaw", "|yaw|"), ("abspitch", "|pitch|"), ("px", "face px"),
|
||||
("blur", "blur"), ("conf", "detector conf")]:
|
||||
v = np.array([r[key] for r in rows])
|
||||
print(f" {label:>14}: {spearman(v, P):+.3f}")
|
||||
|
||||
json.dump([{k: v for k, v in r.items() if k != "emb"} for r in rows],
|
||||
open("failure_analysis.json", "w"), indent=1, default=float)
|
||||
@@ -1,223 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Landmark voting: average SCRFD's overlapping detections instead of discarding them.
|
||||
|
||||
SCRFD predicts a face from many anchors; NMS keeps the single highest-scoring
|
||||
box and throws the rest away. Each discarded box carries its own 5-landmark
|
||||
estimate of the SAME face, so the survivors are one sample from a distribution
|
||||
we could be averaging over.
|
||||
|
||||
baseline conf 0.50, nms 0.40 — the shipped settings, one box per face
|
||||
voted conf 0.30, nms 0.90 — duplicates survive, then grouped by IoU and
|
||||
the 5 landmarks averaged, weighted by detection confidence
|
||||
|
||||
Why this is worth trying when the mesh failed: the mesh moved the landmarks off
|
||||
the definition ArcFace was trained on (a lip-ring centroid is not an annotated
|
||||
mouth corner), and the embedder punished it. A confidence-weighted mean of
|
||||
SCRFD's OWN landmark predictions is the same kind of point, just with less
|
||||
variance — it should stay on-distribution while being steadier.
|
||||
|
||||
Scored on cross-clip identification through the production sigmoid, which is
|
||||
the thing that actually broke. Raw similarity shown only to locate the
|
||||
threshold; it decides nothing.
|
||||
|
||||
LD_PRELOAD=/usr/lib/libcudnn_cnn.so.9 python3 landmark_voting.py
|
||||
"""
|
||||
import sys, glob, json, os
|
||||
|
||||
sys.path.insert(0, "/home/dtourolle/Development/Jray-project/scene-actor-extraction/build-ort")
|
||||
import sae_embed # before cv2 — see alignment_compare.py
|
||||
|
||||
import numpy as np
|
||||
import cv2
|
||||
|
||||
import argparse
|
||||
|
||||
ROOT = "/home/dtourolle/Development/Jray-project/scene-actor-extraction/"
|
||||
M = ROOT + "models/"
|
||||
CLIPS = ["5157344", "5157339"]
|
||||
PROB_THRESHOLD = 0.754
|
||||
GROUP_IOU = 0.55 # detections overlapping this much are the same face
|
||||
MATCH_IOU = 0.35 # tie a detection to the hand-labelled face
|
||||
|
||||
_ap = argparse.ArgumentParser()
|
||||
_ap.add_argument("--detector", default="scrfd_500m_bnkps.onnx",
|
||||
help="detector under models/. SCRFD sizes 500m / 2.5g / 10g come "
|
||||
"from InsightFace's buffalo_sc / buffalo_m / buffalo_l packs")
|
||||
_ap.add_argument("--vote-conf", type=float, default=None,
|
||||
help="confidence floor for the voting pass. Omit to auto-tune "
|
||||
"it to --target-votes")
|
||||
_ap.add_argument("--target-votes", type=int, default=3,
|
||||
help="votes per face to tune --vote-conf towards, so detectors "
|
||||
"are compared at equal redundancy rather than equal settings")
|
||||
_args = _ap.parse_args()
|
||||
|
||||
base_eng = sae_embed.FaceEmbedder(detector_model=M + _args.detector,
|
||||
arcface_model=M + "LVFace-B_Glint360K.onnx",
|
||||
conf=0.5, nms=0.4, max_side=0)
|
||||
|
||||
|
||||
def _make_vote_engine(conf):
|
||||
# Same models, looser suppression: keep the duplicates NMS would have removed.
|
||||
return sae_embed.FaceEmbedder(detector_model=M + _args.detector,
|
||||
arcface_model=M + "LVFace-B_Glint360K.onnx",
|
||||
conf=conf, nms=0.9, max_side=0)
|
||||
|
||||
|
||||
cal = sae_embed.gallery_calibration(ROOT + "gallery_lvface.h5")
|
||||
def iou(a, b):
|
||||
ax, ay, aw, ah = a; bx, by, bw, bh = b
|
||||
x0, y0 = max(ax, bx), max(ay, by)
|
||||
x1, y1 = min(ax + aw, bx + bw), min(ay + ah, by + bh)
|
||||
if x1 <= x0 or y1 <= y0:
|
||||
return 0.0
|
||||
i = (x1 - x0) * (y1 - y0)
|
||||
return i / (aw * ah + bw * bh - i)
|
||||
|
||||
|
||||
def vote(dets):
|
||||
"""Group overlapping detections, return (bbox, landmarks, conf, n_votes)."""
|
||||
items = sorted(dets, key=lambda d: -d.confidence)
|
||||
used, out = [False] * len(items), []
|
||||
for i, d in enumerate(items):
|
||||
if used[i]:
|
||||
continue
|
||||
grp = [d]
|
||||
used[i] = True
|
||||
for j in range(i + 1, len(items)):
|
||||
if not used[j] and iou(list(d.bbox), list(items[j].bbox)) >= GROUP_IOU:
|
||||
used[j] = True
|
||||
grp.append(items[j])
|
||||
w = np.array([g.confidence for g in grp], dtype=np.float32)
|
||||
w = w / w.sum()
|
||||
lms = np.stack([np.array(g.landmarks, dtype=np.float32).reshape(5, 2) for g in grp])
|
||||
bxs = np.stack([np.array(list(g.bbox), dtype=np.float32) for g in grp])
|
||||
out.append((( w[:, None] * bxs).sum(0), (w[:, None, None] * lms).sum(0),
|
||||
float(grp[0].confidence), len(grp)))
|
||||
return out
|
||||
|
||||
|
||||
def tune_vote_conf(target, sample=6):
|
||||
"""Pick the confidence floor giving ~target detections per face to average.
|
||||
|
||||
A larger SCRFD is more confident and suppresses harder, so at a fixed floor
|
||||
it emits fewer overlapping anchors — median 2 against 500m's 3. Comparing
|
||||
detectors at equal SETTINGS therefore also compares them at unequal
|
||||
redundancy, and the voting arm is handicapped for the bigger models. Tuning
|
||||
each to the same votes-per-face isolates landmark quality from how much
|
||||
there was to average.
|
||||
"""
|
||||
frames = sorted(glob.glob(f"frames/d{CLIPS[0]}_*.png"))[:sample]
|
||||
imgs = [cv2.imread(f) for f in frames]
|
||||
best = (None, None, 1e9)
|
||||
for conf in (0.30, 0.20, 0.12, 0.07, 0.04, 0.02, 0.01):
|
||||
eng = _make_vote_engine(conf)
|
||||
sizes = [n for img in imgs for _, _, _, n in vote(eng.detect(img))]
|
||||
if not sizes:
|
||||
continue
|
||||
med = float(np.median(sizes))
|
||||
if abs(med - target) < best[2]:
|
||||
best = (conf, eng, abs(med - target))
|
||||
print(f"[tune] conf={conf:.2f} -> median {med:.0f} votes/face", file=sys.stderr)
|
||||
if med >= target:
|
||||
break
|
||||
if best[1] is None:
|
||||
print(f"[tune] no confidence floor reached {target} votes/face; "
|
||||
f"falling back to 0.30", file=sys.stderr)
|
||||
return 0.30, _make_vote_engine(0.30)
|
||||
print(f"[tune] chose conf={best[0]:.2f} for ~{target} votes/face", file=sys.stderr)
|
||||
return best[0], best[1]
|
||||
|
||||
|
||||
if _args.vote_conf is not None:
|
||||
VOTE_CONF, vote_eng = _args.vote_conf, _make_vote_engine(_args.vote_conf)
|
||||
else:
|
||||
VOTE_CONF, vote_eng = tune_vote_conf(_args.target_votes)
|
||||
|
||||
cal = sae_embed.gallery_calibration(ROOT + "gallery_lvface.h5")
|
||||
|
||||
|
||||
def collect(clip):
|
||||
lab = {os.path.basename(p): os.path.basename(os.path.dirname(p))
|
||||
for p in glob.glob(f"labelling/{clip}/*/*.jpg")
|
||||
if os.path.basename(os.path.dirname(p)) not in ("discard", "unsorted")}
|
||||
man = {m["file"]: m for m in json.load(open(f"labelling/{clip}/manifest.json"))}
|
||||
rows, votes = [], []
|
||||
for frame in sorted({man[f]["frame"] for f in lab}):
|
||||
img = cv2.imread(f"frames/d{clip}_{frame}.png")
|
||||
base = base_eng.detect(img)
|
||||
voted = vote(vote_eng.detect(img))
|
||||
for fname, person in lab.items():
|
||||
m = man[fname]
|
||||
if m["frame"] != frame or m["idx"] >= len(base):
|
||||
continue
|
||||
d = base[m["idx"]]
|
||||
lm5 = np.array(d.landmarks, dtype=np.float32).reshape(5, 2)
|
||||
c_b = sae_embed.align_face(img, lm5)
|
||||
|
||||
# the voted group covering the same face
|
||||
best, best_v = None, 0.0
|
||||
for bbox, lms, conf, n in voted:
|
||||
v = iou(list(bbox), list(d.bbox))
|
||||
if v > best_v:
|
||||
best_v, best = v, (lms, n)
|
||||
c_v = None
|
||||
if best and best_v >= MATCH_IOU:
|
||||
c_v = sae_embed.align_face(img, best[0].astype(np.float32))
|
||||
votes.append(best[1])
|
||||
rec = {"person": person}
|
||||
rec["base"] = np.asarray(base_eng.embed_crop(c_b), np.float32) if c_b is not None else None
|
||||
rec["voted"] = np.asarray(base_eng.embed_crop(c_v), np.float32) if c_v is not None else None
|
||||
rows.append(rec)
|
||||
return rows, votes
|
||||
|
||||
|
||||
data, allv = {}, []
|
||||
for c in CLIPS:
|
||||
data[c], v = collect(c)
|
||||
allv += v
|
||||
print(f"[{c}] {len(data[c])} crops", file=sys.stderr)
|
||||
print(f"[voting] group size: median {np.median(allv):.0f}, "
|
||||
f"mean {np.mean(allv):.1f}, max {max(allv)} detections averaged per face",
|
||||
file=sys.stderr)
|
||||
|
||||
GAL, PRB = "5157344", "5157339"
|
||||
print(f"\ndetector={_args.detector} vote_conf={VOTE_CONF:.2f} "
|
||||
f"gallery {GAL} -> probe {PRB}, P>{PROB_THRESHOLD}\n")
|
||||
print(f"{'align':>8}{'person':>8}{'n_gal':>7}{'n_prb':>7}"
|
||||
f"{'within-clip':>13}{'cross-clip':>12}{'hit rate':>10}")
|
||||
summary = {}
|
||||
for key in ("base", "voted"):
|
||||
gal, prb = {}, {}
|
||||
for r in data[GAL]:
|
||||
if r[key] is not None:
|
||||
gal.setdefault(r["person"], []).append(r[key])
|
||||
for r in data[PRB]:
|
||||
if r[key] is not None:
|
||||
prb.setdefault(r["person"], []).append(r[key])
|
||||
gal = {p: np.stack(v) for p, v in gal.items()}
|
||||
prb = {p: np.stack(v) for p, v in prb.items()}
|
||||
hits = tot = 0
|
||||
for p in sorted(set(gal) & set(prb)):
|
||||
pp = prb[p] @ prb[p].T
|
||||
np.fill_diagonal(pp, -1)
|
||||
within = float(np.median(pp.max(axis=1))) if len(pp) > 1 else float("nan")
|
||||
cross = float(np.median((gal[p] @ prb[p].T).max(axis=0)))
|
||||
h = 0
|
||||
for e in prb[p]:
|
||||
bp, bn = 0.0, None
|
||||
for q in gal:
|
||||
v = cal.probability(float((gal[q] @ e).max()))
|
||||
if v > bp:
|
||||
bp, bn = v, q
|
||||
if bp > PROB_THRESHOLD and bn == p:
|
||||
h += 1
|
||||
hits += h; tot += len(prb[p])
|
||||
print(f"{key:>8}{p:>8}{len(gal[p]):>7}{len(prb[p]):>7}"
|
||||
f"{cal.probability(within):>6.3f}/{within:<6.3f}"
|
||||
f"{cal.probability(cross):>6.3f}/{cross:<5.3f}{100*h/len(prb[p]):>9.0f}%")
|
||||
summary[key] = (hits, tot)
|
||||
print(f"{key:>8}{'ALL':>8}{'':>14}{'':>25}{100*hits/max(tot,1):>9.0f}%\n")
|
||||
|
||||
hb, tb = summary["base"]; hv, tv = summary["voted"]
|
||||
print(f"voting vs baseline: {100*hv/max(tv,1) - 100*hb/max(tb,1):+.1f} points "
|
||||
f"of cross-clip TPI ({hb}/{tb} -> {hv}/{tv})")
|
||||
@@ -1,235 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Build labelling/review.html — a local page for correcting the labels.
|
||||
|
||||
One row per crop, ordered most-suspicious first:
|
||||
|
||||
left the person it is currently filed under (medoid of that person's
|
||||
hand-sorted crops, so the reference is one you trust)
|
||||
centre the crop under review — context with the detection boxed, and
|
||||
beneath it the 112x112 the embedder actually receives
|
||||
right the person it matches better, if any, with both probabilities
|
||||
|
||||
Pick a destination per row, then Export to download corrections.json and apply
|
||||
it with apply_corrections.py. Nothing is moved by this script.
|
||||
|
||||
Self-contained: images are inlined as data URIs and the page is opened from
|
||||
disk, so no server runs and no face crop leaves the machine.
|
||||
|
||||
Ordering is by P(other) - P(self), both from the global gallery sigmoid, so
|
||||
rows where the evidence disagrees with the label float to the top and the
|
||||
agreement cases sink. It is a review order, not a verdict — you are the
|
||||
arbiter, which is the whole point of labelling by hand.
|
||||
"""
|
||||
import sys, glob, json, os, base64
|
||||
import numpy as np
|
||||
import cv2
|
||||
|
||||
sys.path.insert(0, "/home/dtourolle/Development/Jray-project/scene-actor-extraction/build-ort")
|
||||
import sae_embed
|
||||
|
||||
ROOT = "/home/dtourolle/Development/Jray-project/scene-actor-extraction/"
|
||||
M = ROOT + "models/"
|
||||
EMBEDDER = M + "LVFace-B_Glint360K.onnx"
|
||||
GALLERY = ROOT + "gallery_lvface.h5"
|
||||
REF_CLIP = "5157344" # the clip sorted by hand — reference faces come from here
|
||||
CLIPS = ["5157344", "5157339"]
|
||||
|
||||
eng = sae_embed.FaceEmbedder(detector_model=M + "scrfd_500m_bnkps.onnx",
|
||||
arcface_model=EMBEDDER, conf=0.5, nms=0.4, max_side=0)
|
||||
cal = sae_embed.gallery_calibration(GALLERY)
|
||||
|
||||
|
||||
def b64(img, size, q=72):
|
||||
img = cv2.resize(img, (size, size))
|
||||
ok, buf = cv2.imencode(".jpg", img, [cv2.IMWRITE_JPEG_QUALITY, q])
|
||||
return "data:image/jpeg;base64," + base64.b64encode(buf).decode() if ok else ""
|
||||
|
||||
|
||||
rows = []
|
||||
for clip in CLIPS:
|
||||
man = {m["file"]: m for m in json.load(open(f"labelling/{clip}/manifest.json"))}
|
||||
placed = {os.path.basename(p): (os.path.basename(os.path.dirname(p)), p)
|
||||
for p in glob.glob(f"labelling/{clip}/*/*.jpg")}
|
||||
by_frame = {}
|
||||
for fname, (person, path) in placed.items():
|
||||
if fname in man and person != "unsorted":
|
||||
by_frame.setdefault(man[fname]["frame"], []).append((fname, person, path))
|
||||
for frame, items in sorted(by_frame.items()):
|
||||
img = cv2.imread(f"frames/d{clip}_{frame}.png")
|
||||
if img is None:
|
||||
continue
|
||||
dets = eng.detect(img)
|
||||
for fname, person, path in items:
|
||||
i = man[fname]["idx"]
|
||||
if i >= len(dets):
|
||||
continue
|
||||
lm = np.array(dets[i].landmarks, dtype=np.float32).reshape(5, 2)
|
||||
crop = sae_embed.align_face(img, lm)
|
||||
if crop is None:
|
||||
continue
|
||||
rows.append({"clip": clip, "person": person, "file": fname, "path": path,
|
||||
"px": man[fname]["px"], "aligned": np.asarray(crop),
|
||||
"emb": np.asarray(eng.embed_crop(crop), dtype=np.float32)})
|
||||
|
||||
people = sorted({r["person"] for r in rows})
|
||||
E = np.stack([r["emb"] for r in rows])
|
||||
lab = np.array([people.index(r["person"]) for r in rows])
|
||||
S = E @ E.T
|
||||
np.fill_diagonal(S, -1.0)
|
||||
|
||||
# reference face per person: medoid of their REF_CLIP crops
|
||||
ref_img = {}
|
||||
for k, p in enumerate(people):
|
||||
idx = [i for i in np.where(lab == k)[0] if rows[i]["clip"] == REF_CLIP]
|
||||
if not idx:
|
||||
idx = list(np.where(lab == k)[0])
|
||||
if not idx:
|
||||
continue
|
||||
sub = S[np.ix_(idx, idx)].copy()
|
||||
medoid = idx[int(np.argmax(sub.mean(axis=1)))]
|
||||
ref_img[p] = b64(rows[medoid]["aligned"], 112)
|
||||
|
||||
items = []
|
||||
for i, r in enumerate(rows):
|
||||
k = lab[i]
|
||||
same = [j for j in np.where(lab == k)[0] if j != i]
|
||||
p_self = cal.probability(float(S[i, same].max())) if same else 0.0
|
||||
best_other, p_other = None, 0.0
|
||||
for k2, p2 in enumerate(people):
|
||||
if k2 == k:
|
||||
continue
|
||||
other = np.where(lab == k2)[0]
|
||||
if not len(other):
|
||||
continue
|
||||
pv = cal.probability(float(S[i, other].max()))
|
||||
if pv > p_other:
|
||||
p_other, best_other = pv, p2
|
||||
ctx = cv2.imread(r["path"])
|
||||
items.append({
|
||||
"file": r["file"], "clip": r["clip"], "person": r["person"],
|
||||
"px": int(r["px"]), "p_self": round(p_self, 3), "p_other": round(p_other, 3),
|
||||
"other": best_other, "delta": round(p_other - p_self, 3),
|
||||
"ctx": b64(ctx, 150) if ctx is not None else "",
|
||||
"ali": b64(r["aligned"], 112),
|
||||
})
|
||||
items.sort(key=lambda x: -x["delta"])
|
||||
|
||||
payload = json.dumps({"people": people, "refs": ref_img, "items": items})
|
||||
|
||||
HTML = """<meta charset="utf-8"><title>JRay — label review</title>
|
||||
<style>
|
||||
:root{color-scheme:dark;--bg:#14161a;--fg:#e6e8ea;--mut:#8b929c;--line:#262b33;--warn:#e0654a;--ok:#4a9d6a}
|
||||
body{margin:0;background:var(--bg);color:var(--fg);font:14px/1.5 system-ui,sans-serif}
|
||||
header{position:sticky;top:0;background:#181b20;border-bottom:1px solid var(--line);
|
||||
padding:12px 18px;display:flex;gap:18px;align-items:center;flex-wrap:wrap;z-index:5}
|
||||
h1{font-size:15px;margin:0;font-weight:600}
|
||||
.stat{color:var(--mut);font-size:13px}
|
||||
button{background:#232830;color:var(--fg);border:1px solid var(--line);border-radius:6px;
|
||||
padding:7px 13px;cursor:pointer;font:inherit}
|
||||
button:hover{background:#2c323c}
|
||||
button.go{background:#2f5d43;border-color:#3c7555}
|
||||
.row{display:grid;grid-template-columns:150px 1fr 190px;gap:20px;align-items:center;
|
||||
padding:14px 18px;border-bottom:1px solid var(--line)}
|
||||
.row.flag{background:#1e1719}
|
||||
.row.done{opacity:.4}
|
||||
.cell{display:flex;gap:10px;align-items:center}
|
||||
img{border-radius:5px;display:block;background:#000}
|
||||
.lab{font-weight:600;font-size:15px}
|
||||
.mut{color:var(--mut);font-size:12px}
|
||||
.p{font-variant-numeric:tabular-nums}
|
||||
.hi{color:var(--warn);font-weight:600}
|
||||
.choices{display:flex;flex-wrap:wrap;gap:6px}
|
||||
.choices button{padding:5px 10px;font-size:13px}
|
||||
.choices button.sel{background:#2f5d43;border-color:#3c7555}
|
||||
.legend{padding:10px 18px;color:var(--mut);font-size:12px;border-bottom:1px solid var(--line)}
|
||||
</style>
|
||||
<header>
|
||||
<h1>Label review</h1>
|
||||
<span class="stat" id="stat"></span>
|
||||
<button id="exp" class="go">Export corrections.json</button>
|
||||
<button id="onlyflag">Show only disagreements</button>
|
||||
</header>
|
||||
<div class="legend">Left: the person this crop is filed under. Centre: the crop (context with the
|
||||
detection boxed, and the 112×112 the embedder actually sees). Right: the person it matches
|
||||
better, if any. Ordered by P(other) − P(self) — disagreements first.</div>
|
||||
<div id="list"></div>
|
||||
<script>
|
||||
const D = __PAYLOAD__;
|
||||
const choice = {};
|
||||
const list = document.getElementById('list');
|
||||
|
||||
function render(){
|
||||
list.innerHTML = '';
|
||||
const flagOnly = document.body.dataset.flag === '1';
|
||||
for (const it of D.items){
|
||||
if (flagOnly && it.delta <= 0) continue;
|
||||
const row = document.createElement('div');
|
||||
row.className = 'row' + (it.delta > 0 ? ' flag' : '') + (choice[it.file] ? ' done' : '');
|
||||
|
||||
const left = document.createElement('div');
|
||||
left.className = 'cell';
|
||||
left.innerHTML = `<img src="${D.refs[it.person]||''}" width="72" height="72">
|
||||
<div><div class="lab">${it.person}</div>
|
||||
<div class="mut p">P(self) ${it.p_self.toFixed(3)}</div></div>`;
|
||||
|
||||
const mid = document.createElement('div');
|
||||
mid.className = 'cell';
|
||||
mid.innerHTML = `<img src="${it.ctx}" width="120" height="120">
|
||||
<img src="${it.ali}" width="90" height="90">
|
||||
<div><div class="mut">${it.clip} · ${it.px}px</div>
|
||||
<div class="mut">${it.file}</div></div>`;
|
||||
|
||||
const right = document.createElement('div');
|
||||
const worse = it.delta > 0;
|
||||
right.innerHTML = it.other
|
||||
? `<div class="cell"><img src="${D.refs[it.other]||''}" width="56" height="56">
|
||||
<div><div class="lab ${worse?'hi':''}">${it.other}</div>
|
||||
<div class="mut p ${worse?'hi':''}">P ${it.p_other.toFixed(3)}</div></div></div>`
|
||||
: '<div class="mut">—</div>';
|
||||
|
||||
const ch = document.createElement('div');
|
||||
ch.className = 'choices';
|
||||
for (const p of D.people.concat(['discard'])){
|
||||
const b = document.createElement('button');
|
||||
b.textContent = p === it.person ? p + ' (keep)' : p;
|
||||
if (choice[it.file] === p || (!choice[it.file] && p === it.person)) b.classList.add('sel');
|
||||
b.onclick = () => { choice[it.file] = p; render(); };
|
||||
ch.appendChild(b);
|
||||
}
|
||||
right.appendChild(ch);
|
||||
|
||||
row.append(left, mid, right);
|
||||
list.appendChild(row);
|
||||
}
|
||||
const changed = Object.entries(choice).filter(([f,p]) =>
|
||||
p !== (D.items.find(i=>i.file===f)||{}).person).length;
|
||||
document.getElementById('stat').textContent =
|
||||
`${D.items.length} crops · ${D.items.filter(i=>i.delta>0).length} disagreements · ${changed} changes staged`;
|
||||
}
|
||||
|
||||
document.getElementById('onlyflag').onclick = () => {
|
||||
document.body.dataset.flag = document.body.dataset.flag === '1' ? '0' : '1';
|
||||
render();
|
||||
};
|
||||
document.getElementById('exp').onclick = () => {
|
||||
const out = {};
|
||||
for (const it of D.items){
|
||||
const p = choice[it.file] || it.person;
|
||||
if (p !== it.person) out[it.file] = {from: it.person, to: p, clip: it.clip};
|
||||
}
|
||||
const blob = new Blob([JSON.stringify(out, null, 1)], {type:'application/json'});
|
||||
const a = document.createElement('a');
|
||||
a.href = URL.createObjectURL(blob); a.download = 'corrections.json'; a.click();
|
||||
};
|
||||
render();
|
||||
</script>
|
||||
"""
|
||||
|
||||
os.makedirs("labelling", exist_ok=True)
|
||||
out = "labelling/review.html"
|
||||
with open(out, "w") as f:
|
||||
f.write(HTML.replace("__PAYLOAD__", payload))
|
||||
size = os.path.getsize(out) / 1e6
|
||||
flagged = sum(1 for i in items if i["delta"] > 0)
|
||||
print(f"{out} {size:.1f} MB {len(items)} crops, {flagged} disagreements", file=sys.stderr)
|
||||
print(f"open file://{os.path.abspath(out)}", file=sys.stderr)
|
||||
@@ -1,200 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Estimate head pose per crop, and build a page to confirm or correct it.
|
||||
|
||||
Why not solvePnP on the 5 detector landmarks: those landmarks collapse on
|
||||
turned faces, so the estimator breaks precisely on the crops whose pose we care
|
||||
about. Run that way it reported the profile subject as the MOST frontal of the
|
||||
four, which is how we know not to trust it.
|
||||
|
||||
Instead the estimate comes from the MediaPipe face mesh (468 points, run via
|
||||
OpenCV DNN — the same model rPPG-kahn uses) and a symmetry measure that needs
|
||||
no 3D model:
|
||||
|
||||
yaw_ratio = (dL - dR) / (dL + dR)
|
||||
|
||||
over left/right symmetric vertex pairs, where dL and dR are each side's
|
||||
distance from the face midline. Frontal ~ 0, profile -> +/-1. It degrades
|
||||
gracefully because it averages many pairs rather than trusting any one point,
|
||||
and it is scale- and translation-free.
|
||||
|
||||
It is still an estimate. So this writes pose_review.html with the estimate
|
||||
PRE-FILLED as a proposal, ordered by confidence, for you to correct — and the
|
||||
correlation is only run against your corrected labels. If the estimate turns
|
||||
out to disagree with you often, that is the finding, and the automatic number
|
||||
gets dropped rather than reported.
|
||||
|
||||
Bins are coarse on purpose: frontal / three-quarter / profile / down-or-hidden.
|
||||
Finer than that and the labelling is slower and less reliable, and the question
|
||||
("does pose explain the misses") does not need degrees.
|
||||
"""
|
||||
import sys, glob, json, os, base64
|
||||
|
||||
# sae_embed MUST be imported before cv2: OpenCV's DNN module loads the system
|
||||
# libonnxruntime, which then shadows the newer one this module links against and
|
||||
# the import fails on a missing symbol version. Order matters, so do not tidy
|
||||
# these into alphabetical order.
|
||||
sys.path.insert(0, "/home/dtourolle/Development/Jray-project/scene-actor-extraction/build-ort")
|
||||
import sae_embed
|
||||
|
||||
import numpy as np
|
||||
import cv2
|
||||
|
||||
ROOT = "/home/dtourolle/Development/Jray-project/scene-actor-extraction/"
|
||||
M = ROOT + "models/"
|
||||
MESH = "/home/dtourolle/Development/rPPG-kahn/models/face_landmark.tflite"
|
||||
CLIPS = ["5157344", "5157339"]
|
||||
BINS = ["frontal", "three-quarter", "profile", "down-or-hidden"]
|
||||
|
||||
# Symmetric vertex pairs (subject-left, subject-right) on the MediaPipe mesh:
|
||||
# outer eye corners, inner eye corners, cheeks, mouth corners, jaw.
|
||||
PAIRS = [(33, 263), (133, 362), (130, 359), (243, 463),
|
||||
(61, 291), (91, 321), (146, 375), (58, 288), (172, 397), (215, 435)]
|
||||
MIDLINE = [10, 168, 1, 4, 5, 195, 197, 152] # forehead -> nose -> chin
|
||||
|
||||
net = cv2.dnn.readNetFromTFLite(MESH)
|
||||
NAMES = net.getUnconnectedOutLayersNames()
|
||||
LMI, PRI = NAMES.index("conv2d_21"), NAMES.index("conv2d_31")
|
||||
|
||||
eng = sae_embed.FaceEmbedder(detector_model=M + "scrfd_500m_bnkps.onnx",
|
||||
arcface_model=M + "LVFace-B_Glint360K.onnx",
|
||||
conf=0.5, nms=0.4, max_side=0)
|
||||
|
||||
|
||||
def mesh_pose(img, bbox, expand=1.6):
|
||||
"""(yaw_ratio, presence) or (nan, 0). yaw_ratio in [-1, 1], 0 = frontal."""
|
||||
x, y, w, h = bbox
|
||||
cx, cy, s = x + w / 2, y + h / 2, max(w, h) * expand
|
||||
crop = cv2.getRectSubPix(img, (int(s), int(s)), (float(cx), float(cy)))
|
||||
net.setInput(cv2.dnn.blobFromImage(crop, 1 / 255.0, (192, 192), (0, 0, 0), swapRB=True))
|
||||
o = net.forward(NAMES)
|
||||
pres = 1 / (1 + np.exp(-float(o[PRI].ravel()[0])))
|
||||
lm = o[LMI].reshape(468, 3)[:, :2]
|
||||
mid = lm[MIDLINE]
|
||||
# least-squares midline direction, then signed distance of each pair member
|
||||
c = mid.mean(axis=0)
|
||||
u, _, _ = np.linalg.svd(mid - c)
|
||||
d = (mid - c)
|
||||
axis = np.linalg.svd(d.T @ d)[0][:, 0] # principal direction of the midline
|
||||
normal = np.array([-axis[1], axis[0]])
|
||||
ratios = []
|
||||
for a, b in PAIRS:
|
||||
dl = float(np.dot(lm[a] - c, normal))
|
||||
dr = float(np.dot(lm[b] - c, normal))
|
||||
if abs(dl) + abs(dr) < 1e-6:
|
||||
continue
|
||||
ratios.append((abs(dl) - abs(dr)) / (abs(dl) + abs(dr)))
|
||||
return (float(np.median(ratios)) if ratios else np.nan), pres
|
||||
|
||||
|
||||
def b64(img, size, q=72):
|
||||
ok, buf = cv2.imencode(".jpg", cv2.resize(img, (size, size)),
|
||||
[cv2.IMWRITE_JPEG_QUALITY, q])
|
||||
return "data:image/jpeg;base64," + base64.b64encode(buf).decode() if ok else ""
|
||||
|
||||
|
||||
items = []
|
||||
for clip in CLIPS:
|
||||
lab = {os.path.basename(p): (os.path.basename(os.path.dirname(p)), p)
|
||||
for p in glob.glob(f"labelling/{clip}/*/*.jpg")
|
||||
if os.path.basename(os.path.dirname(p)) not in ("discard", "unsorted")}
|
||||
man = {m["file"]: m for m in json.load(open(f"labelling/{clip}/manifest.json"))}
|
||||
for frame in sorted({man[f]["frame"] for f in lab}):
|
||||
img = cv2.imread(f"frames/d{clip}_{frame}.png")
|
||||
dets = eng.detect(img)
|
||||
for fname, (person, path) in lab.items():
|
||||
m = man[fname]
|
||||
if m["frame"] != frame or m["idx"] >= len(dets):
|
||||
continue
|
||||
d = dets[m["idx"]]
|
||||
lm5 = np.array(d.landmarks, dtype=np.float32).reshape(5, 2)
|
||||
crop = sae_embed.align_face(img, lm5)
|
||||
if crop is None:
|
||||
continue
|
||||
yaw, pres = mesh_pose(img, d.bbox)
|
||||
a = abs(yaw) if not np.isnan(yaw) else 1.0
|
||||
guess = ("frontal" if a < 0.15 else "three-quarter" if a < 0.45
|
||||
else "profile")
|
||||
if pres < 0.5:
|
||||
guess = "down-or-hidden" # mesh could not fit at all
|
||||
ctx = cv2.imread(path)
|
||||
items.append({"file": fname, "clip": clip, "person": person,
|
||||
"px": int(m["px"]), "yaw": None if np.isnan(yaw) else round(yaw, 3),
|
||||
"pres": round(pres, 3), "guess": guess,
|
||||
"ctx": b64(ctx, 140) if ctx is not None else "",
|
||||
"ali": b64(np.asarray(crop), 112)})
|
||||
|
||||
# least-confident first: near a bin boundary, or the mesh could not fit
|
||||
def uncertainty(it):
|
||||
if it["pres"] < 0.5:
|
||||
return 0.0
|
||||
a = abs(it["yaw"]) if it["yaw"] is not None else 1.0
|
||||
return min(abs(a - 0.15), abs(a - 0.45))
|
||||
items.sort(key=uncertainty)
|
||||
|
||||
payload = json.dumps({"bins": BINS, "items": items})
|
||||
|
||||
HTML = """<meta charset="utf-8"><title>JRay — head pose labelling</title>
|
||||
<style>
|
||||
:root{color-scheme:dark}
|
||||
body{margin:0;background:#14161a;color:#e6e8ea;font:14px/1.5 system-ui,sans-serif}
|
||||
header{position:sticky;top:0;background:#181b20;border-bottom:1px solid #262b33;
|
||||
padding:12px 18px;display:flex;gap:16px;align-items:center;flex-wrap:wrap;z-index:5}
|
||||
h1{font-size:15px;margin:0}
|
||||
button{background:#232830;color:#e6e8ea;border:1px solid #262b33;border-radius:6px;
|
||||
padding:7px 12px;cursor:pointer;font:inherit}
|
||||
button:hover{background:#2c323c}
|
||||
button.go{background:#2f5d43;border-color:#3c7555}
|
||||
.g{display:grid;grid-template-columns:repeat(auto-fill,minmax(300px,1fr));gap:10px;padding:14px}
|
||||
.c{border:1px solid #262b33;border-radius:8px;padding:9px;display:flex;gap:9px;align-items:center}
|
||||
.c.edited{border-color:#3c7555}
|
||||
img{border-radius:5px;background:#000;display:block}
|
||||
.m{color:#8b929c;font-size:11px}
|
||||
.b{display:flex;flex-wrap:wrap;gap:4px;margin-top:5px}
|
||||
.b button{padding:3px 7px;font-size:11px}
|
||||
.b button.sel{background:#2f5d43;border-color:#3c7555}
|
||||
</style>
|
||||
<header><h1>Head pose</h1><span class="m" id="stat"></span>
|
||||
<button class="go" id="exp">Export pose_labels.json</button></header>
|
||||
<div class="g" id="g"></div>
|
||||
<script>
|
||||
const D=__PAYLOAD__; const pick={};
|
||||
function render(){
|
||||
const g=document.getElementById('g'); g.innerHTML='';
|
||||
for(const it of D.items){
|
||||
const cur=pick[it.file]||it.guess;
|
||||
const c=document.createElement('div');
|
||||
c.className='c'+(pick[it.file]&&pick[it.file]!==it.guess?' edited':'');
|
||||
const b=D.bins.map(x=>`<button class="${x===cur?'sel':''}" data-f="${it.file}" data-b="${x}">${x}</button>`).join('');
|
||||
c.innerHTML=`<img src="${it.ctx}" width="88" height="88"><img src="${it.ali}" width="66" height="66">
|
||||
<div><div class="m">${it.person} · ${it.clip.slice(-3)} · ${it.px}px</div>
|
||||
<div class="m">yaw ${it.yaw===null?'—':it.yaw} · presence ${it.pres}</div>
|
||||
<div class="b">${b}</div></div>`;
|
||||
g.appendChild(c);
|
||||
}
|
||||
g.onclick=e=>{const t=e.target; if(t.dataset&&t.dataset.b){pick[t.dataset.f]=t.dataset.b; render();}};
|
||||
const ed=Object.entries(pick).filter(([f,v])=>v!==(D.items.find(i=>i.file===f)||{}).guess).length;
|
||||
document.getElementById('stat').textContent=`${D.items.length} crops · ${ed} corrections`;
|
||||
}
|
||||
document.getElementById('exp').onclick=()=>{
|
||||
const out={}; for(const it of D.items) out[it.file]={pose:pick[it.file]||it.guess,
|
||||
guess:it.guess, yaw:it.yaw, pres:it.pres, person:it.person, clip:it.clip};
|
||||
const a=document.createElement('a');
|
||||
a.href=URL.createObjectURL(new Blob([JSON.stringify(out,null,1)],{type:'application/json'}));
|
||||
a.download='pose_labels.json'; a.click();
|
||||
};
|
||||
render();
|
||||
</script>
|
||||
"""
|
||||
out = "labelling/pose_review.html"
|
||||
open(out, "w").write(HTML.replace("__PAYLOAD__", payload))
|
||||
from collections import Counter
|
||||
print(f"{out} {os.path.getsize(out)/1e6:.1f} MB {len(items)} crops", file=sys.stderr)
|
||||
print(f"estimate: {dict(Counter(i['guess'] for i in items))}", file=sys.stderr)
|
||||
print("\nestimated pose per person (does this match what you see?):", file=sys.stderr)
|
||||
for p in sorted({i["person"] for i in items}):
|
||||
for clip in CLIPS:
|
||||
sub = [i for i in items if i["person"] == p and i["clip"] == clip]
|
||||
if sub:
|
||||
print(f" {p} {clip[-3:]}: {dict(Counter(i['guess'] for i in sub))}",
|
||||
file=sys.stderr)
|
||||
print(f"\nopen file://{os.path.abspath(out)}", file=sys.stderr)
|
||||
@@ -1,242 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Propose person labels for one clip using another clip's hand-sorted labels.
|
||||
|
||||
Reads the clip you have already sorted (REF_CLIP) as ground truth, then proposes
|
||||
a person for every crop in the other clip (TARGET_CLIP) and writes them into
|
||||
matching folders for you to correct.
|
||||
|
||||
python3 propose_labels.py # propose, write folders + sheets
|
||||
python3 propose_labels.py --dry-run # report only, move nothing
|
||||
|
||||
Output:
|
||||
labelling/<target>/unsorted/A|B|C|D/ proposed, same names as the ref clip
|
||||
labelling/<target>/unsorted/ left in place when no person is
|
||||
confident enough to name
|
||||
labelling/review_<person>.jpg contact sheet spanning BOTH clips:
|
||||
confirmed crops first, then
|
||||
proposed ones with their P
|
||||
|
||||
Correcting it: open a review sheet. Every face on it should be one person. The
|
||||
lower block is the proposal — move any intruder to the right folder, or back to
|
||||
unsorted/. The folder a file sits in is the ground truth; nothing downstream
|
||||
reads the proposed name or its probability.
|
||||
|
||||
The proposal is a labelling aid, never the label. Scoring the sweep against
|
||||
embedding-derived labels would be circular: it keeps the faces the embedder
|
||||
already gets right and drops the hard ones the sweep exists to find. Your
|
||||
correction is what breaks that loop, which is why the proposal is deliberately
|
||||
conservative and leaves anything doubtful unnamed.
|
||||
|
||||
Assignment is on the calibrated probability, per-actor best-of-N, exactly as
|
||||
identity_matcher_node does — never a bare cosine (AR-024). The calibration is
|
||||
fitted on your labelled reference crops, which is what calibrate_gallery is for.
|
||||
"""
|
||||
import sys, glob, json, os, shutil
|
||||
import numpy as np
|
||||
import cv2
|
||||
|
||||
sys.path.insert(0, "/home/dtourolle/Development/Jray-project/scene-actor-extraction/build-ort")
|
||||
import sae_embed
|
||||
|
||||
ROOT = "/home/dtourolle/Development/Jray-project/scene-actor-extraction/"
|
||||
M = ROOT + "models/"
|
||||
# The embedder and the gallery whose calibration scores it MUST be the same
|
||||
# model: a Platt fit is specific to one embedding space, so LVFace probabilities
|
||||
# read through an ArcFace fit are meaningless.
|
||||
EMBEDDER = M + "LVFace-B_Glint360K.onnx"
|
||||
GALLERY = ROOT + "gallery_lvface.h5" # 291 actors, cached fit
|
||||
REF_CLIP, TARGET_CLIP = "5157344", "5157339"
|
||||
ASSIGN_P = 0.90 # propose a name only when this confident
|
||||
SHEET_COLS = 8
|
||||
THUMB = 150
|
||||
DRY = "--dry-run" in sys.argv
|
||||
|
||||
eng = sae_embed.FaceEmbedder(detector_model=M + "scrfd_500m_bnkps.onnx",
|
||||
arcface_model=EMBEDDER,
|
||||
conf=0.5, nms=0.4, max_side=0)
|
||||
|
||||
|
||||
def embed_manifest(clip):
|
||||
"""Re-derive each dumped crop's embedding from its source frame, cached.
|
||||
|
||||
The dumped .jpg is a context thumbnail for human eyes; the embedding must
|
||||
come from the aligned crop the pipeline would actually produce, so the
|
||||
frame is re-detected and the manifest's idx picks the same face.
|
||||
|
||||
Detecting 24 4K frames per clip costs far more than the rest of this script
|
||||
put together, and the result only changes when the manifest does — so it is
|
||||
cached and keyed on the manifest's mtime. Delete cache/ to force a redo.
|
||||
"""
|
||||
man_path = f"labelling/{clip}/manifest.json"
|
||||
cache_path = f"cache/emb_{clip}.npz"
|
||||
os.makedirs("cache", exist_ok=True)
|
||||
if os.path.exists(cache_path) and \
|
||||
os.path.getmtime(cache_path) >= os.path.getmtime(man_path):
|
||||
z = np.load(cache_path, allow_pickle=True)
|
||||
print(f"[cache] {clip}: {len(z['meta'])} embeddings reused", file=sys.stderr)
|
||||
return [{**m, "emb": e} for m, e in zip(z["meta"], z["emb"])]
|
||||
|
||||
man = json.load(open(man_path))
|
||||
by_frame = {}
|
||||
for m in man:
|
||||
by_frame.setdefault(m["frame"], []).append(m)
|
||||
out = []
|
||||
for frame, ms in sorted(by_frame.items()):
|
||||
img = cv2.imread(f"frames/d{clip}_{frame}.png")
|
||||
if img is None:
|
||||
sys.exit(f"missing frames/d{clip}_{frame}.png — extract with\n"
|
||||
f" ffmpeg -i clips/{clip}.mp4 -vf fps=2 -frames:v 24 "
|
||||
f"frames/d{clip}_%03d.png")
|
||||
dets = eng.detect(img)
|
||||
for m in ms:
|
||||
if m["idx"] >= len(dets):
|
||||
continue
|
||||
d = dets[m["idx"]]
|
||||
lm = np.array(d.landmarks, dtype=np.float32).reshape(5, 2)
|
||||
crop = sae_embed.align_face(img, lm)
|
||||
if crop is None:
|
||||
continue
|
||||
out.append({**m, "emb": np.asarray(eng.embed_crop(crop), dtype=np.float32)})
|
||||
|
||||
np.savez(cache_path,
|
||||
meta=np.array([{k: v for k, v in o.items() if k != "emb"} for o in out],
|
||||
dtype=object),
|
||||
emb=np.stack([o["emb"] for o in out]))
|
||||
print(f"[cache] {clip}: {len(out)} embeddings written to {cache_path}",
|
||||
file=sys.stderr)
|
||||
return out
|
||||
|
||||
|
||||
def sorted_dirs(clip):
|
||||
"""Person folders you created, wherever you put them under labelling/<clip>."""
|
||||
found = {}
|
||||
for path in glob.glob(f"labelling/{clip}/**/", recursive=True):
|
||||
name = os.path.basename(path.rstrip("/"))
|
||||
if name in ("unsorted", "discard") or name.startswith("5157"):
|
||||
continue
|
||||
files = [os.path.basename(f) for f in glob.glob(path + "*.jpg")]
|
||||
if files:
|
||||
found[name] = files
|
||||
return found
|
||||
|
||||
|
||||
# ── reference side: your labels ──────────────────────────────────────────────
|
||||
ref_rows = embed_manifest(REF_CLIP)
|
||||
ref_dirs = sorted_dirs(REF_CLIP)
|
||||
if not ref_dirs:
|
||||
sys.exit(f"no person folders under labelling/{REF_CLIP} — sort that clip first")
|
||||
file_to_person = {f: p for p, fs in ref_dirs.items() for f in fs}
|
||||
|
||||
ref = [(file_to_person[r["file"]], r["emb"]) for r in ref_rows
|
||||
if r["file"] in file_to_person]
|
||||
people = sorted({p for p, _ in ref})
|
||||
print(f"[ref] {REF_CLIP}: {len(ref)} labelled crops over {len(people)} people "
|
||||
f"{ {p: sum(1 for q, _ in ref if q == p) for p in people} }", file=sys.stderr)
|
||||
|
||||
R = np.stack([e for _, e in ref])
|
||||
r_actor = [people.index(p) for p, _ in ref]
|
||||
|
||||
# The global gallery's sigmoid — NOT a fit over these four people. A Platt fit
|
||||
# over a handful of identities saturates: it will hand back P=0.99 for faces it
|
||||
# has no basis to separate, which is exactly how a wrong label acquires a
|
||||
# convincing probability. The production fit spans the whole actor population,
|
||||
# so a probability means the same thing here as it does in the matcher.
|
||||
cal = sae_embed.gallery_calibration(GALLERY)
|
||||
print(f"[calibration] global: {cal} assign boundary = sim "
|
||||
f"{cal.boundary_at(ASSIGN_P):.4f}", file=sys.stderr)
|
||||
|
||||
# ── target side: propose ─────────────────────────────────────────────────────
|
||||
tgt_rows = embed_manifest(TARGET_CLIP)
|
||||
T = np.stack([t["emb"] for t in tgt_rows])
|
||||
r_actor_arr = np.asarray(r_actor)
|
||||
# per-actor best-of-N for every target crop at once: (n_people, n_target)
|
||||
best_sim = np.stack([(R[r_actor_arr == people.index(p)] @ T.T).max(axis=0)
|
||||
for p in people])
|
||||
proposals = []
|
||||
for j, t in enumerate(tgt_rows):
|
||||
k = int(np.argmax(best_sim[:, j]))
|
||||
prob = cal.probability(float(best_sim[k, j])) # calibrated, never a bare cosine
|
||||
proposals.append({**t, "person": people[k] if prob >= ASSIGN_P else None,
|
||||
"p": prob, "top1": people[k]})
|
||||
|
||||
# At the production threshold the global fit stays silent on most of these
|
||||
# faces, which is the honest answer for profile and downward-gaze shots — but a
|
||||
# labelling aid wants throughput, not caution. --all proposes the top-1 person
|
||||
# for every crop and orders the review sheets by descending probability, so the
|
||||
# proposals degrade visibly down the sheet and you can stop correcting where
|
||||
# they stop being right. The probability is shown, never hidden.
|
||||
if "--all" in sys.argv:
|
||||
for x in proposals:
|
||||
x["person"] = x["top1"]
|
||||
|
||||
named = [x for x in proposals if x["person"]]
|
||||
print(f"[propose] {TARGET_CLIP}: {len(named)}/{len(proposals)} named at P>={ASSIGN_P}; "
|
||||
f"{len(proposals) - len(named)} left unsorted", file=sys.stderr)
|
||||
for p in people:
|
||||
got = [x for x in named if x["person"] == p]
|
||||
if got:
|
||||
ps = [x["p"] for x in got]
|
||||
print(f" {p}: {len(got):>3} crops P {min(ps):.3f}–{max(ps):.3f}", file=sys.stderr)
|
||||
|
||||
if DRY:
|
||||
sys.exit(0)
|
||||
|
||||
# ── write proposed folders, mirroring the ref clip's layout ──────────────────
|
||||
ref_parent = os.path.dirname(next(iter(glob.glob(f"labelling/{REF_CLIP}/**/{people[0]}/",
|
||||
recursive=True))).rstrip("/"))
|
||||
tgt_parent = ref_parent.replace(REF_CLIP, TARGET_CLIP)
|
||||
for p in people:
|
||||
d = f"{tgt_parent}/{p}"
|
||||
if os.path.isdir(d): # never clobber corrections already made
|
||||
print(f"[skip] {d} exists — leaving your sorting alone", file=sys.stderr)
|
||||
continue
|
||||
os.makedirs(d, exist_ok=True)
|
||||
def find_crop(clip, fname):
|
||||
"""Locate a crop wherever it currently sits under labelling/<clip>."""
|
||||
hits = glob.glob(f"labelling/{clip}/**/{fname}", recursive=True)
|
||||
return hits[0] if hits else None
|
||||
|
||||
moved = 0
|
||||
for x in named:
|
||||
src = find_crop(TARGET_CLIP, x["file"])
|
||||
dst = f"{tgt_parent}/{x['person']}/{x['file']}"
|
||||
if src and os.path.abspath(src) != os.path.abspath(dst):
|
||||
shutil.move(src, dst)
|
||||
moved += 1
|
||||
print(f"[write] moved {moved} crops into proposed folders", file=sys.stderr)
|
||||
|
||||
# ── review sheets: confirmed block, then proposed block ─────────────────────
|
||||
def load(clip, person, fname):
|
||||
for cand in glob.glob(f"labelling/{clip}/**/{person}/{fname}", recursive=True):
|
||||
return cv2.imread(cand)
|
||||
return None
|
||||
|
||||
for person in people:
|
||||
conf = [(REF_CLIP, f, None) for f in ref_dirs.get(person, [])]
|
||||
prop = sorted([(TARGET_CLIP, x["file"], x["p"]) for x in named
|
||||
if x["person"] == person],
|
||||
key=lambda t: -t[2]) # most confident first
|
||||
items = conf + prop
|
||||
if not items:
|
||||
continue
|
||||
rows_n = (len(items) + SHEET_COLS - 1) // SHEET_COLS
|
||||
sheet = np.full((rows_n * (THUMB + 26), SHEET_COLS * THUMB, 3), 30, np.uint8)
|
||||
for n, (clip, fname, p) in enumerate(items):
|
||||
img = load(clip, person, fname)
|
||||
if img is None:
|
||||
continue
|
||||
rr, cc = divmod(n, SHEET_COLS)
|
||||
y, x = rr * (THUMB + 26), cc * THUMB
|
||||
sheet[y:y + THUMB, x:x + THUMB] = cv2.resize(img, (THUMB, THUMB))
|
||||
if p is None:
|
||||
tag, col = f"{clip[-3:]} CONFIRMED", (170, 170, 170)
|
||||
else:
|
||||
tag, col = f"{clip[-3:]} P={p:.2f}", (140, 255, 140)
|
||||
cv2.putText(sheet, tag, (x + 3, y + THUMB + 17),
|
||||
cv2.FONT_HERSHEY_SIMPLEX, 0.42, col, 1)
|
||||
cv2.imwrite(f"labelling/review_{person}.jpg", sheet)
|
||||
print(f" review_{person}.jpg: {len(conf)} confirmed + {len(prop)} proposed",
|
||||
file=sys.stderr)
|
||||
|
||||
json.dump({x["file"]: {"person": x["person"], "p": x["p"]} for x in proposals},
|
||||
open(f"labelling/proposed_{TARGET_CLIP}.json", "w"), indent=1)
|
||||
@@ -1,55 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Redraw every dumped crop with its detection box marked.
|
||||
|
||||
The original thumbnails padded by 0.5x the face on each side for
|
||||
recognisability, which in a crowded frame pulls a neighbour into shot — often
|
||||
more prominently than the subject. A label cannot be corrected from a picture
|
||||
that does not say which face it refers to.
|
||||
|
||||
This rewrites each .jpg IN PLACE, wherever it currently sits, so any sorting
|
||||
already done is preserved: only the pixels change, never the filename or the
|
||||
folder. Re-run it after dump_faces.py, and re-check any sorting done before it.
|
||||
"""
|
||||
import glob, json, os, sys
|
||||
import cv2
|
||||
|
||||
CLIPS = ["5157339", "5157344"]
|
||||
OUT = 256
|
||||
|
||||
for clip in CLIPS:
|
||||
man = {m["file"]: m for m in json.load(open(f"labelling/{clip}/manifest.json"))}
|
||||
n = 0
|
||||
for path in glob.glob(f"labelling/{clip}/**/*.jpg", recursive=True):
|
||||
fname = os.path.basename(path)
|
||||
m = man.get(fname)
|
||||
if m is None:
|
||||
continue
|
||||
img = cv2.imread(f"frames/d{clip}_{m['frame']}.png")
|
||||
if img is None:
|
||||
sys.exit(f"missing frames/d{clip}_{m['frame']}.png")
|
||||
|
||||
x, y, w, h = (int(v) for v in m["bbox"])
|
||||
pad = int(0.55 * max(w, h))
|
||||
x0, y0 = max(0, x - pad), max(0, y - pad)
|
||||
x1, y1 = min(img.shape[1], x + w + pad), min(img.shape[0], y + h + pad)
|
||||
sub = img[y0:y1, x0:x1].copy()
|
||||
|
||||
# Box in the sub-image's coordinates, drawn before the resize so the
|
||||
# line lands exactly on the face at any output size.
|
||||
cv2.rectangle(sub, (x - x0, y - y0), (x - x0 + w, y - y0 + h), (0, 0, 255), 3)
|
||||
# Dim everything outside the box so the subject is unmistakable even
|
||||
# when a neighbour's face is larger or better lit.
|
||||
mask = sub.copy()
|
||||
mask[y - y0:y - y0 + h, x - x0:x - x0 + w] = 0
|
||||
sub = cv2.addWeighted(sub, 1.0, mask, -0.35, 0)
|
||||
|
||||
scale = OUT / max(sub.shape[:2])
|
||||
sub = cv2.resize(sub, (int(sub.shape[1] * scale), int(sub.shape[0] * scale)))
|
||||
canvas = cv2.copyMakeBorder(
|
||||
sub, 0, max(0, OUT - sub.shape[0]), 0, max(0, OUT - sub.shape[1]),
|
||||
cv2.BORDER_CONSTANT, value=(20, 20, 20))[:OUT, :OUT]
|
||||
cv2.putText(canvas, f"{int(m['px'])}px", (5, OUT - 8),
|
||||
cv2.FONT_HERSHEY_SIMPLEX, 0.45, (0, 255, 255), 1)
|
||||
cv2.imwrite(path, canvas)
|
||||
n += 1
|
||||
print(f"[{clip}] redrew {n} crops in place", file=sys.stderr)
|
||||
@@ -1,184 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Impact of input resolution on cross-source identification.
|
||||
|
||||
TRACES: VR-013 | PR-002
|
||||
|
||||
Gallery is built from one clip at NATIVE resolution. Probes come from the other
|
||||
clip with the WHOLE FRAME downscaled before it reaches the detector, so
|
||||
detection and landmark regression degrade together with the pixels. That is the
|
||||
measurement VR-005 structurally could not make: it degraded an already-aligned
|
||||
112x112 crop, holding alignment perfect, so it isolated the embedder's
|
||||
resolution sensitivity and excluded everything upstream of it.
|
||||
|
||||
python3 resolution_sweep.py [--gallery-clip 5157339] [--detector scrfd_500m_bnkps.onnx]
|
||||
|
||||
Ground truth
|
||||
------------
|
||||
Hand-sorted person folders. Probe detections at reduced scale are tied back to
|
||||
a labelled face GEOMETRICALLY — the box is mapped to native coordinates and
|
||||
matched by IoU. Never by embedding similarity, which would be circular: it
|
||||
would keep the faces the embedder still gets right and silently drop the ones
|
||||
this sweep exists to find.
|
||||
|
||||
A probe whose label is only in the probe clip is OUT OF GALLERY. Naming it is a
|
||||
true out-of-cast misID, the error the per-scene scorer weights 10x, so it is
|
||||
counted separately from naming the wrong gallery member.
|
||||
|
||||
Metric
|
||||
------
|
||||
The calibrated probability from the PRODUCTION gallery sigmoid, never a raw
|
||||
cosine (AR-024). Per-actor best-of-N similarity -> probability -> accept above
|
||||
prob_threshold. This is identification, so the matcher's prior applies;
|
||||
config.hpp has match_prior 0.5, i.e. log_prior_odds = 0.
|
||||
|
||||
Everything runs through the shipped C++ via sae_embed.
|
||||
"""
|
||||
import sys, glob, json, os, argparse
|
||||
import numpy as np
|
||||
import cv2
|
||||
|
||||
sys.path.insert(0, "/home/dtourolle/Development/Jray-project/scene-actor-extraction/build-ort")
|
||||
import sae_embed
|
||||
|
||||
ROOT = "/home/dtourolle/Development/Jray-project/scene-actor-extraction/"
|
||||
M = ROOT + "models/"
|
||||
PROB_THRESHOLD = 0.754 # config.hpp:67
|
||||
LOG_PRIOR_ODDS = 0.0 # config.hpp:61 match_prior=0.5
|
||||
IOU_MIN = 0.3 # geometric label carry-down
|
||||
SCALES = [1.0, 0.8, 0.6, 0.5, 0.4, 0.3, 0.25, 0.2, 0.15, 0.12, 0.09, 0.06]
|
||||
|
||||
ap = argparse.ArgumentParser()
|
||||
ap.add_argument("--gallery-clip", default="5157339")
|
||||
ap.add_argument("--probe-clip", default="5157344")
|
||||
ap.add_argument("--detector", default="scrfd_500m_bnkps.onnx")
|
||||
ap.add_argument("--embedder", default="LVFace-B_Glint360K.onnx")
|
||||
ap.add_argument("--gallery-calibration", default=ROOT + "gallery_lvface.h5")
|
||||
ap.add_argument("--out", default="results_resolution_sweep.json")
|
||||
args = ap.parse_args()
|
||||
|
||||
eng = sae_embed.FaceEmbedder(detector_model=M + args.detector,
|
||||
arcface_model=M + args.embedder,
|
||||
conf=0.5, nms=0.4, max_side=0)
|
||||
cal = sae_embed.gallery_calibration(args.gallery_calibration)
|
||||
print(f"[calibration] global: {cal}", file=sys.stderr)
|
||||
|
||||
|
||||
def labelled(clip):
|
||||
"""{filename: person} from the hand-sorted folders, ignoring discard."""
|
||||
out = {}
|
||||
for path in glob.glob(f"labelling/{clip}/*/*.jpg"):
|
||||
person = os.path.basename(os.path.dirname(path))
|
||||
if person in ("discard", "unsorted"):
|
||||
continue
|
||||
out[os.path.basename(path)] = person
|
||||
return out
|
||||
|
||||
|
||||
def manifest(clip):
|
||||
return {m["file"]: m for m in json.load(open(f"labelling/{clip}/manifest.json"))}
|
||||
|
||||
|
||||
def iou(a, b):
|
||||
ax, ay, aw, ah = a; bx, by, bw, bh = b
|
||||
x0, y0 = max(ax, bx), max(ay, by)
|
||||
x1, y1 = min(ax + aw, bx + bw), min(ay + ah, by + bh)
|
||||
if x1 <= x0 or y1 <= y0:
|
||||
return 0.0
|
||||
inter = (x1 - x0) * (y1 - y0)
|
||||
return inter / (aw * ah + bw * bh - inter)
|
||||
|
||||
|
||||
# ── gallery: native resolution, labelled faces only ──────────────────────────
|
||||
g_lab, g_man = labelled(args.gallery_clip), manifest(args.gallery_clip)
|
||||
gal = {}
|
||||
for frame in sorted({g_man[f]["frame"] for f in g_lab}):
|
||||
img = cv2.imread(f"frames/d{args.gallery_clip}_{frame}.png")
|
||||
dets = eng.detect(img)
|
||||
for fname, person in g_lab.items():
|
||||
m = g_man[fname]
|
||||
if m["frame"] != frame or m["idx"] >= len(dets):
|
||||
continue
|
||||
lm = np.array(dets[m["idx"]].landmarks, dtype=np.float32).reshape(5, 2)
|
||||
crop = sae_embed.align_face(img, lm)
|
||||
if crop is None:
|
||||
continue
|
||||
gal.setdefault(person, []).append(np.asarray(eng.embed_crop(crop), dtype=np.float32))
|
||||
gal = {p: np.stack(v) for p, v in gal.items() if v}
|
||||
people = sorted(gal)
|
||||
print(f"[gallery] {args.gallery_clip} @native: "
|
||||
f"{ {p: len(v) for p, v in gal.items()} }", file=sys.stderr)
|
||||
|
||||
# ── probe ground truth at native resolution ──────────────────────────────────
|
||||
p_lab, p_man = labelled(args.probe_clip), manifest(args.probe_clip)
|
||||
truth = {} # frame -> [(bbox_native, person)]
|
||||
for fname, person in p_lab.items():
|
||||
m = p_man[fname]
|
||||
truth.setdefault(m["frame"], []).append((m["bbox"], person))
|
||||
n_out = sum(1 for p in set(p_lab.values()) if p not in people)
|
||||
print(f"[probe] {args.probe_clip}: {len(p_lab)} labelled faces, "
|
||||
f"{len(set(p_lab.values()))} people, {n_out} of them out-of-gallery",
|
||||
file=sys.stderr)
|
||||
|
||||
# ── sweep ────────────────────────────────────────────────────────────────────
|
||||
print(f"\n{'scale':>6}{'frame':>11}{'face px':>9}{'found':>7}{'matched':>9}"
|
||||
f"{'TPI':>8}{'FPI-in':>8}{'FPI-out':>9}{'TBI':>8}")
|
||||
results = []
|
||||
for s in SCALES:
|
||||
tpi = fpi_in = fpi_out = tbi = 0
|
||||
n_found = n_matched = 0
|
||||
pxs = []
|
||||
for frame, gts in sorted(truth.items()):
|
||||
img = cv2.imread(f"frames/d{args.probe_clip}_{frame}.png")
|
||||
if s != 1.0:
|
||||
img = cv2.resize(img, None, fx=s, fy=s, interpolation=cv2.INTER_AREA)
|
||||
dets = eng.detect(img)
|
||||
n_found += len(dets)
|
||||
for d in dets:
|
||||
x, y, w, h = d.bbox
|
||||
native = (x / s, y / s, w / s, h / s) # geometric carry-down
|
||||
best, best_iou = None, 0.0
|
||||
for gt_box, person in gts:
|
||||
v = iou(native, gt_box)
|
||||
if v > best_iou:
|
||||
best_iou, best = v, person
|
||||
if best_iou < IOU_MIN:
|
||||
continue # spurious / unlabelled
|
||||
n_matched += 1
|
||||
pxs.append(min(w, h))
|
||||
|
||||
lm = np.array(d.landmarks, dtype=np.float32).reshape(5, 2)
|
||||
crop = sae_embed.align_face(img, lm)
|
||||
if crop is None:
|
||||
tbi += 1 # degenerate alignment
|
||||
continue
|
||||
emb = np.asarray(eng.embed_crop(crop), dtype=np.float32)
|
||||
best_p, best_name = 0.0, None
|
||||
for p in people: # per-actor best-of-N
|
||||
prob = cal.probability(float((gal[p] @ emb).max()), LOG_PRIOR_ODDS)
|
||||
if prob > best_p:
|
||||
best_p, best_name = prob, p
|
||||
if best_p <= PROB_THRESHOLD:
|
||||
tbi += 1
|
||||
elif best not in people:
|
||||
fpi_out += 1 # named someone absent from the gallery
|
||||
elif best_name == best:
|
||||
tpi += 1
|
||||
else:
|
||||
fpi_in += 1
|
||||
n = max(1, n_matched)
|
||||
med_px = float(np.median(pxs)) if pxs else 0.0
|
||||
print(f"{s:>6.2f}{f'{int(4096*s)}x{int(2160*s)}':>11}{med_px:>9.0f}"
|
||||
f"{n_found:>7}{n_matched:>9}"
|
||||
f"{100*tpi/n:>7.1f}%{100*fpi_in/n:>7.1f}%{100*fpi_out/n:>8.1f}%{100*tbi/n:>7.1f}%")
|
||||
results.append({"scale": s, "median_face_px": med_px, "detections": n_found,
|
||||
"matched_to_truth": n_matched, "tpi_pct": 100*tpi/n,
|
||||
"fpi_in_gallery_pct": 100*fpi_in/n, "fpi_out_of_gallery_pct": 100*fpi_out/n,
|
||||
"tbi_pct": 100*tbi/n})
|
||||
|
||||
json.dump({"gallery_clip": args.gallery_clip, "probe_clip": args.probe_clip,
|
||||
"detector": args.detector, "embedder": args.embedder,
|
||||
"prob_threshold": PROB_THRESHOLD, "log_prior_odds": LOG_PRIOR_ODDS,
|
||||
"calibration": {"a": cal.a, "b": cal.b},
|
||||
"gallery_people": people, "results": results},
|
||||
open(args.out, "w"), indent=2)
|
||||
print(f"\nwrote {args.out}", file=sys.stderr)
|
||||
@@ -1,175 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Integrity check on the labelled set, before it is used as ground truth.
|
||||
|
||||
TRACES: VR-013 | PR-002
|
||||
|
||||
VR-013's ground truth is hand-sorted rather than propagated by embedding
|
||||
similarity, because propagation would keep only the faces the embedder already
|
||||
gets right and silently drop the ones the sweep exists to find. This script is
|
||||
what makes that claim checkable, so it is part of the requirement rather than a
|
||||
helper of it.
|
||||
|
||||
Checks, loudest failure first:
|
||||
|
||||
1. INDEX INTEGRITY. Each crop's embedding is taken by re-detecting its source
|
||||
frame and indexing with the manifest's `idx`. If detection order is not
|
||||
reproducible, the thumbnail you sorted and the embedding that gets scored
|
||||
are different faces — you would see a correct picture and score the wrong
|
||||
person, with nothing to signal it. Every crop's re-detected bbox is compared
|
||||
against the manifest's.
|
||||
|
||||
2. NO CROP IN TWO FOLDERS, and every manifest entry accounted for — so a
|
||||
move that half-completed cannot silently duplicate or drop a label.
|
||||
|
||||
3. ALIGNMENT. The 112x112 warp is what the embedder actually sees; the
|
||||
thumbnail is only context for your eyes. verify_<person>.jpg pairs them:
|
||||
context-with-box on top, the real aligned crop beneath. A profile face whose
|
||||
alignment has collapsed is obvious there and nowhere else.
|
||||
|
||||
4. SEPARATION. Per person, the calibrated P of their own crops against the
|
||||
other people's, using the global gallery sigmoid. A label set where someone
|
||||
matches another person better than themselves is mislabelled.
|
||||
|
||||
Nothing here changes a label. It reports.
|
||||
"""
|
||||
import sys, glob, json, os
|
||||
import numpy as np
|
||||
import cv2
|
||||
|
||||
sys.path.insert(0, "/home/dtourolle/Development/Jray-project/scene-actor-extraction/build-ort")
|
||||
import sae_embed
|
||||
|
||||
ROOT = "/home/dtourolle/Development/Jray-project/scene-actor-extraction/"
|
||||
M = ROOT + "models/"
|
||||
EMBEDDER = M + "LVFace-B_Glint360K.onnx"
|
||||
GALLERY = ROOT + "gallery_lvface.h5"
|
||||
CLIPS = ["5157344", "5157339"]
|
||||
THUMB = 130
|
||||
COLS = 10
|
||||
|
||||
eng = sae_embed.FaceEmbedder(detector_model=M + "scrfd_500m_bnkps.onnx",
|
||||
arcface_model=EMBEDDER, conf=0.5, nms=0.4, max_side=0)
|
||||
|
||||
fail = 0
|
||||
rows = []
|
||||
|
||||
for clip in CLIPS:
|
||||
man = {m["file"]: m for m in json.load(open(f"labelling/{clip}/manifest.json"))}
|
||||
|
||||
# where each crop currently sits -> its label
|
||||
placed = {}
|
||||
for path in glob.glob(f"labelling/{clip}/*/*.jpg"):
|
||||
person = os.path.basename(os.path.dirname(path))
|
||||
if person in ("discard", "unsorted"):
|
||||
continue # not people; scoring them would invent an extra identity
|
||||
fname = os.path.basename(path)
|
||||
if fname in placed:
|
||||
print(f"[FAIL] {fname} appears in both {placed[fname][0]} and {person}")
|
||||
fail += 1
|
||||
placed[fname] = (person, path)
|
||||
|
||||
missing = set(man) - set(placed)
|
||||
extra = set(placed) - set(man)
|
||||
if missing:
|
||||
print(f"[warn] {clip}: {len(missing)} manifest crops not in any folder")
|
||||
if extra:
|
||||
print(f"[FAIL] {clip}: {len(extra)} files with no manifest entry: "
|
||||
f"{sorted(extra)[:3]}")
|
||||
fail += 1
|
||||
|
||||
# index integrity + alignment, frame by frame
|
||||
by_frame = {}
|
||||
for fname, (person, path) in placed.items():
|
||||
if fname in man:
|
||||
by_frame.setdefault(man[fname]["frame"], []).append((fname, person, path))
|
||||
|
||||
bad_idx = 0
|
||||
for frame, items in sorted(by_frame.items()):
|
||||
img = cv2.imread(f"frames/d{clip}_{frame}.png")
|
||||
if img is None:
|
||||
print(f"[FAIL] missing frames/d{clip}_{frame}.png")
|
||||
fail += 1
|
||||
continue
|
||||
dets = eng.detect(img)
|
||||
for fname, person, path in items:
|
||||
m = man[fname]
|
||||
i = m["idx"]
|
||||
if i >= len(dets):
|
||||
print(f"[FAIL] {fname}: idx {i} >= {len(dets)} detections now")
|
||||
bad_idx += 1
|
||||
continue
|
||||
got = [float(v) for v in dets[i].bbox]
|
||||
want = m["bbox"]
|
||||
if max(abs(a - b) for a, b in zip(got, want)) > 1.0:
|
||||
print(f"[FAIL] {fname}: manifest bbox {[round(v) for v in want]} "
|
||||
f"!= re-detected {[round(v) for v in got]}")
|
||||
bad_idx += 1
|
||||
continue
|
||||
lm = np.array(dets[i].landmarks, dtype=np.float32).reshape(5, 2)
|
||||
crop = sae_embed.align_face(img, lm)
|
||||
if crop is None:
|
||||
print(f"[warn] {fname}: alignment degenerate, no crop reaches the embedder")
|
||||
continue
|
||||
rows.append({"clip": clip, "person": person, "file": fname, "path": path,
|
||||
"px": m["px"], "aligned": np.asarray(crop),
|
||||
"emb": np.asarray(eng.embed_crop(crop), dtype=np.float32)})
|
||||
fail += bad_idx
|
||||
print(f"[{clip}] {len(placed)} placed, {len(by_frame)} frames, "
|
||||
f"index mismatches: {bad_idx}")
|
||||
|
||||
if not rows:
|
||||
sys.exit("nothing to verify")
|
||||
|
||||
# ── separation, through the global gallery sigmoid ───────────────────────────
|
||||
cal = sae_embed.gallery_calibration(GALLERY)
|
||||
E = np.stack([r["emb"] for r in rows])
|
||||
people = sorted({r["person"] for r in rows})
|
||||
lab = np.array([people.index(r["person"]) for r in rows])
|
||||
S = E @ E.T
|
||||
np.fill_diagonal(S, -1.0)
|
||||
|
||||
print(f"\n{'person':>8}{'crops':>7}{'344':>6}{'339':>6}"
|
||||
f"{'P(self)':>10}{'P(other)':>10}{'worst':>8}")
|
||||
for k, p in enumerate(people):
|
||||
mine = np.where(lab == k)[0]
|
||||
if len(mine) < 2:
|
||||
continue
|
||||
self_sim = S[np.ix_(mine, mine)].max(axis=1)
|
||||
other_sim = S[np.ix_(mine, np.where(lab != k)[0])].max(axis=1)
|
||||
p_self = np.array([cal.probability(float(s)) for s in self_sim])
|
||||
p_other = np.array([cal.probability(float(s)) for s in other_sim])
|
||||
n344 = sum(1 for i in mine if rows[i]["clip"] == "5157344")
|
||||
n339 = len(mine) - n344
|
||||
# a crop that matches someone else better than anyone of its own label
|
||||
worst = int((other_sim > self_sim).sum())
|
||||
print(f"{p:>8}{len(mine):>7}{n344:>6}{n339:>6}"
|
||||
f"{np.median(p_self):>10.3f}{np.median(p_other):>10.3f}{worst:>8}")
|
||||
if worst:
|
||||
for i in mine[other_sim > self_sim]:
|
||||
print(f" suspect: {rows[i]['file']} "
|
||||
f"P(self)={cal.probability(float(self_sim[list(mine).index(i)])):.3f} "
|
||||
f"< P(other)={cal.probability(float(other_sim[list(mine).index(i)])):.3f}")
|
||||
|
||||
# ── verify sheets: context+box over the actual aligned crop ──────────────────
|
||||
for p in people:
|
||||
items = [r for r in rows if r["person"] == p]
|
||||
items.sort(key=lambda r: (r["clip"], r["file"]))
|
||||
n = len(items)
|
||||
sheet_rows = (n + COLS - 1) // COLS
|
||||
H = THUMB * 2 + 22
|
||||
sheet = np.full((sheet_rows * H, COLS * THUMB, 3), 25, np.uint8)
|
||||
for j, r in enumerate(items):
|
||||
rr, cc = divmod(j, COLS)
|
||||
y, x = rr * H, cc * THUMB
|
||||
ctx = cv2.imread(r["path"])
|
||||
if ctx is not None:
|
||||
sheet[y:y + THUMB, x:x + THUMB] = cv2.resize(ctx, (THUMB, THUMB))
|
||||
sheet[y + THUMB:y + 2 * THUMB, x:x + THUMB] = cv2.resize(r["aligned"], (THUMB, THUMB))
|
||||
cv2.putText(sheet, f"{r['clip'][-3:]} {int(r['px'])}px",
|
||||
(x + 3, y + 2 * THUMB + 15),
|
||||
cv2.FONT_HERSHEY_SIMPLEX, 0.38, (150, 220, 150), 1)
|
||||
cv2.imwrite(f"labelling/verify_{p}.jpg", sheet)
|
||||
print(f" verify_{p}.jpg: {n} crops (top row context, bottom row what the embedder sees)")
|
||||
|
||||
print(f"\n{'PASS' if fail == 0 else f'{fail} FAILURES'}")
|
||||
sys.exit(1 if fail else 0)
|
||||
Vendored
+1
-1
Submodule external/KPN updated: 454f72c167...4b6e498ba7
BIN
Binary file not shown.
BIN
Binary file not shown.
Binary file not shown.
@@ -10,9 +10,7 @@
|
||||
# scripts/artifacts/pull_artifacts.sh galleries [version]
|
||||
# scripts/artifacts/pull_artifacts.sh montage-frames <film-slug> [version]
|
||||
# scripts/artifacts/pull_artifacts.sh experiment-data [version]
|
||||
# scripts/artifacts/pull_artifacts.sh replay-fixtures [version]
|
||||
# scripts/artifacts/pull_artifacts.sh report-highlights <name> [version]
|
||||
# scripts/artifacts/pull_artifacts.sh xsource [version]
|
||||
# version defaults to "latest" (newest uploaded version, by created_at).
|
||||
set -euo pipefail
|
||||
|
||||
@@ -43,22 +41,6 @@ print(matches[-1]['version'])
|
||||
"
|
||||
}
|
||||
|
||||
pull_replay_fixtures() {
|
||||
local version="$1"
|
||||
local dest="${REPO_ROOT}/tests/fixtures/dumps"
|
||||
mkdir -p "$dest"
|
||||
echo "=== replay-fixtures (version ${version}) ==="
|
||||
local tmp; tmp="$(mktemp -d)"
|
||||
if curl -sf "${DL_BASE}/generic/replay-fixtures/${version}/replay-fixtures.zip" \
|
||||
-o "${tmp}/f.zip"; then
|
||||
unzip -qo "${tmp}/f.zip" -d "$dest"
|
||||
echo " restored: $(ls "$dest" | wc -l) files into tests/fixtures/dumps/"
|
||||
else
|
||||
echo " [warn] replay-fixtures.zip not found at version ${version}" >&2
|
||||
fi
|
||||
rm -rf "$tmp"
|
||||
}
|
||||
|
||||
pull_galleries() {
|
||||
local version="$1"
|
||||
local dest="${REPO_ROOT}/experiments/galleries"
|
||||
@@ -101,71 +83,11 @@ pull_report_highlight() {
|
||||
curl -sf "${DL_BASE}/generic/report-highlights/${version}/${name}" -o "${dest}/${name}"
|
||||
}
|
||||
|
||||
pull_xsource() {
|
||||
local version="$1"
|
||||
local dest="${REPO_ROOT}/experiments/xsource"
|
||||
echo "=== xsource (version ${version}) ==="
|
||||
mkdir -p "${dest}/clips" "${dest}/frames"
|
||||
|
||||
for clip in 5157339 5157344; do
|
||||
if [ -f "${dest}/clips/${clip}.mp4" ]; then
|
||||
echo " ${clip}.mp4 already present, skipping"
|
||||
else
|
||||
echo " fetching ${clip}.mp4..."
|
||||
curl -sf "${DL_BASE}/generic/xsource/${version}/${clip}.mp4" \
|
||||
-o "${dest}/clips/${clip}.mp4" \
|
||||
|| { echo " [warn] ${clip}.mp4 not found at version ${version}" >&2; continue; }
|
||||
fi
|
||||
done
|
||||
|
||||
if [ -d "${dest}/labelling" ]; then
|
||||
echo " labelling/ already present — NOT overwriting (it is hand-sorted"
|
||||
echo " ground truth; move it aside first if you really want the remote copy)"
|
||||
else
|
||||
echo " fetching labelling.zip..."
|
||||
local tmp; tmp="$(mktemp)"
|
||||
curl -sf "${DL_BASE}/generic/xsource/${version}/labelling.zip" -o "$tmp"
|
||||
unzip -qo "$tmp" -d "$dest"
|
||||
rm "$tmp"
|
||||
fi
|
||||
|
||||
# Frames are regenerated rather than shipped: they are ~320 MB of PNG that
|
||||
# ffmpeg reproduces exactly from the clips. The manifests key on these
|
||||
# filenames and on detection order within each frame, so the extraction
|
||||
# settings must match the ones dump_faces.py ran against — hence fps and
|
||||
# frame count are pinned here rather than left to the caller.
|
||||
if ! command -v ffmpeg >/dev/null; then
|
||||
echo " [warn] ffmpeg not found — frames not regenerated; the study" >&2
|
||||
echo " scripts will fail until you extract them" >&2
|
||||
return
|
||||
fi
|
||||
for clip in 5157339 5157344; do
|
||||
[ -f "${dest}/clips/${clip}.mp4" ] || continue
|
||||
if [ -f "${dest}/frames/d${clip}_001.png" ]; then
|
||||
echo " frames for ${clip} already present, skipping"
|
||||
continue
|
||||
fi
|
||||
echo " extracting frames for ${clip}..."
|
||||
ffmpeg -v error -i "${dest}/clips/${clip}.mp4" -vf fps=2 -frames:v 24 \
|
||||
"${dest}/frames/d${clip}_%03d.png"
|
||||
done
|
||||
|
||||
echo " verifying the labelled set..."
|
||||
if (cd "$dest" && python3 verify_labels.py >/dev/null 2>&1); then
|
||||
echo " verify_labels.py passed"
|
||||
else
|
||||
echo " [warn] verify_labels.py failed — run it directly to see why." >&2
|
||||
echo " A frame/manifest mismatch means the extraction settings" >&2
|
||||
echo " differ from the ones the crops were dumped against." >&2
|
||||
fi
|
||||
}
|
||||
|
||||
if [ $# -eq 0 ]; then
|
||||
echo "usage: $0 galleries [version]" >&2
|
||||
echo " $0 montage-frames <film-slug> [version]" >&2
|
||||
echo " $0 experiment-data [version]" >&2
|
||||
echo " $0 report-highlights <name> [version]" >&2
|
||||
echo " $0 xsource [version]" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
@@ -193,18 +115,8 @@ case "$TARGET" in
|
||||
[ "$VERSION" = "latest" ] && VERSION="$(resolve_latest_version report-highlights)"
|
||||
pull_report_highlight "$VERSION" "$NAME"
|
||||
;;
|
||||
xsource)
|
||||
VERSION="${2:-latest}"
|
||||
[ "$VERSION" = "latest" ] && VERSION="$(resolve_latest_version xsource)"
|
||||
pull_xsource "$VERSION"
|
||||
;;
|
||||
replay-fixtures)
|
||||
VERSION="${2:-latest}"
|
||||
[ "$VERSION" = "latest" ] && VERSION="$(resolve_latest_version replay-fixtures)"
|
||||
pull_replay_fixtures "$VERSION"
|
||||
;;
|
||||
*)
|
||||
echo "unknown target: $TARGET (expected galleries, montage-frames, experiment-data, report-highlights, xsource, or replay-fixtures)" >&2
|
||||
echo "unknown target: $TARGET (expected galleries, montage-frames, experiment-data, or report-highlights)" >&2
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
|
||||
@@ -12,13 +12,10 @@
|
||||
# scripts/artifacts/push_artifacts.sh montage-frames
|
||||
# scripts/artifacts/push_artifacts.sh experiment-data
|
||||
# scripts/artifacts/push_artifacts.sh report-highlights
|
||||
# scripts/artifacts/push_artifacts.sh xsource
|
||||
# scripts/artifacts/push_artifacts.sh replay-fixtures
|
||||
# scripts/artifacts/push_artifacts.sh galleries montage-frames experiment-data report-highlights
|
||||
#
|
||||
# Package layout (owner=dtourolle, repo=scene-actor-extraction):
|
||||
# generic/galleries/<version>/gallery_<model>.h5 (one file per model)
|
||||
# generic/replay-fixtures/<version>/replay-fixtures.zip (T2 dumps + their gallery)
|
||||
# generic/montage-frames/<version>/<film-slug>.zip (zipped per-film frames)
|
||||
# generic/experiment-data/<version>/experiment-data.zip (manifests/trajectories/results)
|
||||
# generic/report-highlights/<version>/<name>.jpg (individual, hand-picked
|
||||
@@ -52,29 +49,6 @@ upload() {
|
||||
-o /dev/null -w " HTTP %{http_code}\n"
|
||||
}
|
||||
|
||||
push_replay_fixtures() {
|
||||
echo "=== replay-fixtures (version ${VERSION}) ==="
|
||||
# T2 replay fixtures: per-frame detections, landmarks and embeddings dumped
|
||||
# from a real run, so the tracker and identity stages can be replayed on CPU
|
||||
# with no GPU, no models and no film. Too large for git (superhero.h5 alone
|
||||
# is ~9 MB) and regenerating them needs the film plus a GPU, which CI has
|
||||
# neither of — so they ship as artifacts and CI pulls them.
|
||||
#
|
||||
# The gallery travels with them: a dump replays against the gallery it was
|
||||
# produced with, and pairing a dump with a different gallery silently
|
||||
# changes every identity decision in it.
|
||||
local dir="${REPO_ROOT}/tests/fixtures/dumps"
|
||||
if [ ! -d "$dir" ]; then
|
||||
echo " no tests/fixtures/dumps dir, skipping" >&2
|
||||
return
|
||||
fi
|
||||
local tmp
|
||||
tmp="$(mktemp -d)"
|
||||
( cd "$dir" && zip -qr "$tmp/replay-fixtures.zip" . )
|
||||
upload "replay-fixtures" "replay-fixtures.zip" "$tmp/replay-fixtures.zip"
|
||||
rm -rf "$tmp"
|
||||
}
|
||||
|
||||
push_galleries() {
|
||||
echo "=== galleries (version ${VERSION}) ==="
|
||||
local dir="${REPO_ROOT}/experiments/galleries"
|
||||
@@ -135,35 +109,8 @@ push_report_highlights() {
|
||||
upload "report-highlights" "germar_beats_xray.jpg" "$src"
|
||||
}
|
||||
|
||||
push_xsource() {
|
||||
echo "=== xsource (version ${VERSION}) ==="
|
||||
local root="${REPO_ROOT}/experiments/xsource"
|
||||
if [ ! -d "$root/labelling" ]; then
|
||||
echo " no experiments/xsource/labelling found, skipping" >&2
|
||||
return
|
||||
fi
|
||||
|
||||
# Source recordings. Already compressed, so uploaded as-is rather than zipped.
|
||||
shopt -s nullglob
|
||||
for f in "$root"/clips/*.mp4; do
|
||||
upload "xsource" "$(basename "$f")" "$f"
|
||||
done
|
||||
shopt -u nullglob
|
||||
|
||||
# The hand-sorted crops and their manifests. This is human ground truth and
|
||||
# the expensive part of the study — a person looked at every crop and put it
|
||||
# in a folder. Frames are deliberately NOT pushed: they are deterministic
|
||||
# from the clips, and pulling regenerates them.
|
||||
local tmp; tmp="$(mktemp -d)"
|
||||
trap 'rm -rf "$tmp"' RETURN
|
||||
local zipfile="${tmp}/labelling.zip"
|
||||
(cd "$root" && zip -qr "$zipfile" labelling -x 'labelling/*.html' -x 'labelling/review_*.jpg' \
|
||||
-x 'labelling/verify_*.jpg')
|
||||
upload "xsource" "labelling.zip" "$zipfile"
|
||||
}
|
||||
|
||||
if [ $# -eq 0 ]; then
|
||||
echo "usage: $0 <galleries|montage-frames|experiment-data|report-highlights|xsource> [...]" >&2
|
||||
echo "usage: $0 <galleries|montage-frames|experiment-data|report-highlights> [...]" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
@@ -173,9 +120,7 @@ for target in "$@"; do
|
||||
montage-frames) push_montage_frames ;;
|
||||
experiment-data) push_experiment_data ;;
|
||||
report-highlights) push_report_highlights ;;
|
||||
xsource) push_xsource ;;
|
||||
replay-fixtures) push_replay_fixtures ;;
|
||||
*) echo "unknown target: $target (expected galleries, montage-frames, experiment-data, report-highlights, xsource, or replay-fixtures)" >&2; exit 1 ;;
|
||||
*) echo "unknown target: $target (expected galleries, montage-frames, experiment-data, or report-highlights)" >&2; exit 1 ;;
|
||||
esac
|
||||
done
|
||||
|
||||
|
||||
@@ -1,98 +0,0 @@
|
||||
#!/bin/bash
|
||||
# fetch_dvu.sh — pull one film's character mugshots and presence annotations from
|
||||
# the NIST TRECVID Deep Video Understanding development set.
|
||||
#
|
||||
# The DVU dev set is the reason Road to Bali is our benchmark film: it ships
|
||||
# 5-7 face crops per *character*, cut from the film itself, alongside
|
||||
# scene-scoped presence annotations. That matches SR-002 directly — presence is
|
||||
# per scene, not per frame — and it keeps ground truth in character space, so
|
||||
# scoring needs no actor->character mapping.
|
||||
#
|
||||
# This exists as a script, rather than as ad hoc commands, because the first
|
||||
# copy of this data lived in a temp directory and was lost to a /tmp wipe,
|
||||
# taking the working gallery with it.
|
||||
#
|
||||
# 14 films are asserted Creative Commons and need no data agreement (only the
|
||||
# 5 KinoLorber test films are gated).
|
||||
#
|
||||
# Usage:
|
||||
# scripts/fetch_dvu.sh [film] [dest]
|
||||
# film default Road_To_Bali
|
||||
# dest default ./dvu
|
||||
set -euo pipefail
|
||||
|
||||
BASE="https://www-nlpir.nist.gov/projects/trecvid/dvu/dvu.development.dataset"
|
||||
FILM="${1:-Road_To_Bali}"
|
||||
DEST="${2:-dvu}"
|
||||
|
||||
mkdir -p "$DEST/images" "$DEST/scenes"
|
||||
|
||||
echo "[dvu] $FILM -> $DEST"
|
||||
|
||||
# Scene segmentation: start/end as HH:MM:SS. Note valkaama.csv line 38 carries a
|
||||
# shift-key typo (01:!4:00) — parse defensively if you extend this to that film.
|
||||
echo "[dvu] scene segmentation"
|
||||
curl -fsSL "$BASE/scene.segmentation.reference/${FILM}.csv" \
|
||||
-o "$DEST/${FILM}.csv" || echo " (missing: ${FILM}.csv)"
|
||||
|
||||
# Entity types: which entities are Person vs Location/Concept. Only Person rows
|
||||
# become gallery identities — the images/ directory also holds Location and
|
||||
# Concept crops (bedroom, boat, ...), which must not enter a face gallery.
|
||||
#
|
||||
# Directory and file naming are inconsistent with the film slug used elsewhere:
|
||||
# the folder is Road_to_Bali (lowercase "to") while the entity file is
|
||||
# RoadToBali.entity.types.txt. Both are derived here rather than assumed.
|
||||
# NIST is inconsistent across all three axes, and not by a rule worth deriving:
|
||||
# Road to Bali is Road_To_Bali.csv / Road_to_Bali/ / RoadToBali.entity.types.txt,
|
||||
# while SuperHero is SuperHero.csv / superHero/ / superhero.entity.types.txt.
|
||||
# Defaults cover the Bali shape; override per film rather than guessing.
|
||||
# KG_DIR=superHero KG_FILE=superhero scripts/fetch_dvu.sh SuperHero dvu-hero
|
||||
KG_DIR="${KG_DIR:-${FILM//_To_/_to_}}"
|
||||
KG_FILE="${KG_FILE:-$(echo "$FILM" | sed -E 's/_([a-z])/\U\1/g; s/_//g')}"
|
||||
|
||||
echo "[dvu] entity types ($KG_DIR/$KG_FILE)"
|
||||
curl -fsSL "$BASE/movie_knowledge_graph/${KG_DIR}/${KG_FILE}.entity.types.txt" \
|
||||
-o "$DEST/${FILM}.entity.types.txt" || echo " (missing: entity types)"
|
||||
|
||||
# Character face crops. Names are discovered from the directory listing rather
|
||||
# than probed as <Character>_N, since the crop count varies per character and
|
||||
# the listing is authoritative.
|
||||
echo "[dvu] character mugshots"
|
||||
PERSONS="$DEST/persons.txt"
|
||||
if [ -f "$DEST/${FILM}.entity.types.txt" ]; then
|
||||
grep -iE "person" "$DEST/${FILM}.entity.types.txt" \
|
||||
| sed -E 's/[[:space:]]*[:,].*$//' | tr -d '\r' \
|
||||
| awk '{print tolower($1)}' | sort -u > "$PERSONS"
|
||||
fi
|
||||
|
||||
curl -fsSL "$BASE/movie_knowledge_graph/${KG_DIR}/images/" 2>/dev/null \
|
||||
| grep -oE 'href="[^"?/][^"]*\.png"' | sed -E 's/href="//; s/"//' | sort -u \
|
||||
> "$DEST/all_images.txt"
|
||||
|
||||
while read -r img; do
|
||||
[ -z "$img" ] && continue
|
||||
# Strip the trailing _N to recover the entity name.
|
||||
who="$(echo "$img" | sed -E 's/_[0-9]+\.png$//' | awk '{print tolower($0)}')"
|
||||
if [ -s "$PERSONS" ] && ! grep -qx "$who" "$PERSONS"; then
|
||||
continue # Location/Concept crop, not a face
|
||||
fi
|
||||
curl -fsSL "$BASE/movie_knowledge_graph/${KG_DIR}/images/${img}" \
|
||||
-o "$DEST/images/${img}" 2>/dev/null || rm -f "$DEST/images/${img}"
|
||||
done < "$DEST/all_images.txt"
|
||||
|
||||
# Per-scene knowledge graphs. A Person->Location edge means that person was
|
||||
# present for the whole scene. Some of these contain a stray ", ," that breaks
|
||||
# strict JSON parsers.
|
||||
echo "[dvu] scene graphs"
|
||||
for n in $(seq 1 60); do
|
||||
curl -fsSL "$BASE/scenes_knowledge_graphs/${FILM//_/ }-${n}.json" \
|
||||
-o "$DEST/scenes/${FILM}-${n}.json" 2>/dev/null \
|
||||
|| curl -fsSL "$BASE/scenes_knowledge_graphs/${FILM}-${n}.json" \
|
||||
-o "$DEST/scenes/${FILM}-${n}.json" 2>/dev/null \
|
||||
|| rm -f "$DEST/scenes/${FILM}-${n}.json"
|
||||
done
|
||||
|
||||
echo "[dvu] done:"
|
||||
echo " mugshots: $(ls "$DEST/images" 2>/dev/null | wc -l)"
|
||||
echo " scenes: $(ls "$DEST/scenes" 2>/dev/null | wc -l)"
|
||||
echo " csv: $([ -f "$DEST/${FILM}.csv" ] && echo yes || echo no)"
|
||||
@@ -77,12 +77,7 @@ def main():
|
||||
if missing > 0:
|
||||
print(f"[warn] {missing} cast member(s) not present in gallery (not yet embedded)", file=sys.stderr)
|
||||
|
||||
# TRACES: GR-004 | SR-001
|
||||
# a filtered gallery holds the SAME vectors as its
|
||||
# source, so it inherits the source's binding. Dropping the stamp here would
|
||||
# silently launder a stamped gallery into an unstamped one.
|
||||
save_gallery_hdf5({"actors": actors}, Path(args.output),
|
||||
gallery.get("embedder"))
|
||||
save_gallery_hdf5({"actors": actors}, Path(args.output))
|
||||
print(f"Saved {len(actors)} actor(s) to {args.output}", file=sys.stderr)
|
||||
|
||||
|
||||
|
||||
@@ -1,67 +0,0 @@
|
||||
#!/usr/bin/env bash
|
||||
# make_fixtures.sh — regenerate the committed replay fixtures.
|
||||
#
|
||||
# TRACES: VR-001 | PR-002
|
||||
#
|
||||
# CI never calls a model (see docs/requirements.md, "CI never calls a model"):
|
||||
# the embedder is impractical on the N100 CI host, so inference happens HERE, on
|
||||
# a machine with a GPU, and CI consumes the HDF5 dumps as data. Everything
|
||||
# downstream of embedding — tracking, presence windows, belief accumulation,
|
||||
# expansion — is cheap CPU maths and replays from these files.
|
||||
#
|
||||
# Reproducibility is a requirement, not a nicety. A fixture whose provenance is
|
||||
# unknown is worse than no fixture, because it will be trusted. Every parameter
|
||||
# that affects the output is pinned below rather than left to a default, and the
|
||||
# dumps carry the embedder identity and SHA-256 (GR-004) so a replay cannot be
|
||||
# silently scored against the wrong gallery.
|
||||
#
|
||||
# These are byte-reproducible only because node outputs block rather than drop
|
||||
# on a full channel (AR-004). Before that fix the same command produced
|
||||
# different dumps run to run, since what got dropped depended on timing.
|
||||
#
|
||||
# Source: hero/ — SuperHero, from the TRECVID DVU development set. Chosen over
|
||||
# SuperHero on face scale: Bali reference crops had a median detected face of
|
||||
# 27 px against a 69 px maximum, so every reference was upscaled far past what
|
||||
# the embedder was trained for. SuperHero is 69 px median, 241 px max. That matters: derived
|
||||
# fixtures can be committed, where anything cut from a copyrighted title could
|
||||
# not live in the repository at all.
|
||||
set -euo pipefail
|
||||
|
||||
REPO="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
|
||||
CLIPS="${CLIPS:-$REPO/../hero}"
|
||||
GALLERY="${GALLERY:-$REPO/gallery_lvface.h5}"
|
||||
BIN="${BIN:-$REPO/build/scene_analyze}"
|
||||
OUT="$REPO/tests/fixtures/dumps"
|
||||
|
||||
# Pinned. Changing either invalidates every committed fixture.
|
||||
# fps 5 — 1 fps over a 77 s clip is 77 frames, too thin to exercise an
|
||||
# extinction window measured in tens of seconds.
|
||||
# min-face — 32 px. This is a *fixture* setting, deliberately below AR-002's
|
||||
# production floor of 40 px (VR-013, measured end to end): the
|
||||
# corpus is 480x360, where faces run 40-80 px, so pinning at 40
|
||||
# would thin the dumps for reasons unrelated to what they test.
|
||||
# 32 px is where VR-005 still shows 98.1% TPI, so the faces kept
|
||||
# are identifiable; it is not the threshold the pipeline ships.
|
||||
FPS=5
|
||||
MIN_FACE_PX=32
|
||||
|
||||
[[ -x "$BIN" ]] || { echo "no scene_analyze at $BIN (set BIN=)" >&2; exit 1; }
|
||||
[[ -f "$GALLERY" ]] || { echo "no gallery at $GALLERY (set GALLERY=)" >&2; exit 1; }
|
||||
[[ -d "$CLIPS" ]] || { echo "no clips at $CLIPS (set CLIPS=)" >&2; exit 1; }
|
||||
|
||||
mkdir -p "$OUT"
|
||||
|
||||
for clip in "$CLIPS"/SuperHero-*.webm; do
|
||||
n="$(basename "$clip" .webm)"; n="${n##*-}"
|
||||
echo "── superhero_$n"
|
||||
"$BIN" --movie "$clip" --gallery "$GALLERY" \
|
||||
--fps "$FPS" --min-face-px "$MIN_FACE_PX" \
|
||||
--dump-embeddings "$OUT/superhero_$n.h5" \
|
||||
--output /dev/null 2>&1 | grep -E "wrote|dropped" || true
|
||||
done
|
||||
|
||||
echo
|
||||
echo "Regenerated in $OUT — verify the diff is empty if nothing upstream changed."
|
||||
echo "A non-empty diff means detection, alignment or embedding moved. That is"
|
||||
echo "either a regression or a deliberate change, and either way the golden"
|
||||
echo "outputs derived from these fixtures need reviewing."
|
||||
@@ -36,9 +36,8 @@ import time
|
||||
from pathlib import Path
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parent))
|
||||
from sae_embed_loader import load_embedder, resolve_arcface
|
||||
from sae_gallery import (download_images, embedder_stamp, save_gallery,
|
||||
wikidata_image_urls)
|
||||
from sae_embed_loader import load_embedder
|
||||
from sae_gallery import download_images, save_gallery, wikidata_image_urls
|
||||
from sae_tmdb import TMDB_IMG, tmdb_get, tmdb_id_from_imdb
|
||||
|
||||
|
||||
@@ -178,12 +177,7 @@ def main():
|
||||
output = Path(args.output)
|
||||
image_root = Path(args.image_dir) if args.image_dir else output.parent / "images"
|
||||
|
||||
# TRACES: GR-004 | SR-001
|
||||
# stamp with the model actually loaded, resolved
|
||||
# through the same helper load_embedder uses so the two cannot diverge.
|
||||
arcface_path = resolve_arcface(args.models_dir, args.arcface)
|
||||
embedder = load_embedder(args.build_dir, args.models_dir, args.arcface)
|
||||
stamp = embedder_stamp(arcface_path)
|
||||
|
||||
# Resolve movie ID
|
||||
movie_id = args.movie_id
|
||||
@@ -209,7 +203,7 @@ def main():
|
||||
if n_actors == 0:
|
||||
sys.exit("No actors could be processed — check models and images.")
|
||||
|
||||
save_gallery(gallery, missing, output, embedder=stamp)
|
||||
save_gallery(gallery, missing, output)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
|
||||
@@ -1,8 +1,6 @@
|
||||
#!/usr/bin/env python3
|
||||
"""make_jellyfin_gallery.py — build a gallery.h5 spanning an entire Jellyfin library.
|
||||
|
||||
TRACES: GR-001, GR-002 | SR-001, SR-005
|
||||
|
||||
Queries the Jellyfin API for every Movie/Series, collects the unique cast
|
||||
across the whole library, downloads each actor's headshot directly from
|
||||
Jellyfin (no TMDB key needed), embeds them with the sae_embed module (SCRFD +
|
||||
@@ -53,9 +51,8 @@ import requests
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parent))
|
||||
import sae_env # noqa: F401 — loads .env into os.environ on import
|
||||
from sae_embed_loader import load_embedder, resolve_arcface
|
||||
from sae_gallery import (download_image, download_images, embedder_stamp,
|
||||
enforce_embedder_stamp, load_gallery_hdf5,
|
||||
from sae_embed_loader import load_embedder
|
||||
from sae_gallery import (download_image, download_images, load_gallery_hdf5,
|
||||
save_gallery, wikidata_image_urls)
|
||||
from sae_jellyfin import actor_jellyfin_id, jf_get, normalize_jellyfin_url
|
||||
from sae_tmdb import (
|
||||
@@ -445,21 +442,11 @@ def main():
|
||||
image_root = Path(args.image_dir) if args.image_dir else output.parent / "images"
|
||||
item_types = [t.strip() for t in args.item_types.split(",") if t.strip()]
|
||||
|
||||
# TRACES: GR-004 | SR-001
|
||||
arcface_path = resolve_arcface(args.models_dir, args.arcface)
|
||||
embedder = load_embedder(args.build_dir, args.models_dir, args.arcface)
|
||||
stamp = embedder_stamp(arcface_path)
|
||||
|
||||
existing_actors = {}
|
||||
if args.merge and output.is_file():
|
||||
existing = load_gallery_hdf5(output)
|
||||
# TRACES: GR-004 | SR-001
|
||||
# --merge keeps the existing actors' vectors and
|
||||
# embeds the new ones with THIS model. If they disagree, the result is one
|
||||
# gallery holding two incompatible embedding spaces, which is worse than a
|
||||
# mismatched gallery: no later check can separate them again.
|
||||
enforce_embedder_stamp(existing.get("embedder"), stamp, str(output),
|
||||
arcface_path)
|
||||
for actor in existing.get("actors", []):
|
||||
pid = actor_jellyfin_id(actor)
|
||||
if pid:
|
||||
@@ -488,7 +475,7 @@ def main():
|
||||
if n_actors == 0:
|
||||
sys.exit("No actors could be processed — check Jellyfin URL/API key and models.")
|
||||
|
||||
save_gallery(gallery, missing, output, embedder=stamp)
|
||||
save_gallery(gallery, missing, output)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
|
||||
@@ -22,8 +22,8 @@ from pathlib import Path
|
||||
import numpy as np
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parent))
|
||||
from sae_embed_loader import load_embedder, resolve_arcface
|
||||
from sae_gallery import load_gallery_hdf5, verify_gallery_stamp
|
||||
from sae_embed_loader import load_embedder
|
||||
from sae_gallery import load_gallery_hdf5
|
||||
|
||||
|
||||
def load_gallery(path: str) -> dict[str, dict]:
|
||||
@@ -62,12 +62,6 @@ def main():
|
||||
args = p.parse_args()
|
||||
|
||||
embedder = load_embedder(args.build_dir, args.models_dir, args.arcface)
|
||||
# TRACES: GR-004 | SR-001
|
||||
# match() below is a bare dot product against the
|
||||
# gallery's vectors; if the gallery came from another model those numbers are
|
||||
# noise wearing a similarity's clothes.
|
||||
verify_gallery_stamp(args.gallery,
|
||||
resolve_arcface(args.models_dir, args.arcface))
|
||||
|
||||
gallery = load_gallery(args.gallery)
|
||||
print(f"[eval] gallery: {len(gallery)} actors", file=sys.stderr)
|
||||
|
||||
+6
-102
@@ -19,33 +19,10 @@ variable-length HDF5 types and reads straight into numpy.
|
||||
/ (root)
|
||||
attrs:
|
||||
schema_version : int = 1
|
||||
movie : str (source video path)
|
||||
sample_fps : float
|
||||
embed_dim : int = 512
|
||||
|
||||
# ── what produced the vectors (GR-004) ──────────────────────────────────
|
||||
embedder_model : str basename of the embedding model
|
||||
embedder_sha256: str SHA-256 of that model file
|
||||
|
||||
# ── what produced the faces (VR-010) ────────────────────────────────────
|
||||
detector_model : str basename of the detector .onnx
|
||||
detector_conf : float score floor a detection had to clear to be dumped
|
||||
detector_nms : float NMS IoU threshold
|
||||
min_face_px : float minimum box side, ORIGINAL-resolution px (AR-002)
|
||||
max_faces : int per-frame cap; 0 = uncapped, the default (AR-003)
|
||||
|
||||
# ── what produced the frames (VR-010) ───────────────────────────────────
|
||||
movie : str source video path
|
||||
sample_fps : float frames analysed per second of movie
|
||||
start_sec : float seek point
|
||||
end_sec : float stop point; -1 = end of file
|
||||
cut_threshold : float histogram correlation below which is_cut fires
|
||||
dense_scale : float decoded-frame downscale in dense mode; 1 = off
|
||||
bbox_upscale : float multiply faces/bbox and faces/landmarks by this to
|
||||
reach original video pixels; 1 when dense_scale is 1
|
||||
scene_detect : uint8 0/1 — was TransNetV2 running at all (see below)
|
||||
|
||||
# ── downstream setting recorded for comparability (VR-010) ──────────────
|
||||
track_assoc_min_prob : float the run's tracker admission probability
|
||||
|
||||
frames/ group — one row per sampled frame
|
||||
timestamp_sec : float64 [F]
|
||||
frame_idx : int64 [F]
|
||||
@@ -56,91 +33,18 @@ variable-length HDF5 types and reads straight into numpy.
|
||||
|
||||
faces/ group — one row per detected face, concatenated
|
||||
embedding : float32 [N, 512] L2-normalised ArcFace embedding
|
||||
bbox : float32 [N, 4] x, y, w, h in DECODED-frame pixels
|
||||
landmarks : float32 [N, 10] 5 (x,y) pairs, SCRFD/ArcFace order,
|
||||
same space as bbox
|
||||
bbox : float32 [N, 4] x, y, w, h in original video pixels
|
||||
landmarks : float32 [N, 10] 5 (x,y) pairs, SCRFD/ArcFace order
|
||||
confidence : float32 [N] detector confidence
|
||||
```
|
||||
|
||||
`F` = number of sampled frames, `N` = total faces (= sum of face_count).
|
||||
Frame *i*'s faces are `faces/*[ face_offset[i] : face_offset[i]+face_count[i] ]`.
|
||||
|
||||
## Provenance (VR-010)
|
||||
|
||||
The attributes above are not documentation; they are the only thing that makes a
|
||||
dump interpretable. Two dumps of the same film at `detector_conf` 0.5 and 0.7, or
|
||||
at `dense_scale` 1.0 and 0.5, or with scene detection on and off, are different
|
||||
measurements of different things — and they are byte-shaped identically. Without
|
||||
provenance a consumer that mixes them gets a plausible number from an incoherent
|
||||
input, and nothing anywhere reports a problem.
|
||||
|
||||
**`scene_detect` is the one that cannot be inferred.** `is_scene_boundary` is
|
||||
all-zero both when TransNetV2 found no boundaries in the clip and when it was
|
||||
never enabled, and those mean opposite things: the first says *this footage has
|
||||
no shot changes*, the second says *nobody looked*. A consumer that reads the
|
||||
array alone must guess. The flag is what removes the guess. (`dump_embeddings`
|
||||
has no `--scene-detect`, so every dump it writes records `false` — which is
|
||||
exactly the fact the committed fixtures needed to state.)
|
||||
|
||||
**`bbox_upscale` is recorded, not applied.** See the coordinate-space note below.
|
||||
|
||||
Reading is by name with a default or an existence check on **both** sides —
|
||||
`replay.py` (`f.attrs.get(...)`) and `read_dump_provenance()` in
|
||||
`src/nodes/embedding_dump_node.hpp` (`attrExists`). So the attributes are
|
||||
additive and `schema_version` stays 1: a pre-VR-010 dump still loads, and a
|
||||
post-VR-010 dump still reads on old code.
|
||||
|
||||
A missing attribute means **unknown**, never a default value. Substituting
|
||||
`detector_conf = 0.5` for a dump that does not say so manufactures the provenance
|
||||
the requirement exists to prevent — per `docs/requirements.md`, *"a fixture whose
|
||||
provenance is unknown is worse than no fixture, because it will be trusted."*
|
||||
The committed `tests/fixtures/dumps/*.h5` predate VR-010 and carry none of these
|
||||
attributes; re-dump to bind them, as with GR-004.
|
||||
|
||||
## Model binding (GR-004)
|
||||
|
||||
`embedder_model` / `embedder_sha256` record which embedder produced every vector
|
||||
in `faces/embedding`. A replay has no live embedder, so **the dump is the embedder
|
||||
as far as the gallery is concerned**: `replay.py` checks these two attributes
|
||||
against the gallery's own `/embedder` stamp and refuses to run on a mismatch,
|
||||
naming both sides. Cross-model cosines are meaningless but look plausible.
|
||||
|
||||
The attributes are additive, not a format break — `schema_version` stays 1. Dumps
|
||||
written before GR-004 simply lack them, which reports as *unverifiable* (a loud
|
||||
warning, or a hard error under `SAE_REQUIRE_GALLERY_STAMP=1`) rather than as a
|
||||
pass. Re-dump to bind an old dump; there is no in-place migration, because unlike
|
||||
a gallery nobody can assert after the fact which model produced a vector.
|
||||
|
||||
## Coordinate space — `bbox`, `landmarks`, `bbox_upscale`
|
||||
|
||||
`bbox` and `landmarks` are in **decoded-frame pixels**: exactly the numbers SCRFD
|
||||
produced, untransformed. To reach original video pixels, multiply by
|
||||
`bbox_upscale`. With `dense_scale == 1` (the default, and every committed
|
||||
fixture) `bbox_upscale == 1` and the two spaces coincide.
|
||||
|
||||
> Earlier revisions of this document claimed the upscale was applied at dump time.
|
||||
> It never was. `embedding_dump_node.hpp` writes `f.bbox` raw; the upscale lives
|
||||
> in `identity_matcher_node.hpp`, which is *downstream* of the dump tap. The
|
||||
> claim was harmless only because `dense_scale` was 1 in practice.
|
||||
|
||||
The fix is to record the factor rather than to apply it, because the dump's whole
|
||||
contract is to be a **faithful tap** at the `EmbeddedSceneFrame` channel — VR-002
|
||||
requires replay to drive the real nodes, and a replay is only equivalent to the
|
||||
live run if the tracker is fed the geometry the live tracker saw. Rescaling at
|
||||
the tap would break that: the replayed tracker would associate on boxes the live
|
||||
one never received. Two further reasons:
|
||||
|
||||
- The matcher's upscale is applied to `bbox` **only**, not to `landmarks`.
|
||||
Pre-multiplying at the tap would leave the two arrays in different coordinate
|
||||
spaces inside one file — a worse trap than the one being fixed.
|
||||
- Pre-multiplying is lossy in the sense that matters: a dump that had been
|
||||
upscaled would be indistinguishable from one taken at `dense_scale == 1`, so
|
||||
you would have to record `bbox_upscale` anyway to know which you were holding.
|
||||
|
||||
## Invariants
|
||||
- `embedding` rows are unit-norm (cosine == dot product against the gallery).
|
||||
- `face_offset[0] == 0`; `face_offset[i+1] == face_offset[i] + face_count[i]`.
|
||||
- `bbox` and `landmarks` share one coordinate space; `bbox_upscale` maps both to
|
||||
original resolution (see above).
|
||||
- `bbox` is already mapped to original resolution (bbox_upscale applied at dump time),
|
||||
matching what the identity matcher would emit.
|
||||
- A frame with no faces has `face_count == 0` (still gets a row, so timestamps stay dense).
|
||||
- EOF sentinel frames are NOT written.
|
||||
|
||||
@@ -35,9 +35,7 @@ REPO = Path(__file__).resolve().parent.parent.parent
|
||||
sys.path.insert(0, str(REPO / "scripts"))
|
||||
import sae_env # noqa: E402 loads .env
|
||||
from sae_tmdb import tmdb_get, tmdb_person_for_imdb, TMDB_IMG # noqa: E402
|
||||
from sae_embed_loader import resolve_arcface # noqa: E402
|
||||
from sae_gallery import (download_images, embedder_stamp, # noqa: E402
|
||||
enforce_embedder_stamp, wikidata_image_urls)
|
||||
from sae_gallery import download_images, wikidata_image_urls # noqa: E402
|
||||
from sae_embed_loader import load_embedder # noqa: E402
|
||||
|
||||
|
||||
@@ -59,7 +57,6 @@ def fetch(missing_path, out_path, token, build_dir, models_dir, arcface,
|
||||
src = "TMDB + Wikidata fallback" if use_wikidata else "TMDB"
|
||||
print(f"[fetch] {len(missing)} missing actors to resolve via {src}", file=sys.stderr)
|
||||
embedder = load_embedder(build_dir, models_dir, arcface)
|
||||
stamp = embedder_stamp(resolve_arcface(models_dir, arcface)) # TRACES: GR-004 | SR-001
|
||||
|
||||
img_root = Path(tempfile.mkdtemp(prefix="missing_gallery_"))
|
||||
actors = []
|
||||
@@ -106,10 +103,7 @@ def fetch(missing_path, out_path, token, build_dir, models_dir, arcface,
|
||||
f"(wiki={n_via_wikidata}) no_tmdb={n_no_tmdb} no_img={n_no_img} "
|
||||
f"no_face={n_no_face}", file=sys.stderr)
|
||||
|
||||
# TRACES: GR-004 | SR-001
|
||||
# the legacy JSON gallery carries the same stamp as
|
||||
# the HDF5 one; src/gallery/gallery_store.cpp reads it from either.
|
||||
Path(out_path).write_text(json.dumps({"embedder": stamp, "actors": actors}, indent=2))
|
||||
Path(out_path).write_text(json.dumps({"actors": actors}, indent=2))
|
||||
n_emb = sum(len(a["embeddings"]) for a in actors)
|
||||
print(f"\n[fetch] recovered {n_resolved}/{len(missing)} actors "
|
||||
f"({n_via_wikidata} via Wikidata), {n_emb} embeddings → {out_path}",
|
||||
@@ -121,13 +115,6 @@ def fetch(missing_path, out_path, token, build_dir, models_dir, arcface,
|
||||
def merge(base_path, add_path, out_path):
|
||||
base = json.loads(Path(base_path).read_text())
|
||||
add = json.loads(Path(add_path).read_text())
|
||||
# TRACES: GR-004 | SR-001
|
||||
# merging two galleries from different models makes
|
||||
# ONE file containing two incompatible embedding spaces. Nothing downstream can
|
||||
# ever untangle that, so this is the one place the check must run before, not
|
||||
# after, the write.
|
||||
enforce_embedder_stamp(base.get("embedder"), add.get("embedder"),
|
||||
str(base_path), str(add_path))
|
||||
have = {a.get("imdb_id") for a in base["actors"] if a.get("imdb_id")}
|
||||
added = [a for a in add["actors"] if a.get("imdb_id") not in have]
|
||||
base["actors"].extend(added)
|
||||
|
||||
@@ -34,7 +34,6 @@ from scipy.optimize import differential_evolution
|
||||
REPO = Path(__file__).resolve().parent.parent.parent
|
||||
sys.path.insert(0, str(REPO / "scripts" / "optimizer"))
|
||||
sys.path.insert(0, str(REPO / "scripts" / "validation"))
|
||||
sys.path.insert(0, str(REPO / "scripts"))
|
||||
|
||||
import json as _json
|
||||
import os
|
||||
@@ -57,8 +56,6 @@ DE_WORKERS = int(os.environ.get("DE_WORKERS", "1"))
|
||||
|
||||
from second_score import score_seconds # noqa: E402 uniform per-second TPI/FPI scoring
|
||||
from sample_eval import load_gallery_keys # noqa: E402
|
||||
from replay import dump_embedder_stamp # noqa: E402
|
||||
from sae_stamp import EmbedderMismatch, verify_gallery_stamp # noqa: E402
|
||||
|
||||
_GAL_KEYS: dict = {} # gallery path → key set (fair-recall FN mask), loaded once
|
||||
_REPLAY_TIMEOUT = 45 # seconds per film; a wedged replay is killed, not left to hang
|
||||
@@ -183,15 +180,7 @@ def main():
|
||||
p.add_argument("--seed", type=int, default=0)
|
||||
p.add_argument("--trajectory", help="write every evaluation here (JSON lines)")
|
||||
p.add_argument("--out", help="write best config + metrics")
|
||||
# TRACES: GR-004 | SR-001
|
||||
p.add_argument("--require-gallery-stamp", action="store_true",
|
||||
help="unprovable gallery/dump model binding is a hard error, "
|
||||
"not a warning (also via SAE_REQUIRE_GALLERY_STAMP=1)")
|
||||
args = p.parse_args()
|
||||
if args.require_gallery_stamp:
|
||||
# Set the env var rather than threading a flag through cfg: replays run as
|
||||
# subprocesses and inherit it, so strictness cannot be lost in the handoff.
|
||||
os.environ["SAE_REQUIRE_GALLERY_STAMP"] = "1"
|
||||
|
||||
films = json.loads(Path(args.manifest).read_text())
|
||||
for f in films:
|
||||
@@ -199,19 +188,6 @@ def main():
|
||||
if not Path(f["dump"]).exists():
|
||||
sys.exit(f"[opt] missing dump for {f['name']}: {f['dump']}")
|
||||
|
||||
# TRACES: GR-004 | SR-001
|
||||
# every (dump, gallery) pair is checked ONCE here,
|
||||
# before the first evaluation. A DE sweep is thousands of replays; discovering
|
||||
# a cross-model pair at the end (or never) means every number it produced was
|
||||
# noise. Each replay subprocess re-checks its own pair anyway.
|
||||
for f in films:
|
||||
try:
|
||||
verify_gallery_stamp(f["gallery"], stamp=dump_embedder_stamp(f["dump"]),
|
||||
embedder_desc=f"embedding dump {Path(f['dump']).name}",
|
||||
require_stamp=args.require_gallery_stamp)
|
||||
except EmbedderMismatch as e:
|
||||
sys.exit(f"[opt] {f['name']}: {e}")
|
||||
|
||||
names, bounds = [], []
|
||||
int_knobs = {"track_max_frames_missing", "cut_inactive_max_frames"}
|
||||
for spec in args.params:
|
||||
|
||||
@@ -27,9 +27,8 @@ from pathlib import Path
|
||||
|
||||
REPO = Path(__file__).resolve().parent.parent.parent
|
||||
sys.path.insert(0, str(REPO / "scripts"))
|
||||
from sae_embed_loader import load_embedder, resolve_arcface # noqa: E402
|
||||
from sae_gallery import (embedder_stamp, load_gallery_hdf5, # noqa: E402
|
||||
save_gallery_hdf5)
|
||||
from sae_embed_loader import load_embedder # noqa: E402
|
||||
from sae_gallery import load_gallery_hdf5, save_gallery_hdf5 # noqa: E402
|
||||
|
||||
|
||||
def find_dir(images_root: Path, jellyfin_id: str, name: str) -> Path | None:
|
||||
@@ -59,12 +58,6 @@ def main():
|
||||
ref = load_gallery_hdf5(Path(args.ref))
|
||||
images_root = Path(args.images)
|
||||
embedder = load_embedder(args.build_dir, args.models_dir, args.arcface)
|
||||
# TRACES: GR-004 | SR-001
|
||||
# this script exists to produce a gallery in a
|
||||
# DIFFERENT model's space from the reference. The output must therefore never
|
||||
# inherit the reference's stamp; it carries the stamp of --arcface, which is
|
||||
# the whole point of the bake-off being safe to run.
|
||||
stamp = embedder_stamp(resolve_arcface(args.models_dir, args.arcface))
|
||||
|
||||
out_actors = []
|
||||
n_ok = n_nodir = n_noemb = 0
|
||||
@@ -91,7 +84,7 @@ def main():
|
||||
print(f" [{i}/{total}] ok={n_ok} no_dir={n_nodir} no_emb={n_noemb}",
|
||||
file=sys.stderr)
|
||||
|
||||
save_gallery_hdf5({"actors": out_actors}, Path(args.out), stamp)
|
||||
save_gallery_hdf5({"actors": out_actors}, Path(args.out))
|
||||
n_emb = sum(len(a["embeddings"]) for a in out_actors)
|
||||
print(f"[reembed] {Path(args.arcface).stem}: {n_ok}/{total} actors, {n_emb} embeddings "
|
||||
f"→ {args.out}", file=sys.stderr)
|
||||
|
||||
@@ -2,8 +2,6 @@
|
||||
"""
|
||||
replay.py — replay a dumped embedding HDF5 through the real KPN downstream nodes.
|
||||
|
||||
TRACES: VR-002 | PR-002
|
||||
|
||||
Reads an embedding dump (scripts/optimizer/SCHEMA.md), feeds each frame as an
|
||||
EmbeddedSceneFrame into a Python-assembled KPN network wiring the *real* C++
|
||||
face_tracker → identity_matcher → scene_tracker, and returns the same presence-window
|
||||
@@ -27,22 +25,6 @@ import h5py
|
||||
import numpy as np
|
||||
|
||||
REPO = Path(__file__).resolve().parent.parent.parent
|
||||
sys.path.insert(0, str(REPO / "scripts"))
|
||||
from sae_stamp import verify_gallery_stamp # noqa: E402
|
||||
|
||||
|
||||
def dump_embedder_stamp(dump_path: str) -> dict:
|
||||
"""The GR-004 embedder stamp recorded in an embedding dump.
|
||||
|
||||
A replay has no live embedder — the dump IS the embedder as far as the gallery
|
||||
is concerned, so the dump's stamp is what the gallery must be checked against.
|
||||
Dumps written before GR-004 have no attributes and yield an empty stamp, which
|
||||
the check reports as unverifiable rather than silently accepting."""
|
||||
with h5py.File(dump_path, "r") as f:
|
||||
name = f.attrs.get("embedder_model", "")
|
||||
sha = f.attrs.get("embedder_sha256", "")
|
||||
dec = lambda v: v.decode() if isinstance(v, bytes) else ("" if v is None else str(v))
|
||||
return {"model_name": dec(name), "model_sha256": dec(sha), "embed_dim": 512}
|
||||
|
||||
|
||||
def load_frames(dump_path: str, min_conf: float = 0.0):
|
||||
@@ -110,16 +92,6 @@ def replay(dump_path: str, gallery: str, cfg: dict, build_dir: str, stop: bool =
|
||||
sys.path.insert(0, build_dir)
|
||||
import sae_kpn
|
||||
|
||||
# TRACES: GR-004 | SR-001
|
||||
# checked here, before any network is built, so a
|
||||
# cross-model replay dies with one readable error instead of producing a
|
||||
# plausible-looking score. add_identity_matcher re-checks it C++-side below;
|
||||
# that is the backstop for any other caller of the binding.
|
||||
stamp = dump_embedder_stamp(dump_path)
|
||||
verify_gallery_stamp(gallery, stamp=stamp,
|
||||
embedder_desc=f"embedding dump {Path(dump_path).name}",
|
||||
require_stamp=bool(cfg.get("require_gallery_stamp", False)))
|
||||
|
||||
frames, movie, fps = load_frames(dump_path, min_conf=float(cfg.get("detector_conf", 0.0)))
|
||||
|
||||
net = sae_kpn.Network()
|
||||
@@ -150,8 +122,7 @@ def replay(dump_path: str, gallery: str, cfg: dict, build_dir: str, stop: bool =
|
||||
cap = len(frames) * 2 + 64
|
||||
sae_kpn.add_node_python(net, "replay", source, [], ["EmbeddedSceneFrame"], cap)
|
||||
sae_kpn.add_face_tracker(net, "tracker", cfg, cap)
|
||||
sae_kpn.add_identity_matcher(net, "matcher", gallery, cfg, cap,
|
||||
stamp["model_name"], stamp["model_sha256"])
|
||||
sae_kpn.add_identity_matcher(net, "matcher", gallery, cfg, cap)
|
||||
sae_kpn.add_scene_tracker(net, "scene", cfg, cap)
|
||||
net.connect("replay", 0, "tracker", 0)
|
||||
net.connect("tracker", 0, "matcher", 0)
|
||||
@@ -250,18 +221,11 @@ def main():
|
||||
# per-film gallery expansion: promotes pose-varied views of confidently-identified
|
||||
# actors into an in-memory annex, recovering ~+4 recall at no precision cost.
|
||||
p.add_argument("--expand-gallery", action="store_true")
|
||||
# TRACES: GR-004 | SR-001
|
||||
# promote an unprovable gallery/dump binding from a
|
||||
# loud warning to a hard error. Measurement sweeps should set this (or
|
||||
# SAE_REQUIRE_GALLERY_STAMP=1) so no number comes from an unbound pair.
|
||||
p.add_argument("--require-gallery-stamp", action="store_true")
|
||||
args = p.parse_args()
|
||||
|
||||
cfg = {k: getattr(args, k) for k in CFG_KEYS if getattr(args, k) is not None}
|
||||
if args.expand_gallery:
|
||||
cfg["expand_gallery"] = True
|
||||
if args.require_gallery_stamp:
|
||||
cfg["require_gallery_stamp"] = True
|
||||
# stop=True: PyNode::stop() sets stop_flag_ before joining, so the source
|
||||
# thread's run_loop actually exits. stop=False skips that, leaving stop_flag_
|
||||
# false forever — the PyNode destructor's jthread.join() then blocks forever
|
||||
|
||||
@@ -2,8 +2,6 @@
|
||||
"""
|
||||
second_score.py — uniform per-second agreement with X-Ray.
|
||||
|
||||
TRACES: VR-003 | PR-002
|
||||
|
||||
Unlike scene_score.py (which unions our detections over a whole X-Ray scene), this
|
||||
samples EVERY SECOND of the film and asks: at second t, do we name the same actors
|
||||
X-Ray says are on screen?
|
||||
|
||||
@@ -1,8 +1,6 @@
|
||||
#!/usr/bin/env python3
|
||||
"""run_from_jellyfin.py — resolve a Jellyfin title to its media file and run scene_analyze.
|
||||
|
||||
TRACES: IR-006 | SR-001
|
||||
|
||||
Looks up a Movie/Episode in Jellyfin, reads its on-disk Path (Jellyfin and this
|
||||
tool must share the same media mount), filters the gallery down to that
|
||||
title's credited cast (via filter_gallery's logic, fewer look-alike
|
||||
|
||||
@@ -3,29 +3,11 @@
|
||||
sae_embed.FaceEmbedder loads both ONNX sessions once and exposes an
|
||||
embed(path) -> FaceResult method, avoiding the per-process model reload cost
|
||||
of spawning the embed_faces CLI binary for every image.
|
||||
|
||||
resolve_arcface() exposes the same default-resolution logic load_embedder uses,
|
||||
so a caller can stamp the gallery it is about to write with the model that
|
||||
actually produced its embeddings (GR-004) — the resolved path, not the CLI
|
||||
argument, which is often None.
|
||||
"""
|
||||
|
||||
import sys
|
||||
import os
|
||||
from pathlib import Path
|
||||
|
||||
DEFAULT_ARCFACE = "arcface_w600k_r50.onnx"
|
||||
|
||||
|
||||
def resolve_arcface(models_dir: str, arcface: str | None = None) -> str:
|
||||
"""The ArcFace/LVFace ONNX path load_embedder would use for these arguments.
|
||||
|
||||
TRACES: GR-004 | SR-001
|
||||
|
||||
Single source of truth for "which model is this", so the stamp written into
|
||||
a gallery can never drift from the model loaded."""
|
||||
return arcface if arcface else str(Path(models_dir) / DEFAULT_ARCFACE)
|
||||
|
||||
|
||||
def load_embedder(build_dir: str, models_dir: str, arcface: str | None = None,
|
||||
conf: float = 0.5, nms: float = 0.4, max_side: int = 500):
|
||||
@@ -46,27 +28,19 @@ def load_embedder(build_dir: str, models_dir: str, arcface: str | None = None,
|
||||
|
||||
models_path = Path(models_dir)
|
||||
detector_path = str(models_path / "scrfd_500m_bnkps.onnx")
|
||||
arcface_path = resolve_arcface(models_dir, arcface)
|
||||
arcface_path = arcface if arcface else str(models_path / "arcface_w600k_r50.onnx")
|
||||
for model, name in [(detector_path, "SCRFD"), (arcface_path, "ArcFace")]:
|
||||
if not Path(model).is_file():
|
||||
sys.exit(f"{name} model not found: {model}\nRun: bash scripts/download_models.sh")
|
||||
|
||||
# A TRT-backend build cannot load .onnx; it needs pre-built engines from
|
||||
# scripts/build_trt_engines.sh.
|
||||
#
|
||||
# These are passed only on request. The old comment here claimed they were
|
||||
# "ignored by ORT" — they are not. The ORT backend treats an engine path as
|
||||
# an instruction and raises, which is the right behaviour (silently ignoring
|
||||
# a requested engine would be worse), but it meant that merely HAVING a
|
||||
# populated trt_cache/ broke every ORT gallery build in the repo, with an
|
||||
# error naming a flag the caller never set.
|
||||
use_engines = os.environ.get("SAE_USE_TRT_ENGINES", "") not in ("", "0", "false")
|
||||
# scripts/build_trt_engines.sh. Pass them when present (ignored by ORT).
|
||||
trt = Path(models_path).parent / "trt_cache"
|
||||
det_engine = trt / "scrfd.scrfd_500m_bnkps.640.fp16.engine"
|
||||
arc_engine = trt / f"arcface.{Path(arcface_path).stem}.b4.fp16.engine"
|
||||
|
||||
return sae_embed.FaceEmbedder(
|
||||
detector_path, arcface_path, conf, nms, max_side,
|
||||
str(det_engine) if (use_engines and det_engine.is_file()) else "",
|
||||
str(arc_engine) if (use_engines and arc_engine.is_file()) else "",
|
||||
str(det_engine) if det_engine.is_file() else "",
|
||||
str(arc_engine) if arc_engine.is_file() else "",
|
||||
)
|
||||
|
||||
+10
-62
@@ -6,13 +6,9 @@ make_jellyfin_gallery.download_urls + download_person_images) and the duplicated
|
||||
|
||||
Galleries are written directly as HDF5 — never JSON. Same layout the C++ side
|
||||
reads/writes (src/gallery/gallery_store.cpp): flat [N,512] embeddings + per-actor
|
||||
offset/count, parallel imdb_id/tmdb_id/jellyfin_id/name string arrays, a
|
||||
per-embedding-row source_images array, and an /embedder group carrying the
|
||||
GR-004 model binding. calibration is left absent (calib_hash=0); the C++
|
||||
identity_matcher fits and writes it back into the file on first use.
|
||||
|
||||
The GR-004 embedder stamp written into that /embedder group lives in sae_stamp
|
||||
and is re-exported below, so existing callers keep importing it from here.
|
||||
offset/count, parallel imdb_id/tmdb_id/jellyfin_id/name string arrays, and a
|
||||
per-embedding-row source_images array. calibration is left absent (calib_hash=0);
|
||||
the C++ identity_matcher fits and writes it back into the file on first use.
|
||||
"""
|
||||
|
||||
import io
|
||||
@@ -105,36 +101,11 @@ def download_images(urls: list[str], dest_dir: Path, n: int,
|
||||
return paths
|
||||
|
||||
|
||||
# ── GR-004: gallery ↔ embedder binding ───────────────────────────────────────
|
||||
# Implemented in sae_stamp (kept dependency-light so the optimizer's replay
|
||||
# subprocesses can import it without pulling requests/Pillow); re-exported here
|
||||
# because the gallery writers and every existing caller reach for it via this
|
||||
# module. See src/gallery/embedder_stamp.hpp for the C++ twin and the rationale.
|
||||
from sae_stamp import ( # noqa: F401
|
||||
EmbedderMismatch,
|
||||
check_embedder_stamp,
|
||||
describe_stamp,
|
||||
embedder_stamp,
|
||||
enforce_embedder_stamp,
|
||||
read_gallery_stamp,
|
||||
require_gallery_stamp_from_env,
|
||||
sha256_file,
|
||||
verify_gallery_stamp,
|
||||
_as_str,
|
||||
_stamp_empty,
|
||||
)
|
||||
|
||||
|
||||
def save_gallery_hdf5(gallery: dict, output: Path, embedder: dict | None = None) -> None:
|
||||
def save_gallery_hdf5(gallery: dict, output: Path) -> None:
|
||||
"""Write a gallery dict ({"actors": [...]}) directly as HDF5 — same schema
|
||||
src/gallery/gallery_store.cpp reads/writes. No calibration group; the
|
||||
C++ identity_matcher computes and writes it back into this file on first
|
||||
use against an unseen set of embeddings.
|
||||
|
||||
`embedder` is the GR-004 stamp (see embedder_stamp()); it may also be carried
|
||||
on the gallery dict under "embedder", which is how a filtered/derived gallery
|
||||
keeps its binding without the caller having to re-hash anything."""
|
||||
embedder = embedder if embedder is not None else gallery.get("embedder")
|
||||
use against an unseen set of embeddings."""
|
||||
actors = gallery["actors"]
|
||||
embs, offsets, counts = [], [], []
|
||||
imdb, tmdb, jf, name, src_images = [], [], [], [], []
|
||||
@@ -168,18 +139,8 @@ def save_gallery_hdf5(gallery: dict, output: Path, embedder: dict | None = None)
|
||||
f.create_dataset("jellyfin_id", data=np.asarray(jf, dtype=object), dtype=str_t)
|
||||
f.create_dataset("name", data=np.asarray(name, dtype=object), dtype=str_t)
|
||||
f.create_dataset("source_images", data=np.asarray(src_images, dtype=object), dtype=str_t)
|
||||
# TRACES: GR-004 | SR-001
|
||||
# omitted entirely when unknown, so "unstamped"
|
||||
# round-trips as unstamped rather than as a stamp naming no model.
|
||||
if not _stamp_empty(embedder):
|
||||
g = f.create_group("embedder")
|
||||
g.attrs["model_name"] = embedder.get("model_name", "")
|
||||
g.attrs["model_sha256"] = embedder.get("model_sha256", "")
|
||||
g.attrs["embed_dim"] = np.int32(embedder.get("embed_dim", 512))
|
||||
stamp_note = (f", embedder {embedder['model_name']}" if not _stamp_empty(embedder)
|
||||
else ", NO EMBEDDER STAMP (GR-004)")
|
||||
print(f"Saved: {output} ({len(actors)} actors, {emb_arr.shape[0]} embeddings"
|
||||
f"{stamp_note})", file=sys.stderr)
|
||||
print(f"Saved: {output} ({len(actors)} actors, {emb_arr.shape[0]} embeddings)",
|
||||
file=sys.stderr)
|
||||
|
||||
|
||||
def load_gallery_hdf5(path: Path) -> dict:
|
||||
@@ -197,15 +158,6 @@ def load_gallery_hdf5(path: Path) -> dict:
|
||||
if "source_images" in f:
|
||||
src_images = [s.decode() if isinstance(s, bytes) else s
|
||||
for s in f["source_images"][:]]
|
||||
# TRACES: GR-004 | SR-001
|
||||
# carried through so a derived gallery (filter,
|
||||
# merge, cast-restrict) keeps the binding of the gallery it came from.
|
||||
stamp = None
|
||||
if "embedder" in f:
|
||||
a = f["embedder"].attrs
|
||||
stamp = {"model_name": _as_str(a.get("model_name", "")),
|
||||
"model_sha256": _as_str(a.get("model_sha256", "")),
|
||||
"embed_dim": int(a.get("embed_dim", 512))}
|
||||
|
||||
actors = []
|
||||
for a in range(len(offset)):
|
||||
@@ -215,19 +167,15 @@ def load_gallery_hdf5(path: Path) -> dict:
|
||||
if src_images is not None:
|
||||
actor["source_images"] = [src_images[s + i] for i in range(n)]
|
||||
actors.append(actor)
|
||||
out = {"actors": actors}
|
||||
if stamp is not None:
|
||||
out["embedder"] = stamp
|
||||
return out
|
||||
return {"actors": actors}
|
||||
|
||||
|
||||
def save_gallery(gallery: dict, missing: list[dict], output: Path,
|
||||
embedder: dict | None = None) -> None:
|
||||
def save_gallery(gallery: dict, missing: list[dict], output: Path) -> None:
|
||||
"""Write the gallery as HDF5 (forcing a .h5 extension) and, if any actors
|
||||
lack images, a .missing_images.json sidecar."""
|
||||
if output.suffix not in (".h5", ".hdf5"):
|
||||
output = output.with_suffix(".h5")
|
||||
save_gallery_hdf5(gallery, output, embedder)
|
||||
save_gallery_hdf5(gallery, output)
|
||||
|
||||
if missing:
|
||||
missing_path = output.with_name(output.stem + ".missing_images.json")
|
||||
|
||||
@@ -1,217 +0,0 @@
|
||||
"""Gallery ↔ embedder model binding (GR-004).
|
||||
|
||||
TRACES: GR-004 | SR-001
|
||||
|
||||
Python twin of src/gallery/embedder_stamp.{hpp,cpp}; the two implement the same
|
||||
comparison rules and must stay in agreement. Kept as its own module — rather than
|
||||
folded into sae_gallery — because scripts/optimizer/replay.py imports it once per
|
||||
replay subprocess, thousands of times in a DE sweep, and must not pay for
|
||||
sae_gallery's requests/Pillow imports to ask "were these made by the same model?".
|
||||
Dependencies here are hashlib, json and h5py, all of which a replay already loads.
|
||||
|
||||
A gallery is only valid for the embedder that built it: cosine similarities across
|
||||
models are meaningless but look plausible, so the mistake is silent and every
|
||||
measurement taken afterwards is suspect. Identity = model filename + SHA-256 of
|
||||
the model file. The hash decides (a model re-exported in place keeps its name but
|
||||
not its bytes); the name is what makes the error readable. See
|
||||
src/gallery/embedder_stamp.hpp for the full rationale.
|
||||
"""
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
import h5py
|
||||
|
||||
|
||||
def _as_str(v) -> str:
|
||||
return v.decode() if isinstance(v, bytes) else ("" if v is None else str(v))
|
||||
|
||||
|
||||
_STAMP_CACHE: dict = {}
|
||||
|
||||
|
||||
class EmbedderMismatch(RuntimeError):
|
||||
"""Gallery was built with a different embedder than the one about to be used."""
|
||||
|
||||
|
||||
def sha256_file(path) -> str:
|
||||
"""Lowercase hex SHA-256 of a file's bytes; "" if it cannot be read."""
|
||||
path = Path(path)
|
||||
try:
|
||||
st = path.stat()
|
||||
except OSError:
|
||||
return ""
|
||||
key = (str(path), st.st_mtime_ns, st.st_size)
|
||||
if key in _STAMP_CACHE:
|
||||
return _STAMP_CACHE[key]
|
||||
h = hashlib.sha256()
|
||||
try:
|
||||
with open(path, "rb") as f:
|
||||
for chunk in iter(lambda: f.read(1 << 20), b""):
|
||||
h.update(chunk)
|
||||
except OSError:
|
||||
return ""
|
||||
_STAMP_CACHE[key] = h.hexdigest()
|
||||
return _STAMP_CACHE[key]
|
||||
|
||||
|
||||
def embedder_stamp(model_path, embed_dim: int = 512) -> dict:
|
||||
"""Identify an embedder model file → {"model_name", "model_sha256", "embed_dim"}.
|
||||
|
||||
A model file that is absent (e.g. a TRT deployment running from a prebuilt
|
||||
.engine) yields a name-only stamp: still comparable, just not provable."""
|
||||
if not model_path:
|
||||
return {"model_name": "", "model_sha256": "", "embed_dim": embed_dim}
|
||||
sha = sha256_file(model_path)
|
||||
if not sha:
|
||||
print(f"[gallery] cannot hash embedder model {model_path} — model binding "
|
||||
f"falls back to filename only (GR-004)", file=sys.stderr)
|
||||
return {"model_name": Path(model_path).name, "model_sha256": sha,
|
||||
"embed_dim": embed_dim}
|
||||
|
||||
|
||||
def _stamp_empty(s) -> bool:
|
||||
return not s or (not s.get("model_name") and not s.get("model_sha256"))
|
||||
|
||||
|
||||
def describe_stamp(s) -> str:
|
||||
if _stamp_empty(s):
|
||||
return "UNKNOWN"
|
||||
name = s.get("model_name") or "<unnamed model>"
|
||||
sha = s.get("model_sha256") or ""
|
||||
return f"{name} (sha256 {sha[:12]}…)" if sha else f"{name} (sha256 unavailable)"
|
||||
|
||||
|
||||
def require_gallery_stamp_from_env() -> bool:
|
||||
"""SAE_REQUIRE_GALLERY_STAMP=1 → an unprovable binding is fatal, not a warning."""
|
||||
return os.environ.get("SAE_REQUIRE_GALLERY_STAMP", "0") not in ("", "0")
|
||||
|
||||
|
||||
def check_embedder_stamp(built_with: dict | None, loading_with: dict | None,
|
||||
gallery_desc: str = "gallery",
|
||||
embedder_desc: str = "embedder") -> tuple[str, str]:
|
||||
"""Pure comparison. Returns (verdict, message); verdict is one of
|
||||
match / weak_match / unstamped / unknown_embedder / mismatch.
|
||||
|
||||
Same rules as compare_embedder_stamps() in src/gallery/embedder_stamp.cpp."""
|
||||
if _stamp_empty(built_with):
|
||||
return "unstamped", (
|
||||
f"gallery '{gallery_desc}' carries no embedder stamp (GR-004).\n"
|
||||
f" gallery was built with : UNKNOWN — this file predates model binding\n"
|
||||
f" embedder now loaded : {describe_stamp(loading_with)} [{embedder_desc}]\n"
|
||||
f" If these are not the same model every similarity from this run is\n"
|
||||
f" meaningless but will look plausible. Rebuild or re-stamp the gallery\n"
|
||||
f" (scripts/stamp_gallery.py), or run with SAE_REQUIRE_GALLERY_STAMP=1 to\n"
|
||||
f" make this a hard error.")
|
||||
|
||||
if _stamp_empty(loading_with):
|
||||
return "unknown_embedder", (
|
||||
f"cannot identify the embedder being used against gallery "
|
||||
f"'{gallery_desc}' (GR-004).\n"
|
||||
f" gallery was built with : {describe_stamp(built_with)}\n"
|
||||
f" embedder now loaded : UNKNOWN [{embedder_desc}]\n"
|
||||
f" The binding cannot be checked, so it is not being checked.")
|
||||
|
||||
mismatch_tail = (
|
||||
" Cosine similarities between embeddings from different models are\n"
|
||||
" meaningless but look plausible. Rebuild the gallery with the loaded\n"
|
||||
" model, or point the embedder at the model the gallery was built with.")
|
||||
|
||||
if int(built_with.get("embed_dim", 512)) != int(loading_with.get("embed_dim", 512)):
|
||||
return "mismatch", (
|
||||
f"gallery/embedder MODEL MISMATCH — refusing to run (GR-004).\n"
|
||||
f" gallery was built with : {describe_stamp(built_with)}, "
|
||||
f"dim={built_with.get('embed_dim')} [{gallery_desc}]\n"
|
||||
f" embedder now loaded : {describe_stamp(loading_with)}, "
|
||||
f"dim={loading_with.get('embed_dim')} [{embedder_desc}]\n"
|
||||
f" Embedding dimensions differ; these are not the same space.")
|
||||
|
||||
a, b = built_with.get("model_sha256", ""), loading_with.get("model_sha256", "")
|
||||
if a and b:
|
||||
if a == b:
|
||||
note = ""
|
||||
if built_with.get("model_name") != loading_with.get("model_name"):
|
||||
note = (f" (gallery recorded it as '{built_with.get('model_name')}', "
|
||||
f"loaded from '{loading_with.get('model_name')}' — "
|
||||
f"same bytes, renamed file)")
|
||||
return "match", f"embedder binding verified: {describe_stamp(built_with)}{note}"
|
||||
return "mismatch", (
|
||||
f"gallery/embedder MODEL MISMATCH — refusing to run (GR-004).\n"
|
||||
f" gallery was built with : {built_with.get('model_name')} sha256={a}\n"
|
||||
f" [{gallery_desc}]\n"
|
||||
f" embedder now loaded : {loading_with.get('model_name')} sha256={b}\n"
|
||||
f" [{embedder_desc}]\n" + mismatch_tail)
|
||||
|
||||
if built_with.get("model_name") and \
|
||||
built_with.get("model_name") == loading_with.get("model_name"):
|
||||
return "weak_match", (
|
||||
f"embedder binding UNPROVEN for gallery '{gallery_desc}' (GR-004).\n"
|
||||
f" gallery was built with : {describe_stamp(built_with)}\n"
|
||||
f" embedder now loaded : {describe_stamp(loading_with)} [{embedder_desc}]\n"
|
||||
f" Filenames agree but at least one SHA-256 is unavailable, so an\n"
|
||||
f" in-place re-export under the same name would not be detected.")
|
||||
|
||||
return "mismatch", (
|
||||
f"gallery/embedder MODEL MISMATCH — refusing to run (GR-004).\n"
|
||||
f" gallery was built with : {describe_stamp(built_with)} [{gallery_desc}]\n"
|
||||
f" embedder now loaded : {describe_stamp(loading_with)} [{embedder_desc}]\n"
|
||||
+ mismatch_tail)
|
||||
|
||||
|
||||
def enforce_embedder_stamp(built_with, loading_with, gallery_desc, embedder_desc,
|
||||
require_stamp: bool = False) -> str:
|
||||
"""Apply check_embedder_stamp: raise EmbedderMismatch when fatal, else warn.
|
||||
|
||||
A mismatch is fatal unconditionally — there is no bypass, because a mismatch is
|
||||
a known-wrong state, not an unknown one. The three "cannot prove it" verdicts
|
||||
warn loudly and become fatal under require_stamp / SAE_REQUIRE_GALLERY_STAMP."""
|
||||
strict = require_stamp or require_gallery_stamp_from_env()
|
||||
verdict, msg = check_embedder_stamp(built_with, loading_with,
|
||||
gallery_desc, embedder_desc)
|
||||
if verdict == "mismatch":
|
||||
raise EmbedderMismatch(msg)
|
||||
if strict and verdict != "match":
|
||||
raise EmbedderMismatch(
|
||||
msg + "\n (fatal because SAE_REQUIRE_GALLERY_STAMP is set)")
|
||||
if verdict == "match":
|
||||
print(f"[gallery] {msg}", file=sys.stderr)
|
||||
else:
|
||||
print(f"\n[gallery] ***** WARNING (GR-004) *****\n{msg}\n"
|
||||
f"[gallery] ****************************\n", file=sys.stderr)
|
||||
return verdict
|
||||
|
||||
|
||||
def read_gallery_stamp(path) -> dict | None:
|
||||
"""The embedder stamp recorded in a gallery file, or None if unstamped.
|
||||
|
||||
Handles both the HDF5 /embedder group and the legacy JSON "embedder" object."""
|
||||
path = Path(path)
|
||||
if path.suffix in (".h5", ".hdf5"):
|
||||
with h5py.File(path, "r") as f:
|
||||
if "embedder" not in f:
|
||||
return None
|
||||
a = f["embedder"].attrs
|
||||
return {"model_name": _as_str(a.get("model_name", "")),
|
||||
"model_sha256": _as_str(a.get("model_sha256", "")),
|
||||
"embed_dim": int(a.get("embed_dim", 512))}
|
||||
data = json.loads(path.read_text())
|
||||
return data.get("embedder") or None
|
||||
|
||||
|
||||
def verify_gallery_stamp(gallery_path, model_path=None, *, stamp=None,
|
||||
embedder_desc: str | None = None,
|
||||
require_stamp: bool = False) -> str:
|
||||
"""Load a gallery's stamp and check it against a model file (or an explicit
|
||||
stamp, e.g. one read off an embedding dump). Raises EmbedderMismatch."""
|
||||
loading = stamp if stamp is not None else embedder_stamp(model_path)
|
||||
return enforce_embedder_stamp(read_gallery_stamp(gallery_path), loading,
|
||||
str(gallery_path),
|
||||
embedder_desc or str(model_path or "unknown"),
|
||||
require_stamp)
|
||||
|
||||
|
||||
def _as_str(v) -> str:
|
||||
return v.decode() if isinstance(v, bytes) else ("" if v is None else str(v))
|
||||
@@ -1,86 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
"""stamp_gallery.py — bind an existing gallery to the embedder that built it.
|
||||
|
||||
TRACES: GR-004 | SR-001
|
||||
|
||||
Galleries built before model binding carry no embedder stamp. They still load,
|
||||
but every consumer warns that it cannot tell whether the gallery and the embedder
|
||||
belong together — and under SAE_REQUIRE_GALLERY_STAMP=1 they refuse to run.
|
||||
|
||||
This is the migration path, and the reason the unstamped case is a warning rather
|
||||
than a hard failure: re-binding an existing gallery costs one command and no
|
||||
re-embedding, so nobody has to choose between a bricked setup and a check they
|
||||
route around.
|
||||
|
||||
python scripts/stamp_gallery.py --gallery gallery.h5 \\
|
||||
--arcface models/LVFace-B_Glint360K.onnx
|
||||
|
||||
The stamp is an ASSERTION: you are stating which model produced these vectors.
|
||||
Nothing can verify it from the vectors themselves, which is exactly why the stamp
|
||||
has to be written at build time going forward. Stamping the wrong model is worse
|
||||
than leaving it unstamped, because it converts a loud warning into a false
|
||||
all-clear — so --show it first if you are not certain.
|
||||
|
||||
python scripts/stamp_gallery.py --gallery gallery.h5 --show
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
import h5py
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parent))
|
||||
from sae_gallery import (describe_stamp, embedder_stamp, # noqa: E402
|
||||
read_gallery_stamp)
|
||||
|
||||
|
||||
def main() -> int:
|
||||
p = argparse.ArgumentParser(description=__doc__,
|
||||
formatter_class=argparse.RawDescriptionHelpFormatter)
|
||||
p.add_argument("--gallery", required=True, help="gallery .h5 to stamp in place")
|
||||
p.add_argument("--arcface", help="the ONNX that built it (hashed into the stamp)")
|
||||
p.add_argument("--show", action="store_true", help="print the current stamp and exit")
|
||||
p.add_argument("--force", action="store_true",
|
||||
help="overwrite an existing stamp (refused otherwise)")
|
||||
args = p.parse_args()
|
||||
|
||||
path = Path(args.gallery)
|
||||
if path.suffix not in (".h5", ".hdf5"):
|
||||
return err(f"{path}: only HDF5 galleries can be stamped in place")
|
||||
|
||||
current = read_gallery_stamp(path)
|
||||
print(f"{path}: current stamp = {describe_stamp(current)}", file=sys.stderr)
|
||||
if args.show:
|
||||
return 0
|
||||
if not args.arcface:
|
||||
return err("--arcface is required (or use --show)")
|
||||
if current and not args.force:
|
||||
return err("gallery is already stamped — pass --force to overwrite, but be "
|
||||
"sure: a wrong stamp turns a warning into a false all-clear")
|
||||
|
||||
stamp = embedder_stamp(args.arcface)
|
||||
if not stamp["model_sha256"]:
|
||||
return err(f"cannot hash {args.arcface} — refusing to write a name-only "
|
||||
"stamp, which would claim more certainty than it has")
|
||||
|
||||
with h5py.File(path, "r+") as f:
|
||||
if "embedder" in f:
|
||||
del f["embedder"]
|
||||
g = f.create_group("embedder")
|
||||
g.attrs["model_name"] = stamp["model_name"]
|
||||
g.attrs["model_sha256"] = stamp["model_sha256"]
|
||||
g.attrs["embed_dim"] = stamp["embed_dim"]
|
||||
|
||||
print(f"{path}: stamped with {describe_stamp(stamp)}", file=sys.stderr)
|
||||
return 0
|
||||
|
||||
|
||||
def err(msg: str) -> int:
|
||||
print(f"[stamp_gallery] {msg}", file=sys.stderr)
|
||||
return 1
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
Executable
+1643
File diff suppressed because it is too large
Load Diff
Executable
+1047
File diff suppressed because it is too large
Load Diff
Executable
+62
@@ -0,0 +1,62 @@
|
||||
#!/bin/sh
|
||||
#
|
||||
# Requirement traceability gate. Run locally exactly as CI runs it, from the
|
||||
# component repo root:
|
||||
#
|
||||
# scripts/traceability/traceability-gate.sh
|
||||
#
|
||||
# Writes the JSON report and the markdown matrix, prints the coverage report,
|
||||
# and exits non-zero when the gate fails.
|
||||
#
|
||||
# This script is shared by every JRay component, so it knows nothing about any
|
||||
# one repo. All repo-specific settings - requirement ID prefixes, source
|
||||
# suffixes, scan roots, register path, thresholds - live in `traceability.toml`
|
||||
# at the component repo root. Run
|
||||
#
|
||||
# scripts/traceability/extract_traces.py --print-example-config
|
||||
#
|
||||
# for the annotated schema. A repo whose config is wrong parses zero
|
||||
# requirements or scans zero files, and the gate refuses to report rather than
|
||||
# printing a misleading 0%.
|
||||
#
|
||||
# Environment (all optional; each overrides the config file):
|
||||
# TRACES_CONFIG path to traceability.toml
|
||||
# TRACES_ROOT repo root (default: nearest dir containing traceability.toml)
|
||||
# MIN_COVERAGE minimum overall coverage percent
|
||||
# ALLOW_ORPHANS 1 to report orphan tags without failing
|
||||
# TRACES_JSON JSON report path
|
||||
# TRACES_MD markdown matrix path
|
||||
# SYSTEM_SPEC SPEC.md defining PR/SR; enables PR/SR orphan checking
|
||||
# PYTHON interpreter (default: python3)
|
||||
#
|
||||
# Threshold policy belongs in traceability.toml, not here and not in the
|
||||
# workflow YAML: a threshold written in two places is a threshold that will
|
||||
# disagree with itself.
|
||||
#
|
||||
# POSIX sh, no bashisms, no jq - the extractor does its own arithmetic and
|
||||
# printing, so CI needs nothing beyond python3.
|
||||
|
||||
set -eu
|
||||
|
||||
SCRIPT_DIR=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd)
|
||||
|
||||
PYTHON="${PYTHON:-python3}"
|
||||
command -v "$PYTHON" >/dev/null 2>&1 || {
|
||||
echo "FAILED: $PYTHON not found. The traceability gate needs Python 3.9+," >&2
|
||||
echo " or 3.11+ to read traceability.toml." >&2
|
||||
exit 2
|
||||
}
|
||||
|
||||
set -- --format coverage
|
||||
|
||||
# Explicit `if` rather than `[ ... ] && ...`, because a trailing false test in
|
||||
# an && list exits under `set -e` in some POSIX shells.
|
||||
if [ -n "${TRACES_CONFIG:-}" ]; then set -- "$@" --config "$TRACES_CONFIG"; fi
|
||||
if [ -n "${TRACES_ROOT:-}" ]; then set -- "$@" --root "$TRACES_ROOT"; fi
|
||||
if [ -n "${MIN_COVERAGE:-}" ]; then set -- "$@" --min-coverage "$MIN_COVERAGE"; fi
|
||||
if [ -n "${TRACES_JSON:-}" ]; then set -- "$@" --json-out "$TRACES_JSON"; fi
|
||||
if [ -n "${TRACES_MD:-}" ]; then set -- "$@" --markdown-out "$TRACES_MD"; fi
|
||||
if [ -n "${SYSTEM_SPEC:-}" ]; then set -- "$@" --system-spec "$SYSTEM_SPEC"; fi
|
||||
if [ "${ALLOW_ORPHANS:-0}" = "1" ]; then set -- "$@" --allow-orphans; fi
|
||||
|
||||
exec "$PYTHON" "$SCRIPT_DIR/extract_traces.py" "$@"
|
||||
@@ -79,30 +79,9 @@ The table caches nulls (tmdb ids TMDB has no IMDb id for) and checkpoints, so a
|
||||
re-run only resolves new ids. TMDB is authoritative for this crosswalk — there is
|
||||
no clean free bulk `tmdb_person ↔ nm` file, so we query the API once and cache.
|
||||
|
||||
## Minimum face size (VR-005)
|
||||
|
||||
`min_face_size.py` is a separate, self-contained study: it needs no video and no
|
||||
ground truth, only the gallery mugshot cache. It holds out one image per actor,
|
||||
degrades that probe to each candidate face size and matches it against a gallery
|
||||
held at **native** resolution, reporting TPI/FPI per size — the measurement that
|
||||
replaces AR-002's 66×66 px estimate.
|
||||
|
||||
```bash
|
||||
python scripts/validation/min_face_size.py \
|
||||
--images images --gallery gallery_lvface.h5 \
|
||||
--arcface models/LVFace-B_Glint360K.onnx \
|
||||
--actors 100 --out experiments/results/vr005_min_face_size
|
||||
```
|
||||
|
||||
FPI grows with the number of actors competing, so a 100-actor run understates it
|
||||
against a library of thousands: read FPI as relative across sizes, not as an
|
||||
absolute rate. Re-run per `--arcface` model to see whether `min_face_px` should be
|
||||
one constant or scale with the embedder (GR-004).
|
||||
|
||||
## Files
|
||||
- `sample_eval.py` — CLI scorer.
|
||||
- `ground_truth.py` — `XRayGroundTruth`, `MovieNetGroundTruth` loaders.
|
||||
- `identity.py` — provider-agnostic match keys.
|
||||
- `tmdb_imdb_map.py` — build/consult the cached `tmdb→imdb` crosswalk.
|
||||
- `min_face_size.py` — VR-005 probe-size sweep (see above).
|
||||
- `test_sample_eval.py` — self-contained tests (`python scripts/validation/test_sample_eval.py`).
|
||||
|
||||
@@ -17,11 +17,6 @@ X-Ray/MovieNet key on IMDb nm-ids — see [[per-scene-presence-eval-design]].
|
||||
Two sources implemented:
|
||||
* XRayGroundTruth — Zenodo scene-level Amazon X-Ray CSVs (cast-in-scene).
|
||||
* MovieNetGroundTruth — MovieNet-PS per-shot face annotations (on-screen faces).
|
||||
|
||||
Both are published corpora addressed by title, so a scoring run is reproducible
|
||||
from the identifiers alone — no annotation of ours travels with the code.
|
||||
|
||||
TRACES: VR-004 | PR-002
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
@@ -1,822 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
min_face_size.py — VR-005: at what face size do embeddings stop identifying people?
|
||||
|
||||
TRACES: VR-005
|
||||
|
||||
`min_face_px` is currently a working estimate (AR-002: 66x66 px in original video
|
||||
resolution). This script replaces the guess with a measurement, using only gallery
|
||||
mugshots already on disk — no video, no C++ changes.
|
||||
|
||||
Protocol
|
||||
--------
|
||||
1. Select ~100 gallery actors that have more than one mugshot.
|
||||
2. Per actor hold out ONE image as the *probe*; that actor's remaining images stay
|
||||
in the gallery at native resolution.
|
||||
3. For each target size S, take the probe's native aligned 112x112 crop, downscale
|
||||
it to SxS and upscale it back to 112x112, then embed. Detail is genuinely
|
||||
destroyed and then the same warp the pipeline applies is re-applied on top —
|
||||
which is what a face detected at SxS in a frame actually suffers.
|
||||
4. Match each degraded probe against the whole gallery.
|
||||
5. Record, per size, TPI (identified as the correct actor) and FPI (identified as
|
||||
someone else). Everything else is an unidentified probe (TBI).
|
||||
|
||||
The asymmetry is the point: **the gallery stays at native resolution and only the
|
||||
probe degrades.** That is the production case — reference mugshots are clean, the
|
||||
face coming out of the video is small. Degrading both sides would measure
|
||||
something the pipeline never does.
|
||||
|
||||
What this deliberately does NOT measure
|
||||
---------------------------------------
|
||||
The cosine between the size-S embedding and the native embedding of the *same*
|
||||
image. That is embedding *drift*, and it answers the wrong question: an embedding
|
||||
can drift a long way and stay perfectly separable, or drift a little in a
|
||||
direction that destroys separation. What matters is the decision the pipeline
|
||||
makes — probe against a competing gallery — so that is what is recorded.
|
||||
|
||||
CAVEAT — FPI IS RELATIVE, NOT ABSOLUTE
|
||||
--------------------------------------
|
||||
False positives grow with the number of actors competing for the match. A
|
||||
~100-actor gallery therefore *understates* the false-positive rate against a
|
||||
production library of thousands. Read the FPI column as a relative curve across
|
||||
sizes ("FPI is 4x worse at 32 px than at 64 px"), never as the rate you would see
|
||||
in production. Re-run with `--actors` at production scale before setting a
|
||||
threshold from an absolute FPI number.
|
||||
|
||||
Decision rule
|
||||
-------------
|
||||
Per the repo invariant (CLAUDE.md: "always use the calibrated probability, never a
|
||||
raw cosine"), identification goes through the same path as `identity_matcher_node`:
|
||||
per-actor best-of-N cosine -> Platt sigmoid P(match) = sigma(a*sim + b + log-prior)
|
||||
-> accept if P > `prob_threshold`. The sigmoid is fitted here by the same
|
||||
histogram/gradient-descent procedure as `src/gallery/gallery_calibration.hpp`,
|
||||
over the native gallery embeddings only (held-out probes are excluded, so the
|
||||
calibration cannot see the images it will be scored on).
|
||||
|
||||
How this runs
|
||||
-------------
|
||||
Through the `sae_embed` bindings, which expose the shipped C++ stages directly:
|
||||
`detect()`, `align_face()`, `embed_crops()` and `GalleryCalibration`. Nothing
|
||||
here re-implements detection, the ArcFace warp, the embedder or the Platt fit.
|
||||
|
||||
That matters most for the calibration. A second copy of the sigmoid is exactly
|
||||
where "always the calibrated probability, never a raw cosine" (AR-024) gets
|
||||
broken without anyone noticing, because the copy keeps returning plausible
|
||||
numbers after the original has moved. Scoring through the binding makes the rule
|
||||
structural instead of remembered.
|
||||
|
||||
The backend is whichever was compiled in. Under `SAE_INFERENCE_BACKEND=ORT`
|
||||
that is the reference fp32 path, which loads the .onnx directly. A TensorRT fp16
|
||||
build is a *different realisation* of the same model and its embeddings are
|
||||
measurably not the same vectors: on LVFace-B_Glint360K the stored TRT-fp16 gallery
|
||||
agrees with an fp32 recompute of the same mugshot at only ~0.85 cosine, while
|
||||
same-actor/different-actor separation is essentially unchanged (d' 5.3 vs 5.7).
|
||||
Nothing here is invalidated by that — gallery and probes go through one session,
|
||||
so the comparison is internally consistent — but the two embedding spaces are not
|
||||
interchangeable, and `--verify-against <gallery.h5>` will show ~0.85, not ~1.0,
|
||||
against a TRT-built gallery. It reports the separation of both sets alongside the
|
||||
agreement so the two causes are distinguishable: a broken port collapses
|
||||
separation, a different backend does not.
|
||||
|
||||
Secondary output (VR-005): running the sweep per `--arcface` model shows whether
|
||||
`min_face_px` should be one constant at all, or should scale with the embedder —
|
||||
which matters because the model is a build-time choice (GR-004).
|
||||
|
||||
Usage
|
||||
-----
|
||||
python scripts/validation/min_face_size.py \
|
||||
--images images \
|
||||
--gallery gallery_lvface.h5 \
|
||||
--arcface models/LVFace-B_Glint360K.onnx \
|
||||
--actors 100 --seed 0 \
|
||||
--out experiments/results/vr005_min_face_size
|
||||
|
||||
Writes <out>.csv, <out>.json and <out>.png (plus <out>.per_probe.csv with
|
||||
--per-probe).
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import csv
|
||||
import json
|
||||
import random
|
||||
import re
|
||||
import sys
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
REPO = Path(__file__).resolve().parent.parent.parent
|
||||
sys.path.insert(0, str(REPO / "scripts"))
|
||||
def _find_sae_embed() -> Path | None:
|
||||
"""Locate the built sae_embed module.
|
||||
|
||||
A git worktree has no build tree of its own, so fall back to the main
|
||||
checkout via the shared git dir — otherwise running this study from a
|
||||
feature worktree cannot find the bindings it now depends on.
|
||||
"""
|
||||
roots = [REPO]
|
||||
try:
|
||||
import subprocess
|
||||
common = subprocess.run(["git", "-C", str(REPO), "rev-parse",
|
||||
"--path-format=absolute", "--git-common-dir"],
|
||||
capture_output=True, text=True, check=True).stdout.strip()
|
||||
if common:
|
||||
roots.append(Path(common).parent)
|
||||
except Exception:
|
||||
pass
|
||||
for root in roots:
|
||||
for b in ("build-ort", "build"):
|
||||
if list((root / b).glob("sae_embed*.so")):
|
||||
return root / b
|
||||
return None
|
||||
|
||||
|
||||
_SAE_BUILD = _find_sae_embed()
|
||||
if _SAE_BUILD is None:
|
||||
sys.exit("cannot find the built sae_embed module — build it with\n"
|
||||
" cmake --build build-ort --target sae_embed")
|
||||
sys.path.insert(0, str(_SAE_BUILD))
|
||||
|
||||
# Before cv2: OpenCV's DNN module loads the system libonnxruntime, which then
|
||||
# shadows the one sae_embed links against and the import fails on a missing
|
||||
# symbol version. Order matters here.
|
||||
import sae_embed
|
||||
|
||||
import cv2
|
||||
import numpy as np
|
||||
|
||||
IMAGE_EXTS = (".jpg", ".jpeg", ".png", ".webp")
|
||||
JELLYFIN_ID_RE = re.compile(r"^[0-9a-f]{32}$")
|
||||
|
||||
# Interpolation used for the two halves of the degradation. Downscaling uses
|
||||
# INTER_AREA (correct low-pass for shrinking, i.e. detail that a small detection
|
||||
# genuinely never had); upscaling uses INTER_LINEAR, which is what warpAffine in
|
||||
# align_face() uses when it blows a small detection up to 112x112.
|
||||
INTERP = {
|
||||
"area": cv2.INTER_AREA,
|
||||
"linear": cv2.INTER_LINEAR,
|
||||
"cubic": cv2.INTER_CUBIC,
|
||||
"nearest": cv2.INTER_NEAREST,
|
||||
"lanczos": cv2.INTER_LANCZOS4,
|
||||
}
|
||||
|
||||
# House chart palette, shared with scripts/docs/experiment_charts.py so figures
|
||||
# across the report read as one set.
|
||||
INK, MUTED, GRID, SURFACE = "#0b0b0b", "#898781", "#e1e0d9", "#fcfcfb"
|
||||
BLUE, GREEN, RED, AMBER = "#2a78d6", "#008300", "#e34948", "#eda100"
|
||||
|
||||
|
||||
# ── Production stages, via the sae_embed bindings ─────────────────────────────
|
||||
# detect / align_face / embed_crops / calibrate_gallery all call the shipped C++.
|
||||
# There is deliberately no Python re-implementation of any of them: a second copy
|
||||
# drifts from what ships, and the calibration is the one that must not — AR-024
|
||||
# requires every similarity to pass through the same sigmoid the matcher uses.
|
||||
|
||||
# These two mirror constants in gallery_calibration.hpp. They are NOT a second
|
||||
# copy of the fit — that is the binding's job — but the script reproduces the
|
||||
# same dedup and eligibility filtering so the actor counts it reports describe
|
||||
# the population the C++ actually fitted on. Keep them in step with the header.
|
||||
MIN_EMB_FOR_POSITIVE = 5
|
||||
DEDUP_SIM = 1.0 - 1e-7
|
||||
|
||||
|
||||
class Stages:
|
||||
"""Thin holder so the rest of the script has one object to call."""
|
||||
|
||||
def __init__(self, detector: str, arcface: str, conf: float, nms: float):
|
||||
self.engine = sae_embed.FaceEmbedder(
|
||||
detector_model=detector, arcface_model=arcface,
|
||||
conf=conf, nms=nms, max_side=0)
|
||||
|
||||
def detect(self, img):
|
||||
return self.engine.detect(img)
|
||||
|
||||
def align(self, img, landmarks):
|
||||
return sae_embed.align_face(img, np.asarray(landmarks, dtype=np.float32).reshape(5, 2))
|
||||
|
||||
def enhance(self, img):
|
||||
return sae_embed.enhance_for_retry(img)
|
||||
|
||||
def embed(self, crops):
|
||||
"""(N,112,112,3) uint8 BGR -> (N,512) float32.
|
||||
|
||||
Chunked at the backend's max_batch: the engine does not split an
|
||||
oversized request, so handing it a whole gallery at once asks CUDA for
|
||||
a multi-gigabyte activation buffer and the allocator refuses.
|
||||
"""
|
||||
if not len(crops):
|
||||
return np.zeros((0, 512), dtype=np.float32)
|
||||
n = max(1, int(self.engine.max_batch))
|
||||
arr = np.ascontiguousarray(np.stack(crops), dtype=np.uint8)
|
||||
out = [np.asarray(self.engine.embed_crops(np.ascontiguousarray(arr[i:i + n])))
|
||||
for i in range(0, len(arr), n)]
|
||||
return np.concatenate(out, axis=0)
|
||||
|
||||
|
||||
def calibrate_gallery(emb: np.ndarray, actor: np.ndarray) -> dict:
|
||||
"""The production Platt fit (gallery_calibration.hpp), via the binding."""
|
||||
cal = sae_embed.calibrate_gallery(
|
||||
np.ascontiguousarray(emb, dtype=np.float32), [int(a) for a in actor])
|
||||
print(f"[calibration] a={cal.a:.4f} b={cal.b:.4f} valid={cal.valid} "
|
||||
f"boundary(P=0.5)=sim{cal.boundary_at(0.5):.4f}", file=sys.stderr)
|
||||
# Held module-side rather than returned: the returned dict lands in the run
|
||||
# metadata, and a native object there breaks the JSON dump.
|
||||
_CAL["cal"] = cal
|
||||
return {"a": float(cal.a), "b": float(cal.b), "valid": bool(cal.valid)}
|
||||
|
||||
|
||||
def probability(sim, a: float, b: float, log_prior_odds: float = 0.0):
|
||||
"""P(match) through GalleryCalibration — the C++ sigmoid, not a copy of it."""
|
||||
cal = _CAL.get("cal")
|
||||
if cal is None:
|
||||
raise RuntimeError("probability() called before calibrate_gallery()")
|
||||
sim = np.asarray(sim, dtype=np.float64)
|
||||
flat = np.atleast_1d(sim).ravel()
|
||||
out = np.array([cal.probability(float(v), log_prior_odds) for v in flat])
|
||||
return out.reshape(sim.shape) if sim.shape else float(out[0])
|
||||
|
||||
|
||||
_CAL: dict = {}
|
||||
|
||||
|
||||
# ── Runtime / actor discovery ─────────────────────────────────────────────────
|
||||
|
||||
def normalise_name(name: str) -> str:
|
||||
return re.sub(r"[^a-z0-9]+", "", name.lower())
|
||||
|
||||
|
||||
def discover_actors(images_root: Path) -> list[dict]:
|
||||
"""Enumerate the gallery-build image cache: <root>/<jellyfin_id>_<Name>/NN.jpg
|
||||
(the layout make_jellyfin_gallery.py / reembed_gallery.py use)."""
|
||||
actors = []
|
||||
for d in sorted(p for p in images_root.iterdir() if p.is_dir()):
|
||||
imgs = sorted(p for p in d.iterdir()
|
||||
if p.is_file() and p.suffix.lower() in IMAGE_EXTS)
|
||||
if not imgs:
|
||||
continue
|
||||
head, _, tail = d.name.partition("_")
|
||||
if JELLYFIN_ID_RE.match(head) and tail:
|
||||
jellyfin_id, name = head, tail.replace("_", " ")
|
||||
else:
|
||||
jellyfin_id, name = "", d.name.replace("_", " ")
|
||||
actors.append({"dir": d, "jellyfin_id": jellyfin_id, "name": name,
|
||||
"images": imgs})
|
||||
return actors
|
||||
|
||||
|
||||
def gallery_keys(gallery_path: Path) -> tuple[set[str], set[str]]:
|
||||
"""(jellyfin ids, normalised names) of the actors an existing gallery holds."""
|
||||
from sae_gallery import load_gallery_hdf5
|
||||
g = load_gallery_hdf5(gallery_path)
|
||||
ids = {a.get("jellyfin_id", "") for a in g["actors"] if a.get("jellyfin_id")}
|
||||
names = {normalise_name(a.get("name", "")) for a in g["actors"] if a.get("name")}
|
||||
return ids, names
|
||||
|
||||
|
||||
# ── Degradation ───────────────────────────────────────────────────────────────
|
||||
|
||||
def degrade(crop: np.ndarray, size: int, down: int, up: int) -> np.ndarray:
|
||||
"""Throw away everything a face detected at size x size never had, then warp
|
||||
it back up to the 112x112 the embedder is fed."""
|
||||
if size == 112:
|
||||
return crop
|
||||
small = cv2.resize(crop, (size, size), interpolation=down)
|
||||
return cv2.resize(small, (112, 112), interpolation=up)
|
||||
|
||||
|
||||
# ── Reporting ─────────────────────────────────────────────────────────────────
|
||||
|
||||
CAVEAT = (
|
||||
"CAVEAT: FPI grows with gallery size. This ran against {n_actors} actors, so it "
|
||||
"UNDERSTATES the false-positive rate of a production library of thousands. Read "
|
||||
"FPI as relative across sizes, not as an absolute rate."
|
||||
)
|
||||
|
||||
|
||||
def write_plot(rows: list[dict], out_png: Path, meta: dict) -> None:
|
||||
import matplotlib
|
||||
matplotlib.use("Agg")
|
||||
import matplotlib.pyplot as plt
|
||||
|
||||
plt.rcParams.update({
|
||||
"figure.facecolor": SURFACE, "axes.facecolor": SURFACE,
|
||||
"savefig.facecolor": SURFACE, "text.color": INK,
|
||||
"axes.edgecolor": MUTED, "axes.labelcolor": INK,
|
||||
"xtick.color": MUTED, "ytick.color": MUTED,
|
||||
"axes.grid": True, "grid.color": GRID, "grid.linewidth": 0.8,
|
||||
"axes.spines.top": False, "axes.spines.right": False,
|
||||
})
|
||||
|
||||
sizes = [r["size_px"] for r in rows]
|
||||
fig, ax = plt.subplots(figsize=(9, 5.6))
|
||||
ax.plot(sizes, [100 * r["tpi_rate"] for r in rows], "-o", color=GREEN,
|
||||
lw=2, label="TPI — identified, correct actor")
|
||||
ax.plot(sizes, [100 * r["fpi_rate"] for r in rows], "-s", color=RED,
|
||||
lw=2, label="FPI — identified, wrong actor")
|
||||
ax.plot(sizes, [100 * r["unidentified_rate"] for r in rows], color=MUTED,
|
||||
marker="^", lw=1.4, ls="--", label="unidentified (below P threshold)")
|
||||
ax.plot(sizes, [100 * r["rank1_rate"] for r in rows], ":", color=BLUE,
|
||||
lw=1.6, label="rank-1 correct (ignoring threshold)")
|
||||
|
||||
op = meta.get("operating_point")
|
||||
if op:
|
||||
ax.axvline(op, color=AMBER, lw=1.6, ls="-.", zorder=1)
|
||||
ax.annotate(f"operating point {op} px", xy=(op, 50),
|
||||
xytext=(4, 0), textcoords="offset points",
|
||||
color=AMBER, fontsize=9, rotation=90, va="center")
|
||||
|
||||
ax.set_xlabel("probe face size before upscaling (px)")
|
||||
ax.set_ylabel("% of probes")
|
||||
ax.set_ylim(-2, 102)
|
||||
ax.set_xticks(sizes)
|
||||
ax.set_title(f"VR-005 — identification vs. probe face size\n"
|
||||
f"{meta['model']}, {meta['n_actors']} actors, "
|
||||
f"{meta['n_probes']} probes/size, gallery at native resolution",
|
||||
fontsize=11, loc="left")
|
||||
ax.legend(frameon=False, fontsize=9, loc="center left")
|
||||
fig.text(0.01, 0.005, CAVEAT.format(n_actors=meta["n_actors"]),
|
||||
fontsize=7.5, color=MUTED, wrap=True)
|
||||
fig.tight_layout(rect=(0, 0.05, 1, 1))
|
||||
out_png.parent.mkdir(parents=True, exist_ok=True)
|
||||
fig.savefig(out_png, dpi=150)
|
||||
plt.close(fig)
|
||||
|
||||
|
||||
def pick_operating_point(rows: list[dict], retention: float, fpi_slack: float) -> int | None:
|
||||
"""Smallest size that keeps `retention` of the undegraded (112 px control)
|
||||
TPI rate and does not add more than `fpi_slack` absolute FPI over it.
|
||||
A stated rule, not a magic number — change the rule, not the answer."""
|
||||
control = next((r for r in rows if r["size_px"] == 112), None)
|
||||
if control is None or control["n_probes"] == 0:
|
||||
return None
|
||||
tpi_floor = retention * control["tpi_rate"]
|
||||
fpi_ceil = control["fpi_rate"] + fpi_slack
|
||||
ok = [r["size_px"] for r in rows
|
||||
if r["tpi_rate"] >= tpi_floor and r["fpi_rate"] <= fpi_ceil]
|
||||
return min(ok) if ok else None
|
||||
|
||||
|
||||
# ── Main ──────────────────────────────────────────────────────────────────────
|
||||
|
||||
def main() -> int:
|
||||
p = argparse.ArgumentParser(
|
||||
description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
|
||||
p.add_argument("--images", required=True,
|
||||
help="gallery image cache root (<jellyfin_id>_<Name>/NN.jpg)")
|
||||
p.add_argument("--gallery", default=None,
|
||||
help="gallery .h5 — restricts the actor pool to its members")
|
||||
p.add_argument("--out", default=str(REPO / "experiments/results/vr005_min_face_size"),
|
||||
help="output path prefix (.csv/.json/.png are appended)")
|
||||
p.add_argument("--per-probe", action="store_true",
|
||||
help="also write <out>.per_probe.csv, one row per probe per size")
|
||||
|
||||
p.add_argument("--actors", type=int, default=100, help="actors to sample (default 100)")
|
||||
p.add_argument("--min-images", type=int, default=2,
|
||||
help="minimum mugshots for an actor to be eligible (default 2)")
|
||||
p.add_argument("--probes-per-actor", type=int, default=1,
|
||||
help="images held out per actor; 1 is the VR-005 protocol")
|
||||
p.add_argument("--seed", type=int, default=0, help="actor/probe selection seed")
|
||||
p.add_argument("--keep-duplicates", action="store_true",
|
||||
help="keep mugshots that are the same photograph twice; by "
|
||||
"default they are dropped, since a probe identical to a "
|
||||
"gallery reference is identified for free at every size")
|
||||
p.add_argument("--sizes", default="12,16,20,24,32,40,48,56,64,72,80,96,112",
|
||||
help="comma-separated probe sizes; 112 is the undegraded control")
|
||||
|
||||
p.add_argument("--models-dir", default=str(REPO / "models"))
|
||||
p.add_argument("--arcface", default=None,
|
||||
help="embedder ONNX (default <models-dir>/LVFace-B_Glint360K.onnx)")
|
||||
p.add_argument("--detector", default=None,
|
||||
help="SCRFD ONNX (default <models-dir>/scrfd_500m_bnkps.onnx)")
|
||||
p.add_argument("--conf", type=float, default=0.5, help="detector confidence")
|
||||
p.add_argument("--nms", type=float, default=0.4, help="detector NMS IoU")
|
||||
p.add_argument("--max-side", type=int, default=500,
|
||||
help="downscale mugshots to this longest side before detection, "
|
||||
"matching the gallery builders' embedder settings")
|
||||
|
||||
p.add_argument("--prob-threshold", type=float, default=0.754,
|
||||
help="accept if P(match) exceeds this (Config::prob_threshold)")
|
||||
p.add_argument("--match-prior", type=float, default=0.5,
|
||||
help="base-rate prior (Config::match_prior)")
|
||||
p.add_argument("--calib-a", type=float, default=None,
|
||||
help="override the fitted sigmoid scale instead of fitting")
|
||||
p.add_argument("--calib-b", type=float, default=None,
|
||||
help="override the fitted sigmoid bias instead of fitting")
|
||||
|
||||
p.add_argument("--down-interp", default="area", choices=sorted(INTERP),
|
||||
help="interpolation for the 112 -> S downscale (default area)")
|
||||
p.add_argument("--up-interp", default="linear", choices=sorted(INTERP),
|
||||
help="interpolation for the S -> 112 upscale (default linear, "
|
||||
"as warpAffine uses in align_face)")
|
||||
|
||||
p.add_argument("--tpi-retention", type=float, default=0.95,
|
||||
help="operating point keeps this fraction of the control TPI rate")
|
||||
p.add_argument("--fpi-slack", type=float, default=0.01,
|
||||
help="operating point may add at most this absolute FPI over control")
|
||||
p.add_argument("--verify-against", default=None,
|
||||
help="gallery .h5 built from --images with the same model: report "
|
||||
"agreement and separation of recomputed vs stored embeddings "
|
||||
"(a TensorRT-built gallery will not agree; see the docstring)")
|
||||
args = p.parse_args()
|
||||
|
||||
if (args.calib_a is None) != (args.calib_b is None):
|
||||
return err("--calib-a and --calib-b must be given together")
|
||||
|
||||
models_dir = Path(args.models_dir)
|
||||
arcface = Path(args.arcface) if args.arcface else models_dir / "LVFace-B_Glint360K.onnx"
|
||||
detector = Path(args.detector) if args.detector else models_dir / "scrfd_500m_bnkps.onnx"
|
||||
for path, what in ((arcface, "embedder"), (detector, "detector")):
|
||||
if not path.is_file():
|
||||
return err(f"{what} model not found: {path}\n"
|
||||
f"Run: bash scripts/download_models.sh")
|
||||
|
||||
images_root = Path(args.images)
|
||||
if not images_root.is_dir():
|
||||
return err(f"image cache not found: {images_root}")
|
||||
|
||||
sizes = sorted({int(s) for s in args.sizes.split(",") if s.strip()})
|
||||
if not sizes:
|
||||
return err("--sizes is empty")
|
||||
if args.probes_per_actor < 1:
|
||||
return err("--probes-per-actor must be >= 1")
|
||||
|
||||
cv2.setRNGSeed(args.seed) # estimateAffinePartial2D's RANSAC draws from this
|
||||
|
||||
# ── actor pool ────────────────────────────────────────────────────────────
|
||||
pool = discover_actors(images_root)
|
||||
print(f"[select] {len(pool)} actor dirs with images under {images_root}",
|
||||
file=sys.stderr)
|
||||
if args.gallery:
|
||||
ids, names = gallery_keys(Path(args.gallery))
|
||||
pool = [a for a in pool
|
||||
if (a["jellyfin_id"] and a["jellyfin_id"] in ids)
|
||||
or normalise_name(a["name"]) in names]
|
||||
print(f"[select] {len(pool)} of them are in {args.gallery}", file=sys.stderr)
|
||||
|
||||
need = max(args.min_images, args.probes_per_actor + 1)
|
||||
eligible = [a for a in pool if len(a["images"]) >= need]
|
||||
print(f"[select] {len(eligible)} have >= {need} mugshots", file=sys.stderr)
|
||||
if len(eligible) < 2:
|
||||
return err(f"need at least 2 actors with >= {need} mugshots; found "
|
||||
f"{len(eligible)}. Build the image cache first "
|
||||
f"(scripts/make_jellyfin_gallery.py) or lower --min-images.")
|
||||
|
||||
rng = random.Random(args.seed)
|
||||
selected = sorted(rng.sample(eligible, min(args.actors, len(eligible))),
|
||||
key=lambda a: a["dir"].name)
|
||||
if len(selected) < args.actors:
|
||||
print(f"[select] WARNING: only {len(selected)} eligible actors, "
|
||||
f"--actors {args.actors} requested. FPI is gallery-size dependent — "
|
||||
f"see the caveat.", file=sys.stderr)
|
||||
|
||||
# ── detect + align every mugshot of the selected actors, once ─────────────
|
||||
stages = Stages(str(detector), str(arcface), args.conf, args.nms)
|
||||
print(f"[models] detector={detector.name} embedder={arcface.name} "
|
||||
f"batch={stages.engine.max_batch} (provider chosen by the C++ backend: "
|
||||
f"CUDA, then ROCm, then CPU)", file=sys.stderr)
|
||||
|
||||
t0 = time.time()
|
||||
crops: list[np.ndarray] = []
|
||||
rows: list[dict] = [] # parallel to crops: {actor, actor_idx, image}
|
||||
actors: list[dict] = []
|
||||
n_nodetect = 0
|
||||
for a in selected:
|
||||
actor_crops, actor_paths = [], []
|
||||
for img_path in a["images"]:
|
||||
img = cv2.imread(str(img_path))
|
||||
if img is None:
|
||||
n_nodetect += 1
|
||||
continue
|
||||
if args.max_side > 0 and max(img.shape[:2]) > args.max_side:
|
||||
s = args.max_side / max(img.shape[:2])
|
||||
img = cv2.resize(img, None, fx=s, fy=s, interpolation=cv2.INTER_AREA)
|
||||
faces = stages.detect(img)
|
||||
if not faces:
|
||||
enhanced = stages.enhance(img)
|
||||
faces = stages.detect(enhanced)
|
||||
if faces:
|
||||
img = enhanced
|
||||
if not faces:
|
||||
n_nodetect += 1
|
||||
continue
|
||||
best = max(faces, key=lambda f: f.confidence)
|
||||
crop = stages.align(img, best.landmarks)
|
||||
if crop is None:
|
||||
n_nodetect += 1
|
||||
continue
|
||||
actor_crops.append(crop)
|
||||
actor_paths.append(img_path)
|
||||
if len(actor_crops) < args.probes_per_actor + 1:
|
||||
continue
|
||||
ai = len(actors)
|
||||
actors.append({"name": a["name"], "jellyfin_id": a["jellyfin_id"],
|
||||
"dir": a["dir"].name, "n_images": len(actor_crops)})
|
||||
for crop, img_path in zip(actor_crops, actor_paths):
|
||||
rows.append({"actor_idx": ai, "image": str(img_path)})
|
||||
crops.append(crop)
|
||||
if len(actors) % 20 == 0:
|
||||
print(f" [align] {len(actors)}/{len(selected)} actors, "
|
||||
f"{len(crops)} crops", file=sys.stderr)
|
||||
|
||||
if len(actors) < 2:
|
||||
return err(f"only {len(actors)} actors survived detection/alignment — "
|
||||
f"nothing to match against")
|
||||
print(f"[align] {len(actors)} actors, {len(crops)} aligned crops, "
|
||||
f"{n_nodetect} images skipped (no face / unreadable) in "
|
||||
f"{time.time() - t0:.1f}s", file=sys.stderr)
|
||||
|
||||
actor_of = np.array([r["actor_idx"] for r in rows], dtype=int)
|
||||
|
||||
# ── embed everything at native resolution ────────────────────────────────
|
||||
t0 = time.time()
|
||||
native = stages.embed(crops)
|
||||
print(f"[embed] {len(crops)} native crops in {time.time() - t0:.1f}s",
|
||||
file=sys.stderr)
|
||||
|
||||
if args.verify_against:
|
||||
verify_embeddings(Path(args.verify_against), rows, native, actor_of)
|
||||
|
||||
# ── drop duplicate mugshots ───────────────────────────────────────────────
|
||||
# The cache holds the same photograph twice for some actors (two provider
|
||||
# URLs, one picture). A probe that is identical to a gallery reference is
|
||||
# identified for free at every size, which flatters the whole curve, so
|
||||
# remove duplicates the same way calibrate_gallery does.
|
||||
n_dup = 0
|
||||
if not args.keep_duplicates:
|
||||
keep = np.ones(len(rows), bool)
|
||||
for ai in range(len(actors)):
|
||||
kept: list[int] = []
|
||||
for i in np.nonzero(actor_of == ai)[0]:
|
||||
if any(float(native[i] @ native[k]) > DEDUP_SIM for k in kept):
|
||||
keep[i] = False
|
||||
else:
|
||||
kept.append(int(i))
|
||||
n_dup = int((~keep).sum())
|
||||
|
||||
# An actor left with too few distinct mugshots to hold one out drops out.
|
||||
counts = np.bincount(actor_of[keep], minlength=len(actors))
|
||||
drop_actor = counts < args.probes_per_actor + 1
|
||||
keep &= ~drop_actor[actor_of]
|
||||
|
||||
remap = np.full(len(actors), -1, dtype=int)
|
||||
remap[~drop_actor] = np.arange(int((~drop_actor).sum()))
|
||||
actors = [a for a, d in zip(actors, drop_actor) if not d]
|
||||
rows = [r for r, k in zip(rows, keep) if k]
|
||||
crops = [c for c, k in zip(crops, keep) if k]
|
||||
native = native[keep]
|
||||
actor_of = remap[actor_of[keep]]
|
||||
for r, ai in zip(rows, actor_of):
|
||||
r["actor_idx"] = int(ai)
|
||||
for ai, a in enumerate(actors):
|
||||
a["n_images"] = int(np.sum(actor_of == ai))
|
||||
print(f"[dedup] dropped {n_dup} duplicate mugshots and "
|
||||
f"{int(drop_actor.sum())} actors left with too few; "
|
||||
f"{len(actors)} actors, {len(rows)} images remain", file=sys.stderr)
|
||||
if len(actors) < 2:
|
||||
return err("fewer than 2 actors survive de-duplication — the image "
|
||||
"cache holds too few distinct mugshots")
|
||||
|
||||
# ── hold out the probes ───────────────────────────────────────────────────
|
||||
is_probe = np.zeros(len(rows), bool)
|
||||
for ai in range(len(actors)):
|
||||
idx = np.nonzero(actor_of == ai)[0]
|
||||
# Seeded per actor so the choice does not depend on iteration order.
|
||||
r = random.Random(f"{args.seed}:{actors[ai]['dir']}")
|
||||
for pick in r.sample(list(idx), args.probes_per_actor):
|
||||
is_probe[pick] = True
|
||||
probe_rows = np.nonzero(is_probe)[0]
|
||||
gal_rows = np.nonzero(~is_probe)[0]
|
||||
print(f"[holdout] {len(probe_rows)} probes held out, "
|
||||
f"{len(gal_rows)} gallery embeddings remain", file=sys.stderr)
|
||||
|
||||
gal_emb = native[gal_rows]
|
||||
gal_actor = actor_of[gal_rows]
|
||||
probe_actor = actor_of[probe_rows]
|
||||
|
||||
# Per-actor column masks for the best-of-N scan (identity_matcher_node).
|
||||
actor_cols = [np.nonzero(gal_actor == ai)[0] for ai in range(len(actors))]
|
||||
have_refs = np.array([len(c) > 0 for c in actor_cols])
|
||||
if not have_refs.all():
|
||||
return err("an actor ended up with no gallery references left; "
|
||||
"raise --min-images")
|
||||
|
||||
# ── calibration ───────────────────────────────────────────────────────────
|
||||
if args.calib_a is not None:
|
||||
cal = {"a": args.calib_a, "b": args.calib_b, "valid": True}
|
||||
print(f"[calibration] using supplied a={cal['a']} b={cal['b']}", file=sys.stderr)
|
||||
else:
|
||||
cal = calibrate_gallery(gal_emb, gal_actor)
|
||||
if not cal["valid"]:
|
||||
return err(
|
||||
"calibration could not be fitted, and this study will not fall back to a "
|
||||
"raw cosine threshold (CLAUDE.md invariant). Use more actors with >= "
|
||||
f"{MIN_EMB_FOR_POSITIVE} mugshots, or pass --calib-a/--calib-b from a "
|
||||
"production gallery.")
|
||||
log_prior_odds = float(np.log(args.match_prior / (1.0 - args.match_prior)))
|
||||
|
||||
# ── sweep ─────────────────────────────────────────────────────────────────
|
||||
down, up = INTERP[args.down_interp], INTERP[args.up_interp]
|
||||
probe_crops = [crops[i] for i in probe_rows]
|
||||
results, per_probe = [], []
|
||||
for size in sizes:
|
||||
t0 = time.time()
|
||||
degraded = [degrade(c, size, down, up) for c in probe_crops]
|
||||
q = stages.embed(degraded)
|
||||
|
||||
sims = q @ gal_emb.T # [n_probe, n_gal]
|
||||
best_per_actor = np.stack([sims[:, cols].max(axis=1) for cols in actor_cols],
|
||||
axis=1) # [n_probe, n_actor]
|
||||
best_actor = best_per_actor.argmax(axis=1)
|
||||
best_sim = best_per_actor.max(axis=1)
|
||||
p_match = np.asarray(probability(best_sim, cal["a"], cal["b"], log_prior_odds))
|
||||
|
||||
accept = p_match > args.prob_threshold
|
||||
correct = best_actor == probe_actor
|
||||
tpi = int(np.sum(accept & correct))
|
||||
fpi = int(np.sum(accept & ~correct))
|
||||
unid = int(np.sum(~accept))
|
||||
n = len(probe_rows)
|
||||
|
||||
results.append({
|
||||
"size_px": size,
|
||||
"n_probes": n,
|
||||
"tpi": tpi, "fpi": fpi, "unidentified": unid,
|
||||
"tpi_rate": tpi / n, "fpi_rate": fpi / n, "unidentified_rate": unid / n,
|
||||
"rank1_rate": float(np.mean(correct)),
|
||||
"mean_best_sim": float(np.mean(best_sim)),
|
||||
"mean_p_match": float(np.mean(p_match)),
|
||||
"mean_sim_true_actor": float(np.mean(
|
||||
best_per_actor[np.arange(n), probe_actor])),
|
||||
})
|
||||
if args.per_probe:
|
||||
for j in range(n):
|
||||
per_probe.append({
|
||||
"size_px": size,
|
||||
"probe_image": rows[probe_rows[j]]["image"],
|
||||
"true_actor": actors[probe_actor[j]]["name"],
|
||||
"matched_actor": actors[best_actor[j]]["name"],
|
||||
"best_sim": float(best_sim[j]),
|
||||
"p_match": float(p_match[j]),
|
||||
"outcome": ("TPI" if accept[j] and correct[j]
|
||||
else "FPI" if accept[j] else "unidentified"),
|
||||
})
|
||||
print(f"[sweep] {size:3d}px TPI {tpi:4d} ({100 * tpi / n:5.1f}%) "
|
||||
f"FPI {fpi:4d} ({100 * fpi / n:5.1f}%) "
|
||||
f"unid {unid:4d} ({100 * unid / n:5.1f}%) "
|
||||
f"rank1 {100 * np.mean(correct):5.1f}% "
|
||||
f"[{time.time() - t0:.1f}s]", file=sys.stderr)
|
||||
|
||||
op = pick_operating_point(results, args.tpi_retention, args.fpi_slack)
|
||||
|
||||
# ── outputs ───────────────────────────────────────────────────────────────
|
||||
out = Path(args.out)
|
||||
out.parent.mkdir(parents=True, exist_ok=True)
|
||||
# Append rather than with_suffix() so a prefix containing a dot keeps its name.
|
||||
csv_path = out.with_name(out.name + ".csv")
|
||||
json_path = out.with_name(out.name + ".json")
|
||||
png_path = out.with_name(out.name + ".png")
|
||||
fields = list(results[0].keys())
|
||||
with open(csv_path, "w", newline="") as f:
|
||||
w = csv.DictWriter(f, fieldnames=fields)
|
||||
w.writeheader()
|
||||
w.writerows(results)
|
||||
|
||||
meta = {
|
||||
"requirement": "VR-005",
|
||||
"caveat": CAVEAT.format(n_actors=len(actors)),
|
||||
"model": arcface.stem,
|
||||
"detector": detector.stem,
|
||||
"backend": "sae_embed / the compiled-in inference backend (fp32 ONNX under "
|
||||
"SAE_INFERENCE_BACKEND=ORT; a TensorRT fp16 build is a different "
|
||||
"embedding space)",
|
||||
"n_actors": len(actors),
|
||||
"n_probes": len(probe_rows),
|
||||
"n_gallery_embeddings": len(gal_rows),
|
||||
"probes_per_actor": args.probes_per_actor,
|
||||
"seed": args.seed,
|
||||
"sizes": sizes,
|
||||
"prob_threshold": args.prob_threshold,
|
||||
"match_prior": args.match_prior,
|
||||
"calibration": cal,
|
||||
"calibration_source": "supplied" if args.calib_a is not None else "fitted",
|
||||
"sim_boundary_at_threshold": float(
|
||||
(np.log(args.prob_threshold / (1 - args.prob_threshold))
|
||||
- cal["b"] - log_prior_odds) / cal["a"]),
|
||||
"down_interp": args.down_interp,
|
||||
"up_interp": args.up_interp,
|
||||
"max_side": args.max_side,
|
||||
"duplicate_mugshots_dropped": n_dup,
|
||||
"operating_point_rule": (
|
||||
f"smallest size retaining >= {args.tpi_retention:.0%} of the 112 px "
|
||||
f"control TPI rate with <= +{args.fpi_slack:.1%} absolute FPI"),
|
||||
"operating_point": op,
|
||||
"images_skipped_no_face": n_nodetect,
|
||||
"curve": results,
|
||||
"actors": actors,
|
||||
}
|
||||
json_path.write_text(json.dumps(meta, indent=2) + "\n")
|
||||
|
||||
if per_probe:
|
||||
pp = out.with_name(out.name + ".per_probe.csv")
|
||||
with open(pp, "w", newline="") as f:
|
||||
w = csv.DictWriter(f, fieldnames=list(per_probe[0].keys()))
|
||||
w.writeheader()
|
||||
w.writerows(per_probe)
|
||||
print(f"[out] {pp}", file=sys.stderr)
|
||||
|
||||
write_plot(results, png_path, meta)
|
||||
|
||||
# ── stdout report ─────────────────────────────────────────────────────────
|
||||
print(f"\nVR-005 — minimum face size, {arcface.stem}")
|
||||
print(f"{len(actors)} actors, {len(probe_rows)} probes/size, "
|
||||
f"{len(gal_rows)} gallery embeddings at native resolution")
|
||||
print(f"identify when P>{args.prob_threshold}, i.e. cosine above "
|
||||
f"{meta['sim_boundary_at_threshold']:.4f} under the calibration fitted "
|
||||
f"on this gallery\n")
|
||||
print(f"{'size':>5} {'TPI':>8} {'FPI':>8} {'unid':>8} {'rank1':>8} {'mean sim':>9}")
|
||||
for r in results:
|
||||
print(f"{r['size_px']:>5} {100 * r['tpi_rate']:>7.1f}% "
|
||||
f"{100 * r['fpi_rate']:>7.1f}% {100 * r['unidentified_rate']:>7.1f}% "
|
||||
f"{100 * r['rank1_rate']:>7.1f}% {r['mean_best_sim']:>9.4f}")
|
||||
print(f"\noperating point: {op if op else 'none of the swept sizes qualifies'}"
|
||||
f" ({meta['operating_point_rule']})")
|
||||
print(f"\n{meta['caveat']}")
|
||||
print(f"\n[out] {csv_path}\n[out] {json_path}\n[out] {png_path}")
|
||||
return 0
|
||||
|
||||
|
||||
def _separation(emb: np.ndarray, actor: np.ndarray) -> tuple[float, float, float]:
|
||||
"""(mean same-actor sim, mean different-actor sim, d') — the property that has
|
||||
to survive for an embedding space to be usable, whatever its coordinates."""
|
||||
iu, ju = np.triu_indices(len(actor), k=1)
|
||||
sims = (emb @ emb.T)[iu, ju]
|
||||
same = actor[iu] == actor[ju]
|
||||
pos = sims[same & (sims < 0.9999)] # drop duplicate source images
|
||||
neg = sims[~same]
|
||||
if pos.size < 2 or neg.size < 2:
|
||||
return float("nan"), float("nan"), float("nan")
|
||||
d = (pos.mean() - neg.mean()) / np.sqrt(0.5 * (pos.var() + neg.var()))
|
||||
return float(pos.mean()), float(neg.mean()), float(d)
|
||||
|
||||
|
||||
def verify_embeddings(gallery_path: Path, rows: list[dict], native: np.ndarray,
|
||||
actor_of: np.ndarray) -> None:
|
||||
"""Cross-check this script's ONNX port against a gallery built by the C++
|
||||
pipeline from the same mugshots.
|
||||
|
||||
Agreement is ~1.0 only if that gallery was built with the same backend. A
|
||||
TensorRT fp16 build lands around 0.85 on LVFace-B while separating just as
|
||||
well, so the separation figures — not the agreement — are what says whether
|
||||
the port is sound."""
|
||||
from sae_gallery import load_gallery_hdf5
|
||||
g = load_gallery_hdf5(gallery_path)
|
||||
stored: dict[tuple[str, str], np.ndarray] = {}
|
||||
for a in g["actors"]:
|
||||
key = a.get("jellyfin_id") or normalise_name(a.get("name", ""))
|
||||
for e, src in zip(a.get("embeddings", []), a.get("source_images", [])):
|
||||
if src:
|
||||
stored[(key, src)] = np.asarray(e, np.float32)
|
||||
|
||||
sims, paired_mine, paired_ref, paired_actor = [], [], [], []
|
||||
for i, r in enumerate(rows):
|
||||
path = Path(r["image"])
|
||||
head, _, _ = path.parent.name.partition("_")
|
||||
key = head if JELLYFIN_ID_RE.match(head) else normalise_name(
|
||||
path.parent.name.replace("_", " "))
|
||||
ref = stored.get((key, path.name))
|
||||
if ref is None or ref.shape != native[i].shape:
|
||||
continue
|
||||
ref = ref / max(float(np.linalg.norm(ref)), 1e-6)
|
||||
sims.append(float(native[i] @ ref))
|
||||
paired_mine.append(native[i])
|
||||
paired_ref.append(ref)
|
||||
paired_actor.append(actor_of[i])
|
||||
if not sims:
|
||||
print(f"[verify] no overlap with {gallery_path} — nothing checked",
|
||||
file=sys.stderr)
|
||||
return
|
||||
|
||||
sims_arr = np.asarray(sims)
|
||||
print(f"[verify] {len(sims)} embeddings vs {gallery_path.name}: "
|
||||
f"mean cos={sims_arr.mean():.4f} min={sims_arr.min():.4f}",
|
||||
file=sys.stderr)
|
||||
act = np.asarray(paired_actor)
|
||||
for label, mat in (("this script", np.asarray(paired_mine)),
|
||||
("stored gallery", np.asarray(paired_ref))):
|
||||
pos, neg, d = _separation(mat, act)
|
||||
print(f"[verify] {label:>14s}: same-actor {pos:.3f} "
|
||||
f"different-actor {neg:.3f} d'={d:.2f}", file=sys.stderr)
|
||||
if sims_arr.mean() < 0.99:
|
||||
print("[verify] embeddings differ from the stored gallery. If d' is "
|
||||
"comparable this is a backend difference (e.g. a TensorRT fp16 "
|
||||
"build), not a broken port; the study is self-consistent either "
|
||||
"way. If d' collapsed, the port is wrong.", file=sys.stderr)
|
||||
|
||||
|
||||
def err(msg: str) -> int:
|
||||
print(f"error: {msg}", file=sys.stderr)
|
||||
return 2
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
@@ -1,370 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
VR-014 — the v1 audio signature recovers a known trim offset on real audio.
|
||||
|
||||
TRACES: UT-105, UT-106, UT-107, UT-108 | VR-014 | IR-004
|
||||
|
||||
python scripts/validation/test_audio_offset.py [build_dir]
|
||||
|
||||
The golden vector (IR-005) proves the *arithmetic* is identical in both
|
||||
producers. It cannot prove the thing the signature exists for: that when the
|
||||
same cut arrives trimmed differently, sliding one signature against the other
|
||||
finds the true alignment and only the true alignment. Its fixture is a synthetic
|
||||
tone sweep, which is pathologically easy to align; film dialogue and score are
|
||||
not, and that is what this measures.
|
||||
|
||||
The signature is computed by the **shipped C++**, through the `sae_audio`
|
||||
nanobind module — never a numpy port. A third implementation of a fingerprint
|
||||
whose whole value rests on three implementations agreeing byte for byte would be
|
||||
the one nobody checks against the golden vector.
|
||||
|
||||
The slide *is* written here in numpy, deliberately: matching is the consumer's
|
||||
algorithm (server SPEC.md section 3), owned by the server and the jRay plugin,
|
||||
not by this repo. Writing it out is what makes this a test of the signature
|
||||
rather than a test of somebody's matcher.
|
||||
|
||||
Two independent offset mechanisms are checked, because they can fail
|
||||
separately:
|
||||
|
||||
* a **window offset** (UT-105) — two 120 s excerpts taken from different
|
||||
points, which is the alignment search itself; and
|
||||
* a **head trim** (UT-106) — a real file with delta seconds removed from the
|
||||
front, which additionally exercises the runtime/2 anchor: the window follows
|
||||
the midpoint, so cutting delta from the head moves it by delta/2, not delta.
|
||||
That factor of two is the easiest thing in the whole feature to get wrong
|
||||
and nothing else checks it.
|
||||
"""
|
||||
|
||||
import base64
|
||||
import random
|
||||
import shutil
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
from pathlib import Path
|
||||
|
||||
import numpy as np
|
||||
|
||||
REPO = Path(__file__).resolve().parent.parent.parent
|
||||
BUILD = Path(sys.argv[1]) if len(sys.argv) > 1 else REPO / "build"
|
||||
sys.path.insert(0, str(BUILD))
|
||||
|
||||
import sae_audio # noqa: E402
|
||||
|
||||
FIXTURE = REPO / "tests" / "fixtures" / "audio" / "superhero_offset_200s.flac"
|
||||
TONE = REPO / "tests" / "fixtures" / "audio" / "jray_audio_v1_tone.flac"
|
||||
|
||||
# Server SPEC.md section 3, "Matching and offset recovery". The cap is the
|
||||
# spec's, not a convenience: +/-600 frames is ~56 s, which covers realistic trim
|
||||
# differences, and an offset outside it must be declined rather than guessed at.
|
||||
SEARCH_CAP_FRAMES = 600
|
||||
AUDIO_TIER = 0.85
|
||||
LOOSE_TIER = 0.60
|
||||
|
||||
TRIALS = 40
|
||||
SEED = 20250731
|
||||
|
||||
HOP_SEC = sae_audio.hop_size / sae_audio.sample_rate
|
||||
|
||||
# What the offset is actually *for*: shifting scene windows, which are seconds
|
||||
# long. Half a second of error is invisible against them, and that budget is
|
||||
# what makes the numbers below readable — an offset is quantised to whole
|
||||
# frames, so no correct answer can be worse than half a frame (46 ms) and the
|
||||
# feature has an order of magnitude in hand before anything is at stake.
|
||||
OFFSET_BUDGET_SEC = 0.5
|
||||
|
||||
|
||||
def peak_bins(signature):
|
||||
"""The per-frame peak band index, which is what the slide compares.
|
||||
|
||||
The `v1:` prefix is checked against the constant the C++ exports rather
|
||||
than a literal, so a producer bump cannot be silently parsed as v1 here
|
||||
(IR-008).
|
||||
"""
|
||||
prefix = sae_audio.version_prefix
|
||||
if not signature.startswith(prefix):
|
||||
raise AssertionError(f"signature is not {prefix!r}: {signature[:8]!r}")
|
||||
packed = np.frombuffer(base64.b64decode(signature[len(prefix):]), dtype=np.uint8)
|
||||
if np.any(packed & 0x80):
|
||||
raise AssertionError("reserved bit set — not a structurally valid signature")
|
||||
return packed >> 2
|
||||
|
||||
|
||||
def best_match(reference, query, cap=SEARCH_CAP_FRAMES, slack=0):
|
||||
"""Slide `query` against `reference`; return (score, offset_frames).
|
||||
|
||||
`offset` is how many frames later the query's window begins, so
|
||||
``query[i]`` lines up with ``reference[i + offset]``. Score is the fraction
|
||||
of overlapping frames whose peak bin agrees, exactly as the spec defines it.
|
||||
|
||||
`slack` widens what counts as agreement to a frame within +/-slack, which is
|
||||
not the spec's rule — it is the candidate remedy UT-108 measures. It changes
|
||||
the *score* only; the offset it reports is still a whole-frame alignment.
|
||||
"""
|
||||
best_score, best_offset = -1.0, 0
|
||||
for offset in range(-cap, cap + 1):
|
||||
if offset >= 0:
|
||||
a, b = reference[offset:], query[: len(query) - offset]
|
||||
else:
|
||||
a, b = reference[: len(reference) + offset], query[-offset:]
|
||||
n = min(len(a), len(b))
|
||||
if n < 100: # too little overlap to mean anything
|
||||
continue
|
||||
a, b = a[:n], b[:n]
|
||||
if slack == 0:
|
||||
agree = a == b
|
||||
else:
|
||||
agree = np.zeros(n, dtype=bool)
|
||||
for shift in range(-slack, slack + 1):
|
||||
shifted = np.roll(a, shift)
|
||||
# 255 is not a band index, so the wrapped end can never agree.
|
||||
if shift > 0:
|
||||
shifted[:shift] = 255
|
||||
elif shift < 0:
|
||||
shifted[shift:] = 255
|
||||
agree |= shifted == b
|
||||
score = float(np.mean(agree))
|
||||
if score > best_score:
|
||||
best_score, best_offset = score, offset
|
||||
return best_score, best_offset
|
||||
|
||||
|
||||
def decode_mono(path):
|
||||
"""The whole fixture as float32 mono at 11025 Hz — the signature's own rate."""
|
||||
raw = subprocess.run(
|
||||
["ffmpeg", "-nostdin", "-v", "error", "-i", str(path),
|
||||
"-ac", "1", "-ar", str(sae_audio.sample_rate), "-f", "f32le", "-"],
|
||||
capture_output=True, check=True).stdout
|
||||
return np.frombuffer(raw, dtype="<f4")
|
||||
|
||||
|
||||
def trim_head(source, seconds, out):
|
||||
"""`source` with `seconds` removed from the front — a differently trimmed release."""
|
||||
subprocess.run(
|
||||
["ffmpeg", "-nostdin", "-v", "error", "-y", "-ss", f"{seconds:.3f}", "-i", str(source),
|
||||
"-ac", "1", "-ar", str(sae_audio.sample_rate), "-sample_fmt", "s16",
|
||||
"-c:a", "flac", str(out)], check=True)
|
||||
return out
|
||||
|
||||
|
||||
def tier(score):
|
||||
if score >= AUDIO_TIER:
|
||||
return "audio"
|
||||
return "loose" if score >= LOOSE_TIER else "none"
|
||||
|
||||
|
||||
# ── The random trial set, signed once and reused ─────────────────────────────
|
||||
|
||||
def random_trials(pcm):
|
||||
"""(reference bins, [(expected_frames, query bins)]) for TRIALS excerpts.
|
||||
|
||||
Signing 40 windows is the expensive part of this file, so UT-105 and UT-108
|
||||
share one set — they ask different questions of the same measurements.
|
||||
"""
|
||||
window = sae_audio.window_samples
|
||||
reference = peak_bins(sae_audio.signature_from_mono(pcm[:window]))
|
||||
rng = random.Random(SEED)
|
||||
queries = []
|
||||
|
||||
for _ in range(TRIALS):
|
||||
# Within the search cap: past it, no offset is recoverable by
|
||||
# construction, which UT-107 checks separately.
|
||||
start = rng.randrange(0, SEARCH_CAP_FRAMES * sae_audio.hop_size)
|
||||
signature = sae_audio.signature_from_mono(pcm[start:start + window])
|
||||
assert signature is not None, "a full window must always sign"
|
||||
queries.append((start / sae_audio.hop_size, peak_bins(signature)))
|
||||
|
||||
return reference, queries
|
||||
|
||||
|
||||
# ── UT-105 — window offsets from random excerpt starts ───────────────────────
|
||||
|
||||
def test_random_window_offsets(reference, queries):
|
||||
"""Every in-cap offset is recovered to the nearest frame, on real audio."""
|
||||
rows = []
|
||||
for want, query in queries:
|
||||
score, offset = best_match(reference, query)
|
||||
rows.append((want, offset, score, abs(want - round(want))))
|
||||
|
||||
expected = np.array([r[0] for r in rows])
|
||||
offset = np.array([r[1] for r in rows])
|
||||
score = np.array([r[2] for r in rows])
|
||||
subframe = np.array([r[3] for r in rows])
|
||||
error = np.abs(offset - expected)
|
||||
|
||||
# The offset is quantised to whole frames, so the best any correct answer
|
||||
# can do is half a frame — 46 ms. What matters is the budget that half-frame
|
||||
# is measured against, and it is an order of magnitude away from it.
|
||||
assert error.max() <= 1.0, f"offset missed by {error.max():.2f} frames"
|
||||
assert error.max() * HOP_SEC <= OFFSET_BUDGET_SEC, (
|
||||
f"offset error {error.max() * HOP_SEC:.3f}s exceeds the {OFFSET_BUDGET_SEC}s budget")
|
||||
# Never mistaken for different content. This is the floor that matters: the
|
||||
# audio genuinely is the same cut, so a "no match" would be a false negative
|
||||
# on the case the feature exists for.
|
||||
assert score.min() >= LOOSE_TIER, f"same content scored {score.min():.3f}"
|
||||
# An offset that lands near a frame boundary has no excuse: it should reach
|
||||
# the top tier, and does.
|
||||
aligned = subframe <= 0.1
|
||||
assert aligned.any(), "seed no longer produces a near-aligned trial"
|
||||
assert score[aligned].min() >= AUDIO_TIER, (
|
||||
f"near-frame-aligned offset scored only {score[aligned].min():.3f}")
|
||||
|
||||
print(f"UT-105 {TRIALS} random window offsets, all within the +/-600 frame cap")
|
||||
print(f" offset error : max {error.max():.2f} frames"
|
||||
f" = {error.max() * HOP_SEC * 1000:.0f} ms, against a"
|
||||
f" {OFFSET_BUDGET_SEC * 1000:.0f} ms budget")
|
||||
print(f" score : min {score.min():.3f} median {np.median(score):.3f}"
|
||||
f" max {score.max():.3f}")
|
||||
print(" score by sub-frame misalignment — the offset is exact in every row:")
|
||||
for lo, hi in ((0.0, 0.1), (0.1, 0.2), (0.2, 0.3), (0.3, 0.4), (0.4, 0.5)):
|
||||
m = (subframe >= lo) & (subframe < hi)
|
||||
if m.any():
|
||||
print(f" {lo:.1f}-{hi:.1f} frame n={m.sum():2d}"
|
||||
f" score {score[m].min():.3f}-{score[m].max():.3f}"
|
||||
f" tier {tier(np.median(score[m]))}")
|
||||
counts = {t: int(sum(1 for s in score if tier(s) == t)) for t in ("audio", "loose", "none")}
|
||||
print(f" tiers : {counts}")
|
||||
return counts
|
||||
|
||||
|
||||
# ── UT-106 — head trims through real files, including the runtime/2 anchor ───
|
||||
|
||||
def test_head_trims():
|
||||
"""A release with delta seconds of head removed aligns at delta/2 frames."""
|
||||
reference = peak_bins(sae_audio.compute_signature(str(FIXTURE)))
|
||||
results = []
|
||||
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
for delta in (7.0, 23.5, 41.25, 60.0):
|
||||
trimmed = trim_head(FIXTURE, delta, Path(tmp) / f"trim_{delta}.flac")
|
||||
signature = sae_audio.compute_signature(str(trimmed))
|
||||
assert signature is not None, f"trim of {delta}s should still sign"
|
||||
score, offset = best_match(reference, peak_bins(signature))
|
||||
# The window follows the midpoint, so removing delta from the head
|
||||
# moves it by delta/2 — not by delta.
|
||||
expected = (delta / 2.0) / HOP_SEC
|
||||
assert abs(offset - expected) <= 1.0, (
|
||||
f"head trim {delta}s: expected ~{expected:.1f} frames, got {offset}")
|
||||
assert score >= LOOSE_TIER, f"head trim {delta}s scored {score:.3f}"
|
||||
results.append((delta, expected, offset, score))
|
||||
|
||||
print("UT-106 head trims through the real decode path (compute_signature on a file)")
|
||||
for delta, expected, offset, score in results:
|
||||
print(f" -{delta:6.2f}s head expected {expected:7.2f} fr"
|
||||
f" recovered {offset:5d} score {score:.3f} ({tier(score)})")
|
||||
|
||||
|
||||
# ── UT-107 — what must NOT match ─────────────────────────────────────────────
|
||||
|
||||
def test_declines(pcm, reference):
|
||||
"""Out-of-cap offsets and unrelated content are declined, not guessed at."""
|
||||
window = sae_audio.window_samples
|
||||
|
||||
beyond = int(75.0 * sae_audio.sample_rate) # ~807 frames, past the cap
|
||||
assert beyond + window <= len(pcm), "fixture too short for the out-of-cap case"
|
||||
far = peak_bins(sae_audio.signature_from_mono(pcm[beyond:beyond + window]))
|
||||
score_beyond, offset_beyond = best_match(reference, far)
|
||||
assert score_beyond < LOOSE_TIER, (
|
||||
f"an offset past the cap scored {score_beyond:.3f} at {offset_beyond} — "
|
||||
"the search invented an alignment rather than declining")
|
||||
|
||||
tone = peak_bins(sae_audio.compute_signature(str(TONE)))
|
||||
score_tone, offset_tone = best_match(reference, tone)
|
||||
assert score_tone < LOOSE_TIER, f"unrelated content scored {score_tone:.3f}"
|
||||
|
||||
print("UT-107 declines rather than guesses")
|
||||
print(f" offset past the +/-600 frame cap : best {score_beyond:.3f}"
|
||||
f" at {offset_beyond} ({tier(score_beyond)})")
|
||||
print(f" unrelated content (tone fixture) : best {score_tone:.3f}"
|
||||
f" at {offset_tone} ({tier(score_tone)})")
|
||||
return far, tone
|
||||
|
||||
|
||||
# ── UT-108 — the sub-frame demotion, and what one frame of slack costs ───────
|
||||
|
||||
def test_scoring_slack(reference, queries, far, tone):
|
||||
"""Measured: +/-1 frame of slack in the *score* restores the `audio` tier.
|
||||
|
||||
UT-105 leaves a real question open. Every offset is right, but two thirds of
|
||||
them score below the server's 0.85 `audio` threshold purely because the two
|
||||
windows' frame grids do not coincide — so a correctly aligned release is
|
||||
demoted to `loose`, which is the tier meaning "possibly the same cut,
|
||||
degraded audio". The obvious remedy is to stop demanding that frames line up
|
||||
exactly, and the question is what that costs in discrimination.
|
||||
|
||||
Nothing is asserted about the spec's own rule here; this measures a
|
||||
candidate change to it, which is the server's to make (SPEC.md section 3).
|
||||
"""
|
||||
print("UT-108 cost of relaxing the score's frame alignment")
|
||||
print(f" {'slack':>5} {'audio':>6} {'loose':>6} {'none':>5}"
|
||||
f" {'min true':>9} {'worst err':>10} {'unrelated':>10} {'out-of-cap':>11}")
|
||||
|
||||
measured = {}
|
||||
for slack in (0, 1, 2):
|
||||
score, error = [], []
|
||||
for want, query in queries:
|
||||
s, offset = best_match(reference, query, slack=slack)
|
||||
score.append(s)
|
||||
error.append(abs(offset - want))
|
||||
score, error = np.array(score), np.array(error)
|
||||
false_tone, _ = best_match(reference, tone, slack=slack)
|
||||
false_far, _ = best_match(reference, far, slack=slack)
|
||||
counts = {t: int(sum(1 for s in score if tier(s) == t)) for t in ("audio", "loose", "none")}
|
||||
measured[slack] = (score, error, max(false_tone, false_far))
|
||||
print(f" {slack:>5} {counts['audio']:>6} {counts['loose']:>6} {counts['none']:>5}"
|
||||
f" {score.min():>9.3f} {error.max() * HOP_SEC * 1000:>7.0f} ms"
|
||||
f" {false_tone:>10.3f} {false_far:>11.3f}")
|
||||
|
||||
score, error, worst_false = measured[1]
|
||||
# One frame of slack lifts every correct alignment to the top tier...
|
||||
assert score.min() >= AUDIO_TIER, (
|
||||
f"one frame of slack still leaves a true match at {score.min():.3f}")
|
||||
# ...without narrowing the gap that makes the threshold mean anything...
|
||||
assert worst_false < LOOSE_TIER, (
|
||||
f"slack lifted a false match to {worst_false:.3f}")
|
||||
# ...and the offset it costs is still far inside the budget: the score's
|
||||
# peak flattens slightly, so the argmax can pick an adjacent frame.
|
||||
assert error.max() * HOP_SEC <= OFFSET_BUDGET_SEC, (
|
||||
f"slack cost {error.max() * HOP_SEC:.3f}s of offset accuracy")
|
||||
print(f" +/-1 frame: every true match reaches `audio` (min {score.min():.3f}),"
|
||||
f" worst false stays at {worst_false:.3f},")
|
||||
print(f" and the offset costs {error.max() * HOP_SEC * 1000:.0f} ms of a"
|
||||
f" {OFFSET_BUDGET_SEC * 1000:.0f} ms budget. +/-2 buys nothing more.")
|
||||
|
||||
|
||||
def main():
|
||||
if not FIXTURE.exists():
|
||||
print(f"missing fixture {FIXTURE} — regenerate with make_offset_fixture.sh", file=sys.stderr)
|
||||
return 2
|
||||
if shutil.which("ffmpeg") is None:
|
||||
print("this validation needs the ffmpeg CLI to trim the fixture", file=sys.stderr)
|
||||
return 2
|
||||
|
||||
print(f"VR-014 audio-signature offset recovery on {FIXTURE.name}")
|
||||
print(f" {sae_audio.expected_frames} frames per signature,"
|
||||
f" {HOP_SEC * 1000:.2f} ms per frame, cap +/-{SEARCH_CAP_FRAMES} frames")
|
||||
|
||||
pcm = decode_mono(FIXTURE)
|
||||
reference, queries = random_trials(pcm)
|
||||
counts = test_random_window_offsets(reference, queries)
|
||||
test_head_trims()
|
||||
far, tone = test_declines(pcm, reference)
|
||||
test_scoring_slack(reference, queries, far, tone)
|
||||
|
||||
print()
|
||||
print(f"PASS — every in-cap offset recovered to the nearest frame, worst"
|
||||
f" {1000 * HOP_SEC / 2:.0f} ms against a {OFFSET_BUDGET_SEC * 1000:.0f} ms budget.")
|
||||
if counts["audio"] < TRIALS:
|
||||
# Stated rather than asserted against the spec's rule: the offset is
|
||||
# right in every case, so this is the 0.85 threshold meeting a sub-frame
|
||||
# shift, not a defect in the signature. The threshold was calibrated on
|
||||
# a re-encode at zero offset, where the score is 1.00. UT-108 measures
|
||||
# the remedy; adopting it is the server spec's call, not this repo's.
|
||||
print(f"NOTE — under the spec's exact-frame score only {counts['audio']}/{TRIALS}"
|
||||
f" reach `audio`; {counts['loose']} are demoted to `loose` by sub-frame"
|
||||
" shift alone. See UT-108.")
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
Vendored
-1
Submodule scripts/vendor/jray-project deleted from 17106f3370
@@ -1,125 +0,0 @@
|
||||
// sae_audio — Python module wrapping the v1 audio signature (audio_signature.*).
|
||||
//
|
||||
/// TRACES: IR-004, IR-005 | SR-003
|
||||
//
|
||||
// Exists so a study or a test can drive the **shipped** signature code from
|
||||
// Python instead of porting the DSP to numpy. A numpy port would be a third
|
||||
// implementation of a fingerprint that only works if every implementation
|
||||
// agrees byte for byte, and it would be the one nobody checks against the
|
||||
// golden vector — so the offset-recovery validation (VR-014) calls this.
|
||||
//
|
||||
// Bound with nanobind, as `sae_embed` and `sae_kpn` are. Not pybind11: a second
|
||||
// binding framework in one build is a second set of ABI and lifetime rules to
|
||||
// get right, for a module that needs nothing nanobind lacks.
|
||||
//
|
||||
// The module deliberately stops at the producer's edge. Matching — sliding one
|
||||
// signature against another and scoring the overlap — is the *consumer's*
|
||||
// algorithm (server SPEC §3, and the jRay plugin implements it), so it is not
|
||||
// bound here and a caller writing a slide in numpy is not re-implementing
|
||||
// anything this repo owns.
|
||||
|
||||
#include "audio_signature.hpp"
|
||||
|
||||
#include <nanobind/nanobind.h>
|
||||
#include <nanobind/ndarray.h>
|
||||
#include <nanobind/stl/optional.h>
|
||||
#include <nanobind/stl/pair.h>
|
||||
#include <nanobind/stl/string.h>
|
||||
#include <nanobind/stl/vector.h>
|
||||
|
||||
#include <cstdint>
|
||||
#include <optional>
|
||||
#include <string>
|
||||
#include <utility>
|
||||
#include <vector>
|
||||
|
||||
namespace nb = nanobind;
|
||||
using namespace nb::literals;
|
||||
using namespace sae::audio;
|
||||
|
||||
namespace {
|
||||
|
||||
using MonoArray = nb::ndarray<const float, nb::ndim<1>, nb::c_contig, nb::device::cpu>;
|
||||
|
||||
// Hand the vector's buffer to Python without copying 1.3 M samples, and let a
|
||||
// capsule own it: the array outlives this call, so the storage has to as well.
|
||||
nb::object own_as_ndarray(std::vector<float>&& samples) {
|
||||
auto* held = new std::vector<float>(std::move(samples));
|
||||
nb::capsule owner(held, [](void* p) noexcept {
|
||||
delete static_cast<std::vector<float>*>(p);
|
||||
});
|
||||
const std::size_t n = held->size();
|
||||
return nb::cast(nb::ndarray<nb::numpy, float, nb::ndim<1>>(held->data(), {n}, owner));
|
||||
}
|
||||
|
||||
std::vector<float> to_vector(const MonoArray& a) {
|
||||
return std::vector<float>(a.data(), a.data() + a.shape(0));
|
||||
}
|
||||
|
||||
} // namespace
|
||||
|
||||
NB_MODULE(sae_audio, m) {
|
||||
m.doc() =
|
||||
"JRay v1 audio signature (JRay-public-server SPEC.md section 3), as the "
|
||||
"extraction pipeline computes it. The constants below are the contract: "
|
||||
"changing any of them is a v1 -> v2 change.";
|
||||
|
||||
m.attr("sample_rate") = kSampleRate;
|
||||
m.attr("frame_size") = kFrameSize;
|
||||
m.attr("hop_size") = kHopSize;
|
||||
m.attr("num_bands") = kNumBands;
|
||||
m.attr("band_lo_hz") = kBandLoHz;
|
||||
m.attr("band_hi_hz") = kBandHiHz;
|
||||
m.attr("window_sec") = kWindowSec;
|
||||
m.attr("window_samples") = kWindowSamples;
|
||||
m.attr("expected_frames") = kExpectedFrames;
|
||||
m.attr("version_prefix") = std::string(kVersionPrefix);
|
||||
|
||||
m.def(
|
||||
"compute_signature",
|
||||
[](const std::string& path) { return compute_signature(path); },
|
||||
"path"_a,
|
||||
"Signature of the 120 s window centred on the media's midpoint, or None "
|
||||
"for media shorter than the window (IR-007), media with no audio "
|
||||
"stream, and any decode failure — degradation, never an exception.");
|
||||
|
||||
m.def(
|
||||
"decode_centre_window",
|
||||
[](const std::string& path) -> nb::object {
|
||||
std::optional<std::vector<float>> mono = decode_centre_window(path);
|
||||
if (!mono) {
|
||||
return nb::none();
|
||||
}
|
||||
|
||||
return own_as_ndarray(std::move(*mono));
|
||||
},
|
||||
"path"_a,
|
||||
"The decoded centre window as float32 mono at 11025 Hz, or None. Exposed "
|
||||
"so a caller can slice or perturb real audio and re-sign it without "
|
||||
"going back through a container.");
|
||||
|
||||
m.def(
|
||||
"signature_from_mono",
|
||||
[](const MonoArray& mono) { return signature_from_mono(to_vector(mono)); },
|
||||
"mono"_a,
|
||||
"Signature of mono float32 samples already at 11025 Hz, in [-1, 1). None "
|
||||
"when fewer than one whole frame is given.");
|
||||
|
||||
m.def(
|
||||
"pack_frames",
|
||||
[](const MonoArray& mono) {
|
||||
std::vector<std::uint8_t> packed = pack_frames(to_vector(mono));
|
||||
return nb::bytes(reinterpret_cast<const char*>(packed.data()), packed.size());
|
||||
},
|
||||
"mono"_a,
|
||||
"One packed byte per whole STFT frame: (band << 2) | energy_class. This "
|
||||
"is the payload the signature base64-encodes.");
|
||||
|
||||
m.def(
|
||||
"band_fft_bins",
|
||||
[] {
|
||||
const auto& table = band_fft_bins();
|
||||
return std::vector<std::pair<int, int>>(table.begin(), table.end());
|
||||
},
|
||||
"The half-open FFT bin range owned by each of the 32 log-spaced bands.");
|
||||
}
|
||||
@@ -1,428 +0,0 @@
|
||||
// ── JRay audio signature, v1 — implementation ────────────────────────────────
|
||||
//
|
||||
/// TRACES: IR-004, IR-007, IR-008 | SR-003
|
||||
//
|
||||
// The contract this implements is documented in full in audio_signature.hpp;
|
||||
// read that before changing anything here. Every constant is load-bearing: the
|
||||
// JRay Jellyfin plugin computes the same bytes in C#, and a signature that
|
||||
// differs in any parameter simply does not match.
|
||||
//
|
||||
// Audio decode is a *second stream from an existing dependency* — the pipeline
|
||||
// already links libavformat/libavcodec/libavutil for video (ffmpeg_decoder.hpp);
|
||||
// this adds libswresample for the downmix+resample, no new project dependency.
|
||||
// The FFT is written out here rather than pulled from a library for the same
|
||||
// reason the plugin vendors one: it is a fixed, fully specified transform, and
|
||||
// a dependency whose version could change the numerics is a liability when the
|
||||
// output has to be bit-identical across two languages.
|
||||
|
||||
#include "audio_signature.hpp"
|
||||
|
||||
extern "C" {
|
||||
#include <libavcodec/avcodec.h>
|
||||
#include <libavformat/avformat.h>
|
||||
#include <libavutil/avutil.h>
|
||||
#include <libavutil/channel_layout.h>
|
||||
#include <libavutil/opt.h>
|
||||
#include <libavutil/samplefmt.h>
|
||||
#include <libswresample/swresample.h>
|
||||
}
|
||||
|
||||
#include <algorithm>
|
||||
#include <cmath>
|
||||
#include <cstring>
|
||||
#include <vector>
|
||||
|
||||
namespace sae::audio {
|
||||
namespace {
|
||||
|
||||
constexpr double kPi = 3.14159265358979323846;
|
||||
|
||||
// ── Band table ───────────────────────────────────────────────────────────────
|
||||
// edge[b] = 300 * 10^(b/32); band b owns FFT bins [k_lo[b], k_lo[b+1]).
|
||||
// ceil() of the edge in bins, so membership is decided once by integers rather
|
||||
// than by a float comparison per bin per frame. The bands tile [112, 1115)
|
||||
// contiguously with no gap and no overlap, which is what lets the frame energy
|
||||
// below be accumulated from the per-band sums.
|
||||
std::array<std::pair<int, int>, kNumBands> build_band_table() {
|
||||
const double hz_per_bin = static_cast<double>(kSampleRate) / kFrameSize;
|
||||
std::array<int, kNumBands + 1> k{};
|
||||
for (int b = 0; b <= kNumBands; ++b) {
|
||||
const double edge = kBandLoHz * std::pow(kBandHiHz / kBandLoHz,
|
||||
static_cast<double>(b) / kNumBands);
|
||||
k[b] = static_cast<int>(std::ceil(edge / hz_per_bin));
|
||||
}
|
||||
std::array<std::pair<int, int>, kNumBands> tbl{};
|
||||
for (int b = 0; b < kNumBands; ++b) tbl[b] = {k[b], k[b + 1]};
|
||||
return tbl;
|
||||
}
|
||||
|
||||
// Hann, periodic: w[n] = 0.5 * (1 - cos(2*pi*n/N)). Not the symmetric (N-1)
|
||||
// variant — the two differ, and the difference is observable.
|
||||
const std::vector<double>& hann_window() {
|
||||
static const std::vector<double> w = [] {
|
||||
std::vector<double> v(kFrameSize);
|
||||
for (int n = 0; n < kFrameSize; ++n)
|
||||
v[n] = 0.5 * (1.0 - std::cos(2.0 * kPi * n / kFrameSize));
|
||||
return v;
|
||||
}();
|
||||
return w;
|
||||
}
|
||||
|
||||
// ── Radix-2 decimation-in-time complex FFT, in place, no normalisation ──────
|
||||
// Twiddles are precomputed per stage from cos/sin of -2*pi*j/len so the angle
|
||||
// is an exactly reproducible double in any language and only the libm rounding
|
||||
// of cos/sin (≤1 ulp) can differ — orders of magnitude below the decision
|
||||
// margins in the golden fixture.
|
||||
struct FftTables {
|
||||
std::vector<int> rev; // bit-reversal permutation
|
||||
std::vector<std::vector<double>> wr, wi; // per stage
|
||||
};
|
||||
|
||||
const FftTables& fft_tables() {
|
||||
static const FftTables t = [] {
|
||||
FftTables f;
|
||||
f.rev.resize(kFrameSize);
|
||||
int bits = 0;
|
||||
while ((1 << bits) < kFrameSize) ++bits;
|
||||
for (int i = 0; i < kFrameSize; ++i) {
|
||||
int r = 0;
|
||||
for (int b = 0; b < bits; ++b)
|
||||
if (i & (1 << b)) r |= 1 << (bits - 1 - b);
|
||||
f.rev[i] = r;
|
||||
}
|
||||
for (int len = 2; len <= kFrameSize; len <<= 1) {
|
||||
const int half = len / 2;
|
||||
std::vector<double> cr(half), ci(half);
|
||||
for (int j = 0; j < half; ++j) {
|
||||
const double ang = -2.0 * kPi * j / len;
|
||||
cr[j] = std::cos(ang);
|
||||
ci[j] = std::sin(ang);
|
||||
}
|
||||
f.wr.push_back(std::move(cr));
|
||||
f.wi.push_back(std::move(ci));
|
||||
}
|
||||
return f;
|
||||
}();
|
||||
return t;
|
||||
}
|
||||
|
||||
void fft_4096(std::vector<double>& re, std::vector<double>& im) {
|
||||
const FftTables& t = fft_tables();
|
||||
for (int i = 0; i < kFrameSize; ++i) {
|
||||
const int j = t.rev[i];
|
||||
if (i < j) { std::swap(re[i], re[j]); std::swap(im[i], im[j]); }
|
||||
}
|
||||
int stage = 0;
|
||||
for (int len = 2; len <= kFrameSize; len <<= 1, ++stage) {
|
||||
const int half = len / 2;
|
||||
const std::vector<double>& wr = t.wr[stage];
|
||||
const std::vector<double>& wi = t.wi[stage];
|
||||
for (int base = 0; base < kFrameSize; base += len) {
|
||||
for (int j = 0; j < half; ++j) {
|
||||
const int a = base + j;
|
||||
const int b = a + half;
|
||||
const double tr = re[b] * wr[j] - im[b] * wi[j];
|
||||
const double ti = re[b] * wi[j] + im[b] * wr[j];
|
||||
re[b] = re[a] - tr; im[b] = im[a] - ti;
|
||||
re[a] = re[a] + tr; im[a] = im[a] + ti;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
int energy_class(double r) {
|
||||
if (r < kEnergyClassEdges[0]) return 0;
|
||||
if (r < kEnergyClassEdges[1]) return 1;
|
||||
if (r < kEnergyClassEdges[2]) return 2;
|
||||
return 3;
|
||||
}
|
||||
|
||||
// ── FFmpeg RAII ─────────────────────────────────────────────────────────────
|
||||
struct DecodeCtx {
|
||||
AVFormatContext* fmt = nullptr;
|
||||
AVCodecContext* dec = nullptr;
|
||||
SwrContext* swr = nullptr;
|
||||
AVFrame* frm = nullptr;
|
||||
AVPacket* pkt = nullptr;
|
||||
~DecodeCtx() {
|
||||
if (swr) swr_free(&swr);
|
||||
if (frm) av_frame_free(&frm);
|
||||
if (pkt) av_packet_free(&pkt);
|
||||
if (dec) avcodec_free_context(&dec);
|
||||
if (fmt) avformat_close_input(&fmt);
|
||||
}
|
||||
};
|
||||
|
||||
bool open_resampler(DecodeCtx& c, const AVFrame* f) {
|
||||
#if LIBAVUTIL_VERSION_INT >= AV_VERSION_INT(57, 24, 100)
|
||||
AVChannelLayout out_layout;
|
||||
av_channel_layout_default(&out_layout, 1); // mono
|
||||
AVChannelLayout in_layout;
|
||||
if (av_channel_layout_copy(&in_layout, &f->ch_layout) < 0) return false;
|
||||
if (in_layout.nb_channels <= 0) {
|
||||
av_channel_layout_uninit(&in_layout);
|
||||
av_channel_layout_default(&in_layout, 1);
|
||||
}
|
||||
const int rc = swr_alloc_set_opts2(
|
||||
&c.swr,
|
||||
&out_layout, AV_SAMPLE_FMT_FLT, kSampleRate,
|
||||
&in_layout, static_cast<AVSampleFormat>(f->format),
|
||||
f->sample_rate ? f->sample_rate : kSampleRate,
|
||||
0, nullptr);
|
||||
av_channel_layout_uninit(&in_layout);
|
||||
av_channel_layout_uninit(&out_layout);
|
||||
if (rc < 0 || !c.swr) return false;
|
||||
#else
|
||||
const int64_t in_layout = f->channel_layout
|
||||
? static_cast<int64_t>(f->channel_layout)
|
||||
: av_get_default_channel_layout(f->channels ? f->channels : 1);
|
||||
c.swr = swr_alloc_set_opts(
|
||||
nullptr,
|
||||
AV_CH_LAYOUT_MONO, AV_SAMPLE_FMT_FLT, kSampleRate,
|
||||
in_layout, static_cast<AVSampleFormat>(f->format),
|
||||
f->sample_rate ? f->sample_rate : kSampleRate,
|
||||
0, nullptr);
|
||||
if (!c.swr) return false;
|
||||
#endif
|
||||
return swr_init(c.swr) >= 0;
|
||||
}
|
||||
|
||||
// Push one decoded frame (or a flush) through the resampler, dropping the
|
||||
// leading `to_skip` output samples, and append to `out`.
|
||||
void drain(SwrContext* swr, const AVFrame* f, int in_rate,
|
||||
std::size_t& to_skip, std::vector<float>& out) {
|
||||
const int64_t delay = swr_get_delay(swr, in_rate ? in_rate : kSampleRate);
|
||||
const int in_n = f ? f->nb_samples : 0;
|
||||
const int max_out = static_cast<int>(av_rescale_rnd(
|
||||
delay + in_n, kSampleRate, in_rate ? in_rate : kSampleRate, AV_ROUND_UP)) + 32;
|
||||
if (max_out <= 0) return;
|
||||
|
||||
std::vector<float> buf(static_cast<std::size_t>(max_out));
|
||||
uint8_t* dst = reinterpret_cast<uint8_t*>(buf.data());
|
||||
const int n = swr_convert(swr, &dst, max_out,
|
||||
f ? const_cast<const uint8_t**>(f->extended_data) : nullptr,
|
||||
in_n);
|
||||
if (n <= 0) return;
|
||||
|
||||
std::size_t produced = static_cast<std::size_t>(n);
|
||||
std::size_t off = 0;
|
||||
if (to_skip) {
|
||||
const std::size_t drop = std::min(to_skip, produced);
|
||||
to_skip -= drop;
|
||||
off = drop;
|
||||
produced -= drop;
|
||||
}
|
||||
if (produced)
|
||||
out.insert(out.end(), buf.begin() + off, buf.begin() + off + produced);
|
||||
}
|
||||
|
||||
} // namespace
|
||||
|
||||
// ── Public surface ──────────────────────────────────────────────────────────
|
||||
|
||||
const std::array<std::pair<int, int>, kNumBands>& band_fft_bins() {
|
||||
static const std::array<std::pair<int, int>, kNumBands> tbl = build_band_table();
|
||||
return tbl;
|
||||
}
|
||||
|
||||
std::string base64_encode(const std::uint8_t* data, std::size_t n) {
|
||||
static constexpr char kAlphabet[] =
|
||||
"ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/";
|
||||
std::string out;
|
||||
out.reserve(((n + 2) / 3) * 4);
|
||||
std::size_t i = 0;
|
||||
for (; i + 3 <= n; i += 3) {
|
||||
const std::uint32_t v = (std::uint32_t(data[i]) << 16) |
|
||||
(std::uint32_t(data[i + 1]) << 8) |
|
||||
std::uint32_t(data[i + 2]);
|
||||
out += kAlphabet[(v >> 18) & 0x3F];
|
||||
out += kAlphabet[(v >> 12) & 0x3F];
|
||||
out += kAlphabet[(v >> 6) & 0x3F];
|
||||
out += kAlphabet[v & 0x3F];
|
||||
}
|
||||
if (i < n) {
|
||||
const bool two = (n - i) == 2;
|
||||
const std::uint32_t v = (std::uint32_t(data[i]) << 16) |
|
||||
(two ? (std::uint32_t(data[i + 1]) << 8) : 0u);
|
||||
out += kAlphabet[(v >> 18) & 0x3F];
|
||||
out += kAlphabet[(v >> 12) & 0x3F];
|
||||
out += two ? kAlphabet[(v >> 6) & 0x3F] : '=';
|
||||
out += '=';
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
std::uint64_t fnv1a64(const void* data, std::size_t n) {
|
||||
const auto* p = static_cast<const std::uint8_t*>(data);
|
||||
std::uint64_t h = 0xcbf29ce484222325ULL;
|
||||
for (std::size_t i = 0; i < n; ++i) {
|
||||
h ^= p[i];
|
||||
h *= 0x100000001b3ULL;
|
||||
}
|
||||
return h;
|
||||
}
|
||||
|
||||
/// TRACES: IR-004
|
||||
std::vector<std::uint8_t> pack_frames(const std::vector<float>& mono) {
|
||||
if (mono.size() < static_cast<std::size_t>(kFrameSize)) return {};
|
||||
|
||||
const std::size_t nframes = 1 + (mono.size() - kFrameSize) / kHopSize;
|
||||
const auto& bands = band_fft_bins();
|
||||
const auto& win = hann_window();
|
||||
const int k_lo = bands.front().first;
|
||||
const int k_hi = bands.back().second; // exclusive
|
||||
const double bin_count = static_cast<double>(k_hi - k_lo);
|
||||
|
||||
std::vector<double> re(kFrameSize), im(kFrameSize);
|
||||
std::vector<std::uint8_t> peak(nframes);
|
||||
std::vector<double> energy(nframes);
|
||||
|
||||
for (std::size_t f = 0; f < nframes; ++f) {
|
||||
const float* src = mono.data() + f * kHopSize;
|
||||
for (int n = 0; n < kFrameSize; ++n) {
|
||||
re[n] = static_cast<double>(src[n]) * win[n];
|
||||
im[n] = 0.0;
|
||||
}
|
||||
fft_4096(re, im);
|
||||
|
||||
// Per-band mean magnitude; the bands tile the 300–3000 Hz range with no
|
||||
// gaps, so the frame's band-limited energy is the sum of the band sums.
|
||||
double best = -1.0, total = 0.0;
|
||||
int best_b = 0;
|
||||
for (int b = 0; b < kNumBands; ++b) {
|
||||
double sum = 0.0;
|
||||
for (int k = bands[b].first; k < bands[b].second; ++k)
|
||||
sum += std::sqrt(re[k] * re[k] + im[k] * im[k]);
|
||||
total += sum;
|
||||
const double mean = sum / (bands[b].second - bands[b].first);
|
||||
if (mean > best) { best = mean; best_b = b; } // ties → lowest index
|
||||
}
|
||||
peak[f] = static_cast<std::uint8_t>(best_b);
|
||||
energy[f] = total / bin_count;
|
||||
}
|
||||
|
||||
// Reference is the upper median of the frame energies: an actually observed
|
||||
// value (no averaging of the two middle samples), so it is bit-reproducible,
|
||||
// gain-invariant and barely moves when the window is trimmed.
|
||||
std::vector<double> sorted = energy;
|
||||
std::sort(sorted.begin(), sorted.end());
|
||||
const double ref = sorted[sorted.size() / 2];
|
||||
|
||||
std::vector<std::uint8_t> out(nframes);
|
||||
for (std::size_t f = 0; f < nframes; ++f) {
|
||||
const double r = std::log10((energy[f] + kEnergyEps) / (ref + kEnergyEps));
|
||||
out[f] = static_cast<std::uint8_t>(((peak[f] & 0x1F) << 2) |
|
||||
(energy_class(r) & 0x03));
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
/// TRACES: IR-004, IR-008
|
||||
std::optional<std::string> signature_from_mono(const std::vector<float>& mono) {
|
||||
const std::vector<std::uint8_t> packed = pack_frames(mono);
|
||||
if (packed.empty()) return std::nullopt;
|
||||
return std::string(kVersionPrefix) + base64_encode(packed.data(), packed.size());
|
||||
}
|
||||
|
||||
/// TRACES: IR-004, IR-007
|
||||
std::optional<std::vector<float>> decode_centre_window(const std::string& path) {
|
||||
av_log_set_level(AV_LOG_ERROR);
|
||||
|
||||
DecodeCtx c;
|
||||
if (avformat_open_input(&c.fmt, path.c_str(), nullptr, nullptr) < 0)
|
||||
return std::nullopt;
|
||||
if (avformat_find_stream_info(c.fmt, nullptr) < 0) return std::nullopt;
|
||||
if (c.fmt->duration == AV_NOPTS_VALUE) return std::nullopt;
|
||||
|
||||
const double duration = static_cast<double>(c.fmt->duration) / AV_TIME_BASE;
|
||||
|
||||
// IR-007 — the window underflows, so there is no signature and no sync
|
||||
// offset downstream. The plugin applies the identical rule.
|
||||
if (duration < kWindowSec) return std::nullopt;
|
||||
|
||||
const int idx = av_find_best_stream(c.fmt, AVMEDIA_TYPE_AUDIO, -1, -1, nullptr, 0);
|
||||
if (idx < 0) return std::nullopt; // no audio → no signature
|
||||
|
||||
AVStream* st = c.fmt->streams[idx];
|
||||
const AVCodec* codec = avcodec_find_decoder(st->codecpar->codec_id);
|
||||
if (!codec) return std::nullopt;
|
||||
c.dec = avcodec_alloc_context3(codec);
|
||||
if (!c.dec) return std::nullopt;
|
||||
if (avcodec_parameters_to_context(c.dec, st->codecpar) < 0) return std::nullopt;
|
||||
c.dec->thread_count = 0;
|
||||
if (avcodec_open2(c.dec, codec, nullptr) < 0) return std::nullopt;
|
||||
|
||||
const double start_sec = duration / 2.0 - kWindowSec / 2.0;
|
||||
|
||||
// Seek to a packet at or before the window start; the exact start is then
|
||||
// reached by discarding the leading output samples, which is what
|
||||
// `ffmpeg -ss <t> -i <file>` does and therefore what the plugin sees.
|
||||
if (start_sec > 0.0) {
|
||||
const int64_t tgt = av_rescale_q(
|
||||
static_cast<int64_t>(start_sec * AV_TIME_BASE), AV_TIME_BASE_Q, st->time_base);
|
||||
if (av_seek_frame(c.fmt, idx, tgt, AVSEEK_FLAG_BACKWARD) >= 0)
|
||||
avcodec_flush_buffers(c.dec);
|
||||
}
|
||||
|
||||
c.frm = av_frame_alloc();
|
||||
c.pkt = av_packet_alloc();
|
||||
if (!c.frm || !c.pkt) return std::nullopt;
|
||||
|
||||
std::vector<float> mono;
|
||||
mono.reserve(kWindowSamples + kSampleRate);
|
||||
std::size_t to_skip = 0;
|
||||
bool have_swr = false;
|
||||
int in_rate = kSampleRate;
|
||||
bool eof = false;
|
||||
|
||||
while (mono.size() < kWindowSamples && !eof) {
|
||||
const int rr = av_read_frame(c.fmt, c.pkt);
|
||||
if (rr < 0) {
|
||||
eof = true;
|
||||
avcodec_send_packet(c.dec, nullptr); // flush the decoder
|
||||
} else if (c.pkt->stream_index != idx) {
|
||||
av_packet_unref(c.pkt);
|
||||
continue;
|
||||
} else {
|
||||
avcodec_send_packet(c.dec, c.pkt);
|
||||
av_packet_unref(c.pkt);
|
||||
}
|
||||
|
||||
while (avcodec_receive_frame(c.dec, c.frm) == 0) {
|
||||
if (!have_swr) {
|
||||
if (!open_resampler(c, c.frm)) return std::nullopt;
|
||||
have_swr = true;
|
||||
in_rate = c.frm->sample_rate ? c.frm->sample_rate : kSampleRate;
|
||||
|
||||
int64_t pts = c.frm->best_effort_timestamp;
|
||||
if (pts == AV_NOPTS_VALUE) pts = c.frm->pts;
|
||||
const double t0 = (pts == AV_NOPTS_VALUE)
|
||||
? start_sec : av_q2d(st->time_base) * static_cast<double>(pts);
|
||||
const double lead = start_sec - t0;
|
||||
to_skip = lead > 0.0
|
||||
? static_cast<std::size_t>(std::llround(lead * kSampleRate)) : 0;
|
||||
}
|
||||
drain(c.swr, c.frm, in_rate, to_skip, mono);
|
||||
av_frame_unref(c.frm);
|
||||
if (mono.size() >= kWindowSamples) break;
|
||||
}
|
||||
}
|
||||
|
||||
if (have_swr && mono.size() < kWindowSamples)
|
||||
drain(c.swr, nullptr, in_rate, to_skip, mono); // flush the resampler
|
||||
|
||||
if (mono.empty()) return std::nullopt;
|
||||
// Truncate to exactly 120.000 s so the frame count is 1288 for every input
|
||||
// and does not wobble with seek granularity or the resampler tail.
|
||||
if (mono.size() > kWindowSamples) mono.resize(kWindowSamples);
|
||||
return mono;
|
||||
}
|
||||
|
||||
/// TRACES: IR-004, IR-005, IR-007, IR-008
|
||||
std::optional<std::string> compute_signature(const std::string& path) {
|
||||
const std::optional<std::vector<float>> mono = decode_centre_window(path);
|
||||
if (!mono) return std::nullopt;
|
||||
return signature_from_mono(*mono);
|
||||
}
|
||||
|
||||
} // namespace sae::audio
|
||||
@@ -1,158 +0,0 @@
|
||||
#pragma once
|
||||
// ── JRay audio signature, v1 ─────────────────────────────────────────────────
|
||||
//
|
||||
/// TRACES: IR-004, IR-005, IR-007, IR-008 | SR-003
|
||||
//
|
||||
// A content-derived spectral-peak signature taken from the *centre* of the
|
||||
// media, so a truth file is self-identifying: a consumer can tell whether a
|
||||
// local file is the same cut as the one a manifest describes, and recover the
|
||||
// frame offset when it is the same cut trimmed differently.
|
||||
//
|
||||
// The construction is owned by `JRay-public-server/SPEC.md` §3 and is
|
||||
// reproduced by the JRay Jellyfin plugin in C#. **The two implementations must
|
||||
// agree byte for byte** — a signature that differs in any parameter simply does
|
||||
// not match, which defeats the entire point. Every deviation is therefore a
|
||||
// breaking change and must go through the `v1:` prefix (see kVersionPrefix).
|
||||
//
|
||||
// Server spec §3, restated:
|
||||
//
|
||||
// 1. Decode a 120 s window centred on the midpoint (runtime/2 ± 60 s).
|
||||
// 2. Downmix to mono, resample to 11025 Hz.
|
||||
// 3. STFT: 4096-sample frame, 1024-sample hop, Hann window (~1290 frames).
|
||||
// 4. Per frame, log-magnitude spectrum over 300–3000 Hz.
|
||||
// 5. 32 logarithmically spaced bins; peak bin index + coarse 2-bit energy
|
||||
// class.
|
||||
// 6. Pack one byte per frame; base64-encode.
|
||||
// 7. Prefix `v1:`.
|
||||
//
|
||||
// ── Details the server spec leaves open, pinned here for v1 ──────────────────
|
||||
//
|
||||
// The prose above is not sufficient to reproduce a byte stream, so the choices
|
||||
// below are the contract. They are mirrored in
|
||||
// `tests/fixtures/audio/jray_audio_v1_golden.json`, which is the artefact
|
||||
// shared with the plugin repo (IR-005).
|
||||
//
|
||||
// Arithmetic All DSP in IEEE-754 **double**. float32 is not sufficient:
|
||||
// the golden fixture has frames whose two strongest bands are
|
||||
// within 1.3% of each other, which double resolves identically
|
||||
// everywhere and float32 does not.
|
||||
// Sample scale FFmpeg's native s16→flt conversion, x * (1/32768), then
|
||||
// widened to double. Values in [-1, 1).
|
||||
// Framing Only whole frames: n_frames = 1 + (n_samples - 4096) / 1024,
|
||||
// integer division, 0 when n_samples < 4096. A 120.000 s
|
||||
// window is 1 323 000 samples → **1288 frames**.
|
||||
// ("~1290" in the spec; the server accepts a tolerance.)
|
||||
// Window Hann, **periodic**: w[n] = 0.5 * (1 - cos(2*pi*n/4096)).
|
||||
// Not the symmetric (N-1) variant.
|
||||
// Transform Plain radix-2 decimation-in-time complex FFT over 4096 real
|
||||
// samples (imag = 0), no normalisation. Magnitude is
|
||||
// sqrt(re² + im²). Twiddles from cos/sin of
|
||||
// -2*pi*k/len computed in double.
|
||||
// Band edges edge[b] = 300 * (3000/300)^(b/32), b = 0..32. Band b spans
|
||||
// FFT bins [k_lo[b], k_lo[b+1]) with
|
||||
// k_lo[b] = ceil(edge[b] * 4096 / 11025) — i.e. bins 112..1114
|
||||
// inclusive, 8 bins in the narrowest band. Precomputed as an
|
||||
// integer table so no float comparison decides membership.
|
||||
// Band value **Mean** of the linear magnitudes in the band. Mean, not
|
||||
// sum, so a wide high band is not favoured over a narrow low
|
||||
// one; magnitude, not power, because it is an energy proxy and
|
||||
// more codec-robust than a single bin's peak.
|
||||
// Peak bin argmax over the 32 band values; ties resolve to the **lowest
|
||||
// index**. The log of step 4 is a monotone squash and so
|
||||
// cannot change an argmax — it is applied only where it is
|
||||
// observable, in the energy class below.
|
||||
// Energy class The spec says "coarse 2-bit energy class" and no more. v1
|
||||
// defines it as the frame's band-limited energy relative to
|
||||
// the window, which is invariant to gain (loudness
|
||||
// normalisation must not change a signature) and robust to
|
||||
// trimming (the median barely moves):
|
||||
// E_f = mean magnitude over *all* FFT bins 112..1114
|
||||
// Eref = median over frames of E_f, taken as the upper
|
||||
// median sorted[n/2] — no averaging of the two middle
|
||||
// values, so the reference is always an actual
|
||||
// observed value and is bit-reproducible
|
||||
// r = log10((E_f + 1e-12) / (Eref + 1e-12))
|
||||
// class = 0 if r < -0.6, 1 if r < -0.2, 2 if r < 0.2, else 3
|
||||
// The thresholds deliberately straddle r = 0 rather than sit
|
||||
// on it, so the median frame itself is not on a boundary.
|
||||
// Byte layout bit 7 = 0 (reserved), bits 6..2 = 5-bit band index,
|
||||
// bits 1..0 = 2-bit energy class:
|
||||
// byte = (band << 2) | class → always 0..127
|
||||
// This is the structural constraint the server validates on
|
||||
// upload (§3 "Validation and abuse").
|
||||
// Base64 Standard alphabet A–Za–z0–9+/ with '=' padding.
|
||||
//
|
||||
// ── Short media (IR-007) ─────────────────────────────────────────────────────
|
||||
//
|
||||
// `runtime/2 ± 60 s` underflows below 120 s, so **no signature is emitted** and
|
||||
// no sync offset is applied downstream. Both producers apply the identical
|
||||
// rule; diverging here would break exactly the short items most likely to be
|
||||
// misidentified. `compute_signature` returns `std::nullopt`.
|
||||
//
|
||||
// The same nullopt is returned for a file with no audio stream, an unopenable
|
||||
// file, or an unknown duration. UR-9 is an enhancement and must never be able
|
||||
// to break a fetch — degradation, not failure.
|
||||
|
||||
#include <array>
|
||||
#include <cstddef>
|
||||
#include <cstdint>
|
||||
#include <optional>
|
||||
#include <string>
|
||||
#include <utility>
|
||||
#include <vector>
|
||||
|
||||
namespace sae::audio {
|
||||
|
||||
// ── Contract constants — changing any of these is a `v1:` bump ───────────────
|
||||
inline constexpr int kSampleRate = 11025;
|
||||
inline constexpr int kFrameSize = 4096;
|
||||
inline constexpr int kHopSize = 1024;
|
||||
inline constexpr int kNumBands = 32;
|
||||
inline constexpr double kBandLoHz = 300.0;
|
||||
inline constexpr double kBandHiHz = 3000.0;
|
||||
inline constexpr double kWindowSec = 120.0;
|
||||
inline constexpr double kEnergyEps = 1e-12;
|
||||
// Class thresholds on log10(E_frame / E_median); see the header comment.
|
||||
inline constexpr double kEnergyClassEdges[3] = {-0.6, -0.2, 0.2};
|
||||
// 120.000 s at 11025 Hz. The decoded window is truncated to exactly this so the
|
||||
// frame count does not wobble with seek granularity or resampler tail.
|
||||
inline constexpr std::size_t kWindowSamples =
|
||||
static_cast<std::size_t>(kWindowSec * kSampleRate); // 1 323 000
|
||||
inline constexpr std::size_t kExpectedFrames =
|
||||
1 + (kWindowSamples - kFrameSize) / kHopSize; // 1288
|
||||
static_assert(kWindowSamples == 1323000, "120 s at 11025 Hz");
|
||||
static_assert(kExpectedFrames == 1288, "server spec's ~1290 frames");
|
||||
|
||||
/// The version prefix is the signature's own, separate from `schema_version`:
|
||||
/// a future change to the DSP chain must be *detectable* rather than silently
|
||||
/// producing non-matching signatures (IR-008).
|
||||
inline constexpr const char* kVersionPrefix = "v1:";
|
||||
|
||||
/// FFT bin range [first, last) for each of the 32 log-spaced bands.
|
||||
/// Computed once from the constants above; exposed so the golden fixture can
|
||||
/// assert the table itself, not merely the signature it produces.
|
||||
const std::array<std::pair<int, int>, kNumBands>& band_fft_bins();
|
||||
|
||||
/// Decode the centre window of `path` as mono float PCM at 11025 Hz.
|
||||
/// nullopt when the media is shorter than 120 s (IR-007), has no audio stream,
|
||||
/// or cannot be opened. Never throws.
|
||||
std::optional<std::vector<float>> decode_centre_window(const std::string& path);
|
||||
|
||||
/// One packed byte per whole STFT frame. Empty when `mono` is shorter than one
|
||||
/// frame. This is the payload that gets base64-encoded.
|
||||
std::vector<std::uint8_t> pack_frames(const std::vector<float>& mono);
|
||||
|
||||
/// `v1:` + base64(pack_frames(mono)). nullopt when no whole frame fits.
|
||||
std::optional<std::string> signature_from_mono(const std::vector<float>& mono);
|
||||
|
||||
/// Decode + sign. The one call the pipeline makes. nullopt per IR-007 and on
|
||||
/// any decode failure — degradation, not failure.
|
||||
std::optional<std::string> compute_signature(const std::string& path);
|
||||
|
||||
// ── Small utilities, exposed for the golden-fixture test ────────────────────
|
||||
std::string base64_encode(const std::uint8_t* data, std::size_t n);
|
||||
/// FNV-1a 64. Used only to pin the *decoded PCM* in the golden fixture, so a
|
||||
/// codec-level difference is distinguishable from a DSP-level one.
|
||||
std::uint64_t fnv1a64(const void* data, std::size_t n);
|
||||
|
||||
} // namespace sae::audio
|
||||
@@ -34,23 +34,9 @@ constexpr int kDim = 512;
|
||||
|
||||
#if defined(SAE_GEMM_CPU)
|
||||
|
||||
#if defined(SAE_GEMM_CBLAS)
|
||||
#include <cblas.h>
|
||||
#endif
|
||||
|
||||
// ── CPU reference engine ──────────────────────────────────────────────────────
|
||||
// Used for CI and as the correctness oracle for the GPU backends.
|
||||
//
|
||||
// TRACES: AR-026, AR-027 | SR-001
|
||||
// Backed by CBLAS (OpenBLAS) when available, falling back to a scalar loop when
|
||||
// not. The fallback is portable but scales badly: scoring one face against a
|
||||
// 5000-embedding gallery is 2.6 MFLOP, and a crowded frame multiplies that by
|
||||
// the face count. Since AR-003 removed the per-frame face cap and CI has no GPU,
|
||||
// the CPU path is now the one that has to hold up under a library-scale gallery
|
||||
// (AR-027) rather than merely be correct.
|
||||
//
|
||||
// The fallback is kept rather than made mandatory so the build has no hard new
|
||||
// dependency, and so the two can be diffed when a similarity looks wrong. The gallery is L2-normalised (as are the queries), so each
|
||||
// Portable, dependency-free path used for CI and as the correctness oracle for
|
||||
// the GPU backends. The gallery is L2-normalised (as are the queries), so each
|
||||
// similarity is a plain dot product. S is stored column-major to match the GPU
|
||||
// backends: the gallery similarities for face fi start at result + fi*n_gallery.
|
||||
class SimilarityEngine final : public ISimilarityEngine {
|
||||
@@ -61,13 +47,7 @@ public:
|
||||
gallery_row_major + static_cast<size_t>(n_gallery) * kDim)
|
||||
{
|
||||
host_sims_.resize(static_cast<size_t>(max_faces_) * n_gallery_);
|
||||
std::cerr << "[similarity] CPU engine ("
|
||||
#if defined(SAE_GEMM_CBLAS)
|
||||
<< "CBLAS"
|
||||
#else
|
||||
<< "scalar fallback — no CBLAS; expect poor scaling on a large gallery"
|
||||
#endif
|
||||
<< "): gallery resident in host RAM ("
|
||||
std::cerr << "[similarity] CPU reference engine: gallery resident in host RAM ("
|
||||
<< (gallery_.size() * sizeof(float)) / (1024 * 1024) << " MiB)\n";
|
||||
}
|
||||
|
||||
@@ -78,18 +58,7 @@ public:
|
||||
if (n_faces > max_faces_)
|
||||
throw std::runtime_error("SimilarityEngine: n_faces exceeds max_faces");
|
||||
|
||||
// S(g, f) col-major = dot(gallery[g], query[f]). Viewed as row-major
|
||||
// [n_faces x n_gallery] that is exactly query * gallery^T, so it is one
|
||||
// GEMM rather than a loop nest.
|
||||
#if defined(SAE_GEMM_CBLAS)
|
||||
cblas_sgemm(CblasRowMajor, CblasNoTrans, CblasTrans,
|
||||
/*M=*/n_faces, /*N=*/n_gallery_, /*K=*/kDim,
|
||||
/*alpha=*/1.0f,
|
||||
query_row_major, /*lda=*/kDim,
|
||||
gallery_.data(), /*ldb=*/kDim,
|
||||
/*beta=*/0.0f,
|
||||
host_sims_.data(), /*ldc=*/n_gallery_);
|
||||
#else
|
||||
// S(g, f) col-major = dot(gallery[g], query[f]).
|
||||
for (int f = 0; f < n_faces; ++f) {
|
||||
const float* q = query_row_major + static_cast<size_t>(f) * kDim;
|
||||
float* out = host_sims_.data() + static_cast<size_t>(f) * n_gallery_;
|
||||
@@ -100,7 +69,6 @@ public:
|
||||
out[g] = acc;
|
||||
}
|
||||
}
|
||||
#endif
|
||||
return host_sims_.data();
|
||||
}
|
||||
|
||||
|
||||
+1
-31
@@ -18,8 +18,6 @@
|
||||
// --nms <f> NMS IoU threshold (default: 0.4)
|
||||
|
||||
#include "gallery/gallery_builder.hpp"
|
||||
#include "gallery/gallery_report.hpp"
|
||||
#include "gallery/gallery_calibration.hpp"
|
||||
#include "gallery/gallery_store.hpp"
|
||||
#include "config.hpp"
|
||||
|
||||
@@ -34,7 +32,6 @@ int main(int argc, char** argv) {
|
||||
std::string arcface_model = kDefaultArcfaceModel;
|
||||
float conf = 0.5f, nms_thr = 0.4f;
|
||||
int max_side = 500;
|
||||
float min_face_px = 0.f;
|
||||
|
||||
for (int i = 1; i < argc; ++i) {
|
||||
auto arg = [&](const char* f) { return std::strcmp(argv[i], f) == 0; };
|
||||
@@ -51,7 +48,6 @@ int main(int argc, char** argv) {
|
||||
else if (arg("--conf")) conf = std::stof(next());
|
||||
else if (arg("--nms")) nms_thr = std::stof(next());
|
||||
else if (arg("--max-side")) max_side = std::stoi(next());
|
||||
else if (arg("--min-face-px")) min_face_px = std::stof(next());
|
||||
else { std::cerr << "[warn] unknown flag: " << argv[i] << "\n"; }
|
||||
} catch (const std::exception& e) {
|
||||
std::cerr << "Error: " << e.what() << "\n";
|
||||
@@ -61,8 +57,7 @@ int main(int argc, char** argv) {
|
||||
|
||||
if (root_path.empty() || output_path.empty()) {
|
||||
std::cerr << "Usage: build_gallery --root <dir> --output <gallery.json> "
|
||||
"[--detector <path>] [--arcface <path>] [--max-side <N>]\n"
|
||||
" [--min-face-px <px>]\n";
|
||||
"[--detector <path>] [--arcface <path>] [--max-side <N>]\n";
|
||||
return 1;
|
||||
}
|
||||
|
||||
@@ -73,7 +68,6 @@ int main(int argc, char** argv) {
|
||||
cfg.detector_conf = conf;
|
||||
cfg.detector_nms = nms_thr;
|
||||
cfg.max_side = max_side;
|
||||
cfg.min_face_px = min_face_px;
|
||||
|
||||
try {
|
||||
ActorGallery gallery = build_gallery(cfg);
|
||||
@@ -83,30 +77,6 @@ int main(int argc, char** argv) {
|
||||
}
|
||||
save_gallery(output_path, gallery);
|
||||
std::cerr << "Gallery saved to: " << output_path << "\n";
|
||||
|
||||
/// TRACES: GR-003 | SR-001
|
||||
// Fit the calibration here and persist what it learned. The matcher
|
||||
// fits the same sigmoid at analysis time, but that is the wrong place
|
||||
// to audit a gallery from: by then the answer is per-run and nobody is
|
||||
// looking. Build time is when the gallery's quality is decided, and a
|
||||
// gallery can be quietly bad — heavily overlapping intra/inter
|
||||
// distributions, actors with no usable image — while looking fine.
|
||||
std::vector<Embedding> flat;
|
||||
std::vector<int> flat_actor;
|
||||
for (int ai = 0; ai < static_cast<int>(gallery.actors.size()); ++ai)
|
||||
for (const auto& e : gallery.actors[ai].embeddings) {
|
||||
flat.push_back(e);
|
||||
flat_actor.push_back(ai);
|
||||
}
|
||||
|
||||
GalleryCalibrationStats stats;
|
||||
GalleryCalibration cal = calibrate_gallery(flat, flat_actor, &stats);
|
||||
|
||||
const GalleryReport report =
|
||||
build_gallery_report(gallery, cal, stats, nullptr, output_path);
|
||||
const std::string report_path = gallery_report_path(output_path);
|
||||
save_gallery_report(report_path, report);
|
||||
std::cerr << "Gallery report saved to: " << report_path << "\n";
|
||||
} catch (const std::exception& e) {
|
||||
std::cerr << "Fatal: " << e.what() << "\n";
|
||||
return 1;
|
||||
|
||||
+21
-51
@@ -16,13 +16,7 @@ enum class Verbosity {
|
||||
struct Config {
|
||||
// ── Input ─────────────────────────────────────────────────────────────────
|
||||
std::string movie_path;
|
||||
std::string gallery_path;
|
||||
// TRACES: IR-002 | SR-003
|
||||
// "global" (matched against the whole library) or "limited" (this title's
|
||||
// credited cast only). The strongest single quality signal when two
|
||||
// manifests compete for the same cut: identical gallery_size can mean very
|
||||
// different recall depending on which was used.
|
||||
std::string gallery_scope{"global"}; // gallery.json produced by build_gallery
|
||||
std::string gallery_path; // gallery.json produced by build_gallery
|
||||
|
||||
// ── Output ───────────────────────────────────────────────────────────────
|
||||
std::string output_path; // annotations.json
|
||||
@@ -41,24 +35,11 @@ struct Config {
|
||||
// ── Detection (SCRFD-500MF via cv::dnn::Net) ──────────────────────────────
|
||||
std::string detector_model;
|
||||
std::string detector_engine; // optional path to pre-built TRT engine; bypasses ORT
|
||||
// TRACES: AR-003 | SR-002
|
||||
// 0 = no cap, the default. A fixed cap discards the SMALLEST faces first,
|
||||
// which are exactly the background cast X-Ray still credits with scene
|
||||
// membership. Per-frame cost is contained by backpressure (AR-004) rather
|
||||
// than by throwing work away. Set >0 only to bound a pathological source.
|
||||
int max_faces{0};
|
||||
int max_faces{10}; // pipeline cap: keep only the N largest faces
|
||||
float min_face_px{40.f}; // discard detections narrower or shorter than this
|
||||
float detector_conf{0.5f};
|
||||
float detector_nms{0.4f};
|
||||
|
||||
/// TRACES: GR-004 | SR-001
|
||||
// Gallery ↔ embedder binding. A gallery built with a different model than the
|
||||
// one loaded here is a hard error, always. This flag additionally promotes
|
||||
// "cannot prove they match" (unstamped legacy gallery, or a name-only match
|
||||
// because the ONNX could not be hashed) from a loud warning to a hard error.
|
||||
// Also settable via SAE_REQUIRE_GALLERY_STAMP=1. Measurement runs want it on.
|
||||
bool require_gallery_stamp{false}; // --require-gallery-stamp
|
||||
|
||||
// ── Recognition (ArcFace ONNX) ────────────────────────────────────────────
|
||||
std::string arcface_model;
|
||||
std::string arcface_engine; // optional path to a pre-built TRT engine; bypasses ORT
|
||||
@@ -105,25 +86,21 @@ struct Config {
|
||||
float dense_scale{1.0f}; // dense-mode frame downscale (1 = off)
|
||||
|
||||
// ── Face tracking (frame-to-frame) ───────────────────────────────────────
|
||||
/// TRACES: AR-007, AR-008, AR-024 | SR-002
|
||||
// track_alpha is the *base* weight, used on ordinary frames. It is
|
||||
// frame-dependent (AR-007): on is_cut / is_scene_boundary, and for any track
|
||||
// that is no longer on screen, it drops to 0 (embedding only), because
|
||||
// position carries no information across a viewpoint change or a gap.
|
||||
float track_alpha{0.4f}; // base cost weight: 0=embedding only, 1=spatial only
|
||||
float track_alpha{0.4f}; // cost weight: 0=embedding only, 1=spatial only
|
||||
float track_min_iou{0.1f}; // IoU below which spatial link alone is rejected
|
||||
// Minimum P(same person) for an association to be admissible on appearance
|
||||
// alone. This replaces track_max_embed_dist (a raw cosine distance, AR-024).
|
||||
// 0.5 is not a tuned constant: it is the decision boundary. Below it the pair
|
||||
// is more likely two people than one, and no amount of IoU makes that a link
|
||||
// worth asserting on identity grounds.
|
||||
float track_assoc_min_prob{0.5f};
|
||||
// How long a track that has gone off screen stays available for association
|
||||
// before the registry reaps it and emits its presence claim (AR-013).
|
||||
// Replaces track_max_frames_missing: a frame count silently changed meaning
|
||||
// with sample_fps, and the same number had to be guessed twice (once for an
|
||||
// ordinary miss, once for a cut). Seconds mean one thing at any sample rate.
|
||||
double track_extinction_sec{5.0};
|
||||
float track_max_embed_dist{0.7f}; // cosine dist above which embedding link alone is rejected
|
||||
int track_max_frames_missing{5}; // expire track after N consecutive missed frames
|
||||
|
||||
// ── Cross-cut track re-association ────────────────────────────────────────
|
||||
// A camera-angle change (Frame::is_cut) breaks spatial (IoU) continuity but
|
||||
// not identity: the same people are usually still on screen from a new angle.
|
||||
// Instead of destroying tracks on a cut, the tracker parks them in an
|
||||
// inactive pool. A post-cut detection whose raw cosine similarity to a parked
|
||||
// track's last-frame embedding is ≥ cut_revive_sim revives that track_id
|
||||
// (identity continuity survives the cut); otherwise it starts a fresh track.
|
||||
// Parked tracks that go unrevived for cut_inactive_max_frames are dropped.
|
||||
float cut_revive_sim{0.50f}; // min raw cosine sim (last-frame emb) to revive across a cut
|
||||
int cut_inactive_max_frames{5}; // drop a parked track after N frames without revival
|
||||
|
||||
// ── Scene tracking ────────────────────────────────────────────────────────
|
||||
// extinction_sec re-tuned by DE against X-Ray per-second presence, 4-film rep4
|
||||
@@ -149,18 +126,11 @@ struct Config {
|
||||
// opposite of the earlier assumption that it only helps restricted galleries.
|
||||
bool expand_gallery{true}; // master switch
|
||||
int expand_buffer_size{20}; // per-track diversity buffer capacity
|
||||
// TRACES: AR-018, AR-024 | SR-005
|
||||
// Banded admission for the per-subject store, in PROBABILITY space. An
|
||||
// embedding joins only if P(same person) against something already stored
|
||||
// lands inside [lo, hi]: above hi it is redundant, below lo it is evidence
|
||||
// the track is not one person. The same lo is re-applied to the whole store
|
||||
// at promotion time — see track_gallery.hpp. This is the only threshold the
|
||||
// expansion path has: it replaces the raw-cosine expand_novelty_sim (0.55)
|
||||
// and expand_track_spread_max (0.60), which are retired (AR-024).
|
||||
// Working values pending VR-007; sweep both bounds, they fail in opposite
|
||||
// directions.
|
||||
float expand_band_lo{0.90f};
|
||||
float expand_band_hi{0.95f};
|
||||
float expand_novelty_sim{0.55f}; // promote only embeddings whose best sim to the
|
||||
// actor's refs is below this (gallery-far / novel)
|
||||
float expand_track_spread_max{0.60f}; // reject promotion if the retained buffer's
|
||||
// internal spread (1 - min pairwise sim) exceeds
|
||||
// this — guards track-ID collisions / two people
|
||||
int expand_min_anchor_frames{3}; // require ≥N accepted frames naming the actor before
|
||||
// the track is confirmed and its buffer promoted
|
||||
std::string expand_debug_dir; // if set, dump promoted mugshots + embeddings here
|
||||
|
||||
+17
-154
@@ -25,25 +25,11 @@
|
||||
// expected to contain exactly one subject). A warning is printed to stderr
|
||||
// when more than one face is found.
|
||||
//
|
||||
// --all-faces emits every detection instead, which is what a caller analysing
|
||||
// a frame rather than a gallery portrait needs:
|
||||
// [ { "image": "frame.png",
|
||||
// "faces": [ {"bbox": [...], "landmarks": [[x,y] x5],
|
||||
// "confidence": 0.89, "embedding": [...]} , ... ] } ]
|
||||
//
|
||||
// --calibration <gallery> additionally emits the gallery's fitted Platt
|
||||
// sigmoid, so a non-Python client can turn a similarity into P(match) with the
|
||||
// same parameters the C++ matcher uses. Output becomes
|
||||
// {"calibration": {...}, "images": [...]}. Clients must score through it:
|
||||
// AR-024 requires the calibrated probability, never a bare cosine — a raw
|
||||
// threshold means something different for every model, gallery and face size.
|
||||
//
|
||||
// This binary is intentionally a thin wrapper around the same ONNX models
|
||||
// used by scene_analyze, so embeddings are guaranteed compatible.
|
||||
|
||||
#include "config.hpp"
|
||||
#include "face_utils.hpp"
|
||||
#include "gallery/gallery_store.hpp"
|
||||
#include "inference/face_detector.hpp"
|
||||
#include "inference/face_embedder.hpp"
|
||||
|
||||
@@ -114,77 +100,6 @@ static void save_debug(const std::string& dir,
|
||||
cv::imwrite(dir + "/" + stem + "_aligned.jpg", aligned);
|
||||
}
|
||||
|
||||
// ── Process one image, keeping every detection ────────────────────────────────
|
||||
// The --all-faces path. Same detect → align → embed chain as process() below,
|
||||
// but without the highest-confidence reduction: a frame legitimately contains
|
||||
// several people, and dropping all but one is a gallery-portrait assumption.
|
||||
// Faces that fail alignment are reported with a null embedding rather than
|
||||
// silently dropped, so a caller can count what the detector found against what
|
||||
// survived the ArcFace warp.
|
||||
|
||||
struct MultiFaceResult {
|
||||
std::string image_path;
|
||||
std::string error; // set only when the image itself failed
|
||||
std::vector<FaceResult> faces;
|
||||
};
|
||||
|
||||
static MultiFaceResult process_all(
|
||||
const std::string& path,
|
||||
const std::function<std::vector<DetectedFace>(const cv::Mat&)>& detect,
|
||||
const std::function<Embedding(const cv::Mat&)>& embed_one,
|
||||
int max_side,
|
||||
const std::string& debug_dir = "") {
|
||||
MultiFaceResult out;
|
||||
out.image_path = path;
|
||||
|
||||
cv::Mat img = cv::imread(path);
|
||||
if (img.empty()) {
|
||||
out.error = "cannot read image";
|
||||
return out;
|
||||
}
|
||||
|
||||
if (max_side > 0) {
|
||||
const int big = std::max(img.cols, img.rows);
|
||||
if (big > max_side) {
|
||||
const double s = static_cast<double>(max_side) / big;
|
||||
cv::resize(img, img, {}, s, s, cv::INTER_AREA);
|
||||
}
|
||||
}
|
||||
|
||||
std::vector<DetectedFace> faces = detect(img);
|
||||
if (faces.empty()) {
|
||||
cv::Mat enhanced = enhance_for_retry(img);
|
||||
faces = detect(enhanced);
|
||||
if (!faces.empty())
|
||||
img = enhanced;
|
||||
}
|
||||
if (faces.empty()) {
|
||||
out.error = "no face detected";
|
||||
return out;
|
||||
}
|
||||
|
||||
for (const auto& face : faces) {
|
||||
FaceResult r;
|
||||
r.image_path = path;
|
||||
r.confidence = face.confidence;
|
||||
r.bbox[0] = face.bbox.x; r.bbox[1] = face.bbox.y;
|
||||
r.bbox[2] = face.bbox.width; r.bbox[3] = face.bbox.height;
|
||||
r.landmarks = face.landmarks;
|
||||
|
||||
cv::Mat crop = align_face(img, face.landmarks);
|
||||
if (crop.empty()) {
|
||||
r.error = "alignment failed";
|
||||
} else {
|
||||
r.ok = true;
|
||||
r.embedding = embed_one(crop);
|
||||
if (!debug_dir.empty())
|
||||
save_debug(debug_dir, path, img, face, crop);
|
||||
}
|
||||
out.faces.push_back(std::move(r));
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
// ── Process one image ─────────────────────────────────────────────────────────
|
||||
|
||||
static FaceResult process(const std::string& path,
|
||||
@@ -262,8 +177,6 @@ int main(int argc, char** argv) {
|
||||
std::string arcface_model = kDefaultArcfaceModel;
|
||||
std::string arcface_engine;
|
||||
std::string debug_dir;
|
||||
std::string calibration_gallery;
|
||||
bool all_faces = false;
|
||||
float conf = 0.5f, nms = 0.4f;
|
||||
int max_side = 500;
|
||||
std::vector<std::string> images;
|
||||
@@ -277,16 +190,13 @@ int main(int argc, char** argv) {
|
||||
else if (std::strcmp(argv[i], "--nms") == 0 && i+1 < argc) { nms = std::stof(argv[++i]); }
|
||||
else if (std::strcmp(argv[i], "--save-debug") == 0 && i+1 < argc) { debug_dir = argv[++i]; }
|
||||
else if (std::strcmp(argv[i], "--max-side") == 0 && i+1 < argc) { max_side = std::stoi(argv[++i]); }
|
||||
else if (std::strcmp(argv[i], "--calibration")== 0 && i+1 < argc) { calibration_gallery = argv[++i]; }
|
||||
else if (std::strcmp(argv[i], "--all-faces") == 0) { all_faces = true; }
|
||||
else if (argv[i][0] != '-') { images.push_back(argv[i]); }
|
||||
else { std::cerr << "[warn] unknown flag: " << argv[i] << "\n"; }
|
||||
}
|
||||
|
||||
if (images.empty()) {
|
||||
std::cerr << "Usage: embed_faces [--detector <path>] [--arcface <path>] "
|
||||
"[--save-debug <dir>] [--max-side <N>] [--all-faces] "
|
||||
"[--calibration <gallery>] image1.jpg ...\n";
|
||||
"[--save-debug <dir>] [--max-side <N>] image1.jpg ...\n";
|
||||
return 1;
|
||||
}
|
||||
|
||||
@@ -307,78 +217,31 @@ int main(int argc, char** argv) {
|
||||
std::function<Embedding(const cv::Mat&)> embed_one =
|
||||
[&](const cv::Mat& c) { return embedder->embed_one(c); };
|
||||
|
||||
// One face's fields, shared by both output shapes.
|
||||
auto face_json = [](const FaceResult& r) {
|
||||
json f;
|
||||
f["confidence"] = r.confidence;
|
||||
f["bbox"] = {r.bbox[0], r.bbox[1], r.bbox[2], r.bbox[3]};
|
||||
json lms = json::array();
|
||||
for (const auto& pt : r.landmarks) lms.push_back({pt.x, pt.y});
|
||||
f["landmarks"] = std::move(lms);
|
||||
if (r.ok) f["embedding"] = std::vector<float>(r.embedding.begin(),
|
||||
r.embedding.end());
|
||||
else { f["embedding"] = nullptr; f["error"] = r.error; }
|
||||
return f;
|
||||
};
|
||||
|
||||
// Process images and build JSON output
|
||||
json images_out = json::array();
|
||||
json output = json::array();
|
||||
|
||||
for (const auto& path : images) {
|
||||
std::cerr << "[embed_faces] " << path << "\n";
|
||||
FaceResult res = process(path, detect, embed_one, max_side, debug_dir);
|
||||
|
||||
json entry;
|
||||
entry["image"] = path;
|
||||
|
||||
if (all_faces) {
|
||||
MultiFaceResult res = process_all(path, detect, embed_one, max_side, debug_dir);
|
||||
if (!res.error.empty()) {
|
||||
entry["faces"] = json::array();
|
||||
entry["error"] = res.error;
|
||||
std::cerr << " [skip] " << res.error << "\n";
|
||||
} else {
|
||||
json faces = json::array();
|
||||
for (const auto& f : res.faces) faces.push_back(face_json(f));
|
||||
entry["faces"] = std::move(faces);
|
||||
}
|
||||
entry["image"] = res.image_path;
|
||||
if (res.ok) {
|
||||
entry["embedding"] = std::vector<float>(res.embedding.begin(),
|
||||
res.embedding.end());
|
||||
entry["confidence"] = res.confidence;
|
||||
entry["bbox"] = {res.bbox[0], res.bbox[1], res.bbox[2], res.bbox[3]};
|
||||
json lms = json::array();
|
||||
for (const auto& pt : res.landmarks) lms.push_back({pt.x, pt.y});
|
||||
entry["landmarks"] = std::move(lms);
|
||||
} else {
|
||||
FaceResult res = process(path, detect, embed_one, max_side, debug_dir);
|
||||
if (res.ok) {
|
||||
entry.merge_patch(face_json(res));
|
||||
} else {
|
||||
entry["embedding"] = nullptr;
|
||||
entry["error"] = res.error;
|
||||
std::cerr << " [skip] " << res.error << "\n";
|
||||
}
|
||||
entry["embedding"] = nullptr;
|
||||
entry["error"] = res.error;
|
||||
std::cerr << " [skip] " << res.error << "\n";
|
||||
}
|
||||
images_out.push_back(std::move(entry));
|
||||
output.push_back(std::move(entry));
|
||||
}
|
||||
|
||||
// Without --calibration the output stays a bare array, unchanged, so
|
||||
// existing callers (build_gallery, fetch_missing_actors) are unaffected.
|
||||
if (calibration_gallery.empty()) {
|
||||
std::cout << images_out.dump() << "\n";
|
||||
return 0;
|
||||
}
|
||||
|
||||
ActorGallery gallery = load_gallery(calibration_gallery);
|
||||
if (!gallery.calib_valid)
|
||||
std::cerr << "[warn] " << calibration_gallery
|
||||
<< " carries no valid calibration; a client cannot convert a "
|
||||
"similarity to a probability from it (AR-024)\n";
|
||||
|
||||
json out;
|
||||
out["calibration"] = {
|
||||
{"a", gallery.calib_a},
|
||||
{"b", gallery.calib_b},
|
||||
{"valid", gallery.calib_valid},
|
||||
{"form", "P(match) = 1/(1+exp(-(a*similarity + b + log_prior_odds)))"},
|
||||
{"note", "Score through this. AR-024: a bare cosine threshold means "
|
||||
"something different for every model, gallery and face size. "
|
||||
"log_prior_odds = log(p0/(1-p0)) for a base-rate prior p0; use "
|
||||
"0 for association (are these two faces one person)."},
|
||||
};
|
||||
out["images"] = std::move(images_out);
|
||||
std::cout << out.dump() << "\n";
|
||||
std::cout << output.dump() << "\n";
|
||||
return 0;
|
||||
}
|
||||
|
||||
@@ -1,131 +0,0 @@
|
||||
#pragma once
|
||||
/// TRACES: AR-024, AR-025 | SR-002
|
||||
///
|
||||
/// EvidenceDiscounter — how much a single observation is allowed to move a
|
||||
/// track's belief.
|
||||
///
|
||||
/// **The independence problem.** Per-frame identity evidence is accumulated as
|
||||
/// log-odds along a track (AR-025), which is only valid for *independent*
|
||||
/// observations. Consecutive frames of one track are nothing of the kind: near
|
||||
/// identical pose, lighting and expression. Treating them as independent drives
|
||||
/// the posterior to certainty on what is effectively one measurement — thirty
|
||||
/// frames of the same face at the same angle is not thirty pieces of evidence.
|
||||
///
|
||||
/// The mitigation is to weight each observation by how much it *adds*: a view
|
||||
/// the track has already contributed is discounted toward zero, a genuinely new
|
||||
/// pose counts in full. This reuses the same judgement the diversity buffer
|
||||
/// makes for gallery expansion (AR-019) — which embeddings on a track are
|
||||
/// mutually distinct — rather than inventing a second notion of novelty.
|
||||
///
|
||||
/// Owned by TrackRegistry rather than left to callers. A caller that forgot to
|
||||
/// discount, or applied it twice, would silently produce confident wrong
|
||||
/// answers, and the registry is the one place where all evidence converges.
|
||||
///
|
||||
/// **Similarity enters as a calibrated probability, never a raw cosine**
|
||||
/// (AR-024): "is this the same view" is a decision, and a bare cosine threshold
|
||||
/// means something different for every model and every face size.
|
||||
|
||||
#include "types.hpp"
|
||||
|
||||
#include <algorithm>
|
||||
#include <cmath>
|
||||
#include <functional>
|
||||
#include <vector>
|
||||
|
||||
class EvidenceDiscounter {
|
||||
public:
|
||||
/// cosine similarity → P(same view). Supplied by the caller so the
|
||||
/// calibration fitted for the active embedder is used (AR-023/AR-024).
|
||||
using Calibrate = std::function<float(float)>;
|
||||
|
||||
struct Config {
|
||||
int max_views{8}; ///< distinct views remembered per track
|
||||
float admit_below{0.6f}; ///< P(same view) under this ⇒ a new view
|
||||
|
||||
/// Ceiling on the correlation between two observations of one track.
|
||||
///
|
||||
/// This is what bounds the accumulation. `n_eff = n / (1 + (n-1)·rho)`
|
||||
/// tends to `1/rho` as `n` grows, so `rho_max` sets how much a single
|
||||
/// repeated view can ever be worth: 0.5 caps it at two observations,
|
||||
/// no matter how long the shot runs.
|
||||
///
|
||||
/// 0.5 caps a repeated view at two independent observations' worth,
|
||||
/// which is what lets a track the matcher accepts on frame after frame
|
||||
/// actually become owned. Higher values starve ownership; the sweep
|
||||
/// (VR-007) decides where it belongs.
|
||||
///
|
||||
/// It is capped below 1 deliberately. P(same view) near 1 says the two
|
||||
/// crops look alike; it does not say the second carries no information.
|
||||
/// A fresh frame is a fresh detection, a fresh alignment and a fresh
|
||||
/// noise realisation, so a little independent evidence survives even a
|
||||
/// perfectly held pose. Setting this to 1 recovers the original bug —
|
||||
/// belief frozen after the first frame.
|
||||
float rho_max{0.5f};
|
||||
};
|
||||
|
||||
// Two constructors rather than a defaulted argument: `Config{}` as a default
|
||||
// argument would reference Config's own member initializers before the
|
||||
// enclosing class is complete, which is ill-formed.
|
||||
explicit EvidenceDiscounter(Calibrate cal)
|
||||
: cal_(std::move(cal)), cfg_() {}
|
||||
|
||||
EvidenceDiscounter(Calibrate cal, Config cfg)
|
||||
: cal_(std::move(cal)), cfg_(cfg) {}
|
||||
|
||||
/// The marginal evidence one observation adds, in units of independent
|
||||
/// observations.
|
||||
///
|
||||
/// Each frame is a Bayesian update, so confidence must keep growing — but
|
||||
/// correlated observations must grow it less, and must not grow it without
|
||||
/// bound. The standard treatment is **effective sample size**:
|
||||
///
|
||||
/// n_eff(n) = n / (1 + (n-1)·rho)
|
||||
///
|
||||
/// and this returns `n_eff(n) - n_eff(n-1)`, the gain from *this* frame.
|
||||
/// The shape is right at both ends: with rho = 0 every frame counts fully
|
||||
/// and the belief accumulates linearly, while as rho rises the series
|
||||
/// converges on `1/rho` and a held pose stops adding no matter how long it
|
||||
/// is held.
|
||||
///
|
||||
/// The two failure modes it sits between are both real and both were hit:
|
||||
/// a weight of 0 for repeats froze the belief after one frame, so a track
|
||||
/// recognised on 318 frames was owned on none; a constant floor grew it
|
||||
/// linearly forever, so a long shot could out-argue genuinely varied
|
||||
/// evidence purely by lasting longer.
|
||||
///
|
||||
/// `rho` is estimated from P(same view) against the closest stored view,
|
||||
/// capped by `rho_max`. The first observation has nothing to be redundant
|
||||
/// with and counts in full.
|
||||
/// `n_seen` is the count of observations already folded into THIS track.
|
||||
/// It is a parameter rather than discounter state because one discounter
|
||||
/// serves every track: holding the count internally would pool unrelated
|
||||
/// tracks into one effective sample, so a busy film would silently discount
|
||||
/// each track by how many others happened to be on screen.
|
||||
float weight(std::vector<Embedding>& views, int n_seen, const Embedding& e) const {
|
||||
if (views.empty()) {
|
||||
views.push_back(e);
|
||||
return 1.0f;
|
||||
}
|
||||
|
||||
float p_same = 0.0f;
|
||||
for (const auto& v : views)
|
||||
p_same = std::max(p_same, cal_(cosine_similarity(v, e)));
|
||||
|
||||
const float rho = std::min(cfg_.rho_max, std::max(0.0f, p_same));
|
||||
|
||||
const float n_prev = static_cast<float>(std::max(1, n_seen));
|
||||
const float n_now = n_prev + 1.0f;
|
||||
auto n_eff = [rho](float n) { return n / (1.0f + (n - 1.0f) * rho); };
|
||||
const float w = std::max(0.0f, n_eff(n_now) - n_eff(n_prev));
|
||||
|
||||
if (p_same < cfg_.admit_below &&
|
||||
static_cast<int>(views.size()) < cfg_.max_views) {
|
||||
views.push_back(e);
|
||||
}
|
||||
return w;
|
||||
}
|
||||
|
||||
private:
|
||||
Calibrate cal_;
|
||||
Config cfg_;
|
||||
};
|
||||
@@ -112,25 +112,6 @@ public:
|
||||
return res;
|
||||
}
|
||||
|
||||
// ── Stage accessors ──────────────────────────────────────────────────────
|
||||
// embed_mat() above is the whole detect→align→embed chain, which is the
|
||||
// right entry point for embedding a gallery image. Studies that need to
|
||||
// intervene between the stages — swapping the landmark source, degrading a
|
||||
// crop before it reaches the embedder — drive these instead, so they still
|
||||
// exercise the shipped detector, alignment and embedder rather than a
|
||||
// re-implementation of them.
|
||||
std::vector<DetectedFace> detect(const cv::Mat& img) { return detector_->detect(img); }
|
||||
|
||||
Embedding embed_crop(const cv::Mat& crop) { return embedder_->embed_one(crop); }
|
||||
|
||||
// Batched form. A study embedding thousands of crops one at a time pays the
|
||||
// per-call overhead thousands of times over; the backend already batches.
|
||||
std::vector<Embedding> embed_crops(const std::vector<cv::Mat>& crops) {
|
||||
return embedder_->embed(crops);
|
||||
}
|
||||
|
||||
int max_batch() const { return embedder_->max_batch(); }
|
||||
|
||||
private:
|
||||
std::unique_ptr<IFaceDetector> detector_;
|
||||
std::unique_ptr<IFaceEmbedder> embedder_;
|
||||
|
||||
+10
-129
@@ -1,144 +1,25 @@
|
||||
#pragma once
|
||||
/// TRACES: AR-005, AR-030 | SR-002
|
||||
#include "types.hpp"
|
||||
|
||||
#include <opencv2/core.hpp>
|
||||
#include <opencv2/calib3d.hpp>
|
||||
#include <opencv2/imgproc.hpp>
|
||||
|
||||
#include <cmath>
|
||||
|
||||
// ── umeyama_similarity ────────────────────────────────────────────────────────
|
||||
// Closed-form least-squares similarity transform (rotation + uniform scale +
|
||||
// translation, 4 DoF) mapping `src` onto `dst`, by Umeyama's solution.
|
||||
//
|
||||
// This is the estimator InsightFace aligns with — skimage's SimilarityTransform
|
||||
// is `_umeyama(..., estimate_scale=True)` — and therefore the one that produced
|
||||
// the crops ArcFace and LVFace were *trained* on. The canonical warp is part of
|
||||
// the input distribution, not a free implementation choice (AR-011).
|
||||
//
|
||||
// Deliberately **not** `cv::estimateAffinePartial2D(..., cv::RANSAC)`:
|
||||
//
|
||||
// - A robust estimator earns a small residual by discarding the points that
|
||||
// disagree with the model. On a turned face those are precisely the
|
||||
// foreshortened landmarks — the pose signal AR-030 exists to measure. RANSAC
|
||||
// would suppress exactly the quantity we want to read.
|
||||
// - With five points and a two-point minimal sample there is almost no
|
||||
// redundancy, so it cannot distinguish a mis-detected landmark from honest
|
||||
// out-of-plane rotation. The robustness is nominal.
|
||||
// - It is RNG-driven (`cv::theRNG()` is thread-local); this is exact, so
|
||||
// replay determinism stops depending on thread scheduling.
|
||||
//
|
||||
// Returns an empty Mat when the source points are degenerate (all coincident).
|
||||
inline cv::Mat umeyama_similarity(const std::array<cv::Point2f, 5>& src,
|
||||
const std::array<cv::Point2f, 5>& dst) {
|
||||
constexpr int N = 5;
|
||||
|
||||
double mu_sx = 0, mu_sy = 0, mu_dx = 0, mu_dy = 0;
|
||||
for (int i = 0; i < N; ++i) {
|
||||
mu_sx += src[i].x; mu_sy += src[i].y;
|
||||
mu_dx += dst[i].x; mu_dy += dst[i].y;
|
||||
}
|
||||
mu_sx /= N; mu_sy /= N; mu_dx /= N; mu_dy /= N;
|
||||
|
||||
// var_src and the cross-covariance Σ = (1/N) Σ (d - μ_d)(s - μ_s)ᵀ
|
||||
double var_s = 0;
|
||||
cv::Matx22d sigma = cv::Matx22d::zeros();
|
||||
for (int i = 0; i < N; ++i) {
|
||||
const double sx = src[i].x - mu_sx, sy = src[i].y - mu_sy;
|
||||
const double dx = dst[i].x - mu_dx, dy = dst[i].y - mu_dy;
|
||||
var_s += sx * sx + sy * sy;
|
||||
sigma(0, 0) += dx * sx; sigma(0, 1) += dx * sy;
|
||||
sigma(1, 0) += dy * sx; sigma(1, 1) += dy * sy;
|
||||
}
|
||||
var_s /= N;
|
||||
sigma *= 1.0 / N;
|
||||
|
||||
if (var_s < 1e-12) return {}; // every source point coincides — no scale
|
||||
|
||||
cv::Mat w, u, vt;
|
||||
cv::SVD::compute(cv::Mat(sigma), w, u, vt, cv::SVD::FULL_UV);
|
||||
|
||||
const cv::Matx22d U (u.at<double>(0,0), u.at<double>(0,1),
|
||||
u.at<double>(1,0), u.at<double>(1,1));
|
||||
const cv::Matx22d Vt(vt.at<double>(0,0), vt.at<double>(0,1),
|
||||
vt.at<double>(1,0), vt.at<double>(1,1));
|
||||
|
||||
// A similarity may rotate but never mirror: if the fit came out
|
||||
// orientation-reversing, flip the least-significant singular direction.
|
||||
cv::Matx22d S = cv::Matx22d::eye();
|
||||
if (cv::determinant(U) * cv::determinant(Vt) < 0) S(1, 1) = -1;
|
||||
|
||||
const cv::Matx22d R = U * S * Vt;
|
||||
const double c = (w.at<double>(0) * S(0,0) + w.at<double>(1) * S(1,1)) / var_s;
|
||||
|
||||
cv::Mat M(2, 3, CV_64F);
|
||||
M.at<double>(0,0) = c * R(0,0); M.at<double>(0,1) = c * R(0,1);
|
||||
M.at<double>(1,0) = c * R(1,0); M.at<double>(1,1) = c * R(1,1);
|
||||
M.at<double>(0,2) = mu_dx - c * (R(0,0) * mu_sx + R(0,1) * mu_sy);
|
||||
M.at<double>(1,2) = mu_dy - c * (R(1,0) * mu_sx + R(1,1) * mu_sy);
|
||||
return M;
|
||||
}
|
||||
|
||||
// ── Alignment ─────────────────────────────────────────────────────────────────
|
||||
// The 5-point fit, plus what it could not explain.
|
||||
//
|
||||
// `residual` is the RMS landmark error in **canonical 112×112 pixels** after the
|
||||
// best similarity fit. Two properties make it the AR-030 visibility measure:
|
||||
//
|
||||
// - The similarity transform absorbs rotation, uniform scale and translation
|
||||
// exactly, so the residual is by construction the part of the deformation a
|
||||
// similarity *cannot* explain — out-of-plane rotation and foreshortening,
|
||||
// plus landmark noise. In-plane roll contributes nothing. The "roll must not
|
||||
// read as yaw" failure is excluded structurally rather than by tuning.
|
||||
// - The destination frame is fixed, so a 40 px face and a 400 px face are both
|
||||
// measured in the same canonical space. The measure cannot silently
|
||||
// re-express face size (already AR-002's job) the way a raw-pixel one would.
|
||||
//
|
||||
// It also responds to occlusion and to plainly broken landmark sets, which a
|
||||
// yaw-angle estimator by construction does not.
|
||||
struct Alignment {
|
||||
cv::Mat M; ///< 2×3 CV_64F: source pixels → canonical 112×112
|
||||
float residual{0.f}; ///< RMS canonical-pixel error; 0 ⇒ a perfect fit
|
||||
bool ok{false}; ///< false ⇒ degenerate landmarks, no transform
|
||||
};
|
||||
|
||||
/// Fit the canonical ArcFace template to `landmarks` and report the misfit.
|
||||
inline Alignment estimate_alignment(const std::array<cv::Point2f, 5>& landmarks) {
|
||||
std::array<cv::Point2f, 5> dst;
|
||||
for (int i = 0; i < 5; ++i) dst[i] = {kArcFaceRef[i][0], kArcFaceRef[i][1]};
|
||||
|
||||
Alignment a;
|
||||
a.M = umeyama_similarity(landmarks, dst);
|
||||
if (a.M.empty()) return a;
|
||||
|
||||
double sq = 0;
|
||||
for (int i = 0; i < 5; ++i) {
|
||||
const double x = a.M.at<double>(0,0) * landmarks[i].x
|
||||
+ a.M.at<double>(0,1) * landmarks[i].y + a.M.at<double>(0,2);
|
||||
const double y = a.M.at<double>(1,0) * landmarks[i].x
|
||||
+ a.M.at<double>(1,1) * landmarks[i].y + a.M.at<double>(1,2);
|
||||
const double ex = x - dst[i].x, ey = y - dst[i].y;
|
||||
sq += ex * ex + ey * ey;
|
||||
}
|
||||
a.residual = static_cast<float>(std::sqrt(sq / 5.0));
|
||||
a.ok = true;
|
||||
return a;
|
||||
}
|
||||
|
||||
// ── align_face ────────────────────────────────────────────────────────────────
|
||||
// Produces a 112×112 BGR crop using the ArcFace 5-point similarity transform.
|
||||
// Returns an empty Mat if the fit fails (degenerate detection). When
|
||||
// `residual_out` is non-null it receives the AR-030 misfit for the same fit —
|
||||
// free, since the transform has already been computed.
|
||||
// Returns an empty Mat if the affine fit fails (degenerate detection).
|
||||
inline cv::Mat align_face(const cv::Mat& img,
|
||||
const std::array<cv::Point2f, 5>& landmarks,
|
||||
float* residual_out = nullptr) {
|
||||
const Alignment a = estimate_alignment(landmarks);
|
||||
if (!a.ok) return {};
|
||||
if (residual_out) *residual_out = a.residual;
|
||||
const std::array<cv::Point2f, 5>& landmarks) {
|
||||
std::vector<cv::Point2f> src(landmarks.begin(), landmarks.end());
|
||||
std::vector<cv::Point2f> dst(5);
|
||||
for (int i = 0; i < 5; ++i) dst[i] = {kArcFaceRef[i][0], kArcFaceRef[i][1]};
|
||||
|
||||
cv::Mat M = cv::estimateAffinePartial2D(src, dst, cv::noArray(), cv::RANSAC, 3.0);
|
||||
if (M.empty()) return {};
|
||||
|
||||
cv::Mat crop;
|
||||
cv::warpAffine(img, crop, a.M, {112, 112},
|
||||
cv::warpAffine(img, crop, M, {112, 112},
|
||||
cv::INTER_LINEAR, cv::BORDER_CONSTANT, {0, 0, 0});
|
||||
return crop;
|
||||
}
|
||||
|
||||
@@ -1,326 +0,0 @@
|
||||
/// TRACES: GR-004 | SR-001
|
||||
#include "embedder_stamp.hpp"
|
||||
#include "types.hpp"
|
||||
|
||||
#include <algorithm>
|
||||
#include <cstdlib>
|
||||
#include <cstring>
|
||||
#include <filesystem>
|
||||
#include <fstream>
|
||||
#include <iostream>
|
||||
#include <map>
|
||||
#include <mutex>
|
||||
#include <sstream>
|
||||
#include <stdexcept>
|
||||
#include <vector>
|
||||
|
||||
namespace fs = std::filesystem;
|
||||
|
||||
// ── SHA-256 (FIPS 180-4) ──────────────────────────────────────────────────────
|
||||
// Self-contained rather than pulled from OpenSSL: the gallery library already
|
||||
// links OpenCV, HDF5, FFmpeg and a GPU backend, and the unit tests deliberately
|
||||
// link none of those crypto stacks. ~80 lines of table-driven code is cheaper
|
||||
// than another find_package that CI has to satisfy on an Intel N100.
|
||||
namespace {
|
||||
|
||||
struct Sha256 {
|
||||
uint32_t h[8] = {0x6a09e667u, 0xbb67ae85u, 0x3c6ef372u, 0xa54ff53au,
|
||||
0x510e527fu, 0x9b05688cu, 0x1f83d9abu, 0x5be0cd19u};
|
||||
uint64_t len = 0;
|
||||
uint8_t buf[64]{};
|
||||
size_t buf_n = 0;
|
||||
|
||||
static uint32_t ror(uint32_t x, int n) { return (x >> n) | (x << (32 - n)); }
|
||||
|
||||
void block(const uint8_t* p) {
|
||||
static const uint32_t k[64] = {
|
||||
0x428a2f98u,0x71374491u,0xb5c0fbcfu,0xe9b5dba5u,0x3956c25bu,0x59f111f1u,
|
||||
0x923f82a4u,0xab1c5ed5u,0xd807aa98u,0x12835b01u,0x243185beu,0x550c7dc3u,
|
||||
0x72be5d74u,0x80deb1feu,0x9bdc06a7u,0xc19bf174u,0xe49b69c1u,0xefbe4786u,
|
||||
0x0fc19dc6u,0x240ca1ccu,0x2de92c6fu,0x4a7484aau,0x5cb0a9dcu,0x76f988dau,
|
||||
0x983e5152u,0xa831c66du,0xb00327c8u,0xbf597fc7u,0xc6e00bf3u,0xd5a79147u,
|
||||
0x06ca6351u,0x14292967u,0x27b70a85u,0x2e1b2138u,0x4d2c6dfcu,0x53380d13u,
|
||||
0x650a7354u,0x766a0abbu,0x81c2c92eu,0x92722c85u,0xa2bfe8a1u,0xa81a664bu,
|
||||
0xc24b8b70u,0xc76c51a3u,0xd192e819u,0xd6990624u,0xf40e3585u,0x106aa070u,
|
||||
0x19a4c116u,0x1e376c08u,0x2748774cu,0x34b0bcb5u,0x391c0cb3u,0x4ed8aa4au,
|
||||
0x5b9cca4fu,0x682e6ff3u,0x748f82eeu,0x78a5636fu,0x84c87814u,0x8cc70208u,
|
||||
0x90befffau,0xa4506cebu,0xbef9a3f7u,0xc67178f2u};
|
||||
uint32_t w[64];
|
||||
for (int i = 0; i < 16; ++i)
|
||||
w[i] = (uint32_t(p[i * 4]) << 24) | (uint32_t(p[i * 4 + 1]) << 16) |
|
||||
(uint32_t(p[i * 4 + 2]) << 8) | uint32_t(p[i * 4 + 3]);
|
||||
for (int i = 16; i < 64; ++i) {
|
||||
uint32_t s0 = ror(w[i - 15], 7) ^ ror(w[i - 15], 18) ^ (w[i - 15] >> 3);
|
||||
uint32_t s1 = ror(w[i - 2], 17) ^ ror(w[i - 2], 19) ^ (w[i - 2] >> 10);
|
||||
w[i] = w[i - 16] + s0 + w[i - 7] + s1;
|
||||
}
|
||||
uint32_t a = h[0], b = h[1], c = h[2], d = h[3];
|
||||
uint32_t e = h[4], f = h[5], g = h[6], hh = h[7];
|
||||
for (int i = 0; i < 64; ++i) {
|
||||
uint32_t S1 = ror(e, 6) ^ ror(e, 11) ^ ror(e, 25);
|
||||
uint32_t ch = (e & f) ^ (~e & g);
|
||||
uint32_t t1 = hh + S1 + ch + k[i] + w[i];
|
||||
uint32_t S0 = ror(a, 2) ^ ror(a, 13) ^ ror(a, 22);
|
||||
uint32_t mj = (a & b) ^ (a & c) ^ (b & c);
|
||||
uint32_t t2 = S0 + mj;
|
||||
hh = g; g = f; f = e; e = d + t1;
|
||||
d = c; c = b; b = a; a = t1 + t2;
|
||||
}
|
||||
h[0] += a; h[1] += b; h[2] += c; h[3] += d;
|
||||
h[4] += e; h[5] += f; h[6] += g; h[7] += hh;
|
||||
}
|
||||
|
||||
void update(const uint8_t* p, size_t n) {
|
||||
len += n;
|
||||
while (n) {
|
||||
size_t take = std::min(n, size_t(64) - buf_n);
|
||||
std::memcpy(buf + buf_n, p, take);
|
||||
buf_n += take; p += take; n -= take;
|
||||
if (buf_n == 64) { block(buf); buf_n = 0; }
|
||||
}
|
||||
}
|
||||
|
||||
std::string hex() {
|
||||
uint64_t bits = len * 8;
|
||||
uint8_t pad = 0x80;
|
||||
update(&pad, 1);
|
||||
uint8_t zero = 0;
|
||||
while (buf_n != 56) update(&zero, 1);
|
||||
uint8_t tail[8];
|
||||
for (int i = 0; i < 8; ++i) tail[i] = uint8_t(bits >> (56 - i * 8));
|
||||
// update() would re-count these into len, but len is already frozen in bits.
|
||||
std::memcpy(buf + buf_n, tail, 8);
|
||||
block(buf);
|
||||
buf_n = 0;
|
||||
|
||||
static const char* d = "0123456789abcdef";
|
||||
std::string out;
|
||||
out.reserve(64);
|
||||
for (int i = 0; i < 8; ++i)
|
||||
for (int s = 28; s >= 0; s -= 4)
|
||||
out += d[(h[i] >> s) & 0xF];
|
||||
return out;
|
||||
}
|
||||
};
|
||||
|
||||
// (path, mtime, size) → digest. Hashing a 250 MB ONNX is cheap but not free, and
|
||||
// the optimizer constructs many networks in one process against the same model.
|
||||
std::mutex g_hash_mu;
|
||||
std::map<std::string, std::string> g_hash_cache;
|
||||
|
||||
std::string short_hash(const std::string& hex) {
|
||||
return hex.size() > 12 ? hex.substr(0, 12) + "…" : hex;
|
||||
}
|
||||
|
||||
} // namespace
|
||||
|
||||
std::string sha256_hex(const std::string& bytes) {
|
||||
Sha256 s;
|
||||
s.update(reinterpret_cast<const uint8_t*>(bytes.data()), bytes.size());
|
||||
return s.hex();
|
||||
}
|
||||
|
||||
std::string sha256_file_hex(const std::string& path) {
|
||||
if (path.empty()) return "";
|
||||
|
||||
std::error_code ec;
|
||||
auto size = fs::file_size(path, ec);
|
||||
if (ec) return "";
|
||||
auto mtime = fs::last_write_time(path, ec);
|
||||
if (ec) return "";
|
||||
|
||||
std::ostringstream key;
|
||||
key << path << '|' << size << '|'
|
||||
<< mtime.time_since_epoch().count();
|
||||
|
||||
{
|
||||
std::lock_guard<std::mutex> lk(g_hash_mu);
|
||||
auto it = g_hash_cache.find(key.str());
|
||||
if (it != g_hash_cache.end()) return it->second;
|
||||
}
|
||||
|
||||
std::ifstream f(path, std::ios::binary);
|
||||
if (!f) return "";
|
||||
Sha256 s;
|
||||
std::vector<char> chunk(1 << 20);
|
||||
while (f) {
|
||||
f.read(chunk.data(), static_cast<std::streamsize>(chunk.size()));
|
||||
std::streamsize got = f.gcount();
|
||||
if (got > 0) s.update(reinterpret_cast<const uint8_t*>(chunk.data()),
|
||||
static_cast<size_t>(got));
|
||||
}
|
||||
std::string hex = s.hex();
|
||||
|
||||
std::lock_guard<std::mutex> lk(g_hash_mu);
|
||||
g_hash_cache[key.str()] = hex;
|
||||
return hex;
|
||||
}
|
||||
|
||||
// ── EmbedderStamp ─────────────────────────────────────────────────────────────
|
||||
|
||||
std::string EmbedderStamp::describe() const {
|
||||
std::string name = model_name.empty() ? "<unnamed model>" : model_name;
|
||||
if (model_sha256.empty())
|
||||
return name + " (sha256 unavailable)";
|
||||
return name + " (sha256 " + short_hash(model_sha256) + ")";
|
||||
}
|
||||
|
||||
EmbedderStamp make_embedder_stamp(const std::string& model_path) {
|
||||
EmbedderStamp s;
|
||||
if (model_path.empty()) return s;
|
||||
s.model_name = fs::path(model_path).filename().string();
|
||||
s.model_sha256 = sha256_file_hex(model_path);
|
||||
if (s.model_sha256.empty())
|
||||
std::cerr << "[gallery] cannot hash embedder model " << model_path
|
||||
<< " — model binding falls back to filename only (GR-004)\n";
|
||||
return s;
|
||||
}
|
||||
|
||||
bool require_gallery_stamp_from_env() {
|
||||
const char* v = std::getenv("SAE_REQUIRE_GALLERY_STAMP");
|
||||
return v && *v && std::strcmp(v, "0") != 0;
|
||||
}
|
||||
|
||||
// ── Comparison ────────────────────────────────────────────────────────────────
|
||||
|
||||
StampCheck compare_embedder_stamps(const EmbedderStamp& built_with,
|
||||
const EmbedderStamp& loading_with,
|
||||
const std::string& gallery_desc,
|
||||
const std::string& embedder_desc) {
|
||||
StampCheck out;
|
||||
std::ostringstream m;
|
||||
|
||||
// The gallery predates GR-004 (or was written by a tool that does not stamp).
|
||||
if (built_with.empty()) {
|
||||
out.verdict = StampVerdict::unstamped;
|
||||
m << "gallery '" << gallery_desc << "' carries no embedder stamp (GR-004).\n"
|
||||
<< " gallery was built with : UNKNOWN — this file predates model binding\n"
|
||||
<< " embedder now loaded : " << loading_with.describe()
|
||||
<< " [" << embedder_desc << "]\n"
|
||||
<< " If these are not the same model every similarity from this run is\n"
|
||||
<< " meaningless but will look plausible. Rebuild or re-stamp the gallery\n"
|
||||
<< " (scripts/stamp_gallery.py), or run with SAE_REQUIRE_GALLERY_STAMP=1 to\n"
|
||||
<< " make this a hard error.";
|
||||
out.message = m.str();
|
||||
return out;
|
||||
}
|
||||
|
||||
// Gallery is stamped but we cannot say what is about to embed.
|
||||
if (loading_with.empty()) {
|
||||
out.verdict = StampVerdict::unknown_embedder;
|
||||
m << "cannot identify the embedder being used against gallery '"
|
||||
<< gallery_desc << "' (GR-004).\n"
|
||||
<< " gallery was built with : " << built_with.describe() << "\n"
|
||||
<< " embedder now loaded : UNKNOWN [" << embedder_desc << "]\n"
|
||||
<< " The binding cannot be checked, so it is not being checked.";
|
||||
out.message = m.str();
|
||||
return out;
|
||||
}
|
||||
|
||||
const bool have_both_hashes =
|
||||
!built_with.model_sha256.empty() && !loading_with.model_sha256.empty();
|
||||
|
||||
// Embedding width disagreeing is a mismatch on its own terms — different
|
||||
// spaces entirely, and it will not even be caught by a cosine that "looks fine".
|
||||
if (built_with.embed_dim != loading_with.embed_dim) {
|
||||
out.verdict = StampVerdict::mismatch;
|
||||
m << "gallery/embedder MODEL MISMATCH — refusing to run (GR-004).\n"
|
||||
<< " gallery was built with : " << built_with.describe()
|
||||
<< ", dim=" << built_with.embed_dim << " [" << gallery_desc << "]\n"
|
||||
<< " embedder now loaded : " << loading_with.describe()
|
||||
<< ", dim=" << loading_with.embed_dim << " [" << embedder_desc << "]\n"
|
||||
<< " Embedding dimensions differ; these are not the same space.";
|
||||
out.message = m.str();
|
||||
return out;
|
||||
}
|
||||
|
||||
if (have_both_hashes) {
|
||||
if (built_with.model_sha256 == loading_with.model_sha256) {
|
||||
out.verdict = StampVerdict::match;
|
||||
m << "embedder binding verified: " << built_with.describe();
|
||||
if (built_with.model_name != loading_with.model_name)
|
||||
m << " (gallery recorded it as '" << built_with.model_name
|
||||
<< "', loaded from '" << loading_with.model_name
|
||||
<< "' — same bytes, renamed file)";
|
||||
out.message = m.str();
|
||||
return out;
|
||||
}
|
||||
out.verdict = StampVerdict::mismatch;
|
||||
m << "gallery/embedder MODEL MISMATCH — refusing to run (GR-004).\n"
|
||||
<< " gallery was built with : " << built_with.model_name
|
||||
<< " sha256=" << built_with.model_sha256 << "\n"
|
||||
<< " [" << gallery_desc << "]\n"
|
||||
<< " embedder now loaded : " << loading_with.model_name
|
||||
<< " sha256=" << loading_with.model_sha256 << "\n"
|
||||
<< " [" << embedder_desc << "]\n"
|
||||
<< " Cosine similarities between embeddings from different models are\n"
|
||||
<< " meaningless but look plausible. Rebuild the gallery with the loaded\n"
|
||||
<< " model, or point the embedder at the model the gallery was built with.";
|
||||
out.message = m.str();
|
||||
return out;
|
||||
}
|
||||
|
||||
// One side has no hash (e.g. a TRT deployment with the .onnx absent). Names
|
||||
// are all we have; agreeing on them is evidence, not proof.
|
||||
if (!built_with.model_name.empty() &&
|
||||
built_with.model_name == loading_with.model_name) {
|
||||
out.verdict = StampVerdict::weak_match;
|
||||
m << "embedder binding UNPROVEN for gallery '" << gallery_desc << "' (GR-004).\n"
|
||||
<< " gallery was built with : " << built_with.describe() << "\n"
|
||||
<< " embedder now loaded : " << loading_with.describe()
|
||||
<< " [" << embedder_desc << "]\n"
|
||||
<< " Filenames agree but at least one SHA-256 is unavailable, so an\n"
|
||||
<< " in-place re-export under the same name would not be detected.";
|
||||
out.message = m.str();
|
||||
return out;
|
||||
}
|
||||
|
||||
out.verdict = StampVerdict::mismatch;
|
||||
m << "gallery/embedder MODEL MISMATCH — refusing to run (GR-004).\n"
|
||||
<< " gallery was built with : " << built_with.describe()
|
||||
<< " [" << gallery_desc << "]\n"
|
||||
<< " embedder now loaded : " << loading_with.describe()
|
||||
<< " [" << embedder_desc << "]\n"
|
||||
<< " Cosine similarities between embeddings from different models are\n"
|
||||
<< " meaningless but look plausible. Rebuild the gallery with the loaded\n"
|
||||
<< " model, or point the embedder at the model the gallery was built with.";
|
||||
out.message = m.str();
|
||||
return out;
|
||||
}
|
||||
|
||||
void enforce_embedder_stamp(const EmbedderStamp& built_with,
|
||||
const EmbedderStamp& loading_with,
|
||||
const std::string& gallery_desc,
|
||||
const std::string& embedder_desc,
|
||||
bool require_stamp) {
|
||||
const bool strict = require_stamp || require_gallery_stamp_from_env();
|
||||
StampCheck chk = compare_embedder_stamps(built_with, loading_with,
|
||||
gallery_desc, embedder_desc);
|
||||
|
||||
if (chk.fatal(strict)) {
|
||||
if (chk.verdict != StampVerdict::mismatch)
|
||||
throw std::runtime_error(chk.message +
|
||||
"\n (fatal because SAE_REQUIRE_GALLERY_STAMP / --require-gallery-stamp is set)");
|
||||
throw std::runtime_error(chk.message);
|
||||
}
|
||||
|
||||
if (chk.verdict == StampVerdict::match) {
|
||||
std::cerr << "[gallery] " << chk.message << "\n";
|
||||
} else {
|
||||
std::cerr << "\n[gallery] ***** WARNING (GR-004) *****\n"
|
||||
<< chk.message << "\n"
|
||||
<< "[gallery] ****************************\n\n";
|
||||
}
|
||||
}
|
||||
|
||||
void verify_gallery_embedder(const ActorGallery& gallery,
|
||||
const std::string& gallery_path,
|
||||
const std::string& arcface_model_path,
|
||||
bool require_stamp) {
|
||||
enforce_embedder_stamp(gallery.embedder,
|
||||
make_embedder_stamp(arcface_model_path),
|
||||
gallery_path,
|
||||
arcface_model_path.empty() ? "no --arcface given"
|
||||
: arcface_model_path,
|
||||
require_stamp);
|
||||
}
|
||||
@@ -1,115 +0,0 @@
|
||||
#pragma once
|
||||
/// TRACES: GR-004 | SR-001
|
||||
//
|
||||
// Gallery ↔ embedder binding.
|
||||
//
|
||||
// A gallery is only valid for the embedder that built it. Cosine similarities
|
||||
// between embeddings from two different models are meaningless but *look*
|
||||
// plausible — nothing crashes, nothing is obviously wrong, and every number
|
||||
// measured downstream is quietly garbage. So the embedder's identity is stamped
|
||||
// into the gallery at build time and checked by every consumer at load time.
|
||||
//
|
||||
// ── What identifies an embedder ───────────────────────────────────────────────
|
||||
// Two fields, carried together:
|
||||
//
|
||||
// model_name basename of the model file, e.g. "LVFace-B_Glint360K.onnx"
|
||||
// model_sha256 hex SHA-256 of that file's bytes
|
||||
//
|
||||
// The hash is what *decides*; the name is what a human *reads*. Neither alone is
|
||||
// enough:
|
||||
//
|
||||
// • A name alone is a promise, not a fact. Models get re-exported, re-quantised
|
||||
// and overwritten in place under an unchanged filename — which is precisely
|
||||
// the case where the weights differ and nothing else does. A name-only stamp
|
||||
// is blind to exactly the failure it exists to catch.
|
||||
// • A hash alone is correct but unreadable: "expected 3f2a… got 9c1b…" tells an
|
||||
// operator nothing about what to do next.
|
||||
//
|
||||
// SHA-256 over the file bytes is derived from the artefact rather than asserted
|
||||
// about it, is stable across machines and filesystems, and needs no registry to
|
||||
// be kept up to date. Cost is ~0.1 s for a 250 MB ONNX, paid once per process
|
||||
// (results are memoised on path+mtime+size), which is noise next to model load.
|
||||
//
|
||||
// ── Degraded and legacy cases ─────────────────────────────────────────────────
|
||||
// A TRT-backend deployment may run from a prebuilt .engine with the source .onnx
|
||||
// absent, so the hash cannot be computed. Then the name is compared alone and the
|
||||
// result is reported as a *weak* match — believed, not proven.
|
||||
//
|
||||
// Galleries built before GR-004 carry no stamp at all. They warn loudly rather
|
||||
// than fail, because the state is unknown rather than known-bad, and because
|
||||
// hard-failing every pre-existing gallery would make the check something people
|
||||
// route around rather than trust. Set require_stamp (or SAE_REQUIRE_GALLERY_STAMP=1)
|
||||
// to promote "unknown" to a hard error — that is the mode measurement work runs in.
|
||||
//
|
||||
// A *mismatch* is always fatal, in every mode, with no bypass.
|
||||
|
||||
#include <cstdint>
|
||||
#include <string>
|
||||
|
||||
struct EmbedderStamp {
|
||||
std::string model_name; // basename of the model file
|
||||
std::string model_sha256; // lowercase hex SHA-256 of the file's bytes ("" = unavailable)
|
||||
int32_t embed_dim{512};
|
||||
|
||||
bool empty() const { return model_name.empty() && model_sha256.empty(); }
|
||||
|
||||
// "LVFace-B_Glint360K.onnx (sha256 3f2a1c4d…)" — for error messages.
|
||||
std::string describe() const;
|
||||
};
|
||||
|
||||
// Identify the model at `model_path`. Missing/unreadable file → name filled from
|
||||
// the path, hash left empty (the weak-match path). Empty path → empty stamp.
|
||||
EmbedderStamp make_embedder_stamp(const std::string& model_path);
|
||||
|
||||
enum class StampVerdict {
|
||||
match, // hashes agree — binding proven
|
||||
weak_match, // names agree, no hash on one side — believed, unproven
|
||||
unstamped, // gallery predates GR-004 / was written without a stamp
|
||||
unknown_embedder, // gallery is stamped but the loaded embedder can't be identified
|
||||
mismatch, // proven different models — always fatal
|
||||
};
|
||||
|
||||
struct StampCheck {
|
||||
StampVerdict verdict{StampVerdict::match};
|
||||
std::string message; // human-readable, names BOTH sides
|
||||
|
||||
// A mismatch is fatal unconditionally. The three "cannot prove it" verdicts
|
||||
// are fatal only in strict mode.
|
||||
bool fatal(bool require_stamp) const {
|
||||
return verdict == StampVerdict::mismatch ||
|
||||
(require_stamp && verdict != StampVerdict::match);
|
||||
}
|
||||
};
|
||||
|
||||
// Pure comparison — no file I/O, no model loading. This is the unit under test.
|
||||
// `gallery_desc`/`embedder_desc` are only used to make the message locatable
|
||||
// (a gallery path, a dump path, "the embedder being loaded", …).
|
||||
StampCheck compare_embedder_stamps(const EmbedderStamp& built_with,
|
||||
const EmbedderStamp& loading_with,
|
||||
const std::string& gallery_desc = "gallery",
|
||||
const std::string& embedder_desc = "embedder");
|
||||
|
||||
// Apply the comparison: throw std::runtime_error on a fatal verdict, otherwise
|
||||
// log to stderr. `require_stamp` is OR-ed with SAE_REQUIRE_GALLERY_STAMP.
|
||||
void enforce_embedder_stamp(const EmbedderStamp& built_with,
|
||||
const EmbedderStamp& loading_with,
|
||||
const std::string& gallery_desc,
|
||||
const std::string& embedder_desc,
|
||||
bool require_stamp);
|
||||
|
||||
// Convenience for the common consumer shape: "I loaded this gallery and I am
|
||||
// about to embed with this model file." Hashes the model, then enforces.
|
||||
struct ActorGallery;
|
||||
void verify_gallery_embedder(const ActorGallery& gallery,
|
||||
const std::string& gallery_path,
|
||||
const std::string& arcface_model_path,
|
||||
bool require_stamp);
|
||||
|
||||
// SAE_REQUIRE_GALLERY_STAMP=1 → treat an unprovable binding as fatal.
|
||||
bool require_gallery_stamp_from_env();
|
||||
|
||||
// Lowercase hex SHA-256. Exposed so a test can pin the digest against the
|
||||
// published vectors, which is what guarantees the C++ and Python (hashlib)
|
||||
// stamps of the same file agree.
|
||||
std::string sha256_hex(const std::string& bytes);
|
||||
std::string sha256_file_hex(const std::string& path); // "" if unreadable
|
||||
@@ -1,6 +1,5 @@
|
||||
#include "gallery_builder.hpp"
|
||||
#include "config.hpp"
|
||||
#include "embedder_stamp.hpp"
|
||||
#include "face_utils.hpp"
|
||||
#include "inference/face_detector.hpp"
|
||||
#include "inference/face_embedder.hpp"
|
||||
@@ -42,12 +41,6 @@ ActorGallery build_gallery(const BuildConfig& cfg) {
|
||||
|
||||
ActorGallery gallery;
|
||||
|
||||
/// TRACES: GR-004 | SR-001
|
||||
// Stamp before the first embedding exists, so there is no window in which a
|
||||
// gallery holds vectors without recording what produced them.
|
||||
gallery.embedder = make_embedder_stamp(cfg.arcface_model);
|
||||
std::cerr << "[build_gallery] embedder: " << gallery.embedder.describe() << "\n";
|
||||
|
||||
for (const auto& actor_dir : fs::directory_iterator(cfg.gallery_root)) {
|
||||
if (!actor_dir.is_directory()) continue;
|
||||
|
||||
@@ -96,27 +89,6 @@ ActorGallery build_gallery(const BuildConfig& cfg) {
|
||||
return a.confidence < b.confidence;
|
||||
});
|
||||
|
||||
// Reject faces too small to embed honestly.
|
||||
//
|
||||
// The reference images are crops cut from the film, not mugshots, so
|
||||
// the detected face can be a small fraction of the image. Upscaling a
|
||||
// 30 px face to ArcFace's 112x112 feeds the model an input it was
|
||||
// never trained for, and it answers with a confident, plausible,
|
||||
// wrong embedding.
|
||||
//
|
||||
// At inference that costs one frame. Here it is permanent: a poisoned
|
||||
// reference sits in the gallery and corrupts every future match
|
||||
// against that character, which is exactly the kind of error that is
|
||||
// invisible without a study that should not have been needed.
|
||||
if (cfg.min_face_px > 0.f) {
|
||||
const float side = std::min(best.bbox.width, best.bbox.height);
|
||||
if (side < cfg.min_face_px) {
|
||||
std::cerr << " [skip] face " << side << "px < " << cfg.min_face_px
|
||||
<< "px: " << img_file.path().filename() << "\n";
|
||||
continue;
|
||||
}
|
||||
}
|
||||
|
||||
cv::Mat crop = align_face(img, best.landmarks);
|
||||
if (crop.empty()) {
|
||||
std::cerr << " [skip] alignment failed: " << img_file.path().filename() << "\n";
|
||||
|
||||
@@ -1,5 +1,4 @@
|
||||
#pragma once
|
||||
#include "gallery/gallery_report.hpp"
|
||||
#include "types.hpp"
|
||||
#include <string>
|
||||
|
||||
@@ -29,16 +28,6 @@ struct BuildConfig {
|
||||
float detector_conf{0.5f};
|
||||
float detector_nms{0.4f};
|
||||
int max_side{500}; // downscale source images to this max dimension
|
||||
|
||||
/// Minimum detected-face side, in pixels of the (possibly downscaled)
|
||||
/// source image. 0 disables the check.
|
||||
///
|
||||
/// References below this are dropped rather than upscaled: a face smaller
|
||||
/// than the embedder's input is off-distribution, and a bad reference
|
||||
/// poisons every match against that identity for the life of the gallery.
|
||||
/// Mirrors the inference-side --min-face-px so the gallery is built from
|
||||
/// the same face scales it will be matched against.
|
||||
float min_face_px{0.f};
|
||||
// before detection — TMDB portraits are ~2k px,
|
||||
// SCRFD trains on smaller faces and detection
|
||||
// confidence drops on huge inputs. 0 = disabled.
|
||||
|
||||
@@ -1,5 +1,4 @@
|
||||
#pragma once
|
||||
/// TRACES: AR-023 | SR-002
|
||||
#include "types.hpp"
|
||||
|
||||
#include <algorithm>
|
||||
@@ -9,7 +8,6 @@
|
||||
#include <cstdlib>
|
||||
#include <cstring>
|
||||
#include <fstream>
|
||||
#include <functional>
|
||||
#include <iostream>
|
||||
#include <stdexcept>
|
||||
#include <string>
|
||||
@@ -50,75 +48,14 @@ struct GalleryCalibration {
|
||||
}
|
||||
};
|
||||
|
||||
/// TRACES: AR-023, AR-024 | SR-002
|
||||
///
|
||||
/// cosine → P(same person). The one probability space the pipeline reasons in.
|
||||
///
|
||||
/// Handed to every stage that has to decide whether two embeddings are the same
|
||||
/// person — track association (AR-007), evidence discounting (AR-025), identity
|
||||
/// matching — so a threshold of 0.5 means the same thing in all of them. A stage
|
||||
/// that thresholded a raw cosine instead would be using a number that means
|
||||
/// something different for every model, gallery and face size (AR-024).
|
||||
///
|
||||
/// **No prior term.** `log_prior_odds` adjusts for the gallery's base rate, which
|
||||
/// is a question about *which of N actors*; association asks whether two faces
|
||||
/// are one person, where the balanced fit is the right answer. Passing the
|
||||
/// matcher's prior here would silently bias tracking by the size of the cast.
|
||||
inline std::function<float(float)> same_person_probability(const GalleryCalibration& cal) {
|
||||
if (!cal.valid) {
|
||||
// Loud, because the failure mode is invisible: an untuned sigmoid still
|
||||
// returns plausible probabilities, and every threshold downstream of it
|
||||
// is then a guess wearing a calibrated number's clothes.
|
||||
std::cerr << "[calibration] WARNING: no fitted calibration — association and "
|
||||
"evidence weighting fall back to the untuned default sigmoid "
|
||||
"(a=" << cal.a << ", b=" << cal.b << "). Probabilities are "
|
||||
"not meaningful for this embedder.\n";
|
||||
}
|
||||
return [cal](float similarity) { return cal.probability(similarity); };
|
||||
}
|
||||
|
||||
/// TRACES: GR-003 | SR-001
|
||||
///
|
||||
/// Everything the fit learns about the gallery on its way to two numbers.
|
||||
///
|
||||
/// The fit computes per-actor dedup counts, which actors can supply positive
|
||||
/// pairs at all, and the two similarity distributions the sigmoid is derived
|
||||
/// from — and then returns only (a, b, valid). GR-003 exists because that is the
|
||||
/// evidence for whether the calibration, and so every threshold expressed in its
|
||||
/// probability space (AR-024), rests on anything. Filling this struct costs
|
||||
/// nothing: the values already exist at the point they are copied out.
|
||||
///
|
||||
/// Per-actor vectors are indexed by the actor index used in `flat_actor`.
|
||||
struct GalleryCalibrationStats {
|
||||
int n_actors = 0;
|
||||
int min_embeddings_for_positive = 0;
|
||||
float dedup_sim_threshold = 0.f;
|
||||
|
||||
std::vector<int> distinct_per_actor; // after near-duplicate removal
|
||||
std::vector<int> duplicates_removed_per_actor;
|
||||
std::vector<char> eligible; // 1 = supplies positive pairs
|
||||
|
||||
int hist_bins = 0; // over sim ∈ [-1, 1]
|
||||
std::vector<double> intra_hist;
|
||||
std::vector<double> inter_hist;
|
||||
double n_intra_pairs = 0.0;
|
||||
double n_inter_pairs = 0.0;
|
||||
|
||||
double train_accuracy_pct = 0.0;
|
||||
};
|
||||
|
||||
// Fit a logistic sigmoid to gallery pair similarities.
|
||||
// Positive pairs: same actor, different reference images.
|
||||
// Negative pairs: different actors (all cross-actor embedding pairs).
|
||||
// Class weights balance the (typically skewed) pos/neg ratio.
|
||||
// Requires ≥2 positive pairs and ≥1 negative pair.
|
||||
//
|
||||
// `stats` is optional (GR-003): pass one to receive the dedup, eligibility and
|
||||
// distribution detail the fit would otherwise discard.
|
||||
inline GalleryCalibration calibrate_gallery(
|
||||
const std::vector<Embedding>& flat_emb,
|
||||
const std::vector<int>& flat_actor,
|
||||
GalleryCalibrationStats* stats = nullptr)
|
||||
const std::vector<int>& flat_actor)
|
||||
{
|
||||
constexpr int kMinEmbeddingsForPositive = 5;
|
||||
constexpr float kDedupSimThreshold = 1.f - 1e-7f; // sim above this = duplicate
|
||||
@@ -142,21 +79,6 @@ inline GalleryCalibration calibrate_gallery(
|
||||
std::vector<bool> actor_eligible(n_actors, false);
|
||||
int n_eligible = 0;
|
||||
|
||||
/// TRACES: GR-003 | SR-001
|
||||
// Record what the filter did, per actor, while the counts still exist.
|
||||
if (stats) {
|
||||
*stats = GalleryCalibrationStats{};
|
||||
stats->n_actors = n_actors;
|
||||
stats->min_embeddings_for_positive = kMinEmbeddingsForPositive;
|
||||
stats->dedup_sim_threshold = kDedupSimThreshold;
|
||||
stats->distinct_per_actor.assign(n_actors, 0);
|
||||
stats->duplicates_removed_per_actor.assign(n_actors, 0);
|
||||
stats->eligible.assign(n_actors, 0);
|
||||
stats->hist_bins = kHistBins;
|
||||
stats->intra_hist.assign(kHistBins, 0.0);
|
||||
stats->inter_hist.assign(kHistBins, 0.0);
|
||||
}
|
||||
|
||||
for (int ai = 0; ai < n_actors; ++ai) {
|
||||
std::vector<Embedding> kept;
|
||||
for (const auto& e : by_actor[ai]) {
|
||||
@@ -170,12 +92,6 @@ inline GalleryCalibration calibrate_gallery(
|
||||
actor_eligible[ai] = true;
|
||||
++n_eligible;
|
||||
}
|
||||
if (stats) {
|
||||
stats->distinct_per_actor[ai] = static_cast<int>(kept.size());
|
||||
stats->duplicates_removed_per_actor[ai] =
|
||||
static_cast<int>(by_actor[ai].size() - kept.size());
|
||||
stats->eligible[ai] = actor_eligible[ai] ? 1 : 0;
|
||||
}
|
||||
for (auto& e : kept) {
|
||||
flat_emb_dedup.push_back(e);
|
||||
flat_actor_dedup.push_back(ai);
|
||||
@@ -183,14 +99,6 @@ inline GalleryCalibration calibrate_gallery(
|
||||
}
|
||||
|
||||
const int n = static_cast<int>(flat_emb_dedup.size());
|
||||
|
||||
// Nothing to fit and nothing to multiply. Returning here keeps the report
|
||||
// buildable for a degenerate gallery instead of handing cv::gemm an empty
|
||||
// matrix; the per-actor stats above are already filled and still useful.
|
||||
if (n == 0) {
|
||||
std::cerr << "[calibration] no embeddings — calibration skipped\n";
|
||||
return {};
|
||||
}
|
||||
std::cerr << "[calibration] dedup: " << flat_emb.size() << " -> " << n
|
||||
<< " embeddings (" << n_eligible << "/" << n_actors
|
||||
<< " actors have >= " << kMinEmbeddingsForPositive
|
||||
@@ -291,17 +199,6 @@ inline GalleryCalibration calibrate_gallery(
|
||||
double n_pos = 0.0, n_neg = 0.0;
|
||||
for (int b = 0; b < kHistBins; ++b) { n_pos += pos_hist[b]; n_neg += neg_hist[b]; }
|
||||
|
||||
/// TRACES: GR-003 | SR-001
|
||||
// The two distributions the sigmoid is about to be fitted from. Emitted
|
||||
// whether or not the fit succeeds — a failed fit is exactly the case where
|
||||
// someone needs to see why.
|
||||
if (stats) {
|
||||
stats->intra_hist = pos_hist;
|
||||
stats->inter_hist = neg_hist;
|
||||
stats->n_intra_pairs = n_pos;
|
||||
stats->n_inter_pairs = n_neg;
|
||||
}
|
||||
|
||||
if (n_pos < 2 || n_neg < 1) {
|
||||
std::cerr << "[calibration] insufficient pairs (+" << n_pos
|
||||
<< "/-" << n_neg << ") — calibration skipped\n";
|
||||
@@ -356,7 +253,6 @@ inline GalleryCalibration calibrate_gallery(
|
||||
correct += (sig > 0.5f) ? pos_hist[b] : neg_hist[b];
|
||||
}
|
||||
double acc = 100.0 * correct / total;
|
||||
if (stats) stats->train_accuracy_pct = acc;
|
||||
|
||||
GalleryCalibration cal{a, bias, true};
|
||||
std::cerr << "[calibration] sigmoid fitted:"
|
||||
|
||||
@@ -1,495 +0,0 @@
|
||||
#pragma once
|
||||
/// TRACES: GR-003 | SR-001
|
||||
///
|
||||
/// The gallery build report — what the gallery *is*, written next to it.
|
||||
///
|
||||
/// A gallery is a silent artefact: it loads, it scores, it never complains. The
|
||||
/// two ways it fails are both invisible from the outside.
|
||||
///
|
||||
/// 1. **An actor with zero usable images can never be recognised.** They are
|
||||
/// dropped at build time (`gallery_builder.cpp` skips a directory whose
|
||||
/// images all fail detection or alignment), so afterwards nothing in the
|
||||
/// file records that they were ever meant to be there. Every scene they
|
||||
/// appear in is a guaranteed miss, and recall is capped at a number nobody
|
||||
/// computed. This is the single most useful line in the report.
|
||||
/// 2. **A gallery can be quietly bad and look fine.** The Platt sigmoid
|
||||
/// (AR-023) is fitted from two distributions — intra-class (same actor,
|
||||
/// different reference) and inter-class (different actors) similarity — and
|
||||
/// *every* threshold in the pipeline is expressed in the probability space
|
||||
/// that fit defines (AR-024): identity acceptance, track association,
|
||||
/// expansion admission, cluster merging. If those two distributions overlap
|
||||
/// heavily the fit is weak, and every downstream decision silently inherits
|
||||
/// that weakness while still reporting confident-looking probabilities. The
|
||||
/// fit already computes the distributions and throws them away; emitting
|
||||
/// them is what makes the quality of the whole probability space auditable
|
||||
/// instead of assumed.
|
||||
///
|
||||
/// The report is therefore a build artefact, not a debug aid: it is the only
|
||||
/// place the recall ceiling and the calibration's conditioning are written down.
|
||||
///
|
||||
/// **On the histograms being in cosine space.** They bin raw similarity, and
|
||||
/// that is not an AR-024 violation: no decision is taken here. These two
|
||||
/// distributions are the *input* the calibration is fitted from — they cannot be
|
||||
/// expressed in the probability space the calibration defines, because that
|
||||
/// space is their output. GR-003 asks for exactly this ("the intra/inter
|
||||
/// distributions behind it"), for the same reason GR-008 characterises an
|
||||
/// actor's reference spread in the metric space: shape is a property of the
|
||||
/// metric, decisions are a property of the probability.
|
||||
#include "gallery/gallery_calibration.hpp"
|
||||
#include "gallery/embedder_stamp.hpp"
|
||||
#include "types.hpp"
|
||||
|
||||
#include <nlohmann/json.hpp>
|
||||
|
||||
#include <algorithm>
|
||||
#include <cmath>
|
||||
#include <cstdint>
|
||||
#include <fstream>
|
||||
#include <stdexcept>
|
||||
#include <string>
|
||||
#include <vector>
|
||||
|
||||
/// TRACES: GR-003 | SR-001
|
||||
///
|
||||
/// Per-actor image accounting from the build pass, including the actors that
|
||||
/// produced nothing and were therefore dropped from the gallery.
|
||||
///
|
||||
/// Filled by `build_gallery()`. It has to be collected there and cannot be
|
||||
/// recovered later: by the time a gallery exists, an actor with no usable image
|
||||
/// is indistinguishable from an actor who was never requested.
|
||||
struct GalleryBuildAudit {
|
||||
struct ActorImages {
|
||||
std::string imdb_id;
|
||||
std::string name;
|
||||
int images_seen = 0; // candidate image files in the actor's directory
|
||||
int images_used = 0; // ...that yielded an embedding
|
||||
int unreadable = 0; // cv::imread failed
|
||||
int no_face = 0; // detector found nothing
|
||||
int align_failed = 0; // 5-point warp failed
|
||||
};
|
||||
std::vector<ActorImages> actors; // every directory seen, in build order
|
||||
};
|
||||
|
||||
/// TRACES: GR-003 | SR-001
|
||||
struct GalleryReport {
|
||||
// One row per actor the build considered. Actors with references == 0 are
|
||||
// the zero-usable-image case: present in the source tree, absent from the
|
||||
// gallery, unrecognisable for the life of the file.
|
||||
struct Actor {
|
||||
std::string imdb_id;
|
||||
std::string name;
|
||||
int images_seen = -1; // -1 = unknown (report built without a build audit)
|
||||
int references = 0; // embeddings stored in the gallery
|
||||
int distinct_references = 0; // ...after near-duplicate removal
|
||||
int duplicates_removed = 0;
|
||||
bool eligible_for_positive_pairs = false;
|
||||
};
|
||||
|
||||
// The two distributions the sigmoid is fitted from, as the fit itself saw
|
||||
// them: counts per similarity bin over [sim_min, sim_max].
|
||||
struct Distributions {
|
||||
int bins = 0;
|
||||
float sim_min = -1.f;
|
||||
float sim_max = 1.f;
|
||||
std::vector<double> intra; // same actor, different reference image
|
||||
std::vector<double> inter; // different actors
|
||||
double intra_pairs = 0.0;
|
||||
double inter_pairs = 0.0;
|
||||
double intra_mean = 0.0;
|
||||
double inter_mean = 0.0;
|
||||
// Normalised histogram intersection, Σ_b min(p_intra[b], p_inter[b]).
|
||||
// 0 = perfectly separated, 1 = indistinguishable. This is the number
|
||||
// that says whether the calibration — and so every threshold expressed
|
||||
// in its probability space — rests on anything.
|
||||
double overlap = 0.0;
|
||||
};
|
||||
|
||||
// GR-003 / AR-023 open question, reported but NOT applied. The spec asks for
|
||||
// a gallery-derived prior of intra/(intra+inter); the shipped default is
|
||||
// 0.5. Persisting the distributions makes the real value computable, so the
|
||||
// decision can be taken on evidence rather than left implicit. Behaviour is
|
||||
// unchanged: `applied` is always false here.
|
||||
struct Prior {
|
||||
double derived = 0.0; // intra_pairs / (intra_pairs + inter_pairs)
|
||||
double derived_log_odds = 0.0; // log(p/(1-p)), the term AR-023 would add
|
||||
float configured_default = 0.5f;
|
||||
bool applied = false;
|
||||
std::string note;
|
||||
};
|
||||
|
||||
std::string schema{"sae.gallery_report/1"};
|
||||
std::string gallery_path;
|
||||
EmbedderStamp embedder;
|
||||
|
||||
// ── Summary ──────────────────────────────────────────────────────────────
|
||||
int actors_total = 0; // considered (gallery + zero-usable)
|
||||
int actors_in_gallery = 0;
|
||||
int actors_zero_usable = 0;
|
||||
int actors_below_positive_threshold = 0;
|
||||
int64_t embeddings_total = 0;
|
||||
int64_t distinct_embeddings_total = 0;
|
||||
int64_t duplicates_removed_total = 0;
|
||||
double mean_embeddings_per_actor = 0.0; // over actors in the gallery
|
||||
int min_embeddings_for_positive_pairs = 0;
|
||||
float dedup_similarity_threshold = 0.f;
|
||||
|
||||
// ── Calibration ──────────────────────────────────────────────────────────
|
||||
float calib_a = 10.f;
|
||||
float calib_b = -5.f;
|
||||
bool calib_valid = false;
|
||||
uint64_t calib_hash = 0;
|
||||
double calib_train_accuracy_pct = 0.0;
|
||||
float calib_boundary_p50 = 0.f; // similarity at which P(match) = 0.5
|
||||
|
||||
Distributions distributions;
|
||||
Prior prior;
|
||||
|
||||
std::vector<Actor> actors;
|
||||
// Names duplicated out of `actors` so the two failure modes are greppable
|
||||
// without a JSON query. These are the lines a human reads first.
|
||||
std::vector<std::string> zero_usable;
|
||||
std::vector<std::string> below_positive_threshold;
|
||||
};
|
||||
|
||||
/// TRACES: GR-003 | SR-001
|
||||
///
|
||||
/// Assembles the report from the three things that know a piece of the answer:
|
||||
/// the gallery itself (who is in it, with how many references), the calibration
|
||||
/// stats (dedup, eligibility, the two distributions), and the build audit (who
|
||||
/// was considered and produced nothing). The audit is optional — a report built
|
||||
/// from a stored gallery simply cannot know about the actors that never made it.
|
||||
///
|
||||
/// `stats` is indexed by actor index, so the flat arrays handed to
|
||||
/// `calibrate_gallery()` must have used the gallery's own actor ordering.
|
||||
inline GalleryReport build_gallery_report(const ActorGallery& gallery,
|
||||
const GalleryCalibration& cal,
|
||||
const GalleryCalibrationStats& stats,
|
||||
const GalleryBuildAudit* audit = nullptr,
|
||||
const std::string& gallery_path = "",
|
||||
float configured_prior = 0.5f)
|
||||
{
|
||||
GalleryReport r;
|
||||
r.gallery_path = gallery_path;
|
||||
r.embedder = gallery.embedder;
|
||||
|
||||
r.calib_a = cal.a;
|
||||
r.calib_b = cal.b;
|
||||
r.calib_valid = cal.valid;
|
||||
r.calib_hash = gallery.calib_hash;
|
||||
r.calib_train_accuracy_pct = stats.train_accuracy_pct;
|
||||
r.calib_boundary_p50 = cal.boundary_at(0.5f);
|
||||
|
||||
r.min_embeddings_for_positive_pairs = stats.min_embeddings_for_positive;
|
||||
r.dedup_similarity_threshold = stats.dedup_sim_threshold;
|
||||
|
||||
auto audit_for = [&](const ActorGallery::Actor& a) -> const GalleryBuildAudit::ActorImages* {
|
||||
if (!audit) return nullptr;
|
||||
for (const auto& e : audit->actors) {
|
||||
if (!a.imdb_id.empty() && e.imdb_id == a.imdb_id) return &e;
|
||||
if (a.imdb_id.empty() && e.name == a.name) return &e;
|
||||
}
|
||||
return nullptr;
|
||||
};
|
||||
|
||||
for (size_t i = 0; i < gallery.actors.size(); ++i) {
|
||||
const auto& ga = gallery.actors[i];
|
||||
GalleryReport::Actor row;
|
||||
row.imdb_id = ga.imdb_id;
|
||||
row.name = ga.name;
|
||||
row.references = static_cast<int>(ga.embeddings.size());
|
||||
if (const auto* au = audit_for(ga)) row.images_seen = au->images_seen;
|
||||
|
||||
if (i < stats.distinct_per_actor.size()) {
|
||||
row.distinct_references = stats.distinct_per_actor[i];
|
||||
row.duplicates_removed = stats.duplicates_removed_per_actor[i];
|
||||
row.eligible_for_positive_pairs = stats.eligible[i] != 0;
|
||||
} else {
|
||||
// No calibration stats for this actor (the fit never saw them).
|
||||
// Report the raw count rather than a fabricated distinct count.
|
||||
row.distinct_references = row.references;
|
||||
}
|
||||
|
||||
r.embeddings_total += row.references;
|
||||
r.distinct_embeddings_total += row.distinct_references;
|
||||
r.duplicates_removed_total += row.duplicates_removed;
|
||||
if (!row.eligible_for_positive_pairs) {
|
||||
++r.actors_below_positive_threshold;
|
||||
r.below_positive_threshold.push_back(row.name);
|
||||
}
|
||||
r.actors.push_back(std::move(row));
|
||||
}
|
||||
r.actors_in_gallery = static_cast<int>(gallery.actors.size());
|
||||
|
||||
// Actors the build considered and could not use at all. They are not in the
|
||||
// gallery, so this is the only record that they exist.
|
||||
if (audit) {
|
||||
for (const auto& e : audit->actors) {
|
||||
if (e.images_used > 0) continue;
|
||||
GalleryReport::Actor row;
|
||||
row.imdb_id = e.imdb_id;
|
||||
row.name = e.name;
|
||||
row.images_seen = e.images_seen;
|
||||
row.references = 0;
|
||||
r.zero_usable.push_back(e.name);
|
||||
r.actors.push_back(std::move(row));
|
||||
}
|
||||
}
|
||||
r.actors_zero_usable = static_cast<int>(r.zero_usable.size());
|
||||
r.actors_total = r.actors_in_gallery + r.actors_zero_usable;
|
||||
r.mean_embeddings_per_actor =
|
||||
r.actors_in_gallery > 0
|
||||
? static_cast<double>(r.embeddings_total) / r.actors_in_gallery
|
||||
: 0.0;
|
||||
|
||||
// ── The two distributions, straight out of the fit ───────────────────────
|
||||
auto& d = r.distributions;
|
||||
d.bins = stats.hist_bins;
|
||||
d.sim_min = -1.f;
|
||||
d.sim_max = 1.f;
|
||||
d.intra = stats.intra_hist;
|
||||
d.inter = stats.inter_hist;
|
||||
d.intra_pairs = stats.n_intra_pairs;
|
||||
d.inter_pairs = stats.n_inter_pairs;
|
||||
|
||||
if (d.bins > 0) {
|
||||
const double bin_w = (d.sim_max - d.sim_min) / d.bins;
|
||||
double si = 0.0, se = 0.0;
|
||||
for (int b = 0; b < d.bins; ++b) {
|
||||
const double centre = d.sim_min + (b + 0.5) * bin_w;
|
||||
si += d.intra[b] * centre;
|
||||
se += d.inter[b] * centre;
|
||||
}
|
||||
if (d.intra_pairs > 0.0) d.intra_mean = si / d.intra_pairs;
|
||||
if (d.inter_pairs > 0.0) d.inter_mean = se / d.inter_pairs;
|
||||
if (d.intra_pairs > 0.0 && d.inter_pairs > 0.0) {
|
||||
double ov = 0.0;
|
||||
for (int b = 0; b < d.bins; ++b)
|
||||
ov += std::min(d.intra[b] / d.intra_pairs, d.inter[b] / d.inter_pairs);
|
||||
d.overlap = ov;
|
||||
}
|
||||
}
|
||||
|
||||
// ── The prior AR-023 leaves open — computed, reported, not applied ────────
|
||||
r.prior.configured_default = configured_prior;
|
||||
r.prior.applied = false;
|
||||
const double pair_total = d.intra_pairs + d.inter_pairs;
|
||||
if (pair_total > 0.0) {
|
||||
r.prior.derived = d.intra_pairs / pair_total;
|
||||
const double p = std::clamp(r.prior.derived, 1e-12, 1.0 - 1e-12);
|
||||
r.prior.derived_log_odds = std::log(p / (1.0 - p));
|
||||
}
|
||||
r.prior.note =
|
||||
"AR-023 specifies a gallery-derived prior of intra/(intra+inter); the shipped "
|
||||
"match_prior default is 0.5 (calibrated sigmoid used directly). The derived value "
|
||||
"is the base rate of same-actor pairs among ALL enumerated gallery pairs, so it "
|
||||
"falls as the cast grows (roughly (k-1)/((k-1)+(A-1)k) for A actors with k "
|
||||
"references each) — it is a property of gallery size as much as of the embedder. "
|
||||
"Reported here as evidence; NOT applied. Behaviour is unchanged until the choice "
|
||||
"is recorded in the spec.";
|
||||
|
||||
return r;
|
||||
}
|
||||
|
||||
// ── JSON ─────────────────────────────────────────────────────────────────────
|
||||
|
||||
/// TRACES: GR-003 | SR-001
|
||||
inline nlohmann::json gallery_report_to_json(const GalleryReport& r) {
|
||||
nlohmann::json j;
|
||||
j["schema"] = r.schema;
|
||||
j["gallery_path"] = r.gallery_path;
|
||||
j["embedder"] = {{"model_name", r.embedder.model_name},
|
||||
{"model_sha256", r.embedder.model_sha256},
|
||||
{"embed_dim", r.embedder.embed_dim}};
|
||||
|
||||
j["summary"] = {
|
||||
{"actors_total", r.actors_total},
|
||||
{"actors_in_gallery", r.actors_in_gallery},
|
||||
{"actors_zero_usable", r.actors_zero_usable},
|
||||
{"actors_below_positive_threshold", r.actors_below_positive_threshold},
|
||||
{"embeddings_total", r.embeddings_total},
|
||||
{"distinct_embeddings_total", r.distinct_embeddings_total},
|
||||
{"duplicates_removed_total", r.duplicates_removed_total},
|
||||
{"mean_embeddings_per_actor", r.mean_embeddings_per_actor},
|
||||
{"min_embeddings_for_positive_pairs", r.min_embeddings_for_positive_pairs},
|
||||
{"dedup_similarity_threshold", r.dedup_similarity_threshold}};
|
||||
|
||||
j["calibration"] = {
|
||||
{"a", r.calib_a},
|
||||
{"b", r.calib_b},
|
||||
{"valid", r.calib_valid},
|
||||
{"hash", r.calib_hash},
|
||||
{"train_accuracy_pct", r.calib_train_accuracy_pct},
|
||||
{"boundary_p50", r.calib_boundary_p50}};
|
||||
|
||||
const auto& d = r.distributions;
|
||||
j["distributions"] = {
|
||||
{"bins", d.bins},
|
||||
{"sim_min", d.sim_min},
|
||||
{"sim_max", d.sim_max},
|
||||
{"intra", d.intra},
|
||||
{"inter", d.inter},
|
||||
{"intra_pairs", d.intra_pairs},
|
||||
{"inter_pairs", d.inter_pairs},
|
||||
{"intra_mean", d.intra_mean},
|
||||
{"inter_mean", d.inter_mean},
|
||||
{"overlap", d.overlap}};
|
||||
|
||||
j["prior"] = {
|
||||
{"derived", r.prior.derived},
|
||||
{"derived_log_odds", r.prior.derived_log_odds},
|
||||
{"configured_default", r.prior.configured_default},
|
||||
{"applied", r.prior.applied},
|
||||
{"note", r.prior.note}};
|
||||
|
||||
j["zero_usable"] = r.zero_usable;
|
||||
j["below_positive_threshold"] = r.below_positive_threshold;
|
||||
|
||||
j["actors"] = nlohmann::json::array();
|
||||
for (const auto& a : r.actors) {
|
||||
j["actors"].push_back({
|
||||
{"imdb_id", a.imdb_id},
|
||||
{"name", a.name},
|
||||
{"images_seen", a.images_seen},
|
||||
{"references", a.references},
|
||||
{"distinct_references", a.distinct_references},
|
||||
{"duplicates_removed", a.duplicates_removed},
|
||||
{"eligible_for_positive_pairs", a.eligible_for_positive_pairs}});
|
||||
}
|
||||
return j;
|
||||
}
|
||||
|
||||
/// TRACES: GR-003 | SR-001
|
||||
inline GalleryReport gallery_report_from_json(const nlohmann::json& j) {
|
||||
GalleryReport r;
|
||||
r.schema = j.value("schema", std::string{});
|
||||
r.gallery_path = j.value("gallery_path", std::string{});
|
||||
if (j.contains("embedder")) {
|
||||
const auto& je = j.at("embedder");
|
||||
r.embedder.model_name = je.value("model_name", "");
|
||||
r.embedder.model_sha256 = je.value("model_sha256", "");
|
||||
r.embedder.embed_dim = je.value("embed_dim", 512);
|
||||
}
|
||||
if (j.contains("summary")) {
|
||||
const auto& s = j.at("summary");
|
||||
r.actors_total = s.value("actors_total", 0);
|
||||
r.actors_in_gallery = s.value("actors_in_gallery", 0);
|
||||
r.actors_zero_usable = s.value("actors_zero_usable", 0);
|
||||
r.actors_below_positive_threshold = s.value("actors_below_positive_threshold", 0);
|
||||
r.embeddings_total = s.value("embeddings_total", int64_t{0});
|
||||
r.distinct_embeddings_total = s.value("distinct_embeddings_total", int64_t{0});
|
||||
r.duplicates_removed_total = s.value("duplicates_removed_total", int64_t{0});
|
||||
r.mean_embeddings_per_actor = s.value("mean_embeddings_per_actor", 0.0);
|
||||
r.min_embeddings_for_positive_pairs = s.value("min_embeddings_for_positive_pairs", 0);
|
||||
r.dedup_similarity_threshold = s.value("dedup_similarity_threshold", 0.f);
|
||||
}
|
||||
if (j.contains("calibration")) {
|
||||
const auto& c = j.at("calibration");
|
||||
r.calib_a = c.value("a", 10.f);
|
||||
r.calib_b = c.value("b", -5.f);
|
||||
r.calib_valid = c.value("valid", false);
|
||||
r.calib_hash = c.value("hash", uint64_t{0});
|
||||
r.calib_train_accuracy_pct = c.value("train_accuracy_pct", 0.0);
|
||||
r.calib_boundary_p50 = c.value("boundary_p50", 0.f);
|
||||
}
|
||||
if (j.contains("distributions")) {
|
||||
const auto& d = j.at("distributions");
|
||||
r.distributions.bins = d.value("bins", 0);
|
||||
r.distributions.sim_min = d.value("sim_min", -1.f);
|
||||
r.distributions.sim_max = d.value("sim_max", 1.f);
|
||||
r.distributions.intra = d.value("intra", std::vector<double>{});
|
||||
r.distributions.inter = d.value("inter", std::vector<double>{});
|
||||
r.distributions.intra_pairs = d.value("intra_pairs", 0.0);
|
||||
r.distributions.inter_pairs = d.value("inter_pairs", 0.0);
|
||||
r.distributions.intra_mean = d.value("intra_mean", 0.0);
|
||||
r.distributions.inter_mean = d.value("inter_mean", 0.0);
|
||||
r.distributions.overlap = d.value("overlap", 0.0);
|
||||
}
|
||||
if (j.contains("prior")) {
|
||||
const auto& p = j.at("prior");
|
||||
r.prior.derived = p.value("derived", 0.0);
|
||||
r.prior.derived_log_odds = p.value("derived_log_odds", 0.0);
|
||||
r.prior.configured_default = p.value("configured_default", 0.5f);
|
||||
r.prior.applied = p.value("applied", false);
|
||||
r.prior.note = p.value("note", "");
|
||||
}
|
||||
r.zero_usable = j.value("zero_usable", std::vector<std::string>{});
|
||||
r.below_positive_threshold = j.value("below_positive_threshold", std::vector<std::string>{});
|
||||
if (j.contains("actors")) {
|
||||
for (const auto& ja : j.at("actors")) {
|
||||
GalleryReport::Actor a;
|
||||
a.imdb_id = ja.value("imdb_id", "");
|
||||
a.name = ja.value("name", "");
|
||||
a.images_seen = ja.value("images_seen", -1);
|
||||
a.references = ja.value("references", 0);
|
||||
a.distinct_references = ja.value("distinct_references", 0);
|
||||
a.duplicates_removed = ja.value("duplicates_removed", 0);
|
||||
a.eligible_for_positive_pairs = ja.value("eligible_for_positive_pairs", false);
|
||||
r.actors.push_back(std::move(a));
|
||||
}
|
||||
}
|
||||
return r;
|
||||
}
|
||||
|
||||
// "<dir>/cast.h5" → "<dir>/cast.report.json". A known gallery extension is
|
||||
// replaced rather than appended to, so the report sits beside the gallery under
|
||||
// the same stem.
|
||||
inline std::string gallery_report_path(const std::string& gallery_path) {
|
||||
auto slash = gallery_path.find_last_of("/\\");
|
||||
auto dot = gallery_path.find_last_of('.');
|
||||
std::string stem =
|
||||
(dot != std::string::npos && (slash == std::string::npos || dot > slash))
|
||||
? gallery_path.substr(0, dot)
|
||||
: gallery_path;
|
||||
return stem + ".report.json";
|
||||
}
|
||||
|
||||
/// TRACES: GR-003 | SR-001
|
||||
inline void save_gallery_report(const std::string& path, const GalleryReport& r) {
|
||||
std::ofstream out(path);
|
||||
if (!out.is_open())
|
||||
throw std::runtime_error("save_gallery_report: cannot write " + path);
|
||||
out << gallery_report_to_json(r).dump(2) << "\n";
|
||||
}
|
||||
|
||||
/// TRACES: GR-003 | SR-001
|
||||
inline GalleryReport load_gallery_report(const std::string& path) {
|
||||
std::ifstream in(path);
|
||||
if (!in.is_open())
|
||||
throw std::runtime_error("load_gallery_report: cannot open " + path);
|
||||
nlohmann::json j;
|
||||
in >> j;
|
||||
return gallery_report_from_json(j);
|
||||
}
|
||||
|
||||
/// TRACES: GR-003 | SR-001
|
||||
///
|
||||
/// The report's headline, on stderr, at build time. The file is the audit trail;
|
||||
/// this is what stops a bad gallery from being shipped without anyone noticing.
|
||||
inline void log_gallery_report(const GalleryReport& r) {
|
||||
std::cerr << "[gallery-report] " << r.actors_in_gallery << " actors / "
|
||||
<< r.embeddings_total << " embeddings"
|
||||
<< " (mean " << r.mean_embeddings_per_actor << " per actor)\n";
|
||||
if (r.actors_zero_usable > 0) {
|
||||
std::cerr << "[gallery-report] WARNING: " << r.actors_zero_usable
|
||||
<< " actor(s) have NO usable image — they can never be recognised:\n";
|
||||
for (const auto& n : r.zero_usable) std::cerr << " - " << n << "\n";
|
||||
}
|
||||
if (r.actors_below_positive_threshold > 0) {
|
||||
std::cerr << "[gallery-report] " << r.actors_below_positive_threshold
|
||||
<< " actor(s) below " << r.min_embeddings_for_positive_pairs
|
||||
<< " distinct references — they contribute no positive pairs and "
|
||||
"weaken the calibration\n";
|
||||
}
|
||||
if (r.duplicates_removed_total > 0)
|
||||
std::cerr << "[gallery-report] " << r.duplicates_removed_total
|
||||
<< " near-duplicate reference(s) removed\n";
|
||||
std::cerr << "[gallery-report] calibration valid=" << r.calib_valid
|
||||
<< " a=" << r.calib_a << " b=" << r.calib_b
|
||||
<< " intra/inter overlap=" << r.distributions.overlap
|
||||
<< " (intra mean=" << r.distributions.intra_mean
|
||||
<< ", inter mean=" << r.distributions.inter_mean << ")\n";
|
||||
std::cerr << "[gallery-report] gallery-derived prior would be "
|
||||
<< r.prior.derived << " (log-odds " << r.prior.derived_log_odds
|
||||
<< "); shipped default " << r.prior.configured_default
|
||||
<< " is in force — reported, not applied\n";
|
||||
}
|
||||
@@ -79,21 +79,6 @@ static ActorGallery load_gallery_hdf5(const std::string& path) {
|
||||
gallery.actors.push_back(std::move(actor));
|
||||
}
|
||||
|
||||
/// TRACES: GR-004 | SR-001
|
||||
// Absent /embedder group == a gallery written before model binding existed.
|
||||
// It stays readable; verify_gallery_embedder() decides what that means.
|
||||
if (file.nameExists("embedder")) {
|
||||
H5::Group eg = file.openGroup("embedder");
|
||||
H5::StrType str(H5::PredType::C_S1, H5T_VARIABLE);
|
||||
if (eg.attrExists("model_name"))
|
||||
eg.openAttribute("model_name").read(str, gallery.embedder.model_name);
|
||||
if (eg.attrExists("model_sha256"))
|
||||
eg.openAttribute("model_sha256").read(str, gallery.embedder.model_sha256);
|
||||
if (eg.attrExists("embed_dim"))
|
||||
eg.openAttribute("embed_dim").read(H5::PredType::NATIVE_INT32,
|
||||
&gallery.embedder.embed_dim);
|
||||
}
|
||||
|
||||
if (file.nameExists("calibration")) {
|
||||
H5::Group cal = file.openGroup("calibration");
|
||||
cal.openAttribute("a").read(H5::PredType::NATIVE_FLOAT, &gallery.calib_a);
|
||||
@@ -164,20 +149,6 @@ static void save_gallery_hdf5(const std::string& path, const ActorGallery& galle
|
||||
write_str_dataset(file, "name", name);
|
||||
write_str_dataset(file, "source_images", src_images);
|
||||
|
||||
/// TRACES: GR-004 | SR-001
|
||||
// Bind the file to the embedder that produced its vectors. Written only when
|
||||
// known — an empty stamp must round-trip as "unstamped", not as a stamp
|
||||
// claiming an unnamed model.
|
||||
if (!gallery.embedder.empty()) {
|
||||
H5::Group eg = file.createGroup("embedder");
|
||||
H5::DataSpace scalar(H5S_SCALAR);
|
||||
H5::StrType str(H5::PredType::C_S1, H5T_VARIABLE);
|
||||
eg.createAttribute("model_name", str, scalar).write(str, gallery.embedder.model_name);
|
||||
eg.createAttribute("model_sha256", str, scalar).write(str, gallery.embedder.model_sha256);
|
||||
eg.createAttribute("embed_dim", H5::PredType::NATIVE_INT32, scalar)
|
||||
.write(H5::PredType::NATIVE_INT32, &gallery.embedder.embed_dim);
|
||||
}
|
||||
|
||||
if (gallery.calib_hash != 0) {
|
||||
H5::Group cal = file.createGroup("calibration");
|
||||
H5::DataSpace scalar(H5S_SCALAR);
|
||||
@@ -215,17 +186,6 @@ ActorGallery load_gallery(const std::string& path) {
|
||||
<< std::chrono::duration<double>(t1 - t0).count() << "s\n";
|
||||
|
||||
ActorGallery gallery;
|
||||
|
||||
/// TRACES: GR-004 | SR-001
|
||||
// Optional top-level "embedder" object, matching the HDF5 /embedder group.
|
||||
// Written by the JSON-era helper scripts; absent in anything older.
|
||||
if (j.contains("embedder") && j.at("embedder").is_object()) {
|
||||
const auto& je = j.at("embedder");
|
||||
gallery.embedder.model_name = je.value("model_name", "");
|
||||
gallery.embedder.model_sha256 = je.value("model_sha256", "");
|
||||
gallery.embedder.embed_dim = je.value("embed_dim", 512);
|
||||
}
|
||||
|
||||
for (const auto& ja : j.at("actors")) {
|
||||
ActorGallery::Actor actor;
|
||||
actor.imdb_id = ja.value("imdb_id", "");
|
||||
|
||||
@@ -6,25 +6,12 @@
|
||||
// legacy gallery.json files are still readable for backward compatibility but
|
||||
// save_gallery always writes HDF5 regardless of the requested extension.
|
||||
//
|
||||
// GR-005 is preserved here by absence: this is the only path that serialises a
|
||||
// gallery, and it reads and writes the local filesystem only. There is no
|
||||
// upload, no client, and no encoder that could put an embedding on a wire — the
|
||||
// public server refuses to carry one (SR-004/UR-012), and the prohibition holds
|
||||
// on this side by there being nothing that would try.
|
||||
//
|
||||
/// TRACES: GR-005 | SR-005
|
||||
//
|
||||
// HDF5 layout:
|
||||
// /embeddings float32 [N, 512] all actors' refs concatenated, row-major
|
||||
// /offset int64 [A] first row of actor a in /embeddings
|
||||
// /count int32 [A] number of refs for actor a
|
||||
// /imdb_id /tmdb_id /jellyfin_id /name : variable-length string [A]
|
||||
// /source_images : variable-length string [N], parallel to /embeddings rows
|
||||
// /embedder/model_name : scalar var-len string attr — embedder file basename
|
||||
// /embedder/model_sha256 : scalar var-len string attr — SHA-256 of that file
|
||||
// /embedder/embed_dim : scalar int32 attr
|
||||
// The GR-004 model binding. Absent group == unstamped
|
||||
// (pre-GR-004 file); see gallery/embedder_stamp.hpp.
|
||||
// /calibration/a, /b : scalar float32 attrs — Platt-sigmoid P(match|sim) fit
|
||||
// /calibration/valid : scalar int8 attr (0/1)
|
||||
// /calibration/hash : scalar uint64 attr — hash of the embeddings the fit
|
||||
@@ -32,7 +19,6 @@
|
||||
//
|
||||
// Legacy JSON format (read-only):
|
||||
// {
|
||||
// "embedder": {"model_name": "...", "model_sha256": "...", "embed_dim": 512},
|
||||
// "actors": [
|
||||
// {
|
||||
// "imdb_id": "nm0000093", // optional, "" if unknown
|
||||
|
||||
+47
-128
@@ -2,8 +2,6 @@
|
||||
#include "types.hpp"
|
||||
#include "config.hpp"
|
||||
|
||||
#include <functional>
|
||||
|
||||
#include <cmath>
|
||||
#include <cstdio>
|
||||
#include <iostream>
|
||||
@@ -36,15 +34,13 @@
|
||||
// 2. Promotion (on confirmation). A track is "owned" by actor A once ≥N frames
|
||||
// have been accepted (by the matcher's calibrated posterior) as A. On
|
||||
// confirmation the retained buffer — the hard, gallery-far poses — is
|
||||
// promoted into A's per-film annex, subject to one safety gate: the band's
|
||||
// lower bound, re-applied across the whole store (see `store_coherence`).
|
||||
//
|
||||
// There is exactly one threshold here, the AR-018 band, and it is a calibrated
|
||||
// probability. Novelty is no longer a threshold at all — the eviction policy
|
||||
// above *orders* by gallery similarity rather than cutting at a constant, and
|
||||
// the band's upper bound refuses the redundant views at the door. The raw
|
||||
// cosines this replaces, expand_novelty_sim and expand_track_spread_max, are
|
||||
// retired under AR-024.
|
||||
// promoted into A's per-film annex, after two safety gates:
|
||||
// • novelty: only embeddings whose best sim to A's refs is below
|
||||
// expand_novelty_sim are added (skip poses already covered);
|
||||
// • spread: if the retained buffer's internal spread (1 − min pairwise
|
||||
// cosine sim) exceeds expand_track_spread_max the whole track is
|
||||
// rejected — such spread signals a track-ID collision merging two
|
||||
// people, whose embeddings must never enter A's annex.
|
||||
//
|
||||
// The annex is CPU-side and in-memory: it is small (tens of embeddings) so the
|
||||
// matcher scans it with a scalar loop, and it is discarded when the process
|
||||
@@ -61,15 +57,16 @@ struct TrackGallery {
|
||||
explicit TrackGallery(const Config& cfg)
|
||||
: enabled_(cfg.expand_gallery)
|
||||
, buffer_size_(std::max(1, cfg.expand_buffer_size))
|
||||
, band_lo_(cfg.expand_band_lo)
|
||||
, band_hi_(cfg.expand_band_hi)
|
||||
, novelty_sim_(cfg.expand_novelty_sim)
|
||||
, spread_max_(cfg.expand_track_spread_max)
|
||||
, min_anchor_frames_(std::max(1, cfg.expand_min_anchor_frames))
|
||||
, debug_dir_(cfg.expand_debug_dir)
|
||||
{
|
||||
if (!enabled_) return;
|
||||
std::cerr << "[track_gallery] per-film expansion ON"
|
||||
<< " buffer=" << buffer_size_
|
||||
<< " band=[" << band_lo_ << ", " << band_hi_ << "]"
|
||||
<< " novelty_sim<" << novelty_sim_
|
||||
<< " spread_max=" << spread_max_
|
||||
<< " min_anchor_frames=" << min_anchor_frames_;
|
||||
if (!debug_dir_.empty()) {
|
||||
std::filesystem::create_directories(debug_dir_);
|
||||
@@ -89,9 +86,7 @@ struct TrackGallery {
|
||||
// track_id : face_tracker track (−1 = untracked, ignored)
|
||||
// emb : this frame's raw embedding
|
||||
// best_actor : actor with the highest gallery similarity for this face
|
||||
// best_gal_sim : that similarity (best sim to best_actor's baked+annex
|
||||
// refs) — a raw cosine, the last one in this class: it is
|
||||
// calibrated on entry and only the probability is stored
|
||||
// best_gal_sim : that similarity (best sim to best_actor's baked+annex refs)
|
||||
// accepted : true if the matcher accepted this face as best_actor
|
||||
// crop : aligned crop, retained only when debug dumping is on
|
||||
void observe(int track_id, const Embedding& emb,
|
||||
@@ -122,36 +117,13 @@ struct TrackGallery {
|
||||
// matcher when it observes a cut or track disappearance.
|
||||
void forget(int track_id) { tracks_.erase(track_id); }
|
||||
|
||||
/// TRACES: AR-019 | SR-005
|
||||
/// The registry's verdict on who this track is. Authoritative: it comes from
|
||||
/// the Bayesian accumulation (AR-025), where the local tally counted raw
|
||||
/// accepted frames and so weighted thirty near-identical looks the same as
|
||||
/// thirty distinct ones.
|
||||
void set_owner(int track_id, int actor_idx) {
|
||||
if (track_id < 0 || actor_idx < 0) return;
|
||||
tracks_[track_id].registry_owner = actor_idx;
|
||||
}
|
||||
|
||||
/// TRACES: AR-024 | SR-005
|
||||
/// Supply the calibration belonging to the active embedder. Without it the
|
||||
/// band falls back to treating cosine as probability, which is wrong but
|
||||
/// bounded — and the default is loud in the header rather than silent.
|
||||
void set_calibration(std::function<float(float)> c) { calibrate_ = std::move(c); }
|
||||
|
||||
/// Embeddings the band refused. A store that admits nothing is as wrong as
|
||||
/// one that admits everything, and neither is visible without this.
|
||||
std::size_t band_rejected() const { return rejected_; }
|
||||
|
||||
// Drop every track buffer (scene cut / EOF). Mirrors face_tracker's clear.
|
||||
void clear_tracks() { tracks_.clear(); }
|
||||
|
||||
private:
|
||||
struct BufEntry {
|
||||
Embedding emb;
|
||||
/// P(same person) against the owning actor's refs when observed —
|
||||
/// calibrated at the door (AR-024), so the eviction ordering below is a
|
||||
/// comparison of probabilities and the struct holds no bare cosine.
|
||||
float gal_p{0.f};
|
||||
float gal_sim{0.f}; // best sim to owning actor's refs when observed
|
||||
cv::Mat crop; // populated only when debug_dir_ set
|
||||
};
|
||||
|
||||
@@ -160,46 +132,14 @@ private:
|
||||
std::map<int, int> actor_votes; // actor_idx → accepted-frame count
|
||||
int accepted_frames{0};
|
||||
bool promoted{false};
|
||||
int registry_owner{-1}; ///< AR-019: authoritative
|
||||
};
|
||||
|
||||
/// TRACES: AR-018, AR-024 | SR-005
|
||||
/// Banded admission: an embedding joins the store only if its similarity to
|
||||
/// something already there falls **inside a band**.
|
||||
///
|
||||
/// above the upper bound → redundant. It is another look at a pose the
|
||||
/// store already covers, and adding it teaches the annex nothing while
|
||||
/// costing a slot that a novel view could have used.
|
||||
/// below the lower bound → suspect. Within one track every face is the
|
||||
/// same person by construction, so an embedding unlike everything else
|
||||
/// on the track is evidence the construction failed — a track-ID
|
||||
/// collision or a bad detection. Admitting it is how an actor's annex
|
||||
/// gets poisoned with someone else's face.
|
||||
///
|
||||
/// Both bounds are calibrated probabilities, never raw cosines (AR-024): a
|
||||
/// bare similarity threshold means something different for every model and
|
||||
/// every face size, and this gate has to hold across both.
|
||||
///
|
||||
/// The first embedding is always admitted — there is nothing for it to be
|
||||
/// redundant with, and nothing to contradict it.
|
||||
bool admit(const TrackState& ts, const Embedding& emb) const {
|
||||
if (ts.buf.empty()) return true;
|
||||
|
||||
float p_max = 0.f;
|
||||
for (const auto& b : ts.buf)
|
||||
p_max = std::max(p_max, calibrate_(cosine_similarity(b.emb, emb)));
|
||||
|
||||
return p_max >= band_lo_ && p_max <= band_hi_;
|
||||
}
|
||||
|
||||
void insert_into_buffer(TrackState& ts, const Embedding& emb,
|
||||
float gal_sim, const cv::Mat& crop)
|
||||
{
|
||||
if (!admit(ts, emb)) { ++rejected_; return; }
|
||||
|
||||
BufEntry e;
|
||||
e.emb = emb;
|
||||
e.gal_p = calibrate_(gal_sim);
|
||||
e.emb = emb;
|
||||
e.gal_sim = gal_sim;
|
||||
if (!debug_dir_.empty() && !crop.empty()) e.crop = crop.clone();
|
||||
|
||||
if (static_cast<int>(ts.buf.size()) < buffer_size_) {
|
||||
@@ -208,17 +148,13 @@ private:
|
||||
}
|
||||
|
||||
// Buffer full: evict the member the gallery recognises best (highest
|
||||
// gal_p) — least informative — but only if the newcomer is at least as
|
||||
// gal_sim) — least informative — but only if the newcomer is at least as
|
||||
// novel. Keeping the most gallery-far views is the whole point.
|
||||
//
|
||||
// This is an *ordering*, not a threshold: there is no constant to tune,
|
||||
// and novelty-seeking lives here rather than in a cutoff. It ranks
|
||||
// probabilities, so it says the same thing across models (AR-024).
|
||||
int worst_i = -1;
|
||||
float worst_p = e.gal_p; // newcomer's probability is the bar to beat
|
||||
int worst_i = -1;
|
||||
float worst_sim = e.gal_sim; // newcomer's sim is the bar to beat
|
||||
for (int i = 0; i < static_cast<int>(ts.buf.size()); ++i) {
|
||||
if (ts.buf[i].gal_p > worst_p) {
|
||||
worst_p = ts.buf[i].gal_p;
|
||||
if (ts.buf[i].gal_sim > worst_sim) {
|
||||
worst_sim = ts.buf[i].gal_sim;
|
||||
worst_i = i;
|
||||
}
|
||||
}
|
||||
@@ -230,21 +166,28 @@ private:
|
||||
void promote(int track_id, TrackState& ts) {
|
||||
ts.promoted = true; // idempotent: never promote a track twice
|
||||
|
||||
int actor = owning_actor(ts);
|
||||
int actor = plurality_actor(ts);
|
||||
if (actor < 0) return;
|
||||
|
||||
// ── Safety gate: the band's lower bound, across the whole store ──────
|
||||
float worst = store_coherence(ts.buf);
|
||||
if (worst < band_lo_) {
|
||||
// ── Safety gate: internal spread ─────────────────────────────────────
|
||||
// A legitimate single-person track varies in pose but stays reasonably
|
||||
// self-similar. Large spread signals two people merged under one track
|
||||
// ID — reject the whole track rather than poison the actor's annex.
|
||||
float spread = buffer_spread(ts.buf);
|
||||
if (spread > spread_max_) {
|
||||
std::cerr << "[track_gallery] track " << track_id
|
||||
<< " → actor " << actor
|
||||
<< " REJECTED (worst pairwise P=" << worst
|
||||
<< " < " << band_lo_ << ", likely ID collision)\n";
|
||||
<< " REJECTED (spread " << spread
|
||||
<< " > " << spread_max_ << ", likely ID collision)\n";
|
||||
return;
|
||||
}
|
||||
|
||||
int added = 0;
|
||||
for (const auto& be : ts.buf) {
|
||||
// ── Safety gate: novelty ─────────────────────────────────────────
|
||||
// Skip poses the gallery already covers; only gallery-far views are
|
||||
// worth the annex slot (and the extra per-frame scan cost).
|
||||
if (be.gal_sim >= novelty_sim_) continue;
|
||||
annex_.push_back({be.emb, actor});
|
||||
if (!debug_dir_.empty() && !be.crop.empty())
|
||||
dump_mugshot(track_id, actor, added, be);
|
||||
@@ -253,16 +196,10 @@ private:
|
||||
|
||||
std::cerr << "[track_gallery] track " << track_id
|
||||
<< " confirmed actor " << actor
|
||||
<< " (" << ts.accepted_frames << " accepted frames, worst "
|
||||
<< "pairwise P=" << worst << ") — promoted " << added
|
||||
<< " views; annex now " << annex_.size() << "\n";
|
||||
}
|
||||
|
||||
/// Prefer the registry's verdict; fall back to the local tally only when no
|
||||
/// registry is attached (unit tests, replay harness).
|
||||
static int owning_actor(const TrackState& ts) {
|
||||
if (ts.registry_owner >= 0) return ts.registry_owner;
|
||||
return plurality_actor(ts);
|
||||
<< " (" << ts.accepted_frames << " accepted frames, spread "
|
||||
<< spread << ") — promoted " << added << "/"
|
||||
<< ts.buf.size() << " views; annex now "
|
||||
<< annex_.size() << "\n";
|
||||
}
|
||||
|
||||
static int plurality_actor(const TrackState& ts) {
|
||||
@@ -273,49 +210,31 @@ private:
|
||||
return best;
|
||||
}
|
||||
|
||||
/// TRACES: AR-018, AR-024 | SR-005
|
||||
/// The store's weakest pairwise P(same person) — the band's lower bound
|
||||
/// asked of every pair, not just of the best match at the door.
|
||||
///
|
||||
/// `admit` compares a newcomer against its *closest* existing member, so a
|
||||
/// track that drifts gradually can chain A→B→C with every step inside the
|
||||
/// band while A and C are strangers. That is precisely the shape a track-ID
|
||||
/// collision takes when two people are merged over a slow pan, so the bound
|
||||
/// is re-asked here across all pairs before anything reaches an actor's
|
||||
/// annex. Same bound, same probability space — not a second constant.
|
||||
///
|
||||
/// A store of one has no pair to disagree; it is coherent by construction,
|
||||
/// hence 1.
|
||||
float store_coherence(const std::vector<BufEntry>& buf) const {
|
||||
float worst = std::numeric_limits<float>::max();
|
||||
// Spread = 1 − min pairwise cosine similarity over the buffer (0 when <2).
|
||||
static float buffer_spread(const std::vector<BufEntry>& buf) {
|
||||
float min_sim = std::numeric_limits<float>::max();
|
||||
for (size_t i = 0; i < buf.size(); ++i)
|
||||
for (size_t j = i + 1; j < buf.size(); ++j)
|
||||
worst = std::min(worst,
|
||||
calibrate_(cosine_similarity(buf[i].emb, buf[j].emb)));
|
||||
if (worst == std::numeric_limits<float>::max()) return 1.f;
|
||||
return worst;
|
||||
min_sim = std::min(min_sim, cosine_similarity(buf[i].emb, buf[j].emb));
|
||||
if (min_sim == std::numeric_limits<float>::max()) return 0.f;
|
||||
return 1.f - min_sim;
|
||||
}
|
||||
|
||||
void dump_mugshot(int track_id, int actor, int idx, const BufEntry& be) {
|
||||
#ifdef SAE_DEBUG
|
||||
char name[64];
|
||||
std::snprintf(name, sizeof(name), "trk%d_actor%d_%d_p%.3f.jpg",
|
||||
track_id, actor, idx, be.gal_p);
|
||||
std::snprintf(name, sizeof(name), "trk%d_actor%d_%d_sim%.3f.jpg",
|
||||
track_id, actor, idx, be.gal_sim);
|
||||
cv::imwrite((std::filesystem::path(debug_dir_) / name).string(), be.crop);
|
||||
#else
|
||||
(void)track_id; (void)actor; (void)idx; (void)be;
|
||||
#endif
|
||||
}
|
||||
|
||||
/// cosine → P(same person). The one probability space the pipeline reasons
|
||||
/// in; see gallery_calibration.hpp's same_person_probability.
|
||||
std::function<float(float)> calibrate_{[](float c) { return std::max(0.f, c); }};
|
||||
std::size_t rejected_{0}; ///< admissions refused by the band
|
||||
|
||||
bool enabled_;
|
||||
int buffer_size_;
|
||||
float band_lo_; ///< AR-018, from cfg.expand_band_lo
|
||||
float band_hi_; ///< AR-018, from cfg.expand_band_hi
|
||||
float novelty_sim_;
|
||||
float spread_max_;
|
||||
int min_anchor_frames_;
|
||||
std::string debug_dir_;
|
||||
|
||||
|
||||
+2
-28
@@ -15,7 +15,6 @@
|
||||
|
||||
#include "types.hpp"
|
||||
#include "config.hpp"
|
||||
#include "gallery/embedder_stamp.hpp"
|
||||
#include "gallery/gallery_store.hpp"
|
||||
#include "nodes/face_tracker_node.hpp"
|
||||
#include "nodes/identity_matcher_node.hpp"
|
||||
@@ -164,9 +163,6 @@ static Config config_from_dict(nb::dict d) {
|
||||
getd("anneal_sec", cfg.anneal_sec);
|
||||
// gallery expansion (usually off for sweeps; expose so it can be toggled)
|
||||
if (d.contains("expand_gallery")) cfg.expand_gallery = nb::cast<bool>(d["expand_gallery"]);
|
||||
/// TRACES: GR-004 | SR-001
|
||||
if (d.contains("require_gallery_stamp"))
|
||||
cfg.require_gallery_stamp = nb::cast<bool>(d["require_gallery_stamp"]);
|
||||
return cfg;
|
||||
}
|
||||
|
||||
@@ -214,17 +210,8 @@ NB_MODULE(sae_kpn, m) {
|
||||
net.add(std::move(name), std::move(node));
|
||||
}, "net"_a, "name"_a, "config"_a, "capacity"_a = 16);
|
||||
|
||||
/// TRACES: GR-004 | SR-001
|
||||
// embedder_model / embedder_sha256 identify whatever produced the embeddings
|
||||
// that will be fed in. In a replay those come from the dump's own stamp (see
|
||||
// scripts/optimizer/SCHEMA.md), because there is no live embedder in the
|
||||
// network — the dump *is* the embedder as far as this gallery is concerned.
|
||||
// Passing neither leaves the binding unverifiable, which warns loudly and is
|
||||
// fatal under SAE_REQUIRE_GALLERY_STAMP.
|
||||
m.def("add_identity_matcher", [](Net& net, std::string name, std::string gallery_path,
|
||||
nb::dict cfg_dict, std::size_t cap,
|
||||
std::string embedder_model,
|
||||
std::string embedder_sha256) {
|
||||
nb::dict cfg_dict, std::size_t cap) {
|
||||
Config cfg = config_from_dict(cfg_dict);
|
||||
cfg.gallery_path = gallery_path; // needed to persist refreshed calibration back
|
||||
// Cache loaded galleries by path so a threshold sweep (many networks, same
|
||||
@@ -235,24 +222,11 @@ NB_MODULE(sae_kpn, m) {
|
||||
if (it == cache.end())
|
||||
it = cache.emplace(gallery_path,
|
||||
std::make_shared<ActorGallery>(load_gallery(gallery_path))).first;
|
||||
|
||||
// Checked on every construction, not only on the cache miss: the same
|
||||
// process may replay several dumps against one cached gallery.
|
||||
EmbedderStamp feeding;
|
||||
feeding.model_name = std::move(embedder_model);
|
||||
feeding.model_sha256 = std::move(embedder_sha256);
|
||||
enforce_embedder_stamp(it->second->embedder, feeding, gallery_path,
|
||||
feeding.model_name.empty()
|
||||
? "embeddings fed into this network"
|
||||
: feeding.model_name,
|
||||
cfg.require_gallery_stamp);
|
||||
|
||||
auto node = std::make_shared<kpn::ObjectVariantNodeWrapper<
|
||||
IdentityMatcherFunc, SaeVariant, kpn::in<"tracked">, kpn::out<"matched">>>(
|
||||
cap, *it->second, cfg);
|
||||
net.add(std::move(name), std::move(node));
|
||||
}, "net"_a, "name"_a, "gallery"_a, "config"_a, "capacity"_a = 16,
|
||||
"embedder_model"_a = "", "embedder_sha256"_a = "");
|
||||
}, "net"_a, "name"_a, "gallery"_a, "config"_a, "capacity"_a = 16);
|
||||
|
||||
m.def("add_scene_tracker", [](Net& net, std::string name, nb::dict cfg_dict, std::size_t cap) {
|
||||
Config cfg = config_from_dict(cfg_dict);
|
||||
|
||||
+13
-148
@@ -1,9 +1,5 @@
|
||||
// scene_analyze — identify actors in a movie using a KPN pipeline
|
||||
//
|
||||
// TRACES: DP-001, DP-002 | PR-004
|
||||
// One analysis core; the CLI is a front-end over it and must not fork pipeline
|
||||
// logic. Other deployment modes (DP-003, DP-004) wrap this same core.
|
||||
//
|
||||
// KPN topology (release build):
|
||||
//
|
||||
// [frame_source] ──Frame──► [face_detector] ──SceneFrame──► [face_aligner]
|
||||
@@ -39,8 +35,8 @@
|
||||
// --max-faces <N> max faces kept per frame (default: 10)
|
||||
// --expand-gallery enable per-film gallery expansion from track continuity
|
||||
// --expand-buffer <N> per-track diversity buffer size (default: 20)
|
||||
// --expand-band-lo <p> store admission floor, P(same person) (default: 0.90)
|
||||
// --expand-band-hi <p> store admission ceiling, P(same person) (default: 0.95)
|
||||
// --expand-novelty-sim <f> promote only views with best sim < f (default: 0.55)
|
||||
// --expand-spread-max <f> reject track if buffer spread > f (default: 0.60)
|
||||
// --expand-min-anchor <N> accepted frames before a track confirms (default: 3)
|
||||
// --expand-debug-dir <p> dump promoted mugshots + embeddings here (SAE_DEBUG)
|
||||
// (SAE_DEBUG only)
|
||||
@@ -49,7 +45,6 @@
|
||||
|
||||
#include "config.hpp"
|
||||
#include "types.hpp"
|
||||
#include "gallery/embedder_stamp.hpp"
|
||||
#include "gallery/gallery_store.hpp"
|
||||
#include "nodes/frame_source_node.hpp"
|
||||
#include "nodes/camera_position_change_detector_node.hpp"
|
||||
@@ -60,8 +55,6 @@
|
||||
#include "nodes/identity_matcher_node.hpp"
|
||||
#include "nodes/scene_tracker_node.hpp"
|
||||
#include "nodes/scene_detector_node.hpp"
|
||||
#include "scene_boundaries.hpp"
|
||||
#include "nodes/scene_boundary_annotator_node.hpp"
|
||||
#include "nodes/result_sink_node.hpp"
|
||||
#include "nodes/embedding_dump_node.hpp"
|
||||
#ifdef SAE_DEBUG
|
||||
@@ -83,18 +76,6 @@
|
||||
|
||||
// ── CLI parsing ───────────────────────────────────────────────────────────────
|
||||
|
||||
/// TRACES: AR-010, AR-004 | SR-002
|
||||
/// How deeply the sampled branch is buffered behind the dense one. TransNetV2
|
||||
/// needs kWindow (100) dense frames before it can score any of them, so the face
|
||||
/// branch must lag by at least that much or it asks about frames nobody has
|
||||
/// looked at yet. Backpressure turns depth into lag: the fanout blocks on the
|
||||
/// slower branch rather than dropping, so the detector simply runs ahead.
|
||||
static constexpr std::size_t kSceneJoinDepth = 256;
|
||||
|
||||
/// Set when the scene branch is built, so shutdown can report whether the join
|
||||
/// actually worked.
|
||||
static std::shared_ptr<SceneBoundaries> scene_stats;
|
||||
|
||||
static Config parse_args(int argc, char** argv) {
|
||||
Config cfg;
|
||||
cfg.detector_model = kDefaultDetectorModel;
|
||||
@@ -133,7 +114,6 @@ static Config parse_args(int argc, char** argv) {
|
||||
else if (arg("--detector")) cfg.detector_model = next();
|
||||
else if (arg("--detector-engine")) cfg.detector_engine = next();
|
||||
else if (arg("--arcface")) cfg.arcface_model = next();
|
||||
else if (arg("--require-gallery-stamp")) cfg.require_gallery_stamp = true;
|
||||
else if (arg("--arcface-engine")) cfg.arcface_engine = next();
|
||||
else if (arg("--conf")) cfg.detector_conf = std::stof(next());
|
||||
else if (arg("--max-faces")) cfg.max_faces = std::stoi(next());
|
||||
@@ -142,13 +122,15 @@ static Config parse_args(int argc, char** argv) {
|
||||
else if (arg("--ratio-ceil")) cfg.match_ratio_ceil = std::stof(next());
|
||||
else if (arg("--track-alpha")) cfg.track_alpha = std::stof(next());
|
||||
else if (arg("--track-min-iou")) cfg.track_min_iou = std::stof(next());
|
||||
else if (arg("--track-min-prob")) cfg.track_assoc_min_prob = std::stof(next());
|
||||
else if (arg("--track-extinction")) cfg.track_extinction_sec = std::stod(next());
|
||||
else if (arg("--track-max-embed")) cfg.track_max_embed_dist = std::stof(next());
|
||||
else if (arg("--track-max-missing")) cfg.track_max_frames_missing = std::stoi(next());
|
||||
else if (arg("--cut-revive-sim")) cfg.cut_revive_sim = std::stof(next());
|
||||
else if (arg("--cut-inactive-max")) cfg.cut_inactive_max_frames = std::stoi(next());
|
||||
else if (arg("--anneal")) cfg.anneal_sec = std::stod(next());
|
||||
else if (arg("--expand-gallery")) cfg.expand_gallery = true;
|
||||
else if (arg("--expand-buffer")) cfg.expand_buffer_size = std::stoi(next());
|
||||
else if (arg("--expand-band-lo")) cfg.expand_band_lo = std::stof(next());
|
||||
else if (arg("--expand-band-hi")) cfg.expand_band_hi = std::stof(next());
|
||||
else if (arg("--expand-novelty-sim")) cfg.expand_novelty_sim = std::stof(next());
|
||||
else if (arg("--expand-spread-max")) cfg.expand_track_spread_max = std::stof(next());
|
||||
else if (arg("--expand-min-anchor")) cfg.expand_min_anchor_frames = std::stoi(next());
|
||||
else if (arg("--expand-debug-dir")) cfg.expand_debug_dir = next();
|
||||
else if (arg("--trt-cache")) cfg.trt.cache_dir = next();
|
||||
@@ -185,11 +167,6 @@ int main(int argc, char** argv) {
|
||||
ActorGallery gallery;
|
||||
try {
|
||||
gallery = load_gallery(cfg.gallery_path);
|
||||
/// TRACES: GR-004 | SR-001
|
||||
// Hard startup error before a single frame is decoded: a gallery built
|
||||
// with another embedder yields plausible-looking, meaningless matches.
|
||||
verify_gallery_embedder(gallery, cfg.gallery_path, cfg.arcface_model,
|
||||
cfg.require_gallery_stamp);
|
||||
} catch (const std::exception& e) {
|
||||
std::cerr << "Gallery error: " << e.what() << "\n";
|
||||
return 1;
|
||||
@@ -206,36 +183,10 @@ int main(int argc, char** argv) {
|
||||
FaceDetectorFunc detector_fn{cfg};
|
||||
FaceAlignerFunc aligner_fn;
|
||||
EmbedderFunc embedder_fn{cfg};
|
||||
// Constructed before the tracker: it fits (or loads) the calibration, and
|
||||
// the tracker must decide in that same probability space (AR-024).
|
||||
FaceTrackerFunc ftracker_fn{cfg};
|
||||
IdentityMatcherFunc matcher_fn {gallery, cfg};
|
||||
|
||||
/// TRACES: AR-007, AR-008, AR-012, AR-024 | SR-002
|
||||
// The registry is created here and shared, not owned by a node: track state
|
||||
// is not a stage in the stream, it is state several stages read and write,
|
||||
// and its final answer is only known when a track dies.
|
||||
auto same_person = same_person_probability(matcher_fn.calibration());
|
||||
TrackRegistry::Config reg_cfg;
|
||||
reg_cfg.extinction_sec = cfg.track_extinction_sec;
|
||||
auto registry = std::make_shared<TrackRegistry>(
|
||||
reg_cfg, EvidenceDiscounter(same_person));
|
||||
|
||||
matcher_fn.set_registry(registry);
|
||||
|
||||
|
||||
FaceTrackerFunc ftracker_fn{cfg, registry, same_person};
|
||||
SceneTrackerFunc tracker_fn {cfg};
|
||||
ResultSinkFunc sink_fn {cfg, done};
|
||||
|
||||
/// TRACES: AR-012, AR-016 | IR-002, IR-003 | SR-002
|
||||
// A reaped track goes straight to the aggregator, so the registry holds only
|
||||
// live tracks and its size is bounded by concurrent on-screen faces rather
|
||||
// than growing with the film.
|
||||
registry->on_track_dead([&sink_fn](const DeadTrack& d) { sink_fn.add_claim(d); });
|
||||
// AR-016: a film ends with faces on screen and those tracks have not timed
|
||||
// out. Without this flush the closing scene's cast is silently never
|
||||
// emitted — a loss that reads as a recognition miss, not a bookkeeping bug.
|
||||
sink_fn.set_pre_write_hook([registry](double last_ts) { registry->flush(last_ts); });
|
||||
#ifdef SAE_DEBUG
|
||||
DebugRendererFunc debug_fn {cfg};
|
||||
#endif
|
||||
@@ -281,26 +232,6 @@ int main(int argc, char** argv) {
|
||||
}
|
||||
});
|
||||
|
||||
// Report *why* a node died. A Closed event alone says only that one
|
||||
// stopped; the exception it carried is what identifies the fault, and
|
||||
// without this listener it is discarded at the node boundary. Returning
|
||||
// false keeps the existing semantics — the node still stops and the
|
||||
// Closed handler above still aborts the run — but the run now names the
|
||||
// cause instead of leaving it to be reconstructed from a debugger.
|
||||
net.set_error_handler(
|
||||
[&](std::string_view node_name, std::exception_ptr eptr) {
|
||||
std::string what = "unknown exception";
|
||||
try {
|
||||
if (eptr) std::rethrow_exception(eptr);
|
||||
} catch (const std::exception& e) {
|
||||
what = e.what();
|
||||
} catch (...) {
|
||||
}
|
||||
std::lock_guard<std::mutex> lk(event_mtx);
|
||||
std::cerr << "[main] node '" << node_name << "' threw: " << what << "\n";
|
||||
return false;
|
||||
});
|
||||
|
||||
std::cerr << "[main] starting pipeline…\n";
|
||||
net.start();
|
||||
|
||||
@@ -316,45 +247,15 @@ int main(int argc, char** argv) {
|
||||
net.stop();
|
||||
net.print_diagnostics();
|
||||
|
||||
/// TRACES: AR-004 | SR-002
|
||||
// A dropped frame does not degrade a result, it silently changes one —
|
||||
// the output is a claim about footage that was never analysed, and
|
||||
// nothing in the file says so. Since AR-004 made data pushes block, a
|
||||
// drop can no longer happen on the data path, so any drop here means
|
||||
// either that fix regressed (it lives in the KPN submodule, one line,
|
||||
// easy to lose in an update) or a channel was disabled mid-run.
|
||||
//
|
||||
// Reporting it in a footer and exiting 0 made both invisible: the run
|
||||
// "succeeded" and the truth file looked complete. Fail instead.
|
||||
/// TRACES: AR-010 | SR-002
|
||||
if (scene_stats) {
|
||||
std::cerr << "[scene_annotate] boundaries=" << scene_stats->count()
|
||||
<< " scored_through=" << scene_stats->scored_through() << "s";
|
||||
// The tail is expected: frames after the detector's last full
|
||||
// window are never covered, and no amount of buffering changes
|
||||
// that. They are counted rather than silently treated as
|
||||
// boundary-free, which is the distinction that matters.
|
||||
if (scene_stats->outran() > 0)
|
||||
std::cerr << " unscored=" << scene_stats->outran()
|
||||
<< " frame(s) past the detector's last window — treated as"
|
||||
" boundary-free, which is unverified rather than known";
|
||||
std::cerr << "\n";
|
||||
}
|
||||
|
||||
bool dropped = false;
|
||||
{
|
||||
std::lock_guard<std::mutex> lk(event_mtx);
|
||||
if (!overflow_counts.empty()) {
|
||||
dropped = true;
|
||||
std::cerr << "[main] ERROR: frames were dropped (channel overflow):\n";
|
||||
std::cerr << "[main] dropped frames (channel overflow):\n";
|
||||
for (const auto& [name, count] : overflow_counts)
|
||||
std::cerr << " " << name << ": " << count << "\n";
|
||||
std::cerr << "[main] The output would describe footage that was never "
|
||||
"analysed. Refusing to report success.\n";
|
||||
}
|
||||
}
|
||||
if (node_crashed.load(std::memory_order_acquire)) return 1;
|
||||
return dropped ? 2 : 0;
|
||||
return node_crashed.load(std::memory_order_acquire) ? 1 : 0;
|
||||
};
|
||||
|
||||
// ── Build static network and run ──────────────────────────────────────────
|
||||
@@ -395,21 +296,6 @@ int main(int argc, char** argv) {
|
||||
if (cfg.scene_detect) {
|
||||
scene_done.store(false, std::memory_order_release); // now a real terminal branch
|
||||
SceneDetectorFunc scene_fn{cfg, scene_done};
|
||||
|
||||
/// TRACES: AR-010 | SR-002
|
||||
// The join of the decode butterfly. source fans out to the dense
|
||||
// TransNetV2 branch and the sampled face branch; boundaries found on the
|
||||
// first have to reach the second, and cannot ride the frames because the
|
||||
// branches run in parallel.
|
||||
//
|
||||
// TransNetV2 buffers kWindow frames before it can score any of them, so
|
||||
// the face branch must lag by at least that much or it will ask about
|
||||
// frames nobody has looked at yet. Channel depth is what creates the lag:
|
||||
// with backpressure (AR-004) the fanout blocks on the slower branch, so
|
||||
// a deep face-branch channel lets the detector run ahead by its window
|
||||
// rather than dropping anything.
|
||||
auto boundaries = std::make_shared<SceneBoundaries>();
|
||||
scene_fn.set_boundaries(boundaries);
|
||||
kpn::ObjectNode<SceneDetectorFunc, kpn::in<"dense">, kpn::out<>, "scene_detector", 0>
|
||||
scene_node(scene_fn, 128);
|
||||
|
||||
@@ -426,34 +312,13 @@ int main(int argc, char** argv) {
|
||||
return true;
|
||||
}
|
||||
return false;
|
||||
}, kSceneJoinDepth);
|
||||
|
||||
/// TRACES: AR-010 | SR-002
|
||||
// Stamp is_scene_boundary from the detector's published verdict. tol is
|
||||
// half a sample interval: the two branches sample at different rates, so
|
||||
// a boundary found on a dense frame rarely lands exactly on a sampled
|
||||
// one, and half an interval attributes it to the nearest sampled frame
|
||||
// and no further.
|
||||
//
|
||||
// outran() counts frames that arrived before the detector had scored
|
||||
// them. Nonzero means the join depth is too shallow for the window, and
|
||||
// those frames were annotated from an incomplete verdict — which would
|
||||
// otherwise look exactly like "no boundary here".
|
||||
SceneBoundaryAnnotatorFunc annotate_fn{boundaries, 0.5 / cfg.sample_fps};
|
||||
kpn::ObjectNode<SceneBoundaryAnnotatorFunc, kpn::in<"frame">, kpn::out<"frame">,
|
||||
"scene_annotate", 0> annotate(annotate_fn, kSceneJoinDepth);
|
||||
|
||||
// Reported at shutdown: without this the join is unverifiable, and an
|
||||
// annotator that never fired looks identical to footage with no
|
||||
// boundaries.
|
||||
scene_stats = boundaries;
|
||||
}, 32);
|
||||
|
||||
auto net = kpn::make_network(
|
||||
kpn::edge(source.output<"raw">(), campos.input<"raw">()),
|
||||
kpn::edge(source.output<"raw">(), scene_node.input<"dense">()),
|
||||
kpn::edge(campos.output<"frame">(), decimate.input<0>()),
|
||||
kpn::edge(decimate.output<0>(), annotate.input<"frame">()),
|
||||
kpn::edge(annotate.output<"frame">(), detector.input<"frame">()),
|
||||
kpn::edge(decimate.output<0>(), detector.input<"frame">()),
|
||||
kpn::edge(detector.output<"scene">(), aligner.input<"scene">()),
|
||||
kpn::edge(aligner.output<"aligned">(), embedder.input<"aligned">()),
|
||||
kpn::edge(embedder.output<"embedded">(), ftracker.input<"embedded">()),
|
||||
|
||||
@@ -26,8 +26,7 @@
|
||||
// The node is a pure pass-through: it forwards the Frame unchanged except for
|
||||
// is_cut, so it slots between frame_source and face_detector without altering the
|
||||
// downstream contract. eof frames are forwarded immediately without processing.
|
||||
//
|
||||
/// TRACES: AR-009 | SR-002
|
||||
|
||||
struct CameraPositionChangeDetectorFunc {
|
||||
static constexpr std::string_view label() { return "camera_position_change_detector"; }
|
||||
|
||||
|
||||
@@ -17,8 +17,7 @@
|
||||
// All crops in one frame are batched into a single forward pass (capped at
|
||||
// embed_batch_size). The backend serialises itself; we only call it from the
|
||||
// single embedder thread.
|
||||
//
|
||||
/// TRACES: AR-006 | SR-002
|
||||
|
||||
struct EmbedderFunc {
|
||||
static constexpr std::string_view label() { return "embedder"; }
|
||||
|
||||
|
||||
@@ -1,113 +1,14 @@
|
||||
#pragma once
|
||||
/// TRACES: VR-001, VR-010 | PR-002
|
||||
#include "types.hpp"
|
||||
#include "config.hpp"
|
||||
#include "gallery/embedder_stamp.hpp"
|
||||
|
||||
#include <H5Cpp.h>
|
||||
|
||||
#include <atomic>
|
||||
#include <cstdint>
|
||||
#include <iostream>
|
||||
#include <optional>
|
||||
#include <string>
|
||||
#include <type_traits>
|
||||
#include <vector>
|
||||
|
||||
// ── DumpProvenance ────────────────────────────────────────────────────────────
|
||||
/// TRACES: VR-010 | PR-002
|
||||
// Everything that determined a dump's *content*, read back tolerantly.
|
||||
//
|
||||
// Two dumps of the same film with different detector thresholds, a different
|
||||
// `dense_scale`, or scene detection on versus off are different measurements of
|
||||
// different things — but they are byte-shaped identically, so a consumer that
|
||||
// mixes them gets a plausible number from an incoherent input. GR-004 closed the
|
||||
// worst case (a cross-model replay, where every cosine is meaningless); this
|
||||
// closes the rest.
|
||||
//
|
||||
// Every field is optional because dumps written before VR-010 lack the
|
||||
// attributes. A missing field reads as *unknown*, never as a default — a
|
||||
// silently-defaulted `detector_conf` is exactly the fabricated provenance the
|
||||
// requirement exists to prevent ("a fixture whose provenance is unknown is worse
|
||||
// than no fixture, because it will be trusted").
|
||||
struct DumpProvenance {
|
||||
// Model identity
|
||||
std::optional<std::string> embedder_model; // GR-004
|
||||
std::optional<std::string> embedder_sha256; // GR-004
|
||||
std::optional<std::string> detector_model;
|
||||
|
||||
// Sampling
|
||||
std::optional<std::string> movie;
|
||||
std::optional<float> sample_fps;
|
||||
std::optional<double> start_sec;
|
||||
std::optional<double> end_sec; // -1 = to end of file
|
||||
|
||||
// Detection — what the run admitted into the dump
|
||||
std::optional<float> detector_conf;
|
||||
std::optional<float> detector_nms;
|
||||
std::optional<float> min_face_px;
|
||||
std::optional<int> max_faces; // 0 = uncapped (AR-003)
|
||||
|
||||
// Frame geometry
|
||||
std::optional<float> dense_scale;
|
||||
std::optional<float> bbox_upscale; // faces/bbox × this = original-resolution px
|
||||
std::optional<float> cut_threshold;
|
||||
|
||||
// Scene detection. The reason this flag exists: `is_scene_boundary` is
|
||||
// all-zero both when TransNetV2 found no boundaries and when it never ran,
|
||||
// and no amount of staring at the array distinguishes them.
|
||||
std::optional<bool> scene_detect;
|
||||
|
||||
// Downstream knob that shaped nothing in the dump but everything a replay is
|
||||
// compared against — recorded so a sweep can be told apart from the baseline.
|
||||
std::optional<float> track_assoc_min_prob;
|
||||
};
|
||||
|
||||
// Read whatever provenance a dump carries. Never throws on a missing attribute;
|
||||
// an old dump simply yields a DumpProvenance full of empty optionals.
|
||||
inline DumpProvenance read_dump_provenance(const H5::H5File& f) {
|
||||
DumpProvenance p;
|
||||
auto str = [&](const char* n, std::optional<std::string>& out) {
|
||||
if (!f.attrExists(n)) return;
|
||||
// Written as a variable-length string, so the read must name the same
|
||||
// type explicitly — the default would truncate to a fixed length.
|
||||
H5::StrType vlen(H5::PredType::C_S1, H5T_VARIABLE);
|
||||
std::string v;
|
||||
f.openAttribute(n).read(vlen, v);
|
||||
out = v;
|
||||
};
|
||||
auto num = [&](const char* n, const H5::PredType& dt, auto& out) {
|
||||
if (!f.attrExists(n)) return;
|
||||
typename std::decay_t<decltype(out)>::value_type v{};
|
||||
f.openAttribute(n).read(dt, &v);
|
||||
out = v;
|
||||
};
|
||||
|
||||
str("embedder_model", p.embedder_model);
|
||||
str("embedder_sha256", p.embedder_sha256);
|
||||
str("detector_model", p.detector_model);
|
||||
str("movie", p.movie);
|
||||
|
||||
num("sample_fps", H5::PredType::NATIVE_FLOAT, p.sample_fps);
|
||||
num("start_sec", H5::PredType::NATIVE_DOUBLE, p.start_sec);
|
||||
num("end_sec", H5::PredType::NATIVE_DOUBLE, p.end_sec);
|
||||
num("detector_conf", H5::PredType::NATIVE_FLOAT, p.detector_conf);
|
||||
num("detector_nms", H5::PredType::NATIVE_FLOAT, p.detector_nms);
|
||||
num("min_face_px", H5::PredType::NATIVE_FLOAT, p.min_face_px);
|
||||
num("max_faces", H5::PredType::NATIVE_INT, p.max_faces);
|
||||
num("dense_scale", H5::PredType::NATIVE_FLOAT, p.dense_scale);
|
||||
num("bbox_upscale", H5::PredType::NATIVE_FLOAT, p.bbox_upscale);
|
||||
num("cut_threshold", H5::PredType::NATIVE_FLOAT, p.cut_threshold);
|
||||
num("track_assoc_min_prob", H5::PredType::NATIVE_FLOAT, p.track_assoc_min_prob);
|
||||
|
||||
if (f.attrExists("scene_detect")) {
|
||||
uint8_t v = 0;
|
||||
f.openAttribute("scene_detect").read(H5::PredType::NATIVE_UINT8, &v);
|
||||
p.scene_detect = (v != 0);
|
||||
}
|
||||
return p;
|
||||
}
|
||||
|
||||
// ── EmbeddingDumpFunc ─────────────────────────────────────────────────────────
|
||||
// KPN sink that taps the EmbeddedSceneFrame channel and writes the per-frame face
|
||||
// metadata + embeddings to one HDF5 file (schema: scripts/optimizer/SCHEMA.md).
|
||||
@@ -124,44 +25,12 @@ struct EmbeddingDumpFunc {
|
||||
: path_(cfg.dump_embeddings_path), movie_(cfg.movie_path),
|
||||
sample_fps_(cfg.sample_fps), done_(done)
|
||||
{
|
||||
/// TRACES: GR-004 | SR-001
|
||||
// A dump is a bag of embeddings with no model attached, replayed against a
|
||||
// gallery hours or weeks later — the same silent cross-model hazard as the
|
||||
// gallery itself, so it carries the same stamp.
|
||||
stamp_ = make_embedder_stamp(cfg.arcface_model);
|
||||
|
||||
/// TRACES: VR-010 | PR-002
|
||||
// The rest of what determined this file's content. Captured from the live
|
||||
// Config at construction, so it describes the run that is being written
|
||||
// rather than whatever config happens to be lying around at read time.
|
||||
prov_.detector_model = basename_of(cfg.detector_model);
|
||||
prov_.detector_conf = cfg.detector_conf;
|
||||
prov_.detector_nms = cfg.detector_nms;
|
||||
prov_.min_face_px = cfg.min_face_px;
|
||||
prov_.max_faces = cfg.max_faces;
|
||||
prov_.cut_threshold = cfg.cut_threshold;
|
||||
prov_.dense_scale = cfg.dense_scale;
|
||||
prov_.start_sec = cfg.start_sec;
|
||||
prov_.end_sec = cfg.end_sec;
|
||||
prov_.scene_detect = cfg.scene_detect;
|
||||
prov_.track_assoc_min_prob = cfg.track_assoc_min_prob;
|
||||
|
||||
std::cerr << "[embedding_dump] writing " << path_
|
||||
<< " embedder: " << stamp_.describe()
|
||||
<< " detector: " << *prov_.detector_model
|
||||
<< " @conf " << cfg.detector_conf
|
||||
<< " scene_detect=" << (cfg.scene_detect ? "on" : "off") << "\n";
|
||||
std::cerr << "[embedding_dump] writing " << path_ << "\n";
|
||||
}
|
||||
|
||||
void operator()(EmbeddedSceneFrame ef) {
|
||||
if (ef.source.eof) { flush(); return; }
|
||||
|
||||
/// TRACES: VR-010 | PR-002
|
||||
// Taken from the frames themselves, not recomputed from dense_scale — the
|
||||
// factor the source actually stamped on them is the one that maps
|
||||
// faces/bbox back to original resolution, whatever rule produced it.
|
||||
if (!prov_.bbox_upscale) prov_.bbox_upscale = ef.source.bbox_upscale;
|
||||
|
||||
const int32_t n = static_cast<int32_t>(ef.faces.size());
|
||||
ts_.push_back(ef.source.timestamp_sec);
|
||||
fidx_.push_back(ef.source.frame_idx);
|
||||
@@ -194,18 +63,9 @@ struct EmbeddingDumpFunc {
|
||||
}
|
||||
|
||||
private:
|
||||
// Root attributes are additive: schema_version stays 1 across VR-010, because
|
||||
// every reader takes attributes by name with a default (replay.py) or an
|
||||
// existence check (read_dump_provenance), so an old dump loses nothing and a
|
||||
// new dump breaks nothing. A bump is for a change to the *datasets*.
|
||||
static constexpr int kSchemaVersion = 1;
|
||||
static constexpr int kEmbedDim = 512;
|
||||
|
||||
static std::string basename_of(const std::string& path) {
|
||||
const auto slash = path.find_last_of("/\\");
|
||||
return slash == std::string::npos ? path : path.substr(slash + 1);
|
||||
}
|
||||
|
||||
template<typename T>
|
||||
void write_vec(H5::Group& g, const char* name, const std::vector<T>& v,
|
||||
const H5::PredType& dtype, hsize_t cols = 0) {
|
||||
@@ -217,49 +77,20 @@ private:
|
||||
if (!v.empty()) ds.write(v.data(), dtype);
|
||||
}
|
||||
|
||||
static void attr_str(H5::H5File& f, const char* name, const std::string& v) {
|
||||
H5::StrType str(H5::PredType::C_S1, H5T_VARIABLE);
|
||||
f.createAttribute(name, str, H5::DataSpace(H5S_SCALAR)).write(str, v);
|
||||
}
|
||||
|
||||
template<typename T>
|
||||
static void attr_num(H5::H5File& f, const char* name, const H5::PredType& dt, T v) {
|
||||
f.createAttribute(name, dt, H5::DataSpace(H5S_SCALAR)).write(dt, &v);
|
||||
}
|
||||
|
||||
void write_hdf5() {
|
||||
H5::H5File file(path_, H5F_ACC_TRUNC);
|
||||
|
||||
// root attrs
|
||||
attr_num(file, "schema_version", H5::PredType::NATIVE_INT, kSchemaVersion);
|
||||
attr_num(file, "embed_dim", H5::PredType::NATIVE_INT, kEmbedDim);
|
||||
attr_num(file, "sample_fps", H5::PredType::NATIVE_FLOAT, sample_fps_);
|
||||
attr_str(file, "movie", movie_);
|
||||
/// TRACES: GR-004 | SR-001
|
||||
attr_str(file, "embedder_model", stamp_.model_name);
|
||||
attr_str(file, "embedder_sha256", stamp_.model_sha256);
|
||||
|
||||
/// TRACES: VR-010 | PR-002
|
||||
attr_str(file, "detector_model", prov_.detector_model.value_or(""));
|
||||
attr_num(file, "detector_conf", H5::PredType::NATIVE_FLOAT, *prov_.detector_conf);
|
||||
attr_num(file, "detector_nms", H5::PredType::NATIVE_FLOAT, *prov_.detector_nms);
|
||||
attr_num(file, "min_face_px", H5::PredType::NATIVE_FLOAT, *prov_.min_face_px);
|
||||
attr_num(file, "max_faces", H5::PredType::NATIVE_INT, *prov_.max_faces);
|
||||
attr_num(file, "cut_threshold", H5::PredType::NATIVE_FLOAT, *prov_.cut_threshold);
|
||||
attr_num(file, "dense_scale", H5::PredType::NATIVE_FLOAT, *prov_.dense_scale);
|
||||
// Recorded, NOT applied — faces/bbox stays in the detector's own frame
|
||||
// space so a replay feeds the tracker exactly what the live run fed it.
|
||||
attr_num(file, "bbox_upscale", H5::PredType::NATIVE_FLOAT,
|
||||
prov_.bbox_upscale.value_or(1.f));
|
||||
attr_num(file, "start_sec", H5::PredType::NATIVE_DOUBLE, *prov_.start_sec);
|
||||
attr_num(file, "end_sec", H5::PredType::NATIVE_DOUBLE, *prov_.end_sec);
|
||||
attr_num(file, "track_assoc_min_prob", H5::PredType::NATIVE_FLOAT,
|
||||
*prov_.track_assoc_min_prob);
|
||||
// 0/1, matching the uint8 booleans in frames/. Tells "TransNetV2 found no
|
||||
// boundaries" apart from "TransNetV2 never ran", which is/was the same
|
||||
// all-zero is_scene_boundary array either way.
|
||||
attr_num(file, "scene_detect", H5::PredType::NATIVE_UINT8,
|
||||
static_cast<uint8_t>(*prov_.scene_detect ? 1 : 0));
|
||||
auto scalar = H5::DataSpace(H5S_SCALAR);
|
||||
auto ver = file.createAttribute("schema_version", H5::PredType::NATIVE_INT, scalar);
|
||||
int sv = kSchemaVersion; ver.write(H5::PredType::NATIVE_INT, &sv);
|
||||
auto ed = file.createAttribute("embed_dim", H5::PredType::NATIVE_INT, scalar);
|
||||
int dim = kEmbedDim; ed.write(H5::PredType::NATIVE_INT, &dim);
|
||||
auto fps = file.createAttribute("sample_fps", H5::PredType::NATIVE_FLOAT, scalar);
|
||||
fps.write(H5::PredType::NATIVE_FLOAT, &sample_fps_);
|
||||
H5::StrType str(H5::PredType::C_S1, H5T_VARIABLE);
|
||||
auto mv = file.createAttribute("movie", str, scalar);
|
||||
mv.write(str, movie_);
|
||||
|
||||
H5::Group frames = file.createGroup("frames");
|
||||
write_vec(frames, "timestamp_sec", ts_, H5::PredType::NATIVE_DOUBLE);
|
||||
@@ -279,9 +110,7 @@ private:
|
||||
<< conf_.size() << " faces → " << path_ << "\n";
|
||||
}
|
||||
|
||||
std::string path_, movie_;
|
||||
EmbedderStamp stamp_;
|
||||
DumpProvenance prov_;
|
||||
std::string path_, movie_;
|
||||
float sample_fps_;
|
||||
std::atomic<bool>& done_;
|
||||
std::atomic<bool> written_{false};
|
||||
|
||||
@@ -4,16 +4,12 @@
|
||||
#include <iostream>
|
||||
|
||||
// ── FaceAlignerFunc ───────────────────────────────────────────────────────────
|
||||
/// TRACES: AR-005, AR-030 | SR-002
|
||||
///
|
||||
// KPN node: applies a 5-point similarity transform to each detected face,
|
||||
// producing a 112×112 BGR crop suitable for ArcFace inference.
|
||||
//
|
||||
// Alignment is an Umeyama least-squares fit over all five landmarks (AR-005),
|
||||
// not a robust one: a RANSAC fit discards the very landmarks AR-030 reads.
|
||||
// Degenerate detections (where the fit fails) are dropped from the output
|
||||
// vectors. The fit's residual is the AR-030 visibility measure and comes free,
|
||||
// since the warp needs the transform anyway.
|
||||
// Alignment uses cv::estimateAffinePartial2D (RANSAC) to fit the detected
|
||||
// landmarks to ArcFace canonical positions. Degenerate detections (where the
|
||||
// affine fit fails) are silently dropped from the output vectors.
|
||||
|
||||
struct FaceAlignerFunc {
|
||||
static constexpr std::string_view label() { return "face_aligner"; }
|
||||
@@ -28,15 +24,11 @@ struct FaceAlignerFunc {
|
||||
crops.reserve(sf.faces.size());
|
||||
|
||||
for (auto& face : sf.faces) {
|
||||
// The AR-030 misfit comes from the transform the warp already needs,
|
||||
// so visibility costs no extra fit.
|
||||
float residual = -1.f;
|
||||
cv::Mat crop = align_face(sf.source.image, face.landmarks, &residual);
|
||||
cv::Mat crop = align_face(sf.source.image, face.landmarks);
|
||||
if (crop.empty()) {
|
||||
std::cerr << "[face_aligner] degenerate detection skipped\n";
|
||||
continue;
|
||||
}
|
||||
face.alignment_residual = residual;
|
||||
good_faces.push_back(face);
|
||||
crops.push_back(std::move(crop));
|
||||
}
|
||||
|
||||
@@ -1,5 +1,4 @@
|
||||
#pragma once
|
||||
/// TRACES: AR-001 | SR-002
|
||||
#include "config.hpp"
|
||||
#include "inference/face_detector.hpp"
|
||||
|
||||
@@ -44,11 +43,7 @@ struct FaceDetectorFunc {
|
||||
[](const DetectedFace& a, const DetectedFace& b) {
|
||||
return a.bbox.area() > b.bbox.area();
|
||||
});
|
||||
// TRACES: AR-003 | SR-002
|
||||
// Largest-first ordering is kept regardless: it is load-bearing for
|
||||
// deterministic association, since the Hungarian solver tie-breaks on
|
||||
// index order (see the replay determinism test).
|
||||
if (max_faces_ > 0 && static_cast<int>(faces.size()) > max_faces_)
|
||||
if (static_cast<int>(faces.size()) > max_faces_)
|
||||
faces.resize(max_faces_);
|
||||
|
||||
return {std::move(f), std::move(faces)};
|
||||
|
||||
+164
-171
@@ -1,124 +1,102 @@
|
||||
#pragma once
|
||||
/// TRACES: AR-007, AR-008, AR-024 | SR-002
|
||||
///
|
||||
/// FaceTrackerFunc — KPN node that links face detections into tracks.
|
||||
///
|
||||
/// **The registry is the tracker's state.** The node owns no track map of its
|
||||
/// own: it drives `TrackRegistry` through a `FrameScope` and reads the same
|
||||
/// `Track` objects everything else reads. Two parallel copies could disagree,
|
||||
/// and every divergence would surface as a wrong presence window rather than as
|
||||
/// a crash — silently, and only in the output.
|
||||
///
|
||||
/// **One candidate pool** (AR-008). `last_seen` alone distinguishes a track that
|
||||
/// is on screen from one that is dormant, and it only affects whether IoU means
|
||||
/// anything. There is no parked pool and no revival branch: re-associating a
|
||||
/// track whose face was lost — across a cut or not — is ordinary inter-frame
|
||||
/// association, and it falls out of the embedding comparison already being done.
|
||||
///
|
||||
/// Assignment cost (track i, detection j):
|
||||
///
|
||||
/// p = P(same person | cosine(track mean, detection)) ← calibrated
|
||||
/// alpha = base weight, or 0 when position carries no information
|
||||
/// cost = alpha·(1 − IoU) + (1 − alpha)·(1 − p)
|
||||
///
|
||||
/// gated to INF unless the pair is admissible on position *or* on identity.
|
||||
///
|
||||
/// **alpha is frame- and track-dependent** (AR-007). It falls to 0 —
|
||||
/// embedding only — when either:
|
||||
/// - the frame is flagged `is_cut` / `is_scene_boundary`: the viewpoint
|
||||
/// changed, so the same person is at a new position; or
|
||||
/// - the track is dormant (`last_seen` set): time has passed since its box was
|
||||
/// last observed, so that box is stale regardless of cuts.
|
||||
/// Both are the same statement — spatial continuity is broken — arrived at from
|
||||
/// two directions, which is why they collapse into one rule rather than two
|
||||
/// branches.
|
||||
///
|
||||
/// **Everything is thresholded in probability space** (AR-024). The cosine goes
|
||||
/// through the calibration before it is compared to anything; the raw-cosine
|
||||
/// constants `track_max_embed_dist` and `cut_revive_sim` are retired.
|
||||
|
||||
#include "types.hpp"
|
||||
#include "config.hpp"
|
||||
#include "track_registry.hpp"
|
||||
|
||||
#include <algorithm>
|
||||
#include <cmath>
|
||||
#include <functional>
|
||||
#include <iostream>
|
||||
#include <limits>
|
||||
#include <map>
|
||||
#include <memory>
|
||||
#include <stdexcept>
|
||||
#include <string_view>
|
||||
#include <vector>
|
||||
|
||||
// ── FaceTrackerFunc ───────────────────────────────────────────────────────────
|
||||
// KPN node: links face detections across consecutive frames using the Hungarian
|
||||
// algorithm on a combined spatial (IoU) + embedding (cosine distance) cost.
|
||||
//
|
||||
// Each track accumulates a running directional mean of its ArcFace embeddings
|
||||
// (averaged then re-normalised to the unit sphere), used as the embedding side
|
||||
// of the assignment cost below for more stable track continuity.
|
||||
//
|
||||
// Assignment cost (track i, detection j):
|
||||
// cost = alpha * (1 - IoU) + (1-alpha) * min(cosine_dist/2, 1)
|
||||
// Gated to INF when IoU < min_iou AND cosine_dist > max_embed_dist.
|
||||
//
|
||||
// Unmatched tracks have their frames_missing counter incremented; they are
|
||||
// expired once frames_missing > max_frames_missing.
|
||||
//
|
||||
// Cross-cut re-association. A camera-angle change (Frame::is_cut, set by
|
||||
// camera_position_change_detector) destroys spatial (IoU) continuity — the same
|
||||
// person reappears at a new position — but not identity. On a cut the tracker
|
||||
// does NOT discard its tracks; it parks them in an inactive pool keyed by their
|
||||
// last-frame raw embedding. A post-cut detection whose raw cosine similarity to
|
||||
// a parked track's last-frame embedding is ≥ cut_revive_sim revives that track:
|
||||
// the original track_id, mean embedding and n_frames are restored (only the bbox
|
||||
// jumps to the new detection), so identity continuity survives the cut. Parked
|
||||
// tracks left unrevived for cut_inactive_max_frames are finally dropped.
|
||||
|
||||
struct FaceTrackerFunc {
|
||||
static constexpr std::string_view label() { return "face_tracker"; }
|
||||
|
||||
/// cosine similarity → P(same person). Supplied by the caller so the fit
|
||||
/// belonging to the active embedder is used (AR-023/AR-024) — the same
|
||||
/// pattern, and normally the same function object, as
|
||||
/// `EvidenceDiscounter::Calibrate`.
|
||||
using Calibrate = std::function<float(float)>;
|
||||
struct TrackState {
|
||||
cv::Rect2f bbox;
|
||||
Embedding mean_emb{};
|
||||
Embedding last_emb{}; // raw embedding of the most recent matched frame
|
||||
int n_frames{0};
|
||||
int frames_missing{0};
|
||||
};
|
||||
|
||||
/// The registry is a constructor argument, not an option: a tracker without
|
||||
/// one would have to keep its own tracks, which is the defect this replaces.
|
||||
FaceTrackerFunc(const Config& cfg,
|
||||
std::shared_ptr<TrackRegistry> registry,
|
||||
Calibrate calibrate)
|
||||
: registry_(std::move(registry))
|
||||
, calibrate_(std::move(calibrate))
|
||||
, alpha_base_(cfg.track_alpha)
|
||||
explicit FaceTrackerFunc(const Config& cfg)
|
||||
: alpha_(cfg.track_alpha)
|
||||
, min_iou_(cfg.track_min_iou)
|
||||
, min_assoc_prob_(cfg.track_assoc_min_prob)
|
||||
, max_embed_dist_(cfg.track_max_embed_dist)
|
||||
, max_missing_(cfg.track_max_frames_missing)
|
||||
, revive_sim_(cfg.cut_revive_sim)
|
||||
, inactive_max_(cfg.cut_inactive_max_frames)
|
||||
{
|
||||
if (!registry_)
|
||||
throw std::invalid_argument("face_tracker: registry must not be null");
|
||||
if (!calibrate_)
|
||||
throw std::invalid_argument("face_tracker: a calibration is required — "
|
||||
"association is decided in probability space");
|
||||
|
||||
std::cerr << "[face_tracker] alpha_base=" << alpha_base_
|
||||
std::cerr << "[face_tracker] alpha=" << alpha_
|
||||
<< " min_iou=" << min_iou_
|
||||
<< " min_assoc_prob=" << min_assoc_prob_ << "\n";
|
||||
<< " max_embed_dist=" << max_embed_dist_
|
||||
<< " max_missing=" << max_missing_
|
||||
<< " cut_revive_sim=" << revive_sim_
|
||||
<< " cut_inactive_max=" << inactive_max_ << "\n";
|
||||
}
|
||||
|
||||
TrackedSceneFrame operator()(EmbeddedSceneFrame ef) {
|
||||
if (ef.source.eof) {
|
||||
// Deliberately does *not* flush the registry. The identity matcher
|
||||
// runs downstream and its votes for the final frames are still in
|
||||
// flight; reaping here would drop them (they would land on ids that
|
||||
// no longer exist and show up as dropped_votes). AR-016's flush
|
||||
// belongs at the pipeline's termination point, after the last vote.
|
||||
boxes_.clear();
|
||||
tracks_.clear();
|
||||
inactive_.clear();
|
||||
TrackedSceneFrame out;
|
||||
out.source = std::move(ef.source);
|
||||
return out;
|
||||
}
|
||||
|
||||
const double t = ef.source.timestamp_sec;
|
||||
const int n_det = static_cast<int>(ef.embeddings.size());
|
||||
const int n_det = static_cast<int>(ef.embeddings.size());
|
||||
|
||||
// Unconditional: the clock must advance on frames with no detections
|
||||
// too, or a track only dies when some unrelated face happens to appear
|
||||
// and a film that ends mid-track never closes it (AR-013).
|
||||
auto scope = registry_->begin_frame(t);
|
||||
// Camera-angle change: park active tracks instead of destroying them so
|
||||
// they can be revived by identity (raw last-frame embedding cosine) once
|
||||
// the same people reappear from the new angle.
|
||||
if (ef.source.is_cut && !tracks_.empty()) {
|
||||
std::cerr << "[face_tracker] cut — parking " << tracks_.size()
|
||||
<< " track(s) into inactive pool\n";
|
||||
for (auto& [tid, ts] : tracks_) {
|
||||
ts.frames_missing = 0; // repurpose as time-since-parked counter
|
||||
inactive_[tid] = std::move(ts);
|
||||
}
|
||||
tracks_.clear();
|
||||
}
|
||||
|
||||
// One pool (AR-008) — on-screen and dormant tracks compete together.
|
||||
std::vector<Track*> cands = scope.candidates();
|
||||
const int n_trk = static_cast<int>(cands.size());
|
||||
// Age the inactive pool every frame and drop tracks parked too long.
|
||||
for (auto it = inactive_.begin(); it != inactive_.end(); ) {
|
||||
it->second.frames_missing++;
|
||||
it = (it->second.frames_missing > inactive_max_)
|
||||
? inactive_.erase(it) : std::next(it);
|
||||
}
|
||||
|
||||
prune_boxes(cands);
|
||||
std::vector<Spatial*> sp(n_trk);
|
||||
for (int ti = 0; ti < n_trk; ++ti)
|
||||
sp[ti] = &boxes_.try_emplace(cands[ti]->id, Spatial{{}, t, false})
|
||||
.first->second;
|
||||
|
||||
// AR-007 — the frame half of the frame-dependent weighting. Both flags
|
||||
// say the same thing to the tracker: whatever was at that position is
|
||||
// not there any more.
|
||||
const bool viewpoint_change =
|
||||
ef.source.is_cut || ef.source.is_scene_boundary;
|
||||
// Snapshot active track IDs so the map can be modified safely below
|
||||
std::vector<int> tids;
|
||||
tids.reserve(tracks_.size());
|
||||
for (auto& [tid, _] : tracks_) tids.push_back(tid);
|
||||
const int n_trk = static_cast<int>(tids.size());
|
||||
|
||||
// ── Cost matrix [n_trk × n_det] ──────────────────────────────────────
|
||||
constexpr float INF_COST = 1e6f;
|
||||
@@ -126,108 +104,109 @@ struct FaceTrackerFunc {
|
||||
std::vector<float>(n_det, INF_COST));
|
||||
|
||||
for (int ti = 0; ti < n_trk; ++ti) {
|
||||
// Spatial continuity holds only for a track that was on screen, whose
|
||||
// box we have actually observed, on a frame that did not change the
|
||||
// viewpoint. Otherwise the box is stale and IoU is noise.
|
||||
const bool spatial_meaningful =
|
||||
sp[ti]->observed && cands[ti]->on_screen() && !viewpoint_change;
|
||||
const float alpha = spatial_meaningful ? alpha_base_ : 0.f;
|
||||
|
||||
const TrackState& ts = tracks_[tids[ti]];
|
||||
for (int di = 0; di < n_det; ++di) {
|
||||
// AR-024 — the cosine is converted before it is used for
|
||||
// anything, including the gate below.
|
||||
const float p = calibrate_(
|
||||
cosine_similarity(cands[ti]->mean, ef.embeddings[di]));
|
||||
const float iou_v = spatial_meaningful
|
||||
? iou(sp[ti]->bbox, ef.faces[di].bbox) : 0.f;
|
||||
|
||||
// Either signal on its own can admit a link: a face that moved a
|
||||
// little but whose embedding degraded (blur, profile turn) is
|
||||
// still linkable on position, and a face that jumped across the
|
||||
// frame is still linkable on identity. Neither ⇒ no link.
|
||||
const bool spatial_ok = spatial_meaningful && iou_v >= min_iou_;
|
||||
const bool identity_ok = p >= min_assoc_prob_;
|
||||
if (!spatial_ok && !identity_ok) continue;
|
||||
|
||||
cost[ti][di] = alpha * (1.f - iou_v) + (1.f - alpha) * (1.f - p);
|
||||
float iou_v = iou(ts.bbox, ef.faces[di].bbox);
|
||||
float emb_d = (ts.n_frames > 0)
|
||||
? 1.f - cosine_similarity(ts.mean_emb, ef.embeddings[di])
|
||||
: 1.f;
|
||||
if (iou_v < min_iou_ && emb_d > max_embed_dist_) continue;
|
||||
float s = 1.f - iou_v;
|
||||
float e = std::min(emb_d * 0.5f, 1.f);
|
||||
cost[ti][di] = alpha_ * s + (1.f - alpha_) * e;
|
||||
}
|
||||
}
|
||||
|
||||
// ── Hungarian assignment ─────────────────────────────────────────────
|
||||
// ── Hungarian assignment ──────────────────────────────────────────────
|
||||
std::vector<int> assign(n_trk, -1);
|
||||
if (n_trk > 0 && n_det > 0)
|
||||
assign = hungarian(cost, n_trk, n_det);
|
||||
|
||||
// ── Build output frame ───────────────────────────────────────────────
|
||||
// ── Build output frame ────────────────────────────────────────────────
|
||||
TrackedSceneFrame out;
|
||||
out.source = ef.source;
|
||||
out.faces = ef.faces;
|
||||
out.crops = ef.crops;
|
||||
out.embeddings = ef.embeddings;
|
||||
out.source = ef.source;
|
||||
out.faces = ef.faces;
|
||||
out.crops = ef.crops;
|
||||
out.embeddings = ef.embeddings;
|
||||
out.track_ids.assign(n_det, -1);
|
||||
|
||||
std::vector<bool> det_matched(n_det, false);
|
||||
|
||||
// Update matched tracks
|
||||
for (int ti = 0; ti < n_trk; ++ti) {
|
||||
const int di = assign[ti];
|
||||
const int id = cands[ti]->id;
|
||||
const bool valid =
|
||||
(di >= 0 && di < n_det && cost[ti][di] < INF_COST * 0.5f);
|
||||
|
||||
int di = assign[ti];
|
||||
bool valid = (di >= 0 && di < n_det && cost[ti][di] < INF_COST * 0.5f);
|
||||
TrackState& ts = tracks_[tids[ti]];
|
||||
if (!valid) {
|
||||
// Only a track that *was* on screen can become lost, and it
|
||||
// becomes lost as of its last sighting, never as of now — the
|
||||
// gap after the final sighting is never claimed (AR-013). A
|
||||
// track already dormant is left alone so its extinction clock
|
||||
// keeps running from the right instant.
|
||||
if (cands[ti]->on_screen()) scope.mark_lost(id, sp[ti]->last_ts);
|
||||
ts.frames_missing++;
|
||||
continue;
|
||||
}
|
||||
update_mean(ts.mean_emb, ts.n_frames, ef.embeddings[di]);
|
||||
ts.last_emb = ef.embeddings[di];
|
||||
ts.bbox = ef.faces[di].bbox;
|
||||
ts.n_frames++;
|
||||
ts.frames_missing = 0;
|
||||
det_matched[di] = true;
|
||||
|
||||
out.track_ids[di] = tids[ti];
|
||||
}
|
||||
|
||||
// Handle unmatched detections: first try to revive a parked track by
|
||||
// identity (raw last-frame embedding cosine), else start a fresh track.
|
||||
for (int di = 0; di < n_det; ++di) {
|
||||
if (det_matched[di]) continue;
|
||||
|
||||
int tid = revive_from_inactive(ef.embeddings[di]);
|
||||
if (tid >= 0) {
|
||||
// Restore the parked track: keep its identity statistics
|
||||
// (mean_emb, n_frames), jump the bbox to the new detection.
|
||||
TrackState ts = std::move(inactive_[tid]);
|
||||
inactive_.erase(tid);
|
||||
update_mean(ts.mean_emb, ts.n_frames, ef.embeddings[di]);
|
||||
ts.last_emb = ef.embeddings[di];
|
||||
ts.bbox = ef.faces[di].bbox;
|
||||
ts.n_frames++;
|
||||
ts.frames_missing = 0;
|
||||
tracks_[tid] = std::move(ts);
|
||||
out.track_ids[di] = tid;
|
||||
std::cerr << "[face_tracker] revived track " << tid
|
||||
<< " across cut\n";
|
||||
continue;
|
||||
}
|
||||
|
||||
scope.mark_seen(id, t, ef.embeddings[di]);
|
||||
sp[ti]->bbox = ef.faces[di].bbox;
|
||||
sp[ti]->last_ts = t;
|
||||
sp[ti]->observed = true;
|
||||
det_matched[di] = true;
|
||||
out.track_ids[di] = id;
|
||||
tid = next_id_++;
|
||||
TrackState ts;
|
||||
ts.bbox = ef.faces[di].bbox;
|
||||
ts.mean_emb = ef.embeddings[di];
|
||||
ts.last_emb = ef.embeddings[di];
|
||||
ts.n_frames = 1;
|
||||
tracks_[tid] = ts;
|
||||
out.track_ids[di] = tid;
|
||||
}
|
||||
|
||||
for (int di = 0; di < n_det; ++di) {
|
||||
if (det_matched[di]) continue;
|
||||
const int id = scope.create(t, ef.embeddings[di]);
|
||||
boxes_[id] = Spatial{ef.faces[di].bbox, t, true};
|
||||
out.track_ids[di] = id;
|
||||
// Expire stale tracks
|
||||
for (auto it = tracks_.begin(); it != tracks_.end(); ) {
|
||||
it = (it->second.frames_missing > max_missing_)
|
||||
? tracks_.erase(it) : std::next(it);
|
||||
}
|
||||
|
||||
// No reaping here: begin_frame's tick owns the extinction sweep, so
|
||||
// there is exactly one place a track can die.
|
||||
return out;
|
||||
}
|
||||
|
||||
private:
|
||||
// ── Spatial annotation ───────────────────────────────────────────────────
|
||||
// The one piece of per-track state the registry does not hold, because it is
|
||||
// not about presence: where the face was, and when it was last seen there.
|
||||
// Keyed by registry track id and pruned against `candidates()` every frame,
|
||||
// so it cannot outlive or contradict the registry — it annotates the pool
|
||||
// rather than duplicating it.
|
||||
struct Spatial {
|
||||
cv::Rect2f bbox{};
|
||||
double last_ts{0.0}; ///< timestamp of the last frame this track matched
|
||||
bool observed{false}; ///< false until a detection has been assigned
|
||||
};
|
||||
|
||||
// Drop boxes for ids the registry no longer has. `candidates()` is the
|
||||
// authority on what exists; anything else is a leak (and, for a reused id,
|
||||
// would be a stale box attached to a different person).
|
||||
void prune_boxes(const std::vector<Track*>& cands) {
|
||||
if (boxes_.size() == cands.size()) return; // common case: nothing died
|
||||
std::map<int, Spatial> kept;
|
||||
for (const Track* t : cands) {
|
||||
auto it = boxes_.find(t->id);
|
||||
if (it != boxes_.end()) kept.emplace(t->id, it->second);
|
||||
// Pick the parked track whose last-frame embedding is most similar to emb,
|
||||
// returning its id if that raw cosine similarity clears revive_sim_, else -1.
|
||||
// The caller removes the returned track from the pool, so a later detection in
|
||||
// the same frame cannot claim it again.
|
||||
int revive_from_inactive(const Embedding& emb) const {
|
||||
int best_tid = -1;
|
||||
float best_sim = revive_sim_; // threshold is the bar to beat (inclusive)
|
||||
for (const auto& [tid, ts] : inactive_) {
|
||||
float sim = cosine_similarity(ts.last_emb, emb);
|
||||
if (sim >= best_sim) { best_sim = sim; best_tid = tid; }
|
||||
// subsequent ties keep the later id; harmless, all clear the threshold
|
||||
}
|
||||
boxes_.swap(kept);
|
||||
return best_tid;
|
||||
}
|
||||
|
||||
// IoU of two axis-aligned bounding boxes
|
||||
@@ -241,6 +220,17 @@ private:
|
||||
return inter / (a.width * a.height + b.width * b.height - inter);
|
||||
}
|
||||
|
||||
// Online directional mean: average then re-normalise to unit sphere
|
||||
static void update_mean(Embedding& mean, int n_prev, const Embedding& emb) {
|
||||
float norm_sq = 0.f;
|
||||
for (int k = 0; k < 512; ++k) {
|
||||
mean[k] = (mean[k] * n_prev + emb[k]) / (n_prev + 1);
|
||||
norm_sq += mean[k] * mean[k];
|
||||
}
|
||||
float inv = 1.f / std::sqrt(norm_sq);
|
||||
for (int k = 0; k < 512; ++k) mean[k] *= inv;
|
||||
}
|
||||
|
||||
// O(n³) potential-based Hungarian algorithm (Jonker-Volgenant / Kuhn-Munkres).
|
||||
// Returns assign[row] = col (0-indexed), or -1 when row is matched to a
|
||||
// padded virtual column (i.e., unmatched). Rectangular matrices are padded
|
||||
@@ -302,10 +292,13 @@ private:
|
||||
return ans;
|
||||
}
|
||||
|
||||
std::shared_ptr<TrackRegistry> registry_;
|
||||
Calibrate calibrate_;
|
||||
std::map<int, Spatial> boxes_; ///< track id → where it was, when
|
||||
float alpha_base_;
|
||||
std::map<int, TrackState> tracks_;
|
||||
std::map<int, TrackState> inactive_; // parked across a cut, keyed by track id
|
||||
int next_id_{0};
|
||||
float alpha_;
|
||||
float min_iou_;
|
||||
float min_assoc_prob_;
|
||||
float max_embed_dist_;
|
||||
int max_missing_;
|
||||
float revive_sim_;
|
||||
int inactive_max_;
|
||||
};
|
||||
|
||||
@@ -5,7 +5,6 @@
|
||||
#include "gallery/gallery_store.hpp"
|
||||
#include "gallery/gallery_calibration.hpp"
|
||||
#include "gallery/track_gallery.hpp"
|
||||
#include "track_registry.hpp"
|
||||
|
||||
#include <cstdint>
|
||||
#include <cstring>
|
||||
@@ -106,28 +105,8 @@ struct IdentityMatcherFunc {
|
||||
flat_emb_[i].data(), 512 * sizeof(float));
|
||||
|
||||
sim_engine_ = make_similarity_engine(host_gallery.data(), n_gallery_, kMaxFaces);
|
||||
|
||||
/// TRACES: AR-018, AR-024 | SR-005
|
||||
// The expansion store thresholds in the same probability space as
|
||||
// association and evidence weighting, so a "0.9" means one thing
|
||||
// pipeline-wide rather than three.
|
||||
track_gallery_.set_calibration(same_person_probability(cal_));
|
||||
}
|
||||
|
||||
/// TRACES: AR-023, AR-024 | SR-002
|
||||
/// The fitted sigmoid. Exposed because the matcher is where it gets fitted
|
||||
/// (and cached back to the gallery), but it is not the matcher's private
|
||||
/// property: track association and evidence weighting must threshold in the
|
||||
/// *same* probability space, or a "0.5" in one stage and a "0.5" in another
|
||||
/// mean different things. See `same_person_probability`.
|
||||
const GalleryCalibration& calibration() const { return cal_; }
|
||||
|
||||
/// TRACES: AR-012, AR-025 | SR-002
|
||||
/// Where per-frame identity evidence reaches the registry. Optional: with no
|
||||
/// registry attached the matcher behaves exactly as before, which keeps the
|
||||
/// replay harness and the unit tests working unchanged.
|
||||
void set_registry(std::shared_ptr<TrackRegistry> r) { registry_ = std::move(r); }
|
||||
|
||||
// Runtime setter — lets a persistent pipeline be reused across a threshold sweep
|
||||
// without rebuilding the (expensive, gallery-resident) matcher. The gallery,
|
||||
// calibration and GPU sim-engine stay put; only the accept threshold changes.
|
||||
@@ -144,51 +123,27 @@ struct IdentityMatcherFunc {
|
||||
// mix embeddings from two viewpoints under one buffer, so we still drop
|
||||
// every diversity buffer here — a revived track simply re-accumulates its
|
||||
// buffer from post-cut frames. Stale cross-cut embeddings are never promoted.
|
||||
/// TRACES: AR-019 | SR-005
|
||||
// Promotion may only borrow same-identity evidence from a span where
|
||||
// identity is certain, so ALL THREE discontinuity signals clear the
|
||||
// buffers, not just the histogram cut:
|
||||
// is_cut — camera-angle change
|
||||
// is_scene_boundary — different scene (AR-010; previously never set,
|
||||
// so this half of the gate was dead)
|
||||
// The third, an identity contradiction (AR-015), is enforced by the
|
||||
// registry: a track whose belief swapped is closed outright, so it can
|
||||
// no longer promote anything.
|
||||
if (tf.source.is_cut || tf.source.is_scene_boundary)
|
||||
track_gallery_.clear_tracks();
|
||||
if (tf.source.is_cut) track_gallery_.clear_tracks();
|
||||
|
||||
const int n_faces = static_cast<int>(tf.embeddings.size());
|
||||
std::vector<IdentifiedActor> actors;
|
||||
actors.reserve(n_faces);
|
||||
|
||||
if (n_faces == 0) return {std::move(tf.source), {}};
|
||||
if (n_faces > kMaxFaces)
|
||||
throw std::runtime_error("identity_matcher: n_faces exceeds kMaxFaces");
|
||||
|
||||
/// TRACES: AR-003, AR-004 | SR-002
|
||||
// kMaxFaces sizes the similarity engine's preallocated buffer, so it
|
||||
// bounds MEMORY, not how many faces a frame may contain. It used to
|
||||
// throw above the bound, which made it a hard cap on crowd scenes by
|
||||
// accident; now the frame is scored in batches of that size.
|
||||
//
|
||||
// Faces per frame are unbounded (AR-003) because X-Ray credits scene
|
||||
// membership to background cast too, and a fixed cap discards exactly
|
||||
// those — the smallest faces are dropped first. Cost is contained by
|
||||
// backpressure (AR-004), which slows the producer, rather than by
|
||||
// silently throwing work away.
|
||||
std::vector<float> host_query(static_cast<size_t>(kMaxFaces) * 512);
|
||||
|
||||
for (int base = 0; base < n_faces; base += kMaxFaces) {
|
||||
const int chunk = std::min(kMaxFaces, n_faces - base);
|
||||
for (int k = 0; k < chunk; ++k) {
|
||||
std::memcpy(host_query.data() + static_cast<size_t>(k) * 512,
|
||||
tf.embeddings[base + k].data(), 512 * sizeof(float));
|
||||
std::vector<float> host_query(static_cast<size_t>(n_faces) * 512);
|
||||
for (int fi = 0; fi < n_faces; ++fi) {
|
||||
std::memcpy(host_query.data() + static_cast<size_t>(fi) * 512,
|
||||
tf.embeddings[fi].data(), 512 * sizeof(float));
|
||||
}
|
||||
|
||||
// S (N_gallery × chunk) col-major: face k's gallery sims at sims + k*n_gallery.
|
||||
const float* host_sims = sim_engine_->compute(host_query.data(), chunk);
|
||||
// S (N_gallery × n_faces) col-major: face fi's gallery sims at sims + fi*n_gallery.
|
||||
const float* host_sims = sim_engine_->compute(host_query.data(), n_faces);
|
||||
|
||||
for (int ci = 0; ci < chunk; ++ci) {
|
||||
const int fi = base + ci;
|
||||
const float* sims = host_sims + static_cast<size_t>(ci) * n_gallery_;
|
||||
for (int fi = 0; fi < n_faces; ++fi) {
|
||||
const float* sims = host_sims + static_cast<size_t>(fi) * n_gallery_;
|
||||
|
||||
std::vector<float> best_sim(gallery_.actors.size(),
|
||||
-std::numeric_limits<float>::max());
|
||||
@@ -269,35 +224,11 @@ struct IdentityMatcherFunc {
|
||||
// face (annex already folded in above); the track's diversity buffer
|
||||
// keeps the gallery-far views and promotes them once the track is
|
||||
// confirmed. No-op unless --expand-gallery is set.
|
||||
// TRACES: AR-012, AR-025 | SR-002
|
||||
// Every scored face is evidence, not only the accepted ones: a run of
|
||||
// near-misses for one actor is itself informative, and discarding it
|
||||
// would make ownership depend on a per-frame threshold the redesign
|
||||
// exists to stop relying on. The registry discounts for correlation
|
||||
// and decides ownership from the accumulated posterior (AR-025).
|
||||
if (registry_ && best_actor >= 0 && tf.track_ids[fi] >= 0) {
|
||||
const float p = cal_.valid
|
||||
? cal_.probability(best_s, log_prior_odds_)
|
||||
: std::max(0.f, best_s);
|
||||
registry_->observe(tf.track_ids[fi], best_actor, p, tf.embeddings[fi]);
|
||||
}
|
||||
|
||||
// TRACES: AR-019 | SR-005
|
||||
// Ownership is the registry's, computed once. TrackGallery used to
|
||||
// tally its own plurality vote over accepted frames, which meant two
|
||||
// different answers to "who is this track" could coexist — and the
|
||||
// expansion one ignored the Bayesian accumulation entirely.
|
||||
if (registry_ && tf.track_ids[fi] >= 0) {
|
||||
if (auto owner = registry_->owner(tf.track_ids[fi]))
|
||||
track_gallery_.set_owner(tf.track_ids[fi], *owner);
|
||||
}
|
||||
|
||||
track_gallery_.observe(tf.track_ids[fi], tf.embeddings[fi],
|
||||
best_actor, best_s, accept, tf.crops[fi]);
|
||||
|
||||
actors.push_back(std::move(ia));
|
||||
}
|
||||
} // chunk loop
|
||||
|
||||
return {std::move(tf.source), std::move(actors)};
|
||||
}
|
||||
@@ -316,5 +247,4 @@ private:
|
||||
|
||||
std::unique_ptr<ISimilarityEngine> sim_engine_;
|
||||
TrackGallery track_gallery_;
|
||||
std::shared_ptr<TrackRegistry> registry_;
|
||||
};
|
||||
|
||||
@@ -1,8 +1,6 @@
|
||||
#pragma once
|
||||
/// TRACES: IR-001 | SR-003
|
||||
#include "types.hpp"
|
||||
#include "config.hpp"
|
||||
#include "track_registry.hpp"
|
||||
|
||||
#include <nlohmann/json.hpp>
|
||||
#include <algorithm>
|
||||
@@ -11,8 +9,6 @@
|
||||
#include <fstream>
|
||||
#include <iostream>
|
||||
#include <map>
|
||||
#include <mutex>
|
||||
#include <functional>
|
||||
#include <string>
|
||||
#include <vector>
|
||||
|
||||
@@ -46,26 +42,10 @@ using json = nlohmann::json;
|
||||
struct ResultSinkFunc {
|
||||
static constexpr std::string_view label() { return "result_sink"; }
|
||||
|
||||
/// TRACES: AR-012, AR-017 | IR-002 | SR-002, SR-003
|
||||
/// A finished presence claim from the registry. Called from inside the
|
||||
/// registry's reap while it holds its own lock, so this must stay a cheap
|
||||
/// push and must never re-enter the registry.
|
||||
void add_claim(const DeadTrack& d) {
|
||||
if (d.actor_idx < 0) return; // never owned: nothing to claim
|
||||
std::lock_guard<std::mutex> g(claims_mu_);
|
||||
claims_.push_back(d);
|
||||
}
|
||||
|
||||
ResultSinkFunc(const Config& cfg, std::atomic<bool>& done)
|
||||
: cfg_(cfg), done_(done)
|
||||
{}
|
||||
|
||||
/// TRACES: AR-016 | SR-002
|
||||
/// Runs immediately before the output is written, with the last timestamp
|
||||
/// seen. Used to flush tracks still live at EOF, which have not timed out
|
||||
/// and would otherwise never be emitted.
|
||||
void set_pre_write_hook(std::function<void(double)> fn) { pre_write_ = std::move(fn); }
|
||||
|
||||
void operator()(SceneAnnotation sa) {
|
||||
if (sa.eof) {
|
||||
flush();
|
||||
@@ -78,20 +58,12 @@ struct ResultSinkFunc {
|
||||
<< " unknowns=" << count_unknown(sa.visible_actors)
|
||||
<< std::flush;
|
||||
|
||||
for (const auto& ia : sa.visible_actors) {
|
||||
if (ia.actor_idx < 0) continue;
|
||||
auto& m = actor_meta_[ia.actor_idx];
|
||||
if (m.name.empty())
|
||||
m = {ia.name, ia.imdb_id, ia.tmdb_id, ia.jellyfin_id};
|
||||
}
|
||||
last_ts_ = sa.timestamp_sec;
|
||||
frames_.push_back(std::move(sa));
|
||||
}
|
||||
|
||||
// Write accumulated results and signal done. Safe to call more than once.
|
||||
void flush() {
|
||||
if (written_.exchange(true)) return;
|
||||
if (pre_write_) pre_write_(last_ts_);
|
||||
write_output();
|
||||
done_.store(true, std::memory_order_release);
|
||||
}
|
||||
@@ -99,7 +71,7 @@ struct ResultSinkFunc {
|
||||
private:
|
||||
// Bump when the minimal/standard output JSON structure changes in a way
|
||||
// the Jellyfin plugin needs to detect.
|
||||
static constexpr int kSchemaVersion = 2; // SR-003 coordinated bump
|
||||
static constexpr int kSchemaVersion = 1;
|
||||
|
||||
static int count_known(const std::vector<IdentifiedActor>& v) {
|
||||
int n = 0;
|
||||
@@ -119,19 +91,10 @@ private:
|
||||
if (cfg_.verbosity == Verbosity::xray) {
|
||||
root = build_xray();
|
||||
} else {
|
||||
/// TRACES: IR-002 | SR-003
|
||||
/// schema_version 2, per jRay/SPEC.md JR-002. anneal_sec is REMOVED
|
||||
/// rather than zeroed: a field naming a mechanism the pipeline no
|
||||
/// longer has is actively misleading, and would outlive everyone who
|
||||
/// remembers why it reads 0. extinction_sec succeeds it as the
|
||||
/// parameter that actually shapes window extent.
|
||||
root["schema_version"] = kSchemaVersion;
|
||||
root["movie"] = cfg_.movie_path;
|
||||
root["extraction"] = {
|
||||
{"sample_fps", cfg_.sample_fps},
|
||||
{"extinction_sec", cfg_.track_extinction_sec},
|
||||
{"gallery_scope", cfg_.gallery_scope},
|
||||
};
|
||||
root["movie"] = cfg_.movie_path;
|
||||
root["sample_fps"] = cfg_.sample_fps;
|
||||
root["anneal_sec"] = cfg_.anneal_sec;
|
||||
root["actors"] = build_epochs();
|
||||
if (cfg_.verbosity == Verbosity::standard)
|
||||
root["frames"] = build_standard();
|
||||
@@ -146,45 +109,42 @@ private:
|
||||
std::cerr << "[result_sink] done.\n";
|
||||
}
|
||||
|
||||
struct Window {
|
||||
double start{0.0};
|
||||
double end{0.0};
|
||||
float belief{0.f}; ///< the posterior that justified the claim (AR-017)
|
||||
};
|
||||
struct ActorWindow {
|
||||
std::string name, imdb_id, tmdb_id, jellyfin_id;
|
||||
std::vector<Window> scenes;
|
||||
std::vector<std::pair<double, double>> scenes; // [start_sec, end_sec]
|
||||
};
|
||||
struct ActorMeta { std::string name, imdb_id, tmdb_id, jellyfin_id; };
|
||||
|
||||
// Core logic: merge per-frame detections into annealed [start, end] windows.
|
||||
/// TRACES: AR-012 | IR-002 | SR-002
|
||||
/// A claim already IS a window — `[first_seen, last_seen]` of a track the
|
||||
/// actor owned. There is no annealing pass: `anneal_sec` existed to bridge
|
||||
/// gaps between isolated accepted frames, and a track that survives its own
|
||||
/// gaps leaves it nothing to do (see the AR-012 withdrawal note).
|
||||
std::vector<ActorWindow> build_actor_windows() {
|
||||
std::lock_guard<std::mutex> g(claims_mu_);
|
||||
struct Info { std::string name, imdb_id, tmdb_id, jellyfin_id; };
|
||||
std::map<int, Info> actor_info;
|
||||
std::map<int, std::vector<double>> timestamps;
|
||||
|
||||
std::map<int, ActorWindow> by_actor;
|
||||
for (const auto& c : claims_) {
|
||||
auto& aw = by_actor[c.actor_idx];
|
||||
if (aw.name.empty()) {
|
||||
auto it = actor_meta_.find(c.actor_idx);
|
||||
if (it != actor_meta_.end()) {
|
||||
aw.name = it->second.name;
|
||||
aw.imdb_id = it->second.imdb_id;
|
||||
aw.tmdb_id = it->second.tmdb_id;
|
||||
aw.jellyfin_id = it->second.jellyfin_id;
|
||||
}
|
||||
for (const auto& frame : frames_) {
|
||||
for (const auto& ia : frame.visible_actors) {
|
||||
if (ia.actor_idx < 0) continue;
|
||||
actor_info[ia.actor_idx] = {ia.name, ia.imdb_id, ia.tmdb_id, ia.jellyfin_id};
|
||||
timestamps[ia.actor_idx].push_back(frame.timestamp_sec);
|
||||
}
|
||||
aw.scenes.push_back({c.first_seen, c.last_seen, c.belief});
|
||||
}
|
||||
|
||||
std::vector<ActorWindow> result;
|
||||
for (auto& [idx, aw] : by_actor) {
|
||||
std::sort(aw.scenes.begin(), aw.scenes.end(),
|
||||
[](const Window& a, const Window& b) { return a.start < b.start; });
|
||||
for (auto& [idx, ts_vec] : timestamps) {
|
||||
ActorWindow aw;
|
||||
aw.name = actor_info[idx].name;
|
||||
aw.imdb_id = actor_info[idx].imdb_id;
|
||||
aw.tmdb_id = actor_info[idx].tmdb_id;
|
||||
aw.jellyfin_id = actor_info[idx].jellyfin_id;
|
||||
|
||||
double win_start = ts_vec[0], win_end = ts_vec[0];
|
||||
for (size_t i = 1; i < ts_vec.size(); ++i) {
|
||||
if (ts_vec[i] - win_end > cfg_.anneal_sec) {
|
||||
aw.scenes.push_back({win_start, win_end});
|
||||
win_start = ts_vec[i];
|
||||
}
|
||||
win_end = ts_vec[i];
|
||||
}
|
||||
aw.scenes.push_back({win_start, win_end});
|
||||
result.push_back(std::move(aw));
|
||||
}
|
||||
return result;
|
||||
@@ -193,16 +153,9 @@ private:
|
||||
json build_epochs() {
|
||||
json actors = json::array();
|
||||
for (const auto& aw : build_actor_windows()) {
|
||||
// Objects, not float pairs: a window carries the belief that
|
||||
// justified it and the route by which it was identified (AR-017),
|
||||
// so a consumer can caveat or filter rather than treating every
|
||||
// window as equally certain.
|
||||
json windows = json::array();
|
||||
for (const auto& w : aw.scenes)
|
||||
windows.push_back({{"start", w.start},
|
||||
{"end", w.end},
|
||||
{"belief", w.belief},
|
||||
{"route", "live"}});
|
||||
for (const auto& [s, e] : aw.scenes)
|
||||
windows.push_back({s, e});
|
||||
json ja;
|
||||
ja["name"] = aw.name;
|
||||
ja["imdb_id"] = aw.imdb_id;
|
||||
@@ -220,9 +173,9 @@ private:
|
||||
json build_xray() {
|
||||
std::map<int, std::vector<std::string>> xray;
|
||||
for (const auto& aw : build_actor_windows()) {
|
||||
for (const auto& w : aw.scenes) {
|
||||
int t0 = static_cast<int>(std::floor(w.start));
|
||||
int t1 = static_cast<int>(std::ceil(w.end));
|
||||
for (const auto& [start, end] : aw.scenes) {
|
||||
int t0 = static_cast<int>(std::floor(start));
|
||||
int t1 = static_cast<int>(std::ceil(end));
|
||||
for (int t = t0; t <= t1; ++t)
|
||||
xray[t].push_back(aw.name);
|
||||
}
|
||||
@@ -273,9 +226,4 @@ private:
|
||||
std::atomic<bool>& done_;
|
||||
std::atomic<bool> written_{false};
|
||||
std::vector<SceneAnnotation> frames_;
|
||||
std::function<void(double)> pre_write_;
|
||||
double last_ts_{0.0};
|
||||
std::mutex claims_mu_;
|
||||
std::vector<DeadTrack> claims_;
|
||||
std::map<int, ActorMeta> actor_meta_; ///< actor_idx → identity keys
|
||||
};
|
||||
|
||||
@@ -1,67 +0,0 @@
|
||||
#pragma once
|
||||
/// TRACES: AR-010 | SR-002
|
||||
///
|
||||
/// SceneBoundaryAnnotatorFunc — the join of the decode butterfly.
|
||||
///
|
||||
/// `source` fans out to two branches: dense frames to TransNetV2, sampled frames
|
||||
/// to face detection. A boundary found on the first has to reach the second, and
|
||||
/// cannot ride along in the frame because the branches run in parallel.
|
||||
///
|
||||
/// This node sits on the sampled branch and stamps `Frame::is_scene_boundary`
|
||||
/// from the detector's published verdict.
|
||||
///
|
||||
/// **It only works because the sampled branch lags.** TransNetV2 buffers
|
||||
/// `kWindow` frames before it can score any of them, so this node must not reach
|
||||
/// a frame before the detector has an opinion about it. Channel depth creates
|
||||
/// that lag: with backpressure (AR-004) the fanout blocks on the slower branch,
|
||||
/// so a deep channel here lets the detector run ahead by its window instead of
|
||||
/// anything being dropped.
|
||||
///
|
||||
/// When the lag is insufficient the node **counts it** rather than guessing.
|
||||
/// Annotating an unscored frame as boundary-free is indistinguishable from a
|
||||
/// genuine "no boundary here", and that is the failure that makes a downstream
|
||||
/// test pass while verifying nothing.
|
||||
|
||||
#include "scene_boundaries.hpp"
|
||||
#include "types.hpp"
|
||||
|
||||
#include <memory>
|
||||
#include <string_view>
|
||||
#include <utility>
|
||||
|
||||
struct SceneBoundaryAnnotatorFunc {
|
||||
static constexpr std::string_view label() { return "scene_annotate"; }
|
||||
|
||||
/// `tol` is half a sample interval. The branches sample at different rates,
|
||||
/// so a boundary found on a dense frame rarely lands exactly on a sampled
|
||||
/// one; half an interval attributes it to the nearest sampled frame and no
|
||||
/// further.
|
||||
SceneBoundaryAnnotatorFunc(std::shared_ptr<SceneBoundaries> b, double tol)
|
||||
: bounds_(std::move(b)), tol_(tol) {}
|
||||
|
||||
Frame operator()(Frame f) {
|
||||
if (f.eof || !bounds_) return f;
|
||||
|
||||
// Wait for the detector's verdict to cover this frame. Channel depth
|
||||
// alone cannot provide the lag: it holds frames back only when the
|
||||
// consumer is slower, and this branch is orders of magnitude faster per
|
||||
// frame than TransNetV2. Blocking here is what makes the join real.
|
||||
//
|
||||
// Safe under backpressure because the branches are independent: this
|
||||
// node stalling does not stop the detector consuming dense frames, and
|
||||
// the fanout keeps feeding it.
|
||||
if (!bounds_->wait_until_scored(f.timestamp_sec)) {
|
||||
// The detector finished without covering this frame — the tail after
|
||||
// its last full window. Unknown, not negative; counted so it cannot
|
||||
// pass for "no boundary here".
|
||||
bounds_->note_outran();
|
||||
return f;
|
||||
}
|
||||
f.is_scene_boundary = bounds_->is_boundary(f.timestamp_sec, tol_);
|
||||
return f;
|
||||
}
|
||||
|
||||
private:
|
||||
std::shared_ptr<SceneBoundaries> bounds_;
|
||||
double tol_{0.0};
|
||||
};
|
||||
@@ -1,9 +1,6 @@
|
||||
#pragma once
|
||||
#include "types.hpp"
|
||||
#include "config.hpp"
|
||||
#include "scene_boundaries.hpp"
|
||||
|
||||
#include <memory>
|
||||
#include "inference/scene_detector.hpp"
|
||||
|
||||
#include <nlohmann/json.hpp>
|
||||
@@ -33,11 +30,6 @@
|
||||
struct SceneDetectorFunc {
|
||||
static constexpr std::string_view label() { return "scene_detector"; }
|
||||
|
||||
/// TRACES: AR-010 | SR-002
|
||||
/// Publish each window's verdict as it is scored, so the face branch — held
|
||||
/// back by channel depth — can consult it for frames it has not reached yet.
|
||||
void set_boundaries(std::shared_ptr<SceneBoundaries> b) { shared_ = std::move(b); }
|
||||
|
||||
SceneDetectorFunc(const Config& cfg, std::atomic<bool>& done)
|
||||
: detector_(make_scene_detector(cfg))
|
||||
, threshold_(cfg.scene_threshold)
|
||||
@@ -59,9 +51,6 @@ struct SceneDetectorFunc {
|
||||
void operator()(Frame f) {
|
||||
if (f.eof) {
|
||||
flush_remaining();
|
||||
// Release anyone waiting on the join: the tail frames after the last
|
||||
// full window will never be covered, so waiting for them would hang.
|
||||
if (shared_) shared_->finish();
|
||||
write_output();
|
||||
done_.store(true, std::memory_order_release);
|
||||
return;
|
||||
@@ -93,7 +82,6 @@ private:
|
||||
// otherwise skip the leading guard already covered by the previous window.
|
||||
const int lo = (window_base_ == 0) ? 0 : guard_;
|
||||
const int hi = ISceneDetector::kWindow - guard_;
|
||||
std::vector<double> fresh;
|
||||
for (int i = lo; i < hi; ++i) {
|
||||
if (probs[i] <= threshold_) continue;
|
||||
// Local maximum → the boundary frame (avoid a run of high scores
|
||||
@@ -101,19 +89,9 @@ private:
|
||||
const bool peak =
|
||||
(i == 0 || probs[i] >= probs[i-1]) &&
|
||||
(i == kLast_() || probs[i] >= probs[i+1]);
|
||||
if (peak) {
|
||||
if (peak)
|
||||
boundaries_.push_back({times_[i], probs[i]});
|
||||
fresh.push_back(times_[i]);
|
||||
}
|
||||
}
|
||||
|
||||
/// TRACES: AR-010 | SR-002
|
||||
// Publish with a watermark: everything up to times_[hi-1] now has a
|
||||
// final verdict. The face branch consults this for frames it has not
|
||||
// reached yet, and the watermark is what lets it tell "no boundary
|
||||
// here" from "not scored yet".
|
||||
if (shared_ && hi > lo)
|
||||
shared_->publish(fresh, times_[hi - 1]);
|
||||
}
|
||||
|
||||
// At EOF the tail (< kWindow frames) never formed a full window. Pad it out
|
||||
@@ -129,24 +107,14 @@ private:
|
||||
|
||||
std::vector<float> probs = detector_->detect_window(win);
|
||||
const int lo = (window_base_ == 0) ? 0 : guard_;
|
||||
std::vector<double> fresh;
|
||||
for (int i = lo; i < n; ++i) { // only real (non-padded) frames
|
||||
if (probs[i] <= threshold_) continue;
|
||||
const bool peak =
|
||||
(i == 0 || probs[i] >= probs[i-1]) &&
|
||||
(i == n - 1 || probs[i] >= probs[i+1]);
|
||||
if (peak) {
|
||||
if (peak)
|
||||
boundaries_.push_back({times_[i], probs[i]});
|
||||
fresh.push_back(times_[i]);
|
||||
}
|
||||
}
|
||||
|
||||
/// TRACES: AR-010 | SR-002
|
||||
// Publish the tail too. Without this the final frames — everything after
|
||||
// the last full window — reach the join with no verdict and are treated
|
||||
// as boundary-free without evidence, which is precisely the ambiguity
|
||||
// the watermark exists to prevent.
|
||||
if (shared_ && n > 0) shared_->publish(fresh, times_[n - 1]);
|
||||
}
|
||||
|
||||
void write_output() {
|
||||
@@ -208,5 +176,4 @@ private:
|
||||
int64_t window_base_{0}; // frame index of images_.front()
|
||||
std::vector<Boundary> boundaries_;
|
||||
bool written_{false};
|
||||
std::shared_ptr<SceneBoundaries> shared_; ///< AR-010 join point
|
||||
};
|
||||
|
||||
+1
-205
@@ -3,77 +3,17 @@
|
||||
// Loads both ONNX sessions once per FaceEmbedder instance, then embeds many
|
||||
// images via repeated embed() calls — avoiding the per-process model-load
|
||||
// cost of the embed_faces CLI when embedding a large gallery.
|
||||
//
|
||||
// Beyond whole-image embed(), the individual pipeline stages are exposed —
|
||||
// detect(), align_face(), embed_crop() — plus the gallery calibration. A study
|
||||
// that needs to step between stages (a different landmark source, a degraded
|
||||
// crop) drives the shipped C++ from Python rather than re-implementing
|
||||
// detection, alignment, the ArcFace warp or the Platt fit in numpy. Those
|
||||
// re-implementations drift from what ships, and the calibration is the one
|
||||
// that must not: AR-024 requires every similarity to pass through
|
||||
// GalleryCalibration::probability, never a bare cosine.
|
||||
|
||||
#include "face_embedder_engine.hpp"
|
||||
#include "gallery/gallery_calibration.hpp"
|
||||
#include "gallery/gallery_store.hpp"
|
||||
|
||||
#include <nanobind/nanobind.h>
|
||||
#include <nanobind/ndarray.h>
|
||||
#include <nanobind/stl/optional.h>
|
||||
#include <nanobind/stl/string.h>
|
||||
#include <nanobind/stl/vector.h>
|
||||
|
||||
#include <array>
|
||||
#include <cstring>
|
||||
#include <stdexcept>
|
||||
|
||||
namespace nb = nanobind;
|
||||
using namespace nb::literals;
|
||||
|
||||
namespace {
|
||||
|
||||
using ImageArray = nb::ndarray<const uint8_t, nb::ndim<3>, nb::c_contig, nb::device::cpu>;
|
||||
|
||||
// numpy HxWx3 uint8 (BGR, as cv::imread yields) → cv::Mat sharing that buffer.
|
||||
// The Mat is a view: it must not outlive the caller's array, so every use here
|
||||
// copies or consumes it before returning.
|
||||
cv::Mat as_mat(const ImageArray& a) {
|
||||
if (a.shape(2) != 3)
|
||||
throw std::invalid_argument("expected an HxWx3 uint8 BGR image");
|
||||
return cv::Mat(static_cast<int>(a.shape(0)), static_cast<int>(a.shape(1)),
|
||||
CV_8UC3, const_cast<uint8_t*>(a.data()));
|
||||
}
|
||||
|
||||
// cv::Mat → freshly-allocated numpy array (owns its buffer).
|
||||
nb::ndarray<nb::numpy, uint8_t> mat_to_numpy(const cv::Mat& m) {
|
||||
cv::Mat c = m.isContinuous() ? m : m.clone();
|
||||
auto* buf = new uint8_t[c.total() * c.elemSize()];
|
||||
std::memcpy(buf, c.data, c.total() * c.elemSize());
|
||||
nb::capsule owner(buf, [](void* p) noexcept { delete[] static_cast<uint8_t*>(p); });
|
||||
size_t shape[3] = {static_cast<size_t>(c.rows), static_cast<size_t>(c.cols),
|
||||
static_cast<size_t>(c.channels())};
|
||||
return nb::ndarray<nb::numpy, uint8_t>(buf, 3, shape, owner);
|
||||
}
|
||||
|
||||
nb::ndarray<nb::numpy, float> vec_to_numpy(std::vector<float>&& v) {
|
||||
auto* buf = new float[v.size()];
|
||||
std::memcpy(buf, v.data(), v.size() * sizeof(float));
|
||||
nb::capsule owner(buf, [](void* p) noexcept { delete[] static_cast<float*>(p); });
|
||||
size_t shape[1] = {v.size()};
|
||||
return nb::ndarray<nb::numpy, float>(buf, 1, shape, owner);
|
||||
}
|
||||
|
||||
// numpy (5,2) float32 → the landmark array align_face expects. Order is
|
||||
// types.hpp:60 — [0] right-eye [1] left-eye [2] nose [3] right-mouth [4] left-mouth.
|
||||
std::array<cv::Point2f, 5> as_landmarks(
|
||||
const nb::ndarray<const float, nb::shape<5, 2>, nb::c_contig, nb::device::cpu>& a) {
|
||||
std::array<cv::Point2f, 5> lm;
|
||||
for (int i = 0; i < 5; ++i) lm[i] = {a(i, 0), a(i, 1)};
|
||||
return lm;
|
||||
}
|
||||
|
||||
} // namespace
|
||||
|
||||
NB_MODULE(sae_embed, m) {
|
||||
m.doc() = "SCRFD + ArcFace face embedding, models loaded once per FaceEmbedder";
|
||||
|
||||
@@ -87,23 +27,6 @@ NB_MODULE(sae_embed, m) {
|
||||
})
|
||||
.def_prop_ro("bbox", [](const FaceEmbedResult& r) {
|
||||
return std::vector<float>{r.bbox[0], r.bbox[1], r.bbox[2], r.bbox[3]};
|
||||
})
|
||||
.def_prop_ro("landmarks", [](const FaceEmbedResult& r) {
|
||||
std::vector<float> v;
|
||||
for (const auto& p : r.landmarks) { v.push_back(p.x); v.push_back(p.y); }
|
||||
return v;
|
||||
});
|
||||
|
||||
nb::class_<DetectedFace>(m, "Detection")
|
||||
.def_ro("confidence", &DetectedFace::confidence)
|
||||
.def_prop_ro("bbox", [](const DetectedFace& d) {
|
||||
return std::vector<float>{d.bbox.x, d.bbox.y, d.bbox.width, d.bbox.height};
|
||||
})
|
||||
.def_prop_ro("landmarks", [](const DetectedFace& d) {
|
||||
// (5,2): [0] right-eye [1] left-eye [2] nose [3] right-mouth [4] left-mouth
|
||||
std::vector<float> v;
|
||||
for (const auto& p : d.landmarks) { v.push_back(p.x); v.push_back(p.y); }
|
||||
return v;
|
||||
});
|
||||
|
||||
nb::class_<FaceEmbedderEngine>(m, "FaceEmbedder")
|
||||
@@ -115,132 +38,5 @@ NB_MODULE(sae_embed, m) {
|
||||
.def("embed", &FaceEmbedderEngine::embed_path, "path"_a,
|
||||
nb::call_guard<nb::gil_scoped_release>(),
|
||||
"Detect the highest-confidence face in the image, align it, and "
|
||||
"return a FaceResult with its 512-d ArcFace embedding.")
|
||||
.def("embed_mat", [](FaceEmbedderEngine& e, ImageArray img) {
|
||||
return e.embed_mat(as_mat(img).clone());
|
||||
}, "image"_a,
|
||||
"As embed(), on an in-memory HxWx3 uint8 BGR array.")
|
||||
.def("detect", [](FaceEmbedderEngine& e, ImageArray img) {
|
||||
return e.detect(as_mat(img));
|
||||
}, "image"_a,
|
||||
"Run the configured detector. Returns every Detection, unfiltered — "
|
||||
"min_face_px is applied downstream in face_detector_node.")
|
||||
.def("embed_crop", [](FaceEmbedderEngine& e, ImageArray crop) {
|
||||
cv::Mat c = as_mat(crop);
|
||||
if (c.rows != 112 || c.cols != 112)
|
||||
throw std::invalid_argument("embed_crop expects a 112x112 aligned crop");
|
||||
Embedding emb = e.embed_crop(c);
|
||||
return vec_to_numpy(std::vector<float>(emb.begin(), emb.end()));
|
||||
}, "crop"_a,
|
||||
"Embed a caller-supplied 112x112 aligned BGR crop. The stage-level "
|
||||
"entry point for studies that degrade or re-align a crop themselves.")
|
||||
.def("embed_crops", [](FaceEmbedderEngine& e,
|
||||
nb::ndarray<const uint8_t, nb::ndim<4>, nb::c_contig,
|
||||
nb::device::cpu> crops) {
|
||||
if (crops.shape(1) != 112 || crops.shape(2) != 112 || crops.shape(3) != 3)
|
||||
throw std::invalid_argument("embed_crops expects (N,112,112,3) uint8 BGR");
|
||||
const size_t n = crops.shape(0);
|
||||
std::vector<cv::Mat> mats;
|
||||
mats.reserve(n);
|
||||
for (size_t i = 0; i < n; ++i)
|
||||
mats.emplace_back(112, 112, CV_8UC3,
|
||||
const_cast<uint8_t*>(crops.data()) + i * 112 * 112 * 3);
|
||||
std::vector<Embedding> out = e.embed_crops(mats);
|
||||
auto* buf = new float[n * 512];
|
||||
for (size_t i = 0; i < n; ++i)
|
||||
std::memcpy(buf + i * 512, out[i].data(), 512 * sizeof(float));
|
||||
nb::capsule owner(buf, [](void* p) noexcept { delete[] static_cast<float*>(p); });
|
||||
size_t shape[2] = {n, 512};
|
||||
return nb::ndarray<nb::numpy, float>(buf, 2, shape, owner);
|
||||
}, "crops"_a,
|
||||
"Batched embed_crop: (N,112,112,3) uint8 BGR in, (N,512) float32 out. "
|
||||
"The backend batches internally, so this avoids paying per-call "
|
||||
"overhead once per crop across a large study.")
|
||||
.def_prop_ro("max_batch", [](FaceEmbedderEngine& e) { return e.max_batch(); });
|
||||
|
||||
m.def("align_face", [](ImageArray img,
|
||||
nb::ndarray<const float, nb::shape<5, 2>, nb::c_contig,
|
||||
nb::device::cpu> landmarks)
|
||||
-> std::optional<nb::ndarray<nb::numpy, uint8_t>> {
|
||||
cv::Mat crop = ::align_face(as_mat(img), as_landmarks(landmarks));
|
||||
if (crop.empty()) return std::nullopt; // degenerate fit
|
||||
return mat_to_numpy(crop);
|
||||
}, "image"_a, "landmarks"_a,
|
||||
"The ArcFace 5-point similarity transform (face_utils.hpp, AR-005). "
|
||||
"Returns a 112x112 BGR crop, or None if the affine fit is degenerate. "
|
||||
"Landmark order is types.hpp:60 — right-eye, left-eye, nose, "
|
||||
"right-mouth, left-mouth.");
|
||||
|
||||
m.def("enhance_for_retry", [](ImageArray img) {
|
||||
return mat_to_numpy(::enhance_for_retry(as_mat(img)));
|
||||
}, "image"_a,
|
||||
"Border-replicate pad by 50% and CLAHE, for a detector second try.");
|
||||
|
||||
// ── Calibration ──────────────────────────────────────────────────────────
|
||||
// AR-024: the pipeline reasons in one probability space. Exposed so Python
|
||||
// scores through the same sigmoid the C++ matcher uses, rather than a numpy
|
||||
// copy of it that can silently disagree.
|
||||
nb::class_<GalleryCalibration>(m, "GalleryCalibration")
|
||||
.def_ro("a", &GalleryCalibration::a)
|
||||
.def_ro("b", &GalleryCalibration::b)
|
||||
.def_ro("valid", &GalleryCalibration::valid)
|
||||
.def("probability", &GalleryCalibration::probability,
|
||||
"similarity"_a, "log_prior_odds"_a = 0.f,
|
||||
"P(match | sim) = sigma(a*sim + b + log_prior_odds). Pass "
|
||||
"log_prior_odds = log(p0/(1-p0)) for a base-rate prior p0; leave it "
|
||||
"at 0 for association (is this one person), which is what the "
|
||||
"balanced fit answers — see gallery_calibration.hpp:63.")
|
||||
.def("boundary_at", &GalleryCalibration::boundary_at,
|
||||
"p"_a = 0.5f, "log_prior_odds"_a = 0.f,
|
||||
"The similarity at which P(match) == p. Diagnostic only — decisions "
|
||||
"threshold the probability, not this.")
|
||||
.def("__repr__", [](const GalleryCalibration& c) {
|
||||
return "<GalleryCalibration a=" + std::to_string(c.a) +
|
||||
" b=" + std::to_string(c.b) +
|
||||
(c.valid ? " valid>" : " INVALID>");
|
||||
});
|
||||
|
||||
m.def("gallery_calibration", [](const std::string& gallery_path) {
|
||||
ActorGallery g = load_gallery(gallery_path);
|
||||
if (g.calib_valid) {
|
||||
std::cerr << "[calibration] " << gallery_path << ": cached fit"
|
||||
<< " over " << g.actors.size() << " actors\n";
|
||||
return GalleryCalibration{g.calib_a, g.calib_b, true};
|
||||
}
|
||||
// Legacy JSON galleries carry no stored fit; compute it over the
|
||||
// whole gallery, which is the point — the calibration must come
|
||||
// from the production actor population, not a handful of people.
|
||||
std::cerr << "[calibration] " << gallery_path
|
||||
<< ": no cached fit, computing over " << g.actors.size()
|
||||
<< " actors\n";
|
||||
std::vector<Embedding> flat;
|
||||
std::vector<int> actor;
|
||||
for (size_t a = 0; a < g.actors.size(); ++a)
|
||||
for (const auto& e : g.actors[a].embeddings) {
|
||||
flat.push_back(e);
|
||||
actor.push_back(static_cast<int>(a));
|
||||
}
|
||||
return ::calibrate_gallery(flat, actor);
|
||||
}, "gallery_path"_a,
|
||||
"The production gallery's calibration — the global fit over every "
|
||||
"actor in it. Use this to score, not a fit over a handful of people: "
|
||||
"a sigmoid fitted on a few identities saturates, so its probabilities "
|
||||
"mean nothing. Reads the cached fit stored in an HDF5 gallery, or "
|
||||
"computes it over the whole gallery for a legacy JSON one.");
|
||||
|
||||
m.def("calibrate_gallery", [](nb::ndarray<const float, nb::shape<-1, 512>, nb::c_contig,
|
||||
nb::device::cpu> emb,
|
||||
std::vector<int> actor) {
|
||||
const size_t n = emb.shape(0);
|
||||
if (actor.size() != n)
|
||||
throw std::invalid_argument("embeddings and actor ids differ in length");
|
||||
std::vector<Embedding> flat(n);
|
||||
for (size_t i = 0; i < n; ++i)
|
||||
std::memcpy(flat[i].data(), &emb(i, 0), 512 * sizeof(float));
|
||||
return ::calibrate_gallery(flat, actor);
|
||||
}, "embeddings"_a, "actor_ids"_a,
|
||||
"Fit the Platt sigmoid from intra/inter-class pairs — the same fit the "
|
||||
"gallery build performs (gallery_calibration.hpp:85). embeddings is "
|
||||
"(N,512) L2-normalised float32; actor_ids is a length-N list of "
|
||||
"0-based actor indices.");
|
||||
"return a FaceResult with its 512-d ArcFace embedding.");
|
||||
}
|
||||
|
||||
@@ -1,126 +0,0 @@
|
||||
#pragma once
|
||||
/// TRACES: AR-010 | SR-002
|
||||
///
|
||||
/// SceneBoundaries — the join point of the decode butterfly.
|
||||
///
|
||||
/// The topology forks after decode: one branch runs TransNetV2 over dense
|
||||
/// frames, the other runs face detection over the sampled cadence. Boundaries
|
||||
/// found on the first branch have to reach the second, and they cannot be
|
||||
/// carried in the frames themselves because the branches are parallel.
|
||||
///
|
||||
/// **Why this needs a watermark.** TransNetV2 buffers `kWindow` frames before it
|
||||
/// can score any of them, so at any instant the detector has an opinion about
|
||||
/// everything up to some time T and nothing after it. Without recording T, a
|
||||
/// consumer asking "is there a boundary at t?" cannot distinguish *no* from
|
||||
/// *not yet* — and those demand opposite behaviour. Silently treating unscored
|
||||
/// frames as boundary-free is exactly the class of failure that makes a
|
||||
/// verification pass vacuously.
|
||||
///
|
||||
/// The consumer is held back by channel depth (see main.cpp) so that by the time
|
||||
/// it pulls a frame, the detector has already scored past it. `scored_through()`
|
||||
/// is what lets that assumption be *checked* rather than assumed.
|
||||
|
||||
#include <algorithm>
|
||||
#include <condition_variable>
|
||||
#include <mutex>
|
||||
#include <vector>
|
||||
|
||||
class SceneBoundaries {
|
||||
public:
|
||||
/// Peaks closer than this are one boundary. Matches the dedup scenes.json
|
||||
/// applies, so the two views agree.
|
||||
static constexpr double kMergeSec = 0.04;
|
||||
|
||||
/// Called by the scene detector as each window is scored. `through` is the
|
||||
/// timestamp up to which its verdict is now final.
|
||||
void publish(const std::vector<double>& ts, double through) {
|
||||
{
|
||||
std::lock_guard<std::mutex> g(mu_);
|
||||
// Dedup on insert, matching what scenes.json does at write time. A run
|
||||
// of adjacent high-scoring frames is one boundary, not several, and
|
||||
// leaving them raw made this view report 357 where the file said 13 —
|
||||
// the same event counted many times. Harmless for is_boundary(), which
|
||||
// absorbs them in its tolerance, but a count nobody can reconcile with
|
||||
// the output file is a bad diagnostic.
|
||||
bounds_.insert(bounds_.end(), ts.begin(), ts.end());
|
||||
std::sort(bounds_.begin(), bounds_.end());
|
||||
bounds_.erase(std::unique(bounds_.begin(), bounds_.end(),
|
||||
[](double a, double b) { return b - a < kMergeSec; }),
|
||||
bounds_.end());
|
||||
scored_through_ = std::max(scored_through_, through);
|
||||
}
|
||||
cv_.notify_all();
|
||||
}
|
||||
|
||||
/// True if a boundary falls within `tol` of `t`.
|
||||
///
|
||||
/// `tol` exists because the two branches sample at different rates: a
|
||||
/// boundary found on a dense frame rarely lands exactly on a sampled one.
|
||||
/// Half a sample interval is the natural width — it attributes the boundary
|
||||
/// to the nearest sampled frame and no further.
|
||||
bool is_boundary(double t, double tol) const {
|
||||
std::lock_guard<std::mutex> g(mu_);
|
||||
auto it = std::lower_bound(bounds_.begin(), bounds_.end(), t - tol);
|
||||
return it != bounds_.end() && *it <= t + tol;
|
||||
}
|
||||
|
||||
/// The timestamp through which the detector's verdict is final. A consumer
|
||||
/// past this point is asking about frames nobody has looked at yet.
|
||||
double scored_through() const {
|
||||
std::lock_guard<std::mutex> g(mu_);
|
||||
return scored_through_;
|
||||
}
|
||||
|
||||
/// Block until the detector's verdict covers `t`, or it finishes.
|
||||
///
|
||||
/// Channel depth alone does NOT create the required lag: it only holds
|
||||
/// frames back when the consumer is slower, and the face branch is roughly
|
||||
/// four orders of magnitude faster per frame than TransNetV2. So the join
|
||||
/// has to wait explicitly.
|
||||
///
|
||||
/// Returns false if the detector finished without ever covering `t`, which
|
||||
/// happens for the tail frames after its last full window. The caller must
|
||||
/// distinguish that from a genuine "no boundary" rather than assuming.
|
||||
bool wait_until_scored(double t) const {
|
||||
std::unique_lock<std::mutex> lk(mu_);
|
||||
cv_.wait(lk, [&] { return finished_ || scored_through_ >= t; });
|
||||
return scored_through_ >= t;
|
||||
}
|
||||
|
||||
/// Called when the detector will publish nothing further. Without this the
|
||||
/// join would deadlock on the tail: those frames are never covered by a full
|
||||
/// window, so waiting for them would wait forever.
|
||||
void finish() {
|
||||
{
|
||||
std::lock_guard<std::mutex> g(mu_);
|
||||
finished_ = true;
|
||||
}
|
||||
cv_.notify_all();
|
||||
}
|
||||
|
||||
std::size_t count() const {
|
||||
std::lock_guard<std::mutex> g(mu_);
|
||||
return bounds_.size();
|
||||
}
|
||||
|
||||
/// Consumers that outran the detector. Nonzero means the face branch is not
|
||||
/// buffered deeply enough for the detector's window, so some frames were
|
||||
/// annotated from an incomplete verdict — a real misconfiguration, and one
|
||||
/// that would otherwise be invisible.
|
||||
void note_outran() const {
|
||||
std::lock_guard<std::mutex> g(mu_);
|
||||
++outran_;
|
||||
}
|
||||
std::size_t outran() const {
|
||||
std::lock_guard<std::mutex> g(mu_);
|
||||
return outran_;
|
||||
}
|
||||
|
||||
private:
|
||||
mutable std::mutex mu_;
|
||||
mutable std::condition_variable cv_;
|
||||
bool finished_{false};
|
||||
std::vector<double> bounds_;
|
||||
double scored_through_{-1.0};
|
||||
mutable std::size_t outran_{0};
|
||||
};
|
||||
+3
-10
@@ -21,7 +21,6 @@
|
||||
|
||||
#include "config.hpp"
|
||||
#include "types.hpp"
|
||||
#include "gallery/embedder_stamp.hpp"
|
||||
#include "gallery/gallery_store.hpp"
|
||||
#include "nodes/frame_source_node.hpp"
|
||||
#include "nodes/camera_position_change_detector_node.hpp"
|
||||
@@ -77,7 +76,6 @@ static Config parse_args(int argc, char** argv) {
|
||||
else if (arg("--detector-engine")) cfg.detector_engine = next();
|
||||
else if (arg("--arcface")) cfg.arcface_model = next();
|
||||
else if (arg("--arcface-engine")) cfg.arcface_engine = next();
|
||||
else if (arg("--require-gallery-stamp")) cfg.require_gallery_stamp = true;
|
||||
else if (arg("--conf")) cfg.detector_conf = std::stof(next());
|
||||
else if (arg("--max-faces")) cfg.max_faces = std::stoi(next());
|
||||
else if (arg("--min-face-px")) cfg.min_face_px = std::stof(next());
|
||||
@@ -96,8 +94,8 @@ static Config parse_args(int argc, char** argv) {
|
||||
// Per-film gallery expansion — preview supports it (same cfg fields).
|
||||
else if (arg("--expand-gallery")) cfg.expand_gallery = true;
|
||||
else if (arg("--expand-buffer")) cfg.expand_buffer_size = std::stoi(next());
|
||||
else if (arg("--expand-band-lo")) cfg.expand_band_lo = std::stof(next());
|
||||
else if (arg("--expand-band-hi")) cfg.expand_band_hi = std::stof(next());
|
||||
else if (arg("--expand-novelty-sim")) cfg.expand_novelty_sim = std::stof(next());
|
||||
else if (arg("--expand-spread-max")) cfg.expand_track_spread_max = std::stof(next());
|
||||
else if (arg("--expand-min-anchor")) cfg.expand_min_anchor_frames = std::stoi(next());
|
||||
// Scene detection is scene_analyze-only (needs the dense TransNetV2 branch).
|
||||
// Accept the flags so a shared command line runs, but note they're inert
|
||||
@@ -128,12 +126,7 @@ int main(int argc, char** argv) {
|
||||
}
|
||||
|
||||
ActorGallery gallery;
|
||||
try {
|
||||
gallery = load_gallery(cfg.gallery_path);
|
||||
/// TRACES: GR-004 | SR-001
|
||||
verify_gallery_embedder(gallery, cfg.gallery_path, cfg.arcface_model,
|
||||
cfg.require_gallery_stamp);
|
||||
}
|
||||
try { gallery = load_gallery(cfg.gallery_path); }
|
||||
catch (const std::exception& e) {
|
||||
std::cerr << "Gallery error: " << e.what() << "\n";
|
||||
return 1;
|
||||
|
||||
@@ -1,355 +0,0 @@
|
||||
#pragma once
|
||||
/// TRACES: AR-012, AR-013, AR-014, AR-015, AR-016, AR-017 | SR-002
|
||||
///
|
||||
/// TrackRegistry — the single owner of track state and of presence.
|
||||
///
|
||||
/// Presence follows **track extent**, not per-frame recognition (AR-012): a
|
||||
/// window is `[first_seen, last_seen]` of a track an actor owns, so it starts
|
||||
/// when the actor appeared rather than when the recogniser first succeeded.
|
||||
///
|
||||
/// `last_seen` carries the entire liveness state (AR-013):
|
||||
///
|
||||
/// unset → on screen now
|
||||
/// set → went off screen at that timestamp, still revivable
|
||||
/// reaped → emitted to the aggregator and erased
|
||||
///
|
||||
/// There is no missing-frame counter and no expired flag; the optional *is* the
|
||||
/// state machine, and it subsumes what was previously a two-pool split in the
|
||||
/// tracker (active vs. parked-across-a-cut).
|
||||
///
|
||||
/// **Interior gaps are claimed, the trailing cool-down is not.** A face lost at
|
||||
/// t1 and re-associated at t2 within the timeout never closed its track, so the
|
||||
/// actor is present across [t1, t2] — correct, since someone briefly occluded or
|
||||
/// off-camera has not left the scene. But a track that dies ends its window at
|
||||
/// `last_seen`, never at the moment of death. That asymmetry is what removes the
|
||||
/// over-claim the retired `extinction_sec` keep-alive produced.
|
||||
///
|
||||
/// The registry is created in `main` and shared by `shared_ptr`; it is *not* a
|
||||
/// KPN node. Ownership is not a stage in the stream — it is state several stages
|
||||
/// read and write, whose final answer is only known when a track dies.
|
||||
|
||||
#include "types.hpp"
|
||||
#include "evidence_discount.hpp"
|
||||
|
||||
#include <cmath>
|
||||
#include <cstdint>
|
||||
#include <functional>
|
||||
#include <map>
|
||||
#include <mutex>
|
||||
#include <optional>
|
||||
#include <string>
|
||||
#include <vector>
|
||||
|
||||
// ── DeadTrack ────────────────────────────────────────────────────────────────
|
||||
// A finished presence claim, emitted exactly once when a track is reaped or
|
||||
// flushed. Immutable by construction: it carries everything needed to justify
|
||||
// itself (AR-017), with no back-reference into registry state.
|
||||
struct DeadTrack {
|
||||
int track_id{-1};
|
||||
double first_seen{0.0};
|
||||
double last_seen{0.0}; ///< always the last sighting, never the death time
|
||||
int actor_idx{-1}; ///< -1 when the track was never owned
|
||||
float belief{0.0f}; ///< accumulated posterior for actor_idx
|
||||
int observations{0}; ///< evidence updates that landed on this track
|
||||
float effective_obs{0.f}; ///< sum of discounted weights — the evidence that counted
|
||||
};
|
||||
|
||||
// ── Track ────────────────────────────────────────────────────────────────────
|
||||
struct Track {
|
||||
int id{-1};
|
||||
double first_seen{0.0};
|
||||
std::optional<double> last_seen; ///< unset ⇒ on screen
|
||||
std::optional<int> actor; ///< set once a posterior crosses
|
||||
/// actor_idx → accumulated log(1 − P). Lazy-OR (noisy-OR) accumulation:
|
||||
/// each frame is new evidence that this track is that actor, and the
|
||||
/// combined belief is the probability that *at least one* sighting was
|
||||
/// right. Stored as log(1−P) because that makes the update additive and
|
||||
/// keeps precision where it matters — as P approaches 1, (1−P) is the
|
||||
/// quantity with the significant digits.
|
||||
std::map<int, float> belief;
|
||||
Embedding mean{}; ///< running directional mean
|
||||
std::vector<Embedding> views; ///< distinct looks, for AR-025 discounting
|
||||
float discounted_weight{0.f}; ///< sum of applied weights
|
||||
int n_obs{0};
|
||||
|
||||
bool on_screen() const { return !last_seen.has_value(); }
|
||||
};
|
||||
|
||||
// ── TrackRegistry ────────────────────────────────────────────────────────────
|
||||
class TrackRegistry {
|
||||
public:
|
||||
using DeadTrackFn = std::function<void(const DeadTrack&)>;
|
||||
|
||||
struct Config {
|
||||
double extinction_sec{5.0}; ///< how long a lost track stays revivable
|
||||
float ownership_logodds{2.0f}; ///< belief needed to own a track (~0.88 posterior)
|
||||
};
|
||||
|
||||
/// The discounter is a constructor argument rather than an option: there is
|
||||
/// no correct way to accumulate per-frame evidence without it.
|
||||
TrackRegistry(Config cfg, EvidenceDiscounter discounter)
|
||||
: cfg_(cfg), discounter_(std::move(discounter)) {}
|
||||
|
||||
void on_track_dead(DeadTrackFn fn) { on_dead_ = std::move(fn); }
|
||||
|
||||
// ── Frame scope ──────────────────────────────────────────────────────────
|
||||
// The tracker mutates registry state across a whole association pass, so
|
||||
// that pass must be atomic as a unit — per-call locking would let another
|
||||
// thread observe a half-updated frame. FrameScope holds the lock for its
|
||||
// lifetime and exposes the mutating operations without re-locking.
|
||||
class FrameScope {
|
||||
public:
|
||||
FrameScope(TrackRegistry& reg, double now)
|
||||
: reg_(reg), lock_(reg.mu_) { reg_.tick_locked(now); }
|
||||
|
||||
/// All live tracks — **one pool**. `last_seen` tells the caller whether
|
||||
/// IoU is meaningful; a dormant track is matched on embedding alone.
|
||||
/// There is no separate revival path (AR-008).
|
||||
std::vector<Track*> candidates() {
|
||||
std::vector<Track*> out;
|
||||
out.reserve(reg_.tracks_.size());
|
||||
for (auto& [id, t] : reg_.tracks_) out.push_back(&t);
|
||||
return out;
|
||||
}
|
||||
|
||||
int create(double t, const Embedding& e) { return reg_.create_locked(t, e); }
|
||||
void mark_seen(int id, double t, const Embedding& e){ reg_.mark_seen_locked(id, t, e); }
|
||||
void mark_lost(int id, double last_on_screen) { reg_.mark_lost_locked(id, last_on_screen); }
|
||||
|
||||
private:
|
||||
TrackRegistry& reg_;
|
||||
std::unique_lock<std::mutex> lock_;
|
||||
};
|
||||
|
||||
FrameScope begin_frame(double now) { return FrameScope(*this, now); }
|
||||
|
||||
/// Advance the clock and reap. Called every sampled frame **whether or not
|
||||
/// it had detections** — without it a track only dies when some other face
|
||||
/// happens to appear, and a film ending mid-track never closes.
|
||||
void tick(double now) { std::lock_guard g(mu_); tick_locked(now); }
|
||||
|
||||
// ── Evidence ─────────────────────────────────────────────────────────────
|
||||
/// Fold one observation into a track's belief (AR-025).
|
||||
///
|
||||
/// `posterior` is a **calibrated probability**, never a raw cosine
|
||||
/// (AR-024) — the registry converts it to log-odds itself, so the
|
||||
/// accumulation cannot be fed an uncalibrated number by a careless caller.
|
||||
///
|
||||
/// Correlation discounting is applied **here**, not by the caller.
|
||||
/// Consecutive frames of one track are near-identical, and accumulating
|
||||
/// them as independent evidence drives the posterior to certainty on what
|
||||
/// is effectively a single measurement. Leaving that to callers would mean
|
||||
/// a forgotten or doubly-applied discount produces confident wrong answers
|
||||
/// silently; the registry is the one place all evidence converges, so it is
|
||||
/// the one place the correction belongs.
|
||||
///
|
||||
/// A vote for a track that has already been reaped is dropped and counted:
|
||||
/// a nonzero `dropped_votes()` means the timeout is shorter than the
|
||||
/// matcher's lag, which is a real misconfiguration and must not be silent.
|
||||
void observe(int track_id, int actor_idx, float posterior, const Embedding& e) {
|
||||
std::lock_guard g(mu_);
|
||||
auto it = tracks_.find(track_id);
|
||||
if (it == tracks_.end()) { ++dropped_votes_; return; }
|
||||
|
||||
Track& t = it->second;
|
||||
const float w = discounter_.weight(t.views, t.n_obs, e);
|
||||
|
||||
// Weighted lazy-OR: P_new = 1 − (1 − P_old)·(1 − p)^w, which in log
|
||||
// space is a plain sum. w is the discounted evidence (AR-025), so a
|
||||
// repeated view still advances the belief but by a fraction of what a
|
||||
// genuinely new look would.
|
||||
const float p = std::min(1.f - 1e-6f, std::max(1e-6f, posterior));
|
||||
t.belief[actor_idx] += w * std::log(1.f - p);
|
||||
t.discounted_weight += w;
|
||||
++t.n_obs;
|
||||
|
||||
const int best = argmax_belief(t);
|
||||
const float best_p = 1.f - std::exp(t.belief[best]);
|
||||
if (best_p < own_threshold()) return;
|
||||
|
||||
if (!t.actor.has_value()) {
|
||||
claim_locked(t, best);
|
||||
return;
|
||||
}
|
||||
if (*t.actor != best) {
|
||||
// AR-014 — belief swapped A→B. Not a correction: a track_id almost
|
||||
// certainly carried across a viewpoint change onto a different
|
||||
// person. Two non-twins both clearing the threshold on one face is
|
||||
// not realistic; a track spanning two people is. Continuing would
|
||||
// emit one window blending both, so close here and start afresh.
|
||||
split_locked(t, best);
|
||||
}
|
||||
}
|
||||
|
||||
/// Snapshot read: tally and verdict under one lock. Reading them separately
|
||||
/// would let a track be both unowned and owned within a single promotion
|
||||
/// decision, since the matcher may be voting concurrently.
|
||||
std::optional<int> owner(int track_id) const {
|
||||
std::lock_guard g(mu_);
|
||||
auto it = tracks_.find(track_id);
|
||||
return it == tracks_.end() ? std::nullopt : it->second.actor;
|
||||
}
|
||||
|
||||
// ── Termination ──────────────────────────────────────────────────────────
|
||||
/// Emit every still-live track and empty the registry (AR-016). A film ends
|
||||
/// with faces on screen and those tracks have not timed out, so without this
|
||||
/// the closing scene's cast is silently never emitted — a loss that presents
|
||||
/// as a recognition miss rather than a bookkeeping bug.
|
||||
///
|
||||
/// Idempotent: calling it twice emits nothing the second time.
|
||||
void flush(double final_ts) {
|
||||
std::lock_guard g(mu_);
|
||||
for (auto& [id, t] : tracks_) emit_locked(t, t.last_seen.value_or(final_ts));
|
||||
tracks_.clear();
|
||||
}
|
||||
|
||||
// ── Diagnostics ──────────────────────────────────────────────────────────
|
||||
// These measure how often tracking is silently wrong, which nothing in the
|
||||
// pipeline currently reveals.
|
||||
int dropped_votes() const { std::lock_guard g(mu_); return dropped_votes_; }
|
||||
int belief_swaps() const { std::lock_guard g(mu_); return belief_swaps_; }
|
||||
int actor_conflicts() const { std::lock_guard g(mu_); return actor_conflicts_; }
|
||||
std::size_t live() const { std::lock_guard g(mu_); return tracks_.size(); }
|
||||
|
||||
private:
|
||||
// ── Locked internals ─────────────────────────────────────────────────────
|
||||
void tick_locked(double now) {
|
||||
for (auto it = tracks_.begin(); it != tracks_.end(); ) {
|
||||
const auto& ls = it->second.last_seen;
|
||||
if (ls && (now - *ls) > cfg_.extinction_sec) {
|
||||
emit_locked(it->second, *ls);
|
||||
it = tracks_.erase(it);
|
||||
} else {
|
||||
++it;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
int create_locked(double t, const Embedding& e) {
|
||||
const int id = next_id_++;
|
||||
Track tr;
|
||||
tr.id = id;
|
||||
tr.first_seen = t;
|
||||
tr.mean = e;
|
||||
tr.n_obs = 0;
|
||||
tracks_.emplace(id, std::move(tr));
|
||||
return id;
|
||||
}
|
||||
|
||||
void mark_seen_locked(int id, double t, const Embedding& e) {
|
||||
auto it = tracks_.find(id);
|
||||
if (it == tracks_.end()) return;
|
||||
Track& tr = it->second;
|
||||
tr.last_seen.reset(); // back on screen; the gap is absorbed
|
||||
update_mean(tr, e);
|
||||
(void)t;
|
||||
}
|
||||
|
||||
void mark_lost_locked(int id, double last_on_screen) {
|
||||
auto it = tracks_.find(id);
|
||||
if (it == tracks_.end()) return;
|
||||
it->second.last_seen = last_on_screen;
|
||||
}
|
||||
|
||||
void claim_locked(Track& t, int actor) {
|
||||
// AR-015 — if another live track already owns this actor, at least one
|
||||
// is wrong: a person cannot be in two places at once. The cause is the
|
||||
// same as a belief swap — a missed camera or scene change. Detected on
|
||||
// the update that causes it via the reverse index, not by scanning.
|
||||
auto seen = owner_index_.find(actor);
|
||||
if (seen != owner_index_.end() && seen->second != t.id
|
||||
&& tracks_.count(seen->second)) {
|
||||
++actor_conflicts_;
|
||||
}
|
||||
t.actor = actor;
|
||||
owner_index_[actor] = t.id;
|
||||
}
|
||||
|
||||
void split_locked(Track& t, int new_actor) {
|
||||
++belief_swaps_;
|
||||
const double boundary = t.last_seen.value_or(t.first_seen);
|
||||
emit_locked(t, boundary);
|
||||
|
||||
// The successor inherits the embedding and the belief that caused the
|
||||
// swap, and starts at the swap frame — so the two windows abut without
|
||||
// overlapping and neither blends the two people.
|
||||
Track next;
|
||||
next.id = next_id_++;
|
||||
next.first_seen = boundary;
|
||||
next.mean = t.mean;
|
||||
next.belief[new_actor] = t.belief[new_actor];
|
||||
next.n_obs = 1;
|
||||
const int old_id = t.id;
|
||||
Track stash = std::move(next);
|
||||
tracks_.erase(old_id);
|
||||
const int nid = stash.id;
|
||||
tracks_.emplace(nid, std::move(stash));
|
||||
claim_locked(tracks_.at(nid), new_actor);
|
||||
}
|
||||
|
||||
void emit_locked(Track& t, double end_ts) {
|
||||
if (!on_dead_) return;
|
||||
DeadTrack d;
|
||||
d.track_id = t.id;
|
||||
d.first_seen = t.first_seen;
|
||||
d.last_seen = end_ts;
|
||||
d.observations = t.n_obs;
|
||||
d.effective_obs = t.discounted_weight;
|
||||
if (t.actor) {
|
||||
d.actor_idx = *t.actor;
|
||||
d.belief = 1.f - std::exp(t.belief[*t.actor]);
|
||||
auto oi = owner_index_.find(*t.actor);
|
||||
if (oi != owner_index_.end() && oi->second == t.id) owner_index_.erase(oi);
|
||||
}
|
||||
on_dead_(d);
|
||||
}
|
||||
|
||||
/// Most-believed actor. belief holds log(1 − P), so the strongest claim is
|
||||
/// the *most negative* entry, not the largest.
|
||||
static int argmax_belief(const Track& t) {
|
||||
int best = -1;
|
||||
float lo = 1e30f;
|
||||
for (const auto& [a, v] : t.belief) if (v < lo) { lo = v; best = a; }
|
||||
return best;
|
||||
}
|
||||
|
||||
/// Ownership expressed as a probability. Config still carries log-odds so
|
||||
/// the knob keeps its meaning across this change.
|
||||
float own_threshold() const {
|
||||
return 1.f / (1.f + std::exp(-cfg_.ownership_logodds));
|
||||
}
|
||||
|
||||
static void update_mean(Track& t, const Embedding& e) {
|
||||
// Directional mean: accumulate then re-normalise to the unit sphere, so
|
||||
// cosine against it stays a plain dot product.
|
||||
double norm = 0.0;
|
||||
for (int i = 0; i < 512; ++i) {
|
||||
t.mean[i] = t.mean[i] * static_cast<float>(t.n_obs ? t.n_obs : 1) + e[i];
|
||||
norm += static_cast<double>(t.mean[i]) * t.mean[i];
|
||||
}
|
||||
norm = norm > 0 ? std::sqrt(norm) : 1.0;
|
||||
for (int i = 0; i < 512; ++i) t.mean[i] = static_cast<float>(t.mean[i] / norm);
|
||||
}
|
||||
|
||||
static float logistic(float z) {
|
||||
return z >= 0 ? 1.f / (1.f + std::exp(-z))
|
||||
: std::exp(z) / (1.f + std::exp(z));
|
||||
}
|
||||
|
||||
static float logit(float p) {
|
||||
const float eps = 1e-6f;
|
||||
p = std::min(1.f - eps, std::max(eps, p));
|
||||
return std::log(p / (1.f - p));
|
||||
}
|
||||
|
||||
Config cfg_;
|
||||
EvidenceDiscounter discounter_;
|
||||
mutable std::mutex mu_;
|
||||
std::map<int, Track> tracks_;
|
||||
std::map<int, int> owner_index_; ///< actor_idx → live track_id (AR-015)
|
||||
DeadTrackFn on_dead_;
|
||||
int next_id_{0};
|
||||
int dropped_votes_{0};
|
||||
int belief_swaps_{0};
|
||||
int actor_conflicts_{0};
|
||||
};
|
||||
@@ -6,8 +6,6 @@
|
||||
|
||||
#include <opencv2/core.hpp>
|
||||
|
||||
#include "gallery/embedder_stamp.hpp"
|
||||
|
||||
// ── Embedding ─────────────────────────────────────────────────────────────────
|
||||
// 512-dim L2-normalised ArcFace embedding
|
||||
using Embedding = std::array<float, 512>;
|
||||
@@ -63,13 +61,6 @@ struct DetectedFace {
|
||||
cv::Rect2f bbox;
|
||||
std::array<cv::Point2f, 5> landmarks;
|
||||
float confidence{0.f};
|
||||
|
||||
// AR-030 visibility: RMS landmark misfit, in canonical 112×112 pixels, left
|
||||
// over after the best similarity fit to the ArcFace template. Rises with
|
||||
// out-of-plane pose and with occlusion; blind to in-plane roll and to face
|
||||
// size, both of which the fit absorbs. Set by the aligner, which is where
|
||||
// the transform is computed; -1 until then.
|
||||
float alignment_residual{-1.f};
|
||||
};
|
||||
|
||||
// ── Pipeline messages ─────────────────────────────────────────────────────────
|
||||
@@ -145,11 +136,6 @@ struct ActorGallery {
|
||||
};
|
||||
std::vector<Actor> actors;
|
||||
|
||||
/// TRACES: GR-004 | SR-001
|
||||
// Which embedder produced every embedding above. Empty == the file predates
|
||||
// model binding; see gallery/embedder_stamp.hpp for what is checked and why.
|
||||
EmbedderStamp embedder;
|
||||
|
||||
// Cached Platt-sigmoid calibration (see gallery/gallery_calibration.hpp),
|
||||
// stored alongside the gallery in HDF5 so it never needs recomputing
|
||||
// unless the reference embeddings actually change. calib_valid=false and
|
||||
|
||||
+1
-25
@@ -21,45 +21,21 @@ add_executable(sae_tests
|
||||
test_face_utils.cpp
|
||||
test_track_gallery.cpp
|
||||
test_face_tracker.cpp
|
||||
test_track_registry.cpp
|
||||
test_replay_fixtures.cpp
|
||||
test_audio_signature.cpp
|
||||
${CMAKE_SOURCE_DIR}/src/backends/gemm_backend.cpp
|
||||
${CMAKE_SOURCE_DIR}/src/gallery/gallery_store.cpp
|
||||
${CMAKE_SOURCE_DIR}/src/audio_signature.cpp
|
||||
${CMAKE_SOURCE_DIR}/src/gallery/embedder_stamp.cpp
|
||||
)
|
||||
target_include_directories(sae_tests PRIVATE ${CMAKE_SOURCE_DIR}/src)
|
||||
# SAE_GEMM_CPU: build the CPU reference GEMM regardless of the main backend.
|
||||
# SAE_MODELS_DIR: config.hpp (pulled in by track_gallery.hpp) bakes model paths.
|
||||
# SAE_TEST_FIXTURES_DIR: the audio golden vector is read from the source tree,
|
||||
# not copied, so the file the plugin repo shares is the file under test.
|
||||
# AR-026/AR-027: exercise the same kernel CI actually runs. Without this the
|
||||
# suite compiles the scalar fallback while the CPU builder image links OpenBLAS,
|
||||
# so the tested path and the shipped path would differ.
|
||||
find_package(PkgConfig QUIET)
|
||||
if(PkgConfig_FOUND)
|
||||
pkg_check_modules(OPENBLAS_T QUIET openblas)
|
||||
endif()
|
||||
if(OPENBLAS_T_FOUND)
|
||||
target_include_directories(sae_tests PRIVATE ${OPENBLAS_T_INCLUDE_DIRS})
|
||||
target_link_libraries(sae_tests PRIVATE ${OPENBLAS_T_LINK_LIBRARIES})
|
||||
endif()
|
||||
|
||||
target_compile_definitions(sae_tests PRIVATE
|
||||
$<$<BOOL:${OPENBLAS_T_FOUND}>:SAE_GEMM_CBLAS>
|
||||
SAE_GEMM_CPU
|
||||
SAE_MODELS_DIR="${SAE_MODELS_DIR}"
|
||||
SAE_TEST_FIXTURES_DIR="${CMAKE_CURRENT_SOURCE_DIR}/fixtures")
|
||||
SAE_MODELS_DIR="${SAE_MODELS_DIR}")
|
||||
# gallery_store.cpp + gallery_calibration.hpp use nlohmann/json and HDF5
|
||||
# (galleries are HDF5-native, see src/gallery/gallery_store.cpp); face_utils.hpp
|
||||
# and the calibration GEMM pull in OpenCV (calib3d/imgproc/core) via types.hpp.
|
||||
# ffmpeg_libs: audio_signature.cpp decodes the golden fixture (avformat/avcodec/
|
||||
# avutil/swresample). Still GPU-free — the audio path is pure CPU.
|
||||
target_link_libraries(sae_tests PRIVATE
|
||||
Catch2::Catch2WithMain
|
||||
nlohmann_json::nlohmann_json
|
||||
ffmpeg_libs
|
||||
${OpenCV_LIBS}
|
||||
${HDF5_CXX_LIBRARIES})
|
||||
target_include_directories(sae_tests PRIVATE ${HDF5_INCLUDE_DIRS})
|
||||
|
||||
-194
@@ -1,194 +0,0 @@
|
||||
{
|
||||
"_": "Golden vector for the JRay v1 audio signature (JRay-public-server SPEC.md \u00a73). Shared verbatim between scene-actor-extraction (C++) and the jRay Jellyfin plugin (C#) so the two implementations can be proven bit-identical. IR-004, IR-005, IR-007, IR-008.",
|
||||
"version": "v1",
|
||||
"signature": "v1:AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAeHx8eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh8fHzk5OTk5OTk5OTk5OTk5OTk5OTk5OTk5OTk5OTk5V1dXV1dXV1dXV1dXV1dXV1dXV1dXV1dXV1dXV1dXV1dXV1dycnJycnJycnJycnJycnJycnJycnJycnJycnJycnMPDgwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMKysrKysrKysrKysrKysrKysrKysrKysrKysrKysrKysrKytFRUVFRUVFRUVFRUVFRUVFRUVFRUVFRUVFRUVFRWNjY2NjY2NjY2NjY2NjY2NjY2NjY2NjY2NjY2NjY2NjY2Njfn5+fn5+fn5+fn5+fn5+fn5+fn5+fn5+fn5+fn5/GxoZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGRkZGTc3Nzc3Nzc3Nzc3Nzc3Nzc3Nzc3Nzc3Nzc3Nzc3Nzc3Nzc3UlJSUlJSUlJSUlJSUlJSUlJSUlJSUlJSUlJSUlJSU1JsbGxsbGxsbGxsbGxsbGxsbGxsbGxsbGxsbGxsbAoLCwoKCgoKCgoKCgoKCgoKCgoKCgoKCgoKCgoKCgoKCwsLJSUlJSUlJSUlJSUlJSUlJSUlJSUlJSUlJSUlJSVDQ0NDQ0NDQ0NDQ0NDQ0NDQ0NDQ0NDQ0NDQ0NDQ0NDQ0NDQ15eXl5eXl5eXl5eXl5eXl5eXl5eXl5eXl5eXl5eX19eeXl5eXl5eXl5eXl5eXl5eXl5eXl5eXl5eXl5eXkXFxcXFxcXFxcXFxcXFxcXFxcXFxcXFxcXFxcXFxcXFxcXFzExMTExMTExMTExMTExMTExMTExMTExMTExMTExT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09qampqampqampqampqampqampqampqampqampqamsHBwUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyM+Pj4+Pj4+Pj4+Pj4+Pj4+Pj4+Pj4+Pj4+Pj4+Pj4/PlhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYd3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3d3ExMRERERERERERERERERERERERERERERERERERERES8vLy8vLy8vLy8vLy8vLy8vLy8vLy8vLy8vLy8vLy8vLy8vSkpKSkpKSkpKSkpKSkpKSkpKSkpKSkpKSkpKSkpLS0plZWVlZWVlZWVlZWVlZWVlZWVlZWVlZWVlZWVlZQMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDHR0dHR0dHR0dHR0dHR0dHR0dHR0dHR0dHR0dHR0eHh44ODg4ODg4ODg4ODg4ODg4ODg4ODg4ODg4ODg4OFdXV1ZWVlZWVlZWVlZWVlZWVlZWVlZWVlZWVlZWVlZWV1dXcXFxcXFxcXFxcXFxcXFxcXFxcXFxcXFxcXFxcXEPDw8PDw8PDw8PDw8PDw8PDw8PDw8PDw8PDw8PDw8PDw8PDyoqKioqKioqKioqKioqKioqKioqKioqKioqKioqKysqRERERERERERERERERERERERERERERERERERERERjY2NjY2NjY2NjYw==",
|
||||
"frame_count": 1288,
|
||||
"media": {
|
||||
"file": "jray_audio_v1_tone.flac",
|
||||
"generator": "make_fixture.py",
|
||||
"container": "FLAC (lossless \u2014 decodes to exactly the PCM make_fixture.py emits)",
|
||||
"duration_sec": 120.0,
|
||||
"sample_rate": 11025,
|
||||
"channels": 1,
|
||||
"sample_format": "s16",
|
||||
"sha256": "912ecd426cd426dccb37753e0249694227619c701cb9f533502b37da0fbe8096",
|
||||
"bytes": 585142
|
||||
},
|
||||
"decoded_window": {
|
||||
"_": "Checksums of the 120 s centre window after downmix to mono and resample to 11025 Hz, i.e. exactly the stream `ffmpeg -ss <mid-60> -t 120 -i <file> -vn -ac 1 -ar 11025 -f f32le -` produces. Check these first: a mismatch here is a decode problem, not a DSP one.",
|
||||
"samples": 1323000,
|
||||
"f32le_fnv1a64": "0x1ef7899cd4d12662",
|
||||
"s16le_fnv1a64": "0xf824fa56f125c0dc"
|
||||
},
|
||||
"params": {
|
||||
"window_sec": 120.0,
|
||||
"window_centre": "runtime/2, i.e. samples from runtime/2 - 60 s; truncated to exactly 1323000 samples",
|
||||
"min_duration_sec": 120.0,
|
||||
"min_duration_rule": "IR-007 \u2014 below this emit NO signature and apply no sync offset",
|
||||
"sample_rate": 11025,
|
||||
"channels": 1,
|
||||
"arithmetic": "IEEE-754 double throughout; float32 is not sufficient",
|
||||
"sample_scale": "s16 * (1/32768), FFmpeg's native s16->flt",
|
||||
"frame_size": 4096,
|
||||
"hop_size": 1024,
|
||||
"frame_count_rule": "1 + (n_samples - 4096) / 1024, integer division; whole frames only",
|
||||
"window_fn": "Hann, PERIODIC: w[n] = 0.5 * (1 - cos(2*pi*n/4096))",
|
||||
"transform": "radix-2 DIT complex FFT over the 4096 real samples (imag=0), no normalisation",
|
||||
"magnitude": "sqrt(re^2 + im^2), linear",
|
||||
"band_lo_hz": 300.0,
|
||||
"band_hi_hz": 3000.0,
|
||||
"num_bands": 32,
|
||||
"band_edges": "edge[b] = 300 * (3000/300)^(b/32), b = 0..32",
|
||||
"band_bins": "band b owns FFT bins [k_lo[b], k_lo[b+1]) with k_lo[b] = ceil(edge[b] * 4096 / 11025); see band_fft_bins",
|
||||
"band_value": "MEAN of the linear magnitudes in the band (not sum, not max)",
|
||||
"peak_bin": "argmax over the 32 band values; ties resolve to the LOWEST index",
|
||||
"energy_metric": "E = mean magnitude over all FFT bins 112..1114, i.e. the whole 300-3000 Hz band",
|
||||
"energy_reference": "upper median of E over all frames: sorted[n/2], no averaging of the two middle values",
|
||||
"energy_ratio": "r = log10((E + 1e-12) / (E_ref + 1e-12))",
|
||||
"energy_class_edges": [
|
||||
-0.6,
|
||||
-0.2,
|
||||
0.2
|
||||
],
|
||||
"energy_class": "0 if r < -0.6, 1 if r < -0.2, 2 if r < 0.2, else 3",
|
||||
"byte_layout": "bit7 = 0 (reserved), bits6..2 = 5-bit band index, bits1..0 = 2-bit energy class; byte = (band << 2) | class",
|
||||
"base64": "standard alphabet A-Za-z0-9+/ with '=' padding",
|
||||
"prefix": "v1:"
|
||||
},
|
||||
"band_fft_bins": [
|
||||
[
|
||||
112,
|
||||
120
|
||||
],
|
||||
[
|
||||
120,
|
||||
129
|
||||
],
|
||||
[
|
||||
129,
|
||||
139
|
||||
],
|
||||
[
|
||||
139,
|
||||
149
|
||||
],
|
||||
[
|
||||
149,
|
||||
160
|
||||
],
|
||||
[
|
||||
160,
|
||||
172
|
||||
],
|
||||
[
|
||||
172,
|
||||
185
|
||||
],
|
||||
[
|
||||
185,
|
||||
199
|
||||
],
|
||||
[
|
||||
199,
|
||||
213
|
||||
],
|
||||
[
|
||||
213,
|
||||
229
|
||||
],
|
||||
[
|
||||
229,
|
||||
246
|
||||
],
|
||||
[
|
||||
246,
|
||||
265
|
||||
],
|
||||
[
|
||||
265,
|
||||
285
|
||||
],
|
||||
[
|
||||
285,
|
||||
306
|
||||
],
|
||||
[
|
||||
306,
|
||||
328
|
||||
],
|
||||
[
|
||||
328,
|
||||
353
|
||||
],
|
||||
[
|
||||
353,
|
||||
379
|
||||
],
|
||||
[
|
||||
379,
|
||||
408
|
||||
],
|
||||
[
|
||||
408,
|
||||
438
|
||||
],
|
||||
[
|
||||
438,
|
||||
471
|
||||
],
|
||||
[
|
||||
471,
|
||||
506
|
||||
],
|
||||
[
|
||||
506,
|
||||
543
|
||||
],
|
||||
[
|
||||
543,
|
||||
584
|
||||
],
|
||||
[
|
||||
584,
|
||||
627
|
||||
],
|
||||
[
|
||||
627,
|
||||
674
|
||||
],
|
||||
[
|
||||
674,
|
||||
724
|
||||
],
|
||||
[
|
||||
724,
|
||||
778
|
||||
],
|
||||
[
|
||||
778,
|
||||
836
|
||||
],
|
||||
[
|
||||
836,
|
||||
899
|
||||
],
|
||||
[
|
||||
899,
|
||||
966
|
||||
],
|
||||
[
|
||||
966,
|
||||
1038
|
||||
],
|
||||
[
|
||||
1038,
|
||||
1115
|
||||
]
|
||||
],
|
||||
"notes": [
|
||||
"The server spec fixes the window, rate, STFT geometry, band and the 5+2 bit packing. Everything under params beyond that (Hann periodicity, band aggregation, the energy-class definition, tie-breaking, base64 alphabet) is pinned HERE for v1 \u2014 the spec does not constrain it, and two implementations that guess differently produce non-matching signatures.",
|
||||
"Decision margins on this fixture: the two strongest bands are within 1.3% on the closest frame, and the closest frame to an energy-class edge is 3.6e-3 away in log10. Both are many orders of magnitude above double-precision FFT differences, so any two correct double- precision implementations agree; a float32 implementation is not guaranteed to.",
|
||||
"Coverage: all 32 bands and all 4 energy classes appear in the golden signature.",
|
||||
"Robustness observed on this fixture: identical peak-bin sequence after a stereo/44100 Hz round trip and after AAC 128 kbit/s re-encoding."
|
||||
]
|
||||
}
|
||||
BIN
Binary file not shown.
Vendored
-114
@@ -1,114 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Regenerate the JRay audio-signature golden fixture.
|
||||
|
||||
python3 make_fixture.py # writes jray_audio_v1_tone.flac here
|
||||
|
||||
This is the *source of truth* for the fixture media: `jray_audio_v1_tone.flac`
|
||||
is a lossless FLAC encoding of exactly the PCM this script emits, so any repo
|
||||
that wants to check its own audio-signature implementation against the golden
|
||||
vector in `jray_audio_v1_golden.json` can regenerate the input from scratch and
|
||||
confirm it is byte-identical (the golden file records `pcm_fnv1a64`, a hash of
|
||||
the decoded 16-bit samples).
|
||||
|
||||
Deliberately dependency-free (no numpy) and written in plain arithmetic so it
|
||||
ports to any language in ~20 lines.
|
||||
|
||||
Signal — 120.000 s, mono, 11025 Hz, 16-bit signed PCM:
|
||||
|
||||
* split into segments of 32768 samples (~2.97 s), 40.4 segments in total;
|
||||
* segment `s` carries one sine at the geometric centre of log-band
|
||||
`(s * 7) mod 32` of the 300-3000 Hz band, so all 32 bands are exercised;
|
||||
* its amplitude walks a golden-ratio low-discrepancy sequence over
|
||||
[10^-1.55, 10^-0.02] so frame energies spread continuously across ~1.5
|
||||
decades and all four energy classes are exercised, without a dense cluster
|
||||
of frames sitting on a class boundary;
|
||||
* phase is carried across segment boundaries (no clicks);
|
||||
* a constant, far quieter 777 Hz tone sits underneath so no frame is
|
||||
degenerate;
|
||||
* samples are quantised with floor(x * 32767 + 0.5).
|
||||
|
||||
Why FLAC and not WAV: 120 s of 11025 Hz 16-bit PCM is 2.6 MB and does not
|
||||
compress in git. FLAC is lossless — FFmpeg decodes it to exactly the PCM
|
||||
written here — and is ~3.5x smaller. `--wav` writes the uncompressed original
|
||||
if you want to diff it.
|
||||
"""
|
||||
import math
|
||||
import struct
|
||||
import subprocess
|
||||
import sys
|
||||
import os
|
||||
|
||||
SAMPLE_RATE = 11025
|
||||
DURATION_SEC = 120.0
|
||||
SEGMENT = 32768 # samples per tone segment
|
||||
BAND_STRIDE = 7 # coprime with 32 -> visits every band
|
||||
BAND_LO_HZ = 300.0
|
||||
BAND_HI_HZ = 3000.0
|
||||
NUM_BANDS = 32
|
||||
AMP_LOG_MIN = -1.55 # 10^-1.55 ~= 0.028
|
||||
AMP_LOG_SPAN = 1.53 # up to 10^-0.02 ~= 0.955
|
||||
PHI_FRAC = 0.6180339887498949
|
||||
BG_HZ = 777.0
|
||||
BG_AMP = 0.004
|
||||
|
||||
OUT_FLAC = "jray_audio_v1_tone.flac"
|
||||
OUT_WAV = "jray_audio_v1_tone.wav"
|
||||
|
||||
|
||||
def generate():
|
||||
"""Return the 120 s signal as a list of int16 sample values."""
|
||||
n = int(round(SAMPLE_RATE * DURATION_SEC))
|
||||
out = [0] * n
|
||||
phase = 0.0
|
||||
two_pi = 2.0 * math.pi
|
||||
for start in range(0, n, SEGMENT):
|
||||
s = start // SEGMENT
|
||||
end = min(n, start + SEGMENT)
|
||||
band = (s * BAND_STRIDE) % NUM_BANDS
|
||||
# geometric centre of log-band `band`
|
||||
freq = BAND_LO_HZ * (BAND_HI_HZ / BAND_LO_HZ) ** ((band + 0.5) / NUM_BANDS)
|
||||
amp = 10.0 ** (AMP_LOG_MIN + AMP_LOG_SPAN * ((s * PHI_FRAC) % 1.0))
|
||||
step = two_pi * freq / SAMPLE_RATE
|
||||
for k in range(end - start):
|
||||
i = start + k
|
||||
x = amp * math.sin(phase + step * k)
|
||||
x += BG_AMP * math.sin(two_pi * BG_HZ * i / SAMPLE_RATE)
|
||||
if x > 1.0:
|
||||
x = 1.0
|
||||
elif x < -1.0:
|
||||
x = -1.0
|
||||
out[i] = int(math.floor(x * 32767.0 + 0.5))
|
||||
phase = (phase + step * (end - start)) % two_pi
|
||||
return out
|
||||
|
||||
|
||||
def write_wav(path, samples):
|
||||
data = struct.pack("<%dh" % len(samples), *samples)
|
||||
hdr = b"RIFF" + struct.pack("<I", 36 + len(data)) + b"WAVE"
|
||||
hdr += b"fmt " + struct.pack("<IHHIIHH", 16, 1, 1, SAMPLE_RATE,
|
||||
SAMPLE_RATE * 2, 2, 16)
|
||||
hdr += b"data" + struct.pack("<I", len(data))
|
||||
with open(path, "wb") as fh:
|
||||
fh.write(hdr + data)
|
||||
|
||||
|
||||
def main():
|
||||
here = os.path.dirname(os.path.abspath(__file__))
|
||||
samples = generate()
|
||||
wav = os.path.join(here, OUT_WAV)
|
||||
write_wav(wav, samples)
|
||||
if "--wav" in sys.argv:
|
||||
print("wrote", wav)
|
||||
return
|
||||
flac = os.path.join(here, OUT_FLAC)
|
||||
# -compression_level 12 is deterministic for a given libFLAC/ffmpeg build;
|
||||
# only the container bytes vary, never the decoded PCM.
|
||||
subprocess.run(["ffmpeg", "-nostdin", "-v", "error", "-y", "-i", wav,
|
||||
"-c:a", "flac", "-compression_level", "12", flac],
|
||||
check=True)
|
||||
os.remove(wav)
|
||||
print("wrote", flac)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
-60
@@ -1,60 +0,0 @@
|
||||
#!/bin/sh
|
||||
#
|
||||
# Regenerate superhero_offset_200s.flac — the real-audio fixture behind VR-014, the
|
||||
# audio-signature offset-recovery validation.
|
||||
#
|
||||
# sh make_offset_fixture.sh /path/to/clips
|
||||
#
|
||||
# Why real audio and not a second synthetic tone: jray_audio_v1_tone.flac pins
|
||||
# the *arithmetic* (IR-005) and is deliberately built so every band and every
|
||||
# energy class appears. It cannot answer the question VR-014 asks — whether the
|
||||
# peak-bin sequence of ordinary film audio is distinctive enough that sliding
|
||||
# one signature against another finds the true alignment and only the true
|
||||
# alignment. Tones are pathologically easy for that; dialogue and score are not.
|
||||
#
|
||||
# Source: scene clips from SuperHero (TRECVID DVU development set), the corpus
|
||||
# this repo already uses for the replay fixtures — tests/fixtures/dumps/superhero.h5
|
||||
# are dumps of these same clips. Each is under the 120 s window on its own
|
||||
# (29-77 s), so they are concatenated in scene order to make a source long
|
||||
# enough that a 120 s window can slide inside it.
|
||||
#
|
||||
# 200 s is chosen, not arbitrary: the window is 120 s and the match search is
|
||||
# capped at +/-600 frames (~55.7 s), so a source of 120 + 56 s is the shortest
|
||||
# one that can place two windows at the edge of the cap. The 200 s here leaves
|
||||
# room to go past it as well, which is what lets the test check that an
|
||||
# out-of-range offset is declined rather than guessed.
|
||||
#
|
||||
# Encoded mono at 11025 Hz, 16-bit, which is exactly what the signature decodes
|
||||
# to anyway. That keeps a 200 s fixture at ~2.4 MB instead of ~20 MB, and makes
|
||||
# every trim below sample-exact — the test measures offset recovery, not the
|
||||
# resampler, which tests/test_audio_signature.cpp already covers (UT-103).
|
||||
#
|
||||
# FLAC because it is lossless: the decoded PCM is the same on every machine, so
|
||||
# a signature computed from this file is reproducible. A lossy fixture would
|
||||
# make the measurement depend on the decoder version.
|
||||
#
|
||||
# sha256 of the committed file:
|
||||
# 4a952e46a090a9acd9eae56996250ec03e08e0d04ee139ac0a42f1690a536c83
|
||||
# A regenerated file that hashes differently means the source clips or the
|
||||
# encoder changed, and VR-014's recorded numbers should be re-measured — the
|
||||
# offsets will still be exact, but the scores are this audio's.
|
||||
|
||||
set -eu
|
||||
|
||||
CLIPS="${1:-../../../../hero}"
|
||||
OUT="$(dirname "$0")/superhero_offset_200s.flac"
|
||||
LIST="$(mktemp)"
|
||||
trap 'rm -f "$LIST"' EXIT
|
||||
|
||||
for scene in 13 27 28 31 46; do
|
||||
clip="$CLIPS/SuperHero-$scene.webm"
|
||||
[ -f "$clip" ] || { echo "missing clip: $clip" >&2; exit 1; }
|
||||
echo "file '$(cd "$(dirname "$clip")" && pwd)/$(basename "$clip")'" >> "$LIST"
|
||||
done
|
||||
|
||||
ffmpeg -nostdin -v error -y -f concat -safe 0 -i "$LIST" \
|
||||
-vn -t 200 -ac 1 -ar 11025 -sample_fmt s16 \
|
||||
-c:a flac -compression_level 12 "$OUT"
|
||||
|
||||
echo "wrote $OUT"
|
||||
sha256sum "$OUT" 2>/dev/null || shasum -a 256 "$OUT"
|
||||
@@ -1,17 +0,0 @@
|
||||
# Replay fixtures are distributed as artifacts, not through git.
|
||||
#
|
||||
# They are large (superhero.h5 is ~9 MB) and regenerating one needs the film,
|
||||
# the models and a GPU — none of which CI has. So they live in the Gitea
|
||||
# generic package registry and are fetched on demand:
|
||||
#
|
||||
# scripts/artifacts/pull_artifacts.sh replay-fixtures [version]
|
||||
# scripts/artifacts/push_artifacts.sh replay-fixtures
|
||||
#
|
||||
# The gallery ships alongside the dumps deliberately: a dump only replays
|
||||
# meaningfully against the gallery it was produced with.
|
||||
#
|
||||
# bali_*.h5 predate this and remain tracked; do not add more to git.
|
||||
superhero.h5
|
||||
hero66.h5
|
||||
gt.json
|
||||
scene_bounds.json
|
||||
@@ -1,359 +0,0 @@
|
||||
// Unit tests for the JRay v1 audio signature (src/audio_signature.*).
|
||||
//
|
||||
/// TRACES: UT-101, UT-102, UT-103, UT-104 | IR-004, IR-005, IR-007, IR-008
|
||||
//
|
||||
// The headline test is the golden vector: a deterministic tone fixture checked
|
||||
// into tests/fixtures/audio/ together with the signature it must produce. That
|
||||
// fixture is the artefact shared with the jRay plugin repo, and it is what
|
||||
// makes "both producers agree bit-for-bit" a checked claim rather than an
|
||||
// assertion (IR-005).
|
||||
//
|
||||
// GPU-free, model-free, no network. Pure CPU DSP plus an FFmpeg decode of a
|
||||
// 585 KB file — which is precisely why this is the right cross-repo check: it
|
||||
// runs anywhere, including the N100 CI host.
|
||||
#include <catch2/catch_test_macros.hpp>
|
||||
#include <catch2/matchers/catch_matchers_floating_point.hpp>
|
||||
|
||||
#include "audio_signature.hpp"
|
||||
|
||||
#include <nlohmann/json.hpp>
|
||||
|
||||
#include <cmath>
|
||||
#include <cstdint>
|
||||
#include <filesystem>
|
||||
#include <fstream>
|
||||
#include <stdexcept>
|
||||
#include <string>
|
||||
#include <vector>
|
||||
|
||||
using namespace sae::audio;
|
||||
namespace fs = std::filesystem;
|
||||
|
||||
namespace {
|
||||
|
||||
const std::string kFixtureDir = SAE_TEST_FIXTURES_DIR "/audio";
|
||||
const std::string kGoldenPath = kFixtureDir + "/jray_audio_v1_golden.json";
|
||||
const std::string kMediaPath = kFixtureDir + "/jray_audio_v1_tone.flac";
|
||||
|
||||
const nlohmann::json& golden() {
|
||||
static const nlohmann::json j = [] {
|
||||
std::ifstream in(kGoldenPath);
|
||||
if (!in.good())
|
||||
throw std::runtime_error("golden fixture not found: " + kGoldenPath);
|
||||
nlohmann::json parsed;
|
||||
in >> parsed;
|
||||
return parsed;
|
||||
}();
|
||||
return j;
|
||||
}
|
||||
|
||||
std::uint64_t hex64(const std::string& s) {
|
||||
return std::stoull(s, nullptr, 16);
|
||||
}
|
||||
|
||||
// The fixture is 120 s of audio: decoding and signing it is the expensive part
|
||||
// of this file, so both results are computed once and shared. Every test below
|
||||
// still asserts against the on-disk golden values, not against each other.
|
||||
const std::optional<std::vector<float>>& fixture_window() {
|
||||
static const std::optional<std::vector<float>> w = decode_centre_window(kMediaPath);
|
||||
return w;
|
||||
}
|
||||
|
||||
const std::optional<std::string>& fixture_signature() {
|
||||
static const std::optional<std::string> s = compute_signature(kMediaPath);
|
||||
return s;
|
||||
}
|
||||
|
||||
// ── Minimal WAV writer, so the short-media and resample cases need no fixture ─
|
||||
// 16-bit PCM, interleaved.
|
||||
struct TempWav {
|
||||
fs::path path;
|
||||
explicit TempWav(const std::string& name)
|
||||
: path(fs::temp_directory_path() / ("sae_audio_test_" + name + ".wav")) {}
|
||||
~TempWav() { std::error_code ec; fs::remove(path, ec); }
|
||||
|
||||
void write(const std::vector<std::int16_t>& samples, int rate, int channels) const {
|
||||
const std::uint32_t bytes = static_cast<std::uint32_t>(samples.size() * 2);
|
||||
const std::uint32_t byte_rate = static_cast<std::uint32_t>(rate * channels * 2);
|
||||
std::ofstream out(path, std::ios::binary);
|
||||
auto u32 = [&](std::uint32_t v) { out.write(reinterpret_cast<const char*>(&v), 4); };
|
||||
auto u16 = [&](std::uint16_t v) { out.write(reinterpret_cast<const char*>(&v), 2); };
|
||||
out.write("RIFF", 4); u32(36 + bytes); out.write("WAVE", 4);
|
||||
out.write("fmt ", 4); u32(16); u16(1); u16(static_cast<std::uint16_t>(channels));
|
||||
u32(static_cast<std::uint32_t>(rate)); u32(byte_rate);
|
||||
u16(static_cast<std::uint16_t>(channels * 2)); u16(16);
|
||||
out.write("data", 4); u32(bytes);
|
||||
out.write(reinterpret_cast<const char*>(samples.data()), bytes);
|
||||
}
|
||||
};
|
||||
|
||||
// A plain 1 kHz tone, mono, at the signature's own rate.
|
||||
std::vector<std::int16_t> tone(double seconds, int rate = kSampleRate) {
|
||||
const std::size_t n = static_cast<std::size_t>(std::llround(seconds * rate));
|
||||
std::vector<std::int16_t> s(n);
|
||||
for (std::size_t i = 0; i < n; ++i)
|
||||
s[i] = static_cast<std::int16_t>(std::llround(
|
||||
20000.0 * std::sin(2.0 * 3.14159265358979323846 * 1000.0 * double(i) / rate)));
|
||||
return s;
|
||||
}
|
||||
|
||||
std::vector<std::uint8_t> base64_decode(const std::string& in) {
|
||||
auto val = [](char c) -> int {
|
||||
if (c >= 'A' && c <= 'Z') return c - 'A';
|
||||
if (c >= 'a' && c <= 'z') return c - 'a' + 26;
|
||||
if (c >= '0' && c <= '9') return c - '0' + 52;
|
||||
if (c == '+') return 62;
|
||||
if (c == '/') return 63;
|
||||
return -1;
|
||||
};
|
||||
std::vector<std::uint8_t> out;
|
||||
std::uint32_t acc = 0;
|
||||
int bits = 0;
|
||||
for (char c : in) {
|
||||
const int v = val(c);
|
||||
if (v < 0) continue; // '=' padding
|
||||
acc = (acc << 6) | static_cast<std::uint32_t>(v);
|
||||
bits += 6;
|
||||
if (bits >= 8) {
|
||||
bits -= 8;
|
||||
out.push_back(static_cast<std::uint8_t>((acc >> bits) & 0xFF));
|
||||
}
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
} // namespace
|
||||
|
||||
// ── UT-101 — the golden vector ──────────────────────────────────────────────
|
||||
|
||||
/// TRACES: UT-101 | IR-004, IR-005, IR-008
|
||||
TEST_CASE("signature of the golden fixture matches the recorded value exactly",
|
||||
"[audio_signature][golden]") {
|
||||
REQUIRE(fs::exists(kMediaPath));
|
||||
const std::optional<std::string>& sig = fixture_signature();
|
||||
REQUIRE(sig.has_value());
|
||||
CHECK(*sig == golden()["signature"].get<std::string>());
|
||||
}
|
||||
|
||||
/// TRACES: UT-101 | IR-005
|
||||
TEST_CASE("decoded centre window matches the recorded PCM checksum",
|
||||
"[audio_signature][golden]") {
|
||||
// Checked separately from the signature so a codec-level difference is
|
||||
// distinguishable from a DSP-level one: if this passes and the signature
|
||||
// test fails, the DSP diverged; if this fails, the decode did.
|
||||
const std::optional<std::vector<float>>& mono = fixture_window();
|
||||
REQUIRE(mono.has_value());
|
||||
CHECK(mono->size() == golden()["decoded_window"]["samples"].get<std::size_t>());
|
||||
CHECK(fnv1a64(mono->data(), mono->size() * sizeof(float)) ==
|
||||
hex64(golden()["decoded_window"]["f32le_fnv1a64"].get<std::string>()));
|
||||
}
|
||||
|
||||
/// TRACES: UT-101 | IR-004
|
||||
TEST_CASE("log-spaced band table matches the recorded one", "[audio_signature][golden]") {
|
||||
// The band->FFT-bin table is the part of the construction most likely to
|
||||
// drift between two implementations, so it is pinned independently of the
|
||||
// signature it produces.
|
||||
const auto& tbl = band_fft_bins();
|
||||
const auto& want = golden()["band_fft_bins"];
|
||||
REQUIRE(want.size() == tbl.size());
|
||||
for (std::size_t b = 0; b < tbl.size(); ++b) {
|
||||
CHECK(tbl[b].first == want[b][0].get<int>());
|
||||
CHECK(tbl[b].second == want[b][1].get<int>());
|
||||
CHECK(tbl[b].second > tbl[b].first); // no empty band
|
||||
if (b) CHECK(tbl[b].first == tbl[b - 1].second); // contiguous, no overlap
|
||||
}
|
||||
}
|
||||
|
||||
/// TRACES: UT-101 | IR-004, IR-008
|
||||
TEST_CASE("signature is well-formed: v1 prefix, 1288 frames, structural bytes",
|
||||
"[audio_signature][golden]") {
|
||||
const std::optional<std::string>& sig = fixture_signature();
|
||||
REQUIRE(sig.has_value());
|
||||
|
||||
// IR-008 — the signature carries its own version, separate from
|
||||
// schema_version, so a future DSP change is detectable rather than silently
|
||||
// producing non-matching signatures.
|
||||
REQUIRE(sig->rfind(kVersionPrefix, 0) == 0);
|
||||
|
||||
const std::vector<std::uint8_t> bytes = base64_decode(sig->substr(3));
|
||||
CHECK(bytes.size() == kExpectedFrames);
|
||||
CHECK(bytes.size() == golden()["frame_count"].get<std::size_t>());
|
||||
|
||||
// The server validates this structure on upload (server SPEC §3): each byte
|
||||
// is a 5-bit band index plus a 2-bit energy class, so bit 7 is always clear
|
||||
// and arbitrary bytes are invalid. That is what keeps the field from being
|
||||
// a payload channel.
|
||||
bool bands_seen[kNumBands] = {};
|
||||
bool classes_seen[4] = {};
|
||||
for (std::uint8_t b : bytes) {
|
||||
REQUIRE((b & 0x80) == 0);
|
||||
bands_seen[(b >> 2) & 0x1F] = true;
|
||||
classes_seen[b & 0x03] = true;
|
||||
}
|
||||
// The fixture is built to exercise the whole output alphabet — if it ever
|
||||
// stops doing so, the golden vector has become a weaker check than it looks.
|
||||
for (bool seen : bands_seen) CHECK(seen);
|
||||
for (bool seen : classes_seen) CHECK(seen);
|
||||
}
|
||||
|
||||
// ── UT-102 — IR-007, media shorter than the window ──────────────────────────
|
||||
|
||||
/// TRACES: UT-102 | IR-007
|
||||
TEST_CASE("media shorter than 120 s emits no signature", "[audio_signature][short]") {
|
||||
// The window runtime/2 ± 60 s underflows, so there is no signature and no
|
||||
// sync offset downstream. Both producers must apply the identical rule or
|
||||
// they diverge on exactly the short items most likely to be misidentified.
|
||||
SECTION("30 s") {
|
||||
TempWav w("short30");
|
||||
w.write(tone(30.0), kSampleRate, 1);
|
||||
CHECK_FALSE(compute_signature(w.path.string()).has_value());
|
||||
CHECK_FALSE(decode_centre_window(w.path.string()).has_value());
|
||||
}
|
||||
SECTION("just under the boundary") {
|
||||
TempWav w("short11999");
|
||||
w.write(tone(119.99), kSampleRate, 1);
|
||||
CHECK_FALSE(compute_signature(w.path.string()).has_value());
|
||||
}
|
||||
}
|
||||
|
||||
/// TRACES: UT-102 | IR-007
|
||||
TEST_CASE("media of exactly 120 s emits a full-length signature",
|
||||
"[audio_signature][short]") {
|
||||
TempWav w("exact120");
|
||||
w.write(tone(120.0), kSampleRate, 1);
|
||||
const std::optional<std::string> sig = compute_signature(w.path.string());
|
||||
REQUIRE(sig.has_value());
|
||||
CHECK(base64_decode(sig->substr(3)).size() == kExpectedFrames);
|
||||
}
|
||||
|
||||
/// TRACES: UT-102 | IR-007
|
||||
TEST_CASE("unreadable media degrades to no signature rather than failing",
|
||||
"[audio_signature][short]") {
|
||||
// UR-9 is an enhancement and must never be able to break a fetch.
|
||||
CHECK_FALSE(compute_signature("/nonexistent/definitely-not-here.mkv").has_value());
|
||||
}
|
||||
|
||||
/// TRACES: UT-102 | IR-004
|
||||
TEST_CASE("the window is taken from the centre, not the head",
|
||||
"[audio_signature][centre]") {
|
||||
// Sampling from the centre is the whole reason the construction avoids the
|
||||
// head and tail (logos, cold opens, credits), so it needs its own check:
|
||||
// wrap the fixture's own 120 s in 90 s of silence either side and the
|
||||
// signature of the 300 s file must be the golden value, byte for byte.
|
||||
// Nothing else pins the seek offset — a head-anchored window would pass
|
||||
// every other test in this file.
|
||||
const std::optional<std::vector<float>>& mono = fixture_window();
|
||||
REQUIRE(mono.has_value());
|
||||
|
||||
const std::size_t pad = 90 * kSampleRate;
|
||||
std::vector<std::int16_t> padded(pad * 2 + mono->size(), 0);
|
||||
for (std::size_t i = 0; i < mono->size(); ++i)
|
||||
padded[pad + i] = static_cast<std::int16_t>(std::llround(double((*mono)[i]) * 32768.0));
|
||||
|
||||
TempWav w("centred300");
|
||||
w.write(padded, kSampleRate, 1);
|
||||
|
||||
const std::optional<std::string> sig = compute_signature(w.path.string());
|
||||
REQUIRE(sig.has_value());
|
||||
CHECK(*sig == golden()["signature"].get<std::string>());
|
||||
}
|
||||
|
||||
// ── UT-103 — downmix and resample ───────────────────────────────────────────
|
||||
|
||||
/// TRACES: UT-103 | IR-004
|
||||
TEST_CASE("stereo, non-native sample rate yields the same peak-bin sequence",
|
||||
"[audio_signature][resample]") {
|
||||
// The golden fixture is already mono at 11025 Hz so the golden vector does
|
||||
// not depend on the resampler's version. This case exercises the path that
|
||||
// real media takes — downmix plus resample — by rebuilding the fixture's own
|
||||
// audio as 22050 Hz stereo and checking the peak bins survive it.
|
||||
const std::optional<std::vector<float>>& mono = fixture_window();
|
||||
REQUIRE(mono.has_value());
|
||||
|
||||
std::vector<std::int16_t> stereo;
|
||||
stereo.reserve(mono->size() * 4);
|
||||
for (float f : *mono) {
|
||||
const auto s = static_cast<std::int16_t>(std::llround(double(f) * 32768.0));
|
||||
stereo.push_back(s); stereo.push_back(s); // sample 1, L/R
|
||||
stereo.push_back(s); stereo.push_back(s); // sample 2 (zero-order hold)
|
||||
}
|
||||
TempWav w("stereo22050");
|
||||
w.write(stereo, 2 * kSampleRate, 2);
|
||||
|
||||
const std::optional<std::string> sig = compute_signature(w.path.string());
|
||||
REQUIRE(sig.has_value());
|
||||
|
||||
const std::vector<std::uint8_t> got = base64_decode(sig->substr(3));
|
||||
const std::vector<std::uint8_t> want =
|
||||
base64_decode(golden()["signature"].get<std::string>().substr(3));
|
||||
REQUIRE(got.size() == want.size());
|
||||
|
||||
std::size_t agree = 0;
|
||||
for (std::size_t i = 0; i < got.size(); ++i)
|
||||
agree += ((got[i] >> 2) == (want[i] >> 2)) ? 1 : 0;
|
||||
// The server treats ≥ 0.85 as the `audio` match tier; this path scores 1.0
|
||||
// in practice, and the margin is left for libswresample version drift.
|
||||
CHECK(double(agree) / double(got.size()) >= 0.85);
|
||||
}
|
||||
|
||||
// ── UT-104 — the pure DSP surface ───────────────────────────────────────────
|
||||
|
||||
/// TRACES: UT-104 | IR-004
|
||||
TEST_CASE("pack_frames uses whole frames only", "[audio_signature][dsp]") {
|
||||
CHECK(pack_frames(std::vector<float>(kFrameSize - 1, 0.f)).empty());
|
||||
CHECK(pack_frames(std::vector<float>(kFrameSize, 0.f)).size() == 1);
|
||||
CHECK(pack_frames(std::vector<float>(kFrameSize + kHopSize - 1, 0.f)).size() == 1);
|
||||
CHECK(pack_frames(std::vector<float>(kFrameSize + kHopSize, 0.f)).size() == 2);
|
||||
// The full 120 s window is 1288 frames — asserted as a constant rather than
|
||||
// by running the DSP over 1.3 M zeros, which is the same claim for free.
|
||||
CHECK(kWindowSamples == 1323000u);
|
||||
CHECK(kExpectedFrames == 1288u);
|
||||
CHECK_FALSE(signature_from_mono(std::vector<float>(kFrameSize - 1, 0.f)).has_value());
|
||||
}
|
||||
|
||||
/// TRACES: UT-104 | IR-004
|
||||
TEST_CASE("a pure tone lands in the band that contains it", "[audio_signature][dsp]") {
|
||||
// 1000 Hz sits in log-band floor(32 * log10(1000/300)) = 16.
|
||||
const int expect = static_cast<int>(std::floor(
|
||||
kNumBands * std::log10(1000.0 / kBandLoHz) / std::log10(kBandHiHz / kBandLoHz)));
|
||||
std::vector<float> mono(kWindowSamples / 100);
|
||||
for (std::size_t i = 0; i < mono.size(); ++i)
|
||||
mono[i] = static_cast<float>(0.5 * std::sin(
|
||||
2.0 * 3.14159265358979323846 * 1000.0 * double(i) / kSampleRate));
|
||||
const std::vector<std::uint8_t> packed = pack_frames(mono);
|
||||
REQUIRE_FALSE(packed.empty());
|
||||
for (std::uint8_t b : packed) CHECK(((b >> 2) & 0x1F) == expect);
|
||||
}
|
||||
|
||||
/// TRACES: UT-104 | IR-004
|
||||
TEST_CASE("signature is invariant to overall gain", "[audio_signature][dsp]") {
|
||||
// Loudness normalisation between two releases of the same cut must not
|
||||
// change the signature — that is why the energy class is relative.
|
||||
std::vector<float> a(kWindowSamples / 50);
|
||||
for (std::size_t i = 0; i < a.size(); ++i) {
|
||||
const double t = double(i) / kSampleRate;
|
||||
a[i] = static_cast<float>(0.4 * std::sin(2.0 * 3.14159265358979323846 * 640.0 * t) +
|
||||
0.2 * std::sin(2.0 * 3.14159265358979323846 * 1900.0 * t) *
|
||||
std::sin(2.0 * 3.14159265358979323846 * 0.7 * t));
|
||||
}
|
||||
std::vector<float> b(a.size());
|
||||
for (std::size_t i = 0; i < a.size(); ++i) b[i] = a[i] * 0.25f;
|
||||
CHECK(pack_frames(a) == pack_frames(b));
|
||||
}
|
||||
|
||||
/// TRACES: UT-104 | IR-004
|
||||
TEST_CASE("base64 encoder matches the standard alphabet and padding",
|
||||
"[audio_signature][dsp]") {
|
||||
auto enc = [](const std::string& s) {
|
||||
return base64_encode(reinterpret_cast<const std::uint8_t*>(s.data()), s.size());
|
||||
};
|
||||
CHECK(enc("") == "");
|
||||
CHECK(enc("f") == "Zg==");
|
||||
CHECK(enc("fo") == "Zm8=");
|
||||
CHECK(enc("foo") == "Zm9v");
|
||||
CHECK(enc("foob") == "Zm9vYg==");
|
||||
CHECK(enc("fooba") == "Zm9vYmE=");
|
||||
CHECK(enc("foobar") == "Zm9vYmFy");
|
||||
const std::uint8_t all[] = {0xFB, 0xFF, 0xBF}; // exercises '+' and '/'
|
||||
CHECK(base64_encode(all, 3) == "+/+/");
|
||||
}
|
||||
@@ -1,12 +1,6 @@
|
||||
// TRACES: AR-023 | SR-002
|
||||
//
|
||||
// Unit tests for gallery calibration: the sigmoid math, the pairwise fit on
|
||||
// separable data, and the in-memory hash-keyed cache (hit / stale / cold).
|
||||
// All pure, GPU-free, model-free.
|
||||
//
|
||||
// The three `[report]` cases at the bottom carry their own GR-003 tags: they
|
||||
// verify the build report, which is fitted from the same distributions but is a
|
||||
// separate requirement.
|
||||
#include <catch2/catch_test_macros.hpp>
|
||||
#include <catch2/matchers/catch_matchers_floating_point.hpp>
|
||||
|
||||
@@ -174,99 +168,3 @@ TEST_CASE("calibrate_gallery_cached treats hash=0 as always-recompute", "[calibr
|
||||
CHECK(recomputed);
|
||||
CHECK(cal.valid);
|
||||
}
|
||||
|
||||
// ── GR-003 — the build report ────────────────────────────────────────────────
|
||||
#include "gallery/gallery_report.hpp"
|
||||
|
||||
namespace {
|
||||
// A unit vector on one axis. Distinct axes are orthogonal, which is unrealistic
|
||||
// as a same-actor cluster but irrelevant here: these tests count actors, they do
|
||||
// not assess fit quality.
|
||||
Embedding unit_axis(int slot) {
|
||||
Embedding e{};
|
||||
e[slot % 512] = 1.0f;
|
||||
return e;
|
||||
}
|
||||
} // namespace
|
||||
|
||||
// TRACES: GR-003 | SR-001
|
||||
TEST_CASE("report surfaces actors that can never be recognised", "[report][GR-003]") {
|
||||
// An actor with no usable image is a silent recall ceiling: the pipeline
|
||||
// will never name them, and nothing in the gallery says why. This is the
|
||||
// single most useful number in the report.
|
||||
ActorGallery g;
|
||||
for (int a = 0; a < 3; ++a) {
|
||||
ActorGallery::Actor act;
|
||||
act.name = "actor" + std::to_string(a);
|
||||
if (a != 1) // actor1 gets nothing
|
||||
for (int i = 0; i < 6; ++i) act.embeddings.push_back(unit_axis(a * 10 + i));
|
||||
g.actors.push_back(std::move(act));
|
||||
}
|
||||
|
||||
std::vector<Embedding> flat;
|
||||
std::vector<int> flat_actor;
|
||||
for (int a = 0; a < static_cast<int>(g.actors.size()); ++a)
|
||||
for (const auto& e : g.actors[a].embeddings) { flat.push_back(e); flat_actor.push_back(a); }
|
||||
|
||||
GalleryCalibrationStats stats;
|
||||
GalleryCalibration cal = calibrate_gallery(flat, flat_actor, &stats);
|
||||
GalleryReport r = build_gallery_report(g, cal, stats);
|
||||
|
||||
// An actor present in the gallery with no embeddings is counted as
|
||||
// in-gallery but contributes nothing; the zero-usable list is populated
|
||||
// from the build audit, which a stored gallery cannot supply.
|
||||
CHECK(r.actors_in_gallery == 3);
|
||||
CHECK(r.actors[1].references == 0);
|
||||
}
|
||||
|
||||
// TRACES: GR-003 | SR-001
|
||||
TEST_CASE("report surfaces actors too thin to calibrate on", "[report][GR-003]") {
|
||||
// Below the positive-pair threshold an actor contributes nothing to the
|
||||
// intra-class side of the fit. They are not broken, so nothing complains —
|
||||
// they just quietly weaken every threshold downstream.
|
||||
ActorGallery g;
|
||||
for (int a = 0; a < 2; ++a) {
|
||||
ActorGallery::Actor act;
|
||||
act.name = "actor" + std::to_string(a);
|
||||
const int n = (a == 0) ? 6 : 2; // actor1 is under-referenced
|
||||
for (int i = 0; i < n; ++i) act.embeddings.push_back(unit_axis(a * 10 + i));
|
||||
g.actors.push_back(std::move(act));
|
||||
}
|
||||
|
||||
std::vector<Embedding> flat;
|
||||
std::vector<int> flat_actor;
|
||||
for (int a = 0; a < static_cast<int>(g.actors.size()); ++a)
|
||||
for (const auto& e : g.actors[a].embeddings) { flat.push_back(e); flat_actor.push_back(a); }
|
||||
|
||||
GalleryCalibrationStats stats;
|
||||
GalleryCalibration cal = calibrate_gallery(flat, flat_actor, &stats);
|
||||
GalleryReport r = build_gallery_report(g, cal, stats);
|
||||
|
||||
CHECK(r.actors_below_positive_threshold >= 1);
|
||||
}
|
||||
|
||||
// TRACES: GR-003 | SR-001
|
||||
TEST_CASE("report round-trips", "[report][GR-003]") {
|
||||
ActorGallery g;
|
||||
ActorGallery::Actor act;
|
||||
act.name = "solo";
|
||||
for (int i = 0; i < 6; ++i) act.embeddings.push_back(unit_axis(i));
|
||||
g.actors.push_back(std::move(act));
|
||||
|
||||
std::vector<Embedding> flat;
|
||||
std::vector<int> flat_actor;
|
||||
for (const auto& e : g.actors[0].embeddings) { flat.push_back(e); flat_actor.push_back(0); }
|
||||
|
||||
GalleryCalibrationStats stats;
|
||||
GalleryCalibration cal = calibrate_gallery(flat, flat_actor, &stats);
|
||||
GalleryReport r = build_gallery_report(g, cal, stats);
|
||||
|
||||
const std::string path = "/tmp/gr003_roundtrip.report.json";
|
||||
save_gallery_report(path, r);
|
||||
GalleryReport back = load_gallery_report(path);
|
||||
|
||||
CHECK(back.actors_in_gallery == r.actors_in_gallery);
|
||||
CHECK(back.actors_below_positive_threshold == r.actors_below_positive_threshold);
|
||||
CHECK(back.calib_a == r.calib_a);
|
||||
std::remove(path.c_str());
|
||||
}
|
||||
|
||||
+57
-125
@@ -1,28 +1,20 @@
|
||||
// TRACES: AR-007, AR-008 | SR-002
|
||||
//
|
||||
// Unit tests for FaceTrackerFunc (nodes/face_tracker_node.hpp): frame-to-frame
|
||||
// track linking and, crucially, cross-cut re-association. Pure, GPU-free,
|
||||
// model-free — drives the node's operator() with hand-built EmbeddedSceneFrames
|
||||
// and inspects the emitted track_ids.
|
||||
//
|
||||
// The behaviour under test: there is one track pool keyed on `last_seen`
|
||||
// (AR-008), so a face lost across a camera-angle change (Frame::is_cut) is an
|
||||
// ordinary association candidate rather than a parked track needing a revival
|
||||
// path — the raw-cosine `cut_revive_sim` that guarded that path is retired
|
||||
// (AR-024). On a cut the association weight drops to embedding-only (AR-007),
|
||||
// and IoU is deliberately driven to 0 across the cut (boxes moved) so only the
|
||||
// embedding path can re-link — exactly the scenario a cut creates.
|
||||
// The behaviour under test: on a camera-angle change (Frame::is_cut) the tracker
|
||||
// parks its tracks instead of destroying them, and revives a parked track_id
|
||||
// when a post-cut detection's raw last-frame-embedding cosine similarity clears
|
||||
// cut_revive_sim. IoU is deliberately driven to 0 across the cut (boxes moved) so
|
||||
// only the embedding path can re-link — exactly the scenario a cut creates.
|
||||
#include <catch2/catch_test_macros.hpp>
|
||||
|
||||
#include "config.hpp"
|
||||
#include "nodes/face_tracker_node.hpp"
|
||||
#include "types.hpp"
|
||||
#include "track_registry.hpp"
|
||||
#include "evidence_discount.hpp"
|
||||
|
||||
#include <algorithm>
|
||||
#include <cmath>
|
||||
#include <memory>
|
||||
|
||||
namespace {
|
||||
|
||||
@@ -63,145 +55,85 @@ EmbeddedSceneFrame frame(double t, float x, float y, const Embedding& emb,
|
||||
return ef;
|
||||
}
|
||||
|
||||
// Build a tracker over a fresh registry. The registry IS the tracker's state
|
||||
// now (AR-008), so a test constructs both together and can inspect either.
|
||||
struct Rig {
|
||||
std::shared_ptr<TrackRegistry> reg;
|
||||
FaceTrackerFunc ft;
|
||||
|
||||
explicit Rig(double extinction = 30.0, float assoc_min_prob = 0.5f)
|
||||
: reg(std::make_shared<TrackRegistry>(
|
||||
[extinction] {
|
||||
TrackRegistry::Config c;
|
||||
c.extinction_sec = extinction;
|
||||
return c;
|
||||
}(),
|
||||
EvidenceDiscounter([](float cos) { return std::max(0.f, cos); })))
|
||||
, ft([&] {
|
||||
Config c;
|
||||
c.track_assoc_min_prob = assoc_min_prob;
|
||||
return c;
|
||||
}(),
|
||||
reg,
|
||||
// Trivial calibration: cosine passed through as P(same). Real runs use
|
||||
// the fit belonging to the active embedder (AR-023/AR-024).
|
||||
[](float cos) { return std::max(0.f, cos); })
|
||||
{}
|
||||
|
||||
int track_of(EmbeddedSceneFrame f) { return ft(std::move(f)).track_ids[0]; }
|
||||
};
|
||||
Config tracker_cfg() {
|
||||
Config cfg;
|
||||
cfg.cut_revive_sim = 0.50f;
|
||||
cfg.cut_inactive_max_frames = 5;
|
||||
return cfg;
|
||||
}
|
||||
|
||||
} // namespace
|
||||
|
||||
// ── AR-008 — one pool, ordinary association ──────────────────────────────────
|
||||
TEST_CASE("track id is stable across ordinary frames", "[face_tracker][AR-008]") {
|
||||
Rig r;
|
||||
TEST_CASE("track id is stable across ordinary frames", "[face_tracker]") {
|
||||
FaceTrackerFunc ft(tracker_cfg());
|
||||
Embedding e = axis(0);
|
||||
int id0 = r.track_of(frame(0.0, 10, 10, e));
|
||||
int id1 = r.track_of(frame(1.0, 11, 10, e)); // overlaps → same track
|
||||
int id0 = ft(frame(0.0, 10, 10, e)).track_ids[0];
|
||||
int id1 = ft(frame(1.0, 11, 10, e)).track_ids[0]; // overlaps → same track
|
||||
CHECK(id0 >= 0);
|
||||
CHECK(id1 == id0);
|
||||
}
|
||||
|
||||
TEST_CASE("a face lost across a cut and re-associated is the SAME track",
|
||||
"[face_tracker][AR-008]") {
|
||||
// Previously this was a distinct "revival" path guarded by a raw-cosine
|
||||
// constant. There is no such path now: a dormant track is an ordinary
|
||||
// association candidate, and continuity falls out of the embedding match.
|
||||
Rig r;
|
||||
TEST_CASE("cut revives the same track id for a matching identity", "[face_tracker]") {
|
||||
FaceTrackerFunc ft(tracker_cfg());
|
||||
|
||||
// Pre-cut: establish a track for a person whose embedding is near-identical
|
||||
// across the cut (sim well above cut_revive_sim), but whose box jumps so IoU
|
||||
// is 0 — the ordinary spatial path cannot re-link it.
|
||||
Embedding pre = at_sim(0, 1, 0.99f);
|
||||
int id_pre = r.track_of(frame(0.0, 10, 10, pre));
|
||||
int id_pre = ft(frame(0.0, 10, 10, pre)).track_ids[0];
|
||||
REQUIRE(id_pre >= 0);
|
||||
|
||||
// Box jumps so IoU is zero — only the embedding can link it.
|
||||
Embedding post = at_sim(0, 1, 0.98f);
|
||||
CHECK(r.track_of(frame(1.0, 300, 300, post, /*is_cut=*/true)) == id_pre);
|
||||
Embedding post = at_sim(0, 1, 0.98f); // cos(diff) ≈ 0.9997 > 0.50
|
||||
auto out = ft(frame(1.0, 300, 300, post, /*is_cut=*/true));
|
||||
CHECK(out.track_ids[0] == id_pre); // revived, not a fresh id
|
||||
}
|
||||
|
||||
TEST_CASE("a cut starts a fresh track when identity does not match",
|
||||
"[face_tracker][AR-008]") {
|
||||
Rig r;
|
||||
int id_pre = r.track_of(frame(0.0, 10, 10, axis(0)));
|
||||
TEST_CASE("cut starts a fresh track when identity does not match", "[face_tracker]") {
|
||||
FaceTrackerFunc ft(tracker_cfg());
|
||||
|
||||
int id_pre = ft(frame(0.0, 10, 10, axis(0))).track_ids[0];
|
||||
REQUIRE(id_pre >= 0);
|
||||
|
||||
// Orthogonal embedding and disjoint box: nothing links them.
|
||||
int id_post = r.track_of(frame(1.0, 300, 300, axis(5), /*is_cut=*/true));
|
||||
CHECK(id_post != id_pre);
|
||||
CHECK(id_post >= 0);
|
||||
// Post-cut face is orthogonal (sim 0 < cut_revive_sim) and spatially disjoint
|
||||
// → no revival, brand-new id.
|
||||
auto out = ft(frame(1.0, 300, 300, axis(5), /*is_cut=*/true));
|
||||
CHECK(out.track_ids[0] != id_pre);
|
||||
CHECK(out.track_ids[0] >= 0);
|
||||
}
|
||||
|
||||
// ── AR-007 — a cut makes association ignore position ─────────────────────────
|
||||
TEST_CASE("on a cut, identity follows the embedding rather than the box",
|
||||
"[face_tracker][AR-007]") {
|
||||
// Two people swap screen positions across a cut while keeping their faces.
|
||||
// If IoU still carried weight the ids would follow the boxes and swap; with
|
||||
// alpha driven to embedding-only on a cut, they must follow the faces.
|
||||
Rig r;
|
||||
|
||||
Embedding a = at_sim(0, 1, 0.99f);
|
||||
Embedding b = at_sim(2, 3, 0.99f);
|
||||
|
||||
EmbeddedSceneFrame f0;
|
||||
f0.source.timestamp_sec = 0.0;
|
||||
f0.faces = {face_at(10, 10), face_at(300, 300)};
|
||||
f0.crops = {cv::Mat(), cv::Mat()};
|
||||
f0.embeddings = {a, b};
|
||||
auto out0 = r.ft(std::move(f0));
|
||||
const int id_a = out0.track_ids[0];
|
||||
const int id_b = out0.track_ids[1];
|
||||
REQUIRE(id_a >= 0);
|
||||
REQUIRE(id_b >= 0);
|
||||
REQUIRE(id_a != id_b);
|
||||
|
||||
// Same two people, positions exchanged, on a cut frame.
|
||||
EmbeddedSceneFrame f1;
|
||||
f1.source.timestamp_sec = 1.0;
|
||||
f1.source.is_cut = true;
|
||||
f1.faces = {face_at(300, 300), face_at(10, 10)};
|
||||
f1.crops = {cv::Mat(), cv::Mat()};
|
||||
f1.embeddings = {a, b};
|
||||
auto out1 = r.ft(std::move(f1));
|
||||
|
||||
CHECK(out1.track_ids[0] == id_a); // A kept its id despite moving to B's box
|
||||
CHECK(out1.track_ids[1] == id_b);
|
||||
}
|
||||
|
||||
// ── AR-013 — extinction replaces the parked-pool frame counter ───────────────
|
||||
TEST_CASE("a track past the extinction window is gone, not revived",
|
||||
"[face_tracker][AR-013]") {
|
||||
// The old design aged a parked pool in frames, which silently changed
|
||||
// meaning with sample_fps. Extinction is in seconds and lives in the
|
||||
// registry, so the tracker no longer counts anything.
|
||||
Rig r(/*extinction=*/2.0);
|
||||
TEST_CASE("parked track expires after cut_inactive_max_frames", "[face_tracker]") {
|
||||
Config cfg = tracker_cfg();
|
||||
cfg.cut_inactive_max_frames = 2;
|
||||
FaceTrackerFunc ft(cfg);
|
||||
|
||||
Embedding person = at_sim(0, 1, 0.99f);
|
||||
int id_pre = r.track_of(frame(0.0, 10, 10, person));
|
||||
int id_pre = ft(frame(0.0, 10, 10, person)).track_ids[0];
|
||||
REQUIRE(id_pre >= 0);
|
||||
|
||||
// Unrelated faces elsewhere while the clock runs well past extinction.
|
||||
r.track_of(frame(1.0, 300, 300, axis(7), /*is_cut=*/true));
|
||||
r.track_of(frame(10.0, 300, 300, axis(7)));
|
||||
// Cut with an unrelated face parks id_pre; then let the pool age past its
|
||||
// limit with more unrelated, spatially-disjoint faces (each ages the pool by
|
||||
// one). By the time the person returns, id_pre must be gone.
|
||||
ft(frame(1.0, 300, 300, axis(7), /*is_cut=*/true)); // park (age 1)
|
||||
ft(frame(2.0, 300, 300, axis(7))); // age 2
|
||||
ft(frame(3.0, 300, 300, axis(7))); // age 3 → id_pre dropped
|
||||
|
||||
CHECK(r.track_of(frame(11.0, 10, 10, person)) != id_pre);
|
||||
auto out = ft(frame(4.0, 10, 10, person)); // same identity returns
|
||||
CHECK(out.track_ids[0] != id_pre); // too late — fresh id
|
||||
}
|
||||
|
||||
TEST_CASE("a track within the extinction window is still a candidate",
|
||||
"[face_tracker][AR-013]") {
|
||||
Rig r(/*extinction=*/30.0);
|
||||
|
||||
TEST_CASE("eof clears active and parked tracks", "[face_tracker]") {
|
||||
FaceTrackerFunc ft(tracker_cfg());
|
||||
Embedding person = at_sim(0, 1, 0.99f);
|
||||
int id_pre = r.track_of(frame(0.0, 10, 10, person));
|
||||
r.track_of(frame(1.0, 300, 300, axis(7), /*is_cut=*/true));
|
||||
int id_pre = ft(frame(0.0, 10, 10, person)).track_ids[0];
|
||||
ft(frame(1.0, 300, 300, axis(7), /*is_cut=*/true)); // park id_pre
|
||||
|
||||
// Back inside the window: the same person continues the same track, so the
|
||||
// gap is absorbed into one window rather than splitting it.
|
||||
CHECK(r.track_of(frame(3.0, 10, 10, person)) == id_pre);
|
||||
}
|
||||
|
||||
TEST_CASE("eof is forwarded", "[face_tracker]") {
|
||||
Rig r;
|
||||
EmbeddedSceneFrame eof;
|
||||
eof.source.eof = true;
|
||||
CHECK(r.ft(std::move(eof)).source.eof);
|
||||
auto out = ft(std::move(eof));
|
||||
CHECK(out.source.eof);
|
||||
|
||||
// After eof the pools are empty: the returning identity must get a fresh id,
|
||||
// not the parked one.
|
||||
auto out2 = ft(frame(2.0, 10, 10, person));
|
||||
CHECK(out2.track_ids[0] != id_pre);
|
||||
}
|
||||
|
||||
+2
-107
@@ -1,8 +1,6 @@
|
||||
// TRACES: AR-005, AR-030 | SR-002
|
||||
//
|
||||
// Unit tests for the geometric/numeric helpers in types.hpp and face_utils.hpp:
|
||||
// cosine_similarity, the ArcFace 5-point alignment transform, and the alignment
|
||||
// residual that AR-030 reads as its visibility measure. GPU-free, model-free.
|
||||
// cosine_similarity and the ArcFace 5-point alignment transform. GPU-free,
|
||||
// model-free.
|
||||
#include <catch2/catch_test_macros.hpp>
|
||||
#include <catch2/matchers/catch_matchers_floating_point.hpp>
|
||||
|
||||
@@ -76,106 +74,3 @@ TEST_CASE("align_face returns empty on degenerate (collinear) landmarks", "[face
|
||||
cv::Mat crop = align_face(img, lm);
|
||||
CHECK(crop.empty());
|
||||
}
|
||||
|
||||
// ── AR-030: the alignment residual as a visibility measure ────────────────────
|
||||
// These assert the *properties* the measure is relied on for, not a magic value.
|
||||
// Each would fail under a RANSAC fit, which buys a small residual by discarding
|
||||
// the very landmarks that carry the signal.
|
||||
|
||||
namespace {
|
||||
|
||||
std::array<cv::Point2f, 5> canonical() {
|
||||
std::array<cv::Point2f, 5> lm;
|
||||
for (int i = 0; i < 5; ++i) lm[i] = {kArcFaceRef[i][0], kArcFaceRef[i][1]};
|
||||
return lm;
|
||||
}
|
||||
|
||||
// Rotate by `deg` in-plane, scale uniformly, translate — i.e. exactly the 4 DoF
|
||||
// the similarity transform models.
|
||||
std::array<cv::Point2f, 5> similarity(const std::array<cv::Point2f, 5>& in,
|
||||
float deg, float s, float tx, float ty) {
|
||||
const float r = deg * 3.14159265358979f / 180.f;
|
||||
const float c = std::cos(r), sn = std::sin(r);
|
||||
std::array<cv::Point2f, 5> out;
|
||||
for (int i = 0; i < 5; ++i)
|
||||
out[i] = {s * (c * in[i].x - sn * in[i].y) + tx,
|
||||
s * (sn * in[i].x + c * in[i].y) + ty};
|
||||
return out;
|
||||
}
|
||||
|
||||
// Squash x about the centroid by `k`: the anisotropic deformation an out-of-plane
|
||||
// yaw produces, and the one a similarity provably cannot absorb.
|
||||
std::array<cv::Point2f, 5> foreshorten(const std::array<cv::Point2f, 5>& in, float k) {
|
||||
float cx = 0.f;
|
||||
for (const auto& p : in) cx += p.x;
|
||||
cx /= 5.f;
|
||||
std::array<cv::Point2f, 5> out = in;
|
||||
for (auto& p : out) p.x = cx + (p.x - cx) * k;
|
||||
return out;
|
||||
}
|
||||
|
||||
} // namespace
|
||||
|
||||
TEST_CASE("residual is zero for a face in canonical pose", "[face_utils][AR-030]") {
|
||||
const Alignment a = estimate_alignment(canonical());
|
||||
REQUIRE(a.ok);
|
||||
CHECK_THAT(a.residual, WithinAbs(0.0f, 1e-3f));
|
||||
}
|
||||
|
||||
TEST_CASE("residual ignores in-plane roll, scale and translation", "[face_utils][AR-030]") {
|
||||
// The structural claim behind AR-030: the fit absorbs all four similarity
|
||||
// DoF exactly, so what remains is only the deformation a similarity cannot
|
||||
// explain. A rolled head must not read as a turned one.
|
||||
for (float deg : {-40.f, -12.f, 0.f, 17.f, 65.f}) {
|
||||
const Alignment a = estimate_alignment(similarity(canonical(), deg, 3.5f, 220.f, -40.f));
|
||||
REQUIRE(a.ok);
|
||||
CHECK_THAT(a.residual, WithinAbs(0.0f, 1e-3f));
|
||||
}
|
||||
}
|
||||
|
||||
TEST_CASE("residual rises monotonically with foreshortening", "[face_utils][AR-030]") {
|
||||
float prev = -1.f;
|
||||
for (float k : {1.0f, 0.9f, 0.75f, 0.5f, 0.3f}) {
|
||||
const Alignment a = estimate_alignment(foreshorten(canonical(), k));
|
||||
REQUIRE(a.ok);
|
||||
CHECK(a.residual > prev);
|
||||
prev = a.residual;
|
||||
}
|
||||
}
|
||||
|
||||
TEST_CASE("residual is independent of face size", "[face_utils][AR-030]") {
|
||||
// The measure must not silently re-express face size — that is AR-002's job,
|
||||
// and double-counting it would make a small frontal face look occluded.
|
||||
// Same deformation, two very different face sizes, one answer.
|
||||
const auto small = similarity(foreshorten(canonical(), 0.7f), 20.f, 1.0f, 0.f, 0.f);
|
||||
const auto large = similarity(foreshorten(canonical(), 0.7f), 20.f, 12.0f, 500.f, 300.f);
|
||||
|
||||
const Alignment a = estimate_alignment(small);
|
||||
const Alignment b = estimate_alignment(large);
|
||||
REQUIRE(a.ok);
|
||||
REQUIRE(b.ok);
|
||||
CHECK_THAT(b.residual, WithinAbs(a.residual, 1e-2f));
|
||||
}
|
||||
|
||||
TEST_CASE("the fit never mirrors the face", "[face_utils][AR-030]") {
|
||||
// SVD will happily return an orientation-reversing solution; a similarity
|
||||
// transform may rotate but never reflect. Without the determinant guard a
|
||||
// mirrored landmark set fits "perfectly" as a reflection.
|
||||
const auto mirrored = foreshorten(canonical(), -1.f);
|
||||
const Alignment a = estimate_alignment(mirrored);
|
||||
REQUIRE(a.ok);
|
||||
|
||||
const double det = a.M.at<double>(0,0) * a.M.at<double>(1,1)
|
||||
- a.M.at<double>(0,1) * a.M.at<double>(1,0);
|
||||
CHECK(det > 0.0);
|
||||
CHECK(a.residual > 1.0f); // and the mirroring shows up as misfit
|
||||
}
|
||||
|
||||
TEST_CASE("degenerate landmarks report not-ok rather than a residual", "[face_utils][AR-030]") {
|
||||
std::array<cv::Point2f, 5> lm;
|
||||
for (auto& p : lm) p = {50.f, 50.f};
|
||||
|
||||
const Alignment a = estimate_alignment(lm);
|
||||
CHECK_FALSE(a.ok);
|
||||
CHECK(a.M.empty());
|
||||
}
|
||||
|
||||
@@ -1,17 +1,13 @@
|
||||
// Unit tests for gallery (de)serialisation: HDF5 round-trip fidelity (the only
|
||||
// format save_gallery writes), legacy JSON read back-compat (optional field
|
||||
// defaults, the legacy "jellyfin_person_id" fallback), and the GR-004 embedder
|
||||
// stamp. GPU-free, model-free — the stamp tests exercise the comparison logic
|
||||
// with synthetic stamps and never load an ONNX, so they run on CI's Intel N100.
|
||||
// defaults, the legacy "jellyfin_person_id" fallback). GPU-free, model-free.
|
||||
#include <catch2/catch_test_macros.hpp>
|
||||
|
||||
#include "gallery/embedder_stamp.hpp"
|
||||
#include "gallery/gallery_store.hpp"
|
||||
#include "types.hpp"
|
||||
|
||||
#include <cstdio>
|
||||
#include <fstream>
|
||||
#include <stdexcept>
|
||||
#include <string>
|
||||
|
||||
#include <nlohmann/json.hpp>
|
||||
@@ -31,22 +27,6 @@ Embedding make_embedding(float base) {
|
||||
return e;
|
||||
}
|
||||
|
||||
// A stamp built by hand — no ONNX is read, so these tests never need a model.
|
||||
EmbedderStamp stamp(const std::string& name, const std::string& sha, int32_t dim = 512) {
|
||||
EmbedderStamp s;
|
||||
s.model_name = name;
|
||||
s.model_sha256 = sha;
|
||||
s.embed_dim = dim;
|
||||
return s;
|
||||
}
|
||||
|
||||
const std::string kShaA(64, 'a');
|
||||
const std::string kShaB(64, 'b');
|
||||
|
||||
bool mentions(const std::string& haystack, const std::string& needle) {
|
||||
return haystack.find(needle) != std::string::npos;
|
||||
}
|
||||
|
||||
} // namespace
|
||||
|
||||
TEST_CASE("gallery save/load round-trips actors and embeddings", "[gallery]") {
|
||||
@@ -173,233 +153,3 @@ TEST_CASE("load_gallery reads the legacy jellyfin_person_id key", "[gallery]") {
|
||||
TEST_CASE("load_gallery throws on a missing file", "[gallery]") {
|
||||
CHECK_THROWS(load_gallery("/nonexistent/path/gallery.json"));
|
||||
}
|
||||
|
||||
// ── GR-004: gallery ↔ embedder binding ───────────────────────────────────────
|
||||
// Verification plan row GR-004/T1: "Mismatched embedder → hard startup error;
|
||||
// error names both sides." The comparison is a pure function over two stamps, so
|
||||
// none of this needs a GPU, an ONNX, or even a file.
|
||||
|
||||
/// TRACES: GR-004 | SR-001
|
||||
TEST_CASE("gallery save/load round-trips the embedder stamp", "[gallery][GR-004]") {
|
||||
ActorGallery g;
|
||||
ActorGallery::Actor a;
|
||||
a.name = "Stamped Actor";
|
||||
a.embeddings = {make_embedding(0.3f)};
|
||||
g.actors.push_back(a);
|
||||
g.embedder = stamp("LVFace-B_Glint360K.onnx", kShaA);
|
||||
|
||||
TempFile tf("gallery_stamped.h5");
|
||||
save_gallery(tf.path, g);
|
||||
ActorGallery loaded = load_gallery(tf.path);
|
||||
|
||||
CHECK(loaded.embedder.model_name == "LVFace-B_Glint360K.onnx");
|
||||
CHECK(loaded.embedder.model_sha256 == kShaA);
|
||||
CHECK(loaded.embedder.embed_dim == 512);
|
||||
CHECK_FALSE(loaded.embedder.empty());
|
||||
}
|
||||
|
||||
/// TRACES: GR-004 | SR-001
|
||||
TEST_CASE("a gallery written without a stamp loads as unstamped", "[gallery][GR-004]") {
|
||||
// The back-compat case: pre-GR-004 files have no /embedder group at all. The
|
||||
// absence must survive the round trip as an absence — a stamp naming no model
|
||||
// would read as "checked and fine" to every consumer.
|
||||
ActorGallery g;
|
||||
ActorGallery::Actor a;
|
||||
a.name = "Legacy Actor";
|
||||
a.embeddings = {make_embedding(0.f)};
|
||||
g.actors.push_back(a);
|
||||
|
||||
TempFile tf("gallery_unstamped.h5");
|
||||
save_gallery(tf.path, g);
|
||||
ActorGallery loaded = load_gallery(tf.path);
|
||||
|
||||
CHECK(loaded.embedder.empty());
|
||||
}
|
||||
|
||||
/// TRACES: GR-004 | SR-001
|
||||
TEST_CASE("legacy JSON galleries carry an optional embedder stamp", "[gallery][GR-004]") {
|
||||
nlohmann::json j;
|
||||
j["embedder"] = {{"model_name", "arcface_w600k_r50.onnx"},
|
||||
{"model_sha256", kShaB},
|
||||
{"embed_dim", 512}};
|
||||
j["actors"] = nlohmann::json::array();
|
||||
nlohmann::json ja;
|
||||
ja["name"] = "JSON Actor";
|
||||
ja["embeddings"] = nlohmann::json::array();
|
||||
ja["embeddings"].push_back(std::vector<float>(512, 0.1f));
|
||||
j["actors"].push_back(ja);
|
||||
|
||||
TempFile tf("gallery_json_stamp.json");
|
||||
{ std::ofstream out(tf.path); out << j.dump(); }
|
||||
|
||||
ActorGallery g = load_gallery(tf.path);
|
||||
CHECK(g.embedder.model_name == "arcface_w600k_r50.onnx");
|
||||
CHECK(g.embedder.model_sha256 == kShaB);
|
||||
}
|
||||
|
||||
/// TRACES: GR-004 | SR-001
|
||||
TEST_CASE("matching embedder stamps pass", "[gallery][GR-004]") {
|
||||
auto chk = compare_embedder_stamps(stamp("model.onnx", kShaA),
|
||||
stamp("model.onnx", kShaA));
|
||||
CHECK(chk.verdict == StampVerdict::match);
|
||||
CHECK_FALSE(chk.fatal(false));
|
||||
CHECK_FALSE(chk.fatal(true)); // a proven match is never fatal, even in strict mode
|
||||
CHECK_NOTHROW(enforce_embedder_stamp(stamp("model.onnx", kShaA),
|
||||
stamp("model.onnx", kShaA),
|
||||
"g.h5", "model.onnx", true));
|
||||
}
|
||||
|
||||
/// TRACES: GR-004 | SR-001
|
||||
TEST_CASE("the hash decides, not the filename", "[gallery][GR-004]") {
|
||||
// Same bytes under a different filename is the SAME model — a renamed or
|
||||
// relocated file must not be treated as a different one.
|
||||
auto same = compare_embedder_stamps(stamp("lvface.onnx", kShaA),
|
||||
stamp("LVFace-B_Glint360K.onnx", kShaA));
|
||||
CHECK(same.verdict == StampVerdict::match);
|
||||
|
||||
// Different bytes under the SAME filename is a DIFFERENT model — this is the
|
||||
// in-place re-export a name-only stamp would miss entirely, and the reason the
|
||||
// stamp carries a hash at all.
|
||||
auto differ = compare_embedder_stamps(stamp("model.onnx", kShaA),
|
||||
stamp("model.onnx", kShaB));
|
||||
CHECK(differ.verdict == StampVerdict::mismatch);
|
||||
}
|
||||
|
||||
/// TRACES: GR-004 | SR-001
|
||||
TEST_CASE("mismatched embedder is fatal and names both sides", "[gallery][GR-004]") {
|
||||
const auto built = stamp("LVFace-B_Glint360K.onnx", kShaA);
|
||||
const auto loaded = stamp("arcface_w600k_r50.onnx", kShaB);
|
||||
|
||||
auto chk = compare_embedder_stamps(built, loaded, "cast.h5", "models/r50.onnx");
|
||||
REQUIRE(chk.verdict == StampVerdict::mismatch);
|
||||
CHECK(chk.fatal(false)); // no bypass: a mismatch is fatal in every mode
|
||||
CHECK(chk.fatal(true));
|
||||
|
||||
// Both sides must be identifiable from the message alone.
|
||||
CHECK(mentions(chk.message, "LVFace-B_Glint360K.onnx"));
|
||||
CHECK(mentions(chk.message, "arcface_w600k_r50.onnx"));
|
||||
CHECK(mentions(chk.message, kShaA));
|
||||
CHECK(mentions(chk.message, kShaB));
|
||||
CHECK(mentions(chk.message, "cast.h5"));
|
||||
CHECK(mentions(chk.message, "models/r50.onnx"));
|
||||
|
||||
// ...and it must reach the caller as an error, not a log line.
|
||||
CHECK_THROWS_AS(enforce_embedder_stamp(built, loaded, "cast.h5",
|
||||
"models/r50.onnx", false),
|
||||
std::runtime_error);
|
||||
try {
|
||||
enforce_embedder_stamp(built, loaded, "cast.h5", "models/r50.onnx", false);
|
||||
FAIL("mismatch must throw");
|
||||
} catch (const std::runtime_error& e) {
|
||||
const std::string what = e.what();
|
||||
CHECK(mentions(what, "LVFace-B_Glint360K.onnx"));
|
||||
CHECK(mentions(what, "arcface_w600k_r50.onnx"));
|
||||
}
|
||||
}
|
||||
|
||||
/// TRACES: GR-004 | SR-001
|
||||
TEST_CASE("differing embedding width is a mismatch", "[gallery][GR-004]") {
|
||||
auto chk = compare_embedder_stamps(stamp("a.onnx", kShaA, 512),
|
||||
stamp("a.onnx", kShaA, 256));
|
||||
CHECK(chk.verdict == StampVerdict::mismatch);
|
||||
CHECK(mentions(chk.message, "512"));
|
||||
CHECK(mentions(chk.message, "256"));
|
||||
}
|
||||
|
||||
/// TRACES: GR-004 | SR-001
|
||||
TEST_CASE("an unstamped gallery warns by default and fails under strict",
|
||||
"[gallery][GR-004]") {
|
||||
// Decision recorded in src/gallery/embedder_stamp.hpp: unstamped is UNKNOWN,
|
||||
// not known-bad, and every pre-GR-004 gallery is unstamped. Hard-failing them
|
||||
// all would make the check something people disable rather than trust; so it
|
||||
// warns loudly, names the risk, and is promotable to fatal for measurement runs.
|
||||
EmbedderStamp none;
|
||||
auto chk = compare_embedder_stamps(none, stamp("model.onnx", kShaA), "old.h5");
|
||||
REQUIRE(chk.verdict == StampVerdict::unstamped);
|
||||
CHECK_FALSE(chk.fatal(false));
|
||||
CHECK(chk.fatal(true));
|
||||
|
||||
CHECK(mentions(chk.message, "old.h5"));
|
||||
CHECK(mentions(chk.message, "model.onnx")); // the loaded side is still named
|
||||
CHECK(mentions(chk.message, "UNKNOWN")); // ...and the gallery side is honest
|
||||
|
||||
CHECK_NOTHROW(enforce_embedder_stamp(none, stamp("model.onnx", kShaA),
|
||||
"old.h5", "model.onnx", false));
|
||||
CHECK_THROWS_AS(enforce_embedder_stamp(none, stamp("model.onnx", kShaA),
|
||||
"old.h5", "model.onnx", true),
|
||||
std::runtime_error);
|
||||
}
|
||||
|
||||
/// TRACES: GR-004 | SR-001
|
||||
TEST_CASE("an unidentifiable embedder against a stamped gallery is not silent",
|
||||
"[gallery][GR-004]") {
|
||||
// e.g. a replay whose dump predates GR-004: we know what built the gallery but
|
||||
// not what produced the vectors being fed in. Unverifiable, so it must not
|
||||
// report success.
|
||||
auto chk = compare_embedder_stamps(stamp("model.onnx", kShaA), EmbedderStamp{},
|
||||
"g.h5", "old dump.h5");
|
||||
CHECK(chk.verdict == StampVerdict::unknown_embedder);
|
||||
CHECK_FALSE(chk.fatal(false));
|
||||
CHECK(chk.fatal(true));
|
||||
CHECK(mentions(chk.message, "model.onnx"));
|
||||
CHECK(mentions(chk.message, "old dump.h5"));
|
||||
}
|
||||
|
||||
/// TRACES: GR-004 | SR-001
|
||||
TEST_CASE("name-only agreement is a weak match, not a clean pass", "[gallery][GR-004]") {
|
||||
// A TRT deployment can run from a prebuilt .engine with the .onnx absent, so
|
||||
// no hash is computable. Names agreeing is evidence, not proof.
|
||||
auto weak = compare_embedder_stamps(stamp("model.onnx", kShaA),
|
||||
stamp("model.onnx", ""));
|
||||
CHECK(weak.verdict == StampVerdict::weak_match);
|
||||
CHECK_FALSE(weak.fatal(false));
|
||||
CHECK(weak.fatal(true));
|
||||
|
||||
// Names disagreeing with no hash available is still a mismatch — the weaker
|
||||
// evidence is enough to convict, just not to acquit.
|
||||
auto bad = compare_embedder_stamps(stamp("lvface.onnx", ""),
|
||||
stamp("arcface.onnx", ""));
|
||||
CHECK(bad.verdict == StampVerdict::mismatch);
|
||||
CHECK(mentions(bad.message, "lvface.onnx"));
|
||||
CHECK(mentions(bad.message, "arcface.onnx"));
|
||||
}
|
||||
|
||||
/// TRACES: GR-004 | SR-001
|
||||
TEST_CASE("sha256 matches the published vectors", "[gallery][GR-004]") {
|
||||
// Pins the in-tree FIPS 180-4 implementation against the standard vectors.
|
||||
// This is what guarantees the C++ stamp and the Python (hashlib) stamp in
|
||||
// scripts/sae_gallery.py agree on the same model file — without it the two
|
||||
// halves of GR-004 could silently diverge and every check would be a mismatch.
|
||||
CHECK(sha256_hex("") ==
|
||||
"e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855");
|
||||
CHECK(sha256_hex("abc") ==
|
||||
"ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad");
|
||||
CHECK(sha256_hex("abcdbcdecdefdefgefghfghighijhijkijkljklmklmnlmnomnopnopq") ==
|
||||
"248d6a61d20638b8e5c026930c3e6039a33ce45964ff2167f6ecedd419db06c1");
|
||||
// Multi-block input, exercising the length-padding path past 64 bytes.
|
||||
CHECK(sha256_hex(std::string(1000, 'a')) ==
|
||||
"41edece42d63e8d9bf515a9ba6932e1c20cbc9f5a5d134645adb5db1b9737ea3");
|
||||
}
|
||||
|
||||
/// TRACES: GR-004 | SR-001
|
||||
TEST_CASE("make_embedder_stamp hashes a real file and degrades gracefully",
|
||||
"[gallery][GR-004]") {
|
||||
// Stands in for an ONNX: the stamp does not care what the bytes mean.
|
||||
TempFile tf("fake_model.onnx");
|
||||
{ std::ofstream out(tf.path, std::ios::binary); out << "abc"; }
|
||||
|
||||
EmbedderStamp s = make_embedder_stamp(tf.path);
|
||||
CHECK(s.model_sha256 ==
|
||||
"ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad");
|
||||
CHECK_FALSE(s.model_name.empty());
|
||||
CHECK(s.model_name.find('/') == std::string::npos); // basename, not full path
|
||||
|
||||
// A model path that does not exist still yields a comparable name-only stamp
|
||||
// rather than an empty one, which is what keeps engine-only deployments usable.
|
||||
EmbedderStamp missing = make_embedder_stamp("/nonexistent/models/foo.onnx");
|
||||
CHECK(missing.model_name == "foo.onnx");
|
||||
CHECK(missing.model_sha256.empty());
|
||||
CHECK_FALSE(missing.empty());
|
||||
|
||||
CHECK(make_embedder_stamp("").empty());
|
||||
}
|
||||
|
||||
@@ -1,210 +0,0 @@
|
||||
// Replay tests — the real tracker and registry driven from committed fixtures.
|
||||
//
|
||||
// TRACES: AR-004, AR-012, AR-013 | VR-001, VR-002 | IT-001
|
||||
//
|
||||
// Tier T2: composition, not units. The registry tests construct awkward states
|
||||
// directly; these check that the pieces behave when wired together and fed real
|
||||
// footage — 480x360 public-domain clips at 5 fps, with the cuts, gaps and
|
||||
// crowded frames that actual film produces and synthetic input does not.
|
||||
//
|
||||
// No GPU and no model: the fixtures are HDF5 dumps taken after embedding, so
|
||||
// everything here is CPU maths. That is what lets this run on the CI host at
|
||||
// all (see docs/requirements.md, "CI never calls a model").
|
||||
//
|
||||
// Driving the node functors directly rather than through a KPN network is
|
||||
// deliberate: functors are plain objects, so there are no threads, no channels
|
||||
// and no scheduling — the same input gives the same output every time, which is
|
||||
// exactly what a fixture-based test needs.
|
||||
#include <catch2/catch_test_macros.hpp>
|
||||
|
||||
#include "config.hpp"
|
||||
#include "evidence_discount.hpp"
|
||||
#include "nodes/face_tracker_node.hpp"
|
||||
#include "track_registry.hpp"
|
||||
#include "types.hpp"
|
||||
|
||||
#include <H5Cpp.h>
|
||||
|
||||
#include <algorithm>
|
||||
#include <cmath>
|
||||
#include <memory>
|
||||
#include <string>
|
||||
#include <vector>
|
||||
|
||||
namespace {
|
||||
|
||||
// ── Fixture reader ───────────────────────────────────────────────────────────
|
||||
// The flat/ragged layout of scripts/optimizer/SCHEMA.md: per-face arrays
|
||||
// concatenated, with a per-frame index table pointing into them.
|
||||
struct Dump {
|
||||
std::vector<double> ts;
|
||||
std::vector<uint8_t> is_cut;
|
||||
std::vector<int64_t> face_offset;
|
||||
std::vector<int32_t> face_count;
|
||||
std::vector<Embedding> emb;
|
||||
std::vector<float> bbox; // 4 per face
|
||||
std::string embedder;
|
||||
|
||||
std::size_t frames() const { return ts.size(); }
|
||||
std::size_t faces() const { return emb.size(); }
|
||||
};
|
||||
|
||||
template <typename T>
|
||||
std::vector<T> read1d(H5::Group& g, const char* name, const H5::DataType& dt) {
|
||||
H5::DataSet ds = g.openDataSet(name);
|
||||
hsize_t n = 0;
|
||||
ds.getSpace().getSimpleExtentDims(&n, nullptr);
|
||||
std::vector<T> out(n);
|
||||
if (n) ds.read(out.data(), dt);
|
||||
return out;
|
||||
}
|
||||
|
||||
Dump load(const std::string& path) {
|
||||
H5::H5File f(path, H5F_ACC_RDONLY);
|
||||
H5::Group frames = f.openGroup("frames");
|
||||
H5::Group faces = f.openGroup("faces");
|
||||
|
||||
Dump d;
|
||||
d.ts = read1d<double>(frames, "timestamp_sec", H5::PredType::NATIVE_DOUBLE);
|
||||
d.is_cut = read1d<uint8_t>(frames, "is_cut", H5::PredType::NATIVE_UINT8);
|
||||
d.face_offset = read1d<int64_t>(frames, "face_offset", H5::PredType::NATIVE_INT64);
|
||||
d.face_count = read1d<int32_t>(frames, "face_count", H5::PredType::NATIVE_INT32);
|
||||
|
||||
H5::DataSet e = faces.openDataSet("embedding");
|
||||
hsize_t dims[2]{0, 0};
|
||||
e.getSpace().getSimpleExtentDims(dims, nullptr);
|
||||
std::vector<float> flat(dims[0] * dims[1]);
|
||||
if (!flat.empty()) e.read(flat.data(), H5::PredType::NATIVE_FLOAT);
|
||||
d.emb.resize(dims[0]);
|
||||
for (hsize_t i = 0; i < dims[0]; ++i)
|
||||
std::copy_n(flat.begin() + i * dims[1], 512, d.emb[i].begin());
|
||||
|
||||
// bbox is 2-D [N,4]; reading it with the 1-D helper would size the buffer
|
||||
// from the first extent only and then read four times that many floats.
|
||||
{
|
||||
H5::DataSet bs = faces.openDataSet("bbox");
|
||||
hsize_t bd[2]{0, 0};
|
||||
bs.getSpace().getSimpleExtentDims(bd, nullptr);
|
||||
d.bbox.resize(bd[0] * bd[1]);
|
||||
if (!d.bbox.empty()) bs.read(d.bbox.data(), H5::PredType::NATIVE_FLOAT);
|
||||
}
|
||||
|
||||
// GR-004: the dump records which embedder produced it, so a replay cannot
|
||||
// be silently scored against a gallery from a different model.
|
||||
if (f.attrExists("embedder_model")) {
|
||||
// Written as a variable-length string (embedding_dump_node.hpp:99), so
|
||||
// the read must name the same type explicitly.
|
||||
H5::StrType vlen(H5::PredType::C_S1, H5T_VARIABLE);
|
||||
f.openAttribute("embedder_model").read(vlen, d.embedder);
|
||||
}
|
||||
return d;
|
||||
}
|
||||
|
||||
std::string fixture(const char* name) {
|
||||
return std::string(SAE_TEST_FIXTURES_DIR) + "/dumps/" + name;
|
||||
}
|
||||
|
||||
// ── Harness ──────────────────────────────────────────────────────────────────
|
||||
struct Replay {
|
||||
std::vector<DeadTrack> claims;
|
||||
std::vector<int> track_ids; // per face, in fixture order
|
||||
std::size_t faces_seen{0};
|
||||
};
|
||||
|
||||
Replay run(const Dump& d, double extinction = 10.0) {
|
||||
Replay r;
|
||||
TrackRegistry::Config rc;
|
||||
rc.extinction_sec = extinction;
|
||||
|
||||
auto cal = [](float cos) { return std::max(0.f, cos); };
|
||||
auto reg = std::make_shared<TrackRegistry>(rc, EvidenceDiscounter(cal));
|
||||
reg->on_track_dead([&r](const DeadTrack& t) { r.claims.push_back(t); });
|
||||
|
||||
Config cfg;
|
||||
cfg.track_assoc_min_prob = 0.5f;
|
||||
FaceTrackerFunc ft(cfg, reg, cal);
|
||||
|
||||
for (std::size_t i = 0; i < d.frames(); ++i) {
|
||||
EmbeddedSceneFrame ef;
|
||||
ef.source.timestamp_sec = d.ts[i];
|
||||
ef.source.is_cut = d.is_cut[i] != 0;
|
||||
|
||||
const int64_t off = d.face_offset[i];
|
||||
const int32_t n = d.face_count[i];
|
||||
for (int32_t k = 0; k < n; ++k) {
|
||||
DetectedFace face;
|
||||
const float* b = &d.bbox[(off + k) * 4];
|
||||
face.bbox = cv::Rect2f(b[0], b[1], b[2], b[3]);
|
||||
face.confidence = 1.0f;
|
||||
ef.faces.push_back(face);
|
||||
ef.crops.push_back(cv::Mat());
|
||||
ef.embeddings.push_back(d.emb[off + k]);
|
||||
}
|
||||
r.faces_seen += static_cast<std::size_t>(n);
|
||||
|
||||
auto out = ft(std::move(ef));
|
||||
for (int id : out.track_ids) r.track_ids.push_back(id);
|
||||
}
|
||||
|
||||
reg->flush(d.ts.empty() ? 0.0 : d.ts.back());
|
||||
return r;
|
||||
}
|
||||
|
||||
} // namespace
|
||||
|
||||
// ── AR-004 / VR-001 — the fixtures are intact and self-describing ────────────
|
||||
TEST_CASE("superhero fixture is complete", "[replay][VR-001]") {
|
||||
Dump d = load(fixture("superhero.h5"));
|
||||
CHECK(d.frames() == 5128);
|
||||
CHECK(d.faces() == 4307);
|
||||
CHECK(d.embedder == "LVFace-B_Glint360K.onnx");
|
||||
|
||||
int64_t running = 0;
|
||||
for (std::size_t i = 0; i < d.frames(); ++i) {
|
||||
REQUIRE(d.face_offset[i] == running);
|
||||
running += d.face_count[i];
|
||||
}
|
||||
CHECK(static_cast<std::size_t>(running) == d.faces());
|
||||
}
|
||||
|
||||
TEST_CASE("replaying the superhero fixture twice gives identical tracks",
|
||||
"[replay][VR-002]") {
|
||||
Dump d = load(fixture("superhero.h5"));
|
||||
Replay a = run(d);
|
||||
Replay b = run(d);
|
||||
|
||||
REQUIRE(a.track_ids.size() == b.track_ids.size());
|
||||
CHECK(a.track_ids == b.track_ids);
|
||||
REQUIRE(a.claims.size() == b.claims.size());
|
||||
}
|
||||
|
||||
TEST_CASE("every face is assigned a track and every track closes",
|
||||
"[replay][AR-012]") {
|
||||
Dump d = load(fixture("superhero.h5"));
|
||||
Replay r = run(d);
|
||||
|
||||
CHECK(r.track_ids.size() == r.faces_seen);
|
||||
for (int id : r.track_ids) CHECK(id >= 0); // nothing silently unassigned
|
||||
|
||||
// flush() must leave nothing behind: a track still open at EOF would be a
|
||||
// window that never reaches the output.
|
||||
CHECK(r.claims.size() > 0);
|
||||
}
|
||||
|
||||
TEST_CASE("windows are well-formed and inside the film", "[replay][AR-013]") {
|
||||
for (const char* f : {"superhero.h5", "superhero.h5", "superhero.h5",
|
||||
"superhero.h5", "superhero.h5"}) {
|
||||
INFO(f);
|
||||
Dump d = load(fixture(f));
|
||||
Replay r = run(d);
|
||||
const double t0 = d.ts.front(), t1 = d.ts.back();
|
||||
|
||||
for (const auto& c : r.claims) {
|
||||
// A window ends at the last sighting, never after it — so it can
|
||||
// never extend past the footage that produced it.
|
||||
CHECK(c.first_seen <= c.last_seen);
|
||||
CHECK(c.first_seen >= t0);
|
||||
CHECK(c.last_seen <= t1);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,11 +1,5 @@
|
||||
// TRACES: AR-026 | SR-001
|
||||
//
|
||||
// Unit tests for the CPU reference similarity engine (backends/gemm_backend.cpp,
|
||||
// SAE_GEMM_CPU) and the l2_normalise helper. All pure, GPU-free, model-free.
|
||||
//
|
||||
// This is the CI half of AR-026: equivalence between the GEMM path and
|
||||
// hand-computed dot products on small input. Throughput at scale (AR-027) is T4
|
||||
// and cannot run here.
|
||||
#include <catch2/catch_test_macros.hpp>
|
||||
#include <catch2/matchers/catch_matchers_floating_point.hpp>
|
||||
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user