diff --git a/CMakeLists.txt b/CMakeLists.txt index e218fc1..fb8fac0 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -258,6 +258,7 @@ add_library(sae_gallery STATIC src/gallery/gallery_store.cpp src/gallery/gallery_builder.cpp src/audio_signature.cpp # IR-004 — content-derived audio signature + src/gallery/embedder_stamp.cpp # GR-004 — gallery/embedder binding ) set_target_properties(sae_gallery PROPERTIES POSITION_INDEPENDENT_CODE ON) target_include_directories(sae_gallery PUBLIC src ${HDF5_INCLUDE_DIRS}) diff --git a/docs/SPEC.md b/docs/SPEC.md index 49fbaa0..5123d6b 100644 --- a/docs/SPEC.md +++ b/docs/SPEC.md @@ -1083,8 +1083,57 @@ surfaced as a build report. different models are meaningless but *look* plausible — this fails silently and expensively otherwise. -**Gap:** named as step 4 of the `service-conversion.md` implementation plan; -unbuilt. This is the highest-value small fix in the document. +### The stamp + +Two fields, written together: the model file's **basename** and the **SHA-256 of +its bytes** (plus `embed_dim` as a cheap extra guard). Stored as the `/embedder` +group in the gallery HDF5, and as an optional top-level `"embedder"` object in +the legacy JSON format. + +The hash *decides*; the name is what a human *reads*. Neither alone is enough. A +name is a promise rather than a fact — models get re-exported, re-quantised and +overwritten in place under an unchanged filename, which is exactly the case where +the weights differ and nothing else does, so a name-only stamp is blind to the +failure it exists to catch. A hash alone is correct but unactionable: *"expected +3f2a…, got 9c1b…"* tells an operator nothing about what to do next. SHA-256 over +the file is derived from the artefact rather than asserted about it, needs no +registry kept up to date, and costs ~0.1 s for a 250 MB ONNX once per process. + +### Verdicts + +| Verdict | When | Default | Under strict mode | +|---|---|---|---| +| `match` | hashes agree | proceed | proceed | +| `weak_match` | names agree, one side unhashable | **warn** | **error** | +| `unstamped` | gallery predates GR-004 | **warn** | **error** | +| `unknown_embedder` | gallery stamped, embedder unidentifiable | **warn** | **error** | +| `mismatch` | proven different models | **error** | **error** | + +**A mismatch is fatal in every mode, with no bypass**, and the message names both +sides — what the gallery was built with and what is loaded. + +The three "cannot prove it" verdicts warn loudly instead, because they describe an +*unknown* state rather than a *known-bad* one, and because every gallery built +before this requirement is unstamped. Hard-failing all of them would make the +check something people route around rather than trust. Strict mode +(`--require-gallery-stamp`, or `SAE_REQUIRE_GALLERY_STAMP=1`, which propagates to +subprocesses) promotes them to errors — that is the mode measurement work runs in. +`scripts/stamp_gallery.py` re-binds an existing gallery without re-embedding, so +migration costs one command; that is what makes "warn" a temporary state rather +than a permanent one. + +### Scope of the check + +Embedding **dumps** carry the same stamp (`embedder_model` / `embedder_sha256` +root attributes, `scripts/optimizer/SCHEMA.md`): a replay has no live embedder, so +the dump *is* the embedder as far as the gallery is concerned. Derived galleries +(filter, cast-restrict) inherit their source's stamp; `--merge` and the JSON +gallery merge check *before* writing, since a merged file holding two embedding +spaces cannot be untangled afterwards by any later check. + +**Gap:** none. Stamped in `gallery_builder.cpp` and the Python builders; verified +in `scene_analyze`, `scene_preview`, the `sae_kpn` matcher binding, `replay.py`, +`optimize.py`, `movienet_eval.py` and the merge paths. ## GR-006 … GR-009 — Provenance tiers and poisoning guard diff --git a/docs/plan.md b/docs/plan.md index de9dadc..62b5245 100644 --- a/docs/plan.md +++ b/docs/plan.md @@ -260,13 +260,23 @@ Context crops opt-in behind `--dump-unidentified-crops`. # Gallery -## GR-004 — Model binding +## GR-004 — Model binding — **DONE** -**Depends on:** nothing. **Startable immediately, highest value per line.** +**Depended on:** nothing. Landed before any measurement work, as intended. -Stamp embedder identity into the gallery at build; verify at load in -`scene_analyze`, `replay.py` and the optimizer. Mismatch is a hard error naming -both sides. +Stamp = model basename + SHA-256 of the ONNX, written as the `/embedder` group at +build time (`gallery_builder.cpp`, `sae_gallery.save_gallery_hdf5`) and verified +at load in `scene_analyze`, `scene_preview`, the `sae_kpn` matcher binding, +`replay.py`, `optimize.py` and `movienet_eval.py`. Mismatch is a hard error naming +both sides, with no bypass. Embedding dumps carry the same stamp, since a replay +has no live embedder to check against. + +Unstamped legacy galleries **warn loudly and proceed** rather than failing: +unknown is not known-bad, and hard-failing every pre-existing gallery would turn +the check into something people disable. `--require-gallery-stamp` / +`SAE_REQUIRE_GALLERY_STAMP=1` promotes that to a hard error — measurement runs +should set it. `scripts/stamp_gallery.py` re-binds an existing gallery without +re-embedding, so the warning state is cheap to leave. Cross-model similarities are meaningless but *look* plausible — this fails silently and expensively, and it would corrupt every measurement taken during the diff --git a/docs/requirements.md b/docs/requirements.md index 4d83bb1..8191a8d 100644 --- a/docs/requirements.md +++ b/docs/requirements.md @@ -294,7 +294,7 @@ because it will be trusted. | IR-007 | **T1** | Media < 120 s emits no signature at all | Exactly 120 s; just under; zero-length audio. Must match the plugin's cutoff exactly — a caller-varying window length is what SR-004 forbids | | IR-008 | T1 | `v1:` prefix emitted and honoured on read | Unknown prefix rejected, not guessed | | GR-009 | T1 | Human-confirmed associations persist and are tier-tagged | Survives a gallery rebuild; distinguishable from baked and harvested | -| GR-004 | T1 | Mismatched embedder → hard startup error | Error names both sides | +| GR-004 | T1 | Mismatched embedder → hard startup error | Error names both sides; **unstamped warns, and errors under `SAE_REQUIRE_GALLERY_STAMP`**; same filename + different SHA-256 must still be a mismatch | | GR-008 | T1 | Outlier flagged among an actor's references | Injected poisoned embedding detected | | VR-009 | T1 | Posterior calibration holds | A 0.99 posterior is wrong ~1% of the time on held-out tracks | diff --git a/scripts/filter_gallery.py b/scripts/filter_gallery.py index 8610603..c07deb0 100644 --- a/scripts/filter_gallery.py +++ b/scripts/filter_gallery.py @@ -77,7 +77,11 @@ def main(): if missing > 0: print(f"[warn] {missing} cast member(s) not present in gallery (not yet embedded)", file=sys.stderr) - save_gallery_hdf5({"actors": actors}, Path(args.output)) + # TRACES: GR-004 | SR-001 — a filtered gallery holds the SAME vectors as its + # source, so it inherits the source's binding. Dropping the stamp here would + # silently launder a stamped gallery into an unstamped one. + save_gallery_hdf5({"actors": actors}, Path(args.output), + gallery.get("embedder")) print(f"Saved {len(actors)} actor(s) to {args.output}", file=sys.stderr) diff --git a/scripts/make_gallery.py b/scripts/make_gallery.py index 7d3a208..024b935 100755 --- a/scripts/make_gallery.py +++ b/scripts/make_gallery.py @@ -36,8 +36,9 @@ import time from pathlib import Path sys.path.insert(0, str(Path(__file__).resolve().parent)) -from sae_embed_loader import load_embedder -from sae_gallery import download_images, save_gallery, wikidata_image_urls +from sae_embed_loader import load_embedder, resolve_arcface +from sae_gallery import (download_images, embedder_stamp, save_gallery, + wikidata_image_urls) from sae_tmdb import TMDB_IMG, tmdb_get, tmdb_id_from_imdb @@ -177,7 +178,11 @@ def main(): output = Path(args.output) image_root = Path(args.image_dir) if args.image_dir else output.parent / "images" + # TRACES: GR-004 | SR-001 — stamp with the model actually loaded, resolved + # through the same helper load_embedder uses so the two cannot diverge. + arcface_path = resolve_arcface(args.models_dir, args.arcface) embedder = load_embedder(args.build_dir, args.models_dir, args.arcface) + stamp = embedder_stamp(arcface_path) # Resolve movie ID movie_id = args.movie_id @@ -203,7 +208,7 @@ def main(): if n_actors == 0: sys.exit("No actors could be processed — check models and images.") - save_gallery(gallery, missing, output) + save_gallery(gallery, missing, output, embedder=stamp) if __name__ == "__main__": diff --git a/scripts/make_jellyfin_gallery.py b/scripts/make_jellyfin_gallery.py index 45379cd..556757c 100644 --- a/scripts/make_jellyfin_gallery.py +++ b/scripts/make_jellyfin_gallery.py @@ -51,8 +51,9 @@ import requests sys.path.insert(0, str(Path(__file__).resolve().parent)) import sae_env # noqa: F401 — loads .env into os.environ on import -from sae_embed_loader import load_embedder -from sae_gallery import (download_image, download_images, load_gallery_hdf5, +from sae_embed_loader import load_embedder, resolve_arcface +from sae_gallery import (download_image, download_images, embedder_stamp, + enforce_embedder_stamp, load_gallery_hdf5, save_gallery, wikidata_image_urls) from sae_jellyfin import actor_jellyfin_id, jf_get, normalize_jellyfin_url from sae_tmdb import ( @@ -442,11 +443,20 @@ def main(): image_root = Path(args.image_dir) if args.image_dir else output.parent / "images" item_types = [t.strip() for t in args.item_types.split(",") if t.strip()] + # TRACES: GR-004 | SR-001 + arcface_path = resolve_arcface(args.models_dir, args.arcface) embedder = load_embedder(args.build_dir, args.models_dir, args.arcface) + stamp = embedder_stamp(arcface_path) existing_actors = {} if args.merge and output.is_file(): existing = load_gallery_hdf5(output) + # TRACES: GR-004 | SR-001 — --merge keeps the existing actors' vectors and + # embeds the new ones with THIS model. If they disagree, the result is one + # gallery holding two incompatible embedding spaces, which is worse than a + # mismatched gallery: no later check can separate them again. + enforce_embedder_stamp(existing.get("embedder"), stamp, str(output), + arcface_path) for actor in existing.get("actors", []): pid = actor_jellyfin_id(actor) if pid: @@ -475,7 +485,7 @@ def main(): if n_actors == 0: sys.exit("No actors could be processed — check Jellyfin URL/API key and models.") - save_gallery(gallery, missing, output) + save_gallery(gallery, missing, output, embedder=stamp) if __name__ == "__main__": diff --git a/scripts/movienet_eval.py b/scripts/movienet_eval.py index b94abf8..454c832 100644 --- a/scripts/movienet_eval.py +++ b/scripts/movienet_eval.py @@ -22,8 +22,8 @@ from pathlib import Path import numpy as np sys.path.insert(0, str(Path(__file__).resolve().parent)) -from sae_embed_loader import load_embedder -from sae_gallery import load_gallery_hdf5 +from sae_embed_loader import load_embedder, resolve_arcface +from sae_gallery import load_gallery_hdf5, verify_gallery_stamp def load_gallery(path: str) -> dict[str, dict]: @@ -62,6 +62,11 @@ def main(): args = p.parse_args() embedder = load_embedder(args.build_dir, args.models_dir, args.arcface) + # TRACES: GR-004 | SR-001 — match() below is a bare dot product against the + # gallery's vectors; if the gallery came from another model those numbers are + # noise wearing a similarity's clothes. + verify_gallery_stamp(args.gallery, + resolve_arcface(args.models_dir, args.arcface)) gallery = load_gallery(args.gallery) print(f"[eval] gallery: {len(gallery)} actors", file=sys.stderr) diff --git a/scripts/optimizer/SCHEMA.md b/scripts/optimizer/SCHEMA.md index cf2ba34..105cfca 100644 --- a/scripts/optimizer/SCHEMA.md +++ b/scripts/optimizer/SCHEMA.md @@ -22,6 +22,8 @@ variable-length HDF5 types and reads straight into numpy. movie : str (source video path) sample_fps : float embed_dim : int = 512 + embedder_model : str basename of the embedding model (GR-004) + embedder_sha256: str SHA-256 of that model file (GR-004) frames/ group — one row per sampled frame timestamp_sec : float64 [F] @@ -41,6 +43,20 @@ variable-length HDF5 types and reads straight into numpy. `F` = number of sampled frames, `N` = total faces (= sum of face_count). Frame *i*'s faces are `faces/*[ face_offset[i] : face_offset[i]+face_count[i] ]`. +## Model binding (GR-004) + +`embedder_model` / `embedder_sha256` record which embedder produced every vector +in `faces/embedding`. A replay has no live embedder, so **the dump is the embedder +as far as the gallery is concerned**: `replay.py` checks these two attributes +against the gallery's own `/embedder` stamp and refuses to run on a mismatch, +naming both sides. Cross-model cosines are meaningless but look plausible. + +The attributes are additive, not a format break — `schema_version` stays 1. Dumps +written before GR-004 simply lack them, which reports as *unverifiable* (a loud +warning, or a hard error under `SAE_REQUIRE_GALLERY_STAMP=1`) rather than as a +pass. Re-dump to bind an old dump; there is no in-place migration, because unlike +a gallery nobody can assert after the fact which model produced a vector. + ## Invariants - `embedding` rows are unit-norm (cosine == dot product against the gallery). - `face_offset[0] == 0`; `face_offset[i+1] == face_offset[i] + face_count[i]`. diff --git a/scripts/optimizer/fetch_missing_actors.py b/scripts/optimizer/fetch_missing_actors.py index f125c6b..773999b 100644 --- a/scripts/optimizer/fetch_missing_actors.py +++ b/scripts/optimizer/fetch_missing_actors.py @@ -35,7 +35,9 @@ REPO = Path(__file__).resolve().parent.parent.parent sys.path.insert(0, str(REPO / "scripts")) import sae_env # noqa: E402 loads .env from sae_tmdb import tmdb_get, tmdb_person_for_imdb, TMDB_IMG # noqa: E402 -from sae_gallery import download_images, wikidata_image_urls # noqa: E402 +from sae_embed_loader import resolve_arcface # noqa: E402 +from sae_gallery import (download_images, embedder_stamp, # noqa: E402 + enforce_embedder_stamp, wikidata_image_urls) from sae_embed_loader import load_embedder # noqa: E402 @@ -57,6 +59,7 @@ def fetch(missing_path, out_path, token, build_dir, models_dir, arcface, src = "TMDB + Wikidata fallback" if use_wikidata else "TMDB" print(f"[fetch] {len(missing)} missing actors to resolve via {src}", file=sys.stderr) embedder = load_embedder(build_dir, models_dir, arcface) + stamp = embedder_stamp(resolve_arcface(models_dir, arcface)) # TRACES: GR-004 | SR-001 img_root = Path(tempfile.mkdtemp(prefix="missing_gallery_")) actors = [] @@ -103,7 +106,9 @@ def fetch(missing_path, out_path, token, build_dir, models_dir, arcface, f"(wiki={n_via_wikidata}) no_tmdb={n_no_tmdb} no_img={n_no_img} " f"no_face={n_no_face}", file=sys.stderr) - Path(out_path).write_text(json.dumps({"actors": actors}, indent=2)) + # TRACES: GR-004 | SR-001 — the legacy JSON gallery carries the same stamp as + # the HDF5 one; src/gallery/gallery_store.cpp reads it from either. + Path(out_path).write_text(json.dumps({"embedder": stamp, "actors": actors}, indent=2)) n_emb = sum(len(a["embeddings"]) for a in actors) print(f"\n[fetch] recovered {n_resolved}/{len(missing)} actors " f"({n_via_wikidata} via Wikidata), {n_emb} embeddings → {out_path}", @@ -115,6 +120,12 @@ def fetch(missing_path, out_path, token, build_dir, models_dir, arcface, def merge(base_path, add_path, out_path): base = json.loads(Path(base_path).read_text()) add = json.loads(Path(add_path).read_text()) + # TRACES: GR-004 | SR-001 — merging two galleries from different models makes + # ONE file containing two incompatible embedding spaces. Nothing downstream can + # ever untangle that, so this is the one place the check must run before, not + # after, the write. + enforce_embedder_stamp(base.get("embedder"), add.get("embedder"), + str(base_path), str(add_path)) have = {a.get("imdb_id") for a in base["actors"] if a.get("imdb_id")} added = [a for a in add["actors"] if a.get("imdb_id") not in have] base["actors"].extend(added) diff --git a/scripts/optimizer/optimize.py b/scripts/optimizer/optimize.py index e0a5bc0..73e4091 100644 --- a/scripts/optimizer/optimize.py +++ b/scripts/optimizer/optimize.py @@ -34,6 +34,7 @@ from scipy.optimize import differential_evolution REPO = Path(__file__).resolve().parent.parent.parent sys.path.insert(0, str(REPO / "scripts" / "optimizer")) sys.path.insert(0, str(REPO / "scripts" / "validation")) +sys.path.insert(0, str(REPO / "scripts")) import json as _json import os @@ -56,6 +57,8 @@ DE_WORKERS = int(os.environ.get("DE_WORKERS", "1")) from second_score import score_seconds # noqa: E402 uniform per-second TPI/FPI scoring from sample_eval import load_gallery_keys # noqa: E402 +from replay import dump_embedder_stamp # noqa: E402 +from sae_stamp import EmbedderMismatch, verify_gallery_stamp # noqa: E402 _GAL_KEYS: dict = {} # gallery path → key set (fair-recall FN mask), loaded once _REPLAY_TIMEOUT = 45 # seconds per film; a wedged replay is killed, not left to hang @@ -180,7 +183,15 @@ def main(): p.add_argument("--seed", type=int, default=0) p.add_argument("--trajectory", help="write every evaluation here (JSON lines)") p.add_argument("--out", help="write best config + metrics") + # TRACES: GR-004 | SR-001 + p.add_argument("--require-gallery-stamp", action="store_true", + help="unprovable gallery/dump model binding is a hard error, " + "not a warning (also via SAE_REQUIRE_GALLERY_STAMP=1)") args = p.parse_args() + if args.require_gallery_stamp: + # Set the env var rather than threading a flag through cfg: replays run as + # subprocesses and inherit it, so strictness cannot be lost in the handoff. + os.environ["SAE_REQUIRE_GALLERY_STAMP"] = "1" films = json.loads(Path(args.manifest).read_text()) for f in films: @@ -188,6 +199,18 @@ def main(): if not Path(f["dump"]).exists(): sys.exit(f"[opt] missing dump for {f['name']}: {f['dump']}") + # TRACES: GR-004 | SR-001 — every (dump, gallery) pair is checked ONCE here, + # before the first evaluation. A DE sweep is thousands of replays; discovering + # a cross-model pair at the end (or never) means every number it produced was + # noise. Each replay subprocess re-checks its own pair anyway. + for f in films: + try: + verify_gallery_stamp(f["gallery"], stamp=dump_embedder_stamp(f["dump"]), + embedder_desc=f"embedding dump {Path(f['dump']).name}", + require_stamp=args.require_gallery_stamp) + except EmbedderMismatch as e: + sys.exit(f"[opt] {f['name']}: {e}") + names, bounds = [], [] int_knobs = {"track_max_frames_missing", "cut_inactive_max_frames"} for spec in args.params: diff --git a/scripts/optimizer/reembed_gallery.py b/scripts/optimizer/reembed_gallery.py index 603d809..a672d59 100644 --- a/scripts/optimizer/reembed_gallery.py +++ b/scripts/optimizer/reembed_gallery.py @@ -27,8 +27,9 @@ from pathlib import Path REPO = Path(__file__).resolve().parent.parent.parent sys.path.insert(0, str(REPO / "scripts")) -from sae_embed_loader import load_embedder # noqa: E402 -from sae_gallery import load_gallery_hdf5, save_gallery_hdf5 # noqa: E402 +from sae_embed_loader import load_embedder, resolve_arcface # noqa: E402 +from sae_gallery import (embedder_stamp, load_gallery_hdf5, # noqa: E402 + save_gallery_hdf5) def find_dir(images_root: Path, jellyfin_id: str, name: str) -> Path | None: @@ -58,6 +59,11 @@ def main(): ref = load_gallery_hdf5(Path(args.ref)) images_root = Path(args.images) embedder = load_embedder(args.build_dir, args.models_dir, args.arcface) + # TRACES: GR-004 | SR-001 — this script exists to produce a gallery in a + # DIFFERENT model's space from the reference. The output must therefore never + # inherit the reference's stamp; it carries the stamp of --arcface, which is + # the whole point of the bake-off being safe to run. + stamp = embedder_stamp(resolve_arcface(args.models_dir, args.arcface)) out_actors = [] n_ok = n_nodir = n_noemb = 0 @@ -84,7 +90,7 @@ def main(): print(f" [{i}/{total}] ok={n_ok} no_dir={n_nodir} no_emb={n_noemb}", file=sys.stderr) - save_gallery_hdf5({"actors": out_actors}, Path(args.out)) + save_gallery_hdf5({"actors": out_actors}, Path(args.out), stamp) n_emb = sum(len(a["embeddings"]) for a in out_actors) print(f"[reembed] {Path(args.arcface).stem}: {n_ok}/{total} actors, {n_emb} embeddings " f"→ {args.out}", file=sys.stderr) diff --git a/scripts/optimizer/replay.py b/scripts/optimizer/replay.py index 0d6ad17..320c632 100644 --- a/scripts/optimizer/replay.py +++ b/scripts/optimizer/replay.py @@ -25,6 +25,22 @@ import h5py import numpy as np REPO = Path(__file__).resolve().parent.parent.parent +sys.path.insert(0, str(REPO / "scripts")) +from sae_stamp import verify_gallery_stamp # noqa: E402 + + +def dump_embedder_stamp(dump_path: str) -> dict: + """The GR-004 embedder stamp recorded in an embedding dump. + + A replay has no live embedder — the dump IS the embedder as far as the gallery + is concerned, so the dump's stamp is what the gallery must be checked against. + Dumps written before GR-004 have no attributes and yield an empty stamp, which + the check reports as unverifiable rather than silently accepting.""" + with h5py.File(dump_path, "r") as f: + name = f.attrs.get("embedder_model", "") + sha = f.attrs.get("embedder_sha256", "") + dec = lambda v: v.decode() if isinstance(v, bytes) else ("" if v is None else str(v)) + return {"model_name": dec(name), "model_sha256": dec(sha), "embed_dim": 512} def load_frames(dump_path: str, min_conf: float = 0.0): @@ -92,6 +108,15 @@ def replay(dump_path: str, gallery: str, cfg: dict, build_dir: str, stop: bool = sys.path.insert(0, build_dir) import sae_kpn + # TRACES: GR-004 | SR-001 — checked here, before any network is built, so a + # cross-model replay dies with one readable error instead of producing a + # plausible-looking score. add_identity_matcher re-checks it C++-side below; + # that is the backstop for any other caller of the binding. + stamp = dump_embedder_stamp(dump_path) + verify_gallery_stamp(gallery, stamp=stamp, + embedder_desc=f"embedding dump {Path(dump_path).name}", + require_stamp=bool(cfg.get("require_gallery_stamp", False))) + frames, movie, fps = load_frames(dump_path, min_conf=float(cfg.get("detector_conf", 0.0))) net = sae_kpn.Network() @@ -122,7 +147,8 @@ def replay(dump_path: str, gallery: str, cfg: dict, build_dir: str, stop: bool = cap = len(frames) * 2 + 64 sae_kpn.add_node_python(net, "replay", source, [], ["EmbeddedSceneFrame"], cap) sae_kpn.add_face_tracker(net, "tracker", cfg, cap) - sae_kpn.add_identity_matcher(net, "matcher", gallery, cfg, cap) + sae_kpn.add_identity_matcher(net, "matcher", gallery, cfg, cap, + stamp["model_name"], stamp["model_sha256"]) sae_kpn.add_scene_tracker(net, "scene", cfg, cap) net.connect("replay", 0, "tracker", 0) net.connect("tracker", 0, "matcher", 0) @@ -221,11 +247,17 @@ def main(): # per-film gallery expansion: promotes pose-varied views of confidently-identified # actors into an in-memory annex, recovering ~+4 recall at no precision cost. p.add_argument("--expand-gallery", action="store_true") + # TRACES: GR-004 | SR-001 — promote an unprovable gallery/dump binding from a + # loud warning to a hard error. Measurement sweeps should set this (or + # SAE_REQUIRE_GALLERY_STAMP=1) so no number comes from an unbound pair. + p.add_argument("--require-gallery-stamp", action="store_true") args = p.parse_args() cfg = {k: getattr(args, k) for k in CFG_KEYS if getattr(args, k) is not None} if args.expand_gallery: cfg["expand_gallery"] = True + if args.require_gallery_stamp: + cfg["require_gallery_stamp"] = True # stop=True: PyNode::stop() sets stop_flag_ before joining, so the source # thread's run_loop actually exits. stop=False skips that, leaving stop_flag_ # false forever — the PyNode destructor's jthread.join() then blocks forever diff --git a/scripts/sae_embed_loader.py b/scripts/sae_embed_loader.py index eb9c001..a60c13a 100644 --- a/scripts/sae_embed_loader.py +++ b/scripts/sae_embed_loader.py @@ -3,11 +3,26 @@ sae_embed.FaceEmbedder loads both ONNX sessions once and exposes an embed(path) -> FaceResult method, avoiding the per-process model reload cost of spawning the embed_faces CLI binary for every image. + +resolve_arcface() exposes the same default-resolution logic load_embedder uses, +so a caller can stamp the gallery it is about to write with the model that +actually produced its embeddings (GR-004) — the resolved path, not the CLI +argument, which is often None. """ import sys from pathlib import Path +DEFAULT_ARCFACE = "arcface_w600k_r50.onnx" + + +def resolve_arcface(models_dir: str, arcface: str | None = None) -> str: + """The ArcFace/LVFace ONNX path load_embedder would use for these arguments. + + TRACES: GR-004 | SR-001 — single source of truth for "which model is this", + so the stamp written into a gallery can never drift from the model loaded.""" + return arcface if arcface else str(Path(models_dir) / DEFAULT_ARCFACE) + def load_embedder(build_dir: str, models_dir: str, arcface: str | None = None, conf: float = 0.5, nms: float = 0.4, max_side: int = 500): @@ -28,7 +43,7 @@ def load_embedder(build_dir: str, models_dir: str, arcface: str | None = None, models_path = Path(models_dir) detector_path = str(models_path / "scrfd_500m_bnkps.onnx") - arcface_path = arcface if arcface else str(models_path / "arcface_w600k_r50.onnx") + arcface_path = resolve_arcface(models_dir, arcface) for model, name in [(detector_path, "SCRFD"), (arcface_path, "ArcFace")]: if not Path(model).is_file(): sys.exit(f"{name} model not found: {model}\nRun: bash scripts/download_models.sh") diff --git a/scripts/sae_gallery.py b/scripts/sae_gallery.py index ac97d5f..1addf29 100644 --- a/scripts/sae_gallery.py +++ b/scripts/sae_gallery.py @@ -6,9 +6,13 @@ make_jellyfin_gallery.download_urls + download_person_images) and the duplicated Galleries are written directly as HDF5 — never JSON. Same layout the C++ side reads/writes (src/gallery/gallery_store.cpp): flat [N,512] embeddings + per-actor -offset/count, parallel imdb_id/tmdb_id/jellyfin_id/name string arrays, and a -per-embedding-row source_images array. calibration is left absent (calib_hash=0); -the C++ identity_matcher fits and writes it back into the file on first use. +offset/count, parallel imdb_id/tmdb_id/jellyfin_id/name string arrays, a +per-embedding-row source_images array, and an /embedder group carrying the +GR-004 model binding. calibration is left absent (calib_hash=0); the C++ +identity_matcher fits and writes it back into the file on first use. + +The GR-004 embedder stamp written into that /embedder group lives in sae_stamp +and is re-exported below, so existing callers keep importing it from here. """ import io @@ -101,11 +105,36 @@ def download_images(urls: list[str], dest_dir: Path, n: int, return paths -def save_gallery_hdf5(gallery: dict, output: Path) -> None: +# ── GR-004: gallery ↔ embedder binding ─────────────────────────────────────── +# Implemented in sae_stamp (kept dependency-light so the optimizer's replay +# subprocesses can import it without pulling requests/Pillow); re-exported here +# because the gallery writers and every existing caller reach for it via this +# module. See src/gallery/embedder_stamp.hpp for the C++ twin and the rationale. +from sae_stamp import ( # noqa: F401 + EmbedderMismatch, + check_embedder_stamp, + describe_stamp, + embedder_stamp, + enforce_embedder_stamp, + read_gallery_stamp, + require_gallery_stamp_from_env, + sha256_file, + verify_gallery_stamp, + _as_str, + _stamp_empty, +) + + +def save_gallery_hdf5(gallery: dict, output: Path, embedder: dict | None = None) -> None: """Write a gallery dict ({"actors": [...]}) directly as HDF5 — same schema src/gallery/gallery_store.cpp reads/writes. No calibration group; the C++ identity_matcher computes and writes it back into this file on first - use against an unseen set of embeddings.""" + use against an unseen set of embeddings. + + `embedder` is the GR-004 stamp (see embedder_stamp()); it may also be carried + on the gallery dict under "embedder", which is how a filtered/derived gallery + keeps its binding without the caller having to re-hash anything.""" + embedder = embedder if embedder is not None else gallery.get("embedder") actors = gallery["actors"] embs, offsets, counts = [], [], [] imdb, tmdb, jf, name, src_images = [], [], [], [], [] @@ -139,8 +168,17 @@ def save_gallery_hdf5(gallery: dict, output: Path) -> None: f.create_dataset("jellyfin_id", data=np.asarray(jf, dtype=object), dtype=str_t) f.create_dataset("name", data=np.asarray(name, dtype=object), dtype=str_t) f.create_dataset("source_images", data=np.asarray(src_images, dtype=object), dtype=str_t) - print(f"Saved: {output} ({len(actors)} actors, {emb_arr.shape[0]} embeddings)", - file=sys.stderr) + # TRACES: GR-004 | SR-001 — omitted entirely when unknown, so "unstamped" + # round-trips as unstamped rather than as a stamp naming no model. + if not _stamp_empty(embedder): + g = f.create_group("embedder") + g.attrs["model_name"] = embedder.get("model_name", "") + g.attrs["model_sha256"] = embedder.get("model_sha256", "") + g.attrs["embed_dim"] = np.int32(embedder.get("embed_dim", 512)) + stamp_note = (f", embedder {embedder['model_name']}" if not _stamp_empty(embedder) + else ", NO EMBEDDER STAMP (GR-004)") + print(f"Saved: {output} ({len(actors)} actors, {emb_arr.shape[0]} embeddings" + f"{stamp_note})", file=sys.stderr) def load_gallery_hdf5(path: Path) -> dict: @@ -158,6 +196,14 @@ def load_gallery_hdf5(path: Path) -> dict: if "source_images" in f: src_images = [s.decode() if isinstance(s, bytes) else s for s in f["source_images"][:]] + # TRACES: GR-004 | SR-001 — carried through so a derived gallery (filter, + # merge, cast-restrict) keeps the binding of the gallery it came from. + stamp = None + if "embedder" in f: + a = f["embedder"].attrs + stamp = {"model_name": _as_str(a.get("model_name", "")), + "model_sha256": _as_str(a.get("model_sha256", "")), + "embed_dim": int(a.get("embed_dim", 512))} actors = [] for a in range(len(offset)): @@ -167,15 +213,19 @@ def load_gallery_hdf5(path: Path) -> dict: if src_images is not None: actor["source_images"] = [src_images[s + i] for i in range(n)] actors.append(actor) - return {"actors": actors} + out = {"actors": actors} + if stamp is not None: + out["embedder"] = stamp + return out -def save_gallery(gallery: dict, missing: list[dict], output: Path) -> None: +def save_gallery(gallery: dict, missing: list[dict], output: Path, + embedder: dict | None = None) -> None: """Write the gallery as HDF5 (forcing a .h5 extension) and, if any actors lack images, a .missing_images.json sidecar.""" if output.suffix not in (".h5", ".hdf5"): output = output.with_suffix(".h5") - save_gallery_hdf5(gallery, output) + save_gallery_hdf5(gallery, output, embedder) if missing: missing_path = output.with_name(output.stem + ".missing_images.json") diff --git a/scripts/sae_stamp.py b/scripts/sae_stamp.py new file mode 100644 index 0000000..ee33e3b --- /dev/null +++ b/scripts/sae_stamp.py @@ -0,0 +1,217 @@ +"""Gallery ↔ embedder model binding (GR-004). + +TRACES: GR-004 | SR-001 + +Python twin of src/gallery/embedder_stamp.{hpp,cpp}; the two implement the same +comparison rules and must stay in agreement. Kept as its own module — rather than +folded into sae_gallery — because scripts/optimizer/replay.py imports it once per +replay subprocess, thousands of times in a DE sweep, and must not pay for +sae_gallery's requests/Pillow imports to ask "were these made by the same model?". +Dependencies here are hashlib, json and h5py, all of which a replay already loads. + +A gallery is only valid for the embedder that built it: cosine similarities across +models are meaningless but look plausible, so the mistake is silent and every +measurement taken afterwards is suspect. Identity = model filename + SHA-256 of +the model file. The hash decides (a model re-exported in place keeps its name but +not its bytes); the name is what makes the error readable. See +src/gallery/embedder_stamp.hpp for the full rationale. +""" + +import hashlib +import json +import os +import sys +from pathlib import Path + +import h5py + + +def _as_str(v) -> str: + return v.decode() if isinstance(v, bytes) else ("" if v is None else str(v)) + + +_STAMP_CACHE: dict = {} + + +class EmbedderMismatch(RuntimeError): + """Gallery was built with a different embedder than the one about to be used.""" + + +def sha256_file(path) -> str: + """Lowercase hex SHA-256 of a file's bytes; "" if it cannot be read.""" + path = Path(path) + try: + st = path.stat() + except OSError: + return "" + key = (str(path), st.st_mtime_ns, st.st_size) + if key in _STAMP_CACHE: + return _STAMP_CACHE[key] + h = hashlib.sha256() + try: + with open(path, "rb") as f: + for chunk in iter(lambda: f.read(1 << 20), b""): + h.update(chunk) + except OSError: + return "" + _STAMP_CACHE[key] = h.hexdigest() + return _STAMP_CACHE[key] + + +def embedder_stamp(model_path, embed_dim: int = 512) -> dict: + """Identify an embedder model file → {"model_name", "model_sha256", "embed_dim"}. + + A model file that is absent (e.g. a TRT deployment running from a prebuilt + .engine) yields a name-only stamp: still comparable, just not provable.""" + if not model_path: + return {"model_name": "", "model_sha256": "", "embed_dim": embed_dim} + sha = sha256_file(model_path) + if not sha: + print(f"[gallery] cannot hash embedder model {model_path} — model binding " + f"falls back to filename only (GR-004)", file=sys.stderr) + return {"model_name": Path(model_path).name, "model_sha256": sha, + "embed_dim": embed_dim} + + +def _stamp_empty(s) -> bool: + return not s or (not s.get("model_name") and not s.get("model_sha256")) + + +def describe_stamp(s) -> str: + if _stamp_empty(s): + return "UNKNOWN" + name = s.get("model_name") or "" + sha = s.get("model_sha256") or "" + return f"{name} (sha256 {sha[:12]}…)" if sha else f"{name} (sha256 unavailable)" + + +def require_gallery_stamp_from_env() -> bool: + """SAE_REQUIRE_GALLERY_STAMP=1 → an unprovable binding is fatal, not a warning.""" + return os.environ.get("SAE_REQUIRE_GALLERY_STAMP", "0") not in ("", "0") + + +def check_embedder_stamp(built_with: dict | None, loading_with: dict | None, + gallery_desc: str = "gallery", + embedder_desc: str = "embedder") -> tuple[str, str]: + """Pure comparison. Returns (verdict, message); verdict is one of + match / weak_match / unstamped / unknown_embedder / mismatch. + + Same rules as compare_embedder_stamps() in src/gallery/embedder_stamp.cpp.""" + if _stamp_empty(built_with): + return "unstamped", ( + f"gallery '{gallery_desc}' carries no embedder stamp (GR-004).\n" + f" gallery was built with : UNKNOWN — this file predates model binding\n" + f" embedder now loaded : {describe_stamp(loading_with)} [{embedder_desc}]\n" + f" If these are not the same model every similarity from this run is\n" + f" meaningless but will look plausible. Rebuild or re-stamp the gallery\n" + f" (scripts/stamp_gallery.py), or run with SAE_REQUIRE_GALLERY_STAMP=1 to\n" + f" make this a hard error.") + + if _stamp_empty(loading_with): + return "unknown_embedder", ( + f"cannot identify the embedder being used against gallery " + f"'{gallery_desc}' (GR-004).\n" + f" gallery was built with : {describe_stamp(built_with)}\n" + f" embedder now loaded : UNKNOWN [{embedder_desc}]\n" + f" The binding cannot be checked, so it is not being checked.") + + mismatch_tail = ( + " Cosine similarities between embeddings from different models are\n" + " meaningless but look plausible. Rebuild the gallery with the loaded\n" + " model, or point the embedder at the model the gallery was built with.") + + if int(built_with.get("embed_dim", 512)) != int(loading_with.get("embed_dim", 512)): + return "mismatch", ( + f"gallery/embedder MODEL MISMATCH — refusing to run (GR-004).\n" + f" gallery was built with : {describe_stamp(built_with)}, " + f"dim={built_with.get('embed_dim')} [{gallery_desc}]\n" + f" embedder now loaded : {describe_stamp(loading_with)}, " + f"dim={loading_with.get('embed_dim')} [{embedder_desc}]\n" + f" Embedding dimensions differ; these are not the same space.") + + a, b = built_with.get("model_sha256", ""), loading_with.get("model_sha256", "") + if a and b: + if a == b: + note = "" + if built_with.get("model_name") != loading_with.get("model_name"): + note = (f" (gallery recorded it as '{built_with.get('model_name')}', " + f"loaded from '{loading_with.get('model_name')}' — " + f"same bytes, renamed file)") + return "match", f"embedder binding verified: {describe_stamp(built_with)}{note}" + return "mismatch", ( + f"gallery/embedder MODEL MISMATCH — refusing to run (GR-004).\n" + f" gallery was built with : {built_with.get('model_name')} sha256={a}\n" + f" [{gallery_desc}]\n" + f" embedder now loaded : {loading_with.get('model_name')} sha256={b}\n" + f" [{embedder_desc}]\n" + mismatch_tail) + + if built_with.get("model_name") and \ + built_with.get("model_name") == loading_with.get("model_name"): + return "weak_match", ( + f"embedder binding UNPROVEN for gallery '{gallery_desc}' (GR-004).\n" + f" gallery was built with : {describe_stamp(built_with)}\n" + f" embedder now loaded : {describe_stamp(loading_with)} [{embedder_desc}]\n" + f" Filenames agree but at least one SHA-256 is unavailable, so an\n" + f" in-place re-export under the same name would not be detected.") + + return "mismatch", ( + f"gallery/embedder MODEL MISMATCH — refusing to run (GR-004).\n" + f" gallery was built with : {describe_stamp(built_with)} [{gallery_desc}]\n" + f" embedder now loaded : {describe_stamp(loading_with)} [{embedder_desc}]\n" + + mismatch_tail) + + +def enforce_embedder_stamp(built_with, loading_with, gallery_desc, embedder_desc, + require_stamp: bool = False) -> str: + """Apply check_embedder_stamp: raise EmbedderMismatch when fatal, else warn. + + A mismatch is fatal unconditionally — there is no bypass, because a mismatch is + a known-wrong state, not an unknown one. The three "cannot prove it" verdicts + warn loudly and become fatal under require_stamp / SAE_REQUIRE_GALLERY_STAMP.""" + strict = require_stamp or require_gallery_stamp_from_env() + verdict, msg = check_embedder_stamp(built_with, loading_with, + gallery_desc, embedder_desc) + if verdict == "mismatch": + raise EmbedderMismatch(msg) + if strict and verdict != "match": + raise EmbedderMismatch( + msg + "\n (fatal because SAE_REQUIRE_GALLERY_STAMP is set)") + if verdict == "match": + print(f"[gallery] {msg}", file=sys.stderr) + else: + print(f"\n[gallery] ***** WARNING (GR-004) *****\n{msg}\n" + f"[gallery] ****************************\n", file=sys.stderr) + return verdict + + +def read_gallery_stamp(path) -> dict | None: + """The embedder stamp recorded in a gallery file, or None if unstamped. + + Handles both the HDF5 /embedder group and the legacy JSON "embedder" object.""" + path = Path(path) + if path.suffix in (".h5", ".hdf5"): + with h5py.File(path, "r") as f: + if "embedder" not in f: + return None + a = f["embedder"].attrs + return {"model_name": _as_str(a.get("model_name", "")), + "model_sha256": _as_str(a.get("model_sha256", "")), + "embed_dim": int(a.get("embed_dim", 512))} + data = json.loads(path.read_text()) + return data.get("embedder") or None + + +def verify_gallery_stamp(gallery_path, model_path=None, *, stamp=None, + embedder_desc: str | None = None, + require_stamp: bool = False) -> str: + """Load a gallery's stamp and check it against a model file (or an explicit + stamp, e.g. one read off an embedding dump). Raises EmbedderMismatch.""" + loading = stamp if stamp is not None else embedder_stamp(model_path) + return enforce_embedder_stamp(read_gallery_stamp(gallery_path), loading, + str(gallery_path), + embedder_desc or str(model_path or "unknown"), + require_stamp) + + +def _as_str(v) -> str: + return v.decode() if isinstance(v, bytes) else ("" if v is None else str(v)) diff --git a/scripts/stamp_gallery.py b/scripts/stamp_gallery.py new file mode 100644 index 0000000..230d776 --- /dev/null +++ b/scripts/stamp_gallery.py @@ -0,0 +1,86 @@ +#!/usr/bin/env python3 +"""stamp_gallery.py — bind an existing gallery to the embedder that built it. + +TRACES: GR-004 | SR-001 + +Galleries built before model binding carry no embedder stamp. They still load, +but every consumer warns that it cannot tell whether the gallery and the embedder +belong together — and under SAE_REQUIRE_GALLERY_STAMP=1 they refuse to run. + +This is the migration path, and the reason the unstamped case is a warning rather +than a hard failure: re-binding an existing gallery costs one command and no +re-embedding, so nobody has to choose between a bricked setup and a check they +route around. + + python scripts/stamp_gallery.py --gallery gallery.h5 \\ + --arcface models/LVFace-B_Glint360K.onnx + +The stamp is an ASSERTION: you are stating which model produced these vectors. +Nothing can verify it from the vectors themselves, which is exactly why the stamp +has to be written at build time going forward. Stamping the wrong model is worse +than leaving it unstamped, because it converts a loud warning into a false +all-clear — so --show it first if you are not certain. + + python scripts/stamp_gallery.py --gallery gallery.h5 --show +""" +from __future__ import annotations + +import argparse +import sys +from pathlib import Path + +import h5py + +sys.path.insert(0, str(Path(__file__).resolve().parent)) +from sae_gallery import (describe_stamp, embedder_stamp, # noqa: E402 + read_gallery_stamp) + + +def main() -> int: + p = argparse.ArgumentParser(description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter) + p.add_argument("--gallery", required=True, help="gallery .h5 to stamp in place") + p.add_argument("--arcface", help="the ONNX that built it (hashed into the stamp)") + p.add_argument("--show", action="store_true", help="print the current stamp and exit") + p.add_argument("--force", action="store_true", + help="overwrite an existing stamp (refused otherwise)") + args = p.parse_args() + + path = Path(args.gallery) + if path.suffix not in (".h5", ".hdf5"): + return err(f"{path}: only HDF5 galleries can be stamped in place") + + current = read_gallery_stamp(path) + print(f"{path}: current stamp = {describe_stamp(current)}", file=sys.stderr) + if args.show: + return 0 + if not args.arcface: + return err("--arcface is required (or use --show)") + if current and not args.force: + return err("gallery is already stamped — pass --force to overwrite, but be " + "sure: a wrong stamp turns a warning into a false all-clear") + + stamp = embedder_stamp(args.arcface) + if not stamp["model_sha256"]: + return err(f"cannot hash {args.arcface} — refusing to write a name-only " + "stamp, which would claim more certainty than it has") + + with h5py.File(path, "r+") as f: + if "embedder" in f: + del f["embedder"] + g = f.create_group("embedder") + g.attrs["model_name"] = stamp["model_name"] + g.attrs["model_sha256"] = stamp["model_sha256"] + g.attrs["embed_dim"] = stamp["embed_dim"] + + print(f"{path}: stamped with {describe_stamp(stamp)}", file=sys.stderr) + return 0 + + +def err(msg: str) -> int: + print(f"[stamp_gallery] {msg}", file=sys.stderr) + return 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/src/config.hpp b/src/config.hpp index f81ef71..75b7900 100644 --- a/src/config.hpp +++ b/src/config.hpp @@ -40,6 +40,14 @@ struct Config { float detector_conf{0.5f}; float detector_nms{0.4f}; + /// TRACES: GR-004 | SR-001 + // Gallery ↔ embedder binding. A gallery built with a different model than the + // one loaded here is a hard error, always. This flag additionally promotes + // "cannot prove they match" (unstamped legacy gallery, or a name-only match + // because the ONNX could not be hashed) from a loud warning to a hard error. + // Also settable via SAE_REQUIRE_GALLERY_STAMP=1. Measurement runs want it on. + bool require_gallery_stamp{false}; // --require-gallery-stamp + // ── Recognition (ArcFace ONNX) ──────────────────────────────────────────── std::string arcface_model; std::string arcface_engine; // optional path to a pre-built TRT engine; bypasses ORT diff --git a/src/gallery/embedder_stamp.cpp b/src/gallery/embedder_stamp.cpp new file mode 100644 index 0000000..c80e9cd --- /dev/null +++ b/src/gallery/embedder_stamp.cpp @@ -0,0 +1,326 @@ +/// TRACES: GR-004 | SR-001 +#include "embedder_stamp.hpp" +#include "types.hpp" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace fs = std::filesystem; + +// ── SHA-256 (FIPS 180-4) ────────────────────────────────────────────────────── +// Self-contained rather than pulled from OpenSSL: the gallery library already +// links OpenCV, HDF5, FFmpeg and a GPU backend, and the unit tests deliberately +// link none of those crypto stacks. ~80 lines of table-driven code is cheaper +// than another find_package that CI has to satisfy on an Intel N100. +namespace { + +struct Sha256 { + uint32_t h[8] = {0x6a09e667u, 0xbb67ae85u, 0x3c6ef372u, 0xa54ff53au, + 0x510e527fu, 0x9b05688cu, 0x1f83d9abu, 0x5be0cd19u}; + uint64_t len = 0; + uint8_t buf[64]{}; + size_t buf_n = 0; + + static uint32_t ror(uint32_t x, int n) { return (x >> n) | (x << (32 - n)); } + + void block(const uint8_t* p) { + static const uint32_t k[64] = { + 0x428a2f98u,0x71374491u,0xb5c0fbcfu,0xe9b5dba5u,0x3956c25bu,0x59f111f1u, + 0x923f82a4u,0xab1c5ed5u,0xd807aa98u,0x12835b01u,0x243185beu,0x550c7dc3u, + 0x72be5d74u,0x80deb1feu,0x9bdc06a7u,0xc19bf174u,0xe49b69c1u,0xefbe4786u, + 0x0fc19dc6u,0x240ca1ccu,0x2de92c6fu,0x4a7484aau,0x5cb0a9dcu,0x76f988dau, + 0x983e5152u,0xa831c66du,0xb00327c8u,0xbf597fc7u,0xc6e00bf3u,0xd5a79147u, + 0x06ca6351u,0x14292967u,0x27b70a85u,0x2e1b2138u,0x4d2c6dfcu,0x53380d13u, + 0x650a7354u,0x766a0abbu,0x81c2c92eu,0x92722c85u,0xa2bfe8a1u,0xa81a664bu, + 0xc24b8b70u,0xc76c51a3u,0xd192e819u,0xd6990624u,0xf40e3585u,0x106aa070u, + 0x19a4c116u,0x1e376c08u,0x2748774cu,0x34b0bcb5u,0x391c0cb3u,0x4ed8aa4au, + 0x5b9cca4fu,0x682e6ff3u,0x748f82eeu,0x78a5636fu,0x84c87814u,0x8cc70208u, + 0x90befffau,0xa4506cebu,0xbef9a3f7u,0xc67178f2u}; + uint32_t w[64]; + for (int i = 0; i < 16; ++i) + w[i] = (uint32_t(p[i * 4]) << 24) | (uint32_t(p[i * 4 + 1]) << 16) | + (uint32_t(p[i * 4 + 2]) << 8) | uint32_t(p[i * 4 + 3]); + for (int i = 16; i < 64; ++i) { + uint32_t s0 = ror(w[i - 15], 7) ^ ror(w[i - 15], 18) ^ (w[i - 15] >> 3); + uint32_t s1 = ror(w[i - 2], 17) ^ ror(w[i - 2], 19) ^ (w[i - 2] >> 10); + w[i] = w[i - 16] + s0 + w[i - 7] + s1; + } + uint32_t a = h[0], b = h[1], c = h[2], d = h[3]; + uint32_t e = h[4], f = h[5], g = h[6], hh = h[7]; + for (int i = 0; i < 64; ++i) { + uint32_t S1 = ror(e, 6) ^ ror(e, 11) ^ ror(e, 25); + uint32_t ch = (e & f) ^ (~e & g); + uint32_t t1 = hh + S1 + ch + k[i] + w[i]; + uint32_t S0 = ror(a, 2) ^ ror(a, 13) ^ ror(a, 22); + uint32_t mj = (a & b) ^ (a & c) ^ (b & c); + uint32_t t2 = S0 + mj; + hh = g; g = f; f = e; e = d + t1; + d = c; c = b; b = a; a = t1 + t2; + } + h[0] += a; h[1] += b; h[2] += c; h[3] += d; + h[4] += e; h[5] += f; h[6] += g; h[7] += hh; + } + + void update(const uint8_t* p, size_t n) { + len += n; + while (n) { + size_t take = std::min(n, size_t(64) - buf_n); + std::memcpy(buf + buf_n, p, take); + buf_n += take; p += take; n -= take; + if (buf_n == 64) { block(buf); buf_n = 0; } + } + } + + std::string hex() { + uint64_t bits = len * 8; + uint8_t pad = 0x80; + update(&pad, 1); + uint8_t zero = 0; + while (buf_n != 56) update(&zero, 1); + uint8_t tail[8]; + for (int i = 0; i < 8; ++i) tail[i] = uint8_t(bits >> (56 - i * 8)); + // update() would re-count these into len, but len is already frozen in bits. + std::memcpy(buf + buf_n, tail, 8); + block(buf); + buf_n = 0; + + static const char* d = "0123456789abcdef"; + std::string out; + out.reserve(64); + for (int i = 0; i < 8; ++i) + for (int s = 28; s >= 0; s -= 4) + out += d[(h[i] >> s) & 0xF]; + return out; + } +}; + +// (path, mtime, size) → digest. Hashing a 250 MB ONNX is cheap but not free, and +// the optimizer constructs many networks in one process against the same model. +std::mutex g_hash_mu; +std::map g_hash_cache; + +std::string short_hash(const std::string& hex) { + return hex.size() > 12 ? hex.substr(0, 12) + "…" : hex; +} + +} // namespace + +std::string sha256_hex(const std::string& bytes) { + Sha256 s; + s.update(reinterpret_cast(bytes.data()), bytes.size()); + return s.hex(); +} + +std::string sha256_file_hex(const std::string& path) { + if (path.empty()) return ""; + + std::error_code ec; + auto size = fs::file_size(path, ec); + if (ec) return ""; + auto mtime = fs::last_write_time(path, ec); + if (ec) return ""; + + std::ostringstream key; + key << path << '|' << size << '|' + << mtime.time_since_epoch().count(); + + { + std::lock_guard lk(g_hash_mu); + auto it = g_hash_cache.find(key.str()); + if (it != g_hash_cache.end()) return it->second; + } + + std::ifstream f(path, std::ios::binary); + if (!f) return ""; + Sha256 s; + std::vector chunk(1 << 20); + while (f) { + f.read(chunk.data(), static_cast(chunk.size())); + std::streamsize got = f.gcount(); + if (got > 0) s.update(reinterpret_cast(chunk.data()), + static_cast(got)); + } + std::string hex = s.hex(); + + std::lock_guard lk(g_hash_mu); + g_hash_cache[key.str()] = hex; + return hex; +} + +// ── EmbedderStamp ───────────────────────────────────────────────────────────── + +std::string EmbedderStamp::describe() const { + std::string name = model_name.empty() ? "" : model_name; + if (model_sha256.empty()) + return name + " (sha256 unavailable)"; + return name + " (sha256 " + short_hash(model_sha256) + ")"; +} + +EmbedderStamp make_embedder_stamp(const std::string& model_path) { + EmbedderStamp s; + if (model_path.empty()) return s; + s.model_name = fs::path(model_path).filename().string(); + s.model_sha256 = sha256_file_hex(model_path); + if (s.model_sha256.empty()) + std::cerr << "[gallery] cannot hash embedder model " << model_path + << " — model binding falls back to filename only (GR-004)\n"; + return s; +} + +bool require_gallery_stamp_from_env() { + const char* v = std::getenv("SAE_REQUIRE_GALLERY_STAMP"); + return v && *v && std::strcmp(v, "0") != 0; +} + +// ── Comparison ──────────────────────────────────────────────────────────────── + +StampCheck compare_embedder_stamps(const EmbedderStamp& built_with, + const EmbedderStamp& loading_with, + const std::string& gallery_desc, + const std::string& embedder_desc) { + StampCheck out; + std::ostringstream m; + + // The gallery predates GR-004 (or was written by a tool that does not stamp). + if (built_with.empty()) { + out.verdict = StampVerdict::unstamped; + m << "gallery '" << gallery_desc << "' carries no embedder stamp (GR-004).\n" + << " gallery was built with : UNKNOWN — this file predates model binding\n" + << " embedder now loaded : " << loading_with.describe() + << " [" << embedder_desc << "]\n" + << " If these are not the same model every similarity from this run is\n" + << " meaningless but will look plausible. Rebuild or re-stamp the gallery\n" + << " (scripts/stamp_gallery.py), or run with SAE_REQUIRE_GALLERY_STAMP=1 to\n" + << " make this a hard error."; + out.message = m.str(); + return out; + } + + // Gallery is stamped but we cannot say what is about to embed. + if (loading_with.empty()) { + out.verdict = StampVerdict::unknown_embedder; + m << "cannot identify the embedder being used against gallery '" + << gallery_desc << "' (GR-004).\n" + << " gallery was built with : " << built_with.describe() << "\n" + << " embedder now loaded : UNKNOWN [" << embedder_desc << "]\n" + << " The binding cannot be checked, so it is not being checked."; + out.message = m.str(); + return out; + } + + const bool have_both_hashes = + !built_with.model_sha256.empty() && !loading_with.model_sha256.empty(); + + // Embedding width disagreeing is a mismatch on its own terms — different + // spaces entirely, and it will not even be caught by a cosine that "looks fine". + if (built_with.embed_dim != loading_with.embed_dim) { + out.verdict = StampVerdict::mismatch; + m << "gallery/embedder MODEL MISMATCH — refusing to run (GR-004).\n" + << " gallery was built with : " << built_with.describe() + << ", dim=" << built_with.embed_dim << " [" << gallery_desc << "]\n" + << " embedder now loaded : " << loading_with.describe() + << ", dim=" << loading_with.embed_dim << " [" << embedder_desc << "]\n" + << " Embedding dimensions differ; these are not the same space."; + out.message = m.str(); + return out; + } + + if (have_both_hashes) { + if (built_with.model_sha256 == loading_with.model_sha256) { + out.verdict = StampVerdict::match; + m << "embedder binding verified: " << built_with.describe(); + if (built_with.model_name != loading_with.model_name) + m << " (gallery recorded it as '" << built_with.model_name + << "', loaded from '" << loading_with.model_name + << "' — same bytes, renamed file)"; + out.message = m.str(); + return out; + } + out.verdict = StampVerdict::mismatch; + m << "gallery/embedder MODEL MISMATCH — refusing to run (GR-004).\n" + << " gallery was built with : " << built_with.model_name + << " sha256=" << built_with.model_sha256 << "\n" + << " [" << gallery_desc << "]\n" + << " embedder now loaded : " << loading_with.model_name + << " sha256=" << loading_with.model_sha256 << "\n" + << " [" << embedder_desc << "]\n" + << " Cosine similarities between embeddings from different models are\n" + << " meaningless but look plausible. Rebuild the gallery with the loaded\n" + << " model, or point the embedder at the model the gallery was built with."; + out.message = m.str(); + return out; + } + + // One side has no hash (e.g. a TRT deployment with the .onnx absent). Names + // are all we have; agreeing on them is evidence, not proof. + if (!built_with.model_name.empty() && + built_with.model_name == loading_with.model_name) { + out.verdict = StampVerdict::weak_match; + m << "embedder binding UNPROVEN for gallery '" << gallery_desc << "' (GR-004).\n" + << " gallery was built with : " << built_with.describe() << "\n" + << " embedder now loaded : " << loading_with.describe() + << " [" << embedder_desc << "]\n" + << " Filenames agree but at least one SHA-256 is unavailable, so an\n" + << " in-place re-export under the same name would not be detected."; + out.message = m.str(); + return out; + } + + out.verdict = StampVerdict::mismatch; + m << "gallery/embedder MODEL MISMATCH — refusing to run (GR-004).\n" + << " gallery was built with : " << built_with.describe() + << " [" << gallery_desc << "]\n" + << " embedder now loaded : " << loading_with.describe() + << " [" << embedder_desc << "]\n" + << " Cosine similarities between embeddings from different models are\n" + << " meaningless but look plausible. Rebuild the gallery with the loaded\n" + << " model, or point the embedder at the model the gallery was built with."; + out.message = m.str(); + return out; +} + +void enforce_embedder_stamp(const EmbedderStamp& built_with, + const EmbedderStamp& loading_with, + const std::string& gallery_desc, + const std::string& embedder_desc, + bool require_stamp) { + const bool strict = require_stamp || require_gallery_stamp_from_env(); + StampCheck chk = compare_embedder_stamps(built_with, loading_with, + gallery_desc, embedder_desc); + + if (chk.fatal(strict)) { + if (chk.verdict != StampVerdict::mismatch) + throw std::runtime_error(chk.message + + "\n (fatal because SAE_REQUIRE_GALLERY_STAMP / --require-gallery-stamp is set)"); + throw std::runtime_error(chk.message); + } + + if (chk.verdict == StampVerdict::match) { + std::cerr << "[gallery] " << chk.message << "\n"; + } else { + std::cerr << "\n[gallery] ***** WARNING (GR-004) *****\n" + << chk.message << "\n" + << "[gallery] ****************************\n\n"; + } +} + +void verify_gallery_embedder(const ActorGallery& gallery, + const std::string& gallery_path, + const std::string& arcface_model_path, + bool require_stamp) { + enforce_embedder_stamp(gallery.embedder, + make_embedder_stamp(arcface_model_path), + gallery_path, + arcface_model_path.empty() ? "no --arcface given" + : arcface_model_path, + require_stamp); +} diff --git a/src/gallery/embedder_stamp.hpp b/src/gallery/embedder_stamp.hpp new file mode 100644 index 0000000..88eef9a --- /dev/null +++ b/src/gallery/embedder_stamp.hpp @@ -0,0 +1,115 @@ +#pragma once +/// TRACES: GR-004 | SR-001 +// +// Gallery ↔ embedder binding. +// +// A gallery is only valid for the embedder that built it. Cosine similarities +// between embeddings from two different models are meaningless but *look* +// plausible — nothing crashes, nothing is obviously wrong, and every number +// measured downstream is quietly garbage. So the embedder's identity is stamped +// into the gallery at build time and checked by every consumer at load time. +// +// ── What identifies an embedder ─────────────────────────────────────────────── +// Two fields, carried together: +// +// model_name basename of the model file, e.g. "LVFace-B_Glint360K.onnx" +// model_sha256 hex SHA-256 of that file's bytes +// +// The hash is what *decides*; the name is what a human *reads*. Neither alone is +// enough: +// +// • A name alone is a promise, not a fact. Models get re-exported, re-quantised +// and overwritten in place under an unchanged filename — which is precisely +// the case where the weights differ and nothing else does. A name-only stamp +// is blind to exactly the failure it exists to catch. +// • A hash alone is correct but unreadable: "expected 3f2a… got 9c1b…" tells an +// operator nothing about what to do next. +// +// SHA-256 over the file bytes is derived from the artefact rather than asserted +// about it, is stable across machines and filesystems, and needs no registry to +// be kept up to date. Cost is ~0.1 s for a 250 MB ONNX, paid once per process +// (results are memoised on path+mtime+size), which is noise next to model load. +// +// ── Degraded and legacy cases ───────────────────────────────────────────────── +// A TRT-backend deployment may run from a prebuilt .engine with the source .onnx +// absent, so the hash cannot be computed. Then the name is compared alone and the +// result is reported as a *weak* match — believed, not proven. +// +// Galleries built before GR-004 carry no stamp at all. They warn loudly rather +// than fail, because the state is unknown rather than known-bad, and because +// hard-failing every pre-existing gallery would make the check something people +// route around rather than trust. Set require_stamp (or SAE_REQUIRE_GALLERY_STAMP=1) +// to promote "unknown" to a hard error — that is the mode measurement work runs in. +// +// A *mismatch* is always fatal, in every mode, with no bypass. + +#include +#include + +struct EmbedderStamp { + std::string model_name; // basename of the model file + std::string model_sha256; // lowercase hex SHA-256 of the file's bytes ("" = unavailable) + int32_t embed_dim{512}; + + bool empty() const { return model_name.empty() && model_sha256.empty(); } + + // "LVFace-B_Glint360K.onnx (sha256 3f2a1c4d…)" — for error messages. + std::string describe() const; +}; + +// Identify the model at `model_path`. Missing/unreadable file → name filled from +// the path, hash left empty (the weak-match path). Empty path → empty stamp. +EmbedderStamp make_embedder_stamp(const std::string& model_path); + +enum class StampVerdict { + match, // hashes agree — binding proven + weak_match, // names agree, no hash on one side — believed, unproven + unstamped, // gallery predates GR-004 / was written without a stamp + unknown_embedder, // gallery is stamped but the loaded embedder can't be identified + mismatch, // proven different models — always fatal +}; + +struct StampCheck { + StampVerdict verdict{StampVerdict::match}; + std::string message; // human-readable, names BOTH sides + + // A mismatch is fatal unconditionally. The three "cannot prove it" verdicts + // are fatal only in strict mode. + bool fatal(bool require_stamp) const { + return verdict == StampVerdict::mismatch || + (require_stamp && verdict != StampVerdict::match); + } +}; + +// Pure comparison — no file I/O, no model loading. This is the unit under test. +// `gallery_desc`/`embedder_desc` are only used to make the message locatable +// (a gallery path, a dump path, "the embedder being loaded", …). +StampCheck compare_embedder_stamps(const EmbedderStamp& built_with, + const EmbedderStamp& loading_with, + const std::string& gallery_desc = "gallery", + const std::string& embedder_desc = "embedder"); + +// Apply the comparison: throw std::runtime_error on a fatal verdict, otherwise +// log to stderr. `require_stamp` is OR-ed with SAE_REQUIRE_GALLERY_STAMP. +void enforce_embedder_stamp(const EmbedderStamp& built_with, + const EmbedderStamp& loading_with, + const std::string& gallery_desc, + const std::string& embedder_desc, + bool require_stamp); + +// Convenience for the common consumer shape: "I loaded this gallery and I am +// about to embed with this model file." Hashes the model, then enforces. +struct ActorGallery; +void verify_gallery_embedder(const ActorGallery& gallery, + const std::string& gallery_path, + const std::string& arcface_model_path, + bool require_stamp); + +// SAE_REQUIRE_GALLERY_STAMP=1 → treat an unprovable binding as fatal. +bool require_gallery_stamp_from_env(); + +// Lowercase hex SHA-256. Exposed so a test can pin the digest against the +// published vectors, which is what guarantees the C++ and Python (hashlib) +// stamps of the same file agree. +std::string sha256_hex(const std::string& bytes); +std::string sha256_file_hex(const std::string& path); // "" if unreadable diff --git a/src/gallery/gallery_builder.cpp b/src/gallery/gallery_builder.cpp index 3543de2..fa8b549 100644 --- a/src/gallery/gallery_builder.cpp +++ b/src/gallery/gallery_builder.cpp @@ -1,5 +1,6 @@ #include "gallery_builder.hpp" #include "config.hpp" +#include "embedder_stamp.hpp" #include "face_utils.hpp" #include "inference/face_detector.hpp" #include "inference/face_embedder.hpp" @@ -41,6 +42,12 @@ ActorGallery build_gallery(const BuildConfig& cfg) { ActorGallery gallery; + /// TRACES: GR-004 | SR-001 + // Stamp before the first embedding exists, so there is no window in which a + // gallery holds vectors without recording what produced them. + gallery.embedder = make_embedder_stamp(cfg.arcface_model); + std::cerr << "[build_gallery] embedder: " << gallery.embedder.describe() << "\n"; + for (const auto& actor_dir : fs::directory_iterator(cfg.gallery_root)) { if (!actor_dir.is_directory()) continue; diff --git a/src/gallery/gallery_store.cpp b/src/gallery/gallery_store.cpp index 798c351..b8daf62 100644 --- a/src/gallery/gallery_store.cpp +++ b/src/gallery/gallery_store.cpp @@ -79,6 +79,21 @@ static ActorGallery load_gallery_hdf5(const std::string& path) { gallery.actors.push_back(std::move(actor)); } + /// TRACES: GR-004 | SR-001 + // Absent /embedder group == a gallery written before model binding existed. + // It stays readable; verify_gallery_embedder() decides what that means. + if (file.nameExists("embedder")) { + H5::Group eg = file.openGroup("embedder"); + H5::StrType str(H5::PredType::C_S1, H5T_VARIABLE); + if (eg.attrExists("model_name")) + eg.openAttribute("model_name").read(str, gallery.embedder.model_name); + if (eg.attrExists("model_sha256")) + eg.openAttribute("model_sha256").read(str, gallery.embedder.model_sha256); + if (eg.attrExists("embed_dim")) + eg.openAttribute("embed_dim").read(H5::PredType::NATIVE_INT32, + &gallery.embedder.embed_dim); + } + if (file.nameExists("calibration")) { H5::Group cal = file.openGroup("calibration"); cal.openAttribute("a").read(H5::PredType::NATIVE_FLOAT, &gallery.calib_a); @@ -149,6 +164,20 @@ static void save_gallery_hdf5(const std::string& path, const ActorGallery& galle write_str_dataset(file, "name", name); write_str_dataset(file, "source_images", src_images); + /// TRACES: GR-004 | SR-001 + // Bind the file to the embedder that produced its vectors. Written only when + // known — an empty stamp must round-trip as "unstamped", not as a stamp + // claiming an unnamed model. + if (!gallery.embedder.empty()) { + H5::Group eg = file.createGroup("embedder"); + H5::DataSpace scalar(H5S_SCALAR); + H5::StrType str(H5::PredType::C_S1, H5T_VARIABLE); + eg.createAttribute("model_name", str, scalar).write(str, gallery.embedder.model_name); + eg.createAttribute("model_sha256", str, scalar).write(str, gallery.embedder.model_sha256); + eg.createAttribute("embed_dim", H5::PredType::NATIVE_INT32, scalar) + .write(H5::PredType::NATIVE_INT32, &gallery.embedder.embed_dim); + } + if (gallery.calib_hash != 0) { H5::Group cal = file.createGroup("calibration"); H5::DataSpace scalar(H5S_SCALAR); @@ -186,6 +215,17 @@ ActorGallery load_gallery(const std::string& path) { << std::chrono::duration(t1 - t0).count() << "s\n"; ActorGallery gallery; + + /// TRACES: GR-004 | SR-001 + // Optional top-level "embedder" object, matching the HDF5 /embedder group. + // Written by the JSON-era helper scripts; absent in anything older. + if (j.contains("embedder") && j.at("embedder").is_object()) { + const auto& je = j.at("embedder"); + gallery.embedder.model_name = je.value("model_name", ""); + gallery.embedder.model_sha256 = je.value("model_sha256", ""); + gallery.embedder.embed_dim = je.value("embed_dim", 512); + } + for (const auto& ja : j.at("actors")) { ActorGallery::Actor actor; actor.imdb_id = ja.value("imdb_id", ""); diff --git a/src/gallery/gallery_store.hpp b/src/gallery/gallery_store.hpp index c038587..4566811 100644 --- a/src/gallery/gallery_store.hpp +++ b/src/gallery/gallery_store.hpp @@ -12,6 +12,11 @@ // /count int32 [A] number of refs for actor a // /imdb_id /tmdb_id /jellyfin_id /name : variable-length string [A] // /source_images : variable-length string [N], parallel to /embeddings rows +// /embedder/model_name : scalar var-len string attr — embedder file basename +// /embedder/model_sha256 : scalar var-len string attr — SHA-256 of that file +// /embedder/embed_dim : scalar int32 attr +// The GR-004 model binding. Absent group == unstamped +// (pre-GR-004 file); see gallery/embedder_stamp.hpp. // /calibration/a, /b : scalar float32 attrs — Platt-sigmoid P(match|sim) fit // /calibration/valid : scalar int8 attr (0/1) // /calibration/hash : scalar uint64 attr — hash of the embeddings the fit @@ -19,6 +24,7 @@ // // Legacy JSON format (read-only): // { +// "embedder": {"model_name": "...", "model_sha256": "...", "embed_dim": 512}, // "actors": [ // { // "imdb_id": "nm0000093", // optional, "" if unknown diff --git a/src/kpn_bindings.cpp b/src/kpn_bindings.cpp index c105b6b..f518009 100644 --- a/src/kpn_bindings.cpp +++ b/src/kpn_bindings.cpp @@ -15,6 +15,7 @@ #include "types.hpp" #include "config.hpp" +#include "gallery/embedder_stamp.hpp" #include "gallery/gallery_store.hpp" #include "nodes/face_tracker_node.hpp" #include "nodes/identity_matcher_node.hpp" @@ -163,6 +164,9 @@ static Config config_from_dict(nb::dict d) { getd("anneal_sec", cfg.anneal_sec); // gallery expansion (usually off for sweeps; expose so it can be toggled) if (d.contains("expand_gallery")) cfg.expand_gallery = nb::cast(d["expand_gallery"]); + /// TRACES: GR-004 | SR-001 + if (d.contains("require_gallery_stamp")) + cfg.require_gallery_stamp = nb::cast(d["require_gallery_stamp"]); return cfg; } @@ -210,8 +214,17 @@ NB_MODULE(sae_kpn, m) { net.add(std::move(name), std::move(node)); }, "net"_a, "name"_a, "config"_a, "capacity"_a = 16); + /// TRACES: GR-004 | SR-001 + // embedder_model / embedder_sha256 identify whatever produced the embeddings + // that will be fed in. In a replay those come from the dump's own stamp (see + // scripts/optimizer/SCHEMA.md), because there is no live embedder in the + // network — the dump *is* the embedder as far as this gallery is concerned. + // Passing neither leaves the binding unverifiable, which warns loudly and is + // fatal under SAE_REQUIRE_GALLERY_STAMP. m.def("add_identity_matcher", [](Net& net, std::string name, std::string gallery_path, - nb::dict cfg_dict, std::size_t cap) { + nb::dict cfg_dict, std::size_t cap, + std::string embedder_model, + std::string embedder_sha256) { Config cfg = config_from_dict(cfg_dict); cfg.gallery_path = gallery_path; // needed to persist refreshed calibration back // Cache loaded galleries by path so a threshold sweep (many networks, same @@ -222,11 +235,24 @@ NB_MODULE(sae_kpn, m) { if (it == cache.end()) it = cache.emplace(gallery_path, std::make_shared(load_gallery(gallery_path))).first; + + // Checked on every construction, not only on the cache miss: the same + // process may replay several dumps against one cached gallery. + EmbedderStamp feeding; + feeding.model_name = std::move(embedder_model); + feeding.model_sha256 = std::move(embedder_sha256); + enforce_embedder_stamp(it->second->embedder, feeding, gallery_path, + feeding.model_name.empty() + ? "embeddings fed into this network" + : feeding.model_name, + cfg.require_gallery_stamp); + auto node = std::make_shared, kpn::out<"matched">>>( cap, *it->second, cfg); net.add(std::move(name), std::move(node)); - }, "net"_a, "name"_a, "gallery"_a, "config"_a, "capacity"_a = 16); + }, "net"_a, "name"_a, "gallery"_a, "config"_a, "capacity"_a = 16, + "embedder_model"_a = "", "embedder_sha256"_a = ""); m.def("add_scene_tracker", [](Net& net, std::string name, nb::dict cfg_dict, std::size_t cap) { Config cfg = config_from_dict(cfg_dict); diff --git a/src/main.cpp b/src/main.cpp index 7e638ae..8dacbd6 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -45,6 +45,7 @@ #include "config.hpp" #include "types.hpp" +#include "gallery/embedder_stamp.hpp" #include "gallery/gallery_store.hpp" #include "nodes/frame_source_node.hpp" #include "nodes/camera_position_change_detector_node.hpp" @@ -114,6 +115,7 @@ static Config parse_args(int argc, char** argv) { else if (arg("--detector")) cfg.detector_model = next(); else if (arg("--detector-engine")) cfg.detector_engine = next(); else if (arg("--arcface")) cfg.arcface_model = next(); + else if (arg("--require-gallery-stamp")) cfg.require_gallery_stamp = true; else if (arg("--arcface-engine")) cfg.arcface_engine = next(); else if (arg("--conf")) cfg.detector_conf = std::stof(next()); else if (arg("--max-faces")) cfg.max_faces = std::stoi(next()); @@ -167,6 +169,11 @@ int main(int argc, char** argv) { ActorGallery gallery; try { gallery = load_gallery(cfg.gallery_path); + /// TRACES: GR-004 | SR-001 + // Hard startup error before a single frame is decoded: a gallery built + // with another embedder yields plausible-looking, meaningless matches. + verify_gallery_embedder(gallery, cfg.gallery_path, cfg.arcface_model, + cfg.require_gallery_stamp); } catch (const std::exception& e) { std::cerr << "Gallery error: " << e.what() << "\n"; return 1; diff --git a/src/nodes/embedding_dump_node.hpp b/src/nodes/embedding_dump_node.hpp index 1541dc1..13cf361 100644 --- a/src/nodes/embedding_dump_node.hpp +++ b/src/nodes/embedding_dump_node.hpp @@ -1,6 +1,7 @@ #pragma once #include "types.hpp" #include "config.hpp" +#include "gallery/embedder_stamp.hpp" #include @@ -25,7 +26,13 @@ struct EmbeddingDumpFunc { : path_(cfg.dump_embeddings_path), movie_(cfg.movie_path), sample_fps_(cfg.sample_fps), done_(done) { - std::cerr << "[embedding_dump] writing " << path_ << "\n"; + /// TRACES: GR-004 | SR-001 + // A dump is a bag of embeddings with no model attached, replayed against a + // gallery hours or weeks later — the same silent cross-model hazard as the + // gallery itself, so it carries the same stamp. + stamp_ = make_embedder_stamp(cfg.arcface_model); + std::cerr << "[embedding_dump] writing " << path_ + << " embedder: " << stamp_.describe() << "\n"; } void operator()(EmbeddedSceneFrame ef) { @@ -91,6 +98,9 @@ private: H5::StrType str(H5::PredType::C_S1, H5T_VARIABLE); auto mv = file.createAttribute("movie", str, scalar); mv.write(str, movie_); + /// TRACES: GR-004 | SR-001 + file.createAttribute("embedder_model", str, scalar).write(str, stamp_.model_name); + file.createAttribute("embedder_sha256", str, scalar).write(str, stamp_.model_sha256); H5::Group frames = file.createGroup("frames"); write_vec(frames, "timestamp_sec", ts_, H5::PredType::NATIVE_DOUBLE); @@ -110,7 +120,8 @@ private: << conf_.size() << " faces → " << path_ << "\n"; } - std::string path_, movie_; + std::string path_, movie_; + EmbedderStamp stamp_; float sample_fps_; std::atomic& done_; std::atomic written_{false}; diff --git a/src/scene_preview.cpp b/src/scene_preview.cpp index b1a3e27..e7892cf 100644 --- a/src/scene_preview.cpp +++ b/src/scene_preview.cpp @@ -21,6 +21,7 @@ #include "config.hpp" #include "types.hpp" +#include "gallery/embedder_stamp.hpp" #include "gallery/gallery_store.hpp" #include "nodes/frame_source_node.hpp" #include "nodes/camera_position_change_detector_node.hpp" @@ -76,6 +77,7 @@ static Config parse_args(int argc, char** argv) { else if (arg("--detector-engine")) cfg.detector_engine = next(); else if (arg("--arcface")) cfg.arcface_model = next(); else if (arg("--arcface-engine")) cfg.arcface_engine = next(); + else if (arg("--require-gallery-stamp")) cfg.require_gallery_stamp = true; else if (arg("--conf")) cfg.detector_conf = std::stof(next()); else if (arg("--max-faces")) cfg.max_faces = std::stoi(next()); else if (arg("--min-face-px")) cfg.min_face_px = std::stof(next()); @@ -126,7 +128,12 @@ int main(int argc, char** argv) { } ActorGallery gallery; - try { gallery = load_gallery(cfg.gallery_path); } + try { + gallery = load_gallery(cfg.gallery_path); + /// TRACES: GR-004 | SR-001 + verify_gallery_embedder(gallery, cfg.gallery_path, cfg.arcface_model, + cfg.require_gallery_stamp); + } catch (const std::exception& e) { std::cerr << "Gallery error: " << e.what() << "\n"; return 1; diff --git a/src/types.hpp b/src/types.hpp index 08afb42..02ccfc5 100644 --- a/src/types.hpp +++ b/src/types.hpp @@ -6,6 +6,8 @@ #include +#include "gallery/embedder_stamp.hpp" + // ── Embedding ───────────────────────────────────────────────────────────────── // 512-dim L2-normalised ArcFace embedding using Embedding = std::array; @@ -136,6 +138,11 @@ struct ActorGallery { }; std::vector actors; + /// TRACES: GR-004 | SR-001 + // Which embedder produced every embedding above. Empty == the file predates + // model binding; see gallery/embedder_stamp.hpp for what is checked and why. + EmbedderStamp embedder; + // Cached Platt-sigmoid calibration (see gallery/gallery_calibration.hpp), // stored alongside the gallery in HDF5 so it never needs recomputing // unless the reference embeddings actually change. calib_valid=false and diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index c7d2a9b..4763f15 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -25,6 +25,7 @@ add_executable(sae_tests ${CMAKE_SOURCE_DIR}/src/backends/gemm_backend.cpp ${CMAKE_SOURCE_DIR}/src/gallery/gallery_store.cpp ${CMAKE_SOURCE_DIR}/src/audio_signature.cpp + ${CMAKE_SOURCE_DIR}/src/gallery/embedder_stamp.cpp ) target_include_directories(sae_tests PRIVATE ${CMAKE_SOURCE_DIR}/src) # SAE_GEMM_CPU: build the CPU reference GEMM regardless of the main backend. diff --git a/tests/test_gallery_store.cpp b/tests/test_gallery_store.cpp index 40b2567..1979e83 100644 --- a/tests/test_gallery_store.cpp +++ b/tests/test_gallery_store.cpp @@ -1,13 +1,17 @@ // Unit tests for gallery (de)serialisation: HDF5 round-trip fidelity (the only // format save_gallery writes), legacy JSON read back-compat (optional field -// defaults, the legacy "jellyfin_person_id" fallback). GPU-free, model-free. +// defaults, the legacy "jellyfin_person_id" fallback), and the GR-004 embedder +// stamp. GPU-free, model-free — the stamp tests exercise the comparison logic +// with synthetic stamps and never load an ONNX, so they run on CI's Intel N100. #include +#include "gallery/embedder_stamp.hpp" #include "gallery/gallery_store.hpp" #include "types.hpp" #include #include +#include #include #include @@ -27,6 +31,22 @@ Embedding make_embedding(float base) { return e; } +// A stamp built by hand — no ONNX is read, so these tests never need a model. +EmbedderStamp stamp(const std::string& name, const std::string& sha, int32_t dim = 512) { + EmbedderStamp s; + s.model_name = name; + s.model_sha256 = sha; + s.embed_dim = dim; + return s; +} + +const std::string kShaA(64, 'a'); +const std::string kShaB(64, 'b'); + +bool mentions(const std::string& haystack, const std::string& needle) { + return haystack.find(needle) != std::string::npos; +} + } // namespace TEST_CASE("gallery save/load round-trips actors and embeddings", "[gallery]") { @@ -153,3 +173,233 @@ TEST_CASE("load_gallery reads the legacy jellyfin_person_id key", "[gallery]") { TEST_CASE("load_gallery throws on a missing file", "[gallery]") { CHECK_THROWS(load_gallery("/nonexistent/path/gallery.json")); } + +// ── GR-004: gallery ↔ embedder binding ─────────────────────────────────────── +// Verification plan row GR-004/T1: "Mismatched embedder → hard startup error; +// error names both sides." The comparison is a pure function over two stamps, so +// none of this needs a GPU, an ONNX, or even a file. + +/// TRACES: GR-004 | SR-001 +TEST_CASE("gallery save/load round-trips the embedder stamp", "[gallery][GR-004]") { + ActorGallery g; + ActorGallery::Actor a; + a.name = "Stamped Actor"; + a.embeddings = {make_embedding(0.3f)}; + g.actors.push_back(a); + g.embedder = stamp("LVFace-B_Glint360K.onnx", kShaA); + + TempFile tf("gallery_stamped.h5"); + save_gallery(tf.path, g); + ActorGallery loaded = load_gallery(tf.path); + + CHECK(loaded.embedder.model_name == "LVFace-B_Glint360K.onnx"); + CHECK(loaded.embedder.model_sha256 == kShaA); + CHECK(loaded.embedder.embed_dim == 512); + CHECK_FALSE(loaded.embedder.empty()); +} + +/// TRACES: GR-004 | SR-001 +TEST_CASE("a gallery written without a stamp loads as unstamped", "[gallery][GR-004]") { + // The back-compat case: pre-GR-004 files have no /embedder group at all. The + // absence must survive the round trip as an absence — a stamp naming no model + // would read as "checked and fine" to every consumer. + ActorGallery g; + ActorGallery::Actor a; + a.name = "Legacy Actor"; + a.embeddings = {make_embedding(0.f)}; + g.actors.push_back(a); + + TempFile tf("gallery_unstamped.h5"); + save_gallery(tf.path, g); + ActorGallery loaded = load_gallery(tf.path); + + CHECK(loaded.embedder.empty()); +} + +/// TRACES: GR-004 | SR-001 +TEST_CASE("legacy JSON galleries carry an optional embedder stamp", "[gallery][GR-004]") { + nlohmann::json j; + j["embedder"] = {{"model_name", "arcface_w600k_r50.onnx"}, + {"model_sha256", kShaB}, + {"embed_dim", 512}}; + j["actors"] = nlohmann::json::array(); + nlohmann::json ja; + ja["name"] = "JSON Actor"; + ja["embeddings"] = nlohmann::json::array(); + ja["embeddings"].push_back(std::vector(512, 0.1f)); + j["actors"].push_back(ja); + + TempFile tf("gallery_json_stamp.json"); + { std::ofstream out(tf.path); out << j.dump(); } + + ActorGallery g = load_gallery(tf.path); + CHECK(g.embedder.model_name == "arcface_w600k_r50.onnx"); + CHECK(g.embedder.model_sha256 == kShaB); +} + +/// TRACES: GR-004 | SR-001 +TEST_CASE("matching embedder stamps pass", "[gallery][GR-004]") { + auto chk = compare_embedder_stamps(stamp("model.onnx", kShaA), + stamp("model.onnx", kShaA)); + CHECK(chk.verdict == StampVerdict::match); + CHECK_FALSE(chk.fatal(false)); + CHECK_FALSE(chk.fatal(true)); // a proven match is never fatal, even in strict mode + CHECK_NOTHROW(enforce_embedder_stamp(stamp("model.onnx", kShaA), + stamp("model.onnx", kShaA), + "g.h5", "model.onnx", true)); +} + +/// TRACES: GR-004 | SR-001 +TEST_CASE("the hash decides, not the filename", "[gallery][GR-004]") { + // Same bytes under a different filename is the SAME model — a renamed or + // relocated file must not be treated as a different one. + auto same = compare_embedder_stamps(stamp("lvface.onnx", kShaA), + stamp("LVFace-B_Glint360K.onnx", kShaA)); + CHECK(same.verdict == StampVerdict::match); + + // Different bytes under the SAME filename is a DIFFERENT model — this is the + // in-place re-export a name-only stamp would miss entirely, and the reason the + // stamp carries a hash at all. + auto differ = compare_embedder_stamps(stamp("model.onnx", kShaA), + stamp("model.onnx", kShaB)); + CHECK(differ.verdict == StampVerdict::mismatch); +} + +/// TRACES: GR-004 | SR-001 +TEST_CASE("mismatched embedder is fatal and names both sides", "[gallery][GR-004]") { + const auto built = stamp("LVFace-B_Glint360K.onnx", kShaA); + const auto loaded = stamp("arcface_w600k_r50.onnx", kShaB); + + auto chk = compare_embedder_stamps(built, loaded, "cast.h5", "models/r50.onnx"); + REQUIRE(chk.verdict == StampVerdict::mismatch); + CHECK(chk.fatal(false)); // no bypass: a mismatch is fatal in every mode + CHECK(chk.fatal(true)); + + // Both sides must be identifiable from the message alone. + CHECK(mentions(chk.message, "LVFace-B_Glint360K.onnx")); + CHECK(mentions(chk.message, "arcface_w600k_r50.onnx")); + CHECK(mentions(chk.message, kShaA)); + CHECK(mentions(chk.message, kShaB)); + CHECK(mentions(chk.message, "cast.h5")); + CHECK(mentions(chk.message, "models/r50.onnx")); + + // ...and it must reach the caller as an error, not a log line. + CHECK_THROWS_AS(enforce_embedder_stamp(built, loaded, "cast.h5", + "models/r50.onnx", false), + std::runtime_error); + try { + enforce_embedder_stamp(built, loaded, "cast.h5", "models/r50.onnx", false); + FAIL("mismatch must throw"); + } catch (const std::runtime_error& e) { + const std::string what = e.what(); + CHECK(mentions(what, "LVFace-B_Glint360K.onnx")); + CHECK(mentions(what, "arcface_w600k_r50.onnx")); + } +} + +/// TRACES: GR-004 | SR-001 +TEST_CASE("differing embedding width is a mismatch", "[gallery][GR-004]") { + auto chk = compare_embedder_stamps(stamp("a.onnx", kShaA, 512), + stamp("a.onnx", kShaA, 256)); + CHECK(chk.verdict == StampVerdict::mismatch); + CHECK(mentions(chk.message, "512")); + CHECK(mentions(chk.message, "256")); +} + +/// TRACES: GR-004 | SR-001 +TEST_CASE("an unstamped gallery warns by default and fails under strict", + "[gallery][GR-004]") { + // Decision recorded in src/gallery/embedder_stamp.hpp: unstamped is UNKNOWN, + // not known-bad, and every pre-GR-004 gallery is unstamped. Hard-failing them + // all would make the check something people disable rather than trust; so it + // warns loudly, names the risk, and is promotable to fatal for measurement runs. + EmbedderStamp none; + auto chk = compare_embedder_stamps(none, stamp("model.onnx", kShaA), "old.h5"); + REQUIRE(chk.verdict == StampVerdict::unstamped); + CHECK_FALSE(chk.fatal(false)); + CHECK(chk.fatal(true)); + + CHECK(mentions(chk.message, "old.h5")); + CHECK(mentions(chk.message, "model.onnx")); // the loaded side is still named + CHECK(mentions(chk.message, "UNKNOWN")); // ...and the gallery side is honest + + CHECK_NOTHROW(enforce_embedder_stamp(none, stamp("model.onnx", kShaA), + "old.h5", "model.onnx", false)); + CHECK_THROWS_AS(enforce_embedder_stamp(none, stamp("model.onnx", kShaA), + "old.h5", "model.onnx", true), + std::runtime_error); +} + +/// TRACES: GR-004 | SR-001 +TEST_CASE("an unidentifiable embedder against a stamped gallery is not silent", + "[gallery][GR-004]") { + // e.g. a replay whose dump predates GR-004: we know what built the gallery but + // not what produced the vectors being fed in. Unverifiable, so it must not + // report success. + auto chk = compare_embedder_stamps(stamp("model.onnx", kShaA), EmbedderStamp{}, + "g.h5", "old dump.h5"); + CHECK(chk.verdict == StampVerdict::unknown_embedder); + CHECK_FALSE(chk.fatal(false)); + CHECK(chk.fatal(true)); + CHECK(mentions(chk.message, "model.onnx")); + CHECK(mentions(chk.message, "old dump.h5")); +} + +/// TRACES: GR-004 | SR-001 +TEST_CASE("name-only agreement is a weak match, not a clean pass", "[gallery][GR-004]") { + // A TRT deployment can run from a prebuilt .engine with the .onnx absent, so + // no hash is computable. Names agreeing is evidence, not proof. + auto weak = compare_embedder_stamps(stamp("model.onnx", kShaA), + stamp("model.onnx", "")); + CHECK(weak.verdict == StampVerdict::weak_match); + CHECK_FALSE(weak.fatal(false)); + CHECK(weak.fatal(true)); + + // Names disagreeing with no hash available is still a mismatch — the weaker + // evidence is enough to convict, just not to acquit. + auto bad = compare_embedder_stamps(stamp("lvface.onnx", ""), + stamp("arcface.onnx", "")); + CHECK(bad.verdict == StampVerdict::mismatch); + CHECK(mentions(bad.message, "lvface.onnx")); + CHECK(mentions(bad.message, "arcface.onnx")); +} + +/// TRACES: GR-004 | SR-001 +TEST_CASE("sha256 matches the published vectors", "[gallery][GR-004]") { + // Pins the in-tree FIPS 180-4 implementation against the standard vectors. + // This is what guarantees the C++ stamp and the Python (hashlib) stamp in + // scripts/sae_gallery.py agree on the same model file — without it the two + // halves of GR-004 could silently diverge and every check would be a mismatch. + CHECK(sha256_hex("") == + "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"); + CHECK(sha256_hex("abc") == + "ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad"); + CHECK(sha256_hex("abcdbcdecdefdefgefghfghighijhijkijkljklmklmnlmnomnopnopq") == + "248d6a61d20638b8e5c026930c3e6039a33ce45964ff2167f6ecedd419db06c1"); + // Multi-block input, exercising the length-padding path past 64 bytes. + CHECK(sha256_hex(std::string(1000, 'a')) == + "41edece42d63e8d9bf515a9ba6932e1c20cbc9f5a5d134645adb5db1b9737ea3"); +} + +/// TRACES: GR-004 | SR-001 +TEST_CASE("make_embedder_stamp hashes a real file and degrades gracefully", + "[gallery][GR-004]") { + // Stands in for an ONNX: the stamp does not care what the bytes mean. + TempFile tf("fake_model.onnx"); + { std::ofstream out(tf.path, std::ios::binary); out << "abc"; } + + EmbedderStamp s = make_embedder_stamp(tf.path); + CHECK(s.model_sha256 == + "ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad"); + CHECK_FALSE(s.model_name.empty()); + CHECK(s.model_name.find('/') == std::string::npos); // basename, not full path + + // A model path that does not exist still yields a comparable name-only stamp + // rather than an empty one, which is what keeps engine-only deployments usable. + EmbedderStamp missing = make_embedder_stamp("/nonexistent/models/foo.onnx"); + CHECK(missing.model_name == "foo.onnx"); + CHECK(missing.model_sha256.empty()); + CHECK_FALSE(missing.empty()); + + CHECK(make_embedder_stamp("").empty()); +}