From 01d7ead1e781e25fc7af5b63a5d5487a3fb4dcd8 Mon Sep 17 00:00:00 2001 From: Duncan Tourolle Date: Fri, 31 Jul 2026 15:20:18 +0200 Subject: [PATCH] study(VR-013): cross-source identification probe over input resolution MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Gallery from one recording, probes from another, sweeping the probe's input resolution end to end. VR-005 asked the same question over gallery mugshots but degraded an already-aligned 112x112 crop with alignment held perfect, so it isolates the embedder. Here the whole frame is downscaled before the detector, so detection and landmark regression degrade with it — which is most of the difference. Corpus is two 4096x2160 clips of one shoot, four people, hand-sorted. Ground truth is sorted by hand and gated by verify_labels.py; labels carried down the scales geometrically by box position, never by embedding similarity, which would keep only the faces the embedder already gets right and drop the ones the sweep exists to find. Findings, all scored through the production gallery sigmoid at prob_threshold 0.754 — never a raw cosine: - Holding 90% of the plateau needs ~50 px end to end, against VR-005's ~22 px. min_face_px at 40 looks right; 32 would admit faces in the falling region. - FPI is 0.0% at every scale. Resolution loss goes entirely to TBI. - The ceiling is cross-view, not resolution: everyone matches themselves within a recording (0.55-0.85) and collapses across two (0.14-0.45, threshold 0.335). Only the subject with frontal *gallery* references identified reliably, whatever their probe pose — so the lever is gallery pose coverage, not a better landmark source. - Averaging SCRFD's overlapping detections instead of discarding them at NMS lifts cross-recording TPI 41% -> 49%, for one forward pass and no extra model. Four identities and one shoot, so the shape is the result and the absolute rates are not. Both clips contain all four people, so there is no out-of-gallery class and the 10x-weighted out-of-cast misID is untested here. Clips, frames, hand-sorted crops and results are gitignored and belong in the artifact registry — the sorting is human ground truth and expensive to redo. Co-Authored-By: Claude Opus 5 TRACES: VR-013 | AR-002, AR-005, AR-024 --- docs/requirements.md | 2 + experiments/.gitignore | 11 ++ experiments/xsource/README.md | 72 +++++++ experiments/xsource/apply_corrections.py | 44 +++++ experiments/xsource/dump_faces.py | 90 +++++++++ experiments/xsource/failure_analysis.py | 174 ++++++++++++++++ experiments/xsource/landmark_voting.py | 165 ++++++++++++++++ experiments/xsource/make_review_site.py | 235 ++++++++++++++++++++++ experiments/xsource/pose_label.py | 200 +++++++++++++++++++ experiments/xsource/propose_labels.py | 242 +++++++++++++++++++++++ experiments/xsource/redraw_boxes.py | 55 ++++++ experiments/xsource/resolution_sweep.py | 182 +++++++++++++++++ experiments/xsource/verify_labels.py | 167 ++++++++++++++++ 13 files changed, 1639 insertions(+) create mode 100644 experiments/xsource/README.md create mode 100644 experiments/xsource/apply_corrections.py create mode 100644 experiments/xsource/dump_faces.py create mode 100644 experiments/xsource/failure_analysis.py create mode 100644 experiments/xsource/landmark_voting.py create mode 100644 experiments/xsource/make_review_site.py create mode 100644 experiments/xsource/pose_label.py create mode 100644 experiments/xsource/propose_labels.py create mode 100644 experiments/xsource/redraw_boxes.py create mode 100644 experiments/xsource/resolution_sweep.py create mode 100644 experiments/xsource/verify_labels.py diff --git a/docs/requirements.md b/docs/requirements.md index 9446397..0409f4f 100644 --- a/docs/requirements.md +++ b/docs/requirements.md @@ -115,6 +115,7 @@ Status: `Done` · `In Progress` · `Planned` · `TBD` · `Withdrawn` | VR-010 | Dump provenance attributes — embedder model, detector settings, `dense_scale`, `scene_detect`, sample rate | PR-002 | **High** | Planned | | VR-011 | Rewrite the replay harness for the post-AR-012 output contract | PR-002 | High | Planned | | VR-012 | Quality-knee study — TPI/FPI vs sharpness and vs pose, as VR-005 did for size; also settles whether the 5-point pose proxy needs a dedicated landmark model | PR-002 | Medium | Planned | +| VR-013 | Cross-source identification probe — gallery from one recording, probes from another, swept over input resolution end to end | PR-002 | Medium | In Progress | --- @@ -335,6 +336,7 @@ because it will be trusted. | AR-029 | T1 | Synthetic blur ladder → monotonically falling sharpness | Gaussian vs motion blur; **small sharp face vs large soft one** — size must not leak into this axis | | AR-030 | T1 | Alignment residual rises monotonically with foreshortening | **In-plane roll, scale and translation must leave it at zero** — the property that makes it a pose measure rather than a pose-and-everything-else measure; face size must not shift it; degenerate landmarks report not-ok rather than a number | | VR-012 | **T4** | Knee located per axis on held-out films | Report each candidate threshold's cost in **lost true presence**, not only its gain in precision — a gate that improves misID by discarding half the cast has not helped | +| VR-013 | **T4** | Identification holds across two recordings of the same people, and degrades to TBI rather than to a wrong name as input resolution falls | Gallery and probes must come from *different* recordings — a hold-one-out over one recording measures a much easier problem and will not surface the cross-view failure. Ground truth is hand-sorted; labels propagated by embedding similarity would keep only the faces the embedder already gets right | | IR-001/002 | T1 | Serialised output matches golden file | Zero-length window; actor with many windows | | IR-003 | T1 | Output written after deferred pass | Not at EOF | | IR-004/005 | **T1** | Signature matches golden vector bit-for-bit | Identical result in both producer repos | diff --git a/experiments/.gitignore b/experiments/.gitignore index 1ec81d2..5bf6dde 100644 --- a/experiments/.gitignore +++ b/experiments/.gitignore @@ -11,6 +11,17 @@ manifests/ trajectories/ results/ +# Cross-source identification study: source clips and the hand-sorted face +# crops. The sorting is human ground truth and expensive to redo, so it goes to +# the artifact registry rather than being regenerated — push it once sorted. +xsource/clips/ +xsource/labelling/ +xsource/frames/ +xsource/cache/ +xsource/results_*.json +xsource/failure_analysis.json +xsource/*.jpg + # Raw run logs and scratch scripts (regenerated by every run). _scratch/ diff --git a/experiments/xsource/README.md b/experiments/xsource/README.md new file mode 100644 index 0000000..903ceb1 --- /dev/null +++ b/experiments/xsource/README.md @@ -0,0 +1,72 @@ +# xsource — cross-source identification probe (VR-013) + +Gallery from **one** recording, probes from **another**, swept over the probe's +input resolution. Complements VR-005, which asked the same question over gallery +mugshots: that one degrades an already-aligned 112×112 crop, holding alignment +perfect, so it isolates the embedder. This one downscales the **whole frame** +before the detector, so detection and landmark regression degrade with it. + +Corpus: two Pexels clips of one shoot (4096×2160, 25 fps), four people, all four +present in both. Clips and hand-sorted crops are gitignored — push them with +`scripts/artifacts/push_artifacts.sh`, because the sorting is human ground truth +and expensive to redo. + +## Scripts + +| script | does | +|---|---| +| `dump_faces.py` | detect every face, write a context crop per detection + a manifest | +| `redraw_boxes.py` | redraw those crops with the detection boxed, in place | +| `propose_labels.py` | propose labels for one clip from another clip's hand-sorted folders | +| `make_review_site.py` | local `review.html` — current label, crop, better match, correct and export | +| `apply_corrections.py` | apply the exported `corrections.json` | +| `verify_labels.py` | integrity gate: index consistency, duplicates, separation. Exits non-zero on failure | +| `resolution_sweep.py` | the VR-013 measurement | +| `failure_analysis.py` | what explains the misses — pose, size, blur, detector confidence | +| `landmark_voting.py` | average SCRFD's overlapping detections instead of discarding them | +| `pose_label.py` | mesh-estimated head pose, for hand correction (feeds VR-012) | + +Everything drives the shipped C++ through `sae_embed`; nothing reimplements +detection, alignment, the embedder or the calibration. Scoring goes through the +production gallery sigmoid — never a raw cosine (AR-024). + + LD_PRELOAD=/usr/lib/libcudnn_cnn.so.9 python3 resolution_sweep.py + +The preload is needed while ORT's CUDA provider looks for +`cudnnGetConvolutionBackwardDataAlgorithm_v7`, which cuDNN 9 moved into +`libcudnn_cnn.so.9` behind a dispatch stub. Without it everything silently falls +back to CPU. + +## What it found + +**Resolution is not the binding constraint here.** TPI holds ~41–47% from 4096×2160 +down to ~45 px faces, then falls: 23 px → 26%, 18 px → 12%, 14 px → 1.5%. Holding +90% of the plateau needs roughly 50 px end to end, against VR-005's ~22 px — the +gap is detection and landmark error, which VR-005 excludes by construction. + +**FPI is 0.0% at every scale.** Resolution loss goes entirely to TBI: the pipeline +stops naming people rather than naming the wrong one. + +**The ceiling is cross-view, not resolution.** Every person matches themselves +strongly *within* a recording (sim 0.55–0.85) and collapses *across* the two +(0.14–0.45, threshold 0.335). Only the person with frontal **gallery** references +identified reliably, whatever their probe pose — so the lever is gallery pose +coverage (`docs/pose-expansion.md`), not a better landmark model. + +**Landmark voting helps.** SCRFD predicts each face from several anchors and NMS +discards all but one, throwing away a median of 3 landmark estimates per face. +Averaging them, weighted by confidence, lifts cross-clip TPI 41% → 49% for one +forward pass and no extra model. A MediaPipe mesh as landmark source went the +other way (41% → 16%): more stable within a recording, but a ring centroid is not +the annotated landmark ArcFace was trained on, and the embedder punishes the +off-distribution crop. + +## Reading these numbers + +Four identities, 70 probes, one shoot. The ~47% plateau is pose, not resolution — +half these faces are turned away and never clear threshold at any scale, so the +absolute rates say little and the *shape* is the result. Both clips contain all +four people, so there is no out-of-gallery class and the 10×-weighted out-of-cast +misID is **untested** here; holding one identity out of the gallery would fix +that. And the resolution curve is dominated by the single subject whose gallery +references are frontal. diff --git a/experiments/xsource/apply_corrections.py b/experiments/xsource/apply_corrections.py new file mode 100644 index 0000000..5b18f1b --- /dev/null +++ b/experiments/xsource/apply_corrections.py @@ -0,0 +1,44 @@ +#!/usr/bin/env python3 +"""Apply corrections.json exported from review.html. + + python3 apply_corrections.py ~/Downloads/corrections.json [--dry-run] + +Moves each crop to the folder you chose. "discard" goes to labelling//discard/, +which the sweep ignores — nothing is deleted, so a misclick is recoverable. + +Refuses to move a file it cannot find exactly once, rather than guessing: a +half-applied correction set would put a crop in two folders and quietly +duplicate a label. +""" +import sys, json, glob, os, shutil + +if len(sys.argv) < 2: + sys.exit(__doc__) +path = sys.argv[1] +DRY = "--dry-run" in sys.argv +corr = json.load(open(path)) +if not corr: + sys.exit("no corrections in that file") + +moved = skipped = 0 +for fname, c in corr.items(): + clip, to = c["clip"], c["to"] + hits = glob.glob(f"labelling/{clip}/**/{fname}", recursive=True) + if len(hits) != 1: + print(f"[skip] {fname}: found {len(hits)} copies, expected 1") + skipped += 1 + continue + src = hits[0] + dst_dir = f"labelling/{clip}/{to}" + dst = f"{dst_dir}/{fname}" + if os.path.abspath(src) == os.path.abspath(dst): + continue + print(f"{'would move' if DRY else 'move'} {c['from']} -> {to}: {fname}") + if not DRY: + os.makedirs(dst_dir, exist_ok=True) + shutil.move(src, dst) + moved += 1 + +print(f"\n{moved} moved, {skipped} skipped{' (dry run)' if DRY else ''}") +if not DRY and moved: + print("re-run verify_labels.py to confirm the set is still consistent") diff --git a/experiments/xsource/dump_faces.py b/experiments/xsource/dump_faces.py new file mode 100644 index 0000000..4f126fa --- /dev/null +++ b/experiments/xsource/dump_faces.py @@ -0,0 +1,90 @@ +#!/usr/bin/env python3 +"""Dump face crops from both clips for hand-labelling. + +Writes labelling//unsorted/.jpg — a context crop around each +detection, big enough to recognise a person by eye. Move them into +labelling//person_A/, person_B/, ... and the sweep reads those folders as +ground truth. + +Filenames carry a cNN_ cluster-hint prefix so visually similar faces sort next +to each other in a file manager. The hint is only an ordering convenience — +the folder you drop a file into is what counts, and the sweep never reads the +prefix. + +Detection and alignment run through the shipped C++ (sae_embed). Every crop +keeps its clip, frame and native-resolution bbox in manifest.json, so probe +detections at reduced scale can be tied back to a labelled face geometrically, +by position, rather than by embedding similarity — which would be circular. +""" +import sys, glob, json, os, shutil +import numpy as np +import cv2 + +sys.path.insert(0, "/home/dtourolle/Development/Jray-project/scene-actor-extraction/build-ort") +import sae_embed + +M = "/home/dtourolle/Development/Jray-project/scene-actor-extraction/models/" +CLIPS = ["5157339", "5157344"] +MIN_PX = 60 +CTX = 256 # context-crop side, for human recognisability + +eng = sae_embed.FaceEmbedder(detector_model=M + "scrfd_500m_bnkps.onnx", + arcface_model=M + "arcface_w600k_r50.onnx", + conf=0.5, nms=0.4, max_side=0) + +for clip in CLIPS: + out_dir = f"labelling/{clip}/unsorted" + if os.path.isdir(f"labelling/{clip}"): + print(f"[skip] labelling/{clip} exists — not overwriting your sorting", + file=sys.stderr) + continue + os.makedirs(out_dir, exist_ok=True) + + entries = [] + for p in sorted(glob.glob(f"pex/d{clip}_*.png")): + frame = p.rsplit("_", 1)[-1].split(".")[0] + img = cv2.imread(p) + for i, d in enumerate(eng.detect(img)): + x, y, w, h = d.bbox + if min(w, h) < MIN_PX: + continue + lm = np.array(d.landmarks, dtype=np.float32).reshape(5, 2) + crop = sae_embed.align_face(img, lm) + if crop is None: + continue + emb = np.asarray(eng.embed_crop(crop), dtype=np.float32) + + pad = int(0.5 * max(w, h)) + x0, y0 = max(0, int(x) - pad), max(0, int(y) - pad) + x1, y1 = min(img.shape[1], int(x + w) + pad), min(img.shape[0], int(y + h) + pad) + ctx = cv2.resize(img[y0:y1, x0:x1], (CTX, CTX)) + + entries.append({"clip": clip, "frame": frame, "idx": i, + "bbox": [float(x), float(y), float(w), float(h)], + "px": float(min(w, h)), "conf": float(d.confidence), + "emb": emb, "ctx": ctx}) + + # cluster hint only — greedy, purely to group similar faces in the file list + E = np.stack([e["emb"] for e in entries]) + hint = -np.ones(len(entries), int) + k = 0 + for i in range(len(entries)): + if hint[i] >= 0: + continue + hint[i] = k + for j in range(i + 1, len(entries)): + if hint[j] < 0 and float(E[i] @ E[j]) > 0.5: + hint[j] = k + k += 1 + + manifest = [] + for e, h in zip(entries, hint): + name = f"c{h:02d}_{e['clip']}_f{e['frame']}_i{e['idx']}_{int(e['px'])}px.jpg" + cv2.imwrite(f"{out_dir}/{name}", e["ctx"]) + manifest.append({k: v for k, v in e.items() if k not in ("emb", "ctx")} + | {"file": name, "cluster_hint": int(h)}) + + json.dump(manifest, open(f"labelling/{clip}/manifest.json", "w"), indent=1) + print(f"[{clip}] {len(manifest)} crops in {out_dir}, {k} cluster hints, " + f"face px {min(m['px'] for m in manifest):.0f}–{max(m['px'] for m in manifest):.0f}", + file=sys.stderr) diff --git a/experiments/xsource/failure_analysis.py b/experiments/xsource/failure_analysis.py new file mode 100644 index 0000000..df6cd93 --- /dev/null +++ b/experiments/xsource/failure_analysis.py @@ -0,0 +1,174 @@ +#!/usr/bin/env python3 +"""What explains the misses? Head pose, face size, blur, detector confidence. + +For every hand-labelled probe face, computes the calibrated probability against +its OWN gallery entry — so a low value is a false negative, not a mistake about +who it is — and pairs it with covariates that might explain the failure. + +Head pose comes from solvePnP of the 5 landmarks against a canonical 3D face, +giving yaw/pitch/roll in degrees. + + CAVEAT, and it matters: the pose estimate is derived from the same 5 + landmarks the alignment uses. Where those landmarks are unreliable the pose + estimate is unreliable too, and both degrade for the same reason. So this + can show that failures concentrate at high yaw; it cannot cleanly separate + "the head was turned" from "the landmarks were wrong because the head was + turned". Those are the same physical cause, but not the same fix — the + first argues for gallery pose coverage, the second for a better landmark + source. + +A sanity check is printed first: pose is estimated per person, and if it does +not recover what is visible in the review sheets (one subject frontal, another +in profile, another looking down) then the estimate is not worth reading. + +Similarities go through the production gallery sigmoid, never compared raw. +""" +import sys, glob, json, os +import numpy as np +import cv2 + +sys.path.insert(0, "/home/dtourolle/Development/Jray-project/scene-actor-extraction/build-ort") +import sae_embed + +ROOT = "/home/dtourolle/Development/Jray-project/scene-actor-extraction/" +M = ROOT + "models/" +GALLERY_CLIP, PROBE_CLIP = "5157344", "5157339" +PROB_THRESHOLD = 0.754 + +# Canonical 3D face, ordered as types.hpp:60 — +# [0] right-eye [1] left-eye [2] nose [3] right-mouth [4] left-mouth. +# The subject's right eye sits to the LEFT in image space, hence the negative X. +FACE_3D = np.array([ + (-34.0, 35.0, -28.0), + ( 34.0, 35.0, -28.0), + ( 0.0, 0.0, 0.0), + (-26.0, -32.0, -25.0), + ( 26.0, -32.0, -25.0), +], dtype=np.float64) + +eng = sae_embed.FaceEmbedder(detector_model=M + "scrfd_500m_bnkps.onnx", + arcface_model=M + "LVFace-B_Glint360K.onnx", + conf=0.5, nms=0.4, max_side=0) +cal = sae_embed.gallery_calibration(ROOT + "gallery_lvface.h5") + + +def head_pose(lm, w, h): + """yaw, pitch, roll in degrees. Focal length assumed = image width.""" + cam = np.array([[w, 0, w / 2], [0, w, h / 2], [0, 0, 1]], dtype=np.float64) + ok, rvec, _ = cv2.solvePnP(FACE_3D, lm.astype(np.float64), cam, None, + flags=cv2.SOLVEPNP_EPNP) + if not ok: + return None + R, _ = cv2.Rodrigues(rvec) + sy = np.sqrt(R[0, 0] ** 2 + R[1, 0] ** 2) + if sy > 1e-6: + pitch = np.degrees(np.arctan2(-R[2, 0], sy)) + yaw = np.degrees(np.arctan2(R[1, 0], R[0, 0])) + roll = np.degrees(np.arctan2(R[2, 1], R[2, 2])) + else: + pitch = np.degrees(np.arctan2(-R[2, 0], sy)); yaw = 0.0 + roll = np.degrees(np.arctan2(-R[1, 2], R[1, 1])) + # solvePnP's yaw wraps near +/-180 for a face pointing at the camera; + # fold it to a "degrees away from frontal" magnitude. + yaw = ((yaw + 180) % 360) - 180 + if abs(yaw) > 90: + yaw = np.sign(yaw) * (180 - abs(yaw)) + return yaw, pitch, roll + + +def collect(clip): + lab = {os.path.basename(p): os.path.basename(os.path.dirname(p)) + for p in glob.glob(f"labelling/{clip}/*/*.jpg") + if os.path.basename(os.path.dirname(p)) not in ("discard", "unsorted")} + man = {m["file"]: m for m in json.load(open(f"labelling/{clip}/manifest.json"))} + rows = [] + for frame in sorted({man[f]["frame"] for f in lab}): + img = cv2.imread(f"frames/d{clip}_{frame}.png") + dets = eng.detect(img) + H, W = img.shape[:2] + for f, person in lab.items(): + m = man[f] + if m["frame"] != frame or m["idx"] >= len(dets): + continue + d = dets[m["idx"]] + lm = np.array(d.landmarks, dtype=np.float32).reshape(5, 2) + crop = sae_embed.align_face(img, lm) + if crop is None: + continue + pose = head_pose(lm, W, H) + x, y, w, h = d.bbox + g = cv2.cvtColor(np.asarray(crop), cv2.COLOR_BGR2GRAY) + rows.append({ + "person": person, "px": float(min(w, h)), "conf": float(d.confidence), + "yaw": pose[0] if pose else np.nan, "pitch": pose[1] if pose else np.nan, + "roll": pose[2] if pose else np.nan, + "blur": float(cv2.Laplacian(g, cv2.CV_64F).var()), + "emb": np.asarray(eng.embed_crop(crop), dtype=np.float32)}) + return rows + + +gal_rows = collect(GALLERY_CLIP) +prb_rows = collect(PROBE_CLIP) +gal = {} +for r in gal_rows: + gal.setdefault(r["person"], []).append(r["emb"]) +gal = {p: np.stack(v) for p, v in gal.items()} + +for r in prb_rows: + if r["person"] in gal: + s = float((gal[r["person"]] @ r["emb"]).max()) # best-of-N, own actor + r["p"] = cal.probability(s) + r["sim"] = s + else: + r["p"] = np.nan +rows = [r for r in prb_rows if not np.isnan(r.get("p", np.nan))] +print(f"[data] {len(rows)} labelled probe faces with a gallery entry\n", file=sys.stderr) + +# ── sanity check: does the pose estimate recover what the sheets show? ─────── +print("pose by person (does this match the review sheets?)") +print(f"{'person':>7}{'n':>5}{'|yaw| med':>11}{'pitch med':>11}{'P med':>8}{'hit rate':>10}") +for p in sorted({r['person'] for r in rows}): + sub = [r for r in rows if r["person"] == p] + print(f"{p:>7}{len(sub):>5}" + f"{np.median([abs(r['yaw']) for r in sub]):>11.1f}" + f"{np.median([r['pitch'] for r in sub]):>11.1f}" + f"{np.median([r['p'] for r in sub]):>8.3f}" + f"{100*np.mean([r['p'] > PROB_THRESHOLD for r in sub]):>9.0f}%") + +# ── P binned by each covariate ─────────────────────────────────────────────── +def binned(name, key, edges, fmt="{:.0f}"): + print(f"\nP(match) by {name}") + print(f"{'bin':>16}{'n':>5}{'P med':>9}{'hit rate':>10}{'sim med':>9}") + vals = np.array([r[key] for r in rows]) + for lo, hi in zip(edges[:-1], edges[1:]): + sub = [r for r, v in zip(rows, vals) if lo <= v < hi] + if not sub: + continue + lbl = f"{fmt.format(lo)}–{fmt.format(hi)}" + print(f"{lbl:>16}{len(sub):>5}" + f"{np.median([r['p'] for r in sub]):>9.3f}" + f"{100*np.mean([r['p'] > PROB_THRESHOLD for r in sub]):>9.0f}%" + f"{np.median([r['sim'] for r in sub]):>9.3f}") + +for r in rows: + r["absyaw"] = abs(r["yaw"]) + r["abspitch"] = abs(r["pitch"]) +binned("|yaw| (deg from frontal)", "absyaw", [0, 10, 20, 30, 45, 60, 91]) +binned("|pitch| (deg)", "abspitch", [0, 10, 20, 30, 45, 91]) +binned("face size (px)", "px", [0, 130, 150, 175, 200, 400]) +binned("blur (laplacian var)", "blur", [0, 50, 150, 400, 1000, 1e9]) +binned("detector confidence", "conf", [0.5, 0.6, 0.7, 0.8, 0.9, 1.01], "{:.2f}") + +# ── how much does each covariate actually explain? ─────────────────────────── +print("\nSpearman rank correlation with P(match):") +def spearman(a, b): + ra = np.argsort(np.argsort(a)); rb = np.argsort(np.argsort(b)) + return float(np.corrcoef(ra, rb)[0, 1]) +P = np.array([r["p"] for r in rows]) +for key, label in [("absyaw", "|yaw|"), ("abspitch", "|pitch|"), ("px", "face px"), + ("blur", "blur"), ("conf", "detector conf")]: + v = np.array([r[key] for r in rows]) + print(f" {label:>14}: {spearman(v, P):+.3f}") + +json.dump([{k: v for k, v in r.items() if k != "emb"} for r in rows], + open("failure_analysis.json", "w"), indent=1, default=float) diff --git a/experiments/xsource/landmark_voting.py b/experiments/xsource/landmark_voting.py new file mode 100644 index 0000000..1a1bd07 --- /dev/null +++ b/experiments/xsource/landmark_voting.py @@ -0,0 +1,165 @@ +#!/usr/bin/env python3 +"""Landmark voting: average SCRFD's overlapping detections instead of discarding them. + +SCRFD predicts a face from many anchors; NMS keeps the single highest-scoring +box and throws the rest away. Each discarded box carries its own 5-landmark +estimate of the SAME face, so the survivors are one sample from a distribution +we could be averaging over. + + baseline conf 0.50, nms 0.40 — the shipped settings, one box per face + voted conf 0.30, nms 0.90 — duplicates survive, then grouped by IoU and + the 5 landmarks averaged, weighted by detection confidence + +Why this is worth trying when the mesh failed: the mesh moved the landmarks off +the definition ArcFace was trained on (a lip-ring centroid is not an annotated +mouth corner), and the embedder punished it. A confidence-weighted mean of +SCRFD's OWN landmark predictions is the same kind of point, just with less +variance — it should stay on-distribution while being steadier. + +Scored on cross-clip identification through the production sigmoid, which is +the thing that actually broke. Raw similarity shown only to locate the +threshold; it decides nothing. + + LD_PRELOAD=/usr/lib/libcudnn_cnn.so.9 python3 landmark_voting.py +""" +import sys, glob, json, os + +sys.path.insert(0, "/home/dtourolle/Development/Jray-project/scene-actor-extraction/build-ort") +import sae_embed # before cv2 — see alignment_compare.py + +import numpy as np +import cv2 + +ROOT = "/home/dtourolle/Development/Jray-project/scene-actor-extraction/" +M = ROOT + "models/" +CLIPS = ["5157344", "5157339"] +PROB_THRESHOLD = 0.754 +GROUP_IOU = 0.55 # detections overlapping this much are the same face +MATCH_IOU = 0.35 # tie a detection to the hand-labelled face + +base_eng = sae_embed.FaceEmbedder(detector_model=M + "scrfd_500m_bnkps.onnx", + arcface_model=M + "LVFace-B_Glint360K.onnx", + conf=0.5, nms=0.4, max_side=0) +# Same models, looser suppression: keep the duplicates NMS would have removed. +vote_eng = sae_embed.FaceEmbedder(detector_model=M + "scrfd_500m_bnkps.onnx", + arcface_model=M + "LVFace-B_Glint360K.onnx", + conf=0.3, nms=0.9, max_side=0) +cal = sae_embed.gallery_calibration(ROOT + "gallery_lvface.h5") + + +def iou(a, b): + ax, ay, aw, ah = a; bx, by, bw, bh = b + x0, y0 = max(ax, bx), max(ay, by) + x1, y1 = min(ax + aw, bx + bw), min(ay + ah, by + bh) + if x1 <= x0 or y1 <= y0: + return 0.0 + i = (x1 - x0) * (y1 - y0) + return i / (aw * ah + bw * bh - i) + + +def vote(dets): + """Group overlapping detections, return (bbox, landmarks, conf, n_votes).""" + items = sorted(dets, key=lambda d: -d.confidence) + used, out = [False] * len(items), [] + for i, d in enumerate(items): + if used[i]: + continue + grp = [d] + used[i] = True + for j in range(i + 1, len(items)): + if not used[j] and iou(list(d.bbox), list(items[j].bbox)) >= GROUP_IOU: + used[j] = True + grp.append(items[j]) + w = np.array([g.confidence for g in grp], dtype=np.float32) + w = w / w.sum() + lms = np.stack([np.array(g.landmarks, dtype=np.float32).reshape(5, 2) for g in grp]) + bxs = np.stack([np.array(list(g.bbox), dtype=np.float32) for g in grp]) + out.append((( w[:, None] * bxs).sum(0), (w[:, None, None] * lms).sum(0), + float(grp[0].confidence), len(grp))) + return out + + +def collect(clip): + lab = {os.path.basename(p): os.path.basename(os.path.dirname(p)) + for p in glob.glob(f"labelling/{clip}/*/*.jpg") + if os.path.basename(os.path.dirname(p)) not in ("discard", "unsorted")} + man = {m["file"]: m for m in json.load(open(f"labelling/{clip}/manifest.json"))} + rows, votes = [], [] + for frame in sorted({man[f]["frame"] for f in lab}): + img = cv2.imread(f"frames/d{clip}_{frame}.png") + base = base_eng.detect(img) + voted = vote(vote_eng.detect(img)) + for fname, person in lab.items(): + m = man[fname] + if m["frame"] != frame or m["idx"] >= len(base): + continue + d = base[m["idx"]] + lm5 = np.array(d.landmarks, dtype=np.float32).reshape(5, 2) + c_b = sae_embed.align_face(img, lm5) + + # the voted group covering the same face + best, best_v = None, 0.0 + for bbox, lms, conf, n in voted: + v = iou(list(bbox), list(d.bbox)) + if v > best_v: + best_v, best = v, (lms, n) + c_v = None + if best and best_v >= MATCH_IOU: + c_v = sae_embed.align_face(img, best[0].astype(np.float32)) + votes.append(best[1]) + rec = {"person": person} + rec["base"] = np.asarray(base_eng.embed_crop(c_b), np.float32) if c_b is not None else None + rec["voted"] = np.asarray(base_eng.embed_crop(c_v), np.float32) if c_v is not None else None + rows.append(rec) + return rows, votes + + +data, allv = {}, [] +for c in CLIPS: + data[c], v = collect(c) + allv += v + print(f"[{c}] {len(data[c])} crops", file=sys.stderr) +print(f"[voting] group size: median {np.median(allv):.0f}, " + f"mean {np.mean(allv):.1f}, max {max(allv)} detections averaged per face", + file=sys.stderr) + +GAL, PRB = "5157344", "5157339" +print(f"\ngallery {GAL} -> probe {PRB}, P>{PROB_THRESHOLD}\n") +print(f"{'align':>8}{'person':>8}{'n_gal':>7}{'n_prb':>7}" + f"{'within-clip':>13}{'cross-clip':>12}{'hit rate':>10}") +summary = {} +for key in ("base", "voted"): + gal, prb = {}, {} + for r in data[GAL]: + if r[key] is not None: + gal.setdefault(r["person"], []).append(r[key]) + for r in data[PRB]: + if r[key] is not None: + prb.setdefault(r["person"], []).append(r[key]) + gal = {p: np.stack(v) for p, v in gal.items()} + prb = {p: np.stack(v) for p, v in prb.items()} + hits = tot = 0 + for p in sorted(set(gal) & set(prb)): + pp = prb[p] @ prb[p].T + np.fill_diagonal(pp, -1) + within = float(np.median(pp.max(axis=1))) if len(pp) > 1 else float("nan") + cross = float(np.median((gal[p] @ prb[p].T).max(axis=0))) + h = 0 + for e in prb[p]: + bp, bn = 0.0, None + for q in gal: + v = cal.probability(float((gal[q] @ e).max())) + if v > bp: + bp, bn = v, q + if bp > PROB_THRESHOLD and bn == p: + h += 1 + hits += h; tot += len(prb[p]) + print(f"{key:>8}{p:>8}{len(gal[p]):>7}{len(prb[p]):>7}" + f"{cal.probability(within):>6.3f}/{within:<6.3f}" + f"{cal.probability(cross):>6.3f}/{cross:<5.3f}{100*h/len(prb[p]):>9.0f}%") + summary[key] = (hits, tot) + print(f"{key:>8}{'ALL':>8}{'':>14}{'':>25}{100*hits/max(tot,1):>9.0f}%\n") + +hb, tb = summary["base"]; hv, tv = summary["voted"] +print(f"voting vs baseline: {100*hv/max(tv,1) - 100*hb/max(tb,1):+.1f} points " + f"of cross-clip TPI ({hb}/{tb} -> {hv}/{tv})") diff --git a/experiments/xsource/make_review_site.py b/experiments/xsource/make_review_site.py new file mode 100644 index 0000000..11856ae --- /dev/null +++ b/experiments/xsource/make_review_site.py @@ -0,0 +1,235 @@ +#!/usr/bin/env python3 +"""Build labelling/review.html — a local page for correcting the labels. + +One row per crop, ordered most-suspicious first: + + left the person it is currently filed under (medoid of that person's + hand-sorted crops, so the reference is one you trust) + centre the crop under review — context with the detection boxed, and + beneath it the 112x112 the embedder actually receives + right the person it matches better, if any, with both probabilities + +Pick a destination per row, then Export to download corrections.json and apply +it with apply_corrections.py. Nothing is moved by this script. + +Self-contained: images are inlined as data URIs and the page is opened from +disk, so no server runs and no face crop leaves the machine. + +Ordering is by P(other) - P(self), both from the global gallery sigmoid, so +rows where the evidence disagrees with the label float to the top and the +agreement cases sink. It is a review order, not a verdict — you are the +arbiter, which is the whole point of labelling by hand. +""" +import sys, glob, json, os, base64 +import numpy as np +import cv2 + +sys.path.insert(0, "/home/dtourolle/Development/Jray-project/scene-actor-extraction/build-ort") +import sae_embed + +ROOT = "/home/dtourolle/Development/Jray-project/scene-actor-extraction/" +M = ROOT + "models/" +EMBEDDER = M + "LVFace-B_Glint360K.onnx" +GALLERY = ROOT + "gallery_lvface.h5" +REF_CLIP = "5157344" # the clip sorted by hand — reference faces come from here +CLIPS = ["5157344", "5157339"] + +eng = sae_embed.FaceEmbedder(detector_model=M + "scrfd_500m_bnkps.onnx", + arcface_model=EMBEDDER, conf=0.5, nms=0.4, max_side=0) +cal = sae_embed.gallery_calibration(GALLERY) + + +def b64(img, size, q=72): + img = cv2.resize(img, (size, size)) + ok, buf = cv2.imencode(".jpg", img, [cv2.IMWRITE_JPEG_QUALITY, q]) + return "data:image/jpeg;base64," + base64.b64encode(buf).decode() if ok else "" + + +rows = [] +for clip in CLIPS: + man = {m["file"]: m for m in json.load(open(f"labelling/{clip}/manifest.json"))} + placed = {os.path.basename(p): (os.path.basename(os.path.dirname(p)), p) + for p in glob.glob(f"labelling/{clip}/*/*.jpg")} + by_frame = {} + for fname, (person, path) in placed.items(): + if fname in man and person != "unsorted": + by_frame.setdefault(man[fname]["frame"], []).append((fname, person, path)) + for frame, items in sorted(by_frame.items()): + img = cv2.imread(f"frames/d{clip}_{frame}.png") + if img is None: + continue + dets = eng.detect(img) + for fname, person, path in items: + i = man[fname]["idx"] + if i >= len(dets): + continue + lm = np.array(dets[i].landmarks, dtype=np.float32).reshape(5, 2) + crop = sae_embed.align_face(img, lm) + if crop is None: + continue + rows.append({"clip": clip, "person": person, "file": fname, "path": path, + "px": man[fname]["px"], "aligned": np.asarray(crop), + "emb": np.asarray(eng.embed_crop(crop), dtype=np.float32)}) + +people = sorted({r["person"] for r in rows}) +E = np.stack([r["emb"] for r in rows]) +lab = np.array([people.index(r["person"]) for r in rows]) +S = E @ E.T +np.fill_diagonal(S, -1.0) + +# reference face per person: medoid of their REF_CLIP crops +ref_img = {} +for k, p in enumerate(people): + idx = [i for i in np.where(lab == k)[0] if rows[i]["clip"] == REF_CLIP] + if not idx: + idx = list(np.where(lab == k)[0]) + if not idx: + continue + sub = S[np.ix_(idx, idx)].copy() + medoid = idx[int(np.argmax(sub.mean(axis=1)))] + ref_img[p] = b64(rows[medoid]["aligned"], 112) + +items = [] +for i, r in enumerate(rows): + k = lab[i] + same = [j for j in np.where(lab == k)[0] if j != i] + p_self = cal.probability(float(S[i, same].max())) if same else 0.0 + best_other, p_other = None, 0.0 + for k2, p2 in enumerate(people): + if k2 == k: + continue + other = np.where(lab == k2)[0] + if not len(other): + continue + pv = cal.probability(float(S[i, other].max())) + if pv > p_other: + p_other, best_other = pv, p2 + ctx = cv2.imread(r["path"]) + items.append({ + "file": r["file"], "clip": r["clip"], "person": r["person"], + "px": int(r["px"]), "p_self": round(p_self, 3), "p_other": round(p_other, 3), + "other": best_other, "delta": round(p_other - p_self, 3), + "ctx": b64(ctx, 150) if ctx is not None else "", + "ali": b64(r["aligned"], 112), + }) +items.sort(key=lambda x: -x["delta"]) + +payload = json.dumps({"people": people, "refs": ref_img, "items": items}) + +HTML = """JRay — label review + +
+

Label review

+ + + +
+
Left: the person this crop is filed under. Centre: the crop (context with the +detection boxed, and the 112×112 the embedder actually sees). Right: the person it matches +better, if any. Ordered by P(other) − P(self) — disagreements first.
+
+ +""" + +os.makedirs("labelling", exist_ok=True) +out = "labelling/review.html" +with open(out, "w") as f: + f.write(HTML.replace("__PAYLOAD__", payload)) +size = os.path.getsize(out) / 1e6 +flagged = sum(1 for i in items if i["delta"] > 0) +print(f"{out} {size:.1f} MB {len(items)} crops, {flagged} disagreements", file=sys.stderr) +print(f"open file://{os.path.abspath(out)}", file=sys.stderr) diff --git a/experiments/xsource/pose_label.py b/experiments/xsource/pose_label.py new file mode 100644 index 0000000..513d286 --- /dev/null +++ b/experiments/xsource/pose_label.py @@ -0,0 +1,200 @@ +#!/usr/bin/env python3 +"""Estimate head pose per crop, and build a page to confirm or correct it. + +Why not solvePnP on the 5 detector landmarks: those landmarks collapse on +turned faces, so the estimator breaks precisely on the crops whose pose we care +about. Run that way it reported the profile subject as the MOST frontal of the +four, which is how we know not to trust it. + +Instead the estimate comes from the MediaPipe face mesh (468 points, run via +OpenCV DNN — the same model rPPG-kahn uses) and a symmetry measure that needs +no 3D model: + + yaw_ratio = (dL - dR) / (dL + dR) + +over left/right symmetric vertex pairs, where dL and dR are each side's +distance from the face midline. Frontal ~ 0, profile -> +/-1. It degrades +gracefully because it averages many pairs rather than trusting any one point, +and it is scale- and translation-free. + +It is still an estimate. So this writes pose_review.html with the estimate +PRE-FILLED as a proposal, ordered by confidence, for you to correct — and the +correlation is only run against your corrected labels. If the estimate turns +out to disagree with you often, that is the finding, and the automatic number +gets dropped rather than reported. + +Bins are coarse on purpose: frontal / three-quarter / profile / down-or-hidden. +Finer than that and the labelling is slower and less reliable, and the question +("does pose explain the misses") does not need degrees. +""" +import sys, glob, json, os, base64 + +# sae_embed MUST be imported before cv2: OpenCV's DNN module loads the system +# libonnxruntime, which then shadows the newer one this module links against and +# the import fails on a missing symbol version. Order matters, so do not tidy +# these into alphabetical order. +sys.path.insert(0, "/home/dtourolle/Development/Jray-project/scene-actor-extraction/build-ort") +import sae_embed + +import numpy as np +import cv2 + +ROOT = "/home/dtourolle/Development/Jray-project/scene-actor-extraction/" +M = ROOT + "models/" +MESH = "/home/dtourolle/Development/rPPG-kahn/models/face_landmark.tflite" +CLIPS = ["5157344", "5157339"] +BINS = ["frontal", "three-quarter", "profile", "down-or-hidden"] + +# Symmetric vertex pairs (subject-left, subject-right) on the MediaPipe mesh: +# outer eye corners, inner eye corners, cheeks, mouth corners, jaw. +PAIRS = [(33, 263), (133, 362), (130, 359), (243, 463), + (61, 291), (91, 321), (146, 375), (58, 288), (172, 397), (215, 435)] +MIDLINE = [10, 168, 1, 4, 5, 195, 197, 152] # forehead -> nose -> chin + +net = cv2.dnn.readNetFromTFLite(MESH) +NAMES = net.getUnconnectedOutLayersNames() +LMI, PRI = NAMES.index("conv2d_21"), NAMES.index("conv2d_31") + +eng = sae_embed.FaceEmbedder(detector_model=M + "scrfd_500m_bnkps.onnx", + arcface_model=M + "LVFace-B_Glint360K.onnx", + conf=0.5, nms=0.4, max_side=0) + + +def mesh_pose(img, bbox, expand=1.6): + """(yaw_ratio, presence) or (nan, 0). yaw_ratio in [-1, 1], 0 = frontal.""" + x, y, w, h = bbox + cx, cy, s = x + w / 2, y + h / 2, max(w, h) * expand + crop = cv2.getRectSubPix(img, (int(s), int(s)), (float(cx), float(cy))) + net.setInput(cv2.dnn.blobFromImage(crop, 1 / 255.0, (192, 192), (0, 0, 0), swapRB=True)) + o = net.forward(NAMES) + pres = 1 / (1 + np.exp(-float(o[PRI].ravel()[0]))) + lm = o[LMI].reshape(468, 3)[:, :2] + mid = lm[MIDLINE] + # least-squares midline direction, then signed distance of each pair member + c = mid.mean(axis=0) + u, _, _ = np.linalg.svd(mid - c) + d = (mid - c) + axis = np.linalg.svd(d.T @ d)[0][:, 0] # principal direction of the midline + normal = np.array([-axis[1], axis[0]]) + ratios = [] + for a, b in PAIRS: + dl = float(np.dot(lm[a] - c, normal)) + dr = float(np.dot(lm[b] - c, normal)) + if abs(dl) + abs(dr) < 1e-6: + continue + ratios.append((abs(dl) - abs(dr)) / (abs(dl) + abs(dr))) + return (float(np.median(ratios)) if ratios else np.nan), pres + + +def b64(img, size, q=72): + ok, buf = cv2.imencode(".jpg", cv2.resize(img, (size, size)), + [cv2.IMWRITE_JPEG_QUALITY, q]) + return "data:image/jpeg;base64," + base64.b64encode(buf).decode() if ok else "" + + +items = [] +for clip in CLIPS: + lab = {os.path.basename(p): (os.path.basename(os.path.dirname(p)), p) + for p in glob.glob(f"labelling/{clip}/*/*.jpg") + if os.path.basename(os.path.dirname(p)) not in ("discard", "unsorted")} + man = {m["file"]: m for m in json.load(open(f"labelling/{clip}/manifest.json"))} + for frame in sorted({man[f]["frame"] for f in lab}): + img = cv2.imread(f"frames/d{clip}_{frame}.png") + dets = eng.detect(img) + for fname, (person, path) in lab.items(): + m = man[fname] + if m["frame"] != frame or m["idx"] >= len(dets): + continue + d = dets[m["idx"]] + lm5 = np.array(d.landmarks, dtype=np.float32).reshape(5, 2) + crop = sae_embed.align_face(img, lm5) + if crop is None: + continue + yaw, pres = mesh_pose(img, d.bbox) + a = abs(yaw) if not np.isnan(yaw) else 1.0 + guess = ("frontal" if a < 0.15 else "three-quarter" if a < 0.45 + else "profile") + if pres < 0.5: + guess = "down-or-hidden" # mesh could not fit at all + ctx = cv2.imread(path) + items.append({"file": fname, "clip": clip, "person": person, + "px": int(m["px"]), "yaw": None if np.isnan(yaw) else round(yaw, 3), + "pres": round(pres, 3), "guess": guess, + "ctx": b64(ctx, 140) if ctx is not None else "", + "ali": b64(np.asarray(crop), 112)}) + +# least-confident first: near a bin boundary, or the mesh could not fit +def uncertainty(it): + if it["pres"] < 0.5: + return 0.0 + a = abs(it["yaw"]) if it["yaw"] is not None else 1.0 + return min(abs(a - 0.15), abs(a - 0.45)) +items.sort(key=uncertainty) + +payload = json.dumps({"bins": BINS, "items": items}) + +HTML = """JRay — head pose labelling + +

Head pose

+
+
+ +""" +out = "labelling/pose_review.html" +open(out, "w").write(HTML.replace("__PAYLOAD__", payload)) +from collections import Counter +print(f"{out} {os.path.getsize(out)/1e6:.1f} MB {len(items)} crops", file=sys.stderr) +print(f"estimate: {dict(Counter(i['guess'] for i in items))}", file=sys.stderr) +print("\nestimated pose per person (does this match what you see?):", file=sys.stderr) +for p in sorted({i["person"] for i in items}): + for clip in CLIPS: + sub = [i for i in items if i["person"] == p and i["clip"] == clip] + if sub: + print(f" {p} {clip[-3:]}: {dict(Counter(i['guess'] for i in sub))}", + file=sys.stderr) +print(f"\nopen file://{os.path.abspath(out)}", file=sys.stderr) diff --git a/experiments/xsource/propose_labels.py b/experiments/xsource/propose_labels.py new file mode 100644 index 0000000..6d6920f --- /dev/null +++ b/experiments/xsource/propose_labels.py @@ -0,0 +1,242 @@ +#!/usr/bin/env python3 +"""Propose person labels for one clip using another clip's hand-sorted labels. + +Reads the clip you have already sorted (REF_CLIP) as ground truth, then proposes +a person for every crop in the other clip (TARGET_CLIP) and writes them into +matching folders for you to correct. + + python3 propose_labels.py # propose, write folders + sheets + python3 propose_labels.py --dry-run # report only, move nothing + +Output: + labelling//unsorted/A|B|C|D/ proposed, same names as the ref clip + labelling//unsorted/ left in place when no person is + confident enough to name + labelling/review_.jpg contact sheet spanning BOTH clips: + confirmed crops first, then + proposed ones with their P + +Correcting it: open a review sheet. Every face on it should be one person. The +lower block is the proposal — move any intruder to the right folder, or back to +unsorted/. The folder a file sits in is the ground truth; nothing downstream +reads the proposed name or its probability. + +The proposal is a labelling aid, never the label. Scoring the sweep against +embedding-derived labels would be circular: it keeps the faces the embedder +already gets right and drops the hard ones the sweep exists to find. Your +correction is what breaks that loop, which is why the proposal is deliberately +conservative and leaves anything doubtful unnamed. + +Assignment is on the calibrated probability, per-actor best-of-N, exactly as +identity_matcher_node does — never a bare cosine (AR-024). The calibration is +fitted on your labelled reference crops, which is what calibrate_gallery is for. +""" +import sys, glob, json, os, shutil +import numpy as np +import cv2 + +sys.path.insert(0, "/home/dtourolle/Development/Jray-project/scene-actor-extraction/build-ort") +import sae_embed + +ROOT = "/home/dtourolle/Development/Jray-project/scene-actor-extraction/" +M = ROOT + "models/" +# The embedder and the gallery whose calibration scores it MUST be the same +# model: a Platt fit is specific to one embedding space, so LVFace probabilities +# read through an ArcFace fit are meaningless. +EMBEDDER = M + "LVFace-B_Glint360K.onnx" +GALLERY = ROOT + "gallery_lvface.h5" # 291 actors, cached fit +REF_CLIP, TARGET_CLIP = "5157344", "5157339" +ASSIGN_P = 0.90 # propose a name only when this confident +SHEET_COLS = 8 +THUMB = 150 +DRY = "--dry-run" in sys.argv + +eng = sae_embed.FaceEmbedder(detector_model=M + "scrfd_500m_bnkps.onnx", + arcface_model=EMBEDDER, + conf=0.5, nms=0.4, max_side=0) + + +def embed_manifest(clip): + """Re-derive each dumped crop's embedding from its source frame, cached. + + The dumped .jpg is a context thumbnail for human eyes; the embedding must + come from the aligned crop the pipeline would actually produce, so the + frame is re-detected and the manifest's idx picks the same face. + + Detecting 24 4K frames per clip costs far more than the rest of this script + put together, and the result only changes when the manifest does — so it is + cached and keyed on the manifest's mtime. Delete cache/ to force a redo. + """ + man_path = f"labelling/{clip}/manifest.json" + cache_path = f"cache/emb_{clip}.npz" + os.makedirs("cache", exist_ok=True) + if os.path.exists(cache_path) and \ + os.path.getmtime(cache_path) >= os.path.getmtime(man_path): + z = np.load(cache_path, allow_pickle=True) + print(f"[cache] {clip}: {len(z['meta'])} embeddings reused", file=sys.stderr) + return [{**m, "emb": e} for m, e in zip(z["meta"], z["emb"])] + + man = json.load(open(man_path)) + by_frame = {} + for m in man: + by_frame.setdefault(m["frame"], []).append(m) + out = [] + for frame, ms in sorted(by_frame.items()): + img = cv2.imread(f"frames/d{clip}_{frame}.png") + if img is None: + sys.exit(f"missing frames/d{clip}_{frame}.png — extract with\n" + f" ffmpeg -i clips/{clip}.mp4 -vf fps=2 -frames:v 24 " + f"frames/d{clip}_%03d.png") + dets = eng.detect(img) + for m in ms: + if m["idx"] >= len(dets): + continue + d = dets[m["idx"]] + lm = np.array(d.landmarks, dtype=np.float32).reshape(5, 2) + crop = sae_embed.align_face(img, lm) + if crop is None: + continue + out.append({**m, "emb": np.asarray(eng.embed_crop(crop), dtype=np.float32)}) + + np.savez(cache_path, + meta=np.array([{k: v for k, v in o.items() if k != "emb"} for o in out], + dtype=object), + emb=np.stack([o["emb"] for o in out])) + print(f"[cache] {clip}: {len(out)} embeddings written to {cache_path}", + file=sys.stderr) + return out + + +def sorted_dirs(clip): + """Person folders you created, wherever you put them under labelling/.""" + found = {} + for path in glob.glob(f"labelling/{clip}/**/", recursive=True): + name = os.path.basename(path.rstrip("/")) + if name in ("unsorted", "discard") or name.startswith("5157"): + continue + files = [os.path.basename(f) for f in glob.glob(path + "*.jpg")] + if files: + found[name] = files + return found + + +# ── reference side: your labels ────────────────────────────────────────────── +ref_rows = embed_manifest(REF_CLIP) +ref_dirs = sorted_dirs(REF_CLIP) +if not ref_dirs: + sys.exit(f"no person folders under labelling/{REF_CLIP} — sort that clip first") +file_to_person = {f: p for p, fs in ref_dirs.items() for f in fs} + +ref = [(file_to_person[r["file"]], r["emb"]) for r in ref_rows + if r["file"] in file_to_person] +people = sorted({p for p, _ in ref}) +print(f"[ref] {REF_CLIP}: {len(ref)} labelled crops over {len(people)} people " + f"{ {p: sum(1 for q, _ in ref if q == p) for p in people} }", file=sys.stderr) + +R = np.stack([e for _, e in ref]) +r_actor = [people.index(p) for p, _ in ref] + +# The global gallery's sigmoid — NOT a fit over these four people. A Platt fit +# over a handful of identities saturates: it will hand back P=0.99 for faces it +# has no basis to separate, which is exactly how a wrong label acquires a +# convincing probability. The production fit spans the whole actor population, +# so a probability means the same thing here as it does in the matcher. +cal = sae_embed.gallery_calibration(GALLERY) +print(f"[calibration] global: {cal} assign boundary = sim " + f"{cal.boundary_at(ASSIGN_P):.4f}", file=sys.stderr) + +# ── target side: propose ───────────────────────────────────────────────────── +tgt_rows = embed_manifest(TARGET_CLIP) +T = np.stack([t["emb"] for t in tgt_rows]) +r_actor_arr = np.asarray(r_actor) +# per-actor best-of-N for every target crop at once: (n_people, n_target) +best_sim = np.stack([(R[r_actor_arr == people.index(p)] @ T.T).max(axis=0) + for p in people]) +proposals = [] +for j, t in enumerate(tgt_rows): + k = int(np.argmax(best_sim[:, j])) + prob = cal.probability(float(best_sim[k, j])) # calibrated, never a bare cosine + proposals.append({**t, "person": people[k] if prob >= ASSIGN_P else None, + "p": prob, "top1": people[k]}) + +# At the production threshold the global fit stays silent on most of these +# faces, which is the honest answer for profile and downward-gaze shots — but a +# labelling aid wants throughput, not caution. --all proposes the top-1 person +# for every crop and orders the review sheets by descending probability, so the +# proposals degrade visibly down the sheet and you can stop correcting where +# they stop being right. The probability is shown, never hidden. +if "--all" in sys.argv: + for x in proposals: + x["person"] = x["top1"] + +named = [x for x in proposals if x["person"]] +print(f"[propose] {TARGET_CLIP}: {len(named)}/{len(proposals)} named at P>={ASSIGN_P}; " + f"{len(proposals) - len(named)} left unsorted", file=sys.stderr) +for p in people: + got = [x for x in named if x["person"] == p] + if got: + ps = [x["p"] for x in got] + print(f" {p}: {len(got):>3} crops P {min(ps):.3f}–{max(ps):.3f}", file=sys.stderr) + +if DRY: + sys.exit(0) + +# ── write proposed folders, mirroring the ref clip's layout ────────────────── +ref_parent = os.path.dirname(next(iter(glob.glob(f"labelling/{REF_CLIP}/**/{people[0]}/", + recursive=True))).rstrip("/")) +tgt_parent = ref_parent.replace(REF_CLIP, TARGET_CLIP) +for p in people: + d = f"{tgt_parent}/{p}" + if os.path.isdir(d): # never clobber corrections already made + print(f"[skip] {d} exists — leaving your sorting alone", file=sys.stderr) + continue + os.makedirs(d, exist_ok=True) +def find_crop(clip, fname): + """Locate a crop wherever it currently sits under labelling/.""" + hits = glob.glob(f"labelling/{clip}/**/{fname}", recursive=True) + return hits[0] if hits else None + +moved = 0 +for x in named: + src = find_crop(TARGET_CLIP, x["file"]) + dst = f"{tgt_parent}/{x['person']}/{x['file']}" + if src and os.path.abspath(src) != os.path.abspath(dst): + shutil.move(src, dst) + moved += 1 +print(f"[write] moved {moved} crops into proposed folders", file=sys.stderr) + +# ── review sheets: confirmed block, then proposed block ───────────────────── +def load(clip, person, fname): + for cand in glob.glob(f"labelling/{clip}/**/{person}/{fname}", recursive=True): + return cv2.imread(cand) + return None + +for person in people: + conf = [(REF_CLIP, f, None) for f in ref_dirs.get(person, [])] + prop = sorted([(TARGET_CLIP, x["file"], x["p"]) for x in named + if x["person"] == person], + key=lambda t: -t[2]) # most confident first + items = conf + prop + if not items: + continue + rows_n = (len(items) + SHEET_COLS - 1) // SHEET_COLS + sheet = np.full((rows_n * (THUMB + 26), SHEET_COLS * THUMB, 3), 30, np.uint8) + for n, (clip, fname, p) in enumerate(items): + img = load(clip, person, fname) + if img is None: + continue + rr, cc = divmod(n, SHEET_COLS) + y, x = rr * (THUMB + 26), cc * THUMB + sheet[y:y + THUMB, x:x + THUMB] = cv2.resize(img, (THUMB, THUMB)) + if p is None: + tag, col = f"{clip[-3:]} CONFIRMED", (170, 170, 170) + else: + tag, col = f"{clip[-3:]} P={p:.2f}", (140, 255, 140) + cv2.putText(sheet, tag, (x + 3, y + THUMB + 17), + cv2.FONT_HERSHEY_SIMPLEX, 0.42, col, 1) + cv2.imwrite(f"labelling/review_{person}.jpg", sheet) + print(f" review_{person}.jpg: {len(conf)} confirmed + {len(prop)} proposed", + file=sys.stderr) + +json.dump({x["file"]: {"person": x["person"], "p": x["p"]} for x in proposals}, + open(f"labelling/proposed_{TARGET_CLIP}.json", "w"), indent=1) diff --git a/experiments/xsource/redraw_boxes.py b/experiments/xsource/redraw_boxes.py new file mode 100644 index 0000000..7e64269 --- /dev/null +++ b/experiments/xsource/redraw_boxes.py @@ -0,0 +1,55 @@ +#!/usr/bin/env python3 +"""Redraw every dumped crop with its detection box marked. + +The original thumbnails padded by 0.5x the face on each side for +recognisability, which in a crowded frame pulls a neighbour into shot — often +more prominently than the subject. A label cannot be corrected from a picture +that does not say which face it refers to. + +This rewrites each .jpg IN PLACE, wherever it currently sits, so any sorting +already done is preserved: only the pixels change, never the filename or the +folder. Re-run it after dump_faces.py, and re-check any sorting done before it. +""" +import glob, json, os, sys +import cv2 + +CLIPS = ["5157339", "5157344"] +OUT = 256 + +for clip in CLIPS: + man = {m["file"]: m for m in json.load(open(f"labelling/{clip}/manifest.json"))} + n = 0 + for path in glob.glob(f"labelling/{clip}/**/*.jpg", recursive=True): + fname = os.path.basename(path) + m = man.get(fname) + if m is None: + continue + img = cv2.imread(f"frames/d{clip}_{m['frame']}.png") + if img is None: + sys.exit(f"missing frames/d{clip}_{m['frame']}.png") + + x, y, w, h = (int(v) for v in m["bbox"]) + pad = int(0.55 * max(w, h)) + x0, y0 = max(0, x - pad), max(0, y - pad) + x1, y1 = min(img.shape[1], x + w + pad), min(img.shape[0], y + h + pad) + sub = img[y0:y1, x0:x1].copy() + + # Box in the sub-image's coordinates, drawn before the resize so the + # line lands exactly on the face at any output size. + cv2.rectangle(sub, (x - x0, y - y0), (x - x0 + w, y - y0 + h), (0, 0, 255), 3) + # Dim everything outside the box so the subject is unmistakable even + # when a neighbour's face is larger or better lit. + mask = sub.copy() + mask[y - y0:y - y0 + h, x - x0:x - x0 + w] = 0 + sub = cv2.addWeighted(sub, 1.0, mask, -0.35, 0) + + scale = OUT / max(sub.shape[:2]) + sub = cv2.resize(sub, (int(sub.shape[1] * scale), int(sub.shape[0] * scale))) + canvas = cv2.copyMakeBorder( + sub, 0, max(0, OUT - sub.shape[0]), 0, max(0, OUT - sub.shape[1]), + cv2.BORDER_CONSTANT, value=(20, 20, 20))[:OUT, :OUT] + cv2.putText(canvas, f"{int(m['px'])}px", (5, OUT - 8), + cv2.FONT_HERSHEY_SIMPLEX, 0.45, (0, 255, 255), 1) + cv2.imwrite(path, canvas) + n += 1 + print(f"[{clip}] redrew {n} crops in place", file=sys.stderr) diff --git a/experiments/xsource/resolution_sweep.py b/experiments/xsource/resolution_sweep.py new file mode 100644 index 0000000..6603f75 --- /dev/null +++ b/experiments/xsource/resolution_sweep.py @@ -0,0 +1,182 @@ +#!/usr/bin/env python3 +"""Impact of input resolution on cross-source identification. + +Gallery is built from one clip at NATIVE resolution. Probes come from the other +clip with the WHOLE FRAME downscaled before it reaches the detector, so +detection and landmark regression degrade together with the pixels. That is the +measurement VR-005 structurally could not make: it degraded an already-aligned +112x112 crop, holding alignment perfect, so it isolated the embedder's +resolution sensitivity and excluded everything upstream of it. + + python3 resolution_sweep.py [--gallery-clip 5157339] [--detector scrfd_500m_bnkps.onnx] + +Ground truth +------------ +Hand-sorted person folders. Probe detections at reduced scale are tied back to +a labelled face GEOMETRICALLY — the box is mapped to native coordinates and +matched by IoU. Never by embedding similarity, which would be circular: it +would keep the faces the embedder still gets right and silently drop the ones +this sweep exists to find. + +A probe whose label is only in the probe clip is OUT OF GALLERY. Naming it is a +true out-of-cast misID, the error the per-scene scorer weights 10x, so it is +counted separately from naming the wrong gallery member. + +Metric +------ +The calibrated probability from the PRODUCTION gallery sigmoid, never a raw +cosine (AR-024). Per-actor best-of-N similarity -> probability -> accept above +prob_threshold. This is identification, so the matcher's prior applies; +config.hpp has match_prior 0.5, i.e. log_prior_odds = 0. + +Everything runs through the shipped C++ via sae_embed. +""" +import sys, glob, json, os, argparse +import numpy as np +import cv2 + +sys.path.insert(0, "/home/dtourolle/Development/Jray-project/scene-actor-extraction/build-ort") +import sae_embed + +ROOT = "/home/dtourolle/Development/Jray-project/scene-actor-extraction/" +M = ROOT + "models/" +PROB_THRESHOLD = 0.754 # config.hpp:67 +LOG_PRIOR_ODDS = 0.0 # config.hpp:61 match_prior=0.5 +IOU_MIN = 0.3 # geometric label carry-down +SCALES = [1.0, 0.8, 0.6, 0.5, 0.4, 0.3, 0.25, 0.2, 0.15, 0.12, 0.09, 0.06] + +ap = argparse.ArgumentParser() +ap.add_argument("--gallery-clip", default="5157339") +ap.add_argument("--probe-clip", default="5157344") +ap.add_argument("--detector", default="scrfd_500m_bnkps.onnx") +ap.add_argument("--embedder", default="LVFace-B_Glint360K.onnx") +ap.add_argument("--gallery-calibration", default=ROOT + "gallery_lvface.h5") +ap.add_argument("--out", default="results_resolution_sweep.json") +args = ap.parse_args() + +eng = sae_embed.FaceEmbedder(detector_model=M + args.detector, + arcface_model=M + args.embedder, + conf=0.5, nms=0.4, max_side=0) +cal = sae_embed.gallery_calibration(args.gallery_calibration) +print(f"[calibration] global: {cal}", file=sys.stderr) + + +def labelled(clip): + """{filename: person} from the hand-sorted folders, ignoring discard.""" + out = {} + for path in glob.glob(f"labelling/{clip}/*/*.jpg"): + person = os.path.basename(os.path.dirname(path)) + if person in ("discard", "unsorted"): + continue + out[os.path.basename(path)] = person + return out + + +def manifest(clip): + return {m["file"]: m for m in json.load(open(f"labelling/{clip}/manifest.json"))} + + +def iou(a, b): + ax, ay, aw, ah = a; bx, by, bw, bh = b + x0, y0 = max(ax, bx), max(ay, by) + x1, y1 = min(ax + aw, bx + bw), min(ay + ah, by + bh) + if x1 <= x0 or y1 <= y0: + return 0.0 + inter = (x1 - x0) * (y1 - y0) + return inter / (aw * ah + bw * bh - inter) + + +# ── gallery: native resolution, labelled faces only ────────────────────────── +g_lab, g_man = labelled(args.gallery_clip), manifest(args.gallery_clip) +gal = {} +for frame in sorted({g_man[f]["frame"] for f in g_lab}): + img = cv2.imread(f"frames/d{args.gallery_clip}_{frame}.png") + dets = eng.detect(img) + for fname, person in g_lab.items(): + m = g_man[fname] + if m["frame"] != frame or m["idx"] >= len(dets): + continue + lm = np.array(dets[m["idx"]].landmarks, dtype=np.float32).reshape(5, 2) + crop = sae_embed.align_face(img, lm) + if crop is None: + continue + gal.setdefault(person, []).append(np.asarray(eng.embed_crop(crop), dtype=np.float32)) +gal = {p: np.stack(v) for p, v in gal.items() if v} +people = sorted(gal) +print(f"[gallery] {args.gallery_clip} @native: " + f"{ {p: len(v) for p, v in gal.items()} }", file=sys.stderr) + +# ── probe ground truth at native resolution ────────────────────────────────── +p_lab, p_man = labelled(args.probe_clip), manifest(args.probe_clip) +truth = {} # frame -> [(bbox_native, person)] +for fname, person in p_lab.items(): + m = p_man[fname] + truth.setdefault(m["frame"], []).append((m["bbox"], person)) +n_out = sum(1 for p in set(p_lab.values()) if p not in people) +print(f"[probe] {args.probe_clip}: {len(p_lab)} labelled faces, " + f"{len(set(p_lab.values()))} people, {n_out} of them out-of-gallery", + file=sys.stderr) + +# ── sweep ──────────────────────────────────────────────────────────────────── +print(f"\n{'scale':>6}{'frame':>11}{'face px':>9}{'found':>7}{'matched':>9}" + f"{'TPI':>8}{'FPI-in':>8}{'FPI-out':>9}{'TBI':>8}") +results = [] +for s in SCALES: + tpi = fpi_in = fpi_out = tbi = 0 + n_found = n_matched = 0 + pxs = [] + for frame, gts in sorted(truth.items()): + img = cv2.imread(f"frames/d{args.probe_clip}_{frame}.png") + if s != 1.0: + img = cv2.resize(img, None, fx=s, fy=s, interpolation=cv2.INTER_AREA) + dets = eng.detect(img) + n_found += len(dets) + for d in dets: + x, y, w, h = d.bbox + native = (x / s, y / s, w / s, h / s) # geometric carry-down + best, best_iou = None, 0.0 + for gt_box, person in gts: + v = iou(native, gt_box) + if v > best_iou: + best_iou, best = v, person + if best_iou < IOU_MIN: + continue # spurious / unlabelled + n_matched += 1 + pxs.append(min(w, h)) + + lm = np.array(d.landmarks, dtype=np.float32).reshape(5, 2) + crop = sae_embed.align_face(img, lm) + if crop is None: + tbi += 1 # degenerate alignment + continue + emb = np.asarray(eng.embed_crop(crop), dtype=np.float32) + best_p, best_name = 0.0, None + for p in people: # per-actor best-of-N + prob = cal.probability(float((gal[p] @ emb).max()), LOG_PRIOR_ODDS) + if prob > best_p: + best_p, best_name = prob, p + if best_p <= PROB_THRESHOLD: + tbi += 1 + elif best not in people: + fpi_out += 1 # named someone absent from the gallery + elif best_name == best: + tpi += 1 + else: + fpi_in += 1 + n = max(1, n_matched) + med_px = float(np.median(pxs)) if pxs else 0.0 + print(f"{s:>6.2f}{f'{int(4096*s)}x{int(2160*s)}':>11}{med_px:>9.0f}" + f"{n_found:>7}{n_matched:>9}" + f"{100*tpi/n:>7.1f}%{100*fpi_in/n:>7.1f}%{100*fpi_out/n:>8.1f}%{100*tbi/n:>7.1f}%") + results.append({"scale": s, "median_face_px": med_px, "detections": n_found, + "matched_to_truth": n_matched, "tpi_pct": 100*tpi/n, + "fpi_in_gallery_pct": 100*fpi_in/n, "fpi_out_of_gallery_pct": 100*fpi_out/n, + "tbi_pct": 100*tbi/n}) + +json.dump({"gallery_clip": args.gallery_clip, "probe_clip": args.probe_clip, + "detector": args.detector, "embedder": args.embedder, + "prob_threshold": PROB_THRESHOLD, "log_prior_odds": LOG_PRIOR_ODDS, + "calibration": {"a": cal.a, "b": cal.b}, + "gallery_people": people, "results": results}, + open(args.out, "w"), indent=2) +print(f"\nwrote {args.out}", file=sys.stderr) diff --git a/experiments/xsource/verify_labels.py b/experiments/xsource/verify_labels.py new file mode 100644 index 0000000..a556d51 --- /dev/null +++ b/experiments/xsource/verify_labels.py @@ -0,0 +1,167 @@ +#!/usr/bin/env python3 +"""Integrity check on the labelled set, before it is used as ground truth. + +Checks, loudest failure first: + +1. INDEX INTEGRITY. Each crop's embedding is taken by re-detecting its source + frame and indexing with the manifest's `idx`. If detection order is not + reproducible, the thumbnail you sorted and the embedding that gets scored + are different faces — you would see a correct picture and score the wrong + person, with nothing to signal it. Every crop's re-detected bbox is compared + against the manifest's. + +2. NO CROP IN TWO FOLDERS, and every manifest entry accounted for — so a + move that half-completed cannot silently duplicate or drop a label. + +3. ALIGNMENT. The 112x112 warp is what the embedder actually sees; the + thumbnail is only context for your eyes. verify_.jpg pairs them: + context-with-box on top, the real aligned crop beneath. A profile face whose + alignment has collapsed is obvious there and nowhere else. + +4. SEPARATION. Per person, the calibrated P of their own crops against the + other people's, using the global gallery sigmoid. A label set where someone + matches another person better than themselves is mislabelled. + +Nothing here changes a label. It reports. +""" +import sys, glob, json, os +import numpy as np +import cv2 + +sys.path.insert(0, "/home/dtourolle/Development/Jray-project/scene-actor-extraction/build-ort") +import sae_embed + +ROOT = "/home/dtourolle/Development/Jray-project/scene-actor-extraction/" +M = ROOT + "models/" +EMBEDDER = M + "LVFace-B_Glint360K.onnx" +GALLERY = ROOT + "gallery_lvface.h5" +CLIPS = ["5157344", "5157339"] +THUMB = 130 +COLS = 10 + +eng = sae_embed.FaceEmbedder(detector_model=M + "scrfd_500m_bnkps.onnx", + arcface_model=EMBEDDER, conf=0.5, nms=0.4, max_side=0) + +fail = 0 +rows = [] + +for clip in CLIPS: + man = {m["file"]: m for m in json.load(open(f"labelling/{clip}/manifest.json"))} + + # where each crop currently sits -> its label + placed = {} + for path in glob.glob(f"labelling/{clip}/*/*.jpg"): + person = os.path.basename(os.path.dirname(path)) + if person in ("discard", "unsorted"): + continue # not people; scoring them would invent an extra identity + fname = os.path.basename(path) + if fname in placed: + print(f"[FAIL] {fname} appears in both {placed[fname][0]} and {person}") + fail += 1 + placed[fname] = (person, path) + + missing = set(man) - set(placed) + extra = set(placed) - set(man) + if missing: + print(f"[warn] {clip}: {len(missing)} manifest crops not in any folder") + if extra: + print(f"[FAIL] {clip}: {len(extra)} files with no manifest entry: " + f"{sorted(extra)[:3]}") + fail += 1 + + # index integrity + alignment, frame by frame + by_frame = {} + for fname, (person, path) in placed.items(): + if fname in man: + by_frame.setdefault(man[fname]["frame"], []).append((fname, person, path)) + + bad_idx = 0 + for frame, items in sorted(by_frame.items()): + img = cv2.imread(f"frames/d{clip}_{frame}.png") + if img is None: + print(f"[FAIL] missing frames/d{clip}_{frame}.png") + fail += 1 + continue + dets = eng.detect(img) + for fname, person, path in items: + m = man[fname] + i = m["idx"] + if i >= len(dets): + print(f"[FAIL] {fname}: idx {i} >= {len(dets)} detections now") + bad_idx += 1 + continue + got = [float(v) for v in dets[i].bbox] + want = m["bbox"] + if max(abs(a - b) for a, b in zip(got, want)) > 1.0: + print(f"[FAIL] {fname}: manifest bbox {[round(v) for v in want]} " + f"!= re-detected {[round(v) for v in got]}") + bad_idx += 1 + continue + lm = np.array(dets[i].landmarks, dtype=np.float32).reshape(5, 2) + crop = sae_embed.align_face(img, lm) + if crop is None: + print(f"[warn] {fname}: alignment degenerate, no crop reaches the embedder") + continue + rows.append({"clip": clip, "person": person, "file": fname, "path": path, + "px": m["px"], "aligned": np.asarray(crop), + "emb": np.asarray(eng.embed_crop(crop), dtype=np.float32)}) + fail += bad_idx + print(f"[{clip}] {len(placed)} placed, {len(by_frame)} frames, " + f"index mismatches: {bad_idx}") + +if not rows: + sys.exit("nothing to verify") + +# ── separation, through the global gallery sigmoid ─────────────────────────── +cal = sae_embed.gallery_calibration(GALLERY) +E = np.stack([r["emb"] for r in rows]) +people = sorted({r["person"] for r in rows}) +lab = np.array([people.index(r["person"]) for r in rows]) +S = E @ E.T +np.fill_diagonal(S, -1.0) + +print(f"\n{'person':>8}{'crops':>7}{'344':>6}{'339':>6}" + f"{'P(self)':>10}{'P(other)':>10}{'worst':>8}") +for k, p in enumerate(people): + mine = np.where(lab == k)[0] + if len(mine) < 2: + continue + self_sim = S[np.ix_(mine, mine)].max(axis=1) + other_sim = S[np.ix_(mine, np.where(lab != k)[0])].max(axis=1) + p_self = np.array([cal.probability(float(s)) for s in self_sim]) + p_other = np.array([cal.probability(float(s)) for s in other_sim]) + n344 = sum(1 for i in mine if rows[i]["clip"] == "5157344") + n339 = len(mine) - n344 + # a crop that matches someone else better than anyone of its own label + worst = int((other_sim > self_sim).sum()) + print(f"{p:>8}{len(mine):>7}{n344:>6}{n339:>6}" + f"{np.median(p_self):>10.3f}{np.median(p_other):>10.3f}{worst:>8}") + if worst: + for i in mine[other_sim > self_sim]: + print(f" suspect: {rows[i]['file']} " + f"P(self)={cal.probability(float(self_sim[list(mine).index(i)])):.3f} " + f"< P(other)={cal.probability(float(other_sim[list(mine).index(i)])):.3f}") + +# ── verify sheets: context+box over the actual aligned crop ────────────────── +for p in people: + items = [r for r in rows if r["person"] == p] + items.sort(key=lambda r: (r["clip"], r["file"])) + n = len(items) + sheet_rows = (n + COLS - 1) // COLS + H = THUMB * 2 + 22 + sheet = np.full((sheet_rows * H, COLS * THUMB, 3), 25, np.uint8) + for j, r in enumerate(items): + rr, cc = divmod(j, COLS) + y, x = rr * H, cc * THUMB + ctx = cv2.imread(r["path"]) + if ctx is not None: + sheet[y:y + THUMB, x:x + THUMB] = cv2.resize(ctx, (THUMB, THUMB)) + sheet[y + THUMB:y + 2 * THUMB, x:x + THUMB] = cv2.resize(r["aligned"], (THUMB, THUMB)) + cv2.putText(sheet, f"{r['clip'][-3:]} {int(r['px'])}px", + (x + 3, y + 2 * THUMB + 15), + cv2.FONT_HERSHEY_SIMPLEX, 0.38, (150, 220, 150), 1) + cv2.imwrite(f"labelling/verify_{p}.jpg", sheet) + print(f" verify_{p}.jpg: {n} crops (top row context, bottom row what the embedder sees)") + +print(f"\n{'PASS' if fail == 0 else f'{fail} FAILURES'}") +sys.exit(1 if fail else 0)