diff --git a/docs/requirements.md b/docs/requirements.md index 9446397..0409f4f 100644 --- a/docs/requirements.md +++ b/docs/requirements.md @@ -115,6 +115,7 @@ Status: `Done` · `In Progress` · `Planned` · `TBD` · `Withdrawn` | VR-010 | Dump provenance attributes — embedder model, detector settings, `dense_scale`, `scene_detect`, sample rate | PR-002 | **High** | Planned | | VR-011 | Rewrite the replay harness for the post-AR-012 output contract | PR-002 | High | Planned | | VR-012 | Quality-knee study — TPI/FPI vs sharpness and vs pose, as VR-005 did for size; also settles whether the 5-point pose proxy needs a dedicated landmark model | PR-002 | Medium | Planned | +| VR-013 | Cross-source identification probe — gallery from one recording, probes from another, swept over input resolution end to end | PR-002 | Medium | In Progress | --- @@ -335,6 +336,7 @@ because it will be trusted. | AR-029 | T1 | Synthetic blur ladder → monotonically falling sharpness | Gaussian vs motion blur; **small sharp face vs large soft one** — size must not leak into this axis | | AR-030 | T1 | Alignment residual rises monotonically with foreshortening | **In-plane roll, scale and translation must leave it at zero** — the property that makes it a pose measure rather than a pose-and-everything-else measure; face size must not shift it; degenerate landmarks report not-ok rather than a number | | VR-012 | **T4** | Knee located per axis on held-out films | Report each candidate threshold's cost in **lost true presence**, not only its gain in precision — a gate that improves misID by discarding half the cast has not helped | +| VR-013 | **T4** | Identification holds across two recordings of the same people, and degrades to TBI rather than to a wrong name as input resolution falls | Gallery and probes must come from *different* recordings — a hold-one-out over one recording measures a much easier problem and will not surface the cross-view failure. Ground truth is hand-sorted; labels propagated by embedding similarity would keep only the faces the embedder already gets right | | IR-001/002 | T1 | Serialised output matches golden file | Zero-length window; actor with many windows | | IR-003 | T1 | Output written after deferred pass | Not at EOF | | IR-004/005 | **T1** | Signature matches golden vector bit-for-bit | Identical result in both producer repos | diff --git a/experiments/.gitignore b/experiments/.gitignore index 1ec81d2..5bf6dde 100644 --- a/experiments/.gitignore +++ b/experiments/.gitignore @@ -11,6 +11,17 @@ manifests/ trajectories/ results/ +# Cross-source identification study: source clips and the hand-sorted face +# crops. The sorting is human ground truth and expensive to redo, so it goes to +# the artifact registry rather than being regenerated — push it once sorted. +xsource/clips/ +xsource/labelling/ +xsource/frames/ +xsource/cache/ +xsource/results_*.json +xsource/failure_analysis.json +xsource/*.jpg + # Raw run logs and scratch scripts (regenerated by every run). _scratch/ diff --git a/experiments/xsource/README.md b/experiments/xsource/README.md new file mode 100644 index 0000000..903ceb1 --- /dev/null +++ b/experiments/xsource/README.md @@ -0,0 +1,72 @@ +# xsource — cross-source identification probe (VR-013) + +Gallery from **one** recording, probes from **another**, swept over the probe's +input resolution. Complements VR-005, which asked the same question over gallery +mugshots: that one degrades an already-aligned 112×112 crop, holding alignment +perfect, so it isolates the embedder. This one downscales the **whole frame** +before the detector, so detection and landmark regression degrade with it. + +Corpus: two Pexels clips of one shoot (4096×2160, 25 fps), four people, all four +present in both. Clips and hand-sorted crops are gitignored — push them with +`scripts/artifacts/push_artifacts.sh`, because the sorting is human ground truth +and expensive to redo. + +## Scripts + +| script | does | +|---|---| +| `dump_faces.py` | detect every face, write a context crop per detection + a manifest | +| `redraw_boxes.py` | redraw those crops with the detection boxed, in place | +| `propose_labels.py` | propose labels for one clip from another clip's hand-sorted folders | +| `make_review_site.py` | local `review.html` — current label, crop, better match, correct and export | +| `apply_corrections.py` | apply the exported `corrections.json` | +| `verify_labels.py` | integrity gate: index consistency, duplicates, separation. Exits non-zero on failure | +| `resolution_sweep.py` | the VR-013 measurement | +| `failure_analysis.py` | what explains the misses — pose, size, blur, detector confidence | +| `landmark_voting.py` | average SCRFD's overlapping detections instead of discarding them | +| `pose_label.py` | mesh-estimated head pose, for hand correction (feeds VR-012) | + +Everything drives the shipped C++ through `sae_embed`; nothing reimplements +detection, alignment, the embedder or the calibration. Scoring goes through the +production gallery sigmoid — never a raw cosine (AR-024). + + LD_PRELOAD=/usr/lib/libcudnn_cnn.so.9 python3 resolution_sweep.py + +The preload is needed while ORT's CUDA provider looks for +`cudnnGetConvolutionBackwardDataAlgorithm_v7`, which cuDNN 9 moved into +`libcudnn_cnn.so.9` behind a dispatch stub. Without it everything silently falls +back to CPU. + +## What it found + +**Resolution is not the binding constraint here.** TPI holds ~41–47% from 4096×2160 +down to ~45 px faces, then falls: 23 px → 26%, 18 px → 12%, 14 px → 1.5%. Holding +90% of the plateau needs roughly 50 px end to end, against VR-005's ~22 px — the +gap is detection and landmark error, which VR-005 excludes by construction. + +**FPI is 0.0% at every scale.** Resolution loss goes entirely to TBI: the pipeline +stops naming people rather than naming the wrong one. + +**The ceiling is cross-view, not resolution.** Every person matches themselves +strongly *within* a recording (sim 0.55–0.85) and collapses *across* the two +(0.14–0.45, threshold 0.335). Only the person with frontal **gallery** references +identified reliably, whatever their probe pose — so the lever is gallery pose +coverage (`docs/pose-expansion.md`), not a better landmark model. + +**Landmark voting helps.** SCRFD predicts each face from several anchors and NMS +discards all but one, throwing away a median of 3 landmark estimates per face. +Averaging them, weighted by confidence, lifts cross-clip TPI 41% → 49% for one +forward pass and no extra model. A MediaPipe mesh as landmark source went the +other way (41% → 16%): more stable within a recording, but a ring centroid is not +the annotated landmark ArcFace was trained on, and the embedder punishes the +off-distribution crop. + +## Reading these numbers + +Four identities, 70 probes, one shoot. The ~47% plateau is pose, not resolution — +half these faces are turned away and never clear threshold at any scale, so the +absolute rates say little and the *shape* is the result. Both clips contain all +four people, so there is no out-of-gallery class and the 10×-weighted out-of-cast +misID is **untested** here; holding one identity out of the gallery would fix +that. And the resolution curve is dominated by the single subject whose gallery +references are frontal. diff --git a/experiments/xsource/apply_corrections.py b/experiments/xsource/apply_corrections.py new file mode 100644 index 0000000..5b18f1b --- /dev/null +++ b/experiments/xsource/apply_corrections.py @@ -0,0 +1,44 @@ +#!/usr/bin/env python3 +"""Apply corrections.json exported from review.html. + + python3 apply_corrections.py ~/Downloads/corrections.json [--dry-run] + +Moves each crop to the folder you chose. "discard" goes to labelling//discard/, +which the sweep ignores — nothing is deleted, so a misclick is recoverable. + +Refuses to move a file it cannot find exactly once, rather than guessing: a +half-applied correction set would put a crop in two folders and quietly +duplicate a label. +""" +import sys, json, glob, os, shutil + +if len(sys.argv) < 2: + sys.exit(__doc__) +path = sys.argv[1] +DRY = "--dry-run" in sys.argv +corr = json.load(open(path)) +if not corr: + sys.exit("no corrections in that file") + +moved = skipped = 0 +for fname, c in corr.items(): + clip, to = c["clip"], c["to"] + hits = glob.glob(f"labelling/{clip}/**/{fname}", recursive=True) + if len(hits) != 1: + print(f"[skip] {fname}: found {len(hits)} copies, expected 1") + skipped += 1 + continue + src = hits[0] + dst_dir = f"labelling/{clip}/{to}" + dst = f"{dst_dir}/{fname}" + if os.path.abspath(src) == os.path.abspath(dst): + continue + print(f"{'would move' if DRY else 'move'} {c['from']} -> {to}: {fname}") + if not DRY: + os.makedirs(dst_dir, exist_ok=True) + shutil.move(src, dst) + moved += 1 + +print(f"\n{moved} moved, {skipped} skipped{' (dry run)' if DRY else ''}") +if not DRY and moved: + print("re-run verify_labels.py to confirm the set is still consistent") diff --git a/experiments/xsource/dump_faces.py b/experiments/xsource/dump_faces.py new file mode 100644 index 0000000..4f126fa --- /dev/null +++ b/experiments/xsource/dump_faces.py @@ -0,0 +1,90 @@ +#!/usr/bin/env python3 +"""Dump face crops from both clips for hand-labelling. + +Writes labelling//unsorted/.jpg — a context crop around each +detection, big enough to recognise a person by eye. Move them into +labelling//person_A/, person_B/, ... and the sweep reads those folders as +ground truth. + +Filenames carry a cNN_ cluster-hint prefix so visually similar faces sort next +to each other in a file manager. The hint is only an ordering convenience — +the folder you drop a file into is what counts, and the sweep never reads the +prefix. + +Detection and alignment run through the shipped C++ (sae_embed). Every crop +keeps its clip, frame and native-resolution bbox in manifest.json, so probe +detections at reduced scale can be tied back to a labelled face geometrically, +by position, rather than by embedding similarity — which would be circular. +""" +import sys, glob, json, os, shutil +import numpy as np +import cv2 + +sys.path.insert(0, "/home/dtourolle/Development/Jray-project/scene-actor-extraction/build-ort") +import sae_embed + +M = "/home/dtourolle/Development/Jray-project/scene-actor-extraction/models/" +CLIPS = ["5157339", "5157344"] +MIN_PX = 60 +CTX = 256 # context-crop side, for human recognisability + +eng = sae_embed.FaceEmbedder(detector_model=M + "scrfd_500m_bnkps.onnx", + arcface_model=M + "arcface_w600k_r50.onnx", + conf=0.5, nms=0.4, max_side=0) + +for clip in CLIPS: + out_dir = f"labelling/{clip}/unsorted" + if os.path.isdir(f"labelling/{clip}"): + print(f"[skip] labelling/{clip} exists — not overwriting your sorting", + file=sys.stderr) + continue + os.makedirs(out_dir, exist_ok=True) + + entries = [] + for p in sorted(glob.glob(f"pex/d{clip}_*.png")): + frame = p.rsplit("_", 1)[-1].split(".")[0] + img = cv2.imread(p) + for i, d in enumerate(eng.detect(img)): + x, y, w, h = d.bbox + if min(w, h) < MIN_PX: + continue + lm = np.array(d.landmarks, dtype=np.float32).reshape(5, 2) + crop = sae_embed.align_face(img, lm) + if crop is None: + continue + emb = np.asarray(eng.embed_crop(crop), dtype=np.float32) + + pad = int(0.5 * max(w, h)) + x0, y0 = max(0, int(x) - pad), max(0, int(y) - pad) + x1, y1 = min(img.shape[1], int(x + w) + pad), min(img.shape[0], int(y + h) + pad) + ctx = cv2.resize(img[y0:y1, x0:x1], (CTX, CTX)) + + entries.append({"clip": clip, "frame": frame, "idx": i, + "bbox": [float(x), float(y), float(w), float(h)], + "px": float(min(w, h)), "conf": float(d.confidence), + "emb": emb, "ctx": ctx}) + + # cluster hint only — greedy, purely to group similar faces in the file list + E = np.stack([e["emb"] for e in entries]) + hint = -np.ones(len(entries), int) + k = 0 + for i in range(len(entries)): + if hint[i] >= 0: + continue + hint[i] = k + for j in range(i + 1, len(entries)): + if hint[j] < 0 and float(E[i] @ E[j]) > 0.5: + hint[j] = k + k += 1 + + manifest = [] + for e, h in zip(entries, hint): + name = f"c{h:02d}_{e['clip']}_f{e['frame']}_i{e['idx']}_{int(e['px'])}px.jpg" + cv2.imwrite(f"{out_dir}/{name}", e["ctx"]) + manifest.append({k: v for k, v in e.items() if k not in ("emb", "ctx")} + | {"file": name, "cluster_hint": int(h)}) + + json.dump(manifest, open(f"labelling/{clip}/manifest.json", "w"), indent=1) + print(f"[{clip}] {len(manifest)} crops in {out_dir}, {k} cluster hints, " + f"face px {min(m['px'] for m in manifest):.0f}–{max(m['px'] for m in manifest):.0f}", + file=sys.stderr) diff --git a/experiments/xsource/failure_analysis.py b/experiments/xsource/failure_analysis.py new file mode 100644 index 0000000..df6cd93 --- /dev/null +++ b/experiments/xsource/failure_analysis.py @@ -0,0 +1,174 @@ +#!/usr/bin/env python3 +"""What explains the misses? Head pose, face size, blur, detector confidence. + +For every hand-labelled probe face, computes the calibrated probability against +its OWN gallery entry — so a low value is a false negative, not a mistake about +who it is — and pairs it with covariates that might explain the failure. + +Head pose comes from solvePnP of the 5 landmarks against a canonical 3D face, +giving yaw/pitch/roll in degrees. + + CAVEAT, and it matters: the pose estimate is derived from the same 5 + landmarks the alignment uses. Where those landmarks are unreliable the pose + estimate is unreliable too, and both degrade for the same reason. So this + can show that failures concentrate at high yaw; it cannot cleanly separate + "the head was turned" from "the landmarks were wrong because the head was + turned". Those are the same physical cause, but not the same fix — the + first argues for gallery pose coverage, the second for a better landmark + source. + +A sanity check is printed first: pose is estimated per person, and if it does +not recover what is visible in the review sheets (one subject frontal, another +in profile, another looking down) then the estimate is not worth reading. + +Similarities go through the production gallery sigmoid, never compared raw. +""" +import sys, glob, json, os +import numpy as np +import cv2 + +sys.path.insert(0, "/home/dtourolle/Development/Jray-project/scene-actor-extraction/build-ort") +import sae_embed + +ROOT = "/home/dtourolle/Development/Jray-project/scene-actor-extraction/" +M = ROOT + "models/" +GALLERY_CLIP, PROBE_CLIP = "5157344", "5157339" +PROB_THRESHOLD = 0.754 + +# Canonical 3D face, ordered as types.hpp:60 — +# [0] right-eye [1] left-eye [2] nose [3] right-mouth [4] left-mouth. +# The subject's right eye sits to the LEFT in image space, hence the negative X. +FACE_3D = np.array([ + (-34.0, 35.0, -28.0), + ( 34.0, 35.0, -28.0), + ( 0.0, 0.0, 0.0), + (-26.0, -32.0, -25.0), + ( 26.0, -32.0, -25.0), +], dtype=np.float64) + +eng = sae_embed.FaceEmbedder(detector_model=M + "scrfd_500m_bnkps.onnx", + arcface_model=M + "LVFace-B_Glint360K.onnx", + conf=0.5, nms=0.4, max_side=0) +cal = sae_embed.gallery_calibration(ROOT + "gallery_lvface.h5") + + +def head_pose(lm, w, h): + """yaw, pitch, roll in degrees. Focal length assumed = image width.""" + cam = np.array([[w, 0, w / 2], [0, w, h / 2], [0, 0, 1]], dtype=np.float64) + ok, rvec, _ = cv2.solvePnP(FACE_3D, lm.astype(np.float64), cam, None, + flags=cv2.SOLVEPNP_EPNP) + if not ok: + return None + R, _ = cv2.Rodrigues(rvec) + sy = np.sqrt(R[0, 0] ** 2 + R[1, 0] ** 2) + if sy > 1e-6: + pitch = np.degrees(np.arctan2(-R[2, 0], sy)) + yaw = np.degrees(np.arctan2(R[1, 0], R[0, 0])) + roll = np.degrees(np.arctan2(R[2, 1], R[2, 2])) + else: + pitch = np.degrees(np.arctan2(-R[2, 0], sy)); yaw = 0.0 + roll = np.degrees(np.arctan2(-R[1, 2], R[1, 1])) + # solvePnP's yaw wraps near +/-180 for a face pointing at the camera; + # fold it to a "degrees away from frontal" magnitude. + yaw = ((yaw + 180) % 360) - 180 + if abs(yaw) > 90: + yaw = np.sign(yaw) * (180 - abs(yaw)) + return yaw, pitch, roll + + +def collect(clip): + lab = {os.path.basename(p): os.path.basename(os.path.dirname(p)) + for p in glob.glob(f"labelling/{clip}/*/*.jpg") + if os.path.basename(os.path.dirname(p)) not in ("discard", "unsorted")} + man = {m["file"]: m for m in json.load(open(f"labelling/{clip}/manifest.json"))} + rows = [] + for frame in sorted({man[f]["frame"] for f in lab}): + img = cv2.imread(f"frames/d{clip}_{frame}.png") + dets = eng.detect(img) + H, W = img.shape[:2] + for f, person in lab.items(): + m = man[f] + if m["frame"] != frame or m["idx"] >= len(dets): + continue + d = dets[m["idx"]] + lm = np.array(d.landmarks, dtype=np.float32).reshape(5, 2) + crop = sae_embed.align_face(img, lm) + if crop is None: + continue + pose = head_pose(lm, W, H) + x, y, w, h = d.bbox + g = cv2.cvtColor(np.asarray(crop), cv2.COLOR_BGR2GRAY) + rows.append({ + "person": person, "px": float(min(w, h)), "conf": float(d.confidence), + "yaw": pose[0] if pose else np.nan, "pitch": pose[1] if pose else np.nan, + "roll": pose[2] if pose else np.nan, + "blur": float(cv2.Laplacian(g, cv2.CV_64F).var()), + "emb": np.asarray(eng.embed_crop(crop), dtype=np.float32)}) + return rows + + +gal_rows = collect(GALLERY_CLIP) +prb_rows = collect(PROBE_CLIP) +gal = {} +for r in gal_rows: + gal.setdefault(r["person"], []).append(r["emb"]) +gal = {p: np.stack(v) for p, v in gal.items()} + +for r in prb_rows: + if r["person"] in gal: + s = float((gal[r["person"]] @ r["emb"]).max()) # best-of-N, own actor + r["p"] = cal.probability(s) + r["sim"] = s + else: + r["p"] = np.nan +rows = [r for r in prb_rows if not np.isnan(r.get("p", np.nan))] +print(f"[data] {len(rows)} labelled probe faces with a gallery entry\n", file=sys.stderr) + +# ── sanity check: does the pose estimate recover what the sheets show? ─────── +print("pose by person (does this match the review sheets?)") +print(f"{'person':>7}{'n':>5}{'|yaw| med':>11}{'pitch med':>11}{'P med':>8}{'hit rate':>10}") +for p in sorted({r['person'] for r in rows}): + sub = [r for r in rows if r["person"] == p] + print(f"{p:>7}{len(sub):>5}" + f"{np.median([abs(r['yaw']) for r in sub]):>11.1f}" + f"{np.median([r['pitch'] for r in sub]):>11.1f}" + f"{np.median([r['p'] for r in sub]):>8.3f}" + f"{100*np.mean([r['p'] > PROB_THRESHOLD for r in sub]):>9.0f}%") + +# ── P binned by each covariate ─────────────────────────────────────────────── +def binned(name, key, edges, fmt="{:.0f}"): + print(f"\nP(match) by {name}") + print(f"{'bin':>16}{'n':>5}{'P med':>9}{'hit rate':>10}{'sim med':>9}") + vals = np.array([r[key] for r in rows]) + for lo, hi in zip(edges[:-1], edges[1:]): + sub = [r for r, v in zip(rows, vals) if lo <= v < hi] + if not sub: + continue + lbl = f"{fmt.format(lo)}–{fmt.format(hi)}" + print(f"{lbl:>16}{len(sub):>5}" + f"{np.median([r['p'] for r in sub]):>9.3f}" + f"{100*np.mean([r['p'] > PROB_THRESHOLD for r in sub]):>9.0f}%" + f"{np.median([r['sim'] for r in sub]):>9.3f}") + +for r in rows: + r["absyaw"] = abs(r["yaw"]) + r["abspitch"] = abs(r["pitch"]) +binned("|yaw| (deg from frontal)", "absyaw", [0, 10, 20, 30, 45, 60, 91]) +binned("|pitch| (deg)", "abspitch", [0, 10, 20, 30, 45, 91]) +binned("face size (px)", "px", [0, 130, 150, 175, 200, 400]) +binned("blur (laplacian var)", "blur", [0, 50, 150, 400, 1000, 1e9]) +binned("detector confidence", "conf", [0.5, 0.6, 0.7, 0.8, 0.9, 1.01], "{:.2f}") + +# ── how much does each covariate actually explain? ─────────────────────────── +print("\nSpearman rank correlation with P(match):") +def spearman(a, b): + ra = np.argsort(np.argsort(a)); rb = np.argsort(np.argsort(b)) + return float(np.corrcoef(ra, rb)[0, 1]) +P = np.array([r["p"] for r in rows]) +for key, label in [("absyaw", "|yaw|"), ("abspitch", "|pitch|"), ("px", "face px"), + ("blur", "blur"), ("conf", "detector conf")]: + v = np.array([r[key] for r in rows]) + print(f" {label:>14}: {spearman(v, P):+.3f}") + +json.dump([{k: v for k, v in r.items() if k != "emb"} for r in rows], + open("failure_analysis.json", "w"), indent=1, default=float) diff --git a/experiments/xsource/landmark_voting.py b/experiments/xsource/landmark_voting.py new file mode 100644 index 0000000..1a1bd07 --- /dev/null +++ b/experiments/xsource/landmark_voting.py @@ -0,0 +1,165 @@ +#!/usr/bin/env python3 +"""Landmark voting: average SCRFD's overlapping detections instead of discarding them. + +SCRFD predicts a face from many anchors; NMS keeps the single highest-scoring +box and throws the rest away. Each discarded box carries its own 5-landmark +estimate of the SAME face, so the survivors are one sample from a distribution +we could be averaging over. + + baseline conf 0.50, nms 0.40 — the shipped settings, one box per face + voted conf 0.30, nms 0.90 — duplicates survive, then grouped by IoU and + the 5 landmarks averaged, weighted by detection confidence + +Why this is worth trying when the mesh failed: the mesh moved the landmarks off +the definition ArcFace was trained on (a lip-ring centroid is not an annotated +mouth corner), and the embedder punished it. A confidence-weighted mean of +SCRFD's OWN landmark predictions is the same kind of point, just with less +variance — it should stay on-distribution while being steadier. + +Scored on cross-clip identification through the production sigmoid, which is +the thing that actually broke. Raw similarity shown only to locate the +threshold; it decides nothing. + + LD_PRELOAD=/usr/lib/libcudnn_cnn.so.9 python3 landmark_voting.py +""" +import sys, glob, json, os + +sys.path.insert(0, "/home/dtourolle/Development/Jray-project/scene-actor-extraction/build-ort") +import sae_embed # before cv2 — see alignment_compare.py + +import numpy as np +import cv2 + +ROOT = "/home/dtourolle/Development/Jray-project/scene-actor-extraction/" +M = ROOT + "models/" +CLIPS = ["5157344", "5157339"] +PROB_THRESHOLD = 0.754 +GROUP_IOU = 0.55 # detections overlapping this much are the same face +MATCH_IOU = 0.35 # tie a detection to the hand-labelled face + +base_eng = sae_embed.FaceEmbedder(detector_model=M + "scrfd_500m_bnkps.onnx", + arcface_model=M + "LVFace-B_Glint360K.onnx", + conf=0.5, nms=0.4, max_side=0) +# Same models, looser suppression: keep the duplicates NMS would have removed. +vote_eng = sae_embed.FaceEmbedder(detector_model=M + "scrfd_500m_bnkps.onnx", + arcface_model=M + "LVFace-B_Glint360K.onnx", + conf=0.3, nms=0.9, max_side=0) +cal = sae_embed.gallery_calibration(ROOT + "gallery_lvface.h5") + + +def iou(a, b): + ax, ay, aw, ah = a; bx, by, bw, bh = b + x0, y0 = max(ax, bx), max(ay, by) + x1, y1 = min(ax + aw, bx + bw), min(ay + ah, by + bh) + if x1 <= x0 or y1 <= y0: + return 0.0 + i = (x1 - x0) * (y1 - y0) + return i / (aw * ah + bw * bh - i) + + +def vote(dets): + """Group overlapping detections, return (bbox, landmarks, conf, n_votes).""" + items = sorted(dets, key=lambda d: -d.confidence) + used, out = [False] * len(items), [] + for i, d in enumerate(items): + if used[i]: + continue + grp = [d] + used[i] = True + for j in range(i + 1, len(items)): + if not used[j] and iou(list(d.bbox), list(items[j].bbox)) >= GROUP_IOU: + used[j] = True + grp.append(items[j]) + w = np.array([g.confidence for g in grp], dtype=np.float32) + w = w / w.sum() + lms = np.stack([np.array(g.landmarks, dtype=np.float32).reshape(5, 2) for g in grp]) + bxs = np.stack([np.array(list(g.bbox), dtype=np.float32) for g in grp]) + out.append((( w[:, None] * bxs).sum(0), (w[:, None, None] * lms).sum(0), + float(grp[0].confidence), len(grp))) + return out + + +def collect(clip): + lab = {os.path.basename(p): os.path.basename(os.path.dirname(p)) + for p in glob.glob(f"labelling/{clip}/*/*.jpg") + if os.path.basename(os.path.dirname(p)) not in ("discard", "unsorted")} + man = {m["file"]: m for m in json.load(open(f"labelling/{clip}/manifest.json"))} + rows, votes = [], [] + for frame in sorted({man[f]["frame"] for f in lab}): + img = cv2.imread(f"frames/d{clip}_{frame}.png") + base = base_eng.detect(img) + voted = vote(vote_eng.detect(img)) + for fname, person in lab.items(): + m = man[fname] + if m["frame"] != frame or m["idx"] >= len(base): + continue + d = base[m["idx"]] + lm5 = np.array(d.landmarks, dtype=np.float32).reshape(5, 2) + c_b = sae_embed.align_face(img, lm5) + + # the voted group covering the same face + best, best_v = None, 0.0 + for bbox, lms, conf, n in voted: + v = iou(list(bbox), list(d.bbox)) + if v > best_v: + best_v, best = v, (lms, n) + c_v = None + if best and best_v >= MATCH_IOU: + c_v = sae_embed.align_face(img, best[0].astype(np.float32)) + votes.append(best[1]) + rec = {"person": person} + rec["base"] = np.asarray(base_eng.embed_crop(c_b), np.float32) if c_b is not None else None + rec["voted"] = np.asarray(base_eng.embed_crop(c_v), np.float32) if c_v is not None else None + rows.append(rec) + return rows, votes + + +data, allv = {}, [] +for c in CLIPS: + data[c], v = collect(c) + allv += v + print(f"[{c}] {len(data[c])} crops", file=sys.stderr) +print(f"[voting] group size: median {np.median(allv):.0f}, " + f"mean {np.mean(allv):.1f}, max {max(allv)} detections averaged per face", + file=sys.stderr) + +GAL, PRB = "5157344", "5157339" +print(f"\ngallery {GAL} -> probe {PRB}, P>{PROB_THRESHOLD}\n") +print(f"{'align':>8}{'person':>8}{'n_gal':>7}{'n_prb':>7}" + f"{'within-clip':>13}{'cross-clip':>12}{'hit rate':>10}") +summary = {} +for key in ("base", "voted"): + gal, prb = {}, {} + for r in data[GAL]: + if r[key] is not None: + gal.setdefault(r["person"], []).append(r[key]) + for r in data[PRB]: + if r[key] is not None: + prb.setdefault(r["person"], []).append(r[key]) + gal = {p: np.stack(v) for p, v in gal.items()} + prb = {p: np.stack(v) for p, v in prb.items()} + hits = tot = 0 + for p in sorted(set(gal) & set(prb)): + pp = prb[p] @ prb[p].T + np.fill_diagonal(pp, -1) + within = float(np.median(pp.max(axis=1))) if len(pp) > 1 else float("nan") + cross = float(np.median((gal[p] @ prb[p].T).max(axis=0))) + h = 0 + for e in prb[p]: + bp, bn = 0.0, None + for q in gal: + v = cal.probability(float((gal[q] @ e).max())) + if v > bp: + bp, bn = v, q + if bp > PROB_THRESHOLD and bn == p: + h += 1 + hits += h; tot += len(prb[p]) + print(f"{key:>8}{p:>8}{len(gal[p]):>7}{len(prb[p]):>7}" + f"{cal.probability(within):>6.3f}/{within:<6.3f}" + f"{cal.probability(cross):>6.3f}/{cross:<5.3f}{100*h/len(prb[p]):>9.0f}%") + summary[key] = (hits, tot) + print(f"{key:>8}{'ALL':>8}{'':>14}{'':>25}{100*hits/max(tot,1):>9.0f}%\n") + +hb, tb = summary["base"]; hv, tv = summary["voted"] +print(f"voting vs baseline: {100*hv/max(tv,1) - 100*hb/max(tb,1):+.1f} points " + f"of cross-clip TPI ({hb}/{tb} -> {hv}/{tv})") diff --git a/experiments/xsource/make_review_site.py b/experiments/xsource/make_review_site.py new file mode 100644 index 0000000..11856ae --- /dev/null +++ b/experiments/xsource/make_review_site.py @@ -0,0 +1,235 @@ +#!/usr/bin/env python3 +"""Build labelling/review.html — a local page for correcting the labels. + +One row per crop, ordered most-suspicious first: + + left the person it is currently filed under (medoid of that person's + hand-sorted crops, so the reference is one you trust) + centre the crop under review — context with the detection boxed, and + beneath it the 112x112 the embedder actually receives + right the person it matches better, if any, with both probabilities + +Pick a destination per row, then Export to download corrections.json and apply +it with apply_corrections.py. Nothing is moved by this script. + +Self-contained: images are inlined as data URIs and the page is opened from +disk, so no server runs and no face crop leaves the machine. + +Ordering is by P(other) - P(self), both from the global gallery sigmoid, so +rows where the evidence disagrees with the label float to the top and the +agreement cases sink. It is a review order, not a verdict — you are the +arbiter, which is the whole point of labelling by hand. +""" +import sys, glob, json, os, base64 +import numpy as np +import cv2 + +sys.path.insert(0, "/home/dtourolle/Development/Jray-project/scene-actor-extraction/build-ort") +import sae_embed + +ROOT = "/home/dtourolle/Development/Jray-project/scene-actor-extraction/" +M = ROOT + "models/" +EMBEDDER = M + "LVFace-B_Glint360K.onnx" +GALLERY = ROOT + "gallery_lvface.h5" +REF_CLIP = "5157344" # the clip sorted by hand — reference faces come from here +CLIPS = ["5157344", "5157339"] + +eng = sae_embed.FaceEmbedder(detector_model=M + "scrfd_500m_bnkps.onnx", + arcface_model=EMBEDDER, conf=0.5, nms=0.4, max_side=0) +cal = sae_embed.gallery_calibration(GALLERY) + + +def b64(img, size, q=72): + img = cv2.resize(img, (size, size)) + ok, buf = cv2.imencode(".jpg", img, [cv2.IMWRITE_JPEG_QUALITY, q]) + return "data:image/jpeg;base64," + base64.b64encode(buf).decode() if ok else "" + + +rows = [] +for clip in CLIPS: + man = {m["file"]: m for m in json.load(open(f"labelling/{clip}/manifest.json"))} + placed = {os.path.basename(p): (os.path.basename(os.path.dirname(p)), p) + for p in glob.glob(f"labelling/{clip}/*/*.jpg")} + by_frame = {} + for fname, (person, path) in placed.items(): + if fname in man and person != "unsorted": + by_frame.setdefault(man[fname]["frame"], []).append((fname, person, path)) + for frame, items in sorted(by_frame.items()): + img = cv2.imread(f"frames/d{clip}_{frame}.png") + if img is None: + continue + dets = eng.detect(img) + for fname, person, path in items: + i = man[fname]["idx"] + if i >= len(dets): + continue + lm = np.array(dets[i].landmarks, dtype=np.float32).reshape(5, 2) + crop = sae_embed.align_face(img, lm) + if crop is None: + continue + rows.append({"clip": clip, "person": person, "file": fname, "path": path, + "px": man[fname]["px"], "aligned": np.asarray(crop), + "emb": np.asarray(eng.embed_crop(crop), dtype=np.float32)}) + +people = sorted({r["person"] for r in rows}) +E = np.stack([r["emb"] for r in rows]) +lab = np.array([people.index(r["person"]) for r in rows]) +S = E @ E.T +np.fill_diagonal(S, -1.0) + +# reference face per person: medoid of their REF_CLIP crops +ref_img = {} +for k, p in enumerate(people): + idx = [i for i in np.where(lab == k)[0] if rows[i]["clip"] == REF_CLIP] + if not idx: + idx = list(np.where(lab == k)[0]) + if not idx: + continue + sub = S[np.ix_(idx, idx)].copy() + medoid = idx[int(np.argmax(sub.mean(axis=1)))] + ref_img[p] = b64(rows[medoid]["aligned"], 112) + +items = [] +for i, r in enumerate(rows): + k = lab[i] + same = [j for j in np.where(lab == k)[0] if j != i] + p_self = cal.probability(float(S[i, same].max())) if same else 0.0 + best_other, p_other = None, 0.0 + for k2, p2 in enumerate(people): + if k2 == k: + continue + other = np.where(lab == k2)[0] + if not len(other): + continue + pv = cal.probability(float(S[i, other].max())) + if pv > p_other: + p_other, best_other = pv, p2 + ctx = cv2.imread(r["path"]) + items.append({ + "file": r["file"], "clip": r["clip"], "person": r["person"], + "px": int(r["px"]), "p_self": round(p_self, 3), "p_other": round(p_other, 3), + "other": best_other, "delta": round(p_other - p_self, 3), + "ctx": b64(ctx, 150) if ctx is not None else "", + "ali": b64(r["aligned"], 112), + }) +items.sort(key=lambda x: -x["delta"]) + +payload = json.dumps({"people": people, "refs": ref_img, "items": items}) + +HTML = """JRay — label review + +
+

Label review

+ + + +
+
Left: the person this crop is filed under. Centre: the crop (context with the +detection boxed, and the 112×112 the embedder actually sees). Right: the person it matches +better, if any. Ordered by P(other) − P(self) — disagreements first.
+
+ +""" + +os.makedirs("labelling", exist_ok=True) +out = "labelling/review.html" +with open(out, "w") as f: + f.write(HTML.replace("__PAYLOAD__", payload)) +size = os.path.getsize(out) / 1e6 +flagged = sum(1 for i in items if i["delta"] > 0) +print(f"{out} {size:.1f} MB {len(items)} crops, {flagged} disagreements", file=sys.stderr) +print(f"open file://{os.path.abspath(out)}", file=sys.stderr) diff --git a/experiments/xsource/pose_label.py b/experiments/xsource/pose_label.py new file mode 100644 index 0000000..513d286 --- /dev/null +++ b/experiments/xsource/pose_label.py @@ -0,0 +1,200 @@ +#!/usr/bin/env python3 +"""Estimate head pose per crop, and build a page to confirm or correct it. + +Why not solvePnP on the 5 detector landmarks: those landmarks collapse on +turned faces, so the estimator breaks precisely on the crops whose pose we care +about. Run that way it reported the profile subject as the MOST frontal of the +four, which is how we know not to trust it. + +Instead the estimate comes from the MediaPipe face mesh (468 points, run via +OpenCV DNN — the same model rPPG-kahn uses) and a symmetry measure that needs +no 3D model: + + yaw_ratio = (dL - dR) / (dL + dR) + +over left/right symmetric vertex pairs, where dL and dR are each side's +distance from the face midline. Frontal ~ 0, profile -> +/-1. It degrades +gracefully because it averages many pairs rather than trusting any one point, +and it is scale- and translation-free. + +It is still an estimate. So this writes pose_review.html with the estimate +PRE-FILLED as a proposal, ordered by confidence, for you to correct — and the +correlation is only run against your corrected labels. If the estimate turns +out to disagree with you often, that is the finding, and the automatic number +gets dropped rather than reported. + +Bins are coarse on purpose: frontal / three-quarter / profile / down-or-hidden. +Finer than that and the labelling is slower and less reliable, and the question +("does pose explain the misses") does not need degrees. +""" +import sys, glob, json, os, base64 + +# sae_embed MUST be imported before cv2: OpenCV's DNN module loads the system +# libonnxruntime, which then shadows the newer one this module links against and +# the import fails on a missing symbol version. Order matters, so do not tidy +# these into alphabetical order. +sys.path.insert(0, "/home/dtourolle/Development/Jray-project/scene-actor-extraction/build-ort") +import sae_embed + +import numpy as np +import cv2 + +ROOT = "/home/dtourolle/Development/Jray-project/scene-actor-extraction/" +M = ROOT + "models/" +MESH = "/home/dtourolle/Development/rPPG-kahn/models/face_landmark.tflite" +CLIPS = ["5157344", "5157339"] +BINS = ["frontal", "three-quarter", "profile", "down-or-hidden"] + +# Symmetric vertex pairs (subject-left, subject-right) on the MediaPipe mesh: +# outer eye corners, inner eye corners, cheeks, mouth corners, jaw. +PAIRS = [(33, 263), (133, 362), (130, 359), (243, 463), + (61, 291), (91, 321), (146, 375), (58, 288), (172, 397), (215, 435)] +MIDLINE = [10, 168, 1, 4, 5, 195, 197, 152] # forehead -> nose -> chin + +net = cv2.dnn.readNetFromTFLite(MESH) +NAMES = net.getUnconnectedOutLayersNames() +LMI, PRI = NAMES.index("conv2d_21"), NAMES.index("conv2d_31") + +eng = sae_embed.FaceEmbedder(detector_model=M + "scrfd_500m_bnkps.onnx", + arcface_model=M + "LVFace-B_Glint360K.onnx", + conf=0.5, nms=0.4, max_side=0) + + +def mesh_pose(img, bbox, expand=1.6): + """(yaw_ratio, presence) or (nan, 0). yaw_ratio in [-1, 1], 0 = frontal.""" + x, y, w, h = bbox + cx, cy, s = x + w / 2, y + h / 2, max(w, h) * expand + crop = cv2.getRectSubPix(img, (int(s), int(s)), (float(cx), float(cy))) + net.setInput(cv2.dnn.blobFromImage(crop, 1 / 255.0, (192, 192), (0, 0, 0), swapRB=True)) + o = net.forward(NAMES) + pres = 1 / (1 + np.exp(-float(o[PRI].ravel()[0]))) + lm = o[LMI].reshape(468, 3)[:, :2] + mid = lm[MIDLINE] + # least-squares midline direction, then signed distance of each pair member + c = mid.mean(axis=0) + u, _, _ = np.linalg.svd(mid - c) + d = (mid - c) + axis = np.linalg.svd(d.T @ d)[0][:, 0] # principal direction of the midline + normal = np.array([-axis[1], axis[0]]) + ratios = [] + for a, b in PAIRS: + dl = float(np.dot(lm[a] - c, normal)) + dr = float(np.dot(lm[b] - c, normal)) + if abs(dl) + abs(dr) < 1e-6: + continue + ratios.append((abs(dl) - abs(dr)) / (abs(dl) + abs(dr))) + return (float(np.median(ratios)) if ratios else np.nan), pres + + +def b64(img, size, q=72): + ok, buf = cv2.imencode(".jpg", cv2.resize(img, (size, size)), + [cv2.IMWRITE_JPEG_QUALITY, q]) + return "data:image/jpeg;base64," + base64.b64encode(buf).decode() if ok else "" + + +items = [] +for clip in CLIPS: + lab = {os.path.basename(p): (os.path.basename(os.path.dirname(p)), p) + for p in glob.glob(f"labelling/{clip}/*/*.jpg") + if os.path.basename(os.path.dirname(p)) not in ("discard", "unsorted")} + man = {m["file"]: m for m in json.load(open(f"labelling/{clip}/manifest.json"))} + for frame in sorted({man[f]["frame"] for f in lab}): + img = cv2.imread(f"frames/d{clip}_{frame}.png") + dets = eng.detect(img) + for fname, (person, path) in lab.items(): + m = man[fname] + if m["frame"] != frame or m["idx"] >= len(dets): + continue + d = dets[m["idx"]] + lm5 = np.array(d.landmarks, dtype=np.float32).reshape(5, 2) + crop = sae_embed.align_face(img, lm5) + if crop is None: + continue + yaw, pres = mesh_pose(img, d.bbox) + a = abs(yaw) if not np.isnan(yaw) else 1.0 + guess = ("frontal" if a < 0.15 else "three-quarter" if a < 0.45 + else "profile") + if pres < 0.5: + guess = "down-or-hidden" # mesh could not fit at all + ctx = cv2.imread(path) + items.append({"file": fname, "clip": clip, "person": person, + "px": int(m["px"]), "yaw": None if np.isnan(yaw) else round(yaw, 3), + "pres": round(pres, 3), "guess": guess, + "ctx": b64(ctx, 140) if ctx is not None else "", + "ali": b64(np.asarray(crop), 112)}) + +# least-confident first: near a bin boundary, or the mesh could not fit +def uncertainty(it): + if it["pres"] < 0.5: + return 0.0 + a = abs(it["yaw"]) if it["yaw"] is not None else 1.0 + return min(abs(a - 0.15), abs(a - 0.45)) +items.sort(key=uncertainty) + +payload = json.dumps({"bins": BINS, "items": items}) + +HTML = """JRay — head pose labelling + +

Head pose

+
+
+ +""" +out = "labelling/pose_review.html" +open(out, "w").write(HTML.replace("__PAYLOAD__", payload)) +from collections import Counter +print(f"{out} {os.path.getsize(out)/1e6:.1f} MB {len(items)} crops", file=sys.stderr) +print(f"estimate: {dict(Counter(i['guess'] for i in items))}", file=sys.stderr) +print("\nestimated pose per person (does this match what you see?):", file=sys.stderr) +for p in sorted({i["person"] for i in items}): + for clip in CLIPS: + sub = [i for i in items if i["person"] == p and i["clip"] == clip] + if sub: + print(f" {p} {clip[-3:]}: {dict(Counter(i['guess'] for i in sub))}", + file=sys.stderr) +print(f"\nopen file://{os.path.abspath(out)}", file=sys.stderr) diff --git a/experiments/xsource/propose_labels.py b/experiments/xsource/propose_labels.py new file mode 100644 index 0000000..6d6920f --- /dev/null +++ b/experiments/xsource/propose_labels.py @@ -0,0 +1,242 @@ +#!/usr/bin/env python3 +"""Propose person labels for one clip using another clip's hand-sorted labels. + +Reads the clip you have already sorted (REF_CLIP) as ground truth, then proposes +a person for every crop in the other clip (TARGET_CLIP) and writes them into +matching folders for you to correct. + + python3 propose_labels.py # propose, write folders + sheets + python3 propose_labels.py --dry-run # report only, move nothing + +Output: + labelling//unsorted/A|B|C|D/ proposed, same names as the ref clip + labelling//unsorted/ left in place when no person is + confident enough to name + labelling/review_.jpg contact sheet spanning BOTH clips: + confirmed crops first, then + proposed ones with their P + +Correcting it: open a review sheet. Every face on it should be one person. The +lower block is the proposal — move any intruder to the right folder, or back to +unsorted/. The folder a file sits in is the ground truth; nothing downstream +reads the proposed name or its probability. + +The proposal is a labelling aid, never the label. Scoring the sweep against +embedding-derived labels would be circular: it keeps the faces the embedder +already gets right and drops the hard ones the sweep exists to find. Your +correction is what breaks that loop, which is why the proposal is deliberately +conservative and leaves anything doubtful unnamed. + +Assignment is on the calibrated probability, per-actor best-of-N, exactly as +identity_matcher_node does — never a bare cosine (AR-024). The calibration is +fitted on your labelled reference crops, which is what calibrate_gallery is for. +""" +import sys, glob, json, os, shutil +import numpy as np +import cv2 + +sys.path.insert(0, "/home/dtourolle/Development/Jray-project/scene-actor-extraction/build-ort") +import sae_embed + +ROOT = "/home/dtourolle/Development/Jray-project/scene-actor-extraction/" +M = ROOT + "models/" +# The embedder and the gallery whose calibration scores it MUST be the same +# model: a Platt fit is specific to one embedding space, so LVFace probabilities +# read through an ArcFace fit are meaningless. +EMBEDDER = M + "LVFace-B_Glint360K.onnx" +GALLERY = ROOT + "gallery_lvface.h5" # 291 actors, cached fit +REF_CLIP, TARGET_CLIP = "5157344", "5157339" +ASSIGN_P = 0.90 # propose a name only when this confident +SHEET_COLS = 8 +THUMB = 150 +DRY = "--dry-run" in sys.argv + +eng = sae_embed.FaceEmbedder(detector_model=M + "scrfd_500m_bnkps.onnx", + arcface_model=EMBEDDER, + conf=0.5, nms=0.4, max_side=0) + + +def embed_manifest(clip): + """Re-derive each dumped crop's embedding from its source frame, cached. + + The dumped .jpg is a context thumbnail for human eyes; the embedding must + come from the aligned crop the pipeline would actually produce, so the + frame is re-detected and the manifest's idx picks the same face. + + Detecting 24 4K frames per clip costs far more than the rest of this script + put together, and the result only changes when the manifest does — so it is + cached and keyed on the manifest's mtime. Delete cache/ to force a redo. + """ + man_path = f"labelling/{clip}/manifest.json" + cache_path = f"cache/emb_{clip}.npz" + os.makedirs("cache", exist_ok=True) + if os.path.exists(cache_path) and \ + os.path.getmtime(cache_path) >= os.path.getmtime(man_path): + z = np.load(cache_path, allow_pickle=True) + print(f"[cache] {clip}: {len(z['meta'])} embeddings reused", file=sys.stderr) + return [{**m, "emb": e} for m, e in zip(z["meta"], z["emb"])] + + man = json.load(open(man_path)) + by_frame = {} + for m in man: + by_frame.setdefault(m["frame"], []).append(m) + out = [] + for frame, ms in sorted(by_frame.items()): + img = cv2.imread(f"frames/d{clip}_{frame}.png") + if img is None: + sys.exit(f"missing frames/d{clip}_{frame}.png — extract with\n" + f" ffmpeg -i clips/{clip}.mp4 -vf fps=2 -frames:v 24 " + f"frames/d{clip}_%03d.png") + dets = eng.detect(img) + for m in ms: + if m["idx"] >= len(dets): + continue + d = dets[m["idx"]] + lm = np.array(d.landmarks, dtype=np.float32).reshape(5, 2) + crop = sae_embed.align_face(img, lm) + if crop is None: + continue + out.append({**m, "emb": np.asarray(eng.embed_crop(crop), dtype=np.float32)}) + + np.savez(cache_path, + meta=np.array([{k: v for k, v in o.items() if k != "emb"} for o in out], + dtype=object), + emb=np.stack([o["emb"] for o in out])) + print(f"[cache] {clip}: {len(out)} embeddings written to {cache_path}", + file=sys.stderr) + return out + + +def sorted_dirs(clip): + """Person folders you created, wherever you put them under labelling/.""" + found = {} + for path in glob.glob(f"labelling/{clip}/**/", recursive=True): + name = os.path.basename(path.rstrip("/")) + if name in ("unsorted", "discard") or name.startswith("5157"): + continue + files = [os.path.basename(f) for f in glob.glob(path + "*.jpg")] + if files: + found[name] = files + return found + + +# ── reference side: your labels ────────────────────────────────────────────── +ref_rows = embed_manifest(REF_CLIP) +ref_dirs = sorted_dirs(REF_CLIP) +if not ref_dirs: + sys.exit(f"no person folders under labelling/{REF_CLIP} — sort that clip first") +file_to_person = {f: p for p, fs in ref_dirs.items() for f in fs} + +ref = [(file_to_person[r["file"]], r["emb"]) for r in ref_rows + if r["file"] in file_to_person] +people = sorted({p for p, _ in ref}) +print(f"[ref] {REF_CLIP}: {len(ref)} labelled crops over {len(people)} people " + f"{ {p: sum(1 for q, _ in ref if q == p) for p in people} }", file=sys.stderr) + +R = np.stack([e for _, e in ref]) +r_actor = [people.index(p) for p, _ in ref] + +# The global gallery's sigmoid — NOT a fit over these four people. A Platt fit +# over a handful of identities saturates: it will hand back P=0.99 for faces it +# has no basis to separate, which is exactly how a wrong label acquires a +# convincing probability. The production fit spans the whole actor population, +# so a probability means the same thing here as it does in the matcher. +cal = sae_embed.gallery_calibration(GALLERY) +print(f"[calibration] global: {cal} assign boundary = sim " + f"{cal.boundary_at(ASSIGN_P):.4f}", file=sys.stderr) + +# ── target side: propose ───────────────────────────────────────────────────── +tgt_rows = embed_manifest(TARGET_CLIP) +T = np.stack([t["emb"] for t in tgt_rows]) +r_actor_arr = np.asarray(r_actor) +# per-actor best-of-N for every target crop at once: (n_people, n_target) +best_sim = np.stack([(R[r_actor_arr == people.index(p)] @ T.T).max(axis=0) + for p in people]) +proposals = [] +for j, t in enumerate(tgt_rows): + k = int(np.argmax(best_sim[:, j])) + prob = cal.probability(float(best_sim[k, j])) # calibrated, never a bare cosine + proposals.append({**t, "person": people[k] if prob >= ASSIGN_P else None, + "p": prob, "top1": people[k]}) + +# At the production threshold the global fit stays silent on most of these +# faces, which is the honest answer for profile and downward-gaze shots — but a +# labelling aid wants throughput, not caution. --all proposes the top-1 person +# for every crop and orders the review sheets by descending probability, so the +# proposals degrade visibly down the sheet and you can stop correcting where +# they stop being right. The probability is shown, never hidden. +if "--all" in sys.argv: + for x in proposals: + x["person"] = x["top1"] + +named = [x for x in proposals if x["person"]] +print(f"[propose] {TARGET_CLIP}: {len(named)}/{len(proposals)} named at P>={ASSIGN_P}; " + f"{len(proposals) - len(named)} left unsorted", file=sys.stderr) +for p in people: + got = [x for x in named if x["person"] == p] + if got: + ps = [x["p"] for x in got] + print(f" {p}: {len(got):>3} crops P {min(ps):.3f}–{max(ps):.3f}", file=sys.stderr) + +if DRY: + sys.exit(0) + +# ── write proposed folders, mirroring the ref clip's layout ────────────────── +ref_parent = os.path.dirname(next(iter(glob.glob(f"labelling/{REF_CLIP}/**/{people[0]}/", + recursive=True))).rstrip("/")) +tgt_parent = ref_parent.replace(REF_CLIP, TARGET_CLIP) +for p in people: + d = f"{tgt_parent}/{p}" + if os.path.isdir(d): # never clobber corrections already made + print(f"[skip] {d} exists — leaving your sorting alone", file=sys.stderr) + continue + os.makedirs(d, exist_ok=True) +def find_crop(clip, fname): + """Locate a crop wherever it currently sits under labelling/.""" + hits = glob.glob(f"labelling/{clip}/**/{fname}", recursive=True) + return hits[0] if hits else None + +moved = 0 +for x in named: + src = find_crop(TARGET_CLIP, x["file"]) + dst = f"{tgt_parent}/{x['person']}/{x['file']}" + if src and os.path.abspath(src) != os.path.abspath(dst): + shutil.move(src, dst) + moved += 1 +print(f"[write] moved {moved} crops into proposed folders", file=sys.stderr) + +# ── review sheets: confirmed block, then proposed block ───────────────────── +def load(clip, person, fname): + for cand in glob.glob(f"labelling/{clip}/**/{person}/{fname}", recursive=True): + return cv2.imread(cand) + return None + +for person in people: + conf = [(REF_CLIP, f, None) for f in ref_dirs.get(person, [])] + prop = sorted([(TARGET_CLIP, x["file"], x["p"]) for x in named + if x["person"] == person], + key=lambda t: -t[2]) # most confident first + items = conf + prop + if not items: + continue + rows_n = (len(items) + SHEET_COLS - 1) // SHEET_COLS + sheet = np.full((rows_n * (THUMB + 26), SHEET_COLS * THUMB, 3), 30, np.uint8) + for n, (clip, fname, p) in enumerate(items): + img = load(clip, person, fname) + if img is None: + continue + rr, cc = divmod(n, SHEET_COLS) + y, x = rr * (THUMB + 26), cc * THUMB + sheet[y:y + THUMB, x:x + THUMB] = cv2.resize(img, (THUMB, THUMB)) + if p is None: + tag, col = f"{clip[-3:]} CONFIRMED", (170, 170, 170) + else: + tag, col = f"{clip[-3:]} P={p:.2f}", (140, 255, 140) + cv2.putText(sheet, tag, (x + 3, y + THUMB + 17), + cv2.FONT_HERSHEY_SIMPLEX, 0.42, col, 1) + cv2.imwrite(f"labelling/review_{person}.jpg", sheet) + print(f" review_{person}.jpg: {len(conf)} confirmed + {len(prop)} proposed", + file=sys.stderr) + +json.dump({x["file"]: {"person": x["person"], "p": x["p"]} for x in proposals}, + open(f"labelling/proposed_{TARGET_CLIP}.json", "w"), indent=1) diff --git a/experiments/xsource/redraw_boxes.py b/experiments/xsource/redraw_boxes.py new file mode 100644 index 0000000..7e64269 --- /dev/null +++ b/experiments/xsource/redraw_boxes.py @@ -0,0 +1,55 @@ +#!/usr/bin/env python3 +"""Redraw every dumped crop with its detection box marked. + +The original thumbnails padded by 0.5x the face on each side for +recognisability, which in a crowded frame pulls a neighbour into shot — often +more prominently than the subject. A label cannot be corrected from a picture +that does not say which face it refers to. + +This rewrites each .jpg IN PLACE, wherever it currently sits, so any sorting +already done is preserved: only the pixels change, never the filename or the +folder. Re-run it after dump_faces.py, and re-check any sorting done before it. +""" +import glob, json, os, sys +import cv2 + +CLIPS = ["5157339", "5157344"] +OUT = 256 + +for clip in CLIPS: + man = {m["file"]: m for m in json.load(open(f"labelling/{clip}/manifest.json"))} + n = 0 + for path in glob.glob(f"labelling/{clip}/**/*.jpg", recursive=True): + fname = os.path.basename(path) + m = man.get(fname) + if m is None: + continue + img = cv2.imread(f"frames/d{clip}_{m['frame']}.png") + if img is None: + sys.exit(f"missing frames/d{clip}_{m['frame']}.png") + + x, y, w, h = (int(v) for v in m["bbox"]) + pad = int(0.55 * max(w, h)) + x0, y0 = max(0, x - pad), max(0, y - pad) + x1, y1 = min(img.shape[1], x + w + pad), min(img.shape[0], y + h + pad) + sub = img[y0:y1, x0:x1].copy() + + # Box in the sub-image's coordinates, drawn before the resize so the + # line lands exactly on the face at any output size. + cv2.rectangle(sub, (x - x0, y - y0), (x - x0 + w, y - y0 + h), (0, 0, 255), 3) + # Dim everything outside the box so the subject is unmistakable even + # when a neighbour's face is larger or better lit. + mask = sub.copy() + mask[y - y0:y - y0 + h, x - x0:x - x0 + w] = 0 + sub = cv2.addWeighted(sub, 1.0, mask, -0.35, 0) + + scale = OUT / max(sub.shape[:2]) + sub = cv2.resize(sub, (int(sub.shape[1] * scale), int(sub.shape[0] * scale))) + canvas = cv2.copyMakeBorder( + sub, 0, max(0, OUT - sub.shape[0]), 0, max(0, OUT - sub.shape[1]), + cv2.BORDER_CONSTANT, value=(20, 20, 20))[:OUT, :OUT] + cv2.putText(canvas, f"{int(m['px'])}px", (5, OUT - 8), + cv2.FONT_HERSHEY_SIMPLEX, 0.45, (0, 255, 255), 1) + cv2.imwrite(path, canvas) + n += 1 + print(f"[{clip}] redrew {n} crops in place", file=sys.stderr) diff --git a/experiments/xsource/resolution_sweep.py b/experiments/xsource/resolution_sweep.py new file mode 100644 index 0000000..6603f75 --- /dev/null +++ b/experiments/xsource/resolution_sweep.py @@ -0,0 +1,182 @@ +#!/usr/bin/env python3 +"""Impact of input resolution on cross-source identification. + +Gallery is built from one clip at NATIVE resolution. Probes come from the other +clip with the WHOLE FRAME downscaled before it reaches the detector, so +detection and landmark regression degrade together with the pixels. That is the +measurement VR-005 structurally could not make: it degraded an already-aligned +112x112 crop, holding alignment perfect, so it isolated the embedder's +resolution sensitivity and excluded everything upstream of it. + + python3 resolution_sweep.py [--gallery-clip 5157339] [--detector scrfd_500m_bnkps.onnx] + +Ground truth +------------ +Hand-sorted person folders. Probe detections at reduced scale are tied back to +a labelled face GEOMETRICALLY — the box is mapped to native coordinates and +matched by IoU. Never by embedding similarity, which would be circular: it +would keep the faces the embedder still gets right and silently drop the ones +this sweep exists to find. + +A probe whose label is only in the probe clip is OUT OF GALLERY. Naming it is a +true out-of-cast misID, the error the per-scene scorer weights 10x, so it is +counted separately from naming the wrong gallery member. + +Metric +------ +The calibrated probability from the PRODUCTION gallery sigmoid, never a raw +cosine (AR-024). Per-actor best-of-N similarity -> probability -> accept above +prob_threshold. This is identification, so the matcher's prior applies; +config.hpp has match_prior 0.5, i.e. log_prior_odds = 0. + +Everything runs through the shipped C++ via sae_embed. +""" +import sys, glob, json, os, argparse +import numpy as np +import cv2 + +sys.path.insert(0, "/home/dtourolle/Development/Jray-project/scene-actor-extraction/build-ort") +import sae_embed + +ROOT = "/home/dtourolle/Development/Jray-project/scene-actor-extraction/" +M = ROOT + "models/" +PROB_THRESHOLD = 0.754 # config.hpp:67 +LOG_PRIOR_ODDS = 0.0 # config.hpp:61 match_prior=0.5 +IOU_MIN = 0.3 # geometric label carry-down +SCALES = [1.0, 0.8, 0.6, 0.5, 0.4, 0.3, 0.25, 0.2, 0.15, 0.12, 0.09, 0.06] + +ap = argparse.ArgumentParser() +ap.add_argument("--gallery-clip", default="5157339") +ap.add_argument("--probe-clip", default="5157344") +ap.add_argument("--detector", default="scrfd_500m_bnkps.onnx") +ap.add_argument("--embedder", default="LVFace-B_Glint360K.onnx") +ap.add_argument("--gallery-calibration", default=ROOT + "gallery_lvface.h5") +ap.add_argument("--out", default="results_resolution_sweep.json") +args = ap.parse_args() + +eng = sae_embed.FaceEmbedder(detector_model=M + args.detector, + arcface_model=M + args.embedder, + conf=0.5, nms=0.4, max_side=0) +cal = sae_embed.gallery_calibration(args.gallery_calibration) +print(f"[calibration] global: {cal}", file=sys.stderr) + + +def labelled(clip): + """{filename: person} from the hand-sorted folders, ignoring discard.""" + out = {} + for path in glob.glob(f"labelling/{clip}/*/*.jpg"): + person = os.path.basename(os.path.dirname(path)) + if person in ("discard", "unsorted"): + continue + out[os.path.basename(path)] = person + return out + + +def manifest(clip): + return {m["file"]: m for m in json.load(open(f"labelling/{clip}/manifest.json"))} + + +def iou(a, b): + ax, ay, aw, ah = a; bx, by, bw, bh = b + x0, y0 = max(ax, bx), max(ay, by) + x1, y1 = min(ax + aw, bx + bw), min(ay + ah, by + bh) + if x1 <= x0 or y1 <= y0: + return 0.0 + inter = (x1 - x0) * (y1 - y0) + return inter / (aw * ah + bw * bh - inter) + + +# ── gallery: native resolution, labelled faces only ────────────────────────── +g_lab, g_man = labelled(args.gallery_clip), manifest(args.gallery_clip) +gal = {} +for frame in sorted({g_man[f]["frame"] for f in g_lab}): + img = cv2.imread(f"frames/d{args.gallery_clip}_{frame}.png") + dets = eng.detect(img) + for fname, person in g_lab.items(): + m = g_man[fname] + if m["frame"] != frame or m["idx"] >= len(dets): + continue + lm = np.array(dets[m["idx"]].landmarks, dtype=np.float32).reshape(5, 2) + crop = sae_embed.align_face(img, lm) + if crop is None: + continue + gal.setdefault(person, []).append(np.asarray(eng.embed_crop(crop), dtype=np.float32)) +gal = {p: np.stack(v) for p, v in gal.items() if v} +people = sorted(gal) +print(f"[gallery] {args.gallery_clip} @native: " + f"{ {p: len(v) for p, v in gal.items()} }", file=sys.stderr) + +# ── probe ground truth at native resolution ────────────────────────────────── +p_lab, p_man = labelled(args.probe_clip), manifest(args.probe_clip) +truth = {} # frame -> [(bbox_native, person)] +for fname, person in p_lab.items(): + m = p_man[fname] + truth.setdefault(m["frame"], []).append((m["bbox"], person)) +n_out = sum(1 for p in set(p_lab.values()) if p not in people) +print(f"[probe] {args.probe_clip}: {len(p_lab)} labelled faces, " + f"{len(set(p_lab.values()))} people, {n_out} of them out-of-gallery", + file=sys.stderr) + +# ── sweep ──────────────────────────────────────────────────────────────────── +print(f"\n{'scale':>6}{'frame':>11}{'face px':>9}{'found':>7}{'matched':>9}" + f"{'TPI':>8}{'FPI-in':>8}{'FPI-out':>9}{'TBI':>8}") +results = [] +for s in SCALES: + tpi = fpi_in = fpi_out = tbi = 0 + n_found = n_matched = 0 + pxs = [] + for frame, gts in sorted(truth.items()): + img = cv2.imread(f"frames/d{args.probe_clip}_{frame}.png") + if s != 1.0: + img = cv2.resize(img, None, fx=s, fy=s, interpolation=cv2.INTER_AREA) + dets = eng.detect(img) + n_found += len(dets) + for d in dets: + x, y, w, h = d.bbox + native = (x / s, y / s, w / s, h / s) # geometric carry-down + best, best_iou = None, 0.0 + for gt_box, person in gts: + v = iou(native, gt_box) + if v > best_iou: + best_iou, best = v, person + if best_iou < IOU_MIN: + continue # spurious / unlabelled + n_matched += 1 + pxs.append(min(w, h)) + + lm = np.array(d.landmarks, dtype=np.float32).reshape(5, 2) + crop = sae_embed.align_face(img, lm) + if crop is None: + tbi += 1 # degenerate alignment + continue + emb = np.asarray(eng.embed_crop(crop), dtype=np.float32) + best_p, best_name = 0.0, None + for p in people: # per-actor best-of-N + prob = cal.probability(float((gal[p] @ emb).max()), LOG_PRIOR_ODDS) + if prob > best_p: + best_p, best_name = prob, p + if best_p <= PROB_THRESHOLD: + tbi += 1 + elif best not in people: + fpi_out += 1 # named someone absent from the gallery + elif best_name == best: + tpi += 1 + else: + fpi_in += 1 + n = max(1, n_matched) + med_px = float(np.median(pxs)) if pxs else 0.0 + print(f"{s:>6.2f}{f'{int(4096*s)}x{int(2160*s)}':>11}{med_px:>9.0f}" + f"{n_found:>7}{n_matched:>9}" + f"{100*tpi/n:>7.1f}%{100*fpi_in/n:>7.1f}%{100*fpi_out/n:>8.1f}%{100*tbi/n:>7.1f}%") + results.append({"scale": s, "median_face_px": med_px, "detections": n_found, + "matched_to_truth": n_matched, "tpi_pct": 100*tpi/n, + "fpi_in_gallery_pct": 100*fpi_in/n, "fpi_out_of_gallery_pct": 100*fpi_out/n, + "tbi_pct": 100*tbi/n}) + +json.dump({"gallery_clip": args.gallery_clip, "probe_clip": args.probe_clip, + "detector": args.detector, "embedder": args.embedder, + "prob_threshold": PROB_THRESHOLD, "log_prior_odds": LOG_PRIOR_ODDS, + "calibration": {"a": cal.a, "b": cal.b}, + "gallery_people": people, "results": results}, + open(args.out, "w"), indent=2) +print(f"\nwrote {args.out}", file=sys.stderr) diff --git a/experiments/xsource/verify_labels.py b/experiments/xsource/verify_labels.py new file mode 100644 index 0000000..a556d51 --- /dev/null +++ b/experiments/xsource/verify_labels.py @@ -0,0 +1,167 @@ +#!/usr/bin/env python3 +"""Integrity check on the labelled set, before it is used as ground truth. + +Checks, loudest failure first: + +1. INDEX INTEGRITY. Each crop's embedding is taken by re-detecting its source + frame and indexing with the manifest's `idx`. If detection order is not + reproducible, the thumbnail you sorted and the embedding that gets scored + are different faces — you would see a correct picture and score the wrong + person, with nothing to signal it. Every crop's re-detected bbox is compared + against the manifest's. + +2. NO CROP IN TWO FOLDERS, and every manifest entry accounted for — so a + move that half-completed cannot silently duplicate or drop a label. + +3. ALIGNMENT. The 112x112 warp is what the embedder actually sees; the + thumbnail is only context for your eyes. verify_.jpg pairs them: + context-with-box on top, the real aligned crop beneath. A profile face whose + alignment has collapsed is obvious there and nowhere else. + +4. SEPARATION. Per person, the calibrated P of their own crops against the + other people's, using the global gallery sigmoid. A label set where someone + matches another person better than themselves is mislabelled. + +Nothing here changes a label. It reports. +""" +import sys, glob, json, os +import numpy as np +import cv2 + +sys.path.insert(0, "/home/dtourolle/Development/Jray-project/scene-actor-extraction/build-ort") +import sae_embed + +ROOT = "/home/dtourolle/Development/Jray-project/scene-actor-extraction/" +M = ROOT + "models/" +EMBEDDER = M + "LVFace-B_Glint360K.onnx" +GALLERY = ROOT + "gallery_lvface.h5" +CLIPS = ["5157344", "5157339"] +THUMB = 130 +COLS = 10 + +eng = sae_embed.FaceEmbedder(detector_model=M + "scrfd_500m_bnkps.onnx", + arcface_model=EMBEDDER, conf=0.5, nms=0.4, max_side=0) + +fail = 0 +rows = [] + +for clip in CLIPS: + man = {m["file"]: m for m in json.load(open(f"labelling/{clip}/manifest.json"))} + + # where each crop currently sits -> its label + placed = {} + for path in glob.glob(f"labelling/{clip}/*/*.jpg"): + person = os.path.basename(os.path.dirname(path)) + if person in ("discard", "unsorted"): + continue # not people; scoring them would invent an extra identity + fname = os.path.basename(path) + if fname in placed: + print(f"[FAIL] {fname} appears in both {placed[fname][0]} and {person}") + fail += 1 + placed[fname] = (person, path) + + missing = set(man) - set(placed) + extra = set(placed) - set(man) + if missing: + print(f"[warn] {clip}: {len(missing)} manifest crops not in any folder") + if extra: + print(f"[FAIL] {clip}: {len(extra)} files with no manifest entry: " + f"{sorted(extra)[:3]}") + fail += 1 + + # index integrity + alignment, frame by frame + by_frame = {} + for fname, (person, path) in placed.items(): + if fname in man: + by_frame.setdefault(man[fname]["frame"], []).append((fname, person, path)) + + bad_idx = 0 + for frame, items in sorted(by_frame.items()): + img = cv2.imread(f"frames/d{clip}_{frame}.png") + if img is None: + print(f"[FAIL] missing frames/d{clip}_{frame}.png") + fail += 1 + continue + dets = eng.detect(img) + for fname, person, path in items: + m = man[fname] + i = m["idx"] + if i >= len(dets): + print(f"[FAIL] {fname}: idx {i} >= {len(dets)} detections now") + bad_idx += 1 + continue + got = [float(v) for v in dets[i].bbox] + want = m["bbox"] + if max(abs(a - b) for a, b in zip(got, want)) > 1.0: + print(f"[FAIL] {fname}: manifest bbox {[round(v) for v in want]} " + f"!= re-detected {[round(v) for v in got]}") + bad_idx += 1 + continue + lm = np.array(dets[i].landmarks, dtype=np.float32).reshape(5, 2) + crop = sae_embed.align_face(img, lm) + if crop is None: + print(f"[warn] {fname}: alignment degenerate, no crop reaches the embedder") + continue + rows.append({"clip": clip, "person": person, "file": fname, "path": path, + "px": m["px"], "aligned": np.asarray(crop), + "emb": np.asarray(eng.embed_crop(crop), dtype=np.float32)}) + fail += bad_idx + print(f"[{clip}] {len(placed)} placed, {len(by_frame)} frames, " + f"index mismatches: {bad_idx}") + +if not rows: + sys.exit("nothing to verify") + +# ── separation, through the global gallery sigmoid ─────────────────────────── +cal = sae_embed.gallery_calibration(GALLERY) +E = np.stack([r["emb"] for r in rows]) +people = sorted({r["person"] for r in rows}) +lab = np.array([people.index(r["person"]) for r in rows]) +S = E @ E.T +np.fill_diagonal(S, -1.0) + +print(f"\n{'person':>8}{'crops':>7}{'344':>6}{'339':>6}" + f"{'P(self)':>10}{'P(other)':>10}{'worst':>8}") +for k, p in enumerate(people): + mine = np.where(lab == k)[0] + if len(mine) < 2: + continue + self_sim = S[np.ix_(mine, mine)].max(axis=1) + other_sim = S[np.ix_(mine, np.where(lab != k)[0])].max(axis=1) + p_self = np.array([cal.probability(float(s)) for s in self_sim]) + p_other = np.array([cal.probability(float(s)) for s in other_sim]) + n344 = sum(1 for i in mine if rows[i]["clip"] == "5157344") + n339 = len(mine) - n344 + # a crop that matches someone else better than anyone of its own label + worst = int((other_sim > self_sim).sum()) + print(f"{p:>8}{len(mine):>7}{n344:>6}{n339:>6}" + f"{np.median(p_self):>10.3f}{np.median(p_other):>10.3f}{worst:>8}") + if worst: + for i in mine[other_sim > self_sim]: + print(f" suspect: {rows[i]['file']} " + f"P(self)={cal.probability(float(self_sim[list(mine).index(i)])):.3f} " + f"< P(other)={cal.probability(float(other_sim[list(mine).index(i)])):.3f}") + +# ── verify sheets: context+box over the actual aligned crop ────────────────── +for p in people: + items = [r for r in rows if r["person"] == p] + items.sort(key=lambda r: (r["clip"], r["file"])) + n = len(items) + sheet_rows = (n + COLS - 1) // COLS + H = THUMB * 2 + 22 + sheet = np.full((sheet_rows * H, COLS * THUMB, 3), 25, np.uint8) + for j, r in enumerate(items): + rr, cc = divmod(j, COLS) + y, x = rr * H, cc * THUMB + ctx = cv2.imread(r["path"]) + if ctx is not None: + sheet[y:y + THUMB, x:x + THUMB] = cv2.resize(ctx, (THUMB, THUMB)) + sheet[y + THUMB:y + 2 * THUMB, x:x + THUMB] = cv2.resize(r["aligned"], (THUMB, THUMB)) + cv2.putText(sheet, f"{r['clip'][-3:]} {int(r['px'])}px", + (x + 3, y + 2 * THUMB + 15), + cv2.FONT_HERSHEY_SIMPLEX, 0.38, (150, 220, 150), 1) + cv2.imwrite(f"labelling/verify_{p}.jpg", sheet) + print(f" verify_{p}.jpg: {n} crops (top row context, bottom row what the embedder sees)") + +print(f"\n{'PASS' if fail == 0 else f'{fail} FAILURES'}") +sys.exit(1 if fail else 0)