#!/usr/bin/env python3 """Impact of input resolution on cross-source identification. Gallery is built from one clip at NATIVE resolution. Probes come from the other clip with the WHOLE FRAME downscaled before it reaches the detector, so detection and landmark regression degrade together with the pixels. That is the measurement VR-005 structurally could not make: it degraded an already-aligned 112x112 crop, holding alignment perfect, so it isolated the embedder's resolution sensitivity and excluded everything upstream of it. python3 resolution_sweep.py [--gallery-clip 5157339] [--detector scrfd_500m_bnkps.onnx] Ground truth ------------ Hand-sorted person folders. Probe detections at reduced scale are tied back to a labelled face GEOMETRICALLY — the box is mapped to native coordinates and matched by IoU. Never by embedding similarity, which would be circular: it would keep the faces the embedder still gets right and silently drop the ones this sweep exists to find. A probe whose label is only in the probe clip is OUT OF GALLERY. Naming it is a true out-of-cast misID, the error the per-scene scorer weights 10x, so it is counted separately from naming the wrong gallery member. Metric ------ The calibrated probability from the PRODUCTION gallery sigmoid, never a raw cosine (AR-024). Per-actor best-of-N similarity -> probability -> accept above prob_threshold. This is identification, so the matcher's prior applies; config.hpp has match_prior 0.5, i.e. log_prior_odds = 0. Everything runs through the shipped C++ via sae_embed. """ import sys, glob, json, os, argparse import numpy as np import cv2 sys.path.insert(0, "/home/dtourolle/Development/Jray-project/scene-actor-extraction/build-ort") import sae_embed ROOT = "/home/dtourolle/Development/Jray-project/scene-actor-extraction/" M = ROOT + "models/" PROB_THRESHOLD = 0.754 # config.hpp:67 LOG_PRIOR_ODDS = 0.0 # config.hpp:61 match_prior=0.5 IOU_MIN = 0.3 # geometric label carry-down SCALES = [1.0, 0.8, 0.6, 0.5, 0.4, 0.3, 0.25, 0.2, 0.15, 0.12, 0.09, 0.06] ap = argparse.ArgumentParser() ap.add_argument("--gallery-clip", default="5157339") ap.add_argument("--probe-clip", default="5157344") ap.add_argument("--detector", default="scrfd_500m_bnkps.onnx") ap.add_argument("--embedder", default="LVFace-B_Glint360K.onnx") ap.add_argument("--gallery-calibration", default=ROOT + "gallery_lvface.h5") ap.add_argument("--out", default="results_resolution_sweep.json") args = ap.parse_args() eng = sae_embed.FaceEmbedder(detector_model=M + args.detector, arcface_model=M + args.embedder, conf=0.5, nms=0.4, max_side=0) cal = sae_embed.gallery_calibration(args.gallery_calibration) print(f"[calibration] global: {cal}", file=sys.stderr) def labelled(clip): """{filename: person} from the hand-sorted folders, ignoring discard.""" out = {} for path in glob.glob(f"labelling/{clip}/*/*.jpg"): person = os.path.basename(os.path.dirname(path)) if person in ("discard", "unsorted"): continue out[os.path.basename(path)] = person return out def manifest(clip): return {m["file"]: m for m in json.load(open(f"labelling/{clip}/manifest.json"))} def iou(a, b): ax, ay, aw, ah = a; bx, by, bw, bh = b x0, y0 = max(ax, bx), max(ay, by) x1, y1 = min(ax + aw, bx + bw), min(ay + ah, by + bh) if x1 <= x0 or y1 <= y0: return 0.0 inter = (x1 - x0) * (y1 - y0) return inter / (aw * ah + bw * bh - inter) # ── gallery: native resolution, labelled faces only ────────────────────────── g_lab, g_man = labelled(args.gallery_clip), manifest(args.gallery_clip) gal = {} for frame in sorted({g_man[f]["frame"] for f in g_lab}): img = cv2.imread(f"frames/d{args.gallery_clip}_{frame}.png") dets = eng.detect(img) for fname, person in g_lab.items(): m = g_man[fname] if m["frame"] != frame or m["idx"] >= len(dets): continue lm = np.array(dets[m["idx"]].landmarks, dtype=np.float32).reshape(5, 2) crop = sae_embed.align_face(img, lm) if crop is None: continue gal.setdefault(person, []).append(np.asarray(eng.embed_crop(crop), dtype=np.float32)) gal = {p: np.stack(v) for p, v in gal.items() if v} people = sorted(gal) print(f"[gallery] {args.gallery_clip} @native: " f"{ {p: len(v) for p, v in gal.items()} }", file=sys.stderr) # ── probe ground truth at native resolution ────────────────────────────────── p_lab, p_man = labelled(args.probe_clip), manifest(args.probe_clip) truth = {} # frame -> [(bbox_native, person)] for fname, person in p_lab.items(): m = p_man[fname] truth.setdefault(m["frame"], []).append((m["bbox"], person)) n_out = sum(1 for p in set(p_lab.values()) if p not in people) print(f"[probe] {args.probe_clip}: {len(p_lab)} labelled faces, " f"{len(set(p_lab.values()))} people, {n_out} of them out-of-gallery", file=sys.stderr) # ── sweep ──────────────────────────────────────────────────────────────────── print(f"\n{'scale':>6}{'frame':>11}{'face px':>9}{'found':>7}{'matched':>9}" f"{'TPI':>8}{'FPI-in':>8}{'FPI-out':>9}{'TBI':>8}") results = [] for s in SCALES: tpi = fpi_in = fpi_out = tbi = 0 n_found = n_matched = 0 pxs = [] for frame, gts in sorted(truth.items()): img = cv2.imread(f"frames/d{args.probe_clip}_{frame}.png") if s != 1.0: img = cv2.resize(img, None, fx=s, fy=s, interpolation=cv2.INTER_AREA) dets = eng.detect(img) n_found += len(dets) for d in dets: x, y, w, h = d.bbox native = (x / s, y / s, w / s, h / s) # geometric carry-down best, best_iou = None, 0.0 for gt_box, person in gts: v = iou(native, gt_box) if v > best_iou: best_iou, best = v, person if best_iou < IOU_MIN: continue # spurious / unlabelled n_matched += 1 pxs.append(min(w, h)) lm = np.array(d.landmarks, dtype=np.float32).reshape(5, 2) crop = sae_embed.align_face(img, lm) if crop is None: tbi += 1 # degenerate alignment continue emb = np.asarray(eng.embed_crop(crop), dtype=np.float32) best_p, best_name = 0.0, None for p in people: # per-actor best-of-N prob = cal.probability(float((gal[p] @ emb).max()), LOG_PRIOR_ODDS) if prob > best_p: best_p, best_name = prob, p if best_p <= PROB_THRESHOLD: tbi += 1 elif best not in people: fpi_out += 1 # named someone absent from the gallery elif best_name == best: tpi += 1 else: fpi_in += 1 n = max(1, n_matched) med_px = float(np.median(pxs)) if pxs else 0.0 print(f"{s:>6.2f}{f'{int(4096*s)}x{int(2160*s)}':>11}{med_px:>9.0f}" f"{n_found:>7}{n_matched:>9}" f"{100*tpi/n:>7.1f}%{100*fpi_in/n:>7.1f}%{100*fpi_out/n:>8.1f}%{100*tbi/n:>7.1f}%") results.append({"scale": s, "median_face_px": med_px, "detections": n_found, "matched_to_truth": n_matched, "tpi_pct": 100*tpi/n, "fpi_in_gallery_pct": 100*fpi_in/n, "fpi_out_of_gallery_pct": 100*fpi_out/n, "tbi_pct": 100*tbi/n}) json.dump({"gallery_clip": args.gallery_clip, "probe_clip": args.probe_clip, "detector": args.detector, "embedder": args.embedder, "prob_threshold": PROB_THRESHOLD, "log_prior_odds": LOG_PRIOR_ODDS, "calibration": {"a": cal.a, "b": cal.b}, "gallery_people": people, "results": results}, open(args.out, "w"), indent=2) print(f"\nwrote {args.out}", file=sys.stderr)