study(VR-013): cross-source identification probe over input resolution
Gallery from one recording, probes from another, sweeping the probe's input resolution end to end. VR-005 asked the same question over gallery mugshots but degraded an already-aligned 112x112 crop with alignment held perfect, so it isolates the embedder. Here the whole frame is downscaled before the detector, so detection and landmark regression degrade with it — which is most of the difference. Corpus is two 4096x2160 clips of one shoot, four people, hand-sorted. Ground truth is sorted by hand and gated by verify_labels.py; labels carried down the scales geometrically by box position, never by embedding similarity, which would keep only the faces the embedder already gets right and drop the ones the sweep exists to find. Findings, all scored through the production gallery sigmoid at prob_threshold 0.754 — never a raw cosine: - Holding 90% of the plateau needs ~50 px end to end, against VR-005's ~22 px. min_face_px at 40 looks right; 32 would admit faces in the falling region. - FPI is 0.0% at every scale. Resolution loss goes entirely to TBI. - The ceiling is cross-view, not resolution: everyone matches themselves within a recording (0.55-0.85) and collapses across two (0.14-0.45, threshold 0.335). Only the subject with frontal *gallery* references identified reliably, whatever their probe pose — so the lever is gallery pose coverage, not a better landmark source. - Averaging SCRFD's overlapping detections instead of discarding them at NMS lifts cross-recording TPI 41% -> 49%, for one forward pass and no extra model. Four identities and one shoot, so the shape is the result and the absolute rates are not. Both clips contain all four people, so there is no out-of-gallery class and the 10x-weighted out-of-cast misID is untested here. Clips, frames, hand-sorted crops and results are gitignored and belong in the artifact registry — the sorting is human ground truth and expensive to redo. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com> TRACES: VR-013 | AR-002, AR-005, AR-024
This commit is contained in:
Claude Opus 5
@@ -0,0 +1,165 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Landmark voting: average SCRFD's overlapping detections instead of discarding them.
|
||||
|
||||
SCRFD predicts a face from many anchors; NMS keeps the single highest-scoring
|
||||
box and throws the rest away. Each discarded box carries its own 5-landmark
|
||||
estimate of the SAME face, so the survivors are one sample from a distribution
|
||||
we could be averaging over.
|
||||
|
||||
baseline conf 0.50, nms 0.40 — the shipped settings, one box per face
|
||||
voted conf 0.30, nms 0.90 — duplicates survive, then grouped by IoU and
|
||||
the 5 landmarks averaged, weighted by detection confidence
|
||||
|
||||
Why this is worth trying when the mesh failed: the mesh moved the landmarks off
|
||||
the definition ArcFace was trained on (a lip-ring centroid is not an annotated
|
||||
mouth corner), and the embedder punished it. A confidence-weighted mean of
|
||||
SCRFD's OWN landmark predictions is the same kind of point, just with less
|
||||
variance — it should stay on-distribution while being steadier.
|
||||
|
||||
Scored on cross-clip identification through the production sigmoid, which is
|
||||
the thing that actually broke. Raw similarity shown only to locate the
|
||||
threshold; it decides nothing.
|
||||
|
||||
LD_PRELOAD=/usr/lib/libcudnn_cnn.so.9 python3 landmark_voting.py
|
||||
"""
|
||||
import sys, glob, json, os
|
||||
|
||||
sys.path.insert(0, "/home/dtourolle/Development/Jray-project/scene-actor-extraction/build-ort")
|
||||
import sae_embed # before cv2 — see alignment_compare.py
|
||||
|
||||
import numpy as np
|
||||
import cv2
|
||||
|
||||
ROOT = "/home/dtourolle/Development/Jray-project/scene-actor-extraction/"
|
||||
M = ROOT + "models/"
|
||||
CLIPS = ["5157344", "5157339"]
|
||||
PROB_THRESHOLD = 0.754
|
||||
GROUP_IOU = 0.55 # detections overlapping this much are the same face
|
||||
MATCH_IOU = 0.35 # tie a detection to the hand-labelled face
|
||||
|
||||
base_eng = sae_embed.FaceEmbedder(detector_model=M + "scrfd_500m_bnkps.onnx",
|
||||
arcface_model=M + "LVFace-B_Glint360K.onnx",
|
||||
conf=0.5, nms=0.4, max_side=0)
|
||||
# Same models, looser suppression: keep the duplicates NMS would have removed.
|
||||
vote_eng = sae_embed.FaceEmbedder(detector_model=M + "scrfd_500m_bnkps.onnx",
|
||||
arcface_model=M + "LVFace-B_Glint360K.onnx",
|
||||
conf=0.3, nms=0.9, max_side=0)
|
||||
cal = sae_embed.gallery_calibration(ROOT + "gallery_lvface.h5")
|
||||
|
||||
|
||||
def iou(a, b):
|
||||
ax, ay, aw, ah = a; bx, by, bw, bh = b
|
||||
x0, y0 = max(ax, bx), max(ay, by)
|
||||
x1, y1 = min(ax + aw, bx + bw), min(ay + ah, by + bh)
|
||||
if x1 <= x0 or y1 <= y0:
|
||||
return 0.0
|
||||
i = (x1 - x0) * (y1 - y0)
|
||||
return i / (aw * ah + bw * bh - i)
|
||||
|
||||
|
||||
def vote(dets):
|
||||
"""Group overlapping detections, return (bbox, landmarks, conf, n_votes)."""
|
||||
items = sorted(dets, key=lambda d: -d.confidence)
|
||||
used, out = [False] * len(items), []
|
||||
for i, d in enumerate(items):
|
||||
if used[i]:
|
||||
continue
|
||||
grp = [d]
|
||||
used[i] = True
|
||||
for j in range(i + 1, len(items)):
|
||||
if not used[j] and iou(list(d.bbox), list(items[j].bbox)) >= GROUP_IOU:
|
||||
used[j] = True
|
||||
grp.append(items[j])
|
||||
w = np.array([g.confidence for g in grp], dtype=np.float32)
|
||||
w = w / w.sum()
|
||||
lms = np.stack([np.array(g.landmarks, dtype=np.float32).reshape(5, 2) for g in grp])
|
||||
bxs = np.stack([np.array(list(g.bbox), dtype=np.float32) for g in grp])
|
||||
out.append((( w[:, None] * bxs).sum(0), (w[:, None, None] * lms).sum(0),
|
||||
float(grp[0].confidence), len(grp)))
|
||||
return out
|
||||
|
||||
|
||||
def collect(clip):
|
||||
lab = {os.path.basename(p): os.path.basename(os.path.dirname(p))
|
||||
for p in glob.glob(f"labelling/{clip}/*/*.jpg")
|
||||
if os.path.basename(os.path.dirname(p)) not in ("discard", "unsorted")}
|
||||
man = {m["file"]: m for m in json.load(open(f"labelling/{clip}/manifest.json"))}
|
||||
rows, votes = [], []
|
||||
for frame in sorted({man[f]["frame"] for f in lab}):
|
||||
img = cv2.imread(f"frames/d{clip}_{frame}.png")
|
||||
base = base_eng.detect(img)
|
||||
voted = vote(vote_eng.detect(img))
|
||||
for fname, person in lab.items():
|
||||
m = man[fname]
|
||||
if m["frame"] != frame or m["idx"] >= len(base):
|
||||
continue
|
||||
d = base[m["idx"]]
|
||||
lm5 = np.array(d.landmarks, dtype=np.float32).reshape(5, 2)
|
||||
c_b = sae_embed.align_face(img, lm5)
|
||||
|
||||
# the voted group covering the same face
|
||||
best, best_v = None, 0.0
|
||||
for bbox, lms, conf, n in voted:
|
||||
v = iou(list(bbox), list(d.bbox))
|
||||
if v > best_v:
|
||||
best_v, best = v, (lms, n)
|
||||
c_v = None
|
||||
if best and best_v >= MATCH_IOU:
|
||||
c_v = sae_embed.align_face(img, best[0].astype(np.float32))
|
||||
votes.append(best[1])
|
||||
rec = {"person": person}
|
||||
rec["base"] = np.asarray(base_eng.embed_crop(c_b), np.float32) if c_b is not None else None
|
||||
rec["voted"] = np.asarray(base_eng.embed_crop(c_v), np.float32) if c_v is not None else None
|
||||
rows.append(rec)
|
||||
return rows, votes
|
||||
|
||||
|
||||
data, allv = {}, []
|
||||
for c in CLIPS:
|
||||
data[c], v = collect(c)
|
||||
allv += v
|
||||
print(f"[{c}] {len(data[c])} crops", file=sys.stderr)
|
||||
print(f"[voting] group size: median {np.median(allv):.0f}, "
|
||||
f"mean {np.mean(allv):.1f}, max {max(allv)} detections averaged per face",
|
||||
file=sys.stderr)
|
||||
|
||||
GAL, PRB = "5157344", "5157339"
|
||||
print(f"\ngallery {GAL} -> probe {PRB}, P>{PROB_THRESHOLD}\n")
|
||||
print(f"{'align':>8}{'person':>8}{'n_gal':>7}{'n_prb':>7}"
|
||||
f"{'within-clip':>13}{'cross-clip':>12}{'hit rate':>10}")
|
||||
summary = {}
|
||||
for key in ("base", "voted"):
|
||||
gal, prb = {}, {}
|
||||
for r in data[GAL]:
|
||||
if r[key] is not None:
|
||||
gal.setdefault(r["person"], []).append(r[key])
|
||||
for r in data[PRB]:
|
||||
if r[key] is not None:
|
||||
prb.setdefault(r["person"], []).append(r[key])
|
||||
gal = {p: np.stack(v) for p, v in gal.items()}
|
||||
prb = {p: np.stack(v) for p, v in prb.items()}
|
||||
hits = tot = 0
|
||||
for p in sorted(set(gal) & set(prb)):
|
||||
pp = prb[p] @ prb[p].T
|
||||
np.fill_diagonal(pp, -1)
|
||||
within = float(np.median(pp.max(axis=1))) if len(pp) > 1 else float("nan")
|
||||
cross = float(np.median((gal[p] @ prb[p].T).max(axis=0)))
|
||||
h = 0
|
||||
for e in prb[p]:
|
||||
bp, bn = 0.0, None
|
||||
for q in gal:
|
||||
v = cal.probability(float((gal[q] @ e).max()))
|
||||
if v > bp:
|
||||
bp, bn = v, q
|
||||
if bp > PROB_THRESHOLD and bn == p:
|
||||
h += 1
|
||||
hits += h; tot += len(prb[p])
|
||||
print(f"{key:>8}{p:>8}{len(gal[p]):>7}{len(prb[p]):>7}"
|
||||
f"{cal.probability(within):>6.3f}/{within:<6.3f}"
|
||||
f"{cal.probability(cross):>6.3f}/{cross:<5.3f}{100*h/len(prb[p]):>9.0f}%")
|
||||
summary[key] = (hits, tot)
|
||||
print(f"{key:>8}{'ALL':>8}{'':>14}{'':>25}{100*hits/max(tot,1):>9.0f}%\n")
|
||||
|
||||
hb, tb = summary["base"]; hv, tv = summary["voted"]
|
||||
print(f"voting vs baseline: {100*hv/max(tv,1) - 100*hb/max(tb,1):+.1f} points "
|
||||
f"of cross-clip TPI ({hb}/{tb} -> {hv}/{tv})")
|
||||
Reference in New Issue
Block a user