study(VR-013): cross-source identification probe over input resolution
Gallery from one recording, probes from another, sweeping the probe's input resolution end to end. VR-005 asked the same question over gallery mugshots but degraded an already-aligned 112x112 crop with alignment held perfect, so it isolates the embedder. Here the whole frame is downscaled before the detector, so detection and landmark regression degrade with it — which is most of the difference. Corpus is two 4096x2160 clips of one shoot, four people, hand-sorted. Ground truth is sorted by hand and gated by verify_labels.py; labels carried down the scales geometrically by box position, never by embedding similarity, which would keep only the faces the embedder already gets right and drop the ones the sweep exists to find. Findings, all scored through the production gallery sigmoid at prob_threshold 0.754 — never a raw cosine: - Holding 90% of the plateau needs ~50 px end to end, against VR-005's ~22 px. min_face_px at 40 looks right; 32 would admit faces in the falling region. - FPI is 0.0% at every scale. Resolution loss goes entirely to TBI. - The ceiling is cross-view, not resolution: everyone matches themselves within a recording (0.55-0.85) and collapses across two (0.14-0.45, threshold 0.335). Only the subject with frontal *gallery* references identified reliably, whatever their probe pose — so the lever is gallery pose coverage, not a better landmark source. - Averaging SCRFD's overlapping detections instead of discarding them at NMS lifts cross-recording TPI 41% -> 49%, for one forward pass and no extra model. Four identities and one shoot, so the shape is the result and the absolute rates are not. Both clips contain all four people, so there is no out-of-gallery class and the 10x-weighted out-of-cast misID is untested here. Clips, frames, hand-sorted crops and results are gitignored and belong in the artifact registry — the sorting is human ground truth and expensive to redo. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com> TRACES: VR-013 | AR-002, AR-005, AR-024
This commit is contained in:
Claude Opus 5
@@ -0,0 +1,235 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Build labelling/review.html — a local page for correcting the labels.
|
||||
|
||||
One row per crop, ordered most-suspicious first:
|
||||
|
||||
left the person it is currently filed under (medoid of that person's
|
||||
hand-sorted crops, so the reference is one you trust)
|
||||
centre the crop under review — context with the detection boxed, and
|
||||
beneath it the 112x112 the embedder actually receives
|
||||
right the person it matches better, if any, with both probabilities
|
||||
|
||||
Pick a destination per row, then Export to download corrections.json and apply
|
||||
it with apply_corrections.py. Nothing is moved by this script.
|
||||
|
||||
Self-contained: images are inlined as data URIs and the page is opened from
|
||||
disk, so no server runs and no face crop leaves the machine.
|
||||
|
||||
Ordering is by P(other) - P(self), both from the global gallery sigmoid, so
|
||||
rows where the evidence disagrees with the label float to the top and the
|
||||
agreement cases sink. It is a review order, not a verdict — you are the
|
||||
arbiter, which is the whole point of labelling by hand.
|
||||
"""
|
||||
import sys, glob, json, os, base64
|
||||
import numpy as np
|
||||
import cv2
|
||||
|
||||
sys.path.insert(0, "/home/dtourolle/Development/Jray-project/scene-actor-extraction/build-ort")
|
||||
import sae_embed
|
||||
|
||||
ROOT = "/home/dtourolle/Development/Jray-project/scene-actor-extraction/"
|
||||
M = ROOT + "models/"
|
||||
EMBEDDER = M + "LVFace-B_Glint360K.onnx"
|
||||
GALLERY = ROOT + "gallery_lvface.h5"
|
||||
REF_CLIP = "5157344" # the clip sorted by hand — reference faces come from here
|
||||
CLIPS = ["5157344", "5157339"]
|
||||
|
||||
eng = sae_embed.FaceEmbedder(detector_model=M + "scrfd_500m_bnkps.onnx",
|
||||
arcface_model=EMBEDDER, conf=0.5, nms=0.4, max_side=0)
|
||||
cal = sae_embed.gallery_calibration(GALLERY)
|
||||
|
||||
|
||||
def b64(img, size, q=72):
|
||||
img = cv2.resize(img, (size, size))
|
||||
ok, buf = cv2.imencode(".jpg", img, [cv2.IMWRITE_JPEG_QUALITY, q])
|
||||
return "data:image/jpeg;base64," + base64.b64encode(buf).decode() if ok else ""
|
||||
|
||||
|
||||
rows = []
|
||||
for clip in CLIPS:
|
||||
man = {m["file"]: m for m in json.load(open(f"labelling/{clip}/manifest.json"))}
|
||||
placed = {os.path.basename(p): (os.path.basename(os.path.dirname(p)), p)
|
||||
for p in glob.glob(f"labelling/{clip}/*/*.jpg")}
|
||||
by_frame = {}
|
||||
for fname, (person, path) in placed.items():
|
||||
if fname in man and person != "unsorted":
|
||||
by_frame.setdefault(man[fname]["frame"], []).append((fname, person, path))
|
||||
for frame, items in sorted(by_frame.items()):
|
||||
img = cv2.imread(f"frames/d{clip}_{frame}.png")
|
||||
if img is None:
|
||||
continue
|
||||
dets = eng.detect(img)
|
||||
for fname, person, path in items:
|
||||
i = man[fname]["idx"]
|
||||
if i >= len(dets):
|
||||
continue
|
||||
lm = np.array(dets[i].landmarks, dtype=np.float32).reshape(5, 2)
|
||||
crop = sae_embed.align_face(img, lm)
|
||||
if crop is None:
|
||||
continue
|
||||
rows.append({"clip": clip, "person": person, "file": fname, "path": path,
|
||||
"px": man[fname]["px"], "aligned": np.asarray(crop),
|
||||
"emb": np.asarray(eng.embed_crop(crop), dtype=np.float32)})
|
||||
|
||||
people = sorted({r["person"] for r in rows})
|
||||
E = np.stack([r["emb"] for r in rows])
|
||||
lab = np.array([people.index(r["person"]) for r in rows])
|
||||
S = E @ E.T
|
||||
np.fill_diagonal(S, -1.0)
|
||||
|
||||
# reference face per person: medoid of their REF_CLIP crops
|
||||
ref_img = {}
|
||||
for k, p in enumerate(people):
|
||||
idx = [i for i in np.where(lab == k)[0] if rows[i]["clip"] == REF_CLIP]
|
||||
if not idx:
|
||||
idx = list(np.where(lab == k)[0])
|
||||
if not idx:
|
||||
continue
|
||||
sub = S[np.ix_(idx, idx)].copy()
|
||||
medoid = idx[int(np.argmax(sub.mean(axis=1)))]
|
||||
ref_img[p] = b64(rows[medoid]["aligned"], 112)
|
||||
|
||||
items = []
|
||||
for i, r in enumerate(rows):
|
||||
k = lab[i]
|
||||
same = [j for j in np.where(lab == k)[0] if j != i]
|
||||
p_self = cal.probability(float(S[i, same].max())) if same else 0.0
|
||||
best_other, p_other = None, 0.0
|
||||
for k2, p2 in enumerate(people):
|
||||
if k2 == k:
|
||||
continue
|
||||
other = np.where(lab == k2)[0]
|
||||
if not len(other):
|
||||
continue
|
||||
pv = cal.probability(float(S[i, other].max()))
|
||||
if pv > p_other:
|
||||
p_other, best_other = pv, p2
|
||||
ctx = cv2.imread(r["path"])
|
||||
items.append({
|
||||
"file": r["file"], "clip": r["clip"], "person": r["person"],
|
||||
"px": int(r["px"]), "p_self": round(p_self, 3), "p_other": round(p_other, 3),
|
||||
"other": best_other, "delta": round(p_other - p_self, 3),
|
||||
"ctx": b64(ctx, 150) if ctx is not None else "",
|
||||
"ali": b64(r["aligned"], 112),
|
||||
})
|
||||
items.sort(key=lambda x: -x["delta"])
|
||||
|
||||
payload = json.dumps({"people": people, "refs": ref_img, "items": items})
|
||||
|
||||
HTML = """<meta charset="utf-8"><title>JRay — label review</title>
|
||||
<style>
|
||||
:root{color-scheme:dark;--bg:#14161a;--fg:#e6e8ea;--mut:#8b929c;--line:#262b33;--warn:#e0654a;--ok:#4a9d6a}
|
||||
body{margin:0;background:var(--bg);color:var(--fg);font:14px/1.5 system-ui,sans-serif}
|
||||
header{position:sticky;top:0;background:#181b20;border-bottom:1px solid var(--line);
|
||||
padding:12px 18px;display:flex;gap:18px;align-items:center;flex-wrap:wrap;z-index:5}
|
||||
h1{font-size:15px;margin:0;font-weight:600}
|
||||
.stat{color:var(--mut);font-size:13px}
|
||||
button{background:#232830;color:var(--fg);border:1px solid var(--line);border-radius:6px;
|
||||
padding:7px 13px;cursor:pointer;font:inherit}
|
||||
button:hover{background:#2c323c}
|
||||
button.go{background:#2f5d43;border-color:#3c7555}
|
||||
.row{display:grid;grid-template-columns:150px 1fr 190px;gap:20px;align-items:center;
|
||||
padding:14px 18px;border-bottom:1px solid var(--line)}
|
||||
.row.flag{background:#1e1719}
|
||||
.row.done{opacity:.4}
|
||||
.cell{display:flex;gap:10px;align-items:center}
|
||||
img{border-radius:5px;display:block;background:#000}
|
||||
.lab{font-weight:600;font-size:15px}
|
||||
.mut{color:var(--mut);font-size:12px}
|
||||
.p{font-variant-numeric:tabular-nums}
|
||||
.hi{color:var(--warn);font-weight:600}
|
||||
.choices{display:flex;flex-wrap:wrap;gap:6px}
|
||||
.choices button{padding:5px 10px;font-size:13px}
|
||||
.choices button.sel{background:#2f5d43;border-color:#3c7555}
|
||||
.legend{padding:10px 18px;color:var(--mut);font-size:12px;border-bottom:1px solid var(--line)}
|
||||
</style>
|
||||
<header>
|
||||
<h1>Label review</h1>
|
||||
<span class="stat" id="stat"></span>
|
||||
<button id="exp" class="go">Export corrections.json</button>
|
||||
<button id="onlyflag">Show only disagreements</button>
|
||||
</header>
|
||||
<div class="legend">Left: the person this crop is filed under. Centre: the crop (context with the
|
||||
detection boxed, and the 112×112 the embedder actually sees). Right: the person it matches
|
||||
better, if any. Ordered by P(other) − P(self) — disagreements first.</div>
|
||||
<div id="list"></div>
|
||||
<script>
|
||||
const D = __PAYLOAD__;
|
||||
const choice = {};
|
||||
const list = document.getElementById('list');
|
||||
|
||||
function render(){
|
||||
list.innerHTML = '';
|
||||
const flagOnly = document.body.dataset.flag === '1';
|
||||
for (const it of D.items){
|
||||
if (flagOnly && it.delta <= 0) continue;
|
||||
const row = document.createElement('div');
|
||||
row.className = 'row' + (it.delta > 0 ? ' flag' : '') + (choice[it.file] ? ' done' : '');
|
||||
|
||||
const left = document.createElement('div');
|
||||
left.className = 'cell';
|
||||
left.innerHTML = `<img src="${D.refs[it.person]||''}" width="72" height="72">
|
||||
<div><div class="lab">${it.person}</div>
|
||||
<div class="mut p">P(self) ${it.p_self.toFixed(3)}</div></div>`;
|
||||
|
||||
const mid = document.createElement('div');
|
||||
mid.className = 'cell';
|
||||
mid.innerHTML = `<img src="${it.ctx}" width="120" height="120">
|
||||
<img src="${it.ali}" width="90" height="90">
|
||||
<div><div class="mut">${it.clip} · ${it.px}px</div>
|
||||
<div class="mut">${it.file}</div></div>`;
|
||||
|
||||
const right = document.createElement('div');
|
||||
const worse = it.delta > 0;
|
||||
right.innerHTML = it.other
|
||||
? `<div class="cell"><img src="${D.refs[it.other]||''}" width="56" height="56">
|
||||
<div><div class="lab ${worse?'hi':''}">${it.other}</div>
|
||||
<div class="mut p ${worse?'hi':''}">P ${it.p_other.toFixed(3)}</div></div></div>`
|
||||
: '<div class="mut">—</div>';
|
||||
|
||||
const ch = document.createElement('div');
|
||||
ch.className = 'choices';
|
||||
for (const p of D.people.concat(['discard'])){
|
||||
const b = document.createElement('button');
|
||||
b.textContent = p === it.person ? p + ' (keep)' : p;
|
||||
if (choice[it.file] === p || (!choice[it.file] && p === it.person)) b.classList.add('sel');
|
||||
b.onclick = () => { choice[it.file] = p; render(); };
|
||||
ch.appendChild(b);
|
||||
}
|
||||
right.appendChild(ch);
|
||||
|
||||
row.append(left, mid, right);
|
||||
list.appendChild(row);
|
||||
}
|
||||
const changed = Object.entries(choice).filter(([f,p]) =>
|
||||
p !== (D.items.find(i=>i.file===f)||{}).person).length;
|
||||
document.getElementById('stat').textContent =
|
||||
`${D.items.length} crops · ${D.items.filter(i=>i.delta>0).length} disagreements · ${changed} changes staged`;
|
||||
}
|
||||
|
||||
document.getElementById('onlyflag').onclick = () => {
|
||||
document.body.dataset.flag = document.body.dataset.flag === '1' ? '0' : '1';
|
||||
render();
|
||||
};
|
||||
document.getElementById('exp').onclick = () => {
|
||||
const out = {};
|
||||
for (const it of D.items){
|
||||
const p = choice[it.file] || it.person;
|
||||
if (p !== it.person) out[it.file] = {from: it.person, to: p, clip: it.clip};
|
||||
}
|
||||
const blob = new Blob([JSON.stringify(out, null, 1)], {type:'application/json'});
|
||||
const a = document.createElement('a');
|
||||
a.href = URL.createObjectURL(blob); a.download = 'corrections.json'; a.click();
|
||||
};
|
||||
render();
|
||||
</script>
|
||||
"""
|
||||
|
||||
os.makedirs("labelling", exist_ok=True)
|
||||
out = "labelling/review.html"
|
||||
with open(out, "w") as f:
|
||||
f.write(HTML.replace("__PAYLOAD__", payload))
|
||||
size = os.path.getsize(out) / 1e6
|
||||
flagged = sum(1 for i in items if i["delta"] > 0)
|
||||
print(f"{out} {size:.1f} MB {len(items)} crops, {flagged} disagreements", file=sys.stderr)
|
||||
print(f"open file://{os.path.abspath(out)}", file=sys.stderr)
|
||||
Reference in New Issue
Block a user