feat(review): GT-aware TP/FP/FN frame annotation + scene-detector examples
dump_error_frames.py drew every identified box green, so a false positive looked
like a true positive and a missed cast member was invisible. Make the annotation
ground-truth aware, matching what the per-second scorer classifies:
- GREEN true positive — a name X-Ray also credits to this scene
- RED false positive — a name X-Ray does NOT credit here (the real error)
- ORANGE unknown detection
- BLUE a text panel listing X-Ray cast present with no detected face (the
structural false-negatives — no box exists to draw)
Add two representative annotated frames to the scene-detector page: a clean
green-TP second, and the face-vs-scene-cast case (a red FP lead + six off-camera
cast in blue) that makes the recall ceiling visual. Frames are generated by the
script from replay.py --raw-out output; the two committed examples are hand-picked
doc assets (bulk experiments/dump_review is regenerable and gitignored).
This commit is contained in:
@@ -120,3 +120,4 @@ venv/
|
|||||||
Thumbs.db
|
Thumbs.db
|
||||||
.venv-rocm/
|
.venv-rocm/
|
||||||
!models/scene_boundary_xgb.json
|
!models/scene_boundary_xgb.json
|
||||||
|
experiments/dump_review/
|
||||||
|
|||||||
Binary file not shown.
|
After Width: | Height: | Size: 47 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 47 KiB |
@@ -133,6 +133,25 @@ The two headline cases:
|
|||||||
Naive flood-fill barely beat doing nothing (64% vs 62%) and broke a film. With a
|
Naive flood-fill barely beat doing nothing (64% vs 62%) and broke a film. With a
|
||||||
real boundary detector, flood-fill is decisively the right mode.
|
real boundary detector, flood-fill is decisively the right mode.
|
||||||
|
|
||||||
|
### What the frames look like
|
||||||
|
|
||||||
|
`scripts/optimizer/dump_error_frames.py` pulls representative seconds and draws
|
||||||
|
each face box coloured against X-Ray's scene cast: **green** = true positive (a
|
||||||
|
name X-Ray also credits to this scene), **red** = false positive (a name X-Ray
|
||||||
|
does *not* credit here — the real error), **orange** = an unknown detection. Cast
|
||||||
|
X-Ray lists as present but for whom no face was detected — the structural
|
||||||
|
false-negatives a face pipeline can never box — are listed as a **blue** panel.
|
||||||
|
|
||||||
|

|
||||||
|
|
||||||
|
Above: three faces named correctly (green). Below: the face-vs-scene-cast tension
|
||||||
|
made visual — the one visible face is confidently named (here it is a red
|
||||||
|
false-positive, a lead X-Ray did not credit to this exact scene), while six
|
||||||
|
credited cast members are off-camera with no face to detect (blue). This is why
|
||||||
|
recall against X-Ray has a structural ceiling, not a fixable bug.
|
||||||
|
|
||||||
|

|
||||||
|
|
||||||
## In the pipeline
|
## In the pipeline
|
||||||
|
|
||||||
Boundary detection is a **post-EOF step**, like flood-fill itself: the per-film
|
Boundary detection is a **post-EOF step**, like flood-fill itself: the per-film
|
||||||
|
|||||||
@@ -121,23 +121,44 @@ def load_raw_annotations(raw_path: str):
|
|||||||
return by_second
|
return by_second
|
||||||
|
|
||||||
|
|
||||||
def draw_annotations(frame_path: Path, actors: list):
|
def _name_key(name: str) -> str:
|
||||||
|
"""Normalised match key, mirroring identity.py's name: fallback."""
|
||||||
|
return "name:" + "".join(ch for ch in name.lower() if ch.isalnum() or ch == " ").strip()
|
||||||
|
|
||||||
|
|
||||||
|
def draw_annotations(frame_path: Path, actors: list, fp_keys=None, fn_names=None):
|
||||||
|
"""Draw GT-aware boxes: GREEN = true positive (named actor X-Ray also has in
|
||||||
|
this scene), RED = false positive (named actor NOT in the scene → the real
|
||||||
|
error), ORANGE = unknown detection. FN cast (present per X-Ray but no face
|
||||||
|
detected — so no box to draw) is listed as a BLUE text panel bottom-left."""
|
||||||
img = cv2.imread(str(frame_path))
|
img = cv2.imread(str(frame_path))
|
||||||
if img is None:
|
if img is None:
|
||||||
return
|
return
|
||||||
|
fp_keys = fp_keys or set()
|
||||||
|
GREEN, RED, ORANGE, BLUE = (60,200,0), (0,0,230), (220,100,0), (230,150,0)
|
||||||
for a in actors:
|
for a in actors:
|
||||||
known = a.get("actor_idx", -1) >= 0
|
known = a.get("actor_idx", -1) >= 0
|
||||||
colour = (60, 200, 0) if known else (220, 100, 0) # BGR: green / orange
|
if known:
|
||||||
x, y, w, h = a["bbox"]
|
colour = RED if _name_key(a["name"]) in fp_keys else GREEN
|
||||||
x, y, w, h = int(x), int(y), int(w), int(h)
|
label = f"{a['name']} {a['similarity']*100:.0f}%"
|
||||||
|
else:
|
||||||
|
colour = ORANGE; label = f"unknown {a['similarity']*100:.0f}%"
|
||||||
|
x, y, w, h = (int(v) for v in a["bbox"])
|
||||||
cv2.rectangle(img, (x, y), (x + w, y + h), colour, 2)
|
cv2.rectangle(img, (x, y), (x + w, y + h), colour, 2)
|
||||||
|
(tw, th), _ = cv2.getTextSize(label, cv2.FONT_HERSHEY_SIMPLEX, 0.5, 1)
|
||||||
label = f"{a['name']} {a['similarity']*100:.0f}%" if known else f"unknown {a['similarity']*100:.0f}%"
|
cv2.rectangle(img, (x, max(0, y-th-4)), (x+tw+4, y), colour, cv2.FILLED)
|
||||||
(tw, th), baseline = cv2.getTextSize(label, cv2.FONT_HERSHEY_SIMPLEX, 0.5, 1)
|
|
||||||
strip_y0 = max(0, y - th - 4)
|
|
||||||
cv2.rectangle(img, (x, strip_y0), (x + tw + 4, y), colour, cv2.FILLED)
|
|
||||||
cv2.putText(img, label, (x+2, y-2), cv2.FONT_HERSHEY_SIMPLEX, 0.5,
|
cv2.putText(img, label, (x+2, y-2), cv2.FONT_HERSHEY_SIMPLEX, 0.5,
|
||||||
(255,255,255), 1, cv2.LINE_AA)
|
(255,255,255), 1, cv2.LINE_AA)
|
||||||
|
# FN: X-Ray cast present with no detected face — no box exists, so list them.
|
||||||
|
fn = [n for n in (fn_names or []) if n]
|
||||||
|
if fn:
|
||||||
|
H = img.shape[0]
|
||||||
|
cv2.putText(img, "off-screen / missed (X-Ray cast, no face):",
|
||||||
|
(8, H-8-18*len(fn[:6])), cv2.FONT_HERSHEY_SIMPLEX, 0.45, BLUE, 1, cv2.LINE_AA)
|
||||||
|
for i, n in enumerate(fn[:6]):
|
||||||
|
disp = n.replace("name:", "").title()
|
||||||
|
cv2.putText(img, f" {disp}", (8, H-8-18*(len(fn[:6])-1-i)),
|
||||||
|
cv2.FONT_HERSHEY_SIMPLEX, 0.5, BLUE, 1, cv2.LINE_AA)
|
||||||
cv2.imwrite(str(frame_path), img)
|
cv2.imwrite(str(frame_path), img)
|
||||||
|
|
||||||
|
|
||||||
@@ -183,7 +204,9 @@ def main():
|
|||||||
extract_frame(args.movie, r["t"], out_path)
|
extract_frame(args.movie, r["t"], out_path)
|
||||||
ok = True
|
ok = True
|
||||||
if raw_by_second is not None:
|
if raw_by_second is not None:
|
||||||
draw_annotations(out_path, raw_by_second.get(r["t"], []))
|
fp_keys = {_name_key(n) for n in r["fp"]}
|
||||||
|
draw_annotations(out_path, raw_by_second.get(r["t"], []),
|
||||||
|
fp_keys=fp_keys, fn_names=r["fn"])
|
||||||
except subprocess.CalledProcessError as e:
|
except subprocess.CalledProcessError as e:
|
||||||
ok = False
|
ok = False
|
||||||
print(f"[dump_error_frames] ffmpeg failed at t={r['t']}: {e}", file=sys.stderr)
|
print(f"[dump_error_frames] ffmpeg failed at t={r['t']}: {e}", file=sys.stderr)
|
||||||
|
|||||||
Reference in New Issue
Block a user