feat(review): GT-aware TP/FP/FN frame annotation + scene-detector examples

dump_error_frames.py drew every identified box green, so a false positive looked
like a true positive and a missed cast member was invisible. Make the annotation
ground-truth aware, matching what the per-second scorer classifies:
  - GREEN  true positive  — a name X-Ray also credits to this scene
  - RED    false positive — a name X-Ray does NOT credit here (the real error)
  - ORANGE unknown detection
  - BLUE   a text panel listing X-Ray cast present with no detected face (the
           structural false-negatives — no box exists to draw)

Add two representative annotated frames to the scene-detector page: a clean
green-TP second, and the face-vs-scene-cast case (a red FP lead + six off-camera
cast in blue) that makes the recall ceiling visual. Frames are generated by the
script from replay.py --raw-out output; the two committed examples are hand-picked
doc assets (bulk experiments/dump_review is regenerable and gitignored).
This commit is contained in:
2026-08-09 22:28:21 +02:00
parent edf19ab798
commit 0feafec7c9
5 changed files with 55 additions and 12 deletions
+1
View File
@@ -120,3 +120,4 @@ venv/
Thumbs.db Thumbs.db
.venv-rocm/ .venv-rocm/
!models/scene_boundary_xgb.json !models/scene_boundary_xgb.json
experiments/dump_review/
Binary file not shown.

After

Width:  |  Height:  |  Size: 47 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 47 KiB

+19
View File
@@ -133,6 +133,25 @@ The two headline cases:
Naive flood-fill barely beat doing nothing (64% vs 62%) and broke a film. With a Naive flood-fill barely beat doing nothing (64% vs 62%) and broke a film. With a
real boundary detector, flood-fill is decisively the right mode. real boundary detector, flood-fill is decisively the right mode.
### What the frames look like
`scripts/optimizer/dump_error_frames.py` pulls representative seconds and draws
each face box coloured against X-Ray's scene cast: **green** = true positive (a
name X-Ray also credits to this scene), **red** = false positive (a name X-Ray
does *not* credit here — the real error), **orange** = an unknown detection. Cast
X-Ray lists as present but for whom no face was detected — the structural
false-negatives a face pipeline can never box — are listed as a **blue** panel.
![A correctly identified second: green true-positive boxes](assets/images/scarface_tp_example.jpg)
Above: three faces named correctly (green). Below: the face-vs-scene-cast tension
made visual — the one visible face is confidently named (here it is a red
false-positive, a lead X-Ray did not credit to this exact scene), while six
credited cast members are off-camera with no face to detect (blue). This is why
recall against X-Ray has a structural ceiling, not a fixable bug.
![A false-positive box (red) with off-screen cast listed (blue)](assets/images/scarface_fn_fp_example.jpg)
## In the pipeline ## In the pipeline
Boundary detection is a **post-EOF step**, like flood-fill itself: the per-film Boundary detection is a **post-EOF step**, like flood-fill itself: the per-film
+33 -10
View File
@@ -121,23 +121,44 @@ def load_raw_annotations(raw_path: str):
return by_second return by_second
def draw_annotations(frame_path: Path, actors: list): def _name_key(name: str) -> str:
"""Normalised match key, mirroring identity.py's name: fallback."""
return "name:" + "".join(ch for ch in name.lower() if ch.isalnum() or ch == " ").strip()
def draw_annotations(frame_path: Path, actors: list, fp_keys=None, fn_names=None):
"""Draw GT-aware boxes: GREEN = true positive (named actor X-Ray also has in
this scene), RED = false positive (named actor NOT in the scene → the real
error), ORANGE = unknown detection. FN cast (present per X-Ray but no face
detected — so no box to draw) is listed as a BLUE text panel bottom-left."""
img = cv2.imread(str(frame_path)) img = cv2.imread(str(frame_path))
if img is None: if img is None:
return return
fp_keys = fp_keys or set()
GREEN, RED, ORANGE, BLUE = (60,200,0), (0,0,230), (220,100,0), (230,150,0)
for a in actors: for a in actors:
known = a.get("actor_idx", -1) >= 0 known = a.get("actor_idx", -1) >= 0
colour = (60, 200, 0) if known else (220, 100, 0) # BGR: green / orange if known:
x, y, w, h = a["bbox"] colour = RED if _name_key(a["name"]) in fp_keys else GREEN
x, y, w, h = int(x), int(y), int(w), int(h) label = f"{a['name']} {a['similarity']*100:.0f}%"
else:
colour = ORANGE; label = f"unknown {a['similarity']*100:.0f}%"
x, y, w, h = (int(v) for v in a["bbox"])
cv2.rectangle(img, (x, y), (x + w, y + h), colour, 2) cv2.rectangle(img, (x, y), (x + w, y + h), colour, 2)
(tw, th), _ = cv2.getTextSize(label, cv2.FONT_HERSHEY_SIMPLEX, 0.5, 1)
label = f"{a['name']} {a['similarity']*100:.0f}%" if known else f"unknown {a['similarity']*100:.0f}%" cv2.rectangle(img, (x, max(0, y-th-4)), (x+tw+4, y), colour, cv2.FILLED)
(tw, th), baseline = cv2.getTextSize(label, cv2.FONT_HERSHEY_SIMPLEX, 0.5, 1)
strip_y0 = max(0, y - th - 4)
cv2.rectangle(img, (x, strip_y0), (x + tw + 4, y), colour, cv2.FILLED)
cv2.putText(img, label, (x+2, y-2), cv2.FONT_HERSHEY_SIMPLEX, 0.5, cv2.putText(img, label, (x+2, y-2), cv2.FONT_HERSHEY_SIMPLEX, 0.5,
(255,255,255), 1, cv2.LINE_AA) (255,255,255), 1, cv2.LINE_AA)
# FN: X-Ray cast present with no detected face — no box exists, so list them.
fn = [n for n in (fn_names or []) if n]
if fn:
H = img.shape[0]
cv2.putText(img, "off-screen / missed (X-Ray cast, no face):",
(8, H-8-18*len(fn[:6])), cv2.FONT_HERSHEY_SIMPLEX, 0.45, BLUE, 1, cv2.LINE_AA)
for i, n in enumerate(fn[:6]):
disp = n.replace("name:", "").title()
cv2.putText(img, f" {disp}", (8, H-8-18*(len(fn[:6])-1-i)),
cv2.FONT_HERSHEY_SIMPLEX, 0.5, BLUE, 1, cv2.LINE_AA)
cv2.imwrite(str(frame_path), img) cv2.imwrite(str(frame_path), img)
@@ -183,7 +204,9 @@ def main():
extract_frame(args.movie, r["t"], out_path) extract_frame(args.movie, r["t"], out_path)
ok = True ok = True
if raw_by_second is not None: if raw_by_second is not None:
draw_annotations(out_path, raw_by_second.get(r["t"], [])) fp_keys = {_name_key(n) for n in r["fp"]}
draw_annotations(out_path, raw_by_second.get(r["t"], []),
fp_keys=fp_keys, fn_names=r["fn"])
except subprocess.CalledProcessError as e: except subprocess.CalledProcessError as e:
ok = False ok = False
print(f"[dump_error_frames] ffmpeg failed at t={r['t']}: {e}", file=sys.stderr) print(f"[dump_error_frames] ffmpeg failed at t={r['t']}: {e}", file=sys.stderr)