diff --git a/.gitignore b/.gitignore index 6546aea..914c24b 100644 --- a/.gitignore +++ b/.gitignore @@ -120,3 +120,4 @@ venv/ Thumbs.db .venv-rocm/ !models/scene_boundary_xgb.json +experiments/dump_review/ diff --git a/docs/assets/images/scarface_fn_fp_example.jpg b/docs/assets/images/scarface_fn_fp_example.jpg new file mode 100644 index 0000000..c7eff72 Binary files /dev/null and b/docs/assets/images/scarface_fn_fp_example.jpg differ diff --git a/docs/assets/images/scarface_tp_example.jpg b/docs/assets/images/scarface_tp_example.jpg new file mode 100644 index 0000000..fe1d44c Binary files /dev/null and b/docs/assets/images/scarface_tp_example.jpg differ diff --git a/docs/scene-boundary-detector.md b/docs/scene-boundary-detector.md index 815d5c0..6d3dbb0 100644 --- a/docs/scene-boundary-detector.md +++ b/docs/scene-boundary-detector.md @@ -133,6 +133,25 @@ The two headline cases: Naive flood-fill barely beat doing nothing (64% vs 62%) and broke a film. With a real boundary detector, flood-fill is decisively the right mode. +### What the frames look like + +`scripts/optimizer/dump_error_frames.py` pulls representative seconds and draws +each face box coloured against X-Ray's scene cast: **green** = true positive (a +name X-Ray also credits to this scene), **red** = false positive (a name X-Ray +does *not* credit here — the real error), **orange** = an unknown detection. Cast +X-Ray lists as present but for whom no face was detected — the structural +false-negatives a face pipeline can never box — are listed as a **blue** panel. + +![A correctly identified second: green true-positive boxes](assets/images/scarface_tp_example.jpg) + +Above: three faces named correctly (green). Below: the face-vs-scene-cast tension +made visual — the one visible face is confidently named (here it is a red +false-positive, a lead X-Ray did not credit to this exact scene), while six +credited cast members are off-camera with no face to detect (blue). This is why +recall against X-Ray has a structural ceiling, not a fixable bug. + +![A false-positive box (red) with off-screen cast listed (blue)](assets/images/scarface_fn_fp_example.jpg) + ## In the pipeline Boundary detection is a **post-EOF step**, like flood-fill itself: the per-film diff --git a/scripts/optimizer/dump_error_frames.py b/scripts/optimizer/dump_error_frames.py index 2892345..fb70470 100644 --- a/scripts/optimizer/dump_error_frames.py +++ b/scripts/optimizer/dump_error_frames.py @@ -121,23 +121,44 @@ def load_raw_annotations(raw_path: str): return by_second -def draw_annotations(frame_path: Path, actors: list): +def _name_key(name: str) -> str: + """Normalised match key, mirroring identity.py's name: fallback.""" + return "name:" + "".join(ch for ch in name.lower() if ch.isalnum() or ch == " ").strip() + + +def draw_annotations(frame_path: Path, actors: list, fp_keys=None, fn_names=None): + """Draw GT-aware boxes: GREEN = true positive (named actor X-Ray also has in + this scene), RED = false positive (named actor NOT in the scene → the real + error), ORANGE = unknown detection. FN cast (present per X-Ray but no face + detected — so no box to draw) is listed as a BLUE text panel bottom-left.""" img = cv2.imread(str(frame_path)) if img is None: return + fp_keys = fp_keys or set() + GREEN, RED, ORANGE, BLUE = (60,200,0), (0,0,230), (220,100,0), (230,150,0) for a in actors: known = a.get("actor_idx", -1) >= 0 - colour = (60, 200, 0) if known else (220, 100, 0) # BGR: green / orange - x, y, w, h = a["bbox"] - x, y, w, h = int(x), int(y), int(w), int(h) + if known: + colour = RED if _name_key(a["name"]) in fp_keys else GREEN + label = f"{a['name']} {a['similarity']*100:.0f}%" + else: + colour = ORANGE; label = f"unknown {a['similarity']*100:.0f}%" + x, y, w, h = (int(v) for v in a["bbox"]) cv2.rectangle(img, (x, y), (x + w, y + h), colour, 2) - - label = f"{a['name']} {a['similarity']*100:.0f}%" if known else f"unknown {a['similarity']*100:.0f}%" - (tw, th), baseline = cv2.getTextSize(label, cv2.FONT_HERSHEY_SIMPLEX, 0.5, 1) - strip_y0 = max(0, y - th - 4) - cv2.rectangle(img, (x, strip_y0), (x + tw + 4, y), colour, cv2.FILLED) - cv2.putText(img, label, (x + 2, y - 2), cv2.FONT_HERSHEY_SIMPLEX, 0.5, - (255, 255, 255), 1, cv2.LINE_AA) + (tw, th), _ = cv2.getTextSize(label, cv2.FONT_HERSHEY_SIMPLEX, 0.5, 1) + cv2.rectangle(img, (x, max(0, y-th-4)), (x+tw+4, y), colour, cv2.FILLED) + cv2.putText(img, label, (x+2, y-2), cv2.FONT_HERSHEY_SIMPLEX, 0.5, + (255,255,255), 1, cv2.LINE_AA) + # FN: X-Ray cast present with no detected face — no box exists, so list them. + fn = [n for n in (fn_names or []) if n] + if fn: + H = img.shape[0] + cv2.putText(img, "off-screen / missed (X-Ray cast, no face):", + (8, H-8-18*len(fn[:6])), cv2.FONT_HERSHEY_SIMPLEX, 0.45, BLUE, 1, cv2.LINE_AA) + for i, n in enumerate(fn[:6]): + disp = n.replace("name:", "").title() + cv2.putText(img, f" {disp}", (8, H-8-18*(len(fn[:6])-1-i)), + cv2.FONT_HERSHEY_SIMPLEX, 0.5, BLUE, 1, cv2.LINE_AA) cv2.imwrite(str(frame_path), img) @@ -183,7 +204,9 @@ def main(): extract_frame(args.movie, r["t"], out_path) ok = True if raw_by_second is not None: - draw_annotations(out_path, raw_by_second.get(r["t"], [])) + fp_keys = {_name_key(n) for n in r["fp"]} + draw_annotations(out_path, raw_by_second.get(r["t"], []), + fp_keys=fp_keys, fn_names=r["fn"]) except subprocess.CalledProcessError as e: ok = False print(f"[dump_error_frames] ffmpeg failed at t={r['t']}: {e}", file=sys.stderr)