dtourolle
0e35dac951
A boosted-tree scene-boundary detector that replaces the grayscale histogram-correlation cut detector as the flood-fill boundary source, and substantially improves actor-presence accuracy. Downstream result (per-second X-Ray presence F1, macro over 9 films): track_extent 62.3% | flood + histogram cuts 64.0% | flood + this 76.9% +12.9pp, and it wins on every film — notably fixing the histogram flood's Scarface collapse (61 -> 41 -> 71) and lifting Downton 41 -> 84. Design (each choice measured — see the memory / report): - XGBoost REGRESSOR on a ±3s window of DELTA features (symmetric RGB-hist and audio-PSD deltas at k=1,2,4,8s + ramp bank + time-since-last-peak debounce). Raw histograms dilute; deltas separate boundaries ~4-5x. - SOFT Gaussian proximity target (sigma=10s) so near-misses train as near-correct, not hard negatives; regression -> smooth score -> NMS peaks. - KNEE per-film threshold: self-calibrates the boundary count to ~the true scene count, no global rate. Evaluated at ±20s (X-Ray scenes ~170s). - Trained on all 9 films (Cafe/Scarface low-contrast grades must be seen). Honest held-out ~41% boundary-F1 @±20s vs ~27% grayscale. Scripts: train_xgb_boundary.py (shipped detector), extract_audio_features.py (per-second log-PSD), downstream_presence.py (the A/B above), density_floor.py (fallback for detection-starved films), plus the LSTM/DE explorations kept for provenance. Model: models/scene_boundary_xgb.json. Not yet wired into the live C++ pipeline — boundaries are a post-EOF step in the sink (like flood-fill itself); libxgboost C++ integration is the next step.
109 lines
4.5 KiB
Python
109 lines
4.5 KiB
Python
#!/usr/bin/env python3
|
|
"""
|
|
downstream_presence.py — does the XGBoost scene detector actually improve ACTOR
|
|
PRESENCE accuracy? Boundary-F1 is only a proxy; this is the number that decides
|
|
whether the detector ships.
|
|
|
|
For each film, compares presence (per-second X-Ray F1) under three regimes:
|
|
A. track_extent — no flood-fill (claim = [first_seen, last_seen])
|
|
B. flood + histogram cuts — current shipped flood (snaps to is_cut)
|
|
C. flood + XGBoost bounds — inject the detector's boundaries into
|
|
is_scene_boundary (flood prefers it over is_cut)
|
|
|
|
Injection: write a copy of each dump with frames/is_scene_boundary set from the
|
|
XGBoost knee boundaries, then replay --presence-mode flood against that copy.
|
|
Uses the shipped model (all-9 fit). Scored with second_score at the 10-knob
|
|
optimum config.
|
|
"""
|
|
from __future__ import annotations
|
|
import sys, json, shutil, subprocess, tempfile, os
|
|
from pathlib import Path
|
|
import numpy as np
|
|
import h5py
|
|
|
|
sys.path.insert(0, "scripts/scene_detector")
|
|
sys.path.insert(0, "scripts/optimizer")
|
|
sys.path.insert(0, "scripts/validation")
|
|
import train_xgb_boundary as XB
|
|
from second_score import score_seconds
|
|
from sample_eval import load_gallery_keys
|
|
import xgboost as xgb
|
|
|
|
GAL = "experiments/galleries/gallery_LVFace-B_Glint360K.h5"
|
|
MODEL = "experiments/results/scene_boundary/xgb_boundary_shipped.json"
|
|
# 10-knob presence optimum (shipped config)
|
|
CFG = ["--prob-threshold", "0.485", "--ownership-logodds", "1.72",
|
|
"--track-extinction-sec", "31", "--track-alpha", "0.435",
|
|
"--evidence-rho-max", "0.204", "--evidence-admit-below", "0.784",
|
|
"--match-prior", "0.433", "--expand-band-lo", "0.804",
|
|
"--expand-band-hi", "0.952", "--expand-gallery"]
|
|
|
|
|
|
def xgb_boundary_seconds(reg, dump):
|
|
X, yb, ic = XB.per_second_matrix(dump, xr_for(dump), "experiments/dumps/audio_features")
|
|
prob = np.clip(reg.predict(X), 0, 1)
|
|
return set(XB.knee_boundaries(prob))
|
|
|
|
|
|
FILMS = json.load(open("experiments/manifests/films_LVFace_opencv5.json"))
|
|
_XR = {f["dump"]: f["xray"] for f in FILMS}
|
|
def xr_for(dump): return _XR[dump]
|
|
|
|
|
|
def inject_boundaries(dump, second_set, out_path):
|
|
"""Copy dump, set frames/is_scene_boundary=1 at the given integer seconds."""
|
|
shutil.copy(dump, out_path)
|
|
with h5py.File(out_path, "r+") as f:
|
|
ts = f["frames/timestamp_sec"][:]
|
|
bnd = np.zeros(len(ts), np.uint8)
|
|
for i, t in enumerate(ts):
|
|
if int(round(t)) in second_set:
|
|
bnd[i] = 1
|
|
if "frames/is_scene_boundary" in f:
|
|
f["frames/is_scene_boundary"][:] = bnd
|
|
else:
|
|
f["frames"].create_dataset("is_scene_boundary", data=bnd)
|
|
|
|
|
|
def replay(dump, out, mode):
|
|
argv = [".venv-rocm/bin/python" if False else sys.executable,
|
|
"scripts/optimizer/replay.py", "--dump", dump, "--gallery", GAL,
|
|
"--out", out] + CFG
|
|
if mode:
|
|
argv += ["--presence-mode", mode]
|
|
subprocess.run(argv, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, timeout=300)
|
|
return json.loads(Path(out).read_text())
|
|
|
|
|
|
def main():
|
|
reg = xgb.XGBRegressor(); reg.load_model(MODEL)
|
|
gk = load_gallery_keys(GAL)
|
|
tmp = tempfile.mkdtemp()
|
|
print(f"{'film':24s} {'trackext':>9} {'flood+hist':>11} {'flood+XGB':>10}")
|
|
agg = {"track_extent": [], "flood_hist": [], "flood_xgb": []}
|
|
for f in FILMS:
|
|
dump, xr = f["dump"], f["xray"]
|
|
out = f"{tmp}/out.json"
|
|
# A. track_extent
|
|
a = score_seconds(replay(dump, out, "track_extent"), xr, gallery_keys=gk)
|
|
# B. flood + histogram cuts (original dump's is_cut; is_scene_boundary=0)
|
|
b = score_seconds(replay(dump, out, "flood"), xr, gallery_keys=gk)
|
|
# C. flood + XGBoost boundaries injected
|
|
inj = f"{tmp}/inj_{f['slug']}.h5"
|
|
inject_boundaries(dump, xgb_boundary_seconds(reg, dump), inj)
|
|
c = score_seconds(replay(inj, out, "flood"), xr, gallery_keys=gk)
|
|
os.unlink(inj)
|
|
agg["track_extent"].append(a["f1"]); agg["flood_hist"].append(b["f1"])
|
|
agg["flood_xgb"].append(c["f1"])
|
|
print(f"{f['name'][:24]:24s} {a['f1']*100:8.1f}% {b['f1']*100:10.1f}% "
|
|
f"{c['f1']*100:9.1f}%")
|
|
print(f"\n{'MACRO-MEAN':24s} {np.mean(agg['track_extent'])*100:8.1f}% "
|
|
f"{np.mean(agg['flood_hist'])*100:10.1f}% {np.mean(agg['flood_xgb'])*100:9.1f}%")
|
|
json.dump({k: float(np.mean(v)) for k, v in agg.items()},
|
|
open("experiments/results/scene_boundary/downstream_presence.json", "w"),
|
|
indent=2)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|