dtourolle
0e35dac951
A boosted-tree scene-boundary detector that replaces the grayscale histogram-correlation cut detector as the flood-fill boundary source, and substantially improves actor-presence accuracy. Downstream result (per-second X-Ray presence F1, macro over 9 films): track_extent 62.3% | flood + histogram cuts 64.0% | flood + this 76.9% +12.9pp, and it wins on every film — notably fixing the histogram flood's Scarface collapse (61 -> 41 -> 71) and lifting Downton 41 -> 84. Design (each choice measured — see the memory / report): - XGBoost REGRESSOR on a ±3s window of DELTA features (symmetric RGB-hist and audio-PSD deltas at k=1,2,4,8s + ramp bank + time-since-last-peak debounce). Raw histograms dilute; deltas separate boundaries ~4-5x. - SOFT Gaussian proximity target (sigma=10s) so near-misses train as near-correct, not hard negatives; regression -> smooth score -> NMS peaks. - KNEE per-film threshold: self-calibrates the boundary count to ~the true scene count, no global rate. Evaluated at ±20s (X-Ray scenes ~170s). - Trained on all 9 films (Cafe/Scarface low-contrast grades must be seen). Honest held-out ~41% boundary-F1 @±20s vs ~27% grayscale. Scripts: train_xgb_boundary.py (shipped detector), extract_audio_features.py (per-second log-PSD), downstream_presence.py (the A/B above), density_floor.py (fallback for detection-starved films), plus the LSTM/DE explorations kept for provenance. Model: models/scene_boundary_xgb.json. Not yet wired into the live C++ pipeline — boundaries are a post-EOF step in the sink (like flood-fill itself); libxgboost C++ integration is the next step.
57 lines
2.9 KiB
Python
57 lines
2.9 KiB
Python
#!/usr/bin/env python3
|
|
"""Standalone DE-optimised AUDIO scene cutter: tune a matched-filter ramp on the
|
|
audio log-PSD to maximise X-Ray boundary F1. No neural net. Holdout films are
|
|
never seen in training. Writes the tuned filter + held-out performance."""
|
|
import sys, json, os
|
|
import numpy as np
|
|
sys.path.insert(0, "scripts/scene_detector")
|
|
from de_ramp import load_series, xray_bounds, ramp_kernel, response, boundary_f1
|
|
from scipy.optimize import differential_evolution
|
|
|
|
MANIFEST = "experiments/manifests/films_LVFace_opencv5.json"
|
|
AUDIO = "experiments/dumps/audio_features"
|
|
HOLDOUT = {"Scarface", "Sound_of_Metal", "Valerian_and_the_City_of_a_Thousand_Plan"}
|
|
OUT = "experiments/results/scene_boundary/de_audio_cutter.json"
|
|
|
|
films = json.load(open(MANIFEST))
|
|
train = [f for f in films if f["slug"] not in HOLDOUT]
|
|
val = [f for f in films if f["slug"] in HOLDOUT]
|
|
tr = [(load_series(f["dump"], AUDIO, "audio")[0], xray_bounds(f["xray"])) for f in train]
|
|
va = [(f["slug"], load_series(f["dump"], AUDIO, "audio")[0], xray_bounds(f["xray"])) for f in val]
|
|
print(f"DE AUDIO cutter: {len(tr)} train, holdout {sorted(HOLDOUT)}", flush=True)
|
|
|
|
def neg_f1(x):
|
|
H = int(round(x[0])); gamma = x[1]; dead = int(round(x[2])); pct = x[3]
|
|
if H < 1 or dead >= H: return 0.0
|
|
w = ramp_kernel(H, gamma, dead)
|
|
return -float(np.mean([boundary_f1(response(s, w, H), b, pct) for s, b in tr]))
|
|
|
|
evals = [0]
|
|
def cb(xk, convergence):
|
|
evals[0] += 1
|
|
print(f"[de-audio] gen {evals[0]} convergence={convergence:.3f}", flush=True)
|
|
|
|
res = differential_evolution(neg_f1, [(1, 10), (0.3, 3.0), (0, 4), (80, 98)],
|
|
seed=0, popsize=12, maxiter=25, tol=1e-4,
|
|
polish=False, callback=cb)
|
|
H = int(round(res.x[0])); gamma = float(res.x[1]); dead = int(round(res.x[2])); pct = float(res.x[3])
|
|
print(f"\n=== DE-OPTIMISED AUDIO SCENE CUTTER ===", flush=True)
|
|
print(f"tuned ramp: H={H}s gamma={gamma:.2f} dead={dead}s threshold_pct={pct:.0f}", flush=True)
|
|
print(f"train boundary-F1: {-res.fun*100:.1f}%\n", flush=True)
|
|
print("held-out (audio-only, P/R/F1 ±2s):", flush=True)
|
|
w = ramp_kernel(H, gamma, dead)
|
|
rep = {"H": H, "gamma": gamma, "dead": dead, "pct": pct,
|
|
"train_f1": float(-res.fun), "holdout": sorted(HOLDOUT), "films": {}}
|
|
for slug, s, b in va:
|
|
r = response(s, w, H); thr = np.percentile(r, pct); pred = np.where(r > thr)[0]
|
|
bidx = [int(x) for x in b if int(x) < len(r)]
|
|
tp_p = sum(any(abs(p-i) <= 2 for i in bidx) for p in pred)
|
|
tp_t = sum(any(abs(p-i) <= 2 for p in pred) for i in bidx)
|
|
P = tp_p/max(len(pred), 1); R = tp_t/max(len(bidx), 1); F = 2*P*R/(P+R) if P+R else 0
|
|
rep["films"][slug] = {"P": P, "R": R, "F1": F, "n_pred": len(pred), "n_true": len(bidx)}
|
|
print(f" {slug[:26]:26s} P={P*100:4.0f}% R={R*100:4.0f}% F1={F*100:4.0f}% "
|
|
f"({len(pred)} preds/{len(bidx)} true)", flush=True)
|
|
os.makedirs(os.path.dirname(OUT), exist_ok=True)
|
|
json.dump(rep, open(OUT, "w"), indent=2)
|
|
print(f"\nsaved → {OUT}", flush=True)
|