diff --git a/docs/assets/images/scene_detector_evolution.png b/docs/assets/images/scene_detector_evolution.png new file mode 100644 index 0000000..a684941 Binary files /dev/null and b/docs/assets/images/scene_detector_evolution.png differ diff --git a/docs/scene-boundary-detector.md b/docs/scene-boundary-detector.md index 6d3dbb0..9f118a5 100644 --- a/docs/scene-boundary-detector.md +++ b/docs/scene-boundary-detector.md @@ -62,7 +62,13 @@ Everything is per second, aligned to the 1-fps presence grid. the films where video is weak (Downton, Sound of Metal), so it is included and the model uses it where it helps. -![Detector development: features and model](assets/images/scene_detector_evolution.png) +![Detector development at strict ±2 s tolerance, and where the shipped detector landed at the ±20 s tolerance the pipeline uses](assets/images/scene_detector_evolution.png) + +The left panel is the *feature* development, scored at a strict ±2 s tolerance so +each change is visible — this is where "delta beats raw histogram" was measured, not +the shipped tolerance. The right panel is the shipped detector at the ±20 s +tolerance the pipeline actually uses (see below). The two panels are on different +tolerances by design and must not be read as one curve. Dead ends, all measured and discarded: audio-only detection; raw histograms/PSDs as input; a two-tower BiLSTM (no better than the tree, far slower); @@ -81,14 +87,22 @@ and TransNetV2 (a Conv3D net that will not co-reside with the ROCm/VAAPI stack). real boundaries give way to noise. Selecting at the knee **self-calibrates the boundary count** to roughly the true scene count, per film, with no global threshold that would be wrong for every grade. -- **Trained on all nine films** for the shipped model. Café Society and Scarface - (the low-contrast grades) *must* be in training — held out, the model cannot - generalise to them; in training they reach 70–86% boundary-F1. +- **Trained on all nine films** for the shipped model. Keeping the low-contrast + grades (Café Society, Scarface) in training matters most: on its own training + films the shipped model reaches **72.9% macro boundary-F1** (per-film 51–86%), + versus **29.8%** for the grayscale baseline on the same films. Boundary detection, held out (leave-one-out, ±20 s tolerance — appropriate given -~170 s scenes): **~34% F1, versus ~27% for the grayscale baseline.** The absolute -number is capped by the narrative-vs-audiovisual mismatch above; the point is the -downstream effect. +~170 s scenes): **44.1% macro F1, versus 29.8% for the grayscale baseline** — the +honest generalisation number, each film scored by a detector trained on the other +eight. Even the low-contrast grades generalise (Scarface held out 32%, Café Society +51%), where the grayscale detector scores 0% and 31%. The absolute number is capped +by the narrative-vs-audiovisual mismatch above — many boundaries have no +audio-visual signature at all — so the point is the downstream effect, below. + +| boundary-F1 @±20 s | grayscale | learned (LOO) | learned (train-all) | +| ------------------ | --------: | ------------: | ------------------: | +| macro over 9 films | 29.8% | **44.1%** | 72.9% | ## The result that matters: actor presence diff --git a/scripts/scene_detector/make_figures.py b/scripts/scene_detector/make_figures.py index 1087700..b2b6902 100644 --- a/scripts/scene_detector/make_figures.py +++ b/scripts/scene_detector/make_figures.py @@ -57,15 +57,34 @@ def fig_macro(): fig.tight_layout(); fig.savefig(OUT/"scene_presence_macro.png"); plt.close(fig) # ── Figure 3: feature/model evolution (boundary-F1 development) ────────────── +# Two panels, because the development curve and the shipped result are measured +# at DIFFERENT tolerances and must not be plotted on one axis: +# left — relative feature progress at the strict ±2 s tolerance (how the LSTM +# experiments were scored; establishes which features helped) +# right — the shipped XGBoost detector at the ±20 s tolerance the pipeline +# actually uses and scores at (grayscale vs learned-LOO vs train-all) def fig_evolution(): + fig,(axl,axr)=plt.subplots(1,2,figsize=(11,4.5),gridspec_kw={"width_ratios":[1.15,1]}) + steps=["grayscale\nbaseline","raw-hist\nLSTM","delta\nLSTM","XGBoost\n(delta+debounce)"] - f1=[7.2,7.5,10.8,15.2] # boundary-F1 @±2s during development - fig,ax=plt.subplots(figsize=(6.5,4.5)) - ax.plot(steps,f1,marker="o",color="#3d7ea6",lw=2,ms=8) - for i,v in enumerate(f1): ax.text(i,v+0.4,f"{v:.1f}%",ha="center",fontsize=10) - ax.set_ylabel("held-out boundary F1 @±2s (%)") - ax.set_title("Detector development: features + model") - ax.set_ylim(0,18) + dev=[7.2,7.5,10.8,15.2] # boundary-F1 @±2s during LSTM-era development + axl.plot(steps,dev,marker="o",color="#9aa7b4",lw=2,ms=8) + for i,v in enumerate(dev): axl.text(i,v+0.4,f"{v:.1f}%",ha="center",fontsize=9) + axl.set_ylabel("boundary F1 @±2 s (%)") + axl.set_title("Feature progress (strict ±2 s)") + axl.set_ylim(0,18) + + # shipped detector at the ±20s tolerance the pipeline uses — real measured + # macro numbers: grayscale (xgb_report gray_F1), learned LOO, learned train-all + names=["grayscale","learned\n(LOO)","learned\n(train-all)"] + f20=[29.8,44.1,72.9]; cols=["#e07a5f","#3d7ea6","#8fb8cf"] + bars=axr.bar(names,f20,color=cols) + for b,v in zip(bars,f20): axr.text(b.get_x()+b.get_width()/2,v+1.2,f"{v:.1f}%", + ha="center",fontsize=10,fontweight="bold") + axr.set_ylabel("boundary F1 @±20 s (%)") + axr.set_title("Shipped detector (±20 s, macro/9 films)") + axr.set_ylim(0,80) + fig.suptitle("Detector development, and where it landed",fontsize=13) fig.tight_layout(); fig.savefig(OUT/"scene_detector_evolution.png"); plt.close(fig) import csv as _csv