docs: richer report — data figures, success/failure frames, commit-pinned repo links
- experiment_charts.py generates 4 figures from experiments/ artifacts: held-out per-film F1, 16-combo ranking, DE search landscape, and the Downton detector-vs-tracker ghost timeline (replaces the blank title-card screenshot) - new frames: 19-correct wedding shot (success case), Many Saints ghost-vs-unknown frame (three error classes in one image) - rename rep4-optimizer-results.md -> model-bakeoff.md; rep4 kept only as the on-disk artifact prefix, explained once - repo file references are now links via https://REPOLINK/<path> placeholders; build_site.sh pins them to the HEAD commit's raw URLs and fails the build if a linked path doesn't exist at HEAD - drop references to removed scripts (scene_score.py, score_config.py) and to session-memory names; mark artifact-registry paths with their pull commands - commit readme_example.jpg + pipeline_topology.svg so README renders on the plain Gitea repo view - deploy_pages.sh: push built site/ to the gitea-pages branch
This commit is contained in:
@@ -0,0 +1,256 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
experiment_charts.py — generate the rep4/held-out figures referenced by the docs,
|
||||
from the experiment artifacts under experiments/ (no hardcoded numbers).
|
||||
|
||||
Figures:
|
||||
holdout_f1_by_film.png — held-out per-film F1 vs. the training-set fit
|
||||
rep4_matrix_f1.png — all 16 bake-off combos, colored by model
|
||||
de_search_landscape.png — DE search space: prob_threshold x extinction_sec, F1 as color
|
||||
downton_ghost_timeline.png— detector face_count vs. tracker output through the credits
|
||||
|
||||
Usage:
|
||||
python scripts/docs/experiment_charts.py --out-dir docs/assets/images
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
from pathlib import Path
|
||||
|
||||
import matplotlib.pyplot as plt
|
||||
import numpy as np
|
||||
from matplotlib.colors import LinearSegmentedColormap
|
||||
|
||||
REPO = Path(__file__).resolve().parent.parent.parent
|
||||
RESULTS = REPO / "experiments/results"
|
||||
|
||||
# Same model -> color mapping as calibration_chart.py, so identity is stable
|
||||
# across every figure in the report.
|
||||
MODEL_COLOURS = {
|
||||
"arcface_w600k_r50": ("ArcFace w600k-R50", "#2a78d6"),
|
||||
"arcface_r18": ("ArcFace R18", "#008300"),
|
||||
"arcface_w600k_mbf": ("ArcFace w600k-MBF", "#e87ba4"),
|
||||
"LVFace-B_Glint360K": ("LVFace-B Glint360K", "#eda100"),
|
||||
}
|
||||
INK = "#0b0b0b"
|
||||
MUTED = "#898781"
|
||||
GRID = "#e1e0d9"
|
||||
SURFACE = "#fcfcfb"
|
||||
BLUE = "#2a78d6"
|
||||
GREEN = "#008300"
|
||||
RED = "#e34948"
|
||||
|
||||
plt.rcParams.update({
|
||||
"figure.facecolor": SURFACE,
|
||||
"axes.facecolor": SURFACE,
|
||||
"savefig.facecolor": SURFACE,
|
||||
"text.color": INK,
|
||||
"axes.edgecolor": MUTED,
|
||||
"axes.labelcolor": INK,
|
||||
"xtick.color": MUTED,
|
||||
"ytick.color": MUTED,
|
||||
"axes.grid": True,
|
||||
"grid.color": GRID,
|
||||
"grid.linewidth": 0.8,
|
||||
"axes.spines.top": False,
|
||||
"axes.spines.right": False,
|
||||
"font.size": 11,
|
||||
})
|
||||
|
||||
|
||||
def training_best() -> dict:
|
||||
with open(RESULTS / "rep4_best_LVFace-B_Glint360K_full_exp.json") as f:
|
||||
return json.load(f)["best"]
|
||||
|
||||
|
||||
def fig_holdout_f1(out: Path):
|
||||
with open(RESULTS / "holdout/holdout_scores.json") as f:
|
||||
films = json.load(f)["per_film"]
|
||||
films = sorted(films, key=lambda d: d["f1"])
|
||||
names = [d["name"] for d in films]
|
||||
f1 = [d["f1"] * 100 for d in films]
|
||||
train_f1 = training_best()["f1"] * 100
|
||||
macro = float(np.mean(f1))
|
||||
|
||||
fig, ax = plt.subplots(figsize=(9, 4.2))
|
||||
ax.grid(axis="y", visible=False)
|
||||
bars = ax.barh(names, f1, height=0.55, color=BLUE, zorder=3)
|
||||
for b, v, d in zip(bars, f1, films):
|
||||
note = f"{v:.1f}%"
|
||||
if d["FPI_misid"]:
|
||||
note += f" ({d['FPI_misid']} misIDs)"
|
||||
ax.text(v + 1, b.get_y() + b.get_height() / 2, note,
|
||||
va="center", ha="left", fontsize=10, color=INK)
|
||||
ax.axvline(train_f1, color=MUTED, lw=1.5, ls="--", zorder=2)
|
||||
ax.text(train_f1 + 0.7, len(names) - 0.35, f"training-set fit {train_f1:.1f}%",
|
||||
color=MUTED, fontsize=9.5, ha="left", va="center")
|
||||
ax.axvline(macro, color=RED, lw=1.5, ls=":", zorder=2)
|
||||
ax.text(macro - 0.7, -0.72, f"held-out macro avg {macro:.1f}%",
|
||||
color=RED, fontsize=9.5, ha="right", va="center")
|
||||
ax.set_xlim(0, 100)
|
||||
ax.set_ylim(-1.05, len(names) - 0.3 + 0.55)
|
||||
ax.set_xlabel("per-second F1 (%)")
|
||||
ax.set_title("Shipped config on the 5 films the optimizer never saw",
|
||||
loc="left", fontsize=12, pad=12)
|
||||
fig.tight_layout()
|
||||
fig.savefig(out, dpi=160)
|
||||
plt.close(fig)
|
||||
|
||||
|
||||
def fig_rep4_matrix(out: Path):
|
||||
combos = []
|
||||
for path in sorted(RESULTS.glob("rep4_best_*.json")):
|
||||
stem = path.stem[len("rep4_best_"):]
|
||||
for slug in MODEL_COLOURS:
|
||||
if stem.startswith(slug):
|
||||
mode = stem[len(slug) + 1:] # e.g. full_exp
|
||||
with open(path) as f:
|
||||
best = json.load(f)["best"]
|
||||
combos.append((slug, mode, best["f1"] * 100))
|
||||
break
|
||||
combos.sort(key=lambda c: c[2])
|
||||
|
||||
fig, ax = plt.subplots(figsize=(9, 6.2))
|
||||
ax.grid(axis="y", visible=False)
|
||||
labels = []
|
||||
for i, (slug, mode, f1) in enumerate(combos):
|
||||
label, colour = MODEL_COLOURS[slug]
|
||||
scope, exp = mode.rsplit("_", 1)
|
||||
labels.append(f"{scope} · {'expand' if exp == 'exp' else 'no expand'}")
|
||||
ax.hlines(i, 50, f1, color=GRID, lw=1.2, zorder=2)
|
||||
ax.plot(f1, i, "o", ms=9, color=colour, zorder=3,
|
||||
mfc=colour if scope == "restricted" else SURFACE,
|
||||
mec=colour, mew=2)
|
||||
ax.text(f1 + 0.35, i, f"{f1:.1f}", va="center", fontsize=8.5, color=MUTED)
|
||||
ax.set_yticks(range(len(combos)), labels, fontsize=9)
|
||||
ax.set_xlim(65, 80)
|
||||
ax.set_xlabel("training-set per-second F1 (%)")
|
||||
ax.set_title("All 16 combos — filled dot = cast-restricted gallery, open = full",
|
||||
loc="left", fontsize=12, pad=12)
|
||||
handles = [plt.Line2D([], [], marker="o", ls="", ms=9, color=c, label=l)
|
||||
for _, (l, c) in MODEL_COLOURS.items()]
|
||||
ax.legend(handles=handles, loc="lower right", frameon=False, fontsize=9.5)
|
||||
fig.tight_layout()
|
||||
fig.savefig(out, dpi=160)
|
||||
plt.close(fig)
|
||||
|
||||
|
||||
def fig_de_landscape(out: Path):
|
||||
evals = []
|
||||
with open(REPO / "experiments/trajectories/rep4_LVFace-B_Glint360K_full_exp.jsonl") as f:
|
||||
for line in f:
|
||||
d = json.loads(line)
|
||||
evals.append((d["config"]["prob_threshold"],
|
||||
d["config"]["extinction_sec"], d["f1"] * 100))
|
||||
x, y, f1 = map(np.array, zip(*evals))
|
||||
best = training_best()
|
||||
|
||||
# one-hue sequential ramp (light -> dark blue), per the report palette
|
||||
cmap = LinearSegmentedColormap.from_list(
|
||||
"seq_blue", ["#cde2fb", "#86b6ef", "#3987e5", "#1c5cab", "#0d366b"])
|
||||
|
||||
fig, ax = plt.subplots(figsize=(9, 5.2))
|
||||
# clip the color scale to the top of the range — DE spends most evals near
|
||||
# the optimum, so an unclipped scale renders the structure invisible
|
||||
sc = ax.scatter(x, y, c=f1, cmap=cmap, s=22, linewidths=0, zorder=3,
|
||||
vmin=70, vmax=float(f1.max()))
|
||||
ax.plot(best["config"]["prob_threshold"], best["config"]["extinction_sec"],
|
||||
marker="*", ms=18, color=RED, mec=SURFACE, mew=1.2, zorder=4)
|
||||
ax.annotate(f"shipped optimum F1 {best['f1']*100:.1f}%",
|
||||
(best["config"]["prob_threshold"], best["config"]["extinction_sec"]),
|
||||
textcoords="offset points", xytext=(-14, -30),
|
||||
ha="right", fontsize=10, color=RED,
|
||||
arrowprops={"arrowstyle": "-", "color": RED, "lw": 1})
|
||||
cb = fig.colorbar(sc, ax=ax, pad=0.02)
|
||||
cb.set_label("per-second F1 (%)")
|
||||
cb.outline.set_visible(False)
|
||||
ax.set_xlabel("prob_threshold")
|
||||
ax.set_ylabel("extinction_sec")
|
||||
ax.set_title("All 512 DE evaluations, LVFace-B full-gallery + expansion",
|
||||
loc="left", fontsize=12, pad=12)
|
||||
fig.tight_layout()
|
||||
fig.savefig(out, dpi=160)
|
||||
plt.close(fig)
|
||||
|
||||
|
||||
def fig_downton_timeline(out: Path, t0: int = 7100, t1: int = 7340):
|
||||
import h5py
|
||||
|
||||
tracker = {}
|
||||
with open(RESULTS / "holdout/raw_Downton_Abbey__A_New_Era.jsonl") as f:
|
||||
for line in f:
|
||||
d = json.loads(line)
|
||||
tracker[int(d["timestamp_sec"])] = len(d["visible_actors"])
|
||||
with h5py.File(REPO / "experiments/dumps/LVFace-B_Glint360K/"
|
||||
"dump_Downton_Abbey__A_New_Era.h5", "r") as h5:
|
||||
ts = h5["frames/timestamp_sec"][:]
|
||||
fc = h5["frames/face_count"][:]
|
||||
det = {int(t): int(c) for t, c in zip(ts, fc)}
|
||||
|
||||
t = np.arange(t0, t1)
|
||||
# the raw stream occasionally skips a second under replay load — carry the
|
||||
# last seen value forward rather than dropping to 0
|
||||
trk, last = [], 0
|
||||
for s in t:
|
||||
if s in tracker:
|
||||
last = tracker[s]
|
||||
trk.append(last)
|
||||
trk = np.array(trk)
|
||||
dc = np.array([det.get(s, 0) for s in t])
|
||||
|
||||
fig, ax = plt.subplots(figsize=(9.5, 4.4))
|
||||
ax.grid(axis="x", visible=False)
|
||||
ax.fill_between(t, dc, step="mid", color=GREEN, alpha=0.25, zorder=2)
|
||||
ax.step(t, dc, where="mid", color=GREEN, lw=2, zorder=3)
|
||||
ax.step(t, trk, where="mid", color=BLUE, lw=2, zorder=4)
|
||||
|
||||
# longest contiguous run of "detector sees nothing, tracker still reporting"
|
||||
ghost = (dc == 0) & (trk > 0)
|
||||
runs, start = [], None
|
||||
for i, g in enumerate(ghost):
|
||||
if g and start is None:
|
||||
start = i
|
||||
elif not g and start is not None:
|
||||
runs.append((start, i - 1))
|
||||
start = None
|
||||
if start is not None:
|
||||
runs.append((start, len(ghost) - 1))
|
||||
if runs:
|
||||
i0, i1 = max(runs, key=lambda r: r[1] - r[0])
|
||||
g0, g1 = t[i0], t[i1]
|
||||
ax.axvspan(g0, g1, color=RED, alpha=0.08, zorder=1)
|
||||
ax.annotate(f"{g1 - g0}s of credits: 0 faces detected,\n"
|
||||
f"{trk[i0]} actors still reported (frozen boxes)",
|
||||
((g0 + g1) / 2, 20.5), ha="center", va="bottom",
|
||||
fontsize=10, color=RED)
|
||||
ax.text(t0 + 4, 27.3, "actors reported by tracker", color=BLUE,
|
||||
fontsize=10.5, va="bottom")
|
||||
ax.text(t0 + 4, 11.5, "faces seen by detector", color=GREEN,
|
||||
fontsize=10.5, va="bottom")
|
||||
ax.set_xlabel("film time (s)")
|
||||
ax.set_ylabel("count")
|
||||
ax.set_ylim(0, 31)
|
||||
ax.set_title("Downton Abbey: A New Era — the cut to credits, second by second",
|
||||
loc="left", fontsize=12, pad=12)
|
||||
fig.tight_layout()
|
||||
fig.savefig(out, dpi=160)
|
||||
plt.close(fig)
|
||||
|
||||
|
||||
def main():
|
||||
p = argparse.ArgumentParser()
|
||||
p.add_argument("--out-dir", type=Path,
|
||||
default=REPO / "docs/assets/images")
|
||||
args = p.parse_args()
|
||||
args.out_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
fig_holdout_f1(args.out_dir / "holdout_f1_by_film.png")
|
||||
fig_rep4_matrix(args.out_dir / "rep4_matrix_f1.png")
|
||||
fig_de_landscape(args.out_dir / "de_search_landscape.png")
|
||||
fig_downton_timeline(args.out_dir / "downton_ghost_timeline.png")
|
||||
print(f"[experiment_charts] wrote 4 figures to {args.out_dir}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in New Issue
Block a user