dtourolle
6f0ad83a55
Optimizer (scripts/optimizer/): replay.py runs the real C++ tracker/matcher/ scene_tracker chain over a dumped-embeddings HDF5 via sae_kpn, so a threshold sweep never re-decodes video or re-embeds faces. optimize.py drives scipy's differential_evolution over the knob space, with DE-level parallelism (multiple population candidates evaluated concurrently via a ThreadPoolExecutor) on top of per-film replay parallelism. second_score.py is the per-second X-Ray scoring metric (TPI/FPI/FN, out-of-cast misID weighted 10x, fair recall masked to gallery-known cast) that superseded an earlier scene-union metric. dump_error_frames.py / dump_scene_montage.py extract annotated video frames (bounding boxes, TPI/FPI/FN captions, onscreen-vs-offscreen split) for visual review of a replay against ground truth. Gallery utilities: cast_restrict.py, gallery_membership.py, fetch_missing_actors.py, reembed_gallery.py. scripts/validation/: X-Ray ground-truth loading and provider-agnostic identity matching (identity.py's keys_for — an actor is the union of every id we can derive, since pipeline output and ground truth don't share one id space). scripts/artifacts/: push/pull scripts for the Gitea generic package registry — galleries, montage frames, and experiment data (manifests/trajectories/results) are pushed there instead of committed, since none are needed to run the app, only benchmarks. Versioned by git short-SHA. scripts/docs/: MkDocs site build (build_site.sh) and the calibration-curve comparison chart (calibration_chart.py, matplotlib, reads each gallery's embedded calibration). Gallery-building scripts (make_jellyfin_gallery.py, make_gallery.py, filter_gallery.py, run_from_jellyfin.py, movienet_eval.py, movienet_prep.py, sae_gallery.py) updated to read/write HDF5 galleries exclusively, matching the engine-side format switch. run_from_jellyfin.py and the optimizer no longer carry movie source paths in shared manifests (some source filenames include scene-release tags) — resolved locally via a gitignored file-lut.json instead.
113 lines
4.6 KiB
Bash
Executable File
113 lines
4.6 KiB
Bash
Executable File
#!/bin/bash
|
|
# push_artifacts.sh — upload benchmark artifacts (galleries, montage frames) to the
|
|
# Gitea generic package registry, decoupled from git history entirely. These are
|
|
# NOT needed to run the main app — only for benchmarks/experiments/reports.
|
|
#
|
|
# Requires GITEA_TOKEN in the environment (a Gitea access token with package
|
|
# read/write scope). Never hardcode the token; export it in your shell:
|
|
# export GITEA_TOKEN=...
|
|
#
|
|
# Usage:
|
|
# scripts/artifacts/push_artifacts.sh galleries
|
|
# scripts/artifacts/push_artifacts.sh montage-frames
|
|
# scripts/artifacts/push_artifacts.sh experiment-data
|
|
# scripts/artifacts/push_artifacts.sh galleries montage-frames experiment-data
|
|
#
|
|
# Package layout (owner=dtourolle, repo=scene-actor-extraction):
|
|
# generic/galleries/<version>/gallery_<model>.h5 (one file per model)
|
|
# generic/montage-frames/<version>/<film-slug>.zip (zipped per-film frames)
|
|
# generic/experiment-data/<version>/experiment-data.zip (manifests/trajectories/results)
|
|
# version = current git short SHA, so artifacts are traceable to the code that
|
|
# produced them. Re-running with the same SHA overwrites that version's files.
|
|
set -euo pipefail
|
|
|
|
REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
|
|
OWNER="dtourolle"
|
|
PKG_REPO="scene-actor-extraction"
|
|
# NOTE: the generic package registry lives under /api/packages/ (no /v1/) —
|
|
# distinct from the regular REST API under /api/v1/packages/ used for listing.
|
|
BASE_URL="https://gitea.tourolle.paris/api/packages/${OWNER}"
|
|
VERSION="$(git -C "$REPO_ROOT" rev-parse --short HEAD)"
|
|
|
|
if [ -z "${GITEA_TOKEN:-}" ]; then
|
|
echo "error: GITEA_TOKEN is not set. export GITEA_TOKEN=<your token> and retry." >&2
|
|
exit 1
|
|
fi
|
|
|
|
upload() {
|
|
local package="$1" filename="$2" filepath="$3"
|
|
local url="${BASE_URL}/generic/${package}/${VERSION}/${filename}"
|
|
echo " uploading ${filename} -> ${package}/${VERSION}..."
|
|
curl -sf -X PUT "$url" \
|
|
-H "Authorization: token ${GITEA_TOKEN}" \
|
|
-H "Content-Type: application/octet-stream" \
|
|
--upload-file "$filepath" \
|
|
-o /dev/null -w " HTTP %{http_code}\n"
|
|
}
|
|
|
|
push_galleries() {
|
|
echo "=== galleries (version ${VERSION}) ==="
|
|
local dir="${REPO_ROOT}/experiments/galleries"
|
|
shopt -s nullglob
|
|
for f in "$dir"/gallery_*.h5; do
|
|
upload "galleries" "$(basename "$f")" "$f"
|
|
done
|
|
shopt -u nullglob
|
|
}
|
|
|
|
push_montage_frames() {
|
|
echo "=== montage-frames (version ${VERSION}) ==="
|
|
local root="${REPO_ROOT}/experiments/results/holdout/montage_bestworst"
|
|
if [ ! -d "$root" ]; then
|
|
echo " no montage_bestworst dir found, skipping" >&2
|
|
return
|
|
fi
|
|
local tmp
|
|
tmp="$(mktemp -d)"
|
|
trap 'rm -rf "$tmp"' RETURN
|
|
for filmdir in "$root"/*/; do
|
|
[ -d "$filmdir" ] || continue
|
|
local slug ascii_slug zipfile
|
|
slug="$(basename "$filmdir")"
|
|
# Gitea's generic package registry rejects non-ASCII filenames (verified:
|
|
# a bare é in the name 400s). Transliterate for the upload name only —
|
|
# the local directory name (with accents) is untouched.
|
|
ascii_slug="$(echo "$slug" | iconv -f utf-8 -t ascii//translit 2>/dev/null || echo "$slug")"
|
|
zipfile="${tmp}/${ascii_slug}.zip"
|
|
(cd "$filmdir" && zip -qr "$zipfile" .)
|
|
upload "montage-frames" "${ascii_slug}.zip" "$zipfile"
|
|
done
|
|
}
|
|
|
|
push_experiment_data() {
|
|
echo "=== experiment-data (version ${VERSION}) ==="
|
|
# manifests/trajectories/results are all small text/JSON — no source paths
|
|
# (manifests never carry a "movie" field; see experiments/file-lut.json)
|
|
# so this is safe to share as one bundle.
|
|
local tmp; tmp="$(mktemp -d)"
|
|
trap 'rm -rf "$tmp"' RETURN
|
|
local zipfile="${tmp}/experiment-data.zip"
|
|
(cd "$REPO_ROOT" && zip -qr "$zipfile" \
|
|
experiments/manifests experiments/trajectories experiments/results \
|
|
-x '*.log' -x '*/holdout/montage_bestworst/*' -x '*/holdout/frames/*' \
|
|
-x '*/holdout/montage/*' -x '*/holdout/*.jsonl' -x '*/holdout/pred_*.json' \
|
|
-x '*/holdout/raw_*.jsonl')
|
|
upload "experiment-data" "experiment-data.zip" "$zipfile"
|
|
}
|
|
|
|
if [ $# -eq 0 ]; then
|
|
echo "usage: $0 <galleries|montage-frames|experiment-data> [...]" >&2
|
|
exit 1
|
|
fi
|
|
|
|
for target in "$@"; do
|
|
case "$target" in
|
|
galleries) push_galleries ;;
|
|
montage-frames) push_montage_frames ;;
|
|
experiment-data) push_experiment_data ;;
|
|
*) echo "unknown target: $target (expected galleries, montage-frames, or experiment-data)" >&2; exit 1 ;;
|
|
esac
|
|
done
|
|
|
|
echo "Done. Browse at: https://gitea.tourolle.paris/${OWNER}/${PKG_REPO}/packages"
|