feat(artifacts): push and pull the VR-013 corpus
The cross-source study needs two 4K recordings and a hand-sorted set of face crops, neither of which belongs in git. Adds an xsource target to both artifact scripts. Push uploads the clips as-is (already compressed) and zips labelling/. Pull fetches both and regenerates frames with ffmpeg rather than downloading them: ~320 MB of PNG that is deterministic from the clips. The extraction settings are pinned in the script, not left to the caller, because the manifests key on frame filenames and on detection order within each frame — verify_labels.py runs afterwards and fails loudly if they drift. Pull refuses to overwrite an existing labelling/. It is human ground truth: somebody looked at 167 crops and placed each one, and silently replacing that with a remote copy would destroy the expensive half of the study. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com> TRACES: VR-013
This commit is contained in:
Claude Opus 5
@@ -11,6 +11,7 @@
|
||||
# scripts/artifacts/pull_artifacts.sh montage-frames <film-slug> [version]
|
||||
# scripts/artifacts/pull_artifacts.sh experiment-data [version]
|
||||
# scripts/artifacts/pull_artifacts.sh report-highlights <name> [version]
|
||||
# scripts/artifacts/pull_artifacts.sh xsource [version]
|
||||
# version defaults to "latest" (newest uploaded version, by created_at).
|
||||
set -euo pipefail
|
||||
|
||||
@@ -83,11 +84,71 @@ pull_report_highlight() {
|
||||
curl -sf "${DL_BASE}/generic/report-highlights/${version}/${name}" -o "${dest}/${name}"
|
||||
}
|
||||
|
||||
pull_xsource() {
|
||||
local version="$1"
|
||||
local dest="${REPO_ROOT}/experiments/xsource"
|
||||
echo "=== xsource (version ${version}) ==="
|
||||
mkdir -p "${dest}/clips" "${dest}/frames"
|
||||
|
||||
for clip in 5157339 5157344; do
|
||||
if [ -f "${dest}/clips/${clip}.mp4" ]; then
|
||||
echo " ${clip}.mp4 already present, skipping"
|
||||
else
|
||||
echo " fetching ${clip}.mp4..."
|
||||
curl -sf "${DL_BASE}/generic/xsource/${version}/${clip}.mp4" \
|
||||
-o "${dest}/clips/${clip}.mp4" \
|
||||
|| { echo " [warn] ${clip}.mp4 not found at version ${version}" >&2; continue; }
|
||||
fi
|
||||
done
|
||||
|
||||
if [ -d "${dest}/labelling" ]; then
|
||||
echo " labelling/ already present — NOT overwriting (it is hand-sorted"
|
||||
echo " ground truth; move it aside first if you really want the remote copy)"
|
||||
else
|
||||
echo " fetching labelling.zip..."
|
||||
local tmp; tmp="$(mktemp)"
|
||||
curl -sf "${DL_BASE}/generic/xsource/${version}/labelling.zip" -o "$tmp"
|
||||
unzip -qo "$tmp" -d "$dest"
|
||||
rm "$tmp"
|
||||
fi
|
||||
|
||||
# Frames are regenerated rather than shipped: they are ~320 MB of PNG that
|
||||
# ffmpeg reproduces exactly from the clips. The manifests key on these
|
||||
# filenames and on detection order within each frame, so the extraction
|
||||
# settings must match the ones dump_faces.py ran against — hence fps and
|
||||
# frame count are pinned here rather than left to the caller.
|
||||
if ! command -v ffmpeg >/dev/null; then
|
||||
echo " [warn] ffmpeg not found — frames not regenerated; the study" >&2
|
||||
echo " scripts will fail until you extract them" >&2
|
||||
return
|
||||
fi
|
||||
for clip in 5157339 5157344; do
|
||||
[ -f "${dest}/clips/${clip}.mp4" ] || continue
|
||||
if [ -f "${dest}/frames/d${clip}_001.png" ]; then
|
||||
echo " frames for ${clip} already present, skipping"
|
||||
continue
|
||||
fi
|
||||
echo " extracting frames for ${clip}..."
|
||||
ffmpeg -v error -i "${dest}/clips/${clip}.mp4" -vf fps=2 -frames:v 24 \
|
||||
"${dest}/frames/d${clip}_%03d.png"
|
||||
done
|
||||
|
||||
echo " verifying the labelled set..."
|
||||
if (cd "$dest" && python3 verify_labels.py >/dev/null 2>&1); then
|
||||
echo " verify_labels.py passed"
|
||||
else
|
||||
echo " [warn] verify_labels.py failed — run it directly to see why." >&2
|
||||
echo " A frame/manifest mismatch means the extraction settings" >&2
|
||||
echo " differ from the ones the crops were dumped against." >&2
|
||||
fi
|
||||
}
|
||||
|
||||
if [ $# -eq 0 ]; then
|
||||
echo "usage: $0 galleries [version]" >&2
|
||||
echo " $0 montage-frames <film-slug> [version]" >&2
|
||||
echo " $0 experiment-data [version]" >&2
|
||||
echo " $0 report-highlights <name> [version]" >&2
|
||||
echo " $0 xsource [version]" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
@@ -115,8 +176,13 @@ case "$TARGET" in
|
||||
[ "$VERSION" = "latest" ] && VERSION="$(resolve_latest_version report-highlights)"
|
||||
pull_report_highlight "$VERSION" "$NAME"
|
||||
;;
|
||||
xsource)
|
||||
VERSION="${2:-latest}"
|
||||
[ "$VERSION" = "latest" ] && VERSION="$(resolve_latest_version xsource)"
|
||||
pull_xsource "$VERSION"
|
||||
;;
|
||||
*)
|
||||
echo "unknown target: $TARGET (expected galleries, montage-frames, experiment-data, or report-highlights)" >&2
|
||||
echo "unknown target: $TARGET (expected galleries, montage-frames, experiment-data, report-highlights, or xsource)" >&2
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
|
||||
@@ -12,6 +12,7 @@
|
||||
# scripts/artifacts/push_artifacts.sh montage-frames
|
||||
# scripts/artifacts/push_artifacts.sh experiment-data
|
||||
# scripts/artifacts/push_artifacts.sh report-highlights
|
||||
# scripts/artifacts/push_artifacts.sh xsource
|
||||
# scripts/artifacts/push_artifacts.sh galleries montage-frames experiment-data report-highlights
|
||||
#
|
||||
# Package layout (owner=dtourolle, repo=scene-actor-extraction):
|
||||
@@ -109,8 +110,35 @@ push_report_highlights() {
|
||||
upload "report-highlights" "germar_beats_xray.jpg" "$src"
|
||||
}
|
||||
|
||||
push_xsource() {
|
||||
echo "=== xsource (version ${VERSION}) ==="
|
||||
local root="${REPO_ROOT}/experiments/xsource"
|
||||
if [ ! -d "$root/labelling" ]; then
|
||||
echo " no experiments/xsource/labelling found, skipping" >&2
|
||||
return
|
||||
fi
|
||||
|
||||
# Source recordings. Already compressed, so uploaded as-is rather than zipped.
|
||||
shopt -s nullglob
|
||||
for f in "$root"/clips/*.mp4; do
|
||||
upload "xsource" "$(basename "$f")" "$f"
|
||||
done
|
||||
shopt -u nullglob
|
||||
|
||||
# The hand-sorted crops and their manifests. This is human ground truth and
|
||||
# the expensive part of the study — a person looked at every crop and put it
|
||||
# in a folder. Frames are deliberately NOT pushed: they are deterministic
|
||||
# from the clips, and pulling regenerates them.
|
||||
local tmp; tmp="$(mktemp -d)"
|
||||
trap 'rm -rf "$tmp"' RETURN
|
||||
local zipfile="${tmp}/labelling.zip"
|
||||
(cd "$root" && zip -qr "$zipfile" labelling -x 'labelling/*.html' -x 'labelling/review_*.jpg' \
|
||||
-x 'labelling/verify_*.jpg')
|
||||
upload "xsource" "labelling.zip" "$zipfile"
|
||||
}
|
||||
|
||||
if [ $# -eq 0 ]; then
|
||||
echo "usage: $0 <galleries|montage-frames|experiment-data|report-highlights> [...]" >&2
|
||||
echo "usage: $0 <galleries|montage-frames|experiment-data|report-highlights|xsource> [...]" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
@@ -120,7 +148,8 @@ for target in "$@"; do
|
||||
montage-frames) push_montage_frames ;;
|
||||
experiment-data) push_experiment_data ;;
|
||||
report-highlights) push_report_highlights ;;
|
||||
*) echo "unknown target: $target (expected galleries, montage-frames, experiment-data, or report-highlights)" >&2; exit 1 ;;
|
||||
xsource) push_xsource ;;
|
||||
*) echo "unknown target: $target (expected galleries, montage-frames, experiment-data, report-highlights, or xsource)" >&2; exit 1 ;;
|
||||
esac
|
||||
done
|
||||
|
||||
|
||||
Reference in New Issue
Block a user