feat(artifacts): push and pull the VR-013 corpus

The cross-source study needs two 4K recordings and a hand-sorted set of
face crops, neither of which belongs in git. Adds an xsource target to
both artifact scripts.

Push uploads the clips as-is (already compressed) and zips labelling/.
Pull fetches both and regenerates frames with ffmpeg rather than
downloading them: ~320 MB of PNG that is deterministic from the clips.
The extraction settings are pinned in the script, not left to the
caller, because the manifests key on frame filenames and on detection
order within each frame — verify_labels.py runs afterwards and fails
loudly if they drift.

Pull refuses to overwrite an existing labelling/. It is human ground
truth: somebody looked at 167 crops and placed each one, and silently
replacing that with a remote copy would destroy the expensive half of
the study.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>

TRACES: VR-013

experiments/xsource/README.md

@@ -7,9 +7,29 @@ perfect, so it isolates the embedder. This one downscales the **whole frame**
 before the detector, so detection and landmark regression degrade with it.
 
 Corpus: two Pexels clips of one shoot (4096×2160, 25 fps), four people, all four
-present in both. Clips and hand-sorted crops are gitignored — push them with
+present in both.
-`scripts/artifacts/push_artifacts.sh`, because the sorting is human ground truth
+
-and expensive to redo.
+## Getting the data
+
+Clips, frames and hand-sorted crops are gitignored; they live in the artifact
+registry.
+
+    scripts/artifacts/pull_artifacts.sh xsource     # clips + labelling, frames regenerated
+    scripts/artifacts/push_artifacts.sh xsource     # after correcting labels
+
+Pulling fetches the two clips and the hand-sorted crops, then regenerates the
+frames with ffmpeg — ~320 MB of PNG that is deterministic from the clips, so it
+is not worth shipping. Extraction settings are pinned in the pull script because
+the manifests key on frame filenames *and* on detection order within each frame;
+`verify_labels.py` runs at the end and will fail loudly if they drift.
+
+Pull never overwrites an existing `labelling/`. That directory is human ground
+truth — somebody looked at all 167 crops and put each one in a folder — and it
+is the expensive part of this study, so push it once corrected.
+
+Clips are Pexels-licensed: free to use, no attribution required, but not
+CC or MIT. Fine as a frozen CI artifact on private infrastructure; do not
+redistribute them as stock content.
 
 ## Scripts
 

scripts/artifacts/pull_artifacts.sh

@@ -11,6 +11,7 @@
 #   scripts/artifacts/pull_artifacts.sh montage-frames <film-slug> [version]
 #   scripts/artifacts/pull_artifacts.sh experiment-data [version]
 #   scripts/artifacts/pull_artifacts.sh report-highlights <name> [version]
+#   scripts/artifacts/pull_artifacts.sh xsource [version]
 # version defaults to "latest" (newest uploaded version, by created_at).
 set -euo pipefail
 
@@ -83,11 +84,71 @@ pull_report_highlight() {
     curl -sf "${DL_BASE}/generic/report-highlights/${version}/${name}" -o "${dest}/${name}"
 }
 
+pull_xsource() {
+    local version="$1"
+    local dest="${REPO_ROOT}/experiments/xsource"
+    echo "=== xsource (version ${version}) ==="
+    mkdir -p "${dest}/clips" "${dest}/frames"
+
+    for clip in 5157339 5157344; do
+        if [ -f "${dest}/clips/${clip}.mp4" ]; then
+            echo "  ${clip}.mp4 already present, skipping"
+        else
+            echo "  fetching ${clip}.mp4..."
+            curl -sf "${DL_BASE}/generic/xsource/${version}/${clip}.mp4" \
+                -o "${dest}/clips/${clip}.mp4" \
+                || { echo "    [warn] ${clip}.mp4 not found at version ${version}" >&2; continue; }
+        fi
+    done
+
+    if [ -d "${dest}/labelling" ]; then
+        echo "  labelling/ already present — NOT overwriting (it is hand-sorted"
+        echo "  ground truth; move it aside first if you really want the remote copy)"
+    else
+        echo "  fetching labelling.zip..."
+        local tmp; tmp="$(mktemp)"
+        curl -sf "${DL_BASE}/generic/xsource/${version}/labelling.zip" -o "$tmp"
+        unzip -qo "$tmp" -d "$dest"
+        rm "$tmp"
+    fi
+
+    # Frames are regenerated rather than shipped: they are ~320 MB of PNG that
+    # ffmpeg reproduces exactly from the clips. The manifests key on these
+    # filenames and on detection order within each frame, so the extraction
+    # settings must match the ones dump_faces.py ran against — hence fps and
+    # frame count are pinned here rather than left to the caller.
+    if ! command -v ffmpeg >/dev/null; then
+        echo "  [warn] ffmpeg not found — frames not regenerated; the study" >&2
+        echo "         scripts will fail until you extract them" >&2
+        return
+    fi
+    for clip in 5157339 5157344; do
+        [ -f "${dest}/clips/${clip}.mp4" ] || continue
+        if [ -f "${dest}/frames/d${clip}_001.png" ]; then
+            echo "  frames for ${clip} already present, skipping"
+            continue
+        fi
+        echo "  extracting frames for ${clip}..."
+        ffmpeg -v error -i "${dest}/clips/${clip}.mp4" -vf fps=2 -frames:v 24 \
+            "${dest}/frames/d${clip}_%03d.png"
+    done
+
+    echo "  verifying the labelled set..."
+    if (cd "$dest" && python3 verify_labels.py >/dev/null 2>&1); then
+        echo "    verify_labels.py passed"
+    else
+        echo "    [warn] verify_labels.py failed — run it directly to see why." >&2
+        echo "           A frame/manifest mismatch means the extraction settings" >&2
+        echo "           differ from the ones the crops were dumped against." >&2
+    fi
+}
+
 if [ $# -eq 0 ]; then
     echo "usage: $0 galleries [version]" >&2
     echo "       $0 montage-frames <film-slug> [version]" >&2
     echo "       $0 experiment-data [version]" >&2
     echo "       $0 report-highlights <name> [version]" >&2
+    echo "       $0 xsource [version]" >&2
     exit 1
 fi
 
@@ -115,8 +176,13 @@ case "$TARGET" in
         [ "$VERSION" = "latest" ] && VERSION="$(resolve_latest_version report-highlights)"
         pull_report_highlight "$VERSION" "$NAME"
         ;;
+    xsource)
+        VERSION="${2:-latest}"
+        [ "$VERSION" = "latest" ] && VERSION="$(resolve_latest_version xsource)"
+        pull_xsource "$VERSION"
+        ;;
     *)
-        echo "unknown target: $TARGET (expected galleries, montage-frames, experiment-data, or report-highlights)" >&2
+        echo "unknown target: $TARGET (expected galleries, montage-frames, experiment-data, report-highlights, or xsource)" >&2
         exit 1
         ;;
 esac

scripts/artifacts/push_artifacts.sh

@@ -12,6 +12,7 @@
 #   scripts/artifacts/push_artifacts.sh montage-frames
 #   scripts/artifacts/push_artifacts.sh experiment-data
 #   scripts/artifacts/push_artifacts.sh report-highlights
+#   scripts/artifacts/push_artifacts.sh xsource
 #   scripts/artifacts/push_artifacts.sh galleries montage-frames experiment-data report-highlights
 #
 # Package layout (owner=dtourolle, repo=scene-actor-extraction):
@@ -109,8 +110,35 @@ push_report_highlights() {
     upload "report-highlights" "germar_beats_xray.jpg" "$src"
 }
 
+push_xsource() {
+    echo "=== xsource (version ${VERSION}) ==="
+    local root="${REPO_ROOT}/experiments/xsource"
+    if [ ! -d "$root/labelling" ]; then
+        echo "  no experiments/xsource/labelling found, skipping" >&2
+        return
+    fi
+
+    # Source recordings. Already compressed, so uploaded as-is rather than zipped.
+    shopt -s nullglob
+    for f in "$root"/clips/*.mp4; do
+        upload "xsource" "$(basename "$f")" "$f"
+    done
+    shopt -u nullglob
+
+    # The hand-sorted crops and their manifests. This is human ground truth and
+    # the expensive part of the study — a person looked at every crop and put it
+    # in a folder. Frames are deliberately NOT pushed: they are deterministic
+    # from the clips, and pulling regenerates them.
+    local tmp; tmp="$(mktemp -d)"
+    trap 'rm -rf "$tmp"' RETURN
+    local zipfile="${tmp}/labelling.zip"
+    (cd "$root" && zip -qr "$zipfile" labelling -x 'labelling/*.html' -x 'labelling/review_*.jpg' \
+        -x 'labelling/verify_*.jpg')
+    upload "xsource" "labelling.zip" "$zipfile"
+}
+
 if [ $# -eq 0 ]; then
-    echo "usage: $0 <galleries|montage-frames|experiment-data|report-highlights> [...]" >&2
+    echo "usage: $0 <galleries|montage-frames|experiment-data|report-highlights|xsource> [...]" >&2
     exit 1
 fi
 
@@ -120,7 +148,8 @@ for target in "$@"; do
         montage-frames)     push_montage_frames ;;
         experiment-data)    push_experiment_data ;;
         report-highlights)  push_report_highlights ;;
-        *) echo "unknown target: $target (expected galleries, montage-frames, experiment-data, or report-highlights)" >&2; exit 1 ;;
+        xsource)            push_xsource ;;
+        *) echo "unknown target: $target (expected galleries, montage-frames, experiment-data, report-highlights, or xsource)" >&2; exit 1 ;;
     esac
 done