scene-actor-extraction/scripts/artifacts/pull_artifacts.sh

#!/bin/bash
# pull_artifacts.sh — download benchmark artifacts (galleries, montage frames) from
# the Gitea generic package registry. Counterpart to push_artifacts.sh.
#
# Downloads are public (no token needed) as long as the repo/packages are public.
# Resolving "latest" needs GITEA_TOKEN (the list-packages endpoint requires auth
# on this instance even for a public account) — export it before using "latest".
#
# Usage:
#   scripts/artifacts/pull_artifacts.sh galleries [version]
#   scripts/artifacts/pull_artifacts.sh montage-frames <film-slug> [version]
#   scripts/artifacts/pull_artifacts.sh experiment-data [version]
#   scripts/artifacts/pull_artifacts.sh replay-fixtures [version]
#   scripts/artifacts/pull_artifacts.sh report-highlights <name> [version]
#   scripts/artifacts/pull_artifacts.sh xsource [version]
# version defaults to "latest" (newest uploaded version, by created_at).
set -euo pipefail

REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
OWNER="dtourolle"
# NOTE: file download lives under /api/packages/ (no /v1/); listing/metadata
# lives under the regular /api/v1/packages/ REST API. Different base paths,
# both real — see push_artifacts.sh's comment.
DL_BASE="https://gitea.tourolle.paris/api/packages/${OWNER}"

resolve_latest_version() {
    local package="$1"
    if [ -z "${GITEA_TOKEN:-}" ]; then
        echo "error: resolving 'latest' needs GITEA_TOKEN (list-packages requires auth here)." >&2
        echo "  export GITEA_TOKEN=... or pass an explicit version instead of 'latest'." >&2
        exit 1
    fi
    curl -sf "https://gitea.tourolle.paris/api/v1/packages/${OWNER}" \
        -H "Authorization: token ${GITEA_TOKEN}" \
        | python3 -c "
import json, sys
d = json.load(sys.stdin)
matches = [p for p in d if p['name'] == '$package' and p['type'] == 'generic']
if not matches:
    sys.exit('no versions found for package \'$package\'')
matches.sort(key=lambda p: p['created_at'])
print(matches[-1]['version'])
"
}

pull_replay_fixtures() {
    local version="$1"
    local dest="${REPO_ROOT}/tests/fixtures/dumps"
    mkdir -p "$dest"
    echo "=== replay-fixtures (version ${version}) ==="
    local tmp; tmp="$(mktemp -d)"
    if curl -sf "${DL_BASE}/generic/replay-fixtures/${version}/replay-fixtures.zip" \
            -o "${tmp}/f.zip"; then
        unzip -qo "${tmp}/f.zip" -d "$dest"
        echo "  restored: $(ls "$dest" | wc -l) files into tests/fixtures/dumps/"
    else
        echo "    [warn] replay-fixtures.zip not found at version ${version}" >&2
    fi
    rm -rf "$tmp"
}

pull_galleries() {
    local version="$1"
    local dest="${REPO_ROOT}/experiments/galleries"
    mkdir -p "$dest"
    echo "=== galleries (version ${version}) ==="
    for model in arcface_w600k_r50 arcface_r18 arcface_w600k_mbf LVFace-B_Glint360K; do
        local f="gallery_${model}.h5"
        echo "  fetching ${f}..."
        curl -sf "${DL_BASE}/generic/galleries/${version}/${f}" \
            -o "${dest}/${f}" || echo "    [warn] ${f} not found at version ${version}"
    done
}

pull_montage_frames() {
    local version="$1" film="$2"
    local dest="${REPO_ROOT}/experiments/results/holdout/montage_bestworst"
    mkdir -p "$dest"
    echo "=== montage-frames/${film} (version ${version}) ==="
    local tmp; tmp="$(mktemp)"
    curl -sf "${DL_BASE}/generic/montage-frames/${version}/${film}.zip" -o "$tmp"
    mkdir -p "${dest}/${film}"
    unzip -qo "$tmp" -d "${dest}/${film}"
    rm "$tmp"
}

pull_experiment_data() {
    local version="$1"
    echo "=== experiment-data (version ${version}) ==="
    local tmp; tmp="$(mktemp)"
    curl -sf "${DL_BASE}/generic/experiment-data/${version}/experiment-data.zip" -o "$tmp"
    unzip -qo "$tmp" -d "$REPO_ROOT"
    rm "$tmp"
}

pull_report_highlight() {
    local version="$1" name="$2"
    local dest="${REPO_ROOT}/docs/assets/images"
    mkdir -p "$dest"
    echo "=== report-highlights/${name} (version ${version}) ==="
    curl -sf "${DL_BASE}/generic/report-highlights/${version}/${name}" -o "${dest}/${name}"
}

pull_xsource() {
    local version="$1"
    local dest="${REPO_ROOT}/experiments/xsource"
    echo "=== xsource (version ${version}) ==="
    mkdir -p "${dest}/clips" "${dest}/frames"

    for clip in 5157339 5157344; do
        if [ -f "${dest}/clips/${clip}.mp4" ]; then
            echo "  ${clip}.mp4 already present, skipping"
        else
            echo "  fetching ${clip}.mp4..."
            curl -sf "${DL_BASE}/generic/xsource/${version}/${clip}.mp4" \
                -o "${dest}/clips/${clip}.mp4" \
                || { echo "    [warn] ${clip}.mp4 not found at version ${version}" >&2; continue; }
        fi
    done

    if [ -d "${dest}/labelling" ]; then
        echo "  labelling/ already present — NOT overwriting (it is hand-sorted"
        echo "  ground truth; move it aside first if you really want the remote copy)"
    else
        echo "  fetching labelling.zip..."
        local tmp; tmp="$(mktemp)"
        curl -sf "${DL_BASE}/generic/xsource/${version}/labelling.zip" -o "$tmp"
        unzip -qo "$tmp" -d "$dest"
        rm "$tmp"
    fi

    # Frames are regenerated rather than shipped: they are ~320 MB of PNG that
    # ffmpeg reproduces exactly from the clips. The manifests key on these
    # filenames and on detection order within each frame, so the extraction
    # settings must match the ones dump_faces.py ran against — hence fps and
    # frame count are pinned here rather than left to the caller.
    if ! command -v ffmpeg >/dev/null; then
        echo "  [warn] ffmpeg not found — frames not regenerated; the study" >&2
        echo "         scripts will fail until you extract them" >&2
        return
    fi
    for clip in 5157339 5157344; do
        [ -f "${dest}/clips/${clip}.mp4" ] || continue
        if [ -f "${dest}/frames/d${clip}_001.png" ]; then
            echo "  frames for ${clip} already present, skipping"
            continue
        fi
        echo "  extracting frames for ${clip}..."
        ffmpeg -v error -i "${dest}/clips/${clip}.mp4" -vf fps=2 -frames:v 24 \
            "${dest}/frames/d${clip}_%03d.png"
    done

    echo "  verifying the labelled set..."
    if (cd "$dest" && python3 verify_labels.py >/dev/null 2>&1); then
        echo "    verify_labels.py passed"
    else
        echo "    [warn] verify_labels.py failed — run it directly to see why." >&2
        echo "           A frame/manifest mismatch means the extraction settings" >&2
        echo "           differ from the ones the crops were dumped against." >&2
    fi
}

if [ $# -eq 0 ]; then
    echo "usage: $0 galleries [version]" >&2
    echo "       $0 montage-frames <film-slug> [version]" >&2
    echo "       $0 experiment-data [version]" >&2
    echo "       $0 report-highlights <name> [version]" >&2
    echo "       $0 xsource [version]" >&2
    exit 1
fi

TARGET="$1"
case "$TARGET" in
    galleries)
        VERSION="${2:-latest}"
        [ "$VERSION" = "latest" ] && VERSION="$(resolve_latest_version galleries)"
        pull_galleries "$VERSION"
        ;;
    montage-frames)
        FILM="${2:?usage: $0 montage-frames <film-slug> [version]}"
        VERSION="${3:-latest}"
        [ "$VERSION" = "latest" ] && VERSION="$(resolve_latest_version montage-frames)"
        pull_montage_frames "$VERSION" "$FILM"
        ;;
    experiment-data)
        VERSION="${2:-latest}"
        [ "$VERSION" = "latest" ] && VERSION="$(resolve_latest_version experiment-data)"
        pull_experiment_data "$VERSION"
        ;;
    report-highlights)
        NAME="${2:?usage: $0 report-highlights <name> [version]}"
        VERSION="${3:-latest}"
        [ "$VERSION" = "latest" ] && VERSION="$(resolve_latest_version report-highlights)"
        pull_report_highlight "$VERSION" "$NAME"
        ;;
    xsource)
        VERSION="${2:-latest}"
        [ "$VERSION" = "latest" ] && VERSION="$(resolve_latest_version xsource)"
        pull_xsource "$VERSION"
        ;;
    replay-fixtures)
        VERSION="${2:-latest}"
        [ "$VERSION" = "latest" ] && VERSION="$(resolve_latest_version replay-fixtures)"
        pull_replay_fixtures "$VERSION"
        ;;
    *)
        echo "unknown target: $TARGET (expected galleries, montage-frames, experiment-data, report-highlights, xsource, or replay-fixtures)" >&2
        exit 1
        ;;
esac