#!/bin/bash
# fetch_dvu.sh — pull one film's character mugshots and presence annotations from
# the NIST TRECVID Deep Video Understanding development set.
#
# The DVU dev set is the reason Road to Bali is our benchmark film: it ships
# 5-7 face crops per *character*, cut from the film itself, alongside
# scene-scoped presence annotations. That matches SR-002 directly — presence is
# per scene, not per frame — and it keeps ground truth in character space, so
# scoring needs no actor->character mapping.
#
# This exists as a script, rather than as ad hoc commands, because the first
# copy of this data lived in a temp directory and was lost to a /tmp wipe,
# taking the working gallery with it.
#
# 14 films are asserted Creative Commons and need no data agreement (only the
# 5 KinoLorber test films are gated).
#
# Usage:
#   scripts/fetch_dvu.sh [film] [dest]
#     film  default Road_To_Bali
#     dest  default ./dvu
set -euo pipefail

BASE="https://www-nlpir.nist.gov/projects/trecvid/dvu/dvu.development.dataset"
FILM="${1:-Road_To_Bali}"
DEST="${2:-dvu}"

mkdir -p "$DEST/images" "$DEST/scenes"

echo "[dvu] $FILM -> $DEST"

# Scene segmentation: start/end as HH:MM:SS. Note valkaama.csv line 38 carries a
# shift-key typo (01:!4:00) — parse defensively if you extend this to that film.
echo "[dvu] scene segmentation"
curl -fsSL "$BASE/scene.segmentation.reference/${FILM}.csv" \
    -o "$DEST/${FILM}.csv" || echo "  (missing: ${FILM}.csv)"

# Entity types: which entities are Person vs Location/Concept. Only Person rows
# become gallery identities — the images/ directory also holds Location and
# Concept crops (bedroom, boat, ...), which must not enter a face gallery.
#
# Directory and file naming are inconsistent with the film slug used elsewhere:
# the folder is Road_to_Bali (lowercase "to") while the entity file is
# RoadToBali.entity.types.txt. Both are derived here rather than assumed.
# NIST is inconsistent across all three axes, and not by a rule worth deriving:
# Road to Bali is Road_To_Bali.csv / Road_to_Bali/ / RoadToBali.entity.types.txt,
# while SuperHero is SuperHero.csv / superHero/ / superhero.entity.types.txt.
# Defaults cover the Bali shape; override per film rather than guessing.
#   KG_DIR=superHero KG_FILE=superhero scripts/fetch_dvu.sh SuperHero dvu-hero
KG_DIR="${KG_DIR:-${FILM//_To_/_to_}}"
KG_FILE="${KG_FILE:-$(echo "$FILM" | sed -E 's/_([a-z])/\U\1/g; s/_//g')}"

echo "[dvu] entity types ($KG_DIR/$KG_FILE)"
curl -fsSL "$BASE/movie_knowledge_graph/${KG_DIR}/${KG_FILE}.entity.types.txt" \
    -o "$DEST/${FILM}.entity.types.txt" || echo "  (missing: entity types)"

# Character face crops. Names are discovered from the directory listing rather
# than probed as <Character>_N, since the crop count varies per character and
# the listing is authoritative.
echo "[dvu] character mugshots"
PERSONS="$DEST/persons.txt"
if [ -f "$DEST/${FILM}.entity.types.txt" ]; then
    grep -iE "person" "$DEST/${FILM}.entity.types.txt" \
        | sed -E 's/[[:space:]]*[:,].*$//' | tr -d '\r' \
        | awk '{print tolower($1)}' | sort -u > "$PERSONS"
fi

curl -fsSL "$BASE/movie_knowledge_graph/${KG_DIR}/images/" 2>/dev/null \
    | grep -oE 'href="[^"?/][^"]*\.png"' | sed -E 's/href="//; s/"//' | sort -u \
    > "$DEST/all_images.txt"

while read -r img; do
    [ -z "$img" ] && continue
    # Strip the trailing _N to recover the entity name.
    who="$(echo "$img" | sed -E 's/_[0-9]+\.png$//' | awk '{print tolower($0)}')"
    if [ -s "$PERSONS" ] && ! grep -qx "$who" "$PERSONS"; then
        continue   # Location/Concept crop, not a face
    fi
    curl -fsSL "$BASE/movie_knowledge_graph/${KG_DIR}/images/${img}" \
        -o "$DEST/images/${img}" 2>/dev/null || rm -f "$DEST/images/${img}"
done < "$DEST/all_images.txt"

# Per-scene knowledge graphs. A Person->Location edge means that person was
# present for the whole scene. Some of these contain a stray ", ," that breaks
# strict JSON parsers.
echo "[dvu] scene graphs"
for n in $(seq 1 60); do
    curl -fsSL "$BASE/scenes_knowledge_graphs/${FILM//_/ }-${n}.json" \
        -o "$DEST/scenes/${FILM}-${n}.json" 2>/dev/null \
        || curl -fsSL "$BASE/scenes_knowledge_graphs/${FILM}-${n}.json" \
            -o "$DEST/scenes/${FILM}-${n}.json" 2>/dev/null \
            || rm -f "$DEST/scenes/${FILM}-${n}.json"
done

echo "[dvu] done:"
echo "  mugshots: $(ls "$DEST/images" 2>/dev/null | wc -l)"
echo "  scenes:   $(ls "$DEST/scenes" 2>/dev/null | wc -l)"
echo "  csv:      $([ -f "$DEST/${FILM}.csv" ] && echo yes || echo no)"