refactor(scripts): extract shared sae_* helpers and dedupe gallery builders
Consolidate copy-pasted logic across the gallery/run scripts into shared modules: - sae_env.py — zero-dependency .env loader (populates os.environ) - sae_tmdb.py — TMDB API helpers (tmdb_get, person images, id lookups) - sae_jellyfin.py— Jellyfin API helpers (jf_get, id/URL normalisation) - sae_gallery.py — image download + gallery.json writing make_gallery, make_jellyfin_gallery and filter_gallery now import these instead of carrying their own near-identical copies.
This commit is contained in:
@@ -33,57 +33,15 @@ import json
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
import requests
|
||||
|
||||
|
||||
def jf_get(base_url: str, api_key: str, path: str, **params) -> dict:
|
||||
url = base_url.rstrip("/") + path
|
||||
headers = {"X-Emby-Token": api_key, "Accept": "application/json"}
|
||||
r = requests.get(url, params=params, headers=headers, timeout=30)
|
||||
r.raise_for_status()
|
||||
return r.json()
|
||||
|
||||
|
||||
def find_item_id(base_url: str, api_key: str, title: str, item_types: list[str]) -> str:
|
||||
data = jf_get(
|
||||
base_url, api_key, "/Items",
|
||||
Recursive="true",
|
||||
IncludeItemTypes=",".join(item_types),
|
||||
SearchTerm=title,
|
||||
Limit=10,
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parent))
|
||||
# Re-exported for backwards compatibility — run_from_jellyfin.py and external
|
||||
# callers historically imported these from filter_gallery.
|
||||
from sae_jellyfin import ( # noqa: F401
|
||||
jf_get,
|
||||
find_item_id,
|
||||
fetch_cast_person_ids,
|
||||
actor_jellyfin_id,
|
||||
)
|
||||
items = data.get("Items", [])
|
||||
if not items:
|
||||
raise ValueError(f"No item found matching title {title!r}")
|
||||
if len(items) > 1:
|
||||
print("Multiple matches found:", file=sys.stderr)
|
||||
for it in items:
|
||||
print(f" {it['Id']} {it.get('Type')} {it.get('Name')} ({it.get('ProductionYear')})", file=sys.stderr)
|
||||
print(f"Using first match: {items[0]['Name']}", file=sys.stderr)
|
||||
return items[0]["Id"]
|
||||
|
||||
|
||||
def fetch_cast_person_ids(base_url: str, api_key: str, item_id: str) -> set[str]:
|
||||
# /Items/{id} (no user context) returns 400 on recent Jellyfin servers;
|
||||
# /Items?Ids=... works with an API key alone.
|
||||
data = jf_get(base_url, api_key, "/Items", Ids=item_id, Fields="People", Recursive="true")
|
||||
items = data.get("Items", [])
|
||||
if not items:
|
||||
raise ValueError(f"Item {item_id} not found")
|
||||
item = items[0]
|
||||
cast_ids = {p["Id"] for p in item.get("People", []) if p.get("Type") == "Actor"}
|
||||
|
||||
# Episodes generally only carry their own guest stars in "People" — the
|
||||
# regular/recurring cast lives on the parent Series item, so pull that
|
||||
# in too or the filtered gallery ends up missing the main cast.
|
||||
series_id = item.get("SeriesId")
|
||||
if series_id:
|
||||
series_data = jf_get(base_url, api_key, "/Items", Ids=series_id, Fields="People", Recursive="true")
|
||||
series_items = series_data.get("Items", [])
|
||||
if series_items:
|
||||
cast_ids |= {p["Id"] for p in series_items[0].get("People", []) if p.get("Type") == "Actor"}
|
||||
|
||||
return cast_ids
|
||||
|
||||
|
||||
def main():
|
||||
@@ -113,10 +71,8 @@ def main():
|
||||
cast_ids = fetch_cast_person_ids(args.jellyfin_url, args.api_key, item_id)
|
||||
print(f"Title has {len(cast_ids)} credited cast member(s)", file=sys.stderr)
|
||||
|
||||
# current make_jellyfin_gallery.py writes "jellyfin_id"; older galleries used
|
||||
# "jellyfin_person_id" (see gallery_store.cpp's fallback for the same pair).
|
||||
actors = [a for a in gallery.get("actors", [])
|
||||
if (a.get("jellyfin_id") or a.get("jellyfin_person_id")) in cast_ids]
|
||||
if actor_jellyfin_id(a) in cast_ids]
|
||||
missing = len(cast_ids) - len(actors)
|
||||
if missing > 0:
|
||||
print(f"[warn] {missing} cast member(s) not present in gallery (not yet embedded)", file=sys.stderr)
|
||||
|
||||
+4
-86
@@ -31,69 +31,15 @@ Get a free TMDB API key at: https://www.themoviedb.org/settings/api
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import io
|
||||
import json
|
||||
import sys
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
import requests
|
||||
from PIL import Image
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parent))
|
||||
from sae_embed_loader import load_embedder
|
||||
|
||||
TMDB_BASE = "https://api.themoviedb.org/3"
|
||||
TMDB_IMG = "https://image.tmdb.org/t/p/original"
|
||||
WIKIDATA_SPARQL = "https://query.wikidata.org/sparql"
|
||||
WIKIDATA_HEADERS = {
|
||||
"Accept": "application/sparql-results+json",
|
||||
"User-Agent": "scene-actor-extraction/1.0 (https://github.com/; gallery builder)",
|
||||
}
|
||||
|
||||
# ── TMDB helpers ──────────────────────────────────────────────────────────────
|
||||
|
||||
def tmdb_get(path: str, token: str, **params) -> dict:
|
||||
url = TMDB_BASE + path
|
||||
if token.startswith("eyJ"):
|
||||
headers = {"Authorization": f"Bearer {token}", "Accept": "application/json"}
|
||||
r = requests.get(url, params=params, headers=headers, timeout=10)
|
||||
else:
|
||||
params["api_key"] = token
|
||||
r = requests.get(url, params=params, headers={"Accept": "application/json"}, timeout=10)
|
||||
r.raise_for_status()
|
||||
return r.json()
|
||||
|
||||
|
||||
# ── Wikidata helpers ──────────────────────────────────────────────────────────
|
||||
|
||||
def wikidata_image_urls(imdb_person_id: str) -> list[str]:
|
||||
"""Return Commons image URL(s) for a person via their IMDB ID (P345 -> P18)."""
|
||||
if not imdb_person_id:
|
||||
return []
|
||||
query = (
|
||||
"SELECT ?image WHERE { "
|
||||
f'?person wdt:P345 "{imdb_person_id}" . '
|
||||
"?person wdt:P18 ?image . "
|
||||
"}"
|
||||
)
|
||||
try:
|
||||
r = requests.get(WIKIDATA_SPARQL, params={"query": query, "format": "json"},
|
||||
headers=WIKIDATA_HEADERS, timeout=15)
|
||||
r.raise_for_status()
|
||||
bindings = r.json().get("results", {}).get("bindings", [])
|
||||
return [b["image"]["value"] for b in bindings if "image" in b]
|
||||
except Exception as e:
|
||||
print(f" [warn] wikidata lookup failed for {imdb_person_id}: {e}", file=sys.stderr)
|
||||
return []
|
||||
|
||||
|
||||
def tmdb_id_from_imdb(imdb_id: str, key: str) -> int:
|
||||
data = tmdb_get(f"/find/{imdb_id}", key, external_source="imdb_id")
|
||||
results = data.get("movie_results", [])
|
||||
if not results:
|
||||
raise ValueError(f"No TMDB movie found for IMDB ID {imdb_id}")
|
||||
return results[0]["id"]
|
||||
from sae_gallery import download_images, save_gallery, wikidata_image_urls
|
||||
from sae_tmdb import TMDB_IMG, tmdb_get, tmdb_id_from_imdb
|
||||
|
||||
|
||||
def fetch_cast(movie_id: int, key: str) -> list[dict]:
|
||||
@@ -135,27 +81,6 @@ def fetch_cast(movie_id: int, key: str) -> list[dict]:
|
||||
return actors
|
||||
|
||||
|
||||
# ── Image download ────────────────────────────────────────────────────────────
|
||||
|
||||
def download_images(actor: dict, dest_dir: Path, n: int) -> list[Path]:
|
||||
"""Download up to n profile images for an actor into dest_dir."""
|
||||
dest_dir.mkdir(parents=True, exist_ok=True)
|
||||
paths = []
|
||||
for i, url in enumerate(actor["profile_images"][:n]):
|
||||
out = dest_dir / f"{i:02d}.jpg"
|
||||
if out.exists() and out.stat().st_size > 1024:
|
||||
paths.append(out)
|
||||
continue
|
||||
try:
|
||||
r = requests.get(url, timeout=15)
|
||||
r.raise_for_status()
|
||||
Image.open(io.BytesIO(r.content)).convert("RGB").save(out, "JPEG")
|
||||
paths.append(out)
|
||||
except Exception as e:
|
||||
print(f" [warn] download failed: {url}: {e}", file=sys.stderr)
|
||||
return paths
|
||||
|
||||
|
||||
# ── Gallery assembly ─────────────────────────────────────────────────────────
|
||||
|
||||
def build_gallery(movie_id: int, key: str, embedder,
|
||||
@@ -182,7 +107,7 @@ def build_gallery(movie_id: int, key: str, embedder,
|
||||
"tmdb_id": actor["tmdb_id"], "reason": "no images found"})
|
||||
continue
|
||||
|
||||
image_paths = download_images(actor, actor_dir, images_per_actor)
|
||||
image_paths = download_images(actor["profile_images"], actor_dir, images_per_actor)
|
||||
if not image_paths:
|
||||
print(" no images downloaded, skipping", file=sys.stderr)
|
||||
missing.append({"name": actor["name"], "imdb_id": actor["imdb_id"],
|
||||
@@ -279,14 +204,7 @@ def main():
|
||||
if n_actors == 0:
|
||||
sys.exit("No actors could be processed — check models and images.")
|
||||
|
||||
output.parent.mkdir(parents=True, exist_ok=True)
|
||||
output.write_text(json.dumps(gallery, indent=2) + "\n")
|
||||
print(f"Saved: {output}", file=sys.stderr)
|
||||
|
||||
if missing:
|
||||
missing_path = output.with_name(output.stem + ".missing_images.json")
|
||||
missing_path.write_text(json.dumps(missing, indent=2) + "\n")
|
||||
print(f"{len(missing)} actor(s) need images — see {missing_path}", file=sys.stderr)
|
||||
save_gallery(gallery, missing, output)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
|
||||
@@ -43,31 +43,25 @@ Get a free TMDB API key at: https://www.themoviedb.org/settings/api
|
||||
import argparse
|
||||
import concurrent.futures
|
||||
import os
|
||||
import io
|
||||
import json
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
import requests
|
||||
from PIL import Image
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parent))
|
||||
import sae_env # noqa: F401 — loads .env into os.environ on import
|
||||
from sae_embed_loader import load_embedder
|
||||
|
||||
TMDB_BASE = "https://api.themoviedb.org/3"
|
||||
TMDB_IMG = "https://image.tmdb.org/t/p/original"
|
||||
from sae_gallery import download_image, download_images, save_gallery, wikidata_image_urls
|
||||
from sae_jellyfin import actor_jellyfin_id, jf_get, normalize_jellyfin_url
|
||||
from sae_tmdb import (
|
||||
tmdb_person_by_name,
|
||||
tmdb_person_for_imdb,
|
||||
)
|
||||
|
||||
|
||||
# ── Jellyfin API helpers ────────────────────────────────────────────────────
|
||||
|
||||
def jf_get(base_url: str, api_key: str, path: str, **params) -> dict:
|
||||
url = base_url.rstrip("/") + path
|
||||
headers = {"X-Emby-Token": api_key, "Accept": "application/json"}
|
||||
r = requests.get(url, params=params, headers=headers, timeout=30)
|
||||
r.raise_for_status()
|
||||
return r.json()
|
||||
|
||||
|
||||
def fetch_library_items(base_url: str, api_key: str, item_types: list[str]):
|
||||
"""Yield every Movie/Series item dict (with its People field)."""
|
||||
start = 0
|
||||
@@ -125,99 +119,26 @@ def fetch_imdb_id(base_url: str, api_key: str, person_id: str) -> str | None:
|
||||
return data.get("ProviderIds", {}).get("Imdb")
|
||||
|
||||
|
||||
# ── TMDB fallback (for actors with no usable Jellyfin image) ────────────────
|
||||
|
||||
def tmdb_get(path: str, key: str, **params) -> dict:
|
||||
url = TMDB_BASE + path
|
||||
if key.startswith("eyJ"):
|
||||
headers = {"Authorization": f"Bearer {key}", "Accept": "application/json"}
|
||||
r = requests.get(url, params=params, headers=headers, timeout=10)
|
||||
else:
|
||||
params["api_key"] = key
|
||||
r = requests.get(url, params=params, headers={"Accept": "application/json"}, timeout=10)
|
||||
r.raise_for_status()
|
||||
return r.json()
|
||||
|
||||
|
||||
def tmdb_person_images(tmdb_person_id: str, tmdb_key: str) -> list[str]:
|
||||
images_data = tmdb_get(f"/person/{tmdb_person_id}/images", tmdb_key)
|
||||
return [TMDB_IMG + p["file_path"] for p in images_data.get("profiles", []) if p.get("file_path")]
|
||||
|
||||
|
||||
def tmdb_person_for_imdb(imdb_id: str, tmdb_key: str) -> tuple[str | None, list[str]]:
|
||||
"""Return (tmdb_person_id, profile_image_urls) for the TMDB person matching this IMDB person id."""
|
||||
data = tmdb_get(f"/find/{imdb_id}", tmdb_key, external_source="imdb_id")
|
||||
people = data.get("person_results", [])
|
||||
if not people:
|
||||
return None, []
|
||||
tmdb_person_id = str(people[0]["id"])
|
||||
return tmdb_person_id, tmdb_person_images(tmdb_person_id, tmdb_key)
|
||||
|
||||
|
||||
def tmdb_person_by_name(name: str, tmdb_key: str) -> tuple[str | None, list[str]]:
|
||||
"""Return (tmdb_person_id, profile_image_urls) for the best name match on TMDB.
|
||||
|
||||
Used when Jellyfin has no IMDB ProviderId for this person (the common case —
|
||||
Jellyfin rarely populates ProviderIds on Person items), so /find/{imdb_id}
|
||||
isn't an option. /search/person is sorted by popularity; take the top hit.
|
||||
"""
|
||||
data = tmdb_get("/search/person", tmdb_key, query=name)
|
||||
people = data.get("results", [])
|
||||
if not people:
|
||||
return None, []
|
||||
tmdb_person_id = str(people[0]["id"])
|
||||
return tmdb_person_id, tmdb_person_images(tmdb_person_id, tmdb_key)
|
||||
|
||||
|
||||
def download_urls(urls: list[str], dest_dir: Path, n: int, start_index: int = 0) -> list[Path]:
|
||||
"""Download up to n images from urls into dest_dir, numbered from start_index."""
|
||||
dest_dir.mkdir(parents=True, exist_ok=True)
|
||||
paths = []
|
||||
for i, url in enumerate(urls[:n]):
|
||||
out = dest_dir / f"{start_index + i:02d}.jpg"
|
||||
if out.exists() and out.stat().st_size > 1024:
|
||||
paths.append(out)
|
||||
continue
|
||||
try:
|
||||
r = requests.get(url, timeout=15)
|
||||
r.raise_for_status()
|
||||
Image.open(io.BytesIO(r.content)).convert("RGB").save(out, "JPEG")
|
||||
paths.append(out)
|
||||
except Exception as e:
|
||||
print(f" [warn] TMDB image download failed: {url}: {e}", file=sys.stderr)
|
||||
return paths
|
||||
|
||||
|
||||
# ── Image download ──────────────────────────────────────────────────────────
|
||||
|
||||
def download_person_images(base_url: str, api_key: str, person_id: str,
|
||||
dest_dir: Path, n: int) -> list[Path]:
|
||||
"""Download up to n images for a Jellyfin person item into dest_dir."""
|
||||
"""Download up to n images for a Jellyfin person item into dest_dir.
|
||||
|
||||
Jellyfin's per-index Primary image endpoint 404s once you run past the
|
||||
number of images it has, so stop at the first index that fails rather than
|
||||
probing all n.
|
||||
"""
|
||||
dest_dir.mkdir(parents=True, exist_ok=True)
|
||||
headers = {"X-Emby-Token": api_key}
|
||||
paths = []
|
||||
for i in range(n):
|
||||
out = dest_dir / f"{i:02d}.jpg"
|
||||
if out.exists() and out.stat().st_size > 1024:
|
||||
paths.append(out)
|
||||
continue
|
||||
url = base_url.rstrip("/") + f"/Items/{person_id}/Images/Primary/{i}"
|
||||
try:
|
||||
r = requests.get(url, headers=headers, params={"api_key": api_key}, timeout=15)
|
||||
if r.status_code == 404:
|
||||
break
|
||||
r.raise_for_status()
|
||||
ctype = r.headers.get("Content-Type", "")
|
||||
if not ctype.startswith("image/"):
|
||||
print(f" [warn] unexpected response for {person_id} index {i}: "
|
||||
f"status={r.status_code} content-type={ctype!r} "
|
||||
f"len={len(r.content)} body={r.content[:200]!r}", file=sys.stderr)
|
||||
break
|
||||
Image.open(io.BytesIO(r.content)).convert("RGB").save(out, "JPEG")
|
||||
paths.append(out)
|
||||
except Exception as e:
|
||||
print(f" [warn] image download failed for {person_id} index {i}: {e}", file=sys.stderr)
|
||||
path = download_image(url, out, headers=headers, params={"api_key": api_key})
|
||||
if path is None:
|
||||
break
|
||||
paths.append(path)
|
||||
return paths
|
||||
|
||||
|
||||
@@ -232,8 +153,12 @@ def fetch_actor_images(base_url: str, api_key: str, pid: str, info: dict,
|
||||
print(f"{name} ({pid}) — in {len(info['appearances'])} title(s)", file=sys.stderr)
|
||||
image_paths = download_person_images(base_url, api_key, pid, actor_dir, images_per_actor)
|
||||
|
||||
# Need the IMDB id for the TMDB /find lookup, to persist it (--fetch-imdb-ids),
|
||||
# and for the Wikidata fallback below. Fetch it once if Jellyfin's own image
|
||||
# came up short (so Wikidata can fire) or whenever it's otherwise wanted.
|
||||
short = len(image_paths) < images_per_actor
|
||||
imdb_id = None
|
||||
if fetch_imdb_ids or tmdb_key:
|
||||
if fetch_imdb_ids or tmdb_key or short:
|
||||
imdb_id = fetch_imdb_id(base_url, api_key, pid)
|
||||
|
||||
# Resolve the TMDB person id whenever possible — independent of whether
|
||||
@@ -255,7 +180,19 @@ def fetch_actor_images(base_url: str, api_key: str, pid: str, info: dict,
|
||||
needed = images_per_actor - len(image_paths)
|
||||
print(f" {name}: Jellyfin image missing/incomplete, falling back to TMDB "
|
||||
f"({len(tmdb_urls)} image(s) available)…", file=sys.stderr)
|
||||
image_paths += download_urls(tmdb_urls, actor_dir, needed, start_index=len(image_paths))
|
||||
image_paths += download_images(tmdb_urls, actor_dir, needed, start_index=len(image_paths))
|
||||
|
||||
# Last resort: a CC-licensed Commons headshot via Wikidata, keyed by the
|
||||
# actor's IMDB id. Catches actors TMDB has no usable image for (or that the
|
||||
# name search missed entirely).
|
||||
if len(image_paths) < images_per_actor and imdb_id:
|
||||
wiki_urls = wikidata_image_urls(imdb_id)
|
||||
if wiki_urls:
|
||||
needed = images_per_actor - len(image_paths)
|
||||
print(f" {name}: still short, falling back to Wikidata/Commons "
|
||||
f"({len(wiki_urls)} image(s) available)…", file=sys.stderr)
|
||||
image_paths += download_images(wiki_urls, actor_dir, needed,
|
||||
start_index=len(image_paths))
|
||||
|
||||
return image_paths, imdb_id, tmdb_id
|
||||
|
||||
@@ -326,17 +263,29 @@ def build_gallery(base_url: str, api_key: str, embedder, item_types: list[str],
|
||||
gallery_actors = []
|
||||
todo = []
|
||||
n_retry = 0
|
||||
n_topup = 0
|
||||
for pid, info in actors.items():
|
||||
existing = existing_actors.get(pid)
|
||||
if existing is not None and existing.get("embeddings"):
|
||||
n_existing = len(existing.get("embeddings", [])) if existing is not None else 0
|
||||
# Keep an existing actor as-is only once they're at the image target.
|
||||
# Fewer than that (including zero) → re-process to top up / retry, so a
|
||||
# merge run can upgrade 1-image actors to images_per_actor without
|
||||
# re-embedding actors already at the target.
|
||||
if n_existing >= images_per_actor:
|
||||
gallery_actors.append(existing)
|
||||
else:
|
||||
if existing is not None:
|
||||
if n_existing > 0:
|
||||
n_topup += 1
|
||||
elif existing is not None:
|
||||
n_retry += 1
|
||||
todo.append((pid, info))
|
||||
|
||||
if len(gallery_actors):
|
||||
print(f"Skipping {len(gallery_actors)} actor(s) already present in existing gallery", file=sys.stderr)
|
||||
print(f"Skipping {len(gallery_actors)} actor(s) already at {images_per_actor} "
|
||||
f"image(s) in existing gallery", file=sys.stderr)
|
||||
if n_topup:
|
||||
print(f"Topping up {n_topup} actor(s) with fewer than {images_per_actor} "
|
||||
f"embedding(s)", file=sys.stderr)
|
||||
if n_retry:
|
||||
print(f"Retrying {n_retry} actor(s) with no embeddings in existing gallery", file=sys.stderr)
|
||||
print(f"Processing {len(todo)} new actor(s) with {workers} worker(s)…", file=sys.stderr)
|
||||
@@ -393,8 +342,10 @@ def main():
|
||||
help="Directory containing ONNX models (default: models/)")
|
||||
parser.add_argument("--arcface", default=None,
|
||||
help="Path to ArcFace ONNX model (overrides --models-dir selection)")
|
||||
parser.add_argument("--images-per-actor", type=int, default=1,
|
||||
help="Images to download per actor (default: 1 — Jellyfin usually caches one)")
|
||||
parser.add_argument("--images-per-actor", type=int, default=10,
|
||||
help="Images to download per actor (default: 10). Jellyfin usually has "
|
||||
"only 1, so the rest come from the TMDB/Wikidata fallbacks; more "
|
||||
"images per actor means more embeddings and more robust matching.")
|
||||
parser.add_argument("--image-dir", default=None,
|
||||
help="Where to store downloaded images (default: <output_dir>/images)")
|
||||
parser.add_argument("--fetch-imdb-ids", action="store_true",
|
||||
@@ -414,12 +365,7 @@ def main():
|
||||
"on a single dedicated thread regardless of this value.")
|
||||
args = parser.parse_args()
|
||||
|
||||
# Defend against a base URL that accidentally includes an API path,
|
||||
# e.g. "https://host/Items?" — strip any query string and trailing
|
||||
# /Items so we don't build double-nested, query-mangled URLs.
|
||||
jellyfin_url = args.jellyfin_url.split("?", 1)[0].rstrip("/")
|
||||
if jellyfin_url.endswith("/Items"):
|
||||
jellyfin_url = jellyfin_url[: -len("/Items")]
|
||||
jellyfin_url = normalize_jellyfin_url(args.jellyfin_url)
|
||||
|
||||
output = Path(args.output)
|
||||
image_root = Path(args.image_dir) if args.image_dir else output.parent / "images"
|
||||
@@ -431,8 +377,7 @@ def main():
|
||||
if args.merge and output.is_file():
|
||||
existing = json.loads(output.read_text())
|
||||
for actor in existing.get("actors", []):
|
||||
# older galleries used "jellyfin_person_id"
|
||||
pid = actor.get("jellyfin_id") or actor.get("jellyfin_person_id")
|
||||
pid = actor_jellyfin_id(actor)
|
||||
if pid:
|
||||
existing_actors[pid] = actor
|
||||
print(f"Loaded {len(existing_actors)} actor(s) from existing gallery for merge", file=sys.stderr)
|
||||
@@ -457,14 +402,7 @@ def main():
|
||||
if n_actors == 0:
|
||||
sys.exit("No actors could be processed — check Jellyfin URL/API key and models.")
|
||||
|
||||
output.parent.mkdir(parents=True, exist_ok=True)
|
||||
output.write_text(json.dumps(gallery, indent=2) + "\n")
|
||||
print(f"Saved: {output}", file=sys.stderr)
|
||||
|
||||
if missing:
|
||||
missing_path = output.with_name(output.stem + ".missing_images.json")
|
||||
missing_path.write_text(json.dumps(missing, indent=2) + "\n")
|
||||
print(f"{len(missing)} actor(s) need images — see {missing_path}", file=sys.stderr)
|
||||
save_gallery(gallery, missing, output)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
|
||||
@@ -0,0 +1,49 @@
|
||||
"""Zero-dependency .env loader.
|
||||
|
||||
The gallery/run scripts read credentials (JELLYFIN_URL, JELLYFIN_API_KEY,
|
||||
TMDB_API_KEY) from os.environ, but nothing populated os.environ from the
|
||||
project's .env file — so the keys only worked if you'd manually
|
||||
`set -a; source .env`. This loads .env into os.environ on import, without
|
||||
pulling in python-dotenv.
|
||||
|
||||
Import this module early (before argparse reads os.environ defaults) so the
|
||||
.env values are available as defaults.
|
||||
"""
|
||||
|
||||
import os
|
||||
from pathlib import Path
|
||||
|
||||
# Walk up from this file to find the project root's .env (scripts/ is one
|
||||
# level down from the repo root).
|
||||
_DEFAULT_ENV = Path(__file__).resolve().parent.parent / ".env"
|
||||
|
||||
|
||||
def load_env(path: Path | str = _DEFAULT_ENV, *, override: bool = False) -> None:
|
||||
"""Parse a .env file into os.environ.
|
||||
|
||||
Lines are KEY=VALUE; blank lines and #-comments are ignored, surrounding
|
||||
whitespace is stripped, and a single layer of matching quotes around the
|
||||
value is removed. Existing os.environ entries win unless override=True, so
|
||||
an explicitly exported var (or one set on the command line) takes
|
||||
precedence over the file.
|
||||
"""
|
||||
path = Path(path)
|
||||
if not path.is_file():
|
||||
return
|
||||
for raw in path.read_text().splitlines():
|
||||
line = raw.strip()
|
||||
if not line or line.startswith("#") or "=" not in line:
|
||||
continue
|
||||
key, _, value = line.partition("=")
|
||||
key = key.strip()
|
||||
value = value.strip()
|
||||
if len(value) >= 2 and value[0] == value[-1] and value[0] in ("'", '"'):
|
||||
value = value[1:-1]
|
||||
if not key:
|
||||
continue
|
||||
if override or key not in os.environ:
|
||||
os.environ[key] = value
|
||||
|
||||
|
||||
# Load on import so `os.environ.get(...)` argparse defaults see .env values.
|
||||
load_env()
|
||||
@@ -0,0 +1,105 @@
|
||||
"""Shared image-download and gallery-writing helpers.
|
||||
|
||||
Consolidates the three near-identical download loops (make_gallery.download_images,
|
||||
make_jellyfin_gallery.download_urls + download_person_images) and the duplicated
|
||||
"write gallery.json + .missing_images.json" tail from both builders.
|
||||
"""
|
||||
|
||||
import io
|
||||
import json
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
import requests
|
||||
from PIL import Image, UnidentifiedImageError
|
||||
|
||||
# Errors that mean "this one image couldn't be fetched/decoded" rather than a bug:
|
||||
# network/HTTP failures, filesystem errors, and undecodable image bytes. We catch
|
||||
# exactly these around a single download so a coding error isn't swallowed as a
|
||||
# spurious "download failed".
|
||||
DOWNLOAD_ERRORS = (requests.RequestException, OSError, UnidentifiedImageError)
|
||||
|
||||
# Wikidata SPARQL: map an IMDB person id (P345) to their image (P18) on
|
||||
# Wikimedia Commons. Used as a free, CC-licensed headshot fallback when a
|
||||
# provider (TMDB/Jellyfin) has no usable image. Wikidata requires a
|
||||
# descriptive User-Agent.
|
||||
WIKIDATA_SPARQL = "https://query.wikidata.org/sparql"
|
||||
WIKIDATA_HEADERS = {
|
||||
"Accept": "application/sparql-results+json",
|
||||
"User-Agent": "scene-actor-extraction/1.0 (https://github.com/; gallery builder)",
|
||||
}
|
||||
|
||||
|
||||
def wikidata_image_urls(imdb_person_id: str) -> list[str]:
|
||||
"""Return Commons image URL(s) for a person via their IMDB id (P345 -> P18)."""
|
||||
if not imdb_person_id:
|
||||
return []
|
||||
query = (
|
||||
"SELECT ?image WHERE { "
|
||||
f'?person wdt:P345 "{imdb_person_id}" . '
|
||||
"?person wdt:P18 ?image . "
|
||||
"}"
|
||||
)
|
||||
try:
|
||||
r = requests.get(WIKIDATA_SPARQL, params={"query": query, "format": "json"},
|
||||
headers=WIKIDATA_HEADERS, timeout=15)
|
||||
r.raise_for_status()
|
||||
bindings = r.json().get("results", {}).get("bindings", [])
|
||||
return [b["image"]["value"] for b in bindings if "image" in b]
|
||||
except (requests.RequestException, ValueError) as e:
|
||||
print(f" [warn] wikidata lookup failed for {imdb_person_id}: {e}", file=sys.stderr)
|
||||
return []
|
||||
|
||||
|
||||
def download_image(url: str, out_path: Path, *,
|
||||
headers: dict | None = None,
|
||||
params: dict | None = None) -> Path | None:
|
||||
"""Download one image to out_path as JPEG, returning the path or None on failure.
|
||||
|
||||
Skips the download if out_path already exists and is larger than 1 KiB
|
||||
(a previously cached, non-truncated image).
|
||||
"""
|
||||
if out_path.exists() and out_path.stat().st_size > 1024:
|
||||
return out_path
|
||||
try:
|
||||
r = requests.get(url, headers=headers, params=params, timeout=15)
|
||||
r.raise_for_status()
|
||||
ctype = r.headers.get("Content-Type", "")
|
||||
if ctype and not ctype.startswith("image/"):
|
||||
print(f" [warn] unexpected response for {url}: "
|
||||
f"status={r.status_code} content-type={ctype!r} len={len(r.content)}",
|
||||
file=sys.stderr)
|
||||
return None
|
||||
out_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
Image.open(io.BytesIO(r.content)).convert("RGB").save(out_path, "JPEG")
|
||||
return out_path
|
||||
except DOWNLOAD_ERRORS as e:
|
||||
print(f" [warn] image download failed: {url}: {e}", file=sys.stderr)
|
||||
return None
|
||||
|
||||
|
||||
def download_images(urls: list[str], dest_dir: Path, n: int,
|
||||
start_index: int = 0, *,
|
||||
headers: dict | None = None,
|
||||
params: dict | None = None) -> list[Path]:
|
||||
"""Download up to n images from urls into dest_dir, numbered from start_index."""
|
||||
dest_dir.mkdir(parents=True, exist_ok=True)
|
||||
paths = []
|
||||
for i, url in enumerate(urls[:n]):
|
||||
out = dest_dir / f"{start_index + i:02d}.jpg"
|
||||
path = download_image(url, out, headers=headers, params=params)
|
||||
if path is not None:
|
||||
paths.append(path)
|
||||
return paths
|
||||
|
||||
|
||||
def save_gallery(gallery: dict, missing: list[dict], output: Path) -> None:
|
||||
"""Write gallery.json and, if any actors lack images, a .missing_images.json sidecar."""
|
||||
output.parent.mkdir(parents=True, exist_ok=True)
|
||||
output.write_text(json.dumps(gallery, indent=2) + "\n")
|
||||
print(f"Saved: {output}", file=sys.stderr)
|
||||
|
||||
if missing:
|
||||
missing_path = output.with_name(output.stem + ".missing_images.json")
|
||||
missing_path.write_text(json.dumps(missing, indent=2) + "\n")
|
||||
print(f"{len(missing)} actor(s) need images — see {missing_path}", file=sys.stderr)
|
||||
@@ -0,0 +1,119 @@
|
||||
"""Shared Jellyfin API helpers for the gallery/run scripts.
|
||||
|
||||
Consolidates the jf_get/find_item_id/fetch_cast_person_ids helpers that were
|
||||
previously copy-pasted across filter_gallery.py, make_jellyfin_gallery.py and
|
||||
run_from_jellyfin.py, plus the small gallery-id and URL-normalisation helpers.
|
||||
"""
|
||||
|
||||
import sys
|
||||
|
||||
import requests
|
||||
|
||||
|
||||
def jf_get(base_url: str, api_key: str, path: str, **params) -> dict:
|
||||
url = base_url.rstrip("/") + path
|
||||
headers = {"X-Emby-Token": api_key, "Accept": "application/json"}
|
||||
r = requests.get(url, params=params, headers=headers, timeout=30)
|
||||
r.raise_for_status()
|
||||
return r.json()
|
||||
|
||||
|
||||
def normalize_jellyfin_url(url: str) -> str:
|
||||
"""Strip any query string and a trailing /Items from a base URL.
|
||||
|
||||
Defends against a base URL that accidentally includes an API path, e.g.
|
||||
"https://host/Items?" — so we don't build double-nested, query-mangled URLs.
|
||||
"""
|
||||
url = url.split("?", 1)[0].rstrip("/")
|
||||
if url.endswith("/Items"):
|
||||
url = url[: -len("/Items")]
|
||||
return url
|
||||
|
||||
|
||||
def actor_jellyfin_id(actor: dict) -> str | None:
|
||||
"""Return an actor's Jellyfin person id, tolerating the older key name.
|
||||
|
||||
current make_jellyfin_gallery.py writes "jellyfin_id"; older galleries used
|
||||
"jellyfin_person_id" (see gallery_store.cpp's fallback for the same pair).
|
||||
"""
|
||||
return actor.get("jellyfin_id") or actor.get("jellyfin_person_id")
|
||||
|
||||
|
||||
def find_item_id(base_url: str, api_key: str, title: str, item_types: list[str]) -> str:
|
||||
data = jf_get(
|
||||
base_url, api_key, "/Items",
|
||||
Recursive="true",
|
||||
IncludeItemTypes=",".join(item_types),
|
||||
SearchTerm=title,
|
||||
Limit=10,
|
||||
)
|
||||
items = data.get("Items", [])
|
||||
if not items:
|
||||
raise ValueError(f"No item found matching title {title!r}")
|
||||
if len(items) > 1:
|
||||
print("Multiple matches found:", file=sys.stderr)
|
||||
for it in items:
|
||||
print(f" {it['Id']} {it.get('Type')} {it.get('Name')} ({it.get('ProductionYear')})", file=sys.stderr)
|
||||
print(f"Using first match: {items[0]['Name']}", file=sys.stderr)
|
||||
return items[0]["Id"]
|
||||
|
||||
|
||||
def fetch_episode_info(base_url: str, api_key: str, item_id: str) -> dict | None:
|
||||
"""Return episode locator info, or None if the item isn't an Episode.
|
||||
|
||||
Pulls the season/episode numbers off the Episode item and the series'
|
||||
TMDB/IMDB ProviderIds off the parent Series — TMDB's episode-credits
|
||||
endpoint is keyed by (series id, season, episode). Returns a dict with
|
||||
keys: series_id, season, episode, series_tmdb, series_imdb (any of the
|
||||
id/number values may be None if Jellyfin hasn't populated them).
|
||||
"""
|
||||
data = jf_get(base_url, api_key, "/Items", Ids=item_id,
|
||||
Fields="ParentIndexNumber,IndexNumber", Recursive="true")
|
||||
items = data.get("Items", [])
|
||||
if not items:
|
||||
return None
|
||||
item = items[0]
|
||||
if item.get("Type") != "Episode":
|
||||
return None
|
||||
|
||||
series_id = item.get("SeriesId")
|
||||
series_tmdb = series_imdb = None
|
||||
if series_id:
|
||||
series_data = jf_get(base_url, api_key, "/Items", Ids=series_id,
|
||||
Fields="ProviderIds", Recursive="true")
|
||||
series_items = series_data.get("Items", [])
|
||||
if series_items:
|
||||
providers = series_items[0].get("ProviderIds", {})
|
||||
series_tmdb = providers.get("Tmdb")
|
||||
series_imdb = providers.get("Imdb")
|
||||
|
||||
return {
|
||||
"series_id": series_id,
|
||||
"season": item.get("ParentIndexNumber"),
|
||||
"episode": item.get("IndexNumber"),
|
||||
"series_tmdb": series_tmdb,
|
||||
"series_imdb": series_imdb,
|
||||
}
|
||||
|
||||
|
||||
def fetch_cast_person_ids(base_url: str, api_key: str, item_id: str) -> set[str]:
|
||||
# /Items/{id} (no user context) returns 400 on recent Jellyfin servers;
|
||||
# /Items?Ids=... works with an API key alone.
|
||||
data = jf_get(base_url, api_key, "/Items", Ids=item_id, Fields="People", Recursive="true")
|
||||
items = data.get("Items", [])
|
||||
if not items:
|
||||
raise ValueError(f"Item {item_id} not found")
|
||||
item = items[0]
|
||||
cast_ids = {p["Id"] for p in item.get("People", []) if p.get("Type") == "Actor"}
|
||||
|
||||
# Episodes generally only carry their own guest stars in "People" — the
|
||||
# regular/recurring cast lives on the parent Series item, so pull that
|
||||
# in too or the filtered gallery ends up missing the main cast.
|
||||
series_id = item.get("SeriesId")
|
||||
if series_id:
|
||||
series_data = jf_get(base_url, api_key, "/Items", Ids=series_id, Fields="People", Recursive="true")
|
||||
series_items = series_data.get("Items", [])
|
||||
if series_items:
|
||||
cast_ids |= {p["Id"] for p in series_items[0].get("People", []) if p.get("Type") == "Actor"}
|
||||
|
||||
return cast_ids
|
||||
@@ -0,0 +1,88 @@
|
||||
"""Shared TMDB API helpers for the gallery builder scripts.
|
||||
|
||||
Consolidates the tmdb_get/tmdb_person_images/tmdb_person_* helpers that were
|
||||
byte-for-byte identical in make_gallery.py and make_jellyfin_gallery.py.
|
||||
"""
|
||||
|
||||
import requests
|
||||
|
||||
TMDB_BASE = "https://api.themoviedb.org/3"
|
||||
TMDB_IMG = "https://image.tmdb.org/t/p/original"
|
||||
|
||||
|
||||
def tmdb_get(path: str, token: str, **params) -> dict:
|
||||
url = TMDB_BASE + path
|
||||
# TMDB accepts two credential styles. A v4 "API Read Access Token" is a JWT,
|
||||
# which always starts with "eyJ" (base64 of '{"...'); it goes in an
|
||||
# Authorization: Bearer header. A v3 API key is an opaque hex string and goes
|
||||
# in the api_key query param instead. We auto-detect by that "eyJ" prefix.
|
||||
if token.startswith("eyJ"):
|
||||
headers = {"Authorization": f"Bearer {token}", "Accept": "application/json"}
|
||||
r = requests.get(url, params=params, headers=headers, timeout=10)
|
||||
else:
|
||||
params["api_key"] = token
|
||||
r = requests.get(url, params=params, headers={"Accept": "application/json"}, timeout=10)
|
||||
r.raise_for_status()
|
||||
return r.json()
|
||||
|
||||
|
||||
def tmdb_person_images(tmdb_person_id: str, token: str) -> list[str]:
|
||||
images_data = tmdb_get(f"/person/{tmdb_person_id}/images", token)
|
||||
return [TMDB_IMG + p["file_path"] for p in images_data.get("profiles", []) if p.get("file_path")]
|
||||
|
||||
|
||||
def tmdb_person_for_imdb(imdb_id: str, token: str) -> tuple[str | None, list[str]]:
|
||||
"""Return (tmdb_person_id, profile_image_urls) for the TMDB person matching this IMDB person id."""
|
||||
data = tmdb_get(f"/find/{imdb_id}", token, external_source="imdb_id")
|
||||
people = data.get("person_results", [])
|
||||
if not people:
|
||||
return None, []
|
||||
tmdb_person_id = str(people[0]["id"])
|
||||
return tmdb_person_id, tmdb_person_images(tmdb_person_id, token)
|
||||
|
||||
|
||||
def tmdb_person_by_name(name: str, token: str) -> tuple[str | None, list[str]]:
|
||||
"""Return (tmdb_person_id, profile_image_urls) for the best name match on TMDB.
|
||||
|
||||
Used when Jellyfin has no IMDB ProviderId for this person (the common case —
|
||||
Jellyfin rarely populates ProviderIds on Person items), so /find/{imdb_id}
|
||||
isn't an option. /search/person is sorted by popularity; take the top hit.
|
||||
"""
|
||||
data = tmdb_get("/search/person", token, query=name)
|
||||
people = data.get("results", [])
|
||||
if not people:
|
||||
return None, []
|
||||
tmdb_person_id = str(people[0]["id"])
|
||||
return tmdb_person_id, tmdb_person_images(tmdb_person_id, token)
|
||||
|
||||
|
||||
def tmdb_id_from_imdb(imdb_id: str, token: str) -> int:
|
||||
data = tmdb_get(f"/find/{imdb_id}", token, external_source="imdb_id")
|
||||
results = data.get("movie_results", [])
|
||||
if not results:
|
||||
raise ValueError(f"No TMDB movie found for IMDB ID {imdb_id}")
|
||||
return results[0]["id"]
|
||||
|
||||
|
||||
def tmdb_tv_id_from_imdb(imdb_id: str, token: str) -> int | None:
|
||||
"""Return the TMDB TV series id for an IMDB series id, or None if not found."""
|
||||
data = tmdb_get(f"/find/{imdb_id}", token, external_source="imdb_id")
|
||||
results = data.get("tv_results", [])
|
||||
return results[0]["id"] if results else None
|
||||
|
||||
|
||||
def tmdb_episode_cast(tv_id: int | str, season: int, episode: int, token: str) -> list[dict]:
|
||||
"""Return the per-episode cast for one episode (cast + guest_stars).
|
||||
|
||||
Uses /tv/{id}/season/{s}/episode/{e}/credits, which is the episode-level
|
||||
roster — strictly tighter than the whole-series cast Jellyfin exposes.
|
||||
Each dict carries at least "id" (TMDB person id) and "name".
|
||||
"""
|
||||
data = tmdb_get(f"/tv/{tv_id}/season/{season}/episode/{episode}/credits", token)
|
||||
# De-dupe by TMDB person id — a name can appear in both lists.
|
||||
by_id: dict = {}
|
||||
for person in data.get("cast", []) + data.get("guest_stars", []):
|
||||
pid = person.get("id")
|
||||
if pid is not None and pid not in by_id:
|
||||
by_id[pid] = person
|
||||
return list(by_id.values())
|
||||
Reference in New Issue
Block a user