DarkRoom/tools/film-profiles/convert.py

#!/usr/bin/env python3
"""Convert spektrafilm film profiles into DarkRoom's own compact format, and
generate the colour-science tables `dr-film` compiles in.

    python3 tools/film-profiles/convert.py --fetch

Kept in the tree, and kept runnable, so that the conversion from upstream is
reproducible and auditable rather than a one-off paste. CC BY-SA 4.0 requires
that a modified copy say it was modified; this script *is* the statement of
what was done, and `core/dr-film/profiles/CHANGELOG.txt` records it in prose.

The upstream profiles are published on the same 380-780nm, 5nm grid as the
Mallett 2019 sRGB basis and the CIE 1931 observer, so nothing here resamples
anything -- the conversion is a trim and a reformat, not an interpolation.
"""
import argparse
import json
import pathlib
import sys
import urllib.request

ROOT = pathlib.Path(__file__).resolve().parents[2]
PROFILE_DIR = ROOT / "core/dr-film/profiles"
UPSTREAM = "https://raw.githubusercontent.com/andreavolpato/spektrafilm/main"
UPSTREAM_PROFILES = f"{UPSTREAM}/src/spektrafilm/data/profiles"

# Every stock spektrafilm publishes. A print paper is a stock like any other,
# distinguished only by `support: paper`; a cine print film likewise, by
# `stage: printing`. The renderer treats all three the same and the *picker*
# decides what a photographer is offered, which is why they are all converted
# here rather than filtered at the source.
STOCKS = [
    "fujifilm_c200",
    "fujifilm_crystal_archive_typeii",
    "fujifilm_pro_400h",
    "fujifilm_provia_100f",
    "fujifilm_velvia_100",
    "fujifilm_xtra_400",
    "kodak_2383",
    "kodak_2393",
    "kodak_ektachrome_100",
    "kodak_ektacolor_edge",
    "kodak_ektar_100",
    "kodak_endura_premier",
    "kodak_gold_200",
    "kodak_kodachrome_64",
    "kodak_portra_160",
    "kodak_portra_400",
    "kodak_portra_800",
    "kodak_portra_800_push1",
    "kodak_portra_800_push2",
    "kodak_portra_endura",
    "kodak_supra_endura",
    "kodak_ultra_endura",
    "kodak_ultramax_400",
    "kodak_verita_200d",
    "kodak_vision3_50d",
    "kodak_vision3_200t",
    "kodak_vision3_250d",
    "kodak_vision3_500t",
    # Black and white. Upstream ships these on `dev` only, and they are the
    # only open, measured monochrome profiles in existence -- everything else
    # is a datasheet graph nobody has digitised.
    "kodak_trix",
    "kodak_doublex",
    "kodak_2302",
]

# Stocks that live on upstream's `dev` branch rather than `main`.
#
# The black-and-white ones, which is the whole reason for the exception: there
# is no monochrome stock on `main` at all, and these are the only open,
# measured B&W profiles that exist anywhere. Pinned per stock rather than
# moving everything to `dev`, so the colour stocks stay on the released branch.
DEV_BRANCH = {"kodak_trix", "kodak_doublex", "kodak_2302"}

# Which development time to ship, for a stock whose curves are measured at
# several. Index into the profile's own `development_time` array.
#
# The middle of the published range, which is the manufacturer's standard
# process: 6.5 minutes for Double-X, 5 for 2302. The others are push and pull,
# and are the obvious next control -- the data for them ships in the upstream
# file and is dropped here only because nothing can yet ask for it.
DEVELOPMENT_INDEX = 2

CACHE = pathlib.Path(__file__).parent / "upstream"


def fetch():
    CACHE.mkdir(exist_ok=True)
    for stock in STOCKS:
        dest = CACHE / f"{stock}.json"
        if dest.exists():
            continue
        branch = "dev" if stock in DEV_BRANCH else "main"
        print(f"fetching {stock} ({branch})", file=sys.stderr)
        urllib.request.urlretrieve(
            f"https://raw.githubusercontent.com/andreavolpato/spektrafilm/{branch}"
            f"/src/spektrafilm/data/profiles/{stock}.json",
            dest,
        )
    lic = CACHE / "SPEKTRAFILM_LICENSE.txt"
    if not lic.exists():
        urllib.request.urlretrieve(f"{UPSTREAM}/SPEKTRAFILM_LICENSE.txt", lic)


def num(v, places=6):
    """A null becomes an explicit 0, so the YAML has no holes to interpret."""
    if v is None or v != v:
        return "0"
    s = f"{v:.{places}g}"
    return "0" if s in ("-0", "-0.0") else s


def row(values, places=6):
    return "[" + ", ".join(num(v, places) for v in values) + "]"


def columns(rows, want=3):
    """Normalise a table to `want` columns per row.

    Two shapes arrive. A colour stock gives three columns and passes through.
    A black-and-white stock gives **one** -- it has one emulsion -- and is
    spread across all three, which is exact rather than approximate: three
    layers with identical sensitivity and identical curves respond identically,
    which is what one layer does.

    The dye is the exception and is handled by `split_dye`.
    """
    out = []
    for row in rows:
        if len(row) >= want:
            out.append(row[:want])
        else:
            out.append([row[0]] * want)
    return out


def split_dye(rows):
    """A single emulsion's dye, divided across the three layers.

    The renderer sums the three layers' contributions -- Beer-Lambert, so
    densities add -- and a monochrome stock has one dye, not three. Replicating
    it unchanged would treat one emulsion as three stacked copies of itself and
    render everything three times too dense. A third each reconstructs the
    single layer exactly, and stays correct anywhere the three densities differ,
    which is where the baked lookup interpolates.
    """
    out = []
    for row in rows:
        if len(row) >= 3:
            out.append(row[:3])
        else:
            v = row[0]
            out.append([None if v is None else v / 3.0] * 3)
    return out


def pick_development(rows, index):
    """One development time's column, for a stock measured at several."""
    return [[row[min(index, len(row) - 1)]] if isinstance(row, list) else [row] for row in rows]


def convert(stock):
    d = json.loads((CACHE / f"{stock}.json").read_text())
    info, data = d["info"], d["data"]
    monochrome = info.get("channel_model") == "bw"

    # A monochrome stock may be measured at several development times. Take one
    # column before anything else, so everything below sees the usual shape.
    if monochrome:
        curves = data["density_curves"]
        if curves and isinstance(curves[0], list) and len(curves[0]) > 1:
            data["density_curves"] = pick_development(curves, DEVELOPMENT_INDEX)
        base = data.get("base_density")
        if base and isinstance(base[0], list):
            data["base_density"] = [
                row[min(DEVELOPMENT_INDEX, len(row) - 1)] for row in base
            ]
    n = len(data["wavelengths"])
    assert data["wavelengths"][0] == 380.0 and data["wavelengths"][-1] == 780.0 and n == 81, (
        f"{stock}: unexpected wavelength grid; the conversion assumes 380-780nm at 5nm"
    )

    out = [
        "# Generated by tools/film-profiles/convert.py from spektrafilm.",
        "# Do not edit by hand: re-run the converter instead.",
        "#",
        "# spektrafilm by Andrea Volpato, https://github.com/andreavolpato/spektrafilm",
        "# Licensed CC BY-SA 4.0. Modified for DarkRoom: trimmed to the fields the",
        "# renderer uses and reformatted; see profiles/CHANGELOG.txt.",
        "",
        f"version: {d['metadata']['version']!r}",
        f"stock: {info['stock']}",
        f"name: {info['name']!r}",
        f"kind: {info['type']}          # negative | positive",
        f"support: {info['support']}    # film | paper",
        f"stage: {info['stage']}        # filming | printing",
        f"monochrome: {'true' if monochrome else 'false'}",
        f"reference_illuminant: {info['reference_illuminant']}",
        f"viewing_illuminant: {info['viewing_illuminant']}",
    ]
    if info.get("target_print"):
        out.append(f"target_print: {info['target_print']}")
    out += [
        "",
        "# log10 spectral sensitivity per layer, 380-780nm at 5nm, in R,G,B layer",
        "# order. A null upstream means the datasheet has no reading there, which is",
        "# blindness, so it is written as the sentinel the loader reads as such.",
        "log_sensitivity:",
    ]
    for wl, triple in zip(data["wavelengths"], columns(data["log_sensitivity"])):
        vals = ["-9" if (v is None or v != v) else num(v) for v in triple]
        out.append(f"  - [{', '.join(vals)}]   # {wl:.0f}nm")

    out += [
        "",
        "# Spectral density of each layer's dye at unit density, same grid and order.",
        "dye_density:",
    ]
    for wl, triple in zip(data["wavelengths"], split_dye(data["channel_density"])):
        out.append(f"  - {row(triple)}   # {wl:.0f}nm")

    base = data.get("base_density")
    out += [
        "",
        "# The support's own density -- film base plus, for a colour negative, the",
        "# orange mask. Flat zero where the datasheet does not give it.",
        f"base_density: {row(base) if base else row([0.0] * n)}",
        "",
        "# The characteristic curves: density against log10 exposure, sampled",
        f"# uniformly over [{data['log_exposure'][0]:g}, {data['log_exposure'][-1]:g}].",
        f"log_exposure_min: {num(data['log_exposure'][0])}",
        f"log_exposure_max: {num(data['log_exposure'][-1])}",
        "density_curves:",
    ]
    for triple in columns(data["density_curves"]):
        out.append(f"  - {row(triple, 5)}")
    return "\n".join(out) + "\n"


TABLE_HEADER = '''//! Generated by tools/film-profiles/convert.py. Do not edit.
//!
//! The fixed colour science: the observer, the illuminants and the spectral
//! basis. None of it is per-stock, all of it is published data, and together it
//! is under 6 kB of source -- which is the point. A film simulation's data cost
//! is dominated by whatever it uses to turn a pixel back into a spectrum, and a
//! basis is three curves where a coefficient table is megabytes.

/// The lowest wavelength sampled, in nanometres.
pub const LAMBDA_MIN: f32 = 380.0;
/// The spacing between samples, in nanometres.
pub const LAMBDA_STEP: f32 = 5.0;
/// How many wavelengths every spectral table carries.
///
/// The profiles, the observer and the basis all arrive on this grid already, so
/// nothing in this crate resamples anything.
pub const SPECTRUM: usize = 81;
'''


def emit_tables(dest):
    import numpy as np
    import colour

    grid = colour.SpectralShape(380, 780, 5)
    cmf = colour.MSDS_CMFS["CIE 1931 2 Degree Standard Observer"].copy().align(grid).values
    basis = colour.recovery.MSDS_BASIS_FUNCTIONS_sRGB_MALLETT2019.copy().align(grid).values

    def lit(v):
        """Always a float literal: `%g` renders an exact zero as `0`, which is
        an integer in Rust and will not compile in an `[f32; _]`."""
        s = f"{v:.8g}"
        return s if any(ch in s for ch in ".eE") else s + ".0"

    def table(name, doc, values):
        lines = [f"\n{doc}\npub static {name}: [[f32; 3]; SPECTRUM] = ["]
        for wl, triple in zip(grid.wavelengths, values):
            cells = ", ".join(lit(v) for v in triple)
            lines.append(f"    [{cells}],  // {wl:.0f}nm")
        lines.append("];")
        return "\n".join(lines)

    def flat(name, doc, values, wavelengths):
        lines = [f"\n{doc}\npub static {name}: [f32; SPECTRUM] = ["]
        for i in range(0, len(values), 6):
            chunk = ", ".join(lit(v) for v in values[i:i + 6])
            lines.append(f"    {chunk},")
        lines.append("];")
        return "\n".join(lines)

    parts = [TABLE_HEADER]
    parts.append(table(
        "OBSERVER",
        "/// CIE 1931 2-degree standard observer, x-bar/y-bar/z-bar per wavelength.",
        cmf,
    ))
    parts.append(table(
        "SRGB_BASIS",
        "/// Mallett & Yuksel (2019) sRGB reflectance basis: the three smooth,\n"
        "/// non-negative spectra that reconstruct any sRGB colour exactly.\n"
        "///\n"
        "/// This is what makes the exposure step a 3x3 matrix rather than a\n"
        "/// per-pixel spectral integration -- see [`crate::bake`].",
        basis,
    ))
    for name in ("D50", "D55", "D65"):
        sd = colour.SDS_ILLUMINANTS[name].copy().align(grid).values
        parts.append(flat(
            f"ILLUMINANT_{name}",
            f"/// CIE standard illuminant {name}, normalised to unit mean.",
            sd / sd.mean(),
            grid.wavelengths,
        ))
    dest.write_text("\n".join(parts) + "\n")


def main():
    ap = argparse.ArgumentParser()
    ap.add_argument("--fetch", action="store_true", help="download upstream profiles first")
    ap.add_argument(
        "--tables",
        action="store_true",
        help="also regenerate src/tables.rs (needs colour-science; the tables are "
        "fixed colour science and do not change when a stock is added)",
    )
    args = ap.parse_args()

    if args.fetch:
        fetch()
    if not CACHE.exists():
        sys.exit("no upstream cache; run with --fetch")

    PROFILE_DIR.mkdir(parents=True, exist_ok=True)
    for stock in STOCKS:
        dest = PROFILE_DIR / f"{stock}.yaml"
        dest.write_text(convert(stock))
        print(f"{dest.relative_to(ROOT)}  {dest.stat().st_size / 1024:.1f} kB")

    lic = CACHE / "SPEKTRAFILM_LICENSE.txt"
    if lic.exists():
        (PROFILE_DIR / "LICENSE-PROFILES.txt").write_text(lic.read_text())

    if args.tables:
        tables = ROOT / "core/dr-film/src/tables.rs"
        emit_tables(tables)
        print(f"{tables.relative_to(ROOT)}  {tables.stat().st_size / 1024:.1f} kB")

    registry = ROOT / "core/dr-film/src/built_in.rs"
    emit_registry(registry)
    print(f"{registry.relative_to(ROOT)}  {registry.stat().st_size / 1024:.1f} kB")


def emit_registry(dest):
    """The compiled-in stock list.

    Generated rather than hand-maintained, because it has to agree exactly with
    what is in `profiles/` -- a stock converted but never listed is a file that
    ships and cannot be chosen, which looks like a bug in the picker.
    """
    lines = [
        "//! Generated by tools/film-profiles/convert.py. Do not edit.",
        "//!",
        "//! The stocks compiled in as a floor. A floor rather than the whole",
        "//! story: a stock is a file, and the point of the format is that",
        "//! anyone can add one without a release.",
        "",
        "/// Each stock's id and its YAML, in the order the converter ran.",
        "pub static BUILT_IN: &[(&str, &str)] = &[",
    ]
    for stock in STOCKS:
        lines.append(f'    ("{stock}", include_str!("../profiles/{stock}.yaml")),')
    lines.append("];")
    dest.write_text("\n".join(lines) + "\n")


if __name__ == "__main__":
    main()