Merge branch 'feature/ci-images' into feature/opencv5

2026-08-04 14:53:24 +02:00
parent 22758da118 a3827646b9
commit 079b490ede
2 changed files with 460 additions and 0 deletions
@@ -0,0 +1,342 @@
 # sae-builder-cpu — the CI build image
 #
 # TRACES: DP-007 | PR-004
 #
 # Build/push:  scripts/ci/build_builder_image.sh --push
 # Consumed by: .gitea/workflows/unit-tests.yml  (pinned by tag, never :latest)
 # Docs:        docs/ci-image.md
 #
 # This is the CPU corner of the DP-008 builder matrix and the DP-007 CI image at
 # the same time — one artifact, two uses. The CUDA and ROCm siblings differ only
 # in the accelerator stack layered on top of this dependency set.
 #
 # CI runs on an Intel N100 with no discrete GPU. Everything here is chosen so
 # that `-DSAE_INFERENCE_BACKEND=ORT -DSAE_GEMM_BACKEND=CPU -DSAE_BUILD_TESTS=ON`
 # configures, builds and runs without a GPU, without a model, and without
 # reaching GitHub.
 # ─── Base image ──────────────────────────────────────────────────────────────
 #
 # Chosen for the OLDEST glibc to be supported, not for recency. A binary built
 # in a container runs against the *host's* glibc; glibc is backward compatible
 # but not forward, so the build base sets the floor for every machine DP-008's
 # binaries can ever run on. Building on a newer base than the oldest supported
 # host produces the classic `GLIBC_2.xx not found` failure at load time.
 #
 # Debian 12 "bookworm" = glibc 2.36 (Aug 2022). What that floor covers:
 #
 #   Distro                       glibc   Covered?
 #   Arch / CachyOS (rolling)     2.41+   yes
 #   Fedora 37 and later          2.36+   yes   ← DP-005's targets are Fedora+Arch
 #   Debian 12 / 13               2.36+   yes
 #   Ubuntu 24.04 LTS             2.39    yes
 #   Ubuntu 22.04 LTS             2.35    NO
 #   RHEL / Rocky / Alma 9        2.34    NO
 #   Debian 11                    2.31    NO
 #
 # The three misses are accepted deliberately: DP-005 puts Debian/Ubuntu out of
 # installer scope and names Fedora + Arch as the supported distros, and every
 # supported Fedora is 2.36 or newer. Going lower costs the toolchain rather than
 # buying reach — Debian 11 ships GCC 10 (incomplete C++20) and Python 3.9, which
 # has no `tomllib` and therefore cannot read the traceability gate's
 # traceability.toml.
 #
 # Escape hatch, recorded now so it is not rediscovered under pressure: if the
 # floor must drop to glibc 2.28 (RHEL 8 / manylinux_2_28 — the same baseline the
 # ONNX Runtime and PyTorch wheels target), the move is a Rocky 8 base plus
 # gcc-toolset-13, and OpenCV/FFmpeg/HDF5 all leave apt for source or
 # EPEL/RPM Fusion. That is a different image, not a flag on this one.
 #
 # Not a glibc problem but worth stating: the binaries this image produces also
 # link OpenCV, FFmpeg and HDF5 shared objects by soname. Making a *portable*
 # release binary (DP-008) is a separate question from the glibc floor, and is
 # answered by static linking or bundling, not by the base image.
 FROM debian:12-slim
 # Pins. Every version this image installs from source is an ARG so a rebuild is
 # a one-line diff and `docker history` records what a given tag actually holds.
 #
 # ORT 1.28.0 and OpenCV 5.0.0 match the developer machine, so CI and local
 # builds exercise the same libraries rather than merely similar ones.
 # Catch2 / nlohmann_json / nanobind match the FetchContent pins in
 # CMakeLists.txt:248 and tests/CMakeLists.txt:12 exactly — a vendored copy at a
 # different version would be a silent divergence, not a convenience.
 ARG ORT_VERSION=1.28.0
 ARG OPENCV_VERSION=5.0.0
 ARG CATCH2_VERSION=v3.5.3
 ARG NLOHMANN_JSON_VERSION=v3.11.3
 ARG NANOBIND_VERSION=v2.4.0
 # Stamped so a build can prove which image it ran in, and so a green tick can be
 # traced back to a specific dependency set. See the "Confirm the builder image"
 # step in .gitea/workflows/unit-tests.yml.
 ARG IMAGE_TAG=dev
 ENV SAE_BUILDER=cpu \
    SAE_BUILDER_VERSION=${IMAGE_TAG} \
    SAE_ORT_VERSION=${ORT_VERSION} \
    SAE_OPENCV_VERSION=${OPENCV_VERSION} \
    DEBIAN_FRONTEND=noninteractive
 # ─── System dependencies ─────────────────────────────────────────────────────
 #
 # One layer, ordered by why it is here rather than alphabetically.
 RUN apt-get update && apt-get install -y --no-install-recommends \
    # Toolchain. bookworm's default gcc is 12.2 — enough for the C++20 the
    # project sets unconditionally (CMakeLists.txt:4). cmake is 3.25, above the
    # 3.21 minimum. Ninja because the N100 has four cores and every second of
    # build scheduling shows.
    build-essential \
    cmake \
    ninja-build \
    pkg-config \
    git \
    ca-certificates \
    curl \
    # Gitea's act_runner executes JS actions (actions/checkout, upload-artifact)
    # with the `node` found *inside* the container. Without this the job cannot
    # even check the repository out. Same reason as the kpnpp-builder image.
    nodejs \
    # HDF5 with the C++ API: galleries are HDF5-native and it is also the VR-001
    # dump format. find_package(HDF5 COMPONENTS CXX) at CMakeLists.txt:273.
    libhdf5-dev \
    # FFmpeg decode. swresample is on this list deliberately: the audio
    # signature (IR-004) downmixes to mono and resamples to 11025 Hz, and
    # tests/test_audio_signature.cpp decodes the golden FLAC fixture, so the
    # test build needs it as much as the main build does.
    libavformat-dev \
    libavcodec-dev \
    libavutil-dev \
    libswscale-dev \
    libswresample-dev \
    # OpenBLAS — required here, not optional. CI has no GPU, so SAE_GEMM_BACKEND
    # =CPU is the only path it ever exercises, and without OpenBLAS the CPU GEMM
    # falls back to a scalar loop that does not scale against a library-sized
    # gallery (AR-027). The build only *warns* when it is missing so a developer
    # without it still gets a working tree; the image must never be that case.
    # Both the main build (CMakeLists.txt:162) and the test target
    # (tests/CMakeLists.txt:42) discover it through pkg-config `openblas`.
    libopenblas-dev \
    # Python: the build itself needs the interpreter and headers
    # (find_package(Python COMPONENTS Interpreter Development.Module) at
    # CMakeLists.txt:254, for the nanobind modules). numpy/h5py/scipy are for
    # the Python-side tooling — fixture generation, replay, validation scripts.
    # From apt rather than pip: bookworm marks the environment externally
    # managed (PEP 668), and apt's h5py is already linked against the same
    # libhdf5 installed above. bookworm's python3 is 3.11, which has tomllib —
    # the traceability gate needs it to read traceability.toml.
    python3 \
    python3-dev \
    python3-numpy \
    python3-h5py \
    python3-scipy \
    # Image codecs for the OpenCV build below. Without these OpenCV silently
    # builds an imgcodecs that cannot read a JPEG, which fails at run time in a
    # gallery build rather than at compile time here.
    libjpeg62-turbo-dev \
    libpng-dev \
    libtiff-dev \
    libwebp-dev \
    libopenjp2-7-dev \
    zlib1g-dev \
    && rm -rf /var/lib/apt/lists/*
 # Fail the image build, not the CI run, if OpenBLAS or swresample are not
 # discoverable the way CMakeLists.txt discovers them. An image that ships
 # libopenblas but no openblas.pc would compile the scalar fallback in silence.
 RUN set -eux; \
    pkg-config --exists openblas; \
    echo "openblas    $(pkg-config --modversion openblas)"; \
    pkg-config --exists libswresample; \
    echo "swresample  $(pkg-config --modversion libswresample)"
 # ─── ONNX Runtime, CPU provider only ─────────────────────────────────────────
 #
 # The official prebuilt linux-x64 tarball is the CPU build: no CUDA, no
 # TensorRT, no ROCm execution providers. That is the whole requirement here —
 # excluding the GPU providers is not a size optimisation, it is the point.
 #
 # Verified against the 1.28.0 tarball: the shared object's highest versioned
 # symbol requirement is GLIBC_2.27 / GLIBCXX_3.4.21, well under this base's
 # 2.36, so ORT does not raise the floor set above.
 #
 # Installed to /usr/local/{lib,include/onnxruntime} because CMakeLists.txt
 # includes <onnxruntime/onnxruntime_cxx_api.h> and needs the *parent* of that
 # directory on the include path (CMakeLists.txt:108-113).
 #
 # CI never calls a model — the embedder measures ~930 ms/frame on this CPU
 # provider — so ORT is present to satisfy the link, not to run inference.
 RUN set -eux; \
    curl -fsSL -o /tmp/ort.tgz \
      "https://github.com/microsoft/onnxruntime/releases/download/v${ORT_VERSION}/onnxruntime-linux-x64-${ORT_VERSION}.tgz"; \
    mkdir -p /tmp/ort; \
    tar -xzf /tmp/ort.tgz -C /tmp/ort --strip-components=1; \
    cp -a /tmp/ort/lib/libonnxruntime.so* /usr/local/lib/; \
    mkdir -p /usr/local/include/onnxruntime; \
    cp -a /tmp/ort/include/. /usr/local/include/onnxruntime/; \
    ldconfig; \
    rm -rf /tmp/ort /tmp/ort.tgz; \
    test -f /usr/local/include/onnxruntime/onnxruntime_cxx_api.h
 # ─── OpenCV 5, from source ───────────────────────────────────────────────────
 #
 # This is the reason the image is prebuilt at all. CMakeLists.txt:25 probes for
 # OpenCV 5 first and falls back to 4; the branch targets 5, which no Debian
 # release ships (bookworm has 4.6), and building it inside every CI run would
 # dominate the run on an N100.
 #
 # BUILD_LIST is exactly the seven components find_package asks for
 # (CMakeLists.txt:25-29) — OpenCV resolves their internal dependencies itself.
 # Everything else is off: tests, samples, Java/Python bindings, and the apps.
 #
 # No GUI backend. highgui still builds (find_package REQUIREs the component) but
 # with a stub — CI never calls imshow, and pulling GTK/Qt into a headless build
 # image buys nothing. scene_preview is a developer tool, not a CI target.
 #
 # CUDA/cuDNN explicitly off: DP-007 excludes the GPU stack outright.
 #
 # The source tree and build tree are removed in the same layer, so the ~3 GB of
 # intermediates cost nothing in the published image.
 RUN set -eux; \
    curl -fsSL -o /tmp/opencv.tar.gz \
      "https://github.com/opencv/opencv/archive/refs/tags/${OPENCV_VERSION}.tar.gz"; \
    mkdir -p /tmp/opencv-src; \
    tar -xzf /tmp/opencv.tar.gz -C /tmp/opencv-src --strip-components=1; \
    cmake -S /tmp/opencv-src -B /tmp/opencv-build -G Ninja \
      -DCMAKE_BUILD_TYPE=Release \
      -DCMAKE_INSTALL_PREFIX=/usr/local \
      -DBUILD_LIST=core,imgproc,imgcodecs,videoio,dnn,objdetect,highgui \
      -DBUILD_SHARED_LIBS=ON \
      -DBUILD_TESTS=OFF \
      -DBUILD_PERF_TESTS=OFF \
      -DBUILD_EXAMPLES=OFF \
      -DBUILD_DOCS=OFF \
      -DBUILD_opencv_apps=OFF \
      -DBUILD_JAVA=OFF \
      -DBUILD_opencv_python3=OFF \
      -DWITH_FFMPEG=ON \
      -DWITH_GTK=OFF \
      -DWITH_QT=OFF \
      -DWITH_OPENGL=OFF \
      -DWITH_CUDA=OFF \
      -DWITH_CUDNN=OFF \
      -DOPENCV_GENERATE_PKGCONFIG=ON \
      -DCMAKE_INSTALL_RPATH=/usr/local/lib; \
    cmake --build /tmp/opencv-build --parallel; \
    cmake --install /tmp/opencv-build; \
    ldconfig; \
    rm -rf /tmp/opencv-src /tmp/opencv-build /tmp/opencv.tar.gz
 # ─── Vendored dependencies: Catch2, nlohmann/json, nanobind ──────────────────
 #
 # All three are FetchContent'ed by the build today, which makes every CI run
 # depend on GitHub being reachable — a network outage would present as a code
 # failure. Baking them in removes that dependency entirely.
 #
 # Catch2 is *installed*, so tests/CMakeLists.txt:6 `find_package(Catch2 3 QUIET)`
 # succeeds and the FetchContent fallback is never reached. Its source is kept as
 # well so the override below can cover the case where find_package somehow does
 # not fire.
 #
 # nanobind must be cloned with submodules: its `ext/robin_map` is a git
 # submodule, and a GitHub source tarball does not contain it. This is the one
 # dependency where "download the tarball" produces a tree that configures and
 # then fails to compile.
 RUN set -eux; \
    mkdir -p /opt/vendor; \
    git clone --depth 1 --branch "${NLOHMANN_JSON_VERSION}" \
      https://github.com/nlohmann/json.git /opt/vendor/nlohmann_json; \
    git clone --depth 1 --branch "${NANOBIND_VERSION}" --recurse-submodules \
      https://github.com/wjakob/nanobind.git /opt/vendor/nanobind; \
    git clone --depth 1 --branch "${CATCH2_VERSION}" \
      https://github.com/catchorg/Catch2.git /opt/vendor/Catch2; \
    cmake -S /opt/vendor/Catch2 -B /tmp/catch2-build -G Ninja \
      -DCMAKE_BUILD_TYPE=Release \
      -DCMAKE_INSTALL_PREFIX=/usr/local \
      -DBUILD_TESTING=OFF; \
    cmake --build /tmp/catch2-build --parallel; \
    cmake --install /tmp/catch2-build; \
    rm -rf /tmp/catch2-build; \
    find /opt/vendor -maxdepth 2 -name .git -exec rm -rf {} +; \
    ldconfig
 # The initial-cache script the build is configured with. It lives in the image,
 # not in the workflow, so the vendor paths have exactly one owner: move a
 # directory here and no consumer needs editing.
 #
 # FETCHCONTENT_FULLY_DISCONNECTED=ON is the load-bearing line. With it, any
 # FetchContent dependency that is *not* covered by an override above is a hard
 # configure error instead of a silent download — so "this build does not touch
 # GitHub" is enforced by the build system rather than asserted in a comment.
 RUN set -eux; \
    printf '%s\n' \
      '# Baked into sae-builder-cpu. Use with: cmake -C /opt/vendor/vendored-deps.cmake ...' \
      '# TRACES: DP-007' \
      'set(FETCHCONTENT_SOURCE_DIR_NLOHMANN_JSON "/opt/vendor/nlohmann_json" CACHE PATH "vendored in the CI image")' \
      'set(FETCHCONTENT_SOURCE_DIR_NANOBIND      "/opt/vendor/nanobind"      CACHE PATH "vendored in the CI image")' \
      'set(FETCHCONTENT_SOURCE_DIR_CATCH2        "/opt/vendor/Catch2"        CACHE PATH "vendored in the CI image")' \
      'set(FETCHCONTENT_FULLY_DISCONNECTED       ON  CACHE BOOL "no CI build may fetch from the network")' \
      > /opt/vendor/vendored-deps.cmake; \
    cat /opt/vendor/vendored-deps.cmake
 # ─── Self-check ──────────────────────────────────────────────────────────────
 #
 # Run the project's own dependency discovery — the same find_package and
 # pkg_check_modules calls CMakeLists.txt makes — against this image, at image
 # build time. An image that cannot satisfy them should fail here, loudly, once,
 # rather than in every CI run that pulls it.
 #
 # Deliberately not a build of the project: the image must be buildable without
 # the repository, and the repository's own configure step is what CI is for.
 RUN set -eux; \
    mkdir -p /tmp/selfcheck; \
    printf '%s\n' \
      'cmake_minimum_required(VERSION 3.21)' \
      'project(sae_image_selfcheck LANGUAGES CXX)' \
      'set(CMAKE_CXX_STANDARD 20)' \
      'set(CMAKE_CXX_STANDARD_REQUIRED ON)' \
      'find_package(OpenCV 5 REQUIRED COMPONENTS core imgproc imgcodecs videoio dnn objdetect highgui)' \
      'message(STATUS "OpenCV ${OpenCV_VERSION}")' \
      'find_package(HDF5 REQUIRED COMPONENTS CXX)' \
      'message(STATUS "HDF5 ${HDF5_VERSION}")' \
      'find_package(Catch2 3 REQUIRED)' \
      'message(STATUS "Catch2 ${Catch2_VERSION}")' \
      'find_package(Python 3.8 REQUIRED COMPONENTS Interpreter Development.Module)' \
      'find_package(PkgConfig REQUIRED)' \
      'pkg_check_modules(AVFORMAT   REQUIRED libavformat)' \
      'pkg_check_modules(AVCODEC    REQUIRED libavcodec)' \
      'pkg_check_modules(AVUTIL     REQUIRED libavutil)' \
      'pkg_check_modules(SWSCALE    REQUIRED libswscale)' \
      'pkg_check_modules(SWRESAMPLE REQUIRED libswresample)' \
      'pkg_check_modules(OPENBLAS   REQUIRED openblas)' \
      'find_library(ORT_LIB onnxruntime REQUIRED HINTS /usr/lib /usr/local/lib)' \
      'find_path(ORT_INCLUDE onnxruntime_cxx_api.h PATH_SUFFIXES onnxruntime' \
      '          HINTS /usr/include/onnxruntime /usr/local/include/onnxruntime /usr/local/include REQUIRED)' \
      'message(STATUS "ORT ${ORT_LIB} / ${ORT_INCLUDE}")' \
      > /tmp/selfcheck/CMakeLists.txt; \
    cmake -S /tmp/selfcheck -B /tmp/selfcheck/build -G Ninja; \
    rm -rf /tmp/selfcheck
 # Python-side tooling the fixture and validation scripts import. Checked here so
 # a missing wheel is an image failure rather than a mid-run traceback.
 RUN python3 -c "import numpy, h5py, scipy; print('numpy', numpy.__version__, 'h5py', h5py.__version__, 'scipy', scipy.__version__)"
 # ─── What is deliberately NOT here ───────────────────────────────────────────
 #
 #   CUDA, TensorRT, ROCm, and the ORT GPU execution providers
 #       No GPU to use them. They belong to the sae-builder-cuda and
 #       sae-builder-rocm siblings (DP-008).
 #
 #   The ONNX models
 #       Seven files, ~725 MB, in Git LFS. T1/T2 tests are model-free by design
 #       (tests/CMakeLists.txt:1-4), so the CI image needs none of them, and
 #       baking them in would inflate the image roughly tenfold to serve the T3
 #       smoke tests alone. Those pull the model they need via LFS in a separate
 #       job. The CI workflow checks out with LFS off for the same reason.
 #
 #   The repository
 #       Nothing from the source tree is COPYed in. The image is a toolchain, and
 #       a toolchain that embeds the code it builds has to be rebuilt whenever the
 #       code changes — which is exactly the per-run cost this image exists to
 #       avoid.
 WORKDIR /src
@@ -0,0 +1,118 @@
 #!/bin/bash
 # build_builder_image.sh — build and publish the DP-007 CI builder image to the
 # Gitea container registry.
 #
 # TRACES: DP-007 | PR-004
 #
 # Usage:
 #   scripts/ci/build_builder_image.sh                  # build only, tag v1
 #   scripts/ci/build_builder_image.sh --push           # build and push
 #   scripts/ci/build_builder_image.sh --tag v2 --push  # bump the pinned tag
 #   scripts/ci/build_builder_image.sh --no-cache       # force a clean rebuild
 #
 # The tag is the contract with CI. .gitea/workflows/unit-tests.yml names an
 # explicit tag in its `container:` block and never `latest`, so that rebuilding
 # the image cannot silently change what a previous green build meant. Bumping
 # the dependency set means bumping the tag AND editing the workflow — the two
 # edits landing in the same commit is the point, not an inconvenience.
 #
 # Registry auth: this script does not log in. Do it once, out of band:
 #   docker login gitea.tourolle.paris
 # The CI host is already authenticated this way (its cached credentials in
 # ~/.docker/config.json are what the kpnpp-builder push relies on), so a
 # workflow that calls this script needs no secret plumbing.
 set -euo pipefail
 REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
 REGISTRY="gitea.tourolle.paris"
 OWNER="dtourolle"
 IMAGE="sae-builder-cpu"
 DOCKERFILE="Dockerfile.builder-cpu"
 # The tag CI pins to today. Keep this in step with the `container.image` line in
 # .gitea/workflows/unit-tests.yml; the workflow asserts at run time that the
 # image it landed in reports this same version, so a drift shows up as a failed
 # job rather than as a build against the wrong toolchain.
 TAG="v1"
 PUSH=0
 EXTRA_ARGS=()
 while [ $# -gt 0 ]; do
    case "$1" in
        --push)     PUSH=1 ;;
        --tag)      TAG="${2:?--tag needs a value}"; shift ;;
        --no-cache) EXTRA_ARGS+=(--no-cache) ;;
        -h|--help)  sed -n '2,30p' "${BASH_SOURCE[0]}"; exit 0 ;;
        *)          echo "error: unknown argument '$1'" >&2; exit 2 ;;
    esac
    shift
 done
 if [ "$TAG" = "latest" ]; then
    echo "error: refusing to build the tag 'latest'." >&2
    echo "DP-007 requires CI to pin an immutable tag. A moving 'latest' means a" >&2
    echo "rebuild retroactively changes what every earlier green build proved." >&2
    exit 2
 fi
 REF="${REGISTRY}/${OWNER}/${IMAGE}:${TAG}"
 # A second tag carrying the commit that produced the image. The workflow pins
 # the human-readable tag; this one is the audit trail — given any image you can
 # recover the Dockerfile that built it.
 SHA="$(git -C "$REPO_ROOT" rev-parse --short HEAD)"
 REF_SHA="${REGISTRY}/${OWNER}/${IMAGE}:${TAG}-${SHA}"
 # The Dockerfile COPYs nothing from the repository on purpose (see its closing
 # comment), so the build context is an empty directory rather than the repo
 # root. Sending ~1 GB of models, fixtures and experiment data to the daemon for
 # a build that reads none of it is pure latency.
 CONTEXT="$(mktemp -d)"
 trap 'rm -rf "$CONTEXT"' EXIT
 echo "=== building ${REF}"
 echo "    dockerfile: ${REPO_ROOT}/${DOCKERFILE}"
 echo "    context:    (empty — the image embeds no repository content)"
 echo
 echo "    Expect this to take a while: OpenCV 5 is compiled from source because"
 echo "    no Debian release ships it. That cost is paid once per image, which is"
 echo "    the entire reason DP-007 asks for a prebuilt image instead of"
 echo "    installing dependencies inside each CI run."
 echo
 docker build \
    "${EXTRA_ARGS[@]}" \
    --build-arg "IMAGE_TAG=${TAG}" \
    -f "${REPO_ROOT}/${DOCKERFILE}" \
    -t "${REF}" \
    -t "${REF_SHA}" \
    "${CONTEXT}"
 echo
 echo "=== built"
 docker image inspect "${REF}" --format '    {{.RepoTags}}  {{.Size}} bytes'
 docker run --rm "${REF}" sh -c 'echo "    SAE_BUILDER=$SAE_BUILDER version=$SAE_BUILDER_VERSION ort=$SAE_ORT_VERSION opencv=$SAE_OPENCV_VERSION"'
 if [ "$PUSH" -eq 0 ]; then
    echo
    echo "Not pushed. Re-run with --push, or push by hand:"
    echo "    docker push ${REF}"
    echo "    docker push ${REF_SHA}"
    exit 0
 fi
 echo
 echo "=== pushing"
 # No `latest` tag is pushed, by design. Publishing one invites a workflow to use
 # it, and DP-007 exists to prevent exactly that.
 docker push "${REF}"
 docker push "${REF_SHA}"
 echo
 echo "=== published ${REF}"
 echo "If this was a dependency-set change, bump the tag in"
 echo "    .gitea/workflows/unit-tests.yml   (container.image)"
 echo "    scripts/ci/build_builder_image.sh (TAG, above)"
 echo "in the same commit, so no run can build against an image the repository"
 echo "does not describe."