From a3827646b97b9bcf1056e495347767882199fcc7 Mon Sep 17 00:00:00 2001 From: Duncan Tourolle Date: Tue, 4 Aug 2026 14:53:12 +0200 Subject: [PATCH] feat(ci): CPU builder image for the unit-test workflow TRACES: DP-007 | PR-004 Pinned by tag rather than :latest, so a workflow run is reproducible against the image it was written for. --- Dockerfile.builder-cpu | 342 ++++++++++++++++++++++++++++++ scripts/ci/build_builder_image.sh | 118 +++++++++++ 2 files changed, 460 insertions(+) create mode 100644 Dockerfile.builder-cpu create mode 100644 scripts/ci/build_builder_image.sh diff --git a/Dockerfile.builder-cpu b/Dockerfile.builder-cpu new file mode 100644 index 0000000..a81cc54 --- /dev/null +++ b/Dockerfile.builder-cpu @@ -0,0 +1,342 @@ +# sae-builder-cpu — the CI build image +# +# TRACES: DP-007 | PR-004 +# +# Build/push: scripts/ci/build_builder_image.sh --push +# Consumed by: .gitea/workflows/unit-tests.yml (pinned by tag, never :latest) +# Docs: docs/ci-image.md +# +# This is the CPU corner of the DP-008 builder matrix and the DP-007 CI image at +# the same time — one artifact, two uses. The CUDA and ROCm siblings differ only +# in the accelerator stack layered on top of this dependency set. +# +# CI runs on an Intel N100 with no discrete GPU. Everything here is chosen so +# that `-DSAE_INFERENCE_BACKEND=ORT -DSAE_GEMM_BACKEND=CPU -DSAE_BUILD_TESTS=ON` +# configures, builds and runs without a GPU, without a model, and without +# reaching GitHub. + +# ─── Base image ────────────────────────────────────────────────────────────── +# +# Chosen for the OLDEST glibc to be supported, not for recency. A binary built +# in a container runs against the *host's* glibc; glibc is backward compatible +# but not forward, so the build base sets the floor for every machine DP-008's +# binaries can ever run on. Building on a newer base than the oldest supported +# host produces the classic `GLIBC_2.xx not found` failure at load time. +# +# Debian 12 "bookworm" = glibc 2.36 (Aug 2022). What that floor covers: +# +# Distro glibc Covered? +# Arch / CachyOS (rolling) 2.41+ yes +# Fedora 37 and later 2.36+ yes ← DP-005's targets are Fedora+Arch +# Debian 12 / 13 2.36+ yes +# Ubuntu 24.04 LTS 2.39 yes +# Ubuntu 22.04 LTS 2.35 NO +# RHEL / Rocky / Alma 9 2.34 NO +# Debian 11 2.31 NO +# +# The three misses are accepted deliberately: DP-005 puts Debian/Ubuntu out of +# installer scope and names Fedora + Arch as the supported distros, and every +# supported Fedora is 2.36 or newer. Going lower costs the toolchain rather than +# buying reach — Debian 11 ships GCC 10 (incomplete C++20) and Python 3.9, which +# has no `tomllib` and therefore cannot read the traceability gate's +# traceability.toml. +# +# Escape hatch, recorded now so it is not rediscovered under pressure: if the +# floor must drop to glibc 2.28 (RHEL 8 / manylinux_2_28 — the same baseline the +# ONNX Runtime and PyTorch wheels target), the move is a Rocky 8 base plus +# gcc-toolset-13, and OpenCV/FFmpeg/HDF5 all leave apt for source or +# EPEL/RPM Fusion. That is a different image, not a flag on this one. +# +# Not a glibc problem but worth stating: the binaries this image produces also +# link OpenCV, FFmpeg and HDF5 shared objects by soname. Making a *portable* +# release binary (DP-008) is a separate question from the glibc floor, and is +# answered by static linking or bundling, not by the base image. +FROM debian:12-slim + +# Pins. Every version this image installs from source is an ARG so a rebuild is +# a one-line diff and `docker history` records what a given tag actually holds. +# +# ORT 1.28.0 and OpenCV 5.0.0 match the developer machine, so CI and local +# builds exercise the same libraries rather than merely similar ones. +# Catch2 / nlohmann_json / nanobind match the FetchContent pins in +# CMakeLists.txt:248 and tests/CMakeLists.txt:12 exactly — a vendored copy at a +# different version would be a silent divergence, not a convenience. +ARG ORT_VERSION=1.28.0 +ARG OPENCV_VERSION=5.0.0 +ARG CATCH2_VERSION=v3.5.3 +ARG NLOHMANN_JSON_VERSION=v3.11.3 +ARG NANOBIND_VERSION=v2.4.0 + +# Stamped so a build can prove which image it ran in, and so a green tick can be +# traced back to a specific dependency set. See the "Confirm the builder image" +# step in .gitea/workflows/unit-tests.yml. +ARG IMAGE_TAG=dev +ENV SAE_BUILDER=cpu \ + SAE_BUILDER_VERSION=${IMAGE_TAG} \ + SAE_ORT_VERSION=${ORT_VERSION} \ + SAE_OPENCV_VERSION=${OPENCV_VERSION} \ + DEBIAN_FRONTEND=noninteractive + +# ─── System dependencies ───────────────────────────────────────────────────── +# +# One layer, ordered by why it is here rather than alphabetically. +RUN apt-get update && apt-get install -y --no-install-recommends \ + # Toolchain. bookworm's default gcc is 12.2 — enough for the C++20 the + # project sets unconditionally (CMakeLists.txt:4). cmake is 3.25, above the + # 3.21 minimum. Ninja because the N100 has four cores and every second of + # build scheduling shows. + build-essential \ + cmake \ + ninja-build \ + pkg-config \ + git \ + ca-certificates \ + curl \ + # Gitea's act_runner executes JS actions (actions/checkout, upload-artifact) + # with the `node` found *inside* the container. Without this the job cannot + # even check the repository out. Same reason as the kpnpp-builder image. + nodejs \ + # HDF5 with the C++ API: galleries are HDF5-native and it is also the VR-001 + # dump format. find_package(HDF5 COMPONENTS CXX) at CMakeLists.txt:273. + libhdf5-dev \ + # FFmpeg decode. swresample is on this list deliberately: the audio + # signature (IR-004) downmixes to mono and resamples to 11025 Hz, and + # tests/test_audio_signature.cpp decodes the golden FLAC fixture, so the + # test build needs it as much as the main build does. + libavformat-dev \ + libavcodec-dev \ + libavutil-dev \ + libswscale-dev \ + libswresample-dev \ + # OpenBLAS — required here, not optional. CI has no GPU, so SAE_GEMM_BACKEND + # =CPU is the only path it ever exercises, and without OpenBLAS the CPU GEMM + # falls back to a scalar loop that does not scale against a library-sized + # gallery (AR-027). The build only *warns* when it is missing so a developer + # without it still gets a working tree; the image must never be that case. + # Both the main build (CMakeLists.txt:162) and the test target + # (tests/CMakeLists.txt:42) discover it through pkg-config `openblas`. + libopenblas-dev \ + # Python: the build itself needs the interpreter and headers + # (find_package(Python COMPONENTS Interpreter Development.Module) at + # CMakeLists.txt:254, for the nanobind modules). numpy/h5py/scipy are for + # the Python-side tooling — fixture generation, replay, validation scripts. + # From apt rather than pip: bookworm marks the environment externally + # managed (PEP 668), and apt's h5py is already linked against the same + # libhdf5 installed above. bookworm's python3 is 3.11, which has tomllib — + # the traceability gate needs it to read traceability.toml. + python3 \ + python3-dev \ + python3-numpy \ + python3-h5py \ + python3-scipy \ + # Image codecs for the OpenCV build below. Without these OpenCV silently + # builds an imgcodecs that cannot read a JPEG, which fails at run time in a + # gallery build rather than at compile time here. + libjpeg62-turbo-dev \ + libpng-dev \ + libtiff-dev \ + libwebp-dev \ + libopenjp2-7-dev \ + zlib1g-dev \ + && rm -rf /var/lib/apt/lists/* + +# Fail the image build, not the CI run, if OpenBLAS or swresample are not +# discoverable the way CMakeLists.txt discovers them. An image that ships +# libopenblas but no openblas.pc would compile the scalar fallback in silence. +RUN set -eux; \ + pkg-config --exists openblas; \ + echo "openblas $(pkg-config --modversion openblas)"; \ + pkg-config --exists libswresample; \ + echo "swresample $(pkg-config --modversion libswresample)" + +# ─── ONNX Runtime, CPU provider only ───────────────────────────────────────── +# +# The official prebuilt linux-x64 tarball is the CPU build: no CUDA, no +# TensorRT, no ROCm execution providers. That is the whole requirement here — +# excluding the GPU providers is not a size optimisation, it is the point. +# +# Verified against the 1.28.0 tarball: the shared object's highest versioned +# symbol requirement is GLIBC_2.27 / GLIBCXX_3.4.21, well under this base's +# 2.36, so ORT does not raise the floor set above. +# +# Installed to /usr/local/{lib,include/onnxruntime} because CMakeLists.txt +# includes and needs the *parent* of that +# directory on the include path (CMakeLists.txt:108-113). +# +# CI never calls a model — the embedder measures ~930 ms/frame on this CPU +# provider — so ORT is present to satisfy the link, not to run inference. +RUN set -eux; \ + curl -fsSL -o /tmp/ort.tgz \ + "https://github.com/microsoft/onnxruntime/releases/download/v${ORT_VERSION}/onnxruntime-linux-x64-${ORT_VERSION}.tgz"; \ + mkdir -p /tmp/ort; \ + tar -xzf /tmp/ort.tgz -C /tmp/ort --strip-components=1; \ + cp -a /tmp/ort/lib/libonnxruntime.so* /usr/local/lib/; \ + mkdir -p /usr/local/include/onnxruntime; \ + cp -a /tmp/ort/include/. /usr/local/include/onnxruntime/; \ + ldconfig; \ + rm -rf /tmp/ort /tmp/ort.tgz; \ + test -f /usr/local/include/onnxruntime/onnxruntime_cxx_api.h + +# ─── OpenCV 5, from source ─────────────────────────────────────────────────── +# +# This is the reason the image is prebuilt at all. CMakeLists.txt:25 probes for +# OpenCV 5 first and falls back to 4; the branch targets 5, which no Debian +# release ships (bookworm has 4.6), and building it inside every CI run would +# dominate the run on an N100. +# +# BUILD_LIST is exactly the seven components find_package asks for +# (CMakeLists.txt:25-29) — OpenCV resolves their internal dependencies itself. +# Everything else is off: tests, samples, Java/Python bindings, and the apps. +# +# No GUI backend. highgui still builds (find_package REQUIREs the component) but +# with a stub — CI never calls imshow, and pulling GTK/Qt into a headless build +# image buys nothing. scene_preview is a developer tool, not a CI target. +# +# CUDA/cuDNN explicitly off: DP-007 excludes the GPU stack outright. +# +# The source tree and build tree are removed in the same layer, so the ~3 GB of +# intermediates cost nothing in the published image. +RUN set -eux; \ + curl -fsSL -o /tmp/opencv.tar.gz \ + "https://github.com/opencv/opencv/archive/refs/tags/${OPENCV_VERSION}.tar.gz"; \ + mkdir -p /tmp/opencv-src; \ + tar -xzf /tmp/opencv.tar.gz -C /tmp/opencv-src --strip-components=1; \ + cmake -S /tmp/opencv-src -B /tmp/opencv-build -G Ninja \ + -DCMAKE_BUILD_TYPE=Release \ + -DCMAKE_INSTALL_PREFIX=/usr/local \ + -DBUILD_LIST=core,imgproc,imgcodecs,videoio,dnn,objdetect,highgui \ + -DBUILD_SHARED_LIBS=ON \ + -DBUILD_TESTS=OFF \ + -DBUILD_PERF_TESTS=OFF \ + -DBUILD_EXAMPLES=OFF \ + -DBUILD_DOCS=OFF \ + -DBUILD_opencv_apps=OFF \ + -DBUILD_JAVA=OFF \ + -DBUILD_opencv_python3=OFF \ + -DWITH_FFMPEG=ON \ + -DWITH_GTK=OFF \ + -DWITH_QT=OFF \ + -DWITH_OPENGL=OFF \ + -DWITH_CUDA=OFF \ + -DWITH_CUDNN=OFF \ + -DOPENCV_GENERATE_PKGCONFIG=ON \ + -DCMAKE_INSTALL_RPATH=/usr/local/lib; \ + cmake --build /tmp/opencv-build --parallel; \ + cmake --install /tmp/opencv-build; \ + ldconfig; \ + rm -rf /tmp/opencv-src /tmp/opencv-build /tmp/opencv.tar.gz + +# ─── Vendored dependencies: Catch2, nlohmann/json, nanobind ────────────────── +# +# All three are FetchContent'ed by the build today, which makes every CI run +# depend on GitHub being reachable — a network outage would present as a code +# failure. Baking them in removes that dependency entirely. +# +# Catch2 is *installed*, so tests/CMakeLists.txt:6 `find_package(Catch2 3 QUIET)` +# succeeds and the FetchContent fallback is never reached. Its source is kept as +# well so the override below can cover the case where find_package somehow does +# not fire. +# +# nanobind must be cloned with submodules: its `ext/robin_map` is a git +# submodule, and a GitHub source tarball does not contain it. This is the one +# dependency where "download the tarball" produces a tree that configures and +# then fails to compile. +RUN set -eux; \ + mkdir -p /opt/vendor; \ + git clone --depth 1 --branch "${NLOHMANN_JSON_VERSION}" \ + https://github.com/nlohmann/json.git /opt/vendor/nlohmann_json; \ + git clone --depth 1 --branch "${NANOBIND_VERSION}" --recurse-submodules \ + https://github.com/wjakob/nanobind.git /opt/vendor/nanobind; \ + git clone --depth 1 --branch "${CATCH2_VERSION}" \ + https://github.com/catchorg/Catch2.git /opt/vendor/Catch2; \ + cmake -S /opt/vendor/Catch2 -B /tmp/catch2-build -G Ninja \ + -DCMAKE_BUILD_TYPE=Release \ + -DCMAKE_INSTALL_PREFIX=/usr/local \ + -DBUILD_TESTING=OFF; \ + cmake --build /tmp/catch2-build --parallel; \ + cmake --install /tmp/catch2-build; \ + rm -rf /tmp/catch2-build; \ + find /opt/vendor -maxdepth 2 -name .git -exec rm -rf {} +; \ + ldconfig + +# The initial-cache script the build is configured with. It lives in the image, +# not in the workflow, so the vendor paths have exactly one owner: move a +# directory here and no consumer needs editing. +# +# FETCHCONTENT_FULLY_DISCONNECTED=ON is the load-bearing line. With it, any +# FetchContent dependency that is *not* covered by an override above is a hard +# configure error instead of a silent download — so "this build does not touch +# GitHub" is enforced by the build system rather than asserted in a comment. +RUN set -eux; \ + printf '%s\n' \ + '# Baked into sae-builder-cpu. Use with: cmake -C /opt/vendor/vendored-deps.cmake ...' \ + '# TRACES: DP-007' \ + 'set(FETCHCONTENT_SOURCE_DIR_NLOHMANN_JSON "/opt/vendor/nlohmann_json" CACHE PATH "vendored in the CI image")' \ + 'set(FETCHCONTENT_SOURCE_DIR_NANOBIND "/opt/vendor/nanobind" CACHE PATH "vendored in the CI image")' \ + 'set(FETCHCONTENT_SOURCE_DIR_CATCH2 "/opt/vendor/Catch2" CACHE PATH "vendored in the CI image")' \ + 'set(FETCHCONTENT_FULLY_DISCONNECTED ON CACHE BOOL "no CI build may fetch from the network")' \ + > /opt/vendor/vendored-deps.cmake; \ + cat /opt/vendor/vendored-deps.cmake + +# ─── Self-check ────────────────────────────────────────────────────────────── +# +# Run the project's own dependency discovery — the same find_package and +# pkg_check_modules calls CMakeLists.txt makes — against this image, at image +# build time. An image that cannot satisfy them should fail here, loudly, once, +# rather than in every CI run that pulls it. +# +# Deliberately not a build of the project: the image must be buildable without +# the repository, and the repository's own configure step is what CI is for. +RUN set -eux; \ + mkdir -p /tmp/selfcheck; \ + printf '%s\n' \ + 'cmake_minimum_required(VERSION 3.21)' \ + 'project(sae_image_selfcheck LANGUAGES CXX)' \ + 'set(CMAKE_CXX_STANDARD 20)' \ + 'set(CMAKE_CXX_STANDARD_REQUIRED ON)' \ + 'find_package(OpenCV 5 REQUIRED COMPONENTS core imgproc imgcodecs videoio dnn objdetect highgui)' \ + 'message(STATUS "OpenCV ${OpenCV_VERSION}")' \ + 'find_package(HDF5 REQUIRED COMPONENTS CXX)' \ + 'message(STATUS "HDF5 ${HDF5_VERSION}")' \ + 'find_package(Catch2 3 REQUIRED)' \ + 'message(STATUS "Catch2 ${Catch2_VERSION}")' \ + 'find_package(Python 3.8 REQUIRED COMPONENTS Interpreter Development.Module)' \ + 'find_package(PkgConfig REQUIRED)' \ + 'pkg_check_modules(AVFORMAT REQUIRED libavformat)' \ + 'pkg_check_modules(AVCODEC REQUIRED libavcodec)' \ + 'pkg_check_modules(AVUTIL REQUIRED libavutil)' \ + 'pkg_check_modules(SWSCALE REQUIRED libswscale)' \ + 'pkg_check_modules(SWRESAMPLE REQUIRED libswresample)' \ + 'pkg_check_modules(OPENBLAS REQUIRED openblas)' \ + 'find_library(ORT_LIB onnxruntime REQUIRED HINTS /usr/lib /usr/local/lib)' \ + 'find_path(ORT_INCLUDE onnxruntime_cxx_api.h PATH_SUFFIXES onnxruntime' \ + ' HINTS /usr/include/onnxruntime /usr/local/include/onnxruntime /usr/local/include REQUIRED)' \ + 'message(STATUS "ORT ${ORT_LIB} / ${ORT_INCLUDE}")' \ + > /tmp/selfcheck/CMakeLists.txt; \ + cmake -S /tmp/selfcheck -B /tmp/selfcheck/build -G Ninja; \ + rm -rf /tmp/selfcheck + +# Python-side tooling the fixture and validation scripts import. Checked here so +# a missing wheel is an image failure rather than a mid-run traceback. +RUN python3 -c "import numpy, h5py, scipy; print('numpy', numpy.__version__, 'h5py', h5py.__version__, 'scipy', scipy.__version__)" + +# ─── What is deliberately NOT here ─────────────────────────────────────────── +# +# CUDA, TensorRT, ROCm, and the ORT GPU execution providers +# No GPU to use them. They belong to the sae-builder-cuda and +# sae-builder-rocm siblings (DP-008). +# +# The ONNX models +# Seven files, ~725 MB, in Git LFS. T1/T2 tests are model-free by design +# (tests/CMakeLists.txt:1-4), so the CI image needs none of them, and +# baking them in would inflate the image roughly tenfold to serve the T3 +# smoke tests alone. Those pull the model they need via LFS in a separate +# job. The CI workflow checks out with LFS off for the same reason. +# +# The repository +# Nothing from the source tree is COPYed in. The image is a toolchain, and +# a toolchain that embeds the code it builds has to be rebuilt whenever the +# code changes — which is exactly the per-run cost this image exists to +# avoid. + +WORKDIR /src diff --git a/scripts/ci/build_builder_image.sh b/scripts/ci/build_builder_image.sh new file mode 100644 index 0000000..acce6ec --- /dev/null +++ b/scripts/ci/build_builder_image.sh @@ -0,0 +1,118 @@ +#!/bin/bash +# build_builder_image.sh — build and publish the DP-007 CI builder image to the +# Gitea container registry. +# +# TRACES: DP-007 | PR-004 +# +# Usage: +# scripts/ci/build_builder_image.sh # build only, tag v1 +# scripts/ci/build_builder_image.sh --push # build and push +# scripts/ci/build_builder_image.sh --tag v2 --push # bump the pinned tag +# scripts/ci/build_builder_image.sh --no-cache # force a clean rebuild +# +# The tag is the contract with CI. .gitea/workflows/unit-tests.yml names an +# explicit tag in its `container:` block and never `latest`, so that rebuilding +# the image cannot silently change what a previous green build meant. Bumping +# the dependency set means bumping the tag AND editing the workflow — the two +# edits landing in the same commit is the point, not an inconvenience. +# +# Registry auth: this script does not log in. Do it once, out of band: +# docker login gitea.tourolle.paris +# The CI host is already authenticated this way (its cached credentials in +# ~/.docker/config.json are what the kpnpp-builder push relies on), so a +# workflow that calls this script needs no secret plumbing. +set -euo pipefail + +REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" + +REGISTRY="gitea.tourolle.paris" +OWNER="dtourolle" +IMAGE="sae-builder-cpu" +DOCKERFILE="Dockerfile.builder-cpu" + +# The tag CI pins to today. Keep this in step with the `container.image` line in +# .gitea/workflows/unit-tests.yml; the workflow asserts at run time that the +# image it landed in reports this same version, so a drift shows up as a failed +# job rather than as a build against the wrong toolchain. +TAG="v1" + +PUSH=0 +EXTRA_ARGS=() + +while [ $# -gt 0 ]; do + case "$1" in + --push) PUSH=1 ;; + --tag) TAG="${2:?--tag needs a value}"; shift ;; + --no-cache) EXTRA_ARGS+=(--no-cache) ;; + -h|--help) sed -n '2,30p' "${BASH_SOURCE[0]}"; exit 0 ;; + *) echo "error: unknown argument '$1'" >&2; exit 2 ;; + esac + shift +done + +if [ "$TAG" = "latest" ]; then + echo "error: refusing to build the tag 'latest'." >&2 + echo "DP-007 requires CI to pin an immutable tag. A moving 'latest' means a" >&2 + echo "rebuild retroactively changes what every earlier green build proved." >&2 + exit 2 +fi + +REF="${REGISTRY}/${OWNER}/${IMAGE}:${TAG}" +# A second tag carrying the commit that produced the image. The workflow pins +# the human-readable tag; this one is the audit trail — given any image you can +# recover the Dockerfile that built it. +SHA="$(git -C "$REPO_ROOT" rev-parse --short HEAD)" +REF_SHA="${REGISTRY}/${OWNER}/${IMAGE}:${TAG}-${SHA}" + +# The Dockerfile COPYs nothing from the repository on purpose (see its closing +# comment), so the build context is an empty directory rather than the repo +# root. Sending ~1 GB of models, fixtures and experiment data to the daemon for +# a build that reads none of it is pure latency. +CONTEXT="$(mktemp -d)" +trap 'rm -rf "$CONTEXT"' EXIT + +echo "=== building ${REF}" +echo " dockerfile: ${REPO_ROOT}/${DOCKERFILE}" +echo " context: (empty — the image embeds no repository content)" +echo +echo " Expect this to take a while: OpenCV 5 is compiled from source because" +echo " no Debian release ships it. That cost is paid once per image, which is" +echo " the entire reason DP-007 asks for a prebuilt image instead of" +echo " installing dependencies inside each CI run." +echo + +docker build \ + "${EXTRA_ARGS[@]}" \ + --build-arg "IMAGE_TAG=${TAG}" \ + -f "${REPO_ROOT}/${DOCKERFILE}" \ + -t "${REF}" \ + -t "${REF_SHA}" \ + "${CONTEXT}" + +echo +echo "=== built" +docker image inspect "${REF}" --format ' {{.RepoTags}} {{.Size}} bytes' +docker run --rm "${REF}" sh -c 'echo " SAE_BUILDER=$SAE_BUILDER version=$SAE_BUILDER_VERSION ort=$SAE_ORT_VERSION opencv=$SAE_OPENCV_VERSION"' + +if [ "$PUSH" -eq 0 ]; then + echo + echo "Not pushed. Re-run with --push, or push by hand:" + echo " docker push ${REF}" + echo " docker push ${REF_SHA}" + exit 0 +fi + +echo +echo "=== pushing" +# No `latest` tag is pushed, by design. Publishing one invites a workflow to use +# it, and DP-007 exists to prevent exactly that. +docker push "${REF}" +docker push "${REF_SHA}" + +echo +echo "=== published ${REF}" +echo "If this was a dependency-set change, bump the tag in" +echo " .gitea/workflows/unit-tests.yml (container.image)" +echo " scripts/ci/build_builder_image.sh (TAG, above)" +echo "in the same commit, so no run can build against an image the repository" +echo "does not describe."