#!/usr/bin/env bash # Assemble a signed APK from an already-built libdarkroom.so. # # This runs *inside* the Android image, where the SDK lives. It is deliberately # separate from package.sh: package.sh is a host-side convenience that mounts # the repo into a container and drives the whole build, while CI already runs # in that image and needs only this half. Keeping the assembly in one file # means the APK a device gets from `package.sh --install` and the APK CI # publishes are built by the same code, rather than by two copies that drift. # # Everything is overridable, because the two callers disagree about paths: the # container mounts the repo at /work, CI checks it out wherever the runner # likes. # # REPO repo root (default: this script's ../..) # TARGET_DIR cargo target directory (default: $REPO/target-android) # JNILIBS where cargo-ndk wrote the .so (default: $TARGET_DIR/jniLibs) # OUT output directory (default: $TARGET_DIR/apk) # KEYSTORE signing keystore (default: $TARGET_DIR/debug.keystore) # RUNTIME_DIR the inference runtime (default: $TARGET_DIR/runtime, # fetched by tools/fetch-android-runtime.sh) # ABI Android ABI (default: arm64-v8a) # RUST_TARGET Rust target triple (default: aarch64-linux-android) # # Signing. With none of these set the APK is debug-signed with a generated # throwaway key, which is what a test device wants. Set all three for a real # signature: # # KEYSTORE_PASS keystore password — presence of this is what selects # release signing # KEY_PASS key password (default: same as KEYSTORE_PASS) # KEY_ALIAS key alias within the store # # The passwords are read from the environment and handed to apksigner as # `env:`, never `pass:`. `pass:` puts the password in the process table, where # every other process on the machine can read it out of `ps`. set -euo pipefail HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" REPO="$(cd "${REPO:-${HERE}/../..}" && pwd)" TARGET_DIR="${TARGET_DIR:-${REPO}/target-android}" mkdir -p "${TARGET_DIR}" TARGET_DIR="$(cd "${TARGET_DIR}" && pwd)" JNILIBS="${JNILIBS:-${TARGET_DIR}/jniLibs}" OUT="${OUT:-${TARGET_DIR}/apk}" KEYSTORE="${KEYSTORE:-${TARGET_DIR}/debug.keystore}" RUNTIME_DIR="${RUNTIME_DIR:-${TARGET_DIR}/runtime}" # Release signing is selected by supplying a password, not by a flag, so there # is no way to ask for a release build and silently get a debug one. if [[ -n "${KEYSTORE_PASS:-}" ]]; then SIGNING=release KEY_ALIAS="${KEY_ALIAS:?KEY_ALIAS is required when KEYSTORE_PASS is set}" export DR_KS_PASS="${KEYSTORE_PASS}" export DR_KEY_PASS="${KEY_PASS:-${KEYSTORE_PASS}}" else SIGNING=debug KEY_ALIAS="androiddebugkey" export DR_KS_PASS=android export DR_KEY_PASS=android fi ABI="${ABI:-arm64-v8a}" RUST_TARGET="${RUST_TARGET:-aarch64-linux-android}" SDK="${ANDROID_HOME:-/opt/android-sdk}" # Resolved rather than hard-coded: the versions live in the Dockerfile as ARGs, # and a second copy here is a second thing to forget when they move. The newest # installed build-tools wins. BT="$(find "${SDK}/build-tools" -maxdepth 1 -mindepth 1 -type d | sort -V | tail -1)" [[ -n "${BT}" ]] || { echo "error: no build-tools in ${SDK}" >&2; exit 1; } # The Dockerfile sets ANDROID_JAR to the compile SDK; see its comment for why # that is not the same number as MIN_API. ANDROID_JAR="${ANDROID_JAR:-$(find "${SDK}/platforms" -maxdepth 1 -name 'android-*' \ | sort -V | tail -1)/android.jar}" [[ -f "${ANDROID_JAR}" ]] || { echo "error: no android.jar at ${ANDROID_JAR}" >&2; exit 1; } # MIN_API comes from the Dockerfile too, so the manifest the device reads and # the API the linker targeted cannot disagree. MIN_API="$(sed -n 's/^ARG MIN_API=\([0-9]*\).*/\1/p' "${REPO}/docker/android/Dockerfile")" [[ -n "${MIN_API}" ]] || { echo "error: no ARG MIN_API= in docker/android/Dockerfile" >&2; exit 1; } TARGET_API="$(basename "$(dirname "${ANDROID_JAR}")" | sed 's/^android-//')" # The version, taken from the workspace rather than restated here. See # package.sh for why versionCode is packed the way it is. VERSION_NAME="$(sed -n 's/^version = "\(.*\)"$/\1/p' "${REPO}/Cargo.toml" | head -1)" [[ -n "${VERSION_NAME}" ]] || { echo "error: no version in Cargo.toml" >&2; exit 1; } VERSION_CODE="$(awk -F. '{ print $1 * 10000 + $2 * 100 + $3 }' <<< "${VERSION_NAME}")" echo "==> version ${VERSION_NAME} (code ${VERSION_CODE}), min API ${MIN_API}, target API ${TARGET_API}" SO="${JNILIBS}/${ABI}/libdarkroom.so" [[ -f "${SO}" ]] || { echo "error: ${SO} not built" >&2; exit 1; } rm -rf "${OUT}" mkdir -p "${OUT}/staging/lib/${ABI}" # Slint compiles a Java helper (SlintAndroidJavaHelper) in its build script and # dexes it. The build-dir hash changes whenever its inputs change, so find it # rather than hard-coding a path; the newest wins if stale directories from # earlier builds are still around. DEX="$(find "${TARGET_DIR}/${RUST_TARGET}/release/build" \ -path "*i-slint-backend-android-activity*/out/classes.dex" \ -printf "%T@ %p\n" 2>/dev/null | sort -rn | head -1 | cut -d" " -f2-)" [[ -n "${DEX}" ]] || { echo "error: Slint classes.dex not found — did the backend build?" >&2; exit 1; } echo " dex: ${DEX}" # --------------------------------------------------------------------------- # Our own Java. # # Almost all of this app is Rust, and the classes here are the exceptions the # platform forces: Android constructs some things itself, from a class named in # the manifest, and hands the result back. A `ContentProvider` is one — the # system instantiates it, nothing in the process ever calls its constructor — # and reaching the launch `Intent` is another, because it arrives through # `Activity.getIntent()` and android-activity gives Rust a JNI handle to a # stock `NativeActivity` rather than a subclass it could have put code in. # Neither can be written as Rust at any price, so the APK needs a dex of ours. # # Skipped when the tree has no Java, which is the state this build was in until # FR-PLAT-AND-6 and the state a cut-down branch may return to. The step then # costs nothing and the APK carries Slint's dex alone, exactly as before. JAVA_SRC="${REPO}/apps/darkroom-android/android/java" JAVA_FILES=() if [[ -d "${JAVA_SRC}" ]]; then mapfile -t JAVA_FILES < <(find "${JAVA_SRC}" -name '*.java' | sort) fi if [[ ${#JAVA_FILES[@]} -gt 0 ]]; then echo "==> compiling ${#JAVA_FILES[@]} Java source(s)" mkdir -p "${OUT}/classes" "${OUT}/dex" # `-source 8 -target 8` with an explicit `-bootclasspath`, because that is # the last combination in which javac still lets the boot class path be # replaced: from `-target 9` onwards it rejects the flag outright, and the # platform classes then come from the *JDK* rather than from android.jar. # That compiles cleanly and fails on the device — a JDK class Android does # not ship raises NoClassDefFoundError the moment it is touched, with # nothing at build time having said so. Compiling against android.jar and # only android.jar is what makes "it compiled" mean "the device has it". # # `-Xlint:-options` silences one note, "source value 8 is obsolete", which # is advice about a future JDK rather than about this code. The JDK is # pinned in the Dockerfile, so the day it matters is a deliberate bump. # # `-encoding UTF-8` because javac otherwise reads sources in the *platform* # encoding, and the container sets no locale, so that is US-ASCII. Every # curly quote and em dash in a comment then becomes "unmappable character # (0x94)" — 55 errors from prose, in a file whose code is fine. The sources # are UTF-8 like everything else in the repo; this says so rather than # asking the prose to be typed in ASCII. javac \ -source 8 -target 8 \ -encoding UTF-8 \ -bootclasspath "${ANDROID_JAR}" \ -classpath "${ANDROID_JAR}" \ -Xlint:-options \ -d "${OUT}/classes" \ "${JAVA_FILES[@]}" mapfile -t CLASS_FILES < <(find "${OUT}/classes" -name '*.class' | sort) # d8 merges, it does not only translate. Handing it Slint's finished # classes.dex alongside our fresh .class files yields one dex holding both, # which is what the zip step below already expects. The alternative — ours # as a second classes2.dex — works at API 28, where multidex is native, but # leaves two files to keep in step in the staging and zip steps for no gain # at this size. # # `--min-api` is MIN_API for the same reason the linkers use it: d8 decides # what it must desugar from the oldest device this APK may reach, and a # higher number here emits bytecode that verifies against the build # machine's idea of Android and not against that device's. # # `--lib` is android.jar rather than a copy of the classpath: desugaring # needs to see the platform types it is desugaring against, and without it # d8 reports missing classes for anything our code touches. "${BT}/d8" \ --release \ --min-api "${MIN_API}" \ --lib "${ANDROID_JAR}" \ --output "${OUT}/dex" \ "${DEX}" \ "${CLASS_FILES[@]}" DEX="${OUT}/dex/classes.dex" echo " dex: ${DEX} (ours merged with Slint's)" fi # A debug keystore. CI points KEYSTORE at a throwaway directory so nothing is # persisted or published; package.sh keeps one in the cache on purpose, because # Android refuses to update an installed app whose signature changed and a new # key every build would mean uninstalling before every install. # # Debug-signed only. This gets the app onto a test device; it is not a release # signature, and the store password is the Android convention rather than a # secret worth protecting. if [[ "${SIGNING}" == "release" ]]; then # Never generated on demand. A release key is created once, by hand, and # kept; conjuring one here would mean every build signed by a different # identity, which is indistinguishable from having no signing story at all. [[ -f "${KEYSTORE}" ]] || { echo "error: KEYSTORE_PASS is set but ${KEYSTORE} does not exist" >&2 exit 1 } echo " signing with the release key (alias ${KEY_ALIAS})" elif [[ ! -f "${KEYSTORE}" ]]; then echo " generating debug keystore" mkdir -p "$(dirname "${KEYSTORE}")" keytool -genkeypair -keystore "${KEYSTORE}" -alias "${KEY_ALIAS}" \ -storepass:env DR_KS_PASS -keypass:env DR_KEY_PASS \ -keyalg RSA -keysize 2048 -validity 10950 \ -dname "CN=Android Debug,O=Android,C=US" >/dev/null 2>&1 fi # The launcher icon is the only resource the app has, but resources go through # aapt2 in two steps regardless: compile turns the source tree into an # intermediate archive of flat files, link folds that into the APK and builds # the resources.arsc table that @mipmap/ic_launcher in the manifest resolves # against. Skipping compile and handing link the directory does not work — link # only reads compiled input. "${BT}/aapt2" compile \ --dir "${REPO}/apps/darkroom-android/android/res" \ -o "${OUT}/res.zip" # `android:debuggable`, when asked for, and never otherwise. # # Without it `adb shell run-as` refuses — "package not debuggable" — and the # app's own storage cannot be looked at from the host at all. That storage is # where the face shards, the thumbnail store and the catalog live, so when a # device disagrees with the desktop about what it has synced, there is no way # to find out which of them is wrong. # # Set through aapt2 rather than in `AndroidManifest.xml` deliberately: the flag # then exists only for the build that opted in, and a release build cannot # inherit it by someone forgetting to take it back out again. A debuggable APK # lets any process on the device read this app's private files, so it is a # thing to install on a test tablet and not a thing to publish. DEBUG_FLAG=() if [ -n "${DARKROOM_DEBUGGABLE:-}" ]; then echo "==> debuggable build (run-as enabled; do not publish)" DEBUG_FLAG=(--debug-mode) fi "${BT}/aapt2" link \ -I "${ANDROID_JAR}" \ --manifest "${REPO}/apps/darkroom-android/android/AndroidManifest.xml" \ -R "${OUT}/res.zip" \ --min-sdk-version "${MIN_API}" \ --target-sdk-version "${TARGET_API}" \ --version-name "${VERSION_NAME}" \ --version-code "${VERSION_CODE}" \ "${DEBUG_FLAG[@]}" \ -o "${OUT}/base.apk" \ --auto-add-overlay cp "${SO}" "${OUT}/staging/lib/${ABI}/libdarkroom.so" cp "${DEX}" "${OUT}/staging/classes.dex" # The inference runtime (docs/dev/inference.md §3): ONNX Runtime and Qualcomm's # Hexagon backend, beside libdarkroom.so so the app finds them in its own # native library directory. The build links none of it — the app dlopens # `libonnxruntime.so` at launch and runs on tract if it is not there — so an # APK without these is a slower app, not a broken one, and `RUNTIME_DIR=none` # builds exactly that. 174 MB for the default set; the script says which # Hexagon generations that buys. if [[ "${RUNTIME_DIR}" != "none" ]]; then if [[ ! -f "${RUNTIME_DIR}/lib/libonnxruntime.so" ]]; then "${REPO}/tools/fetch-android-runtime.sh" "${RUNTIME_DIR}" fi cp "${RUNTIME_DIR}"/lib/*.so "${OUT}/staging/lib/${ABI}/" mkdir -p "${OUT}/staging/assets/licences" cp "${RUNTIME_DIR}"/QNN-*.* "${OUT}/staging/assets/licences/" 2>/dev/null || true echo " runtime: $(ls "${RUNTIME_DIR}/lib" | wc -l) libraries from ${RUNTIME_DIR}/lib" else echo " runtime: none (tract only)" fi # The models. Android has no other route to one — app-private storage is not # user-reachable and the in-app fetch is unbuilt (docs/dev/faces.md §2.2a) — so # they go in the APK and `android_main` unpacks them on first launch. The # sources are `models/face/` and `models/scene/`, shared with the Arch package # rather than living under this one platform's directory. # # Two directories, and they are not the same kind of thing. The face weights # are absent from most checkouts by design (research-only grant), so finding # none is ordinary. The scene model is committed, so finding none means a # broken checkout — but this script still only warns, because the failure it # would otherwise cause is at APK build time on a machine that may legitimately # be building the face-less variant. # # Through the staging directory rather than aapt2's `-A`: the .so and the dex # already go in with `zip` below, and one mechanism for "extra files in the # APK" is easier to follow than two. # # Cleared first: a previous run that died between staging and cleanup would # otherwise leave models in the APK that are no longer in the tree. rm -rf "${OUT}/staging/assets/models" mkdir -p "${OUT}/staging/assets/models" _bundled="" for _dir in face scene inpaint; do ASSETS="${REPO}/models/${_dir}" compgen -G "${ASSETS}/*.onnx" >/dev/null || continue # An LFS pointer is ~130 bytes and looks exactly like a model to `cp`. Left # unchecked it reaches the device and fails inside tract, which reports a # broken graph rather than a clone that needs `git lfs pull`. Same guard # dr-segment's build script applies to yolo26n-seg.onnx, and the same # reason. Only the weights are checked: the vocabulary and the category # descriptor beside them are legitimately a few kilobytes. for m in "${ASSETS}"/*.onnx; do if [[ "$(stat -c%s "${m}")" -lt 100000 ]]; then echo "error: $(basename "${m}") is $(stat -c%s "${m}") bytes — an LFS pointer, not a model." >&2 echo " run: git lfs pull" >&2 exit 1 fi done # The scene model is three files: the graph, its vocabulary, and the # category descriptor. All three are needed to decode anything, so they # travel together; README.md is documentation and stays out of the APK. for f in "${ASSETS}"/*; do case "$(basename "${f}")" in README.md) continue ;; esac cp "${f}" "${OUT}/staging/assets/models/" _bundled="${_bundled} $(basename "${f}")" done done if [[ -n "${_bundled}" ]]; then echo " assets:${_bundled}" else echo " assets: no models found (face indexing and the scene tab will be off on the device)" fi # -0 "" stores the .so without compression so Android can mmap it directly # (extractNativeLibs=false territory); for a 37 MB library that also keeps # install times sane. cd "${OUT}/staging" cp "${OUT}/base.apk" "${OUT}/unaligned.apk" zip -q -0 -X "${OUT}/unaligned.apk" lib/"${ABI}"/*.so zip -q -X "${OUT}/unaligned.apk" classes.dex # Stored, not deflated: an ONNX graph is mostly incompressible float data, so # deflating it buys a few percent and costs the whole file being inflated into # RAM on the way out. AAssetManager reads a stored entry straight from the # mapped APK. if [[ -d assets ]]; then zip -q -0 -X -r "${OUT}/unaligned.apk" assets fi # zipalign before signing: apksigner preserves alignment, the reverse order # invalidates the signature. "${BT}/zipalign" -p -f 4 "${OUT}/unaligned.apk" "${OUT}/darkroom.apk" "${BT}/apksigner" sign \ --ks "${KEYSTORE}" --ks-key-alias "${KEY_ALIAS}" \ --ks-pass env:DR_KS_PASS --key-pass env:DR_KEY_PASS \ --min-sdk-version "${MIN_API}" \ "${OUT}/darkroom.apk" "${BT}/apksigner" verify --print-certs "${OUT}/darkroom.apk" | head -2 echo " signing: ${SIGNING}" # The intermediates are not the artefact, and leaving them beside it invites # the wrong file being picked up by a glob. rm -rf "${OUT}/staging" "${OUT}/res.zip" "${OUT}/base.apk" "${OUT}/unaligned.apk" \ "${OUT}/classes" "${OUT}/dex" echo "==> ${OUT}/darkroom.apk" ls -la "${OUT}/darkroom.apk"