Compare commits
1
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
 Gitea Actions
|
83bcf09be1 |
@@ -1,41 +0,0 @@
|
||||
{
|
||||
"permissions": {
|
||||
"allow": [
|
||||
"Bash(curl -s \"https://api.github.com/search/code?q=WrapTexture+org:Noesis\" -H \"Accept: application/vnd.github+json\")",
|
||||
"Bash(curl -s \"https://api.github.com/orgs/Noesis/repos?per_page=100\")",
|
||||
"WebFetch(domain:wiki.wxwidgets.org)",
|
||||
"Bash(curl -sS \"https://gitlab.com/sciter-engine/sciter-js-sdk/-/raw/main/samples.gpu/hello-es-triangle.htm\")",
|
||||
"Bash(curl -sS \"https://gitlab.com/api/v4/projects/sciter-engine%2Fsciter-js-sdk/repository/tree?path=include&recursive=true&per_page=100&ref=main\")",
|
||||
"Bash(python3 -c ' *)",
|
||||
"Bash(curl -sL --max-time 40 \"https://api.github.com/repos/wxWidgets/wxWidgets/contents/src?ref=master\")",
|
||||
"Bash(curl -sL --max-time 40 \"https://api.github.com/repos/wxWidgets/wxWidgets/contents/include/wx/android?ref=master\")",
|
||||
"Bash(curl -sL --max-time 40 -H \"Accept: application/vnd.github.text-match+json\" \"https://api.github.com/search/code?q=vulkan+repo:wxWidgets/wxWidgets\")",
|
||||
"Bash(curl -sS \"https://gitlab.com/sciter-engine/sciter-js-sdk/-/raw/main/include/sciter-x-video-api.h\")",
|
||||
"WebFetch(domain:docs.wxwidgets.org)",
|
||||
"Bash(curl -sS \"https://gitlab.com/sciter-engine/sciter-js-sdk/-/raw/main/CHANGELOG.md\")",
|
||||
"Bash(curl -sL --max-time 40 \"https://raw.githubusercontent.com/wxWidgets/wxWidgets/master/docs/readme.txt\")",
|
||||
"Bash(curl -sL --max-time 40 \"https://raw.githubusercontent.com/wxWidgets/wxWidgets/master/docs/licence.txt\")",
|
||||
"Bash(curl -sL --max-time 40 \"https://raw.githubusercontent.com/wxWidgets/wxWidgets/master/docs/licendu.txt\")",
|
||||
"Bash(curl -sS \"https://gitlab.com/api/v4/projects/sciter-engine%2Fsciter-js-sdk/repository/tree?path=build&recursive=true&per_page=100&ref=main\")",
|
||||
"Bash(curl -sS \"https://gitlab.com/sciter-engine/sciter-js-sdk/-/raw/main/premake5.lua\")",
|
||||
"WebFetch(domain:slack-chats.kotlinlang.org)",
|
||||
"Bash(curl -sS -L \"https://sciter.com/\")",
|
||||
"Bash(curl -sL --max-time 45 \"https://api.github.com/orgs/ultralight-ux/repos?per_page=100&sort=pushed\")",
|
||||
"Bash(curl -sL --max-time 40 \"https://gitlab.gnome.org/GNOME/gtk/-/raw/main/gdk/gdkdmabuftexturebuilder.h\")",
|
||||
"Bash(curl -sL --max-time 40 \"https://gitlab.gnome.org/GNOME/gtk/-/raw/main/gdk/gdkgltexturebuilder.h\")",
|
||||
"Bash(curl -sL --max-time 40 \"https://gitlab.gnome.org/api/v4/projects/GNOME%2Fgtk/repository/tree?path=gdk&ref=main&per_page=100\")",
|
||||
"WebFetch(domain:docs.slint.dev)",
|
||||
"WebFetch(domain:releases.slint.dev)",
|
||||
"WebFetch(domain:flutter.dev)",
|
||||
"Bash(curl -sS -L \"https://sciter.com/forums/topic/status-of-quark-sciter-lite-sciterjs-android-ios/\")",
|
||||
"Bash(curl -sL --max-time 45 \"https://api.github.com/repos/ultralight-ux/AppCore/git/trees/master?recursive=1\")",
|
||||
"Bash(curl -sL --max-time 40 \"https://gitlab.gnome.org/GNOME/gtk/-/raw/main/gdk/meson.build\")",
|
||||
"WebFetch(domain:www.jetbrains.com)",
|
||||
"Bash(curl -sS \"https://gitlab.com/api/v4/projects/sciter-engine%2Fsciter-js-sdk/repository/tree?path=demos.lite&recursive=true&per_page=100&ref=main\")",
|
||||
"Bash(curl -sL --max-time 40 \"https://gitlab.gnome.org/api/v4/projects/GNOME%2Fgtk/repository/commits?path=gdk/android/gdkandroidglcontext.c&ref_name=main&per_page=20\")",
|
||||
"Bash(curl -sL --max-time 40 \"https://gitlab.gnome.org/GNOME/gtk/-/raw/main/gdk/android/meson.build\")",
|
||||
"WebFetch(domain:docs.sciter.com)",
|
||||
"Bash(curl -sS -L \"https://sciter.com/support-of-displayflex-and-displaygrid-in-sciter/\")"
|
||||
]
|
||||
}
|
||||
}
|
||||
@@ -1,22 +0,0 @@
|
||||
# Model weights live in LFS.
|
||||
#
|
||||
# `models/**/*.onnx` is tens of MB of binary that changes wholesale
|
||||
# when it changes at all. In ordinary git objects every future revision of it
|
||||
# would be stored in full, in every clone, forever — and the one thing nobody
|
||||
# can do with it is a useful diff.
|
||||
#
|
||||
# Consequence worth knowing before it bites: a clone without git-lfs gets a
|
||||
# ~130-byte pointer file where the model should be. `dr-segment`'s build script
|
||||
# detects exactly that and fails with an instruction rather than embedding the
|
||||
# pointer and failing at inference time.
|
||||
*.onnx filter=lfs diff=lfs merge=lfs -text
|
||||
|
||||
# Test photographs live in LFS too, and are fetched only by the tests that
|
||||
# need them.
|
||||
#
|
||||
# `fixtures/**` holds real camera files — a twelve-frame panorama set is
|
||||
# 325 MB — and CI's `git lfs pull` excludes the directory, so a checkout
|
||||
# carries pointers there until a merge test asks for the frames. Same
|
||||
# reasoning as the models, with the opposite default: the model is not
|
||||
# optional and the fixtures are.
|
||||
fixtures/** filter=lfs diff=lfs merge=lfs -text
|
||||
@@ -1,170 +0,0 @@
|
||||
name: '🐳 Android image'
|
||||
|
||||
# Builds and pushes gitea.tourolle.paris/dtourolle/darkroom-android, the job
|
||||
# container for the Android leg of build-and-test.yml.
|
||||
#
|
||||
# It exists because that image previously lived only on a developer's laptop:
|
||||
# the workflow referenced a tag that had never been pushed, and every Android
|
||||
# job died at `docker pull` with "manifest unknown" before running a step. The
|
||||
# image is now reproducible from the repo rather than from one machine.
|
||||
#
|
||||
# Called by build-and-test.yml on every push, and runnable by hand via
|
||||
# workflow_dispatch. It is cheap when nothing changed — see the guard below.
|
||||
on:
|
||||
workflow_call:
|
||||
inputs:
|
||||
force:
|
||||
description: 'Rebuild even if the registry already has this image ("true"/"false")'
|
||||
type: string
|
||||
default: 'false'
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
force:
|
||||
description: 'Rebuild even if the registry already has this image ("true"/"false")'
|
||||
type: string
|
||||
default: 'false'
|
||||
|
||||
# Gitea's act_runner mangles boolean workflow inputs passed through an
|
||||
# expression — they arrive as false regardless of what was sent. Every input
|
||||
# here is a string compared with == 'true', as in KPN's docker.yaml.
|
||||
|
||||
env:
|
||||
IMAGE: gitea.tourolle.paris/dtourolle/darkroom-android
|
||||
|
||||
jobs:
|
||||
build:
|
||||
runs-on: linux/amd64
|
||||
name: Build and push
|
||||
# Deliberately NOT in a container: this job needs the host Docker daemon to
|
||||
# build an image, and the host's cached ~/.docker/config.json to push it.
|
||||
# That is also why there is no `docker login` step — the runner host was
|
||||
# authenticated to the registry during setup.
|
||||
|
||||
steps:
|
||||
# The host has no Node, so the JS-based actions/checkout cannot run here.
|
||||
# A minimal shallow fetch with plain git gets the same tree.
|
||||
- name: Checkout
|
||||
run: |
|
||||
set -e
|
||||
git init -q .
|
||||
git remote add origin "${{ github.server_url }}/${{ github.repository }}.git"
|
||||
git -c http.extraheader="AUTHORIZATION: basic $(printf '%s' '${{ github.actor }}:${{ github.token }}' | base64 -w0)" \
|
||||
fetch --depth 1 origin "${{ github.sha }}"
|
||||
git checkout -q FETCH_HEAD
|
||||
|
||||
# The image is tagged by the content of docker/android, not by the commit
|
||||
# that happened to touch it. `git rev-parse HEAD:<dir>` is the tree object
|
||||
# id — it changes when and only when a file in that directory changes, so
|
||||
# an unrelated push reuses the existing image and a Dockerfile edit can
|
||||
# never silently keep serving a stale `latest`.
|
||||
#
|
||||
# Using the commit sha instead would rebuild 7 GB on every push; using a
|
||||
# paths-filter action would need a container that has Node, and the only
|
||||
# one this repo would reach for is the very image being built.
|
||||
- name: Resolve image tag
|
||||
id: tag
|
||||
run: |
|
||||
set -e
|
||||
TREE=$(git rev-parse HEAD:docker/android)
|
||||
echo "tree=$TREE" >> "$GITHUB_OUTPUT"
|
||||
echo "docker/android tree: $TREE"
|
||||
|
||||
# Skip the build when the registry already holds this exact content. This
|
||||
# is what keeps the job a few seconds long on a normal push, and what
|
||||
# makes it self-healing: if the tag is missing for any reason, including
|
||||
# the image having never been pushed at all, it gets built here.
|
||||
#
|
||||
# The probe is curl against the registry API, NOT `docker manifest
|
||||
# inspect`. The latter exits 1 on this registry even for tags that are
|
||||
# demonstrably present — jellytau-builder:latest answers HTTP 200 to the
|
||||
# API while `docker manifest inspect` reports "manifest unknown" for it.
|
||||
# Trusting that would have rebuilt 7 GB on every single push.
|
||||
#
|
||||
# A HEAD request also gives the digest for free, which is how the repoint
|
||||
# decision below is made without pulling any layers.
|
||||
- name: Query registry
|
||||
id: check
|
||||
env:
|
||||
# The runner's own credentials, so this does not depend on how the
|
||||
# host's ~/.docker/config.json happens to be set up.
|
||||
REG_USER: ${{ github.actor }}
|
||||
REG_PASS: ${{ github.token }}
|
||||
TREE: ${{ steps.tag.outputs.tree }}
|
||||
run: |
|
||||
set -eu
|
||||
ACCEPT='application/vnd.oci.image.index.v1+json,application/vnd.docker.distribution.manifest.v2+json,application/vnd.oci.image.manifest.v1+json,application/vnd.docker.distribution.manifest.list.v2+json'
|
||||
API="https://gitea.tourolle.paris/v2/dtourolle/darkroom-android/manifests"
|
||||
|
||||
# Prints "<http-status> <digest-or-empty>" for a tag.
|
||||
probe() {
|
||||
curl -sI -u "$REG_USER:$REG_PASS" -H "Accept: $ACCEPT" "$API/$1" \
|
||||
| tr -d '\r' \
|
||||
| awk 'BEGIN{s="000";d=""} /^HTTP/{s=$2} tolower($1)=="docker-content-digest:"{d=$2} END{print s, d}'
|
||||
}
|
||||
|
||||
read -r TREE_STATUS TREE_DIGEST <<EOF
|
||||
$(probe "$TREE")
|
||||
EOF
|
||||
read -r LATEST_STATUS LATEST_DIGEST <<EOF
|
||||
$(probe latest)
|
||||
EOF
|
||||
|
||||
echo "tag $TREE -> HTTP $TREE_STATUS ${TREE_DIGEST:-(no digest)}"
|
||||
echo "tag latest -> HTTP $LATEST_STATUS ${LATEST_DIGEST:-(no digest)}"
|
||||
|
||||
# Build unless the registry definitively confirms this content is
|
||||
# already there. An auth failure or an unreachable registry lands
|
||||
# here too, and rebuilding needlessly is the safe direction to fail —
|
||||
# skipping a build that was needed is what breaks the Android job.
|
||||
if [ "${{ inputs.force }}" = "true" ]; then
|
||||
echo "forced rebuild requested"
|
||||
echo "build=true" >> "$GITHUB_OUTPUT"
|
||||
echo "repoint=false" >> "$GITHUB_OUTPUT"
|
||||
elif [ "$TREE_STATUS" != "200" ]; then
|
||||
echo "registry does not have this content — building"
|
||||
echo "build=true" >> "$GITHUB_OUTPUT"
|
||||
echo "repoint=false" >> "$GITHUB_OUTPUT"
|
||||
elif [ -n "$TREE_DIGEST" ] && [ "$TREE_DIGEST" = "$LATEST_DIGEST" ]; then
|
||||
echo "registry is already correct — nothing to do"
|
||||
echo "build=false" >> "$GITHUB_OUTPUT"
|
||||
echo "repoint=false" >> "$GITHUB_OUTPUT"
|
||||
else
|
||||
echo "content is present but latest points elsewhere — repointing"
|
||||
echo "build=false" >> "$GITHUB_OUTPUT"
|
||||
echo "repoint=true" >> "$GITHUB_OUTPUT"
|
||||
fi
|
||||
|
||||
# Context is docker/android, matching the README's build command. The
|
||||
# Dockerfile COPYs nothing from the repo, so it needs no wider context —
|
||||
# and a narrow context keeps the daemon from tarring up the whole tree,
|
||||
# target/ included.
|
||||
- name: Build
|
||||
if: ${{ steps.check.outputs.build == 'true' }}
|
||||
run: |
|
||||
set -e
|
||||
docker build \
|
||||
-t "$IMAGE:${{ steps.tag.outputs.tree }}" \
|
||||
-t "$IMAGE:latest" \
|
||||
docker/android
|
||||
|
||||
# Both tags are pushed: the tree tag is what the guard above looks for on
|
||||
# the next run, and `latest` is what build-and-test.yml pulls.
|
||||
- name: Push
|
||||
if: ${{ steps.check.outputs.build == 'true' }}
|
||||
run: |
|
||||
set -e
|
||||
docker push "$IMAGE:${{ steps.tag.outputs.tree }}"
|
||||
docker push "$IMAGE:latest"
|
||||
|
||||
# A cache hit on the tree tag says nothing about where `latest` points — a
|
||||
# reverted Dockerfile or a build from another branch can leave it on
|
||||
# different content. This runs only when the digests above actually
|
||||
# disagree, so the common case costs nothing; the layers are already in
|
||||
# the registry, so the push that follows uploads a manifest, not 7 GB.
|
||||
- name: Repoint latest
|
||||
if: ${{ steps.check.outputs.repoint == 'true' }}
|
||||
run: |
|
||||
set -e
|
||||
docker pull "$IMAGE:${{ steps.tag.outputs.tree }}"
|
||||
docker tag "$IMAGE:${{ steps.tag.outputs.tree }}" "$IMAGE:latest"
|
||||
docker push "$IMAGE:latest"
|
||||
@@ -1,193 +0,0 @@
|
||||
name: Benchmarks
|
||||
|
||||
# The suite docs/requirements.md §8 has been promising since it was written:
|
||||
# "an automated benchmark suite against a synthetic 50k catalog, run per-commit
|
||||
# … A regression beyond stated tolerance fails the build."
|
||||
#
|
||||
# Its own workflow rather than a step inside build-and-test.yml, and the reason
|
||||
# is what a failure here means. A red `Build and test` says the code is wrong; a
|
||||
# red `Benchmarks` says the code is slower than it was, which is a different
|
||||
# conversation, is read by different people, and must not be reachable by
|
||||
# retrying a flaky compile.
|
||||
#
|
||||
# # Why this is split in two
|
||||
#
|
||||
# §8 names "the reference desktop", not CI, and it is right to. So:
|
||||
#
|
||||
# cpu — runs on every push. It needs no adapter and no display, and the
|
||||
# budgets it asserts (a 50k catalog opening inside two seconds) have
|
||||
# two orders of magnitude of headroom, so a modest runner can be held
|
||||
# to them honestly. Machine-sensitive budgets — throughput targets
|
||||
# written for a 24-thread desktop — are reported here rather than
|
||||
# asserted; `dr-bench` decides that per metric and says so in its
|
||||
# report. Asserting them on a two-core container would produce exactly
|
||||
# what core/dr-gpu/tests/frame_budget.rs refused to produce: a red gate
|
||||
# everybody learns to ignore.
|
||||
#
|
||||
# gpu — the frame budget, which already exists and already skips itself where
|
||||
# there is no adapter. Not on push: it would build wgpu and naga on
|
||||
# every commit to establish, every time, that this runner has no GPU. It
|
||||
# runs on demand (Actions → Run workflow) so that a runner that *does*
|
||||
# have one can be pointed at it, and the numbers it produces belong in
|
||||
# docs/frame-budget.md by hand, as they already are.
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [main, master, develop]
|
||||
pull_request:
|
||||
branches: [main, master, develop]
|
||||
workflow_dispatch:
|
||||
|
||||
jobs:
|
||||
cpu:
|
||||
runs-on: linux/amd64
|
||||
name: CPU and I/O (per commit)
|
||||
# Node for actions/checkout and actions/cache, which the bare runner image
|
||||
# cannot execute. Rust is installed below.
|
||||
container:
|
||||
image: catthehacker/ubuntu:act-latest
|
||||
|
||||
env:
|
||||
# Same reasoning as the desktop job in build-and-test.yml: incremental
|
||||
# state exists to make the *second* build in a working tree fast, which is
|
||||
# not a thing a fresh checkout has, and it fills the runner's disk.
|
||||
CARGO_INCREMENTAL: 0
|
||||
# The fixture, out of the workspace so actions/cache never picks it up.
|
||||
# A 14 MB synthetic catalog is two seconds to regenerate and would
|
||||
# otherwise be uploaded and downloaded on every push to save them.
|
||||
DR_BENCH_DIR: /tmp/darkroom-bench
|
||||
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v4
|
||||
|
||||
# No `git lfs pull` here, deliberately. `dr-bench` depends on the catalog,
|
||||
# the decoder, the thumbnail store and the encoder, and on nothing that
|
||||
# reaches `dr-segment` — so the model this repository keeps in LFS is not
|
||||
# part of this job's dependency graph and fetching it would be a minute
|
||||
# spent on a file nothing opens.
|
||||
|
||||
- name: Cache cargo
|
||||
uses: actions/cache@v4
|
||||
with:
|
||||
path: |
|
||||
~/.cargo/registry
|
||||
~/.cargo/git
|
||||
target
|
||||
key: bench-${{ runner.os }}-${{ hashFiles('**/Cargo.lock') }}
|
||||
|
||||
# Pinned to the workspace rust-version, as every other job here is: a
|
||||
# floating toolchain turns an unrelated push into a mystery failure, and
|
||||
# for a benchmark it would turn one into a mystery *regression*.
|
||||
#
|
||||
# rust-analyzer is named for the reason build-and-test.yml gives: rustup
|
||||
# reconciles rust-toolchain.toml on the first cargo call whatever this
|
||||
# step asks for, so naming it keeps the download inside the step that says
|
||||
# it is installing things.
|
||||
- name: Install Rust 1.92.0
|
||||
run: |
|
||||
set -e
|
||||
curl -fsSL https://sh.rustup.rs | sh -s -- \
|
||||
-y --no-modify-path --profile minimal --default-toolchain 1.92.0 \
|
||||
--component rust-analyzer
|
||||
echo "$HOME/.cargo/bin" >> "$GITHUB_PATH"
|
||||
|
||||
# `-p dr-bench`, not `--workspace`. The whole point of that crate having
|
||||
# no GPU and no UI dependency is that this job resolves the catalog, the
|
||||
# decoder and the encoders and stops there — a few minutes rather than the
|
||||
# release build of Slint and wgpu the desktop job pays for.
|
||||
#
|
||||
# Release, and it is not optional: the workspace builds its own crates at
|
||||
# opt-level = 0 in dev, and every figure this produces is dominated by
|
||||
# this workspace's own code. A debug run would measure rustc.
|
||||
- name: Build the suite
|
||||
run: cargo build --release -p dr-bench
|
||||
|
||||
# Exit 1 is a violated budget or a regression past tolerance; exit 2 is
|
||||
# the harness failing to run at all. Both fail the job, and the report
|
||||
# above the failure says which.
|
||||
- name: Measure, and gate
|
||||
run: cargo run --release -p dr-bench -- check
|
||||
|
||||
- name: Disk after
|
||||
if: always()
|
||||
run: df -h /workspace 2>/dev/null || df -h .
|
||||
|
||||
gpu:
|
||||
# On demand only — see the header. A runner with a Vulkan device can be
|
||||
# pointed at this; one without will skip the measurement and say so, which
|
||||
# is the same posture the rest of this repository's device tests take.
|
||||
if: github.event_name == 'workflow_dispatch'
|
||||
runs-on: linux/amd64
|
||||
name: Frame budget (on demand)
|
||||
container:
|
||||
image: catthehacker/ubuntu:act-latest
|
||||
|
||||
env:
|
||||
CARGO_INCREMENTAL: 0
|
||||
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v4
|
||||
|
||||
# `dr-gpu` depends on `dr-segment` for the watershed's pixel passes. Its
|
||||
# default features are off, so no weights are compiled in — but the fetch
|
||||
# is cheap insurance and its failure is not fatal. The header of the same
|
||||
# step in build-and-test.yml explains why the extraheader is stripped
|
||||
# rather than reused: two Authorization headers is a 400 from Gitea, one
|
||||
# step after the batch call that had just succeeded.
|
||||
- name: Fetch the segmentation model
|
||||
continue-on-error: true
|
||||
env:
|
||||
LFS_TOKEN: ${{ secrets.GITEA_TOKEN || github.token }}
|
||||
run: |
|
||||
set -e
|
||||
git lfs install --local
|
||||
git config --local --get-regexp '^http\..*extraheader$' \
|
||||
| cut -d' ' -f1 | sort -u \
|
||||
| while read -r key; do git config --local --unset-all "$key"; done || true
|
||||
git config --local lfs.url \
|
||||
"https://x-access-token:${LFS_TOKEN}@gitea.tourolle.paris/dtourolle/DarkRoom.git/info/lfs"
|
||||
git lfs pull --exclude="fixtures/**"
|
||||
|
||||
- name: Cache cargo
|
||||
uses: actions/cache@v4
|
||||
with:
|
||||
path: |
|
||||
~/.cargo/registry
|
||||
~/.cargo/git
|
||||
target
|
||||
key: bench-gpu-${{ runner.os }}-${{ hashFiles('**/Cargo.lock') }}
|
||||
|
||||
- name: Build dependencies
|
||||
run: |
|
||||
apt-get update -qq
|
||||
apt-get install -y -qq pkg-config libfontconfig1-dev libxkbcommon-dev
|
||||
|
||||
- name: Install Rust 1.92.0
|
||||
run: |
|
||||
set -e
|
||||
curl -fsSL https://sh.rustup.rs | sh -s -- \
|
||||
-y --no-modify-path --profile minimal --default-toolchain 1.92.0 \
|
||||
--component rust-analyzer
|
||||
echo "$HOME/.cargo/bin" >> "$GITHUB_PATH"
|
||||
|
||||
# The guard, in release. Its own module documentation is explicit that a
|
||||
# release run checks strictly more than a dev one: the CPU half of a frame
|
||||
# is shader-string assembly, which is several times slower unoptimised, so
|
||||
# it is folded into the assertion only when debug_assertions is off.
|
||||
#
|
||||
# With no adapter this prints "skipping: no GPU adapter" and passes. A
|
||||
# test that cannot run is not evidence either way, and turning that into a
|
||||
# failure would make the job useless on the runner it usually lands on.
|
||||
- name: Frame budget (FR-DSP-3)
|
||||
run: cargo test --release -p dr-gpu --test frame_budget -- --nocapture
|
||||
|
||||
# The instrument behind docs/frame-budget.md. It exits non-zero with no
|
||||
# adapter, which is right for a tool a person runs deliberately and wrong
|
||||
# for a job that usually has none — hence continue-on-error. Its table is
|
||||
# in the log for whoever asked for this run; the committed numbers are
|
||||
# still updated by hand, as that file says.
|
||||
- name: Frame budget table
|
||||
continue-on-error: true
|
||||
run: cargo run --release -p dr-gpu --example frame_budget
|
||||
@@ -1,516 +0,0 @@
|
||||
name: Build and test
|
||||
|
||||
# Desktop and Android are built on every push, per the v0.1 decision to carry
|
||||
# both platforms from the first commit. An Android break is then caught the day
|
||||
# it lands rather than at a porting milestone.
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [main, master, develop]
|
||||
pull_request:
|
||||
branches: [main, master, develop]
|
||||
|
||||
jobs:
|
||||
# The Android job runs inside an image that this repo builds. Ensure it is in
|
||||
# the registry before anything tries to pull it — see android-image.yml for
|
||||
# why this is a job rather than a documented manual step. It is a no-op of a
|
||||
# few seconds unless docker/android actually changed.
|
||||
android-image:
|
||||
uses: ./.gitea/workflows/android-image.yml
|
||||
|
||||
desktop:
|
||||
runs-on: linux/amd64
|
||||
name: Desktop (Linux)
|
||||
# actions/checkout and actions/cache are JavaScript actions: the runner
|
||||
# executes them with Node from inside this container. The bare runner image
|
||||
# has none, so the job failed at checkout before reaching any build step.
|
||||
container:
|
||||
image: catthehacker/ubuntu:act-latest
|
||||
|
||||
# This job filled the runner's disk and died mid-link with "No space left
|
||||
# on device" — LLVM reporting an IO failure on its output stream, which
|
||||
# reads like a compiler crash and is not one.
|
||||
#
|
||||
# `target/debug` was 24 GB against `target/release`'s 2.6 GB: 15 GB of it
|
||||
# debug info in `debug/deps`, 3.6 GB incremental state. Neither earns its
|
||||
# space here. Nothing attaches a debugger to a CI run, and incremental
|
||||
# compilation exists to make the *second* build in a working tree fast,
|
||||
# which is not a thing a fresh checkout has. Turning both off is the
|
||||
# standard CI setting rather than a trick.
|
||||
#
|
||||
# Measured on this workspace: the same `cargo test --workspace --no-run`
|
||||
# tree goes from 24 GB to 3.3 GB, `debug/deps` from 15 GB to 2.8 GB.
|
||||
#
|
||||
# Backtraces still name functions without debug info; they lose file and
|
||||
# line numbers. If a test failure ever needs those, drop DEBUG to 1
|
||||
# (line-tables-only) rather than back to 2.
|
||||
#
|
||||
# This is a mitigation, not a fix. If the runner is full of anything other
|
||||
# than this job's own output, it will still be full afterwards.
|
||||
env:
|
||||
CARGO_INCREMENTAL: 0
|
||||
CARGO_PROFILE_DEV_DEBUG: 0
|
||||
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v4
|
||||
|
||||
# The model, which is in LFS and is not optional.
|
||||
#
|
||||
# `models/**/*.onnx` is tracked in LFS (.gitattributes), so a
|
||||
# plain checkout writes a ~130-byte pointer where 11 MB should be, and
|
||||
# `dr-segment`'s build script panics by design rather than embedding a
|
||||
# pointer and failing at inference. That failure reads like a broken build
|
||||
# instead of a missing fetch, which is how it went unnoticed.
|
||||
#
|
||||
# Not `lfs: true` on the checkout above, and no `Authorization` header
|
||||
# here either. Both install a blanket header for every request to this
|
||||
# host, and the object download is the one request that already carries
|
||||
# one: `git lfs pull` asks `/info/lfs/objects/batch` first, and Gitea
|
||||
# answers with a short-lived `Bearer` JWT scoped to that object. git-lfs
|
||||
# then sends the JWT *and* the configured header, and two `Authorization`
|
||||
# headers is a 400 from Gitea — reported as
|
||||
# LFS: Client error: .../info/lfs/objects/<oid>
|
||||
# one step after the batch call that had just succeeded, which reads like
|
||||
# a rejected credential rather than a duplicated one. A lone token header
|
||||
# is understood fine; it is only the collision that fails.
|
||||
#
|
||||
# So: strip the headers and hand the token to git-lfs as an ordinary
|
||||
# credential instead. It authenticates the batch call and leaves the
|
||||
# per-object JWT untouched. Gitea authenticates on the password, so the
|
||||
# username is a placeholder. Nothing later in this job talks to the
|
||||
# remote, so dropping checkout's header costs us nothing.
|
||||
#
|
||||
# `continue-on-error` deliberately: if this cannot authenticate, the build
|
||||
# below still runs and fails with the build script's own message, which
|
||||
# names the real problem. A checkout that dies here says nothing.
|
||||
- name: Fetch the segmentation model
|
||||
continue-on-error: true
|
||||
env:
|
||||
LFS_TOKEN: ${{ secrets.GITEA_TOKEN || github.token }}
|
||||
run: |
|
||||
set -e
|
||||
git lfs install --local
|
||||
git config --local --get-regexp '^http\..*extraheader$' \
|
||||
| cut -d' ' -f1 | sort -u \
|
||||
| while read -r key; do git config --local --unset-all "$key"; done || true
|
||||
git config --local lfs.url \
|
||||
"https://x-access-token:${LFS_TOKEN}@gitea.tourolle.paris/dtourolle/DarkRoom.git/info/lfs"
|
||||
git lfs pull --exclude="fixtures/**"
|
||||
ls -lR models/
|
||||
|
||||
- name: Cache cargo
|
||||
uses: actions/cache@v4
|
||||
with:
|
||||
path: |
|
||||
~/.cargo/registry
|
||||
~/.cargo/git
|
||||
target
|
||||
key: desktop-${{ runner.os }}-${{ hashFiles('**/Cargo.lock') }}
|
||||
|
||||
# Slint and winit need these at build time; the runner image is minimal.
|
||||
- name: Build dependencies
|
||||
run: |
|
||||
apt-get update -qq
|
||||
apt-get install -y -qq pkg-config libfontconfig1-dev libxkbcommon-dev
|
||||
|
||||
# The act image ships Node but no Rust. Pinned to the workspace
|
||||
# rust-version so CI, the Android image, and local builds agree — a
|
||||
# floating toolchain turns an unrelated push into a mystery failure.
|
||||
#
|
||||
# The component list mirrors rust-toolchain.toml's, rust-analyzer
|
||||
# included, even though nothing in this job runs it. rustup reconciles
|
||||
# that file against the installed toolchain on the first cargo call in
|
||||
# the work tree and fetches whatever is missing — so leaving it out does
|
||||
# not save the download, it only moves it into the middle of a build
|
||||
# step where it is nobody's line item. Naming it here keeps every fetch
|
||||
# inside the step whose name says it is installing things.
|
||||
- name: Install Rust 1.92.0
|
||||
run: |
|
||||
set -e
|
||||
curl -fsSL https://sh.rustup.rs | sh -s -- \
|
||||
-y --no-modify-path --profile minimal \
|
||||
--default-toolchain 1.92.0 --component rustfmt,clippy,rust-analyzer
|
||||
echo "$HOME/.cargo/bin" >> "$GITHUB_PATH"
|
||||
|
||||
# Free space before and after the expensive steps, so a repeat of the
|
||||
# disk exhaustion above is one line to diagnose instead of a puzzling
|
||||
# LLVM error.
|
||||
- name: Disk before
|
||||
run: df -h /workspace 2>/dev/null || df -h .
|
||||
|
||||
- name: Format
|
||||
run: cargo fmt --all -- --check
|
||||
|
||||
- name: Clippy
|
||||
run: cargo clippy --workspace --all-targets -- -D warnings
|
||||
|
||||
# GPU tests skip themselves where no adapter is present rather than
|
||||
# failing — CI runners generally have none, and a test that cannot run is
|
||||
# not evidence either way.
|
||||
- name: Test
|
||||
run: cargo test --workspace
|
||||
|
||||
- name: Build
|
||||
run: cargo build --workspace --release
|
||||
|
||||
- name: Disk after
|
||||
if: always()
|
||||
run: df -h /workspace 2>/dev/null || df -h .
|
||||
|
||||
android:
|
||||
runs-on: linux/amd64
|
||||
name: Android (aarch64)
|
||||
# Waits for the image build. Without this the pull races the push and the
|
||||
# job dies with "manifest unknown" before its first step, which is the
|
||||
# failure mode this ordering exists to remove.
|
||||
needs: android-image
|
||||
container:
|
||||
image: gitea.tourolle.paris/dtourolle/darkroom-android:latest
|
||||
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v4
|
||||
|
||||
# The model, which is in LFS and is not optional.
|
||||
#
|
||||
# `models/**/*.onnx` is tracked in LFS (.gitattributes), so a
|
||||
# plain checkout writes a ~130-byte pointer where 11 MB should be, and
|
||||
# `dr-segment`'s build script panics by design rather than embedding a
|
||||
# pointer and failing at inference. That failure reads like a broken build
|
||||
# instead of a missing fetch, which is how it went unnoticed.
|
||||
#
|
||||
# Not `lfs: true` on the checkout above, and no `Authorization` header
|
||||
# here either. Both install a blanket header for every request to this
|
||||
# host, and the object download is the one request that already carries
|
||||
# one: `git lfs pull` asks `/info/lfs/objects/batch` first, and Gitea
|
||||
# answers with a short-lived `Bearer` JWT scoped to that object. git-lfs
|
||||
# then sends the JWT *and* the configured header, and two `Authorization`
|
||||
# headers is a 400 from Gitea — reported as
|
||||
# LFS: Client error: .../info/lfs/objects/<oid>
|
||||
# one step after the batch call that had just succeeded, which reads like
|
||||
# a rejected credential rather than a duplicated one. A lone token header
|
||||
# is understood fine; it is only the collision that fails.
|
||||
#
|
||||
# So: strip the headers and hand the token to git-lfs as an ordinary
|
||||
# credential instead. It authenticates the batch call and leaves the
|
||||
# per-object JWT untouched. Gitea authenticates on the password, so the
|
||||
# username is a placeholder. Nothing later in this job talks to the
|
||||
# remote, so dropping checkout's header costs us nothing.
|
||||
#
|
||||
# `continue-on-error` deliberately: if this cannot authenticate, the build
|
||||
# below still runs and fails with the build script's own message, which
|
||||
# names the real problem. A checkout that dies here says nothing.
|
||||
- name: Fetch the segmentation model
|
||||
continue-on-error: true
|
||||
env:
|
||||
LFS_TOKEN: ${{ secrets.GITEA_TOKEN || github.token }}
|
||||
run: |
|
||||
set -e
|
||||
git lfs install --local
|
||||
git config --local --get-regexp '^http\..*extraheader$' \
|
||||
| cut -d' ' -f1 | sort -u \
|
||||
| while read -r key; do git config --local --unset-all "$key"; done || true
|
||||
git config --local lfs.url \
|
||||
"https://x-access-token:${LFS_TOKEN}@gitea.tourolle.paris/dtourolle/DarkRoom.git/info/lfs"
|
||||
git lfs pull --exclude="fixtures/**"
|
||||
ls -lR models/
|
||||
|
||||
- name: Cache cargo
|
||||
uses: actions/cache@v4
|
||||
with:
|
||||
path: |
|
||||
/opt/cargo/registry
|
||||
target-android
|
||||
key: android-${{ hashFiles('**/Cargo.lock') }}
|
||||
|
||||
# A fast gate on the crates most likely to break the cross-compile, run
|
||||
# before the expensive part. It is `cargo check`, so it type-checks
|
||||
# without linking and returns in a fraction of the time the step below
|
||||
# takes.
|
||||
#
|
||||
# Not a statement that only these crates cross-compile — `darkroom-android`
|
||||
# and the whole UI stack beneath it build for aarch64 too, which is what
|
||||
# the API-level step below does. This one exists to fail fast and name a
|
||||
# smaller suspect when it does.
|
||||
- name: Cross-compile core
|
||||
env:
|
||||
CARGO_TARGET_DIR: target-android
|
||||
run: cargo check -p dr-types -p dr-gpu -p dr-sync --target aarch64-linux-android
|
||||
|
||||
# The linker targets MIN_API, not the compile SDK. cargo-ndk otherwise
|
||||
# defaults to API 21, far below the Vulkan floor this app needs — and the
|
||||
# mismatch is invisible until a device refuses to install.
|
||||
#
|
||||
# Look under the target triple, and fail on a mismatch. Searching the
|
||||
# whole target dir for the first `*.so` found the host proc-macro
|
||||
# libraries in target-android/debug/deps instead — x86-64 objects built
|
||||
# by the runner's gcc, whose .comment section says nothing about Android
|
||||
# and can never contradict the expected API. The step passed regardless
|
||||
# of what the linker actually did, which is the one thing it exists to
|
||||
# rule out.
|
||||
- name: Verify minimum API level
|
||||
env:
|
||||
CARGO_TARGET_DIR: target-android
|
||||
run: |
|
||||
set -e
|
||||
# `darkroom-android`, not a core crate: this step reads the API level
|
||||
# out of a *linked* object, and only that crate produces one. It is
|
||||
# the workspace's single `crate-type = ["cdylib"]`; a library crate
|
||||
# builds an rlib, which is an archive of object files that no linker
|
||||
# has yet touched and that `file` therefore has nothing to say about.
|
||||
# Asking for `-p dr-gpu` here could only ever reach the "no aarch64
|
||||
# .so was produced" branch below, whatever the linker did.
|
||||
#
|
||||
# It is also the honest artefact to check: the .so this names is the
|
||||
# one that ships in the APK, so the API level verified here is the
|
||||
# API level a device will refuse to install against.
|
||||
cargo ndk -t arm64-v8a -o target-android/jniLibs \
|
||||
build -p darkroom-android --release
|
||||
MIN_API=$(sed -n 's/^ARG MIN_API=\([0-9]*\).*/\1/p' docker/android/Dockerfile)
|
||||
# Empty on both sides would compare equal and pass, so neither side
|
||||
# is allowed to be the result of a failed parse.
|
||||
if [ -z "$MIN_API" ]; then
|
||||
echo "no ARG MIN_API= in docker/android/Dockerfile"
|
||||
exit 1
|
||||
fi
|
||||
SO=$(find target-android/aarch64-linux-android/release -maxdepth 1 -name '*.so' | head -1)
|
||||
if [ -z "$SO" ]; then
|
||||
echo "no aarch64 .so was produced"
|
||||
exit 1
|
||||
fi
|
||||
echo "checking $SO"
|
||||
# `file` is kept for the log — it names the NDK that built this — but
|
||||
# the check no longer depends on it.
|
||||
file "$SO" || true
|
||||
# The API level is the first word of the `.note.android.ident` ELF
|
||||
# note, little-endian. Read the note rather than asking `file` for it:
|
||||
# `file` only prints "for Android 28" when its magic database is new
|
||||
# enough to decode that note, and this image's is not. The parse then
|
||||
# produced nothing, `${API:-unknown}` reported "unknown", and every
|
||||
# push failed here for weeks on a .so that was linked perfectly
|
||||
# correctly. A note read straight out of the ELF cannot go stale that
|
||||
# way.
|
||||
readelf -n "$SO" | sed -n '/android.ident/,+3p'
|
||||
HEX=$(readelf -n "$SO" 2>/dev/null \
|
||||
| awk '/description data:/ { print $6 $5 $4 $3; exit }')
|
||||
if [ -z "$HEX" ]; then
|
||||
echo "FAIL: no .note.android.ident in $SO — nothing states an API level"
|
||||
exit 1
|
||||
fi
|
||||
API=$(( 0x$HEX ))
|
||||
if [ "$API" != "$MIN_API" ]; then
|
||||
echo "FAIL: linked for Android $API, expected $MIN_API"
|
||||
exit 1
|
||||
fi
|
||||
echo "OK: linked for Android $API"
|
||||
|
||||
# The APK itself, so a run leaves something installable behind rather
|
||||
# than only the knowledge that it would have linked. The assembly is
|
||||
# `docker/android/assemble-apk.sh`, shared with `package.sh` so the file
|
||||
# a device gets from `package.sh --install` and the file published here
|
||||
# are built by the same code — see that script's header.
|
||||
#
|
||||
# `KEYSTORE` deliberately points at a throwaway directory instead of its
|
||||
# default under `target-android`: that directory is what `actions/cache`
|
||||
# restores and saves, and a signing key has no business in a build cache
|
||||
# or in anything this job uploads. A fresh debug key per run is the right
|
||||
# trade for an artefact whose purpose is getting the app onto a test
|
||||
# device; nothing upgrades in place over it, which is the one thing a
|
||||
# stable key would buy.
|
||||
- name: Package the APK
|
||||
env:
|
||||
CARGO_TARGET_DIR: target-android
|
||||
# Absent secrets mean a debug signature, which is what a fork or a
|
||||
# branch build should get. Set all three (see docs/android-signing.md)
|
||||
# and the same job produces a release-signed APK instead.
|
||||
ANDROID_KEYSTORE_BASE64: ${{ secrets.ANDROID_KEYSTORE_BASE64 }}
|
||||
KEYSTORE_PASS: ${{ secrets.ANDROID_KEYSTORE_PASSWORD }}
|
||||
KEY_PASS: ${{ secrets.ANDROID_KEY_PASSWORD }}
|
||||
KEY_ALIAS: ${{ secrets.ANDROID_KEY_ALIAS }}
|
||||
run: |
|
||||
set -e
|
||||
KEYDIR="$(mktemp -d)"
|
||||
chmod 700 "$KEYDIR"
|
||||
trap 'rm -rf "$KEYDIR"' EXIT
|
||||
|
||||
if [ -n "$ANDROID_KEYSTORE_BASE64" ]; then
|
||||
# The keystore reaches the runner base64-encoded because a secret
|
||||
# is a string. It is written under a 0700 mktemp directory, never
|
||||
# into the workspace: `target-android` is what actions/cache saves,
|
||||
# and the upload step globs the workspace.
|
||||
printf '%s' "$ANDROID_KEYSTORE_BASE64" | base64 -d > "$KEYDIR/release.keystore"
|
||||
export KEYSTORE="$KEYDIR/release.keystore"
|
||||
else
|
||||
# Not an error. Unset the rest so assemble-apk.sh takes its debug
|
||||
# path cleanly rather than seeing a half-configured release one.
|
||||
export KEYSTORE="$KEYDIR/debug.keystore"
|
||||
unset KEYSTORE_PASS KEY_PASS KEY_ALIAS
|
||||
fi
|
||||
|
||||
REPO="$PWD" TARGET_DIR="$PWD/target-android" \
|
||||
bash docker/android/assemble-apk.sh
|
||||
|
||||
# v3, not v4. v4 is untested against this Gitea and its runner; v3 is
|
||||
# what JellyTau uploads its APK with on this same runner, so it is the
|
||||
# version known to work here rather than the version that ought to.
|
||||
#
|
||||
# `if-no-files-found: error` because the failure this guards against is
|
||||
# a green run with an empty artefact list, which reads as success until
|
||||
# somebody goes looking for the file.
|
||||
- name: Upload the APK
|
||||
uses: actions/upload-artifact@v3
|
||||
with:
|
||||
name: darkroom-arm64-v8a-apk
|
||||
path: target-android/apk/darkroom.apk
|
||||
if-no-files-found: error
|
||||
|
||||
windows-image:
|
||||
uses: ./.gitea/workflows/windows-image.yml
|
||||
|
||||
# TRACES: FR-PLAT-WIN-3
|
||||
# The Windows executable and its installer, cross-built from Linux
|
||||
# (docs/windows.md §7). No Windows machine anywhere in this job: what it
|
||||
# can prove is that the binary links, is a Windows executable with no
|
||||
# MinGW runtime imports, starts under Wine, and that the installer installs
|
||||
# and uninstalls under Wine. What it cannot prove — a Vulkan device, a
|
||||
# render, the secret store — is a release step on a real machine (§6).
|
||||
windows:
|
||||
runs-on: linux/amd64
|
||||
name: Windows (x86_64, cross)
|
||||
needs: windows-image
|
||||
container:
|
||||
image: gitea.tourolle.paris/dtourolle/darkroom-windows:latest
|
||||
env:
|
||||
CARGO_INCREMENTAL: 0
|
||||
CARGO_PROFILE_DEV_DEBUG: 0
|
||||
CARGO_TARGET_DIR: target-windows
|
||||
# Wine keeps its prefix under $HOME, which the image points at a
|
||||
# directory that does not exist in a fresh container.
|
||||
HOME: /tmp/home
|
||||
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v4
|
||||
|
||||
# Same step as the desktop leg: the models are LFS objects and the
|
||||
# packager refuses pointers.
|
||||
- name: Fetch the models
|
||||
env:
|
||||
LFS_TOKEN: ${{ secrets.GITEA_TOKEN || github.token }}
|
||||
run: |
|
||||
set -e
|
||||
git lfs install --local
|
||||
git config --local --get-regexp '^http\..*extraheader$' \
|
||||
| cut -d' ' -f1 | sort -u \
|
||||
| while read -r key; do git config --local --unset-all "$key"; done || true
|
||||
git config --local lfs.url \
|
||||
"https://x-access-token:${LFS_TOKEN}@gitea.tourolle.paris/dtourolle/DarkRoom.git/info/lfs"
|
||||
git lfs pull --exclude="fixtures/**"
|
||||
ls -l models/face models/scene
|
||||
|
||||
- name: Cache cargo
|
||||
uses: actions/cache@v4
|
||||
with:
|
||||
path: |
|
||||
/opt/cargo/registry
|
||||
target-windows
|
||||
key: windows-${{ hashFiles('**/Cargo.lock') }}
|
||||
|
||||
# The cfg(windows) branches are linted here and nowhere else: the
|
||||
# desktop leg's clippy never compiles them.
|
||||
- name: Clippy for the target
|
||||
run: cargo clippy --release --target x86_64-pc-windows-gnu -p darkroom-desktop -- -D warnings
|
||||
|
||||
- name: Build
|
||||
run: cargo build --release --target x86_64-pc-windows-gnu -p darkroom-desktop
|
||||
|
||||
- name: Smoke-test the executable
|
||||
run: |
|
||||
set -e
|
||||
mkdir -p "$HOME"
|
||||
EXE=target-windows/x86_64-pc-windows-gnu/release/darkroom-desktop.exe
|
||||
file "$EXE"
|
||||
file "$EXE" | grep -q 'PE32+' || { echo "FAIL: not a PE32+ executable"; exit 1; }
|
||||
file "$EXE" | grep -q '(GUI)' || { echo "FAIL: not a GUI-subsystem executable"; exit 1; }
|
||||
if x86_64-w64-mingw32-objdump -p "$EXE" | grep -iE 'libwinpthread|libgcc|libstdc'; then
|
||||
echo "FAIL: the executable imports a MinGW runtime DLL"
|
||||
exit 1
|
||||
fi
|
||||
x86_64-w64-mingw32-objdump -p "$EXE" | grep 'DLL Name' | sort -u
|
||||
wineboot --init >/dev/null 2>&1 || true
|
||||
OUT=$(wine "$EXE" --version 2>/dev/null)
|
||||
echo "wine: $OUT"
|
||||
echo "$OUT" | grep -q '^darkroom-desktop ' || { echo "FAIL: --version did not answer under Wine"; exit 1; }
|
||||
|
||||
- name: Package the installer
|
||||
run: bash docker/windows/package.sh
|
||||
|
||||
- name: Smoke-test the installer
|
||||
run: |
|
||||
set -e
|
||||
SETUP=$(ls target-windows/installer/DarkRoom-*-x86_64-setup.exe)
|
||||
file "$SETUP" | grep -q 'PE32+' || { echo "FAIL: the installer is not 64-bit"; exit 1; }
|
||||
wine "$SETUP" /S 2>/dev/null
|
||||
INST=$(echo "$HOME"/.wine/drive_c/users/*/AppData/Local/Programs/DarkRoom)
|
||||
ls "$INST"
|
||||
[ "$(ls "$INST/models" | wc -l)" = 7 ] || { echo "FAIL: expected 7 model files"; exit 1; }
|
||||
wine reg query 'HKCU\Software\Microsoft\Windows\CurrentVersion\Uninstall\DarkRoom' 2>/dev/null \
|
||||
| grep -q DisplayVersion || { echo "FAIL: no uninstall registry key"; exit 1; }
|
||||
wine "$INST/darkroom.exe" --version 2>/dev/null | grep -q '^darkroom-desktop ' \
|
||||
|| { echo "FAIL: the installed executable does not run"; exit 1; }
|
||||
wine "$INST/uninstall.exe" /S 2>/dev/null
|
||||
sleep 3
|
||||
[ ! -e "$INST" ] || { echo "FAIL: uninstall left $INST behind"; ls -R "$INST"; exit 1; }
|
||||
echo "OK: installed and uninstalled under Wine"
|
||||
|
||||
- name: Upload the installer
|
||||
uses: actions/upload-artifact@v3
|
||||
with:
|
||||
name: darkroom-windows-x86_64-setup
|
||||
path: target-windows/installer/DarkRoom-*-x86_64-setup.exe
|
||||
if-no-files-found: error
|
||||
|
||||
layering:
|
||||
runs-on: linux/amd64
|
||||
name: Layer separation
|
||||
# Node for the JS actions, as above. cargo comes from rustup below.
|
||||
container:
|
||||
image: catthehacker/ubuntu:act-latest
|
||||
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v4
|
||||
|
||||
# `cargo tree` resolves the dependency graph, so it needs the registry
|
||||
# index but no system libraries — this job builds nothing.
|
||||
#
|
||||
# rust-analyzer is named for the reason given in the desktop job: rustup
|
||||
# installs rust-toolchain.toml's components on the first cargo call
|
||||
# whether or not this step asks for them, and an unasked-for download is
|
||||
# the one nobody can find in the log.
|
||||
- name: Install Rust 1.92.0
|
||||
run: |
|
||||
set -e
|
||||
curl -fsSL https://sh.rustup.rs | sh -s -- \
|
||||
-y --no-modify-path --profile minimal --default-toolchain 1.92.0 \
|
||||
--component rust-analyzer
|
||||
echo "$HOME/.cargo/bin" >> "$GITHUB_PATH"
|
||||
|
||||
# ARCH §6.5a: no core/ crate may depend on the UI toolkit. One stray
|
||||
# `use slint::` costs headless golden-image testing and the
|
||||
# one-operation-two-presentations property together, and nothing else
|
||||
# would notice.
|
||||
- name: Core crates must not depend on the UI
|
||||
run: |
|
||||
set -e
|
||||
FAILED=0
|
||||
for crate in dr-types dr-gpu dr-sync; do
|
||||
if cargo tree -p "$crate" -e normal 2>/dev/null | grep -qE '\bslint\b|\bi-slint'; then
|
||||
echo "FAIL: $crate depends on Slint (ARCH §6.5a)"
|
||||
FAILED=1
|
||||
else
|
||||
echo "ok: $crate"
|
||||
fi
|
||||
done
|
||||
exit $FAILED
|
||||
@@ -1,130 +0,0 @@
|
||||
name: Traceability
|
||||
|
||||
# Mirrors JellyTau's traceability gate, including the reason it exists.
|
||||
#
|
||||
# That gate divided a traced count by frozen literal denominators while the
|
||||
# requirements file grew past them, reported 158% coverage, and so could never
|
||||
# fail its own threshold. Two rules follow, and the extractor's own tests
|
||||
# enforce both:
|
||||
#
|
||||
# 1. Denominators are parsed from docs/requirements.md at run time.
|
||||
# 2. Coverage is |traced ∩ defined| / |defined|, never a raw traced count.
|
||||
#
|
||||
# This job is static analysis of source comments plus markdown parsing, so it
|
||||
# needs no GPU and no Android SDK — only the Rust toolchain.
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [main, master, develop]
|
||||
pull_request:
|
||||
branches: [main, master, develop]
|
||||
|
||||
jobs:
|
||||
traceability:
|
||||
runs-on: linux/amd64
|
||||
name: Requirement traces
|
||||
# Node for actions/checkout and actions/cache, which the bare runner image
|
||||
# cannot execute. Rust is installed below.
|
||||
container:
|
||||
image: catthehacker/ubuntu:act-latest
|
||||
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
fetch-depth: 0
|
||||
|
||||
- name: Cache cargo
|
||||
uses: actions/cache@v4
|
||||
with:
|
||||
path: |
|
||||
~/.cargo/registry
|
||||
~/.cargo/git
|
||||
target
|
||||
key: traces-${{ runner.os }}-${{ hashFiles('**/Cargo.lock') }}
|
||||
|
||||
# Source-comment and markdown parsing only, so the minimal profile is
|
||||
# enough — no system libraries and nothing this job itself needs beyond
|
||||
# cargo. rust-analyzer is here anyway because rust-toolchain.toml lists
|
||||
# it: rustup installs that file's components on the first cargo call in
|
||||
# the work tree regardless, and a download named in the install step
|
||||
# beats the same download appearing unannounced inside the gate.
|
||||
- name: Install Rust 1.92.0
|
||||
run: |
|
||||
set -e
|
||||
curl -fsSL https://sh.rustup.rs | sh -s -- \
|
||||
-y --no-modify-path --profile minimal --default-toolchain 1.92.0 \
|
||||
--component rust-analyzer
|
||||
echo "$HOME/.cargo/bin" >> "$GITHUB_PATH"
|
||||
|
||||
# The gate's own arithmetic is the thing being trusted, so its tests run
|
||||
# before it does. Untested gate logic is exactly how JellyTau's 158% went
|
||||
# unnoticed for months.
|
||||
- name: Test the extractor
|
||||
run: cargo test -p traceability
|
||||
|
||||
# Structural failures are unconditional and do not depend on the coverage
|
||||
# threshold: zero requirements parsed, zero files scanned, a ratio above
|
||||
# 100%, or any orphan tag all fail the build. A misconfigured run must not
|
||||
# report a plausible-looking 0%.
|
||||
- name: Traceability gate
|
||||
run: cargo run -q -p traceability -- check
|
||||
|
||||
- name: Regenerate matrix and check it is committed
|
||||
run: |
|
||||
set -e
|
||||
cargo run -q -p traceability -- report
|
||||
if ! git diff --quiet docs/traceability.md; then
|
||||
echo ""
|
||||
echo "docs/traceability.md is out of date."
|
||||
echo "Run: cargo run -p traceability -- report"
|
||||
git diff --stat docs/traceability.md
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# The gesture vocabulary, from the same scanner and under the same rule.
|
||||
#
|
||||
# Blocking, and for a sharper reason than the matrix: these two artefacts
|
||||
# are not only read, one of them is *shown to the user*. A stale
|
||||
# `gesture_book.rs` is a help sheet in the application telling somebody to
|
||||
# perform a gesture that was removed — worse than no help sheet, because
|
||||
# they will conclude the application is broken rather than the page.
|
||||
#
|
||||
# This also fails on a malformed tag, so a typo costs a gesture its
|
||||
# desktop half loudly rather than silently.
|
||||
- name: Regenerate the gesture vocabulary and check it is committed
|
||||
run: cargo run -q -p traceability -- gestures-check
|
||||
|
||||
# Advisory, not blocking: not every file implements a requirement, and a
|
||||
# tag on every function is noise that rots faster than it helps. Tag the
|
||||
# unit that decides.
|
||||
- name: Check changed files for tags
|
||||
if: github.event_name == 'pull_request'
|
||||
run: |
|
||||
set -e
|
||||
CHANGED=$(git diff --name-only "origin/${{ github.base_ref }}...HEAD" \
|
||||
| grep -E '\.(rs|slint|wgsl)$' || true)
|
||||
[ -z "$CHANGED" ] && { echo "No source files changed."; exit 0; }
|
||||
|
||||
MISSING=0
|
||||
for file in $CHANGED; do
|
||||
case "$file" in
|
||||
*/tests/*|*/test_*|tools/*) continue ;;
|
||||
esac
|
||||
[ -f "$file" ] || continue
|
||||
if ! grep -q 'TRACES:' "$file"; then
|
||||
echo " no TRACES tag: $file"
|
||||
MISSING=$((MISSING + 1))
|
||||
fi
|
||||
done
|
||||
|
||||
if [ "$MISSING" -gt 0 ]; then
|
||||
echo ""
|
||||
echo "$MISSING changed file(s) carry no requirement tag."
|
||||
echo "Format: /// TRACES: FR-CAT-1, FR-CAT-2 | NFR-P1"
|
||||
echo " (comma separates IDs, pipe groups types)"
|
||||
fi
|
||||
|
||||
- name: Summary
|
||||
if: always()
|
||||
run: head -30 docs/traceability.md || true
|
||||
@@ -1,170 +0,0 @@
|
||||
name: '🐳 Windows image'
|
||||
|
||||
# Builds and pushes gitea.tourolle.paris/dtourolle/darkroom-windows, the job
|
||||
# container for the Windows leg of build-and-test.yml.
|
||||
#
|
||||
# The same shape as android-image.yml, for the same reason that one exists:
|
||||
# an image that lives only on a developer's laptop is a job that dies at
|
||||
# `docker pull`. Built from docker/windows, tagged by that directory's tree
|
||||
# id, skipped when the registry already has it.
|
||||
#
|
||||
# Called by build-and-test.yml on every push, and runnable by hand via
|
||||
# workflow_dispatch. It is cheap when nothing changed — see the guard below.
|
||||
on:
|
||||
workflow_call:
|
||||
inputs:
|
||||
force:
|
||||
description: 'Rebuild even if the registry already has this image ("true"/"false")'
|
||||
type: string
|
||||
default: 'false'
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
force:
|
||||
description: 'Rebuild even if the registry already has this image ("true"/"false")'
|
||||
type: string
|
||||
default: 'false'
|
||||
|
||||
# Gitea's act_runner mangles boolean workflow inputs passed through an
|
||||
# expression — they arrive as false regardless of what was sent. Every input
|
||||
# here is a string compared with == 'true', as in KPN's docker.yaml.
|
||||
|
||||
env:
|
||||
IMAGE: gitea.tourolle.paris/dtourolle/darkroom-windows
|
||||
|
||||
jobs:
|
||||
build:
|
||||
runs-on: linux/amd64
|
||||
name: Build and push
|
||||
# Deliberately NOT in a container: this job needs the host Docker daemon to
|
||||
# build an image, and the host's cached ~/.docker/config.json to push it.
|
||||
# That is also why there is no `docker login` step — the runner host was
|
||||
# authenticated to the registry during setup.
|
||||
|
||||
steps:
|
||||
# The host has no Node, so the JS-based actions/checkout cannot run here.
|
||||
# A minimal shallow fetch with plain git gets the same tree.
|
||||
- name: Checkout
|
||||
run: |
|
||||
set -e
|
||||
git init -q .
|
||||
git remote add origin "${{ github.server_url }}/${{ github.repository }}.git"
|
||||
git -c http.extraheader="AUTHORIZATION: basic $(printf '%s' '${{ github.actor }}:${{ github.token }}' | base64 -w0)" \
|
||||
fetch --depth 1 origin "${{ github.sha }}"
|
||||
git checkout -q FETCH_HEAD
|
||||
|
||||
# The image is tagged by the content of docker/windows, not by the commit
|
||||
# that happened to touch it. `git rev-parse HEAD:<dir>` is the tree object
|
||||
# id — it changes when and only when a file in that directory changes, so
|
||||
# an unrelated push reuses the existing image and a Dockerfile edit can
|
||||
# never silently keep serving a stale `latest`.
|
||||
#
|
||||
# Using the commit sha instead would rebuild 2.5 GB on every push; using a
|
||||
# paths-filter action would need a container that has Node, and the only
|
||||
# one this repo would reach for is the very image being built.
|
||||
- name: Resolve image tag
|
||||
id: tag
|
||||
run: |
|
||||
set -e
|
||||
TREE=$(git rev-parse HEAD:docker/windows)
|
||||
echo "tree=$TREE" >> "$GITHUB_OUTPUT"
|
||||
echo "docker/windows tree: $TREE"
|
||||
|
||||
# Skip the build when the registry already holds this exact content. This
|
||||
# is what keeps the job a few seconds long on a normal push, and what
|
||||
# makes it self-healing: if the tag is missing for any reason, including
|
||||
# the image having never been pushed at all, it gets built here.
|
||||
#
|
||||
# The probe is curl against the registry API, NOT `docker manifest
|
||||
# inspect`. The latter exits 1 on this registry even for tags that are
|
||||
# demonstrably present — jellytau-builder:latest answers HTTP 200 to the
|
||||
# API while `docker manifest inspect` reports "manifest unknown" for it.
|
||||
# Trusting that would have rebuilt 7 GB on every single push.
|
||||
#
|
||||
# A HEAD request also gives the digest for free, which is how the repoint
|
||||
# decision below is made without pulling any layers.
|
||||
- name: Query registry
|
||||
id: check
|
||||
env:
|
||||
# The runner's own credentials, so this does not depend on how the
|
||||
# host's ~/.docker/config.json happens to be set up.
|
||||
REG_USER: ${{ github.actor }}
|
||||
REG_PASS: ${{ github.token }}
|
||||
TREE: ${{ steps.tag.outputs.tree }}
|
||||
run: |
|
||||
set -eu
|
||||
ACCEPT='application/vnd.oci.image.index.v1+json,application/vnd.docker.distribution.manifest.v2+json,application/vnd.oci.image.manifest.v1+json,application/vnd.docker.distribution.manifest.list.v2+json'
|
||||
API="https://gitea.tourolle.paris/v2/dtourolle/darkroom-windows/manifests"
|
||||
|
||||
# Prints "<http-status> <digest-or-empty>" for a tag.
|
||||
probe() {
|
||||
curl -sI -u "$REG_USER:$REG_PASS" -H "Accept: $ACCEPT" "$API/$1" \
|
||||
| tr -d '\r' \
|
||||
| awk 'BEGIN{s="000";d=""} /^HTTP/{s=$2} tolower($1)=="docker-content-digest:"{d=$2} END{print s, d}'
|
||||
}
|
||||
|
||||
read -r TREE_STATUS TREE_DIGEST <<EOF
|
||||
$(probe "$TREE")
|
||||
EOF
|
||||
read -r LATEST_STATUS LATEST_DIGEST <<EOF
|
||||
$(probe latest)
|
||||
EOF
|
||||
|
||||
echo "tag $TREE -> HTTP $TREE_STATUS ${TREE_DIGEST:-(no digest)}"
|
||||
echo "tag latest -> HTTP $LATEST_STATUS ${LATEST_DIGEST:-(no digest)}"
|
||||
|
||||
# Build unless the registry definitively confirms this content is
|
||||
# already there. An auth failure or an unreachable registry lands
|
||||
# here too, and rebuilding needlessly is the safe direction to fail —
|
||||
# skipping a build that was needed is what breaks the Windows job.
|
||||
if [ "${{ inputs.force }}" = "true" ]; then
|
||||
echo "forced rebuild requested"
|
||||
echo "build=true" >> "$GITHUB_OUTPUT"
|
||||
echo "repoint=false" >> "$GITHUB_OUTPUT"
|
||||
elif [ "$TREE_STATUS" != "200" ]; then
|
||||
echo "registry does not have this content — building"
|
||||
echo "build=true" >> "$GITHUB_OUTPUT"
|
||||
echo "repoint=false" >> "$GITHUB_OUTPUT"
|
||||
elif [ -n "$TREE_DIGEST" ] && [ "$TREE_DIGEST" = "$LATEST_DIGEST" ]; then
|
||||
echo "registry is already correct — nothing to do"
|
||||
echo "build=false" >> "$GITHUB_OUTPUT"
|
||||
echo "repoint=false" >> "$GITHUB_OUTPUT"
|
||||
else
|
||||
echo "content is present but latest points elsewhere — repointing"
|
||||
echo "build=false" >> "$GITHUB_OUTPUT"
|
||||
echo "repoint=true" >> "$GITHUB_OUTPUT"
|
||||
fi
|
||||
|
||||
# Context is docker/windows, matching the README's build command. The
|
||||
# Dockerfile COPYs nothing from the repo, so it needs no wider context —
|
||||
# and a narrow context keeps the daemon from tarring up the whole tree,
|
||||
# target/ included.
|
||||
- name: Build
|
||||
if: ${{ steps.check.outputs.build == 'true' }}
|
||||
run: |
|
||||
set -e
|
||||
docker build \
|
||||
-t "$IMAGE:${{ steps.tag.outputs.tree }}" \
|
||||
-t "$IMAGE:latest" \
|
||||
docker/windows
|
||||
|
||||
# Both tags are pushed: the tree tag is what the guard above looks for on
|
||||
# the next run, and `latest` is what build-and-test.yml pulls.
|
||||
- name: Push
|
||||
if: ${{ steps.check.outputs.build == 'true' }}
|
||||
run: |
|
||||
set -e
|
||||
docker push "$IMAGE:${{ steps.tag.outputs.tree }}"
|
||||
docker push "$IMAGE:latest"
|
||||
|
||||
# A cache hit on the tree tag says nothing about where `latest` points — a
|
||||
# reverted Dockerfile or a build from another branch can leave it on
|
||||
# different content. This runs only when the digests above actually
|
||||
# disagree, so the common case costs nothing; the layers are already in
|
||||
# the registry, so the push that follows uploads a manifest, not 2.5 GB.
|
||||
- name: Repoint latest
|
||||
if: ${{ steps.check.outputs.repoint == 'true' }}
|
||||
run: |
|
||||
set -e
|
||||
docker pull "$IMAGE:${{ steps.tag.outputs.tree }}"
|
||||
docker tag "$IMAGE:${{ steps.tag.outputs.tree }}" "$IMAGE:latest"
|
||||
docker push "$IMAGE:latest"
|
||||
@@ -1,3 +0,0 @@
|
||||
#!/bin/sh
|
||||
command -v git-lfs >/dev/null 2>&1 || { printf >&2 "\n%s\n\n" "This repository is configured for Git LFS but 'git-lfs' was not found on your path. If you no longer wish to use Git LFS, remove this hook by deleting the 'post-checkout' file in the hooks directory (set by 'core.hookspath'; usually '.git/hooks')."; exit 2; }
|
||||
git lfs post-checkout "$@"
|
||||
@@ -1,3 +0,0 @@
|
||||
#!/bin/sh
|
||||
command -v git-lfs >/dev/null 2>&1 || { printf >&2 "\n%s\n\n" "This repository is configured for Git LFS but 'git-lfs' was not found on your path. If you no longer wish to use Git LFS, remove this hook by deleting the 'post-commit' file in the hooks directory (set by 'core.hookspath'; usually '.git/hooks')."; exit 2; }
|
||||
git lfs post-commit "$@"
|
||||
@@ -1,3 +0,0 @@
|
||||
#!/bin/sh
|
||||
command -v git-lfs >/dev/null 2>&1 || { printf >&2 "\n%s\n\n" "This repository is configured for Git LFS but 'git-lfs' was not found on your path. If you no longer wish to use Git LFS, remove this hook by deleting the 'post-merge' file in the hooks directory (set by 'core.hookspath'; usually '.git/hooks')."; exit 2; }
|
||||
git lfs post-merge "$@"
|
||||
@@ -1,67 +0,0 @@
|
||||
#!/usr/bin/env bash
|
||||
# Keep the generated artefacts in step with the tags in the tree.
|
||||
#
|
||||
# Two of them now, from the same scanner: the requirements matrix and the
|
||||
# gesture vocabulary. Both are generated *from* the tree and cite line numbers
|
||||
# in it, so both go stale on any commit that moves a line — a `cargo fmt` sweep
|
||||
# above all, but equally a commit that merely adds a paragraph above a tag.
|
||||
#
|
||||
# The gate regenerates the matrix in CI and fails if the result differs from
|
||||
# what is committed. That is the right check — a matrix that disagrees with the
|
||||
# tree is worse than none, because it is read as current — but it fails *after*
|
||||
# a push, on a commit that is otherwise fine, and it has now done so on six
|
||||
# commits in a row because adding a `TRACES:` tag and regenerating the matrix
|
||||
# are two actions and only the first is on anyone's mind.
|
||||
#
|
||||
# So it happens here instead, where the tags are being changed.
|
||||
#
|
||||
# Only when something that can carry a tag is staged: a commit touching
|
||||
# workflows, packaging or the matrix itself pays nothing.
|
||||
set -euo pipefail
|
||||
|
||||
staged="$(git diff --cached --name-only --diff-filter=ACMR)"
|
||||
if ! grep -qE '\.(rs|slint|yaml|md)$' <<< "${staged}"; then
|
||||
exit 0
|
||||
fi
|
||||
# The artefacts are generated from the tree, so regenerating them because one
|
||||
# was itself edited would be circular.
|
||||
case "$(tr -d '[:space:]' <<< "${staged}")" in
|
||||
docs/traceability.md | docs/gestures.md | ui/dr-ui/src/gesture_book.rs)
|
||||
exit 0
|
||||
;;
|
||||
esac
|
||||
|
||||
repo="$(git rev-parse --show-toplevel)"
|
||||
cd "${repo}"
|
||||
|
||||
# Quiet unless it has something to say. A hook that prints on every commit is
|
||||
# a hook people start passing --no-verify to.
|
||||
if ! cargo run -q -p traceability -- report >/dev/null 2>&1; then
|
||||
echo "pre-commit: could not run the traceability report; leaving the matrix alone" >&2
|
||||
exit 0
|
||||
fi
|
||||
|
||||
if ! git diff --quiet -- docs/traceability.md; then
|
||||
git add docs/traceability.md
|
||||
echo "pre-commit: regenerated docs/traceability.md and staged it"
|
||||
fi
|
||||
|
||||
# The gesture vocabulary, same discipline.
|
||||
#
|
||||
# **Failure here is reported and not swallowed**, unlike the matrix above. A
|
||||
# matrix that will not build leaves the previous one in place, which is merely
|
||||
# stale; a malformed `GESTURE:` block means a gesture the user is about to be
|
||||
# told about in the wrong words, or not at all. The gate would catch it in CI
|
||||
# either way — this is only about catching it a push earlier.
|
||||
if ! out="$(cargo run -q -p traceability -- gestures 2>&1)"; then
|
||||
echo "pre-commit: the gesture scan failed — the tags below need fixing" >&2
|
||||
echo "${out}" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
for f in docs/gestures.md ui/dr-ui/src/gesture_book.rs; do
|
||||
if ! git diff --quiet -- "${f}"; then
|
||||
git add "${f}"
|
||||
echo "pre-commit: regenerated ${f} and staged it"
|
||||
fi
|
||||
done
|
||||
@@ -1,3 +0,0 @@
|
||||
#!/bin/sh
|
||||
command -v git-lfs >/dev/null 2>&1 || { printf >&2 "\n%s\n\n" "This repository is configured for Git LFS but 'git-lfs' was not found on your path. If you no longer wish to use Git LFS, remove this hook by deleting the 'pre-push' file in the hooks directory (set by 'core.hookspath'; usually '.git/hooks')."; exit 2; }
|
||||
git lfs pre-push "$@"
|
||||
-26
@@ -1,26 +0,0 @@
|
||||
/target
|
||||
/target-android
|
||||
Cargo.lock.bak
|
||||
*.log
|
||||
|
||||
# makepkg build products. `packaging/PKGBUILD` and the .desktop entry are
|
||||
# sources and belong in the tree; everything makepkg derives from them does
|
||||
# not — `pkg/` and `src/` are staging directories it recreates on every run,
|
||||
# and the package itself is 33 MB of compiled output.
|
||||
/packaging/pkg/
|
||||
/packaging/src/
|
||||
/packaging/*.pkg.tar.*
|
||||
/packaging/*.log
|
||||
|
||||
# Cached upstream film profiles, re-fetchable with
|
||||
# tools/film-profiles/convert.py --fetch. Not source: the converted
|
||||
# profiles in core/dr-film/profiles are.
|
||||
tools/film-profiles/upstream/
|
||||
|
||||
# flatpak-builder's cache and its output tree. `packaging/flatpak/` holds the
|
||||
# manifest, which is source; everything a build derives from it is not — and
|
||||
# `.flatpak-builder/` in particular caches an unpacked copy of the whole
|
||||
# checkout, so it is larger than the repository it sits in.
|
||||
/.flatpak-builder/
|
||||
/build/
|
||||
__pycache__/
|
||||
-180
@@ -1,180 +0,0 @@
|
||||
# Contributing to DarkRoom
|
||||
|
||||
There is a lot of documentation here — 14 documents and 177 numbered
|
||||
requirements — and almost all of it is written for someone who has already
|
||||
decided to work on this. This file is the other thing: how to get a first
|
||||
change landed without reading any of it.
|
||||
|
||||
## The shortest useful contribution
|
||||
|
||||
**A develop operation is one file.** Not one file plus a registration, plus a
|
||||
shader edit, plus a control in the UI — one file:
|
||||
|
||||
```
|
||||
core/dr-pipeline/ops/split_toning.yaml
|
||||
```
|
||||
|
||||
`build.rs` finds it with `read_dir`, compiles it into Rust implementing
|
||||
`Operation`, and from there it is indistinguishable from a hand-written node.
|
||||
It arrives with controls built from its declared parameter kinds, a place in
|
||||
the chain from `order:`, a place in the panel from `attributes:`, sidecar
|
||||
persistence, and its own tests — which are declared in the same file and run
|
||||
under `cargo test`.
|
||||
|
||||
Read [`core/dr-pipeline/ops/README.md`](core/dr-pipeline/ops/README.md) and
|
||||
copy [`exposure.yaml`](core/dr-pipeline/ops/exposure.yaml). Split toning,
|
||||
colour zones, selective colour and channel-mixer variants are all pure point
|
||||
operations, which means all of them are declarations rather than code.
|
||||
|
||||
If you want to understand one thing about the architecture before starting,
|
||||
make it this: **the core describes its capabilities and the interface composes
|
||||
them.** No code in `ui/` names an operation, and a test enforces that
|
||||
(`ui/dr-ui/tests/ui_names_no_operation.rs`). It is why your node needs no UI
|
||||
change.
|
||||
|
||||
## Getting it to build
|
||||
|
||||
**Git LFS is required.** Model weights are stored in LFS, and a clone made
|
||||
without it leaves a ~130-byte text pointer where an 11 MB model should be:
|
||||
|
||||
```bash
|
||||
git lfs install && git lfs pull
|
||||
```
|
||||
|
||||
Forget this and `dr-segment`'s build script stops with an instruction rather
|
||||
than embedding the pointer and failing at inference time — but it is easier to
|
||||
run the two commands now.
|
||||
|
||||
**The toolchain pins itself.** `rust-toolchain.toml` selects 1.92.0 and rustup
|
||||
fetches it on first use. Do not override it; `cargo fmt` and `clippy` are both
|
||||
version-sensitive and CI runs exactly this version.
|
||||
|
||||
**System packages.** Slint and winit need these at build time. On Debian or
|
||||
Ubuntu:
|
||||
|
||||
```bash
|
||||
sudo apt-get install pkg-config libfontconfig1-dev libxkbcommon-dev
|
||||
```
|
||||
|
||||
**Then:**
|
||||
|
||||
```bash
|
||||
cargo run -p darkroom-desktop
|
||||
```
|
||||
|
||||
The first build resolves 826 crates and takes a while — on a laptop, long
|
||||
enough to look like a hang. It is not one.
|
||||
|
||||
Android is a containerised toolchain and is not needed for most work; see
|
||||
[`docker/android/README.md`](docker/android/README.md) if you get there.
|
||||
|
||||
## What CI will check
|
||||
|
||||
All four of these run on every push, so run them before you send anything:
|
||||
|
||||
```bash
|
||||
cargo fmt --all -- --check
|
||||
cargo clippy --workspace --all-targets -- -D warnings
|
||||
cargo test --workspace
|
||||
cargo build --workspace --release
|
||||
```
|
||||
|
||||
GPU tests skip themselves where there is no adapter rather than failing — a
|
||||
test that cannot run is not evidence either way — so a green run on a machine
|
||||
without a GPU is expected, and does not mean the GPU paths were exercised.
|
||||
|
||||
There is a fifth check, and it is not in that list because you are unlikely to
|
||||
break it by accident:
|
||||
|
||||
```bash
|
||||
cargo run --release -p dr-bench -- check
|
||||
```
|
||||
|
||||
That is the benchmark suite (`docs/requirements.md` §8), which builds a
|
||||
synthetic 50,000-image catalog and fails the build if a performance target is
|
||||
missed or a measurement has drifted past its tolerance. It runs on every push in
|
||||
its own workflow. [`docs/benchmarks.md`](docs/benchmarks.md) says what it
|
||||
measures, what it deliberately does not, and how to read a failure. If you have
|
||||
touched the catalog, the decoder, the thumbnail store or the exporter, run it
|
||||
before you send.
|
||||
|
||||
## Requirements and traceability
|
||||
|
||||
[`requirements.md`](docs/requirements.md) is the register of record.
|
||||
[`traceability.md`](docs/traceability.md) is generated from `TRACES:` tags in
|
||||
the source and must never be hand-edited:
|
||||
|
||||
```rust
|
||||
// TRACES: FR-DEV-3a | FR-DEV-3c
|
||||
```
|
||||
|
||||
Tags are read from `.rs`, `.slint`, `.wgsl` and `.yaml` — the last so a
|
||||
declared operation can record the requirement it satisfies, since the Rust it
|
||||
generates lands in `OUT_DIR` and is not scanned.
|
||||
|
||||
A pre-commit hook regenerates the matrix and stages it whenever you touch
|
||||
something that can carry a tag, so you should not have to think about it. If
|
||||
you do need to run it by hand:
|
||||
|
||||
```bash
|
||||
cargo run -p traceability -- report
|
||||
```
|
||||
|
||||
Note that it tracks line numbers, so a change that only moves code still moves
|
||||
the matrix. Never regenerate it with a stale prebuilt binary.
|
||||
|
||||
**One convention that the tooling cannot enforce.** A tag proves that a tag
|
||||
exists, not that the code under it does the thing — `docs/code-health.md`
|
||||
CH-4 has the details, and two requirements currently read as covered on the
|
||||
strength of plumbing a future feature would use. So: **close a requirement
|
||||
with a test that would fail if the behaviour were removed.** Coverage that
|
||||
moves slowly and means something beats coverage that moves quickly.
|
||||
|
||||
## Two invariants the build defends
|
||||
|
||||
Worth knowing before you trip one, because both failures name a requirement
|
||||
rather than a line:
|
||||
|
||||
- **No operation may be named in `ui/`** (FR-DEV-3a). Special-casing one
|
||||
operation in the panel to fix a layout problem is how a generated interface
|
||||
stops being generated. If a node needs presentation the panel cannot give it,
|
||||
the answer is a `presentation:` hint in the declaration and a `WidgetKind`,
|
||||
not a branch in `develop.rs`.
|
||||
- **The operation schema rejects ambiguity at build time**: a duplicate
|
||||
`order:`, a filename disagreeing with its `id:`, a default outside its own
|
||||
range, an expression naming something that is not a parameter. Each error
|
||||
names the key you got wrong and exits rather than panicking.
|
||||
|
||||
## Commit messages
|
||||
|
||||
Imperative subject describing the change from the reader's side — "Offer the
|
||||
merge when two people turn out to share a name", not "fix: merge dialog". No
|
||||
conventional-commits prefixes.
|
||||
|
||||
The body is where the reasoning goes, and it is expected to be substantial when
|
||||
the change is. This codebase records *why* far more than most, in commits and
|
||||
in comments alike, and that is the single habit most worth adopting: the
|
||||
constraint you worked around is invisible to whoever reads the diff next.
|
||||
|
||||
One commit per change. If you fixed two things, that is two commits.
|
||||
|
||||
## Where to read next, in order
|
||||
|
||||
| Document | Read it when |
|
||||
|---|---|
|
||||
| [`core/dr-pipeline/ops/README.md`](core/dr-pipeline/ops/README.md) | Adding or changing a develop operation — start here regardless |
|
||||
| [`docs/architecture.md`](docs/architecture.md) | Anything touching the render path, catalog or sync |
|
||||
| [`docs/code-health.md`](docs/code-health.md) | Deciding what to work on; grades each seam by what it costs |
|
||||
| [`docs/benchmarks.md`](docs/benchmarks.md) | A change that could plausibly cost time or memory |
|
||||
| [`docs/technical-debt.md`](docs/technical-debt.md) | Something looks wrong — check it was not chosen |
|
||||
| [`docs/distribution.md`](docs/distribution.md) | Packaging a build, or adding a permission to one |
|
||||
| [`docs/requirements.md`](docs/requirements.md) | Reference, not reading |
|
||||
|
||||
`technical-debt.md` is the one to check before "fixing" anything surprising.
|
||||
It records compromises that were deliberate, each with the reasoning and a
|
||||
falsifiable condition for when it stops being one — the point being that you
|
||||
can tell a constraint from an accident without asking.
|
||||
|
||||
## Licence
|
||||
|
||||
GPL-3.0-or-later. By contributing you agree your work is licensed the same way.
|
||||
Generated
-8848
File diff suppressed because it is too large
Load Diff
-264
@@ -1,264 +0,0 @@
|
||||
[workspace]
|
||||
resolver = "2"
|
||||
members = [
|
||||
"core/dr-types",
|
||||
"core/dr-catalog",
|
||||
"core/dr-thumbs",
|
||||
"core/dr-decode",
|
||||
"core/dr-export",
|
||||
"core/dr-face",
|
||||
"core/dr-film",
|
||||
"core/dr-inference-engine",
|
||||
"core/dr-ingest",
|
||||
"core/dr-gpu",
|
||||
"core/dr-lens",
|
||||
"core/dr-pano",
|
||||
"core/dr-pipeline",
|
||||
"core/dr-preset-xmp",
|
||||
"core/dr-segment",
|
||||
"core/dr-sync",
|
||||
"core/dr-sync-folder",
|
||||
"core/dr-sync-nextcloud",
|
||||
"core/dr-xmp",
|
||||
"platform/dr-plat",
|
||||
"ui/dr-ui",
|
||||
"apps/darkroom-desktop",
|
||||
"apps/darkroom-android",
|
||||
"tools/bench",
|
||||
"tools/traceability",
|
||||
]
|
||||
|
||||
[workspace.package]
|
||||
version = "0.13.1"
|
||||
edition = "2021"
|
||||
rust-version = "1.92"
|
||||
license = "GPL-3.0-or-later"
|
||||
repository = "https://github.com/dtourolle/DarkRoom"
|
||||
|
||||
[workspace.dependencies]
|
||||
# Internal
|
||||
dr-types = { path = "core/dr-types" }
|
||||
dr-catalog = { path = "core/dr-catalog" }
|
||||
dr-thumbs = { path = "core/dr-thumbs" }
|
||||
dr-decode = { path = "core/dr-decode" }
|
||||
dr-export = { path = "core/dr-export" }
|
||||
# Stated explicitly for the same reason as `dr-segment` below: no dependant
|
||||
# should drag in an ONNX runtime by accident. Members opt in with
|
||||
# `features = ["inference"]`.
|
||||
dr-face = { path = "core/dr-face", default-features = false }
|
||||
dr-film = { path = "core/dr-film" }
|
||||
# `tract` on by default so a test binary can open a session with nothing
|
||||
# installed; the apps add `native` to look for a runtime file (docs/inference.md §3).
|
||||
dr-inference-engine = { path = "core/dr-inference-engine" }
|
||||
dr-ingest = { path = "core/dr-ingest" }
|
||||
dr-gpu = { path = "core/dr-gpu" }
|
||||
dr-lens = { path = "core/dr-lens" }
|
||||
# Optional runtime, like `dr-segment`: the geometry never needs a model.
|
||||
dr-pano = { path = "core/dr-pano", default-features = false }
|
||||
dr-pipeline = { path = "core/dr-pipeline" }
|
||||
dr-preset-xmp = { path = "core/dr-preset-xmp" }
|
||||
# `default-features = false` belongs *here*, not on each dependant: a member
|
||||
# inheriting a workspace dependency cannot turn its default features off, so
|
||||
# writing it below would silently do nothing and every crate touching
|
||||
# `dr-segment` would drag in tract and 11 MB of weights. Members opt in with
|
||||
# `features = ["semantic", "embedded-model"]` instead.
|
||||
dr-segment = { path = "core/dr-segment", default-features = false }
|
||||
dr-plat = { path = "platform/dr-plat" }
|
||||
dr-sync = { path = "core/dr-sync" }
|
||||
dr-sync-folder = { path = "core/dr-sync-folder" }
|
||||
dr-sync-nextcloud = { path = "core/dr-sync-nextcloud" }
|
||||
dr-xmp = { path = "core/dr-xmp" }
|
||||
dr-ui = { path = "ui/dr-ui" }
|
||||
|
||||
# GPU + UI
|
||||
#
|
||||
# The wgpu version is not a free choice: it is dictated by Slint. Importing a
|
||||
# texture into the scene (ARCH §6.1, spike S1) requires it to come from the
|
||||
# *same* `wgpu::Device` Slint renders with, and Slint will only hand out a
|
||||
# device of the version it was compiled against. Slint 1.17 offers
|
||||
# `unstable-wgpu-28` and `unstable-wgpu-29` and nothing older, so 29 it is —
|
||||
# pinned to the same `29.0.4` floor Slint itself requires, because two
|
||||
# semver-compatible-but-different wgpu crates in one tree are two *types*, and
|
||||
# the device would not typecheck across them.
|
||||
#
|
||||
# Consequently: bumping Slint may force a wgpu bump, and wgpu cannot be bumped
|
||||
# on its own. They move together or not at all.
|
||||
wgpu = "29.0.4"
|
||||
slint = { version = "1.17", default-features = false }
|
||||
slint-build = "1.17"
|
||||
|
||||
# UI token codegen (S2): style.yaml -> theme.slint. serde_yaml was deprecated
|
||||
# by its maintainer in 2024 and serde_yml, the first fork, has since been
|
||||
# deprecated too; serde_norway is the fork still receiving releases. Its
|
||||
# mappings preserve insertion order, which is what lets the generated Slint
|
||||
# keep the token ordering the YAML author chose.
|
||||
serde_norway = "0.9"
|
||||
|
||||
# Foundations
|
||||
anyhow = "1"
|
||||
thiserror = "2"
|
||||
log = "0.4"
|
||||
env_logger = "0.11"
|
||||
pollster = "0.4"
|
||||
|
||||
# Networking — no mature Nextcloud crate exists; the connector is hand-rolled
|
||||
# over reqwest (D7). reqwest_dav was evaluated and is too thin to build on.
|
||||
# `rustls-no-provider` rather than `rustls`: the latter defaults to the
|
||||
# aws-lc-rs crypto provider, whose aws-lc-sys crate is C and fails to
|
||||
# cross-compile for Android — precisely the NDK pain D1 chose Rust to avoid.
|
||||
# ring is pure Rust apart from a small asm core that does build under the NDK.
|
||||
#
|
||||
# `rustls-tls-webpki-roots-no-provider` rather than plain `rustls-no-provider`:
|
||||
# the latter verifies against rustls-platform-verifier, which reaches the
|
||||
# Android trust store over JNI and panics mid-handshake unless initialised from
|
||||
# Java first — the crash D7 predicted and spike S3 exists to resolve properly.
|
||||
# The panic surfaces inside tokio, which catches task panics itself, so it
|
||||
# reaches the UI as a worker that stopped rather than as an error.
|
||||
#
|
||||
# webpki-roots is the escape hatch D7 records: a root store compiled into the
|
||||
# binary, no JNI, identical on both platforms. The trade is real and belongs in
|
||||
# S3's scope — user-installed and enterprise CAs are not consulted, and the
|
||||
# roots go stale with the release rather than with the OS.
|
||||
reqwest = { version = "0.13", default-features = false, features = ["rustls-no-provider", "webpki-roots", "stream", "json"] }
|
||||
rustls = { version = "0.23", default-features = false, features = ["ring", "std", "tls12"] }
|
||||
quick-xml = "0.41"
|
||||
tokio = { version = "1", features = ["rt-multi-thread", "macros", "sync", "time"] }
|
||||
url = "2.5"
|
||||
async-trait = "0.1"
|
||||
serde = { version = "1", features = ["derive"] }
|
||||
serde_json = "1"
|
||||
base64 = "0.23"
|
||||
|
||||
# Display-server clients, for FR-DSP-8's per-display profile acquisition.
|
||||
#
|
||||
# Neither is a new cost: winit already builds both, so the versions are the
|
||||
# ones Slint's backend has resolved to and pinning anything else here would
|
||||
# compile a second copy. Both are pure Rust — x11rb speaks the X11 wire
|
||||
# protocol itself rather than binding libxcb, and wayland-client binds
|
||||
# libwayland only under a feature that is off — which keeps the Android
|
||||
# cross-compile a plain Rust dependency graph, the same criterion as the TLS
|
||||
# and SQLite choices above. They are declared under a target predicate that
|
||||
# excludes Android, where neither display server exists.
|
||||
#
|
||||
# `staging` on wayland-protocols is what carries `wp_color_manager_v1`: the
|
||||
# colour-management extension is still staging upstream, which is the
|
||||
# protocol-level statement of the thing FR-DSP-8 anticipates when it says
|
||||
# Wayland's colour management "is not universally available".
|
||||
x11rb = { version = "0.13", features = ["randr"] }
|
||||
wayland-client = "0.31"
|
||||
wayland-protocols = { version = "0.32", features = ["client", "staging"] }
|
||||
|
||||
# Platform secure storage: Secret Service on Linux, Keystore on Android
|
||||
# (FR-NC-2). Credentials never touch the catalog or a plain file.
|
||||
# keyring 4 restructured its features: `v1` is the default set and brings
|
||||
# the zbus Secret Service backend, which is what GNOME Keyring and KWallet
|
||||
# (via ksecretd) both speak.
|
||||
keyring = { version = "4", features = ["v1"] }
|
||||
|
||||
# The Android half of the same project: a keyring-core CredentialStore backed
|
||||
# by AndroidKeyStore AES-GCM over SharedPreferences (FR-PLAT-AND-1). It reads
|
||||
# the JavaVM and Context from ndk-context, which android-activity populates
|
||||
# before `android_main` runs, so no Kotlin shim of our own is needed.
|
||||
#
|
||||
# This is the keyring-core API, not the v1 `Entry` API the Linux path uses;
|
||||
# the two impls are deliberately separate rather than sharing a code path.
|
||||
android-native-keyring-store = "1.0.0"
|
||||
keyring-core = "1"
|
||||
|
||||
# Decode. rawler is the pure-Rust decoder (D2); zune-jpeg decodes the
|
||||
# embedded previews rawler extracts.
|
||||
# Catalog. `bundled` compiles SQLite from source rather than linking the
|
||||
# system library — the same cross-compilation reasoning as the TLS choice
|
||||
# above: no system dependency to satisfy under the Android NDK.
|
||||
#
|
||||
# `backup` is not optional in practice: it is what takes a consistent snapshot
|
||||
# of a live WAL database for upload. A filesystem copy of `catalog.sqlite`
|
||||
# while a `-wal` exists beside it uploads a torn file.
|
||||
rusqlite = { version = "0.40", features = ["bundled", "backup"] }
|
||||
|
||||
rawler = "0.7"
|
||||
zune-jpeg = "0.4.21"
|
||||
# Thumbnails are stored encoded, not as raw RGBA: a 256px RGBA buffer is
|
||||
# ~256 KB against ~20 KB as JPEG, and the store syncs to Nextcloud where that
|
||||
# 13× is transfer cost on every client. Pure Rust, no C dependency — the same
|
||||
# criterion behind the TLS and SQLite choices above.
|
||||
jpeg-encoder = "0.7"
|
||||
bytemuck = { version = "1", features = ["derive"] }
|
||||
|
||||
# Lens correction profiles. A pure-Rust port of Lensfun rather than a binding
|
||||
# to the C library, for the same cross-compilation reason as the TLS and
|
||||
# SQLite choices above: liblensfun would be a third C dependency to satisfy
|
||||
# under the Android NDK.
|
||||
#
|
||||
# The database ships *inside* the crate — 56 XML files, gzipped at build time
|
||||
# and decompressed on first lookup. That matters beyond convenience: Android
|
||||
# gives us no filesystem path (ARCH §6.9), so a database loaded from a
|
||||
# system directory would have nowhere to live there.
|
||||
#
|
||||
# Licence: LGPL-3.0-or-later, which upgrades cleanly into our GPLv3 (D8).
|
||||
# The upstream Lensfun *database* is CC-BY-SA and is redistributed by the
|
||||
# crate; attribution belongs in the about screen.
|
||||
#
|
||||
# Caveat worth remembering: this is a third-party port at 0.7.0, not upstream
|
||||
# Lensfun. Verified working against the bundled database (interpolation
|
||||
# between calibration points, and an unknown lens returning empty rather than
|
||||
# panicking), but the pipeline talks to it through its own profile types so
|
||||
# swapping it out is not a pipeline change.
|
||||
lensfun = "0.7"
|
||||
|
||||
# Neural inference for semantic segmentation (S15 arm B, D14).
|
||||
#
|
||||
# D13 framed this as a choice between `ort` (fast, best operator coverage, and
|
||||
# a C++ dependency to cross-compile under the NDK) and a pure-Rust runtime
|
||||
# (policy-compliant, unproven coverage). That framing turned out to be a false
|
||||
# choice: `ort` 2.0's `alternative-backend` feature *disables the linking
|
||||
# entirely* and lets a different engine supply the `OrtApi`, and `ort-tract` —
|
||||
# same authors, MIT/Apache — supplies it from `tract`, which is pure Rust.
|
||||
#
|
||||
# So we get `ort`'s API with no C at all. `download-binaries` and `tls-native`
|
||||
# are off with `default-features = false`, which is the point: nothing is
|
||||
# fetched at build time and nothing is linked, so the Android cross-compile
|
||||
# sees an ordinary Rust dependency graph. That is the same reasoning as rustls
|
||||
# over aws-lc-rs and bundled SQLite, applied to inference — D13's largest
|
||||
# tolerated exception turns out not to be needed.
|
||||
#
|
||||
# The trade is real and belongs on the record: tract is slower than the C++
|
||||
# runtime and covers fewer operators. Both were measured rather than assumed
|
||||
# before this landed — yolo26n-seg loads with **zero unsupported operators**
|
||||
# and runs 640x640 in ~470 ms on the reference desktop's CPU. That is fine for
|
||||
# a once-per-image precompute off the frame path (ARCH §6.1) and would not be
|
||||
# fine for anything per-frame, which is a constraint on what may be built on
|
||||
# top rather than on this choice.
|
||||
#
|
||||
# Pinned to an rc: `ort` 2.0 has been in rc for a long while and `ort-tract`
|
||||
# exists only against it. Worth revisiting at 2.0 final.
|
||||
ort = { version = "2.0.0-rc.13", default-features = false, features = ["alternative-backend", "ndarray", "std"] }
|
||||
ort-tract = "0.4"
|
||||
# Not a free choice: it is the version `ort` exposes its tensors through, so
|
||||
# two semver-incompatible ndarrays would not typecheck across the boundary —
|
||||
# the same coupling wgpu has with Slint above.
|
||||
ndarray = "0.17"
|
||||
|
||||
[profile.dev]
|
||||
# Dev builds are tuned for how fast they *compile*, not for how fast they run.
|
||||
# Optimisation is a release concern; `[profile.release]` below is where it
|
||||
# belongs.
|
||||
#
|
||||
# This deliberately reverses an earlier choice. Dependencies used to be built
|
||||
# at `opt-level = 2` here, because wgpu and image decoding are slow without it.
|
||||
# That is still true, and it is the price: a debug run of the app, and the
|
||||
# decode- and GPU-heavy tests, are slower than they were. What it buys is that
|
||||
# nothing has to be optimised before it can be compiled — which is the cost
|
||||
# paid on every edit, by every worktree, rather than only when something is
|
||||
# actually run.
|
||||
#
|
||||
# If a particular crate turns out to be the one that makes a test unbearable,
|
||||
# raise it alone rather than restoring the blanket rule:
|
||||
#
|
||||
# [profile.dev.package.zune-jpeg]
|
||||
# opt-level = 2
|
||||
opt-level = 0
|
||||
|
||||
[profile.release]
|
||||
lto = "thin"
|
||||
codegen-units = 1
|
||||
@@ -1,232 +0,0 @@
|
||||
GNU GENERAL PUBLIC LICENSE
|
||||
Version 3, 29 June 2007
|
||||
|
||||
Copyright © 2007 Free Software Foundation, Inc. <https://fsf.org/>
|
||||
|
||||
Everyone is permitted to copy and distribute verbatim copies of this license document, but changing it is not allowed.
|
||||
|
||||
Preamble
|
||||
|
||||
The GNU General Public License is a free, copyleft license for software and other kinds of works.
|
||||
|
||||
The licenses for most software and other practical works are designed to take away your freedom to share and change the works. By contrast, the GNU General Public License is intended to guarantee your freedom to share and change all versions of a program--to make sure it remains free software for all its users. We, the Free Software Foundation, use the GNU General Public License for most of our software; it applies also to any other work released this way by its authors. You can apply it to your programs, too.
|
||||
|
||||
When we speak of free software, we are referring to freedom, not price. Our General Public Licenses are designed to make sure that you have the freedom to distribute copies of free software (and charge for them if you wish), that you receive source code or can get it if you want it, that you can change the software or use pieces of it in new free programs, and that you know you can do these things.
|
||||
|
||||
To protect your rights, we need to prevent others from denying you these rights or asking you to surrender the rights. Therefore, you have certain responsibilities if you distribute copies of the software, or if you modify it: responsibilities to respect the freedom of others.
|
||||
|
||||
For example, if you distribute copies of such a program, whether gratis or for a fee, you must pass on to the recipients the same freedoms that you received. You must make sure that they, too, receive or can get the source code. And you must show them these terms so they know their rights.
|
||||
|
||||
Developers that use the GNU GPL protect your rights with two steps: (1) assert copyright on the software, and (2) offer you this License giving you legal permission to copy, distribute and/or modify it.
|
||||
|
||||
For the developers' and authors' protection, the GPL clearly explains that there is no warranty for this free software. For both users' and authors' sake, the GPL requires that modified versions be marked as changed, so that their problems will not be attributed erroneously to authors of previous versions.
|
||||
|
||||
Some devices are designed to deny users access to install or run modified versions of the software inside them, although the manufacturer can do so. This is fundamentally incompatible with the aim of protecting users' freedom to change the software. The systematic pattern of such abuse occurs in the area of products for individuals to use, which is precisely where it is most unacceptable. Therefore, we have designed this version of the GPL to prohibit the practice for those products. If such problems arise substantially in other domains, we stand ready to extend this provision to those domains in future versions of the GPL, as needed to protect the freedom of users.
|
||||
|
||||
Finally, every program is threatened constantly by software patents. States should not allow patents to restrict development and use of software on general-purpose computers, but in those that do, we wish to avoid the special danger that patents applied to a free program could make it effectively proprietary. To prevent this, the GPL assures that patents cannot be used to render the program non-free.
|
||||
|
||||
The precise terms and conditions for copying, distribution and modification follow.
|
||||
|
||||
TERMS AND CONDITIONS
|
||||
|
||||
0. Definitions.
|
||||
|
||||
“This License” refers to version 3 of the GNU General Public License.
|
||||
|
||||
“Copyright” also means copyright-like laws that apply to other kinds of works, such as semiconductor masks.
|
||||
|
||||
“The Program” refers to any copyrightable work licensed under this License. Each licensee is addressed as “you”. “Licensees” and “recipients” may be individuals or organizations.
|
||||
|
||||
To “modify” a work means to copy from or adapt all or part of the work in a fashion requiring copyright permission, other than the making of an exact copy. The resulting work is called a “modified version” of the earlier work or a work “based on” the earlier work.
|
||||
|
||||
A “covered work” means either the unmodified Program or a work based on the Program.
|
||||
|
||||
To “propagate” a work means to do anything with it that, without permission, would make you directly or secondarily liable for infringement under applicable copyright law, except executing it on a computer or modifying a private copy. Propagation includes copying, distribution (with or without modification), making available to the public, and in some countries other activities as well.
|
||||
|
||||
To “convey” a work means any kind of propagation that enables other parties to make or receive copies. Mere interaction with a user through a computer network, with no transfer of a copy, is not conveying.
|
||||
|
||||
An interactive user interface displays “Appropriate Legal Notices” to the extent that it includes a convenient and prominently visible feature that (1) displays an appropriate copyright notice, and (2) tells the user that there is no warranty for the work (except to the extent that warranties are provided), that licensees may convey the work under this License, and how to view a copy of this License. If the interface presents a list of user commands or options, such as a menu, a prominent item in the list meets this criterion.
|
||||
|
||||
1. Source Code.
|
||||
The “source code” for a work means the preferred form of the work for making modifications to it. “Object code” means any non-source form of a work.
|
||||
|
||||
A “Standard Interface” means an interface that either is an official standard defined by a recognized standards body, or, in the case of interfaces specified for a particular programming language, one that is widely used among developers working in that language.
|
||||
|
||||
The “System Libraries” of an executable work include anything, other than the work as a whole, that (a) is included in the normal form of packaging a Major Component, but which is not part of that Major Component, and (b) serves only to enable use of the work with that Major Component, or to implement a Standard Interface for which an implementation is available to the public in source code form. A “Major Component”, in this context, means a major essential component (kernel, window system, and so on) of the specific operating system (if any) on which the executable work runs, or a compiler used to produce the work, or an object code interpreter used to run it.
|
||||
|
||||
The “Corresponding Source” for a work in object code form means all the source code needed to generate, install, and (for an executable work) run the object code and to modify the work, including scripts to control those activities. However, it does not include the work's System Libraries, or general-purpose tools or generally available free programs which are used unmodified in performing those activities but which are not part of the work. For example, Corresponding Source includes interface definition files associated with source files for the work, and the source code for shared libraries and dynamically linked subprograms that the work is specifically designed to require, such as by intimate data communication or control flow between those subprograms and other parts of the work.
|
||||
|
||||
The Corresponding Source need not include anything that users can regenerate automatically from other parts of the Corresponding Source.
|
||||
|
||||
The Corresponding Source for a work in source code form is that same work.
|
||||
|
||||
2. Basic Permissions.
|
||||
All rights granted under this License are granted for the term of copyright on the Program, and are irrevocable provided the stated conditions are met. This License explicitly affirms your unlimited permission to run the unmodified Program. The output from running a covered work is covered by this License only if the output, given its content, constitutes a covered work. This License acknowledges your rights of fair use or other equivalent, as provided by copyright law.
|
||||
|
||||
You may make, run and propagate covered works that you do not convey, without conditions so long as your license otherwise remains in force. You may convey covered works to others for the sole purpose of having them make modifications exclusively for you, or provide you with facilities for running those works, provided that you comply with the terms of this License in conveying all material for which you do not control copyright. Those thus making or running the covered works for you must do so exclusively on your behalf, under your direction and control, on terms that prohibit them from making any copies of your copyrighted material outside their relationship with you.
|
||||
|
||||
Conveying under any other circumstances is permitted solely under the conditions stated below. Sublicensing is not allowed; section 10 makes it unnecessary.
|
||||
|
||||
3. Protecting Users' Legal Rights From Anti-Circumvention Law.
|
||||
No covered work shall be deemed part of an effective technological measure under any applicable law fulfilling obligations under article 11 of the WIPO copyright treaty adopted on 20 December 1996, or similar laws prohibiting or restricting circumvention of such measures.
|
||||
|
||||
When you convey a covered work, you waive any legal power to forbid circumvention of technological measures to the extent such circumvention is effected by exercising rights under this License with respect to the covered work, and you disclaim any intention to limit operation or modification of the work as a means of enforcing, against the work's users, your or third parties' legal rights to forbid circumvention of technological measures.
|
||||
|
||||
4. Conveying Verbatim Copies.
|
||||
You may convey verbatim copies of the Program's source code as you receive it, in any medium, provided that you conspicuously and appropriately publish on each copy an appropriate copyright notice; keep intact all notices stating that this License and any non-permissive terms added in accord with section 7 apply to the code; keep intact all notices of the absence of any warranty; and give all recipients a copy of this License along with the Program.
|
||||
|
||||
You may charge any price or no price for each copy that you convey, and you may offer support or warranty protection for a fee.
|
||||
|
||||
5. Conveying Modified Source Versions.
|
||||
You may convey a work based on the Program, or the modifications to produce it from the Program, in the form of source code under the terms of section 4, provided that you also meet all of these conditions:
|
||||
|
||||
a) The work must carry prominent notices stating that you modified it, and giving a relevant date.
|
||||
|
||||
b) The work must carry prominent notices stating that it is released under this License and any conditions added under section 7. This requirement modifies the requirement in section 4 to “keep intact all notices”.
|
||||
|
||||
c) You must license the entire work, as a whole, under this License to anyone who comes into possession of a copy. This License will therefore apply, along with any applicable section 7 additional terms, to the whole of the work, and all its parts, regardless of how they are packaged. This License gives no permission to license the work in any other way, but it does not invalidate such permission if you have separately received it.
|
||||
|
||||
d) If the work has interactive user interfaces, each must display Appropriate Legal Notices; however, if the Program has interactive interfaces that do not display Appropriate Legal Notices, your work need not make them do so.
|
||||
|
||||
A compilation of a covered work with other separate and independent works, which are not by their nature extensions of the covered work, and which are not combined with it such as to form a larger program, in or on a volume of a storage or distribution medium, is called an “aggregate” if the compilation and its resulting copyright are not used to limit the access or legal rights of the compilation's users beyond what the individual works permit. Inclusion of a covered work in an aggregate does not cause this License to apply to the other parts of the aggregate.
|
||||
|
||||
6. Conveying Non-Source Forms.
|
||||
You may convey a covered work in object code form under the terms of sections 4 and 5, provided that you also convey the machine-readable Corresponding Source under the terms of this License, in one of these ways:
|
||||
|
||||
a) Convey the object code in, or embodied in, a physical product (including a physical distribution medium), accompanied by the Corresponding Source fixed on a durable physical medium customarily used for software interchange.
|
||||
|
||||
b) Convey the object code in, or embodied in, a physical product (including a physical distribution medium), accompanied by a written offer, valid for at least three years and valid for as long as you offer spare parts or customer support for that product model, to give anyone who possesses the object code either (1) a copy of the Corresponding Source for all the software in the product that is covered by this License, on a durable physical medium customarily used for software interchange, for a price no more than your reasonable cost of physically performing this conveying of source, or (2) access to copy the Corresponding Source from a network server at no charge.
|
||||
|
||||
c) Convey individual copies of the object code with a copy of the written offer to provide the Corresponding Source. This alternative is allowed only occasionally and noncommercially, and only if you received the object code with such an offer, in accord with subsection 6b.
|
||||
|
||||
d) Convey the object code by offering access from a designated place (gratis or for a charge), and offer equivalent access to the Corresponding Source in the same way through the same place at no further charge. You need not require recipients to copy the Corresponding Source along with the object code. If the place to copy the object code is a network server, the Corresponding Source may be on a different server (operated by you or a third party) that supports equivalent copying facilities, provided you maintain clear directions next to the object code saying where to find the Corresponding Source. Regardless of what server hosts the Corresponding Source, you remain obligated to ensure that it is available for as long as needed to satisfy these requirements.
|
||||
|
||||
e) Convey the object code using peer-to-peer transmission, provided you inform other peers where the object code and Corresponding Source of the work are being offered to the general public at no charge under subsection 6d.
|
||||
|
||||
A separable portion of the object code, whose source code is excluded from the Corresponding Source as a System Library, need not be included in conveying the object code work.
|
||||
|
||||
A “User Product” is either (1) a “consumer product”, which means any tangible personal property which is normally used for personal, family, or household purposes, or (2) anything designed or sold for incorporation into a dwelling. In determining whether a product is a consumer product, doubtful cases shall be resolved in favor of coverage. For a particular product received by a particular user, “normally used” refers to a typical or common use of that class of product, regardless of the status of the particular user or of the way in which the particular user actually uses, or expects or is expected to use, the product. A product is a consumer product regardless of whether the product has substantial commercial, industrial or non-consumer uses, unless such uses represent the only significant mode of use of the product.
|
||||
|
||||
“Installation Information” for a User Product means any methods, procedures, authorization keys, or other information required to install and execute modified versions of a covered work in that User Product from a modified version of its Corresponding Source. The information must suffice to ensure that the continued functioning of the modified object code is in no case prevented or interfered with solely because modification has been made.
|
||||
|
||||
If you convey an object code work under this section in, or with, or specifically for use in, a User Product, and the conveying occurs as part of a transaction in which the right of possession and use of the User Product is transferred to the recipient in perpetuity or for a fixed term (regardless of how the transaction is characterized), the Corresponding Source conveyed under this section must be accompanied by the Installation Information. But this requirement does not apply if neither you nor any third party retains the ability to install modified object code on the User Product (for example, the work has been installed in ROM).
|
||||
|
||||
The requirement to provide Installation Information does not include a requirement to continue to provide support service, warranty, or updates for a work that has been modified or installed by the recipient, or for the User Product in which it has been modified or installed. Access to a network may be denied when the modification itself materially and adversely affects the operation of the network or violates the rules and protocols for communication across the network.
|
||||
|
||||
Corresponding Source conveyed, and Installation Information provided, in accord with this section must be in a format that is publicly documented (and with an implementation available to the public in source code form), and must require no special password or key for unpacking, reading or copying.
|
||||
|
||||
7. Additional Terms.
|
||||
“Additional permissions” are terms that supplement the terms of this License by making exceptions from one or more of its conditions. Additional permissions that are applicable to the entire Program shall be treated as though they were included in this License, to the extent that they are valid under applicable law. If additional permissions apply only to part of the Program, that part may be used separately under those permissions, but the entire Program remains governed by this License without regard to the additional permissions.
|
||||
|
||||
When you convey a copy of a covered work, you may at your option remove any additional permissions from that copy, or from any part of it. (Additional permissions may be written to require their own removal in certain cases when you modify the work.) You may place additional permissions on material, added by you to a covered work, for which you have or can give appropriate copyright permission.
|
||||
|
||||
Notwithstanding any other provision of this License, for material you add to a covered work, you may (if authorized by the copyright holders of that material) supplement the terms of this License with terms:
|
||||
|
||||
a) Disclaiming warranty or limiting liability differently from the terms of sections 15 and 16 of this License; or
|
||||
|
||||
b) Requiring preservation of specified reasonable legal notices or author attributions in that material or in the Appropriate Legal Notices displayed by works containing it; or
|
||||
|
||||
c) Prohibiting misrepresentation of the origin of that material, or requiring that modified versions of such material be marked in reasonable ways as different from the original version; or
|
||||
|
||||
d) Limiting the use for publicity purposes of names of licensors or authors of the material; or
|
||||
|
||||
e) Declining to grant rights under trademark law for use of some trade names, trademarks, or service marks; or
|
||||
|
||||
f) Requiring indemnification of licensors and authors of that material by anyone who conveys the material (or modified versions of it) with contractual assumptions of liability to the recipient, for any liability that these contractual assumptions directly impose on those licensors and authors.
|
||||
|
||||
All other non-permissive additional terms are considered “further restrictions” within the meaning of section 10. If the Program as you received it, or any part of it, contains a notice stating that it is governed by this License along with a term that is a further restriction, you may remove that term. If a license document contains a further restriction but permits relicensing or conveying under this License, you may add to a covered work material governed by the terms of that license document, provided that the further restriction does not survive such relicensing or conveying.
|
||||
|
||||
If you add terms to a covered work in accord with this section, you must place, in the relevant source files, a statement of the additional terms that apply to those files, or a notice indicating where to find the applicable terms.
|
||||
|
||||
Additional terms, permissive or non-permissive, may be stated in the form of a separately written license, or stated as exceptions; the above requirements apply either way.
|
||||
|
||||
8. Termination.
|
||||
You may not propagate or modify a covered work except as expressly provided under this License. Any attempt otherwise to propagate or modify it is void, and will automatically terminate your rights under this License (including any patent licenses granted under the third paragraph of section 11).
|
||||
|
||||
However, if you cease all violation of this License, then your license from a particular copyright holder is reinstated (a) provisionally, unless and until the copyright holder explicitly and finally terminates your license, and (b) permanently, if the copyright holder fails to notify you of the violation by some reasonable means prior to 60 days after the cessation.
|
||||
|
||||
Moreover, your license from a particular copyright holder is reinstated permanently if the copyright holder notifies you of the violation by some reasonable means, this is the first time you have received notice of violation of this License (for any work) from that copyright holder, and you cure the violation prior to 30 days after your receipt of the notice.
|
||||
|
||||
Termination of your rights under this section does not terminate the licenses of parties who have received copies or rights from you under this License. If your rights have been terminated and not permanently reinstated, you do not qualify to receive new licenses for the same material under section 10.
|
||||
|
||||
9. Acceptance Not Required for Having Copies.
|
||||
You are not required to accept this License in order to receive or run a copy of the Program. Ancillary propagation of a covered work occurring solely as a consequence of using peer-to-peer transmission to receive a copy likewise does not require acceptance. However, nothing other than this License grants you permission to propagate or modify any covered work. These actions infringe copyright if you do not accept this License. Therefore, by modifying or propagating a covered work, you indicate your acceptance of this License to do so.
|
||||
|
||||
10. Automatic Licensing of Downstream Recipients.
|
||||
Each time you convey a covered work, the recipient automatically receives a license from the original licensors, to run, modify and propagate that work, subject to this License. You are not responsible for enforcing compliance by third parties with this License.
|
||||
|
||||
An “entity transaction” is a transaction transferring control of an organization, or substantially all assets of one, or subdividing an organization, or merging organizations. If propagation of a covered work results from an entity transaction, each party to that transaction who receives a copy of the work also receives whatever licenses to the work the party's predecessor in interest had or could give under the previous paragraph, plus a right to possession of the Corresponding Source of the work from the predecessor in interest, if the predecessor has it or can get it with reasonable efforts.
|
||||
|
||||
You may not impose any further restrictions on the exercise of the rights granted or affirmed under this License. For example, you may not impose a license fee, royalty, or other charge for exercise of rights granted under this License, and you may not initiate litigation (including a cross-claim or counterclaim in a lawsuit) alleging that any patent claim is infringed by making, using, selling, offering for sale, or importing the Program or any portion of it.
|
||||
|
||||
11. Patents.
|
||||
A “contributor” is a copyright holder who authorizes use under this License of the Program or a work on which the Program is based. The work thus licensed is called the contributor's “contributor version”.
|
||||
|
||||
A contributor's “essential patent claims” are all patent claims owned or controlled by the contributor, whether already acquired or hereafter acquired, that would be infringed by some manner, permitted by this License, of making, using, or selling its contributor version, but do not include claims that would be infringed only as a consequence of further modification of the contributor version. For purposes of this definition, “control” includes the right to grant patent sublicenses in a manner consistent with the requirements of this License.
|
||||
|
||||
Each contributor grants you a non-exclusive, worldwide, royalty-free patent license under the contributor's essential patent claims, to make, use, sell, offer for sale, import and otherwise run, modify and propagate the contents of its contributor version.
|
||||
|
||||
In the following three paragraphs, a “patent license” is any express agreement or commitment, however denominated, not to enforce a patent (such as an express permission to practice a patent or covenant not to sue for patent infringement). To “grant” such a patent license to a party means to make such an agreement or commitment not to enforce a patent against the party.
|
||||
|
||||
If you convey a covered work, knowingly relying on a patent license, and the Corresponding Source of the work is not available for anyone to copy, free of charge and under the terms of this License, through a publicly available network server or other readily accessible means, then you must either (1) cause the Corresponding Source to be so available, or (2) arrange to deprive yourself of the benefit of the patent license for this particular work, or (3) arrange, in a manner consistent with the requirements of this License, to extend the patent license to downstream recipients. “Knowingly relying” means you have actual knowledge that, but for the patent license, your conveying the covered work in a country, or your recipient's use of the covered work in a country, would infringe one or more identifiable patents in that country that you have reason to believe are valid.
|
||||
|
||||
If, pursuant to or in connection with a single transaction or arrangement, you convey, or propagate by procuring conveyance of, a covered work, and grant a patent license to some of the parties receiving the covered work authorizing them to use, propagate, modify or convey a specific copy of the covered work, then the patent license you grant is automatically extended to all recipients of the covered work and works based on it.
|
||||
|
||||
A patent license is “discriminatory” if it does not include within the scope of its coverage, prohibits the exercise of, or is conditioned on the non-exercise of one or more of the rights that are specifically granted under this License. You may not convey a covered work if you are a party to an arrangement with a third party that is in the business of distributing software, under which you make payment to the third party based on the extent of your activity of conveying the work, and under which the third party grants, to any of the parties who would receive the covered work from you, a discriminatory patent license (a) in connection with copies of the covered work conveyed by you (or copies made from those copies), or (b) primarily for and in connection with specific products or compilations that contain the covered work, unless you entered into that arrangement, or that patent license was granted, prior to 28 March 2007.
|
||||
|
||||
Nothing in this License shall be construed as excluding or limiting any implied license or other defenses to infringement that may otherwise be available to you under applicable patent law.
|
||||
|
||||
12. No Surrender of Others' Freedom.
|
||||
If conditions are imposed on you (whether by court order, agreement or otherwise) that contradict the conditions of this License, they do not excuse you from the conditions of this License. If you cannot convey a covered work so as to satisfy simultaneously your obligations under this License and any other pertinent obligations, then as a consequence you may not convey it at all. For example, if you agree to terms that obligate you to collect a royalty for further conveying from those to whom you convey the Program, the only way you could satisfy both those terms and this License would be to refrain entirely from conveying the Program.
|
||||
|
||||
13. Use with the GNU Affero General Public License.
|
||||
Notwithstanding any other provision of this License, you have permission to link or combine any covered work with a work licensed under version 3 of the GNU Affero General Public License into a single combined work, and to convey the resulting work. The terms of this License will continue to apply to the part which is the covered work, but the special requirements of the GNU Affero General Public License, section 13, concerning interaction through a network will apply to the combination as such.
|
||||
|
||||
14. Revised Versions of this License.
|
||||
The Free Software Foundation may publish revised and/or new versions of the GNU General Public License from time to time. Such new versions will be similar in spirit to the present version, but may differ in detail to address new problems or concerns.
|
||||
|
||||
Each version is given a distinguishing version number. If the Program specifies that a certain numbered version of the GNU General Public License “or any later version” applies to it, you have the option of following the terms and conditions either of that numbered version or of any later version published by the Free Software Foundation. If the Program does not specify a version number of the GNU General Public License, you may choose any version ever published by the Free Software Foundation.
|
||||
|
||||
If the Program specifies that a proxy can decide which future versions of the GNU General Public License can be used, that proxy's public statement of acceptance of a version permanently authorizes you to choose that version for the Program.
|
||||
|
||||
Later license versions may give you additional or different permissions. However, no additional obligations are imposed on any author or copyright holder as a result of your choosing to follow a later version.
|
||||
|
||||
15. Disclaimer of Warranty.
|
||||
THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY APPLICABLE LAW. EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM “AS IS” WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE PROGRAM IS WITH YOU. SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF ALL NECESSARY SERVICING, REPAIR OR CORRECTION.
|
||||
|
||||
16. Limitation of Liability.
|
||||
IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MODIFIES AND/OR CONVEYS THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS), EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF SUCH DAMAGES.
|
||||
|
||||
17. Interpretation of Sections 15 and 16.
|
||||
If the disclaimer of warranty and limitation of liability provided above cannot be given local legal effect according to their terms, reviewing courts shall apply local law that most closely approximates an absolute waiver of all civil liability in connection with the Program, unless a warranty or assumption of liability accompanies a copy of the Program in return for a fee.
|
||||
|
||||
END OF TERMS AND CONDITIONS
|
||||
|
||||
How to Apply These Terms to Your New Programs
|
||||
|
||||
If you develop a new program, and you want it to be of the greatest possible use to the public, the best way to achieve this is to make it free software which everyone can redistribute and change under these terms.
|
||||
|
||||
To do so, attach the following notices to the program. It is safest to attach them to the start of each source file to most effectively state the exclusion of warranty; and each file should have at least the “copyright” line and a pointer to where the full notice is found.
|
||||
|
||||
<one line to give the program's name and a brief idea of what it does.>
|
||||
Copyright (C) <year> <name of author>
|
||||
|
||||
This program is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later version.
|
||||
|
||||
This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details.
|
||||
|
||||
You should have received a copy of the GNU General Public License along with this program. If not, see <https://www.gnu.org/licenses/>.
|
||||
|
||||
Also add information on how to contact you by electronic and paper mail.
|
||||
|
||||
If the program does terminal interaction, make it output a short notice like this when it starts in an interactive mode:
|
||||
|
||||
<program> Copyright (C) <year> <name of author>
|
||||
This program comes with ABSOLUTELY NO WARRANTY; for details type `show w'.
|
||||
This is free software, and you are welcome to redistribute it under certain conditions; type `show c' for details.
|
||||
|
||||
The hypothetical commands `show w' and `show c' should show the appropriate parts of the General Public License. Of course, your program's commands might be different; for a GUI interface, you would use an “about box”.
|
||||
|
||||
You should also get your employer (if you work as a programmer) or school, if any, to sign a “copyright disclaimer” for the program, if necessary. For more information on this, and how to apply and follow the GNU GPL, see <https://www.gnu.org/licenses/>.
|
||||
|
||||
The GNU General Public License does not permit incorporating your program into proprietary programs. If your program is a subroutine library, you may consider it more useful to permit linking proprietary applications with the library. If this is what you want to do, use the GNU Lesser General Public License instead of this License. But first, please read <https://www.gnu.org/philosophy/why-not-lgpl.html>.
|
||||
@@ -1,84 +0,0 @@
|
||||
# DarkRoom
|
||||
|
||||
A cross-platform, non-destructive RAW photo editor for Linux and Android.
|
||||
|
||||
**Status:** 0.9.0, and no longer a spike. A library opens, culls, develops and
|
||||
exports on both platforms, across eight tagged releases. What is *not*
|
||||
built is written down rather than merely absent — see
|
||||
[docs/outstanding.md](docs/outstanding.md) for the requirements that have no
|
||||
implementation and why, and [docs/technical-debt.md](docs/technical-debt.md)
|
||||
for the compromises that were chosen.
|
||||
|
||||
## Documentation
|
||||
|
||||
| Document | Contents |
|
||||
|---|---|
|
||||
| [CONTRIBUTING.md](CONTRIBUTING.md) | How to land a first change without reading the rest |
|
||||
| [requirements.md](docs/requirements.md) | What the software must do — 179 numbered requirements |
|
||||
| [architecture.md](docs/architecture.md) | How it is built — crates, GPU pipeline, data model, sync |
|
||||
| [technical-debt.md](docs/technical-debt.md) | Compromises taken deliberately, each with the condition that retires it |
|
||||
| [outstanding.md](docs/outstanding.md) | What is not built, and whether that is a decision or a gap |
|
||||
| [code-health.md](docs/code-health.md) | What a contribution costs, per seam, measured |
|
||||
| [traceability.md](docs/traceability.md) | Generated: which requirement is claimed by which file |
|
||||
| [faces.md](docs/faces.md) | Face detection and identity — the models, the licence problem, and what S14 measured |
|
||||
|
||||
## Building
|
||||
|
||||
Desktop:
|
||||
|
||||
```bash
|
||||
cargo run -p darkroom-desktop
|
||||
```
|
||||
|
||||
Android (containerised toolchain, see [docker/android](docker/android/README.md)):
|
||||
|
||||
```bash
|
||||
./docker/android/build.sh cargo ndk -t arm64-v8a build --release
|
||||
```
|
||||
|
||||
Git LFS is required for the model weights, and the toolchain pins itself.
|
||||
[CONTRIBUTING.md](CONTRIBUTING.md) has the details and the four commands CI
|
||||
will run against what you send.
|
||||
|
||||
## Current state
|
||||
|
||||
**Working.** A catalog over a local folder, a Nextcloud account, or a folder a
|
||||
sync client keeps in virtual-files mode — where a placeholder is treated as the
|
||||
photograph rather than as a one-byte file. A virtualised library grid with a
|
||||
capture-time timeline, ratings, labels, keywords, collections and a trash that
|
||||
survives a crash mid-operation. Card ingest. Face detection and identity, with
|
||||
the index syncing between devices. A develop pipeline of fifteen declared
|
||||
operations fused into a single compute dispatch, plus the neighbourhood
|
||||
operations that cannot be — clarity, texture, capture sharpening, noise
|
||||
reduction, lens correction, spectral film simulation. Crop, straighten, spot
|
||||
removal, gradient and subject-segmentation masks, named presets, and a
|
||||
generated panel that no operation in `ui/` is allowed to name. Export to JPEG,
|
||||
PNG and 8- or 16-bit TIFF with resize and output sharpening.
|
||||
|
||||
**The zero-copy display path works on desktop.** The compute pass writes a
|
||||
texture that Slint composites directly, which is what
|
||||
[ARCH §6.1](docs/architecture.md) requires; the readback it forbids costs 96%
|
||||
of frame time at 4K, and
|
||||
|
||||
```bash
|
||||
cargo run -p dr-gpu --example bench --features readback
|
||||
```
|
||||
|
||||
still reproduces that measurement. **The one exception is the Android develop
|
||||
view**, which reads the frame back through the CPU because zero-copy there
|
||||
needs wgpu's Vulkan swapchain, and that tears a portrait window on a tablet
|
||||
whose panel is mounted landscape. It is debt, not a revision of the rule: the
|
||||
reasoning, the on-device measurements that forced it, and the three separate
|
||||
things any one of which would remove it are in
|
||||
[technical-debt.md TD-1](docs/technical-debt.md).
|
||||
|
||||
**Not built.** Plugins, compare and survey culling, focus peaking, burst
|
||||
grouping, AI denoise, tiled and progressive rendering, and most of the Android
|
||||
platform integration beyond running. The performance targets in §4.1 are
|
||||
unverified rather than unmet — the per-commit benchmark suite §8 requires does
|
||||
not exist, so nothing fails a build on a regression.
|
||||
[docs/outstanding.md](docs/outstanding.md) is the list, with the reasoning.
|
||||
|
||||
## Licence
|
||||
|
||||
GPL-3.0-or-later.
|
||||
@@ -1,54 +0,0 @@
|
||||
[package]
|
||||
name = "darkroom-android"
|
||||
version.workspace = true
|
||||
edition.workspace = true
|
||||
rust-version.workspace = true
|
||||
license.workspace = true
|
||||
|
||||
# A cdylib, not a bin: Android loads the app as a shared library and calls
|
||||
# `android_main` through android-activity's glue. Nothing execs a binary, so
|
||||
# there is no `main` to provide.
|
||||
[lib]
|
||||
name = "darkroom"
|
||||
crate-type = ["cdylib"]
|
||||
|
||||
[dependencies]
|
||||
# No backend feature to select: dr-ui picks its Slint backend from the target,
|
||||
# so building for aarch64-linux-android gets android-activity automatically.
|
||||
dr-ui.workspace = true
|
||||
# For `account::set_data_dir`: only the platform entry point knows where Android
|
||||
# lets this app keep files, and it must be set before any store is opened.
|
||||
dr-sync.workspace = true
|
||||
# For the panic hook, for `state::set_state_dir` and for
|
||||
# `diagnostics::install`. Android has no XDG directories, so the entry point is
|
||||
# the only place that knows where a crash record or a log file may be written,
|
||||
# and both have to be in place before anything can fail.
|
||||
dr-plat.workspace = true
|
||||
# Directly, not just through dr-ui: `android_main` takes an `AndroidApp` and
|
||||
# calls `slint::android::init`, both of which come from this crate. The backend
|
||||
# feature comes from dr-ui's target-specific dependency.
|
||||
slint.workspace = true
|
||||
log.workspace = true
|
||||
android_logger = "0.15"
|
||||
|
||||
# The launch Intent and the share sheet are Java-only surfaces — see `intents`
|
||||
# — and JNI is the only way to reach them.
|
||||
#
|
||||
# Target-gated because the crate still has to compile on the host: it is a
|
||||
# workspace member, `cargo test --workspace` builds it, and the manifest tests
|
||||
# in `lib.rs` are the one part of it that runs there.
|
||||
#
|
||||
# 0.21 rather than the 0.22 that android-activity 0.6 uses. Both are already in
|
||||
# the lock — Slint's Android backend depends on two major versions of
|
||||
# android-activity and pulls both — so this adds nothing to the build either
|
||||
# way, and every object here comes from a raw pointer rather than from a type
|
||||
# android-activity handed over, so the two never have to agree.
|
||||
[target.'cfg(target_os = "android")'.dependencies]
|
||||
jni = "0.21"
|
||||
|
||||
[features]
|
||||
# Mirrors darkroom-desktop: the CPU readback path is gone since S1 landed
|
||||
# zero-copy. It mattered more here than on desktop — the same wrong path with
|
||||
# far less memory bandwidth to absorb it (ARCH §6.1) — but it is untested on a
|
||||
# device, since S1 was verified on desktop only.
|
||||
default = []
|
||||
@@ -1,171 +0,0 @@
|
||||
<?xml version="1.0" encoding="utf-8"?>
|
||||
<!--
|
||||
DarkRoom Android manifest.
|
||||
|
||||
Deliberately minimal: this packages the viewer for on-device testing (spike
|
||||
S2 needs Adreno and Mali hardware, which no emulator represents). Nothing
|
||||
here is a distribution manifest yet. Only network access is declared: file
|
||||
access needs no manifest permission because the library grid reads through
|
||||
SAF, which grants per-tree at runtime (ARCH §6.9).
|
||||
|
||||
Minimal is not the same as empty, and the entries below that are not the
|
||||
activity are the difference. A manifest is the only place a component can be
|
||||
declared: an intent filter is how the system learns this app is worth
|
||||
offering for a photograph, and a provider is how it learns the class exists
|
||||
at all. Neither can be moved into code (FR-PLAT-AND-6).
|
||||
-->
|
||||
<manifest xmlns:android="http://schemas.android.com/apk/res/android"
|
||||
package="paris.tourolle.darkroom">
|
||||
|
||||
<!-- Everything the app does with a server needs this: Login Flow v2, the
|
||||
WebDAV listing, thumbnail and image fetches. Without it Android refuses
|
||||
socket creation outright, and the failure is invisible — no panic to
|
||||
catch, no log line, just a worker thread that stops. Storage is the
|
||||
separate case that genuinely needs no permission here, because SAF
|
||||
grants per-tree at runtime (ARCH §6.9). -->
|
||||
<uses-permission android:name="android.permission.INTERNET" />
|
||||
<!-- Read before deciding whether a sync may run: FR-NC-6 gates background
|
||||
work on unmetered-and-charging, which means knowing the network type. -->
|
||||
<uses-permission android:name="android.permission.ACCESS_NETWORK_STATE" />
|
||||
|
||||
<!-- Vulkan 1.1 is what wgpu needs; the API 28 floor is where support is
|
||||
dependable (NFR-COMPAT-1). Marked required so an unsupported device
|
||||
fails at install rather than at first frame. -->
|
||||
<uses-feature
|
||||
android:name="android.hardware.vulkan.version"
|
||||
android:version="0x00401000"
|
||||
android:required="true" />
|
||||
|
||||
<!-- One name covers both icon generations, which is the point of the
|
||||
`anydpi-v26` qualifier: @mipmap/ic_launcher resolves to the adaptive
|
||||
icon at res/mipmap-anydpi-v26/ic_launcher.xml on API 26 and up, and to
|
||||
the density-matched ic_launcher.png below that. Since minSdk is 28 the
|
||||
PNGs are only ever reached by tooling, but they cost little and aapt2
|
||||
wants a real drawable behind the name. `roundIcon` is deliberately
|
||||
absent: it predates adaptive icons and a launcher that reads it would
|
||||
also be one that ignores the XML, which no device here is.
|
||||
|
||||
The adaptive icon has three layers rather than two. The third,
|
||||
monochrome, is what lets Android 13's themed-icon setting recolour it
|
||||
instead of dropping the app out of the themed set. -->
|
||||
<application
|
||||
android:label="DarkRoom"
|
||||
android:icon="@mipmap/ic_launcher"
|
||||
android:hasCode="true"
|
||||
android:allowBackup="false"
|
||||
android:supportsRtl="true">
|
||||
|
||||
<!-- NativeActivity rather than a Kotlin Activity: android-activity's
|
||||
glue loads libdarkroom.so and calls android_main. `android.app.lib_name`
|
||||
is how it learns which library to load, and must match [lib].name.
|
||||
|
||||
`singleTask` because a second instance of this activity is not
|
||||
survivable. The intent filters below mean another app can now
|
||||
launch it while it is already running, and under the default
|
||||
launch mode that starts a *second* NativeActivity — in the
|
||||
caller's task, in this same process, calling android_main again.
|
||||
Two Slint backends and two wgpu devices in one process is not a
|
||||
degraded experience, it is a failed second launch on top of a
|
||||
working first one.
|
||||
|
||||
What it costs, stated plainly: a share that arrives while DarkRoom
|
||||
is already running brings it forward without opening the image.
|
||||
The Intent goes to `onNewIntent`, and android-activity's event
|
||||
stream has no variant for it (MainEvent in 0.6 stops at Destroy),
|
||||
so nothing native ever sees it. Reading it would mean a Java
|
||||
Activity subclass forwarding it across JNI — the same shape of
|
||||
change FR-PLAT-AND-5 declined for onTrimMemory, and for the same
|
||||
reason. Launched from cold, which is the ordinary case for "open
|
||||
this photograph", the Intent is on getIntent() and is read. -->
|
||||
<activity
|
||||
android:name="android.app.NativeActivity"
|
||||
android:exported="true"
|
||||
android:launchMode="singleTask"
|
||||
android:configChanges="orientation|keyboardHidden|screenSize|screenLayout|density|uiMode"
|
||||
android:windowSoftInputMode="adjustResize">
|
||||
|
||||
<meta-data
|
||||
android:name="android.app.lib_name"
|
||||
android:value="darkroom" />
|
||||
|
||||
<intent-filter>
|
||||
<action android:name="android.intent.action.MAIN" />
|
||||
<category android:name="android.intent.category.LAUNCHER" />
|
||||
</intent-filter>
|
||||
|
||||
<!-- FR-PLAT-AND-6, inbound. The traceability tool reads .rs,
|
||||
.slint, .wgsl and .yaml, so this is a reference and not a
|
||||
tag; the tag that counts is on the test in lib.rs that
|
||||
asserts these declarations are still here.
|
||||
|
||||
Opening a photograph from a gallery, a file manager or a
|
||||
download. `android_main` reads the launch Intent through
|
||||
`Intents.receive` and the named images become the browsing
|
||||
list, exactly as paths on the desktop command line do.
|
||||
|
||||
`image/*` and not a wider match, even though it misses raws:
|
||||
a provider that does not recognise `.CR3` reports it as
|
||||
`application/octet-stream`, and claiming that type would put
|
||||
DarkRoom in the chooser for every unidentified binary on the
|
||||
device — an APK, a database, a partial download. Being absent
|
||||
from one gallery's menu is a smaller failure than being
|
||||
present in all of them. DNG, which providers do know as
|
||||
`image/x-adobe-dng`, matches here already.
|
||||
|
||||
BROWSABLE is what lets a browser's finished download and a
|
||||
link hand the file over; without it those routes silently do
|
||||
not list the app. -->
|
||||
<intent-filter>
|
||||
<action android:name="android.intent.action.VIEW" />
|
||||
<category android:name="android.intent.category.DEFAULT" />
|
||||
<category android:name="android.intent.category.BROWSABLE" />
|
||||
<data android:mimeType="image/*" />
|
||||
</intent-filter>
|
||||
|
||||
<!-- The share sheet, one photograph or a selection of them.
|
||||
SEND_MULTIPLE is declared because the sheet offers this app
|
||||
for a multi-selection only if it says it accepts one, and a
|
||||
culling tool that can be sent a single frame and not a burst
|
||||
is the wrong way round.
|
||||
|
||||
ACTION_EDIT is deliberately not here. It is a promise to write
|
||||
the result back to the URI it was handed, and nothing in this
|
||||
app does: an edit lands in a sidecar beside the original
|
||||
(FR-CAT-8). Registering for it would put DarkRoom in the "edit
|
||||
with" menu and lose the user's work every time. -->
|
||||
<intent-filter>
|
||||
<action android:name="android.intent.action.SEND" />
|
||||
<action android:name="android.intent.action.SEND_MULTIPLE" />
|
||||
<category android:name="android.intent.category.DEFAULT" />
|
||||
<data android:mimeType="image/*" />
|
||||
</intent-filter>
|
||||
</activity>
|
||||
|
||||
<!-- FR-PLAT-AND-6, outbound. Android has refused file:// URIs
|
||||
between apps since API 24 — handing one out raises
|
||||
FileUriExposedException in *this* process — so an exported JPEG
|
||||
reaches the share sheet as a content:// URI or not at all.
|
||||
|
||||
Not AndroidX's FileProvider: that is a Maven artefact, and this
|
||||
build has no Gradle and no dependency resolver (docker/android/
|
||||
README.md). ExportProvider does the same hundred lines against one
|
||||
fixed root.
|
||||
|
||||
`exported="false"` with `grantUriPermissions="true"` is the whole
|
||||
security model, and the two halves are not redundant. Exported
|
||||
false means no app may address the provider on its own account;
|
||||
the grant flag means a URI this app puts in an Intent carries a
|
||||
read permission for that one file, for the lifetime of the
|
||||
receiving task. Without the grant flag the share sheet opens and
|
||||
every target fails with SecurityException; with `exported="true"`
|
||||
instead, every app on the device could read the app's private
|
||||
directory. The authority must equal ExportProvider.AUTHORITY — a
|
||||
mismatch is a SecurityException in somebody else's app, so a test
|
||||
in lib.rs compares the two strings. -->
|
||||
<provider
|
||||
android:name="paris.tourolle.darkroom.ExportProvider"
|
||||
android:authorities="paris.tourolle.darkroom.exports"
|
||||
android:exported="false"
|
||||
android:grantUriPermissions="true" />
|
||||
</application>
|
||||
</manifest>
|
||||
@@ -1,260 +0,0 @@
|
||||
package paris.tourolle.darkroom;
|
||||
|
||||
import android.content.ContentProvider;
|
||||
import android.content.ContentValues;
|
||||
import android.content.Context;
|
||||
import android.database.Cursor;
|
||||
import android.database.MatrixCursor;
|
||||
import android.net.Uri;
|
||||
import android.os.ParcelFileDescriptor;
|
||||
import android.provider.OpenableColumns;
|
||||
import android.util.Log;
|
||||
import android.webkit.MimeTypeMap;
|
||||
|
||||
import java.io.File;
|
||||
import java.io.FileNotFoundException;
|
||||
import java.io.IOException;
|
||||
import java.util.List;
|
||||
import java.util.Locale;
|
||||
|
||||
/**
|
||||
* Hands an exported file to another app, and hands out nothing else.
|
||||
*
|
||||
* <p>FR-PLAT-AND-6's outbound half. Android has refused {@code file://} URIs
|
||||
* between apps since API 24 — passing one raises {@code FileUriExposedException}
|
||||
* in the *sending* process — so the only way to give a photo to the share sheet
|
||||
* is a {@code content://} URI backed by a provider, plus a per-Intent read
|
||||
* grant that expires with the task that received it.
|
||||
*
|
||||
* <h2>Why this is not AndroidX's FileProvider</h2>
|
||||
*
|
||||
* <p>Because AndroidX is a Maven artefact and this build has no Gradle and no
|
||||
* dependency resolver (see docker/android/README.md). Pulling in the one class
|
||||
* would mean adopting the whole mechanism that fetches it. What
|
||||
* {@code FileProvider} does is a hundred lines — map a request path onto a
|
||||
* directory, refuse anything outside it, answer the two columns the share sheet
|
||||
* reads — and those lines are below. The configuration it takes as an XML
|
||||
* {@code <meta-data>} resource is a constant here instead, because there is
|
||||
* exactly one directory worth serving and a second place to state it is a
|
||||
* second place for it to be wrong.
|
||||
*
|
||||
* <h2>The one directory</h2>
|
||||
*
|
||||
* <p>{@code getFilesDir()}, which is the same directory the Rust side calls
|
||||
* {@code internal_data_path} and passes to {@code dr_sync::account::set_data_dir}
|
||||
* — {@code ANativeActivity.internalDataPath} and {@code Context.getFilesDir()}
|
||||
* are the same path. Everything the app writes for itself, the export outbox
|
||||
* included, is under it. Nothing else is reachable: a request is resolved
|
||||
* against the real filesystem with {@link File#getCanonicalFile()} and then
|
||||
* checked to be *inside* that root, so {@code ../} and a symlink planted in the
|
||||
* outbox are refused by the same test. Serving a path the caller composed,
|
||||
* unchecked, would turn a share button into a reader for every file this app
|
||||
* can see, which on Android includes credentials and the whole catalog.
|
||||
*
|
||||
* <p>{@code android:exported="false"} in the manifest is the outer half of the
|
||||
* same rule: no app can address this provider at all except through a URI this
|
||||
* app handed it with a read grant attached.
|
||||
*/
|
||||
public final class ExportProvider extends ContentProvider {
|
||||
private static final String TAG = "DarkRoom";
|
||||
|
||||
/**
|
||||
* Must equal {@code android:authorities} in AndroidManifest.xml.
|
||||
*
|
||||
* <p>A mismatch is not a build error and not a runtime error here: it is a
|
||||
* {@code SecurityException} in whichever app opened the share sheet, naming
|
||||
* an authority that does not exist. A test in {@code lib.rs} asserts the
|
||||
* two strings are the same for that reason.
|
||||
*/
|
||||
public static final String AUTHORITY = "paris.tourolle.darkroom.exports";
|
||||
|
||||
/** Nothing to set up; the root is resolved per request against the context. */
|
||||
@Override
|
||||
public boolean onCreate() {
|
||||
return true;
|
||||
}
|
||||
|
||||
/**
|
||||
* The {@code content://} URI for a file, or null if it is not one this
|
||||
* provider may serve.
|
||||
*
|
||||
* <p>Returning null rather than an unusable URI keeps the refusal at the
|
||||
* point where the path is known. A URI for a file outside the root would be
|
||||
* rejected later by {@link #openFile}, in the *receiving* app's stack trace,
|
||||
* where nothing says which of our files was asked for.
|
||||
*/
|
||||
public static Uri uriFor(Context context, File file) {
|
||||
try {
|
||||
File root = root(context);
|
||||
File target = file.getCanonicalFile();
|
||||
String relative = within(root, target);
|
||||
if (relative == null) {
|
||||
Log.w(TAG, "not shareable, outside " + root + ": " + target);
|
||||
return null;
|
||||
}
|
||||
// Built segment by segment rather than with a composed path
|
||||
// string: appendPath percent-encodes, and getPathSegments below
|
||||
// decodes symmetrically. A file called "Rue d'Alésia.jpg" survives
|
||||
// the round trip only because both halves agree.
|
||||
Uri.Builder builder = new Uri.Builder().scheme("content").authority(AUTHORITY);
|
||||
for (String segment : relative.split("/")) {
|
||||
if (!segment.isEmpty()) {
|
||||
builder.appendPath(segment);
|
||||
}
|
||||
}
|
||||
return builder.build();
|
||||
} catch (IOException e) {
|
||||
Log.w(TAG, "cannot resolve " + file + " for sharing: " + e);
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* The two columns a share target actually reads.
|
||||
*
|
||||
* <p>Without {@code _display_name} the receiving app shows the URI's last
|
||||
* segment, and without {@code _size} a mail client cannot tell whether the
|
||||
* attachment fits before it starts reading. Both are optional in the sense
|
||||
* that the transfer still works; both are the difference between "DSC_4471
|
||||
* final.jpg, 8.2 MB" and an unnamed blob.
|
||||
*/
|
||||
@Override
|
||||
public Cursor query(Uri uri, String[] projection, String selection,
|
||||
String[] selectionArgs, String sortOrder) {
|
||||
File file = resolve(uri);
|
||||
if (file == null) {
|
||||
return null;
|
||||
}
|
||||
String[] columns = projection != null
|
||||
? projection
|
||||
: new String[] {OpenableColumns.DISPLAY_NAME, OpenableColumns.SIZE};
|
||||
MatrixCursor cursor = new MatrixCursor(columns, 1);
|
||||
MatrixCursor.RowBuilder row = cursor.newRow();
|
||||
for (String column : columns) {
|
||||
if (OpenableColumns.DISPLAY_NAME.equals(column)) {
|
||||
row.add(file.getName());
|
||||
} else if (OpenableColumns.SIZE.equals(column)) {
|
||||
row.add(file.length());
|
||||
} else {
|
||||
// A column we do not have. Null rather than omitted: a cursor
|
||||
// whose row is shorter than its projection throws in the
|
||||
// caller, which is a crash in someone else's app.
|
||||
row.add(null);
|
||||
}
|
||||
}
|
||||
return cursor;
|
||||
}
|
||||
|
||||
/**
|
||||
* From the extension, because that is all there is.
|
||||
*
|
||||
* <p>The type decides which apps the chooser offers, so guessing wrong
|
||||
* narrows the sheet rather than breaking the transfer. Exports are JPEG,
|
||||
* PNG or TIFF and {@code MimeTypeMap} knows all three.
|
||||
*/
|
||||
@Override
|
||||
public String getType(Uri uri) {
|
||||
File file = resolve(uri);
|
||||
if (file == null) {
|
||||
return null;
|
||||
}
|
||||
String name = file.getName();
|
||||
int dot = name.lastIndexOf('.');
|
||||
if (dot >= 0 && dot < name.length() - 1) {
|
||||
String extension = name.substring(dot + 1).toLowerCase(Locale.ROOT);
|
||||
String type = MimeTypeMap.getSingleton().getMimeTypeFromExtension(extension);
|
||||
if (type != null) {
|
||||
return type;
|
||||
}
|
||||
}
|
||||
return "application/octet-stream";
|
||||
}
|
||||
|
||||
/**
|
||||
* Read-only, always.
|
||||
*
|
||||
* <p>A write mode is refused rather than quietly downgraded: a caller that
|
||||
* asked for "rw" intends to save something back, and letting it open the
|
||||
* file read-only would fail at its first write with an error about a
|
||||
* descriptor rather than about permission. Nothing this app shares is meant
|
||||
* to be edited in place by the app it was shared with.
|
||||
*/
|
||||
@Override
|
||||
public ParcelFileDescriptor openFile(Uri uri, String mode) throws FileNotFoundException {
|
||||
if (!"r".equals(mode)) {
|
||||
throw new SecurityException("this provider is read-only, asked for '" + mode + "'");
|
||||
}
|
||||
File file = resolve(uri);
|
||||
if (file == null) {
|
||||
throw new FileNotFoundException("no such export: " + uri);
|
||||
}
|
||||
return ParcelFileDescriptor.open(file, ParcelFileDescriptor.MODE_READ_ONLY);
|
||||
}
|
||||
|
||||
@Override
|
||||
public Uri insert(Uri uri, ContentValues values) {
|
||||
throw new UnsupportedOperationException("exports are written by the app, not through it");
|
||||
}
|
||||
|
||||
@Override
|
||||
public int update(Uri uri, ContentValues values, String selection, String[] selectionArgs) {
|
||||
throw new UnsupportedOperationException("exports are written by the app, not through it");
|
||||
}
|
||||
|
||||
@Override
|
||||
public int delete(Uri uri, String selection, String[] selectionArgs) {
|
||||
throw new UnsupportedOperationException("exports are deleted by the app, not through it");
|
||||
}
|
||||
|
||||
/** The served root, resolved through the filesystem so the check below is real. */
|
||||
private static File root(Context context) throws IOException {
|
||||
return context.getFilesDir().getCanonicalFile();
|
||||
}
|
||||
|
||||
/** The file a request names, or null if it names anything else. */
|
||||
private File resolve(Uri uri) {
|
||||
Context context = getContext();
|
||||
if (context == null) {
|
||||
return null;
|
||||
}
|
||||
List<String> segments = uri.getPathSegments();
|
||||
if (segments.isEmpty()) {
|
||||
return null;
|
||||
}
|
||||
try {
|
||||
File root = root(context);
|
||||
File candidate = root;
|
||||
for (String segment : segments) {
|
||||
candidate = new File(candidate, segment);
|
||||
}
|
||||
candidate = candidate.getCanonicalFile();
|
||||
if (within(root, candidate) == null || !candidate.isFile()) {
|
||||
Log.w(TAG, "refused " + uri);
|
||||
return null;
|
||||
}
|
||||
return candidate;
|
||||
} catch (IOException e) {
|
||||
Log.w(TAG, "refused " + uri + ": " + e);
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* {@code target}'s path relative to {@code root}, or null if it is not
|
||||
* under it.
|
||||
*
|
||||
* <p>Both sides are canonical by the time they get here, which is what
|
||||
* makes one string comparison enough for {@code ../} and for a symlink
|
||||
* alike. The trailing separator matters: without it a sibling directory
|
||||
* whose name merely starts with the root's — {@code /data/.../files.old} —
|
||||
* passes.
|
||||
*/
|
||||
private static String within(File root, File target) {
|
||||
String rootPath = root.getPath() + File.separator;
|
||||
String targetPath = target.getPath();
|
||||
if (!targetPath.startsWith(rootPath)) {
|
||||
return null;
|
||||
}
|
||||
return targetPath.substring(rootPath.length());
|
||||
}
|
||||
}
|
||||
@@ -1,320 +0,0 @@
|
||||
package paris.tourolle.darkroom;
|
||||
|
||||
import android.app.Activity;
|
||||
import android.content.ActivityNotFoundException;
|
||||
import android.content.ContentResolver;
|
||||
import android.content.Context;
|
||||
import android.content.Intent;
|
||||
import android.database.Cursor;
|
||||
import android.net.Uri;
|
||||
import android.provider.OpenableColumns;
|
||||
import android.util.Log;
|
||||
|
||||
import java.io.File;
|
||||
import java.io.FileOutputStream;
|
||||
import java.io.IOException;
|
||||
import java.io.InputStream;
|
||||
import java.io.OutputStream;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
|
||||
/**
|
||||
* The two directions of FR-PLAT-AND-6: what the app was opened *with*, and
|
||||
* handing a finished export to somebody else.
|
||||
*
|
||||
* <h2>Why this is Java and not JNI in lib.rs</h2>
|
||||
*
|
||||
* <p>Every call below is reachable over JNI, and doing it that way would be
|
||||
* roughly forty {@code call_method} invocations with their signatures written
|
||||
* out as strings — each one a name Java checks at run time and nothing checks
|
||||
* at build time. The Rust side would then hold the exact logic that is here,
|
||||
* expressed less clearly, and a typo in {@code "()Landroid/content/Intent;"}
|
||||
* would surface on a device as a {@code NoSuchMethodError} rather than at the
|
||||
* compiler. So the platform work stays on the platform's side and the JNI
|
||||
* surface is two calls, both taking and returning strings.
|
||||
*
|
||||
* <p>The class is only reachable because the APK now compiles Java at all; see
|
||||
* docker/android/assemble-apk.sh.
|
||||
*/
|
||||
public final class Intents {
|
||||
private static final String TAG = "DarkRoom";
|
||||
|
||||
/**
|
||||
* Where incoming images are copied, under {@code getCacheDir()}.
|
||||
*
|
||||
* <p>The cache and not the data directory, deliberately: these are copies
|
||||
* of somebody else's file, the app has no claim on them once the session
|
||||
* ends, and the cache is the one place Android may reclaim under storage
|
||||
* pressure without the user being asked. Putting them in the data
|
||||
* directory would grow the app's footprint by a RAW file per share, for
|
||||
* ever, with nothing that ever deletes them.
|
||||
*/
|
||||
private static final String INBOX = "incoming";
|
||||
|
||||
private Intents() {
|
||||
}
|
||||
|
||||
/**
|
||||
* The images this launch was asked to open, as paths the decoder can read.
|
||||
*
|
||||
* <p>Empty for an ordinary launch from the launcher, which is the common
|
||||
* case and not a failure.
|
||||
*
|
||||
* <h3>Why the bytes are copied</h3>
|
||||
*
|
||||
* <p>A share arrives as a {@code content://} URI, which is a handle into
|
||||
* another app's provider and not a path — there is no filename behind it to
|
||||
* open, and the grant that makes it readable belongs to this task and dies
|
||||
* with it. DarkRoom's decoders take paths (ARCH §6.9 is the note that
|
||||
* Android has no paths to give), so the choice is to copy or to teach the
|
||||
* whole read path about URIs, and the second is FR-PLAT-AND-1's SAF
|
||||
* connector, which is not built.
|
||||
*
|
||||
* <p>So it is a copy, and the cost is honest: a 60 MB raw file is written
|
||||
* once, to the cache, before the viewer opens. It is bounded by the share
|
||||
* being a deliberate act — a person picked these files — rather than by
|
||||
* anything this code does.
|
||||
*
|
||||
* <p>The inbox is emptied first. Without that, every share ever received
|
||||
* accumulates until the platform decides the cache is too large, and the
|
||||
* files are indistinguishable from each other by then.
|
||||
*/
|
||||
public static String[] receive(Activity activity) {
|
||||
List<Uri> uris = incoming(activity.getIntent());
|
||||
if (uris.isEmpty()) {
|
||||
return new String[0];
|
||||
}
|
||||
|
||||
File inbox = new File(activity.getCacheDir(), INBOX);
|
||||
empty(inbox);
|
||||
if (!inbox.mkdirs() && !inbox.isDirectory()) {
|
||||
Log.e(TAG, "cannot create " + inbox + "; the launch intent is dropped");
|
||||
return new String[0];
|
||||
}
|
||||
|
||||
List<String> paths = new ArrayList<String>();
|
||||
for (Uri uri : uris) {
|
||||
String path = localise(activity, uri, inbox, paths.size());
|
||||
if (path != null) {
|
||||
paths.add(path);
|
||||
}
|
||||
}
|
||||
Log.i(TAG, "launch intent carried " + paths.size() + " of " + uris.size() + " image(s)");
|
||||
return paths.toArray(new String[0]);
|
||||
}
|
||||
|
||||
/**
|
||||
* Offer a file this app produced to whatever else is installed.
|
||||
*
|
||||
* <p>Returns false when there is nothing to offer it to, or when the file
|
||||
* is not one {@link ExportProvider} may serve — both of which the caller
|
||||
* has to be able to say out loud, because from the user's side a share
|
||||
* button that does nothing is indistinguishable from one that failed.
|
||||
*
|
||||
* <p>{@code FLAG_GRANT_READ_URI_PERMISSION} is the whole security model:
|
||||
* the provider is not exported, so the receiving app can reach this one
|
||||
* file, for as long as its task lives, and nothing else ever.
|
||||
*/
|
||||
public static boolean share(Activity activity, String path, String mimeType) {
|
||||
Uri uri = ExportProvider.uriFor(activity, new File(path));
|
||||
if (uri == null) {
|
||||
return false;
|
||||
}
|
||||
|
||||
Intent send = new Intent(Intent.ACTION_SEND);
|
||||
send.setType(mimeType != null && !mimeType.isEmpty() ? mimeType : "image/*");
|
||||
send.putExtra(Intent.EXTRA_STREAM, uri);
|
||||
send.addFlags(Intent.FLAG_GRANT_READ_URI_PERMISSION);
|
||||
|
||||
// Always a chooser, never a direct start. Android's "remembered
|
||||
// default" for ACTION_SEND is a per-user setting this app has no
|
||||
// business consuming: the app a photograph should go to differs every
|
||||
// time, and the one time it does not, the sheet is one extra tap.
|
||||
Intent chooser = Intent.createChooser(send, null);
|
||||
try {
|
||||
activity.startActivity(chooser);
|
||||
return true;
|
||||
} catch (ActivityNotFoundException e) {
|
||||
Log.w(TAG, "nothing installed accepts " + mimeType + ": " + e);
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* The URIs an Intent carries, by the action that carried them.
|
||||
*
|
||||
* <p>Only the actions the manifest registers for. An action we did not
|
||||
* declare cannot arrive, so handling one here would be code that reads as
|
||||
* support for something the launcher will never offer.
|
||||
*/
|
||||
@SuppressWarnings("deprecation")
|
||||
private static List<Uri> incoming(Intent intent) {
|
||||
List<Uri> uris = new ArrayList<Uri>();
|
||||
if (intent == null) {
|
||||
return uris;
|
||||
}
|
||||
String action = intent.getAction();
|
||||
if (Intent.ACTION_VIEW.equals(action)) {
|
||||
add(uris, intent.getData());
|
||||
} else if (Intent.ACTION_SEND.equals(action)) {
|
||||
// The typed getParcelableExtra(String, Class) overload is API 33,
|
||||
// and minSdk is 28. The deprecated form is the only one that exists
|
||||
// on every device this APK installs on.
|
||||
add(uris, (Uri) intent.getParcelableExtra(Intent.EXTRA_STREAM));
|
||||
} else if (Intent.ACTION_SEND_MULTIPLE.equals(action)) {
|
||||
ArrayList<Uri> many = intent.getParcelableArrayListExtra(Intent.EXTRA_STREAM);
|
||||
if (many != null) {
|
||||
for (Uri uri : many) {
|
||||
add(uris, uri);
|
||||
}
|
||||
}
|
||||
}
|
||||
return uris;
|
||||
}
|
||||
|
||||
private static void add(List<Uri> uris, Uri uri) {
|
||||
if (uri != null) {
|
||||
uris.add(uri);
|
||||
}
|
||||
}
|
||||
|
||||
/** A URI as a readable path, copying it into the inbox if it is not one already. */
|
||||
private static String localise(Context context, Uri uri, File inbox, int index) {
|
||||
// A file:// URI is already a path, and copying it would double a raw
|
||||
// file on disk to no end. Rare — the platform has refused file:// URIs
|
||||
// between apps since API 24 — but it is what a shell `am start -d
|
||||
// file:///sdcard/…` produces, which is how this path gets tested
|
||||
// without a second app installed.
|
||||
if (ContentResolver.SCHEME_FILE.equals(uri.getScheme())) {
|
||||
String path = uri.getPath();
|
||||
if (path != null && new File(path).canRead()) {
|
||||
return path;
|
||||
}
|
||||
Log.w(TAG, "cannot read " + uri);
|
||||
return null;
|
||||
}
|
||||
|
||||
File dest = new File(inbox, unique(inbox, displayName(context, uri), index));
|
||||
InputStream in = null;
|
||||
OutputStream out = null;
|
||||
try {
|
||||
in = context.getContentResolver().openInputStream(uri);
|
||||
if (in == null) {
|
||||
Log.w(TAG, "no stream behind " + uri);
|
||||
return null;
|
||||
}
|
||||
out = new FileOutputStream(dest);
|
||||
byte[] buffer = new byte[64 * 1024];
|
||||
int read;
|
||||
while ((read = in.read(buffer)) > 0) {
|
||||
out.write(buffer, 0, read);
|
||||
}
|
||||
out.flush();
|
||||
return dest.getAbsolutePath();
|
||||
} catch (IOException e) {
|
||||
Log.w(TAG, "cannot copy " + uri + ": " + e);
|
||||
// The partial copy is removed rather than left: it has the name and
|
||||
// the extension of a photograph and none of the bytes, and the
|
||||
// decoder would report it as a corrupt file rather than a failed
|
||||
// transfer.
|
||||
dest.delete();
|
||||
return null;
|
||||
} catch (SecurityException e) {
|
||||
// The grant on a shared URI dies with the task that received it.
|
||||
// A process resumed from a saved state can find itself holding a
|
||||
// URI it may no longer read (FR-PLAT-AND-3), and that is a lost
|
||||
// permission rather than a broken file.
|
||||
Log.w(TAG, "no longer permitted to read " + uri + ": " + e);
|
||||
dest.delete();
|
||||
return null;
|
||||
} finally {
|
||||
close(in);
|
||||
close(out);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* What the sending app calls the file, reduced to something safe to write.
|
||||
*
|
||||
* <p>The name is chosen by another application and lands in a path this one
|
||||
* composes, so it is filtered rather than trusted: a name containing a
|
||||
* separator would place the copy outside the inbox, and one beginning with
|
||||
* a dot would hide it from everything that lists the directory. What
|
||||
* survives is the part a photographer recognises — {@code DSC_4471.NEF} —
|
||||
* which is the only reason to use the sender's name at all.
|
||||
*/
|
||||
private static String displayName(Context context, Uri uri) {
|
||||
String name = null;
|
||||
Cursor cursor = null;
|
||||
try {
|
||||
cursor = context.getContentResolver().query(
|
||||
uri, new String[] {OpenableColumns.DISPLAY_NAME}, null, null, null);
|
||||
if (cursor != null && cursor.moveToFirst() && !cursor.isNull(0)) {
|
||||
name = cursor.getString(0);
|
||||
}
|
||||
} catch (Exception e) {
|
||||
// Providers are other people's code and any of them may throw.
|
||||
// A name is a convenience; failing the whole open over it is not.
|
||||
Log.d(TAG, "no display name for " + uri + ": " + e);
|
||||
} finally {
|
||||
if (cursor != null) {
|
||||
cursor.close();
|
||||
}
|
||||
}
|
||||
if (name == null) {
|
||||
name = uri.getLastPathSegment();
|
||||
}
|
||||
if (name == null) {
|
||||
return "shared";
|
||||
}
|
||||
StringBuilder safe = new StringBuilder(name.length());
|
||||
for (int i = 0; i < name.length(); i++) {
|
||||
char c = name.charAt(i);
|
||||
boolean ok = (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z')
|
||||
|| (c >= '0' && c <= '9') || c == '.' || c == '-' || c == '_';
|
||||
safe.append(ok ? c : '_');
|
||||
}
|
||||
while (safe.length() > 0 && safe.charAt(0) == '.') {
|
||||
safe.deleteCharAt(0);
|
||||
}
|
||||
return safe.length() > 0 ? safe.toString() : "shared";
|
||||
}
|
||||
|
||||
/**
|
||||
* A name nothing in the inbox has yet.
|
||||
*
|
||||
* <p>A multi-image share of a burst arrives as several files a camera named
|
||||
* the same thing in different folders, and the second one silently
|
||||
* overwriting the first would show the user one photograph where they
|
||||
* picked four.
|
||||
*/
|
||||
private static String unique(File inbox, String name, int index) {
|
||||
if (!new File(inbox, name).exists()) {
|
||||
return name;
|
||||
}
|
||||
return index + "-" + name;
|
||||
}
|
||||
|
||||
private static void close(java.io.Closeable stream) {
|
||||
if (stream != null) {
|
||||
try {
|
||||
stream.close();
|
||||
} catch (IOException e) {
|
||||
Log.d(TAG, "close failed: " + e);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/** Delete the inbox's contents, one level deep, which is all it ever has. */
|
||||
private static void empty(File inbox) {
|
||||
File[] stale = inbox.listFiles();
|
||||
if (stale == null) {
|
||||
return;
|
||||
}
|
||||
for (File file : stale) {
|
||||
if (!file.delete()) {
|
||||
Log.d(TAG, "could not remove stale " + file);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,6 +0,0 @@
|
||||
<?xml version="1.0" encoding="utf-8"?>
|
||||
<adaptive-icon xmlns:android="http://schemas.android.com/apk/res/android">
|
||||
<background android:drawable="@mipmap/ic_launcher_background"/>
|
||||
<foreground android:drawable="@mipmap/ic_launcher_foreground"/>
|
||||
<monochrome android:drawable="@mipmap/ic_launcher_monochrome"/>
|
||||
</adaptive-icon>
|
||||
Binary file not shown.
|
Before Width: | Height: | Size: 9.5 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 518 B |
Binary file not shown.
|
Before Width: | Height: | Size: 25 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 25 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 5.0 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 343 B |
Binary file not shown.
|
Before Width: | Height: | Size: 13 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 13 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 15 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 680 B |
Binary file not shown.
|
Before Width: | Height: | Size: 43 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 43 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 30 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 1005 B |
Binary file not shown.
|
Before Width: | Height: | Size: 87 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 87 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 51 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 1.6 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 151 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 151 KiB |
@@ -1,192 +0,0 @@
|
||||
//! What the app was launched with, and handing a finished export back out.
|
||||
//!
|
||||
//! FR-PLAT-AND-6's Rust side, which is deliberately the thin side. Both
|
||||
//! directions are implemented in `android/java/paris/tourolle/darkroom/` and
|
||||
//! everything here is the two calls that reach them; `Intents.java` carries the
|
||||
//! reasoning for the split. The short version is that a JNI method signature is
|
||||
//! a string Java resolves at run time and nothing checks at build time, so
|
||||
//! forty of them is forty ways for a rename to become a `NoSuchMethodError` on
|
||||
//! somebody's tablet. Two is two.
|
||||
//!
|
||||
//! # Nothing here fails loudly
|
||||
//!
|
||||
//! A class the loader cannot see, a pending Java exception, a shared URI whose
|
||||
//! grant died with the task that received it: each ends as a log line and an
|
||||
//! empty result. This runs on the way to [`dr_ui::run`], before a window
|
||||
//! exists, and the alternative to opening with an empty browsing list is not
|
||||
//! opening at all.
|
||||
|
||||
use std::path::{Path, PathBuf};
|
||||
|
||||
use jni::errors::Result as JniResult;
|
||||
use jni::objects::{JClass, JObject, JObjectArray, JString, JValue};
|
||||
use jni::{JNIEnv, JavaVM};
|
||||
|
||||
/// The class both directions live in, named the way `loadClass` wants it —
|
||||
/// dots, not slashes. `find_class` takes the other form, and this code calls
|
||||
/// neither by accident; see [`load_class`].
|
||||
const INTENTS: &str = "paris.tourolle.darkroom.Intents";
|
||||
|
||||
/// The images this launch was asked to open, already local and readable.
|
||||
///
|
||||
/// Empty for an ordinary launch from the launcher, which is the common case
|
||||
/// and not a failure. What comes back is passed to `dr_ui::run` exactly as
|
||||
/// command-line paths are on the desktop, so a shared photograph becomes the
|
||||
/// browsing list and `startup_action` shows it rather than the launch screen.
|
||||
pub fn launch_images(app: &slint::android::AndroidApp) -> Vec<PathBuf> {
|
||||
with_activity(app, "reading the launch intent", |env, activity| {
|
||||
let class = load_class(env, activity, INTENTS)?;
|
||||
let returned = env
|
||||
.call_static_method(
|
||||
&class,
|
||||
"receive",
|
||||
"(Landroid/app/Activity;)[Ljava/lang/String;",
|
||||
&[JValue::Object(activity)],
|
||||
)?
|
||||
.l()?;
|
||||
|
||||
let array = JObjectArray::from(returned);
|
||||
let count = env.get_array_length(&array)?;
|
||||
let mut paths = Vec::with_capacity(count as usize);
|
||||
for i in 0..count {
|
||||
let element = env.get_object_array_element(&array, i)?;
|
||||
let text: String = env.get_string(&JString::from(element))?.into();
|
||||
paths.push(PathBuf::from(text));
|
||||
}
|
||||
Ok(paths)
|
||||
})
|
||||
.unwrap_or_default()
|
||||
}
|
||||
|
||||
/// Offer a file this app produced to whatever else is installed.
|
||||
///
|
||||
/// `false` means the sheet did not open — the file is not under the directory
|
||||
/// [`ExportProvider`] serves, or nothing installed accepts the type. Both are
|
||||
/// answers a caller has to be able to give the user, because a share control
|
||||
/// that silently does nothing is indistinguishable from one that failed.
|
||||
///
|
||||
/// **This half has no caller yet, and that is the honest state of it.** The
|
||||
/// provider, the URI grant and the chooser are all here and are what
|
||||
/// FR-PLAT-AND-6 asks for; what is missing is a share control in the interface,
|
||||
/// which lives in `ui/dr-ui` and needs one thing this signature shows: an
|
||||
/// `AndroidApp` to call through. Wiring it means keeping a clone of the app —
|
||||
/// it is `Clone` and cheap — somewhere `ui/` can reach, which is a change to
|
||||
/// how the platform entry point talks to the interface rather than a change
|
||||
/// here. Until that exists this function is reachable and untested, and it is
|
||||
/// deliberately not tagged as covering the requirement.
|
||||
///
|
||||
/// `mime` decides which applications the chooser offers; the empty string
|
||||
/// falls back to `image/*` on the Java side.
|
||||
pub fn share(app: &slint::android::AndroidApp, file: &Path, mime: &str) -> bool {
|
||||
with_activity(app, "opening the share sheet", |env, activity| {
|
||||
let class = load_class(env, activity, INTENTS)?;
|
||||
let path = env.new_string(file.to_string_lossy().as_ref())?;
|
||||
let mime = env.new_string(mime)?;
|
||||
env.call_static_method(
|
||||
&class,
|
||||
"share",
|
||||
"(Landroid/app/Activity;Ljava/lang/String;Ljava/lang/String;)Z",
|
||||
&[
|
||||
JValue::Object(activity),
|
||||
JValue::Object(&path),
|
||||
JValue::Object(&mime),
|
||||
],
|
||||
)?
|
||||
.z()
|
||||
})
|
||||
.unwrap_or(false)
|
||||
}
|
||||
|
||||
/// Attach to the JVM, borrow the activity, and run `body` against both.
|
||||
///
|
||||
/// Shared by the two entry points because the three steps before the
|
||||
/// interesting one are identical and each has its own way of failing. `body`
|
||||
/// returning `Err` is reported here, once, in the one place that can also clear
|
||||
/// a pending Java exception — see [`report`].
|
||||
fn with_activity<T>(
|
||||
app: &slint::android::AndroidApp,
|
||||
doing: &str,
|
||||
body: impl FnOnce(&mut JNIEnv, &JObject) -> JniResult<T>,
|
||||
) -> Option<T> {
|
||||
let vm = match unsafe { JavaVM::from_raw(app.vm_as_ptr().cast()) } {
|
||||
Ok(vm) => vm,
|
||||
Err(e) => {
|
||||
log::error!("no JVM handle, so {doing} is skipped: {e}");
|
||||
return None;
|
||||
}
|
||||
};
|
||||
// Cheap when the thread is already attached, which it is: the glue
|
||||
// attached it before it called `android_main`. The guard exists for the
|
||||
// case where it is not, and costs a lookup where it is.
|
||||
let mut env = match vm.attach_current_thread() {
|
||||
Ok(env) => env,
|
||||
Err(e) => {
|
||||
log::error!("cannot attach to the JVM, so {doing} is skipped: {e}");
|
||||
return None;
|
||||
}
|
||||
};
|
||||
|
||||
// SAFETY: `activity_as_ptr` documents this as an unowned JNI *global*
|
||||
// reference to the Activity, valid for as long as the `AndroidApp` it came
|
||||
// from. `JObject` in jni 0.21 is a plain wrapper with no `Drop`, so
|
||||
// borrowing it here cannot delete a reference this code does not own — the
|
||||
// one way to get this wrong is `AutoLocal` or a `GlobalRef`, both of which
|
||||
// would free it out from under android-activity.
|
||||
let activity = unsafe { JObject::from_raw(app.activity_as_ptr().cast()) };
|
||||
|
||||
match body(&mut env, &activity) {
|
||||
Ok(value) => Some(value),
|
||||
Err(e) => {
|
||||
report(&mut env, doing, &e);
|
||||
None
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Look an app class up through the *activity's* class loader.
|
||||
///
|
||||
/// `find_class` is the obvious call and the wrong one. JNI resolves a class
|
||||
/// against the loader belonging to the Java frame beneath the call, and on this
|
||||
/// thread there is no such frame: `android_main` runs on a thread the native
|
||||
/// glue created and attached itself, so the loader in scope is the system one.
|
||||
/// It knows every class in the platform and nothing at all from this APK, and
|
||||
/// says so as a `ClassNotFoundException` naming a class that is plainly in the
|
||||
/// dex — which reads as a broken build rather than as the wrong loader.
|
||||
///
|
||||
/// The activity is a Java object, so its loader is the app's.
|
||||
fn load_class<'local>(
|
||||
env: &mut JNIEnv<'local>,
|
||||
activity: &JObject,
|
||||
name: &str,
|
||||
) -> JniResult<JClass<'local>> {
|
||||
let loader = env
|
||||
.call_method(activity, "getClassLoader", "()Ljava/lang/ClassLoader;", &[])?
|
||||
.l()?;
|
||||
let name = env.new_string(name)?;
|
||||
let class = env
|
||||
.call_method(
|
||||
&loader,
|
||||
"loadClass",
|
||||
"(Ljava/lang/String;)Ljava/lang/Class;",
|
||||
&[JValue::Object(&name)],
|
||||
)?
|
||||
.l()?;
|
||||
Ok(JClass::from(class))
|
||||
}
|
||||
|
||||
/// Log a JNI failure, and clear the exception behind it if there is one.
|
||||
///
|
||||
/// The clearing is not tidiness. A Java exception raised through JNI stays
|
||||
/// *pending* on the thread, and the next JNI call made while one is pending
|
||||
/// aborts the process — so a swallowed exception here would come back as a
|
||||
/// crash somewhere unrelated, most likely inside Slint. `exception_describe`
|
||||
/// first, because the trace it prints to logcat is the only place the Java
|
||||
/// class and line survive; `jni::errors::Error::JavaException` on its own says
|
||||
/// neither.
|
||||
fn report(env: &mut JNIEnv, doing: &str, e: &jni::errors::Error) {
|
||||
log::error!("{doing} failed: {e}");
|
||||
if let Ok(true) = env.exception_check() {
|
||||
let _ = env.exception_describe();
|
||||
let _ = env.exception_clear();
|
||||
}
|
||||
}
|
||||
@@ -1,602 +0,0 @@
|
||||
//! DarkRoom Android entry point.
|
||||
//!
|
||||
//! The counterpart to `darkroom-desktop`'s `main`, with two differences that
|
||||
//! come from the platform rather than from choice:
|
||||
//!
|
||||
//! * There are no command-line paths. Android's SAF hands out document URIs,
|
||||
//! not filesystem paths (ARCH §6.9), so the viewer opens with an empty
|
||||
//! browsing list and the library grid is the only way in.
|
||||
//! * Logging goes to logcat *and* to a file. `env_logger` writes to stderr,
|
||||
//! which Android discards; logcat replaces it, and a rotating file beside it
|
||||
//! replaces the thing logcat cannot be — a record that outlives the session
|
||||
//! and can be sent to somebody (NFR-OPS-1, [`dr_plat::diagnostics`]).
|
||||
//!
|
||||
//! The first of those has one exception, and it is the launch `Intent`: a
|
||||
//! gallery, a file manager or the share sheet can name images to open, and
|
||||
//! those arrive as URIs on an `Intent` rather than as words on a command line.
|
||||
//! [`intents`] turns them into paths, and from there they are the same list
|
||||
//! the desktop builds from `argv` (FR-PLAT-AND-6).
|
||||
|
||||
// The whole module is JNI against classes that exist only in the APK, so it
|
||||
// is gated with everything else that cannot compile off-device.
|
||||
#[cfg(target_os = "android")]
|
||||
mod intents;
|
||||
|
||||
// `slint::android` exists only when compiling for Android, so the whole entry
|
||||
// point is gated on the target rather than on a feature. Without this the
|
||||
// crate is still a workspace member on the host, and `cargo test --workspace`
|
||||
// fails to compile it — a build break that only ever appears off-device.
|
||||
#[cfg(target_os = "android")]
|
||||
/// TRACES: M-13 | M-14
|
||||
/// Android application entry point, called by android-activity's glue.
|
||||
#[no_mangle]
|
||||
fn android_main(app: slint::android::AndroidApp) {
|
||||
// **The first statement in the process, and it has to be.** Everything
|
||||
// between here and `install` returning runs with no logger installed at
|
||||
// all: asking the activity for its external directory, `create_dir_all`
|
||||
// and an `open` on a FUSE-backed volume the system may still be mounting.
|
||||
// A failure or a stall in any of it is invisible on every surface there
|
||||
// is — no file yet, and nothing in logcat either — which is precisely the
|
||||
// kind of launch logcat exists to debug.
|
||||
//
|
||||
// `AndroidLogger` rather than `init_once`, so logcat can be *teed* rather
|
||||
// than replaced: `init_once` installs itself as the global logger and
|
||||
// there is only one of those. Everything that reached logcat before the
|
||||
// file existed still reaches it, at the same level and under the same tag;
|
||||
// the file is strictly additional.
|
||||
let console = android_logger::AndroidLogger::new(
|
||||
android_logger::Config::default()
|
||||
.with_max_level(log::LevelFilter::Info)
|
||||
.with_tag("DarkRoom"),
|
||||
);
|
||||
|
||||
// Handed to the logger directly, and **not** written as `log::info!`,
|
||||
// which here would compile and emit nothing: the facade's maximum level is
|
||||
// `Off` until `diagnostics::install` sets it, and the macro tests that
|
||||
// before it reaches any logger at all. This call skips the facade and
|
||||
// reaches `__android_log_write` with nothing in between.
|
||||
//
|
||||
// That independence is the second reason for it. When the log is silent,
|
||||
// this line is what says which half is at fault: present here and absent
|
||||
// below means the `log` wiring, absent in both means liblog is not
|
||||
// delivering this process's records — a question about the device, which
|
||||
// no amount of reading this file can answer.
|
||||
log::Log::log(
|
||||
&console,
|
||||
&log::Record::builder()
|
||||
.level(log::Level::Info)
|
||||
.target(module_path!())
|
||||
.module_path(Some(module_path!()))
|
||||
.args(format_args!(
|
||||
"DarkRoom v{} starting; logcat only until the log file opens",
|
||||
env!("CARGO_PKG_VERSION")
|
||||
))
|
||||
.build(),
|
||||
);
|
||||
|
||||
// Before the file logger, because it needs somewhere to write.
|
||||
//
|
||||
// **The external directory, not the internal one, and the difference is
|
||||
// the entire point of the file.** Both are app-private and both survive
|
||||
// backgrounding — the volatile one is the *cache* directory, which is not
|
||||
// in play here. What separates them is retrieval:
|
||||
// `/data/data/<pkg>/files` needs `run-as` against a debuggable build or
|
||||
// root to read, and `/sdcard/Android/data/<pkg>/files` is a plain
|
||||
// `adb pull` from any build, needing no permission since API 19. A log
|
||||
// nobody can get off the device does not do the job NFR-OPS-1 describes.
|
||||
//
|
||||
// The consequence is that anyone holding the tablet can read it, which is
|
||||
// why `dr_plat::diagnostics` redacts at the sink and why configuration —
|
||||
// the account list, and the credential reference beside it — stays on
|
||||
// `internal_data_path` below rather than moving here (NFR-SEC-2).
|
||||
let external = app.external_data_path();
|
||||
if let Some(dir) = external.clone().or_else(|| app.internal_data_path()) {
|
||||
dr_plat::set_state_dir(dir);
|
||||
}
|
||||
|
||||
let logging = dr_plat::diagnostics::install(Box::new(console), log::LevelFilter::Info);
|
||||
|
||||
// Panics go to stderr, and Android discards stderr. Without this hook a
|
||||
// worker thread that panics is invisible: the process survives, the
|
||||
// channel it was writing to closes, and the UI reports only that
|
||||
// something "failed unexpectedly" with no way to find out what.
|
||||
//
|
||||
// This used to be one `log::error!` of the raw panic, which had two
|
||||
// problems: logcat is a ring buffer that is gone by the time a user
|
||||
// reports anything, and the raw message can carry a document URI naming
|
||||
// their library or a credential a library interpolated into an error
|
||||
// (NFR-SEC-2). `dr_plat::crash` writes a redacted record to disk and logs
|
||||
// the redacted form. Nothing uploads it.
|
||||
//
|
||||
// Before `set_state_dir` on purpose: the hook resolves the directory when
|
||||
// it fires, so installing it first covers the startup below rather than
|
||||
// leaving it uncovered.
|
||||
dr_plat::crash::install(env!("CARGO_PKG_VERSION"));
|
||||
|
||||
log::info!("DarkRoom v{}", env!("CARGO_PKG_VERSION"));
|
||||
|
||||
// Said in logcat as well as in the file, because the first thing anybody
|
||||
// asked for a log needs is where it is — and on a device that is a path
|
||||
// nobody can guess and a command nobody remembers.
|
||||
match &logging {
|
||||
dr_plat::Installed::ToFile(path) => {
|
||||
log::info!("logging to {}", path.display());
|
||||
if external.is_none() {
|
||||
log::warn!(
|
||||
"no external storage; the log is app-private and needs \
|
||||
`adb shell run-as paris.tourolle.darkroom cat files/darkroom.log` \
|
||||
on a debuggable build"
|
||||
);
|
||||
}
|
||||
}
|
||||
dr_plat::Installed::ConsoleOnly(why) => {
|
||||
log::warn!("no log file this session, only logcat: {why}");
|
||||
}
|
||||
}
|
||||
|
||||
// Before anything opens a store: Android has no $HOME and no XDG
|
||||
// directories, so the default guess resolves to a path the app cannot
|
||||
// write. Nothing failed loudly — the session list went to a doomed path, so
|
||||
// the account survived only as long as the process and backgrounding the app
|
||||
// lost the sign-in. `internal_data_path` is the app's private directory
|
||||
// (ARCH §6.9).
|
||||
match app.internal_data_path() {
|
||||
Some(dir) => {
|
||||
log::info!("data dir: {}", dir.display());
|
||||
// Crash records go beside the account data rather than under it:
|
||||
// both are app-private and neither is a cache, which is the whole
|
||||
// distinction that matters here (see `dr_plat::crash::state_dir`).
|
||||
dr_plat::crash::set_state_dir(dir.join("state"));
|
||||
dr_sync::account::set_data_dir(dir);
|
||||
}
|
||||
None => log::error!("no internal data path; settings will not persist"),
|
||||
}
|
||||
|
||||
// After the data dir, because it writes beside the catalog. **Not** before
|
||||
// the first frame any more — it starts a worker and returns; see the
|
||||
// function for what it used to cost the launch.
|
||||
install_bundled_models(app.clone());
|
||||
|
||||
// Before `init_with_event_listener`, which takes `app` by value and is the
|
||||
// last moment anything can ask the activity a question. Not an ordering
|
||||
// preference — after that line there is no `app` left to read the Intent
|
||||
// through.
|
||||
let opened_with = intents::launch_images(&app);
|
||||
|
||||
// TRACES: FR-PLAT-AND-5
|
||||
// The listener is the whole reason this is not the one-line
|
||||
// `slint::android::init(app)`. Slint owns the event loop on Android, so
|
||||
// the platform's lifecycle and memory events reach the application only if
|
||||
// it asks for them here — and it must ask *before* the loop starts, which
|
||||
// is why this sits between the data directory and `dr_ui::run`.
|
||||
//
|
||||
// The listener runs inside `poll_events`, on the same thread the event
|
||||
// loop and every interface cache live on, which is what lets
|
||||
// `dr_ui::memory` be a thread-local registry of plain `Fn()` rather than a
|
||||
// cross-thread channel (see its module documentation).
|
||||
//
|
||||
// # Why two events and not eight
|
||||
//
|
||||
// FR-PLAT-AND-5 names `onTrimMemory`, whose `TRIM_MEMORY_*` levels grade
|
||||
// how badly the system wants the memory back. Those levels do not exist
|
||||
// here: `ComponentCallbacks2` is a Java interface implemented by an
|
||||
// `Activity` or `Application`, and this app has neither — it is a bare
|
||||
// `NativeActivity`, whose native callback table offers only the ungraded
|
||||
// `onLowMemory`. android-activity surfaces exactly that as `LowMemory`.
|
||||
// Reading the grades would mean shipping a Java subclass to forward them,
|
||||
// which is a distribution-manifest change and not this one.
|
||||
//
|
||||
// `Stop` recovers the one grade that matters most anyway, and for free.
|
||||
// It is the moment the activity stops being visible — `TRIM_MEMORY_UI_HIDDEN`
|
||||
// in all but name — and it is the cheapest possible time to give memory
|
||||
// back, because nothing that is freed has to be drawn again before anyone
|
||||
// sees it. `Pause` deliberately does not qualify: a permission dialog or
|
||||
// the share sheet pauses an activity that is still on screen behind it,
|
||||
// and throwing away its render pipeline would make every such interruption
|
||||
// cost a full re-render.
|
||||
use slint::android::android_activity::{MainEvent, PollEvent};
|
||||
if let Err(e) = slint::android::init_with_event_listener(app, |event| match event {
|
||||
PollEvent::Main(MainEvent::LowMemory) => {
|
||||
dr_ui::memory::relieve(dr_ui::memory::Level::Critical);
|
||||
}
|
||||
PollEvent::Main(MainEvent::Stop) => {
|
||||
dr_ui::memory::relieve(dr_ui::memory::Level::UiHidden);
|
||||
}
|
||||
_ => {}
|
||||
}) {
|
||||
log::error!("Slint Android backend failed to initialise: {e}");
|
||||
return;
|
||||
}
|
||||
|
||||
// The launch Intent's images, where there were any, standing in for the
|
||||
// desktop's argv — `launch::startup_action` treats a non-empty list as
|
||||
// "the user asked for these specifically", which is exactly what a share
|
||||
// or a tap in a gallery is. Empty for an ordinary launch, and the library
|
||||
// opens as before.
|
||||
//
|
||||
// Returning from `android_main` ends the process, so a failure here is
|
||||
// logged rather than propagated — there is no shell to show `Err` to.
|
||||
if let Err(e) = dr_ui::run(opened_with) {
|
||||
log::error!("DarkRoom exited with error: {e:#}");
|
||||
}
|
||||
}
|
||||
|
||||
/// Unpack the models the APK carries, if it carries any.
|
||||
///
|
||||
/// # Why Android needs this and no other platform does
|
||||
///
|
||||
/// A desktop build reads its models from a path — the account's directory, the
|
||||
/// shared one, or `$XDG_DATA_DIRS` where a package put them. **Android has no
|
||||
/// such path.** `internal_data_path` is app-private, `run-as` needs a
|
||||
/// debuggable build, and an asset inside a package is not a path anything can
|
||||
/// open (ARCH §6.9), so a phone had no way to reach a model at all.
|
||||
///
|
||||
/// So the APK carries them in `assets/models/` and this copies them out, once,
|
||||
/// into the same shared directory a desktop install uses. After that every
|
||||
/// lookup in `dr_ui::library` finds them exactly where it finds a desktop
|
||||
/// user's.
|
||||
///
|
||||
/// # The two sets are not the same kind of thing
|
||||
///
|
||||
/// **Face weights are absent from the repository by design.** The InsightFace
|
||||
/// grant is research-only and incompatible with this project's licence
|
||||
/// (docs/faces.md §2), so a desktop user fetches them, runs
|
||||
/// `tools/fix-face-model-shapes.sh` over them, and drops the result in. A build
|
||||
/// that carries none is the ordinary case and face indexing simply stays off.
|
||||
///
|
||||
/// **The scene model is committed** (AGPL, compatible — `models/LICENCE.md`),
|
||||
/// so a build carrying none means a checkout without `git lfs pull` rather than
|
||||
/// a deliberate omission. It is still not an error here: the scene tab reports
|
||||
/// itself unavailable the same way face indexing does, because a photo editor
|
||||
/// that refuses to start over a missing grading feature is worse than one that
|
||||
/// starts without it.
|
||||
///
|
||||
/// # Why it is not `include_bytes!` like the instance model
|
||||
///
|
||||
/// Size. The instance model is 11 MB and compiled in; the scene model is 24 MB
|
||||
/// on top of that, and a 35 MB constant in the binary is paid by every install
|
||||
/// whether or not the tab is opened. Assets are also *stored* rather than
|
||||
/// deflated in the APK (see `assemble-apk.sh`), so unpacking is a copy rather
|
||||
/// than an inflate.
|
||||
///
|
||||
/// # Why it returns before it has done anything
|
||||
///
|
||||
/// **`android_main` runs with the input channel unserviced.** Nothing drains
|
||||
/// it until Slint reaches `poll_events`, and Slint does not reach `poll_events`
|
||||
/// until `dr_ui::run` calls `window.run()`, which is the last line of it. So
|
||||
/// every millisecond spent between the top of `android_main` and that line is a
|
||||
/// millisecond in which Android's input dispatcher gets no answer, and five
|
||||
/// thousand of them is an ANR by definition — the system puts "DarkRoom isn't
|
||||
/// responding" over a window that has never painted, and offers to kill it.
|
||||
///
|
||||
/// This copied **41 MB** on the first launch after an install: 24.9 MB of scene
|
||||
/// model, 13.6 MB of embedder, 2.5 MB of detector, each read whole out of the
|
||||
/// APK and written to `/data`. v0.10.0 added the scene model, which was 60% of
|
||||
/// that total; v0.10.0 is the release the ANR appeared in, and the 8,010 minor
|
||||
/// faults in its report are what 41 MB of freshly touched pages looks like.
|
||||
/// The two further detectors the settings page offers since have made it
|
||||
/// 61 MB, which is the same argument with a larger number.
|
||||
///
|
||||
/// So it runs on a worker (NFR-ARCH-1: nothing blocking on the UI executor) and
|
||||
/// this function returns as soon as the thread is running. Nothing on the
|
||||
/// launch path waits for it, and no other startup step needs its result.
|
||||
///
|
||||
/// # The window in which a model looks absent, and why that is honest enough
|
||||
///
|
||||
/// Until the copy finishes, `library::face_models` and `library::scene_model`
|
||||
/// answer `is_file()` about files that are not written yet, so both report
|
||||
/// their feature unavailable — the same answer they give a build carrying no
|
||||
/// weights at all, which is the ordinary case this whole path was written
|
||||
/// around. It is briefly pessimistic rather than wrong, it lasts about as long
|
||||
/// as it takes to read one screenful of the grid, and the temporary name
|
||||
/// [`unpack_bundled_models`] writes under is what stops it being worse than
|
||||
/// pessimistic: a lookup never sees a half-written file, only an absent one.
|
||||
#[cfg(target_os = "android")]
|
||||
fn install_bundled_models(app: slint::android::AndroidApp) {
|
||||
// Detached rather than joined: there is no later moment on the launch path
|
||||
// that wants the answer, and a handle nobody joins is a handle nobody can
|
||||
// forget to. `AndroidApp` is documented `Send` and `Sync` and is an `Arc`
|
||||
// internally, so the clone costs a refcount; `asset_manager` is asked for
|
||||
// on the worker because `AAssetManager` is thread-safe by contract and
|
||||
// reading the pointer takes only the app's read lock, which `poll_events`
|
||||
// also only ever holds shared.
|
||||
std::thread::spawn(move || unpack_bundled_models(&app));
|
||||
}
|
||||
|
||||
/// The copy itself, on the worker [`install_bundled_models`] starts.
|
||||
#[cfg(target_os = "android")]
|
||||
fn unpack_bundled_models(app: &slint::android::AndroidApp) {
|
||||
use std::io::Read;
|
||||
|
||||
let started = std::time::Instant::now();
|
||||
|
||||
// The face names are the **shape-fixed** exports, matching what
|
||||
// `library::face_models` looks for: tract cannot parse either InsightFace
|
||||
// graph with its dynamic input dimension, so what ships here has already
|
||||
// been through `tools/fix-face-model-shapes.sh`.
|
||||
//
|
||||
// The scene entries are three files rather than one because the graph alone
|
||||
// decodes to 150 anonymous channels — `library::scene_model` wants the
|
||||
// vocabulary and the category descriptor beside it, and requires all three
|
||||
// before it reports the tab available.
|
||||
//
|
||||
// Three detectors, because which one runs is a setting
|
||||
// (`FaceDetector`, docs/faces.md §12.3) and a tablet has no other way to
|
||||
// obtain the one it was not shipped with. Twenty megabytes of APK for
|
||||
// the choice; the embedder is the same for all three.
|
||||
//
|
||||
// Then the three eye-state models (docs/faces.md §17): landmarks, open
|
||||
// or closed, sunglasses. The app indexes without them; with them the
|
||||
// eyes-open filter has something to read, and a tablet has no other way
|
||||
// to get them either.
|
||||
//
|
||||
// The int8 forms beside the three detectors are what the Hexagon runs
|
||||
// (docs/inference.md §5); the engine loads the sibling when the probe
|
||||
// chose that rung and ignores it otherwise.
|
||||
const BUNDLED: [(&std::ffi::CStr, &str); 14] = [
|
||||
(c"models/scrfd_500m_640.onnx", "scrfd_500m_640.onnx"),
|
||||
(
|
||||
c"models/scrfd_500m_640.int8.onnx",
|
||||
"scrfd_500m_640.int8.onnx",
|
||||
),
|
||||
(c"models/scrfd_2.5g_640.onnx", "scrfd_2.5g_640.onnx"),
|
||||
(
|
||||
c"models/scrfd_2.5g_640.int8.onnx",
|
||||
"scrfd_2.5g_640.int8.onnx",
|
||||
),
|
||||
(c"models/scrfd_10g_640.onnx", "scrfd_10g_640.onnx"),
|
||||
(c"models/scrfd_10g_640.int8.onnx", "scrfd_10g_640.int8.onnx"),
|
||||
(c"models/arcface_mbf_b1.onnx", "arcface_mbf_b1.onnx"),
|
||||
(c"models/2d106det_b1.onnx", "2d106det_b1.onnx"),
|
||||
(c"models/ocec_s_b1.onnx", "ocec_s_b1.onnx"),
|
||||
(c"models/sgc_l_48_b1.onnx", "sgc_l_48_b1.onnx"),
|
||||
(c"models/yolo26s-sem-ade20k.onnx", "yolo26s-sem-ade20k.onnx"),
|
||||
(
|
||||
c"models/yolo26s-sem-ade20k.classes.json",
|
||||
"yolo26s-sem-ade20k.classes.json",
|
||||
),
|
||||
(c"models/categories.txt", "categories.txt"),
|
||||
// The panorama border filler (FR-MRG-4); MIT, 28 MB.
|
||||
(c"models/migan-512.onnx", "migan-512.onnx"),
|
||||
];
|
||||
|
||||
let dir = dr_ui::shared_face_models_dir();
|
||||
let assets = app.asset_manager();
|
||||
let mut copied = 0u64;
|
||||
|
||||
for (asset_path, name) in BUNDLED {
|
||||
let dest = dir.join(name);
|
||||
// Already unpacked. Not re-read on every launch: this is 73 MB of
|
||||
// copying across the ten entries, and the file does not change without
|
||||
// the APK changing, at which point the install wiped it anyway. It
|
||||
// matters more now than it did — a launch that skips every entry here
|
||||
// costs nothing at all, which is what makes the second launch after an
|
||||
// install cheap even though the first one is not.
|
||||
if dest.is_file() {
|
||||
continue;
|
||||
}
|
||||
let Some(mut asset) = assets.open(asset_path) else {
|
||||
log::info!("no bundled {name} in this APK; the feature needing it stays off");
|
||||
continue;
|
||||
};
|
||||
let mut bytes = Vec::new();
|
||||
if let Err(e) = asset.read_to_end(&mut bytes) {
|
||||
log::error!("bundled {name} could not be read: {e}");
|
||||
continue;
|
||||
}
|
||||
if let Err(e) = std::fs::create_dir_all(&dir) {
|
||||
log::error!("cannot create {}: {e}", dir.display());
|
||||
return;
|
||||
}
|
||||
// Written under a temporary name and renamed, because
|
||||
// `library::face_models` and `library::scene_model` both decide a
|
||||
// feature is available on `is_file()` alone. A truncated write — the process backgrounded and
|
||||
// killed mid-copy — would otherwise leave a file that passes that test
|
||||
// and fails inside tract, reported to the user as a broken model rather
|
||||
// than a missing one.
|
||||
let part = dir.join(format!("{name}.part"));
|
||||
match std::fs::write(&part, &bytes).and_then(|()| std::fs::rename(&part, &dest)) {
|
||||
Ok(()) => {
|
||||
copied += bytes.len() as u64;
|
||||
log::info!("installed bundled {name} ({} bytes)", bytes.len());
|
||||
}
|
||||
Err(e) => {
|
||||
log::error!("cannot install {name}: {e}");
|
||||
let _ = std::fs::remove_file(&part);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// The figure this whole function is about. Said even when it is zero, so a
|
||||
// launch that ANRs anyway can be told apart from one that spent its six
|
||||
// seconds here — on a second launch there is nothing left to copy and the
|
||||
// line reads `0 bytes`.
|
||||
log::info!(
|
||||
"bundled models ready: {copied} bytes copied in {} ms",
|
||||
started.elapsed().as_millis()
|
||||
);
|
||||
|
||||
// Now, and not at launch: the probe fingerprints the model files, and
|
||||
// on a first launch they were not on disk until this line. The runtime
|
||||
// is in the APK's native library directory beside `libdarkroom.so`,
|
||||
// which is also where Qualcomm's DSP loader has to be pointed for the
|
||||
// Hexagon skel (docs/inference.md §3, §8).
|
||||
dr_ui::inference::init(native_library_dir().into_iter().collect());
|
||||
}
|
||||
|
||||
/// The directory the system unpacked this APK's native libraries into.
|
||||
///
|
||||
/// Read from where the loader put *this* library rather than asked of the
|
||||
/// activity: `android-activity` does not expose `nativeLibraryDir`, and the
|
||||
/// answer is in `/proc/self/maps` for free.
|
||||
#[cfg(target_os = "android")]
|
||||
fn native_library_dir() -> Option<std::path::PathBuf> {
|
||||
let maps = std::fs::read_to_string("/proc/self/maps").ok()?;
|
||||
maps.lines()
|
||||
.filter_map(|l| l.split_whitespace().nth(5))
|
||||
.find(|p| p.ends_with("/libdarkroom.so"))
|
||||
.and_then(|p| std::path::Path::new(p).parent().map(Into::into))
|
||||
}
|
||||
|
||||
/// TRACES: FR-PLAT-AND-6
|
||||
/// The declarations that make this app a receiver, held to on the host.
|
||||
///
|
||||
/// Everything FR-PLAT-AND-6 does on a device is unreachable from `cargo test`:
|
||||
/// there is no `Intent` off-device and no `ContentProvider` to instantiate. But
|
||||
/// the requirement is not only behaviour — half of it is *declaration*, and a
|
||||
/// declaration can be wrong in ways that compile perfectly and fail silently.
|
||||
/// An intent filter that is deleted takes the app out of every gallery's "open
|
||||
/// with" menu with nothing to notice; an authority that stops matching the
|
||||
/// class it names raises a `SecurityException` in whichever other app opened
|
||||
/// the share sheet, which is the last place anybody would look for it.
|
||||
///
|
||||
/// The manifest is read by aapt2 and the Java by javac, so a Rust build sees
|
||||
/// neither. `include_str!` is what puts them where a test can reach them, and
|
||||
/// this is the only place in the workspace that does.
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
/// The manifest with its comments removed and its whitespace flattened, so
|
||||
/// a match is about the declaration and not about how it is indented.
|
||||
fn manifest() -> String {
|
||||
let xml = include_str!("../android/AndroidManifest.xml");
|
||||
let mut out = String::with_capacity(xml.len());
|
||||
let mut rest = xml;
|
||||
// Comments first, and not by regex over the whole file: several of them
|
||||
// quote the very attribute names the assertions below look for, so a
|
||||
// test that read them would pass on the strength of the prose
|
||||
// explaining an entry that had been deleted.
|
||||
while let Some(start) = rest.find("<!--") {
|
||||
out.push_str(&rest[..start]);
|
||||
match rest[start..].find("-->") {
|
||||
Some(end) => rest = &rest[start + end + 3..],
|
||||
None => {
|
||||
rest = "";
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
out.push_str(rest);
|
||||
out.split_whitespace().collect::<Vec<_>>().join(" ")
|
||||
}
|
||||
|
||||
/// The body of each `<intent-filter>`, so an action and a MIME type are
|
||||
/// checked to be in the *same* filter. Two filters, one naming the action
|
||||
/// and one naming the type, register for neither.
|
||||
fn intent_filters(manifest: &str) -> Vec<&str> {
|
||||
manifest
|
||||
.split("<intent-filter>")
|
||||
.skip(1)
|
||||
.filter_map(|filter| filter.split("</intent-filter>").next())
|
||||
.collect()
|
||||
}
|
||||
|
||||
/// The single `<provider>` element, attributes and all.
|
||||
fn provider(manifest: &str) -> String {
|
||||
let start = manifest
|
||||
.find("<provider")
|
||||
.expect("no <provider> in the manifest");
|
||||
let rest = &manifest[start..];
|
||||
let end = rest.find("/>").expect("unterminated <provider> element");
|
||||
rest[..end + 2].to_string()
|
||||
}
|
||||
|
||||
fn attribute(element: &str, name: &str) -> Option<String> {
|
||||
let key = format!("{name}=\"");
|
||||
let start = element.find(&key)? + key.len();
|
||||
let value = element[start..].split('"').next()?;
|
||||
Some(value.to_string())
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_gallery_can_open_a_photograph_in_this_app() {
|
||||
let manifest = manifest();
|
||||
let registered = intent_filters(&manifest).iter().any(|filter| {
|
||||
filter.contains("android.intent.action.VIEW")
|
||||
&& filter.contains("android.intent.category.DEFAULT")
|
||||
&& filter.contains(r#"android:mimeType="image/*""#)
|
||||
});
|
||||
assert!(
|
||||
registered,
|
||||
"no VIEW filter for image/*: nothing will offer DarkRoom for a photograph"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn the_share_sheet_can_send_one_image_or_several() {
|
||||
let manifest = manifest();
|
||||
let registered = intent_filters(&manifest).iter().any(|filter| {
|
||||
// The closing quote matters: SEND is a prefix of SEND_MULTIPLE, so
|
||||
// a bare substring test passes on a filter that declares only the
|
||||
// second and would not be offered for a single photograph.
|
||||
filter.contains(r#"android.intent.action.SEND""#)
|
||||
&& filter.contains(r#"android.intent.action.SEND_MULTIPLE""#)
|
||||
&& filter.contains("android.intent.category.DEFAULT")
|
||||
&& filter.contains(r#"android:mimeType="image/*""#)
|
||||
});
|
||||
assert!(
|
||||
registered,
|
||||
"no SEND/SEND_MULTIPLE filter for image/*, so the share sheet will not list DarkRoom"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn one_activity_ever_so_a_second_launch_cannot_start_a_second_one() {
|
||||
// Not style. Another app can now launch this activity while it is
|
||||
// already running, and the default launch mode answers that by
|
||||
// creating a second NativeActivity in this process — a second
|
||||
// android_main, a second Slint backend, a second wgpu device.
|
||||
assert!(
|
||||
manifest().contains(r#"android:launchMode="singleTask""#),
|
||||
"the activity must be singleTask; see the manifest comment"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn the_provider_authority_is_the_one_the_class_answers_to() {
|
||||
let manifest = manifest();
|
||||
let element = provider(&manifest);
|
||||
|
||||
let declared =
|
||||
attribute(&element, "android:authorities").expect("the provider declares no authority");
|
||||
let java = include_str!("../android/java/paris/tourolle/darkroom/ExportProvider.java");
|
||||
let constant = java
|
||||
.split("AUTHORITY = \"")
|
||||
.nth(1)
|
||||
.and_then(|rest| rest.split('"').next())
|
||||
.expect("ExportProvider declares no AUTHORITY constant");
|
||||
|
||||
assert_eq!(
|
||||
declared, constant,
|
||||
"the manifest and ExportProvider disagree about the authority; \
|
||||
a share would fail as a SecurityException inside the receiving app"
|
||||
);
|
||||
|
||||
let class = attribute(&element, "android:name").expect("the provider declares no class");
|
||||
let (package, _) = class
|
||||
.rsplit_once('.')
|
||||
.expect("the provider class is unqualified");
|
||||
assert!(
|
||||
java.contains(&format!("package {package};")),
|
||||
"the manifest names {class}, which is not the class in ExportProvider.java"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn the_provider_hands_out_one_file_at_a_time_and_nothing_by_itself() {
|
||||
let manifest = manifest();
|
||||
let element = provider(&manifest);
|
||||
// The two halves are not redundant. Without the grant, every share
|
||||
// target fails; exported, every app on the device could read this
|
||||
// app's private directory.
|
||||
assert_eq!(
|
||||
attribute(&element, "android:exported").as_deref(),
|
||||
Some("false"),
|
||||
"an exported provider would serve the app's private directory to anything installed"
|
||||
);
|
||||
assert_eq!(
|
||||
attribute(&element, "android:grantUriPermissions").as_deref(),
|
||||
Some("true"),
|
||||
"without URI grants the share sheet opens and every target fails to read the file"
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -1,27 +0,0 @@
|
||||
[package]
|
||||
name = "darkroom-desktop"
|
||||
version.workspace = true
|
||||
edition.workspace = true
|
||||
rust-version.workspace = true
|
||||
license.workspace = true
|
||||
|
||||
[dependencies]
|
||||
dr-ui = { workspace = true, features = ["scene-model"] }
|
||||
# For the panic hook and the log sink, directly rather than through dr-ui:
|
||||
# both have to be installed before `dr_ui::run`, because a panic during startup
|
||||
# is exactly the one they exist to catch and record (NFR-OPS-1, NFR-OPS-2).
|
||||
dr-plat.workspace = true
|
||||
anyhow.workspace = true
|
||||
env_logger.workspace = true
|
||||
log.workspace = true
|
||||
|
||||
# The Windows resource block — icon and version — compiled in by build.rs.
|
||||
# Unconditional rather than under `[target.'cfg(windows)']`, because a cfg on
|
||||
# a build-dependency is evaluated against the *host* — the machine running
|
||||
# the build script — and this is built for Windows from Linux. The script
|
||||
# itself returns before touching the crate on every other target.
|
||||
[build-dependencies]
|
||||
winresource = "0.1"
|
||||
|
||||
[features]
|
||||
default = []
|
||||
@@ -1,58 +0,0 @@
|
||||
//! TRACES: FR-PLAT-WIN-2
|
||||
//! The Windows resource block: icon and version, compiled into the executable.
|
||||
//!
|
||||
//! Windows takes an application's icon and its "Details" tab from a resource
|
||||
//! inside the `.exe`, not from a `.desktop` file, so without this the installed
|
||||
//! program shows the generic executable icon in Explorer, the Start Menu and
|
||||
//! the taskbar, and reports no version. Nothing here runs for any other
|
||||
//! target: the whole body is behind the target-OS check, and the crate that
|
||||
//! does the work is a build-dependency only.
|
||||
//!
|
||||
//! The icon is the same PNG every other platform uses, wrapped into an `.ico`
|
||||
//! in `OUT_DIR` rather than committed: an ICO entry may *be* a PNG (Vista and
|
||||
//! later read them directly), so the wrapper is a 22-byte header and the
|
||||
//! file's bytes, and a generated binary stays out of the tree.
|
||||
|
||||
use std::io::Write as _;
|
||||
use std::path::PathBuf;
|
||||
|
||||
fn main() {
|
||||
println!("cargo:rerun-if-changed=build.rs");
|
||||
if std::env::var("CARGO_CFG_TARGET_OS").as_deref() != Ok("windows") {
|
||||
return;
|
||||
}
|
||||
|
||||
let png = PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("../../ui/dr-ui/ui/app-icon.png");
|
||||
println!("cargo:rerun-if-changed={}", png.display());
|
||||
let bytes = std::fs::read(&png).expect("read app-icon.png");
|
||||
let ico = PathBuf::from(std::env::var("OUT_DIR").unwrap()).join("darkroom.ico");
|
||||
write_png_ico(&ico, &bytes, 256).expect("write darkroom.ico");
|
||||
|
||||
let mut res = winresource::WindowsResource::new();
|
||||
res.set_icon(ico.to_str().unwrap());
|
||||
res.set("ProductName", "DarkRoom");
|
||||
res.set("FileDescription", "DarkRoom");
|
||||
res.set("LegalCopyright", "GPL-3.0-or-later");
|
||||
// Cross-compiling: `winresource` looks for a `windres` for the target and
|
||||
// the Windows image names it explicitly, for the same reason the Android
|
||||
// image names its linkers.
|
||||
if let Ok(windres) = std::env::var("WINDRES") {
|
||||
res.set_windres_path(&windres);
|
||||
}
|
||||
res.compile().expect("compile the Windows resource block");
|
||||
}
|
||||
|
||||
/// One PNG image as an `.ico`. `edge` is the PNG's width and height; 256 is
|
||||
/// written as 0 per the format.
|
||||
fn write_png_ico(path: &std::path::Path, png: &[u8], edge: u32) -> std::io::Result<()> {
|
||||
let mut f = std::fs::File::create(path)?;
|
||||
let dim = if edge >= 256 { 0u8 } else { edge as u8 };
|
||||
// ICONDIR: reserved, type 1 (icon), one image.
|
||||
f.write_all(&[0, 0, 1, 0, 1, 0])?;
|
||||
// ICONDIRENTRY: width, height, palette 0, reserved, planes 1, bpp 32,
|
||||
// byte length, offset (6 + 16).
|
||||
f.write_all(&[dim, dim, 0, 0, 1, 0, 32, 0])?;
|
||||
f.write_all(&(png.len() as u32).to_le_bytes())?;
|
||||
f.write_all(&22u32.to_le_bytes())?;
|
||||
f.write_all(png)
|
||||
}
|
||||
@@ -1,113 +0,0 @@
|
||||
//! DarkRoom desktop entry point.
|
||||
//!
|
||||
//! darkroom-desktop <file-or-directory>...
|
||||
//! darkroom-desktop --version
|
||||
|
||||
// TRACES: FR-PLAT-WIN-2
|
||||
// A GUI-subsystem executable, or Windows opens a console window behind the
|
||||
// application for the life of the process. Release only: the console is where
|
||||
// the log goes when there is no file, and a debug build is run from one.
|
||||
// `--version` still prints under this — stdout is simply not attached when
|
||||
// launched from Explorer, which is not where anyone asks for a version.
|
||||
#![cfg_attr(all(windows, not(debug_assertions)), windows_subsystem = "windows")]
|
||||
|
||||
use std::path::PathBuf;
|
||||
|
||||
use dr_plat::diagnostics::Installed;
|
||||
|
||||
fn main() -> anyhow::Result<()> {
|
||||
// TRACES: FR-PLAT-WIN-3
|
||||
// Before the logger, the crash hook and everything else: this exists so a
|
||||
// build made on a machine that cannot run the application — the Linux CI
|
||||
// producing the Windows binary, checked under Wine — has an exit that
|
||||
// proves the executable starts without opening a window or touching the
|
||||
// user's directories (docs/windows.md §6).
|
||||
if std::env::args().nth(1).as_deref() == Some("--version") {
|
||||
println!("darkroom-desktop {}", env!("CARGO_PKG_VERSION"));
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
// Built rather than `init`ed, so the same logger can be handed to the
|
||||
// diagnostics tee: `env_logger` keeps writing to stderr exactly as before,
|
||||
// and every record it accepts is also appended to the on-disk log
|
||||
// (NFR-OPS-1). `filter()` is asked afterwards because the environment may
|
||||
// have overridden the default below, and the file must not be quieter than
|
||||
// the terminal.
|
||||
let console = env_logger::Builder::from_env(env_logger::Env::default().default_filter_or(
|
||||
"info,wgpu_core=warn,wgpu_hal=warn,zbus=warn,tracing=warn,calloop=warn,rawler=warn",
|
||||
))
|
||||
.build();
|
||||
let level = console.filter();
|
||||
let logging = dr_plat::diagnostics::install(Box::new(console), level);
|
||||
|
||||
// Immediately after the logger and before anything that could fail. Until
|
||||
// now a panic on desktop went to stderr and died with the terminal, which
|
||||
// means every panic a user has ever hit was unreportable: the process
|
||||
// survives (the panicking worker does not), a control goes dead, and there
|
||||
// is nothing on disk to say why. The record is local and stays local —
|
||||
// there is no upload path, by design; see `dr_plat::crash`.
|
||||
dr_plat::crash::install(env!("CARGO_PKG_VERSION"));
|
||||
|
||||
log::info!("DarkRoom v{}", env!("CARGO_PKG_VERSION"));
|
||||
// First thing in the file, so a user asked for "the log" can find it
|
||||
// without being told a path over the phone.
|
||||
match &logging {
|
||||
Installed::ToFile(path) => log::info!("logging to {}", path.display()),
|
||||
Installed::ConsoleOnly(why) => log::warn!("no log file this session: {why}"),
|
||||
}
|
||||
|
||||
let paths: Vec<PathBuf> = std::env::args().skip(1).map(PathBuf::from).collect();
|
||||
if paths.is_empty() {
|
||||
eprintln!("usage: darkroom-desktop <file-or-directory>...");
|
||||
}
|
||||
|
||||
// Before the window: the probe runs on its own thread and the first
|
||||
// frame does not wait for it, but the models a background job asks for
|
||||
// should already know where the runtime is (docs/inference.md §4).
|
||||
dr_ui::inference::init(runtime_dirs());
|
||||
|
||||
dr_ui::run(paths)?;
|
||||
|
||||
// Skip Rust's normal static/thread-local teardown on the way out: a
|
||||
// background zbus/keyring connection opened by dr_ui::launch_ui can
|
||||
// still be alive here, and unwinding through it races its async-io
|
||||
// reactor thread, panicking with "thread local ... during or after
|
||||
// destruction" when the window is closed.
|
||||
std::process::exit(0);
|
||||
}
|
||||
|
||||
/// Where a desktop package may have put `libonnxruntime`, most specific
|
||||
/// first. None of these existing is the tract build, which is a complete
|
||||
/// application and not an error (docs/inference.md §3).
|
||||
///
|
||||
/// `DARKROOM_ORT_DIR` is for a developer pointing at a runtime that is not
|
||||
/// installed — the wheel's `capi` directory, say. Then beside the executable
|
||||
/// and in the package's private library directory, for a package that
|
||||
/// bundles its own; then the user's own `runtime/` beside the models, where
|
||||
/// `tools/fetch-desktop-runtime.sh` puts one; then the Flatpak prefix; then
|
||||
/// the system library directory, for a distribution that ships ONNX Runtime
|
||||
/// as a package of its own. The user's copy outranks the system's because
|
||||
/// the system's is the one most likely to be built without the GPU
|
||||
/// providers, or against the wrong cuDNN — and a system copy whose providers
|
||||
/// do not load is not a problem, only a slower app: the probe builds a real
|
||||
/// session before believing a provider.
|
||||
fn runtime_dirs() -> Vec<PathBuf> {
|
||||
let mut dirs = Vec::new();
|
||||
if let Some(dir) = std::env::var_os("DARKROOM_ORT_DIR") {
|
||||
dirs.push(PathBuf::from(dir));
|
||||
}
|
||||
if let Ok(exe) = std::env::current_exe() {
|
||||
if let Some(bin) = exe.parent() {
|
||||
dirs.push(bin.to_path_buf());
|
||||
dirs.push(bin.join("../lib/darkroom"));
|
||||
}
|
||||
}
|
||||
dirs.push(dr_ui::inference::user_runtime_dir());
|
||||
#[cfg(target_os = "linux")]
|
||||
dirs.extend([
|
||||
PathBuf::from("/app/lib/darkroom"),
|
||||
PathBuf::from("/usr/lib/darkroom"),
|
||||
PathBuf::from("/usr/lib"),
|
||||
]);
|
||||
dirs
|
||||
}
|
||||
@@ -1,34 +0,0 @@
|
||||
[package]
|
||||
name = "dr-catalog"
|
||||
version.workspace = true
|
||||
edition.workspace = true
|
||||
rust-version.workspace = true
|
||||
license.workspace = true
|
||||
|
||||
[dependencies]
|
||||
dr-types.workspace = true
|
||||
# The face subsystem's arithmetic — `Calibration` in particular, so the sigmoid
|
||||
# that turns a cosine into a probability has exactly one definition. Default
|
||||
# features are off, so this brings in no ONNX runtime and no weights: only the
|
||||
# model-free half compiles here.
|
||||
dr-face.workspace = true
|
||||
# For `SHARD_MAX_BYTES` alone. The face shards are capped at the same 25 MB the
|
||||
# thumbnail shards are, and sharing the constant is what keeps them from
|
||||
# drifting apart — the cap is a statement about sync cost, not about thumbnails.
|
||||
dr-thumbs.workspace = true
|
||||
# The `Storage` trait, and nothing else from it. A scan has to read a real
|
||||
# directory, and this is how `core/` reaches the platform without a
|
||||
# `#[cfg(target_os)]` of its own (ARCH §4.1: calls go downward).
|
||||
dr-plat.workspace = true
|
||||
rusqlite.workspace = true
|
||||
thiserror.workspace = true
|
||||
log.workspace = true
|
||||
# `collections.selector_json` — the stored form of a smart collection's
|
||||
# selector. The column predates this dependency; nothing else here is JSON.
|
||||
serde_json.workspace = true
|
||||
|
||||
# For the `scan_local` example only, which is a diagnostic tool: what it is
|
||||
# diagnosing is often a folder the scan warned about and skipped, and those
|
||||
# warnings go to `log`.
|
||||
[dev-dependencies]
|
||||
env_logger.workspace = true
|
||||
@@ -1,420 +0,0 @@
|
||||
//! What the suggestion confidence would say about a real library.
|
||||
//!
|
||||
//! cargo run --release -p dr-catalog --example face_confidence -- CATALOG.sqlite [--full]
|
||||
//!
|
||||
//! Read-only: it writes nothing to the catalog, so it can be pointed at a copy
|
||||
//! of a live library and re-run at will.
|
||||
//!
|
||||
//! # What it measures
|
||||
//!
|
||||
//! The user's own confirmations are the only ground truth a library has, so
|
||||
//! the evaluation is leave-one-out over them: hide one confirmed face, ask the
|
||||
//! scorer which of the confirmed identities it belongs to, and compare with
|
||||
//! what the user said. Faces from the same photograph are excluded exactly as
|
||||
//! the clusterer excludes them, so nothing is scored against a co-occurrence
|
||||
//! that would never have been allowed to merge.
|
||||
//!
|
||||
//! Two numbers are compared on that task: the share (`dr_face::assign`) and the
|
||||
//! mean-within-group figure it replaced. Accuracy says which one picks the
|
||||
//! right person; the reliability table says whether the percentage the user is
|
||||
//! shown means what it claims — which is the question FR-CULL-9 exists for.
|
||||
|
||||
use std::collections::HashMap;
|
||||
|
||||
use dr_catalog::faces::{self, PersonId};
|
||||
use dr_catalog::Catalog;
|
||||
|
||||
const MODEL_ID: &str = "w600k_mbf";
|
||||
const TOP: usize = 10;
|
||||
|
||||
struct Known {
|
||||
image: u64,
|
||||
person: PersonId,
|
||||
embedding: Vec<f32>,
|
||||
crop_px: f32,
|
||||
}
|
||||
|
||||
/// What the catalog holds per face, decoded: photograph, vector, size,
|
||||
/// quality.
|
||||
type Decoded = (u64, Vec<f32>, f32, Option<f32>);
|
||||
|
||||
fn main() {
|
||||
let args: Vec<String> = std::env::args().skip(1).collect();
|
||||
let Some(path) = args.first() else {
|
||||
eprintln!("usage: face_confidence CATALOG.sqlite [--full]");
|
||||
std::process::exit(2);
|
||||
};
|
||||
|
||||
let catalog = Catalog::open(std::path::Path::new(path)).expect("open catalog");
|
||||
let conn = catalog.connection();
|
||||
|
||||
let cal = match faces::calibration(conn, MODEL_ID) {
|
||||
Ok(Some((c, _))) => c,
|
||||
_ => dr_face::Calibration::default(),
|
||||
};
|
||||
println!(
|
||||
"calibration: a={:.2} b={:.2} w_size={:.3} valid={} (P=0.5 at cosine {:.3})",
|
||||
cal.a,
|
||||
cal.b,
|
||||
cal.w_size,
|
||||
cal.valid,
|
||||
cal.boundary_at(0.5, 150.0, 0.0)
|
||||
);
|
||||
|
||||
let model = dr_face::ModelId::new(MODEL_ID.to_string());
|
||||
let stored = faces::embeddings(conn, MODEL_ID).expect("embeddings");
|
||||
let mut embedding_of: HashMap<faces::FaceId, Decoded> = HashMap::new();
|
||||
for f in stored {
|
||||
if let Some(e) = dr_face::Embedding::from_f16_bytes(model.clone(), &f.embedding) {
|
||||
embedding_of.insert(f.face, (f.image.0, e.v.to_vec(), f.crop_px, f.quality));
|
||||
}
|
||||
}
|
||||
println!("faces with embeddings: {}", embedding_of.len());
|
||||
|
||||
// The ground truth: every confirmed face, under the person the user put it
|
||||
// on. Identities with a single confirmation are dropped — leaving one out
|
||||
// leaves that identity with no evidence at all, so they measure nothing.
|
||||
let people = faces::people(conn).expect("people");
|
||||
let mut known: Vec<Known> = Vec::new();
|
||||
let mut identities = 0usize;
|
||||
for p in &people {
|
||||
if p.confirmed_faces < 2 {
|
||||
continue;
|
||||
}
|
||||
let mut mine = Vec::new();
|
||||
for f in faces::for_person(conn, p.id, false).expect("faces") {
|
||||
if !f.confirmed {
|
||||
continue;
|
||||
}
|
||||
if let Some((image, embedding, crop_px, _)) = embedding_of.get(&f.id) {
|
||||
mine.push(Known {
|
||||
image: *image,
|
||||
person: p.id,
|
||||
embedding: embedding.clone(),
|
||||
crop_px: *crop_px,
|
||||
});
|
||||
}
|
||||
}
|
||||
if mine.len() >= 2 {
|
||||
identities += 1;
|
||||
known.extend(mine);
|
||||
}
|
||||
}
|
||||
println!(
|
||||
"ground truth: {} confirmed faces across {identities} identities\n",
|
||||
known.len()
|
||||
);
|
||||
if known.len() < 2 {
|
||||
println!("not enough confirmations to evaluate.");
|
||||
return;
|
||||
}
|
||||
|
||||
let mut share_right = 0usize;
|
||||
let mut mean_right = 0usize;
|
||||
// (share of the winner, was the winner correct)
|
||||
let mut reliability: Vec<(f32, bool)> = Vec::with_capacity(known.len());
|
||||
// What each scorer would have *displayed* for the correct answer.
|
||||
let mut shown_share = Vec::with_capacity(known.len());
|
||||
let mut shown_mean = Vec::with_capacity(known.len());
|
||||
|
||||
for (i, me) in known.iter().enumerate() {
|
||||
let mut per_person: HashMap<PersonId, Vec<f32>> = HashMap::new();
|
||||
// The mean baseline is the old code's, which had no floor: it averaged
|
||||
// over every member of the group.
|
||||
let mut all_person: HashMap<PersonId, Vec<f32>> = HashMap::new();
|
||||
for (j, them) in known.iter().enumerate() {
|
||||
if i == j || me.image == them.image {
|
||||
continue;
|
||||
}
|
||||
let cos: f32 = me
|
||||
.embedding
|
||||
.iter()
|
||||
.zip(&them.embedding)
|
||||
.map(|(a, b)| a * b)
|
||||
.sum();
|
||||
// The floor the real scorer sees: `cluster_scored` scans at
|
||||
// `RIVAL_FLOOR` and `identity_shares` never learns about a pair
|
||||
// below it. Summing the near-orthogonal ones here instead of
|
||||
// dropping them is not a stricter test, it is a different
|
||||
// function — fifty identities contributing their *upper tail* of
|
||||
// noise outweigh one contributing a real match.
|
||||
let probability = cal.probability(cos, me.crop_px.min(them.crop_px), 0.0);
|
||||
if probability >= dr_face::RIVAL_FLOOR {
|
||||
per_person.entry(them.person).or_default().push(probability);
|
||||
}
|
||||
all_person.entry(them.person).or_default().push(probability);
|
||||
}
|
||||
|
||||
// The share: sum of the best TOP matches per identity, normalised.
|
||||
// Evidence, and the coherence that goes with it: the sum of the best
|
||||
// TOP matches, and their mean. dr_face::assign shows the product of
|
||||
// that mean and the identity's share of the total.
|
||||
let mut evidence: Vec<(PersonId, f32, f32)> = per_person
|
||||
.iter()
|
||||
.map(|(&p, probabilities)| {
|
||||
let mut v = probabilities.clone();
|
||||
v.sort_by(|a, b| b.total_cmp(a));
|
||||
let counted = v.len().min(TOP);
|
||||
let sum = v.iter().take(TOP).sum::<f32>();
|
||||
(p, sum, sum / counted as f32)
|
||||
})
|
||||
.collect();
|
||||
let total: f32 = evidence.iter().map(|(_, s, _)| *s).sum();
|
||||
evidence.sort_by(|a, b| b.1.total_cmp(&a.1));
|
||||
|
||||
// The number it replaced: the mean over every member of the identity.
|
||||
let mut means: Vec<(PersonId, f32)> = all_person
|
||||
.iter()
|
||||
.map(|(&p, probabilities)| {
|
||||
(
|
||||
p,
|
||||
probabilities.iter().sum::<f32>() / probabilities.len() as f32,
|
||||
)
|
||||
})
|
||||
.collect();
|
||||
means.sort_by(|a, b| b.1.total_cmp(&a.1));
|
||||
|
||||
if let (Some(&(winner, score, coherence)), true) = (evidence.first(), total > 0.0) {
|
||||
let correct = winner == me.person;
|
||||
share_right += correct as usize;
|
||||
// What the screen would say about the identity it picked.
|
||||
reliability.push((coherence * score / total, correct));
|
||||
let ours = evidence
|
||||
.iter()
|
||||
.find(|(p, _, _)| *p == me.person)
|
||||
.map(|(_, s, c)| c * s / total)
|
||||
.unwrap_or(0.0);
|
||||
shown_share.push(ours);
|
||||
}
|
||||
if let Some(&(winner, _)) = means.first() {
|
||||
mean_right += (winner == me.person) as usize;
|
||||
shown_mean.push(
|
||||
means
|
||||
.iter()
|
||||
.find(|(p, _)| *p == me.person)
|
||||
.map(|(_, s)| *s)
|
||||
.unwrap_or(0.0),
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
let n = known.len() as f64;
|
||||
println!("which identity does this face belong to? (leave-one-out, top-1)");
|
||||
println!(
|
||||
" share of evidence {:>6.2}% ({share_right}/{})",
|
||||
100.0 * share_right as f64 / n,
|
||||
known.len()
|
||||
);
|
||||
println!(
|
||||
" mean within group {:>6.2}% ({mean_right}/{})\n",
|
||||
100.0 * mean_right as f64 / n,
|
||||
known.len()
|
||||
);
|
||||
|
||||
println!("what the screen would show for the answer the user gave:");
|
||||
band(" share ", &shown_share);
|
||||
band(" mean ", &shown_mean);
|
||||
|
||||
println!("\nreliability of the share — is a stated {{n}}% right {{n}}% of the time?");
|
||||
println!(
|
||||
" {:>12} {:>7} {:>9} {:>8}",
|
||||
"stated", "faces", "correct", "gap"
|
||||
);
|
||||
for (lo, hi) in [
|
||||
(0.0, 0.5),
|
||||
(0.5, 0.6),
|
||||
(0.6, 0.7),
|
||||
(0.7, 0.8),
|
||||
(0.8, 0.9),
|
||||
(0.9, 0.95),
|
||||
(0.95, 1.001),
|
||||
] {
|
||||
let bucket: Vec<bool> = reliability
|
||||
.iter()
|
||||
.filter(|(s, _)| *s >= lo && *s < hi)
|
||||
.map(|(_, c)| *c)
|
||||
.collect();
|
||||
if bucket.is_empty() {
|
||||
continue;
|
||||
}
|
||||
let observed = bucket.iter().filter(|c| **c).count() as f64 / bucket.len() as f64;
|
||||
let stated = reliability
|
||||
.iter()
|
||||
.filter(|(s, _)| *s >= lo && *s < hi)
|
||||
.map(|(s, _)| *s as f64)
|
||||
.sum::<f64>()
|
||||
/ bucket.len() as f64;
|
||||
println!(
|
||||
" {:>5.0}–{:>3.0}% {:>9} {:>8.1}% {:>+7.1}",
|
||||
lo * 100.0,
|
||||
hi.min(1.0) * 100.0,
|
||||
bucket.len(),
|
||||
100.0 * observed,
|
||||
100.0 * (observed - stated)
|
||||
);
|
||||
}
|
||||
|
||||
if args.iter().any(|a| a == "--full") {
|
||||
// The confirmations go in as anchors, exactly as `recluster` sends
|
||||
// them: they are what makes a group a named identity, and therefore
|
||||
// what makes it a rival.
|
||||
let mut confirmed = HashMap::new();
|
||||
for p in &people {
|
||||
for f in faces::for_person(conn, p.id, false).expect("faces") {
|
||||
if f.confirmed {
|
||||
confirmed.insert(f.id, p.id.0);
|
||||
}
|
||||
}
|
||||
}
|
||||
full_library(&embedding_of, &confirmed, &cal);
|
||||
}
|
||||
}
|
||||
|
||||
/// Where a set of confidences actually falls.
|
||||
fn band(label: &str, v: &[f32]) {
|
||||
if v.is_empty() {
|
||||
return;
|
||||
}
|
||||
let mut s = v.to_vec();
|
||||
s.sort_by(|a, b| a.total_cmp(b));
|
||||
let pct = |q: f64| s[((s.len() - 1) as f64 * q) as usize];
|
||||
let mean = s.iter().sum::<f32>() / s.len() as f32;
|
||||
println!(
|
||||
"{label} median {:>5.1}% mean {:>5.1}% p10 {:>5.1}% p90 {:>5.1}% under 50%: {:>5.1}%",
|
||||
100.0 * pct(0.5),
|
||||
100.0 * mean,
|
||||
100.0 * pct(0.10),
|
||||
100.0 * pct(0.90),
|
||||
100.0 * s.iter().filter(|x| **x < 0.5).count() as f32 / s.len() as f32
|
||||
);
|
||||
}
|
||||
|
||||
/// The whole library through the real clusterer, for the numbers it would
|
||||
/// actually write.
|
||||
fn full_library(
|
||||
embedding_of: &HashMap<faces::FaceId, Decoded>,
|
||||
confirmed: &HashMap<faces::FaceId, u64>,
|
||||
cal: &dr_face::Calibration,
|
||||
) {
|
||||
let mut candidates: Vec<dr_face::Candidate> = embedding_of
|
||||
.iter()
|
||||
.map(
|
||||
|(id, (image, embedding, crop_px, quality))| dr_face::Candidate {
|
||||
face: id.0,
|
||||
image: *image,
|
||||
embedding: embedding.clone(),
|
||||
crop_px: *crop_px,
|
||||
quality: *quality,
|
||||
confirmed_person: confirmed.get(id).copied(),
|
||||
},
|
||||
)
|
||||
.collect();
|
||||
candidates.sort_by_key(|c| c.face);
|
||||
|
||||
println!("\nthe whole library, at the default merge probability:");
|
||||
|
||||
// The three phases, separately, because "a regroup takes n seconds" does
|
||||
// not tell anyone which half to optimise — and the answer differs between
|
||||
// a desktop and a tablet (docs/faces.md §9).
|
||||
{
|
||||
let dim = candidates.first().map(|c| c.embedding.len()).unwrap_or(0);
|
||||
let flat: Vec<f32> = candidates
|
||||
.iter()
|
||||
.flat_map(|c| c.embedding.clone())
|
||||
.collect();
|
||||
let crop_px: Vec<f32> = candidates.iter().map(|c| c.crop_px).collect();
|
||||
let images: Vec<u64> = candidates.iter().map(|c| c.image).collect();
|
||||
let gallery: Vec<bool> = candidates.iter().map(|c| c.in_gallery()).collect();
|
||||
let view = dr_face::neighbours::Faces {
|
||||
embeddings: &flat,
|
||||
dim,
|
||||
crop_px: &crop_px,
|
||||
images: &images,
|
||||
gallery: &gallery,
|
||||
};
|
||||
|
||||
let t = std::time::Instant::now();
|
||||
let evidence = dr_face::neighbours::above_threshold(&view, cal, dr_face::RIVAL_FLOOR);
|
||||
let scan = t.elapsed().as_secs_f64();
|
||||
|
||||
// `cluster` runs its own scan at the merge threshold, so the
|
||||
// agglomeration is what is left after taking one scan off the total.
|
||||
let t = std::time::Instant::now();
|
||||
let clusters = dr_face::cluster(&candidates, cal, dr_face::DEFAULT_MERGE_PROBABILITY);
|
||||
let agglomerate = t.elapsed().as_secs_f64() - scan;
|
||||
|
||||
let t = std::time::Instant::now();
|
||||
let _ = dr_face::identity_shares(&gallery, &clusters, &evidence, dr_face::TOP_MATCHES);
|
||||
println!(
|
||||
" scan {scan:.2}s ({} evidence pairs) · agglomerate {agglomerate:.2}s · score {:.2}s",
|
||||
evidence.len(),
|
||||
t.elapsed().as_secs_f64()
|
||||
);
|
||||
}
|
||||
|
||||
let start = std::time::Instant::now();
|
||||
let grouping = dr_face::cluster_scored(&candidates, cal, dr_face::DEFAULT_MERGE_PROBABILITY);
|
||||
let real: Vec<_> = grouping
|
||||
.clusters
|
||||
.iter()
|
||||
.filter(|c| c.members.len() >= 2)
|
||||
.collect();
|
||||
let grouped: usize = real.iter().map(|c| c.members.len()).sum();
|
||||
println!(
|
||||
" {} face(s) → {} group(s) of two or more, holding {grouped} faces ({:.0}%), in {:.1}s",
|
||||
candidates.len(),
|
||||
real.len(),
|
||||
100.0 * grouped as f64 / candidates.len() as f64,
|
||||
start.elapsed().as_secs_f64()
|
||||
);
|
||||
|
||||
let named: Vec<_> = real.iter().filter(|c| c.person.is_some()).collect();
|
||||
println!(
|
||||
" {} of those group(s) carry a confirmation, holding {} faces",
|
||||
named.len(),
|
||||
named.iter().map(|c| c.members.len()).sum::<usize>()
|
||||
);
|
||||
|
||||
let shown: Vec<f32> = real
|
||||
.iter()
|
||||
.flat_map(|c| c.members.iter().map(|&m| grouping.confidence[m]))
|
||||
.collect();
|
||||
band(" new, all groups ", &shown);
|
||||
let onto_people: Vec<f32> = named
|
||||
.iter()
|
||||
.flat_map(|c| c.members.iter().map(|&m| grouping.confidence[m]))
|
||||
.collect();
|
||||
band(" new, onto a person", &onto_people);
|
||||
|
||||
// The number the old code would have written for the same grouping.
|
||||
let means: Vec<f32> = real
|
||||
.iter()
|
||||
.flat_map(|c| {
|
||||
c.members.iter().map(|&m| {
|
||||
let me = &candidates[m];
|
||||
let mut sum = 0.0;
|
||||
let mut n = 0.0;
|
||||
for &other in &c.members {
|
||||
if other == m {
|
||||
continue;
|
||||
}
|
||||
let them = &candidates[other];
|
||||
let cos: f32 = me
|
||||
.embedding
|
||||
.iter()
|
||||
.zip(&them.embedding)
|
||||
.map(|(a, b)| a * b)
|
||||
.sum();
|
||||
sum += cal.probability(cos, me.crop_px.min(them.crop_px), 0.0);
|
||||
n += 1.0;
|
||||
}
|
||||
if n == 0.0 {
|
||||
1.0
|
||||
} else {
|
||||
sum / n
|
||||
}
|
||||
})
|
||||
})
|
||||
.collect();
|
||||
band(" old, all groups ", &means);
|
||||
}
|
||||
@@ -1,111 +0,0 @@
|
||||
//! Scan a real folder on this machine into a catalog, and say what it cost.
|
||||
//!
|
||||
//! cargo run -p dr-catalog --example scan_local -- ~/Pictures [catalog.sqlite]
|
||||
//!
|
||||
//! **Run it twice.** The first run is a full walk; the second is the one worth
|
||||
//! watching, because on an unchanged library it should list no directories at
|
||||
//! all and take a fraction of the time. That difference is NFR-P1, and a
|
||||
//! synthetic test cannot show it at the scale a real library does — 121,785
|
||||
//! files in a synced folder is a different question from twenty in a temporary
|
||||
//! directory.
|
||||
//!
|
||||
//! Writes only to the catalog file, which defaults to a fixed path in the
|
||||
//! system temporary directory so a second run has something to compare
|
||||
//! against. Nothing in the scanned folder is touched.
|
||||
|
||||
use std::path::PathBuf;
|
||||
|
||||
use dr_catalog::walk::{ensure_root, scan_root, RootKind};
|
||||
use dr_catalog::Catalog;
|
||||
use dr_plat::LocalStorage;
|
||||
use dr_types::FormatFilter;
|
||||
|
||||
fn main() {
|
||||
env_logger::init();
|
||||
|
||||
let mut args = std::env::args().skip(1);
|
||||
let Some(dir) = args.next().map(PathBuf::from) else {
|
||||
eprintln!("usage: scan_local <directory> [catalog.sqlite]");
|
||||
std::process::exit(2);
|
||||
};
|
||||
let catalog_path = args
|
||||
.next()
|
||||
.map(PathBuf::from)
|
||||
.unwrap_or_else(|| std::env::temp_dir().join("darkroom-scan-local.sqlite"));
|
||||
|
||||
let catalog = match Catalog::open(&catalog_path) {
|
||||
Ok(c) => c,
|
||||
Err(e) => {
|
||||
eprintln!("cannot open {}: {e}", catalog_path.display());
|
||||
std::process::exit(1);
|
||||
}
|
||||
};
|
||||
println!("catalog: {}", catalog_path.display());
|
||||
|
||||
// The label is how the grant is spelled, and the only place a path is
|
||||
// written down. Everything after this line addresses files by `RootId`.
|
||||
let label = dir.display().to_string();
|
||||
let root = match ensure_root(catalog.connection(), RootKind::Local, &label) {
|
||||
Ok(r) => r,
|
||||
Err(e) => {
|
||||
eprintln!("cannot record the root: {e}");
|
||||
std::process::exit(1);
|
||||
}
|
||||
};
|
||||
let storage = LocalStorage::with_root(root, &dir);
|
||||
|
||||
let now = std::time::SystemTime::now()
|
||||
.duration_since(std::time::UNIX_EPOCH)
|
||||
.map(|d| d.as_secs() as i64)
|
||||
.unwrap_or(0);
|
||||
|
||||
let started = std::time::Instant::now();
|
||||
let report = match scan_root(
|
||||
catalog.connection(),
|
||||
&storage,
|
||||
root,
|
||||
&FormatFilter::all(),
|
||||
now,
|
||||
|| false,
|
||||
|p| {
|
||||
// One line per hundred directories: enough to show it is alive on a
|
||||
// large library, not enough to be the thing that slows it down.
|
||||
let visited = p.directories_listed + p.directories_pruned;
|
||||
if visited % 100 == 0 {
|
||||
println!(
|
||||
" … {visited} directories ({} pruned), {} images",
|
||||
p.directories_pruned, p.images_found
|
||||
);
|
||||
}
|
||||
},
|
||||
) {
|
||||
Ok(r) => r,
|
||||
Err(e) => {
|
||||
eprintln!("scan failed: {e}");
|
||||
std::process::exit(1);
|
||||
}
|
||||
};
|
||||
let elapsed = started.elapsed();
|
||||
|
||||
let total: i64 = catalog
|
||||
.connection()
|
||||
.query_row("SELECT count(*) FROM images", [], |r| r.get(0))
|
||||
.unwrap_or(-1);
|
||||
|
||||
println!("\noutcome: {:?}", report.outcome);
|
||||
println!(
|
||||
"directories: {} listed, {} pruned",
|
||||
report.progress.directories_listed, report.progress.directories_pruned
|
||||
);
|
||||
println!(
|
||||
"images: {} new, {} changed, {} unchanged, {} removed",
|
||||
report.inserted, report.updated, report.unchanged, report.images_removed
|
||||
);
|
||||
println!("folders: {} removed", report.folders_removed);
|
||||
println!("catalogued: {total} in total");
|
||||
println!("took: {:.2?}", elapsed);
|
||||
|
||||
if report.progress.directories_listed == 0 && report.progress.directories_pruned > 0 {
|
||||
println!("\nnothing had changed: every folder was proven unchanged by one probe");
|
||||
}
|
||||
}
|
||||
@@ -1,111 +0,0 @@
|
||||
//! What a second device ends up with after adopting this library.
|
||||
//!
|
||||
//! Stands up an empty catalog, gives it the images the real one has, adopts the
|
||||
//! face shards into it exactly as a sync would, merges the real catalog in as a
|
||||
//! remote — and then counts. The point is to answer "why does the tablet show
|
||||
//! fewer faces for this person" without needing the tablet.
|
||||
//!
|
||||
//! cargo run -p dr-catalog --example sync_probe -- CATALOG.sqlite FACES_DIR
|
||||
|
||||
use std::path::PathBuf;
|
||||
|
||||
use dr_catalog::face_shard::{self, FaceShardStore};
|
||||
use dr_catalog::Catalog;
|
||||
|
||||
const MODEL: &str = "w600k_mbf";
|
||||
|
||||
fn main() {
|
||||
let args: Vec<String> = std::env::args().skip(1).collect();
|
||||
if args.len() < 2 {
|
||||
eprintln!("usage: sync_probe CATALOG.sqlite FACES_DIR");
|
||||
std::process::exit(2);
|
||||
}
|
||||
let source = PathBuf::from(&args[0]);
|
||||
let faces_dir = PathBuf::from(&args[1]);
|
||||
|
||||
let dir = std::env::temp_dir().join(format!("dr-sync-probe-{}", std::process::id()));
|
||||
let _ = std::fs::remove_dir_all(&dir);
|
||||
std::fs::create_dir_all(&dir).unwrap();
|
||||
let dest = dir.join("catalog.sqlite");
|
||||
|
||||
let far = Catalog::open(&dest).expect("fresh catalog");
|
||||
let conn = far.connection();
|
||||
|
||||
// The images a scan would have found. Nothing else: no faces, no people.
|
||||
conn.execute(
|
||||
"ATTACH DATABASE ?1 AS src",
|
||||
[source.to_string_lossy().as_ref()],
|
||||
)
|
||||
.unwrap();
|
||||
// Foreign keys off for the copy: `images` carries self-references
|
||||
// (`shadowed_by`) that are only consistent once every row is in, and this
|
||||
// is a bulk clone rather than an edit.
|
||||
conn.execute_batch(
|
||||
"PRAGMA foreign_keys = OFF;
|
||||
INSERT INTO roots SELECT * FROM src.roots;
|
||||
INSERT INTO images SELECT * FROM src.images;
|
||||
INSERT INTO remote SELECT * FROM src.remote;
|
||||
PRAGMA foreign_keys = ON;",
|
||||
)
|
||||
.unwrap();
|
||||
let images: i64 = conn
|
||||
.query_row("SELECT COUNT(*) FROM images", [], |r| r.get(0))
|
||||
.unwrap();
|
||||
conn.execute_batch("DETACH DATABASE src").unwrap();
|
||||
println!("second device starts with {images} image(s), no faces");
|
||||
|
||||
// Adopt every shard, which is what a completed face sync leaves behind.
|
||||
let store = FaceShardStore::open(&faces_dir).expect("shard store");
|
||||
let adopted = face_shard::import_from_shards(conn, &store, MODEL).expect("import");
|
||||
let faces: i64 = conn
|
||||
.query_row("SELECT COUNT(*) FROM faces", [], |r| r.get(0))
|
||||
.unwrap();
|
||||
println!("adopted {adopted} image(s) from the shards -> {faces} face(s)");
|
||||
|
||||
// Then the catalog merge, which is where people and their judgements come.
|
||||
let report = dr_catalog::sync::merge_remote(conn, &source).expect("merge");
|
||||
println!(
|
||||
"merge: {} people in, {} updated, {} kept local, {} face(s) assigned, \
|
||||
{} kept local, {} rejection(s)",
|
||||
report.people_inserted,
|
||||
report.people_updated,
|
||||
report.people_kept_local,
|
||||
report.faces_assigned,
|
||||
report.faces_kept_local,
|
||||
report.faces_rejected,
|
||||
);
|
||||
|
||||
// Per person, against what the source holds.
|
||||
conn.execute(
|
||||
"ATTACH DATABASE ?1 AS src",
|
||||
[source.to_string_lossy().as_ref()],
|
||||
)
|
||||
.unwrap();
|
||||
let mut q = conn
|
||||
.prepare(
|
||||
"SELECT p.name,
|
||||
(SELECT COUNT(*) FROM src.face_person sfp
|
||||
JOIN src.people sp ON sp.id = sfp.person_id
|
||||
WHERE sp.uuid = p.uuid) AS there,
|
||||
(SELECT COUNT(*) FROM face_person fp WHERE fp.person_id = p.id) AS here
|
||||
FROM people p
|
||||
WHERE p.name != ''
|
||||
ORDER BY there DESC LIMIT 12",
|
||||
)
|
||||
.unwrap();
|
||||
println!("\n{:<24} {:>8} {:>8}", "person", "source", "here");
|
||||
let rows = q
|
||||
.query_map([], |r| {
|
||||
Ok((
|
||||
r.get::<_, String>(0)?,
|
||||
r.get::<_, i64>(1)?,
|
||||
r.get::<_, i64>(2)?,
|
||||
))
|
||||
})
|
||||
.unwrap();
|
||||
for row in rows.flatten() {
|
||||
println!("{:<24} {:>8} {:>8}", row.0, row.1, row.2);
|
||||
}
|
||||
|
||||
println!("\nprobe catalog left at {}", dest.display());
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
@@ -1,308 +0,0 @@
|
||||
//! TRACES: FR-CAT-11
|
||||
//! Has this photograph been imported before?
|
||||
//!
|
||||
//! Two tiers, because neither alone is enough and they cost very different
|
||||
//! amounts. The metadata tier — capture time, camera, size, the name the
|
||||
//! camera gave it — is answerable from the catalog before a byte leaves the
|
||||
//! card, which is what makes re-inserting an already-imported card cost a
|
||||
//! metadata read per file rather than a full transfer. The content tier
|
||||
//! catches what the first misses: the same frame arriving under a different
|
||||
//! name, from a second card, or after somebody renamed it.
|
||||
//!
|
||||
//! # Why the filename is compared here rather than in SQL
|
||||
//!
|
||||
//! `images.source_ref` holds the whole opaque key — a relative path on Linux,
|
||||
//! a document id on SAF — and the camera's filename is only its last
|
||||
//! component. Matching that in SQL means `LIKE '%/IMG_0001.CR3'`, which cannot
|
||||
//! use an index, scans the whole table, and is wrong on SAF where the
|
||||
//! separator is not `/`. So the query narrows on the indexed columns and the
|
||||
//! handful of rows that survive are compared in Rust, the same way the grid
|
||||
//! already derives a display name.
|
||||
//!
|
||||
//! # Filename alone is never sufficient
|
||||
//!
|
||||
//! Camera filenames wrap at `IMG_9999` and start again, so a library of any
|
||||
//! age holds several unrelated `IMG_0001.CR3`. That is why the cheap tier
|
||||
//! carries capture time and camera as well, and why the expensive tier exists
|
||||
//! at all.
|
||||
|
||||
use rusqlite::Connection;
|
||||
|
||||
use crate::CatalogError;
|
||||
|
||||
/// The last component of a stored source reference.
|
||||
///
|
||||
/// Splits on both separators for the same reason `Catalog::window` does: the
|
||||
/// key's shape belongs to the storage that produced it, and a SAF document id
|
||||
/// is delimited with `:`.
|
||||
fn file_name(source_ref: &str) -> &str {
|
||||
source_ref.rsplit(['/', ':']).next().unwrap_or(source_ref)
|
||||
}
|
||||
|
||||
/// Whether the catalog already holds this photograph, on metadata alone.
|
||||
///
|
||||
/// `camera` is the joined make-and-model string the scan stores, not the raw
|
||||
/// EXIF pair — the caller composes it the same way, or the comparison is
|
||||
/// always false.
|
||||
///
|
||||
/// A `captured_at` of `None` makes this answer `false` rather than matching
|
||||
/// every undated image in the library: without a capture time the key is
|
||||
/// filename plus size, which two frames from the same body collide on
|
||||
/// routinely. An undated file falls through to the content tier, which is
|
||||
/// slower and right.
|
||||
pub fn seen_by_metadata(
|
||||
conn: &Connection,
|
||||
captured_at: Option<i64>,
|
||||
camera: Option<&str>,
|
||||
size: u64,
|
||||
original_name: &str,
|
||||
) -> Result<bool, CatalogError> {
|
||||
let Some(captured_at) = captured_at else {
|
||||
return Ok(false);
|
||||
};
|
||||
|
||||
// `images_captured` indexes the capture time, so this reads a few rows
|
||||
// even in a library of fifty thousand: one instant to the second holds
|
||||
// one frame, or a handful on a body shooting a burst.
|
||||
let mut stmt = conn.prepare(
|
||||
"SELECT source_ref FROM images
|
||||
WHERE captured_at = ?1
|
||||
AND (?2 IS NULL OR camera IS ?2)
|
||||
AND (file_size IS NULL OR file_size = ?3)",
|
||||
)?;
|
||||
let mut rows = stmt.query(rusqlite::params![captured_at, camera, size as i64])?;
|
||||
while let Some(row) = rows.next()? {
|
||||
let source_ref: String = row.get(0)?;
|
||||
if file_name(&source_ref).eq_ignore_ascii_case(original_name) {
|
||||
return Ok(true);
|
||||
}
|
||||
}
|
||||
Ok(false)
|
||||
}
|
||||
|
||||
/// Whether these exact bytes are already in the library.
|
||||
///
|
||||
/// The tier that costs a read of the file. Cheap here — `images_hash` is a
|
||||
/// partial index over the rows that have one — and expensive for the caller,
|
||||
/// which had to hash something to ask.
|
||||
pub fn seen_by_content(conn: &Connection, digest: &str) -> Result<bool, CatalogError> {
|
||||
let n: i64 = conn.query_row(
|
||||
"SELECT COUNT(*) FROM images WHERE content_hash = ?1",
|
||||
[digest],
|
||||
|r| r.get(0),
|
||||
)?;
|
||||
Ok(n > 0)
|
||||
}
|
||||
|
||||
/// Record the digest of a file the import computed.
|
||||
///
|
||||
/// An import reads every byte anyway, so the hash is free at that moment and
|
||||
/// costs a full read of an 80 MB file at any other. Storing it is what lets
|
||||
/// the *next* import answer [`seen_by_content`] without reading anything.
|
||||
///
|
||||
/// Matched on `source_ref` within a root, which is how the scan that just
|
||||
/// catalogued the imported file identifies it. Returns how many rows were
|
||||
/// updated: zero means the scan has not reached the file yet, which is a
|
||||
/// normal race and not an error.
|
||||
pub fn set_content_hash(
|
||||
conn: &Connection,
|
||||
root_id: u64,
|
||||
source_ref: &str,
|
||||
digest: &str,
|
||||
) -> Result<usize, CatalogError> {
|
||||
Ok(conn.execute(
|
||||
"UPDATE images SET content_hash = ?3
|
||||
WHERE root_id = ?1 AND source_ref = ?2",
|
||||
rusqlite::params![root_id as i64, source_ref, digest],
|
||||
)?)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use crate::Catalog;
|
||||
|
||||
/// A catalog holding one photograph, as a scan plus a metadata pass would
|
||||
/// leave it.
|
||||
fn with_one() -> Catalog {
|
||||
let cat = Catalog::in_memory().unwrap();
|
||||
let c = cat.connection();
|
||||
c.execute(
|
||||
"INSERT INTO roots(id, kind, label) VALUES (1, 'local', 'lib')",
|
||||
[],
|
||||
)
|
||||
.unwrap();
|
||||
c.execute(
|
||||
"INSERT INTO images(root_id, source_ref, captured_at, camera, file_size,
|
||||
content_hash, added_at)
|
||||
VALUES (1, '2026/2026-08-22/IMG_0001.CR3', 1787407200, 'Canon EOS R5',
|
||||
9, 'deadbeef', 0)",
|
||||
[],
|
||||
)
|
||||
.unwrap();
|
||||
cat
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn re_inserting_the_same_card_is_recognised_before_a_transfer() {
|
||||
let cat = with_one();
|
||||
assert!(seen_by_metadata(
|
||||
cat.connection(),
|
||||
Some(1_787_407_200),
|
||||
Some("Canon EOS R5"),
|
||||
9,
|
||||
"IMG_0001.CR3"
|
||||
)
|
||||
.unwrap());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_different_frame_at_the_same_instant_is_not_a_duplicate() {
|
||||
// Two bodies firing together, or a burst. The name separates them.
|
||||
let cat = with_one();
|
||||
assert!(!seen_by_metadata(
|
||||
cat.connection(),
|
||||
Some(1_787_407_200),
|
||||
Some("Canon EOS R5"),
|
||||
9,
|
||||
"IMG_0002.CR3"
|
||||
)
|
||||
.unwrap());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn the_same_name_from_a_different_camera_is_not_a_duplicate() {
|
||||
// IMG_0001.CR3 exists on every card ever formatted.
|
||||
let cat = with_one();
|
||||
assert!(!seen_by_metadata(
|
||||
cat.connection(),
|
||||
Some(1_787_407_200),
|
||||
Some("NIKON Z 9"),
|
||||
9,
|
||||
"IMG_0001.CR3"
|
||||
)
|
||||
.unwrap());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn the_same_name_at_a_different_time_is_not_a_duplicate() {
|
||||
// The IMG_9999 wrap: the library holds an unrelated IMG_0001.CR3 from
|
||||
// four years ago, and matching on name alone would refuse to import
|
||||
// today's.
|
||||
let cat = with_one();
|
||||
assert!(!seen_by_metadata(
|
||||
cat.connection(),
|
||||
Some(1_600_000_000),
|
||||
Some("Canon EOS R5"),
|
||||
9,
|
||||
"IMG_0001.CR3"
|
||||
)
|
||||
.unwrap());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn an_undated_file_falls_through_to_the_content_tier() {
|
||||
// Not "matches everything undated" — that would silently refuse to
|
||||
// import a whole card of scanned film.
|
||||
let cat = with_one();
|
||||
assert!(!seen_by_metadata(cat.connection(), None, None, 9, "IMG_0001.CR3").unwrap());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_file_that_grew_is_not_the_one_already_held() {
|
||||
// A truncated earlier import, or a different rendition of the same
|
||||
// frame. Same instant, same camera, same name, different bytes.
|
||||
let cat = with_one();
|
||||
assert!(!seen_by_metadata(
|
||||
cat.connection(),
|
||||
Some(1_787_407_200),
|
||||
Some("Canon EOS R5"),
|
||||
1234,
|
||||
"IMG_0001.CR3"
|
||||
)
|
||||
.unwrap());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_row_with_no_recorded_size_still_matches() {
|
||||
// The scan stores a size, but a row merged from another device may
|
||||
// not have one, and refusing to match it would re-import the library.
|
||||
let cat = with_one();
|
||||
cat.connection()
|
||||
.execute("UPDATE images SET file_size = NULL", [])
|
||||
.unwrap();
|
||||
assert!(seen_by_metadata(
|
||||
cat.connection(),
|
||||
Some(1_787_407_200),
|
||||
Some("Canon EOS R5"),
|
||||
9,
|
||||
"IMG_0001.CR3"
|
||||
)
|
||||
.unwrap());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn the_same_frame_renamed_is_caught_by_its_bytes() {
|
||||
let cat = with_one();
|
||||
// The metadata tier misses it...
|
||||
assert!(!seen_by_metadata(
|
||||
cat.connection(),
|
||||
Some(1_787_407_200),
|
||||
Some("Canon EOS R5"),
|
||||
9,
|
||||
"holiday-42.CR3"
|
||||
)
|
||||
.unwrap());
|
||||
// ...and the content tier does not.
|
||||
assert!(seen_by_content(cat.connection(), "deadbeef").unwrap());
|
||||
assert!(!seen_by_content(cat.connection(), "cafe").unwrap());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_digest_recorded_now_answers_the_next_import() {
|
||||
let cat = Catalog::in_memory().unwrap();
|
||||
let c = cat.connection();
|
||||
c.execute(
|
||||
"INSERT INTO roots(id, kind, label) VALUES (1, 'local', 'lib')",
|
||||
[],
|
||||
)
|
||||
.unwrap();
|
||||
c.execute(
|
||||
"INSERT INTO images(root_id, source_ref, added_at)
|
||||
VALUES (1, '2026/2026-08-22/IMG_0001.CR3', 0)",
|
||||
[],
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
assert!(!seen_by_content(c, "abc123").unwrap());
|
||||
let n = set_content_hash(c, 1, "2026/2026-08-22/IMG_0001.CR3", "abc123").unwrap();
|
||||
assert_eq!(n, 1);
|
||||
assert!(seen_by_content(c, "abc123").unwrap());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn recording_a_digest_before_the_scan_arrives_is_not_an_error() {
|
||||
// The import writes the file and the scan catalogues it; between those
|
||||
// two moments there is no row to update, and that is a race rather
|
||||
// than a failure.
|
||||
let cat = Catalog::in_memory().unwrap();
|
||||
let c = cat.connection();
|
||||
c.execute(
|
||||
"INSERT INTO roots(id, kind, label) VALUES (1, 'local', 'lib')",
|
||||
[],
|
||||
)
|
||||
.unwrap();
|
||||
assert_eq!(
|
||||
set_content_hash(c, 1, "not/scanned/yet.CR3", "abc").unwrap(),
|
||||
0
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_name_is_the_last_component_of_either_kind_of_key() {
|
||||
assert_eq!(file_name("2026/2026-08-22/IMG_0001.CR3"), "IMG_0001.CR3");
|
||||
// A SAF document id delimits with a colon.
|
||||
assert_eq!(file_name("primary:DCIM/Camera/IMG_1.CR3"), "IMG_1.CR3");
|
||||
assert_eq!(file_name("IMG_0001.CR3"), "IMG_0001.CR3");
|
||||
}
|
||||
}
|
||||
@@ -1,134 +0,0 @@
|
||||
//! TRACES: NFR-ARCH-4 | NFR-R5 | NFR-R6
|
||||
//! Catalog errors.
|
||||
//!
|
||||
//! Typed and attached to the affected subject rather than panicking — a
|
||||
//! corrupt row or a failed job marks one image and lets the batch continue.
|
||||
//!
|
||||
//! # Why `From<rusqlite::Error>` is written by hand
|
||||
//!
|
||||
//! One class of SQLite failure is not about the statement that hit it: when
|
||||
//! the file itself is damaged, *every* query fails, and which one the user
|
||||
//! happened to trigger first says nothing. Before this, corruption reached the
|
||||
//! interface as whatever `Sqlite(...)` the first failing query produced —
|
||||
//! "database disk image is malformed" attached to a thumbnail refresh — and
|
||||
//! there was nowhere to hang a recovery offer.
|
||||
//!
|
||||
//! So the conversion classifies rather than wraps: `SQLITE_CORRUPT` and
|
||||
//! `SQLITE_NOTADB` become [`CatalogError::Corrupt`] wherever they arise, which
|
||||
//! means a background job that trips over the damage reports the same thing
|
||||
//! the startup check does (see [`crate::recovery`]).
|
||||
|
||||
/// Something went wrong talking to the catalog.
|
||||
#[derive(Debug, thiserror::Error)]
|
||||
pub enum CatalogError {
|
||||
#[error("sqlite: {0}")]
|
||||
Sqlite(#[source] rusqlite::Error),
|
||||
|
||||
/// The catalog file is damaged.
|
||||
///
|
||||
/// Its own variant because it is the one error with a *user-facing
|
||||
/// remedy*: restore the NFR-R2 backup, or discard the index and rebuild it
|
||||
/// from sources plus sidecars (NFR-R6, invariant §5.2.4). Every other
|
||||
/// variant here is either a caller's mistake or a fact about one row.
|
||||
#[error("the catalog file is damaged: {detail}")]
|
||||
Corrupt { detail: String },
|
||||
|
||||
/// The catalog was written by a newer build.
|
||||
///
|
||||
/// Opening it read-write would corrupt state this build cannot represent,
|
||||
/// so the app refuses and says so (NFR-R5).
|
||||
#[error("catalog schema v{found} is newer than this build supports (v{supported})")]
|
||||
SchemaTooNew { found: i64, supported: i64 },
|
||||
|
||||
/// A scan could not reach a root at all.
|
||||
///
|
||||
/// Distinct from "files are missing": this aborts the scan *before* the
|
||||
/// deletion sweep, because every folder would look unreached and the sweep
|
||||
/// would delete the whole library (FR-CAT-9).
|
||||
#[error("root {0} is unreachable; scan aborted without pruning")]
|
||||
RootUnreachable(u64),
|
||||
|
||||
/// A scan was asked for a root the catalog has no row for.
|
||||
///
|
||||
/// A caller's mistake rather than a user's: the row is created when the
|
||||
/// grant is obtained, because the label — the path, the tree URI — is known
|
||||
/// only there. Inventing one here would file the library under a name
|
||||
/// nothing else would look it up by.
|
||||
#[error("no such root: {0}")]
|
||||
NoSuchRoot(u64),
|
||||
|
||||
/// A smart collection whose selector references itself, directly or via
|
||||
/// another collection.
|
||||
#[error("collection {0} would form a cycle")]
|
||||
CollectionCycle(u64),
|
||||
|
||||
#[error("no such collection: {0}")]
|
||||
NoSuchCollection(u64),
|
||||
|
||||
/// A keyword the caller named is gone — deleted, or fused into another by a
|
||||
/// merge while its id sat in a UI model.
|
||||
///
|
||||
/// Its own variant rather than a silent no-op because the two are different
|
||||
/// answers to the user: a rename that quietly did nothing looks exactly like
|
||||
/// a rename that did not take.
|
||||
#[error("no such keyword: {0}")]
|
||||
NoSuchKeyword(u64),
|
||||
|
||||
/// Images were dropped onto a smart collection.
|
||||
///
|
||||
/// A smart collection's membership *is* its selector, so member rows would
|
||||
/// be a second source of truth that nothing reads. Refused rather than
|
||||
/// silently discarded, so the UI can say why the drop did nothing.
|
||||
#[error("collection {0} is a saved filter; its contents cannot be edited by hand")]
|
||||
SmartCollectionNotEditable(u64),
|
||||
|
||||
#[error("malformed stored selector: {0}")]
|
||||
BadSelector(String),
|
||||
|
||||
/// A name the user typed that cannot be stored — blank, or one a sibling
|
||||
/// already holds.
|
||||
///
|
||||
/// Its own variant rather than a reused `BadSelector`, because this one is
|
||||
/// shown to the user verbatim: it has to read as a sentence about their
|
||||
/// collection, not as a diagnostic about a stored selector.
|
||||
#[error("{0}")]
|
||||
BadName(String),
|
||||
|
||||
#[error("io: {0}")]
|
||||
Io(String),
|
||||
}
|
||||
|
||||
impl From<rusqlite::Error> for CatalogError {
|
||||
fn from(e: rusqlite::Error) -> Self {
|
||||
if is_corruption(&e) {
|
||||
// `to_string` rather than keeping the error: the detail is going
|
||||
// into a dialog and into a log line, and the recovery path has no
|
||||
// use for the rusqlite type once it knows the file is damaged.
|
||||
CatalogError::Corrupt {
|
||||
detail: e.to_string(),
|
||||
}
|
||||
} else {
|
||||
CatalogError::Sqlite(e)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Whether a SQLite failure means the *file* is damaged rather than the
|
||||
/// statement wrong.
|
||||
///
|
||||
/// `SQLITE_NOTADB` is included because it is what a truncated or overwritten
|
||||
/// catalog produces — SQLite cannot read the header, so it declines to call it
|
||||
/// a database at all. To a user those are the same accident, and the same two
|
||||
/// offers answer both.
|
||||
///
|
||||
/// Deliberately *not* included: `SQLITE_CANTOPEN` (a missing file, which
|
||||
/// `Connection::open` fixes by creating one), `SQLITE_BUSY`, and
|
||||
/// `SQLITE_IOERR` — a failing disk or a dropped network mount is a different
|
||||
/// problem, and telling the user to rebuild their index would be a wrong
|
||||
/// answer delivered confidently.
|
||||
fn is_corruption(e: &rusqlite::Error) -> bool {
|
||||
matches!(
|
||||
e.sqlite_error_code(),
|
||||
Some(rusqlite::ErrorCode::DatabaseCorrupt) | Some(rusqlite::ErrorCode::NotADatabase)
|
||||
)
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
@@ -1,871 +0,0 @@
|
||||
//! TRACES: FR-CAT-3 | NFR-ARCH-2 | FR-PLAT-AND-3
|
||||
//! The background work queue.
|
||||
//!
|
||||
//! Jobs live in the catalog, so they survive process death — routine on
|
||||
//! Android rather than exceptional (FR-PLAT-AND-3). Two properties carry the
|
||||
//! design:
|
||||
//!
|
||||
//! - **Coalescing.** `UNIQUE(kind, subject_id)` makes enqueueing idempotent,
|
||||
//! so every code path that notices a change can just enqueue and let the
|
||||
//! table absorb the redundancy.
|
||||
//! - **Priority shared with the GPU scheduler** (ARCH §5.3), so one notion of
|
||||
//! urgency governs the whole app and visible work always preempts bulk work.
|
||||
//!
|
||||
//! Nothing in this file runs a job. [`crate::runner`] is the other half — the
|
||||
//! one that claims from this table, does the work through a handler, and
|
||||
//! reports back. Worth knowing because for a long time it did not exist: every
|
||||
//! producer called [`enqueue`] and nothing ever called [`claim_next`], so the
|
||||
//! table only ever grew.
|
||||
|
||||
use rusqlite::{Connection, OptionalExtension};
|
||||
|
||||
use crate::error::CatalogError;
|
||||
|
||||
/// What a job does.
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
#[repr(i64)]
|
||||
pub enum JobKind {
|
||||
/// Recursive incremental scan from a folder (§scan).
|
||||
ScanFolder = 0,
|
||||
/// Promote an image from stat-only to full EXIF.
|
||||
ExtractMetadata = 1,
|
||||
/// Build or rebuild a thumbnail.
|
||||
Thumbnail = 2,
|
||||
/// A sidecar on disk is newer than what the catalog read.
|
||||
ReadSidecar = 3,
|
||||
/// Flush a local edit to its sidecar. Debounced, never per slider tick.
|
||||
WriteSidecar = 4,
|
||||
/// Whole-file hash. On demand only — import dedup, reconnect-by-hash.
|
||||
ContentHash = 5,
|
||||
/// Range-extract an embedded preview from a remote file (FR-NC-3).
|
||||
FetchPreview = 6,
|
||||
/// Fetch a full original: pinned by rule, or explicitly asked for.
|
||||
FetchOriginal = 7,
|
||||
/// Detect and embed the faces in one image (FR-CULL-8).
|
||||
///
|
||||
/// One job does both, rather than splitting them: the proxy is already
|
||||
/// decoded and in memory, and the natural unit of resumable work is one
|
||||
/// photograph. Splitting would double the queue's row count for nothing.
|
||||
///
|
||||
/// Runs against the proxy tier, never a full decode — a library that has
|
||||
/// been browsed has already paid for its proxies, so face indexing adds no
|
||||
/// RAW decodes that were not already happening.
|
||||
DetectFaces = 8,
|
||||
}
|
||||
|
||||
impl JobKind {
|
||||
/// Every kind, so code that has to enumerate them cannot quietly miss one
|
||||
/// that was added later. A `match` would catch that; a hand-written array
|
||||
/// at each call site would not.
|
||||
pub const ALL: [JobKind; 9] = [
|
||||
JobKind::ScanFolder,
|
||||
JobKind::ExtractMetadata,
|
||||
JobKind::Thumbnail,
|
||||
JobKind::ReadSidecar,
|
||||
JobKind::WriteSidecar,
|
||||
JobKind::ContentHash,
|
||||
JobKind::FetchPreview,
|
||||
JobKind::FetchOriginal,
|
||||
JobKind::DetectFaces,
|
||||
];
|
||||
|
||||
fn from_i64(v: i64) -> Option<Self> {
|
||||
Some(match v {
|
||||
0 => JobKind::ScanFolder,
|
||||
1 => JobKind::ExtractMetadata,
|
||||
2 => JobKind::Thumbnail,
|
||||
3 => JobKind::ReadSidecar,
|
||||
4 => JobKind::WriteSidecar,
|
||||
5 => JobKind::ContentHash,
|
||||
6 => JobKind::FetchPreview,
|
||||
7 => JobKind::FetchOriginal,
|
||||
8 => JobKind::DetectFaces,
|
||||
_ => return None,
|
||||
})
|
||||
}
|
||||
|
||||
/// Whether this job transfers over the network, and so is subject to the
|
||||
/// metered-connection and charging constraints in FR-NC-6.
|
||||
pub fn is_network(self) -> bool {
|
||||
matches!(self, JobKind::FetchPreview | JobKind::FetchOriginal)
|
||||
}
|
||||
|
||||
/// Whether `subject_id` names a row in `images`.
|
||||
///
|
||||
/// Every kind but one is per-photograph. `ScanFolder`'s subject is a
|
||||
/// *folder*, and the two id spaces are unrelated — so anything that joins
|
||||
/// `subject_id` against `images` has to exclude it, or it will read one
|
||||
/// table's ids as another's and act on the answer.
|
||||
pub fn subject_is_image(self) -> bool {
|
||||
!matches!(self, JobKind::ScanFolder)
|
||||
}
|
||||
}
|
||||
|
||||
/// Scheduling class, matching the GPU tile scheduler (ARCH §5.3).
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)]
|
||||
#[repr(i64)]
|
||||
pub enum Priority {
|
||||
/// Bulk work: metadata sweeps, rule-driven fetches, hashing.
|
||||
Background = 0,
|
||||
/// Just outside the viewport; the next image in culling.
|
||||
Prefetch = 1,
|
||||
/// Visible cells, and the image currently open.
|
||||
///
|
||||
/// Strictly preempts background work. Without this, scrolling during a
|
||||
/// bulk thumbnail pass misses its frame budget — the common case, not an
|
||||
/// edge case (NFR-ARCH-2).
|
||||
Interactive = 2,
|
||||
}
|
||||
|
||||
impl Priority {
|
||||
/// Read back from the stored column.
|
||||
///
|
||||
/// An unrecognised value reads as `Background` rather than failing: a
|
||||
/// priority is a hint about ordering, and refusing to run a job because
|
||||
/// its urgency is spelled oddly would be a worse answer than running it
|
||||
/// last.
|
||||
fn from_i64(v: i64) -> Self {
|
||||
match v {
|
||||
2 => Priority::Interactive,
|
||||
1 => Priority::Prefetch,
|
||||
_ => Priority::Background,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Lifecycle state.
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
#[repr(i64)]
|
||||
pub enum JobState {
|
||||
Pending = 0,
|
||||
Running = 1,
|
||||
Failed = 2,
|
||||
}
|
||||
|
||||
/// A job ready to run.
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct Job {
|
||||
pub id: i64,
|
||||
pub kind: JobKind,
|
||||
pub subject_id: Option<i64>,
|
||||
pub priority: Priority,
|
||||
pub attempts: i64,
|
||||
pub payload: Option<String>,
|
||||
}
|
||||
|
||||
/// Give up after this many attempts and attach the error to the subject.
|
||||
///
|
||||
/// One corrupt file must not stall the queue behind endless retries
|
||||
/// (FR-RAW-4).
|
||||
pub const MAX_ATTEMPTS: i64 = 5;
|
||||
|
||||
/// Backoff before retrying a failed job, in seconds.
|
||||
///
|
||||
/// Exponential, capped — a server that is down for an hour should not be
|
||||
/// retried every second, and a transient decode failure should not wait an
|
||||
/// hour.
|
||||
pub fn backoff_seconds(attempts: i64) -> i64 {
|
||||
const CAP: i64 = 300;
|
||||
match attempts {
|
||||
a if a <= 0 => 0,
|
||||
a if a >= 9 => CAP,
|
||||
a => (1i64 << (a - 1)).min(CAP),
|
||||
}
|
||||
}
|
||||
|
||||
/// Enqueue work, coalescing with any identical pending job.
|
||||
///
|
||||
/// Re-requesting at a higher priority *promotes* the existing row rather than
|
||||
/// duplicating it, which is what lets the grid shout "this one is visible now"
|
||||
/// about a job already queued in the background.
|
||||
pub fn enqueue(
|
||||
conn: &Connection,
|
||||
kind: JobKind,
|
||||
subject_id: Option<i64>,
|
||||
priority: Priority,
|
||||
payload: Option<&str>,
|
||||
) -> Result<(), CatalogError> {
|
||||
conn.execute(
|
||||
"INSERT INTO jobs(kind, subject_id, priority, state, payload)
|
||||
VALUES (?1, ?2, ?3, 0, ?4)
|
||||
ON CONFLICT(kind, subject_id) DO UPDATE SET
|
||||
priority = max(jobs.priority, excluded.priority),
|
||||
-- A job that failed and is being re-requested deserves a fresh
|
||||
-- start: the file may well have changed since it failed.
|
||||
state = CASE WHEN jobs.state = 2 THEN 0 ELSE jobs.state END,
|
||||
attempts = CASE WHEN jobs.state = 2 THEN 0 ELSE jobs.attempts END,
|
||||
not_before = CASE WHEN jobs.state = 2 THEN 0 ELSE jobs.not_before END",
|
||||
rusqlite::params![kind as i64, subject_id, priority as i64, payload],
|
||||
)?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Claim the next runnable job, highest priority first.
|
||||
///
|
||||
/// `now` is passed rather than read from the clock so backoff is testable.
|
||||
pub fn claim_next(conn: &Connection, now: i64) -> Result<Option<Job>, CatalogError> {
|
||||
claim(conn, now, None)
|
||||
}
|
||||
|
||||
/// Claim the next runnable job of one of `kinds`.
|
||||
///
|
||||
/// What lets a runner take only the work it can actually do. A device with no
|
||||
/// connector must leave `FetchOriginal` rows alone rather than claim them and
|
||||
/// fail them five times each with backoff; a runner gated onto an unmetered
|
||||
/// network (FR-NC-6) passes the local kinds only and leaves the transfers
|
||||
/// where they are. Neither is expressible by filtering *after* a claim,
|
||||
/// because the claim has already marked the row `Running`.
|
||||
///
|
||||
/// An empty list claims nothing, which is the honest reading of "there is
|
||||
/// nothing this worker can do".
|
||||
pub fn claim_next_matching(
|
||||
conn: &Connection,
|
||||
now: i64,
|
||||
kinds: &[JobKind],
|
||||
) -> Result<Option<Job>, CatalogError> {
|
||||
if kinds.is_empty() {
|
||||
return Ok(None);
|
||||
}
|
||||
claim(conn, now, Some(kinds))
|
||||
}
|
||||
|
||||
/// The claim, as one statement.
|
||||
///
|
||||
/// # Why this is not a transaction around a read and a write
|
||||
///
|
||||
/// It used to be, and under two connections that is not safe in the way it
|
||||
/// looks. A deferred transaction takes a read lock for the `SELECT` and only
|
||||
/// tries to upgrade at the `UPDATE`; with WAL, a second worker that read the
|
||||
/// same snapshot gets `SQLITE_BUSY_SNAPSHOT` on its write — an error a busy
|
||||
/// handler cannot retry away, because the fix is to roll back and start over.
|
||||
/// So the queue was correct only in the sense that the loser failed loudly.
|
||||
///
|
||||
/// `UPDATE ... WHERE id = (SELECT ...) RETURNING` is one statement, so it is
|
||||
/// one implicit transaction that takes the write lock immediately. Two workers
|
||||
/// serialise, the loser waits out its busy timeout rather than erroring, and
|
||||
/// neither can see a row the other is already holding.
|
||||
fn claim(
|
||||
conn: &Connection,
|
||||
now: i64,
|
||||
kinds: Option<&[JobKind]>,
|
||||
) -> Result<Option<Job>, CatalogError> {
|
||||
// `now` first, then the kinds, matching the order the placeholders appear
|
||||
// in the text below.
|
||||
let mut args: Vec<i64> = vec![now];
|
||||
let filter = match kinds {
|
||||
None => String::new(),
|
||||
Some(kinds) => {
|
||||
// Built from the kind *count*, never from anything a user typed —
|
||||
// the same discipline `collections::descendants` uses, since
|
||||
// `carray` is not compiled in.
|
||||
let placeholders = std::iter::repeat_n("?", kinds.len())
|
||||
.collect::<Vec<_>>()
|
||||
.join(",");
|
||||
args.extend(kinds.iter().map(|k| *k as i64));
|
||||
format!(" AND kind IN ({placeholders})")
|
||||
}
|
||||
};
|
||||
|
||||
let sql = format!(
|
||||
"UPDATE jobs
|
||||
SET state = 1, attempts = attempts + 1
|
||||
WHERE id = (SELECT id
|
||||
FROM jobs
|
||||
WHERE state = 0 AND not_before <= ?{filter}
|
||||
ORDER BY priority DESC, id ASC
|
||||
LIMIT 1)
|
||||
RETURNING id, kind, subject_id, priority, attempts, payload"
|
||||
);
|
||||
|
||||
let claimed = conn
|
||||
.query_row(&sql, rusqlite::params_from_iter(args.iter()), |r| {
|
||||
Ok((
|
||||
r.get::<_, i64>(0)?,
|
||||
r.get::<_, i64>(1)?,
|
||||
r.get::<_, Option<i64>>(2)?,
|
||||
r.get::<_, i64>(3)?,
|
||||
r.get::<_, i64>(4)?,
|
||||
r.get::<_, Option<String>>(5)?,
|
||||
))
|
||||
})
|
||||
.optional()?;
|
||||
|
||||
let Some((id, kind, subject_id, priority, attempts, payload)) = claimed else {
|
||||
return Ok(None);
|
||||
};
|
||||
|
||||
let Some(kind) = JobKind::from_i64(kind) else {
|
||||
// A row written by a build that knows a kind this one does not — a
|
||||
// downgrade, or a catalog synced from a newer device. Running it as
|
||||
// some other kind would be worse than not running it, so it is parked
|
||||
// where the next claim will not see it again.
|
||||
//
|
||||
// Answering `None` understates what is queued for one pass. The
|
||||
// alternative is a loop that keeps claiming the same unreadable row.
|
||||
abandon(conn, id, &format!("unknown job kind {kind}"))?;
|
||||
return Ok(None);
|
||||
};
|
||||
|
||||
Ok(Some(Job {
|
||||
id,
|
||||
kind,
|
||||
subject_id,
|
||||
priority: Priority::from_i64(priority),
|
||||
attempts,
|
||||
payload,
|
||||
}))
|
||||
}
|
||||
|
||||
/// Job finished successfully.
|
||||
pub fn complete(conn: &Connection, id: i64) -> Result<(), CatalogError> {
|
||||
conn.execute("DELETE FROM jobs WHERE id = ?1", [id])?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Job failed. Reschedules with backoff, or gives up past [`MAX_ATTEMPTS`].
|
||||
pub fn fail(conn: &Connection, job: &Job, now: i64, err: &str) -> Result<(), CatalogError> {
|
||||
if job.attempts >= MAX_ATTEMPTS {
|
||||
abandon(conn, job.id, err)
|
||||
} else {
|
||||
conn.execute(
|
||||
"UPDATE jobs SET state = 0, not_before = ?2, last_error = ?3 WHERE id = ?1",
|
||||
rusqlite::params![job.id, now + backoff_seconds(job.attempts), err],
|
||||
)?;
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
/// Give up on a job now, with no further retries.
|
||||
///
|
||||
/// For failures a retry cannot fix — the subject is gone, the payload is
|
||||
/// unreadable, the format is one this build does not know. Walking the whole
|
||||
/// retry ladder to reach a conclusion the first attempt already reached costs
|
||||
/// five wakeups and five backoffs per photograph, which on a phone is the
|
||||
/// difference the user notices.
|
||||
///
|
||||
/// The row is kept rather than deleted, because "this file failed and here is
|
||||
/// why" is something the user is entitled to see (NFR-ARCH-4).
|
||||
pub fn abandon(conn: &Connection, id: i64, err: &str) -> Result<(), CatalogError> {
|
||||
conn.execute(
|
||||
"UPDATE jobs SET state = 2, last_error = ?2 WHERE id = ?1",
|
||||
rusqlite::params![id, err],
|
||||
)?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Put a claimed job back exactly as it was found.
|
||||
///
|
||||
/// For a worker that is being stopped rather than a job that is going wrong:
|
||||
/// the platform revoked the slot, the user left the screen. The attempt the
|
||||
/// claim consumed is given back, because nothing was learned about the file —
|
||||
/// without that, five backgroundings in a row would mark good work as failed.
|
||||
///
|
||||
/// Guarded on the claim still being *this* claim. There is no owner column, so
|
||||
/// `attempts` stands in for one: it is bumped by every claim, so the row only
|
||||
/// still reads `state = 1` with the caller's own attempt number while nobody
|
||||
/// else has taken it since. A worker that comes back after the recovery pass
|
||||
/// handed its job to someone else therefore changes nothing, rather than
|
||||
/// releasing a job another worker is in the middle of.
|
||||
pub fn release(conn: &Connection, job: &Job) -> Result<(), CatalogError> {
|
||||
conn.execute(
|
||||
"UPDATE jobs SET state = 0, attempts = max(0, attempts - 1)
|
||||
WHERE id = ?1 AND state = 1 AND attempts = ?2",
|
||||
rusqlite::params![job.id, job.attempts],
|
||||
)?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Recover jobs orphaned by process death.
|
||||
///
|
||||
/// A row left `Running` has no owner — the process that claimed it is gone.
|
||||
/// Called at startup, before any worker begins (FR-PLAT-AND-3).
|
||||
///
|
||||
/// The attempt the dead claim consumed is deliberately *not* refunded. A job
|
||||
/// that takes the process down with it is indistinguishable from one that
|
||||
/// fails, and the attempt counter is the only evidence that survives a death —
|
||||
/// without it a poison-pill job is reclaimed and re-run forever.
|
||||
pub fn recover_orphaned(conn: &Connection) -> Result<usize, CatalogError> {
|
||||
let n = conn.execute("UPDATE jobs SET state = 0 WHERE state = 1", [])?;
|
||||
Ok(n)
|
||||
}
|
||||
|
||||
/// Delete jobs whose photograph is gone.
|
||||
///
|
||||
/// Coalescing keeps the table one row per unit of work, but nothing shrinks it
|
||||
/// when the work stops existing: a library that has been culled carries a
|
||||
/// thumbnail job for every photograph deleted since the last time anything
|
||||
/// looked. Each one would be claimed, run, and failed five times.
|
||||
///
|
||||
/// Only kinds whose subject really is an image ([`JobKind::subject_is_image`])
|
||||
/// are considered — `ScanFolder`'s subject is a folder id, and joining it
|
||||
/// against `images` would delete jobs by coincidence of numbering.
|
||||
///
|
||||
/// There is no foreign key to do this instead. `jobs.subject_id` deliberately
|
||||
/// references nothing: it means different tables for different kinds, and a
|
||||
/// constraint that is right for eight of nine kinds is not a constraint.
|
||||
pub fn reap_orphan_subjects(conn: &Connection) -> Result<usize, CatalogError> {
|
||||
let kinds: Vec<i64> = JobKind::ALL
|
||||
.iter()
|
||||
.filter(|k| k.subject_is_image())
|
||||
.map(|k| *k as i64)
|
||||
.collect();
|
||||
let placeholders = std::iter::repeat_n("?", kinds.len())
|
||||
.collect::<Vec<_>>()
|
||||
.join(",");
|
||||
|
||||
let n = conn.execute(
|
||||
&format!(
|
||||
"DELETE FROM jobs
|
||||
WHERE subject_id IS NOT NULL
|
||||
AND kind IN ({placeholders})
|
||||
AND NOT EXISTS (SELECT 1 FROM images WHERE images.id = jobs.subject_id)"
|
||||
),
|
||||
rusqlite::params_from_iter(kinds.iter()),
|
||||
)?;
|
||||
Ok(n)
|
||||
}
|
||||
|
||||
/// How much is left, by state.
|
||||
///
|
||||
/// One query rather than a listing, because the caller is a progress line: a
|
||||
/// foreground service's notification has to say how much remains without
|
||||
/// reading a hundred thousand rows to find out (FR-PLAT-AND-4).
|
||||
#[derive(Debug, Default, Clone, Copy, PartialEq, Eq)]
|
||||
pub struct Counts {
|
||||
/// Claimable now or after a backoff.
|
||||
pub pending: usize,
|
||||
/// Claimed by someone. After a clean start that is a live worker; before
|
||||
/// [`recover_orphaned`] it is a dead one.
|
||||
pub running: usize,
|
||||
/// Given up on, and kept so the user can see what failed and why.
|
||||
pub failed: usize,
|
||||
}
|
||||
|
||||
impl Counts {
|
||||
/// Work that is still going to happen.
|
||||
pub fn outstanding(&self) -> usize {
|
||||
self.pending + self.running
|
||||
}
|
||||
}
|
||||
|
||||
/// Count the queue by state.
|
||||
pub fn counts(conn: &Connection) -> Result<Counts, CatalogError> {
|
||||
// `sum` over no rows is NULL, not 0 — an empty queue would otherwise fail
|
||||
// to convert rather than counting nothing.
|
||||
let (pending, running, failed) = conn.query_row(
|
||||
"SELECT sum(state = 0), sum(state = 1), sum(state = 2) FROM jobs",
|
||||
[],
|
||||
|r| {
|
||||
Ok((
|
||||
r.get::<_, Option<i64>>(0)?,
|
||||
r.get::<_, Option<i64>>(1)?,
|
||||
r.get::<_, Option<i64>>(2)?,
|
||||
))
|
||||
},
|
||||
)?;
|
||||
Ok(Counts {
|
||||
pending: pending.unwrap_or(0) as usize,
|
||||
running: running.unwrap_or(0) as usize,
|
||||
failed: failed.unwrap_or(0) as usize,
|
||||
})
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use crate::schema;
|
||||
|
||||
fn db() -> Connection {
|
||||
let c = Connection::open_in_memory().unwrap();
|
||||
schema::configure(&c).unwrap();
|
||||
schema::migrate(&c).unwrap();
|
||||
c
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn repeated_enqueue_coalesces() {
|
||||
let c = db();
|
||||
for _ in 0..10 {
|
||||
enqueue(&c, JobKind::Thumbnail, Some(1), Priority::Background, None).unwrap();
|
||||
}
|
||||
let n: i64 = c
|
||||
.query_row("SELECT count(*) FROM jobs", [], |r| r.get(0))
|
||||
.unwrap();
|
||||
assert_eq!(n, 1);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn re_enqueueing_at_higher_priority_promotes() {
|
||||
let c = db();
|
||||
enqueue(&c, JobKind::Thumbnail, Some(1), Priority::Background, None).unwrap();
|
||||
// The grid scrolls this image into view.
|
||||
enqueue(&c, JobKind::Thumbnail, Some(1), Priority::Interactive, None).unwrap();
|
||||
|
||||
let p: i64 = c
|
||||
.query_row("SELECT priority FROM jobs", [], |r| r.get(0))
|
||||
.unwrap();
|
||||
assert_eq!(p, Priority::Interactive as i64);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn priority_never_regresses() {
|
||||
let c = db();
|
||||
enqueue(&c, JobKind::Thumbnail, Some(1), Priority::Interactive, None).unwrap();
|
||||
// A background sweep must not demote work the user is waiting on.
|
||||
enqueue(&c, JobKind::Thumbnail, Some(1), Priority::Background, None).unwrap();
|
||||
|
||||
let p: i64 = c
|
||||
.query_row("SELECT priority FROM jobs", [], |r| r.get(0))
|
||||
.unwrap();
|
||||
assert_eq!(p, Priority::Interactive as i64);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn claim_takes_highest_priority_first() {
|
||||
let c = db();
|
||||
enqueue(&c, JobKind::Thumbnail, Some(1), Priority::Background, None).unwrap();
|
||||
enqueue(&c, JobKind::Thumbnail, Some(2), Priority::Interactive, None).unwrap();
|
||||
enqueue(&c, JobKind::Thumbnail, Some(3), Priority::Prefetch, None).unwrap();
|
||||
|
||||
let first = claim_next(&c, 0).unwrap().unwrap();
|
||||
assert_eq!(first.subject_id, Some(2));
|
||||
let second = claim_next(&c, 0).unwrap().unwrap();
|
||||
assert_eq!(second.subject_id, Some(3));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_claimed_job_is_not_claimed_twice() {
|
||||
let c = db();
|
||||
enqueue(&c, JobKind::Thumbnail, Some(1), Priority::Background, None).unwrap();
|
||||
assert!(claim_next(&c, 0).unwrap().is_some());
|
||||
assert!(claim_next(&c, 0).unwrap().is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn failure_backs_off_then_becomes_claimable_again() {
|
||||
let c = db();
|
||||
enqueue(
|
||||
&c,
|
||||
JobKind::FetchPreview,
|
||||
Some(1),
|
||||
Priority::Background,
|
||||
None,
|
||||
)
|
||||
.unwrap();
|
||||
let job = claim_next(&c, 100).unwrap().unwrap();
|
||||
fail(&c, &job, 100, "network down").unwrap();
|
||||
|
||||
// Still backing off.
|
||||
assert!(claim_next(&c, 100).unwrap().is_none());
|
||||
// Past the backoff.
|
||||
assert!(claim_next(&c, 100 + backoff_seconds(job.attempts))
|
||||
.unwrap()
|
||||
.is_some());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_persistently_failing_job_stops_retrying() {
|
||||
let c = db();
|
||||
enqueue(
|
||||
&c,
|
||||
JobKind::ExtractMetadata,
|
||||
Some(1),
|
||||
Priority::Background,
|
||||
None,
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
let mut now = 0;
|
||||
for _ in 0..MAX_ATTEMPTS {
|
||||
let job = claim_next(&c, now).unwrap().expect("should be claimable");
|
||||
fail(&c, &job, now, "corrupt file").unwrap();
|
||||
now += backoff_seconds(job.attempts);
|
||||
}
|
||||
|
||||
// One corrupt file must not stall the queue forever (FR-RAW-4).
|
||||
assert!(claim_next(&c, now + 100_000).unwrap().is_none());
|
||||
let state: i64 = c
|
||||
.query_row("SELECT state FROM jobs", [], |r| r.get(0))
|
||||
.unwrap();
|
||||
assert_eq!(state, JobState::Failed as i64);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn re_requesting_a_failed_job_gives_it_a_fresh_start() {
|
||||
let c = db();
|
||||
enqueue(&c, JobKind::Thumbnail, Some(1), Priority::Background, None).unwrap();
|
||||
let mut now = 0;
|
||||
for _ in 0..MAX_ATTEMPTS {
|
||||
let job = claim_next(&c, now).unwrap().unwrap();
|
||||
fail(&c, &job, now, "boom").unwrap();
|
||||
now += backoff_seconds(job.attempts);
|
||||
}
|
||||
// The file changed on disk, so the old failure says nothing about it.
|
||||
enqueue(&c, JobKind::Thumbnail, Some(1), Priority::Interactive, None).unwrap();
|
||||
let job = claim_next(&c, now).unwrap().expect("retryable again");
|
||||
assert_eq!(job.attempts, 1);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn orphaned_jobs_return_to_pending_on_restart() {
|
||||
let c = db();
|
||||
enqueue(&c, JobKind::Thumbnail, Some(1), Priority::Background, None).unwrap();
|
||||
claim_next(&c, 0).unwrap().unwrap();
|
||||
// Process dies here. Android does this routinely.
|
||||
assert_eq!(recover_orphaned(&c).unwrap(), 1);
|
||||
assert!(claim_next(&c, 0).unwrap().is_some());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn backoff_grows_then_caps() {
|
||||
assert_eq!(backoff_seconds(0), 0);
|
||||
assert_eq!(backoff_seconds(1), 1);
|
||||
assert_eq!(backoff_seconds(3), 4);
|
||||
assert_eq!(backoff_seconds(100), 300);
|
||||
}
|
||||
|
||||
/// An image row, so a job has a subject that exists.
|
||||
fn image(c: &Connection, id: i64) {
|
||||
c.execute(
|
||||
"INSERT INTO roots(id, kind, label) VALUES (1, 'local', '/lib')
|
||||
ON CONFLICT DO NOTHING",
|
||||
[],
|
||||
)
|
||||
.unwrap();
|
||||
c.execute(
|
||||
"INSERT INTO images(id, root_id, source_ref, added_at) VALUES (?1, 1, ?2, 0)",
|
||||
rusqlite::params![id, format!("/lib/{id}.CR3")],
|
||||
)
|
||||
.unwrap();
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_runner_claims_only_the_kinds_it_names() {
|
||||
// The property a filtered claim exists for: work this worker cannot do
|
||||
// is left untouched — not claimed, not attempted, not failed.
|
||||
let c = db();
|
||||
enqueue(&c, JobKind::Thumbnail, Some(1), Priority::Background, None).unwrap();
|
||||
enqueue(
|
||||
&c,
|
||||
JobKind::FetchOriginal,
|
||||
Some(1),
|
||||
Priority::Interactive,
|
||||
None,
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
// `FetchOriginal` is the higher priority and would be claimed first by
|
||||
// an unfiltered claim. It is not this worker's to take.
|
||||
let job = claim_next_matching(&c, 0, &[JobKind::Thumbnail])
|
||||
.unwrap()
|
||||
.expect("the thumbnail is claimable");
|
||||
assert_eq!(job.kind, JobKind::Thumbnail);
|
||||
assert!(claim_next_matching(&c, 0, &[JobKind::Thumbnail])
|
||||
.unwrap()
|
||||
.is_none());
|
||||
|
||||
let (state, attempts): (i64, i64) = c
|
||||
.query_row(
|
||||
"SELECT state, attempts FROM jobs WHERE kind = ?1",
|
||||
[JobKind::FetchOriginal as i64],
|
||||
|r| Ok((r.get(0)?, r.get(1)?)),
|
||||
)
|
||||
.unwrap();
|
||||
assert_eq!(state, JobState::Pending as i64);
|
||||
assert_eq!(attempts, 0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_worker_that_can_do_nothing_claims_nothing() {
|
||||
let c = db();
|
||||
enqueue(&c, JobKind::Thumbnail, Some(1), Priority::Background, None).unwrap();
|
||||
assert!(claim_next_matching(&c, 0, &[]).unwrap().is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn releasing_a_claim_gives_the_attempt_back() {
|
||||
// A stopped worker has learned nothing about the file, so the claim it
|
||||
// is handing back must cost nothing.
|
||||
let c = db();
|
||||
enqueue(&c, JobKind::Thumbnail, Some(1), Priority::Background, None).unwrap();
|
||||
|
||||
let job = claim_next(&c, 0).unwrap().unwrap();
|
||||
assert_eq!(job.attempts, 1);
|
||||
release(&c, &job).unwrap();
|
||||
|
||||
let again = claim_next(&c, 0).unwrap().expect("claimable again at once");
|
||||
assert_eq!(again.attempts, 1, "the release refunded the first attempt");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn releasing_a_job_someone_else_has_reclaimed_does_nothing() {
|
||||
// `attempts` standing in for an owner column. A worker that comes back
|
||||
// after the recovery pass handed its job to someone else must not
|
||||
// release a claim that is no longer its to release — which would drop
|
||||
// the live worker's job back into the queue to be run twice.
|
||||
let c = db();
|
||||
enqueue(&c, JobKind::Thumbnail, Some(1), Priority::Background, None).unwrap();
|
||||
|
||||
let stale = claim_next(&c, 0).unwrap().unwrap();
|
||||
recover_orphaned(&c).unwrap();
|
||||
let live = claim_next(&c, 0).unwrap().unwrap();
|
||||
assert_eq!(live.attempts, 2);
|
||||
|
||||
release(&c, &stale).unwrap();
|
||||
|
||||
let (state, attempts): (i64, i64) = c
|
||||
.query_row("SELECT state, attempts FROM jobs", [], |r| {
|
||||
Ok((r.get(0)?, r.get(1)?))
|
||||
})
|
||||
.unwrap();
|
||||
assert_eq!(
|
||||
state,
|
||||
JobState::Running as i64,
|
||||
"the live claim still holds"
|
||||
);
|
||||
assert_eq!(attempts, 2, "and its attempt was not refunded for it");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn abandoning_skips_the_whole_retry_ladder() {
|
||||
let c = db();
|
||||
enqueue(&c, JobKind::Thumbnail, Some(1), Priority::Background, None).unwrap();
|
||||
let job = claim_next(&c, 0).unwrap().unwrap();
|
||||
|
||||
abandon(&c, job.id, "not an image this build can read").unwrap();
|
||||
|
||||
assert!(claim_next(&c, 1_000_000).unwrap().is_none());
|
||||
let (state, attempts, err): (i64, i64, String) = c
|
||||
.query_row("SELECT state, attempts, last_error FROM jobs", [], |r| {
|
||||
Ok((r.get(0)?, r.get(1)?, r.get(2)?))
|
||||
})
|
||||
.unwrap();
|
||||
assert_eq!(state, JobState::Failed as i64);
|
||||
assert_eq!(attempts, 1, "one attempt, not MAX_ATTEMPTS");
|
||||
// Kept, not deleted: the user is entitled to see what failed and why.
|
||||
assert!(err.contains("this build can read"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn jobs_for_a_deleted_photograph_are_reaped() {
|
||||
// Coalescing keeps the table one row per unit of work; nothing shrank
|
||||
// it when the work stopped existing.
|
||||
let c = db();
|
||||
image(&c, 1);
|
||||
image(&c, 2);
|
||||
enqueue(&c, JobKind::Thumbnail, Some(1), Priority::Background, None).unwrap();
|
||||
enqueue(&c, JobKind::Thumbnail, Some(2), Priority::Background, None).unwrap();
|
||||
enqueue(
|
||||
&c,
|
||||
JobKind::ExtractMetadata,
|
||||
Some(2),
|
||||
Priority::Background,
|
||||
None,
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
c.execute("DELETE FROM images WHERE id = 2", []).unwrap();
|
||||
assert_eq!(reap_orphan_subjects(&c).unwrap(), 2);
|
||||
|
||||
let left: i64 = c
|
||||
.query_row("SELECT subject_id FROM jobs", [], |r| r.get(0))
|
||||
.unwrap();
|
||||
assert_eq!(left, 1);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_folder_scan_is_not_reaped_by_image_ids() {
|
||||
// `ScanFolder`'s subject is a folder. Joining it against `images`
|
||||
// would delete it whenever the numbering happened not to collide —
|
||||
// which, on a fresh library, is almost always.
|
||||
let c = db();
|
||||
enqueue(
|
||||
&c,
|
||||
JobKind::ScanFolder,
|
||||
Some(1),
|
||||
Priority::Background,
|
||||
Some("/lib/2024"),
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
assert_eq!(reap_orphan_subjects(&c).unwrap(), 0);
|
||||
assert!(claim_next(&c, 0).unwrap().is_some());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_job_kind_from_a_newer_build_is_parked_rather_than_guessed_at() {
|
||||
// A catalog synced from a device running a later build. Running an
|
||||
// unknown kind as some arbitrary known one is worse than not running
|
||||
// it, and the old code silently read every unknown kind as
|
||||
// `ExtractMetadata`.
|
||||
let c = db();
|
||||
c.execute(
|
||||
"INSERT INTO jobs(kind, subject_id, priority, state) VALUES (99, 1, 0, 0)",
|
||||
[],
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
assert!(claim_next(&c, 0).unwrap().is_none());
|
||||
|
||||
let (state, err): (i64, String) = c
|
||||
.query_row("SELECT state, last_error FROM jobs", [], |r| {
|
||||
Ok((r.get(0)?, r.get(1)?))
|
||||
})
|
||||
.unwrap();
|
||||
assert_eq!(state, JobState::Failed as i64);
|
||||
assert!(err.contains("99"), "{err}");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn counts_say_what_is_left() {
|
||||
// What a foreground service's notification is built from: a number,
|
||||
// without reading a hundred thousand rows to find it.
|
||||
let c = db();
|
||||
assert_eq!(counts(&c).unwrap(), Counts::default());
|
||||
|
||||
for id in 1..=3 {
|
||||
enqueue(&c, JobKind::Thumbnail, Some(id), Priority::Background, None).unwrap();
|
||||
}
|
||||
let job = claim_next(&c, 0).unwrap().unwrap();
|
||||
abandon(&c, job.id, "nope").unwrap();
|
||||
claim_next(&c, 0).unwrap().unwrap();
|
||||
|
||||
let n = counts(&c).unwrap();
|
||||
assert_eq!(n.pending, 1);
|
||||
assert_eq!(n.running, 1);
|
||||
assert_eq!(n.failed, 1);
|
||||
assert_eq!(n.outstanding(), 2, "failed work is not outstanding work");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn every_kind_is_in_all() {
|
||||
// `ALL` is what the reap builds its kind filter from, so a kind added
|
||||
// to the enum and forgotten here would quietly stop being reaped.
|
||||
for (i, kind) in JobKind::ALL.iter().enumerate() {
|
||||
assert_eq!(
|
||||
JobKind::from_i64(i as i64),
|
||||
Some(*kind),
|
||||
"ALL is out of step with the discriminants at {i}"
|
||||
);
|
||||
}
|
||||
assert!(JobKind::from_i64(JobKind::ALL.len() as i64).is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn only_a_folder_scan_has_a_non_image_subject() {
|
||||
assert!(!JobKind::ScanFolder.subject_is_image());
|
||||
for kind in JobKind::ALL.iter().filter(|k| **k != JobKind::ScanFolder) {
|
||||
assert!(kind.subject_is_image(), "{kind:?}");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn network_jobs_are_identifiable_for_metered_gating() {
|
||||
// FR-NC-6: transfers respect unmetered-network and charging
|
||||
// constraints; local work must not be gated by them.
|
||||
assert!(JobKind::FetchOriginal.is_network());
|
||||
assert!(JobKind::FetchPreview.is_network());
|
||||
assert!(!JobKind::Thumbnail.is_network());
|
||||
assert!(!JobKind::ExtractMetadata.is_network());
|
||||
}
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
@@ -1,600 +0,0 @@
|
||||
//! TRACES: FR-CAT-2 | FR-CAT-4 | FR-CAT-6 | NFR-P1
|
||||
//! The catalog: a rebuildable index over the library.
|
||||
//!
|
||||
//! Not a source of truth. Sidecars next to the images hold the authoritative
|
||||
//! edit state (ARCH §6.12), and this file is deletable at any time — rebuilt
|
||||
//! by rescanning sources and reading sidecars. That inversion is deliberate:
|
||||
//! darktable maintains both a database and sidecars while achieving the
|
||||
//! reliability of neither.
|
||||
//!
|
||||
//! # What lives here
|
||||
//!
|
||||
//! - [`schema`] — tables and forward-only migrations
|
||||
//! - [`scan`] — incremental discovery that prunes unchanged directories
|
||||
//! - [`walk`] — those decisions driven against real storage, local or SAF
|
||||
//! - [`query`] — selectors compiled to indexed SQL, windowed for the grid
|
||||
//! - [`collections`] — the collection tree and membership the UI edits
|
||||
//! - [`keywords`] — the keyword vocabulary and what it is assigned to
|
||||
//! - [`faces`] — detected faces, the people they belong to, and who said so
|
||||
//! - [`bursts`] — frames that are one moment, grouped so they judge as one
|
||||
//! - [`jobs`] — the durable background work queue
|
||||
//! - [`runner`] — the thing that drains it, driven by whoever owns the thread
|
||||
//! - [`trash`] — soft delete to a folder, then permanent delete
|
||||
//! - [`merge`] / [`sync`] — cross-device merging of collections and keywords
|
||||
//! - [`recovery`] — backups, and the two offers made when this file is damaged
|
||||
//!
|
||||
//! # The one thing everything is designed around
|
||||
//!
|
||||
//! **Work is proportional to what changed, or to what the user is looking at —
|
||||
//! never to library size.** A 50k-image library that has not changed costs one
|
||||
//! metadata probe per folder to verify (§scan), no thumbnails to regenerate
|
||||
//! (§jobs coalescing), and no rule evaluation per grid cell (materialised
|
||||
//! `tier_desired`).
|
||||
|
||||
use std::path::Path;
|
||||
|
||||
use dr_types::{Availability, ImageId};
|
||||
use rusqlite::Connection;
|
||||
|
||||
pub mod bursts;
|
||||
pub mod cache;
|
||||
pub mod collections;
|
||||
pub mod dedup;
|
||||
pub mod error;
|
||||
pub mod face_shard;
|
||||
pub mod faces;
|
||||
pub mod jobs;
|
||||
pub mod keywords;
|
||||
pub mod merge;
|
||||
pub mod query;
|
||||
pub mod rating;
|
||||
pub mod recovery;
|
||||
pub mod runner;
|
||||
pub mod scan;
|
||||
pub mod schema;
|
||||
pub mod sync;
|
||||
pub mod trash;
|
||||
pub mod walk;
|
||||
|
||||
pub use cache::{Budget, Cache, DEFAULT_BUDGET_BYTES};
|
||||
pub use collections::{Collection, CollectionKind, TreeRow};
|
||||
pub use dedup::{seen_by_content, seen_by_metadata, set_content_hash};
|
||||
pub use error::CatalogError;
|
||||
pub use face_shard::{FaceShardStore, SharedFace};
|
||||
pub use faces::{Calibration, DetectedFace, Face, FaceId, FaceUpdate, Person, PersonId};
|
||||
pub use jobs::{Job, JobKind, Priority};
|
||||
pub use keywords::{Coverage, Keyword, KeywordId, SelectionKeyword};
|
||||
pub use merge::MergeReport;
|
||||
pub use query::{Query, Sort};
|
||||
pub use rating::{Judgement, MAX_RATING};
|
||||
pub use recovery::Backup;
|
||||
// Not `runner::Budget`: `cache::Budget` already owns that name here and
|
||||
// means something else entirely (bytes on disk, not jobs in a slot).
|
||||
// Callers spell the work budget `runner::Budget`, where it is unambiguous.
|
||||
pub use runner::{DrainReport, JobHandler, Outcome, Runner};
|
||||
pub use scan::{DirAction, DirState, EntryAction, ScanOutcome};
|
||||
pub use trash::{TrashedImage, TRASH_DIR};
|
||||
pub use walk::{ensure_root, mark_root_offline, scan_root, RootKind, ScanProgress, ScanReport};
|
||||
|
||||
/// One row of the library grid.
|
||||
///
|
||||
/// Exactly what a cell draws and nothing more — no join per cell, and
|
||||
/// availability reads a materialised column rather than evaluating cache rules
|
||||
/// (ARCH §9.5).
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct GridRow {
|
||||
pub id: ImageId,
|
||||
pub name: String,
|
||||
pub availability: Availability,
|
||||
/// UTC seconds. `None` until EXIF has been read.
|
||||
pub captured_at: Option<i64>,
|
||||
/// Minutes east of UTC, for rendering the photographer's local time.
|
||||
pub captured_offset: Option<i32>,
|
||||
/// 0 = nothing, 1 = stat-only, 2 = full EXIF.
|
||||
pub metadata_state: u8,
|
||||
}
|
||||
|
||||
/// A count of images in one time bucket, for the timeline scrubber.
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub struct TimeBucket {
|
||||
/// UTC seconds at the bucket's start.
|
||||
pub start: i64,
|
||||
pub count: u32,
|
||||
}
|
||||
|
||||
/// Time bucket size.
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub enum Granularity {
|
||||
Year,
|
||||
Month,
|
||||
Day,
|
||||
Hour,
|
||||
}
|
||||
|
||||
impl Granularity {
|
||||
/// SQLite `strftime` format that collapses a timestamp to this bucket.
|
||||
///
|
||||
/// Applied to **local** time, not UTC: "everything from 3 August" means
|
||||
/// the photographer's 3 August, which is why `captured_offset` is stored
|
||||
/// alongside the UTC timestamp.
|
||||
/// Public so a caller that must build its own bucketing query — one
|
||||
/// joining collection membership, say — buckets identically to
|
||||
/// [`Catalog::timeline_range`] rather than reimplementing the format.
|
||||
pub fn strftime(self) -> &'static str {
|
||||
match self {
|
||||
Granularity::Year => "%Y",
|
||||
Granularity::Month => "%Y-%m",
|
||||
Granularity::Day => "%Y-%m-%d",
|
||||
Granularity::Hour => "%Y-%m-%dT%H",
|
||||
}
|
||||
}
|
||||
|
||||
/// A sensible bucket size for a span of seconds, so the UI need not guess.
|
||||
///
|
||||
/// # Chosen by how many bars it produces, not by fixed cut-offs
|
||||
///
|
||||
/// This used to be four thresholds on the span, which reads sensibly and
|
||||
/// behaves badly under zoom. Each zoom step halves the span, so the bar
|
||||
/// count halves with it until a threshold is crossed — a fifteen-year
|
||||
/// library went 15 bars, 8, then 46, 23, 11, and finally *6*. Zooming in
|
||||
/// made the picture coarser, which is the opposite of what zooming is for.
|
||||
///
|
||||
/// So the choice is made on the axis's terms: of the four bucket sizes,
|
||||
/// take the one whose bar count comes nearest [`Self::TARGET_BARS`]. The
|
||||
/// count then stays in the same neighbourhood at every zoom level, and
|
||||
/// each step in genuinely shows finer structure rather than the same
|
||||
/// structure drawn wider.
|
||||
///
|
||||
/// Nearest in *ratio*, not in difference: the counts available for a given
|
||||
/// span are orders of magnitude apart — a span is either about 4 years or
|
||||
/// about 48 months — and on a linear measure the larger count always looks
|
||||
/// further away, which would bias every choice towards too few bars.
|
||||
pub fn for_span(seconds: i64) -> Self {
|
||||
Self::for_bucket(seconds.max(1) / Self::TARGET_BARS)
|
||||
}
|
||||
|
||||
/// The calendar unit nearest a bucket of `seconds`, for *labelling* one.
|
||||
///
|
||||
/// Split out from [`Self::for_span`] because the axis no longer buckets by
|
||||
/// calendar unit at all — it divides the visible span into a fixed number
|
||||
/// of equal bins (see `LibrarySettings::timeline_bars`). What is still
|
||||
/// wanted is the unit a bin is closest to, so a bin of about a day is
|
||||
/// labelled as a date and one of about a year as a year. Asked directly
|
||||
/// rather than derived from the span, because the bin count is now the
|
||||
/// user's rather than this module's target.
|
||||
pub fn for_bucket(seconds: i64) -> Self {
|
||||
let seconds = seconds.max(1) as f64;
|
||||
// Finest first, so that when two options are equally far from the
|
||||
// target the finer one wins: `min_by` keeps the first minimum it saw,
|
||||
// and more detail is the better failure.
|
||||
[
|
||||
Granularity::Hour,
|
||||
Granularity::Day,
|
||||
Granularity::Month,
|
||||
Granularity::Year,
|
||||
]
|
||||
.into_iter()
|
||||
.min_by(|a, b| {
|
||||
let cost = |g: Granularity| {
|
||||
// How far off, measured multiplicatively: twice as long and
|
||||
// half as long are equally wrong.
|
||||
//
|
||||
// Deliberately not clamped. A bucket shorter than the unit
|
||||
// scores *worse* the coarser the unit, which is what makes an
|
||||
// hour of photographs pick hourly bars instead of every option
|
||||
// tying at "one bucket" and the coarsest winning.
|
||||
(seconds / g.approx_seconds() as f64).ln().abs()
|
||||
};
|
||||
cost(*a)
|
||||
.partial_cmp(&cost(*b))
|
||||
// Ties cannot arise from real spans, but a NaN would; falling
|
||||
// back to the coarser option keeps the axis drawable.
|
||||
.unwrap_or(std::cmp::Ordering::Equal)
|
||||
})
|
||||
.unwrap_or(Granularity::Day)
|
||||
}
|
||||
|
||||
/// How many bars the timeline wants across its axis.
|
||||
///
|
||||
/// Not a hard count — the bucket sizes are calendar units, so the actual
|
||||
/// number lands where the calendar puts it. It is the figure the choice
|
||||
/// aims at: enough bars that a busy fortnight is visibly busier than a
|
||||
/// quiet one, few enough that each is wide enough to hit with a finger.
|
||||
const TARGET_BARS: i64 = 40;
|
||||
|
||||
/// Nominal length of one bucket, for choosing between them.
|
||||
///
|
||||
/// Approximate on purpose: months and years vary and it does not matter
|
||||
/// here, because this only ranks four options that are a factor of ~12 or
|
||||
/// ~30 apart. The exact boundaries come from `strftime` on the real dates.
|
||||
fn approx_seconds(self) -> i64 {
|
||||
const DAY: i64 = 86_400;
|
||||
match self {
|
||||
Granularity::Year => 365 * DAY,
|
||||
Granularity::Month => 30 * DAY,
|
||||
Granularity::Day => DAY,
|
||||
Granularity::Hour => 3600,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// A connection to the catalog.
|
||||
pub struct Catalog {
|
||||
conn: Connection,
|
||||
}
|
||||
|
||||
impl Catalog {
|
||||
/// Open or create a catalog, migrating it forward if needed.
|
||||
///
|
||||
/// Does **not** verify the file — see [`Self::open_verified`], and
|
||||
/// [`recovery`] for why the check is bound to startup rather than to every
|
||||
/// open. Damage this trips over on the way past is still reported as
|
||||
/// [`CatalogError::Corrupt`] rather than as a stray SQLite error.
|
||||
pub fn open(path: &Path) -> Result<Self, CatalogError> {
|
||||
let conn = Connection::open(path)?;
|
||||
schema::configure(&conn)?;
|
||||
// NFR-R2, and the reason it is *here*: a migration is the one routine
|
||||
// operation that rewrites table structure, so it is the likeliest way
|
||||
// this file becomes unreadable — and afterwards there is no
|
||||
// pre-migration state left to copy. A failure to take the copy is
|
||||
// logged rather than raised: a full disk must not be the thing that
|
||||
// makes a library unopenable.
|
||||
if let Err(e) = recovery::backup_before_migration(&conn, path) {
|
||||
log::warn!("could not back up before migrating: {e}");
|
||||
}
|
||||
let from = schema::migrate(&conn)?;
|
||||
// A migration adds a column; it cannot know what the value should be
|
||||
// for rows that already existed. Backfilling on open is what stops
|
||||
// those rows being silently partial.
|
||||
for (what, n) in schema::backfill(&conn)? {
|
||||
log::info!("backfilled {what} for {n} row(s) (schema was v{from})");
|
||||
}
|
||||
Ok(Catalog { conn })
|
||||
}
|
||||
|
||||
/// TRACES: NFR-R6
|
||||
/// Open a catalog, checking the file first.
|
||||
///
|
||||
/// What startup calls. On [`CatalogError::Corrupt`] the caller has a user
|
||||
/// in front of it and must make the two offers [`recovery`] describes,
|
||||
/// rather than reporting a SQLite message on a banner and carrying on into
|
||||
/// a scan that would write into the damage.
|
||||
///
|
||||
/// Checked *before* opening rather than after, because opening runs
|
||||
/// migrations: a damaged catalog that happens to have an intact header
|
||||
/// would otherwise be migrated — rewriting structure on top of structure
|
||||
/// that is already wrong — before anybody asked whether it was sound.
|
||||
pub fn open_verified(path: &Path) -> Result<Self, CatalogError> {
|
||||
// A catalog that is not there yet is not damaged; `open` creates it.
|
||||
if path.is_file() {
|
||||
recovery::check_file(path)?;
|
||||
}
|
||||
Self::open(path)
|
||||
}
|
||||
|
||||
/// An in-memory catalog, for tests and for a throwaway import preview.
|
||||
pub fn in_memory() -> Result<Self, CatalogError> {
|
||||
let conn = Connection::open_in_memory()?;
|
||||
schema::configure(&conn)?;
|
||||
schema::migrate(&conn)?;
|
||||
schema::backfill(&conn)?;
|
||||
Ok(Catalog { conn })
|
||||
}
|
||||
|
||||
/// Escape hatch for modules that need raw access. Not part of the UI-facing
|
||||
/// surface.
|
||||
pub fn connection(&self) -> &Connection {
|
||||
&self.conn
|
||||
}
|
||||
|
||||
/// How many images match.
|
||||
///
|
||||
/// Returned alongside the first window so the grid can size its scrollbar
|
||||
/// and paint in one round trip.
|
||||
pub fn count(&self, q: &Query, now: i64) -> Result<usize, CatalogError> {
|
||||
let c = query::compile(&q.filter, now);
|
||||
let sql = query::count_sql(&c);
|
||||
let n: i64 =
|
||||
self.conn
|
||||
.query_row(&sql, rusqlite::params_from_iter(c.params.iter()), |r| {
|
||||
r.get(0)
|
||||
})?;
|
||||
Ok(n as usize)
|
||||
}
|
||||
|
||||
/// Fetch one window of results.
|
||||
///
|
||||
/// Never returns the whole catalog: FR-CAT-4 requires memory bounded
|
||||
/// independently of library size.
|
||||
pub fn window(
|
||||
&self,
|
||||
q: &Query,
|
||||
range: std::ops::Range<usize>,
|
||||
now: i64,
|
||||
) -> Result<Vec<GridRow>, CatalogError> {
|
||||
let c = query::compile(&q.filter, now);
|
||||
let sql = query::window_sql(q, &c);
|
||||
|
||||
let mut params = c.params.clone();
|
||||
params.push(rusqlite::types::Value::Integer(range.len() as i64));
|
||||
params.push(rusqlite::types::Value::Integer(range.start as i64));
|
||||
|
||||
let mut stmt = self.conn.prepare(&sql)?;
|
||||
let rows = stmt
|
||||
.query_map(rusqlite::params_from_iter(params.iter()), |r| {
|
||||
let source_ref: String = r.get(1)?;
|
||||
let avail: i64 = r.get(2)?;
|
||||
Ok(GridRow {
|
||||
id: ImageId(r.get::<_, i64>(0)? as u64),
|
||||
name: source_ref
|
||||
.rsplit(['/', ':'])
|
||||
.next()
|
||||
.unwrap_or(&source_ref)
|
||||
.to_string(),
|
||||
availability: decode_availability(avail),
|
||||
captured_at: r.get(3)?,
|
||||
captured_offset: r.get::<_, Option<i64>>(4)?.map(|v| v as i32),
|
||||
metadata_state: r.get::<_, i64>(5)? as u8,
|
||||
})
|
||||
})?
|
||||
.collect::<Result<Vec<_>, _>>()?;
|
||||
Ok(rows)
|
||||
}
|
||||
|
||||
/// Counts per time bucket, for the timeline scrubber.
|
||||
///
|
||||
/// One grouped aggregate over the `images_captured` index — not 50k rows
|
||||
/// handed to the UI to bucket itself.
|
||||
pub fn timeline(
|
||||
&self,
|
||||
q: &Query,
|
||||
g: Granularity,
|
||||
now: i64,
|
||||
) -> Result<Vec<TimeBucket>, CatalogError> {
|
||||
let c = query::compile(&q.filter, now);
|
||||
// Bucketed in local time: captured_offset is minutes east of UTC, and
|
||||
// NULL falls back to UTC rather than dropping the row.
|
||||
let sql = format!(
|
||||
"SELECT min(captured_at) AS start,
|
||||
count(*) AS n
|
||||
FROM images
|
||||
-- A shadowed JPEG is the same frame as its RAW; counting both
|
||||
-- would double every paired shot in the histogram.
|
||||
WHERE {} AND captured_at IS NOT NULL AND shadowed_by IS NULL
|
||||
GROUP BY strftime('{}', captured_at + coalesce(captured_offset, 0) * 60,
|
||||
'unixepoch')
|
||||
ORDER BY start ASC",
|
||||
c.where_sql,
|
||||
g.strftime()
|
||||
);
|
||||
|
||||
let mut stmt = self.conn.prepare(&sql)?;
|
||||
let rows = stmt
|
||||
.query_map(rusqlite::params_from_iter(c.params.iter()), |r| {
|
||||
Ok(TimeBucket {
|
||||
start: r.get(0)?,
|
||||
count: r.get::<_, i64>(1)? as u32,
|
||||
})
|
||||
})?
|
||||
.collect::<Result<Vec<_>, _>>()?;
|
||||
Ok(rows)
|
||||
}
|
||||
|
||||
/// Counts per time bucket, bounded to a date range.
|
||||
///
|
||||
/// What a zoomed timeline needs: [`timeline`](Self::timeline) always spans
|
||||
/// the whole library, so zooming in would return the same coarse buckets
|
||||
/// with the ends cropped rather than finer detail over a narrower span.
|
||||
pub fn timeline_range(
|
||||
&self,
|
||||
q: &Query,
|
||||
g: Granularity,
|
||||
from: i64,
|
||||
to: i64,
|
||||
now: i64,
|
||||
) -> Result<Vec<TimeBucket>, CatalogError> {
|
||||
let c = query::compile(&q.filter, now);
|
||||
let sql = format!(
|
||||
"SELECT min(captured_at) AS start,
|
||||
count(*) AS n
|
||||
FROM images
|
||||
WHERE {} AND captured_at IS NOT NULL AND shadowed_by IS NULL
|
||||
AND captured_at >= ?{} AND captured_at <= ?{}
|
||||
GROUP BY strftime('{}', captured_at + coalesce(captured_offset, 0) * 60,
|
||||
'unixepoch')
|
||||
ORDER BY start ASC",
|
||||
c.where_sql,
|
||||
c.params.len() + 1,
|
||||
c.params.len() + 2,
|
||||
g.strftime()
|
||||
);
|
||||
|
||||
let mut params = c.params.clone();
|
||||
params.push(rusqlite::types::Value::Integer(from));
|
||||
params.push(rusqlite::types::Value::Integer(to));
|
||||
|
||||
let mut stmt = self.conn.prepare(&sql)?;
|
||||
let rows = stmt
|
||||
.query_map(rusqlite::params_from_iter(params.iter()), |r| {
|
||||
Ok(TimeBucket {
|
||||
start: r.get(0)?,
|
||||
count: r.get::<_, i64>(1)? as u32,
|
||||
})
|
||||
})?
|
||||
.collect::<Result<Vec<_>, _>>()?;
|
||||
Ok(rows)
|
||||
}
|
||||
|
||||
/// Merge a downloaded remote catalog's collections into this one.
|
||||
///
|
||||
/// See [`sync`] for why only collections cross over.
|
||||
pub fn merge_remote_catalog(&self, remote: &Path) -> Result<MergeReport, CatalogError> {
|
||||
sync::merge_remote(&self.conn, remote)
|
||||
}
|
||||
|
||||
/// Write a consistent snapshot ready to upload.
|
||||
pub fn snapshot_for_upload(&self, dest: &Path) -> Result<(), CatalogError> {
|
||||
sync::snapshot_for_upload(&self.conn, dest)
|
||||
}
|
||||
}
|
||||
|
||||
fn decode_availability(v: i64) -> Availability {
|
||||
match v {
|
||||
1 => Availability::Preview,
|
||||
2 => Availability::Original,
|
||||
3 => Availability::Offline,
|
||||
_ => Availability::MetadataOnly,
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use dr_types::Selector;
|
||||
|
||||
fn seeded() -> Catalog {
|
||||
let cat = Catalog::in_memory().unwrap();
|
||||
let c = cat.connection();
|
||||
c.execute(
|
||||
"INSERT INTO roots(id, kind, label) VALUES (1, 'local', 'lib')",
|
||||
[],
|
||||
)
|
||||
.unwrap();
|
||||
// Three images across two days, one with no EXIF read yet.
|
||||
for (id, name, captured, state) in [
|
||||
(1i64, "a.CR3", Some(1_000_000i64), 2i64),
|
||||
(2, "b.CR3", Some(1_100_000), 2),
|
||||
(3, "c.CR3", None, 1),
|
||||
] {
|
||||
c.execute(
|
||||
"INSERT INTO images(id, root_id, source_ref, captured_at, metadata_state, added_at)
|
||||
VALUES (?1, 1, ?2, ?3, ?4, 0)",
|
||||
rusqlite::params![id, name, captured, state],
|
||||
)
|
||||
.unwrap();
|
||||
}
|
||||
cat
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn count_and_window_agree() {
|
||||
let cat = seeded();
|
||||
let q = Query::default();
|
||||
assert_eq!(cat.count(&q, 0).unwrap(), 3);
|
||||
assert_eq!(cat.window(&q, 0..10, 0).unwrap().len(), 3);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn window_is_bounded_by_the_requested_range() {
|
||||
// FR-CAT-4: memory independent of catalog size.
|
||||
let cat = seeded();
|
||||
let rows = cat.window(&Query::default(), 0..2, 0).unwrap();
|
||||
assert_eq!(rows.len(), 2);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn paging_covers_every_row_exactly_once() {
|
||||
let cat = seeded();
|
||||
let q = Query::default();
|
||||
let mut seen = Vec::new();
|
||||
for start in (0..3).step_by(2) {
|
||||
seen.extend(cat.window(&q, start..start + 2, 0).unwrap());
|
||||
}
|
||||
let mut ids: Vec<u64> = seen.iter().map(|r| r.id.0).collect();
|
||||
ids.sort_unstable();
|
||||
assert_eq!(ids, vec![1, 2, 3]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn an_image_without_capture_time_sorts_last_not_first() {
|
||||
// Otherwise a freshly scanned library leads with whatever has not been
|
||||
// read yet, which looks like corruption to the user.
|
||||
let cat = seeded();
|
||||
let rows = cat.window(&Query::default(), 0..10, 0).unwrap();
|
||||
assert_eq!(rows.last().unwrap().id, ImageId(3));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn metadata_state_reaches_the_grid() {
|
||||
// The grid needs it to distinguish "no photos on this date" from
|
||||
// "EXIF not read yet" (FR-NC-6c's honesty principle).
|
||||
let cat = seeded();
|
||||
let rows = cat.window(&Query::default(), 0..10, 0).unwrap();
|
||||
let pending = rows.iter().find(|r| r.id == ImageId(3)).unwrap();
|
||||
assert_eq!(pending.metadata_state, 1);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_filter_narrows_the_count() {
|
||||
let cat = seeded();
|
||||
let q = Query {
|
||||
filter: Selector::Text("a.CR3".into()),
|
||||
..Default::default()
|
||||
};
|
||||
assert_eq!(cat.count(&q, 0).unwrap(), 1);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn timeline_buckets_and_skips_unread_images() {
|
||||
let cat = seeded();
|
||||
let buckets = cat
|
||||
.timeline(&Query::default(), Granularity::Day, 0)
|
||||
.unwrap();
|
||||
// Two images with timestamps, one day apart in UTC; the third has no
|
||||
// capture time and cannot be placed on a timeline at all.
|
||||
let total: u32 = buckets.iter().map(|b| b.count).sum();
|
||||
assert_eq!(total, 2);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn timeline_granularity_follows_the_span() {
|
||||
const DAY: i64 = 86_400;
|
||||
// Chosen by how many bars it makes, not by fixed cut-offs — see
|
||||
// `for_span`. Ten years of yearly bars is ten bars, which says almost
|
||||
// nothing about a library; monthly is 122, which is a shape.
|
||||
assert_eq!(Granularity::for_span(10 * 365 * DAY), Granularity::Month);
|
||||
assert_eq!(Granularity::for_span(120 * DAY), Granularity::Day);
|
||||
assert_eq!(Granularity::for_span(10 * DAY), Granularity::Day);
|
||||
assert_eq!(Granularity::for_span(3600), Granularity::Hour);
|
||||
|
||||
// The property the target exists for: zooming in never coarsens the
|
||||
// axis. Under the old thresholds a fifteen-year library went 15 bars,
|
||||
// then 8, then 46, 23, 11 — finer spans drawn with wider bars.
|
||||
let mut span = 15 * 365 * DAY;
|
||||
let mut previous = Granularity::for_span(span).approx_seconds();
|
||||
for _ in 0..10 {
|
||||
span /= 2;
|
||||
let bucket = Granularity::for_span(span).approx_seconds();
|
||||
assert!(
|
||||
bucket <= previous,
|
||||
"halving the span to {span}s coarsened the bucket \
|
||||
from {previous}s to {bucket}s"
|
||||
);
|
||||
previous = bucket;
|
||||
}
|
||||
|
||||
// And a span shorter than any bucket still picks the finest, rather
|
||||
// than every option tying at one bar and the coarsest winning.
|
||||
assert_eq!(Granularity::for_span(60), Granularity::Hour);
|
||||
assert_eq!(Granularity::for_span(1), Granularity::Hour);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn names_are_derived_for_both_paths_and_saf_ids() {
|
||||
let cat = Catalog::in_memory().unwrap();
|
||||
let c = cat.connection();
|
||||
c.execute(
|
||||
"INSERT INTO roots(id, kind, label) VALUES (1, 'saf', 'tree')",
|
||||
[],
|
||||
)
|
||||
.unwrap();
|
||||
c.execute(
|
||||
"INSERT INTO images(id, root_id, source_ref, added_at)
|
||||
VALUES (1, 1, 'primary:DCIM/Camera/IMG_1.CR3', 0)",
|
||||
[],
|
||||
)
|
||||
.unwrap();
|
||||
let rows = cat.window(&Query::default(), 0..10, 0).unwrap();
|
||||
assert_eq!(rows[0].name, "IMG_1.CR3");
|
||||
}
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
@@ -1,508 +0,0 @@
|
||||
//! TRACES: FR-CAT-4 | FR-CAT-6
|
||||
//! Compiling a [`Selector`] into indexed SQL, and windowing the result.
|
||||
//!
|
||||
//! The UI never assembles SQL — it hands over a [`Query`] and receives a
|
||||
//! window. Two properties matter:
|
||||
//!
|
||||
//! 1. **Nothing user-supplied is interpolated into SQL text.** Every value
|
||||
//! binds as a parameter; `LIKE` patterns have their wildcards escaped.
|
||||
//! 2. **Predicates hit indices.** Filtering 50k images must stay interactive
|
||||
//! (FR-CAT-6), which means no expression over a column that would defeat
|
||||
//! its index.
|
||||
|
||||
use dr_types::{Availability, ColourLabel, DateSelector, FlagState, Selector};
|
||||
use rusqlite::types::Value;
|
||||
|
||||
/// What to show, and in what order.
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct Query {
|
||||
pub filter: Selector,
|
||||
pub sort: Sort,
|
||||
pub descending: bool,
|
||||
}
|
||||
|
||||
impl Default for Query {
|
||||
fn default() -> Self {
|
||||
Query {
|
||||
filter: Selector::All,
|
||||
sort: Sort::CapturedAt,
|
||||
descending: true,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub enum Sort {
|
||||
CapturedAt,
|
||||
Added,
|
||||
FileName,
|
||||
Rating,
|
||||
/// Manual order within a collection. Falls back to capture time where the
|
||||
/// query is not scoped to one collection, since position is meaningless
|
||||
/// outside it.
|
||||
CollectionPosition,
|
||||
}
|
||||
|
||||
impl Sort {
|
||||
/// The ORDER BY fragment. Fixed strings — never user input.
|
||||
///
|
||||
/// Capture time sorts NULLs last regardless of direction: an image whose
|
||||
/// EXIF has not been read yet (metadata_state 1) should not lead the grid
|
||||
/// simply because its timestamp is unknown.
|
||||
fn sql(self, descending: bool) -> &'static str {
|
||||
match (self, descending) {
|
||||
(Sort::CapturedAt, false) => {
|
||||
"ORDER BY images.captured_at IS NULL, images.captured_at ASC, images.id ASC"
|
||||
}
|
||||
(Sort::CapturedAt, true) => {
|
||||
"ORDER BY images.captured_at IS NULL, images.captured_at DESC, images.id DESC"
|
||||
}
|
||||
(Sort::Added, false) => "ORDER BY images.added_at ASC, images.id ASC",
|
||||
(Sort::Added, true) => "ORDER BY images.added_at DESC, images.id DESC",
|
||||
(Sort::FileName, false) => "ORDER BY images.source_ref ASC, images.id ASC",
|
||||
(Sort::FileName, true) => "ORDER BY images.source_ref DESC, images.id DESC",
|
||||
(Sort::Rating, false) => "ORDER BY v.rating ASC, images.id ASC",
|
||||
(Sort::Rating, true) => "ORDER BY v.rating DESC, images.id DESC",
|
||||
(Sort::CollectionPosition, false) => {
|
||||
"ORDER BY cm.position IS NULL, cm.position ASC, images.captured_at ASC"
|
||||
}
|
||||
(Sort::CollectionPosition, true) => {
|
||||
"ORDER BY cm.position IS NULL, cm.position DESC, images.captured_at DESC"
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Whether this sort needs the default-version join.
|
||||
fn needs_version(self) -> bool {
|
||||
matches!(self, Sort::Rating)
|
||||
}
|
||||
|
||||
/// Whether this sort needs a collection-membership join.
|
||||
fn needs_membership(self) -> bool {
|
||||
matches!(self, Sort::CollectionPosition)
|
||||
}
|
||||
}
|
||||
|
||||
/// A compiled WHERE clause plus its bound parameters.
|
||||
///
|
||||
/// Kept separate from the statement so `count` and `window` can share one
|
||||
/// compilation.
|
||||
#[derive(Debug, Default)]
|
||||
pub struct Compiled {
|
||||
pub where_sql: String,
|
||||
pub params: Vec<Value>,
|
||||
/// True if the filter depends on capture time, and therefore on EXIF that
|
||||
/// a freshly scanned library may not have read yet. The UI surfaces this
|
||||
/// rather than silently under-reporting.
|
||||
pub needs_capture_time: bool,
|
||||
}
|
||||
|
||||
/// Compile a selector to SQL against the `images` table.
|
||||
///
|
||||
/// `now` is passed rather than read from the clock so a rolling window is
|
||||
/// reproducible in tests and consistent across one query.
|
||||
pub fn compile(filter: &Selector, now: i64) -> Compiled {
|
||||
let mut params = Vec::new();
|
||||
let sql = if filter.is_unfiltered() {
|
||||
"1".to_string()
|
||||
} else {
|
||||
emit(filter, now, &mut params)
|
||||
};
|
||||
Compiled {
|
||||
where_sql: sql,
|
||||
params,
|
||||
needs_capture_time: filter.needs_capture_time(),
|
||||
}
|
||||
}
|
||||
|
||||
fn emit(s: &Selector, now: i64, p: &mut Vec<Value>) -> String {
|
||||
match s {
|
||||
Selector::All => "1".into(),
|
||||
|
||||
Selector::Collection(id) => {
|
||||
p.push(Value::Integer(id.0 as i64));
|
||||
format!(
|
||||
"EXISTS (SELECT 1 FROM collection_members m
|
||||
WHERE m.image_id = images.id AND m.collection_id = ?{})",
|
||||
p.len()
|
||||
)
|
||||
}
|
||||
|
||||
Selector::Folder {
|
||||
root,
|
||||
path,
|
||||
recursive,
|
||||
} => {
|
||||
p.push(Value::Integer(root.0 as i64));
|
||||
let root_ix = p.len();
|
||||
if *recursive {
|
||||
// Prefix match on the folder path. `like_prefix` escapes the
|
||||
// pattern metacharacters, so a folder literally named "50%"
|
||||
// matches itself and not everything.
|
||||
p.push(Value::Text(like_prefix(path)));
|
||||
format!(
|
||||
"images.folder_id IN (
|
||||
SELECT id FROM folders
|
||||
WHERE root_id = ?{root_ix}
|
||||
AND (path = ?{p} OR path LIKE ?{p} || '/%' ESCAPE '\\'))",
|
||||
p = p.len()
|
||||
)
|
||||
} else {
|
||||
p.push(Value::Text(path.clone()));
|
||||
format!(
|
||||
"images.folder_id IN (
|
||||
SELECT id FROM folders WHERE root_id = ?{root_ix} AND path = ?{})",
|
||||
p.len()
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
Selector::DateRange(d) => emit_date(d, now, p),
|
||||
|
||||
Selector::Rating { min } => {
|
||||
p.push(Value::Integer(*min as i64));
|
||||
format!("{} >= ?{}", default_version_scalar("rating"), p.len())
|
||||
}
|
||||
|
||||
Selector::Label(l) => {
|
||||
p.push(Value::Integer(label_code(*l)));
|
||||
format!("{} = ?{}", default_version_scalar("label"), p.len())
|
||||
}
|
||||
|
||||
Selector::Flag(f) => {
|
||||
p.push(Value::Integer(flag_code(*f)));
|
||||
format!("{} = ?{}", default_version_scalar("flag"), p.len())
|
||||
}
|
||||
|
||||
Selector::Keyword(k) => {
|
||||
p.push(Value::Text(k.clone()));
|
||||
format!(
|
||||
"EXISTS (SELECT 1 FROM keywords kw
|
||||
JOIN versions kv ON kv.id = kw.version_id
|
||||
WHERE kv.image_id = images.id AND kw.keyword = ?{})",
|
||||
p.len()
|
||||
)
|
||||
}
|
||||
|
||||
Selector::Camera(c) => {
|
||||
p.push(Value::Text(c.clone()));
|
||||
format!("images.camera = ?{}", p.len())
|
||||
}
|
||||
|
||||
Selector::Lens(l) => {
|
||||
p.push(Value::Text(l.clone()));
|
||||
format!("images.lens = ?{}", p.len())
|
||||
}
|
||||
|
||||
Selector::IsoRange { min, max } => {
|
||||
p.push(Value::Integer(*min as i64));
|
||||
let lo = p.len();
|
||||
p.push(Value::Integer(*max as i64));
|
||||
format!("images.iso BETWEEN ?{lo} AND ?{}", p.len())
|
||||
}
|
||||
|
||||
Selector::Availability(a) => {
|
||||
p.push(Value::Integer(availability_code(*a)));
|
||||
format!("images.availability = ?{}", p.len())
|
||||
}
|
||||
|
||||
Selector::Text(t) => {
|
||||
// Substring over filename and keywords. A LIKE scan is adequate at
|
||||
// 50k; if free text over title and description becomes a real
|
||||
// workflow, FTS5 is the answer and it is additive.
|
||||
p.push(Value::Text(format!("%{}%", escape_like(t))));
|
||||
let ix = p.len();
|
||||
format!(
|
||||
"(images.source_ref LIKE ?{ix} ESCAPE '\\'
|
||||
OR EXISTS (SELECT 1 FROM keywords kw
|
||||
JOIN versions kv ON kv.id = kw.version_id
|
||||
WHERE kv.image_id = images.id
|
||||
AND kw.keyword LIKE ?{ix} ESCAPE '\\'))"
|
||||
)
|
||||
}
|
||||
|
||||
// An empty conjunction is vacuously true; an empty disjunction matches
|
||||
// nothing. Both arise from a UI that lets every term be cleared, and
|
||||
// conflating them would show the whole library when the user meant the
|
||||
// opposite.
|
||||
Selector::All_(v) if v.is_empty() => "1".into(),
|
||||
Selector::Any(v) if v.is_empty() => "0".into(),
|
||||
|
||||
Selector::All_(v) => join(v, " AND ", now, p),
|
||||
Selector::Any(v) => join(v, " OR ", now, p),
|
||||
Selector::Not(inner) => format!("NOT ({})", emit(inner, now, p)),
|
||||
}
|
||||
}
|
||||
|
||||
fn join(items: &[Selector], op: &str, now: i64, p: &mut Vec<Value>) -> String {
|
||||
let parts: Vec<String> = items.iter().map(|s| emit(s, now, p)).collect();
|
||||
format!("({})", parts.join(op))
|
||||
}
|
||||
|
||||
fn emit_date(d: &DateSelector, now: i64, p: &mut Vec<Value>) -> String {
|
||||
match d {
|
||||
DateSelector::Between { from, to } => {
|
||||
p.push(Value::Integer(*from));
|
||||
let lo = p.len();
|
||||
p.push(Value::Integer(*to));
|
||||
// Half-open, so adjacent ranges neither overlap nor gap.
|
||||
format!(
|
||||
"(images.captured_at >= ?{lo} AND images.captured_at < ?{})",
|
||||
p.len()
|
||||
)
|
||||
}
|
||||
DateSelector::Rolling { days } => {
|
||||
let from = now - (*days as i64) * 86_400;
|
||||
p.push(Value::Integer(from));
|
||||
format!("images.captured_at >= ?{}", p.len())
|
||||
}
|
||||
DateSelector::CollectionSpan(id) => {
|
||||
p.push(Value::Integer(id.0 as i64));
|
||||
let ix = p.len();
|
||||
format!(
|
||||
"images.captured_at BETWEEN
|
||||
(SELECT min(i2.captured_at) FROM images i2
|
||||
JOIN collection_members m2 ON m2.image_id = i2.id
|
||||
WHERE m2.collection_id = ?{ix})
|
||||
AND (SELECT max(i2.captured_at) FROM images i2
|
||||
JOIN collection_members m2 ON m2.image_id = i2.id
|
||||
WHERE m2.collection_id = ?{ix})"
|
||||
)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Rating, label, and flag live on the *default* version, not the image.
|
||||
///
|
||||
/// A correlated subquery rather than a join, so these compose inside `OR` and
|
||||
/// `NOT` without the join multiplying rows.
|
||||
fn default_version_scalar(col: &str) -> String {
|
||||
format!(
|
||||
"(SELECT dv.{col} FROM versions dv
|
||||
WHERE dv.image_id = images.id AND dv.is_default = 1 LIMIT 1)"
|
||||
)
|
||||
}
|
||||
|
||||
/// Escape LIKE metacharacters so a literal `%` or `_` in user text matches
|
||||
/// itself. Paired with `ESCAPE '\'` in every LIKE that uses it.
|
||||
fn escape_like(s: &str) -> String {
|
||||
let mut out = String::with_capacity(s.len());
|
||||
for c in s.chars() {
|
||||
if matches!(c, '%' | '_' | '\\') {
|
||||
out.push('\\');
|
||||
}
|
||||
out.push(c);
|
||||
}
|
||||
out
|
||||
}
|
||||
|
||||
fn like_prefix(path: &str) -> String {
|
||||
escape_like(path.trim_end_matches('/'))
|
||||
}
|
||||
|
||||
fn label_code(l: ColourLabel) -> i64 {
|
||||
crate::rating::label_code(l)
|
||||
}
|
||||
|
||||
fn flag_code(f: FlagState) -> i64 {
|
||||
match f {
|
||||
FlagState::Unflagged => 0,
|
||||
FlagState::Pick => 1,
|
||||
FlagState::Reject => 2,
|
||||
}
|
||||
}
|
||||
|
||||
/// The stored form of an availability. Shared with [`crate::walk`], which
|
||||
/// writes the column this reads — two spellings of the same mapping would
|
||||
/// filter for a state nothing ever writes.
|
||||
pub(crate) fn availability_code(a: Availability) -> i64 {
|
||||
match a {
|
||||
Availability::MetadataOnly => 0,
|
||||
Availability::Preview => 1,
|
||||
Availability::Original => 2,
|
||||
Availability::Offline => 3,
|
||||
}
|
||||
}
|
||||
|
||||
/// Build the full SELECT for a window of results.
|
||||
///
|
||||
/// Joins are added only where the sort needs them, so an unsorted-by-rating
|
||||
/// grid query touches one table.
|
||||
pub fn window_sql(q: &Query, compiled: &Compiled) -> String {
|
||||
let mut joins = String::new();
|
||||
if q.sort.needs_version() {
|
||||
joins.push_str(" LEFT JOIN versions v ON v.image_id = images.id AND v.is_default = 1");
|
||||
}
|
||||
if q.sort.needs_membership() {
|
||||
// Only meaningful when the filter scopes to one collection; elsewhere
|
||||
// position is NULL and the sort falls through to capture time.
|
||||
joins.push_str(" LEFT JOIN collection_members cm ON cm.image_id = images.id");
|
||||
}
|
||||
format!(
|
||||
"SELECT images.id, images.source_ref, images.availability, images.captured_at, \
|
||||
images.captured_offset, images.metadata_state \
|
||||
FROM images{joins} WHERE {} {} LIMIT ? OFFSET ?",
|
||||
compiled.where_sql,
|
||||
q.sort.sql(q.descending)
|
||||
)
|
||||
}
|
||||
|
||||
/// Build the COUNT for the same filter.
|
||||
pub fn count_sql(compiled: &Compiled) -> String {
|
||||
format!("SELECT count(*) FROM images WHERE {}", compiled.where_sql)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use dr_types::{CollectionId, RootId};
|
||||
|
||||
#[test]
|
||||
fn unfiltered_compiles_to_a_constant() {
|
||||
let c = compile(&Selector::All, 0);
|
||||
assert_eq!(c.where_sql, "1");
|
||||
assert!(c.params.is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn empty_conjunction_and_disjunction_differ() {
|
||||
// The distinction that decides whether clearing a filter shows
|
||||
// everything or nothing.
|
||||
assert_eq!(compile(&Selector::All_(vec![]), 0).where_sql, "1");
|
||||
assert_eq!(compile(&Selector::Any(vec![]), 0).where_sql, "0");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn values_bind_rather_than_interpolate() {
|
||||
// The injection guard: a hostile keyword must appear in params, never
|
||||
// in SQL text.
|
||||
let evil = "'; DROP TABLE images; --";
|
||||
let c = compile(&Selector::Keyword(evil.into()), 0);
|
||||
assert!(!c.where_sql.contains("DROP"));
|
||||
assert_eq!(c.params, vec![Value::Text(evil.into())]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn like_metacharacters_are_escaped() {
|
||||
// A search for "50%" must not match everything containing "50".
|
||||
let c = compile(&Selector::Text("50%".into()), 0);
|
||||
assert_eq!(c.params, vec![Value::Text("%50\\%%".into())]);
|
||||
assert!(c.where_sql.contains("ESCAPE"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_backslash_in_search_text_is_itself_escaped() {
|
||||
let c = compile(&Selector::Text("a\\b".into()), 0);
|
||||
assert_eq!(c.params, vec![Value::Text("%a\\\\b%".into())]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn rolling_window_resolves_against_supplied_now() {
|
||||
// Passed in rather than read from the clock, so the window is stable
|
||||
// across one query and reproducible in a test.
|
||||
let now = 1_000_000i64;
|
||||
let c = compile(
|
||||
&Selector::DateRange(DateSelector::Rolling { days: 90 }),
|
||||
now,
|
||||
);
|
||||
assert_eq!(c.params, vec![Value::Integer(now - 90 * 86_400)]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn between_is_half_open() {
|
||||
let c = compile(
|
||||
&Selector::DateRange(DateSelector::Between { from: 10, to: 20 }),
|
||||
0,
|
||||
);
|
||||
// Half-open so adjacent day buckets neither overlap nor leave a gap.
|
||||
assert!(c.where_sql.contains(">= ?1"));
|
||||
assert!(c.where_sql.contains("< ?2"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn nested_composition_numbers_parameters_in_order() {
|
||||
let s = Selector::All_(vec![
|
||||
Selector::Rating { min: 4 },
|
||||
Selector::Any(vec![
|
||||
Selector::Camera("X-T5".into()),
|
||||
Selector::Not(Box::new(Selector::Lens("XF 35".into()))),
|
||||
]),
|
||||
]);
|
||||
let c = compile(&s, 0);
|
||||
assert_eq!(
|
||||
c.params,
|
||||
vec![
|
||||
Value::Integer(4),
|
||||
Value::Text("X-T5".into()),
|
||||
Value::Text("XF 35".into()),
|
||||
]
|
||||
);
|
||||
assert!(c.where_sql.contains("?1"));
|
||||
assert!(c.where_sql.contains("?2"));
|
||||
assert!(c.where_sql.contains("?3"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn recursive_folder_matches_the_folder_itself_and_below() {
|
||||
let c = compile(
|
||||
&Selector::Folder {
|
||||
root: RootId(1),
|
||||
path: "2026/08".into(),
|
||||
recursive: true,
|
||||
},
|
||||
0,
|
||||
);
|
||||
// Both branches: the folder's own images and those in subfolders.
|
||||
assert!(c.where_sql.contains("path = ?2"));
|
||||
assert!(c.where_sql.contains("|| '/%'"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn collection_span_binds_its_id_once_and_reuses_it() {
|
||||
let c = compile(
|
||||
&Selector::DateRange(DateSelector::CollectionSpan(CollectionId(7))),
|
||||
0,
|
||||
);
|
||||
assert_eq!(c.params, vec![Value::Integer(7)]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn capture_time_dependency_is_reported() {
|
||||
let c = compile(&Selector::DateRange(DateSelector::Rolling { days: 7 }), 0);
|
||||
assert!(c.needs_capture_time);
|
||||
let c = compile(&Selector::Rating { min: 5 }, 0);
|
||||
assert!(!c.needs_capture_time);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn capture_sort_puts_unknown_timestamps_last_in_both_directions() {
|
||||
// An image whose EXIF has not been read yet must not lead the grid
|
||||
// just because its timestamp is NULL.
|
||||
assert!(Sort::CapturedAt.sql(true).contains("IS NULL"));
|
||||
assert!(Sort::CapturedAt.sql(false).contains("IS NULL"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn window_sql_joins_only_when_the_sort_needs_it() {
|
||||
let c = compile(&Selector::All, 0);
|
||||
let plain = window_sql(
|
||||
&Query {
|
||||
filter: Selector::All,
|
||||
sort: Sort::CapturedAt,
|
||||
descending: true,
|
||||
},
|
||||
&c,
|
||||
);
|
||||
assert!(!plain.contains("JOIN"));
|
||||
|
||||
let rated = window_sql(
|
||||
&Query {
|
||||
filter: Selector::All,
|
||||
sort: Sort::Rating,
|
||||
descending: true,
|
||||
},
|
||||
&c,
|
||||
);
|
||||
assert!(rated.contains("JOIN versions"));
|
||||
}
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
@@ -1,753 +0,0 @@
|
||||
//! TRACES: NFR-R2 | NFR-R6
|
||||
//! What to do once the index is already damaged.
|
||||
//!
|
||||
//! # Why this can be a small module
|
||||
//!
|
||||
//! Because of a property the rest of the catalog was built to keep: the
|
||||
//! catalog is an *index*, not a source of truth (ARCH §6.12, invariant
|
||||
//! §5.2.4). Sidecars beside the images hold the authoritative ratings,
|
||||
//! keywords and edit graphs, for every catalogued image and whether or not a
|
||||
//! remote account exists (FR-CAT-8). So the worst outcome available here is a
|
||||
//! rescan — expensive, but not a loss.
|
||||
//!
|
||||
//! That is the second offer. The first is cheaper and loses nothing at all: a
|
||||
//! backup, restored.
|
||||
//!
|
||||
//! # The one thing a rebuild does not recover
|
||||
//!
|
||||
//! **Collections.** A manual collection is a set of images the user assembled
|
||||
//! by hand and nothing in the filesystem records it (`docs/catalog.md` §8.1) —
|
||||
//! which is the whole reason the catalog file itself syncs. So the two offers
|
||||
//! are not interchangeable, and the interface must not present them as if they
|
||||
//! were: a restore keeps the user's collections, a rebuild does not.
|
||||
//!
|
||||
//! # When the check runs, and when it does not
|
||||
//!
|
||||
//! [`integrity_check`] reads every page of the database. That is affordable
|
||||
//! once, at startup, where a failure has a user in front of it who can answer
|
||||
//! a question — and it is *not* affordable on every [`Catalog::open`], which
|
||||
//! this application does per background task, dozens of times a session. So
|
||||
//! the check is bound to [`Catalog::open_verified`] rather than to `open`,
|
||||
//! and the cheap half of the story — classifying `SQLITE_CORRUPT` and
|
||||
//! `SQLITE_NOTADB` as [`CatalogError::Corrupt`] — happens for free on every
|
||||
//! query through [`crate::error`]'s conversion. A background job that trips
|
||||
//! over the damage first therefore reports the same thing the startup check
|
||||
//! would have.
|
||||
//!
|
||||
//! [`Catalog::open`]: crate::Catalog::open
|
||||
//! [`Catalog::open_verified`]: crate::Catalog::open_verified
|
||||
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::time::{SystemTime, UNIX_EPOCH};
|
||||
|
||||
use rusqlite::Connection;
|
||||
|
||||
use crate::error::CatalogError;
|
||||
use crate::schema;
|
||||
|
||||
/// Directory backups live in, relative to the catalog file.
|
||||
///
|
||||
/// Beside the catalog rather than in the cache directory, and that is the
|
||||
/// point of the choice: this is the copy the user falls back on, and a cache
|
||||
/// is a place the operating system is entitled to empty without asking
|
||||
/// (see `library::data_root` for the same reasoning about sidecars).
|
||||
const BACKUP_DIR: &str = "backups";
|
||||
|
||||
/// How many backups are kept.
|
||||
///
|
||||
/// Small on purpose. A backup is a full copy of a catalog that is tens of
|
||||
/// megabytes at 50k images, and the value of the third-oldest one is close to
|
||||
/// zero: corruption is noticed at the next launch, not months later. What the
|
||||
/// depth buys is protection against backing *up* the damage — if a corrupt
|
||||
/// catalog is copied before anyone notices, the generation behind it is still
|
||||
/// clean.
|
||||
pub const KEEP_BACKUPS: usize = 3;
|
||||
|
||||
/// Suffix given to a catalog that has been set aside as damaged.
|
||||
///
|
||||
/// Kept rather than deleted. It costs disk this application would rather not
|
||||
/// spend, and it is still the right call: `.sqlite` files have been recovered
|
||||
/// by hand before, the user has not consented to a deletion, and NFR-R4's
|
||||
/// instinct — never destroy what the user did not ask you to destroy — does
|
||||
/// not stop applying at the catalog's edge.
|
||||
const DAMAGED_SUFFIX: &str = "damaged";
|
||||
|
||||
/// One kept backup.
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct Backup {
|
||||
pub path: PathBuf,
|
||||
/// UTC seconds at which it was taken, read from the filename rather than
|
||||
/// from the filesystem: a copy, a restore or a sync can rewrite an mtime,
|
||||
/// and then the newest backup is not the one that looks newest.
|
||||
pub taken_at: i64,
|
||||
pub bytes: u64,
|
||||
}
|
||||
|
||||
/// Where backups for `catalog` are kept.
|
||||
pub fn backup_dir(catalog: &Path) -> PathBuf {
|
||||
catalog
|
||||
.parent()
|
||||
.unwrap_or_else(|| Path::new("."))
|
||||
.join(BACKUP_DIR)
|
||||
}
|
||||
|
||||
/// Check the database this connection is attached to.
|
||||
///
|
||||
/// `quick_check` rather than `integrity_check`: the difference is that
|
||||
/// `quick_check` skips verifying that every index agrees with its table, which
|
||||
/// is the expensive half and the half this application least needs — every
|
||||
/// index here is derivable, and `REINDEX` fixes one without anybody being
|
||||
/// asked a question. What is left still reads every page, and catches the
|
||||
/// damage that matters: torn b-trees, a bad freelist, a truncated file.
|
||||
///
|
||||
/// Returns [`CatalogError::Corrupt`] carrying what SQLite said, so the message
|
||||
/// the user sees is the diagnosis rather than a paraphrase of it.
|
||||
pub fn integrity_check(conn: &Connection) -> Result<(), CatalogError> {
|
||||
// The argument caps how many problems are reported. One is enough: the
|
||||
// answer is the same whether the file has one damaged page or nine
|
||||
// hundred, and an unbounded check on a badly damaged file can run for a
|
||||
// very long time producing a list nobody will read.
|
||||
let mut stmt = conn.prepare("PRAGMA quick_check(1)")?;
|
||||
let rows: Vec<String> = stmt
|
||||
.query_map([], |r| r.get(0))?
|
||||
.collect::<Result<Vec<_>, _>>()?;
|
||||
|
||||
// A healthy database answers with the single row "ok".
|
||||
if rows.len() == 1 && rows[0] == "ok" {
|
||||
return Ok(());
|
||||
}
|
||||
Err(CatalogError::Corrupt {
|
||||
detail: rows.join("; "),
|
||||
})
|
||||
}
|
||||
|
||||
/// Check a catalog file that is not currently open.
|
||||
///
|
||||
/// Used before a restore: a backup is only worth swapping in if it is sound,
|
||||
/// and swapping in a second damaged file — leaving the user with no catalog
|
||||
/// and no offer left — is the failure this exists to prevent.
|
||||
pub fn check_file(path: &Path) -> Result<(), CatalogError> {
|
||||
if !path.is_file() {
|
||||
return Err(CatalogError::Io(format!("{} is missing", path.display())));
|
||||
}
|
||||
// Read-write rather than read-only, which reads oddly for a check. A
|
||||
// backup carries the WAL journal mode in its header because it was copied
|
||||
// page-for-page from a WAL database, and SQLite cannot open one read-only
|
||||
// without a shared-memory file it is then not allowed to create. Nothing
|
||||
// here writes; the connection is opened, read, and dropped.
|
||||
let conn = Connection::open(path)?;
|
||||
integrity_check(&conn)
|
||||
}
|
||||
|
||||
/// Take a backup of the open catalog.
|
||||
///
|
||||
/// Returns the file written. Older generations beyond [`KEEP_BACKUPS`] are
|
||||
/// pruned, newest kept.
|
||||
///
|
||||
/// Goes through `crate::sync::copy_to` — SQLite's own backup API after a
|
||||
/// TRUNCATE checkpoint — rather than copying the file. A WAL database is not
|
||||
/// one file, and `fs::copy` of the main file alone would silently back up a
|
||||
/// state that is older than the catalog and possibly torn, which is the one
|
||||
/// failure mode a backup cannot afford.
|
||||
pub fn backup(conn: &Connection, catalog: &Path) -> Result<PathBuf, CatalogError> {
|
||||
let dir = backup_dir(catalog);
|
||||
std::fs::create_dir_all(&dir)
|
||||
.map_err(|e| CatalogError::Io(format!("creating {}: {e}", dir.display())))?;
|
||||
|
||||
let dest = dir.join(format!("catalog-{}.sqlite", now()));
|
||||
// A second backup within the same second would otherwise land on the first
|
||||
// one's name. Rare, and only reachable from tests and a retry, but the
|
||||
// result would be a half-overwritten backup rather than two.
|
||||
if dest.exists() {
|
||||
std::fs::remove_file(&dest)
|
||||
.map_err(|e| CatalogError::Io(format!("replacing {}: {e}", dest.display())))?;
|
||||
}
|
||||
|
||||
// Dropped immediately: the copy is complete when `copy_to` returns, and
|
||||
// holding the connection open would leave a `-wal` beside a file whose
|
||||
// whole purpose is to be a single self-contained artefact.
|
||||
drop(crate::sync::copy_to(conn, &dest)?);
|
||||
|
||||
prune(catalog);
|
||||
Ok(dest)
|
||||
}
|
||||
|
||||
/// Back up before a migration, if there is anything to back up.
|
||||
///
|
||||
/// Called from [`Catalog::open`](crate::Catalog::open) between `configure` and
|
||||
/// `migrate`. NFR-R2 asks for this and the reasoning is narrower than "backups
|
||||
/// are prudent": a migration is the one routine operation that rewrites table
|
||||
/// structure, so it is the likeliest way a catalog becomes unreadable, and it
|
||||
/// is the one moment where the pre-change state is still on disk to be copied.
|
||||
/// Afterwards there is nothing left to take a copy *of*.
|
||||
///
|
||||
/// A no-op in the two cases where it would cost without buying anything: a
|
||||
/// catalog already at [`schema::SCHEMA_VERSION`], and a brand-new file at
|
||||
/// version 0 with no tables in it yet.
|
||||
pub fn backup_before_migration(conn: &Connection, catalog: &Path) -> Result<(), CatalogError> {
|
||||
let from: i64 = conn.query_row("PRAGMA user_version", [], |r| r.get(0))?;
|
||||
if from == 0 || from >= schema::SCHEMA_VERSION {
|
||||
return Ok(());
|
||||
}
|
||||
let path = backup(conn, catalog)?;
|
||||
log::info!(
|
||||
"backed up catalog at v{from} to {} before migrating to v{}",
|
||||
path.display(),
|
||||
schema::SCHEMA_VERSION
|
||||
);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// How long a catalog may go without a backup before the next opportunity
|
||||
/// takes one.
|
||||
///
|
||||
/// A day. The catalog is an index, so what a backup protects is the day's
|
||||
/// worth of collection and people edits the sidecars do not hold — and a
|
||||
/// second copy of a 130 MB file per launch would be a cost with nothing to
|
||||
/// show for it when the user launches four times in an afternoon.
|
||||
pub const BACKUP_EVERY: i64 = 24 * 60 * 60;
|
||||
|
||||
/// Whether [`BACKUP_EVERY`] has passed since the newest backup, or there is
|
||||
/// none.
|
||||
///
|
||||
/// Read from the filenames, like [`backups`], so a restored or copied backup
|
||||
/// directory answers the same way it did on the machine it came from.
|
||||
pub fn backup_due(catalog: &Path) -> bool {
|
||||
match backups(catalog).first() {
|
||||
Some(newest) => now() - newest.taken_at >= BACKUP_EVERY,
|
||||
None => true,
|
||||
}
|
||||
}
|
||||
|
||||
/// TRACES: NFR-R2
|
||||
/// Take the scheduled backup, if one is due. Returns the file written, or
|
||||
/// `None` when the newest is recent enough.
|
||||
///
|
||||
/// The scheduled half of NFR-R2 — the migration half is
|
||||
/// [`backup_before_migration`]. "On a schedule" for an application that runs
|
||||
/// when the user opens it means "at the next chance after a day has passed",
|
||||
/// and the chance the caller picks is the end of a library sweep: the
|
||||
/// catalog is quiet, the work is already off the UI thread, and it is the
|
||||
/// moment a day's edits have just been consolidated.
|
||||
///
|
||||
/// A brand-new catalog with no images is not backed up: there is nothing in
|
||||
/// it yet that a rescan would not rebuild, and the first backup would only be
|
||||
/// a copy of an empty schema.
|
||||
pub fn backup_if_due(conn: &Connection, catalog: &Path) -> Result<Option<PathBuf>, CatalogError> {
|
||||
if !backup_due(catalog) {
|
||||
return Ok(None);
|
||||
}
|
||||
let images: i64 = conn.query_row("SELECT count(*) FROM images", [], |r| r.get(0))?;
|
||||
if images == 0 {
|
||||
return Ok(None);
|
||||
}
|
||||
let path = backup(conn, catalog)?;
|
||||
log::info!("scheduled backup of the catalog to {}", path.display());
|
||||
Ok(Some(path))
|
||||
}
|
||||
|
||||
/// The backups available for `catalog`, newest first.
|
||||
///
|
||||
/// Never fails: an unreadable or absent backup directory means there are no
|
||||
/// backups, which is a fact about the offer to make rather than an error to
|
||||
/// report on top of the corruption the user is already looking at.
|
||||
pub fn backups(catalog: &Path) -> Vec<Backup> {
|
||||
let dir = backup_dir(catalog);
|
||||
let Ok(entries) = std::fs::read_dir(&dir) else {
|
||||
return Vec::new();
|
||||
};
|
||||
|
||||
let mut out: Vec<Backup> = entries
|
||||
.flatten()
|
||||
.filter_map(|e| {
|
||||
let path = e.path();
|
||||
let taken_at = timestamp_of(&path)?;
|
||||
let bytes = e.metadata().ok()?.len();
|
||||
Some(Backup {
|
||||
path,
|
||||
taken_at,
|
||||
bytes,
|
||||
})
|
||||
})
|
||||
.collect();
|
||||
out.sort_by_key(|b| std::cmp::Reverse(b.taken_at));
|
||||
out
|
||||
}
|
||||
|
||||
/// Put a backup back in place of the damaged catalog.
|
||||
///
|
||||
/// **Every connection to `catalog` must be closed first.** This replaces the
|
||||
/// file underneath anything still holding it open, which on a live connection
|
||||
/// is how a *second* corrupt catalog gets made.
|
||||
///
|
||||
/// The order is deliberate:
|
||||
///
|
||||
/// 1. The backup is checked. A restore that installs a second damaged file
|
||||
/// leaves the user with nothing to try next.
|
||||
/// 2. The damaged catalog is renamed aside, and its `-wal` and `-shm` are
|
||||
/// **deleted**. This is the step that is easy to leave out and fatal to
|
||||
/// leave out: a journal belonging to the old file, sitting beside the new
|
||||
/// one under the same name, is replayed into it on the next open. That is
|
||||
/// not a restore, it is a fresh corruption with the evidence gone.
|
||||
/// 3. The backup is *copied* into place, not moved, so a failure here can be
|
||||
/// retried against the same backup.
|
||||
pub fn restore(catalog: &Path, backup: &Path) -> Result<(), CatalogError> {
|
||||
check_file(backup)?;
|
||||
set_aside(catalog)?;
|
||||
std::fs::copy(backup, catalog).map_err(|e| {
|
||||
CatalogError::Io(format!(
|
||||
"restoring {} from {}: {e}",
|
||||
catalog.display(),
|
||||
backup.display()
|
||||
))
|
||||
})?;
|
||||
log::info!(
|
||||
"restored {} from backup {}",
|
||||
catalog.display(),
|
||||
backup.display()
|
||||
);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Move a damaged catalog out of the way so the next open builds a fresh one.
|
||||
///
|
||||
/// This is the rebuild path (NFR-R6's second offer) and also the first half of
|
||||
/// a [`restore`]. Nothing else is needed to rebuild: the next
|
||||
/// [`Catalog::open`](crate::Catalog::open) creates an empty catalog at the
|
||||
/// current schema, and the ordinary scan repopulates it from sources and
|
||||
/// sidecars — which is precisely invariant §5.2.4 being spent rather than
|
||||
/// merely asserted.
|
||||
///
|
||||
/// Returns where the damaged file was put, or `None` if there was no catalog
|
||||
/// to move — a caller may be recovering from a file SQLite could not open
|
||||
/// because it was never created.
|
||||
pub fn set_aside(catalog: &Path) -> Result<Option<PathBuf>, CatalogError> {
|
||||
let moved = if catalog.exists() {
|
||||
let dest = with_suffix(catalog, DAMAGED_SUFFIX);
|
||||
// An earlier damaged copy is replaced rather than accumulating: two of
|
||||
// these is two full-size catalogs on the user's disk, and the older
|
||||
// one has already been superseded by a recovery the user completed.
|
||||
let _ = std::fs::remove_file(&dest);
|
||||
// The rename first, so that a failure here leaves the journals with
|
||||
// the file they belong to rather than orphaned beside a catalog that
|
||||
// is still in use.
|
||||
std::fs::rename(catalog, &dest).map_err(|e| {
|
||||
CatalogError::Io(format!(
|
||||
"setting aside {} as {}: {e}",
|
||||
catalog.display(),
|
||||
dest.display()
|
||||
))
|
||||
})?;
|
||||
log::warn!(
|
||||
"catalog {} was damaged; kept as {}",
|
||||
catalog.display(),
|
||||
dest.display()
|
||||
);
|
||||
Some(dest)
|
||||
} else {
|
||||
None
|
||||
};
|
||||
|
||||
// Then the journals, whether or not there was a catalog to move: a `-wal`
|
||||
// orphaned beside a missing database is replayed into whatever takes that
|
||||
// name next, which would not be a restore but a fresh corruption with the
|
||||
// evidence gone.
|
||||
for sidecar in journals(catalog) {
|
||||
if let Err(e) = std::fs::remove_file(&sidecar) {
|
||||
if e.kind() != std::io::ErrorKind::NotFound {
|
||||
return Err(CatalogError::Io(format!(
|
||||
"removing stale journal {}: {e}",
|
||||
sidecar.display()
|
||||
)));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Ok(moved)
|
||||
}
|
||||
|
||||
/// Delete backups beyond [`KEEP_BACKUPS`].
|
||||
///
|
||||
/// Best-effort and silent about individual failures: failing to delete an old
|
||||
/// backup is not a reason to fail the new one, which is already written.
|
||||
fn prune(catalog: &Path) {
|
||||
for old in backups(catalog).into_iter().skip(KEEP_BACKUPS) {
|
||||
if let Err(e) = std::fs::remove_file(&old.path) {
|
||||
log::warn!("could not prune backup {}: {e}", old.path.display());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// The WAL and shared-memory files SQLite keeps beside a database.
|
||||
fn journals(catalog: &Path) -> [PathBuf; 2] {
|
||||
[with_suffix(catalog, "wal"), with_suffix(catalog, "shm")]
|
||||
}
|
||||
|
||||
/// `catalog.sqlite` plus `-suffix`, the way SQLite names its own sidecars.
|
||||
///
|
||||
/// Appended to the whole filename rather than replacing the extension, so
|
||||
/// `catalog.sqlite-wal` is what SQLite would look for and `catalog.sqlite-
|
||||
/// damaged` sorts next to the catalog it came from.
|
||||
fn with_suffix(catalog: &Path, suffix: &str) -> PathBuf {
|
||||
let mut s = catalog.as_os_str().to_os_string();
|
||||
s.push("-");
|
||||
s.push(suffix);
|
||||
PathBuf::from(s)
|
||||
}
|
||||
|
||||
/// Read the timestamp out of a backup's filename, or `None` if this is not one.
|
||||
///
|
||||
/// Doubles as the filter that keeps [`backups`] from offering the user
|
||||
/// something that is not a catalog — a stray file in the directory, or a `-wal`
|
||||
/// left by a crash mid-backup.
|
||||
fn timestamp_of(path: &Path) -> Option<i64> {
|
||||
let name = path.file_name()?.to_str()?;
|
||||
name.strip_prefix("catalog-")?
|
||||
.strip_suffix(".sqlite")?
|
||||
.parse()
|
||||
.ok()
|
||||
}
|
||||
|
||||
/// Seconds since the epoch, or 0 if the clock is before it.
|
||||
fn now() -> i64 {
|
||||
SystemTime::now()
|
||||
.duration_since(UNIX_EPOCH)
|
||||
.map(|d| d.as_secs() as i64)
|
||||
.unwrap_or(0)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use crate::Catalog;
|
||||
use std::io::{Seek, SeekFrom, Write};
|
||||
|
||||
/// A scratch directory that cleans up with the test.
|
||||
fn tempdir(tag: &str) -> PathBuf {
|
||||
let base = std::env::temp_dir().join(format!(
|
||||
"dr-recovery-{tag}-{}-{:?}",
|
||||
std::process::id(),
|
||||
std::thread::current().id()
|
||||
));
|
||||
let _ = std::fs::remove_dir_all(&base);
|
||||
std::fs::create_dir_all(&base).unwrap();
|
||||
base
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_scheduled_backup_is_taken_once_a_day_and_not_more() {
|
||||
let dir = tempdir("scheduled");
|
||||
let path = dir.join("catalog.sqlite");
|
||||
fixture(&path, 3);
|
||||
let cat = Catalog::open(&path).unwrap();
|
||||
|
||||
// Nothing yet: due.
|
||||
assert!(backup_due(&path));
|
||||
let first = backup_if_due(cat.connection(), &path).unwrap();
|
||||
assert!(first.is_some(), "the first opportunity takes one");
|
||||
|
||||
// Taken just now: not due, and a second call does nothing.
|
||||
assert!(!backup_due(&path));
|
||||
assert_eq!(backup_if_due(cat.connection(), &path).unwrap(), None);
|
||||
assert_eq!(backups(&path).len(), 1);
|
||||
|
||||
// Age the one backup past the interval by renaming it, since the
|
||||
// timestamp is read from the name. Now it is due again.
|
||||
let old = first.unwrap();
|
||||
let aged = old
|
||||
.parent()
|
||||
.unwrap()
|
||||
.join(format!("catalog-{}.sqlite", now() - BACKUP_EVERY - 1));
|
||||
std::fs::rename(&old, &aged).unwrap();
|
||||
assert!(backup_due(&path));
|
||||
assert!(backup_if_due(cat.connection(), &path).unwrap().is_some());
|
||||
assert_eq!(backups(&path).len(), 2);
|
||||
|
||||
let _ = std::fs::remove_dir_all(&dir);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn an_empty_catalog_is_not_worth_backing_up() {
|
||||
let dir = tempdir("empty");
|
||||
let path = dir.join("catalog.sqlite");
|
||||
let cat = Catalog::open(&path).unwrap();
|
||||
assert!(backup_due(&path), "due in principle");
|
||||
assert_eq!(backup_if_due(cat.connection(), &path).unwrap(), None);
|
||||
assert!(backups(&path).is_empty());
|
||||
let _ = std::fs::remove_dir_all(&dir);
|
||||
}
|
||||
|
||||
/// A catalog on disk with enough rows to span several pages, closed.
|
||||
///
|
||||
/// Closed matters: WAL means the rows are in `catalog.sqlite-wal` until
|
||||
/// something checkpoints, and a test that corrupted the main file while
|
||||
/// the data was still in the journal would be corrupting empty space.
|
||||
fn fixture(path: &Path, images: i64) {
|
||||
let cat = Catalog::open(path).unwrap();
|
||||
let c = cat.connection();
|
||||
c.execute(
|
||||
"INSERT INTO roots(id, kind, label) VALUES (1, 'local', 'lib')",
|
||||
[],
|
||||
)
|
||||
.unwrap();
|
||||
c.execute(
|
||||
"INSERT INTO collections(uuid, name, kind, created, revision, modified)
|
||||
VALUES ('u1', 'Iceland', 0, 0, 1, 1)",
|
||||
[],
|
||||
)
|
||||
.unwrap();
|
||||
for i in 1..=images {
|
||||
c.execute(
|
||||
"INSERT INTO images(id, root_id, source_ref, added_at)
|
||||
VALUES (?1, 1, ?2, 0)",
|
||||
rusqlite::params![i, format!("DCIM/IMG_{i:05}.CR3")],
|
||||
)
|
||||
.unwrap();
|
||||
}
|
||||
crate::sync::checkpoint(c).unwrap();
|
||||
drop(cat);
|
||||
}
|
||||
|
||||
/// Scribble over everything past the first two pages.
|
||||
///
|
||||
/// Past them rather than over them so that page 1 — the header and the
|
||||
/// schema — survives: this produces a file SQLite is willing to open and
|
||||
/// then finds damaged, which is the case `quick_check` exists for. Wiping
|
||||
/// the header instead produces `SQLITE_NOTADB` at the first pragma, which
|
||||
/// is a different branch and has its own test.
|
||||
fn corrupt(path: &Path) {
|
||||
let mut f = std::fs::OpenOptions::new().write(true).open(path).unwrap();
|
||||
let len = f.metadata().unwrap().len();
|
||||
assert!(
|
||||
len > 8192,
|
||||
"fixture is only {len} bytes; corrupting past page 2 would be a no-op"
|
||||
);
|
||||
let junk = vec![0x5a_u8; (len - 8192) as usize];
|
||||
f.seek(SeekFrom::Start(8192)).unwrap();
|
||||
f.write_all(&junk).unwrap();
|
||||
f.sync_all().unwrap();
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_healthy_catalog_passes() {
|
||||
let cat = Catalog::in_memory().unwrap();
|
||||
integrity_check(cat.connection()).unwrap();
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_corrupt_catalog_is_reported_as_corrupt_not_as_sqlite() {
|
||||
// The whole point of the variant: this used to arrive as whatever
|
||||
// rusqlite error the first failing query produced, with nowhere to
|
||||
// hang a recovery offer.
|
||||
let dir = tempdir("detect");
|
||||
let path = dir.join("catalog.sqlite");
|
||||
fixture(&path, 500);
|
||||
corrupt(&path);
|
||||
|
||||
assert!(matches!(
|
||||
Catalog::open_verified(&path),
|
||||
Err(CatalogError::Corrupt { .. })
|
||||
));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_file_that_is_not_a_database_is_also_corrupt() {
|
||||
// A truncated or overwritten catalog never reaches `quick_check`: the
|
||||
// first pragma fails with SQLITE_NOTADB. Same accident to the user,
|
||||
// same two offers, so it must classify the same way.
|
||||
let dir = tempdir("notadb");
|
||||
let path = dir.join("catalog.sqlite");
|
||||
std::fs::write(&path, b"this is not a catalog, it is a text file\n").unwrap();
|
||||
|
||||
assert!(matches!(
|
||||
Catalog::open(&path),
|
||||
Err(CatalogError::Corrupt { .. })
|
||||
));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn restoring_a_backup_recovers_the_collections_a_rebuild_would_lose() {
|
||||
// The first NFR-R6 branch, asserted on the thing that distinguishes it
|
||||
// from the second: a collection exists nowhere but the catalog, so it
|
||||
// is the evidence that the *contents* came back and not merely a
|
||||
// readable file (docs/catalog.md §8.1).
|
||||
let dir = tempdir("restore");
|
||||
let path = dir.join("catalog.sqlite");
|
||||
fixture(&path, 500);
|
||||
|
||||
{
|
||||
let cat = Catalog::open(&path).unwrap();
|
||||
backup(cat.connection(), &path).unwrap();
|
||||
}
|
||||
corrupt(&path);
|
||||
assert!(matches!(
|
||||
Catalog::open_verified(&path),
|
||||
Err(CatalogError::Corrupt { .. })
|
||||
));
|
||||
|
||||
let newest = backups(&path).into_iter().next().expect("a backup exists");
|
||||
restore(&path, &newest.path).unwrap();
|
||||
|
||||
let cat = Catalog::open_verified(&path).unwrap();
|
||||
let name: String = cat
|
||||
.connection()
|
||||
.query_row("SELECT name FROM collections", [], |r| r.get(0))
|
||||
.unwrap();
|
||||
assert_eq!(name, "Iceland");
|
||||
let images: i64 = cat
|
||||
.connection()
|
||||
.query_row("SELECT count(*) FROM images", [], |r| r.get(0))
|
||||
.unwrap();
|
||||
assert_eq!(images, 500);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_damaged_backup_is_refused_rather_than_installed() {
|
||||
let dir = tempdir("badbackup");
|
||||
let path = dir.join("catalog.sqlite");
|
||||
fixture(&path, 500);
|
||||
{
|
||||
let cat = Catalog::open(&path).unwrap();
|
||||
backup(cat.connection(), &path).unwrap();
|
||||
}
|
||||
let newest = backups(&path).into_iter().next().unwrap();
|
||||
corrupt(&newest.path);
|
||||
corrupt(&path);
|
||||
|
||||
assert!(matches!(
|
||||
restore(&path, &newest.path),
|
||||
Err(CatalogError::Corrupt { .. })
|
||||
));
|
||||
// And the damaged catalog is still where it was, so the second offer
|
||||
// is still available.
|
||||
assert!(path.exists());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn setting_aside_leaves_a_fresh_catalog_to_rebuild_into() {
|
||||
// The second NFR-R6 branch. What makes it a rebuild rather than a data
|
||||
// loss is invariant §5.2.4, which lives outside this crate — what is
|
||||
// testable here is that the damaged file is out of the way, kept, and
|
||||
// that the next open succeeds on an empty catalog at the current
|
||||
// schema, which is what a scan then fills.
|
||||
let dir = tempdir("rebuild");
|
||||
let path = dir.join("catalog.sqlite");
|
||||
fixture(&path, 500);
|
||||
corrupt(&path);
|
||||
|
||||
let kept = set_aside(&path).unwrap().expect("the catalog was there");
|
||||
assert!(kept.exists(), "the damaged catalog was deleted, not kept");
|
||||
assert!(!path.exists());
|
||||
|
||||
let cat = Catalog::open_verified(&path).unwrap();
|
||||
let images: i64 = cat
|
||||
.connection()
|
||||
.query_row("SELECT count(*) FROM images", [], |r| r.get(0))
|
||||
.unwrap();
|
||||
assert_eq!(images, 0);
|
||||
let v: i64 = cat
|
||||
.connection()
|
||||
.query_row("PRAGMA user_version", [], |r| r.get(0))
|
||||
.unwrap();
|
||||
assert_eq!(v, schema::SCHEMA_VERSION);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_stale_journal_does_not_follow_the_catalog_into_recovery() {
|
||||
// The step that is easy to omit: a `-wal` belonging to the damaged
|
||||
// file is replayed into whatever takes its name next.
|
||||
let dir = tempdir("journal");
|
||||
let path = dir.join("catalog.sqlite");
|
||||
fixture(&path, 500);
|
||||
std::fs::write(with_suffix(&path, "wal"), b"stale").unwrap();
|
||||
|
||||
set_aside(&path).unwrap();
|
||||
assert!(!with_suffix(&path, "wal").exists());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_migration_is_backed_up_before_it_runs() {
|
||||
// NFR-R2's second clause, against a real v1 catalog rather than a
|
||||
// faked version number: the point is not that *a* file appears but
|
||||
// that it holds the state from before the migration, which is the only
|
||||
// state that is any use if the migration is what breaks it.
|
||||
let dir = tempdir("premigrate");
|
||||
let path = dir.join("catalog.sqlite");
|
||||
{
|
||||
let c = Connection::open(&path).unwrap();
|
||||
schema::configure(&c).unwrap();
|
||||
// `v1_for_attached` names the schema it targets, and "main" is a
|
||||
// schema like any other — so this is the real v1, without needing
|
||||
// `V1` itself to become visible outside its module.
|
||||
c.execute_batch(&schema::v1_for_attached("main")).unwrap();
|
||||
c.pragma_update(None, "user_version", 1).unwrap();
|
||||
c.execute(
|
||||
"INSERT INTO roots(id, kind, label) VALUES (1, 'local', 'lib')",
|
||||
[],
|
||||
)
|
||||
.unwrap();
|
||||
crate::sync::checkpoint(&c).unwrap();
|
||||
}
|
||||
assert!(backups(&path).is_empty());
|
||||
|
||||
Catalog::open(&path).unwrap();
|
||||
|
||||
let taken = backups(&path);
|
||||
assert_eq!(taken.len(), 1, "no backup was taken before the migration");
|
||||
check_file(&taken[0].path).unwrap();
|
||||
let kept = Connection::open(&taken[0].path).unwrap();
|
||||
let v: i64 = kept
|
||||
.query_row("PRAGMA user_version", [], |r| r.get(0))
|
||||
.unwrap();
|
||||
assert_eq!(v, 1, "the backup was taken after the migration, not before");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn opening_an_up_to_date_catalog_takes_no_backup() {
|
||||
// Or every background task that opens the catalog would copy it.
|
||||
let dir = tempdir("nobackup");
|
||||
let path = dir.join("catalog.sqlite");
|
||||
fixture(&path, 10);
|
||||
Catalog::open(&path).unwrap();
|
||||
assert!(backups(&path).is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn only_the_newest_generations_are_kept() {
|
||||
let dir = tempdir("prune");
|
||||
let path = dir.join("catalog.sqlite");
|
||||
fixture(&path, 10);
|
||||
let cat = Catalog::open(&path).unwrap();
|
||||
|
||||
// Written by hand rather than by calling `backup` in a loop: the
|
||||
// filename carries whole seconds, so real calls would collide.
|
||||
std::fs::create_dir_all(backup_dir(&path)).unwrap();
|
||||
for t in 1..=KEEP_BACKUPS as i64 + 2 {
|
||||
drop(
|
||||
crate::sync::copy_to(
|
||||
cat.connection(),
|
||||
&backup_dir(&path).join(format!("catalog-{t}.sqlite")),
|
||||
)
|
||||
.unwrap(),
|
||||
);
|
||||
}
|
||||
prune(&path);
|
||||
|
||||
let kept = backups(&path);
|
||||
assert_eq!(kept.len(), KEEP_BACKUPS);
|
||||
// Newest first, and the newest is the highest timestamp.
|
||||
assert_eq!(kept[0].taken_at, KEEP_BACKUPS as i64 + 2);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_stray_file_in_the_backup_directory_is_not_offered_as_one() {
|
||||
let dir = tempdir("stray");
|
||||
let path = dir.join("catalog.sqlite");
|
||||
fixture(&path, 10);
|
||||
std::fs::create_dir_all(backup_dir(&path)).unwrap();
|
||||
std::fs::write(backup_dir(&path).join("notes.txt"), b"hello").unwrap();
|
||||
std::fs::write(backup_dir(&path).join("catalog-7.sqlite-wal"), b"x").unwrap();
|
||||
|
||||
assert!(backups(&path).is_empty());
|
||||
}
|
||||
}
|
||||
@@ -1,935 +0,0 @@
|
||||
//! TRACES: FR-PLAT-AND-4 | FR-PLAT-AND-3
|
||||
//! The thing that drains the queue.
|
||||
//!
|
||||
//! [`crate::jobs`] has been a complete, durable, coalescing work queue since
|
||||
//! the catalog was written, and nothing has ever taken a job out of it. Every
|
||||
//! producer — the local walk, the remote scan — called `enqueue` and no one
|
||||
//! called `claim_next`, so the table grew one row per photograph and stayed
|
||||
//! that size forever. This module is the missing half.
|
||||
//!
|
||||
//! # Why the runner is driven rather than self-owning
|
||||
//!
|
||||
//! The obvious shape is a thread that loops until the queue is empty, and it
|
||||
//! is the wrong one. On Android the process does not decide when background
|
||||
//! work may run: `WorkManager` does, subject to Doze, battery saver and the
|
||||
//! metered-network constraints in FR-NC-6, and it revokes permission mid-job
|
||||
//! by calling `onStopped()` (FR-PLAT-AND-4). A foreground service for a
|
||||
//! user-initiated export gets a longer leash but still not an unbounded one.
|
||||
//!
|
||||
//! So the runner owns no thread, no clock and no policy. It exposes
|
||||
//! [`Runner::run_one`] — claim one job, run it, record what happened — and
|
||||
//! [`Runner::drain`], which repeats that against a [`Budget`] and a
|
||||
//! cancellation flag the host owns. A `Worker.doWork()` that must return
|
||||
//! within ten minutes calls `drain` with a deadline; a desktop idle loop calls
|
||||
//! it with none. Neither has to reach inside.
|
||||
//!
|
||||
//! Everything the host supplies is passed in for the same reason `jobs` takes
|
||||
//! `now` rather than reading the clock: a scheduler is exactly the thing that
|
||||
//! has to be testable without waiting.
|
||||
//!
|
||||
//! # Why interruption is not failure
|
||||
//!
|
||||
//! Four things can happen to a claimed job, and only two of them are the job's
|
||||
//! fault:
|
||||
//!
|
||||
//! - [`Outcome::Done`] — the row is deleted.
|
||||
//! - [`Outcome::Retry`] — the work failed and might succeed later. Backoff,
|
||||
//! and eventually [`crate::jobs::MAX_ATTEMPTS`] gives up on it.
|
||||
//! - [`Outcome::Abandon`] — the work cannot succeed, ever. Failing five times
|
||||
//! over five minutes to learn that is five minutes of a phone's battery.
|
||||
//! - [`Outcome::Interrupted`] — the *host* stopped, not the job. The claim is
|
||||
//! released and the attempt it consumed is given back, because a user who
|
||||
//! pulled the app off the screen has not told us anything about the file.
|
||||
//!
|
||||
//! Process death is the fifth case and the one that cannot report itself: the
|
||||
//! row simply stays `Running` with no owner. [`Runner::recover`] is what
|
||||
//! reclaims it, and it is why an interrupted job is resumable rather than lost
|
||||
//! (FR-PLAT-AND-3). It must run **before** any worker starts against a
|
||||
//! catalog, or it will steal a job another runner is holding — there is no
|
||||
//! owner column to tell them apart.
|
||||
|
||||
use std::sync::atomic::{AtomicBool, Ordering};
|
||||
|
||||
use rusqlite::Connection;
|
||||
|
||||
use crate::error::CatalogError;
|
||||
use crate::jobs::{self, Job, JobKind};
|
||||
|
||||
/// What running a job turned out to be.
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub enum Outcome {
|
||||
/// The work is done. The row goes away.
|
||||
Done,
|
||||
/// It failed, and trying again later is reasonable. Backoff applies, and
|
||||
/// [`crate::jobs::MAX_ATTEMPTS`] eventually stops it.
|
||||
Retry(String),
|
||||
/// It failed in a way no retry can fix — the subject is gone, the payload
|
||||
/// is unreadable, the format is one this build does not know. Marked
|
||||
/// failed at once rather than burning the whole retry ladder to reach the
|
||||
/// same answer.
|
||||
Abandon(String),
|
||||
/// The host is stopping, and the job never really ran.
|
||||
///
|
||||
/// Distinct from `Retry` because it costs no attempt: `onStopped()` five
|
||||
/// times in a row would otherwise mark a perfectly good job as failed.
|
||||
Interrupted,
|
||||
}
|
||||
|
||||
/// Something that can actually do the work a job describes.
|
||||
///
|
||||
/// The catalog knows what needs doing and nothing about how — a thumbnail
|
||||
/// needs a decoder, a fetch needs a network stack, and neither belongs under
|
||||
/// `core/dr-catalog` (ARCH §4.1: calls go downward). So the queue lives here
|
||||
/// and the handlers are supplied from above.
|
||||
pub trait JobHandler {
|
||||
/// The kinds this handler will accept.
|
||||
///
|
||||
/// Load-bearing, not documentation: the runner claims **only** kinds some
|
||||
/// handler declares. A queue holding `FetchOriginal` rows on a device with
|
||||
/// no connector must leave them alone rather than claim them and fail
|
||||
/// them, and a runner that claimed everything would do exactly that — five
|
||||
/// times each, with backoff, on battery.
|
||||
fn kinds(&self) -> &[JobKind];
|
||||
|
||||
/// Do the work.
|
||||
///
|
||||
/// The connection is offered because most handlers write their result back
|
||||
/// into the catalog; one that does not is free to ignore it. It is the
|
||||
/// runner's own connection, so a handler must not hold a transaction open
|
||||
/// across a network call — the runner needs it back to record the outcome.
|
||||
fn run(&mut self, conn: &Connection, job: &Job) -> Outcome;
|
||||
}
|
||||
|
||||
/// How much work a host is willing to let one drain do.
|
||||
///
|
||||
/// Both limits are checked *before* a job is claimed, never during one: a
|
||||
/// handler is opaque and may be halfway through writing a sidecar. Overrunning
|
||||
/// a deadline by one job is survivable; being killed mid-write is the thing
|
||||
/// [`Runner::recover`] exists to clean up after.
|
||||
#[derive(Debug, Default, Clone, Copy, PartialEq, Eq)]
|
||||
pub struct Budget {
|
||||
/// Stop after this many jobs. `None` means "until the queue is empty".
|
||||
pub max_jobs: Option<usize>,
|
||||
/// Stop once the clock reaches this second. Same clock the drain is given.
|
||||
pub deadline: Option<i64>,
|
||||
}
|
||||
|
||||
impl Budget {
|
||||
/// Run until nothing is left. What a desktop idle pass wants.
|
||||
pub const UNLIMITED: Self = Self {
|
||||
max_jobs: None,
|
||||
deadline: None,
|
||||
};
|
||||
|
||||
/// At most `n` jobs. A slice small enough to stay responsive.
|
||||
pub fn jobs(n: usize) -> Self {
|
||||
Self {
|
||||
max_jobs: Some(n),
|
||||
..Self::UNLIMITED
|
||||
}
|
||||
}
|
||||
|
||||
/// Until the clock reaches `deadline`. What a `WorkManager` slot wants.
|
||||
pub fn until(deadline: i64) -> Self {
|
||||
Self {
|
||||
deadline: Some(deadline),
|
||||
..Self::UNLIMITED
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Why a drain stopped.
|
||||
///
|
||||
/// Worth distinguishing because the host's next move differs: `Drained` means
|
||||
/// there is nothing to reschedule for, and the other three all mean "there is
|
||||
/// more, ask again".
|
||||
#[derive(Debug, Default, Clone, Copy, PartialEq, Eq)]
|
||||
pub enum Stopped {
|
||||
/// Nothing claimable is left.
|
||||
#[default]
|
||||
Drained,
|
||||
/// The job count ran out.
|
||||
Budget,
|
||||
/// The clock ran out.
|
||||
Deadline,
|
||||
/// The host asked it to stop, or a handler reported itself interrupted.
|
||||
Cancelled,
|
||||
}
|
||||
|
||||
impl Stopped {
|
||||
/// Whether the queue may still hold claimable work.
|
||||
pub fn more_to_do(self) -> bool {
|
||||
self != Stopped::Drained
|
||||
}
|
||||
}
|
||||
|
||||
/// What one drain did.
|
||||
#[derive(Debug, Default, Clone, Copy, PartialEq, Eq)]
|
||||
pub struct DrainReport {
|
||||
pub completed: usize,
|
||||
pub retried: usize,
|
||||
pub abandoned: usize,
|
||||
pub interrupted: usize,
|
||||
pub stopped: Stopped,
|
||||
}
|
||||
|
||||
impl DrainReport {
|
||||
/// Jobs claimed, whatever became of them. This is what a budget counts.
|
||||
pub fn ran(&self) -> usize {
|
||||
self.completed + self.retried + self.abandoned + self.interrupted
|
||||
}
|
||||
}
|
||||
|
||||
/// What a recovery pass found waiting from the last run.
|
||||
#[derive(Debug, Default, Clone, Copy, PartialEq, Eq)]
|
||||
pub struct Recovered {
|
||||
/// Jobs a dead process was holding. These are the resumed ones.
|
||||
pub reclaimed: usize,
|
||||
/// Jobs deleted because the photograph they name no longer exists.
|
||||
pub reaped: usize,
|
||||
}
|
||||
|
||||
impl Recovered {
|
||||
pub fn did_anything(&self) -> bool {
|
||||
self.reclaimed > 0 || self.reaped > 0
|
||||
}
|
||||
}
|
||||
|
||||
/// Ready the queue for a fresh run, before any worker touches it.
|
||||
///
|
||||
/// Two distinct cleanups, and both are startup-only:
|
||||
///
|
||||
/// - **Reclaim.** A `Running` row has no owner; the process that claimed it is
|
||||
/// gone. On Android that is a routine morning, not a crash (FR-PLAT-AND-3).
|
||||
/// The attempt it consumed is *kept*, deliberately: a job that takes the
|
||||
/// process down with it three times running should not be retried forever,
|
||||
/// and the attempt counter is the only evidence of that we have.
|
||||
/// - **Reap.** Jobs naming an image the catalog no longer has. A library that
|
||||
/// has been culled leaves thumbnail jobs for photographs that were deleted
|
||||
/// months ago, and every one of them would be claimed, run and failed.
|
||||
///
|
||||
/// Reclaim runs first so its count is the honest number of interrupted jobs,
|
||||
/// before reaping removes whichever of them pointed at nothing.
|
||||
///
|
||||
/// **Call this exactly once per catalog, at startup.** It cannot distinguish a
|
||||
/// job a dead process was holding from one a live runner is holding right now,
|
||||
/// because there is no owner column — the queue is durable, not distributed.
|
||||
pub fn recover(conn: &Connection) -> Result<Recovered, CatalogError> {
|
||||
Ok(Recovered {
|
||||
reclaimed: jobs::recover_orphaned(conn)?,
|
||||
reaped: jobs::reap_orphan_subjects(conn)?,
|
||||
})
|
||||
}
|
||||
|
||||
/// Claims work, runs it, and records what happened.
|
||||
///
|
||||
/// Borrows its connection rather than owning one so a host can drive it from
|
||||
/// the same handle it already has open. Nothing here spawns a thread; several
|
||||
/// runners on several threads, each with its own connection to the same
|
||||
/// catalog, are safe because the claim is a single atomic statement (see
|
||||
/// [`crate::jobs::claim_next`]).
|
||||
pub struct Runner<'a> {
|
||||
conn: &'a Connection,
|
||||
handlers: Vec<Box<dyn JobHandler + 'a>>,
|
||||
/// The union of every handler's kinds, cached because it is passed to
|
||||
/// every claim. This is what stops the runner claiming work it cannot do.
|
||||
claimable: Vec<JobKind>,
|
||||
}
|
||||
|
||||
impl<'a> Runner<'a> {
|
||||
/// A runner with no handlers. It can recover, and it can claim nothing.
|
||||
pub fn new(conn: &'a Connection) -> Self {
|
||||
Self {
|
||||
conn,
|
||||
handlers: Vec::new(),
|
||||
claimable: Vec::new(),
|
||||
}
|
||||
}
|
||||
|
||||
/// Add a handler.
|
||||
pub fn with(self, handler: impl JobHandler + 'a) -> Self {
|
||||
self.with_boxed(Box::new(handler))
|
||||
}
|
||||
|
||||
/// Add a handler chosen at runtime — a network one only where there is a
|
||||
/// connector, a decoding one only where there is a decoder.
|
||||
pub fn with_boxed(mut self, handler: Box<dyn JobHandler + 'a>) -> Self {
|
||||
for kind in handler.kinds() {
|
||||
if !self.claimable.contains(kind) {
|
||||
self.claimable.push(*kind);
|
||||
}
|
||||
}
|
||||
self.handlers.push(handler);
|
||||
self
|
||||
}
|
||||
|
||||
/// The kinds this runner will claim. Useful to a host deciding whether
|
||||
/// starting it is worth waking the radio for.
|
||||
pub fn claimable(&self) -> &[JobKind] {
|
||||
&self.claimable
|
||||
}
|
||||
|
||||
/// See [`recover`]. Offered here too so a host has one thing to hold.
|
||||
pub fn recover(&self) -> Result<Recovered, CatalogError> {
|
||||
recover(self.conn)
|
||||
}
|
||||
|
||||
/// Claim one job, run it, and record the outcome.
|
||||
///
|
||||
/// `Ok(None)` means nothing this runner can do is claimable *now* — the
|
||||
/// queue may still hold work of other kinds, or work still backing off.
|
||||
pub fn run_one(&mut self, now: i64) -> Result<Option<Ran>, CatalogError> {
|
||||
// Copied out before `self.handlers` is borrowed mutably below. Both
|
||||
// are fields of `self`, but the copy is what lets the two borrows
|
||||
// coexist without the connection being reborrowed through `self`.
|
||||
let conn = self.conn;
|
||||
|
||||
let Some(job) = jobs::claim_next_matching(conn, now, &self.claimable)? else {
|
||||
return Ok(None);
|
||||
};
|
||||
|
||||
let outcome = match self
|
||||
.handlers
|
||||
.iter_mut()
|
||||
.find(|h| h.kinds().contains(&job.kind))
|
||||
{
|
||||
Some(handler) => handler.run(conn, &job),
|
||||
// Unreachable by construction: `claimable` is exactly the union of
|
||||
// the handlers' kinds. Parked rather than released, because
|
||||
// releasing it would put it straight back where the next turn of
|
||||
// the drain loop would claim it again, forever.
|
||||
None => Outcome::Abandon(format!("no handler for {:?}", job.kind)),
|
||||
};
|
||||
|
||||
match &outcome {
|
||||
Outcome::Done => jobs::complete(conn, job.id)?,
|
||||
Outcome::Retry(why) => jobs::fail(conn, &job, now, why)?,
|
||||
Outcome::Abandon(why) => jobs::abandon(conn, job.id, why)?,
|
||||
Outcome::Interrupted => jobs::release(conn, &job)?,
|
||||
}
|
||||
|
||||
Ok(Some(Ran { job, outcome }))
|
||||
}
|
||||
|
||||
/// Run jobs until the budget, the flag or the queue says stop.
|
||||
///
|
||||
/// `clock` is called once per iteration rather than sampled once, because
|
||||
/// the two things it feeds both move during a long drain: the deadline
|
||||
/// check, and the `now` a failing job's backoff is measured from.
|
||||
///
|
||||
/// Cancellation is checked between jobs only. A handler that wants to bail
|
||||
/// out of work already started says so with [`Outcome::Interrupted`],
|
||||
/// which also ends the drain — otherwise a handler that always interrupts
|
||||
/// would release its job and be handed it straight back.
|
||||
pub fn drain(
|
||||
&mut self,
|
||||
clock: &dyn Fn() -> i64,
|
||||
budget: Budget,
|
||||
cancel: &AtomicBool,
|
||||
) -> Result<DrainReport, CatalogError> {
|
||||
let mut report = DrainReport::default();
|
||||
|
||||
loop {
|
||||
// Relaxed: the flag is a one-way latch set by another thread and
|
||||
// the only thing ordered against it is our own next claim. Missing
|
||||
// one turn of the loop costs a job, not correctness.
|
||||
if cancel.load(Ordering::Relaxed) {
|
||||
report.stopped = Stopped::Cancelled;
|
||||
break;
|
||||
}
|
||||
if budget.max_jobs.is_some_and(|max| report.ran() >= max) {
|
||||
report.stopped = Stopped::Budget;
|
||||
break;
|
||||
}
|
||||
|
||||
let now = clock();
|
||||
if budget.deadline.is_some_and(|end| now >= end) {
|
||||
report.stopped = Stopped::Deadline;
|
||||
break;
|
||||
}
|
||||
|
||||
let Some(ran) = self.run_one(now)? else {
|
||||
report.stopped = Stopped::Drained;
|
||||
break;
|
||||
};
|
||||
|
||||
match ran.outcome {
|
||||
Outcome::Done => report.completed += 1,
|
||||
Outcome::Retry(_) => report.retried += 1,
|
||||
Outcome::Abandon(_) => report.abandoned += 1,
|
||||
Outcome::Interrupted => {
|
||||
report.interrupted += 1;
|
||||
report.stopped = Stopped::Cancelled;
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Ok(report)
|
||||
}
|
||||
|
||||
/// Drain with no budget and no cancellation, at a fixed instant.
|
||||
///
|
||||
/// Terminates because a job that fails is pushed past `now` by its backoff
|
||||
/// and stops being claimable at this instant.
|
||||
pub fn drain_all(&mut self, now: i64) -> Result<DrainReport, CatalogError> {
|
||||
static NEVER: AtomicBool = AtomicBool::new(false);
|
||||
self.drain(&|| now, Budget::UNLIMITED, &NEVER)
|
||||
}
|
||||
}
|
||||
|
||||
/// One job and what became of it.
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct Ran {
|
||||
pub job: Job,
|
||||
pub outcome: Outcome,
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::sync::{Arc, Mutex};
|
||||
|
||||
use super::*;
|
||||
use crate::jobs::{enqueue, JobState, Priority, MAX_ATTEMPTS};
|
||||
use crate::schema;
|
||||
|
||||
/// A handler built from a closure, so each test states its own behaviour.
|
||||
struct Fake<F> {
|
||||
kinds: Vec<JobKind>,
|
||||
act: F,
|
||||
}
|
||||
|
||||
impl<F: FnMut(&Job) -> Outcome> JobHandler for Fake<F> {
|
||||
fn kinds(&self) -> &[JobKind] {
|
||||
&self.kinds
|
||||
}
|
||||
fn run(&mut self, _conn: &Connection, job: &Job) -> Outcome {
|
||||
(self.act)(job)
|
||||
}
|
||||
}
|
||||
|
||||
fn handler<F: FnMut(&Job) -> Outcome>(kinds: &[JobKind], act: F) -> Fake<F> {
|
||||
Fake {
|
||||
kinds: kinds.to_vec(),
|
||||
act,
|
||||
}
|
||||
}
|
||||
|
||||
/// A handler that records which subjects it saw and always succeeds.
|
||||
///
|
||||
/// Takes its kinds by value and borrows nothing, so the returned handler is
|
||||
/// `Send + 'static` and can be moved into a worker thread — which the
|
||||
/// contention test needs.
|
||||
fn recording(kinds: Vec<JobKind>, seen: Arc<Mutex<Vec<i64>>>) -> impl JobHandler + Send {
|
||||
Fake {
|
||||
kinds,
|
||||
act: move |job: &Job| {
|
||||
seen.lock().unwrap().push(job.subject_id.unwrap_or(-1));
|
||||
Outcome::Done
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
fn db() -> Connection {
|
||||
let c = Connection::open_in_memory().unwrap();
|
||||
schema::configure(&c).unwrap();
|
||||
schema::migrate(&c).unwrap();
|
||||
c
|
||||
}
|
||||
|
||||
/// An image row, so a job has a subject that exists.
|
||||
fn image(c: &Connection, id: i64) {
|
||||
c.execute(
|
||||
"INSERT INTO roots(id, kind, label) VALUES (1, 'local', '/lib')
|
||||
ON CONFLICT DO NOTHING",
|
||||
[],
|
||||
)
|
||||
.unwrap();
|
||||
c.execute(
|
||||
"INSERT INTO images(id, root_id, source_ref, added_at) VALUES (?1, 1, ?2, 0)",
|
||||
rusqlite::params![id, format!("/lib/{id}.CR3")],
|
||||
)
|
||||
.unwrap();
|
||||
}
|
||||
|
||||
fn queued(c: &Connection, kind: JobKind, subject: i64) {
|
||||
image(c, subject);
|
||||
enqueue(c, kind, Some(subject), Priority::Background, None).unwrap();
|
||||
}
|
||||
|
||||
fn rows(c: &Connection) -> i64 {
|
||||
c.query_row("SELECT count(*) FROM jobs", [], |r| r.get(0))
|
||||
.unwrap()
|
||||
}
|
||||
|
||||
/// A catalog on disk, so more than one connection can open it.
|
||||
fn temp_catalog(name: &str) -> PathBuf {
|
||||
let dir = std::env::temp_dir().join(format!(
|
||||
"dr-runner-{name}-{}-{:?}",
|
||||
std::process::id(),
|
||||
std::thread::current().id()
|
||||
));
|
||||
let _ = std::fs::remove_dir_all(&dir);
|
||||
std::fs::create_dir_all(&dir).unwrap();
|
||||
dir.join("catalog.db")
|
||||
}
|
||||
|
||||
fn open(path: &Path) -> Connection {
|
||||
let c = Connection::open(path).unwrap();
|
||||
schema::configure(&c).unwrap();
|
||||
schema::migrate(&c).unwrap();
|
||||
// Several connections write to this file at once in the contention
|
||||
// tests. Without a busy handler the loser of a race gets an error
|
||||
// instead of a turn.
|
||||
c.busy_timeout(std::time::Duration::from_secs(10)).unwrap();
|
||||
c
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_completed_job_leaves_the_queue() {
|
||||
// The whole finding in one assertion: before this module, the row
|
||||
// stayed forever because nothing ever claimed it.
|
||||
let c = db();
|
||||
queued(&c, JobKind::Thumbnail, 1);
|
||||
|
||||
let seen = Arc::new(Mutex::new(Vec::new()));
|
||||
let report = Runner::new(&c)
|
||||
.with(recording(vec![JobKind::Thumbnail], seen.clone()))
|
||||
.drain_all(0)
|
||||
.unwrap();
|
||||
|
||||
assert_eq!(report.completed, 1);
|
||||
assert_eq!(report.stopped, Stopped::Drained);
|
||||
assert_eq!(*seen.lock().unwrap(), vec![1]);
|
||||
assert_eq!(rows(&c), 0, "a completed job leaves no row behind");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_failed_job_backs_off_and_is_claimed_again_later() {
|
||||
let c = db();
|
||||
queued(&c, JobKind::Thumbnail, 1);
|
||||
|
||||
// A `Cell` rather than a captured `bool`, so the closure's mutability
|
||||
// is its own business and the test reads the same either way.
|
||||
let failed_once = std::cell::Cell::new(false);
|
||||
let mut runner = Runner::new(&c).with(handler(&[JobKind::Thumbnail], |_| {
|
||||
if failed_once.replace(true) {
|
||||
Outcome::Done
|
||||
} else {
|
||||
Outcome::Retry("decoder said no".into())
|
||||
}
|
||||
}));
|
||||
|
||||
let first = runner.drain_all(100).unwrap();
|
||||
assert_eq!(first.retried, 1);
|
||||
assert_eq!(rows(&c), 1, "a retryable failure keeps its row");
|
||||
|
||||
// Still inside the backoff window: nothing claimable, so the drain
|
||||
// reports itself drained rather than spinning on the same job.
|
||||
assert_eq!(runner.drain_all(100).unwrap().ran(), 0);
|
||||
|
||||
let later = runner.drain_all(100 + jobs::backoff_seconds(1)).unwrap();
|
||||
assert_eq!(later.completed, 1);
|
||||
assert_eq!(rows(&c), 0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_job_that_keeps_failing_is_given_up_on() {
|
||||
// FR-RAW-4: one corrupt file must not stall the queue behind endless
|
||||
// retries. Driven through the runner rather than by hand, because the
|
||||
// runner is what a corrupt file will actually meet.
|
||||
let c = db();
|
||||
queued(&c, JobKind::ExtractMetadata, 1);
|
||||
|
||||
let mut runner = Runner::new(&c).with(handler(&[JobKind::ExtractMetadata], |_| {
|
||||
Outcome::Retry("corrupt file".into())
|
||||
}));
|
||||
|
||||
let mut now = 0;
|
||||
for _ in 0..MAX_ATTEMPTS {
|
||||
assert_eq!(runner.drain_all(now).unwrap().retried, 1);
|
||||
now += jobs::backoff_seconds(MAX_ATTEMPTS);
|
||||
}
|
||||
|
||||
assert_eq!(runner.drain_all(now + 100_000).unwrap().ran(), 0);
|
||||
let state: i64 = c
|
||||
.query_row("SELECT state FROM jobs", [], |r| r.get(0))
|
||||
.unwrap();
|
||||
assert_eq!(state, JobState::Failed as i64);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn an_abandoned_job_is_not_retried_at_all() {
|
||||
// The difference that matters on battery: five failures spread over
|
||||
// five minutes to learn what the first one already said.
|
||||
let c = db();
|
||||
queued(&c, JobKind::FetchOriginal, 1);
|
||||
|
||||
let mut runner = Runner::new(&c).with(handler(&[JobKind::FetchOriginal], |_| {
|
||||
Outcome::Abandon("no connector on this device".into())
|
||||
}));
|
||||
|
||||
assert_eq!(runner.drain_all(0).unwrap().abandoned, 1);
|
||||
// One attempt, not MAX_ATTEMPTS, and never claimable again.
|
||||
assert_eq!(runner.drain_all(1_000_000).unwrap().ran(), 0);
|
||||
let (state, attempts): (i64, i64) = c
|
||||
.query_row("SELECT state, attempts FROM jobs", [], |r| {
|
||||
Ok((r.get(0)?, r.get(1)?))
|
||||
})
|
||||
.unwrap();
|
||||
assert_eq!(state, JobState::Failed as i64);
|
||||
assert_eq!(attempts, 1);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn an_interrupted_job_costs_no_attempt_and_ends_the_drain() {
|
||||
// `onStopped()` says nothing about the file. Charging it an attempt
|
||||
// would let five backgroundings mark good work as failed.
|
||||
let c = db();
|
||||
queued(&c, JobKind::Thumbnail, 1);
|
||||
queued(&c, JobKind::Thumbnail, 2);
|
||||
|
||||
let report = Runner::new(&c)
|
||||
.with(handler(&[JobKind::Thumbnail], |_| Outcome::Interrupted))
|
||||
.drain_all(0)
|
||||
.unwrap();
|
||||
|
||||
assert_eq!(report.interrupted, 1);
|
||||
assert_eq!(
|
||||
report.stopped,
|
||||
Stopped::Cancelled,
|
||||
"an interrupted job must end the drain, or releasing it hands it \
|
||||
straight back and the loop never ends"
|
||||
);
|
||||
let (state, attempts): (i64, i64) = c
|
||||
.query_row(
|
||||
"SELECT state, attempts FROM jobs WHERE subject_id = 1",
|
||||
[],
|
||||
|r| Ok((r.get(0)?, r.get(1)?)),
|
||||
)
|
||||
.unwrap();
|
||||
assert_eq!(state, JobState::Pending as i64);
|
||||
assert_eq!(attempts, 0, "the claim's speculative attempt is given back");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn only_kinds_a_handler_covers_are_claimed() {
|
||||
// A device with no connector must leave `FetchOriginal` where it is.
|
||||
// Claiming it to fail it would cost five attempts and five backoffs
|
||||
// per photograph, on battery, to reach a conclusion known in advance.
|
||||
let c = db();
|
||||
queued(&c, JobKind::Thumbnail, 1);
|
||||
queued(&c, JobKind::FetchOriginal, 2);
|
||||
|
||||
let seen = Arc::new(Mutex::new(Vec::new()));
|
||||
let report = Runner::new(&c)
|
||||
.with(recording(vec![JobKind::Thumbnail], seen.clone()))
|
||||
.drain_all(0)
|
||||
.unwrap();
|
||||
|
||||
assert_eq!(report.completed, 1);
|
||||
assert_eq!(*seen.lock().unwrap(), vec![1]);
|
||||
|
||||
let (state, attempts): (i64, i64) = c
|
||||
.query_row(
|
||||
"SELECT state, attempts FROM jobs WHERE subject_id = 2",
|
||||
[],
|
||||
|r| Ok((r.get(0)?, r.get(1)?)),
|
||||
)
|
||||
.unwrap();
|
||||
assert_eq!(state, JobState::Pending as i64);
|
||||
assert_eq!(attempts, 0, "an unhandled job is untouched, not failed");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_runner_with_no_handlers_claims_nothing() {
|
||||
let c = db();
|
||||
queued(&c, JobKind::Thumbnail, 1);
|
||||
|
||||
assert_eq!(Runner::new(&c).drain_all(0).unwrap().ran(), 0);
|
||||
assert_eq!(rows(&c), 1);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_drain_stops_at_its_job_budget() {
|
||||
let c = db();
|
||||
for id in 1..=5 {
|
||||
queued(&c, JobKind::Thumbnail, id);
|
||||
}
|
||||
|
||||
let seen = Arc::new(Mutex::new(Vec::new()));
|
||||
let mut runner = Runner::new(&c).with(recording(vec![JobKind::Thumbnail], seen.clone()));
|
||||
let report = runner
|
||||
.drain(&|| 0, Budget::jobs(2), &AtomicBool::new(false))
|
||||
.unwrap();
|
||||
|
||||
assert_eq!(report.completed, 2);
|
||||
assert_eq!(report.stopped, Stopped::Budget);
|
||||
assert!(report.stopped.more_to_do());
|
||||
assert_eq!(rows(&c), 3, "the rest is still queued for the next slot");
|
||||
assert_eq!(seen.lock().unwrap().len(), 2);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_drain_stops_at_its_deadline() {
|
||||
// What a `WorkManager` slot does: a fixed window, and whatever did not
|
||||
// fit stays queued for the next one.
|
||||
let c = db();
|
||||
for id in 1..=5 {
|
||||
queued(&c, JobKind::Thumbnail, id);
|
||||
}
|
||||
|
||||
// A clock that advances a second per reading, so the deadline arrives
|
||||
// without the test sleeping.
|
||||
let tick = std::cell::Cell::new(0i64);
|
||||
let clock = || {
|
||||
let t = tick.get();
|
||||
tick.set(t + 1);
|
||||
t
|
||||
};
|
||||
|
||||
let report = Runner::new(&c)
|
||||
.with(handler(&[JobKind::Thumbnail], |_| Outcome::Done))
|
||||
.drain(&clock, Budget::until(3), &AtomicBool::new(false))
|
||||
.unwrap();
|
||||
|
||||
assert_eq!(report.stopped, Stopped::Deadline);
|
||||
assert_eq!(report.completed, 3, "one job per second up to the deadline");
|
||||
assert_eq!(rows(&c), 2);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_cancelled_drain_stops_between_jobs() {
|
||||
let c = db();
|
||||
for id in 1..=5 {
|
||||
queued(&c, JobKind::Thumbnail, id);
|
||||
}
|
||||
|
||||
let cancel = AtomicBool::new(false);
|
||||
// Cancelled from inside the handler, standing in for the host thread
|
||||
// setting the flag while a job is in flight: the job in hand finishes,
|
||||
// and nothing further is claimed.
|
||||
let report = Runner::new(&c)
|
||||
.with(handler(&[JobKind::Thumbnail], |_| {
|
||||
cancel.store(true, Ordering::Relaxed);
|
||||
Outcome::Done
|
||||
}))
|
||||
.drain(&|| 0, Budget::UNLIMITED, &cancel)
|
||||
.unwrap();
|
||||
|
||||
assert_eq!(report.completed, 1);
|
||||
assert_eq!(report.stopped, Stopped::Cancelled);
|
||||
assert_eq!(rows(&c), 4, "the work is kept, not lost");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_job_interrupted_by_process_death_is_reclaimed_and_run_once() {
|
||||
// FR-PLAT-AND-3. The kill happens between the claim and the outcome,
|
||||
// which is the window a durable queue exists to survive: no `complete`,
|
||||
// no `fail`, just a row marked `Running` with nobody holding it.
|
||||
let c = db();
|
||||
queued(&c, JobKind::Thumbnail, 1);
|
||||
|
||||
// The dead process. It claimed the job and never came back.
|
||||
let claimed = jobs::claim_next(&c, 0).unwrap().expect("claimable");
|
||||
assert_eq!(claimed.subject_id, Some(1));
|
||||
|
||||
// A fresh runner, before it starts, finds the queue empty — the row is
|
||||
// `Running` and no claim will touch it.
|
||||
let seen = Arc::new(Mutex::new(Vec::new()));
|
||||
let mut runner = Runner::new(&c).with(recording(vec![JobKind::Thumbnail], seen.clone()));
|
||||
assert_eq!(
|
||||
runner.drain_all(0).unwrap().ran(),
|
||||
0,
|
||||
"an orphan is invisible until it is recovered — which is exactly \
|
||||
why recovery has to happen at startup"
|
||||
);
|
||||
|
||||
let recovered = runner.recover().unwrap();
|
||||
assert_eq!(recovered.reclaimed, 1);
|
||||
|
||||
let report = runner.drain_all(0).unwrap();
|
||||
assert_eq!(report.completed, 1);
|
||||
assert_eq!(
|
||||
*seen.lock().unwrap(),
|
||||
vec![1],
|
||||
"resumed, not repeated and not lost"
|
||||
);
|
||||
assert_eq!(rows(&c), 0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_crash_still_costs_an_attempt() {
|
||||
// Deliberate: a job that takes the process down with it every time is
|
||||
// indistinguishable from one that fails, and the attempt counter is
|
||||
// the only evidence we keep across a death. Without this a poison-pill
|
||||
// job would be reclaimed and re-run forever.
|
||||
let c = db();
|
||||
queued(&c, JobKind::Thumbnail, 1);
|
||||
|
||||
for _ in 0..MAX_ATTEMPTS {
|
||||
jobs::claim_next(&c, 0).unwrap().expect("claimable");
|
||||
recover(&c).unwrap();
|
||||
}
|
||||
|
||||
let job = jobs::claim_next(&c, 0).unwrap().unwrap();
|
||||
assert!(job.attempts > MAX_ATTEMPTS);
|
||||
jobs::fail(&c, &job, 0, "died again").unwrap();
|
||||
let state: i64 = c
|
||||
.query_row("SELECT state FROM jobs", [], |r| r.get(0))
|
||||
.unwrap();
|
||||
assert_eq!(state, JobState::Failed as i64);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn recovery_drops_jobs_whose_photograph_is_gone() {
|
||||
// A culled library leaves thumbnail jobs for images deleted months
|
||||
// ago. Every one would be claimed, run and failed.
|
||||
let c = db();
|
||||
queued(&c, JobKind::Thumbnail, 1);
|
||||
queued(&c, JobKind::Thumbnail, 2);
|
||||
c.execute("DELETE FROM images WHERE id = 2", []).unwrap();
|
||||
|
||||
let recovered = recover(&c).unwrap();
|
||||
assert_eq!(recovered.reaped, 1);
|
||||
assert!(recovered.did_anything());
|
||||
|
||||
let left: i64 = c
|
||||
.query_row("SELECT subject_id FROM jobs", [], |r| r.get(0))
|
||||
.unwrap();
|
||||
assert_eq!(left, 1, "only the job whose subject survives is kept");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_quiet_startup_recovers_nothing() {
|
||||
let c = db();
|
||||
queued(&c, JobKind::Thumbnail, 1);
|
||||
assert_eq!(recover(&c).unwrap(), Recovered::default());
|
||||
assert!(!recover(&c).unwrap().did_anything());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn two_runners_on_one_catalog_never_take_the_same_job() {
|
||||
// Sequential rather than threaded, so the property is asserted without
|
||||
// depending on the scheduler: whatever the second connection claims,
|
||||
// it is not what the first one is holding.
|
||||
let path = temp_catalog("contention-pair");
|
||||
let a = open(&path);
|
||||
let b = open(&path);
|
||||
|
||||
for id in 1..=2 {
|
||||
queued(&a, JobKind::Thumbnail, id);
|
||||
}
|
||||
|
||||
let first = jobs::claim_next(&a, 0).unwrap().expect("one for A");
|
||||
let second = jobs::claim_next(&b, 0).unwrap().expect("one for B");
|
||||
|
||||
assert_ne!(first.id, second.id);
|
||||
assert!(
|
||||
jobs::claim_next(&a, 0).unwrap().is_none(),
|
||||
"a claimed job is invisible to every connection, not just its own"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn concurrent_runners_share_the_queue_without_repeating_work() {
|
||||
// The claim is one atomic statement precisely so this holds: four
|
||||
// threads, four connections, and every job run exactly once.
|
||||
const THREADS: usize = 4;
|
||||
const JOBS: i64 = 24;
|
||||
|
||||
let path = temp_catalog("contention-threads");
|
||||
let seeder = open(&path);
|
||||
for id in 1..=JOBS {
|
||||
queued(&seeder, JobKind::Thumbnail, id);
|
||||
}
|
||||
drop(seeder);
|
||||
|
||||
let seen = Arc::new(Mutex::new(Vec::new()));
|
||||
let mut threads = Vec::new();
|
||||
for _ in 0..THREADS {
|
||||
let path = path.clone();
|
||||
let seen = seen.clone();
|
||||
threads.push(std::thread::spawn(move || {
|
||||
let conn = open(&path);
|
||||
// Bound to a local rather than left as the block's tail: the
|
||||
// `Runner` borrows `conn`, and a tail expression's temporaries
|
||||
// are dropped *after* the block's locals, so the borrow would
|
||||
// outlive what it borrows.
|
||||
let completed = Runner::new(&conn)
|
||||
.with(recording(vec![JobKind::Thumbnail], seen))
|
||||
.drain_all(0)
|
||||
.unwrap()
|
||||
.completed;
|
||||
completed
|
||||
}));
|
||||
}
|
||||
|
||||
let completed: usize = threads.into_iter().map(|t| t.join().unwrap()).sum();
|
||||
assert_eq!(completed, JOBS as usize);
|
||||
|
||||
let mut ran = seen.lock().unwrap().clone();
|
||||
ran.sort_unstable();
|
||||
assert_eq!(
|
||||
ran,
|
||||
(1..=JOBS).collect::<Vec<_>>(),
|
||||
"every job exactly once — no duplicate claim, nothing dropped"
|
||||
);
|
||||
|
||||
let leftover = open(&path);
|
||||
assert_eq!(rows(&leftover), 0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn priority_survives_the_runner() {
|
||||
// NFR-ARCH-2: visible work strictly preempts bulk work, and it has to
|
||||
// still be true when the queue is drained through a handler rather
|
||||
// than by hand.
|
||||
let c = db();
|
||||
image(&c, 1);
|
||||
image(&c, 2);
|
||||
enqueue(&c, JobKind::Thumbnail, Some(1), Priority::Background, None).unwrap();
|
||||
enqueue(&c, JobKind::Thumbnail, Some(2), Priority::Interactive, None).unwrap();
|
||||
|
||||
let seen = Arc::new(Mutex::new(Vec::new()));
|
||||
Runner::new(&c)
|
||||
.with(recording(vec![JobKind::Thumbnail], seen.clone()))
|
||||
.drain_all(0)
|
||||
.unwrap();
|
||||
|
||||
assert_eq!(*seen.lock().unwrap(), vec![2, 1]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_handler_sees_the_payload_and_the_attempt_count() {
|
||||
// Both are how a handler decides what to do: the payload is the only
|
||||
// thing that survives from the enqueue site, and the attempt count is
|
||||
// how it can tell a first try from a last one.
|
||||
let c = db();
|
||||
image(&c, 1);
|
||||
enqueue(
|
||||
&c,
|
||||
JobKind::ScanFolder,
|
||||
Some(1),
|
||||
Priority::Background,
|
||||
Some("/lib/2024"),
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
let payload = Arc::new(Mutex::new(None));
|
||||
let recorded = payload.clone();
|
||||
Runner::new(&c)
|
||||
.with(handler(&[JobKind::ScanFolder], move |job| {
|
||||
*recorded.lock().unwrap() = Some((job.payload.clone(), job.attempts));
|
||||
Outcome::Done
|
||||
}))
|
||||
.drain_all(0)
|
||||
.unwrap();
|
||||
|
||||
assert_eq!(
|
||||
*payload.lock().unwrap(),
|
||||
Some((Some("/lib/2024".to_string()), 1))
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -1,237 +0,0 @@
|
||||
//! TRACES: FR-CAT-1 | FR-CAT-9 | NFR-P1
|
||||
//! Incremental scan: the local analogue of ETag pruning.
|
||||
//!
|
||||
//! Nextcloud propagates ETags up the tree, so one request proves a whole
|
||||
//! library unchanged (ARCH §8.4). A filesystem offers no such guarantee — a
|
||||
//! directory's mtime moves when its *direct* entries change and not when a
|
||||
//! grandchild does, so there is no cheap "did anything below here change"
|
||||
//! probe.
|
||||
//!
|
||||
//! Local scan therefore prunes at each level rather than at the root: one
|
||||
//! metadata probe per directory when nothing changed, instead of one per file.
|
||||
//! A 50k-image library in ~2k folders costs 2k probes, which is the difference
|
||||
//! between meeting and missing NFR-P1 on SAF.
|
||||
//!
|
||||
//! This module holds the decision logic and the deletion-sweep rules; walking
|
||||
//! an actual directory belongs to the platform layer, which supplies
|
||||
//! [`DirState`] and [`DirEntry`]. [`crate::walk`] is what puts the two
|
||||
//! together.
|
||||
|
||||
pub use dr_types::{DirEntry, DirState};
|
||||
|
||||
use dr_types::FormatFilter;
|
||||
|
||||
/// What the scanner should do with a directory, before listing it.
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub enum DirAction {
|
||||
/// Contents unchanged. Skip the listing, but still recurse into known
|
||||
/// children — without upward propagation, a deep change is invisible from
|
||||
/// here.
|
||||
RecurseOnly,
|
||||
/// List and reconcile, then recurse.
|
||||
ListAndRecurse,
|
||||
}
|
||||
|
||||
/// Decide whether a directory needs listing.
|
||||
pub fn classify_dir(stored: Option<DirState>, current: DirState) -> DirAction {
|
||||
match stored {
|
||||
Some(s) if s == current => DirAction::RecurseOnly,
|
||||
_ => DirAction::ListAndRecurse,
|
||||
}
|
||||
}
|
||||
|
||||
/// What reconciling one listed entry against the catalog implies.
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub enum EntryAction {
|
||||
/// Not catalogued. Insert at `metadata_state = 1` and queue EXIF.
|
||||
Insert,
|
||||
/// Catalogued and unchanged. The common case, and it must cost nothing.
|
||||
Unchanged,
|
||||
/// Size or mtime moved: re-read metadata, rebuild the thumbnail, and drop
|
||||
/// the content hash, which is no longer valid.
|
||||
Changed,
|
||||
/// Recognised but not a format the user asked to scan for.
|
||||
Ignored,
|
||||
}
|
||||
|
||||
/// What the catalog already holds for a source.
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub struct KnownFile {
|
||||
pub size: u64,
|
||||
pub mtime: i64,
|
||||
}
|
||||
|
||||
/// Classify one listed file.
|
||||
pub fn classify_entry(
|
||||
entry: &DirEntry,
|
||||
known: Option<KnownFile>,
|
||||
formats: &FormatFilter,
|
||||
) -> EntryAction {
|
||||
if !formats.allows_name(&entry.name) {
|
||||
return EntryAction::Ignored;
|
||||
}
|
||||
match known {
|
||||
None => EntryAction::Insert,
|
||||
Some(k) if k.size == entry.size && k.mtime == entry.mtime => EntryAction::Unchanged,
|
||||
Some(_) => EntryAction::Changed,
|
||||
}
|
||||
}
|
||||
|
||||
/// Outcome of a scan, which decides whether pruning may run.
|
||||
///
|
||||
/// `Cancelled` is the default because a scan that has not run has proven
|
||||
/// nothing absent, and every default in this area must fail towards keeping
|
||||
/// photographs.
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)]
|
||||
pub enum ScanOutcome {
|
||||
/// Every reachable folder was visited.
|
||||
Complete,
|
||||
/// The user cancelled. Partial state is valid — jobs are resumable — but
|
||||
/// unvisited folders must not be read as deleted.
|
||||
#[default]
|
||||
Cancelled,
|
||||
/// The root itself could not be opened: drive unplugged, SAF grant
|
||||
/// revoked, share unmounted.
|
||||
RootUnreachable,
|
||||
/// Some subtree failed while the root was fine.
|
||||
PartialFailure,
|
||||
}
|
||||
|
||||
impl ScanOutcome {
|
||||
/// Whether the deletion sweep may run.
|
||||
///
|
||||
/// **The most dangerous decision in the catalog.** The sweep deletes every
|
||||
/// folder not reached by this scan's generation. After an incomplete scan
|
||||
/// that is most of the library, so it runs only on `Complete`.
|
||||
///
|
||||
/// FR-CAT-9 draws exactly this line: a source *proven absent* may leave
|
||||
/// the catalog; a source merely *unreachable* is marked offline and kept,
|
||||
/// with its ratings and edits intact.
|
||||
pub fn may_prune(self) -> bool {
|
||||
matches!(self, ScanOutcome::Complete)
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use dr_types::Format;
|
||||
|
||||
const A: DirState = DirState {
|
||||
mtime: 100,
|
||||
entry_count: 5,
|
||||
};
|
||||
|
||||
#[test]
|
||||
fn unchanged_directory_is_not_listed() {
|
||||
assert_eq!(classify_dir(Some(A), A), DirAction::RecurseOnly);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_never_seen_directory_is_listed() {
|
||||
assert_eq!(classify_dir(None, A), DirAction::ListAndRecurse);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn changed_mtime_forces_a_listing() {
|
||||
let now = DirState { mtime: 101, ..A };
|
||||
assert_eq!(classify_dir(Some(A), now), DirAction::ListAndRecurse);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn entry_count_catches_what_mtime_misses() {
|
||||
// A file added within the same timestamp tick: mtime is unchanged, so
|
||||
// mtime alone would skip this directory and lose the new image.
|
||||
let now = DirState {
|
||||
mtime: 100,
|
||||
entry_count: 6,
|
||||
};
|
||||
assert_eq!(classify_dir(Some(A), now), DirAction::ListAndRecurse);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn unchanged_file_costs_nothing() {
|
||||
let e = DirEntry {
|
||||
name: "IMG_0001.CR3".into(),
|
||||
is_dir: false,
|
||||
size: 30_000_000,
|
||||
mtime: 500,
|
||||
};
|
||||
let known = KnownFile {
|
||||
size: 30_000_000,
|
||||
mtime: 500,
|
||||
};
|
||||
assert_eq!(
|
||||
classify_entry(&e, Some(known), &FormatFilter::all()),
|
||||
EntryAction::Unchanged
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_resaved_file_is_reprocessed() {
|
||||
let e = DirEntry {
|
||||
name: "IMG_0001.CR3".into(),
|
||||
is_dir: false,
|
||||
size: 30_000_001,
|
||||
mtime: 900,
|
||||
};
|
||||
let known = KnownFile {
|
||||
size: 30_000_000,
|
||||
mtime: 500,
|
||||
};
|
||||
assert_eq!(
|
||||
classify_entry(&e, Some(known), &FormatFilter::all()),
|
||||
EntryAction::Changed
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn format_filter_excludes_unwanted_types() {
|
||||
let jpeg = DirEntry {
|
||||
name: "IMG_0001.JPG".into(),
|
||||
is_dir: false,
|
||||
size: 1,
|
||||
mtime: 1,
|
||||
};
|
||||
assert_eq!(
|
||||
classify_entry(&jpeg, None, &FormatFilter::raw_only()),
|
||||
EntryAction::Ignored
|
||||
);
|
||||
assert_eq!(
|
||||
classify_entry(&jpeg, None, &FormatFilter::all()),
|
||||
EntryAction::Insert
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_placeholder_is_catalogued_as_the_image_it_stands_for() {
|
||||
// 121,785 of these in a real synced folder (ARCH §9.0). Each must
|
||||
// enter the catalog as a CR2 marked offline, not be skipped as an
|
||||
// unknown ".nextcloud" type.
|
||||
let stub = DirEntry {
|
||||
name: "_MG_4130.CR2.nextcloud".into(),
|
||||
is_dir: false,
|
||||
size: 1,
|
||||
mtime: 1,
|
||||
};
|
||||
assert_eq!(
|
||||
classify_entry(&stub, None, &FormatFilter::from_formats([Format::Cr2])),
|
||||
EntryAction::Insert
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn pruning_requires_a_complete_scan() {
|
||||
assert!(ScanOutcome::Complete.may_prune());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn an_unreachable_root_never_prunes() {
|
||||
// The guard that stops an unplugged drive from deleting the library:
|
||||
// every folder would look unreached, so the sweep would take all of
|
||||
// them (FR-CAT-9).
|
||||
assert!(!ScanOutcome::RootUnreachable.may_prune());
|
||||
assert!(!ScanOutcome::Cancelled.may_prune());
|
||||
assert!(!ScanOutcome::PartialFailure.may_prune());
|
||||
}
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
@@ -1,383 +0,0 @@
|
||||
//! TRACES: FR-CAT-7 | FR-NC-9 | NFR-R1
|
||||
//! Preparing the catalog file for upload, and taking in a remote one.
|
||||
//!
|
||||
//! # The hazard this module exists to handle
|
||||
//!
|
||||
//! A WAL-mode SQLite database is not one file. Committed transactions can live
|
||||
//! in `catalog.sqlite-wal` with the main file lagging behind, so copying
|
||||
//! `catalog.sqlite` alone uploads a **torn snapshot**: internally consistent as
|
||||
//! of some older point, missing everything since. Worse, a naive copy taken
|
||||
//! while a writer is mid-transaction can be structurally corrupt.
|
||||
//!
|
||||
//! So an upload never copies the live file. It runs a TRUNCATE checkpoint to
|
||||
//! fold the WAL back into the main file, then uses SQLite's own backup API to
|
||||
//! take a consistent snapshot — which serialises correctly against concurrent
|
||||
//! writers rather than racing them.
|
||||
//!
|
||||
//! # What is actually synced
|
||||
//!
|
||||
//! Only the *user's judgements about their library* merge: collections, and the
|
||||
//! keyword vocabulary with its assignments (see [`crate::merge`]). The rest of
|
||||
//! the catalog is a *local index* of *local* storage — folder mtimes, cache
|
||||
//! paths, job rows — and copying another device's version of those in would be
|
||||
//! actively wrong. The remote file is read for those two and then discarded.
|
||||
//!
|
||||
//! This is why the catalog remains disposable in the ARCH §6.12 sense: nothing
|
||||
//! here makes the local database authoritative for anything a rebuild could
|
||||
//! not recover.
|
||||
|
||||
use std::path::{Path, PathBuf};
|
||||
|
||||
use rusqlite::Connection;
|
||||
|
||||
use crate::error::CatalogError;
|
||||
use crate::merge::{self, MergeReport};
|
||||
|
||||
/// Schema name the downloaded remote catalog is attached under.
|
||||
const REMOTE_SCHEMA: &str = "remote_cat";
|
||||
|
||||
/// Fold the WAL into the main database file.
|
||||
///
|
||||
/// TRUNCATE rather than PASSIVE: passive checkpointing gives up when a reader
|
||||
/// holds the WAL open, which would leave recent commits out of the snapshot
|
||||
/// without saying so.
|
||||
pub fn checkpoint(conn: &Connection) -> Result<(), CatalogError> {
|
||||
conn.pragma_update(None, "wal_checkpoint", "TRUNCATE")?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Write a consistent snapshot of the catalog to `dest`, ready to upload.
|
||||
///
|
||||
/// Uses the backup API rather than a filesystem copy so the snapshot is
|
||||
/// coherent even with writers active. Callers should still prefer a quiet
|
||||
/// moment — this competes with background jobs for the write lock.
|
||||
pub fn snapshot_for_upload(conn: &Connection, dest: &Path) -> Result<(), CatalogError> {
|
||||
let out = copy_to(conn, dest)?;
|
||||
strip_face_crops(&out)?;
|
||||
verify_snapshot(&out)?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// TRACES: NFR-R2
|
||||
/// Refuse to hand over a snapshot that will not pass `quick_check`.
|
||||
///
|
||||
/// The upload is the copy every other device merges from, and a damaged one
|
||||
/// costs far more than the check: each device downloads it, fails, and — for
|
||||
/// a week, once — declines to push over it. `quick_check` reads every page
|
||||
/// but skips index verification, which is the affordable version of "is this
|
||||
/// a database" on a 40 MB file that has just been written and is still in the
|
||||
/// page cache. A failure here is [`CatalogError::Corrupt`], the same thing a
|
||||
/// receiving device would have said, so the sync reports it the same way.
|
||||
fn verify_snapshot(snapshot: &Connection) -> Result<(), CatalogError> {
|
||||
let verdict: String = snapshot.query_row("PRAGMA quick_check", [], |r| r.get(0))?;
|
||||
if verdict == "ok" {
|
||||
Ok(())
|
||||
} else {
|
||||
Err(CatalogError::Corrupt {
|
||||
detail: format!("the snapshot for upload failed quick_check: {verdict}"),
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
/// Checkpoint, then copy the whole database to `dest`, and hand back the
|
||||
/// connection to the copy.
|
||||
///
|
||||
/// Split out from [`snapshot_for_upload`] because [`crate::recovery`] wants
|
||||
/// exactly this and none of what follows it there: an NFR-R2 backup is the
|
||||
/// file the user may have to *live on*, so it keeps the face crops that an
|
||||
/// upload strips. Sharing the copy rather than reimplementing it is what keeps
|
||||
/// the WAL discipline in one place — a backup taken with `fs::copy` would be
|
||||
/// the torn snapshot this module's header exists to warn about.
|
||||
pub(crate) fn copy_to(conn: &Connection, dest: &Path) -> Result<Connection, CatalogError> {
|
||||
checkpoint(conn)?;
|
||||
|
||||
let mut out = Connection::open(dest)?;
|
||||
let backup = rusqlite::backup::Backup::new(conn, &mut out)?;
|
||||
// SQLite's own "copy everything" sentinel is -1, but rusqlite asserts a
|
||||
// positive page count, so ask for more pages than a catalog will ever
|
||||
// have. The effect is the same: one step, no interleaved writers, no
|
||||
// progress callback. A 50k-image catalog is tens of megabytes.
|
||||
backup.run_to_completion(i32::MAX, std::time::Duration::ZERO, None)?;
|
||||
drop(backup);
|
||||
|
||||
Ok(out)
|
||||
}
|
||||
|
||||
/// Drop the stored face crops from a snapshot before it is uploaded.
|
||||
///
|
||||
/// The snapshot is the *whole catalog*, uploaded on every sync and downloaded
|
||||
/// by every device. Face crops are a few KB each and a fully indexed library
|
||||
/// holds tens of thousands of them, so leaving them in would put tens of MB on
|
||||
/// every round trip — the exact cost `face_shard`'s 25 MB cap exists to bound,
|
||||
/// and the reason the bulk per-face data lives in shards in the first place.
|
||||
///
|
||||
/// Crops are not lost by this: they travel in the face shards
|
||||
/// ([`crate::face_shard::export_to_shards`]), which are written once and
|
||||
/// downloaded once. Nothing reads a crop out of a merged remote catalog —
|
||||
/// [`merge_all`] touches collections and keywords only — so removing them here
|
||||
/// costs a receiving device nothing it would otherwise have had.
|
||||
///
|
||||
/// `VACUUM` afterwards because SQLite does not return freed pages to the file
|
||||
/// on its own, and an upload sized by the file rather than by its contents
|
||||
/// would keep paying for bytes that are no longer there.
|
||||
fn strip_face_crops(snapshot: &Connection) -> Result<(), CatalogError> {
|
||||
// A catalog older than the crop column is a legitimate input here — a
|
||||
// snapshot taken mid-migration, or a test fixture built from an earlier
|
||||
// schema — so an absent column is nothing to fail over.
|
||||
let has_crop = snapshot
|
||||
.prepare("SELECT crop FROM faces LIMIT 1")
|
||||
.map(|_| true)
|
||||
.unwrap_or(false);
|
||||
if !has_crop {
|
||||
return Ok(());
|
||||
}
|
||||
snapshot.execute("UPDATE faces SET crop = NULL WHERE crop IS NOT NULL", [])?;
|
||||
snapshot.execute_batch("VACUUM")?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Whether a downloaded remote catalog is worth merging.
|
||||
///
|
||||
/// Cheap guard before attaching: a remote written by a newer build may contain
|
||||
/// tables and columns this one cannot read, and attempting the merge would
|
||||
/// fail mid-transaction rather than declining cleanly.
|
||||
pub fn remote_is_mergeable(remote: &Path) -> Result<bool, CatalogError> {
|
||||
let conn = Connection::open_with_flags(
|
||||
remote,
|
||||
rusqlite::OpenFlags::SQLITE_OPEN_READ_ONLY | rusqlite::OpenFlags::SQLITE_OPEN_NO_MUTEX,
|
||||
)?;
|
||||
let v: i64 = conn.query_row("PRAGMA user_version", [], |r| r.get(0))?;
|
||||
Ok(v <= crate::schema::SCHEMA_VERSION)
|
||||
}
|
||||
|
||||
/// Attach a downloaded remote catalog, merge its collections, detach.
|
||||
///
|
||||
/// The remote file is opened **read-only** — this device never writes to
|
||||
/// another device's catalog, it only reads collections out of it.
|
||||
pub fn merge_remote(conn: &Connection, remote: &Path) -> Result<MergeReport, CatalogError> {
|
||||
if !remote_is_mergeable(remote)? {
|
||||
return Err(CatalogError::SchemaTooNew {
|
||||
found: -1,
|
||||
supported: crate::schema::SCHEMA_VERSION,
|
||||
});
|
||||
}
|
||||
|
||||
// Path binds as a parameter; ATTACH accepts one, so a path containing a
|
||||
// quote cannot break out into SQL.
|
||||
conn.execute(
|
||||
&format!("ATTACH DATABASE ?1 AS {REMOTE_SCHEMA}"),
|
||||
[remote.to_string_lossy().as_ref()],
|
||||
)?;
|
||||
|
||||
let result = merge::merge_all(conn);
|
||||
|
||||
// Detach even if the merge failed, or the next attempt errors with
|
||||
// "database remote_cat is already in use".
|
||||
let detach = conn.execute(&format!("DETACH DATABASE {REMOTE_SCHEMA}"), []);
|
||||
if let Err(e) = detach {
|
||||
log::warn!("failed to detach remote catalog: {e}");
|
||||
}
|
||||
|
||||
result
|
||||
}
|
||||
|
||||
/// Where the catalog snapshot and the downloaded remote live.
|
||||
///
|
||||
/// Both are transient working files, not the catalog itself, so they belong in
|
||||
/// the cache directory rather than beside the live database.
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct SyncPaths {
|
||||
pub upload_snapshot: PathBuf,
|
||||
pub downloaded_remote: PathBuf,
|
||||
}
|
||||
|
||||
impl SyncPaths {
|
||||
pub fn in_dir(cache_dir: &Path) -> Self {
|
||||
SyncPaths {
|
||||
upload_snapshot: cache_dir.join("catalog-upload.sqlite"),
|
||||
downloaded_remote: cache_dir.join("catalog-remote.sqlite"),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use crate::schema;
|
||||
|
||||
fn seeded(path: &Path) -> Connection {
|
||||
let c = Connection::open(path).unwrap();
|
||||
schema::configure(&c).unwrap();
|
||||
schema::migrate(&c).unwrap();
|
||||
c
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn snapshot_captures_committed_data() {
|
||||
let dir = tempdir();
|
||||
let live = dir.join("catalog.sqlite");
|
||||
let snap = dir.join("snap.sqlite");
|
||||
|
||||
let c = seeded(&live);
|
||||
c.execute(
|
||||
"INSERT INTO collections(uuid, name, kind, created, revision, modified)
|
||||
VALUES ('u1', 'Iceland', 0, 0, 1, 1)",
|
||||
[],
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
snapshot_for_upload(&c, &snap).unwrap();
|
||||
|
||||
// The snapshot must hold the row even though it was written after the
|
||||
// database was created — the torn-file failure this guards against.
|
||||
let s = Connection::open(&snap).unwrap();
|
||||
let name: String = s
|
||||
.query_row("SELECT name FROM collections", [], |r| r.get(0))
|
||||
.unwrap();
|
||||
assert_eq!(name, "Iceland");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_remote_from_a_newer_build_is_declined_not_attempted() {
|
||||
let dir = tempdir();
|
||||
let remote = dir.join("remote.sqlite");
|
||||
let r = seeded(&remote);
|
||||
r.pragma_update(None, "user_version", schema::SCHEMA_VERSION + 1)
|
||||
.unwrap();
|
||||
drop(r);
|
||||
|
||||
assert!(!remote_is_mergeable(&remote).unwrap());
|
||||
|
||||
let local = seeded(&dir.join("local.sqlite"));
|
||||
assert!(matches!(
|
||||
merge_remote(&local, &remote),
|
||||
Err(CatalogError::SchemaTooNew { .. })
|
||||
));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn merge_remote_round_trips_a_collection() {
|
||||
let dir = tempdir();
|
||||
let remote_path = dir.join("remote.sqlite");
|
||||
{
|
||||
let r = seeded(&remote_path);
|
||||
r.execute(
|
||||
"INSERT INTO collections(uuid, name, kind, created, revision, modified)
|
||||
VALUES ('u-remote', 'Portugal', 0, 0, 1, 1)",
|
||||
[],
|
||||
)
|
||||
.unwrap();
|
||||
checkpoint(&r).unwrap();
|
||||
}
|
||||
|
||||
let local = seeded(&dir.join("local.sqlite"));
|
||||
local
|
||||
.execute(
|
||||
"INSERT INTO collections(uuid, name, kind, created, revision, modified)
|
||||
VALUES ('u-local', 'Iceland', 0, 0, 1, 1)",
|
||||
[],
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
let report = merge_remote(&local, &remote_path).unwrap();
|
||||
assert_eq!(report.inserted, 1);
|
||||
|
||||
let n: i64 = local
|
||||
.query_row("SELECT count(*) FROM collections", [], |r| r.get(0))
|
||||
.unwrap();
|
||||
assert_eq!(n, 2);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn the_remote_can_be_merged_twice_without_attach_conflict() {
|
||||
// Detach must happen even on the failure path, or the second attempt
|
||||
// errors with "database remote_cat is already in use".
|
||||
let dir = tempdir();
|
||||
let remote_path = dir.join("remote.sqlite");
|
||||
{
|
||||
let r = seeded(&remote_path);
|
||||
r.execute(
|
||||
"INSERT INTO collections(uuid, name, kind, created, revision, modified)
|
||||
VALUES ('u-remote', 'Portugal', 0, 0, 1, 1)",
|
||||
[],
|
||||
)
|
||||
.unwrap();
|
||||
checkpoint(&r).unwrap();
|
||||
}
|
||||
let local = seeded(&dir.join("local.sqlite"));
|
||||
|
||||
merge_remote(&local, &remote_path).unwrap();
|
||||
let second = merge_remote(&local, &remote_path).unwrap();
|
||||
assert!(!second.local_changed());
|
||||
}
|
||||
|
||||
/// A scratch directory that cleans up with the test.
|
||||
fn tempdir() -> PathBuf {
|
||||
let base = std::env::temp_dir().join(format!(
|
||||
"dr-catalog-test-{}-{:?}",
|
||||
std::process::id(),
|
||||
std::thread::current().id()
|
||||
));
|
||||
let _ = std::fs::remove_dir_all(&base);
|
||||
std::fs::create_dir_all(&base).unwrap();
|
||||
base
|
||||
}
|
||||
|
||||
/// The whole reason crops live in the shards: a snapshot is uploaded whole,
|
||||
/// on every sync, to every device.
|
||||
#[test]
|
||||
fn the_snapshot_carries_no_face_crops() {
|
||||
let dir = tempdir();
|
||||
let live = dir.join("catalog.sqlite");
|
||||
let snap = dir.join("snap.sqlite");
|
||||
|
||||
let c = seeded(&live);
|
||||
c.execute(
|
||||
"INSERT OR IGNORE INTO roots(id, kind, label) VALUES (1, 'local', 'lib')",
|
||||
[],
|
||||
)
|
||||
.unwrap();
|
||||
c.execute(
|
||||
"INSERT INTO images(id, root_id, source_ref, added_at) VALUES (1, 1, 'a.CR3', 0)",
|
||||
[],
|
||||
)
|
||||
.unwrap();
|
||||
c.execute(
|
||||
"INSERT INTO faces
|
||||
(image_id, x, y, w, h, landmarks, detector_confidence, embedding,
|
||||
crop_px, model_id, detected_at, crop)
|
||||
VALUES (1, 0.1, 0.1, 0.2, 0.2, X'00', 0.9, X'00', 180.0, 'm', 0, ?1)",
|
||||
[vec![7u8; 4096]],
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
snapshot_for_upload(&c, &snap).unwrap();
|
||||
|
||||
let out = Connection::open(&snap).unwrap();
|
||||
let crops: i64 = out
|
||||
.query_row(
|
||||
"SELECT COUNT(*) FROM faces WHERE crop IS NOT NULL",
|
||||
[],
|
||||
|r| r.get(0),
|
||||
)
|
||||
.unwrap();
|
||||
assert_eq!(crops, 0, "the snapshot still carries face crops");
|
||||
|
||||
// The face itself must still be there — only the pixels are dropped.
|
||||
let faces: i64 = out
|
||||
.query_row("SELECT COUNT(*) FROM faces", [], |r| r.get(0))
|
||||
.unwrap();
|
||||
assert_eq!(faces, 1);
|
||||
|
||||
// And the local catalog keeps its crop: this strips the copy, never
|
||||
// the original.
|
||||
let kept: i64 = c
|
||||
.query_row(
|
||||
"SELECT COUNT(*) FROM faces WHERE crop IS NOT NULL",
|
||||
[],
|
||||
|r| r.get(0),
|
||||
)
|
||||
.unwrap();
|
||||
assert_eq!(kept, 1, "stripping the snapshot damaged the live catalog");
|
||||
}
|
||||
}
|
||||
@@ -1,636 +0,0 @@
|
||||
//! TRACES: FR-CAT-15 | NFR-R2
|
||||
//! Soft delete, restore, and the permanent delete that follows.
|
||||
//!
|
||||
//! # Why the trash is a folder and not a flag
|
||||
//!
|
||||
//! The catalog is a *rebuildable index* (ARCH §6.12): delete `catalog.sqlite`
|
||||
//! and it is reconstructed by rescanning sources. A trash implemented as a
|
||||
//! column alone would therefore not survive its own design — a rebuild would
|
||||
//! find every trashed file still sitting in the library and re-index it as an
|
||||
//! ordinary photograph, silently undoing every delete the user had made.
|
||||
//!
|
||||
//! So a soft delete **moves the file** into `.darkroom-trash/` under the library
|
||||
//! root, and the catalog merely records that this happened. The folder is the
|
||||
//! durable fact; the row is the convenience. Recovering by hand needs no
|
||||
//! DarkRoom at all, which is the property that matters when the thing being
|
||||
//! risked is a photograph.
|
||||
//!
|
||||
//! `dr_sync::scan::is_excluded` keeps the scanner out of that folder. Without
|
||||
//! it the next scan re-indexes the trash and the delete comes undone — the two
|
||||
//! halves are one mechanism and neither works alone.
|
||||
//!
|
||||
//! # The two steps
|
||||
//!
|
||||
//! **Soft** ([`trash`]) — `MOVE` to the trash folder, record `trashed_at` and
|
||||
//! the path it came from. Reversible by [`restore`], which is why the original
|
||||
//! path has to be remembered: the trash is flat, and the folder structure cannot
|
||||
//! be recovered from the trashed name.
|
||||
//!
|
||||
//! **Hard** ([`purge`]) — `DELETE` the file, then delete the row. Irreversible
|
||||
//! from DarkRoom's side, though the server's own trashbin may still hold it.
|
||||
//! Ordered file-first deliberately: see [`purge_order`].
|
||||
//!
|
||||
//! # What this module does not do
|
||||
//!
|
||||
//! It performs no I/O. Every function here records or reads catalog state, and
|
||||
//! the caller pairs it with the remote operation — because the remote call is
|
||||
//! async and the catalog is not, and because the *order* of the two is a
|
||||
//! correctness property that belongs in one visible place rather than buried in
|
||||
//! a transaction.
|
||||
|
||||
use rusqlite::{Connection, OptionalExtension};
|
||||
|
||||
use dr_types::ImageId;
|
||||
|
||||
use crate::error::CatalogError;
|
||||
|
||||
/// Directory holding soft-deleted images, under the library root.
|
||||
///
|
||||
/// The same constant `dr_sync::scan` excludes. Duplicated as a `const` here
|
||||
/// rather than depended upon because `dr-catalog` does not (and should not)
|
||||
/// depend on `dr-sync`; the pairing is asserted by a test.
|
||||
pub const TRASH_DIR: &str = ".darkroom-trash";
|
||||
|
||||
/// One trashed image, as the trash view lists it.
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct TrashedImage {
|
||||
pub image_id: ImageId,
|
||||
/// Where the file is *now* — inside the trash folder.
|
||||
pub source_ref: String,
|
||||
/// Where it was before, and where [`restore`] will put it back.
|
||||
pub trashed_from: String,
|
||||
/// UTC seconds when it was trashed.
|
||||
pub trashed_at: i64,
|
||||
/// `oc:fileid`, preserved across the move. What the thumbnail store keys on,
|
||||
/// and what makes a restore free rather than a re-download.
|
||||
pub file_id: Option<u64>,
|
||||
pub size: u64,
|
||||
}
|
||||
|
||||
/// The path a soft-deleted image should be moved to.
|
||||
///
|
||||
/// Flat: the trash is a holding area, not an archive, and mirroring the library
|
||||
/// tree inside it would mean creating directories on the way to deleting things.
|
||||
/// The original path is remembered in the catalog instead, which is what
|
||||
/// [`restore`] reads.
|
||||
///
|
||||
/// **Collisions are resolved rather than allowed to overwrite.** Two files named
|
||||
/// `IMG_0001.CR2` from different folders are different photographs, and a `MOVE`
|
||||
/// onto an existing name would destroy one of them — the precise failure a trash
|
||||
/// exists to prevent. The image id disambiguates, and being already unique it
|
||||
/// needs no retry loop.
|
||||
pub fn trash_path(root: &str, image: ImageId, original: &str) -> String {
|
||||
let name = original.rsplit(['/', ':']).next().unwrap_or(original);
|
||||
let prefix = if root.is_empty() {
|
||||
String::new()
|
||||
} else {
|
||||
format!("{root}/")
|
||||
};
|
||||
format!("{prefix}{TRASH_DIR}/{}-{name}", image.0)
|
||||
}
|
||||
|
||||
/// Where a trashed image goes back to.
|
||||
///
|
||||
/// The stored original path, verbatim. Returns `None` where the image is not
|
||||
/// trashed, so a caller cannot restore something that was never deleted.
|
||||
pub fn restore_path(conn: &Connection, image: ImageId) -> Result<Option<String>, CatalogError> {
|
||||
let path: Option<String> = conn
|
||||
.query_row(
|
||||
"SELECT trashed_from FROM images
|
||||
WHERE id = ?1 AND trashed_at IS NOT NULL",
|
||||
[image.0 as i64],
|
||||
|r| r.get(0),
|
||||
)
|
||||
.optional()?
|
||||
.flatten();
|
||||
Ok(path)
|
||||
}
|
||||
|
||||
/// Record that images have been moved to the trash.
|
||||
///
|
||||
/// Call **after** the move succeeds. Recording first and moving second would
|
||||
/// leave the catalog claiming a file is trashed while it sits in the library,
|
||||
/// where the next scan finds it — and since the scan excludes the trash folder,
|
||||
/// the row would never be corrected.
|
||||
///
|
||||
/// `moved` pairs each image with the path it now occupies, which is what
|
||||
/// [`trash_path`] produced for it.
|
||||
///
|
||||
/// Idempotent on `trashed_at`: re-trashing an already-trashed image keeps the
|
||||
/// *original* timestamp and original path, so a retry after a partial failure
|
||||
/// cannot rewrite `trashed_from` to a path inside the trash — which would make
|
||||
/// the image unrestorable.
|
||||
pub fn record_trashed(
|
||||
conn: &Connection,
|
||||
moved: &[(ImageId, String)],
|
||||
now: i64,
|
||||
) -> Result<usize, CatalogError> {
|
||||
if moved.is_empty() {
|
||||
return Ok(0);
|
||||
}
|
||||
let tx = conn.unchecked_transaction()?;
|
||||
let mut n = 0;
|
||||
|
||||
{
|
||||
let mut stmt = tx.prepare(
|
||||
"UPDATE images
|
||||
SET trashed_from = CASE
|
||||
WHEN trashed_at IS NULL THEN source_ref
|
||||
ELSE trashed_from
|
||||
END,
|
||||
source_ref = ?2,
|
||||
trashed_at = coalesce(trashed_at, ?3)
|
||||
WHERE id = ?1",
|
||||
)?;
|
||||
for (image, path) in moved {
|
||||
n += stmt.execute(rusqlite::params![image.0 as i64, path, now])?;
|
||||
}
|
||||
}
|
||||
|
||||
tx.commit()?;
|
||||
Ok(n)
|
||||
}
|
||||
|
||||
/// Record that images have been moved back out of the trash.
|
||||
///
|
||||
/// Call after the move succeeds, for the same reason as [`record_trashed`].
|
||||
/// Clears both columns: a restored image is an ordinary one, and leaving
|
||||
/// `trashed_from` set would make the next trash-and-restore cycle restore it to
|
||||
/// a stale location.
|
||||
pub fn record_restored(
|
||||
conn: &Connection,
|
||||
restored: &[(ImageId, String)],
|
||||
) -> Result<usize, CatalogError> {
|
||||
if restored.is_empty() {
|
||||
return Ok(0);
|
||||
}
|
||||
let tx = conn.unchecked_transaction()?;
|
||||
let mut n = 0;
|
||||
|
||||
{
|
||||
let mut stmt = tx.prepare(
|
||||
"UPDATE images
|
||||
SET source_ref = ?2, trashed_at = NULL, trashed_from = NULL
|
||||
WHERE id = ?1 AND trashed_at IS NOT NULL",
|
||||
)?;
|
||||
for (image, path) in restored {
|
||||
n += stmt.execute(rusqlite::params![image.0 as i64, path])?;
|
||||
}
|
||||
}
|
||||
|
||||
tx.commit()?;
|
||||
Ok(n)
|
||||
}
|
||||
|
||||
/// Forget images whose files have been permanently deleted.
|
||||
///
|
||||
/// Call **after** the remote delete succeeds — see [`purge_order`].
|
||||
///
|
||||
/// Deletes the catalog rows outright rather than tombstoning them. There is
|
||||
/// nothing to merge: unlike a collection, an image row is derived from a file
|
||||
/// that no longer exists, so a rescan on another device will not reintroduce it
|
||||
/// and needs no tombstone to be told so. `ON DELETE CASCADE` takes the versions,
|
||||
/// keywords, remote mapping and cache rows with it.
|
||||
///
|
||||
/// Returns how many rows went.
|
||||
pub fn forget(conn: &Connection, images: &[ImageId]) -> Result<usize, CatalogError> {
|
||||
if images.is_empty() {
|
||||
return Ok(0);
|
||||
}
|
||||
let tx = conn.unchecked_transaction()?;
|
||||
let mut n = 0;
|
||||
{
|
||||
let mut stmt = tx.prepare("DELETE FROM images WHERE id = ?1")?;
|
||||
for image in images {
|
||||
n += stmt.execute([image.0 as i64])?;
|
||||
}
|
||||
}
|
||||
tx.commit()?;
|
||||
Ok(n)
|
||||
}
|
||||
|
||||
/// Why the file is deleted before the row.
|
||||
///
|
||||
/// Not a function — a note with a name, so the reasoning is findable from the
|
||||
/// call site.
|
||||
///
|
||||
/// **File first, then the row.** If the delete succeeds and the process dies
|
||||
/// before the row goes, the catalog holds a trashed row whose file is gone; the
|
||||
/// user sees it in the trash, empties again, gets a `404`, and it is treated as
|
||||
/// already-deleted (see [`is_already_gone`]). Recoverable, and visible.
|
||||
///
|
||||
/// The other order loses the file silently. Dropping the row first and dying
|
||||
/// before the delete leaves an orphan in `.darkroom-trash/` that nothing in the
|
||||
/// UI lists, nothing counts, and no scan will ever find — because the scanner
|
||||
/// excludes that folder. It consumes quota forever and the user has no way to
|
||||
/// learn it is there.
|
||||
pub const fn purge_order() {}
|
||||
|
||||
/// Whether a delete failure means the file was already gone.
|
||||
///
|
||||
/// A `404` on the way to deleting something is success: the goal state is
|
||||
/// "this file does not exist", and it does not. Treating it as an error would
|
||||
/// wedge an empty-trash operation on a file the user had removed by hand, and
|
||||
/// no amount of retrying would clear it.
|
||||
pub fn is_already_gone(status: Option<u16>) -> bool {
|
||||
matches!(status, Some(404) | Some(410))
|
||||
}
|
||||
|
||||
/// List what is in the trash, newest first.
|
||||
///
|
||||
/// Newest first because the trash is reviewed to undo a recent mistake, not
|
||||
/// browsed chronologically.
|
||||
pub fn list(conn: &Connection, limit: usize) -> Result<Vec<TrashedImage>, CatalogError> {
|
||||
let mut stmt = conn.prepare(
|
||||
"SELECT i.id, i.source_ref, i.trashed_from, i.trashed_at, r.file_id, i.file_size
|
||||
FROM images i
|
||||
LEFT JOIN remote r ON r.image_id = i.id
|
||||
WHERE i.trashed_at IS NOT NULL
|
||||
ORDER BY i.trashed_at DESC, i.id DESC
|
||||
LIMIT ?1",
|
||||
)?;
|
||||
let rows = stmt
|
||||
.query_map([limit as i64], |r| {
|
||||
let source_ref: String = r.get(1)?;
|
||||
Ok(TrashedImage {
|
||||
image_id: ImageId(r.get::<_, i64>(0)? as u64),
|
||||
// A row with no `trashed_from` predates nothing — it cannot
|
||||
// happen through this module — but a hand-edited or
|
||||
// partially-migrated catalog could produce one. Falling back to
|
||||
// the current path keeps it listed and deletable rather than
|
||||
// invisible; a restore to the trash folder is a no-op the user
|
||||
// can see, where a hidden row is not.
|
||||
trashed_from: r
|
||||
.get::<_, Option<String>>(2)?
|
||||
.unwrap_or_else(|| source_ref.clone()),
|
||||
source_ref,
|
||||
trashed_at: r.get(3)?,
|
||||
file_id: r.get::<_, Option<i64>>(4)?.map(|v| v as u64),
|
||||
size: r.get::<_, Option<i64>>(5)?.unwrap_or(0) as u64,
|
||||
})
|
||||
})?
|
||||
.collect::<Result<Vec<_>, _>>()?;
|
||||
Ok(rows)
|
||||
}
|
||||
|
||||
/// Every trashed image id, for emptying the whole trash.
|
||||
///
|
||||
/// Separate from [`list`] because emptying needs all of them, not a window, and
|
||||
/// wants no per-row detail.
|
||||
pub fn all_trashed(conn: &Connection) -> Result<Vec<ImageId>, CatalogError> {
|
||||
let mut stmt = conn.prepare("SELECT id FROM images WHERE trashed_at IS NOT NULL")?;
|
||||
let rows = stmt
|
||||
.query_map([], |r| Ok(ImageId(r.get::<_, i64>(0)? as u64)))?
|
||||
.collect::<Result<Vec<_>, _>>()?;
|
||||
Ok(rows)
|
||||
}
|
||||
|
||||
/// How many images are in the trash, and how many bytes they hold.
|
||||
///
|
||||
/// The bytes are the point: "empty trash" is a destructive action, and the
|
||||
/// amount being freed is what tells the user whether they meant it.
|
||||
pub fn summary(conn: &Connection) -> Result<(usize, u64), CatalogError> {
|
||||
let (n, bytes): (i64, i64) = conn.query_row(
|
||||
"SELECT count(*), coalesce(sum(file_size), 0)
|
||||
FROM images WHERE trashed_at IS NOT NULL",
|
||||
[],
|
||||
|r| Ok((r.get(0)?, r.get(1)?)),
|
||||
)?;
|
||||
Ok((n as usize, bytes as u64))
|
||||
}
|
||||
|
||||
/// `oc:fileid`s of trashed images, so their thumbnails can be dropped.
|
||||
///
|
||||
/// The thumbnail store is keyed on the stable file id and shared with other
|
||||
/// clients, so a purge that left its entries behind would keep serving previews
|
||||
/// of photographs that no longer exist — and the shards sync, so it would keep
|
||||
/// doing so on every other device too.
|
||||
pub fn file_ids_for(conn: &Connection, images: &[ImageId]) -> Result<Vec<u64>, CatalogError> {
|
||||
if images.is_empty() {
|
||||
return Ok(Vec::new());
|
||||
}
|
||||
let placeholders = std::iter::repeat_n("?", images.len())
|
||||
.collect::<Vec<_>>()
|
||||
.join(",");
|
||||
let sql = format!("SELECT file_id FROM remote WHERE image_id IN ({placeholders})");
|
||||
let params: Vec<rusqlite::types::Value> = images
|
||||
.iter()
|
||||
.map(|i| rusqlite::types::Value::Integer(i.0 as i64))
|
||||
.collect();
|
||||
|
||||
let mut stmt = conn.prepare(&sql)?;
|
||||
let rows = stmt
|
||||
.query_map(rusqlite::params_from_iter(params.iter()), |r| {
|
||||
Ok(r.get::<_, i64>(0)? as u64)
|
||||
})?
|
||||
.collect::<Result<Vec<_>, _>>()?;
|
||||
Ok(rows)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use crate::Catalog;
|
||||
|
||||
fn seeded() -> Catalog {
|
||||
let cat = Catalog::in_memory().unwrap();
|
||||
let c = cat.connection();
|
||||
c.execute(
|
||||
"INSERT INTO roots(id, kind, label) VALUES (1, 'remote', 'PhotosRaw')",
|
||||
[],
|
||||
)
|
||||
.unwrap();
|
||||
for i in 1..=4i64 {
|
||||
c.execute(
|
||||
"INSERT INTO images(id, root_id, source_ref, file_size, added_at)
|
||||
VALUES (?1, 1, ?2, ?3, 0)",
|
||||
rusqlite::params![i, format!("PhotosRaw/2019/IMG_{i:04}.CR2"), 30_000_000 * i],
|
||||
)
|
||||
.unwrap();
|
||||
c.execute(
|
||||
"INSERT INTO remote(image_id, file_id) VALUES (?1, ?2)",
|
||||
rusqlite::params![i, 1000 + i],
|
||||
)
|
||||
.unwrap();
|
||||
}
|
||||
cat
|
||||
}
|
||||
|
||||
fn img(i: u64) -> ImageId {
|
||||
ImageId(i)
|
||||
}
|
||||
|
||||
/// Trash one image the way the UI does: compute the path, then record.
|
||||
fn do_trash(cat: &Catalog, i: u64, now: i64) -> String {
|
||||
let c = cat.connection();
|
||||
let original: String = c
|
||||
.query_row(
|
||||
"SELECT source_ref FROM images WHERE id = ?1",
|
||||
[i as i64],
|
||||
|r| r.get(0),
|
||||
)
|
||||
.unwrap();
|
||||
let to = trash_path("PhotosRaw", img(i), &original);
|
||||
record_trashed(c, &[(img(i), to.clone())], now).unwrap();
|
||||
to
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn the_trash_directory_matches_the_one_the_scanner_excludes() {
|
||||
// These are two constants in two crates that must agree, or the scan
|
||||
// re-indexes the trash and every soft delete comes undone.
|
||||
assert_eq!(TRASH_DIR, dr_sync_trash_dir());
|
||||
}
|
||||
|
||||
/// The scanner's constant, quoted rather than imported — `dr-catalog` does
|
||||
/// not depend on `dr-sync`, and adding that dependency for one string would
|
||||
/// invert the layering.
|
||||
fn dr_sync_trash_dir() -> &'static str {
|
||||
".darkroom-trash"
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn trashing_moves_the_path_and_remembers_where_it_came_from() {
|
||||
let cat = seeded();
|
||||
let c = cat.connection();
|
||||
do_trash(&cat, 1, 5_000);
|
||||
|
||||
let (source, from, at): (String, String, i64) = c
|
||||
.query_row(
|
||||
"SELECT source_ref, trashed_from, trashed_at FROM images WHERE id = 1",
|
||||
[],
|
||||
|r| Ok((r.get(0)?, r.get(1)?, r.get(2)?)),
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
// `source_ref` follows the bytes: this is where a fetch must now look.
|
||||
assert!(source.contains(TRASH_DIR), "{source}");
|
||||
// And the original is remembered, or a restore has nowhere to go.
|
||||
assert_eq!(from, "PhotosRaw/2019/IMG_0001.CR2");
|
||||
assert_eq!(at, 5_000);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn the_trash_path_keeps_the_original_filename_recognisable() {
|
||||
// The user reviewing the trash needs to recognise the photograph; an
|
||||
// opaque id alone would make the list unreadable.
|
||||
let p = trash_path("PhotosRaw", img(7), "PhotosRaw/2019/IMG_0042.CR2");
|
||||
assert!(p.ends_with("IMG_0042.CR2"), "{p}");
|
||||
assert!(p.starts_with("PhotosRaw/.darkroom-trash/"), "{p}");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn two_files_with_the_same_name_do_not_collide_in_the_trash() {
|
||||
// The failure a trash exists to prevent: a MOVE onto an existing name
|
||||
// destroys one of two different photographs.
|
||||
let a = trash_path("PhotosRaw", img(1), "PhotosRaw/2019/IMG_0001.CR2");
|
||||
let b = trash_path("PhotosRaw", img(2), "PhotosRaw/2024/IMG_0001.CR2");
|
||||
assert_ne!(a, b);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_whole_account_root_yields_no_leading_slash() {
|
||||
// The root is empty when the library is the whole account; a path
|
||||
// beginning "/" would resolve differently on the server.
|
||||
let p = trash_path("", img(3), "2019/IMG_0003.CR2");
|
||||
assert_eq!(p, ".darkroom-trash/3-IMG_0003.CR2");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn restoring_puts_the_original_path_back_and_clears_the_flag() {
|
||||
let cat = seeded();
|
||||
let c = cat.connection();
|
||||
do_trash(&cat, 1, 5_000);
|
||||
|
||||
let back = restore_path(c, img(1))
|
||||
.unwrap()
|
||||
.expect("knows where it came from");
|
||||
assert_eq!(back, "PhotosRaw/2019/IMG_0001.CR2");
|
||||
|
||||
record_restored(c, &[(img(1), back.clone())]).unwrap();
|
||||
|
||||
let (source, at): (String, Option<i64>) = c
|
||||
.query_row(
|
||||
"SELECT source_ref, trashed_at FROM images WHERE id = 1",
|
||||
[],
|
||||
|r| Ok((r.get(0)?, r.get(1)?)),
|
||||
)
|
||||
.unwrap();
|
||||
assert_eq!(source, back);
|
||||
assert_eq!(at, None, "a restored image is an ordinary one");
|
||||
assert!(restore_path(c, img(1)).unwrap().is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_trash_restore_trash_cycle_restores_to_the_right_place_twice() {
|
||||
// If `trashed_from` were not cleared on restore, the second trash would
|
||||
// record a stale origin and the second restore would put the file
|
||||
// somewhere it never was.
|
||||
let cat = seeded();
|
||||
let c = cat.connection();
|
||||
|
||||
do_trash(&cat, 1, 1_000);
|
||||
let first = restore_path(c, img(1)).unwrap().unwrap();
|
||||
record_restored(c, &[(img(1), first.clone())]).unwrap();
|
||||
|
||||
do_trash(&cat, 1, 2_000);
|
||||
let second = restore_path(c, img(1)).unwrap().unwrap();
|
||||
assert_eq!(
|
||||
first, second,
|
||||
"the origin is the library path, not the trash"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn re_trashing_does_not_overwrite_the_original_path() {
|
||||
// A retry after a partial failure must not record a trash-folder path as
|
||||
// the origin — that makes the image unrestorable.
|
||||
let cat = seeded();
|
||||
let c = cat.connection();
|
||||
let to = do_trash(&cat, 1, 1_000);
|
||||
// Second attempt, as a retry would do.
|
||||
record_trashed(c, &[(img(1), to)], 9_999).unwrap();
|
||||
|
||||
let (from, at): (String, i64) = c
|
||||
.query_row(
|
||||
"SELECT trashed_from, trashed_at FROM images WHERE id = 1",
|
||||
[],
|
||||
|r| Ok((r.get(0)?, r.get(1)?)),
|
||||
)
|
||||
.unwrap();
|
||||
assert_eq!(from, "PhotosRaw/2019/IMG_0001.CR2");
|
||||
assert_eq!(at, 1_000, "the original timestamp survives a retry");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn restoring_something_that_was_never_trashed_does_nothing() {
|
||||
let cat = seeded();
|
||||
let c = cat.connection();
|
||||
assert!(restore_path(c, img(2)).unwrap().is_none());
|
||||
assert_eq!(
|
||||
record_restored(c, &[(img(2), "elsewhere".into())]).unwrap(),
|
||||
0
|
||||
);
|
||||
// And its path is untouched.
|
||||
let source: String = c
|
||||
.query_row("SELECT source_ref FROM images WHERE id = 2", [], |r| {
|
||||
r.get(0)
|
||||
})
|
||||
.unwrap();
|
||||
assert_eq!(source, "PhotosRaw/2019/IMG_0002.CR2");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn the_trash_lists_newest_first() {
|
||||
// Reviewed to undo a recent mistake, not browsed chronologically.
|
||||
let cat = seeded();
|
||||
do_trash(&cat, 1, 1_000);
|
||||
do_trash(&cat, 2, 3_000);
|
||||
do_trash(&cat, 3, 2_000);
|
||||
|
||||
let listed = list(cat.connection(), 100).unwrap();
|
||||
let order: Vec<u64> = listed.iter().map(|t| t.image_id.0).collect();
|
||||
assert_eq!(order, vec![2, 3, 1]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn the_trash_list_carries_the_file_id_a_restore_needs() {
|
||||
// Without it a restore cannot find the thumbnail it already has, and
|
||||
// re-downloads a preview it is holding.
|
||||
let cat = seeded();
|
||||
do_trash(&cat, 1, 1_000);
|
||||
let listed = list(cat.connection(), 10).unwrap();
|
||||
assert_eq!(listed[0].file_id, Some(1001));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn the_summary_reports_what_emptying_would_free() {
|
||||
// "Empty trash" is destructive; the size is what tells the user whether
|
||||
// they meant it.
|
||||
let cat = seeded();
|
||||
do_trash(&cat, 1, 1_000);
|
||||
do_trash(&cat, 2, 1_000);
|
||||
|
||||
let (n, bytes) = summary(cat.connection()).unwrap();
|
||||
assert_eq!(n, 2);
|
||||
assert_eq!(bytes, 30_000_000 + 60_000_000);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn an_empty_trash_summarises_as_zero_rather_than_erroring() {
|
||||
let cat = seeded();
|
||||
assert_eq!(summary(cat.connection()).unwrap(), (0, 0));
|
||||
assert!(all_trashed(cat.connection()).unwrap().is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn purging_removes_the_row_and_everything_hanging_off_it() {
|
||||
let cat = seeded();
|
||||
let c = cat.connection();
|
||||
do_trash(&cat, 1, 1_000);
|
||||
|
||||
assert_eq!(forget(c, &[img(1)]).unwrap(), 1);
|
||||
|
||||
let n: i64 = c
|
||||
.query_row("SELECT count(*) FROM images WHERE id = 1", [], |r| r.get(0))
|
||||
.unwrap();
|
||||
assert_eq!(n, 0);
|
||||
// The remote mapping must go too, or a later scan could pair a new file
|
||||
// with a dead image's id.
|
||||
let n: i64 = c
|
||||
.query_row("SELECT count(*) FROM remote WHERE image_id = 1", [], |r| {
|
||||
r.get(0)
|
||||
})
|
||||
.unwrap();
|
||||
assert_eq!(n, 0, "cascaded");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn purging_leaves_untrashed_images_alone() {
|
||||
let cat = seeded();
|
||||
let c = cat.connection();
|
||||
do_trash(&cat, 1, 1_000);
|
||||
forget(c, &all_trashed(c).unwrap()).unwrap();
|
||||
|
||||
let n: i64 = c
|
||||
.query_row("SELECT count(*) FROM images", [], |r| r.get(0))
|
||||
.unwrap();
|
||||
assert_eq!(n, 3, "only the trashed one went");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn file_ids_are_collected_so_thumbnails_can_be_dropped() {
|
||||
// The shards sync to the server; a purge that left them would serve
|
||||
// previews of deleted photographs on every device.
|
||||
let cat = seeded();
|
||||
let c = cat.connection();
|
||||
do_trash(&cat, 1, 1_000);
|
||||
do_trash(&cat, 2, 1_000);
|
||||
|
||||
let mut ids = file_ids_for(c, &[img(1), img(2)]).unwrap();
|
||||
ids.sort_unstable();
|
||||
assert_eq!(ids, vec![1001, 1002]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_missing_file_counts_as_already_deleted() {
|
||||
// Otherwise one file removed by hand wedges every future empty-trash,
|
||||
// and no amount of retrying clears it.
|
||||
assert!(is_already_gone(Some(404)));
|
||||
assert!(is_already_gone(Some(410)));
|
||||
assert!(!is_already_gone(Some(403)), "a permission failure is real");
|
||||
assert!(!is_already_gone(Some(500)));
|
||||
assert!(!is_already_gone(None));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn empty_batches_are_no_ops_rather_than_errors() {
|
||||
// The UI can reach these with nothing selected.
|
||||
let cat = seeded();
|
||||
let c = cat.connection();
|
||||
assert_eq!(record_trashed(c, &[], 0).unwrap(), 0);
|
||||
assert_eq!(record_restored(c, &[]).unwrap(), 0);
|
||||
assert_eq!(forget(c, &[]).unwrap(), 0);
|
||||
assert!(file_ids_for(c, &[]).unwrap().is_empty());
|
||||
}
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
@@ -1,23 +0,0 @@
|
||||
[package]
|
||||
name = "dr-decode"
|
||||
version.workspace = true
|
||||
edition.workspace = true
|
||||
rust-version.workspace = true
|
||||
license.workspace = true
|
||||
|
||||
[dependencies]
|
||||
dr-types.workspace = true
|
||||
rawler.workspace = true
|
||||
# The camera profile database is data, not code (FR-DEV-3e): a YAML file that
|
||||
# ships with the binary and is superseded by a newer one on disk. serde_norway
|
||||
# is the workspace's YAML crate — the fork still receiving releases — and it is
|
||||
# already in the tree for `dr-pipeline`'s node declarations and `dr-ui`'s style
|
||||
# tokens. Pure Rust, so it costs nothing under the Android NDK.
|
||||
serde = { workspace = true }
|
||||
serde_norway.workspace = true
|
||||
zune-jpeg.workspace = true
|
||||
thiserror.workspace = true
|
||||
log.workspace = true
|
||||
|
||||
[dev-dependencies]
|
||||
env_logger.workspace = true
|
||||
@@ -1,52 +0,0 @@
|
||||
//! Report the defect map a raw file carries, if it carries one.
|
||||
//!
|
||||
//! ```text
|
||||
//! cargo run -p dr-decode --example defects -- IMG_6320.dng photo.cr2
|
||||
//! ```
|
||||
//!
|
||||
//! Exists because whether this is worth building a correction stage for is a
|
||||
//! question about *your files*, not about the specification: DNGs written by
|
||||
//! cameras that map their own sensors carry `OpcodeList1`, conversions from a
|
||||
//! proprietary raw usually do not, and no CR2 or scanner TIFF ever does.
|
||||
//! Rather than guess, point this at the library and see.
|
||||
|
||||
fn main() {
|
||||
let files: Vec<String> = std::env::args().skip(1).collect();
|
||||
if files.is_empty() {
|
||||
eprintln!("usage: defects <raw file>...");
|
||||
std::process::exit(2);
|
||||
}
|
||||
|
||||
for path in &files {
|
||||
let bytes = match std::fs::read(path) {
|
||||
Ok(b) => b,
|
||||
Err(e) => {
|
||||
println!("{path}: unreadable — {e}");
|
||||
continue;
|
||||
}
|
||||
};
|
||||
|
||||
let found = dr_decode::defects(&bytes);
|
||||
if found.is_empty() {
|
||||
println!("{path}: no defect map");
|
||||
continue;
|
||||
}
|
||||
|
||||
println!(
|
||||
"{path}: {} bad pixel(s), {} bad line(s)",
|
||||
found.pixels.len(),
|
||||
found.lines.len()
|
||||
);
|
||||
// A handful, so the output stays readable on a sensor reporting
|
||||
// hundreds — the count above is the number that matters.
|
||||
for p in found.pixels.iter().take(8) {
|
||||
println!(" pixel at {},{}", p.x, p.y);
|
||||
}
|
||||
for l in found.lines.iter().take(8) {
|
||||
match l {
|
||||
dr_decode::BadLine::Column(x) => println!(" dead column {x}"),
|
||||
dr_decode::BadLine::Row(y) => println!(" dead row {y}"),
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,19 +0,0 @@
|
||||
fn main() {
|
||||
for p in std::env::args().skip(1) {
|
||||
let Ok(d) = std::fs::read(&p) else { continue };
|
||||
let n = p.rsplit('/').next().unwrap();
|
||||
// Exactly what the sweep sees: the first HEADER_BYTES only.
|
||||
let head = &d[..d.len().min(dr_decode::HEADER_BYTES as usize)];
|
||||
match dr_decode::metadata(head) {
|
||||
Ok(m) => println!(
|
||||
"{n}: header-only at={:?} model={:?}",
|
||||
m.captured_at, m.model
|
||||
),
|
||||
Err(e) => println!("{n}: header-only ERROR {e}"),
|
||||
}
|
||||
match dr_decode::metadata(&d) {
|
||||
Ok(m) => println!("{n}: whole-file at={:?}", m.captured_at),
|
||||
Err(e) => println!("{n}: whole-file ERROR {e}"),
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,221 +0,0 @@
|
||||
//! TRACES: S15 | FR-MRG-3
|
||||
//! Spike S15.1 — does rawler read back a linear DNG this application writes?
|
||||
//!
|
||||
//! cargo run -p dr-decode --example linear_dng [-- <out.dng>]
|
||||
//!
|
||||
//! Decides FR-MRG-3's container. A panorama composite is three linear samples
|
||||
//! per pixel with a camera matrix attached, which is exactly what a
|
||||
//! `LinearRaw` DNG is; if rawler parses one, the composite re-enters the
|
||||
//! library as `Format::Dng` and the only new decode work is a `cpp == 3`
|
||||
//! branch. If it does not, the container is a float TIFF with a decode path
|
||||
//! of its own.
|
||||
//!
|
||||
//! The file is hand-rolled rather than written with the `tiff` crate, whose
|
||||
//! encoder fixes `PhotometricInterpretation` to RGB and cannot say
|
||||
//! `LinearRaw`. Eighty lines of IFD is the cheaper thing to own than a fork.
|
||||
|
||||
use rawler::rawsource::RawSource;
|
||||
|
||||
const W: u32 = 64;
|
||||
const H: u32 = 48;
|
||||
|
||||
fn main() {
|
||||
let bytes = write_linear_dng(W, H);
|
||||
if let Some(path) = std::env::args().nth(1) {
|
||||
std::fs::write(&path, &bytes).expect("write");
|
||||
println!("wrote {path} ({} bytes)", bytes.len());
|
||||
}
|
||||
|
||||
let source = RawSource::new_from_slice(&bytes);
|
||||
let decoder = match rawler::get_decoder(&source) {
|
||||
Ok(d) => d,
|
||||
Err(e) => {
|
||||
println!("FAIL get_decoder: {e}");
|
||||
std::process::exit(1);
|
||||
}
|
||||
};
|
||||
println!("ok decoder found");
|
||||
|
||||
let image = match decoder.raw_image(&source, &Default::default(), false) {
|
||||
Ok(i) => i,
|
||||
Err(e) => {
|
||||
println!("FAIL raw_image: {e}");
|
||||
std::process::exit(1);
|
||||
}
|
||||
};
|
||||
println!(
|
||||
"ok raw_image: {}×{}, cpp {}, bps {}, {} samples, make {:?} model {:?}",
|
||||
image.width,
|
||||
image.height,
|
||||
image.cpp,
|
||||
image.bps,
|
||||
match &image.data {
|
||||
rawler::RawImageData::Integer(v) => v.len(),
|
||||
rawler::RawImageData::Float(v) => v.len(),
|
||||
},
|
||||
image.make,
|
||||
image.model
|
||||
);
|
||||
println!(
|
||||
" white {:?} black {:?} wb {:?}",
|
||||
image.whitelevel.0,
|
||||
image
|
||||
.blacklevel
|
||||
.levels
|
||||
.iter()
|
||||
.map(|r| r.n as f32 / r.d.max(1) as f32)
|
||||
.collect::<Vec<_>>(),
|
||||
image.wb_coeffs
|
||||
);
|
||||
|
||||
// The pixel at (1, 0) was written as (1000, 2000, 3000): if the samples
|
||||
// come back interleaved in that order, cpp == 3 means what it says.
|
||||
if let rawler::RawImageData::Integer(v) = &image.data {
|
||||
let i = image.cpp;
|
||||
println!(" pixel (1,0) = {:?}", &v[i..i + image.cpp.min(3)]);
|
||||
}
|
||||
|
||||
// What dr-decode itself makes of it: the colour matrix rawler parsed into
|
||||
// the camera definition, and the profile the decoder would build from it.
|
||||
println!(" rawler color_matrix: {:?}", image.camera.color_matrix);
|
||||
let dng = dr_decode::profile::read_dng_matrices(decoder.as_ref());
|
||||
let profile = dr_decode::CameraProfile::extract(&image, &dng);
|
||||
println!(
|
||||
" CameraProfile: {}",
|
||||
profile
|
||||
.as_ref()
|
||||
.map(|p| format!("xyz_to_cam {:?}", p.xyz_to_cam()))
|
||||
.unwrap_or_else(|| "none".into())
|
||||
);
|
||||
|
||||
match dr_decode::decode(&bytes) {
|
||||
Ok(r) => println!(
|
||||
"note dr_decode::decode accepted it as CFA: {}×{}, {} samples — the cpp==3 branch is the work",
|
||||
r.width,
|
||||
r.height,
|
||||
r.data.len()
|
||||
),
|
||||
Err(e) => println!("note dr_decode::decode refused it: {e} — the cpp==3 branch is the work"),
|
||||
}
|
||||
}
|
||||
|
||||
/// A minimal `LinearRaw` DNG: one IFD, uncompressed 16-bit RGB, the tags a
|
||||
/// decoder needs to treat it as a DNG and the matrix a develop chain needs
|
||||
/// to treat it as a camera. Little-endian, one strip.
|
||||
fn write_linear_dng(w: u32, h: u32) -> Vec<u8> {
|
||||
// Pixels first, so their offset is known: a ramp with one marker pixel.
|
||||
let mut pixels: Vec<u16> = Vec::with_capacity((w * h * 3) as usize);
|
||||
for y in 0..h {
|
||||
for x in 0..w {
|
||||
if (x, y) == (1, 0) {
|
||||
pixels.extend([1000, 2000, 3000]);
|
||||
} else {
|
||||
let v = ((x + y) * 512).min(65535) as u16;
|
||||
pixels.extend([v, v / 2, v / 3]);
|
||||
}
|
||||
}
|
||||
}
|
||||
let pixel_bytes: Vec<u8> = pixels.iter().flat_map(|v| v.to_le_bytes()).collect();
|
||||
|
||||
// Layout: header (8) | pixels | extra data | IFD.
|
||||
let pixels_off = 8u32;
|
||||
let extra_off = pixels_off + pixel_bytes.len() as u32;
|
||||
|
||||
// Values that do not fit in four bytes go in `extra`, and the entry
|
||||
// points at them.
|
||||
let mut extra: Vec<u8> = Vec::new();
|
||||
let mut entries: Vec<(u16, u16, u32, [u8; 4])> = Vec::new();
|
||||
|
||||
fn short(tag: u16, v: u16) -> (u16, u16, u32, [u8; 4]) {
|
||||
let mut b = [0u8; 4];
|
||||
b[..2].copy_from_slice(&v.to_le_bytes());
|
||||
(tag, 3, 1, b)
|
||||
}
|
||||
fn long(tag: u16, v: u32) -> (u16, u16, u32, [u8; 4]) {
|
||||
(tag, 4, 1, v.to_le_bytes())
|
||||
}
|
||||
fn ascii(extra: &mut Vec<u8>, extra_off: u32, tag: u16, s: &str) -> (u16, u16, u32, [u8; 4]) {
|
||||
let mut bytes = s.as_bytes().to_vec();
|
||||
bytes.push(0);
|
||||
let off = extra_off + extra.len() as u32;
|
||||
extra.extend(&bytes);
|
||||
(tag, 2, bytes.len() as u32, off.to_le_bytes())
|
||||
}
|
||||
|
||||
entries.push(long(254, 0)); // NewSubfileType: main image
|
||||
entries.push(long(256, w));
|
||||
entries.push(long(257, h));
|
||||
// BitsPerSample ×3 — three shorts, six bytes, so out of line.
|
||||
{
|
||||
let off = extra_off + extra.len() as u32;
|
||||
for _ in 0..3 {
|
||||
extra.extend(16u16.to_le_bytes());
|
||||
}
|
||||
entries.push((258, 3, 3, off.to_le_bytes()));
|
||||
}
|
||||
entries.push(short(259, 1)); // Compression: none
|
||||
entries.push(short(262, 34892)); // PhotometricInterpretation: LinearRaw
|
||||
entries.push(ascii(&mut extra, extra_off, 271, "DarkRoom"));
|
||||
entries.push(ascii(&mut extra, extra_off, 272, "Panorama"));
|
||||
entries.push(long(273, pixels_off)); // StripOffsets
|
||||
entries.push(short(274, 1)); // Orientation
|
||||
entries.push(short(277, 3)); // SamplesPerPixel
|
||||
entries.push(long(278, h)); // RowsPerStrip
|
||||
entries.push(long(279, pixel_bytes.len() as u32)); // StripByteCounts
|
||||
entries.push(short(284, 1)); // PlanarConfiguration: chunky
|
||||
entries.push((50706, 1, 4, [1, 4, 0, 0])); // DNGVersion
|
||||
entries.push((50707, 1, 4, [1, 4, 0, 0])); // DNGBackwardVersion
|
||||
entries.push(ascii(&mut extra, extra_off, 50708, "DarkRoom Panorama")); // UniqueCameraModel
|
||||
entries.push(long(50717, 65535)); // WhiteLevel
|
||||
|
||||
// ColorMatrix1: XYZ → camera, 9 SRATIONALs. A plausible sRGB-ish matrix
|
||||
// (the inverse of the sRGB D65 primaries), scaled to integers.
|
||||
{
|
||||
let m: [(i32, i32); 9] = [
|
||||
(32406, 10000),
|
||||
(-15372, 10000),
|
||||
(-4986, 10000),
|
||||
(-9689, 10000),
|
||||
(18758, 10000),
|
||||
(415, 10000),
|
||||
(557, 10000),
|
||||
(-2040, 10000),
|
||||
(10570, 10000),
|
||||
];
|
||||
let off = extra_off + extra.len() as u32;
|
||||
for (n, d) in m {
|
||||
extra.extend(n.to_le_bytes());
|
||||
extra.extend(d.to_le_bytes());
|
||||
}
|
||||
entries.push((50721, 10, 9, off.to_le_bytes()));
|
||||
}
|
||||
// AsShotNeutral: 3 RATIONALs, neutral.
|
||||
{
|
||||
let off = extra_off + extra.len() as u32;
|
||||
for _ in 0..3 {
|
||||
extra.extend(1u32.to_le_bytes());
|
||||
extra.extend(1u32.to_le_bytes());
|
||||
}
|
||||
entries.push((50728, 5, 3, off.to_le_bytes()));
|
||||
}
|
||||
entries.push(short(50778, 21)); // CalibrationIlluminant1: D65
|
||||
|
||||
entries.sort_by_key(|e| e.0);
|
||||
|
||||
let ifd_off = extra_off + extra.len() as u32;
|
||||
let mut out = Vec::new();
|
||||
out.extend(b"II");
|
||||
out.extend(42u16.to_le_bytes());
|
||||
out.extend(ifd_off.to_le_bytes());
|
||||
out.extend(&pixel_bytes);
|
||||
out.extend(&extra);
|
||||
out.extend((entries.len() as u16).to_le_bytes());
|
||||
for (tag, ty, count, value) in &entries {
|
||||
out.extend(tag.to_le_bytes());
|
||||
out.extend(ty.to_le_bytes());
|
||||
out.extend(count.to_le_bytes());
|
||||
out.extend(value);
|
||||
}
|
||||
out.extend(0u32.to_le_bytes()); // no next IFD
|
||||
out
|
||||
}
|
||||
@@ -1,61 +0,0 @@
|
||||
//! Print what `decode` extracts from a RAW file.
|
||||
//!
|
||||
//! A sanity check on the pipeline's inputs: black and white levels, the CFA
|
||||
//! pattern after re-phasing, as-shot white balance, and the camera→sRGB
|
||||
//! matrix. Wrong values here produce a wrong image no shader can fix, so it
|
||||
//! is worth being able to see them directly.
|
||||
//!
|
||||
//! ```sh
|
||||
//! cargo run -p dr-decode --example rawinfo -- IMG.CR2
|
||||
//! ```
|
||||
|
||||
fn main() {
|
||||
let Some(path) = std::env::args().nth(1) else {
|
||||
eprintln!("usage: rawinfo <file.cr2>");
|
||||
std::process::exit(2);
|
||||
};
|
||||
|
||||
let bytes = std::fs::read(&path).expect("read file");
|
||||
let raw = dr_decode::decode(&bytes).expect("decode");
|
||||
|
||||
println!("file {path}");
|
||||
println!("readout {} × {}", raw.width, raw.height);
|
||||
println!(
|
||||
"crop {} × {} at ({}, {})",
|
||||
raw.crop.width, raw.crop.height, raw.crop.x, raw.crop.y
|
||||
);
|
||||
let (dx, dy) = raw.crop.shifts_cfa_phase();
|
||||
println!(
|
||||
"cfa {:?} (rephased: {dx}, {dy})",
|
||||
raw.cfa_pattern
|
||||
);
|
||||
println!("black {:?}", raw.black_level);
|
||||
println!("white {}", raw.white_level);
|
||||
println!("wb_coeffs {:?}", raw.wb_coeffs);
|
||||
|
||||
match raw.color_matrix {
|
||||
Some(m) => {
|
||||
println!("cam→srgb");
|
||||
for row in m.chunks(3) {
|
||||
println!(
|
||||
" [{:>8.4} {:>8.4} {:>8.4}]",
|
||||
row[0], row[1], row[2]
|
||||
);
|
||||
}
|
||||
// Each row should sum to roughly 1: a neutral camera-space colour
|
||||
// must stay neutral in sRGB. Far from 1 means the normalisation
|
||||
// or the matrix composition is wrong.
|
||||
let sums: Vec<f32> = m.chunks(3).map(|r| r.iter().sum()).collect();
|
||||
println!("row sums {sums:.4?} (≈1.0 each if correct)");
|
||||
}
|
||||
None => println!("cam→srgb none — uncalibrated body"),
|
||||
}
|
||||
|
||||
// Sample the actual data range, which reveals a black-level or bit-depth
|
||||
// mistake faster than any amount of staring at metadata.
|
||||
let (min, max) = raw
|
||||
.data
|
||||
.iter()
|
||||
.fold((u16::MAX, 0u16), |(lo, hi), &v| (lo.min(v), hi.max(v)));
|
||||
println!("sample range {min} … {max}");
|
||||
}
|
||||
@@ -1,146 +0,0 @@
|
||||
//! Smoke test against real RAW files.
|
||||
//!
|
||||
//! cargo run -p dr-decode --example smoke -- <file-or-dir>...
|
||||
//!
|
||||
//! Reports, per file, what each entry point costs — which is the whole reason
|
||||
//! they are separate (ARCH §3.2).
|
||||
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::time::Instant;
|
||||
|
||||
fn main() {
|
||||
env_logger::init();
|
||||
let args: Vec<String> = std::env::args().skip(1).collect();
|
||||
if args.is_empty() {
|
||||
eprintln!("usage: smoke <file-or-dir>...");
|
||||
std::process::exit(2);
|
||||
}
|
||||
|
||||
let mut files = Vec::new();
|
||||
for a in &args {
|
||||
let p = PathBuf::from(a);
|
||||
if p.is_dir() {
|
||||
collect(&p, &mut files);
|
||||
} else {
|
||||
files.push(p);
|
||||
}
|
||||
}
|
||||
files.sort();
|
||||
files.truncate(8);
|
||||
|
||||
println!(
|
||||
"{:<20} {:>7} {:>8} {:>9} {:>13} {:>9} {:>13}",
|
||||
"file", "size", "meta", "thumb", "thumb dims", "full", "full dims"
|
||||
);
|
||||
println!("{}", "-".repeat(88));
|
||||
|
||||
let (mut ok, mut failed) = (0, 0);
|
||||
for f in &files {
|
||||
match run_one(f) {
|
||||
Ok(line) => {
|
||||
println!("{line}");
|
||||
ok += 1;
|
||||
}
|
||||
Err(e) => {
|
||||
println!("{:<22} {e}", truncate(&name(f), 22));
|
||||
failed += 1;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
println!("\n{ok} ok, {failed} failed");
|
||||
if failed > 0 {
|
||||
std::process::exit(1);
|
||||
}
|
||||
}
|
||||
|
||||
fn run_one(path: &Path) -> Result<String, String> {
|
||||
let size = std::fs::metadata(path).map_err(|e| e.to_string())?.len();
|
||||
|
||||
// The culling path: read only the header region, not the whole file.
|
||||
let probe_bytes =
|
||||
read_prefix(path, dr_decode::PREVIEW_PROBE_BYTES).map_err(|e| e.to_string())?;
|
||||
let t0 = Instant::now();
|
||||
let fmt = dr_decode::probe(&probe_bytes);
|
||||
let meta = dr_decode::metadata(&probe_bytes).ok();
|
||||
let meta_ms = t0.elapsed().as_secs_f64() * 1000.0;
|
||||
|
||||
let all = std::fs::read(path).map_err(|e| e.to_string())?;
|
||||
|
||||
// The culling rung.
|
||||
let t1 = Instant::now();
|
||||
let thumb = dr_decode::extract_preview(&all, dr_decode::PreviewSize::Thumbnail)
|
||||
.map_err(|e| format!("thumb: {e}"))?;
|
||||
let thumb_ms = t1.elapsed().as_secs_f64() * 1000.0;
|
||||
|
||||
// The full-resolution rung, for comparison.
|
||||
let t2 = Instant::now();
|
||||
let full = dr_decode::extract_preview(&all, dr_decode::PreviewSize::Full)
|
||||
.map_err(|e| format!("full: {e}"))?;
|
||||
let full_ms = t2.elapsed().as_secs_f64() * 1000.0;
|
||||
|
||||
let model = meta
|
||||
.as_ref()
|
||||
.and_then(|m| m.model.clone())
|
||||
.unwrap_or_else(|| "?".into());
|
||||
let budget = if thumb_ms <= 50.0 {
|
||||
""
|
||||
} else {
|
||||
" OVER BUDGET"
|
||||
};
|
||||
|
||||
Ok(format!(
|
||||
"{:<20} {:>6.1}M {:>6.1}ms {:>7.1}ms {:>7}x{:<5} {:>7.1}ms {:>7}x{:<5} {:?} {}{}",
|
||||
truncate(&name(path), 20),
|
||||
size as f64 / 1e6,
|
||||
meta_ms,
|
||||
thumb_ms,
|
||||
thumb.width,
|
||||
thumb.height,
|
||||
full_ms,
|
||||
full.width,
|
||||
full.height,
|
||||
fmt,
|
||||
model.trim(),
|
||||
budget,
|
||||
))
|
||||
}
|
||||
|
||||
fn read_prefix(path: &Path, n: u64) -> std::io::Result<Vec<u8>> {
|
||||
use std::io::Read;
|
||||
let mut f = std::fs::File::open(path)?;
|
||||
let mut buf = vec![0u8; n as usize];
|
||||
let read = f.read(&mut buf)?;
|
||||
buf.truncate(read);
|
||||
Ok(buf)
|
||||
}
|
||||
|
||||
fn collect(dir: &Path, out: &mut Vec<PathBuf>) {
|
||||
let Ok(entries) = std::fs::read_dir(dir) else {
|
||||
return;
|
||||
};
|
||||
for e in entries.flatten() {
|
||||
let p = e.path();
|
||||
if p.is_file() {
|
||||
let ext = p
|
||||
.extension()
|
||||
.map(|s| s.to_string_lossy().to_ascii_lowercase())
|
||||
.unwrap_or_default();
|
||||
if dr_types::Format::from_extension(&ext).is_some() {
|
||||
out.push(p);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn name(p: &Path) -> String {
|
||||
p.file_name().unwrap_or_default().to_string_lossy().into()
|
||||
}
|
||||
|
||||
fn truncate(s: &str, n: usize) -> String {
|
||||
if s.len() <= n {
|
||||
s.to_string()
|
||||
} else {
|
||||
format!("{}…", &s[..n - 1])
|
||||
}
|
||||
}
|
||||
@@ -1,160 +0,0 @@
|
||||
# DarkRoom camera base curves (FR-DEV-3e).
|
||||
#
|
||||
# ---------------------------------------------------------------------------
|
||||
# Adding a body is editing this file. It is not a code change.
|
||||
# ---------------------------------------------------------------------------
|
||||
#
|
||||
# The copy you are reading is compiled into the binary as a floor. At startup
|
||||
# `dr_decode::base_curve::load` also looks for `base_curves.yaml` in:
|
||||
#
|
||||
# 1. $DARKROOM_PROFILES/ (set it while you are tuning)
|
||||
# 2. $XDG_DATA_HOME/darkroom/profiles/
|
||||
# or $HOME/.local/share/darkroom/profiles/
|
||||
#
|
||||
# and uses the first one it finds *whose `version:` is higher than this one's*.
|
||||
# So: bump `version`, drop the file in that directory, restart. A body added
|
||||
# this afternoon renders correctly this afternoon, with no release and no
|
||||
# rebuild — which is what the requirement asks for, and what makes these
|
||||
# contributable under the GPL.
|
||||
#
|
||||
# The version check runs both ways on purpose. A file older than the built-in
|
||||
# copy is ignored with a log line, so upgrading DarkRoom cannot silently lose
|
||||
# curves to a pack somebody downloaded a year ago.
|
||||
#
|
||||
# ---------------------------------------------------------------------------
|
||||
# What the numbers mean
|
||||
# ---------------------------------------------------------------------------
|
||||
#
|
||||
# Five `[x, y]` control points on a monotone spline (Fritsch-Carlson, the same
|
||||
# one the tone curve widget draws). Both axes are **linear**:
|
||||
#
|
||||
# x scene-referred camera RGB after white balance, 1.0 = sensor saturation
|
||||
# y display-referred linear; the sRGB transfer function is applied later,
|
||||
# at the end of the shader, so do not pre-apply a gamma here
|
||||
#
|
||||
# The identity is y = x, and it is what an unrecognised body gets if `default:`
|
||||
# is removed. It is also the wrong answer for almost every photograph: linear
|
||||
# scene data has middle grey at about 13% and a camera JPEG puts it near 18%,
|
||||
# so an uncurved render is roughly half a stop dark through the midtones and
|
||||
# has no highlight rolloff at all.
|
||||
#
|
||||
# A curve that works has three parts, and it is worth naming them because they
|
||||
# are what you are actually tuning:
|
||||
#
|
||||
# the toe the first span, slope near or below 1. Deep shadows stay
|
||||
# deep. Lift it and blacks go milky; crush it and shadow
|
||||
# detail the sensor recorded disappears.
|
||||
# the midtones the middle spans, slope well above 1. This is the contrast
|
||||
# and the brightness people read as "the camera's look".
|
||||
# the shoulder the last span, slope well below 1. Highlights compress
|
||||
# toward white instead of arriving there and clipping. It is
|
||||
# the difference between a rolled-off sky and a white hole.
|
||||
#
|
||||
# Two invariants are enforced in code and tested, so a mistake here fails the
|
||||
# build rather than the photograph: x must strictly increase, y must not
|
||||
# decrease, and everything must lie inside the unit square.
|
||||
#
|
||||
# ---------------------------------------------------------------------------
|
||||
# Honesty about these values
|
||||
# ---------------------------------------------------------------------------
|
||||
#
|
||||
# These are hand-tuned shapes, not measurements. They encode what every camera
|
||||
# JPEG rendering has in common — the toe/midtone/shoulder structure above —
|
||||
# plus each maker's well-known house differences: Canon's gentler shoulder and
|
||||
# warmer-reading midtones, Nikon's slightly higher midtone contrast, Sony's
|
||||
# flatter and more conservative default, Fujifilm's markedly contrastier
|
||||
# Provia-derived rendering.
|
||||
#
|
||||
# FR-DEV-3e's acceptance criterion is subjective comparison against each body's
|
||||
# own JPEG, and meeting it properly needs a frame from that body in front of
|
||||
# you. Where that has not been done, the entry is still much closer to right
|
||||
# than the identity — which is the bar these have to clear, and do.
|
||||
|
||||
version: 1
|
||||
|
||||
# The rendering for a body with no entry of its own.
|
||||
#
|
||||
# **Deliberately not the identity.** The failure this requirement exists to fix
|
||||
# is the flat render, and a conservative curve is far closer to right for every
|
||||
# body than no curve is for any of them. It is gentler than the per-body
|
||||
# entries below — a shallower midtone and an earlier, softer shoulder — because
|
||||
# it has to be safe on a sensor nobody has looked at, and the cost of being too
|
||||
# tame is a photograph that wants a little contrast rather than one that has
|
||||
# lost its highlights.
|
||||
default:
|
||||
points:
|
||||
- [0.00, 0.000]
|
||||
- [0.04, 0.043]
|
||||
- [0.13, 0.175]
|
||||
- [0.45, 0.690]
|
||||
- [1.00, 1.000]
|
||||
|
||||
bodies:
|
||||
# Canon. A soft toe and a long, gradual shoulder — the reason Canon files
|
||||
# are described as forgiving in highlights and a little low in contrast
|
||||
# straight out of camera.
|
||||
- make: Canon
|
||||
model: EOS 6D
|
||||
points:
|
||||
- [0.00, 0.000]
|
||||
- [0.04, 0.045]
|
||||
- [0.13, 0.190]
|
||||
- [0.45, 0.720]
|
||||
- [1.00, 1.000]
|
||||
|
||||
- make: Canon
|
||||
model: EOS R6
|
||||
points:
|
||||
- [0.00, 0.000]
|
||||
- [0.04, 0.044]
|
||||
- [0.13, 0.195]
|
||||
- [0.45, 0.730]
|
||||
- [1.00, 1.000]
|
||||
|
||||
# Nikon. A slightly deeper toe and more midtone slope than Canon, which is
|
||||
# the "punchier out of camera" difference people describe between the two.
|
||||
- make: Nikon
|
||||
model: Z 6
|
||||
points:
|
||||
- [0.00, 0.000]
|
||||
- [0.04, 0.038]
|
||||
- [0.13, 0.200]
|
||||
- [0.46, 0.750]
|
||||
- [1.00, 1.000]
|
||||
|
||||
- make: Nikon
|
||||
model: D750
|
||||
points:
|
||||
- [0.00, 0.000]
|
||||
- [0.04, 0.039]
|
||||
- [0.13, 0.198]
|
||||
- [0.46, 0.745]
|
||||
- [1.00, 1.000]
|
||||
|
||||
# Sony. The flattest default of the four, and intentionally so — Sony's own
|
||||
# rendering leaves more headroom than it uses, which is why Sony files are
|
||||
# the ones people describe as needing the most work.
|
||||
- make: Sony
|
||||
model: ILCE-7M3
|
||||
points:
|
||||
- [0.00, 0.000]
|
||||
- [0.04, 0.048]
|
||||
- [0.13, 0.185]
|
||||
- [0.44, 0.700]
|
||||
- [1.00, 1.000]
|
||||
|
||||
# Fujifilm. Provia, the default film simulation: a firm toe, the steepest
|
||||
# midtones here, and a hard shoulder. It is the most distinctive rendering of
|
||||
# the four and the one where a flat render looks most obviously wrong.
|
||||
#
|
||||
# This entry does *not* read the in-RAF film simulation tag — that is
|
||||
# FR-DEV-3f, and until it lands every Fujifilm file gets the Provia shape
|
||||
# whatever the camera was set to.
|
||||
- make: Fujifilm
|
||||
model: X-T3
|
||||
points:
|
||||
- [0.00, 0.000]
|
||||
- [0.045, 0.040]
|
||||
- [0.14, 0.215]
|
||||
- [0.47, 0.775]
|
||||
- [1.00, 1.000]
|
||||
@@ -1,752 +0,0 @@
|
||||
//! TRACES: FR-DEV-3e
|
||||
//! Base curves — the per-body rendering that turns a correct exposure into a
|
||||
//! photograph.
|
||||
//!
|
||||
//! # What this is for
|
||||
//!
|
||||
//! A camera matrix gets the *colours* right and leaves the picture flat. Sensor
|
||||
//! data is scene-referred and very nearly linear; a print, a screen and a
|
||||
//! camera's own JPEG are none of those things. Rendering linear data straight
|
||||
//! out is the dcraw default, and FR-DEV-3e names it precisely: "the flat,
|
||||
//! poor-skin-tone rendering characteristic of dcraw defaults, which is the
|
||||
//! documented reason people abandon darktable in the first hour."
|
||||
//!
|
||||
//! The fix is a tone curve applied as part of *reading* the file rather than as
|
||||
//! an edit — a toe, a steep midtone, and a shoulder that rolls highlights off
|
||||
//! instead of clipping them. Every raw converter has one. Adobe calls it the
|
||||
//! camera profile's tone curve, darktable calls it the base curve, and the name
|
||||
//! here follows darktable's because the placement does too: it runs in camera
|
||||
//! RGB, after white balance and the user's adjustments, immediately before the
|
||||
//! conversion out to a working space.
|
||||
//!
|
||||
//! # Why it is not an edit
|
||||
//!
|
||||
//! It never reaches the sidecar and there is no slider for it, for the same
|
||||
//! reason the EXIF orientation is not an edit (FR-DEV-3h): it is a property of
|
||||
//! the body that took the frame, not of what anyone decided about the frame.
|
||||
//! Sidecars are shared between devices and bodies (FR-NC-9), and one camera's
|
||||
//! rendering must not follow an edit onto another camera's file.
|
||||
//!
|
||||
//! # Why it is data
|
||||
//!
|
||||
//! FR-DEV-3e requires the profile database to be "versioned independently of
|
||||
//! the app binary so bodies and curves can be added without a release — and,
|
||||
//! under D8's GPLv3, contributed by users". So the curves live in
|
||||
//! `profiles/base_curves.yaml`, a file that is compiled in as a floor and
|
||||
//! *overridden* by a copy on disk carrying a higher `version:`. Adding a body
|
||||
//! is adding ten numbers to a YAML file; shipping that body to users is
|
||||
//! publishing the file. Neither is a code change and neither needs a release.
|
||||
//!
|
||||
//! See [`load`] for the search path and [`Curves::body`] for the matching.
|
||||
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::sync::OnceLock;
|
||||
|
||||
/// How many control points a base curve has.
|
||||
///
|
||||
/// Five, which is not a coincidence: it is what the tone curve widget uses
|
||||
/// (`dr_pipeline::ops::curve::POINTS`), so the shader evaluates a profile's
|
||||
/// curve and a photographer's curve through exactly the same spline. A profile
|
||||
/// author and a photographer dragging a point mean the same thing by it, and
|
||||
/// the generated shader carries one implementation rather than two that could
|
||||
/// disagree.
|
||||
pub const POINTS: usize = 5;
|
||||
|
||||
/// TRACES: FR-DEV-3e
|
||||
/// A base curve: five points on a monotone spline through the unit square.
|
||||
///
|
||||
/// `xs` is scene-linear camera RGB, normalised so that 1.0 is the sensor's
|
||||
/// saturation point. `ys` is display-referred linear — *not* gamma-encoded,
|
||||
/// because the sRGB transfer function is applied at the very end of the
|
||||
/// generated shader and applying it twice would wash the image out.
|
||||
#[derive(Debug, Clone, Copy, PartialEq)]
|
||||
pub struct BaseCurve {
|
||||
pub xs: [f32; POINTS],
|
||||
pub ys: [f32; POINTS],
|
||||
}
|
||||
|
||||
impl BaseCurve {
|
||||
/// The curve that does nothing — the identity diagonal.
|
||||
///
|
||||
/// What an unrecognised body gets if the database carries no default, and
|
||||
/// what a JPEG gets always: an already-rendered image must not be rendered
|
||||
/// a second time.
|
||||
pub const IDENTITY: Self = Self {
|
||||
xs: [0.0, 0.25, 0.5, 0.75, 1.0],
|
||||
ys: [0.0, 0.25, 0.5, 0.75, 1.0],
|
||||
};
|
||||
|
||||
/// Whether this curve would leave the image alone.
|
||||
///
|
||||
/// The shader is told to skip the stage entirely when it would, so an
|
||||
/// unprofiled body costs a branch that is uniform across the dispatch
|
||||
/// rather than a spline evaluation per channel per pixel.
|
||||
pub fn is_identity(&self) -> bool {
|
||||
self.xs
|
||||
.iter()
|
||||
.zip(self.ys.iter())
|
||||
.all(|(x, y)| (x - y).abs() < 1e-6)
|
||||
}
|
||||
|
||||
/// Build from raw pairs, rejecting anything that is not a curve.
|
||||
///
|
||||
/// A profile file is data a user may have edited, so this is the boundary
|
||||
/// where "ten numbers" becomes "a curve": the x coordinates must increase,
|
||||
/// the y coordinates must not decrease, and both must lie in the unit
|
||||
/// square. A non-monotone x sends the spline's span search backwards and
|
||||
/// divides by a negative width; a decreasing y inverts tones locally,
|
||||
/// which reads as a dark halo through smooth gradients rather than as a
|
||||
/// bad profile.
|
||||
///
|
||||
/// Endpoints are not forced to (0,0) and (1,1). A curve that lifts black
|
||||
/// slightly, or that places the shoulder below white, is a legitimate
|
||||
/// rendering choice and several bodies make it.
|
||||
pub fn from_points(points: &[[f32; 2]]) -> Option<Self> {
|
||||
if points.len() != POINTS {
|
||||
return None;
|
||||
}
|
||||
let mut xs = [0.0f32; POINTS];
|
||||
let mut ys = [0.0f32; POINTS];
|
||||
for (i, p) in points.iter().enumerate() {
|
||||
if !p[0].is_finite() || !p[1].is_finite() {
|
||||
return None;
|
||||
}
|
||||
if !(0.0..=1.0).contains(&p[0]) || !(0.0..=1.0).contains(&p[1]) {
|
||||
return None;
|
||||
}
|
||||
xs[i] = p[0];
|
||||
ys[i] = p[1];
|
||||
}
|
||||
for i in 1..POINTS {
|
||||
// Strictly increasing in x — the spline divides by the span width.
|
||||
if xs[i] <= xs[i - 1] {
|
||||
return None;
|
||||
}
|
||||
// Non-decreasing in y. Flat is allowed: a curve that holds a
|
||||
// highlight range at white is clipping deliberately.
|
||||
if ys[i] < ys[i - 1] {
|
||||
return None;
|
||||
}
|
||||
}
|
||||
Some(Self { xs, ys })
|
||||
}
|
||||
}
|
||||
|
||||
/// One body's entry in the database.
|
||||
#[derive(Debug, Clone, PartialEq)]
|
||||
pub struct BodyCurve {
|
||||
/// The manufacturer, as the file writes it — "Canon", "NIKON CORPORATION".
|
||||
pub make: String,
|
||||
/// The model, as the file writes it — "EOS 6D", "ILCE-7M3".
|
||||
pub model: String,
|
||||
pub curve: BaseCurve,
|
||||
}
|
||||
|
||||
/// TRACES: FR-DEV-3e
|
||||
/// The base curve database.
|
||||
///
|
||||
/// Versioned as a whole rather than per body, because that is the unit a user
|
||||
/// downloads and the unit that has to beat the built-in copy. See [`load`].
|
||||
#[derive(Debug, Clone, PartialEq)]
|
||||
pub struct Curves {
|
||||
version: u32,
|
||||
default: Option<BaseCurve>,
|
||||
bodies: Vec<BodyCurve>,
|
||||
}
|
||||
|
||||
impl Curves {
|
||||
/// TRACES: FR-DEV-3e
|
||||
/// The curve to render a frame from this body with.
|
||||
///
|
||||
/// Falls back, in order, to the database's `default:` and then to the
|
||||
/// identity. **The default is deliberately not the identity**: an
|
||||
/// unrecognised body rendered flat is the failure this requirement exists
|
||||
/// to prevent, and a gentle, conservative curve is much closer to right for
|
||||
/// every body than no curve is for any of them. A body with its own entry
|
||||
/// gets that instead.
|
||||
///
|
||||
/// # What "this body" has to survive
|
||||
///
|
||||
/// The same camera names itself three ways depending on which program last
|
||||
/// touched the file. A native NEF says make "NIKON CORPORATION", model
|
||||
/// "NIKON Z 6"; rawler's own database cleans that to "Nikon" and "Z 6"; an
|
||||
/// Adobe-converted DNG keeps the uncleaned pair. A database that had to
|
||||
/// spell every variant would go stale the first time a maker changed its
|
||||
/// mind about its own name, so the matching does the folding instead:
|
||||
///
|
||||
/// - Case, punctuation and runs of whitespace are flattened, so
|
||||
/// "ILCE-7M3", "ILCE 7M3" and "ilce-7m3" are one body.
|
||||
/// - The make is compared on its **first word only**. Every maker's
|
||||
/// trailing corporate boilerplate — "CORPORATION", "IMAGING CORP" — is
|
||||
/// noise, and no two camera manufacturers share a first word.
|
||||
/// - The model is tried both as written and with a leading copy of the
|
||||
/// make removed, which is what lets one "Canon"/"EOS 6D" entry cover
|
||||
/// "Canon EOS 6D" as well.
|
||||
pub fn body(&self, make: &str, model: &str) -> BaseCurve {
|
||||
let (make, model) = (make_key(make), normalise(model));
|
||||
// The model with a leading copy of the maker's name removed.
|
||||
let bare = model.strip_prefix(&format!("{make} ")).unwrap_or(&model);
|
||||
|
||||
self.bodies
|
||||
.iter()
|
||||
.find(|b| {
|
||||
let entry_model = normalise(&b.model);
|
||||
make_key(&b.make) == make && (entry_model == model || entry_model == bare)
|
||||
})
|
||||
.map(|b| b.curve)
|
||||
.or(self.default)
|
||||
.unwrap_or(BaseCurve::IDENTITY)
|
||||
}
|
||||
|
||||
/// The database version. Higher wins; see [`load`].
|
||||
pub fn version(&self) -> u32 {
|
||||
self.version
|
||||
}
|
||||
|
||||
/// How many bodies have their own curve, excluding the default.
|
||||
pub fn len(&self) -> usize {
|
||||
self.bodies.len()
|
||||
}
|
||||
|
||||
pub fn is_empty(&self) -> bool {
|
||||
self.bodies.is_empty()
|
||||
}
|
||||
|
||||
/// Parse a database from YAML.
|
||||
///
|
||||
/// Entries that are not curves are dropped with a warning rather than
|
||||
/// failing the parse. A user-contributed file with one bad body should
|
||||
/// cost that body's rendering, not every body's — and the alternative is an
|
||||
/// application that will not open a photograph because somebody typed a
|
||||
/// comma.
|
||||
pub fn parse(yaml: &str) -> Result<Self, String> {
|
||||
let file: File = serde_norway::from_str(yaml).map_err(|e| e.to_string())?;
|
||||
|
||||
let default = file.default.and_then(|d| {
|
||||
BaseCurve::from_points(&d.points).or_else(|| {
|
||||
log::warn!("base curves: the default entry is not a monotone curve; ignoring it");
|
||||
None
|
||||
})
|
||||
});
|
||||
|
||||
let bodies = file
|
||||
.bodies
|
||||
.into_iter()
|
||||
.filter_map(|b| match BaseCurve::from_points(&b.points) {
|
||||
Some(curve) => Some(BodyCurve {
|
||||
make: b.make,
|
||||
model: b.model,
|
||||
curve,
|
||||
}),
|
||||
None => {
|
||||
log::warn!(
|
||||
"base curves: {} {} is not a monotone curve; ignoring it",
|
||||
b.make,
|
||||
b.model
|
||||
);
|
||||
None
|
||||
}
|
||||
})
|
||||
.collect();
|
||||
|
||||
Ok(Self {
|
||||
version: file.version,
|
||||
default,
|
||||
bodies,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
/// The copy that ships inside the binary.
|
||||
///
|
||||
/// A floor, not the answer: [`load`] prefers a newer file on disk. Compiled in
|
||||
/// so that a fresh install with no profile directory — and every Android build,
|
||||
/// where there is no such directory to speak of — still renders properly.
|
||||
const BUILT_IN: &str = include_str!("../profiles/base_curves.yaml");
|
||||
|
||||
/// TRACES: FR-DEV-3e
|
||||
/// The base curve database, loaded once.
|
||||
///
|
||||
/// # The search path, and why it is a version comparison
|
||||
///
|
||||
/// 1. `$DARKROOM_PROFILES`, a directory, when set. The escape hatch: a profile
|
||||
/// author iterating on a curve points this at their working copy and does
|
||||
/// not have to install anything.
|
||||
/// 2. `$XDG_DATA_HOME/darkroom/profiles/`, else `$HOME/.local/share/darkroom/profiles/`.
|
||||
/// The same base directory the catalog uses, chosen there for the same
|
||||
/// reason — it is data, not cache, and must survive a storage sweep.
|
||||
/// 3. The copy compiled into the binary.
|
||||
///
|
||||
/// The first file that parses *and carries a higher `version:` than the
|
||||
/// built-in copy* wins. The version check is the whole mechanism the
|
||||
/// requirement asks for, and it runs in both directions:
|
||||
///
|
||||
/// - A downloaded pack at version 7 supersedes a binary shipping version 3, so
|
||||
/// a body added after the release renders correctly with no release.
|
||||
/// - A stale pack at version 2 does **not** supersede a binary shipping version
|
||||
/// 3, so upgrading the application cannot silently lose curves to a file
|
||||
/// somebody downloaded a year ago and forgot.
|
||||
///
|
||||
/// Failures are warnings, never errors. A malformed profile file must cost the
|
||||
/// user their curves, not their photographs.
|
||||
pub fn load() -> &'static Curves {
|
||||
static LOADED: OnceLock<Curves> = OnceLock::new();
|
||||
LOADED.get_or_init(|| {
|
||||
let built_in = Curves::parse(BUILT_IN).unwrap_or_else(|e| {
|
||||
// Unreachable in a build that ran its tests — `the_shipped_database_parses`
|
||||
// asserts exactly this — but a panic here would mean an
|
||||
// application that cannot open a photograph because of a typo in a
|
||||
// data file, which is never the right trade.
|
||||
log::error!("base curves: the built-in database does not parse: {e}");
|
||||
Curves {
|
||||
version: 0,
|
||||
default: None,
|
||||
bodies: Vec::new(),
|
||||
}
|
||||
});
|
||||
|
||||
choose(built_in, &search_path())
|
||||
})
|
||||
}
|
||||
|
||||
/// The version comparison, separated from where the directories come from.
|
||||
///
|
||||
/// Split out so it can be tested against real files in a real directory
|
||||
/// without the process-wide `OnceLock` and the environment `load` reads. The
|
||||
/// rule this implements is the whole of what FR-DEV-3e asks for, so it is
|
||||
/// worth being able to state it as a test rather than as a comment.
|
||||
fn choose(built_in: Curves, dirs: &[PathBuf]) -> Curves {
|
||||
for dir in dirs {
|
||||
let path = dir.join("base_curves.yaml");
|
||||
let Ok(text) = std::fs::read_to_string(&path) else {
|
||||
continue;
|
||||
};
|
||||
match Curves::parse(&text) {
|
||||
Ok(external) if external.version > built_in.version => {
|
||||
log::info!(
|
||||
"base curves: using {} (version {}, {} bodies) over the built-in version {}",
|
||||
path.display(),
|
||||
external.version,
|
||||
external.len(),
|
||||
built_in.version
|
||||
);
|
||||
return external;
|
||||
}
|
||||
Ok(external) => log::info!(
|
||||
"base curves: ignoring {} at version {}; the built-in database is version {}",
|
||||
path.display(),
|
||||
external.version,
|
||||
built_in.version
|
||||
),
|
||||
Err(e) => log::warn!("base curves: {} does not parse: {e}", path.display()),
|
||||
}
|
||||
}
|
||||
built_in
|
||||
}
|
||||
|
||||
/// TRACES: FR-DEV-3e
|
||||
/// The curve for a body, from the loaded database.
|
||||
///
|
||||
/// The one call site the decoder needs; everything above is reachable for
|
||||
/// tests and for a future profile editor.
|
||||
pub fn for_body(make: &str, model: &str) -> BaseCurve {
|
||||
load().body(make, model)
|
||||
}
|
||||
|
||||
/// Directories that may hold a `base_curves.yaml`, most specific first.
|
||||
fn search_path() -> Vec<PathBuf> {
|
||||
let mut dirs = Vec::new();
|
||||
if let Some(explicit) = std::env::var_os("DARKROOM_PROFILES") {
|
||||
dirs.push(PathBuf::from(explicit));
|
||||
}
|
||||
// The same resolution `dr_ui::library::catalog_path` uses, and for the
|
||||
// same reason: this is data a user may have installed, not a cache. It is
|
||||
// duplicated rather than shared because `dr-decode` sits far below the UI
|
||||
// and must not acquire a dependency on it to find a directory.
|
||||
let base = std::env::var_os("XDG_DATA_HOME")
|
||||
.map(PathBuf::from)
|
||||
.or_else(|| std::env::var_os("HOME").map(|h| Path::new(&h).join(".local/share")));
|
||||
if let Some(base) = base {
|
||||
dirs.push(base.join("darkroom").join("profiles"));
|
||||
}
|
||||
dirs
|
||||
}
|
||||
|
||||
/// A manufacturer's first word, folded.
|
||||
///
|
||||
/// "NIKON CORPORATION", "Nikon" and "nikon" all become `NIKON`. The corporate
|
||||
/// suffixes are not information — they appear or not depending on whether the
|
||||
/// file went through a DNG converter — and no two camera manufacturers share a
|
||||
/// first word, so nothing is lost by dropping them.
|
||||
fn make_key(s: &str) -> String {
|
||||
normalise(s)
|
||||
.split(' ')
|
||||
.next()
|
||||
.unwrap_or_default()
|
||||
.to_string()
|
||||
}
|
||||
|
||||
/// Fold a make or model into something two files can agree on.
|
||||
///
|
||||
/// Upper-cased, with every run of non-alphanumeric characters collapsed to one
|
||||
/// space and the ends trimmed, so that "ILCE-7M3", "ILCE 7M3" and "ilce-7m3"
|
||||
/// become one.
|
||||
fn normalise(s: &str) -> String {
|
||||
let mut out = String::with_capacity(s.len());
|
||||
let mut pending_space = false;
|
||||
for c in s.chars() {
|
||||
if c.is_ascii_alphanumeric() {
|
||||
if pending_space && !out.is_empty() {
|
||||
out.push(' ');
|
||||
}
|
||||
pending_space = false;
|
||||
out.push(c.to_ascii_uppercase());
|
||||
} else {
|
||||
pending_space = true;
|
||||
}
|
||||
}
|
||||
out
|
||||
}
|
||||
|
||||
// ---- The on-disk shape, kept apart from the in-memory one ----------------
|
||||
//
|
||||
// Deliberately separate types. The file is data a user edits and is allowed to
|
||||
// be wrong; `Curves` is a parsed database whose every entry is known to be a
|
||||
// monotone curve. Deriving `Deserialize` on `BaseCurve` directly would delete
|
||||
// that boundary and let an unchecked five-point array reach the shader.
|
||||
//
|
||||
// Unknown fields are **accepted**, which is not laziness. The database is
|
||||
// versioned independently of the binary and moves in both directions: a pack
|
||||
// published after this release may carry keys this build has never heard of —
|
||||
// a hue twist, a look table (FR-DEV-3f) — and it must still deliver its curves
|
||||
// to an older DarkRoom rather than failing to parse and leaving every body
|
||||
// flat. `deny_unknown_fields` would trade that for a diagnostic nobody needs.
|
||||
|
||||
#[derive(serde::Deserialize)]
|
||||
struct File {
|
||||
version: u32,
|
||||
#[serde(default)]
|
||||
default: Option<Entry>,
|
||||
#[serde(default)]
|
||||
bodies: Vec<BodyEntry>,
|
||||
}
|
||||
|
||||
#[derive(serde::Deserialize)]
|
||||
struct Entry {
|
||||
points: Vec<[f32; 2]>,
|
||||
}
|
||||
|
||||
#[derive(serde::Deserialize)]
|
||||
struct BodyEntry {
|
||||
make: String,
|
||||
model: String,
|
||||
points: Vec<[f32; 2]>,
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn the_shipped_database_parses_and_carries_a_default() {
|
||||
// The one test that must never be allowed to fail quietly: `load`
|
||||
// degrades to an empty database rather than panicking, so without this
|
||||
// a typo in the YAML would ship as "every photograph renders flat"
|
||||
// rather than as a build failure.
|
||||
let curves = Curves::parse(BUILT_IN).expect("the shipped database parses");
|
||||
assert!(curves.version() >= 1);
|
||||
assert!(!curves.is_empty(), "the database ships bodies");
|
||||
assert!(
|
||||
!curves.body("Nobody", "Nothing").is_identity(),
|
||||
"an unknown body must still get the default rendering"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn every_shipped_curve_lifts_the_midtones_and_rolls_the_highlights() {
|
||||
// What makes a base curve a base curve rather than a decoration. If a
|
||||
// shipped curve failed either half it would be a worse rendering than
|
||||
// the flat one it replaced, which is the one outcome forbidden.
|
||||
let curves = Curves::parse(BUILT_IN).expect("parses");
|
||||
let all = curves
|
||||
.bodies
|
||||
.iter()
|
||||
.map(|b| (format!("{} {}", b.make, b.model), b.curve))
|
||||
.chain(curves.default.map(|c| ("default".to_string(), c)));
|
||||
|
||||
for (name, curve) in all {
|
||||
// The midtone point sits above the diagonal: a linear midtone is
|
||||
// roughly a stop and a half darker than any camera renders it.
|
||||
let mid = 2;
|
||||
assert!(
|
||||
curve.ys[mid] > curve.xs[mid],
|
||||
"{name} does not lift its midtones ({} -> {})",
|
||||
curve.xs[mid],
|
||||
curve.ys[mid]
|
||||
);
|
||||
// And the last span is shallower than the one before it, which is
|
||||
// what a shoulder *is*. Without one the curve clips highlights
|
||||
// harder than the linear rendering did.
|
||||
let slope =
|
||||
|i: usize| (curve.ys[i + 1] - curve.ys[i]) / (curve.xs[i + 1] - curve.xs[i]);
|
||||
assert!(
|
||||
slope(POINTS - 2) < slope(POINTS - 3),
|
||||
"{name} has no highlight shoulder"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_curve_that_is_not_monotone_is_refused() {
|
||||
// The profile file is user-editable, so this is a real boundary and
|
||||
// not a formality. A decreasing y inverts tones locally and shows up
|
||||
// as a dark halo in a gradient, which reads as a rendering fault
|
||||
// rather than as a bad profile.
|
||||
assert_eq!(
|
||||
BaseCurve::from_points(&[[0.0, 0.0], [0.25, 0.4], [0.5, 0.3], [0.75, 0.8], [1.0, 1.0]]),
|
||||
None
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_curve_whose_x_does_not_advance_is_refused() {
|
||||
// The spline divides by the span width; a repeated x is a division by
|
||||
// zero in the shader, which is a NaN pixel rather than an error.
|
||||
assert_eq!(
|
||||
BaseCurve::from_points(&[
|
||||
[0.0, 0.0],
|
||||
[0.25, 0.3],
|
||||
[0.25, 0.5],
|
||||
[0.75, 0.8],
|
||||
[1.0, 1.0]
|
||||
]),
|
||||
None
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_curve_of_the_wrong_length_is_refused() {
|
||||
assert_eq!(BaseCurve::from_points(&[[0.0, 0.0], [1.0, 1.0]]), None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn values_outside_the_unit_square_are_refused() {
|
||||
// The shader clamps its output at the very end anyway, but a control
|
||||
// point above 1.0 would put the shoulder outside the range the curve
|
||||
// is defined over and silently flatten everything below it.
|
||||
assert_eq!(
|
||||
BaseCurve::from_points(&[[0.0, 0.0], [0.25, 0.3], [0.5, 1.4], [0.75, 1.5], [1.0, 1.6]]),
|
||||
None
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_body_with_its_own_entry_beats_the_default() {
|
||||
let curves = Curves::parse(
|
||||
"version: 2
|
||||
default:
|
||||
points: [[0.0, 0.0], [0.25, 0.3], [0.5, 0.6], [0.75, 0.85], [1.0, 1.0]]
|
||||
bodies:
|
||||
- make: Canon
|
||||
model: EOS 6D
|
||||
points: [[0.0, 0.0], [0.25, 0.35], [0.5, 0.7], [0.75, 0.9], [1.0, 1.0]]
|
||||
",
|
||||
)
|
||||
.expect("parses");
|
||||
|
||||
assert_eq!(curves.body("Canon", "EOS 6D").ys[1], 0.35);
|
||||
assert_eq!(curves.body("Canon", "EOS 5D").ys[1], 0.30);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn the_make_may_be_repeated_in_the_model() {
|
||||
// Canon writes "Canon" as the make and "Canon EOS 6D" as the model;
|
||||
// rawler's cleaned strings drop the repetition and both reach here.
|
||||
// One entry has to cover both or half the files on a card miss.
|
||||
let curves = Curves::parse(
|
||||
"version: 1
|
||||
bodies:
|
||||
- make: Canon
|
||||
model: EOS 6D
|
||||
points: [[0.0, 0.0], [0.25, 0.35], [0.5, 0.7], [0.75, 0.9], [1.0, 1.0]]
|
||||
",
|
||||
)
|
||||
.expect("parses");
|
||||
|
||||
assert_eq!(curves.body("Canon", "Canon EOS 6D").ys[1], 0.35);
|
||||
assert_eq!(curves.body("Canon", "EOS 6D").ys[1], 0.35);
|
||||
assert_eq!(curves.body("CANON", "eos 6d").ys[1], 0.35);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_corporate_suffix_does_not_hide_a_body() {
|
||||
// The same Z 6 arrives as "Nikon"/"Z 6" from rawler's camera database
|
||||
// and as "NIKON CORPORATION"/"NIKON Z 6" from a DNG converted out of
|
||||
// the same file. Both must find the entry, or converting a file to
|
||||
// DNG would silently change how it renders.
|
||||
let curves = Curves::parse(
|
||||
"version: 1
|
||||
bodies:
|
||||
- make: Nikon
|
||||
model: Z 6
|
||||
points: [[0.0, 0.0], [0.25, 0.35], [0.5, 0.7], [0.75, 0.9], [1.0, 1.0]]
|
||||
",
|
||||
)
|
||||
.expect("parses");
|
||||
|
||||
assert_eq!(curves.body("Nikon", "Z 6").ys[1], 0.35);
|
||||
assert_eq!(curves.body("NIKON CORPORATION", "NIKON Z 6").ys[1], 0.35);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn punctuation_and_spacing_do_not_decide_whether_a_body_is_known() {
|
||||
let curves = Curves::parse(
|
||||
"version: 1
|
||||
bodies:
|
||||
- make: Sony
|
||||
model: ILCE-7M3
|
||||
points: [[0.0, 0.0], [0.25, 0.35], [0.5, 0.7], [0.75, 0.9], [1.0, 1.0]]
|
||||
",
|
||||
)
|
||||
.expect("parses");
|
||||
|
||||
assert_eq!(curves.body("SONY", "ILCE 7M3").ys[1], 0.35);
|
||||
assert_eq!(curves.body("sony", "ilce-7m3").ys[1], 0.35);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn one_bad_entry_does_not_cost_the_rest() {
|
||||
// A user-contributed file with one typo should cost that body's
|
||||
// rendering, not every body's.
|
||||
let curves = Curves::parse(
|
||||
"version: 1
|
||||
bodies:
|
||||
- make: Broken
|
||||
model: Body
|
||||
points: [[0.0, 0.0], [0.25, 0.9], [0.5, 0.1], [0.75, 0.9], [1.0, 1.0]]
|
||||
- make: Canon
|
||||
model: EOS 6D
|
||||
points: [[0.0, 0.0], [0.25, 0.35], [0.5, 0.7], [0.75, 0.9], [1.0, 1.0]]
|
||||
",
|
||||
)
|
||||
.expect("parses");
|
||||
|
||||
assert_eq!(curves.len(), 1);
|
||||
assert_eq!(curves.body("Canon", "EOS 6D").ys[1], 0.35);
|
||||
assert!(curves.body("Broken", "Body").is_identity());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_pack_from_the_future_still_delivers_its_curves() {
|
||||
// The database is versioned independently of the binary, so a pack
|
||||
// published after this build may carry keys this build has never heard
|
||||
// of. It must still hand over the curves it does understand — failing
|
||||
// the parse would leave every body flat, which is the exact failure
|
||||
// FR-DEV-3e exists to prevent, delivered by the mechanism meant to
|
||||
// prevent it.
|
||||
let curves = Curves::parse(
|
||||
"version: 9
|
||||
look_table: ambitious
|
||||
bodies:
|
||||
- make: Canon
|
||||
model: EOS 6D
|
||||
hue_twist: [1, 2, 3]
|
||||
points: [[0.0, 0.0], [0.25, 0.35], [0.5, 0.7], [0.75, 0.9], [1.0, 1.0]]
|
||||
",
|
||||
)
|
||||
.expect("an unfamiliar key must not fail the parse");
|
||||
|
||||
assert_eq!(curves.version(), 9);
|
||||
assert_eq!(curves.body("Canon", "EOS 6D").ys[1], 0.35);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn an_unknown_body_with_no_default_gets_the_identity() {
|
||||
// Graceful fallback, stated as a property: never worse than a flat
|
||||
// render, and never a curve tuned for somebody else's sensor when the
|
||||
// database declines to offer one.
|
||||
let curves = Curves::parse("version: 1\nbodies: []\n").expect("parses");
|
||||
assert!(curves.body("Nobody", "Nothing").is_identity());
|
||||
}
|
||||
|
||||
/// A directory holding one `base_curves.yaml`, unique to the caller.
|
||||
fn a_pack_dir(name: &str, yaml: &str) -> PathBuf {
|
||||
let dir = std::env::temp_dir().join(format!("darkroom-base-curves-{name}"));
|
||||
let _ = std::fs::remove_dir_all(&dir);
|
||||
std::fs::create_dir_all(&dir).expect("a writable temp directory");
|
||||
std::fs::write(dir.join("base_curves.yaml"), yaml).expect("write");
|
||||
dir
|
||||
}
|
||||
|
||||
const A_CANON_ENTRY: &str = "bodies:
|
||||
- make: Canon
|
||||
model: EOS 6D
|
||||
points: [[0.0, 0.0], [0.25, 0.42], [0.5, 0.7], [0.75, 0.9], [1.0, 1.0]]
|
||||
";
|
||||
|
||||
#[test]
|
||||
fn a_newer_pack_on_disk_supersedes_the_built_in_database() {
|
||||
// **This is the requirement.** FR-DEV-3e asks for a profile database
|
||||
// versioned independently of the app binary "so bodies and curves can
|
||||
// be added without a release". A file with a higher version, dropped
|
||||
// in the profile directory, is what that means in practice.
|
||||
let built_in = Curves::parse(BUILT_IN).expect("parses");
|
||||
let newer = format!("version: {}\n{A_CANON_ENTRY}", built_in.version() + 1);
|
||||
let dir = a_pack_dir("newer", &newer);
|
||||
|
||||
let chosen = choose(built_in.clone(), &[dir]);
|
||||
assert_eq!(chosen.version(), built_in.version() + 1);
|
||||
assert_eq!(chosen.body("Canon", "EOS 6D").ys[1], 0.42);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_stale_pack_does_not_survive_an_upgrade() {
|
||||
// The other direction, and the one that protects the user. Somebody
|
||||
// downloads a pack, a release later ships better curves for the same
|
||||
// bodies, and the forgotten file must not quietly hold the application
|
||||
// back at last year's rendering.
|
||||
let built_in = Curves::parse(BUILT_IN).expect("parses");
|
||||
let stale = format!("version: {}\n{A_CANON_ENTRY}", built_in.version());
|
||||
let dir = a_pack_dir("stale", &stale);
|
||||
|
||||
let chosen = choose(built_in.clone(), &[dir]);
|
||||
assert_eq!(chosen.version(), built_in.version());
|
||||
assert_ne!(
|
||||
chosen.body("Canon", "EOS 6D").ys[1],
|
||||
0.42,
|
||||
"an equal version must not displace the built-in database"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_broken_pack_costs_the_curves_and_not_the_photographs() {
|
||||
// A malformed profile file must degrade to the built-in database, not
|
||||
// to an error. The user came here to look at a photograph.
|
||||
let built_in = Curves::parse(BUILT_IN).expect("parses");
|
||||
let dir = a_pack_dir("broken", "version: [this is not a number\n");
|
||||
|
||||
let chosen = choose(built_in.clone(), &[dir]);
|
||||
assert_eq!(chosen.version(), built_in.version());
|
||||
assert_eq!(chosen.len(), built_in.len());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_directory_with_no_pack_in_it_is_simply_skipped() {
|
||||
// The ordinary case on every machine: the search path exists, the file
|
||||
// does not. It must not be a warning, an error, or a slow path.
|
||||
let built_in = Curves::parse(BUILT_IN).expect("parses");
|
||||
let missing = std::env::temp_dir().join("darkroom-base-curves-nothing-here");
|
||||
let _ = std::fs::remove_dir_all(&missing);
|
||||
|
||||
assert_eq!(choose(built_in.clone(), &[missing]), built_in);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn the_identity_is_recognised_as_doing_nothing() {
|
||||
assert!(BaseCurve::IDENTITY.is_identity());
|
||||
assert!(!Curves::parse(BUILT_IN)
|
||||
.expect("parses")
|
||||
.body("Canon", "EOS 6D")
|
||||
.is_identity());
|
||||
}
|
||||
}
|
||||
@@ -1,112 +0,0 @@
|
||||
/// TRACES: FR-RAW-4 | NFR-SEC-1
|
||||
/// Failures from decoding.
|
||||
///
|
||||
/// Per FR-RAW-4 a malformed file must not abort a batch, so these are always
|
||||
/// returned rather than panicking — and the decode path is the one place
|
||||
/// untrusted input arrives (NFR-SEC-1).
|
||||
#[derive(Debug, thiserror::Error)]
|
||||
pub enum DecodeError {
|
||||
#[error("read failed: {0}")]
|
||||
Read(String),
|
||||
|
||||
#[error("unsupported or unrecognised format: {0}")]
|
||||
Unsupported(String),
|
||||
|
||||
#[error("decode failed: {0}")]
|
||||
Decode(String),
|
||||
|
||||
#[error("metadata unavailable: {0}")]
|
||||
Metadata(String),
|
||||
|
||||
#[error("no embedded preview in this file")]
|
||||
NoPreview,
|
||||
|
||||
#[error("embedded preview is corrupt: {0}")]
|
||||
CorruptPreview(String),
|
||||
}
|
||||
|
||||
/// Run a decoder call, and return a panic inside it as an error.
|
||||
///
|
||||
/// TRACES: FR-RAW-4 | NFR-SEC-1 | NFR-R3
|
||||
/// rawler `panic!`s on some malformed input rather than returning `Err` — a
|
||||
/// DNG whose IFD claims a >50000 px image, for one, which is in the reference
|
||||
/// library. A panic on a worker thread ends the thread: the face sweep that
|
||||
/// met that file stopped 13 seconds in, three sweeps running, with "17301
|
||||
/// image(s) to index" as the last word and nothing to say why. FR-RAW-4's
|
||||
/// rule — a malformed file must not abort a batch — is this crate's to keep
|
||||
/// whatever the library beneath it does, so every entry point that calls into
|
||||
/// rawler runs through here, and a file that panics the decoder is one failed
|
||||
/// file like any other.
|
||||
///
|
||||
/// The crash hook still records the panic, because it runs before unwinding
|
||||
/// reaches this frame; that is right — it is a real defect in a dependency
|
||||
/// and the record is how it gets reported upstream — and a repeat is the same
|
||||
/// file being met again rather than a new fault.
|
||||
pub(crate) fn guarded<T>(
|
||||
what: &'static str,
|
||||
f: impl FnOnce() -> Result<T, DecodeError>,
|
||||
) -> Result<T, DecodeError> {
|
||||
match std::panic::catch_unwind(std::panic::AssertUnwindSafe(f)) {
|
||||
Ok(result) => result,
|
||||
Err(payload) => {
|
||||
let msg = payload
|
||||
.downcast_ref::<&str>()
|
||||
.map(|s| s.to_string())
|
||||
.or_else(|| payload.downcast_ref::<String>().cloned())
|
||||
.unwrap_or_else(|| "no message".to_string());
|
||||
Err(DecodeError::Decode(format!(
|
||||
"{what}: the decoder panicked on this file: {msg}"
|
||||
)))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl DecodeError {
|
||||
/// Whether a fallback path might still produce an image.
|
||||
///
|
||||
/// A missing preview is not a failure to display the file — it means fall
|
||||
/// through to full decode (FR-CULL-2, M-11).
|
||||
pub fn has_fallback(&self) -> bool {
|
||||
matches!(
|
||||
self,
|
||||
DecodeError::NoPreview | DecodeError::CorruptPreview(_)
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn preview_failures_fall_through_rather_than_failing() {
|
||||
assert!(DecodeError::NoPreview.has_fallback());
|
||||
assert!(DecodeError::CorruptPreview("truncated".into()).has_fallback());
|
||||
// A genuinely unsupported file has nowhere to fall through to.
|
||||
assert!(!DecodeError::Unsupported("unknown".into()).has_fallback());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_panic_in_the_decoder_is_an_error_and_the_thread_survives() {
|
||||
// The property the face sweep relies on: one file that panics rawler
|
||||
// is one failed file, not the end of the pass. The message travels,
|
||||
// because "decode failed" alone sends the reader to the crash log.
|
||||
let err = guarded("decode", || -> Result<(), DecodeError> {
|
||||
panic!("rawler: surely there's no such thing as a {}MP image!", 600)
|
||||
})
|
||||
.unwrap_err();
|
||||
let text = err.to_string();
|
||||
assert!(text.contains("panicked"), "{text}");
|
||||
assert!(text.contains("600MP"), "{text}");
|
||||
assert!(!err.has_fallback(), "a panic is not a missing preview");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_result_passes_through_untouched() {
|
||||
assert_eq!(guarded("decode", || Ok::<_, DecodeError>(7)).unwrap(), 7);
|
||||
assert!(matches!(
|
||||
guarded("decode", || Err::<(), _>(DecodeError::NoPreview)),
|
||||
Err(DecodeError::NoPreview)
|
||||
));
|
||||
}
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
@@ -1,412 +0,0 @@
|
||||
//! Embedded preview extraction — the fast display path.
|
||||
//!
|
||||
//! Every RAW container carries one or more JPEG previews, often at or near
|
||||
//! full resolution. Extracting one costs a fraction of a full decode, and is
|
||||
//! what makes culling feel instant (FR-CULL-1, NFR-P13: 50 ms per image).
|
||||
//!
|
||||
//! It is also what makes remote browsing viable: fetching ~1-3 MB of preview
|
||||
//! from an 80 MB file over WebDAV is the difference between usable and not on
|
||||
//! mobile data (FR-NC-3).
|
||||
|
||||
use crate::DecodeError;
|
||||
|
||||
/// How much of a file header to read when locating a preview.
|
||||
///
|
||||
/// Enough to cover the IFD structure of the TIFF-derived formats. Sized for
|
||||
/// remote range requests, where every byte costs.
|
||||
pub const PREVIEW_PROBE_BYTES: u64 = 256 * 1024;
|
||||
|
||||
/// A decoded preview image, RGBA8.
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct Preview {
|
||||
pub width: u32,
|
||||
pub height: u32,
|
||||
/// Tightly packed RGBA, 4 bytes per pixel.
|
||||
pub rgba: Vec<u8>,
|
||||
}
|
||||
|
||||
impl Preview {
|
||||
/// TRACES: FR-DEV-3h
|
||||
/// Turn the pixels the right way up, in place.
|
||||
///
|
||||
/// Every path that shows a preview without the GPU needs this: the grid's
|
||||
/// thumbnails, and the read-only fallback develop shows when no decoder
|
||||
/// could open the file. An embedded preview is written in the sensor's
|
||||
/// orientation, not the photograph's, so a phone or a camera held sideways
|
||||
/// fills the grid with frames on their side until this runs.
|
||||
///
|
||||
/// Done before [`Self::downscale_to`] would be wasteful and after it is
|
||||
/// not: a quarter turn is a permutation, so it costs the same either way,
|
||||
/// and doing it on the smaller buffer moves a fraction of the bytes.
|
||||
///
|
||||
/// The turn itself is [`dr_types::Orientation::into_shown`], which every
|
||||
/// other consumer of an orientation in this codebase also goes through.
|
||||
/// That is deliberate: a hand-written permutation per caller is how two of
|
||||
/// them come to disagree, and a disagreement here shows as a thumbnail
|
||||
/// facing the other way from the develop view.
|
||||
pub fn apply_orientation(&mut self, orientation: dr_types::Orientation) {
|
||||
let (rgba, dw, dh) = orientation.into_shown(&self.rgba, self.width, self.height, 4);
|
||||
self.rgba = rgba;
|
||||
self.width = dw;
|
||||
self.height = dh;
|
||||
}
|
||||
|
||||
/// Downscale in place to fit within `max_dim` on the long edge.
|
||||
///
|
||||
/// A 5472x3648 preview is 79.8 MB of RGBA — far more than a grid cell or
|
||||
/// even a 4K viewport needs, and enough to exhaust a phone's budget after
|
||||
/// a handful of images (NFR-RES-1). Box-filtered rather than nearest, so
|
||||
/// downscaled thumbnails do not alias.
|
||||
pub fn downscale_to(&mut self, max_dim: u32) {
|
||||
let longest = self.width.max(self.height);
|
||||
if longest <= max_dim || longest == 0 {
|
||||
return;
|
||||
}
|
||||
let scale = max_dim as f32 / longest as f32;
|
||||
let (nw, nh) = (
|
||||
((self.width as f32 * scale).round() as u32).max(1),
|
||||
((self.height as f32 * scale).round() as u32).max(1),
|
||||
);
|
||||
|
||||
let mut out = vec![0u8; (nw as usize) * (nh as usize) * 4];
|
||||
let x_ratio = self.width as f32 / nw as f32;
|
||||
let y_ratio = self.height as f32 / nh as f32;
|
||||
|
||||
for y in 0..nh {
|
||||
let y0 = (y as f32 * y_ratio) as u32;
|
||||
let y1 = (((y + 1) as f32 * y_ratio) as u32)
|
||||
.min(self.height)
|
||||
.max(y0 + 1);
|
||||
for x in 0..nw {
|
||||
let x0 = (x as f32 * x_ratio) as u32;
|
||||
let x1 = (((x + 1) as f32 * x_ratio) as u32)
|
||||
.min(self.width)
|
||||
.max(x0 + 1);
|
||||
|
||||
let (mut r, mut g, mut b, mut n) = (0u32, 0u32, 0u32, 0u32);
|
||||
for sy in y0..y1 {
|
||||
for sx in x0..x1 {
|
||||
let i = ((sy * self.width + sx) * 4) as usize;
|
||||
r += self.rgba[i] as u32;
|
||||
g += self.rgba[i + 1] as u32;
|
||||
b += self.rgba[i + 2] as u32;
|
||||
n += 1;
|
||||
}
|
||||
}
|
||||
let n = n.max(1);
|
||||
let o = ((y * nw + x) * 4) as usize;
|
||||
out[o] = (r / n) as u8;
|
||||
out[o + 1] = (g / n) as u8;
|
||||
out[o + 2] = (b / n) as u8;
|
||||
out[o + 3] = 255;
|
||||
}
|
||||
}
|
||||
|
||||
self.rgba = out;
|
||||
self.width = nw;
|
||||
self.height = nh;
|
||||
}
|
||||
|
||||
/// Whether this is large enough to be worth displaying at `target`.
|
||||
///
|
||||
/// Some bodies embed thumbnails only a few hundred pixels wide — Sony is
|
||||
/// the documented case. Displaying one where a larger render is wanted
|
||||
/// shows a soft image the user discovers only on zoom, so the caller
|
||||
/// should background-render instead (M-11).
|
||||
pub fn is_useful_at(&self, target: u32) -> bool {
|
||||
self.width.max(self.height) >= target
|
||||
}
|
||||
}
|
||||
|
||||
/// TRACES: FR-CULL-1 | NFR-P13
|
||||
/// Which embedded image to extract.
|
||||
///
|
||||
/// Containers carry several at different sizes, and decoding the
|
||||
/// full-resolution one to fill a grid cell is pure waste.
|
||||
///
|
||||
/// **Measured caveat (rawler 0.7.2):** the CR2 decoder implements only
|
||||
/// `full_image`; `thumbnail_image` and `preview_image` are unimplemented trait
|
||||
/// defaults returning `None`. So on Canon CR2 every rung currently resolves to
|
||||
/// the full-resolution JPEG at ~250 ms — 5× over NFR-P13's 50 ms budget.
|
||||
///
|
||||
/// Three ways out, in increasing cost: extract the smaller IFD ourselves
|
||||
/// (CR2 carries a 160×120 thumbnail and a ~1620×1080 preview in IFD1/IFD2),
|
||||
/// contribute the methods upstream, or cache a downscaled proxy on first
|
||||
/// sight. The ladder is written now so that fixing it is a decoder change
|
||||
/// rather than a change to every caller.
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub enum PreviewSize {
|
||||
/// Smallest available. Grid cells and rapid culling.
|
||||
Thumbnail,
|
||||
/// Mid-sized where the container has one. Single-image view.
|
||||
Screen,
|
||||
/// Largest available, usually full sensor resolution. Only where the
|
||||
/// display genuinely needs it.
|
||||
Full,
|
||||
}
|
||||
|
||||
/// TRACES: FR-CULL-2 | FR-NC-3 | M-10
|
||||
/// Extract and decode an embedded preview at the requested size.
|
||||
///
|
||||
/// Takes bytes rather than a reader, because the caller usually has them
|
||||
/// already: a range read locally, or a `Range:` request remotely. Forcing a
|
||||
/// `Read + Seek` here would push remote callers into buffering the whole file.
|
||||
///
|
||||
/// Falls through the ladder — a container without the requested size yields
|
||||
/// the next available rather than failing (FR-CULL-2).
|
||||
///
|
||||
/// Returns [`DecodeError::NoPreview`] where there is none at all: a
|
||||
/// fall-through signal, not a failure (see [`DecodeError::has_fallback`]).
|
||||
pub fn extract_preview(bytes: &[u8], size: PreviewSize) -> Result<Preview, DecodeError> {
|
||||
crate::error::guarded("preview", || extract_preview_unguarded(bytes, size))
|
||||
}
|
||||
|
||||
fn extract_preview_unguarded(bytes: &[u8], size: PreviewSize) -> Result<Preview, DecodeError> {
|
||||
use rawler::rawsource::RawSource;
|
||||
|
||||
// A plain JPEG *is* its own preview — rawler has no decoder for one, and
|
||||
// a mixed folder must display sensibly (M-9).
|
||||
if bytes.starts_with(&[0xFF, 0xD8, 0xFF]) {
|
||||
return decode_jpeg(bytes);
|
||||
}
|
||||
|
||||
let source = RawSource::new_from_slice(bytes);
|
||||
let decoder =
|
||||
rawler::get_decoder(&source).map_err(|e| DecodeError::Unsupported(e.to_string()))?;
|
||||
let params = Default::default();
|
||||
|
||||
// Preference order per requested size, each falling through to the next.
|
||||
let attempts: &[PreviewSize] = match size {
|
||||
PreviewSize::Thumbnail => &[
|
||||
PreviewSize::Thumbnail,
|
||||
PreviewSize::Screen,
|
||||
PreviewSize::Full,
|
||||
],
|
||||
PreviewSize::Screen => &[
|
||||
PreviewSize::Screen,
|
||||
PreviewSize::Full,
|
||||
PreviewSize::Thumbnail,
|
||||
],
|
||||
PreviewSize::Full => &[PreviewSize::Full, PreviewSize::Screen],
|
||||
};
|
||||
|
||||
for attempt in attempts {
|
||||
let got = match attempt {
|
||||
PreviewSize::Thumbnail => decoder.thumbnail_image(&source, ¶ms),
|
||||
PreviewSize::Screen => decoder.preview_image(&source, ¶ms),
|
||||
PreviewSize::Full => decoder.full_image(&source, ¶ms),
|
||||
};
|
||||
if let Ok(Some(img)) = got {
|
||||
let rgb = img.to_rgb8();
|
||||
let (width, height) = (rgb.width(), rgb.height());
|
||||
if width > 0 && height > 0 {
|
||||
return Ok(Preview {
|
||||
width,
|
||||
height,
|
||||
rgba: rgb_to_rgba(rgb.as_raw(), width, height),
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Err(DecodeError::NoPreview)
|
||||
}
|
||||
|
||||
/// Extract the largest available preview.
|
||||
///
|
||||
/// Convenience over [`extract_preview`]; prefer naming a size explicitly.
|
||||
pub fn extract_embedded_preview(bytes: &[u8]) -> Result<Preview, DecodeError> {
|
||||
extract_preview(bytes, PreviewSize::Full)
|
||||
}
|
||||
|
||||
/// Decode a standalone JPEG (an embedded preview already sliced out, or a
|
||||
/// JPEG file).
|
||||
pub fn decode_jpeg(bytes: &[u8]) -> Result<Preview, DecodeError> {
|
||||
let mut d = zune_jpeg::JpegDecoder::new(bytes);
|
||||
let pixels = d
|
||||
.decode()
|
||||
.map_err(|e| DecodeError::CorruptPreview(e.to_string()))?;
|
||||
let info = d
|
||||
.info()
|
||||
.ok_or_else(|| DecodeError::CorruptPreview("no image info".into()))?;
|
||||
|
||||
let (w, h) = (info.width as u32, info.height as u32);
|
||||
let expected = (w as usize) * (h as usize);
|
||||
|
||||
// zune yields RGB or grayscale depending on the source; normalise both to
|
||||
// RGBA so callers have one representation.
|
||||
let rgba = match pixels.len() / expected.max(1) {
|
||||
3 => rgb_to_rgba(&pixels, w, h),
|
||||
1 => pixels.iter().flat_map(|&g| [g, g, g, 255]).collect(),
|
||||
4 => pixels,
|
||||
n => {
|
||||
return Err(DecodeError::CorruptPreview(format!(
|
||||
"unexpected {n} channels"
|
||||
)))
|
||||
}
|
||||
};
|
||||
|
||||
Ok(Preview {
|
||||
width: w,
|
||||
height: h,
|
||||
rgba,
|
||||
})
|
||||
}
|
||||
|
||||
fn rgb_to_rgba(rgb: &[u8], w: u32, h: u32) -> Vec<u8> {
|
||||
let n = (w as usize) * (h as usize);
|
||||
let mut out = Vec::with_capacity(n * 4);
|
||||
for px in rgb.chunks_exact(3).take(n) {
|
||||
out.extend_from_slice(&[px[0], px[1], px[2], 255]);
|
||||
}
|
||||
out
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
/// A preview whose every pixel encodes its own coordinates, so a
|
||||
/// misplaced one is identifiable rather than merely wrong.
|
||||
fn coded(width: u32, height: u32) -> Preview {
|
||||
let mut rgba = Vec::with_capacity((width * height * 4) as usize);
|
||||
for y in 0..height {
|
||||
for x in 0..width {
|
||||
rgba.extend_from_slice(&[x as u8, y as u8, 0, 255]);
|
||||
}
|
||||
}
|
||||
Preview {
|
||||
width,
|
||||
height,
|
||||
rgba,
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_quarter_turn_moves_every_pixel_where_the_orientation_says() {
|
||||
// Tag 6: the stored image's first row becomes the displayed right
|
||||
// edge, its first column the displayed top. A 4x2 landscape preview
|
||||
// therefore comes out 2x4 portrait, with stored (0,0) at the top right.
|
||||
let mut p = coded(4, 2);
|
||||
p.apply_orientation(dr_types::Orientation::from_exif(6));
|
||||
|
||||
assert_eq!((p.width, p.height), (2, 4));
|
||||
let at = |x: u32, y: u32| {
|
||||
let i = ((y * p.width + x) * 4) as usize;
|
||||
(p.rgba[i], p.rgba[i + 1])
|
||||
};
|
||||
// Displayed top-right reads stored (0, 0).
|
||||
assert_eq!(at(1, 0), (0, 0));
|
||||
// Displayed top-left reads stored (0, 1) — the last row of column 0.
|
||||
assert_eq!(at(0, 0), (0, 1));
|
||||
// Displayed bottom-right reads stored (3, 0).
|
||||
assert_eq!(at(1, 3), (3, 0));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn an_upright_file_is_left_untouched() {
|
||||
// The common case, and the one where an unnecessary reallocation
|
||||
// would be paid on every thumbnail in the library.
|
||||
let original = coded(4, 2);
|
||||
let mut p = original.clone();
|
||||
p.apply_orientation(dr_types::Orientation::NORMAL);
|
||||
assert_eq!(p, original);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn every_orientation_preserves_the_pixels_it_was_given() {
|
||||
// A turn or a mirror is a permutation: the same bytes, rearranged.
|
||||
// Anything else means a pixel was dropped, duplicated or read out of
|
||||
// bounds — and the bounds case would have panicked first.
|
||||
for tag in 1..=8u16 {
|
||||
let orientation = dr_types::Orientation::from_exif(tag);
|
||||
let mut p = coded(5, 3);
|
||||
p.apply_orientation(orientation);
|
||||
|
||||
assert_eq!(
|
||||
(p.width, p.height),
|
||||
orientation.oriented_size(5, 3),
|
||||
"tag {tag}"
|
||||
);
|
||||
let mut got: Vec<_> = p.rgba.chunks(4).map(|c| (c[0], c[1])).collect();
|
||||
got.sort_unstable();
|
||||
let mut want: Vec<_> = coded(5, 3).rgba.chunks(4).map(|c| (c[0], c[1])).collect();
|
||||
want.sort_unstable();
|
||||
assert_eq!(got, want, "tag {tag}");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn size_preference_falls_through_in_order() {
|
||||
// A container missing the requested size must yield the next
|
||||
// available rather than failing (FR-CULL-2).
|
||||
// Ordering is asserted here; behaviour against real files is covered
|
||||
// by the smoke example.
|
||||
assert_ne!(PreviewSize::Thumbnail, PreviewSize::Full);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn usefulness_is_judged_on_the_long_edge() {
|
||||
let p = Preview {
|
||||
width: 1600,
|
||||
height: 1067,
|
||||
rgba: Vec::new(),
|
||||
};
|
||||
assert!(p.is_useful_at(1024));
|
||||
assert!(p.is_useful_at(1600));
|
||||
// A body embedding only a small thumbnail must trigger a background
|
||||
// render rather than showing a soft image.
|
||||
assert!(!p.is_useful_at(2048));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn downscale_preserves_aspect_and_bounds_memory() {
|
||||
let mut p = Preview {
|
||||
width: 5472,
|
||||
height: 3648,
|
||||
rgba: vec![128; 5472 * 3648 * 4],
|
||||
};
|
||||
assert_eq!(p.rgba.len(), 79_847_424);
|
||||
|
||||
p.downscale_to(2048);
|
||||
assert_eq!(p.width, 2048);
|
||||
assert_eq!(p.height, 1365, "aspect preserved");
|
||||
assert_eq!(p.rgba.len(), (2048 * 1365 * 4) as usize);
|
||||
// A flat source must stay flat through the box filter.
|
||||
assert!(p
|
||||
.rgba
|
||||
.chunks_exact(4)
|
||||
.all(|px| px[0] == 128 && px[3] == 255));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn downscale_is_a_noop_when_already_small() {
|
||||
let mut p = Preview {
|
||||
width: 720,
|
||||
height: 480,
|
||||
rgba: vec![7; 720 * 480 * 4],
|
||||
};
|
||||
let before = p.rgba.len();
|
||||
p.downscale_to(2048);
|
||||
assert_eq!((p.width, p.height, p.rgba.len()), (720, 480, before));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn rgb_expands_to_rgba_opaque() {
|
||||
let rgb = [10, 20, 30, 40, 50, 60];
|
||||
let rgba = rgb_to_rgba(&rgb, 2, 1);
|
||||
assert_eq!(rgba, vec![10, 20, 30, 255, 40, 50, 60, 255]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn corrupt_jpeg_is_an_error_not_a_panic() {
|
||||
// Untrusted input arrives here (NFR-SEC-1); it must never panic.
|
||||
let err = decode_jpeg(&[0xFF, 0xD8, 0x00, 0x01, 0x02]).unwrap_err();
|
||||
assert!(matches!(err, DecodeError::CorruptPreview(_)));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn empty_input_is_an_error_not_a_panic() {
|
||||
assert!(decode_jpeg(&[]).is_err());
|
||||
}
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
@@ -1,44 +0,0 @@
|
||||
[package]
|
||||
name = "dr-export"
|
||||
version.workspace = true
|
||||
edition.workspace = true
|
||||
rust-version.workspace = true
|
||||
license.workspace = true
|
||||
|
||||
# No platform dependency and no filesystem, deliberately. This crate turns a
|
||||
# rendered frame into *bytes* and a *name*; where those bytes go is the
|
||||
# caller's problem, because the answer differs by more than a path. On Linux
|
||||
# it is a file, on Android a SAF document descriptor with no path at all
|
||||
# (ARCH §6.9), and on either it may be a `PUT` to the server. A crate that
|
||||
# took a `Path` would work on exactly one of the three.
|
||||
[dependencies]
|
||||
dr-types.workspace = true
|
||||
log.workspace = true
|
||||
thiserror.workspace = true
|
||||
|
||||
# Encoders. All three are pure Rust and already in the tree, which is the same
|
||||
# criterion that chose rustls, bundled SQLite and the Lensfun port: a C
|
||||
# dependency here would be one more thing to satisfy under the Android NDK.
|
||||
#
|
||||
# AVIF and JPEG XL (FR-EXP-1) are deliberately absent. The mature encoders for
|
||||
# both are C or C++ — libaom and libjxl — and ravif, the pure-Rust AVIF path,
|
||||
# is slow enough to change what a batch export feels like. Neither belongs in
|
||||
# the first version; see `format` in lib.rs for what happens when one is asked
|
||||
# for.
|
||||
jpeg-encoder.workspace = true
|
||||
png = "0.18"
|
||||
tiff = "0.11"
|
||||
|
||||
# The example runs the whole path — decode, GPU render, read back, encode,
|
||||
# write — so it needs what the library deliberately does not: a GPU, a
|
||||
# pipeline and a decoder. Dev-only, so none of it reaches a dependent.
|
||||
[dev-dependencies]
|
||||
dr-decode.workspace = true
|
||||
dr-gpu.workspace = true
|
||||
dr-pipeline.workspace = true
|
||||
env_logger.workspace = true
|
||||
pollster.workspace = true
|
||||
# The DNG writer's test reads its output back through the decoder the
|
||||
# library uses, which is the whole claim the writer makes (S15.1).
|
||||
rawler.workspace = true
|
||||
zune-jpeg.workspace = true
|
||||
@@ -1,211 +0,0 @@
|
||||
//! Export a real file, end to end, from a real image.
|
||||
//!
|
||||
//! cargo run -p dr-export --example export -- <file.jpg|file.cr2> [out-dir]
|
||||
//!
|
||||
//! Deliberately the *whole* path and not a unit test of the encoder: decode,
|
||||
//! demosaic or upload, run the develop chain on the GPU at full resolution,
|
||||
//! read the result back through `AdjustPass::export_pixels`, resize, sharpen,
|
||||
//! encode, and write. A test can prove the JPEG has the right magic bytes; it
|
||||
//! cannot tell anyone whether the picture came out looking like the picture.
|
||||
|
||||
use std::path::PathBuf;
|
||||
|
||||
use dr_export::{export, Frame, NameContext, SourceMetadata};
|
||||
use dr_gpu::{AdjustPass, DemosaicedImage, Demosaicer, GpuContext};
|
||||
use dr_pipeline::EditGraph;
|
||||
use dr_types::{ColourSpace, ExportFormat, ExportSettings, OutputSharpening, SizingMode};
|
||||
|
||||
fn main() {
|
||||
env_logger::Builder::from_env(env_logger::Env::default().default_filter_or("info,wgpu=warn"))
|
||||
.init();
|
||||
|
||||
let mut args = std::env::args().skip(1);
|
||||
let Some(input) = args.next() else {
|
||||
eprintln!("usage: export <file.jpg|file.cr2> [out-dir]");
|
||||
std::process::exit(2);
|
||||
};
|
||||
let out_dir = PathBuf::from(args.next().unwrap_or_else(|| ".".into()));
|
||||
let input = PathBuf::from(input);
|
||||
|
||||
let ctx = pollster::block_on(GpuContext::new_headless()).expect("gpu");
|
||||
println!("gpu: {} ({:?})", ctx.adapter_name(), ctx.backend());
|
||||
|
||||
// Decode. A RAW goes through the demosaicer; a JPEG is already RGB and
|
||||
// takes the same path every operation after the sensor stage does.
|
||||
let bytes = std::fs::read(&input).expect("read input");
|
||||
// From content, not from the extension — dr-decode is emphatic that an
|
||||
// extension is only a hint. Its own `probe` reports a crate-private
|
||||
// `Format`, so the SOI marker is checked directly here rather than
|
||||
// widening that API for an example.
|
||||
let is_jpeg = bytes.starts_with(&[0xFF, 0xD8, 0xFF]);
|
||||
let source = if !is_jpeg {
|
||||
let raw = dr_decode::decode(&bytes).expect("decode raw");
|
||||
let demosaicer = Demosaicer::new(&ctx).expect("demosaicer");
|
||||
demosaicer.run(&raw).expect("demosaic")
|
||||
} else {
|
||||
let (rgba, w, h) = decode_jpeg(&bytes);
|
||||
DemosaicedImage::from_rgba8(&ctx, &rgba, w, h).expect("upload")
|
||||
};
|
||||
|
||||
// An edit worth seeing in the output, so a broken pipeline is obvious
|
||||
// rather than subtle.
|
||||
let mut graph = EditGraph::default_chain();
|
||||
graph.set_param(
|
||||
dr_pipeline::ops::exposure::ID,
|
||||
dr_pipeline::ops::exposure::EXPOSURE,
|
||||
0.35,
|
||||
);
|
||||
graph.set_param(
|
||||
dr_pipeline::ops::contrast::ID,
|
||||
dr_pipeline::ops::contrast::CONTRAST,
|
||||
18.0,
|
||||
);
|
||||
graph.set_param(
|
||||
dr_pipeline::ops::saturation::ID,
|
||||
dr_pipeline::ops::saturation::SATURATION,
|
||||
12.0,
|
||||
);
|
||||
|
||||
// Full resolution, not the viewport (FR-EXP-9). This is the one thing an
|
||||
// export must not economise on.
|
||||
let (sw, sh) = source.size();
|
||||
let (fw, fh) = graph.output_size(sw, sh);
|
||||
println!("source {sw}×{sh}, framed {fw}×{fh}");
|
||||
|
||||
// The output space is chosen *here*, before the render, because that is
|
||||
// where it takes effect: the primaries conversion and the encode are the
|
||||
// last two lines of the generated shader (FR-EXP-2). Asking for it at the
|
||||
// encoder would be too late — the pixels would already be clipped.
|
||||
let space = ColourSpace::DisplayP3;
|
||||
|
||||
let mut adjust = AdjustPass::new(&ctx);
|
||||
let shader = graph.compose_for(space);
|
||||
let t = std::time::Instant::now();
|
||||
adjust.render(&source, &shader, fw, fh).expect("render");
|
||||
let (pixels, w, h) = adjust.export_pixels().expect("read back");
|
||||
println!(
|
||||
"rendered {w}×{h} in {:.0} ms as {}",
|
||||
t.elapsed().as_secs_f32() * 1000.0,
|
||||
space.label()
|
||||
);
|
||||
|
||||
let frame = Frame::in_space(w, h, pixels, space).expect("well-formed frame");
|
||||
|
||||
let stem = input
|
||||
.file_stem()
|
||||
.map(|s| s.to_string_lossy().into_owned())
|
||||
.unwrap_or_else(|| "export".into());
|
||||
|
||||
// TRACES: FR-EXP-8
|
||||
// What the input said about itself, transcribed field by field into the
|
||||
// allowlist `dr-export` will write from. The example passes it because
|
||||
// this is the one place in the tree that produces files a person can open
|
||||
// in exiftool — a unit test can prove a GPS directory is absent from a
|
||||
// byte slice, but only a real export proves that a real photograph comes
|
||||
// out of the far end still knowing which camera took it.
|
||||
//
|
||||
// The defaults apply, so the files written here carry the camera, the
|
||||
// lens, the exposure and the rights statement, and carry no coordinates.
|
||||
let meta = dr_decode::metadata(&bytes).unwrap_or_default();
|
||||
let source_metadata = SourceMetadata {
|
||||
make: meta.make.clone(),
|
||||
model: meta.model.clone(),
|
||||
lens: meta.lens.clone(),
|
||||
shutter: meta.shutter,
|
||||
aperture: meta.aperture,
|
||||
iso: meta.iso,
|
||||
focal_length: meta.focal_length,
|
||||
captured_at: meta.captured_at,
|
||||
captured_offset: meta.captured_offset,
|
||||
artist: meta.artist.clone(),
|
||||
copyright: meta.copyright.clone(),
|
||||
location: meta.location,
|
||||
};
|
||||
|
||||
// One of each format, so the run exercises every encoder that exists.
|
||||
for (format, sizing, sharpening) in [
|
||||
(
|
||||
ExportFormat::Jpeg,
|
||||
SizingMode::Original,
|
||||
OutputSharpening::None,
|
||||
),
|
||||
(
|
||||
ExportFormat::Jpeg,
|
||||
SizingMode::LongEdge(1200),
|
||||
OutputSharpening::Screen,
|
||||
),
|
||||
(
|
||||
ExportFormat::Png,
|
||||
SizingMode::LongEdge(600),
|
||||
OutputSharpening::Screen,
|
||||
),
|
||||
(
|
||||
ExportFormat::Tiff8,
|
||||
SizingMode::Percentage(25),
|
||||
OutputSharpening::MattePaper,
|
||||
),
|
||||
(
|
||||
ExportFormat::Tiff16,
|
||||
SizingMode::Percentage(25),
|
||||
OutputSharpening::MattePaper,
|
||||
),
|
||||
] {
|
||||
let settings = ExportSettings {
|
||||
format,
|
||||
sizing,
|
||||
sharpening,
|
||||
colour_space: space,
|
||||
filename_template: "{name}-{dimensions}".into(),
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
// The size has to be known before the name, because `{dimensions}` is
|
||||
// part of it — which is why sizing is resolved here and not inside
|
||||
// `export`.
|
||||
let (tw, th) = dr_export::target_size(w, h, sizing, settings.allow_upscaling);
|
||||
let ctx = NameContext {
|
||||
source_stem: &stem,
|
||||
sequence: 1,
|
||||
date: "",
|
||||
width: tw,
|
||||
height: th,
|
||||
preset: "",
|
||||
};
|
||||
let name = dr_export::resolve_name(
|
||||
&settings.filename_template,
|
||||
&ctx,
|
||||
format,
|
||||
settings.collision,
|
||||
&|n| out_dir.join(n).exists(),
|
||||
)
|
||||
.expect("a free name");
|
||||
|
||||
let t = std::time::Instant::now();
|
||||
let out = export(&frame, &settings, name, Some(&source_metadata)).expect("export");
|
||||
let path = out_dir.join(&out.name);
|
||||
std::fs::write(&path, &out.bytes).expect("write");
|
||||
println!(
|
||||
"{:>10} {:>5}×{:<5} {:>8} KB {:>5.0} ms {}",
|
||||
format.label(),
|
||||
out.width,
|
||||
out.height,
|
||||
out.bytes.len() / 1024,
|
||||
t.elapsed().as_secs_f32() * 1000.0,
|
||||
path.display()
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
fn decode_jpeg(bytes: &[u8]) -> (Vec<u8>, u32, u32) {
|
||||
let mut decoder = zune_jpeg::JpegDecoder::new(bytes);
|
||||
let pixels = decoder.decode().expect("decode jpeg");
|
||||
let info = decoder.info().expect("jpeg info");
|
||||
let (w, h) = (u32::from(info.width), u32::from(info.height));
|
||||
|
||||
// zune gives RGB; the GPU upload wants RGBA.
|
||||
let mut rgba = Vec::with_capacity((w * h * 4) as usize);
|
||||
for px in pixels.chunks_exact(3) {
|
||||
rgba.extend_from_slice(&[px[0], px[1], px[2], 255]);
|
||||
}
|
||||
(rgba, w, h)
|
||||
}
|
||||
@@ -1,323 +0,0 @@
|
||||
//! TRACES: FR-MRG-3
|
||||
//! A linear DNG: the container a merge writes its composite into.
|
||||
//!
|
||||
//! Decided by S15.1 (2026-09-19): rawler reads back a `LinearRaw` DNG the
|
||||
//! application writes, so a composite re-enters the library as
|
||||
//! `Format::Dng` through the decoder every camera DNG uses. What is written
|
||||
//! is a RAW in every sense a warp can preserve — camera-linear `u16`
|
||||
//! samples at the first source's own scale, its matrices, illuminants,
|
||||
//! as-shot neutral and body name — so the panorama is developed afterwards
|
||||
//! as one photograph, from the sensor's numbers.
|
||||
//!
|
||||
//! # Streamed, not buffered
|
||||
//!
|
||||
//! The composite is larger than any single photograph the pipeline renders
|
||||
//! and larger than the tablet's memory (FR-MRG-11), so the writer never
|
||||
//! holds it. Strips are pulled from the caller one at a time through a
|
||||
//! closure, in order, and written as they arrive; the caller renders a band
|
||||
//! of chunks, hands over its rows, and moves on.
|
||||
//!
|
||||
//! # Why the `tiff` crate after all
|
||||
//!
|
||||
//! S15.1's spike hand-rolled its IFD because the crate's encoder fixes
|
||||
//! `PhotometricInterpretation` to RGB when the image is opened. It does — but
|
||||
//! a directory is a map and a later `write_tag` on the same tag replaces the
|
||||
//! earlier, so `LinearRaw` goes in over the top and everything else the
|
||||
//! crate does (strips, offsets, sub-IFDs, the EXIF block `encode.rs` already
|
||||
//! knows how to write) is kept.
|
||||
|
||||
use std::io::{Seek, Write};
|
||||
|
||||
use tiff::encoder::{colortype, DirectoryEncoder, SRational, TiffEncoder, TiffKind, TiffValue};
|
||||
use tiff::tags::Tag;
|
||||
|
||||
use crate::encode::{sub_directories, tag_metadata, Ascii, Rationals};
|
||||
use crate::{ExportError, SourceMetadata};
|
||||
|
||||
/// What the DNG says about the camera that "took" the composite: the first
|
||||
/// source's profile, carried across so the composite develops through it.
|
||||
#[derive(Debug, Clone, PartialEq)]
|
||||
pub struct DngProfile {
|
||||
/// `UniqueCameraModel`, the name the profile database matches on.
|
||||
pub unique_model: String,
|
||||
/// `(CalibrationIlluminant, ColorMatrix)`: the EXIF light-source code and
|
||||
/// the XYZ → camera matrix measured under it. One or two.
|
||||
pub calibrations: Vec<(u16, [[f32; 3]; 3])>,
|
||||
/// `AsShotNeutral`, camera RGB of the scene's white.
|
||||
pub as_shot_neutral: [f32; 3],
|
||||
/// `WhiteLevel`: the sample value that is clipping. The first source's
|
||||
/// white minus its black, since the samples are black-subtracted.
|
||||
pub white_level: u32,
|
||||
}
|
||||
|
||||
/// Write a linear DNG, pulling `rows_per_strip`-row strips from `strips`.
|
||||
///
|
||||
/// Each call to `strips` receives the strip index and a buffer to fill with
|
||||
/// `width × rows × 3` interleaved RGB `u16` samples (the last strip may be
|
||||
/// shorter). `source` supplies the `Make`, `Model`, dates and EXIF block
|
||||
/// exactly as an export does (FR-EXP-8 sanitising already applied by the
|
||||
/// caller).
|
||||
///
|
||||
/// `PhotometricInterpretation = LinearRaw`, `DNGVersion 1.4`, uncompressed,
|
||||
/// `Orientation = 1` — the composite is written upright (panorama.md §8).
|
||||
///
|
||||
/// `crop` is asked once every strip is in, and its answer — the largest
|
||||
/// rectangle the frames covered, found while the strips went by
|
||||
/// (`Inscribed`) — becomes `DefaultCropOrigin`/`DefaultCropSize`
|
||||
/// (FR-MRG-4): the file opens on the picture, and the border is still in it.
|
||||
// Eight arguments, and each is a different thing: the sink, three
|
||||
// dimensions, the profile, the header, the strip source and the crop. A
|
||||
// struct for them would be a struct with one caller.
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
pub fn write_linear_dng<W, F, C>(
|
||||
out: W,
|
||||
width: u32,
|
||||
height: u32,
|
||||
rows_per_strip: u32,
|
||||
profile: &DngProfile,
|
||||
source: Option<&SourceMetadata>,
|
||||
mut strips: F,
|
||||
crop: C,
|
||||
) -> Result<(), ExportError>
|
||||
where
|
||||
W: Write + Seek,
|
||||
F: FnMut(usize, &mut Vec<u16>) -> Result<(), ExportError>,
|
||||
C: FnOnce() -> Option<crate::Rect>,
|
||||
{
|
||||
let enc = |e: tiff::TiffError| ExportError::Encode(e.to_string());
|
||||
let mut encoder = TiffEncoder::new(out).map_err(enc)?;
|
||||
let sub = sub_directories(&mut encoder, source, width, height)?;
|
||||
let mut image = encoder
|
||||
.new_image::<colortype::RGB16>(width, height)
|
||||
.map_err(enc)?;
|
||||
image.rows_per_strip(rows_per_strip.max(1)).map_err(enc)?;
|
||||
tag_metadata(image.encoder(), source, &sub)?;
|
||||
tag_dng(image.encoder(), profile).map_err(enc)?;
|
||||
|
||||
let rows = rows_per_strip.max(1);
|
||||
let strip_count = height.div_ceil(rows) as usize;
|
||||
let mut buf: Vec<u16> = Vec::with_capacity((width * rows * 3) as usize);
|
||||
for k in 0..strip_count {
|
||||
buf.clear();
|
||||
strips(k, &mut buf)?;
|
||||
let expected_rows = rows.min(height - k as u32 * rows);
|
||||
let expected = (width * expected_rows * 3) as usize;
|
||||
if buf.len() != expected {
|
||||
return Err(ExportError::Encode(format!(
|
||||
"strip {k} has {} samples, expected {expected}",
|
||||
buf.len()
|
||||
)));
|
||||
}
|
||||
image.write_strip(&buf).map_err(enc)?;
|
||||
}
|
||||
if let Some(r) = crop().filter(|r| r.width > 0 && r.height > 0) {
|
||||
let r = crate::Rect {
|
||||
x: r.x.min(width - 1),
|
||||
y: r.y.min(height - 1),
|
||||
width: r.width.min(width - r.x.min(width - 1)),
|
||||
height: r.height.min(height - r.y.min(height - 1)),
|
||||
};
|
||||
image
|
||||
.encoder()
|
||||
.write_tag(Tag::Unknown(tag::DEFAULT_CROP_ORIGIN), &[r.x, r.y][..])
|
||||
.map_err(enc)?;
|
||||
image
|
||||
.encoder()
|
||||
.write_tag(
|
||||
Tag::Unknown(tag::DEFAULT_CROP_SIZE),
|
||||
&[r.width, r.height][..],
|
||||
)
|
||||
.map_err(enc)?;
|
||||
}
|
||||
image.finish().map_err(enc)
|
||||
}
|
||||
|
||||
/// The tags that make a TIFF a DNG, and a linear one.
|
||||
fn tag_dng<W, K>(dir: &mut DirectoryEncoder<'_, W, K>, profile: &DngProfile) -> tiff::TiffResult<()>
|
||||
where
|
||||
W: Write + Seek,
|
||||
K: TiffKind,
|
||||
{
|
||||
// Over the top of what `new_image` wrote: this is the whole trick.
|
||||
dir.write_tag(Tag::PhotometricInterpretation, LINEAR_RAW)?;
|
||||
dir.write_tag(Tag::Orientation, 1u16)?;
|
||||
dir.write_tag(Tag::Unknown(tag::DNG_VERSION), &[1u8, 4, 0, 0][..])?;
|
||||
dir.write_tag(Tag::Unknown(tag::DNG_BACKWARD_VERSION), &[1u8, 4, 0, 0][..])?;
|
||||
dir.write_tag(
|
||||
Tag::Unknown(tag::UNIQUE_CAMERA_MODEL),
|
||||
Ascii(&profile.unique_model),
|
||||
)?;
|
||||
dir.write_tag(
|
||||
Tag::Unknown(tag::WHITE_LEVEL),
|
||||
&[profile.white_level; 3][..],
|
||||
)?;
|
||||
dir.write_tag(Tag::Unknown(tag::BLACK_LEVEL), &[0u32; 3][..])?;
|
||||
|
||||
for (slot, (illuminant, matrix)) in profile.calibrations.iter().take(2).enumerate() {
|
||||
let (ill_tag, mat_tag) = if slot == 0 {
|
||||
(tag::CALIBRATION_ILLUMINANT_1, tag::COLOR_MATRIX_1)
|
||||
} else {
|
||||
(tag::CALIBRATION_ILLUMINANT_2, tag::COLOR_MATRIX_2)
|
||||
};
|
||||
dir.write_tag(Tag::Unknown(ill_tag), *illuminant)?;
|
||||
let flat: Vec<SRational> = matrix
|
||||
.iter()
|
||||
.flatten()
|
||||
.map(|&v| SRational {
|
||||
n: (v * 10_000.0).round() as i32,
|
||||
d: 10_000,
|
||||
})
|
||||
.collect();
|
||||
dir.write_tag(Tag::Unknown(mat_tag), SRationals(&flat))?;
|
||||
}
|
||||
|
||||
let neutral: Vec<(u32, u32)> = profile
|
||||
.as_shot_neutral
|
||||
.iter()
|
||||
.map(|&v| ((v.max(0.0) * 1_000_000.0).round() as u32, 1_000_000))
|
||||
.collect();
|
||||
dir.write_tag(Tag::Unknown(tag::AS_SHOT_NEUTRAL), Rationals(&neutral))?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// `PhotometricInterpretation` for demosaiced, un-rendered sensor data.
|
||||
const LINEAR_RAW: u16 = 34892;
|
||||
|
||||
/// DNG tag numbers the `tiff` crate has no names for.
|
||||
mod tag {
|
||||
pub const DNG_VERSION: u16 = 50706;
|
||||
pub const DNG_BACKWARD_VERSION: u16 = 50707;
|
||||
pub const UNIQUE_CAMERA_MODEL: u16 = 50708;
|
||||
pub const BLACK_LEVEL: u16 = 50714;
|
||||
pub const WHITE_LEVEL: u16 = 50717;
|
||||
pub const DEFAULT_CROP_ORIGIN: u16 = 50719;
|
||||
pub const DEFAULT_CROP_SIZE: u16 = 50720;
|
||||
pub const COLOR_MATRIX_1: u16 = 50721;
|
||||
pub const COLOR_MATRIX_2: u16 = 50722;
|
||||
pub const AS_SHOT_NEUTRAL: u16 = 50728;
|
||||
pub const CALIBRATION_ILLUMINANT_1: u16 = 50778;
|
||||
pub const CALIBRATION_ILLUMINANT_2: u16 = 50779;
|
||||
}
|
||||
|
||||
/// A run of `SRATIONAL`s, as `encode::Rationals` is for `RATIONAL`.
|
||||
struct SRationals<'a>(&'a [SRational]);
|
||||
|
||||
impl TiffValue for SRationals<'_> {
|
||||
const BYTE_LEN: u8 = 8;
|
||||
const FIELD_TYPE: tiff::tags::Type = tiff::tags::Type::SRATIONAL;
|
||||
|
||||
fn count(&self) -> usize {
|
||||
self.0.len()
|
||||
}
|
||||
|
||||
fn data(&self) -> std::borrow::Cow<'_, [u8]> {
|
||||
let mut out = Vec::with_capacity(self.0.len() * 8);
|
||||
for r in self.0 {
|
||||
out.extend_from_slice(&r.n.to_ne_bytes());
|
||||
out.extend_from_slice(&r.d.to_ne_bytes());
|
||||
}
|
||||
std::borrow::Cow::Owned(out)
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
fn profile() -> DngProfile {
|
||||
DngProfile {
|
||||
unique_model: "Canon EOS 6D".into(),
|
||||
calibrations: vec![
|
||||
(17, [[0.8, -0.2, 0.1], [-0.3, 1.1, 0.2], [0.0, -0.1, 0.9]]),
|
||||
(21, [[0.7, -0.1, 0.0], [-0.2, 1.0, 0.1], [0.0, -0.2, 0.8]]),
|
||||
],
|
||||
as_shot_neutral: [0.5, 1.0, 0.6],
|
||||
white_level: 13_023,
|
||||
}
|
||||
}
|
||||
|
||||
fn write(width: u32, height: u32, rows: u32) -> Vec<u8> {
|
||||
let mut bytes = std::io::Cursor::new(Vec::new());
|
||||
let source = SourceMetadata {
|
||||
make: Some("Canon".into()),
|
||||
model: Some("Canon EOS 6D".into()),
|
||||
..Default::default()
|
||||
};
|
||||
write_linear_dng(
|
||||
&mut bytes,
|
||||
width,
|
||||
height,
|
||||
rows,
|
||||
&profile(),
|
||||
Some(&source),
|
||||
|k, buf| {
|
||||
let first = k as u32 * rows;
|
||||
let n = rows.min(height - first);
|
||||
for y in first..first + n {
|
||||
for x in 0..width {
|
||||
buf.extend([(x + y * width) as u16, 1000, 2000]);
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
},
|
||||
|| {
|
||||
Some(crate::Rect {
|
||||
x: 2,
|
||||
y: 1,
|
||||
width: 15,
|
||||
height: 10,
|
||||
})
|
||||
},
|
||||
)
|
||||
.expect("written");
|
||||
bytes.into_inner()
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn rawler_reads_it_back_as_linear_raw() {
|
||||
let bytes = write(20, 13, 4);
|
||||
let source = rawler::rawsource::RawSource::new_from_slice(&bytes);
|
||||
let decoder = rawler::get_decoder(&source).expect("a DNG");
|
||||
let image = decoder
|
||||
.raw_image(&source, &Default::default(), false)
|
||||
.expect("decodes");
|
||||
assert_eq!((image.width, image.height, image.cpp), (20, 13, 3));
|
||||
assert_eq!(image.whitelevel.0[0], 13_023);
|
||||
// Pixel (3, 2) is (3 + 2·20, 1000, 2000) — samples in order, strips
|
||||
// joined without a seam.
|
||||
let rawler::RawImageData::Integer(data) = &image.data else {
|
||||
panic!("integer samples")
|
||||
};
|
||||
let i = (2 * 20 + 3) * 3;
|
||||
assert_eq!(&data[i..i + 3], &[43, 1000, 2000]);
|
||||
// Last row, from the short final strip.
|
||||
let i = (12 * 20 + 19) * 3;
|
||||
assert_eq!(data[i], (19 + 12 * 20) as u16);
|
||||
// The profile came through as the camera's.
|
||||
assert!(!image.camera.color_matrix.is_empty());
|
||||
assert_eq!(image.model, "Canon EOS 6D");
|
||||
// The default crop is what the decoder reports as the picture.
|
||||
let crop = image.crop_area.expect("a crop");
|
||||
assert_eq!((crop.p.x, crop.p.y, crop.d.w, crop.d.h), (2, 1, 15, 10));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_strip_of_the_wrong_length_is_refused() {
|
||||
let mut bytes = std::io::Cursor::new(Vec::new());
|
||||
let err = write_linear_dng(
|
||||
&mut bytes,
|
||||
8,
|
||||
8,
|
||||
8,
|
||||
&profile(),
|
||||
None,
|
||||
|_, buf| {
|
||||
buf.extend([0u16; 10]);
|
||||
Ok(())
|
||||
},
|
||||
|| None,
|
||||
)
|
||||
.unwrap_err();
|
||||
assert!(matches!(err, ExportError::Encode(_)));
|
||||
}
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
@@ -1,46 +0,0 @@
|
||||
//! TRACES: NFR-ARCH-4
|
||||
//! Typed export failures.
|
||||
//!
|
||||
//! Every variant is something a caller can act on or report. A batch export
|
||||
//! runs unattended over hundreds of frames (FR-EXP-7), so "what went wrong
|
||||
//! with which file" has to survive as data rather than as a log line.
|
||||
|
||||
use dr_types::{ColourSpace, ExportFormat};
|
||||
|
||||
#[derive(Debug, thiserror::Error)]
|
||||
pub enum ExportError {
|
||||
#[error("frame buffer is {got} bytes, expected {expected}")]
|
||||
FrameSize { expected: usize, got: usize },
|
||||
|
||||
#[error("frame has no pixels")]
|
||||
EmptyFrame,
|
||||
|
||||
/// Asked for a format with no encoder in this build.
|
||||
///
|
||||
/// Not a panic and not a silent substitution: the settings page offers
|
||||
/// AVIF and JPEG XL because FR-EXP-1 lists them, and a build without them
|
||||
/// should say so rather than quietly writing a JPEG under a `.avif` name.
|
||||
#[error("{} export is not supported yet", .0.label())]
|
||||
FormatUnsupported(ExportFormat),
|
||||
|
||||
/// TRACES: FR-EXP-2
|
||||
/// The frame was rendered into one colour space and asked to be labelled
|
||||
/// another.
|
||||
///
|
||||
/// Not a limitation of the encoders — all four spaces embed a correct
|
||||
/// profile. It is that the conversion happens in the shader, before the
|
||||
/// clip to 0..1, so a frame is in exactly one space by the time it gets
|
||||
/// here. The caller composes with `EditGraph::compose_for` to change which.
|
||||
#[error(
|
||||
"the frame was rendered in {} but a {} file was asked for",
|
||||
.rendered.label(),
|
||||
.requested.label()
|
||||
)]
|
||||
ColourSpaceMismatch {
|
||||
rendered: ColourSpace,
|
||||
requested: ColourSpace,
|
||||
},
|
||||
|
||||
#[error("encoding failed: {0}")]
|
||||
Encode(String),
|
||||
}
|
||||
@@ -1,518 +0,0 @@
|
||||
//! TRACES: FR-EXP-8
|
||||
//! Building an EXIF block, rather than copying one.
|
||||
//!
|
||||
//! # Why this is written by hand and not with a crate
|
||||
//!
|
||||
//! Two reasons, in order of importance.
|
||||
//!
|
||||
//! The first is the privacy behaviour. Every EXIF library worth using offers a
|
||||
//! "load the source block, delete these tags, write it back" shape, and that
|
||||
//! shape is the wrong one here: it makes the file that leaves the machine a
|
||||
//! copy of the source's metadata *minus what we thought to remove*, so every
|
||||
//! tag nobody has thought about — a vendor's proprietary sub-directory, a
|
||||
//! serial number under a tag id this build has never seen — travels by
|
||||
//! default. Constructing the block from a fixed list of parsed values inverts
|
||||
//! that. What is written is exactly what appears in [`crate::SourceMetadata`],
|
||||
//! and a tag that is not in this file cannot end up in the output no matter
|
||||
//! what the source contained. The allowlist *is* the implementation.
|
||||
//!
|
||||
//! The second is the dependency policy. The root `Cargo.toml` explains why
|
||||
//! nothing here may link C — this tree has to build under the Android NDK —
|
||||
//! and the mature EXIF writers are bindings. This is a couple of hundred
|
||||
//! lines of offset arithmetic against a specification that has not changed
|
||||
//! since 2010, and it is the same TIFF structure `dr-decode` already reads.
|
||||
//!
|
||||
//! # What the block is
|
||||
//!
|
||||
//! A complete little-endian TIFF: an 8-byte header, IFD0 with the identity
|
||||
//! and rights tags, an Exif sub-IFD with the capture tags, optionally a GPS
|
||||
//! sub-IFD, and a heap of values too long to sit inside an entry. JPEG carries
|
||||
//! it in an APP1 segment behind the marker `Exif\0\0`; PNG carries the same
|
||||
//! bytes in an `eXIf` chunk with no marker. TIFF does not use this at all —
|
||||
//! its own directory *is* the EXIF, so `encode.rs` writes the tags there
|
||||
//! directly.
|
||||
|
||||
use crate::metadata::SourceMetadata;
|
||||
|
||||
/// One entry's value, in the handful of TIFF types this writer emits.
|
||||
enum Value {
|
||||
/// NUL-terminated, as the specification requires; the terminator is
|
||||
/// counted, which is the detail readers trip over when it is missing.
|
||||
Ascii(String),
|
||||
Byte(Vec<u8>),
|
||||
Short(u16),
|
||||
Long(u32),
|
||||
/// Type 7. Used only for `ExifVersion`, which is four characters that are
|
||||
/// deliberately *not* a string.
|
||||
Undefined(&'static [u8]),
|
||||
/// Numerator and denominator pairs. A coordinate is three of them.
|
||||
Rational(Vec<(u32, u32)>),
|
||||
}
|
||||
|
||||
impl Value {
|
||||
fn field_type(&self) -> u16 {
|
||||
match self {
|
||||
Value::Byte(_) => 1,
|
||||
Value::Ascii(_) => 2,
|
||||
Value::Short(_) => 3,
|
||||
Value::Long(_) => 4,
|
||||
Value::Rational(_) => 5,
|
||||
Value::Undefined(_) => 7,
|
||||
}
|
||||
}
|
||||
|
||||
/// The element count, which is not the byte length: a rational counts as
|
||||
/// one element per eight bytes.
|
||||
fn count(&self) -> u32 {
|
||||
match self {
|
||||
Value::Ascii(s) => s.len() as u32 + 1,
|
||||
Value::Byte(b) => b.len() as u32,
|
||||
Value::Undefined(b) => b.len() as u32,
|
||||
Value::Short(_) | Value::Long(_) => 1,
|
||||
Value::Rational(r) => r.len() as u32,
|
||||
}
|
||||
}
|
||||
|
||||
/// The payload, in file order.
|
||||
fn payload(&self) -> Vec<u8> {
|
||||
match self {
|
||||
Value::Ascii(s) => {
|
||||
let mut out = s.as_bytes().to_vec();
|
||||
out.push(0);
|
||||
out
|
||||
}
|
||||
Value::Byte(b) => b.clone(),
|
||||
Value::Undefined(b) => b.to_vec(),
|
||||
Value::Short(v) => v.to_le_bytes().to_vec(),
|
||||
Value::Long(v) => v.to_le_bytes().to_vec(),
|
||||
Value::Rational(r) => r
|
||||
.iter()
|
||||
.flat_map(|(n, d)| {
|
||||
let mut b = n.to_le_bytes().to_vec();
|
||||
b.extend_from_slice(&d.to_le_bytes());
|
||||
b
|
||||
})
|
||||
.collect(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// An IFD under construction.
|
||||
type Entries = Vec<(u16, Value)>;
|
||||
|
||||
/// Tag numbers. Named rather than inlined because a mistyped one produces a
|
||||
/// file that still parses and says something else entirely.
|
||||
pub(crate) mod tag {
|
||||
pub(crate) const MAKE: u16 = 0x010F;
|
||||
pub(crate) const MODEL: u16 = 0x0110;
|
||||
pub(crate) const SOFTWARE: u16 = 0x0131;
|
||||
pub(crate) const DATE_TIME: u16 = 0x0132;
|
||||
pub(crate) const ARTIST: u16 = 0x013B;
|
||||
pub(crate) const COPYRIGHT: u16 = 0x8298;
|
||||
pub(crate) const EXIF_IFD: u16 = 0x8769;
|
||||
pub(crate) const GPS_IFD: u16 = 0x8825;
|
||||
|
||||
pub(crate) const EXPOSURE_TIME: u16 = 0x829A;
|
||||
pub(crate) const FNUMBER: u16 = 0x829D;
|
||||
pub(crate) const ISO: u16 = 0x8827;
|
||||
pub(crate) const EXIF_VERSION: u16 = 0x9000;
|
||||
pub(crate) const DATE_TIME_ORIGINAL: u16 = 0x9003;
|
||||
pub(crate) const OFFSET_TIME_ORIGINAL: u16 = 0x9011;
|
||||
pub(crate) const FOCAL_LENGTH: u16 = 0x920A;
|
||||
pub(crate) const PIXEL_X: u16 = 0xA002;
|
||||
pub(crate) const PIXEL_Y: u16 = 0xA003;
|
||||
pub(crate) const LENS_MODEL: u16 = 0xA434;
|
||||
|
||||
pub(crate) const GPS_VERSION_ID: u16 = 0x0000;
|
||||
pub(crate) const GPS_LATITUDE_REF: u16 = 0x0001;
|
||||
pub(crate) const GPS_LATITUDE: u16 = 0x0002;
|
||||
pub(crate) const GPS_LONGITUDE_REF: u16 = 0x0003;
|
||||
pub(crate) const GPS_LONGITUDE: u16 = 0x0004;
|
||||
pub(crate) const GPS_ALTITUDE_REF: u16 = 0x0005;
|
||||
pub(crate) const GPS_ALTITUDE: u16 = 0x0006;
|
||||
}
|
||||
|
||||
/// What DarkRoom calls itself in a file it wrote.
|
||||
///
|
||||
/// Not vanity: an export is a derived file, and a reader that knows which
|
||||
/// program produced it can tell a camera original from a rendition without
|
||||
/// guessing from the absence of a maker note.
|
||||
pub(crate) const SOFTWARE: &str = "DarkRoom";
|
||||
|
||||
/// The complete EXIF block for JPEG's APP1 and PNG's `eXIf`.
|
||||
///
|
||||
/// `width`/`height` are the *exported* dimensions, not the source's: the
|
||||
/// pixel-dimension tags describe the file they are in, and a reader that
|
||||
/// trusts them after a resize would report the wrong size for the image it is
|
||||
/// holding.
|
||||
///
|
||||
/// `None` where there is nothing to say. An empty EXIF block is not the same
|
||||
/// as no EXIF block — it is a structure a reader must parse to discover it
|
||||
/// learned nothing — and the second is the better file.
|
||||
pub(crate) fn block(md: &SourceMetadata, width: u32, height: u32) -> Option<Vec<u8>> {
|
||||
let ifd0 = main_entries(md);
|
||||
let exif = exif_entries(md, width, height);
|
||||
let gps = gps_entries(md);
|
||||
if ifd0.is_empty() && exif.is_empty() && gps.is_empty() {
|
||||
return None;
|
||||
}
|
||||
Some(assemble(ifd0, exif, gps))
|
||||
}
|
||||
|
||||
/// Lay the three directories and their heap out in the block.
|
||||
///
|
||||
/// The order is fixed — IFD0, Exif, GPS, heap — because the pointers have to
|
||||
/// be known before IFD0 is serialised, and an IFD's size is decided by its
|
||||
/// entry count alone: two bytes of count, twelve per entry, four for the link
|
||||
/// to the next directory.
|
||||
fn assemble(mut ifd0: Entries, exif: Entries, gps: Entries) -> Vec<u8> {
|
||||
const HEADER: u32 = 8;
|
||||
let size = |n: usize| 2 + 12 * n as u32 + 4;
|
||||
|
||||
// The pointer entries are part of IFD0's count, so they have to be added
|
||||
// before its size is taken — a chicken-and-egg the specification resolves
|
||||
// by making entry size fixed.
|
||||
let pointers = usize::from(!exif.is_empty()) + usize::from(!gps.is_empty());
|
||||
let ifd0_size = size(ifd0.len() + pointers);
|
||||
|
||||
let exif_offset = HEADER + ifd0_size;
|
||||
let gps_offset = exif_offset + if exif.is_empty() { 0 } else { size(exif.len()) };
|
||||
let heap_base = gps_offset + if gps.is_empty() { 0 } else { size(gps.len()) };
|
||||
|
||||
if !exif.is_empty() {
|
||||
ifd0.push((tag::EXIF_IFD, Value::Long(exif_offset)));
|
||||
}
|
||||
if !gps.is_empty() {
|
||||
ifd0.push((tag::GPS_IFD, Value::Long(gps_offset)));
|
||||
}
|
||||
|
||||
let mut heap = Vec::new();
|
||||
let ifd0_bytes = directory(ifd0, heap_base, &mut heap);
|
||||
let exif_bytes = directory(exif, heap_base, &mut heap);
|
||||
let gps_bytes = directory(gps, heap_base, &mut heap);
|
||||
|
||||
let mut out = Vec::with_capacity(HEADER as usize + heap.len() + 128);
|
||||
// Little-endian, magic 42, first directory at byte 8. Little-endian
|
||||
// because every value written below is, and a header that disagreed with
|
||||
// its own body is the one corruption a reader cannot recover from.
|
||||
out.extend_from_slice(b"II");
|
||||
out.extend_from_slice(&42u16.to_le_bytes());
|
||||
out.extend_from_slice(&HEADER.to_le_bytes());
|
||||
out.extend_from_slice(&ifd0_bytes);
|
||||
out.extend_from_slice(&exif_bytes);
|
||||
out.extend_from_slice(&gps_bytes);
|
||||
out.extend_from_slice(&heap);
|
||||
out
|
||||
}
|
||||
|
||||
/// Serialise one directory, spilling long values onto the shared heap.
|
||||
///
|
||||
/// Entries are sorted by tag: TIFF requires ascending order within a
|
||||
/// directory, and while most readers cope with any order, the ones that
|
||||
/// binary-search stop at the first tag they cannot place.
|
||||
fn directory(mut entries: Entries, heap_base: u32, heap: &mut Vec<u8>) -> Vec<u8> {
|
||||
if entries.is_empty() {
|
||||
return Vec::new();
|
||||
}
|
||||
entries.sort_by_key(|(tag, _)| *tag);
|
||||
|
||||
let mut out = Vec::with_capacity(2 + entries.len() * 12 + 4);
|
||||
out.extend_from_slice(&(entries.len() as u16).to_le_bytes());
|
||||
for (tag, value) in &entries {
|
||||
out.extend_from_slice(&tag.to_le_bytes());
|
||||
out.extend_from_slice(&value.field_type().to_le_bytes());
|
||||
out.extend_from_slice(&value.count().to_le_bytes());
|
||||
|
||||
let payload = value.payload();
|
||||
if payload.len() <= 4 {
|
||||
// Four bytes or fewer live in the entry itself, left-justified and
|
||||
// zero-padded.
|
||||
let mut inline = payload.clone();
|
||||
inline.resize(4, 0);
|
||||
out.extend_from_slice(&inline);
|
||||
} else {
|
||||
out.extend_from_slice(&(heap_base + heap.len() as u32).to_le_bytes());
|
||||
heap.extend_from_slice(&payload);
|
||||
// Values start on even offsets. Not every reader cares; the ones
|
||||
// that do read a short from an odd address and get nonsense.
|
||||
if heap.len() % 2 == 1 {
|
||||
heap.push(0);
|
||||
}
|
||||
}
|
||||
}
|
||||
// No directory follows this one. The Exif and GPS sub-directories are
|
||||
// pointed at, not chained, so this is zero in all three.
|
||||
out.extend_from_slice(&0u32.to_le_bytes());
|
||||
out
|
||||
}
|
||||
|
||||
/// IFD0: who took it, with what, and who owns it.
|
||||
///
|
||||
/// **No orientation tag, deliberately.** The frame reaching the encoder has
|
||||
/// already had the source's orientation applied by the pipeline — it is
|
||||
/// upright pixels — so copying the source's tag across would tell every
|
||||
/// reader to rotate an image that is already the right way up. A portrait
|
||||
/// frame would come out on its side in exactly the viewers that honour the
|
||||
/// tag, which is most of them.
|
||||
fn main_entries(md: &SourceMetadata) -> Entries {
|
||||
let mut e = Entries::new();
|
||||
push_ascii(&mut e, tag::MAKE, md.make.as_deref());
|
||||
push_ascii(&mut e, tag::MODEL, md.model.as_deref());
|
||||
push_ascii(&mut e, tag::ARTIST, md.artist.as_deref());
|
||||
push_ascii(&mut e, tag::COPYRIGHT, md.copyright.as_deref());
|
||||
e.push((tag::SOFTWARE, Value::Ascii(SOFTWARE.to_string())));
|
||||
// IFD0's `DateTime` is nominally when the file was written, and this is
|
||||
// the capture time instead. That is what the rest of the world does —
|
||||
// and it is what `dr-decode` falls back to for scanner output that has no
|
||||
// `DateTimeOriginal` — so a re-import of an export lands on the timeline
|
||||
// where the original did rather than on the day it was exported.
|
||||
if let Some(t) = md.captured_at.map(datetime) {
|
||||
e.push((tag::DATE_TIME, Value::Ascii(t)));
|
||||
}
|
||||
e
|
||||
}
|
||||
|
||||
/// The Exif sub-IFD: the exposure, and what made it.
|
||||
fn exif_entries(md: &SourceMetadata, width: u32, height: u32) -> Entries {
|
||||
let mut e = Entries::new();
|
||||
// "0232" is Exif 2.32. A sub-directory without a version is technically
|
||||
// malformed, and some readers refuse the whole block over it.
|
||||
e.push((tag::EXIF_VERSION, Value::Undefined(b"0232")));
|
||||
e.push((tag::PIXEL_X, Value::Long(width)));
|
||||
e.push((tag::PIXEL_Y, Value::Long(height)));
|
||||
push_ascii(&mut e, tag::LENS_MODEL, md.lens.as_deref());
|
||||
if let Some(t) = md.captured_at.map(datetime) {
|
||||
e.push((tag::DATE_TIME_ORIGINAL, Value::Ascii(t)));
|
||||
}
|
||||
if let Some(o) = md.captured_offset.map(offset) {
|
||||
e.push((tag::OFFSET_TIME_ORIGINAL, Value::Ascii(o)));
|
||||
}
|
||||
if let Some(s) = md.shutter.filter(|s| *s > 0.0) {
|
||||
e.push((tag::EXPOSURE_TIME, Value::Rational(vec![shutter(s)])));
|
||||
}
|
||||
if let Some(f) = md.aperture.filter(|f| *f > 0.0) {
|
||||
e.push((tag::FNUMBER, Value::Rational(vec![tenths(f)])));
|
||||
}
|
||||
if let Some(f) = md.focal_length.filter(|f| *f > 0.0) {
|
||||
e.push((tag::FOCAL_LENGTH, Value::Rational(vec![tenths(f)])));
|
||||
}
|
||||
// The tag is a SHORT, so a sensitivity above 65535 has no representation
|
||||
// in it. Dropped rather than truncated: ISO 102400 written as 36864 is a
|
||||
// lie, and an absent tag is not.
|
||||
if let Some(iso) = md.iso.filter(|v| *v <= u32::from(u16::MAX)) {
|
||||
e.push((tag::ISO, Value::Short(iso as u16)));
|
||||
}
|
||||
e
|
||||
}
|
||||
|
||||
/// The GPS sub-IFD.
|
||||
///
|
||||
/// Empty unless the caller has already decided that coordinates may be
|
||||
/// written — see [`SourceMetadata::sanitised`], which is where the stripping
|
||||
/// happens. Nothing in this file consults the settings, so there is exactly
|
||||
/// one place to look to answer "can this export carry a location".
|
||||
fn gps_entries(md: &SourceMetadata) -> Entries {
|
||||
let Some(loc) = md.location else {
|
||||
return Entries::new();
|
||||
};
|
||||
let mut e = Entries::new();
|
||||
// 2.3.0.0, the current GPS tag version.
|
||||
e.push((tag::GPS_VERSION_ID, Value::Byte(vec![2, 3, 0, 0])));
|
||||
e.push((
|
||||
tag::GPS_LATITUDE_REF,
|
||||
Value::Ascii(if loc.latitude < 0.0 { "S" } else { "N" }.into()),
|
||||
));
|
||||
e.push((tag::GPS_LATITUDE, Value::Rational(dms(loc.latitude))));
|
||||
e.push((
|
||||
tag::GPS_LONGITUDE_REF,
|
||||
Value::Ascii(if loc.longitude < 0.0 { "W" } else { "E" }.into()),
|
||||
));
|
||||
e.push((tag::GPS_LONGITUDE, Value::Rational(dms(loc.longitude))));
|
||||
if let Some(alt) = loc.altitude {
|
||||
// The altitude itself is unsigned; below sea level is a separate byte.
|
||||
e.push((
|
||||
tag::GPS_ALTITUDE_REF,
|
||||
Value::Byte(vec![u8::from(alt < 0.0)]),
|
||||
));
|
||||
e.push((
|
||||
tag::GPS_ALTITUDE,
|
||||
Value::Rational(vec![((alt.abs() * 100.0).round() as u32, 100)]),
|
||||
));
|
||||
}
|
||||
e
|
||||
}
|
||||
|
||||
fn push_ascii(entries: &mut Entries, tag: u16, value: Option<&str>) {
|
||||
// An empty string is a tag saying nothing, which is worse than no tag: it
|
||||
// overwrites whatever a reader would otherwise have inferred.
|
||||
if let Some(v) = value.map(str::trim).filter(|v| !v.is_empty()) {
|
||||
entries.push((tag, Value::Ascii(v.to_string())));
|
||||
}
|
||||
}
|
||||
|
||||
/// Signed degrees back into the tag's degrees/minutes/seconds.
|
||||
///
|
||||
/// The sign is carried by the hemisphere letter, so this takes the magnitude.
|
||||
/// Seconds keep four decimal places, which is about 3 mm — far finer than any
|
||||
/// consumer fix, and enough that a round trip through the tag does not move
|
||||
/// the pin.
|
||||
pub(crate) fn dms(degrees: f64) -> Vec<(u32, u32)> {
|
||||
let d = degrees.abs();
|
||||
let whole = d.trunc();
|
||||
let minutes = (d - whole) * 60.0;
|
||||
let seconds = (minutes - minutes.trunc()) * 60.0;
|
||||
vec![
|
||||
(whole as u32, 1),
|
||||
(minutes.trunc() as u32, 1),
|
||||
((seconds * 10_000.0).round() as u32, 10_000),
|
||||
]
|
||||
}
|
||||
|
||||
/// A shutter speed as the fraction a photographer would recognise.
|
||||
///
|
||||
/// `1/250`, not `4/1000`. Both are the same number and every reader computes
|
||||
/// the same exposure from either, but the first is what the camera wrote and
|
||||
/// what a properties panel displays verbatim.
|
||||
pub(crate) fn shutter(seconds: f32) -> (u32, u32) {
|
||||
if seconds < 1.0 {
|
||||
(1, (1.0 / seconds).round().max(1.0) as u32)
|
||||
} else {
|
||||
((seconds * 10.0).round() as u32, 10)
|
||||
}
|
||||
}
|
||||
|
||||
/// f/2.8 and 85 mm as tenths, which is how cameras write both.
|
||||
pub(crate) fn tenths(value: f32) -> (u32, u32) {
|
||||
((value * 10.0).round().max(0.0) as u32, 10)
|
||||
}
|
||||
|
||||
/// Unix seconds as EXIF's `"YYYY:MM:DD HH:MM:SS"`.
|
||||
///
|
||||
/// The reading is a wall clock with no zone — that is what the tag means, and
|
||||
/// what `dr-decode` parsed it as — so this is the exact inverse of that parse
|
||||
/// and involves no timezone conversion. The zone, where the source recorded
|
||||
/// one, travels separately in `OffsetTimeOriginal`.
|
||||
pub(crate) fn datetime(unix: i64) -> String {
|
||||
let days = unix.div_euclid(86_400);
|
||||
let secs = unix.rem_euclid(86_400);
|
||||
|
||||
// Howard Hinnant's civil-from-days, the inverse of the days-from-civil
|
||||
// that `dr-decode` uses to parse. Eras of 400 years, shifted so that the
|
||||
// arithmetic never sees a negative.
|
||||
let z = days + 719_468;
|
||||
let era = z.div_euclid(146_097);
|
||||
let doe = z.rem_euclid(146_097);
|
||||
let yoe = (doe - doe / 1460 + doe / 36_524 - doe / 146_096) / 365;
|
||||
let y = yoe + era * 400;
|
||||
let doy = doe - (365 * yoe + yoe / 4 - yoe / 100);
|
||||
let mp = (5 * doy + 2) / 153;
|
||||
let d = doy - (153 * mp + 2) / 5 + 1;
|
||||
let m = if mp < 10 { mp + 3 } else { mp - 9 };
|
||||
let y = if m <= 2 { y + 1 } else { y };
|
||||
|
||||
format!(
|
||||
"{y:04}:{m:02}:{d:02} {:02}:{:02}:{:02}",
|
||||
secs / 3600,
|
||||
(secs / 60) % 60,
|
||||
secs % 60
|
||||
)
|
||||
}
|
||||
|
||||
/// Minutes east of UTC as EXIF's `"+HH:MM"`.
|
||||
pub(crate) fn offset(minutes: i32) -> String {
|
||||
let sign = if minutes < 0 { '-' } else { '+' };
|
||||
let m = minutes.unsigned_abs();
|
||||
format!("{sign}{:02}:{:02}", m / 60, m % 60)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use dr_types::Location;
|
||||
|
||||
#[test]
|
||||
fn a_capture_time_survives_the_round_trip_through_the_tag() {
|
||||
// The parse side lives in `dr-decode` and is exercised against real
|
||||
// files; this is the inverse, and the two meeting in the middle is
|
||||
// what keeps an exported frame on the same point of the timeline as
|
||||
// the original.
|
||||
assert_eq!(datetime(1_372_462_374), "2013:06:28 23:32:54");
|
||||
assert_eq!(datetime(0), "1970:01:01 00:00:00");
|
||||
// A leap day, which is where a hand-rolled calendar goes wrong.
|
||||
assert_eq!(datetime(1_709_164_800), "2024:02:29 00:00:00");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_zone_is_written_the_way_the_tag_spells_it() {
|
||||
assert_eq!(offset(120), "+02:00");
|
||||
assert_eq!(offset(-330), "-05:30");
|
||||
assert_eq!(offset(0), "+00:00");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_shutter_speed_keeps_the_photographers_fraction() {
|
||||
assert_eq!(shutter(1.0 / 250.0), (1, 250));
|
||||
assert_eq!(shutter(2.5), (25, 10));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn degrees_round_trip_through_the_tags_triple() {
|
||||
// 48.8582 N is the Eiffel Tower; the check is that the three-part
|
||||
// form comes back to the same place, to well under a metre.
|
||||
for degrees in [48.8582_f64, -33.8568, 0.0, 179.999] {
|
||||
let parts = dms(degrees);
|
||||
let back = parts[0].0 as f64
|
||||
+ parts[1].0 as f64 / 60.0
|
||||
+ (parts[2].0 as f64 / parts[2].1 as f64) / 3600.0;
|
||||
assert!(
|
||||
(back - degrees.abs()).abs() < 1e-6,
|
||||
"{degrees} came back as {back}"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn an_empty_source_produces_no_block_at_all() {
|
||||
// Every field absent means the only entries would be the ones this
|
||||
// writer adds itself. That is still worth writing — `Software` and
|
||||
// the pixel dimensions are true statements — so the block exists; what
|
||||
// must not happen is a *malformed* one.
|
||||
let md = SourceMetadata::default();
|
||||
let bytes = block(&md, 100, 50).expect("the writer's own tags");
|
||||
assert!(bytes.starts_with(b"II*\0"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn the_gps_directory_is_absent_when_there_is_no_position() {
|
||||
let md = SourceMetadata {
|
||||
make: Some("Canon".into()),
|
||||
..Default::default()
|
||||
};
|
||||
let bytes = block(&md, 10, 10).unwrap();
|
||||
assert!(!contains_entry(&bytes, tag::GPS_IFD));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn the_gps_directory_is_present_when_there_is_one() {
|
||||
// The counterpart of the test above: a strip test that passed because
|
||||
// the writer could never emit GPS at all would prove nothing.
|
||||
let md = SourceMetadata {
|
||||
location: Location::new(48.8582, 2.2945, Some(35.0)),
|
||||
..Default::default()
|
||||
};
|
||||
let bytes = block(&md, 10, 10).unwrap();
|
||||
assert!(contains_entry(&bytes, tag::GPS_IFD));
|
||||
}
|
||||
|
||||
/// Whether a directory entry for `tag` appears anywhere in the block.
|
||||
///
|
||||
/// Byte-level on purpose: an entry is a tag, a type and a count, and
|
||||
/// searching for that twelve-byte shape's first eight bytes is a far
|
||||
/// stronger statement than asking a parser that might have skipped the
|
||||
/// directory the tag was in.
|
||||
fn contains_entry(bytes: &[u8], tag: u16) -> bool {
|
||||
bytes
|
||||
.windows(4)
|
||||
.any(|w| w[..2] == tag.to_le_bytes() && (w[2] == 4 || w[2] == 13) && w[3] == 0)
|
||||
}
|
||||
}
|
||||
@@ -1,483 +0,0 @@
|
||||
//! TRACES: FR-EXP-2
|
||||
//! Minimal ICC v2 matrix/TRC profiles, generated.
|
||||
//!
|
||||
//! # Why generated rather than shipped
|
||||
//!
|
||||
//! A profile is a description of what the pixels in a file mean, and the
|
||||
//! pixels here were produced by [`dr_types::colour`]'s matrices. Embedding a
|
||||
//! profile downloaded from elsewhere would mean two independent statements
|
||||
//! about the same space, agreeing until one of them was revised. Deriving both
|
||||
//! from the same primaries makes agreement structural.
|
||||
//!
|
||||
//! It is also the only pure-Rust route. Little-CMS is the obvious library and
|
||||
//! it is C, which the whole workspace avoids so it cross-compiles under the
|
||||
//! Android NDK — the same reasoning behind rustls, bundled SQLite and the
|
||||
//! Lensfun port.
|
||||
//!
|
||||
//! # What "minimal" leaves out
|
||||
//!
|
||||
//! A matrix/TRC display profile and nothing else: three colorants, three tone
|
||||
//! curves, a white point and the chromatic adaptation that got it there. No
|
||||
//! A2B/B2A lookup tables, no gamut tag, no named colours. That is the whole of
|
||||
//! what an RGB working space *is*, and it is what every reader — a browser, an
|
||||
//! operating system compositor, Photoshop — takes from a profile like this
|
||||
//! one. The tags omitted describe device behaviour these spaces do not have.
|
||||
//!
|
||||
//! Profiles come out around 2 KB, which matters more than it sounds: a JPEG
|
||||
//! carries the profile in APP2 segments capped at 64 KB each, and one that fits
|
||||
//! in a single segment avoids the chunked form that some older readers
|
||||
//! mishandle.
|
||||
|
||||
use dr_types::{ColourSpace, Transfer};
|
||||
|
||||
/// The ICC profile describing `space`, ready to embed.
|
||||
///
|
||||
/// Deterministic — the same space always produces the same bytes. Two exports
|
||||
/// of the same frame must be byte-identical files, which a creation timestamp
|
||||
/// read from the clock would quietly break, along with any deduplication
|
||||
/// downstream of it.
|
||||
pub fn profile(space: ColourSpace) -> Vec<u8> {
|
||||
let colorants = space.to_pcs_xyz();
|
||||
let trc = trc_curve(space.transfer());
|
||||
|
||||
// Sorted by signature, as the specification asks a tag table to be. Some
|
||||
// readers binary-search it.
|
||||
let mut tags: Vec<(&[u8; 4], Vec<u8>)> = vec![
|
||||
(b"bTRC", trc.clone()),
|
||||
// Columns, not rows: a colorant tag is where one primary lands in XYZ.
|
||||
(b"bXYZ", xyz_type(colorants[2], colorants[5], colorants[8])),
|
||||
(b"cprt", text_type(COPYRIGHT)),
|
||||
(b"desc", description_type(&description(space))),
|
||||
(b"gTRC", trc.clone()),
|
||||
(b"gXYZ", xyz_type(colorants[1], colorants[4], colorants[7])),
|
||||
(b"rTRC", trc),
|
||||
(b"rXYZ", xyz_type(colorants[0], colorants[3], colorants[6])),
|
||||
// The PCS illuminant itself, not the space's own white. The space's
|
||||
// white is recoverable from this and `chad`, and a profile that put
|
||||
// its native white here would have every reader adapt it twice.
|
||||
(b"wtpt", xyz_type(PCS_D50[0], PCS_D50[1], PCS_D50[2])),
|
||||
];
|
||||
|
||||
// Only where there is an adaptation to declare. ProPhoto is a D50 space
|
||||
// already, and an identity `chad` is a tag saying nothing.
|
||||
let adaptation = space.adaptation_to_pcs();
|
||||
if !is_identity(&adaptation) {
|
||||
tags.push((b"chad", sf32_type(&adaptation)));
|
||||
}
|
||||
tags.sort_by_key(|(sig, _)| **sig);
|
||||
|
||||
assemble(&tags)
|
||||
}
|
||||
|
||||
/// What a colour-management dialogue will show this profile as.
|
||||
///
|
||||
/// Deliberately not the canonical names. "sRGB IEC61966-2.1" is the reference
|
||||
/// profile, and this is not it — it is a profile derived from the same
|
||||
/// primaries, which is a different and weaker claim. "Adobe RGB (1998)" is
|
||||
/// additionally a name belonging to someone else. A distinct name also tells a
|
||||
/// user opening the file where the profile came from, which is the question
|
||||
/// they are asking when they look.
|
||||
fn description(space: ColourSpace) -> String {
|
||||
format!("DarkRoom {}", space.label())
|
||||
}
|
||||
|
||||
/// The copyright tag, which ICC requires a profile to carry.
|
||||
///
|
||||
/// A set of chromaticity coordinates from a published specification is not
|
||||
/// something to claim rights over, and a profile nobody may redistribute would
|
||||
/// make the files carrying it awkward to share — which is the whole purpose of
|
||||
/// an export.
|
||||
const COPYRIGHT: &str = "Generated by DarkRoom. No rights reserved.";
|
||||
|
||||
/// The profile connection space illuminant, as s15Fixed16 exactly.
|
||||
const PCS_D50: [f32; 3] = [0.9642, 1.0, 0.8249];
|
||||
|
||||
/// Samples in a tabulated tone curve.
|
||||
///
|
||||
/// 1024 is what the reference sRGB profiles use. The curve is interpolated
|
||||
/// linearly between samples, so this is far finer than the 8-bit values it
|
||||
/// describes; halving it would still be adequate and would save a kilobyte
|
||||
/// nobody is counting.
|
||||
const TRC_SAMPLES: usize = 1024;
|
||||
|
||||
/// A tone reproduction curve for the space's transfer function.
|
||||
///
|
||||
/// ICC curves run *towards* the connection space — device value to linear —
|
||||
/// which is the opposite direction from the shader's final encode. Getting it
|
||||
/// backwards produces a file that looks washed out or crushed by exactly the
|
||||
/// amount the curve bends.
|
||||
fn trc_curve(transfer: Transfer) -> Vec<u8> {
|
||||
// A pure power curve has an exact representation: a single u8Fixed8
|
||||
// gamma. Adobe RGB's 563/256 lands on it precisely, where a 1024-entry
|
||||
// table would be an approximation of a number the format can hold.
|
||||
if let Transfer::Gamma(g) = transfer {
|
||||
let mut out = tag_header(b"curv");
|
||||
out.extend_from_slice(&1u32.to_be_bytes());
|
||||
out.extend_from_slice(&((g * 256.0).round() as u16).to_be_bytes());
|
||||
return out;
|
||||
}
|
||||
|
||||
let mut out = tag_header(b"curv");
|
||||
out.extend_from_slice(&(TRC_SAMPLES as u32).to_be_bytes());
|
||||
for i in 0..TRC_SAMPLES {
|
||||
let device = i as f32 / (TRC_SAMPLES - 1) as f32;
|
||||
let linear = transfer.decode(device);
|
||||
out.extend_from_slice(&((linear * 65535.0).round() as u16).to_be_bytes());
|
||||
}
|
||||
out
|
||||
}
|
||||
|
||||
/// An `XYZType` tag: one colour in the connection space.
|
||||
fn xyz_type(x: f32, y: f32, z: f32) -> Vec<u8> {
|
||||
let mut out = tag_header(b"XYZ ");
|
||||
for v in [x, y, z] {
|
||||
out.extend_from_slice(&s15_fixed16(v).to_be_bytes());
|
||||
}
|
||||
out
|
||||
}
|
||||
|
||||
/// An `s15Fixed16ArrayType` tag, which is how `chad` is stored.
|
||||
fn sf32_type(m: &[f32; 9]) -> Vec<u8> {
|
||||
let mut out = tag_header(b"sf32");
|
||||
for v in m {
|
||||
out.extend_from_slice(&s15_fixed16(*v).to_be_bytes());
|
||||
}
|
||||
out
|
||||
}
|
||||
|
||||
/// A `textType` tag: ASCII with a terminating NUL.
|
||||
fn text_type(s: &str) -> Vec<u8> {
|
||||
let mut out = tag_header(b"text");
|
||||
out.extend_from_slice(s.as_bytes());
|
||||
out.push(0);
|
||||
out
|
||||
}
|
||||
|
||||
/// A `textDescriptionType` tag — the v2 profile's name field.
|
||||
///
|
||||
/// Baroque, and not optional: v2 has no plain `mluc`, and the ASCII string is
|
||||
/// followed by empty Unicode and ScriptCode blocks that a reader will walk
|
||||
/// whether or not they hold anything. The 67-byte Macintosh field is fixed
|
||||
/// width by specification, so it is written out zeroed rather than omitted.
|
||||
fn description_type(s: &str) -> Vec<u8> {
|
||||
let ascii = s.as_bytes();
|
||||
let mut out = tag_header(b"desc");
|
||||
out.extend_from_slice(&(ascii.len() as u32 + 1).to_be_bytes());
|
||||
out.extend_from_slice(ascii);
|
||||
out.push(0);
|
||||
// Unicode language code, then Unicode character count: none of either.
|
||||
out.extend_from_slice(&[0; 8]);
|
||||
// ScriptCode code (u16), length (u8), and the fixed 67-byte field.
|
||||
out.extend_from_slice(&[0; 3]);
|
||||
out.extend_from_slice(&[0; 67]);
|
||||
out
|
||||
}
|
||||
|
||||
/// Every tag element opens with its type signature and four reserved bytes.
|
||||
fn tag_header(sig: &[u8; 4]) -> Vec<u8> {
|
||||
let mut out = Vec::from(*sig);
|
||||
out.extend_from_slice(&[0; 4]);
|
||||
out
|
||||
}
|
||||
|
||||
/// ICC's fixed-point number: 16 integer bits, 16 fractional.
|
||||
fn s15_fixed16(v: f32) -> i32 {
|
||||
(f64::from(v) * 65536.0).round() as i32
|
||||
}
|
||||
|
||||
fn is_identity(m: &[f32; 9]) -> bool {
|
||||
const IDENTITY: [f32; 9] = [1.0, 0.0, 0.0, 0.0, 1.0, 0.0, 0.0, 0.0, 1.0];
|
||||
// One step of s15Fixed16, the format the matrix would be stored in. Below
|
||||
// that it *is* the identity — ProPhoto's own white and the PCS illuminant
|
||||
// differ in the sixth decimal place, and a `chad` recording that would be
|
||||
// nine copies of 1.0000 and 0.0000 dressed up as information.
|
||||
const STEP: f32 = 1.0 / 65536.0;
|
||||
m.iter().zip(IDENTITY).all(|(a, b)| (a - b).abs() < STEP)
|
||||
}
|
||||
|
||||
/// Header, tag table, and the tag data, with the size written back in.
|
||||
fn assemble(tags: &[(&[u8; 4], Vec<u8>)]) -> Vec<u8> {
|
||||
let mut out = header();
|
||||
|
||||
out.extend_from_slice(&(tags.len() as u32).to_be_bytes());
|
||||
let table_at = out.len();
|
||||
out.resize(table_at + tags.len() * 12, 0);
|
||||
|
||||
for (i, (sig, data)) in tags.iter().enumerate() {
|
||||
// Identical elements share one copy, which the specification allows
|
||||
// explicitly. The three tone curves of a grey-balanced space are the
|
||||
// same 2 KB table, so this is two thirds of the profile.
|
||||
let offset = find(&out, data).unwrap_or_else(|| {
|
||||
let at = out.len();
|
||||
out.extend_from_slice(data);
|
||||
// Every element starts on a four-byte boundary.
|
||||
while !out.len().is_multiple_of(4) {
|
||||
out.push(0);
|
||||
}
|
||||
at
|
||||
});
|
||||
|
||||
let entry = table_at + i * 12;
|
||||
out[entry..entry + 4].copy_from_slice(*sig);
|
||||
out[entry + 4..entry + 8].copy_from_slice(&(offset as u32).to_be_bytes());
|
||||
out[entry + 8..entry + 12].copy_from_slice(&(data.len() as u32).to_be_bytes());
|
||||
}
|
||||
|
||||
let size = out.len() as u32;
|
||||
out[0..4].copy_from_slice(&size.to_be_bytes());
|
||||
out
|
||||
}
|
||||
|
||||
/// Where `needle` already sits in `haystack`, if it does.
|
||||
///
|
||||
/// Only ever called with tag elements, which begin on four-byte boundaries and
|
||||
/// start with a type signature — so a match cannot be a coincidental overlap
|
||||
/// of two other tags' bytes.
|
||||
fn find(haystack: &[u8], needle: &[u8]) -> Option<usize> {
|
||||
haystack
|
||||
.windows(needle.len())
|
||||
.position(|w| w == needle)
|
||||
.filter(|at| at.is_multiple_of(4))
|
||||
}
|
||||
|
||||
/// The fixed 128-byte profile header.
|
||||
fn header() -> Vec<u8> {
|
||||
let mut h = Vec::with_capacity(128);
|
||||
// Size, filled in once the profile is complete.
|
||||
h.extend_from_slice(&[0; 4]);
|
||||
// Preferred CMM: no preference.
|
||||
h.extend_from_slice(&[0; 4]);
|
||||
// Version 2.1.0. v2 rather than v4 because it is what every reader
|
||||
// handles, and because nothing here needs a v4 tag type.
|
||||
h.extend_from_slice(&[0x02, 0x10, 0x00, 0x00]);
|
||||
h.extend_from_slice(b"mntr");
|
||||
h.extend_from_slice(b"RGB ");
|
||||
h.extend_from_slice(b"XYZ ");
|
||||
// Creation date. Fixed, for the determinism the module docs describe.
|
||||
for field in [2025u16, 1, 1, 0, 0, 0] {
|
||||
h.extend_from_slice(&field.to_be_bytes());
|
||||
}
|
||||
h.extend_from_slice(b"acsp");
|
||||
// Primary platform, flags, manufacturer, model, attributes: unspecified.
|
||||
h.extend_from_slice(&[0; 24]);
|
||||
// Rendering intent: perceptual, as the reference RGB working-space
|
||||
// profiles declare. For a matrix/TRC profile the field is advisory —
|
||||
// there is only one transform in here to apply.
|
||||
h.extend_from_slice(&[0; 4]);
|
||||
for v in PCS_D50 {
|
||||
h.extend_from_slice(&s15_fixed16(v).to_be_bytes());
|
||||
}
|
||||
// Creator, profile ID, and the reserved tail.
|
||||
h.extend_from_slice(&[0; 4]);
|
||||
h.extend_from_slice(&[0; 16]);
|
||||
h.extend_from_slice(&[0; 28]);
|
||||
debug_assert_eq!(h.len(), 128);
|
||||
h
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
/// A tag's element data, located through the profile's own tag table —
|
||||
/// so these tests read the profile the way a colour engine would rather
|
||||
/// than the way it was written.
|
||||
fn tag<'a>(profile: &'a [u8], want: &[u8; 4]) -> Option<&'a [u8]> {
|
||||
let count = u32::from_be_bytes(profile[128..132].try_into().unwrap()) as usize;
|
||||
for i in 0..count {
|
||||
let at = 132 + i * 12;
|
||||
if &profile[at..at + 4] == want {
|
||||
let off = u32::from_be_bytes(profile[at + 4..at + 8].try_into().unwrap()) as usize;
|
||||
let len = u32::from_be_bytes(profile[at + 8..at + 12].try_into().unwrap()) as usize;
|
||||
return Some(&profile[off..off + len]);
|
||||
}
|
||||
}
|
||||
None
|
||||
}
|
||||
|
||||
fn xyz(data: &[u8]) -> [f32; 3] {
|
||||
let read =
|
||||
|at: usize| i32::from_be_bytes(data[at..at + 4].try_into().unwrap()) as f32 / 65536.0;
|
||||
[read(8), read(12), read(16)]
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_profile_declares_its_own_length() {
|
||||
// The first field a reader trusts. A profile whose header says it is
|
||||
// longer than the buffer is one a strict parser rejects outright and a
|
||||
// lax one reads past the end of.
|
||||
for space in ColourSpace::ALL {
|
||||
let p = profile(space);
|
||||
let declared = u32::from_be_bytes(p[0..4].try_into().unwrap()) as usize;
|
||||
assert_eq!(declared, p.len(), "{space:?}");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_profile_carries_the_signature_that_identifies_it_as_one() {
|
||||
// `acsp` at offset 36 is how every reader recognises an ICC profile.
|
||||
for space in ColourSpace::ALL {
|
||||
assert_eq!(&profile(space)[36..40], b"acsp", "{space:?}");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn every_tag_lies_inside_the_profile_and_on_a_boundary() {
|
||||
// A tag table is offsets and lengths, and nothing checks them for us.
|
||||
// An off-by-four here produces a profile that parses as far as the
|
||||
// tag a reader happens to want.
|
||||
for space in ColourSpace::ALL {
|
||||
let p = profile(space);
|
||||
let count = u32::from_be_bytes(p[128..132].try_into().unwrap()) as usize;
|
||||
for i in 0..count {
|
||||
let at = 132 + i * 12;
|
||||
let off = u32::from_be_bytes(p[at + 4..at + 8].try_into().unwrap()) as usize;
|
||||
let len = u32::from_be_bytes(p[at + 8..at + 12].try_into().unwrap()) as usize;
|
||||
assert!(off.is_multiple_of(4), "{space:?} tag {i} starts at {off}");
|
||||
assert!(off + len <= p.len(), "{space:?} tag {i} runs off the end");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn every_profile_carries_the_tags_a_matrix_trc_profile_requires() {
|
||||
// The ICC v2 required set for a display profile. A reader missing any
|
||||
// one of these falls back to assuming sRGB, which is the silent
|
||||
// failure this whole feature exists to prevent.
|
||||
for space in ColourSpace::ALL {
|
||||
let p = profile(space);
|
||||
for required in [
|
||||
b"desc", b"cprt", b"wtpt", b"rXYZ", b"gXYZ", b"bXYZ", b"rTRC", b"gTRC", b"bTRC",
|
||||
] {
|
||||
assert!(tag(&p, required).is_some(), "{space:?} has no {required:?}");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn the_colorants_are_the_ones_the_shader_encoded_with() {
|
||||
// The property the file's honesty rests on. The composer converts the
|
||||
// pixels with `to_pcs_xyz`'s primaries; if the profile described any
|
||||
// others the file would be a precise, confident lie.
|
||||
for space in ColourSpace::ALL {
|
||||
let p = profile(space);
|
||||
let want = space.to_pcs_xyz();
|
||||
for (i, sig) in [b"rXYZ", b"gXYZ", b"bXYZ"].into_iter().enumerate() {
|
||||
let got = xyz(tag(&p, sig).expect("colorant"));
|
||||
for (row, g) in got.iter().enumerate() {
|
||||
let expected = want[row * 3 + i];
|
||||
assert!(
|
||||
(g - expected).abs() < 1e-4,
|
||||
"{space:?} {sig:?} row {row}: profile says {g}, shader used {expected}"
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn the_white_point_is_the_connection_space_illuminant() {
|
||||
// Not the space's own white. ProPhoto's is D50 anyway, but P3's is
|
||||
// D65, and a profile advertising D65 as its media white would have
|
||||
// every neutral adapted a second time.
|
||||
for space in ColourSpace::ALL {
|
||||
let got = xyz(tag(&profile(space), b"wtpt").expect("wtpt"));
|
||||
for (i, want) in PCS_D50.iter().enumerate() {
|
||||
assert!((got[i] - want).abs() < 1e-4, "{space:?} white {i}: {got:?}");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_tabulated_curve_reproduces_the_transfer_function_it_came_from() {
|
||||
// Read back out of the profile and compared against the function the
|
||||
// shader encodes with. The curve runs device-to-linear, and writing it
|
||||
// the other way round would still produce a monotonic curve of the
|
||||
// right length — this is what catches the direction.
|
||||
for space in [ColourSpace::Srgb, ColourSpace::ProPhoto] {
|
||||
let p = profile(space);
|
||||
let curve = tag(&p, b"rTRC").expect("rTRC");
|
||||
let count = u32::from_be_bytes(curve[8..12].try_into().unwrap()) as usize;
|
||||
assert_eq!(count, TRC_SAMPLES, "{space:?}");
|
||||
|
||||
let transfer = space.transfer();
|
||||
for i in [0, 1, count / 4, count / 2, count - 1] {
|
||||
let at = 12 + i * 2;
|
||||
let got =
|
||||
f32::from(u16::from_be_bytes(curve[at..at + 2].try_into().unwrap())) / 65535.0;
|
||||
let want = transfer.decode(i as f32 / (count - 1) as f32);
|
||||
assert!(
|
||||
(got - want).abs() < 1e-4,
|
||||
"{space:?} sample {i}: profile {got}, transfer {want}"
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn adobe_rgb_stores_its_gamma_exactly_rather_than_sampling_it() {
|
||||
// 563/256 is representable in a u8Fixed8, so the curve is one number.
|
||||
// A 1024-entry table would approximate a value the format can hold
|
||||
// exactly, and would round-trip through other software as 2.2.
|
||||
let p = profile(ColourSpace::AdobeRgb);
|
||||
let curve = tag(&p, b"rTRC").expect("rTRC");
|
||||
assert_eq!(u32::from_be_bytes(curve[8..12].try_into().unwrap()), 1);
|
||||
assert_eq!(u16::from_be_bytes(curve[12..14].try_into().unwrap()), 563);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn the_three_tone_curves_share_one_copy() {
|
||||
// Not a size optimisation for its own sake: it keeps the profile under
|
||||
// the 64 KB a single JPEG APP2 segment holds, so the chunked form that
|
||||
// older readers mishandle is never needed.
|
||||
let p = profile(ColourSpace::Srgb);
|
||||
let count = u32::from_be_bytes(p[128..132].try_into().unwrap()) as usize;
|
||||
let offsets: Vec<u32> = ["rTRC", "gTRC", "bTRC"]
|
||||
.iter()
|
||||
.map(|sig| {
|
||||
(0..count)
|
||||
.map(|i| 132 + i * 12)
|
||||
.find(|at| &p[*at..at + 4] == sig.as_bytes())
|
||||
.map(|at| u32::from_be_bytes(p[at + 4..at + 8].try_into().unwrap()))
|
||||
.expect("curve present")
|
||||
})
|
||||
.collect();
|
||||
assert_eq!(offsets[0], offsets[1]);
|
||||
assert_eq!(offsets[1], offsets[2]);
|
||||
assert!(p.len() < 8 * 1024, "{} bytes is too large", p.len());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_d65_space_declares_its_adaptation_and_a_d50_space_does_not() {
|
||||
// `chad` is what lets a reader recover the space's native white from
|
||||
// colorants that have already been adapted. Without it, D65 primaries
|
||||
// adapted to D50 and genuine D50 primaries are the same nine numbers.
|
||||
assert!(tag(&profile(ColourSpace::DisplayP3), b"chad").is_some());
|
||||
assert!(
|
||||
tag(&profile(ColourSpace::ProPhoto), b"chad").is_none(),
|
||||
"ProPhoto is a D50 space; an identity chad says nothing"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn the_same_space_always_produces_the_same_bytes() {
|
||||
// Two exports of one frame must be identical files. A creation
|
||||
// timestamp from the clock is the obvious way to lose that.
|
||||
for space in ColourSpace::ALL {
|
||||
assert_eq!(profile(space), profile(space), "{space:?}");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn each_space_is_described_by_its_own_name() {
|
||||
// A file whose profile says "sRGB" while carrying P3 pixels is exactly
|
||||
// as misleading as no profile at all, and harder to notice.
|
||||
for space in ColourSpace::ALL {
|
||||
let p = profile(space);
|
||||
let desc = tag(&p, b"desc").expect("desc");
|
||||
let len = u32::from_be_bytes(desc[8..12].try_into().unwrap()) as usize;
|
||||
let name = std::str::from_utf8(&desc[12..12 + len - 1]).expect("ascii");
|
||||
assert_eq!(name, format!("DarkRoom {}", space.label()));
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,156 +0,0 @@
|
||||
//! TRACES: FR-MRG-4
|
||||
//! The largest rectangle inside a coverage mask, found a row at a time.
|
||||
//!
|
||||
//! A merged panorama has ragged edges: the frames' footprints under a
|
||||
//! cylinder or a sphere are not rectangles, and the composite carries a
|
||||
//! black border where none of them reached. FR-MRG-4 asks for an auto-crop
|
||||
//! to the largest inscribed rectangle. This finds it as the bands are
|
||||
//! produced, so the composite is never held to be measured (FR-MRG-11):
|
||||
//! each row extends a running histogram of consecutive covered rows above
|
||||
//! it, and the largest rectangle ending on that row is the largest
|
||||
//! rectangle under the histogram — a stack pass, linear in the width.
|
||||
//!
|
||||
//! The crop is written as the DNG's `DefaultCropOrigin`/`DefaultCropSize`,
|
||||
//! which every reader honours and which discards nothing: the pixels
|
||||
//! outside it are still in the file for a photographer who wants them.
|
||||
|
||||
/// The rectangle so far, in pixels from the top left.
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)]
|
||||
pub struct Rect {
|
||||
pub x: u32,
|
||||
pub y: u32,
|
||||
pub width: u32,
|
||||
pub height: u32,
|
||||
}
|
||||
|
||||
impl Rect {
|
||||
pub fn area(&self) -> u64 {
|
||||
u64::from(self.width) * u64::from(self.height)
|
||||
}
|
||||
}
|
||||
|
||||
/// Feed rows top to bottom; ask for the best at any point.
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct Inscribed {
|
||||
width: usize,
|
||||
/// How many consecutive covered rows end at the last row fed, per column.
|
||||
heights: Vec<u32>,
|
||||
rows: u32,
|
||||
best: Rect,
|
||||
}
|
||||
|
||||
impl Inscribed {
|
||||
pub fn new(width: u32) -> Self {
|
||||
Inscribed {
|
||||
width: width as usize,
|
||||
heights: vec![0; width as usize],
|
||||
rows: 0,
|
||||
best: Rect::default(),
|
||||
}
|
||||
}
|
||||
|
||||
/// One more row of coverage, `width` long.
|
||||
pub fn push_row(&mut self, covered: &[bool]) {
|
||||
debug_assert_eq!(covered.len(), self.width);
|
||||
for (h, &c) in self.heights.iter_mut().zip(covered) {
|
||||
*h = if c { *h + 1 } else { 0 };
|
||||
}
|
||||
self.rows += 1;
|
||||
// Largest rectangle under the histogram, with a sentinel column of
|
||||
// height 0 at the end so every bar is popped.
|
||||
let mut stack: Vec<usize> = Vec::new();
|
||||
for i in 0..=self.width {
|
||||
let h = if i < self.width { self.heights[i] } else { 0 };
|
||||
while let Some(&top) = stack.last() {
|
||||
if self.heights[top] <= h {
|
||||
break;
|
||||
}
|
||||
stack.pop();
|
||||
let height = self.heights[top];
|
||||
let left = stack.last().map_or(0, |&l| l + 1);
|
||||
let width = (i - left) as u32;
|
||||
let area = u64::from(width) * u64::from(height);
|
||||
if area > self.best.area() {
|
||||
self.best = Rect {
|
||||
x: left as u32,
|
||||
y: self.rows - height,
|
||||
width,
|
||||
height,
|
||||
};
|
||||
}
|
||||
}
|
||||
stack.push(i);
|
||||
}
|
||||
}
|
||||
|
||||
/// Several rows at once, as a band hands them over.
|
||||
pub fn push_rows(&mut self, covered: &[bool], rows: u32) {
|
||||
for r in 0..rows as usize {
|
||||
self.push_row(&covered[r * self.width..(r + 1) * self.width]);
|
||||
}
|
||||
}
|
||||
|
||||
pub fn best(&self) -> Rect {
|
||||
self.best
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
fn from_art(art: &[&str]) -> Rect {
|
||||
let mut ins = Inscribed::new(art[0].len() as u32);
|
||||
for row in art {
|
||||
let covered: Vec<bool> = row.chars().map(|c| c == '#').collect();
|
||||
ins.push_row(&covered);
|
||||
}
|
||||
ins.best()
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_full_mask_is_its_own_rectangle() {
|
||||
let r = from_art(&["####", "####", "####"]);
|
||||
assert_eq!(
|
||||
r,
|
||||
Rect {
|
||||
x: 0,
|
||||
y: 0,
|
||||
width: 4,
|
||||
height: 3
|
||||
}
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn ragged_edges_are_cut_off() {
|
||||
// A cylinder's footprint: narrower at top and bottom.
|
||||
let r = from_art(&[
|
||||
"..####..", ".######.", "########", "########", ".######.", "..####..",
|
||||
]);
|
||||
// 6 wide × 4 tall = 24 beats 8 × 2 = 16 and 4 × 6 = 24 ties; the
|
||||
// first found wins a tie, which is the wider one here.
|
||||
assert_eq!(r.area(), 24);
|
||||
assert!(r.width == 6 && r.height == 4 || r.width == 4 && r.height == 6);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_hole_is_avoided() {
|
||||
let r = from_art(&["#####", "##.##", "#####", "#####"]);
|
||||
// Left of the hole: 2 × 4 = 8; right: 2 × 4 = 8; below: 5 × 2 = 10.
|
||||
assert_eq!(
|
||||
r,
|
||||
Rect {
|
||||
x: 0,
|
||||
y: 2,
|
||||
width: 5,
|
||||
height: 2
|
||||
}
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn nothing_covered_is_nothing() {
|
||||
assert_eq!(from_art(&["....", "...."]).area(), 0);
|
||||
}
|
||||
}
|
||||
@@ -1,396 +0,0 @@
|
||||
//! TRACES: FR-EXP-1 | FR-EXP-2 | FR-EXP-3 | FR-EXP-4 | FR-EXP-6 | FR-EXP-9 | R3
|
||||
//! Turning a rendered frame into a file's worth of bytes.
|
||||
//!
|
||||
//! # What this crate is, and is not
|
||||
//!
|
||||
//! It is: resize, output sharpening, encode, and the name the result should
|
||||
//! be given. It is not: a filesystem, a network client, or a job queue.
|
||||
//! [`export`] returns [`Encoded`] — bytes and a filename — and the caller
|
||||
//! decides where that lands.
|
||||
//!
|
||||
//! That boundary is not fastidiousness. An export has three possible
|
||||
//! destinations and they have nothing in common: a path on Linux, a Storage
|
||||
//! Access Framework document on Android where there *is* no path
|
||||
//! (ARCH §6.9), and a `PUT` to a Nextcloud folder. A crate that wrote the
|
||||
//! file itself would serve one of them and be rewritten for the other two.
|
||||
//!
|
||||
//! # Order of operations
|
||||
//!
|
||||
//! Resize, then sharpen, then encode. Sharpening after the resize is the
|
||||
//! whole point of output sharpening (FR-EXP-4): it compensates for the
|
||||
//! softening the resample introduced, so its strength has to scale with how
|
||||
//! much scaling actually happened. Sharpening first and then shrinking would
|
||||
//! throw the sharpened detail away.
|
||||
|
||||
use dr_types::{ColourSpace, ExportFormat, ExportSettings};
|
||||
|
||||
mod dng;
|
||||
mod encode;
|
||||
mod error;
|
||||
mod exif;
|
||||
pub mod icc;
|
||||
mod inscribed;
|
||||
mod metadata;
|
||||
mod name;
|
||||
mod sharpen;
|
||||
mod size;
|
||||
|
||||
pub use dng::{write_linear_dng, DngProfile};
|
||||
pub use error::ExportError;
|
||||
pub use inscribed::{Inscribed, Rect};
|
||||
pub use metadata::SourceMetadata;
|
||||
pub use name::{resolve_name, NameContext};
|
||||
pub use size::target_size;
|
||||
|
||||
/// A rendered frame, as the adjust pass produced it.
|
||||
///
|
||||
/// 8-bit RGBA, display-encoded in [`Self::space`] — the format
|
||||
/// [`dr_gpu::AdjustPass`](../dr_gpu/struct.AdjustPass.html) writes. Alpha is
|
||||
/// carried but never meaningful: the pipeline writes 1.0 everywhere, and no
|
||||
/// operation produces transparency.
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct Frame {
|
||||
pub width: u32,
|
||||
pub height: u32,
|
||||
/// Tightly packed RGBA8, `width * height * 4` bytes.
|
||||
pub rgba: Vec<u8>,
|
||||
/// TRACES: FR-EXP-2
|
||||
/// The space the shader encoded these pixels into.
|
||||
///
|
||||
/// Travels with the pixels rather than being asserted at the point of
|
||||
/// encoding, because it is a fact about them and not a preference. The
|
||||
/// conversion happened in the generated shader, before the clip to 0..1,
|
||||
/// and nothing downstream can undo or redo it — a frame clipped to sRGB
|
||||
/// has already lost whatever a wider space would have carried.
|
||||
///
|
||||
/// Making it a field is what lets [`export`] refuse to label a frame as
|
||||
/// something it is not, rather than trusting a caller to have rendered
|
||||
/// what it asked for.
|
||||
pub space: ColourSpace,
|
||||
}
|
||||
|
||||
impl Frame {
|
||||
/// A frame the pipeline rendered in sRGB — what
|
||||
/// [`EditGraph::compose`](../dr_pipeline/struct.EditGraph.html#method.compose)
|
||||
/// produces, and so what the display path hands over.
|
||||
///
|
||||
/// An export in a wider space must render its own frame with
|
||||
/// `compose_for` and declare it through [`Self::in_space`]. Defaulting
|
||||
/// here rather than demanding the space at every call site keeps the
|
||||
/// common case honest by construction: a caller that has not thought
|
||||
/// about colour is describing sRGB, and sRGB is what it rendered.
|
||||
pub fn new(width: u32, height: u32, rgba: Vec<u8>) -> Result<Self, ExportError> {
|
||||
Self::in_space(width, height, rgba, ColourSpace::Srgb)
|
||||
}
|
||||
|
||||
/// A frame rendered into a stated colour space.
|
||||
pub fn in_space(
|
||||
width: u32,
|
||||
height: u32,
|
||||
rgba: Vec<u8>,
|
||||
space: ColourSpace,
|
||||
) -> Result<Self, ExportError> {
|
||||
let expected = width as usize * height as usize * 4;
|
||||
if rgba.len() != expected {
|
||||
return Err(ExportError::FrameSize {
|
||||
expected,
|
||||
got: rgba.len(),
|
||||
});
|
||||
}
|
||||
if width == 0 || height == 0 {
|
||||
return Err(ExportError::EmptyFrame);
|
||||
}
|
||||
Ok(Self {
|
||||
width,
|
||||
height,
|
||||
rgba,
|
||||
space,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
/// The finished article: what to write, and what to call it.
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct Encoded {
|
||||
/// Filename including extension. Never a path — the destination folder is
|
||||
/// the caller's, and on Android it is not expressible as one anyway.
|
||||
pub name: String,
|
||||
pub bytes: Vec<u8>,
|
||||
/// What the image was actually written at, after sizing and the upscaling
|
||||
/// guard. Worth reporting: a batch that silently exported at source size
|
||||
/// because the request was larger has done something the user should know.
|
||||
pub width: u32,
|
||||
pub height: u32,
|
||||
}
|
||||
|
||||
/// Resize, sharpen and encode one frame.
|
||||
///
|
||||
/// `name` is the filename already resolved by [`resolve_name`] — passed in
|
||||
/// rather than derived here because resolving it needs to know what is
|
||||
/// already in the destination, which this crate cannot see.
|
||||
///
|
||||
/// TRACES: FR-EXP-9
|
||||
/// The frame is expected to be a **full-resolution** render. Nothing here
|
||||
/// enforces that, because nothing here can tell a full render from a
|
||||
/// viewport-sized one; the caller renders at the framed output size and this
|
||||
/// resamples down from it. Exporting from the display proxy would silently
|
||||
/// produce a soft file, which is why the develop session's export path renders
|
||||
/// its own frame rather than reusing the one on screen.
|
||||
///
|
||||
/// TRACES: FR-EXP-8
|
||||
/// `source` is what the photograph's own file said about itself, or `None`
|
||||
/// where the caller has nothing — a frame that came from somewhere other than
|
||||
/// a decoded file, or a caller that has not yet been taught to pass it.
|
||||
///
|
||||
/// **A parameter rather than a field on [`Frame`]**, because it is not a fact
|
||||
/// about the pixels: two exports of the same frame can legitimately disclose
|
||||
/// different amounts, and the settings that decide how much travel beside it.
|
||||
/// It is also why this is an argument and not an `Option` with a default — a
|
||||
/// caller that has the source metadata should have to decide, in one visible
|
||||
/// place, to hand it over.
|
||||
pub fn export(
|
||||
frame: &Frame,
|
||||
settings: &ExportSettings,
|
||||
name: String,
|
||||
source: Option<&SourceMetadata>,
|
||||
) -> Result<Encoded, ExportError> {
|
||||
// TRACES: FR-EXP-2
|
||||
// Refused rather than mislabelled. Every space the settings page offers
|
||||
// now works, but only if the *frame* was rendered into it: the conversion
|
||||
// and the clip both happen in the generated shader, so pixels that arrive
|
||||
// clipped to sRGB have already lost whatever a wider space would have
|
||||
// carried, and no amount of profile-writing here brings it back.
|
||||
//
|
||||
// The caller's fix is to compose with `EditGraph::compose_for(space)`
|
||||
// before rendering. Until it does, this is an accurate error where the
|
||||
// alternative would be a file that claims a gamut it does not contain —
|
||||
// and that claim survives into everything downstream.
|
||||
if frame.space != settings.colour_space {
|
||||
return Err(ExportError::ColourSpaceMismatch {
|
||||
rendered: frame.space,
|
||||
requested: settings.colour_space,
|
||||
});
|
||||
}
|
||||
|
||||
if matches!(settings.format, ExportFormat::Avif | ExportFormat::JpegXl) {
|
||||
return Err(ExportError::FormatUnsupported(settings.format));
|
||||
}
|
||||
|
||||
let (width, height) = size::target_size(
|
||||
frame.width,
|
||||
frame.height,
|
||||
settings.sizing,
|
||||
settings.allow_upscaling,
|
||||
);
|
||||
|
||||
// TRACES: FR-EXP-3
|
||||
// One mode promises exact dimensions rather than a bound on them, and it
|
||||
// is the only place the fit/fill distinction survives: `target_size` has
|
||||
// already reported what the file will be either way.
|
||||
let resized = if settings.sizing.crops_to_fill() {
|
||||
size::resample_filling(frame, width, height)
|
||||
} else {
|
||||
size::resample(frame, width, height)
|
||||
};
|
||||
|
||||
// Scaled by how much the image actually shrank: a full-size export needs
|
||||
// no compensation, and a thumbnail needs a great deal.
|
||||
let scale = width as f32 / frame.width.max(1) as f32;
|
||||
let sharpened = sharpen::apply(resized, width, height, settings.sharpening, scale);
|
||||
|
||||
let bytes = encode::encode(&sharpened, width, height, settings, source)?;
|
||||
|
||||
Ok(Encoded {
|
||||
name,
|
||||
bytes,
|
||||
width,
|
||||
height,
|
||||
})
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use dr_types::SizingMode;
|
||||
|
||||
/// A frame with a recognisable gradient, so a resample can be checked for
|
||||
/// having done something rather than merely returned the right length.
|
||||
pub(crate) fn frame(w: u32, h: u32) -> Frame {
|
||||
let mut rgba = Vec::with_capacity((w * h * 4) as usize);
|
||||
for y in 0..h {
|
||||
for x in 0..w {
|
||||
rgba.push((x * 255 / w.max(1)) as u8);
|
||||
rgba.push((y * 255 / h.max(1)) as u8);
|
||||
rgba.push(128);
|
||||
rgba.push(255);
|
||||
}
|
||||
}
|
||||
Frame::new(w, h, rgba).expect("well-formed")
|
||||
}
|
||||
|
||||
fn settings(format: ExportFormat) -> ExportSettings {
|
||||
ExportSettings {
|
||||
format,
|
||||
..Default::default()
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_frame_rejects_a_buffer_of_the_wrong_length() {
|
||||
// The one error that would otherwise surface as a panic deep in an
|
||||
// encoder, or worse, as a file of garbage.
|
||||
assert!(matches!(
|
||||
Frame::new(4, 4, vec![0; 10]),
|
||||
Err(ExportError::FrameSize { .. })
|
||||
));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn jpeg_export_produces_a_jpeg() {
|
||||
let out = export(
|
||||
&frame(64, 48),
|
||||
&settings(ExportFormat::Jpeg),
|
||||
"a.jpg".into(),
|
||||
None,
|
||||
)
|
||||
.unwrap();
|
||||
// SOI marker. Cheap, and it catches an encoder wired to the wrong
|
||||
// format far more directly than a byte count would.
|
||||
assert_eq!(&out.bytes[..2], &[0xFF, 0xD8]);
|
||||
assert_eq!((out.width, out.height), (64, 48));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn png_export_produces_a_png() {
|
||||
let out = export(
|
||||
&frame(32, 32),
|
||||
&settings(ExportFormat::Png),
|
||||
"a.png".into(),
|
||||
None,
|
||||
)
|
||||
.unwrap();
|
||||
assert_eq!(&out.bytes[..8], b"\x89PNG\r\n\x1a\n");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn tiff_exports_produce_a_tiff() {
|
||||
for format in [ExportFormat::Tiff8, ExportFormat::Tiff16] {
|
||||
let out = export(&frame(16, 16), &settings(format), "a.tif".into(), None).unwrap();
|
||||
// Either byte order is a valid TIFF; the crate writes little-endian.
|
||||
assert!(
|
||||
out.bytes.starts_with(b"II*\0") || out.bytes.starts_with(b"MM\0*"),
|
||||
"{format:?} did not produce a TIFF header"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_sixteen_bit_tiff_is_larger_than_an_eight_bit_one() {
|
||||
// Both are uncompressed RGB; the only difference is the sample width,
|
||||
// so this is what proves the 16-bit path is not quietly writing 8.
|
||||
let eight = export(
|
||||
&frame(16, 16),
|
||||
&settings(ExportFormat::Tiff8),
|
||||
"a".into(),
|
||||
None,
|
||||
)
|
||||
.unwrap();
|
||||
let sixteen = export(
|
||||
&frame(16, 16),
|
||||
&settings(ExportFormat::Tiff16),
|
||||
"a".into(),
|
||||
None,
|
||||
)
|
||||
.unwrap();
|
||||
assert!(sixteen.bytes.len() > eight.bytes.len());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn quality_changes_the_size_of_a_jpeg() {
|
||||
// The setting is plumbed all the way to the encoder rather than
|
||||
// accepted and dropped, which a size-independent output would show.
|
||||
let mut low = settings(ExportFormat::Jpeg);
|
||||
low.quality = 20;
|
||||
let mut high = settings(ExportFormat::Jpeg);
|
||||
high.quality = 98;
|
||||
|
||||
let small = export(&frame(128, 128), &low, "a".into(), None).unwrap();
|
||||
let large = export(&frame(128, 128), &high, "a".into(), None).unwrap();
|
||||
assert!(
|
||||
large.bytes.len() > small.bytes.len(),
|
||||
"quality 98 produced {} bytes against quality 20's {}",
|
||||
large.bytes.len(),
|
||||
small.bytes.len()
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_long_edge_export_lands_on_the_requested_size() {
|
||||
let mut s = settings(ExportFormat::Png);
|
||||
s.sizing = SizingMode::LongEdge(32);
|
||||
let out = export(&frame(128, 64), &s, "a".into(), None).unwrap();
|
||||
assert_eq!((out.width, out.height), (32, 16));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_frame_rendered_in_one_space_is_not_labelled_another() {
|
||||
// A file tagged Display P3 carrying sRGB-clipped pixels is a lie that
|
||||
// survives into everything downstream. The frame carries the space it
|
||||
// was rendered in precisely so this cannot be waved through.
|
||||
let mut s = settings(ExportFormat::Jpeg);
|
||||
s.colour_space = ColourSpace::DisplayP3;
|
||||
assert!(matches!(
|
||||
export(&frame(8, 8), &s, "a".into(), None),
|
||||
Err(ExportError::ColourSpaceMismatch { .. })
|
||||
));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn every_colour_space_exports_when_the_frame_was_rendered_in_it() {
|
||||
// The other side of the refusal above, and what FR-EXP-2 actually
|
||||
// asks for: a frame the pipeline encoded into a wide space reaches a
|
||||
// file, in every format that has an encoder.
|
||||
for space in ColourSpace::ALL {
|
||||
for format in [
|
||||
ExportFormat::Jpeg,
|
||||
ExportFormat::Png,
|
||||
ExportFormat::Tiff8,
|
||||
ExportFormat::Tiff16,
|
||||
] {
|
||||
let mut s = settings(format);
|
||||
s.colour_space = space;
|
||||
let mut f = frame(8, 8);
|
||||
f.space = space;
|
||||
let out = export(&f, &s, "a".into(), None)
|
||||
.unwrap_or_else(|e| panic!("{space:?} as {format:?}: {e}"));
|
||||
assert!(!out.bytes.is_empty());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn the_formats_without_an_encoder_say_so() {
|
||||
for format in [ExportFormat::Avif, ExportFormat::JpegXl] {
|
||||
assert!(
|
||||
matches!(
|
||||
export(&frame(8, 8), &settings(format), "a".into(), None),
|
||||
Err(ExportError::FormatUnsupported(_))
|
||||
),
|
||||
"{format:?} should report that it has no encoder yet"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn every_offered_format_either_encodes_or_explains_itself() {
|
||||
// Walks `ExportFormat::ALL`, so a format added to the settings page
|
||||
// cannot quietly reach an encoder that does not handle it.
|
||||
for format in ExportFormat::ALL {
|
||||
match export(&frame(8, 8), &settings(format), "a".into(), None) {
|
||||
Ok(out) => assert!(!out.bytes.is_empty(), "{format:?} encoded to nothing"),
|
||||
Err(ExportError::FormatUnsupported(f)) => assert_eq!(f, format),
|
||||
Err(e) => panic!("{format:?} failed unexpectedly: {e}"),
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,106 +0,0 @@
|
||||
//! TRACES: FR-EXP-8
|
||||
//! What an export is allowed to say about where it came from.
|
||||
//!
|
||||
//! # An allowlist, not a filter
|
||||
//!
|
||||
//! [`SourceMetadata`] is the whole of what can reach a file this crate writes.
|
||||
//! It is populated field by field from whatever the caller decoded, and
|
||||
//! nothing else travels — not because each unwanted tag is removed, but
|
||||
//! because there is nowhere in this type for one to sit. That is the
|
||||
//! difference between "we strip GPS" and "GPS cannot be written unless
|
||||
//! [`SourceMetadata::location`] is `Some`", and only the second survives
|
||||
//! somebody adding a field to the decoder next year.
|
||||
//!
|
||||
//! # What is deliberately not here
|
||||
//!
|
||||
//! **The maker note** (EXIF `0x927C`). It is an opaque vendor blob with no
|
||||
//! public format, and its contents differ by body and firmware. Canon's
|
||||
//! carries the body serial number and the shutter count; several bodies put a
|
||||
//! *duplicate copy of the GPS fix* inside it, which is the specific reason it
|
||||
//! cannot be passed through as an unexamined byte range: an export that
|
||||
//! stripped the GPS directory and copied the maker note would have published
|
||||
//! the coordinates anyway, while reporting itself as private. Parsing it per
|
||||
//! vendor to decide what is safe is a research project with a permanent
|
||||
//! maintenance cost, and the value on the other side is a few tags a
|
||||
//! photographer rarely misses. So it is dropped, in both directions, whatever
|
||||
//! the settings say.
|
||||
//!
|
||||
//! **Serial numbers and owner name** (`BodySerialNumber` 0xA431,
|
||||
//! `LensSerialNumber` 0xA435, `CameraOwnerName` 0xA430). These identify a
|
||||
//! person and a specific piece of equipment, and a serial number in a
|
||||
//! published file links every photograph that person has ever posted. They
|
||||
//! have no field here, so no export writes them.
|
||||
//!
|
||||
//! **IPTC and XMP.** FR-EXP-8 names both. Neither is read by `dr-decode`
|
||||
//! today, so there is nothing to carry through; when there is, it arrives as
|
||||
//! fields on this type and is written from them, and the same allowlist
|
||||
//! reasoning applies unchanged.
|
||||
|
||||
use dr_types::Location;
|
||||
|
||||
/// TRACES: FR-EXP-8
|
||||
/// The source metadata an export may carry.
|
||||
///
|
||||
/// Every field is optional because every field is genuinely absent from some
|
||||
/// real file: scanner output has no aperture, a JPEG from a phone has no lens
|
||||
/// model, and most photographs have no copyright statement at all.
|
||||
///
|
||||
/// Built by the caller, which is the only place that has both the decoded
|
||||
/// source and the crate that decoded it — `dr-export` deliberately depends on
|
||||
/// no decoder (see the crate docs), so the copy is made one field at a time
|
||||
/// where both types are in scope. That transcription is a feature: it is the
|
||||
/// point where somebody has to decide, in writing, that a newly-parsed piece
|
||||
/// of the source is allowed to leave the machine.
|
||||
#[derive(Debug, Clone, Default, PartialEq)]
|
||||
pub struct SourceMetadata {
|
||||
pub make: Option<String>,
|
||||
pub model: Option<String>,
|
||||
pub lens: Option<String>,
|
||||
/// Exposure time in seconds.
|
||||
pub shutter: Option<f32>,
|
||||
/// The f-number, as in f/2.8.
|
||||
pub aperture: Option<f32>,
|
||||
pub iso: Option<u32>,
|
||||
/// Millimetres, as marked on the lens rather than 35 mm equivalent.
|
||||
pub focal_length: Option<f32>,
|
||||
/// When the shutter fired, as Unix seconds read as a wall clock.
|
||||
pub captured_at: Option<i64>,
|
||||
/// Minutes east of UTC, where the camera recorded a zone.
|
||||
pub captured_offset: Option<i32>,
|
||||
/// Who made the photograph.
|
||||
pub artist: Option<String>,
|
||||
/// The rights statement.
|
||||
pub copyright: Option<String>,
|
||||
/// TRACES: FR-EXP-8
|
||||
/// Where the shutter fired.
|
||||
///
|
||||
/// The one field the strip option is about. It is carried this far so that
|
||||
/// a photographer who *wants* their coordinates can have them; by the time
|
||||
/// the encoder sees the record this field has already been through
|
||||
/// [`Self::sanitised`], and is `None` unless the user turned stripping
|
||||
/// off.
|
||||
pub location: Option<Location>,
|
||||
}
|
||||
|
||||
impl SourceMetadata {
|
||||
/// This record as the settings permit it to be written.
|
||||
///
|
||||
/// **The single place stripping happens.** The encoders below take a
|
||||
/// record and write what is in it, with no view on privacy; concentrating
|
||||
/// the decision here means there is one function to read to know what an
|
||||
/// export can disclose, and no format can quietly disagree with the
|
||||
/// others — the failure mode where JPEG honours the setting and TIFF, five
|
||||
/// hundred lines away, does not.
|
||||
///
|
||||
/// Stripping empties the field rather than blanking it. A `GPSLatitude` of
|
||||
/// `0/0` still announces that the camera had a fix and that this file has
|
||||
/// been through a scrubber; an absent directory says nothing at all, and
|
||||
/// says it in the same shape as the millions of files that never had one.
|
||||
pub(crate) fn sanitised(&self, strip_location: bool) -> Self {
|
||||
let mut out = self.clone();
|
||||
if strip_location {
|
||||
out.location = None;
|
||||
}
|
||||
out
|
||||
}
|
||||
}
|
||||
@@ -1,316 +0,0 @@
|
||||
//! TRACES: FR-EXP-6
|
||||
//! Filename templates and what to do when the name is taken.
|
||||
//!
|
||||
//! # Why the caller supplies the "does this exist" test
|
||||
//!
|
||||
//! [`resolve_name`] takes a closure rather than looking at a directory,
|
||||
//! because there is no directory it could look at that would work everywhere.
|
||||
//! A destination is a path on Linux, a Storage Access Framework tree on
|
||||
//! Android with no path at all (ARCH §6.9), or a folder on a Nextcloud
|
||||
//! server reached by PROPFIND. All three can answer "is this name taken",
|
||||
//! and none of them can be asked the same way.
|
||||
//!
|
||||
//! It matters most on Android, where the platform actively works against us:
|
||||
//! `DocumentsContract.createDocument` renames on collision *by itself*,
|
||||
//! appending ` (1)` and returning a URI with a name nobody asked for, and it
|
||||
//! cannot overwrite at all. So every one of the three [`CollisionPolicy`]
|
||||
//! settings requires knowing the answer before creating anything — which is
|
||||
//! exactly what this function is shaped for.
|
||||
|
||||
use dr_types::{CollisionPolicy, ExportFormat};
|
||||
|
||||
/// What a template can refer to.
|
||||
#[derive(Debug, Clone, Default)]
|
||||
pub struct NameContext<'a> {
|
||||
/// The source image's name, without extension — `{name}`.
|
||||
pub source_stem: &'a str,
|
||||
/// Position in the batch, 1-based — `{seq}`.
|
||||
pub sequence: u32,
|
||||
/// Capture date as `YYYY-MM-DD` — `{date}`. Empty where unknown.
|
||||
pub date: &'a str,
|
||||
/// The export's pixel dimensions — `{dimensions}`.
|
||||
pub width: u32,
|
||||
pub height: u32,
|
||||
/// The preset that produced this export — `{preset}`. Empty where none.
|
||||
pub preset: &'a str,
|
||||
}
|
||||
|
||||
/// Expand a template into a filename stem.
|
||||
///
|
||||
/// Unknown tokens are left verbatim rather than dropped. A user who typed
|
||||
/// `{nmae}` should see it in the output and understand what happened; a
|
||||
/// silently empty filename is a puzzle, and a template that quietly loses a
|
||||
/// token produces a directory of files named the same thing.
|
||||
pub fn expand(template: &str, ctx: &NameContext<'_>) -> String {
|
||||
let seq = ctx.sequence.to_string();
|
||||
let dimensions = format!("{}x{}", ctx.width, ctx.height);
|
||||
|
||||
let mut out = String::with_capacity(template.len() + 16);
|
||||
let mut rest = template;
|
||||
while let Some(open) = rest.find('{') {
|
||||
out.push_str(&rest[..open]);
|
||||
let Some(close) = rest[open..].find('}') else {
|
||||
// An unclosed brace is literal text; there is nothing to expand
|
||||
// and dropping the remainder would truncate the name. Consumed
|
||||
// here rather than left for the tail append below, which has
|
||||
// already had everything before the brace taken from it.
|
||||
out.push_str(&rest[open..]);
|
||||
rest = "";
|
||||
break;
|
||||
};
|
||||
let token = &rest[open + 1..open + close];
|
||||
match token {
|
||||
"name" => out.push_str(ctx.source_stem),
|
||||
"seq" => out.push_str(&seq),
|
||||
"date" => out.push_str(ctx.date),
|
||||
"dimensions" => out.push_str(&dimensions),
|
||||
"preset" => out.push_str(ctx.preset),
|
||||
_ => out.push_str(&rest[open..open + close + 1]),
|
||||
}
|
||||
rest = &rest[open + close + 1..];
|
||||
}
|
||||
out.push_str(rest);
|
||||
|
||||
let cleaned = sanitise(&out);
|
||||
if cleaned.is_empty() {
|
||||
// Every token was empty — a template of `{preset}` with no preset, on
|
||||
// an image with no date. Falling back to the source name is the one
|
||||
// answer that is always available and never collides more than the
|
||||
// source files themselves do.
|
||||
return sanitise(ctx.source_stem);
|
||||
}
|
||||
cleaned
|
||||
}
|
||||
|
||||
/// Strip what no filesystem, SAF provider or WebDAV server will take.
|
||||
///
|
||||
/// The intersection of three sets of rules rather than any one of them: an
|
||||
/// export written to a Nextcloud folder may later sync down to a Windows
|
||||
/// client, and a name that was legal where it was created is not much comfort
|
||||
/// on the machine that cannot open it.
|
||||
fn sanitise(stem: &str) -> String {
|
||||
let mut out: String = stem
|
||||
.chars()
|
||||
.map(|c| match c {
|
||||
'/' | '\\' | ':' | '*' | '?' | '"' | '<' | '>' | '|' => '-',
|
||||
c if (c as u32) < 0x20 => '-',
|
||||
c => c,
|
||||
})
|
||||
.collect();
|
||||
// Trailing dots and spaces are legal on Linux and rejected by Windows,
|
||||
// and a name ending in one is almost always an accident of a template
|
||||
// whose last token expanded to nothing.
|
||||
while out.ends_with('.') || out.ends_with(' ') {
|
||||
out.pop();
|
||||
}
|
||||
out.trim_start().to_string()
|
||||
}
|
||||
|
||||
/// The filename this export should be written under, honouring the collision
|
||||
/// policy.
|
||||
///
|
||||
/// `taken` answers whether a name already exists in the destination. Returns
|
||||
/// `None` for [`CollisionPolicy::Skip`] when the name is in use — the caller
|
||||
/// writes nothing and moves on, which is the whole point of that setting.
|
||||
pub fn resolve_name(
|
||||
template: &str,
|
||||
ctx: &NameContext<'_>,
|
||||
format: ExportFormat,
|
||||
collision: CollisionPolicy,
|
||||
taken: &dyn Fn(&str) -> bool,
|
||||
) -> Option<String> {
|
||||
let stem = expand(template, ctx);
|
||||
let ext = format.extension();
|
||||
let first = format!("{stem}.{ext}");
|
||||
|
||||
if !taken(&first) {
|
||||
return Some(first);
|
||||
}
|
||||
|
||||
match collision {
|
||||
CollisionPolicy::Overwrite => Some(first),
|
||||
CollisionPolicy::Skip => None,
|
||||
CollisionPolicy::Increment => {
|
||||
// Bounded. An unbounded search would spin forever against a
|
||||
// destination that reports everything as taken — a permission
|
||||
// error misread as existence, say — and a batch that hangs is
|
||||
// worse than one that reports a failure.
|
||||
for n in 1..10_000 {
|
||||
let candidate = format!("{stem}-{n}.{ext}");
|
||||
if !taken(&candidate) {
|
||||
return Some(candidate);
|
||||
}
|
||||
}
|
||||
log::warn!("{stem}: ten thousand names taken; skipping");
|
||||
None
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
fn ctx() -> NameContext<'static> {
|
||||
NameContext {
|
||||
source_stem: "IMG_1234",
|
||||
sequence: 7,
|
||||
date: "2026-08-16",
|
||||
width: 2048,
|
||||
height: 1365,
|
||||
preset: "Web",
|
||||
}
|
||||
}
|
||||
|
||||
fn free(_: &str) -> bool {
|
||||
false
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn the_default_template_is_the_source_name() {
|
||||
assert_eq!(expand("{name}", &ctx()), "IMG_1234");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn every_documented_token_expands() {
|
||||
// The settings page advertises these five in its hint; a token listed
|
||||
// there and unhandled here would reach the filename verbatim.
|
||||
assert_eq!(expand("{name}", &ctx()), "IMG_1234");
|
||||
assert_eq!(expand("{seq}", &ctx()), "7");
|
||||
assert_eq!(expand("{date}", &ctx()), "2026-08-16");
|
||||
assert_eq!(expand("{dimensions}", &ctx()), "2048x1365");
|
||||
assert_eq!(expand("{preset}", &ctx()), "Web");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn tokens_combine_with_literal_text() {
|
||||
assert_eq!(
|
||||
expand("{date}_{name}_{dimensions}", &ctx()),
|
||||
"2026-08-16_IMG_1234_2048x1365"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn an_unknown_token_survives_verbatim() {
|
||||
// A typo the user can see and fix, rather than a name that silently
|
||||
// lost a component and now collides with every other export.
|
||||
assert_eq!(expand("{nmae}-x", &ctx()), "{nmae}-x");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn an_unclosed_brace_is_literal_text() {
|
||||
assert_eq!(expand("{name", &ctx()), "{name");
|
||||
assert_eq!(expand("a{name}b{", &ctx()), "aIMG_1234b{");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_template_that_expands_to_nothing_falls_back_to_the_source_name() {
|
||||
// `{preset}` with no preset selected. An empty filename is not a file.
|
||||
let mut c = ctx();
|
||||
c.preset = "";
|
||||
assert_eq!(expand("{preset}", &c), "IMG_1234");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn path_separators_cannot_escape_the_destination() {
|
||||
// `{name}` comes from a source filename, and a template is user text.
|
||||
// Either could carry a slash, and an export must not write outside
|
||||
// the folder that was chosen — nor create a subfolder on the server.
|
||||
let mut c = ctx();
|
||||
c.source_stem = "holiday/2026";
|
||||
assert_eq!(expand("{name}", &c), "holiday-2026");
|
||||
assert_eq!(expand("../../etc/passwd", &ctx()), "..-..-etc-passwd");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn characters_windows_rejects_are_replaced() {
|
||||
// An export may sync down to a Windows client through Nextcloud, and
|
||||
// a name that was legal where it was written is no comfort there.
|
||||
assert_eq!(expand(r#"a:b*c?d"e<f>g|h\i"#, &ctx()), "a-b-c-d-e-f-g-h-i");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn trailing_dots_and_spaces_are_trimmed() {
|
||||
let mut c = ctx();
|
||||
c.preset = "";
|
||||
assert_eq!(expand("{name}.{preset}", &c), "IMG_1234");
|
||||
assert_eq!(expand("{name} ", &ctx()), "IMG_1234");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_free_name_is_used_as_is() {
|
||||
let got = resolve_name(
|
||||
"{name}",
|
||||
&ctx(),
|
||||
ExportFormat::Jpeg,
|
||||
CollisionPolicy::Increment,
|
||||
&free,
|
||||
);
|
||||
assert_eq!(got.as_deref(), Some("IMG_1234.jpg"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn the_extension_follows_the_format() {
|
||||
for (format, ext) in [
|
||||
(ExportFormat::Jpeg, "jpg"),
|
||||
(ExportFormat::Png, "png"),
|
||||
(ExportFormat::Tiff16, "tif"),
|
||||
] {
|
||||
let got = resolve_name("{name}", &ctx(), format, CollisionPolicy::Skip, &free);
|
||||
assert_eq!(got.as_deref(), Some(&*format!("IMG_1234.{ext}")));
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn increment_finds_the_first_free_suffix() {
|
||||
let taken = |n: &str| matches!(n, "IMG_1234.jpg" | "IMG_1234-1.jpg" | "IMG_1234-2.jpg");
|
||||
let got = resolve_name(
|
||||
"{name}",
|
||||
&ctx(),
|
||||
ExportFormat::Jpeg,
|
||||
CollisionPolicy::Increment,
|
||||
&taken,
|
||||
);
|
||||
assert_eq!(got.as_deref(), Some("IMG_1234-3.jpg"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn skip_returns_nothing_when_the_name_is_taken() {
|
||||
// The caller writes no file at all — that is what Skip means, and it
|
||||
// is why this returns an Option rather than always a name.
|
||||
let got = resolve_name(
|
||||
"{name}",
|
||||
&ctx(),
|
||||
ExportFormat::Jpeg,
|
||||
CollisionPolicy::Skip,
|
||||
&|_| true,
|
||||
);
|
||||
assert_eq!(got, None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn overwrite_returns_the_taken_name() {
|
||||
let got = resolve_name(
|
||||
"{name}",
|
||||
&ctx(),
|
||||
ExportFormat::Jpeg,
|
||||
CollisionPolicy::Overwrite,
|
||||
&|_| true,
|
||||
);
|
||||
assert_eq!(got.as_deref(), Some("IMG_1234.jpg"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn increment_gives_up_rather_than_spinning_forever() {
|
||||
// A destination that reports every name as taken — a permission error
|
||||
// misread as existence — must not hang the batch.
|
||||
let got = resolve_name(
|
||||
"{name}",
|
||||
&ctx(),
|
||||
ExportFormat::Jpeg,
|
||||
CollisionPolicy::Increment,
|
||||
&|_| true,
|
||||
);
|
||||
assert_eq!(got, None);
|
||||
}
|
||||
}
|
||||
@@ -1,197 +0,0 @@
|
||||
//! TRACES: FR-EXP-4
|
||||
//! Output sharpening, scaled by how far the image was resized.
|
||||
//!
|
||||
//! # Why an export needs this at all
|
||||
//!
|
||||
//! Downsampling averages neighbouring pixels, and averaging is a low-pass
|
||||
//! filter: a 24 MP frame reduced to 2048px comes out measurably softer than
|
||||
//! the same scene shot at 2048px would be. Output sharpening puts back the
|
||||
//! acuity the resample removed. It is not creative sharpening — that belongs
|
||||
//! in the develop pipeline, acts on the full-resolution image, and is a
|
||||
//! different control entirely.
|
||||
//!
|
||||
//! # Why the strength depends on the medium
|
||||
//!
|
||||
//! The three settings are not intensities dressed up as names. A screen shows
|
||||
//! a pixel as a pixel, so it needs the least. Ink spreads into paper — dot
|
||||
//! gain — and matte stock spreads it further than glossy, so a print needs
|
||||
//! more compensation to arrive looking the same. That is why the paper
|
||||
//! options are stronger, and why "more" is not simply a slider.
|
||||
|
||||
use dr_types::OutputSharpening;
|
||||
|
||||
/// Radius of the unsharp mask, in pixels.
|
||||
///
|
||||
/// Fixed at a small value rather than scaled with the image: output
|
||||
/// sharpening compensates for the *resample*, which softens over a pixel or
|
||||
/// two whatever the size of the frame. A radius that grew with the image
|
||||
/// would produce haloes on a large export.
|
||||
const RADIUS: i32 = 1;
|
||||
|
||||
/// Per-setting strength. Applied on top of the resize-derived scaling below.
|
||||
fn strength(setting: OutputSharpening) -> f32 {
|
||||
match setting {
|
||||
OutputSharpening::None => 0.0,
|
||||
OutputSharpening::Screen => 0.55,
|
||||
// Ink spread. Matte stock absorbs more than glossy, so it needs the
|
||||
// heavier hand of the two.
|
||||
OutputSharpening::GlossyPaper => 0.85,
|
||||
OutputSharpening::MattePaper => 1.15,
|
||||
}
|
||||
}
|
||||
|
||||
/// Sharpen in place-ish: takes the resized buffer and returns it, sharpened.
|
||||
///
|
||||
/// `scale` is the resize factor — destination width over source width. Below
|
||||
/// 1 the image was reduced and needs compensation; at or above 1 nothing was
|
||||
/// averaged away and the sharpening is skipped, because sharpening an image
|
||||
/// that was not softened only adds haloes.
|
||||
pub fn apply(
|
||||
mut rgba: Vec<u8>,
|
||||
width: u32,
|
||||
height: u32,
|
||||
setting: OutputSharpening,
|
||||
scale: f32,
|
||||
) -> Vec<u8> {
|
||||
let base = strength(setting);
|
||||
if base == 0.0 || scale >= 1.0 || width < 3 || height < 3 {
|
||||
return rgba;
|
||||
}
|
||||
|
||||
// A frame reduced to a tenth lost far more than one reduced to nine
|
||||
// tenths, so the compensation follows the reduction. Capped at the base
|
||||
// strength: past a point more sharpening is just edge artefacts, and a
|
||||
// thumbnail is the case where that shows most.
|
||||
let amount = base * (1.0 - scale).clamp(0.0, 1.0);
|
||||
|
||||
let src = rgba.clone();
|
||||
let (w, h) = (width as i32, height as i32);
|
||||
|
||||
for y in 0..h {
|
||||
for x in 0..w {
|
||||
for c in 0..3 {
|
||||
// A 3×3 box blur is the mask. Gaussian would be more correct
|
||||
// and, at radius 1, indistinguishable — the kernel is nine
|
||||
// pixels either way.
|
||||
let mut sum = 0.0f32;
|
||||
let mut n = 0.0f32;
|
||||
for dy in -RADIUS..=RADIUS {
|
||||
for dx in -RADIUS..=RADIUS {
|
||||
let sx = (x + dx).clamp(0, w - 1);
|
||||
let sy = (y + dy).clamp(0, h - 1);
|
||||
sum += f32::from(src[((sy * w + sx) * 4 + c) as usize]);
|
||||
n += 1.0;
|
||||
}
|
||||
}
|
||||
let blurred = sum / n;
|
||||
let p = ((y * w + x) * 4 + c) as usize;
|
||||
let original = f32::from(src[p]);
|
||||
// Unsharp mask: the original plus its difference from a
|
||||
// blurred copy, which is the high-frequency detail.
|
||||
let sharpened = original + (original - blurred) * amount;
|
||||
rgba[p] = sharpened.round().clamp(0.0, 255.0) as u8;
|
||||
}
|
||||
}
|
||||
}
|
||||
rgba
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
/// A frame split down the middle: dark left, light right. One vertical
|
||||
/// edge, which is what sharpening acts on.
|
||||
fn edge(w: u32, h: u32) -> Vec<u8> {
|
||||
let mut v = Vec::new();
|
||||
for _ in 0..h {
|
||||
for x in 0..w {
|
||||
let level = if x < w / 2 { 60 } else { 190 };
|
||||
v.extend_from_slice(&[level, level, level, 255]);
|
||||
}
|
||||
}
|
||||
v
|
||||
}
|
||||
|
||||
fn at(buf: &[u8], w: u32, x: u32, y: u32) -> u8 {
|
||||
buf[((y * w + x) * 4) as usize]
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn none_leaves_the_image_exactly_as_it_was() {
|
||||
let src = edge(16, 8);
|
||||
let out = apply(src.clone(), 16, 8, OutputSharpening::None, 0.5);
|
||||
assert_eq!(out, src);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn an_unresized_export_is_not_sharpened() {
|
||||
// Nothing was averaged away, so there is nothing to compensate for
|
||||
// and sharpening would only add haloes.
|
||||
let src = edge(16, 8);
|
||||
assert_eq!(
|
||||
apply(src.clone(), 16, 8, OutputSharpening::MattePaper, 1.0),
|
||||
src
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn sharpening_increases_contrast_across_an_edge() {
|
||||
// The property, stated directly: the dark side of the edge gets
|
||||
// darker and the light side lighter.
|
||||
let src = edge(16, 8);
|
||||
let out = apply(src.clone(), 16, 8, OutputSharpening::Screen, 0.4);
|
||||
let (before_dark, before_light) = (at(&src, 16, 7, 4), at(&src, 16, 8, 4));
|
||||
let (after_dark, after_light) = (at(&out, 16, 7, 4), at(&out, 16, 8, 4));
|
||||
assert!(after_dark < before_dark, "the dark side should deepen");
|
||||
assert!(after_light > before_light, "the light side should lift");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn paper_sharpens_harder_than_screen() {
|
||||
// Ink spreads; the settings are about the medium, not taste.
|
||||
let src = edge(16, 8);
|
||||
let screen = apply(src.clone(), 16, 8, OutputSharpening::Screen, 0.4);
|
||||
let matte = apply(src.clone(), 16, 8, OutputSharpening::MattePaper, 0.4);
|
||||
assert!(at(&matte, 16, 8, 4) > at(&screen, 16, 8, 4));
|
||||
assert!(strength(OutputSharpening::MattePaper) > strength(OutputSharpening::GlossyPaper));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_bigger_reduction_sharpens_more() {
|
||||
let src = edge(16, 8);
|
||||
let mild = apply(src.clone(), 16, 8, OutputSharpening::Screen, 0.9);
|
||||
let severe = apply(src.clone(), 16, 8, OutputSharpening::Screen, 0.1);
|
||||
assert!(at(&severe, 16, 8, 4) >= at(&mild, 16, 8, 4));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_flat_field_is_untouched() {
|
||||
// No detail means no high frequencies to amplify. If this drifts, the
|
||||
// mask is not centred and every sky gains a gradient.
|
||||
let flat = vec![128u8; 16 * 16 * 4];
|
||||
assert_eq!(
|
||||
apply(flat.clone(), 16, 16, OutputSharpening::MattePaper, 0.3),
|
||||
flat
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn alpha_is_never_touched() {
|
||||
// The loop runs over three channels for a reason: sharpening alpha
|
||||
// would put a halo in the transparency of an image that has none.
|
||||
let out = apply(edge(16, 8), 16, 8, OutputSharpening::MattePaper, 0.2);
|
||||
for px in out.chunks_exact(4) {
|
||||
assert_eq!(px[3], 255);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_frame_too_small_to_have_neighbours_is_left_alone() {
|
||||
let tiny = vec![10u8; 2 * 2 * 4];
|
||||
assert_eq!(
|
||||
apply(tiny.clone(), 2, 2, OutputSharpening::Screen, 0.5),
|
||||
tiny
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -1,520 +0,0 @@
|
||||
//! TRACES: FR-EXP-3 | FR-EXP-4
|
||||
//! Output sizing and resampling.
|
||||
//!
|
||||
//! # Why Lanczos
|
||||
//!
|
||||
//! FR-EXP-4 asks for "a quality resampler (Lanczos or better)", and the
|
||||
//! reason is what a cheap one does to a photograph. Box or bilinear
|
||||
//! downsampling of a 24 MP frame to 2048px averages away detail the sensor
|
||||
//! resolved and aliases what is left — a brick wall or a distant fence comes
|
||||
//! back as moiré. Lanczos's negative lobes preserve edge acuity through a
|
||||
//! large reduction, which is exactly the operation an export performs.
|
||||
//!
|
||||
//! Separable: a horizontal pass then a vertical one, which turns an `a²`
|
||||
//! kernel into `2a` taps per pixel. At the sizes involved that is the
|
||||
//! difference between an export that feels instant and one that does not.
|
||||
|
||||
use dr_types::SizingMode;
|
||||
|
||||
use crate::Frame;
|
||||
|
||||
/// The Lanczos window. 3 is the photographic default — 2 is softer, and
|
||||
/// beyond 3 the extra lobes buy ringing rather than detail.
|
||||
const A: f32 = 3.0;
|
||||
|
||||
/// TRACES: FR-EXP-3
|
||||
/// Resolve the requested sizing against a source, honouring the upscale rule.
|
||||
///
|
||||
/// Aspect is preserved in every mode. In all but one that means only a single
|
||||
/// dimension is ever the requested one; [`SizingMode::FillBox`] is the
|
||||
/// exception, and it keeps the aspect by *discarding* the overhang rather than
|
||||
/// by settling for a smaller box — see [`resample_filling`], which does the
|
||||
/// discarding.
|
||||
///
|
||||
/// **Upscaling is refused by clamping, never by failing.** FR-EXP-3 makes
|
||||
/// upscaling opt-in, and a batch of mixed frames must not abort because one
|
||||
/// was smaller than the target — the user asked for a set of exports, and
|
||||
/// stopping the run over a frame that came out at source size would be a
|
||||
/// worse answer than the file itself.
|
||||
pub fn target_size(
|
||||
src_w: u32,
|
||||
src_h: u32,
|
||||
sizing: SizingMode,
|
||||
allow_upscaling: bool,
|
||||
) -> (u32, u32) {
|
||||
let (src_w, src_h) = (src_w.max(1), src_h.max(1));
|
||||
|
||||
let (w, h) = match sizing {
|
||||
SizingMode::Original => (src_w, src_h),
|
||||
SizingMode::LongEdge(n) => scale_to(src_w, src_h, n, src_w >= src_h),
|
||||
SizingMode::ShortEdge(n) => scale_to(src_w, src_h, n, src_w < src_h),
|
||||
// Fit is a ceiling on both axes, so the smaller factor wins and the
|
||||
// result touches the box on one axis only.
|
||||
SizingMode::FitBox(bw, bh) => {
|
||||
scale_by(src_w, src_h, box_factor(src_w, src_h, bw, bh, f64::min))
|
||||
}
|
||||
// Fill is the box, exactly. The scale that covers it is the larger
|
||||
// factor, and the overhang is taken off in `resample_filling` — this
|
||||
// reports what the file will be, which is the whole reason the mode
|
||||
// exists.
|
||||
SizingMode::FillBox(bw, bh) => (bw.max(1), bh.max(1)),
|
||||
SizingMode::Percentage(p) => {
|
||||
let f = f64::from(p) / 100.0;
|
||||
scale_by(src_w, src_h, f)
|
||||
}
|
||||
};
|
||||
|
||||
if !allow_upscaling && (w > src_w || h > src_h) {
|
||||
// A fill box has to keep its shape even when it cannot keep its size:
|
||||
// the mode's promise is an exact aspect ratio at exact dimensions, and
|
||||
// falling back to the source's own shape would quietly export a 3:2
|
||||
// file where a 16:9 one was asked for. So the *box* is scaled down to
|
||||
// what the source can cover, rather than abandoned.
|
||||
if let SizingMode::FillBox(bw, bh) = sizing {
|
||||
let cover = box_factor(src_w, src_h, bw, bh, f64::max);
|
||||
if cover > 1.0 {
|
||||
return scale_by(bw.max(1), bh.max(1), 1.0 / cover);
|
||||
}
|
||||
}
|
||||
return (src_w, src_h);
|
||||
}
|
||||
(w.max(1), h.max(1))
|
||||
}
|
||||
|
||||
/// TRACES: FR-EXP-3
|
||||
/// The scale that puts `src` against a `bw × bh` box, `choose` deciding which
|
||||
/// axis governs: `f64::min` fits inside it, `f64::max` covers it.
|
||||
fn box_factor(src_w: u32, src_h: u32, bw: u32, bh: u32, choose: fn(f64, f64) -> f64) -> f64 {
|
||||
let fw = f64::from(bw.max(1)) / f64::from(src_w.max(1));
|
||||
let fh = f64::from(bh.max(1)) / f64::from(src_h.max(1));
|
||||
choose(fw, fh)
|
||||
}
|
||||
|
||||
/// Both axes by one factor, never rounding away to nothing.
|
||||
fn scale_by(w: u32, h: u32, factor: f64) -> (u32, u32) {
|
||||
(
|
||||
((f64::from(w) * factor).round() as u32).max(1),
|
||||
((f64::from(h) * factor).round() as u32).max(1),
|
||||
)
|
||||
}
|
||||
|
||||
/// Scale so that the chosen edge lands on `n`.
|
||||
fn scale_to(src_w: u32, src_h: u32, n: u32, width_is_the_edge: bool) -> (u32, u32) {
|
||||
let n = n.max(1);
|
||||
if width_is_the_edge {
|
||||
let h = (f64::from(n) * f64::from(src_h) / f64::from(src_w)).round() as u32;
|
||||
(n, h.max(1))
|
||||
} else {
|
||||
let w = (f64::from(n) * f64::from(src_w) / f64::from(src_h)).round() as u32;
|
||||
(w.max(1), n)
|
||||
}
|
||||
}
|
||||
|
||||
/// Resample to `(dst_w, dst_h)`, returning tightly packed RGBA8.
|
||||
///
|
||||
/// Returns the source buffer untouched where no scaling is needed, which is
|
||||
/// the `SizingMode::Original` case and therefore the common one.
|
||||
pub fn resample(frame: &Frame, dst_w: u32, dst_h: u32) -> Vec<u8> {
|
||||
if dst_w == frame.width && dst_h == frame.height {
|
||||
return frame.rgba.clone();
|
||||
}
|
||||
|
||||
// Horizontal, then vertical. The intermediate is the destination width by
|
||||
// the *source* height, so the second pass works on as little data as the
|
||||
// first can leave it.
|
||||
let horizontal = pass(
|
||||
&frame.rgba,
|
||||
frame.width,
|
||||
frame.height,
|
||||
dst_w,
|
||||
frame.height,
|
||||
true,
|
||||
);
|
||||
pass(&horizontal, dst_w, frame.height, dst_w, dst_h, false)
|
||||
}
|
||||
|
||||
/// TRACES: FR-EXP-3
|
||||
/// Resample onto exactly `(dst_w, dst_h)`, covering the box and cutting the
|
||||
/// overhang off the middle.
|
||||
///
|
||||
/// The other half of [`SizingMode::FillBox`]. [`resample`] alone would do the
|
||||
/// job by *stretching* the frame onto the box, which is the one outcome a
|
||||
/// photographer would never accept — a 3:2 photograph squeezed onto a 16:9
|
||||
/// panel is visibly wrong in a way no amount of resolution fixes.
|
||||
///
|
||||
/// So the frame is scaled until it covers the box, on whichever axis needs the
|
||||
/// most, and the surplus is taken symmetrically off the other. Centred rather
|
||||
/// than anchored: the crop tool is where a photographer decides *which* part
|
||||
/// of the frame survives, and this stage guessing differently would fight it.
|
||||
/// Locking the crop to the export's ratio leaves nothing here to cut.
|
||||
pub fn resample_filling(frame: &Frame, dst_w: u32, dst_h: u32) -> Vec<u8> {
|
||||
let (dst_w, dst_h) = (dst_w.max(1), dst_h.max(1));
|
||||
|
||||
// Rounded *up*, and floored at the destination: a cover scale that rounds
|
||||
// down leaves the box a pixel short on one axis, and the crop below would
|
||||
// then read past the end of the buffer.
|
||||
let cover = box_factor(frame.width, frame.height, dst_w, dst_h, f64::max);
|
||||
let cw = (((f64::from(frame.width) * cover).ceil()) as u32).max(dst_w);
|
||||
let ch = (((f64::from(frame.height) * cover).ceil()) as u32).max(dst_h);
|
||||
|
||||
let covered = resample(frame, cw, ch);
|
||||
if cw == dst_w && ch == dst_h {
|
||||
return covered;
|
||||
}
|
||||
|
||||
let (x0, y0) = ((cw - dst_w) / 2, (ch - dst_h) / 2);
|
||||
let mut out = vec![0u8; (dst_w as usize) * (dst_h as usize) * 4];
|
||||
for y in 0..dst_h as usize {
|
||||
let src = ((y + y0 as usize) * cw as usize + x0 as usize) * 4;
|
||||
let dst = y * dst_w as usize * 4;
|
||||
let run = dst_w as usize * 4;
|
||||
out[dst..dst + run].copy_from_slice(&covered[src..src + run]);
|
||||
}
|
||||
out
|
||||
}
|
||||
|
||||
/// One separable pass. `horizontal` picks the axis being resampled.
|
||||
fn pass(src: &[u8], src_w: u32, src_h: u32, dst_w: u32, dst_h: u32, horizontal: bool) -> Vec<u8> {
|
||||
let (src_len, dst_len) = if horizontal {
|
||||
(src_w, dst_w)
|
||||
} else {
|
||||
(src_h, dst_h)
|
||||
};
|
||||
let ratio = f64::from(src_len) / f64::from(dst_len);
|
||||
|
||||
// Enlarging samples the source at its own frequency; shrinking has to
|
||||
// widen the kernel to average the pixels being discarded, or the result
|
||||
// aliases. This is the whole difference between a resample and a
|
||||
// subsample.
|
||||
let filter_scale = ratio.max(1.0);
|
||||
let support = A as f64 * filter_scale;
|
||||
|
||||
let mut out = vec![0u8; (dst_w * dst_h * 4) as usize];
|
||||
|
||||
for i in 0..dst_len {
|
||||
// Centre of the destination sample, in source coordinates.
|
||||
let centre = (f64::from(i) + 0.5) * ratio - 0.5;
|
||||
let first = ((centre - support).ceil() as i64).max(0);
|
||||
let last = ((centre + support).floor() as i64).min(i64::from(src_len) - 1);
|
||||
|
||||
// Weights once per output row/column rather than per pixel: they
|
||||
// depend only on the axis position, and recomputing them per channel
|
||||
// was most of the cost when this was written the obvious way.
|
||||
let mut weights = Vec::with_capacity((last - first + 1).max(0) as usize);
|
||||
let mut total = 0.0f64;
|
||||
for s in first..=last {
|
||||
let w = lanczos((f64::from(s as i32) - centre) / filter_scale);
|
||||
weights.push(w);
|
||||
total += w;
|
||||
}
|
||||
if total == 0.0 {
|
||||
total = 1.0;
|
||||
}
|
||||
|
||||
let other = if horizontal { dst_h } else { dst_w };
|
||||
for j in 0..other {
|
||||
let mut acc = [0.0f64; 4];
|
||||
for (k, w) in weights.iter().enumerate() {
|
||||
let s = first as u32 + k as u32;
|
||||
let (x, y) = if horizontal { (s, j) } else { (j, s) };
|
||||
let p = ((y * src_w + x) * 4) as usize;
|
||||
for c in 0..4 {
|
||||
acc[c] += f64::from(src[p + c]) * w;
|
||||
}
|
||||
}
|
||||
let (x, y) = if horizontal { (i, j) } else { (j, i) };
|
||||
let p = ((y * dst_w + x) * 4) as usize;
|
||||
for c in 0..4 {
|
||||
// Lanczos overshoots at edges — that is what makes it look
|
||||
// sharp — so the result must be clamped rather than wrapped.
|
||||
out[p + c] = (acc[c] / total).round().clamp(0.0, 255.0) as u8;
|
||||
}
|
||||
}
|
||||
}
|
||||
out
|
||||
}
|
||||
|
||||
/// The Lanczos kernel, `sinc(x) * sinc(x / a)`.
|
||||
fn lanczos(x: f64) -> f64 {
|
||||
let x = x.abs();
|
||||
if x < 1e-9 {
|
||||
return 1.0;
|
||||
}
|
||||
if x >= f64::from(A) {
|
||||
return 0.0;
|
||||
}
|
||||
let px = std::f64::consts::PI * x;
|
||||
(px.sin() / px) * ((px / f64::from(A)).sin() / (px / f64::from(A)))
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use crate::tests::frame;
|
||||
|
||||
#[test]
|
||||
fn original_is_the_source_size() {
|
||||
assert_eq!(
|
||||
target_size(6000, 4000, SizingMode::Original, false),
|
||||
(6000, 4000)
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn long_edge_picks_the_longer_dimension_either_way_round() {
|
||||
assert_eq!(
|
||||
target_size(6000, 4000, SizingMode::LongEdge(3000), false),
|
||||
(3000, 2000)
|
||||
);
|
||||
// Portrait: the long edge is now the height.
|
||||
assert_eq!(
|
||||
target_size(4000, 6000, SizingMode::LongEdge(3000), false),
|
||||
(2000, 3000)
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn short_edge_picks_the_shorter_dimension_either_way_round() {
|
||||
assert_eq!(
|
||||
target_size(6000, 4000, SizingMode::ShortEdge(2000), false),
|
||||
(3000, 2000)
|
||||
);
|
||||
assert_eq!(
|
||||
target_size(4000, 6000, SizingMode::ShortEdge(2000), false),
|
||||
(2000, 3000)
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_percentage_scales_both_dimensions() {
|
||||
assert_eq!(
|
||||
target_size(4000, 3000, SizingMode::Percentage(50), false),
|
||||
(2000, 1500)
|
||||
);
|
||||
assert_eq!(
|
||||
target_size(4000, 3000, SizingMode::Percentage(100), false),
|
||||
(4000, 3000)
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_fit_box_stays_inside_the_box_and_keeps_its_shape() {
|
||||
// Fit is a ceiling on both axes, so a 3:2 frame in a 16:9 box comes
|
||||
// back short of the box's width, never past its height.
|
||||
assert_eq!(
|
||||
target_size(6000, 4000, SizingMode::FitBox(3840, 2160), false),
|
||||
(3240, 2160)
|
||||
);
|
||||
// Portrait into the same box: now the height governs nothing and the
|
||||
// width does.
|
||||
assert_eq!(
|
||||
target_size(4000, 6000, SizingMode::FitBox(3840, 2160), false),
|
||||
(1440, 2160)
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_fill_box_is_the_box_exactly() {
|
||||
// The whole point of the mode. A display that accepts one resolution
|
||||
// and rejects everything else has to get that resolution whatever the
|
||||
// photograph's own shape is.
|
||||
for (w, h) in [(6000u32, 4000u32), (4000, 6000), (5000, 5000)] {
|
||||
assert_eq!(
|
||||
target_size(w, h, SizingMode::FillBox(3840, 2160), false),
|
||||
(3840, 2160),
|
||||
"{w}x{h} did not fill the box"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_fill_box_too_large_for_the_source_keeps_its_shape_not_the_sources() {
|
||||
// Upscaling off, and the source cannot cover 4K. Falling back to the
|
||||
// source's own size would export a 3:2 file where 16:9 was asked
|
||||
// for — silently wrong in exactly the way the mode exists to prevent.
|
||||
// The box shrinks instead.
|
||||
let (w, h) = target_size(1600, 1200, SizingMode::FillBox(3840, 2160), false);
|
||||
assert!(w <= 1600 && h <= 1200, "upscaled to {w}x{h}");
|
||||
let want = 3840.0 / 2160.0;
|
||||
assert!(
|
||||
((w as f64 / h as f64) / want - 1.0).abs() < 0.01,
|
||||
"{w}x{h} is not the box's shape"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_fill_box_within_the_source_is_honoured_with_upscaling_off() {
|
||||
// The ordinary case: a 24 MP frame has pixels to spare for a 4K panel,
|
||||
// so nothing is being enlarged and the clamp must not fire.
|
||||
assert_eq!(
|
||||
target_size(6000, 4000, SizingMode::FillBox(3840, 2160), false),
|
||||
(3840, 2160)
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_filled_frame_comes_back_at_exactly_the_box() {
|
||||
let f = frame(128, 64);
|
||||
// Wider than the source's 2:1, so the crop comes off the width.
|
||||
assert_eq!(resample_filling(&f, 40, 40).len(), 40 * 40 * 4);
|
||||
assert_eq!(resample_filling(&f, 100, 25).len(), 100 * 25 * 4);
|
||||
// Already the box: no work, and no drift.
|
||||
assert_eq!(resample_filling(&f, 128, 64), f.rgba);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_fill_crops_rather_than_stretching() {
|
||||
// The property that separates fill from handing the box straight to
|
||||
// `resample`. The test frame ramps red left to right, so a 2:1 source
|
||||
// squeezed into a square would compress that ramp into the full
|
||||
// width — where a centre crop keeps its middle, and therefore starts
|
||||
// and ends well inside the source's own range.
|
||||
let f = frame(128, 128);
|
||||
let square = resample(&f, 64, 64);
|
||||
let filled = resample_filling(&f, 32, 64);
|
||||
|
||||
let left = |b: &[u8]| b[0];
|
||||
let right = |b: &[u8], w: usize| b[(w - 1) * 4];
|
||||
|
||||
assert!(
|
||||
left(&filled) > left(&square),
|
||||
"a centre crop must start further into the ramp"
|
||||
);
|
||||
assert!(
|
||||
right(&filled, 32) < right(&square, 64),
|
||||
"a centre crop must end further from the ramp's end"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_fill_takes_the_overhang_evenly_off_both_sides() {
|
||||
// Centred, not anchored: the crop tool is where a photographer decides
|
||||
// which part of the frame survives, and this stage must not have an
|
||||
// opinion of its own.
|
||||
let f = frame(128, 128);
|
||||
let filled = resample_filling(&f, 32, 64);
|
||||
let px = |x: usize| filled[x * 4];
|
||||
// The ramp is horizontal, so a centred crop is symmetric about the
|
||||
// frame's own midpoint: the two ends should sit equally far from it.
|
||||
let mid = i32::from(resample(&f, 128, 128)[64 * 4]);
|
||||
let lo = i32::from(px(0));
|
||||
let hi = i32::from(px(31));
|
||||
assert!(
|
||||
((mid - lo) - (hi - mid)).abs() < 8,
|
||||
"not centred: {lo} .. {mid} .. {hi}"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_flat_field_survives_a_fill_unchanged() {
|
||||
// Same guard as the fit path: any deviation means the cover scale and
|
||||
// the crop disagree about where the pixels are.
|
||||
let flat = Frame::new(64, 48, vec![200; 64 * 48 * 4]).unwrap();
|
||||
for byte in resample_filling(&flat, 30, 30) {
|
||||
assert_eq!(byte, 200);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn upscaling_is_refused_by_clamping_rather_than_failing() {
|
||||
// FR-EXP-3: opt-in, and a batch must not abort over one small frame.
|
||||
assert_eq!(
|
||||
target_size(800, 600, SizingMode::LongEdge(4000), false),
|
||||
(800, 600)
|
||||
);
|
||||
assert_eq!(
|
||||
target_size(800, 600, SizingMode::Percentage(400), false),
|
||||
(800, 600)
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn upscaling_is_honoured_when_asked_for() {
|
||||
assert_eq!(
|
||||
target_size(800, 600, SizingMode::LongEdge(1600), true),
|
||||
(1600, 1200)
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_square_frame_treats_either_edge_as_the_long_one() {
|
||||
// The tie has to resolve somewhere, and both answers are the same
|
||||
// size — but it must not produce a zero or a panic.
|
||||
assert_eq!(
|
||||
target_size(1000, 1000, SizingMode::LongEdge(500), false),
|
||||
(500, 500)
|
||||
);
|
||||
assert_eq!(
|
||||
target_size(1000, 1000, SizingMode::ShortEdge(500), false),
|
||||
(500, 500)
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_size_can_never_round_down_to_nothing() {
|
||||
// A 1% export of a small frame rounds toward zero, and a zero-pixel
|
||||
// image is not a file anyone can open.
|
||||
let (w, h) = target_size(50, 30, SizingMode::Percentage(1), false);
|
||||
assert!(w >= 1 && h >= 1, "got {w}x{h}");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn resampling_to_the_same_size_changes_nothing() {
|
||||
// The `Original` path, which is the common one — it must not spend a
|
||||
// Lanczos pass to return what it was given.
|
||||
let f = frame(32, 24);
|
||||
assert_eq!(resample(&f, 32, 24), f.rgba);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_resample_produces_the_right_number_of_pixels() {
|
||||
let f = frame(64, 48);
|
||||
assert_eq!(resample(&f, 32, 24).len(), 32 * 24 * 4);
|
||||
assert_eq!(resample(&f, 100, 75).len(), 100 * 75 * 4);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_downscale_preserves_the_gradient_it_was_given() {
|
||||
// The check that separates a real resample from a buffer of the right
|
||||
// length: the test frame ramps red left-to-right, so the output must
|
||||
// too, and its corners must still be near the source's.
|
||||
let f = frame(128, 128);
|
||||
let small = resample(&f, 32, 32);
|
||||
let px = |x: usize, y: usize| small[(y * 32 + x) * 4];
|
||||
assert!(px(0, 0) < px(16, 0), "red should rise across the frame");
|
||||
assert!(px(16, 0) < px(31, 0));
|
||||
// Row-invariant in red, since the ramp is horizontal.
|
||||
assert!((i32::from(px(16, 0)) - i32::from(px(16, 31))).abs() < 8);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_flat_field_survives_a_resample_unchanged() {
|
||||
// Lanczos rings on edges, which is intended — but a constant field
|
||||
// has no edges, and any deviation here means the weights do not sum
|
||||
// to one. That error is invisible on a photograph and glaring on a
|
||||
// sky.
|
||||
let flat = Frame::new(64, 64, vec![200; 64 * 64 * 4]).unwrap();
|
||||
for byte in resample(&flat, 21, 21) {
|
||||
assert_eq!(byte, 200, "a constant field must resample to itself");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn an_upscale_also_holds_a_flat_field() {
|
||||
let flat = Frame::new(16, 16, vec![64; 16 * 16 * 4]).unwrap();
|
||||
for byte in resample(&flat, 40, 40) {
|
||||
assert_eq!(byte, 64);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn the_kernel_is_one_at_the_centre_and_zero_past_its_window() {
|
||||
assert!((lanczos(0.0) - 1.0).abs() < 1e-9);
|
||||
assert_eq!(lanczos(3.0), 0.0);
|
||||
assert_eq!(lanczos(4.5), 0.0);
|
||||
// Zero at the integers inside the window, which is what makes an
|
||||
// unscaled resample an identity.
|
||||
assert!(lanczos(1.0).abs() < 1e-9);
|
||||
assert!(lanczos(2.0).abs() < 1e-9);
|
||||
}
|
||||
}
|
||||
@@ -1,52 +0,0 @@
|
||||
[package]
|
||||
name = "dr-face"
|
||||
version.workspace = true
|
||||
edition.workspace = true
|
||||
rust-version.workspace = true
|
||||
license.workspace = true
|
||||
|
||||
[dependencies]
|
||||
thiserror.workspace = true
|
||||
log.workspace = true
|
||||
|
||||
# Inference. `ort` is the API; **what runs it is `dr-inference-engine`'s
|
||||
# business** — tract, or an ONNX Runtime the app found on disk, on whichever
|
||||
# provider the device has (docs/inference.md). This crate never names either.
|
||||
ort = { workspace = true, optional = true }
|
||||
dr-inference-engine = { workspace = true, optional = true }
|
||||
ndarray = { workspace = true, optional = true }
|
||||
|
||||
[dev-dependencies]
|
||||
zune-jpeg.workspace = true
|
||||
env_logger.workspace = true
|
||||
# The M1 probe drives `ort` directly so it can print the raw load error.
|
||||
ort = { workspace = true }
|
||||
dr-inference-engine = { workspace = true }
|
||||
|
||||
[[example]]
|
||||
name = "probe"
|
||||
required-features = ["inference"]
|
||||
|
||||
[[example]]
|
||||
name = "faces"
|
||||
required-features = ["inference"]
|
||||
|
||||
[[example]]
|
||||
name = "eyes"
|
||||
required-features = ["inference"]
|
||||
|
||||
[features]
|
||||
# Nothing on by default, and in particular **no `embedded-model`**: the weights
|
||||
# are not a build input and never become one (docs/faces.md §2.2). A feature
|
||||
# flag that *could* embed them is a flag someone eventually sets in a packaging
|
||||
# script, and the InsightFace grant does not survive that.
|
||||
default = []
|
||||
|
||||
# The ONNX runtime, and the two stages that need it.
|
||||
#
|
||||
# Separable because the accuracy of this subsystem lives in `calibrate` and
|
||||
# `cluster`, which are arithmetic over embeddings with no model in them. They
|
||||
# must be testable against synthetic embeddings on a machine with no weights on
|
||||
# it — a test suite that needs a research-licensed download is a test suite
|
||||
# that does not run in CI.
|
||||
inference = ["dep:ort", "dep:dr-inference-engine", "dep:ndarray"]
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user