Add folder scan with format selection; validate A3 on a real library

Library setup as the user described it: pick a folder, choose which RAW
types to look for, scan recursively.

  dr-types::FormatFilter  the tick-box selection, seeing through VFS
                          placeholder suffixes so a dehydrated CR2 still
                          matches as a CR2
  dr-sync::scan           recursive walk, Depth:1 per directory, pruning
                          unchanged subtrees where the backend propagates
                          directory ETags

Verified against nextcloud.tourolle.paris (34.0.2) on a real library:

  browse root      32 entries, 98ms
  scan PhotosRaw   17,185 RAW files in 334 directories, 34.1s
                   (7,836 CR2 + 9,349 DNG)
  range read       262KB of a 21.5MB DNG in 119ms — 1.22% of the file,
                   and enough to read "Canon EOS 6D | ISO 100"

That last line is assumption A3 validated on real data. Cataloguing this
library by whole-file fetch would move roughly 370GB; the range path
moves a few MB.

Pruning is capability-gated rather than assumed: with per-entry ETags a
probe costs a request and proves nothing about children, so it is skipped
entirely. A test asserts zero probes in that case.

Still unresolved: /core/preview returns 400 for every parameter
combination tried, including on a JPEG the server reports as having a
preview. Not a request-shape bug — it fails identically bare. Recorded
rather than worked around; ARCH §6.7 already treats server previews as
opportunistic, so nothing depends on it.
This commit is contained in:
2026-08-09 12:22:31 +02:00
parent fbadf9afc8
commit c8bb08e661
29 changed files with 7193 additions and 234 deletions
+166 -12
View File
@@ -23,6 +23,7 @@
use std::fmt::Write as _;
use crate::descriptor::{OpDescriptor, ParamId};
use crate::framing::{Framing, FRAMING_UNIFORM_FIELDS};
/// What an operation's parameters affect, for cache invalidation scoping.
///
@@ -131,7 +132,19 @@ const BASE_UNIFORM_FIELDS: usize = 16;
///
/// Inactive operations are skipped entirely — they contribute no code, no
/// uniforms, and nothing to the structure hash.
///
/// Equivalent to [`compose_with_framing`] with neutral framing.
pub fn compose(ops: &[Box<dyn Operation>]) -> ComposedShader {
compose_with_framing(ops, &Framing::new())
}
/// Compose operations and framing into a single compute shader.
///
/// Framing generates the shader's **prologue** — the map from an output pixel
/// back to a source position — where [`compose`] would emit a fixed identity
/// scale. The fused-dispatch property is unaffected: a cropped, straightened
/// edit with three adjustments is still one dispatch, one read, one write.
pub fn compose_with_framing(ops: &[Box<dyn Operation>], framing: &Framing) -> ComposedShader {
let active: Vec<&dyn Operation> = ops
.iter()
.map(|o| o.as_ref())
@@ -157,6 +170,18 @@ pub fn compose(ops: &[Box<dyn Operation>]) -> ComposedShader {
);
uniform_values.resize(BASE_UNIFORM_FIELDS, 0.0);
// Framing's block follows the base one at a fixed offset, for the same
// reason: the prologue is emitted whether or not any operation is active,
// so these slots cannot be positioned by the op loop below.
uniform_fields.push_str(
" // Framing: the crop rect (origin, extent) and the straightening\n\
\x20 // angle as sin/cos — a trig call per pixel would recompute a\n\
\x20 // value that is constant across the dispatch.\n\
\x20 crop_rect: vec4<f32>,\n\
\x20 framing_angle: vec4<f32>,\n",
);
uniform_values.extend_from_slice(&framing.uniforms());
for op in &active {
let id = op.descriptor().id.0;
let prefix = sanitise(id);
@@ -206,6 +231,19 @@ pub fn compose(ops: &[Box<dyn Operation>]) -> ComposedShader {
let _ = writeln!(helper_src, "{}\n", h.source.trim_end());
}
// The coordinate stage: output pixel -> source position -> colour. Emitted
// ahead of the operation fragments, which receive the sampled `c`.
let prologue = format!(
"{}\n{}",
framing.wgsl_prologue(),
sample_source(framing.needs_interpolation())
);
let sampler_helper = if framing.needs_interpolation() {
BILINEAR_HELPER
} else {
""
};
let source = format!(
"// GENERATED — do not edit.
//
@@ -221,7 +259,7 @@ struct Params {{
@group(0) @binding(1) var<uniform> u: Params;
@group(0) @binding(2) var output: texture_storage_2d<rgba8unorm, write>;
{helper_src}// Linear sRGB to the display transfer function.
{sampler_helper}{helper_src}// Linear sRGB to the display transfer function.
//
// The one place quantisation happens: everything above runs in linear f16,
// and this is the final encode (ARCH §5.2).
@@ -238,14 +276,7 @@ fn main(@builtin(global_invocation_id) gid: vec3<u32>) {{
return;
}}
// Source is the demosaiced image: linear, scene-referred, camera space.
let src_dims = textureDimensions(source);
let coord = vec2<i32>(
i32(gid.x * src_dims.x / dims.x),
i32(gid.y * src_dims.y / dims.y),
);
var c = textureLoad(source, coord, 0).rgb;
{prologue}
// As-shot white balance. Applied unconditionally, before any operation,
// because it is part of *interpreting* the sensor rather than an edit: a
// Bayer sensor's green photosites collect far more signal than its red
@@ -271,7 +302,10 @@ fn main(@builtin(global_invocation_id) gid: vec3<u32>) {{
active.len()
);
let structure_hash = hash_structure(&active);
// Framing enters the hash by structure only — which branches its prologue
// generated, never how far a slider moved. Dragging the crop handles must
// reuse the compiled pipeline and re-upload uniforms.
let structure_hash = mix(hash_structure(&active), framing.structure_key());
ComposedShader {
source,
@@ -280,6 +314,83 @@ fn main(@builtin(global_invocation_id) gid: vec3<u32>) {{
}
}
/// The WGSL turning the framed source position `p` into the colour `c`.
///
/// Split out because it is the join between the coordinate stage and the
/// colour stage, and because the choice it makes — an exact integer load, or
/// a filtered sample — is the one thing the free-angle case changes.
fn sample_source(interpolate: bool) -> &'static str {
if interpolate {
" // Back to texture coordinates.
let uv_src = p / aspect + vec2<f32>(0.5);
// Outside the source there is no pixel. A straightened frame exposes its
// corners; render them black rather than clamping, which would smear an
// edge pixel across them.
if (any(uv_src < vec2<f32>(0.0)) || any(uv_src >= vec2<f32>(1.0))) {
textureStore(output, vec2<i32>(gid.xy), vec4<f32>(0.0, 0.0, 0.0, 1.0));
return;
}
// A free angle puts output pixels between source pixels. Nearest-neighbour
// here is what makes a straightened horizon stair-step, so interpolate.
var c = sample_bilinear(uv_src, src_dims);
"
} else {
" // Back to texture coordinates.
let uv_src = p / aspect + vec2<f32>(0.5);
// Outside the source there is no pixel — possible once the frame has been
// transformed at all. Render it black rather than clamping, which would
// smear an edge pixel across the gap.
if (any(uv_src < vec2<f32>(0.0)) || any(uv_src >= vec2<f32>(1.0))) {
textureStore(output, vec2<i32>(gid.xy), vec4<f32>(0.0, 0.0, 0.0, 1.0));
return;
}
// Every output pixel lands on a source pixel, so load it directly: exact,
// and with no interpolation to soften detail.
let coord = min(vec2<i32>(uv_src * vec2<f32>(src_dims)), vec2<i32>(src_dims) - vec2<i32>(1));
var c = textureLoad(source, coord, 0).rgb;
"
}
}
/// Bilinear sampling against an unfiltered `texture_2d`.
///
/// Hand-rolled rather than done with a sampler: the source is bound as a plain
/// texture, and adding a sampler for the straightening case alone would change
/// a bind group layout that every pass shares.
const BILINEAR_HELPER: &str = "fn sample_bilinear(uv: vec2<f32>, dims: vec2<u32>) -> vec3<f32> {
let last = vec2<i32>(dims) - vec2<i32>(1);
// Half-texel offset: sample positions are texel *centres*. Without it the
// image shifts by half a pixel and every rotation comes out slightly soft.
let q = uv * vec2<f32>(dims) - vec2<f32>(0.5);
let base = floor(q);
let f = q - base;
let i0 = clamp(vec2<i32>(base), vec2<i32>(0), last);
let i1 = min(i0 + vec2<i32>(1), last);
let c00 = textureLoad(source, vec2<i32>(i0.x, i0.y), 0).rgb;
let c10 = textureLoad(source, vec2<i32>(i1.x, i0.y), 0).rgb;
let c01 = textureLoad(source, vec2<i32>(i0.x, i1.y), 0).rgb;
let c11 = textureLoad(source, vec2<i32>(i1.x, i1.y), 0).rgb;
return mix(mix(c00, c10, f.x), mix(c01, c11, f.x), f.y);
}
";
/// Fold a value into a hash. FNV-1a's mixing step, over eight bytes.
fn mix(mut h: u64, value: u64) -> u64 {
for byte in value.to_le_bytes() {
h ^= u64::from(byte);
h = h.wrapping_mul(0x100_0000_01b3);
}
h
}
/// Hash the op-set and order — the structure, not the values.
///
/// Two edits with the same operations at different slider positions produce
@@ -305,13 +416,29 @@ fn hash_structure(active: &[&dyn Operation]) -> u64 {
///
/// Whole-word matching matters: an operation with uniforms `amount` and
/// `amount_hi` must not have the first rewrite corrupt the second.
fn rewrite_uniform(src: &str, name: &str, replacement: &str) -> String {
///
/// Shared with [`crate::warp`], which prefixes its uniforms by the same rule
/// and must not diverge from it.
/// Comments are skipped. A fragment explaining what `factor` does should not
/// have its prose rewritten to `u.saturation_factor` — the generated source
/// is meant to be read when a shader fails to compile, and mangled comments
/// make that harder rather than easier.
pub(crate) fn rewrite_uniform(src: &str, name: &str, replacement: &str) -> String {
let mut out = String::with_capacity(src.len());
let bytes = src.as_bytes();
let mut i = 0;
// Tracks whether the cursor sits inside a `//` comment. WGSL fragments
// use line comments only, so this needs no block-comment handling.
let mut in_comment = false;
while i < src.len() {
if src[i..].starts_with(name) {
if bytes[i] == b'\n' {
in_comment = false;
} else if !in_comment && src[i..].starts_with("//") {
in_comment = true;
}
if !in_comment && src[i..].starts_with(name) {
let before_ok = i == 0 || !is_ident_byte(bytes[i - 1]);
let after = i + name.len();
let after_ok = after >= src.len() || !is_ident_byte(bytes[after]);
@@ -511,6 +638,33 @@ mod tests {
assert_eq!(got, "x = u.p_amount + amount_hi;");
}
#[test]
fn rewriting_leaves_comments_alone() {
// Found in generated source: a comment reading "A factor of 0 is
// monochrome" came out as "A u.saturation_factor of 0 is monochrome".
// The generated source is what gets read when a shader fails to
// compile, so mangling it works against the one time it matters.
let got = rewrite_uniform(
"// A factor of 0 is monochrome\nc = c * factor;",
"factor",
"u.op_factor",
);
assert_eq!(got, "// A factor of 0 is monochrome\nc = c * u.op_factor;");
}
#[test]
fn rewriting_resumes_after_a_comment_ends() {
let got = rewrite_uniform(
"// factor here is prose\nlet x = factor;\n// factor again\n",
"factor",
"u.p",
);
assert_eq!(
got,
"// factor here is prose\nlet x = u.p;\n// factor again\n"
);
}
#[test]
fn rewriting_leaves_substrings_alone() {
let got = rewrite_uniform("total_amount = 1.0;", "amount", "u.a");