Add dr-thumbs: a sharded, syncable thumbnail store

A thumbnail is the one derived artefact worth sending over the wire: it costs
a range fetch plus a decode to produce and is identical for every client
looking at the same file. A second device that downloads a shard gets a full
grid without fetching a byte of RAW.

Sharded at 25 MB, filled sequentially. The cap is about sync granularity, not
SQLite's limits — one growing database means every client re-downloads it
whenever a single thumbnail is added, whereas with sequential fill only the
newest shard is ever dirty and sealed shards are safe to cache forever.

Stored JPEG-encoded rather than as raw RGBA: a 256px RGBA buffer is ~256 KB
against ~20 KB encoded, and that 13x is transfer cost on every client.

Keyed on Nextcloud's oc:fileid, stable across server-side rename and move.

Assisted-by: LLM
This commit is contained in:
2026-08-09 20:38:12 +02:00
parent b6a5c0f090
commit d6ddd0703b
6 changed files with 1011 additions and 0 deletions
+163
View File
@@ -0,0 +1,163 @@
//! TRACES: FR-CAT-3 | NFR-RES-4
//! Encoding and decoding stored thumbnails.
//!
//! Thumbnails are stored as JPEG rather than raw pixels. A 256×170 RGBA buffer
//! is ~174 KB; the same image as JPEG is ~15 KB. That ratio decides whether a
//! 25 MB shard holds a few hundred thumbnails or a few thousand — and since
//! shards sync to Nextcloud, it is transfer cost paid by every client, not
//! just disk.
use crate::error::ThumbError;
/// JPEG quality for stored thumbnails.
///
/// 82 is the knee: visible artefacts start below it, and above it size climbs
/// with no benefit at 256px. These are browsing aids — a culling decision is
/// made against raw data (FR-CULL-3), never against one of these.
const QUALITY: u8 = 82;
/// Encode RGBA pixels to JPEG.
///
/// Alpha is discarded: a camera preview has none, and JPEG cannot carry it.
pub fn encode_rgba(width: u32, height: u32, rgba: &[u8]) -> Result<Vec<u8>, ThumbError> {
let expected = (width as usize)
.checked_mul(height as usize)
.and_then(|n| n.checked_mul(4))
.ok_or_else(|| ThumbError::Io("thumbnail dimensions overflow".into()))?;
if rgba.len() < expected {
return Err(ThumbError::Io(format!(
"thumbnail buffer is {} bytes, expected {expected}",
rgba.len()
)));
}
let mut out = Vec::new();
let encoder = jpeg_encoder::Encoder::new(&mut out, QUALITY);
encoder
.encode(
&rgba[..expected],
width as u16,
height as u16,
jpeg_encoder::ColorType::Rgba,
)
.map_err(|e| ThumbError::Io(e.to_string()))?;
Ok(out)
}
/// Decode a stored JPEG back to RGBA for display.
pub fn decode_rgba(bytes: &[u8]) -> Result<(u32, u32, Vec<u8>), ThumbError> {
let mut d = zune_jpeg::JpegDecoder::new(bytes);
let pixels = d
.decode()
.map_err(|e| ThumbError::Io(format!("decoding stored thumbnail: {e}")))?;
let info = d
.info()
.ok_or_else(|| ThumbError::Io("stored thumbnail has no header".into()))?;
let (w, h) = (info.width as u32, info.height as u32);
let px = (w as usize) * (h as usize);
// zune yields RGB for a colour JPEG and one channel for a greyscale one.
// A greyscale camera preview is rare but real, and treating it as RGB
// would produce a garbled cell rather than a grey one.
let rgba = match pixels.len() / px.max(1) {
3 => {
let mut out = Vec::with_capacity(px * 4);
for c in pixels.chunks_exact(3) {
out.extend_from_slice(&[c[0], c[1], c[2], 255]);
}
out
}
1 => {
let mut out = Vec::with_capacity(px * 4);
for g in &pixels {
out.extend_from_slice(&[*g, *g, *g, 255]);
}
out
}
_ => {
return Err(ThumbError::Io(format!(
"unexpected {} bytes for {w}×{h}",
pixels.len()
)))
}
};
Ok((w, h, rgba))
}
#[cfg(test)]
mod tests {
use super::*;
fn gradient(w: u32, h: u32) -> Vec<u8> {
let mut v = Vec::with_capacity((w * h * 4) as usize);
for y in 0..h {
for x in 0..w {
v.extend_from_slice(&[(x * 4) as u8, (y * 4) as u8, 128, 255]);
}
}
v
}
#[test]
fn encoding_then_decoding_preserves_dimensions() {
let bytes = encode_rgba(64, 48, &gradient(64, 48)).unwrap();
let (w, h, rgba) = decode_rgba(&bytes).unwrap();
assert_eq!((w, h), (64, 48));
assert_eq!(rgba.len(), 64 * 48 * 4);
}
#[test]
fn encoding_is_far_smaller_than_raw() {
// The entire reason for encoding: shards sync, so this ratio is
// transfer cost on every client.
let raw = gradient(256, 170);
let encoded = encode_rgba(256, 170, &raw).unwrap();
assert!(
encoded.len() * 5 < raw.len(),
"expected a large saving, got {} vs {}",
encoded.len(),
raw.len()
);
}
#[test]
fn decoded_pixels_are_opaque() {
let bytes = encode_rgba(16, 16, &gradient(16, 16)).unwrap();
let (_, _, rgba) = decode_rgba(&bytes).unwrap();
assert!(rgba.chunks_exact(4).all(|p| p[3] == 255));
}
#[test]
fn a_short_buffer_errors_rather_than_reading_past_it() {
// Decoders handle untrusted input (NFR-SEC-1); a truncated preview
// must not become an out-of-bounds read.
assert!(encode_rgba(64, 64, &[0u8; 16]).is_err());
}
#[test]
fn absurd_dimensions_do_not_overflow() {
assert!(encode_rgba(u32::MAX, u32::MAX, &[0u8; 4]).is_err());
}
#[test]
fn corrupt_stored_bytes_error_rather_than_panic() {
assert!(decode_rgba(&[0xFF, 0xD8, 0xFF, 0x00, 0x01]).is_err());
assert!(decode_rgba(&[]).is_err());
}
#[test]
fn roughly_round_trips_the_image() {
// Lossy, so exact equality is wrong to assert — but a mid-grey pixel
// must not come back as black or white.
let w = 32;
let src: Vec<u8> = std::iter::repeat_n([128u8, 128, 128, 255], (w * w) as usize)
.flatten()
.collect();
let (_, _, out) = decode_rgba(&encode_rgba(w, w, &src).unwrap()).unwrap();
assert!(out[0].abs_diff(128) < 12, "got {}", out[0]);
}
}