wip: ingest

This commit is contained in:
2026-08-22 14:12:40 +02:00
parent 98e0ad0537
commit 735683b849
10 changed files with 1267 additions and 11 deletions
Generated
+1
View File
@@ -1578,6 +1578,7 @@ dependencies = [
"dr-decode", "dr-decode",
"dr-export", "dr-export",
"dr-gpu", "dr-gpu",
"dr-ingest",
"dr-pipeline", "dr-pipeline",
"dr-plat", "dr-plat",
"dr-segment", "dr-segment",
+308
View File
@@ -0,0 +1,308 @@
//! TRACES: FR-CAT-11
//! Has this photograph been imported before?
//!
//! Two tiers, because neither alone is enough and they cost very different
//! amounts. The metadata tier — capture time, camera, size, the name the
//! camera gave it — is answerable from the catalog before a byte leaves the
//! card, which is what makes re-inserting an already-imported card cost a
//! metadata read per file rather than a full transfer. The content tier
//! catches what the first misses: the same frame arriving under a different
//! name, from a second card, or after somebody renamed it.
//!
//! # Why the filename is compared here rather than in SQL
//!
//! `images.source_ref` holds the whole opaque key — a relative path on Linux,
//! a document id on SAF — and the camera's filename is only its last
//! component. Matching that in SQL means `LIKE '%/IMG_0001.CR3'`, which cannot
//! use an index, scans the whole table, and is wrong on SAF where the
//! separator is not `/`. So the query narrows on the indexed columns and the
//! handful of rows that survive are compared in Rust, the same way the grid
//! already derives a display name.
//!
//! # Filename alone is never sufficient
//!
//! Camera filenames wrap at `IMG_9999` and start again, so a library of any
//! age holds several unrelated `IMG_0001.CR3`. That is why the cheap tier
//! carries capture time and camera as well, and why the expensive tier exists
//! at all.
use rusqlite::Connection;
use crate::CatalogError;
/// The last component of a stored source reference.
///
/// Splits on both separators for the same reason `Catalog::window` does: the
/// key's shape belongs to the storage that produced it, and a SAF document id
/// is delimited with `:`.
fn file_name(source_ref: &str) -> &str {
source_ref.rsplit(['/', ':']).next().unwrap_or(source_ref)
}
/// Whether the catalog already holds this photograph, on metadata alone.
///
/// `camera` is the joined make-and-model string the scan stores, not the raw
/// EXIF pair — the caller composes it the same way, or the comparison is
/// always false.
///
/// A `captured_at` of `None` makes this answer `false` rather than matching
/// every undated image in the library: without a capture time the key is
/// filename plus size, which two frames from the same body collide on
/// routinely. An undated file falls through to the content tier, which is
/// slower and right.
pub fn seen_by_metadata(
conn: &Connection,
captured_at: Option<i64>,
camera: Option<&str>,
size: u64,
original_name: &str,
) -> Result<bool, CatalogError> {
let Some(captured_at) = captured_at else {
return Ok(false);
};
// `images_captured` indexes the capture time, so this reads a few rows
// even in a library of fifty thousand: one instant to the second holds
// one frame, or a handful on a body shooting a burst.
let mut stmt = conn.prepare(
"SELECT source_ref FROM images
WHERE captured_at = ?1
AND (?2 IS NULL OR camera IS ?2)
AND (file_size IS NULL OR file_size = ?3)",
)?;
let mut rows = stmt.query(rusqlite::params![captured_at, camera, size as i64])?;
while let Some(row) = rows.next()? {
let source_ref: String = row.get(0)?;
if file_name(&source_ref).eq_ignore_ascii_case(original_name) {
return Ok(true);
}
}
Ok(false)
}
/// Whether these exact bytes are already in the library.
///
/// The tier that costs a read of the file. Cheap here — `images_hash` is a
/// partial index over the rows that have one — and expensive for the caller,
/// which had to hash something to ask.
pub fn seen_by_content(conn: &Connection, digest: &str) -> Result<bool, CatalogError> {
let n: i64 = conn.query_row(
"SELECT COUNT(*) FROM images WHERE content_hash = ?1",
[digest],
|r| r.get(0),
)?;
Ok(n > 0)
}
/// Record the digest of a file the import computed.
///
/// An import reads every byte anyway, so the hash is free at that moment and
/// costs a full read of an 80 MB file at any other. Storing it is what lets
/// the *next* import answer [`seen_by_content`] without reading anything.
///
/// Matched on `source_ref` within a root, which is how the scan that just
/// catalogued the imported file identifies it. Returns how many rows were
/// updated: zero means the scan has not reached the file yet, which is a
/// normal race and not an error.
pub fn set_content_hash(
conn: &Connection,
root_id: u64,
source_ref: &str,
digest: &str,
) -> Result<usize, CatalogError> {
Ok(conn.execute(
"UPDATE images SET content_hash = ?3
WHERE root_id = ?1 AND source_ref = ?2",
rusqlite::params![root_id as i64, source_ref, digest],
)?)
}
#[cfg(test)]
mod tests {
use super::*;
use crate::Catalog;
/// A catalog holding one photograph, as a scan plus a metadata pass would
/// leave it.
fn with_one() -> Catalog {
let cat = Catalog::in_memory().unwrap();
let c = cat.connection();
c.execute(
"INSERT INTO roots(id, kind, label) VALUES (1, 'local', 'lib')",
[],
)
.unwrap();
c.execute(
"INSERT INTO images(root_id, source_ref, captured_at, camera, file_size,
content_hash, added_at)
VALUES (1, '2026/2026-08-22/IMG_0001.CR3', 1787407200, 'Canon EOS R5',
9, 'deadbeef', 0)",
[],
)
.unwrap();
cat
}
#[test]
fn re_inserting_the_same_card_is_recognised_before_a_transfer() {
let cat = with_one();
assert!(seen_by_metadata(
cat.connection(),
Some(1_787_407_200),
Some("Canon EOS R5"),
9,
"IMG_0001.CR3"
)
.unwrap());
}
#[test]
fn a_different_frame_at_the_same_instant_is_not_a_duplicate() {
// Two bodies firing together, or a burst. The name separates them.
let cat = with_one();
assert!(!seen_by_metadata(
cat.connection(),
Some(1_787_407_200),
Some("Canon EOS R5"),
9,
"IMG_0002.CR3"
)
.unwrap());
}
#[test]
fn the_same_name_from_a_different_camera_is_not_a_duplicate() {
// IMG_0001.CR3 exists on every card ever formatted.
let cat = with_one();
assert!(!seen_by_metadata(
cat.connection(),
Some(1_787_407_200),
Some("NIKON Z 9"),
9,
"IMG_0001.CR3"
)
.unwrap());
}
#[test]
fn the_same_name_at_a_different_time_is_not_a_duplicate() {
// The IMG_9999 wrap: the library holds an unrelated IMG_0001.CR3 from
// four years ago, and matching on name alone would refuse to import
// today's.
let cat = with_one();
assert!(!seen_by_metadata(
cat.connection(),
Some(1_600_000_000),
Some("Canon EOS R5"),
9,
"IMG_0001.CR3"
)
.unwrap());
}
#[test]
fn an_undated_file_falls_through_to_the_content_tier() {
// Not "matches everything undated" — that would silently refuse to
// import a whole card of scanned film.
let cat = with_one();
assert!(!seen_by_metadata(cat.connection(), None, None, 9, "IMG_0001.CR3").unwrap());
}
#[test]
fn a_file_that_grew_is_not_the_one_already_held() {
// A truncated earlier import, or a different rendition of the same
// frame. Same instant, same camera, same name, different bytes.
let cat = with_one();
assert!(!seen_by_metadata(
cat.connection(),
Some(1_787_407_200),
Some("Canon EOS R5"),
1234,
"IMG_0001.CR3"
)
.unwrap());
}
#[test]
fn a_row_with_no_recorded_size_still_matches() {
// The scan stores a size, but a row merged from another device may
// not have one, and refusing to match it would re-import the library.
let cat = with_one();
cat.connection()
.execute("UPDATE images SET file_size = NULL", [])
.unwrap();
assert!(seen_by_metadata(
cat.connection(),
Some(1_787_407_200),
Some("Canon EOS R5"),
9,
"IMG_0001.CR3"
)
.unwrap());
}
#[test]
fn the_same_frame_renamed_is_caught_by_its_bytes() {
let cat = with_one();
// The metadata tier misses it...
assert!(!seen_by_metadata(
cat.connection(),
Some(1_787_407_200),
Some("Canon EOS R5"),
9,
"holiday-42.CR3"
)
.unwrap());
// ...and the content tier does not.
assert!(seen_by_content(cat.connection(), "deadbeef").unwrap());
assert!(!seen_by_content(cat.connection(), "cafe").unwrap());
}
#[test]
fn a_digest_recorded_now_answers_the_next_import() {
let cat = Catalog::in_memory().unwrap();
let c = cat.connection();
c.execute(
"INSERT INTO roots(id, kind, label) VALUES (1, 'local', 'lib')",
[],
)
.unwrap();
c.execute(
"INSERT INTO images(root_id, source_ref, added_at)
VALUES (1, '2026/2026-08-22/IMG_0001.CR3', 0)",
[],
)
.unwrap();
assert!(!seen_by_content(c, "abc123").unwrap());
let n = set_content_hash(c, 1, "2026/2026-08-22/IMG_0001.CR3", "abc123").unwrap();
assert_eq!(n, 1);
assert!(seen_by_content(c, "abc123").unwrap());
}
#[test]
fn recording_a_digest_before_the_scan_arrives_is_not_an_error() {
// The import writes the file and the scan catalogues it; between those
// two moments there is no row to update, and that is a race rather
// than a failure.
let cat = Catalog::in_memory().unwrap();
let c = cat.connection();
c.execute(
"INSERT INTO roots(id, kind, label) VALUES (1, 'local', 'lib')",
[],
)
.unwrap();
assert_eq!(
set_content_hash(c, 1, "not/scanned/yet.CR3", "abc").unwrap(),
0
);
}
#[test]
fn a_name_is_the_last_component_of_either_kind_of_key() {
assert_eq!(file_name("2026/2026-08-22/IMG_0001.CR3"), "IMG_0001.CR3");
// A SAF document id delimits with a colon.
assert_eq!(file_name("primary:DCIM/Camera/IMG_1.CR3"), "IMG_1.CR3");
assert_eq!(file_name("IMG_0001.CR3"), "IMG_0001.CR3");
}
}
+2
View File
@@ -33,6 +33,7 @@ use rusqlite::Connection;
pub mod cache; pub mod cache;
pub mod collections; pub mod collections;
pub mod dedup;
pub mod error; pub mod error;
pub mod jobs; pub mod jobs;
pub mod merge; pub mod merge;
@@ -46,6 +47,7 @@ pub mod walk;
pub use cache::{Budget, Cache, DEFAULT_BUDGET_BYTES}; pub use cache::{Budget, Cache, DEFAULT_BUDGET_BYTES};
pub use collections::{Collection, CollectionKind, TreeRow}; pub use collections::{Collection, CollectionKind, TreeRow};
pub use dedup::{seen_by_content, seen_by_metadata, set_content_hash};
pub use error::CatalogError; pub use error::CatalogError;
pub use jobs::{Job, JobKind, Priority}; pub use jobs::{Job, JobKind, Priority};
pub use merge::MergeReport; pub use merge::MergeReport;
+1 -1
View File
@@ -131,7 +131,7 @@ impl Default for Options {
} }
/// One file offered for import. /// One file offered for import.
#[derive(Debug, Clone)] #[derive(Debug, Clone, PartialEq, Eq)]
pub struct Candidate { pub struct Candidate {
/// Where it is now — on the card. /// Where it is now — on the card.
pub source: SourceRef, pub source: SourceRef,
+2
View File
@@ -6,6 +6,7 @@
pub mod secrets; pub mod secrets;
pub mod storage; pub mod storage;
pub mod volumes;
pub use secrets::{ pub use secrets::{
EphemeralSecretStore, PlatformSecretStore, SecretError, SecretKind, SecretRef, SecretStore, EphemeralSecretStore, PlatformSecretStore, SecretError, SecretKind, SecretRef, SecretStore,
@@ -14,3 +15,4 @@ pub use storage::{
DirRef, Entry, LocalStorage, NewFile, Node, SeekableRead, Storage, StorageError, DirRef, Entry, LocalStorage, NewFile, Node, SeekableRead, Storage, StorageError,
WritableStorage, WritableStorage,
}; };
pub use volumes::{volumes, Volume};
+331
View File
@@ -0,0 +1,331 @@
//! TRACES: FR-CAT-10 | NFR-PORT-1
//! Finding the card.
//!
//! FR-CAT-10 asks for removable-volume insertion to be detected "where the
//! platform permits", which is a careful phrase and this module is why. There
//! is no portable answer: Linux has a mount table and a sysfs flag, Android
//! has neither and hands out a document tree the user picked (ARCH §6.9).
//! So this reports what it can and returns an empty list where it cannot,
//! and every caller must still offer the user a way to say where the card is.
//!
//! # This is the one place besides `grant` that names a path
//!
//! `storage` explains why nothing above it takes a `Path`: a library location
//! is a [`crate::storage::LocalStorage::grant`] away from being a `RootId`,
//! and Android has no path to give. Volume discovery sits on the same side of
//! that line — it produces exactly what `grant` consumes, the folder that is
//! about to become a root. It is discovery of a path rather than use of one.
//!
//! # Why a heuristic rather than a device manager
//!
//! Talking to udisks2 over D-Bus would be authoritative and would add a D-Bus
//! dependency, a service that may not be running, and a permission dialog on
//! some desktops — for a question the filesystem can answer directly. The
//! mount table plus the sysfs `removable` flag covers USB card readers,
//! phones mounted as storage, and external drives. What it does not cover is
//! an internal SD reader that reports itself fixed, which is why a `DCIM`
//! directory is reported alongside and weighted more heavily than the flag:
//! a volume with `DCIM` on it is a camera card whatever sysfs believes.
use std::path::{Path, PathBuf};
/// Somewhere a card might be.
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct Volume {
/// What to call it in a list. The mount point's last component, which is
/// what desktops name after the filesystem label.
pub label: String,
/// Where it is mounted. Hand this to
/// [`LocalStorage::grant`](crate::storage::LocalStorage::grant).
pub path: PathBuf,
/// Whether the kernel calls the underlying device removable.
pub removable: bool,
/// Whether it holds a `DCIM` directory — the strongest signal there is
/// that this is a camera card rather than a backup drive.
pub has_dcim: bool,
}
impl Volume {
/// Whether to show this one first.
///
/// `DCIM` outranks the kernel flag: an internal SD reader often reports
/// its device as fixed, and the user with a card in it does not care what
/// sysfs thinks.
pub fn is_likely_card(&self) -> bool {
self.has_dcim || self.removable
}
}
/// Every mounted volume that might hold photographs.
///
/// Ordered with the likely cards first and, within each group, by label, so
/// the list does not reorder itself between two calls a second apart.
/// Returns empty where the platform does not permit the question — which is
/// not an error and must not be presented as one.
pub fn volumes() -> Vec<Volume> {
let mut found = platform_volumes();
found.sort_by(|a, b| {
b.is_likely_card()
.cmp(&a.is_likely_card())
.then_with(|| a.label.cmp(&b.label))
});
found
}
#[cfg(target_os = "linux")]
fn platform_volumes() -> Vec<Volume> {
let table = match std::fs::read_to_string("/proc/mounts") {
Ok(t) => t,
Err(e) => {
log::debug!("no mount table: {e}");
return Vec::new();
}
};
parse_mounts(&table)
.into_iter()
.filter(|m| is_candidate(m))
.map(|m| {
let removable = device_is_removable(&m.device).unwrap_or(false)
// A desktop that automounts under /run/media or /media has
// already decided this is removable, and it had udisks2 to ask.
|| under_media_dir(&m.mount_point);
Volume {
label: label_for(&m.mount_point),
has_dcim: m.mount_point.join("DCIM").is_dir(),
removable,
path: m.mount_point,
}
})
.collect()
}
/// Everywhere else: no answer, and saying so is the honest result.
///
/// On Android the question is not merely unanswerable but wrong — storage is
/// reached through a tree the user granted, and a card appears there or not at
/// all (ARCH §6.9, FR-PLAT-AND-1).
#[cfg(not(target_os = "linux"))]
fn platform_volumes() -> Vec<Volume> {
Vec::new()
}
/// One line of the mount table.
#[derive(Debug, Clone, PartialEq, Eq)]
struct Mount {
device: String,
mount_point: PathBuf,
fs_type: String,
}
/// Parse `/proc/mounts`.
///
/// Split out from the reading so it can be tested against real tables without
/// a card plugged in — which is the only way this file is testable at all.
fn parse_mounts(table: &str) -> Vec<Mount> {
table
.lines()
.filter_map(|line| {
let mut fields = line.split_whitespace();
let device = fields.next()?;
let mount_point = fields.next()?;
let fs_type = fields.next()?;
Some(Mount {
device: unescape(device),
mount_point: PathBuf::from(unescape(mount_point)),
fs_type: fs_type.to_string(),
})
})
.collect()
}
/// Undo the octal escaping the kernel applies to spaces and tabs.
///
/// A card is very often labelled with a space in it — `EOS DIGITAL` is what
/// Canon writes — and mounted at `/run/media/duncan/EOS\040DIGITAL`. Without
/// this the path does not exist and the card is invisible.
fn unescape(field: &str) -> String {
let mut out = String::with_capacity(field.len());
let bytes = field.as_bytes();
let mut i = 0;
while i < bytes.len() {
if bytes[i] == b'\\' && i + 3 < bytes.len() {
let digits = &field[i + 1..i + 4];
if let Ok(c) = u8::from_str_radix(digits, 8) {
out.push(c as char);
i += 4;
continue;
}
}
out.push(bytes[i] as char);
i += 1;
}
out
}
/// Whether a mount could hold a photograph library at all.
///
/// Excludes the pseudo-filesystems, which is most of the table, and anything
/// not backed by a block device. A card reader always is.
fn is_candidate(m: &Mount) -> bool {
const FILESYSTEMS: &[&str] = &[
// What cameras format cards as, in practice: FAT32 below 32 GB,
// exFAT above it, which is why an SDXC card is always exfat.
"vfat", "exfat", "ntfs", "ntfs3",
"fuseblk", // And what an external drive carrying an archive is.
"ext2", "ext3", "ext4", "btrfs", "xfs", "f2fs", "hfsplus", "apfs",
];
m.device.starts_with("/dev/") && FILESYSTEMS.contains(&m.fs_type.as_str())
}
/// Whether the desktop automounted this, which means it already decided.
fn under_media_dir(mount_point: &Path) -> bool {
let s = mount_point.to_string_lossy();
s.starts_with("/run/media/") || s.starts_with("/media/") || s.starts_with("/mnt/media/")
}
/// Ask sysfs whether the device behind a partition is removable.
///
/// `/dev/sdb1` is a partition; the flag lives on its whole-disk parent,
/// `/sys/block/sdb/removable`. Trailing digits are stripped to get there,
/// except on `mmcblk0p1` and `nvme0n1p1`, whose parents keep their digits and
/// lose only the `pN` suffix — strip naively and `mmcblk0p1` becomes
/// `mmcblk`, which does not exist, and every SD card in an internal reader
/// reports itself fixed.
fn device_is_removable(device: &str) -> Option<bool> {
let name = device.strip_prefix("/dev/")?;
let disk = whole_disk(name);
let flag = std::fs::read_to_string(format!("/sys/block/{disk}/removable")).ok()?;
Some(flag.trim() == "1")
}
/// The whole-disk name a partition belongs to.
fn whole_disk(partition: &str) -> String {
// `mmcblk0p1` -> `mmcblk0`, `nvme0n1p3` -> `nvme0n1`: these spell the
// partition with a `p`, and the disk name legitimately ends in a digit.
if partition.starts_with("mmcblk") || partition.starts_with("nvme") {
if let Some((disk, tail)) = partition.rsplit_once('p') {
if !tail.is_empty() && tail.chars().all(|c| c.is_ascii_digit()) {
return disk.to_string();
}
}
return partition.to_string();
}
// `sdb1` -> `sdb`. A SCSI-style disk name never ends in a digit.
partition
.trim_end_matches(|c: char| c.is_ascii_digit())
.to_string()
}
/// What to call a volume in a list.
fn label_for(mount_point: &Path) -> String {
mount_point
.file_name()
.map(|n| n.to_string_lossy().to_string())
.unwrap_or_else(|| mount_point.to_string_lossy().to_string())
}
#[cfg(test)]
mod tests {
use super::*;
const TABLE: &str = "\
proc /proc proc rw,nosuid 0 0
sys /sys sysfs rw,nosuid 0 0
/dev/nvme0n1p2 / btrfs rw,noatime 0 0
/dev/nvme0n1p1 /boot vfat rw,relatime 0 0
/dev/sdb1 /run/media/duncan/EOS\\040DIGITAL exfat rw,nosuid 0 0
tmpfs /run/user/1000 tmpfs rw,nosuid 0 0
";
#[test]
fn a_mount_table_yields_its_block_devices() {
let mounts = parse_mounts(TABLE);
assert_eq!(mounts.len(), 6);
let candidates: Vec<_> = mounts.iter().filter(|m| is_candidate(m)).collect();
// Three block-backed filesystems; the pseudo ones are gone.
assert_eq!(candidates.len(), 3);
assert!(candidates.iter().all(|m| m.device.starts_with("/dev/")));
}
#[test]
fn a_card_labelled_with_a_space_is_not_lost() {
// Canon writes `EOS DIGITAL`, the kernel escapes the space, and a path
// that keeps the escape does not exist.
let mounts = parse_mounts(TABLE);
let card = mounts
.iter()
.find(|m| m.device == "/dev/sdb1")
.expect("the card");
assert_eq!(
card.mount_point,
PathBuf::from("/run/media/duncan/EOS DIGITAL")
);
assert_eq!(label_for(&card.mount_point), "EOS DIGITAL");
}
#[test]
fn an_automounted_volume_is_treated_as_removable() {
// The desktop had udisks2 to ask and already decided.
assert!(under_media_dir(Path::new("/run/media/duncan/EOS DIGITAL")));
assert!(under_media_dir(Path::new("/media/usb0")));
assert!(!under_media_dir(Path::new("/home/duncan/Photos")));
// Not a prefix match on the word: `/media-server` is not automounted.
assert!(!under_media_dir(Path::new("/mediaserver/x")));
}
#[test]
fn a_partition_resolves_to_the_disk_that_carries_the_flag() {
assert_eq!(whole_disk("sdb1"), "sdb");
assert_eq!(whole_disk("sda12"), "sda");
// The naive strip turns these into `mmcblk` and `nvme`, neither of
// which exists — and every SD card in an internal reader then reports
// itself fixed.
assert_eq!(whole_disk("mmcblk0p1"), "mmcblk0");
assert_eq!(whole_disk("nvme0n1p3"), "nvme0n1");
// A whole disk named directly is already the answer.
assert_eq!(whole_disk("mmcblk0"), "mmcblk0");
assert_eq!(whole_disk("sdb"), "sdb");
}
#[test]
fn a_camera_card_outranks_a_backup_drive() {
let card = Volume {
label: "EOS DIGITAL".into(),
path: "/run/media/duncan/EOS DIGITAL".into(),
removable: false,
has_dcim: true,
};
let drive = Volume {
label: "archive".into(),
path: "/run/media/duncan/archive".into(),
removable: true,
has_dcim: false,
};
let fixed = Volume {
label: "boot".into(),
path: "/boot".into(),
removable: false,
has_dcim: false,
};
// An internal reader reports its device fixed, and the user with a
// card in it does not care what sysfs thinks.
assert!(card.is_likely_card());
assert!(drive.is_likely_card());
assert!(!fixed.is_likely_card());
}
#[test]
fn asking_where_there_is_no_answer_is_not_an_error() {
// The call must work on a machine with no card, in CI, and on a
// platform that cannot answer at all — an empty list, never a panic.
let _ = volumes();
}
#[test]
fn an_octal_escape_that_is_not_one_is_left_alone() {
// A backslash near the end of a field must not read past it.
assert_eq!(unescape("/mnt/odd\\"), "/mnt/odd\\");
assert_eq!(unescape("/mnt/a\\04"), "/mnt/a\\04");
assert_eq!(unescape("/mnt/a\\040b"), "/mnt/a b");
}
}
+1
View File
@@ -27,6 +27,7 @@ dr-plat.workspace = true
dr-sync.workspace = true dr-sync.workspace = true
dr-sync-nextcloud.workspace = true dr-sync-nextcloud.workspace = true
dr-export.workspace = true dr-export.workspace = true
dr-ingest.workspace = true
dr-pipeline.workspace = true dr-pipeline.workspace = true
dr-catalog.workspace = true dr-catalog.workspace = true
dr-thumbs.workspace = true dr-thumbs.workspace = true
+600
View File
@@ -0,0 +1,600 @@
//! TRACES: FR-CAT-10 | FR-CAT-11 | FR-NC-7a
//! Bringing a card into the library: the work, off the interface thread.
//!
//! [`dr_ingest`] does the copying and knows nothing about cards, catalogs or
//! windows — it is handed a list of files, a probe for their metadata and a
//! question about duplicates. This supplies all three, and runs the result on
//! a worker.
//!
//! The shape is the one every background operation in this crate has: a
//! thread with an `mpsc` channel, drained by a `slint::Timer` on the UI
//! thread, reporting into [`crate::activity`]. See `import_ui` for the
//! draining half.
//!
//! # Two phases, one worker
//!
//! Surveying a card is itself slow — a full card is two thousand files across
//! a directory tree on a bus that manages 40 MB/s on a good day — so it runs
//! on the worker too and reports what it found before any copying starts. The
//! user sees "1,847 photographs, 61.2 GB" and then a progress bar, rather than
//! a frozen window and then a progress bar.
//!
//! # The import does not catalogue what it wrote
//!
//! It writes files into the library and then asks for a scan, rather than
//! inserting rows itself. FR-CAT-1 already turns files on disk into catalog
//! rows, and a second path into the `images` table would be a second set of
//! bugs about folders, formats and metadata state. What the import *does*
//! record afterwards is each file's digest (`dr_catalog::dedup`), which the
//! scan cannot know because it never reads a whole file.
use std::path::{Path, PathBuf};
use std::sync::atomic::{AtomicBool, Ordering};
use std::sync::mpsc::{Receiver, Sender};
use std::sync::Arc;
use dr_ingest::{Candidate, DupKey, Imported, Ingest, Options, Report, Shot};
use dr_plat::{DirRef, LocalStorage, Storage, WritableStorage};
use dr_types::{FormatFilter, RootId};
/// Which root the card is granted as, and which the library is.
///
/// Two distinct ids in one `LocalStorage` would also work; separate storages
/// keep the read-only source and the writable destination separate all the
/// way down, which is the distinction `WritableStorage` exists to make.
const CARD: RootId = RootId(9001);
const LIBRARY: RootId = RootId(9002);
const BACKUP: RootId = RootId(9003);
/// A cancel flag shared with the worker.
pub type Cancel = Arc<AtomicBool>;
/// Everything an import needs to run.
#[derive(Debug, Clone)]
pub struct Request {
/// Where the card is mounted.
pub card: PathBuf,
/// The library root the template's folders are created under.
pub library: PathBuf,
/// The optional second destination (FR-CAT-10).
pub backup: Option<PathBuf>,
/// The catalog, opened by the worker for its duplicate queries.
///
/// A path rather than a connection: `rusqlite::Connection` is not `Sync`,
/// and every other worker in this crate opens its own for the same reason.
pub catalog: PathBuf,
/// The library's root id *in the catalog*, which is what
/// [`dr_catalog::set_content_hash`] matches on. Unrelated to [`LIBRARY`],
/// which is this module's handle on the same folder.
pub catalog_root: u64,
/// Which file types to take off the card.
pub filter: FormatFilter,
pub options: Options,
}
/// What the worker sends back.
#[derive(Debug, Clone)]
pub enum Message {
/// The card has been walked. Sent once, before any copying.
Surveyed { files: usize, bytes: u64 },
/// One file finished — imported, skipped or failed.
Progress {
done: usize,
total: usize,
bytes: u64,
name: String,
},
/// The run ended. Always the last message.
Finished(Outcome),
/// The run could not start at all.
Failed(String),
}
/// What an import did, in the form the interface reports it.
#[derive(Debug, Clone, Default, PartialEq, Eq)]
pub struct Outcome {
pub imported: usize,
pub duplicates: usize,
pub failed: usize,
/// Files dated by modification time rather than by the shutter
/// (FR-NC-7a). Named rather than counted: the user needs to know *which*
/// photographs are filed under a date that is not when they were taken.
pub undated: Vec<String>,
pub bytes: u64,
pub cancelled: bool,
/// The folders that were written into, for the message at the end and for
/// the upload that follows (FR-NC-7a).
pub folders: Vec<String>,
/// Card files that may now be deleted, for a move-import (FR-NC-7b).
///
/// Carried out rather than acted on here: this crate knows the local write
/// succeeded, and not whether the upload did.
pub retirable: usize,
}
/// Start an import.
///
/// Returns immediately; the work happens on a thread and reports through the
/// channel. A dropped receiver does not stop the worker — the cancel flag
/// does, and the caller holds it.
pub fn spawn(request: Request, cancel: Cancel) -> Receiver<Message> {
let (tx, rx) = std::sync::mpsc::channel();
std::thread::Builder::new()
.name("import".into())
.spawn(move || {
if let Err(e) = run(request, &cancel, &tx) {
let _ = tx.send(Message::Failed(e));
}
})
.expect("spawning the import worker");
rx
}
fn run(request: Request, cancel: &Cancel, tx: &Sender<Message>) -> Result<(), String> {
let card = LocalStorage::with_root(CARD, request.card.clone());
let library = LocalStorage::with_root(LIBRARY, request.library.clone());
let backup = request
.backup
.as_ref()
.map(|p| LocalStorage::with_root(BACKUP, p.clone()));
let candidates = survey(&card, CARD, &request.filter, &|| {
cancel.load(Ordering::Relaxed)
})
.map_err(|e| format!("could not read the card: {e}"))?;
let bytes: u64 = candidates.iter().map(|c| c.size).sum();
let _ = tx.send(Message::Surveyed {
files: candidates.len(),
bytes,
});
// Opened after the survey rather than before: a card that turns out to be
// empty should not also report a catalog it never needed.
let catalog = dr_catalog::Catalog::open(&request.catalog)
.map_err(|e| format!("could not open the catalog: {e}"))?;
let library_root = DirRef::root(LIBRARY);
let backup_root = DirRef::root(BACKUP);
let ingest = Ingest {
source: &card,
dest: &library,
dest_root: &library_root,
backup: backup.as_ref().map(|b| (b as &dyn WritableStorage, &backup_root)),
options: request.options.clone(),
};
let report = ingest.run(
&candidates,
&|c| probe(&card, c),
&|key| is_duplicate(catalog.connection(), key),
&|| cancel.load(Ordering::Relaxed),
&mut |p| {
let _ = tx.send(Message::Progress {
done: p.done,
total: p.total,
bytes: p.bytes,
name: p.name.clone(),
});
},
);
// The digests, so the *next* import can answer the content tier without
// reading anything. Best-effort and after the fact: the rows do not exist
// until a scan has caught up, and a hash that misses its row costs one
// wasted transfer next time rather than a failure now.
record_digests(catalog.connection(), request.catalog_root, &report.imported);
let _ = tx.send(Message::Finished(summarise(&report)));
Ok(())
}
/// Walk a card for files worth importing.
///
/// Recursive rather than `DCIM`-only: a card carries `DCIM/100CANON`, a phone
/// carries `DCIM/Camera`, and a card that has been through a computer carries
/// whatever somebody put on it. Walking the whole volume finds all three, and
/// the format filter is what keeps the result to photographs.
pub fn survey(
storage: &dyn Storage,
root: RootId,
filter: &FormatFilter,
cancel: &dyn Fn() -> bool,
) -> Result<Vec<Candidate>, dr_plat::StorageError> {
let mut out = Vec::new();
let mut queue = vec![storage.root_dir(root)?];
while let Some(dir) = queue.pop() {
if cancel() {
break;
}
// A directory that cannot be read is skipped rather than fatal: cards
// carry vendor folders with odd permissions, and one of them must not
// cost the user the other nine hundred photographs (FR-CAT-9).
let entries = match storage.list(&dir) {
Ok(e) => e,
Err(e) => {
log::warn!("skipping {dir}: {e}");
continue;
}
};
for entry in entries {
if entry.meta.is_dir {
if !dr_catalog::walk::is_excluded(&entry.meta.name) {
if let dr_plat::Node::Dir(d) = entry.node {
queue.push(d);
}
}
continue;
}
if !filter.allows_name(&entry.meta.name) {
continue;
}
if let dr_plat::Node::File(source) = entry.node {
out.push(Candidate {
source,
name: entry.meta.name,
size: entry.meta.size,
// The listing's reading, which is the fallback the folder
// template uses when a file carries no capture time.
modified_at: entry.meta.mtime / 1000,
});
}
}
}
// A card walks in whatever order the filesystem hands back, which for FAT
// is creation order per directory and arbitrary across them. Sorted so an
// import reports its progress through a card in the order a photographer
// would expect, and so two runs of the same card behave the same way.
out.sort_by(|a, b| a.name.cmp(&b.name));
Ok(out)
}
/// Read one candidate's capture metadata.
///
/// A header read rather than the whole file: `dr_decode::HEADER_BYTES` is
/// enough for EXIF on every format in the tree, and reading 80 MB per file to
/// learn a date would take longer than the import itself.
fn probe(card: &dyn Storage, candidate: &Candidate) -> Option<Shot> {
let header = card
.read_range(&candidate.source, 0..dr_decode::HEADER_BYTES)
.ok()?;
let md = dr_decode::metadata(&header).ok()?;
Some(Shot {
captured_at: md.captured_at,
captured_offset: md.captured_offset,
make: md.make,
model: md.model,
modified_at: candidate.modified_at,
})
}
/// FR-CAT-11's two tiers, against the catalog.
///
/// The camera string is composed the way the scan composes it
/// ([`crate::library::camera_label`]) — the comparison is against what a scan
/// wrote, so a second spelling here would silently disable the cheap tier.
fn is_duplicate(conn: &rusqlite::Connection, key: &DupKey) -> bool {
if let Some(digest) = &key.digest {
return dr_catalog::seen_by_content(conn, digest).unwrap_or(false);
}
let camera = crate::library::camera_label(key.make.as_deref(), key.model.as_deref());
dr_catalog::seen_by_metadata(
conn,
key.captured_at,
camera.as_deref(),
key.size,
&key.original_name,
)
.unwrap_or(false)
}
/// Store what the import learned that a scan cannot.
fn record_digests(conn: &rusqlite::Connection, root: u64, imported: &[Imported]) {
for image in imported {
if let Err(e) = dr_catalog::set_content_hash(conn, root, image.written.key(), &image.digest)
{
log::warn!("recording the digest of {}: {e}", image.name);
}
}
}
/// Reduce a report to what the interface says about it.
pub fn summarise(report: &Report) -> Outcome {
let mut folders: Vec<String> = report
.imported
.iter()
.map(|i| i.folders.join("/"))
.collect();
folders.sort();
folders.dedup();
Outcome {
imported: report.imported.len(),
duplicates: report.duplicates.len(),
failed: report.failed.len(),
undated: report.undated.clone(),
bytes: report.bytes,
cancelled: report.cancelled,
folders,
retirable: report.retirable.len(),
}
}
/// The sentence shown when an import ends.
///
/// Says what happened to every file, because the counts that are zero are the
/// ones worth not mentioning and the ones that are not are the whole point.
/// A run that imported nothing says so rather than reporting success in the
/// abstract.
pub fn describe(outcome: &Outcome) -> String {
let mut parts = Vec::new();
if outcome.imported > 0 {
parts.push(format!(
"{} imported ({})",
outcome.imported,
crate::activity::describe_bytes(outcome.bytes)
));
}
if outcome.duplicates > 0 {
parts.push(format!("{} already in the library", outcome.duplicates));
}
if outcome.failed > 0 {
parts.push(format!("{} failed", outcome.failed));
}
let head = if parts.is_empty() {
"Nothing to import".to_string()
} else {
parts.join(", ")
};
let mut out = if outcome.cancelled {
format!("Stopped — {head}")
} else {
head
};
// Where they went, which is the question the folder template raises.
match outcome.folders.len() {
0 => {}
1 => out.push_str(&format!(" into {}", outcome.folders[0])),
n => out.push_str(&format!(" into {n} folders")),
}
// FR-NC-7a: the mtime fallback is reported rather than silent.
if !outcome.undated.is_empty() {
out.push_str(&format!(
". {} had no capture time and {} filed by modification date",
outcome.undated.len(),
if outcome.undated.len() == 1 {
"was"
} else {
"were"
}
));
}
out
}
/// Whether a folder looks like somewhere a card would be.
///
/// Used to warn before an import writes into the wrong place: a library root
/// and a card mount point are both just directories, and the two are very
/// easy to swap in a dialog.
pub fn looks_like_a_card(path: &Path) -> bool {
path.join("DCIM").is_dir()
}
#[cfg(test)]
mod tests {
use super::*;
use dr_ingest::DateSource;
struct Tree(PathBuf);
impl Tree {
fn new(name: &str) -> Self {
let dir = std::env::temp_dir().join(format!(
"dr-ui-import-{name}-{}-{:?}",
std::process::id(),
std::thread::current().id()
));
let _ = std::fs::remove_dir_all(&dir);
std::fs::create_dir_all(&dir).unwrap();
Tree(dir)
}
fn file(&self, rel: &str, bytes: &[u8]) -> &Self {
let p = self.0.join(rel);
std::fs::create_dir_all(p.parent().unwrap()).unwrap();
std::fs::write(p, bytes).unwrap();
self
}
}
impl Drop for Tree {
fn drop(&mut self) {
let _ = std::fs::remove_dir_all(&self.0);
}
}
fn survey_of(t: &Tree) -> Vec<Candidate> {
let storage = LocalStorage::with_root(CARD, t.0.clone());
survey(&storage, CARD, &FormatFilter::all(), &|| false).unwrap()
}
#[test]
fn a_survey_finds_photographs_wherever_the_camera_put_them() {
let t = Tree::new("survey");
// Three real layouts at once: a Canon card, a phone, and files
// somebody dropped at the top level.
t.file("DCIM/100CANON/IMG_0001.CR3", b"a")
.file("DCIM/Camera/PXL_0002.dng", b"bb")
.file("loose.NEF", b"ccc");
let found = survey_of(&t);
assert_eq!(found.len(), 3);
assert_eq!(
found.iter().map(|c| c.name.as_str()).collect::<Vec<_>>(),
["IMG_0001.CR3", "PXL_0002.dng", "loose.NEF"]
);
}
#[test]
fn a_survey_leaves_what_is_not_a_photograph() {
let t = Tree::new("survey-filter");
t.file("DCIM/100CANON/IMG_0001.CR3", b"a")
// Every card carries these, and none of them is an import.
.file("DCIM/100CANON/IMG_0001.THM", b"x")
.file("MISC/settings.dat", b"x")
.file("readme.txt", b"x");
let found = survey_of(&t);
assert_eq!(found.len(), 1);
assert_eq!(found[0].name, "IMG_0001.CR3");
}
#[test]
fn a_survey_is_ordered_the_same_way_twice() {
// FAT hands directories back in creation order and volumes in none at
// all, so an unsorted survey reports its progress through a card in an
// order that changes between runs.
let t = Tree::new("survey-order");
t.file("DCIM/100CANON/IMG_0003.CR3", b"c")
.file("DCIM/100CANON/IMG_0001.CR3", b"a")
.file("DCIM/101CANON/IMG_0002.CR3", b"b");
assert_eq!(survey_of(&t), survey_of(&t));
}
#[test]
fn a_survey_carries_what_the_folder_template_needs() {
let t = Tree::new("survey-meta");
t.file("DCIM/100CANON/IMG_0001.CR3", b"12345");
let found = survey_of(&t);
assert_eq!(found[0].size, 5);
// Seconds, not milliseconds — a template handed a millisecond reading
// files everything in the year 56000.
let year = dr_types::civil_from_unix(found[0].modified_at).year;
assert!((2000..2200).contains(&year), "mtime looked like {year}");
}
#[test]
fn a_survey_can_be_stopped() {
let t = Tree::new("survey-cancel");
t.file("DCIM/100CANON/IMG_0001.CR3", b"a");
let storage = LocalStorage::with_root(CARD, t.0.clone());
let found = survey(&storage, CARD, &FormatFilter::all(), &|| true).unwrap();
assert!(found.is_empty());
}
#[test]
fn a_card_is_recognised_by_its_dcim_folder() {
let t = Tree::new("looks-like");
t.file("DCIM/100CANON/IMG_0001.CR3", b"a");
assert!(looks_like_a_card(&t.0));
assert!(!looks_like_a_card(&t.0.join("DCIM/100CANON")));
}
// ---- what the interface says -----------------------------------------
fn outcome() -> Outcome {
Outcome {
imported: 3,
bytes: 3 * 1024 * 1024,
folders: vec!["2026/2026-08-22".into()],
..Default::default()
}
}
#[test]
fn a_finished_import_says_what_it_did_and_where() {
let text = describe(&outcome());
assert!(text.starts_with("3 imported ("), "{text}");
assert!(text.ends_with(" into 2026/2026-08-22"), "{text}");
}
#[test]
fn a_card_that_was_already_imported_says_so_rather_than_nothing() {
let o = Outcome {
imported: 0,
duplicates: 12,
bytes: 0,
folders: vec![],
..Default::default()
};
// The failure mode this guards against is a dialog that closes with a
// cheerful "done" after transferring nothing.
assert_eq!(describe(&o), "12 already in the library");
}
#[test]
fn an_empty_card_is_not_reported_as_a_success() {
assert_eq!(describe(&Outcome::default()), "Nothing to import");
}
#[test]
fn a_cancelled_run_leads_with_the_fact_that_it_stopped() {
let o = Outcome {
cancelled: true,
..outcome()
};
assert!(describe(&o).starts_with("Stopped — 3 imported"), "{}", describe(&o));
}
#[test]
fn undated_files_are_named_in_the_result() {
// FR-NC-7a: a file dated by mtime is filed under when it was last
// written, which for a card through a reader is often today.
let o = Outcome {
undated: vec!["SCAN_01.TIF".into()],
..outcome()
};
let text = describe(&o);
assert!(text.contains("1 had no capture time"), "{text}");
assert!(text.contains("was filed by modification date"), "{text}");
let many = Outcome {
undated: vec!["a".into(), "b".into()],
..outcome()
};
assert!(describe(&many).contains("2 had no capture time"));
assert!(describe(&many).contains("were filed"));
}
#[test]
fn several_days_on_one_card_are_counted_rather_than_listed() {
let o = Outcome {
folders: vec!["2026/2026-08-21".into(), "2026/2026-08-22".into()],
..outcome()
};
assert!(describe(&o).ends_with("into 2 folders"), "{}", describe(&o));
}
#[test]
fn a_summary_collapses_a_days_worth_of_files_into_one_folder() {
let report = Report {
imported: (0..3)
.map(|i| Imported {
source: dr_types::SourceRef::Local {
root: CARD,
relative: format!("IMG_{i}.CR3"),
},
written: dr_types::SourceRef::Local {
root: LIBRARY,
relative: format!("2026/2026-08-22/IMG_{i}.CR3"),
},
folders: vec!["2026".into(), "2026-08-22".into()],
name: format!("IMG_{i}.CR3"),
digest: "x".into(),
size: 10,
dated_from: DateSource::Capture,
backup: None,
})
.collect(),
bytes: 30,
..Default::default()
};
let o = summarise(&report);
assert_eq!(o.imported, 3);
assert_eq!(o.folders, ["2026/2026-08-22"]);
}
}
+1
View File
@@ -26,6 +26,7 @@ mod develop;
mod export; mod export;
mod gradient; mod gradient;
mod histogram; mod histogram;
mod import;
mod labels; mod labels;
mod library; mod library;
mod library_ui; mod library_ui;
+20 -10
View File
@@ -1981,21 +1981,31 @@ fn collect_metadata(header: &[u8], req: &ThumbnailRequest, out: &mut Vec<Metadat
image_id: req.image_id, image_id: req.image_id,
captured_at: md.captured_at, captured_at: md.captured_at,
captured_offset: md.captured_offset, captured_offset: md.captured_offset,
camera: match (&md.make, &md.model) { camera: camera_label(md.make.as_deref(), md.model.as_deref()),
// Bodies repeat the make inside the model ("Canon EOS 6D"), so
// joining unconditionally yields "Canon Canon EOS 6D".
(Some(make), Some(model)) if model.starts_with(make.as_str()) => {
Some(model.trim().to_string())
}
(Some(make), Some(model)) => Some(format!("{} {}", make.trim(), model.trim())),
(None, Some(model)) => Some(model.trim().to_string()),
_ => None,
},
lens: md.lens.map(|l| l.trim().to_string()), lens: md.lens.map(|l| l.trim().to_string()),
iso: md.iso, iso: md.iso,
}); });
} }
/// TRACES: FR-CAT-11
/// The camera string the catalog stores, from an EXIF make and model.
///
/// One definition rather than one per caller, because an import's duplicate
/// check compares against what a scan wrote (`dr_catalog::dedup`). Two
/// spellings of the same body would not fail loudly — they would silently
/// disable the cheap tier, and every re-inserted card would transfer in full
/// before the digest caught it.
pub fn camera_label(make: Option<&str>, model: Option<&str>) -> Option<String> {
match (make, model) {
// Bodies repeat the make inside the model ("Canon EOS 6D"), so
// joining unconditionally yields "Canon Canon EOS 6D".
(Some(make), Some(model)) if model.starts_with(make) => Some(model.trim().to_string()),
(Some(make), Some(model)) => Some(format!("{} {}", make.trim(), model.trim())),
(None, Some(model)) => Some(model.trim().to_string()),
_ => None,
}
}
/// Write a batch of dates and tell the UI, draining `found`. /// Write a batch of dates and tell the UI, draining `found`.
/// ///
/// Separate from the loop so the same path serves both the periodic flush and /// Separate from the loop so the same path serves both the periodic flush and