wip: ingest
This commit is contained in:
Generated
+1
@@ -1578,6 +1578,7 @@ dependencies = [
|
||||
"dr-decode",
|
||||
"dr-export",
|
||||
"dr-gpu",
|
||||
"dr-ingest",
|
||||
"dr-pipeline",
|
||||
"dr-plat",
|
||||
"dr-segment",
|
||||
|
||||
@@ -0,0 +1,308 @@
|
||||
//! TRACES: FR-CAT-11
|
||||
//! Has this photograph been imported before?
|
||||
//!
|
||||
//! Two tiers, because neither alone is enough and they cost very different
|
||||
//! amounts. The metadata tier — capture time, camera, size, the name the
|
||||
//! camera gave it — is answerable from the catalog before a byte leaves the
|
||||
//! card, which is what makes re-inserting an already-imported card cost a
|
||||
//! metadata read per file rather than a full transfer. The content tier
|
||||
//! catches what the first misses: the same frame arriving under a different
|
||||
//! name, from a second card, or after somebody renamed it.
|
||||
//!
|
||||
//! # Why the filename is compared here rather than in SQL
|
||||
//!
|
||||
//! `images.source_ref` holds the whole opaque key — a relative path on Linux,
|
||||
//! a document id on SAF — and the camera's filename is only its last
|
||||
//! component. Matching that in SQL means `LIKE '%/IMG_0001.CR3'`, which cannot
|
||||
//! use an index, scans the whole table, and is wrong on SAF where the
|
||||
//! separator is not `/`. So the query narrows on the indexed columns and the
|
||||
//! handful of rows that survive are compared in Rust, the same way the grid
|
||||
//! already derives a display name.
|
||||
//!
|
||||
//! # Filename alone is never sufficient
|
||||
//!
|
||||
//! Camera filenames wrap at `IMG_9999` and start again, so a library of any
|
||||
//! age holds several unrelated `IMG_0001.CR3`. That is why the cheap tier
|
||||
//! carries capture time and camera as well, and why the expensive tier exists
|
||||
//! at all.
|
||||
|
||||
use rusqlite::Connection;
|
||||
|
||||
use crate::CatalogError;
|
||||
|
||||
/// The last component of a stored source reference.
|
||||
///
|
||||
/// Splits on both separators for the same reason `Catalog::window` does: the
|
||||
/// key's shape belongs to the storage that produced it, and a SAF document id
|
||||
/// is delimited with `:`.
|
||||
fn file_name(source_ref: &str) -> &str {
|
||||
source_ref.rsplit(['/', ':']).next().unwrap_or(source_ref)
|
||||
}
|
||||
|
||||
/// Whether the catalog already holds this photograph, on metadata alone.
|
||||
///
|
||||
/// `camera` is the joined make-and-model string the scan stores, not the raw
|
||||
/// EXIF pair — the caller composes it the same way, or the comparison is
|
||||
/// always false.
|
||||
///
|
||||
/// A `captured_at` of `None` makes this answer `false` rather than matching
|
||||
/// every undated image in the library: without a capture time the key is
|
||||
/// filename plus size, which two frames from the same body collide on
|
||||
/// routinely. An undated file falls through to the content tier, which is
|
||||
/// slower and right.
|
||||
pub fn seen_by_metadata(
|
||||
conn: &Connection,
|
||||
captured_at: Option<i64>,
|
||||
camera: Option<&str>,
|
||||
size: u64,
|
||||
original_name: &str,
|
||||
) -> Result<bool, CatalogError> {
|
||||
let Some(captured_at) = captured_at else {
|
||||
return Ok(false);
|
||||
};
|
||||
|
||||
// `images_captured` indexes the capture time, so this reads a few rows
|
||||
// even in a library of fifty thousand: one instant to the second holds
|
||||
// one frame, or a handful on a body shooting a burst.
|
||||
let mut stmt = conn.prepare(
|
||||
"SELECT source_ref FROM images
|
||||
WHERE captured_at = ?1
|
||||
AND (?2 IS NULL OR camera IS ?2)
|
||||
AND (file_size IS NULL OR file_size = ?3)",
|
||||
)?;
|
||||
let mut rows = stmt.query(rusqlite::params![captured_at, camera, size as i64])?;
|
||||
while let Some(row) = rows.next()? {
|
||||
let source_ref: String = row.get(0)?;
|
||||
if file_name(&source_ref).eq_ignore_ascii_case(original_name) {
|
||||
return Ok(true);
|
||||
}
|
||||
}
|
||||
Ok(false)
|
||||
}
|
||||
|
||||
/// Whether these exact bytes are already in the library.
|
||||
///
|
||||
/// The tier that costs a read of the file. Cheap here — `images_hash` is a
|
||||
/// partial index over the rows that have one — and expensive for the caller,
|
||||
/// which had to hash something to ask.
|
||||
pub fn seen_by_content(conn: &Connection, digest: &str) -> Result<bool, CatalogError> {
|
||||
let n: i64 = conn.query_row(
|
||||
"SELECT COUNT(*) FROM images WHERE content_hash = ?1",
|
||||
[digest],
|
||||
|r| r.get(0),
|
||||
)?;
|
||||
Ok(n > 0)
|
||||
}
|
||||
|
||||
/// Record the digest of a file the import computed.
|
||||
///
|
||||
/// An import reads every byte anyway, so the hash is free at that moment and
|
||||
/// costs a full read of an 80 MB file at any other. Storing it is what lets
|
||||
/// the *next* import answer [`seen_by_content`] without reading anything.
|
||||
///
|
||||
/// Matched on `source_ref` within a root, which is how the scan that just
|
||||
/// catalogued the imported file identifies it. Returns how many rows were
|
||||
/// updated: zero means the scan has not reached the file yet, which is a
|
||||
/// normal race and not an error.
|
||||
pub fn set_content_hash(
|
||||
conn: &Connection,
|
||||
root_id: u64,
|
||||
source_ref: &str,
|
||||
digest: &str,
|
||||
) -> Result<usize, CatalogError> {
|
||||
Ok(conn.execute(
|
||||
"UPDATE images SET content_hash = ?3
|
||||
WHERE root_id = ?1 AND source_ref = ?2",
|
||||
rusqlite::params![root_id as i64, source_ref, digest],
|
||||
)?)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use crate::Catalog;
|
||||
|
||||
/// A catalog holding one photograph, as a scan plus a metadata pass would
|
||||
/// leave it.
|
||||
fn with_one() -> Catalog {
|
||||
let cat = Catalog::in_memory().unwrap();
|
||||
let c = cat.connection();
|
||||
c.execute(
|
||||
"INSERT INTO roots(id, kind, label) VALUES (1, 'local', 'lib')",
|
||||
[],
|
||||
)
|
||||
.unwrap();
|
||||
c.execute(
|
||||
"INSERT INTO images(root_id, source_ref, captured_at, camera, file_size,
|
||||
content_hash, added_at)
|
||||
VALUES (1, '2026/2026-08-22/IMG_0001.CR3', 1787407200, 'Canon EOS R5',
|
||||
9, 'deadbeef', 0)",
|
||||
[],
|
||||
)
|
||||
.unwrap();
|
||||
cat
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn re_inserting_the_same_card_is_recognised_before_a_transfer() {
|
||||
let cat = with_one();
|
||||
assert!(seen_by_metadata(
|
||||
cat.connection(),
|
||||
Some(1_787_407_200),
|
||||
Some("Canon EOS R5"),
|
||||
9,
|
||||
"IMG_0001.CR3"
|
||||
)
|
||||
.unwrap());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_different_frame_at_the_same_instant_is_not_a_duplicate() {
|
||||
// Two bodies firing together, or a burst. The name separates them.
|
||||
let cat = with_one();
|
||||
assert!(!seen_by_metadata(
|
||||
cat.connection(),
|
||||
Some(1_787_407_200),
|
||||
Some("Canon EOS R5"),
|
||||
9,
|
||||
"IMG_0002.CR3"
|
||||
)
|
||||
.unwrap());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn the_same_name_from_a_different_camera_is_not_a_duplicate() {
|
||||
// IMG_0001.CR3 exists on every card ever formatted.
|
||||
let cat = with_one();
|
||||
assert!(!seen_by_metadata(
|
||||
cat.connection(),
|
||||
Some(1_787_407_200),
|
||||
Some("NIKON Z 9"),
|
||||
9,
|
||||
"IMG_0001.CR3"
|
||||
)
|
||||
.unwrap());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn the_same_name_at_a_different_time_is_not_a_duplicate() {
|
||||
// The IMG_9999 wrap: the library holds an unrelated IMG_0001.CR3 from
|
||||
// four years ago, and matching on name alone would refuse to import
|
||||
// today's.
|
||||
let cat = with_one();
|
||||
assert!(!seen_by_metadata(
|
||||
cat.connection(),
|
||||
Some(1_600_000_000),
|
||||
Some("Canon EOS R5"),
|
||||
9,
|
||||
"IMG_0001.CR3"
|
||||
)
|
||||
.unwrap());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn an_undated_file_falls_through_to_the_content_tier() {
|
||||
// Not "matches everything undated" — that would silently refuse to
|
||||
// import a whole card of scanned film.
|
||||
let cat = with_one();
|
||||
assert!(!seen_by_metadata(cat.connection(), None, None, 9, "IMG_0001.CR3").unwrap());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_file_that_grew_is_not_the_one_already_held() {
|
||||
// A truncated earlier import, or a different rendition of the same
|
||||
// frame. Same instant, same camera, same name, different bytes.
|
||||
let cat = with_one();
|
||||
assert!(!seen_by_metadata(
|
||||
cat.connection(),
|
||||
Some(1_787_407_200),
|
||||
Some("Canon EOS R5"),
|
||||
1234,
|
||||
"IMG_0001.CR3"
|
||||
)
|
||||
.unwrap());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_row_with_no_recorded_size_still_matches() {
|
||||
// The scan stores a size, but a row merged from another device may
|
||||
// not have one, and refusing to match it would re-import the library.
|
||||
let cat = with_one();
|
||||
cat.connection()
|
||||
.execute("UPDATE images SET file_size = NULL", [])
|
||||
.unwrap();
|
||||
assert!(seen_by_metadata(
|
||||
cat.connection(),
|
||||
Some(1_787_407_200),
|
||||
Some("Canon EOS R5"),
|
||||
9,
|
||||
"IMG_0001.CR3"
|
||||
)
|
||||
.unwrap());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn the_same_frame_renamed_is_caught_by_its_bytes() {
|
||||
let cat = with_one();
|
||||
// The metadata tier misses it...
|
||||
assert!(!seen_by_metadata(
|
||||
cat.connection(),
|
||||
Some(1_787_407_200),
|
||||
Some("Canon EOS R5"),
|
||||
9,
|
||||
"holiday-42.CR3"
|
||||
)
|
||||
.unwrap());
|
||||
// ...and the content tier does not.
|
||||
assert!(seen_by_content(cat.connection(), "deadbeef").unwrap());
|
||||
assert!(!seen_by_content(cat.connection(), "cafe").unwrap());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_digest_recorded_now_answers_the_next_import() {
|
||||
let cat = Catalog::in_memory().unwrap();
|
||||
let c = cat.connection();
|
||||
c.execute(
|
||||
"INSERT INTO roots(id, kind, label) VALUES (1, 'local', 'lib')",
|
||||
[],
|
||||
)
|
||||
.unwrap();
|
||||
c.execute(
|
||||
"INSERT INTO images(root_id, source_ref, added_at)
|
||||
VALUES (1, '2026/2026-08-22/IMG_0001.CR3', 0)",
|
||||
[],
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
assert!(!seen_by_content(c, "abc123").unwrap());
|
||||
let n = set_content_hash(c, 1, "2026/2026-08-22/IMG_0001.CR3", "abc123").unwrap();
|
||||
assert_eq!(n, 1);
|
||||
assert!(seen_by_content(c, "abc123").unwrap());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn recording_a_digest_before_the_scan_arrives_is_not_an_error() {
|
||||
// The import writes the file and the scan catalogues it; between those
|
||||
// two moments there is no row to update, and that is a race rather
|
||||
// than a failure.
|
||||
let cat = Catalog::in_memory().unwrap();
|
||||
let c = cat.connection();
|
||||
c.execute(
|
||||
"INSERT INTO roots(id, kind, label) VALUES (1, 'local', 'lib')",
|
||||
[],
|
||||
)
|
||||
.unwrap();
|
||||
assert_eq!(
|
||||
set_content_hash(c, 1, "not/scanned/yet.CR3", "abc").unwrap(),
|
||||
0
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_name_is_the_last_component_of_either_kind_of_key() {
|
||||
assert_eq!(file_name("2026/2026-08-22/IMG_0001.CR3"), "IMG_0001.CR3");
|
||||
// A SAF document id delimits with a colon.
|
||||
assert_eq!(file_name("primary:DCIM/Camera/IMG_1.CR3"), "IMG_1.CR3");
|
||||
assert_eq!(file_name("IMG_0001.CR3"), "IMG_0001.CR3");
|
||||
}
|
||||
}
|
||||
@@ -33,6 +33,7 @@ use rusqlite::Connection;
|
||||
|
||||
pub mod cache;
|
||||
pub mod collections;
|
||||
pub mod dedup;
|
||||
pub mod error;
|
||||
pub mod jobs;
|
||||
pub mod merge;
|
||||
@@ -46,6 +47,7 @@ pub mod walk;
|
||||
|
||||
pub use cache::{Budget, Cache, DEFAULT_BUDGET_BYTES};
|
||||
pub use collections::{Collection, CollectionKind, TreeRow};
|
||||
pub use dedup::{seen_by_content, seen_by_metadata, set_content_hash};
|
||||
pub use error::CatalogError;
|
||||
pub use jobs::{Job, JobKind, Priority};
|
||||
pub use merge::MergeReport;
|
||||
|
||||
@@ -131,7 +131,7 @@ impl Default for Options {
|
||||
}
|
||||
|
||||
/// One file offered for import.
|
||||
#[derive(Debug, Clone)]
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct Candidate {
|
||||
/// Where it is now — on the card.
|
||||
pub source: SourceRef,
|
||||
|
||||
@@ -6,6 +6,7 @@
|
||||
|
||||
pub mod secrets;
|
||||
pub mod storage;
|
||||
pub mod volumes;
|
||||
|
||||
pub use secrets::{
|
||||
EphemeralSecretStore, PlatformSecretStore, SecretError, SecretKind, SecretRef, SecretStore,
|
||||
@@ -14,3 +15,4 @@ pub use storage::{
|
||||
DirRef, Entry, LocalStorage, NewFile, Node, SeekableRead, Storage, StorageError,
|
||||
WritableStorage,
|
||||
};
|
||||
pub use volumes::{volumes, Volume};
|
||||
|
||||
@@ -0,0 +1,331 @@
|
||||
//! TRACES: FR-CAT-10 | NFR-PORT-1
|
||||
//! Finding the card.
|
||||
//!
|
||||
//! FR-CAT-10 asks for removable-volume insertion to be detected "where the
|
||||
//! platform permits", which is a careful phrase and this module is why. There
|
||||
//! is no portable answer: Linux has a mount table and a sysfs flag, Android
|
||||
//! has neither and hands out a document tree the user picked (ARCH §6.9).
|
||||
//! So this reports what it can and returns an empty list where it cannot,
|
||||
//! and every caller must still offer the user a way to say where the card is.
|
||||
//!
|
||||
//! # This is the one place besides `grant` that names a path
|
||||
//!
|
||||
//! `storage` explains why nothing above it takes a `Path`: a library location
|
||||
//! is a [`crate::storage::LocalStorage::grant`] away from being a `RootId`,
|
||||
//! and Android has no path to give. Volume discovery sits on the same side of
|
||||
//! that line — it produces exactly what `grant` consumes, the folder that is
|
||||
//! about to become a root. It is discovery of a path rather than use of one.
|
||||
//!
|
||||
//! # Why a heuristic rather than a device manager
|
||||
//!
|
||||
//! Talking to udisks2 over D-Bus would be authoritative and would add a D-Bus
|
||||
//! dependency, a service that may not be running, and a permission dialog on
|
||||
//! some desktops — for a question the filesystem can answer directly. The
|
||||
//! mount table plus the sysfs `removable` flag covers USB card readers,
|
||||
//! phones mounted as storage, and external drives. What it does not cover is
|
||||
//! an internal SD reader that reports itself fixed, which is why a `DCIM`
|
||||
//! directory is reported alongside and weighted more heavily than the flag:
|
||||
//! a volume with `DCIM` on it is a camera card whatever sysfs believes.
|
||||
|
||||
use std::path::{Path, PathBuf};
|
||||
|
||||
/// Somewhere a card might be.
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct Volume {
|
||||
/// What to call it in a list. The mount point's last component, which is
|
||||
/// what desktops name after the filesystem label.
|
||||
pub label: String,
|
||||
/// Where it is mounted. Hand this to
|
||||
/// [`LocalStorage::grant`](crate::storage::LocalStorage::grant).
|
||||
pub path: PathBuf,
|
||||
/// Whether the kernel calls the underlying device removable.
|
||||
pub removable: bool,
|
||||
/// Whether it holds a `DCIM` directory — the strongest signal there is
|
||||
/// that this is a camera card rather than a backup drive.
|
||||
pub has_dcim: bool,
|
||||
}
|
||||
|
||||
impl Volume {
|
||||
/// Whether to show this one first.
|
||||
///
|
||||
/// `DCIM` outranks the kernel flag: an internal SD reader often reports
|
||||
/// its device as fixed, and the user with a card in it does not care what
|
||||
/// sysfs thinks.
|
||||
pub fn is_likely_card(&self) -> bool {
|
||||
self.has_dcim || self.removable
|
||||
}
|
||||
}
|
||||
|
||||
/// Every mounted volume that might hold photographs.
|
||||
///
|
||||
/// Ordered with the likely cards first and, within each group, by label, so
|
||||
/// the list does not reorder itself between two calls a second apart.
|
||||
/// Returns empty where the platform does not permit the question — which is
|
||||
/// not an error and must not be presented as one.
|
||||
pub fn volumes() -> Vec<Volume> {
|
||||
let mut found = platform_volumes();
|
||||
found.sort_by(|a, b| {
|
||||
b.is_likely_card()
|
||||
.cmp(&a.is_likely_card())
|
||||
.then_with(|| a.label.cmp(&b.label))
|
||||
});
|
||||
found
|
||||
}
|
||||
|
||||
#[cfg(target_os = "linux")]
|
||||
fn platform_volumes() -> Vec<Volume> {
|
||||
let table = match std::fs::read_to_string("/proc/mounts") {
|
||||
Ok(t) => t,
|
||||
Err(e) => {
|
||||
log::debug!("no mount table: {e}");
|
||||
return Vec::new();
|
||||
}
|
||||
};
|
||||
parse_mounts(&table)
|
||||
.into_iter()
|
||||
.filter(|m| is_candidate(m))
|
||||
.map(|m| {
|
||||
let removable = device_is_removable(&m.device).unwrap_or(false)
|
||||
// A desktop that automounts under /run/media or /media has
|
||||
// already decided this is removable, and it had udisks2 to ask.
|
||||
|| under_media_dir(&m.mount_point);
|
||||
Volume {
|
||||
label: label_for(&m.mount_point),
|
||||
has_dcim: m.mount_point.join("DCIM").is_dir(),
|
||||
removable,
|
||||
path: m.mount_point,
|
||||
}
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
/// Everywhere else: no answer, and saying so is the honest result.
|
||||
///
|
||||
/// On Android the question is not merely unanswerable but wrong — storage is
|
||||
/// reached through a tree the user granted, and a card appears there or not at
|
||||
/// all (ARCH §6.9, FR-PLAT-AND-1).
|
||||
#[cfg(not(target_os = "linux"))]
|
||||
fn platform_volumes() -> Vec<Volume> {
|
||||
Vec::new()
|
||||
}
|
||||
|
||||
/// One line of the mount table.
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
struct Mount {
|
||||
device: String,
|
||||
mount_point: PathBuf,
|
||||
fs_type: String,
|
||||
}
|
||||
|
||||
/// Parse `/proc/mounts`.
|
||||
///
|
||||
/// Split out from the reading so it can be tested against real tables without
|
||||
/// a card plugged in — which is the only way this file is testable at all.
|
||||
fn parse_mounts(table: &str) -> Vec<Mount> {
|
||||
table
|
||||
.lines()
|
||||
.filter_map(|line| {
|
||||
let mut fields = line.split_whitespace();
|
||||
let device = fields.next()?;
|
||||
let mount_point = fields.next()?;
|
||||
let fs_type = fields.next()?;
|
||||
Some(Mount {
|
||||
device: unescape(device),
|
||||
mount_point: PathBuf::from(unescape(mount_point)),
|
||||
fs_type: fs_type.to_string(),
|
||||
})
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
/// Undo the octal escaping the kernel applies to spaces and tabs.
|
||||
///
|
||||
/// A card is very often labelled with a space in it — `EOS DIGITAL` is what
|
||||
/// Canon writes — and mounted at `/run/media/duncan/EOS\040DIGITAL`. Without
|
||||
/// this the path does not exist and the card is invisible.
|
||||
fn unescape(field: &str) -> String {
|
||||
let mut out = String::with_capacity(field.len());
|
||||
let bytes = field.as_bytes();
|
||||
let mut i = 0;
|
||||
while i < bytes.len() {
|
||||
if bytes[i] == b'\\' && i + 3 < bytes.len() {
|
||||
let digits = &field[i + 1..i + 4];
|
||||
if let Ok(c) = u8::from_str_radix(digits, 8) {
|
||||
out.push(c as char);
|
||||
i += 4;
|
||||
continue;
|
||||
}
|
||||
}
|
||||
out.push(bytes[i] as char);
|
||||
i += 1;
|
||||
}
|
||||
out
|
||||
}
|
||||
|
||||
/// Whether a mount could hold a photograph library at all.
|
||||
///
|
||||
/// Excludes the pseudo-filesystems, which is most of the table, and anything
|
||||
/// not backed by a block device. A card reader always is.
|
||||
fn is_candidate(m: &Mount) -> bool {
|
||||
const FILESYSTEMS: &[&str] = &[
|
||||
// What cameras format cards as, in practice: FAT32 below 32 GB,
|
||||
// exFAT above it, which is why an SDXC card is always exfat.
|
||||
"vfat", "exfat", "ntfs", "ntfs3",
|
||||
"fuseblk", // And what an external drive carrying an archive is.
|
||||
"ext2", "ext3", "ext4", "btrfs", "xfs", "f2fs", "hfsplus", "apfs",
|
||||
];
|
||||
m.device.starts_with("/dev/") && FILESYSTEMS.contains(&m.fs_type.as_str())
|
||||
}
|
||||
|
||||
/// Whether the desktop automounted this, which means it already decided.
|
||||
fn under_media_dir(mount_point: &Path) -> bool {
|
||||
let s = mount_point.to_string_lossy();
|
||||
s.starts_with("/run/media/") || s.starts_with("/media/") || s.starts_with("/mnt/media/")
|
||||
}
|
||||
|
||||
/// Ask sysfs whether the device behind a partition is removable.
|
||||
///
|
||||
/// `/dev/sdb1` is a partition; the flag lives on its whole-disk parent,
|
||||
/// `/sys/block/sdb/removable`. Trailing digits are stripped to get there,
|
||||
/// except on `mmcblk0p1` and `nvme0n1p1`, whose parents keep their digits and
|
||||
/// lose only the `pN` suffix — strip naively and `mmcblk0p1` becomes
|
||||
/// `mmcblk`, which does not exist, and every SD card in an internal reader
|
||||
/// reports itself fixed.
|
||||
fn device_is_removable(device: &str) -> Option<bool> {
|
||||
let name = device.strip_prefix("/dev/")?;
|
||||
let disk = whole_disk(name);
|
||||
let flag = std::fs::read_to_string(format!("/sys/block/{disk}/removable")).ok()?;
|
||||
Some(flag.trim() == "1")
|
||||
}
|
||||
|
||||
/// The whole-disk name a partition belongs to.
|
||||
fn whole_disk(partition: &str) -> String {
|
||||
// `mmcblk0p1` -> `mmcblk0`, `nvme0n1p3` -> `nvme0n1`: these spell the
|
||||
// partition with a `p`, and the disk name legitimately ends in a digit.
|
||||
if partition.starts_with("mmcblk") || partition.starts_with("nvme") {
|
||||
if let Some((disk, tail)) = partition.rsplit_once('p') {
|
||||
if !tail.is_empty() && tail.chars().all(|c| c.is_ascii_digit()) {
|
||||
return disk.to_string();
|
||||
}
|
||||
}
|
||||
return partition.to_string();
|
||||
}
|
||||
// `sdb1` -> `sdb`. A SCSI-style disk name never ends in a digit.
|
||||
partition
|
||||
.trim_end_matches(|c: char| c.is_ascii_digit())
|
||||
.to_string()
|
||||
}
|
||||
|
||||
/// What to call a volume in a list.
|
||||
fn label_for(mount_point: &Path) -> String {
|
||||
mount_point
|
||||
.file_name()
|
||||
.map(|n| n.to_string_lossy().to_string())
|
||||
.unwrap_or_else(|| mount_point.to_string_lossy().to_string())
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
const TABLE: &str = "\
|
||||
proc /proc proc rw,nosuid 0 0
|
||||
sys /sys sysfs rw,nosuid 0 0
|
||||
/dev/nvme0n1p2 / btrfs rw,noatime 0 0
|
||||
/dev/nvme0n1p1 /boot vfat rw,relatime 0 0
|
||||
/dev/sdb1 /run/media/duncan/EOS\\040DIGITAL exfat rw,nosuid 0 0
|
||||
tmpfs /run/user/1000 tmpfs rw,nosuid 0 0
|
||||
";
|
||||
|
||||
#[test]
|
||||
fn a_mount_table_yields_its_block_devices() {
|
||||
let mounts = parse_mounts(TABLE);
|
||||
assert_eq!(mounts.len(), 6);
|
||||
let candidates: Vec<_> = mounts.iter().filter(|m| is_candidate(m)).collect();
|
||||
// Three block-backed filesystems; the pseudo ones are gone.
|
||||
assert_eq!(candidates.len(), 3);
|
||||
assert!(candidates.iter().all(|m| m.device.starts_with("/dev/")));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_card_labelled_with_a_space_is_not_lost() {
|
||||
// Canon writes `EOS DIGITAL`, the kernel escapes the space, and a path
|
||||
// that keeps the escape does not exist.
|
||||
let mounts = parse_mounts(TABLE);
|
||||
let card = mounts
|
||||
.iter()
|
||||
.find(|m| m.device == "/dev/sdb1")
|
||||
.expect("the card");
|
||||
assert_eq!(
|
||||
card.mount_point,
|
||||
PathBuf::from("/run/media/duncan/EOS DIGITAL")
|
||||
);
|
||||
assert_eq!(label_for(&card.mount_point), "EOS DIGITAL");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn an_automounted_volume_is_treated_as_removable() {
|
||||
// The desktop had udisks2 to ask and already decided.
|
||||
assert!(under_media_dir(Path::new("/run/media/duncan/EOS DIGITAL")));
|
||||
assert!(under_media_dir(Path::new("/media/usb0")));
|
||||
assert!(!under_media_dir(Path::new("/home/duncan/Photos")));
|
||||
// Not a prefix match on the word: `/media-server` is not automounted.
|
||||
assert!(!under_media_dir(Path::new("/mediaserver/x")));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_partition_resolves_to_the_disk_that_carries_the_flag() {
|
||||
assert_eq!(whole_disk("sdb1"), "sdb");
|
||||
assert_eq!(whole_disk("sda12"), "sda");
|
||||
// The naive strip turns these into `mmcblk` and `nvme`, neither of
|
||||
// which exists — and every SD card in an internal reader then reports
|
||||
// itself fixed.
|
||||
assert_eq!(whole_disk("mmcblk0p1"), "mmcblk0");
|
||||
assert_eq!(whole_disk("nvme0n1p3"), "nvme0n1");
|
||||
// A whole disk named directly is already the answer.
|
||||
assert_eq!(whole_disk("mmcblk0"), "mmcblk0");
|
||||
assert_eq!(whole_disk("sdb"), "sdb");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_camera_card_outranks_a_backup_drive() {
|
||||
let card = Volume {
|
||||
label: "EOS DIGITAL".into(),
|
||||
path: "/run/media/duncan/EOS DIGITAL".into(),
|
||||
removable: false,
|
||||
has_dcim: true,
|
||||
};
|
||||
let drive = Volume {
|
||||
label: "archive".into(),
|
||||
path: "/run/media/duncan/archive".into(),
|
||||
removable: true,
|
||||
has_dcim: false,
|
||||
};
|
||||
let fixed = Volume {
|
||||
label: "boot".into(),
|
||||
path: "/boot".into(),
|
||||
removable: false,
|
||||
has_dcim: false,
|
||||
};
|
||||
// An internal reader reports its device fixed, and the user with a
|
||||
// card in it does not care what sysfs thinks.
|
||||
assert!(card.is_likely_card());
|
||||
assert!(drive.is_likely_card());
|
||||
assert!(!fixed.is_likely_card());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn asking_where_there_is_no_answer_is_not_an_error() {
|
||||
// The call must work on a machine with no card, in CI, and on a
|
||||
// platform that cannot answer at all — an empty list, never a panic.
|
||||
let _ = volumes();
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn an_octal_escape_that_is_not_one_is_left_alone() {
|
||||
// A backslash near the end of a field must not read past it.
|
||||
assert_eq!(unescape("/mnt/odd\\"), "/mnt/odd\\");
|
||||
assert_eq!(unescape("/mnt/a\\04"), "/mnt/a\\04");
|
||||
assert_eq!(unescape("/mnt/a\\040b"), "/mnt/a b");
|
||||
}
|
||||
}
|
||||
@@ -27,6 +27,7 @@ dr-plat.workspace = true
|
||||
dr-sync.workspace = true
|
||||
dr-sync-nextcloud.workspace = true
|
||||
dr-export.workspace = true
|
||||
dr-ingest.workspace = true
|
||||
dr-pipeline.workspace = true
|
||||
dr-catalog.workspace = true
|
||||
dr-thumbs.workspace = true
|
||||
|
||||
@@ -0,0 +1,600 @@
|
||||
//! TRACES: FR-CAT-10 | FR-CAT-11 | FR-NC-7a
|
||||
//! Bringing a card into the library: the work, off the interface thread.
|
||||
//!
|
||||
//! [`dr_ingest`] does the copying and knows nothing about cards, catalogs or
|
||||
//! windows — it is handed a list of files, a probe for their metadata and a
|
||||
//! question about duplicates. This supplies all three, and runs the result on
|
||||
//! a worker.
|
||||
//!
|
||||
//! The shape is the one every background operation in this crate has: a
|
||||
//! thread with an `mpsc` channel, drained by a `slint::Timer` on the UI
|
||||
//! thread, reporting into [`crate::activity`]. See `import_ui` for the
|
||||
//! draining half.
|
||||
//!
|
||||
//! # Two phases, one worker
|
||||
//!
|
||||
//! Surveying a card is itself slow — a full card is two thousand files across
|
||||
//! a directory tree on a bus that manages 40 MB/s on a good day — so it runs
|
||||
//! on the worker too and reports what it found before any copying starts. The
|
||||
//! user sees "1,847 photographs, 61.2 GB" and then a progress bar, rather than
|
||||
//! a frozen window and then a progress bar.
|
||||
//!
|
||||
//! # The import does not catalogue what it wrote
|
||||
//!
|
||||
//! It writes files into the library and then asks for a scan, rather than
|
||||
//! inserting rows itself. FR-CAT-1 already turns files on disk into catalog
|
||||
//! rows, and a second path into the `images` table would be a second set of
|
||||
//! bugs about folders, formats and metadata state. What the import *does*
|
||||
//! record afterwards is each file's digest (`dr_catalog::dedup`), which the
|
||||
//! scan cannot know because it never reads a whole file.
|
||||
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::sync::atomic::{AtomicBool, Ordering};
|
||||
use std::sync::mpsc::{Receiver, Sender};
|
||||
use std::sync::Arc;
|
||||
|
||||
use dr_ingest::{Candidate, DupKey, Imported, Ingest, Options, Report, Shot};
|
||||
use dr_plat::{DirRef, LocalStorage, Storage, WritableStorage};
|
||||
use dr_types::{FormatFilter, RootId};
|
||||
|
||||
/// Which root the card is granted as, and which the library is.
|
||||
///
|
||||
/// Two distinct ids in one `LocalStorage` would also work; separate storages
|
||||
/// keep the read-only source and the writable destination separate all the
|
||||
/// way down, which is the distinction `WritableStorage` exists to make.
|
||||
const CARD: RootId = RootId(9001);
|
||||
const LIBRARY: RootId = RootId(9002);
|
||||
const BACKUP: RootId = RootId(9003);
|
||||
|
||||
/// A cancel flag shared with the worker.
|
||||
pub type Cancel = Arc<AtomicBool>;
|
||||
|
||||
/// Everything an import needs to run.
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct Request {
|
||||
/// Where the card is mounted.
|
||||
pub card: PathBuf,
|
||||
/// The library root the template's folders are created under.
|
||||
pub library: PathBuf,
|
||||
/// The optional second destination (FR-CAT-10).
|
||||
pub backup: Option<PathBuf>,
|
||||
/// The catalog, opened by the worker for its duplicate queries.
|
||||
///
|
||||
/// A path rather than a connection: `rusqlite::Connection` is not `Sync`,
|
||||
/// and every other worker in this crate opens its own for the same reason.
|
||||
pub catalog: PathBuf,
|
||||
/// The library's root id *in the catalog*, which is what
|
||||
/// [`dr_catalog::set_content_hash`] matches on. Unrelated to [`LIBRARY`],
|
||||
/// which is this module's handle on the same folder.
|
||||
pub catalog_root: u64,
|
||||
/// Which file types to take off the card.
|
||||
pub filter: FormatFilter,
|
||||
pub options: Options,
|
||||
}
|
||||
|
||||
/// What the worker sends back.
|
||||
#[derive(Debug, Clone)]
|
||||
pub enum Message {
|
||||
/// The card has been walked. Sent once, before any copying.
|
||||
Surveyed { files: usize, bytes: u64 },
|
||||
/// One file finished — imported, skipped or failed.
|
||||
Progress {
|
||||
done: usize,
|
||||
total: usize,
|
||||
bytes: u64,
|
||||
name: String,
|
||||
},
|
||||
/// The run ended. Always the last message.
|
||||
Finished(Outcome),
|
||||
/// The run could not start at all.
|
||||
Failed(String),
|
||||
}
|
||||
|
||||
/// What an import did, in the form the interface reports it.
|
||||
#[derive(Debug, Clone, Default, PartialEq, Eq)]
|
||||
pub struct Outcome {
|
||||
pub imported: usize,
|
||||
pub duplicates: usize,
|
||||
pub failed: usize,
|
||||
/// Files dated by modification time rather than by the shutter
|
||||
/// (FR-NC-7a). Named rather than counted: the user needs to know *which*
|
||||
/// photographs are filed under a date that is not when they were taken.
|
||||
pub undated: Vec<String>,
|
||||
pub bytes: u64,
|
||||
pub cancelled: bool,
|
||||
/// The folders that were written into, for the message at the end and for
|
||||
/// the upload that follows (FR-NC-7a).
|
||||
pub folders: Vec<String>,
|
||||
/// Card files that may now be deleted, for a move-import (FR-NC-7b).
|
||||
///
|
||||
/// Carried out rather than acted on here: this crate knows the local write
|
||||
/// succeeded, and not whether the upload did.
|
||||
pub retirable: usize,
|
||||
}
|
||||
|
||||
/// Start an import.
|
||||
///
|
||||
/// Returns immediately; the work happens on a thread and reports through the
|
||||
/// channel. A dropped receiver does not stop the worker — the cancel flag
|
||||
/// does, and the caller holds it.
|
||||
pub fn spawn(request: Request, cancel: Cancel) -> Receiver<Message> {
|
||||
let (tx, rx) = std::sync::mpsc::channel();
|
||||
std::thread::Builder::new()
|
||||
.name("import".into())
|
||||
.spawn(move || {
|
||||
if let Err(e) = run(request, &cancel, &tx) {
|
||||
let _ = tx.send(Message::Failed(e));
|
||||
}
|
||||
})
|
||||
.expect("spawning the import worker");
|
||||
rx
|
||||
}
|
||||
|
||||
fn run(request: Request, cancel: &Cancel, tx: &Sender<Message>) -> Result<(), String> {
|
||||
let card = LocalStorage::with_root(CARD, request.card.clone());
|
||||
let library = LocalStorage::with_root(LIBRARY, request.library.clone());
|
||||
let backup = request
|
||||
.backup
|
||||
.as_ref()
|
||||
.map(|p| LocalStorage::with_root(BACKUP, p.clone()));
|
||||
|
||||
let candidates = survey(&card, CARD, &request.filter, &|| {
|
||||
cancel.load(Ordering::Relaxed)
|
||||
})
|
||||
.map_err(|e| format!("could not read the card: {e}"))?;
|
||||
|
||||
let bytes: u64 = candidates.iter().map(|c| c.size).sum();
|
||||
let _ = tx.send(Message::Surveyed {
|
||||
files: candidates.len(),
|
||||
bytes,
|
||||
});
|
||||
|
||||
// Opened after the survey rather than before: a card that turns out to be
|
||||
// empty should not also report a catalog it never needed.
|
||||
let catalog = dr_catalog::Catalog::open(&request.catalog)
|
||||
.map_err(|e| format!("could not open the catalog: {e}"))?;
|
||||
|
||||
let library_root = DirRef::root(LIBRARY);
|
||||
let backup_root = DirRef::root(BACKUP);
|
||||
let ingest = Ingest {
|
||||
source: &card,
|
||||
dest: &library,
|
||||
dest_root: &library_root,
|
||||
backup: backup.as_ref().map(|b| (b as &dyn WritableStorage, &backup_root)),
|
||||
options: request.options.clone(),
|
||||
};
|
||||
|
||||
let report = ingest.run(
|
||||
&candidates,
|
||||
&|c| probe(&card, c),
|
||||
&|key| is_duplicate(catalog.connection(), key),
|
||||
&|| cancel.load(Ordering::Relaxed),
|
||||
&mut |p| {
|
||||
let _ = tx.send(Message::Progress {
|
||||
done: p.done,
|
||||
total: p.total,
|
||||
bytes: p.bytes,
|
||||
name: p.name.clone(),
|
||||
});
|
||||
},
|
||||
);
|
||||
|
||||
// The digests, so the *next* import can answer the content tier without
|
||||
// reading anything. Best-effort and after the fact: the rows do not exist
|
||||
// until a scan has caught up, and a hash that misses its row costs one
|
||||
// wasted transfer next time rather than a failure now.
|
||||
record_digests(catalog.connection(), request.catalog_root, &report.imported);
|
||||
|
||||
let _ = tx.send(Message::Finished(summarise(&report)));
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Walk a card for files worth importing.
|
||||
///
|
||||
/// Recursive rather than `DCIM`-only: a card carries `DCIM/100CANON`, a phone
|
||||
/// carries `DCIM/Camera`, and a card that has been through a computer carries
|
||||
/// whatever somebody put on it. Walking the whole volume finds all three, and
|
||||
/// the format filter is what keeps the result to photographs.
|
||||
pub fn survey(
|
||||
storage: &dyn Storage,
|
||||
root: RootId,
|
||||
filter: &FormatFilter,
|
||||
cancel: &dyn Fn() -> bool,
|
||||
) -> Result<Vec<Candidate>, dr_plat::StorageError> {
|
||||
let mut out = Vec::new();
|
||||
let mut queue = vec![storage.root_dir(root)?];
|
||||
while let Some(dir) = queue.pop() {
|
||||
if cancel() {
|
||||
break;
|
||||
}
|
||||
// A directory that cannot be read is skipped rather than fatal: cards
|
||||
// carry vendor folders with odd permissions, and one of them must not
|
||||
// cost the user the other nine hundred photographs (FR-CAT-9).
|
||||
let entries = match storage.list(&dir) {
|
||||
Ok(e) => e,
|
||||
Err(e) => {
|
||||
log::warn!("skipping {dir}: {e}");
|
||||
continue;
|
||||
}
|
||||
};
|
||||
for entry in entries {
|
||||
if entry.meta.is_dir {
|
||||
if !dr_catalog::walk::is_excluded(&entry.meta.name) {
|
||||
if let dr_plat::Node::Dir(d) = entry.node {
|
||||
queue.push(d);
|
||||
}
|
||||
}
|
||||
continue;
|
||||
}
|
||||
if !filter.allows_name(&entry.meta.name) {
|
||||
continue;
|
||||
}
|
||||
if let dr_plat::Node::File(source) = entry.node {
|
||||
out.push(Candidate {
|
||||
source,
|
||||
name: entry.meta.name,
|
||||
size: entry.meta.size,
|
||||
// The listing's reading, which is the fallback the folder
|
||||
// template uses when a file carries no capture time.
|
||||
modified_at: entry.meta.mtime / 1000,
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
// A card walks in whatever order the filesystem hands back, which for FAT
|
||||
// is creation order per directory and arbitrary across them. Sorted so an
|
||||
// import reports its progress through a card in the order a photographer
|
||||
// would expect, and so two runs of the same card behave the same way.
|
||||
out.sort_by(|a, b| a.name.cmp(&b.name));
|
||||
Ok(out)
|
||||
}
|
||||
|
||||
/// Read one candidate's capture metadata.
|
||||
///
|
||||
/// A header read rather than the whole file: `dr_decode::HEADER_BYTES` is
|
||||
/// enough for EXIF on every format in the tree, and reading 80 MB per file to
|
||||
/// learn a date would take longer than the import itself.
|
||||
fn probe(card: &dyn Storage, candidate: &Candidate) -> Option<Shot> {
|
||||
let header = card
|
||||
.read_range(&candidate.source, 0..dr_decode::HEADER_BYTES)
|
||||
.ok()?;
|
||||
let md = dr_decode::metadata(&header).ok()?;
|
||||
Some(Shot {
|
||||
captured_at: md.captured_at,
|
||||
captured_offset: md.captured_offset,
|
||||
make: md.make,
|
||||
model: md.model,
|
||||
modified_at: candidate.modified_at,
|
||||
})
|
||||
}
|
||||
|
||||
/// FR-CAT-11's two tiers, against the catalog.
|
||||
///
|
||||
/// The camera string is composed the way the scan composes it
|
||||
/// ([`crate::library::camera_label`]) — the comparison is against what a scan
|
||||
/// wrote, so a second spelling here would silently disable the cheap tier.
|
||||
fn is_duplicate(conn: &rusqlite::Connection, key: &DupKey) -> bool {
|
||||
if let Some(digest) = &key.digest {
|
||||
return dr_catalog::seen_by_content(conn, digest).unwrap_or(false);
|
||||
}
|
||||
let camera = crate::library::camera_label(key.make.as_deref(), key.model.as_deref());
|
||||
dr_catalog::seen_by_metadata(
|
||||
conn,
|
||||
key.captured_at,
|
||||
camera.as_deref(),
|
||||
key.size,
|
||||
&key.original_name,
|
||||
)
|
||||
.unwrap_or(false)
|
||||
}
|
||||
|
||||
/// Store what the import learned that a scan cannot.
|
||||
fn record_digests(conn: &rusqlite::Connection, root: u64, imported: &[Imported]) {
|
||||
for image in imported {
|
||||
if let Err(e) = dr_catalog::set_content_hash(conn, root, image.written.key(), &image.digest)
|
||||
{
|
||||
log::warn!("recording the digest of {}: {e}", image.name);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Reduce a report to what the interface says about it.
|
||||
pub fn summarise(report: &Report) -> Outcome {
|
||||
let mut folders: Vec<String> = report
|
||||
.imported
|
||||
.iter()
|
||||
.map(|i| i.folders.join("/"))
|
||||
.collect();
|
||||
folders.sort();
|
||||
folders.dedup();
|
||||
|
||||
Outcome {
|
||||
imported: report.imported.len(),
|
||||
duplicates: report.duplicates.len(),
|
||||
failed: report.failed.len(),
|
||||
undated: report.undated.clone(),
|
||||
bytes: report.bytes,
|
||||
cancelled: report.cancelled,
|
||||
folders,
|
||||
retirable: report.retirable.len(),
|
||||
}
|
||||
}
|
||||
|
||||
/// The sentence shown when an import ends.
|
||||
///
|
||||
/// Says what happened to every file, because the counts that are zero are the
|
||||
/// ones worth not mentioning and the ones that are not are the whole point.
|
||||
/// A run that imported nothing says so rather than reporting success in the
|
||||
/// abstract.
|
||||
pub fn describe(outcome: &Outcome) -> String {
|
||||
let mut parts = Vec::new();
|
||||
if outcome.imported > 0 {
|
||||
parts.push(format!(
|
||||
"{} imported ({})",
|
||||
outcome.imported,
|
||||
crate::activity::describe_bytes(outcome.bytes)
|
||||
));
|
||||
}
|
||||
if outcome.duplicates > 0 {
|
||||
parts.push(format!("{} already in the library", outcome.duplicates));
|
||||
}
|
||||
if outcome.failed > 0 {
|
||||
parts.push(format!("{} failed", outcome.failed));
|
||||
}
|
||||
|
||||
let head = if parts.is_empty() {
|
||||
"Nothing to import".to_string()
|
||||
} else {
|
||||
parts.join(", ")
|
||||
};
|
||||
|
||||
let mut out = if outcome.cancelled {
|
||||
format!("Stopped — {head}")
|
||||
} else {
|
||||
head
|
||||
};
|
||||
|
||||
// Where they went, which is the question the folder template raises.
|
||||
match outcome.folders.len() {
|
||||
0 => {}
|
||||
1 => out.push_str(&format!(" into {}", outcome.folders[0])),
|
||||
n => out.push_str(&format!(" into {n} folders")),
|
||||
}
|
||||
|
||||
// FR-NC-7a: the mtime fallback is reported rather than silent.
|
||||
if !outcome.undated.is_empty() {
|
||||
out.push_str(&format!(
|
||||
". {} had no capture time and {} filed by modification date",
|
||||
outcome.undated.len(),
|
||||
if outcome.undated.len() == 1 {
|
||||
"was"
|
||||
} else {
|
||||
"were"
|
||||
}
|
||||
));
|
||||
}
|
||||
out
|
||||
}
|
||||
|
||||
/// Whether a folder looks like somewhere a card would be.
|
||||
///
|
||||
/// Used to warn before an import writes into the wrong place: a library root
|
||||
/// and a card mount point are both just directories, and the two are very
|
||||
/// easy to swap in a dialog.
|
||||
pub fn looks_like_a_card(path: &Path) -> bool {
|
||||
path.join("DCIM").is_dir()
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use dr_ingest::DateSource;
|
||||
|
||||
struct Tree(PathBuf);
|
||||
|
||||
impl Tree {
|
||||
fn new(name: &str) -> Self {
|
||||
let dir = std::env::temp_dir().join(format!(
|
||||
"dr-ui-import-{name}-{}-{:?}",
|
||||
std::process::id(),
|
||||
std::thread::current().id()
|
||||
));
|
||||
let _ = std::fs::remove_dir_all(&dir);
|
||||
std::fs::create_dir_all(&dir).unwrap();
|
||||
Tree(dir)
|
||||
}
|
||||
fn file(&self, rel: &str, bytes: &[u8]) -> &Self {
|
||||
let p = self.0.join(rel);
|
||||
std::fs::create_dir_all(p.parent().unwrap()).unwrap();
|
||||
std::fs::write(p, bytes).unwrap();
|
||||
self
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for Tree {
|
||||
fn drop(&mut self) {
|
||||
let _ = std::fs::remove_dir_all(&self.0);
|
||||
}
|
||||
}
|
||||
|
||||
fn survey_of(t: &Tree) -> Vec<Candidate> {
|
||||
let storage = LocalStorage::with_root(CARD, t.0.clone());
|
||||
survey(&storage, CARD, &FormatFilter::all(), &|| false).unwrap()
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_survey_finds_photographs_wherever_the_camera_put_them() {
|
||||
let t = Tree::new("survey");
|
||||
// Three real layouts at once: a Canon card, a phone, and files
|
||||
// somebody dropped at the top level.
|
||||
t.file("DCIM/100CANON/IMG_0001.CR3", b"a")
|
||||
.file("DCIM/Camera/PXL_0002.dng", b"bb")
|
||||
.file("loose.NEF", b"ccc");
|
||||
|
||||
let found = survey_of(&t);
|
||||
assert_eq!(found.len(), 3);
|
||||
assert_eq!(
|
||||
found.iter().map(|c| c.name.as_str()).collect::<Vec<_>>(),
|
||||
["IMG_0001.CR3", "PXL_0002.dng", "loose.NEF"]
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_survey_leaves_what_is_not_a_photograph() {
|
||||
let t = Tree::new("survey-filter");
|
||||
t.file("DCIM/100CANON/IMG_0001.CR3", b"a")
|
||||
// Every card carries these, and none of them is an import.
|
||||
.file("DCIM/100CANON/IMG_0001.THM", b"x")
|
||||
.file("MISC/settings.dat", b"x")
|
||||
.file("readme.txt", b"x");
|
||||
|
||||
let found = survey_of(&t);
|
||||
assert_eq!(found.len(), 1);
|
||||
assert_eq!(found[0].name, "IMG_0001.CR3");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_survey_is_ordered_the_same_way_twice() {
|
||||
// FAT hands directories back in creation order and volumes in none at
|
||||
// all, so an unsorted survey reports its progress through a card in an
|
||||
// order that changes between runs.
|
||||
let t = Tree::new("survey-order");
|
||||
t.file("DCIM/100CANON/IMG_0003.CR3", b"c")
|
||||
.file("DCIM/100CANON/IMG_0001.CR3", b"a")
|
||||
.file("DCIM/101CANON/IMG_0002.CR3", b"b");
|
||||
assert_eq!(survey_of(&t), survey_of(&t));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_survey_carries_what_the_folder_template_needs() {
|
||||
let t = Tree::new("survey-meta");
|
||||
t.file("DCIM/100CANON/IMG_0001.CR3", b"12345");
|
||||
let found = survey_of(&t);
|
||||
assert_eq!(found[0].size, 5);
|
||||
// Seconds, not milliseconds — a template handed a millisecond reading
|
||||
// files everything in the year 56000.
|
||||
let year = dr_types::civil_from_unix(found[0].modified_at).year;
|
||||
assert!((2000..2200).contains(&year), "mtime looked like {year}");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_survey_can_be_stopped() {
|
||||
let t = Tree::new("survey-cancel");
|
||||
t.file("DCIM/100CANON/IMG_0001.CR3", b"a");
|
||||
let storage = LocalStorage::with_root(CARD, t.0.clone());
|
||||
let found = survey(&storage, CARD, &FormatFilter::all(), &|| true).unwrap();
|
||||
assert!(found.is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_card_is_recognised_by_its_dcim_folder() {
|
||||
let t = Tree::new("looks-like");
|
||||
t.file("DCIM/100CANON/IMG_0001.CR3", b"a");
|
||||
assert!(looks_like_a_card(&t.0));
|
||||
assert!(!looks_like_a_card(&t.0.join("DCIM/100CANON")));
|
||||
}
|
||||
|
||||
// ---- what the interface says -----------------------------------------
|
||||
|
||||
fn outcome() -> Outcome {
|
||||
Outcome {
|
||||
imported: 3,
|
||||
bytes: 3 * 1024 * 1024,
|
||||
folders: vec!["2026/2026-08-22".into()],
|
||||
..Default::default()
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_finished_import_says_what_it_did_and_where() {
|
||||
let text = describe(&outcome());
|
||||
assert!(text.starts_with("3 imported ("), "{text}");
|
||||
assert!(text.ends_with(" into 2026/2026-08-22"), "{text}");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_card_that_was_already_imported_says_so_rather_than_nothing() {
|
||||
let o = Outcome {
|
||||
imported: 0,
|
||||
duplicates: 12,
|
||||
bytes: 0,
|
||||
folders: vec![],
|
||||
..Default::default()
|
||||
};
|
||||
// The failure mode this guards against is a dialog that closes with a
|
||||
// cheerful "done" after transferring nothing.
|
||||
assert_eq!(describe(&o), "12 already in the library");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn an_empty_card_is_not_reported_as_a_success() {
|
||||
assert_eq!(describe(&Outcome::default()), "Nothing to import");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_cancelled_run_leads_with_the_fact_that_it_stopped() {
|
||||
let o = Outcome {
|
||||
cancelled: true,
|
||||
..outcome()
|
||||
};
|
||||
assert!(describe(&o).starts_with("Stopped — 3 imported"), "{}", describe(&o));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn undated_files_are_named_in_the_result() {
|
||||
// FR-NC-7a: a file dated by mtime is filed under when it was last
|
||||
// written, which for a card through a reader is often today.
|
||||
let o = Outcome {
|
||||
undated: vec!["SCAN_01.TIF".into()],
|
||||
..outcome()
|
||||
};
|
||||
let text = describe(&o);
|
||||
assert!(text.contains("1 had no capture time"), "{text}");
|
||||
assert!(text.contains("was filed by modification date"), "{text}");
|
||||
|
||||
let many = Outcome {
|
||||
undated: vec!["a".into(), "b".into()],
|
||||
..outcome()
|
||||
};
|
||||
assert!(describe(&many).contains("2 had no capture time"));
|
||||
assert!(describe(&many).contains("were filed"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn several_days_on_one_card_are_counted_rather_than_listed() {
|
||||
let o = Outcome {
|
||||
folders: vec!["2026/2026-08-21".into(), "2026/2026-08-22".into()],
|
||||
..outcome()
|
||||
};
|
||||
assert!(describe(&o).ends_with("into 2 folders"), "{}", describe(&o));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_summary_collapses_a_days_worth_of_files_into_one_folder() {
|
||||
let report = Report {
|
||||
imported: (0..3)
|
||||
.map(|i| Imported {
|
||||
source: dr_types::SourceRef::Local {
|
||||
root: CARD,
|
||||
relative: format!("IMG_{i}.CR3"),
|
||||
},
|
||||
written: dr_types::SourceRef::Local {
|
||||
root: LIBRARY,
|
||||
relative: format!("2026/2026-08-22/IMG_{i}.CR3"),
|
||||
},
|
||||
folders: vec!["2026".into(), "2026-08-22".into()],
|
||||
name: format!("IMG_{i}.CR3"),
|
||||
digest: "x".into(),
|
||||
size: 10,
|
||||
dated_from: DateSource::Capture,
|
||||
backup: None,
|
||||
})
|
||||
.collect(),
|
||||
bytes: 30,
|
||||
..Default::default()
|
||||
};
|
||||
let o = summarise(&report);
|
||||
assert_eq!(o.imported, 3);
|
||||
assert_eq!(o.folders, ["2026/2026-08-22"]);
|
||||
}
|
||||
}
|
||||
@@ -26,6 +26,7 @@ mod develop;
|
||||
mod export;
|
||||
mod gradient;
|
||||
mod histogram;
|
||||
mod import;
|
||||
mod labels;
|
||||
mod library;
|
||||
mod library_ui;
|
||||
|
||||
+20
-10
@@ -1981,21 +1981,31 @@ fn collect_metadata(header: &[u8], req: &ThumbnailRequest, out: &mut Vec<Metadat
|
||||
image_id: req.image_id,
|
||||
captured_at: md.captured_at,
|
||||
captured_offset: md.captured_offset,
|
||||
camera: match (&md.make, &md.model) {
|
||||
// Bodies repeat the make inside the model ("Canon EOS 6D"), so
|
||||
// joining unconditionally yields "Canon Canon EOS 6D".
|
||||
(Some(make), Some(model)) if model.starts_with(make.as_str()) => {
|
||||
Some(model.trim().to_string())
|
||||
}
|
||||
(Some(make), Some(model)) => Some(format!("{} {}", make.trim(), model.trim())),
|
||||
(None, Some(model)) => Some(model.trim().to_string()),
|
||||
_ => None,
|
||||
},
|
||||
camera: camera_label(md.make.as_deref(), md.model.as_deref()),
|
||||
lens: md.lens.map(|l| l.trim().to_string()),
|
||||
iso: md.iso,
|
||||
});
|
||||
}
|
||||
|
||||
/// TRACES: FR-CAT-11
|
||||
/// The camera string the catalog stores, from an EXIF make and model.
|
||||
///
|
||||
/// One definition rather than one per caller, because an import's duplicate
|
||||
/// check compares against what a scan wrote (`dr_catalog::dedup`). Two
|
||||
/// spellings of the same body would not fail loudly — they would silently
|
||||
/// disable the cheap tier, and every re-inserted card would transfer in full
|
||||
/// before the digest caught it.
|
||||
pub fn camera_label(make: Option<&str>, model: Option<&str>) -> Option<String> {
|
||||
match (make, model) {
|
||||
// Bodies repeat the make inside the model ("Canon EOS 6D"), so
|
||||
// joining unconditionally yields "Canon Canon EOS 6D".
|
||||
(Some(make), Some(model)) if model.starts_with(make) => Some(model.trim().to_string()),
|
||||
(Some(make), Some(model)) => Some(format!("{} {}", make.trim(), model.trim())),
|
||||
(None, Some(model)) => Some(model.trim().to_string()),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
|
||||
/// Write a batch of dates and tell the UI, draining `found`.
|
||||
///
|
||||
/// Separate from the loop so the same path serves both the periodic flush and
|
||||
|
||||
Reference in New Issue
Block a user