Date a photograph from its name when its header has none

WhatsApp strips every EXIF tag and names the file "WhatsApp Image
2023-06-15 at 07.00.42.jpeg"; Windows Phone, Android cameras and
darktable's import put the date in the name too. Those images sorted
after everything else and were absent from the timeline.

name_dates reads a date (and a time, when one follows) from the file
name, then from the innermost folder that states one. A sequence number
after a date is not read as a time, and a bare year folder is not a date.
EXIF always wins: only examined rows still undated are filled.

The sweep and the metadata repair fill as they mark an image examined,
and the open backfill fills catalogs examined by earlier builds. On the
reference library that takes 274 undated images to 10; the no-op case is
a seek on images_captured, 0.6 ms an open.

core/dr-catalog/examples/catalog_bench.rs

@@ -27,7 +27,7 @@
 use std::path::PathBuf;
 use std::time::{Duration, Instant};
 
-use dr_catalog::{keywords, rating, schema, Catalog};
+use dr_catalog::{keywords, name_dates, rating, schema, Catalog};
 
 fn main() {
     let mut args: Vec<String> = std::env::args().skip(1).collect();
@@ -108,6 +108,9 @@ fn main() {
     time("  keywords::adopt_orphan_terms", 20, || {
         keywords::adopt_orphan_terms(conn).unwrap();
     });
+    time("  name_dates::fill", 20, || {
+        name_dates::fill(conn, None).unwrap();
+    });
 
     interactive(conn);
 

core/dr-catalog/src/lib.rs

@@ -50,6 +50,7 @@ pub mod faces;
 pub mod jobs;
 pub mod keywords;
 pub mod merge;
+pub mod name_dates;
 pub mod query;
 pub mod rating;
 pub mod recovery;

core/dr-catalog/src/name_dates.rs

@@ -0,0 +1,387 @@
+//! TRACES: FR-CAT-5
+//! A capture time read from the file's name, for an image whose header has
+//! none.
+//!
+//! # Why
+//!
+//! A photograph with no EXIF date sorts after everything else, so it is lost
+//! at the end of the grid and absent from the timeline. The files that end up
+//! there are rarely without a date — they are without *EXIF*: WhatsApp strips
+//! every tag and names the file `WhatsApp Image 2023-06-15 at 07.00.42.jpeg`,
+//! a Windows Phone wrote `WP_20140922_14_16_27_Pro.jpg`, a phone camera
+//! `IMG_20190812_153012.jpg`, and darktable's import renames to
+//! `20230629_0001.jpeg`. On the reference library 250 of 274 undated images
+//! carried their date in the name or in the folder above it.
+//!
+//! # What is accepted
+//!
+//! A date is `YYYYMMDD` as a whole run of digits, or `YYYY`, `MM` and `DD`
+//! joined by `-`, `_` or `.`. A time may follow it — `HHMMSS` as one run (or
+//! nine digits, milliseconds appended), or three two-digit runs joined by
+//! `-`, `_`, `.` or `:` — after `_`, `-`, `.`, `T`, a space or ` at `.
+//! Anything else after the date leaves it at midnight: `_0059` in
+//! `20230628_0059` is a sequence number, not 00:59, and reading it as a time
+//! would invent one.
+//!
+//! The name is tried first and then each folder above it, innermost first —
+//! `2016/2016-11-11/IMG_7910.jpg` is dated by its folder. A bare year folder
+//! is not a date: putting a photograph at 1 January is a wrong answer, and an
+//! undated one at least says it does not know.
+//!
+//! The reading is wall-clock time with no zone, stored as EXIF's is
+//! (`dr_decode::parse_exif_datetime`), and EXIF always wins: this only fills
+//! rows whose `captured_at` is still empty.
+
+use rusqlite::Connection;
+
+use crate::CatalogError;
+
+/// The capture time a path's name states, as wall-clock Unix seconds.
+pub fn date_from_path(source_ref: &str) -> Option<i64> {
+    let mut parts = source_ref.rsplit(['/', '\\']);
+    let name = parts.next()?;
+    let stem = name.rsplit_once('.').map_or(name, |(stem, _)| stem);
+    date_in(stem).or_else(|| parts.find_map(date_in))
+}
+
+/// Date every examined, undated image whose name states one.
+///
+/// `only` limits the pass to the images just examined — what the sweep hands
+/// in — and `None` visits every undated image, which is the backfill's case.
+/// Both read the undated side alone (`images_captured` answers
+/// `captured_at IS NULL` with a seek), never the library.
+///
+/// Returns how many images were dated.
+pub fn fill(conn: &Connection, only: Option<&[i64]>) -> Result<usize, CatalogError> {
+    let rows: Vec<(i64, String)> = match only {
+        None => {
+            let mut stmt = conn.prepare(
+                "SELECT id, source_ref FROM images
+                  WHERE captured_at IS NULL AND metadata_state >= 2",
+            )?;
+            let rows = stmt
+                .query_map([], |r| Ok((r.get(0)?, r.get(1)?)))?
+                .collect::<Result<_, _>>()?;
+            rows
+        }
+        Some(ids) => {
+            let mut stmt = conn.prepare_cached(
+                "SELECT source_ref FROM images
+                  WHERE id = ?1 AND captured_at IS NULL AND metadata_state >= 2",
+            )?;
+            let mut rows = Vec::new();
+            for &id in ids {
+                let mut q = stmt.query([id])?;
+                if let Some(r) = q.next()? {
+                    rows.push((id, r.get(0)?));
+                }
+            }
+            rows
+        }
+    };
+
+    let dated: Vec<(i64, i64)> = rows
+        .iter()
+        .filter_map(|(id, path)| date_from_path(path).map(|at| (*id, at)))
+        .collect();
+    if dated.is_empty() {
+        return Ok(0);
+    }
+
+    // A savepoint rather than a transaction, so a caller already inside one
+    // can still call this: the backfill's 250 rows are one commit, not 250.
+    conn.execute_batch("SAVEPOINT name_dates")?;
+    let written = (|| {
+        let mut stmt = conn.prepare_cached(
+            "UPDATE images SET captured_at = ?2 WHERE id = ?1 AND captured_at IS NULL",
+        )?;
+        let mut n = 0;
+        for (id, at) in &dated {
+            n += stmt.execute(rusqlite::params![id, at])?;
+        }
+        Ok::<_, CatalogError>(n)
+    })();
+    match written {
+        Ok(n) => {
+            conn.execute_batch("RELEASE name_dates")?;
+            Ok(n)
+        }
+        Err(e) => {
+            let _ = conn.execute_batch("ROLLBACK TO name_dates; RELEASE name_dates");
+            Err(e)
+        }
+    }
+}
+
+/// The first date, with its time if one follows, in one name component.
+fn date_in(s: &str) -> Option<i64> {
+    let b = s.as_bytes();
+    let mut i = 0;
+    while i < b.len() {
+        // Only at the start of a run of digits: a date inside a longer number
+        // is a coincidence, not a date.
+        if b[i].is_ascii_digit() && (i == 0 || !b[i - 1].is_ascii_digit()) {
+            if let Some(at) = date_at(b, i) {
+                return Some(at);
+            }
+        }
+        i += 1;
+    }
+    None
+}
+
+/// A date starting at `i`, and the time after it if there is one.
+fn date_at(b: &[u8], i: usize) -> Option<i64> {
+    let run = digits(b, i);
+    let ((y, mo, d), after) = match run.len() {
+        // YYYYMMDD, or YYYYMMDDHHMMSS written as one number.
+        8 | 14 => ((num(&run[..4]), num(&run[4..6]), num(&run[6..8])), i + 8),
+        4 => {
+            let sep = |at: usize| matches!(b.get(at), Some(b'-' | b'_' | b'.'));
+            let mo_at = i + 4 + 1;
+            let d_at = mo_at + 2 + 1;
+            if !(sep(i + 4) && digits(b, mo_at).len() == 2 && sep(mo_at + 2))
+                || digits(b, d_at).len() != 2
+            {
+                return None;
+            }
+            (
+                (num(run), num(&b[mo_at..mo_at + 2]), num(&b[d_at..d_at + 2])),
+                d_at + 2,
+            )
+        }
+        _ => return None,
+    };
+    let day = civil_days(y, mo, d)?;
+
+    let time = if run.len() == 14 {
+        hms(num(&run[8..10]), num(&run[10..12]), num(&run[12..14]))
+    } else {
+        time_at(b, after)
+    };
+    Some(day * 86_400 + time.unwrap_or(0))
+}
+
+/// The time following a date that ends at `i`, as seconds into the day.
+fn time_at(b: &[u8], i: usize) -> Option<i64> {
+    let rest = &b[i..];
+    let start = if rest.starts_with(b" at ") {
+        i + 4
+    } else if matches!(rest.first(), Some(b'_' | b'-' | b'.' | b'T' | b' ')) {
+        i + 1
+    } else {
+        return None;
+    };
+
+    let run = digits(b, start);
+    match run.len() {
+        // HHMMSS, or with milliseconds appended (Pixel's PXL_…_123456789).
+        6 | 9 => hms(num(&run[..2]), num(&run[2..4]), num(&run[4..6])),
+        2 => {
+            let sep = |at: usize| matches!(b.get(at), Some(b'-' | b'_' | b'.' | b':'));
+            let (m_at, s_at) = (start + 3, start + 6);
+            if !(sep(start + 2) && digits(b, m_at).len() == 2 && sep(m_at + 2))
+                || digits(b, s_at).len() != 2
+            {
+                return None;
+            }
+            hms(num(run), num(&b[m_at..m_at + 2]), num(&b[s_at..s_at + 2]))
+        }
+        _ => None,
+    }
+}
+
+/// The run of ASCII digits starting at `i`.
+fn digits(b: &[u8], i: usize) -> &[u8] {
+    let rest = b.get(i..).unwrap_or(&[]);
+    let n = rest.iter().take_while(|c| c.is_ascii_digit()).count();
+    &rest[..n]
+}
+
+fn num(d: &[u8]) -> i64 {
+    d.iter().fold(0, |n, c| n * 10 + i64::from(c - b'0'))
+}
+
+fn hms(h: i64, m: i64, s: i64) -> Option<i64> {
+    ((0..24).contains(&h) && (0..60).contains(&m) && (0..61).contains(&s))
+        .then_some(h * 3_600 + m * 60 + s)
+}
+
+/// Days since 1970-01-01 for a valid civil date, `None` for anything else.
+///
+/// The year range is EXIF's (`parse_exif_datetime`): wide enough for scanned
+/// film, narrow enough that a counter such as `12345678` is not a date.
+fn civil_days(y: i64, mo: i64, d: i64) -> Option<i64> {
+    let leap = y % 4 == 0 && (y % 100 != 0 || y % 400 == 0);
+    let month_len = match mo {
+        1 | 3 | 5 | 7 | 8 | 10 | 12 => 31,
+        4 | 6 | 9 | 11 => 30,
+        2 if leap => 29,
+        2 => 28,
+        _ => return None,
+    };
+    if !(1900..=2200).contains(&y) || !(1..=month_len).contains(&d) {
+        return None;
+    }
+    let y_adj = if mo <= 2 { y - 1 } else { y };
+    let era = y_adj.div_euclid(400);
+    let yoe = y_adj - era * 400;
+    let mp = (mo + 9) % 12;
+    let doy = (153 * mp + 2) / 5 + d - 1;
+    let doe = yoe * 365 + yoe / 4 - yoe / 100 + doy;
+    Some(era * 146_097 + doe - 719_468)
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    /// Wall-clock seconds for a date and time, the expected side of each case.
+    fn at(y: i64, mo: i64, d: i64, h: i64, mi: i64, s: i64) -> Option<i64> {
+        Some(civil_days(y, mo, d).unwrap() * 86_400 + h * 3_600 + mi * 60 + s)
+    }
+
+    #[test]
+    fn the_names_in_the_reference_library_are_read() {
+        // Every shape here is a file that sat undated at the end of the grid.
+        for (path, want) in [
+            (
+                "PhotosRaw/alps trip/alps whatsapp/WhatsApp Image 2023-06-15 at 07.00.42.jpeg",
+                at(2023, 6, 15, 7, 0, 42),
+            ),
+            (
+                "PhotosRaw/alps trip/alps whatsapp/WhatsApp Image 2023-06-17 at 12.45.52 (1).jpeg",
+                at(2023, 6, 17, 12, 45, 52),
+            ),
+            (
+                "PhotosRaw/WP_20140922_14_16_27_Pro.jpg",
+                at(2014, 9, 22, 14, 16, 27),
+            ),
+            // A sequence number after the date is not a time.
+            (
+                "PhotosRaw/Darktable/20230629_no_name/20230629_0001.jpeg",
+                at(2023, 6, 29, 0, 0, 0),
+            ),
+            ("PhotosRaw/20230628_0059.jpg", at(2023, 6, 28, 0, 0, 0)),
+            (
+                "PhotosRaw/backdrops/IMG_20130625_0021.jpg",
+                at(2013, 6, 25, 0, 0, 0),
+            ),
+            (
+                "PhotosRaw/alps trip/20230628_0641 - 20230628_0661.jpg",
+                at(2023, 6, 28, 0, 0, 0),
+            ),
+        ] {
+            assert_eq!(date_from_path(path), want, "{path}");
+        }
+    }
+
+    #[test]
+    fn common_camera_and_app_names_are_read() {
+        for (path, want) in [
+            ("IMG_20190812_153012.jpg", at(2019, 8, 12, 15, 30, 12)),
+            ("PXL_20210101_123456789.jpg", at(2021, 1, 1, 12, 34, 56)),
+            (
+                "Screenshot_2021-03-04-12-30-45.png",
+                at(2021, 3, 4, 12, 30, 45),
+            ),
+            (
+                "Screenshot from 2021-03-04 12-30-45.png",
+                at(2021, 3, 4, 12, 30, 45),
+            ),
+            ("IMG-20210304-WA0001.jpg", at(2021, 3, 4, 0, 0, 0)),
+            ("20210304143012.jpg", at(2021, 3, 4, 14, 30, 12)),
+            ("2019.12.25 party.jpg", at(2019, 12, 25, 0, 0, 0)),
+            ("signal-2022-01-02-101112.jpg", at(2022, 1, 2, 10, 11, 12)),
+            ("2022-01-02T10:11:12.jpg", at(2022, 1, 2, 10, 11, 12)),
+        ] {
+            assert_eq!(date_from_path(path), want, "{path}");
+        }
+    }
+
+    #[test]
+    fn a_folder_dates_a_name_that_does_not() {
+        assert_eq!(
+            date_from_path("PhotosRaw/2016/2016-11-11/IMG_7910.jpg"),
+            at(2016, 11, 11, 0, 0, 0)
+        );
+        // The innermost folder that states a date wins.
+        assert_eq!(
+            date_from_path("2016-01-01 trip/2016-01-03/_MG_1.jpg"),
+            at(2016, 1, 3, 0, 0, 0)
+        );
+        // The name beats its folder.
+        assert_eq!(
+            date_from_path("2016-11-11/IMG_20161112_080000.jpg"),
+            at(2016, 11, 12, 8, 0, 0)
+        );
+    }
+
+    #[test]
+    fn numbers_that_are_not_dates_are_left_alone() {
+        for path in [
+            "PhotosRaw/_MG_9002.jpg",
+            "PhotosRaw/scanning/fau_2.jpg",
+            // A year folder is not a day.
+            "PhotosRaw/2016/_MG_1.jpg",
+            "IMG_1999.jpg",
+            "DSC_12345678.jpg",  // month 56
+            "20230230_0001.jpg", // 30 February
+            "120230615.jpg",     // the date is inside a longer number
+            "1612345678901.jpg", // a millisecond epoch, not a civil date
+            "2023-6-15.jpg",     // a one-digit month is too loose to trust
+        ] {
+            assert_eq!(date_from_path(path), None, "{path}");
+        }
+    }
+
+    #[test]
+    fn a_time_that_cannot_be_is_dropped_and_the_date_kept() {
+        assert_eq!(
+            date_from_path("20230615_256199.jpg"),
+            at(2023, 6, 15, 0, 0, 0)
+        );
+    }
+
+    #[test]
+    fn fill_dates_only_examined_undated_rows_and_never_overrides_exif() {
+        let c = Connection::open_in_memory().unwrap();
+        crate::schema::migrate(&c).unwrap();
+        c.execute(
+            "INSERT INTO roots(id, kind, label) VALUES (1, 'remote', 'lib')",
+            [],
+        )
+        .unwrap();
+        // (id, name, captured_at, metadata_state)
+        for (id, name, captured, state) in [
+            (1i64, "IMG_20190812_153012.jpg", None, 2i64),
+            // EXIF already answered; the name disagrees and loses.
+            (2, "IMG_20190812_153012b.jpg", Some(42i64), 2),
+            // Not yet examined: EXIF may still come, so the name waits.
+            (3, "IMG_20190813_000000.jpg", None, 1),
+            (4, "_MG_9002.jpg", None, 2),
+        ] {
+            c.execute(
+                "INSERT INTO images(id, root_id, source_ref, captured_at, metadata_state, added_at)
+                 VALUES (?1, 1, ?2, ?3, ?4, 0)",
+                rusqlite::params![id, name, captured, state],
+            )
+            .unwrap();
+        }
+        let captured = |id: i64| -> Option<i64> {
+            c.query_row("SELECT captured_at FROM images WHERE id = ?1", [id], |r| {
+                r.get(0)
+            })
+            .unwrap()
+        };
+
+        assert_eq!(fill(&c, Some(&[2, 3, 4])).unwrap(), 0);
+        assert_eq!(fill(&c, None).unwrap(), 1);
+        assert_eq!(captured(1), at(2019, 8, 12, 15, 30, 12));
+        assert_eq!(captured(2), Some(42));
+        assert_eq!(captured(3), None);
+        assert_eq!(captured(4), None);
+        // Nothing left to do is a no-op, not a rewrite.
+        assert_eq!(fill(&c, None).unwrap(), 0);
+    }
+}

core/dr-catalog/src/schema.rs

@@ -356,6 +356,15 @@ pub fn backfill(conn: &Connection) -> Result<Vec<(&'static str, usize)>, Catalog
         out.push(("keyword_terms", n));
     }
 
+    // TRACES: FR-CAT-5
+    // A date from the file's name for every examined image EXIF left undated.
+    // The sweep does this as it examines each image; this is for the images
+    // examined by a build that did not, and reads the undated side alone.
+    let n = crate::name_dates::fill(conn, None)?;
+    if n > 0 {
+        out.push(("dates_from_names", n));
+    }
+
     Ok(out)
 }
 

ui/dr-ui/src/library/sweep.rs

@@ -522,11 +522,16 @@ pub(super) fn flush_sweep(catalog: &Catalog, found: &mut Vec<MetadataFound>) {
         .filter(|m| m.captured_at.is_none())
         .map(|m| m.image_id)
         .collect();
-    for id in ids {
+    for &id in &ids {
         let _ = catalog
             .connection()
             .execute("UPDATE images SET metadata_state = 2 WHERE id = ?1", [id]);
     }
+    // A header with no date is not a photograph with no date: WhatsApp and
+    // most re-exports strip EXIF and keep the date in the name.
+    if let Err(e) = dr_catalog::name_dates::fill(catalog.connection(), Some(&ids)) {
+        log::warn!("sweep: dating from names: {e}");
+    }
     found.clear();
 }
 

ui/dr-ui/src/repairs.rs

@@ -740,6 +740,9 @@ fn metadata(
                 [target.image.0 as i64],
             )
             .map_err(|e| Failure::Other(e.to_string()))?;
+        // The name, when the header had no date — as the sweep does.
+        dr_catalog::name_dates::fill(catalog.connection(), Some(&[target.image.0 as i64]))
+            .map_err(|e| Failure::Other(e.to_string()))?;
     }
     Ok(1)
 }