//! Where things live on disk: the catalog and cache directories per //! account, and the bundled/downloaded face and inpainting model files. use dr_sync::Account; use std::path::PathBuf; /// Where the catalog for an account lives. /// /// Keyed by [`Account::namespace`] so two accounts do not share an index — /// two servers, two logins on one server, or two folders on one disk. Under /// the XDG data directory, not cache: the catalog is rebuildable but /// rebuilding it costs a full rescan, so it is not something to discard on a /// cache sweep. /// /// The namespace is the account's to compute, not this function's, because it /// is also frozen: it names the directory an existing install's catalog, /// thumbnail shards and un-uploaded sidecars are already in. pub fn catalog_path(account: &Account) -> PathBuf { data_root().join(account.namespace()).join("catalog.sqlite") } /// TRACES: FR-UI-8 /// Where this library's last position is remembered. /// /// Beside the catalog, under the same account namespace, for the reason /// `catalog_path` gives: a place belongs to one library, and two folders on one /// disk are two libraries with two positions. /// /// **The name matches the file that travels.** The copy on the server is /// `place.json` under `.darkroom-derived/`, and the exchange between them is a /// straight newest-wins swap of the same bytes — so calling the local one /// anything else would be one more thing to keep in step for no gain. /// /// In the data directory rather than the cache one. The consequence is milder /// here than for the sidecars `data_root` was moved for — losing a place costs /// a scroll, not a day of culling — but a file the system is free to delete is /// one that would rarely survive long enough to be read. pub fn place_path(account: &Account) -> PathBuf { data_root().join(account.namespace()).join("place.json") } /// TRACES: FR-DEV-6 /// The preset library as this device and this library's server last agreed /// on it — the base `PresetLibrary::merge` decides deletions against. /// /// Per library, beside the place, because each library's server keeps its /// own copy. In the data directory for the reason the place is: a base the /// system deleted would turn the next exchange into a union, and every /// preset deleted since the last one would come back. pub fn presets_base_path(account: &Account) -> PathBuf { data_root() .join(account.namespace()) .join("presets.base.drpl") } /// The directory every account's data hangs off. /// /// **Not the cache directory, and on Android that distinction is the whole /// point.** Neither `XDG_DATA_HOME` nor `HOME` is set there, so this used /// to fall through to `temp_dir()` — which Android resolves to the app's /// *cache*, a directory the system deletes without asking under storage /// pressure. /// /// What sits beside a catalog is not disposable. `sidecars/` is the /// commit point for every rating and edit made offline (see /// `sidecar_cache`), and `outbox/` holds exports the user has been told /// succeeded. A day of culling on a train, evicted by the OS before it ever /// reached the server, is the worst failure this application can have, and /// it would be silent. /// /// `dr_sync::account::declared_data_dir` is the persistent per-app directory /// the Android entry point establishes before anything opens a store. A /// desktop declares nothing and takes the platform's data directory from /// `dr_plat::dirs` — XDG on Linux, `%LOCALAPPDATA%` on Windows — which keeps /// the established location on Linux rather than moving anyone's catalog. pub(crate) fn data_root() -> PathBuf { match dr_sync::account::declared_data_dir() { Some(declared) => declared.join("darkroom"), None => dr_plat::base_dir(dr_plat::Base::Data), } } /// TRACES: FR-NC-10 | NFR-R1 /// Move an account's data out of the cache directory it used to live in. /// /// Called once at startup, before anything opens a store. The durable /// location changed when `catalog_path` stopped falling through to /// `temp_dir()` on Android, and without this the app would find no catalog, /// rescan a library of tens of thousands of images over the network, and /// re-fetch every thumbnail — while the old copy sat in a directory the /// system was free to delete. /// /// Worse than the cost: `sidecars/` and `outbox/` hold work that exists /// nowhere else. Abandoning them would discard offline ratings and edits that /// had not yet synced, silently, as an upgrade. /// /// A rename, not a copy: both directories are inside the app's own data on /// one filesystem, so it is atomic and cannot half-finish. If the destination /// already exists this does nothing — the migration has run, or this is a /// fresh install, and in neither case may it overwrite live data. pub fn migrate_legacy_cache_data(account: &Account) { // Only meaningful where the old fallback and the new one differ, which is // exactly the platform that had the problem. On a desktop with XDG set, // both resolve to the same place and this returns immediately. let legacy_base = std::env::temp_dir(); let Some(current) = catalog_path(account).parent().map(|p| p.to_path_buf()) else { return; }; let Some(account) = current.file_name() else { return; }; let legacy = legacy_base.join("darkroom").join(account); move_account_dir(&legacy, ¤t); } /// The move itself, separated so it can be tested against ordinary /// directories rather than the platform's idea of a cache. pub(super) fn move_account_dir(legacy: &std::path::Path, current: &std::path::Path) { if legacy == current || !legacy.is_dir() || current.exists() { return; } if let Some(parent) = current.parent() { if let Err(e) = std::fs::create_dir_all(parent) { log::warn!("preparing {}: {e}", parent.display()); return; } } match std::fs::rename(legacy, current) { Ok(()) => log::info!( "moved library data out of the cache: {} -> {}", legacy.display(), current.display() ), // Reported rather than fatal: a failed move leaves the old copy where // it was and costs a rescan, which is recoverable. Stopping the app // over it would not be. Err(e) => log::warn!( "could not move {} to {}: {e}", legacy.display(), current.display() ), } } /// Where an account's thumbnail shards live. /// /// Beside the catalog rather than in the cache directory: these sync to the /// server and are shared with other clients, so discarding them on a cache /// sweep would cost a re-download for everyone. pub fn thumbs_dir(account: &Account) -> PathBuf { catalog_path(account) .parent() .map(|p| p.join("thumbs")) .unwrap_or_else(|| std::env::temp_dir().join("darkroom-thumbs")) } /// Where the face models live, beside the catalog and the thumbnails. /// /// **Not shipped with the application** and not a build input: the InsightFace /// weights carry a non-commercial research grant incompatible with this /// project's licence, so the user obtains them and the app loads them from here /// (docs/dev/faces.md §2). An absent directory is the ordinary state of a fresh /// install, not an error. pub fn face_models_dir(account: &Account) -> PathBuf { catalog_path(account) .parent() .map(|p| p.join("models")) .unwrap_or_else(|| std::env::temp_dir().join("darkroom-models")) } /// Where face models live for *every* account on this device. /// /// Account-independent, unlike the catalog: a model is identified by /// `faces.model_id` (catalog.md §10.1) and not by who is signed in, so two /// accounts have no reason to hold two 15 MB copies of the same weights. This /// is also the only directory an Android build can populate for itself — the /// entry point extracts the APK's bundled copy here, and no session exists at /// that point to key a per-account path off. /// /// **That extraction races the first seconds of a launch and is meant to.** It /// is 41 MB of copying and it used to happen before the first frame, which on a /// tablet is an ANR (`darkroom-android`'s `install_bundled_models`). So a /// lookup here can answer "absent" for a model that is on its way; each file is /// renamed into place, so what a lookup never sees is a half-written one. pub fn shared_face_models_dir() -> PathBuf { data_root().join("models") } /// The names of the three eye-state models, as shipped in `models/face/`. /// /// The shape-fixed exports, like the face pair: `tools/fix-face-model-shapes.sh` /// pins each one's batch dimension to 1 before tract will analyse it. pub const LANDMARK_MODEL: &str = "2d106det_b1.onnx"; pub const EYE_MODEL: &str = "ocec_s_b1.onnx"; pub const SUNGLASSES_MODEL: &str = "sgc_l_48_b1.onnx"; /// Where the face models are on this machine. /// /// The detector and the embedder are required — see [`face_models`] — and /// the eye pair is not: a library indexes people without it and simply has /// no eye readings, which every reader treats as "unknown" rather than as a /// verdict (`dr_face::eyes`). Found beside the pair, in the same directory, /// so a hand-placed pair with no eye models beside it does not pick up the /// package's eye models from a directory it otherwise outranks. #[derive(Debug, Clone, PartialEq, Eq)] pub struct FaceModelPaths { pub detector: PathBuf, pub embedder: PathBuf, /// `(landmarks, eyes, sunglasses)` — [`LANDMARK_MODEL`], [`EYE_MODEL`] /// and [`SUNGLASSES_MODEL`] — where all three are present, or `None`. /// All or none: a partial reading is not a reading /// (`dr_face::classify::EyeModels`). pub eyes: Option<(PathBuf, PathBuf, PathBuf)>, } impl FaceModelPaths { /// Load the eye models, if there are any. /// /// A pair that is present but refuses to load is logged and treated as /// absent: a broken eye model must not stop the detector and embedder, /// which are the ones the People screen cannot do without. pub fn load_eyes(&self) -> Option { let (landmarks, eyes, sunglasses) = self.eyes.as_ref()?; match dr_face::EyeModels::from_paths(landmarks, eyes, sunglasses) { Ok(m) => Some(m), Err(e) => { log::warn!("face models: eye models present but unusable, indexing without: {e}"); None } } } } /// Where the inference engine keeps what it derives per device: the probe /// result and compiled engines (docs/dev/inference.md §4, §5). A peer of /// `thumbs`, not of the catalog: disposable, regenerable, never synced. pub fn inference_cache_dir() -> PathBuf { data_root().join("inference") } /// Somewhere to put a file that only needs to exist for a moment. /// /// Under the data root rather than `std::env::temp_dir()`, which on Android /// names a directory the app cannot write to. Nothing here survives a /// launch on purpose: whoever writes into it deletes what they wrote. pub fn scratch_dir() -> PathBuf { data_root().join("scratch") } /// The detector and embedder files, if both are present — and the eye /// models beside them, if those are. /// /// Both or neither: an embedder with no detector has nothing to embed, and a /// detector with no embedder finds faces it cannot tell apart. Reporting the /// pair missing is more useful than half-starting. /// /// The names are the **shape-fixed** exports, not what InsightFace ships: /// `tools/fix-face-model-shapes.sh` has to run over the originals first, /// because tract cannot parse either graph with a dynamic input. /// /// Searched in three places, most specific first: /// /// 1. **The account's own directory.** A library can be pinned to its own /// weights — a model swap is a `model_id` change and a re-index, and someone /// mid-migration needs one account's pair to stay put without holding the /// other back. /// 2. **The shared user directory.** Where a user drops a pair by hand, and /// where the Android entry point unpacks the copy the APK carries. /// 3. **The system directories.** Where a package installs them — the Arch /// package puts the pair in `/usr/share/darkroom/models`. Last, so anything /// the user placed themselves outranks what the package shipped. pub fn face_models(account: &Account, detector: dr_types::FaceDetector) -> Option { let pair = |dir: &PathBuf| { let det = dir.join(detector.file_name()); let embedder = dir.join("arcface_mbf_b1.onnx"); (det.is_file() && embedder.is_file()).then(|| { let landmarks = dir.join(LANDMARK_MODEL); let eyes = dir.join(EYE_MODEL); let sunglasses = dir.join(SUNGLASSES_MODEL); FaceModelPaths { detector: det, embedder, eyes: (landmarks.is_file() && eyes.is_file() && sunglasses.is_file()) .then_some((landmarks, eyes, sunglasses)), } }) }; let mut searched = vec![face_models_dir(account), shared_face_models_dir()]; searched.extend(system_face_models_dirs()); let found = searched.iter().find_map(pair); match &found { None => { // The settings page can only say "not installed". This is the line // that says where it looked, which is the whole of what a user with // the files in the wrong place needs — and the first thing to read // when a freshly installed package reports no model. log::warn!( "face models: no directory holds both {} and arcface_mbf_b1.onnx; searched {}", detector.file_name(), searched .iter() .map(|d| d.display().to_string()) .collect::>() .join(", ") ); } Some(m) if m.eyes.is_none() => { log::info!( "face models: no {LANDMARK_MODEL}, {EYE_MODEL} and {SUNGLASSES_MODEL} beside {}; indexing without eye readings", m.detector.display() ); } Some(_) => {} } found } /// The scene model, its vocabulary and its category descriptor, if all three /// are present. /// /// All three or none, for the same reason `face_models` insists on its pair: /// the graph alone decodes to 150 anonymous channels, and a descriptor naming /// classes a different model does not have is refused by /// `dr_segment::scene::parse_categories` anyway. Reporting the set missing is /// more useful than starting and failing at the first inference. /// /// Searched in the same three places, most specific first — the account's own /// directory, the shared one, then wherever a package installed them. Android /// only ever finds the second, which is where `install_bundled_models` unpacks /// the APK's copy before any store opens. /// /// Unlike the face weights this model *is* in the repository, so a desktop /// build from a complete checkout has it. Absent means either a checkout /// without `git lfs pull` or a package that chose not to carry 24 MB, and the /// scene tab reports itself unavailable rather than the app refusing to run. pub fn scene_model(account: &Account) -> Option<(PathBuf, PathBuf, PathBuf)> { fn set(dir: PathBuf) -> Option<(PathBuf, PathBuf, PathBuf)> { let model = dir.join("yolo26s-sem-ade20k.onnx"); let classes = dir.join("yolo26s-sem-ade20k.classes.json"); let categories = dir.join("categories.txt"); (model.is_file() && classes.is_file() && categories.is_file()) .then_some((model, classes, categories)) } set(face_models_dir(account)) .or_else(|| set(shared_face_models_dir())) .or_else(|| system_face_models_dirs().into_iter().find_map(set)) } /// Where a *package* may have installed the models. /// /// `dr_plat::system_data_dirs` has the rule per platform: `$XDG_DATA_DIRS` on /// Linux, the executable's own directory on Windows, nothing on Android — /// there the APK's copy is unpacked into the shared user directory instead, /// because an asset inside a package is not a path anything can read from /// (ARCH §6.9). Last in the search order on every platform, so a pair the /// user placed by hand outranks the installed one. pub(super) fn system_face_models_dirs() -> Vec { dr_plat::system_data_dirs() .into_iter() .map(|d| d.join("models")) .collect() } /// The file `name` in the shared user models directory, else in the first /// system directory that has it — the search every account-independent /// model lookup makes, and the one the inference engine is told about, so /// that a package's models under `/usr/share` are probed and compiled for /// exactly as a user's own would be. pub fn shared_model(name: &str) -> Option { std::iter::once(shared_face_models_dir()) .chain(system_face_models_dirs()) .map(|d| d.join(name)) .find(|p| p.is_file()) } /// TRACES: FR-MRG-4 /// The panorama border filler, as shipped in `models/inpaint/`. pub const INPAINT_MODEL: &str = "migan-512.onnx"; /// TRACES: FR-MRG-4 /// Where the border filler is, if it is anywhere: the shared user /// directory, then the system ones — the same search as the faces', minus /// the per-account step, because a fill is not identity-bearing and no /// library has a reason to pin its own. pub fn inpaint_model() -> Option { shared_model(INPAINT_MODEL) } /// TRACES: FR-DEV-3g /// The network a denoise method runs, as shipped in `models/denoise/`; /// `None` for the classical demosaic, which runs none. pub fn denoise_network( method: dr_pipeline::learned_denoise::Method, ) -> Option { use dr_pipeline::learned_denoise::Method; match method { Method::Bilinear => None, Method::Fast => Some(dr_denoise::FAST), Method::Medium => Some(dr_denoise::MEDIUM), Method::Best => Some(dr_denoise::BEST), } } /// TRACES: FR-DEV-3g /// Where a method's network is, by the border filler's search, and the /// context it needs. pub fn denoise_model( method: dr_pipeline::learned_denoise::Method, ) -> Option<(PathBuf, dr_denoise::Shipped)> { let net = denoise_network(method)?; Some((shared_model(net.file)?, net)) } #[cfg(test)] mod tests { use super::*; /// An account for the path tests, defaulting to the connector every /// existing install uses. fn account(endpoint: &str, user: &str) -> Account { Account::new("nextcloud", endpoint).with_login(user, user) } #[test] fn catalog_paths_separate_accounts() { // Two accounts on one machine must not share an index, or one // library's images appear in the other. let a = catalog_path(&account("https://cloud.example", "duncan")); let b = catalog_path(&account("https://cloud.example", "someone")); let c = catalog_path(&account("https://other.example", "duncan")); assert_ne!(a, b); assert_ne!(a, c); } #[test] fn a_folder_library_gets_its_own_catalog() { // The same rule across backends: a folder library on this machine // must not land in the directory a server account is already using. let server = catalog_path(&account("https://cloud.example", "duncan")); let folder = catalog_path(&Account::new("folder", "/mnt/photos")); assert_ne!(server, folder); assert_ne!( folder, catalog_path(&Account::new("folder", "/mnt/other-photos")) ); } #[test] fn a_legacy_cache_directory_is_moved_rather_than_abandoned() { // The upgrade hazard: `sidecars/` and `outbox/` hold work that exists // nowhere else, so leaving them behind in a directory the system may // empty would discard unsynced ratings and edits as a side effect of // installing a new build. let root = std::env::temp_dir().join(format!("dr-migrate-{}", std::process::id())); let _ = std::fs::remove_dir_all(&root); let legacy = root.join("darkroom").join("cloud-example-duncan"); std::fs::create_dir_all(legacy.join("sidecars")).unwrap(); std::fs::write(legacy.join("catalog.sqlite"), b"catalog").unwrap(); std::fs::write(legacy.join("sidecars").join("a.drsc"), b"an unsynced edit").unwrap(); let current = root.join("new").join("cloud-example-duncan"); move_account_dir(&legacy, ¤t); assert!(!legacy.exists(), "the old copy must not be left behind"); assert_eq!( std::fs::read(current.join("catalog.sqlite")).unwrap(), b"catalog" ); assert_eq!( std::fs::read(current.join("sidecars").join("a.drsc")).unwrap(), b"an unsynced edit", "an unsynced edit must survive the move" ); let _ = std::fs::remove_dir_all(&root); } #[test] fn a_migration_never_overwrites_live_data() { // Running twice, or a fresh install that already has a catalog. The // destination wins: it is the one the application is using. let root = std::env::temp_dir().join(format!("dr-migrate2-{}", std::process::id())); let _ = std::fs::remove_dir_all(&root); let legacy = root.join("old").join("acct"); let current = root.join("new").join("acct"); std::fs::create_dir_all(&legacy).unwrap(); std::fs::create_dir_all(¤t).unwrap(); std::fs::write(legacy.join("catalog.sqlite"), b"stale").unwrap(); std::fs::write(current.join("catalog.sqlite"), b"live").unwrap(); move_account_dir(&legacy, ¤t); assert_eq!( std::fs::read(current.join("catalog.sqlite")).unwrap(), b"live" ); let _ = std::fs::remove_dir_all(&root); } #[test] fn durable_data_never_lands_in_a_cache_directory() { // The fault this guards against is silent and total: on Android the // fallback used to be `temp_dir()`, which resolves to the app's cache // — a directory the system empties under storage pressure. Beside this // catalog sit `sidecars/`, the commit point for every offline rating // and edit, and `outbox/`, holding exports the user was told had // succeeded. Losing a day of culling to an OS housekeeping pass, with // no error and no trace, is the worst outcome this application has. let path = catalog_path(&account("https://cloud.example", "duncan")); let text = path.to_string_lossy().to_lowercase(); assert!( !text.contains("/cache/") && !text.contains("/tmp/"), "the catalog and everything beside it must be durable, got {}", path.display() ); } #[test] fn the_outbox_and_sidecars_sit_beside_the_catalog() { // Stated as a test because three separate call sites derive their // location by taking this path's parent, and a change here moves all // of them at once — including the two holding unsynced user work. let catalog = catalog_path(&account("https://cloud.example", "duncan")); let parent = catalog.parent().expect("a parent"); assert_eq!( crate::export::outbox_dir(&account("https://cloud.example", "duncan")), parent.join("outbox") ); } #[test] fn catalog_path_is_filesystem_safe() { let p = catalog_path(&account("https://cloud.example.com:8443/nc", "duncan")); let s = p.to_string_lossy(); assert!(!s.contains("://")); assert!(!s.contains(':') || cfg!(windows)); } #[test] fn thumbs_live_beside_the_catalog_not_in_the_cache() { // They sync to the server and are shared with other clients, so a // cache sweep must not discard them. let cat = catalog_path(&account("https://cloud.example", "duncan")); let thumbs = thumbs_dir(&account("https://cloud.example", "duncan")); assert_eq!(thumbs.parent(), cat.parent()); } }