Sync camera profiles through the library, and name the tone curves
camera-profiles.md §13: the derived sync pass gains a profiles step, after the catalog and before the place, that exchanges the profiles directory with <library>/.darkroom-derived/profiles. Profiles are immutable and named for what they hold, so name and size decide: it uploads what the server lacks or holds at another size and downloads what this device lacks — parsed before it is kept, written beside its name and renamed — then reloads the set, so a profile copied out of a DNG on the desktop renders the body's CR2s on the tablet after its next sync. Like the place it never fails the pass. Not a catalog table: a schema change would stop an older peer merging at all. Labels for the view transform's new curve choice: Curve, Camera Raw, Sigmoid.
This commit is contained in:
@@ -91,6 +91,11 @@ pub struct SyncReport {
|
||||
/// is a fact only the other device knew.
|
||||
pub place_adopted: bool,
|
||||
pub place_uploaded: bool,
|
||||
/// TRACES: FR-DEV-3e
|
||||
/// Camera profiles exchanged with the library's `profiles` folder
|
||||
/// (camera-profiles.md §13).
|
||||
pub profiles_uploaded: usize,
|
||||
pub profiles_downloaded: usize,
|
||||
}
|
||||
|
||||
impl SyncReport {
|
||||
@@ -103,6 +108,7 @@ impl SyncReport {
|
||||
|| self.face_shards_uploaded > 0
|
||||
|| self.face_shards_downloaded > 0
|
||||
|| self.place_adopted
|
||||
|| self.profiles_downloaded > 0
|
||||
}
|
||||
}
|
||||
|
||||
@@ -224,6 +230,16 @@ async fn run(
|
||||
let _ = tx.send(SyncMessage::Status("checking thumbnails…".into()));
|
||||
sync_shards(backend, &base, thumbs_dir, scratch, &mut report).await?;
|
||||
|
||||
// TRACES: FR-DEV-3e
|
||||
// The camera profiles, so a profile copied out of a DNG on one device
|
||||
// renders that body's raws on every device (camera-profiles.md §13).
|
||||
// After the catalog and before the place; like the place it never fails
|
||||
// the pass.
|
||||
if let Some(dir) = dr_decode::dcp::profiles_directory() {
|
||||
let _ = tx.send(SyncMessage::Status("checking camera profiles…".into()));
|
||||
sync_profiles(backend, &base, &dir, &mut report).await;
|
||||
}
|
||||
|
||||
// TRACES: FR-UI-8
|
||||
// Last, and it costs one small GET plus at most one small PUT. Last because
|
||||
// it is the only thing here that is not derived state and so the only thing
|
||||
@@ -995,6 +1011,117 @@ async fn sync_place(
|
||||
}
|
||||
}
|
||||
|
||||
/// TRACES: FR-DEV-3e
|
||||
/// Exchange camera profiles with `<derived>/profiles` (camera-profiles.md
|
||||
/// §13).
|
||||
///
|
||||
/// Profiles are immutable and named for what they hold (`<camera>
|
||||
/// <profile>.dcp`, `dcp::save`), so a name and a size say whether two copies
|
||||
/// are the same: upload what the server lacks or holds at another size,
|
||||
/// download what this device lacks. A download is parsed before it is kept,
|
||||
/// and written beside its name then renamed, because the profile loader
|
||||
/// reads every `.dcp` in the folder and a half-written one would be skipped
|
||||
/// with a warning rather than retried. Never fails the pass: a profile that
|
||||
/// did not travel this time travels next time.
|
||||
async fn sync_profiles(
|
||||
backend: &dyn RemoteBackend,
|
||||
base: &RemotePath,
|
||||
local_dir: &Path,
|
||||
report: &mut SyncReport,
|
||||
) {
|
||||
let dir = RemotePath::new(format!("{}/profiles", base.as_str()));
|
||||
let is_profile = |name: &str| {
|
||||
Path::new(name)
|
||||
.extension()
|
||||
.is_some_and(|x| x.eq_ignore_ascii_case("dcp"))
|
||||
};
|
||||
|
||||
let remote: std::collections::HashMap<String, u64> = backend
|
||||
.list(&dir, None)
|
||||
.await
|
||||
.map(|entries| {
|
||||
entries
|
||||
.into_iter()
|
||||
.filter(|e| e.kind == dr_sync::EntryKind::File && is_profile(e.path.name()))
|
||||
.map(|e| (e.path.name().to_string(), e.size))
|
||||
.collect()
|
||||
})
|
||||
// Absent until the first device uploads one; an empty answer.
|
||||
.unwrap_or_default();
|
||||
|
||||
let local: std::collections::HashMap<String, PathBuf> = std::fs::read_dir(local_dir)
|
||||
.map(|entries| {
|
||||
entries
|
||||
.flatten()
|
||||
.filter_map(|e| {
|
||||
let name = e.file_name().to_string_lossy().into_owned();
|
||||
is_profile(&name).then(|| (name, e.path()))
|
||||
})
|
||||
.collect()
|
||||
})
|
||||
.unwrap_or_default();
|
||||
|
||||
// ---- upload ----------------------------------------------------------
|
||||
let to_upload: Vec<(&String, &PathBuf)> = local
|
||||
.iter()
|
||||
.filter(|(name, path)| {
|
||||
let size = std::fs::metadata(path).map(|m| m.len()).unwrap_or(0);
|
||||
remote.get(*name) != Some(&size)
|
||||
})
|
||||
.collect();
|
||||
if !to_upload.is_empty() {
|
||||
let _ = backend.create_dir(&dir).await;
|
||||
}
|
||||
for (name, path) in to_upload {
|
||||
let Ok(bytes) = std::fs::read(path) else {
|
||||
continue;
|
||||
};
|
||||
let target = RemotePath::new(format!("{}/{name}", dir.as_str()));
|
||||
match backend.put(&target, bytes, None).await {
|
||||
Ok(_) => report.profiles_uploaded += 1,
|
||||
Err(e) => log::debug!("uploading camera profile {name}: {e}"),
|
||||
}
|
||||
}
|
||||
|
||||
// ---- download --------------------------------------------------------
|
||||
for name in remote.keys().filter(|n| !local.contains_key(*n)) {
|
||||
let source = RemotePath::new(format!("{}/{name}", dir.as_str()));
|
||||
let bytes = match read_derived(backend, &source).await {
|
||||
Ok(b) => b,
|
||||
Err(e) => {
|
||||
log::debug!("fetching camera profile {name}: {e}");
|
||||
continue;
|
||||
}
|
||||
};
|
||||
if let Err(why) = dr_decode::dcp::Dcp::parse(&bytes) {
|
||||
log::warn!("camera profile {name} on the server is not one: {why}");
|
||||
continue;
|
||||
}
|
||||
let path = local_dir.join(name);
|
||||
let partial = local_dir.join(format!("{name}.part"));
|
||||
let written = std::fs::create_dir_all(local_dir)
|
||||
.and_then(|()| std::fs::write(&partial, &bytes))
|
||||
.and_then(|()| std::fs::rename(&partial, &path));
|
||||
match written {
|
||||
Ok(()) => report.profiles_downloaded += 1,
|
||||
Err(e) => {
|
||||
let _ = std::fs::remove_file(&partial);
|
||||
log::warn!("saving camera profile {name}: {e}");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if report.profiles_downloaded > 0 {
|
||||
log::info!(
|
||||
"camera profiles: {} fetched from the library",
|
||||
report.profiles_downloaded
|
||||
);
|
||||
// Decodes from now on see them; one already in flight keeps the set
|
||||
// it started with.
|
||||
dr_decode::dcp::set_profiles_directory(local_dir.to_path_buf());
|
||||
}
|
||||
}
|
||||
|
||||
/// TRACES: FR-UI-8
|
||||
/// Fetch just the place, for the handover at launch.
|
||||
///
|
||||
@@ -1734,6 +1861,79 @@ mod derived_guard_tests {
|
||||
(puts.load(Ordering::SeqCst), report, after, sent)
|
||||
}
|
||||
|
||||
/// A small valid profile, as bytes, named for `model`.
|
||||
fn profile_bytes(model: &str) -> Vec<u8> {
|
||||
dr_decode::dcp::Dcp {
|
||||
name: "Test Standard".into(),
|
||||
unique_camera_model: Some(model.into()),
|
||||
copyright: None,
|
||||
calibration_signature: None,
|
||||
embed_policy: 0,
|
||||
illuminants: [Some(21), None],
|
||||
color_matrix: [
|
||||
Some([[1.0, 0.0, 0.0], [0.0, 1.0, 0.0], [0.0, 0.0, 1.0]]),
|
||||
None,
|
||||
],
|
||||
forward_matrix: [None, None],
|
||||
hue_sat: [None, None],
|
||||
look: dr_types::HueSatTable::new(2, 2, 1, false, vec![[5.0, 1.1, 1.0]; 4]),
|
||||
tone_curve: None,
|
||||
baseline_exposure_offset: 0.0,
|
||||
}
|
||||
.to_bytes()
|
||||
.unwrap()
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn camera_profiles_travel_both_ways_and_once() {
|
||||
// TRACES: FR-DEV-3e
|
||||
// camera-profiles.md §13: ours goes up, theirs comes down, and a
|
||||
// second pass with nothing new moves nothing.
|
||||
let root = std::env::temp_dir().join(format!("dr-profile-sync-{}", std::process::id()));
|
||||
let _ = std::fs::remove_dir_all(&root);
|
||||
let server = root.join("server");
|
||||
let device = root.join("device");
|
||||
std::fs::create_dir_all(server.join(".darkroom-derived/profiles")).unwrap();
|
||||
std::fs::create_dir_all(&device).unwrap();
|
||||
std::fs::write(device.join("Ours A.dcp"), profile_bytes("Ours A")).unwrap();
|
||||
std::fs::write(
|
||||
server.join(".darkroom-derived/profiles/Theirs B.dcp"),
|
||||
profile_bytes("Theirs B"),
|
||||
)
|
||||
.unwrap();
|
||||
// Not a profile, and must be left alone in both directions.
|
||||
std::fs::write(server.join(".darkroom-derived/profiles/notes.txt"), b"x").unwrap();
|
||||
std::fs::write(
|
||||
server.join(".darkroom-derived/profiles/Broken C.dcp"),
|
||||
b"not a profile",
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
let backend = dr_sync_folder::FolderBackend::new(&server).unwrap();
|
||||
let base = RemotePath::new(".darkroom-derived");
|
||||
let mut report = SyncReport::default();
|
||||
sync_profiles(&backend, &base, &device, &mut report).await;
|
||||
|
||||
assert_eq!(report.profiles_uploaded, 1);
|
||||
assert_eq!(report.profiles_downloaded, 1, "the broken one is refused");
|
||||
assert!(server
|
||||
.join(".darkroom-derived/profiles/Ours A.dcp")
|
||||
.exists());
|
||||
assert!(device.join("Theirs B.dcp").exists());
|
||||
assert!(!device.join("Broken C.dcp").exists());
|
||||
assert!(!device.join("notes.txt").exists());
|
||||
assert!(
|
||||
dr_decode::dcp::find(Some("Theirs B"), "", "").is_some(),
|
||||
"a fetched profile is matched on the next decode"
|
||||
);
|
||||
|
||||
let mut again = SyncReport::default();
|
||||
sync_profiles(&backend, &base, &device, &mut again).await;
|
||||
assert_eq!((again.profiles_uploaded, again.profiles_downloaded), (0, 0));
|
||||
|
||||
let _ = std::fs::remove_dir_all(&root);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn a_place_that_could_not_be_read_is_never_written_over() {
|
||||
// A dehydrated placeholder, and the record on the server may well be
|
||||
|
||||
Reference in New Issue
Block a user