feat(engine): HDF5-native galleries with embedded calibration; TensorRT backends; scene detection
Gallery format switches from JSON to HDF5 exclusively (JSON read-only kept for back-compat): save_gallery always writes HDF5, and the fitted Platt-sigmoid calibration (a, b, valid, hash) is now embedded directly in the gallery file instead of a sidecar .calib_cache.json — identity_matcher reads it from the loaded gallery and writes back only when the embeddings actually changed (hash mismatch), skipping the O(n^2) refit otherwise. Also includes: TensorRT inference backend support (ort_backend.cpp, trt_backend.cpp), gemm_backend improvements, TransNetV2-based scene-boundary detection wired through frame_source/face_tracker/main, and CMake build target updates for the new sources. Bumps the KPN submodule to feature/persistent-pipeline-reuse (push_blocking backpressure, node_ptr/node_stats introspection, ObjectVariantNodeWrapper for stateful functors) — needed by the optimizer's sae_kpn Python bindings.
This commit is contained in:
@@ -335,63 +335,48 @@ inline uint64_t hash_gallery_embeddings(
|
||||
return h;
|
||||
}
|
||||
|
||||
// Calibrates the gallery, caching the fitted (a, b, valid) result on disk
|
||||
// keyed by a hash of the reference embeddings. The O(n^2) pairwise fit only
|
||||
// re-runs when the gallery's embeddings/actor assignments actually change.
|
||||
// Calibrates the gallery, reusing (cached_a, cached_b, cached_valid) if
|
||||
// cached_hash matches a fresh hash of the current embeddings/actor
|
||||
// assignments — the O(n^2) pairwise fit only re-runs when they actually
|
||||
// change. Distinct from calibrate_gallery_cached's old sidecar-JSON-file
|
||||
// design: the cache now lives in the gallery HDF5 itself (ActorGallery::
|
||||
// calib_*, see gallery_store.hpp), so this takes the previous values
|
||||
// in-memory rather than a file path. Sets `recomputed` so the caller (which
|
||||
// holds the open gallery file/struct) knows whether it needs to persist the
|
||||
// refreshed values back.
|
||||
inline GalleryCalibration calibrate_gallery_cached(
|
||||
const std::vector<Embedding>& flat_emb,
|
||||
const std::vector<int>& flat_actor,
|
||||
const std::string& cache_path)
|
||||
float cached_a,
|
||||
float cached_b,
|
||||
bool cached_valid,
|
||||
uint64_t cached_hash,
|
||||
const std::string& curve_base_path,
|
||||
bool& recomputed)
|
||||
{
|
||||
uint64_t hash = hash_gallery_embeddings(flat_emb, flat_actor);
|
||||
recomputed = false;
|
||||
|
||||
std::string base_path = cache_path;
|
||||
constexpr std::string_view kJsonExt = ".json";
|
||||
if (base_path.size() >= kJsonExt.size() &&
|
||||
base_path.compare(base_path.size() - kJsonExt.size(), kJsonExt.size(), kJsonExt) == 0)
|
||||
base_path.resize(base_path.size() - kJsonExt.size());
|
||||
|
||||
std::ifstream in(cache_path);
|
||||
if (in.is_open()) {
|
||||
try {
|
||||
nlohmann::json j;
|
||||
in >> j;
|
||||
if (j.at("hash").get<uint64_t>() == hash) {
|
||||
GalleryCalibration cal;
|
||||
cal.a = j.at("a").get<float>();
|
||||
cal.b = j.at("b").get<float>();
|
||||
cal.valid = j.at("valid").get<bool>();
|
||||
std::cerr << "[calibration] using cached calibration from "
|
||||
<< cache_path << " (a=" << cal.a << " b=" << cal.b
|
||||
<< " valid=" << cal.valid << ")\n";
|
||||
save_calibration_curve(cal, base_path);
|
||||
return cal;
|
||||
}
|
||||
std::cerr << "[calibration] cache at " << cache_path
|
||||
<< " is stale, recomputing\n";
|
||||
} catch (const std::exception&) {
|
||||
std::cerr << "[calibration] cache at " << cache_path
|
||||
<< " is unreadable, recomputing\n";
|
||||
}
|
||||
if (cached_hash != 0 && cached_hash == hash) {
|
||||
GalleryCalibration cal{cached_a, cached_b, cached_valid};
|
||||
std::cerr << "[calibration] using cached calibration from gallery"
|
||||
<< " (a=" << cal.a << " b=" << cal.b
|
||||
<< " valid=" << cal.valid << ")\n";
|
||||
if (!curve_base_path.empty()) save_calibration_curve(cal, curve_base_path);
|
||||
return cal;
|
||||
}
|
||||
if (cached_hash != 0)
|
||||
std::cerr << "[calibration] cached calibration is stale (embeddings changed), "
|
||||
"recomputing\n";
|
||||
|
||||
auto t0 = std::chrono::steady_clock::now();
|
||||
GalleryCalibration cal = calibrate_gallery(flat_emb, flat_actor);
|
||||
auto t1 = std::chrono::steady_clock::now();
|
||||
double secs = std::chrono::duration<double>(t1 - t0).count();
|
||||
std::cerr << "[calibration] fit took " << secs << "s for "
|
||||
std::cerr << "[calibration] fit took "
|
||||
<< std::chrono::duration<double>(t1 - t0).count() << "s for "
|
||||
<< flat_emb.size() << " embeddings\n";
|
||||
|
||||
nlohmann::json j;
|
||||
j["hash"] = hash;
|
||||
j["a"] = cal.a;
|
||||
j["b"] = cal.b;
|
||||
j["valid"] = cal.valid;
|
||||
j["fit_secs"] = secs;
|
||||
std::ofstream out(cache_path);
|
||||
if (out.is_open()) out << j.dump(2) << "\n";
|
||||
|
||||
save_calibration_curve(cal, base_path);
|
||||
|
||||
if (!curve_base_path.empty()) save_calibration_curve(cal, curve_base_path);
|
||||
recomputed = true;
|
||||
return cal;
|
||||
}
|
||||
|
||||
+171
-22
@@ -1,6 +1,7 @@
|
||||
#include "gallery_store.hpp"
|
||||
|
||||
#include <nlohmann/json.hpp>
|
||||
#include <H5Cpp.h>
|
||||
#include <chrono>
|
||||
#include <fstream>
|
||||
#include <iostream>
|
||||
@@ -8,7 +9,168 @@
|
||||
|
||||
using json = nlohmann::json;
|
||||
|
||||
// ── HDF5 (see gallery_store.hpp for the full layout) ─────────────────────────
|
||||
// A 170MB gallery JSON parses in ~18s (nlohmann). The same data as HDF5 loads in
|
||||
// ~1s — a big win for the optimizer, which reloads the gallery per replay subprocess.
|
||||
static bool ends_with(const std::string& s, const std::string& suf) {
|
||||
return s.size() >= suf.size() &&
|
||||
s.compare(s.size() - suf.size(), suf.size(), suf) == 0;
|
||||
}
|
||||
|
||||
static std::vector<std::string> read_str_dataset(H5::H5File& file, const char* name, hsize_t a) {
|
||||
H5::DataSet ds = file.openDataSet(name);
|
||||
H5::StrType st = ds.getStrType();
|
||||
std::vector<std::string> out(a);
|
||||
if (st.isVariableStr()) {
|
||||
std::vector<char*> raw(a);
|
||||
ds.read(raw.data(), st);
|
||||
for (hsize_t i = 0; i < a; ++i) { out[i] = raw[i] ? raw[i] : ""; }
|
||||
H5::DataSpace sp = ds.getSpace();
|
||||
H5Dvlen_reclaim(st.getId(), sp.getId(), H5P_DEFAULT, raw.data());
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
static ActorGallery load_gallery_hdf5(const std::string& path) {
|
||||
std::cerr << "[gallery] loading " << path << " (HDF5)..." << std::flush;
|
||||
auto t0 = std::chrono::steady_clock::now();
|
||||
|
||||
H5::H5File file(path, H5F_ACC_RDONLY);
|
||||
H5::DataSet emb_ds = file.openDataSet("embeddings");
|
||||
hsize_t dims[2];
|
||||
emb_ds.getSpace().getSimpleExtentDims(dims); // [N, 512]
|
||||
const hsize_t N = dims[0];
|
||||
if (dims[1] != 512) throw std::runtime_error("gallery HDF5: embedding dim != 512");
|
||||
|
||||
std::vector<float> flat(N * 512);
|
||||
emb_ds.read(flat.data(), H5::PredType::NATIVE_FLOAT);
|
||||
|
||||
H5::DataSet off_ds = file.openDataSet("offset");
|
||||
hsize_t adim[1];
|
||||
off_ds.getSpace().getSimpleExtentDims(adim);
|
||||
const hsize_t A = adim[0];
|
||||
std::vector<int64_t> offset(A);
|
||||
off_ds.read(offset.data(), H5::PredType::NATIVE_INT64);
|
||||
std::vector<int32_t> count(A);
|
||||
file.openDataSet("count").read(count.data(), H5::PredType::NATIVE_INT32);
|
||||
|
||||
auto imdb = read_str_dataset(file, "imdb_id", A);
|
||||
auto tmdb = read_str_dataset(file, "tmdb_id", A);
|
||||
auto jf = read_str_dataset(file, "jellyfin_id", A);
|
||||
auto name = read_str_dataset(file, "name", A);
|
||||
std::vector<std::string> src_images;
|
||||
if (file.nameExists("source_images"))
|
||||
src_images = read_str_dataset(file, "source_images", N);
|
||||
|
||||
ActorGallery gallery;
|
||||
gallery.actors.reserve(A);
|
||||
for (hsize_t a = 0; a < A; ++a) {
|
||||
ActorGallery::Actor actor;
|
||||
actor.imdb_id = imdb[a]; actor.tmdb_id = tmdb[a];
|
||||
actor.jellyfin_id = jf[a]; actor.name = name[a];
|
||||
for (int32_t e = 0; e < count[a]; ++e) {
|
||||
hsize_t row = offset[a] + e;
|
||||
Embedding emb;
|
||||
std::copy_n(flat.data() + row * 512, 512, emb.begin());
|
||||
actor.embeddings.push_back(emb);
|
||||
if (!src_images.empty())
|
||||
actor.source_images.push_back(src_images[row]);
|
||||
}
|
||||
gallery.actors.push_back(std::move(actor));
|
||||
}
|
||||
|
||||
if (file.nameExists("calibration")) {
|
||||
H5::Group cal = file.openGroup("calibration");
|
||||
cal.openAttribute("a").read(H5::PredType::NATIVE_FLOAT, &gallery.calib_a);
|
||||
cal.openAttribute("b").read(H5::PredType::NATIVE_FLOAT, &gallery.calib_b);
|
||||
int8_t valid = 0;
|
||||
cal.openAttribute("valid").read(H5::PredType::NATIVE_INT8, &valid);
|
||||
gallery.calib_valid = valid != 0;
|
||||
cal.openAttribute("hash").read(H5::PredType::NATIVE_UINT64, &gallery.calib_hash);
|
||||
}
|
||||
|
||||
auto t1 = std::chrono::steady_clock::now();
|
||||
std::cerr << " built " << A << " actors / " << N << " embeddings in "
|
||||
<< std::chrono::duration<double>(t1 - t0).count() << "s";
|
||||
if (gallery.calib_hash != 0)
|
||||
std::cerr << " (calibration cached: a=" << gallery.calib_a
|
||||
<< " b=" << gallery.calib_b << " valid=" << gallery.calib_valid << ")";
|
||||
std::cerr << "\n";
|
||||
return gallery;
|
||||
}
|
||||
|
||||
static void write_str_dataset(H5::H5File& file, const char* name,
|
||||
const std::vector<std::string>& values) {
|
||||
H5::StrType str_t(H5::PredType::C_S1, H5T_VARIABLE);
|
||||
hsize_t n = values.size();
|
||||
H5::DataSpace space(1, &n);
|
||||
H5::DataSet ds = file.createDataSet(name, str_t, space);
|
||||
std::vector<const char*> raw(n);
|
||||
for (hsize_t i = 0; i < n; ++i) raw[i] = values[i].c_str();
|
||||
ds.write(raw.data(), str_t);
|
||||
}
|
||||
|
||||
static void save_gallery_hdf5(const std::string& path, const ActorGallery& gallery) {
|
||||
H5::H5File file(path, H5F_ACC_TRUNC);
|
||||
|
||||
std::vector<float> flat;
|
||||
std::vector<int64_t> offset;
|
||||
std::vector<int32_t> count;
|
||||
std::vector<std::string> imdb, tmdb, jf, name, src_images;
|
||||
int64_t row = 0;
|
||||
for (const auto& a : gallery.actors) {
|
||||
offset.push_back(row);
|
||||
count.push_back(static_cast<int32_t>(a.embeddings.size()));
|
||||
row += static_cast<int64_t>(a.embeddings.size());
|
||||
for (size_t i = 0; i < a.embeddings.size(); ++i) {
|
||||
flat.insert(flat.end(), a.embeddings[i].begin(), a.embeddings[i].end());
|
||||
src_images.push_back(i < a.source_images.size() ? a.source_images[i] : "");
|
||||
}
|
||||
imdb.push_back(a.imdb_id); tmdb.push_back(a.tmdb_id);
|
||||
jf.push_back(a.jellyfin_id); name.push_back(a.name);
|
||||
}
|
||||
|
||||
hsize_t N = flat.size() / 512;
|
||||
hsize_t emb_dims[2] = {N, 512};
|
||||
H5::DataSpace emb_space(2, emb_dims);
|
||||
file.createDataSet("embeddings", H5::PredType::NATIVE_FLOAT, emb_space)
|
||||
.write(flat.data(), H5::PredType::NATIVE_FLOAT);
|
||||
|
||||
hsize_t A = gallery.actors.size();
|
||||
H5::DataSpace a_space(1, &A);
|
||||
file.createDataSet("offset", H5::PredType::NATIVE_INT64, a_space)
|
||||
.write(offset.data(), H5::PredType::NATIVE_INT64);
|
||||
file.createDataSet("count", H5::PredType::NATIVE_INT32, a_space)
|
||||
.write(count.data(), H5::PredType::NATIVE_INT32);
|
||||
|
||||
write_str_dataset(file, "imdb_id", imdb);
|
||||
write_str_dataset(file, "tmdb_id", tmdb);
|
||||
write_str_dataset(file, "jellyfin_id", jf);
|
||||
write_str_dataset(file, "name", name);
|
||||
write_str_dataset(file, "source_images", src_images);
|
||||
|
||||
if (gallery.calib_hash != 0) {
|
||||
H5::Group cal = file.createGroup("calibration");
|
||||
H5::DataSpace scalar(H5S_SCALAR);
|
||||
cal.createAttribute("a", H5::PredType::NATIVE_FLOAT, scalar)
|
||||
.write(H5::PredType::NATIVE_FLOAT, &gallery.calib_a);
|
||||
cal.createAttribute("b", H5::PredType::NATIVE_FLOAT, scalar)
|
||||
.write(H5::PredType::NATIVE_FLOAT, &gallery.calib_b);
|
||||
int8_t valid = gallery.calib_valid ? 1 : 0;
|
||||
cal.createAttribute("valid", H5::PredType::NATIVE_INT8, scalar)
|
||||
.write(H5::PredType::NATIVE_INT8, &valid);
|
||||
cal.createAttribute("hash", H5::PredType::NATIVE_UINT64, scalar)
|
||||
.write(H5::PredType::NATIVE_UINT64, &gallery.calib_hash);
|
||||
}
|
||||
|
||||
std::cerr << "[gallery] saved " << A << " actors / " << N
|
||||
<< " embeddings to " << path << " (HDF5)\n";
|
||||
}
|
||||
|
||||
ActorGallery load_gallery(const std::string& path) {
|
||||
if (ends_with(path, ".h5") || ends_with(path, ".hdf5"))
|
||||
return load_gallery_hdf5(path);
|
||||
|
||||
std::ifstream f(path);
|
||||
if (!f.is_open())
|
||||
throw std::runtime_error("load_gallery: cannot open " + path);
|
||||
@@ -53,28 +215,15 @@ ActorGallery load_gallery(const std::string& path) {
|
||||
return gallery;
|
||||
}
|
||||
|
||||
// Always writes HDF5. If `path` doesn't already end in .h5/.hdf5, the
|
||||
// extension is replaced (galleries are never written as JSON anymore).
|
||||
void save_gallery(const std::string& path, const ActorGallery& gallery) {
|
||||
json j;
|
||||
j["actors"] = json::array();
|
||||
|
||||
for (const auto& actor : gallery.actors) {
|
||||
json ja;
|
||||
ja["imdb_id"] = actor.imdb_id;
|
||||
ja["tmdb_id"] = actor.tmdb_id;
|
||||
ja["jellyfin_id"] = actor.jellyfin_id;
|
||||
ja["name"] = actor.name;
|
||||
ja["source_images"] = actor.source_images;
|
||||
|
||||
ja["embeddings"] = json::array();
|
||||
for (const auto& emb : actor.embeddings) {
|
||||
ja["embeddings"].push_back(
|
||||
std::vector<float>(emb.begin(), emb.end()));
|
||||
}
|
||||
j["actors"].push_back(std::move(ja));
|
||||
std::string out_path = path;
|
||||
if (!ends_with(out_path, ".h5") && !ends_with(out_path, ".hdf5")) {
|
||||
auto dot = out_path.find_last_of('.');
|
||||
out_path = (dot == std::string::npos ? out_path : out_path.substr(0, dot)) + ".h5";
|
||||
std::cerr << "[gallery] save_gallery: writing HDF5 to " << out_path
|
||||
<< " (galleries are no longer written as JSON)\n";
|
||||
}
|
||||
|
||||
std::ofstream f(path);
|
||||
if (!f.is_open())
|
||||
throw std::runtime_error("save_gallery: cannot write " + path);
|
||||
f << j.dump(2) << "\n";
|
||||
save_gallery_hdf5(out_path, gallery);
|
||||
}
|
||||
|
||||
@@ -2,9 +2,22 @@
|
||||
#include "types.hpp"
|
||||
#include <string>
|
||||
|
||||
// Load/save the actor gallery from/to a JSON file.
|
||||
// Load/save the actor gallery. HDF5 (.h5/.hdf5) is the only format written;
|
||||
// legacy gallery.json files are still readable for backward compatibility but
|
||||
// save_gallery always writes HDF5 regardless of the requested extension.
|
||||
//
|
||||
// JSON format:
|
||||
// HDF5 layout:
|
||||
// /embeddings float32 [N, 512] all actors' refs concatenated, row-major
|
||||
// /offset int64 [A] first row of actor a in /embeddings
|
||||
// /count int32 [A] number of refs for actor a
|
||||
// /imdb_id /tmdb_id /jellyfin_id /name : variable-length string [A]
|
||||
// /source_images : variable-length string [N], parallel to /embeddings rows
|
||||
// /calibration/a, /b : scalar float32 attrs — Platt-sigmoid P(match|sim) fit
|
||||
// /calibration/valid : scalar int8 attr (0/1)
|
||||
// /calibration/hash : scalar uint64 attr — hash of the embeddings the fit
|
||||
// was computed from; a mismatch means "recompute"
|
||||
//
|
||||
// Legacy JSON format (read-only):
|
||||
// {
|
||||
// "actors": [
|
||||
// {
|
||||
|
||||
Reference in New Issue
Block a user