feat(engine): HDF5-native galleries with embedded calibration; TensorRT backends; scene detection
Gallery format switches from JSON to HDF5 exclusively (JSON read-only kept for back-compat): save_gallery always writes HDF5, and the fitted Platt-sigmoid calibration (a, b, valid, hash) is now embedded directly in the gallery file instead of a sidecar .calib_cache.json — identity_matcher reads it from the loaded gallery and writes back only when the embeddings actually changed (hash mismatch), skipping the O(n^2) refit otherwise. Also includes: TensorRT inference backend support (ort_backend.cpp, trt_backend.cpp), gemm_backend improvements, TransNetV2-based scene-boundary detection wired through frame_source/face_tracker/main, and CMake build target updates for the new sources. Bumps the KPN submodule to feature/persistent-pipeline-reuse (push_blocking backpressure, node_ptr/node_stats introspection, ObjectVariantNodeWrapper for stateful functors) — needed by the optimizer's sae_kpn Python bindings.
This commit is contained in:
+171
-22
@@ -1,6 +1,7 @@
|
||||
#include "gallery_store.hpp"
|
||||
|
||||
#include <nlohmann/json.hpp>
|
||||
#include <H5Cpp.h>
|
||||
#include <chrono>
|
||||
#include <fstream>
|
||||
#include <iostream>
|
||||
@@ -8,7 +9,168 @@
|
||||
|
||||
using json = nlohmann::json;
|
||||
|
||||
// ── HDF5 (see gallery_store.hpp for the full layout) ─────────────────────────
|
||||
// A 170MB gallery JSON parses in ~18s (nlohmann). The same data as HDF5 loads in
|
||||
// ~1s — a big win for the optimizer, which reloads the gallery per replay subprocess.
|
||||
static bool ends_with(const std::string& s, const std::string& suf) {
|
||||
return s.size() >= suf.size() &&
|
||||
s.compare(s.size() - suf.size(), suf.size(), suf) == 0;
|
||||
}
|
||||
|
||||
static std::vector<std::string> read_str_dataset(H5::H5File& file, const char* name, hsize_t a) {
|
||||
H5::DataSet ds = file.openDataSet(name);
|
||||
H5::StrType st = ds.getStrType();
|
||||
std::vector<std::string> out(a);
|
||||
if (st.isVariableStr()) {
|
||||
std::vector<char*> raw(a);
|
||||
ds.read(raw.data(), st);
|
||||
for (hsize_t i = 0; i < a; ++i) { out[i] = raw[i] ? raw[i] : ""; }
|
||||
H5::DataSpace sp = ds.getSpace();
|
||||
H5Dvlen_reclaim(st.getId(), sp.getId(), H5P_DEFAULT, raw.data());
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
static ActorGallery load_gallery_hdf5(const std::string& path) {
|
||||
std::cerr << "[gallery] loading " << path << " (HDF5)..." << std::flush;
|
||||
auto t0 = std::chrono::steady_clock::now();
|
||||
|
||||
H5::H5File file(path, H5F_ACC_RDONLY);
|
||||
H5::DataSet emb_ds = file.openDataSet("embeddings");
|
||||
hsize_t dims[2];
|
||||
emb_ds.getSpace().getSimpleExtentDims(dims); // [N, 512]
|
||||
const hsize_t N = dims[0];
|
||||
if (dims[1] != 512) throw std::runtime_error("gallery HDF5: embedding dim != 512");
|
||||
|
||||
std::vector<float> flat(N * 512);
|
||||
emb_ds.read(flat.data(), H5::PredType::NATIVE_FLOAT);
|
||||
|
||||
H5::DataSet off_ds = file.openDataSet("offset");
|
||||
hsize_t adim[1];
|
||||
off_ds.getSpace().getSimpleExtentDims(adim);
|
||||
const hsize_t A = adim[0];
|
||||
std::vector<int64_t> offset(A);
|
||||
off_ds.read(offset.data(), H5::PredType::NATIVE_INT64);
|
||||
std::vector<int32_t> count(A);
|
||||
file.openDataSet("count").read(count.data(), H5::PredType::NATIVE_INT32);
|
||||
|
||||
auto imdb = read_str_dataset(file, "imdb_id", A);
|
||||
auto tmdb = read_str_dataset(file, "tmdb_id", A);
|
||||
auto jf = read_str_dataset(file, "jellyfin_id", A);
|
||||
auto name = read_str_dataset(file, "name", A);
|
||||
std::vector<std::string> src_images;
|
||||
if (file.nameExists("source_images"))
|
||||
src_images = read_str_dataset(file, "source_images", N);
|
||||
|
||||
ActorGallery gallery;
|
||||
gallery.actors.reserve(A);
|
||||
for (hsize_t a = 0; a < A; ++a) {
|
||||
ActorGallery::Actor actor;
|
||||
actor.imdb_id = imdb[a]; actor.tmdb_id = tmdb[a];
|
||||
actor.jellyfin_id = jf[a]; actor.name = name[a];
|
||||
for (int32_t e = 0; e < count[a]; ++e) {
|
||||
hsize_t row = offset[a] + e;
|
||||
Embedding emb;
|
||||
std::copy_n(flat.data() + row * 512, 512, emb.begin());
|
||||
actor.embeddings.push_back(emb);
|
||||
if (!src_images.empty())
|
||||
actor.source_images.push_back(src_images[row]);
|
||||
}
|
||||
gallery.actors.push_back(std::move(actor));
|
||||
}
|
||||
|
||||
if (file.nameExists("calibration")) {
|
||||
H5::Group cal = file.openGroup("calibration");
|
||||
cal.openAttribute("a").read(H5::PredType::NATIVE_FLOAT, &gallery.calib_a);
|
||||
cal.openAttribute("b").read(H5::PredType::NATIVE_FLOAT, &gallery.calib_b);
|
||||
int8_t valid = 0;
|
||||
cal.openAttribute("valid").read(H5::PredType::NATIVE_INT8, &valid);
|
||||
gallery.calib_valid = valid != 0;
|
||||
cal.openAttribute("hash").read(H5::PredType::NATIVE_UINT64, &gallery.calib_hash);
|
||||
}
|
||||
|
||||
auto t1 = std::chrono::steady_clock::now();
|
||||
std::cerr << " built " << A << " actors / " << N << " embeddings in "
|
||||
<< std::chrono::duration<double>(t1 - t0).count() << "s";
|
||||
if (gallery.calib_hash != 0)
|
||||
std::cerr << " (calibration cached: a=" << gallery.calib_a
|
||||
<< " b=" << gallery.calib_b << " valid=" << gallery.calib_valid << ")";
|
||||
std::cerr << "\n";
|
||||
return gallery;
|
||||
}
|
||||
|
||||
static void write_str_dataset(H5::H5File& file, const char* name,
|
||||
const std::vector<std::string>& values) {
|
||||
H5::StrType str_t(H5::PredType::C_S1, H5T_VARIABLE);
|
||||
hsize_t n = values.size();
|
||||
H5::DataSpace space(1, &n);
|
||||
H5::DataSet ds = file.createDataSet(name, str_t, space);
|
||||
std::vector<const char*> raw(n);
|
||||
for (hsize_t i = 0; i < n; ++i) raw[i] = values[i].c_str();
|
||||
ds.write(raw.data(), str_t);
|
||||
}
|
||||
|
||||
static void save_gallery_hdf5(const std::string& path, const ActorGallery& gallery) {
|
||||
H5::H5File file(path, H5F_ACC_TRUNC);
|
||||
|
||||
std::vector<float> flat;
|
||||
std::vector<int64_t> offset;
|
||||
std::vector<int32_t> count;
|
||||
std::vector<std::string> imdb, tmdb, jf, name, src_images;
|
||||
int64_t row = 0;
|
||||
for (const auto& a : gallery.actors) {
|
||||
offset.push_back(row);
|
||||
count.push_back(static_cast<int32_t>(a.embeddings.size()));
|
||||
row += static_cast<int64_t>(a.embeddings.size());
|
||||
for (size_t i = 0; i < a.embeddings.size(); ++i) {
|
||||
flat.insert(flat.end(), a.embeddings[i].begin(), a.embeddings[i].end());
|
||||
src_images.push_back(i < a.source_images.size() ? a.source_images[i] : "");
|
||||
}
|
||||
imdb.push_back(a.imdb_id); tmdb.push_back(a.tmdb_id);
|
||||
jf.push_back(a.jellyfin_id); name.push_back(a.name);
|
||||
}
|
||||
|
||||
hsize_t N = flat.size() / 512;
|
||||
hsize_t emb_dims[2] = {N, 512};
|
||||
H5::DataSpace emb_space(2, emb_dims);
|
||||
file.createDataSet("embeddings", H5::PredType::NATIVE_FLOAT, emb_space)
|
||||
.write(flat.data(), H5::PredType::NATIVE_FLOAT);
|
||||
|
||||
hsize_t A = gallery.actors.size();
|
||||
H5::DataSpace a_space(1, &A);
|
||||
file.createDataSet("offset", H5::PredType::NATIVE_INT64, a_space)
|
||||
.write(offset.data(), H5::PredType::NATIVE_INT64);
|
||||
file.createDataSet("count", H5::PredType::NATIVE_INT32, a_space)
|
||||
.write(count.data(), H5::PredType::NATIVE_INT32);
|
||||
|
||||
write_str_dataset(file, "imdb_id", imdb);
|
||||
write_str_dataset(file, "tmdb_id", tmdb);
|
||||
write_str_dataset(file, "jellyfin_id", jf);
|
||||
write_str_dataset(file, "name", name);
|
||||
write_str_dataset(file, "source_images", src_images);
|
||||
|
||||
if (gallery.calib_hash != 0) {
|
||||
H5::Group cal = file.createGroup("calibration");
|
||||
H5::DataSpace scalar(H5S_SCALAR);
|
||||
cal.createAttribute("a", H5::PredType::NATIVE_FLOAT, scalar)
|
||||
.write(H5::PredType::NATIVE_FLOAT, &gallery.calib_a);
|
||||
cal.createAttribute("b", H5::PredType::NATIVE_FLOAT, scalar)
|
||||
.write(H5::PredType::NATIVE_FLOAT, &gallery.calib_b);
|
||||
int8_t valid = gallery.calib_valid ? 1 : 0;
|
||||
cal.createAttribute("valid", H5::PredType::NATIVE_INT8, scalar)
|
||||
.write(H5::PredType::NATIVE_INT8, &valid);
|
||||
cal.createAttribute("hash", H5::PredType::NATIVE_UINT64, scalar)
|
||||
.write(H5::PredType::NATIVE_UINT64, &gallery.calib_hash);
|
||||
}
|
||||
|
||||
std::cerr << "[gallery] saved " << A << " actors / " << N
|
||||
<< " embeddings to " << path << " (HDF5)\n";
|
||||
}
|
||||
|
||||
ActorGallery load_gallery(const std::string& path) {
|
||||
if (ends_with(path, ".h5") || ends_with(path, ".hdf5"))
|
||||
return load_gallery_hdf5(path);
|
||||
|
||||
std::ifstream f(path);
|
||||
if (!f.is_open())
|
||||
throw std::runtime_error("load_gallery: cannot open " + path);
|
||||
@@ -53,28 +215,15 @@ ActorGallery load_gallery(const std::string& path) {
|
||||
return gallery;
|
||||
}
|
||||
|
||||
// Always writes HDF5. If `path` doesn't already end in .h5/.hdf5, the
|
||||
// extension is replaced (galleries are never written as JSON anymore).
|
||||
void save_gallery(const std::string& path, const ActorGallery& gallery) {
|
||||
json j;
|
||||
j["actors"] = json::array();
|
||||
|
||||
for (const auto& actor : gallery.actors) {
|
||||
json ja;
|
||||
ja["imdb_id"] = actor.imdb_id;
|
||||
ja["tmdb_id"] = actor.tmdb_id;
|
||||
ja["jellyfin_id"] = actor.jellyfin_id;
|
||||
ja["name"] = actor.name;
|
||||
ja["source_images"] = actor.source_images;
|
||||
|
||||
ja["embeddings"] = json::array();
|
||||
for (const auto& emb : actor.embeddings) {
|
||||
ja["embeddings"].push_back(
|
||||
std::vector<float>(emb.begin(), emb.end()));
|
||||
}
|
||||
j["actors"].push_back(std::move(ja));
|
||||
std::string out_path = path;
|
||||
if (!ends_with(out_path, ".h5") && !ends_with(out_path, ".hdf5")) {
|
||||
auto dot = out_path.find_last_of('.');
|
||||
out_path = (dot == std::string::npos ? out_path : out_path.substr(0, dot)) + ".h5";
|
||||
std::cerr << "[gallery] save_gallery: writing HDF5 to " << out_path
|
||||
<< " (galleries are no longer written as JSON)\n";
|
||||
}
|
||||
|
||||
std::ofstream f(path);
|
||||
if (!f.is_open())
|
||||
throw std::runtime_error("save_gallery: cannot write " + path);
|
||||
f << j.dump(2) << "\n";
|
||||
save_gallery_hdf5(out_path, gallery);
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user