dtourolle/scene-actor-extraction
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

faster calibration curve generation

Browse Source 
jellyfin intergration
This commit is contained in:
dtourolle
2026-06-12 17:54:23 +02:00
parent d753062c6c
commit a1d6759abc
17 changed files with 1379 additions and 166 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/nodes/identity_matcher_node.hpp
+10 -4
View File
@@ -37,6 +37,7 @@ struct IdentityMatcherFunc {
, ratio_(cfg.match_ratio)
, ratio_ceil_(cfg.match_ratio_ceil)
{
std::cerr << "[identity_matcher] flattening gallery embeddings...\n";
for (int ai = 0; ai < static_cast<int>(gallery_.actors.size()); ++ai) {
for (const auto& emb : gallery_.actors[ai].embeddings) {
flat_emb_.push_back(emb);
@@ -44,7 +45,10 @@ struct IdentityMatcherFunc {
}
}
cal_ = calibrate_gallery(flat_emb_, flat_actor_);
std::cerr << "[identity_matcher] starting calibration ("
<< flat_emb_.size() << " embeddings)...\n";
cal_ = calibrate_gallery_cached(flat_emb_, flat_actor_,
cfg.gallery_path + ".calib_cache.json");
if (cal_.valid) {
std::cerr << "[identity_matcher] calibrated Bayesian matching"
@@ -126,9 +130,11 @@ struct IdentityMatcherFunc {
ia.track_id = tf.track_ids[fi];
if (accept) {
ia.actor_idx = best_actor;
ia.name = gallery_.actors[best_actor].name;
ia.imdb_id = gallery_.actors[best_actor].imdb_id;
ia.actor_idx = best_actor;
ia.name = gallery_.actors[best_actor].name;
ia.imdb_id = gallery_.actors[best_actor].imdb_id;
ia.tmdb_id = gallery_.actors[best_actor].tmdb_id;
ia.jellyfin_id = gallery_.actors[best_actor].jellyfin_id;
ia.similarity = cal_.valid
? cal_.probability(best_s, log_prior_odds_)
: best_s;
src/nodes/result_sink_node.hpp
+31 -11
View File
@@ -18,7 +18,16 @@ using json = nlohmann::json;
// KPN sink node: accumulates SceneAnnotations and writes the final JSON on EOF.
//
// Verbosity::minimal — merges per-frame presence into contiguous time windows.
// Output: { "movie": "...", "actors": [{ "name", "imdb_id", "scenes": [[t0,t1], ...] }] }
// Output: {
// "schema_version": 1, "movie": "...", "sample_fps": ..., "anneal_sec": ...,
// "actors": [{ "name", "imdb_id", "tmdb_id", "jellyfin_id", "scenes": [[t0,t1], ...] }]
// }
// This is the spec consumed by the Jellyfin plugin: each actor carries every
// identity key the gallery knows (empty string if not resolved). The plugin
// should prefer "jellyfin_id" (direct Person item GUID) when non-empty, and
// otherwise resolve "imdb_id"/"tmdb_id" against the item's People ProviderIds.
// To find who's on screen at timestamp t, scan each actor's "scenes" for a
// window where start <= t <= end.
//
// Verbosity::standard — per-frame detail including bboxes, similarity, unknowns.
// Output: { "frames": [{ "t", "identified": [...], "unknowns": [...] }] }
@@ -55,6 +64,10 @@ struct ResultSinkFunc {
}
private:
// Bump when the minimal/standard output JSON structure changes in a way
// the Jellyfin plugin needs to detect.
static constexpr int kSchemaVersion = 1;
static int count_known(const std::vector<IdentifiedActor>& v) {
int n = 0;
for (const auto& a : v) if (a.actor_idx >= 0) ++n;
@@ -73,6 +86,7 @@ private:
if (cfg_.verbosity == Verbosity::xray) {
root = build_xray();
} else {
root["schema_version"] = kSchemaVersion;
root["movie"] = cfg_.movie_path;
root["sample_fps"] = cfg_.sample_fps;
root["anneal_sec"] = cfg_.anneal_sec;
@@ -91,20 +105,20 @@ private:
}
struct ActorWindow {
std::string name, imdb_id;
std::string name, imdb_id, tmdb_id, jellyfin_id;
std::vector<std::pair<double, double>> scenes; // [start_sec, end_sec]
};
// Core logic: merge per-frame detections into annealed [start, end] windows.
std::vector<ActorWindow> build_actor_windows() {
struct Info { std::string name, imdb_id; };
struct Info { std::string name, imdb_id, tmdb_id, jellyfin_id; };
std::map<int, Info> actor_info;
std::map<int, std::vector<double>> timestamps;
for (const auto& frame : frames_) {
for (const auto& ia : frame.visible_actors) {
if (ia.actor_idx < 0) continue;
actor_info[ia.actor_idx] = {ia.name, ia.imdb_id};
actor_info[ia.actor_idx] = {ia.name, ia.imdb_id, ia.tmdb_id, ia.jellyfin_id};
timestamps[ia.actor_idx].push_back(frame.timestamp_sec);
}
}
@@ -112,8 +126,10 @@ private:
std::vector<ActorWindow> result;
for (auto& [idx, ts_vec] : timestamps) {
ActorWindow aw;
aw.name = actor_info[idx].name;
aw.imdb_id = actor_info[idx].imdb_id;
aw.name = actor_info[idx].name;
aw.imdb_id = actor_info[idx].imdb_id;
aw.tmdb_id = actor_info[idx].tmdb_id;
aw.jellyfin_id = actor_info[idx].jellyfin_id;
double win_start = ts_vec[0], win_end = ts_vec[0];
for (size_t i = 1; i < ts_vec.size(); ++i) {
@@ -136,9 +152,11 @@ private:
for (const auto& [s, e] : aw.scenes)
windows.push_back({s, e});
json ja;
ja["name"] = aw.name;
ja["imdb_id"] = aw.imdb_id;
ja["scenes"] = std::move(windows);
ja["name"] = aw.name;
ja["imdb_id"] = aw.imdb_id;
ja["tmdb_id"] = aw.tmdb_id;
ja["jellyfin_id"] = aw.jellyfin_id;
ja["scenes"] = std::move(windows);
actors.push_back(std::move(ja));
}
return actors;
@@ -178,8 +196,10 @@ private:
if (ia.actor_idx >= 0) {
json ja;
ja["name"] = ia.name;
ja["imdb_id"] = ia.imdb_id;
ja["name"] = ia.name;
ja["imdb_id"] = ia.imdb_id;
ja["tmdb_id"] = ia.tmdb_id;
ja["jellyfin_id"] = ia.jellyfin_id;
ja["similarity"] = ia.similarity;
ja["track_id"] = ia.track_id;
ja["bbox"] = jbox;
src/nodes/scene_tracker_node.hpp
+9 -3
View File
@@ -41,6 +41,8 @@ struct SceneTrackerFunc {
slot.last_crop = ia.crop;
slot.name = ia.name;
slot.imdb_id = ia.imdb_id;
slot.tmdb_id = ia.tmdb_id;
slot.jellyfin_id = ia.jellyfin_id;
// Keep the best (highest) similarity seen in this window
if (ia.similarity > slot.best_similarity)
slot.best_similarity = ia.similarity;
@@ -60,9 +62,11 @@ struct SceneTrackerFunc {
for (const auto& [actor_idx, slot] : active_) {
IdentifiedActor ia;
ia.actor_idx = actor_idx;
ia.name = slot.name;
ia.imdb_id = slot.imdb_id;
ia.actor_idx = actor_idx;
ia.name = slot.name;
ia.imdb_id = slot.imdb_id;
ia.tmdb_id = slot.tmdb_id;
ia.jellyfin_id = slot.jellyfin_id;
ia.similarity = slot.best_similarity;
ia.bbox = slot.last_bbox;
ia.crop = slot.last_crop;
@@ -85,6 +89,8 @@ private:
cv::Mat last_crop;
std::string name;
std::string imdb_id;
std::string tmdb_id;
std::string jellyfin_id;
};
double extinction_sec_;