// sae_kpn — run the real downstream pipeline inside a Python-assembled KPN // network, fed by a Python HDF5 replay source. Lets a parameter sweep re-run the // exact C++ tracking/matching/presence logic over dumped embeddings — no video // decode, no GPU — with different Config knobs each run. // /// TRACES: VR-011, VR-002 | PR-002 // // **The whole chain is C++, including the sink.** That is the VR-011 change and // it is the point of the requirement: replay must drive the real nodes, not a // reimplementation. Two things were wrong before. // // 1. It did not compile. `add_face_tracker` built `FaceTrackerFunc` from a // Config alone, and the tracker has required a TrackRegistry and a // calibration since AR-007/AR-008 moved association into probability // space. Any .so in a stale build/ predates that. // // 2. Presence was rebuilt in Python. `replay.py::build_minimal` merged // per-frame detections into windows by annealing gaps — which is what the // pipeline did before AR-012. The sink now builds a window from a // TrackRegistry claim: the extent of a track an actor owned, starting when // they appeared rather than when recognition first succeeded. Those answer // different questions, so every sweep was tuning against a contract the // shipped code had stopped honouring. // // Both had the same root cause, which is why this is one binding and not three. // The chain has a construction ORDER — the matcher fits the calibration, the // registry needs a discounter built from it, the tracker needs both, and the // sink needs the registry's claims — and a factory-per-node API cannot express // it. `add_pipeline` mirrors main.cpp exactly and is the only way to build the // chain, so the ordering cannot be got wrong again from Python. // // Boundary types (cross the Python seam): // EmbeddedSceneFrame IN (built by the Python replay source from HDF5 arrays) // SceneAnnotation OUT (optional tee for per-frame debug rendering only — // the presence output is written by the C++ sink) // Intermediate types (TrackedSceneFrame, MatchedSceneFrame) flow C++→C++ only, but // still need channel factories + converters registered so PyNetwork can wire them. #define KPN_BUILD_PYTHON #include #include #include "types.hpp" #include "config.hpp" #include "gallery/embedder_stamp.hpp" #include "gallery/gallery_store.hpp" #include "nodes/face_tracker_node.hpp" #include "nodes/identity_matcher_node.hpp" #include "nodes/frame_annotation_node.hpp" #include "nodes/result_sink_node.hpp" #include "track_registry.hpp" #include "evidence_discount.hpp" #include #include #include #include #include #include #include #include #include #include namespace nb = nanobind; using namespace nb::literals; // ── ReplaySession ───────────────────────────────────────────────────────────── /// TRACES: VR-011 | PR-002 /// State the network's nodes reference but do not own. /// /// ResultSinkFunc holds `std::atomic&`, exactly as it does under main(), /// where it is a stack local in a function that outlives the pipeline. There is /// no such frame here -- the network is built and torn down from Python -- so /// the flag lives in a session held for the network's lifetime and released /// explicitly. The registry is here for the same reason: the sink's claim /// callback captures it. struct ReplaySession { /// Owns the Config, and must. ResultSinkFunc holds `const Config&` -- under /// main() that is a stack local in a frame which outlives the pipeline, so /// the reference is fine there. There is no such frame here: the network is /// built inside a binding call and torn down from Python, so a Config local /// to add_pipeline dies the moment it returns and the sink is left reading /// freed memory. It presented as an empty output_path -- the sink announced /// `[result_sink] writing ` and wrote nothing. Config cfg; std::atomic done{false}; std::shared_ptr registry; }; // Function-local static so ordering against other translation units cannot bite. inline std::map>& sessions() { static std::map> s; return s; } // The variant spanning every type that flows on a channel in the replay chain. using SaeVariant = std::variant; // ── Node wrapper aliases ────────────────────────────────────────────────────── // Named once so add_pipeline and the runtime setters cannot disagree about a // node's port names: a mismatch there is a dynamic_cast that returns null, i.e. // a runtime setter that silently does nothing. using MatcherWrap = kpn::ObjectVariantNodeWrapper< IdentityMatcherFunc, SaeVariant, kpn::in<"tracked">, kpn::out<"matched">>; using TrackerWrap = kpn::ObjectVariantNodeWrapper< FaceTrackerFunc, SaeVariant, kpn::in<"embedded">, kpn::out<"tracked">>; using AnnotWrap = kpn::ObjectVariantNodeWrapper< FrameAnnotationFunc, SaeVariant, kpn::in<"matched">, kpn::out<"annotation">>; using SinkWrap = kpn::ObjectVariantNodeWrapper< ResultSinkFunc, SaeVariant, kpn::in<"annotation">, kpn::out<>>; // ── Converters ───────────────────────────────────────────────────────────────── // Only EmbeddedSceneFrame (in) and SceneAnnotation (out) actually cross the seam; // the two intermediates get identity-ish stubs (never converted in practice) so the // variant's converter map is complete. namespace kpn { // EmbeddedSceneFrame: built FROM Python (a dict of numpy arrays). to_python is a // stub (the replay source only produces it; nothing reads it back). template<> struct PythonConverter { static constexpr const char* type_name = "EmbeddedSceneFrame"; static nb::object to_python(const EmbeddedSceneFrame&) { // Not needed downstream; return None. (Kept total for map completeness.) return nb::none(); } static EmbeddedSceneFrame from_python(nb::object o) { nb::dict d = nb::cast(o); EmbeddedSceneFrame ef; ef.source.timestamp_sec = nb::cast(d["timestamp_sec"]); ef.source.frame_idx = d.contains("frame_idx") ? nb::cast(d["frame_idx"]) : -1; ef.source.eof = d.contains("eof") ? nb::cast(d["eof"]) : false; ef.source.is_cut = d.contains("is_cut") ? nb::cast(d["is_cut"]) : false; ef.source.is_scene_boundary = d.contains("is_scene_boundary") ? nb::cast(d["is_scene_boundary"]) : false; if (ef.source.eof) return ef; // faces: (N,4) bbox, (N,10) landmarks, (N,) confidence, (N,512) embeddings auto bbox = nb::cast, nb::c_contig>>(d["bbox"]); auto lmk = nb::cast, nb::c_contig>>(d["landmarks"]); auto conf = nb::cast, nb::c_contig>>(d["confidence"]); auto emb = nb::cast, nb::c_contig>>(d["embeddings"]); // AR-028 quality vector. Optional because a v1 dump predates it — absent // leaves the DetectedFace sentinels at -1, which reads as *unscored*, not // as a bad face. There is no live aligner on this path to recompute it: // the replay starts at the embedded-frame channel, so what the dump does // not carry is genuinely gone. // // Held in named locals, like the four above, because the ndarray owns the // reference that keeps the buffer alive — reading .data() off a temporary // would leave the pointer dangling at the end of the statement. using FloatCol = nb::ndarray, nb::c_contig>; std::optional sharp_col, resid_col; if (d.contains("sharpness")) sharp_col = nb::cast(d["sharpness"]); if (d.contains("alignment_residual")) resid_col = nb::cast(d["alignment_residual"]); const float* sp = sharp_col ? sharp_col->data() : nullptr; const float* rp = resid_col ? resid_col->data() : nullptr; const size_t n = bbox.shape(0); ef.faces.reserve(n); ef.embeddings.reserve(n); const float* bp = bbox.data(); const float* lp = lmk.data(); const float* cp = conf.data(); const float* ep = emb.data(); for (size_t i = 0; i < n; ++i) { DetectedFace f; f.bbox = cv::Rect2f(bp[i*4+0], bp[i*4+1], bp[i*4+2], bp[i*4+3]); for (int k = 0; k < 5; ++k) f.landmarks[k] = cv::Point2f(lp[i*10 + k*2], lp[i*10 + k*2 + 1]); f.confidence = cp[i]; if (sp) f.sharpness = sp[i]; if (rp) f.alignment_residual = rp[i]; ef.faces.push_back(f); Embedding e; for (int k = 0; k < 512; ++k) e[k] = ep[i*512 + k]; ef.embeddings.push_back(e); } // crops left empty: tracker/matcher only forward them for debug rendering. ef.crops.resize(n); return ef; } }; // SceneAnnotation: read INTO Python. from_python is a stub (Python never builds one). template<> struct PythonConverter { static constexpr const char* type_name = "SceneAnnotation"; static nb::object to_python(const SceneAnnotation& sa) { nb::dict d; d["timestamp_sec"] = sa.timestamp_sec; d["eof"] = sa.eof; nb::list actors; for (const auto& a : sa.visible_actors) { nb::dict ad; ad["actor_idx"] = a.actor_idx; ad["track_id"] = a.track_id; ad["name"] = a.name; ad["imdb_id"] = a.imdb_id; ad["tmdb_id"] = a.tmdb_id; ad["jellyfin_id"] = a.jellyfin_id; ad["similarity"] = a.similarity; ad["bbox"] = nb::make_tuple(a.bbox.x, a.bbox.y, a.bbox.width, a.bbox.height); actors.append(ad); } d["visible_actors"] = actors; return d; } static SceneAnnotation from_python(nb::object) { return {}; // never called } }; // Intermediates: never cross the seam. Provide stubs so register_full_type compiles. template<> struct PythonConverter { static constexpr const char* type_name = "TrackedSceneFrame"; static nb::object to_python(const TrackedSceneFrame&) { return nb::none(); } static TrackedSceneFrame from_python(nb::object) { return {}; } }; template<> struct PythonConverter { static constexpr const char* type_name = "MatchedSceneFrame"; static nb::object to_python(const MatchedSceneFrame&) { return nb::none(); } static MatchedSceneFrame from_python(nb::object) { return {}; } }; } // namespace kpn // ── Config from Python dict ───────────────────────────────────────────────────── // Only the knobs relevant to the replayed chain; everything else keeps its default. static Config config_from_dict(nb::dict d) { Config cfg; auto getf = [&](const char* k, float& dst) { if (d.contains(k)) dst = nb::cast(d[k]); }; auto geti = [&](const char* k, int& dst) { if (d.contains(k)) dst = nb::cast(d[k]); }; auto getd = [&](const char* k, double& dst){ if (d.contains(k)) dst = nb::cast(d[k]); }; // identity matcher getf("match_prior", cfg.match_prior); getf("prob_threshold", cfg.prob_threshold); // face tracker getf("track_alpha", cfg.track_alpha); getf("track_min_iou", cfg.track_min_iou); getf("track_assoc_min_prob", cfg.track_assoc_min_prob); getd("track_extinction_sec", cfg.track_extinction_sec); // AR-025: swept knobs, previously unreachable from any config. getf("ownership_logodds", cfg.ownership_logodds); getf("evidence_rho_max", cfg.evidence_rho_max); getf("evidence_admit_below", cfg.evidence_admit_below); geti("evidence_max_views", cfg.evidence_max_views); // gallery expansion (usually off for sweeps; expose so it can be toggled) if (d.contains("expand_gallery")) cfg.expand_gallery = nb::cast(d["expand_gallery"]); // AR-018: banded admission bounds for the per-film annex, in probability // space. Reachable from a sweep — the config comment asks for both to be // swept, and they are ignored unless expand_gallery is on. See track_gallery.hpp. getf("expand_band_lo", cfg.expand_band_lo); getf("expand_band_hi", cfg.expand_band_hi); // Presence derivation. Accepts a string ("flood"/"track_extent") or a // number (DE only produces floats: >=0.5 → flood) so the sweep can toggle // it as a sixth knob. flood snaps to boundaries in the replayed frames // (is_scene_boundary if present, else is_cut). if (d.contains("presence_mode")) { const auto& pm = d["presence_mode"]; bool flood = false; if (nb::isinstance(pm)) flood = (nb::cast(pm) == "flood"); else flood = (nb::cast(pm) >= 0.5); cfg.presence_mode = flood ? PresenceMode::flood : PresenceMode::track_extent; } /// TRACES: GR-004 | SR-001 if (d.contains("require_gallery_stamp")) cfg.require_gallery_stamp = nb::cast(d["require_gallery_stamp"]); /// TRACES: VR-011 | IR-001 | PR-002 | SR-003 // The sink is a real node in this network now, so it needs the two things // that decide what it writes and where. Both used to be irrelevant here // because the replay never had a sink -- Python rebuilt presence instead, // which is the reimplementation VR-002 forbids and VR-011 removes. if (d.contains("output_path")) cfg.output_path = nb::cast(d["output_path"]); if (d.contains("verbosity")) { const int v = nb::cast(d["verbosity"]); cfg.verbosity = v == 2 ? Verbosity::xray : v == 1 ? Verbosity::standard : Verbosity::minimal; } // Reported verbatim in the truth file's extraction block, so a replayed // manifest says which gallery scope produced it (IR-002). if (d.contains("gallery_scope")) cfg.gallery_scope = nb::cast(d["gallery_scope"]); if (d.contains("sample_fps")) cfg.sample_fps = nb::cast(d["sample_fps"]); if (d.contains("movie_path")) cfg.movie_path = nb::cast(d["movie_path"]); return cfg; } using Net = kpn::python::PyNetwork; NB_MODULE(sae_kpn, m) { m.doc() = "Real KPN downstream nodes (tracker/matcher/frame_annotation) for Python replay sweeps"; kpn::python::register_py_network(m, "Network"); // Register converters + channel factories for all four channel types on a net. // register_py_network doesn't do this (auto_bind does); we patch __init__ to. // Simpler: expose a free helper the Python side calls right after construction. m.def("_register_types", [](Net& net) { net.register_full_type( [](const EmbeddedSceneFrame& v){ return kpn::PythonConverter::to_python(v); }, [](nb::object o){ return kpn::PythonConverter::from_python(std::move(o)); }, "EmbeddedSceneFrame"); net.register_full_type( [](const TrackedSceneFrame& v){ return kpn::PythonConverter::to_python(v); }, [](nb::object o){ return kpn::PythonConverter::from_python(std::move(o)); }, "TrackedSceneFrame"); net.register_full_type( [](const MatchedSceneFrame& v){ return kpn::PythonConverter::to_python(v); }, [](nb::object o){ return kpn::PythonConverter::from_python(std::move(o)); }, "MatchedSceneFrame"); net.register_full_type( [](const SceneAnnotation& v){ return kpn::PythonConverter::to_python(v); }, [](nb::object o){ return kpn::PythonConverter::from_python(std::move(o)); }, "SceneAnnotation"); }); m.def("add_node_python", [](Net& net, std::string name, nb::object callable, std::vector ins, std::vector outs, std::size_t cap) { net.add_node_python(std::move(name), std::move(callable), std::move(ins), std::move(outs), cap); }, "net"_a, "name"_a, "callable"_a, "inputs"_a, "outputs"_a, "capacity"_a = 5); // ── The pipeline ──────────────────────────────────────────────────────────── /// TRACES: VR-011, VR-002 | DP-001 | PR-002, PR-004 /// /// One call builds the whole downstream chain, in the one order that works: /// /// matcher (fits the calibration) /// -> registry (needs a discounter built from it) /// -> tracker (needs both) /// -> frame_annotation /// -> result_sink (needs the registry's claims) /// /// This replaces add_face_tracker / add_identity_matcher / add_frame_annotation. /// They were separate because the network is assembled node by node from /// Python -- and that is exactly how the seam broke: the tracker's dependency /// on a calibration that only exists once the matcher is built cannot be /// expressed as three independent factories, so the tracker factory kept /// constructing FaceTrackerFunc{cfg} against a signature that no longer /// existed. A binding that cannot represent the order will eventually be /// called in the wrong one. /// /// DP-001 -- "modes are front-ends and must not fork pipeline logic" -- is /// the requirement this serves. The replay harness is a front-end. Its job is /// to supply frames and read the result, not to re-derive presence. m.def("add_pipeline", [](Net& net, std::string gallery_path, nb::dict cfg_dict, std::size_t cap, std::string embedder_model, std::string embedder_sha256) { Config cfg = config_from_dict(cfg_dict); cfg.gallery_path = gallery_path; // so a refreshed calibration persists back // Cache loaded galleries by path so a threshold sweep (many networks, same // gallery) pays the parse once. The matcher holds a const ref; the cache // keeps the gallery alive for the process lifetime. static std::map> cache; auto it = cache.find(gallery_path); if (it == cache.end()) it = cache.emplace(gallery_path, std::make_shared(load_gallery(gallery_path))).first; /// TRACES: GR-004 | SR-001 // embedder_model / embedder_sha256 identify whatever produced the // embeddings that will be fed in. In a replay those come from the dump's // own stamp: there is no live embedder here, so the dump *is* the // embedder as far as this gallery is concerned. Checked on every // construction, not only on a cache miss -- one process may replay // several dumps against one cached gallery. EmbedderStamp feeding; feeding.model_name = std::move(embedder_model); feeding.model_sha256 = std::move(embedder_sha256); enforce_embedder_stamp(it->second->embedder, feeding, gallery_path, feeding.model_name.empty() ? "embeddings fed into this network" : feeding.model_name, cfg.require_gallery_stamp); // 1. Matcher first: its constructor fits (or loads) the calibration. auto matcher = std::make_shared(cap, *it->second, cfg); // 2. The calibration every other stage must decide in (AR-024). auto same_person = same_person_probability(matcher->functor().calibration()); // 3. Registry + discounter, from Config (AR-025). TrackRegistry::Config reg_cfg; reg_cfg.track_extinction_sec = cfg.track_extinction_sec; reg_cfg.ownership_logodds = cfg.ownership_logodds; EvidenceDiscounter::Config disc_cfg; disc_cfg.max_views = cfg.evidence_max_views; disc_cfg.admit_below = cfg.evidence_admit_below; disc_cfg.rho_max = cfg.evidence_rho_max; auto registry = std::make_shared( reg_cfg, EvidenceDiscounter(same_person, disc_cfg)); matcher->functor().set_registry(registry); // 4. Tracker, which needs both. auto tracker = std::make_shared(cap, cfg, registry, same_person); // 5. Projection, stateless. auto annot = std::make_shared(cap); // 6. The real sink. `done` outlives the network via the session below; // ResultSinkFunc holds it by reference, as it does in main.cpp. auto session = std::make_shared(); session->cfg = cfg; // the sink holds this by reference session->registry = registry; auto sink = std::make_shared(cap, session->cfg, session->done); /// TRACES: AR-012, AR-016 | IR-003 | SR-002 // The claim path, identical to main.cpp's. Without the flush hook every // track still live at EOF is silently dropped -- which in a replay is // most of the closing scene, and reads as a recognition miss rather than // as a missing wire. ResultSinkFunc& sink_fn = sink->functor(); registry->on_track_dead([&sink_fn](const DeadTrack& d) { sink_fn.add_claim(d); }); sink_fn.set_pre_write_hook([registry](double last_ts) { registry->flush(last_ts); }); net.add("tracker", tracker); net.add("matcher", matcher); net.add("annotation", annot); net.add("sink", sink); // Keyed by network so release_pipeline can free it. Not a leak-by-design: // a sweep builds one network per replay, and the sink accumulates every // annotation, so holding these forever would grow with films x configs. sessions()[&net] = session; }, "net"_a, "gallery"_a, "config"_a, "capacity"_a = 16, "embedder_model"_a = "", "embedder_sha256"_a = ""); /// Drop the session for a network. Idempotent. Call after net.stop(); not /// calling it holds one registry and one sink's accumulated frames per /// replay, which a long sweep will notice. m.def("release_pipeline", [](Net& net) { sessions().erase(&net); }, "net"_a); /// TRACES: VR-011 | AR-025 | PR-002 /// The registry's own count of how often it was wrong, exposed so a replay /// can fail on it instead of returning a plausible-looking empty answer. /// /// `dropped_votes` is the one that matters here and it earned its keep /// immediately. A vote lands on a track the registry has already reaped when /// the matcher lags the tracker by more than track_extinction_sec of film. /// In scene_analyze that cannot happen -- channels are 16-64 deep, so /// backpressure pins the two nodes within a few frames of each other. This /// harness sized every channel to the whole film to avoid a PyNode overflow /// drop, which removed the backpressure entirely: the tracker ran the film /// to the end while the matcher was still in its first minute, every vote /// arrived after its track was gone, no track was ever owned, and the run /// produced zero presence windows while cheerfully reporting 1647 frames /// with an identified face. m.def("pipeline_diagnostics", [](Net& net) { nb::dict d; auto it = sessions().find(&net); if (it == sessions().end() || !it->second->registry) return d; const auto& r = *it->second->registry; d["dropped_votes"] = r.dropped_votes(); d["belief_swaps"] = r.belief_swaps(); d["actor_conflicts"] = r.actor_conflicts(); d["live_tracks"] = static_cast(r.live()); return d; }, "net"_a); /// True once the sink has written its output. The sink flushes on the EOF /// annotation, so a caller that reads the file before this is racing it. m.def("pipeline_done", [](Net& net) { auto it = sessions().find(&net); return it != sessions().end() && it->second->done.load(std::memory_order_acquire); }, "net"_a); // ── Runtime setters (persistent-pipeline reuse across a threshold sweep) ───── // Build the network once, then change thresholds between replays — no rebuild, // no teardown (which is where the ROCm deadlock lives), no gallery reload. m.def("set_prob_threshold", [](Net& net, std::string name, float t) { auto* w = dynamic_cast(net.node_ptr(name)); if (!w) throw std::runtime_error("set_prob_threshold: '" + name + "' is not an identity_matcher"); w->functor().set_prob_threshold(t); }, "net"_a, "name"_a, "value"_a); }