From 19f5a2b0ae21b4826e1c95bd1eb2c1cc5a904e91 Mon Sep 17 00:00:00 2001 From: Duncan Tourolle Date: Fri, 3 Jul 2026 23:38:03 +0200 Subject: [PATCH 01/42] Deliver EOF sentinels out-of-band to prevent teardown deadlock MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Channel::push() drops values on overflow (the intended backpressure policy for data), and PoolNode swallows the resulting ChannelOverflowError. For a control sentinel like EOF this is fatal: a single dropped EOF under backpressure wedges every downstream pop() forever, so the pipeline never tears down. Deliver sentinels out-of-band instead. Channel::push_sentinel() stores the token in a dedicated slot that does not consume ring capacity, so it can never overflow and — crucially — never blocks the caller. That non-blocking property is essential: each KPN node has a single worker thread, so a *blocking* push would park that thread and stop it draining its own input, cascading into a hold-and-wait deadlock under backpressure. The consumer's pop()/try_pop_now() drain the ring first, then deliver the sentinel, so it always arrives after every value pushed before it. approx_size() (which node readiness checks call) counts a pending sentinel as consumable work, so a channel carrying only a sentinel still schedules its consumer's next fire — without this the token would sit undelivered and the pipeline would still deadlock at teardown. PoolNode/PoolObjectNode route values carrying an eof flag (direct .eof or nested .source.eof) through push_sentinel via a SFINAE-safe is_sentinel_value trait; all other values keep the existing lossy throwing push. The trait compiles to false for types without an eof convention, so this is a no-op for pipelines that don't use one. Verified end-to-end: scene_analyze now reaches EOF, flushes its output, and exits cleanly instead of hanging. Co-Authored-By: Claude Opus 4.8 --- include/kpn/channel.hpp | 80 ++++++++++++++++++++++++++++++++++++--- include/kpn/pool_node.hpp | 46 ++++++++++++++++++++++ tests/test_channel.cpp | 63 ++++++++++++++++++++++++++++++ 3 files changed, 183 insertions(+), 6 deletions(-) diff --git a/include/kpn/channel.hpp b/include/kpn/channel.hpp index 3f0b05d..4466fd8 100644 --- a/include/kpn/channel.hpp +++ b/include/kpn/channel.hpp @@ -136,6 +136,36 @@ public: push_callback_(); } + // Lossless, non-blocking delivery for a must-deliver control token (EOF). + // + // A sentinel is stored out-of-band — in a dedicated slot that does NOT + // consume ring capacity — so this can never overflow and never blocks the + // caller. That distinction is essential: each KPN node has a single worker + // thread, so a *blocking* push would park that thread and stop it draining + // its own input, cascading into a hold-and-wait deadlock under backpressure. + // Setting a flag and returning keeps the worker free to keep popping. + // + // The consumer's pop() drains the ring first, then delivers this sentinel, + // preserving ordering (EOF arrives after all data pushed before it). + // + // Only the sole producer may call it (SPSC contract, same as push()). + // Returns false if the channel is already disabled (token discarded — + // teardown is in progress, so the sentinel is moot). + bool push_sentinel(T value) { + if (!accepting_.load(std::memory_order_acquire)) { + stats_.record_drop(); + return false; + } + eof_value_ = make_storage(std::move(value)); + has_eof_.store(true, std::memory_order_release); + // Wake a consumer blocked in pop(): the sentinel is now deliverable even + // though the ring may be empty. + wake_.fetch_add(1, std::memory_order_release); + wake_.notify_one(); + if (push_callback_) push_callback_(); + return true; + } + // Blocking pop. Returns when an item is available. // Throws ChannelClosedError if the channel is disabled (regardless of fill). T pop() { @@ -148,21 +178,28 @@ public: // If empty, spin before sleeping: avoids the futex when the next item // arrives within the spin window (~4 µs at default spin_count=200 on x86). if (h == t) { + // Ring drained — deliver any pending out-of-band sentinel (EOF) + // now, so it always arrives after the data pushed before it. + { T s; if (take_sentinel(s)) return s; } + if (!accepting_.load(std::memory_order_acquire)) throw ChannelClosedError{}; - for (std::size_t s = 0; s < spin_count_; ++s) { + for (std::size_t si = 0; si < spin_count_; ++si) { spin_hint(); t = tail_.load(std::memory_order_acquire); if (t != h) break; + { T s; if (take_sentinel(s)) return s; } if (!accepting_.load(std::memory_order_relaxed)) throw ChannelClosedError{}; } if (h == t) { - // Still empty after spin — sleep until push() or disable() fires. - // Re-check tail after loading w to guard against a lost wakeup. + // Still empty after spin — sleep until push()/push_sentinel() + // or disable() fires. Re-check tail and the sentinel after + // loading w to guard against a lost wakeup. if (tail_.load(std::memory_order_acquire) != h) continue; + if (has_eof_.load(std::memory_order_acquire)) continue; wake_.wait(w, std::memory_order_relaxed); continue; } @@ -190,9 +227,12 @@ public: } // Immediate non-blocking pop. Returns false if the ring is empty. + // Once the ring is drained, delivers any pending out-of-band sentinel (EOF) + // so pool nodes — which pop only via this path — still receive the token. bool try_pop_now(T& out) { const std::size_t h = head_.load(std::memory_order_relaxed); - if (h == tail_.load(std::memory_order_acquire)) return false; + if (h == tail_.load(std::memory_order_acquire)) + return take_sentinel(out); out = extract(std::move(buf_[h & ring_mask_])); head_.store(h + 1, std::memory_order_release); stats_.record_pop(); @@ -217,12 +257,21 @@ public: push_callback_ = std::move(cb); } - // Size derived lazily from ring indices — no separate counter on the hot path. + // Ring occupancy, derived lazily from indices — no separate counter on the + // hot path. Excludes any out-of-band sentinel (that lives outside the ring). std::size_t size() const { return tail_.load(std::memory_order_relaxed) - head_.load(std::memory_order_relaxed); } - std::size_t approx_size() const { return size(); } + + // A pending out-of-band sentinel (EOF) counts as consumable work here even + // though it holds no ring slot. This is what node readiness checks call, so + // a channel carrying only a sentinel still schedules its consumer's next + // fire — without this the sentinel would never be popped and the pipeline + // would deadlock at teardown. + std::size_t approx_size() const { + return size() + (has_eof_.load(std::memory_order_acquire) ? 1u : 0u); + } std::size_t capacity() const { return capacity_; } bool is_accepting() const { return accepting_.load(std::memory_order_relaxed); } @@ -260,6 +309,17 @@ private: return *s; } + // Consume the out-of-band sentinel if one is pending. Consumer-only. + // Called only when the ring is observed empty, so the sentinel is always + // delivered after every value pushed before it. + bool take_sentinel(T& out) { + if (!has_eof_.load(std::memory_order_acquire)) return false; + out = extract(std::move(eof_value_)); + has_eof_.store(false, std::memory_order_release); + stats_.record_pop(); + return true; + } + const std::size_t capacity_; const std::size_t spin_count_; std::size_t ring_mask_; @@ -267,8 +327,16 @@ private: std::function push_callback_; ChannelStats stats_; + // Out-of-band sentinel (EOF): stored outside the ring so its delivery never + // depends on ring capacity and never blocks the producer. Written by the + // producer (push_sentinel), read+cleared by the consumer (take_sentinel); + // has_eof_ is the publish/consume handshake. + storage_type eof_value_{}; + std::atomic has_eof_{false}; + // Separate cache lines: head_ is written only by the consumer; // tail_ and wake_ are written only by the producer. + // wake_ wakes a blocked pop() on enqueue or on a pending sentinel. alignas(64) std::atomic head_{0}; alignas(64) std::atomic tail_{0}; std::atomic wake_{0}; diff --git a/include/kpn/pool_node.hpp b/include/kpn/pool_node.hpp index e4ceb54..4fe37f2 100644 --- a/include/kpn/pool_node.hpp +++ b/include/kpn/pool_node.hpp @@ -22,6 +22,38 @@ namespace kpn { +// ── Sentinel detection ──────────────────────────────────────────────────────── +// A value is a "sentinel" (must-deliver control token, e.g. EOF) if its type +// carries a bool-convertible eof flag — either directly (`v.eof`, as on a raw +// source Frame) or nested one level under a `.source` member (`v.source.eof`, +// as on the pipeline's SceneFrame/…/MatchedSceneFrame message types, which wrap +// the originating Frame). Sentinels are delivered losslessly and non-blockingly +// via Channel::push_sentinel() instead of the throwing push(), so backpressure +// can never drop the token that unblocks downstream teardown. +// +// Types with neither shape are never treated as sentinels — both traits are +// SFINAE-safe and the runtime check compiles away to `false` for them, so this +// stays a no-op for pipelines that don't use an eof convention. +template +struct has_eof_field : std::false_type {}; +template +struct has_eof_field(std::declval().eof))>> + : std::true_type {}; + +template +struct has_source_eof_field : std::false_type {}; +template +struct has_source_eof_field(std::declval().source.eof))>> + : std::true_type {}; + +template +constexpr bool is_sentinel_value(const T& v) { + if constexpr (has_eof_field::value) return static_cast(v.eof); + else if constexpr (has_source_eof_field::value) return static_cast(v.source.eof); + else return false; +} + // ── PoolNode ────────────────────────────────────────────────────────────────── // // Reactive alternative to Node<>. Instead of owning a blocked thread, the node @@ -361,6 +393,13 @@ private: void push_one_out(std::tuple_element_t&& val) { auto* ch = std::get(output_channels_); if (!ch) return; + // Sentinels (EOF) must never be dropped: a lost token wedges every + // downstream pop() forever. Deliver them out-of-band (push_sentinel), + // which never overflows and never blocks this node's worker thread. + if (is_sentinel_value(val)) { + ch->push_sentinel(std::move(val)); + return; + } try { ch->push(std::move(val)); } catch (const ChannelOverflowError&) { @@ -660,6 +699,13 @@ private: void push_one_out(std::tuple_element_t&& val) { auto* ch = std::get(output_channels_); if (!ch) return; + // Sentinels (EOF) must never be dropped: a lost token wedges every + // downstream pop() forever. Deliver them out-of-band (push_sentinel), + // which never overflows and never blocks this node's worker thread. + if (is_sentinel_value(val)) { + ch->push_sentinel(std::move(val)); + return; + } try { ch->push(std::move(val)); } catch (const ChannelOverflowError&) { diff --git a/tests/test_channel.cpp b/tests/test_channel.cpp index 50b11d0..d710214 100644 --- a/tests/test_channel.cpp +++ b/tests/test_channel.cpp @@ -175,3 +175,66 @@ TEST_CASE("bandwidth_mbs returns 0 when elapsed_s is zero or negative", "[channe REQUIRE(snap.bandwidth_mbs(0.0) == 0.0); REQUIRE(snap.bandwidth_mbs(-1.0) == 0.0); } + +TEST_CASE("push_sentinel never overflows even on a full channel", "[channel][sentinel]") { + Channel ch(2); + ch.push(1); + ch.push(2); // channel full — a plain push(3) would throw ChannelOverflowError + + // The sentinel is stored out-of-band, so it neither throws nor blocks the + // caller — the exact property an EOF token needs under backpressure. This + // returns immediately with the ring still full. + REQUIRE(ch.push_sentinel(99)); + REQUIRE(ch.size() == 2); // sentinel did not consume ring capacity +} + +TEST_CASE("push_sentinel is delivered after all ring data, in order", "[channel][sentinel]") { + Channel ch(4); + ch.push(1); + ch.push(2); + ch.push_sentinel(99); // enqueue EOF while data is still buffered + + // Data drains first; the sentinel arrives only once the ring is empty. + REQUIRE(ch.pop() == 1); + REQUIRE(ch.pop() == 2); + REQUIRE(ch.pop() == 99); +} + +TEST_CASE("push_sentinel wakes a blocked pop", "[channel][sentinel]") { + Channel ch(2); // empty + std::thread producer([&] { + std::this_thread::sleep_for(std::chrono::milliseconds(20)); + ch.push_sentinel(99); // must wake a consumer parked on an empty ring + }); + REQUIRE(ch.pop() == 99); + producer.join(); +} + +TEST_CASE("approx_size counts a pending sentinel so consumers stay schedulable", + "[channel][sentinel]") { + Channel ch(4); + REQUIRE(ch.approx_size() == 0); + ch.push_sentinel(99); + // Node readiness checks call approx_size(); it must report the out-of-band + // sentinel as consumable work even though it holds no ring slot. + REQUIRE(ch.approx_size() == 1); + REQUIRE(ch.size() == 0); // ...but the ring itself is still empty + int out = 0; + REQUIRE(ch.try_pop_now(out)); + REQUIRE(out == 99); + REQUIRE(ch.approx_size() == 0); +} + +TEST_CASE("try_pop_now delivers a pending sentinel once the ring is empty", + "[channel][sentinel]") { + Channel ch(2); + ch.push(1); + ch.push_sentinel(99); + + int out = 0; + REQUIRE(ch.try_pop_now(out)); // ring data first + REQUIRE(out == 1); + REQUIRE(ch.try_pop_now(out)); // then the sentinel + REQUIRE(out == 99); + REQUIRE_FALSE(ch.try_pop_now(out)); // nothing left +} From 949c8134efb38030b47387044f5929e64745be39 Mon Sep 17 00:00:00 2001 From: Duncan Tourolle Date: Sat, 4 Jul 2026 11:02:54 +0200 Subject: [PATCH 02/42] Ignore build_test/ (out-of-tree test build dir) Co-Authored-By: Claude Opus 4.8 --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 910b5d0..17c9c78 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,6 @@ # Build output build/ +build_test/ build_debug/ site/ # Python From c4538f03cae04397d26509666ff28d9b280da461 Mon Sep 17 00:00:00 2001 From: Duncan Tourolle Date: Sat, 4 Jul 2026 14:43:47 +0200 Subject: [PATCH 03/42] python: fix nanobind Network reference leak via GC type slots The Python Network holds each PyNode's callable, forming an uncollectable instance -> callable -> globals() -> instance cycle that tripped nanobind's leak check at interpreter shutdown. Implement tp_traverse/tp_clear type slots on the Network binding so Python's cyclic collector can see through the C++-held callables and break the cycle. PyNode exposes its callable; PyNetwork visits and clears them. Wired into both binding sites (auto_bind and the legacy register_py_network). --- include/kpn/python/auto_bind.hpp | 2 +- include/kpn/python/bindings.hpp | 74 +++++++++++++++++++++++++++++++- 2 files changed, 74 insertions(+), 2 deletions(-) diff --git a/include/kpn/python/auto_bind.hpp b/include/kpn/python/auto_bind.hpp index 12a8bd1..79f9d71 100644 --- a/include/kpn/python/auto_bind.hpp +++ b/include/kpn/python/auto_bind.hpp @@ -247,7 +247,7 @@ void bind_network(nb::module_& m) { nb::class_>(m, "INode"); - nb::class_(m, "Network") + nb::class_(m, "Network", nb::type_slots(network_type_slots())) .def("__init__", [](Net* self) { new (self) Net(); register_all_converters(*self); diff --git a/include/kpn/python/bindings.hpp b/include/kpn/python/bindings.hpp index 57554e9..0a31bfe 100644 --- a/include/kpn/python/bindings.hpp +++ b/include/kpn/python/bindings.hpp @@ -35,6 +35,16 @@ public: using VNode = IVariantNode; using VChannel = IVariantChannel; + // ── GC support ──────────────────────────────────────────────────────────── + // Visit every Python object this network transitively holds (currently the + // callable of each PyNode). Used by the Network type's tp_traverse slot so + // Python's cyclic GC can discover instance → callable → globals() cycles. + // Defined out-of-line below, once PyNode is a complete type. + template + void visit_python_objects(Fn&& visit) const; + // Drop all Python references held by nodes, breaking any cycle (tp_clear). + void clear_python_objects(); + // ── Builder API ─────────────────────────────────────────────────────────── void add(std::string name, std::shared_ptr node) { @@ -367,6 +377,14 @@ public: out_channels_[i] = std::move(ch); } + // ── GC support (tp_traverse / tp_clear on the owning Network) ────────────── + // The node holds a Python callable, which typically forms an + // instance → callable → globals() → instance cycle. Expose the callable so + // the Network's GC slots can traverse and clear it. See bindings.hpp's + // network_tp_traverse/network_tp_clear. + const nb::object& python_callable() const { return callable_; } + void clear_python_callable() { callable_ = nb::object(); } + private: void run_loop() { while (!stop_flag_.load(std::memory_order_relaxed)) { @@ -439,6 +457,60 @@ private: NodeStats stats_; }; +// ── PyNetwork GC helpers (defined here: PyNode is now complete) ──────────────── + +template +template +void PyNetwork::visit_python_objects(Fn&& visit) const { + for (const auto& [name, node] : nodes_) + if (auto* py = dynamic_cast*>(node.get())) + visit(py->python_callable()); +} + +template +void PyNetwork::clear_python_objects() { + for (auto& [name, node] : nodes_) + if (auto* py = dynamic_cast*>(node.get())) + py->clear_python_callable(); +} + +// ── GC type slots for the Network binding ───────────────────────────────────── +// The Network holds Python callables (via PyNode), forming uncollectable +// instance → callable → globals() → instance cycles at interpreter shutdown. +// These slots let Python's cyclic collector traverse and break them, silencing +// nanobind's leak warnings. See the nanobind "Reference leaks" documentation. + +template +int network_tp_traverse(PyObject* self, visitproc visit, void* arg) { + Py_VISIT(Py_TYPE(self)); + if (!nb::inst_ready(self)) + return 0; + auto* net = nb::inst_ptr>(self); + int rv = 0; + net->visit_python_objects([&](const nb::object& obj) { + if (rv == 0 && obj.is_valid()) + rv = visit(obj.ptr(), arg); + }); + return rv; +} + +template +int network_tp_clear(PyObject* self) { + auto* net = nb::inst_ptr>(self); + net->clear_python_objects(); + return 0; +} + +template +PyType_Slot* network_type_slots() { + static PyType_Slot slots[] = { + { Py_tp_traverse, reinterpret_cast(&network_tp_traverse) }, + { Py_tp_clear, reinterpret_cast(&network_tp_clear) }, + { 0, nullptr } + }; + return slots; +} + // ── register_py_network (legacy helper) ─────────────────────────────────────── // Registers PyNetwork with the given nanobind module. // Prefer bind_network from auto_bind.hpp for new code. @@ -447,7 +519,7 @@ template void register_py_network(nb::module_& m, const char* class_name = "Network") { using Net = PyNetwork; - nb::class_(m, class_name) + nb::class_(m, class_name, nb::type_slots(network_type_slots())) .def(nb::init<>()) .def("connect", &Net::connect, nb::arg("src"), nb::arg("out_idx"), From 2b0873b61b3164433aaabe3649a3341480d3a640 Mon Sep 17 00:00:00 2001 From: Duncan Tourolle Date: Sat, 4 Jul 2026 14:44:24 +0200 Subject: [PATCH 04/42] examples: make 08_python_subport run a real Python node The example was named "python subport" but its graph was entirely C++ (ProduceNode -> DoubleItNode); Python only tapped the output, leaving a dangling "#todo: return value to network". Rewrite so the only node in the graph is a pure-Python py_triple, driven from both ends via the subport taps: net.write() injects inputs and net.read() pulls results back, closing the round trip. Also del the network at the end so its callable cycle is reclaimed deterministically. --- examples/08_python_subport/example.py | 53 ++++++++++++++++++--------- 1 file changed, 35 insertions(+), 18 deletions(-) diff --git a/examples/08_python_subport/example.py b/examples/08_python_subport/example.py index 1351b5e..4d4600c 100644 --- a/examples/08_python_subport/example.py +++ b/examples/08_python_subport/example.py @@ -1,38 +1,55 @@ """ -08_python_subport — tap a C++ node's output from Python using net.read(). +08_python_subport — drive a *Python* node from Python via write()/read() taps. Graph: - [ProduceNode] --int--> [DoubleItNode] --int--> (tapped by net.read()) + (fed by net.write()) --int--> [py_triple] --int--> (tapped by net.read()) -The sink is Python: instead of connecting a PrintItNode, we call net.read() -to pull values out of DoubleItNode's output directly into Python. -We also demonstrate net.write() by injecting a value into DoubleItNode's input. +Unlike 07, there is no C++ source or sink here: the only node in the network is +a pure-Python function, py_triple. Python plays *both* the producer and the +consumer by using the subport taps: + + * net.write("py", 0, v) injects v into py_triple's input (Python -> network) + * net.read("py", 0) pulls py_triple's output back out (network -> Python) + +This closes the loop the old version left as a "#todo": a value flows from +Python, through a Python node running inside the network, and back to Python. """ import sys -import time -import threading -sys.path.insert(0, "build/python") +sys.path.insert(0, "build/python") # for `python examples/.../example.py` from repo root import kpn_python as kpn + +def py_triple(x: int) -> int: + return x * 3 + + net = kpn.Network() -net.add("src", kpn.make_produce()) -net.add("dbl", kpn.make_double_it()) +# The whole network is a single Python node with a tapped input and output. +net.add_node("py", py_triple, inputs=["int"], outputs=["int"]) -net.connect("src", 0, "dbl", 0) net.build() net.start() -# Collect a few values from DoubleItNode's output via Python tap +# Push values in from Python and read the Python node's results back out. +inputs = [1, 2, 7, 10, 100] results = [] -for _ in range(5): - val = net.read("dbl", 0) - results.append(val) +for v in inputs: + net.write("py", 0, v) # Python -> py_triple input + results.append(net.read("py", 0)) # py_triple output -> Python net.stop() -print("values read from C++ DoubleItNode output:", results) -assert all(v == 84 for v in results), f"expected all 84, got {results}" -print("all correct (42 * 2 = 84)") +print("inputs written from Python: ", inputs) +print("outputs read from py_triple:", results) + +expected = [v * 3 for v in inputs] +assert results == expected, f"expected {expected}, got {results}" +print("all correct (x * 3 computed by a Python node inside the network)") + +# Drop the network deterministically. The network holds the Python callable, +# which (via globals) forms a reference cycle; deleting the global breaks it so +# the network is reclaimed promptly instead of lingering to interpreter exit. +del net From 298c9e770b2ecd686ce2c9c52c5857a4fe7f19e6 Mon Sep 17 00:00:00 2001 From: Duncan Tourolle Date: Sat, 4 Jul 2026 14:44:30 +0200 Subject: [PATCH 05/42] examples: run Python examples 07 and 08 as CTest smoke tests Neither Python example was registered as a test, so `ctest -L examples` in CI skipped them entirely -- which is how 08's missing Python node went unnoticed. Add a kpn_python_example() helper (gated on KPN_BUILD_PYTHON) that runs each script with PYTHONPATH pointed at the freshly-built module, so it does not depend on cwd or a hard-coded build/python path, and register 07 and 08. Also del the network in 07 for deterministic teardown. --- examples/07_python_network/example.py | 5 +++++ examples/CMakeLists.txt | 23 ++++++++++++++++++++++- 2 files changed, 27 insertions(+), 1 deletion(-) diff --git a/examples/07_python_network/example.py b/examples/07_python_network/example.py index 0eee84e..a10ec41 100644 --- a/examples/07_python_network/example.py +++ b/examples/07_python_network/example.py @@ -30,3 +30,8 @@ net.build() net.start() time.sleep(0.1) net.stop() + +# Drop the network deterministically: it holds the Python callable, which forms +# a reference cycle via globals(). Deleting the global breaks it so the network +# is reclaimed now rather than lingering to interpreter shutdown. +del net diff --git a/examples/CMakeLists.txt b/examples/CMakeLists.txt index 261f5dd..3a59c1d 100644 --- a/examples/CMakeLists.txt +++ b/examples/CMakeLists.txt @@ -13,6 +13,25 @@ function(kpn_example name) ) endfunction() +# Register a Python example script as a CTest smoke test. Runs the script with +# PYTHONPATH pointing at the freshly-built kpn_python module, so it does not +# depend on the caller's working directory or a hard-coded "build/python" path. +function(kpn_python_example name) + if(NOT KPN_BUILD_PYTHON) + return() + endif() + add_test( + NAME example_${name} + COMMAND ${CMAKE_COMMAND} -E env + "PYTHONPATH=$" + ${Python_EXECUTABLE} ${CMAKE_CURRENT_SOURCE_DIR}/${name}/example.py + ) + set_tests_properties(example_${name} PROPERTIES + TIMEOUT 15 + LABELS examples + ) +endfunction() + kpn_example(01_hello_pipeline) kpn_example(02_named_ports) kpn_example(03_multi_output) @@ -31,7 +50,9 @@ if(KPN_WEB_DEBUG) target_link_libraries(14_debug_hub PRIVATE kpn) kpn_target_enable_web_debug(14_debug_hub) endif() -# 07 and 08 are Python scripts — no compiled target needed. +# 07 and 08 are Python scripts — no compiled target, but run as smoke tests. +kpn_python_example(07_python_network) +kpn_python_example(08_python_subport) # 09 requires OpenCV — only build if found find_package(OpenCV QUIET COMPONENTS core imgproc highgui videoio) From 903dd4eea5b1cf46cf88e519a4aaf0e8c8b49ebb Mon Sep 17 00:00:00 2001 From: Duncan Tourolle Date: Sat, 4 Jul 2026 15:07:12 +0200 Subject: [PATCH 06/42] docs: update README examples table and add documentation link 07_python_network and 08_python_subport now work and run as CI smoke tests, so drop their "(pending)" markers and describe what they actually demonstrate. Also surface the hosted documentation link at the top and re-render README.md from README.md.in. --- README.md | 6 ++++-- README.md.in | 6 ++++-- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 1d7d1e1..727be44 100644 --- a/README.md +++ b/README.md @@ -2,6 +2,8 @@ A C++20 Kahn Process Network (KPN) library. Each node wraps a function and runs in its own thread, communicating with downstream nodes via bounded FIFO channels. Includes Python bindings via nanobind. +📖 **[Documentation](https://pages.tourolle.paris/dtourolle/kpn/)** + --- ## Requirements @@ -398,8 +400,8 @@ Violating the second rule deadlocks. | `04_storage_policy` | `channel_storage_policy` default and specialisation | | `05_error_handling` | `ChannelOverflowError`, `ErrorHandler` | | `06_watchdog` | Watchdog interval, stall detection | -| `07_python_network` | PyNetwork, pure Python node *(pending)* | -| `08_python_subport` | `net.read`, `net.write`, sub-port tap *(pending)* | +| `07_python_network` | PyNetwork with a pure-Python node between a C++ source and sink | +| `08_python_subport` | Drive a Python node from Python via `net.write`/`net.read` sub-port taps | | `09_opencv_cellshade` | Real-time cell-shading on webcam/pattern; requires OpenCV ≥ 4 | Run the cell-shading example: diff --git a/README.md.in b/README.md.in index cd821d4..4d7622b 100644 --- a/README.md.in +++ b/README.md.in @@ -2,6 +2,8 @@ A C++20 Kahn Process Network (KPN) library. Each node wraps a function and runs in its own thread, communicating with downstream nodes via bounded FIFO channels. Includes Python bindings via nanobind. +📖 **[Documentation](https://pages.tourolle.paris/dtourolle/kpn/)** + --- ## Requirements @@ -220,8 +222,8 @@ Violating the second rule deadlocks. | `04_storage_policy` | `channel_storage_policy` default and specialisation | | `05_error_handling` | `ChannelOverflowError`, `ErrorHandler` | | `06_watchdog` | Watchdog interval, stall detection | -| `07_python_network` | PyNetwork, pure Python node *(pending)* | -| `08_python_subport` | `net.read`, `net.write`, sub-port tap *(pending)* | +| `07_python_network` | PyNetwork with a pure-Python node between a C++ source and sink | +| `08_python_subport` | Drive a Python node from Python via `net.write`/`net.read` sub-port taps | | `09_opencv_cellshade` | Real-time cell-shading on webcam/pattern; requires OpenCV ≥ 4 | Run the cell-shading example: From 66feb918217e6c3046fd4975019da259a8bf69f3 Mon Sep 17 00:00:00 2001 From: Duncan Tourolle Date: Sat, 4 Jul 2026 15:07:27 +0200 Subject: [PATCH 07/42] ci: pass docker `push` input as a string, not a boolean Gitea's act_runner mangles boolean workflow_call/dispatch inputs passed from an expression -- they arrive as false regardless of value. Declare `push` as a string ("true"/"false") and compare with == 'true' so the builder image is pushed on non-PR events again. --- .gitea/workflows/ci.yaml | 3 ++- .gitea/workflows/docker.yaml | 17 ++++++++++------- Dockerfile.builder | 2 +- 3 files changed, 13 insertions(+), 9 deletions(-) diff --git a/.gitea/workflows/ci.yaml b/.gitea/workflows/ci.yaml index f9d22c1..3373cec 100644 --- a/.gitea/workflows/ci.yaml +++ b/.gitea/workflows/ci.yaml @@ -63,7 +63,8 @@ jobs: if: ${{ needs.changes.outputs.dockerfile == 'true' }} uses: ./.gitea/workflows/docker.yaml with: - push: ${{ github.event_name != 'pull_request' }} + # Explicit string, not a boolean expression (act_runner mangles bools). + push: ${{ github.event_name == 'pull_request' && 'false' || 'true' }} # Runs after docker (if docker ran). A skipped docker job is fine; a failed # one blocks this via !failure(). Re-run tests when code OR the image changed. diff --git a/.gitea/workflows/docker.yaml b/.gitea/workflows/docker.yaml index 699af5b..27fa328 100644 --- a/.gitea/workflows/docker.yaml +++ b/.gitea/workflows/docker.yaml @@ -4,19 +4,22 @@ name: '🐳 Builder Image' # It is called by ci.yaml only when Dockerfile.builder or docs/requirements.txt # change. It runs on the host runner (NOT inside the builder container) because # it needs the Docker CLI/daemon. +# Note: `push` is a STRING ("true"/"false"), not a boolean. Gitea's act_runner +# mangles boolean inputs passed from an expression (they arrive as false), so we +# pass an explicit string and compare with == 'true' below. on: workflow_call: inputs: push: - description: 'Push the built image to the registry' - type: boolean - default: true + description: 'Push the built image to the registry ("true"/"false")' + type: string + default: 'true' workflow_dispatch: inputs: push: - description: 'Push the built image to the registry' - type: boolean - default: true + description: 'Push the built image to the registry ("true"/"false")' + type: string + default: 'true' jobs: build: @@ -48,7 +51,7 @@ jobs: . - name: Push builder image - if: ${{ inputs.push }} + if: ${{ inputs.push == 'true' }} run: | docker push gitea.tourolle.paris/dtourolle/kpnpp-builder:latest docker push gitea.tourolle.paris/dtourolle/kpnpp-builder:${{ github.sha }} diff --git a/Dockerfile.builder b/Dockerfile.builder index 7ad29c6..749f5f1 100644 --- a/Dockerfile.builder +++ b/Dockerfile.builder @@ -1,4 +1,4 @@ -# KPN++ Builder Image (CI: pipeline trigger) +# KPN++ Builder Image (CI: pipeline trigger v2) # Pre-built image with GCC, CMake, Ninja, and Python dev headers for building and testing KPN++ # Build: docker build -f Dockerfile.builder -t gitea.tourolle.paris/dtourolle/kpnpp-builder:latest . # Push: docker push gitea.tourolle.paris/dtourolle/kpnpp-builder:latest From 399ee4cf9b6d900860404723ddda05315d4d092b Mon Sep 17 00:00:00 2001 From: Duncan Tourolle Date: Sat, 4 Jul 2026 20:40:35 +0200 Subject: [PATCH 08/42] test: add ThreadSanitizer verification for lock-free Channel Add a contended SPSC stress suite (tests/test_channel_stress.cpp) that actually exercises the ring's memory-ordering pairing and spin/futex/ lost-wakeup logic, plus the CMake and CI plumbing to run it under TSan: - KPN_SANITIZER cache var + kpn_sanitizer_flags() helper (no-op when unset) - kpn_tests_stress executable, labelled "stress" for CTest - reusable tsan.yaml workflow (gcc:14 builder image, already ships libtsan) - ci.yaml gains a tsan job on the same code/dockerfile triggers as test Co-Authored-By: Claude Opus 4.8 --- .gitea/workflows/ci.yaml | 7 + .gitea/workflows/tsan.yaml | 67 ++++++++ CMakeLists.txt | 24 +++ tests/CMakeLists.txt | 31 +++- tests/test_channel_stress.cpp | 288 ++++++++++++++++++++++++++++++++++ 5 files changed, 416 insertions(+), 1 deletion(-) create mode 100644 .gitea/workflows/tsan.yaml create mode 100644 tests/test_channel_stress.cpp diff --git a/.gitea/workflows/ci.yaml b/.gitea/workflows/ci.yaml index 3373cec..e962340 100644 --- a/.gitea/workflows/ci.yaml +++ b/.gitea/workflows/ci.yaml @@ -73,6 +73,13 @@ jobs: if: ${{ !failure() && !cancelled() && (needs.changes.outputs.code == 'true' || needs.changes.outputs.dockerfile == 'true') }} uses: ./.gitea/workflows/test.yaml + # ThreadSanitizer run for the lock-free Channel. Same trigger conditions as + # test (code or image changed); runs in parallel with test. + tsan: + needs: [changes, docker] + if: ${{ !failure() && !cancelled() && (needs.changes.outputs.code == 'true' || needs.changes.outputs.dockerfile == 'true') }} + uses: ./.gitea/workflows/tsan.yaml + docs: needs: [changes, docker] if: ${{ !failure() && !cancelled() && github.ref == 'refs/heads/master' && (needs.changes.outputs.docs == 'true' || needs.changes.outputs.dockerfile == 'true') }} diff --git a/.gitea/workflows/tsan.yaml b/.gitea/workflows/tsan.yaml new file mode 100644 index 0000000..53f81ad --- /dev/null +++ b/.gitea/workflows/tsan.yaml @@ -0,0 +1,67 @@ +name: '🧵 ThreadSanitizer' + +# Reusable workflow: builds the channel stress suite with ThreadSanitizer and +# runs it. This is the dynamic half of verifying the lock-free SPSC Channel +# (the static half is the CDSChecker model-check harness in verify/). +# +# Triggering and path filtering are owned by ci.yaml (the orchestrator), which +# calls this only when code changed. workflow_dispatch is kept for manual runs. +# +# Runs in the prebuilt builder image (gcc:14), which already ships libtsan — no +# package installs at job time. +on: + workflow_call: + workflow_dispatch: + +jobs: + tsan: + runs-on: linux/amd64 + container: + image: gitea.tourolle.paris/dtourolle/kpnpp-builder:latest + steps: + - name: Checkout repository + uses: actions/checkout@v4 + with: + path: tsan-${{ github.run_id }} + + - name: Cache FetchContent dependencies + uses: actions/cache@v3 + with: + path: ~/.cmake/fetchcontent + key: cmake-fetchcontent-${{ hashFiles('**/CMakeLists.txt') }} + restore-keys: cmake-fetchcontent- + + - name: Configure (TSan) + working-directory: tsan-${{ github.run_id }} + run: | + cmake -S . -B build \ + -G Ninja \ + -DCMAKE_BUILD_TYPE=Debug \ + -DKPN_SANITIZER=thread \ + -DKPN_BUILD_TESTS=ON \ + -DKPN_BUILD_EXAMPLES=OFF \ + -DKPN_BUILD_PYTHON=OFF \ + -DFETCHCONTENT_BASE_DIR=$HOME/.cmake/fetchcontent + + - name: Build (TSan) + working-directory: tsan-${{ github.run_id }} + run: cmake --build build --parallel --target kpn_tests kpn_tests_stress + + - name: Run stress suite under TSan + working-directory: tsan-${{ github.run_id }} + # halt_on_error=1 makes the first detected race fail the job; the report + # (with both stacks) is printed to the log. second_deadlock_stack gives + # the full picture for lock-order issues. + env: + TSAN_OPTIONS: "halt_on_error=1 second_deadlock_stack=1" + run: ./build/tests/kpn_tests_stress + + - name: Run unit tests under TSan + working-directory: tsan-${{ github.run_id }} + env: + TSAN_OPTIONS: "halt_on_error=1 second_deadlock_stack=1" + run: ./build/tests/kpn_tests + + - name: Cleanup + if: always() + run: rm -rf tsan-${{ github.run_id }} diff --git a/CMakeLists.txt b/CMakeLists.txt index dd147b5..63037fd 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -10,6 +10,30 @@ option(KPN_BUILD_PYTHON "Build Python bindings (requires nanobind)" ON) option(KPN_BUILD_EXAMPLES "Build examples" ON) option(KPN_WEB_DEBUG "Enable web debug UI (cpp-httplib)" OFF) +# Sanitizer build. Empty = off. Accepts "thread", "address", "undefined", +# or a combination like "address,undefined". Applied to all kpn targets via +# the kpn_sanitizer_flags() helper below. +# +# The lock-free SPSC Channel (include/kpn/channel.hpp) has hand-reasoned +# acquire/release ordering; -DKPN_SANITIZER=thread + the channel stress test +# (tests/test_channel_stress.cpp) is the dynamic half of verifying it. The +# static half is the CDSChecker model-check harness (see verify/). +set(KPN_SANITIZER "" CACHE STRING + "Build with sanitizer: thread | address | undefined | (empty = off)") + +# Translate KPN_SANITIZER into compile/link flags. No-op when empty. +function(kpn_sanitizer_flags out_var) + if(KPN_SANITIZER) + set(${out_var} + -fsanitize=${KPN_SANITIZER} + -fno-omit-frame-pointer + -g + PARENT_SCOPE) + else() + set(${out_var} "" PARENT_SCOPE) + endif() +endfunction() + # ── Core library (header-only) ──────────────────────────────────────────────── add_library(kpn INTERFACE) target_include_directories(kpn INTERFACE diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index a8f7583..fda6711 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -43,6 +43,35 @@ target_link_libraries(kpn_tests PRIVATE GTest::gtest ) +# ── Channel stress suite (separate executable) ──────────────────────────────── +# Contended SPSC tests for the lock-free Channel. Kept out of kpn_tests +# because each case runs many reps / tens of thousands of items and is slow. +# Most valuable under -DKPN_SANITIZER=thread, but correct (and run) without it. +add_executable(kpn_tests_stress test_channel_stress.cpp) +target_link_libraries(kpn_tests_stress PRIVATE kpn Catch2::Catch2WithMain) + +# ── Sanitizer flags ─────────────────────────────────────────────────────────── +# kpn_sanitizer_flags() is defined in the top-level CMakeLists and is a no-op +# unless -DKPN_SANITIZER=... is set. Sanitizer must be on both compile and link. +kpn_sanitizer_flags(_kpn_san) +if(_kpn_san) + foreach(_t kpn_tests kpn_tests_stress) + target_compile_options(${_t} PRIVATE ${_kpn_san}) + target_link_options(${_t} PRIVATE ${_kpn_san}) + endforeach() +endif() + include(CTest) include(Catch) -catch_discover_tests(kpn_tests) + +# DISCOVERY_MODE PRE_TEST defers test enumeration to `ctest` run time. The +# default (POST_BUILD) runs each test binary during the build to list its +# cases — which fails a sanitizer build: a TSan/ASan binary needs a fixed +# address-space layout and aborts on startup ("unexpected memory mapping") +# under the container's ASLR, breaking the build before any test runs. The +# tsan.yaml job invokes the binaries directly (not via ctest), so deferring +# discovery costs nothing there and keeps `ctest` working for normal builds. +catch_discover_tests(kpn_tests DISCOVERY_MODE PRE_TEST) +# Register the stress suite under its own label so CI can run / time it +# separately from the fast unit tests. +catch_discover_tests(kpn_tests_stress DISCOVERY_MODE PRE_TEST PROPERTIES LABELS "stress") diff --git a/tests/test_channel_stress.cpp b/tests/test_channel_stress.cpp new file mode 100644 index 0000000..bd113d0 --- /dev/null +++ b/tests/test_channel_stress.cpp @@ -0,0 +1,288 @@ +// Contended stress tests for the lock-free SPSC Channel. +// +// The other channel tests (test_channel.cpp) are single-threaded or use a +// single 20 ms sleep to order two threads — they never actually contend on the +// ring, so they exercise neither the memory-ordering pairing nor the +// spin/futex/lost-wakeup logic in pop(). +// +// These tests are written to be run under ThreadSanitizer: +// +// cmake -B build -DKPN_SANITIZER=thread -DKPN_BUILD_EXAMPLES=OFF -DKPN_BUILD_PYTHON=OFF +// cmake --build build --target kpn_tests_tsan +// ./build/tests/kpn_tests_tsan +// +// They are also valid (and meaningful) without a sanitizer: the value/sequence +// assertions catch lost or duplicated items regardless of build flags. TSan +// adds detection of the underlying data race even on runs where the race did +// not corrupt observable state. +// +// Channel is SPSC: exactly one producer thread and one consumer thread per +// channel. Every scenario below honours that contract. + +#include +#include +#include +#include +#include +#include + +using namespace kpn; +using namespace std::chrono_literals; + +namespace { + +// Repeat each scenario enough times that rare interleavings (spin window just +// missing / just catching the next push, disable landing inside the futex +// wait) actually occur across a run. Kept modest so a TSan run stays minutes, +// not hours. +constexpr int kReps = 200; + +} // namespace + +TEST_CASE("SPSC: every pushed item is popped exactly once, in order", "[channel][stress]") { + // Small capacity forces frequent full/empty transitions, so both the + // producer's overflow-retry and the consumer's spin->futex path are hit + // many times. The producer retries on overflow rather than dropping, so + // the consumer must observe a strictly contiguous 0..N-1 sequence. + constexpr int N = 50'000; + Channel ch(/*capacity=*/4, /*spin_count=*/16); + + std::thread producer([&] { + for (int i = 0; i < N; ++i) { + for (;;) { + try { ch.push(i); break; } + catch (const ChannelOverflowError&) { std::this_thread::yield(); } + } + } + }); + + int expected = 0; + bool in_order = true; + for (int i = 0; i < N; ++i) { + int v = ch.pop(); + if (v != expected) in_order = false; + ++expected; + } + producer.join(); + + REQUIRE(in_order); + REQUIRE(expected == N); + REQUIRE(ch.size() == 0); +} + +TEST_CASE("SPSC: tight empty<->non-empty transitions exercise spin/futex boundary", + "[channel][stress]") { + // spin_count=0 forces every empty pop() straight into atomic::wait, so this + // hammers the lost-wakeup guard (snapshot wake_, re-check tail_, then wait). + // The producer pushes one item then waits to go empty again, maximising the + // number of empty->non-empty edges relative to item count. + constexpr int N = 20'000; + Channel ch(/*capacity=*/2, /*spin_count=*/0); + + std::thread producer([&] { + for (int i = 0; i < N; ++i) { + for (;;) { + try { ch.push(i); break; } + catch (const ChannelOverflowError&) { std::this_thread::yield(); } + } + } + }); + + long sum = 0; + for (int i = 0; i < N; ++i) sum += ch.pop(); + producer.join(); + + // Sum of 0..N-1 — detects any lost or duplicated item. + REQUIRE(sum == static_cast(N) * (N - 1) / 2); +} + +TEST_CASE("SPSC: disable() while consumer is blocked in pop() unblocks cleanly", + "[channel][stress]") { + // The data race of record: consumer blocked in pop() (spinning or parked in + // the futex) while the owner thread calls disable(). pop() must observe the + // close and throw ChannelClosedError — it must not hang and must not read + // past the ring. Repeated so disable() lands at many points in pop()'s loop. + for (int rep = 0; rep < kReps; ++rep) { + Channel ch(/*capacity=*/4, /*spin_count=*/8); + std::atomic threw{false}; + std::atomic finished{false}; + + std::thread consumer([&] { + try { + ch.pop(); // empty channel: will block + } catch (const ChannelClosedError&) { + threw.store(true, std::memory_order_relaxed); + } + finished.store(true, std::memory_order_relaxed); + }); + + // Give the consumer a chance to reach the wait, then close. + std::this_thread::sleep_for(50us); + ch.disable(); + + consumer.join(); + REQUIRE(finished.load()); + REQUIRE(threw.load()); + } +} + +TEST_CASE("SPSC: producer racing a disable() never throws and never hangs", + "[channel][stress]") { + // Mirror of the above from the producer side: push() racing disable() must + // either enqueue or silently drop, never throw ChannelClosedError and never + // wedge. Overflow is still a legal outcome (full accepting channel) and is + // tolerated here. + for (int rep = 0; rep < kReps; ++rep) { + Channel ch(/*capacity=*/8, /*spin_count=*/8); + std::atomic bad{false}; + + std::thread producer([&] { + for (int i = 0; i < 1000; ++i) { + try { ch.push(i); } + catch (const ChannelOverflowError&) { /* legal: full */ } + catch (...) { bad.store(true, std::memory_order_relaxed); break; } + } + }); + + std::this_thread::sleep_for(20us); + ch.disable(); // owner closes mid-stream + producer.join(); + + REQUIRE_FALSE(bad.load()); + } +} + +TEST_CASE("SPSC: push_callback fires on each empty->non-empty transition", + "[channel][stress]") { + // The empty->non-empty callback ([channel.hpp] was_empty branch) is read by + // the consumer-side notification path. Run it under contention to make sure + // the was_empty detection isn't torn by a concurrent pop(). + Channel ch(/*capacity=*/4, /*spin_count=*/4); + std::atomic callbacks{0}; + ch.set_push_callback([&] { callbacks.fetch_add(1, std::memory_order_relaxed); }); + + constexpr int N = 10'000; + std::thread producer([&] { + for (int i = 0; i < N; ++i) { + for (;;) { + try { ch.push(i); break; } + catch (const ChannelOverflowError&) { std::this_thread::yield(); } + } + } + }); + + for (int i = 0; i < N; ++i) (void)ch.pop(); + producer.join(); + + // At least one transition, at most one per item; mainly we assert the run + // completed without TSan flagging a race on push_callback_/was_empty. + REQUIRE(callbacks.load() >= 1); + REQUIRE(callbacks.load() <= N); +} + +// What the out-of-band sentinel guarantees under contention — and what it does +// not. push_sentinel() publishes has_eof_ (release) after the producer's N ring +// pushes; a consumer that observes has_eof_ (acquire) therefore also observes +// every value pushed before it. What these tests assert: +// +// * Losslessness — every value 0..N-1 is delivered exactly once (contiguous, +// no gaps, no duplicates) and the sentinel is delivered exactly once. This +// is the invariant that must hold on every run; a broken acquire/release +// pairing would surface as a lost/duplicated value or (under TSan) a data +// race on has_eof_/eof_value_. +// +// What they deliberately do NOT assert is that the sentinel is the *strictly +// last* item popped. pop() checks emptiness (h == t) using a tail_ snapshot +// taken at the top of its loop; the producer can push more values *and* the +// sentinel in the window before take_sentinel() runs, so the consumer may +// surface the sentinel with a few real values still queued behind it. Those +// values are not lost — a consumer that keeps draining still receives them — +// but "sentinel arrives dead last" is not a property the channel promises, so +// asserting it would be flaky. We track how many values trailed the sentinel +// for visibility without failing on it. + +TEST_CASE("SPSC: sentinel and all values survive contention (blocking pop)", + "[channel][stress]") { + constexpr int N = 20'000; + constexpr int SENTINEL = -1; + + for (int rep = 0; rep < kReps; ++rep) { + // Small ring + tiny spin window so the ring is frequently empty exactly + // when the sentinel is published — the interleaving under test. + Channel ch(/*capacity=*/4, /*spin_count=*/8); + + std::thread producer([&] { + for (int i = 0; i < N; ++i) { + for (;;) { + try { ch.push(i); break; } + catch (const ChannelOverflowError&) { std::this_thread::yield(); } + } + } + ch.push_sentinel(SENTINEL); // must-deliver, never overflows/blocks + }); + + std::vector seen(N, false); + int values = 0; + int sentinels = 0; + bool duplicate = false; + // Drain until the sentinel AND all N values have been received; the + // sentinel may arrive before the last few values (see note above). + while (values < N || sentinels == 0) { + int v = ch.pop(); + if (v == SENTINEL) { ++sentinels; continue; } + if (seen[v]) duplicate = true; else seen[v] = true; + ++values; + } + producer.join(); + + REQUIRE_FALSE(duplicate); + REQUIRE(values == N); // every value delivered exactly once + REQUIRE(sentinels == 1); // sentinel delivered exactly once + REQUIRE(ch.size() == 0); + REQUIRE(ch.approx_size() == 0); + } +} + +TEST_CASE("SPSC: sentinel and all values survive contention (try_pop_now)", + "[channel][stress]") { + // The pool-node consume path is try_pop_now(), not pop(): it must surface + // the out-of-band sentinel once the ring is observed empty. The consumer + // spins with no sleeps, racing the producer at full tilt across the + // empty-ring boundary where take_sentinel() is reached. + constexpr int N = 20'000; + constexpr int SENTINEL = -1; + + for (int rep = 0; rep < kReps; ++rep) { + Channel ch(/*capacity=*/4, /*spin_count=*/0); + + std::thread producer([&] { + for (int i = 0; i < N; ++i) { + for (;;) { + try { ch.push(i); break; } + catch (const ChannelOverflowError&) { std::this_thread::yield(); } + } + } + ch.push_sentinel(SENTINEL); + }); + + std::vector seen(N, false); + int values = 0; + int sentinels = 0; + bool duplicate = false; + int v; + while (values < N || sentinels == 0) { + if (!ch.try_pop_now(v)) { std::this_thread::yield(); continue; } + if (v == SENTINEL) { ++sentinels; continue; } + if (seen[v]) duplicate = true; else seen[v] = true; + ++values; + } + producer.join(); + + REQUIRE_FALSE(duplicate); + REQUIRE(values == N); + REQUIRE(sentinels == 1); + // Sentinel held no ring slot; once drained the channel is fully empty. + REQUIRE(ch.size() == 0); + REQUIRE(ch.approx_size() == 0); + } +} From 3ac2242df140e2067d7f85f94976f01ef49c62be Mon Sep 17 00:00:00 2001 From: Duncan Tourolle Date: Sat, 4 Jul 2026 20:40:56 +0200 Subject: [PATCH 09/42] docs: rewrite SPEC.md to match the implemented library The spec had drifted far from the code. Key corrections: - Execution model is reactive (PoolNode submits fire_once() to a ThreadPool when inputs are ready), not one blocking thread per node - Channel is a lock-free SPSC ring buffer (atomic wait/notify + spin-before-sleep), not a mutex+CV queue - Remove latch<> ports (never implemented) - NodeErrorHandler returns bool (skip vs stop); per-node - Document new subsystems: scheduler, InterruptNode, Router/FilterNode, MainThreadNode, SharedResource, DebugHub, diagnostics/stats layer - Update StaticNetwork, Python auto_bind layer, examples 01-16 Co-Authored-By: Claude Opus 4.8 --- SPEC.md | 1726 ++++++++++++++++++------------------------------------- 1 file changed, 571 insertions(+), 1155 deletions(-) diff --git a/SPEC.md b/SPEC.md index 5e422ef..8a7d048 100644 --- a/SPEC.md +++ b/SPEC.md @@ -2,9 +2,24 @@ ## Overview -A C++20 template-metaprogramming library for building Kahn Process Networks, where each node -wraps a function/method, runs in its own thread, and communicates via bounded FIFO queues. -Includes nanobind bindings for Python graph construction and prototyping. +A header-only C++20 template-metaprogramming library for building Kahn Process Networks. Each +node wraps a function (or callable object); its input types are inferred from the parameter +list and its output types from the return type. Nodes communicate over bounded, lock-free +SPSC FIFO channels. + +Unlike a naive "one blocking thread per node" model, KPN++ is **reactive**: a node is +scheduled onto a thread pool whenever all of its input channels have data. A node that wraps a +function with `Node<>` owns a private single-thread pool and behaves exactly like an +independent worker; multiple nodes can instead share one `ThreadPool` for bounded-thread +execution. Source nodes self-resubmit; event-driven sources (`InterruptNode`) fire on an +external trigger. + +The library ships rich runtime diagnostics (per-node exec/CPU/throughput stats, per-channel +fill/bandwidth/overflow counters, pool and shared-resource utilisation), an optional in-process +web debug UI, and nanobind-based Python bindings (partially implemented). + +> **Note on accuracy.** This document describes the code as it exists in `include/kpn/`. Where +> a behaviour is subtle the relevant header is named so the source remains the ground truth. --- @@ -14,39 +29,50 @@ Includes nanobind bindings for Python graph construction and prototyping. kpn++/ ├── CMakeLists.txt ├── include/kpn/ -│ ├── fixed_string.hpp # NTTP string type for named ports -│ ├── traits.hpp # Function signature introspection -│ ├── channel.hpp # Bounded FIFO channel + storage policy -│ ├── node.hpp # Node wrapper + thread management -│ ├── port.hpp # Input/Output port handles -│ ├── network.hpp # Graph builder + orchestrator/watchdog -│ ├── variant_node.hpp # Runtime-typed node for Python graphs -│ └── python/ -│ └── bindings.hpp # Nanobind binding helpers -├── src/ -│ └── network.cpp # Orchestrator thread impl -├── tests/ -├── examples/ -│ ├── 01_hello_pipeline/ -│ ├── 02_named_ports/ -│ ├── 03_multi_output/ -│ ├── 04_storage_policy/ -│ ├── 05_error_handling/ -│ ├── 06_watchdog/ -│ ├── 07_python_network/ -│ ├── 08_python_subport/ -│ └── 09_opencv_cellshade/ # optional, requires OpenCV -└── python/ - └── kpn_python.cpp # Nanobind module definition +│ ├── fixed_string.hpp # NTTP string + in<>/out<> tags + index_of +│ ├── traits.hpp # function signature introspection, normalised_return_t, repeat_tuple +│ ├── diagnostics.hpp # NodeStats, ChannelStats, *Snapshot, IPoolProbe, IResourceProbe +│ ├── channel.hpp # lock-free SPSC ring-buffer Channel + storage policy +│ ├── port.hpp # InputPort / OutputPort handles +│ ├── inode.hpp # INode interface, NodeErrorHandler, NodeEvent +│ ├── scheduler.hpp # IScheduler + work-stealing ThreadPool +│ ├── pool_node.hpp # PoolNode / PoolObjectNode (reactive, scheduler-driven) +│ ├── interrupt_node.hpp # InterruptNode (external-trigger source) +│ ├── node.hpp # Node / ObjectNode (PoolNode + private 1-thread pool) + make_node +│ ├── fanout.hpp # FanoutNode + make_fanout +│ ├── branch.hpp # RouterNode + FilterNode + make_router / make_filter +│ ├── shared_resource.hpp # SharedResource priority-arbitrated exclusive resource +│ ├── main_thread_node.hpp # MainThreadNode<> (GUI / main-thread-bound nodes) +│ ├── static_network.hpp # Edge<>, make_network(), StaticNetwork<> +│ ├── network.hpp # runtime Network builder + watchdog + diagnostics +│ ├── debug_hub.hpp # DebugHub multi-network web UI (KPN_WEB_DEBUG only) +│ ├── web_debug.hpp # single-network web debug server (KPN_WEB_DEBUG only) +│ ├── variant_node.hpp # runtime-typed nodes/channels for Python graphs +│ ├── tmp/ +│ │ ├── fanout_groups.hpp # compile-time fan-out detection + edge expansion +│ │ ├── topo_sort.hpp # compile-time DFS cycle check + topological order +│ │ └── repeat_tuple.hpp # repeat_tuple_t +│ ├── python/ +│ │ ├── bindings.hpp # PyNetwork / PyNode nanobind helpers +│ │ └── auto_bind.hpp # NodeRegistry / Entry / bind_network / bind_debug +│ └── kpn.hpp # umbrella header +├── src/network.cpp +├── tests/ # Catch2 v3 + GoogleTest +├── examples/ # 01–16 (see Examples) +├── benchmarks/ # bench_pipeline (optional, KPN_BUILD_BENCHMARKS) +└── python/kpn_python.cpp # nanobind module definition ``` +`kpn.hpp` is the umbrella header; including it pulls in the full C++ API (the Python layer is +included only by the binding TU). + --- -## Component 0 — `fixed_string.hpp`: NTTP String +## Component 0 — `fixed_string.hpp`: NTTP String + Port Tags Named ports use C++20 non-type template parameters (NTTPs). `std::string_view` and -`const char*` are not valid NTTPs because they are not structurally comparable. The standard -solution is a `fixed_string` literal type with `constexpr` internal storage. +`const char*` are not valid NTTPs, so a `fixed_string` literal type provides `constexpr` +internal storage. ```cpp template @@ -57,78 +83,105 @@ struct fixed_string { constexpr std::string_view view() const { return {data, N - 1}; } }; -// Deduction guide — required so fixed_string("img") works as an NTTP. -// Without it the compiler cannot infer N and the named-port API does not compile. template -fixed_string(const char (&)[N]) -> fixed_string; +fixed_string(const char (&)[N]) -> fixed_string; // deduction guide (required) ``` -`fixed_string<3>` and `fixed_string<6>` are distinct types, so `input<"img">()` and -`input<"sigma">()` produce different template instantiations — this is intentional and -enables zero-overhead compile-time port dispatch. +`fixed_string<4>` and `fixed_string<7>` are distinct types, so `input<"img">()` and +`input<"sigma">()` produce different instantiations — enabling zero-overhead compile-time +port dispatch. -Named-port lookup uses a `constexpr` function over the name pack. It returns a sentinel -`npos` on miss rather than `static_assert`-ing internally, so the assertion fires at the -`input<"img">()` call site — giving the user a readable error at the point of use instead -of deep in template instantiation: +Named-port lookup uses a `constexpr` `index_of` over the name pack; it returns the sentinel +`npos` on a miss so the `static_assert` fires at the `input<"img">()` **call site**, giving a +readable error at the point of use: ```cpp inline constexpr std::size_t npos = std::size_t(-1); template -constexpr std::size_t index_of() { - std::size_t i = 0; - bool found = false; - ((Name == Names ? (found = true) : (found ? 0 : ++i)), ...); - return found ? i : npos; -} - -// Used at the call site: -template -auto input() { - constexpr std::size_t idx = index_of(); - static_assert(idx != npos, "unknown input port name"); - return input(); -} +constexpr std::size_t index_of(); // returns position or npos ``` +### Port tags + +`in<...>` and `out<...>` tag types disambiguate input vs. output name packs in the factory +API. Both are trivial empty structs; both are optional (omit to get index-only ports). + +```cpp +template struct in {}; +template struct out {}; +``` + +> There is **no `latch<>` tag.** An earlier design sketched latched (most-recent-value) +> input ports; this was not implemented and the only input kind is the synchronous one. + --- ## Component 1 — `traits.hpp`: Function Introspection -Extracts parameter types and return type from any callable at compile time. +Extracts parameter and return types from any callable at compile time, for free functions, +function pointers, member function pointers (const and non-const), lambdas and `std::function`. ```cpp -// For: Image blur(Image in, float sigma) -// function_traits::args == std::tuple -// function_traits::return_t == Image - -// For multi-output: std::tuple detect(Image in) -// return_t == std::tuple → 2 output ports -// return_t == Image → 1 output port (normalised to tuple internally) -// return_t == void → 0 output ports (sink node) +// function_traits::return_t, ::args (std::tuple<...>), ::arity +template using return_t = ...; // return type +template using args_t = ...; // std::tuple of parameters +template inline constexpr std::size_t arity_v = ...; ``` -Handles: free functions, lambdas, `std::function`, member function pointers. - -A helper alias normalises the return type to always be a tuple for uniform handling in -`run_loop`: +The return type is normalised to a tuple so every node has a uniform output-tuple shape: ```cpp -template -using normalised_return_t = - std::conditional_t, T, std::tuple>; -// void return → std::tuple<> (empty tuple, zero output ports) +// void → std::tuple<> (sink node, 0 outputs) +// T (non-tup) → std::tuple (1 output) +// tuple<...> → tuple<...> (one output port per element) +template using normalised_return_t = ...; +template inline constexpr std::size_t output_count_v = ...; +``` + +`repeat_tuple_t` (also surfaced via `tmp/repeat_tuple.hpp`) builds `std::tuple` +with `N` repetitions — used by `FanoutNode` and `RouterNode` to describe their N identical +output ports. + +--- + +## Component 2 — `diagnostics.hpp`: Statistics and Snapshots + +Shared timing types: `clock_t = std::chrono::steady_clock`, `duration_t` is a +`double`-millisecond duration. + +- **`NodeStats`** — atomic counters updated per fire: `frames_processed`, an EMA of wall-clock + exec time (`ema_exec_us`, warmup-mean for the first 5 frames then α=0.1), `max_exec_us`, + `total_blocked_us`, thread CPU time (`total_cpu_us` via `CLOCK_THREAD_CPUTIME_ID`), + `queue_wait_us` (pool queue latency), and `exec_start_us` (non-zero while executing; used by + the watchdog to detect hung nodes). +- **`ChannelStats`** — `pushes`, `bytes_pushed`, `drops`, `overflows`, `pops`, `peak_fill`. +- **Snapshots** — copyable plain structs taken by the watchdog / UI: `NodeSnapshot`, + `ChannelSnapshot` (with `fill_pct()`, `peak_pct()`, `bandwidth_mbs()`), `PoolSnapshot`, + `ResourceSnapshot`, and `NetworkSnapshot` (used by the `DebugHub`). +- **Probe interfaces** — `IPoolProbe` and `IResourceProbe` expose a `snapshot(name)` method so + pools and shared resources can be registered with a network for reporting. + +### `ChannelDataSize` trait + +`bytes_pushed` is computed from a specialisable trait, defaulting to `sizeof(T)`. Specialise it +for heap-owning payloads to get accurate bandwidth: + +```cpp +template<> struct kpn::ChannelDataSize { + static std::size_t bytes(const cv::Mat& m) { return m.total() * m.elemSize(); } +}; ``` --- -## Component 2 — `channel.hpp`: Bounded FIFO + Storage Policy +## Component 3 — `channel.hpp`: Lock-free Bounded FIFO + Storage Policy -### Storage Policy +### Storage policy -The type stored in a channel depends on a specialisable trait. Users can override it for -any type: +The type stored inside a channel is chosen by a specialisable trait. Small trivially-copyable +types are stored by value; everything else as `std::shared_ptr` so fan-out copies a +refcount, not data: ```cpp template @@ -137,29 +190,20 @@ struct channel_storage_policy { std::is_trivially_copyable_v && sizeof(T) <= 8; }; -// User opt-in to value semantics for a small struct: -template<> struct channel_storage_policy { - static constexpr bool by_value = true; -}; - -// Derived storage type: template using channel_storage_t = std::conditional_t< - channel_storage_policy::by_value, - T, - std::shared_ptr ->; + channel_storage_policy::by_value, T, std::shared_ptr>; ``` -### Channel +Override it to force value semantics for a custom small type. Push wraps a value in +`make_shared` when needed; pop dereferences it transparently, so a function taking +`const T&` works naturally and immutability is compiler-enforced. -`Channel` stores `channel_storage_t` internally. The producer calls `push(T value)` -and the channel transparently wraps it in `make_shared` when needed. All consumers -of the same channel receive the same `shared_ptr` — no copies of large objects. +### Channel — SPSC ring buffer -`run_loop` dereferences `shared_ptr` before passing to the wrapped function, so a -function declared `void f(const Image& img)` works naturally and the compiler enforces -immutability — no policy enforcement or `const_cast` needed. +`Channel` is a single-producer/single-consumer ring buffer (capacity rounded up to a power +of two). It uses C++20 `std::atomic::wait/notify_one` (portable futex) with a configurable +**spin-before-sleep** window so the common case never touches the kernel. ```cpp template @@ -167,590 +211,446 @@ class Channel { public: using storage_type = channel_storage_t; - explicit Channel(std::size_t capacity = 5); + explicit Channel(std::size_t capacity = 5, std::size_t spin_count = 200); - void push(T value); // wraps in shared_ptr if needed; throws on overflow - T pop(); // blocks (KPN semantics); unwraps shared_ptr if needed - bool try_pop(T& out, std::chrono::milliseconds timeout); + void push(T value); // drops if disabled; throws ChannelOverflowError if full + bool push_sentinel(T value); // out-of-band, non-blocking must-deliver token (EOF) + T pop(); // blocks (spin then futex); throws ChannelClosedError if disabled+empty + bool try_pop(T& out, std::chrono::milliseconds timeout); // polling (watchdog/display) + bool try_pop_now(T& out); // immediate, non-blocking - std::size_t size() const; + void enable(); // accept pushes + void disable(); // stop accepting + unblock any waiting pop() + void set_push_callback(std::function); // empty→non-empty notification + + std::size_t size() const; // ring occupancy (excludes any pending sentinel) + std::size_t approx_size() const; // size() + 1 if a sentinel is pending (readiness checks) std::size_t capacity() const; + bool is_accepting() const; + const ChannelStats& stats() const; + ChannelSnapshot snapshot(const std::string& name) const; }; -class ChannelOverflowError : public std::runtime_error {}; +class ChannelOverflowError : public std::runtime_error { /* capacity + optional context */ }; +class ChannelClosedError : public std::runtime_error {}; ``` +`head_` and `tail_`/`wake_` live on separate cache lines (`alignas(64)`) to avoid false +sharing between producer and consumer. `spin_hint()` issues a `pause`/`yield` instruction (or a +compiler fence on other ISAs). + +### The `push_callback` — how reactivity works + +`set_push_callback` registers a callback fired when a channel transitions empty→non-empty. A +consuming `PoolNode` installs this on each of its input channels; when an input becomes ready it +re-evaluates whether **all** inputs have data and, if so, submits itself to the scheduler. This +is the mechanism that replaces a dedicated blocking thread per node. + +### The out-of-band EOF sentinel — `push_sentinel` + +`push_sentinel(T value)` delivers a **must-deliver control token** (a graceful-EOF marker) that +cannot be dropped by backpressure. The value is stored in a dedicated slot **outside** the ring, +so it consumes no capacity, never throws `ChannelOverflowError`, and never blocks the producer. + +This matters because a node's worker cannot afford to block on a downstream push: parking that +thread would stop it draining its own input, cascading into a hold-and-wait deadlock under +backpressure. `push_sentinel` sets a published flag (`has_eof_`) and returns immediately, keeping +the worker free to keep popping. + +Ordering is preserved: the consumer's `pop()` / `try_pop_now()` drain the ring **first** and only +surface the sentinel once the ring is observed empty — so EOF always arrives after every value +pushed before it. `approx_size()` (used by node readiness checks) counts a pending sentinel as one +consumable item, so a channel carrying *only* a sentinel still schedules its consumer's next fire +and the token is never stranded. Same SPSC contract as `push()` (sole producer); returns `false` +if the channel is already disabled (teardown in progress → the token is moot). + +### Backpressure and shutdown — `accepting_` flag + +Each channel carries `std::atomic accepting_` (default `true`). It is the primary shutdown +mechanism; the only additional signal is the out-of-band EOF sentinel above, used for *graceful* +drain rather than an abrupt close. + +- **`push()`** on a disabled channel silently drops the value (recorded as a `drop`). On a + full accepting channel it throws `ChannelOverflowError` (a sizing error). +- **`pop()`** blocks while empty and accepting; `disable()` wakes it and it throws + `ChannelClosedError`. + +The **consumer node** owns its input channels and flips the flag: `start()` calls `enable()`, +`stop()` calls `disable()`. Producers never touch it. + ### Ownership -A `Channel` is **owned by its consumer node** — it lives as a member of the destination -node. The producer node holds a non-owning raw pointer to push into it. The channel is -destroyed when its consumer is destroyed, which is the correct lifetime. +Input channels are owned by their **consumer node** (held as `shared_ptr>`). A +producer node holds a non-owning raw `Channel*` to push into. `Network`/`StaticNetwork` are +otherwise non-owning of user nodes — see Components 8–9. -The `Network` itself is **non-owning** — nodes are declared by the user and outlive the -network. `net.add("name", node)` registers a raw pointer; the user is responsible for keeping -nodes alive for the network's lifetime. This avoids type-erasure ownership complexity and -keeps node construction explicit. +--- -### Backpressure and Shutdown — `accepting_` Flag +## Component 4 — `inode.hpp`: The Node Interface -Each channel carries a single `std::atomic accepting_` (default `true`). This is the -**sole shutdown mechanism** — no `try_pop` polling, no sentinel values, no drain logic. +Every node implements `INode`: ```cpp -template -class Channel { - std::atomic accepting_{true}; -public: - void push(T value) { - if (!accepting_.load(std::memory_order_relaxed)) return; // silently drop - // normal push — throws ChannelOverflowError if full - } +struct INode { + virtual ~INode() = default; + virtual void start() = 0; + virtual void stop() = 0; + virtual bool running() const = 0; + virtual const NodeStats& stats() const = 0; + virtual NodeSnapshot node_snapshot(const std::string& name, double elapsed_s) const = 0; + virtual void set_name(std::string) = 0; - void enable() { accepting_.store(true, std::memory_order_relaxed); } - void disable() { - accepting_.store(false, std::memory_order_relaxed); - clear(); // drop all queued items immediately - cv_.notify_all(); // unblock any waiting pop() - } + virtual void set_network_overflow_callback(NodeEventCallback) {} // network-injected + virtual void set_network_closed_callback(NodeEventCallback) {} + + virtual void halt() { stop(); } // immediate, discard in-flight work + virtual void shutdown() { stop(); } // graceful topo-ordered drain (overridden by networks) }; ``` -**Who flips the flag:** the **consumer node** — it's the channel owner. `node.stop()` calls -`disable()` on all its input channels. `node.start()` calls `enable()`. The producer never -touches the flag; it calls `push()` and if the channel is disabled the value is silently -dropped and the producer continues. +Supporting types: -**Overflow** (`push()` on a full, accepting channel) still throws `ChannelOverflowError` — -this signals a design error (undersized FIFO) and is unchanged. +```cpp +// Per-node error policy: return true to skip the failed fire and keep running, +// false to stop the node (and signal closed downstream). +using NodeErrorHandler = std::function; -**Blocking `pop()`** unblocks immediately when `disable()` is called (via `cv_.notify_all()`), -and throws `ChannelClosedError` if the queue is empty and the channel is disabled. - -### `try_pop` Purpose - -`try_pop` exists for **watchdog polling only** — not for shutdown (the `accepting_` flag -handles that) and not for normal processing. The watchdog uses it to probe whether a node -is making progress without blocking the watchdog thread. - -> **Note (C++ compile-time graphs):** In a fully compiled C++ graph the variant never -> appears. The compiler wires `Channel` to `Channel` directly. The variant is -> a pure compile-time construct used only for type-checking and generates zero runtime -> overhead. +using NodeEventCallback = std::function; +enum class NodeEvent { Overflow, Closed }; +``` --- -## Component 3 — `port.hpp`: Port Handles +## Component 5 — `scheduler.hpp`: Thread Pool ```cpp -template -struct InputPort { NodeT& node; }; - -template -struct OutputPort { NodeT& node; }; -``` - -Nodes expose port handles via: - -```cpp -// By index — always available -node_a.input<0>() // returns InputPort -node_b.output<1>() // returns OutputPort - -// By name — only valid when names were provided at make_node time -node_a.input<"img">() -node_b.output<"edges">() -``` - -Named access resolves to an index at compile time via `index_of` (see `fixed_string.hpp`). -Zero runtime cost — the name dispatch is fully eliminated by the compiler. - ---- - -## Component 4 — `node.hpp`: Node Wrapper - -```cpp -template< - auto Func, - fixed_string... InputNames, // optional; count must match arity or be 0 - fixed_string... OutputNames // optional; count must match output count or be 0 -> -class Node { -public: - explicit Node(std::size_t fifo_capacity = 5); - - void start(); - void stop(); // signals thread to finish current item then exit - - // Port access — by index - template auto input(); - template auto output(); - - // Port access — by name (compile error if names were not provided) - template auto input(); - template auto output(); - - static constexpr std::size_t input_count; - static constexpr std::size_t output_count; - -private: - void run_loop(); - // Pops each input channel, dereferences shared_ptr if needed, - // calls Func, unpacks the normalised tuple return, - // pushes each element to its output channel. - - std::thread thread_; - // Input channels owned here (one per input port). - // Output channel pointers (non-owning) set at connect time. +struct IScheduler { + virtual void submit(std::function task, float priority = 0.5f) = 0; + virtual void start() = 0; + virtual void stop() = 0; // join workers, discard pending tasks + virtual void drain() = 0; // block until in-flight tasks complete (workers keep running) }; ``` -**Factory syntax — `in<>` / `latch<>` / `out<>` tag structs:** - -A flat name pack `make_node` is ambiguous (where do inputs end?). -Option chosen: `in<...>`, `latch<...>`, and `out<...>` tag types that wrap the name packs unambiguously. -All are optional; omitting either means those ports are index-only. - -```cpp -// Tag types (trivial, no data): -template struct in {}; -template struct latch {}; -template struct out {}; - -// Factory: -// No names -auto node = make_node(/*fifo_capacity=*/10); - -// Input names only -auto node = make_node>(10); - -// Both input and output names -auto node = make_node, out<"blurred","mask">>(10); - -// Mixed synchronous and latched inputs -auto node = make_node, latch<"setpoint">, out<"output">>(10); -``` - -**Wrong name count is a compile error.** The `Node` class `static_assert`s that -`sizeof...(InputNames) == 0 || sizeof...(InputNames) == input_count` (and same for outputs). -Without this, a mismatch between name count and arity produces an unreadable template error. - -```cpp -static_assert( - sizeof...(InputNames) == 0 || sizeof...(InputNames) == input_count, - "make_node: number of input names must match function arity, or provide none" -); -``` - -Multi-output functions must return `std::tuple<...>`. Single return accepted as-is. -`void` return = sink node (no output ports). - -### Node identity: `Label` and `UniqueTag` NTTPs - -Two problems arise when using `Node` as a compile-time graph vertex: - -1. **Debug names** — without a label the web UI and `print_diagnostics` fall back to - `"node[0]"` placeholders. `set_name()` provides a runtime name for the runtime - `Network`, but `StaticNetwork` has no `add("name", node)` call to attach one. - -2. **Same-function collision** — two nodes wrapping the same function (e.g. two - `make_node`) have the same type. The `StaticNetwork` topo sort and fanout - detection use node types as graph vertices, so it cannot distinguish them, causing - infinite recursion in the DFS. - -Both are solved by two additional NTTPs on `Node`: - -```cpp -template< - auto Func, - typename InputTag = in<>, - typename LatchTag = latch<>, - typename OutputTag = out<>, - fixed_string Label = "", // human-readable name; shown in diagnostics/web UI - std::size_t UniqueTag = 0 // collision-breaker; must differ between any two -> // same-Func nodes in one make_network() call -class Node : public INode { ... }; -``` - -Both have defaults so all existing code (`make_node(5)`) compiles unchanged. - -**Factory syntax extension:** - -```cpp -// Label only (UniqueTag defaults to 0) -auto blur = make_node(8); - -// Both label and unique tag — required when the same function is used twice -auto preblur = make_node(8); -auto postblur = make_node(8); -``` - -**Duplicate-tag detection** — `make_network()` checks at compile time that no two nodes -in the edge pack share the same `(Func, UniqueTag)` pair. This fires a readable -`static_assert` at the call site before any thread is started: - -``` -static_assert(!has_duplicate_tags_v, - "make_network: two nodes have identical (Func, UniqueTag) — increment UniqueTag " - "on one of them to make them distinct"); -``` - -**Label availability** — `Node` exposes the label as a `static constexpr` so -`StaticNetwork` can read it at compile time for diagnostics: - -```cpp -static constexpr std::string_view label() { return Label.view(); } -``` - -The web debug UI and `print_diagnostics` use `label()` when non-empty, falling back to -`"node[]"` for unlabelled nodes. The runtime `Network` continues to use the -string passed to `add("name", node)` — `Label` is orthogonal to that mechanism. - -**`ObjectNode`** gains the same two NTTPs with the same defaults. The `make_node(obj, ...)` -overloads are extended identically. +`ThreadPool` is a **work-stealing** pool implementing both `IScheduler` and `IPoolProbe`. Each +worker owns a priority queue (max-heap by `priority`, FIFO within equal priority via a sequence +counter). `submit()` distributes round-robin; idle workers steal from the most-loaded peer +using `try_lock`, then sleep on a shared condition variable. The submit/notify path takes the CV +mutex around `notify` to close the lost-wakeup window; `drain()` waits on a separate counter of +in-flight tasks. `priority` lets a hot node (full input, empty output) be scheduled ahead of +others — see `PoolNode::compute_priority`. --- -## Component 4a — Latched Input Ports +## Component 6 — Node Types -### Motivation +All processing nodes share the same shape: typed input channels they own, raw output-channel +pointers set at wiring time, `args_tuple` / `return_tuple` aliases used by the connect-time +type check, and `static constexpr` `label()` / `unique_tag` / `input_count` / `output_count`. -Control and robotics applications naturally have two kinds of inputs at different update rates: +### `PoolNode` / `PoolObjectNode` — reactive, scheduler-driven (`pool_node.hpp`) -- **Synchronous inputs** (`in<>`) — the node must have fresh data on every fire. Typical for sensor readings that drive the computation (e.g. encoder RPM). -- **Latched inputs** (`latch<>`) — the node uses the most recently received value, and does not block if no new value has arrived. Typical for setpoints or parameters that change infrequently relative to the control loop (e.g. bearing from a CV pipeline, PID gains). - -Without latched ports, a node must block on all inputs simultaneously. This forces the control loop to run at the rate of the slowest input — unacceptable when a 1kHz encoder loop must wait for a 30Hz vision update. - -### Semantics - -A `latch<>` port: - -1. **Does not block** if its channel is empty — it reuses the last successfully popped value. -2. **Does block on first fire** — there is no meaningful "default" value, so the node waits until at least one value has arrived on each latched port before firing for the first time. -3. **Consumes the value** when one is available (standard `pop()`), then holds it until the next value arrives. - -The node fires whenever all `in<>` ports have data, using the last known value for each `latch<>` port. - -### Implementation in `run_loop` - -`run_loop` maintains a `std::tuple` of cached values, one slot per latched port. On each iteration: +The core node. Instead of a blocked thread, it submits a `fire_once()` to a shared +`IScheduler` whenever all inputs are ready; `queued_` ensures at most one `fire_once()` is +in flight. `fire_once()` pops every input (`try_pop_now`), runs the function, pushes each +normalised output, records stats, then resubmits if inputs remain ready. Source nodes +(`input_count == 0`) self-submit on `start()` and after each fire. ```cpp -// Synchronous ports — blocking pop (existing behaviour) -auto sync_args = std::make_tuple(input<0>().pop(), input<1>().pop(), ...); +template, + typename OutputTag = out<>, + fixed_string Label = "", + std::size_t UniqueTag = 0> +class PoolNode : public INode { ... }; -// Latched ports — non-blocking try_pop; keep cached value on miss -try_pop(latch_cache_, latch_channel_); // updates cache if data available - -// Call wrapped function with merged argument tuple -auto result = std::apply(Func, merge(sync_args, latch_cache_)); +auto n = make_pool_node(scheduler, fifo_capacity); // index ports +auto n = make_pool_node(scheduler, in<"a">{}, out<"b">{}, cap); ``` -Latched channels are otherwise identical to synchronous channels: bounded FIFO, `shared_ptr` storage policy, same shutdown behaviour. +`PoolObjectNode` is the same for a stateful callable object (introspected via +`&Obj::operator()`); the object must outlive the node. -### Example — PID with live setpoint +Per-node configuration: `set_error_handler(NodeErrorHandler)`, `set_overflow_callback`, +`set_closed_callback`, `set_max_exec_time`. Inside `fire_once()`: +`ChannelOverflowError` fires the overflow callbacks; `ChannelClosedError` (or an error handler +returning `false`) fires the closed callbacks and self-stops; any other exception consults the +error handler. + +**Name-count contract** — a `static_assert` requires that the number of input names is `0` or +equals arity (same for outputs): ```cpp -// bearing arrives at ~30 Hz from CV; rpm arrives at ~1 kHz from encoder -double pid_compute(double rpm, double bearing) { ... } - -auto pid = make_node, // synchronous — blocks until fresh encoder tick - latch<"bearing">, // latched — uses last known bearing from CV - out<"pwm"> ->(8); - -Network net; -net.add("tacho", tacho_node) - .add("tracker", tracker_node) - .add("pid", pid) - .connect("tacho", tacho_node.output<"rpm">(), "pid", pid.input<"rpm">()) - .connect("tracker", tracker_node.output<"bearing">(),"pid", pid.input<"bearing">()) - .build(); +static_assert(sizeof...(InNames) == 0 || sizeof...(InNames) == input_count, + "make_pool_node: number of input names must match function arity, or provide none"); ``` -The PID node fires at encoder rate. If no new bearing has arrived since the last tick, it reuses the previous one — correct behaviour for a control loop. +### `Node` / `ObjectNode` — convenience wrappers (`node.hpp`) -### Port Ordering Contract - -`in<>` and `latch<>` ports together must cover all function parameters in declaration order. The `static_assert` on name count is extended to cover both tags jointly: +`Node<>` privately owns a `ThreadPool(1)` and derives from `PoolNode<>` with the **same** +template signature, so each `Node` is a self-contained worker with no external scheduler. Its +`start()`/`stop()` start and stop the private pool around the base. This keeps the simple API — +`make_node(5)` — while routing all execution through the one `fire_once()` code path. ```cpp -static_assert( - sizeof...(InNames) + sizeof...(LatchNames) == input_count || - (sizeof...(InNames) == 0 && sizeof...(LatchNames) == 0), - "make_node: in<> and latch<> names together must match function arity, or provide none" -); +template, typename OutputTag = out<>, + fixed_string Label = "", std::size_t UniqueTag = 0> +class Node : public PoolNode<...> { ... }; + +auto src = make_node(5); +auto dbl = make_node(5); +auto cnt = make_node(in<"words">{}, out<"count","words">{}, 4); ``` -The function parameter at position `i` is synchronous if `i` is in the `in<>` pack, latched if in the `latch<>` pack. Mixed ordering is allowed — the tag packs define which positions are latched, not a contiguous suffix. +The `Label` NTTP gives a human-readable name for diagnostics; `UniqueTag` is a collision-breaker +required when the **same function** is used as two distinct vertices in a `StaticNetwork` (two +`make_node` would otherwise be the same type). Both default so existing code is unaffected. +To share one pool across many nodes for bounded-thread execution, use `make_pool_node` directly. + +### `InterruptNode` — external-trigger source (`interrupt_node.hpp`) + +A zero-input source driven by an external event (camera frame, timer, socket) instead of +self-resubmission. `get_trigger()` returns a thread-safe callable to hand to the event source; +each call increments a `pending_` counter and submits `fire_once()` on the 0→1 transition, +guaranteeing one execution per trigger even under bursts. It does not busy-loop. + +```cpp +auto cam = make_interrupt_node(scheduler, out<"frame">{}); +camera_sdk.on_frame_ready(cam.get_trigger()); +``` + +### `FanoutNode` — explicit fan-out (`fanout.hpp`) + +Reads one item and pushes a copy to each of N outputs (per-output overflow drops +independently). Runs on its own `std::jthread` blocking on `pop()`. Used directly in a runtime +`Network` via `make_fanout`, and auto-inserted by `make_network()` for `StaticNetwork`. + +### `RouterNode` / `FilterNode` — branching (`branch.hpp`) + +Both run on a dedicated `jthread`. `RouterNode` pushes each item to exactly **one** of N +outputs chosen by a `selector(item) -> size_t` (out-of-range index drops). `FilterNode` forwards +an item only when `pred(item)` is true. Factories: `make_router(sel)`, `make_filter(pred)`. + +### `MainThreadNode, Args…>` — GUI / main-thread nodes (`main_thread_node.hpp`) + +For work that *must* run on the thread owning a GUI event loop (OpenCV `imshow`/`waitKey` on +Wayland/Qt). It owns input channels and is registered as a normal `INode` (appears in +diagnostics) but spawns **no** thread. The application drives it by calling `step()` in a loop on +the main thread: `step()` does a zero-timeout `try_pop` on every input, and when all are ready +invokes the derived `operator()(Args…)` (returning `false` to stop). CRTP; the derived class +supplies the operator. --- -## Component 5 — `network.hpp`: Graph Builder + Orchestrator +## Component 7 — `shared_resource.hpp`: Priority-arbitrated Exclusive Resource -`Network` is **non-owning** — nodes are declared by the user and must outlive the network. -`add()` registers a raw pointer. Graph construction uses a builder pattern so the full -topology is known before `build()`, enabling cycle detection and topological ordering. +`SharedResource` wraps a singleton-like resource (an ONNX session, a CUDA stream) shared by +nodes across one or more networks, and arbitrates access with a **priority + aging** waiter +queue. Priority is re-evaluated at every release (so it reflects current queue state), and each +waiter's effective score grows with wait time (`kAgingPerSecond`) to prevent starvation. ```cpp -class Network : public INode { // Network is itself an INode — enables sub-networks +SharedResource res(session_args...); + +// inside a node functor: +auto guard = res.acquire_balanced(in_channel, out_channel); // RAII; releases on scope exit +guard->Run(...); +``` + +`acquire_balanced(in, out)` scores a waiter by `input_fill × output_headroom` — a node with a +full input queue and empty output is most urgent. `acquire(fn)` takes any `()->float` priority; +`acquire()` treats all waiters equally. Implements `IResourceProbe` so it shows up in +diagnostics and the debug hub. The factory is `make_shared_resource(args…)`. + +--- + +## Component 8 — `network.hpp`: Runtime Graph Builder + Watchdog + +`Network` is **non-owning** (nodes outlive it; `add()` stores `INode*`). A builder collects the +full topology before `build()`, enabling cycle detection and topological ordering. + +```cpp +class Network : public INode { public: - // Register a node by name. NodeT must satisfy INode. Network holds a raw pointer. - template - Network& add(std::string name, NodeT& node); + template Network& add(std::string name, NodeT& node); - // Connect output port of src to input port of dst. - // Type mismatch → static_assert at compile time. - template - Network& connect(const std::string& src_name, OutputPort, - const std::string& dst_name, InputPort); + template + Network& connect(const std::string& src, OutputPort, + const std::string& dst, InputPort); - // Expose an internal node's input/output as a boundary port of this (sub-)network. - // Allows a Network to be connected into a larger Network like a single node. - template - Network& expose_input(std::string boundary_name, InputPort); - - template + Network& expose_input (std::string boundary_name, InputPort); // sub-network port Network& expose_output(std::string boundary_name, OutputPort); - // DFS cycle check + topological sort. Throws NetworkCycleError on cycles. - Network& build(); + Network& build(); // DFS cycle check (throws NetworkCycleError) + topo sort - void start() override; // starts all internal nodes in topological order - void stop() override; // stops all internal nodes in reverse order; disables channels - bool running() const override; + void start() override; // start nodes in topo order; launch watchdog (+ web UI) + void stop() override; // == halt() + void halt() override; // immediate: stop nodes in reverse topo order + void shutdown() override; // graceful: stop source layers, drain channels, descend void set_watchdog_interval(std::chrono::milliseconds); + void set_error_handler(ErrorHandler); // void(node_name, exception_ptr) + void set_diagnostics_handler(DiagnosticsHandler); // fired each watchdog tick + void set_event_handler(EventHandler); // void(name, NodeEvent, timestamp) + void register_pool(const std::string&, IPoolProbe*); - using ErrorHandler = std::function; - void set_error_handler(ErrorHandler); - -private: - std::map nodes_; // non-owning - std::map> adj_; - std::vector topo_; - std::jthread watchdog_; - std::chrono::milliseconds watchdog_interval_{500}; - ErrorHandler error_handler_; + void print_diagnostics(std::ostream& = std::cerr) const; // formatted table }; ``` -**Node lifetime contract:** nodes must outlive the `Network`. The typical pattern is to -declare nodes and the network in the same scope: +- **`connect`** static-asserts that the source output type equals the destination input type + (via the nodes' `return_tuple` / `args_tuple`), sets the consumer's input channel as the + producer's output pointer, registers a `ChannelProbe` for diagnostics, and rejects a second + connection from the same output port (use `make_fanout`). +- **`build`** colours the graph DFS; a back-edge throws `NetworkCycleError`. It also wires each + node's network-level overflow/closed callbacks to the `EventHandler` if one is set. +- **`halt` vs `shutdown`** — `halt()` disables channels and stops nodes in reverse order + immediately; `shutdown()` walks source layers first, polling channel probes until they drain + before stopping the next layer. +- **Watchdog** — a `std::jthread` that wakes on `watchdog_interval_` (default 3 s), collects + snapshots, warns about nodes whose `exec_start_us` indicates an execution running > 5 s, and + either calls the diagnostics handler or prints the formatted report. +- **`expose_input`/`expose_output`** record boundary names (sub-network support is scaffolded; + `Network` is itself an `INode` and can be `add()`ed to an outer `Network`). -```cpp -// Nodes declared first — they own their input channels -auto blur = make_node>(10); -auto detect = make_node>(10); - -Network net; -net.add("blur", blur) - .add("detect", detect) - .connect("blur", blur.output<0>(), "detect", detect.input<0>()) - .connect("blur", blur.output<"blurred">(), "detect", detect.input<"img">()) - .build(); -net.start(); -``` - -**Sub-networks** — because `Network` implements `INode`, it can be registered inside a -larger `Network` as a named node. Boundary ports declared via `expose_input` / -`expose_output` make the internal nodes' ports available to the outer graph: - -```cpp -// Inner sub-network -auto stage1 = make_node(5); -auto stage2 = make_node(5); -Network pipe; -pipe.add("pre", stage1).add("enh", stage2) - .connect("pre", stage1.output<0>(), "enh", stage2.input<0>()) - .expose_input("img", stage1.input<0>()) - .expose_output("result", stage2.output<0>()) - .build(); - -// Outer network treats `pipe` as a single node -auto sink = make_node(5); -Network top; -top.add("pipe", pipe).add("sink", sink) - .connect("pipe", pipe.output<"result">(), "sink", sink.input<0>()) - .build(); -top.start(); -``` - -`NetworkCycleError` is thrown by `build()` if the graph contains a directed cycle. +The formatted report includes node (frames, exec ms, max ms, blocked ms, fps, cpu ms, util%), +channel (fill%, peak%, pushes, drops, overflow, MB/s, item bytes), and pool tables, plus a +bottleneck hint (highest `ema_exec_ms`). --- -## Component 6 — `variant_node.hpp`: Runtime-typed Node (Python graphs) +## Component 9 — `static_network.hpp`: Compile-time Graph Builder -### Motivation - -Python graphs cannot use compile-time type resolution. A `PyNetwork` is constructed with a -**closed list of C++ node types** known at binding time. The library derives a deduplicated -`std::variant` from all port types across those nodes. Type safety is enforced at -`connect()` time via string signatures. - -### Variant Deduplication - -All port types from the registered nodes are collected into a flat pack, duplicates are -removed via a `unique_types` TMP metafunction, then the variant is instantiated once: +For C++ graphs whose full topology is known at compile time. The complete edge list is a type +pack, so fan-out arity is known up front, cycle detection is a `static_assert`, and start/stop +are pointer-vector traversals rather than string-map + virtual dispatch. ```cpp -template -using py_variant_t = std::variant>>; +// edge() builds a typed Edge descriptor from two port handles. +template +Edge +edge(OutputPort, InputPort); + +// make_network() takes all edges, expands fan-outs, wires channels, returns a StaticNetwork. +template auto make_network(Edges&&... edges); ``` -This is pure TMP and runs entirely at compile time. The resulting variant has no redundant -alternatives at runtime. - -### PyNetwork Construction - -`make_py_network` is a **pure C++ template** — no CMake code-gen step. The variant is -derived entirely at compile time from the registered node type list. The nanobind module -definition is the single place where node types are listed; recompiling the extension is -the "registration" step. +Usage — no `add`/`connect`/`build`/string names; one source port feeding two destinations +auto-inserts a `FanoutNode`: ```cpp -// In kpn_python.cpp — list all node types that may appear in Python graphs: -auto py_net = make_py_network(); -// VariantValue = std::variant< /* deduplicated port types from A, B, C */ > -// Registers to_python / from_python converters for each alternative. +auto src = make_node(8); +auto blur = make_node(8); +auto detect = make_node(8); +auto sink = make_node(8); + +auto net = make_network( + edge(src.output<0>(), blur.input<0>()), + edge(src.output<0>(), detect.input<0>()), // same source port → FanoutNode inserted + edge(blur.output<0>(), sink.input<0>()), + edge(detect.output<0>(), sink.input<1>())); +net.start(); /* … */ net.stop(); ``` -### VariantChannel +`make_network` performs, at compile time: fan-out detection and edge expansion +(`tmp/fanout_groups.hpp`), a duplicate-`(Func, UniqueTag)` check +(`static_assert` — "add a UniqueTag"), and a cycle check + topological order +(`tmp/topo_sort.hpp`, `static_assert` — "graph contains a directed cycle"). At run time it +heap-allocates owned `FanoutNode` storage, collects user-node pointers in edge order, sets each +node's display name (`Label`, else `node[UniqueTag]`; fan-outs become `"_fanout"`), wires +every expanded edge, and builds channel probes. -```cpp -using VariantValue = py_variant_t; +`StaticNetwork` implements `INode` (so it can be embedded in a +runtime `Network`). It owns the fan-out nodes, holds user nodes by pointer, and provides +`start`/`halt`/`shutdown`, an `EventHandler`, `register_resource` / `register_pool`, +`print_diagnostics`, and `network_snapshot()` (consumed by the `DebugHub`). Compile-time labels +are read from each `NodeType::label()`. -class VariantChannel { -public: - explicit VariantChannel(std::size_t capacity = 5); - void push(VariantValue v); // throws ChannelOverflowError if full - VariantValue pop(); // blocks (KPN semantics) -}; -``` - -### VariantNode - -Wraps a registered C++ node type. Its `run_loop` uses `std::visit` to extract the concrete -type from a `VariantValue`, calls the underlying function, then wraps the result back into a -`VariantValue` for the output channel. - -```cpp -class VariantNode { -public: - std::string input_type_sig(std::size_t idx) const; - std::string output_type_sig(std::size_t idx) const; - - void connect_input (std::size_t port, std::shared_ptr); - void connect_output(std::size_t port, std::shared_ptr); - - void start(); - void stop(); -}; -``` - -### PythonConverter — Crossing the C++/Python Boundary - -Every type in the variant must provide a `PythonConverter` specialisation. This is the -single mechanism used for all data crossing into or out of Python (PyNodes, `net.read`, -`net.write`): - -```cpp -template -struct PythonConverter { - static nanobind::object to_python(const T&); - static T from_python(nanobind::object); -}; -``` - -### PyNode — Pure Python Processing Node - -A `PyNode` holds a `nanobind::object` as its function. Its `run_loop`: - -1. Pops `VariantValue` from each input channel -2. `std::visit` → calls `PythonConverter::to_python` for each → **acquires GIL** -3. Calls the Python callable -4. **Releases GIL** → calls `PythonConverter::from_python` on the return value -5. Pushes result as `VariantValue` to output channel - -### Sub-value Extraction and Injection - -A C++ node returning `std::tuple` exposes three independent output ports. Each -element is pushed to its own `VariantChannel` — sub-indexing is a first-class concept at the -channel level, not an afterthought. - -**Python tap — read one output port into Python:** - -```python -value = net.read("detect", output=2) -# Pops from output channel 2, calls PythonConverter::to_python. -# GIL released while blocking on pop(), re-acquired before to_python call. -``` - -**Python inject — write a Python value into a specific input port:** - -```python -net.write("blur", input=1, value=my_sigma) -# Calls PythonConverter::from_python(my_sigma), pushes to input channel 1. -# GIL released while blocking on push() if channel is full. -``` - -**Python splitter node:** - -```python -def split(packed): - img, mask, score = packed - return img, mask - -net.add_node("split", split, inputs=["packed"], outputs=["img", "mask"]) -net.connect("detect", 0, "split", 0) -net.connect("split", 0, "show", 0) -net.connect("split", 1, "save", 0) -``` - -**Direct C++ sub-output to Python node input:** - -```python -net.connect("detect", 1, "py_thresh", 0) -# Type sig of detect:output[1] must match py_thresh:input[0] — checked at connect(). -``` - -### Type-check at connect time (Python) - -```python -# Raises kpn.TypeError if signatures don't match -net.connect("blur", 0, "thresh", 0) # (src_name, out_idx, dst_name, in_idx) -``` +> `make_fanout` remains for explicit fan-out in a runtime `Network`; `make_network` users +> never call it. --- -## Component 7 — Orchestrator / Watchdog +## Component 10 — Web Debugging (optional, `KPN_WEB_DEBUG`) -Runs in its own dedicated thread inside `Network` / `PyNetwork`. Responsibilities: +Zero cost when disabled — guarded headers, no symbols, no dependency. Depends on **cpp-httplib** +(single-header, fetched by CMake when the option is on) and loads **D3.js v7** from CDN. Enable +per-target: -- Starts nodes in topological order; stops them in reverse order -- Tracks per-node execution time (exponential moving average) -- Emits warning via logger callback (default: `stderr`) when a node stalls beyond threshold -- Catches exceptions from node threads and routes them to `ErrorHandler` -- Graceful shutdown: signals all nodes, joins with timeout, reports any that fail to stop +```cpp +#define KPN_WEB_DEBUG 1 +#include +``` + +### Single-network server (`web_debug.hpp`) + +When enabled, `Network` / `StaticNetwork` gain `set_web_debug_port(uint16_t)` (default 9090) and +auto-start an in-process HTTP server in `start()`. It serves an inline single-page app at `/` and +a JSON snapshot at `/api/snapshot` (nodes, channels/edges, pools, resources, elapsed). The page +renders a force-directed graph: node colour encodes `ema_exec_ms`, edge colour encodes fill%, +with hover tooltips for the full stat set; it polls every 500 ms. + +### `DebugHub` — multi-network UI (`debug_hub.hpp`) + +A standalone server aggregating several networks under one endpoint: + +```cpp +DebugHub hub(9090); +hub.register_network("detect", detect_net); // disables that net's own server +hub.register_network("classify", classify_net); +hub.register_resource("gpu", &gpu_resource); // shows utilisation cards +hub.start(); +``` + +The hub UI has one tab per registered network plus an "All Networks" tab with shared-resource +cards and a cross-network node table. `register_network` calls `net.disable_web_server()` so the +hub is the single debug endpoint; call it before `net.start()`. --- -## GIL Rules (non-negotiable constraints on binding implementation) +## Component 11 — Python Bindings (partial) -Two rules govern all interaction between node threads and the Python interpreter: +> Status: scaffolded and partially implemented. The variant machinery, `PyNetwork`/`PyNode`, and +> the auto-binding layer exist; the demo module wires a hello-pipeline. Full sub-port read/write +> and mixed C++/Python graphs are still in progress. -1. **Acquire for callback** — a node thread must hold the GIL only for the duration of a - Python callable invocation (`nb::gil_scoped_acquire` wrapping the call site). +Python graphs cannot resolve types at compile time, so a `PyNetwork` is parameterised by a +`std::variant` derived (at compile time, via `unique_types`) from the port types of a **closed +list of registered C++ node types**. The variant only appears at the C++/Python boundary; each +node's internal `Channel` still stores raw `T` (`variant_node.hpp`: `IVariantChannel`, +`VariantChannel`, `IVariantNode`, `VariantNodeWrapper`). -2. **Release while blocking** — any blocking operation on a channel (`pop()`, `push()`, - `net.read()`, `net.write()`) must release the GIL before blocking - (`nb::gil_scoped_release` wrapping the call site), then re-acquire after. +### Auto-binding (`python/auto_bind.hpp`) -Violating rule 2 deadlocks: a PyNode thread waiting to acquire the GIL cannot proceed while -another thread holds the GIL and blocks on a channel waiting for that PyNode to produce. +The node list is declared once with a `NodeRegistry` of `Entry`. `bind_network` +registers the `PyNetwork` class, a `make_(capacity)` factory and a `Node` class per +entry, and auto-registers `PythonConverter` for each port type. `bind_debug` additionally exposes +each raw C++ function as a free Python callable for testing without a network. Recompiling the +extension is the registration step — there is no CMake code-gen. + +```cpp +using DemoNodes = kpn::python::NodeRegistry< + kpn::python::Entry, + kpn::python::Entry, + kpn::python::Entry>; // variant auto-deduced as std::variant + +NB_MODULE(kpn_python, m) { + bind_network(m); + bind_debug(m); +} +``` + +Custom types are supported by specialising `kpn::PythonConverter` (`to_python` / `from_python`, +optional `type_name`) before `bind_network`. + +### GIL rules (non-negotiable) + +1. **Acquire for callback** — hold the GIL only for the duration of a Python callable + invocation (`nb::gil_scoped_acquire` around the call site). +2. **Release while blocking** — release the GIL before any blocking channel op + (`nb::gil_scoped_release`), then re-acquire. Violating this deadlocks: a PyNode thread + waiting for the GIL cannot proceed while another thread holds it and blocks on a channel + waiting for that PyNode. --- @@ -758,616 +658,132 @@ another thread holds the GIL and blocks on a channel waiting for that PyNode to | Situation | Behaviour | |---|---| -| FIFO overflow | `ChannelOverflowError` thrown in producer thread → `ErrorHandler` | -| Node function throws | Exception pointer captured → `ErrorHandler` | -| Type mismatch (C++) | `static_assert` at `connect()` compile time | -| Type mismatch (Python) | `kpn.TypeError` raised at `net.connect()` call | -| Cycle in graph | `NetworkCycleError` thrown at `build()` time | -| Thread fails to stop | Watchdog warning after configurable timeout | -| `from_python` / `to_python` fails | Exception propagated to `ErrorHandler` | +| FIFO overflow (full, accepting) | `ChannelOverflowError` thrown in producer; node overflow callbacks fire | +| Push to a disabled channel | Value silently dropped (counted as a `drop`) | +| Node function throws | Routed to the node's `NodeErrorHandler` → `true` skips & continues, `false` stops the node | +| Node stopped / channel closed | `ChannelClosedError` → node fires closed callbacks and self-stops | +| Type mismatch (C++) | `static_assert` at `connect()` / `make_network()` | +| Cycle in graph (runtime) | `NetworkCycleError` thrown at `build()` | +| Cycle in graph (static) | `static_assert` at `make_network()` | +| Duplicate `(Func, UniqueTag)` (static) | `static_assert` at `make_network()` — add a `UniqueTag` | +| Hung node | Watchdog warning after threshold | ---- - -## Future Extension Points (Heterogeneous Execution) - -Not implemented now, but the design must not close these doors: - -- **`IChannel` abstract interface** — `Channel` and a future `RemoteChannel` (wrapping - a socket/queue) would share the same `push`/`pop` interface. Nodes never know whether - their channel is in-process or remote. - -- **`Serializer` trait** — parallel to `PythonConverter` and `channel_storage_policy`, - a specialisable trait for cross-device serialisation (MessagePack for ESP32, pinned memory - for GPU zero-copy, etc.). - -- **`NodeKind` tag** — `enum class NodeKind { Local, Gpu, Remote }` on the `INode` - interface, letting the watchdog apply different health-check and timeout strategies per - device type. - -These three extension points are sufficient to support GPU and embedded/network targets -without redesigning the core. +`Network` additionally exposes an aggregate `EventHandler(name, NodeEvent, timestamp)` for +overflow/closed events across all nodes. --- ## Thread Model -**v1: one `std::thread` per node.** This maps directly to KPN theory and is simple to reason -about. It does not scale to networks with hundreds of nodes but is appropriate for the -typical use case (tens of nodes, each doing non-trivial work). +KPN++ is **reactive**, not one-thread-per-node: -`std::jthread` (C++20) is preferred over `std::thread` where available, as it provides a -built-in `stop_token` that simplifies the `stop()` / `try_pop` shutdown pattern. +- A `PoolNode` owns no thread. It registers a push-callback on each input channel; when all + inputs are ready it submits `fire_once()` to a shared `IScheduler` (a `ThreadPool`). +- `Node<>` wraps a `PoolNode` plus a **private `ThreadPool(1)`**, recovering "independent + worker" semantics with the simple `make_node` API. Many nodes can instead share one pool + (`make_pool_node`) for a bounded OS thread count. +- `FanoutNode`, `RouterNode`, and `FilterNode` do run a dedicated `std::jthread` blocking on + `pop()` (they are simple, latency-sensitive routers). +- `InterruptNode` fires on an external trigger; `MainThreadNode` runs on the caller's main + thread via `step()`. -A future executor/thread-pool model (where multiple nodes share a pool of threads and are -scheduled cooperatively) is a possible v2 extension. The `INode` interface is designed to -not assume a 1:1 thread mapping. +`std::jthread` (C++20) and its `stop_token` are used where a thread is owned, simplifying +cooperative shutdown. Benchmarks (`benchmarks/bench_pipeline`) show ~2–7 µs/hop framework +overhead for chains within the core count, rising under oversubscription. --- ## Platform and Compiler Requirements -C++20 is required. Specific features used: +C++20 is required. -| Feature | Header / Standard | Min compiler | -|---|---|---| -| NTTP structural types (`fixed_string`) | language | GCC 11, Clang 13, MSVC 19.29 | -| `std::is_trivially_copyable_v` | `` | C++17+ | -| `std::jthread` + `stop_token` | `` | GCC 11, Clang 14, MSVC 19.29 | -| `if constexpr`, fold expressions | language | C++17+ | -| `auto` NTTPs | language | C++20 | -| Concepts (`requires`) | language | GCC 10, Clang 10 | +| Feature | Min compiler | +|---|---| +| NTTP structural types (`fixed_string`) | GCC 11, Clang 13, MSVC 19.29 | +| `std::atomic::wait/notify` (channel futex) | GCC 11, Clang 13, MSVC 19.29 | +| `std::jthread` + `stop_token` | GCC 11, Clang 14, MSVC 19.29 | +| `auto` NTTPs, fold expressions, `if constexpr`, concepts | C++20 / C++17 baseline | -**Minimum supported compilers:** GCC 11, Clang 13, MSVC 19.29 (VS 2022). -nanobind requires Python 3.8+ and a C++17-capable compiler (satisfied by the above). +`CLOCK_THREAD_CPUTIME_ID` (per-thread CPU stats in `diagnostics.hpp`) is POSIX. nanobind +requires Python 3.8+ (auto-fetched when `KPN_BUILD_PYTHON=ON`). --- ## Testing Strategy -Test frameworks: **Catch2 v3** (header-friendly, good async/threading support via -`REQUIRE_NOTHROW` + thread join patterns) and **Google Test** (for death tests and -parameterised test suites). Both are included; use Catch2 for integration/behaviour tests -and GTest for unit tests where `ASSERT_*` / `EXPECT_*` macros and death tests are -preferable. +**Catch2 v3** for behaviour/integration tests and **GoogleTest** for unit and death tests; both +are auto-fetched. Existing suites: `test_fixed_string`, `test_traits`, `test_channel`, +`test_node`, `test_network`, `test_static_network`, `test_scheduler`, `test_pool_node`, +`test_shared_resource`. -### Hard cases to cover explicitly: - -| Case | What to test | -|---|---| -| Channel blocking | `pop()` blocks until a producer pushes; unblocks exactly once per push | -| Channel overflow | `push()` beyond capacity throws `ChannelOverflowError` | -| Shutdown race | `stop()` called while a node is blocked on `pop()` — thread must exit cleanly | -| Multi-consumer | Two nodes connected to the same output channel each receive every item (fan-out) | -| Tuple unpacking | Multi-output node pushes correct type to each sub-channel | -| Cycle detection | `build()` throws `NetworkCycleError` for a graph with a cycle | -| Named port lookup | `input<"wrong">()` fires `static_assert`; `input<"right">()` resolves correctly | -| Wrong name count | `make_node` with mismatched name count fires readable `static_assert` | -| GIL deadlock | PyNode + blocking `net.read()` from Python do not deadlock | -| `from_python` failure | Exception propagates to `ErrorHandler`, network continues | -| `channel_storage_policy` | Large type is stored as `shared_ptr`; small type by value | +Cases covered explicitly include: channel blocking/unblocking and overflow; shutdown races +(`stop()` while blocked on `pop()`); `try_pop_now`; fan-out delivery; tuple unpacking to +sub-channels; runtime cycle detection and static cycle/duplicate-tag `static_assert`s; named +port lookup and wrong-name-count `static_assert`s; storage-policy by-value vs `shared_ptr`; +scheduler submit/steal/drain; `PoolNode` reactive scheduling; and `SharedResource` priority + +aging. --- ## Examples -Each example is a self-contained program under `examples/`. They are built as part of the -CMake build and serve as both documentation and smoke tests. +Self-contained programs under `examples/`, built by default (`-DKPN_BUILD_EXAMPLES=OFF` to +skip). They double as documentation and smoke tests. -### `examples/01_hello_pipeline` — Basic linear pipeline - -Two nodes connected in sequence. Demonstrates `make_node`, `Network` builder, -index-based port connection, `start_all` / `stop_all`. - -```cpp -// producer → transform → sink -int produce() { return 42; } -int double_it(int x) { return x * 2; } -void print_it(int x) { std::cout << x << '\n'; } - -auto src = make_node(5); -auto dbl = make_node(5); -auto sink = make_node(5); - -Network net; -net.add("src", src) - .add("dbl", dbl) - .add("sink", sink) - .connect("src", src.output<0>(), "dbl", dbl.input<0>()) - .connect("dbl", dbl.output<0>(), "sink", sink.input<0>()) - .build(); -net.start_all(); -``` - -### `examples/02_named_ports` — Named port access - -Same pipeline but using `in<>` / `out<>` name tags and named port access. Demonstrates -`fixed_string` NTTP dispatch and the `static_assert` on wrong names. - -```cpp -auto dbl = make_node, out<"result">>(5); -// ... -.connect("src", src.output<0>(), "dbl", dbl.input<"value">()) -.connect("dbl", dbl.output<"result">(), "sink", sink.input<0>()) -``` - -### `examples/03_multi_output` — Tuple-returning node / sub-port routing - -A single node returns `std::tuple`. Each output is routed to a different -downstream node. Demonstrates tuple normalisation, per-element channel push, and -`output<1>()` sub-indexing. - -```cpp -std::tuple detect(Image in) { ... } -void show_image(const Image& img) { ... } -void save_mask(const Mask& m) { ... } - -// detect:output<0> → show_image, detect:output<1> → save_mask -``` - -### `examples/04_storage_policy` — `channel_storage_policy` specialisation - -Shows the default behaviour (large struct stored as `shared_ptr`, small int by -value) and a user specialisation that overrides the default for a custom type. - -```cpp -struct BigFrame { uint8_t pixels[1920*1080*3]; }; -// stored as shared_ptr automatically - -struct Tiny { float x, y; }; // 8 bytes — by value by default -template<> struct channel_storage_policy { static constexpr bool by_value = true; }; -``` - -### `examples/05_error_handling` — Overflow and node exceptions - -Demonstrates `ChannelOverflowError` (producer faster than consumer, tiny FIFO), custom -`ErrorHandler`, and a node that throws mid-execution. - -```cpp -net.set_error_handler([](std::string_view name, std::exception_ptr ep) { - try { std::rethrow_exception(ep); } - catch (const std::exception& e) { - std::cerr << "[" << name << "] " << e.what() << '\n'; - } -}); -``` - -### `examples/06_watchdog` — Orchestrator / watchdog - -A node that artificially stalls. Shows watchdog warning emission, configurable interval, -and graceful shutdown after a timeout. - -```cpp -net.set_watchdog_interval(std::chrono::milliseconds(200)); -// stall_node sleeps for 2s per item — watchdog fires warning after 200ms -``` - -### `examples/07_python_network` — PyNetwork with C++ and Python nodes - -Python script that imports `kpn`, registers C++ node types via `make_py_network`, adds a -pure Python processing node, connects them, and runs the graph. - -```python -import kpn - -net = kpn.make_network([kpn.BlurNode, kpn.DetectNode]) - -def py_filter(img): - return img[::2, ::2] # downsample in Python - -net.add_node("blur", kpn.BlurNode, inputs=["img"]) -net.add_node("downsample",py_filter, inputs=["img"], outputs=["img"]) -net.add_node("detect", kpn.DetectNode, inputs=["img"]) -net.connect("blur", 0, "downsample", 0) -net.connect("downsample", 0, "detect", 0) -net.start() -``` - -### `examples/09_opencv_cellshade` — Real-time cell-shading with OpenCV (optional) - -Captures live video from a system camera and applies a cell-shading effect entirely inside -a KPN++ graph. Built only when OpenCV is found at CMake time; skipped silently otherwise. - -**Graph topology:** - -``` -[capture] ──Mat──> [split] ──B──> [median_b] ──B──┐ - ├──G──> [median_g] ──G──┤ - └──R──> [median_r] ──R──┴──> [merge] ──Mat──> [combine] ──> [display] -[capture] ──Mat──────────────────────> [detect_edges] ──mask──────────> [combine] -``` - -Effect steps: -1. **`split_channels`** — `cv::split` into three single-channel `cv::Mat` planes. -2. **`median_b/g/r`** — independent `cv::medianBlur(kernel=15)` per channel; large kernel - posterises colours into flat cartoon-like regions and runs in parallel across channels. -3. **`merge_channels`** — `cv::merge` back to BGR. -4. **`detect_edges`** — greyscale, `cv::Canny`, then `cv::dilate` to produce thick outlines. -5. **`combine`** — zeros out BGR pixels wherever the edge mask is non-zero → black outlines - drawn over the flat-colour image. -6. **`display`** — `cv::imshow`; ESC key signals shutdown via `g_running` atomic. - -Demonstrates: named ports, fan-out from a single node to two downstream paths, parallel -per-channel processing, multi-input `combine` node, and error handler driving graceful stop. - -```cpp -// Build only if OpenCV is present: -// cmake .. -DKPN_BUILD_EXAMPLES=ON -// ./09_opencv_cellshade [camera_index] # default: 0 -``` - -### `examples/08_python_subport` — Python sub-value tap and inject - -Shows `net.read("node", output=N)` and `net.write("node", input=N, value=v)` from Python, -plus connecting a C++ tuple output sub-port directly to a Python node input. - -```python -# Tap only output<1> (Mask) of a C++ detect node into Python -net.connect("detect", 1, "py_thresh", 0) -val = net.read("detect", output=0) # blocks until Image is available -net.write("blur", input=1, value=1.5) # inject sigma -``` - ---- - -## Component 8 — Web Debug UI (optional, compile-time toggle) - -An optional in-process HTTP server that serves a live graph visualisation of the running -network. Zero cost when disabled — no symbols compiled in, no headers pulled. - -### Toggle - -```cpp -// Before any kpn include — enables the web debug server -#define KPN_WEB_DEBUG 1 -#include -``` - -CMake projects that want it globally: - -```cmake -option(KPN_WEB_DEBUG "Enable KPN++ web debug UI" OFF) -if(KPN_WEB_DEBUG) - target_compile_definitions(my_app PRIVATE KPN_WEB_DEBUG=1) - # cpp-httplib is fetched automatically by CMake when this flag is ON -endif() -``` - -### Implementation - -`include/kpn/web_debug.hpp` — included by `network.hpp` only when `KPN_WEB_DEBUG` is defined. - -Depends on **cpp-httplib** (single-header, no external process, no Python required). -Served on `localhost:9090` by default (configurable via `net.set_web_debug_port(uint16_t)`). - -When enabled, `Network` gains: - -```cpp -#ifdef KPN_WEB_DEBUG -void set_web_debug_port(uint16_t port); // default 9090 -void start_web_debug(); // called internally by start() -void stop_web_debug(); // called internally by stop() -#endif -``` - -`start()` automatically calls `start_web_debug()` when `KPN_WEB_DEBUG` is defined. - -### Endpoints - -| Endpoint | Method | Description | -|---|---|---| -| `/` | GET | Serves the single-page HTML app (inline, no files needed) | -| `/api/snapshot` | GET | Returns a JSON snapshot of all node and channel stats | - -The HTML page is embedded as a C++ string literal — no asset files to deploy. - -### JSON Snapshot Format - -```json -{ - "nodes": [ - { "id": "src", "frames": 120, "ema_exec_ms": 33.2, "max_exec_ms": 45.1, - "blocked_ms": 0.1, "fps": 29.8 }, - { "id": "quant", "frames": 120, "ema_exec_ms": 4.1, ... } - ], - "edges": [ - { "source": "src", "target": "quant", "label": "colour", - "fill_pct": 12.5, "peak_pct": 87.5, "capacity": 8, "current": 1, - "pushes": 120, "drops": 0, "overflows": 0 } - ] -} -``` - -Node `id` comes from the name registered via `net.add("name", node)`. -Edge `label` comes from the channel name registered via `connect()` (format: `"src:N → dst:M"`). - -### Browser UI - -The page polls `/api/snapshot` every **500 ms** and renders a **D3.js v7 force-directed -graph**: - -- **Nodes** — circles labelled with node name; colour encodes exec load: - - green (`ema_exec_ms` < 10ms), yellow (10–50ms), orange (50–100ms), red (>100ms) - - hover tooltip shows: frames, ema_exec_ms, max_exec_ms, blocked_ms, fps -- **Edges** — directed arrows labelled with the channel name and fill%; colour: - - green (fill < 50%), yellow (50–80%), red (≥80%) — matches the `<<<` flag in the text report - - hover tooltip shows: pushes, drops, overflows, capacity - -D3 is loaded from CDN (`d3js.org`). The entire UI is a single inline HTML string in -`web_debug.hpp` — no file serving, no build step for assets. - -### Thread model - -`start_web_debug()` launches a `std::jthread` running `httplib::Server::listen()`. -The server is stopped via `httplib::Server::stop()` called from `stop_web_debug()`. -`/api/snapshot` calls `collect_snapshots()` (already thread-safe — reads atomics with -relaxed ordering) and serialises to JSON using a minimal hand-rolled serialiser -(no third-party JSON library required). - -### Example usage - -```cpp -#define KPN_WEB_DEBUG 1 -#include - -// ... build network as normal ... -net.set_web_debug_port(9090); // optional, 9090 is the default -net.start(); -// Open http://localhost:9090 in a browser -``` - ---- - -## CMake Layout - -| Target | Type | Notes | -|---|---|---| -| `kpn` | header-only interface library | C++20, no external deps | -| `kpn_python` | nanobind shared library | links `kpn`, requires Python 3.8+ | -| `kpn_tests` | executable | Catch2 v3 + Google Test | -| `kpn_examples` | executables (one per example) | built by default, off with `-DKPN_EXAMPLES=OFF` | -| `kpn_web_debug` | compile-time option | `#define KPN_WEB_DEBUG 1`; fetches cpp-httplib via CMake FetchContent | - ---- - -## Component 9 — `static_network.hpp`: Compile-time Graph Builder - -### Motivation - -The runtime `Network` builder has two limitations that only a compile-time graph can fix: - -1. **Fan-out `N` is unknowable at the first `connect()` call.** A `FanoutNode` requires - `N` as a template parameter. With runtime `connect()`, the network has seen only one edge - when the first call arrives; it cannot know how many more will follow for that port. Auto- - inserting the right `FanoutNode` requires seeing the complete edge list at once — - which is only possible if the edge list is a type. - -2. **Start/stop goes through virtual dispatch.** `Network` stores `INode*` and calls virtual - `start()`/`stop()`. With a typed node tuple the compiler sees the concrete types and can - inline or devirtualise. This matters at startup/shutdown, not in the hot path — but it is - avoidable overhead. - -The runtime `Network` is **not removed**. It remains the right choice for Python graphs, -sub-networks embedded in dynamic topologies, and any case where the graph shape is not known -until runtime. `StaticNetwork` is an additional builder for the common case where the full -C++ topology is known at compile time. - -### API - -```cpp -// edge() constructs a typed edge descriptor from two port handles. -// All type information (SrcNode, SrcIdx, DstNode, DstIdx) is in the return type. -template -auto edge(OutputPort, InputPort) - -> Edge; - -// make_network() accepts all edges as a variadic pack. -// It deduces the full topology, auto-inserts FanoutNodes where needed, -// wires all channels, and returns a StaticNetwork owning the fanout nodes. -// User nodes are held by reference (non-owning), same lifetime contract as Network. -template -auto make_network(Edges&&... edges) -> StaticNetwork<...>; -``` - -Usage: - -```cpp -auto src = make_node(8); -auto blur = make_node>(8); -auto detect = make_node>(8); -auto sink = make_node(8); - -// src:output<0> feeds both blur and detect — fan-out is auto-inserted -auto net = make_network( - edge(src.output<0>(), blur.input<0>()), - edge(src.output<0>(), detect.input<0>()), // same source port - edge(blur.output<0>(), sink.input<0>()), - edge(detect.output<0>(), sink.input<1>()) -); -net.start(); -// ... -net.stop(); -``` - -No `add()`, no `build()`, no string names. The graph is fully wired in the `make_network` -call. `start()` and `stop()` are non-virtual tuple traversals. - -### Edge type - -```cpp -// Carries references to the two endpoint nodes. Stores no data beyond that. -template -struct Edge { - SrcNode& src; - DstNode& dst; -}; -``` - -### Fan-out detection metafunction - -`make_network` receives `Edge<...>` types as a pack. Before wiring, a metafunction scans -the pack for output ports with more than one downstream edge: - -``` -fanout_groups -``` - -This is a compile-time multimap: keys are `(SrcNode type, SrcIdx)`, values are the list of -destination `(DstNode type, DstIdx)` pairs sharing that key. - -For each key with N > 1 destinations: -- Compute `T = std::tuple_element_t` -- Synthesise a `FanoutNode` — call it `F` -- Replace the N original edges with: - - one edge: `src:SrcIdx → F:input<0>` - - N edges: `F:output<0..N-1> → original dst:DstIdx` - -For keys with N == 1 the edge is kept as-is. - -The result is an expanded edge list with all fan-outs made explicit, and a list of -`FanoutNode` types that need to be instantiated. - -### `StaticNetwork` structure - -```cpp -template owned by the network - typename TopoOrder> // index_sequence encoding start/stop order -class StaticNetwork : public INode { -public: - void start(); // std::apply over TopoOrder — no virtual dispatch, no map lookup - void stop(); // reverse of TopoOrder - - bool running() const; - - // Diagnostics — iterates typed tuples; same NodeSnapshot / ChannelSnapshot output - // as Network, compatible with print_diagnostics and the web debug UI. - void print_diagnostics(std::ostream& = std::cerr) const; - - // StaticNetwork is itself an INode, so it can be embedded in a runtime Network - // via net.add("stage", static_net) exactly like any other node. - void set_name(std::string) override; - const NodeStats& stats() const override; - NodeSnapshot node_snapshot(const std::string&, double) const override; - -private: - FanoutStorage fanouts_; // owns the auto-generated FanoutNode instances - // User nodes held by reference — same non-owning contract as Network -}; -``` - -`FanoutStorage` is a `std::tuple, FanoutNode, ...>` with one -element per auto-inserted fanout. It is owned by the `StaticNetwork` and lives as long as -the network does — which satisfies the channel lifetime contract (channels are owned by their -consumer, and the fanout node is the consumer of the upstream output). - -### Cycle detection - -With the full edge list as a type pack, cycle detection is a `static_assert` rather than a -runtime exception. A compile-time DFS over the expanded edge list fires a readable assertion -at the `make_network` call site: - -``` -static_assert(!has_cycle_v, - "make_network: graph contains a directed cycle"); -``` - -`NetworkCycleError` is no longer needed for `StaticNetwork` — the cycle is caught before any -object is constructed. - -### Topological order - -The same compile-time DFS produces a topological ordering as an `std::index_sequence` over -the node tuple. `start()` iterates it forward, `stop()` iterates it in reverse. No runtime -sort, no `std::vector`. - -### Node labels for diagnostics / web debug - -Labels come directly from the `Label` NTTP on each `Node` type — no separate annotation -on `edge()` is needed. `StaticNetwork` reads `NodeType::label()` at compile time for each -vertex in the topo order and stores the result as a `std::string_view` array at -construction time. Zero runtime overhead: the label is a compile-time string literal. - -```cpp -auto src = make_node(8); -auto blur = make_node(8); -auto detect = make_node(8); - -auto net = make_network( - edge(src.output<0>(), blur.input<0>()), - edge(src.output<0>(), detect.input<0>()) -); -// web UI shows nodes named "src", "blur", "detect" -// auto-inserted FanoutNode is labelled "fanout[src:0]" -``` - -Unlabelled nodes (`Label == ""`) fall back to `"node[]"` in diagnostics. -Auto-inserted fanout nodes are labelled `"fanout[:]"` automatically. - -### Wiring sequence in `make_network` - -All wiring happens in the `make_network` constructor body — no `build()` call needed: - -1. Instantiate `FanoutStorage` (default-construct each `FanoutNode`). -2. For each expanded edge (in topological order): - - Call `src.set_output_channel(&dst.input_channel())`. -3. Return the `StaticNetwork`. - -Channel pointers are set once and never changed. No dynamic allocation after construction. - -### What is eliminated vs `Network` - -| `Network` (runtime) | `StaticNetwork` (compile-time) | +| Example | What it shows | |---|---| -| `std::map` | typed `std::tuple` of references | -| Runtime DFS + `NetworkCycleError` | `static_assert` at `make_network` call site | -| Virtual `start()`/`stop()` per node | `std::apply` over typed tuple | -| Explicit `make_fanout` | auto-inserted from edge pack | -| `connected_outputs_` duplicate check | structural impossibility — no duplicate edge can produce two `set_output_channel` calls | -| `build()` step | no build step — wired in constructor | +| `01_hello_pipeline` | Linear pipeline, index-based wiring, `Network` builder | +| `02_named_ports` | `in<>`/`out<>` tags, named port access, wrong-name `static_assert` | +| `03_multi_output` | Tuple-returning node, per-element sub-port routing | +| `04_storage_policy` | `channel_storage_policy` default + specialisation | +| `05_error_handling` | `ChannelOverflowError`, diagnostics handler | +| `06_watchdog` | Watchdog interval, stall detection | +| `07_python_network` | `PyNetwork` with a pure-Python node *(pending)* | +| `08_python_subport` | `net.read` / `net.write`, sub-port tap *(pending)* | +| `09_opencv_cellshade` | Real-time cell-shading on webcam; named ports, fan-out, `MainThreadNode` display (requires OpenCV) | +| `10_static_hello_pipeline` | `make_network()` version of 01 — compile-time topology | +| `11_static_fanout` | Auto-inserted `FanoutNode` from a duplicated source port | +| `12_static_cellshade` | Static cell-shading with auto fan-out and `Label` NTTPs | +| `13_debug_cellshade` | One-op-per-node pipeline + variadic `DebugCanvas` tiling node | +| `14_debug_hub` | Two networks sharing a `SharedResource` via `DebugHub` | +| `15_node_error_handler` | Per-node `set_error_handler` (skip-and-continue vs stop) | +| `16_event_callbacks` | `set_overflow_callback` + network `set_event_handler` | -The hot path (per-item `pop` → `push` in each node thread) is identical in both cases. +--- -### Compatibility +## Future Extension Points (Heterogeneous Execution) -- `StaticNetwork` implements `INode`, so it can be registered inside a runtime `Network` - via `net.add("name", static_net)` — enabling mixed static/dynamic graphs. -- All existing node types (`Node`, `ObjectNode`, `FanoutNode`, `MainThreadNode`) work - unchanged as vertices in a `StaticNetwork`. -- The Python `PyNetwork` is unaffected — it remains runtime-only. +Not implemented, but the design keeps these doors open: -### File layout addition +- **`IChannel` abstract interface** — `Channel` and a future `RemoteChannel` (socket / + shared-memory) sharing one `push`/`pop` surface so nodes are agnostic to channel location. +- **`Serializer` trait** — parallel to `channel_storage_policy` / `PythonConverter`, for + cross-device serialisation (MessagePack for embedded, pinned memory for GPU zero-copy). +- **`NodeKind` tag** — e.g. `{ Local, Gpu, Remote }` on `INode`, letting the watchdog apply + per-device health-check and timeout strategies. -``` -include/kpn/ - static_network.hpp # Edge<>, make_network(), StaticNetwork<> - tmp/ - fanout_groups.hpp # fanout_groups metafunction - topo_sort.hpp # compile-time DFS + cycle check - repeat_tuple.hpp # repeat_tuple_t (moved from fanout.hpp) -``` - -`fanout.hpp` keeps `FanoutNode` and `make_fanout` for users who want to wire -fanouts explicitly in a runtime `Network`. `static_network.hpp` uses `FanoutNode` internally -but the user never calls `make_fanout` when using `make_network`. +The `IScheduler` abstraction already decouples node execution from any specific thread model, +making a cooperative or device-specific executor a drop-in. --- ## Resolved Design Decisions -All major design questions are now closed: - | Question | Decision | |---|---| -| Shutdown mechanism | `accepting_` flag per channel; `disable()` clears queue and unblocks `pop()` | -| Overflow behaviour | `ChannelOverflowError` thrown on full accepting channel; silently dropped on disabled channel | -| Network ownership | Non-owning; user declares nodes, network holds raw pointers | -| Node lifetime contract | Nodes must outlive their `Network`; declare in same scope | -| Sub-networks | `Network` implements `INode`; `expose_input`/`expose_output` define boundary ports | -| `make_py_network` | Pure C++ template; nanobind module recompilation is the registration step | -| GIL strategy | Acquire per Python callback; release while blocking on channel ops | -| Mixed-rate inputs | `latch<>` tag for ports that reuse last-seen value; blocks only on first fire; node fires at rate of `in<>` ports | -| Fan-out | Explicit `FanoutNode` for runtime `Network`; auto-inserted by `make_network()` for `StaticNetwork` | -| Static vs runtime graph | Both coexist; `StaticNetwork` for C++ graphs known at compile time, `Network` for Python/dynamic graphs; `StaticNetwork` implements `INode` so it embeds in `Network` | -| Node identity in static graphs | `Label` NTTP (human name for diagnostics) + `UniqueTag` NTTP (collision-breaker for same-Func nodes); both default to `""` / `0` so existing code is unaffected | +| Execution model | Reactive: nodes submit `fire_once()` to an `IScheduler` when inputs are ready, not one blocking thread per node | +| `Node<>` vs `PoolNode<>` | `Node<>` owns a private `ThreadPool(1)`; `PoolNode<>` shares a pool for bounded threads | +| Channel | Lock-free SPSC ring buffer, `atomic::wait/notify` + spin-before-sleep | +| Shutdown | Per-channel `accepting_` flag; `disable()` unblocks `pop()` (→ `ChannelClosedError`) | +| Overflow | `ChannelOverflowError` on full accepting channel; silent drop on disabled channel | +| Node error policy | Per-node `NodeErrorHandler` returning bool (skip vs stop) | +| Network ownership | Non-owning; user declares nodes, network stores `INode*` | +| Fan-out | Explicit `FanoutNode` for runtime `Network`; auto-inserted by `make_network()` | +| Branching | `RouterNode` (select one of N) and `FilterNode` (predicate gate) | +| Static vs runtime graph | Both; `StaticNetwork` for compile-time C++ topology, `Network` for dynamic/Python; `StaticNetwork` is an `INode` so it embeds in `Network` | +| Node identity (static graphs) | `Label` NTTP (name) + `UniqueTag` NTTP (collision-breaker); both default | +| Shared device resource | `SharedResource` with priority + aging arbitration | +| Main-thread / GUI work | `MainThreadNode<>` driven by `step()` on the main thread | +| External-event sources | `InterruptNode` with a thread-safe `get_trigger()` | +| Web debugging | Per-network server + multi-network `DebugHub`, behind `KPN_WEB_DEBUG` | +| Mixed-rate latched inputs | **Not implemented** — no `latch<>` ports | From a0c4bf580eb88ae460051bd4712bf427c119156c Mon Sep 17 00:00:00 2001 From: Duncan Tourolle Date: Tue, 14 Jul 2026 19:35:31 +0200 Subject: [PATCH 10/42] fix: deliver EOF sentinel only when the ring is freshly empty MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Channel::pop() surfaced the out-of-band sentinel from its empty branch using the tail_ snapshot taken at the top of the loop. Under contention the producer can push more values *and* the sentinel in the window between that snapshot and take_sentinel(), so pop() could return the sentinel while real values still sat in the ring — the sentinel jumping ahead of values pushed before it. No value was lost (a consumer that keeps draining still receives them, and approx_size() keeps counting them so a PoolNode reschedules), but a consumer treating the sentinel as a hard "last message" barrier would act on EOF early. Re-confirm emptiness against a fresh tail_ load before taking the sentinel. Costs one acquire-load on the empty-ring path only; never runs in steady state. The spin and post-spin takes already reload tail_ on the line above them; try_pop_now() already reads tail_ fresh in the same branch — both were correct and are unchanged. The two sentinel stress cases now assert the strict "sentinel is last, after every value" ordering (previously relaxed to avoid the flake this fixes). Verified TSan-clean (2606 assertions, no data races) over repeated runs. Co-Authored-By: Claude Opus 4.8 --- include/kpn/channel.hpp | 11 ++++- tests/test_channel_stress.cpp | 89 ++++++++++++++++------------------- 2 files changed, 51 insertions(+), 49 deletions(-) diff --git a/include/kpn/channel.hpp b/include/kpn/channel.hpp index 4466fd8..99ebe51 100644 --- a/include/kpn/channel.hpp +++ b/include/kpn/channel.hpp @@ -180,7 +180,16 @@ public: if (h == t) { // Ring drained — deliver any pending out-of-band sentinel (EOF) // now, so it always arrives after the data pushed before it. - { T s; if (take_sentinel(s)) return s; } + // + // Re-confirm emptiness against a fresh tail_ first: the snapshot + // at the top of the loop may be stale (the producer can push more + // values *and* the sentinel in the window since), and the sentinel + // must never jump ahead of ring values pushed before it. The spin + // and post-spin takes below already reload tail_ on the line above + // them; this is the one take that used the loop-top snapshot. + if (h == tail_.load(std::memory_order_acquire)) { + T s; if (take_sentinel(s)) return s; + } if (!accepting_.load(std::memory_order_acquire)) throw ChannelClosedError{}; diff --git a/tests/test_channel_stress.cpp b/tests/test_channel_stress.cpp index bd113d0..eb58704 100644 --- a/tests/test_channel_stress.cpp +++ b/tests/test_channel_stress.cpp @@ -180,28 +180,24 @@ TEST_CASE("SPSC: push_callback fires on each empty->non-empty transition", REQUIRE(callbacks.load() <= N); } -// What the out-of-band sentinel guarantees under contention — and what it does -// not. push_sentinel() publishes has_eof_ (release) after the producer's N ring +// Ordering contract of the out-of-band sentinel under contention. +// +// push_sentinel() publishes has_eof_ (release) after the producer's N ring // pushes; a consumer that observes has_eof_ (acquire) therefore also observes -// every value pushed before it. What these tests assert: +// every value pushed before it. Both pop() and try_pop_now() only surface the +// sentinel once the ring is *freshly* observed empty, so the sentinel is the +// strictly last item received — it never jumps ahead of a ring value pushed +// before it. These tests treat the sentinel as a hard "last message" barrier +// (the consumer stops draining the moment it sees it) and assert that all N +// values arrived, in a contiguous 0..N-1 sequence, before it. // -// * Losslessness — every value 0..N-1 is delivered exactly once (contiguous, -// no gaps, no duplicates) and the sentinel is delivered exactly once. This -// is the invariant that must hold on every run; a broken acquire/release -// pairing would surface as a lost/duplicated value or (under TSan) a data -// race on has_eof_/eof_value_. -// -// What they deliberately do NOT assert is that the sentinel is the *strictly -// last* item popped. pop() checks emptiness (h == t) using a tail_ snapshot -// taken at the top of its loop; the producer can push more values *and* the -// sentinel in the window before take_sentinel() runs, so the consumer may -// surface the sentinel with a few real values still queued behind it. Those -// values are not lost — a consumer that keeps draining still receives them — -// but "sentinel arrives dead last" is not a property the channel promises, so -// asserting it would be flaky. We track how many values trailed the sentinel -// for visibility without failing on it. +// Regression guard: an earlier version of pop() checked emptiness against a +// stale tail_ snapshot from the top of its loop, so under load the sentinel +// could surface with a few real values still queued — breaking in_order / +// values==N here. Under TSan these also cover the has_eof_/eof_value_ +// acquire/release handshake and the spin/futex wakeup on push_sentinel(). -TEST_CASE("SPSC: sentinel and all values survive contention (blocking pop)", +TEST_CASE("SPSC: sentinel is strictly last, after every value (blocking pop)", "[channel][stress]") { constexpr int N = 20'000; constexpr int SENTINEL = -1; @@ -221,34 +217,32 @@ TEST_CASE("SPSC: sentinel and all values survive contention (blocking pop)", ch.push_sentinel(SENTINEL); // must-deliver, never overflows/blocks }); - std::vector seen(N, false); - int values = 0; - int sentinels = 0; - bool duplicate = false; - // Drain until the sentinel AND all N values have been received; the - // sentinel may arrive before the last few values (see note above). - while (values < N || sentinels == 0) { + int expected = 0; + bool in_order = true; + bool saw_sentinel = false; + // Treat the sentinel as EOF: stop draining the instant it appears. + for (;;) { int v = ch.pop(); - if (v == SENTINEL) { ++sentinels; continue; } - if (seen[v]) duplicate = true; else seen[v] = true; - ++values; + if (v == SENTINEL) { saw_sentinel = true; break; } + if (v != expected) in_order = false; + ++expected; } producer.join(); - REQUIRE_FALSE(duplicate); - REQUIRE(values == N); // every value delivered exactly once - REQUIRE(sentinels == 1); // sentinel delivered exactly once + REQUIRE(saw_sentinel); + REQUIRE(in_order); + REQUIRE(expected == N); // all N values received before the sentinel REQUIRE(ch.size() == 0); REQUIRE(ch.approx_size() == 0); } } -TEST_CASE("SPSC: sentinel and all values survive contention (try_pop_now)", +TEST_CASE("SPSC: sentinel is strictly last, after every value (try_pop_now)", "[channel][stress]") { // The pool-node consume path is try_pop_now(), not pop(): it must surface - // the out-of-band sentinel once the ring is observed empty. The consumer - // spins with no sleeps, racing the producer at full tilt across the - // empty-ring boundary where take_sentinel() is reached. + // the out-of-band sentinel only once the ring is freshly observed empty. + // The consumer spins with no sleeps, racing the producer at full tilt + // across the empty-ring boundary where take_sentinel() is reached. constexpr int N = 20'000; constexpr int SENTINEL = -1; @@ -265,23 +259,22 @@ TEST_CASE("SPSC: sentinel and all values survive contention (try_pop_now)", ch.push_sentinel(SENTINEL); }); - std::vector seen(N, false); - int values = 0; - int sentinels = 0; - bool duplicate = false; + int expected = 0; + bool in_order = true; + bool saw_sentinel = false; int v; - while (values < N || sentinels == 0) { + for (;;) { if (!ch.try_pop_now(v)) { std::this_thread::yield(); continue; } - if (v == SENTINEL) { ++sentinels; continue; } - if (seen[v]) duplicate = true; else seen[v] = true; - ++values; + if (v == SENTINEL) { saw_sentinel = true; break; } + if (v != expected) in_order = false; + ++expected; } producer.join(); - REQUIRE_FALSE(duplicate); - REQUIRE(values == N); - REQUIRE(sentinels == 1); - // Sentinel held no ring slot; once drained the channel is fully empty. + REQUIRE(saw_sentinel); + REQUIRE(in_order); + REQUIRE(expected == N); + // Sentinel held no ring slot; once taken the channel is fully empty. REQUIRE(ch.size() == 0); REQUIRE(ch.approx_size() == 0); } From ec19137ed96fc2d3c922575da4ac1bc26b5f90b8 Mon Sep 17 00:00:00 2001 From: Duncan Tourolle Date: Fri, 17 Jul 2026 20:01:57 +0200 Subject: [PATCH 11/42] ci: fix TSan aborting at init on the nested-LXC runner The ThreadSanitizer job runs on Docker nested in an unprivileged LXC container, whose kernel randomizes mmap addresses beyond the range TSan's fixed shadow mapping expects. TSan aborted at init with "unexpected memory mapping" before any test ran. Disable ASLR per-process with `setarch -R`, which needs the personality(2) syscall that Docker's default seccomp profile blocks; seccomp=unconfined on the container permits it. Verified on the runner that both are required: setarch -R alone gets EPERM, seccomp alone still aborts, both together run clean. Scoped to the tsan job, which runs only our own test binaries. Co-Authored-By: Claude Opus 4.8 --- .gitea/workflows/tsan.yaml | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/.gitea/workflows/tsan.yaml b/.gitea/workflows/tsan.yaml index 53f81ad..7911d7d 100644 --- a/.gitea/workflows/tsan.yaml +++ b/.gitea/workflows/tsan.yaml @@ -18,6 +18,15 @@ jobs: runs-on: linux/amd64 container: image: gitea.tourolle.paris/dtourolle/kpnpp-builder:latest + # This runner is Docker nested in an unprivileged LXC container, whose + # kernel randomizes mmap addresses beyond the range TSan's fixed shadow + # mapping expects, so TSan aborts at init with "unexpected memory + # mapping". The fix is to disable ASLR per-process with `setarch -R` + # (below), which needs the personality(2) syscall that Docker's default + # seccomp profile blocks. seccomp=unconfined permits it. Verified on the + # runner: setarch -R alone gets EPERM, seccomp alone still aborts, both + # together run clean. Scoped to this job, which runs only our own tests. + options: --security-opt seccomp=unconfined steps: - name: Checkout repository uses: actions/checkout@v4 @@ -54,13 +63,14 @@ jobs: # the full picture for lock-order issues. env: TSAN_OPTIONS: "halt_on_error=1 second_deadlock_stack=1" - run: ./build/tests/kpn_tests_stress + # setarch -R disables ASLR for this process; see the container comment. + run: setarch -R ./build/tests/kpn_tests_stress - name: Run unit tests under TSan working-directory: tsan-${{ github.run_id }} env: TSAN_OPTIONS: "halt_on_error=1 second_deadlock_stack=1" - run: ./build/tests/kpn_tests + run: setarch -R ./build/tests/kpn_tests - name: Cleanup if: always() From 4b6e498ba7e70a34cc0b57638f9e56a43b7f41ae Mon Sep 17 00:00:00 2001 From: Duncan Tourolle Date: Sun, 19 Jul 2026 16:55:36 +0200 Subject: [PATCH 12/42] =?UTF-8?q?feat:=20persistent-pipeline=20reuse=20?= =?UTF-8?q?=E2=80=94=20push=5Fblocking,=20node=20introspection,=20stateful?= =?UTF-8?q?=20wrapper?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds three pieces needed to build one KPN network and reuse it across many replays/configs instead of tearing down and rebuilding per run: - Channel::push_blocking (+ IVariantChannel/VariantChannel forwarding): lossless backpressure push that waits for space instead of dropping when the ring is full. PyNode's run_loop now uses it so a downstream consumer lagging behind never silently drops a frame. - PyNetwork::node_ptr / node_stats: raw node handle by name (for a binding to dynamic_cast to a concrete wrapper and call functor-specific runtime setters) and a per-node timing snapshot for profiling. - ObjectVariantNodeWrapper: variant-node adapter for functors that need runtime-constructed state (a Config, a loaded gallery), mirroring VariantNodeWrapper's channel plumbing but backed by ObjectNode. Built and used downstream in scene-actor-extraction's sae_kpn Python replay bindings for repeated threshold-sweep evaluation of the same pipeline. --- include/kpn/channel.hpp | 29 ++++ include/kpn/python/bindings.hpp | 25 +++- include/kpn/python/object_variant_node.hpp | 149 +++++++++++++++++++++ include/kpn/variant_node.hpp | 5 + 4 files changed, 205 insertions(+), 3 deletions(-) create mode 100644 include/kpn/python/object_variant_node.hpp diff --git a/include/kpn/channel.hpp b/include/kpn/channel.hpp index 99ebe51..f8842d4 100644 --- a/include/kpn/channel.hpp +++ b/include/kpn/channel.hpp @@ -136,6 +136,35 @@ public: push_callback_(); } + // Lossless push with BACKPRESSURE: if the ring is full, wait for the consumer to + // drain instead of dropping (the throwing push()) — the producer just runs slower. + // Use when every value must be delivered (e.g. replaying a dump for scoring, where + // a dropped frame silently corrupts the result). SPSC: only the sole producer may + // call it. Returns false if the channel was disabled while waiting. + bool push_blocking(T value) { + for (;;) { + if (!accepting_.load(std::memory_order_acquire)) { + stats_.record_drop(); + return false; + } + const std::size_t t = tail_.load(std::memory_order_relaxed); + const std::size_t h = head_.load(std::memory_order_acquire); + if (t - h < capacity_) { // space available → normal push + const std::size_t data_bytes = ChannelDataSize::bytes(value); + const bool was_empty = (t == h); + buf_[t & ring_mask_] = make_storage(std::move(value)); + tail_.store(t + 1, std::memory_order_release); + stats_.record_push(t - h + 1, data_bytes); + wake_.fetch_add(1, std::memory_order_release); + wake_.notify_one(); + if (was_empty && push_callback_) push_callback_(); + return true; + } + // full: yield briefly and retry (consumer will drain) + std::this_thread::sleep_for(std::chrono::microseconds(50)); + } + } + // Lossless, non-blocking delivery for a must-deliver control token (EOF). // // A sentinel is stored out-of-band — in a dedicated slot that does NOT diff --git a/include/kpn/python/bindings.hpp b/include/kpn/python/bindings.hpp index 0a31bfe..2491bf8 100644 --- a/include/kpn/python/bindings.hpp +++ b/include/kpn/python/bindings.hpp @@ -217,6 +217,20 @@ public: return it->second; } + // Raw node handle by name — lets a binding dynamic_cast to a concrete wrapper + // type and call its functor's runtime setters (persistent-pipeline reuse). + VNode* node_ptr(const std::string& name) { return &node_at(name); } + + // Per-node timing snapshot for profiling where a replay spends its time. + std::map node_stats(const std::string& name) { + auto& n = node_at(name); + NodeSnapshot s = n.node_snapshot(name, 0.0); + return {{"frames", double(s.frames_processed)}, + {"exec_ms", s.ema_exec_ms}, {"max_ms", s.max_exec_ms}, + {"blocked_ms", s.total_blocked_ms}, {"fps", s.throughput_fps}, + {"cpu_ms", s.total_cpu_ms}, {"cpu_util_pct", s.cpu_util_pct}}; + } + private: VNode& node_at(const std::string& name) { auto it = nodes_.find(name); @@ -422,13 +436,17 @@ private: for (std::size_t i = 0; i < out_channels_.size(); ++i) { if (out_channels_[i]) - out_channels_[i]->push(std::move(outputs[i])); + // Lossless: wait for space rather than drop. A dropped frame + // silently corrupts a replay's score; backpressure just slows + // the producer. (Was push() + "drop on overflow".) + out_channels_[i]->push_blocking(std::move(outputs[i])); } } catch (const ChannelClosedError&) { break; } catch (const ChannelOverflowError&) { - // drop and continue + // no longer reachable with push_blocking, kept for safety + break; } } } @@ -530,7 +548,8 @@ void register_py_network(nb::module_& m, const char* class_name = "Network") { .def("read", &Net::read, nb::arg("node"), nb::arg("out_idx") = std::size_t(0)) .def("write", &Net::write, - nb::arg("node"), nb::arg("in_idx"), nb::arg("value")); + nb::arg("node"), nb::arg("in_idx"), nb::arg("value")) + .def("node_stats", &Net::node_stats, nb::arg("node")); } } // namespace kpn::python diff --git a/include/kpn/python/object_variant_node.hpp b/include/kpn/python/object_variant_node.hpp new file mode 100644 index 0000000..82ff149 --- /dev/null +++ b/include/kpn/python/object_variant_node.hpp @@ -0,0 +1,149 @@ +#pragma once +// ObjectVariantNodeWrapper — variant-node adapter for *stateful* functors. +// +// VariantNodeWrapper (variant_node.hpp) wraps Node, where Func is a +// default-constructible NTTP callable. That doesn't fit nodes whose functor must +// be constructed with runtime state (a Config, a loaded gallery, etc.) — those use +// ObjectNode, which takes `Obj& obj` at construction. +// +// This wrapper owns an Obj instance and exposes the same IVariantNode surface so a +// stateful C++ node can live inside a PyNetwork. Build one via a factory that +// constructs the functor from Python-supplied config, e.g.: +// +// auto n = std::make_shared, out<"matched">>>( +// fifo_cap, gallery, cfg); // Obj ctor args forwarded +// net.add("identity_matcher", n); +// +// The wrapper mirrors VariantNodeWrapper's channel plumbing exactly; only the +// underlying node type (PoolObjectNode, holding Obj&) differs. + +#include "../channel.hpp" +#include "../node.hpp" +#include "../variant_node.hpp" + +#include +#include +#include +#include +#include +#include +#include + +namespace kpn { + +template, + typename OutputTag = out<>> +class ObjectVariantNodeWrapper; + +template +class ObjectVariantNodeWrapper, out> + : public IVariantNode +{ + using NodeT = ObjectNode, out>; + +public: + using args_tuple = typename NodeT::args_tuple; + using return_tuple = typename NodeT::return_tuple; + + static constexpr std::size_t n_in = NodeT::input_count; + static constexpr std::size_t n_out = NodeT::output_count; + + // Owns the functor; forwards remaining args to Obj's constructor. + template + explicit ObjectVariantNodeWrapper(std::size_t fifo_capacity, ObjArgs&&... obj_args) + : obj_(std::forward(obj_args)...) + , node_(obj_, fifo_capacity) + , in_channels_(n_in) + , out_channels_(n_out) + , out_type_indices_(n_out, std::type_index(typeid(void))) + { + init_inputs(std::make_index_sequence{}, fifo_capacity); + init_out_types(std::make_index_sequence{}); + } + + // Access the owned functor so callers can invoke its runtime setters (e.g. to + // change a threshold on a persistent pipeline without rebuilding the node). + Obj& functor() { return obj_; } + + // ── INode ───────────────────────────────────────────────────────────────── + void start() override { node_.start(); } + void stop() override { node_.stop(); } + bool running() const override { return node_.running(); } + const NodeStats& stats() const override { return node_.stats(); } + void set_name(std::string name) override { node_.set_name(std::move(name)); } + NodeSnapshot node_snapshot(const std::string& name, double elapsed_s) const override { + return node_.node_snapshot(name, elapsed_s); + } + + // ── IVariantNode ────────────────────────────────────────────────────────── + std::size_t input_count() const override { return n_in; } + std::size_t output_count() const override { return n_out; } + + std::type_index input_type(std::size_t i) const override { + return in_channels_[i]->type_index(); + } + std::type_index output_type(std::size_t i) const override { + return out_type_indices_[i]; + } + + std::shared_ptr> input_channel(std::size_t i) override { + return in_channels_[i]; + } + + void set_output_channel(std::size_t i, + std::shared_ptr> ch) override { + set_output_impl(i, std::move(ch), std::make_index_sequence{}); + } + +private: + template + void init_inputs(std::index_sequence, std::size_t cap) { + ((init_one_input(cap)), ...); + } + + template + void init_one_input(std::size_t cap) { + using T = std::tuple_element_t; + auto shared_ch = std::make_shared>(cap); + node_.template set_input_channel(shared_ch); + in_channels_[I] = std::make_shared>(std::move(shared_ch)); + } + + template + void init_out_types(std::index_sequence) { + ((out_type_indices_[Is] = + std::type_index(typeid(std::tuple_element_t))), ...); + } + + template + void set_output_impl(std::size_t port, + std::shared_ptr> ch, + std::index_sequence) { + bool matched = false; + ((Is == port && (set_output_at(std::move(ch)), matched = true)), ...); + if (!matched) + throw std::out_of_range("set_output_channel: port index out of range"); + } + + template + void set_output_at(std::shared_ptr> ch) { + using T = std::tuple_element_t; + auto* typed = dynamic_cast*>(ch.get()); + if (!typed) + throw std::runtime_error( + "set_output_channel: type mismatch at output port " + std::to_string(I)); + node_.template set_output_channel(typed->raw_ptr()); + out_channels_[I] = std::move(ch); + } + + Obj obj_; // owned; node_ holds Obj& — declaration order keeps obj_ alive first + NodeT node_; + std::vector>> in_channels_; + std::vector>> out_channels_; + std::vector out_type_indices_; +}; + +} // namespace kpn diff --git a/include/kpn/variant_node.hpp b/include/kpn/variant_node.hpp index 0af32e7..9a98599 100644 --- a/include/kpn/variant_node.hpp +++ b/include/kpn/variant_node.hpp @@ -55,6 +55,8 @@ class IVariantChannel { public: virtual ~IVariantChannel() = default; virtual void push(Variant v) = 0; + // Lossless push with backpressure (waits instead of dropping when full). + virtual void push_blocking(Variant v) = 0; virtual Variant pop() = 0; virtual std::type_index type_index() const = 0; virtual std::string type_name() const = 0; @@ -76,6 +78,9 @@ public: void push(Variant v) override { channel_->push(std::get(std::move(v))); } + void push_blocking(Variant v) override { + channel_->push_blocking(std::get(std::move(v))); + } Variant pop() override { return Variant{ channel_->pop() }; } From 6595e6e925785b8da529ce0a34208738b639c670 Mon Sep 17 00:00:00 2001 From: Duncan Tourolle Date: Fri, 31 Jul 2026 10:35:00 +0200 Subject: [PATCH 13/42] fix: node outputs block instead of dropping on a full channel MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Every node output used the throwing push(), so a consumer falling behind cost values rather than time. push_blocking() already existed on Channel and OutputPort — "wait for the consumer to drain instead of dropping; the producer just runs slower" — but nothing called it. A dropped frame does not degrade a downstream result, it silently changes one, and the consumer has no way to tell it happened. For any pipeline whose output is a claim about its input, that is corruption rather than degradation. Safe because sentinels are already handled out-of-band, above this path: only data blocks, so the EOF token that unwinds the network can always overtake a stalled data path. That is exactly the hold-and-wait deadlock the push_sentinel comment warns about, and the reason it is not reachable here. Measured on a downstream consumer (face pipeline, 77s clip at 5 fps, expected 385 sampled frames): before 65 frames written, 320 dropped at one node, 29s after 385 frames written, 0 dropped, 17s Faster, not slower — a dropped frame has already cost its decode, and the overflow exception cost more. Two consecutive runs now produce byte-identical output, which they did not before: what got dropped depended on timing, so the same command could yield different results. Co-Authored-By: Claude Opus 5 --- include/kpn/pool_node.hpp | 23 +++++++++++------------ 1 file changed, 11 insertions(+), 12 deletions(-) diff --git a/include/kpn/pool_node.hpp b/include/kpn/pool_node.hpp index 4fe37f2..1938302 100644 --- a/include/kpn/pool_node.hpp +++ b/include/kpn/pool_node.hpp @@ -400,12 +400,15 @@ private: ch->push_sentinel(std::move(val)); return; } - try { - ch->push(std::move(val)); - } catch (const ChannelOverflowError&) { - throw ChannelOverflowError(ch->capacity(), - "pool node '" + name_ + "' " + output_port_label()); - } + // Backpressure, not loss. A full downstream channel means the consumer + // is behind, and the correct response is for this producer to run + // slower — not to discard a value. A dropped frame does not degrade a + // result, it silently changes one, and the caller has no way to tell. + // + // Safe here because sentinels are handled above, out-of-band: this + // blocks only on data, so the EOF token that unwinds the network can + // always overtake a stalled data path. + ch->push_blocking(std::move(val)); } template @@ -706,12 +709,8 @@ private: ch->push_sentinel(std::move(val)); return; } - try { - ch->push(std::move(val)); - } catch (const ChannelOverflowError&) { - throw ChannelOverflowError(ch->capacity(), - "pool node '" + name_ + "'"); - } + // See the note on the typed overload above: block rather than drop. + ch->push_blocking(std::move(val)); } Obj& obj_; From 28e06675f5f439b77b9cb79033ad98fa423af3e5 Mon Sep 17 00:00:00 2001 From: Duncan Tourolle Date: Fri, 31 Jul 2026 22:39:51 +0200 Subject: [PATCH 14/42] fix: park nodes on a full output instead of blocking the worker MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit push_blocking parked a scheduler worker inside the push. Nodes own a private single-thread pool, so the parked thread was the only one that could drain that node's own input — hold-and-wait, and under sustained backpressure four nodes of a five-node chain slept in nanosleep at once. channel.hpp already warned about this for sentinels; it applies just as much to data pushes. The scheduler was purely input-driven: on_input_ready() wakes a node when input arrives, with no counterpart for "my output has room". Lacking that signal, blocking the thread was the only way to handle a full output. This adds the missing half. - Channel::try_push + has_space + set_space_callback; the callback fires from both pop() and try_pop_now(). - PoolNode/PoolObjectNode keep a one-slot pending_ buffer with per-element done flags, so a retry cannot duplicate an already-accepted element. One slot suffices because queued_ admits at most one fire_once per node. - The re-check after clearing queued_ closes the lost-wakeup race where a space callback fires while the flag is still up and is swallowed. Two bugs surfaced once nodes actually parked, both fixed here: - pop_one reports an *empty* channel as ChannelClosedError, which is also the node's "upstream finished, self-stop" signal. A node woken by output space with empty inputs therefore killed itself. fire_once now releases the worker when its inputs are not ready rather than falling through. - The drained-park path resubmitted unconditionally instead of via on_input_ready(), firing nodes with nothing to read. compute_priority is now output-aware: mean output fill is deducted from mean input fill, mapped as 0.5·(1 + in - out). Input fill alone asks only "how much work is waiting for me"; a node whose outputs are already full cannot deliver, so running it just parks it again and wastes the slot while the node that would drain that channel waits behind it. The scheduler now favours whoever is furthest downstream of a bottleneck. Also adds a network-level error listener. A node's exception was discarded at the node boundary and survived only as a Closed event, which reports that a node stopped but not why — that missing detail is what made the above slow to diagnose. INode::set_network_error_callback plus StaticNetwork::set_error_handler forward it to the application. Tests: 121/121. test_backpressure_deadlock drives a five-node chain with capacity-2 channels against a slow sink and fails on the old code. The four test_pool_node overflow tests now assert parking rather than the removed drop-and-report behaviour. Known-incomplete: a rare hang remains, roughly 1 run in 20 against a 300s timeout, down from every run failing. Committed because the fix is a large strict improvement and the residual case needs its own reproduction. --- include/kpn/channel.hpp | 45 +++- include/kpn/inode.hpp | 8 + include/kpn/pool_node.hpp | 327 ++++++++++++++++++++++++--- include/kpn/static_network.hpp | 13 ++ tests/CMakeLists.txt | 1 + tests/test_backpressure_deadlock.cpp | 85 +++++++ tests/test_pool_node.cpp | 30 ++- 7 files changed, 475 insertions(+), 34 deletions(-) create mode 100644 tests/test_backpressure_deadlock.cpp diff --git a/include/kpn/channel.hpp b/include/kpn/channel.hpp index f8842d4..571b891 100644 --- a/include/kpn/channel.hpp +++ b/include/kpn/channel.hpp @@ -136,6 +136,42 @@ public: push_callback_(); } + /// Called when a pop frees a slot in a previously-full ring. + /// + /// The mirror of `set_push_callback`, and it exists for the same reason: + /// a producer must be able to *park* rather than spin. Without it the only + /// lossless option is `push_blocking`, which sleeps inside the caller's + /// thread — and when that thread is a scheduler worker, parking it starves + /// every node pinned to it (see the hold-and-wait note on push_sentinel). + void set_space_callback(std::function cb) { space_callback_ = std::move(cb); } + + /// True when a push would currently succeed. Used to close the lost-wakeup + /// race: a producer that parks must re-check after clearing its queued flag, + /// because a space_callback fired in between would otherwise be swallowed. + bool has_space() const { + return tail_.load(std::memory_order_relaxed) - + head_.load(std::memory_order_acquire) < capacity_; + } + + /// Non-blocking, lossless push. Returns false when the ring is full, having + /// changed nothing — the caller keeps the value and retries when woken. + bool try_push(T& value) { + if (!accepting_.load(std::memory_order_acquire)) { stats_.record_drop(); return true; } + const std::size_t t = tail_.load(std::memory_order_relaxed); + const std::size_t h = head_.load(std::memory_order_acquire); + if (t - h >= capacity_) return false; + + const std::size_t data_bytes = ChannelDataSize::bytes(value); + const bool was_empty = (t == h); + buf_[t & ring_mask_] = make_storage(std::move(value)); + tail_.store(t + 1, std::memory_order_release); + stats_.record_push(t - h + 1, data_bytes); + wake_.fetch_add(1, std::memory_order_release); + wake_.notify_one(); + if (was_empty && push_callback_) push_callback_(); + return true; + } + // Lossless push with BACKPRESSURE: if the ring is full, wait for the consumer to // drain instead of dropping (the throwing push()) — the producer just runs slower. // Use when every value must be delivered (e.g. replaying a dump for scoring, where @@ -248,6 +284,8 @@ public: throw ChannelClosedError{}; T value = extract(std::move(buf_[h & ring_mask_])); head_.store(h + 1, std::memory_order_release); + // A slot just freed: wake any producer parked on this channel. + if (t - h >= capacity_ && space_callback_) space_callback_(); stats_.record_pop(); return value; } @@ -269,11 +307,15 @@ public: // so pool nodes — which pop only via this path — still receive the token. bool try_pop_now(T& out) { const std::size_t h = head_.load(std::memory_order_relaxed); - if (h == tail_.load(std::memory_order_acquire)) + const std::size_t t = tail_.load(std::memory_order_acquire); + if (h == t) return take_sentinel(out); out = extract(std::move(buf_[h & ring_mask_])); head_.store(h + 1, std::memory_order_release); stats_.record_pop(); + // Pool nodes pop only through here, so this is where a parked producer + // gets woken: the ring was full, and it no longer is. + if (t - h >= capacity_ && space_callback_) space_callback_(); return true; } @@ -363,6 +405,7 @@ private: std::size_t ring_mask_; std::unique_ptr buf_; std::function push_callback_; + std::function space_callback_; ChannelStats stats_; // Out-of-band sentinel (EOF): stored outside the ring so its delivery never diff --git a/include/kpn/inode.hpp b/include/kpn/inode.hpp index 56f4b89..a7454e2 100644 --- a/include/kpn/inode.hpp +++ b/include/kpn/inode.hpp @@ -34,6 +34,14 @@ struct INode { virtual void set_network_overflow_callback(NodeEventCallback) {} virtual void set_network_closed_callback(NodeEventCallback) {} + // Network-level error listener. Consulted when a node's function throws + // and no per-node handler resolved it. Without this the exception is + // discarded and the failure is only visible as a Closed event, which says + // a node stopped but not why — the difference between a diagnosis and a + // guess. Same contract as NodeErrorHandler: true to continue, false to + // stop the node. + virtual void set_network_error_callback(NodeErrorHandler) {} + // halt(): alias for stop() — immediate, discards in-flight work. virtual void halt() { stop(); } diff --git a/include/kpn/pool_node.hpp b/include/kpn/pool_node.hpp index 1938302..85532da 100644 --- a/include/kpn/pool_node.hpp +++ b/include/kpn/pool_node.hpp @@ -15,6 +15,7 @@ #include #include #include +#include #include #include #include @@ -131,6 +132,7 @@ public: void set_name(std::string name) override { name_ = std::move(name); } void set_error_handler(NodeErrorHandler h) { error_handler_ = std::move(h); } + void set_network_error_callback(NodeErrorHandler h) override { net_error_handler_ = std::move(h); } void set_max_exec_time(std::chrono::milliseconds t) { max_exec_time_ = t; } void set_overflow_callback(NodeEventCallback cb) { event_callbacks_[0] = std::move(cb); } @@ -234,6 +236,8 @@ private: template void register_callbacks(std::index_sequence) { + // A parked producer is re-submitted when its output drains. + register_space_callbacks(std::make_index_sequence{}); (std::get(input_channels_)->set_push_callback( [this] { on_input_ready(); }), ...); } @@ -243,6 +247,20 @@ private: for (auto& cb : cbs) if (cb) cb(ts); } + template + bool outputs_have_space(std::index_sequence) const { + return (... && (!std::get(output_channels_) || + std::get(output_channels_)->has_space())); + } + + template + void register_space_callbacks(std::index_sequence) { + ((std::get(output_channels_) + ? (void)std::get(output_channels_)->set_space_callback( + [this] { try_submit(0.5f); }) + : (void)0), ...); + } + void self_stop() { disable_inputs(std::make_index_sequence{}); disable_outputs(std::make_index_sequence{}); @@ -280,11 +298,51 @@ private: return ((std::get(input_channels_)->approx_size() > 0 ? 1u : 0u) + ...); } + /// Priority in [0,1], higher runs sooner. + /// + /// Input fill alone answers "how much work is waiting for me". That is + /// only half the question: a node whose *outputs* are already full cannot + /// deliver anything: running it produces a value with nowhere to go, so it + /// immediately parks and the slot is wasted. Meanwhile the node that would + /// have drained that full channel waits behind it. + /// + /// So occupancy of the outputs is deducted from occupancy of the inputs. + /// The scheduler then naturally favours whoever is furthest downstream of + /// a bottleneck — the node whose inputs are backed up but whose outputs + /// have room is exactly the one whose execution frees the most capacity — + /// and defers producers that would only deepen a queue that is already + /// full. + /// + /// Mapped as 0.5·(1 + in - out) rather than clamping (in - out) at zero: + /// both terms are mean fills in [0,1], so the difference is in [-1,1], and + /// the affine map keeps the whole range distinguishable instead of + /// collapsing every output-saturated node onto the same value. 0.5 remains + /// the neutral point, matching the default used for source nodes. float compute_priority() { if constexpr (input_count == 0) return 0.5f; - float sum = 0.0f; - sum_fill(sum, std::make_index_sequence{}); - return sum / static_cast(input_count); + float in = 0.0f; + sum_fill(in, std::make_index_sequence{}); + in /= static_cast(input_count); + + if constexpr (output_count == 0) return in; + + float out = 0.0f; + sum_output_fill(out, std::make_index_sequence{}); + out /= static_cast(output_count); + + const float p = 0.5f * (1.0f + in - out); + return p < 0.0f ? 0.0f : (p > 1.0f ? 1.0f : p); + } + + /// Mean fill of the output channels, same normalisation as sum_fill. + /// An unconnected output holds nothing back, so it contributes 0. + template + void sum_output_fill(float& sum, std::index_sequence) { + ((sum += (std::get(output_channels_) && + std::get(output_channels_)->capacity() > 0) + ? float(std::get(output_channels_)->approx_size()) + / float(std::get(output_channels_)->capacity()) + : 0.0f), ...); } template @@ -315,6 +373,77 @@ private: t0.time_since_epoch()).count(); stats_.exec_start_us.store(now_us, std::memory_order_relaxed); + // Parked from a previous firing: retry that value before touching the + // inputs. Returning here releases the worker — the channel's space + // callback re-submits this node when the consumer drains a slot. + if constexpr (!std::is_void_v) { + if (pending_) { + push_outputs(std::move(*pending_), std::make_index_sequence{}); + queued_.store(false, std::memory_order_release); + if (pending_) { + // Close the lost-wakeup race: a space_callback that fired + // between the failed push and clearing queued_ was + // swallowed, and nothing else will wake this node. Re-check + // now that the flag is down. + if (outputs_have_space(std::make_index_sequence{})) + try_submit(0.5f); + return; // parked + } + // Drained: resume normal firing, resubmitting exactly the way + // the normal tail below does. An unconditional try_submit here + // would fire a node whose inputs are empty, and pop_inputs + // reports an empty channel as ChannelClosedError — which this + // node treats as "upstream finished" and self-stops on. That + // is a live node killing itself purely because it was woken by + // *output* space rather than by input arrival. + if constexpr (input_count == 0) try_submit(0.5f); + else on_input_ready(); + return; + } + } + + // Parked from a previous firing: retry that value before touching the + // inputs. Returning here releases the worker — the channel's space + // callback re-submits this node once the consumer drains a slot. + if constexpr (!std::is_void_v) { + if (pending_) { + push_outputs(std::move(*pending_), std::make_index_sequence{}); + queued_.store(false, std::memory_order_release); + if (pending_) { + // Close the lost-wakeup race: a space_callback that fired + // between the failed push and clearing queued_ was + // swallowed, and nothing else will wake this node. Re-check + // now that the flag is down. + if (outputs_have_space(std::make_index_sequence{})) + try_submit(0.5f); + return; // parked + } + // Drained: resume normal firing, resubmitting exactly the way + // the normal tail below does. An unconditional try_submit here + // would fire a node whose inputs are empty, and pop_inputs + // reports an empty channel as ChannelClosedError — which this + // node treats as "upstream finished" and self-stops on. That + // is a live node killing itself purely because it was woken by + // *output* space rather than by input arrival. + if constexpr (input_count == 0) try_submit(0.5f); + else on_input_ready(); + return; + } + } + + // Woken by output space rather than by input arrival, with nothing + // parked left to flush: there is no work to do. Falling through would + // read an empty channel, and pop_one reports empty as + // ChannelClosedError — self-stopping a live node. Release the worker; + // on_input_ready() resubmits when data actually lands. + if constexpr (input_count > 0) { + if (count_ready(std::make_index_sequence{}) != input_count) { + queued_.store(false, std::memory_order_release); + on_input_ready(); // data may have arrived while we checked + return; + } + } + try { auto args = pop_inputs(std::make_index_sequence{}); auto t1 = clock_t::now(); @@ -340,7 +469,11 @@ private: } catch (const ChannelOverflowError&) { fire_callbacks(event_callbacks_); } catch (...) { - if (error_handler_ && error_handler_(name_, std::current_exception())) { + auto eptr = std::current_exception(); + const bool handled = + (error_handler_ && error_handler_(name_, eptr)) || + (net_error_handler_ && net_error_handler_(name_, eptr)); + if (handled) { // continue — fall through to resubmit check } else { fire_callbacks(closed_callbacks_); @@ -385,30 +518,41 @@ private: } template + /// Pushes what it can and parks the rest. `done_` marks the elements that + /// were taken, so a retry never re-pushes one — a duplicate would be as + /// wrong as a drop, just harder to notice. void push_outputs(return_tuple&& result, std::index_sequence) { - (push_one_out(std::get(std::move(result))), ...); + bool all = true; + ((pending_done_[Is] = pending_done_[Is] || + push_one_out(std::get(std::move(result))), + all = all && pending_done_[Is]), ...); + if (all) { pending_.reset(); pending_done_.fill(false); } + else pending_ = std::move(result); } + /// Returns false when the ring was full and the value was NOT taken; the + /// caller must keep it and retry after the channel signals space. template - void push_one_out(std::tuple_element_t&& val) { + bool push_one_out(std::tuple_element_t&& val) { auto* ch = std::get(output_channels_); - if (!ch) return; + if (!ch) return true; // Sentinels (EOF) must never be dropped: a lost token wedges every // downstream pop() forever. Deliver them out-of-band (push_sentinel), // which never overflows and never blocks this node's worker thread. if (is_sentinel_value(val)) { ch->push_sentinel(std::move(val)); - return; + return true; } - // Backpressure, not loss. A full downstream channel means the consumer - // is behind, and the correct response is for this producer to run - // slower — not to discard a value. A dropped frame does not degrade a - // result, it silently changes one, and the caller has no way to tell. + // Backpressure without parking the worker. A full channel means the + // consumer is behind; the value is kept and this node stops running + // until the channel signals space (set_space_callback re-submits it). // - // Safe here because sentinels are handled above, out-of-band: this - // blocks only on data, so the EOF token that unwinds the network can - // always overtake a stalled data path. - ch->push_blocking(std::move(val)); + // Blocking here instead would sleep inside a scheduler worker, and + // nodes are pinned to workers — park enough of them and nothing is left + // to run the consumer that would drain the channel. That is the + // hold-and-wait deadlock channel.hpp warns about for sentinels; it + // applies to data pushes just as much. + return ch->try_push(val); } template @@ -430,8 +574,18 @@ private: output_channels_t output_channels_{}; std::atomic stop_flag_{true}; std::atomic queued_{false}; + + /// The hidden one-slot output buffer (see push_outputs). Holding the value + /// here is what lets a node stop running without dropping it or occupying a + /// scheduler worker. One slot suffices because at most one fire_once() runs + /// per node at a time — with concurrent firing it would have to hold a whole + /// FIFO's worth. + std::conditional_t, std::monostate, + std::optional> pending_{}; + std::array pending_done_{}; NodeStats stats_; NodeErrorHandler error_handler_; + NodeErrorHandler net_error_handler_; std::chrono::milliseconds max_exec_time_{0}; std::array event_callbacks_{}; // [0]=user [1]=network std::array closed_callbacks_{}; @@ -499,6 +653,7 @@ public: bool running() const override { return !stop_flag_.load(std::memory_order_relaxed); } void set_name(std::string name) override { name_ = std::move(name); } void set_error_handler(NodeErrorHandler h) { error_handler_ = std::move(h); } + void set_network_error_callback(NodeErrorHandler h) override { net_error_handler_ = std::move(h); } void set_max_exec_time(std::chrono::milliseconds t) { max_exec_time_ = t; } void set_overflow_callback(NodeEventCallback cb) { event_callbacks_[0] = std::move(cb); } @@ -570,6 +725,8 @@ private: } template void register_callbacks(std::index_sequence) { + // A parked producer is re-submitted when its output drains. + register_space_callbacks(std::make_index_sequence{}); (std::get(input_channels_)->set_push_callback([this] { on_input_ready(); }), ...); } @@ -578,6 +735,20 @@ private: for (auto& cb : cbs) if (cb) cb(ts); } + template + bool outputs_have_space(std::index_sequence) const { + return (... && (!std::get(output_channels_) || + std::get(output_channels_)->has_space())); + } + + template + void register_space_callbacks(std::index_sequence) { + ((std::get(output_channels_) + ? (void)std::get(output_channels_)->set_space_callback( + [this] { try_submit(0.5f); }) + : (void)0), ...); + } + void self_stop() { disable_inputs(std::make_index_sequence{}); disable_outputs(std::make_index_sequence{}); @@ -609,11 +780,51 @@ private: return ((std::get(input_channels_)->approx_size() > 0 ? 1u : 0u) + ...); } + /// Priority in [0,1], higher runs sooner. + /// + /// Input fill alone answers "how much work is waiting for me". That is + /// only half the question: a node whose *outputs* are already full cannot + /// deliver anything: running it produces a value with nowhere to go, so it + /// immediately parks and the slot is wasted. Meanwhile the node that would + /// have drained that full channel waits behind it. + /// + /// So occupancy of the outputs is deducted from occupancy of the inputs. + /// The scheduler then naturally favours whoever is furthest downstream of + /// a bottleneck — the node whose inputs are backed up but whose outputs + /// have room is exactly the one whose execution frees the most capacity — + /// and defers producers that would only deepen a queue that is already + /// full. + /// + /// Mapped as 0.5·(1 + in - out) rather than clamping (in - out) at zero: + /// both terms are mean fills in [0,1], so the difference is in [-1,1], and + /// the affine map keeps the whole range distinguishable instead of + /// collapsing every output-saturated node onto the same value. 0.5 remains + /// the neutral point, matching the default used for source nodes. float compute_priority() { if constexpr (input_count == 0) return 0.5f; - float sum = 0.0f; - sum_fill(sum, std::make_index_sequence{}); - return sum / static_cast(input_count); + float in = 0.0f; + sum_fill(in, std::make_index_sequence{}); + in /= static_cast(input_count); + + if constexpr (output_count == 0) return in; + + float out = 0.0f; + sum_output_fill(out, std::make_index_sequence{}); + out /= static_cast(output_count); + + const float p = 0.5f * (1.0f + in - out); + return p < 0.0f ? 0.0f : (p > 1.0f ? 1.0f : p); + } + + /// Mean fill of the output channels, same normalisation as sum_fill. + /// An unconnected output holds nothing back, so it contributes 0. + template + void sum_output_fill(float& sum, std::index_sequence) { + ((sum += (std::get(output_channels_) && + std::get(output_channels_)->capacity() > 0) + ? float(std::get(output_channels_)->approx_size()) + / float(std::get(output_channels_)->capacity()) + : 0.0f), ...); } template void sum_fill(float& sum, std::index_sequence) { @@ -639,6 +850,46 @@ private: t0.time_since_epoch()).count(); stats_.exec_start_us.store(now_us, std::memory_order_relaxed); + // Parked from a previous firing: retry that value before touching the + // inputs. Returning here releases the worker — the channel's space + // callback re-submits this node once the consumer drains a slot. + if constexpr (!std::is_void_v) { + if (pending_) { + push_outputs(std::move(*pending_), std::make_index_sequence{}); + queued_.store(false, std::memory_order_release); + if (pending_) { + // Close the lost-wakeup race: a space_callback that fired + // between the failed push and clearing queued_ was + // swallowed, and nothing else will wake this node. Re-check + // now that the flag is down. + if (outputs_have_space(std::make_index_sequence{})) + try_submit(0.5f); + return; // parked + } + // Drained: resume normal firing, resubmitting exactly the way + // the normal tail below does. An unconditional try_submit here + // would fire a node whose inputs are empty, and pop_inputs + // reports an empty channel as ChannelClosedError — which this + // node treats as "upstream finished" and self-stops on. That + // is a live node killing itself purely because it was woken by + // *output* space rather than by input arrival. + if constexpr (input_count == 0) try_submit(0.5f); + else on_input_ready(); + return; + } + } + + // See the equivalent guard in PoolNode::fire_once: a space-callback + // wake with nothing parked must release the worker, not fall through + // into pop_inputs on an empty channel. + if constexpr (input_count > 0) { + if (count_ready(std::make_index_sequence{}) != input_count) { + queued_.store(false, std::memory_order_release); + on_input_ready(); + return; + } + } + try { auto args = pop_inputs(std::make_index_sequence{}); auto t1 = clock_t::now(); @@ -662,7 +913,11 @@ private: } catch (const ChannelOverflowError&) { fire_callbacks(event_callbacks_); } catch (...) { - if (error_handler_ && error_handler_(name_, std::current_exception())) { + auto eptr = std::current_exception(); + const bool handled = + (error_handler_ && error_handler_(name_, eptr)) || + (net_error_handler_ && net_error_handler_(name_, eptr)); + if (handled) { } else { fire_callbacks(closed_callbacks_); self_stop(); @@ -695,22 +950,32 @@ private: } template + /// Pushes what it can and parks the rest. `pending_done_` marks the elements + /// already taken, so a retry never re-pushes one — a duplicate is as wrong as + /// a drop and harder to notice. void push_outputs(return_tuple&& result, std::index_sequence) { - (push_one_out(std::get(std::move(result))), ...); + bool all = true; + ((pending_done_[Is] = pending_done_[Is] || + push_one_out(std::get(std::move(result))), + all = all && pending_done_[Is]), ...); + if (all) { pending_.reset(); pending_done_.fill(false); } + else pending_ = std::move(result); } + /// Returns false when the ring was full and the value was NOT taken; the + /// caller must keep it and retry after the channel signals space. template - void push_one_out(std::tuple_element_t&& val) { + bool push_one_out(std::tuple_element_t&& val) { auto* ch = std::get(output_channels_); - if (!ch) return; + if (!ch) return true; // Sentinels (EOF) must never be dropped: a lost token wedges every // downstream pop() forever. Deliver them out-of-band (push_sentinel), // which never overflows and never blocks this node's worker thread. if (is_sentinel_value(val)) { ch->push_sentinel(std::move(val)); - return; + return true; } - // See the note on the typed overload above: block rather than drop. - ch->push_blocking(std::move(val)); + // See the note on the typed overload above: park rather than block. + return ch->try_push(val); } Obj& obj_; @@ -721,8 +986,18 @@ private: output_channels_t output_channels_{}; std::atomic stop_flag_{true}; std::atomic queued_{false}; + + /// The hidden one-slot output buffer (see push_outputs). Holding the value + /// here is what lets a node stop running without dropping it or occupying a + /// scheduler worker. One slot suffices because at most one fire_once() runs + /// per node at a time — with concurrent firing it would have to hold a whole + /// FIFO's worth. + std::conditional_t, std::monostate, + std::optional> pending_{}; + std::array pending_done_{}; NodeStats stats_; NodeErrorHandler error_handler_; + NodeErrorHandler net_error_handler_; std::chrono::milliseconds max_exec_time_{0}; std::array event_callbacks_{}; // [0]=user [1]=network std::array closed_callbacks_{}; diff --git a/include/kpn/static_network.hpp b/include/kpn/static_network.hpp index eeb5304..bb34a2c 100644 --- a/include/kpn/static_network.hpp +++ b/include/kpn/static_network.hpp @@ -122,6 +122,10 @@ public: [this, n](auto ts) { event_handler_(n, NodeEvent::Closed, ts); }); } } + if (error_handler_) { + for (auto* node : user_nodes_topo_) + node->set_network_error_callback(error_handler_); + } for (auto* n : user_nodes_topo_) n->start(); for (auto* n : fanout_nodes_ptr_) n->start(); #ifdef KPN_WEB_DEBUG @@ -181,6 +185,14 @@ public: void set_event_handler(EventHandler h) { event_handler_ = std::move(h); } + /// Application-level error listener. Receives the exception any node's + /// function throws, after that node's own handler (if any) declined it. + /// Return true to skip the failed invocation and keep the node running, + /// false to let it stop. Without a listener the exception is discarded + /// and only a Closed event survives, which reports that a node stopped + /// but not why. + void set_error_handler(NodeErrorHandler h) { error_handler_ = std::move(h); } + #ifdef KPN_WEB_DEBUG void set_web_debug_port(uint16_t port) { web_debug_port_ = port; } // Called by DebugHub::register_network() so the hub owns the debug server. @@ -276,6 +288,7 @@ private: std::vector> resource_probes_; std::vector> pool_probes_; EventHandler event_handler_; + NodeErrorHandler error_handler_; clock_t::time_point start_time_; #ifdef KPN_WEB_DEBUG uint16_t web_debug_port_{9090}; diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index fda6711..9e43788 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -34,6 +34,7 @@ add_executable(kpn_tests test_static_network.cpp test_shared_resource.cpp test_pool_node.cpp + test_backpressure_deadlock.cpp test_scheduler.cpp ) diff --git a/tests/test_backpressure_deadlock.cpp b/tests/test_backpressure_deadlock.cpp new file mode 100644 index 0000000..4679439 --- /dev/null +++ b/tests/test_backpressure_deadlock.cpp @@ -0,0 +1,85 @@ +// Regression: a blocking push must not park a pool worker. +// +// Node outputs use push_blocking so a full channel costs time rather than data +// (a dropped frame does not degrade a downstream result, it silently changes +// one). But push_blocking sleeps *inside* fire_once, which runs on a pool +// worker — and nodes are pinned to workers by index. Park enough workers in +// that retry loop and there is nobody left to run the consumer that would drain +// the channel, so the whole chain wedges. +// +// This is the failure channel.hpp:174 already warns about for sentinels +// ("a blocking push would park that thread and stop it draining its own input, +// cascading into a hold-and-wait deadlock under backpressure"). The warning +// applies to data pushes too. +// +// Observed in the field as an intermittent hang: frame_source, camera_pos, +// face_detector and face_aligner all asleep in push_blocking at once. +#include +#include + +#include +#include +#include + +namespace { + +struct Produce { + static constexpr std::string_view label() { return "produce"; } + int n{0}; + int operator()() { return n++; } +}; + +struct Relay { + static constexpr std::string_view label() { return "relay"; } + int operator()(int v) { return v; } +}; + +// Deliberately slower than the producer, so the channels between them fill. +struct SlowSink { + static constexpr std::string_view label() { return "slow_sink"; } + std::atomic* seen; + void operator()(int) { + std::this_thread::sleep_for(std::chrono::milliseconds(2)); + seen->fetch_add(1, std::memory_order_relaxed); + } +}; + +} // namespace + +TEST_CASE("a saturated chain keeps draining", "[backpressure][deadlock]") { + std::atomic seen{0}; + + Produce p_fn; + Relay r1_fn, r2_fn, r3_fn; + SlowSink s_fn{&seen}; + + // Small channels so they saturate immediately, and a chain longer than a + // modest pool — the shape that starves workers. + kpn::ObjectNode, kpn::out<"a">, "produce", 0> p (p_fn, 2); + kpn::ObjectNode, kpn::out<"b">, "relay1", 0> r1(r1_fn, 2); + kpn::ObjectNode, kpn::out<"c">, "relay2", 0> r2(r2_fn, 2); + kpn::ObjectNode, kpn::out<"d">, "relay3", 0> r3(r3_fn, 2); + kpn::ObjectNode, kpn::out<>, "slow_sink", 0> s (s_fn, 2); + + auto net = kpn::make_network( + kpn::edge(p.output<"a">(), r1.input<"a">()), + kpn::edge(r1.output<"b">(), r2.input<"b">()), + kpn::edge(r2.output<"c">(), r3.input<"c">()), + kpn::edge(r3.output<"d">(), s.input<"d">()) + ); + net.start(); + + // The sink is the slowest stage at 2 ms/item, so 40 items is ~80 ms of real + // work. Anything approaching the timeout means the chain stopped draining + // rather than merely running slowly. + const auto deadline = std::chrono::steady_clock::now() + std::chrono::seconds(20); + while (seen.load(std::memory_order_relaxed) < 40 && + std::chrono::steady_clock::now() < deadline) + std::this_thread::sleep_for(std::chrono::milliseconds(5)); + + const int got = seen.load(std::memory_order_relaxed); + net.stop(); + + INFO("items drained: " << got << " of 40"); + CHECK(got >= 40); +} diff --git a/tests/test_pool_node.cpp b/tests/test_pool_node.cpp index 72d60ba..779572b 100644 --- a/tests/test_pool_node.cpp +++ b/tests/test_pool_node.cpp @@ -243,7 +243,13 @@ TEST_CASE("interrupt node: trigger after stop is ignored", "[interrupt_node]") { // ── Overflow callback ───────────────────────────────────────────────────────── -TEST_CASE("pool node overflow callback fires on full output channel", "[pool_node][overflow]") { +// NOTE: output overflow is no longer reachable on the data path. A node whose +// output channel is full now PARKS — it keeps the value in a hidden one-slot +// buffer, releases its scheduler worker, and is re-submitted when the consumer +// frees a slot. The overflow callback survives for other producers (a direct +// Channel::push by non-node code still throws), but a pool node cannot trigger +// it, so these cases assert the stronger property instead: nothing is dropped. +TEST_CASE("pool node parks instead of overflowing a full output", "[pool_node][overflow]") { auto pool = std::make_shared(2); pool->start(); @@ -264,10 +270,13 @@ TEST_CASE("pool node overflow callback fires on full output channel", "[pool_nod node.stop(); pool->stop(); - REQUIRE(overflow_count.load() > 0); + // Parked, not overflowed: the value is still owned by the node. + REQUIRE(overflow_count.load() == 0); + // And it was never handed downstream, so nothing was lost or duplicated. + REQUIRE(full_ch.size() == 1); } -TEST_CASE("pool node overflow callback is independent per instance", "[pool_node][overflow]") { +TEST_CASE("parking is per node, not shared", "[pool_node][overflow]") { auto pool = std::make_shared(2); pool->start(); @@ -296,7 +305,10 @@ TEST_CASE("pool node overflow callback is independent per instance", "[pool_node nodeB.stop(); pool->stop(); - REQUIRE(a_overflows.load() > 0); + // Neither overflows now: A parks on its full output, B runs normally. The + // point of the case is unchanged — one node's backpressure must not leak + // into another's callbacks. + REQUIRE(a_overflows.load() == 0); REQUIRE(b_overflows.load() == 0); } @@ -412,7 +424,9 @@ TEST_CASE("network_overflow_callback fires on overflow", "[pool_node][network]") node.stop(); pool->stop(); - REQUIRE(net_overflows.load() > 0); + // Parking replaced overflow on the data path, so the network callback no + // longer fires for a pool node's own output. See the note above. + REQUIRE(net_overflows.load() == 0); } TEST_CASE("network_closed_callback fires on crash", "[pool_node][network]") { @@ -459,6 +473,8 @@ TEST_CASE("per-node and network overflow callbacks both fire independently", "[p node.stop(); pool->stop(); - REQUIRE(per_node.load() > 0); - REQUIRE(network.load() > 0); + // Both zero now: the node parks rather than overflowing. The case still + // guards that the two callbacks are wired independently. + REQUIRE(per_node.load() == 0); + REQUIRE(network.load() == 0); } From 9c5ce5f34a93945e19c357feef6adbd0e9ed8f4c Mon Sep 17 00:00:00 2001 From: Duncan Tourolle Date: Sun, 2 Aug 2026 18:33:17 +0200 Subject: [PATCH 15/42] =?UTF-8?q?fix:=20never=20drop=20a=20wake=20?= =?UTF-8?q?=E2=80=94=20a=20node=20must=20not=20sleep=20with=20one=20outsta?= =?UTF-8?q?nding?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 28e0667 stopped nodes blocking a worker on a full output, but replaced an intermittent hang with a quieter one: the pipeline still wedged about 2 runs in 30, now with every worker idle in pthread_cond_wait rather than asleep in a push. Nothing was blocked; nothing had been woken. try_submit discarded any wake arriving while queued_ was up: if (queued_.compare_exchange_strong(expected, true, ...)) scheduler_->submit(...); // else: silently gone Wakes are edge-triggered — a channel fires its space callback once, on the transition — so a dropped one never returns. A node could park a value, release its worker, and sleep forever holding exactly the output its consumer was waiting for, while its producer parked on an input channel that would never drain. try_submit now records the drop in wake_pending_, and release_and_recheck() consumes it at every site that releases a node, giving one invariant: a node never sleeps with a wake outstanding. This subsumes the two ad-hoc re-checks added for the parked-retry and normal push paths, which only moved the stall (1211 items to 6472) because each new early return was a fresh chance to drop a wake. self_stop keeps a plain store — honouring a pending wake there would resubmit a dead node. Adds "a saturated chain never stalls", which reproduces this in about a second where the pipeline needed ~30 runs. It asserts *progress does not freeze* rather than a completion total: capacity-1 channels are slow, and slow must never be reported as wedged. Verified in both directions — it stalls after 1211 items on 28e0667 and passes here. The existing chain test could not catch it: 40 items drain before any strand occurs, and its producer emits forever, so fresh input keeps re-triggering on_input_ready() and flushing the stranded value. Tests: 122/122. --- include/kpn/pool_node.hpp | 101 +++++++++++++++++++++++++-- tests/test_backpressure_deadlock.cpp | 86 +++++++++++++++++++++++ 2 files changed, 180 insertions(+), 7 deletions(-) diff --git a/include/kpn/pool_node.hpp b/include/kpn/pool_node.hpp index 85532da..2d12cbe 100644 --- a/include/kpn/pool_node.hpp +++ b/include/kpn/pool_node.hpp @@ -265,6 +265,8 @@ private: disable_inputs(std::make_index_sequence{}); disable_outputs(std::make_index_sequence{}); stats_.exec_start_us.store(0, std::memory_order_relaxed); + // Plain store, not release_and_recheck(): this node is stopping, and + // honouring a pending wake here would resubmit a dead node. queued_.store(false, std::memory_order_release); stop_flag_.store(true, std::memory_order_relaxed); } @@ -353,10 +355,32 @@ private: : 0.5f), ...); } + /// Submit unless already queued. A wake that arrives while this node is + /// queued or running is *recorded*, never dropped. + /// + /// Wakes are edge-triggered: a channel fires its space callback on the + /// transition, once. If that lands while queued_ is up, the CAS below fails + /// and — before wake_pending_ — the wake was gone. A node could then park a + /// value, release its worker, and sleep forever holding output its consumer + /// was waiting for, with every worker idle in cond_wait and nothing left to + /// re-trigger it. Recording the drop turns the signal level-triggered: the + /// invariant is that a node never sleeps with a wake outstanding, enforced + /// by release_and_recheck() at every point that releases the node. void try_submit(float priority) { bool expected = false; if (queued_.compare_exchange_strong(expected, true, std::memory_order_acq_rel)) scheduler_->submit([this] { fire_once(); }, priority); + else + wake_pending_.store(true, std::memory_order_release); + } + + /// Clear queued_, then honour any wake that was dropped while it was up. + /// Every path that finishes or parks a firing must release the node through + /// here rather than storing queued_ directly. + void release_and_recheck(float priority = 0.5f) { + queued_.store(false, std::memory_order_release); + if (wake_pending_.exchange(false, std::memory_order_acq_rel)) + try_submit(priority); } // ── Execution ───────────────────────────────────────────────────────────── @@ -379,7 +403,7 @@ private: if constexpr (!std::is_void_v) { if (pending_) { push_outputs(std::move(*pending_), std::make_index_sequence{}); - queued_.store(false, std::memory_order_release); + release_and_recheck(); if (pending_) { // Close the lost-wakeup race: a space_callback that fired // between the failed push and clearing queued_ was @@ -408,7 +432,7 @@ private: if constexpr (!std::is_void_v) { if (pending_) { push_outputs(std::move(*pending_), std::make_index_sequence{}); - queued_.store(false, std::memory_order_release); + release_and_recheck(); if (pending_) { // Close the lost-wakeup race: a space_callback that fired // between the failed push and clearing queued_ was @@ -438,7 +462,7 @@ private: // on_input_ready() resubmits when data actually lands. if constexpr (input_count > 0) { if (count_ready(std::make_index_sequence{}) != input_count) { - queued_.store(false, std::memory_order_release); + release_and_recheck(); on_input_ready(); // data may have arrived while we checked return; } @@ -483,10 +507,27 @@ private: } stats_.exec_start_us.store(0, std::memory_order_relaxed); - queued_.store(false, std::memory_order_release); + release_and_recheck(); if (stop_flag_.load(std::memory_order_relaxed)) return; + // Parked by the push above. Same situation as the retry path at the top + // of fire_once — and the same lost-wakeup race, which that path closes + // and this one did not. A space callback that fired while queued_ was + // still up got swallowed by try_submit's CAS, and the resubmit below + // cannot cover it: this firing consumed its input, so inputs are empty + // and on_input_ready() will not resubmit. The node would then hold its + // value forever while its consumer waits for exactly that value and its + // producer parks on an input channel that never drains. Re-check now + // that the flag is down. + if constexpr (!std::is_void_v) { + if (pending_) { + if (outputs_have_space(std::make_index_sequence{})) + try_submit(0.5f); + return; // parked + } + } + // Source nodes always resubmit; others resubmit only if inputs are ready. if constexpr (input_count == 0) { try_submit(0.5f); @@ -574,6 +615,8 @@ private: output_channels_t output_channels_{}; std::atomic stop_flag_{true}; std::atomic queued_{false}; + /// A wake that arrived while queued_ was up. See try_submit. + std::atomic wake_pending_{false}; /// The hidden one-slot output buffer (see push_outputs). Holding the value /// here is what lets a node stop running without dropping it or occupying a @@ -753,6 +796,8 @@ private: disable_inputs(std::make_index_sequence{}); disable_outputs(std::make_index_sequence{}); stats_.exec_start_us.store(0, std::memory_order_relaxed); + // Plain store, not release_and_recheck(): this node is stopping, and + // honouring a pending wake here would resubmit a dead node. queued_.store(false, std::memory_order_release); stop_flag_.store(true, std::memory_order_relaxed); } @@ -834,10 +879,32 @@ private: : 0.5f), ...); } + /// Submit unless already queued. A wake that arrives while this node is + /// queued or running is *recorded*, never dropped. + /// + /// Wakes are edge-triggered: a channel fires its space callback on the + /// transition, once. If that lands while queued_ is up, the CAS below fails + /// and — before wake_pending_ — the wake was gone. A node could then park a + /// value, release its worker, and sleep forever holding output its consumer + /// was waiting for, with every worker idle in cond_wait and nothing left to + /// re-trigger it. Recording the drop turns the signal level-triggered: the + /// invariant is that a node never sleeps with a wake outstanding, enforced + /// by release_and_recheck() at every point that releases the node. void try_submit(float priority) { bool expected = false; if (queued_.compare_exchange_strong(expected, true, std::memory_order_acq_rel)) scheduler_->submit([this] { fire_once(); }, priority); + else + wake_pending_.store(true, std::memory_order_release); + } + + /// Clear queued_, then honour any wake that was dropped while it was up. + /// Every path that finishes or parks a firing must release the node through + /// here rather than storing queued_ directly. + void release_and_recheck(float priority = 0.5f) { + queued_.store(false, std::memory_order_release); + if (wake_pending_.exchange(false, std::memory_order_acq_rel)) + try_submit(priority); } void fire_once() { @@ -856,7 +923,7 @@ private: if constexpr (!std::is_void_v) { if (pending_) { push_outputs(std::move(*pending_), std::make_index_sequence{}); - queued_.store(false, std::memory_order_release); + release_and_recheck(); if (pending_) { // Close the lost-wakeup race: a space_callback that fired // between the failed push and clearing queued_ was @@ -884,7 +951,7 @@ private: // into pop_inputs on an empty channel. if constexpr (input_count > 0) { if (count_ready(std::make_index_sequence{}) != input_count) { - queued_.store(false, std::memory_order_release); + release_and_recheck(); on_input_ready(); return; } @@ -926,8 +993,26 @@ private: } stats_.exec_start_us.store(0, std::memory_order_relaxed); - queued_.store(false, std::memory_order_release); + release_and_recheck(); if (stop_flag_.load(std::memory_order_relaxed)) return; + + // Parked by the push above. Same situation as the retry path at the top + // of fire_once — and the same lost-wakeup race, which that path closes + // and this one did not. A space callback that fired while queued_ was + // still up got swallowed by try_submit's CAS, and the resubmit below + // cannot cover it: this firing consumed its input, so inputs are empty + // and on_input_ready() will not resubmit. The node would then hold its + // value forever while its consumer waits for exactly that value and its + // producer parks on an input channel that never drains. Re-check now + // that the flag is down. + if constexpr (!std::is_void_v) { + if (pending_) { + if (outputs_have_space(std::make_index_sequence{})) + try_submit(0.5f); + return; // parked + } + } + if constexpr (input_count == 0) try_submit(0.5f); else on_input_ready(); } @@ -986,6 +1071,8 @@ private: output_channels_t output_channels_{}; std::atomic stop_flag_{true}; std::atomic queued_{false}; + /// A wake that arrived while queued_ was up. See try_submit. + std::atomic wake_pending_{false}; /// The hidden one-slot output buffer (see push_outputs). Holding the value /// here is what lets a node stop running without dropping it or occupying a diff --git a/tests/test_backpressure_deadlock.cpp b/tests/test_backpressure_deadlock.cpp index 4679439..17835ec 100644 --- a/tests/test_backpressure_deadlock.cpp +++ b/tests/test_backpressure_deadlock.cpp @@ -83,3 +83,89 @@ TEST_CASE("a saturated chain keeps draining", "[backpressure][deadlock]") { INFO("items drained: " << got << " of 40"); CHECK(got >= 40); } + +// Regression: a saturated chain must never stall. +// +// push_outputs parks from two places: the retry at the top of fire_once, and +// the ordinary push after the node function returns. Both release the worker, +// so both face the same lost wakeup — a space callback firing while queued_ is +// still up is swallowed by try_submit's CAS. Only the retry path re-checked for +// space afterwards. The normal path fell through to on_input_ready(), which +// resubmits only if inputs are ready — and the firing that just parked had +// consumed its input, so they are not. +// +// The strand is permanent under saturation: the node holds its value, its +// consumer waits for exactly that value, and its producer fills the node's +// input channel and parks too. Nothing moves again. +// +// The test above cannot catch it — 40 items drain before any strand occurs. +// This one runs the chain saturated and watches for progress to *freeze*, which +// is the signature of the deadlock. It deliberately does not assert a total: +// capacity-1 channels are slow, and "slow" must never be reported as "wedged". +namespace { + +struct FreeRun { + static constexpr std::string_view label() { return "free_run"; } + int n{0}; + int operator()() { return n++; } +}; + +struct CountingSink { + static constexpr std::string_view label() { return "counting_sink"; } + std::atomic* seen; + void operator()(int) { seen->fetch_add(1, std::memory_order_relaxed); } +}; + +} // namespace + +TEST_CASE("a saturated chain never stalls", "[backpressure][deadlock]") { + std::atomic seen{0}; + + FreeRun p_fn; + Relay r1_fn, r2_fn; + CountingSink s_fn{&seen}; + + // Capacity 1 everywhere: every push contends, so the park path is taken + // constantly and the race window is sampled millions of times. + kpn::ObjectNode, kpn::out<"a">, "free_run", 0> p (p_fn, 1); + kpn::ObjectNode, kpn::out<"b">, "relay1", 0> r1(r1_fn, 1); + kpn::ObjectNode, kpn::out<"c">, "relay2", 0> r2(r2_fn, 1); + kpn::ObjectNode, kpn::out<>, "sink", 0> s (s_fn, 1); + + auto net = kpn::make_network( + kpn::edge(p.output<"a">(), r1.input<"a">()), + kpn::edge(r1.output<"b">(), r2.input<"b">()), + kpn::edge(r2.output<"c">(), s.input<"c">()) + ); + net.start(); + + // A live chain moves thousands of items a second, so 3 s with no movement + // at all is a wedge, not a slow patch. Sampling for 25 s gives the race + // ample opportunity: the pipeline hit it roughly twice in 30 runs. + const auto giveup = std::chrono::steady_clock::now() + std::chrono::seconds(25); + int last = 0; + auto last_move = std::chrono::steady_clock::now(); + bool stalled = false; + int stall_at = 0; + + while (std::chrono::steady_clock::now() < giveup) { + std::this_thread::sleep_for(std::chrono::milliseconds(100)); + const int now_seen = seen.load(std::memory_order_relaxed); + if (now_seen != last) { + last = now_seen; + last_move = std::chrono::steady_clock::now(); + } else if (std::chrono::steady_clock::now() - last_move > + std::chrono::seconds(3)) { + stalled = true; + stall_at = now_seen; + break; + } + } + + net.stop(); + + INFO("chain stalled after " << stall_at << " items"); + CHECK_FALSE(stalled); + // Guard against the test passing because nothing ever ran. + CHECK(last > 1000); +} From 454f72c1674d34e8ee552ed3859936d10e8466e1 Mon Sep 17 00:00:00 2001 From: Duncan Tourolle Date: Tue, 4 Aug 2026 14:04:50 +0200 Subject: [PATCH 16/42] chore: ignore generated ORT engine cache ort_cache/ holds .ort engines built on first run from the .onnx models. They are machine- and version-specific build products, not sources. --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 17c9c78..483232f 100644 --- a/.gitignore +++ b/.gitignore @@ -27,3 +27,4 @@ Thumbs.db # Claude Code local settings .claude/settings.local.json +include/kpn/ort_cache/ From a8cfe7300af4782adff531563d8f44b04717d6fd Mon Sep 17 00:00:00 2001 From: Duncan Tourolle Date: Wed, 5 Aug 2026 12:40:03 +0200 Subject: [PATCH 17/42] fix: a lossless fanout, a node that starts awake, and the instrumentation that found them MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Three changes from one debugging session on the intermittent wedge, kept together because the instrumentation is what made the other two findable. **Fanout was never made lossless.** 6595e6e made node outputs lossless and 28e0667 stopped them parking a worker; FanoutNode was in neither and kept `catch (ChannelOverflowError&) {}` per output. Whichever branch fell behind lost items, silently, by an amount that depended on timing — so two runs of the same input could disagree. deliver() now retries each output independently until it is taken, rechecking stop_flag_ every pass so teardown cannot hang on a full output. A fanout owns a private thread, so waiting costs no scheduler worker. **A node could start with a wake already outstanding.** start() enables the input channel several statements before it installs the push callback, and StaticNetwork starts nodes sources-first, so an upstream node is already firing into the gap. A push landing there is accepted by the ring but wakes nobody: push_callback_ fires only on the empty→non-empty transition, and at that instant the callback is null. Every later push sees a non-empty ring and stays silent, so the node is never submitted. Asking on_input_ready() once at the end of start() converts the missed edge into a state check. The signature is distinctive — zero items delivered, not a stall partway. Under `ctest -j4` on a loaded machine it reproduced 7 times in 24 and never in 10 unloaded runs, which is almost certainly the "~1 run in 20" hang 28e0667 recorded as known-incomplete. **NodeSnapshot now carries scheduling state and a true exec total.** queued and wake_pending make the 9c5ce5f invariant observable at runtime; it could previously only be inspected in a debugger, and the bug does not reproduce under one. total_exec_us is a real sum — frames × ema_exec_us tracks the tail of a run, not the whole of it, and diverges badly on a workload whose per-frame cost varies. Both are exposed over the web debug JSON so a wedged pipeline can be interrogated without attaching to it. --- include/kpn/diagnostics.hpp | 27 +++++ include/kpn/fanout.hpp | 91 ++++++++++++++-- include/kpn/interrupt_node.hpp | 1 + include/kpn/main_thread_node.hpp | 2 + include/kpn/pool_node.hpp | 26 +++++ include/kpn/web_debug.hpp | 5 + tests/test_backpressure_deadlock.cpp | 153 +++++++++++++++++++++++++++ 7 files changed, 297 insertions(+), 8 deletions(-) diff --git a/include/kpn/diagnostics.hpp b/include/kpn/diagnostics.hpp index f5080b1..2bc1e4c 100644 --- a/include/kpn/diagnostics.hpp +++ b/include/kpn/diagnostics.hpp @@ -51,6 +51,14 @@ struct NodeStats { std::atomic max_exec_us{0}; std::atomic total_blocked_us{0}; + // Cumulative wall time inside fire_once, summed over every invocation. + // The EMA above cannot be turned into a total: it is exponentially + // weighted, so frames * ema_exec_us tracks the tail of the run rather than + // the whole of it, and on a workload whose per-frame cost varies (a face + // detector on a film: crowd scenes then empty landscapes) the two differ by + // a lot. Answering "how much time went into this node" needs a real sum. + std::atomic total_exec_us{0}; + // Thread CPU time — actual CPU consumed by this node's thread, // measured via CLOCK_THREAD_CPUTIME_ID. Excludes time sleeping or // blocked on mutexes/channels. Sampled once per frame. @@ -89,6 +97,7 @@ struct NodeStats { frames_processed.fetch_add(1, std::memory_order_relaxed); int64_t us = static_cast(exec_time.count() * 1000.0); + total_exec_us.fetch_add(us, std::memory_order_relaxed); uint64_t n = frames_processed.load(std::memory_order_relaxed); int64_t prev = ema_exec_us.load(std::memory_order_relaxed); @@ -147,6 +156,24 @@ struct NodeSnapshot { double total_cpu_ms; // cumulative CPU time consumed by this node's thread double cpu_util_pct; // exec_ms / (exec_ms + blocked_ms) * 100 double queue_wait_ms{0}; // PoolNode: cumulative time spent in pool queue + + // Live scheduling state, for observing the AR-004 invariant "a node never + // sleeps with a wake outstanding". The invariant was previously asserted in + // comments but invisible at runtime, so a lost wake could only be found in a + // debugger — and this bug does not reproduce under one (it needs full speed). + // Two atomic loads at snapshot time, nothing on the hot path. + // + // Read them together with the node's channel fill: + // queued=0, wake=1 -> wake recorded and never consumed + // queued=0, wake=0, input full -> wake never generated at all + // queued=1 while nothing running -> submitted but never scheduled + bool queued{false}; + bool wake_pending{false}; + // Cumulative wall time inside fire_once. Unlike ema_exec_ms this is a true + // sum, so it is the field to use for "share of the run spent in this node". + // Note it still includes time parked pushing into a full output channel; + // total_cpu_ms is the part that backpressure cannot inflate. + double total_exec_ms{0}; }; // ── Pool statistics + snapshot ──────────────────────────────────────────────── diff --git a/include/kpn/fanout.hpp b/include/kpn/fanout.hpp index 180b602..1d6bf9e 100644 --- a/include/kpn/fanout.hpp +++ b/include/kpn/fanout.hpp @@ -7,8 +7,10 @@ #include #include +#include #include #include +#include #include #include #include @@ -78,7 +80,9 @@ public: blocked_ms, elapsed_s > 0 ? frames / elapsed_s : 0.0, stats_.total_cpu_us.load(std::memory_order_relaxed) / 1000.0, - total_ms > 0 ? 100.0 * exec_ms / total_ms : 0.0}; + total_ms > 0 ? 100.0 * exec_ms / total_ms : 0.0, + 0.0, // queue_wait_ms — fanout is not pool-scheduled + stats_.total_exec_us.load(std::memory_order_relaxed) / 1000.0}; } // ── Port access ─────────────────────────────────────────────────────────── @@ -116,6 +120,76 @@ public: } private: + // Deliver `val` to every connected output, losslessly. + // + // Previously a full output cost the value: push() threw and the exception was + // swallowed per output. A dropped item does not degrade a downstream result, + // it silently changes one, and the consumer cannot tell it happened — so the + // fanout waits instead, and the producer upstream runs slower. + // + // Unlike a pool node, a fanout owns a private thread, so waiting here costs + // no scheduler worker and needs no space-callback park; a bounded retry is + // enough. `stop_flag_` is re-checked every pass so teardown cannot hang on a + // full output regardless of the order the network stops its nodes in. + // + // Outputs are retried independently, so a full output never delays delivery + // to one with room. Note what that does *not* buy: the next input is not + // popped until every output has accepted the current item, so one branch can + // never run ahead of another by more than the slower branch's buffering. + // + // **That bound is a precondition on any topology where the branches rejoin.** + // If a consumer on branch B blocks waiting for something branch A computes, + // B's buffering must exceed the lead A needs, or the two wedge — B waiting on + // A, A starved because the fanout is holding an item B will not take. Making + // the fanout lossless is what puts that precondition on the topology; while + // it dropped, the question could not arise. + // + // `parked` receives the time spent waiting on a full output, which the caller + // charges to blocked rather than exec. + // + // Returns false if stopped with the value undelivered. + bool deliver(const T& val, duration_t& parked) { + std::array, N> pending; + std::size_t outstanding = 0; + for (std::size_t i = 0; i < N; ++i) + if (out_channels_[i]) { pending[i].emplace(val); ++outstanding; } + + bool first_pass = true; + auto park_from = clock_t::now(); + + for (;;) { + for (std::size_t i = 0; i < N; ++i) { + if (!pending[i]) continue; + if (out_channels_[i]->try_push(*pending[i])) { + pending[i].reset(); + --outstanding; + } + } + if (first_pass) { park_from = clock_t::now(); first_pass = false; } + + if (outstanding == 0) { + parked = duration_t(clock_t::now() - park_from); + return true; + } + if (stop_flag_.load(std::memory_order_relaxed)) { + // Teardown with work in hand. One last throwing push per + // outstanding output, purely so the channel's own stats record + // the loss (drop if it is disabled, overflow if it is merely + // full). The whole point of the lossless path is that a loss is + // never invisible, and a silent `return` here would reintroduce + // exactly the hole this function exists to close. + for (std::size_t i = 0; i < N; ++i) { + if (!pending[i]) continue; + try { out_channels_[i]->push(std::move(*pending[i])); } + catch (const ChannelOverflowError&) {} + } + parked = duration_t(clock_t::now() - park_from); + return false; + } + std::this_thread::sleep_for(std::chrono::microseconds(50)); + } + } + void run_loop() { while (!stop_flag_.load(std::memory_order_relaxed)) { try { @@ -124,16 +198,17 @@ private: auto t1 = clock_t::now(); auto cpu0 = NodeStats::cpu_now(); - for (std::size_t i = 0; i < N; ++i) { - if (out_channels_[i]) { - try { out_channels_[i]->push(val); } - catch (const ChannelOverflowError&) {} // drop for this output independently - } - } + duration_t parked{0}; + const bool delivered = deliver(val, parked); auto cpu1 = NodeStats::cpu_now(); auto t2 = clock_t::now(); - stats_.record_exec(duration_t(t2 - t1), duration_t(t1 - t0), cpu0, cpu1); + // Time spent waiting on a full output is *blocked*, not exec: a + // parked fanout is idle, and charging it to exec would report the + // node as busy exactly when it is the one being held up. + stats_.record_exec(duration_t(t2 - t1) - parked, + duration_t(t1 - t0) + parked, cpu0, cpu1); + if (!delivered) break; } catch (const ChannelClosedError&) { break; } diff --git a/include/kpn/interrupt_node.hpp b/include/kpn/interrupt_node.hpp index 6e58c24..0b8c830 100644 --- a/include/kpn/interrupt_node.hpp +++ b/include/kpn/interrupt_node.hpp @@ -104,6 +104,7 @@ public: stats_.total_cpu_us.load(std::memory_order_relaxed) / 1000.0, total_ms > 0 ? 100.0 : 0.0, qwait_ms, + stats_.total_exec_us.load(std::memory_order_relaxed) / 1000.0, }; } diff --git a/include/kpn/main_thread_node.hpp b/include/kpn/main_thread_node.hpp index 47ecb18..17ff222 100644 --- a/include/kpn/main_thread_node.hpp +++ b/include/kpn/main_thread_node.hpp @@ -90,6 +90,8 @@ public: elapsed_s > 0 ? frames / elapsed_s : 0.0, stats_.total_cpu_us.load(std::memory_order_relaxed) / 1000.0, total_ms > 0 ? 100.0 * exec_ms / total_ms : 0.0, + 0.0, // queue_wait_ms — main-thread node is not pool-scheduled + stats_.total_exec_us.load(std::memory_order_relaxed) / 1000.0, }; } diff --git a/include/kpn/pool_node.hpp b/include/kpn/pool_node.hpp index 2d12cbe..e9d8d1a 100644 --- a/include/kpn/pool_node.hpp +++ b/include/kpn/pool_node.hpp @@ -116,6 +116,23 @@ public: register_callbacks(std::make_index_sequence{}); if constexpr (input_count == 0) try_submit(0.5f); + else + // Never start with a wake already outstanding — the startup case of + // the invariant 9c5ce5f established for the running pipeline. + // + // enable_inputs() opens the channel several statements before + // register_callbacks() installs the push callback, and the network + // starts nodes sources-first, so an upstream node is already firing + // into this one during that gap. A push landing there is accepted by + // the ring but wakes nobody: Channel::push only invokes the callback + // on the empty→non-empty transition, and at that instant the + // callback is still null. Every later push sees a non-empty ring and + // stays silent, so the node is never submitted — the pipeline reads + // as wedged from the first frame, with no item ever delivered. + // + // on_input_ready() is the level-triggered form of the same question, + // so asking it once here converts the missed edge into a state check. + on_input_ready(); } void stop() override { @@ -156,6 +173,9 @@ public: stats_.total_cpu_us.load(std::memory_order_relaxed) / 1000.0, total_ms > 0 ? 100.0 * exec_ms / total_ms : 0.0, qwait_ms, + stats_.total_exec_us.load(std::memory_order_relaxed) / 1000.0, + queued_.load(std::memory_order_relaxed), + wake_pending_.load(std::memory_order_relaxed), }; } @@ -686,6 +706,9 @@ public: register_callbacks(std::make_index_sequence{}); if constexpr (input_count == 0) try_submit(0.5f); + else + // Never start with a wake already outstanding — see PoolNode::start(). + on_input_ready(); } void stop() override { @@ -720,6 +743,9 @@ public: stats_.total_cpu_us.load(std::memory_order_relaxed) / 1000.0, total_ms > 0 ? 100.0 * exec_ms / total_ms : 0.0, qwait_ms, + stats_.total_exec_us.load(std::memory_order_relaxed) / 1000.0, + queued_.load(std::memory_order_relaxed), + wake_pending_.load(std::memory_order_relaxed), }; } diff --git a/include/kpn/web_debug.hpp b/include/kpn/web_debug.hpp index cd325af..27748be 100644 --- a/include/kpn/web_debug.hpp +++ b/include/kpn/web_debug.hpp @@ -65,6 +65,11 @@ static std::string to_json(const std::vector& nodes, << ",\"fps\":" << n.throughput_fps << ",\"total_cpu_ms\":" << n.total_cpu_ms << ",\"cpu_util_pct\":" << n.cpu_util_pct + // Scheduling state — lets a WEDGED pipeline be interrogated over HTTP + // without a debugger, which matters because the lost-wake bug does not + // reproduce under one. See NodeSnapshot for how to read the pair. + << ",\"queued\":" << (n.queued ? "true" : "false") + << ",\"wake_pending\":" << (n.wake_pending ? "true" : "false") << "}"; } o << "],\"edges\":["; diff --git a/tests/test_backpressure_deadlock.cpp b/tests/test_backpressure_deadlock.cpp index 17835ec..c1ee551 100644 --- a/tests/test_backpressure_deadlock.cpp +++ b/tests/test_backpressure_deadlock.cpp @@ -169,3 +169,156 @@ TEST_CASE("a saturated chain never stalls", "[backpressure][deadlock]") { // Guard against the test passing because nothing ever ran. CHECK(last > 1000); } + +// Regression: a node must not start with a wake already outstanding. +// +// 9c5ce5f established the invariant for the running pipeline — a node never +// sleeps with a wake it dropped. start() broke the same invariant before the +// pipeline was even running: +// +// enable_inputs(...); // channel goes live here +// stop_flag_.store(false); +// queued_.store(false); +// register_callbacks(...); // push callback installed here +// +// StaticNetwork starts nodes sources-first, so an upstream node is already +// firing into this one during that gap. A push landing there is accepted by the +// ring but wakes nobody: Channel::push invokes push_callback_ only on the +// empty→non-empty transition, and at that instant the callback is null. Every +// later push sees a non-empty ring and stays silent. The node is never +// submitted, and since a sink has no outputs there is no space callback to +// rescue it either. +// +// The signature is distinctive: **zero** items delivered, not a stall partway. +// The chain reads as wedged from the first frame. Under `ctest -j4` on a loaded +// machine it reproduced 7 times in 24, and never once in 10 unloaded runs — +// contention widens the window between those two statements. That is almost +// certainly the "rare hang, ~1 run in 20 at a 300 s timeout" 28e0667 recorded as +// known-incomplete. +// +// This test needs no contention: it constructs the state the race leaves behind +// directly, by enabling the input and pushing before start() is ever called. +namespace { + +int passthrough(int x) { return x; } + +} // namespace + +TEST_CASE("a node started with data already queued still fires", + "[backpressure][startup]") { + auto pool = std::make_shared(2); + pool->start(); + + auto node = kpn::make_pool_node(pool, 8); + kpn::Channel out_ch(8); + node.set_output_channel<0>(&out_ch); + + // The missed edge: the channel is live and already holds a value, but no + // callback was installed when it arrived, so the wake has been and gone. + node.input_channel<0>().enable(); + node.input_channel<0>().push(21); + + node.start(); + + // Bounded wait — a plain pop() would hang rather than fail on a regression. + const auto deadline = std::chrono::steady_clock::now() + std::chrono::seconds(5); + while (out_ch.size() == 0 && std::chrono::steady_clock::now() < deadline) + std::this_thread::sleep_for(std::chrono::milliseconds(5)); + + const bool delivered = out_ch.size() > 0; + const int got = delivered ? out_ch.pop() : -1; + + node.stop(); + pool->stop(); + + INFO("value delivered: " << got); + REQUIRE(delivered); + CHECK(got == 21); +} + +// Regression: a fanout absorbs an unequal pair of consumers by slowing, not by +// dropping. +// +// 6595e6e made node outputs lossless and 28e0667 stopped them parking a worker, +// but FanoutNode was in neither: it kept `catch (ChannelOverflowError&) {}` per +// output, so whichever branch fell behind lost items — silently, and by an +// amount that depended on timing. Two runs of the same input could therefore +// disagree, which is fatal for a fixture the rest of the suite is scored +// against. +// +// The two assertions are the two halves of the requirement: +// - no gaps: the slow branch receives *every* item, not most of them; +// - bounded lead: the fast branch is throttled to the slow one rather than +// racing ahead over a drain that is quietly discarding the difference. +// +// Either alone would pass on a broken implementation. A fanout that pushed only +// to the slow branch has no gaps; one that dropped everything for the slow +// branch keeps a bounded lead by never letting it fall behind. +namespace { + +// Records the sequence it sees, so a dropped item shows up as a gap rather than +// merely as a smaller total. +struct SeqCheck { + std::atomic* next_expected; + std::atomic* saw_gap; + int delay_us{0}; + + void record(int v) const { + if (delay_us) + std::this_thread::sleep_for(std::chrono::microseconds(delay_us)); + const int want = next_expected->load(std::memory_order_relaxed); + if (v != want) saw_gap->store(true, std::memory_order_relaxed); + else next_expected->store(want + 1, std::memory_order_relaxed); + } +}; + +struct FastBranch : SeqCheck { + static constexpr std::string_view label() { return "fast_branch"; } + void operator()(int v) { record(v); } +}; + +struct SlowBranch : SeqCheck { + static constexpr std::string_view label() { return "slow_branch"; } + void operator()(int v) { record(v); } +}; + +} // namespace + +TEST_CASE("a fanout absorbs an unequal pair by slowing, not dropping", + "[backpressure][fanout]") { + std::atomic fast_next{0}, slow_next{0}; + std::atomic fast_gap{false}, slow_gap{false}; + + FreeRun p_fn; + FastBranch fast_fn{{&fast_next, &fast_gap, 0}}; + SlowBranch slow_fn{{&slow_next, &slow_gap, 500}}; // 0.5 ms/item + + // Small channels so the slow branch saturates in the first few milliseconds + // and stays saturated for the whole run. + kpn::ObjectNode, kpn::out<"v">, "free_run", 0> p (p_fn, 8); + kpn::ObjectNode, kpn::out<>, "fast", 0> fa(fast_fn, 8); + kpn::ObjectNode, kpn::out<>, "slow", 0> sl(slow_fn, 8); + + // Two edges from one output port: make_network auto-inserts FanoutNode. + auto net = kpn::make_network( + kpn::edge(p.output<"v">(), fa.input<"fast">()), + kpn::edge(p.output<"v">(), sl.input<"slow">()) + ); + net.start(); + std::this_thread::sleep_for(std::chrono::seconds(1)); + net.stop(); + + const int fast_seen = fast_next.load(std::memory_order_relaxed); + const int slow_seen = slow_next.load(std::memory_order_relaxed); + + INFO("fast branch " << fast_seen << " items, slow branch " << slow_seen); + CHECK_FALSE(fast_gap.load(std::memory_order_relaxed)); + CHECK_FALSE(slow_gap.load(std::memory_order_relaxed)); + // Guard against passing because nothing ran: 1 s at 0.5 ms/item is ~2000. + CHECK(slow_seen > 200); + // The lead is bounded by the buffering between the two — the fanout's own + // input, the two output channels, and one item in each node's hand. A + // dropping fanout has no such bound: the fast branch runs at full speed and + // the difference is the loss. + CHECK(fast_seen - slow_seen < 200); +} From 091211cb198e5b0b2d9d650919845fb4ba24f2c2 Mon Sep 17 00:00:00 2001 From: Duncan Tourolle Date: Wed, 5 Aug 2026 12:46:56 +0200 Subject: [PATCH 18/42] fix: NodeSnapshot fields must line up with what nodes supply MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit a8cfe73 appended queued/wake_pending/total_exec_ms to the NodeSnapshot aggregate in an order no call site used. Every node type fills the aggregate positionally and all of them supply total_exec_ms as the element straight after queue_wait_ms — but the struct declared the two bools there. So the exec total landed in `queued`, `queued` landed in `wake_pending`, and `wake_pending` landed in total_exec_ms. GCC reported it as -Wnarrowing (bool to double), 88 times, once per node instantiation across the test build. The build carried on. This is worse than a cosmetic mix-up, because all three fields were added specifically to diagnose a wedge. A wedged pipeline reported total_exec_ms as 0.0 or 1.0, and `queued` as "has this node ever run" — true for every node that had, including ones asleep with nothing to do. The web debug JSON served the same values. Anyone reading them to find the stalled node would have been pointed at the wrong one. Moves total_exec_ms above the two bools to match every call site, and notes in the struct why the order is load-bearing. Verified in both directions: on a8cfe73 the new case reports frames=8 total=0 queued=true; here total >= ema and both flags are false. --- include/kpn/diagnostics.hpp | 15 ++++++++---- tests/test_pool_node.cpp | 46 +++++++++++++++++++++++++++++++++++++ 2 files changed, 56 insertions(+), 5 deletions(-) diff --git a/include/kpn/diagnostics.hpp b/include/kpn/diagnostics.hpp index 2bc1e4c..b94c8f2 100644 --- a/include/kpn/diagnostics.hpp +++ b/include/kpn/diagnostics.hpp @@ -157,6 +157,16 @@ struct NodeSnapshot { double cpu_util_pct; // exec_ms / (exec_ms + blocked_ms) * 100 double queue_wait_ms{0}; // PoolNode: cumulative time spent in pool queue + // Cumulative wall time inside fire_once. Unlike ema_exec_ms this is a true + // sum, so it is the field to use for "share of the run spent in this node". + // Note it still includes time parked pushing into a full output channel; + // total_cpu_ms is the part that backpressure cannot inflate. + // + // Declared before the two bools below because every node type initialises + // this aggregate positionally, and all of them supply total_exec_ms as the + // element after queue_wait_ms. + double total_exec_ms{0}; + // Live scheduling state, for observing the AR-004 invariant "a node never // sleeps with a wake outstanding". The invariant was previously asserted in // comments but invisible at runtime, so a lost wake could only be found in a @@ -169,11 +179,6 @@ struct NodeSnapshot { // queued=1 while nothing running -> submitted but never scheduled bool queued{false}; bool wake_pending{false}; - // Cumulative wall time inside fire_once. Unlike ema_exec_ms this is a true - // sum, so it is the field to use for "share of the run spent in this node". - // Note it still includes time parked pushing into a full output channel; - // total_cpu_ms is the part that backpressure cannot inflate. - double total_exec_ms{0}; }; // ── Pool statistics + snapshot ──────────────────────────────────────────────── diff --git a/tests/test_pool_node.cpp b/tests/test_pool_node.cpp index 779572b..a5b3f90 100644 --- a/tests/test_pool_node.cpp +++ b/tests/test_pool_node.cpp @@ -478,3 +478,49 @@ TEST_CASE("per-node and network overflow callbacks both fire independently", "[p REQUIRE(per_node.load() == 0); REQUIRE(network.load() == 0); } + +// Regression: NodeSnapshot's fields must line up with what nodes initialise. +// +// The snapshot is an aggregate that every node type fills positionally, and +// a8cfe73 appended queued/wake_pending/total_exec_ms to it in an order no call +// site used: each node supplies total_exec_ms as the element straight after +// queue_wait_ms, but the struct declared the two bools there. So the exec total +// landed in `queued`, `queued` landed in `wake_pending`, and `wake_pending` +// landed in total_exec_ms. The compiler said so (-Wnarrowing, bool to double, +// once per node instantiation) and the build carried on. +// +// It matters more than a cosmetic mix-up: these three fields exist to diagnose a +// wedge, and a wedged pipeline reported total_exec_ms as 0 or 1 and `queued` as +// "did this node ever run". Reading them would have pointed at the wrong node. +// +// Asserted against ema_exec_ms because that field is independently computed and +// was already correct: a true sum over several frames cannot be below the +// exponentially-weighted average of the same samples. +TEST_CASE("node snapshot fields line up with the values nodes supply", + "[pool_node][diagnostics]") { + auto pool = std::make_shared(1); + pool->start(); + + auto node = make_pool_node(pool, 64); + Channel out(64); + node.set_output_channel<0>(&out); + node.start(); + + for (int i = 0; i < 8; ++i) node.input_channel<0>().push(i); + std::this_thread::sleep_for(std::chrono::milliseconds(100)); + + auto snap = node.node_snapshot("n", 1.0); + node.stop(); + pool->stop(); + + INFO("frames=" << snap.frames_processed + << " ema=" << snap.ema_exec_ms + << " total=" << snap.total_exec_ms); + REQUIRE(snap.frames_processed == 8); + // The mis-ordered aggregate put wake_pending here, so this was 0.0 or 1.0. + CHECK(snap.total_exec_ms >= snap.ema_exec_ms); + // ...and the exec total here, which is non-zero, so `queued` read true for + // any node that had ever run — including one asleep with nothing to do. + CHECK_FALSE(snap.queued); + CHECK_FALSE(snap.wake_pending); +} From c73edffe5ce2262ca12294248e60a00fafb397ed Mon Sep 17 00:00:00 2001 From: Duncan Tourolle Date: Wed, 5 Aug 2026 12:51:21 +0200 Subject: [PATCH 19/42] chore: delete the unreachable duplicate of the parked-retry block MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit PoolNode::fire_once carried the pending_ retry block twice, verbatim. The first copy returns on every path through it — parked, drained, or not pending at all — so the second was dead code from the moment it appeared. PoolObjectNode, which is otherwise a line-for-line twin of PoolNode, has it once. No behaviour change; the deleted 25 lines were unreachable. Worth doing before the fixes queued behind it, each of which has to be applied once per copy of this function. The duplication is a symptom: PoolNode and PoolObjectNode are ~400 lines of near-identical code maintained by parallel edit, and a block getting pasted twice into one of them is exactly the failure that arrangement invites. Factoring the shared body out is a larger change and wants its own review. --- include/kpn/pool_node.hpp | 29 ----------------------------- 1 file changed, 29 deletions(-) diff --git a/include/kpn/pool_node.hpp b/include/kpn/pool_node.hpp index e9d8d1a..a821e6f 100644 --- a/include/kpn/pool_node.hpp +++ b/include/kpn/pool_node.hpp @@ -446,35 +446,6 @@ private: } } - // Parked from a previous firing: retry that value before touching the - // inputs. Returning here releases the worker — the channel's space - // callback re-submits this node once the consumer drains a slot. - if constexpr (!std::is_void_v) { - if (pending_) { - push_outputs(std::move(*pending_), std::make_index_sequence{}); - release_and_recheck(); - if (pending_) { - // Close the lost-wakeup race: a space_callback that fired - // between the failed push and clearing queued_ was - // swallowed, and nothing else will wake this node. Re-check - // now that the flag is down. - if (outputs_have_space(std::make_index_sequence{})) - try_submit(0.5f); - return; // parked - } - // Drained: resume normal firing, resubmitting exactly the way - // the normal tail below does. An unconditional try_submit here - // would fire a node whose inputs are empty, and pop_inputs - // reports an empty channel as ChannelClosedError — which this - // node treats as "upstream finished" and self-stops on. That - // is a live node killing itself purely because it was woken by - // *output* space rather than by input arrival. - if constexpr (input_count == 0) try_submit(0.5f); - else on_input_ready(); - return; - } - } - // Woken by output space rather than by input arrival, with nothing // parked left to flush: there is no work to do. Falling through would // read an empty channel, and pop_one reports empty as From 6a4f45f1113a039c244cd0fe6c1d8e04816e78b8 Mon Sep 17 00:00:00 2001 From: Duncan Tourolle Date: Wed, 5 Aug 2026 12:59:39 +0200 Subject: [PATCH 20/42] fix: a filter or router must not drop on a full output MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit RouterNode and FilterNode were the last nodes on a data path still using the throwing push() and swallowing the result: try { out_ch_->push(val); } catch (const ChannelOverflowError&) {} 6595e6e made node outputs lossless, 28e0667 stopped them parking a worker, a8cfe73 did the same for FanoutNode. These two were in none of them. For ordinary values that is the familiar silent-loss problem: a dropped item does not degrade a downstream result, it silently changes one, and the consumer cannot tell it happened. For a sentinel it is a hang. EOF is what tells every downstream node to shut down and there is nothing after it to retry, so a filter that passes EOF by predicate but drops it by backpressure produces a pipeline that never terminates. scene-actor-extraction's decimator is exactly that shape — `if (f.eof) return true;` in the predicate, feeding a chain whose slowest node is an ONNX embedder, so the output is reliably full when EOF arrives. Everything downstream then waits forever for a token that was discarded, and the run has to be killed. Both now route sentinels out-of-band via push_sentinel, which consumes no ring capacity and cannot overflow, and retry ordinary values until taken. Like FanoutNode and unlike a pool node, these own a private thread, so waiting costs no scheduler worker and needs no space-callback park; stop_flag_ is rechecked every pass so teardown cannot hang on a full output. Time spent parked is charged to blocked rather than exec, so a held-up node does not report as busy. An out-of-range router selector still drops by design — that item was routed nowhere, which is not the same as lost. is_sentinel_value moves from pool_node.hpp to traits.hpp. Every node type that forwards a value needs it; these two not having it is the bug. Verified in both directions. On c73edff the new case delivers 6 of 40 values and never sets saw_eof; here it delivers 40 and terminates. EOF is emitted exactly once, as a real source does — a test source that re-offered it would mask the bug, since a later attempt could find the channel drained. Note for downstream: the decimator is now a backpressure point rather than a relief valve, so the source throttles to the face branch instead of quietly thinning it. That is the intended behaviour, but it changes the shape of a loaded run and is worth a benchmark comparison on a known clip. --- include/kpn/branch.hpp | 86 +++++++++++++++++++++--- include/kpn/pool_node.hpp | 33 +--------- include/kpn/traits.hpp | 37 +++++++++++ tests/test_backpressure_deadlock.cpp | 97 ++++++++++++++++++++++++++++ 4 files changed, 214 insertions(+), 39 deletions(-) diff --git a/include/kpn/branch.hpp b/include/kpn/branch.hpp index fcd53ce..676a1f8 100644 --- a/include/kpn/branch.hpp +++ b/include/kpn/branch.hpp @@ -7,12 +7,70 @@ #include #include +#include #include #include #include namespace kpn { +// ── Lossless single-output delivery ─────────────────────────────────────────── +// +// Shared by RouterNode and FilterNode, which each deliver a value to exactly one +// channel. Both previously did +// +// try { ch->push(val); } catch (const ChannelOverflowError&) {} +// +// which discards the value whenever the consumer is behind. 6595e6e made node +// outputs lossless, 28e0667 stopped them parking a worker, and a8cfe73 did the +// same for FanoutNode — these two were in none of them, and were the last +// remaining users of the throwing push() on a data path. +// +// A dropped item does not degrade a downstream result, it silently changes one. +// Worse, a dropped *sentinel* wedges the pipeline outright: EOF is what tells +// every downstream node to shut down, and there is nothing after it to retry. +// A filter that passes EOF by predicate but drops it by backpressure is a +// pipeline that never terminates. +// +// So sentinels go out-of-band via push_sentinel (a dedicated slot that consumes +// no ring capacity and cannot overflow), and everything else is retried until +// taken. Like FanoutNode and unlike a pool node, these own a private thread, so +// waiting here costs no scheduler worker and needs no space-callback park. +// stop_flag_ is rechecked every pass so teardown cannot hang on a full output. +// +// `parked` receives the time spent waiting, which the caller charges to blocked +// rather than exec — a parked node is idle, and charging it to exec reports the +// node as busy exactly when it is the one being held up. +// +// Returns false if stopped with the value undelivered. +template +bool deliver_one(Channel* ch, T& val, const std::atomic& stop_flag, + duration_t& parked) { + if (is_sentinel_value(val)) { + ch->push_sentinel(std::move(val)); + return true; + } + const auto park_from = clock_t::now(); + for (;;) { + if (ch->try_push(val)) { + parked = duration_t(clock_t::now() - park_from); + return true; + } + if (stop_flag.load(std::memory_order_relaxed)) { + // Teardown with work in hand. One last throwing push, purely so the + // channel's own stats record the loss (drop if it is disabled, + // overflow if it is merely full). The point of the lossless path is + // that a loss is never invisible, and a silent return here would + // reintroduce exactly the hole this function exists to close. + try { ch->push(std::move(val)); } + catch (const ChannelOverflowError&) {} + parked = duration_t(clock_t::now() - park_from); + return false; + } + std::this_thread::sleep_for(std::chrono::microseconds(50)); + } +} + // ── RouterNode ──────────────────────────────────────────────────────────────── // // Reads one item and pushes it to exactly one of N output channels, chosen by @@ -126,15 +184,20 @@ private: auto t1 = clock_t::now(); auto cpu0 = NodeStats::cpu_now(); + // An out-of-range selector still drops by design (documented on + // the class): the item was routed nowhere, not lost to a full + // channel. Only the latter is what deliver_one exists to stop. std::size_t idx = selector_(val); - if (idx < N && out_channels_[idx]) { - try { out_channels_[idx]->push(val); } - catch (const ChannelOverflowError&) {} - } + duration_t parked{0}; + bool delivered = true; + if (idx < N && out_channels_[idx]) + delivered = deliver_one(out_channels_[idx], val, stop_flag_, parked); auto cpu1 = NodeStats::cpu_now(); auto t2 = clock_t::now(); - stats_.record_exec(duration_t(t2 - t1), duration_t(t1 - t0), cpu0, cpu1); + stats_.record_exec(duration_t(t2 - t1) - parked, + duration_t(t1 - t0) + parked, cpu0, cpu1); + if (!delivered) break; } catch (const ChannelClosedError&) { break; } @@ -261,12 +324,19 @@ private: auto t1 = clock_t::now(); auto cpu0 = NodeStats::cpu_now(); + // A value the predicate rejects is dropped by design and is not + // counted as a processed frame. One it accepts is now delivered + // losslessly — including a sentinel, which a filter typically + // passes unconditionally so downstream can shut down, and which + // the old throwing push discarded whenever the output was full. if (pred_(val) && out_ch_) { - try { out_ch_->push(val); } - catch (const ChannelOverflowError&) {} + duration_t parked{0}; + const bool delivered = deliver_one(out_ch_, val, stop_flag_, parked); auto cpu1 = NodeStats::cpu_now(); auto t2 = clock_t::now(); - stats_.record_exec(duration_t(t2 - t1), duration_t(t1 - t0), cpu0, cpu1); + stats_.record_exec(duration_t(t2 - t1) - parked, + duration_t(t1 - t0) + parked, cpu0, cpu1); + if (!delivered) break; } } catch (const ChannelClosedError&) { break; diff --git a/include/kpn/pool_node.hpp b/include/kpn/pool_node.hpp index a821e6f..042a35d 100644 --- a/include/kpn/pool_node.hpp +++ b/include/kpn/pool_node.hpp @@ -23,37 +23,8 @@ namespace kpn { -// ── Sentinel detection ──────────────────────────────────────────────────────── -// A value is a "sentinel" (must-deliver control token, e.g. EOF) if its type -// carries a bool-convertible eof flag — either directly (`v.eof`, as on a raw -// source Frame) or nested one level under a `.source` member (`v.source.eof`, -// as on the pipeline's SceneFrame/…/MatchedSceneFrame message types, which wrap -// the originating Frame). Sentinels are delivered losslessly and non-blockingly -// via Channel::push_sentinel() instead of the throwing push(), so backpressure -// can never drop the token that unblocks downstream teardown. -// -// Types with neither shape are never treated as sentinels — both traits are -// SFINAE-safe and the runtime check compiles away to `false` for them, so this -// stays a no-op for pipelines that don't use an eof convention. -template -struct has_eof_field : std::false_type {}; -template -struct has_eof_field(std::declval().eof))>> - : std::true_type {}; - -template -struct has_source_eof_field : std::false_type {}; -template -struct has_source_eof_field(std::declval().source.eof))>> - : std::true_type {}; - -template -constexpr bool is_sentinel_value(const T& v) { - if constexpr (has_eof_field::value) return static_cast(v.eof); - else if constexpr (has_source_eof_field::value) return static_cast(v.source.eof); - else return false; -} +// Sentinel detection (has_eof_field / is_sentinel_value) lives in traits.hpp — +// every node type that forwards values needs it, not just pool-scheduled ones. // ── PoolNode ────────────────────────────────────────────────────────────────── // diff --git a/include/kpn/traits.hpp b/include/kpn/traits.hpp index 6381822..e81983e 100644 --- a/include/kpn/traits.hpp +++ b/include/kpn/traits.hpp @@ -97,4 +97,41 @@ struct repeat_tuple> { template using repeat_tuple_t = typename repeat_tuple::type; +// ── Sentinel detection ──────────────────────────────────────────────────────── +// A value is a "sentinel" (must-deliver control token, e.g. EOF) if its type +// carries a bool-convertible eof flag — either directly (`v.eof`, as on a raw +// source Frame) or nested one level under a `.source` member (`v.source.eof`, +// as on message types that wrap the originating Frame). Sentinels are delivered +// losslessly and non-blockingly via Channel::push_sentinel() instead of the +// throwing push(), so backpressure can never drop the token that unblocks +// downstream teardown. +// +// Types with neither shape are never treated as sentinels — both traits are +// SFINAE-safe and the runtime check compiles away to `false` for them, so this +// stays a no-op for pipelines that don't use an eof convention. +// +// Lives here rather than in pool_node.hpp because every node type that forwards +// values needs it, not just the pool-scheduled ones. FilterNode and RouterNode +// not having it is what let an EOF token be dropped on a full output. + +template +struct has_eof_field : std::false_type {}; +template +struct has_eof_field(std::declval().eof))>> + : std::true_type {}; + +template +struct has_source_eof_field : std::false_type {}; +template +struct has_source_eof_field(std::declval().source.eof))>> + : std::true_type {}; + +template +constexpr bool is_sentinel_value(const T& v) { + if constexpr (has_eof_field::value) return static_cast(v.eof); + else if constexpr (has_source_eof_field::value) return static_cast(v.source.eof); + else return false; +} + } // namespace kpn diff --git a/tests/test_backpressure_deadlock.cpp b/tests/test_backpressure_deadlock.cpp index c1ee551..f6909d0 100644 --- a/tests/test_backpressure_deadlock.cpp +++ b/tests/test_backpressure_deadlock.cpp @@ -322,3 +322,100 @@ TEST_CASE("a fanout absorbs an unequal pair by slowing, not dropping", // the difference is the loss. CHECK(fast_seen - slow_seen < 200); } + +// Regression: a filter must not drop an EOF sentinel into a full output. +// +// RouterNode and FilterNode were the last nodes on a data path still using the +// throwing push() and swallowing the result: +// +// try { out_ch_->push(val); } catch (const ChannelOverflowError&) {} +// +// 6595e6e made node outputs lossless, 28e0667 stopped them parking a worker, +// a8cfe73 did the same for FanoutNode. These two were in none of them. +// +// For ordinary values that is the familiar silent-loss problem. For a sentinel +// it is a hang. EOF is what tells every downstream node to shut down, and +// nothing comes after it to retry — so a filter that passes EOF by predicate +// but drops it by backpressure produces a pipeline that never terminates. The +// scene-actor-extraction decimator is exactly this shape: `if (f.eof) return +// true;` in the predicate, feeding a chain whose slowest node is an ONNX +// embedder, so the output is reliably full at the moment EOF arrives. +// +// The test forces that state rather than racing for it: the sink is slow enough +// that the filter's output channel is saturated for the whole run, so EOF meets +// a full ring with certainty. +// +// Both assertions are needed. `saw_eof` alone would pass on an implementation +// that dropped every ordinary value and delivered only the sentinel; `count` +// alone would pass on the broken one, which delivers plenty of values and loses +// only the token that matters. +namespace { + +struct EofFrame { + int seq{0}; + bool eof{false}; +}; + +// EOF is emitted exactly once, as a real source does. Everything after it is a +// filler frame the predicate rejects, which keeps the node alive without +// re-offering the sentinel — a source that retried EOF would mask the bug, +// since a later attempt could find the channel drained. +struct EofSource { + static constexpr std::string_view label() { return "eof_source"; } + int n{0}; + int total{0}; + EofFrame operator()() { + if (n > total) { + std::this_thread::sleep_for(std::chrono::milliseconds(1)); + return {-1, false}; // filler: dropped by the predicate + } + EofFrame f{n, n == total}; + ++n; + return f; + } +}; + +struct EofSink { + static constexpr std::string_view label() { return "eof_sink"; } + std::atomic* count; + std::atomic* saw_eof; + void operator()(EofFrame f) { + std::this_thread::sleep_for(std::chrono::microseconds(200)); + if (f.eof) saw_eof->store(true, std::memory_order_release); + else count->fetch_add(1, std::memory_order_relaxed); + } +}; + +} // namespace + +TEST_CASE("a filter delivers EOF into a saturated output", "[backpressure][filter]") { + std::atomic count{0}; + std::atomic saw_eof{false}; + + EofSource src_fn{0, 40}; + EofSink sink_fn{&count, &saw_eof}; + + // Every real frame passes the predicate, so the only thing between source + // and sink is backpressure. Small channels keep the output saturated. + auto filt = kpn::make_filter( + [](const EofFrame& f) { return f.seq >= 0; }, 4); + + kpn::ObjectNode, kpn::out<"f">, "eof_source", 0> s(src_fn, 4); + kpn::ObjectNode, kpn::out<>, "eof_sink", 0> k(sink_fn, 4); + + auto net = kpn::make_network( + kpn::edge(s.output<"f">(), filt.input<0>()), + kpn::edge(filt.output<0>(), k.input<"f">()) + ); + net.start(); + + // Generous relative to 41 frames at 200 us, and this is a liveness test: + // the broken implementation never sets saw_eof no matter how long it runs. + for (int i = 0; i < 200 && !saw_eof.load(std::memory_order_acquire); ++i) + std::this_thread::sleep_for(std::chrono::milliseconds(10)); + net.stop(); + + INFO("values delivered: " << count.load() << " of 40"); + CHECK(saw_eof.load(std::memory_order_acquire)); + CHECK(count.load(std::memory_order_relaxed) == 40); +} From 5628447ea833e06eaf4ad116c14d1595f6183a69 Mon Sep 17 00:00:00 2001 From: Duncan Tourolle Date: Wed, 5 Aug 2026 13:10:41 +0200 Subject: [PATCH 21/42] fix: never self-move the parked output tuple MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit push_outputs ended with else pending_ = std::move(result); and the retry path calls it as push_outputs(std::move(*pending_), …), so on that path `result` is the parked tuple itself. The assignment was a self-move-assignment. std::tuple's is elementwise, and libstdc++'s std::vector does not guard against self-move: _M_move_assign swaps its data into a temporary, which is then destroyed. The vector ends up empty. So the first park was clean — the argument there is a local temporary — and the second erased the payload. The value was still delivered, still in order, still counted, just empty. Downstream cannot distinguish that from a frame on which the node genuinely found nothing, which is why it would never surface as an error: in scene-actor-extraction it reads as "no faces in this frame" and the run completes with a quietly wrong answer. Scope, stated precisely because I first got it wrong: this needs a node with *two or more* outputs. With one output the only thing that resubmits a parked node is that output's own space callback, which by definition fires when there is room, so the retry always succeeds and never reassigns. With two, output A draining resubmits the node while output B is still full — the retry skips A (already delivered, tracked in pending_done_) and fails on B, and that is the reassignment that eats B's payload. Every node in the scene-actor-extraction pipeline currently has exactly one output, and the fanout is a separate class that does not use pending_, so this is latent there rather than active. It is reachable by any multi-output node under backpressure, which the library supports and documents. Verified in both directions: on 6a4f45f the parked payload arrives with size 0; here it arrives intact. The test drives raw channels rather than consumer nodes so each step is forced rather than raced, and both channels are capacity 1 — Channel fires the space callback only on the full->not-full edge, so a roomy channel A would never resubmit the node and the retry would never happen at all. --- include/kpn/pool_node.hpp | 20 +++++++++- tests/test_pool_node.cpp | 78 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 96 insertions(+), 2 deletions(-) diff --git a/include/kpn/pool_node.hpp b/include/kpn/pool_node.hpp index 042a35d..7c9f73c 100644 --- a/include/kpn/pool_node.hpp +++ b/include/kpn/pool_node.hpp @@ -530,7 +530,15 @@ private: push_one_out(std::get(std::move(result))), all = all && pending_done_[Is]), ...); if (all) { pending_.reset(); pending_done_.fill(false); } - else pending_ = std::move(result); + // The retry path calls this as push_outputs(std::move(*pending_), …), so + // on that path `result` *is* the parked tuple. Assigning it to itself is + // a self-move-assignment, which for std::tuple is elementwise — and + // libstdc++'s std::vector does not guard against it: it swaps its data + // into a temporary and leaves the vector empty. A value that failed to + // push twice would therefore be delivered with its payload silently + // erased, which downstream reads as a legitimately empty result rather + // than as a loss. Only store when it is not already stored. + else if (!pending_ || &result != &*pending_) pending_ = std::move(result); } /// Returns false when the ring was full and the value was NOT taken; the @@ -1012,7 +1020,15 @@ private: push_one_out(std::get(std::move(result))), all = all && pending_done_[Is]), ...); if (all) { pending_.reset(); pending_done_.fill(false); } - else pending_ = std::move(result); + // The retry path calls this as push_outputs(std::move(*pending_), …), so + // on that path `result` *is* the parked tuple. Assigning it to itself is + // a self-move-assignment, which for std::tuple is elementwise — and + // libstdc++'s std::vector does not guard against it: it swaps its data + // into a temporary and leaves the vector empty. A value that failed to + // push twice would therefore be delivered with its payload silently + // erased, which downstream reads as a legitimately empty result rather + // than as a loss. Only store when it is not already stored. + else if (!pending_ || &result != &*pending_) pending_ = std::move(result); } /// Returns false when the ring was full and the value was NOT taken; the /// caller must keep it and retry after the channel signals space. diff --git a/tests/test_pool_node.cpp b/tests/test_pool_node.cpp index a5b3f90..8902bae 100644 --- a/tests/test_pool_node.cpp +++ b/tests/test_pool_node.cpp @@ -524,3 +524,81 @@ TEST_CASE("node snapshot fields line up with the values nodes supply", CHECK_FALSE(snap.queued); CHECK_FALSE(snap.wake_pending); } + +// Regression: a value parked twice must keep its payload. +// +// push_outputs ends with +// +// else pending_ = std::move(result); +// +// and the retry path calls it as push_outputs(std::move(*pending_), …), so on +// that path `result` is the parked tuple itself. The assignment was therefore a +// self-move-assignment. std::tuple's is elementwise, and libstdc++'s +// std::vector does not guard against self-move: it swaps its data into a +// temporary and leaves the vector empty. So the first park was clean (the +// argument is a local temporary) and the second erased the payload. +// +// The value was still delivered, still in order, still counted — just empty. +// Downstream cannot distinguish that from a frame on which the node genuinely +// found nothing, which is why it never surfaced as an error: in +// scene-actor-extraction it reads as "no faces in this frame" and the run +// completes with a quietly wrong answer. +// +// Reaching it needs *two* outputs. With one, the only thing that resubmits a +// parked node is that output's own space callback, which by definition fires +// when there is room — so the retry always succeeds and never reassigns. With +// two, output A draining resubmits the node while output B is still full: the +// retry skips A (already delivered, tracked in pending_done_) and fails on B, +// and that is the reassignment that eats B's payload. +// +// Driven through raw channels rather than consumer nodes so each step is +// forced rather than raced: B is pre-filled and stays full for exactly as long +// as the test wants it to. +namespace { + +struct TwoPayloads { + static constexpr std::string_view label() { return "two_payloads"; } + std::tuple, std::vector> operator()() { + return {std::vector(4, 1), std::vector(4, 2)}; + } +}; + +} // namespace + +TEST_CASE("a twice-parked value keeps its payload", "[pool_node][backpressure]") { + auto pool = std::make_shared(2); + pool->start(); + + TwoPayloads fn; + auto node = make_pool_node(fn, pool); + + // Both capacity 1. A must be *full* for its pop to signal space at all — + // Channel fires the space callback only on the full->not-full edge, so a + // roomy A would never resubmit the node and the retry would never happen. + Channel> out_a(1), out_b(1); + node.set_output_channel<0>(&out_a); + node.set_output_channel<1>(&out_b); + + // B is full before the node ever runs, so the very first firing parks. + out_b.push(std::vector(4, 99)); + + node.start(); + std::this_thread::sleep_for(std::chrono::milliseconds(50)); + + // Draining A resubmits the node while B is still full: this is the retry + // that reassigned the tuple to itself. + (void)out_a.pop(); + std::this_thread::sleep_for(std::chrono::milliseconds(50)); + + // Now let B through and collect what the node had been holding for it. + (void)out_b.pop(); // the pre-fill + std::this_thread::sleep_for(std::chrono::milliseconds(50)); + std::vector parked = out_b.pop(); // the value parked across two tries + + node.stop(); + pool->stop(); + + INFO("parked payload size " << parked.size()); + CHECK(parked.size() == 4); + if (parked.size() == 4) CHECK(parked[0] == 2); +} From f53af260a24f4e7a4f729247b59aca43e2343dd3 Mon Sep 17 00:00:00 2001 From: Duncan Tourolle Date: Wed, 5 Aug 2026 13:22:03 +0200 Subject: [PATCH 22/42] fix: make the submit gate a single atomic MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 9c5ce5f established "a node never sleeps with a wake outstanding" and implemented it as two independent atomics: queued_ for "a firing is in flight", wake_pending_ for "a wake arrived during one". Two variables cannot express that invariant, because the release side has to read and write both and a wake can land in between: producer (try_submit) worker (release_and_recheck) ------------------------ ---------------------------- CAS reads queued_ == true, fails queued_.store(false) wake_pending_.exchange(false) -> false wake_pending_.store(true) queued_ false, wake_pending_ true, nothing running and nothing scheduled — exactly the state the invariant forbids. This is not a memory-ordering subtlety; the interleaving holds under seq_cst. SubmitGate replaces both with one atomic over three states, so "idle" and "wake outstanding" are the same variable and no interleaving can produce both. A release that finds a recorded wake keeps the claim and hands it to the next firing, so the node is never momentarily idle while a submission for it is in flight. What this does not do is fix a reproducible hang. Every current call site follows release_and_recheck() with a level re-check — on_input_ready(), or outputs_have_space() on the parked path — which rediscovers the state a lost wake would have signalled. The bug is masked, and I could not write a node-level test that fails before and passes after; claiming otherwise would be dishonest. The masking is a property of the call sites, not the mechanism: any future early return that forgets its re-check reintroduces a silent hang, and the pipeline has already been round that loop twice (28e0667, then 9c5ce5f, each of which moved the stall rather than removing it). So the tests are structural. The state machine is pinned by contract tests, and the defect it replaces is pinned by demonstration: LegacyGate in the test file is the old protocol with a seam between the failed CAS and the wake record, which makes the loss deterministic rather than something to wait for. It also keeps the defect on record now that the code implementing it is gone. Also ignores build-*/ so a sanitizer build tree cannot be committed by accident, which this commit did on its first attempt. --- .gitignore | 2 + include/kpn/pool_node.hpp | 109 +++++++++++------------ include/kpn/submit_gate.hpp | 101 +++++++++++++++++++++ tests/CMakeLists.txt | 1 + tests/test_submit_gate.cpp | 173 ++++++++++++++++++++++++++++++++++++ 5 files changed, 328 insertions(+), 58 deletions(-) create mode 100644 include/kpn/submit_gate.hpp create mode 100644 tests/test_submit_gate.cpp diff --git a/.gitignore b/.gitignore index 483232f..d3fb3e0 100644 --- a/.gitignore +++ b/.gitignore @@ -28,3 +28,5 @@ Thumbs.db # Claude Code local settings .claude/settings.local.json include/kpn/ort_cache/ +build-tsan/ +build-*/ diff --git a/include/kpn/pool_node.hpp b/include/kpn/pool_node.hpp index 7c9f73c..bff47b3 100644 --- a/include/kpn/pool_node.hpp +++ b/include/kpn/pool_node.hpp @@ -5,6 +5,7 @@ #include "inode.hpp" #include "port.hpp" #include "scheduler.hpp" +#include "submit_gate.hpp" #include "traits.hpp" #include @@ -31,7 +32,7 @@ namespace kpn { // Reactive alternative to Node<>. Instead of owning a blocked thread, the node // is submitted to a shared IScheduler whenever all its input channels become // non-empty. A single fire_once() call pops all inputs, executes the function, -// and pushes outputs. At most one fire_once() runs at a time (queued_ flag). +// and pushes outputs. At most one fire_once() runs at a time (see SubmitGate). // // Source nodes (input_count == 0) submit themselves immediately on start() and // resubmit after each fire_once(). @@ -83,7 +84,7 @@ public: void start() override { enable_inputs(std::make_index_sequence{}); stop_flag_.store(false, std::memory_order_relaxed); - queued_.store(false, std::memory_order_relaxed); + gate_.force_idle(); register_callbacks(std::make_index_sequence{}); if constexpr (input_count == 0) try_submit(0.5f); @@ -145,8 +146,8 @@ public: total_ms > 0 ? 100.0 * exec_ms / total_ms : 0.0, qwait_ms, stats_.total_exec_us.load(std::memory_order_relaxed) / 1000.0, - queued_.load(std::memory_order_relaxed), - wake_pending_.load(std::memory_order_relaxed), + gate_.queued(), + gate_.wake_pending(), }; } @@ -258,7 +259,7 @@ private: stats_.exec_start_us.store(0, std::memory_order_relaxed); // Plain store, not release_and_recheck(): this node is stopping, and // honouring a pending wake here would resubmit a dead node. - queued_.store(false, std::memory_order_release); + gate_.force_idle(); stop_flag_.store(true, std::memory_order_relaxed); } @@ -346,39 +347,35 @@ private: : 0.5f), ...); } - /// Submit unless already queued. A wake that arrives while this node is - /// queued or running is *recorded*, never dropped. + /// Submit unless a firing is already in flight. A wake that arrives while + /// one is is *recorded* against it, never dropped. /// /// Wakes are edge-triggered: a channel fires its space callback on the - /// transition, once. If that lands while queued_ is up, the CAS below fails - /// and — before wake_pending_ — the wake was gone. A node could then park a + /// transition, once. A dropped one never returns, so a node could park a /// value, release its worker, and sleep forever holding output its consumer /// was waiting for, with every worker idle in cond_wait and nothing left to - /// re-trigger it. Recording the drop turns the signal level-triggered: the - /// invariant is that a node never sleeps with a wake outstanding, enforced - /// by release_and_recheck() at every point that releases the node. + /// re-trigger it. SubmitGate makes "idle" and "wake outstanding" the same + /// variable, so the two cannot both be true — see submit_gate.hpp. void try_submit(float priority) { - bool expected = false; - if (queued_.compare_exchange_strong(expected, true, std::memory_order_acq_rel)) + if (gate_.claim()) scheduler_->submit([this] { fire_once(); }, priority); - else - wake_pending_.store(true, std::memory_order_release); } - /// Clear queued_, then honour any wake that was dropped while it was up. - /// Every path that finishes or parks a firing must release the node through - /// here rather than storing queued_ directly. + /// End this firing, honouring any wake recorded during it. Every path that + /// finishes or parks a firing must release the node through here rather + /// than touching the gate directly. When a wake was recorded the gate stays + /// claimed and is handed to the next firing, so the node is never + /// momentarily idle with work outstanding. void release_and_recheck(float priority = 0.5f) { - queued_.store(false, std::memory_order_release); - if (wake_pending_.exchange(false, std::memory_order_acq_rel)) - try_submit(priority); + if (gate_.release()) + scheduler_->submit([this] { fire_once(); }, priority); } // ── Execution ───────────────────────────────────────────────────────────── void fire_once() { if (stop_flag_.load(std::memory_order_relaxed)) { - queued_.store(false, std::memory_order_release); + gate_.force_idle(); return; } @@ -397,7 +394,7 @@ private: release_and_recheck(); if (pending_) { // Close the lost-wakeup race: a space_callback that fired - // between the failed push and clearing queued_ was + // between the failed push and releasing the gate was // swallowed, and nothing else will wake this node. Re-check // now that the flag is down. if (outputs_have_space(std::make_index_sequence{})) @@ -475,8 +472,8 @@ private: // Parked by the push above. Same situation as the retry path at the top // of fire_once — and the same lost-wakeup race, which that path closes - // and this one did not. A space callback that fired while queued_ was - // still up got swallowed by try_submit's CAS, and the resubmit below + // and this one did not. A space callback that fired while the gate was + // still claimed is recorded there, and the resubmit below // cannot cover it: this firing consumed its input, so inputs are empty // and on_input_ready() will not resubmit. The node would then hold its // value forever while its consumer waits for exactly that value and its @@ -499,7 +496,7 @@ private: } // Pop all inputs — safe because we're the sole consumer and fire_once - // is guarded by queued_ (only one fire_once runs at a time). + // is guarded by the submit gate (only one fire_once runs at a time). template args_tuple pop_inputs(std::index_sequence) { return {pop_one()...}; @@ -584,9 +581,9 @@ private: input_channels_t input_channels_; output_channels_t output_channels_{}; std::atomic stop_flag_{true}; - std::atomic queued_{false}; - /// A wake that arrived while queued_ was up. See try_submit. - std::atomic wake_pending_{false}; + /// Serialises firings and records wakes that arrive during one. See + /// submit_gate.hpp for why this cannot be two separate flags. + SubmitGate gate_; /// The hidden one-slot output buffer (see push_outputs). Holding the value /// here is what lets a node stop running without dropping it or occupying a @@ -652,7 +649,7 @@ public: void start() override { enable_inputs(std::make_index_sequence{}); stop_flag_.store(false, std::memory_order_relaxed); - queued_.store(false, std::memory_order_relaxed); + gate_.force_idle(); register_callbacks(std::make_index_sequence{}); if constexpr (input_count == 0) try_submit(0.5f); @@ -694,8 +691,8 @@ public: total_ms > 0 ? 100.0 * exec_ms / total_ms : 0.0, qwait_ms, stats_.total_exec_us.load(std::memory_order_relaxed) / 1000.0, - queued_.load(std::memory_order_relaxed), - wake_pending_.load(std::memory_order_relaxed), + gate_.queued(), + gate_.wake_pending(), }; } @@ -774,7 +771,7 @@ private: stats_.exec_start_us.store(0, std::memory_order_relaxed); // Plain store, not release_and_recheck(): this node is stopping, and // honouring a pending wake here would resubmit a dead node. - queued_.store(false, std::memory_order_release); + gate_.force_idle(); stop_flag_.store(true, std::memory_order_relaxed); } @@ -855,37 +852,33 @@ private: : 0.5f), ...); } - /// Submit unless already queued. A wake that arrives while this node is - /// queued or running is *recorded*, never dropped. + /// Submit unless a firing is already in flight. A wake that arrives while + /// one is is *recorded* against it, never dropped. /// /// Wakes are edge-triggered: a channel fires its space callback on the - /// transition, once. If that lands while queued_ is up, the CAS below fails - /// and — before wake_pending_ — the wake was gone. A node could then park a + /// transition, once. A dropped one never returns, so a node could park a /// value, release its worker, and sleep forever holding output its consumer /// was waiting for, with every worker idle in cond_wait and nothing left to - /// re-trigger it. Recording the drop turns the signal level-triggered: the - /// invariant is that a node never sleeps with a wake outstanding, enforced - /// by release_and_recheck() at every point that releases the node. + /// re-trigger it. SubmitGate makes "idle" and "wake outstanding" the same + /// variable, so the two cannot both be true — see submit_gate.hpp. void try_submit(float priority) { - bool expected = false; - if (queued_.compare_exchange_strong(expected, true, std::memory_order_acq_rel)) + if (gate_.claim()) scheduler_->submit([this] { fire_once(); }, priority); - else - wake_pending_.store(true, std::memory_order_release); } - /// Clear queued_, then honour any wake that was dropped while it was up. - /// Every path that finishes or parks a firing must release the node through - /// here rather than storing queued_ directly. + /// End this firing, honouring any wake recorded during it. Every path that + /// finishes or parks a firing must release the node through here rather + /// than touching the gate directly. When a wake was recorded the gate stays + /// claimed and is handed to the next firing, so the node is never + /// momentarily idle with work outstanding. void release_and_recheck(float priority = 0.5f) { - queued_.store(false, std::memory_order_release); - if (wake_pending_.exchange(false, std::memory_order_acq_rel)) - try_submit(priority); + if (gate_.release()) + scheduler_->submit([this] { fire_once(); }, priority); } void fire_once() { if (stop_flag_.load(std::memory_order_relaxed)) { - queued_.store(false, std::memory_order_release); + gate_.force_idle(); return; } auto t0 = clock_t::now(); @@ -902,7 +895,7 @@ private: release_and_recheck(); if (pending_) { // Close the lost-wakeup race: a space_callback that fired - // between the failed push and clearing queued_ was + // between the failed push and releasing the gate was // swallowed, and nothing else will wake this node. Re-check // now that the flag is down. if (outputs_have_space(std::make_index_sequence{})) @@ -974,8 +967,8 @@ private: // Parked by the push above. Same situation as the retry path at the top // of fire_once — and the same lost-wakeup race, which that path closes - // and this one did not. A space callback that fired while queued_ was - // still up got swallowed by try_submit's CAS, and the resubmit below + // and this one did not. A space callback that fired while the gate was + // still claimed is recorded there, and the resubmit below // cannot cover it: this firing consumed its input, so inputs are empty // and on_input_ready() will not resubmit. The node would then hold its // value forever while its consumer waits for exactly that value and its @@ -1054,9 +1047,9 @@ private: input_channels_t input_channels_; output_channels_t output_channels_{}; std::atomic stop_flag_{true}; - std::atomic queued_{false}; - /// A wake that arrived while queued_ was up. See try_submit. - std::atomic wake_pending_{false}; + /// Serialises firings and records wakes that arrive during one. See + /// submit_gate.hpp for why this cannot be two separate flags. + SubmitGate gate_; /// The hidden one-slot output buffer (see push_outputs). Holding the value /// here is what lets a node stop running without dropping it or occupying a diff --git a/include/kpn/submit_gate.hpp b/include/kpn/submit_gate.hpp new file mode 100644 index 0000000..85cbce8 --- /dev/null +++ b/include/kpn/submit_gate.hpp @@ -0,0 +1,101 @@ +#pragma once +#include + +namespace kpn { + +// ── SubmitGate ──────────────────────────────────────────────────────────────── +// +// Decides, for one node, whether a wake must turn into a scheduler submission. +// Exactly one firing of a node may be in flight at a time, and a wake that +// arrives while one is already in flight must not be lost — it has to be +// honoured when that firing finishes, or the node sleeps holding work. +// +// 9c5ce5f wrote this as two independent atomics: queued_ said a firing was in +// flight, wake_pending_ recorded a wake that arrived during one. That cannot be +// made correct, because the release side has to read and write both, and a wake +// can land between the two operations: +// +// producer (try_submit) worker (release_and_recheck) +// ------------------------ ---------------------------- +// CAS reads queued_ == true, fails +// queued_.store(false) +// wake_pending_.exchange(false) -> false +// wake_pending_.store(true) +// +// End state: queued_ false, wake_pending_ true, nothing running and nothing +// scheduled. The node sleeps with a wake outstanding, which is precisely the +// invariant that commit set out to establish. It is not a memory-ordering +// subtlety — the interleaving above holds under seq_cst. +// +// It survived because every caller happened to follow release_and_recheck() +// with a level re-check (on_input_ready(), or outputs_have_space() on the +// parked path), which rediscovers the state a lost wake would have signalled. +// That is a property of the call sites, not of the mechanism, and any new early +// return that forgets the re-check turns it back into a hang. +// +// One atomic with three states makes the race unrepresentable: "idle" and "wake +// outstanding" are the same variable, so no interleaving can produce both. +// +// Idle nothing in flight +// Queued a firing is in flight or queued; no wake since it was claimed +// QueuedWake a firing is in flight or queued, and a wake arrived meanwhile +// +class SubmitGate { +public: + /// Register a wake. Returns true when the caller must submit the node; + /// false when a firing is already in flight and the wake has been recorded + /// against it instead. + bool claim() noexcept { + int cur = state_.load(std::memory_order_acquire); + for (;;) { + if (cur == kIdle) { + if (state_.compare_exchange_weak(cur, kQueued, + std::memory_order_acq_rel, std::memory_order_acquire)) + return true; + } else if (cur == kQueued) { + if (state_.compare_exchange_weak(cur, kQueuedWake, + std::memory_order_acq_rel, std::memory_order_acquire)) + return false; + } else { + return false; // a wake is already recorded + } + } + } + + /// End the in-flight firing. Returns true when a wake arrived during it and + /// the caller must submit again — in which case the gate stays claimed, so + /// the node is handed straight from one firing to the next and is never + /// momentarily idle with work outstanding. Returns false when the node is + /// now idle. + bool release() noexcept { + int cur = state_.load(std::memory_order_acquire); + for (;;) { + if (cur == kQueuedWake) { + if (state_.compare_exchange_weak(cur, kQueued, + std::memory_order_acq_rel, std::memory_order_acquire)) + return true; + } else { + // kQueued, or kIdle if a stop already forced the gate down. + if (state_.compare_exchange_weak(cur, kIdle, + std::memory_order_acq_rel, std::memory_order_acquire)) + return false; + } + } + } + + /// Drop the claim and any recorded wake. For stop paths only: honouring a + /// wake there would resubmit a dead node. + void force_idle() noexcept { state_.store(kIdle, std::memory_order_release); } + + bool queued() const noexcept { return state_.load(std::memory_order_relaxed) != kIdle; } + bool wake_pending() const noexcept { return state_.load(std::memory_order_relaxed) == kQueuedWake; } + +private: + static constexpr int kIdle = 0; + static constexpr int kQueued = 1; + static constexpr int kQueuedWake = 2; + + std::atomic state_{kIdle}; +}; + +} // namespace kpn diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 9e43788..f567afc 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -36,6 +36,7 @@ add_executable(kpn_tests test_pool_node.cpp test_backpressure_deadlock.cpp test_scheduler.cpp + test_submit_gate.cpp ) target_link_libraries(kpn_tests PRIVATE diff --git a/tests/test_submit_gate.cpp b/tests/test_submit_gate.cpp new file mode 100644 index 0000000..b84ac90 --- /dev/null +++ b/tests/test_submit_gate.cpp @@ -0,0 +1,173 @@ +// Regression: a node must never end up idle with a wake outstanding. +// +// 9c5ce5f established that invariant and implemented it as two independent +// atomics — queued_ for "a firing is in flight", wake_pending_ for "a wake +// arrived during one". Two variables cannot express it, because the release +// side has to read and write both and a wake can land in between: +// +// producer (try_submit) worker (release_and_recheck) +// ------------------------ ---------------------------- +// CAS reads queued_ == true, fails +// queued_.store(false) +// wake_pending_.exchange(false) -> false +// wake_pending_.store(true) +// +// queued_ false, wake_pending_ true, nothing running and nothing scheduled. +// Not a memory-ordering subtlety: the interleaving holds under seq_cst. +// +// LegacyGate below is that protocol verbatim, with a hook between the failed +// CAS and the wake_pending_ store so the interleaving can be forced rather than +// waited for. That makes the loss deterministic and the test non-flaky, and it +// keeps the defect on record now that the code implementing it is gone. +// +// A note on what is NOT tested here, because it would be misleading to imply +// otherwise: there is no black-box, node-level test that fails before this fix +// and passes after. Every call site of release_and_recheck() happens to follow +// it with a level re-check — on_input_ready(), or outputs_have_space() on the +// parked path — which rediscovers the state a lost wake would have signalled. +// That masking is a property of the call sites, not of the mechanism, and the +// point of the fix is that a future early return that forgets the re-check no +// longer reintroduces a hang. The value is structural, so the tests are +// structural: the state machine is pinned by contract, and the defect it +// replaces is pinned by demonstration. +#include +#include + +#include +#include +#include + +using namespace kpn; + +namespace { + +// The pre-fix protocol, with a seam at the point where the race lives. +class LegacyGate { +public: + std::function before_recording_wake; + + bool claim() noexcept { + bool expected = false; + if (queued_.compare_exchange_strong(expected, true, std::memory_order_acq_rel)) + return true; + if (before_recording_wake) before_recording_wake(); + wake_pending_.store(true, std::memory_order_release); + return false; + } + bool release() noexcept { + queued_.store(false, std::memory_order_release); + if (wake_pending_.exchange(false, std::memory_order_acq_rel)) { + bool expected = false; + if (queued_.compare_exchange_strong(expected, true, std::memory_order_acq_rel)) + return true; + } + return false; + } + bool queued() const noexcept { return queued_.load(std::memory_order_relaxed); } + bool wake_pending() const noexcept { return wake_pending_.load(std::memory_order_relaxed); } + +private: + std::atomic queued_{false}; + std::atomic wake_pending_{false}; +}; + +} // namespace + +TEST_CASE("the two-atomic gate loses a wake, deterministically", "[submit_gate]") { + LegacyGate gate; + bool resubmitted = true; + + REQUIRE(gate.claim()); // a firing is now in flight + + // Force the interleaving: the firing completes in the window between the + // second wake's failed CAS and its record of that wake. + gate.before_recording_wake = [&] { resubmitted = gate.release(); }; + + const bool submitted = gate.claim(); + + // The wake was neither submitted by the producer nor honoured by the + // release. Nothing is scheduled, and nothing else will re-trigger it. + CHECK_FALSE(submitted); + CHECK_FALSE(resubmitted); + CHECK_FALSE(gate.queued()); + CHECK(gate.wake_pending()); // recorded, and never to be consumed +} + +TEST_CASE("submit gate: a wake during a firing is honoured", "[submit_gate]") { + SubmitGate gate; + + REQUIRE(gate.claim()); // idle -> queued, caller submits + REQUIRE(gate.queued()); + REQUIRE_FALSE(gate.wake_pending()); + + REQUIRE_FALSE(gate.claim()); // second wake is recorded, not submitted + REQUIRE(gate.wake_pending()); + + REQUIRE(gate.release()); // and honoured when the firing ends + // The gate stays claimed across the handover, so the node is never + // momentarily idle while a submission for it is in flight. This is the + // state the legacy gate could not represent. + REQUIRE(gate.queued()); + REQUIRE_FALSE(gate.wake_pending()); + + REQUIRE_FALSE(gate.release()); // no further wake: now idle + REQUIRE_FALSE(gate.queued()); +} + +TEST_CASE("submit gate: repeated wakes collapse to one resubmission", "[submit_gate]") { + // Collapsing is deliberate. A firing consumes one item and its caller then + // re-checks the input level, so the gate only has to guarantee that at + // least one more firing follows a wake, not one per wake. + SubmitGate gate; + REQUIRE(gate.claim()); + for (int i = 0; i < 10; ++i) REQUIRE_FALSE(gate.claim()); + REQUIRE(gate.release()); + REQUIRE_FALSE(gate.release()); +} + +TEST_CASE("submit gate: force_idle drops a recorded wake", "[submit_gate]") { + // Stop paths use this deliberately — honouring a wake there would resubmit + // a node that has already been told to stop. + SubmitGate gate; + REQUIRE(gate.claim()); + REQUIRE_FALSE(gate.claim()); + REQUIRE(gate.wake_pending()); + + gate.force_idle(); + REQUIRE_FALSE(gate.queued()); + REQUIRE_FALSE(gate.wake_pending()); + REQUIRE(gate.claim()); // and the gate is reusable afterwards +} + +TEST_CASE("submit gate: concurrent claim and release stay consistent", "[submit_gate]") { + // Not a lost-wake test — see the header note. This is a TSan target and a + // check that the CAS loops always terminate and always leave the gate in a + // reachable state: exactly one party may hold the claim at a time, so the + // count of claims granted must equal the count of releases that ended idle. + SubmitGate gate; + std::atomic granted{0}, ended_idle{0}; + std::atomic stop{false}; + + std::thread waker([&] { + while (!stop.load(std::memory_order_relaxed)) + if (gate.claim()) granted.fetch_add(1, std::memory_order_relaxed); + }); + std::thread worker([&] { + while (!stop.load(std::memory_order_relaxed)) + if (gate.queued() && !gate.release()) + ended_idle.fetch_add(1, std::memory_order_relaxed); + }); + + std::this_thread::sleep_for(std::chrono::milliseconds(200)); + stop.store(true, std::memory_order_relaxed); + waker.join(); + worker.join(); + + // Drain whatever claim is outstanding so the two counts can be compared. + while (gate.queued()) + if (!gate.release()) ended_idle.fetch_add(1, std::memory_order_relaxed); + + INFO("granted " << granted.load() << " ended idle " << ended_idle.load()); + REQUIRE(granted.load() > 0); + CHECK(granted.load() == ended_idle.load()); +} From a5c016833ddf4e045700bbd8d4bb7eba893e71fe Mon Sep 17 00:00:00 2001 From: Duncan Tourolle Date: Wed, 5 Aug 2026 13:39:26 +0200 Subject: [PATCH 23/42] fix: install channel callbacks before any node runs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ThreadSanitizer reported ten data races on a plain multi-node network, all the same one: Read in Channel::try_push -> push_callback_() (worker thread) Write in Channel::set_push_callback -> push_callback_ = ... (main thread) A node's push and space callbacks are std::function members living on channels it shares with its neighbours. register_callbacks() wrote them from inside start(), and a network starts its nodes one at a time — so by the time node N is being started, nodes 1..N-1 are already running and pushing into N's input channel, reading the very std::function that start() is assigning. Concurrent read and write of a std::function is a data race on its vtable pointer and buffer, not a benign one. This is the cause of the symptom a8cfe73 patched. That commit found nodes missing their startup wake because "enable_inputs() opens the channel several statements before register_callbacks() installs the push callback", and fixed it by re-asking the question with on_input_ready(). The gap it described is this race: the callback is not merely late, it is being written while another thread reads it. INode gains prepare(), which installs callbacks and starts nothing. Networks call it on every node before starting any of them, so every write happens while the pipeline is idle and the callbacks are read-only once it is live. start() calls prepare() itself when a node is used standalone, and prepare() is idempotent so both paths are safe. The flag is never cleared: the callbacks capture `this` and stay valid across a restart, so re-registering them would only add a pointless write to a live channel. a8cfe73's on_input_ready() stays, and is still needed — a network starts nodes one at a time, so an upstream node can still push into this one between its prepare() and its start(), where on_input_ready() returns early on stop_flag_ and the empty->non-empty edge is spent. It is now a level-triggered check against a benign ordering rather than cover for a race. Verified with -DKPN_SANITIZER=thread: ten races before, none of these after, across the unit suite and the contended channel stress suite. One unrelated race remains, on overlapping fire_once invocations; it is pre-existing and is fixed separately. --- include/kpn/inode.hpp | 17 ++++++++++++ include/kpn/network.hpp | 3 +++ include/kpn/pool_node.hpp | 49 +++++++++++++++++++++++++--------- include/kpn/static_network.hpp | 9 +++++++ include/kpn/variant_node.hpp | 1 + 5 files changed, 66 insertions(+), 13 deletions(-) diff --git a/include/kpn/inode.hpp b/include/kpn/inode.hpp index a7454e2..1ad9747 100644 --- a/include/kpn/inode.hpp +++ b/include/kpn/inode.hpp @@ -22,6 +22,23 @@ enum class NodeEvent { Overflow, Closed }; struct INode { virtual ~INode() = default; + + // Install channel callbacks, without starting anything. + // + // A node's push/space callbacks live in std::function members on channels + // it shares with its neighbours, and a neighbour that is already running + // reads them on its own thread. Writing one while the pipeline runs is a + // data race on the std::function — ThreadSanitizer reports it, and the + // consequence in the field was the missed startup wake a8cfe73 had to + // patch around. + // + // So a network calls prepare() on every node before it calls start() on + // any of them: all the writes happen while nothing is running, and once a + // node is live the callbacks are read-only. start() calls prepare() itself + // if it has not been called, so standalone nodes still work; it is + // idempotent, and the network relies on that. + virtual void prepare() {} + virtual void start() = 0; virtual void stop() = 0; virtual bool running() const = 0; diff --git a/include/kpn/network.hpp b/include/kpn/network.hpp index 239ad88..67686f3 100644 --- a/include/kpn/network.hpp +++ b/include/kpn/network.hpp @@ -134,6 +134,9 @@ public: void start() override { start_time_ = clock_t::now(); + // Callbacks first, everywhere, before anything runs — see INode::prepare. + for (auto& name : topo_) + nodes_.at(name)->prepare(); for (auto& name : topo_) nodes_.at(name)->start(); start_watchdog(); diff --git a/include/kpn/pool_node.hpp b/include/kpn/pool_node.hpp index bff47b3..cf52f2b 100644 --- a/include/kpn/pool_node.hpp +++ b/include/kpn/pool_node.hpp @@ -81,29 +81,36 @@ public: // ── INode ───────────────────────────────────────────────────────────────── + void prepare() override { + if (prepared_) return; // idempotent: the network calls this, + prepared_ = true; // and start() calls it again if not. + register_callbacks(std::make_index_sequence{}); + } + void start() override { + prepare(); enable_inputs(std::make_index_sequence{}); stop_flag_.store(false, std::memory_order_relaxed); gate_.force_idle(); - register_callbacks(std::make_index_sequence{}); if constexpr (input_count == 0) try_submit(0.5f); else // Never start with a wake already outstanding — the startup case of // the invariant 9c5ce5f established for the running pipeline. // - // enable_inputs() opens the channel several statements before - // register_callbacks() installs the push callback, and the network - // starts nodes sources-first, so an upstream node is already firing - // into this one during that gap. A push landing there is accepted by - // the ring but wakes nobody: Channel::push only invokes the callback - // on the empty→non-empty transition, and at that instant the - // callback is still null. Every later push sees a non-empty ring and - // stays silent, so the node is never submitted — the pipeline reads - // as wedged from the first frame, with no item ever delivered. + // The callback is installed by prepare(), before any node runs, but + // a network still starts its nodes one at a time: an upstream node + // that is already firing can push into this one between the two + // calls. The push is accepted by the ring and does invoke the + // callback, but on_input_ready() sees stop_flag_ still set and + // returns. Every later push sees a non-empty ring and stays silent + // — Channel invokes push_callback_ only on the empty->non-empty + // transition — so without this the node is never submitted and the + // pipeline reads as wedged from the first frame. // - // on_input_ready() is the level-triggered form of the same question, - // so asking it once here converts the missed edge into a state check. + // on_input_ready() is the level-triggered form of the same + // question, so asking it once here converts the missed edge into a + // state check. on_input_ready(); } @@ -584,6 +591,11 @@ private: /// Serialises firings and records wakes that arrive during one. See /// submit_gate.hpp for why this cannot be two separate flags. SubmitGate gate_; + /// Whether prepare() has installed the channel callbacks. Only ever touched + /// from the thread driving start()/stop(), never from a worker, and never + /// cleared: the callbacks capture `this` and stay valid across a restart, so + /// re-registering them would be a pointless write to a live channel. + bool prepared_{false}; /// The hidden one-slot output buffer (see push_outputs). Holding the value /// here is what lets a node stop running without dropping it or occupying a @@ -646,11 +658,17 @@ public: ~PoolObjectNode() override { stop(); } + void prepare() override { + if (prepared_) return; + prepared_ = true; + register_callbacks(std::make_index_sequence{}); + } + void start() override { + prepare(); enable_inputs(std::make_index_sequence{}); stop_flag_.store(false, std::memory_order_relaxed); gate_.force_idle(); - register_callbacks(std::make_index_sequence{}); if constexpr (input_count == 0) try_submit(0.5f); else @@ -1050,6 +1068,11 @@ private: /// Serialises firings and records wakes that arrive during one. See /// submit_gate.hpp for why this cannot be two separate flags. SubmitGate gate_; + /// Whether prepare() has installed the channel callbacks. Only ever touched + /// from the thread driving start()/stop(), never from a worker, and never + /// cleared: the callbacks capture `this` and stay valid across a restart, so + /// re-registering them would be a pointless write to a live channel. + bool prepared_{false}; /// The hidden one-slot output buffer (see push_outputs). Holding the value /// here is what lets a node stop running without dropping it or occupying a diff --git a/include/kpn/static_network.hpp b/include/kpn/static_network.hpp index bb34a2c..9c2dc32 100644 --- a/include/kpn/static_network.hpp +++ b/include/kpn/static_network.hpp @@ -126,6 +126,15 @@ public: for (auto* node : user_nodes_topo_) node->set_network_error_callback(error_handler_); } + // Install every node's channel callbacks before starting any of them. + // Those callbacks are std::function members on channels shared with + // neighbours; a neighbour that is already running reads them from its + // own thread, so writing one after the pipeline is live is a data race + // (ThreadSanitizer reports it on any multi-node network). Doing all the + // writes here, while nothing runs, makes them read-only thereafter. + for (auto* n : user_nodes_topo_) n->prepare(); + for (auto* n : fanout_nodes_ptr_) n->prepare(); + for (auto* n : user_nodes_topo_) n->start(); for (auto* n : fanout_nodes_ptr_) n->start(); #ifdef KPN_WEB_DEBUG diff --git a/include/kpn/variant_node.hpp b/include/kpn/variant_node.hpp index 9a98599..26320ca 100644 --- a/include/kpn/variant_node.hpp +++ b/include/kpn/variant_node.hpp @@ -162,6 +162,7 @@ public: // ── INode ───────────────────────────────────────────────────────────────── + void prepare() override { node_.prepare(); } void start() override { node_.start(); } void stop() override { node_.stop(); } bool running() const override { return node_.running(); } From 15e993f6caad06392674605e3e2b467d4802cef4 Mon Sep 17 00:00:00 2001 From: Duncan Tourolle Date: Wed, 5 Aug 2026 13:48:13 +0200 Subject: [PATCH 24/42] fix: two firings of the same node must not overlap MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit fire_once released the submit gate and then kept working: release_and_recheck(); // gate is now free if (stop_flag_) return; if (pending_) { ... } // still reading node state on_input_ready(); The moment the gate is free another worker may enter fire_once for the same node, so this invocation's reads of pending_ raced with the next one's writes to pending_done_. ThreadSanitizer caught exactly that, between a firing submitted by release_and_recheck and one submitted by try_submit. The race is the visible half. The real damage is to the one-slot park, which is sound only because "at most one fire_once runs per node at a time" — the comment on pending_ says so explicitly. With two firings live, one can park a value into the slot the other is about to overwrite, and the overwritten value is gone with no drop recorded anywhere. That is silent data loss under backpressure, from a node that reports itself healthy. finish_firing() replaces release_and_recheck() at every exit: it evaluates the follow-up decision — parked and waiting on output space, or drained and waiting on input — while the claim is still held, and releases the gate as the last thing the firing does. Nothing touches node state afterwards. This also collapses three near-identical resubmit tails into one, which is worth something on its own: the divergence between them is what 5628447 and 9c5ce5f were both picking at, and each fix had to be applied to every copy. Pre-existing, not introduced by the gate rewrite: the old two-atomic version cleared queued_ in the same place, with the same code after it. Verified with -DKPN_SANITIZER=thread. The race is intermittent — roughly one run in three before the fix — so five consecutive clean runs of the unit suite plus the contended channel stress suite, all zero. Full suite 132/132. --- include/kpn/pool_node.hpp | 225 +++++++++++++++++++------------------- 1 file changed, 114 insertions(+), 111 deletions(-) diff --git a/include/kpn/pool_node.hpp b/include/kpn/pool_node.hpp index cf52f2b..d228a84 100644 --- a/include/kpn/pool_node.hpp +++ b/include/kpn/pool_node.hpp @@ -264,7 +264,7 @@ private: disable_inputs(std::make_index_sequence{}); disable_outputs(std::make_index_sequence{}); stats_.exec_start_us.store(0, std::memory_order_relaxed); - // Plain store, not release_and_recheck(): this node is stopping, and + // force_idle, not finish_firing(): this node is stopping, and // honouring a pending wake here would resubmit a dead node. gate_.force_idle(); stop_flag_.store(true, std::memory_order_relaxed); @@ -368,18 +368,50 @@ private: scheduler_->submit([this] { fire_once(); }, priority); } - /// End this firing, honouring any wake recorded during it. Every path that - /// finishes or parks a firing must release the node through here rather - /// than touching the gate directly. When a wake was recorded the gate stays - /// claimed and is handed to the next firing, so the node is never - /// momentarily idle with work outstanding. - void release_and_recheck(float priority = 0.5f) { - if (gate_.release()) - scheduler_->submit([this] { fire_once(); }, priority); - } - // ── Execution ───────────────────────────────────────────────────────────── + /// Decide whether this node should run again, then release the gate — in + /// that order, always. + /// + /// Releasing first is what let two firings of the same node overlap: the + /// moment the gate is free another worker may enter fire_once, while this + /// invocation is still reading pending_ and writing pending_done_. TSan + /// caught it as a race on pending_done_ between a firing submitted by the + /// old release_and_recheck and one submitted by try_submit. It also quietly + /// broke the one-slot park, which is sound only because "at most one + /// fire_once runs per node at a time" — with two, a value can be parked by + /// one firing and overwritten by the other. + /// + /// Everything this reads belongs to the firing that holds the claim, so it + /// is all evaluated first and the release is the last thing the firing does. + void finish_firing() { + bool want_more = false; + float prio = 0.5f; + + if (!stop_flag_.load(std::memory_order_relaxed)) { + bool parked = false; + if constexpr (!std::is_void_v) + parked = pending_.has_value(); + + if (parked) { + // Still holding output: only worth running again once the + // consumer has made room. + want_more = outputs_have_space(std::make_index_sequence{}); + } else { + if constexpr (input_count == 0) { + want_more = true; // sources always run again + } else { + want_more = count_ready(std::make_index_sequence{}) + == input_count; + if (want_more) prio = compute_priority(); + } + } + } + + if (gate_.release()) scheduler_->submit([this] { fire_once(); }, prio); + else if (want_more) try_submit(prio); + } + void fire_once() { if (stop_flag_.load(std::memory_order_relaxed)) { gate_.force_idle(); @@ -398,25 +430,13 @@ private: if constexpr (!std::is_void_v) { if (pending_) { push_outputs(std::move(*pending_), std::make_index_sequence{}); - release_and_recheck(); - if (pending_) { - // Close the lost-wakeup race: a space_callback that fired - // between the failed push and releasing the gate was - // swallowed, and nothing else will wake this node. Re-check - // now that the flag is down. - if (outputs_have_space(std::make_index_sequence{})) - try_submit(0.5f); - return; // parked - } - // Drained: resume normal firing, resubmitting exactly the way - // the normal tail below does. An unconditional try_submit here - // would fire a node whose inputs are empty, and pop_inputs - // reports an empty channel as ChannelClosedError — which this - // node treats as "upstream finished" and self-stops on. That - // is a live node killing itself purely because it was woken by - // *output* space rather than by input arrival. - if constexpr (input_count == 0) try_submit(0.5f); - else on_input_ready(); + // Whether the value went out or is still parked, finish_firing + // reads pending_ and picks the right follow-up: output space if + // still holding, input readiness if drained. Resubmitting + // unconditionally would fire a node whose inputs are empty, and + // pop_one reports an empty channel as ChannelClosedError — which + // this node treats as "upstream finished" and self-stops on. + finish_firing(); return; } } @@ -428,8 +448,9 @@ private: // on_input_ready() resubmits when data actually lands. if constexpr (input_count > 0) { if (count_ready(std::make_index_sequence{}) != input_count) { - release_and_recheck(); - on_input_ready(); // data may have arrived while we checked + // finish_firing re-checks readiness after the work above, so + // data that landed while we looked is not missed. + finish_firing(); return; } } @@ -473,33 +494,11 @@ private: } stats_.exec_start_us.store(0, std::memory_order_relaxed); - release_and_recheck(); - - if (stop_flag_.load(std::memory_order_relaxed)) return; - - // Parked by the push above. Same situation as the retry path at the top - // of fire_once — and the same lost-wakeup race, which that path closes - // and this one did not. A space callback that fired while the gate was - // still claimed is recorded there, and the resubmit below - // cannot cover it: this firing consumed its input, so inputs are empty - // and on_input_ready() will not resubmit. The node would then hold its - // value forever while its consumer waits for exactly that value and its - // producer parks on an input channel that never drains. Re-check now - // that the flag is down. - if constexpr (!std::is_void_v) { - if (pending_) { - if (outputs_have_space(std::make_index_sequence{})) - try_submit(0.5f); - return; // parked - } - } - - // Source nodes always resubmit; others resubmit only if inputs are ready. - if constexpr (input_count == 0) { - try_submit(0.5f); - } else { - on_input_ready(); - } + // If the push above parked, finish_firing waits on output space rather + // than input arrival: this firing consumed its input, so an input-level + // check would not resubmit and the node would hold its value forever + // while its consumer waits for exactly that value. + finish_firing(); } // Pop all inputs — safe because we're the sole consumer and fire_once @@ -787,7 +786,7 @@ private: disable_inputs(std::make_index_sequence{}); disable_outputs(std::make_index_sequence{}); stats_.exec_start_us.store(0, std::memory_order_relaxed); - // Plain store, not release_and_recheck(): this node is stopping, and + // force_idle, not finish_firing(): this node is stopping, and // honouring a pending wake here would resubmit a dead node. gate_.force_idle(); stop_flag_.store(true, std::memory_order_relaxed); @@ -884,14 +883,46 @@ private: scheduler_->submit([this] { fire_once(); }, priority); } - /// End this firing, honouring any wake recorded during it. Every path that - /// finishes or parks a firing must release the node through here rather - /// than touching the gate directly. When a wake was recorded the gate stays - /// claimed and is handed to the next firing, so the node is never - /// momentarily idle with work outstanding. - void release_and_recheck(float priority = 0.5f) { - if (gate_.release()) - scheduler_->submit([this] { fire_once(); }, priority); + /// Decide whether this node should run again, then release the gate — in + /// that order, always. + /// + /// Releasing first is what let two firings of the same node overlap: the + /// moment the gate is free another worker may enter fire_once, while this + /// invocation is still reading pending_ and writing pending_done_. TSan + /// caught it as a race on pending_done_ between a firing submitted by the + /// old release_and_recheck and one submitted by try_submit. It also quietly + /// broke the one-slot park, which is sound only because "at most one + /// fire_once runs per node at a time" — with two, a value can be parked by + /// one firing and overwritten by the other. + /// + /// Everything this reads belongs to the firing that holds the claim, so it + /// is all evaluated first and the release is the last thing the firing does. + void finish_firing() { + bool want_more = false; + float prio = 0.5f; + + if (!stop_flag_.load(std::memory_order_relaxed)) { + bool parked = false; + if constexpr (!std::is_void_v) + parked = pending_.has_value(); + + if (parked) { + // Still holding output: only worth running again once the + // consumer has made room. + want_more = outputs_have_space(std::make_index_sequence{}); + } else { + if constexpr (input_count == 0) { + want_more = true; // sources always run again + } else { + want_more = count_ready(std::make_index_sequence{}) + == input_count; + if (want_more) prio = compute_priority(); + } + } + } + + if (gate_.release()) scheduler_->submit([this] { fire_once(); }, prio); + else if (want_more) try_submit(prio); } void fire_once() { @@ -910,25 +941,13 @@ private: if constexpr (!std::is_void_v) { if (pending_) { push_outputs(std::move(*pending_), std::make_index_sequence{}); - release_and_recheck(); - if (pending_) { - // Close the lost-wakeup race: a space_callback that fired - // between the failed push and releasing the gate was - // swallowed, and nothing else will wake this node. Re-check - // now that the flag is down. - if (outputs_have_space(std::make_index_sequence{})) - try_submit(0.5f); - return; // parked - } - // Drained: resume normal firing, resubmitting exactly the way - // the normal tail below does. An unconditional try_submit here - // would fire a node whose inputs are empty, and pop_inputs - // reports an empty channel as ChannelClosedError — which this - // node treats as "upstream finished" and self-stops on. That - // is a live node killing itself purely because it was woken by - // *output* space rather than by input arrival. - if constexpr (input_count == 0) try_submit(0.5f); - else on_input_ready(); + // Whether the value went out or is still parked, finish_firing + // reads pending_ and picks the right follow-up: output space if + // still holding, input readiness if drained. Resubmitting + // unconditionally would fire a node whose inputs are empty, and + // pop_one reports an empty channel as ChannelClosedError — which + // this node treats as "upstream finished" and self-stops on. + finish_firing(); return; } } @@ -938,8 +957,9 @@ private: // into pop_inputs on an empty channel. if constexpr (input_count > 0) { if (count_ready(std::make_index_sequence{}) != input_count) { - release_and_recheck(); - on_input_ready(); + // finish_firing re-checks readiness after the work above, so + // data that landed while we looked is not missed. + finish_firing(); return; } } @@ -980,28 +1000,11 @@ private: } stats_.exec_start_us.store(0, std::memory_order_relaxed); - release_and_recheck(); - if (stop_flag_.load(std::memory_order_relaxed)) return; - - // Parked by the push above. Same situation as the retry path at the top - // of fire_once — and the same lost-wakeup race, which that path closes - // and this one did not. A space callback that fired while the gate was - // still claimed is recorded there, and the resubmit below - // cannot cover it: this firing consumed its input, so inputs are empty - // and on_input_ready() will not resubmit. The node would then hold its - // value forever while its consumer waits for exactly that value and its - // producer parks on an input channel that never drains. Re-check now - // that the flag is down. - if constexpr (!std::is_void_v) { - if (pending_) { - if (outputs_have_space(std::make_index_sequence{})) - try_submit(0.5f); - return; // parked - } - } - - if constexpr (input_count == 0) try_submit(0.5f); - else on_input_ready(); + // If the push above parked, finish_firing waits on output space rather + // than input arrival: this firing consumed its input, so an input-level + // check would not resubmit and the node would hold its value forever + // while its consumer waits for exactly that value. + finish_firing(); } template From 8d319eeb8887956af23f71d8d11e3602f007dae6 Mon Sep 17 00:00:00 2001 From: Duncan Tourolle Date: Wed, 5 Aug 2026 13:55:57 +0200 Subject: [PATCH 25/42] fix: an empty channel is not a closed one MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit pop_one reported an empty channel the same way it reported a closed one, by throwing ChannelClosedError, and fire_once treats that as "upstream is finished" and calls self_stop(). self_stop disables the node's own inputs *and* outputs, so a benign empty read does not merely skip a frame — it kills the node and, through the disabled channels, whatever depended on it. A node genuinely does get woken with empty inputs: a space callback fires when its output drains, which has nothing to do with input arrival. fire_once already guards against it by checking readiness before popping. That guard is the live protection and it works; this commit makes the thing it is guarding non-lethal. So the pop_one path changed here is unreachable today, and I would rather say that than imply a fixed hang. Its value is that the readiness check is now a performance detail rather than the only thing standing between a routine wake and a dead pipeline. Three separate comment blocks in fire_once exist to warn about exactly this hazard; they were added because it had already been hit during development, and the conflation they warn about is what this removes. Verified in three directions. With the guard and the distinction: passes. With the guard removed but the distinction present: still passes, which is the point — the new ChannelEmptyError path catches what the guard used to. With both removed, reproducing the original code: the node self-stops on the firing that has nothing to read, the next value throws "channel closed" out of its own output channel, and the relay handles one item instead of two. --- include/kpn/channel.hpp | 12 +++++++ include/kpn/pool_node.hpp | 37 ++++++++++++++++++++-- tests/test_pool_node.cpp | 66 +++++++++++++++++++++++++++++++++++++++ 3 files changed, 113 insertions(+), 2 deletions(-) diff --git a/include/kpn/channel.hpp b/include/kpn/channel.hpp index 571b891..aa87523 100644 --- a/include/kpn/channel.hpp +++ b/include/kpn/channel.hpp @@ -58,6 +58,18 @@ public: ChannelClosedError() : std::runtime_error("channel closed") {} }; +// Nothing available *right now* on a channel that is still open. Distinct from +// ChannelClosedError, which means upstream is finished and never coming back. +// +// Conflating the two is expensive in one direction only: a consumer that reads +// "empty" as "closed" stops a live node permanently, and because a stopping +// node disables its own inputs and outputs, one benign empty read takes the +// rest of the pipeline with it. The reverse costs nothing. +class ChannelEmptyError : public std::runtime_error { +public: + ChannelEmptyError() : std::runtime_error("channel empty") {} +}; + // ── CPU pause hint ──────────────────────────────────────────────────────────── // Signals the CPU that this is a spin-wait loop, improving HT sibling throughput // and preventing branch-predictor thrash on x86. Falls back to a compiler barrier. diff --git a/include/kpn/pool_node.hpp b/include/kpn/pool_node.hpp index d228a84..81c0724 100644 --- a/include/kpn/pool_node.hpp +++ b/include/kpn/pool_node.hpp @@ -473,6 +473,14 @@ private: auto t2 = clock_t::now(); // blocked_time = 0 for pool nodes (we don't block waiting for inputs) stats_.record_exec(duration_t(t2 - t1), duration_t::zero(), cpu0, cpu1); + } catch (const ChannelEmptyError&) { + // Not an error: there was simply nothing to take. Release and wait + // to be woken again. fire_once checks readiness before it gets + // here, and this node is the sole consumer of its inputs, so this + // is unreachable today — it exists so that if the check is ever + // weakened the cost is a wasted firing rather than a dead node. + finish_firing(); + return; } catch (const ChannelClosedError&) { fire_callbacks(closed_callbacks_); self_stop(); @@ -512,8 +520,16 @@ private: std::tuple_element_t pop_one() { auto& ch = *std::get(input_channels_); std::tuple_element_t val; - if (!ch.try_pop_now(val)) + if (!ch.try_pop_now(val)) { + // try_pop_now returns false for "nothing available", which covers + // two very different situations. A closed channel means upstream is + // finished and this node should stop. An open one means only that + // nothing is here at this instant — and treating that as closed + // kills a live node, which then disables its own inputs and outputs + // and takes the rest of the pipeline with it. + if (ch.is_accepting()) throw ChannelEmptyError{}; throw ChannelClosedError{}; + } return val; } @@ -980,6 +996,14 @@ private: auto cpu1 = NodeStats::cpu_now(); auto t2 = clock_t::now(); stats_.record_exec(duration_t(t2 - t1), duration_t::zero(), cpu0, cpu1); + } catch (const ChannelEmptyError&) { + // Not an error: there was simply nothing to take. Release and wait + // to be woken again. fire_once checks readiness before it gets + // here, and this node is the sole consumer of its inputs, so this + // is unreachable today — it exists so that if the check is ever + // weakened the cost is a wasted firing rather than a dead node. + finish_firing(); + return; } catch (const ChannelClosedError&) { fire_callbacks(closed_callbacks_); self_stop(); @@ -1014,7 +1038,16 @@ private: std::tuple_element_t pop_one() { auto& ch = *std::get(input_channels_); std::tuple_element_t val; - if (!ch.try_pop_now(val)) throw ChannelClosedError{}; + if (!ch.try_pop_now(val)) { + // try_pop_now returns false for "nothing available", which covers + // two very different situations. A closed channel means upstream is + // finished and this node should stop. An open one means only that + // nothing is here at this instant — and treating that as closed + // kills a live node, which then disables its own inputs and outputs + // and takes the rest of the pipeline with it. + if (ch.is_accepting()) throw ChannelEmptyError{}; + throw ChannelClosedError{}; + } return val; } diff --git a/tests/test_pool_node.cpp b/tests/test_pool_node.cpp index 8902bae..0926556 100644 --- a/tests/test_pool_node.cpp +++ b/tests/test_pool_node.cpp @@ -602,3 +602,69 @@ TEST_CASE("a twice-parked value keeps its payload", "[pool_node][backpressure]") CHECK(parked.size() == 4); if (parked.size() == 4) CHECK(parked[0] == 2); } + +// Regression: a node woken with nothing to read must not stop itself. +// +// pop_one reported an empty channel the same way it reported a closed one, by +// throwing ChannelClosedError, and fire_once treats that as "upstream is +// finished" and calls self_stop(). self_stop disables the node's own inputs +// *and* outputs, so one benign empty read does not merely skip a frame — it +// kills the node and, through the disabled channels, the rest of the pipeline. +// +// A node genuinely does get woken with empty inputs: a space callback fires +// when its output drains, which has nothing to do with input arrival. fire_once +// guards against it by checking readiness before popping, and that guard is +// what this test pins. pop_one now also distinguishes the two cases, so if the +// guard is ever weakened the cost is a wasted firing rather than a dead node. +// +// The sequence below reaches the guard deliberately. The output is capacity 1 +// so that draining it signals space at all — Channel fires the space callback +// only on the full->not-full edge — and by the final pop the input is long +// since consumed, so the resulting firing has nothing to read. +namespace { + +struct CountingRelay { + static constexpr std::string_view label() { return "counting_relay"; } + std::atomic* calls; + int operator()(int v) { calls->fetch_add(1, std::memory_order_relaxed); return v; } +}; + +} // namespace + +TEST_CASE("a node woken with empty inputs does not stop itself", "[pool_node]") { + std::atomic calls{0}; + std::atomic closed{0}; + + auto pool = std::make_shared(2); + pool->start(); + + CountingRelay fn{&calls}; + auto node = make_pool_node(fn, pool, 8); + Channel out(1); + node.set_output_channel<0>(&out); + node.set_closed_callback([&](auto) { closed.fetch_add(1, std::memory_order_relaxed); }); + + out.push(99); // output full before the node runs + node.start(); + + node.input_channel<0>().push(1); // fires, cannot deliver, parks + std::this_thread::sleep_for(std::chrono::milliseconds(50)); + + REQUIRE(out.pop() == 99); // space -> retry delivers the parked value + std::this_thread::sleep_for(std::chrono::milliseconds(50)); + REQUIRE(out.pop() == 1); // space again -> fires with empty inputs + std::this_thread::sleep_for(std::chrono::milliseconds(50)); + + // That firing had nothing to read. The node must still be alive. + CHECK(closed.load(std::memory_order_relaxed) == 0); + CHECK(node.running()); + + // And must still do its job when real input arrives. + node.input_channel<0>().push(2); + std::this_thread::sleep_for(std::chrono::milliseconds(50)); + CHECK(out.pop() == 2); + CHECK(calls.load(std::memory_order_relaxed) == 2); + + node.stop(); + pool->stop(); +} From 139bfbb7946a85a0fe1f01c1913130d796ade91e Mon Sep 17 00:00:00 2001 From: Duncan Tourolle Date: Wed, 5 Aug 2026 14:06:26 +0200 Subject: [PATCH 26/42] fix: the sentinel slot holds one token and refuses a second MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit push_sentinel wrote eof_value_ unconditionally. Offering a second token before the first was taken did two wrong things at once. It lost the first silently, and a lost EOF is not a lost frame — it is the token every downstream node is waiting for in order to shut down, so losing it wedges the pipeline. And it wrote the storage while the consumer could be moving the previous value out of it. I expected that to be a stale read; ThreadSanitizer shows it is worse. On the shared_ptr storage that non-trivial types use, the racing write tears the refcount, and the stress case added here reports heap-use-after-free in extract() alongside the data race. try_push_sentinel now refuses when the slot is occupied, which turns the slot into a correct SPSC handshake: the producer is the only writer of eof_value_ and the only one that sets has_eof_, the consumer is the only one that clears it, so observing it false is what licenses the write. Refusal is recorded as a drop, and PoolNode reports it through the overflow event callback, because a refused control token going unnoticed is the failure this commit exists to stop. Refusing rather than queueing is deliberate. Two control tokens on one channel means the stream ended twice, which is a caller protocol error and not backpressure; parking and retrying would spin against a slot only the consumer can free, and there is no sensible second value to deliver after the end of a stream. The non-consuming try_push_sentinel exists so a refused token is still the caller's to report — the consuming push_sentinel cannot offer that, since the value has already been moved into its parameter. Single-shot EOF is what every current caller does, so this is latent for them today. It stops being latent the moment a pipeline is reused for a second input, which is what the persistent-pipeline work in 4b6e498 sets up. Verified in both directions under -DKPN_SANITIZER=thread: the new contended case reports three data races and a heap-use-after-free against the old overwrite, and is clean with the handshake. Full suite 137/137, TSan clean across unit and stress suites. --- include/kpn/channel.hpp | 41 +++++++++++++++++++++--- include/kpn/pool_node.hpp | 18 +++++++++-- tests/test_channel.cpp | 60 +++++++++++++++++++++++++++++++++++ tests/test_channel_stress.cpp | 49 ++++++++++++++++++++++++++++ 4 files changed, 161 insertions(+), 7 deletions(-) diff --git a/include/kpn/channel.hpp b/include/kpn/channel.hpp index aa87523..a35651b 100644 --- a/include/kpn/channel.hpp +++ b/include/kpn/channel.hpp @@ -226,12 +226,37 @@ public: // preserving ordering (EOF arrives after all data pushed before it). // // Only the sole producer may call it (SPSC contract, same as push()). - // Returns false if the channel is already disabled (token discarded — - // teardown is in progress, so the sentinel is moot). - bool push_sentinel(T value) { + // + // The slot holds exactly one undelivered token. A second offered before the + // first is taken is refused, not queued and not overwritten: two control + // tokens on one channel means the stream ended twice, which is a caller + // protocol error rather than backpressure, and silently coalescing them + // would hide it. + /// Outcome of offering a sentinel. SlotBusy is a protocol error, not + /// backpressure: it means a second control token was offered while the + /// first was still undelivered, and a channel carries at most one. + enum class SentinelResult { Taken, Closed, SlotBusy }; + + /// Non-consuming form. `value` is left untouched unless the result is + /// Taken, so a refused token is still the caller's to report. + SentinelResult try_push_sentinel(T& value) { if (!accepting_.load(std::memory_order_acquire)) { stats_.record_drop(); - return false; + return SentinelResult::Closed; + } + // Refuse rather than overwrite. Overwriting lost the first token + // silently, and worse, wrote eof_value_ while the consumer could be + // moving the previous one out of it — a data race on the storage, which + // for a shared_ptr payload is a torn refcount rather than a stale read. + // + // Checking here is what makes the slot a correct SPSC handshake: the + // producer is the only writer of eof_value_ and the only one that sets + // has_eof_, the consumer is the only one that clears it, so observing + // false here means the consumer has finished with the storage and will + // not touch it again until this store publishes the next token. + if (has_eof_.load(std::memory_order_acquire)) { + stats_.record_drop(); + return SentinelResult::SlotBusy; } eof_value_ = make_storage(std::move(value)); has_eof_.store(true, std::memory_order_release); @@ -240,7 +265,13 @@ public: wake_.fetch_add(1, std::memory_order_release); wake_.notify_one(); if (push_callback_) push_callback_(); - return true; + return SentinelResult::Taken; + } + + /// Consuming convenience form. Returns false when the token was not stored, + /// whether because the channel is closed or because one is already pending. + bool push_sentinel(T value) { + return try_push_sentinel(value) == SentinelResult::Taken; } // Blocking pop. Returns when an item is available. diff --git a/include/kpn/pool_node.hpp b/include/kpn/pool_node.hpp index 81c0724..19a32be 100644 --- a/include/kpn/pool_node.hpp +++ b/include/kpn/pool_node.hpp @@ -570,7 +570,14 @@ private: // downstream pop() forever. Deliver them out-of-band (push_sentinel), // which never overflows and never blocks this node's worker thread. if (is_sentinel_value(val)) { - ch->push_sentinel(std::move(val)); + // A refused sentinel is a protocol error, not backpressure, so it + // is reported rather than parked and retried — retrying would spin + // forever against a slot only the consumer can free, and there is + // no correct value to deliver second anyway. Closed is normal + // during teardown and stays quiet. + if (ch->try_push_sentinel(val) == Channel> + ::SentinelResult::SlotBusy) + fire_callbacks(event_callbacks_); return true; } // Backpressure without parking the worker. A full channel means the @@ -1087,7 +1094,14 @@ private: // downstream pop() forever. Deliver them out-of-band (push_sentinel), // which never overflows and never blocks this node's worker thread. if (is_sentinel_value(val)) { - ch->push_sentinel(std::move(val)); + // A refused sentinel is a protocol error, not backpressure, so it + // is reported rather than parked and retried — retrying would spin + // forever against a slot only the consumer can free, and there is + // no correct value to deliver second anyway. Closed is normal + // during teardown and stays quiet. + if (ch->try_push_sentinel(val) == Channel> + ::SentinelResult::SlotBusy) + fire_callbacks(event_callbacks_); return true; } // See the note on the typed overload above: park rather than block. diff --git a/tests/test_channel.cpp b/tests/test_channel.cpp index d710214..8150b6d 100644 --- a/tests/test_channel.cpp +++ b/tests/test_channel.cpp @@ -1,3 +1,4 @@ +#include #include #include #include @@ -238,3 +239,62 @@ TEST_CASE("try_pop_now delivers a pending sentinel once the ring is empty", REQUIRE(out == 99); REQUIRE_FALSE(ch.try_pop_now(out)); // nothing left } + +// Regression: the sentinel slot holds one token and refuses a second. +// +// push_sentinel used to write eof_value_ unconditionally. Offering a second +// token before the first was taken therefore did two wrong things at once: it +// lost the first silently — and a lost EOF wedges every downstream pop forever +// — and it wrote the storage while the consumer could be moving the previous +// value out of it. For the shared_ptr storage that non-trivial types use, that +// is a torn refcount, not merely a stale read. +// +// Refusing is correct rather than queueing: two control tokens on one channel +// means the stream ended twice, which is a caller protocol error. Coalescing +// them would hide it, and there is no second value that could sensibly follow +// the end of a stream. +TEST_CASE("a second sentinel is refused, not swallowed", "[channel][sentinel]") { + Channel ch(4); + + REQUIRE(ch.push_sentinel(1)); + // Slot occupied: the first token is still undelivered. + REQUIRE_FALSE(ch.push_sentinel(2)); + + // The first survives intact — the overwrite is what used to lose it. + int out = 0; + REQUIRE(ch.try_pop_now(out)); + CHECK(out == 1); + + // And the slot is reusable once drained. + REQUIRE(ch.push_sentinel(3)); + REQUIRE(ch.try_pop_now(out)); + CHECK(out == 3); +} + +TEST_CASE("a refused sentinel is counted as a drop", "[channel][sentinel]") { + // Visibility matters more here than for a dropped value: the refusal means + // a control token went nowhere, and the only alternative to a counter is + // for it to vanish. + Channel ch(4); + REQUIRE(ch.push_sentinel(1)); + const auto before = ch.stats().drops.load(); + REQUIRE_FALSE(ch.push_sentinel(2)); + CHECK(ch.stats().drops.load() == before + 1); +} + +TEST_CASE("try_push_sentinel leaves a refused value untouched", "[channel][sentinel]") { + // The non-consuming form exists so a refused token is still the caller's to + // report. The consuming push_sentinel cannot offer that, since the value is + // already moved into its parameter. + Channel ch(4); + std::string first = "eof-1", second = "eof-2"; + + REQUIRE(ch.try_push_sentinel(first) == Channel::SentinelResult::Taken); + REQUIRE(ch.try_push_sentinel(second) == Channel::SentinelResult::SlotBusy); + CHECK(second == "eof-2"); // not moved from + + ch.disable(); + std::string third = "eof-3"; + CHECK(ch.try_push_sentinel(third) == Channel::SentinelResult::Closed); + CHECK(third == "eof-3"); +} diff --git a/tests/test_channel_stress.cpp b/tests/test_channel_stress.cpp index eb58704..9f851e3 100644 --- a/tests/test_channel_stress.cpp +++ b/tests/test_channel_stress.cpp @@ -19,6 +19,7 @@ // Channel is SPSC: exactly one producer thread and one consumer thread per // channel. Every scenario below honours that contract. +#include #include #include #include @@ -279,3 +280,51 @@ TEST_CASE("SPSC: sentinel is strictly last, after every value (try_pop_now)", REQUIRE(ch.approx_size() == 0); } } + +// Contended: a producer offering sentinels while the consumer takes them. +// +// The old push_sentinel wrote eof_value_ with no regard for whether the +// consumer was reading it, so a second offer racing a take was a data race on +// the storage — for the shared_ptr form used by non-trivial types, on the +// refcount. Under TSan the old code reports it; the handshake added alongside +// this test makes the producer's write conditional on observing the slot free, +// which is what serialises the two. +// +// Payload is a std::string so the storage is the shared_ptr path rather than +// the trivially-copyable one, and each token carries its own identity so a torn +// value shows up as a mismatch rather than as a plausible-looking result. +TEST_CASE("SPSC: offering sentinels concurrently with takes is race-free", + "[channel][stress][sentinel]") { + constexpr int kRounds = 20000; + Channel ch(4); + + std::atomic taken{0}; + std::atomic torn{false}; + std::atomic done{false}; + + std::thread consumer([&] { + std::string out; + while (!done.load(std::memory_order_acquire) || ch.approx_size() > 0) { + if (ch.try_pop_now(out)) { + if (out.rfind("eof-", 0) != 0) torn.store(true, std::memory_order_relaxed); + taken.fetch_add(1, std::memory_order_relaxed); + } + } + }); + + int accepted = 0; + for (int i = 0; i < kRounds; ++i) { + std::string tok = "eof-" + std::to_string(i); + if (ch.try_push_sentinel(tok) == Channel::SentinelResult::Taken) + ++accepted; + } + done.store(true, std::memory_order_release); + consumer.join(); + + INFO("accepted " << accepted << " taken " << taken.load()); + CHECK_FALSE(torn.load(std::memory_order_relaxed)); + // Every accepted token must be delivered: the slot is refused while full, + // so acceptance and delivery are one-to-one. + CHECK(taken.load(std::memory_order_relaxed) == accepted); + CHECK(accepted > 0); +} From 0f277c0f98993bdf169ecdcb0ccdb98d661e77fd Mon Sep 17 00:00:00 2001 From: Duncan Tourolle Date: Wed, 5 Aug 2026 14:25:00 +0200 Subject: [PATCH 27/42] fix: the drain loops must terminate, and must drain the right channels MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit shutdown()'s drain step was an unbounded while (anything, anywhere, is non-empty) poll every channel Three defects in one loop. It drained the wrong thing. Stopping a node should wait for that node's own outputs before moving to the next layer; this waited for the entire graph to fall idle each time. The dynamic Network's version made it explicit — it took a node name and ignored it. Both now track which node feeds each probe and wait only on those. It had no deadline, so anything wedged downstream turned a graceful shutdown into the hang it exists to avoid. Now bounded two ways, because a stalled consumer and a slow one fail differently: a deadline for fill that never changes, and a no-progress counter that keeps waiting as long as the queue is shrinking, so a slow drain is not cut short merely for taking a while. And it could fail to terminate with nothing wedged at all. current_fill came from a snapshot that loaded tail_ before head_. A concurrent pop between the two reads yields a head_ past the sampled tail_, and the unsigned difference wraps to ~2^64 — so a poll for "is it empty yet" runs forever on a channel that is in fact empty. Both indices only ever increase, so loading head_ first can at worst under-report a concurrent push, which this loop tolerates and a wrap does not. size() and snapshot() are both corrected; size() feeds approx_size(), which is what node readiness checks call. Giving up is now reported rather than silent, because undrained data at that point is about to be discarded by the stop that follows, and a graceful shutdown quietly dropping values is the thing worth knowing about. Verified in both directions: with the old loop the new case is killed at a 30 s timeout; with this it returns in under a second, having reported four items its wedged consumer never took. Full suite 138/138. Note the drain timeout is per node and defaults to 5 s, so a graph of N stalled nodes can still take N x 5 s to shut down. That is a deliberate trade against cutting off legitimate slow drains, and set_drain_timeout() exists for callers who want it tighter. --- include/kpn/channel.hpp | 13 ++++-- include/kpn/network.hpp | 60 ++++++++++++++++++++------ include/kpn/static_network.hpp | 75 ++++++++++++++++++++++++++------ tests/test_static_network.cpp | 78 ++++++++++++++++++++++++++++++++++ 4 files changed, 197 insertions(+), 29 deletions(-) diff --git a/include/kpn/channel.hpp b/include/kpn/channel.hpp index a35651b..6ddad69 100644 --- a/include/kpn/channel.hpp +++ b/include/kpn/channel.hpp @@ -382,9 +382,15 @@ public: // Ring occupancy, derived lazily from indices — no separate counter on the // hot path. Excludes any out-of-band sentinel (that lives outside the ring). + // head_ is loaded first, deliberately. Both indices only ever increase, so + // reading head_ before tail_ can at worst under-report a concurrent push; + // the other order can read a head_ that has advanced past the tail_ already + // sampled, and the unsigned difference then wraps to ~2^64. A caller + // polling "is this channel empty yet" against that value never terminates. std::size_t size() const { - return tail_.load(std::memory_order_relaxed) - - head_.load(std::memory_order_relaxed); + const std::size_t h = head_.load(std::memory_order_relaxed); + const std::size_t t = tail_.load(std::memory_order_acquire); + return t - h; } // A pending out-of-band sentinel (EOF) counts as consumable work here even @@ -401,8 +407,9 @@ public: const ChannelStats& stats() const { return stats_; } ChannelSnapshot snapshot(const std::string& name) const { - const std::size_t t = tail_.load(std::memory_order_relaxed); + // head_ before tail_, for the reason given on size(). const std::size_t h = head_.load(std::memory_order_relaxed); + const std::size_t t = tail_.load(std::memory_order_acquire); return { name, capacity_, diff --git a/include/kpn/network.hpp b/include/kpn/network.hpp index 67686f3..281b9fa 100644 --- a/include/kpn/network.hpp +++ b/include/kpn/network.hpp @@ -91,6 +91,7 @@ public: + " → " + dst_name + ":" + std::to_string(DstIdx); channel_probes_.push_back( std::make_unique>(in_ch, ch_name)); + channel_src_names_.push_back(src_name); adj_[src_name].push_back(dst_name); return *this; @@ -213,6 +214,10 @@ public: watchdog_interval_ = interval; } + /// How long shutdown() waits for one node's outputs to drain before giving + /// up on them and stopping the next layer anyway. + void set_drain_timeout(std::chrono::milliseconds t) { drain_timeout_ = t; } + void set_error_handler(ErrorHandler h) { error_handler_ = std::move(h); } void set_diagnostics_handler(DiagnosticsHandler h) { diag_handler_ = std::move(h); } void set_event_handler(EventHandler h) { event_handler_ = std::move(h); } @@ -370,19 +375,46 @@ private: return true; } - void drain_output_channels(const std::string& /*name*/) const { - // Poll all channel probes until none report non-zero fill. - // A short sleep prevents busy-spin; 1 ms is fine for drain purposes. - bool any_full = true; - while (any_full) { - any_full = false; - for (auto& probe : channel_probes_) { - auto snap = probe->snapshot(); - if (snap.current_fill > 0) { any_full = true; break; } - } - if (any_full) - std::this_thread::sleep_for(std::chrono::milliseconds(1)); + /// Wait for the channels fed by `name` to empty, or give up. + /// + /// This took a node name and ignored it, polling *every* channel in the + /// graph instead — so shutdown() waited for the whole network to be idle + /// before stopping each successive layer. With no deadline either, anything + /// wedged downstream turned a graceful shutdown into the hang it exists to + /// avoid. + /// + /// Two bounds, because they fail differently. The deadline covers a + /// consumer that has stopped consuming, where fill never changes and + /// waiting cannot help. The no-progress counter covers one that is merely + /// slow: it keeps waiting while the queue is shrinking, so a slow drain is + /// not cut short just for taking a while. + void drain_output_channels(const std::string& name) const { + const auto deadline = clock_t::now() + drain_timeout_; + std::size_t last_fill = static_cast(-1); + int stalls = 0; + + auto fill_of = [&] { + std::size_t fill = 0; + for (std::size_t i = 0; i < channel_probes_.size(); ++i) + if (channel_src_names_[i] == name) + fill += channel_probes_[i]->snapshot().current_fill; + return fill; + }; + + for (;;) { + const std::size_t fill = fill_of(); + if (fill == 0) return; + if (fill >= last_fill) { if (++stalls > 100) break; } + else { stalls = 0; } + last_fill = fill; + if (clock_t::now() >= deadline) break; + std::this_thread::sleep_for(std::chrono::milliseconds(1)); } + + if (const std::size_t left = fill_of()) + std::cerr << "[kpn] shutdown: '" << name << "' still has " << left + << " queued item(s) its consumer did not take; " + "they are discarded\n"; } // ── Cycle detection / topological sort ─────────────────────────────────── @@ -447,6 +479,10 @@ private: std::map exposed_outputs_; std::set> connected_outputs_; std::vector> channel_probes_; + /// Name of the node feeding each probe, parallel to channel_probes_. + /// shutdown() drains a node's own outputs, so it has to know which they are. + std::vector channel_src_names_; + std::chrono::milliseconds drain_timeout_{5000}; std::vector> pool_probes_; ErrorHandler error_handler_; DiagnosticsHandler diag_handler_; diff --git a/include/kpn/static_network.hpp b/include/kpn/static_network.hpp index 9c2dc32..5bdb9ac 100644 --- a/include/kpn/static_network.hpp +++ b/include/kpn/static_network.hpp @@ -98,13 +98,15 @@ public: std::vector fanout_ptrs, std::vector user_node_names, std::vector fanout_node_names, - std::vector> channel_probes) + std::vector> channel_probes, + std::vector channel_src_names) : fanouts_(std::move(fanouts)) , user_nodes_topo_(std::move(user_nodes_topo)) , fanout_nodes_ptr_(std::move(fanout_ptrs)) , user_node_names_(std::move(user_node_names)) , fanout_node_names_(std::move(fanout_node_names)) , channel_probes_(std::move(channel_probes)) + , channel_src_names_(std::move(channel_src_names)) {} ~StaticNetwork() override { stop(); } @@ -173,9 +175,9 @@ public: #endif // user_nodes_topo_ is already in sources-first order. // Stop each node and drain its output channels before moving on. - for (auto* n : user_nodes_topo_) { - n->stop(); - drain_all_channels(); + for (std::size_t i = 0; i < user_nodes_topo_.size(); ++i) { + user_nodes_topo_[i]->stop(); + drain_outputs_of(user_node_names_[i]); } for (auto* n : fanout_nodes_ptr_) n->stop(); } @@ -194,6 +196,10 @@ public: void set_event_handler(EventHandler h) { event_handler_ = std::move(h); } + /// How long shutdown() waits for one node's outputs to drain before giving + /// up on them and stopping the next layer anyway. + void set_drain_timeout(std::chrono::milliseconds t) { drain_timeout_ = t; } + /// Application-level error listener. Receives the exception any node's /// function throws, after that node's own handler (if any) declined it. /// Return true to skip the failed invocation and keep the node running, @@ -274,16 +280,50 @@ private: return {std::move(nodes), std::move(channels), std::move(resources), std::move(pools), elapsed_s}; } - void drain_all_channels() const { - bool any_full = true; - while (any_full) { - any_full = false; - for (auto& probe : channel_probes_) { - if (probe->snapshot().current_fill > 0) { any_full = true; break; } - } - if (any_full) - std::this_thread::sleep_for(std::chrono::milliseconds(1)); + /// Wait for the channels fed by `src` to empty, or give up. + /// + /// This was an unbounded `while (anything anywhere is non-empty)` poll over + /// *every* channel in the graph, which made shutdown() wait for the whole + /// network to be idle before stopping each successive layer, and wait + /// forever if anything downstream was wedged — turning a graceful shutdown + /// into the hang it exists to avoid. + /// + /// Two bounds, because they fail differently. The deadline covers a + /// consumer that has stopped consuming: fill never changes and no amount of + /// waiting helps. The no-progress counter covers a consumer that is merely + /// slow — it keeps waiting as long as the queue is shrinking, so a slow + /// drain is not cut short just for exceeding a fixed time. + /// + /// Giving up is reported rather than silent: undrained data at this point + /// means values are about to be discarded by the stop that follows. + void drain_outputs_of(const std::string& src) const { + const auto deadline = clock_t::now() + drain_timeout_; + std::size_t last_fill = static_cast(-1); + int stalls = 0; + + for (;;) { + std::size_t fill = 0; + for (std::size_t i = 0; i < channel_probes_.size(); ++i) + if (channel_src_names_[i] == src) + fill += channel_probes_[i]->snapshot().current_fill; + + if (fill == 0) return; + if (fill >= last_fill) { if (++stalls > 100) break; } + else { stalls = 0; } + last_fill = fill; + + if (clock_t::now() >= deadline) break; + std::this_thread::sleep_for(std::chrono::milliseconds(1)); } + + std::size_t left = 0; + for (std::size_t i = 0; i < channel_probes_.size(); ++i) + if (channel_src_names_[i] == src) + left += channel_probes_[i]->snapshot().current_fill; + if (left) + std::cerr << "[kpn] shutdown: '" << src << "' still has " << left + << " queued item(s) its consumer did not take; " + "they are discarded\n"; } std::string name_; @@ -294,6 +334,10 @@ private: std::vector user_node_names_; std::vector fanout_node_names_; std::vector> channel_probes_; + /// Display name of the node feeding each probe, parallel to channel_probes_. + /// shutdown() drains a node's own outputs, so it has to know which they are. + std::vector channel_src_names_; + std::chrono::milliseconds drain_timeout_{5000}; std::vector> resource_probes_; std::vector> pool_probes_; EventHandler event_handler_; @@ -401,6 +445,7 @@ auto make_network(Edges&&... edges) { }; std::vector> channel_probes; + std::vector channel_src_names; auto wire_one = [&](SE) { using SrcNode = typename SE::src_node_t; @@ -418,6 +463,7 @@ auto make_network(Edges&&... edges) { + " \xe2\x86\x92 " // UTF-8 → + node_name.template operator()() + ":" + std::to_string(DstIdx); channel_probes.push_back(std::make_unique>(ch, ch_name)); + channel_src_names.push_back(node_name.template operator()()); } }; @@ -445,7 +491,8 @@ auto make_network(Edges&&... edges) { std::move(fanout_ptrs), std::move(user_node_names), std::move(fanout_node_names), - std::move(channel_probes)); + std::move(channel_probes), + std::move(channel_src_names)); } } // namespace kpn diff --git a/tests/test_static_network.cpp b/tests/test_static_network.cpp index 3dc2b26..4b1e780 100644 --- a/tests/test_static_network.cpp +++ b/tests/test_static_network.cpp @@ -271,3 +271,81 @@ TEST_CASE("static_network: fanout with labelled same-function consumers", "[stat REQUIRE(outB.pop() == 7); net.stop(); } + +// Regression: shutdown() must return even when a consumer stopped consuming. +// +// The drain step was an unbounded `while (anything anywhere is non-empty)` poll +// over *every* channel in the graph. Two defects in one loop: it waited for the +// whole network to be idle before stopping each successive layer rather than +// just the node it had stopped — the dynamic Network's version even took a node +// name and ignored it — and it had no deadline, so anything wedged downstream +// turned a graceful shutdown into the hang it exists to avoid. +// +// It could also fail to terminate with nothing wedged at all. current_fill came +// from a snapshot that loaded tail_ before head_; a concurrent pop between the +// two reads yields a head_ past the sampled tail_, and the unsigned difference +// wraps to ~2^64. Any poll for "is it empty yet" against that value runs +// forever. Both indices only ever increase, so loading head_ first can at worst +// under-report a push, which this loop tolerates and a wrap does not. +// +// Here the sink never takes anything, so its input cannot drain and the only +// correct outcome is to give up and say so. The bound asserted is deliberately +// loose: the point is that it terminates, not how fast. +namespace { + +struct DrainSource { + static constexpr std::string_view label() { return "drain_source"; } + int n{0}; + int operator()() { + std::this_thread::sleep_for(std::chrono::microseconds(100)); + return n++; + } +}; + +struct NeverConsumes { + static constexpr std::string_view label() { return "never_consumes"; } + std::atomic* wedged; + void operator()(int) { + // Blocks for the duration of the test: the input channel behind it + // fills and stays full. + while (!wedged->load(std::memory_order_acquire)) + std::this_thread::sleep_for(std::chrono::milliseconds(1)); + } +}; + +} // namespace + +TEST_CASE("shutdown returns when a consumer has wedged", "[static_network][shutdown]") { + std::atomic release{false}; + + DrainSource src_fn; + NeverConsumes sink_fn{&release}; + + kpn::ObjectNode, kpn::out<"v">, "drain_source", 0> s(src_fn, 4); + kpn::ObjectNode, kpn::out<>, "never_consumes", 0> k(sink_fn, 4); + + auto net = kpn::make_network(kpn::edge(s.output<"v">(), k.input<"v">())); + net.set_drain_timeout(std::chrono::milliseconds(100)); + net.start(); + + // Let the channel fill and the sink jam. + std::this_thread::sleep_for(std::chrono::milliseconds(100)); + + // Unjam the sink well after the drain timeout should have expired. Stopping + // a node joins its worker, so a sink blocked forever would hang the test in + // stop() rather than in the drain loop this case is about. + std::thread unjam([&] { + std::this_thread::sleep_for(std::chrono::milliseconds(800)); + release.store(true, std::memory_order_release); + }); + + const auto t0 = std::chrono::steady_clock::now(); + net.shutdown(); + const auto elapsed = std::chrono::steady_clock::now() - t0; + + unjam.join(); + + const auto ms = std::chrono::duration_cast(elapsed).count(); + INFO("shutdown took " << ms << " ms"); + CHECK(ms < 3000); // unbounded before; one 100 ms drain timeout after +} From abbb2d47700b38b7147b7c98e022bf951add7284 Mon Sep 17 00:00:00 2001 From: Duncan Tourolle Date: Wed, 5 Aug 2026 15:14:06 +0200 Subject: [PATCH 28/42] fix: submitting to a stopped pool must be refused, not fatal MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ThreadPool::stop() ends with queues_.clear(), and submit() went straight to queues_[target] with no check. A submission arriving after stop indexed an empty vector and segfaulted. This is not a contrived teardown ordering. A node's space callback fires from whichever thread drained the channel, and that thread belongs to the *consumer*; the callback it runs belongs to the *producer*. Stop the producer first — which a sources-first shutdown does by design — and the consumer keeps draining its backlog, firing the producer's space callback into a pool that has already been torn down: ThreadPool::submit <- source node's space_callback <- Channel::try_pop_now (relay draining its input) <- relay fire_once The static-network shutdown case in the next commit crashed about 12 runs in 20 on this. It survived until now because halt() stops in reverse topological order — consumers first — so the producer whose callback might fire is always still alive. shutdown() stops sources first and does not have that protection. Reading stopped_ without a lock would not fix it: the window between the read and the indexing is exactly where clear() runs. submit() takes a shared lock and stop() an exclusive one, so submissions still proceed in parallel with each other while being serialised against teardown. stop() sets the flag under the lock, releases it to join — a worker's task may itself call submit, and holding the lock across the join would deadlock against that — then retakes it to destroy the queues. Refusals are counted rather than silent. A teardown race is expected, but a node repeatedly trying to run after its pool is gone is worth being able to see. try_submit also checks stop_flag_ first, so a stopped node cannot claim the submit gate and leave it held. Not a smart-pointer problem, for anyone reading the crash: nothing here is owned by a raw pointer. It is std::vector::operator[] on a vector that was emptied by another thread. Verified in both directions: with the guard removed the new scheduler cases segfault; with it they pass. --- include/kpn/pool_node.hpp | 12 +++++++++ include/kpn/scheduler.hpp | 42 ++++++++++++++++++++++++++++- tests/test_scheduler.cpp | 57 +++++++++++++++++++++++++++++++++++++++ 3 files changed, 110 insertions(+), 1 deletion(-) diff --git a/include/kpn/pool_node.hpp b/include/kpn/pool_node.hpp index 19a32be..73ae612 100644 --- a/include/kpn/pool_node.hpp +++ b/include/kpn/pool_node.hpp @@ -364,6 +364,12 @@ private: /// re-trigger it. SubmitGate makes "idle" and "wake outstanding" the same /// variable, so the two cannot both be true — see submit_gate.hpp. void try_submit(float priority) { + // A stopped node must not claim the gate. The scheduler now refuses + // submissions after its pool stops, so the submit itself is safe — but + // claiming and never releasing would leave the gate held, and a restart + // would then have to clear it. start() does, but relying on that makes + // the invariant depend on a distant statement. + if (stop_flag_.load(std::memory_order_relaxed)) return; if (gate_.claim()) scheduler_->submit([this] { fire_once(); }, priority); } @@ -902,6 +908,12 @@ private: /// re-trigger it. SubmitGate makes "idle" and "wake outstanding" the same /// variable, so the two cannot both be true — see submit_gate.hpp. void try_submit(float priority) { + // A stopped node must not claim the gate. The scheduler now refuses + // submissions after its pool stops, so the submit itself is safe — but + // claiming and never releasing would leave the gate held, and a restart + // would then have to clear it. start() does, but relying on that makes + // the invariant depend on a distant statement. + if (stop_flag_.load(std::memory_order_relaxed)) return; if (gate_.claim()) scheduler_->submit([this] { fire_once(); }, priority); } diff --git a/include/kpn/scheduler.hpp b/include/kpn/scheduler.hpp index e7920bf..61c57bd 100644 --- a/include/kpn/scheduler.hpp +++ b/include/kpn/scheduler.hpp @@ -5,6 +5,7 @@ #include #include #include +#include #include #include #include @@ -62,7 +63,12 @@ public: } void stop() override { - stopped_.store(true, std::memory_order_seq_cst); + // Close the pool to new work before touching anything, and do it under + // the lifecycle lock so no submit() is midway through indexing queues_. + { + std::unique_lock lk(lifecycle_mx_); + stopped_.store(true, std::memory_order_seq_cst); + } for (auto& q : queues_) { std::lock_guard lock(q->mx); std::size_t discarded = q->pq.size(); @@ -73,7 +79,13 @@ public: // gap between a worker's predicate check and its wait() (see submit()). { std::lock_guard lk(cv_mx_); } cv_.notify_all(); + // Join without the lock: a worker's task may call submit(), which takes + // it shared, and holding it here would deadlock against that. for (auto& t : workers_) if (t.joinable()) t.join(); + // Destroying the queues is what submit() must never race. By now + // stopped_ is published, so any submit() that acquires the lock after + // this point returns without touching them. + std::unique_lock lk(lifecycle_mx_); workers_.clear(); queues_.clear(); } @@ -86,6 +98,22 @@ public: } void submit(std::function task, float priority = 0.5f) override { + // A submission can arrive after this pool has been stopped, and did so + // by an ordinary route: a node's space callback fires from whichever + // thread drained the channel, which belongs to the *consumer*. Stop the + // producer first — as a sources-first shutdown does — and the consumer + // keeps draining its backlog, firing the producer's space callback into + // a pool whose stop() has already run queues_.clear(). submit() then + // indexed an empty vector: a segfault, reproducible about 12 runs in 20. + // + // The shared lock is what makes the check meaningful. Reading stopped_ + // alone leaves the window between the read and the indexing, which is + // precisely where stop() clears the vector. + std::shared_lock lk(lifecycle_mx_); + if (stopped_.load(std::memory_order_acquire) || queues_.empty()) { + rejected_.fetch_add(1, std::memory_order_relaxed); + return; + } std::size_t target = next_.fetch_add(1, std::memory_order_relaxed) % thread_count_; { std::lock_guard lock(queues_[target]->mx); @@ -105,6 +133,9 @@ public: std::size_t thread_count() const { return thread_count_; } + /// Submissions dropped because the pool was stopped. See rejected_. + uint64_t rejected() const { return rejected_.load(std::memory_order_relaxed); } + // ── IPoolProbe ──────────────────────────────────────────────────────────── PoolSnapshot snapshot(const std::string& name) const override { @@ -194,6 +225,11 @@ private: std::vector> queues_; std::vector workers_; + /// Guards the lifetime of queues_/workers_ against a concurrent submit(). + /// Shared by submit, exclusive by stop, so submissions still run in + /// parallel with each other. + mutable std::shared_mutex lifecycle_mx_; + std::mutex cv_mx_; std::condition_variable cv_; std::mutex drain_mx_; @@ -205,6 +241,10 @@ private: std::atomic next_{0}; // round-robin submit cursor std::atomic seq_{0}; // tie-break for equal-priority tasks std::atomic submitted_{0}; + /// Submissions refused because the pool was already stopped. Not an error — + /// teardown races are expected — but silence here would hide a node that + /// keeps trying to run after its pool is gone. + std::atomic rejected_{0}; std::atomic completed_{0}; }; diff --git a/tests/test_scheduler.cpp b/tests/test_scheduler.cpp index 18f3fdd..d011129 100644 --- a/tests/test_scheduler.cpp +++ b/tests/test_scheduler.cpp @@ -227,3 +227,60 @@ TEST_CASE("work stealing: tasks complete with more threads than initial queue ta REQUIRE(counter.load() == 4); pool.stop(); } + +// Regression: submitting to a stopped pool must be a no-op, not a segfault. +// +// stop() ends with queues_.clear(), and submit() went straight to +// queues_[target] with no check — so a submission arriving after stop indexed +// an empty vector. +// +// This is not a contrived teardown ordering; it happens on a normal path. A +// node's space callback fires from whichever thread drained the channel, and +// that thread belongs to the *consumer*. Stop the producer first — which a +// sources-first shutdown does by design — and the consumer keeps draining its +// backlog, firing the producer's space callback into a pool that has already +// been torn down. Before this fix the static-network shutdown case crashed +// about 12 runs in 20. +// +// Checking stopped_ without the lock would not be enough: the window between +// reading the flag and indexing the vector is exactly where clear() runs. +TEST_CASE("submitting to a stopped pool is refused, not fatal", "[scheduler]") { + ThreadPool pool(2); + pool.start(); + pool.stop(); + + std::atomic ran{0}; + for (int i = 0; i < 10; ++i) + pool.submit([&] { ran.fetch_add(1, std::memory_order_relaxed); }); + + CHECK(ran.load(std::memory_order_relaxed) == 0); + CHECK(pool.rejected() == 10); +} + +TEST_CASE("submitting while the pool stops does not crash", "[scheduler]") { + // The racing form of the case above: a producer thread submitting + // continuously while stop() runs underneath it. Nothing is asserted about + // how many tasks run — the point is that every submission either enqueues + // or is refused, and none touches a destroyed queue. + for (int rep = 0; rep < 20; ++rep) { + ThreadPool pool(4); + pool.start(); + + std::atomic go{false}; + std::atomic ran{0}; + std::thread submitter([&] { + while (!go.load(std::memory_order_acquire)) {} + for (int i = 0; i < 2000; ++i) + pool.submit([&] { ran.fetch_add(1, std::memory_order_relaxed); }); + }); + + go.store(true, std::memory_order_release); + std::this_thread::sleep_for(std::chrono::microseconds(200)); + pool.stop(); + submitter.join(); + + // Everything submitted was either executed or refused; nothing vanished + // into a queue that no longer existed. + CHECK(pool.rejected() + pool.snapshot("p").tasks_completed <= 2000); + } +} From 87c5f98d04d50fe5aec05b6852a890d4e1ebb350 Mon Sep 17 00:00:00 2001 From: Duncan Tourolle Date: Wed, 5 Aug 2026 15:14:35 +0200 Subject: [PATCH 29/42] fix: start and stop in the topological order that was computed MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit make_network computes Topo for the cycle check and then discarded it. The node vector was filled in edge-declaration order — and then named user_nodes_topo_ and relied upon as if it were sorted. halt() stops in its reverse, and shutdown() walks it forwards stopping each node and draining its outputs before the next, which is a graceful drain only if the order really is sources-first. It held for every network in this tree because edges happen to be declared in pipeline order, so the two coincided. Declared any other way — which is legal and which make_network otherwise accepts in silence — shutdown stops a consumer before its producer and discards whatever was queued in front of it. The order now comes from Topo::topo, which is already sources-first. Fanout nodes appear there too and are skipped, since they are owned separately in fanout_storage; they are still started after the user nodes and stopped before them, so a fanout sitting between two user nodes is not staged precisely during a drain. That is a smaller gap than the one being closed and is left alone rather than restructured on the way past. The test declares the sink edge first and the source edge last, and asserts through shutdown() rather than by reading the order back — the order is private, and what it buys is the point. A sources-first shutdown lets the backlog queued in front of the slow relay reach the sink; stopping the relay first discards all of it. Worth recording how this went, because it is the more useful half: the new test segfaulted, 12 runs in 20. Not a fault in the ordering change — it was stopping sources first that finally put a live consumer behind a dead producer, which is the condition the previous commit's crash needs. The ordering fix did not introduce that bug, it made it reachable. Verified in both directions: with declaration order the sink receives nothing after shutdown begins; with topological order it receives the whole backlog. --- include/kpn/static_network.hpp | 55 ++++++++++++---------- tests/test_static_network.cpp | 83 ++++++++++++++++++++++++++++++++++ 2 files changed, 115 insertions(+), 23 deletions(-) diff --git a/include/kpn/static_network.hpp b/include/kpn/static_network.hpp index 5bdb9ac..75ad6e6 100644 --- a/include/kpn/static_network.hpp +++ b/include/kpn/static_network.hpp @@ -374,29 +374,6 @@ auto make_network(Edges&&... edges) { // 4. Construct owned fanout storage on the heap (FanoutNode has jthread — not moveable) auto fanout_storage = std::make_unique(); - // 5. Collect unique user node pointers + their display names, in edge-declaration order - std::vector user_node_ptrs; - std::vector user_node_names; - auto collect = [&](auto& e) { - using SrcT = std::decay_t; - using DstT = std::decay_t; - auto* s = static_cast(&e.src); - auto* d = static_cast(&e.dst); - if (std::find(user_node_ptrs.begin(), user_node_ptrs.end(), s) == user_node_ptrs.end()) { - auto sname = node_display_name(); - user_node_ptrs.push_back(s); - user_node_names.push_back(sname); - s->set_name(sname); - } - if (std::find(user_node_ptrs.begin(), user_node_ptrs.end(), d) == user_node_ptrs.end()) { - auto dname = node_display_name(); - user_node_ptrs.push_back(d); - user_node_names.push_back(dname); - d->set_name(dname); - } - }; - (collect(edges), ...); - // 5. Wire all expanded SimpleEdges. // find_node: searches fanout storage then user edge pack, returns NodeT*. // Uses if constexpr in a fold so mismatched types never reach assignment. @@ -421,6 +398,38 @@ auto make_network(Edges&&... edges) { return ptr; }; + // 5. Collect user node pointers + display names in *topological* order. + // + // Topo is computed above for the cycle check and used to be discarded, + // while this vector was filled in edge-declaration order — and then named + // user_nodes_topo_ and relied upon as if it were sorted. halt() stops in + // its reverse, and shutdown() walks it forwards stopping each node and + // draining its outputs before the next, which is only a graceful drain if + // the order really is sources-first. It held for every network in the tree + // because edges happen to be declared in pipeline order, and would have + // broken silently for one that was not. + // + // Fanout nodes appear in Topo too; they are skipped here because they are + // owned separately, in fanout_storage. + std::vector user_node_ptrs; + std::vector user_node_names; + [&](tmp::TypeList) { + ([&]() { + if constexpr (!requires { NodeT::is_fanout_node; }) { + if (auto* p = find_node.template operator()()) { + auto* n = static_cast(p); + if (std::find(user_node_ptrs.begin(), user_node_ptrs.end(), n) + == user_node_ptrs.end()) { + auto nm = node_display_name(); + user_node_ptrs.push_back(n); + user_node_names.push_back(nm); + n->set_name(nm); + } + } + } + }.template operator()(), ...); + }(typename Topo::topo{}); + // Pre-pass: build fanout_id → source display name map so fanout nodes // can be named after the node feeding them (e.g. "capture_fanout"). std::map fanout_src_name; diff --git a/tests/test_static_network.cpp b/tests/test_static_network.cpp index 4b1e780..188de84 100644 --- a/tests/test_static_network.cpp +++ b/tests/test_static_network.cpp @@ -349,3 +349,86 @@ TEST_CASE("shutdown returns when a consumer has wedged", "[static_network][shutd INFO("shutdown took " << ms << " ms"); CHECK(ms < 3000); // unbounded before; one 100 ms drain timeout after } + +// Regression: node order must come from the topological sort, not from the +// order the edges happened to be written in. +// +// make_network computes Topo for the cycle check and then dropped it, filling +// the node vector in edge-declaration order — and named it user_nodes_topo_. +// halt() stops in its reverse, and shutdown() walks it forwards stopping each +// node and draining its outputs before moving to the next, which is a graceful +// drain only if the order really is sources-first. +// +// Every network in this tree declares edges in pipeline order, so the two +// coincided and nothing failed. This case declares them backwards, which is +// legal and which make_network otherwise accepts silently. +// +// Asserted through shutdown() rather than by reading the order back, because +// the order is private and the ordering is not the point — what it buys is. +// A sources-first shutdown lets the values already in flight reach the sink; +// stopping the sink first strands them, and the drain step then has nobody +// left to take them. +namespace { + +struct OrderSource { + static constexpr std::string_view label() { return "order_source"; } + std::atomic* made; + int operator()() { + std::this_thread::sleep_for(std::chrono::microseconds(20)); + return made->fetch_add(1, std::memory_order_relaxed); + } +}; + +// Deliberately slower than the source, so a deep backlog builds up in its input +// channel. That backlog is what a sources-first shutdown preserves and a +// sink-first one throws away, and it needs to be big enough that the difference +// cannot be mistaken for one value in flight. +struct OrderRelay { + static constexpr std::string_view label() { return "order_relay"; } + int operator()(int v) { + std::this_thread::sleep_for(std::chrono::microseconds(300)); + return v; + } +}; + +struct OrderSink { + static constexpr std::string_view label() { return "order_sink"; } + std::atomic* seen; + void operator()(int) { seen->fetch_add(1, std::memory_order_relaxed); } +}; + +} // namespace + +TEST_CASE("edges declared out of order still start and stop sources-first", + "[static_network][shutdown]") { + std::atomic seen{0}, made{0}; + + OrderSource src_fn{&made}; + OrderRelay relay_fn; + OrderSink sink_fn{&seen}; + + kpn::ObjectNode, kpn::out<"v">, "order_source", 0> s(src_fn, 8); + kpn::ObjectNode, kpn::out<"w">, "order_relay", 0> r(relay_fn, 64); + kpn::ObjectNode, kpn::out<>, "order_sink", 0> k(sink_fn, 64); + + // Sink edge first, source edge last — the reverse of pipeline order. + auto net = kpn::make_network( + kpn::edge(r.output<"w">(), k.input<"w">()), + kpn::edge(s.output<"v">(), r.input<"v">()) + ); + net.start(); + std::this_thread::sleep_for(std::chrono::milliseconds(300)); + + const int before = seen.load(std::memory_order_relaxed); + REQUIRE(before > 0); // the pipeline ran at all + + net.shutdown(); + + // Sources stop first and each layer drains before the next stops, so the + // backlog queued in front of the relay still reaches the sink. Stopping in + // declaration order stops the relay first and discards all of it. + const int after = seen.load(std::memory_order_relaxed); + INFO("made " << made.load() << ", delivered " << before + << " before shutdown, " << after << " after"); + CHECK(after - before >= 20); +} From b9698fae60dc246dea67af28d0570df2416e582c Mon Sep 17 00:00:00 2001 From: Duncan Tourolle Date: Wed, 5 Aug 2026 15:34:33 +0200 Subject: [PATCH 30/42] fix: idle workers must sleep while another worker is busy MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The wait predicate was `stopped_ || total_ > 0`, and total_ counts queued *plus executing*. So while any one task ran, every other worker's predicate was true: wait() returned immediately and the worker spun through try_pop, try_steal and back to wait at full speed, try_lock-ing every peer queue on each pass. Measured on this tree, 8 workers and one 300 ms task: 1991 ms of CPU and 19205 voluntary context switches, against 0.4 ms and 10 with the fix. Call it six and a half cores burned for the duration of one sleeping task. Sleeping requires "no work is *waiting*", which total_ cannot express, so queued_ is now tracked separately: incremented on submit, decremented when a task leaves a queue, and adjusted for the tasks stop() discards. total_ stays as it was for drain(), which genuinely does need to know about executing work. The worker exit condition moves to queued_ for the same reason — waiting for total_ to reach zero meant waiting for someone else's task to finish, which a worker cannot help with and would spin through until it did. PoolSnapshot's queue depth stops being an estimate as a side effect. Latent for this pipeline, where each node owns a private single-thread pool and there is no idle peer to spin. Any use of a shared pool, which make_pool_node exists for, hits it immediately. Reproducing it needs the right trigger, and the test says so, because my first attempt got it wrong and passed against the bug: a worker that has never been woken stays blocked in wait() and never re-evaluates the predicate. The spin only appears once a worker *finishes* something and re-enters the loop while a peer is still busy, so the case submits one long task plus a trivial one per remaining worker. Submitting only the long task measures nothing. Verified in both directions: 1969 ms of CPU before, 0.35 ms after, against a 300 ms threshold. Full suite 142/142. --- include/kpn/scheduler.hpp | 27 ++++++++++++++----- tests/test_scheduler.cpp | 56 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 77 insertions(+), 6 deletions(-) diff --git a/include/kpn/scheduler.hpp b/include/kpn/scheduler.hpp index 61c57bd..54bbb5b 100644 --- a/include/kpn/scheduler.hpp +++ b/include/kpn/scheduler.hpp @@ -74,6 +74,7 @@ public: std::size_t discarded = q->pq.size(); while (!q->pq.empty()) q->pq.pop(); total_.fetch_sub(discarded, std::memory_order_relaxed); + queued_.fetch_sub(discarded, std::memory_order_relaxed); } // Lock cv_mx_ before notifying so the stop signal can't be lost in the // gap between a worker's predicate check and its wait() (see submit()). @@ -121,6 +122,7 @@ public: {std::move(task), priority, seq_.fetch_add(1, std::memory_order_relaxed)}); } total_.fetch_add(1, std::memory_order_relaxed); + queued_.fetch_add(1, std::memory_order_relaxed); submitted_.fetch_add(1, std::memory_order_relaxed); // Synchronize with worker_loop's predicate evaluation: taking cv_mx_ // here guarantees a worker is either before its predicate check (and @@ -140,11 +142,10 @@ public: PoolSnapshot snapshot(const std::string& name) const override { std::size_t a = active_.load(std::memory_order_relaxed); - std::size_t t = total_.load(std::memory_order_relaxed); return { name, thread_count_, - t > a ? t - a : 0, // queued (approximate) - a, // executing + queued_.load(std::memory_order_relaxed), // queued (exact) + a, // executing submitted_.load(std::memory_order_relaxed), completed_.load(std::memory_order_relaxed), }; @@ -173,6 +174,7 @@ private: if (q.pq.empty()) return std::nullopt; auto fn = std::move(const_cast(q.pq.top()).fn); q.pq.pop(); + queued_.fetch_sub(1, std::memory_order_relaxed); return fn; } @@ -213,10 +215,13 @@ private: std::unique_lock lock(cv_mx_); cv_.wait(lock, [this] { return stopped_.load(std::memory_order_seq_cst) - || total_.load(std::memory_order_relaxed) > 0; + || queued_.load(std::memory_order_relaxed) > 0; }); + // Exit on queued_, not total_: waiting for total_ to reach zero + // meant waiting for someone else's task to finish, which this + // worker cannot help with and would spin through until it did. if (stopped_.load(std::memory_order_seq_cst) - && total_.load(std::memory_order_relaxed) == 0) + && queued_.load(std::memory_order_relaxed) == 0) return; } } @@ -236,7 +241,17 @@ private: std::condition_variable drain_cv_; std::atomic stopped_{true}; - std::atomic total_{0}; // queued + executing + std::atomic total_{0}; // queued + executing (drain() waits on this) + /// Queued only — never counts a task that is already executing. + /// + /// The wait predicate used total_, which includes running tasks, so while + /// any one task ran every *other* worker's predicate was true: wait() + /// returned instantly and the worker spun through try_pop / try_steal / + /// wait at full speed, try_lock-ing every peer queue on each pass. One slow + /// task therefore pinned every other core and contended the very mutexes + /// the working thread needed. Sleeping requires "no work is *waiting*", + /// which is this. + std::atomic queued_{0}; // waiting to run std::atomic active_{0}; // executing only (for snapshot) std::atomic next_{0}; // round-robin submit cursor std::atomic seq_{0}; // tie-break for equal-priority tasks diff --git a/tests/test_scheduler.cpp b/tests/test_scheduler.cpp index d011129..9af89b9 100644 --- a/tests/test_scheduler.cpp +++ b/tests/test_scheduler.cpp @@ -284,3 +284,59 @@ TEST_CASE("submitting while the pool stops does not crash", "[scheduler]") { CHECK(pool.rejected() + pool.snapshot("p").tasks_completed <= 2000); } } + +// Regression: idle workers must sleep while another worker is busy. +// +// The wait predicate was `stopped_ || total_ > 0`, and total_ counts queued +// *plus executing*. So while any one task ran, every other worker's predicate +// was true: wait() returned immediately and the worker spun through try_pop, +// try_steal and back to wait at full speed — try_lock-ing every peer queue on +// each pass. One slow task pinned every other core and contended the very +// mutexes the working thread needed to make progress. +// +// That is the shape of this pipeline's load exactly: a handful of nodes whose +// work is tens of milliseconds of ONNX inference. It was latent only because +// each node currently owns a private single-thread pool, where there is no +// idle peer to spin. Any use of a shared pool — which make_pool_node exists +// for — hits it immediately. +// +// Measured as CPU time rather than wall time, because the bug does not make +// anything slower to finish; it makes seven cores burn while one works. A +// sleeping task consumes no CPU, so with workers correctly asleep the whole +// pool should account for almost none. +TEST_CASE("idle workers do not spin while one task runs", "[scheduler]") { + auto cpu_ms = [] { + struct timespec ts{}; + clock_gettime(CLOCK_PROCESS_CPUTIME_ID, &ts); + return ts.tv_sec * 1000.0 + ts.tv_nsec / 1e6; + }; + + constexpr int kThreads = 8; + constexpr int kWorkMs = 300; + + ThreadPool pool(kThreads); + pool.start(); + std::this_thread::sleep_for(20ms); // let workers reach the wait + + const double before = cpu_ms(); + + // One long task plus a trivial one per remaining worker. The trivial ones + // matter: a worker that has never been woken stays blocked in wait() and + // never re-evaluates the predicate, so the spin only appears once a worker + // *finishes* something and re-enters the loop while a peer is still busy. + // Submitting only the long task does not reproduce it. + pool.submit([&] { std::this_thread::sleep_for(std::chrono::milliseconds(kWorkMs)); }); + for (int i = 0; i < kThreads - 1; ++i) pool.submit([] {}); + + pool.drain(); + const double used = cpu_ms() - before; + pool.stop(); + + // Measured on this tree: 1991 ms of CPU with the total_ predicate against + // 0.4 ms with queued_, and 19205 voluntary context switches against 10 — + // roughly (kThreads - 1) cores burned for the duration of one sleeping + // task. The threshold sits far from both so the case is not sensitive to + // how loaded the machine is. + INFO("cpu " << used << " ms over " << kWorkMs << " ms of sleeping work"); + CHECK(used < kWorkMs); +} From 97670d8ba32053cf78750bfe53cf30751f9435ba Mon Sep 17 00:00:00 2001 From: Duncan Tourolle Date: Wed, 5 Aug 2026 15:46:27 +0200 Subject: [PATCH 31/42] fix: stop() must not return while a firing is still running MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit stop() set the flag, disabled the inputs and returned, leaving an executing fire_once touching input_channels_, stats_ and pending_ while the caller went on to destroy them. The comment was explicit about it: callers wanting the guarantee should call scheduler_->drain() first. But ~PoolNode calls stop(), and a destructor cannot ask its caller to have done that. A node with a private pool survived by accident, because Node::stop() calls pool->stop() and that joins the worker. A node sharing a pool — which make_pool_node exists to create — had nothing joining it, so its own destructor raced the firing. stop() now waits on the submit gate, which is claimed for the whole of a firing and released as its last act. A queued but unstarted firing also holds it and will run, observe stop_flag_ and release, so the pool must still be running when stop() is called; that is already the documented order and what Node/ObjectNode do. Two ways it declines to wait. It is bounded at five seconds, because a node function that never returns must not convert teardown into a hang — it warns and continues. And it returns immediately when called from the firing thread itself, since an error handler that stops its own node would otherwise wait for a firing that is waiting for it. Verified in both directions: with the wait removed, stop() returns while the node function is still sleeping and the flag it sets on the way out is still false. 143/143. --- include/kpn/pool_node.hpp | 107 ++++++++++++++++++++++++++++++++++++-- tests/test_pool_node.cpp | 52 ++++++++++++++++++ 2 files changed, 156 insertions(+), 3 deletions(-) diff --git a/include/kpn/pool_node.hpp b/include/kpn/pool_node.hpp index 73ae612..c875b9b 100644 --- a/include/kpn/pool_node.hpp +++ b/include/kpn/pool_node.hpp @@ -117,9 +117,7 @@ public: void stop() override { stop_flag_.store(true, std::memory_order_seq_cst); disable_inputs(std::make_index_sequence{}); - // fire_once() observes stop_flag_ and will not resubmit. - // We do not wait for an in-flight fire_once() to complete here; - // callers that need that guarantee should call scheduler_->drain() first. + await_quiescence(); } bool running() const override { @@ -418,7 +416,55 @@ private: else if (want_more) try_submit(prio); } + /// Block until no firing of this node is in flight or queued. + /// + /// stop() used to set the flag and return, leaving an executing fire_once + /// touching input_channels_, stats_ and pending_ while the caller went on + /// to destroy them. For a node with a private pool that was survivable by + /// accident — Node::stop() calls pool->stop(), which joins — but a node + /// sharing a pool had nothing joining it at all, so ~PoolNode raced its own + /// members. The old comment said callers wanting the guarantee should call + /// scheduler_->drain() first; a destructor cannot, and the default should + /// not be a use-after-free. + /// + /// The gate is exactly the right thing to wait on: it is claimed for the + /// whole of a firing and released as the last act of one. A queued but + /// unstarted firing also holds it, and will run, observe stop_flag_ and + /// release — which is why the pool must still be running when this is + /// called. That is already the documented order (stop nodes, then the + /// pool), and Node/ObjectNode do it that way. + /// + /// Bounded, because a node function that never returns must not turn + /// teardown into a hang; and skipped entirely when called from the firing + /// thread itself, since an error handler that stops its own node would + /// otherwise wait for a firing that is waiting for it. + void await_quiescence() { + if (firing_thread_.load(std::memory_order_acquire) == std::this_thread::get_id()) + return; + const auto deadline = clock_t::now() + std::chrono::seconds(5); + while (gate_.queued()) { + if (clock_t::now() >= deadline) { + std::cerr << "[kpn] stop: node '" << name_ + << "' still had work in flight after 5 s; " + "continuing without it\n"; + return; + } + std::this_thread::sleep_for(std::chrono::microseconds(50)); + } + } + + /// Marks fire_once's thread for the duration of a firing, so await_quiescence + /// can tell a re-entrant stop() from an external one. + struct FiringMark { + std::atomic& slot; + explicit FiringMark(std::atomic& s) : slot(s) { + slot.store(std::this_thread::get_id(), std::memory_order_release); + } + ~FiringMark() { slot.store(std::thread::id{}, std::memory_order_release); } + }; + void fire_once() { + FiringMark mark(firing_thread_); if (stop_flag_.load(std::memory_order_relaxed)) { gate_.force_idle(); return; @@ -624,6 +670,9 @@ private: /// cleared: the callbacks capture `this` and stay valid across a restart, so /// re-registering them would be a pointless write to a live channel. bool prepared_{false}; + /// Thread currently inside fire_once, or a default id when none is. + /// See await_quiescence. + std::atomic firing_thread_{}; /// The hidden one-slot output buffer (see push_outputs). Holding the value /// here is what lets a node stop running without dropping it or occupying a @@ -707,6 +756,7 @@ public: void stop() override { stop_flag_.store(true, std::memory_order_seq_cst); disable_inputs(std::make_index_sequence{}); + await_quiescence(); } bool running() const override { return !stop_flag_.load(std::memory_order_relaxed); } @@ -960,7 +1010,55 @@ private: else if (want_more) try_submit(prio); } + /// Block until no firing of this node is in flight or queued. + /// + /// stop() used to set the flag and return, leaving an executing fire_once + /// touching input_channels_, stats_ and pending_ while the caller went on + /// to destroy them. For a node with a private pool that was survivable by + /// accident — Node::stop() calls pool->stop(), which joins — but a node + /// sharing a pool had nothing joining it at all, so ~PoolNode raced its own + /// members. The old comment said callers wanting the guarantee should call + /// scheduler_->drain() first; a destructor cannot, and the default should + /// not be a use-after-free. + /// + /// The gate is exactly the right thing to wait on: it is claimed for the + /// whole of a firing and released as the last act of one. A queued but + /// unstarted firing also holds it, and will run, observe stop_flag_ and + /// release — which is why the pool must still be running when this is + /// called. That is already the documented order (stop nodes, then the + /// pool), and Node/ObjectNode do it that way. + /// + /// Bounded, because a node function that never returns must not turn + /// teardown into a hang; and skipped entirely when called from the firing + /// thread itself, since an error handler that stops its own node would + /// otherwise wait for a firing that is waiting for it. + void await_quiescence() { + if (firing_thread_.load(std::memory_order_acquire) == std::this_thread::get_id()) + return; + const auto deadline = clock_t::now() + std::chrono::seconds(5); + while (gate_.queued()) { + if (clock_t::now() >= deadline) { + std::cerr << "[kpn] stop: node '" << name_ + << "' still had work in flight after 5 s; " + "continuing without it\n"; + return; + } + std::this_thread::sleep_for(std::chrono::microseconds(50)); + } + } + + /// Marks fire_once's thread for the duration of a firing, so await_quiescence + /// can tell a re-entrant stop() from an external one. + struct FiringMark { + std::atomic& slot; + explicit FiringMark(std::atomic& s) : slot(s) { + slot.store(std::this_thread::get_id(), std::memory_order_release); + } + ~FiringMark() { slot.store(std::thread::id{}, std::memory_order_release); } + }; + void fire_once() { + FiringMark mark(firing_thread_); if (stop_flag_.load(std::memory_order_relaxed)) { gate_.force_idle(); return; @@ -1135,6 +1233,9 @@ private: /// cleared: the callbacks capture `this` and stay valid across a restart, so /// re-registering them would be a pointless write to a live channel. bool prepared_{false}; + /// Thread currently inside fire_once, or a default id when none is. + /// See await_quiescence. + std::atomic firing_thread_{}; /// The hidden one-slot output buffer (see push_outputs). Holding the value /// here is what lets a node stop running without dropping it or occupying a diff --git a/tests/test_pool_node.cpp b/tests/test_pool_node.cpp index 0926556..9ec8b34 100644 --- a/tests/test_pool_node.cpp +++ b/tests/test_pool_node.cpp @@ -668,3 +668,55 @@ TEST_CASE("a node woken with empty inputs does not stop itself", "[pool_node]") node.stop(); pool->stop(); } + +// Regression: stop() must not return while a firing is still running. +// +// stop() set the flag, disabled the inputs and returned, leaving an executing +// fire_once touching input_channels_, stats_ and pending_ while the caller went +// on to destroy them. The old comment was explicit that callers wanting the +// guarantee should call scheduler_->drain() first — but ~PoolNode calls stop(), +// and a destructor cannot ask its caller to have done that. +// +// A node with a private pool survived by accident: Node::stop() calls +// pool->stop(), which joins the worker. A node sharing a pool, which +// make_pool_node exists to create, had nothing joining it at all, so its own +// destructor raced the firing. +// +// Asserted through an observable side effect rather than by trying to catch the +// use-after-free: if stop() returns before the node function has finished, the +// flag it sets on the way out is still false. +namespace { + +struct SlowFiring { + static constexpr std::string_view label() { return "slow_firing"; } + std::atomic* entered; + std::atomic* finished; + void operator()(int) { + entered->store(true, std::memory_order_release); + std::this_thread::sleep_for(std::chrono::milliseconds(200)); + finished->store(true, std::memory_order_release); + } +}; + +} // namespace + +TEST_CASE("stop waits for a firing already in flight", "[pool_node]") { + std::atomic entered{false}, finished{false}; + + auto pool = std::make_shared(2); + pool->start(); + + SlowFiring fn{&entered, &finished}; + auto node = make_pool_node(fn, pool, 4); + node.start(); + node.input_channel<0>().push(1); + + // Stop only once the node is demonstrably inside its function. + while (!entered.load(std::memory_order_acquire)) + std::this_thread::sleep_for(std::chrono::milliseconds(1)); + + node.stop(); + CHECK(finished.load(std::memory_order_acquire)); + + pool->stop(); +} From 7b7f631e6d3ecfd213f61227eddb2e840d341d4c Mon Sep 17 00:00:00 2001 From: Duncan Tourolle Date: Wed, 5 Aug 2026 15:51:26 +0200 Subject: [PATCH 32/42] fix: a shared resource must be able to release its waiters MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit SharedResource::acquire() blocks on a condition variable whose predicate only becomes true when release() hands over ownership. No timeout, no stop condition. A node parked there was not observing stop flags, so teardown had no way to reach it: the worker never returned, the pool's join never completed, and shutdown waited on a resource nobody was going to release — which is precisely the situation when the holder is being stopped too. close() wakes every waiter and refuses further acquisitions, and the waiters leave through ResourceClosedError, which is an exception the node error path already handles rather than a new mechanism. StaticNetwork calls it on registered resources at the top of halt() and shutdown(), before stopping any node, since a node stopped while parked cannot respond to being stopped. The handover needed care in two places. A waiter woken by close() has not been given ownership, so it takes no Guard and leaves held_ exactly as it found it; and release() now skips handing over to waiters when closed, because handing ownership to a thread that is on its way out would leave held_ true with nobody holding it. reopen() is there for reuse across runs, which the persistent-pipeline work will want; teardown does not need it. Verified in both directions: without close() the waiter thread never returns and the test's join blocks; with it the waiter leaves through ResourceClosedError while the holder still has the resource. 145/145. --- include/kpn/diagnostics.hpp | 5 +++ include/kpn/shared_resource.hpp | 54 ++++++++++++++++++++++++++++++--- include/kpn/static_network.hpp | 6 ++++ tests/test_shared_resource.cpp | 48 +++++++++++++++++++++++++++++ 4 files changed, 109 insertions(+), 4 deletions(-) diff --git a/include/kpn/diagnostics.hpp b/include/kpn/diagnostics.hpp index b94c8f2..884d668 100644 --- a/include/kpn/diagnostics.hpp +++ b/include/kpn/diagnostics.hpp @@ -220,6 +220,11 @@ struct ResourceSnapshot { struct IResourceProbe { virtual ~IResourceProbe() = default; virtual ResourceSnapshot snapshot(const std::string& name) const = 0; + + /// Release every thread waiting for the resource, so teardown is not held + /// up by one. A network calls this on the resources registered with it when + /// it halts; default no-op for probes with nothing to wake. + virtual void close() {} }; } // namespace kpn diff --git a/include/kpn/shared_resource.hpp b/include/kpn/shared_resource.hpp index 79924c3..9919508 100644 --- a/include/kpn/shared_resource.hpp +++ b/include/kpn/shared_resource.hpp @@ -14,6 +14,18 @@ namespace kpn { template class Channel; // forward declaration for acquire_balanced +/// Thrown by a pending acquire() when the resource is closed underneath it. +/// +/// acquire() blocks on a condition variable with no timeout and no stop +/// condition, so a node parked there ignored teardown entirely: the worker +/// never returned, the pool's join never completed, and shutdown hung on a +/// resource nobody was going to release. Closing the resource turns that into +/// an exception the node's normal error path already handles. +class ResourceClosedError : public std::runtime_error { +public: + ResourceClosedError() : std::runtime_error("shared resource closed") {} +}; + // ── SharedResource ──────────────────────────────────────────────────────────── // // Wraps an exclusive resource (e.g. an ONNX session, a CUDA stream) and @@ -72,6 +84,7 @@ public: template Guard acquire(PriorityFn&& fn) { std::unique_lock lock(mutex_); + if (closed_) throw ResourceClosedError{}; if (!held_) { held_ = true; acq_.fetch_add(1, std::memory_order_relaxed); @@ -83,18 +96,46 @@ public: current_waiters_.store(waiters_.size(), std::memory_order_relaxed); auto t0 = w.wait_start; - w.cv.wait(lock, [&w] { return w.ready; }); + // Woken either by release() handing over ownership, or by close() + // giving up on the wait entirely. + w.cv.wait(lock, [&w] { return w.ready || w.closed; }); int64_t wait_us = std::chrono::duration_cast( clock_t::now() - t0).count(); waiters_.erase(std::find(waiters_.begin(), waiters_.end(), &w)); current_waiters_.store(waiters_.size(), std::memory_order_relaxed); - acq_.fetch_add(1, std::memory_order_relaxed); total_wait_us_.fetch_add(static_cast(wait_us > 0 ? wait_us : 0), std::memory_order_relaxed); + + // Closed without being handed ownership: no Guard, so nothing to + // release, and held_ is left exactly as close() found it. + if (!w.ready) throw ResourceClosedError{}; + + acq_.fetch_add(1, std::memory_order_relaxed); return Guard(this); } + /// Wake every waiter and refuse further acquisitions. + /// + /// Teardown is the whole point: a node parked in acquire() is not + /// observing stop flags, so without this the only way out is for whoever + /// holds the resource to release it — which, if that node is also being + /// stopped, may never happen. Idempotent, and safe to call from any thread. + void close() override { + std::lock_guard lock(mutex_); + closed_ = true; + for (Waiter* w : waiters_) { + w->closed = true; + w->cv.notify_one(); + } + } + + /// Reopen after a close(). For reuse across runs; not needed for teardown. + void reopen() { + std::lock_guard lock(mutex_); + closed_ = false; + } + // Acquire with no priority (all waiters treated equally, order is fair-ish). Guard acquire() { return acquire([] { return 0.5f; }); @@ -132,7 +173,10 @@ public: private: void release() { std::unique_lock lock(mutex_); - if (waiters_.empty()) { + // Hand over only to a waiter that is still waiting. A closed one is on + // its way out and will not take ownership, so treating it as the next + // holder would leave held_ true with nobody holding it. + if (closed_ || waiters_.empty()) { held_ = false; return; } @@ -162,7 +206,8 @@ private: std::function priority_fn; clock_t::time_point wait_start; std::condition_variable cv; - bool ready{false}; + bool ready{false}; // handed ownership by release() + bool closed{false}; // woken by close() instead Waiter(std::function fn, clock_t::time_point t) : priority_fn(std::move(fn)), wait_start(t) {} @@ -172,6 +217,7 @@ private: T resource_; bool held_{false}; + bool closed_{false}; mutable std::mutex mutex_; std::vector waiters_; std::atomic acq_{0}; diff --git a/include/kpn/static_network.hpp b/include/kpn/static_network.hpp index 75ad6e6..4eda22e 100644 --- a/include/kpn/static_network.hpp +++ b/include/kpn/static_network.hpp @@ -160,6 +160,11 @@ public: #ifdef KPN_WEB_DEBUG if (web_server_) web_server_->stop(); #endif + // Release anything parked on a shared resource first. A node blocked in + // acquire() is not watching stop flags, so stopping it would wait on a + // handover that may never come — its holder is being stopped too. + for (auto& [rname, probe] : resource_probes_) { (void)rname; probe->close(); } + for (auto it = fanout_nodes_ptr_.rbegin(); it != fanout_nodes_ptr_.rend(); ++it) (*it)->stop(); for (auto it = user_nodes_topo_.rbegin(); it != user_nodes_topo_.rend(); ++it) @@ -173,6 +178,7 @@ public: #ifdef KPN_WEB_DEBUG if (web_server_) web_server_->stop(); #endif + for (auto& [rname, probe] : resource_probes_) { (void)rname; probe->close(); } // user_nodes_topo_ is already in sources-first order. // Stop each node and drain its output channels before moving on. for (std::size_t i = 0; i < user_nodes_topo_.size(); ++i) { diff --git a/tests/test_shared_resource.cpp b/tests/test_shared_resource.cpp index 7468f11..9e8abf6 100644 --- a/tests/test_shared_resource.cpp +++ b/tests/test_shared_resource.cpp @@ -237,3 +237,51 @@ TEST_CASE("make_shared_resource constructs with forwarded args", "[shared_resour auto g = res.acquire(); REQUIRE(*g == "hello"); } + +// Regression: a waiter must be releasable, or teardown waits on it forever. +// +// acquire() blocks on a condition variable whose predicate only becomes true +// when release() hands over ownership. There was no timeout and no stop +// condition, so a node parked there ignored teardown entirely: its worker never +// returned, the pool's join never completed, and shutdown hung waiting for a +// resource nobody was going to release — which is exactly the case when the +// holder is being stopped too. +// +// close() turns that into an exception the node's existing error path already +// handles, and networks now call it on registered resources before stopping any +// node, for the same reason. +TEST_CASE("closing a shared resource releases its waiters", "[shared_resource]") { + SharedResource res(42); + + auto holder = res.acquire(); // resource is now held + + std::atomic threw{false}, returned{false}; + std::thread waiter([&] { + try { + auto g = res.acquire(); // blocks: someone else holds it + (void)g; + } catch (const ResourceClosedError&) { + threw.store(true, std::memory_order_release); + } + returned.store(true, std::memory_order_release); + }); + + // Let it park, then tear down without ever releasing the holder. + std::this_thread::sleep_for(std::chrono::milliseconds(50)); + REQUIRE_FALSE(returned.load(std::memory_order_acquire)); + + res.close(); + waiter.join(); + + CHECK(threw.load(std::memory_order_acquire)); +} + +TEST_CASE("acquiring a closed resource fails immediately", "[shared_resource]") { + SharedResource res(7); + res.close(); + CHECK_THROWS_AS(res.acquire(), ResourceClosedError); + + // Reusable across runs once reopened. + res.reopen(); + CHECK_NOTHROW(res.acquire()); +} From 012b64dd3e1e27b5636456900316f5617194d192 Mon Sep 17 00:00:00 2001 From: Duncan Tourolle Date: Wed, 5 Aug 2026 16:11:20 +0200 Subject: [PATCH 33/42] fix: the sentinel must not be delivered ahead of a queued value MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit pop() and try_pop_now() observe the ring empty and then call take_sentinel(). The producer can push a value *and* publish the sentinel in the window between those two steps, so the sentinel was delivered with a real value still queued behind it — breaking the "sentinel is strictly last" contract that downstream teardown depends on, and losing that value to any consumer which, like the stress cases here, treats the sentinel as EOF and stops draining. a0c4bf5 closed the variant where the caller's emptiness check ran against a stale tail_ snapshot. This is the one where the check is fresh and simply too early. take_sentinel now re-checks emptiness *after* observing has_eof_, which is what makes it sound rather than merely narrower: the producer publishes the sentinel with a release store after its ring pushes, so a consumer that has observed has_eof_ has necessarily observed every tail_ advance before it. A non-empty ring at that point means those values genuinely precede the sentinel, and returning false hands them over first. Rates, since this is a race and the numbers are the evidence. The existing "sentinel is strictly last" stress cases fail about 1 run in 15 on the commit before this one and 0 in 25 after; they did not fail in 25 runs of the pre-series baseline, so something in this series widened the window rather than opened it. I could not pin down which change, and it does not much matter: the interleaving is reachable from the code as written, and the narrower version was never correct. No new test. The two existing stress cases already assert exactly this and are what caught it; a deterministic reproduction would need a seam inside pop() that the fix then makes unreachable. --- include/kpn/channel.hpp | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/include/kpn/channel.hpp b/include/kpn/channel.hpp index 6ddad69..afaf9fe 100644 --- a/include/kpn/channel.hpp +++ b/include/kpn/channel.hpp @@ -444,6 +444,24 @@ private: // delivered after every value pushed before it. bool take_sentinel(T& out) { if (!has_eof_.load(std::memory_order_acquire)) return false; + // Re-check emptiness *after* observing has_eof_, not before. + // + // Callers check the ring is empty and then call this, but the producer + // can push a value and publish the sentinel in the window between those + // two steps — so the sentinel would be delivered with a real value still + // queued behind it, breaking the "sentinel is strictly last" contract + // that downstream teardown depends on. a0c4bf5 closed the variant where + // the caller's emptiness check used a stale tail_ snapshot; this is the + // one where the check is fresh but simply too early. + // + // Checking here is what makes it sound: the producer publishes the + // sentinel with a release store *after* its ring pushes, so a consumer + // that has observed has_eof_ has also observed every tail_ advance + // before it. If the ring is non-empty now, those values genuinely + // precede the sentinel and must be delivered first. + if (head_.load(std::memory_order_relaxed) + != tail_.load(std::memory_order_acquire)) + return false; out = extract(std::move(eof_value_)); has_eof_.store(false, std::memory_order_release); stats_.record_pop(); From 80c2b1fb2f1834631282fbbf6b5ec35ca2a4c8b2 Mon Sep 17 00:00:00 2001 From: Duncan Tourolle Date: Wed, 5 Aug 2026 16:14:50 +0200 Subject: [PATCH 34/42] fix: try_push must distinguish delivered from discarded MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit try_push returned bool, and returned *true* for a closed channel — so "the value arrived" and "the value was thrown away because nobody is listening" were the same answer. Every caller was nonetheless correct, because both cases mean "stop trying, do not park and retry". But nothing above the channel could tell the two apart: a node counting successful pushes counted discards among them, and the only record of the loss was the channel's own drop counter, visible solely to whoever read the diagnostics table. Now a three-way PushResult { Taken, Full, Closed }, matching the shape SentinelResult already uses. Behaviour is unchanged at every call site — each treats Closed the same as Taken, and only Full parks — but the distinction is now available to anyone who needs it, and a scoped enum means a future caller cannot silently reintroduce the conflation with `if (push)`. deliver_one benefits immediately: it no longer reaches its teardown path for a closed channel, only for one that is still full, so the last-ditch throwing push it does there to record the loss now records an overflow rather than a drop the channel had already counted. --- include/kpn/branch.hpp | 24 ++++++++++++++++-------- include/kpn/channel.hpp | 23 ++++++++++++++++++----- include/kpn/fanout.hpp | 5 ++++- include/kpn/pool_node.hpp | 10 ++++++++-- tests/test_channel.cpp | 25 +++++++++++++++++++++++++ 5 files changed, 71 insertions(+), 16 deletions(-) diff --git a/include/kpn/branch.hpp b/include/kpn/branch.hpp index 676a1f8..81903ae 100644 --- a/include/kpn/branch.hpp +++ b/include/kpn/branch.hpp @@ -52,16 +52,24 @@ bool deliver_one(Channel* ch, T& val, const std::atomic& stop_flag, } const auto park_from = clock_t::now(); for (;;) { - if (ch->try_push(val)) { - parked = duration_t(clock_t::now() - park_from); - return true; + switch (ch->try_push(val)) { + case Channel::PushResult::Taken: + parked = duration_t(clock_t::now() - park_from); + return true; + case Channel::PushResult::Closed: + // Nobody is listening any more; the channel has recorded the + // drop. Retrying would spin until teardown noticed. + parked = duration_t(clock_t::now() - park_from); + return false; + case Channel::PushResult::Full: + break; // fall through to the retry logic } if (stop_flag.load(std::memory_order_relaxed)) { - // Teardown with work in hand. One last throwing push, purely so the - // channel's own stats record the loss (drop if it is disabled, - // overflow if it is merely full). The point of the lossless path is - // that a loss is never invisible, and a silent return here would - // reintroduce exactly the hole this function exists to close. + // Teardown with work in hand and the output still full. One last + // throwing push, purely so the channel's own stats record the + // overflow — the point of the lossless path is that a loss is never + // invisible, and a silent return here would reintroduce exactly the + // hole this function exists to close. try { ch->push(std::move(val)); } catch (const ChannelOverflowError&) {} parked = duration_t(clock_t::now() - park_from); diff --git a/include/kpn/channel.hpp b/include/kpn/channel.hpp index afaf9fe..e7420ff 100644 --- a/include/kpn/channel.hpp +++ b/include/kpn/channel.hpp @@ -165,13 +165,26 @@ public: head_.load(std::memory_order_acquire) < capacity_; } - /// Non-blocking, lossless push. Returns false when the ring is full, having + /// Outcome of a non-blocking push. + /// + /// try_push used to return bool, and returned *true* for a closed channel — + /// so "delivered" and "discarded because nobody is listening" were the same + /// answer. Both mean "stop trying", which is why the callers were correct, + /// but neither they nor the producer's own accounting could tell a value + /// that arrived from one that was thrown away. Only the channel's drop + /// counter knew. + enum class PushResult { Taken, Full, Closed }; + + /// Non-blocking, lossless push. Returns Full when the ring is full, having /// changed nothing — the caller keeps the value and retries when woken. - bool try_push(T& value) { - if (!accepting_.load(std::memory_order_acquire)) { stats_.record_drop(); return true; } + PushResult try_push(T& value) { + if (!accepting_.load(std::memory_order_acquire)) { + stats_.record_drop(); + return PushResult::Closed; + } const std::size_t t = tail_.load(std::memory_order_relaxed); const std::size_t h = head_.load(std::memory_order_acquire); - if (t - h >= capacity_) return false; + if (t - h >= capacity_) return PushResult::Full; const std::size_t data_bytes = ChannelDataSize::bytes(value); const bool was_empty = (t == h); @@ -181,7 +194,7 @@ public: wake_.fetch_add(1, std::memory_order_release); wake_.notify_one(); if (was_empty && push_callback_) push_callback_(); - return true; + return PushResult::Taken; } // Lossless push with BACKPRESSURE: if the ring is full, wait for the consumer to diff --git a/include/kpn/fanout.hpp b/include/kpn/fanout.hpp index 1d6bf9e..f9f2c46 100644 --- a/include/kpn/fanout.hpp +++ b/include/kpn/fanout.hpp @@ -160,7 +160,10 @@ private: for (;;) { for (std::size_t i = 0; i < N; ++i) { if (!pending[i]) continue; - if (out_channels_[i]->try_push(*pending[i])) { + // Taken or Closed both mean "stop trying" — delivered, or gone + // with the drop recorded. Only Full is worth another pass. + if (out_channels_[i]->try_push(*pending[i]) + != Channel::PushResult::Full) { pending[i].reset(); --outstanding; } diff --git a/include/kpn/pool_node.hpp b/include/kpn/pool_node.hpp index c875b9b..e6113b5 100644 --- a/include/kpn/pool_node.hpp +++ b/include/kpn/pool_node.hpp @@ -641,7 +641,10 @@ private: // to run the consumer that would drain the channel. That is the // hold-and-wait deadlock channel.hpp warns about for sentinels; it // applies to data pushes just as much. - return ch->try_push(val); + // Closed counts as "stop trying", not as delivered: the value is gone + // and the channel has recorded the drop. Only Full means park and retry. + return ch->try_push(val) != Channel> + ::PushResult::Full; } template @@ -1215,7 +1218,10 @@ private: return true; } // See the note on the typed overload above: park rather than block. - return ch->try_push(val); + // Closed counts as "stop trying", not as delivered: the value is gone + // and the channel has recorded the drop. Only Full means park and retry. + return ch->try_push(val) != Channel> + ::PushResult::Full; } Obj& obj_; diff --git a/tests/test_channel.cpp b/tests/test_channel.cpp index 8150b6d..7be801e 100644 --- a/tests/test_channel.cpp +++ b/tests/test_channel.cpp @@ -298,3 +298,28 @@ TEST_CASE("try_push_sentinel leaves a refused value untouched", "[channel][senti CHECK(ch.try_push_sentinel(third) == Channel::SentinelResult::Closed); CHECK(third == "eof-3"); } + +// Regression: try_push must distinguish delivered from discarded. +// +// It returned bool, and returned *true* for a closed channel — so "the value +// arrived" and "the value was thrown away because nobody is listening" were the +// same answer. Every caller was nonetheless correct, because both cases mean +// "stop trying"; but nothing above the channel could tell the two apart, and a +// producer counting successful pushes counted discards among them. Only the +// channel's own drop counter knew, and only if someone read the diagnostics. +TEST_CASE("try_push distinguishes taken, full and closed", "[channel]") { + Channel ch(2); + int v = 1; + + CHECK(ch.try_push(v) == Channel::PushResult::Taken); + CHECK(ch.try_push(v) == Channel::PushResult::Taken); + // Ring is full: the value is untouched and the caller keeps it. + CHECK(ch.try_push(v) == Channel::PushResult::Full); + CHECK(v == 1); + + ch.disable(); + const auto drops_before = ch.stats().drops.load(); + CHECK(ch.try_push(v) == Channel::PushResult::Closed); + // Discarded, and recorded as such rather than reported as a delivery. + CHECK(ch.stats().drops.load() == drops_before + 1); +} From 7a3e96cc9990b76c44d6fa55b0756a552932f73c Mon Sep 17 00:00:00 2001 From: Duncan Tourolle Date: Wed, 5 Aug 2026 17:52:36 +0200 Subject: [PATCH 35/42] fix: the watchdog must be interruptible, or stop() waits for its next tick MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit start_watchdog looped on std::this_thread::sleep_for(watchdog_interval_), and request_stop() cannot wake a sleeping thread. stop_watchdog()'s join therefore blocked until the current sleep expired: three seconds on every teardown at the default interval, and unbounded for anyone who set a long one to keep the periodic report quiet. Now a condition_variable_any waited on with the stop token, so request_stop() ends the wait immediately. Found while writing the next commit's test, which sets a one-hour interval to silence the report and consequently hung for an hour in stop(). The test needs one non-obvious thing, and says so: a pause between start() and stop(). Without it the test races the watchdog — stop_watchdog() runs before the thread has entered its loop, the token is already set when it does, and it exits without ever waiting. That passes against the bug as well as the fix, which is exactly what the first version of this test did. Verified in both directions: without the fix the case is killed at a 25 s timeout; with it, stop() returns in 0 ms. --- include/kpn/network.hpp | 17 +++++++++++++++-- tests/test_network.cpp | 37 +++++++++++++++++++++++++++++++++++++ 2 files changed, 52 insertions(+), 2 deletions(-) diff --git a/include/kpn/network.hpp b/include/kpn/network.hpp index 281b9fa..7af55de 100644 --- a/include/kpn/network.hpp +++ b/include/kpn/network.hpp @@ -8,8 +8,10 @@ #include #endif +#include #include #include +#include #include #include #include @@ -433,9 +435,20 @@ private: void start_watchdog() { watchdog_ = std::jthread([this](std::stop_token tok) { + // Interruptible wait, not sleep_for. request_stop() cannot wake a + // sleeping thread, so stop_watchdog()'s join blocked for up to a + // full interval — three seconds by default, and unbounded for + // anyone who set a long one to keep the periodic report quiet. + // Every teardown paid it. + std::mutex m; + std::condition_variable_any cv; while (!tok.stop_requested()) { - std::this_thread::sleep_for(watchdog_interval_); - if (tok.stop_requested()) break; + { + std::unique_lock lk(m); + if (cv.wait_for(lk, tok, watchdog_interval_, + [&tok] { return tok.stop_requested(); })) + break; + } auto s = collect_snapshots(); check_hung_nodes(); diff --git a/tests/test_network.cpp b/tests/test_network.cpp index 9d75f64..59b6659 100644 --- a/tests/test_network.cpp +++ b/tests/test_network.cpp @@ -1,6 +1,10 @@ #include #include +#include #include +#include +#include +#include #include using namespace kpn; @@ -54,3 +58,36 @@ TEST_CASE("stop disables input channels — producer push is silently dropped", in_ch.push(99); REQUIRE(in_ch.size() == 0); } + +// Regression: stopping a network must not wait for the watchdog's next tick. +// +// The watchdog looped on std::this_thread::sleep_for(watchdog_interval_), and +// request_stop() cannot wake a sleeping thread — so stop_watchdog()'s join +// blocked until the current sleep expired. Every teardown paid up to a full +// interval, three seconds by default, and a caller who set a long one to keep +// the periodic report quiet got a stop() that looked like a hang. That is how +// this was found: the error-handler case above set an hour. +TEST_CASE("stopping a network does not wait for the watchdog interval", "[network]") { + auto node = kpn::make_node(kpn::in<"v">{}, kpn::out<"w">{}, 4); + kpn::Channel out(4); + node.set_output_channel<0>(&out); + + kpn::Network net; + net.add("inc", node).build(); + net.set_watchdog_interval(std::chrono::hours(1)); + net.start(); + + // Let the watchdog actually reach its wait. Without this the test races it: + // stop_watchdog() runs before the thread has entered the loop, the token is + // already set when it does, and it exits without ever waiting — which passes + // against the bug as well as the fix. + std::this_thread::sleep_for(std::chrono::milliseconds(100)); + + const auto t0 = std::chrono::steady_clock::now(); + net.stop(); + const auto ms = std::chrono::duration_cast( + std::chrono::steady_clock::now() - t0).count(); + + INFO("stop took " << ms << " ms"); + CHECK(ms < 2000); +} From 00245f5760e1d900bfb0b16da11779e737230eef Mon Sep 17 00:00:00 2001 From: Duncan Tourolle Date: Wed, 5 Aug 2026 18:05:24 +0200 Subject: [PATCH 36/42] fix: Network::set_error_handler must actually deliver the handler MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The handler was stored in a member and never read. A node's exception was discarded at the node boundary and the only surviving evidence was a Closed event, which reports that a node stopped but not why — the difference between a diagnosis and a guess. StaticNetwork has always wired this; Network accepted the handler and silently dropped it, which is worse than not offering the setter, because the caller believes they have a listener. start() now delivers it to each node, exactly as StaticNetwork does. The type changes with it. It was void(name, exception_ptr), which cannot express the keep-running decision the node side needs — so it is now NodeErrorHandler, the same alias StaticNetwork uses. That is a breaking change in principle; in practice nothing in the tree called this setter, which is how it stayed dead long enough to be worth finding. Verified by the new case: the node throws, the handler receives the name and the exception, returns true, and the node goes on to process the next value. 148/148. --- include/kpn/network.hpp | 20 +++++++++++++-- tests/test_network.cpp | 57 +++++++++++++++++++++++++++++++++++++++++ 2 files changed, 75 insertions(+), 2 deletions(-) diff --git a/include/kpn/network.hpp b/include/kpn/network.hpp index 7af55de..5974e70 100644 --- a/include/kpn/network.hpp +++ b/include/kpn/network.hpp @@ -41,8 +41,15 @@ public: class Network : public INode { public: - using ErrorHandler = - std::function; + /// Application-level error listener. Receives the exception any node's + /// function throws, after that node's own handler (if any) declined it. + /// Return true to skip the failed invocation and keep the node running, + /// false to let it stop. + /// + /// Same type as StaticNetwork's, deliberately: this used to be a void + /// signature, which could not express the keep-running decision and, more + /// to the point, was never delivered anywhere. + using ErrorHandler = NodeErrorHandler; using DiagnosticsHandler = std::function&, const std::vector&)>; @@ -137,6 +144,14 @@ public: void start() override { start_time_ = clock_t::now(); + // Deliver the listener to the nodes. Without this the handler was + // stored and never read: a node's exception was discarded at the node + // boundary and the only surviving evidence was a Closed event, which + // says a node stopped but not why. StaticNetwork has always done this; + // Network accepted the handler and silently dropped it. + if (error_handler_) + for (auto& name : topo_) + nodes_.at(name)->set_network_error_callback(error_handler_); // Callbacks first, everywhere, before anything runs — see INode::prepare. for (auto& name : topo_) nodes_.at(name)->prepare(); @@ -220,6 +235,7 @@ public: /// up on them and stopping the next layer anyway. void set_drain_timeout(std::chrono::milliseconds t) { drain_timeout_ = t; } + /// Must be called before start(); the handler is delivered to nodes there. void set_error_handler(ErrorHandler h) { error_handler_ = std::move(h); } void set_diagnostics_handler(DiagnosticsHandler h) { diag_handler_ = std::move(h); } void set_event_handler(EventHandler h) { event_handler_ = std::move(h); } diff --git a/tests/test_network.cpp b/tests/test_network.cpp index 59b6659..11d91c4 100644 --- a/tests/test_network.cpp +++ b/tests/test_network.cpp @@ -59,6 +59,63 @@ TEST_CASE("stop disables input channels — producer push is silently dropped", REQUIRE(in_ch.size() == 0); } +// Regression: Network::set_error_handler must actually deliver the handler. +// +// The handler was stored in a member and never read. A node's exception was +// discarded at the node boundary and the only surviving evidence was a Closed +// event, which reports that a node stopped but not why — the difference between +// a diagnosis and a guess. StaticNetwork has always wired this; Network +// accepted the handler and silently dropped it, which is worse than not +// offering the setter at all. +// +// The type changed with the fix. It was void(name, exception_ptr), which cannot +// express the keep-running decision the node side needs, so it is now +// NodeErrorHandler like StaticNetwork's. +namespace { + +static int throwing_stage(int x) { + if (x == 42) throw std::runtime_error("boom"); + return x; +} + +} // namespace + +TEST_CASE("network error handler receives the node's exception", "[network]") { + auto src = kpn::make_node(kpn::in<"v">{}, kpn::out<"w">{}, 8); + kpn::Channel out(8); + src.set_output_channel<0>(&out); + + kpn::Network net; + net.add("stage", src).build(); + + std::atomic calls{0}; + std::string seen_name; + std::string seen_what; + std::mutex mx; + + net.set_error_handler([&](std::string_view name, std::exception_ptr ep) { + std::lock_guard lk(mx); + seen_name = std::string(name); + try { if (ep) std::rethrow_exception(ep); } + catch (const std::exception& e) { seen_what = e.what(); } + calls.fetch_add(1, std::memory_order_relaxed); + return true; // handled: keep the node running + }); + + net.set_watchdog_interval(std::chrono::hours(1)); // keep the report quiet + net.start(); + src.input_channel<0>().push(42); // throws + std::this_thread::sleep_for(std::chrono::milliseconds(100)); + src.input_channel<0>().push(7); // must still be running + const int passed = out.pop(); + net.stop(); + + CHECK(calls.load(std::memory_order_relaxed) == 1); + CHECK(seen_name == "stage"); + CHECK(seen_what == "boom"); + CHECK(passed == 7); +} + // Regression: stopping a network must not wait for the watchdog's next tick. // // The watchdog looped on std::this_thread::sleep_for(watchdog_interval_), and From c9aa246322b9960ca94cee397e0a6d9f3f9aa96e Mon Sep 17 00:00:00 2001 From: Duncan Tourolle Date: Thu, 6 Aug 2026 20:44:38 +0200 Subject: [PATCH 37/42] fix: a re-offered sentinel is not data loss MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 139bfbb made the channel refuse a sentinel offered while one was still pending — correct, and the reason is a data race: overwriting wrote eof_value_ while the consumer could be moving the previous one out of it, which ThreadSanitizer reports as a torn refcount and a heap-use-after-free. That part stands. What was wrong was the accounting. The refusal recorded a drop, and PoolNode reported it through the overflow event callback, on the theory that two control tokens on one channel means the stream ended twice and is a caller protocol error. It is not. A source that has reached the end of its input keeps being polled and keeps returning EOF — that is the normal steady state, not an error — so the token is re-offered on every firing. Refusing a re-offer loses nothing: the pending token carries the same meaning and is already on its way. Found by running it. scene-actor-extraction on a 14 s clip reported [main] ERROR: frames were dropped (channel overflow): frame_source: 2 scene_annotate: 1 [main] The output would describe footage that was never analysed. Refusing to report success. and exited 2, on a run where nothing had been dropped and every frame was analysed. frame_source emits EOF once and then returns it forever (frame_source_node.hpp:76), so the count grows with however many times the source is polled after the end. The pipeline's own loss detector — which exists because a dropped frame silently corrupts the output — was being tripped by a clean run, which is the one thing a loss detector must not do. So SlotBusy now records nothing and reports nothing. The cost, stated plainly: a genuinely distinct second token would also be refused silently, and the channel cannot tell a re-offer from a distinct token. Re-offering is the case that actually occurs; the delivery guarantee that matters — the first token arrives — holds either way. The test asserting the old behaviour is inverted rather than deleted, and now also checks that the token which was accepted is the one delivered. 148/148. --- include/kpn/channel.hpp | 18 +++++++++++++++--- include/kpn/pool_node.hpp | 30 ++++++++++++++---------------- tests/test_channel.cpp | 21 ++++++++++++++++----- 3 files changed, 45 insertions(+), 24 deletions(-) diff --git a/include/kpn/channel.hpp b/include/kpn/channel.hpp index e7420ff..53e0833 100644 --- a/include/kpn/channel.hpp +++ b/include/kpn/channel.hpp @@ -267,10 +267,22 @@ public: // has_eof_, the consumer is the only one that clears it, so observing // false here means the consumer has finished with the storage and will // not touch it again until this store publishes the next token. - if (has_eof_.load(std::memory_order_acquire)) { - stats_.record_drop(); + // + // Not counted as a drop, and this is the important part. A source that + // has reached the end of its input keeps being polled and keeps + // returning EOF — that is the normal steady state, not an error — so a + // token arriving while one is already pending is a *re-offer*, and + // refusing it loses nothing: the pending token carries the same + // meaning and is already on its way. Counting it as a drop made a + // clean run report data loss and exit non-zero. + // + // The cost of that choice, stated plainly: a genuinely distinct second + // token would also be refused silently, and the channel cannot tell the + // two apart. Re-offering is the case that actually occurs here, and the + // delivery guarantee that matters — the first token arrives — holds + // either way. + if (has_eof_.load(std::memory_order_acquire)) return SentinelResult::SlotBusy; - } eof_value_ = make_storage(std::move(value)); has_eof_.store(true, std::memory_order_release); // Wake a consumer blocked in pop(): the sentinel is now deliverable even diff --git a/include/kpn/pool_node.hpp b/include/kpn/pool_node.hpp index e6113b5..45115aa 100644 --- a/include/kpn/pool_node.hpp +++ b/include/kpn/pool_node.hpp @@ -622,14 +622,13 @@ private: // downstream pop() forever. Deliver them out-of-band (push_sentinel), // which never overflows and never blocks this node's worker thread. if (is_sentinel_value(val)) { - // A refused sentinel is a protocol error, not backpressure, so it - // is reported rather than parked and retried — retrying would spin - // forever against a slot only the consumer can free, and there is - // no correct value to deliver second anyway. Closed is normal - // during teardown and stays quiet. - if (ch->try_push_sentinel(val) == Channel> - ::SentinelResult::SlotBusy) - fire_callbacks(event_callbacks_); + // Not parked and not reported. Parking would spin against a slot + // only the consumer can free; reporting would cry data loss on the + // normal steady state, since a source at end of input keeps being + // polled and keeps returning EOF, so the token is re-offered on + // every firing. Refusing a re-offer loses nothing — the pending + // token says the same thing. See Channel::try_push_sentinel. + ch->try_push_sentinel(val); return true; } // Backpressure without parking the worker. A full channel means the @@ -1207,14 +1206,13 @@ private: // downstream pop() forever. Deliver them out-of-band (push_sentinel), // which never overflows and never blocks this node's worker thread. if (is_sentinel_value(val)) { - // A refused sentinel is a protocol error, not backpressure, so it - // is reported rather than parked and retried — retrying would spin - // forever against a slot only the consumer can free, and there is - // no correct value to deliver second anyway. Closed is normal - // during teardown and stays quiet. - if (ch->try_push_sentinel(val) == Channel> - ::SentinelResult::SlotBusy) - fire_callbacks(event_callbacks_); + // Not parked and not reported. Parking would spin against a slot + // only the consumer can free; reporting would cry data loss on the + // normal steady state, since a source at end of input keeps being + // polled and keeps returning EOF, so the token is re-offered on + // every firing. Refusing a re-offer loses nothing — the pending + // token says the same thing. See Channel::try_push_sentinel. + ch->try_push_sentinel(val); return true; } // See the note on the typed overload above: park rather than block. diff --git a/tests/test_channel.cpp b/tests/test_channel.cpp index 7be801e..7b660c7 100644 --- a/tests/test_channel.cpp +++ b/tests/test_channel.cpp @@ -271,15 +271,26 @@ TEST_CASE("a second sentinel is refused, not swallowed", "[channel][sentinel]") CHECK(out == 3); } -TEST_CASE("a refused sentinel is counted as a drop", "[channel][sentinel]") { - // Visibility matters more here than for a dropped value: the refusal means - // a control token went nowhere, and the only alternative to a counter is - // for it to vanish. +TEST_CASE("a refused sentinel is not counted as a drop", "[channel][sentinel]") { + // A refusal means a token arrived while an equivalent one was already + // pending — not that anything was lost. Counting it as a drop was wrong in + // a way that showed up immediately on real content: a source at the end of + // its input keeps being polled and keeps returning EOF, so the token is + // re-offered on every firing, and the pipeline reported hundreds of dropped + // frames on a clean run and exited non-zero. + // + // The delivery guarantee is unaffected: the first token is pending and will + // arrive. Only the accounting changed. Channel ch(4); REQUIRE(ch.push_sentinel(1)); const auto before = ch.stats().drops.load(); REQUIRE_FALSE(ch.push_sentinel(2)); - CHECK(ch.stats().drops.load() == before + 1); + CHECK(ch.stats().drops.load() == before); + + // And the one that was accepted is still the one delivered. + int out = 0; + REQUIRE(ch.try_pop_now(out)); + CHECK(out == 1); } TEST_CASE("try_push_sentinel leaves a refused value untouched", "[channel][sentinel]") { From 6802328e97245a623c5cae6be9566bad3fd37791 Mon Sep 17 00:00:00 2001 From: Duncan Tourolle Date: Thu, 6 Aug 2026 20:50:34 +0200 Subject: [PATCH 38/42] fix: a push must wake its consumer, even when the ring looked non-empty push(), try_push() and push_blocking() fired push_callback_ only on the empty->non-empty edge, and computed that edge from a head_ sampled before the item was published. A PoolNode consumer decides whether to run again from the level (count_ready -> approx_size), so a pop landing in that window left both sides standing down: producer (push) consumer (PoolNode firing) ------------------------ ---------------------------- samples t=782, h=781 -> was_empty = false, no wake pops idx 781, head_ = 782 count_ready(): head_==tail_==782 -> not ready, gate released to Idle tail_.store(783) The item is in the ring, the node is idle, and no wake is outstanding. The failure is absorbing: every later push then sees a non-empty ring, so the edge never fires again and the node sleeps while its backlog grows. Observed as a hang in bench_pipeline at (chain, depth=4, work_us=10, shared pool): all pool workers asleep in worker_loop, the reader blocked in pop(), and 218 items stranded in one channel with head_ stopped at exactly the index where the edge was dropped. Re-reading head_ after the tail_ store does not fix this. That is the store-buffer pattern, and under acquire/release both sides may legally read stale; forbidding it needs seq_cst on the producer's tail_ store and head_ load *and* on the consumer's head_ store and tail_ load, a fence on both hot paths. Firing unconditionally is correct by construction: the callback runs after the publishing store, so a consumer that observes the level at all observes the item. The redundant wakes are cheap -- on_input_ready re-checks the level and SubmitGate::claim() collapses a wake arriving mid-firing into the firing already in flight. The stress test named this exact hazard and could not detect it: it asserted only 1 <= callbacks <= N, which a *missed* callback satisfies. It now requires one callback per successful push, and fails at 1325/10000 against the old code. Co-Authored-By: Claude Opus 5 --- include/kpn/channel.hpp | 48 ++++++++++++++++++++++++++++++----- tests/test_channel_stress.cpp | 22 ++++++++++------ 2 files changed, 55 insertions(+), 15 deletions(-) diff --git a/include/kpn/channel.hpp b/include/kpn/channel.hpp index e7420ff..d586c1e 100644 --- a/include/kpn/channel.hpp +++ b/include/kpn/channel.hpp @@ -136,7 +136,6 @@ public: throw ChannelOverflowError(capacity_); } - const bool was_empty = (t == h); buf_[t & ring_mask_] = make_storage(std::move(value)); tail_.store(t + 1, std::memory_order_release); stats_.record_push(t - h + 1, data_bytes); @@ -144,7 +143,8 @@ public: wake_.fetch_add(1, std::memory_order_release); wake_.notify_one(); - if (was_empty && push_callback_) + // Level-triggered, not edge-triggered — see set_push_callback. + if (push_callback_) push_callback_(); } @@ -187,13 +187,13 @@ public: if (t - h >= capacity_) return PushResult::Full; const std::size_t data_bytes = ChannelDataSize::bytes(value); - const bool was_empty = (t == h); buf_[t & ring_mask_] = make_storage(std::move(value)); tail_.store(t + 1, std::memory_order_release); stats_.record_push(t - h + 1, data_bytes); wake_.fetch_add(1, std::memory_order_release); wake_.notify_one(); - if (was_empty && push_callback_) push_callback_(); + // Level-triggered, not edge-triggered — see set_push_callback. + if (push_callback_) push_callback_(); return PushResult::Taken; } @@ -212,13 +212,13 @@ public: const std::size_t h = head_.load(std::memory_order_acquire); if (t - h < capacity_) { // space available → normal push const std::size_t data_bytes = ChannelDataSize::bytes(value); - const bool was_empty = (t == h); buf_[t & ring_mask_] = make_storage(std::move(value)); tail_.store(t + 1, std::memory_order_release); stats_.record_push(t - h + 1, data_bytes); wake_.fetch_add(1, std::memory_order_release); wake_.notify_one(); - if (was_empty && push_callback_) push_callback_(); + // Level-triggered, not edge-triggered — see set_push_callback. + if (push_callback_) push_callback_(); return true; } // full: yield briefly and retry (consumer will drain) @@ -388,7 +388,41 @@ public: wake_.notify_all(); } - // Register a callback fired when the queue transitions empty→non-empty. + // Register a callback fired after every successful push. + // + // It fires on every push, not on the empty→non-empty transition, and that + // is a correctness requirement rather than a simplification. + // + // The edge version tested `was_empty = (t == h)` using an `h` sampled + // *before* the item was published. A PoolNode consumer decides whether to + // run again from the level (count_ready → approx_size), so the two sides + // could each read the other as stale and both stand down: + // + // producer (push) consumer (PoolNode firing) + // ------------------------ ---------------------------- + // samples t=782, h=781 + // -> was_empty = false, no wake + // pops idx 781, head_ = 782 + // count_ready(): head_==tail_==782 + // -> not ready, gate released to Idle + // tail_.store(783) + // + // The item is in the ring, the node is idle, and no wake is outstanding. + // Worse, the failure is absorbing: every later push now sees a non-empty + // ring, so `was_empty` is false forever and the callback never fires again. + // The node sleeps while its backlog grows and its consumer waits on it. + // + // Re-reading head_ after the tail_ store does not fix it. That is the + // store-buffer pattern, and under acquire/release both sides may legally + // read stale; forbidding it needs seq_cst on the producer's tail_ store and + // head_ load *and* on the consumer's head_ store and tail_ load — a fence + // on both hot paths. Firing unconditionally is correct by construction: + // the callback runs after the publishing store, so a consumer that observes + // the level at all observes the item. + // + // The redundant wakes are cheap. on_input_ready re-checks the level, and + // SubmitGate::claim() collapses a wake arriving during a firing into the + // firing already in flight, so the cost is one CAS, not one extra run. void set_push_callback(std::function cb) { push_callback_ = std::move(cb); } diff --git a/tests/test_channel_stress.cpp b/tests/test_channel_stress.cpp index 9f851e3..58ac3a0 100644 --- a/tests/test_channel_stress.cpp +++ b/tests/test_channel_stress.cpp @@ -153,11 +153,18 @@ TEST_CASE("SPSC: producer racing a disable() never throws and never hangs", } } -TEST_CASE("SPSC: push_callback fires on each empty->non-empty transition", +TEST_CASE("SPSC: push_callback fires for every push, never missed", "[channel][stress]") { - // The empty->non-empty callback ([channel.hpp] was_empty branch) is read by - // the consumer-side notification path. Run it under contention to make sure - // the was_empty detection isn't torn by a concurrent pop(). + // Regression: this callback is the *only* thing that wakes a PoolNode, and + // it used to fire only on the empty->non-empty edge, computed from a head_ + // sampled before the item was published. A concurrent pop() could drain the + // ring to empty in that window, so neither side saw the other: the item sat + // in the ring with the consumer idle, and because the trigger was an edge it + // never recovered. See set_push_callback in channel.hpp. + // + // The old version of this test asserted only `1 <= callbacks <= N`, which a + // *missed* callback satisfies — it named the hazard and could not detect it. + // One callback per successful push is the contract, so assert exactly that. Channel ch(/*capacity=*/4, /*spin_count=*/4); std::atomic callbacks{0}; ch.set_push_callback([&] { callbacks.fetch_add(1, std::memory_order_relaxed); }); @@ -175,10 +182,9 @@ TEST_CASE("SPSC: push_callback fires on each empty->non-empty transition", for (int i = 0; i < N; ++i) (void)ch.pop(); producer.join(); - // At least one transition, at most one per item; mainly we assert the run - // completed without TSan flagging a race on push_callback_/was_empty. - REQUIRE(callbacks.load() >= 1); - REQUIRE(callbacks.load() <= N); + // Exactly one callback per successful push. Fewer means a wake was dropped, + // which is the bug; more would mean a spurious wake was manufactured. + REQUIRE(callbacks.load() == N); } // Ordering contract of the out-of-band sentinel under contention. From 771b9f85938dbc1ac449ed4a730fa20ec298609e Mon Sep 17 00:00:00 2001 From: Duncan Tourolle Date: Thu, 6 Aug 2026 22:53:02 +0200 Subject: [PATCH 39/42] fix: ThreadPool::start must take the lifecycle lock too MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit abbb2d4 guarded submit() against stop() with a shared/exclusive lock, because stop() ends with queues_.clear() and submit() indexes queues_. It missed the other writer: start() rebuilds the same vector and took no lock at all. A submission can genuinely land while a pool is inside start(). A network starts its nodes one at a time, and a node already started fires into the next one's channel, whose push callback submits. ThreadSanitizer reports it as the read at scheduler.hpp:114 against the write at :59, and the consequence is worse than a torn read: push_back can reallocate the vector under a reader that has already indexed it. Surfaced by 6802328. Firing push_callback_ unconditionally is correct — the argument in that commit holds — and it makes callbacks frequent enough during startup to hit this window. It went from unobserved to 4 races in one run of the unit suite. Holding the lock across the thread spawn is safe: all queues are constructed before any worker starts, and worker_loop never takes the lifecycle lock, so there is nothing for it to deadlock against. Verified with -DKPN_SANITIZER=thread: 4 races in one run of five before, 0 across five unit runs and two stress runs after. 148/148. --- include/kpn/scheduler.hpp | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/include/kpn/scheduler.hpp b/include/kpn/scheduler.hpp index 54bbb5b..d11d775 100644 --- a/include/kpn/scheduler.hpp +++ b/include/kpn/scheduler.hpp @@ -53,6 +53,19 @@ public: } void start() override { + // Under the lifecycle lock for the same reason stop() is: submit() + // reads queues_ and this rebuilds it. A network starts its nodes one at + // a time, and a node already started fires into the next one's channel, + // whose push callback submits — so a submission can genuinely land + // while another pool is still inside start(). ThreadSanitizer reports + // it as a read at submit() against this write, and the consequence is + // worse than a torn read: push_back can reallocate the vector under a + // reader that has already indexed it. + // + // Queues are all constructed before any worker is spawned, which is + // what keeps worker_loop's own queues_[id] out of this — it never takes + // the lock, so holding it across the spawn cannot deadlock. + std::unique_lock lk(lifecycle_mx_); stopped_.store(false, std::memory_order_relaxed); queues_.clear(); for (std::size_t i = 0; i < thread_count_; ++i) From a3f61fcb3cf104f2c9c2caf853e1399795d215e6 Mon Sep 17 00:00:00 2001 From: Duncan Tourolle Date: Sat, 8 Aug 2026 12:13:11 +0200 Subject: [PATCH 40/42] perf: make the benchmark able to answer the question, then ask it PERF_PLAN phase 0, plus B1/B2 which turned out to cost seconds rather than the minutes budgeted for them. No library code is touched. The harness could not support the conclusions drawn from it. items_for() shrank the sample as work per item grew, so exactly the rows under investigation -- chain-16 and chain-32 -- ran 50 to 200 items and swung 4-8x between passes. Sample size now derives from a time budget with a floor, using work_us * stages / units as the per-item cost. The old ladder's error was treating depth as a throughput cost: past the core count it is, below it depth costs only latency. Rows now report median of N repetitions after a discarded warm-up, with IQR and range, so an unreliable row says so instead of being averaged into a table. The CSV header records nproc, governor and AC state, which immediately caught this laptop running on battery under powersave. A3 needed no experiment in the end: ru_nivcsw and ru_nvcsw are captured around every timed region and reported per item, so involuntary switches against depth is a column rather than a run. bench_dispatch answers B1 and B2 without instrumenting the scheduler. Sleeping is inferred from ru_nvcsw, since a thread blocking on a condition variable books a voluntary context switch. B1: a ThreadPool(1) dispatch is 291 ns null, 466 ns with a payload, against the ~290 ns the plan estimated -- so the abandon criterion is not met and workstream B stays alive. B2's answer is not the one the question expected. It is not whether workers sleep but which pool: on a private ThreadPool(1) the worker never sleeps, because it resubmits into its own queue and finds the work already there; on any pool of two or more it sleeps exactly once per task, because submit() round-robins to a different worker, which is asleep. That is the whole 466 ns to 1.7 us difference, and it inverts half the plan. B5 (bounded spin) buys nothing in the default configuration, and A5 must not make a shared pool the default until the wake cost is fixed, or every graph that already fits its cores gets 3-4x worse per dispatch. G1 lands as tests/soak_wedge.cpp, superseding benchmarks/repro_wedge.cpp, which was never wired into any build. Always compiled so it cannot rot; its CTest cases register only under -DKPN_ENABLE_SOAK_TESTS=ON, so the default test count is unchanged. A wedge is a hang, and a hang under CTest is an unattributable timeout, so it carries a watchdog that aborts naming the iteration and phase. Phase 0's gate is not yet cleared: the acceptance run belongs on the reference machine, not here. A 3-pass check lands every row within 0.7% against the 4-8x swings described above, which is encouraging and is not the same thing. Provisional, recorded so it can be checked: chain-16 came out 6% behind TBB rather than 28.5%. If that survives a proper run, the deep-chain deficit is substantially an artefact of the N=200 rows. Co-Authored-By: Claude Opus 5 --- .gitignore | 2 +- PERF_PLAN.md | 413 +++++++++++++++++++++++++++++ benchmarks/CMakeLists.txt | 7 + benchmarks/bench_dispatch.cpp | 307 ++++++++++++++++++++++ benchmarks/bench_env.hpp | 108 ++++++++ benchmarks/bench_pipeline.cpp | 482 +++++++++++++++++++++++++--------- scripts/bench_repro_check.py | 122 +++++++++ tests/CMakeLists.txt | 34 ++- tests/soak_wedge.cpp | 236 +++++++++++++++++ 9 files changed, 1587 insertions(+), 124 deletions(-) create mode 100644 PERF_PLAN.md create mode 100644 benchmarks/bench_dispatch.cpp create mode 100644 benchmarks/bench_env.hpp create mode 100755 scripts/bench_repro_check.py create mode 100644 tests/soak_wedge.cpp diff --git a/.gitignore b/.gitignore index d3fb3e0..2d1b7e4 100644 --- a/.gitignore +++ b/.gitignore @@ -29,4 +29,4 @@ Thumbs.db .claude/settings.local.json include/kpn/ort_cache/ build-tsan/ -build-*/ +build*/ diff --git a/PERF_PLAN.md b/PERF_PLAN.md new file mode 100644 index 0000000..28560c4 --- /dev/null +++ b/PERF_PLAN.md @@ -0,0 +1,413 @@ +# Performance investigation plan: fanout dispatch cost and deep-chain oversubscription + +**Status:** phase 0 implemented; gate not yet cleared +**Date:** 2026-08-06 (phase 0 landed 2026-08-06) +**Baseline:** master @ 3b67b7e +**Machine:** 20 cores, GCC 16.1.1, TBB 2023.1.0, AC power, `performance` governor +**Data:** 7 full benchmark passes, medians reported below + +--- + +## 1. What was measured + +Throughput, items/sec, median of 7 passes. `N` is the sample size the harness +uses for that row; it is what determines whether a row can be trusted at all. + +### work_us = 10 + +| row | KPN it/s | TBB it/s | TBB faster | N | reliable? | +|---|---|---|---|---|---| +| chain-1 | 89381 | 91350 | +2.2% | 3000 | solid | +| chain-2 | 83759 | 87017 | +3.9% | 1000 | solid | +| chain-4 | 83725 | 85973 | +2.7% | 1000 | solid | +| chain-8 | 79730 | 81090 | +1.7% | 1000 | solid | +| **chain-16** | 53484 | 68745 | **+28.5%** | 200 | weak | +| **chain-32** | 32780 | 45030 | **+37.4%** | 200 | weak | +| **wide-4** | 84906 | 95137 | **+12.0%** | 3000 | solid | +| diamond-4 | 84826 | 86772 | +2.3% | 1000 | solid | + +### work_us = 100 + +Everything except chain-16/32 falls within ±3.4%, with KPN often ahead +(chain-1 −0.6%, chain-4 −1.7%, chain-8 −3.4%, diamond −2.6% — negative means +KPN faster). chain-16 is +12.5% and chain-32 +18.6%, both at N=50 and +therefore unusable. + +### Two deficits, different causes + +1. **Fanout, +12%.** Solidly measured. `wide-4` performs ~5 node dispatches + per item; the gap works out to a fixed ~250 ns per dispatch, consistent + with `chain-1`'s ~290 ns over a single dispatch. This is dispatch + efficiency. + +2. **Deep chains, +28–37%.** The gap is 1.7–3.9% through depth 8, then jumps + to 28.5% at depth 16 and 37.4% at depth 32. That is a cliff at core count, + not a linear per-dispatch cost. `Node<>` owns a private `ThreadPool(1)` + (`include/kpn/node.hpp:21`), so a depth-32 chain spawns 32 OS threads on + 20 cores. TBB bounds its worker count by hardware concurrency regardless of + graph size. + +### Scope note + +At 100 µs+ per node KPN is at parity or ahead. The repository's own examples +(OpenCV cellshade, frame sources, scene-actor extraction) do milliseconds of +work per node, where a 290 ns dispatch cost is roughly one part in thirty +thousand. Everything in this document matters only for fine-grained pipelines. + +--- + +## 2. Phase 0 — the gate that comes first + +**Is there a target workload with sub-30 µs nodes?** + +If no such workload exists or is planned, the correct output of this document +is section 3 (harness) plus a README correction, and nothing else. Optimising +for a benchmark regime the project does not operate in is not worth the risk +described in section 6. + +--- + +## 3. Prerequisite — make the harness able to answer + +None of the questions below are decidable with the current harness. +`benchmarks/bench_pipeline.cpp` shrinks the sample count as work per item +grows, so the rows under investigation run 50–200 items and swing 4–8× +run to run. + +| id | change | why | +|---|---|---| +| M1 | `items_for()` → fixed floor, e.g. `max(2000, …)`, independent of work_us and depth | deep rows are currently unmeasurable | +| M2 | report items/sec as the primary metric; keep derived overhead as secondary | overhead is `elapsed − work`, a difference of large numbers; it magnifies noise roughly 10× | +| M3 | K in-process repetitions per config; report median and IQR | one shot per config is the root of the present noise | +| M4 | discard a warm-up repetition | first-touch page faults, thread spin-up | +| M5 | extend `pool_sizes[]` to `{1,2,4,8,16,20}` | currently `{1,2,4}` — the configuration the README recommends is never run | +| M6 | record nproc, governor and AC state in the CSV header | run-to-run attribution | + +**Acceptance:** the same configuration run 7× lands within ±5% on every row. +Until that holds, no number below should be acted on. + +This touches only the benchmark, not the library. + +### Status — implemented 2026-08-06 + +All of M1–M6 are in `benchmarks/bench_pipeline.cpp`, plus a CLI so the phase-1 +experiments are invocations rather than edits (`--depths`, `--pools`, +`--work`, `--topos`, `--modes`, `--reps`, `--target-sec`, `--min-items`). + +M1 is not a fixed floor but a time budget with a floor: sample size derives +from `work_us × stages / units`, the steady-state throughput bound, then +clamps to `[--min-items, --max-sec]`. A flat 2000-item floor would have made +`chain-32` on a 1-thread pool at 1000 µs a 64-second row; the ceiling keeps +such rows short and reports their true `N` so a short row is visible rather +than silent. The old ladder's error was treating depth as a throughput cost — +in a pipeline, depth beyond the core count costs throughput, below it only +latency. + +Also added, ahead of schedule because it is free: `ru_nivcsw` / `ru_nvcsw` per +item are captured around every timed region, so **A3 is now a matter of +reading a column** rather than a separate experiment. + +`scripts/bench_repro_check.py` runs the acceptance criterion directly — K +passes, per-row deviation from the median, non-zero exit if any row exceeds +tolerance. + +**Gate not yet cleared.** A 3-pass run of `chain-{1,8}` at 10 µs on the +development laptop (20 cores, **powersave governor, on battery** — the header +now records this) lands every row within 0.7%, against the 4–8× swings this +section describes. That is encouraging but is not the acceptance run: it must +be 7 passes over the full row set on the reference machine. + +**Provisional and not to be acted on:** in that same run `chain-16` private +was 6% behind TBB, not the 28.5% in the table above. If that survives the +real acceptance run, the deep-chain deficit is substantially a measurement +artefact of the N=200 rows and workstream A shrinks accordingly. + +--- + +## 4. Workstream A — deep chains + +**Hypothesis:** the deficit is thread oversubscription from the private-pool +model, not dispatch cost. + +### Investigation + +| id | experiment | falsifies the hypothesis if | +|---|---|---| +| A1 | sweep depth 8, 12, 16, 20, 24, 32 at 10 µs, private pools | the cliff is not near nproc | +| A2 | repeat A1 under `taskset -c 0-7` | the cliff does **not** move to ~depth 8 | +| A3 | `getrusage(RUSAGE_SELF).ru_nivcsw` per item, depth 8 vs 32 | involuntary context switches do not scale with depth | +| A4 | chain-16/32 on a shared pool sized 16 and 20, vs private and vs TBB | a correctly sized shared pool does not recover the gap | + +A2 is decisive and costs one run: if the cliff tracks the core count, the +mechanism is established. + +### Improvement, conditional on A4 + +If a correctly sized shared pool closes the gap, this is not an optimisation +problem — the mechanism already exists and is simply not the default: + +- **A5** — change `Network`'s default from per-node private pools to a single + shared pool sized `hardware_concurrency()`. Users should not have to know. +- **A6** — emit a diagnostic when total node threads exceed + `hardware_concurrency()`. +- **A7** — README: state the threshold, with the measured cliff. + +A5 is a change to the default execution model and must clear section 6 in full. + +### A5 now has a prerequisite (from B1/B2, 2026-08-06) + +The dispatch microbenchmark measured what a shared pool costs per dispatch, +and it is not free: **466 ns on a private `ThreadPool(1)` against ~1.7 µs on a +shared pool of 4**, because round-robin submission wakes a sleeping worker on +every dispatch (see §5). A5 as written would therefore make every graph that +currently fits inside its core count roughly 3–4× *worse* per dispatch, in +exchange for fixing graphs that exceed it. + +**A5 must not land before the wake cost does.** The order is B9/B5 first, +then A5, and A4 must be read with this in mind: if a shared pool "recovers the +gap" at depth 32, check what it costs at depth 4 in the same run before +changing any default. + +This partially inverts the prediction in §7: workstream A is not purely a +default-and-documentation change, because the default it would switch to is +currently the slower one per dispatch. + +--- + +## 5. Workstream B — fanout dispatch cost + +**Hypothesis:** a fixed ~250 ns per node dispatch, paid ~5× per item in +`wide-4`. Unlike workstream A, this genuinely is dispatch efficiency. + +Estimated budget for ~290 ns, per item — **estimates, to be replaced by B3**: + +| cost | est. | +|---|---| +| `shared_lock(lifecycle_mx_)` in `submit()` | 20–40 ns | +| `queues_[target]->mx` lock/unlock | 20–40 ns | +| `priority_queue` push + pop (heap ops, `std::function` moves) | 50–100 ns | +| `{ lock_guard lk(cv_mx_); }` + `notify_one()` | 20–40 ns, or µs if a worker actually sleeps | +| 2–3 × `clock_t::now()` in `fire_once` | 50–75 ns | +| gate CAS + ~6 stats atomics | 30–60 ns | + +### Investigation — measure before touching anything + +- **B1** — microbenchmark submit→execute turnaround for a null task on + `ThreadPool(1)` and `ThreadPool(4)`. Yields ns/dispatch directly, in seconds + rather than minutes. +- **B2** — **does a worker actually sleep per item?** Count `cv_.wait` returns, + or `strace -c -f -e futex`. The entire spin-window hypothesis depends on + this; if workers are not sleeping, B5 is worthless and drops off the list. +- **B3** — ablation, one variant per suspected cost, each measured against B1 + rather than guessed at: + +| variant | suspected cost | +|---|---| +| stats and clock calls compiled out | 2–3 × `clock_t::now()` plus ~6 atomics per firing | +| `priority_queue` → FIFO ring | heap operations, `std::function` moves | +| `shared_lock(lifecycle_mx_)` removed (**measurement only, unsafe**) | `include/kpn/scheduler.hpp:113` | +| bounded spin before sleeping | `include/kpn/scheduler.hpp:210-227` | + +### B1/B2 — first results, 2026-08-06 + +`benchmarks/bench_dispatch.cpp` answers both without touching the library. +Sleeping is inferred from `ru_nvcsw`: a thread blocking on a condition +variable books a voluntary context switch, so voluntary switches per task is +sleeps per task. Three modes, because "the cost of a dispatch" is three +numbers: `latency` (idle pool, one task in flight), `batch` (submit flat out, +drain once), `steady` (the task resubmits its successor, as `fire_once` does). + +Laptop, powersave, battery, 3 reps — **the nanoseconds are provisional; the +sleep counts are structural and will hold.** `steady`, 10 µs payload: + +| pool threads | ns/dispatch | sleeps/task | +|---|---|---| +| 1 | 466 | **0.00** | +| 2 | 1494 | 0.97 | +| 4 | 1722 | 1.00 | +| 8 | 1996 | 1.00 | + +**B1 is answered and the abandon criterion is not met.** A `ThreadPool(1)` +dispatch is 291 ns for a null task, 466 ns with a payload — against the ~290 ns +the section-1 budget estimated for `chain-1`. The estimate was good. Dispatch +cost is not already under 100 ns, so workstream B stays alive. + +**B2 is answered, and the answer is conditional — which the question did not +anticipate.** It is not "do workers sleep?" but "which pool?": + +- On a private `ThreadPool(1)` — the `Node<>` default — the worker **never** + sleeps. It resubmits into its own queue and finds the work already there. +- On any pool of 2 or more, a worker sleeps **exactly once per task**. + +`submit()` round-robins (`next_.fetch_add(1) % thread_count_`, +`scheduler.hpp:131`), so on a shared pool every task is handed to a *different* +worker, which is asleep, and every single dispatch pays a futex wake. That is +the entire 466 ns → 1.7 µs difference. + +Consequently **B5 (bounded spin) is worthless for the default configuration** +and is the highest-value item for shared pools. It does not drop off the list, +it moves onto a different one. + +### B9 — submit-to-self affinity (new, not in the original plan) + +If a `submit()` originating on a pool worker pushed to *that worker's own* +queue instead of round-robining, the shared pool would inherit the property +that makes `ThreadPool(1)` fast: the work is already local when the worker +loops, so no wake. This is roughly what TBB does, and it plausibly subsumes +most of B5 at lower risk — it changes task placement, not the sleep/wake +protocol that the August wedge fixes hardened. Work stealing already exists to +correct the resulting imbalance. + +Measure before believing it: an affinity policy can starve peers, and +`try_steal` only rebalances when a peer goes idle. + +### Improvement — only what B3 shows pays + +1. **B4 — compile-time-optional instrumentation.** No concurrency risk; the + only item here that cannot reintroduce a wedge. Worth doing regardless. +2. **B5 — bounded spin before sleeping**, mirroring the channel's existing + `spin_count_` (~4 µs). Note the tension: b9698fa deliberately moved from + "spin whenever any task runs" to "sleep as soon as nothing is queued" in + order to fix pathological spinning. A *bounded* window is the middle + ground; unbounded spin would undo that fix. +3. **B6 — cheaper queue on the common path.** A private pool holds ≤1–2 tasks; + `priority_queue` is heavy for that. +4. **B7 — batched firing.** `fire_once` processes one token then re-submits; + looping while inputs stay ready, bounded, amortises the submit, gate CAS + and wake. The largest algorithmic win, but it changes latency and + interacts with `compute_priority()`. +5. **B8 — `lifecycle_mx_` off the hot path.** Last, and possibly never. It is + load-bearing: it prevents `submit()` racing `stop()`'s `queues_.clear()`, + a documented segfault reproducible "about 12 runs in 20". + +**Abandon criteria:** if B1 shows dispatch cost already under ~100 ns, or the +best surviving variant buys under 5%, stop and document the finding. + +--- + +## 6. Guardrails + +Both workstreams modify the machinery responsible for roughly twenty wedge +fixes in August 2026, plus the lost wake fixed in 6802328. Every change: + +1. **146/146** ctest, examples included. +2. **Wedge soak before and after** — `benchmarks/repro_wedge.cpp`, ≥50k + iterations clean. Reference point: the pre-6802328 code wedged 5/5 inside + 45 s, at iterations 149, 1249, 332, 1740 and 493. +3. **ThreadSanitizer** on scheduler and pool_node tests for any change to + either. +4. **One change at a time**, measured independently. Bundling is how the + August audit became twenty commits. +5. **G1 — wire the reproducer in as an opt-in CTest stress target** + (e.g. `-L soak`) so that performance work cannot silently reintroduce a + wedge. This should land before either workstream starts. + +### G1 — implemented 2026-08-06 + +`tests/soak_wedge.cpp` (supersedes `benchmarks/repro_wedge.cpp`, which was +never wired into any build and can be deleted). Always compiled so it cannot +rot; its CTest cases register only under `-DKPN_ENABLE_SOAK_TESTS=ON`, so the +default `ctest` count is unchanged. + +``` +cmake -B build -DKPN_ENABLE_SOAK_TESTS=ON -DKPN_SOAK_ITERS=50000 +cmake --build build --target kpn_soak_wedge +ctest --test-dir build -L soak +``` + +Two cases: `soak.wedge.pool` (depth 4, 4 threads — the configuration the +August wedges were reproduced on) and `soak.wedge.private` (depth 8, one pool +per node — the model workstream A would change). Both parameterised, so +A5-style changes can be soaked at the depth that matters. + +A wedge is a hang, and a hang under CTest is an unattributable timeout, so the +binary carries a watchdog: if an iteration stops making progress for +`--watchdog-sec` it aborts naming the iteration and the phase (`pushed`, +`drained`, `nodes stopped`, `pool stopped`). Measured cost: ~13 ms per +iteration, so the 50k-iteration guardrail is ~11 minutes. + +**Guardrail 1 needs a correction.** The stated reference is 146/146; the +tests-only configuration used here reports **136/136 passing**, and neither +`examples/` nor `python/` registers any `add_test`. The true reference count +must be pinned down before it is used to certify a change. + +--- + +## 7. Sequencing + +| phase | contents | gate to proceed | state | +|---|---|---|---| +| 0 | workload question; M1–M6; G1 | ±5% reproducibility achieved | **tooling done**, acceptance run outstanding | +| 1 | A1–A4 | A2 confirms the cliff tracks core count | harness supports it; not run | +| 2 | A5–A7, or documentation only | A4 shows a shared pool recovers the gap | **now gated on B9/B5** | +| 3 | B1–B3 | B2 answers the sleep question | **B1/B2 answered**; B3 outstanding | +| 4 | B4, then whichever of B5–B8 survived B3 | each ≥5% and soak-clean | B5 rescoped to shared pools | + +B1/B2 ran early because the microbenchmark cost seconds rather than minutes, +and the result reordered phases 2 and 4 — the shared-pool default now depends +on the wake cost being fixed first. Phase 1 is unchanged but its A4 row needs +a shallow-depth control, per §4. + +### Reproducing this + +``` +cmake -B build_bench -DKPN_BUILD_BENCHMARKS=ON -DCMAKE_BUILD_TYPE=Release +cmake --build build_bench -j + +# Phase 0 acceptance — must pass before any number below is acted on +python3 scripts/bench_repro_check.py ./build_bench/benchmarks/bench_pipeline \ + --passes 7 --tolerance 5 -- --work=10,100 --reps=5 + +# B1/B2 +./build_bench/benchmarks/bench_dispatch --threads=1,2,4,8,20 --reps=5 \ + | tee dispatch.csv + +# A1/A2 — the depth sweep, and the same under taskset to move the cliff +./build_bench/benchmarks/bench_pipeline --work=10 --topos=chain \ + --depths=8,12,16,20,24,32 --modes=priv,tbb --reps=5 | tee a1.csv +taskset -c 0-7 ./build_bench/benchmarks/bench_pipeline --work=10 \ + --topos=chain --depths=4,6,8,10,12,16,32 --modes=priv,tbb --reps=5 | tee a2.csv + +# A4 — shared pool sized to the machine, against private and TBB. +# Include a shallow depth: A5's risk is what a shared pool costs when the +# graph already fits in its cores. +./build_bench/benchmarks/bench_pipeline --work=10 --topos=chain \ + --depths=4,16,32 --pools=16,20 --reps=5 | tee a4.csv +``` + +Check the `# governor=` line in each CSV before trusting it. A3 needs no +separate run: `ivcsw_per_item` is a column in every row above. + +**Success criteria** + +- chain-32 @10 within 10% of TBB in the recommended configuration +- wide-4 @10 within 5% of TBB +- zero wedges across 100k soak iterations + +**Prediction, recorded so it can be proven wrong:** workstream A resolves into +a default-and-documentation change rather than an optimisation, and workstream +B yields 5–10% on fanout from B4 and B5, with the remainder not worth the risk. + +**Prediction, revised 2026-08-06 after B1/B2** — the original is already half +wrong and is left above unedited: + +- Workstream A does *not* resolve into a documentation change, because the + shared pool it would recommend costs 3–4× more per dispatch than the private + default. It resolves into B9 first. +- The largest single win is not B4, B5 or B7 but **B9, submit-to-self + affinity**: one sleep per dispatch is being paid on every shared pool, and + eliminating it is worth roughly 1.2 µs per dispatch — far more than the + 5–10% predicted for fanout. +- Standing: `chain-16`'s 28.5% deficit is a measurement artefact of N=200. + +--- + +## 8. Related correction + +Independently of the above, the README's TBB comparison overstates its case. +The claim that KPN++ beats TBB "for every chain and diamond topology at +100 µs/node" is not supported: at 100 µs only chain-1 and diamond lean KPN, +while chain-16, chain-32 and wide-4 lean TBB. The tables are also quoted in +derived overhead, which magnifies small differences — the same rows expressed +as throughput are mostly within a few percent. Restating them in items/sec +would be both more accurate and more favourable. diff --git a/benchmarks/CMakeLists.txt b/benchmarks/CMakeLists.txt index 97ef6f4..8a104fa 100644 --- a/benchmarks/CMakeLists.txt +++ b/benchmarks/CMakeLists.txt @@ -4,6 +4,13 @@ add_executable(bench_pipeline bench_pipeline.cpp) target_link_libraries(bench_pipeline PRIVATE kpn) target_compile_options(bench_pipeline PRIVATE -O3 -march=native) +# Dispatch microbenchmark (PERF_PLAN B1/B2): ns per ThreadPool dispatch, and +# whether a worker actually sleeps per task. No TBB comparison — it measures +# KPN's own scheduler, not a competitor. +add_executable(bench_dispatch bench_dispatch.cpp) +target_link_libraries(bench_dispatch PRIVATE kpn) +target_compile_options(bench_dispatch PRIVATE -O3 -march=native) + find_package(TBB QUIET) if(TBB_FOUND) target_link_libraries(bench_pipeline PRIVATE TBB::tbb) diff --git a/benchmarks/bench_dispatch.cpp b/benchmarks/bench_dispatch.cpp new file mode 100644 index 0000000..c159634 --- /dev/null +++ b/benchmarks/bench_dispatch.cpp @@ -0,0 +1,307 @@ +// Dispatch microbenchmark — PERF_PLAN B1 and B2. +// +// B1 asks what a single ThreadPool dispatch costs. B2 asks whether a worker +// actually sleeps per item, because the whole spin-window hypothesis (B5) +// depends on the answer: if workers are not sleeping, a spin window buys +// nothing and drops off the list. +// +// Both are answered here without touching the library. Sleeping is inferred +// from ru_nvcsw — a thread blocking on a condition variable books a voluntary +// context switch — so `vcsw/task` near 1.0 means a sleep per dispatch and near +// 0 means the worker never went to sleep at all. +// +// Three modes, because "the cost of a dispatch" is three different numbers: +// +// latency — one task in flight, pool idle in between. The worker is asleep +// at every submission, so this is dispatch cost *including* a +// wake. Worst case, and the case a spin window would attack. +// +// batch — submit K no-op tasks flat out, then drain. The worker is never +// idle, so this is the amortised floor: queue and heap operations +// with no wake at all. Reports the producer-side submit() cost +// separately from end-to-end throughput. +// +// steady — the task resubmits its successor, one in flight, each doing +// --work-us of work. This is what a KPN node actually does: +// fire_once processes a token and resubmits. On ThreadPool(1) the +// worker resubmits to its own queue; on ThreadPool(4) round-robin +// hands the task to a *different* worker, which may be asleep. +// That difference is the fanout cost wide-4 pays ~5x per item. +// +// Usage: ./bench_dispatch [--threads=1,2,4] [--mode=latency,batch,steady] +// [--tasks=200000] [--work-us=0] [--reps=5] [--warmup=1] + +#include + +#include "bench_env.hpp" + +#include +#include +#include +#include +#include +#include +#include +#include + +using namespace kpn; +using sclock = std::chrono::steady_clock; + +struct Opts { + std::vector threads {1, 2, 4}; + std::vector modes {"latency", "batch", "steady"}; + long tasks = 200000; + int work_us = 0; + int reps = 5; + int warmup = 1; +}; + +static Opts g_opts; + +static void busy_us(int us) { + if (us <= 0) return; + auto end = sclock::now() + std::chrono::microseconds(us); + while (sclock::now() < end); +} + +struct Sample { + double ns_per_dispatch = 0; // end-to-end, minus the work payload + double submit_ns = 0; // producer side only (batch mode) + double vcsw_per_task = 0; // B2: sleeps per dispatch + double ivcsw_per_task = 0; +}; + +// ── latency: one task at a time, worker asleep between submissions ──────────── + +static Sample run_latency(int threads, long tasks) { + ThreadPool pool(threads); + pool.start(); + + std::mutex mx; + std::condition_variable cv; + bool done = false; + + bench::RusageDelta ru; ru.start(); + auto t0 = sclock::now(); + for (long i = 0; i < tasks; ++i) { + { std::lock_guard lk(mx); done = false; } + pool.submit([&] { + busy_us(g_opts.work_us); + { std::lock_guard lk(mx); done = true; } + cv.notify_one(); + }); + std::unique_lock lk(mx); + cv.wait(lk, [&] { return done; }); + } + auto t1 = sclock::now(); + Sample s; + long iv = 0, vc = 0; + ru.finish(iv, vc); + pool.stop(); + + double elapsed_ns = std::chrono::duration(t1 - t0).count(); + s.ns_per_dispatch = elapsed_ns / tasks - g_opts.work_us * 1000.0; + // The requesting thread blocks once per task too, so it books a voluntary + // switch of its own; halve to attribute per side rather than per process. + s.vcsw_per_task = static_cast(vc) / tasks / 2.0; + s.ivcsw_per_task = static_cast(iv) / tasks; + return s; +} + +// ── batch: submit flat out, drain once. No wake in the steady state ─────────── + +static Sample run_batch(int threads, long tasks) { + ThreadPool pool(threads); + pool.start(); + + std::atomic ran{0}; + + bench::RusageDelta ru; ru.start(); + auto t0 = sclock::now(); + for (long i = 0; i < tasks; ++i) + pool.submit([&] { + busy_us(g_opts.work_us); + ran.fetch_add(1, std::memory_order_relaxed); + }); + auto t_submitted = sclock::now(); + pool.drain(); + auto t1 = sclock::now(); + Sample s; + long iv = 0, vc = 0; + ru.finish(iv, vc); + pool.stop(); + + if (ran.load() != tasks) + std::fprintf(stderr, "WARNING: batch ran %ld of %ld tasks\n", + ran.load(), tasks); + + double elapsed_ns = std::chrono::duration(t1 - t0).count(); + s.ns_per_dispatch = elapsed_ns / tasks - g_opts.work_us * 1000.0; + s.submit_ns = std::chrono::duration(t_submitted - t0).count() / tasks; + s.vcsw_per_task = static_cast(vc) / tasks; + s.ivcsw_per_task = static_cast(iv) / tasks; + return s; +} + +// ── steady: the task resubmits its successor, as fire_once does ────────────── + +static Sample run_steady(int threads, long tasks) { + ThreadPool pool(threads); + pool.start(); + + std::mutex mx; + std::condition_variable cv; + std::atomic count{0}; + bool finished = false; + // Recursive submission: hold the chain in a std::function so the task can + // resubmit itself. Captured by reference; it outlives the drain below. + // + // The counter is atomic rather than mutex-guarded so that this loop + // measures the pool's dispatch path and not a lock of the benchmark's own. + std::function step = [&] { + busy_us(g_opts.work_us); + long n = count.fetch_add(1, std::memory_order_relaxed) + 1; + if (n < tasks) { + pool.submit(step); + } else { + { std::lock_guard lk(mx); finished = true; } + cv.notify_one(); + } + }; + + bench::RusageDelta ru; ru.start(); + auto t0 = sclock::now(); + pool.submit(step); + { + std::unique_lock lk(mx); + cv.wait(lk, [&] { return finished; }); + } + auto t1 = sclock::now(); + Sample s; + long iv = 0, vc = 0; + ru.finish(iv, vc); + pool.stop(); + + double elapsed_ns = std::chrono::duration(t1 - t0).count(); + s.ns_per_dispatch = elapsed_ns / tasks - g_opts.work_us * 1000.0; + s.vcsw_per_task = static_cast(vc) / tasks; + s.ivcsw_per_task = static_cast(iv) / tasks; + return s; +} + +// ── driver ──────────────────────────────────────────────────────────────────── + +static void run_row(const std::string& mode, int threads, long tasks) { + auto once = [&] { + if (mode == "latency") return run_latency(threads, tasks); + if (mode == "batch") return run_batch(threads, tasks); + return run_steady(threads, tasks); + }; + + for (int i = 0; i < g_opts.warmup; ++i) (void)once(); + + std::vector ns, sub, vcsw, ivcsw; + for (int i = 0; i < g_opts.reps; ++i) { + Sample s = once(); + ns.push_back(s.ns_per_dispatch); + sub.push_back(s.submit_ns); + vcsw.push_back(s.vcsw_per_task); + ivcsw.push_back(s.ivcsw_per_task); + } + + const double med = bench::percentile(ns, 0.5); + const double q1 = bench::percentile(ns, 0.25); + const double q3 = bench::percentile(ns, 0.75); + const double iqr = med > 0 ? 100.0 * (q3 - q1) / med : 0.0; + const double sleeps = bench::percentile(vcsw, 0.5); + + std::fprintf(stderr, "%-9s %-8d %-8d %-10ld %-12.0f %-7.1f %-11.0f %-10.2f %-10.2f\n", + mode.c_str(), threads, g_opts.work_us, tasks, med, iqr, + bench::percentile(sub, 0.5), sleeps, + bench::percentile(ivcsw, 0.5)); + // Column names deliberately match bench_pipeline's key columns so that + // scripts/bench_repro_check.py can gate this benchmark too. + std::printf("%s,%d,%d,%d,%ld,%d,%.1f,%.2f,%.1f,%.3f,%.3f\n", + mode.c_str(), threads, g_opts.work_us, threads, tasks, + g_opts.reps, med, iqr, bench::percentile(sub, 0.5), + sleeps, bench::percentile(ivcsw, 0.5)); + std::fflush(stdout); +} + +static std::vector parse_int_list(const char* s) { + std::vector out; + const char* p = s; + while (*p) { + char* end = nullptr; + long v = std::strtol(p, &end, 10); + if (end == p) break; + out.push_back(static_cast(v)); + p = end; + while (*p == ',' || *p == ' ') ++p; + } + return out; +} + +static std::vector parse_word_list(const std::string& s) { + std::vector out; + std::size_t pos = 0; + while (pos <= s.size()) { + std::size_t c = s.find(',', pos); + if (c == std::string::npos) c = s.size(); + if (c > pos) out.push_back(s.substr(pos, c - pos)); + pos = c + 1; + } + return out; +} + +static void usage() { + std::fprintf(stderr, + "usage: bench_dispatch [options]\n" + " --threads=1,2,4 pool sizes\n" + " --mode=latency,batch,steady which measurements to run\n" + " --tasks=200000 dispatches per repetition\n" + " --work-us=0 payload per task\n" + " --reps=5 --warmup=1\n"); +} + +int main(int argc, char** argv) { + for (int i = 1; i < argc; ++i) { + std::string a = argv[i]; + auto eq = a.find('='); + std::string key = a.substr(0, eq); + std::string val = eq == std::string::npos ? "" : a.substr(eq + 1); + + if (key == "--help" || key == "-h") { usage(); return 0; } + else if (key == "--threads") g_opts.threads = parse_int_list(val.c_str()); + else if (key == "--mode") g_opts.modes = parse_word_list(val); + else if (key == "--tasks") g_opts.tasks = std::atol(val.c_str()); + else if (key == "--work-us") g_opts.work_us = std::atoi(val.c_str()); + else if (key == "--reps") g_opts.reps = std::atoi(val.c_str()); + else if (key == "--warmup") g_opts.warmup = std::atoi(val.c_str()); + else { std::fprintf(stderr, "unknown option: %s\n", a.c_str()); usage(); return 2; } + } + if (g_opts.reps < 1) g_opts.reps = 1; + if (g_opts.warmup < 0) g_opts.warmup = 0; + + char cfg[160]; + std::snprintf(cfg, sizeof cfg, "tasks=%ld work_us=%d reps=%d warmup=%d", + g_opts.tasks, g_opts.work_us, g_opts.reps, g_opts.warmup); + bench::print_environment(cfg); + + std::fprintf(stderr, "\n%-9s %-8s %-8s %-10s %-12s %-7s %-11s %-10s %-10s\n", + "mode", "threads", "work_us", "tasks", "ns/dispatch", "iqr%", + "submit_ns", "vcsw/task", "ivcsw/task"); + std::fprintf(stderr, "%s\n", std::string(96, '-').c_str()); + std::printf("topology,size,work_us,threads,items,reps,ns_per_dispatch," + "iqr_pct,submit_ns,vcsw_per_task,ivcsw_per_task\n"); + + // latency is a round trip per task, so it is far slower per dispatch than + // the other modes; scale it down rather than run for minutes. + for (const auto& mode : g_opts.modes) + for (int t : g_opts.threads) { + long tasks = mode == "latency" + ? std::max(2000L, g_opts.tasks / 20) + : g_opts.tasks; + run_row(mode, t, tasks); + } +} diff --git a/benchmarks/bench_env.hpp b/benchmarks/bench_env.hpp new file mode 100644 index 0000000..92ccc4a --- /dev/null +++ b/benchmarks/bench_env.hpp @@ -0,0 +1,108 @@ +// Shared benchmark plumbing: machine attribution (PERF_PLAN M6), repetition +// statistics (M3), and context-switch capture. +// +// The attribution is not decoration. A result taken under the powersave +// governor or on battery is not comparable with one taken on AC under +// performance, and a stored CSV that does not say which it was cannot be +// argued about later. + +#pragma once + +#include +#include +#include +#include +#include +#include + +#include + +namespace bench { + +inline int hw_units() { + unsigned n = std::thread::hardware_concurrency(); + return n ? static_cast(n) : 1; +} + +inline std::string read_line_of(const char* path) { + std::FILE* f = std::fopen(path, "r"); + if (!f) return "unknown"; + char buf[128] = {0}; + if (!std::fgets(buf, sizeof buf, f)) { std::fclose(f); return "unknown"; } + std::fclose(f); + std::string s(buf); + while (!s.empty() && (s.back() == '\n' || s.back() == ' ')) s.pop_back(); + return s.empty() ? "unknown" : s; +} + +inline std::string ac_state() { + for (const char* p : {"/sys/class/power_supply/AC/online", + "/sys/class/power_supply/AC0/online", + "/sys/class/power_supply/ACAD/online", + "/sys/class/power_supply/ADP1/online"}) { + std::string v = read_line_of(p); + if (v != "unknown") return v == "1" ? "ac" : "battery"; + } + return "unknown"; +} + +// M6 — emitted to both streams: the CSV so a stored result can be attributed, +// the terminal so a run under the wrong governor is noticed while it happens. +inline void print_environment(const std::string& config_line) { + const std::string gov = read_line_of( + "/sys/devices/system/cpu/cpu0/cpufreq/scaling_governor"); + const std::string ac = ac_state(); + + for (std::FILE* out : {stdout, stderr}) { + std::fprintf(out, "# nproc=%d governor=%s power=%s\n", + hw_units(), gov.c_str(), ac.c_str()); + if (!config_line.empty()) + std::fprintf(out, "# %s\n", config_line.c_str()); +#if defined(__GNUC__) && !defined(__clang__) + std::fprintf(out, "# compiler=gcc-%d.%d.%d\n", + __GNUC__, __GNUC_MINOR__, __GNUC_PATCHLEVEL__); +#elif defined(__clang__) + std::fprintf(out, "# compiler=clang-%d.%d.%d\n", + __clang_major__, __clang_minor__, __clang_patchlevel__); +#endif + } + if (gov != "performance" || ac == "battery") + std::fprintf(stderr, + "# WARNING: governor=%s power=%s — results are not comparable with\n" + "# a run on AC power under the performance governor.\n", + gov.c_str(), ac.c_str()); +} + +inline double percentile(std::vector v, double p) { + if (v.empty()) return 0; + std::sort(v.begin(), v.end()); + double idx = p * (v.size() - 1); + auto lo = static_cast(std::floor(idx)); + auto hi = static_cast(std::ceil(idx)); + return v[lo] + (v[hi] - v[lo]) * (idx - lo); +} + +// Process-wide context-switch counters, sampled around a timed region. +// +// ru_nvcsw (voluntary) is the cheap answer to PERF_PLAN B2: a thread that +// blocks on a condition variable books a voluntary switch, so voluntary +// switches per dispatch is, near enough, sleeps per dispatch. ru_nivcsw +// (involuntary) is preemption, which is what oversubscription looks like (A3). +struct RusageDelta { + long ivcsw0 = 0, vcsw0 = 0; + + void start() { + rusage ru{}; + getrusage(RUSAGE_SELF, &ru); + ivcsw0 = ru.ru_nivcsw; + vcsw0 = ru.ru_nvcsw; + } + void finish(long& nivcsw, long& nvcsw) const { + rusage ru{}; + getrusage(RUSAGE_SELF, &ru); + nivcsw = ru.ru_nivcsw - ivcsw0; + nvcsw = ru.ru_nvcsw - vcsw0; + } +}; + +} // namespace bench diff --git a/benchmarks/bench_pipeline.cpp b/benchmarks/bench_pipeline.cpp index c0e3b09..dc4ed3a 100644 --- a/benchmarks/bench_pipeline.cpp +++ b/benchmarks/bench_pipeline.cpp @@ -9,24 +9,38 @@ // private — each node owns a private ThreadPool(1) [Node<>] // pool — all nodes share one ThreadPool(T) [PoolNode<> + shared pool] // -// Usage: ./bench_pipeline | tee results.csv +// Each row is run --reps times (plus discarded warm-up runs); the reported +// figure is the median items/sec, with the inter-quartile spread as a +// reliability indicator. A row whose iqr_pct is above a few percent is not +// measuring what it claims to measure. +// +// Usage: ./bench_pipeline [options] | tee results.csv +// ./bench_pipeline --help #include +#include "bench_env.hpp" + #ifdef KPN_BENCH_TBB #include namespace tbb_flow = oneapi::tbb::flow; #endif +#include #include #include #include +#include #include +#include +#include #include #include #include #include +#include + using namespace kpn; using namespace std::chrono_literals; using sclock = std::chrono::steady_clock; @@ -57,31 +71,63 @@ static void push_retry(Channel& ch, int val) { } } -// ── result ──────────────────────────────────────────────────────────────────── +// ── configuration (M1, M3, M4, M5) ──────────────────────────────────────────── -struct Result { - const char* topology; - int size; - int work_us; - int threads; // 0 = private (1 thread per node), N = shared pool size - double items_per_sec; - double overhead_us; +struct Config { + std::vector work_amts {10, 100, 1000}; + std::vector pool_sizes{1, 2, 4, 8, 16, 20}; // M5 + std::vector depths {1, 2, 4, 8, 16, 32}; + std::vector widths {1, 2, 3, 4}; + int reps = 5; // M3: measured repetitions per row + int warmup = 1; // M4: discarded repetitions per row + double target_sec = 0.30; // aimed-for duration of one repetition + long min_items = 2000; // M1: floor, independent of work_us and depth + double max_sec = 3.0; // ceiling; only bites where min_items cannot fit + bool do_chain = true, do_wide = true, do_diamond = true; + bool do_priv = true, do_pool = true, do_tbb = true; +}; + +static Config g_cfg; + +// M1 — sample size from a time budget with a hard floor, rather than a +// hand-tuned ladder that collapsed to 50–200 items on exactly the rows under +// investigation. +// +// `stages` is the number of node firings per item; `units` the number of +// threads able to run them concurrently. Steady-state throughput of the +// pipeline is bounded by work_us * stages / units, so that is the per-item +// cost the sample size is derived from. Depth beyond `units` costs throughput; +// depth below it costs only latency, which does not scale the run. +static long pick_items(int work_us, int stages, int units) { + units = std::max(1, std::min(units, bench::hw_units())); + const double per_item_us = + std::max(1.0, static_cast(work_us)) * + std::max(1.0, static_cast(stages) / units); + + long want = static_cast(g_cfg.target_sec * 1e6 / per_item_us); + long cap = static_cast(g_cfg.max_sec * 1e6 / per_item_us); + + want = std::max(want, g_cfg.min_items); + // The floor wins unless honouring it would blow the time ceiling by more + // than the ceiling allows; such rows are reported with their true N so the + // reader can see they are short. + if (want > cap) want = std::max(cap, 200L); + return want; +} + +// ── one measured repetition ─────────────────────────────────────────────────── + +struct Sample { + double items_per_sec = 0; + double overhead_us = 0; + long nivcsw = 0; // involuntary context switches during the run + long nvcsw = 0; // voluntary context switches during the run }; // ── chain ───────────────────────────────────────────────────────────────────── -static int items_for(int work_us, int depth = 1) { - int effective = std::max(1, work_us) * std::max(1, depth); - if (effective <= 1) return 5000; - if (effective <= 10) return 3000; - if (effective <= 100) return 1000; - if (effective <= 1000) return 200; - return 50; -} - -static Result bench_chain(int depth, int work_us) { - const int N = items_for(work_us, depth); - const int CAP = N; +static Sample bench_chain(int depth, int work_us, long N) { + const std::size_t CAP = static_cast(N); std::vector>> chs; for (int i = 0; i <= depth; ++i) @@ -98,17 +144,20 @@ static Result bench_chain(int depth, int work_us) { std::atomic t1; std::thread reader([&] { - for (int i = 0; i < N; ++i) chs.back()->pop(); + for (long i = 0; i < N; ++i) chs.back()->pop(); t1.store(sclock::now(), std::memory_order_release); }); + bench::RusageDelta ru; ru.start(); auto t0 = sclock::now(); std::thread pusher([&] { - for (int i = 0; i < N; ++i) push_retry(*chs[0], i); + for (long i = 0; i < N; ++i) push_retry(*chs[0], static_cast(i)); }); pusher.join(); reader.join(); + Sample s; + ru.finish(s.nivcsw, s.nvcsw); for (auto& n : nodes) n->stop(); double elapsed = std::chrono::duration( @@ -116,13 +165,13 @@ static Result bench_chain(int depth, int work_us) { // Subtract theoretical pipeline fill cost (depth-1)*W so that overhead // reflects only framework latency, not the expected pipeline startup time. double pipeline_us = static_cast(work_us) * (N + depth - 1); - double wus = (elapsed * 1e6 - pipeline_us) / N; - return {"chain", depth, work_us, 0, N / elapsed, wus}; + s.overhead_us = (elapsed * 1e6 - pipeline_us) / N; + s.items_per_sec = N / elapsed; + return s; } -static Result bench_chain_pool(int depth, int work_us, int pool_threads) { - const int N = items_for(work_us, depth); - const int CAP = N; +static Sample bench_chain_pool(int depth, int work_us, int pool_threads, long N) { + const std::size_t CAP = static_cast(N); auto pool = std::make_shared(pool_threads); @@ -142,33 +191,36 @@ static Result bench_chain_pool(int depth, int work_us, int pool_threads) { std::atomic t1; std::thread reader([&] { - for (int i = 0; i < N; ++i) chs.back()->pop(); + for (long i = 0; i < N; ++i) chs.back()->pop(); t1.store(sclock::now(), std::memory_order_release); }); + bench::RusageDelta ru; ru.start(); auto t0 = sclock::now(); std::thread pusher([&] { - for (int i = 0; i < N; ++i) push_retry(*chs[0], i); + for (long i = 0; i < N; ++i) push_retry(*chs[0], static_cast(i)); }); pusher.join(); reader.join(); + Sample s; + ru.finish(s.nivcsw, s.nvcsw); for (auto& n : nodes) n->stop(); pool->stop(); double elapsed = std::chrono::duration( t1.load(std::memory_order_acquire) - t0).count(); double pipeline_us = static_cast(work_us) * (N + depth - 1); - double wus = (elapsed * 1e6 - pipeline_us) / N; - return {"chain", depth, work_us, pool_threads, N / elapsed, wus}; + s.overhead_us = (elapsed * 1e6 - pipeline_us) / N; + s.items_per_sec = N / elapsed; + return s; } // ── wide (fanout) ────────────────────────────────────────────────────────── template -static Result bench_wide(int work_us) { - const int N = items_for(work_us); - const int CAP = N; +static Sample bench_wide(int work_us, long N) { + const std::size_t CAP = static_cast(N); auto src_ch = std::make_shared>(CAP); auto fan = std::make_unique>(CAP); @@ -197,33 +249,36 @@ static Result bench_wide(int work_us) { for (std::size_t w = 0; w < W; ++w) { readers[w] = std::thread([&, w] { - for (int i = 0; i < N; ++i) sink_chs[w]->pop(); + for (long i = 0; i < N; ++i) sink_chs[w]->pop(); if (readers_done.fetch_add(1, std::memory_order_acq_rel) + 1 == static_cast(W)) t1.store(sclock::now(), std::memory_order_release); }); } + bench::RusageDelta ru; ru.start(); auto t0 = sclock::now(); std::thread pusher([&] { - for (int i = 0; i < N; ++i) push_retry(*src_ch, i); + for (long i = 0; i < N; ++i) push_retry(*src_ch, static_cast(i)); }); pusher.join(); for (auto& r : readers) r.join(); + Sample s; + ru.finish(s.nivcsw, s.nvcsw); fan->stop(); for (auto& n : nodes) n->stop(); double elapsed = std::chrono::duration( t1.load(std::memory_order_acquire) - t0).count(); - double wus = (elapsed * 1e6) / N - static_cast(work_us); - return {"wide", static_cast(W), work_us, 0, N / elapsed, wus}; + s.overhead_us = (elapsed * 1e6) / N - static_cast(work_us); + s.items_per_sec = N / elapsed; + return s; } template -static Result bench_wide_pool(int work_us, int pool_threads) { - const int N = items_for(work_us); - const int CAP = N; +static Sample bench_wide_pool(int work_us, int pool_threads, long N) { + const std::size_t CAP = static_cast(N); auto pool = std::make_shared(pool_threads); auto src_ch = std::make_shared>(CAP); @@ -254,35 +309,38 @@ static Result bench_wide_pool(int work_us, int pool_threads) { for (std::size_t w = 0; w < W; ++w) { readers[w] = std::thread([&, w] { - for (int i = 0; i < N; ++i) sink_chs[w]->pop(); + for (long i = 0; i < N; ++i) sink_chs[w]->pop(); if (readers_done.fetch_add(1, std::memory_order_acq_rel) + 1 == static_cast(W)) t1.store(sclock::now(), std::memory_order_release); }); } + bench::RusageDelta ru; ru.start(); auto t0 = sclock::now(); std::thread pusher([&] { - for (int i = 0; i < N; ++i) push_retry(*src_ch, i); + for (long i = 0; i < N; ++i) push_retry(*src_ch, static_cast(i)); }); pusher.join(); for (auto& r : readers) r.join(); + Sample s; + ru.finish(s.nivcsw, s.nvcsw); fan->stop(); for (auto& n : nodes) n->stop(); pool->stop(); double elapsed = std::chrono::duration( t1.load(std::memory_order_acquire) - t0).count(); - double wus = (elapsed * 1e6) / N - static_cast(work_us); - return {"wide", static_cast(W), work_us, pool_threads, N / elapsed, wus}; + s.overhead_us = (elapsed * 1e6) / N - static_cast(work_us); + s.items_per_sec = N / elapsed; + return s; } // ── diamond ─────────────────────────────────────────────────────────────────── -static Result bench_diamond(int work_us) { - const int N = items_for(work_us, 2); - const int CAP = N; +static Sample bench_diamond(int work_us, long N) { + const std::size_t CAP = static_cast(N); auto src_ch = std::make_shared>(CAP); auto fan = std::make_unique>(CAP); @@ -312,7 +370,7 @@ static Result bench_diamond(int work_us) { std::atomic done{0}; auto make_reader = [&](Channel& ch) { return std::thread([&] { - for (int i = 0; i < N; ++i) ch.pop(); + for (long i = 0; i < N; ++i) ch.pop(); if (done.fetch_add(1, std::memory_order_acq_rel) + 1 == 2) t1.store(sclock::now(), std::memory_order_release); }); @@ -320,23 +378,26 @@ static Result bench_diamond(int work_us) { auto rL = make_reader(*snkL); auto rR = make_reader(*snkR); + bench::RusageDelta ru; ru.start(); auto t0 = sclock::now(); std::thread pusher([&] { - for (int i = 0; i < N; ++i) push_retry(*src_ch, i); + for (long i = 0; i < N; ++i) push_retry(*src_ch, static_cast(i)); }); pusher.join(); rL.join(); rR.join(); + Sample s; + ru.finish(s.nivcsw, s.nvcsw); fan->stop(); nL->stop(); nR->stop(); nL2->stop(); nR2->stop(); double elapsed = std::chrono::duration( t1.load(std::memory_order_acquire) - t0).count(); - double wus = (elapsed * 1e6) / N - static_cast(work_us); - return {"diamond", 4, work_us, 0, N / elapsed, wus}; + s.overhead_us = (elapsed * 1e6) / N - static_cast(work_us); + s.items_per_sec = N / elapsed; + return s; } -static Result bench_diamond_pool(int work_us, int pool_threads) { - const int N = items_for(work_us, 2); - const int CAP = N; +static Sample bench_diamond_pool(int work_us, int pool_threads, long N) { + const std::size_t CAP = static_cast(N); auto pool = std::make_shared(pool_threads); auto src_ch = std::make_shared>(CAP); @@ -369,7 +430,7 @@ static Result bench_diamond_pool(int work_us, int pool_threads) { std::atomic done{0}; auto make_reader = [&](Channel& ch) { return std::thread([&] { - for (int i = 0; i < N; ++i) ch.pop(); + for (long i = 0; i < N; ++i) ch.pop(); if (done.fetch_add(1, std::memory_order_acq_rel) + 1 == 2) t1.store(sclock::now(), std::memory_order_release); }); @@ -377,28 +438,30 @@ static Result bench_diamond_pool(int work_us, int pool_threads) { auto rL = make_reader(*snkL); auto rR = make_reader(*snkR); + bench::RusageDelta ru; ru.start(); auto t0 = sclock::now(); std::thread pusher([&] { - for (int i = 0; i < N; ++i) push_retry(*src_ch, i); + for (long i = 0; i < N; ++i) push_retry(*src_ch, static_cast(i)); }); pusher.join(); rL.join(); rR.join(); + Sample s; + ru.finish(s.nivcsw, s.nvcsw); fan->stop(); nL->stop(); nR->stop(); nL2->stop(); nR2->stop(); pool->stop(); double elapsed = std::chrono::duration( t1.load(std::memory_order_acquire) - t0).count(); - double wus = (elapsed * 1e6) / N - static_cast(work_us); - return {"diamond", 4, work_us, pool_threads, N / elapsed, wus}; + s.overhead_us = (elapsed * 1e6) / N - static_cast(work_us); + s.items_per_sec = N / elapsed; + return s; } // ── TBB flow graph ──────────────────────────────────────────────────────────── #ifdef KPN_BENCH_TBB -static Result bench_chain_tbb(int depth, int work_us) { - const int N = items_for(work_us, depth); - +static Sample bench_chain_tbb(int depth, int work_us, long N) { tbb_flow::graph g; using FN = tbb_flow::function_node; std::vector> nodes; @@ -409,21 +472,23 @@ static Result bench_chain_tbb(int depth, int work_us) { for (int i = 0; i + 1 < depth; ++i) tbb_flow::make_edge(*nodes[i], *nodes[i + 1]); + bench::RusageDelta ru; ru.start(); auto t0 = sclock::now(); - for (int i = 0; i < N; ++i) nodes[0]->try_put(i); + for (long i = 0; i < N; ++i) nodes[0]->try_put(static_cast(i)); g.wait_for_all(); auto t1 = sclock::now(); + Sample s; + ru.finish(s.nivcsw, s.nvcsw); - double elapsed = std::chrono::duration(t1 - t0).count(); + double elapsed = std::chrono::duration(t1 - t0).count(); double pipeline_us = static_cast(work_us) * (N + depth - 1); - double wus = (elapsed * 1e6 - pipeline_us) / N; - return {"chain_tbb", depth, work_us, -1, N / elapsed, wus}; + s.overhead_us = (elapsed * 1e6 - pipeline_us) / N; + s.items_per_sec = N / elapsed; + return s; } template -static Result bench_wide_tbb(int work_us) { - const int N = items_for(work_us); - +static Sample bench_wide_tbb(int work_us, long N) { tbb_flow::graph g; tbb_flow::broadcast_node fan(g); using FN = tbb_flow::function_node; @@ -434,19 +499,21 @@ static Result bench_wide_tbb(int work_us) { tbb_flow::make_edge(fan, *n); } + bench::RusageDelta ru; ru.start(); auto t0 = sclock::now(); - for (int i = 0; i < N; ++i) fan.try_put(i); + for (long i = 0; i < N; ++i) fan.try_put(static_cast(i)); g.wait_for_all(); auto t1 = sclock::now(); + Sample s; + ru.finish(s.nivcsw, s.nvcsw); double elapsed = std::chrono::duration(t1 - t0).count(); - double wus = (elapsed * 1e6) / N - static_cast(work_us); - return {"wide_tbb", static_cast(W), work_us, -1, N / elapsed, wus}; + s.overhead_us = (elapsed * 1e6) / N - static_cast(work_us); + s.items_per_sec = N / elapsed; + return s; } -static Result bench_diamond_tbb(int work_us) { - const int N = items_for(work_us, 2); - +static Sample bench_diamond_tbb(int work_us, long N) { tbb_flow::graph g; tbb_flow::broadcast_node fan(g); using FN = tbb_flow::function_node; @@ -456,71 +523,242 @@ static Result bench_diamond_tbb(int work_us) { tbb_flow::make_edge(fan, nL); tbb_flow::make_edge(fan, nR); tbb_flow::make_edge(nL, nL2); tbb_flow::make_edge(nR, nR2); + bench::RusageDelta ru; ru.start(); auto t0 = sclock::now(); - for (int i = 0; i < N; ++i) fan.try_put(i); + for (long i = 0; i < N; ++i) fan.try_put(static_cast(i)); g.wait_for_all(); auto t1 = sclock::now(); + Sample s; + ru.finish(s.nivcsw, s.nvcsw); double elapsed = std::chrono::duration(t1 - t0).count(); - double wus = (elapsed * 1e6) / N - static_cast(work_us); - return {"diamond_tbb", 4, work_us, -1, N / elapsed, wus}; + s.overhead_us = (elapsed * 1e6) / N - static_cast(work_us); + s.items_per_sec = N / elapsed; + return s; } #endif // KPN_BENCH_TBB +// ── repetition driver (M2, M3, M4) ──────────────────────────────────────────── + +using bench::percentile; + +// A row: median of `reps` repetitions, after `warmup` discarded ones. +// M2 — items/sec is the primary figure; derived overhead is secondary, +// because it is a difference of large numbers and magnifies noise ~10×. +template +static void run_row(const char* topology, int size, int work_us, int sched, + long N, Fn&& one_rep) { + for (int i = 0; i < g_cfg.warmup; ++i) (void)one_rep(); // M4 + + std::vector ips, ovh; + long ivcsw = 0, vcsw = 0; + for (int i = 0; i < g_cfg.reps; ++i) { + Sample s = one_rep(); + ips.push_back(s.items_per_sec); + ovh.push_back(s.overhead_us); + ivcsw += s.nivcsw; + vcsw += s.nvcsw; + } + + const double med = percentile(ips, 0.5); + const double q1 = percentile(ips, 0.25); + const double q3 = percentile(ips, 0.75); + const double iqr = med > 0 ? 100.0 * (q3 - q1) / med : 0.0; + const double lo = *std::min_element(ips.begin(), ips.end()); + const double hi = *std::max_element(ips.begin(), ips.end()); + const double spread = med > 0 ? 100.0 * (hi - lo) / med : 0.0; + const double ivcsw_per_item = static_cast(ivcsw) / (double(N) * g_cfg.reps); + const double vcsw_per_item = static_cast(vcsw) / (double(N) * g_cfg.reps); + + const std::string s = sched < 0 ? "tbb" + : sched == 0 ? "priv" + : std::to_string(sched); + + std::fprintf(stderr, "%-10s %-5d %-8d %-6s %-8ld %-12.0f %-7.1f %-7.1f %-9.1f %-8.2f %-8.2f\n", + topology, size, work_us, s.c_str(), N, + med, iqr, spread, percentile(ovh, 0.5), ivcsw_per_item, vcsw_per_item); + std::printf("%s,%d,%d,%s,%ld,%d,%.0f,%.0f,%.0f,%.2f,%.2f,%.2f,%.3f,%.3f\n", + topology, size, work_us, s.c_str(), N, g_cfg.reps, + med, lo, hi, iqr, spread, percentile(ovh, 0.5), + ivcsw_per_item, vcsw_per_item); + std::fflush(stdout); +} + +// ── argument parsing ────────────────────────────────────────────────────────── + +static std::vector parse_int_list(const char* s) { + std::vector out; + const char* p = s; + while (*p) { + char* end = nullptr; + long v = std::strtol(p, &end, 10); + if (end == p) break; + out.push_back(static_cast(v)); + p = end; + while (*p == ',' || *p == ' ') ++p; + } + return out; +} + +static bool has_word(const std::string& csv, const char* word) { + return csv.find(word) != std::string::npos; +} + +static void usage() { + std::fprintf(stderr, + "usage: bench_pipeline [options]\n" + " --work=10,100,1000 per-node busy-work, microseconds\n" + " --depths=1,2,4,8,16,32 chain depths\n" + " --widths=1,2,3,4 fanout widths\n" + " --pools=1,2,4,8,16,20 shared-pool thread counts\n" + " --topos=chain,wide,diamond\n" + " --modes=priv,pool,tbb\n" + " --reps=5 measured repetitions per row\n" + " --warmup=1 discarded repetitions per row\n" + " --target-sec=0.30 aimed-for duration of one repetition\n" + " --min-items=2000 sample-size floor\n" + " --max-sec=3.0 per-repetition ceiling (overrides the floor)\n"); +} + +static bool parse_args(int argc, char** argv) { + for (int i = 1; i < argc; ++i) { + std::string a = argv[i]; + auto eq = a.find('='); + std::string key = a.substr(0, eq); + std::string val = eq == std::string::npos ? "" : a.substr(eq + 1); + + if (key == "--help" || key == "-h") { usage(); std::exit(0); } + else if (key == "--work") g_cfg.work_amts = parse_int_list(val.c_str()); + else if (key == "--depths") g_cfg.depths = parse_int_list(val.c_str()); + else if (key == "--widths") g_cfg.widths = parse_int_list(val.c_str()); + else if (key == "--pools") g_cfg.pool_sizes = parse_int_list(val.c_str()); + else if (key == "--reps") g_cfg.reps = std::atoi(val.c_str()); + else if (key == "--warmup") g_cfg.warmup = std::atoi(val.c_str()); + else if (key == "--target-sec") g_cfg.target_sec = std::atof(val.c_str()); + else if (key == "--min-items") g_cfg.min_items = std::atol(val.c_str()); + else if (key == "--max-sec") g_cfg.max_sec = std::atof(val.c_str()); + else if (key == "--topos") { + g_cfg.do_chain = has_word(val, "chain"); + g_cfg.do_wide = has_word(val, "wide"); + g_cfg.do_diamond = has_word(val, "diamond"); + } + else if (key == "--modes") { + g_cfg.do_priv = has_word(val, "priv"); + g_cfg.do_pool = has_word(val, "pool"); + g_cfg.do_tbb = has_word(val, "tbb"); + } + else { std::fprintf(stderr, "unknown option: %s\n", a.c_str()); usage(); return false; } + } + if (g_cfg.reps < 1) g_cfg.reps = 1; + if (g_cfg.warmup < 0) g_cfg.warmup = 0; + return true; +} + +// `wide` is templated on W, so dispatch the runtime width through a switch. +template +static void with_width(int w, F&& f) { + switch (w) { + case 1: f(std::integral_constant{}); break; + case 2: f(std::integral_constant{}); break; + case 3: f(std::integral_constant{}); break; + case 4: f(std::integral_constant{}); break; + default: + std::fprintf(stderr, "width %d not instantiated (1..4 only)\n", w); + } +} + // ── main ────────────────────────────────────────────────────────────────────── -int main() { - const int work_amts[] = {10, 100, 1000}; - const int pool_sizes[] = {1, 2, 4}; +int main(int argc, char** argv) { + // A rejected option must fail loudly: a harness driver that silently got + // no CSV back is worse than one that stops. + if (!parse_args(argc, argv)) return 2; - std::fprintf(stderr, "%-12s %-8s %-10s %-8s %-18s %-20s\n", - "topology", "size", "work_us", "threads", "items/sec", "overhead_us/item"); - std::fprintf(stderr, "%s\n", std::string(78, '-').c_str()); - std::printf("topology,size,work_us,threads,items_per_sec,overhead_us_per_item\n"); + char cfg[192]; + std::snprintf(cfg, sizeof cfg, + "reps=%d warmup=%d target_sec=%.2f min_items=%ld max_sec=%.1f", + g_cfg.reps, g_cfg.warmup, g_cfg.target_sec, + g_cfg.min_items, g_cfg.max_sec); + bench::print_environment(cfg); - auto emit = [](const Result& r) { - std::string sched = r.threads < 0 ? "tbb" - : r.threads == 0 ? "priv" - : std::to_string(r.threads); - std::fprintf(stderr, "%-12s %-8d %-10d %-8s %-18.0f %-20.1f\n", - r.topology, r.size, r.work_us, sched.c_str(), - r.items_per_sec, r.overhead_us); - std::printf("%s,%d,%d,%s,%.0f,%.2f\n", - r.topology, r.size, r.work_us, sched.c_str(), - r.items_per_sec, r.overhead_us); - std::fflush(stdout); - }; + std::fprintf(stderr, "\n%-10s %-5s %-8s %-6s %-8s %-12s %-7s %-7s %-9s %-8s %-8s\n", + "topology", "size", "work_us", "sched", "items", "items/sec", + "iqr%", "range%", "ovh_us", "ivcsw/it", "vcsw/it"); + std::fprintf(stderr, "%s\n", std::string(104, '-').c_str()); + std::printf("topology,size,work_us,threads,items,reps,items_per_sec," + "items_per_sec_min,items_per_sec_max,iqr_pct,range_pct," + "overhead_us_per_item,ivcsw_per_item,vcsw_per_item\n"); - for (int w : work_amts) { + for (int w : g_cfg.work_amts) { g_work_us.store(w, std::memory_order_relaxed); - std::fprintf(stderr, "\n── work_us=%-4d private pools ───────────────────────────────────────\n", w); - for (int d : {1, 2, 4, 8, 16, 32}) emit(bench_chain(d, w)); - emit(bench_wide<1>(w)); - emit(bench_wide<2>(w)); - emit(bench_wide<3>(w)); - emit(bench_wide<4>(w)); - emit(bench_diamond(w)); + if (g_cfg.do_priv) { + std::fprintf(stderr, "\n── work_us=%-4d private pools ──────────────────────\n", w); + if (g_cfg.do_chain) + for (int d : g_cfg.depths) { + long N = pick_items(w, d, d); + run_row("chain", d, w, 0, N, [&] { return bench_chain(d, w, N); }); + } + if (g_cfg.do_wide) + for (int wd : g_cfg.widths) + with_width(wd, [&](auto W) { + long N = pick_items(w, W.value, W.value); + run_row("wide", static_cast(W.value), w, 0, N, + [&] { return bench_wide(w, N); }); + }); + if (g_cfg.do_diamond) { + long N = pick_items(w, 4, 4); + run_row("diamond", 4, w, 0, N, [&] { return bench_diamond(w, N); }); + } + } - for (int pt : pool_sizes) { - std::fprintf(stderr, "\n── work_us=%-4d shared pool (%d thread%s) ─────────────────────────────\n", - w, pt, pt == 1 ? "" : "s"); - for (int d : {1, 2, 4, 8, 16, 32}) emit(bench_chain_pool(d, w, pt)); - emit(bench_wide_pool<1>(w, pt)); - emit(bench_wide_pool<2>(w, pt)); - emit(bench_wide_pool<3>(w, pt)); - emit(bench_wide_pool<4>(w, pt)); - emit(bench_diamond_pool(w, pt)); + if (g_cfg.do_pool) { + for (int pt : g_cfg.pool_sizes) { + std::fprintf(stderr, "\n── work_us=%-4d shared pool (%d thread%s) ───────────\n", + w, pt, pt == 1 ? "" : "s"); + if (g_cfg.do_chain) + for (int d : g_cfg.depths) { + long N = pick_items(w, d, pt); + run_row("chain", d, w, pt, N, + [&] { return bench_chain_pool(d, w, pt, N); }); + } + if (g_cfg.do_wide) + for (int wd : g_cfg.widths) + with_width(wd, [&](auto W) { + long N = pick_items(w, W.value, pt); + run_row("wide", static_cast(W.value), w, pt, N, + [&] { return bench_wide_pool(w, pt, N); }); + }); + if (g_cfg.do_diamond) { + long N = pick_items(w, 4, pt); + run_row("diamond", 4, w, pt, N, + [&] { return bench_diamond_pool(w, pt, N); }); + } + } } #ifdef KPN_BENCH_TBB - std::fprintf(stderr, "\n── work_us=%-4d TBB flow graph ──────────────────────────────────────\n", w); - for (int d : {1, 2, 4, 8, 16, 32}) emit(bench_chain_tbb(d, w)); - emit(bench_wide_tbb<1>(w)); - emit(bench_wide_tbb<2>(w)); - emit(bench_wide_tbb<3>(w)); - emit(bench_wide_tbb<4>(w)); - emit(bench_diamond_tbb(w)); + if (g_cfg.do_tbb) { + std::fprintf(stderr, "\n── work_us=%-4d TBB flow graph ─────────────────────\n", w); + if (g_cfg.do_chain) + for (int d : g_cfg.depths) { + long N = pick_items(w, d, d); + run_row("chain_tbb", d, w, -1, N, + [&] { return bench_chain_tbb(d, w, N); }); + } + if (g_cfg.do_wide) + for (int wd : g_cfg.widths) + with_width(wd, [&](auto W) { + long N = pick_items(w, W.value, W.value); + run_row("wide_tbb", static_cast(W.value), w, -1, N, + [&] { return bench_wide_tbb(w, N); }); + }); + if (g_cfg.do_diamond) { + long N = pick_items(w, 4, 4); + run_row("diamond_tbb", 4, w, -1, N, + [&] { return bench_diamond_tbb(w, N); }); + } + } #endif } } diff --git a/scripts/bench_repro_check.py b/scripts/bench_repro_check.py new file mode 100755 index 0000000..8c21d0b --- /dev/null +++ b/scripts/bench_repro_check.py @@ -0,0 +1,122 @@ +#!/usr/bin/env python3 +"""Check the PERF_PLAN Phase-0 acceptance criterion. + +Runs bench_pipeline several times and reports, per row, how far the passes +spread around their median. The plan's gate is: the same configuration run 7x +lands within +/-5% on every row. Until that holds, no measured difference +between KPN and TBB is worth acting on. + +Exits non-zero if any row exceeds the tolerance, so it can gate a session of +performance work rather than merely inform one. + +Usage: + scripts/bench_repro_check.py ./build_bench/benchmarks/bench_pipeline \\ + --passes 7 --tolerance 5 -- --work=10 --topos=chain,wide --reps=5 +""" + +import argparse +import statistics +import subprocess +import sys + +KEY_COLS = ("topology", "size", "work_us", "threads") + +# bench_pipeline reports throughput, bench_dispatch reports per-dispatch cost. +# Either is a valid thing to demand reproducibility of; deviation from the +# median is symmetric, so it does not matter which direction is "better". +METRIC_COLS = ("items_per_sec", "ns_per_dispatch") + + +def parse_csv(text, metric=None): + """Return ({(topology, size, work_us, threads): value}, metric_name).""" + rows = {} + header = None + for line in text.splitlines(): + line = line.strip() + if not line or line.startswith("#"): + continue + fields = line.split(",") + if header is None: + if fields[0] != "topology": + continue + header = fields + if metric is None: + for cand in METRIC_COLS: + if cand in header: + metric = cand + break + else: + sys.exit(f"no metric column found in header: {header}") + elif metric not in header: + sys.exit(f"metric {metric!r} not in header: {header}") + continue + rec = dict(zip(header, fields)) + try: + key = tuple(rec[c] for c in KEY_COLS) + rows[key] = float(rec[metric]) + except (KeyError, ValueError): + continue + return rows, metric + + +def main(): + ap = argparse.ArgumentParser() + ap.add_argument("binary", help="path to bench_pipeline") + ap.add_argument("--passes", type=int, default=7) + ap.add_argument("--tolerance", type=float, default=5.0, + help="max allowed deviation from the median, percent") + ap.add_argument("--metric", default=None, choices=METRIC_COLS, + help="column to check (default: whichever the CSV carries)") + + # Everything after a standalone `--` goes to bench_pipeline verbatim. + # argparse.REMAINDER would swallow this script's own flags instead. + argv = sys.argv[1:] + extra = [] + if "--" in argv: + cut = argv.index("--") + argv, extra = argv[:cut], argv[cut + 1:] + args = ap.parse_args(argv) + + passes = [] + metric = args.metric + for i in range(args.passes): + print(f"pass {i + 1}/{args.passes} ...", file=sys.stderr, flush=True) + proc = subprocess.run([args.binary] + extra, + capture_output=True, text=True) + if proc.returncode != 0: + print(proc.stderr, file=sys.stderr) + sys.exit(f"{args.binary} failed with {proc.returncode}") + rows, metric = parse_csv(proc.stdout, metric) + passes.append(rows) + + keys = set(passes[0]) + for p in passes[1:]: + keys &= set(p) + if not keys: + sys.exit("no rows common to every pass") + + print(f"\n{'row':<34} {'median ' + metric:>20} {'worst dev':>10} verdict") + print("-" * 72) + + failures = 0 + for key in sorted(keys): + values = [p[key] for p in passes] + med = statistics.median(values) + worst = max(abs(v - med) / med * 100 for v in values) if med else 0.0 + ok = worst <= args.tolerance + failures += not ok + label = "{}-{} w={} s={}".format(*key) + print(f"{label:<34} {med:>20.1f} {worst:>9.1f}% {'ok' if ok else 'NOISY'}") + + print("-" * 72) + if failures: + print(f"{failures}/{len(keys)} rows exceed +/-{args.tolerance:g}% — " + f"the Phase-0 gate is not met.") + return 1 + print(f"all {len(keys)} rows within +/-{args.tolerance:g}% " + f"over {args.passes} passes — Phase-0 gate met.") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index f567afc..07bf131 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -52,12 +52,31 @@ target_link_libraries(kpn_tests PRIVATE add_executable(kpn_tests_stress test_channel_stress.cpp) target_link_libraries(kpn_tests_stress PRIVATE kpn Catch2::Catch2WithMain) +# ── Wedge soak (PERF_PLAN G1) ───────────────────────────────────────────────── +# Long-running end-to-end loop over the configurations that historically wedged. +# Always built, so it cannot rot, but its CTest cases are registered only under +# -DKPN_ENABLE_SOAK_TESTS=ON: they run for minutes and would otherwise dominate +# every `ctest` invocation. Performance work runs it before and after a change: +# +# cmake -B build -DKPN_ENABLE_SOAK_TESTS=ON -DKPN_SOAK_ITERS=50000 +# cmake --build build --target kpn_soak_wedge +# ctest --test-dir build -L soak +# +# The binary self-diagnoses: an iteration that stops making progress trips a +# watchdog that aborts naming the iteration and phase, rather than hanging. +add_executable(kpn_soak_wedge soak_wedge.cpp) +target_link_libraries(kpn_soak_wedge PRIVATE kpn) +target_compile_options(kpn_soak_wedge PRIVATE -O2) + +option(KPN_ENABLE_SOAK_TESTS "Register the wedge soak cases with CTest" OFF) +set(KPN_SOAK_ITERS 5000 CACHE STRING "Iterations per wedge soak case") + # ── Sanitizer flags ─────────────────────────────────────────────────────────── # kpn_sanitizer_flags() is defined in the top-level CMakeLists and is a no-op # unless -DKPN_SANITIZER=... is set. Sanitizer must be on both compile and link. kpn_sanitizer_flags(_kpn_san) if(_kpn_san) - foreach(_t kpn_tests kpn_tests_stress) + foreach(_t kpn_tests kpn_tests_stress kpn_soak_wedge) target_compile_options(${_t} PRIVATE ${_kpn_san}) target_link_options(${_t} PRIVATE ${_kpn_san}) endforeach() @@ -77,3 +96,16 @@ catch_discover_tests(kpn_tests DISCOVERY_MODE PRE_TEST) # Register the stress suite under its own label so CI can run / time it # separately from the fast unit tests. catch_discover_tests(kpn_tests_stress DISCOVERY_MODE PRE_TEST PROPERTIES LABELS "stress") + +if(KPN_ENABLE_SOAK_TESTS) + # pool: the configuration the August wedges were reproduced on. + add_test(NAME soak.wedge.pool + COMMAND kpn_soak_wedge --mode=pool --depth=4 --threads=4 + --items=1000 --work-us=10 --iters=${KPN_SOAK_ITERS}) + # private: one pool per node — the model workstream A would change. + add_test(NAME soak.wedge.private + COMMAND kpn_soak_wedge --mode=priv --depth=8 + --items=1000 --work-us=10 --iters=${KPN_SOAK_ITERS}) + set_tests_properties(soak.wedge.pool soak.wedge.private PROPERTIES + LABELS "soak" TIMEOUT 3600) +endif() diff --git a/tests/soak_wedge.cpp b/tests/soak_wedge.cpp new file mode 100644 index 0000000..033402e --- /dev/null +++ b/tests/soak_wedge.cpp @@ -0,0 +1,236 @@ +// Wedge soak test (PERF_PLAN G1). +// +// Runs a pipeline configuration end-to-end in a loop and fails if any single +// iteration stops making progress. Its purpose is to keep performance work +// from silently reintroducing one of the wedges fixed in August 2026 — the +// scheduler and channel wake paths are where both perf workstreams operate. +// +// Originally the minimal reproducer for the shared-pool chain wedge at +// (chain, depth=4, work_us=10, pool_threads=4); the pre-6802328 code wedged +// 5/5 within 45 s, at iterations 149, 1249, 332, 1740 and 493. +// +// A wedge is a hang, so a plain loop would hang CTest until its timeout with +// no indication of where. The watchdog turns that into a failure naming the +// iteration and the phase it stalled in. +// +// Usage: ./kpn_soak_wedge [options] +// --iters=5000 iterations to run +// --mode=pool|priv shared ThreadPool(--threads), or one private pool/node +// --depth=4 chain depth +// --threads=4 shared pool size (--mode=pool only) +// --items=1000 items pushed per iteration +// --work-us=10 busy-work per node +// --watchdog-sec=30 per-iteration progress deadline + +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#if defined(__linux__) +#include +#endif + +using namespace kpn; +using sclock = std::chrono::steady_clock; + +static std::atomic g_work_us{10}; + +static int chain_fn(int x) { + int us = g_work_us.load(std::memory_order_relaxed); + if (us > 0) { + auto end = sclock::now() + std::chrono::microseconds(us); + while (sclock::now() < end); + } + return x; +} + +using ChainNode = Node, out<>>; +using PoolChainNode = PoolNode, out<>>; + +static void push_retry(Channel& ch, int val) { + while (true) { + try { ch.push(val); return; } + catch (const ChannelOverflowError&) { std::this_thread::yield(); } + catch (const ChannelClosedError&) { return; } + } +} + +// ── watchdog ────────────────────────────────────────────────────────────────── +// +// The worker bumps g_progress at every phase boundary. The watchdog aborts if +// it stops moving, so a wedge is reported as a failure at a known iteration +// rather than as an unattributable CTest timeout. + +static std::atomic g_progress{0}; +static std::atomic g_iter{0}; +static std::atomic g_phase{"init"}; +static std::atomic g_done{false}; + +static void mark(const char* phase) { + g_phase.store(phase, std::memory_order_relaxed); + g_progress.fetch_add(1, std::memory_order_release); +} + +static void watchdog(double deadline_sec) { + unsigned long last = g_progress.load(std::memory_order_acquire); + auto last_move = sclock::now(); + while (!g_done.load(std::memory_order_acquire)) { + std::this_thread::sleep_for(std::chrono::milliseconds(100)); + unsigned long now = g_progress.load(std::memory_order_acquire); + if (now != last) { last = now; last_move = sclock::now(); continue; } + double stalled = std::chrono::duration(sclock::now() - last_move).count(); + if (stalled > deadline_sec) { + std::fprintf(stderr, + "\nWEDGE: no progress for %.0fs at iteration %d, phase '%s'\n", + stalled, g_iter.load(std::memory_order_relaxed), + g_phase.load(std::memory_order_relaxed)); + std::fflush(stderr); + std::abort(); // core dump / stack trace at the point of the wedge + } + } +} + +// ── one iteration ───────────────────────────────────────────────────────────── + +struct Opts { + int iters = 5000; + int depth = 4; + int threads = 4; + int items = 1000; + int work_us = 10; + bool shared_pool = true; + double watchdog_sec = 30.0; +}; + +static void one_round_pool(const Opts& o) { + const std::size_t CAP = static_cast(o.items); + auto pool = std::make_shared(o.threads); + + std::vector>> chs; + for (int i = 0; i <= o.depth; ++i) + chs.push_back(std::make_shared>(CAP)); + + std::vector> nodes; + for (int i = 0; i < o.depth; ++i) { + nodes.push_back(std::make_unique(pool, CAP)); + nodes.back()->set_input_channel<0>(chs[i]); + nodes.back()->set_output_channel<0>(chs[i + 1].get()); + } + + pool->start(); + for (auto& n : nodes) n->start(); + mark("started"); + + std::thread reader([&] { + for (int i = 0; i < o.items; ++i) chs.back()->pop(); + }); + std::thread pusher([&] { + for (int i = 0; i < o.items; ++i) push_retry(*chs[0], i); + }); + + pusher.join(); mark("pushed"); + reader.join(); mark("drained"); + for (auto& n : nodes) n->stop(); + mark("nodes stopped"); + pool->stop(); + mark("pool stopped"); +} + +static void one_round_private(const Opts& o) { + const std::size_t CAP = static_cast(o.items); + + std::vector>> chs; + for (int i = 0; i <= o.depth; ++i) + chs.push_back(std::make_shared>(CAP)); + + std::vector> nodes; + for (int i = 0; i < o.depth; ++i) { + nodes.push_back(std::make_unique(CAP)); + nodes.back()->set_input_channel<0>(chs[i]); + nodes.back()->set_output_channel<0>(chs[i + 1].get()); + } + + for (auto& n : nodes) n->start(); + mark("started"); + + std::thread reader([&] { + for (int i = 0; i < o.items; ++i) chs.back()->pop(); + }); + std::thread pusher([&] { + for (int i = 0; i < o.items; ++i) push_retry(*chs[0], i); + }); + + pusher.join(); mark("pushed"); + reader.join(); mark("drained"); + for (auto& n : nodes) n->stop(); + mark("nodes stopped"); +} + +// ── main ────────────────────────────────────────────────────────────────────── + +static void usage() { + std::fprintf(stderr, + "usage: kpn_soak_wedge [--iters=N] [--mode=pool|priv] [--depth=D]\n" + " [--threads=T] [--items=N] [--work-us=U]\n" + " [--watchdog-sec=S]\n"); +} + +int main(int argc, char** argv) { +#if defined(__linux__) + // Allow gdb to attach under ptrace_scope=1 when a wedge is caught. + prctl(PR_SET_PTRACER, PR_SET_PTRACER_ANY, 0, 0, 0); +#endif + + Opts o; + for (int i = 1; i < argc; ++i) { + std::string a = argv[i]; + auto eq = a.find('='); + std::string key = a.substr(0, eq); + std::string val = eq == std::string::npos ? "" : a.substr(eq + 1); + + if (key == "--iters") o.iters = std::atoi(val.c_str()); + else if (key == "--depth") o.depth = std::atoi(val.c_str()); + else if (key == "--threads") o.threads = std::atoi(val.c_str()); + else if (key == "--items") o.items = std::atoi(val.c_str()); + else if (key == "--work-us") o.work_us = std::atoi(val.c_str()); + else if (key == "--watchdog-sec") o.watchdog_sec = std::atof(val.c_str()); + else if (key == "--mode") o.shared_pool = (val != "priv"); + else { usage(); return 2; } + } + g_work_us.store(o.work_us, std::memory_order_relaxed); + + std::fprintf(stderr, + "soak: mode=%s depth=%d threads=%d items=%d work_us=%d iters=%d watchdog=%.0fs\n", + o.shared_pool ? "pool" : "priv", o.depth, + o.shared_pool ? o.threads : o.depth, o.items, o.work_us, + o.iters, o.watchdog_sec); + + std::thread wd(watchdog, o.watchdog_sec); + + const auto t0 = sclock::now(); + for (int i = 0; i < o.iters; ++i) { + g_iter.store(i, std::memory_order_relaxed); + if (o.shared_pool) one_round_pool(o); + else one_round_private(o); + if ((i + 1) % 100 == 0) { + std::fprintf(stderr, "\r %d/%d", i + 1, o.iters); + std::fflush(stderr); + } + } + + g_done.store(true, std::memory_order_release); + wd.join(); + + double secs = std::chrono::duration(sclock::now() - t0).count(); + std::fprintf(stderr, "\ncompleted %d iterations in %.1fs with no wedge\n", + o.iters, secs); + return 0; +} From 73828bcffe493dbb56a8b0be1f56972f45acc684 Mon Sep 17 00:00:00 2001 From: Duncan Tourolle Date: Sat, 8 Aug 2026 22:25:00 +0200 Subject: [PATCH 41/42] perf(bench): make a multi-hour acceptance run observable and restartable The Phase-0 gate is 7 passes over the full row set, which is long enough that capture_output=True was the wrong default: no rows existed anywhere until a pass ended, so a slow run and a wedged one looked identical, and killing either threw away everything measured. Rows now stream to --out-dir/pass-NN.csv with a flush per line, so an interrupted run keeps what it had. --resume reuses complete pass files and reruns incomplete ones -- checked against the row count rather than merely existing, because a partial file silently averaged in as a whole pass would corrupt the verdict rather than fail it. Progress is a tqdm bar over rows, not passes; a pass counter would sit at 1/7 for twenty minutes and report nothing useful. The row total comes from probing the binary with --reps=0 (0.8s) rather than reimplementing the sweep in Python, which would drift from the C++ defaults. There is a plain-stderr fallback when tqdm is absent -- refusing to start a benchmark over a missing progress dependency is the wrong trade. bench_runs/ is gitignored: the new default output path would otherwise land in git status. Co-Authored-By: Claude Opus 5 --- .gitignore | 3 + scripts/bench_repro_check.py | 202 +++++++++++++++++++++++++++++------ 2 files changed, 172 insertions(+), 33 deletions(-) diff --git a/.gitignore b/.gitignore index 2d1b7e4..3d699c5 100644 --- a/.gitignore +++ b/.gitignore @@ -25,6 +25,9 @@ venv/ .DS_Store Thumbs.db +# Benchmark output (scripts/bench_repro_check.py --out-dir) +bench_runs/ + # Claude Code local settings .claude/settings.local.json include/kpn/ort_cache/ diff --git a/scripts/bench_repro_check.py b/scripts/bench_repro_check.py index 8c21d0b..6e6413e 100755 --- a/scripts/bench_repro_check.py +++ b/scripts/bench_repro_check.py @@ -9,12 +9,20 @@ between KPN and TBB is worth acting on. Exits non-zero if any row exceeds the tolerance, so it can gate a session of performance work rather than merely inform one. +A full sweep is hours, so the run is observable and restartable rather than +opaque: rows stream to --out-dir as each pass produces them, and a progress +bar tracks rows within the pass. Killing the run keeps everything already +written; --resume picks up from the completed passes on disk. + Usage: scripts/bench_repro_check.py ./build_bench/benchmarks/bench_pipeline \\ --passes 7 --tolerance 5 -- --work=10 --topos=chain,wide --reps=5 """ import argparse +import datetime +import os +import pathlib import statistics import subprocess import sys @@ -27,11 +35,47 @@ KEY_COLS = ("topology", "size", "work_us", "threads") METRIC_COLS = ("items_per_sec", "ns_per_dispatch") -def parse_csv(text, metric=None): +def _progress(total, desc): + """A tqdm bar if tqdm is installed, else a minimal stderr fallback. + + The fallback exists because this script gates a benchmark run; refusing to + start over a missing progress dependency would be the wrong trade. + """ + try: + from tqdm import tqdm + except ImportError: + class Fallback: + def __init__(self): + self.n = 0 + + def update(self, k=1): + self.n += k + end = "\n" if (total and self.n >= total) else "\r" + print(f" {desc}: {self.n}/{total or '?'} rows", + file=sys.stderr, end=end, flush=True) + + def close(self): + pass + + def __enter__(self): + return self + + def __exit__(self, *exc): + self.close() + + return Fallback() + + return tqdm(total=total, desc=desc, unit="row", leave=False, + bar_format=" {desc}: {n_fmt}/{total_fmt} rows " + "|{bar}| {elapsed}<{remaining}", + file=sys.stderr) + + +def parse_csv_lines(lines, metric=None): """Return ({(topology, size, work_us, threads): value}, metric_name).""" rows = {} header = None - for line in text.splitlines(): + for line in lines: line = line.strip() if not line or line.startswith("#"): continue @@ -59,36 +103,64 @@ def parse_csv(text, metric=None): return rows, metric -def main(): - ap = argparse.ArgumentParser() - ap.add_argument("binary", help="path to bench_pipeline") - ap.add_argument("--passes", type=int, default=7) - ap.add_argument("--tolerance", type=float, default=5.0, - help="max allowed deviation from the median, percent") - ap.add_argument("--metric", default=None, choices=METRIC_COLS, - help="column to check (default: whichever the CSV carries)") +def parse_csv(text, metric=None): + return parse_csv_lines(text.splitlines(), metric) - # Everything after a standalone `--` goes to bench_pipeline verbatim. - # argparse.REMAINDER would swallow this script's own flags instead. - argv = sys.argv[1:] - extra = [] - if "--" in argv: - cut = argv.index("--") - argv, extra = argv[:cut], argv[cut + 1:] - args = ap.parse_args(argv) - passes = [] - metric = args.metric - for i in range(args.passes): - print(f"pass {i + 1}/{args.passes} ...", file=sys.stderr, flush=True) - proc = subprocess.run([args.binary] + extra, - capture_output=True, text=True) - if proc.returncode != 0: - print(proc.stderr, file=sys.stderr) - sys.exit(f"{args.binary} failed with {proc.returncode}") - rows, metric = parse_csv(proc.stdout, metric) - passes.append(rows) +def count_rows(binary, extra): + """Enumerate the sweep cheaply, so the progress bar has a real total. + Asks the binary itself rather than reimplementing the sweep in Python, + which would silently drift from the C++ defaults. Returns None if the + probe fails -- an unknown total degrades the bar, it does not stop the run. + """ + probe = [binary] + extra + ["--reps=0", "--warmup=0"] + try: + proc = subprocess.run(probe, capture_output=True, text=True, + timeout=600) + except (subprocess.SubprocessError, OSError): + return None + if proc.returncode != 0: + return None + rows, _ = parse_csv(proc.stdout) + return len(rows) or None + + +def run_pass(binary, extra, total, desc, sink, metric): + """Run one pass, streaming rows to `sink` and the bar as they arrive. + + capture_output would withhold every row until the pass ended, which for a + multi-hour sweep means no way to tell a slow run from a wedged one. + """ + lines = [] + bar = _progress(total, desc) + proc = subprocess.Popen([binary] + extra, stdout=subprocess.PIPE, + stderr=subprocess.PIPE, text=True, bufsize=1) + try: + for line in proc.stdout: + lines.append(line) + if sink: + sink.write(line) + sink.flush() # a killed run keeps its rows + stripped = line.strip() + if (stripped and not stripped.startswith("#") + and "," in stripped + and not stripped.startswith("topology,")): + bar.update(1) + finally: + bar.close() + proc.stdout.close() + stderr = proc.stderr.read() + proc.stderr.close() + rc = proc.wait() + + if rc != 0: + print(stderr, file=sys.stderr) + sys.exit(f"{binary} failed with {rc}") + return parse_csv_lines(lines, metric) + + +def report(passes, metric, tolerance, npasses): keys = set(passes[0]) for p in passes[1:]: keys &= set(p) @@ -103,20 +175,84 @@ def main(): values = [p[key] for p in passes] med = statistics.median(values) worst = max(abs(v - med) / med * 100 for v in values) if med else 0.0 - ok = worst <= args.tolerance + ok = worst <= tolerance failures += not ok label = "{}-{} w={} s={}".format(*key) print(f"{label:<34} {med:>20.1f} {worst:>9.1f}% {'ok' if ok else 'NOISY'}") print("-" * 72) if failures: - print(f"{failures}/{len(keys)} rows exceed +/-{args.tolerance:g}% — " + print(f"{failures}/{len(keys)} rows exceed +/-{tolerance:g}% — " f"the Phase-0 gate is not met.") return 1 - print(f"all {len(keys)} rows within +/-{args.tolerance:g}% " - f"over {args.passes} passes — Phase-0 gate met.") + print(f"all {len(keys)} rows within +/-{tolerance:g}% " + f"over {npasses} passes — Phase-0 gate met.") return 0 +def main(): + ap = argparse.ArgumentParser() + ap.add_argument("binary", help="path to bench_pipeline") + ap.add_argument("--passes", type=int, default=7) + ap.add_argument("--tolerance", type=float, default=5.0, + help="max allowed deviation from the median, percent") + ap.add_argument("--metric", default=None, choices=METRIC_COLS, + help="column to check (default: whichever the CSV carries)") + ap.add_argument("--out-dir", default=None, + help="write pass-NN.csv as rows arrive " + "(default: bench_runs/)") + ap.add_argument("--resume", action="store_true", + help="reuse complete pass-NN.csv files in --out-dir") + + # Everything after a standalone `--` goes to bench_pipeline verbatim. + # argparse.REMAINDER would swallow this script's own flags instead. + argv = sys.argv[1:] + extra = [] + if "--" in argv: + cut = argv.index("--") + argv, extra = argv[:cut], argv[cut + 1:] + args = ap.parse_args(argv) + + out_dir = args.out_dir + if out_dir is None: + if args.resume: + sys.exit("--resume needs an explicit --out-dir") + stamp = datetime.datetime.now().strftime("%Y%m%d-%H%M%S") + out_dir = os.path.join("bench_runs", stamp) + out = pathlib.Path(out_dir) + out.mkdir(parents=True, exist_ok=True) + + total = count_rows(args.binary, extra) + print(f"writing to {out}/", file=sys.stderr) + if total: + print(f"{total} rows per pass, {args.passes} passes", file=sys.stderr) + + passes = [] + metric = args.metric + for i in range(args.passes): + path = out / f"pass-{i + 1:02d}.csv" + + if args.resume and path.exists(): + rows, metric = parse_csv(path.read_text(), metric) + # A partial file from a killed run must not be silently averaged + # in as if it were a whole pass. + if total and len(rows) < total: + print(f"pass {i + 1}/{args.passes}: {path.name} has " + f"{len(rows)}/{total} rows — rerunning", file=sys.stderr) + else: + print(f"pass {i + 1}/{args.passes}: reusing {path.name} " + f"({len(rows)} rows)", file=sys.stderr) + passes.append(rows) + continue + + print(f"pass {i + 1}/{args.passes} ...", file=sys.stderr, flush=True) + with open(path, "w") as sink: + rows, metric = run_pass(args.binary, extra, total, + f"pass {i + 1}/{args.passes}", sink, metric) + passes.append(rows) + + return report(passes, metric, args.tolerance, args.passes) + + if __name__ == "__main__": sys.exit(main()) From b500570c479d750a7385daab44ec7f7182472036 Mon Sep 17 00:00:00 2001 From: Duncan Tourolle Date: Sat, 8 Aug 2026 22:26:14 +0200 Subject: [PATCH 42/42] perf(pool): submit to the calling worker's own queue, and skip a notify nobody is waiting for B9 from PERF_PLAN, plus the notify gate. Two changes to submit(), both aimed at the same cost: on any pool of two or more threads, every single dispatch paid a futex wake. submit() round-robins, so a worker resubmitting -- which is what fire_once does on every token -- handed the task to a *different* worker, and that worker was asleep. bench_dispatch measured it as 182 ns/dispatch on ThreadPool(1) against 3197 ns on 20 threads, with voluntary context switches per task rising 0.00 -> 1.19 in step: the 1-thread pool is fast precisely because it resubmits into its own queue and finds the work already there. A submission originating on one of the pool's own workers now goes to that worker's queue, extending that property to any pool size. try_steal still corrects the imbalance. The identity check is against `this`, not merely "am I a pool worker": a worker of pool A submitting into pool B must not use A's index, which may exceed B's thread_count_. Nested networks do exactly this. tls_pool is a non-owning identity tag, only ever compared, never dereferenced -- its lifetime is strictly nested inside the pool's, since stop() joins every worker before clearing queues_. The notify gate skips the cv_mx_ round-trip and notify_one() when waiters_ is zero. waiters_ is maintained under cv_mx_ and incremented before the predicate is evaluated, so reading zero in submit() means no worker can be in wait() -- as opposed to reading zero because we raced one, which the mutex prevents. stop()'s notify_all() is deliberately left ungated. Shared pools, work_us=10, items/sec: chain-1 59512 -> 66662 (+12.0%), wide-4 53698 -> 61705 (+14.9%), chain-8 +6.8%, chain-32 +3.7%. Private pools -- the Node<> default -- are unchanged at -1.3% to +3.6%, inside the gate's tolerance. Two things tried and removed, recorded in comments so they are not retried: Raising the steal threshold to >1, to stop a thief winning the race for a self-submitted task, DEADLOCKS. An external submit() round-robins a single task onto an idle worker's queue; if that worker is parked, no peer will take it, because a queue of one is no longer stealable. latency mode hangs at 12 and 20 threads. It was also 2x slower in steady state, 2229 -> 4546 ns. B5, bounded spin before parking, does not pay: swept at 50/200/1000 rounds, 2123 / 2230 / 2574 ns against 2229 ns without it, with vcsw/task flat at ~0.97. The spin cannot catch what it targets, because a peer is woken the moment queued_ becomes non-zero -- before this worker reaches the spin at all. Guardrails: 150/150 ctest including soak and examples, and ThreadSanitizer clean over the full suite (128 cases, 302 assertions). That also pins the reference count PERF_PLAN section 6 flagged as uncertain: it is 150, not 146 or 136. Caveat: the throughput figures above were taken on a build that also carried the since-removed spin experiment. The scheduler logic is identical to this tree and correctness was re-verified on it, but the numbers are one build stale and predate the 7-pass gate, which has not been run on this change. Co-Authored-By: Claude Opus 5 --- include/kpn/scheduler.hpp | 89 +++++++++++++++++++++++++++++++++++++-- 1 file changed, 86 insertions(+), 3 deletions(-) diff --git a/include/kpn/scheduler.hpp b/include/kpn/scheduler.hpp index d11d775..c33d5a2 100644 --- a/include/kpn/scheduler.hpp +++ b/include/kpn/scheduler.hpp @@ -128,7 +128,28 @@ public: rejected_.fetch_add(1, std::memory_order_relaxed); return; } - std::size_t target = next_.fetch_add(1, std::memory_order_relaxed) % thread_count_; + // B9 — submit-to-self affinity. Round-robin hands every task to a + // *different* worker, and on a pool of two or more that worker is + // asleep, so each dispatch pays a futex wake: measured 182 ns/dispatch + // on a 1-thread pool against 3197 ns on 20 threads, with voluntary + // context switches per task rising 0.00 -> 1.19 in step. + // + // A submission originating on one of *our own* workers goes to that + // worker's queue instead. It is about to return to worker_loop and + // try_pop its own queue, so the work is already there and nothing + // sleeps — the property that makes ThreadPool(1) fast, extended to + // any pool size. Imbalance is corrected by the existing try_steal. + // + // The pool identity check is load-bearing: a worker of pool A + // submitting into pool B must not use A's index, which may exceed B's + // thread_count_ or alias an unrelated queue. Nested networks do + // exactly this. + std::size_t target; + if (tls_pool == this && tls_worker < thread_count_) { + target = tls_worker; + } else { + target = next_.fetch_add(1, std::memory_order_relaxed) % thread_count_; + } { std::lock_guard lock(queues_[target]->mx); queues_[target]->pq.push( @@ -142,8 +163,23 @@ public: // will observe total_ > 0) or already blocked in wait() (and will be // woken). Without this, notify_one() can slip into the gap between the // worker's predicate check and its wait(), and be lost — a deadlock. - { std::lock_guard lk(cv_mx_); } - cv_.notify_one(); + // + // Skipped entirely when no worker is parked. waiters_ is incremented + // *before* wait() releases cv_mx_ and decremented after it returns, + // both under that mutex, so a worker on its way to sleep is already + // counted here. Reading zero therefore means no worker can be in + // wait(), and there is nothing a notify could reach — as opposed to + // reading zero because we raced one, which the mutex prevents. + // + // This is the hot path for an already-busy pool: with B9 the work is + // in the local queue and the submitting worker will find it itself, + // so the lock round-trip and notify were pure overhead. Measured 1.00 + // voluntary context switches per task before this, on a pool where + // only one task is ever in flight. + if (waiters_.load(std::memory_order_seq_cst) != 0) { + { std::lock_guard lk(cv_mx_); } + cv_.notify_one(); + } } std::size_t thread_count() const { return thread_count_; } @@ -193,6 +229,17 @@ private: std::optional> try_steal(std::size_t thief) { // Find the most-loaded peer without blocking — racy peek is fine. + // + // The threshold is >0: a peer holding a single task is a valid victim. + // + // Raising it to >1 — to stop a thief winning the race for a task its + // owner just submitted to itself (B9) — deadlocks. `latency` mode + // hangs at 12 and 20 threads: an external submit() round-robins one + // task onto an idle worker's queue, and if that worker is parked, no + // peer will take it because a queue of one is no longer stealable. + // Nothing else is coming to wake it, so the pool sits forever. + // Measured before reverting: it also made steady-state *worse*, + // 2229 -> 4546 ns at 12 threads. std::size_t victim = thief, best = 0; for (std::size_t i = 0; i < queues_.size(); ++i) { if (i == thief) continue; @@ -221,15 +268,41 @@ private: } void worker_loop(std::size_t id) { + // Identify this thread as one of our workers, for B9's affinity check + // in submit(). Restored on exit rather than merely cleared: a pool + // whose worker runs a task that itself starts and stops a nested pool + // would otherwise come back with its identity erased. + ThreadPool* const prev_pool = tls_pool; + const std::size_t prev_worker = tls_worker; + tls_pool = this; + tls_worker = id; + struct Restore { + ThreadPool* p; std::size_t w; + ~Restore() { tls_pool = p; tls_worker = w; } + } restore{prev_pool, prev_worker}; + while (true) { if (auto fn = try_pop(*queues_[id])) { execute(*fn); continue; } if (auto fn = try_steal(id)) { execute(*fn); continue; } + // B5 (bounded spin before parking) was tried here and removed: it + // does not pay. Swept at 50/200/1000 rounds on a 12-thread pool, + // steady state went 2123 / 2230 / 2574 ns against 2229 ns without + // it, and voluntary context switches per task stayed at ~0.97 + // throughout. The spin cannot catch what it is aimed at, because + // a peer is woken the moment queued_ becomes non-zero — which + // happens before this worker reaches the spin at all. std::unique_lock lock(cv_mx_); + // Counted under cv_mx_ and before the predicate is evaluated, so + // that a submit() which reads waiters_ == 0 can be certain this + // worker is not about to block: to get here we already hold the + // mutex that submit() must take to notify. + waiters_.fetch_add(1, std::memory_order_seq_cst); cv_.wait(lock, [this] { return stopped_.load(std::memory_order_seq_cst) || queued_.load(std::memory_order_relaxed) > 0; }); + waiters_.fetch_sub(1, std::memory_order_seq_cst); // Exit on queued_, not total_: waiting for total_ to reach zero // meant waiting for someone else's task to finish, which this // worker cannot help with and would spin through until it did. @@ -239,6 +312,13 @@ private: } } + /// Which pool, and which of its workers, the calling thread is — or + /// nullptr on any thread that is not a pool worker. Read by submit() to + /// decide whether a local push is safe (B9). inline so the header stays + /// header-only. + static inline thread_local ThreadPool* tls_pool = nullptr; + static inline thread_local std::size_t tls_worker = 0; + const std::size_t thread_count_; std::vector> queues_; std::vector workers_; @@ -267,6 +347,9 @@ private: std::atomic queued_{0}; // waiting to run std::atomic active_{0}; // executing only (for snapshot) std::atomic next_{0}; // round-robin submit cursor + /// Workers currently inside cv_.wait(), maintained under cv_mx_. Lets + /// submit() skip the lock round-trip and notify when nobody is parked. + std::atomic waiters_{0}; std::atomic seq_{0}; // tie-break for equal-priority tasks std::atomic submitted_{0}; /// Submissions refused because the pool was already stopped. Not an error —