// Regression: a blocking push must not park a pool worker. // // Node outputs use push_blocking so a full channel costs time rather than data // (a dropped frame does not degrade a downstream result, it silently changes // one). But push_blocking sleeps *inside* fire_once, which runs on a pool // worker — and nodes are pinned to workers by index. Park enough workers in // that retry loop and there is nobody left to run the consumer that would drain // the channel, so the whole chain wedges. // // This is the failure channel.hpp:174 already warns about for sentinels // ("a blocking push would park that thread and stop it draining its own input, // cascading into a hold-and-wait deadlock under backpressure"). The warning // applies to data pushes too. // // Observed in the field as an intermittent hang: frame_source, camera_pos, // face_detector and face_aligner all asleep in push_blocking at once. #include #include #include #include #include namespace { struct Produce { static constexpr std::string_view label() { return "produce"; } int n{0}; int operator()() { return n++; } }; struct Relay { static constexpr std::string_view label() { return "relay"; } int operator()(int v) { return v; } }; // Deliberately slower than the producer, so the channels between them fill. struct SlowSink { static constexpr std::string_view label() { return "slow_sink"; } std::atomic* seen; void operator()(int) { std::this_thread::sleep_for(std::chrono::milliseconds(2)); seen->fetch_add(1, std::memory_order_relaxed); } }; } // namespace TEST_CASE("a saturated chain keeps draining", "[backpressure][deadlock]") { std::atomic seen{0}; Produce p_fn; Relay r1_fn, r2_fn, r3_fn; SlowSink s_fn{&seen}; // Small channels so they saturate immediately, and a chain longer than a // modest pool — the shape that starves workers. kpn::ObjectNode, kpn::out<"a">, "produce", 0> p (p_fn, 2); kpn::ObjectNode, kpn::out<"b">, "relay1", 0> r1(r1_fn, 2); kpn::ObjectNode, kpn::out<"c">, "relay2", 0> r2(r2_fn, 2); kpn::ObjectNode, kpn::out<"d">, "relay3", 0> r3(r3_fn, 2); kpn::ObjectNode, kpn::out<>, "slow_sink", 0> s (s_fn, 2); auto net = kpn::make_network( kpn::edge(p.output<"a">(), r1.input<"a">()), kpn::edge(r1.output<"b">(), r2.input<"b">()), kpn::edge(r2.output<"c">(), r3.input<"c">()), kpn::edge(r3.output<"d">(), s.input<"d">()) ); net.start(); // The sink is the slowest stage at 2 ms/item, so 40 items is ~80 ms of real // work. Anything approaching the timeout means the chain stopped draining // rather than merely running slowly. const auto deadline = std::chrono::steady_clock::now() + std::chrono::seconds(20); while (seen.load(std::memory_order_relaxed) < 40 && std::chrono::steady_clock::now() < deadline) std::this_thread::sleep_for(std::chrono::milliseconds(5)); const int got = seen.load(std::memory_order_relaxed); net.stop(); INFO("items drained: " << got << " of 40"); CHECK(got >= 40); } // Regression: a saturated chain must never stall. // // push_outputs parks from two places: the retry at the top of fire_once, and // the ordinary push after the node function returns. Both release the worker, // so both face the same lost wakeup — a space callback firing while queued_ is // still up is swallowed by try_submit's CAS. Only the retry path re-checked for // space afterwards. The normal path fell through to on_input_ready(), which // resubmits only if inputs are ready — and the firing that just parked had // consumed its input, so they are not. // // The strand is permanent under saturation: the node holds its value, its // consumer waits for exactly that value, and its producer fills the node's // input channel and parks too. Nothing moves again. // // The test above cannot catch it — 40 items drain before any strand occurs. // This one runs the chain saturated and watches for progress to *freeze*, which // is the signature of the deadlock. It deliberately does not assert a total: // capacity-1 channels are slow, and "slow" must never be reported as "wedged". namespace { struct FreeRun { static constexpr std::string_view label() { return "free_run"; } int n{0}; int operator()() { return n++; } }; struct CountingSink { static constexpr std::string_view label() { return "counting_sink"; } std::atomic* seen; void operator()(int) { seen->fetch_add(1, std::memory_order_relaxed); } }; } // namespace TEST_CASE("a saturated chain never stalls", "[backpressure][deadlock]") { std::atomic seen{0}; FreeRun p_fn; Relay r1_fn, r2_fn; CountingSink s_fn{&seen}; // Capacity 1 everywhere: every push contends, so the park path is taken // constantly and the race window is sampled millions of times. kpn::ObjectNode, kpn::out<"a">, "free_run", 0> p (p_fn, 1); kpn::ObjectNode, kpn::out<"b">, "relay1", 0> r1(r1_fn, 1); kpn::ObjectNode, kpn::out<"c">, "relay2", 0> r2(r2_fn, 1); kpn::ObjectNode, kpn::out<>, "sink", 0> s (s_fn, 1); auto net = kpn::make_network( kpn::edge(p.output<"a">(), r1.input<"a">()), kpn::edge(r1.output<"b">(), r2.input<"b">()), kpn::edge(r2.output<"c">(), s.input<"c">()) ); net.start(); // A live chain moves thousands of items a second, so 3 s with no movement // at all is a wedge, not a slow patch. Sampling for 25 s gives the race // ample opportunity: the pipeline hit it roughly twice in 30 runs. const auto giveup = std::chrono::steady_clock::now() + std::chrono::seconds(25); int last = 0; auto last_move = std::chrono::steady_clock::now(); bool stalled = false; int stall_at = 0; while (std::chrono::steady_clock::now() < giveup) { std::this_thread::sleep_for(std::chrono::milliseconds(100)); const int now_seen = seen.load(std::memory_order_relaxed); if (now_seen != last) { last = now_seen; last_move = std::chrono::steady_clock::now(); } else if (std::chrono::steady_clock::now() - last_move > std::chrono::seconds(3)) { stalled = true; stall_at = now_seen; break; } } net.stop(); INFO("chain stalled after " << stall_at << " items"); CHECK_FALSE(stalled); // Guard against the test passing because nothing ever ran. CHECK(last > 1000); }