feat(pipeline): detect node crashes and tally dropped frames
Register a KPN event handler in both scene_analyze and scene_preview:
- Overflow events accumulate per-node dropped-frame counts, printed on exit.
- A Closed event from any node other than result_sink at EOF means a stage
died; trip an atomic so the main loop bails out instead of hanging on
'done' forever, and exit non-zero.
Bumps external/KPN to the commit that exposes set_event_handler / NodeEvent.
This commit is contained in:
Vendored
+1
-1
Submodule external/KPN updated: 7c6a8be2b7...949c8134ef
+44
-5
@@ -48,8 +48,11 @@
|
|||||||
#include <chrono>
|
#include <chrono>
|
||||||
#include <cstring>
|
#include <cstring>
|
||||||
#include <iostream>
|
#include <iostream>
|
||||||
|
#include <map>
|
||||||
|
#include <mutex>
|
||||||
#include <stdexcept>
|
#include <stdexcept>
|
||||||
#include <string>
|
#include <string>
|
||||||
|
#include <string_view>
|
||||||
#include <thread>
|
#include <thread>
|
||||||
|
|
||||||
// ── CLI parsing ───────────────────────────────────────────────────────────────
|
// ── CLI parsing ───────────────────────────────────────────────────────────────
|
||||||
@@ -192,17 +195,53 @@ int main(int argc, char** argv) {
|
|||||||
);
|
);
|
||||||
#endif
|
#endif
|
||||||
|
|
||||||
|
// ── Pipeline observability (KPN event handler) ────────────────────────────
|
||||||
|
// Tally dropped frames per node (channel overflow) and detect a node that
|
||||||
|
// stops unexpectedly. A Closed event from any node other than result_sink at
|
||||||
|
// EOF means a stage crashed — without this the main loop below would hang on
|
||||||
|
// `done` forever, so we trip a flag to unblock it and exit non-zero.
|
||||||
|
std::mutex event_mtx;
|
||||||
|
std::map<std::string, long> overflow_counts;
|
||||||
|
std::atomic<bool> node_crashed{false};
|
||||||
|
|
||||||
|
net.set_event_handler(
|
||||||
|
[&](std::string_view node_name, kpn::NodeEvent ev,
|
||||||
|
std::chrono::steady_clock::time_point) {
|
||||||
|
if (ev == kpn::NodeEvent::Overflow) {
|
||||||
|
std::lock_guard<std::mutex> lk(event_mtx);
|
||||||
|
++overflow_counts[std::string(node_name)];
|
||||||
|
} else { // NodeEvent::Closed
|
||||||
|
// result_sink closing once EOF has been signalled is the normal
|
||||||
|
// shutdown path, not a crash.
|
||||||
|
if (node_name == "result_sink" && done.load(std::memory_order_acquire))
|
||||||
|
return;
|
||||||
|
std::cerr << "[main] node '" << node_name
|
||||||
|
<< "' stopped unexpectedly — aborting pipeline\n";
|
||||||
|
node_crashed.store(true, std::memory_order_release);
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
// ── Run ───────────────────────────────────────────────────────────────────
|
// ── Run ───────────────────────────────────────────────────────────────────
|
||||||
// Note: StaticNetwork does not expose set_error_handler; node exceptions
|
|
||||||
// are printed to stderr by the KPN run_loop and the network continues.
|
|
||||||
std::cerr << "[main] starting pipeline…\n";
|
std::cerr << "[main] starting pipeline…\n";
|
||||||
net.start();
|
net.start();
|
||||||
|
|
||||||
// Main thread waits until ResultSinkFunc signals EOF completion
|
// Main thread waits until ResultSinkFunc signals EOF completion, or a node
|
||||||
while (!done.load(std::memory_order_acquire))
|
// crash trips node_crashed.
|
||||||
|
while (!done.load(std::memory_order_acquire) &&
|
||||||
|
!node_crashed.load(std::memory_order_acquire))
|
||||||
std::this_thread::sleep_for(std::chrono::milliseconds(100));
|
std::this_thread::sleep_for(std::chrono::milliseconds(100));
|
||||||
|
|
||||||
net.stop();
|
net.stop();
|
||||||
net.print_diagnostics();
|
net.print_diagnostics();
|
||||||
return 0;
|
|
||||||
|
{
|
||||||
|
std::lock_guard<std::mutex> lk(event_mtx);
|
||||||
|
if (!overflow_counts.empty()) {
|
||||||
|
std::cerr << "[main] dropped frames (channel overflow):\n";
|
||||||
|
for (const auto& [name, count] : overflow_counts)
|
||||||
|
std::cerr << " " << name << ": " << count << "\n";
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return node_crashed.load(std::memory_order_acquire) ? 1 : 0;
|
||||||
}
|
}
|
||||||
|
|||||||
+40
-3
@@ -35,8 +35,11 @@
|
|||||||
#include <chrono>
|
#include <chrono>
|
||||||
#include <cstring>
|
#include <cstring>
|
||||||
#include <iostream>
|
#include <iostream>
|
||||||
|
#include <map>
|
||||||
|
#include <mutex>
|
||||||
#include <stdexcept>
|
#include <stdexcept>
|
||||||
#include <string>
|
#include <string>
|
||||||
|
#include <string_view>
|
||||||
#include <thread>
|
#include <thread>
|
||||||
|
|
||||||
static Config parse_args(int argc, char** argv) {
|
static Config parse_args(int argc, char** argv) {
|
||||||
@@ -143,12 +146,36 @@ int main(int argc, char** argv) {
|
|||||||
kpn::edge(tracker.output<"annotation">(), sink.input<"annotation">())
|
kpn::edge(tracker.output<"annotation">(), sink.input<"annotation">())
|
||||||
);
|
);
|
||||||
|
|
||||||
|
// ── Pipeline observability (KPN event handler) ────────────────────────────
|
||||||
|
// Tally dropped frames per node (channel overflow) and detect a node that
|
||||||
|
// stops unexpectedly so the preview loop below can bail out instead of
|
||||||
|
// spinning on a dead pipeline.
|
||||||
|
std::mutex event_mtx;
|
||||||
|
std::map<std::string, long> overflow_counts;
|
||||||
|
std::atomic<bool> node_crashed{false};
|
||||||
|
|
||||||
|
net.set_event_handler(
|
||||||
|
[&](std::string_view node_name, kpn::NodeEvent ev,
|
||||||
|
std::chrono::steady_clock::time_point) {
|
||||||
|
if (ev == kpn::NodeEvent::Overflow) {
|
||||||
|
std::lock_guard<std::mutex> lk(event_mtx);
|
||||||
|
++overflow_counts[std::string(node_name)];
|
||||||
|
} else { // NodeEvent::Closed
|
||||||
|
if (node_name == "result_sink" && done.load(std::memory_order_acquire))
|
||||||
|
return;
|
||||||
|
std::cerr << "[main] node '" << node_name
|
||||||
|
<< "' stopped unexpectedly — aborting pipeline\n";
|
||||||
|
node_crashed.store(true, std::memory_order_release);
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
// ── Run ───────────────────────────────────────────────────────────────────
|
// ── Run ───────────────────────────────────────────────────────────────────
|
||||||
std::cerr << "[main] starting pipeline — press q or Esc to quit early\n";
|
std::cerr << "[main] starting pipeline — press q or Esc to quit early\n";
|
||||||
net.start();
|
net.start();
|
||||||
|
|
||||||
// Main thread drives the display window; returns false on EOF or q/Esc
|
// Main thread drives the display window; returns false on EOF or q/Esc.
|
||||||
while (preview.step()) {
|
// Bail out early if a node crashes.
|
||||||
|
while (!node_crashed.load(std::memory_order_acquire) && preview.step()) {
|
||||||
cv::waitKey(1); // pump OS events between frames
|
cv::waitKey(1); // pump OS events between frames
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -156,5 +183,15 @@ int main(int argc, char** argv) {
|
|||||||
sink_fn.flush(); // write whatever was accumulated (no-op if EOF already flushed)
|
sink_fn.flush(); // write whatever was accumulated (no-op if EOF already flushed)
|
||||||
cv::destroyAllWindows();
|
cv::destroyAllWindows();
|
||||||
net.print_diagnostics();
|
net.print_diagnostics();
|
||||||
return 0;
|
|
||||||
|
{
|
||||||
|
std::lock_guard<std::mutex> lk(event_mtx);
|
||||||
|
if (!overflow_counts.empty()) {
|
||||||
|
std::cerr << "[main] dropped frames (channel overflow):\n";
|
||||||
|
for (const auto& [name, count] : overflow_counts)
|
||||||
|
std::cerr << " " << name << ": " << count << "\n";
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return node_crashed.load(std::memory_order_acquire) ? 1 : 0;
|
||||||
}
|
}
|
||||||
|
|||||||
Reference in New Issue
Block a user