dtourolle/KPN
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Performance improvements, better readme and complete python bindings
🧪 Test / test (push) Failing after 28m30s
Details

Browse Source
This commit is contained in:
dtourolle
2026-05-12 21:23:33 +02:00
parent c39db82763
commit f6bcaa15b0
38 changed files with 4679 additions and 846 deletions
Download Patch File Download Diff File Expand all files Collapse all files
include/kpn/network.hpp
+118 -6
View File
@@ -1,6 +1,6 @@
#pragma once
#include "diagnostics.hpp"
#include "node.hpp"
#include "inode.hpp"
#include "port.hpp"
#ifdef KPN_WEB_DEBUG
@@ -126,14 +126,17 @@ public:
web_debug_port_,
[this]() {
auto s = collect_snapshots();
return web_debug::to_json(s.nodes, s.channels, {}, s.elapsed_s);
return web_debug::to_json(s.nodes, s.channels, {}, s.elapsed_s, s.pools);
});
web_server_->start();
std::cerr << "[kpn] web debug UI: http://localhost:" << web_debug_port_ << "\n";
#endif
}
void stop() override {
void stop() override { halt(); }
// halt(): immediate stop — broadcasts disable to all channels and joins threads.
void halt() override {
#ifdef KPN_WEB_DEBUG
if (web_server_) web_server_->stop();
#endif
@@ -142,6 +145,37 @@ public:
nodes_.at(*it)->stop();
}
// shutdown(): graceful drain in topological order.
// Stops source nodes first, polls until their output channels drain to zero,
// then stops the next layer, and so on.
void shutdown() override {
#ifdef KPN_WEB_DEBUG
if (web_server_) web_server_->stop();
#endif
stop_watchdog();
// Identify which nodes have no incoming edges (sources).
std::map<std::string, std::size_t> in_degree;
for (auto& [name, _] : nodes_) in_degree[name] = 0;
for (auto& [src, dsts] : adj_)
for (auto& dst : dsts) in_degree[dst]++;
// Walk topo order: stop each source layer, wait for its output channels
// to drain, then proceed to the next layer.
std::set<std::string> stopped;
for (auto& name : topo_) {
if (in_degree[name] == 0 || all_predecessors_stopped(name, stopped)) {
nodes_.at(name)->stop();
stopped.insert(name);
// Wait for output channels of this node to drain.
drain_output_channels(name);
}
}
// Stop any remaining nodes (sinks / nodes not yet stopped).
for (auto it = topo_.rbegin(); it != topo_.rend(); ++it)
if (!stopped.count(*it)) nodes_.at(*it)->stop();
}
bool running() const override { return watchdog_.joinable(); }
void set_name(std::string) override {}
@@ -163,6 +197,10 @@ public:
void set_error_handler(ErrorHandler h) { error_handler_ = std::move(h); }
void set_diagnostics_handler(DiagnosticsHandler h) { diag_handler_ = std::move(h); }
void register_pool(const std::string& name, IPoolProbe* probe) {
pool_probes_.emplace_back(name, probe);
}
#ifdef KPN_WEB_DEBUG
void set_web_debug_port(uint16_t port) { web_debug_port_ = port; }
#endif
@@ -171,7 +209,7 @@ public:
// Can be called at any time; thread-safe (reads atomics with relaxed ordering).
void print_diagnostics(std::ostream& os = std::cerr) const {
auto s = collect_snapshots();
os << format_report(s.nodes, s.channels, s.elapsed_s);
os << format_report(s.nodes, s.channels, s.pools, s.elapsed_s);
}
private:
@@ -180,6 +218,7 @@ private:
struct Snapshots {
std::vector<NodeSnapshot> nodes;
std::vector<ChannelSnapshot> channels;
std::vector<PoolSnapshot> pools;
double elapsed_s;
};
@@ -195,11 +234,16 @@ private:
for (auto& probe : channel_probes_)
channels.push_back(probe->snapshot());
return {std::move(nodes), std::move(channels), elapsed_s};
std::vector<PoolSnapshot> pools;
for (auto& [name, probe] : pool_probes_)
pools.push_back(probe->snapshot(name));
return {std::move(nodes), std::move(channels), std::move(pools), elapsed_s};
}
static std::string format_report(const std::vector<NodeSnapshot>& nodes,
const std::vector<ChannelSnapshot>& channels,
const std::vector<PoolSnapshot>& pools = {},
double elapsed_s = 0.0) {
std::ostringstream os;
os << std::fixed << std::setprecision(1);
@@ -258,6 +302,31 @@ private:
<< flag << "\n";
}
// Pool table
if (!pools.empty()) {
os << "│\n│ Thread Pools:\n";
os << "│ " << std::left
<< std::setw(16) << "name"
<< std::setw(10) << "threads"
<< std::setw(12) << "queued"
<< std::setw(12) << "active"
<< std::setw(14) << "in/s"
<< std::setw(14) << "out/s"
<< "\n│ " << std::string(78, '-') << "\n";
for (auto& p : pools) {
double in_rate = elapsed_s > 0.0 ? p.tasks_submitted / elapsed_s : 0.0;
double out_rate = elapsed_s > 0.0 ? p.tasks_completed / elapsed_s : 0.0;
os << "│ " << std::left
<< std::setw(16) << p.name
<< std::setw(10) << p.thread_count
<< std::setw(12) << p.queue_depth
<< std::setw(12) << p.active_count
<< std::setw(14) << in_rate
<< std::setw(14) << out_rate
<< "\n";
}
}
// Bottleneck hint: node with highest ema_exec_ms
if (!nodes.empty()) {
auto it = std::max_element(nodes.begin(), nodes.end(),
@@ -271,6 +340,31 @@ private:
return os.str();
}
// ── Shutdown helpers ──────────────────────────────────────────────────────
bool all_predecessors_stopped(const std::string& name,
const std::set<std::string>& stopped) const {
for (auto& [src, dsts] : adj_)
for (auto& dst : dsts)
if (dst == name && !stopped.count(src)) return false;
return true;
}
void drain_output_channels(const std::string& /*name*/) const {
// Poll all channel probes until none report non-zero fill.
// A short sleep prevents busy-spin; 1 ms is fine for drain purposes.
bool any_full = true;
while (any_full) {
any_full = false;
for (auto& probe : channel_probes_) {
auto snap = probe->snapshot();
if (snap.current_fill > 0) { any_full = true; break; }
}
if (any_full)
std::this_thread::sleep_for(std::chrono::milliseconds(1));
}
}
// ── Cycle detection / topological sort ───────────────────────────────────
void dfs(const std::string& name, std::map<std::string, int>& color) {
@@ -292,16 +386,33 @@ private:
if (tok.stop_requested()) break;
auto s = collect_snapshots();
check_hung_nodes();
if (diag_handler_) {
diag_handler_(s.nodes, s.channels);
} else {
std::cerr << format_report(s.nodes, s.channels, s.elapsed_s);
std::cerr << format_report(s.nodes, s.channels, s.pools, s.elapsed_s);
}
}
});
}
void check_hung_nodes() const {
auto now_us = std::chrono::duration_cast<std::chrono::microseconds>(
clock_t::now().time_since_epoch()).count();
for (auto& [name, node] : nodes_) {
int64_t start = node->stats().exec_start_us.load(std::memory_order_relaxed);
if (start == 0) continue;
int64_t elapsed_ms = (now_us - start) / 1000;
// Warn if a node has been executing for > 5 s with no max_exec_time set,
// or if it exceeds its configured max. Threshold: 5000 ms default.
if (elapsed_ms > 5000) {
std::cerr << "[kpn] WARNING: node '" << name
<< "' has been executing for " << elapsed_ms << " ms\n";
}
}
}
void stop_watchdog() {
if (watchdog_.joinable())
watchdog_.request_stop(), watchdog_.join();
@@ -316,6 +427,7 @@ private:
std::map<std::string, std::string> exposed_outputs_;
std::set<std::pair<std::string, std::size_t>> connected_outputs_;
std::vector<std::unique_ptr<IChannelProbe>> channel_probes_;
std::vector<std::pair<std::string, IPoolProbe*>> pool_probes_;
ErrorHandler error_handler_;
DiagnosticsHandler diag_handler_;
std::chrono::milliseconds watchdog_interval_{3000};