feat(benchmark): per-node cost and bottleneck attribution for a run
--benchmark <path> reports cumulative CPU and wall time per node and names the node pacing the run. The pacing node is located from sampled channel occupancy, not from time-in-node: backpressure inflates time-in-node for everything downstream of the real bottleneck, so the obvious measure names the victim rather than the cause. Sampling starts with the network and stops before it is destroyed. Channel fill is instantaneous and everything has drained by shutdown, so a single read at the end reports an idle pipeline however congested it was. kill -USR1 dumps the table from a running or wedged process. Channel occupancy identifies a stalled node -- full input, empty output -- without a debug build or a debugger, which is the difference between diagnosing the AR-004 hang in seconds and reproducing it under gdb. Two knobs this exposes for measurement rather than sets: SAE_CV_THREADS, because OpenCV's TBB arena and KPN's thread-per-node are two schedulers unaware of each other on the same cores; and SAE_CUDA_BLOCKING_SYNC, because the default spin-wait held the embedder thread at 99.7% user time while nvidia-powerd cut the GPU's clock from 1005 to 210 MHz. Neither default changes until a measurement says it should. TRACES: VR-015 | PR-004
This commit is contained in:
@@ -0,0 +1,221 @@
|
||||
// Cost attribution for the pipeline benchmark.
|
||||
//
|
||||
// TRACES: VR-015 | UT-120, UT-121, UT-122, UT-123, UT-124 | PR-004
|
||||
//
|
||||
// `attribute_cost` is pure — no clock, no thread, no network — precisely so the
|
||||
// ranking can be tested on CI hardware that can never run the pipeline. These
|
||||
// are T1 tests: they build the snapshots the KPN network would have produced
|
||||
// and assert which node gets blamed.
|
||||
//
|
||||
// The case that matters is UT-121. On SuperHero the real run reported
|
||||
// `frame_source ema=141.899ms` against a decoder logging 12-18 ms, because
|
||||
// `fire_once` bills time parked pushing into a full downstream channel to the
|
||||
// node doing the pushing. Any metric that ranks nodes by wall time inside the
|
||||
// node picks the source — the fastest node in the graph — as the thing to
|
||||
// optimise. That is the mistake this file exists to prevent regressing.
|
||||
|
||||
#include "benchmark.hpp"
|
||||
|
||||
#include <catch2/catch_test_macros.hpp>
|
||||
#include <catch2/matchers/catch_matchers_floating_point.hpp>
|
||||
|
||||
#include <stdexcept>
|
||||
#include <string_view>
|
||||
|
||||
using Catch::Matchers::WithinAbs;
|
||||
using Catch::Matchers::WithinRel;
|
||||
using namespace sae::bench;
|
||||
|
||||
namespace {
|
||||
|
||||
/// A KPN node snapshot with only the fields attribution reads.
|
||||
kpn::NodeSnapshot node(std::string name, std::uint64_t frames,
|
||||
double ema_ms, double cpu_ms, double exec_ms) {
|
||||
kpn::NodeSnapshot s{};
|
||||
s.name = std::move(name);
|
||||
s.frames_processed = frames;
|
||||
s.ema_exec_ms = ema_ms;
|
||||
s.total_cpu_ms = cpu_ms;
|
||||
s.total_exec_ms = exec_ms;
|
||||
return s;
|
||||
}
|
||||
|
||||
/// A channel whose mean fill is `fill_pct` of `capacity`.
|
||||
ChannelOccupancy chan(const std::string& producer, const std::string& consumer,
|
||||
std::size_t capacity, double fill_pct) {
|
||||
ChannelOccupancy c;
|
||||
c.name = producer + ":0 \xe2\x86\x92 " + consumer + ":0";
|
||||
c.producer = producer;
|
||||
c.consumer = consumer;
|
||||
c.capacity = capacity;
|
||||
c.samples = 1000;
|
||||
c.fill_sum = static_cast<double>(c.samples) * static_cast<double>(capacity) * fill_pct / 100.0;
|
||||
return c;
|
||||
}
|
||||
|
||||
// Returned by value: a reference into `v` bound to a name built from a string
|
||||
// literal trips -Wdangling-reference, and the struct is small enough not to care.
|
||||
NodeCost by_name(const std::vector<NodeCost>& v, std::string_view name) {
|
||||
for (const auto& c : v) if (c.name == name) return c;
|
||||
throw std::runtime_error("no such node: " + std::string(name));
|
||||
}
|
||||
|
||||
NodeCost bottleneck(const std::vector<NodeCost>& v) {
|
||||
for (const auto& c : v) if (c.is_bottleneck) return c;
|
||||
throw std::runtime_error("no bottleneck flagged");
|
||||
}
|
||||
|
||||
} // namespace
|
||||
|
||||
// UT-120 — the node work queues up in front of is the one blamed.
|
||||
TEST_CASE("attribute_cost blames the node with a full input and an empty output",
|
||||
"[benchmark][VR-015]") {
|
||||
// source → mid → sink. mid is slow: its input backs up, its output drains.
|
||||
const auto nodes = std::vector<kpn::NodeSnapshot>{
|
||||
node("source", 1000, 10.0, 2'000.0, 10'000.0),
|
||||
node("mid", 1000, 50.0, 45'000.0, 50'000.0),
|
||||
node("sink", 1000, 0.5, 400.0, 500.0),
|
||||
};
|
||||
const auto channels = std::vector<ChannelOccupancy>{
|
||||
chan("source", "mid", 32, 95.0), // full: work piling up in front of mid
|
||||
chan("mid", "sink", 16, 2.0), // empty: mid starves everything after
|
||||
};
|
||||
|
||||
const auto costs = attribute_cost(nodes, channels, /*wall_sec=*/50.0);
|
||||
|
||||
CHECK(bottleneck(costs).name == "mid");
|
||||
CHECK(by_name(costs, "mid").pressure > by_name(costs, "source").pressure);
|
||||
CHECK(by_name(costs, "mid").pressure > by_name(costs, "sink").pressure);
|
||||
}
|
||||
|
||||
// UT-121 — the SuperHero regression: a backpressured source reports a huge
|
||||
// wall time per frame and must NOT be mistaken for the bottleneck.
|
||||
TEST_CASE("a backpressured source is not blamed for the time it spent parked",
|
||||
"[benchmark][VR-015]") {
|
||||
// Numbers taken from the real SuperHero TRT run: the source reports
|
||||
// 141.9 ms/frame inside fire_once while its decoder logs ~15 ms, because
|
||||
// the remaining ~127 ms is spent parked on a full output channel.
|
||||
const auto nodes = std::vector<kpn::NodeSnapshot>{
|
||||
node("frame_source", 5132, 141.899, 77'000.0, 728'000.0),
|
||||
node("face_detector", 5129, 6.830, 480'000.0, 35'000.0),
|
||||
node("result_sink", 5129, 0.080, 410.0, 410.0),
|
||||
};
|
||||
const auto channels = std::vector<ChannelOccupancy>{
|
||||
chan("frame_source", "face_detector", 32, 99.0), // source blocked on this
|
||||
chan("face_detector", "result_sink", 16, 1.0),
|
||||
};
|
||||
|
||||
const auto costs = attribute_cost(nodes, channels, /*wall_sec=*/500.0);
|
||||
|
||||
const auto& src = by_name(costs, "frame_source");
|
||||
const auto& det = by_name(costs, "face_detector");
|
||||
|
||||
// The trap: by wall time inside the node, the source looks 20x costlier.
|
||||
REQUIRE(src.exec_ms_per_frame > det.exec_ms_per_frame * 10.0);
|
||||
// The fix: it is not blamed, because its own output channel is the thing
|
||||
// that is full — it is waiting, not working.
|
||||
CHECK_FALSE(src.is_bottleneck);
|
||||
CHECK(bottleneck(costs).name == "face_detector");
|
||||
// And CPU time, which parking cannot inflate, agrees: the detector burns
|
||||
// 480 s of thread time against the source's 77 s.
|
||||
CHECK(det.cpu_ms > src.cpu_ms);
|
||||
CHECK(det.cpu_pct_of_pipeline > src.cpu_pct_of_pipeline);
|
||||
}
|
||||
|
||||
// UT-122 — terminals stay rankable via the infinite-reservoir convention.
|
||||
TEST_CASE("a source with an empty output is blamed; a sink with a full input is too",
|
||||
"[benchmark][VR-015]") {
|
||||
SECTION("starved pipeline: the source cannot keep up") {
|
||||
const auto nodes = std::vector<kpn::NodeSnapshot>{
|
||||
node("source", 100, 90.0, 9'000.0, 9'000.0),
|
||||
node("mid", 100, 1.0, 100.0, 100.0),
|
||||
};
|
||||
// Nothing ever accumulates: the source is the constraint.
|
||||
const auto costs = attribute_cost(nodes, {chan("source", "mid", 32, 1.0)}, 10.0);
|
||||
CHECK(bottleneck(costs).name == "source");
|
||||
// Source has no input channel, so it is treated as always having work.
|
||||
CHECK_THAT(by_name(costs, "source").in_fill_pct, WithinAbs(100.0, 1e-9));
|
||||
}
|
||||
|
||||
SECTION("congested pipeline: the sink cannot drain") {
|
||||
const auto nodes = std::vector<kpn::NodeSnapshot>{
|
||||
node("mid", 100, 1.0, 100.0, 100.0),
|
||||
node("sink", 100, 90.0, 9'000.0, 9'000.0),
|
||||
};
|
||||
const auto costs = attribute_cost(nodes, {chan("mid", "sink", 16, 98.0)}, 10.0);
|
||||
CHECK(bottleneck(costs).name == "sink");
|
||||
// Sink has no output channel, so it is treated as never blocking.
|
||||
CHECK_THAT(by_name(costs, "sink").out_fill_pct, WithinAbs(0.0, 1e-9));
|
||||
}
|
||||
}
|
||||
|
||||
// UT-123 — the per-node time figures are the ones an optimiser would act on.
|
||||
TEST_CASE("cost shares are computed against wall clock and pipeline total",
|
||||
"[benchmark][VR-015]") {
|
||||
const auto nodes = std::vector<kpn::NodeSnapshot>{
|
||||
node("a", 100, 1.0, 30'000.0, 40'000.0), // 30 s CPU
|
||||
node("b", 100, 1.0, 10'000.0, 12'000.0), // 10 s CPU
|
||||
};
|
||||
const auto costs = attribute_cost(nodes, {chan("a", "b", 8, 50.0)}, /*wall_sec=*/50.0);
|
||||
|
||||
const auto& a = by_name(costs, "a");
|
||||
CHECK_THAT(a.cpu_ms_per_frame, WithinRel(300.0, 1e-9)); // 30 s / 100 frames
|
||||
CHECK_THAT(a.exec_ms_per_frame, WithinRel(400.0, 1e-9));
|
||||
CHECK_THAT(a.cpu_share, WithinRel(0.6, 1e-9)); // 30 s of a 50 s run
|
||||
CHECK_THAT(a.exec_share, WithinRel(0.8, 1e-9));
|
||||
CHECK_THAT(a.cpu_pct_of_pipeline, WithinRel(75.0, 1e-9)); // 30 of 40 s total
|
||||
// Time inside the node that was not spent on its own CPU: parked, or on GPU.
|
||||
CHECK_THAT(a.stall_ms_per_frame, WithinRel(100.0, 1e-9));
|
||||
|
||||
// A node that never ran cannot be the bottleneck, and contributes no cost.
|
||||
const auto idle = std::vector<kpn::NodeSnapshot>{
|
||||
node("ran", 10, 1.0, 100.0, 100.0),
|
||||
node("idle", 0, 0.0, 0.0, 0.0),
|
||||
};
|
||||
const auto idle_costs = attribute_cost(idle, {}, 10.0);
|
||||
CHECK(bottleneck(idle_costs).name == "ran");
|
||||
CHECK_FALSE(by_name(idle_costs, "idle").is_bottleneck);
|
||||
}
|
||||
|
||||
// UT-124 — the node graph is recovered from KPN's channel names, which is what
|
||||
// keeps attribution working when the topology changes.
|
||||
TEST_CASE("channel names split back into producer and consumer",
|
||||
"[benchmark][VR-015]") {
|
||||
std::string p, c;
|
||||
split_edge_name("frame_source:0 \xe2\x86\x92 camera_pos:0", p, c);
|
||||
CHECK(p == "frame_source");
|
||||
CHECK(c == "camera_pos");
|
||||
|
||||
// Multi-port nodes: the port index is stripped, the node name is not.
|
||||
split_edge_name("detector:2 \xe2\x86\x92 aligner:1", p, c);
|
||||
CHECK(p == "detector");
|
||||
CHECK(c == "aligner");
|
||||
|
||||
// A name with no arrow leaves both untouched rather than inventing an edge.
|
||||
std::string q = "unset", r = "unset";
|
||||
split_edge_name("not an edge", q, r);
|
||||
CHECK(q == "unset");
|
||||
CHECK(r == "unset");
|
||||
}
|
||||
|
||||
// A node with several inputs is gated by its emptiest one, and blocked by its
|
||||
// fullest output — the multi-branch case the scene-detect topology creates.
|
||||
TEST_CASE("multi-port nodes take min input fill and max output fill",
|
||||
"[benchmark][VR-015]") {
|
||||
const auto nodes = std::vector<kpn::NodeSnapshot>{
|
||||
node("join", 100, 1.0, 1'000.0, 1'000.0),
|
||||
};
|
||||
const auto channels = std::vector<ChannelOccupancy>{
|
||||
chan("up_a", "join", 32, 99.0), // full, but...
|
||||
chan("up_b", "join", 32, 4.0), // ...this one gates the node
|
||||
chan("join", "down_a", 16, 10.0),
|
||||
chan("join", "down_b", 16, 80.0), // parking on this stops the node
|
||||
};
|
||||
|
||||
const auto costs = attribute_cost(nodes, channels, 10.0);
|
||||
const auto& j = by_name(costs, "join");
|
||||
|
||||
CHECK_THAT(j.in_fill_pct, WithinAbs( 4.0, 1e-9));
|
||||
CHECK_THAT(j.out_fill_pct, WithinAbs(80.0, 1e-9));
|
||||
CHECK(j.pressure < 0.0); // starved, not congested
|
||||
}
|
||||
Reference in New Issue
Block a user