// Cost attribution for the pipeline benchmark. // // TRACES: VR-015 | UT-120, UT-121, UT-122, UT-123, UT-124 | PR-004 // // `attribute_cost` is pure — no clock, no thread, no network — precisely so the // ranking can be tested on CI hardware that can never run the pipeline. These // are T1 tests: they build the snapshots the KPN network would have produced // and assert which node gets blamed. // // The case that matters is UT-121. On SuperHero the real run reported // `frame_source ema=141.899ms` against a decoder logging 12-18 ms, because // `fire_once` bills time parked pushing into a full downstream channel to the // node doing the pushing. Any metric that ranks nodes by wall time inside the // node picks the source — the fastest node in the graph — as the thing to // optimise. That is the mistake this file exists to prevent regressing. #include "benchmark.hpp" #include #include #include #include using Catch::Matchers::WithinAbs; using Catch::Matchers::WithinRel; using namespace sae::bench; namespace { /// A KPN node snapshot with only the fields attribution reads. kpn::NodeSnapshot node(std::string name, std::uint64_t frames, double ema_ms, double cpu_ms, double exec_ms) { kpn::NodeSnapshot s{}; s.name = std::move(name); s.frames_processed = frames; s.ema_exec_ms = ema_ms; s.total_cpu_ms = cpu_ms; s.total_exec_ms = exec_ms; return s; } /// A channel whose mean fill is `fill_pct` of `capacity`. ChannelOccupancy chan(const std::string& producer, const std::string& consumer, std::size_t capacity, double fill_pct) { ChannelOccupancy c; c.name = producer + ":0 \xe2\x86\x92 " + consumer + ":0"; c.producer = producer; c.consumer = consumer; c.capacity = capacity; c.samples = 1000; c.fill_sum = static_cast(c.samples) * static_cast(capacity) * fill_pct / 100.0; return c; } // Returned by value: a reference into `v` bound to a name built from a string // literal trips -Wdangling-reference, and the struct is small enough not to care. NodeCost by_name(const std::vector& v, std::string_view name) { for (const auto& c : v) if (c.name == name) return c; throw std::runtime_error("no such node: " + std::string(name)); } NodeCost bottleneck(const std::vector& v) { for (const auto& c : v) if (c.is_bottleneck) return c; throw std::runtime_error("no bottleneck flagged"); } } // namespace // UT-120 — the node work queues up in front of is the one blamed. TEST_CASE("attribute_cost blames the node with a full input and an empty output", "[benchmark][VR-015]") { // source → mid → sink. mid is slow: its input backs up, its output drains. const auto nodes = std::vector{ node("source", 1000, 10.0, 2'000.0, 10'000.0), node("mid", 1000, 50.0, 45'000.0, 50'000.0), node("sink", 1000, 0.5, 400.0, 500.0), }; const auto channels = std::vector{ chan("source", "mid", 32, 95.0), // full: work piling up in front of mid chan("mid", "sink", 16, 2.0), // empty: mid starves everything after }; const auto costs = attribute_cost(nodes, channels, /*wall_sec=*/50.0); CHECK(bottleneck(costs).name == "mid"); CHECK(by_name(costs, "mid").pressure > by_name(costs, "source").pressure); CHECK(by_name(costs, "mid").pressure > by_name(costs, "sink").pressure); } // UT-121 — the SuperHero regression: a backpressured source reports a huge // wall time per frame and must NOT be mistaken for the bottleneck. TEST_CASE("a backpressured source is not blamed for the time it spent parked", "[benchmark][VR-015]") { // Numbers taken from the real SuperHero TRT run: the source reports // 141.9 ms/frame inside fire_once while its decoder logs ~15 ms, because // the remaining ~127 ms is spent parked on a full output channel. const auto nodes = std::vector{ node("frame_source", 5132, 141.899, 77'000.0, 728'000.0), node("face_detector", 5129, 6.830, 480'000.0, 35'000.0), node("result_sink", 5129, 0.080, 410.0, 410.0), }; const auto channels = std::vector{ chan("frame_source", "face_detector", 32, 99.0), // source blocked on this chan("face_detector", "result_sink", 16, 1.0), }; const auto costs = attribute_cost(nodes, channels, /*wall_sec=*/500.0); const auto& src = by_name(costs, "frame_source"); const auto& det = by_name(costs, "face_detector"); // The trap: by wall time inside the node, the source looks 20x costlier. REQUIRE(src.exec_ms_per_frame > det.exec_ms_per_frame * 10.0); // The fix: it is not blamed, because its own output channel is the thing // that is full — it is waiting, not working. CHECK_FALSE(src.is_bottleneck); CHECK(bottleneck(costs).name == "face_detector"); // And CPU time, which parking cannot inflate, agrees: the detector burns // 480 s of thread time against the source's 77 s. CHECK(det.cpu_ms > src.cpu_ms); CHECK(det.cpu_pct_of_pipeline > src.cpu_pct_of_pipeline); } // UT-122 — terminals stay rankable via the infinite-reservoir convention. TEST_CASE("a source with an empty output is blamed; a sink with a full input is too", "[benchmark][VR-015]") { SECTION("starved pipeline: the source cannot keep up") { const auto nodes = std::vector{ node("source", 100, 90.0, 9'000.0, 9'000.0), node("mid", 100, 1.0, 100.0, 100.0), }; // Nothing ever accumulates: the source is the constraint. const auto costs = attribute_cost(nodes, {chan("source", "mid", 32, 1.0)}, 10.0); CHECK(bottleneck(costs).name == "source"); // Source has no input channel, so it is treated as always having work. CHECK_THAT(by_name(costs, "source").in_fill_pct, WithinAbs(100.0, 1e-9)); } SECTION("congested pipeline: the sink cannot drain") { const auto nodes = std::vector{ node("mid", 100, 1.0, 100.0, 100.0), node("sink", 100, 90.0, 9'000.0, 9'000.0), }; const auto costs = attribute_cost(nodes, {chan("mid", "sink", 16, 98.0)}, 10.0); CHECK(bottleneck(costs).name == "sink"); // Sink has no output channel, so it is treated as never blocking. CHECK_THAT(by_name(costs, "sink").out_fill_pct, WithinAbs(0.0, 1e-9)); } } // UT-123 — the per-node time figures are the ones an optimiser would act on. TEST_CASE("cost shares are computed against wall clock and pipeline total", "[benchmark][VR-015]") { const auto nodes = std::vector{ node("a", 100, 1.0, 30'000.0, 40'000.0), // 30 s CPU node("b", 100, 1.0, 10'000.0, 12'000.0), // 10 s CPU }; const auto costs = attribute_cost(nodes, {chan("a", "b", 8, 50.0)}, /*wall_sec=*/50.0); const auto& a = by_name(costs, "a"); CHECK_THAT(a.cpu_ms_per_frame, WithinRel(300.0, 1e-9)); // 30 s / 100 frames CHECK_THAT(a.exec_ms_per_frame, WithinRel(400.0, 1e-9)); CHECK_THAT(a.cpu_share, WithinRel(0.6, 1e-9)); // 30 s of a 50 s run CHECK_THAT(a.exec_share, WithinRel(0.8, 1e-9)); CHECK_THAT(a.cpu_pct_of_pipeline, WithinRel(75.0, 1e-9)); // 30 of 40 s total // Time inside the node that was not spent on its own CPU: parked, or on GPU. CHECK_THAT(a.stall_ms_per_frame, WithinRel(100.0, 1e-9)); // A node that never ran cannot be the bottleneck, and contributes no cost. const auto idle = std::vector{ node("ran", 10, 1.0, 100.0, 100.0), node("idle", 0, 0.0, 0.0, 0.0), }; const auto idle_costs = attribute_cost(idle, {}, 10.0); CHECK(bottleneck(idle_costs).name == "ran"); CHECK_FALSE(by_name(idle_costs, "idle").is_bottleneck); } // UT-124 — the node graph is recovered from KPN's channel names, which is what // keeps attribution working when the topology changes. TEST_CASE("channel names split back into producer and consumer", "[benchmark][VR-015]") { std::string p, c; split_edge_name("frame_source:0 \xe2\x86\x92 camera_pos:0", p, c); CHECK(p == "frame_source"); CHECK(c == "camera_pos"); // Multi-port nodes: the port index is stripped, the node name is not. split_edge_name("detector:2 \xe2\x86\x92 aligner:1", p, c); CHECK(p == "detector"); CHECK(c == "aligner"); // A name with no arrow leaves both untouched rather than inventing an edge. std::string q = "unset", r = "unset"; split_edge_name("not an edge", q, r); CHECK(q == "unset"); CHECK(r == "unset"); } // A node with several inputs is gated by its emptiest one, and blocked by its // fullest output — the multi-branch case the scene-detect topology creates. TEST_CASE("multi-port nodes take min input fill and max output fill", "[benchmark][VR-015]") { const auto nodes = std::vector{ node("join", 100, 1.0, 1'000.0, 1'000.0), }; const auto channels = std::vector{ chan("up_a", "join", 32, 99.0), // full, but... chan("up_b", "join", 32, 4.0), // ...this one gates the node chan("join", "down_a", 16, 10.0), chan("join", "down_b", 16, 80.0), // parking on this stops the node }; const auto costs = attribute_cost(nodes, channels, 10.0); const auto& j = by_name(costs, "join"); CHECK_THAT(j.in_fill_pct, WithinAbs( 4.0, 1e-9)); CHECK_THAT(j.out_fill_pct, WithinAbs(80.0, 1e-9)); CHECK(j.pressure < 0.0); // starved, not congested }