diff --git a/.gitea/workflows/ci.yaml b/.gitea/workflows/ci.yaml index f9d22c1..e962340 100644 --- a/.gitea/workflows/ci.yaml +++ b/.gitea/workflows/ci.yaml @@ -63,7 +63,8 @@ jobs: if: ${{ needs.changes.outputs.dockerfile == 'true' }} uses: ./.gitea/workflows/docker.yaml with: - push: ${{ github.event_name != 'pull_request' }} + # Explicit string, not a boolean expression (act_runner mangles bools). + push: ${{ github.event_name == 'pull_request' && 'false' || 'true' }} # Runs after docker (if docker ran). A skipped docker job is fine; a failed # one blocks this via !failure(). Re-run tests when code OR the image changed. @@ -72,6 +73,13 @@ jobs: if: ${{ !failure() && !cancelled() && (needs.changes.outputs.code == 'true' || needs.changes.outputs.dockerfile == 'true') }} uses: ./.gitea/workflows/test.yaml + # ThreadSanitizer run for the lock-free Channel. Same trigger conditions as + # test (code or image changed); runs in parallel with test. + tsan: + needs: [changes, docker] + if: ${{ !failure() && !cancelled() && (needs.changes.outputs.code == 'true' || needs.changes.outputs.dockerfile == 'true') }} + uses: ./.gitea/workflows/tsan.yaml + docs: needs: [changes, docker] if: ${{ !failure() && !cancelled() && github.ref == 'refs/heads/master' && (needs.changes.outputs.docs == 'true' || needs.changes.outputs.dockerfile == 'true') }} diff --git a/.gitea/workflows/docker.yaml b/.gitea/workflows/docker.yaml index 699af5b..27fa328 100644 --- a/.gitea/workflows/docker.yaml +++ b/.gitea/workflows/docker.yaml @@ -4,19 +4,22 @@ name: '🐳 Builder Image' # It is called by ci.yaml only when Dockerfile.builder or docs/requirements.txt # change. It runs on the host runner (NOT inside the builder container) because # it needs the Docker CLI/daemon. +# Note: `push` is a STRING ("true"/"false"), not a boolean. Gitea's act_runner +# mangles boolean inputs passed from an expression (they arrive as false), so we +# pass an explicit string and compare with == 'true' below. on: workflow_call: inputs: push: - description: 'Push the built image to the registry' - type: boolean - default: true + description: 'Push the built image to the registry ("true"/"false")' + type: string + default: 'true' workflow_dispatch: inputs: push: - description: 'Push the built image to the registry' - type: boolean - default: true + description: 'Push the built image to the registry ("true"/"false")' + type: string + default: 'true' jobs: build: @@ -48,7 +51,7 @@ jobs: . - name: Push builder image - if: ${{ inputs.push }} + if: ${{ inputs.push == 'true' }} run: | docker push gitea.tourolle.paris/dtourolle/kpnpp-builder:latest docker push gitea.tourolle.paris/dtourolle/kpnpp-builder:${{ github.sha }} diff --git a/.gitea/workflows/tsan.yaml b/.gitea/workflows/tsan.yaml new file mode 100644 index 0000000..7911d7d --- /dev/null +++ b/.gitea/workflows/tsan.yaml @@ -0,0 +1,77 @@ +name: '🧡 ThreadSanitizer' + +# Reusable workflow: builds the channel stress suite with ThreadSanitizer and +# runs it. This is the dynamic half of verifying the lock-free SPSC Channel +# (the static half is the CDSChecker model-check harness in verify/). +# +# Triggering and path filtering are owned by ci.yaml (the orchestrator), which +# calls this only when code changed. workflow_dispatch is kept for manual runs. +# +# Runs in the prebuilt builder image (gcc:14), which already ships libtsan β€” no +# package installs at job time. +on: + workflow_call: + workflow_dispatch: + +jobs: + tsan: + runs-on: linux/amd64 + container: + image: gitea.tourolle.paris/dtourolle/kpnpp-builder:latest + # This runner is Docker nested in an unprivileged LXC container, whose + # kernel randomizes mmap addresses beyond the range TSan's fixed shadow + # mapping expects, so TSan aborts at init with "unexpected memory + # mapping". The fix is to disable ASLR per-process with `setarch -R` + # (below), which needs the personality(2) syscall that Docker's default + # seccomp profile blocks. seccomp=unconfined permits it. Verified on the + # runner: setarch -R alone gets EPERM, seccomp alone still aborts, both + # together run clean. Scoped to this job, which runs only our own tests. + options: --security-opt seccomp=unconfined + steps: + - name: Checkout repository + uses: actions/checkout@v4 + with: + path: tsan-${{ github.run_id }} + + - name: Cache FetchContent dependencies + uses: actions/cache@v3 + with: + path: ~/.cmake/fetchcontent + key: cmake-fetchcontent-${{ hashFiles('**/CMakeLists.txt') }} + restore-keys: cmake-fetchcontent- + + - name: Configure (TSan) + working-directory: tsan-${{ github.run_id }} + run: | + cmake -S . -B build \ + -G Ninja \ + -DCMAKE_BUILD_TYPE=Debug \ + -DKPN_SANITIZER=thread \ + -DKPN_BUILD_TESTS=ON \ + -DKPN_BUILD_EXAMPLES=OFF \ + -DKPN_BUILD_PYTHON=OFF \ + -DFETCHCONTENT_BASE_DIR=$HOME/.cmake/fetchcontent + + - name: Build (TSan) + working-directory: tsan-${{ github.run_id }} + run: cmake --build build --parallel --target kpn_tests kpn_tests_stress + + - name: Run stress suite under TSan + working-directory: tsan-${{ github.run_id }} + # halt_on_error=1 makes the first detected race fail the job; the report + # (with both stacks) is printed to the log. second_deadlock_stack gives + # the full picture for lock-order issues. + env: + TSAN_OPTIONS: "halt_on_error=1 second_deadlock_stack=1" + # setarch -R disables ASLR for this process; see the container comment. + run: setarch -R ./build/tests/kpn_tests_stress + + - name: Run unit tests under TSan + working-directory: tsan-${{ github.run_id }} + env: + TSAN_OPTIONS: "halt_on_error=1 second_deadlock_stack=1" + run: setarch -R ./build/tests/kpn_tests + + - name: Cleanup + if: always() + run: rm -rf tsan-${{ github.run_id }} diff --git a/.gitignore b/.gitignore index 910b5d0..3d699c5 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,6 @@ # Build output build/ +build_test/ build_debug/ site/ # Python @@ -24,5 +25,11 @@ venv/ .DS_Store Thumbs.db +# Benchmark output (scripts/bench_repro_check.py --out-dir) +bench_runs/ + # Claude Code local settings .claude/settings.local.json +include/kpn/ort_cache/ +build-tsan/ +build*/ diff --git a/CMakeLists.txt b/CMakeLists.txt index dd147b5..63037fd 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -10,6 +10,30 @@ option(KPN_BUILD_PYTHON "Build Python bindings (requires nanobind)" ON) option(KPN_BUILD_EXAMPLES "Build examples" ON) option(KPN_WEB_DEBUG "Enable web debug UI (cpp-httplib)" OFF) +# Sanitizer build. Empty = off. Accepts "thread", "address", "undefined", +# or a combination like "address,undefined". Applied to all kpn targets via +# the kpn_sanitizer_flags() helper below. +# +# The lock-free SPSC Channel (include/kpn/channel.hpp) has hand-reasoned +# acquire/release ordering; -DKPN_SANITIZER=thread + the channel stress test +# (tests/test_channel_stress.cpp) is the dynamic half of verifying it. The +# static half is the CDSChecker model-check harness (see verify/). +set(KPN_SANITIZER "" CACHE STRING + "Build with sanitizer: thread | address | undefined | (empty = off)") + +# Translate KPN_SANITIZER into compile/link flags. No-op when empty. +function(kpn_sanitizer_flags out_var) + if(KPN_SANITIZER) + set(${out_var} + -fsanitize=${KPN_SANITIZER} + -fno-omit-frame-pointer + -g + PARENT_SCOPE) + else() + set(${out_var} "" PARENT_SCOPE) + endif() +endfunction() + # ── Core library (header-only) ──────────────────────────────────────────────── add_library(kpn INTERFACE) target_include_directories(kpn INTERFACE diff --git a/Dockerfile.builder b/Dockerfile.builder index 7ad29c6..749f5f1 100644 --- a/Dockerfile.builder +++ b/Dockerfile.builder @@ -1,4 +1,4 @@ -# KPN++ Builder Image (CI: pipeline trigger) +# KPN++ Builder Image (CI: pipeline trigger v2) # Pre-built image with GCC, CMake, Ninja, and Python dev headers for building and testing KPN++ # Build: docker build -f Dockerfile.builder -t gitea.tourolle.paris/dtourolle/kpnpp-builder:latest . # Push: docker push gitea.tourolle.paris/dtourolle/kpnpp-builder:latest diff --git a/PERF_PLAN.md b/PERF_PLAN.md new file mode 100644 index 0000000..28560c4 --- /dev/null +++ b/PERF_PLAN.md @@ -0,0 +1,413 @@ +# Performance investigation plan: fanout dispatch cost and deep-chain oversubscription + +**Status:** phase 0 implemented; gate not yet cleared +**Date:** 2026-08-06 (phase 0 landed 2026-08-06) +**Baseline:** master @ 3b67b7e +**Machine:** 20 cores, GCC 16.1.1, TBB 2023.1.0, AC power, `performance` governor +**Data:** 7 full benchmark passes, medians reported below + +--- + +## 1. What was measured + +Throughput, items/sec, median of 7 passes. `N` is the sample size the harness +uses for that row; it is what determines whether a row can be trusted at all. + +### work_us = 10 + +| row | KPN it/s | TBB it/s | TBB faster | N | reliable? | +|---|---|---|---|---|---| +| chain-1 | 89381 | 91350 | +2.2% | 3000 | solid | +| chain-2 | 83759 | 87017 | +3.9% | 1000 | solid | +| chain-4 | 83725 | 85973 | +2.7% | 1000 | solid | +| chain-8 | 79730 | 81090 | +1.7% | 1000 | solid | +| **chain-16** | 53484 | 68745 | **+28.5%** | 200 | weak | +| **chain-32** | 32780 | 45030 | **+37.4%** | 200 | weak | +| **wide-4** | 84906 | 95137 | **+12.0%** | 3000 | solid | +| diamond-4 | 84826 | 86772 | +2.3% | 1000 | solid | + +### work_us = 100 + +Everything except chain-16/32 falls within Β±3.4%, with KPN often ahead +(chain-1 βˆ’0.6%, chain-4 βˆ’1.7%, chain-8 βˆ’3.4%, diamond βˆ’2.6% β€” negative means +KPN faster). chain-16 is +12.5% and chain-32 +18.6%, both at N=50 and +therefore unusable. + +### Two deficits, different causes + +1. **Fanout, +12%.** Solidly measured. `wide-4` performs ~5 node dispatches + per item; the gap works out to a fixed ~250 ns per dispatch, consistent + with `chain-1`'s ~290 ns over a single dispatch. This is dispatch + efficiency. + +2. **Deep chains, +28–37%.** The gap is 1.7–3.9% through depth 8, then jumps + to 28.5% at depth 16 and 37.4% at depth 32. That is a cliff at core count, + not a linear per-dispatch cost. `Node<>` owns a private `ThreadPool(1)` + (`include/kpn/node.hpp:21`), so a depth-32 chain spawns 32 OS threads on + 20 cores. TBB bounds its worker count by hardware concurrency regardless of + graph size. + +### Scope note + +At 100 Β΅s+ per node KPN is at parity or ahead. The repository's own examples +(OpenCV cellshade, frame sources, scene-actor extraction) do milliseconds of +work per node, where a 290 ns dispatch cost is roughly one part in thirty +thousand. Everything in this document matters only for fine-grained pipelines. + +--- + +## 2. Phase 0 β€” the gate that comes first + +**Is there a target workload with sub-30 Β΅s nodes?** + +If no such workload exists or is planned, the correct output of this document +is section 3 (harness) plus a README correction, and nothing else. Optimising +for a benchmark regime the project does not operate in is not worth the risk +described in section 6. + +--- + +## 3. Prerequisite β€” make the harness able to answer + +None of the questions below are decidable with the current harness. +`benchmarks/bench_pipeline.cpp` shrinks the sample count as work per item +grows, so the rows under investigation run 50–200 items and swing 4–8Γ— +run to run. + +| id | change | why | +|---|---|---| +| M1 | `items_for()` β†’ fixed floor, e.g. `max(2000, …)`, independent of work_us and depth | deep rows are currently unmeasurable | +| M2 | report items/sec as the primary metric; keep derived overhead as secondary | overhead is `elapsed βˆ’ work`, a difference of large numbers; it magnifies noise roughly 10Γ— | +| M3 | K in-process repetitions per config; report median and IQR | one shot per config is the root of the present noise | +| M4 | discard a warm-up repetition | first-touch page faults, thread spin-up | +| M5 | extend `pool_sizes[]` to `{1,2,4,8,16,20}` | currently `{1,2,4}` β€” the configuration the README recommends is never run | +| M6 | record nproc, governor and AC state in the CSV header | run-to-run attribution | + +**Acceptance:** the same configuration run 7Γ— lands within Β±5% on every row. +Until that holds, no number below should be acted on. + +This touches only the benchmark, not the library. + +### Status β€” implemented 2026-08-06 + +All of M1–M6 are in `benchmarks/bench_pipeline.cpp`, plus a CLI so the phase-1 +experiments are invocations rather than edits (`--depths`, `--pools`, +`--work`, `--topos`, `--modes`, `--reps`, `--target-sec`, `--min-items`). + +M1 is not a fixed floor but a time budget with a floor: sample size derives +from `work_us Γ— stages / units`, the steady-state throughput bound, then +clamps to `[--min-items, --max-sec]`. A flat 2000-item floor would have made +`chain-32` on a 1-thread pool at 1000 Β΅s a 64-second row; the ceiling keeps +such rows short and reports their true `N` so a short row is visible rather +than silent. The old ladder's error was treating depth as a throughput cost β€” +in a pipeline, depth beyond the core count costs throughput, below it only +latency. + +Also added, ahead of schedule because it is free: `ru_nivcsw` / `ru_nvcsw` per +item are captured around every timed region, so **A3 is now a matter of +reading a column** rather than a separate experiment. + +`scripts/bench_repro_check.py` runs the acceptance criterion directly β€” K +passes, per-row deviation from the median, non-zero exit if any row exceeds +tolerance. + +**Gate not yet cleared.** A 3-pass run of `chain-{1,8}` at 10 Β΅s on the +development laptop (20 cores, **powersave governor, on battery** β€” the header +now records this) lands every row within 0.7%, against the 4–8Γ— swings this +section describes. That is encouraging but is not the acceptance run: it must +be 7 passes over the full row set on the reference machine. + +**Provisional and not to be acted on:** in that same run `chain-16` private +was 6% behind TBB, not the 28.5% in the table above. If that survives the +real acceptance run, the deep-chain deficit is substantially a measurement +artefact of the N=200 rows and workstream A shrinks accordingly. + +--- + +## 4. Workstream A β€” deep chains + +**Hypothesis:** the deficit is thread oversubscription from the private-pool +model, not dispatch cost. + +### Investigation + +| id | experiment | falsifies the hypothesis if | +|---|---|---| +| A1 | sweep depth 8, 12, 16, 20, 24, 32 at 10 Β΅s, private pools | the cliff is not near nproc | +| A2 | repeat A1 under `taskset -c 0-7` | the cliff does **not** move to ~depth 8 | +| A3 | `getrusage(RUSAGE_SELF).ru_nivcsw` per item, depth 8 vs 32 | involuntary context switches do not scale with depth | +| A4 | chain-16/32 on a shared pool sized 16 and 20, vs private and vs TBB | a correctly sized shared pool does not recover the gap | + +A2 is decisive and costs one run: if the cliff tracks the core count, the +mechanism is established. + +### Improvement, conditional on A4 + +If a correctly sized shared pool closes the gap, this is not an optimisation +problem β€” the mechanism already exists and is simply not the default: + +- **A5** β€” change `Network`'s default from per-node private pools to a single + shared pool sized `hardware_concurrency()`. Users should not have to know. +- **A6** β€” emit a diagnostic when total node threads exceed + `hardware_concurrency()`. +- **A7** β€” README: state the threshold, with the measured cliff. + +A5 is a change to the default execution model and must clear section 6 in full. + +### A5 now has a prerequisite (from B1/B2, 2026-08-06) + +The dispatch microbenchmark measured what a shared pool costs per dispatch, +and it is not free: **466 ns on a private `ThreadPool(1)` against ~1.7 Β΅s on a +shared pool of 4**, because round-robin submission wakes a sleeping worker on +every dispatch (see Β§5). A5 as written would therefore make every graph that +currently fits inside its core count roughly 3–4Γ— *worse* per dispatch, in +exchange for fixing graphs that exceed it. + +**A5 must not land before the wake cost does.** The order is B9/B5 first, +then A5, and A4 must be read with this in mind: if a shared pool "recovers the +gap" at depth 32, check what it costs at depth 4 in the same run before +changing any default. + +This partially inverts the prediction in Β§7: workstream A is not purely a +default-and-documentation change, because the default it would switch to is +currently the slower one per dispatch. + +--- + +## 5. Workstream B β€” fanout dispatch cost + +**Hypothesis:** a fixed ~250 ns per node dispatch, paid ~5Γ— per item in +`wide-4`. Unlike workstream A, this genuinely is dispatch efficiency. + +Estimated budget for ~290 ns, per item β€” **estimates, to be replaced by B3**: + +| cost | est. | +|---|---| +| `shared_lock(lifecycle_mx_)` in `submit()` | 20–40 ns | +| `queues_[target]->mx` lock/unlock | 20–40 ns | +| `priority_queue` push + pop (heap ops, `std::function` moves) | 50–100 ns | +| `{ lock_guard lk(cv_mx_); }` + `notify_one()` | 20–40 ns, or Β΅s if a worker actually sleeps | +| 2–3 Γ— `clock_t::now()` in `fire_once` | 50–75 ns | +| gate CAS + ~6 stats atomics | 30–60 ns | + +### Investigation β€” measure before touching anything + +- **B1** β€” microbenchmark submitβ†’execute turnaround for a null task on + `ThreadPool(1)` and `ThreadPool(4)`. Yields ns/dispatch directly, in seconds + rather than minutes. +- **B2** β€” **does a worker actually sleep per item?** Count `cv_.wait` returns, + or `strace -c -f -e futex`. The entire spin-window hypothesis depends on + this; if workers are not sleeping, B5 is worthless and drops off the list. +- **B3** β€” ablation, one variant per suspected cost, each measured against B1 + rather than guessed at: + +| variant | suspected cost | +|---|---| +| stats and clock calls compiled out | 2–3 Γ— `clock_t::now()` plus ~6 atomics per firing | +| `priority_queue` β†’ FIFO ring | heap operations, `std::function` moves | +| `shared_lock(lifecycle_mx_)` removed (**measurement only, unsafe**) | `include/kpn/scheduler.hpp:113` | +| bounded spin before sleeping | `include/kpn/scheduler.hpp:210-227` | + +### B1/B2 β€” first results, 2026-08-06 + +`benchmarks/bench_dispatch.cpp` answers both without touching the library. +Sleeping is inferred from `ru_nvcsw`: a thread blocking on a condition +variable books a voluntary context switch, so voluntary switches per task is +sleeps per task. Three modes, because "the cost of a dispatch" is three +numbers: `latency` (idle pool, one task in flight), `batch` (submit flat out, +drain once), `steady` (the task resubmits its successor, as `fire_once` does). + +Laptop, powersave, battery, 3 reps β€” **the nanoseconds are provisional; the +sleep counts are structural and will hold.** `steady`, 10 Β΅s payload: + +| pool threads | ns/dispatch | sleeps/task | +|---|---|---| +| 1 | 466 | **0.00** | +| 2 | 1494 | 0.97 | +| 4 | 1722 | 1.00 | +| 8 | 1996 | 1.00 | + +**B1 is answered and the abandon criterion is not met.** A `ThreadPool(1)` +dispatch is 291 ns for a null task, 466 ns with a payload β€” against the ~290 ns +the section-1 budget estimated for `chain-1`. The estimate was good. Dispatch +cost is not already under 100 ns, so workstream B stays alive. + +**B2 is answered, and the answer is conditional β€” which the question did not +anticipate.** It is not "do workers sleep?" but "which pool?": + +- On a private `ThreadPool(1)` β€” the `Node<>` default β€” the worker **never** + sleeps. It resubmits into its own queue and finds the work already there. +- On any pool of 2 or more, a worker sleeps **exactly once per task**. + +`submit()` round-robins (`next_.fetch_add(1) % thread_count_`, +`scheduler.hpp:131`), so on a shared pool every task is handed to a *different* +worker, which is asleep, and every single dispatch pays a futex wake. That is +the entire 466 ns β†’ 1.7 Β΅s difference. + +Consequently **B5 (bounded spin) is worthless for the default configuration** +and is the highest-value item for shared pools. It does not drop off the list, +it moves onto a different one. + +### B9 β€” submit-to-self affinity (new, not in the original plan) + +If a `submit()` originating on a pool worker pushed to *that worker's own* +queue instead of round-robining, the shared pool would inherit the property +that makes `ThreadPool(1)` fast: the work is already local when the worker +loops, so no wake. This is roughly what TBB does, and it plausibly subsumes +most of B5 at lower risk β€” it changes task placement, not the sleep/wake +protocol that the August wedge fixes hardened. Work stealing already exists to +correct the resulting imbalance. + +Measure before believing it: an affinity policy can starve peers, and +`try_steal` only rebalances when a peer goes idle. + +### Improvement β€” only what B3 shows pays + +1. **B4 β€” compile-time-optional instrumentation.** No concurrency risk; the + only item here that cannot reintroduce a wedge. Worth doing regardless. +2. **B5 β€” bounded spin before sleeping**, mirroring the channel's existing + `spin_count_` (~4 Β΅s). Note the tension: b9698fa deliberately moved from + "spin whenever any task runs" to "sleep as soon as nothing is queued" in + order to fix pathological spinning. A *bounded* window is the middle + ground; unbounded spin would undo that fix. +3. **B6 β€” cheaper queue on the common path.** A private pool holds ≀1–2 tasks; + `priority_queue` is heavy for that. +4. **B7 β€” batched firing.** `fire_once` processes one token then re-submits; + looping while inputs stay ready, bounded, amortises the submit, gate CAS + and wake. The largest algorithmic win, but it changes latency and + interacts with `compute_priority()`. +5. **B8 β€” `lifecycle_mx_` off the hot path.** Last, and possibly never. It is + load-bearing: it prevents `submit()` racing `stop()`'s `queues_.clear()`, + a documented segfault reproducible "about 12 runs in 20". + +**Abandon criteria:** if B1 shows dispatch cost already under ~100 ns, or the +best surviving variant buys under 5%, stop and document the finding. + +--- + +## 6. Guardrails + +Both workstreams modify the machinery responsible for roughly twenty wedge +fixes in August 2026, plus the lost wake fixed in 6802328. Every change: + +1. **146/146** ctest, examples included. +2. **Wedge soak before and after** β€” `benchmarks/repro_wedge.cpp`, β‰₯50k + iterations clean. Reference point: the pre-6802328 code wedged 5/5 inside + 45 s, at iterations 149, 1249, 332, 1740 and 493. +3. **ThreadSanitizer** on scheduler and pool_node tests for any change to + either. +4. **One change at a time**, measured independently. Bundling is how the + August audit became twenty commits. +5. **G1 β€” wire the reproducer in as an opt-in CTest stress target** + (e.g. `-L soak`) so that performance work cannot silently reintroduce a + wedge. This should land before either workstream starts. + +### G1 β€” implemented 2026-08-06 + +`tests/soak_wedge.cpp` (supersedes `benchmarks/repro_wedge.cpp`, which was +never wired into any build and can be deleted). Always compiled so it cannot +rot; its CTest cases register only under `-DKPN_ENABLE_SOAK_TESTS=ON`, so the +default `ctest` count is unchanged. + +``` +cmake -B build -DKPN_ENABLE_SOAK_TESTS=ON -DKPN_SOAK_ITERS=50000 +cmake --build build --target kpn_soak_wedge +ctest --test-dir build -L soak +``` + +Two cases: `soak.wedge.pool` (depth 4, 4 threads β€” the configuration the +August wedges were reproduced on) and `soak.wedge.private` (depth 8, one pool +per node β€” the model workstream A would change). Both parameterised, so +A5-style changes can be soaked at the depth that matters. + +A wedge is a hang, and a hang under CTest is an unattributable timeout, so the +binary carries a watchdog: if an iteration stops making progress for +`--watchdog-sec` it aborts naming the iteration and the phase (`pushed`, +`drained`, `nodes stopped`, `pool stopped`). Measured cost: ~13 ms per +iteration, so the 50k-iteration guardrail is ~11 minutes. + +**Guardrail 1 needs a correction.** The stated reference is 146/146; the +tests-only configuration used here reports **136/136 passing**, and neither +`examples/` nor `python/` registers any `add_test`. The true reference count +must be pinned down before it is used to certify a change. + +--- + +## 7. Sequencing + +| phase | contents | gate to proceed | state | +|---|---|---|---| +| 0 | workload question; M1–M6; G1 | Β±5% reproducibility achieved | **tooling done**, acceptance run outstanding | +| 1 | A1–A4 | A2 confirms the cliff tracks core count | harness supports it; not run | +| 2 | A5–A7, or documentation only | A4 shows a shared pool recovers the gap | **now gated on B9/B5** | +| 3 | B1–B3 | B2 answers the sleep question | **B1/B2 answered**; B3 outstanding | +| 4 | B4, then whichever of B5–B8 survived B3 | each β‰₯5% and soak-clean | B5 rescoped to shared pools | + +B1/B2 ran early because the microbenchmark cost seconds rather than minutes, +and the result reordered phases 2 and 4 β€” the shared-pool default now depends +on the wake cost being fixed first. Phase 1 is unchanged but its A4 row needs +a shallow-depth control, per Β§4. + +### Reproducing this + +``` +cmake -B build_bench -DKPN_BUILD_BENCHMARKS=ON -DCMAKE_BUILD_TYPE=Release +cmake --build build_bench -j + +# Phase 0 acceptance β€” must pass before any number below is acted on +python3 scripts/bench_repro_check.py ./build_bench/benchmarks/bench_pipeline \ + --passes 7 --tolerance 5 -- --work=10,100 --reps=5 + +# B1/B2 +./build_bench/benchmarks/bench_dispatch --threads=1,2,4,8,20 --reps=5 \ + | tee dispatch.csv + +# A1/A2 β€” the depth sweep, and the same under taskset to move the cliff +./build_bench/benchmarks/bench_pipeline --work=10 --topos=chain \ + --depths=8,12,16,20,24,32 --modes=priv,tbb --reps=5 | tee a1.csv +taskset -c 0-7 ./build_bench/benchmarks/bench_pipeline --work=10 \ + --topos=chain --depths=4,6,8,10,12,16,32 --modes=priv,tbb --reps=5 | tee a2.csv + +# A4 β€” shared pool sized to the machine, against private and TBB. +# Include a shallow depth: A5's risk is what a shared pool costs when the +# graph already fits in its cores. +./build_bench/benchmarks/bench_pipeline --work=10 --topos=chain \ + --depths=4,16,32 --pools=16,20 --reps=5 | tee a4.csv +``` + +Check the `# governor=` line in each CSV before trusting it. A3 needs no +separate run: `ivcsw_per_item` is a column in every row above. + +**Success criteria** + +- chain-32 @10 within 10% of TBB in the recommended configuration +- wide-4 @10 within 5% of TBB +- zero wedges across 100k soak iterations + +**Prediction, recorded so it can be proven wrong:** workstream A resolves into +a default-and-documentation change rather than an optimisation, and workstream +B yields 5–10% on fanout from B4 and B5, with the remainder not worth the risk. + +**Prediction, revised 2026-08-06 after B1/B2** β€” the original is already half +wrong and is left above unedited: + +- Workstream A does *not* resolve into a documentation change, because the + shared pool it would recommend costs 3–4Γ— more per dispatch than the private + default. It resolves into B9 first. +- The largest single win is not B4, B5 or B7 but **B9, submit-to-self + affinity**: one sleep per dispatch is being paid on every shared pool, and + eliminating it is worth roughly 1.2 Β΅s per dispatch β€” far more than the + 5–10% predicted for fanout. +- Standing: `chain-16`'s 28.5% deficit is a measurement artefact of N=200. + +--- + +## 8. Related correction + +Independently of the above, the README's TBB comparison overstates its case. +The claim that KPN++ beats TBB "for every chain and diamond topology at +100 Β΅s/node" is not supported: at 100 Β΅s only chain-1 and diamond lean KPN, +while chain-16, chain-32 and wide-4 lean TBB. The tables are also quoted in +derived overhead, which magnifies small differences β€” the same rows expressed +as throughput are mostly within a few percent. Restating them in items/sec +would be both more accurate and more favourable. diff --git a/README.md b/README.md index 1d7d1e1..727be44 100644 --- a/README.md +++ b/README.md @@ -2,6 +2,8 @@ A C++20 Kahn Process Network (KPN) library. Each node wraps a function and runs in its own thread, communicating with downstream nodes via bounded FIFO channels. Includes Python bindings via nanobind. +πŸ“– **[Documentation](https://pages.tourolle.paris/dtourolle/kpn/)** + --- ## Requirements @@ -398,8 +400,8 @@ Violating the second rule deadlocks. | `04_storage_policy` | `channel_storage_policy` default and specialisation | | `05_error_handling` | `ChannelOverflowError`, `ErrorHandler` | | `06_watchdog` | Watchdog interval, stall detection | -| `07_python_network` | PyNetwork, pure Python node *(pending)* | -| `08_python_subport` | `net.read`, `net.write`, sub-port tap *(pending)* | +| `07_python_network` | PyNetwork with a pure-Python node between a C++ source and sink | +| `08_python_subport` | Drive a Python node from Python via `net.write`/`net.read` sub-port taps | | `09_opencv_cellshade` | Real-time cell-shading on webcam/pattern; requires OpenCV β‰₯ 4 | Run the cell-shading example: diff --git a/README.md.in b/README.md.in index cd821d4..4d7622b 100644 --- a/README.md.in +++ b/README.md.in @@ -2,6 +2,8 @@ A C++20 Kahn Process Network (KPN) library. Each node wraps a function and runs in its own thread, communicating with downstream nodes via bounded FIFO channels. Includes Python bindings via nanobind. +πŸ“– **[Documentation](https://pages.tourolle.paris/dtourolle/kpn/)** + --- ## Requirements @@ -220,8 +222,8 @@ Violating the second rule deadlocks. | `04_storage_policy` | `channel_storage_policy` default and specialisation | | `05_error_handling` | `ChannelOverflowError`, `ErrorHandler` | | `06_watchdog` | Watchdog interval, stall detection | -| `07_python_network` | PyNetwork, pure Python node *(pending)* | -| `08_python_subport` | `net.read`, `net.write`, sub-port tap *(pending)* | +| `07_python_network` | PyNetwork with a pure-Python node between a C++ source and sink | +| `08_python_subport` | Drive a Python node from Python via `net.write`/`net.read` sub-port taps | | `09_opencv_cellshade` | Real-time cell-shading on webcam/pattern; requires OpenCV β‰₯ 4 | Run the cell-shading example: diff --git a/SPEC.md b/SPEC.md index 5e422ef..8a7d048 100644 --- a/SPEC.md +++ b/SPEC.md @@ -2,9 +2,24 @@ ## Overview -A C++20 template-metaprogramming library for building Kahn Process Networks, where each node -wraps a function/method, runs in its own thread, and communicates via bounded FIFO queues. -Includes nanobind bindings for Python graph construction and prototyping. +A header-only C++20 template-metaprogramming library for building Kahn Process Networks. Each +node wraps a function (or callable object); its input types are inferred from the parameter +list and its output types from the return type. Nodes communicate over bounded, lock-free +SPSC FIFO channels. + +Unlike a naive "one blocking thread per node" model, KPN++ is **reactive**: a node is +scheduled onto a thread pool whenever all of its input channels have data. A node that wraps a +function with `Node<>` owns a private single-thread pool and behaves exactly like an +independent worker; multiple nodes can instead share one `ThreadPool` for bounded-thread +execution. Source nodes self-resubmit; event-driven sources (`InterruptNode`) fire on an +external trigger. + +The library ships rich runtime diagnostics (per-node exec/CPU/throughput stats, per-channel +fill/bandwidth/overflow counters, pool and shared-resource utilisation), an optional in-process +web debug UI, and nanobind-based Python bindings (partially implemented). + +> **Note on accuracy.** This document describes the code as it exists in `include/kpn/`. Where +> a behaviour is subtle the relevant header is named so the source remains the ground truth. --- @@ -14,39 +29,50 @@ Includes nanobind bindings for Python graph construction and prototyping. kpn++/ β”œβ”€β”€ CMakeLists.txt β”œβ”€β”€ include/kpn/ -β”‚ β”œβ”€β”€ fixed_string.hpp # NTTP string type for named ports -β”‚ β”œβ”€β”€ traits.hpp # Function signature introspection -β”‚ β”œβ”€β”€ channel.hpp # Bounded FIFO channel + storage policy -β”‚ β”œβ”€β”€ node.hpp # Node wrapper + thread management -β”‚ β”œβ”€β”€ port.hpp # Input/Output port handles -β”‚ β”œβ”€β”€ network.hpp # Graph builder + orchestrator/watchdog -β”‚ β”œβ”€β”€ variant_node.hpp # Runtime-typed node for Python graphs -β”‚ └── python/ -β”‚ └── bindings.hpp # Nanobind binding helpers -β”œβ”€β”€ src/ -β”‚ └── network.cpp # Orchestrator thread impl -β”œβ”€β”€ tests/ -β”œβ”€β”€ examples/ -β”‚ β”œβ”€β”€ 01_hello_pipeline/ -β”‚ β”œβ”€β”€ 02_named_ports/ -β”‚ β”œβ”€β”€ 03_multi_output/ -β”‚ β”œβ”€β”€ 04_storage_policy/ -β”‚ β”œβ”€β”€ 05_error_handling/ -β”‚ β”œβ”€β”€ 06_watchdog/ -β”‚ β”œβ”€β”€ 07_python_network/ -β”‚ β”œβ”€β”€ 08_python_subport/ -β”‚ └── 09_opencv_cellshade/ # optional, requires OpenCV -└── python/ - └── kpn_python.cpp # Nanobind module definition +β”‚ β”œβ”€β”€ fixed_string.hpp # NTTP string + in<>/out<> tags + index_of +β”‚ β”œβ”€β”€ traits.hpp # function signature introspection, normalised_return_t, repeat_tuple +β”‚ β”œβ”€β”€ diagnostics.hpp # NodeStats, ChannelStats, *Snapshot, IPoolProbe, IResourceProbe +β”‚ β”œβ”€β”€ channel.hpp # lock-free SPSC ring-buffer Channel + storage policy +β”‚ β”œβ”€β”€ port.hpp # InputPort / OutputPort handles +β”‚ β”œβ”€β”€ inode.hpp # INode interface, NodeErrorHandler, NodeEvent +β”‚ β”œβ”€β”€ scheduler.hpp # IScheduler + work-stealing ThreadPool +β”‚ β”œβ”€β”€ pool_node.hpp # PoolNode / PoolObjectNode (reactive, scheduler-driven) +β”‚ β”œβ”€β”€ interrupt_node.hpp # InterruptNode (external-trigger source) +β”‚ β”œβ”€β”€ node.hpp # Node / ObjectNode (PoolNode + private 1-thread pool) + make_node +β”‚ β”œβ”€β”€ fanout.hpp # FanoutNode + make_fanout +β”‚ β”œβ”€β”€ branch.hpp # RouterNode + FilterNode + make_router / make_filter +β”‚ β”œβ”€β”€ shared_resource.hpp # SharedResource priority-arbitrated exclusive resource +β”‚ β”œβ”€β”€ main_thread_node.hpp # MainThreadNode<> (GUI / main-thread-bound nodes) +β”‚ β”œβ”€β”€ static_network.hpp # Edge<>, make_network(), StaticNetwork<> +β”‚ β”œβ”€β”€ network.hpp # runtime Network builder + watchdog + diagnostics +β”‚ β”œβ”€β”€ debug_hub.hpp # DebugHub multi-network web UI (KPN_WEB_DEBUG only) +β”‚ β”œβ”€β”€ web_debug.hpp # single-network web debug server (KPN_WEB_DEBUG only) +β”‚ β”œβ”€β”€ variant_node.hpp # runtime-typed nodes/channels for Python graphs +β”‚ β”œβ”€β”€ tmp/ +β”‚ β”‚ β”œβ”€β”€ fanout_groups.hpp # compile-time fan-out detection + edge expansion +β”‚ β”‚ β”œβ”€β”€ topo_sort.hpp # compile-time DFS cycle check + topological order +β”‚ β”‚ └── repeat_tuple.hpp # repeat_tuple_t +β”‚ β”œβ”€β”€ python/ +β”‚ β”‚ β”œβ”€β”€ bindings.hpp # PyNetwork / PyNode nanobind helpers +β”‚ β”‚ └── auto_bind.hpp # NodeRegistry / Entry / bind_network / bind_debug +β”‚ └── kpn.hpp # umbrella header +β”œβ”€β”€ src/network.cpp +β”œβ”€β”€ tests/ # Catch2 v3 + GoogleTest +β”œβ”€β”€ examples/ # 01–16 (see Examples) +β”œβ”€β”€ benchmarks/ # bench_pipeline (optional, KPN_BUILD_BENCHMARKS) +└── python/kpn_python.cpp # nanobind module definition ``` +`kpn.hpp` is the umbrella header; including it pulls in the full C++ API (the Python layer is +included only by the binding TU). + --- -## Component 0 β€” `fixed_string.hpp`: NTTP String +## Component 0 β€” `fixed_string.hpp`: NTTP String + Port Tags Named ports use C++20 non-type template parameters (NTTPs). `std::string_view` and -`const char*` are not valid NTTPs because they are not structurally comparable. The standard -solution is a `fixed_string` literal type with `constexpr` internal storage. +`const char*` are not valid NTTPs, so a `fixed_string` literal type provides `constexpr` +internal storage. ```cpp template @@ -57,78 +83,105 @@ struct fixed_string { constexpr std::string_view view() const { return {data, N - 1}; } }; -// Deduction guide β€” required so fixed_string("img") works as an NTTP. -// Without it the compiler cannot infer N and the named-port API does not compile. template -fixed_string(const char (&)[N]) -> fixed_string; +fixed_string(const char (&)[N]) -> fixed_string; // deduction guide (required) ``` -`fixed_string<3>` and `fixed_string<6>` are distinct types, so `input<"img">()` and -`input<"sigma">()` produce different template instantiations β€” this is intentional and -enables zero-overhead compile-time port dispatch. +`fixed_string<4>` and `fixed_string<7>` are distinct types, so `input<"img">()` and +`input<"sigma">()` produce different instantiations β€” enabling zero-overhead compile-time +port dispatch. -Named-port lookup uses a `constexpr` function over the name pack. It returns a sentinel -`npos` on miss rather than `static_assert`-ing internally, so the assertion fires at the -`input<"img">()` call site β€” giving the user a readable error at the point of use instead -of deep in template instantiation: +Named-port lookup uses a `constexpr` `index_of` over the name pack; it returns the sentinel +`npos` on a miss so the `static_assert` fires at the `input<"img">()` **call site**, giving a +readable error at the point of use: ```cpp inline constexpr std::size_t npos = std::size_t(-1); template -constexpr std::size_t index_of() { - std::size_t i = 0; - bool found = false; - ((Name == Names ? (found = true) : (found ? 0 : ++i)), ...); - return found ? i : npos; -} - -// Used at the call site: -template -auto input() { - constexpr std::size_t idx = index_of(); - static_assert(idx != npos, "unknown input port name"); - return input(); -} +constexpr std::size_t index_of(); // returns position or npos ``` +### Port tags + +`in<...>` and `out<...>` tag types disambiguate input vs. output name packs in the factory +API. Both are trivial empty structs; both are optional (omit to get index-only ports). + +```cpp +template struct in {}; +template struct out {}; +``` + +> There is **no `latch<>` tag.** An earlier design sketched latched (most-recent-value) +> input ports; this was not implemented and the only input kind is the synchronous one. + --- ## Component 1 β€” `traits.hpp`: Function Introspection -Extracts parameter types and return type from any callable at compile time. +Extracts parameter and return types from any callable at compile time, for free functions, +function pointers, member function pointers (const and non-const), lambdas and `std::function`. ```cpp -// For: Image blur(Image in, float sigma) -// function_traits::args == std::tuple -// function_traits::return_t == Image - -// For multi-output: std::tuple detect(Image in) -// return_t == std::tuple β†’ 2 output ports -// return_t == Image β†’ 1 output port (normalised to tuple internally) -// return_t == void β†’ 0 output ports (sink node) +// function_traits::return_t, ::args (std::tuple<...>), ::arity +template using return_t = ...; // return type +template using args_t = ...; // std::tuple of parameters +template inline constexpr std::size_t arity_v = ...; ``` -Handles: free functions, lambdas, `std::function`, member function pointers. - -A helper alias normalises the return type to always be a tuple for uniform handling in -`run_loop`: +The return type is normalised to a tuple so every node has a uniform output-tuple shape: ```cpp -template -using normalised_return_t = - std::conditional_t, T, std::tuple>; -// void return β†’ std::tuple<> (empty tuple, zero output ports) +// void β†’ std::tuple<> (sink node, 0 outputs) +// T (non-tup) β†’ std::tuple (1 output) +// tuple<...> β†’ tuple<...> (one output port per element) +template using normalised_return_t = ...; +template inline constexpr std::size_t output_count_v = ...; +``` + +`repeat_tuple_t` (also surfaced via `tmp/repeat_tuple.hpp`) builds `std::tuple` +with `N` repetitions β€” used by `FanoutNode` and `RouterNode` to describe their N identical +output ports. + +--- + +## Component 2 β€” `diagnostics.hpp`: Statistics and Snapshots + +Shared timing types: `clock_t = std::chrono::steady_clock`, `duration_t` is a +`double`-millisecond duration. + +- **`NodeStats`** β€” atomic counters updated per fire: `frames_processed`, an EMA of wall-clock + exec time (`ema_exec_us`, warmup-mean for the first 5 frames then Ξ±=0.1), `max_exec_us`, + `total_blocked_us`, thread CPU time (`total_cpu_us` via `CLOCK_THREAD_CPUTIME_ID`), + `queue_wait_us` (pool queue latency), and `exec_start_us` (non-zero while executing; used by + the watchdog to detect hung nodes). +- **`ChannelStats`** β€” `pushes`, `bytes_pushed`, `drops`, `overflows`, `pops`, `peak_fill`. +- **Snapshots** β€” copyable plain structs taken by the watchdog / UI: `NodeSnapshot`, + `ChannelSnapshot` (with `fill_pct()`, `peak_pct()`, `bandwidth_mbs()`), `PoolSnapshot`, + `ResourceSnapshot`, and `NetworkSnapshot` (used by the `DebugHub`). +- **Probe interfaces** β€” `IPoolProbe` and `IResourceProbe` expose a `snapshot(name)` method so + pools and shared resources can be registered with a network for reporting. + +### `ChannelDataSize` trait + +`bytes_pushed` is computed from a specialisable trait, defaulting to `sizeof(T)`. Specialise it +for heap-owning payloads to get accurate bandwidth: + +```cpp +template<> struct kpn::ChannelDataSize { + static std::size_t bytes(const cv::Mat& m) { return m.total() * m.elemSize(); } +}; ``` --- -## Component 2 β€” `channel.hpp`: Bounded FIFO + Storage Policy +## Component 3 β€” `channel.hpp`: Lock-free Bounded FIFO + Storage Policy -### Storage Policy +### Storage policy -The type stored in a channel depends on a specialisable trait. Users can override it for -any type: +The type stored inside a channel is chosen by a specialisable trait. Small trivially-copyable +types are stored by value; everything else as `std::shared_ptr` so fan-out copies a +refcount, not data: ```cpp template @@ -137,29 +190,20 @@ struct channel_storage_policy { std::is_trivially_copyable_v && sizeof(T) <= 8; }; -// User opt-in to value semantics for a small struct: -template<> struct channel_storage_policy { - static constexpr bool by_value = true; -}; - -// Derived storage type: template using channel_storage_t = std::conditional_t< - channel_storage_policy::by_value, - T, - std::shared_ptr ->; + channel_storage_policy::by_value, T, std::shared_ptr>; ``` -### Channel +Override it to force value semantics for a custom small type. Push wraps a value in +`make_shared` when needed; pop dereferences it transparently, so a function taking +`const T&` works naturally and immutability is compiler-enforced. -`Channel` stores `channel_storage_t` internally. The producer calls `push(T value)` -and the channel transparently wraps it in `make_shared` when needed. All consumers -of the same channel receive the same `shared_ptr` β€” no copies of large objects. +### Channel β€” SPSC ring buffer -`run_loop` dereferences `shared_ptr` before passing to the wrapped function, so a -function declared `void f(const Image& img)` works naturally and the compiler enforces -immutability β€” no policy enforcement or `const_cast` needed. +`Channel` is a single-producer/single-consumer ring buffer (capacity rounded up to a power +of two). It uses C++20 `std::atomic::wait/notify_one` (portable futex) with a configurable +**spin-before-sleep** window so the common case never touches the kernel. ```cpp template @@ -167,590 +211,446 @@ class Channel { public: using storage_type = channel_storage_t; - explicit Channel(std::size_t capacity = 5); + explicit Channel(std::size_t capacity = 5, std::size_t spin_count = 200); - void push(T value); // wraps in shared_ptr if needed; throws on overflow - T pop(); // blocks (KPN semantics); unwraps shared_ptr if needed - bool try_pop(T& out, std::chrono::milliseconds timeout); + void push(T value); // drops if disabled; throws ChannelOverflowError if full + bool push_sentinel(T value); // out-of-band, non-blocking must-deliver token (EOF) + T pop(); // blocks (spin then futex); throws ChannelClosedError if disabled+empty + bool try_pop(T& out, std::chrono::milliseconds timeout); // polling (watchdog/display) + bool try_pop_now(T& out); // immediate, non-blocking - std::size_t size() const; + void enable(); // accept pushes + void disable(); // stop accepting + unblock any waiting pop() + void set_push_callback(std::function); // emptyβ†’non-empty notification + + std::size_t size() const; // ring occupancy (excludes any pending sentinel) + std::size_t approx_size() const; // size() + 1 if a sentinel is pending (readiness checks) std::size_t capacity() const; + bool is_accepting() const; + const ChannelStats& stats() const; + ChannelSnapshot snapshot(const std::string& name) const; }; -class ChannelOverflowError : public std::runtime_error {}; +class ChannelOverflowError : public std::runtime_error { /* capacity + optional context */ }; +class ChannelClosedError : public std::runtime_error {}; ``` +`head_` and `tail_`/`wake_` live on separate cache lines (`alignas(64)`) to avoid false +sharing between producer and consumer. `spin_hint()` issues a `pause`/`yield` instruction (or a +compiler fence on other ISAs). + +### The `push_callback` β€” how reactivity works + +`set_push_callback` registers a callback fired when a channel transitions emptyβ†’non-empty. A +consuming `PoolNode` installs this on each of its input channels; when an input becomes ready it +re-evaluates whether **all** inputs have data and, if so, submits itself to the scheduler. This +is the mechanism that replaces a dedicated blocking thread per node. + +### The out-of-band EOF sentinel β€” `push_sentinel` + +`push_sentinel(T value)` delivers a **must-deliver control token** (a graceful-EOF marker) that +cannot be dropped by backpressure. The value is stored in a dedicated slot **outside** the ring, +so it consumes no capacity, never throws `ChannelOverflowError`, and never blocks the producer. + +This matters because a node's worker cannot afford to block on a downstream push: parking that +thread would stop it draining its own input, cascading into a hold-and-wait deadlock under +backpressure. `push_sentinel` sets a published flag (`has_eof_`) and returns immediately, keeping +the worker free to keep popping. + +Ordering is preserved: the consumer's `pop()` / `try_pop_now()` drain the ring **first** and only +surface the sentinel once the ring is observed empty β€” so EOF always arrives after every value +pushed before it. `approx_size()` (used by node readiness checks) counts a pending sentinel as one +consumable item, so a channel carrying *only* a sentinel still schedules its consumer's next fire +and the token is never stranded. Same SPSC contract as `push()` (sole producer); returns `false` +if the channel is already disabled (teardown in progress β†’ the token is moot). + +### Backpressure and shutdown β€” `accepting_` flag + +Each channel carries `std::atomic accepting_` (default `true`). It is the primary shutdown +mechanism; the only additional signal is the out-of-band EOF sentinel above, used for *graceful* +drain rather than an abrupt close. + +- **`push()`** on a disabled channel silently drops the value (recorded as a `drop`). On a + full accepting channel it throws `ChannelOverflowError` (a sizing error). +- **`pop()`** blocks while empty and accepting; `disable()` wakes it and it throws + `ChannelClosedError`. + +The **consumer node** owns its input channels and flips the flag: `start()` calls `enable()`, +`stop()` calls `disable()`. Producers never touch it. + ### Ownership -A `Channel` is **owned by its consumer node** β€” it lives as a member of the destination -node. The producer node holds a non-owning raw pointer to push into it. The channel is -destroyed when its consumer is destroyed, which is the correct lifetime. +Input channels are owned by their **consumer node** (held as `shared_ptr>`). A +producer node holds a non-owning raw `Channel*` to push into. `Network`/`StaticNetwork` are +otherwise non-owning of user nodes β€” see Components 8–9. -The `Network` itself is **non-owning** β€” nodes are declared by the user and outlive the -network. `net.add("name", node)` registers a raw pointer; the user is responsible for keeping -nodes alive for the network's lifetime. This avoids type-erasure ownership complexity and -keeps node construction explicit. +--- -### Backpressure and Shutdown β€” `accepting_` Flag +## Component 4 β€” `inode.hpp`: The Node Interface -Each channel carries a single `std::atomic accepting_` (default `true`). This is the -**sole shutdown mechanism** β€” no `try_pop` polling, no sentinel values, no drain logic. +Every node implements `INode`: ```cpp -template -class Channel { - std::atomic accepting_{true}; -public: - void push(T value) { - if (!accepting_.load(std::memory_order_relaxed)) return; // silently drop - // normal push β€” throws ChannelOverflowError if full - } +struct INode { + virtual ~INode() = default; + virtual void start() = 0; + virtual void stop() = 0; + virtual bool running() const = 0; + virtual const NodeStats& stats() const = 0; + virtual NodeSnapshot node_snapshot(const std::string& name, double elapsed_s) const = 0; + virtual void set_name(std::string) = 0; - void enable() { accepting_.store(true, std::memory_order_relaxed); } - void disable() { - accepting_.store(false, std::memory_order_relaxed); - clear(); // drop all queued items immediately - cv_.notify_all(); // unblock any waiting pop() - } + virtual void set_network_overflow_callback(NodeEventCallback) {} // network-injected + virtual void set_network_closed_callback(NodeEventCallback) {} + + virtual void halt() { stop(); } // immediate, discard in-flight work + virtual void shutdown() { stop(); } // graceful topo-ordered drain (overridden by networks) }; ``` -**Who flips the flag:** the **consumer node** β€” it's the channel owner. `node.stop()` calls -`disable()` on all its input channels. `node.start()` calls `enable()`. The producer never -touches the flag; it calls `push()` and if the channel is disabled the value is silently -dropped and the producer continues. +Supporting types: -**Overflow** (`push()` on a full, accepting channel) still throws `ChannelOverflowError` β€” -this signals a design error (undersized FIFO) and is unchanged. +```cpp +// Per-node error policy: return true to skip the failed fire and keep running, +// false to stop the node (and signal closed downstream). +using NodeErrorHandler = std::function; -**Blocking `pop()`** unblocks immediately when `disable()` is called (via `cv_.notify_all()`), -and throws `ChannelClosedError` if the queue is empty and the channel is disabled. - -### `try_pop` Purpose - -`try_pop` exists for **watchdog polling only** β€” not for shutdown (the `accepting_` flag -handles that) and not for normal processing. The watchdog uses it to probe whether a node -is making progress without blocking the watchdog thread. - -> **Note (C++ compile-time graphs):** In a fully compiled C++ graph the variant never -> appears. The compiler wires `Channel` to `Channel` directly. The variant is -> a pure compile-time construct used only for type-checking and generates zero runtime -> overhead. +using NodeEventCallback = std::function; +enum class NodeEvent { Overflow, Closed }; +``` --- -## Component 3 β€” `port.hpp`: Port Handles +## Component 5 β€” `scheduler.hpp`: Thread Pool ```cpp -template -struct InputPort { NodeT& node; }; - -template -struct OutputPort { NodeT& node; }; -``` - -Nodes expose port handles via: - -```cpp -// By index β€” always available -node_a.input<0>() // returns InputPort -node_b.output<1>() // returns OutputPort - -// By name β€” only valid when names were provided at make_node time -node_a.input<"img">() -node_b.output<"edges">() -``` - -Named access resolves to an index at compile time via `index_of` (see `fixed_string.hpp`). -Zero runtime cost β€” the name dispatch is fully eliminated by the compiler. - ---- - -## Component 4 β€” `node.hpp`: Node Wrapper - -```cpp -template< - auto Func, - fixed_string... InputNames, // optional; count must match arity or be 0 - fixed_string... OutputNames // optional; count must match output count or be 0 -> -class Node { -public: - explicit Node(std::size_t fifo_capacity = 5); - - void start(); - void stop(); // signals thread to finish current item then exit - - // Port access β€” by index - template auto input(); - template auto output(); - - // Port access β€” by name (compile error if names were not provided) - template auto input(); - template auto output(); - - static constexpr std::size_t input_count; - static constexpr std::size_t output_count; - -private: - void run_loop(); - // Pops each input channel, dereferences shared_ptr if needed, - // calls Func, unpacks the normalised tuple return, - // pushes each element to its output channel. - - std::thread thread_; - // Input channels owned here (one per input port). - // Output channel pointers (non-owning) set at connect time. +struct IScheduler { + virtual void submit(std::function task, float priority = 0.5f) = 0; + virtual void start() = 0; + virtual void stop() = 0; // join workers, discard pending tasks + virtual void drain() = 0; // block until in-flight tasks complete (workers keep running) }; ``` -**Factory syntax β€” `in<>` / `latch<>` / `out<>` tag structs:** - -A flat name pack `make_node` is ambiguous (where do inputs end?). -Option chosen: `in<...>`, `latch<...>`, and `out<...>` tag types that wrap the name packs unambiguously. -All are optional; omitting either means those ports are index-only. - -```cpp -// Tag types (trivial, no data): -template struct in {}; -template struct latch {}; -template struct out {}; - -// Factory: -// No names -auto node = make_node(/*fifo_capacity=*/10); - -// Input names only -auto node = make_node>(10); - -// Both input and output names -auto node = make_node, out<"blurred","mask">>(10); - -// Mixed synchronous and latched inputs -auto node = make_node, latch<"setpoint">, out<"output">>(10); -``` - -**Wrong name count is a compile error.** The `Node` class `static_assert`s that -`sizeof...(InputNames) == 0 || sizeof...(InputNames) == input_count` (and same for outputs). -Without this, a mismatch between name count and arity produces an unreadable template error. - -```cpp -static_assert( - sizeof...(InputNames) == 0 || sizeof...(InputNames) == input_count, - "make_node: number of input names must match function arity, or provide none" -); -``` - -Multi-output functions must return `std::tuple<...>`. Single return accepted as-is. -`void` return = sink node (no output ports). - -### Node identity: `Label` and `UniqueTag` NTTPs - -Two problems arise when using `Node` as a compile-time graph vertex: - -1. **Debug names** β€” without a label the web UI and `print_diagnostics` fall back to - `"node[0]"` placeholders. `set_name()` provides a runtime name for the runtime - `Network`, but `StaticNetwork` has no `add("name", node)` call to attach one. - -2. **Same-function collision** β€” two nodes wrapping the same function (e.g. two - `make_node`) have the same type. The `StaticNetwork` topo sort and fanout - detection use node types as graph vertices, so it cannot distinguish them, causing - infinite recursion in the DFS. - -Both are solved by two additional NTTPs on `Node`: - -```cpp -template< - auto Func, - typename InputTag = in<>, - typename LatchTag = latch<>, - typename OutputTag = out<>, - fixed_string Label = "", // human-readable name; shown in diagnostics/web UI - std::size_t UniqueTag = 0 // collision-breaker; must differ between any two -> // same-Func nodes in one make_network() call -class Node : public INode { ... }; -``` - -Both have defaults so all existing code (`make_node(5)`) compiles unchanged. - -**Factory syntax extension:** - -```cpp -// Label only (UniqueTag defaults to 0) -auto blur = make_node(8); - -// Both label and unique tag β€” required when the same function is used twice -auto preblur = make_node(8); -auto postblur = make_node(8); -``` - -**Duplicate-tag detection** β€” `make_network()` checks at compile time that no two nodes -in the edge pack share the same `(Func, UniqueTag)` pair. This fires a readable -`static_assert` at the call site before any thread is started: - -``` -static_assert(!has_duplicate_tags_v, - "make_network: two nodes have identical (Func, UniqueTag) β€” increment UniqueTag " - "on one of them to make them distinct"); -``` - -**Label availability** β€” `Node` exposes the label as a `static constexpr` so -`StaticNetwork` can read it at compile time for diagnostics: - -```cpp -static constexpr std::string_view label() { return Label.view(); } -``` - -The web debug UI and `print_diagnostics` use `label()` when non-empty, falling back to -`"node[]"` for unlabelled nodes. The runtime `Network` continues to use the -string passed to `add("name", node)` β€” `Label` is orthogonal to that mechanism. - -**`ObjectNode`** gains the same two NTTPs with the same defaults. The `make_node(obj, ...)` -overloads are extended identically. +`ThreadPool` is a **work-stealing** pool implementing both `IScheduler` and `IPoolProbe`. Each +worker owns a priority queue (max-heap by `priority`, FIFO within equal priority via a sequence +counter). `submit()` distributes round-robin; idle workers steal from the most-loaded peer +using `try_lock`, then sleep on a shared condition variable. The submit/notify path takes the CV +mutex around `notify` to close the lost-wakeup window; `drain()` waits on a separate counter of +in-flight tasks. `priority` lets a hot node (full input, empty output) be scheduled ahead of +others β€” see `PoolNode::compute_priority`. --- -## Component 4a β€” Latched Input Ports +## Component 6 β€” Node Types -### Motivation +All processing nodes share the same shape: typed input channels they own, raw output-channel +pointers set at wiring time, `args_tuple` / `return_tuple` aliases used by the connect-time +type check, and `static constexpr` `label()` / `unique_tag` / `input_count` / `output_count`. -Control and robotics applications naturally have two kinds of inputs at different update rates: +### `PoolNode` / `PoolObjectNode` β€” reactive, scheduler-driven (`pool_node.hpp`) -- **Synchronous inputs** (`in<>`) β€” the node must have fresh data on every fire. Typical for sensor readings that drive the computation (e.g. encoder RPM). -- **Latched inputs** (`latch<>`) β€” the node uses the most recently received value, and does not block if no new value has arrived. Typical for setpoints or parameters that change infrequently relative to the control loop (e.g. bearing from a CV pipeline, PID gains). - -Without latched ports, a node must block on all inputs simultaneously. This forces the control loop to run at the rate of the slowest input β€” unacceptable when a 1kHz encoder loop must wait for a 30Hz vision update. - -### Semantics - -A `latch<>` port: - -1. **Does not block** if its channel is empty β€” it reuses the last successfully popped value. -2. **Does block on first fire** β€” there is no meaningful "default" value, so the node waits until at least one value has arrived on each latched port before firing for the first time. -3. **Consumes the value** when one is available (standard `pop()`), then holds it until the next value arrives. - -The node fires whenever all `in<>` ports have data, using the last known value for each `latch<>` port. - -### Implementation in `run_loop` - -`run_loop` maintains a `std::tuple` of cached values, one slot per latched port. On each iteration: +The core node. Instead of a blocked thread, it submits a `fire_once()` to a shared +`IScheduler` whenever all inputs are ready; `queued_` ensures at most one `fire_once()` is +in flight. `fire_once()` pops every input (`try_pop_now`), runs the function, pushes each +normalised output, records stats, then resubmits if inputs remain ready. Source nodes +(`input_count == 0`) self-submit on `start()` and after each fire. ```cpp -// Synchronous ports β€” blocking pop (existing behaviour) -auto sync_args = std::make_tuple(input<0>().pop(), input<1>().pop(), ...); +template, + typename OutputTag = out<>, + fixed_string Label = "", + std::size_t UniqueTag = 0> +class PoolNode : public INode { ... }; -// Latched ports β€” non-blocking try_pop; keep cached value on miss -try_pop(latch_cache_, latch_channel_); // updates cache if data available - -// Call wrapped function with merged argument tuple -auto result = std::apply(Func, merge(sync_args, latch_cache_)); +auto n = make_pool_node(scheduler, fifo_capacity); // index ports +auto n = make_pool_node(scheduler, in<"a">{}, out<"b">{}, cap); ``` -Latched channels are otherwise identical to synchronous channels: bounded FIFO, `shared_ptr` storage policy, same shutdown behaviour. +`PoolObjectNode` is the same for a stateful callable object (introspected via +`&Obj::operator()`); the object must outlive the node. -### Example β€” PID with live setpoint +Per-node configuration: `set_error_handler(NodeErrorHandler)`, `set_overflow_callback`, +`set_closed_callback`, `set_max_exec_time`. Inside `fire_once()`: +`ChannelOverflowError` fires the overflow callbacks; `ChannelClosedError` (or an error handler +returning `false`) fires the closed callbacks and self-stops; any other exception consults the +error handler. + +**Name-count contract** β€” a `static_assert` requires that the number of input names is `0` or +equals arity (same for outputs): ```cpp -// bearing arrives at ~30 Hz from CV; rpm arrives at ~1 kHz from encoder -double pid_compute(double rpm, double bearing) { ... } - -auto pid = make_node, // synchronous β€” blocks until fresh encoder tick - latch<"bearing">, // latched β€” uses last known bearing from CV - out<"pwm"> ->(8); - -Network net; -net.add("tacho", tacho_node) - .add("tracker", tracker_node) - .add("pid", pid) - .connect("tacho", tacho_node.output<"rpm">(), "pid", pid.input<"rpm">()) - .connect("tracker", tracker_node.output<"bearing">(),"pid", pid.input<"bearing">()) - .build(); +static_assert(sizeof...(InNames) == 0 || sizeof...(InNames) == input_count, + "make_pool_node: number of input names must match function arity, or provide none"); ``` -The PID node fires at encoder rate. If no new bearing has arrived since the last tick, it reuses the previous one β€” correct behaviour for a control loop. +### `Node` / `ObjectNode` β€” convenience wrappers (`node.hpp`) -### Port Ordering Contract - -`in<>` and `latch<>` ports together must cover all function parameters in declaration order. The `static_assert` on name count is extended to cover both tags jointly: +`Node<>` privately owns a `ThreadPool(1)` and derives from `PoolNode<>` with the **same** +template signature, so each `Node` is a self-contained worker with no external scheduler. Its +`start()`/`stop()` start and stop the private pool around the base. This keeps the simple API β€” +`make_node(5)` β€” while routing all execution through the one `fire_once()` code path. ```cpp -static_assert( - sizeof...(InNames) + sizeof...(LatchNames) == input_count || - (sizeof...(InNames) == 0 && sizeof...(LatchNames) == 0), - "make_node: in<> and latch<> names together must match function arity, or provide none" -); +template, typename OutputTag = out<>, + fixed_string Label = "", std::size_t UniqueTag = 0> +class Node : public PoolNode<...> { ... }; + +auto src = make_node(5); +auto dbl = make_node(5); +auto cnt = make_node(in<"words">{}, out<"count","words">{}, 4); ``` -The function parameter at position `i` is synchronous if `i` is in the `in<>` pack, latched if in the `latch<>` pack. Mixed ordering is allowed β€” the tag packs define which positions are latched, not a contiguous suffix. +The `Label` NTTP gives a human-readable name for diagnostics; `UniqueTag` is a collision-breaker +required when the **same function** is used as two distinct vertices in a `StaticNetwork` (two +`make_node` would otherwise be the same type). Both default so existing code is unaffected. +To share one pool across many nodes for bounded-thread execution, use `make_pool_node` directly. + +### `InterruptNode` β€” external-trigger source (`interrupt_node.hpp`) + +A zero-input source driven by an external event (camera frame, timer, socket) instead of +self-resubmission. `get_trigger()` returns a thread-safe callable to hand to the event source; +each call increments a `pending_` counter and submits `fire_once()` on the 0β†’1 transition, +guaranteeing one execution per trigger even under bursts. It does not busy-loop. + +```cpp +auto cam = make_interrupt_node(scheduler, out<"frame">{}); +camera_sdk.on_frame_ready(cam.get_trigger()); +``` + +### `FanoutNode` β€” explicit fan-out (`fanout.hpp`) + +Reads one item and pushes a copy to each of N outputs (per-output overflow drops +independently). Runs on its own `std::jthread` blocking on `pop()`. Used directly in a runtime +`Network` via `make_fanout`, and auto-inserted by `make_network()` for `StaticNetwork`. + +### `RouterNode` / `FilterNode` β€” branching (`branch.hpp`) + +Both run on a dedicated `jthread`. `RouterNode` pushes each item to exactly **one** of N +outputs chosen by a `selector(item) -> size_t` (out-of-range index drops). `FilterNode` forwards +an item only when `pred(item)` is true. Factories: `make_router(sel)`, `make_filter(pred)`. + +### `MainThreadNode, Args…>` β€” GUI / main-thread nodes (`main_thread_node.hpp`) + +For work that *must* run on the thread owning a GUI event loop (OpenCV `imshow`/`waitKey` on +Wayland/Qt). It owns input channels and is registered as a normal `INode` (appears in +diagnostics) but spawns **no** thread. The application drives it by calling `step()` in a loop on +the main thread: `step()` does a zero-timeout `try_pop` on every input, and when all are ready +invokes the derived `operator()(Args…)` (returning `false` to stop). CRTP; the derived class +supplies the operator. --- -## Component 5 β€” `network.hpp`: Graph Builder + Orchestrator +## Component 7 β€” `shared_resource.hpp`: Priority-arbitrated Exclusive Resource -`Network` is **non-owning** β€” nodes are declared by the user and must outlive the network. -`add()` registers a raw pointer. Graph construction uses a builder pattern so the full -topology is known before `build()`, enabling cycle detection and topological ordering. +`SharedResource` wraps a singleton-like resource (an ONNX session, a CUDA stream) shared by +nodes across one or more networks, and arbitrates access with a **priority + aging** waiter +queue. Priority is re-evaluated at every release (so it reflects current queue state), and each +waiter's effective score grows with wait time (`kAgingPerSecond`) to prevent starvation. ```cpp -class Network : public INode { // Network is itself an INode β€” enables sub-networks +SharedResource res(session_args...); + +// inside a node functor: +auto guard = res.acquire_balanced(in_channel, out_channel); // RAII; releases on scope exit +guard->Run(...); +``` + +`acquire_balanced(in, out)` scores a waiter by `input_fill Γ— output_headroom` β€” a node with a +full input queue and empty output is most urgent. `acquire(fn)` takes any `()->float` priority; +`acquire()` treats all waiters equally. Implements `IResourceProbe` so it shows up in +diagnostics and the debug hub. The factory is `make_shared_resource(args…)`. + +--- + +## Component 8 β€” `network.hpp`: Runtime Graph Builder + Watchdog + +`Network` is **non-owning** (nodes outlive it; `add()` stores `INode*`). A builder collects the +full topology before `build()`, enabling cycle detection and topological ordering. + +```cpp +class Network : public INode { public: - // Register a node by name. NodeT must satisfy INode. Network holds a raw pointer. - template - Network& add(std::string name, NodeT& node); + template Network& add(std::string name, NodeT& node); - // Connect output port of src to input port of dst. - // Type mismatch β†’ static_assert at compile time. - template - Network& connect(const std::string& src_name, OutputPort, - const std::string& dst_name, InputPort); + template + Network& connect(const std::string& src, OutputPort, + const std::string& dst, InputPort); - // Expose an internal node's input/output as a boundary port of this (sub-)network. - // Allows a Network to be connected into a larger Network like a single node. - template - Network& expose_input(std::string boundary_name, InputPort); - - template + Network& expose_input (std::string boundary_name, InputPort); // sub-network port Network& expose_output(std::string boundary_name, OutputPort); - // DFS cycle check + topological sort. Throws NetworkCycleError on cycles. - Network& build(); + Network& build(); // DFS cycle check (throws NetworkCycleError) + topo sort - void start() override; // starts all internal nodes in topological order - void stop() override; // stops all internal nodes in reverse order; disables channels - bool running() const override; + void start() override; // start nodes in topo order; launch watchdog (+ web UI) + void stop() override; // == halt() + void halt() override; // immediate: stop nodes in reverse topo order + void shutdown() override; // graceful: stop source layers, drain channels, descend void set_watchdog_interval(std::chrono::milliseconds); + void set_error_handler(ErrorHandler); // void(node_name, exception_ptr) + void set_diagnostics_handler(DiagnosticsHandler); // fired each watchdog tick + void set_event_handler(EventHandler); // void(name, NodeEvent, timestamp) + void register_pool(const std::string&, IPoolProbe*); - using ErrorHandler = std::function; - void set_error_handler(ErrorHandler); - -private: - std::map nodes_; // non-owning - std::map> adj_; - std::vector topo_; - std::jthread watchdog_; - std::chrono::milliseconds watchdog_interval_{500}; - ErrorHandler error_handler_; + void print_diagnostics(std::ostream& = std::cerr) const; // formatted table }; ``` -**Node lifetime contract:** nodes must outlive the `Network`. The typical pattern is to -declare nodes and the network in the same scope: +- **`connect`** static-asserts that the source output type equals the destination input type + (via the nodes' `return_tuple` / `args_tuple`), sets the consumer's input channel as the + producer's output pointer, registers a `ChannelProbe` for diagnostics, and rejects a second + connection from the same output port (use `make_fanout`). +- **`build`** colours the graph DFS; a back-edge throws `NetworkCycleError`. It also wires each + node's network-level overflow/closed callbacks to the `EventHandler` if one is set. +- **`halt` vs `shutdown`** β€” `halt()` disables channels and stops nodes in reverse order + immediately; `shutdown()` walks source layers first, polling channel probes until they drain + before stopping the next layer. +- **Watchdog** β€” a `std::jthread` that wakes on `watchdog_interval_` (default 3 s), collects + snapshots, warns about nodes whose `exec_start_us` indicates an execution running > 5 s, and + either calls the diagnostics handler or prints the formatted report. +- **`expose_input`/`expose_output`** record boundary names (sub-network support is scaffolded; + `Network` is itself an `INode` and can be `add()`ed to an outer `Network`). -```cpp -// Nodes declared first β€” they own their input channels -auto blur = make_node>(10); -auto detect = make_node>(10); - -Network net; -net.add("blur", blur) - .add("detect", detect) - .connect("blur", blur.output<0>(), "detect", detect.input<0>()) - .connect("blur", blur.output<"blurred">(), "detect", detect.input<"img">()) - .build(); -net.start(); -``` - -**Sub-networks** β€” because `Network` implements `INode`, it can be registered inside a -larger `Network` as a named node. Boundary ports declared via `expose_input` / -`expose_output` make the internal nodes' ports available to the outer graph: - -```cpp -// Inner sub-network -auto stage1 = make_node(5); -auto stage2 = make_node(5); -Network pipe; -pipe.add("pre", stage1).add("enh", stage2) - .connect("pre", stage1.output<0>(), "enh", stage2.input<0>()) - .expose_input("img", stage1.input<0>()) - .expose_output("result", stage2.output<0>()) - .build(); - -// Outer network treats `pipe` as a single node -auto sink = make_node(5); -Network top; -top.add("pipe", pipe).add("sink", sink) - .connect("pipe", pipe.output<"result">(), "sink", sink.input<0>()) - .build(); -top.start(); -``` - -`NetworkCycleError` is thrown by `build()` if the graph contains a directed cycle. +The formatted report includes node (frames, exec ms, max ms, blocked ms, fps, cpu ms, util%), +channel (fill%, peak%, pushes, drops, overflow, MB/s, item bytes), and pool tables, plus a +bottleneck hint (highest `ema_exec_ms`). --- -## Component 6 β€” `variant_node.hpp`: Runtime-typed Node (Python graphs) +## Component 9 β€” `static_network.hpp`: Compile-time Graph Builder -### Motivation - -Python graphs cannot use compile-time type resolution. A `PyNetwork` is constructed with a -**closed list of C++ node types** known at binding time. The library derives a deduplicated -`std::variant` from all port types across those nodes. Type safety is enforced at -`connect()` time via string signatures. - -### Variant Deduplication - -All port types from the registered nodes are collected into a flat pack, duplicates are -removed via a `unique_types` TMP metafunction, then the variant is instantiated once: +For C++ graphs whose full topology is known at compile time. The complete edge list is a type +pack, so fan-out arity is known up front, cycle detection is a `static_assert`, and start/stop +are pointer-vector traversals rather than string-map + virtual dispatch. ```cpp -template -using py_variant_t = std::variant>>; +// edge() builds a typed Edge descriptor from two port handles. +template +Edge +edge(OutputPort, InputPort); + +// make_network() takes all edges, expands fan-outs, wires channels, returns a StaticNetwork. +template auto make_network(Edges&&... edges); ``` -This is pure TMP and runs entirely at compile time. The resulting variant has no redundant -alternatives at runtime. - -### PyNetwork Construction - -`make_py_network` is a **pure C++ template** β€” no CMake code-gen step. The variant is -derived entirely at compile time from the registered node type list. The nanobind module -definition is the single place where node types are listed; recompiling the extension is -the "registration" step. +Usage β€” no `add`/`connect`/`build`/string names; one source port feeding two destinations +auto-inserts a `FanoutNode`: ```cpp -// In kpn_python.cpp β€” list all node types that may appear in Python graphs: -auto py_net = make_py_network(); -// VariantValue = std::variant< /* deduplicated port types from A, B, C */ > -// Registers to_python / from_python converters for each alternative. +auto src = make_node(8); +auto blur = make_node(8); +auto detect = make_node(8); +auto sink = make_node(8); + +auto net = make_network( + edge(src.output<0>(), blur.input<0>()), + edge(src.output<0>(), detect.input<0>()), // same source port β†’ FanoutNode inserted + edge(blur.output<0>(), sink.input<0>()), + edge(detect.output<0>(), sink.input<1>())); +net.start(); /* … */ net.stop(); ``` -### VariantChannel +`make_network` performs, at compile time: fan-out detection and edge expansion +(`tmp/fanout_groups.hpp`), a duplicate-`(Func, UniqueTag)` check +(`static_assert` β€” "add a UniqueTag"), and a cycle check + topological order +(`tmp/topo_sort.hpp`, `static_assert` β€” "graph contains a directed cycle"). At run time it +heap-allocates owned `FanoutNode` storage, collects user-node pointers in edge order, sets each +node's display name (`Label`, else `node[UniqueTag]`; fan-outs become `"_fanout"`), wires +every expanded edge, and builds channel probes. -```cpp -using VariantValue = py_variant_t; +`StaticNetwork` implements `INode` (so it can be embedded in a +runtime `Network`). It owns the fan-out nodes, holds user nodes by pointer, and provides +`start`/`halt`/`shutdown`, an `EventHandler`, `register_resource` / `register_pool`, +`print_diagnostics`, and `network_snapshot()` (consumed by the `DebugHub`). Compile-time labels +are read from each `NodeType::label()`. -class VariantChannel { -public: - explicit VariantChannel(std::size_t capacity = 5); - void push(VariantValue v); // throws ChannelOverflowError if full - VariantValue pop(); // blocks (KPN semantics) -}; -``` - -### VariantNode - -Wraps a registered C++ node type. Its `run_loop` uses `std::visit` to extract the concrete -type from a `VariantValue`, calls the underlying function, then wraps the result back into a -`VariantValue` for the output channel. - -```cpp -class VariantNode { -public: - std::string input_type_sig(std::size_t idx) const; - std::string output_type_sig(std::size_t idx) const; - - void connect_input (std::size_t port, std::shared_ptr); - void connect_output(std::size_t port, std::shared_ptr); - - void start(); - void stop(); -}; -``` - -### PythonConverter β€” Crossing the C++/Python Boundary - -Every type in the variant must provide a `PythonConverter` specialisation. This is the -single mechanism used for all data crossing into or out of Python (PyNodes, `net.read`, -`net.write`): - -```cpp -template -struct PythonConverter { - static nanobind::object to_python(const T&); - static T from_python(nanobind::object); -}; -``` - -### PyNode β€” Pure Python Processing Node - -A `PyNode` holds a `nanobind::object` as its function. Its `run_loop`: - -1. Pops `VariantValue` from each input channel -2. `std::visit` β†’ calls `PythonConverter::to_python` for each β†’ **acquires GIL** -3. Calls the Python callable -4. **Releases GIL** β†’ calls `PythonConverter::from_python` on the return value -5. Pushes result as `VariantValue` to output channel - -### Sub-value Extraction and Injection - -A C++ node returning `std::tuple` exposes three independent output ports. Each -element is pushed to its own `VariantChannel` β€” sub-indexing is a first-class concept at the -channel level, not an afterthought. - -**Python tap β€” read one output port into Python:** - -```python -value = net.read("detect", output=2) -# Pops from output channel 2, calls PythonConverter::to_python. -# GIL released while blocking on pop(), re-acquired before to_python call. -``` - -**Python inject β€” write a Python value into a specific input port:** - -```python -net.write("blur", input=1, value=my_sigma) -# Calls PythonConverter::from_python(my_sigma), pushes to input channel 1. -# GIL released while blocking on push() if channel is full. -``` - -**Python splitter node:** - -```python -def split(packed): - img, mask, score = packed - return img, mask - -net.add_node("split", split, inputs=["packed"], outputs=["img", "mask"]) -net.connect("detect", 0, "split", 0) -net.connect("split", 0, "show", 0) -net.connect("split", 1, "save", 0) -``` - -**Direct C++ sub-output to Python node input:** - -```python -net.connect("detect", 1, "py_thresh", 0) -# Type sig of detect:output[1] must match py_thresh:input[0] β€” checked at connect(). -``` - -### Type-check at connect time (Python) - -```python -# Raises kpn.TypeError if signatures don't match -net.connect("blur", 0, "thresh", 0) # (src_name, out_idx, dst_name, in_idx) -``` +> `make_fanout` remains for explicit fan-out in a runtime `Network`; `make_network` users +> never call it. --- -## Component 7 β€” Orchestrator / Watchdog +## Component 10 β€” Web Debugging (optional, `KPN_WEB_DEBUG`) -Runs in its own dedicated thread inside `Network` / `PyNetwork`. Responsibilities: +Zero cost when disabled β€” guarded headers, no symbols, no dependency. Depends on **cpp-httplib** +(single-header, fetched by CMake when the option is on) and loads **D3.js v7** from CDN. Enable +per-target: -- Starts nodes in topological order; stops them in reverse order -- Tracks per-node execution time (exponential moving average) -- Emits warning via logger callback (default: `stderr`) when a node stalls beyond threshold -- Catches exceptions from node threads and routes them to `ErrorHandler` -- Graceful shutdown: signals all nodes, joins with timeout, reports any that fail to stop +```cpp +#define KPN_WEB_DEBUG 1 +#include +``` + +### Single-network server (`web_debug.hpp`) + +When enabled, `Network` / `StaticNetwork` gain `set_web_debug_port(uint16_t)` (default 9090) and +auto-start an in-process HTTP server in `start()`. It serves an inline single-page app at `/` and +a JSON snapshot at `/api/snapshot` (nodes, channels/edges, pools, resources, elapsed). The page +renders a force-directed graph: node colour encodes `ema_exec_ms`, edge colour encodes fill%, +with hover tooltips for the full stat set; it polls every 500 ms. + +### `DebugHub` β€” multi-network UI (`debug_hub.hpp`) + +A standalone server aggregating several networks under one endpoint: + +```cpp +DebugHub hub(9090); +hub.register_network("detect", detect_net); // disables that net's own server +hub.register_network("classify", classify_net); +hub.register_resource("gpu", &gpu_resource); // shows utilisation cards +hub.start(); +``` + +The hub UI has one tab per registered network plus an "All Networks" tab with shared-resource +cards and a cross-network node table. `register_network` calls `net.disable_web_server()` so the +hub is the single debug endpoint; call it before `net.start()`. --- -## GIL Rules (non-negotiable constraints on binding implementation) +## Component 11 β€” Python Bindings (partial) -Two rules govern all interaction between node threads and the Python interpreter: +> Status: scaffolded and partially implemented. The variant machinery, `PyNetwork`/`PyNode`, and +> the auto-binding layer exist; the demo module wires a hello-pipeline. Full sub-port read/write +> and mixed C++/Python graphs are still in progress. -1. **Acquire for callback** β€” a node thread must hold the GIL only for the duration of a - Python callable invocation (`nb::gil_scoped_acquire` wrapping the call site). +Python graphs cannot resolve types at compile time, so a `PyNetwork` is parameterised by a +`std::variant` derived (at compile time, via `unique_types`) from the port types of a **closed +list of registered C++ node types**. The variant only appears at the C++/Python boundary; each +node's internal `Channel` still stores raw `T` (`variant_node.hpp`: `IVariantChannel`, +`VariantChannel`, `IVariantNode`, `VariantNodeWrapper`). -2. **Release while blocking** β€” any blocking operation on a channel (`pop()`, `push()`, - `net.read()`, `net.write()`) must release the GIL before blocking - (`nb::gil_scoped_release` wrapping the call site), then re-acquire after. +### Auto-binding (`python/auto_bind.hpp`) -Violating rule 2 deadlocks: a PyNode thread waiting to acquire the GIL cannot proceed while -another thread holds the GIL and blocks on a channel waiting for that PyNode to produce. +The node list is declared once with a `NodeRegistry` of `Entry`. `bind_network` +registers the `PyNetwork` class, a `make_(capacity)` factory and a `Node` class per +entry, and auto-registers `PythonConverter` for each port type. `bind_debug` additionally exposes +each raw C++ function as a free Python callable for testing without a network. Recompiling the +extension is the registration step β€” there is no CMake code-gen. + +```cpp +using DemoNodes = kpn::python::NodeRegistry< + kpn::python::Entry, + kpn::python::Entry, + kpn::python::Entry>; // variant auto-deduced as std::variant + +NB_MODULE(kpn_python, m) { + bind_network(m); + bind_debug(m); +} +``` + +Custom types are supported by specialising `kpn::PythonConverter` (`to_python` / `from_python`, +optional `type_name`) before `bind_network`. + +### GIL rules (non-negotiable) + +1. **Acquire for callback** β€” hold the GIL only for the duration of a Python callable + invocation (`nb::gil_scoped_acquire` around the call site). +2. **Release while blocking** β€” release the GIL before any blocking channel op + (`nb::gil_scoped_release`), then re-acquire. Violating this deadlocks: a PyNode thread + waiting for the GIL cannot proceed while another thread holds it and blocks on a channel + waiting for that PyNode. --- @@ -758,616 +658,132 @@ another thread holds the GIL and blocks on a channel waiting for that PyNode to | Situation | Behaviour | |---|---| -| FIFO overflow | `ChannelOverflowError` thrown in producer thread β†’ `ErrorHandler` | -| Node function throws | Exception pointer captured β†’ `ErrorHandler` | -| Type mismatch (C++) | `static_assert` at `connect()` compile time | -| Type mismatch (Python) | `kpn.TypeError` raised at `net.connect()` call | -| Cycle in graph | `NetworkCycleError` thrown at `build()` time | -| Thread fails to stop | Watchdog warning after configurable timeout | -| `from_python` / `to_python` fails | Exception propagated to `ErrorHandler` | +| FIFO overflow (full, accepting) | `ChannelOverflowError` thrown in producer; node overflow callbacks fire | +| Push to a disabled channel | Value silently dropped (counted as a `drop`) | +| Node function throws | Routed to the node's `NodeErrorHandler` β†’ `true` skips & continues, `false` stops the node | +| Node stopped / channel closed | `ChannelClosedError` β†’ node fires closed callbacks and self-stops | +| Type mismatch (C++) | `static_assert` at `connect()` / `make_network()` | +| Cycle in graph (runtime) | `NetworkCycleError` thrown at `build()` | +| Cycle in graph (static) | `static_assert` at `make_network()` | +| Duplicate `(Func, UniqueTag)` (static) | `static_assert` at `make_network()` β€” add a `UniqueTag` | +| Hung node | Watchdog warning after threshold | ---- - -## Future Extension Points (Heterogeneous Execution) - -Not implemented now, but the design must not close these doors: - -- **`IChannel` abstract interface** β€” `Channel` and a future `RemoteChannel` (wrapping - a socket/queue) would share the same `push`/`pop` interface. Nodes never know whether - their channel is in-process or remote. - -- **`Serializer` trait** β€” parallel to `PythonConverter` and `channel_storage_policy`, - a specialisable trait for cross-device serialisation (MessagePack for ESP32, pinned memory - for GPU zero-copy, etc.). - -- **`NodeKind` tag** β€” `enum class NodeKind { Local, Gpu, Remote }` on the `INode` - interface, letting the watchdog apply different health-check and timeout strategies per - device type. - -These three extension points are sufficient to support GPU and embedded/network targets -without redesigning the core. +`Network` additionally exposes an aggregate `EventHandler(name, NodeEvent, timestamp)` for +overflow/closed events across all nodes. --- ## Thread Model -**v1: one `std::thread` per node.** This maps directly to KPN theory and is simple to reason -about. It does not scale to networks with hundreds of nodes but is appropriate for the -typical use case (tens of nodes, each doing non-trivial work). +KPN++ is **reactive**, not one-thread-per-node: -`std::jthread` (C++20) is preferred over `std::thread` where available, as it provides a -built-in `stop_token` that simplifies the `stop()` / `try_pop` shutdown pattern. +- A `PoolNode` owns no thread. It registers a push-callback on each input channel; when all + inputs are ready it submits `fire_once()` to a shared `IScheduler` (a `ThreadPool`). +- `Node<>` wraps a `PoolNode` plus a **private `ThreadPool(1)`**, recovering "independent + worker" semantics with the simple `make_node` API. Many nodes can instead share one pool + (`make_pool_node`) for a bounded OS thread count. +- `FanoutNode`, `RouterNode`, and `FilterNode` do run a dedicated `std::jthread` blocking on + `pop()` (they are simple, latency-sensitive routers). +- `InterruptNode` fires on an external trigger; `MainThreadNode` runs on the caller's main + thread via `step()`. -A future executor/thread-pool model (where multiple nodes share a pool of threads and are -scheduled cooperatively) is a possible v2 extension. The `INode` interface is designed to -not assume a 1:1 thread mapping. +`std::jthread` (C++20) and its `stop_token` are used where a thread is owned, simplifying +cooperative shutdown. Benchmarks (`benchmarks/bench_pipeline`) show ~2–7 Β΅s/hop framework +overhead for chains within the core count, rising under oversubscription. --- ## Platform and Compiler Requirements -C++20 is required. Specific features used: +C++20 is required. -| Feature | Header / Standard | Min compiler | -|---|---|---| -| NTTP structural types (`fixed_string`) | language | GCC 11, Clang 13, MSVC 19.29 | -| `std::is_trivially_copyable_v` | `` | C++17+ | -| `std::jthread` + `stop_token` | `` | GCC 11, Clang 14, MSVC 19.29 | -| `if constexpr`, fold expressions | language | C++17+ | -| `auto` NTTPs | language | C++20 | -| Concepts (`requires`) | language | GCC 10, Clang 10 | +| Feature | Min compiler | +|---|---| +| NTTP structural types (`fixed_string`) | GCC 11, Clang 13, MSVC 19.29 | +| `std::atomic::wait/notify` (channel futex) | GCC 11, Clang 13, MSVC 19.29 | +| `std::jthread` + `stop_token` | GCC 11, Clang 14, MSVC 19.29 | +| `auto` NTTPs, fold expressions, `if constexpr`, concepts | C++20 / C++17 baseline | -**Minimum supported compilers:** GCC 11, Clang 13, MSVC 19.29 (VS 2022). -nanobind requires Python 3.8+ and a C++17-capable compiler (satisfied by the above). +`CLOCK_THREAD_CPUTIME_ID` (per-thread CPU stats in `diagnostics.hpp`) is POSIX. nanobind +requires Python 3.8+ (auto-fetched when `KPN_BUILD_PYTHON=ON`). --- ## Testing Strategy -Test frameworks: **Catch2 v3** (header-friendly, good async/threading support via -`REQUIRE_NOTHROW` + thread join patterns) and **Google Test** (for death tests and -parameterised test suites). Both are included; use Catch2 for integration/behaviour tests -and GTest for unit tests where `ASSERT_*` / `EXPECT_*` macros and death tests are -preferable. +**Catch2 v3** for behaviour/integration tests and **GoogleTest** for unit and death tests; both +are auto-fetched. Existing suites: `test_fixed_string`, `test_traits`, `test_channel`, +`test_node`, `test_network`, `test_static_network`, `test_scheduler`, `test_pool_node`, +`test_shared_resource`. -### Hard cases to cover explicitly: - -| Case | What to test | -|---|---| -| Channel blocking | `pop()` blocks until a producer pushes; unblocks exactly once per push | -| Channel overflow | `push()` beyond capacity throws `ChannelOverflowError` | -| Shutdown race | `stop()` called while a node is blocked on `pop()` β€” thread must exit cleanly | -| Multi-consumer | Two nodes connected to the same output channel each receive every item (fan-out) | -| Tuple unpacking | Multi-output node pushes correct type to each sub-channel | -| Cycle detection | `build()` throws `NetworkCycleError` for a graph with a cycle | -| Named port lookup | `input<"wrong">()` fires `static_assert`; `input<"right">()` resolves correctly | -| Wrong name count | `make_node` with mismatched name count fires readable `static_assert` | -| GIL deadlock | PyNode + blocking `net.read()` from Python do not deadlock | -| `from_python` failure | Exception propagates to `ErrorHandler`, network continues | -| `channel_storage_policy` | Large type is stored as `shared_ptr`; small type by value | +Cases covered explicitly include: channel blocking/unblocking and overflow; shutdown races +(`stop()` while blocked on `pop()`); `try_pop_now`; fan-out delivery; tuple unpacking to +sub-channels; runtime cycle detection and static cycle/duplicate-tag `static_assert`s; named +port lookup and wrong-name-count `static_assert`s; storage-policy by-value vs `shared_ptr`; +scheduler submit/steal/drain; `PoolNode` reactive scheduling; and `SharedResource` priority + +aging. --- ## Examples -Each example is a self-contained program under `examples/`. They are built as part of the -CMake build and serve as both documentation and smoke tests. +Self-contained programs under `examples/`, built by default (`-DKPN_BUILD_EXAMPLES=OFF` to +skip). They double as documentation and smoke tests. -### `examples/01_hello_pipeline` β€” Basic linear pipeline - -Two nodes connected in sequence. Demonstrates `make_node`, `Network` builder, -index-based port connection, `start_all` / `stop_all`. - -```cpp -// producer β†’ transform β†’ sink -int produce() { return 42; } -int double_it(int x) { return x * 2; } -void print_it(int x) { std::cout << x << '\n'; } - -auto src = make_node(5); -auto dbl = make_node(5); -auto sink = make_node(5); - -Network net; -net.add("src", src) - .add("dbl", dbl) - .add("sink", sink) - .connect("src", src.output<0>(), "dbl", dbl.input<0>()) - .connect("dbl", dbl.output<0>(), "sink", sink.input<0>()) - .build(); -net.start_all(); -``` - -### `examples/02_named_ports` β€” Named port access - -Same pipeline but using `in<>` / `out<>` name tags and named port access. Demonstrates -`fixed_string` NTTP dispatch and the `static_assert` on wrong names. - -```cpp -auto dbl = make_node, out<"result">>(5); -// ... -.connect("src", src.output<0>(), "dbl", dbl.input<"value">()) -.connect("dbl", dbl.output<"result">(), "sink", sink.input<0>()) -``` - -### `examples/03_multi_output` β€” Tuple-returning node / sub-port routing - -A single node returns `std::tuple`. Each output is routed to a different -downstream node. Demonstrates tuple normalisation, per-element channel push, and -`output<1>()` sub-indexing. - -```cpp -std::tuple detect(Image in) { ... } -void show_image(const Image& img) { ... } -void save_mask(const Mask& m) { ... } - -// detect:output<0> β†’ show_image, detect:output<1> β†’ save_mask -``` - -### `examples/04_storage_policy` β€” `channel_storage_policy` specialisation - -Shows the default behaviour (large struct stored as `shared_ptr`, small int by -value) and a user specialisation that overrides the default for a custom type. - -```cpp -struct BigFrame { uint8_t pixels[1920*1080*3]; }; -// stored as shared_ptr automatically - -struct Tiny { float x, y; }; // 8 bytes β€” by value by default -template<> struct channel_storage_policy { static constexpr bool by_value = true; }; -``` - -### `examples/05_error_handling` β€” Overflow and node exceptions - -Demonstrates `ChannelOverflowError` (producer faster than consumer, tiny FIFO), custom -`ErrorHandler`, and a node that throws mid-execution. - -```cpp -net.set_error_handler([](std::string_view name, std::exception_ptr ep) { - try { std::rethrow_exception(ep); } - catch (const std::exception& e) { - std::cerr << "[" << name << "] " << e.what() << '\n'; - } -}); -``` - -### `examples/06_watchdog` β€” Orchestrator / watchdog - -A node that artificially stalls. Shows watchdog warning emission, configurable interval, -and graceful shutdown after a timeout. - -```cpp -net.set_watchdog_interval(std::chrono::milliseconds(200)); -// stall_node sleeps for 2s per item β€” watchdog fires warning after 200ms -``` - -### `examples/07_python_network` β€” PyNetwork with C++ and Python nodes - -Python script that imports `kpn`, registers C++ node types via `make_py_network`, adds a -pure Python processing node, connects them, and runs the graph. - -```python -import kpn - -net = kpn.make_network([kpn.BlurNode, kpn.DetectNode]) - -def py_filter(img): - return img[::2, ::2] # downsample in Python - -net.add_node("blur", kpn.BlurNode, inputs=["img"]) -net.add_node("downsample",py_filter, inputs=["img"], outputs=["img"]) -net.add_node("detect", kpn.DetectNode, inputs=["img"]) -net.connect("blur", 0, "downsample", 0) -net.connect("downsample", 0, "detect", 0) -net.start() -``` - -### `examples/09_opencv_cellshade` β€” Real-time cell-shading with OpenCV (optional) - -Captures live video from a system camera and applies a cell-shading effect entirely inside -a KPN++ graph. Built only when OpenCV is found at CMake time; skipped silently otherwise. - -**Graph topology:** - -``` -[capture] ──Mat──> [split] ──B──> [median_b] ──B──┐ - β”œβ”€β”€G──> [median_g] ──G─── - └──R──> [median_r] ──R──┴──> [merge] ──Mat──> [combine] ──> [display] -[capture] ──Mat──────────────────────> [detect_edges] ──mask──────────> [combine] -``` - -Effect steps: -1. **`split_channels`** β€” `cv::split` into three single-channel `cv::Mat` planes. -2. **`median_b/g/r`** β€” independent `cv::medianBlur(kernel=15)` per channel; large kernel - posterises colours into flat cartoon-like regions and runs in parallel across channels. -3. **`merge_channels`** β€” `cv::merge` back to BGR. -4. **`detect_edges`** β€” greyscale, `cv::Canny`, then `cv::dilate` to produce thick outlines. -5. **`combine`** β€” zeros out BGR pixels wherever the edge mask is non-zero β†’ black outlines - drawn over the flat-colour image. -6. **`display`** β€” `cv::imshow`; ESC key signals shutdown via `g_running` atomic. - -Demonstrates: named ports, fan-out from a single node to two downstream paths, parallel -per-channel processing, multi-input `combine` node, and error handler driving graceful stop. - -```cpp -// Build only if OpenCV is present: -// cmake .. -DKPN_BUILD_EXAMPLES=ON -// ./09_opencv_cellshade [camera_index] # default: 0 -``` - -### `examples/08_python_subport` β€” Python sub-value tap and inject - -Shows `net.read("node", output=N)` and `net.write("node", input=N, value=v)` from Python, -plus connecting a C++ tuple output sub-port directly to a Python node input. - -```python -# Tap only output<1> (Mask) of a C++ detect node into Python -net.connect("detect", 1, "py_thresh", 0) -val = net.read("detect", output=0) # blocks until Image is available -net.write("blur", input=1, value=1.5) # inject sigma -``` - ---- - -## Component 8 β€” Web Debug UI (optional, compile-time toggle) - -An optional in-process HTTP server that serves a live graph visualisation of the running -network. Zero cost when disabled β€” no symbols compiled in, no headers pulled. - -### Toggle - -```cpp -// Before any kpn include β€” enables the web debug server -#define KPN_WEB_DEBUG 1 -#include -``` - -CMake projects that want it globally: - -```cmake -option(KPN_WEB_DEBUG "Enable KPN++ web debug UI" OFF) -if(KPN_WEB_DEBUG) - target_compile_definitions(my_app PRIVATE KPN_WEB_DEBUG=1) - # cpp-httplib is fetched automatically by CMake when this flag is ON -endif() -``` - -### Implementation - -`include/kpn/web_debug.hpp` β€” included by `network.hpp` only when `KPN_WEB_DEBUG` is defined. - -Depends on **cpp-httplib** (single-header, no external process, no Python required). -Served on `localhost:9090` by default (configurable via `net.set_web_debug_port(uint16_t)`). - -When enabled, `Network` gains: - -```cpp -#ifdef KPN_WEB_DEBUG -void set_web_debug_port(uint16_t port); // default 9090 -void start_web_debug(); // called internally by start() -void stop_web_debug(); // called internally by stop() -#endif -``` - -`start()` automatically calls `start_web_debug()` when `KPN_WEB_DEBUG` is defined. - -### Endpoints - -| Endpoint | Method | Description | -|---|---|---| -| `/` | GET | Serves the single-page HTML app (inline, no files needed) | -| `/api/snapshot` | GET | Returns a JSON snapshot of all node and channel stats | - -The HTML page is embedded as a C++ string literal β€” no asset files to deploy. - -### JSON Snapshot Format - -```json -{ - "nodes": [ - { "id": "src", "frames": 120, "ema_exec_ms": 33.2, "max_exec_ms": 45.1, - "blocked_ms": 0.1, "fps": 29.8 }, - { "id": "quant", "frames": 120, "ema_exec_ms": 4.1, ... } - ], - "edges": [ - { "source": "src", "target": "quant", "label": "colour", - "fill_pct": 12.5, "peak_pct": 87.5, "capacity": 8, "current": 1, - "pushes": 120, "drops": 0, "overflows": 0 } - ] -} -``` - -Node `id` comes from the name registered via `net.add("name", node)`. -Edge `label` comes from the channel name registered via `connect()` (format: `"src:N β†’ dst:M"`). - -### Browser UI - -The page polls `/api/snapshot` every **500 ms** and renders a **D3.js v7 force-directed -graph**: - -- **Nodes** β€” circles labelled with node name; colour encodes exec load: - - green (`ema_exec_ms` < 10ms), yellow (10–50ms), orange (50–100ms), red (>100ms) - - hover tooltip shows: frames, ema_exec_ms, max_exec_ms, blocked_ms, fps -- **Edges** β€” directed arrows labelled with the channel name and fill%; colour: - - green (fill < 50%), yellow (50–80%), red (β‰₯80%) β€” matches the `<<<` flag in the text report - - hover tooltip shows: pushes, drops, overflows, capacity - -D3 is loaded from CDN (`d3js.org`). The entire UI is a single inline HTML string in -`web_debug.hpp` β€” no file serving, no build step for assets. - -### Thread model - -`start_web_debug()` launches a `std::jthread` running `httplib::Server::listen()`. -The server is stopped via `httplib::Server::stop()` called from `stop_web_debug()`. -`/api/snapshot` calls `collect_snapshots()` (already thread-safe β€” reads atomics with -relaxed ordering) and serialises to JSON using a minimal hand-rolled serialiser -(no third-party JSON library required). - -### Example usage - -```cpp -#define KPN_WEB_DEBUG 1 -#include - -// ... build network as normal ... -net.set_web_debug_port(9090); // optional, 9090 is the default -net.start(); -// Open http://localhost:9090 in a browser -``` - ---- - -## CMake Layout - -| Target | Type | Notes | -|---|---|---| -| `kpn` | header-only interface library | C++20, no external deps | -| `kpn_python` | nanobind shared library | links `kpn`, requires Python 3.8+ | -| `kpn_tests` | executable | Catch2 v3 + Google Test | -| `kpn_examples` | executables (one per example) | built by default, off with `-DKPN_EXAMPLES=OFF` | -| `kpn_web_debug` | compile-time option | `#define KPN_WEB_DEBUG 1`; fetches cpp-httplib via CMake FetchContent | - ---- - -## Component 9 β€” `static_network.hpp`: Compile-time Graph Builder - -### Motivation - -The runtime `Network` builder has two limitations that only a compile-time graph can fix: - -1. **Fan-out `N` is unknowable at the first `connect()` call.** A `FanoutNode` requires - `N` as a template parameter. With runtime `connect()`, the network has seen only one edge - when the first call arrives; it cannot know how many more will follow for that port. Auto- - inserting the right `FanoutNode` requires seeing the complete edge list at once β€” - which is only possible if the edge list is a type. - -2. **Start/stop goes through virtual dispatch.** `Network` stores `INode*` and calls virtual - `start()`/`stop()`. With a typed node tuple the compiler sees the concrete types and can - inline or devirtualise. This matters at startup/shutdown, not in the hot path β€” but it is - avoidable overhead. - -The runtime `Network` is **not removed**. It remains the right choice for Python graphs, -sub-networks embedded in dynamic topologies, and any case where the graph shape is not known -until runtime. `StaticNetwork` is an additional builder for the common case where the full -C++ topology is known at compile time. - -### API - -```cpp -// edge() constructs a typed edge descriptor from two port handles. -// All type information (SrcNode, SrcIdx, DstNode, DstIdx) is in the return type. -template -auto edge(OutputPort, InputPort) - -> Edge; - -// make_network() accepts all edges as a variadic pack. -// It deduces the full topology, auto-inserts FanoutNodes where needed, -// wires all channels, and returns a StaticNetwork owning the fanout nodes. -// User nodes are held by reference (non-owning), same lifetime contract as Network. -template -auto make_network(Edges&&... edges) -> StaticNetwork<...>; -``` - -Usage: - -```cpp -auto src = make_node(8); -auto blur = make_node>(8); -auto detect = make_node>(8); -auto sink = make_node(8); - -// src:output<0> feeds both blur and detect β€” fan-out is auto-inserted -auto net = make_network( - edge(src.output<0>(), blur.input<0>()), - edge(src.output<0>(), detect.input<0>()), // same source port - edge(blur.output<0>(), sink.input<0>()), - edge(detect.output<0>(), sink.input<1>()) -); -net.start(); -// ... -net.stop(); -``` - -No `add()`, no `build()`, no string names. The graph is fully wired in the `make_network` -call. `start()` and `stop()` are non-virtual tuple traversals. - -### Edge type - -```cpp -// Carries references to the two endpoint nodes. Stores no data beyond that. -template -struct Edge { - SrcNode& src; - DstNode& dst; -}; -``` - -### Fan-out detection metafunction - -`make_network` receives `Edge<...>` types as a pack. Before wiring, a metafunction scans -the pack for output ports with more than one downstream edge: - -``` -fanout_groups -``` - -This is a compile-time multimap: keys are `(SrcNode type, SrcIdx)`, values are the list of -destination `(DstNode type, DstIdx)` pairs sharing that key. - -For each key with N > 1 destinations: -- Compute `T = std::tuple_element_t` -- Synthesise a `FanoutNode` β€” call it `F` -- Replace the N original edges with: - - one edge: `src:SrcIdx β†’ F:input<0>` - - N edges: `F:output<0..N-1> β†’ original dst:DstIdx` - -For keys with N == 1 the edge is kept as-is. - -The result is an expanded edge list with all fan-outs made explicit, and a list of -`FanoutNode` types that need to be instantiated. - -### `StaticNetwork` structure - -```cpp -template owned by the network - typename TopoOrder> // index_sequence encoding start/stop order -class StaticNetwork : public INode { -public: - void start(); // std::apply over TopoOrder β€” no virtual dispatch, no map lookup - void stop(); // reverse of TopoOrder - - bool running() const; - - // Diagnostics β€” iterates typed tuples; same NodeSnapshot / ChannelSnapshot output - // as Network, compatible with print_diagnostics and the web debug UI. - void print_diagnostics(std::ostream& = std::cerr) const; - - // StaticNetwork is itself an INode, so it can be embedded in a runtime Network - // via net.add("stage", static_net) exactly like any other node. - void set_name(std::string) override; - const NodeStats& stats() const override; - NodeSnapshot node_snapshot(const std::string&, double) const override; - -private: - FanoutStorage fanouts_; // owns the auto-generated FanoutNode instances - // User nodes held by reference β€” same non-owning contract as Network -}; -``` - -`FanoutStorage` is a `std::tuple, FanoutNode, ...>` with one -element per auto-inserted fanout. It is owned by the `StaticNetwork` and lives as long as -the network does β€” which satisfies the channel lifetime contract (channels are owned by their -consumer, and the fanout node is the consumer of the upstream output). - -### Cycle detection - -With the full edge list as a type pack, cycle detection is a `static_assert` rather than a -runtime exception. A compile-time DFS over the expanded edge list fires a readable assertion -at the `make_network` call site: - -``` -static_assert(!has_cycle_v, - "make_network: graph contains a directed cycle"); -``` - -`NetworkCycleError` is no longer needed for `StaticNetwork` β€” the cycle is caught before any -object is constructed. - -### Topological order - -The same compile-time DFS produces a topological ordering as an `std::index_sequence` over -the node tuple. `start()` iterates it forward, `stop()` iterates it in reverse. No runtime -sort, no `std::vector`. - -### Node labels for diagnostics / web debug - -Labels come directly from the `Label` NTTP on each `Node` type β€” no separate annotation -on `edge()` is needed. `StaticNetwork` reads `NodeType::label()` at compile time for each -vertex in the topo order and stores the result as a `std::string_view` array at -construction time. Zero runtime overhead: the label is a compile-time string literal. - -```cpp -auto src = make_node(8); -auto blur = make_node(8); -auto detect = make_node(8); - -auto net = make_network( - edge(src.output<0>(), blur.input<0>()), - edge(src.output<0>(), detect.input<0>()) -); -// web UI shows nodes named "src", "blur", "detect" -// auto-inserted FanoutNode is labelled "fanout[src:0]" -``` - -Unlabelled nodes (`Label == ""`) fall back to `"node[]"` in diagnostics. -Auto-inserted fanout nodes are labelled `"fanout[:]"` automatically. - -### Wiring sequence in `make_network` - -All wiring happens in the `make_network` constructor body β€” no `build()` call needed: - -1. Instantiate `FanoutStorage` (default-construct each `FanoutNode`). -2. For each expanded edge (in topological order): - - Call `src.set_output_channel(&dst.input_channel())`. -3. Return the `StaticNetwork`. - -Channel pointers are set once and never changed. No dynamic allocation after construction. - -### What is eliminated vs `Network` - -| `Network` (runtime) | `StaticNetwork` (compile-time) | +| Example | What it shows | |---|---| -| `std::map` | typed `std::tuple` of references | -| Runtime DFS + `NetworkCycleError` | `static_assert` at `make_network` call site | -| Virtual `start()`/`stop()` per node | `std::apply` over typed tuple | -| Explicit `make_fanout` | auto-inserted from edge pack | -| `connected_outputs_` duplicate check | structural impossibility β€” no duplicate edge can produce two `set_output_channel` calls | -| `build()` step | no build step β€” wired in constructor | +| `01_hello_pipeline` | Linear pipeline, index-based wiring, `Network` builder | +| `02_named_ports` | `in<>`/`out<>` tags, named port access, wrong-name `static_assert` | +| `03_multi_output` | Tuple-returning node, per-element sub-port routing | +| `04_storage_policy` | `channel_storage_policy` default + specialisation | +| `05_error_handling` | `ChannelOverflowError`, diagnostics handler | +| `06_watchdog` | Watchdog interval, stall detection | +| `07_python_network` | `PyNetwork` with a pure-Python node *(pending)* | +| `08_python_subport` | `net.read` / `net.write`, sub-port tap *(pending)* | +| `09_opencv_cellshade` | Real-time cell-shading on webcam; named ports, fan-out, `MainThreadNode` display (requires OpenCV) | +| `10_static_hello_pipeline` | `make_network()` version of 01 β€” compile-time topology | +| `11_static_fanout` | Auto-inserted `FanoutNode` from a duplicated source port | +| `12_static_cellshade` | Static cell-shading with auto fan-out and `Label` NTTPs | +| `13_debug_cellshade` | One-op-per-node pipeline + variadic `DebugCanvas` tiling node | +| `14_debug_hub` | Two networks sharing a `SharedResource` via `DebugHub` | +| `15_node_error_handler` | Per-node `set_error_handler` (skip-and-continue vs stop) | +| `16_event_callbacks` | `set_overflow_callback` + network `set_event_handler` | -The hot path (per-item `pop` β†’ `push` in each node thread) is identical in both cases. +--- -### Compatibility +## Future Extension Points (Heterogeneous Execution) -- `StaticNetwork` implements `INode`, so it can be registered inside a runtime `Network` - via `net.add("name", static_net)` β€” enabling mixed static/dynamic graphs. -- All existing node types (`Node`, `ObjectNode`, `FanoutNode`, `MainThreadNode`) work - unchanged as vertices in a `StaticNetwork`. -- The Python `PyNetwork` is unaffected β€” it remains runtime-only. +Not implemented, but the design keeps these doors open: -### File layout addition +- **`IChannel` abstract interface** β€” `Channel` and a future `RemoteChannel` (socket / + shared-memory) sharing one `push`/`pop` surface so nodes are agnostic to channel location. +- **`Serializer` trait** β€” parallel to `channel_storage_policy` / `PythonConverter`, for + cross-device serialisation (MessagePack for embedded, pinned memory for GPU zero-copy). +- **`NodeKind` tag** β€” e.g. `{ Local, Gpu, Remote }` on `INode`, letting the watchdog apply + per-device health-check and timeout strategies. -``` -include/kpn/ - static_network.hpp # Edge<>, make_network(), StaticNetwork<> - tmp/ - fanout_groups.hpp # fanout_groups metafunction - topo_sort.hpp # compile-time DFS + cycle check - repeat_tuple.hpp # repeat_tuple_t (moved from fanout.hpp) -``` - -`fanout.hpp` keeps `FanoutNode` and `make_fanout` for users who want to wire -fanouts explicitly in a runtime `Network`. `static_network.hpp` uses `FanoutNode` internally -but the user never calls `make_fanout` when using `make_network`. +The `IScheduler` abstraction already decouples node execution from any specific thread model, +making a cooperative or device-specific executor a drop-in. --- ## Resolved Design Decisions -All major design questions are now closed: - | Question | Decision | |---|---| -| Shutdown mechanism | `accepting_` flag per channel; `disable()` clears queue and unblocks `pop()` | -| Overflow behaviour | `ChannelOverflowError` thrown on full accepting channel; silently dropped on disabled channel | -| Network ownership | Non-owning; user declares nodes, network holds raw pointers | -| Node lifetime contract | Nodes must outlive their `Network`; declare in same scope | -| Sub-networks | `Network` implements `INode`; `expose_input`/`expose_output` define boundary ports | -| `make_py_network` | Pure C++ template; nanobind module recompilation is the registration step | -| GIL strategy | Acquire per Python callback; release while blocking on channel ops | -| Mixed-rate inputs | `latch<>` tag for ports that reuse last-seen value; blocks only on first fire; node fires at rate of `in<>` ports | -| Fan-out | Explicit `FanoutNode` for runtime `Network`; auto-inserted by `make_network()` for `StaticNetwork` | -| Static vs runtime graph | Both coexist; `StaticNetwork` for C++ graphs known at compile time, `Network` for Python/dynamic graphs; `StaticNetwork` implements `INode` so it embeds in `Network` | -| Node identity in static graphs | `Label` NTTP (human name for diagnostics) + `UniqueTag` NTTP (collision-breaker for same-Func nodes); both default to `""` / `0` so existing code is unaffected | +| Execution model | Reactive: nodes submit `fire_once()` to an `IScheduler` when inputs are ready, not one blocking thread per node | +| `Node<>` vs `PoolNode<>` | `Node<>` owns a private `ThreadPool(1)`; `PoolNode<>` shares a pool for bounded threads | +| Channel | Lock-free SPSC ring buffer, `atomic::wait/notify` + spin-before-sleep | +| Shutdown | Per-channel `accepting_` flag; `disable()` unblocks `pop()` (β†’ `ChannelClosedError`) | +| Overflow | `ChannelOverflowError` on full accepting channel; silent drop on disabled channel | +| Node error policy | Per-node `NodeErrorHandler` returning bool (skip vs stop) | +| Network ownership | Non-owning; user declares nodes, network stores `INode*` | +| Fan-out | Explicit `FanoutNode` for runtime `Network`; auto-inserted by `make_network()` | +| Branching | `RouterNode` (select one of N) and `FilterNode` (predicate gate) | +| Static vs runtime graph | Both; `StaticNetwork` for compile-time C++ topology, `Network` for dynamic/Python; `StaticNetwork` is an `INode` so it embeds in `Network` | +| Node identity (static graphs) | `Label` NTTP (name) + `UniqueTag` NTTP (collision-breaker); both default | +| Shared device resource | `SharedResource` with priority + aging arbitration | +| Main-thread / GUI work | `MainThreadNode<>` driven by `step()` on the main thread | +| External-event sources | `InterruptNode` with a thread-safe `get_trigger()` | +| Web debugging | Per-network server + multi-network `DebugHub`, behind `KPN_WEB_DEBUG` | +| Mixed-rate latched inputs | **Not implemented** β€” no `latch<>` ports | diff --git a/benchmarks/CMakeLists.txt b/benchmarks/CMakeLists.txt index 97ef6f4..8a104fa 100644 --- a/benchmarks/CMakeLists.txt +++ b/benchmarks/CMakeLists.txt @@ -4,6 +4,13 @@ add_executable(bench_pipeline bench_pipeline.cpp) target_link_libraries(bench_pipeline PRIVATE kpn) target_compile_options(bench_pipeline PRIVATE -O3 -march=native) +# Dispatch microbenchmark (PERF_PLAN B1/B2): ns per ThreadPool dispatch, and +# whether a worker actually sleeps per task. No TBB comparison β€” it measures +# KPN's own scheduler, not a competitor. +add_executable(bench_dispatch bench_dispatch.cpp) +target_link_libraries(bench_dispatch PRIVATE kpn) +target_compile_options(bench_dispatch PRIVATE -O3 -march=native) + find_package(TBB QUIET) if(TBB_FOUND) target_link_libraries(bench_pipeline PRIVATE TBB::tbb) diff --git a/benchmarks/bench_dispatch.cpp b/benchmarks/bench_dispatch.cpp new file mode 100644 index 0000000..c159634 --- /dev/null +++ b/benchmarks/bench_dispatch.cpp @@ -0,0 +1,307 @@ +// Dispatch microbenchmark β€” PERF_PLAN B1 and B2. +// +// B1 asks what a single ThreadPool dispatch costs. B2 asks whether a worker +// actually sleeps per item, because the whole spin-window hypothesis (B5) +// depends on the answer: if workers are not sleeping, a spin window buys +// nothing and drops off the list. +// +// Both are answered here without touching the library. Sleeping is inferred +// from ru_nvcsw β€” a thread blocking on a condition variable books a voluntary +// context switch β€” so `vcsw/task` near 1.0 means a sleep per dispatch and near +// 0 means the worker never went to sleep at all. +// +// Three modes, because "the cost of a dispatch" is three different numbers: +// +// latency β€” one task in flight, pool idle in between. The worker is asleep +// at every submission, so this is dispatch cost *including* a +// wake. Worst case, and the case a spin window would attack. +// +// batch β€” submit K no-op tasks flat out, then drain. The worker is never +// idle, so this is the amortised floor: queue and heap operations +// with no wake at all. Reports the producer-side submit() cost +// separately from end-to-end throughput. +// +// steady β€” the task resubmits its successor, one in flight, each doing +// --work-us of work. This is what a KPN node actually does: +// fire_once processes a token and resubmits. On ThreadPool(1) the +// worker resubmits to its own queue; on ThreadPool(4) round-robin +// hands the task to a *different* worker, which may be asleep. +// That difference is the fanout cost wide-4 pays ~5x per item. +// +// Usage: ./bench_dispatch [--threads=1,2,4] [--mode=latency,batch,steady] +// [--tasks=200000] [--work-us=0] [--reps=5] [--warmup=1] + +#include + +#include "bench_env.hpp" + +#include +#include +#include +#include +#include +#include +#include +#include + +using namespace kpn; +using sclock = std::chrono::steady_clock; + +struct Opts { + std::vector threads {1, 2, 4}; + std::vector modes {"latency", "batch", "steady"}; + long tasks = 200000; + int work_us = 0; + int reps = 5; + int warmup = 1; +}; + +static Opts g_opts; + +static void busy_us(int us) { + if (us <= 0) return; + auto end = sclock::now() + std::chrono::microseconds(us); + while (sclock::now() < end); +} + +struct Sample { + double ns_per_dispatch = 0; // end-to-end, minus the work payload + double submit_ns = 0; // producer side only (batch mode) + double vcsw_per_task = 0; // B2: sleeps per dispatch + double ivcsw_per_task = 0; +}; + +// ── latency: one task at a time, worker asleep between submissions ──────────── + +static Sample run_latency(int threads, long tasks) { + ThreadPool pool(threads); + pool.start(); + + std::mutex mx; + std::condition_variable cv; + bool done = false; + + bench::RusageDelta ru; ru.start(); + auto t0 = sclock::now(); + for (long i = 0; i < tasks; ++i) { + { std::lock_guard lk(mx); done = false; } + pool.submit([&] { + busy_us(g_opts.work_us); + { std::lock_guard lk(mx); done = true; } + cv.notify_one(); + }); + std::unique_lock lk(mx); + cv.wait(lk, [&] { return done; }); + } + auto t1 = sclock::now(); + Sample s; + long iv = 0, vc = 0; + ru.finish(iv, vc); + pool.stop(); + + double elapsed_ns = std::chrono::duration(t1 - t0).count(); + s.ns_per_dispatch = elapsed_ns / tasks - g_opts.work_us * 1000.0; + // The requesting thread blocks once per task too, so it books a voluntary + // switch of its own; halve to attribute per side rather than per process. + s.vcsw_per_task = static_cast(vc) / tasks / 2.0; + s.ivcsw_per_task = static_cast(iv) / tasks; + return s; +} + +// ── batch: submit flat out, drain once. No wake in the steady state ─────────── + +static Sample run_batch(int threads, long tasks) { + ThreadPool pool(threads); + pool.start(); + + std::atomic ran{0}; + + bench::RusageDelta ru; ru.start(); + auto t0 = sclock::now(); + for (long i = 0; i < tasks; ++i) + pool.submit([&] { + busy_us(g_opts.work_us); + ran.fetch_add(1, std::memory_order_relaxed); + }); + auto t_submitted = sclock::now(); + pool.drain(); + auto t1 = sclock::now(); + Sample s; + long iv = 0, vc = 0; + ru.finish(iv, vc); + pool.stop(); + + if (ran.load() != tasks) + std::fprintf(stderr, "WARNING: batch ran %ld of %ld tasks\n", + ran.load(), tasks); + + double elapsed_ns = std::chrono::duration(t1 - t0).count(); + s.ns_per_dispatch = elapsed_ns / tasks - g_opts.work_us * 1000.0; + s.submit_ns = std::chrono::duration(t_submitted - t0).count() / tasks; + s.vcsw_per_task = static_cast(vc) / tasks; + s.ivcsw_per_task = static_cast(iv) / tasks; + return s; +} + +// ── steady: the task resubmits its successor, as fire_once does ────────────── + +static Sample run_steady(int threads, long tasks) { + ThreadPool pool(threads); + pool.start(); + + std::mutex mx; + std::condition_variable cv; + std::atomic count{0}; + bool finished = false; + // Recursive submission: hold the chain in a std::function so the task can + // resubmit itself. Captured by reference; it outlives the drain below. + // + // The counter is atomic rather than mutex-guarded so that this loop + // measures the pool's dispatch path and not a lock of the benchmark's own. + std::function step = [&] { + busy_us(g_opts.work_us); + long n = count.fetch_add(1, std::memory_order_relaxed) + 1; + if (n < tasks) { + pool.submit(step); + } else { + { std::lock_guard lk(mx); finished = true; } + cv.notify_one(); + } + }; + + bench::RusageDelta ru; ru.start(); + auto t0 = sclock::now(); + pool.submit(step); + { + std::unique_lock lk(mx); + cv.wait(lk, [&] { return finished; }); + } + auto t1 = sclock::now(); + Sample s; + long iv = 0, vc = 0; + ru.finish(iv, vc); + pool.stop(); + + double elapsed_ns = std::chrono::duration(t1 - t0).count(); + s.ns_per_dispatch = elapsed_ns / tasks - g_opts.work_us * 1000.0; + s.vcsw_per_task = static_cast(vc) / tasks; + s.ivcsw_per_task = static_cast(iv) / tasks; + return s; +} + +// ── driver ──────────────────────────────────────────────────────────────────── + +static void run_row(const std::string& mode, int threads, long tasks) { + auto once = [&] { + if (mode == "latency") return run_latency(threads, tasks); + if (mode == "batch") return run_batch(threads, tasks); + return run_steady(threads, tasks); + }; + + for (int i = 0; i < g_opts.warmup; ++i) (void)once(); + + std::vector ns, sub, vcsw, ivcsw; + for (int i = 0; i < g_opts.reps; ++i) { + Sample s = once(); + ns.push_back(s.ns_per_dispatch); + sub.push_back(s.submit_ns); + vcsw.push_back(s.vcsw_per_task); + ivcsw.push_back(s.ivcsw_per_task); + } + + const double med = bench::percentile(ns, 0.5); + const double q1 = bench::percentile(ns, 0.25); + const double q3 = bench::percentile(ns, 0.75); + const double iqr = med > 0 ? 100.0 * (q3 - q1) / med : 0.0; + const double sleeps = bench::percentile(vcsw, 0.5); + + std::fprintf(stderr, "%-9s %-8d %-8d %-10ld %-12.0f %-7.1f %-11.0f %-10.2f %-10.2f\n", + mode.c_str(), threads, g_opts.work_us, tasks, med, iqr, + bench::percentile(sub, 0.5), sleeps, + bench::percentile(ivcsw, 0.5)); + // Column names deliberately match bench_pipeline's key columns so that + // scripts/bench_repro_check.py can gate this benchmark too. + std::printf("%s,%d,%d,%d,%ld,%d,%.1f,%.2f,%.1f,%.3f,%.3f\n", + mode.c_str(), threads, g_opts.work_us, threads, tasks, + g_opts.reps, med, iqr, bench::percentile(sub, 0.5), + sleeps, bench::percentile(ivcsw, 0.5)); + std::fflush(stdout); +} + +static std::vector parse_int_list(const char* s) { + std::vector out; + const char* p = s; + while (*p) { + char* end = nullptr; + long v = std::strtol(p, &end, 10); + if (end == p) break; + out.push_back(static_cast(v)); + p = end; + while (*p == ',' || *p == ' ') ++p; + } + return out; +} + +static std::vector parse_word_list(const std::string& s) { + std::vector out; + std::size_t pos = 0; + while (pos <= s.size()) { + std::size_t c = s.find(',', pos); + if (c == std::string::npos) c = s.size(); + if (c > pos) out.push_back(s.substr(pos, c - pos)); + pos = c + 1; + } + return out; +} + +static void usage() { + std::fprintf(stderr, + "usage: bench_dispatch [options]\n" + " --threads=1,2,4 pool sizes\n" + " --mode=latency,batch,steady which measurements to run\n" + " --tasks=200000 dispatches per repetition\n" + " --work-us=0 payload per task\n" + " --reps=5 --warmup=1\n"); +} + +int main(int argc, char** argv) { + for (int i = 1; i < argc; ++i) { + std::string a = argv[i]; + auto eq = a.find('='); + std::string key = a.substr(0, eq); + std::string val = eq == std::string::npos ? "" : a.substr(eq + 1); + + if (key == "--help" || key == "-h") { usage(); return 0; } + else if (key == "--threads") g_opts.threads = parse_int_list(val.c_str()); + else if (key == "--mode") g_opts.modes = parse_word_list(val); + else if (key == "--tasks") g_opts.tasks = std::atol(val.c_str()); + else if (key == "--work-us") g_opts.work_us = std::atoi(val.c_str()); + else if (key == "--reps") g_opts.reps = std::atoi(val.c_str()); + else if (key == "--warmup") g_opts.warmup = std::atoi(val.c_str()); + else { std::fprintf(stderr, "unknown option: %s\n", a.c_str()); usage(); return 2; } + } + if (g_opts.reps < 1) g_opts.reps = 1; + if (g_opts.warmup < 0) g_opts.warmup = 0; + + char cfg[160]; + std::snprintf(cfg, sizeof cfg, "tasks=%ld work_us=%d reps=%d warmup=%d", + g_opts.tasks, g_opts.work_us, g_opts.reps, g_opts.warmup); + bench::print_environment(cfg); + + std::fprintf(stderr, "\n%-9s %-8s %-8s %-10s %-12s %-7s %-11s %-10s %-10s\n", + "mode", "threads", "work_us", "tasks", "ns/dispatch", "iqr%", + "submit_ns", "vcsw/task", "ivcsw/task"); + std::fprintf(stderr, "%s\n", std::string(96, '-').c_str()); + std::printf("topology,size,work_us,threads,items,reps,ns_per_dispatch," + "iqr_pct,submit_ns,vcsw_per_task,ivcsw_per_task\n"); + + // latency is a round trip per task, so it is far slower per dispatch than + // the other modes; scale it down rather than run for minutes. + for (const auto& mode : g_opts.modes) + for (int t : g_opts.threads) { + long tasks = mode == "latency" + ? std::max(2000L, g_opts.tasks / 20) + : g_opts.tasks; + run_row(mode, t, tasks); + } +} diff --git a/benchmarks/bench_env.hpp b/benchmarks/bench_env.hpp new file mode 100644 index 0000000..92ccc4a --- /dev/null +++ b/benchmarks/bench_env.hpp @@ -0,0 +1,108 @@ +// Shared benchmark plumbing: machine attribution (PERF_PLAN M6), repetition +// statistics (M3), and context-switch capture. +// +// The attribution is not decoration. A result taken under the powersave +// governor or on battery is not comparable with one taken on AC under +// performance, and a stored CSV that does not say which it was cannot be +// argued about later. + +#pragma once + +#include +#include +#include +#include +#include +#include + +#include + +namespace bench { + +inline int hw_units() { + unsigned n = std::thread::hardware_concurrency(); + return n ? static_cast(n) : 1; +} + +inline std::string read_line_of(const char* path) { + std::FILE* f = std::fopen(path, "r"); + if (!f) return "unknown"; + char buf[128] = {0}; + if (!std::fgets(buf, sizeof buf, f)) { std::fclose(f); return "unknown"; } + std::fclose(f); + std::string s(buf); + while (!s.empty() && (s.back() == '\n' || s.back() == ' ')) s.pop_back(); + return s.empty() ? "unknown" : s; +} + +inline std::string ac_state() { + for (const char* p : {"/sys/class/power_supply/AC/online", + "/sys/class/power_supply/AC0/online", + "/sys/class/power_supply/ACAD/online", + "/sys/class/power_supply/ADP1/online"}) { + std::string v = read_line_of(p); + if (v != "unknown") return v == "1" ? "ac" : "battery"; + } + return "unknown"; +} + +// M6 β€” emitted to both streams: the CSV so a stored result can be attributed, +// the terminal so a run under the wrong governor is noticed while it happens. +inline void print_environment(const std::string& config_line) { + const std::string gov = read_line_of( + "/sys/devices/system/cpu/cpu0/cpufreq/scaling_governor"); + const std::string ac = ac_state(); + + for (std::FILE* out : {stdout, stderr}) { + std::fprintf(out, "# nproc=%d governor=%s power=%s\n", + hw_units(), gov.c_str(), ac.c_str()); + if (!config_line.empty()) + std::fprintf(out, "# %s\n", config_line.c_str()); +#if defined(__GNUC__) && !defined(__clang__) + std::fprintf(out, "# compiler=gcc-%d.%d.%d\n", + __GNUC__, __GNUC_MINOR__, __GNUC_PATCHLEVEL__); +#elif defined(__clang__) + std::fprintf(out, "# compiler=clang-%d.%d.%d\n", + __clang_major__, __clang_minor__, __clang_patchlevel__); +#endif + } + if (gov != "performance" || ac == "battery") + std::fprintf(stderr, + "# WARNING: governor=%s power=%s β€” results are not comparable with\n" + "# a run on AC power under the performance governor.\n", + gov.c_str(), ac.c_str()); +} + +inline double percentile(std::vector v, double p) { + if (v.empty()) return 0; + std::sort(v.begin(), v.end()); + double idx = p * (v.size() - 1); + auto lo = static_cast(std::floor(idx)); + auto hi = static_cast(std::ceil(idx)); + return v[lo] + (v[hi] - v[lo]) * (idx - lo); +} + +// Process-wide context-switch counters, sampled around a timed region. +// +// ru_nvcsw (voluntary) is the cheap answer to PERF_PLAN B2: a thread that +// blocks on a condition variable books a voluntary switch, so voluntary +// switches per dispatch is, near enough, sleeps per dispatch. ru_nivcsw +// (involuntary) is preemption, which is what oversubscription looks like (A3). +struct RusageDelta { + long ivcsw0 = 0, vcsw0 = 0; + + void start() { + rusage ru{}; + getrusage(RUSAGE_SELF, &ru); + ivcsw0 = ru.ru_nivcsw; + vcsw0 = ru.ru_nvcsw; + } + void finish(long& nivcsw, long& nvcsw) const { + rusage ru{}; + getrusage(RUSAGE_SELF, &ru); + nivcsw = ru.ru_nivcsw - ivcsw0; + nvcsw = ru.ru_nvcsw - vcsw0; + } +}; + +} // namespace bench diff --git a/benchmarks/bench_pipeline.cpp b/benchmarks/bench_pipeline.cpp index c0e3b09..dc4ed3a 100644 --- a/benchmarks/bench_pipeline.cpp +++ b/benchmarks/bench_pipeline.cpp @@ -9,24 +9,38 @@ // private β€” each node owns a private ThreadPool(1) [Node<>] // pool β€” all nodes share one ThreadPool(T) [PoolNode<> + shared pool] // -// Usage: ./bench_pipeline | tee results.csv +// Each row is run --reps times (plus discarded warm-up runs); the reported +// figure is the median items/sec, with the inter-quartile spread as a +// reliability indicator. A row whose iqr_pct is above a few percent is not +// measuring what it claims to measure. +// +// Usage: ./bench_pipeline [options] | tee results.csv +// ./bench_pipeline --help #include +#include "bench_env.hpp" + #ifdef KPN_BENCH_TBB #include namespace tbb_flow = oneapi::tbb::flow; #endif +#include #include #include #include +#include #include +#include +#include #include #include #include #include +#include + using namespace kpn; using namespace std::chrono_literals; using sclock = std::chrono::steady_clock; @@ -57,31 +71,63 @@ static void push_retry(Channel& ch, int val) { } } -// ── result ──────────────────────────────────────────────────────────────────── +// ── configuration (M1, M3, M4, M5) ──────────────────────────────────────────── -struct Result { - const char* topology; - int size; - int work_us; - int threads; // 0 = private (1 thread per node), N = shared pool size - double items_per_sec; - double overhead_us; +struct Config { + std::vector work_amts {10, 100, 1000}; + std::vector pool_sizes{1, 2, 4, 8, 16, 20}; // M5 + std::vector depths {1, 2, 4, 8, 16, 32}; + std::vector widths {1, 2, 3, 4}; + int reps = 5; // M3: measured repetitions per row + int warmup = 1; // M4: discarded repetitions per row + double target_sec = 0.30; // aimed-for duration of one repetition + long min_items = 2000; // M1: floor, independent of work_us and depth + double max_sec = 3.0; // ceiling; only bites where min_items cannot fit + bool do_chain = true, do_wide = true, do_diamond = true; + bool do_priv = true, do_pool = true, do_tbb = true; +}; + +static Config g_cfg; + +// M1 β€” sample size from a time budget with a hard floor, rather than a +// hand-tuned ladder that collapsed to 50–200 items on exactly the rows under +// investigation. +// +// `stages` is the number of node firings per item; `units` the number of +// threads able to run them concurrently. Steady-state throughput of the +// pipeline is bounded by work_us * stages / units, so that is the per-item +// cost the sample size is derived from. Depth beyond `units` costs throughput; +// depth below it costs only latency, which does not scale the run. +static long pick_items(int work_us, int stages, int units) { + units = std::max(1, std::min(units, bench::hw_units())); + const double per_item_us = + std::max(1.0, static_cast(work_us)) * + std::max(1.0, static_cast(stages) / units); + + long want = static_cast(g_cfg.target_sec * 1e6 / per_item_us); + long cap = static_cast(g_cfg.max_sec * 1e6 / per_item_us); + + want = std::max(want, g_cfg.min_items); + // The floor wins unless honouring it would blow the time ceiling by more + // than the ceiling allows; such rows are reported with their true N so the + // reader can see they are short. + if (want > cap) want = std::max(cap, 200L); + return want; +} + +// ── one measured repetition ─────────────────────────────────────────────────── + +struct Sample { + double items_per_sec = 0; + double overhead_us = 0; + long nivcsw = 0; // involuntary context switches during the run + long nvcsw = 0; // voluntary context switches during the run }; // ── chain ───────────────────────────────────────────────────────────────────── -static int items_for(int work_us, int depth = 1) { - int effective = std::max(1, work_us) * std::max(1, depth); - if (effective <= 1) return 5000; - if (effective <= 10) return 3000; - if (effective <= 100) return 1000; - if (effective <= 1000) return 200; - return 50; -} - -static Result bench_chain(int depth, int work_us) { - const int N = items_for(work_us, depth); - const int CAP = N; +static Sample bench_chain(int depth, int work_us, long N) { + const std::size_t CAP = static_cast(N); std::vector>> chs; for (int i = 0; i <= depth; ++i) @@ -98,17 +144,20 @@ static Result bench_chain(int depth, int work_us) { std::atomic t1; std::thread reader([&] { - for (int i = 0; i < N; ++i) chs.back()->pop(); + for (long i = 0; i < N; ++i) chs.back()->pop(); t1.store(sclock::now(), std::memory_order_release); }); + bench::RusageDelta ru; ru.start(); auto t0 = sclock::now(); std::thread pusher([&] { - for (int i = 0; i < N; ++i) push_retry(*chs[0], i); + for (long i = 0; i < N; ++i) push_retry(*chs[0], static_cast(i)); }); pusher.join(); reader.join(); + Sample s; + ru.finish(s.nivcsw, s.nvcsw); for (auto& n : nodes) n->stop(); double elapsed = std::chrono::duration( @@ -116,13 +165,13 @@ static Result bench_chain(int depth, int work_us) { // Subtract theoretical pipeline fill cost (depth-1)*W so that overhead // reflects only framework latency, not the expected pipeline startup time. double pipeline_us = static_cast(work_us) * (N + depth - 1); - double wus = (elapsed * 1e6 - pipeline_us) / N; - return {"chain", depth, work_us, 0, N / elapsed, wus}; + s.overhead_us = (elapsed * 1e6 - pipeline_us) / N; + s.items_per_sec = N / elapsed; + return s; } -static Result bench_chain_pool(int depth, int work_us, int pool_threads) { - const int N = items_for(work_us, depth); - const int CAP = N; +static Sample bench_chain_pool(int depth, int work_us, int pool_threads, long N) { + const std::size_t CAP = static_cast(N); auto pool = std::make_shared(pool_threads); @@ -142,33 +191,36 @@ static Result bench_chain_pool(int depth, int work_us, int pool_threads) { std::atomic t1; std::thread reader([&] { - for (int i = 0; i < N; ++i) chs.back()->pop(); + for (long i = 0; i < N; ++i) chs.back()->pop(); t1.store(sclock::now(), std::memory_order_release); }); + bench::RusageDelta ru; ru.start(); auto t0 = sclock::now(); std::thread pusher([&] { - for (int i = 0; i < N; ++i) push_retry(*chs[0], i); + for (long i = 0; i < N; ++i) push_retry(*chs[0], static_cast(i)); }); pusher.join(); reader.join(); + Sample s; + ru.finish(s.nivcsw, s.nvcsw); for (auto& n : nodes) n->stop(); pool->stop(); double elapsed = std::chrono::duration( t1.load(std::memory_order_acquire) - t0).count(); double pipeline_us = static_cast(work_us) * (N + depth - 1); - double wus = (elapsed * 1e6 - pipeline_us) / N; - return {"chain", depth, work_us, pool_threads, N / elapsed, wus}; + s.overhead_us = (elapsed * 1e6 - pipeline_us) / N; + s.items_per_sec = N / elapsed; + return s; } // ── wide (fanout) ────────────────────────────────────────────────────────── template -static Result bench_wide(int work_us) { - const int N = items_for(work_us); - const int CAP = N; +static Sample bench_wide(int work_us, long N) { + const std::size_t CAP = static_cast(N); auto src_ch = std::make_shared>(CAP); auto fan = std::make_unique>(CAP); @@ -197,33 +249,36 @@ static Result bench_wide(int work_us) { for (std::size_t w = 0; w < W; ++w) { readers[w] = std::thread([&, w] { - for (int i = 0; i < N; ++i) sink_chs[w]->pop(); + for (long i = 0; i < N; ++i) sink_chs[w]->pop(); if (readers_done.fetch_add(1, std::memory_order_acq_rel) + 1 == static_cast(W)) t1.store(sclock::now(), std::memory_order_release); }); } + bench::RusageDelta ru; ru.start(); auto t0 = sclock::now(); std::thread pusher([&] { - for (int i = 0; i < N; ++i) push_retry(*src_ch, i); + for (long i = 0; i < N; ++i) push_retry(*src_ch, static_cast(i)); }); pusher.join(); for (auto& r : readers) r.join(); + Sample s; + ru.finish(s.nivcsw, s.nvcsw); fan->stop(); for (auto& n : nodes) n->stop(); double elapsed = std::chrono::duration( t1.load(std::memory_order_acquire) - t0).count(); - double wus = (elapsed * 1e6) / N - static_cast(work_us); - return {"wide", static_cast(W), work_us, 0, N / elapsed, wus}; + s.overhead_us = (elapsed * 1e6) / N - static_cast(work_us); + s.items_per_sec = N / elapsed; + return s; } template -static Result bench_wide_pool(int work_us, int pool_threads) { - const int N = items_for(work_us); - const int CAP = N; +static Sample bench_wide_pool(int work_us, int pool_threads, long N) { + const std::size_t CAP = static_cast(N); auto pool = std::make_shared(pool_threads); auto src_ch = std::make_shared>(CAP); @@ -254,35 +309,38 @@ static Result bench_wide_pool(int work_us, int pool_threads) { for (std::size_t w = 0; w < W; ++w) { readers[w] = std::thread([&, w] { - for (int i = 0; i < N; ++i) sink_chs[w]->pop(); + for (long i = 0; i < N; ++i) sink_chs[w]->pop(); if (readers_done.fetch_add(1, std::memory_order_acq_rel) + 1 == static_cast(W)) t1.store(sclock::now(), std::memory_order_release); }); } + bench::RusageDelta ru; ru.start(); auto t0 = sclock::now(); std::thread pusher([&] { - for (int i = 0; i < N; ++i) push_retry(*src_ch, i); + for (long i = 0; i < N; ++i) push_retry(*src_ch, static_cast(i)); }); pusher.join(); for (auto& r : readers) r.join(); + Sample s; + ru.finish(s.nivcsw, s.nvcsw); fan->stop(); for (auto& n : nodes) n->stop(); pool->stop(); double elapsed = std::chrono::duration( t1.load(std::memory_order_acquire) - t0).count(); - double wus = (elapsed * 1e6) / N - static_cast(work_us); - return {"wide", static_cast(W), work_us, pool_threads, N / elapsed, wus}; + s.overhead_us = (elapsed * 1e6) / N - static_cast(work_us); + s.items_per_sec = N / elapsed; + return s; } // ── diamond ─────────────────────────────────────────────────────────────────── -static Result bench_diamond(int work_us) { - const int N = items_for(work_us, 2); - const int CAP = N; +static Sample bench_diamond(int work_us, long N) { + const std::size_t CAP = static_cast(N); auto src_ch = std::make_shared>(CAP); auto fan = std::make_unique>(CAP); @@ -312,7 +370,7 @@ static Result bench_diamond(int work_us) { std::atomic done{0}; auto make_reader = [&](Channel& ch) { return std::thread([&] { - for (int i = 0; i < N; ++i) ch.pop(); + for (long i = 0; i < N; ++i) ch.pop(); if (done.fetch_add(1, std::memory_order_acq_rel) + 1 == 2) t1.store(sclock::now(), std::memory_order_release); }); @@ -320,23 +378,26 @@ static Result bench_diamond(int work_us) { auto rL = make_reader(*snkL); auto rR = make_reader(*snkR); + bench::RusageDelta ru; ru.start(); auto t0 = sclock::now(); std::thread pusher([&] { - for (int i = 0; i < N; ++i) push_retry(*src_ch, i); + for (long i = 0; i < N; ++i) push_retry(*src_ch, static_cast(i)); }); pusher.join(); rL.join(); rR.join(); + Sample s; + ru.finish(s.nivcsw, s.nvcsw); fan->stop(); nL->stop(); nR->stop(); nL2->stop(); nR2->stop(); double elapsed = std::chrono::duration( t1.load(std::memory_order_acquire) - t0).count(); - double wus = (elapsed * 1e6) / N - static_cast(work_us); - return {"diamond", 4, work_us, 0, N / elapsed, wus}; + s.overhead_us = (elapsed * 1e6) / N - static_cast(work_us); + s.items_per_sec = N / elapsed; + return s; } -static Result bench_diamond_pool(int work_us, int pool_threads) { - const int N = items_for(work_us, 2); - const int CAP = N; +static Sample bench_diamond_pool(int work_us, int pool_threads, long N) { + const std::size_t CAP = static_cast(N); auto pool = std::make_shared(pool_threads); auto src_ch = std::make_shared>(CAP); @@ -369,7 +430,7 @@ static Result bench_diamond_pool(int work_us, int pool_threads) { std::atomic done{0}; auto make_reader = [&](Channel& ch) { return std::thread([&] { - for (int i = 0; i < N; ++i) ch.pop(); + for (long i = 0; i < N; ++i) ch.pop(); if (done.fetch_add(1, std::memory_order_acq_rel) + 1 == 2) t1.store(sclock::now(), std::memory_order_release); }); @@ -377,28 +438,30 @@ static Result bench_diamond_pool(int work_us, int pool_threads) { auto rL = make_reader(*snkL); auto rR = make_reader(*snkR); + bench::RusageDelta ru; ru.start(); auto t0 = sclock::now(); std::thread pusher([&] { - for (int i = 0; i < N; ++i) push_retry(*src_ch, i); + for (long i = 0; i < N; ++i) push_retry(*src_ch, static_cast(i)); }); pusher.join(); rL.join(); rR.join(); + Sample s; + ru.finish(s.nivcsw, s.nvcsw); fan->stop(); nL->stop(); nR->stop(); nL2->stop(); nR2->stop(); pool->stop(); double elapsed = std::chrono::duration( t1.load(std::memory_order_acquire) - t0).count(); - double wus = (elapsed * 1e6) / N - static_cast(work_us); - return {"diamond", 4, work_us, pool_threads, N / elapsed, wus}; + s.overhead_us = (elapsed * 1e6) / N - static_cast(work_us); + s.items_per_sec = N / elapsed; + return s; } // ── TBB flow graph ──────────────────────────────────────────────────────────── #ifdef KPN_BENCH_TBB -static Result bench_chain_tbb(int depth, int work_us) { - const int N = items_for(work_us, depth); - +static Sample bench_chain_tbb(int depth, int work_us, long N) { tbb_flow::graph g; using FN = tbb_flow::function_node; std::vector> nodes; @@ -409,21 +472,23 @@ static Result bench_chain_tbb(int depth, int work_us) { for (int i = 0; i + 1 < depth; ++i) tbb_flow::make_edge(*nodes[i], *nodes[i + 1]); + bench::RusageDelta ru; ru.start(); auto t0 = sclock::now(); - for (int i = 0; i < N; ++i) nodes[0]->try_put(i); + for (long i = 0; i < N; ++i) nodes[0]->try_put(static_cast(i)); g.wait_for_all(); auto t1 = sclock::now(); + Sample s; + ru.finish(s.nivcsw, s.nvcsw); - double elapsed = std::chrono::duration(t1 - t0).count(); + double elapsed = std::chrono::duration(t1 - t0).count(); double pipeline_us = static_cast(work_us) * (N + depth - 1); - double wus = (elapsed * 1e6 - pipeline_us) / N; - return {"chain_tbb", depth, work_us, -1, N / elapsed, wus}; + s.overhead_us = (elapsed * 1e6 - pipeline_us) / N; + s.items_per_sec = N / elapsed; + return s; } template -static Result bench_wide_tbb(int work_us) { - const int N = items_for(work_us); - +static Sample bench_wide_tbb(int work_us, long N) { tbb_flow::graph g; tbb_flow::broadcast_node fan(g); using FN = tbb_flow::function_node; @@ -434,19 +499,21 @@ static Result bench_wide_tbb(int work_us) { tbb_flow::make_edge(fan, *n); } + bench::RusageDelta ru; ru.start(); auto t0 = sclock::now(); - for (int i = 0; i < N; ++i) fan.try_put(i); + for (long i = 0; i < N; ++i) fan.try_put(static_cast(i)); g.wait_for_all(); auto t1 = sclock::now(); + Sample s; + ru.finish(s.nivcsw, s.nvcsw); double elapsed = std::chrono::duration(t1 - t0).count(); - double wus = (elapsed * 1e6) / N - static_cast(work_us); - return {"wide_tbb", static_cast(W), work_us, -1, N / elapsed, wus}; + s.overhead_us = (elapsed * 1e6) / N - static_cast(work_us); + s.items_per_sec = N / elapsed; + return s; } -static Result bench_diamond_tbb(int work_us) { - const int N = items_for(work_us, 2); - +static Sample bench_diamond_tbb(int work_us, long N) { tbb_flow::graph g; tbb_flow::broadcast_node fan(g); using FN = tbb_flow::function_node; @@ -456,71 +523,242 @@ static Result bench_diamond_tbb(int work_us) { tbb_flow::make_edge(fan, nL); tbb_flow::make_edge(fan, nR); tbb_flow::make_edge(nL, nL2); tbb_flow::make_edge(nR, nR2); + bench::RusageDelta ru; ru.start(); auto t0 = sclock::now(); - for (int i = 0; i < N; ++i) fan.try_put(i); + for (long i = 0; i < N; ++i) fan.try_put(static_cast(i)); g.wait_for_all(); auto t1 = sclock::now(); + Sample s; + ru.finish(s.nivcsw, s.nvcsw); double elapsed = std::chrono::duration(t1 - t0).count(); - double wus = (elapsed * 1e6) / N - static_cast(work_us); - return {"diamond_tbb", 4, work_us, -1, N / elapsed, wus}; + s.overhead_us = (elapsed * 1e6) / N - static_cast(work_us); + s.items_per_sec = N / elapsed; + return s; } #endif // KPN_BENCH_TBB +// ── repetition driver (M2, M3, M4) ──────────────────────────────────────────── + +using bench::percentile; + +// A row: median of `reps` repetitions, after `warmup` discarded ones. +// M2 β€” items/sec is the primary figure; derived overhead is secondary, +// because it is a difference of large numbers and magnifies noise ~10Γ—. +template +static void run_row(const char* topology, int size, int work_us, int sched, + long N, Fn&& one_rep) { + for (int i = 0; i < g_cfg.warmup; ++i) (void)one_rep(); // M4 + + std::vector ips, ovh; + long ivcsw = 0, vcsw = 0; + for (int i = 0; i < g_cfg.reps; ++i) { + Sample s = one_rep(); + ips.push_back(s.items_per_sec); + ovh.push_back(s.overhead_us); + ivcsw += s.nivcsw; + vcsw += s.nvcsw; + } + + const double med = percentile(ips, 0.5); + const double q1 = percentile(ips, 0.25); + const double q3 = percentile(ips, 0.75); + const double iqr = med > 0 ? 100.0 * (q3 - q1) / med : 0.0; + const double lo = *std::min_element(ips.begin(), ips.end()); + const double hi = *std::max_element(ips.begin(), ips.end()); + const double spread = med > 0 ? 100.0 * (hi - lo) / med : 0.0; + const double ivcsw_per_item = static_cast(ivcsw) / (double(N) * g_cfg.reps); + const double vcsw_per_item = static_cast(vcsw) / (double(N) * g_cfg.reps); + + const std::string s = sched < 0 ? "tbb" + : sched == 0 ? "priv" + : std::to_string(sched); + + std::fprintf(stderr, "%-10s %-5d %-8d %-6s %-8ld %-12.0f %-7.1f %-7.1f %-9.1f %-8.2f %-8.2f\n", + topology, size, work_us, s.c_str(), N, + med, iqr, spread, percentile(ovh, 0.5), ivcsw_per_item, vcsw_per_item); + std::printf("%s,%d,%d,%s,%ld,%d,%.0f,%.0f,%.0f,%.2f,%.2f,%.2f,%.3f,%.3f\n", + topology, size, work_us, s.c_str(), N, g_cfg.reps, + med, lo, hi, iqr, spread, percentile(ovh, 0.5), + ivcsw_per_item, vcsw_per_item); + std::fflush(stdout); +} + +// ── argument parsing ────────────────────────────────────────────────────────── + +static std::vector parse_int_list(const char* s) { + std::vector out; + const char* p = s; + while (*p) { + char* end = nullptr; + long v = std::strtol(p, &end, 10); + if (end == p) break; + out.push_back(static_cast(v)); + p = end; + while (*p == ',' || *p == ' ') ++p; + } + return out; +} + +static bool has_word(const std::string& csv, const char* word) { + return csv.find(word) != std::string::npos; +} + +static void usage() { + std::fprintf(stderr, + "usage: bench_pipeline [options]\n" + " --work=10,100,1000 per-node busy-work, microseconds\n" + " --depths=1,2,4,8,16,32 chain depths\n" + " --widths=1,2,3,4 fanout widths\n" + " --pools=1,2,4,8,16,20 shared-pool thread counts\n" + " --topos=chain,wide,diamond\n" + " --modes=priv,pool,tbb\n" + " --reps=5 measured repetitions per row\n" + " --warmup=1 discarded repetitions per row\n" + " --target-sec=0.30 aimed-for duration of one repetition\n" + " --min-items=2000 sample-size floor\n" + " --max-sec=3.0 per-repetition ceiling (overrides the floor)\n"); +} + +static bool parse_args(int argc, char** argv) { + for (int i = 1; i < argc; ++i) { + std::string a = argv[i]; + auto eq = a.find('='); + std::string key = a.substr(0, eq); + std::string val = eq == std::string::npos ? "" : a.substr(eq + 1); + + if (key == "--help" || key == "-h") { usage(); std::exit(0); } + else if (key == "--work") g_cfg.work_amts = parse_int_list(val.c_str()); + else if (key == "--depths") g_cfg.depths = parse_int_list(val.c_str()); + else if (key == "--widths") g_cfg.widths = parse_int_list(val.c_str()); + else if (key == "--pools") g_cfg.pool_sizes = parse_int_list(val.c_str()); + else if (key == "--reps") g_cfg.reps = std::atoi(val.c_str()); + else if (key == "--warmup") g_cfg.warmup = std::atoi(val.c_str()); + else if (key == "--target-sec") g_cfg.target_sec = std::atof(val.c_str()); + else if (key == "--min-items") g_cfg.min_items = std::atol(val.c_str()); + else if (key == "--max-sec") g_cfg.max_sec = std::atof(val.c_str()); + else if (key == "--topos") { + g_cfg.do_chain = has_word(val, "chain"); + g_cfg.do_wide = has_word(val, "wide"); + g_cfg.do_diamond = has_word(val, "diamond"); + } + else if (key == "--modes") { + g_cfg.do_priv = has_word(val, "priv"); + g_cfg.do_pool = has_word(val, "pool"); + g_cfg.do_tbb = has_word(val, "tbb"); + } + else { std::fprintf(stderr, "unknown option: %s\n", a.c_str()); usage(); return false; } + } + if (g_cfg.reps < 1) g_cfg.reps = 1; + if (g_cfg.warmup < 0) g_cfg.warmup = 0; + return true; +} + +// `wide` is templated on W, so dispatch the runtime width through a switch. +template +static void with_width(int w, F&& f) { + switch (w) { + case 1: f(std::integral_constant{}); break; + case 2: f(std::integral_constant{}); break; + case 3: f(std::integral_constant{}); break; + case 4: f(std::integral_constant{}); break; + default: + std::fprintf(stderr, "width %d not instantiated (1..4 only)\n", w); + } +} + // ── main ────────────────────────────────────────────────────────────────────── -int main() { - const int work_amts[] = {10, 100, 1000}; - const int pool_sizes[] = {1, 2, 4}; +int main(int argc, char** argv) { + // A rejected option must fail loudly: a harness driver that silently got + // no CSV back is worse than one that stops. + if (!parse_args(argc, argv)) return 2; - std::fprintf(stderr, "%-12s %-8s %-10s %-8s %-18s %-20s\n", - "topology", "size", "work_us", "threads", "items/sec", "overhead_us/item"); - std::fprintf(stderr, "%s\n", std::string(78, '-').c_str()); - std::printf("topology,size,work_us,threads,items_per_sec,overhead_us_per_item\n"); + char cfg[192]; + std::snprintf(cfg, sizeof cfg, + "reps=%d warmup=%d target_sec=%.2f min_items=%ld max_sec=%.1f", + g_cfg.reps, g_cfg.warmup, g_cfg.target_sec, + g_cfg.min_items, g_cfg.max_sec); + bench::print_environment(cfg); - auto emit = [](const Result& r) { - std::string sched = r.threads < 0 ? "tbb" - : r.threads == 0 ? "priv" - : std::to_string(r.threads); - std::fprintf(stderr, "%-12s %-8d %-10d %-8s %-18.0f %-20.1f\n", - r.topology, r.size, r.work_us, sched.c_str(), - r.items_per_sec, r.overhead_us); - std::printf("%s,%d,%d,%s,%.0f,%.2f\n", - r.topology, r.size, r.work_us, sched.c_str(), - r.items_per_sec, r.overhead_us); - std::fflush(stdout); - }; + std::fprintf(stderr, "\n%-10s %-5s %-8s %-6s %-8s %-12s %-7s %-7s %-9s %-8s %-8s\n", + "topology", "size", "work_us", "sched", "items", "items/sec", + "iqr%", "range%", "ovh_us", "ivcsw/it", "vcsw/it"); + std::fprintf(stderr, "%s\n", std::string(104, '-').c_str()); + std::printf("topology,size,work_us,threads,items,reps,items_per_sec," + "items_per_sec_min,items_per_sec_max,iqr_pct,range_pct," + "overhead_us_per_item,ivcsw_per_item,vcsw_per_item\n"); - for (int w : work_amts) { + for (int w : g_cfg.work_amts) { g_work_us.store(w, std::memory_order_relaxed); - std::fprintf(stderr, "\n── work_us=%-4d private pools ───────────────────────────────────────\n", w); - for (int d : {1, 2, 4, 8, 16, 32}) emit(bench_chain(d, w)); - emit(bench_wide<1>(w)); - emit(bench_wide<2>(w)); - emit(bench_wide<3>(w)); - emit(bench_wide<4>(w)); - emit(bench_diamond(w)); + if (g_cfg.do_priv) { + std::fprintf(stderr, "\n── work_us=%-4d private pools ──────────────────────\n", w); + if (g_cfg.do_chain) + for (int d : g_cfg.depths) { + long N = pick_items(w, d, d); + run_row("chain", d, w, 0, N, [&] { return bench_chain(d, w, N); }); + } + if (g_cfg.do_wide) + for (int wd : g_cfg.widths) + with_width(wd, [&](auto W) { + long N = pick_items(w, W.value, W.value); + run_row("wide", static_cast(W.value), w, 0, N, + [&] { return bench_wide(w, N); }); + }); + if (g_cfg.do_diamond) { + long N = pick_items(w, 4, 4); + run_row("diamond", 4, w, 0, N, [&] { return bench_diamond(w, N); }); + } + } - for (int pt : pool_sizes) { - std::fprintf(stderr, "\n── work_us=%-4d shared pool (%d thread%s) ─────────────────────────────\n", - w, pt, pt == 1 ? "" : "s"); - for (int d : {1, 2, 4, 8, 16, 32}) emit(bench_chain_pool(d, w, pt)); - emit(bench_wide_pool<1>(w, pt)); - emit(bench_wide_pool<2>(w, pt)); - emit(bench_wide_pool<3>(w, pt)); - emit(bench_wide_pool<4>(w, pt)); - emit(bench_diamond_pool(w, pt)); + if (g_cfg.do_pool) { + for (int pt : g_cfg.pool_sizes) { + std::fprintf(stderr, "\n── work_us=%-4d shared pool (%d thread%s) ───────────\n", + w, pt, pt == 1 ? "" : "s"); + if (g_cfg.do_chain) + for (int d : g_cfg.depths) { + long N = pick_items(w, d, pt); + run_row("chain", d, w, pt, N, + [&] { return bench_chain_pool(d, w, pt, N); }); + } + if (g_cfg.do_wide) + for (int wd : g_cfg.widths) + with_width(wd, [&](auto W) { + long N = pick_items(w, W.value, pt); + run_row("wide", static_cast(W.value), w, pt, N, + [&] { return bench_wide_pool(w, pt, N); }); + }); + if (g_cfg.do_diamond) { + long N = pick_items(w, 4, pt); + run_row("diamond", 4, w, pt, N, + [&] { return bench_diamond_pool(w, pt, N); }); + } + } } #ifdef KPN_BENCH_TBB - std::fprintf(stderr, "\n── work_us=%-4d TBB flow graph ──────────────────────────────────────\n", w); - for (int d : {1, 2, 4, 8, 16, 32}) emit(bench_chain_tbb(d, w)); - emit(bench_wide_tbb<1>(w)); - emit(bench_wide_tbb<2>(w)); - emit(bench_wide_tbb<3>(w)); - emit(bench_wide_tbb<4>(w)); - emit(bench_diamond_tbb(w)); + if (g_cfg.do_tbb) { + std::fprintf(stderr, "\n── work_us=%-4d TBB flow graph ─────────────────────\n", w); + if (g_cfg.do_chain) + for (int d : g_cfg.depths) { + long N = pick_items(w, d, d); + run_row("chain_tbb", d, w, -1, N, + [&] { return bench_chain_tbb(d, w, N); }); + } + if (g_cfg.do_wide) + for (int wd : g_cfg.widths) + with_width(wd, [&](auto W) { + long N = pick_items(w, W.value, W.value); + run_row("wide_tbb", static_cast(W.value), w, -1, N, + [&] { return bench_wide_tbb(w, N); }); + }); + if (g_cfg.do_diamond) { + long N = pick_items(w, 4, 4); + run_row("diamond_tbb", 4, w, -1, N, + [&] { return bench_diamond_tbb(w, N); }); + } + } #endif } } diff --git a/examples/07_python_network/example.py b/examples/07_python_network/example.py index 0eee84e..a10ec41 100644 --- a/examples/07_python_network/example.py +++ b/examples/07_python_network/example.py @@ -30,3 +30,8 @@ net.build() net.start() time.sleep(0.1) net.stop() + +# Drop the network deterministically: it holds the Python callable, which forms +# a reference cycle via globals(). Deleting the global breaks it so the network +# is reclaimed now rather than lingering to interpreter shutdown. +del net diff --git a/examples/08_python_subport/example.py b/examples/08_python_subport/example.py index 1351b5e..4d4600c 100644 --- a/examples/08_python_subport/example.py +++ b/examples/08_python_subport/example.py @@ -1,38 +1,55 @@ """ -08_python_subport β€” tap a C++ node's output from Python using net.read(). +08_python_subport β€” drive a *Python* node from Python via write()/read() taps. Graph: - [ProduceNode] --int--> [DoubleItNode] --int--> (tapped by net.read()) + (fed by net.write()) --int--> [py_triple] --int--> (tapped by net.read()) -The sink is Python: instead of connecting a PrintItNode, we call net.read() -to pull values out of DoubleItNode's output directly into Python. -We also demonstrate net.write() by injecting a value into DoubleItNode's input. +Unlike 07, there is no C++ source or sink here: the only node in the network is +a pure-Python function, py_triple. Python plays *both* the producer and the +consumer by using the subport taps: + + * net.write("py", 0, v) injects v into py_triple's input (Python -> network) + * net.read("py", 0) pulls py_triple's output back out (network -> Python) + +This closes the loop the old version left as a "#todo": a value flows from +Python, through a Python node running inside the network, and back to Python. """ import sys -import time -import threading -sys.path.insert(0, "build/python") +sys.path.insert(0, "build/python") # for `python examples/.../example.py` from repo root import kpn_python as kpn + +def py_triple(x: int) -> int: + return x * 3 + + net = kpn.Network() -net.add("src", kpn.make_produce()) -net.add("dbl", kpn.make_double_it()) +# The whole network is a single Python node with a tapped input and output. +net.add_node("py", py_triple, inputs=["int"], outputs=["int"]) -net.connect("src", 0, "dbl", 0) net.build() net.start() -# Collect a few values from DoubleItNode's output via Python tap +# Push values in from Python and read the Python node's results back out. +inputs = [1, 2, 7, 10, 100] results = [] -for _ in range(5): - val = net.read("dbl", 0) - results.append(val) +for v in inputs: + net.write("py", 0, v) # Python -> py_triple input + results.append(net.read("py", 0)) # py_triple output -> Python net.stop() -print("values read from C++ DoubleItNode output:", results) -assert all(v == 84 for v in results), f"expected all 84, got {results}" -print("all correct (42 * 2 = 84)") +print("inputs written from Python: ", inputs) +print("outputs read from py_triple:", results) + +expected = [v * 3 for v in inputs] +assert results == expected, f"expected {expected}, got {results}" +print("all correct (x * 3 computed by a Python node inside the network)") + +# Drop the network deterministically. The network holds the Python callable, +# which (via globals) forms a reference cycle; deleting the global breaks it so +# the network is reclaimed promptly instead of lingering to interpreter exit. +del net diff --git a/examples/CMakeLists.txt b/examples/CMakeLists.txt index 261f5dd..3a59c1d 100644 --- a/examples/CMakeLists.txt +++ b/examples/CMakeLists.txt @@ -13,6 +13,25 @@ function(kpn_example name) ) endfunction() +# Register a Python example script as a CTest smoke test. Runs the script with +# PYTHONPATH pointing at the freshly-built kpn_python module, so it does not +# depend on the caller's working directory or a hard-coded "build/python" path. +function(kpn_python_example name) + if(NOT KPN_BUILD_PYTHON) + return() + endif() + add_test( + NAME example_${name} + COMMAND ${CMAKE_COMMAND} -E env + "PYTHONPATH=$" + ${Python_EXECUTABLE} ${CMAKE_CURRENT_SOURCE_DIR}/${name}/example.py + ) + set_tests_properties(example_${name} PROPERTIES + TIMEOUT 15 + LABELS examples + ) +endfunction() + kpn_example(01_hello_pipeline) kpn_example(02_named_ports) kpn_example(03_multi_output) @@ -31,7 +50,9 @@ if(KPN_WEB_DEBUG) target_link_libraries(14_debug_hub PRIVATE kpn) kpn_target_enable_web_debug(14_debug_hub) endif() -# 07 and 08 are Python scripts β€” no compiled target needed. +# 07 and 08 are Python scripts β€” no compiled target, but run as smoke tests. +kpn_python_example(07_python_network) +kpn_python_example(08_python_subport) # 09 requires OpenCV β€” only build if found find_package(OpenCV QUIET COMPONENTS core imgproc highgui videoio) diff --git a/include/kpn/branch.hpp b/include/kpn/branch.hpp index fcd53ce..81903ae 100644 --- a/include/kpn/branch.hpp +++ b/include/kpn/branch.hpp @@ -7,12 +7,78 @@ #include #include +#include #include #include #include namespace kpn { +// ── Lossless single-output delivery ─────────────────────────────────────────── +// +// Shared by RouterNode and FilterNode, which each deliver a value to exactly one +// channel. Both previously did +// +// try { ch->push(val); } catch (const ChannelOverflowError&) {} +// +// which discards the value whenever the consumer is behind. 6595e6e made node +// outputs lossless, 28e0667 stopped them parking a worker, and a8cfe73 did the +// same for FanoutNode β€” these two were in none of them, and were the last +// remaining users of the throwing push() on a data path. +// +// A dropped item does not degrade a downstream result, it silently changes one. +// Worse, a dropped *sentinel* wedges the pipeline outright: EOF is what tells +// every downstream node to shut down, and there is nothing after it to retry. +// A filter that passes EOF by predicate but drops it by backpressure is a +// pipeline that never terminates. +// +// So sentinels go out-of-band via push_sentinel (a dedicated slot that consumes +// no ring capacity and cannot overflow), and everything else is retried until +// taken. Like FanoutNode and unlike a pool node, these own a private thread, so +// waiting here costs no scheduler worker and needs no space-callback park. +// stop_flag_ is rechecked every pass so teardown cannot hang on a full output. +// +// `parked` receives the time spent waiting, which the caller charges to blocked +// rather than exec β€” a parked node is idle, and charging it to exec reports the +// node as busy exactly when it is the one being held up. +// +// Returns false if stopped with the value undelivered. +template +bool deliver_one(Channel* ch, T& val, const std::atomic& stop_flag, + duration_t& parked) { + if (is_sentinel_value(val)) { + ch->push_sentinel(std::move(val)); + return true; + } + const auto park_from = clock_t::now(); + for (;;) { + switch (ch->try_push(val)) { + case Channel::PushResult::Taken: + parked = duration_t(clock_t::now() - park_from); + return true; + case Channel::PushResult::Closed: + // Nobody is listening any more; the channel has recorded the + // drop. Retrying would spin until teardown noticed. + parked = duration_t(clock_t::now() - park_from); + return false; + case Channel::PushResult::Full: + break; // fall through to the retry logic + } + if (stop_flag.load(std::memory_order_relaxed)) { + // Teardown with work in hand and the output still full. One last + // throwing push, purely so the channel's own stats record the + // overflow β€” the point of the lossless path is that a loss is never + // invisible, and a silent return here would reintroduce exactly the + // hole this function exists to close. + try { ch->push(std::move(val)); } + catch (const ChannelOverflowError&) {} + parked = duration_t(clock_t::now() - park_from); + return false; + } + std::this_thread::sleep_for(std::chrono::microseconds(50)); + } +} + // ── RouterNode ──────────────────────────────────────────────────────────────── // // Reads one item and pushes it to exactly one of N output channels, chosen by @@ -126,15 +192,20 @@ private: auto t1 = clock_t::now(); auto cpu0 = NodeStats::cpu_now(); + // An out-of-range selector still drops by design (documented on + // the class): the item was routed nowhere, not lost to a full + // channel. Only the latter is what deliver_one exists to stop. std::size_t idx = selector_(val); - if (idx < N && out_channels_[idx]) { - try { out_channels_[idx]->push(val); } - catch (const ChannelOverflowError&) {} - } + duration_t parked{0}; + bool delivered = true; + if (idx < N && out_channels_[idx]) + delivered = deliver_one(out_channels_[idx], val, stop_flag_, parked); auto cpu1 = NodeStats::cpu_now(); auto t2 = clock_t::now(); - stats_.record_exec(duration_t(t2 - t1), duration_t(t1 - t0), cpu0, cpu1); + stats_.record_exec(duration_t(t2 - t1) - parked, + duration_t(t1 - t0) + parked, cpu0, cpu1); + if (!delivered) break; } catch (const ChannelClosedError&) { break; } @@ -261,12 +332,19 @@ private: auto t1 = clock_t::now(); auto cpu0 = NodeStats::cpu_now(); + // A value the predicate rejects is dropped by design and is not + // counted as a processed frame. One it accepts is now delivered + // losslessly β€” including a sentinel, which a filter typically + // passes unconditionally so downstream can shut down, and which + // the old throwing push discarded whenever the output was full. if (pred_(val) && out_ch_) { - try { out_ch_->push(val); } - catch (const ChannelOverflowError&) {} + duration_t parked{0}; + const bool delivered = deliver_one(out_ch_, val, stop_flag_, parked); auto cpu1 = NodeStats::cpu_now(); auto t2 = clock_t::now(); - stats_.record_exec(duration_t(t2 - t1), duration_t(t1 - t0), cpu0, cpu1); + stats_.record_exec(duration_t(t2 - t1) - parked, + duration_t(t1 - t0) + parked, cpu0, cpu1); + if (!delivered) break; } } catch (const ChannelClosedError&) { break; diff --git a/include/kpn/channel.hpp b/include/kpn/channel.hpp index 3f0b05d..4dff40c 100644 --- a/include/kpn/channel.hpp +++ b/include/kpn/channel.hpp @@ -58,6 +58,18 @@ public: ChannelClosedError() : std::runtime_error("channel closed") {} }; +// Nothing available *right now* on a channel that is still open. Distinct from +// ChannelClosedError, which means upstream is finished and never coming back. +// +// Conflating the two is expensive in one direction only: a consumer that reads +// "empty" as "closed" stops a live node permanently, and because a stopping +// node disables its own inputs and outputs, one benign empty read takes the +// rest of the pipeline with it. The reverse costs nothing. +class ChannelEmptyError : public std::runtime_error { +public: + ChannelEmptyError() : std::runtime_error("channel empty") {} +}; + // ── CPU pause hint ──────────────────────────────────────────────────────────── // Signals the CPU that this is a spin-wait loop, improving HT sibling throughput // and preventing branch-predictor thrash on x86. Falls back to a compiler barrier. @@ -124,7 +136,6 @@ public: throw ChannelOverflowError(capacity_); } - const bool was_empty = (t == h); buf_[t & ring_mask_] = make_storage(std::move(value)); tail_.store(t + 1, std::memory_order_release); stats_.record_push(t - h + 1, data_bytes); @@ -132,10 +143,162 @@ public: wake_.fetch_add(1, std::memory_order_release); wake_.notify_one(); - if (was_empty && push_callback_) + // Level-triggered, not edge-triggered β€” see set_push_callback. + if (push_callback_) push_callback_(); } + /// Called when a pop frees a slot in a previously-full ring. + /// + /// The mirror of `set_push_callback`, and it exists for the same reason: + /// a producer must be able to *park* rather than spin. Without it the only + /// lossless option is `push_blocking`, which sleeps inside the caller's + /// thread β€” and when that thread is a scheduler worker, parking it starves + /// every node pinned to it (see the hold-and-wait note on push_sentinel). + void set_space_callback(std::function cb) { space_callback_ = std::move(cb); } + + /// True when a push would currently succeed. Used to close the lost-wakeup + /// race: a producer that parks must re-check after clearing its queued flag, + /// because a space_callback fired in between would otherwise be swallowed. + bool has_space() const { + return tail_.load(std::memory_order_relaxed) - + head_.load(std::memory_order_acquire) < capacity_; + } + + /// Outcome of a non-blocking push. + /// + /// try_push used to return bool, and returned *true* for a closed channel β€” + /// so "delivered" and "discarded because nobody is listening" were the same + /// answer. Both mean "stop trying", which is why the callers were correct, + /// but neither they nor the producer's own accounting could tell a value + /// that arrived from one that was thrown away. Only the channel's drop + /// counter knew. + enum class PushResult { Taken, Full, Closed }; + + /// Non-blocking, lossless push. Returns Full when the ring is full, having + /// changed nothing β€” the caller keeps the value and retries when woken. + PushResult try_push(T& value) { + if (!accepting_.load(std::memory_order_acquire)) { + stats_.record_drop(); + return PushResult::Closed; + } + const std::size_t t = tail_.load(std::memory_order_relaxed); + const std::size_t h = head_.load(std::memory_order_acquire); + if (t - h >= capacity_) return PushResult::Full; + + const std::size_t data_bytes = ChannelDataSize::bytes(value); + buf_[t & ring_mask_] = make_storage(std::move(value)); + tail_.store(t + 1, std::memory_order_release); + stats_.record_push(t - h + 1, data_bytes); + wake_.fetch_add(1, std::memory_order_release); + wake_.notify_one(); + // Level-triggered, not edge-triggered β€” see set_push_callback. + if (push_callback_) push_callback_(); + return PushResult::Taken; + } + + // Lossless push with BACKPRESSURE: if the ring is full, wait for the consumer to + // drain instead of dropping (the throwing push()) β€” the producer just runs slower. + // Use when every value must be delivered (e.g. replaying a dump for scoring, where + // a dropped frame silently corrupts the result). SPSC: only the sole producer may + // call it. Returns false if the channel was disabled while waiting. + bool push_blocking(T value) { + for (;;) { + if (!accepting_.load(std::memory_order_acquire)) { + stats_.record_drop(); + return false; + } + const std::size_t t = tail_.load(std::memory_order_relaxed); + const std::size_t h = head_.load(std::memory_order_acquire); + if (t - h < capacity_) { // space available β†’ normal push + const std::size_t data_bytes = ChannelDataSize::bytes(value); + buf_[t & ring_mask_] = make_storage(std::move(value)); + tail_.store(t + 1, std::memory_order_release); + stats_.record_push(t - h + 1, data_bytes); + wake_.fetch_add(1, std::memory_order_release); + wake_.notify_one(); + // Level-triggered, not edge-triggered β€” see set_push_callback. + if (push_callback_) push_callback_(); + return true; + } + // full: yield briefly and retry (consumer will drain) + std::this_thread::sleep_for(std::chrono::microseconds(50)); + } + } + + // Lossless, non-blocking delivery for a must-deliver control token (EOF). + // + // A sentinel is stored out-of-band β€” in a dedicated slot that does NOT + // consume ring capacity β€” so this can never overflow and never blocks the + // caller. That distinction is essential: each KPN node has a single worker + // thread, so a *blocking* push would park that thread and stop it draining + // its own input, cascading into a hold-and-wait deadlock under backpressure. + // Setting a flag and returning keeps the worker free to keep popping. + // + // The consumer's pop() drains the ring first, then delivers this sentinel, + // preserving ordering (EOF arrives after all data pushed before it). + // + // Only the sole producer may call it (SPSC contract, same as push()). + // + // The slot holds exactly one undelivered token. A second offered before the + // first is taken is refused, not queued and not overwritten: two control + // tokens on one channel means the stream ended twice, which is a caller + // protocol error rather than backpressure, and silently coalescing them + // would hide it. + /// Outcome of offering a sentinel. SlotBusy is a protocol error, not + /// backpressure: it means a second control token was offered while the + /// first was still undelivered, and a channel carries at most one. + enum class SentinelResult { Taken, Closed, SlotBusy }; + + /// Non-consuming form. `value` is left untouched unless the result is + /// Taken, so a refused token is still the caller's to report. + SentinelResult try_push_sentinel(T& value) { + if (!accepting_.load(std::memory_order_acquire)) { + stats_.record_drop(); + return SentinelResult::Closed; + } + // Refuse rather than overwrite. Overwriting lost the first token + // silently, and worse, wrote eof_value_ while the consumer could be + // moving the previous one out of it β€” a data race on the storage, which + // for a shared_ptr payload is a torn refcount rather than a stale read. + // + // Checking here is what makes the slot a correct SPSC handshake: the + // producer is the only writer of eof_value_ and the only one that sets + // has_eof_, the consumer is the only one that clears it, so observing + // false here means the consumer has finished with the storage and will + // not touch it again until this store publishes the next token. + // + // Not counted as a drop, and this is the important part. A source that + // has reached the end of its input keeps being polled and keeps + // returning EOF β€” that is the normal steady state, not an error β€” so a + // token arriving while one is already pending is a *re-offer*, and + // refusing it loses nothing: the pending token carries the same + // meaning and is already on its way. Counting it as a drop made a + // clean run report data loss and exit non-zero. + // + // The cost of that choice, stated plainly: a genuinely distinct second + // token would also be refused silently, and the channel cannot tell the + // two apart. Re-offering is the case that actually occurs here, and the + // delivery guarantee that matters β€” the first token arrives β€” holds + // either way. + if (has_eof_.load(std::memory_order_acquire)) + return SentinelResult::SlotBusy; + eof_value_ = make_storage(std::move(value)); + has_eof_.store(true, std::memory_order_release); + // Wake a consumer blocked in pop(): the sentinel is now deliverable even + // though the ring may be empty. + wake_.fetch_add(1, std::memory_order_release); + wake_.notify_one(); + if (push_callback_) push_callback_(); + return SentinelResult::Taken; + } + + /// Consuming convenience form. Returns false when the token was not stored, + /// whether because the channel is closed or because one is already pending. + bool push_sentinel(T value) { + return try_push_sentinel(value) == SentinelResult::Taken; + } + // Blocking pop. Returns when an item is available. // Throws ChannelClosedError if the channel is disabled (regardless of fill). T pop() { @@ -148,21 +311,37 @@ public: // If empty, spin before sleeping: avoids the futex when the next item // arrives within the spin window (~4 Β΅s at default spin_count=200 on x86). if (h == t) { + // Ring drained β€” deliver any pending out-of-band sentinel (EOF) + // now, so it always arrives after the data pushed before it. + // + // Re-confirm emptiness against a fresh tail_ first: the snapshot + // at the top of the loop may be stale (the producer can push more + // values *and* the sentinel in the window since), and the sentinel + // must never jump ahead of ring values pushed before it. The spin + // and post-spin takes below already reload tail_ on the line above + // them; this is the one take that used the loop-top snapshot. + if (h == tail_.load(std::memory_order_acquire)) { + T s; if (take_sentinel(s)) return s; + } + if (!accepting_.load(std::memory_order_acquire)) throw ChannelClosedError{}; - for (std::size_t s = 0; s < spin_count_; ++s) { + for (std::size_t si = 0; si < spin_count_; ++si) { spin_hint(); t = tail_.load(std::memory_order_acquire); if (t != h) break; + { T s; if (take_sentinel(s)) return s; } if (!accepting_.load(std::memory_order_relaxed)) throw ChannelClosedError{}; } if (h == t) { - // Still empty after spin β€” sleep until push() or disable() fires. - // Re-check tail after loading w to guard against a lost wakeup. + // Still empty after spin β€” sleep until push()/push_sentinel() + // or disable() fires. Re-check tail and the sentinel after + // loading w to guard against a lost wakeup. if (tail_.load(std::memory_order_acquire) != h) continue; + if (has_eof_.load(std::memory_order_acquire)) continue; wake_.wait(w, std::memory_order_relaxed); continue; } @@ -173,6 +352,8 @@ public: throw ChannelClosedError{}; T value = extract(std::move(buf_[h & ring_mask_])); head_.store(h + 1, std::memory_order_release); + // A slot just freed: wake any producer parked on this channel. + if (t - h >= capacity_ && space_callback_) space_callback_(); stats_.record_pop(); return value; } @@ -190,12 +371,19 @@ public: } // Immediate non-blocking pop. Returns false if the ring is empty. + // Once the ring is drained, delivers any pending out-of-band sentinel (EOF) + // so pool nodes β€” which pop only via this path β€” still receive the token. bool try_pop_now(T& out) { const std::size_t h = head_.load(std::memory_order_relaxed); - if (h == tail_.load(std::memory_order_acquire)) return false; + const std::size_t t = tail_.load(std::memory_order_acquire); + if (h == t) + return take_sentinel(out); out = extract(std::move(buf_[h & ring_mask_])); head_.store(h + 1, std::memory_order_release); stats_.record_pop(); + // Pool nodes pop only through here, so this is where a parked producer + // gets woken: the ring was full, and it no longer is. + if (t - h >= capacity_ && space_callback_) space_callback_(); return true; } @@ -212,25 +400,75 @@ public: wake_.notify_all(); } - // Register a callback fired when the queue transitions emptyβ†’non-empty. + // Register a callback fired after every successful push. + // + // It fires on every push, not on the emptyβ†’non-empty transition, and that + // is a correctness requirement rather than a simplification. + // + // The edge version tested `was_empty = (t == h)` using an `h` sampled + // *before* the item was published. A PoolNode consumer decides whether to + // run again from the level (count_ready β†’ approx_size), so the two sides + // could each read the other as stale and both stand down: + // + // producer (push) consumer (PoolNode firing) + // ------------------------ ---------------------------- + // samples t=782, h=781 + // -> was_empty = false, no wake + // pops idx 781, head_ = 782 + // count_ready(): head_==tail_==782 + // -> not ready, gate released to Idle + // tail_.store(783) + // + // The item is in the ring, the node is idle, and no wake is outstanding. + // Worse, the failure is absorbing: every later push now sees a non-empty + // ring, so `was_empty` is false forever and the callback never fires again. + // The node sleeps while its backlog grows and its consumer waits on it. + // + // Re-reading head_ after the tail_ store does not fix it. That is the + // store-buffer pattern, and under acquire/release both sides may legally + // read stale; forbidding it needs seq_cst on the producer's tail_ store and + // head_ load *and* on the consumer's head_ store and tail_ load β€” a fence + // on both hot paths. Firing unconditionally is correct by construction: + // the callback runs after the publishing store, so a consumer that observes + // the level at all observes the item. + // + // The redundant wakes are cheap. on_input_ready re-checks the level, and + // SubmitGate::claim() collapses a wake arriving during a firing into the + // firing already in flight, so the cost is one CAS, not one extra run. void set_push_callback(std::function cb) { push_callback_ = std::move(cb); } - // Size derived lazily from ring indices β€” no separate counter on the hot path. + // Ring occupancy, derived lazily from indices β€” no separate counter on the + // hot path. Excludes any out-of-band sentinel (that lives outside the ring). + // head_ is loaded first, deliberately. Both indices only ever increase, so + // reading head_ before tail_ can at worst under-report a concurrent push; + // the other order can read a head_ that has advanced past the tail_ already + // sampled, and the unsigned difference then wraps to ~2^64. A caller + // polling "is this channel empty yet" against that value never terminates. std::size_t size() const { - return tail_.load(std::memory_order_relaxed) - - head_.load(std::memory_order_relaxed); + const std::size_t h = head_.load(std::memory_order_relaxed); + const std::size_t t = tail_.load(std::memory_order_acquire); + return t - h; + } + + // A pending out-of-band sentinel (EOF) counts as consumable work here even + // though it holds no ring slot. This is what node readiness checks call, so + // a channel carrying only a sentinel still schedules its consumer's next + // fire β€” without this the sentinel would never be popped and the pipeline + // would deadlock at teardown. + std::size_t approx_size() const { + return size() + (has_eof_.load(std::memory_order_acquire) ? 1u : 0u); } - std::size_t approx_size() const { return size(); } std::size_t capacity() const { return capacity_; } bool is_accepting() const { return accepting_.load(std::memory_order_relaxed); } const ChannelStats& stats() const { return stats_; } ChannelSnapshot snapshot(const std::string& name) const { - const std::size_t t = tail_.load(std::memory_order_relaxed); + // head_ before tail_, for the reason given on size(). const std::size_t h = head_.load(std::memory_order_relaxed); + const std::size_t t = tail_.load(std::memory_order_acquire); return { name, capacity_, @@ -260,15 +498,53 @@ private: return *s; } + // Consume the out-of-band sentinel if one is pending. Consumer-only. + // Called only when the ring is observed empty, so the sentinel is always + // delivered after every value pushed before it. + bool take_sentinel(T& out) { + if (!has_eof_.load(std::memory_order_acquire)) return false; + // Re-check emptiness *after* observing has_eof_, not before. + // + // Callers check the ring is empty and then call this, but the producer + // can push a value and publish the sentinel in the window between those + // two steps β€” so the sentinel would be delivered with a real value still + // queued behind it, breaking the "sentinel is strictly last" contract + // that downstream teardown depends on. a0c4bf5 closed the variant where + // the caller's emptiness check used a stale tail_ snapshot; this is the + // one where the check is fresh but simply too early. + // + // Checking here is what makes it sound: the producer publishes the + // sentinel with a release store *after* its ring pushes, so a consumer + // that has observed has_eof_ has also observed every tail_ advance + // before it. If the ring is non-empty now, those values genuinely + // precede the sentinel and must be delivered first. + if (head_.load(std::memory_order_relaxed) + != tail_.load(std::memory_order_acquire)) + return false; + out = extract(std::move(eof_value_)); + has_eof_.store(false, std::memory_order_release); + stats_.record_pop(); + return true; + } + const std::size_t capacity_; const std::size_t spin_count_; std::size_t ring_mask_; std::unique_ptr buf_; std::function push_callback_; + std::function space_callback_; ChannelStats stats_; + // Out-of-band sentinel (EOF): stored outside the ring so its delivery never + // depends on ring capacity and never blocks the producer. Written by the + // producer (push_sentinel), read+cleared by the consumer (take_sentinel); + // has_eof_ is the publish/consume handshake. + storage_type eof_value_{}; + std::atomic has_eof_{false}; + // Separate cache lines: head_ is written only by the consumer; // tail_ and wake_ are written only by the producer. + // wake_ wakes a blocked pop() on enqueue or on a pending sentinel. alignas(64) std::atomic head_{0}; alignas(64) std::atomic tail_{0}; std::atomic wake_{0}; diff --git a/include/kpn/diagnostics.hpp b/include/kpn/diagnostics.hpp index f5080b1..884d668 100644 --- a/include/kpn/diagnostics.hpp +++ b/include/kpn/diagnostics.hpp @@ -51,6 +51,14 @@ struct NodeStats { std::atomic max_exec_us{0}; std::atomic total_blocked_us{0}; + // Cumulative wall time inside fire_once, summed over every invocation. + // The EMA above cannot be turned into a total: it is exponentially + // weighted, so frames * ema_exec_us tracks the tail of the run rather than + // the whole of it, and on a workload whose per-frame cost varies (a face + // detector on a film: crowd scenes then empty landscapes) the two differ by + // a lot. Answering "how much time went into this node" needs a real sum. + std::atomic total_exec_us{0}; + // Thread CPU time β€” actual CPU consumed by this node's thread, // measured via CLOCK_THREAD_CPUTIME_ID. Excludes time sleeping or // blocked on mutexes/channels. Sampled once per frame. @@ -89,6 +97,7 @@ struct NodeStats { frames_processed.fetch_add(1, std::memory_order_relaxed); int64_t us = static_cast(exec_time.count() * 1000.0); + total_exec_us.fetch_add(us, std::memory_order_relaxed); uint64_t n = frames_processed.load(std::memory_order_relaxed); int64_t prev = ema_exec_us.load(std::memory_order_relaxed); @@ -147,6 +156,29 @@ struct NodeSnapshot { double total_cpu_ms; // cumulative CPU time consumed by this node's thread double cpu_util_pct; // exec_ms / (exec_ms + blocked_ms) * 100 double queue_wait_ms{0}; // PoolNode: cumulative time spent in pool queue + + // Cumulative wall time inside fire_once. Unlike ema_exec_ms this is a true + // sum, so it is the field to use for "share of the run spent in this node". + // Note it still includes time parked pushing into a full output channel; + // total_cpu_ms is the part that backpressure cannot inflate. + // + // Declared before the two bools below because every node type initialises + // this aggregate positionally, and all of them supply total_exec_ms as the + // element after queue_wait_ms. + double total_exec_ms{0}; + + // Live scheduling state, for observing the AR-004 invariant "a node never + // sleeps with a wake outstanding". The invariant was previously asserted in + // comments but invisible at runtime, so a lost wake could only be found in a + // debugger β€” and this bug does not reproduce under one (it needs full speed). + // Two atomic loads at snapshot time, nothing on the hot path. + // + // Read them together with the node's channel fill: + // queued=0, wake=1 -> wake recorded and never consumed + // queued=0, wake=0, input full -> wake never generated at all + // queued=1 while nothing running -> submitted but never scheduled + bool queued{false}; + bool wake_pending{false}; }; // ── Pool statistics + snapshot ──────────────────────────────────────────────── @@ -188,6 +220,11 @@ struct ResourceSnapshot { struct IResourceProbe { virtual ~IResourceProbe() = default; virtual ResourceSnapshot snapshot(const std::string& name) const = 0; + + /// Release every thread waiting for the resource, so teardown is not held + /// up by one. A network calls this on the resources registered with it when + /// it halts; default no-op for probes with nothing to wake. + virtual void close() {} }; } // namespace kpn diff --git a/include/kpn/fanout.hpp b/include/kpn/fanout.hpp index 180b602..f9f2c46 100644 --- a/include/kpn/fanout.hpp +++ b/include/kpn/fanout.hpp @@ -7,8 +7,10 @@ #include #include +#include #include #include +#include #include #include #include @@ -78,7 +80,9 @@ public: blocked_ms, elapsed_s > 0 ? frames / elapsed_s : 0.0, stats_.total_cpu_us.load(std::memory_order_relaxed) / 1000.0, - total_ms > 0 ? 100.0 * exec_ms / total_ms : 0.0}; + total_ms > 0 ? 100.0 * exec_ms / total_ms : 0.0, + 0.0, // queue_wait_ms β€” fanout is not pool-scheduled + stats_.total_exec_us.load(std::memory_order_relaxed) / 1000.0}; } // ── Port access ─────────────────────────────────────────────────────────── @@ -116,6 +120,79 @@ public: } private: + // Deliver `val` to every connected output, losslessly. + // + // Previously a full output cost the value: push() threw and the exception was + // swallowed per output. A dropped item does not degrade a downstream result, + // it silently changes one, and the consumer cannot tell it happened β€” so the + // fanout waits instead, and the producer upstream runs slower. + // + // Unlike a pool node, a fanout owns a private thread, so waiting here costs + // no scheduler worker and needs no space-callback park; a bounded retry is + // enough. `stop_flag_` is re-checked every pass so teardown cannot hang on a + // full output regardless of the order the network stops its nodes in. + // + // Outputs are retried independently, so a full output never delays delivery + // to one with room. Note what that does *not* buy: the next input is not + // popped until every output has accepted the current item, so one branch can + // never run ahead of another by more than the slower branch's buffering. + // + // **That bound is a precondition on any topology where the branches rejoin.** + // If a consumer on branch B blocks waiting for something branch A computes, + // B's buffering must exceed the lead A needs, or the two wedge β€” B waiting on + // A, A starved because the fanout is holding an item B will not take. Making + // the fanout lossless is what puts that precondition on the topology; while + // it dropped, the question could not arise. + // + // `parked` receives the time spent waiting on a full output, which the caller + // charges to blocked rather than exec. + // + // Returns false if stopped with the value undelivered. + bool deliver(const T& val, duration_t& parked) { + std::array, N> pending; + std::size_t outstanding = 0; + for (std::size_t i = 0; i < N; ++i) + if (out_channels_[i]) { pending[i].emplace(val); ++outstanding; } + + bool first_pass = true; + auto park_from = clock_t::now(); + + for (;;) { + for (std::size_t i = 0; i < N; ++i) { + if (!pending[i]) continue; + // Taken or Closed both mean "stop trying" β€” delivered, or gone + // with the drop recorded. Only Full is worth another pass. + if (out_channels_[i]->try_push(*pending[i]) + != Channel::PushResult::Full) { + pending[i].reset(); + --outstanding; + } + } + if (first_pass) { park_from = clock_t::now(); first_pass = false; } + + if (outstanding == 0) { + parked = duration_t(clock_t::now() - park_from); + return true; + } + if (stop_flag_.load(std::memory_order_relaxed)) { + // Teardown with work in hand. One last throwing push per + // outstanding output, purely so the channel's own stats record + // the loss (drop if it is disabled, overflow if it is merely + // full). The whole point of the lossless path is that a loss is + // never invisible, and a silent `return` here would reintroduce + // exactly the hole this function exists to close. + for (std::size_t i = 0; i < N; ++i) { + if (!pending[i]) continue; + try { out_channels_[i]->push(std::move(*pending[i])); } + catch (const ChannelOverflowError&) {} + } + parked = duration_t(clock_t::now() - park_from); + return false; + } + std::this_thread::sleep_for(std::chrono::microseconds(50)); + } + } + void run_loop() { while (!stop_flag_.load(std::memory_order_relaxed)) { try { @@ -124,16 +201,17 @@ private: auto t1 = clock_t::now(); auto cpu0 = NodeStats::cpu_now(); - for (std::size_t i = 0; i < N; ++i) { - if (out_channels_[i]) { - try { out_channels_[i]->push(val); } - catch (const ChannelOverflowError&) {} // drop for this output independently - } - } + duration_t parked{0}; + const bool delivered = deliver(val, parked); auto cpu1 = NodeStats::cpu_now(); auto t2 = clock_t::now(); - stats_.record_exec(duration_t(t2 - t1), duration_t(t1 - t0), cpu0, cpu1); + // Time spent waiting on a full output is *blocked*, not exec: a + // parked fanout is idle, and charging it to exec would report the + // node as busy exactly when it is the one being held up. + stats_.record_exec(duration_t(t2 - t1) - parked, + duration_t(t1 - t0) + parked, cpu0, cpu1); + if (!delivered) break; } catch (const ChannelClosedError&) { break; } diff --git a/include/kpn/inode.hpp b/include/kpn/inode.hpp index 56f4b89..1ad9747 100644 --- a/include/kpn/inode.hpp +++ b/include/kpn/inode.hpp @@ -22,6 +22,23 @@ enum class NodeEvent { Overflow, Closed }; struct INode { virtual ~INode() = default; + + // Install channel callbacks, without starting anything. + // + // A node's push/space callbacks live in std::function members on channels + // it shares with its neighbours, and a neighbour that is already running + // reads them on its own thread. Writing one while the pipeline runs is a + // data race on the std::function β€” ThreadSanitizer reports it, and the + // consequence in the field was the missed startup wake a8cfe73 had to + // patch around. + // + // So a network calls prepare() on every node before it calls start() on + // any of them: all the writes happen while nothing is running, and once a + // node is live the callbacks are read-only. start() calls prepare() itself + // if it has not been called, so standalone nodes still work; it is + // idempotent, and the network relies on that. + virtual void prepare() {} + virtual void start() = 0; virtual void stop() = 0; virtual bool running() const = 0; @@ -34,6 +51,14 @@ struct INode { virtual void set_network_overflow_callback(NodeEventCallback) {} virtual void set_network_closed_callback(NodeEventCallback) {} + // Network-level error listener. Consulted when a node's function throws + // and no per-node handler resolved it. Without this the exception is + // discarded and the failure is only visible as a Closed event, which says + // a node stopped but not why β€” the difference between a diagnosis and a + // guess. Same contract as NodeErrorHandler: true to continue, false to + // stop the node. + virtual void set_network_error_callback(NodeErrorHandler) {} + // halt(): alias for stop() β€” immediate, discards in-flight work. virtual void halt() { stop(); } diff --git a/include/kpn/interrupt_node.hpp b/include/kpn/interrupt_node.hpp index 6e58c24..0b8c830 100644 --- a/include/kpn/interrupt_node.hpp +++ b/include/kpn/interrupt_node.hpp @@ -104,6 +104,7 @@ public: stats_.total_cpu_us.load(std::memory_order_relaxed) / 1000.0, total_ms > 0 ? 100.0 : 0.0, qwait_ms, + stats_.total_exec_us.load(std::memory_order_relaxed) / 1000.0, }; } diff --git a/include/kpn/main_thread_node.hpp b/include/kpn/main_thread_node.hpp index 47ecb18..17ff222 100644 --- a/include/kpn/main_thread_node.hpp +++ b/include/kpn/main_thread_node.hpp @@ -90,6 +90,8 @@ public: elapsed_s > 0 ? frames / elapsed_s : 0.0, stats_.total_cpu_us.load(std::memory_order_relaxed) / 1000.0, total_ms > 0 ? 100.0 * exec_ms / total_ms : 0.0, + 0.0, // queue_wait_ms β€” main-thread node is not pool-scheduled + stats_.total_exec_us.load(std::memory_order_relaxed) / 1000.0, }; } diff --git a/include/kpn/network.hpp b/include/kpn/network.hpp index 239ad88..5974e70 100644 --- a/include/kpn/network.hpp +++ b/include/kpn/network.hpp @@ -8,8 +8,10 @@ #include #endif +#include #include #include +#include #include #include #include @@ -39,8 +41,15 @@ public: class Network : public INode { public: - using ErrorHandler = - std::function; + /// Application-level error listener. Receives the exception any node's + /// function throws, after that node's own handler (if any) declined it. + /// Return true to skip the failed invocation and keep the node running, + /// false to let it stop. + /// + /// Same type as StaticNetwork's, deliberately: this used to be a void + /// signature, which could not express the keep-running decision and, more + /// to the point, was never delivered anywhere. + using ErrorHandler = NodeErrorHandler; using DiagnosticsHandler = std::function&, const std::vector&)>; @@ -91,6 +100,7 @@ public: + " β†’ " + dst_name + ":" + std::to_string(DstIdx); channel_probes_.push_back( std::make_unique>(in_ch, ch_name)); + channel_src_names_.push_back(src_name); adj_[src_name].push_back(dst_name); return *this; @@ -134,6 +144,17 @@ public: void start() override { start_time_ = clock_t::now(); + // Deliver the listener to the nodes. Without this the handler was + // stored and never read: a node's exception was discarded at the node + // boundary and the only surviving evidence was a Closed event, which + // says a node stopped but not why. StaticNetwork has always done this; + // Network accepted the handler and silently dropped it. + if (error_handler_) + for (auto& name : topo_) + nodes_.at(name)->set_network_error_callback(error_handler_); + // Callbacks first, everywhere, before anything runs β€” see INode::prepare. + for (auto& name : topo_) + nodes_.at(name)->prepare(); for (auto& name : topo_) nodes_.at(name)->start(); start_watchdog(); @@ -210,6 +231,11 @@ public: watchdog_interval_ = interval; } + /// How long shutdown() waits for one node's outputs to drain before giving + /// up on them and stopping the next layer anyway. + void set_drain_timeout(std::chrono::milliseconds t) { drain_timeout_ = t; } + + /// Must be called before start(); the handler is delivered to nodes there. void set_error_handler(ErrorHandler h) { error_handler_ = std::move(h); } void set_diagnostics_handler(DiagnosticsHandler h) { diag_handler_ = std::move(h); } void set_event_handler(EventHandler h) { event_handler_ = std::move(h); } @@ -367,19 +393,46 @@ private: return true; } - void drain_output_channels(const std::string& /*name*/) const { - // Poll all channel probes until none report non-zero fill. - // A short sleep prevents busy-spin; 1 ms is fine for drain purposes. - bool any_full = true; - while (any_full) { - any_full = false; - for (auto& probe : channel_probes_) { - auto snap = probe->snapshot(); - if (snap.current_fill > 0) { any_full = true; break; } - } - if (any_full) - std::this_thread::sleep_for(std::chrono::milliseconds(1)); + /// Wait for the channels fed by `name` to empty, or give up. + /// + /// This took a node name and ignored it, polling *every* channel in the + /// graph instead β€” so shutdown() waited for the whole network to be idle + /// before stopping each successive layer. With no deadline either, anything + /// wedged downstream turned a graceful shutdown into the hang it exists to + /// avoid. + /// + /// Two bounds, because they fail differently. The deadline covers a + /// consumer that has stopped consuming, where fill never changes and + /// waiting cannot help. The no-progress counter covers one that is merely + /// slow: it keeps waiting while the queue is shrinking, so a slow drain is + /// not cut short just for taking a while. + void drain_output_channels(const std::string& name) const { + const auto deadline = clock_t::now() + drain_timeout_; + std::size_t last_fill = static_cast(-1); + int stalls = 0; + + auto fill_of = [&] { + std::size_t fill = 0; + for (std::size_t i = 0; i < channel_probes_.size(); ++i) + if (channel_src_names_[i] == name) + fill += channel_probes_[i]->snapshot().current_fill; + return fill; + }; + + for (;;) { + const std::size_t fill = fill_of(); + if (fill == 0) return; + if (fill >= last_fill) { if (++stalls > 100) break; } + else { stalls = 0; } + last_fill = fill; + if (clock_t::now() >= deadline) break; + std::this_thread::sleep_for(std::chrono::milliseconds(1)); } + + if (const std::size_t left = fill_of()) + std::cerr << "[kpn] shutdown: '" << name << "' still has " << left + << " queued item(s) its consumer did not take; " + "they are discarded\n"; } // ── Cycle detection / topological sort ─────────────────────────────────── @@ -398,9 +451,20 @@ private: void start_watchdog() { watchdog_ = std::jthread([this](std::stop_token tok) { + // Interruptible wait, not sleep_for. request_stop() cannot wake a + // sleeping thread, so stop_watchdog()'s join blocked for up to a + // full interval β€” three seconds by default, and unbounded for + // anyone who set a long one to keep the periodic report quiet. + // Every teardown paid it. + std::mutex m; + std::condition_variable_any cv; while (!tok.stop_requested()) { - std::this_thread::sleep_for(watchdog_interval_); - if (tok.stop_requested()) break; + { + std::unique_lock lk(m); + if (cv.wait_for(lk, tok, watchdog_interval_, + [&tok] { return tok.stop_requested(); })) + break; + } auto s = collect_snapshots(); check_hung_nodes(); @@ -444,6 +508,10 @@ private: std::map exposed_outputs_; std::set> connected_outputs_; std::vector> channel_probes_; + /// Name of the node feeding each probe, parallel to channel_probes_. + /// shutdown() drains a node's own outputs, so it has to know which they are. + std::vector channel_src_names_; + std::chrono::milliseconds drain_timeout_{5000}; std::vector> pool_probes_; ErrorHandler error_handler_; DiagnosticsHandler diag_handler_; diff --git a/include/kpn/pool_node.hpp b/include/kpn/pool_node.hpp index e4ceb54..45115aa 100644 --- a/include/kpn/pool_node.hpp +++ b/include/kpn/pool_node.hpp @@ -5,6 +5,7 @@ #include "inode.hpp" #include "port.hpp" #include "scheduler.hpp" +#include "submit_gate.hpp" #include "traits.hpp" #include @@ -15,6 +16,7 @@ #include #include #include +#include #include #include #include @@ -22,12 +24,15 @@ namespace kpn { +// Sentinel detection (has_eof_field / is_sentinel_value) lives in traits.hpp β€” +// every node type that forwards values needs it, not just pool-scheduled ones. + // ── PoolNode ────────────────────────────────────────────────────────────────── // // Reactive alternative to Node<>. Instead of owning a blocked thread, the node // is submitted to a shared IScheduler whenever all its input channels become // non-empty. A single fire_once() call pops all inputs, executes the function, -// and pushes outputs. At most one fire_once() runs at a time (queued_ flag). +// and pushes outputs. At most one fire_once() runs at a time (see SubmitGate). // // Source nodes (input_count == 0) submit themselves immediately on start() and // resubmit after each fire_once(). @@ -76,21 +81,43 @@ public: // ── INode ───────────────────────────────────────────────────────────────── + void prepare() override { + if (prepared_) return; // idempotent: the network calls this, + prepared_ = true; // and start() calls it again if not. + register_callbacks(std::make_index_sequence{}); + } + void start() override { + prepare(); enable_inputs(std::make_index_sequence{}); stop_flag_.store(false, std::memory_order_relaxed); - queued_.store(false, std::memory_order_relaxed); - register_callbacks(std::make_index_sequence{}); + gate_.force_idle(); if constexpr (input_count == 0) try_submit(0.5f); + else + // Never start with a wake already outstanding β€” the startup case of + // the invariant 9c5ce5f established for the running pipeline. + // + // The callback is installed by prepare(), before any node runs, but + // a network still starts its nodes one at a time: an upstream node + // that is already firing can push into this one between the two + // calls. The push is accepted by the ring and does invoke the + // callback, but on_input_ready() sees stop_flag_ still set and + // returns. Every later push sees a non-empty ring and stays silent + // β€” Channel invokes push_callback_ only on the empty->non-empty + // transition β€” so without this the node is never submitted and the + // pipeline reads as wedged from the first frame. + // + // on_input_ready() is the level-triggered form of the same + // question, so asking it once here converts the missed edge into a + // state check. + on_input_ready(); } void stop() override { stop_flag_.store(true, std::memory_order_seq_cst); disable_inputs(std::make_index_sequence{}); - // fire_once() observes stop_flag_ and will not resubmit. - // We do not wait for an in-flight fire_once() to complete here; - // callers that need that guarantee should call scheduler_->drain() first. + await_quiescence(); } bool running() const override { @@ -99,6 +126,7 @@ public: void set_name(std::string name) override { name_ = std::move(name); } void set_error_handler(NodeErrorHandler h) { error_handler_ = std::move(h); } + void set_network_error_callback(NodeErrorHandler h) override { net_error_handler_ = std::move(h); } void set_max_exec_time(std::chrono::milliseconds t) { max_exec_time_ = t; } void set_overflow_callback(NodeEventCallback cb) { event_callbacks_[0] = std::move(cb); } @@ -122,6 +150,9 @@ public: stats_.total_cpu_us.load(std::memory_order_relaxed) / 1000.0, total_ms > 0 ? 100.0 * exec_ms / total_ms : 0.0, qwait_ms, + stats_.total_exec_us.load(std::memory_order_relaxed) / 1000.0, + gate_.queued(), + gate_.wake_pending(), }; } @@ -202,6 +233,8 @@ private: template void register_callbacks(std::index_sequence) { + // A parked producer is re-submitted when its output drains. + register_space_callbacks(std::make_index_sequence{}); (std::get(input_channels_)->set_push_callback( [this] { on_input_ready(); }), ...); } @@ -211,11 +244,27 @@ private: for (auto& cb : cbs) if (cb) cb(ts); } + template + bool outputs_have_space(std::index_sequence) const { + return (... && (!std::get(output_channels_) || + std::get(output_channels_)->has_space())); + } + + template + void register_space_callbacks(std::index_sequence) { + ((std::get(output_channels_) + ? (void)std::get(output_channels_)->set_space_callback( + [this] { try_submit(0.5f); }) + : (void)0), ...); + } + void self_stop() { disable_inputs(std::make_index_sequence{}); disable_outputs(std::make_index_sequence{}); stats_.exec_start_us.store(0, std::memory_order_relaxed); - queued_.store(false, std::memory_order_release); + // force_idle, not finish_firing(): this node is stopping, and + // honouring a pending wake here would resubmit a dead node. + gate_.force_idle(); stop_flag_.store(true, std::memory_order_relaxed); } @@ -248,11 +297,51 @@ private: return ((std::get(input_channels_)->approx_size() > 0 ? 1u : 0u) + ...); } + /// Priority in [0,1], higher runs sooner. + /// + /// Input fill alone answers "how much work is waiting for me". That is + /// only half the question: a node whose *outputs* are already full cannot + /// deliver anything: running it produces a value with nowhere to go, so it + /// immediately parks and the slot is wasted. Meanwhile the node that would + /// have drained that full channel waits behind it. + /// + /// So occupancy of the outputs is deducted from occupancy of the inputs. + /// The scheduler then naturally favours whoever is furthest downstream of + /// a bottleneck β€” the node whose inputs are backed up but whose outputs + /// have room is exactly the one whose execution frees the most capacity β€” + /// and defers producers that would only deepen a queue that is already + /// full. + /// + /// Mapped as 0.5Β·(1 + in - out) rather than clamping (in - out) at zero: + /// both terms are mean fills in [0,1], so the difference is in [-1,1], and + /// the affine map keeps the whole range distinguishable instead of + /// collapsing every output-saturated node onto the same value. 0.5 remains + /// the neutral point, matching the default used for source nodes. float compute_priority() { if constexpr (input_count == 0) return 0.5f; - float sum = 0.0f; - sum_fill(sum, std::make_index_sequence{}); - return sum / static_cast(input_count); + float in = 0.0f; + sum_fill(in, std::make_index_sequence{}); + in /= static_cast(input_count); + + if constexpr (output_count == 0) return in; + + float out = 0.0f; + sum_output_fill(out, std::make_index_sequence{}); + out /= static_cast(output_count); + + const float p = 0.5f * (1.0f + in - out); + return p < 0.0f ? 0.0f : (p > 1.0f ? 1.0f : p); + } + + /// Mean fill of the output channels, same normalisation as sum_fill. + /// An unconnected output holds nothing back, so it contributes 0. + template + void sum_output_fill(float& sum, std::index_sequence) { + ((sum += (std::get(output_channels_) && + std::get(output_channels_)->capacity() > 0) + ? float(std::get(output_channels_)->approx_size()) + / float(std::get(output_channels_)->capacity()) + : 0.0f), ...); } template @@ -263,17 +352,121 @@ private: : 0.5f), ...); } + /// Submit unless a firing is already in flight. A wake that arrives while + /// one is is *recorded* against it, never dropped. + /// + /// Wakes are edge-triggered: a channel fires its space callback on the + /// transition, once. A dropped one never returns, so a node could park a + /// value, release its worker, and sleep forever holding output its consumer + /// was waiting for, with every worker idle in cond_wait and nothing left to + /// re-trigger it. SubmitGate makes "idle" and "wake outstanding" the same + /// variable, so the two cannot both be true β€” see submit_gate.hpp. void try_submit(float priority) { - bool expected = false; - if (queued_.compare_exchange_strong(expected, true, std::memory_order_acq_rel)) + // A stopped node must not claim the gate. The scheduler now refuses + // submissions after its pool stops, so the submit itself is safe β€” but + // claiming and never releasing would leave the gate held, and a restart + // would then have to clear it. start() does, but relying on that makes + // the invariant depend on a distant statement. + if (stop_flag_.load(std::memory_order_relaxed)) return; + if (gate_.claim()) scheduler_->submit([this] { fire_once(); }, priority); } // ── Execution ───────────────────────────────────────────────────────────── + /// Decide whether this node should run again, then release the gate β€” in + /// that order, always. + /// + /// Releasing first is what let two firings of the same node overlap: the + /// moment the gate is free another worker may enter fire_once, while this + /// invocation is still reading pending_ and writing pending_done_. TSan + /// caught it as a race on pending_done_ between a firing submitted by the + /// old release_and_recheck and one submitted by try_submit. It also quietly + /// broke the one-slot park, which is sound only because "at most one + /// fire_once runs per node at a time" β€” with two, a value can be parked by + /// one firing and overwritten by the other. + /// + /// Everything this reads belongs to the firing that holds the claim, so it + /// is all evaluated first and the release is the last thing the firing does. + void finish_firing() { + bool want_more = false; + float prio = 0.5f; + + if (!stop_flag_.load(std::memory_order_relaxed)) { + bool parked = false; + if constexpr (!std::is_void_v) + parked = pending_.has_value(); + + if (parked) { + // Still holding output: only worth running again once the + // consumer has made room. + want_more = outputs_have_space(std::make_index_sequence{}); + } else { + if constexpr (input_count == 0) { + want_more = true; // sources always run again + } else { + want_more = count_ready(std::make_index_sequence{}) + == input_count; + if (want_more) prio = compute_priority(); + } + } + } + + if (gate_.release()) scheduler_->submit([this] { fire_once(); }, prio); + else if (want_more) try_submit(prio); + } + + /// Block until no firing of this node is in flight or queued. + /// + /// stop() used to set the flag and return, leaving an executing fire_once + /// touching input_channels_, stats_ and pending_ while the caller went on + /// to destroy them. For a node with a private pool that was survivable by + /// accident β€” Node::stop() calls pool->stop(), which joins β€” but a node + /// sharing a pool had nothing joining it at all, so ~PoolNode raced its own + /// members. The old comment said callers wanting the guarantee should call + /// scheduler_->drain() first; a destructor cannot, and the default should + /// not be a use-after-free. + /// + /// The gate is exactly the right thing to wait on: it is claimed for the + /// whole of a firing and released as the last act of one. A queued but + /// unstarted firing also holds it, and will run, observe stop_flag_ and + /// release β€” which is why the pool must still be running when this is + /// called. That is already the documented order (stop nodes, then the + /// pool), and Node/ObjectNode do it that way. + /// + /// Bounded, because a node function that never returns must not turn + /// teardown into a hang; and skipped entirely when called from the firing + /// thread itself, since an error handler that stops its own node would + /// otherwise wait for a firing that is waiting for it. + void await_quiescence() { + if (firing_thread_.load(std::memory_order_acquire) == std::this_thread::get_id()) + return; + const auto deadline = clock_t::now() + std::chrono::seconds(5); + while (gate_.queued()) { + if (clock_t::now() >= deadline) { + std::cerr << "[kpn] stop: node '" << name_ + << "' still had work in flight after 5 s; " + "continuing without it\n"; + return; + } + std::this_thread::sleep_for(std::chrono::microseconds(50)); + } + } + + /// Marks fire_once's thread for the duration of a firing, so await_quiescence + /// can tell a re-entrant stop() from an external one. + struct FiringMark { + std::atomic& slot; + explicit FiringMark(std::atomic& s) : slot(s) { + slot.store(std::this_thread::get_id(), std::memory_order_release); + } + ~FiringMark() { slot.store(std::thread::id{}, std::memory_order_release); } + }; + void fire_once() { + FiringMark mark(firing_thread_); if (stop_flag_.load(std::memory_order_relaxed)) { - queued_.store(false, std::memory_order_release); + gate_.force_idle(); return; } @@ -283,6 +476,37 @@ private: t0.time_since_epoch()).count(); stats_.exec_start_us.store(now_us, std::memory_order_relaxed); + // Parked from a previous firing: retry that value before touching the + // inputs. Returning here releases the worker β€” the channel's space + // callback re-submits this node when the consumer drains a slot. + if constexpr (!std::is_void_v) { + if (pending_) { + push_outputs(std::move(*pending_), std::make_index_sequence{}); + // Whether the value went out or is still parked, finish_firing + // reads pending_ and picks the right follow-up: output space if + // still holding, input readiness if drained. Resubmitting + // unconditionally would fire a node whose inputs are empty, and + // pop_one reports an empty channel as ChannelClosedError β€” which + // this node treats as "upstream finished" and self-stops on. + finish_firing(); + return; + } + } + + // Woken by output space rather than by input arrival, with nothing + // parked left to flush: there is no work to do. Falling through would + // read an empty channel, and pop_one reports empty as + // ChannelClosedError β€” self-stopping a live node. Release the worker; + // on_input_ready() resubmits when data actually lands. + if constexpr (input_count > 0) { + if (count_ready(std::make_index_sequence{}) != input_count) { + // finish_firing re-checks readiness after the work above, so + // data that landed while we looked is not missed. + finish_firing(); + return; + } + } + try { auto args = pop_inputs(std::make_index_sequence{}); auto t1 = clock_t::now(); @@ -301,6 +525,14 @@ private: auto t2 = clock_t::now(); // blocked_time = 0 for pool nodes (we don't block waiting for inputs) stats_.record_exec(duration_t(t2 - t1), duration_t::zero(), cpu0, cpu1); + } catch (const ChannelEmptyError&) { + // Not an error: there was simply nothing to take. Release and wait + // to be woken again. fire_once checks readiness before it gets + // here, and this node is the sole consumer of its inputs, so this + // is unreachable today β€” it exists so that if the check is ever + // weakened the cost is a wasted firing rather than a dead node. + finish_firing(); + return; } catch (const ChannelClosedError&) { fire_callbacks(closed_callbacks_); self_stop(); @@ -308,7 +540,11 @@ private: } catch (const ChannelOverflowError&) { fire_callbacks(event_callbacks_); } catch (...) { - if (error_handler_ && error_handler_(name_, std::current_exception())) { + auto eptr = std::current_exception(); + const bool handled = + (error_handler_ && error_handler_(name_, eptr)) || + (net_error_handler_ && net_error_handler_(name_, eptr)); + if (handled) { // continue β€” fall through to resubmit check } else { fire_callbacks(closed_callbacks_); @@ -318,20 +554,15 @@ private: } stats_.exec_start_us.store(0, std::memory_order_relaxed); - queued_.store(false, std::memory_order_release); - - if (stop_flag_.load(std::memory_order_relaxed)) return; - - // Source nodes always resubmit; others resubmit only if inputs are ready. - if constexpr (input_count == 0) { - try_submit(0.5f); - } else { - on_input_ready(); - } + // If the push above parked, finish_firing waits on output space rather + // than input arrival: this firing consumed its input, so an input-level + // check would not resubmit and the node would hold its value forever + // while its consumer waits for exactly that value. + finish_firing(); } // Pop all inputs β€” safe because we're the sole consumer and fire_once - // is guarded by queued_ (only one fire_once runs at a time). + // is guarded by the submit gate (only one fire_once runs at a time). template args_tuple pop_inputs(std::index_sequence) { return {pop_one()...}; @@ -341,8 +572,16 @@ private: std::tuple_element_t pop_one() { auto& ch = *std::get(input_channels_); std::tuple_element_t val; - if (!ch.try_pop_now(val)) + if (!ch.try_pop_now(val)) { + // try_pop_now returns false for "nothing available", which covers + // two very different situations. A closed channel means upstream is + // finished and this node should stop. An open one means only that + // nothing is here at this instant β€” and treating that as closed + // kills a live node, which then disables its own inputs and outputs + // and takes the rest of the pipeline with it. + if (ch.is_accepting()) throw ChannelEmptyError{}; throw ChannelClosedError{}; + } return val; } @@ -353,20 +592,58 @@ private: } template + /// Pushes what it can and parks the rest. `done_` marks the elements that + /// were taken, so a retry never re-pushes one β€” a duplicate would be as + /// wrong as a drop, just harder to notice. void push_outputs(return_tuple&& result, std::index_sequence) { - (push_one_out(std::get(std::move(result))), ...); + bool all = true; + ((pending_done_[Is] = pending_done_[Is] || + push_one_out(std::get(std::move(result))), + all = all && pending_done_[Is]), ...); + if (all) { pending_.reset(); pending_done_.fill(false); } + // The retry path calls this as push_outputs(std::move(*pending_), …), so + // on that path `result` *is* the parked tuple. Assigning it to itself is + // a self-move-assignment, which for std::tuple is elementwise β€” and + // libstdc++'s std::vector does not guard against it: it swaps its data + // into a temporary and leaves the vector empty. A value that failed to + // push twice would therefore be delivered with its payload silently + // erased, which downstream reads as a legitimately empty result rather + // than as a loss. Only store when it is not already stored. + else if (!pending_ || &result != &*pending_) pending_ = std::move(result); } + /// Returns false when the ring was full and the value was NOT taken; the + /// caller must keep it and retry after the channel signals space. template - void push_one_out(std::tuple_element_t&& val) { + bool push_one_out(std::tuple_element_t&& val) { auto* ch = std::get(output_channels_); - if (!ch) return; - try { - ch->push(std::move(val)); - } catch (const ChannelOverflowError&) { - throw ChannelOverflowError(ch->capacity(), - "pool node '" + name_ + "' " + output_port_label()); + if (!ch) return true; + // Sentinels (EOF) must never be dropped: a lost token wedges every + // downstream pop() forever. Deliver them out-of-band (push_sentinel), + // which never overflows and never blocks this node's worker thread. + if (is_sentinel_value(val)) { + // Not parked and not reported. Parking would spin against a slot + // only the consumer can free; reporting would cry data loss on the + // normal steady state, since a source at end of input keeps being + // polled and keeps returning EOF, so the token is re-offered on + // every firing. Refusing a re-offer loses nothing β€” the pending + // token says the same thing. See Channel::try_push_sentinel. + ch->try_push_sentinel(val); + return true; } + // Backpressure without parking the worker. A full channel means the + // consumer is behind; the value is kept and this node stops running + // until the channel signals space (set_space_callback re-submits it). + // + // Blocking here instead would sleep inside a scheduler worker, and + // nodes are pinned to workers β€” park enough of them and nothing is left + // to run the consumer that would drain the channel. That is the + // hold-and-wait deadlock channel.hpp warns about for sentinels; it + // applies to data pushes just as much. + // Closed counts as "stop trying", not as delivered: the value is gone + // and the channel has recorded the drop. Only Full means park and retry. + return ch->try_push(val) != Channel> + ::PushResult::Full; } template @@ -387,9 +664,29 @@ private: input_channels_t input_channels_; output_channels_t output_channels_{}; std::atomic stop_flag_{true}; - std::atomic queued_{false}; + /// Serialises firings and records wakes that arrive during one. See + /// submit_gate.hpp for why this cannot be two separate flags. + SubmitGate gate_; + /// Whether prepare() has installed the channel callbacks. Only ever touched + /// from the thread driving start()/stop(), never from a worker, and never + /// cleared: the callbacks capture `this` and stay valid across a restart, so + /// re-registering them would be a pointless write to a live channel. + bool prepared_{false}; + /// Thread currently inside fire_once, or a default id when none is. + /// See await_quiescence. + std::atomic firing_thread_{}; + + /// The hidden one-slot output buffer (see push_outputs). Holding the value + /// here is what lets a node stop running without dropping it or occupying a + /// scheduler worker. One slot suffices because at most one fire_once() runs + /// per node at a time β€” with concurrent firing it would have to hold a whole + /// FIFO's worth. + std::conditional_t, std::monostate, + std::optional> pending_{}; + std::array pending_done_{}; NodeStats stats_; NodeErrorHandler error_handler_; + NodeErrorHandler net_error_handler_; std::chrono::milliseconds max_exec_time_{0}; std::array event_callbacks_{}; // [0]=user [1]=network std::array closed_callbacks_{}; @@ -440,23 +737,34 @@ public: ~PoolObjectNode() override { stop(); } + void prepare() override { + if (prepared_) return; + prepared_ = true; + register_callbacks(std::make_index_sequence{}); + } + void start() override { + prepare(); enable_inputs(std::make_index_sequence{}); stop_flag_.store(false, std::memory_order_relaxed); - queued_.store(false, std::memory_order_relaxed); - register_callbacks(std::make_index_sequence{}); + gate_.force_idle(); if constexpr (input_count == 0) try_submit(0.5f); + else + // Never start with a wake already outstanding β€” see PoolNode::start(). + on_input_ready(); } void stop() override { stop_flag_.store(true, std::memory_order_seq_cst); disable_inputs(std::make_index_sequence{}); + await_quiescence(); } bool running() const override { return !stop_flag_.load(std::memory_order_relaxed); } void set_name(std::string name) override { name_ = std::move(name); } void set_error_handler(NodeErrorHandler h) { error_handler_ = std::move(h); } + void set_network_error_callback(NodeErrorHandler h) override { net_error_handler_ = std::move(h); } void set_max_exec_time(std::chrono::milliseconds t) { max_exec_time_ = t; } void set_overflow_callback(NodeEventCallback cb) { event_callbacks_[0] = std::move(cb); } @@ -480,6 +788,9 @@ public: stats_.total_cpu_us.load(std::memory_order_relaxed) / 1000.0, total_ms > 0 ? 100.0 * exec_ms / total_ms : 0.0, qwait_ms, + stats_.total_exec_us.load(std::memory_order_relaxed) / 1000.0, + gate_.queued(), + gate_.wake_pending(), }; } @@ -528,6 +839,8 @@ private: } template void register_callbacks(std::index_sequence) { + // A parked producer is re-submitted when its output drains. + register_space_callbacks(std::make_index_sequence{}); (std::get(input_channels_)->set_push_callback([this] { on_input_ready(); }), ...); } @@ -536,11 +849,27 @@ private: for (auto& cb : cbs) if (cb) cb(ts); } + template + bool outputs_have_space(std::index_sequence) const { + return (... && (!std::get(output_channels_) || + std::get(output_channels_)->has_space())); + } + + template + void register_space_callbacks(std::index_sequence) { + ((std::get(output_channels_) + ? (void)std::get(output_channels_)->set_space_callback( + [this] { try_submit(0.5f); }) + : (void)0), ...); + } + void self_stop() { disable_inputs(std::make_index_sequence{}); disable_outputs(std::make_index_sequence{}); stats_.exec_start_us.store(0, std::memory_order_relaxed); - queued_.store(false, std::memory_order_release); + // force_idle, not finish_firing(): this node is stopping, and + // honouring a pending wake here would resubmit a dead node. + gate_.force_idle(); stop_flag_.store(true, std::memory_order_relaxed); } @@ -567,11 +896,51 @@ private: return ((std::get(input_channels_)->approx_size() > 0 ? 1u : 0u) + ...); } + /// Priority in [0,1], higher runs sooner. + /// + /// Input fill alone answers "how much work is waiting for me". That is + /// only half the question: a node whose *outputs* are already full cannot + /// deliver anything: running it produces a value with nowhere to go, so it + /// immediately parks and the slot is wasted. Meanwhile the node that would + /// have drained that full channel waits behind it. + /// + /// So occupancy of the outputs is deducted from occupancy of the inputs. + /// The scheduler then naturally favours whoever is furthest downstream of + /// a bottleneck β€” the node whose inputs are backed up but whose outputs + /// have room is exactly the one whose execution frees the most capacity β€” + /// and defers producers that would only deepen a queue that is already + /// full. + /// + /// Mapped as 0.5Β·(1 + in - out) rather than clamping (in - out) at zero: + /// both terms are mean fills in [0,1], so the difference is in [-1,1], and + /// the affine map keeps the whole range distinguishable instead of + /// collapsing every output-saturated node onto the same value. 0.5 remains + /// the neutral point, matching the default used for source nodes. float compute_priority() { if constexpr (input_count == 0) return 0.5f; - float sum = 0.0f; - sum_fill(sum, std::make_index_sequence{}); - return sum / static_cast(input_count); + float in = 0.0f; + sum_fill(in, std::make_index_sequence{}); + in /= static_cast(input_count); + + if constexpr (output_count == 0) return in; + + float out = 0.0f; + sum_output_fill(out, std::make_index_sequence{}); + out /= static_cast(output_count); + + const float p = 0.5f * (1.0f + in - out); + return p < 0.0f ? 0.0f : (p > 1.0f ? 1.0f : p); + } + + /// Mean fill of the output channels, same normalisation as sum_fill. + /// An unconnected output holds nothing back, so it contributes 0. + template + void sum_output_fill(float& sum, std::index_sequence) { + ((sum += (std::get(output_channels_) && + std::get(output_channels_)->capacity() > 0) + ? float(std::get(output_channels_)->approx_size()) + / float(std::get(output_channels_)->capacity()) + : 0.0f), ...); } template void sum_fill(float& sum, std::index_sequence) { @@ -581,15 +950,119 @@ private: : 0.5f), ...); } + /// Submit unless a firing is already in flight. A wake that arrives while + /// one is is *recorded* against it, never dropped. + /// + /// Wakes are edge-triggered: a channel fires its space callback on the + /// transition, once. A dropped one never returns, so a node could park a + /// value, release its worker, and sleep forever holding output its consumer + /// was waiting for, with every worker idle in cond_wait and nothing left to + /// re-trigger it. SubmitGate makes "idle" and "wake outstanding" the same + /// variable, so the two cannot both be true β€” see submit_gate.hpp. void try_submit(float priority) { - bool expected = false; - if (queued_.compare_exchange_strong(expected, true, std::memory_order_acq_rel)) + // A stopped node must not claim the gate. The scheduler now refuses + // submissions after its pool stops, so the submit itself is safe β€” but + // claiming and never releasing would leave the gate held, and a restart + // would then have to clear it. start() does, but relying on that makes + // the invariant depend on a distant statement. + if (stop_flag_.load(std::memory_order_relaxed)) return; + if (gate_.claim()) scheduler_->submit([this] { fire_once(); }, priority); } + /// Decide whether this node should run again, then release the gate β€” in + /// that order, always. + /// + /// Releasing first is what let two firings of the same node overlap: the + /// moment the gate is free another worker may enter fire_once, while this + /// invocation is still reading pending_ and writing pending_done_. TSan + /// caught it as a race on pending_done_ between a firing submitted by the + /// old release_and_recheck and one submitted by try_submit. It also quietly + /// broke the one-slot park, which is sound only because "at most one + /// fire_once runs per node at a time" β€” with two, a value can be parked by + /// one firing and overwritten by the other. + /// + /// Everything this reads belongs to the firing that holds the claim, so it + /// is all evaluated first and the release is the last thing the firing does. + void finish_firing() { + bool want_more = false; + float prio = 0.5f; + + if (!stop_flag_.load(std::memory_order_relaxed)) { + bool parked = false; + if constexpr (!std::is_void_v) + parked = pending_.has_value(); + + if (parked) { + // Still holding output: only worth running again once the + // consumer has made room. + want_more = outputs_have_space(std::make_index_sequence{}); + } else { + if constexpr (input_count == 0) { + want_more = true; // sources always run again + } else { + want_more = count_ready(std::make_index_sequence{}) + == input_count; + if (want_more) prio = compute_priority(); + } + } + } + + if (gate_.release()) scheduler_->submit([this] { fire_once(); }, prio); + else if (want_more) try_submit(prio); + } + + /// Block until no firing of this node is in flight or queued. + /// + /// stop() used to set the flag and return, leaving an executing fire_once + /// touching input_channels_, stats_ and pending_ while the caller went on + /// to destroy them. For a node with a private pool that was survivable by + /// accident β€” Node::stop() calls pool->stop(), which joins β€” but a node + /// sharing a pool had nothing joining it at all, so ~PoolNode raced its own + /// members. The old comment said callers wanting the guarantee should call + /// scheduler_->drain() first; a destructor cannot, and the default should + /// not be a use-after-free. + /// + /// The gate is exactly the right thing to wait on: it is claimed for the + /// whole of a firing and released as the last act of one. A queued but + /// unstarted firing also holds it, and will run, observe stop_flag_ and + /// release β€” which is why the pool must still be running when this is + /// called. That is already the documented order (stop nodes, then the + /// pool), and Node/ObjectNode do it that way. + /// + /// Bounded, because a node function that never returns must not turn + /// teardown into a hang; and skipped entirely when called from the firing + /// thread itself, since an error handler that stops its own node would + /// otherwise wait for a firing that is waiting for it. + void await_quiescence() { + if (firing_thread_.load(std::memory_order_acquire) == std::this_thread::get_id()) + return; + const auto deadline = clock_t::now() + std::chrono::seconds(5); + while (gate_.queued()) { + if (clock_t::now() >= deadline) { + std::cerr << "[kpn] stop: node '" << name_ + << "' still had work in flight after 5 s; " + "continuing without it\n"; + return; + } + std::this_thread::sleep_for(std::chrono::microseconds(50)); + } + } + + /// Marks fire_once's thread for the duration of a firing, so await_quiescence + /// can tell a re-entrant stop() from an external one. + struct FiringMark { + std::atomic& slot; + explicit FiringMark(std::atomic& s) : slot(s) { + slot.store(std::this_thread::get_id(), std::memory_order_release); + } + ~FiringMark() { slot.store(std::thread::id{}, std::memory_order_release); } + }; + void fire_once() { + FiringMark mark(firing_thread_); if (stop_flag_.load(std::memory_order_relaxed)) { - queued_.store(false, std::memory_order_release); + gate_.force_idle(); return; } auto t0 = clock_t::now(); @@ -597,6 +1070,35 @@ private: t0.time_since_epoch()).count(); stats_.exec_start_us.store(now_us, std::memory_order_relaxed); + // Parked from a previous firing: retry that value before touching the + // inputs. Returning here releases the worker β€” the channel's space + // callback re-submits this node once the consumer drains a slot. + if constexpr (!std::is_void_v) { + if (pending_) { + push_outputs(std::move(*pending_), std::make_index_sequence{}); + // Whether the value went out or is still parked, finish_firing + // reads pending_ and picks the right follow-up: output space if + // still holding, input readiness if drained. Resubmitting + // unconditionally would fire a node whose inputs are empty, and + // pop_one reports an empty channel as ChannelClosedError β€” which + // this node treats as "upstream finished" and self-stops on. + finish_firing(); + return; + } + } + + // See the equivalent guard in PoolNode::fire_once: a space-callback + // wake with nothing parked must release the worker, not fall through + // into pop_inputs on an empty channel. + if constexpr (input_count > 0) { + if (count_ready(std::make_index_sequence{}) != input_count) { + // finish_firing re-checks readiness after the work above, so + // data that landed while we looked is not missed. + finish_firing(); + return; + } + } + try { auto args = pop_inputs(std::make_index_sequence{}); auto t1 = clock_t::now(); @@ -613,6 +1115,14 @@ private: auto cpu1 = NodeStats::cpu_now(); auto t2 = clock_t::now(); stats_.record_exec(duration_t(t2 - t1), duration_t::zero(), cpu0, cpu1); + } catch (const ChannelEmptyError&) { + // Not an error: there was simply nothing to take. Release and wait + // to be woken again. fire_once checks readiness before it gets + // here, and this node is the sole consumer of its inputs, so this + // is unreachable today β€” it exists so that if the check is ever + // weakened the cost is a wasted firing rather than a dead node. + finish_firing(); + return; } catch (const ChannelClosedError&) { fire_callbacks(closed_callbacks_); self_stop(); @@ -620,7 +1130,11 @@ private: } catch (const ChannelOverflowError&) { fire_callbacks(event_callbacks_); } catch (...) { - if (error_handler_ && error_handler_(name_, std::current_exception())) { + auto eptr = std::current_exception(); + const bool handled = + (error_handler_ && error_handler_(name_, eptr)) || + (net_error_handler_ && net_error_handler_(name_, eptr)); + if (handled) { } else { fire_callbacks(closed_callbacks_); self_stop(); @@ -629,10 +1143,11 @@ private: } stats_.exec_start_us.store(0, std::memory_order_relaxed); - queued_.store(false, std::memory_order_release); - if (stop_flag_.load(std::memory_order_relaxed)) return; - if constexpr (input_count == 0) try_submit(0.5f); - else on_input_ready(); + // If the push above parked, finish_firing waits on output space rather + // than input arrival: this firing consumed its input, so an input-level + // check would not resubmit and the node would hold its value forever + // while its consumer waits for exactly that value. + finish_firing(); } template @@ -642,7 +1157,16 @@ private: std::tuple_element_t pop_one() { auto& ch = *std::get(input_channels_); std::tuple_element_t val; - if (!ch.try_pop_now(val)) throw ChannelClosedError{}; + if (!ch.try_pop_now(val)) { + // try_pop_now returns false for "nothing available", which covers + // two very different situations. A closed channel means upstream is + // finished and this node should stop. An open one means only that + // nothing is here at this instant β€” and treating that as closed + // kills a live node, which then disables its own inputs and outputs + // and takes the rest of the pipeline with it. + if (ch.is_accepting()) throw ChannelEmptyError{}; + throw ChannelClosedError{}; + } return val; } @@ -653,19 +1177,49 @@ private: } template + /// Pushes what it can and parks the rest. `pending_done_` marks the elements + /// already taken, so a retry never re-pushes one β€” a duplicate is as wrong as + /// a drop and harder to notice. void push_outputs(return_tuple&& result, std::index_sequence) { - (push_one_out(std::get(std::move(result))), ...); + bool all = true; + ((pending_done_[Is] = pending_done_[Is] || + push_one_out(std::get(std::move(result))), + all = all && pending_done_[Is]), ...); + if (all) { pending_.reset(); pending_done_.fill(false); } + // The retry path calls this as push_outputs(std::move(*pending_), …), so + // on that path `result` *is* the parked tuple. Assigning it to itself is + // a self-move-assignment, which for std::tuple is elementwise β€” and + // libstdc++'s std::vector does not guard against it: it swaps its data + // into a temporary and leaves the vector empty. A value that failed to + // push twice would therefore be delivered with its payload silently + // erased, which downstream reads as a legitimately empty result rather + // than as a loss. Only store when it is not already stored. + else if (!pending_ || &result != &*pending_) pending_ = std::move(result); } + /// Returns false when the ring was full and the value was NOT taken; the + /// caller must keep it and retry after the channel signals space. template - void push_one_out(std::tuple_element_t&& val) { + bool push_one_out(std::tuple_element_t&& val) { auto* ch = std::get(output_channels_); - if (!ch) return; - try { - ch->push(std::move(val)); - } catch (const ChannelOverflowError&) { - throw ChannelOverflowError(ch->capacity(), - "pool node '" + name_ + "'"); + if (!ch) return true; + // Sentinels (EOF) must never be dropped: a lost token wedges every + // downstream pop() forever. Deliver them out-of-band (push_sentinel), + // which never overflows and never blocks this node's worker thread. + if (is_sentinel_value(val)) { + // Not parked and not reported. Parking would spin against a slot + // only the consumer can free; reporting would cry data loss on the + // normal steady state, since a source at end of input keeps being + // polled and keeps returning EOF, so the token is re-offered on + // every firing. Refusing a re-offer loses nothing β€” the pending + // token says the same thing. See Channel::try_push_sentinel. + ch->try_push_sentinel(val); + return true; } + // See the note on the typed overload above: park rather than block. + // Closed counts as "stop trying", not as delivered: the value is gone + // and the channel has recorded the drop. Only Full means park and retry. + return ch->try_push(val) != Channel> + ::PushResult::Full; } Obj& obj_; @@ -675,9 +1229,29 @@ private: input_channels_t input_channels_; output_channels_t output_channels_{}; std::atomic stop_flag_{true}; - std::atomic queued_{false}; + /// Serialises firings and records wakes that arrive during one. See + /// submit_gate.hpp for why this cannot be two separate flags. + SubmitGate gate_; + /// Whether prepare() has installed the channel callbacks. Only ever touched + /// from the thread driving start()/stop(), never from a worker, and never + /// cleared: the callbacks capture `this` and stay valid across a restart, so + /// re-registering them would be a pointless write to a live channel. + bool prepared_{false}; + /// Thread currently inside fire_once, or a default id when none is. + /// See await_quiescence. + std::atomic firing_thread_{}; + + /// The hidden one-slot output buffer (see push_outputs). Holding the value + /// here is what lets a node stop running without dropping it or occupying a + /// scheduler worker. One slot suffices because at most one fire_once() runs + /// per node at a time β€” with concurrent firing it would have to hold a whole + /// FIFO's worth. + std::conditional_t, std::monostate, + std::optional> pending_{}; + std::array pending_done_{}; NodeStats stats_; NodeErrorHandler error_handler_; + NodeErrorHandler net_error_handler_; std::chrono::milliseconds max_exec_time_{0}; std::array event_callbacks_{}; // [0]=user [1]=network std::array closed_callbacks_{}; diff --git a/include/kpn/python/auto_bind.hpp b/include/kpn/python/auto_bind.hpp index 12a8bd1..79f9d71 100644 --- a/include/kpn/python/auto_bind.hpp +++ b/include/kpn/python/auto_bind.hpp @@ -247,7 +247,7 @@ void bind_network(nb::module_& m) { nb::class_>(m, "INode"); - nb::class_(m, "Network") + nb::class_(m, "Network", nb::type_slots(network_type_slots())) .def("__init__", [](Net* self) { new (self) Net(); register_all_converters(*self); diff --git a/include/kpn/python/bindings.hpp b/include/kpn/python/bindings.hpp index 57554e9..2491bf8 100644 --- a/include/kpn/python/bindings.hpp +++ b/include/kpn/python/bindings.hpp @@ -35,6 +35,16 @@ public: using VNode = IVariantNode; using VChannel = IVariantChannel; + // ── GC support ──────────────────────────────────────────────────────────── + // Visit every Python object this network transitively holds (currently the + // callable of each PyNode). Used by the Network type's tp_traverse slot so + // Python's cyclic GC can discover instance β†’ callable β†’ globals() cycles. + // Defined out-of-line below, once PyNode is a complete type. + template + void visit_python_objects(Fn&& visit) const; + // Drop all Python references held by nodes, breaking any cycle (tp_clear). + void clear_python_objects(); + // ── Builder API ─────────────────────────────────────────────────────────── void add(std::string name, std::shared_ptr node) { @@ -207,6 +217,20 @@ public: return it->second; } + // Raw node handle by name β€” lets a binding dynamic_cast to a concrete wrapper + // type and call its functor's runtime setters (persistent-pipeline reuse). + VNode* node_ptr(const std::string& name) { return &node_at(name); } + + // Per-node timing snapshot for profiling where a replay spends its time. + std::map node_stats(const std::string& name) { + auto& n = node_at(name); + NodeSnapshot s = n.node_snapshot(name, 0.0); + return {{"frames", double(s.frames_processed)}, + {"exec_ms", s.ema_exec_ms}, {"max_ms", s.max_exec_ms}, + {"blocked_ms", s.total_blocked_ms}, {"fps", s.throughput_fps}, + {"cpu_ms", s.total_cpu_ms}, {"cpu_util_pct", s.cpu_util_pct}}; + } + private: VNode& node_at(const std::string& name) { auto it = nodes_.find(name); @@ -367,6 +391,14 @@ public: out_channels_[i] = std::move(ch); } + // ── GC support (tp_traverse / tp_clear on the owning Network) ────────────── + // The node holds a Python callable, which typically forms an + // instance β†’ callable β†’ globals() β†’ instance cycle. Expose the callable so + // the Network's GC slots can traverse and clear it. See bindings.hpp's + // network_tp_traverse/network_tp_clear. + const nb::object& python_callable() const { return callable_; } + void clear_python_callable() { callable_ = nb::object(); } + private: void run_loop() { while (!stop_flag_.load(std::memory_order_relaxed)) { @@ -404,13 +436,17 @@ private: for (std::size_t i = 0; i < out_channels_.size(); ++i) { if (out_channels_[i]) - out_channels_[i]->push(std::move(outputs[i])); + // Lossless: wait for space rather than drop. A dropped frame + // silently corrupts a replay's score; backpressure just slows + // the producer. (Was push() + "drop on overflow".) + out_channels_[i]->push_blocking(std::move(outputs[i])); } } catch (const ChannelClosedError&) { break; } catch (const ChannelOverflowError&) { - // drop and continue + // no longer reachable with push_blocking, kept for safety + break; } } } @@ -439,6 +475,60 @@ private: NodeStats stats_; }; +// ── PyNetwork GC helpers (defined here: PyNode is now complete) ──────────────── + +template +template +void PyNetwork::visit_python_objects(Fn&& visit) const { + for (const auto& [name, node] : nodes_) + if (auto* py = dynamic_cast*>(node.get())) + visit(py->python_callable()); +} + +template +void PyNetwork::clear_python_objects() { + for (auto& [name, node] : nodes_) + if (auto* py = dynamic_cast*>(node.get())) + py->clear_python_callable(); +} + +// ── GC type slots for the Network binding ───────────────────────────────────── +// The Network holds Python callables (via PyNode), forming uncollectable +// instance β†’ callable β†’ globals() β†’ instance cycles at interpreter shutdown. +// These slots let Python's cyclic collector traverse and break them, silencing +// nanobind's leak warnings. See the nanobind "Reference leaks" documentation. + +template +int network_tp_traverse(PyObject* self, visitproc visit, void* arg) { + Py_VISIT(Py_TYPE(self)); + if (!nb::inst_ready(self)) + return 0; + auto* net = nb::inst_ptr>(self); + int rv = 0; + net->visit_python_objects([&](const nb::object& obj) { + if (rv == 0 && obj.is_valid()) + rv = visit(obj.ptr(), arg); + }); + return rv; +} + +template +int network_tp_clear(PyObject* self) { + auto* net = nb::inst_ptr>(self); + net->clear_python_objects(); + return 0; +} + +template +PyType_Slot* network_type_slots() { + static PyType_Slot slots[] = { + { Py_tp_traverse, reinterpret_cast(&network_tp_traverse) }, + { Py_tp_clear, reinterpret_cast(&network_tp_clear) }, + { 0, nullptr } + }; + return slots; +} + // ── register_py_network (legacy helper) ─────────────────────────────────────── // Registers PyNetwork with the given nanobind module. // Prefer bind_network from auto_bind.hpp for new code. @@ -447,7 +537,7 @@ template void register_py_network(nb::module_& m, const char* class_name = "Network") { using Net = PyNetwork; - nb::class_(m, class_name) + nb::class_(m, class_name, nb::type_slots(network_type_slots())) .def(nb::init<>()) .def("connect", &Net::connect, nb::arg("src"), nb::arg("out_idx"), @@ -458,7 +548,8 @@ void register_py_network(nb::module_& m, const char* class_name = "Network") { .def("read", &Net::read, nb::arg("node"), nb::arg("out_idx") = std::size_t(0)) .def("write", &Net::write, - nb::arg("node"), nb::arg("in_idx"), nb::arg("value")); + nb::arg("node"), nb::arg("in_idx"), nb::arg("value")) + .def("node_stats", &Net::node_stats, nb::arg("node")); } } // namespace kpn::python diff --git a/include/kpn/python/object_variant_node.hpp b/include/kpn/python/object_variant_node.hpp new file mode 100644 index 0000000..82ff149 --- /dev/null +++ b/include/kpn/python/object_variant_node.hpp @@ -0,0 +1,149 @@ +#pragma once +// ObjectVariantNodeWrapper β€” variant-node adapter for *stateful* functors. +// +// VariantNodeWrapper (variant_node.hpp) wraps Node, where Func is a +// default-constructible NTTP callable. That doesn't fit nodes whose functor must +// be constructed with runtime state (a Config, a loaded gallery, etc.) β€” those use +// ObjectNode, which takes `Obj& obj` at construction. +// +// This wrapper owns an Obj instance and exposes the same IVariantNode surface so a +// stateful C++ node can live inside a PyNetwork. Build one via a factory that +// constructs the functor from Python-supplied config, e.g.: +// +// auto n = std::make_shared, out<"matched">>>( +// fifo_cap, gallery, cfg); // Obj ctor args forwarded +// net.add("identity_matcher", n); +// +// The wrapper mirrors VariantNodeWrapper's channel plumbing exactly; only the +// underlying node type (PoolObjectNode, holding Obj&) differs. + +#include "../channel.hpp" +#include "../node.hpp" +#include "../variant_node.hpp" + +#include +#include +#include +#include +#include +#include +#include + +namespace kpn { + +template, + typename OutputTag = out<>> +class ObjectVariantNodeWrapper; + +template +class ObjectVariantNodeWrapper, out> + : public IVariantNode +{ + using NodeT = ObjectNode, out>; + +public: + using args_tuple = typename NodeT::args_tuple; + using return_tuple = typename NodeT::return_tuple; + + static constexpr std::size_t n_in = NodeT::input_count; + static constexpr std::size_t n_out = NodeT::output_count; + + // Owns the functor; forwards remaining args to Obj's constructor. + template + explicit ObjectVariantNodeWrapper(std::size_t fifo_capacity, ObjArgs&&... obj_args) + : obj_(std::forward(obj_args)...) + , node_(obj_, fifo_capacity) + , in_channels_(n_in) + , out_channels_(n_out) + , out_type_indices_(n_out, std::type_index(typeid(void))) + { + init_inputs(std::make_index_sequence{}, fifo_capacity); + init_out_types(std::make_index_sequence{}); + } + + // Access the owned functor so callers can invoke its runtime setters (e.g. to + // change a threshold on a persistent pipeline without rebuilding the node). + Obj& functor() { return obj_; } + + // ── INode ───────────────────────────────────────────────────────────────── + void start() override { node_.start(); } + void stop() override { node_.stop(); } + bool running() const override { return node_.running(); } + const NodeStats& stats() const override { return node_.stats(); } + void set_name(std::string name) override { node_.set_name(std::move(name)); } + NodeSnapshot node_snapshot(const std::string& name, double elapsed_s) const override { + return node_.node_snapshot(name, elapsed_s); + } + + // ── IVariantNode ────────────────────────────────────────────────────────── + std::size_t input_count() const override { return n_in; } + std::size_t output_count() const override { return n_out; } + + std::type_index input_type(std::size_t i) const override { + return in_channels_[i]->type_index(); + } + std::type_index output_type(std::size_t i) const override { + return out_type_indices_[i]; + } + + std::shared_ptr> input_channel(std::size_t i) override { + return in_channels_[i]; + } + + void set_output_channel(std::size_t i, + std::shared_ptr> ch) override { + set_output_impl(i, std::move(ch), std::make_index_sequence{}); + } + +private: + template + void init_inputs(std::index_sequence, std::size_t cap) { + ((init_one_input(cap)), ...); + } + + template + void init_one_input(std::size_t cap) { + using T = std::tuple_element_t; + auto shared_ch = std::make_shared>(cap); + node_.template set_input_channel(shared_ch); + in_channels_[I] = std::make_shared>(std::move(shared_ch)); + } + + template + void init_out_types(std::index_sequence) { + ((out_type_indices_[Is] = + std::type_index(typeid(std::tuple_element_t))), ...); + } + + template + void set_output_impl(std::size_t port, + std::shared_ptr> ch, + std::index_sequence) { + bool matched = false; + ((Is == port && (set_output_at(std::move(ch)), matched = true)), ...); + if (!matched) + throw std::out_of_range("set_output_channel: port index out of range"); + } + + template + void set_output_at(std::shared_ptr> ch) { + using T = std::tuple_element_t; + auto* typed = dynamic_cast*>(ch.get()); + if (!typed) + throw std::runtime_error( + "set_output_channel: type mismatch at output port " + std::to_string(I)); + node_.template set_output_channel(typed->raw_ptr()); + out_channels_[I] = std::move(ch); + } + + Obj obj_; // owned; node_ holds Obj& β€” declaration order keeps obj_ alive first + NodeT node_; + std::vector>> in_channels_; + std::vector>> out_channels_; + std::vector out_type_indices_; +}; + +} // namespace kpn diff --git a/include/kpn/scheduler.hpp b/include/kpn/scheduler.hpp index e7920bf..c33d5a2 100644 --- a/include/kpn/scheduler.hpp +++ b/include/kpn/scheduler.hpp @@ -5,6 +5,7 @@ #include #include #include +#include #include #include #include @@ -52,6 +53,19 @@ public: } void start() override { + // Under the lifecycle lock for the same reason stop() is: submit() + // reads queues_ and this rebuilds it. A network starts its nodes one at + // a time, and a node already started fires into the next one's channel, + // whose push callback submits β€” so a submission can genuinely land + // while another pool is still inside start(). ThreadSanitizer reports + // it as a read at submit() against this write, and the consequence is + // worse than a torn read: push_back can reallocate the vector under a + // reader that has already indexed it. + // + // Queues are all constructed before any worker is spawned, which is + // what keeps worker_loop's own queues_[id] out of this β€” it never takes + // the lock, so holding it across the spawn cannot deadlock. + std::unique_lock lk(lifecycle_mx_); stopped_.store(false, std::memory_order_relaxed); queues_.clear(); for (std::size_t i = 0; i < thread_count_; ++i) @@ -62,18 +76,30 @@ public: } void stop() override { - stopped_.store(true, std::memory_order_seq_cst); + // Close the pool to new work before touching anything, and do it under + // the lifecycle lock so no submit() is midway through indexing queues_. + { + std::unique_lock lk(lifecycle_mx_); + stopped_.store(true, std::memory_order_seq_cst); + } for (auto& q : queues_) { std::lock_guard lock(q->mx); std::size_t discarded = q->pq.size(); while (!q->pq.empty()) q->pq.pop(); total_.fetch_sub(discarded, std::memory_order_relaxed); + queued_.fetch_sub(discarded, std::memory_order_relaxed); } // Lock cv_mx_ before notifying so the stop signal can't be lost in the // gap between a worker's predicate check and its wait() (see submit()). { std::lock_guard lk(cv_mx_); } cv_.notify_all(); + // Join without the lock: a worker's task may call submit(), which takes + // it shared, and holding it here would deadlock against that. for (auto& t : workers_) if (t.joinable()) t.join(); + // Destroying the queues is what submit() must never race. By now + // stopped_ is published, so any submit() that acquires the lock after + // this point returns without touching them. + std::unique_lock lk(lifecycle_mx_); workers_.clear(); queues_.clear(); } @@ -86,34 +112,89 @@ public: } void submit(std::function task, float priority = 0.5f) override { - std::size_t target = next_.fetch_add(1, std::memory_order_relaxed) % thread_count_; + // A submission can arrive after this pool has been stopped, and did so + // by an ordinary route: a node's space callback fires from whichever + // thread drained the channel, which belongs to the *consumer*. Stop the + // producer first β€” as a sources-first shutdown does β€” and the consumer + // keeps draining its backlog, firing the producer's space callback into + // a pool whose stop() has already run queues_.clear(). submit() then + // indexed an empty vector: a segfault, reproducible about 12 runs in 20. + // + // The shared lock is what makes the check meaningful. Reading stopped_ + // alone leaves the window between the read and the indexing, which is + // precisely where stop() clears the vector. + std::shared_lock lk(lifecycle_mx_); + if (stopped_.load(std::memory_order_acquire) || queues_.empty()) { + rejected_.fetch_add(1, std::memory_order_relaxed); + return; + } + // B9 β€” submit-to-self affinity. Round-robin hands every task to a + // *different* worker, and on a pool of two or more that worker is + // asleep, so each dispatch pays a futex wake: measured 182 ns/dispatch + // on a 1-thread pool against 3197 ns on 20 threads, with voluntary + // context switches per task rising 0.00 -> 1.19 in step. + // + // A submission originating on one of *our own* workers goes to that + // worker's queue instead. It is about to return to worker_loop and + // try_pop its own queue, so the work is already there and nothing + // sleeps β€” the property that makes ThreadPool(1) fast, extended to + // any pool size. Imbalance is corrected by the existing try_steal. + // + // The pool identity check is load-bearing: a worker of pool A + // submitting into pool B must not use A's index, which may exceed B's + // thread_count_ or alias an unrelated queue. Nested networks do + // exactly this. + std::size_t target; + if (tls_pool == this && tls_worker < thread_count_) { + target = tls_worker; + } else { + target = next_.fetch_add(1, std::memory_order_relaxed) % thread_count_; + } { std::lock_guard lock(queues_[target]->mx); queues_[target]->pq.push( {std::move(task), priority, seq_.fetch_add(1, std::memory_order_relaxed)}); } total_.fetch_add(1, std::memory_order_relaxed); + queued_.fetch_add(1, std::memory_order_relaxed); submitted_.fetch_add(1, std::memory_order_relaxed); // Synchronize with worker_loop's predicate evaluation: taking cv_mx_ // here guarantees a worker is either before its predicate check (and // will observe total_ > 0) or already blocked in wait() (and will be // woken). Without this, notify_one() can slip into the gap between the // worker's predicate check and its wait(), and be lost β€” a deadlock. - { std::lock_guard lk(cv_mx_); } - cv_.notify_one(); + // + // Skipped entirely when no worker is parked. waiters_ is incremented + // *before* wait() releases cv_mx_ and decremented after it returns, + // both under that mutex, so a worker on its way to sleep is already + // counted here. Reading zero therefore means no worker can be in + // wait(), and there is nothing a notify could reach β€” as opposed to + // reading zero because we raced one, which the mutex prevents. + // + // This is the hot path for an already-busy pool: with B9 the work is + // in the local queue and the submitting worker will find it itself, + // so the lock round-trip and notify were pure overhead. Measured 1.00 + // voluntary context switches per task before this, on a pool where + // only one task is ever in flight. + if (waiters_.load(std::memory_order_seq_cst) != 0) { + { std::lock_guard lk(cv_mx_); } + cv_.notify_one(); + } } std::size_t thread_count() const { return thread_count_; } + /// Submissions dropped because the pool was stopped. See rejected_. + uint64_t rejected() const { return rejected_.load(std::memory_order_relaxed); } + // ── IPoolProbe ──────────────────────────────────────────────────────────── PoolSnapshot snapshot(const std::string& name) const override { std::size_t a = active_.load(std::memory_order_relaxed); - std::size_t t = total_.load(std::memory_order_relaxed); return { name, thread_count_, - t > a ? t - a : 0, // queued (approximate) - a, // executing + queued_.load(std::memory_order_relaxed), // queued (exact) + a, // executing submitted_.load(std::memory_order_relaxed), completed_.load(std::memory_order_relaxed), }; @@ -142,11 +223,23 @@ private: if (q.pq.empty()) return std::nullopt; auto fn = std::move(const_cast(q.pq.top()).fn); q.pq.pop(); + queued_.fetch_sub(1, std::memory_order_relaxed); return fn; } std::optional> try_steal(std::size_t thief) { // Find the most-loaded peer without blocking β€” racy peek is fine. + // + // The threshold is >0: a peer holding a single task is a valid victim. + // + // Raising it to >1 β€” to stop a thief winning the race for a task its + // owner just submitted to itself (B9) β€” deadlocks. `latency` mode + // hangs at 12 and 20 threads: an external submit() round-robins one + // task onto an idle worker's queue, and if that worker is parked, no + // peer will take it because a queue of one is no longer stealable. + // Nothing else is coming to wake it, so the pool sits forever. + // Measured before reverting: it also made steady-state *worse*, + // 2229 -> 4546 ns at 12 threads. std::size_t victim = thief, best = 0; for (std::size_t i = 0; i < queues_.size(); ++i) { if (i == thief) continue; @@ -175,36 +268,94 @@ private: } void worker_loop(std::size_t id) { + // Identify this thread as one of our workers, for B9's affinity check + // in submit(). Restored on exit rather than merely cleared: a pool + // whose worker runs a task that itself starts and stops a nested pool + // would otherwise come back with its identity erased. + ThreadPool* const prev_pool = tls_pool; + const std::size_t prev_worker = tls_worker; + tls_pool = this; + tls_worker = id; + struct Restore { + ThreadPool* p; std::size_t w; + ~Restore() { tls_pool = p; tls_worker = w; } + } restore{prev_pool, prev_worker}; + while (true) { if (auto fn = try_pop(*queues_[id])) { execute(*fn); continue; } if (auto fn = try_steal(id)) { execute(*fn); continue; } + // B5 (bounded spin before parking) was tried here and removed: it + // does not pay. Swept at 50/200/1000 rounds on a 12-thread pool, + // steady state went 2123 / 2230 / 2574 ns against 2229 ns without + // it, and voluntary context switches per task stayed at ~0.97 + // throughout. The spin cannot catch what it is aimed at, because + // a peer is woken the moment queued_ becomes non-zero β€” which + // happens before this worker reaches the spin at all. std::unique_lock lock(cv_mx_); + // Counted under cv_mx_ and before the predicate is evaluated, so + // that a submit() which reads waiters_ == 0 can be certain this + // worker is not about to block: to get here we already hold the + // mutex that submit() must take to notify. + waiters_.fetch_add(1, std::memory_order_seq_cst); cv_.wait(lock, [this] { return stopped_.load(std::memory_order_seq_cst) - || total_.load(std::memory_order_relaxed) > 0; + || queued_.load(std::memory_order_relaxed) > 0; }); + waiters_.fetch_sub(1, std::memory_order_seq_cst); + // Exit on queued_, not total_: waiting for total_ to reach zero + // meant waiting for someone else's task to finish, which this + // worker cannot help with and would spin through until it did. if (stopped_.load(std::memory_order_seq_cst) - && total_.load(std::memory_order_relaxed) == 0) + && queued_.load(std::memory_order_relaxed) == 0) return; } } + /// Which pool, and which of its workers, the calling thread is β€” or + /// nullptr on any thread that is not a pool worker. Read by submit() to + /// decide whether a local push is safe (B9). inline so the header stays + /// header-only. + static inline thread_local ThreadPool* tls_pool = nullptr; + static inline thread_local std::size_t tls_worker = 0; + const std::size_t thread_count_; std::vector> queues_; std::vector workers_; + /// Guards the lifetime of queues_/workers_ against a concurrent submit(). + /// Shared by submit, exclusive by stop, so submissions still run in + /// parallel with each other. + mutable std::shared_mutex lifecycle_mx_; + std::mutex cv_mx_; std::condition_variable cv_; std::mutex drain_mx_; std::condition_variable drain_cv_; std::atomic stopped_{true}; - std::atomic total_{0}; // queued + executing + std::atomic total_{0}; // queued + executing (drain() waits on this) + /// Queued only β€” never counts a task that is already executing. + /// + /// The wait predicate used total_, which includes running tasks, so while + /// any one task ran every *other* worker's predicate was true: wait() + /// returned instantly and the worker spun through try_pop / try_steal / + /// wait at full speed, try_lock-ing every peer queue on each pass. One slow + /// task therefore pinned every other core and contended the very mutexes + /// the working thread needed. Sleeping requires "no work is *waiting*", + /// which is this. + std::atomic queued_{0}; // waiting to run std::atomic active_{0}; // executing only (for snapshot) std::atomic next_{0}; // round-robin submit cursor + /// Workers currently inside cv_.wait(), maintained under cv_mx_. Lets + /// submit() skip the lock round-trip and notify when nobody is parked. + std::atomic waiters_{0}; std::atomic seq_{0}; // tie-break for equal-priority tasks std::atomic submitted_{0}; + /// Submissions refused because the pool was already stopped. Not an error β€” + /// teardown races are expected β€” but silence here would hide a node that + /// keeps trying to run after its pool is gone. + std::atomic rejected_{0}; std::atomic completed_{0}; }; diff --git a/include/kpn/shared_resource.hpp b/include/kpn/shared_resource.hpp index 79924c3..9919508 100644 --- a/include/kpn/shared_resource.hpp +++ b/include/kpn/shared_resource.hpp @@ -14,6 +14,18 @@ namespace kpn { template class Channel; // forward declaration for acquire_balanced +/// Thrown by a pending acquire() when the resource is closed underneath it. +/// +/// acquire() blocks on a condition variable with no timeout and no stop +/// condition, so a node parked there ignored teardown entirely: the worker +/// never returned, the pool's join never completed, and shutdown hung on a +/// resource nobody was going to release. Closing the resource turns that into +/// an exception the node's normal error path already handles. +class ResourceClosedError : public std::runtime_error { +public: + ResourceClosedError() : std::runtime_error("shared resource closed") {} +}; + // ── SharedResource ──────────────────────────────────────────────────────────── // // Wraps an exclusive resource (e.g. an ONNX session, a CUDA stream) and @@ -72,6 +84,7 @@ public: template Guard acquire(PriorityFn&& fn) { std::unique_lock lock(mutex_); + if (closed_) throw ResourceClosedError{}; if (!held_) { held_ = true; acq_.fetch_add(1, std::memory_order_relaxed); @@ -83,18 +96,46 @@ public: current_waiters_.store(waiters_.size(), std::memory_order_relaxed); auto t0 = w.wait_start; - w.cv.wait(lock, [&w] { return w.ready; }); + // Woken either by release() handing over ownership, or by close() + // giving up on the wait entirely. + w.cv.wait(lock, [&w] { return w.ready || w.closed; }); int64_t wait_us = std::chrono::duration_cast( clock_t::now() - t0).count(); waiters_.erase(std::find(waiters_.begin(), waiters_.end(), &w)); current_waiters_.store(waiters_.size(), std::memory_order_relaxed); - acq_.fetch_add(1, std::memory_order_relaxed); total_wait_us_.fetch_add(static_cast(wait_us > 0 ? wait_us : 0), std::memory_order_relaxed); + + // Closed without being handed ownership: no Guard, so nothing to + // release, and held_ is left exactly as close() found it. + if (!w.ready) throw ResourceClosedError{}; + + acq_.fetch_add(1, std::memory_order_relaxed); return Guard(this); } + /// Wake every waiter and refuse further acquisitions. + /// + /// Teardown is the whole point: a node parked in acquire() is not + /// observing stop flags, so without this the only way out is for whoever + /// holds the resource to release it β€” which, if that node is also being + /// stopped, may never happen. Idempotent, and safe to call from any thread. + void close() override { + std::lock_guard lock(mutex_); + closed_ = true; + for (Waiter* w : waiters_) { + w->closed = true; + w->cv.notify_one(); + } + } + + /// Reopen after a close(). For reuse across runs; not needed for teardown. + void reopen() { + std::lock_guard lock(mutex_); + closed_ = false; + } + // Acquire with no priority (all waiters treated equally, order is fair-ish). Guard acquire() { return acquire([] { return 0.5f; }); @@ -132,7 +173,10 @@ public: private: void release() { std::unique_lock lock(mutex_); - if (waiters_.empty()) { + // Hand over only to a waiter that is still waiting. A closed one is on + // its way out and will not take ownership, so treating it as the next + // holder would leave held_ true with nobody holding it. + if (closed_ || waiters_.empty()) { held_ = false; return; } @@ -162,7 +206,8 @@ private: std::function priority_fn; clock_t::time_point wait_start; std::condition_variable cv; - bool ready{false}; + bool ready{false}; // handed ownership by release() + bool closed{false}; // woken by close() instead Waiter(std::function fn, clock_t::time_point t) : priority_fn(std::move(fn)), wait_start(t) {} @@ -172,6 +217,7 @@ private: T resource_; bool held_{false}; + bool closed_{false}; mutable std::mutex mutex_; std::vector waiters_; std::atomic acq_{0}; diff --git a/include/kpn/static_network.hpp b/include/kpn/static_network.hpp index eeb5304..4eda22e 100644 --- a/include/kpn/static_network.hpp +++ b/include/kpn/static_network.hpp @@ -98,13 +98,15 @@ public: std::vector fanout_ptrs, std::vector user_node_names, std::vector fanout_node_names, - std::vector> channel_probes) + std::vector> channel_probes, + std::vector channel_src_names) : fanouts_(std::move(fanouts)) , user_nodes_topo_(std::move(user_nodes_topo)) , fanout_nodes_ptr_(std::move(fanout_ptrs)) , user_node_names_(std::move(user_node_names)) , fanout_node_names_(std::move(fanout_node_names)) , channel_probes_(std::move(channel_probes)) + , channel_src_names_(std::move(channel_src_names)) {} ~StaticNetwork() override { stop(); } @@ -122,6 +124,19 @@ public: [this, n](auto ts) { event_handler_(n, NodeEvent::Closed, ts); }); } } + if (error_handler_) { + for (auto* node : user_nodes_topo_) + node->set_network_error_callback(error_handler_); + } + // Install every node's channel callbacks before starting any of them. + // Those callbacks are std::function members on channels shared with + // neighbours; a neighbour that is already running reads them from its + // own thread, so writing one after the pipeline is live is a data race + // (ThreadSanitizer reports it on any multi-node network). Doing all the + // writes here, while nothing runs, makes them read-only thereafter. + for (auto* n : user_nodes_topo_) n->prepare(); + for (auto* n : fanout_nodes_ptr_) n->prepare(); + for (auto* n : user_nodes_topo_) n->start(); for (auto* n : fanout_nodes_ptr_) n->start(); #ifdef KPN_WEB_DEBUG @@ -145,6 +160,11 @@ public: #ifdef KPN_WEB_DEBUG if (web_server_) web_server_->stop(); #endif + // Release anything parked on a shared resource first. A node blocked in + // acquire() is not watching stop flags, so stopping it would wait on a + // handover that may never come β€” its holder is being stopped too. + for (auto& [rname, probe] : resource_probes_) { (void)rname; probe->close(); } + for (auto it = fanout_nodes_ptr_.rbegin(); it != fanout_nodes_ptr_.rend(); ++it) (*it)->stop(); for (auto it = user_nodes_topo_.rbegin(); it != user_nodes_topo_.rend(); ++it) @@ -158,11 +178,12 @@ public: #ifdef KPN_WEB_DEBUG if (web_server_) web_server_->stop(); #endif + for (auto& [rname, probe] : resource_probes_) { (void)rname; probe->close(); } // user_nodes_topo_ is already in sources-first order. // Stop each node and drain its output channels before moving on. - for (auto* n : user_nodes_topo_) { - n->stop(); - drain_all_channels(); + for (std::size_t i = 0; i < user_nodes_topo_.size(); ++i) { + user_nodes_topo_[i]->stop(); + drain_outputs_of(user_node_names_[i]); } for (auto* n : fanout_nodes_ptr_) n->stop(); } @@ -181,6 +202,18 @@ public: void set_event_handler(EventHandler h) { event_handler_ = std::move(h); } + /// How long shutdown() waits for one node's outputs to drain before giving + /// up on them and stopping the next layer anyway. + void set_drain_timeout(std::chrono::milliseconds t) { drain_timeout_ = t; } + + /// Application-level error listener. Receives the exception any node's + /// function throws, after that node's own handler (if any) declined it. + /// Return true to skip the failed invocation and keep the node running, + /// false to let it stop. Without a listener the exception is discarded + /// and only a Closed event survives, which reports that a node stopped + /// but not why. + void set_error_handler(NodeErrorHandler h) { error_handler_ = std::move(h); } + #ifdef KPN_WEB_DEBUG void set_web_debug_port(uint16_t port) { web_debug_port_ = port; } // Called by DebugHub::register_network() so the hub owns the debug server. @@ -253,16 +286,50 @@ private: return {std::move(nodes), std::move(channels), std::move(resources), std::move(pools), elapsed_s}; } - void drain_all_channels() const { - bool any_full = true; - while (any_full) { - any_full = false; - for (auto& probe : channel_probes_) { - if (probe->snapshot().current_fill > 0) { any_full = true; break; } - } - if (any_full) - std::this_thread::sleep_for(std::chrono::milliseconds(1)); + /// Wait for the channels fed by `src` to empty, or give up. + /// + /// This was an unbounded `while (anything anywhere is non-empty)` poll over + /// *every* channel in the graph, which made shutdown() wait for the whole + /// network to be idle before stopping each successive layer, and wait + /// forever if anything downstream was wedged β€” turning a graceful shutdown + /// into the hang it exists to avoid. + /// + /// Two bounds, because they fail differently. The deadline covers a + /// consumer that has stopped consuming: fill never changes and no amount of + /// waiting helps. The no-progress counter covers a consumer that is merely + /// slow β€” it keeps waiting as long as the queue is shrinking, so a slow + /// drain is not cut short just for exceeding a fixed time. + /// + /// Giving up is reported rather than silent: undrained data at this point + /// means values are about to be discarded by the stop that follows. + void drain_outputs_of(const std::string& src) const { + const auto deadline = clock_t::now() + drain_timeout_; + std::size_t last_fill = static_cast(-1); + int stalls = 0; + + for (;;) { + std::size_t fill = 0; + for (std::size_t i = 0; i < channel_probes_.size(); ++i) + if (channel_src_names_[i] == src) + fill += channel_probes_[i]->snapshot().current_fill; + + if (fill == 0) return; + if (fill >= last_fill) { if (++stalls > 100) break; } + else { stalls = 0; } + last_fill = fill; + + if (clock_t::now() >= deadline) break; + std::this_thread::sleep_for(std::chrono::milliseconds(1)); } + + std::size_t left = 0; + for (std::size_t i = 0; i < channel_probes_.size(); ++i) + if (channel_src_names_[i] == src) + left += channel_probes_[i]->snapshot().current_fill; + if (left) + std::cerr << "[kpn] shutdown: '" << src << "' still has " << left + << " queued item(s) its consumer did not take; " + "they are discarded\n"; } std::string name_; @@ -273,9 +340,14 @@ private: std::vector user_node_names_; std::vector fanout_node_names_; std::vector> channel_probes_; + /// Display name of the node feeding each probe, parallel to channel_probes_. + /// shutdown() drains a node's own outputs, so it has to know which they are. + std::vector channel_src_names_; + std::chrono::milliseconds drain_timeout_{5000}; std::vector> resource_probes_; std::vector> pool_probes_; EventHandler event_handler_; + NodeErrorHandler error_handler_; clock_t::time_point start_time_; #ifdef KPN_WEB_DEBUG uint16_t web_debug_port_{9090}; @@ -308,29 +380,6 @@ auto make_network(Edges&&... edges) { // 4. Construct owned fanout storage on the heap (FanoutNode has jthread β€” not moveable) auto fanout_storage = std::make_unique(); - // 5. Collect unique user node pointers + their display names, in edge-declaration order - std::vector user_node_ptrs; - std::vector user_node_names; - auto collect = [&](auto& e) { - using SrcT = std::decay_t; - using DstT = std::decay_t; - auto* s = static_cast(&e.src); - auto* d = static_cast(&e.dst); - if (std::find(user_node_ptrs.begin(), user_node_ptrs.end(), s) == user_node_ptrs.end()) { - auto sname = node_display_name(); - user_node_ptrs.push_back(s); - user_node_names.push_back(sname); - s->set_name(sname); - } - if (std::find(user_node_ptrs.begin(), user_node_ptrs.end(), d) == user_node_ptrs.end()) { - auto dname = node_display_name(); - user_node_ptrs.push_back(d); - user_node_names.push_back(dname); - d->set_name(dname); - } - }; - (collect(edges), ...); - // 5. Wire all expanded SimpleEdges. // find_node: searches fanout storage then user edge pack, returns NodeT*. // Uses if constexpr in a fold so mismatched types never reach assignment. @@ -355,6 +404,38 @@ auto make_network(Edges&&... edges) { return ptr; }; + // 5. Collect user node pointers + display names in *topological* order. + // + // Topo is computed above for the cycle check and used to be discarded, + // while this vector was filled in edge-declaration order β€” and then named + // user_nodes_topo_ and relied upon as if it were sorted. halt() stops in + // its reverse, and shutdown() walks it forwards stopping each node and + // draining its outputs before the next, which is only a graceful drain if + // the order really is sources-first. It held for every network in the tree + // because edges happen to be declared in pipeline order, and would have + // broken silently for one that was not. + // + // Fanout nodes appear in Topo too; they are skipped here because they are + // owned separately, in fanout_storage. + std::vector user_node_ptrs; + std::vector user_node_names; + [&](tmp::TypeList) { + ([&]() { + if constexpr (!requires { NodeT::is_fanout_node; }) { + if (auto* p = find_node.template operator()()) { + auto* n = static_cast(p); + if (std::find(user_node_ptrs.begin(), user_node_ptrs.end(), n) + == user_node_ptrs.end()) { + auto nm = node_display_name(); + user_node_ptrs.push_back(n); + user_node_names.push_back(nm); + n->set_name(nm); + } + } + } + }.template operator()(), ...); + }(typename Topo::topo{}); + // Pre-pass: build fanout_id β†’ source display name map so fanout nodes // can be named after the node feeding them (e.g. "capture_fanout"). std::map fanout_src_name; @@ -379,6 +460,7 @@ auto make_network(Edges&&... edges) { }; std::vector> channel_probes; + std::vector channel_src_names; auto wire_one = [&](SE) { using SrcNode = typename SE::src_node_t; @@ -396,6 +478,7 @@ auto make_network(Edges&&... edges) { + " \xe2\x86\x92 " // UTF-8 β†’ + node_name.template operator()() + ":" + std::to_string(DstIdx); channel_probes.push_back(std::make_unique>(ch, ch_name)); + channel_src_names.push_back(node_name.template operator()()); } }; @@ -423,7 +506,8 @@ auto make_network(Edges&&... edges) { std::move(fanout_ptrs), std::move(user_node_names), std::move(fanout_node_names), - std::move(channel_probes)); + std::move(channel_probes), + std::move(channel_src_names)); } } // namespace kpn diff --git a/include/kpn/submit_gate.hpp b/include/kpn/submit_gate.hpp new file mode 100644 index 0000000..85cbce8 --- /dev/null +++ b/include/kpn/submit_gate.hpp @@ -0,0 +1,101 @@ +#pragma once +#include + +namespace kpn { + +// ── SubmitGate ──────────────────────────────────────────────────────────────── +// +// Decides, for one node, whether a wake must turn into a scheduler submission. +// Exactly one firing of a node may be in flight at a time, and a wake that +// arrives while one is already in flight must not be lost β€” it has to be +// honoured when that firing finishes, or the node sleeps holding work. +// +// 9c5ce5f wrote this as two independent atomics: queued_ said a firing was in +// flight, wake_pending_ recorded a wake that arrived during one. That cannot be +// made correct, because the release side has to read and write both, and a wake +// can land between the two operations: +// +// producer (try_submit) worker (release_and_recheck) +// ------------------------ ---------------------------- +// CAS reads queued_ == true, fails +// queued_.store(false) +// wake_pending_.exchange(false) -> false +// wake_pending_.store(true) +// +// End state: queued_ false, wake_pending_ true, nothing running and nothing +// scheduled. The node sleeps with a wake outstanding, which is precisely the +// invariant that commit set out to establish. It is not a memory-ordering +// subtlety β€” the interleaving above holds under seq_cst. +// +// It survived because every caller happened to follow release_and_recheck() +// with a level re-check (on_input_ready(), or outputs_have_space() on the +// parked path), which rediscovers the state a lost wake would have signalled. +// That is a property of the call sites, not of the mechanism, and any new early +// return that forgets the re-check turns it back into a hang. +// +// One atomic with three states makes the race unrepresentable: "idle" and "wake +// outstanding" are the same variable, so no interleaving can produce both. +// +// Idle nothing in flight +// Queued a firing is in flight or queued; no wake since it was claimed +// QueuedWake a firing is in flight or queued, and a wake arrived meanwhile +// +class SubmitGate { +public: + /// Register a wake. Returns true when the caller must submit the node; + /// false when a firing is already in flight and the wake has been recorded + /// against it instead. + bool claim() noexcept { + int cur = state_.load(std::memory_order_acquire); + for (;;) { + if (cur == kIdle) { + if (state_.compare_exchange_weak(cur, kQueued, + std::memory_order_acq_rel, std::memory_order_acquire)) + return true; + } else if (cur == kQueued) { + if (state_.compare_exchange_weak(cur, kQueuedWake, + std::memory_order_acq_rel, std::memory_order_acquire)) + return false; + } else { + return false; // a wake is already recorded + } + } + } + + /// End the in-flight firing. Returns true when a wake arrived during it and + /// the caller must submit again β€” in which case the gate stays claimed, so + /// the node is handed straight from one firing to the next and is never + /// momentarily idle with work outstanding. Returns false when the node is + /// now idle. + bool release() noexcept { + int cur = state_.load(std::memory_order_acquire); + for (;;) { + if (cur == kQueuedWake) { + if (state_.compare_exchange_weak(cur, kQueued, + std::memory_order_acq_rel, std::memory_order_acquire)) + return true; + } else { + // kQueued, or kIdle if a stop already forced the gate down. + if (state_.compare_exchange_weak(cur, kIdle, + std::memory_order_acq_rel, std::memory_order_acquire)) + return false; + } + } + } + + /// Drop the claim and any recorded wake. For stop paths only: honouring a + /// wake there would resubmit a dead node. + void force_idle() noexcept { state_.store(kIdle, std::memory_order_release); } + + bool queued() const noexcept { return state_.load(std::memory_order_relaxed) != kIdle; } + bool wake_pending() const noexcept { return state_.load(std::memory_order_relaxed) == kQueuedWake; } + +private: + static constexpr int kIdle = 0; + static constexpr int kQueued = 1; + static constexpr int kQueuedWake = 2; + + std::atomic state_{kIdle}; +}; + +} // namespace kpn diff --git a/include/kpn/traits.hpp b/include/kpn/traits.hpp index 6381822..e81983e 100644 --- a/include/kpn/traits.hpp +++ b/include/kpn/traits.hpp @@ -97,4 +97,41 @@ struct repeat_tuple> { template using repeat_tuple_t = typename repeat_tuple::type; +// ── Sentinel detection ──────────────────────────────────────────────────────── +// A value is a "sentinel" (must-deliver control token, e.g. EOF) if its type +// carries a bool-convertible eof flag β€” either directly (`v.eof`, as on a raw +// source Frame) or nested one level under a `.source` member (`v.source.eof`, +// as on message types that wrap the originating Frame). Sentinels are delivered +// losslessly and non-blockingly via Channel::push_sentinel() instead of the +// throwing push(), so backpressure can never drop the token that unblocks +// downstream teardown. +// +// Types with neither shape are never treated as sentinels β€” both traits are +// SFINAE-safe and the runtime check compiles away to `false` for them, so this +// stays a no-op for pipelines that don't use an eof convention. +// +// Lives here rather than in pool_node.hpp because every node type that forwards +// values needs it, not just the pool-scheduled ones. FilterNode and RouterNode +// not having it is what let an EOF token be dropped on a full output. + +template +struct has_eof_field : std::false_type {}; +template +struct has_eof_field(std::declval().eof))>> + : std::true_type {}; + +template +struct has_source_eof_field : std::false_type {}; +template +struct has_source_eof_field(std::declval().source.eof))>> + : std::true_type {}; + +template +constexpr bool is_sentinel_value(const T& v) { + if constexpr (has_eof_field::value) return static_cast(v.eof); + else if constexpr (has_source_eof_field::value) return static_cast(v.source.eof); + else return false; +} + } // namespace kpn diff --git a/include/kpn/variant_node.hpp b/include/kpn/variant_node.hpp index 0af32e7..26320ca 100644 --- a/include/kpn/variant_node.hpp +++ b/include/kpn/variant_node.hpp @@ -55,6 +55,8 @@ class IVariantChannel { public: virtual ~IVariantChannel() = default; virtual void push(Variant v) = 0; + // Lossless push with backpressure (waits instead of dropping when full). + virtual void push_blocking(Variant v) = 0; virtual Variant pop() = 0; virtual std::type_index type_index() const = 0; virtual std::string type_name() const = 0; @@ -76,6 +78,9 @@ public: void push(Variant v) override { channel_->push(std::get(std::move(v))); } + void push_blocking(Variant v) override { + channel_->push_blocking(std::get(std::move(v))); + } Variant pop() override { return Variant{ channel_->pop() }; } @@ -157,6 +162,7 @@ public: // ── INode ───────────────────────────────────────────────────────────────── + void prepare() override { node_.prepare(); } void start() override { node_.start(); } void stop() override { node_.stop(); } bool running() const override { return node_.running(); } diff --git a/include/kpn/web_debug.hpp b/include/kpn/web_debug.hpp index cd325af..27748be 100644 --- a/include/kpn/web_debug.hpp +++ b/include/kpn/web_debug.hpp @@ -65,6 +65,11 @@ static std::string to_json(const std::vector& nodes, << ",\"fps\":" << n.throughput_fps << ",\"total_cpu_ms\":" << n.total_cpu_ms << ",\"cpu_util_pct\":" << n.cpu_util_pct + // Scheduling state β€” lets a WEDGED pipeline be interrogated over HTTP + // without a debugger, which matters because the lost-wake bug does not + // reproduce under one. See NodeSnapshot for how to read the pair. + << ",\"queued\":" << (n.queued ? "true" : "false") + << ",\"wake_pending\":" << (n.wake_pending ? "true" : "false") << "}"; } o << "],\"edges\":["; diff --git a/scripts/bench_repro_check.py b/scripts/bench_repro_check.py new file mode 100755 index 0000000..6e6413e --- /dev/null +++ b/scripts/bench_repro_check.py @@ -0,0 +1,258 @@ +#!/usr/bin/env python3 +"""Check the PERF_PLAN Phase-0 acceptance criterion. + +Runs bench_pipeline several times and reports, per row, how far the passes +spread around their median. The plan's gate is: the same configuration run 7x +lands within +/-5% on every row. Until that holds, no measured difference +between KPN and TBB is worth acting on. + +Exits non-zero if any row exceeds the tolerance, so it can gate a session of +performance work rather than merely inform one. + +A full sweep is hours, so the run is observable and restartable rather than +opaque: rows stream to --out-dir as each pass produces them, and a progress +bar tracks rows within the pass. Killing the run keeps everything already +written; --resume picks up from the completed passes on disk. + +Usage: + scripts/bench_repro_check.py ./build_bench/benchmarks/bench_pipeline \\ + --passes 7 --tolerance 5 -- --work=10 --topos=chain,wide --reps=5 +""" + +import argparse +import datetime +import os +import pathlib +import statistics +import subprocess +import sys + +KEY_COLS = ("topology", "size", "work_us", "threads") + +# bench_pipeline reports throughput, bench_dispatch reports per-dispatch cost. +# Either is a valid thing to demand reproducibility of; deviation from the +# median is symmetric, so it does not matter which direction is "better". +METRIC_COLS = ("items_per_sec", "ns_per_dispatch") + + +def _progress(total, desc): + """A tqdm bar if tqdm is installed, else a minimal stderr fallback. + + The fallback exists because this script gates a benchmark run; refusing to + start over a missing progress dependency would be the wrong trade. + """ + try: + from tqdm import tqdm + except ImportError: + class Fallback: + def __init__(self): + self.n = 0 + + def update(self, k=1): + self.n += k + end = "\n" if (total and self.n >= total) else "\r" + print(f" {desc}: {self.n}/{total or '?'} rows", + file=sys.stderr, end=end, flush=True) + + def close(self): + pass + + def __enter__(self): + return self + + def __exit__(self, *exc): + self.close() + + return Fallback() + + return tqdm(total=total, desc=desc, unit="row", leave=False, + bar_format=" {desc}: {n_fmt}/{total_fmt} rows " + "|{bar}| {elapsed}<{remaining}", + file=sys.stderr) + + +def parse_csv_lines(lines, metric=None): + """Return ({(topology, size, work_us, threads): value}, metric_name).""" + rows = {} + header = None + for line in lines: + line = line.strip() + if not line or line.startswith("#"): + continue + fields = line.split(",") + if header is None: + if fields[0] != "topology": + continue + header = fields + if metric is None: + for cand in METRIC_COLS: + if cand in header: + metric = cand + break + else: + sys.exit(f"no metric column found in header: {header}") + elif metric not in header: + sys.exit(f"metric {metric!r} not in header: {header}") + continue + rec = dict(zip(header, fields)) + try: + key = tuple(rec[c] for c in KEY_COLS) + rows[key] = float(rec[metric]) + except (KeyError, ValueError): + continue + return rows, metric + + +def parse_csv(text, metric=None): + return parse_csv_lines(text.splitlines(), metric) + + +def count_rows(binary, extra): + """Enumerate the sweep cheaply, so the progress bar has a real total. + + Asks the binary itself rather than reimplementing the sweep in Python, + which would silently drift from the C++ defaults. Returns None if the + probe fails -- an unknown total degrades the bar, it does not stop the run. + """ + probe = [binary] + extra + ["--reps=0", "--warmup=0"] + try: + proc = subprocess.run(probe, capture_output=True, text=True, + timeout=600) + except (subprocess.SubprocessError, OSError): + return None + if proc.returncode != 0: + return None + rows, _ = parse_csv(proc.stdout) + return len(rows) or None + + +def run_pass(binary, extra, total, desc, sink, metric): + """Run one pass, streaming rows to `sink` and the bar as they arrive. + + capture_output would withhold every row until the pass ended, which for a + multi-hour sweep means no way to tell a slow run from a wedged one. + """ + lines = [] + bar = _progress(total, desc) + proc = subprocess.Popen([binary] + extra, stdout=subprocess.PIPE, + stderr=subprocess.PIPE, text=True, bufsize=1) + try: + for line in proc.stdout: + lines.append(line) + if sink: + sink.write(line) + sink.flush() # a killed run keeps its rows + stripped = line.strip() + if (stripped and not stripped.startswith("#") + and "," in stripped + and not stripped.startswith("topology,")): + bar.update(1) + finally: + bar.close() + proc.stdout.close() + stderr = proc.stderr.read() + proc.stderr.close() + rc = proc.wait() + + if rc != 0: + print(stderr, file=sys.stderr) + sys.exit(f"{binary} failed with {rc}") + return parse_csv_lines(lines, metric) + + +def report(passes, metric, tolerance, npasses): + keys = set(passes[0]) + for p in passes[1:]: + keys &= set(p) + if not keys: + sys.exit("no rows common to every pass") + + print(f"\n{'row':<34} {'median ' + metric:>20} {'worst dev':>10} verdict") + print("-" * 72) + + failures = 0 + for key in sorted(keys): + values = [p[key] for p in passes] + med = statistics.median(values) + worst = max(abs(v - med) / med * 100 for v in values) if med else 0.0 + ok = worst <= tolerance + failures += not ok + label = "{}-{} w={} s={}".format(*key) + print(f"{label:<34} {med:>20.1f} {worst:>9.1f}% {'ok' if ok else 'NOISY'}") + + print("-" * 72) + if failures: + print(f"{failures}/{len(keys)} rows exceed +/-{tolerance:g}% β€” " + f"the Phase-0 gate is not met.") + return 1 + print(f"all {len(keys)} rows within +/-{tolerance:g}% " + f"over {npasses} passes β€” Phase-0 gate met.") + return 0 + + +def main(): + ap = argparse.ArgumentParser() + ap.add_argument("binary", help="path to bench_pipeline") + ap.add_argument("--passes", type=int, default=7) + ap.add_argument("--tolerance", type=float, default=5.0, + help="max allowed deviation from the median, percent") + ap.add_argument("--metric", default=None, choices=METRIC_COLS, + help="column to check (default: whichever the CSV carries)") + ap.add_argument("--out-dir", default=None, + help="write pass-NN.csv as rows arrive " + "(default: bench_runs/)") + ap.add_argument("--resume", action="store_true", + help="reuse complete pass-NN.csv files in --out-dir") + + # Everything after a standalone `--` goes to bench_pipeline verbatim. + # argparse.REMAINDER would swallow this script's own flags instead. + argv = sys.argv[1:] + extra = [] + if "--" in argv: + cut = argv.index("--") + argv, extra = argv[:cut], argv[cut + 1:] + args = ap.parse_args(argv) + + out_dir = args.out_dir + if out_dir is None: + if args.resume: + sys.exit("--resume needs an explicit --out-dir") + stamp = datetime.datetime.now().strftime("%Y%m%d-%H%M%S") + out_dir = os.path.join("bench_runs", stamp) + out = pathlib.Path(out_dir) + out.mkdir(parents=True, exist_ok=True) + + total = count_rows(args.binary, extra) + print(f"writing to {out}/", file=sys.stderr) + if total: + print(f"{total} rows per pass, {args.passes} passes", file=sys.stderr) + + passes = [] + metric = args.metric + for i in range(args.passes): + path = out / f"pass-{i + 1:02d}.csv" + + if args.resume and path.exists(): + rows, metric = parse_csv(path.read_text(), metric) + # A partial file from a killed run must not be silently averaged + # in as if it were a whole pass. + if total and len(rows) < total: + print(f"pass {i + 1}/{args.passes}: {path.name} has " + f"{len(rows)}/{total} rows β€” rerunning", file=sys.stderr) + else: + print(f"pass {i + 1}/{args.passes}: reusing {path.name} " + f"({len(rows)} rows)", file=sys.stderr) + passes.append(rows) + continue + + print(f"pass {i + 1}/{args.passes} ...", file=sys.stderr, flush=True) + with open(path, "w") as sink: + rows, metric = run_pass(args.binary, extra, total, + f"pass {i + 1}/{args.passes}", sink, metric) + passes.append(rows) + + return report(passes, metric, args.tolerance, args.passes) + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index a8f7583..07bf131 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -34,7 +34,9 @@ add_executable(kpn_tests test_static_network.cpp test_shared_resource.cpp test_pool_node.cpp + test_backpressure_deadlock.cpp test_scheduler.cpp + test_submit_gate.cpp ) target_link_libraries(kpn_tests PRIVATE @@ -43,6 +45,67 @@ target_link_libraries(kpn_tests PRIVATE GTest::gtest ) +# ── Channel stress suite (separate executable) ──────────────────────────────── +# Contended SPSC tests for the lock-free Channel. Kept out of kpn_tests +# because each case runs many reps / tens of thousands of items and is slow. +# Most valuable under -DKPN_SANITIZER=thread, but correct (and run) without it. +add_executable(kpn_tests_stress test_channel_stress.cpp) +target_link_libraries(kpn_tests_stress PRIVATE kpn Catch2::Catch2WithMain) + +# ── Wedge soak (PERF_PLAN G1) ───────────────────────────────────────────────── +# Long-running end-to-end loop over the configurations that historically wedged. +# Always built, so it cannot rot, but its CTest cases are registered only under +# -DKPN_ENABLE_SOAK_TESTS=ON: they run for minutes and would otherwise dominate +# every `ctest` invocation. Performance work runs it before and after a change: +# +# cmake -B build -DKPN_ENABLE_SOAK_TESTS=ON -DKPN_SOAK_ITERS=50000 +# cmake --build build --target kpn_soak_wedge +# ctest --test-dir build -L soak +# +# The binary self-diagnoses: an iteration that stops making progress trips a +# watchdog that aborts naming the iteration and phase, rather than hanging. +add_executable(kpn_soak_wedge soak_wedge.cpp) +target_link_libraries(kpn_soak_wedge PRIVATE kpn) +target_compile_options(kpn_soak_wedge PRIVATE -O2) + +option(KPN_ENABLE_SOAK_TESTS "Register the wedge soak cases with CTest" OFF) +set(KPN_SOAK_ITERS 5000 CACHE STRING "Iterations per wedge soak case") + +# ── Sanitizer flags ─────────────────────────────────────────────────────────── +# kpn_sanitizer_flags() is defined in the top-level CMakeLists and is a no-op +# unless -DKPN_SANITIZER=... is set. Sanitizer must be on both compile and link. +kpn_sanitizer_flags(_kpn_san) +if(_kpn_san) + foreach(_t kpn_tests kpn_tests_stress kpn_soak_wedge) + target_compile_options(${_t} PRIVATE ${_kpn_san}) + target_link_options(${_t} PRIVATE ${_kpn_san}) + endforeach() +endif() + include(CTest) include(Catch) -catch_discover_tests(kpn_tests) + +# DISCOVERY_MODE PRE_TEST defers test enumeration to `ctest` run time. The +# default (POST_BUILD) runs each test binary during the build to list its +# cases β€” which fails a sanitizer build: a TSan/ASan binary needs a fixed +# address-space layout and aborts on startup ("unexpected memory mapping") +# under the container's ASLR, breaking the build before any test runs. The +# tsan.yaml job invokes the binaries directly (not via ctest), so deferring +# discovery costs nothing there and keeps `ctest` working for normal builds. +catch_discover_tests(kpn_tests DISCOVERY_MODE PRE_TEST) +# Register the stress suite under its own label so CI can run / time it +# separately from the fast unit tests. +catch_discover_tests(kpn_tests_stress DISCOVERY_MODE PRE_TEST PROPERTIES LABELS "stress") + +if(KPN_ENABLE_SOAK_TESTS) + # pool: the configuration the August wedges were reproduced on. + add_test(NAME soak.wedge.pool + COMMAND kpn_soak_wedge --mode=pool --depth=4 --threads=4 + --items=1000 --work-us=10 --iters=${KPN_SOAK_ITERS}) + # private: one pool per node β€” the model workstream A would change. + add_test(NAME soak.wedge.private + COMMAND kpn_soak_wedge --mode=priv --depth=8 + --items=1000 --work-us=10 --iters=${KPN_SOAK_ITERS}) + set_tests_properties(soak.wedge.pool soak.wedge.private PROPERTIES + LABELS "soak" TIMEOUT 3600) +endif() diff --git a/tests/soak_wedge.cpp b/tests/soak_wedge.cpp new file mode 100644 index 0000000..033402e --- /dev/null +++ b/tests/soak_wedge.cpp @@ -0,0 +1,236 @@ +// Wedge soak test (PERF_PLAN G1). +// +// Runs a pipeline configuration end-to-end in a loop and fails if any single +// iteration stops making progress. Its purpose is to keep performance work +// from silently reintroducing one of the wedges fixed in August 2026 β€” the +// scheduler and channel wake paths are where both perf workstreams operate. +// +// Originally the minimal reproducer for the shared-pool chain wedge at +// (chain, depth=4, work_us=10, pool_threads=4); the pre-6802328 code wedged +// 5/5 within 45 s, at iterations 149, 1249, 332, 1740 and 493. +// +// A wedge is a hang, so a plain loop would hang CTest until its timeout with +// no indication of where. The watchdog turns that into a failure naming the +// iteration and the phase it stalled in. +// +// Usage: ./kpn_soak_wedge [options] +// --iters=5000 iterations to run +// --mode=pool|priv shared ThreadPool(--threads), or one private pool/node +// --depth=4 chain depth +// --threads=4 shared pool size (--mode=pool only) +// --items=1000 items pushed per iteration +// --work-us=10 busy-work per node +// --watchdog-sec=30 per-iteration progress deadline + +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#if defined(__linux__) +#include +#endif + +using namespace kpn; +using sclock = std::chrono::steady_clock; + +static std::atomic g_work_us{10}; + +static int chain_fn(int x) { + int us = g_work_us.load(std::memory_order_relaxed); + if (us > 0) { + auto end = sclock::now() + std::chrono::microseconds(us); + while (sclock::now() < end); + } + return x; +} + +using ChainNode = Node, out<>>; +using PoolChainNode = PoolNode, out<>>; + +static void push_retry(Channel& ch, int val) { + while (true) { + try { ch.push(val); return; } + catch (const ChannelOverflowError&) { std::this_thread::yield(); } + catch (const ChannelClosedError&) { return; } + } +} + +// ── watchdog ────────────────────────────────────────────────────────────────── +// +// The worker bumps g_progress at every phase boundary. The watchdog aborts if +// it stops moving, so a wedge is reported as a failure at a known iteration +// rather than as an unattributable CTest timeout. + +static std::atomic g_progress{0}; +static std::atomic g_iter{0}; +static std::atomic g_phase{"init"}; +static std::atomic g_done{false}; + +static void mark(const char* phase) { + g_phase.store(phase, std::memory_order_relaxed); + g_progress.fetch_add(1, std::memory_order_release); +} + +static void watchdog(double deadline_sec) { + unsigned long last = g_progress.load(std::memory_order_acquire); + auto last_move = sclock::now(); + while (!g_done.load(std::memory_order_acquire)) { + std::this_thread::sleep_for(std::chrono::milliseconds(100)); + unsigned long now = g_progress.load(std::memory_order_acquire); + if (now != last) { last = now; last_move = sclock::now(); continue; } + double stalled = std::chrono::duration(sclock::now() - last_move).count(); + if (stalled > deadline_sec) { + std::fprintf(stderr, + "\nWEDGE: no progress for %.0fs at iteration %d, phase '%s'\n", + stalled, g_iter.load(std::memory_order_relaxed), + g_phase.load(std::memory_order_relaxed)); + std::fflush(stderr); + std::abort(); // core dump / stack trace at the point of the wedge + } + } +} + +// ── one iteration ───────────────────────────────────────────────────────────── + +struct Opts { + int iters = 5000; + int depth = 4; + int threads = 4; + int items = 1000; + int work_us = 10; + bool shared_pool = true; + double watchdog_sec = 30.0; +}; + +static void one_round_pool(const Opts& o) { + const std::size_t CAP = static_cast(o.items); + auto pool = std::make_shared(o.threads); + + std::vector>> chs; + for (int i = 0; i <= o.depth; ++i) + chs.push_back(std::make_shared>(CAP)); + + std::vector> nodes; + for (int i = 0; i < o.depth; ++i) { + nodes.push_back(std::make_unique(pool, CAP)); + nodes.back()->set_input_channel<0>(chs[i]); + nodes.back()->set_output_channel<0>(chs[i + 1].get()); + } + + pool->start(); + for (auto& n : nodes) n->start(); + mark("started"); + + std::thread reader([&] { + for (int i = 0; i < o.items; ++i) chs.back()->pop(); + }); + std::thread pusher([&] { + for (int i = 0; i < o.items; ++i) push_retry(*chs[0], i); + }); + + pusher.join(); mark("pushed"); + reader.join(); mark("drained"); + for (auto& n : nodes) n->stop(); + mark("nodes stopped"); + pool->stop(); + mark("pool stopped"); +} + +static void one_round_private(const Opts& o) { + const std::size_t CAP = static_cast(o.items); + + std::vector>> chs; + for (int i = 0; i <= o.depth; ++i) + chs.push_back(std::make_shared>(CAP)); + + std::vector> nodes; + for (int i = 0; i < o.depth; ++i) { + nodes.push_back(std::make_unique(CAP)); + nodes.back()->set_input_channel<0>(chs[i]); + nodes.back()->set_output_channel<0>(chs[i + 1].get()); + } + + for (auto& n : nodes) n->start(); + mark("started"); + + std::thread reader([&] { + for (int i = 0; i < o.items; ++i) chs.back()->pop(); + }); + std::thread pusher([&] { + for (int i = 0; i < o.items; ++i) push_retry(*chs[0], i); + }); + + pusher.join(); mark("pushed"); + reader.join(); mark("drained"); + for (auto& n : nodes) n->stop(); + mark("nodes stopped"); +} + +// ── main ────────────────────────────────────────────────────────────────────── + +static void usage() { + std::fprintf(stderr, + "usage: kpn_soak_wedge [--iters=N] [--mode=pool|priv] [--depth=D]\n" + " [--threads=T] [--items=N] [--work-us=U]\n" + " [--watchdog-sec=S]\n"); +} + +int main(int argc, char** argv) { +#if defined(__linux__) + // Allow gdb to attach under ptrace_scope=1 when a wedge is caught. + prctl(PR_SET_PTRACER, PR_SET_PTRACER_ANY, 0, 0, 0); +#endif + + Opts o; + for (int i = 1; i < argc; ++i) { + std::string a = argv[i]; + auto eq = a.find('='); + std::string key = a.substr(0, eq); + std::string val = eq == std::string::npos ? "" : a.substr(eq + 1); + + if (key == "--iters") o.iters = std::atoi(val.c_str()); + else if (key == "--depth") o.depth = std::atoi(val.c_str()); + else if (key == "--threads") o.threads = std::atoi(val.c_str()); + else if (key == "--items") o.items = std::atoi(val.c_str()); + else if (key == "--work-us") o.work_us = std::atoi(val.c_str()); + else if (key == "--watchdog-sec") o.watchdog_sec = std::atof(val.c_str()); + else if (key == "--mode") o.shared_pool = (val != "priv"); + else { usage(); return 2; } + } + g_work_us.store(o.work_us, std::memory_order_relaxed); + + std::fprintf(stderr, + "soak: mode=%s depth=%d threads=%d items=%d work_us=%d iters=%d watchdog=%.0fs\n", + o.shared_pool ? "pool" : "priv", o.depth, + o.shared_pool ? o.threads : o.depth, o.items, o.work_us, + o.iters, o.watchdog_sec); + + std::thread wd(watchdog, o.watchdog_sec); + + const auto t0 = sclock::now(); + for (int i = 0; i < o.iters; ++i) { + g_iter.store(i, std::memory_order_relaxed); + if (o.shared_pool) one_round_pool(o); + else one_round_private(o); + if ((i + 1) % 100 == 0) { + std::fprintf(stderr, "\r %d/%d", i + 1, o.iters); + std::fflush(stderr); + } + } + + g_done.store(true, std::memory_order_release); + wd.join(); + + double secs = std::chrono::duration(sclock::now() - t0).count(); + std::fprintf(stderr, "\ncompleted %d iterations in %.1fs with no wedge\n", + o.iters, secs); + return 0; +} diff --git a/tests/test_backpressure_deadlock.cpp b/tests/test_backpressure_deadlock.cpp new file mode 100644 index 0000000..f6909d0 --- /dev/null +++ b/tests/test_backpressure_deadlock.cpp @@ -0,0 +1,421 @@ +// Regression: a blocking push must not park a pool worker. +// +// Node outputs use push_blocking so a full channel costs time rather than data +// (a dropped frame does not degrade a downstream result, it silently changes +// one). But push_blocking sleeps *inside* fire_once, which runs on a pool +// worker β€” and nodes are pinned to workers by index. Park enough workers in +// that retry loop and there is nobody left to run the consumer that would drain +// the channel, so the whole chain wedges. +// +// This is the failure channel.hpp:174 already warns about for sentinels +// ("a blocking push would park that thread and stop it draining its own input, +// cascading into a hold-and-wait deadlock under backpressure"). The warning +// applies to data pushes too. +// +// Observed in the field as an intermittent hang: frame_source, camera_pos, +// face_detector and face_aligner all asleep in push_blocking at once. +#include +#include + +#include +#include +#include + +namespace { + +struct Produce { + static constexpr std::string_view label() { return "produce"; } + int n{0}; + int operator()() { return n++; } +}; + +struct Relay { + static constexpr std::string_view label() { return "relay"; } + int operator()(int v) { return v; } +}; + +// Deliberately slower than the producer, so the channels between them fill. +struct SlowSink { + static constexpr std::string_view label() { return "slow_sink"; } + std::atomic* seen; + void operator()(int) { + std::this_thread::sleep_for(std::chrono::milliseconds(2)); + seen->fetch_add(1, std::memory_order_relaxed); + } +}; + +} // namespace + +TEST_CASE("a saturated chain keeps draining", "[backpressure][deadlock]") { + std::atomic seen{0}; + + Produce p_fn; + Relay r1_fn, r2_fn, r3_fn; + SlowSink s_fn{&seen}; + + // Small channels so they saturate immediately, and a chain longer than a + // modest pool β€” the shape that starves workers. + kpn::ObjectNode, kpn::out<"a">, "produce", 0> p (p_fn, 2); + kpn::ObjectNode, kpn::out<"b">, "relay1", 0> r1(r1_fn, 2); + kpn::ObjectNode, kpn::out<"c">, "relay2", 0> r2(r2_fn, 2); + kpn::ObjectNode, kpn::out<"d">, "relay3", 0> r3(r3_fn, 2); + kpn::ObjectNode, kpn::out<>, "slow_sink", 0> s (s_fn, 2); + + auto net = kpn::make_network( + kpn::edge(p.output<"a">(), r1.input<"a">()), + kpn::edge(r1.output<"b">(), r2.input<"b">()), + kpn::edge(r2.output<"c">(), r3.input<"c">()), + kpn::edge(r3.output<"d">(), s.input<"d">()) + ); + net.start(); + + // The sink is the slowest stage at 2 ms/item, so 40 items is ~80 ms of real + // work. Anything approaching the timeout means the chain stopped draining + // rather than merely running slowly. + const auto deadline = std::chrono::steady_clock::now() + std::chrono::seconds(20); + while (seen.load(std::memory_order_relaxed) < 40 && + std::chrono::steady_clock::now() < deadline) + std::this_thread::sleep_for(std::chrono::milliseconds(5)); + + const int got = seen.load(std::memory_order_relaxed); + net.stop(); + + INFO("items drained: " << got << " of 40"); + CHECK(got >= 40); +} + +// Regression: a saturated chain must never stall. +// +// push_outputs parks from two places: the retry at the top of fire_once, and +// the ordinary push after the node function returns. Both release the worker, +// so both face the same lost wakeup β€” a space callback firing while queued_ is +// still up is swallowed by try_submit's CAS. Only the retry path re-checked for +// space afterwards. The normal path fell through to on_input_ready(), which +// resubmits only if inputs are ready β€” and the firing that just parked had +// consumed its input, so they are not. +// +// The strand is permanent under saturation: the node holds its value, its +// consumer waits for exactly that value, and its producer fills the node's +// input channel and parks too. Nothing moves again. +// +// The test above cannot catch it β€” 40 items drain before any strand occurs. +// This one runs the chain saturated and watches for progress to *freeze*, which +// is the signature of the deadlock. It deliberately does not assert a total: +// capacity-1 channels are slow, and "slow" must never be reported as "wedged". +namespace { + +struct FreeRun { + static constexpr std::string_view label() { return "free_run"; } + int n{0}; + int operator()() { return n++; } +}; + +struct CountingSink { + static constexpr std::string_view label() { return "counting_sink"; } + std::atomic* seen; + void operator()(int) { seen->fetch_add(1, std::memory_order_relaxed); } +}; + +} // namespace + +TEST_CASE("a saturated chain never stalls", "[backpressure][deadlock]") { + std::atomic seen{0}; + + FreeRun p_fn; + Relay r1_fn, r2_fn; + CountingSink s_fn{&seen}; + + // Capacity 1 everywhere: every push contends, so the park path is taken + // constantly and the race window is sampled millions of times. + kpn::ObjectNode, kpn::out<"a">, "free_run", 0> p (p_fn, 1); + kpn::ObjectNode, kpn::out<"b">, "relay1", 0> r1(r1_fn, 1); + kpn::ObjectNode, kpn::out<"c">, "relay2", 0> r2(r2_fn, 1); + kpn::ObjectNode, kpn::out<>, "sink", 0> s (s_fn, 1); + + auto net = kpn::make_network( + kpn::edge(p.output<"a">(), r1.input<"a">()), + kpn::edge(r1.output<"b">(), r2.input<"b">()), + kpn::edge(r2.output<"c">(), s.input<"c">()) + ); + net.start(); + + // A live chain moves thousands of items a second, so 3 s with no movement + // at all is a wedge, not a slow patch. Sampling for 25 s gives the race + // ample opportunity: the pipeline hit it roughly twice in 30 runs. + const auto giveup = std::chrono::steady_clock::now() + std::chrono::seconds(25); + int last = 0; + auto last_move = std::chrono::steady_clock::now(); + bool stalled = false; + int stall_at = 0; + + while (std::chrono::steady_clock::now() < giveup) { + std::this_thread::sleep_for(std::chrono::milliseconds(100)); + const int now_seen = seen.load(std::memory_order_relaxed); + if (now_seen != last) { + last = now_seen; + last_move = std::chrono::steady_clock::now(); + } else if (std::chrono::steady_clock::now() - last_move > + std::chrono::seconds(3)) { + stalled = true; + stall_at = now_seen; + break; + } + } + + net.stop(); + + INFO("chain stalled after " << stall_at << " items"); + CHECK_FALSE(stalled); + // Guard against the test passing because nothing ever ran. + CHECK(last > 1000); +} + +// Regression: a node must not start with a wake already outstanding. +// +// 9c5ce5f established the invariant for the running pipeline β€” a node never +// sleeps with a wake it dropped. start() broke the same invariant before the +// pipeline was even running: +// +// enable_inputs(...); // channel goes live here +// stop_flag_.store(false); +// queued_.store(false); +// register_callbacks(...); // push callback installed here +// +// StaticNetwork starts nodes sources-first, so an upstream node is already +// firing into this one during that gap. A push landing there is accepted by the +// ring but wakes nobody: Channel::push invokes push_callback_ only on the +// emptyβ†’non-empty transition, and at that instant the callback is null. Every +// later push sees a non-empty ring and stays silent. The node is never +// submitted, and since a sink has no outputs there is no space callback to +// rescue it either. +// +// The signature is distinctive: **zero** items delivered, not a stall partway. +// The chain reads as wedged from the first frame. Under `ctest -j4` on a loaded +// machine it reproduced 7 times in 24, and never once in 10 unloaded runs β€” +// contention widens the window between those two statements. That is almost +// certainly the "rare hang, ~1 run in 20 at a 300 s timeout" 28e0667 recorded as +// known-incomplete. +// +// This test needs no contention: it constructs the state the race leaves behind +// directly, by enabling the input and pushing before start() is ever called. +namespace { + +int passthrough(int x) { return x; } + +} // namespace + +TEST_CASE("a node started with data already queued still fires", + "[backpressure][startup]") { + auto pool = std::make_shared(2); + pool->start(); + + auto node = kpn::make_pool_node(pool, 8); + kpn::Channel out_ch(8); + node.set_output_channel<0>(&out_ch); + + // The missed edge: the channel is live and already holds a value, but no + // callback was installed when it arrived, so the wake has been and gone. + node.input_channel<0>().enable(); + node.input_channel<0>().push(21); + + node.start(); + + // Bounded wait β€” a plain pop() would hang rather than fail on a regression. + const auto deadline = std::chrono::steady_clock::now() + std::chrono::seconds(5); + while (out_ch.size() == 0 && std::chrono::steady_clock::now() < deadline) + std::this_thread::sleep_for(std::chrono::milliseconds(5)); + + const bool delivered = out_ch.size() > 0; + const int got = delivered ? out_ch.pop() : -1; + + node.stop(); + pool->stop(); + + INFO("value delivered: " << got); + REQUIRE(delivered); + CHECK(got == 21); +} + +// Regression: a fanout absorbs an unequal pair of consumers by slowing, not by +// dropping. +// +// 6595e6e made node outputs lossless and 28e0667 stopped them parking a worker, +// but FanoutNode was in neither: it kept `catch (ChannelOverflowError&) {}` per +// output, so whichever branch fell behind lost items β€” silently, and by an +// amount that depended on timing. Two runs of the same input could therefore +// disagree, which is fatal for a fixture the rest of the suite is scored +// against. +// +// The two assertions are the two halves of the requirement: +// - no gaps: the slow branch receives *every* item, not most of them; +// - bounded lead: the fast branch is throttled to the slow one rather than +// racing ahead over a drain that is quietly discarding the difference. +// +// Either alone would pass on a broken implementation. A fanout that pushed only +// to the slow branch has no gaps; one that dropped everything for the slow +// branch keeps a bounded lead by never letting it fall behind. +namespace { + +// Records the sequence it sees, so a dropped item shows up as a gap rather than +// merely as a smaller total. +struct SeqCheck { + std::atomic* next_expected; + std::atomic* saw_gap; + int delay_us{0}; + + void record(int v) const { + if (delay_us) + std::this_thread::sleep_for(std::chrono::microseconds(delay_us)); + const int want = next_expected->load(std::memory_order_relaxed); + if (v != want) saw_gap->store(true, std::memory_order_relaxed); + else next_expected->store(want + 1, std::memory_order_relaxed); + } +}; + +struct FastBranch : SeqCheck { + static constexpr std::string_view label() { return "fast_branch"; } + void operator()(int v) { record(v); } +}; + +struct SlowBranch : SeqCheck { + static constexpr std::string_view label() { return "slow_branch"; } + void operator()(int v) { record(v); } +}; + +} // namespace + +TEST_CASE("a fanout absorbs an unequal pair by slowing, not dropping", + "[backpressure][fanout]") { + std::atomic fast_next{0}, slow_next{0}; + std::atomic fast_gap{false}, slow_gap{false}; + + FreeRun p_fn; + FastBranch fast_fn{{&fast_next, &fast_gap, 0}}; + SlowBranch slow_fn{{&slow_next, &slow_gap, 500}}; // 0.5 ms/item + + // Small channels so the slow branch saturates in the first few milliseconds + // and stays saturated for the whole run. + kpn::ObjectNode, kpn::out<"v">, "free_run", 0> p (p_fn, 8); + kpn::ObjectNode, kpn::out<>, "fast", 0> fa(fast_fn, 8); + kpn::ObjectNode, kpn::out<>, "slow", 0> sl(slow_fn, 8); + + // Two edges from one output port: make_network auto-inserts FanoutNode. + auto net = kpn::make_network( + kpn::edge(p.output<"v">(), fa.input<"fast">()), + kpn::edge(p.output<"v">(), sl.input<"slow">()) + ); + net.start(); + std::this_thread::sleep_for(std::chrono::seconds(1)); + net.stop(); + + const int fast_seen = fast_next.load(std::memory_order_relaxed); + const int slow_seen = slow_next.load(std::memory_order_relaxed); + + INFO("fast branch " << fast_seen << " items, slow branch " << slow_seen); + CHECK_FALSE(fast_gap.load(std::memory_order_relaxed)); + CHECK_FALSE(slow_gap.load(std::memory_order_relaxed)); + // Guard against passing because nothing ran: 1 s at 0.5 ms/item is ~2000. + CHECK(slow_seen > 200); + // The lead is bounded by the buffering between the two β€” the fanout's own + // input, the two output channels, and one item in each node's hand. A + // dropping fanout has no such bound: the fast branch runs at full speed and + // the difference is the loss. + CHECK(fast_seen - slow_seen < 200); +} + +// Regression: a filter must not drop an EOF sentinel into a full output. +// +// RouterNode and FilterNode were the last nodes on a data path still using the +// throwing push() and swallowing the result: +// +// try { out_ch_->push(val); } catch (const ChannelOverflowError&) {} +// +// 6595e6e made node outputs lossless, 28e0667 stopped them parking a worker, +// a8cfe73 did the same for FanoutNode. These two were in none of them. +// +// For ordinary values that is the familiar silent-loss problem. For a sentinel +// it is a hang. EOF is what tells every downstream node to shut down, and +// nothing comes after it to retry β€” so a filter that passes EOF by predicate +// but drops it by backpressure produces a pipeline that never terminates. The +// scene-actor-extraction decimator is exactly this shape: `if (f.eof) return +// true;` in the predicate, feeding a chain whose slowest node is an ONNX +// embedder, so the output is reliably full at the moment EOF arrives. +// +// The test forces that state rather than racing for it: the sink is slow enough +// that the filter's output channel is saturated for the whole run, so EOF meets +// a full ring with certainty. +// +// Both assertions are needed. `saw_eof` alone would pass on an implementation +// that dropped every ordinary value and delivered only the sentinel; `count` +// alone would pass on the broken one, which delivers plenty of values and loses +// only the token that matters. +namespace { + +struct EofFrame { + int seq{0}; + bool eof{false}; +}; + +// EOF is emitted exactly once, as a real source does. Everything after it is a +// filler frame the predicate rejects, which keeps the node alive without +// re-offering the sentinel β€” a source that retried EOF would mask the bug, +// since a later attempt could find the channel drained. +struct EofSource { + static constexpr std::string_view label() { return "eof_source"; } + int n{0}; + int total{0}; + EofFrame operator()() { + if (n > total) { + std::this_thread::sleep_for(std::chrono::milliseconds(1)); + return {-1, false}; // filler: dropped by the predicate + } + EofFrame f{n, n == total}; + ++n; + return f; + } +}; + +struct EofSink { + static constexpr std::string_view label() { return "eof_sink"; } + std::atomic* count; + std::atomic* saw_eof; + void operator()(EofFrame f) { + std::this_thread::sleep_for(std::chrono::microseconds(200)); + if (f.eof) saw_eof->store(true, std::memory_order_release); + else count->fetch_add(1, std::memory_order_relaxed); + } +}; + +} // namespace + +TEST_CASE("a filter delivers EOF into a saturated output", "[backpressure][filter]") { + std::atomic count{0}; + std::atomic saw_eof{false}; + + EofSource src_fn{0, 40}; + EofSink sink_fn{&count, &saw_eof}; + + // Every real frame passes the predicate, so the only thing between source + // and sink is backpressure. Small channels keep the output saturated. + auto filt = kpn::make_filter( + [](const EofFrame& f) { return f.seq >= 0; }, 4); + + kpn::ObjectNode, kpn::out<"f">, "eof_source", 0> s(src_fn, 4); + kpn::ObjectNode, kpn::out<>, "eof_sink", 0> k(sink_fn, 4); + + auto net = kpn::make_network( + kpn::edge(s.output<"f">(), filt.input<0>()), + kpn::edge(filt.output<0>(), k.input<"f">()) + ); + net.start(); + + // Generous relative to 41 frames at 200 us, and this is a liveness test: + // the broken implementation never sets saw_eof no matter how long it runs. + for (int i = 0; i < 200 && !saw_eof.load(std::memory_order_acquire); ++i) + std::this_thread::sleep_for(std::chrono::milliseconds(10)); + net.stop(); + + INFO("values delivered: " << count.load() << " of 40"); + CHECK(saw_eof.load(std::memory_order_acquire)); + CHECK(count.load(std::memory_order_relaxed) == 40); +} diff --git a/tests/test_channel.cpp b/tests/test_channel.cpp index 50b11d0..7b660c7 100644 --- a/tests/test_channel.cpp +++ b/tests/test_channel.cpp @@ -1,3 +1,4 @@ +#include #include #include #include @@ -175,3 +176,161 @@ TEST_CASE("bandwidth_mbs returns 0 when elapsed_s is zero or negative", "[channe REQUIRE(snap.bandwidth_mbs(0.0) == 0.0); REQUIRE(snap.bandwidth_mbs(-1.0) == 0.0); } + +TEST_CASE("push_sentinel never overflows even on a full channel", "[channel][sentinel]") { + Channel ch(2); + ch.push(1); + ch.push(2); // channel full β€” a plain push(3) would throw ChannelOverflowError + + // The sentinel is stored out-of-band, so it neither throws nor blocks the + // caller β€” the exact property an EOF token needs under backpressure. This + // returns immediately with the ring still full. + REQUIRE(ch.push_sentinel(99)); + REQUIRE(ch.size() == 2); // sentinel did not consume ring capacity +} + +TEST_CASE("push_sentinel is delivered after all ring data, in order", "[channel][sentinel]") { + Channel ch(4); + ch.push(1); + ch.push(2); + ch.push_sentinel(99); // enqueue EOF while data is still buffered + + // Data drains first; the sentinel arrives only once the ring is empty. + REQUIRE(ch.pop() == 1); + REQUIRE(ch.pop() == 2); + REQUIRE(ch.pop() == 99); +} + +TEST_CASE("push_sentinel wakes a blocked pop", "[channel][sentinel]") { + Channel ch(2); // empty + std::thread producer([&] { + std::this_thread::sleep_for(std::chrono::milliseconds(20)); + ch.push_sentinel(99); // must wake a consumer parked on an empty ring + }); + REQUIRE(ch.pop() == 99); + producer.join(); +} + +TEST_CASE("approx_size counts a pending sentinel so consumers stay schedulable", + "[channel][sentinel]") { + Channel ch(4); + REQUIRE(ch.approx_size() == 0); + ch.push_sentinel(99); + // Node readiness checks call approx_size(); it must report the out-of-band + // sentinel as consumable work even though it holds no ring slot. + REQUIRE(ch.approx_size() == 1); + REQUIRE(ch.size() == 0); // ...but the ring itself is still empty + int out = 0; + REQUIRE(ch.try_pop_now(out)); + REQUIRE(out == 99); + REQUIRE(ch.approx_size() == 0); +} + +TEST_CASE("try_pop_now delivers a pending sentinel once the ring is empty", + "[channel][sentinel]") { + Channel ch(2); + ch.push(1); + ch.push_sentinel(99); + + int out = 0; + REQUIRE(ch.try_pop_now(out)); // ring data first + REQUIRE(out == 1); + REQUIRE(ch.try_pop_now(out)); // then the sentinel + REQUIRE(out == 99); + REQUIRE_FALSE(ch.try_pop_now(out)); // nothing left +} + +// Regression: the sentinel slot holds one token and refuses a second. +// +// push_sentinel used to write eof_value_ unconditionally. Offering a second +// token before the first was taken therefore did two wrong things at once: it +// lost the first silently β€” and a lost EOF wedges every downstream pop forever +// β€” and it wrote the storage while the consumer could be moving the previous +// value out of it. For the shared_ptr storage that non-trivial types use, that +// is a torn refcount, not merely a stale read. +// +// Refusing is correct rather than queueing: two control tokens on one channel +// means the stream ended twice, which is a caller protocol error. Coalescing +// them would hide it, and there is no second value that could sensibly follow +// the end of a stream. +TEST_CASE("a second sentinel is refused, not swallowed", "[channel][sentinel]") { + Channel ch(4); + + REQUIRE(ch.push_sentinel(1)); + // Slot occupied: the first token is still undelivered. + REQUIRE_FALSE(ch.push_sentinel(2)); + + // The first survives intact β€” the overwrite is what used to lose it. + int out = 0; + REQUIRE(ch.try_pop_now(out)); + CHECK(out == 1); + + // And the slot is reusable once drained. + REQUIRE(ch.push_sentinel(3)); + REQUIRE(ch.try_pop_now(out)); + CHECK(out == 3); +} + +TEST_CASE("a refused sentinel is not counted as a drop", "[channel][sentinel]") { + // A refusal means a token arrived while an equivalent one was already + // pending β€” not that anything was lost. Counting it as a drop was wrong in + // a way that showed up immediately on real content: a source at the end of + // its input keeps being polled and keeps returning EOF, so the token is + // re-offered on every firing, and the pipeline reported hundreds of dropped + // frames on a clean run and exited non-zero. + // + // The delivery guarantee is unaffected: the first token is pending and will + // arrive. Only the accounting changed. + Channel ch(4); + REQUIRE(ch.push_sentinel(1)); + const auto before = ch.stats().drops.load(); + REQUIRE_FALSE(ch.push_sentinel(2)); + CHECK(ch.stats().drops.load() == before); + + // And the one that was accepted is still the one delivered. + int out = 0; + REQUIRE(ch.try_pop_now(out)); + CHECK(out == 1); +} + +TEST_CASE("try_push_sentinel leaves a refused value untouched", "[channel][sentinel]") { + // The non-consuming form exists so a refused token is still the caller's to + // report. The consuming push_sentinel cannot offer that, since the value is + // already moved into its parameter. + Channel ch(4); + std::string first = "eof-1", second = "eof-2"; + + REQUIRE(ch.try_push_sentinel(first) == Channel::SentinelResult::Taken); + REQUIRE(ch.try_push_sentinel(second) == Channel::SentinelResult::SlotBusy); + CHECK(second == "eof-2"); // not moved from + + ch.disable(); + std::string third = "eof-3"; + CHECK(ch.try_push_sentinel(third) == Channel::SentinelResult::Closed); + CHECK(third == "eof-3"); +} + +// Regression: try_push must distinguish delivered from discarded. +// +// It returned bool, and returned *true* for a closed channel β€” so "the value +// arrived" and "the value was thrown away because nobody is listening" were the +// same answer. Every caller was nonetheless correct, because both cases mean +// "stop trying"; but nothing above the channel could tell the two apart, and a +// producer counting successful pushes counted discards among them. Only the +// channel's own drop counter knew, and only if someone read the diagnostics. +TEST_CASE("try_push distinguishes taken, full and closed", "[channel]") { + Channel ch(2); + int v = 1; + + CHECK(ch.try_push(v) == Channel::PushResult::Taken); + CHECK(ch.try_push(v) == Channel::PushResult::Taken); + // Ring is full: the value is untouched and the caller keeps it. + CHECK(ch.try_push(v) == Channel::PushResult::Full); + CHECK(v == 1); + + ch.disable(); + const auto drops_before = ch.stats().drops.load(); + CHECK(ch.try_push(v) == Channel::PushResult::Closed); + // Discarded, and recorded as such rather than reported as a delivery. + CHECK(ch.stats().drops.load() == drops_before + 1); +} diff --git a/tests/test_channel_stress.cpp b/tests/test_channel_stress.cpp new file mode 100644 index 0000000..58ac3a0 --- /dev/null +++ b/tests/test_channel_stress.cpp @@ -0,0 +1,336 @@ +// Contended stress tests for the lock-free SPSC Channel. +// +// The other channel tests (test_channel.cpp) are single-threaded or use a +// single 20 ms sleep to order two threads β€” they never actually contend on the +// ring, so they exercise neither the memory-ordering pairing nor the +// spin/futex/lost-wakeup logic in pop(). +// +// These tests are written to be run under ThreadSanitizer: +// +// cmake -B build -DKPN_SANITIZER=thread -DKPN_BUILD_EXAMPLES=OFF -DKPN_BUILD_PYTHON=OFF +// cmake --build build --target kpn_tests_tsan +// ./build/tests/kpn_tests_tsan +// +// They are also valid (and meaningful) without a sanitizer: the value/sequence +// assertions catch lost or duplicated items regardless of build flags. TSan +// adds detection of the underlying data race even on runs where the race did +// not corrupt observable state. +// +// Channel is SPSC: exactly one producer thread and one consumer thread per +// channel. Every scenario below honours that contract. + +#include +#include +#include +#include +#include +#include +#include + +using namespace kpn; +using namespace std::chrono_literals; + +namespace { + +// Repeat each scenario enough times that rare interleavings (spin window just +// missing / just catching the next push, disable landing inside the futex +// wait) actually occur across a run. Kept modest so a TSan run stays minutes, +// not hours. +constexpr int kReps = 200; + +} // namespace + +TEST_CASE("SPSC: every pushed item is popped exactly once, in order", "[channel][stress]") { + // Small capacity forces frequent full/empty transitions, so both the + // producer's overflow-retry and the consumer's spin->futex path are hit + // many times. The producer retries on overflow rather than dropping, so + // the consumer must observe a strictly contiguous 0..N-1 sequence. + constexpr int N = 50'000; + Channel ch(/*capacity=*/4, /*spin_count=*/16); + + std::thread producer([&] { + for (int i = 0; i < N; ++i) { + for (;;) { + try { ch.push(i); break; } + catch (const ChannelOverflowError&) { std::this_thread::yield(); } + } + } + }); + + int expected = 0; + bool in_order = true; + for (int i = 0; i < N; ++i) { + int v = ch.pop(); + if (v != expected) in_order = false; + ++expected; + } + producer.join(); + + REQUIRE(in_order); + REQUIRE(expected == N); + REQUIRE(ch.size() == 0); +} + +TEST_CASE("SPSC: tight empty<->non-empty transitions exercise spin/futex boundary", + "[channel][stress]") { + // spin_count=0 forces every empty pop() straight into atomic::wait, so this + // hammers the lost-wakeup guard (snapshot wake_, re-check tail_, then wait). + // The producer pushes one item then waits to go empty again, maximising the + // number of empty->non-empty edges relative to item count. + constexpr int N = 20'000; + Channel ch(/*capacity=*/2, /*spin_count=*/0); + + std::thread producer([&] { + for (int i = 0; i < N; ++i) { + for (;;) { + try { ch.push(i); break; } + catch (const ChannelOverflowError&) { std::this_thread::yield(); } + } + } + }); + + long sum = 0; + for (int i = 0; i < N; ++i) sum += ch.pop(); + producer.join(); + + // Sum of 0..N-1 β€” detects any lost or duplicated item. + REQUIRE(sum == static_cast(N) * (N - 1) / 2); +} + +TEST_CASE("SPSC: disable() while consumer is blocked in pop() unblocks cleanly", + "[channel][stress]") { + // The data race of record: consumer blocked in pop() (spinning or parked in + // the futex) while the owner thread calls disable(). pop() must observe the + // close and throw ChannelClosedError β€” it must not hang and must not read + // past the ring. Repeated so disable() lands at many points in pop()'s loop. + for (int rep = 0; rep < kReps; ++rep) { + Channel ch(/*capacity=*/4, /*spin_count=*/8); + std::atomic threw{false}; + std::atomic finished{false}; + + std::thread consumer([&] { + try { + ch.pop(); // empty channel: will block + } catch (const ChannelClosedError&) { + threw.store(true, std::memory_order_relaxed); + } + finished.store(true, std::memory_order_relaxed); + }); + + // Give the consumer a chance to reach the wait, then close. + std::this_thread::sleep_for(50us); + ch.disable(); + + consumer.join(); + REQUIRE(finished.load()); + REQUIRE(threw.load()); + } +} + +TEST_CASE("SPSC: producer racing a disable() never throws and never hangs", + "[channel][stress]") { + // Mirror of the above from the producer side: push() racing disable() must + // either enqueue or silently drop, never throw ChannelClosedError and never + // wedge. Overflow is still a legal outcome (full accepting channel) and is + // tolerated here. + for (int rep = 0; rep < kReps; ++rep) { + Channel ch(/*capacity=*/8, /*spin_count=*/8); + std::atomic bad{false}; + + std::thread producer([&] { + for (int i = 0; i < 1000; ++i) { + try { ch.push(i); } + catch (const ChannelOverflowError&) { /* legal: full */ } + catch (...) { bad.store(true, std::memory_order_relaxed); break; } + } + }); + + std::this_thread::sleep_for(20us); + ch.disable(); // owner closes mid-stream + producer.join(); + + REQUIRE_FALSE(bad.load()); + } +} + +TEST_CASE("SPSC: push_callback fires for every push, never missed", + "[channel][stress]") { + // Regression: this callback is the *only* thing that wakes a PoolNode, and + // it used to fire only on the empty->non-empty edge, computed from a head_ + // sampled before the item was published. A concurrent pop() could drain the + // ring to empty in that window, so neither side saw the other: the item sat + // in the ring with the consumer idle, and because the trigger was an edge it + // never recovered. See set_push_callback in channel.hpp. + // + // The old version of this test asserted only `1 <= callbacks <= N`, which a + // *missed* callback satisfies β€” it named the hazard and could not detect it. + // One callback per successful push is the contract, so assert exactly that. + Channel ch(/*capacity=*/4, /*spin_count=*/4); + std::atomic callbacks{0}; + ch.set_push_callback([&] { callbacks.fetch_add(1, std::memory_order_relaxed); }); + + constexpr int N = 10'000; + std::thread producer([&] { + for (int i = 0; i < N; ++i) { + for (;;) { + try { ch.push(i); break; } + catch (const ChannelOverflowError&) { std::this_thread::yield(); } + } + } + }); + + for (int i = 0; i < N; ++i) (void)ch.pop(); + producer.join(); + + // Exactly one callback per successful push. Fewer means a wake was dropped, + // which is the bug; more would mean a spurious wake was manufactured. + REQUIRE(callbacks.load() == N); +} + +// Ordering contract of the out-of-band sentinel under contention. +// +// push_sentinel() publishes has_eof_ (release) after the producer's N ring +// pushes; a consumer that observes has_eof_ (acquire) therefore also observes +// every value pushed before it. Both pop() and try_pop_now() only surface the +// sentinel once the ring is *freshly* observed empty, so the sentinel is the +// strictly last item received β€” it never jumps ahead of a ring value pushed +// before it. These tests treat the sentinel as a hard "last message" barrier +// (the consumer stops draining the moment it sees it) and assert that all N +// values arrived, in a contiguous 0..N-1 sequence, before it. +// +// Regression guard: an earlier version of pop() checked emptiness against a +// stale tail_ snapshot from the top of its loop, so under load the sentinel +// could surface with a few real values still queued β€” breaking in_order / +// values==N here. Under TSan these also cover the has_eof_/eof_value_ +// acquire/release handshake and the spin/futex wakeup on push_sentinel(). + +TEST_CASE("SPSC: sentinel is strictly last, after every value (blocking pop)", + "[channel][stress]") { + constexpr int N = 20'000; + constexpr int SENTINEL = -1; + + for (int rep = 0; rep < kReps; ++rep) { + // Small ring + tiny spin window so the ring is frequently empty exactly + // when the sentinel is published β€” the interleaving under test. + Channel ch(/*capacity=*/4, /*spin_count=*/8); + + std::thread producer([&] { + for (int i = 0; i < N; ++i) { + for (;;) { + try { ch.push(i); break; } + catch (const ChannelOverflowError&) { std::this_thread::yield(); } + } + } + ch.push_sentinel(SENTINEL); // must-deliver, never overflows/blocks + }); + + int expected = 0; + bool in_order = true; + bool saw_sentinel = false; + // Treat the sentinel as EOF: stop draining the instant it appears. + for (;;) { + int v = ch.pop(); + if (v == SENTINEL) { saw_sentinel = true; break; } + if (v != expected) in_order = false; + ++expected; + } + producer.join(); + + REQUIRE(saw_sentinel); + REQUIRE(in_order); + REQUIRE(expected == N); // all N values received before the sentinel + REQUIRE(ch.size() == 0); + REQUIRE(ch.approx_size() == 0); + } +} + +TEST_CASE("SPSC: sentinel is strictly last, after every value (try_pop_now)", + "[channel][stress]") { + // The pool-node consume path is try_pop_now(), not pop(): it must surface + // the out-of-band sentinel only once the ring is freshly observed empty. + // The consumer spins with no sleeps, racing the producer at full tilt + // across the empty-ring boundary where take_sentinel() is reached. + constexpr int N = 20'000; + constexpr int SENTINEL = -1; + + for (int rep = 0; rep < kReps; ++rep) { + Channel ch(/*capacity=*/4, /*spin_count=*/0); + + std::thread producer([&] { + for (int i = 0; i < N; ++i) { + for (;;) { + try { ch.push(i); break; } + catch (const ChannelOverflowError&) { std::this_thread::yield(); } + } + } + ch.push_sentinel(SENTINEL); + }); + + int expected = 0; + bool in_order = true; + bool saw_sentinel = false; + int v; + for (;;) { + if (!ch.try_pop_now(v)) { std::this_thread::yield(); continue; } + if (v == SENTINEL) { saw_sentinel = true; break; } + if (v != expected) in_order = false; + ++expected; + } + producer.join(); + + REQUIRE(saw_sentinel); + REQUIRE(in_order); + REQUIRE(expected == N); + // Sentinel held no ring slot; once taken the channel is fully empty. + REQUIRE(ch.size() == 0); + REQUIRE(ch.approx_size() == 0); + } +} + +// Contended: a producer offering sentinels while the consumer takes them. +// +// The old push_sentinel wrote eof_value_ with no regard for whether the +// consumer was reading it, so a second offer racing a take was a data race on +// the storage β€” for the shared_ptr form used by non-trivial types, on the +// refcount. Under TSan the old code reports it; the handshake added alongside +// this test makes the producer's write conditional on observing the slot free, +// which is what serialises the two. +// +// Payload is a std::string so the storage is the shared_ptr path rather than +// the trivially-copyable one, and each token carries its own identity so a torn +// value shows up as a mismatch rather than as a plausible-looking result. +TEST_CASE("SPSC: offering sentinels concurrently with takes is race-free", + "[channel][stress][sentinel]") { + constexpr int kRounds = 20000; + Channel ch(4); + + std::atomic taken{0}; + std::atomic torn{false}; + std::atomic done{false}; + + std::thread consumer([&] { + std::string out; + while (!done.load(std::memory_order_acquire) || ch.approx_size() > 0) { + if (ch.try_pop_now(out)) { + if (out.rfind("eof-", 0) != 0) torn.store(true, std::memory_order_relaxed); + taken.fetch_add(1, std::memory_order_relaxed); + } + } + }); + + int accepted = 0; + for (int i = 0; i < kRounds; ++i) { + std::string tok = "eof-" + std::to_string(i); + if (ch.try_push_sentinel(tok) == Channel::SentinelResult::Taken) + ++accepted; + } + done.store(true, std::memory_order_release); + consumer.join(); + + INFO("accepted " << accepted << " taken " << taken.load()); + CHECK_FALSE(torn.load(std::memory_order_relaxed)); + // Every accepted token must be delivered: the slot is refused while full, + // so acceptance and delivery are one-to-one. + CHECK(taken.load(std::memory_order_relaxed) == accepted); + CHECK(accepted > 0); +} diff --git a/tests/test_network.cpp b/tests/test_network.cpp index 9d75f64..11d91c4 100644 --- a/tests/test_network.cpp +++ b/tests/test_network.cpp @@ -1,6 +1,10 @@ #include #include +#include #include +#include +#include +#include #include using namespace kpn; @@ -54,3 +58,93 @@ TEST_CASE("stop disables input channels β€” producer push is silently dropped", in_ch.push(99); REQUIRE(in_ch.size() == 0); } + +// Regression: Network::set_error_handler must actually deliver the handler. +// +// The handler was stored in a member and never read. A node's exception was +// discarded at the node boundary and the only surviving evidence was a Closed +// event, which reports that a node stopped but not why β€” the difference between +// a diagnosis and a guess. StaticNetwork has always wired this; Network +// accepted the handler and silently dropped it, which is worse than not +// offering the setter at all. +// +// The type changed with the fix. It was void(name, exception_ptr), which cannot +// express the keep-running decision the node side needs, so it is now +// NodeErrorHandler like StaticNetwork's. +namespace { + +static int throwing_stage(int x) { + if (x == 42) throw std::runtime_error("boom"); + return x; +} + +} // namespace + +TEST_CASE("network error handler receives the node's exception", "[network]") { + auto src = kpn::make_node(kpn::in<"v">{}, kpn::out<"w">{}, 8); + kpn::Channel out(8); + src.set_output_channel<0>(&out); + + kpn::Network net; + net.add("stage", src).build(); + + std::atomic calls{0}; + std::string seen_name; + std::string seen_what; + std::mutex mx; + + net.set_error_handler([&](std::string_view name, std::exception_ptr ep) { + std::lock_guard lk(mx); + seen_name = std::string(name); + try { if (ep) std::rethrow_exception(ep); } + catch (const std::exception& e) { seen_what = e.what(); } + calls.fetch_add(1, std::memory_order_relaxed); + return true; // handled: keep the node running + }); + + net.set_watchdog_interval(std::chrono::hours(1)); // keep the report quiet + net.start(); + src.input_channel<0>().push(42); // throws + std::this_thread::sleep_for(std::chrono::milliseconds(100)); + src.input_channel<0>().push(7); // must still be running + const int passed = out.pop(); + net.stop(); + + CHECK(calls.load(std::memory_order_relaxed) == 1); + CHECK(seen_name == "stage"); + CHECK(seen_what == "boom"); + CHECK(passed == 7); +} + +// Regression: stopping a network must not wait for the watchdog's next tick. +// +// The watchdog looped on std::this_thread::sleep_for(watchdog_interval_), and +// request_stop() cannot wake a sleeping thread β€” so stop_watchdog()'s join +// blocked until the current sleep expired. Every teardown paid up to a full +// interval, three seconds by default, and a caller who set a long one to keep +// the periodic report quiet got a stop() that looked like a hang. That is how +// this was found: the error-handler case above set an hour. +TEST_CASE("stopping a network does not wait for the watchdog interval", "[network]") { + auto node = kpn::make_node(kpn::in<"v">{}, kpn::out<"w">{}, 4); + kpn::Channel out(4); + node.set_output_channel<0>(&out); + + kpn::Network net; + net.add("inc", node).build(); + net.set_watchdog_interval(std::chrono::hours(1)); + net.start(); + + // Let the watchdog actually reach its wait. Without this the test races it: + // stop_watchdog() runs before the thread has entered the loop, the token is + // already set when it does, and it exits without ever waiting β€” which passes + // against the bug as well as the fix. + std::this_thread::sleep_for(std::chrono::milliseconds(100)); + + const auto t0 = std::chrono::steady_clock::now(); + net.stop(); + const auto ms = std::chrono::duration_cast( + std::chrono::steady_clock::now() - t0).count(); + + INFO("stop took " << ms << " ms"); + CHECK(ms < 2000); +} diff --git a/tests/test_pool_node.cpp b/tests/test_pool_node.cpp index 72d60ba..9ec8b34 100644 --- a/tests/test_pool_node.cpp +++ b/tests/test_pool_node.cpp @@ -243,7 +243,13 @@ TEST_CASE("interrupt node: trigger after stop is ignored", "[interrupt_node]") { // ── Overflow callback ───────────────────────────────────────────────────────── -TEST_CASE("pool node overflow callback fires on full output channel", "[pool_node][overflow]") { +// NOTE: output overflow is no longer reachable on the data path. A node whose +// output channel is full now PARKS β€” it keeps the value in a hidden one-slot +// buffer, releases its scheduler worker, and is re-submitted when the consumer +// frees a slot. The overflow callback survives for other producers (a direct +// Channel::push by non-node code still throws), but a pool node cannot trigger +// it, so these cases assert the stronger property instead: nothing is dropped. +TEST_CASE("pool node parks instead of overflowing a full output", "[pool_node][overflow]") { auto pool = std::make_shared(2); pool->start(); @@ -264,10 +270,13 @@ TEST_CASE("pool node overflow callback fires on full output channel", "[pool_nod node.stop(); pool->stop(); - REQUIRE(overflow_count.load() > 0); + // Parked, not overflowed: the value is still owned by the node. + REQUIRE(overflow_count.load() == 0); + // And it was never handed downstream, so nothing was lost or duplicated. + REQUIRE(full_ch.size() == 1); } -TEST_CASE("pool node overflow callback is independent per instance", "[pool_node][overflow]") { +TEST_CASE("parking is per node, not shared", "[pool_node][overflow]") { auto pool = std::make_shared(2); pool->start(); @@ -296,7 +305,10 @@ TEST_CASE("pool node overflow callback is independent per instance", "[pool_node nodeB.stop(); pool->stop(); - REQUIRE(a_overflows.load() > 0); + // Neither overflows now: A parks on its full output, B runs normally. The + // point of the case is unchanged β€” one node's backpressure must not leak + // into another's callbacks. + REQUIRE(a_overflows.load() == 0); REQUIRE(b_overflows.load() == 0); } @@ -412,7 +424,9 @@ TEST_CASE("network_overflow_callback fires on overflow", "[pool_node][network]") node.stop(); pool->stop(); - REQUIRE(net_overflows.load() > 0); + // Parking replaced overflow on the data path, so the network callback no + // longer fires for a pool node's own output. See the note above. + REQUIRE(net_overflows.load() == 0); } TEST_CASE("network_closed_callback fires on crash", "[pool_node][network]") { @@ -459,6 +473,250 @@ TEST_CASE("per-node and network overflow callbacks both fire independently", "[p node.stop(); pool->stop(); - REQUIRE(per_node.load() > 0); - REQUIRE(network.load() > 0); + // Both zero now: the node parks rather than overflowing. The case still + // guards that the two callbacks are wired independently. + REQUIRE(per_node.load() == 0); + REQUIRE(network.load() == 0); +} + +// Regression: NodeSnapshot's fields must line up with what nodes initialise. +// +// The snapshot is an aggregate that every node type fills positionally, and +// a8cfe73 appended queued/wake_pending/total_exec_ms to it in an order no call +// site used: each node supplies total_exec_ms as the element straight after +// queue_wait_ms, but the struct declared the two bools there. So the exec total +// landed in `queued`, `queued` landed in `wake_pending`, and `wake_pending` +// landed in total_exec_ms. The compiler said so (-Wnarrowing, bool to double, +// once per node instantiation) and the build carried on. +// +// It matters more than a cosmetic mix-up: these three fields exist to diagnose a +// wedge, and a wedged pipeline reported total_exec_ms as 0 or 1 and `queued` as +// "did this node ever run". Reading them would have pointed at the wrong node. +// +// Asserted against ema_exec_ms because that field is independently computed and +// was already correct: a true sum over several frames cannot be below the +// exponentially-weighted average of the same samples. +TEST_CASE("node snapshot fields line up with the values nodes supply", + "[pool_node][diagnostics]") { + auto pool = std::make_shared(1); + pool->start(); + + auto node = make_pool_node(pool, 64); + Channel out(64); + node.set_output_channel<0>(&out); + node.start(); + + for (int i = 0; i < 8; ++i) node.input_channel<0>().push(i); + std::this_thread::sleep_for(std::chrono::milliseconds(100)); + + auto snap = node.node_snapshot("n", 1.0); + node.stop(); + pool->stop(); + + INFO("frames=" << snap.frames_processed + << " ema=" << snap.ema_exec_ms + << " total=" << snap.total_exec_ms); + REQUIRE(snap.frames_processed == 8); + // The mis-ordered aggregate put wake_pending here, so this was 0.0 or 1.0. + CHECK(snap.total_exec_ms >= snap.ema_exec_ms); + // ...and the exec total here, which is non-zero, so `queued` read true for + // any node that had ever run β€” including one asleep with nothing to do. + CHECK_FALSE(snap.queued); + CHECK_FALSE(snap.wake_pending); +} + +// Regression: a value parked twice must keep its payload. +// +// push_outputs ends with +// +// else pending_ = std::move(result); +// +// and the retry path calls it as push_outputs(std::move(*pending_), …), so on +// that path `result` is the parked tuple itself. The assignment was therefore a +// self-move-assignment. std::tuple's is elementwise, and libstdc++'s +// std::vector does not guard against self-move: it swaps its data into a +// temporary and leaves the vector empty. So the first park was clean (the +// argument is a local temporary) and the second erased the payload. +// +// The value was still delivered, still in order, still counted β€” just empty. +// Downstream cannot distinguish that from a frame on which the node genuinely +// found nothing, which is why it never surfaced as an error: in +// scene-actor-extraction it reads as "no faces in this frame" and the run +// completes with a quietly wrong answer. +// +// Reaching it needs *two* outputs. With one, the only thing that resubmits a +// parked node is that output's own space callback, which by definition fires +// when there is room β€” so the retry always succeeds and never reassigns. With +// two, output A draining resubmits the node while output B is still full: the +// retry skips A (already delivered, tracked in pending_done_) and fails on B, +// and that is the reassignment that eats B's payload. +// +// Driven through raw channels rather than consumer nodes so each step is +// forced rather than raced: B is pre-filled and stays full for exactly as long +// as the test wants it to. +namespace { + +struct TwoPayloads { + static constexpr std::string_view label() { return "two_payloads"; } + std::tuple, std::vector> operator()() { + return {std::vector(4, 1), std::vector(4, 2)}; + } +}; + +} // namespace + +TEST_CASE("a twice-parked value keeps its payload", "[pool_node][backpressure]") { + auto pool = std::make_shared(2); + pool->start(); + + TwoPayloads fn; + auto node = make_pool_node(fn, pool); + + // Both capacity 1. A must be *full* for its pop to signal space at all β€” + // Channel fires the space callback only on the full->not-full edge, so a + // roomy A would never resubmit the node and the retry would never happen. + Channel> out_a(1), out_b(1); + node.set_output_channel<0>(&out_a); + node.set_output_channel<1>(&out_b); + + // B is full before the node ever runs, so the very first firing parks. + out_b.push(std::vector(4, 99)); + + node.start(); + std::this_thread::sleep_for(std::chrono::milliseconds(50)); + + // Draining A resubmits the node while B is still full: this is the retry + // that reassigned the tuple to itself. + (void)out_a.pop(); + std::this_thread::sleep_for(std::chrono::milliseconds(50)); + + // Now let B through and collect what the node had been holding for it. + (void)out_b.pop(); // the pre-fill + std::this_thread::sleep_for(std::chrono::milliseconds(50)); + std::vector parked = out_b.pop(); // the value parked across two tries + + node.stop(); + pool->stop(); + + INFO("parked payload size " << parked.size()); + CHECK(parked.size() == 4); + if (parked.size() == 4) CHECK(parked[0] == 2); +} + +// Regression: a node woken with nothing to read must not stop itself. +// +// pop_one reported an empty channel the same way it reported a closed one, by +// throwing ChannelClosedError, and fire_once treats that as "upstream is +// finished" and calls self_stop(). self_stop disables the node's own inputs +// *and* outputs, so one benign empty read does not merely skip a frame β€” it +// kills the node and, through the disabled channels, the rest of the pipeline. +// +// A node genuinely does get woken with empty inputs: a space callback fires +// when its output drains, which has nothing to do with input arrival. fire_once +// guards against it by checking readiness before popping, and that guard is +// what this test pins. pop_one now also distinguishes the two cases, so if the +// guard is ever weakened the cost is a wasted firing rather than a dead node. +// +// The sequence below reaches the guard deliberately. The output is capacity 1 +// so that draining it signals space at all β€” Channel fires the space callback +// only on the full->not-full edge β€” and by the final pop the input is long +// since consumed, so the resulting firing has nothing to read. +namespace { + +struct CountingRelay { + static constexpr std::string_view label() { return "counting_relay"; } + std::atomic* calls; + int operator()(int v) { calls->fetch_add(1, std::memory_order_relaxed); return v; } +}; + +} // namespace + +TEST_CASE("a node woken with empty inputs does not stop itself", "[pool_node]") { + std::atomic calls{0}; + std::atomic closed{0}; + + auto pool = std::make_shared(2); + pool->start(); + + CountingRelay fn{&calls}; + auto node = make_pool_node(fn, pool, 8); + Channel out(1); + node.set_output_channel<0>(&out); + node.set_closed_callback([&](auto) { closed.fetch_add(1, std::memory_order_relaxed); }); + + out.push(99); // output full before the node runs + node.start(); + + node.input_channel<0>().push(1); // fires, cannot deliver, parks + std::this_thread::sleep_for(std::chrono::milliseconds(50)); + + REQUIRE(out.pop() == 99); // space -> retry delivers the parked value + std::this_thread::sleep_for(std::chrono::milliseconds(50)); + REQUIRE(out.pop() == 1); // space again -> fires with empty inputs + std::this_thread::sleep_for(std::chrono::milliseconds(50)); + + // That firing had nothing to read. The node must still be alive. + CHECK(closed.load(std::memory_order_relaxed) == 0); + CHECK(node.running()); + + // And must still do its job when real input arrives. + node.input_channel<0>().push(2); + std::this_thread::sleep_for(std::chrono::milliseconds(50)); + CHECK(out.pop() == 2); + CHECK(calls.load(std::memory_order_relaxed) == 2); + + node.stop(); + pool->stop(); +} + +// Regression: stop() must not return while a firing is still running. +// +// stop() set the flag, disabled the inputs and returned, leaving an executing +// fire_once touching input_channels_, stats_ and pending_ while the caller went +// on to destroy them. The old comment was explicit that callers wanting the +// guarantee should call scheduler_->drain() first β€” but ~PoolNode calls stop(), +// and a destructor cannot ask its caller to have done that. +// +// A node with a private pool survived by accident: Node::stop() calls +// pool->stop(), which joins the worker. A node sharing a pool, which +// make_pool_node exists to create, had nothing joining it at all, so its own +// destructor raced the firing. +// +// Asserted through an observable side effect rather than by trying to catch the +// use-after-free: if stop() returns before the node function has finished, the +// flag it sets on the way out is still false. +namespace { + +struct SlowFiring { + static constexpr std::string_view label() { return "slow_firing"; } + std::atomic* entered; + std::atomic* finished; + void operator()(int) { + entered->store(true, std::memory_order_release); + std::this_thread::sleep_for(std::chrono::milliseconds(200)); + finished->store(true, std::memory_order_release); + } +}; + +} // namespace + +TEST_CASE("stop waits for a firing already in flight", "[pool_node]") { + std::atomic entered{false}, finished{false}; + + auto pool = std::make_shared(2); + pool->start(); + + SlowFiring fn{&entered, &finished}; + auto node = make_pool_node(fn, pool, 4); + node.start(); + node.input_channel<0>().push(1); + + // Stop only once the node is demonstrably inside its function. + while (!entered.load(std::memory_order_acquire)) + std::this_thread::sleep_for(std::chrono::milliseconds(1)); + + node.stop(); + CHECK(finished.load(std::memory_order_acquire)); + + pool->stop(); } diff --git a/tests/test_scheduler.cpp b/tests/test_scheduler.cpp index 18f3fdd..9af89b9 100644 --- a/tests/test_scheduler.cpp +++ b/tests/test_scheduler.cpp @@ -227,3 +227,116 @@ TEST_CASE("work stealing: tasks complete with more threads than initial queue ta REQUIRE(counter.load() == 4); pool.stop(); } + +// Regression: submitting to a stopped pool must be a no-op, not a segfault. +// +// stop() ends with queues_.clear(), and submit() went straight to +// queues_[target] with no check β€” so a submission arriving after stop indexed +// an empty vector. +// +// This is not a contrived teardown ordering; it happens on a normal path. A +// node's space callback fires from whichever thread drained the channel, and +// that thread belongs to the *consumer*. Stop the producer first β€” which a +// sources-first shutdown does by design β€” and the consumer keeps draining its +// backlog, firing the producer's space callback into a pool that has already +// been torn down. Before this fix the static-network shutdown case crashed +// about 12 runs in 20. +// +// Checking stopped_ without the lock would not be enough: the window between +// reading the flag and indexing the vector is exactly where clear() runs. +TEST_CASE("submitting to a stopped pool is refused, not fatal", "[scheduler]") { + ThreadPool pool(2); + pool.start(); + pool.stop(); + + std::atomic ran{0}; + for (int i = 0; i < 10; ++i) + pool.submit([&] { ran.fetch_add(1, std::memory_order_relaxed); }); + + CHECK(ran.load(std::memory_order_relaxed) == 0); + CHECK(pool.rejected() == 10); +} + +TEST_CASE("submitting while the pool stops does not crash", "[scheduler]") { + // The racing form of the case above: a producer thread submitting + // continuously while stop() runs underneath it. Nothing is asserted about + // how many tasks run β€” the point is that every submission either enqueues + // or is refused, and none touches a destroyed queue. + for (int rep = 0; rep < 20; ++rep) { + ThreadPool pool(4); + pool.start(); + + std::atomic go{false}; + std::atomic ran{0}; + std::thread submitter([&] { + while (!go.load(std::memory_order_acquire)) {} + for (int i = 0; i < 2000; ++i) + pool.submit([&] { ran.fetch_add(1, std::memory_order_relaxed); }); + }); + + go.store(true, std::memory_order_release); + std::this_thread::sleep_for(std::chrono::microseconds(200)); + pool.stop(); + submitter.join(); + + // Everything submitted was either executed or refused; nothing vanished + // into a queue that no longer existed. + CHECK(pool.rejected() + pool.snapshot("p").tasks_completed <= 2000); + } +} + +// Regression: idle workers must sleep while another worker is busy. +// +// The wait predicate was `stopped_ || total_ > 0`, and total_ counts queued +// *plus executing*. So while any one task ran, every other worker's predicate +// was true: wait() returned immediately and the worker spun through try_pop, +// try_steal and back to wait at full speed β€” try_lock-ing every peer queue on +// each pass. One slow task pinned every other core and contended the very +// mutexes the working thread needed to make progress. +// +// That is the shape of this pipeline's load exactly: a handful of nodes whose +// work is tens of milliseconds of ONNX inference. It was latent only because +// each node currently owns a private single-thread pool, where there is no +// idle peer to spin. Any use of a shared pool β€” which make_pool_node exists +// for β€” hits it immediately. +// +// Measured as CPU time rather than wall time, because the bug does not make +// anything slower to finish; it makes seven cores burn while one works. A +// sleeping task consumes no CPU, so with workers correctly asleep the whole +// pool should account for almost none. +TEST_CASE("idle workers do not spin while one task runs", "[scheduler]") { + auto cpu_ms = [] { + struct timespec ts{}; + clock_gettime(CLOCK_PROCESS_CPUTIME_ID, &ts); + return ts.tv_sec * 1000.0 + ts.tv_nsec / 1e6; + }; + + constexpr int kThreads = 8; + constexpr int kWorkMs = 300; + + ThreadPool pool(kThreads); + pool.start(); + std::this_thread::sleep_for(20ms); // let workers reach the wait + + const double before = cpu_ms(); + + // One long task plus a trivial one per remaining worker. The trivial ones + // matter: a worker that has never been woken stays blocked in wait() and + // never re-evaluates the predicate, so the spin only appears once a worker + // *finishes* something and re-enters the loop while a peer is still busy. + // Submitting only the long task does not reproduce it. + pool.submit([&] { std::this_thread::sleep_for(std::chrono::milliseconds(kWorkMs)); }); + for (int i = 0; i < kThreads - 1; ++i) pool.submit([] {}); + + pool.drain(); + const double used = cpu_ms() - before; + pool.stop(); + + // Measured on this tree: 1991 ms of CPU with the total_ predicate against + // 0.4 ms with queued_, and 19205 voluntary context switches against 10 β€” + // roughly (kThreads - 1) cores burned for the duration of one sleeping + // task. The threshold sits far from both so the case is not sensitive to + // how loaded the machine is. + INFO("cpu " << used << " ms over " << kWorkMs << " ms of sleeping work"); + CHECK(used < kWorkMs); +} diff --git a/tests/test_shared_resource.cpp b/tests/test_shared_resource.cpp index 7468f11..9e8abf6 100644 --- a/tests/test_shared_resource.cpp +++ b/tests/test_shared_resource.cpp @@ -237,3 +237,51 @@ TEST_CASE("make_shared_resource constructs with forwarded args", "[shared_resour auto g = res.acquire(); REQUIRE(*g == "hello"); } + +// Regression: a waiter must be releasable, or teardown waits on it forever. +// +// acquire() blocks on a condition variable whose predicate only becomes true +// when release() hands over ownership. There was no timeout and no stop +// condition, so a node parked there ignored teardown entirely: its worker never +// returned, the pool's join never completed, and shutdown hung waiting for a +// resource nobody was going to release β€” which is exactly the case when the +// holder is being stopped too. +// +// close() turns that into an exception the node's existing error path already +// handles, and networks now call it on registered resources before stopping any +// node, for the same reason. +TEST_CASE("closing a shared resource releases its waiters", "[shared_resource]") { + SharedResource res(42); + + auto holder = res.acquire(); // resource is now held + + std::atomic threw{false}, returned{false}; + std::thread waiter([&] { + try { + auto g = res.acquire(); // blocks: someone else holds it + (void)g; + } catch (const ResourceClosedError&) { + threw.store(true, std::memory_order_release); + } + returned.store(true, std::memory_order_release); + }); + + // Let it park, then tear down without ever releasing the holder. + std::this_thread::sleep_for(std::chrono::milliseconds(50)); + REQUIRE_FALSE(returned.load(std::memory_order_acquire)); + + res.close(); + waiter.join(); + + CHECK(threw.load(std::memory_order_acquire)); +} + +TEST_CASE("acquiring a closed resource fails immediately", "[shared_resource]") { + SharedResource res(7); + res.close(); + CHECK_THROWS_AS(res.acquire(), ResourceClosedError); + + // Reusable across runs once reopened. + res.reopen(); + CHECK_NOTHROW(res.acquire()); +} diff --git a/tests/test_static_network.cpp b/tests/test_static_network.cpp index 3dc2b26..188de84 100644 --- a/tests/test_static_network.cpp +++ b/tests/test_static_network.cpp @@ -271,3 +271,164 @@ TEST_CASE("static_network: fanout with labelled same-function consumers", "[stat REQUIRE(outB.pop() == 7); net.stop(); } + +// Regression: shutdown() must return even when a consumer stopped consuming. +// +// The drain step was an unbounded `while (anything anywhere is non-empty)` poll +// over *every* channel in the graph. Two defects in one loop: it waited for the +// whole network to be idle before stopping each successive layer rather than +// just the node it had stopped β€” the dynamic Network's version even took a node +// name and ignored it β€” and it had no deadline, so anything wedged downstream +// turned a graceful shutdown into the hang it exists to avoid. +// +// It could also fail to terminate with nothing wedged at all. current_fill came +// from a snapshot that loaded tail_ before head_; a concurrent pop between the +// two reads yields a head_ past the sampled tail_, and the unsigned difference +// wraps to ~2^64. Any poll for "is it empty yet" against that value runs +// forever. Both indices only ever increase, so loading head_ first can at worst +// under-report a push, which this loop tolerates and a wrap does not. +// +// Here the sink never takes anything, so its input cannot drain and the only +// correct outcome is to give up and say so. The bound asserted is deliberately +// loose: the point is that it terminates, not how fast. +namespace { + +struct DrainSource { + static constexpr std::string_view label() { return "drain_source"; } + int n{0}; + int operator()() { + std::this_thread::sleep_for(std::chrono::microseconds(100)); + return n++; + } +}; + +struct NeverConsumes { + static constexpr std::string_view label() { return "never_consumes"; } + std::atomic* wedged; + void operator()(int) { + // Blocks for the duration of the test: the input channel behind it + // fills and stays full. + while (!wedged->load(std::memory_order_acquire)) + std::this_thread::sleep_for(std::chrono::milliseconds(1)); + } +}; + +} // namespace + +TEST_CASE("shutdown returns when a consumer has wedged", "[static_network][shutdown]") { + std::atomic release{false}; + + DrainSource src_fn; + NeverConsumes sink_fn{&release}; + + kpn::ObjectNode, kpn::out<"v">, "drain_source", 0> s(src_fn, 4); + kpn::ObjectNode, kpn::out<>, "never_consumes", 0> k(sink_fn, 4); + + auto net = kpn::make_network(kpn::edge(s.output<"v">(), k.input<"v">())); + net.set_drain_timeout(std::chrono::milliseconds(100)); + net.start(); + + // Let the channel fill and the sink jam. + std::this_thread::sleep_for(std::chrono::milliseconds(100)); + + // Unjam the sink well after the drain timeout should have expired. Stopping + // a node joins its worker, so a sink blocked forever would hang the test in + // stop() rather than in the drain loop this case is about. + std::thread unjam([&] { + std::this_thread::sleep_for(std::chrono::milliseconds(800)); + release.store(true, std::memory_order_release); + }); + + const auto t0 = std::chrono::steady_clock::now(); + net.shutdown(); + const auto elapsed = std::chrono::steady_clock::now() - t0; + + unjam.join(); + + const auto ms = std::chrono::duration_cast(elapsed).count(); + INFO("shutdown took " << ms << " ms"); + CHECK(ms < 3000); // unbounded before; one 100 ms drain timeout after +} + +// Regression: node order must come from the topological sort, not from the +// order the edges happened to be written in. +// +// make_network computes Topo for the cycle check and then dropped it, filling +// the node vector in edge-declaration order β€” and named it user_nodes_topo_. +// halt() stops in its reverse, and shutdown() walks it forwards stopping each +// node and draining its outputs before moving to the next, which is a graceful +// drain only if the order really is sources-first. +// +// Every network in this tree declares edges in pipeline order, so the two +// coincided and nothing failed. This case declares them backwards, which is +// legal and which make_network otherwise accepts silently. +// +// Asserted through shutdown() rather than by reading the order back, because +// the order is private and the ordering is not the point β€” what it buys is. +// A sources-first shutdown lets the values already in flight reach the sink; +// stopping the sink first strands them, and the drain step then has nobody +// left to take them. +namespace { + +struct OrderSource { + static constexpr std::string_view label() { return "order_source"; } + std::atomic* made; + int operator()() { + std::this_thread::sleep_for(std::chrono::microseconds(20)); + return made->fetch_add(1, std::memory_order_relaxed); + } +}; + +// Deliberately slower than the source, so a deep backlog builds up in its input +// channel. That backlog is what a sources-first shutdown preserves and a +// sink-first one throws away, and it needs to be big enough that the difference +// cannot be mistaken for one value in flight. +struct OrderRelay { + static constexpr std::string_view label() { return "order_relay"; } + int operator()(int v) { + std::this_thread::sleep_for(std::chrono::microseconds(300)); + return v; + } +}; + +struct OrderSink { + static constexpr std::string_view label() { return "order_sink"; } + std::atomic* seen; + void operator()(int) { seen->fetch_add(1, std::memory_order_relaxed); } +}; + +} // namespace + +TEST_CASE("edges declared out of order still start and stop sources-first", + "[static_network][shutdown]") { + std::atomic seen{0}, made{0}; + + OrderSource src_fn{&made}; + OrderRelay relay_fn; + OrderSink sink_fn{&seen}; + + kpn::ObjectNode, kpn::out<"v">, "order_source", 0> s(src_fn, 8); + kpn::ObjectNode, kpn::out<"w">, "order_relay", 0> r(relay_fn, 64); + kpn::ObjectNode, kpn::out<>, "order_sink", 0> k(sink_fn, 64); + + // Sink edge first, source edge last β€” the reverse of pipeline order. + auto net = kpn::make_network( + kpn::edge(r.output<"w">(), k.input<"w">()), + kpn::edge(s.output<"v">(), r.input<"v">()) + ); + net.start(); + std::this_thread::sleep_for(std::chrono::milliseconds(300)); + + const int before = seen.load(std::memory_order_relaxed); + REQUIRE(before > 0); // the pipeline ran at all + + net.shutdown(); + + // Sources stop first and each layer drains before the next stops, so the + // backlog queued in front of the relay still reaches the sink. Stopping in + // declaration order stops the relay first and discards all of it. + const int after = seen.load(std::memory_order_relaxed); + INFO("made " << made.load() << ", delivered " << before + << " before shutdown, " << after << " after"); + CHECK(after - before >= 20); +} diff --git a/tests/test_submit_gate.cpp b/tests/test_submit_gate.cpp new file mode 100644 index 0000000..b84ac90 --- /dev/null +++ b/tests/test_submit_gate.cpp @@ -0,0 +1,173 @@ +// Regression: a node must never end up idle with a wake outstanding. +// +// 9c5ce5f established that invariant and implemented it as two independent +// atomics β€” queued_ for "a firing is in flight", wake_pending_ for "a wake +// arrived during one". Two variables cannot express it, because the release +// side has to read and write both and a wake can land in between: +// +// producer (try_submit) worker (release_and_recheck) +// ------------------------ ---------------------------- +// CAS reads queued_ == true, fails +// queued_.store(false) +// wake_pending_.exchange(false) -> false +// wake_pending_.store(true) +// +// queued_ false, wake_pending_ true, nothing running and nothing scheduled. +// Not a memory-ordering subtlety: the interleaving holds under seq_cst. +// +// LegacyGate below is that protocol verbatim, with a hook between the failed +// CAS and the wake_pending_ store so the interleaving can be forced rather than +// waited for. That makes the loss deterministic and the test non-flaky, and it +// keeps the defect on record now that the code implementing it is gone. +// +// A note on what is NOT tested here, because it would be misleading to imply +// otherwise: there is no black-box, node-level test that fails before this fix +// and passes after. Every call site of release_and_recheck() happens to follow +// it with a level re-check β€” on_input_ready(), or outputs_have_space() on the +// parked path β€” which rediscovers the state a lost wake would have signalled. +// That masking is a property of the call sites, not of the mechanism, and the +// point of the fix is that a future early return that forgets the re-check no +// longer reintroduces a hang. The value is structural, so the tests are +// structural: the state machine is pinned by contract, and the defect it +// replaces is pinned by demonstration. +#include +#include + +#include +#include +#include + +using namespace kpn; + +namespace { + +// The pre-fix protocol, with a seam at the point where the race lives. +class LegacyGate { +public: + std::function before_recording_wake; + + bool claim() noexcept { + bool expected = false; + if (queued_.compare_exchange_strong(expected, true, std::memory_order_acq_rel)) + return true; + if (before_recording_wake) before_recording_wake(); + wake_pending_.store(true, std::memory_order_release); + return false; + } + bool release() noexcept { + queued_.store(false, std::memory_order_release); + if (wake_pending_.exchange(false, std::memory_order_acq_rel)) { + bool expected = false; + if (queued_.compare_exchange_strong(expected, true, std::memory_order_acq_rel)) + return true; + } + return false; + } + bool queued() const noexcept { return queued_.load(std::memory_order_relaxed); } + bool wake_pending() const noexcept { return wake_pending_.load(std::memory_order_relaxed); } + +private: + std::atomic queued_{false}; + std::atomic wake_pending_{false}; +}; + +} // namespace + +TEST_CASE("the two-atomic gate loses a wake, deterministically", "[submit_gate]") { + LegacyGate gate; + bool resubmitted = true; + + REQUIRE(gate.claim()); // a firing is now in flight + + // Force the interleaving: the firing completes in the window between the + // second wake's failed CAS and its record of that wake. + gate.before_recording_wake = [&] { resubmitted = gate.release(); }; + + const bool submitted = gate.claim(); + + // The wake was neither submitted by the producer nor honoured by the + // release. Nothing is scheduled, and nothing else will re-trigger it. + CHECK_FALSE(submitted); + CHECK_FALSE(resubmitted); + CHECK_FALSE(gate.queued()); + CHECK(gate.wake_pending()); // recorded, and never to be consumed +} + +TEST_CASE("submit gate: a wake during a firing is honoured", "[submit_gate]") { + SubmitGate gate; + + REQUIRE(gate.claim()); // idle -> queued, caller submits + REQUIRE(gate.queued()); + REQUIRE_FALSE(gate.wake_pending()); + + REQUIRE_FALSE(gate.claim()); // second wake is recorded, not submitted + REQUIRE(gate.wake_pending()); + + REQUIRE(gate.release()); // and honoured when the firing ends + // The gate stays claimed across the handover, so the node is never + // momentarily idle while a submission for it is in flight. This is the + // state the legacy gate could not represent. + REQUIRE(gate.queued()); + REQUIRE_FALSE(gate.wake_pending()); + + REQUIRE_FALSE(gate.release()); // no further wake: now idle + REQUIRE_FALSE(gate.queued()); +} + +TEST_CASE("submit gate: repeated wakes collapse to one resubmission", "[submit_gate]") { + // Collapsing is deliberate. A firing consumes one item and its caller then + // re-checks the input level, so the gate only has to guarantee that at + // least one more firing follows a wake, not one per wake. + SubmitGate gate; + REQUIRE(gate.claim()); + for (int i = 0; i < 10; ++i) REQUIRE_FALSE(gate.claim()); + REQUIRE(gate.release()); + REQUIRE_FALSE(gate.release()); +} + +TEST_CASE("submit gate: force_idle drops a recorded wake", "[submit_gate]") { + // Stop paths use this deliberately β€” honouring a wake there would resubmit + // a node that has already been told to stop. + SubmitGate gate; + REQUIRE(gate.claim()); + REQUIRE_FALSE(gate.claim()); + REQUIRE(gate.wake_pending()); + + gate.force_idle(); + REQUIRE_FALSE(gate.queued()); + REQUIRE_FALSE(gate.wake_pending()); + REQUIRE(gate.claim()); // and the gate is reusable afterwards +} + +TEST_CASE("submit gate: concurrent claim and release stay consistent", "[submit_gate]") { + // Not a lost-wake test β€” see the header note. This is a TSan target and a + // check that the CAS loops always terminate and always leave the gate in a + // reachable state: exactly one party may hold the claim at a time, so the + // count of claims granted must equal the count of releases that ended idle. + SubmitGate gate; + std::atomic granted{0}, ended_idle{0}; + std::atomic stop{false}; + + std::thread waker([&] { + while (!stop.load(std::memory_order_relaxed)) + if (gate.claim()) granted.fetch_add(1, std::memory_order_relaxed); + }); + std::thread worker([&] { + while (!stop.load(std::memory_order_relaxed)) + if (gate.queued() && !gate.release()) + ended_idle.fetch_add(1, std::memory_order_relaxed); + }); + + std::this_thread::sleep_for(std::chrono::milliseconds(200)); + stop.store(true, std::memory_order_relaxed); + waker.join(); + worker.join(); + + // Drain whatever claim is outstanding so the two counts can be compared. + while (gate.queued()) + if (!gate.release()) ended_idle.fetch_add(1, std::memory_order_relaxed); + + INFO("granted " << granted.load() << " ended idle " << ended_idle.load()); + REQUIRE(granted.load() > 0); + CHECK(granted.load() == ended_idle.load()); +}