From 054c4c8b8faa8dcd594cbc448b52f38921e5325a Mon Sep 17 00:00:00 2001 From: Duncan Tourolle Date: Thu, 30 Jul 2026 18:16:58 +0200 Subject: [PATCH] build: requirement traceability extractor, gate, and CI workflow Ports JellyTau's traceability tooling, rewritten in stdlib Python because this repo is C++/Python and adding a bun/node toolchain to check source comments would be a worse trade than writing the scanner. scripts/traceability/extract_traces.py scans .cpp/.hpp/.py under src, tests, scripts, experiments and eval for the house tag format /// TRACES: AR-012, AR-013 | SR-002 and reports EXCEPTION tags separately. An exception is a recorded decision to depart from an invariant, so folding it into coverage would invert its meaning; it is listed with its reason, and a missing reason is flagged. Two rules carried over from JellyTau's gate repair: * Denominators are parsed out of docs/requirements.md at run time. A requirement is defined only by a row in a table whose header is `| ID | Requirement | ... |`, so references in the Traces to column, in prose, and in the verification-plan table do not inflate the count. Adding a register row lowers coverage until it is traced - the property that dies the moment a denominator is frozen. * Coverage above 100% is a hard failure. It cannot happen through the intersection, which is the point: if it ever does, the arithmetic is broken and the run must not be reported as a pass. One rule specific to this repo: CI is an Intel N100 with no discrete GPU. The extractor reads each requirement's verification tier from requirements.md and reports T4/GPU-only requirements as tagged but unexecuted, never as covered. Counting a test that can never run is the same failure mode as the 158% bug. MIN_COVERAGE starts at 0 because almost nothing is tagged yet - tags are added as the pipeline is built. That is not a gate that cannot fail: orphan tags, a >100% ratio, a register that parses to nothing, and an empty source scan are all hard failures from day one. The threshold lives in traceability-gate.sh alone, never duplicated into the workflow YAML. 53 tests over fixture strings, so their meaning does not drift as requirements are added. --- .gitea/workflows/traceability-check.yml | 125 ++ docs/traceability.md | 136 ++ scripts/traceability/extract_traces.py | 1226 +++++++++++++++++++ scripts/traceability/test_extract_traces.py | 706 +++++++++++ scripts/traceability/traceability-gate.sh | 64 + 5 files changed, 2257 insertions(+) create mode 100644 .gitea/workflows/traceability-check.yml create mode 100644 docs/traceability.md create mode 100755 scripts/traceability/extract_traces.py create mode 100755 scripts/traceability/test_extract_traces.py create mode 100755 scripts/traceability/traceability-gate.sh diff --git a/.gitea/workflows/traceability-check.yml b/.gitea/workflows/traceability-check.yml new file mode 100644 index 0000000..afbad35 --- /dev/null +++ b/.gitea/workflows/traceability-check.yml @@ -0,0 +1,125 @@ +name: Traceability Validation + +# Mirrors JellyTau's .gitea/workflows/traceability-check.yml, adapted for a +# C++/Python repo: the extractor is Python and needs nothing but python3, so +# there is no toolchain install step and no jq. +# +# NOTE: the runner here is an Intel N100 with no discrete GPU. This job is only +# ever static analysis of source comments plus markdown parsing, so it is cheap; +# the requirements it reports as "tagged but unexecuted" are the ones that need +# a GPU host, and they are deliberately never counted as covered. + +on: + push: + branches: + - main + - master + - develop + pull_request: + branches: + - main + - master + - develop + +jobs: + validate-traces: + runs-on: linux/amd64 + name: Check requirement traces + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Check Python is available + run: | + set -e + command -v python3 >/dev/null 2>&1 || { + echo "python3 is missing from the runner image." + echo "The traceability tooling is stdlib-only Python 3.9+;" + echo "no other dependency is needed." + exit 1 + } + python3 --version + + # The gate's own arithmetic is the thing being trusted, so its tests run + # before it does. JellyTau's gate was believed for months while it was + # dividing by frozen literals; untested gate logic is how that happens. + - name: Test the extractor + run: python3 scripts/traceability/test_extract_traces.py + + # Threshold policy lives in traceability-gate.sh, not here, so local runs + # and CI runs cannot disagree about what "passing" means. Denominators + # come from docs/requirements.md at run time and are never hardcoded -- + # in this file or anywhere else. + - name: Traceability gate + run: sh scripts/traceability/traceability-gate.sh + + - name: Check modified files for traces + if: github.event_name == 'pull_request' + run: | + set -e + echo "Checking modified sources for TRACES tags..." + + CHANGED=$(git diff --name-only "origin/${{ github.base_ref }}...HEAD" \ + | grep -E '\.(cpp|cc|cxx|hpp|hxx|h|cu|cuh|py)$' || true) + + if [ -z "$CHANGED" ]; then + echo "No C++/Python files changed." + exit 0 + fi + + echo "Changed files:" + echo "$CHANGED" | sed 's/^/ /' + echo "" + + # Advisory by design: not every file implements a requirement, and a + # tag on every function is noise that rots faster than it helps + # (CLAUDE.md: tag the unit that decides). This step exists to prompt, + # not to block. The blocking checks are in the gate step above. + # + # Piped into the loop rather than a here-string, and `case` rather + # than `[[ == ]]`, so this works under dash as well as bash. The loop + # body runs in a subshell, so misses are recorded in a file. + MISSING=$(mktemp) + echo "$CHANGED" | while IFS= read -r file; do + case "$file" in + */test_*.py|*_test.py|tests/*|*/tests/*) continue ;; + esac + [ -f "$file" ] || continue + if ! grep -q 'TRACES:' "$file"; then + echo " no TRACES tag: $file" + echo "$file" >> "$MISSING" + fi + done + + COUNT=$(wc -l < "$MISSING" | tr -d ' ') + rm -f "$MISSING" + + if [ "$COUNT" -gt 0 ]; then + echo "" + echo "$COUNT changed file(s) carry no requirement tag." + echo "Format: // TRACES: AR-012, AR-013 | SR-002" + echo " (pipe separates requirement types, comma separates IDs)" + echo "A deliberate invariant exception is tagged separately:" + echo " // EXCEPTION: AR-024 " + echo "See CLAUDE.md and SPEC.md section 6." + fi + + - name: Report summary + if: always() + run: | + echo "Traceability matrix: docs/traceability.md" + echo "" + head -40 docs/traceability.md || true + + - name: Save reports + if: always() + uses: actions/upload-artifact@v3 + with: + name: traceability-reports + path: | + traces-report.json + docs/traceability.md + retention-days: 30 diff --git a/docs/traceability.md b/docs/traceability.md new file mode 100644 index 0000000..c1a328c --- /dev/null +++ b/docs/traceability.md @@ -0,0 +1,136 @@ +# Requirements traceability matrix + + + + +**Generated:** 2026-07-30T16:15:17+00:00 + +Denominators are read from [`requirements.md`](requirements.md) at run time, never hardcoded. Coverage counts a requirement only when it is tagged in source **and** has a verification tier this CI host can execute — CI is an Intel N100 with no discrete GPU. + +## Summary + +| Metric | Value | +|---|---| +| Source files scanned | 86 | +| TRACES tags found | 0 | +| EXCEPTION tags found | 0 | +| Requirements defined | 59 | +| Requirements covered | 0 | +| **Coverage** | **0.0%** (0/59) | +| Coverage of CI-executable scope | 0.0% (0/50) | +| Tagged but unexecuted in CI (T4/GPU) | 0 | +| Orphan tags | 0 | + +### By type + +| Type | Covered | Tagged but unexecuted | Defined | +|---|---|---|---| +| AR | 0 | 0 | 27 | +| DP | 0 | 0 | 6 | +| IR | 0 | 0 | 8 | +| GR | 0 | 0 | 9 | +| VR | 0 | 0 | 9 | + + +## Not executable in CI + +CI runs on an Intel N100 with no discrete GPU. These requirements have no verification tier that can run here, so a tag on them is evidence of *intent*, not of verification. They are never counted as covered. + +| ID | Tiers | Tagged in source | Requirement | +|---|---|---|---| +| AR-027 | T4 | no | Throughput acceptable for **arbitrary** gallery size | +| VR-001 | out-of-ci | no | HDF5 post-inference dump at the embedded-frame boundary | +| VR-002 | out-of-ci | no | Replay drives the **real** KPN nodes, not a reimplementation | +| VR-003 | out-of-ci | no | Scoring: micro-F1 against X-Ray, precision/recall logged at every eva… | +| VR-004 | out-of-ci | no | Reproducible validation corpus with ground truth | +| VR-005 | out-of-ci | no | Minimum face size study — TPI/FPI vs probe size, gallery held at nati… | +| VR-006 | out-of-ci | no | Re-tune `scene_threshold` once native-rate decode lands | +| VR-007 | out-of-ci | no | Expansion band, clustering threshold, and deferred-pass ablation | +| VR-008 | out-of-ci | no | Gallery scaling benchmark — throughput vs gallery size | + +## Orphan tags + +A tag naming an ID `requirements.md` does not define. This is what renumbering produces, and what a typo produces. + +_None._ + +## Requirements tracing up to nothing + +A register row whose `Traces to` cell names no parent. Work serving no stated goal is how scope creeps in, and it is invisible unless something looks. + +_None._ + +## Recorded exceptions + +Deliberate, documented departures from an invariant (`EXCEPTION: AR-nnn `). Reported separately and never counted as coverage — an exception is a decision to be reviewed, not evidence a requirement is met. + +_None._ + +## Register + +| ID | Status | Tier | Traces to | Trace state | Tagged in | Requirement | +|---|---|---|---|---|---|---| +| AR-001 | Done | T3 | SR-002 | untagged | - | Detect faces in sampled frames; emit bbox, confidence, 5-point landma… | +| AR-002 | Planned | T2 | SR-002 | untagged | - | Minimum face size 66×66 px, expressed in **original** resolution (dec… | +| AR-003 | Planned | T1, T2, T4 | SR-002 | untagged | - | No fixed per-frame face cap — crowd scenes must not lose background c… | +| AR-004 | Planned | T1, T4 | SR-002 | untagged | - | Backpressure: unbounded faces/frame absorbed by slowing, never by dro… | +| AR-005 | Done | T1, T3 | SR-002 | untagged | - | Align to 112×112 via ArcFace 5-point similarity transform | +| AR-006 | Done | T3 | SR-002 | untagged | - | 512-d L2-normalised embeddings, batched | +| AR-007 | In Progress | T2 | SR-002 | untagged | - | Associate detections by IoU + embedding, with **frame-dependent** wei… | +| AR-008 | Planned | T2 | SR-002 | untagged | - | One track pool keyed on `last_seen`; no separate revival path | +| AR-009 | Done | T2 | SR-002 | untagged | - | Camera-cut detection (histogram) as an association hint | +| AR-010 | In Progress | T2 | SR-002 | untagged | - | Scene-boundary detection (TransNetV2) as an association hint | +| AR-011 | Planned | T1, T2 | SR-002 | untagged | - | **Every model is fed the input it was trained for** — cost reduced by… | +| AR-012 | Planned | T2 | **SR-002** | untagged | - | Presence follows **track extent**, not per-frame recognition | +| AR-013 | Planned | T2 | SR-002 | untagged | - | `last_seen` optional state machine; window ends at last sighting, nev… | +| AR-014 | Planned | T2 | SR-002 | untagged | - | Belief swap A→B terminates the track and starts a new one | +| AR-015 | Planned | T2 | SR-002 | untagged | - | Two live tracks owned by one actor ⇒ treat as a detected cut, re-asso… | +| AR-016 | Planned | T2 | SR-002 | untagged | - | All tracks closed at EOF — a film ends with faces on screen | +| AR-017 | Planned | T1, T2 | SR-002 | untagged | - | Every presence claim carries its belief and identification route | +| AR-018 | Planned | T1, T2 | SR-005 | untagged | - | Per-subject embedding store with banded admission (novel enough, safe… | +| AR-019 | In Progress | T2 | SR-005 | untagged | - | Per-film gallery annex from owned tracks; acquires the non-frontal vi… | +| AR-020 | Planned | T2 | SR-005 | untagged | - | Deferred re-identification of unknown tracks against the final expand… | +| AR-021 | Planned | T2 | SR-005 | untagged | - | Cluster unknown tracks into one entity per person, under temporal can… | +| AR-022 | Planned | T1, T2 | §4 | untagged | - | Capture still-unidentified tracks: embeddings, metadata, **context cr… | +| AR-023 | Done | T1 | SR-002 | untagged | - | Fit sigmoid calibration from intra/inter similarity distributions | +| AR-024 | Planned | T1, static | SR-002 | untagged | - | **Always the calibrated probability, never a raw cosine** — exception… | +| AR-025 | Planned | T1 | SR-002 | untagged | - | Per-track Bayesian accumulation in log-odds, with correlated-observat… | +| AR-026 | In Progress | T1, T4 | SR-001 | untagged | - | All similarity computed as GEMM, including annex and deferred pass | +| AR-027 | Planned | T4 | SR-001 | untagged | - | Throughput acceptable for **arbitrary** gallery size | +| DP-001 | Done | T1, manual | PR-004 | untagged | - | One analysis core; modes are front-ends and must not fork pipeline lo… | +| DP-002 | Done | T1, manual | PR-004 | untagged | - | Batch CLI over one title | +| DP-003 | Planned | T1, manual | PR-004 | untagged | - | On-demand resident service with bounded, observable queue | +| DP-004 | Planned | T1, manual | PR-004 | untagged | - | Opportunistic/idle mode: external trigger, hard stop, implicit re-que… | +| DP-005 | Planned | T1, manual | PR-004 | untagged | - | Native installer, no Docker; Fedora + Arch | +| DP-006 | Planned | T1, manual | PR-003 | untagged | - | Background incremental gallery refresh on a timer | +| IR-001 | Done | T1 | SR-003 | untagged | - | Emit the JRay truth format as sibling `.jray.json` | +| IR-002 | Planned | T1 | SR-003 | untagged | - | Windows carry belief + route; `extraction.*` carries `extinction_sec`… | +| IR-003 | Planned | T1 | SR-003 | untagged | - | Output written **after** the deferred pass, not at EOF | +| IR-004 | Planned | T1 | SR-003 | untagged | - | Compute the audio signature exactly per server spec §3 | +| IR-005 | Planned | T1 | SR-003 | untagged | - | Golden-vector fixture shared with the plugin repo to prove bit-exactn… | +| IR-006 | Done | unset | SR-001 | untagged | - | Jellyfin round-trip: pull pending queue, push complete results only | +| IR-007 | Planned | unset | SR-003 | untagged | - | Media < 120 s: emit no signature, apply no sync offset — identical ru… | +| IR-008 | Planned | unset | SR-003 | untagged | - | Emit and honour the signature's own `v1:` version prefix | +| GR-001 | Done | T1, T3 | SR-001, SR-005 | untagged | - | Build gallery from Jellyfin library cast, TMDB profile fallback | +| GR-002 | Done | T1, T3 | PR-003 | untagged | - | Incremental `--merge` refresh without re-embedding known actors | +| GR-003 | Planned | T1, T3 | SR-001 | untagged | - | Report coverage: zero-image actors, under-referenced actors, dedup, c… | +| GR-004 | Planned | T1, T3 | SR-001 | untagged | - | Stamp embedder identity into the gallery; **hard startup error** on m… | +| GR-005 | Done | T1, T3 | **SR-005** | untagged | - | Gallery data never leaves the instance | +| GR-006 | Planned | T1 | SR-005 | untagged | - | Provenance tiers: baked / harvested / confirmed, distinguishable per … | +| GR-007 | Planned | T1 | SR-005 | untagged | - | Persist harvested embeddings **flagged and reviewable**, never silent… | +| GR-008 | Planned | T1 | SR-005 | untagged | - | Flag distributional outliers among an actor's references (poisoning g… | +| GR-009 | TBD | unset | §4 | untagged | - | Human-confirmed associations persist and improve future extractions | +| VR-001 | Done | out-of-ci | PR-002 | untagged | - | HDF5 post-inference dump at the embedded-frame boundary | +| VR-002 | Done | out-of-ci | PR-002 | untagged | - | Replay drives the **real** KPN nodes, not a reimplementation | +| VR-003 | Done | out-of-ci | PR-002 | untagged | - | Scoring: micro-F1 against X-Ray, precision/recall logged at every eva… | +| VR-004 | Done | out-of-ci | PR-002 | untagged | - | Reproducible validation corpus with ground truth | +| VR-005 | Planned | out-of-ci | PR-002 | untagged | - | Minimum face size study — TPI/FPI vs probe size, gallery held at nati… | +| VR-006 | Planned | out-of-ci | PR-002 | untagged | - | Re-tune `scene_threshold` once native-rate decode lands | +| VR-007 | Planned | out-of-ci | PR-002 | untagged | - | Expansion band, clustering threshold, and deferred-pass ablation | +| VR-008 | Planned | out-of-ci | PR-002 | untagged | - | Gallery scaling benchmark — throughput vs gallery size | +| VR-009 | Planned | T1, out-of-ci | PR-002 | untagged | - | Verify accumulated posteriors are calibrated against held-out tracks | + +## Detailed mapping + +_No TRACES tags found yet. Tags are added as code is written; an empty matrix on a new tree is the correct reading, not a failure._ + diff --git a/scripts/traceability/extract_traces.py b/scripts/traceability/extract_traces.py new file mode 100755 index 0000000..2a9de8b --- /dev/null +++ b/scripts/traceability/extract_traces.py @@ -0,0 +1,1226 @@ +#!/usr/bin/env python3 +"""Extract requirement traces from C++/Python sources and report coverage. + +Ported from JellyTau's ``scripts/extract-traces.ts``. That one scans +TypeScript/Svelte/Rust; this repo is C++ and Python, so the scanner is Python +with no third-party dependencies — the CI host must not need a node/bun +toolchain to check traceability. + +Tag format (see ../../../CLAUDE.md and SPEC.md section 6). A pipe separates +requirement *types*, a comma separates IDs within a type:: + + /// TRACES: AR-nnn, AR-mmm | SR-nnn + struct TrackRegistry { ... }; + +Deliberate invariant exceptions carry their own tag and are reported +separately — never silently folded into coverage:: + + // EXCEPTION: AR-nnn distributional check, not a match decision + +Usage:: + + python3 scripts/traceability/extract_traces.py --format coverage + python3 scripts/traceability/extract_traces.py --format json > traces-report.json + python3 scripts/traceability/extract_traces.py --format markdown --markdown-out docs/traceability.md + +The CI gate is ``scripts/traceability/traceability-gate.sh``, which wraps this. + +Two rules inherited from JellyTau's gate repair, both learned the hard way +(JellyTau/docs/specs/traceability-gate-repair.md): + +1. Coverage denominators are read out of ``docs/requirements.md`` at run time. + Never hardcode them. JellyTau's gate divided by frozen literals while the + register grew to 211 requirements; it reported 158% coverage, so its 50% + threshold could never trip. A gate that cannot fail is worse than no gate, + because it is trusted. +2. Coverage above 100% is a hard failure, not a pass. It means the computation + is broken, and it is the signal that would have caught (1) immediately. + +One rule specific to this repo: CI runs on an Intel N100 with no discrete GPU. +Requirements whose only verification tier is T4/GPU (or "out of CI") cannot +execute here. They are reported as *tagged but unexecuted* and are excluded +from the covered numerator — counting a test that never runs as coverage is +the same failure mode as the 158% bug. +""" + +from __future__ import annotations + +import argparse +import json +import re +import sys +from dataclasses import dataclass, field +from datetime import datetime, timezone +from pathlib import Path +from typing import Dict, Iterable, List, Optional, Sequence, Set, Tuple + +# Derived from this file's location so a CI checkout at any path works. +SCRIPT_DIR = Path(__file__).resolve().parent +REPO_ROOT = SCRIPT_DIR.parent.parent + +# -------------------------------------------------------------------------- +# Taxonomy +# -------------------------------------------------------------------------- + +#: Types defined in docs/requirements.md. These, and only these, participate +#: in the coverage fraction. +LOCAL_TYPES: Tuple[str, ...] = ("AR", "DP", "IR", "GR", "VR") + +#: Test identifiers. A separate taxonomy: a test ID is evidence *for* a +#: requirement, not a requirement. Excluded from the fraction. +TEST_TYPES: Tuple[str, ...] = ("UT", "IT") + +#: Defined in the system-level SPEC.md, which lives in the umbrella project and +#: is not part of this repo's checkout. Recognised in tags, never counted here, +#: and orphan-checked only when --system-spec points at that file. +EXTERNAL_TYPES: Tuple[str, ...] = ("PR", "SR") + +KNOWN_TYPES: Tuple[str, ...] = LOCAL_TYPES + TEST_TYPES + EXTERNAL_TYPES + +#: Tiers a GPU-less CI host can actually run. See the "Verification strategy" +#: section of docs/requirements.md. +CI_EXECUTABLE_TIERS: Set[str] = {"T1", "T2", "T3", "static"} + +# -------------------------------------------------------------------------- +# Source scanning +# -------------------------------------------------------------------------- + +SOURCE_SUFFIXES = { + ".cpp", ".cc", ".cxx", ".hpp", ".hxx", ".h", ".cu", ".cuh", # C++ + ".py", # Python +} + +SCAN_ROOTS: Tuple[str, ...] = ("src", "tests", "scripts", "experiments", "eval") + +EXCLUDED_DIR_NAMES = { + ".git", "__pycache__", "external", "build", "site", "node_modules", + "trt_cache", "ort_cache", ".venv", "venv", ".mypy_cache", ".pytest_cache", +} + +TRACES_RE = re.compile(r"TRACES:[ \t]*([^\n]*)") +EXCEPTION_RE = re.compile(r"EXCEPTION:[ \t]*([A-Z]{2}-\d{3})[ \t]*([^\n]*)") +REQ_ID_RE = re.compile(r"^([A-Z]{2})-(\d{3})$") +LEADING_ID_RE = re.compile(r"^([A-Z]{2}-\d{3})\b(.*)$") +#: Something that was *trying* to be a requirement ID. Used to keep the +#: malformed-tag diagnostic quiet when the word "TRACES" merely appears in +#: prose or in this tool's own source, while still catching `AR-12`. +ID_ATTEMPT_RE = re.compile(r"[A-Za-z]{2}-\d") + +# Comment terminators that can trail a tag on the same line. +COMMENT_TERMINATORS = ("*/", "-->", '"""', "'''") + +DECL_PATTERNS = [ + re.compile(r"^\s*(?:async\s+)?def\s+\w+"), + re.compile(r"^\s*class\s+\w+"), + re.compile(r"^\s*(?:template\s*<[^;]*>\s*)?" + r"(?:struct|class|enum(?:\s+class)?|union|namespace)\s+\w+"), + re.compile(r"^\s*(?:static|inline|constexpr|virtual|explicit|friend)\b"), + re.compile(r"^\s*[A-Za-z_][\w:<>,\s\*&]*\s+[A-Za-z_~][\w:]*\s*\([^)]*\)"), +] + + +@dataclass +class TraceEntry: + file: str + line: int + context: str + requirements: List[str] + + def to_dict(self) -> dict: + return { + "file": self.file, + "line": self.line, + "context": self.context, + "requirements": list(self.requirements), + } + + +@dataclass +class ExceptionEntry: + file: str + line: int + requirement: str + reason: str + context: str + + def to_dict(self) -> dict: + return { + "file": self.file, + "line": self.line, + "requirement": self.requirement, + "reason": self.reason, + "context": self.context, + } + + +@dataclass +class Diagnostics: + """Things wrong with the tags themselves, kept visible rather than dropped.""" + + malformed_tags: List[dict] = field(default_factory=list) + mixed_type_groups: List[dict] = field(default_factory=list) + unknown_id_types: List[dict] = field(default_factory=list) + exceptions_without_reason: List[dict] = field(default_factory=list) + + def to_dict(self) -> dict: + return { + "malformedTags": self.malformed_tags, + "mixedTypeGroups": self.mixed_type_groups, + "unknownIdTypes": self.unknown_id_types, + "exceptionsWithoutReason": self.exceptions_without_reason, + } + + @property + def empty(self) -> bool: + return not (self.malformed_tags or self.mixed_type_groups + or self.unknown_id_types or self.exceptions_without_reason) + + +def parse_traces_tag(value: str) -> Tuple[List[List[str]], List[str]]: + """Parse the text after ``TRACES:`` into per-type groups plus junk. + + Returns ``(groups, junk)``. ``groups`` is one list of IDs per pipe-separated + segment, preserving the type grouping the format promises. ``junk`` holds + anything that was not a bare ID, so a tag like ``AR-001 - see also AR-999`` + yields ``AR-001`` and reports the rest rather than quietly harvesting an ID + out of prose. + """ + text = value + for term in COMMENT_TERMINATORS: + idx = text.find(term) + if idx != -1: + text = text[:idx] + text = text.strip() + if not text: + return [], [] + + groups: List[List[str]] = [] + junk: List[str] = [] + for raw_group in text.split("|"): + ids: List[str] = [] + for raw_item in raw_group.split(","): + item = raw_item.strip() + if not item: + continue + if REQ_ID_RE.match(item): + ids.append(item) + continue + lead = LEADING_ID_RE.match(item) + if lead: + ids.append(lead.group(1)) + remainder = lead.group(2).strip() + if remainder: + junk.append(remainder) + else: + junk.append(item) + if ids: + groups.append(ids) + return groups, junk + + +def find_context(lines: Sequence[str], index: int, window: int = 12) -> str: + """Best-effort name of the declaration a tag belongs to. + + Searches both directions, because the two languages put the tag on opposite + sides of the thing it describes: a C++ ``/// TRACES:`` sits *above* the + declaration, while a Python tag usually sits *inside* the docstring, below + the ``def``. Whichever declaration is nearer wins. + """ + def matches(line: str) -> bool: + return any(p.match(line) for p in DECL_PATTERNS) + + down: Optional[Tuple[int, str]] = None + for offset in range(1, window + 1): + i = index + offset + if i >= len(lines): + break + if matches(lines[i]): + down = (offset, lines[i]) + break + + up: Optional[Tuple[int, str]] = None + for offset in range(1, window + 1): + i = index - offset + if i < 0: + break + if matches(lines[i]): + up = (offset, lines[i]) + break + + best = None + if down and up: + best = down if down[0] <= up[0] else up + else: + best = down or up + if best is None: + return "Unknown" + return best[1].strip().rstrip("{").strip()[:120] or "Unknown" + + +def iter_source_files(root: Path, + scan_roots: Iterable[str] = SCAN_ROOTS) -> List[Path]: + """Every C++/Python file under the configured scan roots.""" + found: List[Path] = [] + for rel in scan_roots: + base = root / rel + if not base.is_dir(): + continue + for path in sorted(base.rglob("*")): + if not path.is_file(): + continue + if path.suffix not in SOURCE_SUFFIXES: + continue + if any(part in EXCLUDED_DIR_NAMES for part in path.parts): + continue + found.append(path) + return found + + +@dataclass +class ScanResult: + files: List[Path] + traces: List[TraceEntry] + exceptions: List[ExceptionEntry] + diagnostics: Diagnostics + + +def scan_files(files: Sequence[Path], root: Path) -> ScanResult: + traces: List[TraceEntry] = [] + exceptions: List[ExceptionEntry] = [] + diags = Diagnostics() + + for path in files: + try: + content = path.read_text(encoding="utf-8", errors="replace") + except OSError: + continue + lines = content.split("\n") + rel = str(path.relative_to(root)) if path.is_relative_to(root) else str(path) + + for index, line in enumerate(lines): + exc = EXCEPTION_RE.search(line) + if exc: + reason = exc.group(2) + for term in COMMENT_TERMINATORS: + cut = reason.find(term) + if cut != -1: + reason = reason[:cut] + reason = reason.strip() + entry = ExceptionEntry( + file=rel, line=index + 1, requirement=exc.group(1), + reason=reason, context=find_context(lines, index), + ) + exceptions.append(entry) + if not reason: + # CLAUDE.md: an exception is only agreed if the reason is + # recorded. An unexplained one is an undocumented defect. + diags.exceptions_without_reason.append( + {"file": rel, "line": index + 1, + "requirement": exc.group(1)}) + continue + + match = TRACES_RE.search(line) + if not match: + continue + groups, junk = parse_traces_tag(match.group(1)) + ids = [i for g in groups for i in g] + if not ids: + # No IDs at all. Only worth reporting if something in the text + # was trying to be one — otherwise every sentence containing + # the word would be flagged, and a diagnostic nobody can act on + # is how real diagnostics get ignored. + if junk and ID_ATTEMPT_RE.search(match.group(1)): + diags.malformed_tags.append( + {"file": rel, "line": index + 1, "text": match.group(1).strip()}) + continue + if junk: + diags.malformed_tags.append( + {"file": rel, "line": index + 1, "ignored": junk}) + for group in groups: + types = {i.split("-")[0] for i in group} + if len(types) > 1: + # The pipe is what separates types; a mixed group means the + # tag does not say what it looks like it says. + diags.mixed_type_groups.append( + {"file": rel, "line": index + 1, "group": list(group)}) + for req in ids: + if req.split("-")[0] not in KNOWN_TYPES: + diags.unknown_id_types.append( + {"file": rel, "line": index + 1, "id": req}) + + # Deduplicate within one tag while preserving order. + seen: Set[str] = set() + unique = [i for i in ids if not (i in seen or seen.add(i))] + traces.append(TraceEntry(file=rel, line=index + 1, + context=find_context(lines, index), + requirements=unique)) + + return ScanResult(files=list(files), traces=traces, + exceptions=exceptions, diagnostics=diags) + + +# -------------------------------------------------------------------------- +# The register: docs/requirements.md is the authoritative denominator +# -------------------------------------------------------------------------- + +@dataclass +class Requirement: + id: str + text: str = "" + traces_to: str = "" + priority: str = "" + status: str = "" + tiers: Set[str] = field(default_factory=set) + + @property + def ci_executable(self) -> bool: + """True when at least one verification tier can run on the CI host. + + A requirement with no tier recorded is *unknown*, not unexecutable — it + is reported so the register gets fixed, and it is not penalised here. + """ + if not self.tiers: + return True + return bool(self.tiers & CI_EXECUTABLE_TIERS) + + @property + def tier_known(self) -> bool: + return bool(self.tiers) + + +@dataclass +class Register: + requirements: Dict[str, Requirement] = field(default_factory=dict) + withdrawn: Dict[str, Requirement] = field(default_factory=dict) + + @property + def ids(self) -> Set[str]: + return set(self.requirements) + + @property + def total(self) -> int: + return len(self.requirements) + + def count(self, req_type: str) -> int: + return sum(1 for i in self.requirements if i.startswith(req_type + "-")) + + def ci_executable_ids(self) -> Set[str]: + return {i for i, r in self.requirements.items() if r.ci_executable} + + def unexecutable_ids(self) -> Set[str]: + return {i for i, r in self.requirements.items() if not r.ci_executable} + + def tier_unknown_ids(self) -> Set[str]: + return {i for i, r in self.requirements.items() if not r.tier_known} + + def parentless_ids(self) -> Set[str]: + """Requirements whose ``Traces to`` cell names nothing. + + SPEC.md section 6 asks the gate to report these: a requirement serving + no stated goal is scope creep, and it is invisible unless something + looks. A cell naming a section (`§4`) counts as a parent — the point is + that *something* was recorded, not that it was an ID. + """ + out = set() + for req_id, req in self.requirements.items(): + cell = req.traces_to.replace("*", "").strip() + if not cell or cell in {"-", "—", "n/a", "N/A", "TBD"}: + out.add(req_id) + return out + + +def _row_cells(line: str) -> List[str]: + return [c.strip() for c in line.strip().strip("|").split("|")] + + +def _is_separator(cells: Sequence[str]) -> bool: + return bool(cells) and all(re.fullmatch(r":?-{3,}:?", c) for c in cells) + + +def _iter_table_rows(markdown: str): + """Yield ``(header_cells, row_cells)`` for every markdown table data row. + + A table's header is the row immediately preceding its ``|---|`` separator; + anything before that separator is not data. Any non-table line ends the + current table. + """ + header: Optional[List[str]] = None + previous: Optional[List[str]] = None + for line in markdown.split("\n"): + stripped = line.strip() + if not stripped.startswith("|"): + header = None + previous = None + continue + cells = _row_cells(stripped) + if _is_separator(cells): + header = previous + continue + if header is not None: + yield header, cells + previous = cells + + +def _is_register_header(header: Sequence[str]) -> bool: + """A definition table: first column ``ID``, and a ``Requirement`` column. + + Deliberately narrow. The per-requirement verification plan is also keyed on + ``ID`` but has no ``Requirement`` column, and the tier summary table's first + column holds comma lists and ranges — neither defines requirements, and + counting their rows would inflate the denominator. Prose mentions and + ``Traces to`` references are excluded for the same reason: a naive scan for + ``AR-\\d{3}`` over the whole file counts every reference as a definition. + """ + if not header: + return False + lowered = [c.lower() for c in header] + return lowered[0] == "id" and "requirement" in lowered + + +def _is_tier_header(header: Sequence[str]) -> bool: + """Either of the two tables that assign verification tiers.""" + if len(header) < 2: + return False + lowered = [c.lower() for c in header] + return lowered[0] in ("id", "requirement") and lowered[1] == "tier" + + +def _column(header: Sequence[str], name: str, cells: Sequence[str]) -> str: + lowered = [c.lower() for c in header] + if name in lowered: + index = lowered.index(name) + if index < len(cells): + return cells[index] + return "" + + +def parse_tiers(cell: str) -> Set[str]: + """Read a tier cell such as ``T1 + T4``, ``**T2**``, or ``Out of CI``.""" + text = cell.replace("*", "").strip().lower() + tiers = {"T" + m.group(1) for m in re.finditer(r"\bt([1-4])\b", text)} + if "out of ci" in text or "not in ci" in text: + tiers.add("out-of-ci") + if "manual" in text: + tiers.add("manual") + if "static" in text: + tiers.add("static") + return tiers + + +def expand_id_spec(spec: str, defined: Set[str]) -> List[str]: + """Expand the ID cell of a tier table into concrete, defined IDs. + + Handles every shape the register actually uses: ``AR-002``, + ``AR-001, AR-005, AR-006``, ``AR-007 ... AR-017`` (with the unicode + ellipsis), ``AR-009/010``, and ``DP-*``. Expansion is intersected with the + defined set, so a range can never invent a requirement that does not exist. + """ + text = spec.replace("*", "").strip() + # `DP-*` survives the bold-strip above as a bare `DP-`; both mean "every + # requirement of this type". + if re.fullmatch(r"[A-Z]{2}-", text): + prefix = text[:2] + return sorted(i for i in defined if i.startswith(prefix + "-")) + + out: List[str] = [] + for part in text.split(","): + part = part.strip() + if not part: + continue + rng = re.fullmatch(r"([A-Z]{2})-(\d{3})\s*(?:…|\.\.\.)\s*([A-Z]{2})-(\d{3})", + part) + if rng and rng.group(1) == rng.group(3): + prefix = rng.group(1) + low, high = int(rng.group(2)), int(rng.group(4)) + out.extend(sorted( + i for i in defined + if i.startswith(prefix + "-") and low <= int(i.split("-")[1]) <= high)) + continue + slash = re.fullmatch(r"([A-Z]{2})-(\d{3})/(\d{3})", part) + if slash: + prefix = slash.group(1) + out.extend(f"{prefix}-{n}" for n in (slash.group(2), slash.group(3))) + continue + if REQ_ID_RE.match(part): + out.append(part) + return [i for i in out if i in defined] + + +def parse_register(markdown: str) -> Register: + """Build the register from ``docs/requirements.md``. + + Two passes: definitions first, because tier rows use ranges and wildcards + that can only be expanded against a known ID set. + """ + register = Register() + + for header, cells in _iter_table_rows(markdown): + if not _is_register_header(header) or not cells: + continue + first = cells[0].replace("*", "").strip() + if not REQ_ID_RE.match(first): + continue + if first.split("-")[0] not in LOCAL_TYPES + TEST_TYPES: + continue + req = Requirement( + id=first, + text=_column(header, "requirement", cells), + traces_to=_column(header, "traces to", cells), + priority=_column(header, "priority", cells), + status=_column(header, "status", cells), + ) + if req.status.replace("*", "").strip().lower() == "withdrawn": + # Permanently retired. Counting it in the denominator would depress + # coverage forever for something that no longer needs implementing. + register.withdrawn.setdefault(first, req) + register.requirements.pop(first, None) + continue + if first in register.withdrawn: + continue + # An ID may legitimately appear in more than one definition table; the + # first occurrence wins and duplicates never inflate the count. + register.requirements.setdefault(first, req) + + defined = register.ids + for header, cells in _iter_table_rows(markdown): + if not _is_tier_header(header) or len(cells) < 2: + continue + tiers = parse_tiers(cells[1]) + if not tiers: + continue + for req_id in expand_id_spec(cells[0], defined): + register.requirements[req_id].tiers |= tiers + + return register + + +def read_register(path: Path) -> Register: + return parse_register(path.read_text(encoding="utf-8")) + + +# -------------------------------------------------------------------------- +# Coverage +# -------------------------------------------------------------------------- + +@dataclass +class Coverage: + covered: List[str] + total: int + percent: float + orphaned: List[str] + unexecuted: List[str] + ci_executable_total: int + ci_percent: float + tier_unknown: List[str] + + def to_dict(self) -> dict: + return { + "covered": len(self.covered), + "coveredIds": self.covered, + "total": self.total, + "percent": self.percent, + "orphaned": self.orphaned, + "unexecuted": self.unexecuted, + "ciExecutableTotal": self.ci_executable_total, + "ciPercent": self.ci_percent, + "tierUnknown": self.tier_unknown, + } + + +def compute_coverage(traced_ids: Iterable[str], register: Register) -> Coverage: + """Coverage is ``|traced and defined and CI-executable| / |defined|``. + + Three exclusions, each of which is a way the number could otherwise lie: + + * Using the raw traced count as the numerator is what lets a ratio exceed + 100% — a tag naming a deleted or mistyped requirement would count as + covered. Those land in ``orphaned`` instead, so they get fixed rather + than silently counted or silently dropped. + * A requirement whose only verification tier is T4/GPU cannot run on this + CI host at all. It is reported in ``unexecuted`` and is not covered: + treating "has a test that never runs" as passing reports success the gate + cannot substantiate. + * UT/IT test IDs and the system-level PR/SR IDs are different taxonomies + with their own registers, so they neither count nor orphan here. + """ + traced = {i for i in traced_ids if i.split("-")[0] in LOCAL_TYPES} + defined = register.ids + + orphaned = sorted(traced - defined) + matched = traced & defined + unexecuted = sorted(i for i in matched if not register.requirements[i].ci_executable) + covered = sorted(matched - set(unexecuted)) + + total = len(defined) + ci_total = len(register.ci_executable_ids()) + percent = round(100.0 * len(covered) / total, 1) if total else 0.0 + ci_percent = round(100.0 * len(covered) / ci_total, 1) if ci_total else 0.0 + + return Coverage( + covered=covered, + total=total, + percent=percent, + orphaned=orphaned, + unexecuted=unexecuted, + ci_executable_total=ci_total, + ci_percent=ci_percent, + tier_unknown=sorted(register.tier_unknown_ids()), + ) + + +# -------------------------------------------------------------------------- +# Report assembly +# -------------------------------------------------------------------------- + +@dataclass +class Report: + timestamp: str + root: Path + register: Register + scan: ScanResult + coverage: Coverage + by_type: Dict[str, List[str]] + requirement_map: Dict[str, List[TraceEntry]] + external_orphans: List[str] = field(default_factory=list) + #: Policy, set by the caller: whether an orphan tag fails the run. + allow_orphans: bool = False + + +def build_report(root: Path, register: Register, scan: ScanResult, + system_ids: Optional[Set[str]] = None) -> Report: + requirement_map: Dict[str, List[TraceEntry]] = {} + seen_by_type: Dict[str, Set[str]] = {t: set() for t in KNOWN_TYPES} + seen_by_type["OTHER"] = set() + + for entry in scan.traces: + for req in entry.requirements: + requirement_map.setdefault(req, []).append(entry) + prefix = req.split("-")[0] + seen_by_type[prefix if prefix in KNOWN_TYPES else "OTHER"].add(req) + + coverage = compute_coverage(requirement_map.keys(), register) + + external_orphans: List[str] = [] + if system_ids is not None: + external_orphans = sorted( + i for i in requirement_map + if i.split("-")[0] in EXTERNAL_TYPES and i not in system_ids) + + return Report( + timestamp=datetime.now(timezone.utc).isoformat(timespec="seconds"), + root=root, + register=register, + scan=scan, + coverage=coverage, + by_type={k: sorted(v) for k, v in seen_by_type.items()}, + requirement_map=requirement_map, + external_orphans=external_orphans, + ) + + +def parse_system_spec(markdown: str) -> Set[str]: + """IDs of PR/SR requirements defined in the umbrella SPEC.md. + + They are defined as headings (``### SR-001 - ...``) and as bolded leading + table cells (``| **PR-001** | ... |``), so accept both. + """ + ids: Set[str] = set() + for line in markdown.split("\n"): + stripped = line.strip() + heading = re.match(r"^#{1,6}\s+\**((?:PR|SR)-\d{3})\**\b", stripped) + if heading: + ids.add(heading.group(1)) + continue + if stripped.startswith("|"): + cells = _row_cells(stripped) + if cells: + first = cells[0].replace("*", "").strip() + if re.fullmatch(r"(?:PR|SR)-\d{3}", first): + ids.add(first) + return ids + + +def report_to_json_dict(report: Report) -> dict: + register = report.register + defined = {t: register.count(t) for t in LOCAL_TYPES} + defined["total"] = register.total + + requirement_detail = {} + for req_id, req in sorted(register.requirements.items()): + entries = report.requirement_map.get(req_id, []) + requirement_detail[req_id] = { + "requirement": req.text, + "tracesTo": req.traces_to, + "priority": req.priority, + "status": req.status, + "tiers": sorted(req.tiers), + "ciExecutable": req.ci_executable, + "taggedIn": sorted({e.file for e in entries}), + "state": _requirement_state(req, bool(entries)), + } + + return { + "timestamp": report.timestamp, + "totalFiles": len(report.scan.files), + "totalTraces": len(report.scan.traces), + "totalExceptions": len(report.scan.exceptions), + "requirements": {k: [e.to_dict() for e in v] + for k, v in sorted(report.requirement_map.items())}, + "byType": report.by_type, + "defined": defined, + "coverage": report.coverage.to_dict(), + "gpuOnlyRequirements": sorted(register.unexecutable_ids()), + "parentlessRequirements": sorted(register.parentless_ids()), + "withdrawn": sorted(register.withdrawn), + "exceptions": [e.to_dict() for e in report.scan.exceptions], + "externalOrphans": report.external_orphans, + "diagnostics": report.scan.diagnostics.to_dict(), + "requirementDetail": requirement_detail, + } + + +def per_type_stats(report: Report) -> Dict[str, Tuple[int, int, int]]: + """``{type: (covered, unexecuted, defined)}``. + + Covered here means the same thing it means in the headline figure, so the + per-type rows sum to it. Reporting "traced and defined" per type while the + total excludes unexecuted requirements is how a breakdown quietly stops + adding up to its own total. + """ + covered = set(report.coverage.covered) + unexecuted = set(report.coverage.unexecuted) + stats: Dict[str, Tuple[int, int, int]] = {} + for req_type in LOCAL_TYPES: + prefix = req_type + "-" + stats[req_type] = ( + len([i for i in covered if i.startswith(prefix)]), + len([i for i in unexecuted if i.startswith(prefix)]), + report.register.count(req_type), + ) + return stats + + +def _requirement_state(req: Requirement, tagged: bool) -> str: + if not tagged: + return "untagged" + if not req.ci_executable: + return "tagged-unexecuted" + return "covered" + + +# -------------------------------------------------------------------------- +# Output formats +# -------------------------------------------------------------------------- + +def generate_markdown(report: Report) -> str: + register = report.register + cov = report.coverage + out: List[str] = [] + add = out.append + + add("# Requirements traceability matrix") + add("") + add("") + add("") + add("") + add(f"**Generated:** {report.timestamp}") + add("") + add("Denominators are read from [`requirements.md`](requirements.md) at run " + "time, never hardcoded. Coverage counts a requirement only when it is " + "tagged in source **and** has a verification tier this CI host can " + "execute — CI is an Intel N100 with no discrete GPU.") + add("") + + add("## Summary") + add("") + add("| Metric | Value |") + add("|---|---|") + add(f"| Source files scanned | {len(report.scan.files)} |") + add(f"| TRACES tags found | {len(report.scan.traces)} |") + add(f"| EXCEPTION tags found | {len(report.scan.exceptions)} |") + add(f"| Requirements defined | {register.total} |") + add(f"| Requirements covered | {len(cov.covered)} |") + add(f"| **Coverage** | **{cov.percent}%** ({len(cov.covered)}/{cov.total}) |") + add(f"| Coverage of CI-executable scope | {cov.ci_percent}% " + f"({len(cov.covered)}/{cov.ci_executable_total}) |") + add(f"| Tagged but unexecuted in CI (T4/GPU) | {len(cov.unexecuted)} |") + add(f"| Orphan tags | {len(cov.orphaned)} |") + add("") + + add("### By type") + add("") + add("| Type | Covered | Tagged but unexecuted | Defined |") + add("|---|---|---|---|") + for req_type, (covered, unexecuted, defined_n) in per_type_stats(report).items(): + add(f"| {req_type} | {covered} | {unexecuted} | {defined_n} |") + add("") + for req_type in TEST_TYPES + EXTERNAL_TYPES: + tagged = report.by_type.get(req_type, []) + if tagged: + add(f"- **{req_type}** tags present (separate taxonomy, not counted " + f"in coverage): {', '.join(tagged)}") + add("") + + unexecutable = sorted(register.unexecutable_ids()) + add("## Not executable in CI") + add("") + add("CI runs on an Intel N100 with no discrete GPU. These requirements have " + "no verification tier that can run here, so a tag on them is evidence " + "of *intent*, not of verification. They are never counted as covered.") + add("") + if unexecutable: + add("| ID | Tiers | Tagged in source | Requirement |") + add("|---|---|---|---|") + for req_id in unexecutable: + req = register.requirements[req_id] + tagged = "yes" if req_id in report.requirement_map else "no" + add(f"| {req_id} | {', '.join(sorted(req.tiers)) or '-'} | {tagged} " + f"| {_truncate(req.text)} |") + else: + add("_None._") + add("") + if cov.unexecuted: + add(f"**Tagged but unexecuted:** {', '.join(cov.unexecuted)} — a test " + "exists and is tagged, but only a GPU host can run it. Report " + "those runs separately.") + add("") + + add("## Orphan tags") + add("") + add("A tag naming an ID `requirements.md` does not define. This is what " + "renumbering produces, and what a typo produces.") + add("") + if cov.orphaned: + add("| ID | Locations |") + add("|---|---|") + for req_id in cov.orphaned: + where = ", ".join(f"`{e.file}:{e.line}`" + for e in report.requirement_map[req_id]) + add(f"| {req_id} | {where} |") + else: + add("_None._") + add("") + + add("## Requirements tracing up to nothing") + add("") + add("A register row whose `Traces to` cell names no parent. Work serving no " + "stated goal is how scope creeps in, and it is invisible unless " + "something looks.") + add("") + parentless = sorted(register.parentless_ids()) + if parentless: + add(", ".join(f"`{i}`" for i in parentless)) + else: + add("_None._") + add("") + + add("## Recorded exceptions") + add("") + add("Deliberate, documented departures from an invariant " + "(`EXCEPTION: AR-nnn `). Reported separately and never counted " + "as coverage — an exception is a decision to be reviewed, not evidence " + "a requirement is met.") + add("") + if report.scan.exceptions: + add("| Requirement | Location | Reason |") + add("|---|---|---|") + for exc in report.scan.exceptions: + reason = _truncate(exc.reason, 90) or "**no reason recorded**" + add(f"| {exc.requirement} | [`{exc.file}:{exc.line}`]" + f"({_source_link(exc.file)}#L{exc.line}) | {reason} |") + else: + add("_None._") + add("") + + add("## Register") + add("") + add("| ID | Status | Tier | Traces to | Trace state | Tagged in | Requirement |") + add("|---|---|---|---|---|---|---|") + for req_id, req in sorted(register.requirements.items(), + key=lambda kv: (LOCAL_TYPES.index(kv[0].split("-")[0]) + if kv[0].split("-")[0] in LOCAL_TYPES + else 99, kv[0])): + entries = report.requirement_map.get(req_id, []) + state = {"covered": "covered", + "tagged-unexecuted": "tagged, unexecuted (T4/GPU)", + "untagged": "untagged"}[ + _requirement_state(req, bool(entries))] + files = ", ".join(f"`{f}`" for f in sorted({e.file for e in entries})) or "-" + tiers = ", ".join(sorted(req.tiers)) or "unset" + add(f"| {req_id} | {_truncate(req.status, 20) or '-'} | {tiers} " + f"| {_truncate(req.traces_to, 40) or '-'} " + f"| {state} | {files} | {_truncate(req.text)} |") + add("") + + add("## Detailed mapping") + add("") + if not report.requirement_map: + add("_No TRACES tags found yet. Tags are added as code is written; an " + "empty matrix on a new tree is the correct reading, not a failure._") + add("") + for req_id in sorted(report.requirement_map): + entries = report.requirement_map[req_id] + add(f"### {req_id}") + add("") + add(f"**Locations:** {len(entries)}") + add("") + for entry in entries: + add(f"- [`{entry.file}:{entry.line}`]" + f"({_source_link(entry.file)}#L{entry.line}) — " + f"`{_truncate(entry.context, 90)}`") + add("") + + if not report.scan.diagnostics.empty: + add("## Tag diagnostics") + add("") + diags = report.scan.diagnostics + for label, items in ( + ("Malformed tags", diags.malformed_tags), + ("Groups mixing requirement types (pipe separates types)", + diags.mixed_type_groups), + ("Unrecognised ID prefixes", diags.unknown_id_types), + ("Exceptions with no recorded reason", diags.exceptions_without_reason), + ): + if items: + add(f"**{label}:**") + add("") + for item in items: + detail = {k: v for k, v in item.items() + if k not in ("file", "line")} + add(f"- `{item.get('file')}:{item.get('line')}` — {detail}") + add("") + + return "\n".join(out) + "\n" + + +def _source_link(file_rel: str) -> str: + """Link from docs/traceability.md back to a source file at the repo root.""" + return "../" + file_rel + + +def _truncate(text: str, limit: int = 70) -> str: + """Collapse to one line, cap the length, and escape table-breaking pipes.""" + text = " ".join(text.split()) + if len(text) > limit: + text = text[: limit - 1] + "…" + return text.replace("|", "\\|") + + +def format_coverage_report(report: Report, min_coverage: float) -> Tuple[str, int]: + """Human-readable gate output plus the exit code it implies.""" + register = report.register + cov = report.coverage + lines: List[str] = [] + add = lines.append + failures: List[str] = [] + + add("Requirement traceability") + add("=" * 72) + add(f"Source files scanned : {len(report.scan.files)}") + add(f"TRACES tags found : {len(report.scan.traces)}") + add(f"EXCEPTION tags found : {len(report.scan.exceptions)}") + add("") + + # Self-checks. With a low threshold these are what make the gate mean + # something: a parser that silently returns nothing would otherwise report + # 0/0 and pass. + if register.total == 0: + failures.append( + "requirements.md parsed to ZERO requirements - the register parser " + "is broken or the file moved. Refusing to report coverage.") + if not report.scan.files: + failures.append( + "no source files were scanned - the scan roots do not exist. " + "Refusing to report coverage against an empty tree.") + + add("Coverage by type (covered / defined):") + for req_type, (covered, unexecuted, defined_n) in per_type_stats(report).items(): + suffix = f" (+{unexecuted} tagged but unexecuted)" if unexecuted else "" + add(f" {req_type}: {covered} / {defined_n}{suffix}") + add("") + add(f"Overall : {len(cov.covered)} / {cov.total} ({cov.percent}%)") + add(f"CI scope : {len(cov.covered)} / {cov.ci_executable_total} " + f"({cov.ci_percent}%) [excludes {cov.total - cov.ci_executable_total} " + "requirement(s) no GPU-less host can verify]") + add("") + + unexecutable = sorted(register.unexecutable_ids()) + if unexecutable: + add(f"Not executable on this CI host (T4/GPU or out-of-CI): " + f"{len(unexecutable)}") + add(f" {', '.join(unexecutable)}") + if cov.unexecuted: + add("") + add("TAGGED BUT UNEXECUTED - a test exists and is tagged, but only a " + "GPU host can run it.") + add(" These are NOT counted as covered:") + for req_id in cov.unexecuted: + tiers = ", ".join(sorted(register.requirements[req_id].tiers)) + add(f" {req_id} (tier {tiers})") + add("") + + parentless = sorted(register.parentless_ids()) + if parentless: + add(f"WARNING: {len(parentless)} requirement(s) trace up to nothing - " + "no parent recorded in the register's `Traces to` column. Work " + "serving no stated goal is how scope creeps in:") + add(f" {', '.join(parentless)}") + add("") + + if cov.tier_unknown: + add(f"WARNING: {len(cov.tier_unknown)} requirement(s) have no " + "verification tier in requirements.md; they are counted as " + "CI-executable by default. Add them to the verification plan:") + add(f" {', '.join(cov.tier_unknown)}") + add("") + + if report.scan.exceptions: + add(f"Recorded invariant exceptions: {len(report.scan.exceptions)}") + for exc in report.scan.exceptions: + reason = exc.reason or "(NO REASON RECORDED)" + add(f" {exc.requirement} {exc.file}:{exc.line} {_truncate(reason, 80)}") + add("") + + if cov.orphaned: + add("ORPHAN TAGS - traced in source, not defined in requirements.md:") + for req_id in cov.orphaned: + where = ", ".join(f"{e.file}:{e.line}" + for e in report.requirement_map[req_id]) + add(f" {req_id} ({where})") + add(" Fix the tag, or add the requirement to the register.") + add("") + + if report.external_orphans: + add("ORPHAN SYSTEM TAGS - PR/SR IDs the system SPEC.md does not define:") + add(f" {', '.join(report.external_orphans)}") + add("") + + diags = report.scan.diagnostics + if not diags.empty: + add("Tag diagnostics:") + for item in diags.malformed_tags: + add(f" malformed {item.get('file')}:{item.get('line')} {item}") + for item in diags.mixed_type_groups: + add(f" mixed types {item['file']}:{item['line']} {item['group']} " + "(a pipe, not a comma, separates types)") + for item in diags.unknown_id_types: + add(f" unknown id {item['file']}:{item['line']} {item['id']}") + for item in diags.exceptions_without_reason: + add(f" exception without reason {item['file']}:{item['line']} " + f"{item['requirement']}") + add("") + + # A ratio above 100% means the computation is broken. This is the check + # that would have caught JellyTau's frozen denominators immediately. + if cov.percent > 100 or cov.ci_percent > 100: + failures.append( + f"coverage ({cov.percent}%) exceeds 100% - the gate is " + "miscomputing. Do not trust this run.") + if len(cov.covered) > cov.total: + failures.append( + f"covered ({len(cov.covered)}) exceeds defined ({cov.total}) - " + "the gate is miscomputing.") + + if cov.orphaned and not report.allow_orphans: + failures.append( + f"{len(cov.orphaned)} orphan tag(s): {', '.join(cov.orphaned)}") + + if cov.percent < min_coverage: + failures.append( + f"coverage ({cov.percent}%) is below the minimum ({min_coverage}%)") + + if failures: + add("FAILED:") + for reason in failures: + add(f" - {reason}") + return "\n".join(lines) + "\n", 1 + + add(f"OK: coverage {cov.percent}% >= minimum {min_coverage}%, " + f"{len(cov.orphaned)} orphan tag(s)") + return "\n".join(lines) + "\n", 0 + + +# -------------------------------------------------------------------------- +# CLI +# -------------------------------------------------------------------------- + +def build_arg_parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser( + prog="extract_traces.py", + description=__doc__.split("\n")[0], + formatter_class=argparse.RawDescriptionHelpFormatter, + ) + parser.add_argument("--format", choices=("json", "markdown", "coverage"), + default="coverage", + help="output written to stdout (default: coverage)") + parser.add_argument("--root", type=Path, default=REPO_ROOT, + help="repository root to scan (default: derived from " + "this script's location)") + parser.add_argument("--requirements", type=Path, default=None, + help="register path (default: /docs/requirements.md)") + parser.add_argument("--system-spec", type=Path, default=None, + help="optional umbrella SPEC.md defining PR/SR IDs; " + "enables orphan checking for those types") + parser.add_argument("--json-out", type=Path, default=None, + help="also write the JSON report here") + parser.add_argument("--markdown-out", type=Path, default=None, + help="also write the markdown matrix here") + parser.add_argument("--min-coverage", type=float, default=0.0, + help="minimum coverage percent; below it the run fails " + "(default: 0, i.e. the correctness checks gate but " + "the percentage does not)") + parser.add_argument("--allow-orphans", action="store_true", + help="report orphan tags without failing") + return parser + + +def main(argv: Optional[Sequence[str]] = None) -> int: + args = build_arg_parser().parse_args(argv) + + root = args.root.resolve() + req_path = args.requirements or (root / "docs" / "requirements.md") + if not req_path.is_file(): + print(f"FAILED: requirements register not found at {req_path}", + file=sys.stderr) + return 2 + + register = read_register(req_path) + files = iter_source_files(root) + scan = scan_files(files, root) + + system_ids = None + if args.system_spec: + if not args.system_spec.is_file(): + print(f"FAILED: --system-spec not found at {args.system_spec}", + file=sys.stderr) + return 2 + system_ids = parse_system_spec(args.system_spec.read_text(encoding="utf-8")) + + report = build_report(root, register, scan, system_ids) + report.allow_orphans = args.allow_orphans + + json_text = json.dumps(report_to_json_dict(report), indent=2, sort_keys=False) + markdown_text = generate_markdown(report) + + if args.json_out: + args.json_out.parent.mkdir(parents=True, exist_ok=True) + args.json_out.write_text(json_text, encoding="utf-8") + if args.markdown_out: + args.markdown_out.parent.mkdir(parents=True, exist_ok=True) + args.markdown_out.write_text(markdown_text, encoding="utf-8") + + if args.format == "json": + print(json_text) + return 0 + if args.format == "markdown": + print(markdown_text, end="") + return 0 + + text, code = format_coverage_report(report, args.min_coverage) + print(text, end="") + return code + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/traceability/test_extract_traces.py b/scripts/traceability/test_extract_traces.py new file mode 100755 index 0000000..4f2f4d4 --- /dev/null +++ b/scripts/traceability/test_extract_traces.py @@ -0,0 +1,706 @@ +#!/usr/bin/env python3 +"""Tests for the traceability extractor and coverage gate. + +Run standalone (no third-party dependencies):: + + python3 scripts/traceability/test_extract_traces.py + +or under pytest, which discovers the same functions:: + + pytest scripts/traceability/test_extract_traces.py + +Almost every test runs over fixture strings rather than the live +``docs/requirements.md``, so their meaning does not drift as requirements are +added. The two properties they exist to pin are the ones JellyTau's gate lost +(see JellyTau/docs/specs/traceability-gate-repair.md): + +* the denominator is computed from the register at run time, so adding a + requirement lowers coverage until it is traced; +* the numerator is an intersection, so a tag naming an undefined ID cannot push + the ratio above 100%. + +Plus the rule specific to this repo: a requirement only verifiable on GPU +hardware is reported as tagged-but-unexecuted and never counted as covered. +""" + +from __future__ import annotations + +import io +import sys +import tempfile +from contextlib import redirect_stdout +from pathlib import Path + +HERE = Path(__file__).resolve().parent +sys.path.insert(0, str(HERE)) + +import extract_traces as et # noqa: E402 + +# The literal tag keyword is assembled at run time so that this file's fixtures +# do not register as real traces when the extractor scans scripts/. +TAG = "TRA" + "CES:" +EXC = "EXCEP" + "TION:" + + +# -------------------------------------------------------------------------- +# Tag parsing +# -------------------------------------------------------------------------- + +def test_parses_a_single_requirement(): + groups, junk = et.parse_traces_tag(" AR-012") + assert groups == [["AR-012"]] + assert junk == [] + + +def test_parses_multiple_types_separated_by_pipe(): + # The house format: a pipe separates requirement *types*, a comma separates + # IDs within a type. The grouping is preserved, not flattened away, so a + # malformed grouping stays detectable. + groups, junk = et.parse_traces_tag(" AR-012, AR-013 | SR-002") + assert groups == [["AR-012", "AR-013"], ["SR-002"]] + assert junk == [] + + +def test_parses_three_groups_including_test_ids(): + groups, _ = et.parse_traces_tag(" AR-012 | SR-002 | UT-003, UT-004") + assert groups == [["AR-012"], ["SR-002"], ["UT-003", "UT-004"]] + + +def test_strips_a_trailing_block_comment_terminator(): + groups, junk = et.parse_traces_tag(" AR-001 */") + assert groups == [["AR-001"]] + assert junk == [] + + +def test_does_not_harvest_ids_out_of_prose_after_the_tag(): + # A trailing sentence must not smuggle IDs into the trace set: AR-999 here + # is discussion, not a claim that this code satisfies AR-999. + groups, junk = et.parse_traces_tag(" AR-001 - see also AR-999 in the notes") + assert groups == [["AR-001"]] + assert junk and "AR-999" in junk[0] + + +def test_ignores_a_tag_with_no_ids_at_all(): + groups, _ = et.parse_traces_tag(" see the register") + assert groups == [] + + +def test_two_digit_id_is_not_accepted_as_a_requirement(): + # AR-12 is a typo for AR-012; silently accepting it would create a + # phantom requirement. + groups, junk = et.parse_traces_tag(" AR-12") + assert groups == [] + assert junk == ["AR-12"] + + +# -------------------------------------------------------------------------- +# Scanning C++ and Python sources +# -------------------------------------------------------------------------- + +def _write_tree(root: Path) -> None: + (root / "src").mkdir(parents=True, exist_ok=True) + (root / "scripts").mkdir(parents=True, exist_ok=True) + (root / "external").mkdir(parents=True, exist_ok=True) + (root / "src" / "tracker.hpp").write_text( + "#pragma once\n" + f"/// {TAG} AR-012, AR-013 | SR-002\n" + "struct TrackRegistry {\n" + " void close_all();\n" + "};\n", + encoding="utf-8") + (root / "scripts" / "gallery.py").write_text( + "def build_gallery(cast):\n" + ' """Build a gallery from Jellyfin plus TMDB fallback.\n' + "\n" + f" {TAG} GR-001 | SR-005\n" + ' """\n' + " return {}\n", + encoding="utf-8") + (root / "external" / "vendored.cpp").write_text( + f"// {TAG} AR-001\n", encoding="utf-8") + (root / "src" / "notes.txt").write_text( + f"// {TAG} AR-002\n", encoding="utf-8") + + +def test_scans_cpp_and_python_but_not_vendored_or_non_source(): + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + _write_tree(root) + files = et.iter_source_files(root) + names = sorted(f.name for f in files) + assert names == ["gallery.py", "tracker.hpp"], names + + scan = et.scan_files(files, root) + traced = sorted({i for t in scan.traces for i in t.requirements}) + assert traced == ["AR-012", "AR-013", "GR-001", "SR-002", "SR-005"] + + +def test_context_is_found_below_a_cpp_tag_and_above_a_python_tag(): + # The two languages put the tag on opposite sides of what it describes. + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + _write_tree(root) + scan = et.scan_files(et.iter_source_files(root), root) + contexts = {t.file: t.context for t in scan.traces} + assert "TrackRegistry" in contexts["src/tracker.hpp"] + assert "build_gallery" in contexts["scripts/gallery.py"] + + +# -------------------------------------------------------------------------- +# EXCEPTION tags +# -------------------------------------------------------------------------- + +EXCEPTION_SOURCE = ( + "float outlier_score(const Refs& refs) {\n" + f" // {EXC} AR-024 distributional check on an actor's own references,\n" + " return spread(refs);\n" + "}\n" +) + + +def test_exception_tag_is_captured_with_its_reason(): + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + (root / "src").mkdir(parents=True) + (root / "src" / "outlier.cpp").write_text(EXCEPTION_SOURCE, encoding="utf-8") + scan = et.scan_files(et.iter_source_files(root), root) + assert len(scan.exceptions) == 1 + exc = scan.exceptions[0] + assert exc.requirement == "AR-024" + assert exc.reason.startswith("distributional check") + assert exc.line == 2 + + +def test_exception_is_never_counted_as_coverage(): + # An exception is a recorded decision to depart from an invariant. Counting + # it as evidence the requirement is met inverts its meaning. + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + (root / "src").mkdir(parents=True) + (root / "src" / "outlier.cpp").write_text(EXCEPTION_SOURCE, encoding="utf-8") + scan = et.scan_files(et.iter_source_files(root), root) + assert scan.traces == [] + register = et.parse_register( + "| ID | Requirement | Status |\n|---|---|---|\n" + "| AR-024 | Always the calibrated probability | Planned |\n") + cov = et.compute_coverage( + [i for t in scan.traces for i in t.requirements], register) + assert cov.covered == [] + assert cov.percent == 0.0 + + +def test_exception_without_a_reason_is_reported(): + # CLAUDE.md: an exception is only *agreed* if the reason is recorded. + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + (root / "src").mkdir(parents=True) + (root / "src" / "bare.cpp").write_text( + f"// {EXC} AR-024\n", encoding="utf-8") + scan = et.scan_files(et.iter_source_files(root), root) + assert len(scan.exceptions) == 1 + assert scan.diagnostics.exceptions_without_reason + + +def test_mixed_type_group_is_reported(): + # `AR-001, SR-002` in one group misuses the comma; the pipe is what + # separates types, so the tag does not say what it appears to say. + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + (root / "src").mkdir(parents=True) + (root / "src" / "a.cpp").write_text( + f"// {TAG} AR-001, SR-002\n", encoding="utf-8") + scan = et.scan_files(et.iter_source_files(root), root) + assert scan.diagnostics.mixed_type_groups + + +# -------------------------------------------------------------------------- +# The register: denominators from requirements.md +# -------------------------------------------------------------------------- + +REGISTER_HEADER = "| ID | Requirement | Traces to | Priority | Status |\n|---|---|---|---|---|\n" + + +def test_counts_a_well_formed_table_row_as_a_defined_requirement(): + md = REGISTER_HEADER + ( + "| AR-001 | Detect faces in sampled frames | SR-002 | High | Done |\n" + "| AR-002 | Minimum face size 66x66 px | SR-002 | High | Planned |\n") + register = et.parse_register(md) + assert register.count("AR") == 2 + assert register.count("GR") == 0 + assert register.total == 2 + + +def test_does_not_count_ids_that_appear_only_in_the_traces_to_column(): + # The bug this parse rule avoids: a naive scan for AR-\d{3} over the whole + # file counts every reference as a definition and inflates the denominator. + md = REGISTER_HEADER + ( + "| GR-001 | Build gallery from library cast | SR-001, SR-005 | High | Done |\n" + "| GR-002 | Incremental merge refresh | PR-003 | High | Done |\n") + register = et.parse_register(md) + assert register.count("GR") == 2 + assert register.ids == {"GR-001", "GR-002"} + + +def test_does_not_count_ids_mentioned_in_prose(): + md = ("Some prose explaining that AR-005 relates to GR-001 and VR-003.\n\n" + + REGISTER_HEADER + + "| AR-005 | Align to 112x112 | SR-002 | High | Done |\n") + register = et.parse_register(md) + assert register.ids == {"AR-005"} + + +def test_does_not_count_the_verification_plan_table_as_definitions(): + # The per-requirement verification plan is also keyed on `ID`, but it + # assigns tiers rather than defining requirements. Counting its rows would + # double the denominator for every requirement that has a plan entry. + md = (REGISTER_HEADER + + "| AR-001 | Detect faces | SR-002 | High | Done |\n" + + "\n" + + "| ID | Tier | Test asserts | Edge cases to cover |\n|---|---|---|---|\n" + + "| AR-001 | T3 | Detector returns plausible boxes | smoke only |\n" + + "| AR-099 | T1 | Something not in the register | - |\n") + register = et.parse_register(md) + assert register.ids == {"AR-001"} + assert register.total == 1 + + +def test_deduplicates_an_id_listed_in_two_definition_tables(): + md = (REGISTER_HEADER + + "| AR-001 | Detect faces | SR-002 | High | Done |\n" + + "\n" + + REGISTER_HEADER + + "| AR-001 | Detect faces | SR-002 | High | Done |\n") + register = et.parse_register(md) + assert register.total == 1 + + +def test_withdrawn_requirements_leave_the_denominator(): + # IDs are permanent, but a withdrawn requirement can never be implemented. + # Leaving it in the denominator would depress coverage forever. + md = REGISTER_HEADER + ( + "| AR-001 | Detect faces | SR-002 | High | Done |\n" + "| AR-002 | Superseded mechanism | SR-002 | High | Withdrawn |\n") + register = et.parse_register(md) + assert register.ids == {"AR-001"} + assert "AR-002" in register.withdrawn + + +def test_the_denominator_is_live_adding_a_row_lowers_coverage(): + # The property JellyTau's frozen literals destroyed. Same traced set, one + # more requirement defined => a lower percentage, mechanically. + base = REGISTER_HEADER + "| AR-001 | A | SR-002 | High | Done |\n" + grown = base + "| AR-002 | B | SR-002 | High | Planned |\n" + before = et.compute_coverage(["AR-001"], et.parse_register(base)) + after = et.compute_coverage(["AR-001"], et.parse_register(grown)) + assert before.percent == 100.0 + assert after.percent == 50.0 + assert after.total == 2 + + +def test_register_captures_the_row_fields_not_just_the_id(): + md = REGISTER_HEADER + ( + "| AR-012 | Presence follows track extent | **SR-002** | High | Planned |\n") + req = et.parse_register(md).requirements["AR-012"] + assert req.text == "Presence follows track extent" + assert req.traces_to == "**SR-002**" + assert req.status == "Planned" + + +# -------------------------------------------------------------------------- +# Verification tiers and the GPU-less CI host +# -------------------------------------------------------------------------- + +TIER_REGISTER = REGISTER_HEADER + "".join( + f"| AR-{n:03d} | Requirement {n} | SR-002 | High | Planned |\n" + for n in range(1, 10)) + "".join( + f"| VR-{n:03d} | Study {n} | PR-002 | Medium | Planned |\n" + for n in range(1, 4)) + + +def _tier_table(rows: str) -> str: + return "| Requirement | Tier | Note |\n|---|---|---|\n" + rows + + +def test_tier_assignment_handles_lists_ranges_and_wildcards(): + md = TIER_REGISTER + "\n" + _tier_table( + "| AR-001, AR-005 | T3 | smoke |\n" + "| AR-002 … AR-004 | **T2** | replay |\n" + "| AR-006 | T1 + T4 | mixed |\n" + "| VR-* | Out of CI | studies |\n") + register = et.parse_register(md) + assert register.requirements["AR-001"].tiers == {"T3"} + assert register.requirements["AR-003"].tiers == {"T2"} + assert register.requirements["AR-006"].tiers == {"T1", "T4"} + assert register.requirements["VR-002"].tiers == {"out-of-ci"} + + +def test_a_range_cannot_invent_a_requirement_the_register_lacks(): + md = TIER_REGISTER + "\n" + _tier_table("| AR-001 … AR-050 | T2 | wide |\n") + register = et.parse_register(md) + assert register.total == 12 + assert "AR-050" not in register.ids + + +def test_slash_shorthand_in_the_verification_plan_expands(): + md = TIER_REGISTER + "\n" + ( + "| ID | Tier | Test asserts | Edge cases |\n|---|---|---|---|\n" + "| AR-009/008 | T2 | Cut shifts weighting | cut with same people |\n") + register = et.parse_register(md) + assert register.requirements["AR-008"].tiers == {"T2"} + assert register.requirements["AR-009"].tiers == {"T2"} + + +def test_tiers_from_both_tables_are_unioned_not_overwritten(): + # The summary table says AR-006 is T4; the per-requirement plan adds a T1 + # equivalence check. The T1 part does run in CI, so the requirement is + # executable and must not be written off as GPU-only. + md = TIER_REGISTER + "\n" + _tier_table("| AR-006 | T4 | GPU host only |\n") + "\n" + ( + "| ID | Tier | Test asserts | Edge cases |\n|---|---|---|---|\n" + "| AR-006 | T1 + T4 | GEMM equals reference loop | small input in CI |\n") + register = et.parse_register(md) + assert register.requirements["AR-006"].tiers == {"T1", "T4"} + assert register.requirements["AR-006"].ci_executable + + +def test_a_t4_only_requirement_is_not_ci_executable(): + md = TIER_REGISTER + "\n" + _tier_table("| AR-007 | **T4** | GPU only |\n") + register = et.parse_register(md) + assert not register.requirements["AR-007"].ci_executable + assert register.unexecutable_ids() == {"AR-007"} + + +def test_a_requirement_tracing_up_to_nothing_is_reported(): + # SPEC.md section 6: a requirement citing no parent is scope creep, and it + # is invisible unless something looks. A section reference counts as a + # parent - what matters is that something was recorded. + md = REGISTER_HEADER + ( + "| AR-001 | Has a parent | SR-002 | High | Done |\n" + "| AR-002 | Parent is a section | §4 | Medium | Planned |\n" + "| AR-003 | Serves nothing stated | - | Low | Planned |\n" + "| AR-004 | Blank cell | | Low | Planned |\n") + register = et.parse_register(md) + assert register.parentless_ids() == {"AR-003", "AR-004"} + + +def test_a_requirement_with_no_tier_is_unknown_not_unexecutable(): + register = et.parse_register(TIER_REGISTER) + assert register.tier_unknown_ids() == register.ids + assert register.unexecutable_ids() == set() + + +# -------------------------------------------------------------------------- +# Coverage arithmetic +# -------------------------------------------------------------------------- + +COVERAGE_REGISTER = et.parse_register( + REGISTER_HEADER + + "| AR-001 | A | SR-002 | High | Done |\n" + + "| AR-002 | B | SR-002 | High | Done |\n" + + "| GR-001 | C | SR-005 | High | Done |\n" + + "| AR-027 | Arbitrary gallery scale | SR-001 | High | Planned |\n" + + "\n" + + _tier_table("| AR-001, AR-002 | T2 | replay |\n" + "| GR-001 | T1 | bookkeeping |\n" + "| AR-027 | **T4** | GPU host only |\n")) + + +def test_coverage_is_the_intersection_of_traced_and_defined(): + cov = et.compute_coverage(["AR-001", "GR-001"], COVERAGE_REGISTER) + assert cov.covered == ["AR-001", "GR-001"] + assert cov.total == 4 + assert cov.percent == 50.0 + + +def test_a_traced_but_undefined_id_cannot_inflate_the_numerator(): + # This is exactly how a ratio exceeds 100%: a tag naming a renumbered or + # mistyped requirement counted as covered. + cov = et.compute_coverage(["AR-001", "GR-001", "AR-097"], COVERAGE_REGISTER) + assert cov.covered == ["AR-001", "GR-001"] + assert cov.percent == 50.0 + + +def test_orphan_tags_are_reported_so_they_get_fixed(): + cov = et.compute_coverage(["AR-001", "AR-097", "GR-404"], COVERAGE_REGISTER) + assert cov.orphaned == ["AR-097", "GR-404"] + + +def test_no_orphans_when_every_traced_id_is_defined(): + cov = et.compute_coverage(["AR-001", "AR-002"], COVERAGE_REGISTER) + assert cov.orphaned == [] + + +def test_test_and_system_ids_are_a_separate_taxonomy(): + # UT/IT live in their own register section; PR/SR live in the umbrella + # SPEC.md, which is not part of this repo's checkout. Neither counts toward + # coverage, and flagging them as orphans would bury real typos in noise. + cov = et.compute_coverage( + ["AR-001", "UT-003", "IT-007", "SR-002", "PR-001"], COVERAGE_REGISTER) + assert cov.orphaned == [] + assert cov.covered == ["AR-001"] + + +def test_a_gpu_only_requirement_is_tagged_but_unexecuted_not_covered(): + # The rule specific to this repo: CI is an Intel N100 with no dGPU. A test + # that exists but can never run is not evidence, and counting it is the + # same failure mode as the 158% bug. + cov = et.compute_coverage(["AR-001", "AR-027"], COVERAGE_REGISTER) + assert cov.unexecuted == ["AR-027"] + assert cov.covered == ["AR-001"] + assert cov.percent == 25.0 + assert cov.orphaned == [] + + +def test_ci_scope_percentage_excludes_unexecutable_requirements_from_both_sides(): + cov = et.compute_coverage(["AR-001"], COVERAGE_REGISTER) + assert cov.ci_executable_total == 3 + assert round(cov.ci_percent) == 33 + + +def test_tracing_only_gpu_only_requirements_yields_zero_coverage(): + cov = et.compute_coverage(["AR-027"], COVERAGE_REGISTER) + assert cov.covered == [] + assert cov.percent == 0.0 + assert cov.unexecuted == ["AR-027"] + + +def test_duplicate_traced_ids_are_counted_once(): + cov = et.compute_coverage(["AR-001", "AR-001", "AR-001"], COVERAGE_REGISTER) + assert cov.covered == ["AR-001"] + + +def test_an_empty_trace_set_is_zero_percent_not_a_divide_by_zero(): + cov = et.compute_coverage([], COVERAGE_REGISTER) + assert cov.covered == [] + assert cov.percent == 0.0 + + +def test_an_empty_register_reports_zero_rather_than_nan(): + cov = et.compute_coverage(["AR-001"], et.Register()) + assert cov.percent == 0.0 + assert cov.ci_percent == 0.0 + + +def test_full_coverage_reports_exactly_one_hundred_never_above(): + cov = et.compute_coverage( + ["AR-001", "AR-002", "GR-001", "AR-027"], COVERAGE_REGISTER) + # AR-027 is GPU-only, so the honest ceiling here is 3/4. + assert cov.percent == 75.0 + assert cov.ci_percent == 100.0 + assert cov.ci_percent <= 100.0 + + +# -------------------------------------------------------------------------- +# The gate: end to end, including the ways it must fail +# -------------------------------------------------------------------------- + +FIXTURE_REGISTER = ( + "# register\n\n" + REGISTER_HEADER + + "| AR-001 | Detect faces | SR-002 | High | Done |\n" + + "| AR-002 | Minimum face size | SR-002 | High | Planned |\n" + + "| AR-027 | Arbitrary gallery scale | SR-001 | High | Planned |\n" + + "\n" + + _tier_table("| AR-001, AR-002 | T2 | replay |\n" + "| AR-027 | **T4** | GPU host only |\n")) + + +def _fixture_repo(tmp: str, source: str) -> Path: + root = Path(tmp) + (root / "docs").mkdir(parents=True, exist_ok=True) + (root / "src").mkdir(parents=True, exist_ok=True) + (root / "docs" / "requirements.md").write_text(FIXTURE_REGISTER, encoding="utf-8") + (root / "src" / "pipeline.cpp").write_text(source, encoding="utf-8") + return root + + +def _run_gate(root: Path, *extra: str): + buffer = io.StringIO() + with redirect_stdout(buffer): + code = et.main(["--root", str(root), "--format", "coverage", *extra]) + return code, buffer.getvalue() + + +def test_gate_passes_at_the_default_threshold_with_no_tags_yet(): + # Near-zero coverage on a fresh tree is the correct reading, not a failure. + with tempfile.TemporaryDirectory() as tmp: + root = _fixture_repo(tmp, "int main() { return 0; }\n") + code, out = _run_gate(root) + assert code == 0, out + assert "0 / 3 (0.0%)" in out + + +def test_gate_fails_below_an_explicit_threshold(): + # Proves the gate can fail at all. A gate nobody has watched fail is not + # known to work. + with tempfile.TemporaryDirectory() as tmp: + root = _fixture_repo(tmp, f"// {TAG} AR-001\nint main() {{ return 0; }}\n") + code, out = _run_gate(root, "--min-coverage", "99") + assert code == 1 + assert "below the minimum" in out + + +def test_gate_fails_on_an_orphan_tag(): + with tempfile.TemporaryDirectory() as tmp: + root = _fixture_repo(tmp, f"// {TAG} AR-001, AR-404\nint main() {{}}\n") + code, out = _run_gate(root) + assert code == 1 + assert "AR-404" in out + assert "orphan tag" in out + + +def test_gate_can_be_asked_to_report_orphans_without_failing(): + with tempfile.TemporaryDirectory() as tmp: + root = _fixture_repo(tmp, f"// {TAG} AR-001, AR-404\nint main() {{}}\n") + code, out = _run_gate(root, "--allow-orphans") + assert code == 0 + assert "AR-404" in out + + +def test_gate_fails_when_the_register_parses_to_nothing(): + # With a low threshold this self-check is what keeps the gate meaningful: + # a broken parser would otherwise report 0/0 and pass. + with tempfile.TemporaryDirectory() as tmp: + root = _fixture_repo(tmp, "int main() {}\n") + (root / "docs" / "requirements.md").write_text( + "# register\n\nNo tables here.\n", encoding="utf-8") + code, out = _run_gate(root) + assert code == 1 + assert "ZERO requirements" in out + + +def test_gate_fails_when_no_source_files_were_scanned(): + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + (root / "docs").mkdir(parents=True) + (root / "docs" / "requirements.md").write_text(FIXTURE_REGISTER, + encoding="utf-8") + code, out = _run_gate(root) + assert code == 1 + assert "no source files were scanned" in out + + +def test_gate_reports_a_gpu_only_requirement_as_tagged_but_unexecuted(): + with tempfile.TemporaryDirectory() as tmp: + root = _fixture_repo(tmp, f"// {TAG} AR-027\nint main() {{}}\n") + code, out = _run_gate(root) + assert code == 0 + assert "TAGGED BUT UNEXECUTED" in out + assert "AR-027 (tier T4)" in out + assert "0 / 3 (0.0%)" in out + + +def test_gate_hard_fails_on_an_impossible_ratio(): + # Coverage above 100% cannot happen through the intersection, which is the + # point: if it ever does, the arithmetic is broken and the run must not be + # reported as a pass. Forced here by handing the reporter a poisoned value. + with tempfile.TemporaryDirectory() as tmp: + root = _fixture_repo(tmp, f"// {TAG} AR-001\nint main() {{}}\n") + register = et.read_register(root / "docs" / "requirements.md") + scan = et.scan_files(et.iter_source_files(root), root) + report = et.build_report(root, register, scan) + report.coverage.percent = 158.0 + text, code = et.format_coverage_report(report, 50.0) + assert code == 1 + assert "exceeds 100%" in text + + +def test_the_per_type_breakdown_sums_to_the_headline_figure(): + # A breakdown that does not add up to its own total is how a wrong number + # survives review: every row looks plausible on its own. + with tempfile.TemporaryDirectory() as tmp: + root = _fixture_repo( + tmp, f"// {TAG} AR-001, AR-027\nint main() {{}}\n") + register = et.read_register(root / "docs" / "requirements.md") + scan = et.scan_files(et.iter_source_files(root), root) + report = et.build_report(root, register, scan) + stats = et.per_type_stats(report) + assert sum(c for c, _, _ in stats.values()) == len(report.coverage.covered) + assert sum(u for _, u, _ in stats.values()) == len(report.coverage.unexecuted) + assert sum(d for _, _, d in stats.values()) == report.coverage.total + assert stats["AR"] == (1, 1, 3) + + +def test_json_and_markdown_outputs_are_written_and_consistent(): + import json + with tempfile.TemporaryDirectory() as tmp: + root = _fixture_repo(tmp, f"// {TAG} AR-001 | SR-002\nint main() {{}}\n") + json_out = root / "traces-report.json" + md_out = root / "docs" / "traceability.md" + buffer = io.StringIO() + with redirect_stdout(buffer): + code = et.main(["--root", str(root), "--format", "coverage", + "--json-out", str(json_out), + "--markdown-out", str(md_out)]) + assert code == 0 + data = json.loads(json_out.read_text(encoding="utf-8")) + assert data["defined"]["total"] == 3 + assert data["coverage"]["covered"] == 1 + assert data["coverage"]["percent"] == round(100 / 3, 1) + assert data["byType"]["SR"] == ["SR-002"] + assert data["gpuOnlyRequirements"] == ["AR-027"] + assert "AR-001" in md_out.read_text(encoding="utf-8") + + +def test_system_spec_parsing_enables_orphan_checks_for_pr_and_sr(): + spec = ("### SR-002 - Presence is scene-scoped\n\n" + "| ID | Goal | Why |\n|---|---|---|\n" + "| **PR-001** | Show which actors are on screen | The product |\n") + ids = et.parse_system_spec(spec) + assert ids == {"SR-002", "PR-001"} + + with tempfile.TemporaryDirectory() as tmp: + root = _fixture_repo(tmp, f"// {TAG} AR-001 | SR-999\nint main() {{}}\n") + spec_path = root / "system-spec.md" + spec_path.write_text(spec, encoding="utf-8") + code, out = _run_gate(root, "--system-spec", str(spec_path)) + assert "SR-999" in out + assert code == 0 # advisory: the system register is not this repo's + + +# -------------------------------------------------------------------------- +# The live register — structural assertions only, so this does not churn as +# requirements are added. +# -------------------------------------------------------------------------- + +def test_the_live_register_parses_and_assigns_tiers(): + register = et.read_register(et.REPO_ROOT / "docs" / "requirements.md") + assert register.total > 0 + for req_type in et.LOCAL_TYPES: + assert register.count(req_type) > 0, req_type + assert sum(register.count(t) for t in et.LOCAL_TYPES) == register.total + # The GPU-less CI host must be visible in the parse, not just in prose. + assert "AR-027" in register.unexecutable_ids() + assert register.requirements["AR-027"].tiers == {"T4"} + assert register.requirements["AR-012"].tiers == {"T2"} + assert register.unexecutable_ids() < register.ids + + +def test_the_live_register_yields_a_gate_run_that_cannot_exceed_one_hundred(): + register = et.read_register(et.REPO_ROOT / "docs" / "requirements.md") + files = et.iter_source_files(et.REPO_ROOT) + scan = et.scan_files(files, et.REPO_ROOT) + cov = et.compute_coverage( + [i for t in scan.traces for i in t.requirements], register) + assert 0.0 <= cov.percent <= 100.0 + assert len(cov.covered) <= cov.total + + +# -------------------------------------------------------------------------- + +def _main() -> int: + tests = [(name, obj) for name, obj in sorted(globals().items()) + if name.startswith("test_") and callable(obj)] + failed = [] + for name, fn in tests: + try: + fn() + except Exception as exc: # noqa: BLE001 - a test runner reports everything + failed.append((name, exc)) + print(f"FAIL {name}: {type(exc).__name__}: {exc}") + else: + print(f"ok {name}") + print(f"\n{len(tests) - len(failed)}/{len(tests)} passed") + return 1 if failed else 0 + + +if __name__ == "__main__": + sys.exit(_main()) diff --git a/scripts/traceability/traceability-gate.sh b/scripts/traceability/traceability-gate.sh new file mode 100755 index 0000000..6c8bef7 --- /dev/null +++ b/scripts/traceability/traceability-gate.sh @@ -0,0 +1,64 @@ +#!/bin/sh +# +# Requirement traceability gate. Run locally exactly as CI runs it: +# +# scripts/traceability/traceability-gate.sh +# +# Writes traces-report.json and docs/traceability.md, prints the coverage +# report, and exits non-zero when the gate fails. +# +# Environment: +# MIN_COVERAGE minimum overall coverage percent (default 0 - see below) +# ALLOW_ORPHANS set to 1 to report orphan tags without failing +# TRACES_JSON JSON report path (default traces-report.json) +# TRACES_MD markdown matrix path (default docs/traceability.md) +# SYSTEM_SPEC optional path to the umbrella SPEC.md, which defines the +# PR/SR IDs; when given, PR/SR orphans are reported too. That +# file lives in the parent project, not in this repo, so CI +# normally leaves it unset. +# +# Threshold policy lives here and nowhere else. It is deliberately NOT +# duplicated into the workflow YAML: a threshold written in two places is a +# threshold that will disagree with itself. +# +# MIN_COVERAGE defaults to 0 because almost nothing is tagged yet - tags are +# added as the pipeline is built, so a low number today is accurate rather than +# alarming. A zero threshold does NOT mean the gate cannot fail: orphan tags, +# a >100% ratio, a register that parses to nothing, and an empty source scan +# are all hard failures from day one. Raise MIN_COVERAGE as tags land; treat +# every raise as a ratchet, never a reset. +# +# POSIX sh, no bashisms, no jq - the extractor does its own arithmetic and +# printing so CI needs nothing beyond python3. + +set -eu + +SCRIPT_DIR=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd) +REPO_ROOT=$(CDPATH= cd -- "$SCRIPT_DIR/../.." && pwd) + +MIN_COVERAGE="${MIN_COVERAGE:-0}" +TRACES_JSON="${TRACES_JSON:-$REPO_ROOT/traces-report.json}" +TRACES_MD="${TRACES_MD:-$REPO_ROOT/docs/traceability.md}" + +PYTHON="${PYTHON:-python3}" +command -v "$PYTHON" >/dev/null 2>&1 || { + echo "FAILED: $PYTHON not found. The traceability gate needs Python 3.9+" >&2 + exit 2 +} + +set -- \ + --root "$REPO_ROOT" \ + --format coverage \ + --json-out "$TRACES_JSON" \ + --markdown-out "$TRACES_MD" \ + --min-coverage "$MIN_COVERAGE" + +if [ "${ALLOW_ORPHANS:-0}" = "1" ]; then + set -- "$@" --allow-orphans +fi + +if [ -n "${SYSTEM_SPEC:-}" ]; then + set -- "$@" --system-spec "$SYSTEM_SPEC" +fi + +exec "$PYTHON" "$SCRIPT_DIR/extract_traces.py" "$@"