diff --git a/.claude/settings.local.json.tmp.271576.c40f99ae1801 b/.claude/settings.local.json.tmp.271576.c40f99ae1801 new file mode 100644 index 0000000..b82640d --- /dev/null +++ b/.claude/settings.local.json.tmp.271576.c40f99ae1801 @@ -0,0 +1,107 @@ +{ + "permissions": { + "allow": [ + "Bash(python3 -c ' *)", + "WebFetch(domain:docs.turso.tech)", + "Bash(awk '/^```/{n++} END{print \"fences:\",n,\\(n%2==0?\"balanced\":\"UNBALANCED\"\\)}' SPEC.md)", + "Bash(python3 -c \"import h5py; print\\('h5py', h5py.__version__\\)\")", + "Bash(python3 scripts/make_jellyfin_gallery.py --help)", + "Bash(timeout 30 python3 -c ' *)", + "Bash(ps -o pid,etime,cmd -C cmake)", + "Bash(cargo check *)", + "Bash(cargo test *)", + "Bash(cargo clippy *)", + "Bash(cargo deny *)", + "Bash(rustc --version)", + "Bash(cargo metadata *)", + "Bash(python3 -c \"import json,sys; d=json.load\\(sys.stdin\\); print\\(d['packages'][0].get\\('license'\\)\\)\")", + "Bash(timeout 900 cargo install cargo-deny --locked)", + "Bash(timeout 1500 docker build -t jray-server:test .)", + "Bash(timeout 900 docker build -f /tmp/claude-1000/-home-dtourolle-Development-Jray-project/d861f764-2add-4f42-a069-0954ad1f9565/scratchpad/Dockerfile.probe -t jray-probe .)", + "Bash(docker rm *)", + "Bash(docker volume *)", + "Bash(timeout 900 docker build -t jray-server:test .)", + "Bash(docker run *)", + "Bash(xargs -I{} echo \"clippy: {}\")", + "Bash(timeout 900 docker build -q -t jray-server:test .)", + "Bash(awk '{s+=$4} END {print \"total: \" s \" passing\"}')", + "Bash(xargs -I{} echo \"clippy issues: {}\")", + "Bash(timeout 600 dotnet build Jellyfin.Plugin.JRay/Jellyfin.Plugin.JRay.csproj -v q --nologo)", + "Bash(awk '{s+=$4} END {print \"server tests: \" s \" passing\"}')", + "Bash(timeout 300 dotnet build Jellyfin.Plugin.JRay/Jellyfin.Plugin.JRay.csproj -v q --nologo)", + "Bash(cp SPEC.md /tmp/claude-1000/-home-dtourolle-Development-Jray-project/de7e497c-da04-462d-a971-d0fcd36bda2a/scratchpad/SPEC.md.bak)", + "Bash(perl -pi -e ' *)", + "Bash(grep -vE \"AR-|DP-|IR-|GR-|VR-|SR-|PR-|H[0-9]|[0-9]{3,}\")", + "Bash(git update-index *)", + "WebFetch(domain:github.com)", + "Bash(git -C /home/dtourolle/Development/Jray-worktrees/audio-signature status --short)", + "Bash(git -C /home/dtourolle/Development/Jray-worktrees/audio-signature branch --show-current)", + "Bash(python3 -c \"import pytest; print\\(pytest.__version__\\)\")", + "Bash(python3 -c \"import numpy; print\\('numpy', numpy.__version__\\)\")", + "Read(//usr/lib/cmake/**)", + "Read(//usr/include/**)", + "Bash(grep -n \"^#\\\\{1,3\\\\} \" JRay-public-server/SPEC.md)", + "Bash(grep -n \"^#\\\\{1,3\\\\} \" scene-actor-extraction/docs/SPEC.md)", + "Bash(cargo build *)", + "Bash(python3 gen.py tone.wav)", + "Bash(python3 ref.py tone.wav)", + "Bash(python3 -c \"import json;d=json.load\\(open\\('package.json'\\)\\);print\\(json.dumps\\({k:v for k,v in d.get\\('scripts',{}\\).items\\(\\) if 'trace' in k.lower\\(\\)},indent=2\\)\\)\")", + "Bash(python3 scripts/traceability/extract_traces.py --format json)", + "Bash(python3 scripts/test_extract_traces.py)", + "Bash(python3 -m pytest scripts/traceability/test_extract_traces.py -q)", + "Bash(python3 scripts/traceability/extract_traces.py --format coverage)", + "Bash(python3 scripts/extract_traces.py)", + "Bash(python3 scripts/extract_traces.py --format json)", + "Bash(git -C /home/dtourolle/Development/Jray-worktrees/traceability-tooling log --oneline -3)", + "Bash(python3 /home/dtourolle/Development/Jray-worktrees/traceability-tooling/scripts/traceability/extract_traces.py --root /home/dtourolle/Development/Jray-project/JRay-public-server --requirements /home/dtourolle/Development/Jray-project/JRay-public-server/docs/requirements.md --system-spec /home/dtourolle/Development/Jray-project/SPEC.md --format coverage)", + "Bash(python3 scripts/traceability/test_extract_traces.py)", + "Bash(sh scripts/traceability/traceability-gate.sh)", + "Bash(python3 -c \"import pyflakes; print\\('pyflakes', pyflakes.__version__\\)\")", + "Bash(timeout 300 dotnet build -v q --nologo)", + "Bash(git check-ignore *)", + "Bash(rm -rf /tmp/claude-1000/-home-dtourolle-Development-Jray-project/de7e497c-da04-462d-a971-d0fcd36bda2a/scratchpad/build-full)", + "Bash(cmake -S . -B /tmp/claude-1000/-home-dtourolle-Development-Jray-project/de7e497c-da04-462d-a971-d0fcd36bda2a/scratchpad/build-full -DSAE_BUILD_TESTS=ON)", + "Bash(pkg-config --cflags opencv4)", + "Bash(pkg-config --cflags opencv)", + "Bash(awk '{s+=$4} END {print \"tests: \" s}')", + "Bash(python3 -m pyflakes scripts/validation/min_face_size.py)", + "Bash(python3 -m flake8 --select=F scripts/validation/min_face_size.py)", + "Bash(git -c user.name=\"Duncan Tourolle\" -c user.email=\"duncan@tourolle.paris\" commit -q -F -)", + "Bash(cmake -S . -B /tmp/claude-1000/-home-dtourolle-Development-Jray-project/de7e497c-da04-462d-a971-d0fcd36bda2a/scratchpad/build-full -DSAE_BUILD_TESTS=ON -DSAE_GEMM_BACKEND=CPU)", + "Bash(python3 -c \"import h5py, numpy, requests, PIL; print\\('deps ok'\\)\")", + "Bash(python3 -m py_compile scripts/validation/min_face_size.py)", + "Bash(git remote *)", + "Bash(python3 et.py --root /home/dtourolle/Development/Jray-project/jRay --requirements /home/dtourolle/Development/Jray-project/jRay/docs/requirements.md --system-spec /home/dtourolle/Development/Jray-project/SPEC.md --format coverage)", + "Bash(git -C /home/dtourolle/Development/Jray-worktrees/traceability-tooling status --short)", + "Bash(git -C /home/dtourolle/Development/Jray-worktrees/traceability-tooling log --oneline -2 -- scripts/traceability/)", + "Bash(timeout 240 python3 scripts/stamp_gallery.py --gallery /tmp/claude-1000/-home-dtourolle-Development-Jray-project/de7e497c-da04-462d-a971-d0fcd36bda2a/scratchpad/g_plain.h5 --show)", + "Bash(cp /tmp/claude-1000/-home-dtourolle-Development-Jray-project/de7e497c-da04-462d-a971-d0fcd36bda2a/scratchpad/g_plain.h5 /tmp/claude-1000/-home-dtourolle-Development-Jray-project/de7e497c-da04-462d-a971-d0fcd36bda2a/scratchpad/g_mig.h5)", + "Bash(python3 scripts/stamp_gallery.py --gallery /tmp/claude-1000/-home-dtourolle-Development-Jray-project/de7e497c-da04-462d-a971-d0fcd36bda2a/scratchpad/g_mig.h5 --arcface /tmp/claude-1000/-home-dtourolle-Development-Jray-project/de7e497c-da04-462d-a971-d0fcd36bda2a/scratchpad/fake.onnx)", + "Bash(timeout 590 cmake -S . -B /tmp/claude-1000/-home-dtourolle-Development-Jray-project/de7e497c-da04-462d-a971-d0fcd36bda2a/scratchpad/bt -DSAE_BUILD_TESTS=ON -DCMAKE_BUILD_TYPE=Release)", + "Bash(grep -n \"^#\\\\{1,3\\\\} \\\\|^| ID \\\\|^| Requirement \\\\|^|---\" jRay/docs/requirements.md)", + "Bash(./build-full/tests/sae_tests)", + "Bash(ls /usr/include/catch2/catch_test_macros.hpp 2>/dev/null && echo \"catch2 system\"; ls /tmp/claude-1000/-home-dtourolle-Development-Jray-project/de7e497c-da04-462d-a971-d0fcd36bda2a/scratchpad/build-full/_deps/ 2>/dev/null | head; ls /usr/lib/libCatch2* 2>/dev/null | head)", + "Read(//usr/lib/**)", + "Bash(find /tmp/claude-1000/-home-dtourolle-Development-Jray-project/de7e497c-da04-462d-a971-d0fcd36bda2a/scratchpad/build-full/_deps/catch2-build -name libCatch2*)", + "Bash(timeout 590 g++ -std=c++20 -O1 -I/home/dtourolle/Development/Jray-worktrees/gallery-model-binding/src -I/usr/include/opencv5 -I/tmp/claude-1000/-home-dtourolle-Development-Jray-project/de7e497c-da04-462d-a971-d0fcd36bda2a/scratchpad/build-full/_deps/nlohmann_json-src/single_include -I/tmp/claude-1000/-home-dtourolle-Development-Jray-project/de7e497c-da04-462d-a971-d0fcd36bda2a/scratchpad/build-full/_deps/catch2-src/src -I/tmp/claude-1000/-home-dtourolle-Development-Jray-project/de7e497c-da04-462d-a971-d0fcd36bda2a/scratchpad/build-full/_deps/catch2-build/generated-includes '-DSAE_MODELS_DIR=\"/home/dtourolle/Development/Jray-worktrees/gallery-model-binding/models\"' -o gallery_tests /home/dtourolle/Development/Jray-worktrees/gallery-model-binding/tests/test_gallery_store.cpp /home/dtourolle/Development/Jray-worktrees/gallery-model-binding/src/gallery/gallery_store.cpp /home/dtourolle/Development/Jray-worktrees/gallery-model-binding/src/gallery/embedder_stamp.cpp /tmp/claude-1000/-home-dtourolle-Development-Jray-project/de7e497c-da04-462d-a971-d0fcd36bda2a/scratchpad/build-full/_deps/catch2-build/src/libCatch2Main.a /tmp/claude-1000/-home-dtourolle-Development-Jray-project/de7e497c-da04-462d-a971-d0fcd36bda2a/scratchpad/build-full/_deps/catch2-build/src/libCatch2.a -lhdf5_cpp -lhdf5 -lopencv_core)", + "Bash(./gallery_tests)", + "Bash(grep -viE \"^\\\\[gallery\\\\]|^ |^$|WARNING \\\\\\(GR-004\\\\\\)|\\\\*\\\\*\\\\*\\\\*\")", + "Bash(python3 scripts/traceability/extract_traces.py --root jRay --requirements jRay/docs/requirements.md --system-spec SPEC.md --types JR --suffixes .cs,.js --scan-roots Jellyfin.Plugin.JRay --format coverage)", + "Bash(python3 scripts/traceability/extract_traces.py --root /home/dtourolle/Development/Jray-project/jRay --requirements /home/dtourolle/Development/Jray-project/jRay/docs/requirements.md --system-spec /home/dtourolle/Development/Jray-project/SPEC.md --types JR --suffixes .cs,.js --scan-roots Jellyfin.Plugin.JRay --format coverage)", + "Bash(cp -r /home/dtourolle/Development/Jray-project/scene-actor-extraction/external/KPN/. external/KPN/)", + "Bash(rm -rf external/KPN/.git)", + "Bash(rm -rf /tmp/claude-1000/-home-dtourolle-Development-Jray-project/de7e497c-da04-462d-a971-d0fcd36bda2a/scratchpad/bt)", + "Bash(timeout 590 cmake -S . -B /tmp/claude-1000/-home-dtourolle-Development-Jray-project/de7e497c-da04-462d-a971-d0fcd36bda2a/scratchpad/bt -DSAE_BUILD_TESTS=ON -DSAE_GEMM_BACKEND=CPU -DCMAKE_BUILD_TYPE=Release)", + "Bash(python3 scripts/traceability/extract_traces.py --root scene-actor-extraction --requirements scene-actor-extraction/docs/requirements.md --format coverage)", + "Bash(timeout 590 cmake --build /tmp/claude-1000/-home-dtourolle-Development-Jray-project/de7e497c-da04-462d-a971-d0fcd36bda2a/scratchpad/bt --target sae_tests -j8)", + "Bash(sh -n scripts/traceability/traceability-gate.sh)", + "Bash(cd /home/dtourolle/Development/Jray-project *)" + ], + "additionalDirectories": [ + "/home/dtourolle/Development/Jray-worktrees/gallery-model-binding/src/gallery", + "/home/dtourolle/Development/Jray-worktrees/min-face-study/scripts/validation", + "/home/dtourolle/Development/Jray-worktrees/audio-signature/tests", + "/home/dtourolle/Development/Jray-worktrees/traceability-tooling/scripts/traceability" + ] + } +} diff --git a/scripts/traceability/extract_traces.py b/scripts/traceability/extract_traces.py new file mode 100755 index 0000000..0b946d0 --- /dev/null +++ b/scripts/traceability/extract_traces.py @@ -0,0 +1,1299 @@ +#!/usr/bin/env python3 +"""Extract requirement traces from C++/Python sources and report coverage. + +Ported from JellyTau's ``scripts/extract-traces.ts``. That one scans +TypeScript/Svelte/Rust; this repo is C++ and Python, so the scanner is Python +with no third-party dependencies — the CI host must not need a node/bun +toolchain to check traceability. + +Tag format (see ../../../CLAUDE.md and SPEC.md section 6). A pipe separates +requirement *types*, a comma separates IDs within a type:: + + /// TRACES: AR-nnn, AR-mmm | SR-nnn + struct TrackRegistry { ... }; + +Deliberate invariant exceptions carry their own tag and are reported +separately — never silently folded into coverage:: + + // EXCEPTION: AR-nnn distributional check, not a match decision + +Usage:: + + python3 scripts/traceability/extract_traces.py --format coverage + python3 scripts/traceability/extract_traces.py --format json > traces-report.json + python3 scripts/traceability/extract_traces.py --format markdown --markdown-out docs/traceability.md + +The CI gate is ``scripts/traceability/traceability-gate.sh``, which wraps this. + +Two rules inherited from JellyTau's gate repair, both learned the hard way +(JellyTau/docs/specs/traceability-gate-repair.md): + +1. Coverage denominators are read out of ``docs/requirements.md`` at run time. + Never hardcode them. JellyTau's gate divided by frozen literals while the + register grew to 211 requirements; it reported 158% coverage, so its 50% + threshold could never trip. A gate that cannot fail is worse than no gate, + because it is trusted. +2. Coverage above 100% is a hard failure, not a pass. It means the computation + is broken, and it is the signal that would have caught (1) immediately. + +One rule specific to this repo: CI runs on an Intel N100 with no discrete GPU. +Requirements whose only verification tier is T4/GPU (or "out of CI") cannot +execute here. They are reported as *tagged but unexecuted* and are excluded +from the covered numerator — counting a test that never runs as coverage is +the same failure mode as the 158% bug. +""" + +from __future__ import annotations + +import argparse +import json +import re +import sys +from dataclasses import dataclass, field +from datetime import datetime, timezone +from pathlib import Path +from typing import Dict, Iterable, List, Optional, Sequence, Set, Tuple + +# Derived from this file's location so a CI checkout at any path works. +SCRIPT_DIR = Path(__file__).resolve().parent +REPO_ROOT = SCRIPT_DIR.parent.parent + +# -------------------------------------------------------------------------- +# Taxonomy +# +# These are *defaults*, not fixed policy. This tool is shared by every component +# in the project (it is vendored in as a submodule from `jray-project`), and the +# components differ in both their requirement prefixes and their languages: +# +# scene-actor-extraction AR/DP/IR/GR/VR C++, Python +# JRay-public-server UR/DR Rust +# jRay UR/DR C# +# +# `--types` and `--suffixes` override the two that vary. Leaving them hardcoded +# is not a small problem: a repo whose prefixes are absent parses to **zero** +# defined requirements, and one whose language is absent scans **zero** files. +# The gate refuses to report coverage in either state rather than printing a +# misleading 0%, which is the behaviour that surfaced this. +# -------------------------------------------------------------------------- + +#: Types defined in docs/requirements.md. These, and only these, participate +#: in the coverage fraction. Override with --types. +LOCAL_TYPES: Tuple[str, ...] = ("AR", "DP", "IR", "GR", "VR") + +#: Test identifiers. A separate taxonomy: a test ID is evidence *for* a +#: requirement, not a requirement. Excluded from the fraction. +TEST_TYPES: Tuple[str, ...] = ("UT", "IT") + +#: Defined in the system-level SPEC.md, which lives in the project home +#: (`jray-project`) rather than in any component checkout. Recognised in tags, +#: never counted here, and orphan-checked only when --system-spec is given. +EXTERNAL_TYPES: Tuple[str, ...] = ("PR", "SR") + +KNOWN_TYPES: Tuple[str, ...] = LOCAL_TYPES + TEST_TYPES + EXTERNAL_TYPES + +#: Tiers a GPU-less CI host can actually run. See the "Verification strategy" +#: section of docs/requirements.md. +CI_EXECUTABLE_TIERS: Set[str] = {"T1", "T2", "T3", "static"} + +# -------------------------------------------------------------------------- +# Source scanning +# -------------------------------------------------------------------------- + +#: Override with --suffixes. Defaults cover the extraction pipeline; the server +#: passes `.rs` and the plugin `.cs`. +SOURCE_SUFFIXES = { + ".cpp", ".cc", ".cxx", ".hpp", ".hxx", ".h", ".cu", ".cuh", # C++ + ".py", # Python +} + +SCAN_ROOTS: Tuple[str, ...] = ("src", "tests", "scripts", "experiments", "eval") + + +def configure_taxonomy(local_types: Sequence[str]) -> None: + """Rebind the requirement prefixes this run recognises. + + Module-level rather than threaded through every call site: the constants are + read in a dozen places, and adding a config parameter to each would be a far + larger diff than the change warrants — more risk of a missed site than the + global buys. Called once from main() before any scanning. + """ + global LOCAL_TYPES, KNOWN_TYPES + LOCAL_TYPES = tuple(local_types) + KNOWN_TYPES = LOCAL_TYPES + TEST_TYPES + EXTERNAL_TYPES + + +def configure_suffixes(suffixes: Sequence[str]) -> None: + """Rebind the file extensions scanned for TRACES tags.""" + global SOURCE_SUFFIXES + SOURCE_SUFFIXES = {s if s.startswith(".") else f".{s}" for s in suffixes} + + +def configure_scan_roots(roots: Sequence[str]) -> None: + """Rebind the directories walked, relative to --root.""" + global SCAN_ROOTS + SCAN_ROOTS = tuple(roots) + +EXCLUDED_DIR_NAMES = { + ".git", "__pycache__", "external", "build", "site", "node_modules", + "trt_cache", "ort_cache", ".venv", "venv", ".mypy_cache", ".pytest_cache", +} + +TRACES_RE = re.compile(r"TRACES:[ \t]*([^\n]*)") +EXCEPTION_RE = re.compile(r"EXCEPTION:[ \t]*([A-Z]{2}-\d{3})[ \t]*([^\n]*)") +REQ_ID_RE = re.compile(r"^([A-Z]{2})-(\d{3})$") +LEADING_ID_RE = re.compile(r"^([A-Z]{2}-\d{3})\b(.*)$") +#: Something that was *trying* to be a requirement ID. Used to keep the +#: malformed-tag diagnostic quiet when the word "TRACES" merely appears in +#: prose or in this tool's own source, while still catching `AR-12`. +ID_ATTEMPT_RE = re.compile(r"[A-Za-z]{2}-\d") + +# Comment terminators that can trail a tag on the same line. +COMMENT_TERMINATORS = ("*/", "-->", '"""', "'''") + +DECL_PATTERNS = [ + re.compile(r"^\s*(?:async\s+)?def\s+\w+"), + re.compile(r"^\s*class\s+\w+"), + re.compile(r"^\s*(?:template\s*<[^;]*>\s*)?" + r"(?:struct|class|enum(?:\s+class)?|union|namespace)\s+\w+"), + re.compile(r"^\s*(?:static|inline|constexpr|virtual|explicit|friend)\b"), + re.compile(r"^\s*[A-Za-z_][\w:<>,\s\*&]*\s+[A-Za-z_~][\w:]*\s*\([^)]*\)"), +] + + +@dataclass +class TraceEntry: + file: str + line: int + context: str + requirements: List[str] + + def to_dict(self) -> dict: + return { + "file": self.file, + "line": self.line, + "context": self.context, + "requirements": list(self.requirements), + } + + +@dataclass +class ExceptionEntry: + file: str + line: int + requirement: str + reason: str + context: str + + def to_dict(self) -> dict: + return { + "file": self.file, + "line": self.line, + "requirement": self.requirement, + "reason": self.reason, + "context": self.context, + } + + +@dataclass +class Diagnostics: + """Things wrong with the tags themselves, kept visible rather than dropped.""" + + malformed_tags: List[dict] = field(default_factory=list) + mixed_type_groups: List[dict] = field(default_factory=list) + unknown_id_types: List[dict] = field(default_factory=list) + exceptions_without_reason: List[dict] = field(default_factory=list) + + def to_dict(self) -> dict: + return { + "malformedTags": self.malformed_tags, + "mixedTypeGroups": self.mixed_type_groups, + "unknownIdTypes": self.unknown_id_types, + "exceptionsWithoutReason": self.exceptions_without_reason, + } + + @property + def empty(self) -> bool: + return not (self.malformed_tags or self.mixed_type_groups + or self.unknown_id_types or self.exceptions_without_reason) + + +def parse_traces_tag(value: str) -> Tuple[List[List[str]], List[str]]: + """Parse the text after ``TRACES:`` into per-type groups plus junk. + + Returns ``(groups, junk)``. ``groups`` is one list of IDs per pipe-separated + segment, preserving the type grouping the format promises. ``junk`` holds + anything that was not a bare ID, so a tag like ``AR-001 - see also AR-999`` + yields ``AR-001`` and reports the rest rather than quietly harvesting an ID + out of prose. + """ + text = value + for term in COMMENT_TERMINATORS: + idx = text.find(term) + if idx != -1: + text = text[:idx] + text = text.strip() + if not text: + return [], [] + + groups: List[List[str]] = [] + junk: List[str] = [] + for raw_group in text.split("|"): + ids: List[str] = [] + for raw_item in raw_group.split(","): + item = raw_item.strip() + if not item: + continue + if REQ_ID_RE.match(item): + ids.append(item) + continue + lead = LEADING_ID_RE.match(item) + if lead: + ids.append(lead.group(1)) + remainder = lead.group(2).strip() + if remainder: + junk.append(remainder) + else: + junk.append(item) + if ids: + groups.append(ids) + return groups, junk + + +def find_context(lines: Sequence[str], index: int, window: int = 12) -> str: + """Best-effort name of the declaration a tag belongs to. + + Searches both directions, because the two languages put the tag on opposite + sides of the thing it describes: a C++ ``/// TRACES:`` sits *above* the + declaration, while a Python tag usually sits *inside* the docstring, below + the ``def``. Whichever declaration is nearer wins. + """ + def matches(line: str) -> bool: + return any(p.match(line) for p in DECL_PATTERNS) + + down: Optional[Tuple[int, str]] = None + for offset in range(1, window + 1): + i = index + offset + if i >= len(lines): + break + if matches(lines[i]): + down = (offset, lines[i]) + break + + up: Optional[Tuple[int, str]] = None + for offset in range(1, window + 1): + i = index - offset + if i < 0: + break + if matches(lines[i]): + up = (offset, lines[i]) + break + + best = None + if down and up: + best = down if down[0] <= up[0] else up + else: + best = down or up + if best is None: + return "Unknown" + return best[1].strip().rstrip("{").strip()[:120] or "Unknown" + + +def iter_source_files(root: Path, + scan_roots: Optional[Iterable[str]] = None) -> List[Path]: + """Every source file, of the configured suffixes, under the scan roots. + + `scan_roots` defaults to `None` rather than to `SCAN_ROOTS` directly: a + default argument is bound once at import, so it would capture the value from + before `configure_scan_roots()` ran and silently ignore the override. + """ + found: List[Path] = [] + for rel in (SCAN_ROOTS if scan_roots is None else scan_roots): + base = root / rel + if not base.is_dir(): + continue + for path in sorted(base.rglob("*")): + if not path.is_file(): + continue + if path.suffix not in SOURCE_SUFFIXES: + continue + if any(part in EXCLUDED_DIR_NAMES for part in path.parts): + continue + found.append(path) + return found + + +@dataclass +class ScanResult: + files: List[Path] + traces: List[TraceEntry] + exceptions: List[ExceptionEntry] + diagnostics: Diagnostics + + +def scan_files(files: Sequence[Path], root: Path) -> ScanResult: + traces: List[TraceEntry] = [] + exceptions: List[ExceptionEntry] = [] + diags = Diagnostics() + + for path in files: + try: + content = path.read_text(encoding="utf-8", errors="replace") + except OSError: + continue + lines = content.split("\n") + rel = str(path.relative_to(root)) if path.is_relative_to(root) else str(path) + + for index, line in enumerate(lines): + exc = EXCEPTION_RE.search(line) + if exc: + reason = exc.group(2) + for term in COMMENT_TERMINATORS: + cut = reason.find(term) + if cut != -1: + reason = reason[:cut] + reason = reason.strip() + entry = ExceptionEntry( + file=rel, line=index + 1, requirement=exc.group(1), + reason=reason, context=find_context(lines, index), + ) + exceptions.append(entry) + if not reason: + # CLAUDE.md: an exception is only agreed if the reason is + # recorded. An unexplained one is an undocumented defect. + diags.exceptions_without_reason.append( + {"file": rel, "line": index + 1, + "requirement": exc.group(1)}) + continue + + match = TRACES_RE.search(line) + if not match: + continue + groups, junk = parse_traces_tag(match.group(1)) + ids = [i for g in groups for i in g] + if not ids: + # No IDs at all. Only worth reporting if something in the text + # was trying to be one — otherwise every sentence containing + # the word would be flagged, and a diagnostic nobody can act on + # is how real diagnostics get ignored. + if junk and ID_ATTEMPT_RE.search(match.group(1)): + diags.malformed_tags.append( + {"file": rel, "line": index + 1, "text": match.group(1).strip()}) + continue + if junk: + diags.malformed_tags.append( + {"file": rel, "line": index + 1, "ignored": junk}) + for group in groups: + types = {i.split("-")[0] for i in group} + if len(types) > 1: + # The pipe is what separates types; a mixed group means the + # tag does not say what it looks like it says. + diags.mixed_type_groups.append( + {"file": rel, "line": index + 1, "group": list(group)}) + for req in ids: + if req.split("-")[0] not in KNOWN_TYPES: + diags.unknown_id_types.append( + {"file": rel, "line": index + 1, "id": req}) + + # Deduplicate within one tag while preserving order. + seen: Set[str] = set() + unique = [i for i in ids if not (i in seen or seen.add(i))] + traces.append(TraceEntry(file=rel, line=index + 1, + context=find_context(lines, index), + requirements=unique)) + + return ScanResult(files=list(files), traces=traces, + exceptions=exceptions, diagnostics=diags) + + +# -------------------------------------------------------------------------- +# The register: docs/requirements.md is the authoritative denominator +# -------------------------------------------------------------------------- + +@dataclass +class Requirement: + id: str + text: str = "" + traces_to: str = "" + priority: str = "" + status: str = "" + tiers: Set[str] = field(default_factory=set) + + @property + def ci_executable(self) -> bool: + """True when at least one verification tier can run on the CI host. + + A requirement with no tier recorded is *unknown*, not unexecutable — it + is reported so the register gets fixed, and it is not penalised here. + """ + if not self.tiers: + return True + return bool(self.tiers & CI_EXECUTABLE_TIERS) + + @property + def tier_known(self) -> bool: + return bool(self.tiers) + + +@dataclass +class Register: + requirements: Dict[str, Requirement] = field(default_factory=dict) + withdrawn: Dict[str, Requirement] = field(default_factory=dict) + + @property + def ids(self) -> Set[str]: + return set(self.requirements) + + @property + def total(self) -> int: + return len(self.requirements) + + def count(self, req_type: str) -> int: + return sum(1 for i in self.requirements if i.startswith(req_type + "-")) + + def ci_executable_ids(self) -> Set[str]: + return {i for i, r in self.requirements.items() if r.ci_executable} + + def unexecutable_ids(self) -> Set[str]: + return {i for i, r in self.requirements.items() if not r.ci_executable} + + def tier_unknown_ids(self) -> Set[str]: + return {i for i, r in self.requirements.items() if not r.tier_known} + + def parentless_ids(self) -> Set[str]: + """Requirements whose ``Traces to`` cell names nothing. + + SPEC.md section 6 asks the gate to report these: a requirement serving + no stated goal is scope creep, and it is invisible unless something + looks. A cell naming a section (`§4`) counts as a parent — the point is + that *something* was recorded, not that it was an ID. + """ + out = set() + for req_id, req in self.requirements.items(): + cell = req.traces_to.replace("*", "").strip() + if not cell or cell in {"-", "—", "n/a", "N/A", "TBD"}: + out.add(req_id) + return out + + +def _row_cells(line: str) -> List[str]: + return [c.strip() for c in line.strip().strip("|").split("|")] + + +def _is_separator(cells: Sequence[str]) -> bool: + return bool(cells) and all(re.fullmatch(r":?-{3,}:?", c) for c in cells) + + +def _iter_table_rows(markdown: str): + """Yield ``(header_cells, row_cells)`` for every markdown table data row. + + A table's header is the row immediately preceding its ``|---|`` separator; + anything before that separator is not data. Any non-table line ends the + current table. + """ + header: Optional[List[str]] = None + previous: Optional[List[str]] = None + for line in markdown.split("\n"): + stripped = line.strip() + if not stripped.startswith("|"): + header = None + previous = None + continue + cells = _row_cells(stripped) + if _is_separator(cells): + header = previous + continue + if header is not None: + yield header, cells + previous = cells + + +def _is_register_header(header: Sequence[str]) -> bool: + """A definition table: first column ``ID``, and a ``Requirement`` column. + + Deliberately narrow. The per-requirement verification plan is also keyed on + ``ID`` but has no ``Requirement`` column, and the tier summary table's first + column holds comma lists and ranges — neither defines requirements, and + counting their rows would inflate the denominator. Prose mentions and + ``Traces to`` references are excluded for the same reason: a naive scan for + ``AR-\\d{3}`` over the whole file counts every reference as a definition. + """ + if not header: + return False + lowered = [c.lower() for c in header] + return lowered[0] == "id" and "requirement" in lowered + + +def _is_tier_header(header: Sequence[str]) -> bool: + """Either of the two tables that assign verification tiers.""" + if len(header) < 2: + return False + lowered = [c.lower() for c in header] + return lowered[0] in ("id", "requirement") and lowered[1] == "tier" + + +def _column(header: Sequence[str], name: str, cells: Sequence[str]) -> str: + lowered = [c.lower() for c in header] + if name in lowered: + index = lowered.index(name) + if index < len(cells): + return cells[index] + return "" + + +def parse_tiers(cell: str) -> Set[str]: + """Read a tier cell such as ``T1 + T4``, ``**T2**``, or ``Out of CI``.""" + text = cell.replace("*", "").strip().lower() + tiers = {"T" + m.group(1) for m in re.finditer(r"\bt([1-4])\b", text)} + if "out of ci" in text or "not in ci" in text: + tiers.add("out-of-ci") + if "manual" in text: + tiers.add("manual") + if "static" in text: + tiers.add("static") + return tiers + + +def expand_id_spec(spec: str, defined: Set[str]) -> List[str]: + """Expand the ID cell of a tier table into concrete, defined IDs. + + Handles every shape the register actually uses: ``AR-002``, + ``AR-001, AR-005, AR-006``, ``AR-007 ... AR-017`` (with the unicode + ellipsis), ``AR-009/010``, and ``DP-*``. Expansion is intersected with the + defined set, so a range can never invent a requirement that does not exist. + """ + text = spec.replace("*", "").strip() + # `DP-*` survives the bold-strip above as a bare `DP-`; both mean "every + # requirement of this type". + if re.fullmatch(r"[A-Z]{2}-", text): + prefix = text[:2] + return sorted(i for i in defined if i.startswith(prefix + "-")) + + out: List[str] = [] + for part in text.split(","): + part = part.strip() + if not part: + continue + rng = re.fullmatch(r"([A-Z]{2})-(\d{3})\s*(?:…|\.\.\.)\s*([A-Z]{2})-(\d{3})", + part) + if rng and rng.group(1) == rng.group(3): + prefix = rng.group(1) + low, high = int(rng.group(2)), int(rng.group(4)) + out.extend(sorted( + i for i in defined + if i.startswith(prefix + "-") and low <= int(i.split("-")[1]) <= high)) + continue + slash = re.fullmatch(r"([A-Z]{2})-(\d{3})/(\d{3})", part) + if slash: + prefix = slash.group(1) + out.extend(f"{prefix}-{n}" for n in (slash.group(2), slash.group(3))) + continue + if REQ_ID_RE.match(part): + out.append(part) + return [i for i in out if i in defined] + + +def parse_register(markdown: str) -> Register: + """Build the register from ``docs/requirements.md``. + + Two passes: definitions first, because tier rows use ranges and wildcards + that can only be expanded against a known ID set. + """ + register = Register() + + for header, cells in _iter_table_rows(markdown): + if not _is_register_header(header) or not cells: + continue + first = cells[0].replace("*", "").strip() + if not REQ_ID_RE.match(first): + continue + if first.split("-")[0] not in LOCAL_TYPES + TEST_TYPES: + continue + req = Requirement( + id=first, + text=_column(header, "requirement", cells), + traces_to=_column(header, "traces to", cells), + priority=_column(header, "priority", cells), + status=_column(header, "status", cells), + ) + if req.status.replace("*", "").strip().lower() == "withdrawn": + # Permanently retired. Counting it in the denominator would depress + # coverage forever for something that no longer needs implementing. + register.withdrawn.setdefault(first, req) + register.requirements.pop(first, None) + continue + if first in register.withdrawn: + continue + # An ID may legitimately appear in more than one definition table; the + # first occurrence wins and duplicates never inflate the count. + register.requirements.setdefault(first, req) + + defined = register.ids + for header, cells in _iter_table_rows(markdown): + if not _is_tier_header(header) or len(cells) < 2: + continue + tiers = parse_tiers(cells[1]) + if not tiers: + continue + for req_id in expand_id_spec(cells[0], defined): + register.requirements[req_id].tiers |= tiers + + return register + + +def read_register(path: Path) -> Register: + return parse_register(path.read_text(encoding="utf-8")) + + +# -------------------------------------------------------------------------- +# Coverage +# -------------------------------------------------------------------------- + +@dataclass +class Coverage: + covered: List[str] + total: int + percent: float + orphaned: List[str] + unexecuted: List[str] + ci_executable_total: int + ci_percent: float + tier_unknown: List[str] + + def to_dict(self) -> dict: + return { + "covered": len(self.covered), + "coveredIds": self.covered, + "total": self.total, + "percent": self.percent, + "orphaned": self.orphaned, + "unexecuted": self.unexecuted, + "ciExecutableTotal": self.ci_executable_total, + "ciPercent": self.ci_percent, + "tierUnknown": self.tier_unknown, + } + + +def compute_coverage(traced_ids: Iterable[str], register: Register) -> Coverage: + """Coverage is ``|traced and defined and CI-executable| / |defined|``. + + Three exclusions, each of which is a way the number could otherwise lie: + + * Using the raw traced count as the numerator is what lets a ratio exceed + 100% — a tag naming a deleted or mistyped requirement would count as + covered. Those land in ``orphaned`` instead, so they get fixed rather + than silently counted or silently dropped. + * A requirement whose only verification tier is T4/GPU cannot run on this + CI host at all. It is reported in ``unexecuted`` and is not covered: + treating "has a test that never runs" as passing reports success the gate + cannot substantiate. + * UT/IT test IDs and the system-level PR/SR IDs are different taxonomies + with their own registers, so they neither count nor orphan here. + """ + traced = {i for i in traced_ids if i.split("-")[0] in LOCAL_TYPES} + defined = register.ids + + orphaned = sorted(traced - defined) + matched = traced & defined + unexecuted = sorted(i for i in matched if not register.requirements[i].ci_executable) + covered = sorted(matched - set(unexecuted)) + + total = len(defined) + ci_total = len(register.ci_executable_ids()) + percent = round(100.0 * len(covered) / total, 1) if total else 0.0 + ci_percent = round(100.0 * len(covered) / ci_total, 1) if ci_total else 0.0 + + return Coverage( + covered=covered, + total=total, + percent=percent, + orphaned=orphaned, + unexecuted=unexecuted, + ci_executable_total=ci_total, + ci_percent=ci_percent, + tier_unknown=sorted(register.tier_unknown_ids()), + ) + + +# -------------------------------------------------------------------------- +# Report assembly +# -------------------------------------------------------------------------- + +@dataclass +class Report: + timestamp: str + root: Path + register: Register + scan: ScanResult + coverage: Coverage + by_type: Dict[str, List[str]] + requirement_map: Dict[str, List[TraceEntry]] + external_orphans: List[str] = field(default_factory=list) + #: Policy, set by the caller: whether an orphan tag fails the run. + allow_orphans: bool = False + + +def build_report(root: Path, register: Register, scan: ScanResult, + system_ids: Optional[Set[str]] = None) -> Report: + requirement_map: Dict[str, List[TraceEntry]] = {} + seen_by_type: Dict[str, Set[str]] = {t: set() for t in KNOWN_TYPES} + seen_by_type["OTHER"] = set() + + for entry in scan.traces: + for req in entry.requirements: + requirement_map.setdefault(req, []).append(entry) + prefix = req.split("-")[0] + seen_by_type[prefix if prefix in KNOWN_TYPES else "OTHER"].add(req) + + coverage = compute_coverage(requirement_map.keys(), register) + + external_orphans: List[str] = [] + if system_ids is not None: + external_orphans = sorted( + i for i in requirement_map + if i.split("-")[0] in EXTERNAL_TYPES and i not in system_ids) + + return Report( + timestamp=datetime.now(timezone.utc).isoformat(timespec="seconds"), + root=root, + register=register, + scan=scan, + coverage=coverage, + by_type={k: sorted(v) for k, v in seen_by_type.items()}, + requirement_map=requirement_map, + external_orphans=external_orphans, + ) + + +def parse_system_spec(markdown: str) -> Set[str]: + """IDs of PR/SR requirements defined in the umbrella SPEC.md. + + They are defined as headings (``### SR-001 - ...``) and as bolded leading + table cells (``| **PR-001** | ... |``), so accept both. + """ + ids: Set[str] = set() + for line in markdown.split("\n"): + stripped = line.strip() + heading = re.match(r"^#{1,6}\s+\**((?:PR|SR)-\d{3})\**\b", stripped) + if heading: + ids.add(heading.group(1)) + continue + if stripped.startswith("|"): + cells = _row_cells(stripped) + if cells: + first = cells[0].replace("*", "").strip() + if re.fullmatch(r"(?:PR|SR)-\d{3}", first): + ids.add(first) + return ids + + +def report_to_json_dict(report: Report) -> dict: + register = report.register + defined = {t: register.count(t) for t in LOCAL_TYPES} + defined["total"] = register.total + + requirement_detail = {} + for req_id, req in sorted(register.requirements.items()): + entries = report.requirement_map.get(req_id, []) + requirement_detail[req_id] = { + "requirement": req.text, + "tracesTo": req.traces_to, + "priority": req.priority, + "status": req.status, + "tiers": sorted(req.tiers), + "ciExecutable": req.ci_executable, + "taggedIn": sorted({e.file for e in entries}), + "state": _requirement_state(req, bool(entries)), + } + + return { + "timestamp": report.timestamp, + "totalFiles": len(report.scan.files), + "totalTraces": len(report.scan.traces), + "totalExceptions": len(report.scan.exceptions), + "requirements": {k: [e.to_dict() for e in v] + for k, v in sorted(report.requirement_map.items())}, + "byType": report.by_type, + "defined": defined, + "coverage": report.coverage.to_dict(), + "gpuOnlyRequirements": sorted(register.unexecutable_ids()), + "parentlessRequirements": sorted(register.parentless_ids()), + "withdrawn": sorted(register.withdrawn), + "exceptions": [e.to_dict() for e in report.scan.exceptions], + "externalOrphans": report.external_orphans, + "diagnostics": report.scan.diagnostics.to_dict(), + "requirementDetail": requirement_detail, + } + + +def per_type_stats(report: Report) -> Dict[str, Tuple[int, int, int]]: + """``{type: (covered, unexecuted, defined)}``. + + Covered here means the same thing it means in the headline figure, so the + per-type rows sum to it. Reporting "traced and defined" per type while the + total excludes unexecuted requirements is how a breakdown quietly stops + adding up to its own total. + """ + covered = set(report.coverage.covered) + unexecuted = set(report.coverage.unexecuted) + stats: Dict[str, Tuple[int, int, int]] = {} + for req_type in LOCAL_TYPES: + prefix = req_type + "-" + stats[req_type] = ( + len([i for i in covered if i.startswith(prefix)]), + len([i for i in unexecuted if i.startswith(prefix)]), + report.register.count(req_type), + ) + return stats + + +def _requirement_state(req: Requirement, tagged: bool) -> str: + if not tagged: + return "untagged" + if not req.ci_executable: + return "tagged-unexecuted" + return "covered" + + +# -------------------------------------------------------------------------- +# Output formats +# -------------------------------------------------------------------------- + +def generate_markdown(report: Report) -> str: + register = report.register + cov = report.coverage + out: List[str] = [] + add = out.append + + add("# Requirements traceability matrix") + add("") + add("") + add("") + add("") + add(f"**Generated:** {report.timestamp}") + add("") + add("Denominators are read from [`requirements.md`](requirements.md) at run " + "time, never hardcoded. Coverage counts a requirement only when it is " + "tagged in source **and** has a verification tier this CI host can " + "execute — CI is an Intel N100 with no discrete GPU.") + add("") + + add("## Summary") + add("") + add("| Metric | Value |") + add("|---|---|") + add(f"| Source files scanned | {len(report.scan.files)} |") + add(f"| TRACES tags found | {len(report.scan.traces)} |") + add(f"| EXCEPTION tags found | {len(report.scan.exceptions)} |") + add(f"| Requirements defined | {register.total} |") + add(f"| Requirements covered | {len(cov.covered)} |") + add(f"| **Coverage** | **{cov.percent}%** ({len(cov.covered)}/{cov.total}) |") + add(f"| Coverage of CI-executable scope | {cov.ci_percent}% " + f"({len(cov.covered)}/{cov.ci_executable_total}) |") + add(f"| Tagged but unexecuted in CI (T4/GPU) | {len(cov.unexecuted)} |") + add(f"| Orphan tags | {len(cov.orphaned)} |") + add("") + + add("### By type") + add("") + add("| Type | Covered | Tagged but unexecuted | Defined |") + add("|---|---|---|---|") + for req_type, (covered, unexecuted, defined_n) in per_type_stats(report).items(): + add(f"| {req_type} | {covered} | {unexecuted} | {defined_n} |") + add("") + for req_type in TEST_TYPES + EXTERNAL_TYPES: + tagged = report.by_type.get(req_type, []) + if tagged: + add(f"- **{req_type}** tags present (separate taxonomy, not counted " + f"in coverage): {', '.join(tagged)}") + add("") + + unexecutable = sorted(register.unexecutable_ids()) + add("## Not executable in CI") + add("") + add("CI runs on an Intel N100 with no discrete GPU. These requirements have " + "no verification tier that can run here, so a tag on them is evidence " + "of *intent*, not of verification. They are never counted as covered.") + add("") + if unexecutable: + add("| ID | Tiers | Tagged in source | Requirement |") + add("|---|---|---|---|") + for req_id in unexecutable: + req = register.requirements[req_id] + tagged = "yes" if req_id in report.requirement_map else "no" + add(f"| {req_id} | {', '.join(sorted(req.tiers)) or '-'} | {tagged} " + f"| {_truncate(req.text)} |") + else: + add("_None._") + add("") + if cov.unexecuted: + add(f"**Tagged but unexecuted:** {', '.join(cov.unexecuted)} — a test " + "exists and is tagged, but only a GPU host can run it. Report " + "those runs separately.") + add("") + + add("## Orphan tags") + add("") + add("A tag naming an ID `requirements.md` does not define. This is what " + "renumbering produces, and what a typo produces.") + add("") + if cov.orphaned: + add("| ID | Locations |") + add("|---|---|") + for req_id in cov.orphaned: + where = ", ".join(f"`{e.file}:{e.line}`" + for e in report.requirement_map[req_id]) + add(f"| {req_id} | {where} |") + else: + add("_None._") + add("") + + add("## Requirements tracing up to nothing") + add("") + add("A register row whose `Traces to` cell names no parent. Work serving no " + "stated goal is how scope creeps in, and it is invisible unless " + "something looks.") + add("") + parentless = sorted(register.parentless_ids()) + if parentless: + add(", ".join(f"`{i}`" for i in parentless)) + else: + add("_None._") + add("") + + add("## Recorded exceptions") + add("") + add("Deliberate, documented departures from an invariant " + "(`EXCEPTION: AR-nnn `). Reported separately and never counted " + "as coverage — an exception is a decision to be reviewed, not evidence " + "a requirement is met.") + add("") + if report.scan.exceptions: + add("| Requirement | Location | Reason |") + add("|---|---|---|") + for exc in report.scan.exceptions: + reason = _truncate(exc.reason, 90) or "**no reason recorded**" + add(f"| {exc.requirement} | [`{exc.file}:{exc.line}`]" + f"({_source_link(exc.file)}#L{exc.line}) | {reason} |") + else: + add("_None._") + add("") + + add("## Register") + add("") + add("| ID | Status | Tier | Traces to | Trace state | Tagged in | Requirement |") + add("|---|---|---|---|---|---|---|") + for req_id, req in sorted(register.requirements.items(), + key=lambda kv: (LOCAL_TYPES.index(kv[0].split("-")[0]) + if kv[0].split("-")[0] in LOCAL_TYPES + else 99, kv[0])): + entries = report.requirement_map.get(req_id, []) + state = {"covered": "covered", + "tagged-unexecuted": "tagged, unexecuted (T4/GPU)", + "untagged": "untagged"}[ + _requirement_state(req, bool(entries))] + files = ", ".join(f"`{f}`" for f in sorted({e.file for e in entries})) or "-" + tiers = ", ".join(sorted(req.tiers)) or "unset" + add(f"| {req_id} | {_truncate(req.status, 20) or '-'} | {tiers} " + f"| {_truncate(req.traces_to, 40) or '-'} " + f"| {state} | {files} | {_truncate(req.text)} |") + add("") + + add("## Detailed mapping") + add("") + if not report.requirement_map: + add("_No TRACES tags found yet. Tags are added as code is written; an " + "empty matrix on a new tree is the correct reading, not a failure._") + add("") + for req_id in sorted(report.requirement_map): + entries = report.requirement_map[req_id] + add(f"### {req_id}") + add("") + add(f"**Locations:** {len(entries)}") + add("") + for entry in entries: + add(f"- [`{entry.file}:{entry.line}`]" + f"({_source_link(entry.file)}#L{entry.line}) — " + f"`{_truncate(entry.context, 90)}`") + add("") + + if not report.scan.diagnostics.empty: + add("## Tag diagnostics") + add("") + diags = report.scan.diagnostics + for label, items in ( + ("Malformed tags", diags.malformed_tags), + ("Groups mixing requirement types (pipe separates types)", + diags.mixed_type_groups), + ("Unrecognised ID prefixes", diags.unknown_id_types), + ("Exceptions with no recorded reason", diags.exceptions_without_reason), + ): + if items: + add(f"**{label}:**") + add("") + for item in items: + detail = {k: v for k, v in item.items() + if k not in ("file", "line")} + add(f"- `{item.get('file')}:{item.get('line')}` — {detail}") + add("") + + return "\n".join(out) + "\n" + + +def _source_link(file_rel: str) -> str: + """Link from docs/traceability.md back to a source file at the repo root.""" + return "../" + file_rel + + +def _truncate(text: str, limit: int = 70) -> str: + """Collapse to one line, cap the length, and escape table-breaking pipes.""" + text = " ".join(text.split()) + if len(text) > limit: + text = text[: limit - 1] + "…" + return text.replace("|", "\\|") + + +def format_coverage_report(report: Report, min_coverage: float) -> Tuple[str, int]: + """Human-readable gate output plus the exit code it implies.""" + register = report.register + cov = report.coverage + lines: List[str] = [] + add = lines.append + failures: List[str] = [] + + add("Requirement traceability") + add("=" * 72) + add(f"Source files scanned : {len(report.scan.files)}") + add(f"TRACES tags found : {len(report.scan.traces)}") + add(f"EXCEPTION tags found : {len(report.scan.exceptions)}") + add("") + + # Self-checks. With a low threshold these are what make the gate mean + # something: a parser that silently returns nothing would otherwise report + # 0/0 and pass. + if register.total == 0: + failures.append( + "requirements.md parsed to ZERO requirements - the register parser " + "is broken or the file moved. Refusing to report coverage.") + if not report.scan.files: + failures.append( + "no source files were scanned - the scan roots do not exist. " + "Refusing to report coverage against an empty tree.") + + add("Coverage by type (covered / defined):") + for req_type, (covered, unexecuted, defined_n) in per_type_stats(report).items(): + suffix = f" (+{unexecuted} tagged but unexecuted)" if unexecuted else "" + add(f" {req_type}: {covered} / {defined_n}{suffix}") + add("") + add(f"Overall : {len(cov.covered)} / {cov.total} ({cov.percent}%)") + add(f"CI scope : {len(cov.covered)} / {cov.ci_executable_total} " + f"({cov.ci_percent}%) [excludes {cov.total - cov.ci_executable_total} " + "requirement(s) no GPU-less host can verify]") + add("") + + unexecutable = sorted(register.unexecutable_ids()) + if unexecutable: + add(f"Not executable on this CI host (T4/GPU or out-of-CI): " + f"{len(unexecutable)}") + add(f" {', '.join(unexecutable)}") + if cov.unexecuted: + add("") + add("TAGGED BUT UNEXECUTED - a test exists and is tagged, but only a " + "GPU host can run it.") + add(" These are NOT counted as covered:") + for req_id in cov.unexecuted: + tiers = ", ".join(sorted(register.requirements[req_id].tiers)) + add(f" {req_id} (tier {tiers})") + add("") + + parentless = sorted(register.parentless_ids()) + if parentless: + add(f"WARNING: {len(parentless)} requirement(s) trace up to nothing - " + "no parent recorded in the register's `Traces to` column. Work " + "serving no stated goal is how scope creeps in:") + add(f" {', '.join(parentless)}") + add("") + + if cov.tier_unknown: + add(f"WARNING: {len(cov.tier_unknown)} requirement(s) have no " + "verification tier in requirements.md; they are counted as " + "CI-executable by default. Add them to the verification plan:") + add(f" {', '.join(cov.tier_unknown)}") + add("") + + if report.scan.exceptions: + add(f"Recorded invariant exceptions: {len(report.scan.exceptions)}") + for exc in report.scan.exceptions: + reason = exc.reason or "(NO REASON RECORDED)" + add(f" {exc.requirement} {exc.file}:{exc.line} {_truncate(reason, 80)}") + add("") + + if cov.orphaned: + add("ORPHAN TAGS - traced in source, not defined in requirements.md:") + for req_id in cov.orphaned: + where = ", ".join(f"{e.file}:{e.line}" + for e in report.requirement_map[req_id]) + add(f" {req_id} ({where})") + add(" Fix the tag, or add the requirement to the register.") + add("") + + if report.external_orphans: + add("ORPHAN SYSTEM TAGS - PR/SR IDs the system SPEC.md does not define:") + add(f" {', '.join(report.external_orphans)}") + add("") + + diags = report.scan.diagnostics + if not diags.empty: + add("Tag diagnostics:") + for item in diags.malformed_tags: + add(f" malformed {item.get('file')}:{item.get('line')} {item}") + for item in diags.mixed_type_groups: + add(f" mixed types {item['file']}:{item['line']} {item['group']} " + "(a pipe, not a comma, separates types)") + for item in diags.unknown_id_types: + add(f" unknown id {item['file']}:{item['line']} {item['id']}") + for item in diags.exceptions_without_reason: + add(f" exception without reason {item['file']}:{item['line']} " + f"{item['requirement']}") + add("") + + # A ratio above 100% means the computation is broken. This is the check + # that would have caught JellyTau's frozen denominators immediately. + if cov.percent > 100 or cov.ci_percent > 100: + failures.append( + f"coverage ({cov.percent}%) exceeds 100% - the gate is " + "miscomputing. Do not trust this run.") + if len(cov.covered) > cov.total: + failures.append( + f"covered ({len(cov.covered)}) exceeds defined ({cov.total}) - " + "the gate is miscomputing.") + + if cov.orphaned and not report.allow_orphans: + failures.append( + f"{len(cov.orphaned)} orphan tag(s): {', '.join(cov.orphaned)}") + + if cov.percent < min_coverage: + failures.append( + f"coverage ({cov.percent}%) is below the minimum ({min_coverage}%)") + + if failures: + add("FAILED:") + for reason in failures: + add(f" - {reason}") + return "\n".join(lines) + "\n", 1 + + add(f"OK: coverage {cov.percent}% >= minimum {min_coverage}%, " + f"{len(cov.orphaned)} orphan tag(s)") + return "\n".join(lines) + "\n", 0 + + +# -------------------------------------------------------------------------- +# CLI +# -------------------------------------------------------------------------- + +def build_arg_parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser( + prog="extract_traces.py", + description=__doc__.split("\n")[0], + formatter_class=argparse.RawDescriptionHelpFormatter, + ) + parser.add_argument("--format", choices=("json", "markdown", "coverage"), + default="coverage", + help="output written to stdout (default: coverage)") + parser.add_argument("--root", type=Path, default=REPO_ROOT, + help="repository root to scan (default: derived from " + "this script's location)") + parser.add_argument("--requirements", type=Path, default=None, + help="register path (default: /docs/requirements.md)") + parser.add_argument("--system-spec", type=Path, default=None, + help="optional umbrella SPEC.md defining PR/SR IDs; " + "enables orphan checking for those types") + parser.add_argument("--json-out", type=Path, default=None, + help="also write the JSON report here") + parser.add_argument("--markdown-out", type=Path, default=None, + help="also write the markdown matrix here") + parser.add_argument("--min-coverage", type=float, default=0.0, + help="minimum coverage percent; below it the run fails " + "(default: 0, i.e. the correctness checks gate but " + "the percentage does not)") + parser.add_argument("--allow-orphans", action="store_true", + help="report orphan tags without failing") + parser.add_argument("--types", default=None, metavar="UR,DR", + help="comma-separated requirement prefixes defined in " + "this repo's register (default: " + f"{','.join(LOCAL_TYPES)}). A repo whose prefixes " + "are absent parses to zero requirements") + parser.add_argument("--suffixes", default=None, metavar=".rs,.cs", + help="comma-separated file extensions to scan " + "(default: the C++/Python set). A repo whose " + "language is absent scans zero files") + parser.add_argument("--scan-roots", default=None, metavar="src,tests", + help="comma-separated directories to walk, relative to " + f"--root (default: {','.join(SCAN_ROOTS)})") + return parser + + +def _split_csv(value: Optional[str]) -> Optional[List[str]]: + if value is None: + return None + items = [v.strip() for v in value.split(",") if v.strip()] + return items or None + + +def main(argv: Optional[Sequence[str]] = None) -> int: + args = build_arg_parser().parse_args(argv) + + # Rebind the per-repo taxonomy before anything reads it. + if types := _split_csv(args.types): + configure_taxonomy(types) + if suffixes := _split_csv(args.suffixes): + configure_suffixes(suffixes) + if scan_roots := _split_csv(args.scan_roots): + configure_scan_roots(scan_roots) + + root = args.root.resolve() + req_path = args.requirements or (root / "docs" / "requirements.md") + if not req_path.is_file(): + print(f"FAILED: requirements register not found at {req_path}", + file=sys.stderr) + return 2 + + register = read_register(req_path) + files = iter_source_files(root) + scan = scan_files(files, root) + + system_ids = None + if args.system_spec: + if not args.system_spec.is_file(): + print(f"FAILED: --system-spec not found at {args.system_spec}", + file=sys.stderr) + return 2 + system_ids = parse_system_spec(args.system_spec.read_text(encoding="utf-8")) + + report = build_report(root, register, scan, system_ids) + report.allow_orphans = args.allow_orphans + + json_text = json.dumps(report_to_json_dict(report), indent=2, sort_keys=False) + markdown_text = generate_markdown(report) + + if args.json_out: + args.json_out.parent.mkdir(parents=True, exist_ok=True) + args.json_out.write_text(json_text, encoding="utf-8") + if args.markdown_out: + args.markdown_out.parent.mkdir(parents=True, exist_ok=True) + args.markdown_out.write_text(markdown_text, encoding="utf-8") + + if args.format == "json": + print(json_text) + return 0 + if args.format == "markdown": + print(markdown_text, end="") + return 0 + + text, code = format_coverage_report(report, args.min_coverage) + print(text, end="") + return code + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/traceability/test_extract_traces.py b/scripts/traceability/test_extract_traces.py new file mode 100755 index 0000000..d965e75 --- /dev/null +++ b/scripts/traceability/test_extract_traces.py @@ -0,0 +1,822 @@ +#!/usr/bin/env python3 +"""Tests for the traceability extractor and coverage gate. + +Run standalone (no third-party dependencies):: + + python3 scripts/traceability/test_extract_traces.py + +or under pytest, which discovers the same functions:: + + pytest scripts/traceability/test_extract_traces.py + +Almost every test runs over fixture strings rather than the live +``docs/requirements.md``, so their meaning does not drift as requirements are +added. The two properties they exist to pin are the ones JellyTau's gate lost +(see JellyTau/docs/specs/traceability-gate-repair.md): + +* the denominator is computed from the register at run time, so adding a + requirement lowers coverage until it is traced; +* the numerator is an intersection, so a tag naming an undefined ID cannot push + the ratio above 100%. + +Plus the rule specific to this repo: a requirement only verifiable on GPU +hardware is reported as tagged-but-unexecuted and never counted as covered. +""" + +from __future__ import annotations + +import io +import os +import sys +import tempfile +from contextlib import redirect_stdout +from pathlib import Path + +HERE = Path(__file__).resolve().parent +sys.path.insert(0, str(HERE)) + +import extract_traces as et # noqa: E402 + +# The literal tag keyword is assembled at run time so that this file's fixtures +# do not register as real traces when the extractor scans scripts/. +TAG = "TRA" + "CES:" +EXC = "EXCEP" + "TION:" + + +# -------------------------------------------------------------------------- +# Tag parsing +# -------------------------------------------------------------------------- + +def test_parses_a_single_requirement(): + groups, junk = et.parse_traces_tag(" AR-012") + assert groups == [["AR-012"]] + assert junk == [] + + +def test_parses_multiple_types_separated_by_pipe(): + # The house format: a pipe separates requirement *types*, a comma separates + # IDs within a type. The grouping is preserved, not flattened away, so a + # malformed grouping stays detectable. + groups, junk = et.parse_traces_tag(" AR-012, AR-013 | SR-002") + assert groups == [["AR-012", "AR-013"], ["SR-002"]] + assert junk == [] + + +def test_parses_three_groups_including_test_ids(): + groups, _ = et.parse_traces_tag(" AR-012 | SR-002 | UT-003, UT-004") + assert groups == [["AR-012"], ["SR-002"], ["UT-003", "UT-004"]] + + +def test_strips_a_trailing_block_comment_terminator(): + groups, junk = et.parse_traces_tag(" AR-001 */") + assert groups == [["AR-001"]] + assert junk == [] + + +def test_does_not_harvest_ids_out_of_prose_after_the_tag(): + # A trailing sentence must not smuggle IDs into the trace set: AR-999 here + # is discussion, not a claim that this code satisfies AR-999. + groups, junk = et.parse_traces_tag(" AR-001 - see also AR-999 in the notes") + assert groups == [["AR-001"]] + assert junk and "AR-999" in junk[0] + + +def test_ignores_a_tag_with_no_ids_at_all(): + groups, _ = et.parse_traces_tag(" see the register") + assert groups == [] + + +def test_two_digit_id_is_not_accepted_as_a_requirement(): + # AR-12 is a typo for AR-012; silently accepting it would create a + # phantom requirement. + groups, junk = et.parse_traces_tag(" AR-12") + assert groups == [] + assert junk == ["AR-12"] + + +# -------------------------------------------------------------------------- +# Scanning C++ and Python sources +# -------------------------------------------------------------------------- + +def _write_tree(root: Path) -> None: + (root / "src").mkdir(parents=True, exist_ok=True) + (root / "scripts").mkdir(parents=True, exist_ok=True) + (root / "external").mkdir(parents=True, exist_ok=True) + (root / "src" / "tracker.hpp").write_text( + "#pragma once\n" + f"/// {TAG} AR-012, AR-013 | SR-002\n" + "struct TrackRegistry {\n" + " void close_all();\n" + "};\n", + encoding="utf-8") + (root / "scripts" / "gallery.py").write_text( + "def build_gallery(cast):\n" + ' """Build a gallery from Jellyfin plus TMDB fallback.\n' + "\n" + f" {TAG} GR-001 | SR-005\n" + ' """\n' + " return {}\n", + encoding="utf-8") + (root / "external" / "vendored.cpp").write_text( + f"// {TAG} AR-001\n", encoding="utf-8") + (root / "src" / "notes.txt").write_text( + f"// {TAG} AR-002\n", encoding="utf-8") + + +def test_scans_cpp_and_python_but_not_vendored_or_non_source(): + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + _write_tree(root) + files = et.iter_source_files(root) + names = sorted(f.name for f in files) + assert names == ["gallery.py", "tracker.hpp"], names + + scan = et.scan_files(files, root) + traced = sorted({i for t in scan.traces for i in t.requirements}) + assert traced == ["AR-012", "AR-013", "GR-001", "SR-002", "SR-005"] + + +def test_context_is_found_below_a_cpp_tag_and_above_a_python_tag(): + # The two languages put the tag on opposite sides of what it describes. + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + _write_tree(root) + scan = et.scan_files(et.iter_source_files(root), root) + contexts = {t.file: t.context for t in scan.traces} + assert "TrackRegistry" in contexts["src/tracker.hpp"] + assert "build_gallery" in contexts["scripts/gallery.py"] + + +# -------------------------------------------------------------------------- +# EXCEPTION tags +# -------------------------------------------------------------------------- + +EXCEPTION_SOURCE = ( + "float outlier_score(const Refs& refs) {\n" + f" // {EXC} AR-024 distributional check on an actor's own references,\n" + " return spread(refs);\n" + "}\n" +) + + +def test_exception_tag_is_captured_with_its_reason(): + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + (root / "src").mkdir(parents=True) + (root / "src" / "outlier.cpp").write_text(EXCEPTION_SOURCE, encoding="utf-8") + scan = et.scan_files(et.iter_source_files(root), root) + assert len(scan.exceptions) == 1 + exc = scan.exceptions[0] + assert exc.requirement == "AR-024" + assert exc.reason.startswith("distributional check") + assert exc.line == 2 + + +def test_exception_is_never_counted_as_coverage(): + # An exception is a recorded decision to depart from an invariant. Counting + # it as evidence the requirement is met inverts its meaning. + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + (root / "src").mkdir(parents=True) + (root / "src" / "outlier.cpp").write_text(EXCEPTION_SOURCE, encoding="utf-8") + scan = et.scan_files(et.iter_source_files(root), root) + assert scan.traces == [] + register = et.parse_register( + "| ID | Requirement | Status |\n|---|---|---|\n" + "| AR-024 | Always the calibrated probability | Planned |\n") + cov = et.compute_coverage( + [i for t in scan.traces for i in t.requirements], register) + assert cov.covered == [] + assert cov.percent == 0.0 + + +def test_exception_without_a_reason_is_reported(): + # CLAUDE.md: an exception is only *agreed* if the reason is recorded. + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + (root / "src").mkdir(parents=True) + (root / "src" / "bare.cpp").write_text( + f"// {EXC} AR-024\n", encoding="utf-8") + scan = et.scan_files(et.iter_source_files(root), root) + assert len(scan.exceptions) == 1 + assert scan.diagnostics.exceptions_without_reason + + +def test_mixed_type_group_is_reported(): + # `AR-001, SR-002` in one group misuses the comma; the pipe is what + # separates types, so the tag does not say what it appears to say. + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + (root / "src").mkdir(parents=True) + (root / "src" / "a.cpp").write_text( + f"// {TAG} AR-001, SR-002\n", encoding="utf-8") + scan = et.scan_files(et.iter_source_files(root), root) + assert scan.diagnostics.mixed_type_groups + + +# -------------------------------------------------------------------------- +# The register: denominators from requirements.md +# -------------------------------------------------------------------------- + +REGISTER_HEADER = "| ID | Requirement | Traces to | Priority | Status |\n|---|---|---|---|---|\n" + + +def test_counts_a_well_formed_table_row_as_a_defined_requirement(): + md = REGISTER_HEADER + ( + "| AR-001 | Detect faces in sampled frames | SR-002 | High | Done |\n" + "| AR-002 | Minimum face size 66x66 px | SR-002 | High | Planned |\n") + register = et.parse_register(md) + assert register.count("AR") == 2 + assert register.count("GR") == 0 + assert register.total == 2 + + +def test_does_not_count_ids_that_appear_only_in_the_traces_to_column(): + # The bug this parse rule avoids: a naive scan for AR-\d{3} over the whole + # file counts every reference as a definition and inflates the denominator. + md = REGISTER_HEADER + ( + "| GR-001 | Build gallery from library cast | SR-001, SR-005 | High | Done |\n" + "| GR-002 | Incremental merge refresh | PR-003 | High | Done |\n") + register = et.parse_register(md) + assert register.count("GR") == 2 + assert register.ids == {"GR-001", "GR-002"} + + +def test_does_not_count_ids_mentioned_in_prose(): + md = ("Some prose explaining that AR-005 relates to GR-001 and VR-003.\n\n" + + REGISTER_HEADER + + "| AR-005 | Align to 112x112 | SR-002 | High | Done |\n") + register = et.parse_register(md) + assert register.ids == {"AR-005"} + + +def test_does_not_count_the_verification_plan_table_as_definitions(): + # The per-requirement verification plan is also keyed on `ID`, but it + # assigns tiers rather than defining requirements. Counting its rows would + # double the denominator for every requirement that has a plan entry. + md = (REGISTER_HEADER + + "| AR-001 | Detect faces | SR-002 | High | Done |\n" + + "\n" + + "| ID | Tier | Test asserts | Edge cases to cover |\n|---|---|---|---|\n" + + "| AR-001 | T3 | Detector returns plausible boxes | smoke only |\n" + + "| AR-099 | T1 | Something not in the register | - |\n") + register = et.parse_register(md) + assert register.ids == {"AR-001"} + assert register.total == 1 + + +def test_deduplicates_an_id_listed_in_two_definition_tables(): + md = (REGISTER_HEADER + + "| AR-001 | Detect faces | SR-002 | High | Done |\n" + + "\n" + + REGISTER_HEADER + + "| AR-001 | Detect faces | SR-002 | High | Done |\n") + register = et.parse_register(md) + assert register.total == 1 + + +def test_withdrawn_requirements_leave_the_denominator(): + # IDs are permanent, but a withdrawn requirement can never be implemented. + # Leaving it in the denominator would depress coverage forever. + md = REGISTER_HEADER + ( + "| AR-001 | Detect faces | SR-002 | High | Done |\n" + "| AR-002 | Superseded mechanism | SR-002 | High | Withdrawn |\n") + register = et.parse_register(md) + assert register.ids == {"AR-001"} + assert "AR-002" in register.withdrawn + + +def test_the_denominator_is_live_adding_a_row_lowers_coverage(): + # The property JellyTau's frozen literals destroyed. Same traced set, one + # more requirement defined => a lower percentage, mechanically. + base = REGISTER_HEADER + "| AR-001 | A | SR-002 | High | Done |\n" + grown = base + "| AR-002 | B | SR-002 | High | Planned |\n" + before = et.compute_coverage(["AR-001"], et.parse_register(base)) + after = et.compute_coverage(["AR-001"], et.parse_register(grown)) + assert before.percent == 100.0 + assert after.percent == 50.0 + assert after.total == 2 + + +def test_register_captures_the_row_fields_not_just_the_id(): + md = REGISTER_HEADER + ( + "| AR-012 | Presence follows track extent | **SR-002** | High | Planned |\n") + req = et.parse_register(md).requirements["AR-012"] + assert req.text == "Presence follows track extent" + assert req.traces_to == "**SR-002**" + assert req.status == "Planned" + + +# -------------------------------------------------------------------------- +# Verification tiers and the GPU-less CI host +# -------------------------------------------------------------------------- + +TIER_REGISTER = REGISTER_HEADER + "".join( + f"| AR-{n:03d} | Requirement {n} | SR-002 | High | Planned |\n" + for n in range(1, 10)) + "".join( + f"| VR-{n:03d} | Study {n} | PR-002 | Medium | Planned |\n" + for n in range(1, 4)) + + +def _tier_table(rows: str) -> str: + return "| Requirement | Tier | Note |\n|---|---|---|\n" + rows + + +def test_tier_assignment_handles_lists_ranges_and_wildcards(): + md = TIER_REGISTER + "\n" + _tier_table( + "| AR-001, AR-005 | T3 | smoke |\n" + "| AR-002 … AR-004 | **T2** | replay |\n" + "| AR-006 | T1 + T4 | mixed |\n" + "| VR-* | Out of CI | studies |\n") + register = et.parse_register(md) + assert register.requirements["AR-001"].tiers == {"T3"} + assert register.requirements["AR-003"].tiers == {"T2"} + assert register.requirements["AR-006"].tiers == {"T1", "T4"} + assert register.requirements["VR-002"].tiers == {"out-of-ci"} + + +def test_a_range_cannot_invent_a_requirement_the_register_lacks(): + md = TIER_REGISTER + "\n" + _tier_table("| AR-001 … AR-050 | T2 | wide |\n") + register = et.parse_register(md) + assert register.total == 12 + assert "AR-050" not in register.ids + + +def test_slash_shorthand_in_the_verification_plan_expands(): + md = TIER_REGISTER + "\n" + ( + "| ID | Tier | Test asserts | Edge cases |\n|---|---|---|---|\n" + "| AR-009/008 | T2 | Cut shifts weighting | cut with same people |\n") + register = et.parse_register(md) + assert register.requirements["AR-008"].tiers == {"T2"} + assert register.requirements["AR-009"].tiers == {"T2"} + + +def test_tiers_from_both_tables_are_unioned_not_overwritten(): + # The summary table says AR-006 is T4; the per-requirement plan adds a T1 + # equivalence check. The T1 part does run in CI, so the requirement is + # executable and must not be written off as GPU-only. + md = TIER_REGISTER + "\n" + _tier_table("| AR-006 | T4 | GPU host only |\n") + "\n" + ( + "| ID | Tier | Test asserts | Edge cases |\n|---|---|---|---|\n" + "| AR-006 | T1 + T4 | GEMM equals reference loop | small input in CI |\n") + register = et.parse_register(md) + assert register.requirements["AR-006"].tiers == {"T1", "T4"} + assert register.requirements["AR-006"].ci_executable + + +def test_a_t4_only_requirement_is_not_ci_executable(): + md = TIER_REGISTER + "\n" + _tier_table("| AR-007 | **T4** | GPU only |\n") + register = et.parse_register(md) + assert not register.requirements["AR-007"].ci_executable + assert register.unexecutable_ids() == {"AR-007"} + + +def test_a_requirement_tracing_up_to_nothing_is_reported(): + # SPEC.md section 6: a requirement citing no parent is scope creep, and it + # is invisible unless something looks. A section reference counts as a + # parent - what matters is that something was recorded. + md = REGISTER_HEADER + ( + "| AR-001 | Has a parent | SR-002 | High | Done |\n" + "| AR-002 | Parent is a section | §4 | Medium | Planned |\n" + "| AR-003 | Serves nothing stated | - | Low | Planned |\n" + "| AR-004 | Blank cell | | Low | Planned |\n") + register = et.parse_register(md) + assert register.parentless_ids() == {"AR-003", "AR-004"} + + +def test_a_requirement_with_no_tier_is_unknown_not_unexecutable(): + register = et.parse_register(TIER_REGISTER) + assert register.tier_unknown_ids() == register.ids + assert register.unexecutable_ids() == set() + + +# -------------------------------------------------------------------------- +# Coverage arithmetic +# -------------------------------------------------------------------------- + +COVERAGE_REGISTER = et.parse_register( + REGISTER_HEADER + + "| AR-001 | A | SR-002 | High | Done |\n" + + "| AR-002 | B | SR-002 | High | Done |\n" + + "| GR-001 | C | SR-005 | High | Done |\n" + + "| AR-027 | Arbitrary gallery scale | SR-001 | High | Planned |\n" + + "\n" + + _tier_table("| AR-001, AR-002 | T2 | replay |\n" + "| GR-001 | T1 | bookkeeping |\n" + "| AR-027 | **T4** | GPU host only |\n")) + + +def test_coverage_is_the_intersection_of_traced_and_defined(): + cov = et.compute_coverage(["AR-001", "GR-001"], COVERAGE_REGISTER) + assert cov.covered == ["AR-001", "GR-001"] + assert cov.total == 4 + assert cov.percent == 50.0 + + +def test_a_traced_but_undefined_id_cannot_inflate_the_numerator(): + # This is exactly how a ratio exceeds 100%: a tag naming a renumbered or + # mistyped requirement counted as covered. + cov = et.compute_coverage(["AR-001", "GR-001", "AR-097"], COVERAGE_REGISTER) + assert cov.covered == ["AR-001", "GR-001"] + assert cov.percent == 50.0 + + +def test_orphan_tags_are_reported_so_they_get_fixed(): + cov = et.compute_coverage(["AR-001", "AR-097", "GR-404"], COVERAGE_REGISTER) + assert cov.orphaned == ["AR-097", "GR-404"] + + +def test_no_orphans_when_every_traced_id_is_defined(): + cov = et.compute_coverage(["AR-001", "AR-002"], COVERAGE_REGISTER) + assert cov.orphaned == [] + + +def test_test_and_system_ids_are_a_separate_taxonomy(): + # UT/IT live in their own register section; PR/SR live in the umbrella + # SPEC.md, which is not part of this repo's checkout. Neither counts toward + # coverage, and flagging them as orphans would bury real typos in noise. + cov = et.compute_coverage( + ["AR-001", "UT-003", "IT-007", "SR-002", "PR-001"], COVERAGE_REGISTER) + assert cov.orphaned == [] + assert cov.covered == ["AR-001"] + + +def test_a_gpu_only_requirement_is_tagged_but_unexecuted_not_covered(): + # The rule specific to this repo: CI is an Intel N100 with no dGPU. A test + # that exists but can never run is not evidence, and counting it is the + # same failure mode as the 158% bug. + cov = et.compute_coverage(["AR-001", "AR-027"], COVERAGE_REGISTER) + assert cov.unexecuted == ["AR-027"] + assert cov.covered == ["AR-001"] + assert cov.percent == 25.0 + assert cov.orphaned == [] + + +def test_ci_scope_percentage_excludes_unexecutable_requirements_from_both_sides(): + cov = et.compute_coverage(["AR-001"], COVERAGE_REGISTER) + assert cov.ci_executable_total == 3 + assert round(cov.ci_percent) == 33 + + +def test_tracing_only_gpu_only_requirements_yields_zero_coverage(): + cov = et.compute_coverage(["AR-027"], COVERAGE_REGISTER) + assert cov.covered == [] + assert cov.percent == 0.0 + assert cov.unexecuted == ["AR-027"] + + +def test_duplicate_traced_ids_are_counted_once(): + cov = et.compute_coverage(["AR-001", "AR-001", "AR-001"], COVERAGE_REGISTER) + assert cov.covered == ["AR-001"] + + +def test_an_empty_trace_set_is_zero_percent_not_a_divide_by_zero(): + cov = et.compute_coverage([], COVERAGE_REGISTER) + assert cov.covered == [] + assert cov.percent == 0.0 + + +def test_an_empty_register_reports_zero_rather_than_nan(): + cov = et.compute_coverage(["AR-001"], et.Register()) + assert cov.percent == 0.0 + assert cov.ci_percent == 0.0 + + +def test_full_coverage_reports_exactly_one_hundred_never_above(): + cov = et.compute_coverage( + ["AR-001", "AR-002", "GR-001", "AR-027"], COVERAGE_REGISTER) + # AR-027 is GPU-only, so the honest ceiling here is 3/4. + assert cov.percent == 75.0 + assert cov.ci_percent == 100.0 + assert cov.ci_percent <= 100.0 + + +# -------------------------------------------------------------------------- +# The gate: end to end, including the ways it must fail +# -------------------------------------------------------------------------- + +FIXTURE_REGISTER = ( + "# register\n\n" + REGISTER_HEADER + + "| AR-001 | Detect faces | SR-002 | High | Done |\n" + + "| AR-002 | Minimum face size | SR-002 | High | Planned |\n" + + "| AR-027 | Arbitrary gallery scale | SR-001 | High | Planned |\n" + + "\n" + + _tier_table("| AR-001, AR-002 | T2 | replay |\n" + "| AR-027 | **T4** | GPU host only |\n")) + + +def _fixture_repo(tmp: str, source: str) -> Path: + root = Path(tmp) + (root / "docs").mkdir(parents=True, exist_ok=True) + (root / "src").mkdir(parents=True, exist_ok=True) + (root / "docs" / "requirements.md").write_text(FIXTURE_REGISTER, encoding="utf-8") + (root / "src" / "pipeline.cpp").write_text(source, encoding="utf-8") + return root + + +def _run_gate(root: Path, *extra: str): + buffer = io.StringIO() + with redirect_stdout(buffer): + code = et.main(["--root", str(root), "--format", "coverage", *extra]) + return code, buffer.getvalue() + + +def test_gate_passes_at_the_default_threshold_with_no_tags_yet(): + # Near-zero coverage on a fresh tree is the correct reading, not a failure. + with tempfile.TemporaryDirectory() as tmp: + root = _fixture_repo(tmp, "int main() { return 0; }\n") + code, out = _run_gate(root) + assert code == 0, out + assert "0 / 3 (0.0%)" in out + + +def test_gate_fails_below_an_explicit_threshold(): + # Proves the gate can fail at all. A gate nobody has watched fail is not + # known to work. + with tempfile.TemporaryDirectory() as tmp: + root = _fixture_repo(tmp, f"// {TAG} AR-001\nint main() {{ return 0; }}\n") + code, out = _run_gate(root, "--min-coverage", "99") + assert code == 1 + assert "below the minimum" in out + + +def test_gate_fails_on_an_orphan_tag(): + with tempfile.TemporaryDirectory() as tmp: + root = _fixture_repo(tmp, f"// {TAG} AR-001, AR-404\nint main() {{}}\n") + code, out = _run_gate(root) + assert code == 1 + assert "AR-404" in out + assert "orphan tag" in out + + +def test_gate_can_be_asked_to_report_orphans_without_failing(): + with tempfile.TemporaryDirectory() as tmp: + root = _fixture_repo(tmp, f"// {TAG} AR-001, AR-404\nint main() {{}}\n") + code, out = _run_gate(root, "--allow-orphans") + assert code == 0 + assert "AR-404" in out + + +def test_gate_fails_when_the_register_parses_to_nothing(): + # With a low threshold this self-check is what keeps the gate meaningful: + # a broken parser would otherwise report 0/0 and pass. + with tempfile.TemporaryDirectory() as tmp: + root = _fixture_repo(tmp, "int main() {}\n") + (root / "docs" / "requirements.md").write_text( + "# register\n\nNo tables here.\n", encoding="utf-8") + code, out = _run_gate(root) + assert code == 1 + assert "ZERO requirements" in out + + +def test_gate_fails_when_no_source_files_were_scanned(): + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + (root / "docs").mkdir(parents=True) + (root / "docs" / "requirements.md").write_text(FIXTURE_REGISTER, + encoding="utf-8") + code, out = _run_gate(root) + assert code == 1 + assert "no source files were scanned" in out + + +def test_gate_reports_a_gpu_only_requirement_as_tagged_but_unexecuted(): + with tempfile.TemporaryDirectory() as tmp: + root = _fixture_repo(tmp, f"// {TAG} AR-027\nint main() {{}}\n") + code, out = _run_gate(root) + assert code == 0 + assert "TAGGED BUT UNEXECUTED" in out + assert "AR-027 (tier T4)" in out + assert "0 / 3 (0.0%)" in out + + +def test_gate_hard_fails_on_an_impossible_ratio(): + # Coverage above 100% cannot happen through the intersection, which is the + # point: if it ever does, the arithmetic is broken and the run must not be + # reported as a pass. Forced here by handing the reporter a poisoned value. + with tempfile.TemporaryDirectory() as tmp: + root = _fixture_repo(tmp, f"// {TAG} AR-001\nint main() {{}}\n") + register = et.read_register(root / "docs" / "requirements.md") + scan = et.scan_files(et.iter_source_files(root), root) + report = et.build_report(root, register, scan) + report.coverage.percent = 158.0 + text, code = et.format_coverage_report(report, 50.0) + assert code == 1 + assert "exceeds 100%" in text + + +def test_the_per_type_breakdown_sums_to_the_headline_figure(): + # A breakdown that does not add up to its own total is how a wrong number + # survives review: every row looks plausible on its own. + with tempfile.TemporaryDirectory() as tmp: + root = _fixture_repo( + tmp, f"// {TAG} AR-001, AR-027\nint main() {{}}\n") + register = et.read_register(root / "docs" / "requirements.md") + scan = et.scan_files(et.iter_source_files(root), root) + report = et.build_report(root, register, scan) + stats = et.per_type_stats(report) + assert sum(c for c, _, _ in stats.values()) == len(report.coverage.covered) + assert sum(u for _, u, _ in stats.values()) == len(report.coverage.unexecuted) + assert sum(d for _, _, d in stats.values()) == report.coverage.total + assert stats["AR"] == (1, 1, 3) + + +def test_json_and_markdown_outputs_are_written_and_consistent(): + import json + with tempfile.TemporaryDirectory() as tmp: + root = _fixture_repo(tmp, f"// {TAG} AR-001 | SR-002\nint main() {{}}\n") + json_out = root / "traces-report.json" + md_out = root / "docs" / "traceability.md" + buffer = io.StringIO() + with redirect_stdout(buffer): + code = et.main(["--root", str(root), "--format", "coverage", + "--json-out", str(json_out), + "--markdown-out", str(md_out)]) + assert code == 0 + data = json.loads(json_out.read_text(encoding="utf-8")) + assert data["defined"]["total"] == 3 + assert data["coverage"]["covered"] == 1 + assert data["coverage"]["percent"] == round(100 / 3, 1) + assert data["byType"]["SR"] == ["SR-002"] + assert data["gpuOnlyRequirements"] == ["AR-027"] + assert "AR-001" in md_out.read_text(encoding="utf-8") + + +def test_system_spec_parsing_enables_orphan_checks_for_pr_and_sr(): + spec = ("### SR-002 - Presence is scene-scoped\n\n" + "| ID | Goal | Why |\n|---|---|---|\n" + "| **PR-001** | Show which actors are on screen | The product |\n") + ids = et.parse_system_spec(spec) + assert ids == {"SR-002", "PR-001"} + + with tempfile.TemporaryDirectory() as tmp: + root = _fixture_repo(tmp, f"// {TAG} AR-001 | SR-999\nint main() {{}}\n") + spec_path = root / "system-spec.md" + spec_path.write_text(spec, encoding="utf-8") + code, out = _run_gate(root, "--system-spec", str(spec_path)) + assert "SR-999" in out + assert code == 0 # advisory: the system register is not this repo's + + +# -------------------------------------------------------------------------- +# The live register — structural assertions only, so this does not churn as +# requirements are added. +# -------------------------------------------------------------------------- + +#: The live-register tests below ran against this repo's own register when the +#: tool lived inside `scene-actor-extraction`. Now that it is shared, this repo +#: is the project home and holds no component register of its own, so they take +#: a path from the environment and skip when it is absent. +#: +#: LIVE_REGISTER=../scene-actor-extraction/docs/requirements.md \ +#: python3 test_extract_traces.py +#: +#: They are kept rather than deleted because they assert something the synthetic +#: fixtures cannot: that a *real* register, with all its formatting accidents, +#: parses at all. +LIVE_REGISTER = os.environ.get("LIVE_REGISTER") + + +def test_the_live_register_parses_and_assigns_tiers(): + if not LIVE_REGISTER: + return # skipped: no component register to point at + register = et.read_register(Path(LIVE_REGISTER)) + assert register.total > 0 + assert sum(register.count(t) for t in et.LOCAL_TYPES) == register.total + # A tier that no GPU-less host can run must be visible in the parse, not + # merely stated in prose. + assert register.unexecutable_ids() <= register.ids + + +def test_the_live_register_yields_a_gate_run_that_cannot_exceed_one_hundred(): + if not LIVE_REGISTER: + return + register = et.read_register(Path(LIVE_REGISTER)) + root = Path(LIVE_REGISTER).resolve().parent.parent + scan = et.scan_files(et.iter_source_files(root), root) + cov = et.compute_coverage( + [i for t in scan.traces for i in t.requirements], register) + assert 0.0 <= cov.percent <= 100.0 + assert len(cov.covered) <= cov.total + + +# -------------------------------------------------------------------------- +# Per-repo configuration — the flags that make this tool shareable. +# +# Without them a repo whose prefixes or language differ from the defaults gets +# zero requirements and zero files, which the gate correctly refuses to report +# as coverage. These assert the overrides actually take effect. +# -------------------------------------------------------------------------- + +def _restore_taxonomy(fn): + """Run `fn` with the module defaults restored afterwards.""" + local, suffixes, roots = et.LOCAL_TYPES, set(et.SOURCE_SUFFIXES), et.SCAN_ROOTS + try: + fn() + finally: + et.configure_taxonomy(local) + et.configure_suffixes(suffixes) + et.configure_scan_roots(roots) + + +def test_types_override_changes_which_prefixes_are_counted(): + def body(): + register_md = ( + "| ID | Requirement | Traces to | Priority | Status |\n" + "|---|---|---|---|---|\n" + "| UR-001 | A user requirement | SR-001 | High | Done |\n" + "| DR-001 | A dev requirement | PR-004 | High | Done |\n" + ) + with tempfile.TemporaryDirectory() as tmp: + path = Path(tmp) / "requirements.md" + path.write_text(register_md, encoding="utf-8") + + # Under the defaults these prefixes are unknown, so nothing counts. + et.configure_taxonomy(("AR", "DP")) + assert et.read_register(path).total == 0 + + et.configure_taxonomy(("UR", "DR")) + register = et.read_register(path) + assert register.total == 2 + assert register.count("UR") == 1 + assert register.count("DR") == 1 + + _restore_taxonomy(body) + + +def test_suffix_override_changes_which_files_are_scanned(): + def body(): + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + (root / "src").mkdir() + (root / "src" / "lib.rs").write_text( + "/// TRACES: UR-001 | SR-004\npub fn f() {}\n", encoding="utf-8") + + # Default suffixes are C++/Python, so a Rust tree scans as empty — + # the failure mode this override exists to fix. + assert et.iter_source_files(root) == [] + + et.configure_suffixes([".rs"]) + files = et.iter_source_files(root) + assert len(files) == 1 + scan = et.scan_files(files, root) + assert [i for t in scan.traces for i in t.requirements] == [ + "UR-001", "SR-004"] + + _restore_taxonomy(body) + + +def test_suffix_override_accepts_extensions_with_or_without_a_dot(): + def body(): + et.configure_suffixes(["rs", ".cs"]) + assert et.SOURCE_SUFFIXES == {".rs", ".cs"} + + _restore_taxonomy(body) + + +def test_scan_root_override_limits_the_walk(): + def body(): + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + # Not "vendor" — that is in EXCLUDED_DIR_NAMES and would be + # filtered whatever the scan roots say, testing the wrong thing. + for d in ("src", "eval"): + (root / d).mkdir() + (root / d / "f.py").write_text("# TRACES: AR-001\n", encoding="utf-8") + + et.configure_scan_roots(["src"]) + assert len(et.iter_source_files(root)) == 1 + + et.configure_scan_roots(["src", "eval"]) + assert len(et.iter_source_files(root)) == 2 + + _restore_taxonomy(body) + + +def test_defaults_are_unchanged_by_the_overrides_existing(): + # The extraction pipeline must keep working with no flags at all, or moving + # the tool here would have broken the repo it came from. + assert et.LOCAL_TYPES == ("AR", "DP", "IR", "GR", "VR") + assert ".py" in et.SOURCE_SUFFIXES and ".cpp" in et.SOURCE_SUFFIXES + assert ".rs" not in et.SOURCE_SUFFIXES + + +# -------------------------------------------------------------------------- + +def _main() -> int: + tests = [(name, obj) for name, obj in sorted(globals().items()) + if name.startswith("test_") and callable(obj)] + failed = [] + for name, fn in tests: + try: + fn() + except Exception as exc: # noqa: BLE001 - a test runner reports everything + failed.append((name, exc)) + print(f"FAIL {name}: {type(exc).__name__}: {exc}") + else: + print(f"ok {name}") + print(f"\n{len(tests) - len(failed)}/{len(tests)} passed") + return 1 if failed else 0 + + +if __name__ == "__main__": + sys.exit(_main()) diff --git a/scripts/traceability/traceability-gate.sh b/scripts/traceability/traceability-gate.sh new file mode 100755 index 0000000..a41dbb8 --- /dev/null +++ b/scripts/traceability/traceability-gate.sh @@ -0,0 +1,89 @@ +#!/bin/sh +# +# Requirement traceability gate. Run locally exactly as CI runs it: +# +# scripts/traceability/traceability-gate.sh +# +# Writes traces-report.json and docs/traceability.md, prints the coverage +# report, and exits non-zero when the gate fails. +# +# Environment: +# MIN_COVERAGE minimum overall coverage percent (default 0 - see below) +# ALLOW_ORPHANS set to 1 to report orphan tags without failing +# TRACES_JSON JSON report path (default traces-report.json) +# TRACES_MD markdown matrix path (default docs/traceability.md) +# REPO_ROOT repository to scan. Defaults to two levels above this +# script, which is correct when the tooling lives in the repo +# it checks. **When vendored as a submodule that default is +# the submodule itself**, so a consuming repo must set this — +# its wrapper does. +# TYPES comma-separated requirement prefixes (e.g. UR,DR). Defaults +# to the extraction set; a repo whose prefixes differ parses +# to zero requirements without this. +# SUFFIXES comma-separated file extensions (e.g. .rs). Defaults to the +# C++/Python set; a repo whose language differs scans zero +# files without this. +# SCAN_ROOTS comma-separated directories to walk, relative to REPO_ROOT. +# SYSTEM_SPEC optional path to the system SPEC.md, which defines the PR/SR +# IDs; when given, PR/SR orphans are reported too. It lives in +# the project home (jray-project) — when this tooling is +# vendored from there, it is a sibling of this script. +# +# Threshold policy lives here and nowhere else. It is deliberately NOT +# duplicated into the workflow YAML: a threshold written in two places is a +# threshold that will disagree with itself. +# +# MIN_COVERAGE defaults to 0 because almost nothing is tagged yet - tags are +# added as the pipeline is built, so a low number today is accurate rather than +# alarming. A zero threshold does NOT mean the gate cannot fail: orphan tags, +# a >100% ratio, a register that parses to nothing, and an empty source scan +# are all hard failures from day one. Raise MIN_COVERAGE as tags land; treat +# every raise as a ratchet, never a reset. +# +# POSIX sh, no bashisms, no jq - the extractor does its own arithmetic and +# printing so CI needs nothing beyond python3. + +set -eu + +SCRIPT_DIR=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd) +REPO_ROOT="${REPO_ROOT:-$(CDPATH= cd -- "$SCRIPT_DIR/../.." && pwd)}" + +MIN_COVERAGE="${MIN_COVERAGE:-0}" +TRACES_JSON="${TRACES_JSON:-$REPO_ROOT/traces-report.json}" +TRACES_MD="${TRACES_MD:-$REPO_ROOT/docs/traceability.md}" + +PYTHON="${PYTHON:-python3}" +command -v "$PYTHON" >/dev/null 2>&1 || { + echo "FAILED: $PYTHON not found. The traceability gate needs Python 3.9+" >&2 + exit 2 +} + +set -- \ + --root "$REPO_ROOT" \ + --requirements "${REQUIREMENTS:-$REPO_ROOT/docs/requirements.md}" \ + --format coverage \ + --json-out "$TRACES_JSON" \ + --markdown-out "$TRACES_MD" \ + --min-coverage "$MIN_COVERAGE" + +if [ "${ALLOW_ORPHANS:-0}" = "1" ]; then + set -- "$@" --allow-orphans +fi + +if [ -n "${SYSTEM_SPEC:-}" ]; then + set -- "$@" --system-spec "$SYSTEM_SPEC" +fi + +if [ -n "${TYPES:-}" ]; then + set -- "$@" --types "$TYPES" +fi + +if [ -n "${SUFFIXES:-}" ]; then + set -- "$@" --suffixes "$SUFFIXES" +fi + +if [ -n "${SCAN_ROOTS:-}" ]; then + set -- "$@" --scan-roots "$SCAN_ROOTS" +fi + +exec "$PYTHON" "$SCRIPT_DIR/extract_traces.py" "$@"