From faaa71fa09f9a7d3263f26122e267797bea3c962 Mon Sep 17 00:00:00 2001 From: Duncan Tourolle Date: Thu, 30 Jul 2026 18:35:56 +0200 Subject: [PATCH] refactor(traceability): parameterise the extractor for all three components The tool is moving into the jray-project submodule to be shared by scene-actor-extraction (C++/Python), jRay (C#) and JRay-public-server (Rust). Two constants blocked that: LOCAL_TYPES and SOURCE_SUFFIXES were hardcoded to this repo, so either sibling parsed zero requirements and scanned zero files. Both, plus the register path, scan roots, system-spec path, exclude list and CI-executable tier set, are now configuration. One implementation, parameterised. A second copy for "the other language" is how two implementations start drifting apart, so there is exactly one - the same code path now produces: scene-actor-extraction AR/DP/IR/GR/VR 59 defined 0 tagged 0.0% jRay JR 46 defined 24 covered 52.2% JRay-public-server UR/DR 32 defined 23 covered 71.9% Configuration is traceability.toml at the component repo root, CLI flags, or both (flags win). Its directory defines the repo root, so the gate works from any subdirectory. `--print-example-config` emits the annotated schema. The JSON report echoes the settings it ran with, since a shared tool's output is otherwise ambiguous about which repo it describes. The refusal behaviour is kept and sharpened, because parameterising is exactly what makes it easy to point a repo at the wrong prefixes or the wrong suffixes. Zero requirements parsed or zero files scanned is still a hard failure, and the message now names the setting that is wrong rather than printing a plausible 0%. Config errors exit 2, not 1: a broken config is not a coverage failure, and conflating them makes CI logs lie about why the job went red. Also fixed while adapting to the sibling registers, which are read but not modified here: * escaped `\|` inside a markdown cell no longer shifts every later column (the server's register contains `small-\|M\|`); * a tag above an attribute-decorated declaration attributes to the declaration, not to `[HttpGet(...)]` or `#[derive(...)]` - the gap jRay's register calls out; * Rust and C# declaration patterns for context extraction; * the missing-tier warning is suppressed for a register that assigns no tiers at all, rather than listing every requirement in it. The workflow is now component-agnostic too: the changed-file check reads its extension list out of the report the gate just wrote, so the definition of "source file" lives in one place. 79 tests, still fixture-based, now including the cross-repo cases: the same parser over JR and UR/DR registers, the same scanner over Rust and C#, and both misconfigurations failing loudly. --- .gitea/workflows/traceability-check.yml | 44 +- docs/traceability.md | 12 +- scripts/traceability/extract_traces.py | 939 ++++++++++++++------ scripts/traceability/test_extract_traces.py | 595 ++++++++++--- scripts/traceability/traceability-gate.sh | 80 +- traceability.toml | 34 + 6 files changed, 1257 insertions(+), 447 deletions(-) create mode 100644 traceability.toml diff --git a/.gitea/workflows/traceability-check.yml b/.gitea/workflows/traceability-check.yml index afbad35..36240b4 100644 --- a/.gitea/workflows/traceability-check.yml +++ b/.gitea/workflows/traceability-check.yml @@ -1,8 +1,13 @@ name: Traceability Validation -# Mirrors JellyTau's .gitea/workflows/traceability-check.yml, adapted for a -# C++/Python repo: the extractor is Python and needs nothing but python3, so -# there is no toolchain install step and no jq. +# Mirrors JellyTau's .gitea/workflows/traceability-check.yml. The extractor is +# stdlib Python, so there is no toolchain install step and no jq. +# +# This workflow is component-agnostic: every repo-specific setting - which ID +# prefixes count, which file suffixes are source, which directories to scan, +# the threshold - lives in traceability.toml at the repo root, and the same +# extractor is shared by all three JRay components. Copying this file into +# another component needs no edits. # # NOTE: the runner here is an Intel N100 with no discrete GPU. This job is only # ever static analysis of source comments plus markdown parsing, so it is cheap; @@ -31,14 +36,15 @@ jobs: uses: actions/checkout@v4 with: fetch-depth: 0 + submodules: recursive - name: Check Python is available run: | set -e command -v python3 >/dev/null 2>&1 || { echo "python3 is missing from the runner image." - echo "The traceability tooling is stdlib-only Python 3.9+;" - echo "no other dependency is needed." + echo "The traceability tooling is stdlib-only Python;" + echo "3.9+ with CLI flags, 3.11+ to read traceability.toml." exit 1 } python3 --version @@ -49,10 +55,13 @@ jobs: - name: Test the extractor run: python3 scripts/traceability/test_extract_traces.py - # Threshold policy lives in traceability-gate.sh, not here, so local runs - # and CI runs cannot disagree about what "passing" means. Denominators - # come from docs/requirements.md at run time and are never hardcoded -- - # in this file or anywhere else. + # Threshold policy and every other repo-specific setting live in + # traceability.toml, not here, so local runs and CI runs cannot disagree + # about what "passing" means. Denominators come from docs/requirements.md + # at run time and are never hardcoded -- in this file or anywhere else. + # + # A misconfigured run (zero requirements parsed, zero files scanned) is a + # hard failure rather than a plausible-looking 0%. - name: Traceability gate run: sh scripts/traceability/traceability-gate.sh @@ -62,11 +71,22 @@ jobs: set -e echo "Checking modified sources for TRACES tags..." + # The extensions come from the report the gate just wrote, which got + # them from traceability.toml. Restating them here would be a second + # place for the source-file definition to live, and the two would + # drift the first time a language is added. + PATTERN=$(python3 -c " + import json, re, sys + suffixes = json.load(open('traces-report.json'))['config']['sourceSuffixes'] + print('(' + '|'.join(re.escape(s) + '\$' for s in suffixes) + ')') + ") + echo "Source suffixes from traceability.toml: $PATTERN" + CHANGED=$(git diff --name-only "origin/${{ github.base_ref }}...HEAD" \ - | grep -E '\.(cpp|cc|cxx|hpp|hxx|h|cu|cuh|py)$' || true) + | grep -E "$PATTERN" || true) if [ -z "$CHANGED" ]; then - echo "No C++/Python files changed." + echo "No source files changed." exit 0 fi @@ -85,7 +105,7 @@ jobs: MISSING=$(mktemp) echo "$CHANGED" | while IFS= read -r file; do case "$file" in - */test_*.py|*_test.py|tests/*|*/tests/*) continue ;; + */test_*.py|*_test.py|*Tests.cs|tests/*|*/tests/*) continue ;; esac [ -f "$file" ] || continue if ! grep -q 'TRACES:' "$file"; then diff --git a/docs/traceability.md b/docs/traceability.md index c1a328c..28dbc67 100644 --- a/docs/traceability.md +++ b/docs/traceability.md @@ -1,11 +1,11 @@ # Requirements traceability matrix - + -**Generated:** 2026-07-30T16:15:17+00:00 +**Generated:** 2026-07-30T16:35:36+00:00 -Denominators are read from [`requirements.md`](requirements.md) at run time, never hardcoded. Coverage counts a requirement only when it is tagged in source **and** has a verification tier this CI host can execute — CI is an Intel N100 with no discrete GPU. +Denominators are read from [`requirements.md`](requirements.md) at run time, never hardcoded. Coverage counts a requirement only when it is tagged in source **and** has a verification tier this repo's CI host can execute (`T1, T2, T3, static`). ## Summary @@ -18,7 +18,7 @@ Denominators are read from [`requirements.md`](requirements.md) at run time, nev | Requirements covered | 0 | | **Coverage** | **0.0%** (0/59) | | Coverage of CI-executable scope | 0.0% (0/50) | -| Tagged but unexecuted in CI (T4/GPU) | 0 | +| Tagged but unexecuted in CI | 0 | | Orphan tags | 0 | ### By type @@ -34,7 +34,7 @@ Denominators are read from [`requirements.md`](requirements.md) at run time, nev ## Not executable in CI -CI runs on an Intel N100 with no discrete GPU. These requirements have no verification tier that can run here, so a tag on them is evidence of *intent*, not of verification. They are never counted as covered. +These requirements have no verification tier this repo's CI host can run, so a tag on them is evidence of *intent*, not of verification. They are never counted as covered. | ID | Tiers | Tagged in source | Requirement | |---|---|---|---| @@ -62,7 +62,7 @@ _None._ ## Recorded exceptions -Deliberate, documented departures from an invariant (`EXCEPTION: AR-nnn `). Reported separately and never counted as coverage — an exception is a decision to be reviewed, not evidence a requirement is met. +Deliberate, documented departures from an invariant (`EXCEPTION: XX-nnn `). Reported separately and never counted as coverage — an exception is a decision to be reviewed, not evidence a requirement is met. _None._ diff --git a/scripts/traceability/extract_traces.py b/scripts/traceability/extract_traces.py index 2a9de8b..e368c9b 100755 --- a/scripts/traceability/extract_traces.py +++ b/scripts/traceability/extract_traces.py @@ -1,12 +1,26 @@ #!/usr/bin/env python3 -"""Extract requirement traces from C++/Python sources and report coverage. +"""Extract requirement traces from source and report coverage against a register. -Ported from JellyTau's ``scripts/extract-traces.ts``. That one scans -TypeScript/Svelte/Rust; this repo is C++ and Python, so the scanner is Python -with no third-party dependencies — the CI host must not need a node/bun -toolchain to check traceability. +Ported from JellyTau's ``scripts/extract-traces.ts`` and written in stdlib +Python so no repo needs a bun/node toolchain to check its source comments. -Tag format (see ../../../CLAUDE.md and SPEC.md section 6). A pipe separates +**One implementation, parameterised.** This tool is shared by every JRay +component repo — C++/Python extraction, C# plugin, Rust server — via the +``jray-project`` submodule. Nothing about a single repo is baked in: the +requirement ID prefixes, the source file suffixes, the directories to scan, the +register path and the system-spec path are all configuration. A second copy for +"the other language" is how two implementations start drifting apart, so there +is exactly one. + +Configuration comes from ``traceability.toml`` at the component repo root, from +CLI flags, or both (flags win). ``--print-example-config`` emits the full +annotated schema; the three required keys are:: + + requirement_types = ["AR", "DP", "IR", "GR", "VR"] + languages = ["cpp", "python"] + source_roots = ["src", "tests", "scripts"] + +Tag format (see the system CLAUDE.md and SPEC.md section 6). A pipe separates requirement *types*, a comma separates IDs within a type:: /// TRACES: AR-nnn, AR-mmm | SR-nnn @@ -19,28 +33,32 @@ separately — never silently folded into coverage:: Usage:: - python3 scripts/traceability/extract_traces.py --format coverage - python3 scripts/traceability/extract_traces.py --format json > traces-report.json - python3 scripts/traceability/extract_traces.py --format markdown --markdown-out docs/traceability.md + python3 extract_traces.py --format coverage + python3 extract_traces.py --format json > report.json + python3 extract_traces.py --config path/to/traceability.toml -The CI gate is ``scripts/traceability/traceability-gate.sh``, which wraps this. +The CI gate is ``traceability-gate.sh``, which wraps this. -Two rules inherited from JellyTau's gate repair, both learned the hard way -(JellyTau/docs/specs/traceability-gate-repair.md): +Three rules the gate exists to enforce. The first two are inherited from +JellyTau's gate repair, both learned the hard way +(``JellyTau/docs/specs/traceability-gate-repair.md``): -1. Coverage denominators are read out of ``docs/requirements.md`` at run time. - Never hardcode them. JellyTau's gate divided by frozen literals while the - register grew to 211 requirements; it reported 158% coverage, so its 50% - threshold could never trip. A gate that cannot fail is worse than no gate, - because it is trusted. +1. Coverage denominators are read out of the register at run time. Never + hardcode them. JellyTau's gate divided by frozen literals while the register + grew to 211 requirements; it reported 158% coverage, so its 50% threshold + could never trip. A gate that cannot fail is worse than no gate, because it + is trusted. 2. Coverage above 100% is a hard failure, not a pass. It means the computation is broken, and it is the signal that would have caught (1) immediately. +3. A misconfigured run refuses to report at all. Parsing zero requirements or + scanning zero files prints a failure, not a plausible-looking 0% — the same + family of error as (1), and the one a shared, parameterised tool makes easy + to hit. -One rule specific to this repo: CI runs on an Intel N100 with no discrete GPU. -Requirements whose only verification tier is T4/GPU (or "out of CI") cannot -execute here. They are reported as *tagged but unexecuted* and are excluded -from the covered numerator — counting a test that never runs as coverage is -the same failure mode as the 158% bug. +The GPU-less-CI rule is configuration rather than code: requirements whose +verification tiers all fall outside ``ci_executable_tiers`` are reported as +*tagged but unexecuted* and excluded from the covered numerator. Counting a +test that never runs is the same failure mode as the 158% bug. """ from __future__ import annotations @@ -49,75 +67,365 @@ import argparse import json import re import sys -from dataclasses import dataclass, field +from dataclasses import dataclass, field, replace from datetime import datetime, timezone from pathlib import Path -from typing import Dict, Iterable, List, Optional, Sequence, Set, Tuple - -# Derived from this file's location so a CI checkout at any path works. -SCRIPT_DIR = Path(__file__).resolve().parent -REPO_ROOT = SCRIPT_DIR.parent.parent +from typing import (Dict, FrozenSet, Iterable, List, Optional, Sequence, Set, + Tuple) # -------------------------------------------------------------------------- -# Taxonomy +# Defaults. Everything here is overridable; nothing here names a single repo. # -------------------------------------------------------------------------- -#: Types defined in docs/requirements.md. These, and only these, participate -#: in the coverage fraction. -LOCAL_TYPES: Tuple[str, ...] = ("AR", "DP", "IR", "GR", "VR") - #: Test identifiers. A separate taxonomy: a test ID is evidence *for* a -#: requirement, not a requirement. Excluded from the fraction. -TEST_TYPES: Tuple[str, ...] = ("UT", "IT") +#: requirement, not a requirement, so it never enters the coverage fraction. +#: House-wide (SPEC.md section 6), hence a default rather than an argument. +DEFAULT_TEST_TYPES: Tuple[str, ...] = ("UT", "IT") -#: Defined in the system-level SPEC.md, which lives in the umbrella project and -#: is not part of this repo's checkout. Recognised in tags, never counted here, -#: and orphan-checked only when --system-spec points at that file. -EXTERNAL_TYPES: Tuple[str, ...] = ("PR", "SR") +#: Defined in the system-level SPEC.md that ``jray-project`` owns. Recognised +#: in tags everywhere, never counted in any component's fraction, and +#: orphan-checked only when ``system_spec`` points at that file. +DEFAULT_EXTERNAL_TYPES: Tuple[str, ...] = ("PR", "SR") -KNOWN_TYPES: Tuple[str, ...] = LOCAL_TYPES + TEST_TYPES + EXTERNAL_TYPES +#: Tiers a CI host can actually run. The extraction pipeline's CI is a GPU-less +#: Intel N100, so its T4 is excluded; jRay numbers its live-only tier T4 for +#: exactly this reason rather than renumbering the shared constant. +DEFAULT_CI_EXECUTABLE_TIERS: FrozenSet[str] = frozenset({"T1", "T2", "T3", "static"}) -#: Tiers a GPU-less CI host can actually run. See the "Verification strategy" -#: section of docs/requirements.md. -CI_EXECUTABLE_TIERS: Set[str] = {"T1", "T2", "T3", "static"} +DEFAULT_EXCLUDE_DIRS: FrozenSet[str] = frozenset({ + ".git", "__pycache__", "external", "vendor", "build", "site", "dist", + "node_modules", "target", "bin", "obj", "trt_cache", "ort_cache", + ".venv", "venv", ".mypy_cache", ".pytest_cache", +}) -# -------------------------------------------------------------------------- -# Source scanning -# -------------------------------------------------------------------------- +DEFAULT_REQUIREMENTS_PATH = "docs/requirements.md" +DEFAULT_MATRIX_PATH = "docs/traceability.md" +DEFAULT_JSON_PATH = "traces-report.json" +CONFIG_FILENAME = "traceability.toml" -SOURCE_SUFFIXES = { - ".cpp", ".cc", ".cxx", ".hpp", ".hxx", ".h", ".cu", ".cuh", # C++ - ".py", # Python +#: Named suffix groups, so a repo declares "Rust" rather than remembering to +#: spell every extension. Additive with an explicit ``source_suffixes`` list. +LANGUAGE_SUFFIXES: Dict[str, FrozenSet[str]] = { + "cpp": frozenset({".cpp", ".cc", ".cxx", ".hpp", ".hxx", ".h", ".cu", ".cuh"}), + "python": frozenset({".py", ".pyi"}), + "rust": frozenset({".rs"}), + "csharp": frozenset({".cs"}), + "javascript": frozenset({".js", ".mjs", ".cjs", ".jsx"}), + "typescript": frozenset({".ts", ".tsx"}), + "svelte": frozenset({".svelte"}), + "go": frozenset({".go"}), + "java": frozenset({".java"}), } -SCAN_ROOTS: Tuple[str, ...] = ("src", "tests", "scripts", "experiments", "eval") -EXCLUDED_DIR_NAMES = { - ".git", "__pycache__", "external", "build", "site", "node_modules", - "trt_cache", "ort_cache", ".venv", "venv", ".mypy_cache", ".pytest_cache", +# -------------------------------------------------------------------------- +# Configuration +# -------------------------------------------------------------------------- + +class ConfigError(Exception): + """A configuration mistake, reported rather than papered over.""" + + +@dataclass(frozen=True) +class Config: + """Everything that differs between component repos. + + The three required fields are exactly the three things that were hardcoded + before this tool was shared: which ID prefixes the repo's register defines, + which files count as source, and where those files live. A repo that gets + any of them wrong parses zero requirements or scans zero files — and the + gate refuses to report rather than printing a misleading 0%. + """ + + requirement_types: Tuple[str, ...] + source_suffixes: FrozenSet[str] + source_roots: Tuple[str, ...] + + root: Path = Path(".") + test_types: Tuple[str, ...] = DEFAULT_TEST_TYPES + external_types: Tuple[str, ...] = DEFAULT_EXTERNAL_TYPES + exclude_dirs: FrozenSet[str] = DEFAULT_EXCLUDE_DIRS + ci_executable_tiers: FrozenSet[str] = DEFAULT_CI_EXECUTABLE_TIERS + requirements_path: str = DEFAULT_REQUIREMENTS_PATH + system_spec_path: Optional[str] = None + matrix_path: Optional[str] = DEFAULT_MATRIX_PATH + json_path: Optional[str] = DEFAULT_JSON_PATH + min_coverage: float = 0.0 + allow_orphans: bool = False + source: str = "defaults" + + @property + def known_types(self) -> Tuple[str, ...]: + return (tuple(self.requirement_types) + tuple(self.test_types) + + tuple(self.external_types)) + + def resolve(self, relative: Optional[str]) -> Optional[Path]: + """A configured path, made absolute against the repo root.""" + if relative is None: + return None + path = Path(relative) + return path if path.is_absolute() else (self.root / path) + + def validate(self) -> None: + problems: List[str] = [] + if not self.requirement_types: + problems.append( + "requirement_types is empty - the register defines IDs with " + "some prefix (AR/DP/IR/GR/VR in extraction, JR in jRay, UR/DR " + "in the server) and the tool cannot guess it") + for name, values in (("requirement_types", self.requirement_types), + ("test_types", self.test_types), + ("external_types", self.external_types)): + for value in values: + if not re.fullmatch(r"[A-Z]{2}", value): + problems.append( + f"{name}: {value!r} is not a two-letter uppercase " + "prefix; IDs are PREFIX-nnn") + overlaps = ((set(self.requirement_types) & set(self.test_types)) + | (set(self.requirement_types) & set(self.external_types))) + if overlaps: + problems.append( + f"prefixes {sorted(overlaps)} are declared both as " + "requirement_types and as another taxonomy; a prefix counted " + "in the fraction cannot also be excluded from it") + if not self.source_suffixes: + problems.append( + 'no source suffixes - set `languages` (e.g. ["rust"]) or ' + '`source_suffixes` (e.g. [".rs"])') + for suffix in sorted(self.source_suffixes): + if not suffix.startswith("."): + problems.append(f"source suffix {suffix!r} must start with a dot") + if not self.source_roots: + problems.append( + "no source roots - set `source_roots` to the directories " + "holding this repo's code") + if not 0.0 <= self.min_coverage <= 100.0: + problems.append( + f"min_coverage {self.min_coverage} is not a percentage") + if problems: + raise ConfigError("; ".join(problems)) + + +CONFIG_KEYS = { + "requirement_types", "test_types", "external_types", "languages", + "source_suffixes", "source_roots", "exclude_dirs", "ci_executable_tiers", + "requirements", "system_spec", "matrix", "json_report", "min_coverage", + "allow_orphans", } + +def example_config() -> str: + """The full schema, emitted by ``--print-example-config``.""" + return """\ +# traceability.toml - per-repo configuration for the shared trace extractor. +# Lives at the component repo root; its directory is taken as the repo root. + +# REQUIRED. The ID prefixes this repo's register defines. These, and only +# these, form the coverage fraction. +# scene-actor-extraction: ["AR", "DP", "IR", "GR", "VR"] +# jRay: ["JR"] +# JRay-public-server: ["UR", "DR"] +requirement_types = ["AR", "DP", "IR", "GR", "VR"] + +# REQUIRED (at least one of the two). `languages` names suffix groups; +# `source_suffixes` adds anything else. Known groups: cpp, python, rust, +# csharp, javascript, typescript, svelte, go, java. +languages = ["cpp", "python"] +# source_suffixes = [".inl"] + +# REQUIRED. Directories to scan, relative to the repo root. +source_roots = ["src", "tests", "scripts"] + +# Optional; defaults shown. +# test_types = ["UT", "IT"] # evidence for requirements, not counted +# external_types = ["PR", "SR"] # owned by the system spec, not counted +# requirements = "docs/requirements.md" +# matrix = "docs/traceability.md" +# json_report = "traces-report.json" +# min_coverage = 0.0 +# allow_orphans = false +# exclude_dirs = ["external", "build", "target", "node_modules"] + +# Which verification tiers this repo's CI host can actually execute. A +# requirement whose tiers all fall outside this set is reported as tagged but +# unexecuted, and is never counted as covered. +# ci_executable_tiers = ["T1", "T2", "T3", "static"] + +# The system spec defining PR/SR, vendored as a submodule in each component so +# the check is runnable in CI. Omit to skip PR/SR orphan checking. +# system_spec = "scripts/vendor/jray-project/SPEC.md" +""" + + +def find_config(start: Path) -> Optional[Path]: + """Nearest ``traceability.toml`` at or above ``start``. + + The file's directory defines the repo root, which is what lets the tool be + run from a subdirectory without silently scanning the wrong tree. + """ + current = start.resolve() + for candidate in (current, *current.parents): + path = candidate / CONFIG_FILENAME + if path.is_file(): + return path + return None + + +def _load_toml(path: Path) -> dict: + try: + import tomllib + except ModuleNotFoundError as exc: # pragma: no cover - version dependent + raise ConfigError( + f"reading {path} needs Python 3.11+ for tomllib (this is " + f"{sys.version_info.major}.{sys.version_info.minor}). Either " + "upgrade, or pass --requirement-type/--language/--source-root on " + "the command line instead of using a config file.") from exc + with path.open("rb") as handle: + return tomllib.load(handle) + + +def config_from_dict(data: dict, root: Path, source: str) -> Config: + unknown = sorted(set(data) - CONFIG_KEYS) + if unknown: + # A typo'd key would otherwise leave a required field empty, and the + # user would be debugging "zero requirements" instead of a spelling. + raise ConfigError( + f"unknown key(s) in {source}: {', '.join(unknown)}. " + f"Known keys: {', '.join(sorted(CONFIG_KEYS))}") + + suffixes: Set[str] = set() + for language in data.get("languages", []): + if language not in LANGUAGE_SUFFIXES: + raise ConfigError( + f"unknown language {language!r} in {source}. Known: " + f"{', '.join(sorted(LANGUAGE_SUFFIXES))}") + suffixes |= LANGUAGE_SUFFIXES[language] + suffixes |= set(data.get("source_suffixes", [])) + + return Config( + requirement_types=tuple(data.get("requirement_types", ())), + source_suffixes=frozenset(suffixes), + source_roots=tuple(data.get("source_roots", ())), + root=root, + test_types=tuple(data.get("test_types", DEFAULT_TEST_TYPES)), + external_types=tuple(data.get("external_types", DEFAULT_EXTERNAL_TYPES)), + exclude_dirs=frozenset(data.get("exclude_dirs", DEFAULT_EXCLUDE_DIRS)), + ci_executable_tiers=frozenset( + data.get("ci_executable_tiers", DEFAULT_CI_EXECUTABLE_TIERS)), + requirements_path=data.get("requirements", DEFAULT_REQUIREMENTS_PATH), + system_spec_path=data.get("system_spec"), + matrix_path=data.get("matrix", DEFAULT_MATRIX_PATH), + json_path=data.get("json_report", DEFAULT_JSON_PATH), + min_coverage=float(data.get("min_coverage", 0.0)), + allow_orphans=bool(data.get("allow_orphans", False)), + source=source, + ) + + +def load_config(args: argparse.Namespace, cwd: Optional[Path] = None) -> Config: + """Merge config file and CLI flags, per field. Flags win.""" + cwd = (cwd or Path.cwd()).resolve() + + config_path: Optional[Path] = args.config + if config_path is None and not args.no_config: + config_path = find_config(cwd) + if config_path is not None and not Path(config_path).is_file(): + raise ConfigError(f"config file not found: {config_path}") + + if config_path is not None: + config_path = Path(config_path).resolve() + config = config_from_dict(_load_toml(config_path), config_path.parent, + str(config_path)) + else: + config = Config(requirement_types=(), source_suffixes=frozenset(), + source_roots=(), root=cwd, source="command line only") + + if args.root is not None: + config = replace(config, root=Path(args.root).resolve()) + + if args.requirement_type: + config = replace(config, requirement_types=tuple(args.requirement_type)) + if args.test_type: + config = replace(config, test_types=tuple(args.test_type)) + if args.external_type: + config = replace(config, external_types=tuple(args.external_type)) + + # Languages and suffixes are additive on top of whatever the file declared, + # so `--language rust` extends rather than silently replacing. + extra: Set[str] = set() + for language in args.language or []: + if language not in LANGUAGE_SUFFIXES: + raise ConfigError( + f"unknown language {language!r}. Known: " + f"{', '.join(sorted(LANGUAGE_SUFFIXES))}") + extra |= LANGUAGE_SUFFIXES[language] + extra |= set(args.source_suffix or []) + if extra: + config = replace(config, source_suffixes=config.source_suffixes | extra) + + if args.source_root: + config = replace(config, source_roots=tuple(args.source_root)) + if args.exclude_dir: + config = replace(config, + exclude_dirs=config.exclude_dirs | set(args.exclude_dir)) + if args.ci_executable_tier: + config = replace(config, + ci_executable_tiers=frozenset(args.ci_executable_tier)) + if args.requirements is not None: + config = replace(config, requirements_path=str(args.requirements)) + if args.system_spec is not None: + config = replace(config, system_spec_path=str(args.system_spec)) + if args.markdown_out is not None: + config = replace(config, matrix_path=str(args.markdown_out)) + if args.json_out is not None: + config = replace(config, json_path=str(args.json_out)) + if args.no_write: + config = replace(config, matrix_path=None, json_path=None) + if args.min_coverage is not None: + config = replace(config, min_coverage=float(args.min_coverage)) + if args.allow_orphans: + config = replace(config, allow_orphans=True) + + config.validate() + return config + + +# -------------------------------------------------------------------------- +# Tag parsing +# -------------------------------------------------------------------------- + TRACES_RE = re.compile(r"TRACES:[ \t]*([^\n]*)") EXCEPTION_RE = re.compile(r"EXCEPTION:[ \t]*([A-Z]{2}-\d{3})[ \t]*([^\n]*)") REQ_ID_RE = re.compile(r"^([A-Z]{2})-(\d{3})$") LEADING_ID_RE = re.compile(r"^([A-Z]{2}-\d{3})\b(.*)$") -#: Something that was *trying* to be a requirement ID. Used to keep the -#: malformed-tag diagnostic quiet when the word "TRACES" merely appears in -#: prose or in this tool's own source, while still catching `AR-12`. +#: Something that was *trying* to be a requirement ID. Keeps the malformed-tag +#: diagnostic quiet when the word "TRACES" merely appears in prose or in this +#: tool's own source, while still catching `AR-12`. ID_ATTEMPT_RE = re.compile(r"[A-Za-z]{2}-\d") -# Comment terminators that can trail a tag on the same line. +#: Comment terminators that can trail a tag on the same line. COMMENT_TERMINATORS = ("*/", "-->", '"""', "'''") DECL_PATTERNS = [ - re.compile(r"^\s*(?:async\s+)?def\s+\w+"), - re.compile(r"^\s*class\s+\w+"), + re.compile(r"^\s*(?:async\s+)?def\s+\w+"), # Python + re.compile(r"^\s*(?:pub(?:\([^)]*\))?\s+)?(?:async\s+)?" + r"(?:unsafe\s+)?(?:extern\s+\"[^\"]*\"\s+)?fn\s+\w+"), # Rust + re.compile(r"^\s*(?:pub(?:\([^)]*\))?\s+)?" + r"(?:impl|trait|mod|type)\b"), # Rust re.compile(r"^\s*(?:template\s*<[^;]*>\s*)?" - r"(?:struct|class|enum(?:\s+class)?|union|namespace)\s+\w+"), - re.compile(r"^\s*(?:static|inline|constexpr|virtual|explicit|friend)\b"), - re.compile(r"^\s*[A-Za-z_][\w:<>,\s\*&]*\s+[A-Za-z_~][\w:]*\s*\([^)]*\)"), + r"(?:struct|class|enum(?:\s+class)?|union|namespace|" + r"interface|record)\s+\w+"), # C++/C# + re.compile(r"^\s*(?:public|private|protected|internal|static|inline|" + r"constexpr|virtual|explicit|friend|override|abstract|sealed|" + r"partial|extern)\b"), + re.compile(r"^\s*[A-Za-z_][\w:<>,\s\*&\.\[\]]*\s+[A-Za-z_~][\w:]*\s*\([^)]*\)"), ] +#: Attribute/annotation lines: C# ``[HttpGet("x")]``, Rust ``#[derive(...)]``, +#: Java ``@Override``. A tag above a decorated declaration must attribute to +#: the declaration, not to its decoration. +ATTRIBUTE_RE = re.compile(r"^\s*(?:\[[^\]]*\]|#\s*\[|@\w+)") + @dataclass class TraceEntry: @@ -221,12 +529,16 @@ def parse_traces_tag(value: str) -> Tuple[List[List[str]], List[str]]: def find_context(lines: Sequence[str], index: int, window: int = 12) -> str: """Best-effort name of the declaration a tag belongs to. - Searches both directions, because the two languages put the tag on opposite - sides of the thing it describes: a C++ ``/// TRACES:`` sits *above* the + Searches both directions, because languages put the tag on opposite sides + of the thing it describes: a C++/C#/Rust ``///`` tag sits *above* the declaration, while a Python tag usually sits *inside* the docstring, below - the ``def``. Whichever declaration is nearer wins. + the ``def``. Whichever declaration is nearer wins. Attribute lines + (``[HttpGet]``, ``#[derive]``) are skipped so a tag above a decorated + method attributes to the method rather than to its decoration. """ def matches(line: str) -> bool: + if ATTRIBUTE_RE.match(line): + return False return any(p.match(line) for p in DECL_PATTERNS) down: Optional[Tuple[int, str]] = None @@ -247,7 +559,6 @@ def find_context(lines: Sequence[str], index: int, window: int = 12) -> str: up = (offset, lines[i]) break - best = None if down and up: best = down if down[0] <= up[0] else up else: @@ -257,20 +568,19 @@ def find_context(lines: Sequence[str], index: int, window: int = 12) -> str: return best[1].strip().rstrip("{").strip()[:120] or "Unknown" -def iter_source_files(root: Path, - scan_roots: Iterable[str] = SCAN_ROOTS) -> List[Path]: - """Every C++/Python file under the configured scan roots.""" +def iter_source_files(config: Config) -> List[Path]: + """Every source file under the configured roots, by configured suffix.""" found: List[Path] = [] - for rel in scan_roots: - base = root / rel + for rel in config.source_roots: + base = config.root / rel if not base.is_dir(): continue for path in sorted(base.rglob("*")): if not path.is_file(): continue - if path.suffix not in SOURCE_SUFFIXES: + if path.suffix not in config.source_suffixes: continue - if any(part in EXCLUDED_DIR_NAMES for part in path.parts): + if any(part in config.exclude_dirs for part in path.parts): continue found.append(path) return found @@ -284,10 +594,12 @@ class ScanResult: diagnostics: Diagnostics -def scan_files(files: Sequence[Path], root: Path) -> ScanResult: +def scan_files(files: Sequence[Path], config: Config) -> ScanResult: traces: List[TraceEntry] = [] exceptions: List[ExceptionEntry] = [] diags = Diagnostics() + root = config.root + known = set(config.known_types) for path in files: try: @@ -306,14 +618,12 @@ def scan_files(files: Sequence[Path], root: Path) -> ScanResult: if cut != -1: reason = reason[:cut] reason = reason.strip() - entry = ExceptionEntry( + exceptions.append(ExceptionEntry( file=rel, line=index + 1, requirement=exc.group(1), - reason=reason, context=find_context(lines, index), - ) - exceptions.append(entry) + reason=reason, context=find_context(lines, index))) if not reason: - # CLAUDE.md: an exception is only agreed if the reason is - # recorded. An unexplained one is an undocumented defect. + # An exception is only *agreed* if the reason is recorded; + # an unexplained one is an undocumented defect. diags.exceptions_without_reason.append( {"file": rel, "line": index + 1, "requirement": exc.group(1)}) @@ -326,12 +636,13 @@ def scan_files(files: Sequence[Path], root: Path) -> ScanResult: ids = [i for g in groups for i in g] if not ids: # No IDs at all. Only worth reporting if something in the text - # was trying to be one — otherwise every sentence containing + # was trying to be one - otherwise every sentence containing # the word would be flagged, and a diagnostic nobody can act on # is how real diagnostics get ignored. if junk and ID_ATTEMPT_RE.search(match.group(1)): diags.malformed_tags.append( - {"file": rel, "line": index + 1, "text": match.group(1).strip()}) + {"file": rel, "line": index + 1, + "text": match.group(1).strip()}) continue if junk: diags.malformed_tags.append( @@ -344,11 +655,10 @@ def scan_files(files: Sequence[Path], root: Path) -> ScanResult: diags.mixed_type_groups.append( {"file": rel, "line": index + 1, "group": list(group)}) for req in ids: - if req.split("-")[0] not in KNOWN_TYPES: + if req.split("-")[0] not in known: diags.unknown_id_types.append( {"file": rel, "line": index + 1, "id": req}) - # Deduplicate within one tag while preserving order. seen: Set[str] = set() unique = [i for i in ids if not (i in seen or seen.add(i))] traces.append(TraceEntry(file=rel, line=index + 1, @@ -360,7 +670,7 @@ def scan_files(files: Sequence[Path], root: Path) -> ScanResult: # -------------------------------------------------------------------------- -# The register: docs/requirements.md is the authoritative denominator +# The register: requirements.md is the authoritative denominator # -------------------------------------------------------------------------- @dataclass @@ -372,17 +682,6 @@ class Requirement: status: str = "" tiers: Set[str] = field(default_factory=set) - @property - def ci_executable(self) -> bool: - """True when at least one verification tier can run on the CI host. - - A requirement with no tier recorded is *unknown*, not unexecutable — it - is reported so the register gets fixed, and it is not penalised here. - """ - if not self.tiers: - return True - return bool(self.tiers & CI_EXECUTABLE_TIERS) - @property def tier_known(self) -> bool: return bool(self.tiers) @@ -390,6 +689,10 @@ class Requirement: @dataclass class Register: + #: The ID prefixes this register defines, from configuration. + types: Tuple[str, ...] = () + #: Verification tiers the consuming repo's CI host can execute. + ci_tiers: FrozenSet[str] = DEFAULT_CI_EXECUTABLE_TIERS requirements: Dict[str, Requirement] = field(default_factory=dict) withdrawn: Dict[str, Requirement] = field(default_factory=dict) @@ -401,14 +704,35 @@ class Register: def total(self) -> int: return len(self.requirements) + @property + def uses_tiers(self) -> bool: + """Whether this register assigns verification tiers at all. + + A repo with no tier tables is not 'missing' tiers; it does not use the + mechanism. Warning about every requirement there would be noise that + trains people to ignore the warning that matters. + """ + return any(r.tier_known for r in self.requirements.values()) + + def is_ci_executable(self, req_id: str) -> bool: + """True when at least one verification tier can run on the CI host. + + A requirement with no tier recorded is *unknown*, not unexecutable - it + is reported so the register gets fixed, and it is not penalised here. + """ + req = self.requirements[req_id] + if not req.tiers: + return True + return bool(req.tiers & self.ci_tiers) + def count(self, req_type: str) -> int: return sum(1 for i in self.requirements if i.startswith(req_type + "-")) def ci_executable_ids(self) -> Set[str]: - return {i for i, r in self.requirements.items() if r.ci_executable} + return {i for i in self.requirements if self.is_ci_executable(i)} def unexecutable_ids(self) -> Set[str]: - return {i for i, r in self.requirements.items() if not r.ci_executable} + return {i for i in self.requirements if not self.is_ci_executable(i)} def tier_unknown_ids(self) -> Set[str]: return {i for i, r in self.requirements.items() if not r.tier_known} @@ -418,7 +742,7 @@ class Register: SPEC.md section 6 asks the gate to report these: a requirement serving no stated goal is scope creep, and it is invisible unless something - looks. A cell naming a section (`§4`) counts as a parent — the point is + looks. A cell naming a section (`§4`) counts as a parent - the point is that *something* was recorded, not that it was an ID. """ out = set() @@ -430,7 +754,11 @@ class Register: def _row_cells(line: str) -> List[str]: - return [c.strip() for c in line.strip().strip("|").split("|")] + r"""Split a markdown table row, honouring escaped ``\|`` inside cells.""" + placeholder = "\x00" + stripped = line.strip().replace("\\|", placeholder) + cells = stripped.strip("|").split("|") + return [c.strip().replace(placeholder, "|") for c in cells] def _is_separator(cells: Sequence[str]) -> bool: @@ -466,7 +794,7 @@ def _is_register_header(header: Sequence[str]) -> bool: Deliberately narrow. The per-requirement verification plan is also keyed on ``ID`` but has no ``Requirement`` column, and the tier summary table's first - column holds comma lists and ranges — neither defines requirements, and + column holds comma lists and ranges - neither defines requirements, and counting their rows would inflate the denominator. Prose mentions and ``Traces to`` references are excluded for the same reason: a naive scan for ``AR-\\d{3}`` over the whole file counts every reference as a definition. @@ -478,7 +806,7 @@ def _is_register_header(header: Sequence[str]) -> bool: def _is_tier_header(header: Sequence[str]) -> bool: - """Either of the two tables that assign verification tiers.""" + """Either of the two table shapes that assign verification tiers.""" if len(header) < 2: return False lowered = [c.lower() for c in header] @@ -497,7 +825,7 @@ def _column(header: Sequence[str], name: str, cells: Sequence[str]) -> str: def parse_tiers(cell: str) -> Set[str]: """Read a tier cell such as ``T1 + T4``, ``**T2**``, or ``Out of CI``.""" text = cell.replace("*", "").strip().lower() - tiers = {"T" + m.group(1) for m in re.finditer(r"\bt([1-4])\b", text)} + tiers = {"T" + m.group(1) for m in re.finditer(r"\bt(\d)\b", text)} if "out of ci" in text or "not in ci" in text: tiers.add("out-of-ci") if "manual" in text: @@ -510,7 +838,7 @@ def parse_tiers(cell: str) -> Set[str]: def expand_id_spec(spec: str, defined: Set[str]) -> List[str]: """Expand the ID cell of a tier table into concrete, defined IDs. - Handles every shape the register actually uses: ``AR-002``, + Handles every shape the registers actually use: ``AR-002``, ``AR-001, AR-005, AR-006``, ``AR-007 ... AR-017`` (with the unicode ellipsis), ``AR-009/010``, and ``DP-*``. Expansion is intersected with the defined set, so a range can never invent a requirement that does not exist. @@ -546,13 +874,15 @@ def expand_id_spec(spec: str, defined: Set[str]) -> List[str]: return [i for i in out if i in defined] -def parse_register(markdown: str) -> Register: - """Build the register from ``docs/requirements.md``. +def parse_register(markdown: str, config: Config) -> Register: + """Build the register from the repo's requirements.md. Two passes: definitions first, because tier rows use ranges and wildcards that can only be expanded against a known ID set. """ - register = Register() + register = Register(types=tuple(config.requirement_types), + ci_tiers=frozenset(config.ci_executable_tiers)) + definable = set(config.requirement_types) | set(config.test_types) for header, cells in _iter_table_rows(markdown): if not _is_register_header(header) or not cells: @@ -560,7 +890,7 @@ def parse_register(markdown: str) -> Register: first = cells[0].replace("*", "").strip() if not REQ_ID_RE.match(first): continue - if first.split("-")[0] not in LOCAL_TYPES + TEST_TYPES: + if first.split("-")[0] not in definable: continue req = Requirement( id=first, @@ -594,8 +924,10 @@ def parse_register(markdown: str) -> Register: return register -def read_register(path: Path) -> Register: - return parse_register(path.read_text(encoding="utf-8")) +def read_register(config: Config) -> Register: + path = config.resolve(config.requirements_path) + assert path is not None + return parse_register(path.read_text(encoding="utf-8"), config) # -------------------------------------------------------------------------- @@ -633,22 +965,22 @@ def compute_coverage(traced_ids: Iterable[str], register: Register) -> Coverage: Three exclusions, each of which is a way the number could otherwise lie: * Using the raw traced count as the numerator is what lets a ratio exceed - 100% — a tag naming a deleted or mistyped requirement would count as + 100% - a tag naming a deleted or mistyped requirement would count as covered. Those land in ``orphaned`` instead, so they get fixed rather than silently counted or silently dropped. - * A requirement whose only verification tier is T4/GPU cannot run on this - CI host at all. It is reported in ``unexecuted`` and is not covered: - treating "has a test that never runs" as passing reports success the gate - cannot substantiate. - * UT/IT test IDs and the system-level PR/SR IDs are different taxonomies - with their own registers, so they neither count nor orphan here. + * A requirement whose verification tiers all fall outside what the CI host + can execute cannot be verified there at all. It is reported in + ``unexecuted`` and is not covered: treating "has a test that never runs" + as passing reports success the gate cannot substantiate. + * Test IDs and system-level IDs are separate taxonomies with their own + registers, so they neither count nor orphan here. """ - traced = {i for i in traced_ids if i.split("-")[0] in LOCAL_TYPES} + traced = {i for i in traced_ids if i.split("-")[0] in register.types} defined = register.ids orphaned = sorted(traced - defined) matched = traced & defined - unexecuted = sorted(i for i in matched if not register.requirements[i].ci_executable) + unexecuted = sorted(i for i in matched if not register.is_ci_executable(i)) covered = sorted(matched - set(unexecuted)) total = len(defined) @@ -675,28 +1007,26 @@ def compute_coverage(traced_ids: Iterable[str], register: Register) -> Coverage: @dataclass class Report: timestamp: str - root: Path + config: Config register: Register scan: ScanResult coverage: Coverage by_type: Dict[str, List[str]] requirement_map: Dict[str, List[TraceEntry]] external_orphans: List[str] = field(default_factory=list) - #: Policy, set by the caller: whether an orphan tag fails the run. - allow_orphans: bool = False -def build_report(root: Path, register: Register, scan: ScanResult, +def build_report(config: Config, register: Register, scan: ScanResult, system_ids: Optional[Set[str]] = None) -> Report: requirement_map: Dict[str, List[TraceEntry]] = {} - seen_by_type: Dict[str, Set[str]] = {t: set() for t in KNOWN_TYPES} + seen_by_type: Dict[str, Set[str]] = {t: set() for t in config.known_types} seen_by_type["OTHER"] = set() for entry in scan.traces: for req in entry.requirements: requirement_map.setdefault(req, []).append(entry) prefix = req.split("-")[0] - seen_by_type[prefix if prefix in KNOWN_TYPES else "OTHER"].add(req) + seen_by_type[prefix if prefix in seen_by_type else "OTHER"].add(req) coverage = compute_coverage(requirement_map.keys(), register) @@ -704,11 +1034,11 @@ def build_report(root: Path, register: Register, scan: ScanResult, if system_ids is not None: external_orphans = sorted( i for i in requirement_map - if i.split("-")[0] in EXTERNAL_TYPES and i not in system_ids) + if i.split("-")[0] in config.external_types and i not in system_ids) return Report( timestamp=datetime.now(timezone.utc).isoformat(timespec="seconds"), - root=root, + config=config, register=register, scan=scan, coverage=coverage, @@ -718,16 +1048,20 @@ def build_report(root: Path, register: Register, scan: ScanResult, ) -def parse_system_spec(markdown: str) -> Set[str]: - """IDs of PR/SR requirements defined in the umbrella SPEC.md. +def parse_system_spec(markdown: str, + types: Sequence[str] = DEFAULT_EXTERNAL_TYPES) -> Set[str]: + """IDs of the system requirements defined in the umbrella SPEC.md. - They are defined as headings (``### SR-001 - ...``) and as bolded leading - table cells (``| **PR-001** | ... |``), so accept both. + They appear as headings (``### SR-001 - ...``) and as bolded leading table + cells (``| **PR-001** | ... |``), so accept both. """ + alternation = "|".join(re.escape(t) for t in types) + heading_re = re.compile(rf"^#{{1,6}}\s+\**((?:{alternation})-\d{{3}})\**\b") + cell_re = re.compile(rf"(?:{alternation})-\d{{3}}") ids: Set[str] = set() for line in markdown.split("\n"): stripped = line.strip() - heading = re.match(r"^#{1,6}\s+\**((?:PR|SR)-\d{3})\**\b", stripped) + heading = heading_re.match(stripped) if heading: ids.add(heading.group(1)) continue @@ -735,50 +1069,11 @@ def parse_system_spec(markdown: str) -> Set[str]: cells = _row_cells(stripped) if cells: first = cells[0].replace("*", "").strip() - if re.fullmatch(r"(?:PR|SR)-\d{3}", first): + if cell_re.fullmatch(first): ids.add(first) return ids -def report_to_json_dict(report: Report) -> dict: - register = report.register - defined = {t: register.count(t) for t in LOCAL_TYPES} - defined["total"] = register.total - - requirement_detail = {} - for req_id, req in sorted(register.requirements.items()): - entries = report.requirement_map.get(req_id, []) - requirement_detail[req_id] = { - "requirement": req.text, - "tracesTo": req.traces_to, - "priority": req.priority, - "status": req.status, - "tiers": sorted(req.tiers), - "ciExecutable": req.ci_executable, - "taggedIn": sorted({e.file for e in entries}), - "state": _requirement_state(req, bool(entries)), - } - - return { - "timestamp": report.timestamp, - "totalFiles": len(report.scan.files), - "totalTraces": len(report.scan.traces), - "totalExceptions": len(report.scan.exceptions), - "requirements": {k: [e.to_dict() for e in v] - for k, v in sorted(report.requirement_map.items())}, - "byType": report.by_type, - "defined": defined, - "coverage": report.coverage.to_dict(), - "gpuOnlyRequirements": sorted(register.unexecutable_ids()), - "parentlessRequirements": sorted(register.parentless_ids()), - "withdrawn": sorted(register.withdrawn), - "exceptions": [e.to_dict() for e in report.scan.exceptions], - "externalOrphans": report.external_orphans, - "diagnostics": report.scan.diagnostics.to_dict(), - "requirementDetail": requirement_detail, - } - - def per_type_stats(report: Report) -> Dict[str, Tuple[int, int, int]]: """``{type: (covered, unexecuted, defined)}``. @@ -790,7 +1085,7 @@ def per_type_stats(report: Report) -> Dict[str, Tuple[int, int, int]]: covered = set(report.coverage.covered) unexecuted = set(report.coverage.unexecuted) stats: Dict[str, Tuple[int, int, int]] = {} - for req_type in LOCAL_TYPES: + for req_type in report.register.types: prefix = req_type + "-" stats[req_type] = ( len([i for i in covered if i.startswith(prefix)]), @@ -800,36 +1095,93 @@ def per_type_stats(report: Report) -> Dict[str, Tuple[int, int, int]]: return stats -def _requirement_state(req: Requirement, tagged: bool) -> str: +def _requirement_state(register: Register, req_id: str, tagged: bool) -> str: if not tagged: return "untagged" - if not req.ci_executable: + if not register.is_ci_executable(req_id): return "tagged-unexecuted" return "covered" +def report_to_json_dict(report: Report) -> dict: + register = report.register + config = report.config + defined = {t: register.count(t) for t in register.types} + defined["total"] = register.total + + requirement_detail = {} + for req_id, req in sorted(register.requirements.items()): + entries = report.requirement_map.get(req_id, []) + requirement_detail[req_id] = { + "requirement": req.text, + "tracesTo": req.traces_to, + "priority": req.priority, + "status": req.status, + "tiers": sorted(req.tiers), + "ciExecutable": register.is_ci_executable(req_id), + "taggedIn": sorted({e.file for e in entries}), + "state": _requirement_state(register, req_id, bool(entries)), + } + + return { + "timestamp": report.timestamp, + # Echoed so a report can be read without guessing how it was produced - + # a shared tool's output is ambiguous otherwise. + "config": { + "source": config.source, + "root": str(config.root), + "requirementTypes": list(config.requirement_types), + "testTypes": list(config.test_types), + "externalTypes": list(config.external_types), + "sourceSuffixes": sorted(config.source_suffixes), + "sourceRoots": list(config.source_roots), + "requirements": config.requirements_path, + "systemSpec": config.system_spec_path, + "ciExecutableTiers": sorted(config.ci_executable_tiers), + "minCoverage": config.min_coverage, + }, + "totalFiles": len(report.scan.files), + "totalTraces": len(report.scan.traces), + "totalExceptions": len(report.scan.exceptions), + "requirements": {k: [e.to_dict() for e in v] + for k, v in sorted(report.requirement_map.items())}, + "byType": report.by_type, + "defined": defined, + "coverage": report.coverage.to_dict(), + "unexecutableRequirements": sorted(register.unexecutable_ids()), + "parentlessRequirements": sorted(register.parentless_ids()), + "withdrawn": sorted(register.withdrawn), + "exceptions": [e.to_dict() for e in report.scan.exceptions], + "externalOrphans": report.external_orphans, + "diagnostics": report.scan.diagnostics.to_dict(), + "requirementDetail": requirement_detail, + } + + # -------------------------------------------------------------------------- # Output formats # -------------------------------------------------------------------------- def generate_markdown(report: Report) -> str: register = report.register + config = report.config cov = report.coverage out: List[str] = [] add = out.append + req_link = Path(config.requirements_path).name + add("# Requirements traceability matrix") add("") add("") - add("") + add("") add("") add(f"**Generated:** {report.timestamp}") add("") - add("Denominators are read from [`requirements.md`](requirements.md) at run " - "time, never hardcoded. Coverage counts a requirement only when it is " - "tagged in source **and** has a verification tier this CI host can " - "execute — CI is an Intel N100 with no discrete GPU.") + add(f"Denominators are read from [`{req_link}`]({req_link}) at run time, " + "never hardcoded. Coverage counts a requirement only when it is tagged " + "in source **and** has a verification tier this repo's CI host can " + f"execute (`{', '.join(sorted(config.ci_executable_tiers))}`).") add("") add("## Summary") @@ -844,7 +1196,7 @@ def generate_markdown(report: Report) -> str: add(f"| **Coverage** | **{cov.percent}%** ({len(cov.covered)}/{cov.total}) |") add(f"| Coverage of CI-executable scope | {cov.ci_percent}% " f"({len(cov.covered)}/{cov.ci_executable_total}) |") - add(f"| Tagged but unexecuted in CI (T4/GPU) | {len(cov.unexecuted)} |") + add(f"| Tagged but unexecuted in CI | {len(cov.unexecuted)} |") add(f"| Orphan tags | {len(cov.orphaned)} |") add("") @@ -855,7 +1207,7 @@ def generate_markdown(report: Report) -> str: for req_type, (covered, unexecuted, defined_n) in per_type_stats(report).items(): add(f"| {req_type} | {covered} | {unexecuted} | {defined_n} |") add("") - for req_type in TEST_TYPES + EXTERNAL_TYPES: + for req_type in tuple(config.test_types) + tuple(config.external_types): tagged = report.by_type.get(req_type, []) if tagged: add(f"- **{req_type}** tags present (separate taxonomy, not counted " @@ -865,9 +1217,9 @@ def generate_markdown(report: Report) -> str: unexecutable = sorted(register.unexecutable_ids()) add("## Not executable in CI") add("") - add("CI runs on an Intel N100 with no discrete GPU. These requirements have " - "no verification tier that can run here, so a tag on them is evidence " - "of *intent*, not of verification. They are never counted as covered.") + add("These requirements have no verification tier this repo's CI host can " + "run, so a tag on them is evidence of *intent*, not of verification. " + "They are never counted as covered.") add("") if unexecutable: add("| ID | Tiers | Tagged in source | Requirement |") @@ -882,13 +1234,13 @@ def generate_markdown(report: Report) -> str: add("") if cov.unexecuted: add(f"**Tagged but unexecuted:** {', '.join(cov.unexecuted)} — a test " - "exists and is tagged, but only a GPU host can run it. Report " + "exists and is tagged, but this CI host cannot run it. Report " "those runs separately.") add("") add("## Orphan tags") add("") - add("A tag naming an ID `requirements.md` does not define. This is what " + add(f"A tag naming an ID `{req_link}` does not define. This is what " "renumbering produces, and what a typo produces.") add("") if cov.orphaned: @@ -909,16 +1261,13 @@ def generate_markdown(report: Report) -> str: "something looks.") add("") parentless = sorted(register.parentless_ids()) - if parentless: - add(", ".join(f"`{i}`" for i in parentless)) - else: - add("_None._") + add(", ".join(f"`{i}`" for i in parentless) if parentless else "_None._") add("") add("## Recorded exceptions") add("") add("Deliberate, documented departures from an invariant " - "(`EXCEPTION: AR-nnn `). Reported separately and never counted " + "(`EXCEPTION: XX-nnn `). Reported separately and never counted " "as coverage — an exception is a decision to be reviewed, not evidence " "a requirement is met.") add("") @@ -937,15 +1286,15 @@ def generate_markdown(report: Report) -> str: add("") add("| ID | Status | Tier | Traces to | Trace state | Tagged in | Requirement |") add("|---|---|---|---|---|---|---|") - for req_id, req in sorted(register.requirements.items(), - key=lambda kv: (LOCAL_TYPES.index(kv[0].split("-")[0]) - if kv[0].split("-")[0] in LOCAL_TYPES - else 99, kv[0])): + order = {t: n for n, t in enumerate(register.types)} + for req_id, req in sorted( + register.requirements.items(), + key=lambda kv: (order.get(kv[0].split("-")[0], 99), kv[0])): entries = report.requirement_map.get(req_id, []) state = {"covered": "covered", - "tagged-unexecuted": "tagged, unexecuted (T4/GPU)", + "tagged-unexecuted": "tagged, unexecuted", "untagged": "untagged"}[ - _requirement_state(req, bool(entries))] + _requirement_state(register, req_id, bool(entries))] files = ", ".join(f"`{f}`" for f in sorted({e.file for e in entries})) or "-" tiers = ", ".join(sorted(req.tiers)) or "unset" add(f"| {req_id} | {_truncate(req.status, 20) or '-'} | {tiers} " @@ -995,7 +1344,7 @@ def generate_markdown(report: Report) -> str: def _source_link(file_rel: str) -> str: - """Link from docs/traceability.md back to a source file at the repo root.""" + """Link from the matrix (in docs/) back to a source file at the repo root.""" return "../" + file_rel @@ -1007,9 +1356,10 @@ def _truncate(text: str, limit: int = 70) -> str: return text.replace("|", "\\|") -def format_coverage_report(report: Report, min_coverage: float) -> Tuple[str, int]: +def format_coverage_report(report: Report) -> Tuple[str, int]: """Human-readable gate output plus the exit code it implies.""" register = report.register + config = report.config cov = report.coverage lines: List[str] = [] add = lines.append @@ -1017,22 +1367,34 @@ def format_coverage_report(report: Report, min_coverage: float) -> Tuple[str, in add("Requirement traceability") add("=" * 72) - add(f"Source files scanned : {len(report.scan.files)}") + add(f"Config : {config.source}") + add(f"Repo root : {config.root}") + add(f"Requirement types : {', '.join(config.requirement_types)}") + add(f"Source files scanned : {len(report.scan.files)} " + f"({', '.join(sorted(config.source_suffixes))} under " + f"{', '.join(config.source_roots)})") add(f"TRACES tags found : {len(report.scan.traces)}") add(f"EXCEPTION tags found : {len(report.scan.exceptions)}") add("") - # Self-checks. With a low threshold these are what make the gate mean - # something: a parser that silently returns nothing would otherwise report - # 0/0 and pass. + # Self-checks. These are what make the gate mean something at a low + # threshold, and what makes a misconfigured shared tool loud instead of + # silent: a repo whose requirement_types or source_suffixes are wrong parses + # nothing and scans nothing, and a plausible-looking 0% would hide it. if register.total == 0: failures.append( - "requirements.md parsed to ZERO requirements - the register parser " - "is broken or the file moved. Refusing to report coverage.") + f"{config.requirements_path} parsed to ZERO requirements. Either " + "the path is wrong, the register parser is broken, or " + f"requirement_types ({', '.join(config.requirement_types) or 'none'}) " + "does not match the prefixes it defines. Refusing to report " + "coverage.") if not report.scan.files: failures.append( - "no source files were scanned - the scan roots do not exist. " - "Refusing to report coverage against an empty tree.") + "no source files were scanned. Either source_roots " + f"({', '.join(config.source_roots) or 'none'}) do not exist, or the " + f"suffixes ({', '.join(sorted(config.source_suffixes)) or 'none'}) " + "do not match this repo's languages. Refusing to report coverage " + "against an empty tree.") add("Coverage by type (covered / defined):") for req_type, (covered, unexecuted, defined_n) in per_type_stats(report).items(): @@ -1042,18 +1404,18 @@ def format_coverage_report(report: Report, min_coverage: float) -> Tuple[str, in add(f"Overall : {len(cov.covered)} / {cov.total} ({cov.percent}%)") add(f"CI scope : {len(cov.covered)} / {cov.ci_executable_total} " f"({cov.ci_percent}%) [excludes {cov.total - cov.ci_executable_total} " - "requirement(s) no GPU-less host can verify]") + "requirement(s) this CI host cannot verify]") add("") unexecutable = sorted(register.unexecutable_ids()) if unexecutable: - add(f"Not executable on this CI host (T4/GPU or out-of-CI): " - f"{len(unexecutable)}") + add("Not executable on this CI host (tiers outside " + f"{', '.join(sorted(config.ci_executable_tiers))}): {len(unexecutable)}") add(f" {', '.join(unexecutable)}") if cov.unexecuted: add("") - add("TAGGED BUT UNEXECUTED - a test exists and is tagged, but only a " - "GPU host can run it.") + add("TAGGED BUT UNEXECUTED - a test exists and is tagged, but this CI " + "host cannot run it.") add(" These are NOT counted as covered:") for req_id in cov.unexecuted: tiers = ", ".join(sorted(register.requirements[req_id].tiers)) @@ -1068,12 +1430,18 @@ def format_coverage_report(report: Report, min_coverage: float) -> Tuple[str, in add(f" {', '.join(parentless)}") add("") - if cov.tier_unknown: + if cov.tier_unknown and register.uses_tiers: + # Only meaningful where the register uses tiers at all; otherwise every + # requirement is listed and the warning trains people to ignore it. add(f"WARNING: {len(cov.tier_unknown)} requirement(s) have no " - "verification tier in requirements.md; they are counted as " + "verification tier in the register; they are counted as " "CI-executable by default. Add them to the verification plan:") add(f" {', '.join(cov.tier_unknown)}") add("") + elif register.total and not register.uses_tiers: + add("Note: this register assigns no verification tiers, so nothing is " + "excluded as unexecutable.") + add("") if report.scan.exceptions: add(f"Recorded invariant exceptions: {len(report.scan.exceptions)}") @@ -1083,7 +1451,8 @@ def format_coverage_report(report: Report, min_coverage: float) -> Tuple[str, in add("") if cov.orphaned: - add("ORPHAN TAGS - traced in source, not defined in requirements.md:") + add("ORPHAN TAGS - traced in source, not defined in " + f"{config.requirements_path}:") for req_id in cov.orphaned: where = ", ".join(f"{e.file}:{e.line}" for e in report.requirement_map[req_id]) @@ -1092,7 +1461,7 @@ def format_coverage_report(report: Report, min_coverage: float) -> Tuple[str, in add("") if report.external_orphans: - add("ORPHAN SYSTEM TAGS - PR/SR IDs the system SPEC.md does not define:") + add("ORPHAN SYSTEM TAGS - IDs the system spec does not define:") add(f" {', '.join(report.external_orphans)}") add("") @@ -1122,13 +1491,14 @@ def format_coverage_report(report: Report, min_coverage: float) -> Tuple[str, in f"covered ({len(cov.covered)}) exceeds defined ({cov.total}) - " "the gate is miscomputing.") - if cov.orphaned and not report.allow_orphans: + if cov.orphaned and not config.allow_orphans: failures.append( f"{len(cov.orphaned)} orphan tag(s): {', '.join(cov.orphaned)}") - if cov.percent < min_coverage: + if cov.percent < config.min_coverage: failures.append( - f"coverage ({cov.percent}%) is below the minimum ({min_coverage}%)") + f"coverage ({cov.percent}%) is below the minimum " + f"({config.min_coverage}%)") if failures: add("FAILED:") @@ -1136,7 +1506,7 @@ def format_coverage_report(report: Report, min_coverage: float) -> Tuple[str, in add(f" - {reason}") return "\n".join(lines) + "\n", 1 - add(f"OK: coverage {cov.percent}% >= minimum {min_coverage}%, " + add(f"OK: coverage {cov.percent}% >= minimum {config.min_coverage}%, " f"{len(cov.orphaned)} orphan tag(s)") return "\n".join(lines) + "\n", 0 @@ -1148,67 +1518,114 @@ def format_coverage_report(report: Report, min_coverage: float) -> Tuple[str, in def build_arg_parser() -> argparse.ArgumentParser: parser = argparse.ArgumentParser( prog="extract_traces.py", - description=__doc__.split("\n")[0], + description="Extract requirement traces and report coverage. Shared by " + "every JRay component repo; configured per repo via " + f"{CONFIG_FILENAME} or the flags below.", formatter_class=argparse.RawDescriptionHelpFormatter, ) parser.add_argument("--format", choices=("json", "markdown", "coverage"), default="coverage", help="output written to stdout (default: coverage)") - parser.add_argument("--root", type=Path, default=REPO_ROOT, - help="repository root to scan (default: derived from " - "this script's location)") - parser.add_argument("--requirements", type=Path, default=None, - help="register path (default: /docs/requirements.md)") - parser.add_argument("--system-spec", type=Path, default=None, - help="optional umbrella SPEC.md defining PR/SR IDs; " - "enables orphan checking for those types") - parser.add_argument("--json-out", type=Path, default=None, - help="also write the JSON report here") - parser.add_argument("--markdown-out", type=Path, default=None, - help="also write the markdown matrix here") - parser.add_argument("--min-coverage", type=float, default=0.0, - help="minimum coverage percent; below it the run fails " - "(default: 0, i.e. the correctness checks gate but " - "the percentage does not)") - parser.add_argument("--allow-orphans", action="store_true", - help="report orphan tags without failing") + parser.add_argument("--config", type=Path, default=None, + help=f"path to {CONFIG_FILENAME} (default: the nearest " + "one at or above the working directory)") + parser.add_argument("--no-config", action="store_true", + help="ignore any config file; use flags only") + parser.add_argument("--print-example-config", action="store_true", + help=f"print an annotated {CONFIG_FILENAME} and exit") + parser.add_argument("--root", type=Path, default=None, + help="repository root (default: the config file's " + "directory, else the working directory)") + + group = parser.add_argument_group("what this repo defines") + group.add_argument("--requirement-type", action="append", metavar="XX", + help="ID prefix this repo's register defines; repeatable. " + "Replaces the configured list") + group.add_argument("--test-type", action="append", metavar="XX", + help="test ID prefix, excluded from coverage; repeatable") + group.add_argument("--external-type", action="append", metavar="XX", + help="system-spec ID prefix, excluded from coverage; " + "repeatable") + + group = parser.add_argument_group("what this repo scans") + group.add_argument("--language", action="append", + choices=sorted(LANGUAGE_SUFFIXES), + help="named suffix group; repeatable, additive") + group.add_argument("--source-suffix", action="append", metavar=".EXT", + help="extra source suffix; repeatable, additive") + group.add_argument("--source-root", action="append", metavar="DIR", + help="directory to scan; repeatable. Replaces the " + "configured list") + group.add_argument("--exclude-dir", action="append", metavar="NAME", + help="directory name to skip anywhere in the tree; " + "repeatable, additive") + + group = parser.add_argument_group("paths") + group.add_argument("--requirements", type=Path, default=None, + help=f"register path (default: {DEFAULT_REQUIREMENTS_PATH})") + group.add_argument("--system-spec", type=Path, default=None, + help="SPEC.md defining the external types; enables " + "orphan checking for them") + group.add_argument("--json-out", type=Path, default=None, + help=f"JSON report path (default: {DEFAULT_JSON_PATH})") + group.add_argument("--markdown-out", type=Path, default=None, + help=f"matrix path (default: {DEFAULT_MATRIX_PATH})") + group.add_argument("--no-write", action="store_true", + help="do not write report files, only print") + + group = parser.add_argument_group("policy") + group.add_argument("--min-coverage", type=float, default=None, + help="minimum coverage percent; below it the run fails") + group.add_argument("--allow-orphans", action="store_true", + help="report orphan tags without failing") + group.add_argument("--ci-executable-tier", action="append", metavar="TIER", + help="verification tier this CI host can run; " + "repeatable. Replaces the configured set") return parser def main(argv: Optional[Sequence[str]] = None) -> int: args = build_arg_parser().parse_args(argv) - root = args.root.resolve() - req_path = args.requirements or (root / "docs" / "requirements.md") - if not req_path.is_file(): + if args.print_example_config: + print(example_config(), end="") + return 0 + + try: + config = load_config(args) + except ConfigError as exc: + print(f"FAILED: {exc}", file=sys.stderr) + return 2 + + req_path = config.resolve(config.requirements_path) + if req_path is None or not req_path.is_file(): print(f"FAILED: requirements register not found at {req_path}", file=sys.stderr) return 2 - register = read_register(req_path) - files = iter_source_files(root) - scan = scan_files(files, root) + register = read_register(config) + files = iter_source_files(config) + scan = scan_files(files, config) system_ids = None - if args.system_spec: - if not args.system_spec.is_file(): - print(f"FAILED: --system-spec not found at {args.system_spec}", - file=sys.stderr) + spec_path = config.resolve(config.system_spec_path) + if spec_path is not None: + if not spec_path.is_file(): + print(f"FAILED: system spec not found at {spec_path}", file=sys.stderr) return 2 - system_ids = parse_system_spec(args.system_spec.read_text(encoding="utf-8")) + system_ids = parse_system_spec(spec_path.read_text(encoding="utf-8"), + config.external_types) - report = build_report(root, register, scan, system_ids) - report.allow_orphans = args.allow_orphans + report = build_report(config, register, scan, system_ids) json_text = json.dumps(report_to_json_dict(report), indent=2, sort_keys=False) markdown_text = generate_markdown(report) - if args.json_out: - args.json_out.parent.mkdir(parents=True, exist_ok=True) - args.json_out.write_text(json_text, encoding="utf-8") - if args.markdown_out: - args.markdown_out.parent.mkdir(parents=True, exist_ok=True) - args.markdown_out.write_text(markdown_text, encoding="utf-8") + for target, text in ((config.resolve(config.json_path), json_text), + (config.resolve(config.matrix_path), markdown_text)): + if target is not None: + target.parent.mkdir(parents=True, exist_ok=True) + target.write_text(text, encoding="utf-8") if args.format == "json": print(json_text) @@ -1217,7 +1634,7 @@ def main(argv: Optional[Sequence[str]] = None) -> int: print(markdown_text, end="") return 0 - text, code = format_coverage_report(report, args.min_coverage) + text, code = format_coverage_report(report) print(text, end="") return code diff --git a/scripts/traceability/test_extract_traces.py b/scripts/traceability/test_extract_traces.py index 4f2f4d4..44d9dfd 100755 --- a/scripts/traceability/test_extract_traces.py +++ b/scripts/traceability/test_extract_traces.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -"""Tests for the traceability extractor and coverage gate. +"""Tests for the shared traceability extractor and coverage gate. Run standalone (no third-party dependencies):: @@ -9,26 +9,32 @@ or under pytest, which discovers the same functions:: pytest scripts/traceability/test_extract_traces.py -Almost every test runs over fixture strings rather than the live -``docs/requirements.md``, so their meaning does not drift as requirements are -added. The two properties they exist to pin are the ones JellyTau's gate lost -(see JellyTau/docs/specs/traceability-gate-repair.md): +Almost every test runs over fixture strings rather than any live +``requirements.md``, so their meaning does not drift as requirements are added, +and so they say the same thing in whichever component repo this tool is +vendored into. + +Four properties these exist to pin: * the denominator is computed from the register at run time, so adding a - requirement lowers coverage until it is traced; + requirement lowers coverage until it is traced (JellyTau's gate lost this - + see JellyTau/docs/specs/traceability-gate-repair.md); * the numerator is an intersection, so a tag naming an undefined ID cannot push - the ratio above 100%. - -Plus the rule specific to this repo: a requirement only verifiable on GPU -hardware is reported as tagged-but-unexecuted and never counted as covered. + the ratio above 100%; +* a requirement whose only verification tier is outside what the CI host can + execute is reported as tagged-but-unexecuted, never counted as covered; +* nothing about one repo is baked in - the same code parses the C++/Python, + C# and Rust components - and a misconfigured run refuses to report rather + than printing a plausible 0%. """ from __future__ import annotations import io +import json import sys import tempfile -from contextlib import redirect_stdout +from contextlib import redirect_stderr, redirect_stdout from pathlib import Path HERE = Path(__file__).resolve().parent @@ -36,12 +42,125 @@ sys.path.insert(0, str(HERE)) import extract_traces as et # noqa: E402 -# The literal tag keyword is assembled at run time so that this file's fixtures -# do not register as real traces when the extractor scans scripts/. +# The literal tag keywords are assembled at run time so that this file's +# fixtures do not register as real traces when the extractor scans scripts/. TAG = "TRA" + "CES:" EXC = "EXCEP" + "TION:" +def cfg(**overrides) -> et.Config: + """A minimal valid config; overrides name whatever a test cares about.""" + base = dict(requirement_types=("AR", "DP", "IR", "GR", "VR"), + source_suffixes=et.LANGUAGE_SUFFIXES["cpp"] + | et.LANGUAGE_SUFFIXES["python"], + source_roots=("src", "scripts")) + base.update(overrides) + return et.Config(**base) + + +CPP_PY = cfg() + + +# -------------------------------------------------------------------------- +# Configuration: the thing that makes one implementation serve three repos +# -------------------------------------------------------------------------- + +def test_config_rejects_an_empty_requirement_type_list(): + # The prefix cannot be guessed, and guessing wrong means parsing zero + # requirements - the failure this whole class of check exists to prevent. + try: + cfg(requirement_types=()).validate() + except et.ConfigError as exc: + assert "requirement_types is empty" in str(exc) + else: + raise AssertionError("expected ConfigError") + + +def test_config_rejects_a_prefix_that_is_not_two_uppercase_letters(): + try: + cfg(requirement_types=("Jr",)).validate() + except et.ConfigError as exc: + assert "two-letter uppercase" in str(exc) + else: + raise AssertionError("expected ConfigError") + + +def test_config_rejects_a_prefix_that_is_both_counted_and_excluded(): + # A prefix cannot be in the fraction and out of it at once; silently + # picking one would make the reported number unexplainable. + try: + cfg(requirement_types=("UR", "UT")).validate() + except et.ConfigError as exc: + assert "cannot also be excluded" in str(exc) + else: + raise AssertionError("expected ConfigError") + + +def test_config_rejects_empty_suffixes_and_roots(): + for kwargs, expected in ((dict(source_suffixes=frozenset()), "no source suffixes"), + (dict(source_roots=()), "no source roots")): + try: + cfg(**kwargs).validate() + except et.ConfigError as exc: + assert expected in str(exc) + else: + raise AssertionError(f"expected ConfigError for {kwargs}") + + +def test_config_rejects_a_suffix_missing_its_dot(): + try: + cfg(source_suffixes=frozenset({"rs"})).validate() + except et.ConfigError as exc: + assert "must start with a dot" in str(exc) + else: + raise AssertionError("expected ConfigError") + + +def test_config_from_dict_expands_language_groups(): + config = et.config_from_dict( + {"requirement_types": ["UR", "DR"], "languages": ["rust"], + "source_roots": ["src", "tests"]}, Path("/repo"), "fixture") + config.validate() + assert config.source_suffixes == frozenset({".rs"}) + assert config.requirement_types == ("UR", "DR") + + +def test_config_from_dict_rejects_an_unknown_key(): + # A typo'd key would leave a required field empty, and the user would be + # debugging "zero requirements" instead of a misspelling. + try: + et.config_from_dict({"requirement_type": ["JR"]}, Path("/repo"), "fixture") + except et.ConfigError as exc: + assert "unknown key" in str(exc) + else: + raise AssertionError("expected ConfigError") + + +def test_config_from_dict_rejects_an_unknown_language(): + try: + et.config_from_dict({"languages": ["cobol"]}, Path("/repo"), "fixture") + except et.ConfigError as exc: + assert "unknown language" in str(exc) + else: + raise AssertionError("expected ConfigError") + + +def test_example_config_is_itself_valid_and_complete(): + # The documentation people copy must parse, or it teaches the wrong schema. + tomllib = __import__("tomllib") + data = tomllib.loads(et.example_config()) + config = et.config_from_dict(data, Path("/repo"), "example") + config.validate() + assert config.requirement_types == ("AR", "DP", "IR", "GR", "VR") + + +def test_config_paths_resolve_against_the_repo_root_not_the_cwd(): + config = cfg(root=Path("/repo"), requirements_path="docs/requirements.md") + assert config.resolve("docs/requirements.md") == Path("/repo/docs/requirements.md") + assert config.resolve("/abs/spec.md") == Path("/abs/spec.md") + assert config.resolve(None) is None + + # -------------------------------------------------------------------------- # Tag parsing # -------------------------------------------------------------------------- @@ -62,8 +181,8 @@ def test_parses_multiple_types_separated_by_pipe(): def test_parses_three_groups_including_test_ids(): - groups, _ = et.parse_traces_tag(" AR-012 | SR-002 | UT-003, UT-004") - assert groups == [["AR-012"], ["SR-002"], ["UT-003", "UT-004"]] + groups, _ = et.parse_traces_tag(" JR-012 | SR-002 | UT-003, UT-004") + assert groups == [["JR-012"], ["SR-002"], ["UT-003", "UT-004"]] def test_strips_a_trailing_block_comment_terminator(): @@ -94,7 +213,7 @@ def test_two_digit_id_is_not_accepted_as_a_requirement(): # -------------------------------------------------------------------------- -# Scanning C++ and Python sources +# Scanning: the same scanner over every component's language # -------------------------------------------------------------------------- def _write_tree(root: Path) -> None: @@ -126,26 +245,82 @@ def test_scans_cpp_and_python_but_not_vendored_or_non_source(): with tempfile.TemporaryDirectory() as tmp: root = Path(tmp) _write_tree(root) - files = et.iter_source_files(root) - names = sorted(f.name for f in files) - assert names == ["gallery.py", "tracker.hpp"], names + config = cfg(root=root) + files = et.iter_source_files(config) + assert sorted(f.name for f in files) == ["gallery.py", "tracker.hpp"] - scan = et.scan_files(files, root) + scan = et.scan_files(files, config) traced = sorted({i for t in scan.traces for i in t.requirements}) assert traced == ["AR-012", "AR-013", "GR-001", "SR-002", "SR-005"] +def test_the_same_scanner_reads_rust_when_configured_for_it(): + # JRay-public-server: Rust sources, UR/DR prefixes. No second extractor. + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + (root / "src").mkdir(parents=True) + (root / "src" / "manifest.rs").write_text( + f"/// {TAG} UR-002 | SR-004\n" + "pub fn accept_upload(body: Body) -> Response {\n" + " todo!()\n" + "}\n", encoding="utf-8") + config = cfg(root=root, requirement_types=("UR", "DR"), + source_suffixes=et.LANGUAGE_SUFFIXES["rust"], + source_roots=("src",)) + scan = et.scan_files(et.iter_source_files(config), config) + assert len(scan.traces) == 1 + assert scan.traces[0].requirements == ["UR-002", "SR-004"] + assert "accept_upload" in scan.traces[0].context + + +def test_the_same_scanner_reads_csharp_when_configured_for_it(): + # jRay: C# plugin, JR prefix, and a tag above an attribute-decorated action. + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + (root / "Jellyfin.Plugin.JRay").mkdir(parents=True) + (root / "Jellyfin.Plugin.JRay" / "TruthController.cs").write_text( + f" /// {TAG} JR-012 | SR-001\n" + ' [HttpGet("{itemId}")]\n' + " public ActionResult GetTruth(Guid itemId)\n" + " {\n" + " return Ok();\n" + " }\n", encoding="utf-8") + config = cfg(root=root, requirement_types=("JR",), + source_suffixes=et.LANGUAGE_SUFFIXES["csharp"], + source_roots=("Jellyfin.Plugin.JRay",)) + scan = et.scan_files(et.iter_source_files(config), config) + assert len(scan.traces) == 1 + assert scan.traces[0].requirements == ["JR-012", "SR-001"] + # The attribute must not steal the context from the method it decorates. + assert "GetTruth" in scan.traces[0].context + assert "HttpGet" not in scan.traces[0].context + + def test_context_is_found_below_a_cpp_tag_and_above_a_python_tag(): # The two languages put the tag on opposite sides of what it describes. with tempfile.TemporaryDirectory() as tmp: root = Path(tmp) _write_tree(root) - scan = et.scan_files(et.iter_source_files(root), root) + config = cfg(root=root) + scan = et.scan_files(et.iter_source_files(config), config) contexts = {t.file: t.context for t in scan.traces} assert "TrackRegistry" in contexts["src/tracker.hpp"] assert "build_gallery" in contexts["scripts/gallery.py"] +def test_an_id_of_an_unconfigured_prefix_is_reported_as_unknown(): + # In the Rust server, an `AR-001` tag is a copy-paste from another repo. + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + (root / "src").mkdir(parents=True) + (root / "src" / "a.rs").write_text(f"// {TAG} AR-001\n", encoding="utf-8") + config = cfg(root=root, requirement_types=("UR", "DR"), + source_suffixes=et.LANGUAGE_SUFFIXES["rust"], + source_roots=("src",)) + scan = et.scan_files(et.iter_source_files(config), config) + assert scan.diagnostics.unknown_id_types + + # -------------------------------------------------------------------------- # EXCEPTION tags # -------------------------------------------------------------------------- @@ -158,12 +333,17 @@ EXCEPTION_SOURCE = ( ) +def _exception_repo(tmp: str) -> et.Config: + root = Path(tmp) + (root / "src").mkdir(parents=True) + (root / "src" / "outlier.cpp").write_text(EXCEPTION_SOURCE, encoding="utf-8") + return cfg(root=root) + + def test_exception_tag_is_captured_with_its_reason(): with tempfile.TemporaryDirectory() as tmp: - root = Path(tmp) - (root / "src").mkdir(parents=True) - (root / "src" / "outlier.cpp").write_text(EXCEPTION_SOURCE, encoding="utf-8") - scan = et.scan_files(et.iter_source_files(root), root) + config = _exception_repo(tmp) + scan = et.scan_files(et.iter_source_files(config), config) assert len(scan.exceptions) == 1 exc = scan.exceptions[0] assert exc.requirement == "AR-024" @@ -175,14 +355,12 @@ def test_exception_is_never_counted_as_coverage(): # An exception is a recorded decision to depart from an invariant. Counting # it as evidence the requirement is met inverts its meaning. with tempfile.TemporaryDirectory() as tmp: - root = Path(tmp) - (root / "src").mkdir(parents=True) - (root / "src" / "outlier.cpp").write_text(EXCEPTION_SOURCE, encoding="utf-8") - scan = et.scan_files(et.iter_source_files(root), root) + config = _exception_repo(tmp) + scan = et.scan_files(et.iter_source_files(config), config) assert scan.traces == [] register = et.parse_register( "| ID | Requirement | Status |\n|---|---|---|\n" - "| AR-024 | Always the calibrated probability | Planned |\n") + "| AR-024 | Always the calibrated probability | Planned |\n", config) cov = et.compute_coverage( [i for t in scan.traces for i in t.requirements], register) assert cov.covered == [] @@ -190,13 +368,13 @@ def test_exception_is_never_counted_as_coverage(): def test_exception_without_a_reason_is_reported(): - # CLAUDE.md: an exception is only *agreed* if the reason is recorded. + # An exception is only *agreed* if the reason is recorded. with tempfile.TemporaryDirectory() as tmp: root = Path(tmp) (root / "src").mkdir(parents=True) - (root / "src" / "bare.cpp").write_text( - f"// {EXC} AR-024\n", encoding="utf-8") - scan = et.scan_files(et.iter_source_files(root), root) + (root / "src" / "bare.cpp").write_text(f"// {EXC} AR-024\n", encoding="utf-8") + config = cfg(root=root) + scan = et.scan_files(et.iter_source_files(config), config) assert len(scan.exceptions) == 1 assert scan.diagnostics.exceptions_without_reason @@ -209,7 +387,8 @@ def test_mixed_type_group_is_reported(): (root / "src").mkdir(parents=True) (root / "src" / "a.cpp").write_text( f"// {TAG} AR-001, SR-002\n", encoding="utf-8") - scan = et.scan_files(et.iter_source_files(root), root) + config = cfg(root=root) + scan = et.scan_files(et.iter_source_files(config), config) assert scan.diagnostics.mixed_type_groups @@ -224,20 +403,41 @@ def test_counts_a_well_formed_table_row_as_a_defined_requirement(): md = REGISTER_HEADER + ( "| AR-001 | Detect faces in sampled frames | SR-002 | High | Done |\n" "| AR-002 | Minimum face size 66x66 px | SR-002 | High | Planned |\n") - register = et.parse_register(md) + register = et.parse_register(md, CPP_PY) assert register.count("AR") == 2 assert register.count("GR") == 0 assert register.total == 2 +def test_the_same_parser_reads_another_repos_prefixes(): + # Identical table shape, different prefixes. This is the change that made + # the tool shareable: with requirement_types hardcoded, both of these + # parsed to zero. + md = REGISTER_HEADER + ( + "| JR-001 | Truth-file schema | SR-003 | High | Done |\n" + "| UR-001 | Availability without payload | SR-001 | High | Done |\n" + "| DR-003 | One writer connection plus a read pool | SR-001 | High | Done |\n") + jray = et.parse_register(md, cfg(requirement_types=("JR",))) + server = et.parse_register(md, cfg(requirement_types=("UR", "DR"))) + assert jray.ids == {"JR-001"} + assert server.ids == {"UR-001", "DR-003"} + + +def test_a_prefix_this_repo_does_not_define_is_not_counted_as_defined(): + # The server's register must not acquire a denominator from an AR row + # pasted in from the extraction repo. + md = REGISTER_HEADER + "| AR-001 | Belongs to another repo | SR-002 | High | Done |\n" + register = et.parse_register(md, cfg(requirement_types=("UR", "DR"))) + assert register.total == 0 + + def test_does_not_count_ids_that_appear_only_in_the_traces_to_column(): # The bug this parse rule avoids: a naive scan for AR-\d{3} over the whole # file counts every reference as a definition and inflates the denominator. md = REGISTER_HEADER + ( "| GR-001 | Build gallery from library cast | SR-001, SR-005 | High | Done |\n" "| GR-002 | Incremental merge refresh | PR-003 | High | Done |\n") - register = et.parse_register(md) - assert register.count("GR") == 2 + register = et.parse_register(md, CPP_PY) assert register.ids == {"GR-001", "GR-002"} @@ -245,7 +445,7 @@ def test_does_not_count_ids_mentioned_in_prose(): md = ("Some prose explaining that AR-005 relates to GR-001 and VR-003.\n\n" + REGISTER_HEADER + "| AR-005 | Align to 112x112 | SR-002 | High | Done |\n") - register = et.parse_register(md) + register = et.parse_register(md, CPP_PY) assert register.ids == {"AR-005"} @@ -259,9 +459,8 @@ def test_does_not_count_the_verification_plan_table_as_definitions(): + "| ID | Tier | Test asserts | Edge cases to cover |\n|---|---|---|---|\n" + "| AR-001 | T3 | Detector returns plausible boxes | smoke only |\n" + "| AR-099 | T1 | Something not in the register | - |\n") - register = et.parse_register(md) + register = et.parse_register(md, CPP_PY) assert register.ids == {"AR-001"} - assert register.total == 1 def test_deduplicates_an_id_listed_in_two_definition_tables(): @@ -270,8 +469,7 @@ def test_deduplicates_an_id_listed_in_two_definition_tables(): + "\n" + REGISTER_HEADER + "| AR-001 | Detect faces | SR-002 | High | Done |\n") - register = et.parse_register(md) - assert register.total == 1 + assert et.parse_register(md, CPP_PY).total == 1 def test_withdrawn_requirements_leave_the_denominator(): @@ -280,18 +478,29 @@ def test_withdrawn_requirements_leave_the_denominator(): md = REGISTER_HEADER + ( "| AR-001 | Detect faces | SR-002 | High | Done |\n" "| AR-002 | Superseded mechanism | SR-002 | High | Withdrawn |\n") - register = et.parse_register(md) + register = et.parse_register(md, CPP_PY) assert register.ids == {"AR-001"} assert "AR-002" in register.withdrawn +def test_an_escaped_pipe_inside_a_cell_does_not_split_the_row(): + # The server's register contains `small-\|M\| all-but-one rule`; a naive + # split shifts every later column by one. + md = REGISTER_HEADER + ( + r"| UR-003 | Strict schema with a small-\|M\| rule | SR-001 | High | Done |" + "\n") + register = et.parse_register(md, cfg(requirement_types=("UR",))) + assert register.requirements["UR-003"].status == "Done" + assert "|M|" in register.requirements["UR-003"].text + + def test_the_denominator_is_live_adding_a_row_lowers_coverage(): # The property JellyTau's frozen literals destroyed. Same traced set, one # more requirement defined => a lower percentage, mechanically. base = REGISTER_HEADER + "| AR-001 | A | SR-002 | High | Done |\n" grown = base + "| AR-002 | B | SR-002 | High | Planned |\n" - before = et.compute_coverage(["AR-001"], et.parse_register(base)) - after = et.compute_coverage(["AR-001"], et.parse_register(grown)) + before = et.compute_coverage(["AR-001"], et.parse_register(base, CPP_PY)) + after = et.compute_coverage(["AR-001"], et.parse_register(grown, CPP_PY)) assert before.percent == 100.0 assert after.percent == 50.0 assert after.total == 2 @@ -300,14 +509,26 @@ def test_the_denominator_is_live_adding_a_row_lowers_coverage(): def test_register_captures_the_row_fields_not_just_the_id(): md = REGISTER_HEADER + ( "| AR-012 | Presence follows track extent | **SR-002** | High | Planned |\n") - req = et.parse_register(md).requirements["AR-012"] + req = et.parse_register(md, CPP_PY).requirements["AR-012"] assert req.text == "Presence follows track extent" assert req.traces_to == "**SR-002**" assert req.status == "Planned" +def test_a_requirement_tracing_up_to_nothing_is_reported(): + # SPEC.md section 6: a requirement citing no parent is scope creep, and it + # is invisible unless something looks. A section reference counts as a + # parent - what matters is that something was recorded. + md = REGISTER_HEADER + ( + "| AR-001 | Has a parent | SR-002 | High | Done |\n" + "| AR-002 | Parent is a section | §4 | Medium | Planned |\n" + "| AR-003 | Serves nothing stated | - | Low | Planned |\n" + "| AR-004 | Blank cell | | Low | Planned |\n") + assert et.parse_register(md, CPP_PY).parentless_ids() == {"AR-003", "AR-004"} + + # -------------------------------------------------------------------------- -# Verification tiers and the GPU-less CI host +# Verification tiers and the CI host's limits # -------------------------------------------------------------------------- TIER_REGISTER = REGISTER_HEADER + "".join( @@ -327,7 +548,7 @@ def test_tier_assignment_handles_lists_ranges_and_wildcards(): "| AR-002 … AR-004 | **T2** | replay |\n" "| AR-006 | T1 + T4 | mixed |\n" "| VR-* | Out of CI | studies |\n") - register = et.parse_register(md) + register = et.parse_register(md, CPP_PY) assert register.requirements["AR-001"].tiers == {"T3"} assert register.requirements["AR-003"].tiers == {"T2"} assert register.requirements["AR-006"].tiers == {"T1", "T4"} @@ -336,7 +557,7 @@ def test_tier_assignment_handles_lists_ranges_and_wildcards(): def test_a_range_cannot_invent_a_requirement_the_register_lacks(): md = TIER_REGISTER + "\n" + _tier_table("| AR-001 … AR-050 | T2 | wide |\n") - register = et.parse_register(md) + register = et.parse_register(md, CPP_PY) assert register.total == 12 assert "AR-050" not in register.ids @@ -345,7 +566,7 @@ def test_slash_shorthand_in_the_verification_plan_expands(): md = TIER_REGISTER + "\n" + ( "| ID | Tier | Test asserts | Edge cases |\n|---|---|---|---|\n" "| AR-009/008 | T2 | Cut shifts weighting | cut with same people |\n") - register = et.parse_register(md) + register = et.parse_register(md, CPP_PY) assert register.requirements["AR-008"].tiers == {"T2"} assert register.requirements["AR-009"].tiers == {"T2"} @@ -357,35 +578,43 @@ def test_tiers_from_both_tables_are_unioned_not_overwritten(): md = TIER_REGISTER + "\n" + _tier_table("| AR-006 | T4 | GPU host only |\n") + "\n" + ( "| ID | Tier | Test asserts | Edge cases |\n|---|---|---|---|\n" "| AR-006 | T1 + T4 | GEMM equals reference loop | small input in CI |\n") - register = et.parse_register(md) + register = et.parse_register(md, CPP_PY) assert register.requirements["AR-006"].tiers == {"T1", "T4"} - assert register.requirements["AR-006"].ci_executable + assert register.is_ci_executable("AR-006") def test_a_t4_only_requirement_is_not_ci_executable(): md = TIER_REGISTER + "\n" + _tier_table("| AR-007 | **T4** | GPU only |\n") - register = et.parse_register(md) - assert not register.requirements["AR-007"].ci_executable + register = et.parse_register(md, CPP_PY) + assert not register.is_ci_executable("AR-007") assert register.unexecutable_ids() == {"AR-007"} -def test_a_requirement_tracing_up_to_nothing_is_reported(): - # SPEC.md section 6: a requirement citing no parent is scope creep, and it - # is invisible unless something looks. A section reference counts as a - # parent - what matters is that something was recorded. - md = REGISTER_HEADER + ( - "| AR-001 | Has a parent | SR-002 | High | Done |\n" - "| AR-002 | Parent is a section | §4 | Medium | Planned |\n" - "| AR-003 | Serves nothing stated | - | Low | Planned |\n" - "| AR-004 | Blank cell | | Low | Planned |\n") - register = et.parse_register(md) - assert register.parentless_ids() == {"AR-003", "AR-004"} +def test_which_tiers_count_as_executable_is_configuration_not_code(): + # A repo whose CI host can run T4 says so, and the same register then + # yields full executability. Nothing about one host is baked in. + md = TIER_REGISTER + "\n" + _tier_table("| AR-007 | **T4** | GPU only |\n") + permissive = et.parse_register( + md, cfg(ci_executable_tiers=frozenset({"T1", "T2", "T3", "T4", "static"}))) + assert permissive.unexecutable_ids() == set() + assert permissive.is_ci_executable("AR-007") def test_a_requirement_with_no_tier_is_unknown_not_unexecutable(): - register = et.parse_register(TIER_REGISTER) + register = et.parse_register(TIER_REGISTER, CPP_PY) assert register.tier_unknown_ids() == register.ids assert register.unexecutable_ids() == set() + assert not register.uses_tiers + + +def test_a_register_that_uses_tiers_is_distinguishable_from_one_that_does_not(): + # The distinction drives whether "no tier recorded" is worth warning about; + # warning on every requirement of a tierless register trains people to + # ignore the warning. + tiered = et.parse_register( + TIER_REGISTER + "\n" + _tier_table("| AR-001 | T1 | unit |\n"), CPP_PY) + assert tiered.uses_tiers + assert not et.parse_register(TIER_REGISTER, CPP_PY).uses_tiers # -------------------------------------------------------------------------- @@ -401,7 +630,8 @@ COVERAGE_REGISTER = et.parse_register( + "\n" + _tier_table("| AR-001, AR-002 | T2 | replay |\n" "| GR-001 | T1 | bookkeeping |\n" - "| AR-027 | **T4** | GPU host only |\n")) + "| AR-027 | **T4** | GPU host only |\n"), + CPP_PY) def test_coverage_is_the_intersection_of_traced_and_defined(): @@ -425,14 +655,13 @@ def test_orphan_tags_are_reported_so_they_get_fixed(): def test_no_orphans_when_every_traced_id_is_defined(): - cov = et.compute_coverage(["AR-001", "AR-002"], COVERAGE_REGISTER) - assert cov.orphaned == [] + assert et.compute_coverage(["AR-001", "AR-002"], COVERAGE_REGISTER).orphaned == [] def test_test_and_system_ids_are_a_separate_taxonomy(): - # UT/IT live in their own register section; PR/SR live in the umbrella - # SPEC.md, which is not part of this repo's checkout. Neither counts toward - # coverage, and flagging them as orphans would bury real typos in noise. + # UT/IT are evidence for requirements; PR/SR live in the system spec, which + # is a different register. Neither counts toward coverage, and flagging + # them as orphans would bury real typos in noise. cov = et.compute_coverage( ["AR-001", "UT-003", "IT-007", "SR-002", "PR-001"], COVERAGE_REGISTER) assert cov.orphaned == [] @@ -440,9 +669,8 @@ def test_test_and_system_ids_are_a_separate_taxonomy(): def test_a_gpu_only_requirement_is_tagged_but_unexecuted_not_covered(): - # The rule specific to this repo: CI is an Intel N100 with no dGPU. A test - # that exists but can never run is not evidence, and counting it is the - # same failure mode as the 158% bug. + # A test that exists but can never run on the CI host is not evidence, and + # counting it is the same failure mode as the 158% bug. cov = et.compute_coverage(["AR-001", "AR-027"], COVERAGE_REGISTER) assert cov.unexecuted == ["AR-027"] assert cov.covered == ["AR-001"] @@ -475,7 +703,7 @@ def test_an_empty_trace_set_is_zero_percent_not_a_divide_by_zero(): def test_an_empty_register_reports_zero_rather_than_nan(): - cov = et.compute_coverage(["AR-001"], et.Register()) + cov = et.compute_coverage(["AR-001"], et.Register(types=("AR",))) assert cov.percent == 0.0 assert cov.ci_percent == 0.0 @@ -486,7 +714,6 @@ def test_full_coverage_reports_exactly_one_hundred_never_above(): # AR-027 is GPU-only, so the honest ceiling here is 3/4. assert cov.percent == 75.0 assert cov.ci_percent == 100.0 - assert cov.ci_percent <= 100.0 # -------------------------------------------------------------------------- @@ -502,23 +729,52 @@ FIXTURE_REGISTER = ( + _tier_table("| AR-001, AR-002 | T2 | replay |\n" "| AR-027 | **T4** | GPU host only |\n")) +FIXTURE_CONFIG_TOML = """\ +requirement_types = ["AR"] +languages = ["cpp"] +source_roots = ["src"] +""" -def _fixture_repo(tmp: str, source: str) -> Path: + +def _fixture_repo(tmp: str, source: str, config_toml: str = FIXTURE_CONFIG_TOML, + register: str = FIXTURE_REGISTER) -> Path: root = Path(tmp) (root / "docs").mkdir(parents=True, exist_ok=True) (root / "src").mkdir(parents=True, exist_ok=True) - (root / "docs" / "requirements.md").write_text(FIXTURE_REGISTER, encoding="utf-8") + (root / "docs" / "requirements.md").write_text(register, encoding="utf-8") (root / "src" / "pipeline.cpp").write_text(source, encoding="utf-8") + (root / et.CONFIG_FILENAME).write_text(config_toml, encoding="utf-8") return root def _run_gate(root: Path, *extra: str): buffer = io.StringIO() with redirect_stdout(buffer): - code = et.main(["--root", str(root), "--format", "coverage", *extra]) + code = et.main(["--config", str(root / et.CONFIG_FILENAME), + "--format", "coverage", *extra]) return code, buffer.getvalue() +def test_gate_reads_its_settings_from_the_repos_config_file(): + with tempfile.TemporaryDirectory() as tmp: + root = _fixture_repo(tmp, f"// {TAG} AR-001\nint main() {{}}\n") + code, out = _run_gate(root) + assert code == 0, out + assert "Requirement types : AR" in out + assert str(root / et.CONFIG_FILENAME) in out + + +def test_config_is_discovered_from_a_subdirectory(): + # The config file's directory is the repo root, so running the gate from + # deep inside the tree does not silently scan a subtree. + with tempfile.TemporaryDirectory() as tmp: + root = _fixture_repo(tmp, f"// {TAG} AR-001\nint main() {{}}\n") + nested = root / "src" / "nodes" + nested.mkdir(parents=True, exist_ok=True) + found = et.find_config(nested) + assert found == root / et.CONFIG_FILENAME + + def test_gate_passes_at_the_default_threshold_with_no_tags_yet(): # Near-zero coverage on a fresh tree is the correct reading, not a failure. with tempfile.TemporaryDirectory() as tmp: @@ -559,20 +815,48 @@ def test_gate_fails_when_the_register_parses_to_nothing(): # With a low threshold this self-check is what keeps the gate meaningful: # a broken parser would otherwise report 0/0 and pass. with tempfile.TemporaryDirectory() as tmp: - root = _fixture_repo(tmp, "int main() {}\n") - (root / "docs" / "requirements.md").write_text( - "# register\n\nNo tables here.\n", encoding="utf-8") + root = _fixture_repo(tmp, "int main() {}\n", + register="# register\n\nNo tables here.\n") code, out = _run_gate(root) assert code == 1 assert "ZERO requirements" in out -def test_gate_fails_when_no_source_files_were_scanned(): +def test_gate_fails_when_the_prefixes_are_configured_wrong(): + # The misconfiguration a shared tool invites: the server's config pointed + # at the extraction register, or a typo'd prefix. Every row is skipped, and + # a plausible 0% would hide it completely. with tempfile.TemporaryDirectory() as tmp: - root = Path(tmp) - (root / "docs").mkdir(parents=True) - (root / "docs" / "requirements.md").write_text(FIXTURE_REGISTER, - encoding="utf-8") + root = _fixture_repo( + tmp, f"// {TAG} AR-001\nint main() {{}}\n", + config_toml='requirement_types = ["JR"]\nlanguages = ["cpp"]\n' + 'source_roots = ["src"]\n') + code, out = _run_gate(root) + assert code == 1 + assert "ZERO requirements" in out + assert "requirement_types (JR)" in out + + +def test_gate_fails_when_the_suffixes_are_configured_wrong(): + # The Rust config applied to a C++ repo: nothing to scan, and coverage + # against an empty tree means nothing. + with tempfile.TemporaryDirectory() as tmp: + root = _fixture_repo( + tmp, f"// {TAG} AR-001\nint main() {{}}\n", + config_toml='requirement_types = ["AR"]\nlanguages = ["rust"]\n' + 'source_roots = ["src"]\n') + code, out = _run_gate(root) + assert code == 1 + assert "no source files were scanned" in out + assert ".rs" in out + + +def test_gate_fails_when_the_source_roots_do_not_exist(): + with tempfile.TemporaryDirectory() as tmp: + root = _fixture_repo( + tmp, f"// {TAG} AR-001\nint main() {{}}\n", + config_toml='requirement_types = ["AR"]\nlanguages = ["cpp"]\n' + 'source_roots = ["nowhere"]\n') code, out = _run_gate(root) assert code == 1 assert "no source files were scanned" in out @@ -594,11 +878,14 @@ def test_gate_hard_fails_on_an_impossible_ratio(): # reported as a pass. Forced here by handing the reporter a poisoned value. with tempfile.TemporaryDirectory() as tmp: root = _fixture_repo(tmp, f"// {TAG} AR-001\nint main() {{}}\n") - register = et.read_register(root / "docs" / "requirements.md") - scan = et.scan_files(et.iter_source_files(root), root) - report = et.build_report(root, register, scan) + config = et.config_from_dict( + {"requirement_types": ["AR"], "languages": ["cpp"], + "source_roots": ["src"]}, root, "fixture") + register = et.read_register(config) + scan = et.scan_files(et.iter_source_files(config), config) + report = et.build_report(config, register, scan) report.coverage.percent = 158.0 - text, code = et.format_coverage_report(report, 50.0) + text, code = et.format_coverage_report(report) assert code == 1 assert "exceeds 100%" in text @@ -607,11 +894,13 @@ def test_the_per_type_breakdown_sums_to_the_headline_figure(): # A breakdown that does not add up to its own total is how a wrong number # survives review: every row looks plausible on its own. with tempfile.TemporaryDirectory() as tmp: - root = _fixture_repo( - tmp, f"// {TAG} AR-001, AR-027\nint main() {{}}\n") - register = et.read_register(root / "docs" / "requirements.md") - scan = et.scan_files(et.iter_source_files(root), root) - report = et.build_report(root, register, scan) + root = _fixture_repo(tmp, f"// {TAG} AR-001, AR-027\nint main() {{}}\n") + config = et.config_from_dict( + {"requirement_types": ["AR"], "languages": ["cpp"], + "source_roots": ["src"]}, root, "fixture") + register = et.read_register(config) + scan = et.scan_files(et.iter_source_files(config), config) + report = et.build_report(config, register, scan) stats = et.per_type_stats(report) assert sum(c for c, _, _ in stats.values()) == len(report.coverage.covered) assert sum(u for _, u, _ in stats.values()) == len(report.coverage.unexecuted) @@ -620,64 +909,116 @@ def test_the_per_type_breakdown_sums_to_the_headline_figure(): def test_json_and_markdown_outputs_are_written_and_consistent(): - import json with tempfile.TemporaryDirectory() as tmp: root = _fixture_repo(tmp, f"// {TAG} AR-001 | SR-002\nint main() {{}}\n") - json_out = root / "traces-report.json" - md_out = root / "docs" / "traceability.md" - buffer = io.StringIO() - with redirect_stdout(buffer): - code = et.main(["--root", str(root), "--format", "coverage", - "--json-out", str(json_out), - "--markdown-out", str(md_out)]) + code, _ = _run_gate(root) assert code == 0 - data = json.loads(json_out.read_text(encoding="utf-8")) + data = json.loads((root / "traces-report.json").read_text(encoding="utf-8")) assert data["defined"]["total"] == 3 assert data["coverage"]["covered"] == 1 assert data["coverage"]["percent"] == round(100 / 3, 1) assert data["byType"]["SR"] == ["SR-002"] - assert data["gpuOnlyRequirements"] == ["AR-027"] - assert "AR-001" in md_out.read_text(encoding="utf-8") + assert data["unexecutableRequirements"] == ["AR-027"] + # The report says how it was produced; a shared tool's output is + # otherwise ambiguous about which repo and settings it describes. + assert data["config"]["requirementTypes"] == ["AR"] + assert data["config"]["sourceRoots"] == ["src"] + assert "AR-001" in (root / "docs" / "traceability.md").read_text( + encoding="utf-8") + + +def test_cli_flags_alone_work_with_no_config_file_at_all(): + # The path a repo takes before it has a traceability.toml, and the escape + # hatch on a Python too old for tomllib. + with tempfile.TemporaryDirectory() as tmp: + root = _fixture_repo(tmp, f"// {TAG} AR-001\nint main() {{}}\n") + (root / et.CONFIG_FILENAME).unlink() + buffer = io.StringIO() + with redirect_stdout(buffer): + code = et.main(["--no-config", "--root", str(root), + "--requirement-type", "AR", + "--language", "cpp", "--source-root", "src", + "--format", "coverage", "--no-write"]) + assert code == 0, buffer.getvalue() + assert "1 / 3" in buffer.getvalue() + assert not (root / "traces-report.json").exists() + + +def test_cli_flags_override_the_config_file(): + with tempfile.TemporaryDirectory() as tmp: + root = _fixture_repo(tmp, f"// {TAG} AR-001\nint main() {{}}\n") + code, out = _run_gate(root, "--requirement-type", "JR", "--no-write") + assert code == 1 + assert "requirement_types (JR)" in out + + +def test_language_flags_extend_rather_than_replace_the_configured_set(): + with tempfile.TemporaryDirectory() as tmp: + root = _fixture_repo(tmp, f"// {TAG} AR-001\nint main() {{}}\n") + (root / "src" / "extra.rs").write_text( + f"/// {TAG} AR-002\npub fn go() {{}}\n", encoding="utf-8") + code, out = _run_gate(root, "--language", "rust", "--no-write") + assert code == 0, out + assert "2 / 3" in out + + +def test_an_invalid_configuration_is_a_distinct_exit_code(): + # 2, not 1: a broken config is not a coverage failure, and conflating them + # makes CI logs lie about why the job went red. + with tempfile.TemporaryDirectory() as tmp: + root = _fixture_repo(tmp, "int main() {}\n", + config_toml='languages = ["cpp"]\nsource_roots = ["src"]\n') + buffer = io.StringIO() + with redirect_stdout(buffer), redirect_stderr(buffer): + code = et.main(["--config", str(root / et.CONFIG_FILENAME)]) + assert code == 2 + assert "requirement_types is empty" in buffer.getvalue() def test_system_spec_parsing_enables_orphan_checks_for_pr_and_sr(): spec = ("### SR-002 - Presence is scene-scoped\n\n" "| ID | Goal | Why |\n|---|---|---|\n" "| **PR-001** | Show which actors are on screen | The product |\n") - ids = et.parse_system_spec(spec) - assert ids == {"SR-002", "PR-001"} + assert et.parse_system_spec(spec) == {"SR-002", "PR-001"} with tempfile.TemporaryDirectory() as tmp: root = _fixture_repo(tmp, f"// {TAG} AR-001 | SR-999\nint main() {{}}\n") spec_path = root / "system-spec.md" spec_path.write_text(spec, encoding="utf-8") - code, out = _run_gate(root, "--system-spec", str(spec_path)) + code, out = _run_gate(root, "--system-spec", str(spec_path), "--no-write") assert "SR-999" in out assert code == 0 # advisory: the system register is not this repo's # -------------------------------------------------------------------------- -# The live register — structural assertions only, so this does not churn as -# requirements are added. +# The live repo — structural assertions only, so this does not churn as +# requirements are added, and so it still means something once this file is +# vendored into another component. # -------------------------------------------------------------------------- -def test_the_live_register_parses_and_assigns_tiers(): - register = et.read_register(et.REPO_ROOT / "docs" / "requirements.md") +def _live_config() -> et.Config: + config_path = et.find_config(HERE) + assert config_path is not None, "no traceability.toml found above this file" + return et.config_from_dict( + __import__("tomllib").loads(config_path.read_text(encoding="utf-8")), + config_path.parent, str(config_path)) + + +def test_this_repos_config_is_valid_and_its_register_parses(): + config = _live_config() + config.validate() + register = et.read_register(config) assert register.total > 0 - for req_type in et.LOCAL_TYPES: + for req_type in config.requirement_types: assert register.count(req_type) > 0, req_type - assert sum(register.count(t) for t in et.LOCAL_TYPES) == register.total - # The GPU-less CI host must be visible in the parse, not just in prose. - assert "AR-027" in register.unexecutable_ids() - assert register.requirements["AR-027"].tiers == {"T4"} - assert register.requirements["AR-012"].tiers == {"T2"} - assert register.unexecutable_ids() < register.ids + assert sum(register.count(t) for t in config.requirement_types) == register.total -def test_the_live_register_yields_a_gate_run_that_cannot_exceed_one_hundred(): - register = et.read_register(et.REPO_ROOT / "docs" / "requirements.md") - files = et.iter_source_files(et.REPO_ROOT) - scan = et.scan_files(files, et.REPO_ROOT) +def test_this_repos_gate_run_cannot_exceed_one_hundred_percent(): + config = _live_config() + register = et.read_register(config) + scan = et.scan_files(et.iter_source_files(config), config) + assert scan.files, "configured source roots matched no files" cov = et.compute_coverage( [i for t in scan.traces for i in t.requirements], register) assert 0.0 <= cov.percent <= 100.0 diff --git a/scripts/traceability/traceability-gate.sh b/scripts/traceability/traceability-gate.sh index 6c8bef7..588848b 100755 --- a/scripts/traceability/traceability-gate.sh +++ b/scripts/traceability/traceability-gate.sh @@ -1,64 +1,62 @@ #!/bin/sh # -# Requirement traceability gate. Run locally exactly as CI runs it: +# Requirement traceability gate. Run locally exactly as CI runs it, from the +# component repo root: # # scripts/traceability/traceability-gate.sh # -# Writes traces-report.json and docs/traceability.md, prints the coverage -# report, and exits non-zero when the gate fails. +# Writes the JSON report and the markdown matrix, prints the coverage report, +# and exits non-zero when the gate fails. # -# Environment: -# MIN_COVERAGE minimum overall coverage percent (default 0 - see below) -# ALLOW_ORPHANS set to 1 to report orphan tags without failing -# TRACES_JSON JSON report path (default traces-report.json) -# TRACES_MD markdown matrix path (default docs/traceability.md) -# SYSTEM_SPEC optional path to the umbrella SPEC.md, which defines the -# PR/SR IDs; when given, PR/SR orphans are reported too. That -# file lives in the parent project, not in this repo, so CI -# normally leaves it unset. +# This script is shared by every JRay component, so it knows nothing about any +# one repo. All repo-specific settings - requirement ID prefixes, source +# suffixes, scan roots, register path, thresholds - live in `traceability.toml` +# at the component repo root. Run # -# Threshold policy lives here and nowhere else. It is deliberately NOT -# duplicated into the workflow YAML: a threshold written in two places is a -# threshold that will disagree with itself. +# scripts/traceability/extract_traces.py --print-example-config # -# MIN_COVERAGE defaults to 0 because almost nothing is tagged yet - tags are -# added as the pipeline is built, so a low number today is accurate rather than -# alarming. A zero threshold does NOT mean the gate cannot fail: orphan tags, -# a >100% ratio, a register that parses to nothing, and an empty source scan -# are all hard failures from day one. Raise MIN_COVERAGE as tags land; treat -# every raise as a ratchet, never a reset. +# for the annotated schema. A repo whose config is wrong parses zero +# requirements or scans zero files, and the gate refuses to report rather than +# printing a misleading 0%. +# +# Environment (all optional; each overrides the config file): +# TRACES_CONFIG path to traceability.toml +# TRACES_ROOT repo root (default: nearest dir containing traceability.toml) +# MIN_COVERAGE minimum overall coverage percent +# ALLOW_ORPHANS 1 to report orphan tags without failing +# TRACES_JSON JSON report path +# TRACES_MD markdown matrix path +# SYSTEM_SPEC SPEC.md defining PR/SR; enables PR/SR orphan checking +# PYTHON interpreter (default: python3) +# +# Threshold policy belongs in traceability.toml, not here and not in the +# workflow YAML: a threshold written in two places is a threshold that will +# disagree with itself. # # POSIX sh, no bashisms, no jq - the extractor does its own arithmetic and -# printing so CI needs nothing beyond python3. +# printing, so CI needs nothing beyond python3. set -eu SCRIPT_DIR=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd) -REPO_ROOT=$(CDPATH= cd -- "$SCRIPT_DIR/../.." && pwd) - -MIN_COVERAGE="${MIN_COVERAGE:-0}" -TRACES_JSON="${TRACES_JSON:-$REPO_ROOT/traces-report.json}" -TRACES_MD="${TRACES_MD:-$REPO_ROOT/docs/traceability.md}" PYTHON="${PYTHON:-python3}" command -v "$PYTHON" >/dev/null 2>&1 || { - echo "FAILED: $PYTHON not found. The traceability gate needs Python 3.9+" >&2 + echo "FAILED: $PYTHON not found. The traceability gate needs Python 3.9+," >&2 + echo " or 3.11+ to read traceability.toml." >&2 exit 2 } -set -- \ - --root "$REPO_ROOT" \ - --format coverage \ - --json-out "$TRACES_JSON" \ - --markdown-out "$TRACES_MD" \ - --min-coverage "$MIN_COVERAGE" +set -- --format coverage -if [ "${ALLOW_ORPHANS:-0}" = "1" ]; then - set -- "$@" --allow-orphans -fi - -if [ -n "${SYSTEM_SPEC:-}" ]; then - set -- "$@" --system-spec "$SYSTEM_SPEC" -fi +# Explicit `if` rather than `[ ... ] && ...`, because a trailing false test in +# an && list exits under `set -e` in some POSIX shells. +if [ -n "${TRACES_CONFIG:-}" ]; then set -- "$@" --config "$TRACES_CONFIG"; fi +if [ -n "${TRACES_ROOT:-}" ]; then set -- "$@" --root "$TRACES_ROOT"; fi +if [ -n "${MIN_COVERAGE:-}" ]; then set -- "$@" --min-coverage "$MIN_COVERAGE"; fi +if [ -n "${TRACES_JSON:-}" ]; then set -- "$@" --json-out "$TRACES_JSON"; fi +if [ -n "${TRACES_MD:-}" ]; then set -- "$@" --markdown-out "$TRACES_MD"; fi +if [ -n "${SYSTEM_SPEC:-}" ]; then set -- "$@" --system-spec "$SYSTEM_SPEC"; fi +if [ "${ALLOW_ORPHANS:-0}" = "1" ]; then set -- "$@" --allow-orphans; fi exec "$PYTHON" "$SCRIPT_DIR/extract_traces.py" "$@" diff --git a/traceability.toml b/traceability.toml new file mode 100644 index 0000000..47c73fb --- /dev/null +++ b/traceability.toml @@ -0,0 +1,34 @@ +# Traceability configuration for scene-actor-extraction. +# +# Read by the shared extractor (scripts/traceability/extract_traces.py), which +# is the same implementation every JRay component uses. Everything repo-specific +# lives here rather than in the tool; run `extract_traces.py +# --print-example-config` for the annotated schema. +# +# This file's directory is taken as the repo root, so the gate works from any +# subdirectory. + +# The prefixes this repo's register defines. Nothing else enters the fraction: +# UT/IT are evidence for requirements, PR/SR belong to the system spec. +requirement_types = ["AR", "DP", "IR", "GR", "VR"] + +# C++ pipeline plus the Python tooling, optimizer and validation scripts. +languages = ["cpp", "python"] + +source_roots = ["src", "tests", "scripts", "experiments", "eval"] + +# CI is an Intel N100 with no discrete GPU. T4 is deliberately absent: a +# requirement verifiable only on GPU hardware is reported as tagged but +# unexecuted and never counted as covered, because counting a test that cannot +# run is the same failure mode as JellyTau's 158% coverage bug. +ci_executable_tiers = ["T1", "T2", "T3", "static"] + +# Threshold policy. 0 today because almost nothing is tagged yet - tags land as +# the pipeline is built. This is not a gate that cannot fail: orphan tags, a +# >100% ratio, a register that parses to nothing and an empty source scan are +# all hard failures already. Ratchet this up as tags land; never reset it down. +min_coverage = 0.0 + +# The system spec owning PR/SR is vendored per-component as a submodule. Point +# at it once that lands to turn on PR/SR orphan checking: +# system_spec = "scripts/vendor/jray-project/SPEC.md"