#!/usr/bin/env python3 """Extract requirement traces from C++/Python sources and report coverage. Ported from JellyTau's ``scripts/extract-traces.ts``. That one scans TypeScript/Svelte/Rust; this repo is C++ and Python, so the scanner is Python with no third-party dependencies — the CI host must not need a node/bun toolchain to check traceability. Tag format (see ../../../CLAUDE.md and SPEC.md section 6). A pipe separates requirement *types*, a comma separates IDs within a type:: /// TRACES: AR-nnn, AR-mmm | SR-nnn struct TrackRegistry { ... }; Deliberate invariant exceptions carry their own tag and are reported separately — never silently folded into coverage:: // EXCEPTION: AR-nnn distributional check, not a match decision Usage:: python3 scripts/traceability/extract_traces.py --format coverage python3 scripts/traceability/extract_traces.py --format json > traces-report.json python3 scripts/traceability/extract_traces.py --format markdown --markdown-out docs/traceability.md The CI gate is ``scripts/traceability/traceability-gate.sh``, which wraps this. Two rules inherited from JellyTau's gate repair, both learned the hard way (JellyTau/docs/specs/traceability-gate-repair.md): 1. Coverage denominators are read out of ``docs/requirements.md`` at run time. Never hardcode them. JellyTau's gate divided by frozen literals while the register grew to 211 requirements; it reported 158% coverage, so its 50% threshold could never trip. A gate that cannot fail is worse than no gate, because it is trusted. 2. Coverage above 100% is a hard failure, not a pass. It means the computation is broken, and it is the signal that would have caught (1) immediately. One rule specific to this repo: CI runs on an Intel N100 with no discrete GPU. Requirements whose only verification tier is T4/GPU (or "out of CI") cannot execute here. They are reported as *tagged but unexecuted* and are excluded from the covered numerator — counting a test that never runs as coverage is the same failure mode as the 158% bug. """ from __future__ import annotations import argparse import json import re import sys from dataclasses import dataclass, field from datetime import datetime, timezone from pathlib import Path from typing import Dict, Iterable, List, Optional, Sequence, Set, Tuple # Derived from this file's location so a CI checkout at any path works. SCRIPT_DIR = Path(__file__).resolve().parent REPO_ROOT = SCRIPT_DIR.parent.parent # -------------------------------------------------------------------------- # Taxonomy # -------------------------------------------------------------------------- #: Types defined in docs/requirements.md. These, and only these, participate #: in the coverage fraction. LOCAL_TYPES: Tuple[str, ...] = ("AR", "DP", "IR", "GR", "VR") #: Test identifiers. A separate taxonomy: a test ID is evidence *for* a #: requirement, not a requirement. Excluded from the fraction. TEST_TYPES: Tuple[str, ...] = ("UT", "IT") #: Defined in the system-level SPEC.md, which lives in the umbrella project and #: is not part of this repo's checkout. Recognised in tags, never counted here, #: and orphan-checked only when --system-spec points at that file. EXTERNAL_TYPES: Tuple[str, ...] = ("PR", "SR") KNOWN_TYPES: Tuple[str, ...] = LOCAL_TYPES + TEST_TYPES + EXTERNAL_TYPES #: Tiers a GPU-less CI host can actually run. See the "Verification strategy" #: section of docs/requirements.md. CI_EXECUTABLE_TIERS: Set[str] = {"T1", "T2", "T3", "static"} # -------------------------------------------------------------------------- # Source scanning # -------------------------------------------------------------------------- SOURCE_SUFFIXES = { ".cpp", ".cc", ".cxx", ".hpp", ".hxx", ".h", ".cu", ".cuh", # C++ ".py", # Python } SCAN_ROOTS: Tuple[str, ...] = ("src", "tests", "scripts", "experiments", "eval") EXCLUDED_DIR_NAMES = { ".git", "__pycache__", "external", "build", "site", "node_modules", "trt_cache", "ort_cache", ".venv", "venv", ".mypy_cache", ".pytest_cache", } TRACES_RE = re.compile(r"TRACES:[ \t]*([^\n]*)") EXCEPTION_RE = re.compile(r"EXCEPTION:[ \t]*([A-Z]{2}-\d{3})[ \t]*([^\n]*)") REQ_ID_RE = re.compile(r"^([A-Z]{2})-(\d{3})$") LEADING_ID_RE = re.compile(r"^([A-Z]{2}-\d{3})\b(.*)$") #: Something that was *trying* to be a requirement ID. Used to keep the #: malformed-tag diagnostic quiet when the word "TRACES" merely appears in #: prose or in this tool's own source, while still catching `AR-12`. ID_ATTEMPT_RE = re.compile(r"[A-Za-z]{2}-\d") # Comment terminators that can trail a tag on the same line. COMMENT_TERMINATORS = ("*/", "-->", '"""', "'''") DECL_PATTERNS = [ re.compile(r"^\s*(?:async\s+)?def\s+\w+"), re.compile(r"^\s*class\s+\w+"), re.compile(r"^\s*(?:template\s*<[^;]*>\s*)?" r"(?:struct|class|enum(?:\s+class)?|union|namespace)\s+\w+"), re.compile(r"^\s*(?:static|inline|constexpr|virtual|explicit|friend)\b"), re.compile(r"^\s*[A-Za-z_][\w:<>,\s\*&]*\s+[A-Za-z_~][\w:]*\s*\([^)]*\)"), ] @dataclass class TraceEntry: file: str line: int context: str requirements: List[str] def to_dict(self) -> dict: return { "file": self.file, "line": self.line, "context": self.context, "requirements": list(self.requirements), } @dataclass class ExceptionEntry: file: str line: int requirement: str reason: str context: str def to_dict(self) -> dict: return { "file": self.file, "line": self.line, "requirement": self.requirement, "reason": self.reason, "context": self.context, } @dataclass class Diagnostics: """Things wrong with the tags themselves, kept visible rather than dropped.""" malformed_tags: List[dict] = field(default_factory=list) mixed_type_groups: List[dict] = field(default_factory=list) unknown_id_types: List[dict] = field(default_factory=list) exceptions_without_reason: List[dict] = field(default_factory=list) def to_dict(self) -> dict: return { "malformedTags": self.malformed_tags, "mixedTypeGroups": self.mixed_type_groups, "unknownIdTypes": self.unknown_id_types, "exceptionsWithoutReason": self.exceptions_without_reason, } @property def empty(self) -> bool: return not (self.malformed_tags or self.mixed_type_groups or self.unknown_id_types or self.exceptions_without_reason) def parse_traces_tag(value: str) -> Tuple[List[List[str]], List[str]]: """Parse the text after ``TRACES:`` into per-type groups plus junk. Returns ``(groups, junk)``. ``groups`` is one list of IDs per pipe-separated segment, preserving the type grouping the format promises. ``junk`` holds anything that was not a bare ID, so a tag like ``AR-001 - see also AR-999`` yields ``AR-001`` and reports the rest rather than quietly harvesting an ID out of prose. """ text = value for term in COMMENT_TERMINATORS: idx = text.find(term) if idx != -1: text = text[:idx] text = text.strip() if not text: return [], [] groups: List[List[str]] = [] junk: List[str] = [] for raw_group in text.split("|"): ids: List[str] = [] for raw_item in raw_group.split(","): item = raw_item.strip() if not item: continue if REQ_ID_RE.match(item): ids.append(item) continue lead = LEADING_ID_RE.match(item) if lead: ids.append(lead.group(1)) remainder = lead.group(2).strip() if remainder: junk.append(remainder) else: junk.append(item) if ids: groups.append(ids) return groups, junk def find_context(lines: Sequence[str], index: int, window: int = 12) -> str: """Best-effort name of the declaration a tag belongs to. Searches both directions, because the two languages put the tag on opposite sides of the thing it describes: a C++ ``/// TRACES:`` sits *above* the declaration, while a Python tag usually sits *inside* the docstring, below the ``def``. Whichever declaration is nearer wins. """ def matches(line: str) -> bool: return any(p.match(line) for p in DECL_PATTERNS) down: Optional[Tuple[int, str]] = None for offset in range(1, window + 1): i = index + offset if i >= len(lines): break if matches(lines[i]): down = (offset, lines[i]) break up: Optional[Tuple[int, str]] = None for offset in range(1, window + 1): i = index - offset if i < 0: break if matches(lines[i]): up = (offset, lines[i]) break best = None if down and up: best = down if down[0] <= up[0] else up else: best = down or up if best is None: return "Unknown" return best[1].strip().rstrip("{").strip()[:120] or "Unknown" def iter_source_files(root: Path, scan_roots: Iterable[str] = SCAN_ROOTS) -> List[Path]: """Every C++/Python file under the configured scan roots.""" found: List[Path] = [] for rel in scan_roots: base = root / rel if not base.is_dir(): continue for path in sorted(base.rglob("*")): if not path.is_file(): continue if path.suffix not in SOURCE_SUFFIXES: continue if any(part in EXCLUDED_DIR_NAMES for part in path.parts): continue found.append(path) return found @dataclass class ScanResult: files: List[Path] traces: List[TraceEntry] exceptions: List[ExceptionEntry] diagnostics: Diagnostics def scan_files(files: Sequence[Path], root: Path) -> ScanResult: traces: List[TraceEntry] = [] exceptions: List[ExceptionEntry] = [] diags = Diagnostics() for path in files: try: content = path.read_text(encoding="utf-8", errors="replace") except OSError: continue lines = content.split("\n") rel = str(path.relative_to(root)) if path.is_relative_to(root) else str(path) for index, line in enumerate(lines): exc = EXCEPTION_RE.search(line) if exc: reason = exc.group(2) for term in COMMENT_TERMINATORS: cut = reason.find(term) if cut != -1: reason = reason[:cut] reason = reason.strip() entry = ExceptionEntry( file=rel, line=index + 1, requirement=exc.group(1), reason=reason, context=find_context(lines, index), ) exceptions.append(entry) if not reason: # CLAUDE.md: an exception is only agreed if the reason is # recorded. An unexplained one is an undocumented defect. diags.exceptions_without_reason.append( {"file": rel, "line": index + 1, "requirement": exc.group(1)}) continue match = TRACES_RE.search(line) if not match: continue groups, junk = parse_traces_tag(match.group(1)) ids = [i for g in groups for i in g] if not ids: # No IDs at all. Only worth reporting if something in the text # was trying to be one — otherwise every sentence containing # the word would be flagged, and a diagnostic nobody can act on # is how real diagnostics get ignored. if junk and ID_ATTEMPT_RE.search(match.group(1)): diags.malformed_tags.append( {"file": rel, "line": index + 1, "text": match.group(1).strip()}) continue if junk: diags.malformed_tags.append( {"file": rel, "line": index + 1, "ignored": junk}) for group in groups: types = {i.split("-")[0] for i in group} if len(types) > 1: # The pipe is what separates types; a mixed group means the # tag does not say what it looks like it says. diags.mixed_type_groups.append( {"file": rel, "line": index + 1, "group": list(group)}) for req in ids: if req.split("-")[0] not in KNOWN_TYPES: diags.unknown_id_types.append( {"file": rel, "line": index + 1, "id": req}) # Deduplicate within one tag while preserving order. seen: Set[str] = set() unique = [i for i in ids if not (i in seen or seen.add(i))] traces.append(TraceEntry(file=rel, line=index + 1, context=find_context(lines, index), requirements=unique)) return ScanResult(files=list(files), traces=traces, exceptions=exceptions, diagnostics=diags) # -------------------------------------------------------------------------- # The register: docs/requirements.md is the authoritative denominator # -------------------------------------------------------------------------- @dataclass class Requirement: id: str text: str = "" traces_to: str = "" priority: str = "" status: str = "" tiers: Set[str] = field(default_factory=set) @property def ci_executable(self) -> bool: """True when at least one verification tier can run on the CI host. A requirement with no tier recorded is *unknown*, not unexecutable — it is reported so the register gets fixed, and it is not penalised here. """ if not self.tiers: return True return bool(self.tiers & CI_EXECUTABLE_TIERS) @property def tier_known(self) -> bool: return bool(self.tiers) @dataclass class Register: requirements: Dict[str, Requirement] = field(default_factory=dict) withdrawn: Dict[str, Requirement] = field(default_factory=dict) @property def ids(self) -> Set[str]: return set(self.requirements) @property def total(self) -> int: return len(self.requirements) def count(self, req_type: str) -> int: return sum(1 for i in self.requirements if i.startswith(req_type + "-")) def ci_executable_ids(self) -> Set[str]: return {i for i, r in self.requirements.items() if r.ci_executable} def unexecutable_ids(self) -> Set[str]: return {i for i, r in self.requirements.items() if not r.ci_executable} def tier_unknown_ids(self) -> Set[str]: return {i for i, r in self.requirements.items() if not r.tier_known} def parentless_ids(self) -> Set[str]: """Requirements whose ``Traces to`` cell names nothing. SPEC.md section 6 asks the gate to report these: a requirement serving no stated goal is scope creep, and it is invisible unless something looks. A cell naming a section (`§4`) counts as a parent — the point is that *something* was recorded, not that it was an ID. """ out = set() for req_id, req in self.requirements.items(): cell = req.traces_to.replace("*", "").strip() if not cell or cell in {"-", "—", "n/a", "N/A", "TBD"}: out.add(req_id) return out def _row_cells(line: str) -> List[str]: return [c.strip() for c in line.strip().strip("|").split("|")] def _is_separator(cells: Sequence[str]) -> bool: return bool(cells) and all(re.fullmatch(r":?-{3,}:?", c) for c in cells) def _iter_table_rows(markdown: str): """Yield ``(header_cells, row_cells)`` for every markdown table data row. A table's header is the row immediately preceding its ``|---|`` separator; anything before that separator is not data. Any non-table line ends the current table. """ header: Optional[List[str]] = None previous: Optional[List[str]] = None for line in markdown.split("\n"): stripped = line.strip() if not stripped.startswith("|"): header = None previous = None continue cells = _row_cells(stripped) if _is_separator(cells): header = previous continue if header is not None: yield header, cells previous = cells def _is_register_header(header: Sequence[str]) -> bool: """A definition table: first column ``ID``, and a ``Requirement`` column. Deliberately narrow. The per-requirement verification plan is also keyed on ``ID`` but has no ``Requirement`` column, and the tier summary table's first column holds comma lists and ranges — neither defines requirements, and counting their rows would inflate the denominator. Prose mentions and ``Traces to`` references are excluded for the same reason: a naive scan for ``AR-\\d{3}`` over the whole file counts every reference as a definition. """ if not header: return False lowered = [c.lower() for c in header] return lowered[0] == "id" and "requirement" in lowered def _is_tier_header(header: Sequence[str]) -> bool: """Either of the two tables that assign verification tiers.""" if len(header) < 2: return False lowered = [c.lower() for c in header] return lowered[0] in ("id", "requirement") and lowered[1] == "tier" def _column(header: Sequence[str], name: str, cells: Sequence[str]) -> str: lowered = [c.lower() for c in header] if name in lowered: index = lowered.index(name) if index < len(cells): return cells[index] return "" def parse_tiers(cell: str) -> Set[str]: """Read a tier cell such as ``T1 + T4``, ``**T2**``, or ``Out of CI``.""" text = cell.replace("*", "").strip().lower() tiers = {"T" + m.group(1) for m in re.finditer(r"\bt([1-4])\b", text)} if "out of ci" in text or "not in ci" in text: tiers.add("out-of-ci") if "manual" in text: tiers.add("manual") if "static" in text: tiers.add("static") return tiers def expand_id_spec(spec: str, defined: Set[str]) -> List[str]: """Expand the ID cell of a tier table into concrete, defined IDs. Handles every shape the register actually uses: ``AR-002``, ``AR-001, AR-005, AR-006``, ``AR-007 ... AR-017`` (with the unicode ellipsis), ``AR-009/010``, and ``DP-*``. Expansion is intersected with the defined set, so a range can never invent a requirement that does not exist. """ text = spec.replace("*", "").strip() # `DP-*` survives the bold-strip above as a bare `DP-`; both mean "every # requirement of this type". if re.fullmatch(r"[A-Z]{2}-", text): prefix = text[:2] return sorted(i for i in defined if i.startswith(prefix + "-")) out: List[str] = [] for part in text.split(","): part = part.strip() if not part: continue rng = re.fullmatch(r"([A-Z]{2})-(\d{3})\s*(?:…|\.\.\.)\s*([A-Z]{2})-(\d{3})", part) if rng and rng.group(1) == rng.group(3): prefix = rng.group(1) low, high = int(rng.group(2)), int(rng.group(4)) out.extend(sorted( i for i in defined if i.startswith(prefix + "-") and low <= int(i.split("-")[1]) <= high)) continue slash = re.fullmatch(r"([A-Z]{2})-(\d{3})/(\d{3})", part) if slash: prefix = slash.group(1) out.extend(f"{prefix}-{n}" for n in (slash.group(2), slash.group(3))) continue if REQ_ID_RE.match(part): out.append(part) return [i for i in out if i in defined] def parse_register(markdown: str) -> Register: """Build the register from ``docs/requirements.md``. Two passes: definitions first, because tier rows use ranges and wildcards that can only be expanded against a known ID set. """ register = Register() for header, cells in _iter_table_rows(markdown): if not _is_register_header(header) or not cells: continue first = cells[0].replace("*", "").strip() if not REQ_ID_RE.match(first): continue if first.split("-")[0] not in LOCAL_TYPES + TEST_TYPES: continue req = Requirement( id=first, text=_column(header, "requirement", cells), traces_to=_column(header, "traces to", cells), priority=_column(header, "priority", cells), status=_column(header, "status", cells), ) if req.status.replace("*", "").strip().lower() == "withdrawn": # Permanently retired. Counting it in the denominator would depress # coverage forever for something that no longer needs implementing. register.withdrawn.setdefault(first, req) register.requirements.pop(first, None) continue if first in register.withdrawn: continue # An ID may legitimately appear in more than one definition table; the # first occurrence wins and duplicates never inflate the count. register.requirements.setdefault(first, req) defined = register.ids for header, cells in _iter_table_rows(markdown): if not _is_tier_header(header) or len(cells) < 2: continue tiers = parse_tiers(cells[1]) if not tiers: continue for req_id in expand_id_spec(cells[0], defined): register.requirements[req_id].tiers |= tiers return register def read_register(path: Path) -> Register: return parse_register(path.read_text(encoding="utf-8")) # -------------------------------------------------------------------------- # Coverage # -------------------------------------------------------------------------- @dataclass class Coverage: covered: List[str] total: int percent: float orphaned: List[str] unexecuted: List[str] ci_executable_total: int ci_percent: float tier_unknown: List[str] def to_dict(self) -> dict: return { "covered": len(self.covered), "coveredIds": self.covered, "total": self.total, "percent": self.percent, "orphaned": self.orphaned, "unexecuted": self.unexecuted, "ciExecutableTotal": self.ci_executable_total, "ciPercent": self.ci_percent, "tierUnknown": self.tier_unknown, } def compute_coverage(traced_ids: Iterable[str], register: Register) -> Coverage: """Coverage is ``|traced and defined and CI-executable| / |defined|``. Three exclusions, each of which is a way the number could otherwise lie: * Using the raw traced count as the numerator is what lets a ratio exceed 100% — a tag naming a deleted or mistyped requirement would count as covered. Those land in ``orphaned`` instead, so they get fixed rather than silently counted or silently dropped. * A requirement whose only verification tier is T4/GPU cannot run on this CI host at all. It is reported in ``unexecuted`` and is not covered: treating "has a test that never runs" as passing reports success the gate cannot substantiate. * UT/IT test IDs and the system-level PR/SR IDs are different taxonomies with their own registers, so they neither count nor orphan here. """ traced = {i for i in traced_ids if i.split("-")[0] in LOCAL_TYPES} defined = register.ids orphaned = sorted(traced - defined) matched = traced & defined unexecuted = sorted(i for i in matched if not register.requirements[i].ci_executable) covered = sorted(matched - set(unexecuted)) total = len(defined) ci_total = len(register.ci_executable_ids()) percent = round(100.0 * len(covered) / total, 1) if total else 0.0 ci_percent = round(100.0 * len(covered) / ci_total, 1) if ci_total else 0.0 return Coverage( covered=covered, total=total, percent=percent, orphaned=orphaned, unexecuted=unexecuted, ci_executable_total=ci_total, ci_percent=ci_percent, tier_unknown=sorted(register.tier_unknown_ids()), ) # -------------------------------------------------------------------------- # Report assembly # -------------------------------------------------------------------------- @dataclass class Report: timestamp: str root: Path register: Register scan: ScanResult coverage: Coverage by_type: Dict[str, List[str]] requirement_map: Dict[str, List[TraceEntry]] external_orphans: List[str] = field(default_factory=list) #: Policy, set by the caller: whether an orphan tag fails the run. allow_orphans: bool = False def build_report(root: Path, register: Register, scan: ScanResult, system_ids: Optional[Set[str]] = None) -> Report: requirement_map: Dict[str, List[TraceEntry]] = {} seen_by_type: Dict[str, Set[str]] = {t: set() for t in KNOWN_TYPES} seen_by_type["OTHER"] = set() for entry in scan.traces: for req in entry.requirements: requirement_map.setdefault(req, []).append(entry) prefix = req.split("-")[0] seen_by_type[prefix if prefix in KNOWN_TYPES else "OTHER"].add(req) coverage = compute_coverage(requirement_map.keys(), register) external_orphans: List[str] = [] if system_ids is not None: external_orphans = sorted( i for i in requirement_map if i.split("-")[0] in EXTERNAL_TYPES and i not in system_ids) return Report( timestamp=datetime.now(timezone.utc).isoformat(timespec="seconds"), root=root, register=register, scan=scan, coverage=coverage, by_type={k: sorted(v) for k, v in seen_by_type.items()}, requirement_map=requirement_map, external_orphans=external_orphans, ) def parse_system_spec(markdown: str) -> Set[str]: """IDs of PR/SR requirements defined in the umbrella SPEC.md. They are defined as headings (``### SR-001 - ...``) and as bolded leading table cells (``| **PR-001** | ... |``), so accept both. """ ids: Set[str] = set() for line in markdown.split("\n"): stripped = line.strip() heading = re.match(r"^#{1,6}\s+\**((?:PR|SR)-\d{3})\**\b", stripped) if heading: ids.add(heading.group(1)) continue if stripped.startswith("|"): cells = _row_cells(stripped) if cells: first = cells[0].replace("*", "").strip() if re.fullmatch(r"(?:PR|SR)-\d{3}", first): ids.add(first) return ids def report_to_json_dict(report: Report) -> dict: register = report.register defined = {t: register.count(t) for t in LOCAL_TYPES} defined["total"] = register.total requirement_detail = {} for req_id, req in sorted(register.requirements.items()): entries = report.requirement_map.get(req_id, []) requirement_detail[req_id] = { "requirement": req.text, "tracesTo": req.traces_to, "priority": req.priority, "status": req.status, "tiers": sorted(req.tiers), "ciExecutable": req.ci_executable, "taggedIn": sorted({e.file for e in entries}), "state": _requirement_state(req, bool(entries)), } return { "timestamp": report.timestamp, "totalFiles": len(report.scan.files), "totalTraces": len(report.scan.traces), "totalExceptions": len(report.scan.exceptions), "requirements": {k: [e.to_dict() for e in v] for k, v in sorted(report.requirement_map.items())}, "byType": report.by_type, "defined": defined, "coverage": report.coverage.to_dict(), "gpuOnlyRequirements": sorted(register.unexecutable_ids()), "parentlessRequirements": sorted(register.parentless_ids()), "withdrawn": sorted(register.withdrawn), "exceptions": [e.to_dict() for e in report.scan.exceptions], "externalOrphans": report.external_orphans, "diagnostics": report.scan.diagnostics.to_dict(), "requirementDetail": requirement_detail, } def per_type_stats(report: Report) -> Dict[str, Tuple[int, int, int]]: """``{type: (covered, unexecuted, defined)}``. Covered here means the same thing it means in the headline figure, so the per-type rows sum to it. Reporting "traced and defined" per type while the total excludes unexecuted requirements is how a breakdown quietly stops adding up to its own total. """ covered = set(report.coverage.covered) unexecuted = set(report.coverage.unexecuted) stats: Dict[str, Tuple[int, int, int]] = {} for req_type in LOCAL_TYPES: prefix = req_type + "-" stats[req_type] = ( len([i for i in covered if i.startswith(prefix)]), len([i for i in unexecuted if i.startswith(prefix)]), report.register.count(req_type), ) return stats def _requirement_state(req: Requirement, tagged: bool) -> str: if not tagged: return "untagged" if not req.ci_executable: return "tagged-unexecuted" return "covered" # -------------------------------------------------------------------------- # Output formats # -------------------------------------------------------------------------- def generate_markdown(report: Report) -> str: register = report.register cov = report.coverage out: List[str] = [] add = out.append add("# Requirements traceability matrix") add("") add("") add("") add("") add(f"**Generated:** {report.timestamp}") add("") add("Denominators are read from [`requirements.md`](requirements.md) at run " "time, never hardcoded. Coverage counts a requirement only when it is " "tagged in source **and** has a verification tier this CI host can " "execute — CI is an Intel N100 with no discrete GPU.") add("") add("## Summary") add("") add("| Metric | Value |") add("|---|---|") add(f"| Source files scanned | {len(report.scan.files)} |") add(f"| TRACES tags found | {len(report.scan.traces)} |") add(f"| EXCEPTION tags found | {len(report.scan.exceptions)} |") add(f"| Requirements defined | {register.total} |") add(f"| Requirements covered | {len(cov.covered)} |") add(f"| **Coverage** | **{cov.percent}%** ({len(cov.covered)}/{cov.total}) |") add(f"| Coverage of CI-executable scope | {cov.ci_percent}% " f"({len(cov.covered)}/{cov.ci_executable_total}) |") add(f"| Tagged but unexecuted in CI (T4/GPU) | {len(cov.unexecuted)} |") add(f"| Orphan tags | {len(cov.orphaned)} |") add("") add("### By type") add("") add("| Type | Covered | Tagged but unexecuted | Defined |") add("|---|---|---|---|") for req_type, (covered, unexecuted, defined_n) in per_type_stats(report).items(): add(f"| {req_type} | {covered} | {unexecuted} | {defined_n} |") add("") for req_type in TEST_TYPES + EXTERNAL_TYPES: tagged = report.by_type.get(req_type, []) if tagged: add(f"- **{req_type}** tags present (separate taxonomy, not counted " f"in coverage): {', '.join(tagged)}") add("") unexecutable = sorted(register.unexecutable_ids()) add("## Not executable in CI") add("") add("CI runs on an Intel N100 with no discrete GPU. These requirements have " "no verification tier that can run here, so a tag on them is evidence " "of *intent*, not of verification. They are never counted as covered.") add("") if unexecutable: add("| ID | Tiers | Tagged in source | Requirement |") add("|---|---|---|---|") for req_id in unexecutable: req = register.requirements[req_id] tagged = "yes" if req_id in report.requirement_map else "no" add(f"| {req_id} | {', '.join(sorted(req.tiers)) or '-'} | {tagged} " f"| {_truncate(req.text)} |") else: add("_None._") add("") if cov.unexecuted: add(f"**Tagged but unexecuted:** {', '.join(cov.unexecuted)} — a test " "exists and is tagged, but only a GPU host can run it. Report " "those runs separately.") add("") add("## Orphan tags") add("") add("A tag naming an ID `requirements.md` does not define. This is what " "renumbering produces, and what a typo produces.") add("") if cov.orphaned: add("| ID | Locations |") add("|---|---|") for req_id in cov.orphaned: where = ", ".join(f"`{e.file}:{e.line}`" for e in report.requirement_map[req_id]) add(f"| {req_id} | {where} |") else: add("_None._") add("") add("## Requirements tracing up to nothing") add("") add("A register row whose `Traces to` cell names no parent. Work serving no " "stated goal is how scope creeps in, and it is invisible unless " "something looks.") add("") parentless = sorted(register.parentless_ids()) if parentless: add(", ".join(f"`{i}`" for i in parentless)) else: add("_None._") add("") add("## Recorded exceptions") add("") add("Deliberate, documented departures from an invariant " "(`EXCEPTION: AR-nnn `). Reported separately and never counted " "as coverage — an exception is a decision to be reviewed, not evidence " "a requirement is met.") add("") if report.scan.exceptions: add("| Requirement | Location | Reason |") add("|---|---|---|") for exc in report.scan.exceptions: reason = _truncate(exc.reason, 90) or "**no reason recorded**" add(f"| {exc.requirement} | [`{exc.file}:{exc.line}`]" f"({_source_link(exc.file)}#L{exc.line}) | {reason} |") else: add("_None._") add("") add("## Register") add("") add("| ID | Status | Tier | Traces to | Trace state | Tagged in | Requirement |") add("|---|---|---|---|---|---|---|") for req_id, req in sorted(register.requirements.items(), key=lambda kv: (LOCAL_TYPES.index(kv[0].split("-")[0]) if kv[0].split("-")[0] in LOCAL_TYPES else 99, kv[0])): entries = report.requirement_map.get(req_id, []) state = {"covered": "covered", "tagged-unexecuted": "tagged, unexecuted (T4/GPU)", "untagged": "untagged"}[ _requirement_state(req, bool(entries))] files = ", ".join(f"`{f}`" for f in sorted({e.file for e in entries})) or "-" tiers = ", ".join(sorted(req.tiers)) or "unset" add(f"| {req_id} | {_truncate(req.status, 20) or '-'} | {tiers} " f"| {_truncate(req.traces_to, 40) or '-'} " f"| {state} | {files} | {_truncate(req.text)} |") add("") add("## Detailed mapping") add("") if not report.requirement_map: add("_No TRACES tags found yet. Tags are added as code is written; an " "empty matrix on a new tree is the correct reading, not a failure._") add("") for req_id in sorted(report.requirement_map): entries = report.requirement_map[req_id] add(f"### {req_id}") add("") add(f"**Locations:** {len(entries)}") add("") for entry in entries: add(f"- [`{entry.file}:{entry.line}`]" f"({_source_link(entry.file)}#L{entry.line}) — " f"`{_truncate(entry.context, 90)}`") add("") if not report.scan.diagnostics.empty: add("## Tag diagnostics") add("") diags = report.scan.diagnostics for label, items in ( ("Malformed tags", diags.malformed_tags), ("Groups mixing requirement types (pipe separates types)", diags.mixed_type_groups), ("Unrecognised ID prefixes", diags.unknown_id_types), ("Exceptions with no recorded reason", diags.exceptions_without_reason), ): if items: add(f"**{label}:**") add("") for item in items: detail = {k: v for k, v in item.items() if k not in ("file", "line")} add(f"- `{item.get('file')}:{item.get('line')}` — {detail}") add("") return "\n".join(out) + "\n" def _source_link(file_rel: str) -> str: """Link from docs/traceability.md back to a source file at the repo root.""" return "../" + file_rel def _truncate(text: str, limit: int = 70) -> str: """Collapse to one line, cap the length, and escape table-breaking pipes.""" text = " ".join(text.split()) if len(text) > limit: text = text[: limit - 1] + "…" return text.replace("|", "\\|") def format_coverage_report(report: Report, min_coverage: float) -> Tuple[str, int]: """Human-readable gate output plus the exit code it implies.""" register = report.register cov = report.coverage lines: List[str] = [] add = lines.append failures: List[str] = [] add("Requirement traceability") add("=" * 72) add(f"Source files scanned : {len(report.scan.files)}") add(f"TRACES tags found : {len(report.scan.traces)}") add(f"EXCEPTION tags found : {len(report.scan.exceptions)}") add("") # Self-checks. With a low threshold these are what make the gate mean # something: a parser that silently returns nothing would otherwise report # 0/0 and pass. if register.total == 0: failures.append( "requirements.md parsed to ZERO requirements - the register parser " "is broken or the file moved. Refusing to report coverage.") if not report.scan.files: failures.append( "no source files were scanned - the scan roots do not exist. " "Refusing to report coverage against an empty tree.") add("Coverage by type (covered / defined):") for req_type, (covered, unexecuted, defined_n) in per_type_stats(report).items(): suffix = f" (+{unexecuted} tagged but unexecuted)" if unexecuted else "" add(f" {req_type}: {covered} / {defined_n}{suffix}") add("") add(f"Overall : {len(cov.covered)} / {cov.total} ({cov.percent}%)") add(f"CI scope : {len(cov.covered)} / {cov.ci_executable_total} " f"({cov.ci_percent}%) [excludes {cov.total - cov.ci_executable_total} " "requirement(s) no GPU-less host can verify]") add("") unexecutable = sorted(register.unexecutable_ids()) if unexecutable: add(f"Not executable on this CI host (T4/GPU or out-of-CI): " f"{len(unexecutable)}") add(f" {', '.join(unexecutable)}") if cov.unexecuted: add("") add("TAGGED BUT UNEXECUTED - a test exists and is tagged, but only a " "GPU host can run it.") add(" These are NOT counted as covered:") for req_id in cov.unexecuted: tiers = ", ".join(sorted(register.requirements[req_id].tiers)) add(f" {req_id} (tier {tiers})") add("") parentless = sorted(register.parentless_ids()) if parentless: add(f"WARNING: {len(parentless)} requirement(s) trace up to nothing - " "no parent recorded in the register's `Traces to` column. Work " "serving no stated goal is how scope creeps in:") add(f" {', '.join(parentless)}") add("") if cov.tier_unknown: add(f"WARNING: {len(cov.tier_unknown)} requirement(s) have no " "verification tier in requirements.md; they are counted as " "CI-executable by default. Add them to the verification plan:") add(f" {', '.join(cov.tier_unknown)}") add("") if report.scan.exceptions: add(f"Recorded invariant exceptions: {len(report.scan.exceptions)}") for exc in report.scan.exceptions: reason = exc.reason or "(NO REASON RECORDED)" add(f" {exc.requirement} {exc.file}:{exc.line} {_truncate(reason, 80)}") add("") if cov.orphaned: add("ORPHAN TAGS - traced in source, not defined in requirements.md:") for req_id in cov.orphaned: where = ", ".join(f"{e.file}:{e.line}" for e in report.requirement_map[req_id]) add(f" {req_id} ({where})") add(" Fix the tag, or add the requirement to the register.") add("") if report.external_orphans: add("ORPHAN SYSTEM TAGS - PR/SR IDs the system SPEC.md does not define:") add(f" {', '.join(report.external_orphans)}") add("") diags = report.scan.diagnostics if not diags.empty: add("Tag diagnostics:") for item in diags.malformed_tags: add(f" malformed {item.get('file')}:{item.get('line')} {item}") for item in diags.mixed_type_groups: add(f" mixed types {item['file']}:{item['line']} {item['group']} " "(a pipe, not a comma, separates types)") for item in diags.unknown_id_types: add(f" unknown id {item['file']}:{item['line']} {item['id']}") for item in diags.exceptions_without_reason: add(f" exception without reason {item['file']}:{item['line']} " f"{item['requirement']}") add("") # A ratio above 100% means the computation is broken. This is the check # that would have caught JellyTau's frozen denominators immediately. if cov.percent > 100 or cov.ci_percent > 100: failures.append( f"coverage ({cov.percent}%) exceeds 100% - the gate is " "miscomputing. Do not trust this run.") if len(cov.covered) > cov.total: failures.append( f"covered ({len(cov.covered)}) exceeds defined ({cov.total}) - " "the gate is miscomputing.") if cov.orphaned and not report.allow_orphans: failures.append( f"{len(cov.orphaned)} orphan tag(s): {', '.join(cov.orphaned)}") if cov.percent < min_coverage: failures.append( f"coverage ({cov.percent}%) is below the minimum ({min_coverage}%)") if failures: add("FAILED:") for reason in failures: add(f" - {reason}") return "\n".join(lines) + "\n", 1 add(f"OK: coverage {cov.percent}% >= minimum {min_coverage}%, " f"{len(cov.orphaned)} orphan tag(s)") return "\n".join(lines) + "\n", 0 # -------------------------------------------------------------------------- # CLI # -------------------------------------------------------------------------- def build_arg_parser() -> argparse.ArgumentParser: parser = argparse.ArgumentParser( prog="extract_traces.py", description=__doc__.split("\n")[0], formatter_class=argparse.RawDescriptionHelpFormatter, ) parser.add_argument("--format", choices=("json", "markdown", "coverage"), default="coverage", help="output written to stdout (default: coverage)") parser.add_argument("--root", type=Path, default=REPO_ROOT, help="repository root to scan (default: derived from " "this script's location)") parser.add_argument("--requirements", type=Path, default=None, help="register path (default: /docs/requirements.md)") parser.add_argument("--system-spec", type=Path, default=None, help="optional umbrella SPEC.md defining PR/SR IDs; " "enables orphan checking for those types") parser.add_argument("--json-out", type=Path, default=None, help="also write the JSON report here") parser.add_argument("--markdown-out", type=Path, default=None, help="also write the markdown matrix here") parser.add_argument("--min-coverage", type=float, default=0.0, help="minimum coverage percent; below it the run fails " "(default: 0, i.e. the correctness checks gate but " "the percentage does not)") parser.add_argument("--allow-orphans", action="store_true", help="report orphan tags without failing") return parser def main(argv: Optional[Sequence[str]] = None) -> int: args = build_arg_parser().parse_args(argv) root = args.root.resolve() req_path = args.requirements or (root / "docs" / "requirements.md") if not req_path.is_file(): print(f"FAILED: requirements register not found at {req_path}", file=sys.stderr) return 2 register = read_register(req_path) files = iter_source_files(root) scan = scan_files(files, root) system_ids = None if args.system_spec: if not args.system_spec.is_file(): print(f"FAILED: --system-spec not found at {args.system_spec}", file=sys.stderr) return 2 system_ids = parse_system_spec(args.system_spec.read_text(encoding="utf-8")) report = build_report(root, register, scan, system_ids) report.allow_orphans = args.allow_orphans json_text = json.dumps(report_to_json_dict(report), indent=2, sort_keys=False) markdown_text = generate_markdown(report) if args.json_out: args.json_out.parent.mkdir(parents=True, exist_ok=True) args.json_out.write_text(json_text, encoding="utf-8") if args.markdown_out: args.markdown_out.parent.mkdir(parents=True, exist_ok=True) args.markdown_out.write_text(markdown_text, encoding="utf-8") if args.format == "json": print(json_text) return 0 if args.format == "markdown": print(markdown_text, end="") return 0 text, code = format_coverage_report(report, args.min_coverage) print(text, end="") return code if __name__ == "__main__": sys.exit(main())