dtourolle
054c4c8b8f
Ports JellyTau's traceability tooling, rewritten in stdlib Python because
this repo is C++/Python and adding a bun/node toolchain to check source
comments would be a worse trade than writing the scanner.
scripts/traceability/extract_traces.py scans .cpp/.hpp/.py under src, tests,
scripts, experiments and eval for the house tag format
/// TRACES: AR-012, AR-013 | SR-002
and reports EXCEPTION tags separately. An exception is a recorded decision to
depart from an invariant, so folding it into coverage would invert its
meaning; it is listed with its reason, and a missing reason is flagged.
Two rules carried over from JellyTau's gate repair:
* Denominators are parsed out of docs/requirements.md at run time. A
requirement is defined only by a row in a table whose header is
`| ID | Requirement | ... |`, so references in the Traces to column, in
prose, and in the verification-plan table do not inflate the count.
Adding a register row lowers coverage until it is traced - the property
that dies the moment a denominator is frozen.
* Coverage above 100% is a hard failure. It cannot happen through the
intersection, which is the point: if it ever does, the arithmetic is
broken and the run must not be reported as a pass.
One rule specific to this repo: CI is an Intel N100 with no discrete GPU. The
extractor reads each requirement's verification tier from requirements.md and
reports T4/GPU-only requirements as tagged but unexecuted, never as covered.
Counting a test that can never run is the same failure mode as the 158% bug.
MIN_COVERAGE starts at 0 because almost nothing is tagged yet - tags are added
as the pipeline is built. That is not a gate that cannot fail: orphan tags, a
>100% ratio, a register that parses to nothing, and an empty source scan are
all hard failures from day one. The threshold lives in traceability-gate.sh
alone, never duplicated into the workflow YAML.
53 tests over fixture strings, so their meaning does not drift as requirements
are added.
1227 lines
46 KiB
Python
Executable File
1227 lines
46 KiB
Python
Executable File
#!/usr/bin/env python3
|
|
"""Extract requirement traces from C++/Python sources and report coverage.
|
|
|
|
Ported from JellyTau's ``scripts/extract-traces.ts``. That one scans
|
|
TypeScript/Svelte/Rust; this repo is C++ and Python, so the scanner is Python
|
|
with no third-party dependencies — the CI host must not need a node/bun
|
|
toolchain to check traceability.
|
|
|
|
Tag format (see ../../../CLAUDE.md and SPEC.md section 6). A pipe separates
|
|
requirement *types*, a comma separates IDs within a type::
|
|
|
|
/// TRACES: AR-nnn, AR-mmm | SR-nnn
|
|
struct TrackRegistry { ... };
|
|
|
|
Deliberate invariant exceptions carry their own tag and are reported
|
|
separately — never silently folded into coverage::
|
|
|
|
// EXCEPTION: AR-nnn distributional check, not a match decision
|
|
|
|
Usage::
|
|
|
|
python3 scripts/traceability/extract_traces.py --format coverage
|
|
python3 scripts/traceability/extract_traces.py --format json > traces-report.json
|
|
python3 scripts/traceability/extract_traces.py --format markdown --markdown-out docs/traceability.md
|
|
|
|
The CI gate is ``scripts/traceability/traceability-gate.sh``, which wraps this.
|
|
|
|
Two rules inherited from JellyTau's gate repair, both learned the hard way
|
|
(JellyTau/docs/specs/traceability-gate-repair.md):
|
|
|
|
1. Coverage denominators are read out of ``docs/requirements.md`` at run time.
|
|
Never hardcode them. JellyTau's gate divided by frozen literals while the
|
|
register grew to 211 requirements; it reported 158% coverage, so its 50%
|
|
threshold could never trip. A gate that cannot fail is worse than no gate,
|
|
because it is trusted.
|
|
2. Coverage above 100% is a hard failure, not a pass. It means the computation
|
|
is broken, and it is the signal that would have caught (1) immediately.
|
|
|
|
One rule specific to this repo: CI runs on an Intel N100 with no discrete GPU.
|
|
Requirements whose only verification tier is T4/GPU (or "out of CI") cannot
|
|
execute here. They are reported as *tagged but unexecuted* and are excluded
|
|
from the covered numerator — counting a test that never runs as coverage is
|
|
the same failure mode as the 158% bug.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
import json
|
|
import re
|
|
import sys
|
|
from dataclasses import dataclass, field
|
|
from datetime import datetime, timezone
|
|
from pathlib import Path
|
|
from typing import Dict, Iterable, List, Optional, Sequence, Set, Tuple
|
|
|
|
# Derived from this file's location so a CI checkout at any path works.
|
|
SCRIPT_DIR = Path(__file__).resolve().parent
|
|
REPO_ROOT = SCRIPT_DIR.parent.parent
|
|
|
|
# --------------------------------------------------------------------------
|
|
# Taxonomy
|
|
# --------------------------------------------------------------------------
|
|
|
|
#: Types defined in docs/requirements.md. These, and only these, participate
|
|
#: in the coverage fraction.
|
|
LOCAL_TYPES: Tuple[str, ...] = ("AR", "DP", "IR", "GR", "VR")
|
|
|
|
#: Test identifiers. A separate taxonomy: a test ID is evidence *for* a
|
|
#: requirement, not a requirement. Excluded from the fraction.
|
|
TEST_TYPES: Tuple[str, ...] = ("UT", "IT")
|
|
|
|
#: Defined in the system-level SPEC.md, which lives in the umbrella project and
|
|
#: is not part of this repo's checkout. Recognised in tags, never counted here,
|
|
#: and orphan-checked only when --system-spec points at that file.
|
|
EXTERNAL_TYPES: Tuple[str, ...] = ("PR", "SR")
|
|
|
|
KNOWN_TYPES: Tuple[str, ...] = LOCAL_TYPES + TEST_TYPES + EXTERNAL_TYPES
|
|
|
|
#: Tiers a GPU-less CI host can actually run. See the "Verification strategy"
|
|
#: section of docs/requirements.md.
|
|
CI_EXECUTABLE_TIERS: Set[str] = {"T1", "T2", "T3", "static"}
|
|
|
|
# --------------------------------------------------------------------------
|
|
# Source scanning
|
|
# --------------------------------------------------------------------------
|
|
|
|
SOURCE_SUFFIXES = {
|
|
".cpp", ".cc", ".cxx", ".hpp", ".hxx", ".h", ".cu", ".cuh", # C++
|
|
".py", # Python
|
|
}
|
|
|
|
SCAN_ROOTS: Tuple[str, ...] = ("src", "tests", "scripts", "experiments", "eval")
|
|
|
|
EXCLUDED_DIR_NAMES = {
|
|
".git", "__pycache__", "external", "build", "site", "node_modules",
|
|
"trt_cache", "ort_cache", ".venv", "venv", ".mypy_cache", ".pytest_cache",
|
|
}
|
|
|
|
TRACES_RE = re.compile(r"TRACES:[ \t]*([^\n]*)")
|
|
EXCEPTION_RE = re.compile(r"EXCEPTION:[ \t]*([A-Z]{2}-\d{3})[ \t]*([^\n]*)")
|
|
REQ_ID_RE = re.compile(r"^([A-Z]{2})-(\d{3})$")
|
|
LEADING_ID_RE = re.compile(r"^([A-Z]{2}-\d{3})\b(.*)$")
|
|
#: Something that was *trying* to be a requirement ID. Used to keep the
|
|
#: malformed-tag diagnostic quiet when the word "TRACES" merely appears in
|
|
#: prose or in this tool's own source, while still catching `AR-12`.
|
|
ID_ATTEMPT_RE = re.compile(r"[A-Za-z]{2}-\d")
|
|
|
|
# Comment terminators that can trail a tag on the same line.
|
|
COMMENT_TERMINATORS = ("*/", "-->", '"""', "'''")
|
|
|
|
DECL_PATTERNS = [
|
|
re.compile(r"^\s*(?:async\s+)?def\s+\w+"),
|
|
re.compile(r"^\s*class\s+\w+"),
|
|
re.compile(r"^\s*(?:template\s*<[^;]*>\s*)?"
|
|
r"(?:struct|class|enum(?:\s+class)?|union|namespace)\s+\w+"),
|
|
re.compile(r"^\s*(?:static|inline|constexpr|virtual|explicit|friend)\b"),
|
|
re.compile(r"^\s*[A-Za-z_][\w:<>,\s\*&]*\s+[A-Za-z_~][\w:]*\s*\([^)]*\)"),
|
|
]
|
|
|
|
|
|
@dataclass
|
|
class TraceEntry:
|
|
file: str
|
|
line: int
|
|
context: str
|
|
requirements: List[str]
|
|
|
|
def to_dict(self) -> dict:
|
|
return {
|
|
"file": self.file,
|
|
"line": self.line,
|
|
"context": self.context,
|
|
"requirements": list(self.requirements),
|
|
}
|
|
|
|
|
|
@dataclass
|
|
class ExceptionEntry:
|
|
file: str
|
|
line: int
|
|
requirement: str
|
|
reason: str
|
|
context: str
|
|
|
|
def to_dict(self) -> dict:
|
|
return {
|
|
"file": self.file,
|
|
"line": self.line,
|
|
"requirement": self.requirement,
|
|
"reason": self.reason,
|
|
"context": self.context,
|
|
}
|
|
|
|
|
|
@dataclass
|
|
class Diagnostics:
|
|
"""Things wrong with the tags themselves, kept visible rather than dropped."""
|
|
|
|
malformed_tags: List[dict] = field(default_factory=list)
|
|
mixed_type_groups: List[dict] = field(default_factory=list)
|
|
unknown_id_types: List[dict] = field(default_factory=list)
|
|
exceptions_without_reason: List[dict] = field(default_factory=list)
|
|
|
|
def to_dict(self) -> dict:
|
|
return {
|
|
"malformedTags": self.malformed_tags,
|
|
"mixedTypeGroups": self.mixed_type_groups,
|
|
"unknownIdTypes": self.unknown_id_types,
|
|
"exceptionsWithoutReason": self.exceptions_without_reason,
|
|
}
|
|
|
|
@property
|
|
def empty(self) -> bool:
|
|
return not (self.malformed_tags or self.mixed_type_groups
|
|
or self.unknown_id_types or self.exceptions_without_reason)
|
|
|
|
|
|
def parse_traces_tag(value: str) -> Tuple[List[List[str]], List[str]]:
|
|
"""Parse the text after ``TRACES:`` into per-type groups plus junk.
|
|
|
|
Returns ``(groups, junk)``. ``groups`` is one list of IDs per pipe-separated
|
|
segment, preserving the type grouping the format promises. ``junk`` holds
|
|
anything that was not a bare ID, so a tag like ``AR-001 - see also AR-999``
|
|
yields ``AR-001`` and reports the rest rather than quietly harvesting an ID
|
|
out of prose.
|
|
"""
|
|
text = value
|
|
for term in COMMENT_TERMINATORS:
|
|
idx = text.find(term)
|
|
if idx != -1:
|
|
text = text[:idx]
|
|
text = text.strip()
|
|
if not text:
|
|
return [], []
|
|
|
|
groups: List[List[str]] = []
|
|
junk: List[str] = []
|
|
for raw_group in text.split("|"):
|
|
ids: List[str] = []
|
|
for raw_item in raw_group.split(","):
|
|
item = raw_item.strip()
|
|
if not item:
|
|
continue
|
|
if REQ_ID_RE.match(item):
|
|
ids.append(item)
|
|
continue
|
|
lead = LEADING_ID_RE.match(item)
|
|
if lead:
|
|
ids.append(lead.group(1))
|
|
remainder = lead.group(2).strip()
|
|
if remainder:
|
|
junk.append(remainder)
|
|
else:
|
|
junk.append(item)
|
|
if ids:
|
|
groups.append(ids)
|
|
return groups, junk
|
|
|
|
|
|
def find_context(lines: Sequence[str], index: int, window: int = 12) -> str:
|
|
"""Best-effort name of the declaration a tag belongs to.
|
|
|
|
Searches both directions, because the two languages put the tag on opposite
|
|
sides of the thing it describes: a C++ ``/// TRACES:`` sits *above* the
|
|
declaration, while a Python tag usually sits *inside* the docstring, below
|
|
the ``def``. Whichever declaration is nearer wins.
|
|
"""
|
|
def matches(line: str) -> bool:
|
|
return any(p.match(line) for p in DECL_PATTERNS)
|
|
|
|
down: Optional[Tuple[int, str]] = None
|
|
for offset in range(1, window + 1):
|
|
i = index + offset
|
|
if i >= len(lines):
|
|
break
|
|
if matches(lines[i]):
|
|
down = (offset, lines[i])
|
|
break
|
|
|
|
up: Optional[Tuple[int, str]] = None
|
|
for offset in range(1, window + 1):
|
|
i = index - offset
|
|
if i < 0:
|
|
break
|
|
if matches(lines[i]):
|
|
up = (offset, lines[i])
|
|
break
|
|
|
|
best = None
|
|
if down and up:
|
|
best = down if down[0] <= up[0] else up
|
|
else:
|
|
best = down or up
|
|
if best is None:
|
|
return "Unknown"
|
|
return best[1].strip().rstrip("{").strip()[:120] or "Unknown"
|
|
|
|
|
|
def iter_source_files(root: Path,
|
|
scan_roots: Iterable[str] = SCAN_ROOTS) -> List[Path]:
|
|
"""Every C++/Python file under the configured scan roots."""
|
|
found: List[Path] = []
|
|
for rel in scan_roots:
|
|
base = root / rel
|
|
if not base.is_dir():
|
|
continue
|
|
for path in sorted(base.rglob("*")):
|
|
if not path.is_file():
|
|
continue
|
|
if path.suffix not in SOURCE_SUFFIXES:
|
|
continue
|
|
if any(part in EXCLUDED_DIR_NAMES for part in path.parts):
|
|
continue
|
|
found.append(path)
|
|
return found
|
|
|
|
|
|
@dataclass
|
|
class ScanResult:
|
|
files: List[Path]
|
|
traces: List[TraceEntry]
|
|
exceptions: List[ExceptionEntry]
|
|
diagnostics: Diagnostics
|
|
|
|
|
|
def scan_files(files: Sequence[Path], root: Path) -> ScanResult:
|
|
traces: List[TraceEntry] = []
|
|
exceptions: List[ExceptionEntry] = []
|
|
diags = Diagnostics()
|
|
|
|
for path in files:
|
|
try:
|
|
content = path.read_text(encoding="utf-8", errors="replace")
|
|
except OSError:
|
|
continue
|
|
lines = content.split("\n")
|
|
rel = str(path.relative_to(root)) if path.is_relative_to(root) else str(path)
|
|
|
|
for index, line in enumerate(lines):
|
|
exc = EXCEPTION_RE.search(line)
|
|
if exc:
|
|
reason = exc.group(2)
|
|
for term in COMMENT_TERMINATORS:
|
|
cut = reason.find(term)
|
|
if cut != -1:
|
|
reason = reason[:cut]
|
|
reason = reason.strip()
|
|
entry = ExceptionEntry(
|
|
file=rel, line=index + 1, requirement=exc.group(1),
|
|
reason=reason, context=find_context(lines, index),
|
|
)
|
|
exceptions.append(entry)
|
|
if not reason:
|
|
# CLAUDE.md: an exception is only agreed if the reason is
|
|
# recorded. An unexplained one is an undocumented defect.
|
|
diags.exceptions_without_reason.append(
|
|
{"file": rel, "line": index + 1,
|
|
"requirement": exc.group(1)})
|
|
continue
|
|
|
|
match = TRACES_RE.search(line)
|
|
if not match:
|
|
continue
|
|
groups, junk = parse_traces_tag(match.group(1))
|
|
ids = [i for g in groups for i in g]
|
|
if not ids:
|
|
# No IDs at all. Only worth reporting if something in the text
|
|
# was trying to be one — otherwise every sentence containing
|
|
# the word would be flagged, and a diagnostic nobody can act on
|
|
# is how real diagnostics get ignored.
|
|
if junk and ID_ATTEMPT_RE.search(match.group(1)):
|
|
diags.malformed_tags.append(
|
|
{"file": rel, "line": index + 1, "text": match.group(1).strip()})
|
|
continue
|
|
if junk:
|
|
diags.malformed_tags.append(
|
|
{"file": rel, "line": index + 1, "ignored": junk})
|
|
for group in groups:
|
|
types = {i.split("-")[0] for i in group}
|
|
if len(types) > 1:
|
|
# The pipe is what separates types; a mixed group means the
|
|
# tag does not say what it looks like it says.
|
|
diags.mixed_type_groups.append(
|
|
{"file": rel, "line": index + 1, "group": list(group)})
|
|
for req in ids:
|
|
if req.split("-")[0] not in KNOWN_TYPES:
|
|
diags.unknown_id_types.append(
|
|
{"file": rel, "line": index + 1, "id": req})
|
|
|
|
# Deduplicate within one tag while preserving order.
|
|
seen: Set[str] = set()
|
|
unique = [i for i in ids if not (i in seen or seen.add(i))]
|
|
traces.append(TraceEntry(file=rel, line=index + 1,
|
|
context=find_context(lines, index),
|
|
requirements=unique))
|
|
|
|
return ScanResult(files=list(files), traces=traces,
|
|
exceptions=exceptions, diagnostics=diags)
|
|
|
|
|
|
# --------------------------------------------------------------------------
|
|
# The register: docs/requirements.md is the authoritative denominator
|
|
# --------------------------------------------------------------------------
|
|
|
|
@dataclass
|
|
class Requirement:
|
|
id: str
|
|
text: str = ""
|
|
traces_to: str = ""
|
|
priority: str = ""
|
|
status: str = ""
|
|
tiers: Set[str] = field(default_factory=set)
|
|
|
|
@property
|
|
def ci_executable(self) -> bool:
|
|
"""True when at least one verification tier can run on the CI host.
|
|
|
|
A requirement with no tier recorded is *unknown*, not unexecutable — it
|
|
is reported so the register gets fixed, and it is not penalised here.
|
|
"""
|
|
if not self.tiers:
|
|
return True
|
|
return bool(self.tiers & CI_EXECUTABLE_TIERS)
|
|
|
|
@property
|
|
def tier_known(self) -> bool:
|
|
return bool(self.tiers)
|
|
|
|
|
|
@dataclass
|
|
class Register:
|
|
requirements: Dict[str, Requirement] = field(default_factory=dict)
|
|
withdrawn: Dict[str, Requirement] = field(default_factory=dict)
|
|
|
|
@property
|
|
def ids(self) -> Set[str]:
|
|
return set(self.requirements)
|
|
|
|
@property
|
|
def total(self) -> int:
|
|
return len(self.requirements)
|
|
|
|
def count(self, req_type: str) -> int:
|
|
return sum(1 for i in self.requirements if i.startswith(req_type + "-"))
|
|
|
|
def ci_executable_ids(self) -> Set[str]:
|
|
return {i for i, r in self.requirements.items() if r.ci_executable}
|
|
|
|
def unexecutable_ids(self) -> Set[str]:
|
|
return {i for i, r in self.requirements.items() if not r.ci_executable}
|
|
|
|
def tier_unknown_ids(self) -> Set[str]:
|
|
return {i for i, r in self.requirements.items() if not r.tier_known}
|
|
|
|
def parentless_ids(self) -> Set[str]:
|
|
"""Requirements whose ``Traces to`` cell names nothing.
|
|
|
|
SPEC.md section 6 asks the gate to report these: a requirement serving
|
|
no stated goal is scope creep, and it is invisible unless something
|
|
looks. A cell naming a section (`§4`) counts as a parent — the point is
|
|
that *something* was recorded, not that it was an ID.
|
|
"""
|
|
out = set()
|
|
for req_id, req in self.requirements.items():
|
|
cell = req.traces_to.replace("*", "").strip()
|
|
if not cell or cell in {"-", "—", "n/a", "N/A", "TBD"}:
|
|
out.add(req_id)
|
|
return out
|
|
|
|
|
|
def _row_cells(line: str) -> List[str]:
|
|
return [c.strip() for c in line.strip().strip("|").split("|")]
|
|
|
|
|
|
def _is_separator(cells: Sequence[str]) -> bool:
|
|
return bool(cells) and all(re.fullmatch(r":?-{3,}:?", c) for c in cells)
|
|
|
|
|
|
def _iter_table_rows(markdown: str):
|
|
"""Yield ``(header_cells, row_cells)`` for every markdown table data row.
|
|
|
|
A table's header is the row immediately preceding its ``|---|`` separator;
|
|
anything before that separator is not data. Any non-table line ends the
|
|
current table.
|
|
"""
|
|
header: Optional[List[str]] = None
|
|
previous: Optional[List[str]] = None
|
|
for line in markdown.split("\n"):
|
|
stripped = line.strip()
|
|
if not stripped.startswith("|"):
|
|
header = None
|
|
previous = None
|
|
continue
|
|
cells = _row_cells(stripped)
|
|
if _is_separator(cells):
|
|
header = previous
|
|
continue
|
|
if header is not None:
|
|
yield header, cells
|
|
previous = cells
|
|
|
|
|
|
def _is_register_header(header: Sequence[str]) -> bool:
|
|
"""A definition table: first column ``ID``, and a ``Requirement`` column.
|
|
|
|
Deliberately narrow. The per-requirement verification plan is also keyed on
|
|
``ID`` but has no ``Requirement`` column, and the tier summary table's first
|
|
column holds comma lists and ranges — neither defines requirements, and
|
|
counting their rows would inflate the denominator. Prose mentions and
|
|
``Traces to`` references are excluded for the same reason: a naive scan for
|
|
``AR-\\d{3}`` over the whole file counts every reference as a definition.
|
|
"""
|
|
if not header:
|
|
return False
|
|
lowered = [c.lower() for c in header]
|
|
return lowered[0] == "id" and "requirement" in lowered
|
|
|
|
|
|
def _is_tier_header(header: Sequence[str]) -> bool:
|
|
"""Either of the two tables that assign verification tiers."""
|
|
if len(header) < 2:
|
|
return False
|
|
lowered = [c.lower() for c in header]
|
|
return lowered[0] in ("id", "requirement") and lowered[1] == "tier"
|
|
|
|
|
|
def _column(header: Sequence[str], name: str, cells: Sequence[str]) -> str:
|
|
lowered = [c.lower() for c in header]
|
|
if name in lowered:
|
|
index = lowered.index(name)
|
|
if index < len(cells):
|
|
return cells[index]
|
|
return ""
|
|
|
|
|
|
def parse_tiers(cell: str) -> Set[str]:
|
|
"""Read a tier cell such as ``T1 + T4``, ``**T2**``, or ``Out of CI``."""
|
|
text = cell.replace("*", "").strip().lower()
|
|
tiers = {"T" + m.group(1) for m in re.finditer(r"\bt([1-4])\b", text)}
|
|
if "out of ci" in text or "not in ci" in text:
|
|
tiers.add("out-of-ci")
|
|
if "manual" in text:
|
|
tiers.add("manual")
|
|
if "static" in text:
|
|
tiers.add("static")
|
|
return tiers
|
|
|
|
|
|
def expand_id_spec(spec: str, defined: Set[str]) -> List[str]:
|
|
"""Expand the ID cell of a tier table into concrete, defined IDs.
|
|
|
|
Handles every shape the register actually uses: ``AR-002``,
|
|
``AR-001, AR-005, AR-006``, ``AR-007 ... AR-017`` (with the unicode
|
|
ellipsis), ``AR-009/010``, and ``DP-*``. Expansion is intersected with the
|
|
defined set, so a range can never invent a requirement that does not exist.
|
|
"""
|
|
text = spec.replace("*", "").strip()
|
|
# `DP-*` survives the bold-strip above as a bare `DP-`; both mean "every
|
|
# requirement of this type".
|
|
if re.fullmatch(r"[A-Z]{2}-", text):
|
|
prefix = text[:2]
|
|
return sorted(i for i in defined if i.startswith(prefix + "-"))
|
|
|
|
out: List[str] = []
|
|
for part in text.split(","):
|
|
part = part.strip()
|
|
if not part:
|
|
continue
|
|
rng = re.fullmatch(r"([A-Z]{2})-(\d{3})\s*(?:…|\.\.\.)\s*([A-Z]{2})-(\d{3})",
|
|
part)
|
|
if rng and rng.group(1) == rng.group(3):
|
|
prefix = rng.group(1)
|
|
low, high = int(rng.group(2)), int(rng.group(4))
|
|
out.extend(sorted(
|
|
i for i in defined
|
|
if i.startswith(prefix + "-") and low <= int(i.split("-")[1]) <= high))
|
|
continue
|
|
slash = re.fullmatch(r"([A-Z]{2})-(\d{3})/(\d{3})", part)
|
|
if slash:
|
|
prefix = slash.group(1)
|
|
out.extend(f"{prefix}-{n}" for n in (slash.group(2), slash.group(3)))
|
|
continue
|
|
if REQ_ID_RE.match(part):
|
|
out.append(part)
|
|
return [i for i in out if i in defined]
|
|
|
|
|
|
def parse_register(markdown: str) -> Register:
|
|
"""Build the register from ``docs/requirements.md``.
|
|
|
|
Two passes: definitions first, because tier rows use ranges and wildcards
|
|
that can only be expanded against a known ID set.
|
|
"""
|
|
register = Register()
|
|
|
|
for header, cells in _iter_table_rows(markdown):
|
|
if not _is_register_header(header) or not cells:
|
|
continue
|
|
first = cells[0].replace("*", "").strip()
|
|
if not REQ_ID_RE.match(first):
|
|
continue
|
|
if first.split("-")[0] not in LOCAL_TYPES + TEST_TYPES:
|
|
continue
|
|
req = Requirement(
|
|
id=first,
|
|
text=_column(header, "requirement", cells),
|
|
traces_to=_column(header, "traces to", cells),
|
|
priority=_column(header, "priority", cells),
|
|
status=_column(header, "status", cells),
|
|
)
|
|
if req.status.replace("*", "").strip().lower() == "withdrawn":
|
|
# Permanently retired. Counting it in the denominator would depress
|
|
# coverage forever for something that no longer needs implementing.
|
|
register.withdrawn.setdefault(first, req)
|
|
register.requirements.pop(first, None)
|
|
continue
|
|
if first in register.withdrawn:
|
|
continue
|
|
# An ID may legitimately appear in more than one definition table; the
|
|
# first occurrence wins and duplicates never inflate the count.
|
|
register.requirements.setdefault(first, req)
|
|
|
|
defined = register.ids
|
|
for header, cells in _iter_table_rows(markdown):
|
|
if not _is_tier_header(header) or len(cells) < 2:
|
|
continue
|
|
tiers = parse_tiers(cells[1])
|
|
if not tiers:
|
|
continue
|
|
for req_id in expand_id_spec(cells[0], defined):
|
|
register.requirements[req_id].tiers |= tiers
|
|
|
|
return register
|
|
|
|
|
|
def read_register(path: Path) -> Register:
|
|
return parse_register(path.read_text(encoding="utf-8"))
|
|
|
|
|
|
# --------------------------------------------------------------------------
|
|
# Coverage
|
|
# --------------------------------------------------------------------------
|
|
|
|
@dataclass
|
|
class Coverage:
|
|
covered: List[str]
|
|
total: int
|
|
percent: float
|
|
orphaned: List[str]
|
|
unexecuted: List[str]
|
|
ci_executable_total: int
|
|
ci_percent: float
|
|
tier_unknown: List[str]
|
|
|
|
def to_dict(self) -> dict:
|
|
return {
|
|
"covered": len(self.covered),
|
|
"coveredIds": self.covered,
|
|
"total": self.total,
|
|
"percent": self.percent,
|
|
"orphaned": self.orphaned,
|
|
"unexecuted": self.unexecuted,
|
|
"ciExecutableTotal": self.ci_executable_total,
|
|
"ciPercent": self.ci_percent,
|
|
"tierUnknown": self.tier_unknown,
|
|
}
|
|
|
|
|
|
def compute_coverage(traced_ids: Iterable[str], register: Register) -> Coverage:
|
|
"""Coverage is ``|traced and defined and CI-executable| / |defined|``.
|
|
|
|
Three exclusions, each of which is a way the number could otherwise lie:
|
|
|
|
* Using the raw traced count as the numerator is what lets a ratio exceed
|
|
100% — a tag naming a deleted or mistyped requirement would count as
|
|
covered. Those land in ``orphaned`` instead, so they get fixed rather
|
|
than silently counted or silently dropped.
|
|
* A requirement whose only verification tier is T4/GPU cannot run on this
|
|
CI host at all. It is reported in ``unexecuted`` and is not covered:
|
|
treating "has a test that never runs" as passing reports success the gate
|
|
cannot substantiate.
|
|
* UT/IT test IDs and the system-level PR/SR IDs are different taxonomies
|
|
with their own registers, so they neither count nor orphan here.
|
|
"""
|
|
traced = {i for i in traced_ids if i.split("-")[0] in LOCAL_TYPES}
|
|
defined = register.ids
|
|
|
|
orphaned = sorted(traced - defined)
|
|
matched = traced & defined
|
|
unexecuted = sorted(i for i in matched if not register.requirements[i].ci_executable)
|
|
covered = sorted(matched - set(unexecuted))
|
|
|
|
total = len(defined)
|
|
ci_total = len(register.ci_executable_ids())
|
|
percent = round(100.0 * len(covered) / total, 1) if total else 0.0
|
|
ci_percent = round(100.0 * len(covered) / ci_total, 1) if ci_total else 0.0
|
|
|
|
return Coverage(
|
|
covered=covered,
|
|
total=total,
|
|
percent=percent,
|
|
orphaned=orphaned,
|
|
unexecuted=unexecuted,
|
|
ci_executable_total=ci_total,
|
|
ci_percent=ci_percent,
|
|
tier_unknown=sorted(register.tier_unknown_ids()),
|
|
)
|
|
|
|
|
|
# --------------------------------------------------------------------------
|
|
# Report assembly
|
|
# --------------------------------------------------------------------------
|
|
|
|
@dataclass
|
|
class Report:
|
|
timestamp: str
|
|
root: Path
|
|
register: Register
|
|
scan: ScanResult
|
|
coverage: Coverage
|
|
by_type: Dict[str, List[str]]
|
|
requirement_map: Dict[str, List[TraceEntry]]
|
|
external_orphans: List[str] = field(default_factory=list)
|
|
#: Policy, set by the caller: whether an orphan tag fails the run.
|
|
allow_orphans: bool = False
|
|
|
|
|
|
def build_report(root: Path, register: Register, scan: ScanResult,
|
|
system_ids: Optional[Set[str]] = None) -> Report:
|
|
requirement_map: Dict[str, List[TraceEntry]] = {}
|
|
seen_by_type: Dict[str, Set[str]] = {t: set() for t in KNOWN_TYPES}
|
|
seen_by_type["OTHER"] = set()
|
|
|
|
for entry in scan.traces:
|
|
for req in entry.requirements:
|
|
requirement_map.setdefault(req, []).append(entry)
|
|
prefix = req.split("-")[0]
|
|
seen_by_type[prefix if prefix in KNOWN_TYPES else "OTHER"].add(req)
|
|
|
|
coverage = compute_coverage(requirement_map.keys(), register)
|
|
|
|
external_orphans: List[str] = []
|
|
if system_ids is not None:
|
|
external_orphans = sorted(
|
|
i for i in requirement_map
|
|
if i.split("-")[0] in EXTERNAL_TYPES and i not in system_ids)
|
|
|
|
return Report(
|
|
timestamp=datetime.now(timezone.utc).isoformat(timespec="seconds"),
|
|
root=root,
|
|
register=register,
|
|
scan=scan,
|
|
coverage=coverage,
|
|
by_type={k: sorted(v) for k, v in seen_by_type.items()},
|
|
requirement_map=requirement_map,
|
|
external_orphans=external_orphans,
|
|
)
|
|
|
|
|
|
def parse_system_spec(markdown: str) -> Set[str]:
|
|
"""IDs of PR/SR requirements defined in the umbrella SPEC.md.
|
|
|
|
They are defined as headings (``### SR-001 - ...``) and as bolded leading
|
|
table cells (``| **PR-001** | ... |``), so accept both.
|
|
"""
|
|
ids: Set[str] = set()
|
|
for line in markdown.split("\n"):
|
|
stripped = line.strip()
|
|
heading = re.match(r"^#{1,6}\s+\**((?:PR|SR)-\d{3})\**\b", stripped)
|
|
if heading:
|
|
ids.add(heading.group(1))
|
|
continue
|
|
if stripped.startswith("|"):
|
|
cells = _row_cells(stripped)
|
|
if cells:
|
|
first = cells[0].replace("*", "").strip()
|
|
if re.fullmatch(r"(?:PR|SR)-\d{3}", first):
|
|
ids.add(first)
|
|
return ids
|
|
|
|
|
|
def report_to_json_dict(report: Report) -> dict:
|
|
register = report.register
|
|
defined = {t: register.count(t) for t in LOCAL_TYPES}
|
|
defined["total"] = register.total
|
|
|
|
requirement_detail = {}
|
|
for req_id, req in sorted(register.requirements.items()):
|
|
entries = report.requirement_map.get(req_id, [])
|
|
requirement_detail[req_id] = {
|
|
"requirement": req.text,
|
|
"tracesTo": req.traces_to,
|
|
"priority": req.priority,
|
|
"status": req.status,
|
|
"tiers": sorted(req.tiers),
|
|
"ciExecutable": req.ci_executable,
|
|
"taggedIn": sorted({e.file for e in entries}),
|
|
"state": _requirement_state(req, bool(entries)),
|
|
}
|
|
|
|
return {
|
|
"timestamp": report.timestamp,
|
|
"totalFiles": len(report.scan.files),
|
|
"totalTraces": len(report.scan.traces),
|
|
"totalExceptions": len(report.scan.exceptions),
|
|
"requirements": {k: [e.to_dict() for e in v]
|
|
for k, v in sorted(report.requirement_map.items())},
|
|
"byType": report.by_type,
|
|
"defined": defined,
|
|
"coverage": report.coverage.to_dict(),
|
|
"gpuOnlyRequirements": sorted(register.unexecutable_ids()),
|
|
"parentlessRequirements": sorted(register.parentless_ids()),
|
|
"withdrawn": sorted(register.withdrawn),
|
|
"exceptions": [e.to_dict() for e in report.scan.exceptions],
|
|
"externalOrphans": report.external_orphans,
|
|
"diagnostics": report.scan.diagnostics.to_dict(),
|
|
"requirementDetail": requirement_detail,
|
|
}
|
|
|
|
|
|
def per_type_stats(report: Report) -> Dict[str, Tuple[int, int, int]]:
|
|
"""``{type: (covered, unexecuted, defined)}``.
|
|
|
|
Covered here means the same thing it means in the headline figure, so the
|
|
per-type rows sum to it. Reporting "traced and defined" per type while the
|
|
total excludes unexecuted requirements is how a breakdown quietly stops
|
|
adding up to its own total.
|
|
"""
|
|
covered = set(report.coverage.covered)
|
|
unexecuted = set(report.coverage.unexecuted)
|
|
stats: Dict[str, Tuple[int, int, int]] = {}
|
|
for req_type in LOCAL_TYPES:
|
|
prefix = req_type + "-"
|
|
stats[req_type] = (
|
|
len([i for i in covered if i.startswith(prefix)]),
|
|
len([i for i in unexecuted if i.startswith(prefix)]),
|
|
report.register.count(req_type),
|
|
)
|
|
return stats
|
|
|
|
|
|
def _requirement_state(req: Requirement, tagged: bool) -> str:
|
|
if not tagged:
|
|
return "untagged"
|
|
if not req.ci_executable:
|
|
return "tagged-unexecuted"
|
|
return "covered"
|
|
|
|
|
|
# --------------------------------------------------------------------------
|
|
# Output formats
|
|
# --------------------------------------------------------------------------
|
|
|
|
def generate_markdown(report: Report) -> str:
|
|
register = report.register
|
|
cov = report.coverage
|
|
out: List[str] = []
|
|
add = out.append
|
|
|
|
add("# Requirements traceability matrix")
|
|
add("")
|
|
add("<!-- GENERATED FILE - do not edit by hand. -->")
|
|
add("<!-- Regenerate: python3 scripts/traceability/extract_traces.py "
|
|
"--format markdown --markdown-out docs/traceability.md -->")
|
|
add("")
|
|
add(f"**Generated:** {report.timestamp}")
|
|
add("")
|
|
add("Denominators are read from [`requirements.md`](requirements.md) at run "
|
|
"time, never hardcoded. Coverage counts a requirement only when it is "
|
|
"tagged in source **and** has a verification tier this CI host can "
|
|
"execute — CI is an Intel N100 with no discrete GPU.")
|
|
add("")
|
|
|
|
add("## Summary")
|
|
add("")
|
|
add("| Metric | Value |")
|
|
add("|---|---|")
|
|
add(f"| Source files scanned | {len(report.scan.files)} |")
|
|
add(f"| TRACES tags found | {len(report.scan.traces)} |")
|
|
add(f"| EXCEPTION tags found | {len(report.scan.exceptions)} |")
|
|
add(f"| Requirements defined | {register.total} |")
|
|
add(f"| Requirements covered | {len(cov.covered)} |")
|
|
add(f"| **Coverage** | **{cov.percent}%** ({len(cov.covered)}/{cov.total}) |")
|
|
add(f"| Coverage of CI-executable scope | {cov.ci_percent}% "
|
|
f"({len(cov.covered)}/{cov.ci_executable_total}) |")
|
|
add(f"| Tagged but unexecuted in CI (T4/GPU) | {len(cov.unexecuted)} |")
|
|
add(f"| Orphan tags | {len(cov.orphaned)} |")
|
|
add("")
|
|
|
|
add("### By type")
|
|
add("")
|
|
add("| Type | Covered | Tagged but unexecuted | Defined |")
|
|
add("|---|---|---|---|")
|
|
for req_type, (covered, unexecuted, defined_n) in per_type_stats(report).items():
|
|
add(f"| {req_type} | {covered} | {unexecuted} | {defined_n} |")
|
|
add("")
|
|
for req_type in TEST_TYPES + EXTERNAL_TYPES:
|
|
tagged = report.by_type.get(req_type, [])
|
|
if tagged:
|
|
add(f"- **{req_type}** tags present (separate taxonomy, not counted "
|
|
f"in coverage): {', '.join(tagged)}")
|
|
add("")
|
|
|
|
unexecutable = sorted(register.unexecutable_ids())
|
|
add("## Not executable in CI")
|
|
add("")
|
|
add("CI runs on an Intel N100 with no discrete GPU. These requirements have "
|
|
"no verification tier that can run here, so a tag on them is evidence "
|
|
"of *intent*, not of verification. They are never counted as covered.")
|
|
add("")
|
|
if unexecutable:
|
|
add("| ID | Tiers | Tagged in source | Requirement |")
|
|
add("|---|---|---|---|")
|
|
for req_id in unexecutable:
|
|
req = register.requirements[req_id]
|
|
tagged = "yes" if req_id in report.requirement_map else "no"
|
|
add(f"| {req_id} | {', '.join(sorted(req.tiers)) or '-'} | {tagged} "
|
|
f"| {_truncate(req.text)} |")
|
|
else:
|
|
add("_None._")
|
|
add("")
|
|
if cov.unexecuted:
|
|
add(f"**Tagged but unexecuted:** {', '.join(cov.unexecuted)} — a test "
|
|
"exists and is tagged, but only a GPU host can run it. Report "
|
|
"those runs separately.")
|
|
add("")
|
|
|
|
add("## Orphan tags")
|
|
add("")
|
|
add("A tag naming an ID `requirements.md` does not define. This is what "
|
|
"renumbering produces, and what a typo produces.")
|
|
add("")
|
|
if cov.orphaned:
|
|
add("| ID | Locations |")
|
|
add("|---|---|")
|
|
for req_id in cov.orphaned:
|
|
where = ", ".join(f"`{e.file}:{e.line}`"
|
|
for e in report.requirement_map[req_id])
|
|
add(f"| {req_id} | {where} |")
|
|
else:
|
|
add("_None._")
|
|
add("")
|
|
|
|
add("## Requirements tracing up to nothing")
|
|
add("")
|
|
add("A register row whose `Traces to` cell names no parent. Work serving no "
|
|
"stated goal is how scope creeps in, and it is invisible unless "
|
|
"something looks.")
|
|
add("")
|
|
parentless = sorted(register.parentless_ids())
|
|
if parentless:
|
|
add(", ".join(f"`{i}`" for i in parentless))
|
|
else:
|
|
add("_None._")
|
|
add("")
|
|
|
|
add("## Recorded exceptions")
|
|
add("")
|
|
add("Deliberate, documented departures from an invariant "
|
|
"(`EXCEPTION: AR-nnn <reason>`). Reported separately and never counted "
|
|
"as coverage — an exception is a decision to be reviewed, not evidence "
|
|
"a requirement is met.")
|
|
add("")
|
|
if report.scan.exceptions:
|
|
add("| Requirement | Location | Reason |")
|
|
add("|---|---|---|")
|
|
for exc in report.scan.exceptions:
|
|
reason = _truncate(exc.reason, 90) or "**no reason recorded**"
|
|
add(f"| {exc.requirement} | [`{exc.file}:{exc.line}`]"
|
|
f"({_source_link(exc.file)}#L{exc.line}) | {reason} |")
|
|
else:
|
|
add("_None._")
|
|
add("")
|
|
|
|
add("## Register")
|
|
add("")
|
|
add("| ID | Status | Tier | Traces to | Trace state | Tagged in | Requirement |")
|
|
add("|---|---|---|---|---|---|---|")
|
|
for req_id, req in sorted(register.requirements.items(),
|
|
key=lambda kv: (LOCAL_TYPES.index(kv[0].split("-")[0])
|
|
if kv[0].split("-")[0] in LOCAL_TYPES
|
|
else 99, kv[0])):
|
|
entries = report.requirement_map.get(req_id, [])
|
|
state = {"covered": "covered",
|
|
"tagged-unexecuted": "tagged, unexecuted (T4/GPU)",
|
|
"untagged": "untagged"}[
|
|
_requirement_state(req, bool(entries))]
|
|
files = ", ".join(f"`{f}`" for f in sorted({e.file for e in entries})) or "-"
|
|
tiers = ", ".join(sorted(req.tiers)) or "unset"
|
|
add(f"| {req_id} | {_truncate(req.status, 20) or '-'} | {tiers} "
|
|
f"| {_truncate(req.traces_to, 40) or '-'} "
|
|
f"| {state} | {files} | {_truncate(req.text)} |")
|
|
add("")
|
|
|
|
add("## Detailed mapping")
|
|
add("")
|
|
if not report.requirement_map:
|
|
add("_No TRACES tags found yet. Tags are added as code is written; an "
|
|
"empty matrix on a new tree is the correct reading, not a failure._")
|
|
add("")
|
|
for req_id in sorted(report.requirement_map):
|
|
entries = report.requirement_map[req_id]
|
|
add(f"### {req_id}")
|
|
add("")
|
|
add(f"**Locations:** {len(entries)}")
|
|
add("")
|
|
for entry in entries:
|
|
add(f"- [`{entry.file}:{entry.line}`]"
|
|
f"({_source_link(entry.file)}#L{entry.line}) — "
|
|
f"`{_truncate(entry.context, 90)}`")
|
|
add("")
|
|
|
|
if not report.scan.diagnostics.empty:
|
|
add("## Tag diagnostics")
|
|
add("")
|
|
diags = report.scan.diagnostics
|
|
for label, items in (
|
|
("Malformed tags", diags.malformed_tags),
|
|
("Groups mixing requirement types (pipe separates types)",
|
|
diags.mixed_type_groups),
|
|
("Unrecognised ID prefixes", diags.unknown_id_types),
|
|
("Exceptions with no recorded reason", diags.exceptions_without_reason),
|
|
):
|
|
if items:
|
|
add(f"**{label}:**")
|
|
add("")
|
|
for item in items:
|
|
detail = {k: v for k, v in item.items()
|
|
if k not in ("file", "line")}
|
|
add(f"- `{item.get('file')}:{item.get('line')}` — {detail}")
|
|
add("")
|
|
|
|
return "\n".join(out) + "\n"
|
|
|
|
|
|
def _source_link(file_rel: str) -> str:
|
|
"""Link from docs/traceability.md back to a source file at the repo root."""
|
|
return "../" + file_rel
|
|
|
|
|
|
def _truncate(text: str, limit: int = 70) -> str:
|
|
"""Collapse to one line, cap the length, and escape table-breaking pipes."""
|
|
text = " ".join(text.split())
|
|
if len(text) > limit:
|
|
text = text[: limit - 1] + "…"
|
|
return text.replace("|", "\\|")
|
|
|
|
|
|
def format_coverage_report(report: Report, min_coverage: float) -> Tuple[str, int]:
|
|
"""Human-readable gate output plus the exit code it implies."""
|
|
register = report.register
|
|
cov = report.coverage
|
|
lines: List[str] = []
|
|
add = lines.append
|
|
failures: List[str] = []
|
|
|
|
add("Requirement traceability")
|
|
add("=" * 72)
|
|
add(f"Source files scanned : {len(report.scan.files)}")
|
|
add(f"TRACES tags found : {len(report.scan.traces)}")
|
|
add(f"EXCEPTION tags found : {len(report.scan.exceptions)}")
|
|
add("")
|
|
|
|
# Self-checks. With a low threshold these are what make the gate mean
|
|
# something: a parser that silently returns nothing would otherwise report
|
|
# 0/0 and pass.
|
|
if register.total == 0:
|
|
failures.append(
|
|
"requirements.md parsed to ZERO requirements - the register parser "
|
|
"is broken or the file moved. Refusing to report coverage.")
|
|
if not report.scan.files:
|
|
failures.append(
|
|
"no source files were scanned - the scan roots do not exist. "
|
|
"Refusing to report coverage against an empty tree.")
|
|
|
|
add("Coverage by type (covered / defined):")
|
|
for req_type, (covered, unexecuted, defined_n) in per_type_stats(report).items():
|
|
suffix = f" (+{unexecuted} tagged but unexecuted)" if unexecuted else ""
|
|
add(f" {req_type}: {covered} / {defined_n}{suffix}")
|
|
add("")
|
|
add(f"Overall : {len(cov.covered)} / {cov.total} ({cov.percent}%)")
|
|
add(f"CI scope : {len(cov.covered)} / {cov.ci_executable_total} "
|
|
f"({cov.ci_percent}%) [excludes {cov.total - cov.ci_executable_total} "
|
|
"requirement(s) no GPU-less host can verify]")
|
|
add("")
|
|
|
|
unexecutable = sorted(register.unexecutable_ids())
|
|
if unexecutable:
|
|
add(f"Not executable on this CI host (T4/GPU or out-of-CI): "
|
|
f"{len(unexecutable)}")
|
|
add(f" {', '.join(unexecutable)}")
|
|
if cov.unexecuted:
|
|
add("")
|
|
add("TAGGED BUT UNEXECUTED - a test exists and is tagged, but only a "
|
|
"GPU host can run it.")
|
|
add(" These are NOT counted as covered:")
|
|
for req_id in cov.unexecuted:
|
|
tiers = ", ".join(sorted(register.requirements[req_id].tiers))
|
|
add(f" {req_id} (tier {tiers})")
|
|
add("")
|
|
|
|
parentless = sorted(register.parentless_ids())
|
|
if parentless:
|
|
add(f"WARNING: {len(parentless)} requirement(s) trace up to nothing - "
|
|
"no parent recorded in the register's `Traces to` column. Work "
|
|
"serving no stated goal is how scope creeps in:")
|
|
add(f" {', '.join(parentless)}")
|
|
add("")
|
|
|
|
if cov.tier_unknown:
|
|
add(f"WARNING: {len(cov.tier_unknown)} requirement(s) have no "
|
|
"verification tier in requirements.md; they are counted as "
|
|
"CI-executable by default. Add them to the verification plan:")
|
|
add(f" {', '.join(cov.tier_unknown)}")
|
|
add("")
|
|
|
|
if report.scan.exceptions:
|
|
add(f"Recorded invariant exceptions: {len(report.scan.exceptions)}")
|
|
for exc in report.scan.exceptions:
|
|
reason = exc.reason or "(NO REASON RECORDED)"
|
|
add(f" {exc.requirement} {exc.file}:{exc.line} {_truncate(reason, 80)}")
|
|
add("")
|
|
|
|
if cov.orphaned:
|
|
add("ORPHAN TAGS - traced in source, not defined in requirements.md:")
|
|
for req_id in cov.orphaned:
|
|
where = ", ".join(f"{e.file}:{e.line}"
|
|
for e in report.requirement_map[req_id])
|
|
add(f" {req_id} ({where})")
|
|
add(" Fix the tag, or add the requirement to the register.")
|
|
add("")
|
|
|
|
if report.external_orphans:
|
|
add("ORPHAN SYSTEM TAGS - PR/SR IDs the system SPEC.md does not define:")
|
|
add(f" {', '.join(report.external_orphans)}")
|
|
add("")
|
|
|
|
diags = report.scan.diagnostics
|
|
if not diags.empty:
|
|
add("Tag diagnostics:")
|
|
for item in diags.malformed_tags:
|
|
add(f" malformed {item.get('file')}:{item.get('line')} {item}")
|
|
for item in diags.mixed_type_groups:
|
|
add(f" mixed types {item['file']}:{item['line']} {item['group']} "
|
|
"(a pipe, not a comma, separates types)")
|
|
for item in diags.unknown_id_types:
|
|
add(f" unknown id {item['file']}:{item['line']} {item['id']}")
|
|
for item in diags.exceptions_without_reason:
|
|
add(f" exception without reason {item['file']}:{item['line']} "
|
|
f"{item['requirement']}")
|
|
add("")
|
|
|
|
# A ratio above 100% means the computation is broken. This is the check
|
|
# that would have caught JellyTau's frozen denominators immediately.
|
|
if cov.percent > 100 or cov.ci_percent > 100:
|
|
failures.append(
|
|
f"coverage ({cov.percent}%) exceeds 100% - the gate is "
|
|
"miscomputing. Do not trust this run.")
|
|
if len(cov.covered) > cov.total:
|
|
failures.append(
|
|
f"covered ({len(cov.covered)}) exceeds defined ({cov.total}) - "
|
|
"the gate is miscomputing.")
|
|
|
|
if cov.orphaned and not report.allow_orphans:
|
|
failures.append(
|
|
f"{len(cov.orphaned)} orphan tag(s): {', '.join(cov.orphaned)}")
|
|
|
|
if cov.percent < min_coverage:
|
|
failures.append(
|
|
f"coverage ({cov.percent}%) is below the minimum ({min_coverage}%)")
|
|
|
|
if failures:
|
|
add("FAILED:")
|
|
for reason in failures:
|
|
add(f" - {reason}")
|
|
return "\n".join(lines) + "\n", 1
|
|
|
|
add(f"OK: coverage {cov.percent}% >= minimum {min_coverage}%, "
|
|
f"{len(cov.orphaned)} orphan tag(s)")
|
|
return "\n".join(lines) + "\n", 0
|
|
|
|
|
|
# --------------------------------------------------------------------------
|
|
# CLI
|
|
# --------------------------------------------------------------------------
|
|
|
|
def build_arg_parser() -> argparse.ArgumentParser:
|
|
parser = argparse.ArgumentParser(
|
|
prog="extract_traces.py",
|
|
description=__doc__.split("\n")[0],
|
|
formatter_class=argparse.RawDescriptionHelpFormatter,
|
|
)
|
|
parser.add_argument("--format", choices=("json", "markdown", "coverage"),
|
|
default="coverage",
|
|
help="output written to stdout (default: coverage)")
|
|
parser.add_argument("--root", type=Path, default=REPO_ROOT,
|
|
help="repository root to scan (default: derived from "
|
|
"this script's location)")
|
|
parser.add_argument("--requirements", type=Path, default=None,
|
|
help="register path (default: <root>/docs/requirements.md)")
|
|
parser.add_argument("--system-spec", type=Path, default=None,
|
|
help="optional umbrella SPEC.md defining PR/SR IDs; "
|
|
"enables orphan checking for those types")
|
|
parser.add_argument("--json-out", type=Path, default=None,
|
|
help="also write the JSON report here")
|
|
parser.add_argument("--markdown-out", type=Path, default=None,
|
|
help="also write the markdown matrix here")
|
|
parser.add_argument("--min-coverage", type=float, default=0.0,
|
|
help="minimum coverage percent; below it the run fails "
|
|
"(default: 0, i.e. the correctness checks gate but "
|
|
"the percentage does not)")
|
|
parser.add_argument("--allow-orphans", action="store_true",
|
|
help="report orphan tags without failing")
|
|
return parser
|
|
|
|
|
|
def main(argv: Optional[Sequence[str]] = None) -> int:
|
|
args = build_arg_parser().parse_args(argv)
|
|
|
|
root = args.root.resolve()
|
|
req_path = args.requirements or (root / "docs" / "requirements.md")
|
|
if not req_path.is_file():
|
|
print(f"FAILED: requirements register not found at {req_path}",
|
|
file=sys.stderr)
|
|
return 2
|
|
|
|
register = read_register(req_path)
|
|
files = iter_source_files(root)
|
|
scan = scan_files(files, root)
|
|
|
|
system_ids = None
|
|
if args.system_spec:
|
|
if not args.system_spec.is_file():
|
|
print(f"FAILED: --system-spec not found at {args.system_spec}",
|
|
file=sys.stderr)
|
|
return 2
|
|
system_ids = parse_system_spec(args.system_spec.read_text(encoding="utf-8"))
|
|
|
|
report = build_report(root, register, scan, system_ids)
|
|
report.allow_orphans = args.allow_orphans
|
|
|
|
json_text = json.dumps(report_to_json_dict(report), indent=2, sort_keys=False)
|
|
markdown_text = generate_markdown(report)
|
|
|
|
if args.json_out:
|
|
args.json_out.parent.mkdir(parents=True, exist_ok=True)
|
|
args.json_out.write_text(json_text, encoding="utf-8")
|
|
if args.markdown_out:
|
|
args.markdown_out.parent.mkdir(parents=True, exist_ok=True)
|
|
args.markdown_out.write_text(markdown_text, encoding="utf-8")
|
|
|
|
if args.format == "json":
|
|
print(json_text)
|
|
return 0
|
|
if args.format == "markdown":
|
|
print(markdown_text, end="")
|
|
return 0
|
|
|
|
text, code = format_coverage_report(report, args.min_coverage)
|
|
print(text, end="")
|
|
return code
|
|
|
|
|
|
if __name__ == "__main__":
|
|
sys.exit(main())
|