|
|
@@ -1,5 +1,5 @@
|
|
|
|
#!/usr/bin/env python3
|
|
|
|
#!/usr/bin/env python3
|
|
|
|
"""Tests for the traceability extractor and coverage gate.
|
|
|
|
"""Tests for the shared traceability extractor and coverage gate.
|
|
|
|
|
|
|
|
|
|
|
|
Run standalone (no third-party dependencies)::
|
|
|
|
Run standalone (no third-party dependencies)::
|
|
|
|
|
|
|
|
|
|
|
@@ -9,26 +9,32 @@ or under pytest, which discovers the same functions::
|
|
|
|
|
|
|
|
|
|
|
|
pytest scripts/traceability/test_extract_traces.py
|
|
|
|
pytest scripts/traceability/test_extract_traces.py
|
|
|
|
|
|
|
|
|
|
|
|
Almost every test runs over fixture strings rather than the live
|
|
|
|
Almost every test runs over fixture strings rather than any live
|
|
|
|
``docs/requirements.md``, so their meaning does not drift as requirements are
|
|
|
|
``requirements.md``, so their meaning does not drift as requirements are added,
|
|
|
|
added. The two properties they exist to pin are the ones JellyTau's gate lost
|
|
|
|
and so they say the same thing in whichever component repo this tool is
|
|
|
|
(see JellyTau/docs/specs/traceability-gate-repair.md):
|
|
|
|
vendored into.
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
Four properties these exist to pin:
|
|
|
|
|
|
|
|
|
|
|
|
* the denominator is computed from the register at run time, so adding a
|
|
|
|
* the denominator is computed from the register at run time, so adding a
|
|
|
|
requirement lowers coverage until it is traced;
|
|
|
|
requirement lowers coverage until it is traced (JellyTau's gate lost this -
|
|
|
|
|
|
|
|
see JellyTau/docs/specs/traceability-gate-repair.md);
|
|
|
|
* the numerator is an intersection, so a tag naming an undefined ID cannot push
|
|
|
|
* the numerator is an intersection, so a tag naming an undefined ID cannot push
|
|
|
|
the ratio above 100%.
|
|
|
|
the ratio above 100%;
|
|
|
|
|
|
|
|
* a requirement whose only verification tier is outside what the CI host can
|
|
|
|
Plus the rule specific to this repo: a requirement only verifiable on GPU
|
|
|
|
execute is reported as tagged-but-unexecuted, never counted as covered;
|
|
|
|
hardware is reported as tagged-but-unexecuted and never counted as covered.
|
|
|
|
* nothing about one repo is baked in - the same code parses the C++/Python,
|
|
|
|
|
|
|
|
C# and Rust components - and a misconfigured run refuses to report rather
|
|
|
|
|
|
|
|
than printing a plausible 0%.
|
|
|
|
"""
|
|
|
|
"""
|
|
|
|
|
|
|
|
|
|
|
|
from __future__ import annotations
|
|
|
|
from __future__ import annotations
|
|
|
|
|
|
|
|
|
|
|
|
import io
|
|
|
|
import io
|
|
|
|
|
|
|
|
import json
|
|
|
|
import sys
|
|
|
|
import sys
|
|
|
|
import tempfile
|
|
|
|
import tempfile
|
|
|
|
from contextlib import redirect_stdout
|
|
|
|
from contextlib import redirect_stderr, redirect_stdout
|
|
|
|
from pathlib import Path
|
|
|
|
from pathlib import Path
|
|
|
|
|
|
|
|
|
|
|
|
HERE = Path(__file__).resolve().parent
|
|
|
|
HERE = Path(__file__).resolve().parent
|
|
|
@@ -36,12 +42,125 @@ sys.path.insert(0, str(HERE))
|
|
|
|
|
|
|
|
|
|
|
|
import extract_traces as et # noqa: E402
|
|
|
|
import extract_traces as et # noqa: E402
|
|
|
|
|
|
|
|
|
|
|
|
# The literal tag keyword is assembled at run time so that this file's fixtures
|
|
|
|
# The literal tag keywords are assembled at run time so that this file's
|
|
|
|
# do not register as real traces when the extractor scans scripts/.
|
|
|
|
# fixtures do not register as real traces when the extractor scans scripts/.
|
|
|
|
TAG = "TRA" + "CES:"
|
|
|
|
TAG = "TRA" + "CES:"
|
|
|
|
EXC = "EXCEP" + "TION:"
|
|
|
|
EXC = "EXCEP" + "TION:"
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def cfg(**overrides) -> et.Config:
|
|
|
|
|
|
|
|
"""A minimal valid config; overrides name whatever a test cares about."""
|
|
|
|
|
|
|
|
base = dict(requirement_types=("AR", "DP", "IR", "GR", "VR"),
|
|
|
|
|
|
|
|
source_suffixes=et.LANGUAGE_SUFFIXES["cpp"]
|
|
|
|
|
|
|
|
| et.LANGUAGE_SUFFIXES["python"],
|
|
|
|
|
|
|
|
source_roots=("src", "scripts"))
|
|
|
|
|
|
|
|
base.update(overrides)
|
|
|
|
|
|
|
|
return et.Config(**base)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
CPP_PY = cfg()
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
# --------------------------------------------------------------------------
|
|
|
|
|
|
|
|
# Configuration: the thing that makes one implementation serve three repos
|
|
|
|
|
|
|
|
# --------------------------------------------------------------------------
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def test_config_rejects_an_empty_requirement_type_list():
|
|
|
|
|
|
|
|
# The prefix cannot be guessed, and guessing wrong means parsing zero
|
|
|
|
|
|
|
|
# requirements - the failure this whole class of check exists to prevent.
|
|
|
|
|
|
|
|
try:
|
|
|
|
|
|
|
|
cfg(requirement_types=()).validate()
|
|
|
|
|
|
|
|
except et.ConfigError as exc:
|
|
|
|
|
|
|
|
assert "requirement_types is empty" in str(exc)
|
|
|
|
|
|
|
|
else:
|
|
|
|
|
|
|
|
raise AssertionError("expected ConfigError")
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def test_config_rejects_a_prefix_that_is_not_two_uppercase_letters():
|
|
|
|
|
|
|
|
try:
|
|
|
|
|
|
|
|
cfg(requirement_types=("Jr",)).validate()
|
|
|
|
|
|
|
|
except et.ConfigError as exc:
|
|
|
|
|
|
|
|
assert "two-letter uppercase" in str(exc)
|
|
|
|
|
|
|
|
else:
|
|
|
|
|
|
|
|
raise AssertionError("expected ConfigError")
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def test_config_rejects_a_prefix_that_is_both_counted_and_excluded():
|
|
|
|
|
|
|
|
# A prefix cannot be in the fraction and out of it at once; silently
|
|
|
|
|
|
|
|
# picking one would make the reported number unexplainable.
|
|
|
|
|
|
|
|
try:
|
|
|
|
|
|
|
|
cfg(requirement_types=("UR", "UT")).validate()
|
|
|
|
|
|
|
|
except et.ConfigError as exc:
|
|
|
|
|
|
|
|
assert "cannot also be excluded" in str(exc)
|
|
|
|
|
|
|
|
else:
|
|
|
|
|
|
|
|
raise AssertionError("expected ConfigError")
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def test_config_rejects_empty_suffixes_and_roots():
|
|
|
|
|
|
|
|
for kwargs, expected in ((dict(source_suffixes=frozenset()), "no source suffixes"),
|
|
|
|
|
|
|
|
(dict(source_roots=()), "no source roots")):
|
|
|
|
|
|
|
|
try:
|
|
|
|
|
|
|
|
cfg(**kwargs).validate()
|
|
|
|
|
|
|
|
except et.ConfigError as exc:
|
|
|
|
|
|
|
|
assert expected in str(exc)
|
|
|
|
|
|
|
|
else:
|
|
|
|
|
|
|
|
raise AssertionError(f"expected ConfigError for {kwargs}")
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def test_config_rejects_a_suffix_missing_its_dot():
|
|
|
|
|
|
|
|
try:
|
|
|
|
|
|
|
|
cfg(source_suffixes=frozenset({"rs"})).validate()
|
|
|
|
|
|
|
|
except et.ConfigError as exc:
|
|
|
|
|
|
|
|
assert "must start with a dot" in str(exc)
|
|
|
|
|
|
|
|
else:
|
|
|
|
|
|
|
|
raise AssertionError("expected ConfigError")
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def test_config_from_dict_expands_language_groups():
|
|
|
|
|
|
|
|
config = et.config_from_dict(
|
|
|
|
|
|
|
|
{"requirement_types": ["UR", "DR"], "languages": ["rust"],
|
|
|
|
|
|
|
|
"source_roots": ["src", "tests"]}, Path("/repo"), "fixture")
|
|
|
|
|
|
|
|
config.validate()
|
|
|
|
|
|
|
|
assert config.source_suffixes == frozenset({".rs"})
|
|
|
|
|
|
|
|
assert config.requirement_types == ("UR", "DR")
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def test_config_from_dict_rejects_an_unknown_key():
|
|
|
|
|
|
|
|
# A typo'd key would leave a required field empty, and the user would be
|
|
|
|
|
|
|
|
# debugging "zero requirements" instead of a misspelling.
|
|
|
|
|
|
|
|
try:
|
|
|
|
|
|
|
|
et.config_from_dict({"requirement_type": ["JR"]}, Path("/repo"), "fixture")
|
|
|
|
|
|
|
|
except et.ConfigError as exc:
|
|
|
|
|
|
|
|
assert "unknown key" in str(exc)
|
|
|
|
|
|
|
|
else:
|
|
|
|
|
|
|
|
raise AssertionError("expected ConfigError")
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def test_config_from_dict_rejects_an_unknown_language():
|
|
|
|
|
|
|
|
try:
|
|
|
|
|
|
|
|
et.config_from_dict({"languages": ["cobol"]}, Path("/repo"), "fixture")
|
|
|
|
|
|
|
|
except et.ConfigError as exc:
|
|
|
|
|
|
|
|
assert "unknown language" in str(exc)
|
|
|
|
|
|
|
|
else:
|
|
|
|
|
|
|
|
raise AssertionError("expected ConfigError")
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def test_example_config_is_itself_valid_and_complete():
|
|
|
|
|
|
|
|
# The documentation people copy must parse, or it teaches the wrong schema.
|
|
|
|
|
|
|
|
tomllib = __import__("tomllib")
|
|
|
|
|
|
|
|
data = tomllib.loads(et.example_config())
|
|
|
|
|
|
|
|
config = et.config_from_dict(data, Path("/repo"), "example")
|
|
|
|
|
|
|
|
config.validate()
|
|
|
|
|
|
|
|
assert config.requirement_types == ("AR", "DP", "IR", "GR", "VR")
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def test_config_paths_resolve_against_the_repo_root_not_the_cwd():
|
|
|
|
|
|
|
|
config = cfg(root=Path("/repo"), requirements_path="docs/requirements.md")
|
|
|
|
|
|
|
|
assert config.resolve("docs/requirements.md") == Path("/repo/docs/requirements.md")
|
|
|
|
|
|
|
|
assert config.resolve("/abs/spec.md") == Path("/abs/spec.md")
|
|
|
|
|
|
|
|
assert config.resolve(None) is None
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
# --------------------------------------------------------------------------
|
|
|
|
# --------------------------------------------------------------------------
|
|
|
|
# Tag parsing
|
|
|
|
# Tag parsing
|
|
|
|
# --------------------------------------------------------------------------
|
|
|
|
# --------------------------------------------------------------------------
|
|
|
@@ -62,8 +181,8 @@ def test_parses_multiple_types_separated_by_pipe():
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def test_parses_three_groups_including_test_ids():
|
|
|
|
def test_parses_three_groups_including_test_ids():
|
|
|
|
groups, _ = et.parse_traces_tag(" AR-012 | SR-002 | UT-003, UT-004")
|
|
|
|
groups, _ = et.parse_traces_tag(" JR-012 | SR-002 | UT-003, UT-004")
|
|
|
|
assert groups == [["AR-012"], ["SR-002"], ["UT-003", "UT-004"]]
|
|
|
|
assert groups == [["JR-012"], ["SR-002"], ["UT-003", "UT-004"]]
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def test_strips_a_trailing_block_comment_terminator():
|
|
|
|
def test_strips_a_trailing_block_comment_terminator():
|
|
|
@@ -94,7 +213,7 @@ def test_two_digit_id_is_not_accepted_as_a_requirement():
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
# --------------------------------------------------------------------------
|
|
|
|
# --------------------------------------------------------------------------
|
|
|
|
# Scanning C++ and Python sources
|
|
|
|
# Scanning: the same scanner over every component's language
|
|
|
|
# --------------------------------------------------------------------------
|
|
|
|
# --------------------------------------------------------------------------
|
|
|
|
|
|
|
|
|
|
|
|
def _write_tree(root: Path) -> None:
|
|
|
|
def _write_tree(root: Path) -> None:
|
|
|
@@ -126,26 +245,82 @@ def test_scans_cpp_and_python_but_not_vendored_or_non_source():
|
|
|
|
with tempfile.TemporaryDirectory() as tmp:
|
|
|
|
with tempfile.TemporaryDirectory() as tmp:
|
|
|
|
root = Path(tmp)
|
|
|
|
root = Path(tmp)
|
|
|
|
_write_tree(root)
|
|
|
|
_write_tree(root)
|
|
|
|
files = et.iter_source_files(root)
|
|
|
|
config = cfg(root=root)
|
|
|
|
names = sorted(f.name for f in files)
|
|
|
|
files = et.iter_source_files(config)
|
|
|
|
assert names == ["gallery.py", "tracker.hpp"], names
|
|
|
|
assert sorted(f.name for f in files) == ["gallery.py", "tracker.hpp"]
|
|
|
|
|
|
|
|
|
|
|
|
scan = et.scan_files(files, root)
|
|
|
|
scan = et.scan_files(files, config)
|
|
|
|
traced = sorted({i for t in scan.traces for i in t.requirements})
|
|
|
|
traced = sorted({i for t in scan.traces for i in t.requirements})
|
|
|
|
assert traced == ["AR-012", "AR-013", "GR-001", "SR-002", "SR-005"]
|
|
|
|
assert traced == ["AR-012", "AR-013", "GR-001", "SR-002", "SR-005"]
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def test_the_same_scanner_reads_rust_when_configured_for_it():
|
|
|
|
|
|
|
|
# JRay-public-server: Rust sources, UR/DR prefixes. No second extractor.
|
|
|
|
|
|
|
|
with tempfile.TemporaryDirectory() as tmp:
|
|
|
|
|
|
|
|
root = Path(tmp)
|
|
|
|
|
|
|
|
(root / "src").mkdir(parents=True)
|
|
|
|
|
|
|
|
(root / "src" / "manifest.rs").write_text(
|
|
|
|
|
|
|
|
f"/// {TAG} UR-002 | SR-004\n"
|
|
|
|
|
|
|
|
"pub fn accept_upload(body: Body) -> Response {\n"
|
|
|
|
|
|
|
|
" todo!()\n"
|
|
|
|
|
|
|
|
"}\n", encoding="utf-8")
|
|
|
|
|
|
|
|
config = cfg(root=root, requirement_types=("UR", "DR"),
|
|
|
|
|
|
|
|
source_suffixes=et.LANGUAGE_SUFFIXES["rust"],
|
|
|
|
|
|
|
|
source_roots=("src",))
|
|
|
|
|
|
|
|
scan = et.scan_files(et.iter_source_files(config), config)
|
|
|
|
|
|
|
|
assert len(scan.traces) == 1
|
|
|
|
|
|
|
|
assert scan.traces[0].requirements == ["UR-002", "SR-004"]
|
|
|
|
|
|
|
|
assert "accept_upload" in scan.traces[0].context
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def test_the_same_scanner_reads_csharp_when_configured_for_it():
|
|
|
|
|
|
|
|
# jRay: C# plugin, JR prefix, and a tag above an attribute-decorated action.
|
|
|
|
|
|
|
|
with tempfile.TemporaryDirectory() as tmp:
|
|
|
|
|
|
|
|
root = Path(tmp)
|
|
|
|
|
|
|
|
(root / "Jellyfin.Plugin.JRay").mkdir(parents=True)
|
|
|
|
|
|
|
|
(root / "Jellyfin.Plugin.JRay" / "TruthController.cs").write_text(
|
|
|
|
|
|
|
|
f" /// {TAG} JR-012 | SR-001\n"
|
|
|
|
|
|
|
|
' [HttpGet("{itemId}")]\n'
|
|
|
|
|
|
|
|
" public ActionResult<TruthFile> GetTruth(Guid itemId)\n"
|
|
|
|
|
|
|
|
" {\n"
|
|
|
|
|
|
|
|
" return Ok();\n"
|
|
|
|
|
|
|
|
" }\n", encoding="utf-8")
|
|
|
|
|
|
|
|
config = cfg(root=root, requirement_types=("JR",),
|
|
|
|
|
|
|
|
source_suffixes=et.LANGUAGE_SUFFIXES["csharp"],
|
|
|
|
|
|
|
|
source_roots=("Jellyfin.Plugin.JRay",))
|
|
|
|
|
|
|
|
scan = et.scan_files(et.iter_source_files(config), config)
|
|
|
|
|
|
|
|
assert len(scan.traces) == 1
|
|
|
|
|
|
|
|
assert scan.traces[0].requirements == ["JR-012", "SR-001"]
|
|
|
|
|
|
|
|
# The attribute must not steal the context from the method it decorates.
|
|
|
|
|
|
|
|
assert "GetTruth" in scan.traces[0].context
|
|
|
|
|
|
|
|
assert "HttpGet" not in scan.traces[0].context
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def test_context_is_found_below_a_cpp_tag_and_above_a_python_tag():
|
|
|
|
def test_context_is_found_below_a_cpp_tag_and_above_a_python_tag():
|
|
|
|
# The two languages put the tag on opposite sides of what it describes.
|
|
|
|
# The two languages put the tag on opposite sides of what it describes.
|
|
|
|
with tempfile.TemporaryDirectory() as tmp:
|
|
|
|
with tempfile.TemporaryDirectory() as tmp:
|
|
|
|
root = Path(tmp)
|
|
|
|
root = Path(tmp)
|
|
|
|
_write_tree(root)
|
|
|
|
_write_tree(root)
|
|
|
|
scan = et.scan_files(et.iter_source_files(root), root)
|
|
|
|
config = cfg(root=root)
|
|
|
|
|
|
|
|
scan = et.scan_files(et.iter_source_files(config), config)
|
|
|
|
contexts = {t.file: t.context for t in scan.traces}
|
|
|
|
contexts = {t.file: t.context for t in scan.traces}
|
|
|
|
assert "TrackRegistry" in contexts["src/tracker.hpp"]
|
|
|
|
assert "TrackRegistry" in contexts["src/tracker.hpp"]
|
|
|
|
assert "build_gallery" in contexts["scripts/gallery.py"]
|
|
|
|
assert "build_gallery" in contexts["scripts/gallery.py"]
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def test_an_id_of_an_unconfigured_prefix_is_reported_as_unknown():
|
|
|
|
|
|
|
|
# In the Rust server, an `AR-001` tag is a copy-paste from another repo.
|
|
|
|
|
|
|
|
with tempfile.TemporaryDirectory() as tmp:
|
|
|
|
|
|
|
|
root = Path(tmp)
|
|
|
|
|
|
|
|
(root / "src").mkdir(parents=True)
|
|
|
|
|
|
|
|
(root / "src" / "a.rs").write_text(f"// {TAG} AR-001\n", encoding="utf-8")
|
|
|
|
|
|
|
|
config = cfg(root=root, requirement_types=("UR", "DR"),
|
|
|
|
|
|
|
|
source_suffixes=et.LANGUAGE_SUFFIXES["rust"],
|
|
|
|
|
|
|
|
source_roots=("src",))
|
|
|
|
|
|
|
|
scan = et.scan_files(et.iter_source_files(config), config)
|
|
|
|
|
|
|
|
assert scan.diagnostics.unknown_id_types
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
# --------------------------------------------------------------------------
|
|
|
|
# --------------------------------------------------------------------------
|
|
|
|
# EXCEPTION tags
|
|
|
|
# EXCEPTION tags
|
|
|
|
# --------------------------------------------------------------------------
|
|
|
|
# --------------------------------------------------------------------------
|
|
|
@@ -158,12 +333,17 @@ EXCEPTION_SOURCE = (
|
|
|
|
)
|
|
|
|
)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def _exception_repo(tmp: str) -> et.Config:
|
|
|
|
|
|
|
|
root = Path(tmp)
|
|
|
|
|
|
|
|
(root / "src").mkdir(parents=True)
|
|
|
|
|
|
|
|
(root / "src" / "outlier.cpp").write_text(EXCEPTION_SOURCE, encoding="utf-8")
|
|
|
|
|
|
|
|
return cfg(root=root)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def test_exception_tag_is_captured_with_its_reason():
|
|
|
|
def test_exception_tag_is_captured_with_its_reason():
|
|
|
|
with tempfile.TemporaryDirectory() as tmp:
|
|
|
|
with tempfile.TemporaryDirectory() as tmp:
|
|
|
|
root = Path(tmp)
|
|
|
|
config = _exception_repo(tmp)
|
|
|
|
(root / "src").mkdir(parents=True)
|
|
|
|
scan = et.scan_files(et.iter_source_files(config), config)
|
|
|
|
(root / "src" / "outlier.cpp").write_text(EXCEPTION_SOURCE, encoding="utf-8")
|
|
|
|
|
|
|
|
scan = et.scan_files(et.iter_source_files(root), root)
|
|
|
|
|
|
|
|
assert len(scan.exceptions) == 1
|
|
|
|
assert len(scan.exceptions) == 1
|
|
|
|
exc = scan.exceptions[0]
|
|
|
|
exc = scan.exceptions[0]
|
|
|
|
assert exc.requirement == "AR-024"
|
|
|
|
assert exc.requirement == "AR-024"
|
|
|
@@ -175,14 +355,12 @@ def test_exception_is_never_counted_as_coverage():
|
|
|
|
# An exception is a recorded decision to depart from an invariant. Counting
|
|
|
|
# An exception is a recorded decision to depart from an invariant. Counting
|
|
|
|
# it as evidence the requirement is met inverts its meaning.
|
|
|
|
# it as evidence the requirement is met inverts its meaning.
|
|
|
|
with tempfile.TemporaryDirectory() as tmp:
|
|
|
|
with tempfile.TemporaryDirectory() as tmp:
|
|
|
|
root = Path(tmp)
|
|
|
|
config = _exception_repo(tmp)
|
|
|
|
(root / "src").mkdir(parents=True)
|
|
|
|
scan = et.scan_files(et.iter_source_files(config), config)
|
|
|
|
(root / "src" / "outlier.cpp").write_text(EXCEPTION_SOURCE, encoding="utf-8")
|
|
|
|
|
|
|
|
scan = et.scan_files(et.iter_source_files(root), root)
|
|
|
|
|
|
|
|
assert scan.traces == []
|
|
|
|
assert scan.traces == []
|
|
|
|
register = et.parse_register(
|
|
|
|
register = et.parse_register(
|
|
|
|
"| ID | Requirement | Status |\n|---|---|---|\n"
|
|
|
|
"| ID | Requirement | Status |\n|---|---|---|\n"
|
|
|
|
"| AR-024 | Always the calibrated probability | Planned |\n")
|
|
|
|
"| AR-024 | Always the calibrated probability | Planned |\n", config)
|
|
|
|
cov = et.compute_coverage(
|
|
|
|
cov = et.compute_coverage(
|
|
|
|
[i for t in scan.traces for i in t.requirements], register)
|
|
|
|
[i for t in scan.traces for i in t.requirements], register)
|
|
|
|
assert cov.covered == []
|
|
|
|
assert cov.covered == []
|
|
|
@@ -190,13 +368,13 @@ def test_exception_is_never_counted_as_coverage():
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def test_exception_without_a_reason_is_reported():
|
|
|
|
def test_exception_without_a_reason_is_reported():
|
|
|
|
# CLAUDE.md: an exception is only *agreed* if the reason is recorded.
|
|
|
|
# An exception is only *agreed* if the reason is recorded.
|
|
|
|
with tempfile.TemporaryDirectory() as tmp:
|
|
|
|
with tempfile.TemporaryDirectory() as tmp:
|
|
|
|
root = Path(tmp)
|
|
|
|
root = Path(tmp)
|
|
|
|
(root / "src").mkdir(parents=True)
|
|
|
|
(root / "src").mkdir(parents=True)
|
|
|
|
(root / "src" / "bare.cpp").write_text(
|
|
|
|
(root / "src" / "bare.cpp").write_text(f"// {EXC} AR-024\n", encoding="utf-8")
|
|
|
|
f"// {EXC} AR-024\n", encoding="utf-8")
|
|
|
|
config = cfg(root=root)
|
|
|
|
scan = et.scan_files(et.iter_source_files(root), root)
|
|
|
|
scan = et.scan_files(et.iter_source_files(config), config)
|
|
|
|
assert len(scan.exceptions) == 1
|
|
|
|
assert len(scan.exceptions) == 1
|
|
|
|
assert scan.diagnostics.exceptions_without_reason
|
|
|
|
assert scan.diagnostics.exceptions_without_reason
|
|
|
|
|
|
|
|
|
|
|
@@ -209,7 +387,8 @@ def test_mixed_type_group_is_reported():
|
|
|
|
(root / "src").mkdir(parents=True)
|
|
|
|
(root / "src").mkdir(parents=True)
|
|
|
|
(root / "src" / "a.cpp").write_text(
|
|
|
|
(root / "src" / "a.cpp").write_text(
|
|
|
|
f"// {TAG} AR-001, SR-002\n", encoding="utf-8")
|
|
|
|
f"// {TAG} AR-001, SR-002\n", encoding="utf-8")
|
|
|
|
scan = et.scan_files(et.iter_source_files(root), root)
|
|
|
|
config = cfg(root=root)
|
|
|
|
|
|
|
|
scan = et.scan_files(et.iter_source_files(config), config)
|
|
|
|
assert scan.diagnostics.mixed_type_groups
|
|
|
|
assert scan.diagnostics.mixed_type_groups
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
@@ -224,20 +403,41 @@ def test_counts_a_well_formed_table_row_as_a_defined_requirement():
|
|
|
|
md = REGISTER_HEADER + (
|
|
|
|
md = REGISTER_HEADER + (
|
|
|
|
"| AR-001 | Detect faces in sampled frames | SR-002 | High | Done |\n"
|
|
|
|
"| AR-001 | Detect faces in sampled frames | SR-002 | High | Done |\n"
|
|
|
|
"| AR-002 | Minimum face size 66x66 px | SR-002 | High | Planned |\n")
|
|
|
|
"| AR-002 | Minimum face size 66x66 px | SR-002 | High | Planned |\n")
|
|
|
|
register = et.parse_register(md)
|
|
|
|
register = et.parse_register(md, CPP_PY)
|
|
|
|
assert register.count("AR") == 2
|
|
|
|
assert register.count("AR") == 2
|
|
|
|
assert register.count("GR") == 0
|
|
|
|
assert register.count("GR") == 0
|
|
|
|
assert register.total == 2
|
|
|
|
assert register.total == 2
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def test_the_same_parser_reads_another_repos_prefixes():
|
|
|
|
|
|
|
|
# Identical table shape, different prefixes. This is the change that made
|
|
|
|
|
|
|
|
# the tool shareable: with requirement_types hardcoded, both of these
|
|
|
|
|
|
|
|
# parsed to zero.
|
|
|
|
|
|
|
|
md = REGISTER_HEADER + (
|
|
|
|
|
|
|
|
"| JR-001 | Truth-file schema | SR-003 | High | Done |\n"
|
|
|
|
|
|
|
|
"| UR-001 | Availability without payload | SR-001 | High | Done |\n"
|
|
|
|
|
|
|
|
"| DR-003 | One writer connection plus a read pool | SR-001 | High | Done |\n")
|
|
|
|
|
|
|
|
jray = et.parse_register(md, cfg(requirement_types=("JR",)))
|
|
|
|
|
|
|
|
server = et.parse_register(md, cfg(requirement_types=("UR", "DR")))
|
|
|
|
|
|
|
|
assert jray.ids == {"JR-001"}
|
|
|
|
|
|
|
|
assert server.ids == {"UR-001", "DR-003"}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def test_a_prefix_this_repo_does_not_define_is_not_counted_as_defined():
|
|
|
|
|
|
|
|
# The server's register must not acquire a denominator from an AR row
|
|
|
|
|
|
|
|
# pasted in from the extraction repo.
|
|
|
|
|
|
|
|
md = REGISTER_HEADER + "| AR-001 | Belongs to another repo | SR-002 | High | Done |\n"
|
|
|
|
|
|
|
|
register = et.parse_register(md, cfg(requirement_types=("UR", "DR")))
|
|
|
|
|
|
|
|
assert register.total == 0
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def test_does_not_count_ids_that_appear_only_in_the_traces_to_column():
|
|
|
|
def test_does_not_count_ids_that_appear_only_in_the_traces_to_column():
|
|
|
|
# The bug this parse rule avoids: a naive scan for AR-\d{3} over the whole
|
|
|
|
# The bug this parse rule avoids: a naive scan for AR-\d{3} over the whole
|
|
|
|
# file counts every reference as a definition and inflates the denominator.
|
|
|
|
# file counts every reference as a definition and inflates the denominator.
|
|
|
|
md = REGISTER_HEADER + (
|
|
|
|
md = REGISTER_HEADER + (
|
|
|
|
"| GR-001 | Build gallery from library cast | SR-001, SR-005 | High | Done |\n"
|
|
|
|
"| GR-001 | Build gallery from library cast | SR-001, SR-005 | High | Done |\n"
|
|
|
|
"| GR-002 | Incremental merge refresh | PR-003 | High | Done |\n")
|
|
|
|
"| GR-002 | Incremental merge refresh | PR-003 | High | Done |\n")
|
|
|
|
register = et.parse_register(md)
|
|
|
|
register = et.parse_register(md, CPP_PY)
|
|
|
|
assert register.count("GR") == 2
|
|
|
|
|
|
|
|
assert register.ids == {"GR-001", "GR-002"}
|
|
|
|
assert register.ids == {"GR-001", "GR-002"}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
@@ -245,7 +445,7 @@ def test_does_not_count_ids_mentioned_in_prose():
|
|
|
|
md = ("Some prose explaining that AR-005 relates to GR-001 and VR-003.\n\n"
|
|
|
|
md = ("Some prose explaining that AR-005 relates to GR-001 and VR-003.\n\n"
|
|
|
|
+ REGISTER_HEADER
|
|
|
|
+ REGISTER_HEADER
|
|
|
|
+ "| AR-005 | Align to 112x112 | SR-002 | High | Done |\n")
|
|
|
|
+ "| AR-005 | Align to 112x112 | SR-002 | High | Done |\n")
|
|
|
|
register = et.parse_register(md)
|
|
|
|
register = et.parse_register(md, CPP_PY)
|
|
|
|
assert register.ids == {"AR-005"}
|
|
|
|
assert register.ids == {"AR-005"}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
@@ -259,9 +459,8 @@ def test_does_not_count_the_verification_plan_table_as_definitions():
|
|
|
|
+ "| ID | Tier | Test asserts | Edge cases to cover |\n|---|---|---|---|\n"
|
|
|
|
+ "| ID | Tier | Test asserts | Edge cases to cover |\n|---|---|---|---|\n"
|
|
|
|
+ "| AR-001 | T3 | Detector returns plausible boxes | smoke only |\n"
|
|
|
|
+ "| AR-001 | T3 | Detector returns plausible boxes | smoke only |\n"
|
|
|
|
+ "| AR-099 | T1 | Something not in the register | - |\n")
|
|
|
|
+ "| AR-099 | T1 | Something not in the register | - |\n")
|
|
|
|
register = et.parse_register(md)
|
|
|
|
register = et.parse_register(md, CPP_PY)
|
|
|
|
assert register.ids == {"AR-001"}
|
|
|
|
assert register.ids == {"AR-001"}
|
|
|
|
assert register.total == 1
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def test_deduplicates_an_id_listed_in_two_definition_tables():
|
|
|
|
def test_deduplicates_an_id_listed_in_two_definition_tables():
|
|
|
@@ -270,8 +469,7 @@ def test_deduplicates_an_id_listed_in_two_definition_tables():
|
|
|
|
+ "\n"
|
|
|
|
+ "\n"
|
|
|
|
+ REGISTER_HEADER
|
|
|
|
+ REGISTER_HEADER
|
|
|
|
+ "| AR-001 | Detect faces | SR-002 | High | Done |\n")
|
|
|
|
+ "| AR-001 | Detect faces | SR-002 | High | Done |\n")
|
|
|
|
register = et.parse_register(md)
|
|
|
|
assert et.parse_register(md, CPP_PY).total == 1
|
|
|
|
assert register.total == 1
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def test_withdrawn_requirements_leave_the_denominator():
|
|
|
|
def test_withdrawn_requirements_leave_the_denominator():
|
|
|
@@ -280,18 +478,29 @@ def test_withdrawn_requirements_leave_the_denominator():
|
|
|
|
md = REGISTER_HEADER + (
|
|
|
|
md = REGISTER_HEADER + (
|
|
|
|
"| AR-001 | Detect faces | SR-002 | High | Done |\n"
|
|
|
|
"| AR-001 | Detect faces | SR-002 | High | Done |\n"
|
|
|
|
"| AR-002 | Superseded mechanism | SR-002 | High | Withdrawn |\n")
|
|
|
|
"| AR-002 | Superseded mechanism | SR-002 | High | Withdrawn |\n")
|
|
|
|
register = et.parse_register(md)
|
|
|
|
register = et.parse_register(md, CPP_PY)
|
|
|
|
assert register.ids == {"AR-001"}
|
|
|
|
assert register.ids == {"AR-001"}
|
|
|
|
assert "AR-002" in register.withdrawn
|
|
|
|
assert "AR-002" in register.withdrawn
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def test_an_escaped_pipe_inside_a_cell_does_not_split_the_row():
|
|
|
|
|
|
|
|
# The server's register contains `small-\|M\| all-but-one rule`; a naive
|
|
|
|
|
|
|
|
# split shifts every later column by one.
|
|
|
|
|
|
|
|
md = REGISTER_HEADER + (
|
|
|
|
|
|
|
|
r"| UR-003 | Strict schema with a small-\|M\| rule | SR-001 | High | Done |"
|
|
|
|
|
|
|
|
"\n")
|
|
|
|
|
|
|
|
register = et.parse_register(md, cfg(requirement_types=("UR",)))
|
|
|
|
|
|
|
|
assert register.requirements["UR-003"].status == "Done"
|
|
|
|
|
|
|
|
assert "|M|" in register.requirements["UR-003"].text
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def test_the_denominator_is_live_adding_a_row_lowers_coverage():
|
|
|
|
def test_the_denominator_is_live_adding_a_row_lowers_coverage():
|
|
|
|
# The property JellyTau's frozen literals destroyed. Same traced set, one
|
|
|
|
# The property JellyTau's frozen literals destroyed. Same traced set, one
|
|
|
|
# more requirement defined => a lower percentage, mechanically.
|
|
|
|
# more requirement defined => a lower percentage, mechanically.
|
|
|
|
base = REGISTER_HEADER + "| AR-001 | A | SR-002 | High | Done |\n"
|
|
|
|
base = REGISTER_HEADER + "| AR-001 | A | SR-002 | High | Done |\n"
|
|
|
|
grown = base + "| AR-002 | B | SR-002 | High | Planned |\n"
|
|
|
|
grown = base + "| AR-002 | B | SR-002 | High | Planned |\n"
|
|
|
|
before = et.compute_coverage(["AR-001"], et.parse_register(base))
|
|
|
|
before = et.compute_coverage(["AR-001"], et.parse_register(base, CPP_PY))
|
|
|
|
after = et.compute_coverage(["AR-001"], et.parse_register(grown))
|
|
|
|
after = et.compute_coverage(["AR-001"], et.parse_register(grown, CPP_PY))
|
|
|
|
assert before.percent == 100.0
|
|
|
|
assert before.percent == 100.0
|
|
|
|
assert after.percent == 50.0
|
|
|
|
assert after.percent == 50.0
|
|
|
|
assert after.total == 2
|
|
|
|
assert after.total == 2
|
|
|
@@ -300,14 +509,26 @@ def test_the_denominator_is_live_adding_a_row_lowers_coverage():
|
|
|
|
def test_register_captures_the_row_fields_not_just_the_id():
|
|
|
|
def test_register_captures_the_row_fields_not_just_the_id():
|
|
|
|
md = REGISTER_HEADER + (
|
|
|
|
md = REGISTER_HEADER + (
|
|
|
|
"| AR-012 | Presence follows track extent | **SR-002** | High | Planned |\n")
|
|
|
|
"| AR-012 | Presence follows track extent | **SR-002** | High | Planned |\n")
|
|
|
|
req = et.parse_register(md).requirements["AR-012"]
|
|
|
|
req = et.parse_register(md, CPP_PY).requirements["AR-012"]
|
|
|
|
assert req.text == "Presence follows track extent"
|
|
|
|
assert req.text == "Presence follows track extent"
|
|
|
|
assert req.traces_to == "**SR-002**"
|
|
|
|
assert req.traces_to == "**SR-002**"
|
|
|
|
assert req.status == "Planned"
|
|
|
|
assert req.status == "Planned"
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def test_a_requirement_tracing_up_to_nothing_is_reported():
|
|
|
|
|
|
|
|
# SPEC.md section 6: a requirement citing no parent is scope creep, and it
|
|
|
|
|
|
|
|
# is invisible unless something looks. A section reference counts as a
|
|
|
|
|
|
|
|
# parent - what matters is that something was recorded.
|
|
|
|
|
|
|
|
md = REGISTER_HEADER + (
|
|
|
|
|
|
|
|
"| AR-001 | Has a parent | SR-002 | High | Done |\n"
|
|
|
|
|
|
|
|
"| AR-002 | Parent is a section | §4 | Medium | Planned |\n"
|
|
|
|
|
|
|
|
"| AR-003 | Serves nothing stated | - | Low | Planned |\n"
|
|
|
|
|
|
|
|
"| AR-004 | Blank cell | | Low | Planned |\n")
|
|
|
|
|
|
|
|
assert et.parse_register(md, CPP_PY).parentless_ids() == {"AR-003", "AR-004"}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
# --------------------------------------------------------------------------
|
|
|
|
# --------------------------------------------------------------------------
|
|
|
|
# Verification tiers and the GPU-less CI host
|
|
|
|
# Verification tiers and the CI host's limits
|
|
|
|
# --------------------------------------------------------------------------
|
|
|
|
# --------------------------------------------------------------------------
|
|
|
|
|
|
|
|
|
|
|
|
TIER_REGISTER = REGISTER_HEADER + "".join(
|
|
|
|
TIER_REGISTER = REGISTER_HEADER + "".join(
|
|
|
@@ -327,7 +548,7 @@ def test_tier_assignment_handles_lists_ranges_and_wildcards():
|
|
|
|
"| AR-002 … AR-004 | **T2** | replay |\n"
|
|
|
|
"| AR-002 … AR-004 | **T2** | replay |\n"
|
|
|
|
"| AR-006 | T1 + T4 | mixed |\n"
|
|
|
|
"| AR-006 | T1 + T4 | mixed |\n"
|
|
|
|
"| VR-* | Out of CI | studies |\n")
|
|
|
|
"| VR-* | Out of CI | studies |\n")
|
|
|
|
register = et.parse_register(md)
|
|
|
|
register = et.parse_register(md, CPP_PY)
|
|
|
|
assert register.requirements["AR-001"].tiers == {"T3"}
|
|
|
|
assert register.requirements["AR-001"].tiers == {"T3"}
|
|
|
|
assert register.requirements["AR-003"].tiers == {"T2"}
|
|
|
|
assert register.requirements["AR-003"].tiers == {"T2"}
|
|
|
|
assert register.requirements["AR-006"].tiers == {"T1", "T4"}
|
|
|
|
assert register.requirements["AR-006"].tiers == {"T1", "T4"}
|
|
|
@@ -336,7 +557,7 @@ def test_tier_assignment_handles_lists_ranges_and_wildcards():
|
|
|
|
|
|
|
|
|
|
|
|
def test_a_range_cannot_invent_a_requirement_the_register_lacks():
|
|
|
|
def test_a_range_cannot_invent_a_requirement_the_register_lacks():
|
|
|
|
md = TIER_REGISTER + "\n" + _tier_table("| AR-001 … AR-050 | T2 | wide |\n")
|
|
|
|
md = TIER_REGISTER + "\n" + _tier_table("| AR-001 … AR-050 | T2 | wide |\n")
|
|
|
|
register = et.parse_register(md)
|
|
|
|
register = et.parse_register(md, CPP_PY)
|
|
|
|
assert register.total == 12
|
|
|
|
assert register.total == 12
|
|
|
|
assert "AR-050" not in register.ids
|
|
|
|
assert "AR-050" not in register.ids
|
|
|
|
|
|
|
|
|
|
|
@@ -345,7 +566,7 @@ def test_slash_shorthand_in_the_verification_plan_expands():
|
|
|
|
md = TIER_REGISTER + "\n" + (
|
|
|
|
md = TIER_REGISTER + "\n" + (
|
|
|
|
"| ID | Tier | Test asserts | Edge cases |\n|---|---|---|---|\n"
|
|
|
|
"| ID | Tier | Test asserts | Edge cases |\n|---|---|---|---|\n"
|
|
|
|
"| AR-009/008 | T2 | Cut shifts weighting | cut with same people |\n")
|
|
|
|
"| AR-009/008 | T2 | Cut shifts weighting | cut with same people |\n")
|
|
|
|
register = et.parse_register(md)
|
|
|
|
register = et.parse_register(md, CPP_PY)
|
|
|
|
assert register.requirements["AR-008"].tiers == {"T2"}
|
|
|
|
assert register.requirements["AR-008"].tiers == {"T2"}
|
|
|
|
assert register.requirements["AR-009"].tiers == {"T2"}
|
|
|
|
assert register.requirements["AR-009"].tiers == {"T2"}
|
|
|
|
|
|
|
|
|
|
|
@@ -357,35 +578,43 @@ def test_tiers_from_both_tables_are_unioned_not_overwritten():
|
|
|
|
md = TIER_REGISTER + "\n" + _tier_table("| AR-006 | T4 | GPU host only |\n") + "\n" + (
|
|
|
|
md = TIER_REGISTER + "\n" + _tier_table("| AR-006 | T4 | GPU host only |\n") + "\n" + (
|
|
|
|
"| ID | Tier | Test asserts | Edge cases |\n|---|---|---|---|\n"
|
|
|
|
"| ID | Tier | Test asserts | Edge cases |\n|---|---|---|---|\n"
|
|
|
|
"| AR-006 | T1 + T4 | GEMM equals reference loop | small input in CI |\n")
|
|
|
|
"| AR-006 | T1 + T4 | GEMM equals reference loop | small input in CI |\n")
|
|
|
|
register = et.parse_register(md)
|
|
|
|
register = et.parse_register(md, CPP_PY)
|
|
|
|
assert register.requirements["AR-006"].tiers == {"T1", "T4"}
|
|
|
|
assert register.requirements["AR-006"].tiers == {"T1", "T4"}
|
|
|
|
assert register.requirements["AR-006"].ci_executable
|
|
|
|
assert register.is_ci_executable("AR-006")
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def test_a_t4_only_requirement_is_not_ci_executable():
|
|
|
|
def test_a_t4_only_requirement_is_not_ci_executable():
|
|
|
|
md = TIER_REGISTER + "\n" + _tier_table("| AR-007 | **T4** | GPU only |\n")
|
|
|
|
md = TIER_REGISTER + "\n" + _tier_table("| AR-007 | **T4** | GPU only |\n")
|
|
|
|
register = et.parse_register(md)
|
|
|
|
register = et.parse_register(md, CPP_PY)
|
|
|
|
assert not register.requirements["AR-007"].ci_executable
|
|
|
|
assert not register.is_ci_executable("AR-007")
|
|
|
|
assert register.unexecutable_ids() == {"AR-007"}
|
|
|
|
assert register.unexecutable_ids() == {"AR-007"}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def test_a_requirement_tracing_up_to_nothing_is_reported():
|
|
|
|
def test_which_tiers_count_as_executable_is_configuration_not_code():
|
|
|
|
# SPEC.md section 6: a requirement citing no parent is scope creep, and it
|
|
|
|
# A repo whose CI host can run T4 says so, and the same register then
|
|
|
|
# is invisible unless something looks. A section reference counts as a
|
|
|
|
# yields full executability. Nothing about one host is baked in.
|
|
|
|
# parent - what matters is that something was recorded.
|
|
|
|
md = TIER_REGISTER + "\n" + _tier_table("| AR-007 | **T4** | GPU only |\n")
|
|
|
|
md = REGISTER_HEADER + (
|
|
|
|
permissive = et.parse_register(
|
|
|
|
"| AR-001 | Has a parent | SR-002 | High | Done |\n"
|
|
|
|
md, cfg(ci_executable_tiers=frozenset({"T1", "T2", "T3", "T4", "static"})))
|
|
|
|
"| AR-002 | Parent is a section | §4 | Medium | Planned |\n"
|
|
|
|
assert permissive.unexecutable_ids() == set()
|
|
|
|
"| AR-003 | Serves nothing stated | - | Low | Planned |\n"
|
|
|
|
assert permissive.is_ci_executable("AR-007")
|
|
|
|
"| AR-004 | Blank cell | | Low | Planned |\n")
|
|
|
|
|
|
|
|
register = et.parse_register(md)
|
|
|
|
|
|
|
|
assert register.parentless_ids() == {"AR-003", "AR-004"}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def test_a_requirement_with_no_tier_is_unknown_not_unexecutable():
|
|
|
|
def test_a_requirement_with_no_tier_is_unknown_not_unexecutable():
|
|
|
|
register = et.parse_register(TIER_REGISTER)
|
|
|
|
register = et.parse_register(TIER_REGISTER, CPP_PY)
|
|
|
|
assert register.tier_unknown_ids() == register.ids
|
|
|
|
assert register.tier_unknown_ids() == register.ids
|
|
|
|
assert register.unexecutable_ids() == set()
|
|
|
|
assert register.unexecutable_ids() == set()
|
|
|
|
|
|
|
|
assert not register.uses_tiers
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def test_a_register_that_uses_tiers_is_distinguishable_from_one_that_does_not():
|
|
|
|
|
|
|
|
# The distinction drives whether "no tier recorded" is worth warning about;
|
|
|
|
|
|
|
|
# warning on every requirement of a tierless register trains people to
|
|
|
|
|
|
|
|
# ignore the warning.
|
|
|
|
|
|
|
|
tiered = et.parse_register(
|
|
|
|
|
|
|
|
TIER_REGISTER + "\n" + _tier_table("| AR-001 | T1 | unit |\n"), CPP_PY)
|
|
|
|
|
|
|
|
assert tiered.uses_tiers
|
|
|
|
|
|
|
|
assert not et.parse_register(TIER_REGISTER, CPP_PY).uses_tiers
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
# --------------------------------------------------------------------------
|
|
|
|
# --------------------------------------------------------------------------
|
|
|
@@ -401,7 +630,8 @@ COVERAGE_REGISTER = et.parse_register(
|
|
|
|
+ "\n"
|
|
|
|
+ "\n"
|
|
|
|
+ _tier_table("| AR-001, AR-002 | T2 | replay |\n"
|
|
|
|
+ _tier_table("| AR-001, AR-002 | T2 | replay |\n"
|
|
|
|
"| GR-001 | T1 | bookkeeping |\n"
|
|
|
|
"| GR-001 | T1 | bookkeeping |\n"
|
|
|
|
"| AR-027 | **T4** | GPU host only |\n"))
|
|
|
|
"| AR-027 | **T4** | GPU host only |\n"),
|
|
|
|
|
|
|
|
CPP_PY)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def test_coverage_is_the_intersection_of_traced_and_defined():
|
|
|
|
def test_coverage_is_the_intersection_of_traced_and_defined():
|
|
|
@@ -425,14 +655,13 @@ def test_orphan_tags_are_reported_so_they_get_fixed():
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def test_no_orphans_when_every_traced_id_is_defined():
|
|
|
|
def test_no_orphans_when_every_traced_id_is_defined():
|
|
|
|
cov = et.compute_coverage(["AR-001", "AR-002"], COVERAGE_REGISTER)
|
|
|
|
assert et.compute_coverage(["AR-001", "AR-002"], COVERAGE_REGISTER).orphaned == []
|
|
|
|
assert cov.orphaned == []
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def test_test_and_system_ids_are_a_separate_taxonomy():
|
|
|
|
def test_test_and_system_ids_are_a_separate_taxonomy():
|
|
|
|
# UT/IT live in their own register section; PR/SR live in the umbrella
|
|
|
|
# UT/IT are evidence for requirements; PR/SR live in the system spec, which
|
|
|
|
# SPEC.md, which is not part of this repo's checkout. Neither counts toward
|
|
|
|
# is a different register. Neither counts toward coverage, and flagging
|
|
|
|
# coverage, and flagging them as orphans would bury real typos in noise.
|
|
|
|
# them as orphans would bury real typos in noise.
|
|
|
|
cov = et.compute_coverage(
|
|
|
|
cov = et.compute_coverage(
|
|
|
|
["AR-001", "UT-003", "IT-007", "SR-002", "PR-001"], COVERAGE_REGISTER)
|
|
|
|
["AR-001", "UT-003", "IT-007", "SR-002", "PR-001"], COVERAGE_REGISTER)
|
|
|
|
assert cov.orphaned == []
|
|
|
|
assert cov.orphaned == []
|
|
|
@@ -440,9 +669,8 @@ def test_test_and_system_ids_are_a_separate_taxonomy():
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def test_a_gpu_only_requirement_is_tagged_but_unexecuted_not_covered():
|
|
|
|
def test_a_gpu_only_requirement_is_tagged_but_unexecuted_not_covered():
|
|
|
|
# The rule specific to this repo: CI is an Intel N100 with no dGPU. A test
|
|
|
|
# A test that exists but can never run on the CI host is not evidence, and
|
|
|
|
# that exists but can never run is not evidence, and counting it is the
|
|
|
|
# counting it is the same failure mode as the 158% bug.
|
|
|
|
# same failure mode as the 158% bug.
|
|
|
|
|
|
|
|
cov = et.compute_coverage(["AR-001", "AR-027"], COVERAGE_REGISTER)
|
|
|
|
cov = et.compute_coverage(["AR-001", "AR-027"], COVERAGE_REGISTER)
|
|
|
|
assert cov.unexecuted == ["AR-027"]
|
|
|
|
assert cov.unexecuted == ["AR-027"]
|
|
|
|
assert cov.covered == ["AR-001"]
|
|
|
|
assert cov.covered == ["AR-001"]
|
|
|
@@ -475,7 +703,7 @@ def test_an_empty_trace_set_is_zero_percent_not_a_divide_by_zero():
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def test_an_empty_register_reports_zero_rather_than_nan():
|
|
|
|
def test_an_empty_register_reports_zero_rather_than_nan():
|
|
|
|
cov = et.compute_coverage(["AR-001"], et.Register())
|
|
|
|
cov = et.compute_coverage(["AR-001"], et.Register(types=("AR",)))
|
|
|
|
assert cov.percent == 0.0
|
|
|
|
assert cov.percent == 0.0
|
|
|
|
assert cov.ci_percent == 0.0
|
|
|
|
assert cov.ci_percent == 0.0
|
|
|
|
|
|
|
|
|
|
|
@@ -486,7 +714,6 @@ def test_full_coverage_reports_exactly_one_hundred_never_above():
|
|
|
|
# AR-027 is GPU-only, so the honest ceiling here is 3/4.
|
|
|
|
# AR-027 is GPU-only, so the honest ceiling here is 3/4.
|
|
|
|
assert cov.percent == 75.0
|
|
|
|
assert cov.percent == 75.0
|
|
|
|
assert cov.ci_percent == 100.0
|
|
|
|
assert cov.ci_percent == 100.0
|
|
|
|
assert cov.ci_percent <= 100.0
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
# --------------------------------------------------------------------------
|
|
|
|
# --------------------------------------------------------------------------
|
|
|
@@ -502,23 +729,52 @@ FIXTURE_REGISTER = (
|
|
|
|
+ _tier_table("| AR-001, AR-002 | T2 | replay |\n"
|
|
|
|
+ _tier_table("| AR-001, AR-002 | T2 | replay |\n"
|
|
|
|
"| AR-027 | **T4** | GPU host only |\n"))
|
|
|
|
"| AR-027 | **T4** | GPU host only |\n"))
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
FIXTURE_CONFIG_TOML = """\
|
|
|
|
|
|
|
|
requirement_types = ["AR"]
|
|
|
|
|
|
|
|
languages = ["cpp"]
|
|
|
|
|
|
|
|
source_roots = ["src"]
|
|
|
|
|
|
|
|
"""
|
|
|
|
|
|
|
|
|
|
|
|
def _fixture_repo(tmp: str, source: str) -> Path:
|
|
|
|
|
|
|
|
|
|
|
|
def _fixture_repo(tmp: str, source: str, config_toml: str = FIXTURE_CONFIG_TOML,
|
|
|
|
|
|
|
|
register: str = FIXTURE_REGISTER) -> Path:
|
|
|
|
root = Path(tmp)
|
|
|
|
root = Path(tmp)
|
|
|
|
(root / "docs").mkdir(parents=True, exist_ok=True)
|
|
|
|
(root / "docs").mkdir(parents=True, exist_ok=True)
|
|
|
|
(root / "src").mkdir(parents=True, exist_ok=True)
|
|
|
|
(root / "src").mkdir(parents=True, exist_ok=True)
|
|
|
|
(root / "docs" / "requirements.md").write_text(FIXTURE_REGISTER, encoding="utf-8")
|
|
|
|
(root / "docs" / "requirements.md").write_text(register, encoding="utf-8")
|
|
|
|
(root / "src" / "pipeline.cpp").write_text(source, encoding="utf-8")
|
|
|
|
(root / "src" / "pipeline.cpp").write_text(source, encoding="utf-8")
|
|
|
|
|
|
|
|
(root / et.CONFIG_FILENAME).write_text(config_toml, encoding="utf-8")
|
|
|
|
return root
|
|
|
|
return root
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def _run_gate(root: Path, *extra: str):
|
|
|
|
def _run_gate(root: Path, *extra: str):
|
|
|
|
buffer = io.StringIO()
|
|
|
|
buffer = io.StringIO()
|
|
|
|
with redirect_stdout(buffer):
|
|
|
|
with redirect_stdout(buffer):
|
|
|
|
code = et.main(["--root", str(root), "--format", "coverage", *extra])
|
|
|
|
code = et.main(["--config", str(root / et.CONFIG_FILENAME),
|
|
|
|
|
|
|
|
"--format", "coverage", *extra])
|
|
|
|
return code, buffer.getvalue()
|
|
|
|
return code, buffer.getvalue()
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def test_gate_reads_its_settings_from_the_repos_config_file():
|
|
|
|
|
|
|
|
with tempfile.TemporaryDirectory() as tmp:
|
|
|
|
|
|
|
|
root = _fixture_repo(tmp, f"// {TAG} AR-001\nint main() {{}}\n")
|
|
|
|
|
|
|
|
code, out = _run_gate(root)
|
|
|
|
|
|
|
|
assert code == 0, out
|
|
|
|
|
|
|
|
assert "Requirement types : AR" in out
|
|
|
|
|
|
|
|
assert str(root / et.CONFIG_FILENAME) in out
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def test_config_is_discovered_from_a_subdirectory():
|
|
|
|
|
|
|
|
# The config file's directory is the repo root, so running the gate from
|
|
|
|
|
|
|
|
# deep inside the tree does not silently scan a subtree.
|
|
|
|
|
|
|
|
with tempfile.TemporaryDirectory() as tmp:
|
|
|
|
|
|
|
|
root = _fixture_repo(tmp, f"// {TAG} AR-001\nint main() {{}}\n")
|
|
|
|
|
|
|
|
nested = root / "src" / "nodes"
|
|
|
|
|
|
|
|
nested.mkdir(parents=True, exist_ok=True)
|
|
|
|
|
|
|
|
found = et.find_config(nested)
|
|
|
|
|
|
|
|
assert found == root / et.CONFIG_FILENAME
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def test_gate_passes_at_the_default_threshold_with_no_tags_yet():
|
|
|
|
def test_gate_passes_at_the_default_threshold_with_no_tags_yet():
|
|
|
|
# Near-zero coverage on a fresh tree is the correct reading, not a failure.
|
|
|
|
# Near-zero coverage on a fresh tree is the correct reading, not a failure.
|
|
|
|
with tempfile.TemporaryDirectory() as tmp:
|
|
|
|
with tempfile.TemporaryDirectory() as tmp:
|
|
|
@@ -559,20 +815,48 @@ def test_gate_fails_when_the_register_parses_to_nothing():
|
|
|
|
# With a low threshold this self-check is what keeps the gate meaningful:
|
|
|
|
# With a low threshold this self-check is what keeps the gate meaningful:
|
|
|
|
# a broken parser would otherwise report 0/0 and pass.
|
|
|
|
# a broken parser would otherwise report 0/0 and pass.
|
|
|
|
with tempfile.TemporaryDirectory() as tmp:
|
|
|
|
with tempfile.TemporaryDirectory() as tmp:
|
|
|
|
root = _fixture_repo(tmp, "int main() {}\n")
|
|
|
|
root = _fixture_repo(tmp, "int main() {}\n",
|
|
|
|
(root / "docs" / "requirements.md").write_text(
|
|
|
|
register="# register\n\nNo tables here.\n")
|
|
|
|
"# register\n\nNo tables here.\n", encoding="utf-8")
|
|
|
|
|
|
|
|
code, out = _run_gate(root)
|
|
|
|
code, out = _run_gate(root)
|
|
|
|
assert code == 1
|
|
|
|
assert code == 1
|
|
|
|
assert "ZERO requirements" in out
|
|
|
|
assert "ZERO requirements" in out
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def test_gate_fails_when_no_source_files_were_scanned():
|
|
|
|
def test_gate_fails_when_the_prefixes_are_configured_wrong():
|
|
|
|
|
|
|
|
# The misconfiguration a shared tool invites: the server's config pointed
|
|
|
|
|
|
|
|
# at the extraction register, or a typo'd prefix. Every row is skipped, and
|
|
|
|
|
|
|
|
# a plausible 0% would hide it completely.
|
|
|
|
with tempfile.TemporaryDirectory() as tmp:
|
|
|
|
with tempfile.TemporaryDirectory() as tmp:
|
|
|
|
root = Path(tmp)
|
|
|
|
root = _fixture_repo(
|
|
|
|
(root / "docs").mkdir(parents=True)
|
|
|
|
tmp, f"// {TAG} AR-001\nint main() {{}}\n",
|
|
|
|
(root / "docs" / "requirements.md").write_text(FIXTURE_REGISTER,
|
|
|
|
config_toml='requirement_types = ["JR"]\nlanguages = ["cpp"]\n'
|
|
|
|
encoding="utf-8")
|
|
|
|
'source_roots = ["src"]\n')
|
|
|
|
|
|
|
|
code, out = _run_gate(root)
|
|
|
|
|
|
|
|
assert code == 1
|
|
|
|
|
|
|
|
assert "ZERO requirements" in out
|
|
|
|
|
|
|
|
assert "requirement_types (JR)" in out
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def test_gate_fails_when_the_suffixes_are_configured_wrong():
|
|
|
|
|
|
|
|
# The Rust config applied to a C++ repo: nothing to scan, and coverage
|
|
|
|
|
|
|
|
# against an empty tree means nothing.
|
|
|
|
|
|
|
|
with tempfile.TemporaryDirectory() as tmp:
|
|
|
|
|
|
|
|
root = _fixture_repo(
|
|
|
|
|
|
|
|
tmp, f"// {TAG} AR-001\nint main() {{}}\n",
|
|
|
|
|
|
|
|
config_toml='requirement_types = ["AR"]\nlanguages = ["rust"]\n'
|
|
|
|
|
|
|
|
'source_roots = ["src"]\n')
|
|
|
|
|
|
|
|
code, out = _run_gate(root)
|
|
|
|
|
|
|
|
assert code == 1
|
|
|
|
|
|
|
|
assert "no source files were scanned" in out
|
|
|
|
|
|
|
|
assert ".rs" in out
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def test_gate_fails_when_the_source_roots_do_not_exist():
|
|
|
|
|
|
|
|
with tempfile.TemporaryDirectory() as tmp:
|
|
|
|
|
|
|
|
root = _fixture_repo(
|
|
|
|
|
|
|
|
tmp, f"// {TAG} AR-001\nint main() {{}}\n",
|
|
|
|
|
|
|
|
config_toml='requirement_types = ["AR"]\nlanguages = ["cpp"]\n'
|
|
|
|
|
|
|
|
'source_roots = ["nowhere"]\n')
|
|
|
|
code, out = _run_gate(root)
|
|
|
|
code, out = _run_gate(root)
|
|
|
|
assert code == 1
|
|
|
|
assert code == 1
|
|
|
|
assert "no source files were scanned" in out
|
|
|
|
assert "no source files were scanned" in out
|
|
|
@@ -594,11 +878,14 @@ def test_gate_hard_fails_on_an_impossible_ratio():
|
|
|
|
# reported as a pass. Forced here by handing the reporter a poisoned value.
|
|
|
|
# reported as a pass. Forced here by handing the reporter a poisoned value.
|
|
|
|
with tempfile.TemporaryDirectory() as tmp:
|
|
|
|
with tempfile.TemporaryDirectory() as tmp:
|
|
|
|
root = _fixture_repo(tmp, f"// {TAG} AR-001\nint main() {{}}\n")
|
|
|
|
root = _fixture_repo(tmp, f"// {TAG} AR-001\nint main() {{}}\n")
|
|
|
|
register = et.read_register(root / "docs" / "requirements.md")
|
|
|
|
config = et.config_from_dict(
|
|
|
|
scan = et.scan_files(et.iter_source_files(root), root)
|
|
|
|
{"requirement_types": ["AR"], "languages": ["cpp"],
|
|
|
|
report = et.build_report(root, register, scan)
|
|
|
|
"source_roots": ["src"]}, root, "fixture")
|
|
|
|
|
|
|
|
register = et.read_register(config)
|
|
|
|
|
|
|
|
scan = et.scan_files(et.iter_source_files(config), config)
|
|
|
|
|
|
|
|
report = et.build_report(config, register, scan)
|
|
|
|
report.coverage.percent = 158.0
|
|
|
|
report.coverage.percent = 158.0
|
|
|
|
text, code = et.format_coverage_report(report, 50.0)
|
|
|
|
text, code = et.format_coverage_report(report)
|
|
|
|
assert code == 1
|
|
|
|
assert code == 1
|
|
|
|
assert "exceeds 100%" in text
|
|
|
|
assert "exceeds 100%" in text
|
|
|
|
|
|
|
|
|
|
|
@@ -607,11 +894,13 @@ def test_the_per_type_breakdown_sums_to_the_headline_figure():
|
|
|
|
# A breakdown that does not add up to its own total is how a wrong number
|
|
|
|
# A breakdown that does not add up to its own total is how a wrong number
|
|
|
|
# survives review: every row looks plausible on its own.
|
|
|
|
# survives review: every row looks plausible on its own.
|
|
|
|
with tempfile.TemporaryDirectory() as tmp:
|
|
|
|
with tempfile.TemporaryDirectory() as tmp:
|
|
|
|
root = _fixture_repo(
|
|
|
|
root = _fixture_repo(tmp, f"// {TAG} AR-001, AR-027\nint main() {{}}\n")
|
|
|
|
tmp, f"// {TAG} AR-001, AR-027\nint main() {{}}\n")
|
|
|
|
config = et.config_from_dict(
|
|
|
|
register = et.read_register(root / "docs" / "requirements.md")
|
|
|
|
{"requirement_types": ["AR"], "languages": ["cpp"],
|
|
|
|
scan = et.scan_files(et.iter_source_files(root), root)
|
|
|
|
"source_roots": ["src"]}, root, "fixture")
|
|
|
|
report = et.build_report(root, register, scan)
|
|
|
|
register = et.read_register(config)
|
|
|
|
|
|
|
|
scan = et.scan_files(et.iter_source_files(config), config)
|
|
|
|
|
|
|
|
report = et.build_report(config, register, scan)
|
|
|
|
stats = et.per_type_stats(report)
|
|
|
|
stats = et.per_type_stats(report)
|
|
|
|
assert sum(c for c, _, _ in stats.values()) == len(report.coverage.covered)
|
|
|
|
assert sum(c for c, _, _ in stats.values()) == len(report.coverage.covered)
|
|
|
|
assert sum(u for _, u, _ in stats.values()) == len(report.coverage.unexecuted)
|
|
|
|
assert sum(u for _, u, _ in stats.values()) == len(report.coverage.unexecuted)
|
|
|
@@ -620,64 +909,116 @@ def test_the_per_type_breakdown_sums_to_the_headline_figure():
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def test_json_and_markdown_outputs_are_written_and_consistent():
|
|
|
|
def test_json_and_markdown_outputs_are_written_and_consistent():
|
|
|
|
import json
|
|
|
|
|
|
|
|
with tempfile.TemporaryDirectory() as tmp:
|
|
|
|
with tempfile.TemporaryDirectory() as tmp:
|
|
|
|
root = _fixture_repo(tmp, f"// {TAG} AR-001 | SR-002\nint main() {{}}\n")
|
|
|
|
root = _fixture_repo(tmp, f"// {TAG} AR-001 | SR-002\nint main() {{}}\n")
|
|
|
|
json_out = root / "traces-report.json"
|
|
|
|
code, _ = _run_gate(root)
|
|
|
|
md_out = root / "docs" / "traceability.md"
|
|
|
|
|
|
|
|
buffer = io.StringIO()
|
|
|
|
|
|
|
|
with redirect_stdout(buffer):
|
|
|
|
|
|
|
|
code = et.main(["--root", str(root), "--format", "coverage",
|
|
|
|
|
|
|
|
"--json-out", str(json_out),
|
|
|
|
|
|
|
|
"--markdown-out", str(md_out)])
|
|
|
|
|
|
|
|
assert code == 0
|
|
|
|
assert code == 0
|
|
|
|
data = json.loads(json_out.read_text(encoding="utf-8"))
|
|
|
|
data = json.loads((root / "traces-report.json").read_text(encoding="utf-8"))
|
|
|
|
assert data["defined"]["total"] == 3
|
|
|
|
assert data["defined"]["total"] == 3
|
|
|
|
assert data["coverage"]["covered"] == 1
|
|
|
|
assert data["coverage"]["covered"] == 1
|
|
|
|
assert data["coverage"]["percent"] == round(100 / 3, 1)
|
|
|
|
assert data["coverage"]["percent"] == round(100 / 3, 1)
|
|
|
|
assert data["byType"]["SR"] == ["SR-002"]
|
|
|
|
assert data["byType"]["SR"] == ["SR-002"]
|
|
|
|
assert data["gpuOnlyRequirements"] == ["AR-027"]
|
|
|
|
assert data["unexecutableRequirements"] == ["AR-027"]
|
|
|
|
assert "AR-001" in md_out.read_text(encoding="utf-8")
|
|
|
|
# The report says how it was produced; a shared tool's output is
|
|
|
|
|
|
|
|
# otherwise ambiguous about which repo and settings it describes.
|
|
|
|
|
|
|
|
assert data["config"]["requirementTypes"] == ["AR"]
|
|
|
|
|
|
|
|
assert data["config"]["sourceRoots"] == ["src"]
|
|
|
|
|
|
|
|
assert "AR-001" in (root / "docs" / "traceability.md").read_text(
|
|
|
|
|
|
|
|
encoding="utf-8")
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def test_cli_flags_alone_work_with_no_config_file_at_all():
|
|
|
|
|
|
|
|
# The path a repo takes before it has a traceability.toml, and the escape
|
|
|
|
|
|
|
|
# hatch on a Python too old for tomllib.
|
|
|
|
|
|
|
|
with tempfile.TemporaryDirectory() as tmp:
|
|
|
|
|
|
|
|
root = _fixture_repo(tmp, f"// {TAG} AR-001\nint main() {{}}\n")
|
|
|
|
|
|
|
|
(root / et.CONFIG_FILENAME).unlink()
|
|
|
|
|
|
|
|
buffer = io.StringIO()
|
|
|
|
|
|
|
|
with redirect_stdout(buffer):
|
|
|
|
|
|
|
|
code = et.main(["--no-config", "--root", str(root),
|
|
|
|
|
|
|
|
"--requirement-type", "AR",
|
|
|
|
|
|
|
|
"--language", "cpp", "--source-root", "src",
|
|
|
|
|
|
|
|
"--format", "coverage", "--no-write"])
|
|
|
|
|
|
|
|
assert code == 0, buffer.getvalue()
|
|
|
|
|
|
|
|
assert "1 / 3" in buffer.getvalue()
|
|
|
|
|
|
|
|
assert not (root / "traces-report.json").exists()
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def test_cli_flags_override_the_config_file():
|
|
|
|
|
|
|
|
with tempfile.TemporaryDirectory() as tmp:
|
|
|
|
|
|
|
|
root = _fixture_repo(tmp, f"// {TAG} AR-001\nint main() {{}}\n")
|
|
|
|
|
|
|
|
code, out = _run_gate(root, "--requirement-type", "JR", "--no-write")
|
|
|
|
|
|
|
|
assert code == 1
|
|
|
|
|
|
|
|
assert "requirement_types (JR)" in out
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def test_language_flags_extend_rather_than_replace_the_configured_set():
|
|
|
|
|
|
|
|
with tempfile.TemporaryDirectory() as tmp:
|
|
|
|
|
|
|
|
root = _fixture_repo(tmp, f"// {TAG} AR-001\nint main() {{}}\n")
|
|
|
|
|
|
|
|
(root / "src" / "extra.rs").write_text(
|
|
|
|
|
|
|
|
f"/// {TAG} AR-002\npub fn go() {{}}\n", encoding="utf-8")
|
|
|
|
|
|
|
|
code, out = _run_gate(root, "--language", "rust", "--no-write")
|
|
|
|
|
|
|
|
assert code == 0, out
|
|
|
|
|
|
|
|
assert "2 / 3" in out
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def test_an_invalid_configuration_is_a_distinct_exit_code():
|
|
|
|
|
|
|
|
# 2, not 1: a broken config is not a coverage failure, and conflating them
|
|
|
|
|
|
|
|
# makes CI logs lie about why the job went red.
|
|
|
|
|
|
|
|
with tempfile.TemporaryDirectory() as tmp:
|
|
|
|
|
|
|
|
root = _fixture_repo(tmp, "int main() {}\n",
|
|
|
|
|
|
|
|
config_toml='languages = ["cpp"]\nsource_roots = ["src"]\n')
|
|
|
|
|
|
|
|
buffer = io.StringIO()
|
|
|
|
|
|
|
|
with redirect_stdout(buffer), redirect_stderr(buffer):
|
|
|
|
|
|
|
|
code = et.main(["--config", str(root / et.CONFIG_FILENAME)])
|
|
|
|
|
|
|
|
assert code == 2
|
|
|
|
|
|
|
|
assert "requirement_types is empty" in buffer.getvalue()
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def test_system_spec_parsing_enables_orphan_checks_for_pr_and_sr():
|
|
|
|
def test_system_spec_parsing_enables_orphan_checks_for_pr_and_sr():
|
|
|
|
spec = ("### SR-002 - Presence is scene-scoped\n\n"
|
|
|
|
spec = ("### SR-002 - Presence is scene-scoped\n\n"
|
|
|
|
"| ID | Goal | Why |\n|---|---|---|\n"
|
|
|
|
"| ID | Goal | Why |\n|---|---|---|\n"
|
|
|
|
"| **PR-001** | Show which actors are on screen | The product |\n")
|
|
|
|
"| **PR-001** | Show which actors are on screen | The product |\n")
|
|
|
|
ids = et.parse_system_spec(spec)
|
|
|
|
assert et.parse_system_spec(spec) == {"SR-002", "PR-001"}
|
|
|
|
assert ids == {"SR-002", "PR-001"}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
with tempfile.TemporaryDirectory() as tmp:
|
|
|
|
with tempfile.TemporaryDirectory() as tmp:
|
|
|
|
root = _fixture_repo(tmp, f"// {TAG} AR-001 | SR-999\nint main() {{}}\n")
|
|
|
|
root = _fixture_repo(tmp, f"// {TAG} AR-001 | SR-999\nint main() {{}}\n")
|
|
|
|
spec_path = root / "system-spec.md"
|
|
|
|
spec_path = root / "system-spec.md"
|
|
|
|
spec_path.write_text(spec, encoding="utf-8")
|
|
|
|
spec_path.write_text(spec, encoding="utf-8")
|
|
|
|
code, out = _run_gate(root, "--system-spec", str(spec_path))
|
|
|
|
code, out = _run_gate(root, "--system-spec", str(spec_path), "--no-write")
|
|
|
|
assert "SR-999" in out
|
|
|
|
assert "SR-999" in out
|
|
|
|
assert code == 0 # advisory: the system register is not this repo's
|
|
|
|
assert code == 0 # advisory: the system register is not this repo's
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
# --------------------------------------------------------------------------
|
|
|
|
# --------------------------------------------------------------------------
|
|
|
|
# The live register — structural assertions only, so this does not churn as
|
|
|
|
# The live repo — structural assertions only, so this does not churn as
|
|
|
|
# requirements are added.
|
|
|
|
# requirements are added, and so it still means something once this file is
|
|
|
|
|
|
|
|
# vendored into another component.
|
|
|
|
# --------------------------------------------------------------------------
|
|
|
|
# --------------------------------------------------------------------------
|
|
|
|
|
|
|
|
|
|
|
|
def test_the_live_register_parses_and_assigns_tiers():
|
|
|
|
def _live_config() -> et.Config:
|
|
|
|
register = et.read_register(et.REPO_ROOT / "docs" / "requirements.md")
|
|
|
|
config_path = et.find_config(HERE)
|
|
|
|
|
|
|
|
assert config_path is not None, "no traceability.toml found above this file"
|
|
|
|
|
|
|
|
return et.config_from_dict(
|
|
|
|
|
|
|
|
__import__("tomllib").loads(config_path.read_text(encoding="utf-8")),
|
|
|
|
|
|
|
|
config_path.parent, str(config_path))
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def test_this_repos_config_is_valid_and_its_register_parses():
|
|
|
|
|
|
|
|
config = _live_config()
|
|
|
|
|
|
|
|
config.validate()
|
|
|
|
|
|
|
|
register = et.read_register(config)
|
|
|
|
assert register.total > 0
|
|
|
|
assert register.total > 0
|
|
|
|
for req_type in et.LOCAL_TYPES:
|
|
|
|
for req_type in config.requirement_types:
|
|
|
|
assert register.count(req_type) > 0, req_type
|
|
|
|
assert register.count(req_type) > 0, req_type
|
|
|
|
assert sum(register.count(t) for t in et.LOCAL_TYPES) == register.total
|
|
|
|
assert sum(register.count(t) for t in config.requirement_types) == register.total
|
|
|
|
# The GPU-less CI host must be visible in the parse, not just in prose.
|
|
|
|
|
|
|
|
assert "AR-027" in register.unexecutable_ids()
|
|
|
|
|
|
|
|
assert register.requirements["AR-027"].tiers == {"T4"}
|
|
|
|
|
|
|
|
assert register.requirements["AR-012"].tiers == {"T2"}
|
|
|
|
|
|
|
|
assert register.unexecutable_ids() < register.ids
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def test_the_live_register_yields_a_gate_run_that_cannot_exceed_one_hundred():
|
|
|
|
def test_this_repos_gate_run_cannot_exceed_one_hundred_percent():
|
|
|
|
register = et.read_register(et.REPO_ROOT / "docs" / "requirements.md")
|
|
|
|
config = _live_config()
|
|
|
|
files = et.iter_source_files(et.REPO_ROOT)
|
|
|
|
register = et.read_register(config)
|
|
|
|
scan = et.scan_files(files, et.REPO_ROOT)
|
|
|
|
scan = et.scan_files(et.iter_source_files(config), config)
|
|
|
|
|
|
|
|
assert scan.files, "configured source roots matched no files"
|
|
|
|
cov = et.compute_coverage(
|
|
|
|
cov = et.compute_coverage(
|
|
|
|
[i for t in scan.traces for i in t.requirements], register)
|
|
|
|
[i for t in scan.traces for i in t.requirements], register)
|
|
|
|
assert 0.0 <= cov.percent <= 100.0
|
|
|
|
assert 0.0 <= cov.percent <= 100.0
|
|
|
|