dtourolle
faaa71fa09
The tool is moving into the jray-project submodule to be shared by
scene-actor-extraction (C++/Python), jRay (C#) and JRay-public-server
(Rust). Two constants blocked that: LOCAL_TYPES and SOURCE_SUFFIXES were
hardcoded to this repo, so either sibling parsed zero requirements and
scanned zero files. Both, plus the register path, scan roots, system-spec
path, exclude list and CI-executable tier set, are now configuration.
One implementation, parameterised. A second copy for "the other language"
is how two implementations start drifting apart, so there is exactly one -
the same code path now produces:
scene-actor-extraction AR/DP/IR/GR/VR 59 defined 0 tagged 0.0%
jRay JR 46 defined 24 covered 52.2%
JRay-public-server UR/DR 32 defined 23 covered 71.9%
Configuration is traceability.toml at the component repo root, CLI flags,
or both (flags win). Its directory defines the repo root, so the gate works
from any subdirectory. `--print-example-config` emits the annotated schema.
The JSON report echoes the settings it ran with, since a shared tool's
output is otherwise ambiguous about which repo it describes.
The refusal behaviour is kept and sharpened, because parameterising is
exactly what makes it easy to point a repo at the wrong prefixes or the
wrong suffixes. Zero requirements parsed or zero files scanned is still a
hard failure, and the message now names the setting that is wrong rather
than printing a plausible 0%. Config errors exit 2, not 1: a broken config
is not a coverage failure, and conflating them makes CI logs lie about why
the job went red.
Also fixed while adapting to the sibling registers, which are read but not
modified here:
* escaped `\|` inside a markdown cell no longer shifts every later column
(the server's register contains `small-\|M\|`);
* a tag above an attribute-decorated declaration attributes to the
declaration, not to `[HttpGet(...)]` or `#[derive(...)]` - the gap
jRay's register calls out;
* Rust and C# declaration patterns for context extraction;
* the missing-tier warning is suppressed for a register that assigns no
tiers at all, rather than listing every requirement in it.
The workflow is now component-agnostic too: the changed-file check reads
its extension list out of the report the gate just wrote, so the definition
of "source file" lives in one place.
79 tests, still fixture-based, now including the cross-repo cases: the same
parser over JR and UR/DR registers, the same scanner over Rust and C#, and
both misconfigurations failing loudly.
1048 lines
43 KiB
Python
Executable File
1048 lines
43 KiB
Python
Executable File
#!/usr/bin/env python3
|
|
"""Tests for the shared traceability extractor and coverage gate.
|
|
|
|
Run standalone (no third-party dependencies)::
|
|
|
|
python3 scripts/traceability/test_extract_traces.py
|
|
|
|
or under pytest, which discovers the same functions::
|
|
|
|
pytest scripts/traceability/test_extract_traces.py
|
|
|
|
Almost every test runs over fixture strings rather than any live
|
|
``requirements.md``, so their meaning does not drift as requirements are added,
|
|
and so they say the same thing in whichever component repo this tool is
|
|
vendored into.
|
|
|
|
Four properties these exist to pin:
|
|
|
|
* the denominator is computed from the register at run time, so adding a
|
|
requirement lowers coverage until it is traced (JellyTau's gate lost this -
|
|
see JellyTau/docs/specs/traceability-gate-repair.md);
|
|
* the numerator is an intersection, so a tag naming an undefined ID cannot push
|
|
the ratio above 100%;
|
|
* a requirement whose only verification tier is outside what the CI host can
|
|
execute is reported as tagged-but-unexecuted, never counted as covered;
|
|
* nothing about one repo is baked in - the same code parses the C++/Python,
|
|
C# and Rust components - and a misconfigured run refuses to report rather
|
|
than printing a plausible 0%.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import io
|
|
import json
|
|
import sys
|
|
import tempfile
|
|
from contextlib import redirect_stderr, redirect_stdout
|
|
from pathlib import Path
|
|
|
|
HERE = Path(__file__).resolve().parent
|
|
sys.path.insert(0, str(HERE))
|
|
|
|
import extract_traces as et # noqa: E402
|
|
|
|
# The literal tag keywords are assembled at run time so that this file's
|
|
# fixtures do not register as real traces when the extractor scans scripts/.
|
|
TAG = "TRA" + "CES:"
|
|
EXC = "EXCEP" + "TION:"
|
|
|
|
|
|
def cfg(**overrides) -> et.Config:
|
|
"""A minimal valid config; overrides name whatever a test cares about."""
|
|
base = dict(requirement_types=("AR", "DP", "IR", "GR", "VR"),
|
|
source_suffixes=et.LANGUAGE_SUFFIXES["cpp"]
|
|
| et.LANGUAGE_SUFFIXES["python"],
|
|
source_roots=("src", "scripts"))
|
|
base.update(overrides)
|
|
return et.Config(**base)
|
|
|
|
|
|
CPP_PY = cfg()
|
|
|
|
|
|
# --------------------------------------------------------------------------
|
|
# Configuration: the thing that makes one implementation serve three repos
|
|
# --------------------------------------------------------------------------
|
|
|
|
def test_config_rejects_an_empty_requirement_type_list():
|
|
# The prefix cannot be guessed, and guessing wrong means parsing zero
|
|
# requirements - the failure this whole class of check exists to prevent.
|
|
try:
|
|
cfg(requirement_types=()).validate()
|
|
except et.ConfigError as exc:
|
|
assert "requirement_types is empty" in str(exc)
|
|
else:
|
|
raise AssertionError("expected ConfigError")
|
|
|
|
|
|
def test_config_rejects_a_prefix_that_is_not_two_uppercase_letters():
|
|
try:
|
|
cfg(requirement_types=("Jr",)).validate()
|
|
except et.ConfigError as exc:
|
|
assert "two-letter uppercase" in str(exc)
|
|
else:
|
|
raise AssertionError("expected ConfigError")
|
|
|
|
|
|
def test_config_rejects_a_prefix_that_is_both_counted_and_excluded():
|
|
# A prefix cannot be in the fraction and out of it at once; silently
|
|
# picking one would make the reported number unexplainable.
|
|
try:
|
|
cfg(requirement_types=("UR", "UT")).validate()
|
|
except et.ConfigError as exc:
|
|
assert "cannot also be excluded" in str(exc)
|
|
else:
|
|
raise AssertionError("expected ConfigError")
|
|
|
|
|
|
def test_config_rejects_empty_suffixes_and_roots():
|
|
for kwargs, expected in ((dict(source_suffixes=frozenset()), "no source suffixes"),
|
|
(dict(source_roots=()), "no source roots")):
|
|
try:
|
|
cfg(**kwargs).validate()
|
|
except et.ConfigError as exc:
|
|
assert expected in str(exc)
|
|
else:
|
|
raise AssertionError(f"expected ConfigError for {kwargs}")
|
|
|
|
|
|
def test_config_rejects_a_suffix_missing_its_dot():
|
|
try:
|
|
cfg(source_suffixes=frozenset({"rs"})).validate()
|
|
except et.ConfigError as exc:
|
|
assert "must start with a dot" in str(exc)
|
|
else:
|
|
raise AssertionError("expected ConfigError")
|
|
|
|
|
|
def test_config_from_dict_expands_language_groups():
|
|
config = et.config_from_dict(
|
|
{"requirement_types": ["UR", "DR"], "languages": ["rust"],
|
|
"source_roots": ["src", "tests"]}, Path("/repo"), "fixture")
|
|
config.validate()
|
|
assert config.source_suffixes == frozenset({".rs"})
|
|
assert config.requirement_types == ("UR", "DR")
|
|
|
|
|
|
def test_config_from_dict_rejects_an_unknown_key():
|
|
# A typo'd key would leave a required field empty, and the user would be
|
|
# debugging "zero requirements" instead of a misspelling.
|
|
try:
|
|
et.config_from_dict({"requirement_type": ["JR"]}, Path("/repo"), "fixture")
|
|
except et.ConfigError as exc:
|
|
assert "unknown key" in str(exc)
|
|
else:
|
|
raise AssertionError("expected ConfigError")
|
|
|
|
|
|
def test_config_from_dict_rejects_an_unknown_language():
|
|
try:
|
|
et.config_from_dict({"languages": ["cobol"]}, Path("/repo"), "fixture")
|
|
except et.ConfigError as exc:
|
|
assert "unknown language" in str(exc)
|
|
else:
|
|
raise AssertionError("expected ConfigError")
|
|
|
|
|
|
def test_example_config_is_itself_valid_and_complete():
|
|
# The documentation people copy must parse, or it teaches the wrong schema.
|
|
tomllib = __import__("tomllib")
|
|
data = tomllib.loads(et.example_config())
|
|
config = et.config_from_dict(data, Path("/repo"), "example")
|
|
config.validate()
|
|
assert config.requirement_types == ("AR", "DP", "IR", "GR", "VR")
|
|
|
|
|
|
def test_config_paths_resolve_against_the_repo_root_not_the_cwd():
|
|
config = cfg(root=Path("/repo"), requirements_path="docs/requirements.md")
|
|
assert config.resolve("docs/requirements.md") == Path("/repo/docs/requirements.md")
|
|
assert config.resolve("/abs/spec.md") == Path("/abs/spec.md")
|
|
assert config.resolve(None) is None
|
|
|
|
|
|
# --------------------------------------------------------------------------
|
|
# Tag parsing
|
|
# --------------------------------------------------------------------------
|
|
|
|
def test_parses_a_single_requirement():
|
|
groups, junk = et.parse_traces_tag(" AR-012")
|
|
assert groups == [["AR-012"]]
|
|
assert junk == []
|
|
|
|
|
|
def test_parses_multiple_types_separated_by_pipe():
|
|
# The house format: a pipe separates requirement *types*, a comma separates
|
|
# IDs within a type. The grouping is preserved, not flattened away, so a
|
|
# malformed grouping stays detectable.
|
|
groups, junk = et.parse_traces_tag(" AR-012, AR-013 | SR-002")
|
|
assert groups == [["AR-012", "AR-013"], ["SR-002"]]
|
|
assert junk == []
|
|
|
|
|
|
def test_parses_three_groups_including_test_ids():
|
|
groups, _ = et.parse_traces_tag(" JR-012 | SR-002 | UT-003, UT-004")
|
|
assert groups == [["JR-012"], ["SR-002"], ["UT-003", "UT-004"]]
|
|
|
|
|
|
def test_strips_a_trailing_block_comment_terminator():
|
|
groups, junk = et.parse_traces_tag(" AR-001 */")
|
|
assert groups == [["AR-001"]]
|
|
assert junk == []
|
|
|
|
|
|
def test_does_not_harvest_ids_out_of_prose_after_the_tag():
|
|
# A trailing sentence must not smuggle IDs into the trace set: AR-999 here
|
|
# is discussion, not a claim that this code satisfies AR-999.
|
|
groups, junk = et.parse_traces_tag(" AR-001 - see also AR-999 in the notes")
|
|
assert groups == [["AR-001"]]
|
|
assert junk and "AR-999" in junk[0]
|
|
|
|
|
|
def test_ignores_a_tag_with_no_ids_at_all():
|
|
groups, _ = et.parse_traces_tag(" see the register")
|
|
assert groups == []
|
|
|
|
|
|
def test_two_digit_id_is_not_accepted_as_a_requirement():
|
|
# AR-12 is a typo for AR-012; silently accepting it would create a
|
|
# phantom requirement.
|
|
groups, junk = et.parse_traces_tag(" AR-12")
|
|
assert groups == []
|
|
assert junk == ["AR-12"]
|
|
|
|
|
|
# --------------------------------------------------------------------------
|
|
# Scanning: the same scanner over every component's language
|
|
# --------------------------------------------------------------------------
|
|
|
|
def _write_tree(root: Path) -> None:
|
|
(root / "src").mkdir(parents=True, exist_ok=True)
|
|
(root / "scripts").mkdir(parents=True, exist_ok=True)
|
|
(root / "external").mkdir(parents=True, exist_ok=True)
|
|
(root / "src" / "tracker.hpp").write_text(
|
|
"#pragma once\n"
|
|
f"/// {TAG} AR-012, AR-013 | SR-002\n"
|
|
"struct TrackRegistry {\n"
|
|
" void close_all();\n"
|
|
"};\n",
|
|
encoding="utf-8")
|
|
(root / "scripts" / "gallery.py").write_text(
|
|
"def build_gallery(cast):\n"
|
|
' """Build a gallery from Jellyfin plus TMDB fallback.\n'
|
|
"\n"
|
|
f" {TAG} GR-001 | SR-005\n"
|
|
' """\n'
|
|
" return {}\n",
|
|
encoding="utf-8")
|
|
(root / "external" / "vendored.cpp").write_text(
|
|
f"// {TAG} AR-001\n", encoding="utf-8")
|
|
(root / "src" / "notes.txt").write_text(
|
|
f"// {TAG} AR-002\n", encoding="utf-8")
|
|
|
|
|
|
def test_scans_cpp_and_python_but_not_vendored_or_non_source():
|
|
with tempfile.TemporaryDirectory() as tmp:
|
|
root = Path(tmp)
|
|
_write_tree(root)
|
|
config = cfg(root=root)
|
|
files = et.iter_source_files(config)
|
|
assert sorted(f.name for f in files) == ["gallery.py", "tracker.hpp"]
|
|
|
|
scan = et.scan_files(files, config)
|
|
traced = sorted({i for t in scan.traces for i in t.requirements})
|
|
assert traced == ["AR-012", "AR-013", "GR-001", "SR-002", "SR-005"]
|
|
|
|
|
|
def test_the_same_scanner_reads_rust_when_configured_for_it():
|
|
# JRay-public-server: Rust sources, UR/DR prefixes. No second extractor.
|
|
with tempfile.TemporaryDirectory() as tmp:
|
|
root = Path(tmp)
|
|
(root / "src").mkdir(parents=True)
|
|
(root / "src" / "manifest.rs").write_text(
|
|
f"/// {TAG} UR-002 | SR-004\n"
|
|
"pub fn accept_upload(body: Body) -> Response {\n"
|
|
" todo!()\n"
|
|
"}\n", encoding="utf-8")
|
|
config = cfg(root=root, requirement_types=("UR", "DR"),
|
|
source_suffixes=et.LANGUAGE_SUFFIXES["rust"],
|
|
source_roots=("src",))
|
|
scan = et.scan_files(et.iter_source_files(config), config)
|
|
assert len(scan.traces) == 1
|
|
assert scan.traces[0].requirements == ["UR-002", "SR-004"]
|
|
assert "accept_upload" in scan.traces[0].context
|
|
|
|
|
|
def test_the_same_scanner_reads_csharp_when_configured_for_it():
|
|
# jRay: C# plugin, JR prefix, and a tag above an attribute-decorated action.
|
|
with tempfile.TemporaryDirectory() as tmp:
|
|
root = Path(tmp)
|
|
(root / "Jellyfin.Plugin.JRay").mkdir(parents=True)
|
|
(root / "Jellyfin.Plugin.JRay" / "TruthController.cs").write_text(
|
|
f" /// {TAG} JR-012 | SR-001\n"
|
|
' [HttpGet("{itemId}")]\n'
|
|
" public ActionResult<TruthFile> GetTruth(Guid itemId)\n"
|
|
" {\n"
|
|
" return Ok();\n"
|
|
" }\n", encoding="utf-8")
|
|
config = cfg(root=root, requirement_types=("JR",),
|
|
source_suffixes=et.LANGUAGE_SUFFIXES["csharp"],
|
|
source_roots=("Jellyfin.Plugin.JRay",))
|
|
scan = et.scan_files(et.iter_source_files(config), config)
|
|
assert len(scan.traces) == 1
|
|
assert scan.traces[0].requirements == ["JR-012", "SR-001"]
|
|
# The attribute must not steal the context from the method it decorates.
|
|
assert "GetTruth" in scan.traces[0].context
|
|
assert "HttpGet" not in scan.traces[0].context
|
|
|
|
|
|
def test_context_is_found_below_a_cpp_tag_and_above_a_python_tag():
|
|
# The two languages put the tag on opposite sides of what it describes.
|
|
with tempfile.TemporaryDirectory() as tmp:
|
|
root = Path(tmp)
|
|
_write_tree(root)
|
|
config = cfg(root=root)
|
|
scan = et.scan_files(et.iter_source_files(config), config)
|
|
contexts = {t.file: t.context for t in scan.traces}
|
|
assert "TrackRegistry" in contexts["src/tracker.hpp"]
|
|
assert "build_gallery" in contexts["scripts/gallery.py"]
|
|
|
|
|
|
def test_an_id_of_an_unconfigured_prefix_is_reported_as_unknown():
|
|
# In the Rust server, an `AR-001` tag is a copy-paste from another repo.
|
|
with tempfile.TemporaryDirectory() as tmp:
|
|
root = Path(tmp)
|
|
(root / "src").mkdir(parents=True)
|
|
(root / "src" / "a.rs").write_text(f"// {TAG} AR-001\n", encoding="utf-8")
|
|
config = cfg(root=root, requirement_types=("UR", "DR"),
|
|
source_suffixes=et.LANGUAGE_SUFFIXES["rust"],
|
|
source_roots=("src",))
|
|
scan = et.scan_files(et.iter_source_files(config), config)
|
|
assert scan.diagnostics.unknown_id_types
|
|
|
|
|
|
# --------------------------------------------------------------------------
|
|
# EXCEPTION tags
|
|
# --------------------------------------------------------------------------
|
|
|
|
EXCEPTION_SOURCE = (
|
|
"float outlier_score(const Refs& refs) {\n"
|
|
f" // {EXC} AR-024 distributional check on an actor's own references,\n"
|
|
" return spread(refs);\n"
|
|
"}\n"
|
|
)
|
|
|
|
|
|
def _exception_repo(tmp: str) -> et.Config:
|
|
root = Path(tmp)
|
|
(root / "src").mkdir(parents=True)
|
|
(root / "src" / "outlier.cpp").write_text(EXCEPTION_SOURCE, encoding="utf-8")
|
|
return cfg(root=root)
|
|
|
|
|
|
def test_exception_tag_is_captured_with_its_reason():
|
|
with tempfile.TemporaryDirectory() as tmp:
|
|
config = _exception_repo(tmp)
|
|
scan = et.scan_files(et.iter_source_files(config), config)
|
|
assert len(scan.exceptions) == 1
|
|
exc = scan.exceptions[0]
|
|
assert exc.requirement == "AR-024"
|
|
assert exc.reason.startswith("distributional check")
|
|
assert exc.line == 2
|
|
|
|
|
|
def test_exception_is_never_counted_as_coverage():
|
|
# An exception is a recorded decision to depart from an invariant. Counting
|
|
# it as evidence the requirement is met inverts its meaning.
|
|
with tempfile.TemporaryDirectory() as tmp:
|
|
config = _exception_repo(tmp)
|
|
scan = et.scan_files(et.iter_source_files(config), config)
|
|
assert scan.traces == []
|
|
register = et.parse_register(
|
|
"| ID | Requirement | Status |\n|---|---|---|\n"
|
|
"| AR-024 | Always the calibrated probability | Planned |\n", config)
|
|
cov = et.compute_coverage(
|
|
[i for t in scan.traces for i in t.requirements], register)
|
|
assert cov.covered == []
|
|
assert cov.percent == 0.0
|
|
|
|
|
|
def test_exception_without_a_reason_is_reported():
|
|
# An exception is only *agreed* if the reason is recorded.
|
|
with tempfile.TemporaryDirectory() as tmp:
|
|
root = Path(tmp)
|
|
(root / "src").mkdir(parents=True)
|
|
(root / "src" / "bare.cpp").write_text(f"// {EXC} AR-024\n", encoding="utf-8")
|
|
config = cfg(root=root)
|
|
scan = et.scan_files(et.iter_source_files(config), config)
|
|
assert len(scan.exceptions) == 1
|
|
assert scan.diagnostics.exceptions_without_reason
|
|
|
|
|
|
def test_mixed_type_group_is_reported():
|
|
# `AR-001, SR-002` in one group misuses the comma; the pipe is what
|
|
# separates types, so the tag does not say what it appears to say.
|
|
with tempfile.TemporaryDirectory() as tmp:
|
|
root = Path(tmp)
|
|
(root / "src").mkdir(parents=True)
|
|
(root / "src" / "a.cpp").write_text(
|
|
f"// {TAG} AR-001, SR-002\n", encoding="utf-8")
|
|
config = cfg(root=root)
|
|
scan = et.scan_files(et.iter_source_files(config), config)
|
|
assert scan.diagnostics.mixed_type_groups
|
|
|
|
|
|
# --------------------------------------------------------------------------
|
|
# The register: denominators from requirements.md
|
|
# --------------------------------------------------------------------------
|
|
|
|
REGISTER_HEADER = "| ID | Requirement | Traces to | Priority | Status |\n|---|---|---|---|---|\n"
|
|
|
|
|
|
def test_counts_a_well_formed_table_row_as_a_defined_requirement():
|
|
md = REGISTER_HEADER + (
|
|
"| AR-001 | Detect faces in sampled frames | SR-002 | High | Done |\n"
|
|
"| AR-002 | Minimum face size 66x66 px | SR-002 | High | Planned |\n")
|
|
register = et.parse_register(md, CPP_PY)
|
|
assert register.count("AR") == 2
|
|
assert register.count("GR") == 0
|
|
assert register.total == 2
|
|
|
|
|
|
def test_the_same_parser_reads_another_repos_prefixes():
|
|
# Identical table shape, different prefixes. This is the change that made
|
|
# the tool shareable: with requirement_types hardcoded, both of these
|
|
# parsed to zero.
|
|
md = REGISTER_HEADER + (
|
|
"| JR-001 | Truth-file schema | SR-003 | High | Done |\n"
|
|
"| UR-001 | Availability without payload | SR-001 | High | Done |\n"
|
|
"| DR-003 | One writer connection plus a read pool | SR-001 | High | Done |\n")
|
|
jray = et.parse_register(md, cfg(requirement_types=("JR",)))
|
|
server = et.parse_register(md, cfg(requirement_types=("UR", "DR")))
|
|
assert jray.ids == {"JR-001"}
|
|
assert server.ids == {"UR-001", "DR-003"}
|
|
|
|
|
|
def test_a_prefix_this_repo_does_not_define_is_not_counted_as_defined():
|
|
# The server's register must not acquire a denominator from an AR row
|
|
# pasted in from the extraction repo.
|
|
md = REGISTER_HEADER + "| AR-001 | Belongs to another repo | SR-002 | High | Done |\n"
|
|
register = et.parse_register(md, cfg(requirement_types=("UR", "DR")))
|
|
assert register.total == 0
|
|
|
|
|
|
def test_does_not_count_ids_that_appear_only_in_the_traces_to_column():
|
|
# The bug this parse rule avoids: a naive scan for AR-\d{3} over the whole
|
|
# file counts every reference as a definition and inflates the denominator.
|
|
md = REGISTER_HEADER + (
|
|
"| GR-001 | Build gallery from library cast | SR-001, SR-005 | High | Done |\n"
|
|
"| GR-002 | Incremental merge refresh | PR-003 | High | Done |\n")
|
|
register = et.parse_register(md, CPP_PY)
|
|
assert register.ids == {"GR-001", "GR-002"}
|
|
|
|
|
|
def test_does_not_count_ids_mentioned_in_prose():
|
|
md = ("Some prose explaining that AR-005 relates to GR-001 and VR-003.\n\n"
|
|
+ REGISTER_HEADER
|
|
+ "| AR-005 | Align to 112x112 | SR-002 | High | Done |\n")
|
|
register = et.parse_register(md, CPP_PY)
|
|
assert register.ids == {"AR-005"}
|
|
|
|
|
|
def test_does_not_count_the_verification_plan_table_as_definitions():
|
|
# The per-requirement verification plan is also keyed on `ID`, but it
|
|
# assigns tiers rather than defining requirements. Counting its rows would
|
|
# double the denominator for every requirement that has a plan entry.
|
|
md = (REGISTER_HEADER
|
|
+ "| AR-001 | Detect faces | SR-002 | High | Done |\n"
|
|
+ "\n"
|
|
+ "| ID | Tier | Test asserts | Edge cases to cover |\n|---|---|---|---|\n"
|
|
+ "| AR-001 | T3 | Detector returns plausible boxes | smoke only |\n"
|
|
+ "| AR-099 | T1 | Something not in the register | - |\n")
|
|
register = et.parse_register(md, CPP_PY)
|
|
assert register.ids == {"AR-001"}
|
|
|
|
|
|
def test_deduplicates_an_id_listed_in_two_definition_tables():
|
|
md = (REGISTER_HEADER
|
|
+ "| AR-001 | Detect faces | SR-002 | High | Done |\n"
|
|
+ "\n"
|
|
+ REGISTER_HEADER
|
|
+ "| AR-001 | Detect faces | SR-002 | High | Done |\n")
|
|
assert et.parse_register(md, CPP_PY).total == 1
|
|
|
|
|
|
def test_withdrawn_requirements_leave_the_denominator():
|
|
# IDs are permanent, but a withdrawn requirement can never be implemented.
|
|
# Leaving it in the denominator would depress coverage forever.
|
|
md = REGISTER_HEADER + (
|
|
"| AR-001 | Detect faces | SR-002 | High | Done |\n"
|
|
"| AR-002 | Superseded mechanism | SR-002 | High | Withdrawn |\n")
|
|
register = et.parse_register(md, CPP_PY)
|
|
assert register.ids == {"AR-001"}
|
|
assert "AR-002" in register.withdrawn
|
|
|
|
|
|
def test_an_escaped_pipe_inside_a_cell_does_not_split_the_row():
|
|
# The server's register contains `small-\|M\| all-but-one rule`; a naive
|
|
# split shifts every later column by one.
|
|
md = REGISTER_HEADER + (
|
|
r"| UR-003 | Strict schema with a small-\|M\| rule | SR-001 | High | Done |"
|
|
"\n")
|
|
register = et.parse_register(md, cfg(requirement_types=("UR",)))
|
|
assert register.requirements["UR-003"].status == "Done"
|
|
assert "|M|" in register.requirements["UR-003"].text
|
|
|
|
|
|
def test_the_denominator_is_live_adding_a_row_lowers_coverage():
|
|
# The property JellyTau's frozen literals destroyed. Same traced set, one
|
|
# more requirement defined => a lower percentage, mechanically.
|
|
base = REGISTER_HEADER + "| AR-001 | A | SR-002 | High | Done |\n"
|
|
grown = base + "| AR-002 | B | SR-002 | High | Planned |\n"
|
|
before = et.compute_coverage(["AR-001"], et.parse_register(base, CPP_PY))
|
|
after = et.compute_coverage(["AR-001"], et.parse_register(grown, CPP_PY))
|
|
assert before.percent == 100.0
|
|
assert after.percent == 50.0
|
|
assert after.total == 2
|
|
|
|
|
|
def test_register_captures_the_row_fields_not_just_the_id():
|
|
md = REGISTER_HEADER + (
|
|
"| AR-012 | Presence follows track extent | **SR-002** | High | Planned |\n")
|
|
req = et.parse_register(md, CPP_PY).requirements["AR-012"]
|
|
assert req.text == "Presence follows track extent"
|
|
assert req.traces_to == "**SR-002**"
|
|
assert req.status == "Planned"
|
|
|
|
|
|
def test_a_requirement_tracing_up_to_nothing_is_reported():
|
|
# SPEC.md section 6: a requirement citing no parent is scope creep, and it
|
|
# is invisible unless something looks. A section reference counts as a
|
|
# parent - what matters is that something was recorded.
|
|
md = REGISTER_HEADER + (
|
|
"| AR-001 | Has a parent | SR-002 | High | Done |\n"
|
|
"| AR-002 | Parent is a section | §4 | Medium | Planned |\n"
|
|
"| AR-003 | Serves nothing stated | - | Low | Planned |\n"
|
|
"| AR-004 | Blank cell | | Low | Planned |\n")
|
|
assert et.parse_register(md, CPP_PY).parentless_ids() == {"AR-003", "AR-004"}
|
|
|
|
|
|
# --------------------------------------------------------------------------
|
|
# Verification tiers and the CI host's limits
|
|
# --------------------------------------------------------------------------
|
|
|
|
TIER_REGISTER = REGISTER_HEADER + "".join(
|
|
f"| AR-{n:03d} | Requirement {n} | SR-002 | High | Planned |\n"
|
|
for n in range(1, 10)) + "".join(
|
|
f"| VR-{n:03d} | Study {n} | PR-002 | Medium | Planned |\n"
|
|
for n in range(1, 4))
|
|
|
|
|
|
def _tier_table(rows: str) -> str:
|
|
return "| Requirement | Tier | Note |\n|---|---|---|\n" + rows
|
|
|
|
|
|
def test_tier_assignment_handles_lists_ranges_and_wildcards():
|
|
md = TIER_REGISTER + "\n" + _tier_table(
|
|
"| AR-001, AR-005 | T3 | smoke |\n"
|
|
"| AR-002 … AR-004 | **T2** | replay |\n"
|
|
"| AR-006 | T1 + T4 | mixed |\n"
|
|
"| VR-* | Out of CI | studies |\n")
|
|
register = et.parse_register(md, CPP_PY)
|
|
assert register.requirements["AR-001"].tiers == {"T3"}
|
|
assert register.requirements["AR-003"].tiers == {"T2"}
|
|
assert register.requirements["AR-006"].tiers == {"T1", "T4"}
|
|
assert register.requirements["VR-002"].tiers == {"out-of-ci"}
|
|
|
|
|
|
def test_a_range_cannot_invent_a_requirement_the_register_lacks():
|
|
md = TIER_REGISTER + "\n" + _tier_table("| AR-001 … AR-050 | T2 | wide |\n")
|
|
register = et.parse_register(md, CPP_PY)
|
|
assert register.total == 12
|
|
assert "AR-050" not in register.ids
|
|
|
|
|
|
def test_slash_shorthand_in_the_verification_plan_expands():
|
|
md = TIER_REGISTER + "\n" + (
|
|
"| ID | Tier | Test asserts | Edge cases |\n|---|---|---|---|\n"
|
|
"| AR-009/008 | T2 | Cut shifts weighting | cut with same people |\n")
|
|
register = et.parse_register(md, CPP_PY)
|
|
assert register.requirements["AR-008"].tiers == {"T2"}
|
|
assert register.requirements["AR-009"].tiers == {"T2"}
|
|
|
|
|
|
def test_tiers_from_both_tables_are_unioned_not_overwritten():
|
|
# The summary table says AR-006 is T4; the per-requirement plan adds a T1
|
|
# equivalence check. The T1 part does run in CI, so the requirement is
|
|
# executable and must not be written off as GPU-only.
|
|
md = TIER_REGISTER + "\n" + _tier_table("| AR-006 | T4 | GPU host only |\n") + "\n" + (
|
|
"| ID | Tier | Test asserts | Edge cases |\n|---|---|---|---|\n"
|
|
"| AR-006 | T1 + T4 | GEMM equals reference loop | small input in CI |\n")
|
|
register = et.parse_register(md, CPP_PY)
|
|
assert register.requirements["AR-006"].tiers == {"T1", "T4"}
|
|
assert register.is_ci_executable("AR-006")
|
|
|
|
|
|
def test_a_t4_only_requirement_is_not_ci_executable():
|
|
md = TIER_REGISTER + "\n" + _tier_table("| AR-007 | **T4** | GPU only |\n")
|
|
register = et.parse_register(md, CPP_PY)
|
|
assert not register.is_ci_executable("AR-007")
|
|
assert register.unexecutable_ids() == {"AR-007"}
|
|
|
|
|
|
def test_which_tiers_count_as_executable_is_configuration_not_code():
|
|
# A repo whose CI host can run T4 says so, and the same register then
|
|
# yields full executability. Nothing about one host is baked in.
|
|
md = TIER_REGISTER + "\n" + _tier_table("| AR-007 | **T4** | GPU only |\n")
|
|
permissive = et.parse_register(
|
|
md, cfg(ci_executable_tiers=frozenset({"T1", "T2", "T3", "T4", "static"})))
|
|
assert permissive.unexecutable_ids() == set()
|
|
assert permissive.is_ci_executable("AR-007")
|
|
|
|
|
|
def test_a_requirement_with_no_tier_is_unknown_not_unexecutable():
|
|
register = et.parse_register(TIER_REGISTER, CPP_PY)
|
|
assert register.tier_unknown_ids() == register.ids
|
|
assert register.unexecutable_ids() == set()
|
|
assert not register.uses_tiers
|
|
|
|
|
|
def test_a_register_that_uses_tiers_is_distinguishable_from_one_that_does_not():
|
|
# The distinction drives whether "no tier recorded" is worth warning about;
|
|
# warning on every requirement of a tierless register trains people to
|
|
# ignore the warning.
|
|
tiered = et.parse_register(
|
|
TIER_REGISTER + "\n" + _tier_table("| AR-001 | T1 | unit |\n"), CPP_PY)
|
|
assert tiered.uses_tiers
|
|
assert not et.parse_register(TIER_REGISTER, CPP_PY).uses_tiers
|
|
|
|
|
|
# --------------------------------------------------------------------------
|
|
# Coverage arithmetic
|
|
# --------------------------------------------------------------------------
|
|
|
|
COVERAGE_REGISTER = et.parse_register(
|
|
REGISTER_HEADER
|
|
+ "| AR-001 | A | SR-002 | High | Done |\n"
|
|
+ "| AR-002 | B | SR-002 | High | Done |\n"
|
|
+ "| GR-001 | C | SR-005 | High | Done |\n"
|
|
+ "| AR-027 | Arbitrary gallery scale | SR-001 | High | Planned |\n"
|
|
+ "\n"
|
|
+ _tier_table("| AR-001, AR-002 | T2 | replay |\n"
|
|
"| GR-001 | T1 | bookkeeping |\n"
|
|
"| AR-027 | **T4** | GPU host only |\n"),
|
|
CPP_PY)
|
|
|
|
|
|
def test_coverage_is_the_intersection_of_traced_and_defined():
|
|
cov = et.compute_coverage(["AR-001", "GR-001"], COVERAGE_REGISTER)
|
|
assert cov.covered == ["AR-001", "GR-001"]
|
|
assert cov.total == 4
|
|
assert cov.percent == 50.0
|
|
|
|
|
|
def test_a_traced_but_undefined_id_cannot_inflate_the_numerator():
|
|
# This is exactly how a ratio exceeds 100%: a tag naming a renumbered or
|
|
# mistyped requirement counted as covered.
|
|
cov = et.compute_coverage(["AR-001", "GR-001", "AR-097"], COVERAGE_REGISTER)
|
|
assert cov.covered == ["AR-001", "GR-001"]
|
|
assert cov.percent == 50.0
|
|
|
|
|
|
def test_orphan_tags_are_reported_so_they_get_fixed():
|
|
cov = et.compute_coverage(["AR-001", "AR-097", "GR-404"], COVERAGE_REGISTER)
|
|
assert cov.orphaned == ["AR-097", "GR-404"]
|
|
|
|
|
|
def test_no_orphans_when_every_traced_id_is_defined():
|
|
assert et.compute_coverage(["AR-001", "AR-002"], COVERAGE_REGISTER).orphaned == []
|
|
|
|
|
|
def test_test_and_system_ids_are_a_separate_taxonomy():
|
|
# UT/IT are evidence for requirements; PR/SR live in the system spec, which
|
|
# is a different register. Neither counts toward coverage, and flagging
|
|
# them as orphans would bury real typos in noise.
|
|
cov = et.compute_coverage(
|
|
["AR-001", "UT-003", "IT-007", "SR-002", "PR-001"], COVERAGE_REGISTER)
|
|
assert cov.orphaned == []
|
|
assert cov.covered == ["AR-001"]
|
|
|
|
|
|
def test_a_gpu_only_requirement_is_tagged_but_unexecuted_not_covered():
|
|
# A test that exists but can never run on the CI host is not evidence, and
|
|
# counting it is the same failure mode as the 158% bug.
|
|
cov = et.compute_coverage(["AR-001", "AR-027"], COVERAGE_REGISTER)
|
|
assert cov.unexecuted == ["AR-027"]
|
|
assert cov.covered == ["AR-001"]
|
|
assert cov.percent == 25.0
|
|
assert cov.orphaned == []
|
|
|
|
|
|
def test_ci_scope_percentage_excludes_unexecutable_requirements_from_both_sides():
|
|
cov = et.compute_coverage(["AR-001"], COVERAGE_REGISTER)
|
|
assert cov.ci_executable_total == 3
|
|
assert round(cov.ci_percent) == 33
|
|
|
|
|
|
def test_tracing_only_gpu_only_requirements_yields_zero_coverage():
|
|
cov = et.compute_coverage(["AR-027"], COVERAGE_REGISTER)
|
|
assert cov.covered == []
|
|
assert cov.percent == 0.0
|
|
assert cov.unexecuted == ["AR-027"]
|
|
|
|
|
|
def test_duplicate_traced_ids_are_counted_once():
|
|
cov = et.compute_coverage(["AR-001", "AR-001", "AR-001"], COVERAGE_REGISTER)
|
|
assert cov.covered == ["AR-001"]
|
|
|
|
|
|
def test_an_empty_trace_set_is_zero_percent_not_a_divide_by_zero():
|
|
cov = et.compute_coverage([], COVERAGE_REGISTER)
|
|
assert cov.covered == []
|
|
assert cov.percent == 0.0
|
|
|
|
|
|
def test_an_empty_register_reports_zero_rather_than_nan():
|
|
cov = et.compute_coverage(["AR-001"], et.Register(types=("AR",)))
|
|
assert cov.percent == 0.0
|
|
assert cov.ci_percent == 0.0
|
|
|
|
|
|
def test_full_coverage_reports_exactly_one_hundred_never_above():
|
|
cov = et.compute_coverage(
|
|
["AR-001", "AR-002", "GR-001", "AR-027"], COVERAGE_REGISTER)
|
|
# AR-027 is GPU-only, so the honest ceiling here is 3/4.
|
|
assert cov.percent == 75.0
|
|
assert cov.ci_percent == 100.0
|
|
|
|
|
|
# --------------------------------------------------------------------------
|
|
# The gate: end to end, including the ways it must fail
|
|
# --------------------------------------------------------------------------
|
|
|
|
FIXTURE_REGISTER = (
|
|
"# register\n\n" + REGISTER_HEADER
|
|
+ "| AR-001 | Detect faces | SR-002 | High | Done |\n"
|
|
+ "| AR-002 | Minimum face size | SR-002 | High | Planned |\n"
|
|
+ "| AR-027 | Arbitrary gallery scale | SR-001 | High | Planned |\n"
|
|
+ "\n"
|
|
+ _tier_table("| AR-001, AR-002 | T2 | replay |\n"
|
|
"| AR-027 | **T4** | GPU host only |\n"))
|
|
|
|
FIXTURE_CONFIG_TOML = """\
|
|
requirement_types = ["AR"]
|
|
languages = ["cpp"]
|
|
source_roots = ["src"]
|
|
"""
|
|
|
|
|
|
def _fixture_repo(tmp: str, source: str, config_toml: str = FIXTURE_CONFIG_TOML,
|
|
register: str = FIXTURE_REGISTER) -> Path:
|
|
root = Path(tmp)
|
|
(root / "docs").mkdir(parents=True, exist_ok=True)
|
|
(root / "src").mkdir(parents=True, exist_ok=True)
|
|
(root / "docs" / "requirements.md").write_text(register, encoding="utf-8")
|
|
(root / "src" / "pipeline.cpp").write_text(source, encoding="utf-8")
|
|
(root / et.CONFIG_FILENAME).write_text(config_toml, encoding="utf-8")
|
|
return root
|
|
|
|
|
|
def _run_gate(root: Path, *extra: str):
|
|
buffer = io.StringIO()
|
|
with redirect_stdout(buffer):
|
|
code = et.main(["--config", str(root / et.CONFIG_FILENAME),
|
|
"--format", "coverage", *extra])
|
|
return code, buffer.getvalue()
|
|
|
|
|
|
def test_gate_reads_its_settings_from_the_repos_config_file():
|
|
with tempfile.TemporaryDirectory() as tmp:
|
|
root = _fixture_repo(tmp, f"// {TAG} AR-001\nint main() {{}}\n")
|
|
code, out = _run_gate(root)
|
|
assert code == 0, out
|
|
assert "Requirement types : AR" in out
|
|
assert str(root / et.CONFIG_FILENAME) in out
|
|
|
|
|
|
def test_config_is_discovered_from_a_subdirectory():
|
|
# The config file's directory is the repo root, so running the gate from
|
|
# deep inside the tree does not silently scan a subtree.
|
|
with tempfile.TemporaryDirectory() as tmp:
|
|
root = _fixture_repo(tmp, f"// {TAG} AR-001\nint main() {{}}\n")
|
|
nested = root / "src" / "nodes"
|
|
nested.mkdir(parents=True, exist_ok=True)
|
|
found = et.find_config(nested)
|
|
assert found == root / et.CONFIG_FILENAME
|
|
|
|
|
|
def test_gate_passes_at_the_default_threshold_with_no_tags_yet():
|
|
# Near-zero coverage on a fresh tree is the correct reading, not a failure.
|
|
with tempfile.TemporaryDirectory() as tmp:
|
|
root = _fixture_repo(tmp, "int main() { return 0; }\n")
|
|
code, out = _run_gate(root)
|
|
assert code == 0, out
|
|
assert "0 / 3 (0.0%)" in out
|
|
|
|
|
|
def test_gate_fails_below_an_explicit_threshold():
|
|
# Proves the gate can fail at all. A gate nobody has watched fail is not
|
|
# known to work.
|
|
with tempfile.TemporaryDirectory() as tmp:
|
|
root = _fixture_repo(tmp, f"// {TAG} AR-001\nint main() {{ return 0; }}\n")
|
|
code, out = _run_gate(root, "--min-coverage", "99")
|
|
assert code == 1
|
|
assert "below the minimum" in out
|
|
|
|
|
|
def test_gate_fails_on_an_orphan_tag():
|
|
with tempfile.TemporaryDirectory() as tmp:
|
|
root = _fixture_repo(tmp, f"// {TAG} AR-001, AR-404\nint main() {{}}\n")
|
|
code, out = _run_gate(root)
|
|
assert code == 1
|
|
assert "AR-404" in out
|
|
assert "orphan tag" in out
|
|
|
|
|
|
def test_gate_can_be_asked_to_report_orphans_without_failing():
|
|
with tempfile.TemporaryDirectory() as tmp:
|
|
root = _fixture_repo(tmp, f"// {TAG} AR-001, AR-404\nint main() {{}}\n")
|
|
code, out = _run_gate(root, "--allow-orphans")
|
|
assert code == 0
|
|
assert "AR-404" in out
|
|
|
|
|
|
def test_gate_fails_when_the_register_parses_to_nothing():
|
|
# With a low threshold this self-check is what keeps the gate meaningful:
|
|
# a broken parser would otherwise report 0/0 and pass.
|
|
with tempfile.TemporaryDirectory() as tmp:
|
|
root = _fixture_repo(tmp, "int main() {}\n",
|
|
register="# register\n\nNo tables here.\n")
|
|
code, out = _run_gate(root)
|
|
assert code == 1
|
|
assert "ZERO requirements" in out
|
|
|
|
|
|
def test_gate_fails_when_the_prefixes_are_configured_wrong():
|
|
# The misconfiguration a shared tool invites: the server's config pointed
|
|
# at the extraction register, or a typo'd prefix. Every row is skipped, and
|
|
# a plausible 0% would hide it completely.
|
|
with tempfile.TemporaryDirectory() as tmp:
|
|
root = _fixture_repo(
|
|
tmp, f"// {TAG} AR-001\nint main() {{}}\n",
|
|
config_toml='requirement_types = ["JR"]\nlanguages = ["cpp"]\n'
|
|
'source_roots = ["src"]\n')
|
|
code, out = _run_gate(root)
|
|
assert code == 1
|
|
assert "ZERO requirements" in out
|
|
assert "requirement_types (JR)" in out
|
|
|
|
|
|
def test_gate_fails_when_the_suffixes_are_configured_wrong():
|
|
# The Rust config applied to a C++ repo: nothing to scan, and coverage
|
|
# against an empty tree means nothing.
|
|
with tempfile.TemporaryDirectory() as tmp:
|
|
root = _fixture_repo(
|
|
tmp, f"// {TAG} AR-001\nint main() {{}}\n",
|
|
config_toml='requirement_types = ["AR"]\nlanguages = ["rust"]\n'
|
|
'source_roots = ["src"]\n')
|
|
code, out = _run_gate(root)
|
|
assert code == 1
|
|
assert "no source files were scanned" in out
|
|
assert ".rs" in out
|
|
|
|
|
|
def test_gate_fails_when_the_source_roots_do_not_exist():
|
|
with tempfile.TemporaryDirectory() as tmp:
|
|
root = _fixture_repo(
|
|
tmp, f"// {TAG} AR-001\nint main() {{}}\n",
|
|
config_toml='requirement_types = ["AR"]\nlanguages = ["cpp"]\n'
|
|
'source_roots = ["nowhere"]\n')
|
|
code, out = _run_gate(root)
|
|
assert code == 1
|
|
assert "no source files were scanned" in out
|
|
|
|
|
|
def test_gate_reports_a_gpu_only_requirement_as_tagged_but_unexecuted():
|
|
with tempfile.TemporaryDirectory() as tmp:
|
|
root = _fixture_repo(tmp, f"// {TAG} AR-027\nint main() {{}}\n")
|
|
code, out = _run_gate(root)
|
|
assert code == 0
|
|
assert "TAGGED BUT UNEXECUTED" in out
|
|
assert "AR-027 (tier T4)" in out
|
|
assert "0 / 3 (0.0%)" in out
|
|
|
|
|
|
def test_gate_hard_fails_on_an_impossible_ratio():
|
|
# Coverage above 100% cannot happen through the intersection, which is the
|
|
# point: if it ever does, the arithmetic is broken and the run must not be
|
|
# reported as a pass. Forced here by handing the reporter a poisoned value.
|
|
with tempfile.TemporaryDirectory() as tmp:
|
|
root = _fixture_repo(tmp, f"// {TAG} AR-001\nint main() {{}}\n")
|
|
config = et.config_from_dict(
|
|
{"requirement_types": ["AR"], "languages": ["cpp"],
|
|
"source_roots": ["src"]}, root, "fixture")
|
|
register = et.read_register(config)
|
|
scan = et.scan_files(et.iter_source_files(config), config)
|
|
report = et.build_report(config, register, scan)
|
|
report.coverage.percent = 158.0
|
|
text, code = et.format_coverage_report(report)
|
|
assert code == 1
|
|
assert "exceeds 100%" in text
|
|
|
|
|
|
def test_the_per_type_breakdown_sums_to_the_headline_figure():
|
|
# A breakdown that does not add up to its own total is how a wrong number
|
|
# survives review: every row looks plausible on its own.
|
|
with tempfile.TemporaryDirectory() as tmp:
|
|
root = _fixture_repo(tmp, f"// {TAG} AR-001, AR-027\nint main() {{}}\n")
|
|
config = et.config_from_dict(
|
|
{"requirement_types": ["AR"], "languages": ["cpp"],
|
|
"source_roots": ["src"]}, root, "fixture")
|
|
register = et.read_register(config)
|
|
scan = et.scan_files(et.iter_source_files(config), config)
|
|
report = et.build_report(config, register, scan)
|
|
stats = et.per_type_stats(report)
|
|
assert sum(c for c, _, _ in stats.values()) == len(report.coverage.covered)
|
|
assert sum(u for _, u, _ in stats.values()) == len(report.coverage.unexecuted)
|
|
assert sum(d for _, _, d in stats.values()) == report.coverage.total
|
|
assert stats["AR"] == (1, 1, 3)
|
|
|
|
|
|
def test_json_and_markdown_outputs_are_written_and_consistent():
|
|
with tempfile.TemporaryDirectory() as tmp:
|
|
root = _fixture_repo(tmp, f"// {TAG} AR-001 | SR-002\nint main() {{}}\n")
|
|
code, _ = _run_gate(root)
|
|
assert code == 0
|
|
data = json.loads((root / "traces-report.json").read_text(encoding="utf-8"))
|
|
assert data["defined"]["total"] == 3
|
|
assert data["coverage"]["covered"] == 1
|
|
assert data["coverage"]["percent"] == round(100 / 3, 1)
|
|
assert data["byType"]["SR"] == ["SR-002"]
|
|
assert data["unexecutableRequirements"] == ["AR-027"]
|
|
# The report says how it was produced; a shared tool's output is
|
|
# otherwise ambiguous about which repo and settings it describes.
|
|
assert data["config"]["requirementTypes"] == ["AR"]
|
|
assert data["config"]["sourceRoots"] == ["src"]
|
|
assert "AR-001" in (root / "docs" / "traceability.md").read_text(
|
|
encoding="utf-8")
|
|
|
|
|
|
def test_cli_flags_alone_work_with_no_config_file_at_all():
|
|
# The path a repo takes before it has a traceability.toml, and the escape
|
|
# hatch on a Python too old for tomllib.
|
|
with tempfile.TemporaryDirectory() as tmp:
|
|
root = _fixture_repo(tmp, f"// {TAG} AR-001\nint main() {{}}\n")
|
|
(root / et.CONFIG_FILENAME).unlink()
|
|
buffer = io.StringIO()
|
|
with redirect_stdout(buffer):
|
|
code = et.main(["--no-config", "--root", str(root),
|
|
"--requirement-type", "AR",
|
|
"--language", "cpp", "--source-root", "src",
|
|
"--format", "coverage", "--no-write"])
|
|
assert code == 0, buffer.getvalue()
|
|
assert "1 / 3" in buffer.getvalue()
|
|
assert not (root / "traces-report.json").exists()
|
|
|
|
|
|
def test_cli_flags_override_the_config_file():
|
|
with tempfile.TemporaryDirectory() as tmp:
|
|
root = _fixture_repo(tmp, f"// {TAG} AR-001\nint main() {{}}\n")
|
|
code, out = _run_gate(root, "--requirement-type", "JR", "--no-write")
|
|
assert code == 1
|
|
assert "requirement_types (JR)" in out
|
|
|
|
|
|
def test_language_flags_extend_rather_than_replace_the_configured_set():
|
|
with tempfile.TemporaryDirectory() as tmp:
|
|
root = _fixture_repo(tmp, f"// {TAG} AR-001\nint main() {{}}\n")
|
|
(root / "src" / "extra.rs").write_text(
|
|
f"/// {TAG} AR-002\npub fn go() {{}}\n", encoding="utf-8")
|
|
code, out = _run_gate(root, "--language", "rust", "--no-write")
|
|
assert code == 0, out
|
|
assert "2 / 3" in out
|
|
|
|
|
|
def test_an_invalid_configuration_is_a_distinct_exit_code():
|
|
# 2, not 1: a broken config is not a coverage failure, and conflating them
|
|
# makes CI logs lie about why the job went red.
|
|
with tempfile.TemporaryDirectory() as tmp:
|
|
root = _fixture_repo(tmp, "int main() {}\n",
|
|
config_toml='languages = ["cpp"]\nsource_roots = ["src"]\n')
|
|
buffer = io.StringIO()
|
|
with redirect_stdout(buffer), redirect_stderr(buffer):
|
|
code = et.main(["--config", str(root / et.CONFIG_FILENAME)])
|
|
assert code == 2
|
|
assert "requirement_types is empty" in buffer.getvalue()
|
|
|
|
|
|
def test_system_spec_parsing_enables_orphan_checks_for_pr_and_sr():
|
|
spec = ("### SR-002 - Presence is scene-scoped\n\n"
|
|
"| ID | Goal | Why |\n|---|---|---|\n"
|
|
"| **PR-001** | Show which actors are on screen | The product |\n")
|
|
assert et.parse_system_spec(spec) == {"SR-002", "PR-001"}
|
|
|
|
with tempfile.TemporaryDirectory() as tmp:
|
|
root = _fixture_repo(tmp, f"// {TAG} AR-001 | SR-999\nint main() {{}}\n")
|
|
spec_path = root / "system-spec.md"
|
|
spec_path.write_text(spec, encoding="utf-8")
|
|
code, out = _run_gate(root, "--system-spec", str(spec_path), "--no-write")
|
|
assert "SR-999" in out
|
|
assert code == 0 # advisory: the system register is not this repo's
|
|
|
|
|
|
# --------------------------------------------------------------------------
|
|
# The live repo — structural assertions only, so this does not churn as
|
|
# requirements are added, and so it still means something once this file is
|
|
# vendored into another component.
|
|
# --------------------------------------------------------------------------
|
|
|
|
def _live_config() -> et.Config:
|
|
config_path = et.find_config(HERE)
|
|
assert config_path is not None, "no traceability.toml found above this file"
|
|
return et.config_from_dict(
|
|
__import__("tomllib").loads(config_path.read_text(encoding="utf-8")),
|
|
config_path.parent, str(config_path))
|
|
|
|
|
|
def test_this_repos_config_is_valid_and_its_register_parses():
|
|
config = _live_config()
|
|
config.validate()
|
|
register = et.read_register(config)
|
|
assert register.total > 0
|
|
for req_type in config.requirement_types:
|
|
assert register.count(req_type) > 0, req_type
|
|
assert sum(register.count(t) for t in config.requirement_types) == register.total
|
|
|
|
|
|
def test_this_repos_gate_run_cannot_exceed_one_hundred_percent():
|
|
config = _live_config()
|
|
register = et.read_register(config)
|
|
scan = et.scan_files(et.iter_source_files(config), config)
|
|
assert scan.files, "configured source roots matched no files"
|
|
cov = et.compute_coverage(
|
|
[i for t in scan.traces for i in t.requirements], register)
|
|
assert 0.0 <= cov.percent <= 100.0
|
|
assert len(cov.covered) <= cov.total
|
|
|
|
|
|
# --------------------------------------------------------------------------
|
|
|
|
def _main() -> int:
|
|
tests = [(name, obj) for name, obj in sorted(globals().items())
|
|
if name.startswith("test_") and callable(obj)]
|
|
failed = []
|
|
for name, fn in tests:
|
|
try:
|
|
fn()
|
|
except Exception as exc: # noqa: BLE001 - a test runner reports everything
|
|
failed.append((name, exc))
|
|
print(f"FAIL {name}: {type(exc).__name__}: {exc}")
|
|
else:
|
|
print(f"ok {name}")
|
|
print(f"\n{len(tests) - len(failed)}/{len(tests)} passed")
|
|
return 1 if failed else 0
|
|
|
|
|
|
if __name__ == "__main__":
|
|
sys.exit(_main())
|