dtourolle
054c4c8b8f
Ports JellyTau's traceability tooling, rewritten in stdlib Python because
this repo is C++/Python and adding a bun/node toolchain to check source
comments would be a worse trade than writing the scanner.
scripts/traceability/extract_traces.py scans .cpp/.hpp/.py under src, tests,
scripts, experiments and eval for the house tag format
/// TRACES: AR-012, AR-013 | SR-002
and reports EXCEPTION tags separately. An exception is a recorded decision to
depart from an invariant, so folding it into coverage would invert its
meaning; it is listed with its reason, and a missing reason is flagged.
Two rules carried over from JellyTau's gate repair:
* Denominators are parsed out of docs/requirements.md at run time. A
requirement is defined only by a row in a table whose header is
`| ID | Requirement | ... |`, so references in the Traces to column, in
prose, and in the verification-plan table do not inflate the count.
Adding a register row lowers coverage until it is traced - the property
that dies the moment a denominator is frozen.
* Coverage above 100% is a hard failure. It cannot happen through the
intersection, which is the point: if it ever does, the arithmetic is
broken and the run must not be reported as a pass.
One rule specific to this repo: CI is an Intel N100 with no discrete GPU. The
extractor reads each requirement's verification tier from requirements.md and
reports T4/GPU-only requirements as tagged but unexecuted, never as covered.
Counting a test that can never run is the same failure mode as the 158% bug.
MIN_COVERAGE starts at 0 because almost nothing is tagged yet - tags are added
as the pipeline is built. That is not a gate that cannot fail: orphan tags, a
>100% ratio, a register that parses to nothing, and an empty source scan are
all hard failures from day one. The threshold lives in traceability-gate.sh
alone, never duplicated into the workflow YAML.
53 tests over fixture strings, so their meaning does not drift as requirements
are added.
707 lines
28 KiB
Python
Executable File
707 lines
28 KiB
Python
Executable File
#!/usr/bin/env python3
|
|
"""Tests for the traceability extractor and coverage gate.
|
|
|
|
Run standalone (no third-party dependencies)::
|
|
|
|
python3 scripts/traceability/test_extract_traces.py
|
|
|
|
or under pytest, which discovers the same functions::
|
|
|
|
pytest scripts/traceability/test_extract_traces.py
|
|
|
|
Almost every test runs over fixture strings rather than the live
|
|
``docs/requirements.md``, so their meaning does not drift as requirements are
|
|
added. The two properties they exist to pin are the ones JellyTau's gate lost
|
|
(see JellyTau/docs/specs/traceability-gate-repair.md):
|
|
|
|
* the denominator is computed from the register at run time, so adding a
|
|
requirement lowers coverage until it is traced;
|
|
* the numerator is an intersection, so a tag naming an undefined ID cannot push
|
|
the ratio above 100%.
|
|
|
|
Plus the rule specific to this repo: a requirement only verifiable on GPU
|
|
hardware is reported as tagged-but-unexecuted and never counted as covered.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import io
|
|
import sys
|
|
import tempfile
|
|
from contextlib import redirect_stdout
|
|
from pathlib import Path
|
|
|
|
HERE = Path(__file__).resolve().parent
|
|
sys.path.insert(0, str(HERE))
|
|
|
|
import extract_traces as et # noqa: E402
|
|
|
|
# The literal tag keyword is assembled at run time so that this file's fixtures
|
|
# do not register as real traces when the extractor scans scripts/.
|
|
TAG = "TRA" + "CES:"
|
|
EXC = "EXCEP" + "TION:"
|
|
|
|
|
|
# --------------------------------------------------------------------------
|
|
# Tag parsing
|
|
# --------------------------------------------------------------------------
|
|
|
|
def test_parses_a_single_requirement():
|
|
groups, junk = et.parse_traces_tag(" AR-012")
|
|
assert groups == [["AR-012"]]
|
|
assert junk == []
|
|
|
|
|
|
def test_parses_multiple_types_separated_by_pipe():
|
|
# The house format: a pipe separates requirement *types*, a comma separates
|
|
# IDs within a type. The grouping is preserved, not flattened away, so a
|
|
# malformed grouping stays detectable.
|
|
groups, junk = et.parse_traces_tag(" AR-012, AR-013 | SR-002")
|
|
assert groups == [["AR-012", "AR-013"], ["SR-002"]]
|
|
assert junk == []
|
|
|
|
|
|
def test_parses_three_groups_including_test_ids():
|
|
groups, _ = et.parse_traces_tag(" AR-012 | SR-002 | UT-003, UT-004")
|
|
assert groups == [["AR-012"], ["SR-002"], ["UT-003", "UT-004"]]
|
|
|
|
|
|
def test_strips_a_trailing_block_comment_terminator():
|
|
groups, junk = et.parse_traces_tag(" AR-001 */")
|
|
assert groups == [["AR-001"]]
|
|
assert junk == []
|
|
|
|
|
|
def test_does_not_harvest_ids_out_of_prose_after_the_tag():
|
|
# A trailing sentence must not smuggle IDs into the trace set: AR-999 here
|
|
# is discussion, not a claim that this code satisfies AR-999.
|
|
groups, junk = et.parse_traces_tag(" AR-001 - see also AR-999 in the notes")
|
|
assert groups == [["AR-001"]]
|
|
assert junk and "AR-999" in junk[0]
|
|
|
|
|
|
def test_ignores_a_tag_with_no_ids_at_all():
|
|
groups, _ = et.parse_traces_tag(" see the register")
|
|
assert groups == []
|
|
|
|
|
|
def test_two_digit_id_is_not_accepted_as_a_requirement():
|
|
# AR-12 is a typo for AR-012; silently accepting it would create a
|
|
# phantom requirement.
|
|
groups, junk = et.parse_traces_tag(" AR-12")
|
|
assert groups == []
|
|
assert junk == ["AR-12"]
|
|
|
|
|
|
# --------------------------------------------------------------------------
|
|
# Scanning C++ and Python sources
|
|
# --------------------------------------------------------------------------
|
|
|
|
def _write_tree(root: Path) -> None:
|
|
(root / "src").mkdir(parents=True, exist_ok=True)
|
|
(root / "scripts").mkdir(parents=True, exist_ok=True)
|
|
(root / "external").mkdir(parents=True, exist_ok=True)
|
|
(root / "src" / "tracker.hpp").write_text(
|
|
"#pragma once\n"
|
|
f"/// {TAG} AR-012, AR-013 | SR-002\n"
|
|
"struct TrackRegistry {\n"
|
|
" void close_all();\n"
|
|
"};\n",
|
|
encoding="utf-8")
|
|
(root / "scripts" / "gallery.py").write_text(
|
|
"def build_gallery(cast):\n"
|
|
' """Build a gallery from Jellyfin plus TMDB fallback.\n'
|
|
"\n"
|
|
f" {TAG} GR-001 | SR-005\n"
|
|
' """\n'
|
|
" return {}\n",
|
|
encoding="utf-8")
|
|
(root / "external" / "vendored.cpp").write_text(
|
|
f"// {TAG} AR-001\n", encoding="utf-8")
|
|
(root / "src" / "notes.txt").write_text(
|
|
f"// {TAG} AR-002\n", encoding="utf-8")
|
|
|
|
|
|
def test_scans_cpp_and_python_but_not_vendored_or_non_source():
|
|
with tempfile.TemporaryDirectory() as tmp:
|
|
root = Path(tmp)
|
|
_write_tree(root)
|
|
files = et.iter_source_files(root)
|
|
names = sorted(f.name for f in files)
|
|
assert names == ["gallery.py", "tracker.hpp"], names
|
|
|
|
scan = et.scan_files(files, root)
|
|
traced = sorted({i for t in scan.traces for i in t.requirements})
|
|
assert traced == ["AR-012", "AR-013", "GR-001", "SR-002", "SR-005"]
|
|
|
|
|
|
def test_context_is_found_below_a_cpp_tag_and_above_a_python_tag():
|
|
# The two languages put the tag on opposite sides of what it describes.
|
|
with tempfile.TemporaryDirectory() as tmp:
|
|
root = Path(tmp)
|
|
_write_tree(root)
|
|
scan = et.scan_files(et.iter_source_files(root), root)
|
|
contexts = {t.file: t.context for t in scan.traces}
|
|
assert "TrackRegistry" in contexts["src/tracker.hpp"]
|
|
assert "build_gallery" in contexts["scripts/gallery.py"]
|
|
|
|
|
|
# --------------------------------------------------------------------------
|
|
# EXCEPTION tags
|
|
# --------------------------------------------------------------------------
|
|
|
|
EXCEPTION_SOURCE = (
|
|
"float outlier_score(const Refs& refs) {\n"
|
|
f" // {EXC} AR-024 distributional check on an actor's own references,\n"
|
|
" return spread(refs);\n"
|
|
"}\n"
|
|
)
|
|
|
|
|
|
def test_exception_tag_is_captured_with_its_reason():
|
|
with tempfile.TemporaryDirectory() as tmp:
|
|
root = Path(tmp)
|
|
(root / "src").mkdir(parents=True)
|
|
(root / "src" / "outlier.cpp").write_text(EXCEPTION_SOURCE, encoding="utf-8")
|
|
scan = et.scan_files(et.iter_source_files(root), root)
|
|
assert len(scan.exceptions) == 1
|
|
exc = scan.exceptions[0]
|
|
assert exc.requirement == "AR-024"
|
|
assert exc.reason.startswith("distributional check")
|
|
assert exc.line == 2
|
|
|
|
|
|
def test_exception_is_never_counted_as_coverage():
|
|
# An exception is a recorded decision to depart from an invariant. Counting
|
|
# it as evidence the requirement is met inverts its meaning.
|
|
with tempfile.TemporaryDirectory() as tmp:
|
|
root = Path(tmp)
|
|
(root / "src").mkdir(parents=True)
|
|
(root / "src" / "outlier.cpp").write_text(EXCEPTION_SOURCE, encoding="utf-8")
|
|
scan = et.scan_files(et.iter_source_files(root), root)
|
|
assert scan.traces == []
|
|
register = et.parse_register(
|
|
"| ID | Requirement | Status |\n|---|---|---|\n"
|
|
"| AR-024 | Always the calibrated probability | Planned |\n")
|
|
cov = et.compute_coverage(
|
|
[i for t in scan.traces for i in t.requirements], register)
|
|
assert cov.covered == []
|
|
assert cov.percent == 0.0
|
|
|
|
|
|
def test_exception_without_a_reason_is_reported():
|
|
# CLAUDE.md: an exception is only *agreed* if the reason is recorded.
|
|
with tempfile.TemporaryDirectory() as tmp:
|
|
root = Path(tmp)
|
|
(root / "src").mkdir(parents=True)
|
|
(root / "src" / "bare.cpp").write_text(
|
|
f"// {EXC} AR-024\n", encoding="utf-8")
|
|
scan = et.scan_files(et.iter_source_files(root), root)
|
|
assert len(scan.exceptions) == 1
|
|
assert scan.diagnostics.exceptions_without_reason
|
|
|
|
|
|
def test_mixed_type_group_is_reported():
|
|
# `AR-001, SR-002` in one group misuses the comma; the pipe is what
|
|
# separates types, so the tag does not say what it appears to say.
|
|
with tempfile.TemporaryDirectory() as tmp:
|
|
root = Path(tmp)
|
|
(root / "src").mkdir(parents=True)
|
|
(root / "src" / "a.cpp").write_text(
|
|
f"// {TAG} AR-001, SR-002\n", encoding="utf-8")
|
|
scan = et.scan_files(et.iter_source_files(root), root)
|
|
assert scan.diagnostics.mixed_type_groups
|
|
|
|
|
|
# --------------------------------------------------------------------------
|
|
# The register: denominators from requirements.md
|
|
# --------------------------------------------------------------------------
|
|
|
|
REGISTER_HEADER = "| ID | Requirement | Traces to | Priority | Status |\n|---|---|---|---|---|\n"
|
|
|
|
|
|
def test_counts_a_well_formed_table_row_as_a_defined_requirement():
|
|
md = REGISTER_HEADER + (
|
|
"| AR-001 | Detect faces in sampled frames | SR-002 | High | Done |\n"
|
|
"| AR-002 | Minimum face size 66x66 px | SR-002 | High | Planned |\n")
|
|
register = et.parse_register(md)
|
|
assert register.count("AR") == 2
|
|
assert register.count("GR") == 0
|
|
assert register.total == 2
|
|
|
|
|
|
def test_does_not_count_ids_that_appear_only_in_the_traces_to_column():
|
|
# The bug this parse rule avoids: a naive scan for AR-\d{3} over the whole
|
|
# file counts every reference as a definition and inflates the denominator.
|
|
md = REGISTER_HEADER + (
|
|
"| GR-001 | Build gallery from library cast | SR-001, SR-005 | High | Done |\n"
|
|
"| GR-002 | Incremental merge refresh | PR-003 | High | Done |\n")
|
|
register = et.parse_register(md)
|
|
assert register.count("GR") == 2
|
|
assert register.ids == {"GR-001", "GR-002"}
|
|
|
|
|
|
def test_does_not_count_ids_mentioned_in_prose():
|
|
md = ("Some prose explaining that AR-005 relates to GR-001 and VR-003.\n\n"
|
|
+ REGISTER_HEADER
|
|
+ "| AR-005 | Align to 112x112 | SR-002 | High | Done |\n")
|
|
register = et.parse_register(md)
|
|
assert register.ids == {"AR-005"}
|
|
|
|
|
|
def test_does_not_count_the_verification_plan_table_as_definitions():
|
|
# The per-requirement verification plan is also keyed on `ID`, but it
|
|
# assigns tiers rather than defining requirements. Counting its rows would
|
|
# double the denominator for every requirement that has a plan entry.
|
|
md = (REGISTER_HEADER
|
|
+ "| AR-001 | Detect faces | SR-002 | High | Done |\n"
|
|
+ "\n"
|
|
+ "| ID | Tier | Test asserts | Edge cases to cover |\n|---|---|---|---|\n"
|
|
+ "| AR-001 | T3 | Detector returns plausible boxes | smoke only |\n"
|
|
+ "| AR-099 | T1 | Something not in the register | - |\n")
|
|
register = et.parse_register(md)
|
|
assert register.ids == {"AR-001"}
|
|
assert register.total == 1
|
|
|
|
|
|
def test_deduplicates_an_id_listed_in_two_definition_tables():
|
|
md = (REGISTER_HEADER
|
|
+ "| AR-001 | Detect faces | SR-002 | High | Done |\n"
|
|
+ "\n"
|
|
+ REGISTER_HEADER
|
|
+ "| AR-001 | Detect faces | SR-002 | High | Done |\n")
|
|
register = et.parse_register(md)
|
|
assert register.total == 1
|
|
|
|
|
|
def test_withdrawn_requirements_leave_the_denominator():
|
|
# IDs are permanent, but a withdrawn requirement can never be implemented.
|
|
# Leaving it in the denominator would depress coverage forever.
|
|
md = REGISTER_HEADER + (
|
|
"| AR-001 | Detect faces | SR-002 | High | Done |\n"
|
|
"| AR-002 | Superseded mechanism | SR-002 | High | Withdrawn |\n")
|
|
register = et.parse_register(md)
|
|
assert register.ids == {"AR-001"}
|
|
assert "AR-002" in register.withdrawn
|
|
|
|
|
|
def test_the_denominator_is_live_adding_a_row_lowers_coverage():
|
|
# The property JellyTau's frozen literals destroyed. Same traced set, one
|
|
# more requirement defined => a lower percentage, mechanically.
|
|
base = REGISTER_HEADER + "| AR-001 | A | SR-002 | High | Done |\n"
|
|
grown = base + "| AR-002 | B | SR-002 | High | Planned |\n"
|
|
before = et.compute_coverage(["AR-001"], et.parse_register(base))
|
|
after = et.compute_coverage(["AR-001"], et.parse_register(grown))
|
|
assert before.percent == 100.0
|
|
assert after.percent == 50.0
|
|
assert after.total == 2
|
|
|
|
|
|
def test_register_captures_the_row_fields_not_just_the_id():
|
|
md = REGISTER_HEADER + (
|
|
"| AR-012 | Presence follows track extent | **SR-002** | High | Planned |\n")
|
|
req = et.parse_register(md).requirements["AR-012"]
|
|
assert req.text == "Presence follows track extent"
|
|
assert req.traces_to == "**SR-002**"
|
|
assert req.status == "Planned"
|
|
|
|
|
|
# --------------------------------------------------------------------------
|
|
# Verification tiers and the GPU-less CI host
|
|
# --------------------------------------------------------------------------
|
|
|
|
TIER_REGISTER = REGISTER_HEADER + "".join(
|
|
f"| AR-{n:03d} | Requirement {n} | SR-002 | High | Planned |\n"
|
|
for n in range(1, 10)) + "".join(
|
|
f"| VR-{n:03d} | Study {n} | PR-002 | Medium | Planned |\n"
|
|
for n in range(1, 4))
|
|
|
|
|
|
def _tier_table(rows: str) -> str:
|
|
return "| Requirement | Tier | Note |\n|---|---|---|\n" + rows
|
|
|
|
|
|
def test_tier_assignment_handles_lists_ranges_and_wildcards():
|
|
md = TIER_REGISTER + "\n" + _tier_table(
|
|
"| AR-001, AR-005 | T3 | smoke |\n"
|
|
"| AR-002 … AR-004 | **T2** | replay |\n"
|
|
"| AR-006 | T1 + T4 | mixed |\n"
|
|
"| VR-* | Out of CI | studies |\n")
|
|
register = et.parse_register(md)
|
|
assert register.requirements["AR-001"].tiers == {"T3"}
|
|
assert register.requirements["AR-003"].tiers == {"T2"}
|
|
assert register.requirements["AR-006"].tiers == {"T1", "T4"}
|
|
assert register.requirements["VR-002"].tiers == {"out-of-ci"}
|
|
|
|
|
|
def test_a_range_cannot_invent_a_requirement_the_register_lacks():
|
|
md = TIER_REGISTER + "\n" + _tier_table("| AR-001 … AR-050 | T2 | wide |\n")
|
|
register = et.parse_register(md)
|
|
assert register.total == 12
|
|
assert "AR-050" not in register.ids
|
|
|
|
|
|
def test_slash_shorthand_in_the_verification_plan_expands():
|
|
md = TIER_REGISTER + "\n" + (
|
|
"| ID | Tier | Test asserts | Edge cases |\n|---|---|---|---|\n"
|
|
"| AR-009/008 | T2 | Cut shifts weighting | cut with same people |\n")
|
|
register = et.parse_register(md)
|
|
assert register.requirements["AR-008"].tiers == {"T2"}
|
|
assert register.requirements["AR-009"].tiers == {"T2"}
|
|
|
|
|
|
def test_tiers_from_both_tables_are_unioned_not_overwritten():
|
|
# The summary table says AR-006 is T4; the per-requirement plan adds a T1
|
|
# equivalence check. The T1 part does run in CI, so the requirement is
|
|
# executable and must not be written off as GPU-only.
|
|
md = TIER_REGISTER + "\n" + _tier_table("| AR-006 | T4 | GPU host only |\n") + "\n" + (
|
|
"| ID | Tier | Test asserts | Edge cases |\n|---|---|---|---|\n"
|
|
"| AR-006 | T1 + T4 | GEMM equals reference loop | small input in CI |\n")
|
|
register = et.parse_register(md)
|
|
assert register.requirements["AR-006"].tiers == {"T1", "T4"}
|
|
assert register.requirements["AR-006"].ci_executable
|
|
|
|
|
|
def test_a_t4_only_requirement_is_not_ci_executable():
|
|
md = TIER_REGISTER + "\n" + _tier_table("| AR-007 | **T4** | GPU only |\n")
|
|
register = et.parse_register(md)
|
|
assert not register.requirements["AR-007"].ci_executable
|
|
assert register.unexecutable_ids() == {"AR-007"}
|
|
|
|
|
|
def test_a_requirement_tracing_up_to_nothing_is_reported():
|
|
# SPEC.md section 6: a requirement citing no parent is scope creep, and it
|
|
# is invisible unless something looks. A section reference counts as a
|
|
# parent - what matters is that something was recorded.
|
|
md = REGISTER_HEADER + (
|
|
"| AR-001 | Has a parent | SR-002 | High | Done |\n"
|
|
"| AR-002 | Parent is a section | §4 | Medium | Planned |\n"
|
|
"| AR-003 | Serves nothing stated | - | Low | Planned |\n"
|
|
"| AR-004 | Blank cell | | Low | Planned |\n")
|
|
register = et.parse_register(md)
|
|
assert register.parentless_ids() == {"AR-003", "AR-004"}
|
|
|
|
|
|
def test_a_requirement_with_no_tier_is_unknown_not_unexecutable():
|
|
register = et.parse_register(TIER_REGISTER)
|
|
assert register.tier_unknown_ids() == register.ids
|
|
assert register.unexecutable_ids() == set()
|
|
|
|
|
|
# --------------------------------------------------------------------------
|
|
# Coverage arithmetic
|
|
# --------------------------------------------------------------------------
|
|
|
|
COVERAGE_REGISTER = et.parse_register(
|
|
REGISTER_HEADER
|
|
+ "| AR-001 | A | SR-002 | High | Done |\n"
|
|
+ "| AR-002 | B | SR-002 | High | Done |\n"
|
|
+ "| GR-001 | C | SR-005 | High | Done |\n"
|
|
+ "| AR-027 | Arbitrary gallery scale | SR-001 | High | Planned |\n"
|
|
+ "\n"
|
|
+ _tier_table("| AR-001, AR-002 | T2 | replay |\n"
|
|
"| GR-001 | T1 | bookkeeping |\n"
|
|
"| AR-027 | **T4** | GPU host only |\n"))
|
|
|
|
|
|
def test_coverage_is_the_intersection_of_traced_and_defined():
|
|
cov = et.compute_coverage(["AR-001", "GR-001"], COVERAGE_REGISTER)
|
|
assert cov.covered == ["AR-001", "GR-001"]
|
|
assert cov.total == 4
|
|
assert cov.percent == 50.0
|
|
|
|
|
|
def test_a_traced_but_undefined_id_cannot_inflate_the_numerator():
|
|
# This is exactly how a ratio exceeds 100%: a tag naming a renumbered or
|
|
# mistyped requirement counted as covered.
|
|
cov = et.compute_coverage(["AR-001", "GR-001", "AR-097"], COVERAGE_REGISTER)
|
|
assert cov.covered == ["AR-001", "GR-001"]
|
|
assert cov.percent == 50.0
|
|
|
|
|
|
def test_orphan_tags_are_reported_so_they_get_fixed():
|
|
cov = et.compute_coverage(["AR-001", "AR-097", "GR-404"], COVERAGE_REGISTER)
|
|
assert cov.orphaned == ["AR-097", "GR-404"]
|
|
|
|
|
|
def test_no_orphans_when_every_traced_id_is_defined():
|
|
cov = et.compute_coverage(["AR-001", "AR-002"], COVERAGE_REGISTER)
|
|
assert cov.orphaned == []
|
|
|
|
|
|
def test_test_and_system_ids_are_a_separate_taxonomy():
|
|
# UT/IT live in their own register section; PR/SR live in the umbrella
|
|
# SPEC.md, which is not part of this repo's checkout. Neither counts toward
|
|
# coverage, and flagging them as orphans would bury real typos in noise.
|
|
cov = et.compute_coverage(
|
|
["AR-001", "UT-003", "IT-007", "SR-002", "PR-001"], COVERAGE_REGISTER)
|
|
assert cov.orphaned == []
|
|
assert cov.covered == ["AR-001"]
|
|
|
|
|
|
def test_a_gpu_only_requirement_is_tagged_but_unexecuted_not_covered():
|
|
# The rule specific to this repo: CI is an Intel N100 with no dGPU. A test
|
|
# that exists but can never run is not evidence, and counting it is the
|
|
# same failure mode as the 158% bug.
|
|
cov = et.compute_coverage(["AR-001", "AR-027"], COVERAGE_REGISTER)
|
|
assert cov.unexecuted == ["AR-027"]
|
|
assert cov.covered == ["AR-001"]
|
|
assert cov.percent == 25.0
|
|
assert cov.orphaned == []
|
|
|
|
|
|
def test_ci_scope_percentage_excludes_unexecutable_requirements_from_both_sides():
|
|
cov = et.compute_coverage(["AR-001"], COVERAGE_REGISTER)
|
|
assert cov.ci_executable_total == 3
|
|
assert round(cov.ci_percent) == 33
|
|
|
|
|
|
def test_tracing_only_gpu_only_requirements_yields_zero_coverage():
|
|
cov = et.compute_coverage(["AR-027"], COVERAGE_REGISTER)
|
|
assert cov.covered == []
|
|
assert cov.percent == 0.0
|
|
assert cov.unexecuted == ["AR-027"]
|
|
|
|
|
|
def test_duplicate_traced_ids_are_counted_once():
|
|
cov = et.compute_coverage(["AR-001", "AR-001", "AR-001"], COVERAGE_REGISTER)
|
|
assert cov.covered == ["AR-001"]
|
|
|
|
|
|
def test_an_empty_trace_set_is_zero_percent_not_a_divide_by_zero():
|
|
cov = et.compute_coverage([], COVERAGE_REGISTER)
|
|
assert cov.covered == []
|
|
assert cov.percent == 0.0
|
|
|
|
|
|
def test_an_empty_register_reports_zero_rather_than_nan():
|
|
cov = et.compute_coverage(["AR-001"], et.Register())
|
|
assert cov.percent == 0.0
|
|
assert cov.ci_percent == 0.0
|
|
|
|
|
|
def test_full_coverage_reports_exactly_one_hundred_never_above():
|
|
cov = et.compute_coverage(
|
|
["AR-001", "AR-002", "GR-001", "AR-027"], COVERAGE_REGISTER)
|
|
# AR-027 is GPU-only, so the honest ceiling here is 3/4.
|
|
assert cov.percent == 75.0
|
|
assert cov.ci_percent == 100.0
|
|
assert cov.ci_percent <= 100.0
|
|
|
|
|
|
# --------------------------------------------------------------------------
|
|
# The gate: end to end, including the ways it must fail
|
|
# --------------------------------------------------------------------------
|
|
|
|
FIXTURE_REGISTER = (
|
|
"# register\n\n" + REGISTER_HEADER
|
|
+ "| AR-001 | Detect faces | SR-002 | High | Done |\n"
|
|
+ "| AR-002 | Minimum face size | SR-002 | High | Planned |\n"
|
|
+ "| AR-027 | Arbitrary gallery scale | SR-001 | High | Planned |\n"
|
|
+ "\n"
|
|
+ _tier_table("| AR-001, AR-002 | T2 | replay |\n"
|
|
"| AR-027 | **T4** | GPU host only |\n"))
|
|
|
|
|
|
def _fixture_repo(tmp: str, source: str) -> Path:
|
|
root = Path(tmp)
|
|
(root / "docs").mkdir(parents=True, exist_ok=True)
|
|
(root / "src").mkdir(parents=True, exist_ok=True)
|
|
(root / "docs" / "requirements.md").write_text(FIXTURE_REGISTER, encoding="utf-8")
|
|
(root / "src" / "pipeline.cpp").write_text(source, encoding="utf-8")
|
|
return root
|
|
|
|
|
|
def _run_gate(root: Path, *extra: str):
|
|
buffer = io.StringIO()
|
|
with redirect_stdout(buffer):
|
|
code = et.main(["--root", str(root), "--format", "coverage", *extra])
|
|
return code, buffer.getvalue()
|
|
|
|
|
|
def test_gate_passes_at_the_default_threshold_with_no_tags_yet():
|
|
# Near-zero coverage on a fresh tree is the correct reading, not a failure.
|
|
with tempfile.TemporaryDirectory() as tmp:
|
|
root = _fixture_repo(tmp, "int main() { return 0; }\n")
|
|
code, out = _run_gate(root)
|
|
assert code == 0, out
|
|
assert "0 / 3 (0.0%)" in out
|
|
|
|
|
|
def test_gate_fails_below_an_explicit_threshold():
|
|
# Proves the gate can fail at all. A gate nobody has watched fail is not
|
|
# known to work.
|
|
with tempfile.TemporaryDirectory() as tmp:
|
|
root = _fixture_repo(tmp, f"// {TAG} AR-001\nint main() {{ return 0; }}\n")
|
|
code, out = _run_gate(root, "--min-coverage", "99")
|
|
assert code == 1
|
|
assert "below the minimum" in out
|
|
|
|
|
|
def test_gate_fails_on_an_orphan_tag():
|
|
with tempfile.TemporaryDirectory() as tmp:
|
|
root = _fixture_repo(tmp, f"// {TAG} AR-001, AR-404\nint main() {{}}\n")
|
|
code, out = _run_gate(root)
|
|
assert code == 1
|
|
assert "AR-404" in out
|
|
assert "orphan tag" in out
|
|
|
|
|
|
def test_gate_can_be_asked_to_report_orphans_without_failing():
|
|
with tempfile.TemporaryDirectory() as tmp:
|
|
root = _fixture_repo(tmp, f"// {TAG} AR-001, AR-404\nint main() {{}}\n")
|
|
code, out = _run_gate(root, "--allow-orphans")
|
|
assert code == 0
|
|
assert "AR-404" in out
|
|
|
|
|
|
def test_gate_fails_when_the_register_parses_to_nothing():
|
|
# With a low threshold this self-check is what keeps the gate meaningful:
|
|
# a broken parser would otherwise report 0/0 and pass.
|
|
with tempfile.TemporaryDirectory() as tmp:
|
|
root = _fixture_repo(tmp, "int main() {}\n")
|
|
(root / "docs" / "requirements.md").write_text(
|
|
"# register\n\nNo tables here.\n", encoding="utf-8")
|
|
code, out = _run_gate(root)
|
|
assert code == 1
|
|
assert "ZERO requirements" in out
|
|
|
|
|
|
def test_gate_fails_when_no_source_files_were_scanned():
|
|
with tempfile.TemporaryDirectory() as tmp:
|
|
root = Path(tmp)
|
|
(root / "docs").mkdir(parents=True)
|
|
(root / "docs" / "requirements.md").write_text(FIXTURE_REGISTER,
|
|
encoding="utf-8")
|
|
code, out = _run_gate(root)
|
|
assert code == 1
|
|
assert "no source files were scanned" in out
|
|
|
|
|
|
def test_gate_reports_a_gpu_only_requirement_as_tagged_but_unexecuted():
|
|
with tempfile.TemporaryDirectory() as tmp:
|
|
root = _fixture_repo(tmp, f"// {TAG} AR-027\nint main() {{}}\n")
|
|
code, out = _run_gate(root)
|
|
assert code == 0
|
|
assert "TAGGED BUT UNEXECUTED" in out
|
|
assert "AR-027 (tier T4)" in out
|
|
assert "0 / 3 (0.0%)" in out
|
|
|
|
|
|
def test_gate_hard_fails_on_an_impossible_ratio():
|
|
# Coverage above 100% cannot happen through the intersection, which is the
|
|
# point: if it ever does, the arithmetic is broken and the run must not be
|
|
# reported as a pass. Forced here by handing the reporter a poisoned value.
|
|
with tempfile.TemporaryDirectory() as tmp:
|
|
root = _fixture_repo(tmp, f"// {TAG} AR-001\nint main() {{}}\n")
|
|
register = et.read_register(root / "docs" / "requirements.md")
|
|
scan = et.scan_files(et.iter_source_files(root), root)
|
|
report = et.build_report(root, register, scan)
|
|
report.coverage.percent = 158.0
|
|
text, code = et.format_coverage_report(report, 50.0)
|
|
assert code == 1
|
|
assert "exceeds 100%" in text
|
|
|
|
|
|
def test_the_per_type_breakdown_sums_to_the_headline_figure():
|
|
# A breakdown that does not add up to its own total is how a wrong number
|
|
# survives review: every row looks plausible on its own.
|
|
with tempfile.TemporaryDirectory() as tmp:
|
|
root = _fixture_repo(
|
|
tmp, f"// {TAG} AR-001, AR-027\nint main() {{}}\n")
|
|
register = et.read_register(root / "docs" / "requirements.md")
|
|
scan = et.scan_files(et.iter_source_files(root), root)
|
|
report = et.build_report(root, register, scan)
|
|
stats = et.per_type_stats(report)
|
|
assert sum(c for c, _, _ in stats.values()) == len(report.coverage.covered)
|
|
assert sum(u for _, u, _ in stats.values()) == len(report.coverage.unexecuted)
|
|
assert sum(d for _, _, d in stats.values()) == report.coverage.total
|
|
assert stats["AR"] == (1, 1, 3)
|
|
|
|
|
|
def test_json_and_markdown_outputs_are_written_and_consistent():
|
|
import json
|
|
with tempfile.TemporaryDirectory() as tmp:
|
|
root = _fixture_repo(tmp, f"// {TAG} AR-001 | SR-002\nint main() {{}}\n")
|
|
json_out = root / "traces-report.json"
|
|
md_out = root / "docs" / "traceability.md"
|
|
buffer = io.StringIO()
|
|
with redirect_stdout(buffer):
|
|
code = et.main(["--root", str(root), "--format", "coverage",
|
|
"--json-out", str(json_out),
|
|
"--markdown-out", str(md_out)])
|
|
assert code == 0
|
|
data = json.loads(json_out.read_text(encoding="utf-8"))
|
|
assert data["defined"]["total"] == 3
|
|
assert data["coverage"]["covered"] == 1
|
|
assert data["coverage"]["percent"] == round(100 / 3, 1)
|
|
assert data["byType"]["SR"] == ["SR-002"]
|
|
assert data["gpuOnlyRequirements"] == ["AR-027"]
|
|
assert "AR-001" in md_out.read_text(encoding="utf-8")
|
|
|
|
|
|
def test_system_spec_parsing_enables_orphan_checks_for_pr_and_sr():
|
|
spec = ("### SR-002 - Presence is scene-scoped\n\n"
|
|
"| ID | Goal | Why |\n|---|---|---|\n"
|
|
"| **PR-001** | Show which actors are on screen | The product |\n")
|
|
ids = et.parse_system_spec(spec)
|
|
assert ids == {"SR-002", "PR-001"}
|
|
|
|
with tempfile.TemporaryDirectory() as tmp:
|
|
root = _fixture_repo(tmp, f"// {TAG} AR-001 | SR-999\nint main() {{}}\n")
|
|
spec_path = root / "system-spec.md"
|
|
spec_path.write_text(spec, encoding="utf-8")
|
|
code, out = _run_gate(root, "--system-spec", str(spec_path))
|
|
assert "SR-999" in out
|
|
assert code == 0 # advisory: the system register is not this repo's
|
|
|
|
|
|
# --------------------------------------------------------------------------
|
|
# The live register — structural assertions only, so this does not churn as
|
|
# requirements are added.
|
|
# --------------------------------------------------------------------------
|
|
|
|
def test_the_live_register_parses_and_assigns_tiers():
|
|
register = et.read_register(et.REPO_ROOT / "docs" / "requirements.md")
|
|
assert register.total > 0
|
|
for req_type in et.LOCAL_TYPES:
|
|
assert register.count(req_type) > 0, req_type
|
|
assert sum(register.count(t) for t in et.LOCAL_TYPES) == register.total
|
|
# The GPU-less CI host must be visible in the parse, not just in prose.
|
|
assert "AR-027" in register.unexecutable_ids()
|
|
assert register.requirements["AR-027"].tiers == {"T4"}
|
|
assert register.requirements["AR-012"].tiers == {"T2"}
|
|
assert register.unexecutable_ids() < register.ids
|
|
|
|
|
|
def test_the_live_register_yields_a_gate_run_that_cannot_exceed_one_hundred():
|
|
register = et.read_register(et.REPO_ROOT / "docs" / "requirements.md")
|
|
files = et.iter_source_files(et.REPO_ROOT)
|
|
scan = et.scan_files(files, et.REPO_ROOT)
|
|
cov = et.compute_coverage(
|
|
[i for t in scan.traces for i in t.requirements], register)
|
|
assert 0.0 <= cov.percent <= 100.0
|
|
assert len(cov.covered) <= cov.total
|
|
|
|
|
|
# --------------------------------------------------------------------------
|
|
|
|
def _main() -> int:
|
|
tests = [(name, obj) for name, obj in sorted(globals().items())
|
|
if name.startswith("test_") and callable(obj)]
|
|
failed = []
|
|
for name, fn in tests:
|
|
try:
|
|
fn()
|
|
except Exception as exc: # noqa: BLE001 - a test runner reports everything
|
|
failed.append((name, exc))
|
|
print(f"FAIL {name}: {type(exc).__name__}: {exc}")
|
|
else:
|
|
print(f"ok {name}")
|
|
print(f"\n{len(tests) - len(failed)}/{len(tests)} passed")
|
|
return 1 if failed else 0
|
|
|
|
|
|
if __name__ == "__main__":
|
|
sys.exit(_main())
|