#!/usr/bin/env python3 """Tests for the traceability extractor and coverage gate. Run standalone (no third-party dependencies):: python3 scripts/traceability/test_extract_traces.py or under pytest, which discovers the same functions:: pytest scripts/traceability/test_extract_traces.py Almost every test runs over fixture strings rather than the live ``docs/requirements.md``, so their meaning does not drift as requirements are added. The two properties they exist to pin are the ones JellyTau's gate lost (see JellyTau/docs/specs/traceability-gate-repair.md): * the denominator is computed from the register at run time, so adding a requirement lowers coverage until it is traced; * the numerator is an intersection, so a tag naming an undefined ID cannot push the ratio above 100%. Plus the rule specific to this repo: a requirement only verifiable on GPU hardware is reported as tagged-but-unexecuted and never counted as covered. """ from __future__ import annotations import io import sys import tempfile from contextlib import redirect_stdout from pathlib import Path HERE = Path(__file__).resolve().parent sys.path.insert(0, str(HERE)) import extract_traces as et # noqa: E402 # The literal tag keyword is assembled at run time so that this file's fixtures # do not register as real traces when the extractor scans scripts/. TAG = "TRA" + "CES:" EXC = "EXCEP" + "TION:" # -------------------------------------------------------------------------- # Tag parsing # -------------------------------------------------------------------------- def test_parses_a_single_requirement(): groups, junk = et.parse_traces_tag(" AR-012") assert groups == [["AR-012"]] assert junk == [] def test_parses_multiple_types_separated_by_pipe(): # The house format: a pipe separates requirement *types*, a comma separates # IDs within a type. The grouping is preserved, not flattened away, so a # malformed grouping stays detectable. groups, junk = et.parse_traces_tag(" AR-012, AR-013 | SR-002") assert groups == [["AR-012", "AR-013"], ["SR-002"]] assert junk == [] def test_parses_three_groups_including_test_ids(): groups, _ = et.parse_traces_tag(" AR-012 | SR-002 | UT-003, UT-004") assert groups == [["AR-012"], ["SR-002"], ["UT-003", "UT-004"]] def test_strips_a_trailing_block_comment_terminator(): groups, junk = et.parse_traces_tag(" AR-001 */") assert groups == [["AR-001"]] assert junk == [] def test_does_not_harvest_ids_out_of_prose_after_the_tag(): # A trailing sentence must not smuggle IDs into the trace set: AR-999 here # is discussion, not a claim that this code satisfies AR-999. groups, junk = et.parse_traces_tag(" AR-001 - see also AR-999 in the notes") assert groups == [["AR-001"]] assert junk and "AR-999" in junk[0] def test_ignores_a_tag_with_no_ids_at_all(): groups, _ = et.parse_traces_tag(" see the register") assert groups == [] def test_two_digit_id_is_not_accepted_as_a_requirement(): # AR-12 is a typo for AR-012; silently accepting it would create a # phantom requirement. groups, junk = et.parse_traces_tag(" AR-12") assert groups == [] assert junk == ["AR-12"] # -------------------------------------------------------------------------- # Scanning C++ and Python sources # -------------------------------------------------------------------------- def _write_tree(root: Path) -> None: (root / "src").mkdir(parents=True, exist_ok=True) (root / "scripts").mkdir(parents=True, exist_ok=True) (root / "external").mkdir(parents=True, exist_ok=True) (root / "src" / "tracker.hpp").write_text( "#pragma once\n" f"/// {TAG} AR-012, AR-013 | SR-002\n" "struct TrackRegistry {\n" " void close_all();\n" "};\n", encoding="utf-8") (root / "scripts" / "gallery.py").write_text( "def build_gallery(cast):\n" ' """Build a gallery from Jellyfin plus TMDB fallback.\n' "\n" f" {TAG} GR-001 | SR-005\n" ' """\n' " return {}\n", encoding="utf-8") (root / "external" / "vendored.cpp").write_text( f"// {TAG} AR-001\n", encoding="utf-8") (root / "src" / "notes.txt").write_text( f"// {TAG} AR-002\n", encoding="utf-8") def test_scans_cpp_and_python_but_not_vendored_or_non_source(): with tempfile.TemporaryDirectory() as tmp: root = Path(tmp) _write_tree(root) files = et.iter_source_files(root) names = sorted(f.name for f in files) assert names == ["gallery.py", "tracker.hpp"], names scan = et.scan_files(files, root) traced = sorted({i for t in scan.traces for i in t.requirements}) assert traced == ["AR-012", "AR-013", "GR-001", "SR-002", "SR-005"] def test_context_is_found_below_a_cpp_tag_and_above_a_python_tag(): # The two languages put the tag on opposite sides of what it describes. with tempfile.TemporaryDirectory() as tmp: root = Path(tmp) _write_tree(root) scan = et.scan_files(et.iter_source_files(root), root) contexts = {t.file: t.context for t in scan.traces} assert "TrackRegistry" in contexts["src/tracker.hpp"] assert "build_gallery" in contexts["scripts/gallery.py"] # -------------------------------------------------------------------------- # EXCEPTION tags # -------------------------------------------------------------------------- EXCEPTION_SOURCE = ( "float outlier_score(const Refs& refs) {\n" f" // {EXC} AR-024 distributional check on an actor's own references,\n" " return spread(refs);\n" "}\n" ) def test_exception_tag_is_captured_with_its_reason(): with tempfile.TemporaryDirectory() as tmp: root = Path(tmp) (root / "src").mkdir(parents=True) (root / "src" / "outlier.cpp").write_text(EXCEPTION_SOURCE, encoding="utf-8") scan = et.scan_files(et.iter_source_files(root), root) assert len(scan.exceptions) == 1 exc = scan.exceptions[0] assert exc.requirement == "AR-024" assert exc.reason.startswith("distributional check") assert exc.line == 2 def test_exception_is_never_counted_as_coverage(): # An exception is a recorded decision to depart from an invariant. Counting # it as evidence the requirement is met inverts its meaning. with tempfile.TemporaryDirectory() as tmp: root = Path(tmp) (root / "src").mkdir(parents=True) (root / "src" / "outlier.cpp").write_text(EXCEPTION_SOURCE, encoding="utf-8") scan = et.scan_files(et.iter_source_files(root), root) assert scan.traces == [] register = et.parse_register( "| ID | Requirement | Status |\n|---|---|---|\n" "| AR-024 | Always the calibrated probability | Planned |\n") cov = et.compute_coverage( [i for t in scan.traces for i in t.requirements], register) assert cov.covered == [] assert cov.percent == 0.0 def test_exception_without_a_reason_is_reported(): # CLAUDE.md: an exception is only *agreed* if the reason is recorded. with tempfile.TemporaryDirectory() as tmp: root = Path(tmp) (root / "src").mkdir(parents=True) (root / "src" / "bare.cpp").write_text( f"// {EXC} AR-024\n", encoding="utf-8") scan = et.scan_files(et.iter_source_files(root), root) assert len(scan.exceptions) == 1 assert scan.diagnostics.exceptions_without_reason def test_mixed_type_group_is_reported(): # `AR-001, SR-002` in one group misuses the comma; the pipe is what # separates types, so the tag does not say what it appears to say. with tempfile.TemporaryDirectory() as tmp: root = Path(tmp) (root / "src").mkdir(parents=True) (root / "src" / "a.cpp").write_text( f"// {TAG} AR-001, SR-002\n", encoding="utf-8") scan = et.scan_files(et.iter_source_files(root), root) assert scan.diagnostics.mixed_type_groups # -------------------------------------------------------------------------- # The register: denominators from requirements.md # -------------------------------------------------------------------------- REGISTER_HEADER = "| ID | Requirement | Traces to | Priority | Status |\n|---|---|---|---|---|\n" def test_counts_a_well_formed_table_row_as_a_defined_requirement(): md = REGISTER_HEADER + ( "| AR-001 | Detect faces in sampled frames | SR-002 | High | Done |\n" "| AR-002 | Minimum face size 66x66 px | SR-002 | High | Planned |\n") register = et.parse_register(md) assert register.count("AR") == 2 assert register.count("GR") == 0 assert register.total == 2 def test_does_not_count_ids_that_appear_only_in_the_traces_to_column(): # The bug this parse rule avoids: a naive scan for AR-\d{3} over the whole # file counts every reference as a definition and inflates the denominator. md = REGISTER_HEADER + ( "| GR-001 | Build gallery from library cast | SR-001, SR-005 | High | Done |\n" "| GR-002 | Incremental merge refresh | PR-003 | High | Done |\n") register = et.parse_register(md) assert register.count("GR") == 2 assert register.ids == {"GR-001", "GR-002"} def test_does_not_count_ids_mentioned_in_prose(): md = ("Some prose explaining that AR-005 relates to GR-001 and VR-003.\n\n" + REGISTER_HEADER + "| AR-005 | Align to 112x112 | SR-002 | High | Done |\n") register = et.parse_register(md) assert register.ids == {"AR-005"} def test_does_not_count_the_verification_plan_table_as_definitions(): # The per-requirement verification plan is also keyed on `ID`, but it # assigns tiers rather than defining requirements. Counting its rows would # double the denominator for every requirement that has a plan entry. md = (REGISTER_HEADER + "| AR-001 | Detect faces | SR-002 | High | Done |\n" + "\n" + "| ID | Tier | Test asserts | Edge cases to cover |\n|---|---|---|---|\n" + "| AR-001 | T3 | Detector returns plausible boxes | smoke only |\n" + "| AR-099 | T1 | Something not in the register | - |\n") register = et.parse_register(md) assert register.ids == {"AR-001"} assert register.total == 1 def test_deduplicates_an_id_listed_in_two_definition_tables(): md = (REGISTER_HEADER + "| AR-001 | Detect faces | SR-002 | High | Done |\n" + "\n" + REGISTER_HEADER + "| AR-001 | Detect faces | SR-002 | High | Done |\n") register = et.parse_register(md) assert register.total == 1 def test_withdrawn_requirements_leave_the_denominator(): # IDs are permanent, but a withdrawn requirement can never be implemented. # Leaving it in the denominator would depress coverage forever. md = REGISTER_HEADER + ( "| AR-001 | Detect faces | SR-002 | High | Done |\n" "| AR-002 | Superseded mechanism | SR-002 | High | Withdrawn |\n") register = et.parse_register(md) assert register.ids == {"AR-001"} assert "AR-002" in register.withdrawn def test_the_denominator_is_live_adding_a_row_lowers_coverage(): # The property JellyTau's frozen literals destroyed. Same traced set, one # more requirement defined => a lower percentage, mechanically. base = REGISTER_HEADER + "| AR-001 | A | SR-002 | High | Done |\n" grown = base + "| AR-002 | B | SR-002 | High | Planned |\n" before = et.compute_coverage(["AR-001"], et.parse_register(base)) after = et.compute_coverage(["AR-001"], et.parse_register(grown)) assert before.percent == 100.0 assert after.percent == 50.0 assert after.total == 2 def test_register_captures_the_row_fields_not_just_the_id(): md = REGISTER_HEADER + ( "| AR-012 | Presence follows track extent | **SR-002** | High | Planned |\n") req = et.parse_register(md).requirements["AR-012"] assert req.text == "Presence follows track extent" assert req.traces_to == "**SR-002**" assert req.status == "Planned" # -------------------------------------------------------------------------- # Verification tiers and the GPU-less CI host # -------------------------------------------------------------------------- TIER_REGISTER = REGISTER_HEADER + "".join( f"| AR-{n:03d} | Requirement {n} | SR-002 | High | Planned |\n" for n in range(1, 10)) + "".join( f"| VR-{n:03d} | Study {n} | PR-002 | Medium | Planned |\n" for n in range(1, 4)) def _tier_table(rows: str) -> str: return "| Requirement | Tier | Note |\n|---|---|---|\n" + rows def test_tier_assignment_handles_lists_ranges_and_wildcards(): md = TIER_REGISTER + "\n" + _tier_table( "| AR-001, AR-005 | T3 | smoke |\n" "| AR-002 … AR-004 | **T2** | replay |\n" "| AR-006 | T1 + T4 | mixed |\n" "| VR-* | Out of CI | studies |\n") register = et.parse_register(md) assert register.requirements["AR-001"].tiers == {"T3"} assert register.requirements["AR-003"].tiers == {"T2"} assert register.requirements["AR-006"].tiers == {"T1", "T4"} assert register.requirements["VR-002"].tiers == {"out-of-ci"} def test_a_range_cannot_invent_a_requirement_the_register_lacks(): md = TIER_REGISTER + "\n" + _tier_table("| AR-001 … AR-050 | T2 | wide |\n") register = et.parse_register(md) assert register.total == 12 assert "AR-050" not in register.ids def test_slash_shorthand_in_the_verification_plan_expands(): md = TIER_REGISTER + "\n" + ( "| ID | Tier | Test asserts | Edge cases |\n|---|---|---|---|\n" "| AR-009/008 | T2 | Cut shifts weighting | cut with same people |\n") register = et.parse_register(md) assert register.requirements["AR-008"].tiers == {"T2"} assert register.requirements["AR-009"].tiers == {"T2"} def test_tiers_from_both_tables_are_unioned_not_overwritten(): # The summary table says AR-006 is T4; the per-requirement plan adds a T1 # equivalence check. The T1 part does run in CI, so the requirement is # executable and must not be written off as GPU-only. md = TIER_REGISTER + "\n" + _tier_table("| AR-006 | T4 | GPU host only |\n") + "\n" + ( "| ID | Tier | Test asserts | Edge cases |\n|---|---|---|---|\n" "| AR-006 | T1 + T4 | GEMM equals reference loop | small input in CI |\n") register = et.parse_register(md) assert register.requirements["AR-006"].tiers == {"T1", "T4"} assert register.requirements["AR-006"].ci_executable def test_a_t4_only_requirement_is_not_ci_executable(): md = TIER_REGISTER + "\n" + _tier_table("| AR-007 | **T4** | GPU only |\n") register = et.parse_register(md) assert not register.requirements["AR-007"].ci_executable assert register.unexecutable_ids() == {"AR-007"} def test_a_requirement_tracing_up_to_nothing_is_reported(): # SPEC.md section 6: a requirement citing no parent is scope creep, and it # is invisible unless something looks. A section reference counts as a # parent - what matters is that something was recorded. md = REGISTER_HEADER + ( "| AR-001 | Has a parent | SR-002 | High | Done |\n" "| AR-002 | Parent is a section | §4 | Medium | Planned |\n" "| AR-003 | Serves nothing stated | - | Low | Planned |\n" "| AR-004 | Blank cell | | Low | Planned |\n") register = et.parse_register(md) assert register.parentless_ids() == {"AR-003", "AR-004"} def test_a_requirement_with_no_tier_is_unknown_not_unexecutable(): register = et.parse_register(TIER_REGISTER) assert register.tier_unknown_ids() == register.ids assert register.unexecutable_ids() == set() # -------------------------------------------------------------------------- # Coverage arithmetic # -------------------------------------------------------------------------- COVERAGE_REGISTER = et.parse_register( REGISTER_HEADER + "| AR-001 | A | SR-002 | High | Done |\n" + "| AR-002 | B | SR-002 | High | Done |\n" + "| GR-001 | C | SR-005 | High | Done |\n" + "| AR-027 | Arbitrary gallery scale | SR-001 | High | Planned |\n" + "\n" + _tier_table("| AR-001, AR-002 | T2 | replay |\n" "| GR-001 | T1 | bookkeeping |\n" "| AR-027 | **T4** | GPU host only |\n")) def test_coverage_is_the_intersection_of_traced_and_defined(): cov = et.compute_coverage(["AR-001", "GR-001"], COVERAGE_REGISTER) assert cov.covered == ["AR-001", "GR-001"] assert cov.total == 4 assert cov.percent == 50.0 def test_a_traced_but_undefined_id_cannot_inflate_the_numerator(): # This is exactly how a ratio exceeds 100%: a tag naming a renumbered or # mistyped requirement counted as covered. cov = et.compute_coverage(["AR-001", "GR-001", "AR-097"], COVERAGE_REGISTER) assert cov.covered == ["AR-001", "GR-001"] assert cov.percent == 50.0 def test_orphan_tags_are_reported_so_they_get_fixed(): cov = et.compute_coverage(["AR-001", "AR-097", "GR-404"], COVERAGE_REGISTER) assert cov.orphaned == ["AR-097", "GR-404"] def test_no_orphans_when_every_traced_id_is_defined(): cov = et.compute_coverage(["AR-001", "AR-002"], COVERAGE_REGISTER) assert cov.orphaned == [] def test_test_and_system_ids_are_a_separate_taxonomy(): # UT/IT live in their own register section; PR/SR live in the umbrella # SPEC.md, which is not part of this repo's checkout. Neither counts toward # coverage, and flagging them as orphans would bury real typos in noise. cov = et.compute_coverage( ["AR-001", "UT-003", "IT-007", "SR-002", "PR-001"], COVERAGE_REGISTER) assert cov.orphaned == [] assert cov.covered == ["AR-001"] def test_a_gpu_only_requirement_is_tagged_but_unexecuted_not_covered(): # The rule specific to this repo: CI is an Intel N100 with no dGPU. A test # that exists but can never run is not evidence, and counting it is the # same failure mode as the 158% bug. cov = et.compute_coverage(["AR-001", "AR-027"], COVERAGE_REGISTER) assert cov.unexecuted == ["AR-027"] assert cov.covered == ["AR-001"] assert cov.percent == 25.0 assert cov.orphaned == [] def test_ci_scope_percentage_excludes_unexecutable_requirements_from_both_sides(): cov = et.compute_coverage(["AR-001"], COVERAGE_REGISTER) assert cov.ci_executable_total == 3 assert round(cov.ci_percent) == 33 def test_tracing_only_gpu_only_requirements_yields_zero_coverage(): cov = et.compute_coverage(["AR-027"], COVERAGE_REGISTER) assert cov.covered == [] assert cov.percent == 0.0 assert cov.unexecuted == ["AR-027"] def test_duplicate_traced_ids_are_counted_once(): cov = et.compute_coverage(["AR-001", "AR-001", "AR-001"], COVERAGE_REGISTER) assert cov.covered == ["AR-001"] def test_an_empty_trace_set_is_zero_percent_not_a_divide_by_zero(): cov = et.compute_coverage([], COVERAGE_REGISTER) assert cov.covered == [] assert cov.percent == 0.0 def test_an_empty_register_reports_zero_rather_than_nan(): cov = et.compute_coverage(["AR-001"], et.Register()) assert cov.percent == 0.0 assert cov.ci_percent == 0.0 def test_full_coverage_reports_exactly_one_hundred_never_above(): cov = et.compute_coverage( ["AR-001", "AR-002", "GR-001", "AR-027"], COVERAGE_REGISTER) # AR-027 is GPU-only, so the honest ceiling here is 3/4. assert cov.percent == 75.0 assert cov.ci_percent == 100.0 assert cov.ci_percent <= 100.0 # -------------------------------------------------------------------------- # The gate: end to end, including the ways it must fail # -------------------------------------------------------------------------- FIXTURE_REGISTER = ( "# register\n\n" + REGISTER_HEADER + "| AR-001 | Detect faces | SR-002 | High | Done |\n" + "| AR-002 | Minimum face size | SR-002 | High | Planned |\n" + "| AR-027 | Arbitrary gallery scale | SR-001 | High | Planned |\n" + "\n" + _tier_table("| AR-001, AR-002 | T2 | replay |\n" "| AR-027 | **T4** | GPU host only |\n")) def _fixture_repo(tmp: str, source: str) -> Path: root = Path(tmp) (root / "docs").mkdir(parents=True, exist_ok=True) (root / "src").mkdir(parents=True, exist_ok=True) (root / "docs" / "requirements.md").write_text(FIXTURE_REGISTER, encoding="utf-8") (root / "src" / "pipeline.cpp").write_text(source, encoding="utf-8") return root def _run_gate(root: Path, *extra: str): buffer = io.StringIO() with redirect_stdout(buffer): code = et.main(["--root", str(root), "--format", "coverage", *extra]) return code, buffer.getvalue() def test_gate_passes_at_the_default_threshold_with_no_tags_yet(): # Near-zero coverage on a fresh tree is the correct reading, not a failure. with tempfile.TemporaryDirectory() as tmp: root = _fixture_repo(tmp, "int main() { return 0; }\n") code, out = _run_gate(root) assert code == 0, out assert "0 / 3 (0.0%)" in out def test_gate_fails_below_an_explicit_threshold(): # Proves the gate can fail at all. A gate nobody has watched fail is not # known to work. with tempfile.TemporaryDirectory() as tmp: root = _fixture_repo(tmp, f"// {TAG} AR-001\nint main() {{ return 0; }}\n") code, out = _run_gate(root, "--min-coverage", "99") assert code == 1 assert "below the minimum" in out def test_gate_fails_on_an_orphan_tag(): with tempfile.TemporaryDirectory() as tmp: root = _fixture_repo(tmp, f"// {TAG} AR-001, AR-404\nint main() {{}}\n") code, out = _run_gate(root) assert code == 1 assert "AR-404" in out assert "orphan tag" in out def test_gate_can_be_asked_to_report_orphans_without_failing(): with tempfile.TemporaryDirectory() as tmp: root = _fixture_repo(tmp, f"// {TAG} AR-001, AR-404\nint main() {{}}\n") code, out = _run_gate(root, "--allow-orphans") assert code == 0 assert "AR-404" in out def test_gate_fails_when_the_register_parses_to_nothing(): # With a low threshold this self-check is what keeps the gate meaningful: # a broken parser would otherwise report 0/0 and pass. with tempfile.TemporaryDirectory() as tmp: root = _fixture_repo(tmp, "int main() {}\n") (root / "docs" / "requirements.md").write_text( "# register\n\nNo tables here.\n", encoding="utf-8") code, out = _run_gate(root) assert code == 1 assert "ZERO requirements" in out def test_gate_fails_when_no_source_files_were_scanned(): with tempfile.TemporaryDirectory() as tmp: root = Path(tmp) (root / "docs").mkdir(parents=True) (root / "docs" / "requirements.md").write_text(FIXTURE_REGISTER, encoding="utf-8") code, out = _run_gate(root) assert code == 1 assert "no source files were scanned" in out def test_gate_reports_a_gpu_only_requirement_as_tagged_but_unexecuted(): with tempfile.TemporaryDirectory() as tmp: root = _fixture_repo(tmp, f"// {TAG} AR-027\nint main() {{}}\n") code, out = _run_gate(root) assert code == 0 assert "TAGGED BUT UNEXECUTED" in out assert "AR-027 (tier T4)" in out assert "0 / 3 (0.0%)" in out def test_gate_hard_fails_on_an_impossible_ratio(): # Coverage above 100% cannot happen through the intersection, which is the # point: if it ever does, the arithmetic is broken and the run must not be # reported as a pass. Forced here by handing the reporter a poisoned value. with tempfile.TemporaryDirectory() as tmp: root = _fixture_repo(tmp, f"// {TAG} AR-001\nint main() {{}}\n") register = et.read_register(root / "docs" / "requirements.md") scan = et.scan_files(et.iter_source_files(root), root) report = et.build_report(root, register, scan) report.coverage.percent = 158.0 text, code = et.format_coverage_report(report, 50.0) assert code == 1 assert "exceeds 100%" in text def test_the_per_type_breakdown_sums_to_the_headline_figure(): # A breakdown that does not add up to its own total is how a wrong number # survives review: every row looks plausible on its own. with tempfile.TemporaryDirectory() as tmp: root = _fixture_repo( tmp, f"// {TAG} AR-001, AR-027\nint main() {{}}\n") register = et.read_register(root / "docs" / "requirements.md") scan = et.scan_files(et.iter_source_files(root), root) report = et.build_report(root, register, scan) stats = et.per_type_stats(report) assert sum(c for c, _, _ in stats.values()) == len(report.coverage.covered) assert sum(u for _, u, _ in stats.values()) == len(report.coverage.unexecuted) assert sum(d for _, _, d in stats.values()) == report.coverage.total assert stats["AR"] == (1, 1, 3) def test_json_and_markdown_outputs_are_written_and_consistent(): import json with tempfile.TemporaryDirectory() as tmp: root = _fixture_repo(tmp, f"// {TAG} AR-001 | SR-002\nint main() {{}}\n") json_out = root / "traces-report.json" md_out = root / "docs" / "traceability.md" buffer = io.StringIO() with redirect_stdout(buffer): code = et.main(["--root", str(root), "--format", "coverage", "--json-out", str(json_out), "--markdown-out", str(md_out)]) assert code == 0 data = json.loads(json_out.read_text(encoding="utf-8")) assert data["defined"]["total"] == 3 assert data["coverage"]["covered"] == 1 assert data["coverage"]["percent"] == round(100 / 3, 1) assert data["byType"]["SR"] == ["SR-002"] assert data["gpuOnlyRequirements"] == ["AR-027"] assert "AR-001" in md_out.read_text(encoding="utf-8") def test_system_spec_parsing_enables_orphan_checks_for_pr_and_sr(): spec = ("### SR-002 - Presence is scene-scoped\n\n" "| ID | Goal | Why |\n|---|---|---|\n" "| **PR-001** | Show which actors are on screen | The product |\n") ids = et.parse_system_spec(spec) assert ids == {"SR-002", "PR-001"} with tempfile.TemporaryDirectory() as tmp: root = _fixture_repo(tmp, f"// {TAG} AR-001 | SR-999\nint main() {{}}\n") spec_path = root / "system-spec.md" spec_path.write_text(spec, encoding="utf-8") code, out = _run_gate(root, "--system-spec", str(spec_path)) assert "SR-999" in out assert code == 0 # advisory: the system register is not this repo's # -------------------------------------------------------------------------- # The live register — structural assertions only, so this does not churn as # requirements are added. # -------------------------------------------------------------------------- def test_the_live_register_parses_and_assigns_tiers(): register = et.read_register(et.REPO_ROOT / "docs" / "requirements.md") assert register.total > 0 for req_type in et.LOCAL_TYPES: assert register.count(req_type) > 0, req_type assert sum(register.count(t) for t in et.LOCAL_TYPES) == register.total # The GPU-less CI host must be visible in the parse, not just in prose. assert "AR-027" in register.unexecutable_ids() assert register.requirements["AR-027"].tiers == {"T4"} assert register.requirements["AR-012"].tiers == {"T2"} assert register.unexecutable_ids() < register.ids def test_the_live_register_yields_a_gate_run_that_cannot_exceed_one_hundred(): register = et.read_register(et.REPO_ROOT / "docs" / "requirements.md") files = et.iter_source_files(et.REPO_ROOT) scan = et.scan_files(files, et.REPO_ROOT) cov = et.compute_coverage( [i for t in scan.traces for i in t.requirements], register) assert 0.0 <= cov.percent <= 100.0 assert len(cov.covered) <= cov.total # -------------------------------------------------------------------------- def _main() -> int: tests = [(name, obj) for name, obj in sorted(globals().items()) if name.startswith("test_") and callable(obj)] failed = [] for name, fn in tests: try: fn() except Exception as exc: # noqa: BLE001 - a test runner reports everything failed.append((name, exc)) print(f"FAIL {name}: {type(exc).__name__}: {exc}") else: print(f"ok {name}") print(f"\n{len(tests) - len(failed)}/{len(tests)} passed") return 1 if failed else 0 if __name__ == "__main__": sys.exit(_main())