Shared traceability tooling, generalised for every component
Moves the extractor, its tests and the gate from scene-actor-extraction into the project home, so all three components run one implementation rather than each growing its own. The gate logic is unchanged; what changes is that the two repo-specific constants become arguments. Both were hard blockers rather than inconveniences. LOCAL_TYPES was fixed at the extraction set, so a register using UR/DR parsed to zero requirements; SOURCE_SUFFIXES covered C++/Python only, so a Rust or C# tree scanned zero files. The gate correctly refuses to report coverage in either state, which is how both surfaced. They are now --types, --suffixes and --scan-roots, with the extraction defaults preserved: that repo runs unchanged with no flags. traceability-gate.sh gains REPO_ROOT, REQUIREMENTS, TYPES, SUFFIXES and SCAN_ROOTS environment overrides. Its REPO_ROOT default of SCRIPT_DIR/../.. is correct when the tooling sits in the repo it checks, but resolves to the submodule itself once vendored, so a consuming repo must set it. Fixes a latent bug found while testing the override: iter_source_files bound SCAN_ROOTS as a default argument, evaluated at import, so configure_scan_roots could never affect it. 58 tests pass. The two that read a live register now take LIVE_REGISTER from the environment and skip without it — they asserted against AR-012/AR-027, which belong to scene-actor-extraction rather than to a shared tool. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
Claude Opus 5
@@ -0,0 +1,107 @@
|
||||
{
|
||||
"permissions": {
|
||||
"allow": [
|
||||
"Bash(python3 -c ' *)",
|
||||
"WebFetch(domain:docs.turso.tech)",
|
||||
"Bash(awk '/^```/{n++} END{print \"fences:\",n,\\(n%2==0?\"balanced\":\"UNBALANCED\"\\)}' SPEC.md)",
|
||||
"Bash(python3 -c \"import h5py; print\\('h5py', h5py.__version__\\)\")",
|
||||
"Bash(python3 scripts/make_jellyfin_gallery.py --help)",
|
||||
"Bash(timeout 30 python3 -c ' *)",
|
||||
"Bash(ps -o pid,etime,cmd -C cmake)",
|
||||
"Bash(cargo check *)",
|
||||
"Bash(cargo test *)",
|
||||
"Bash(cargo clippy *)",
|
||||
"Bash(cargo deny *)",
|
||||
"Bash(rustc --version)",
|
||||
"Bash(cargo metadata *)",
|
||||
"Bash(python3 -c \"import json,sys; d=json.load\\(sys.stdin\\); print\\(d['packages'][0].get\\('license'\\)\\)\")",
|
||||
"Bash(timeout 900 cargo install cargo-deny --locked)",
|
||||
"Bash(timeout 1500 docker build -t jray-server:test .)",
|
||||
"Bash(timeout 900 docker build -f /tmp/claude-1000/-home-dtourolle-Development-Jray-project/d861f764-2add-4f42-a069-0954ad1f9565/scratchpad/Dockerfile.probe -t jray-probe .)",
|
||||
"Bash(docker rm *)",
|
||||
"Bash(docker volume *)",
|
||||
"Bash(timeout 900 docker build -t jray-server:test .)",
|
||||
"Bash(docker run *)",
|
||||
"Bash(xargs -I{} echo \"clippy: {}\")",
|
||||
"Bash(timeout 900 docker build -q -t jray-server:test .)",
|
||||
"Bash(awk '{s+=$4} END {print \"total: \" s \" passing\"}')",
|
||||
"Bash(xargs -I{} echo \"clippy issues: {}\")",
|
||||
"Bash(timeout 600 dotnet build Jellyfin.Plugin.JRay/Jellyfin.Plugin.JRay.csproj -v q --nologo)",
|
||||
"Bash(awk '{s+=$4} END {print \"server tests: \" s \" passing\"}')",
|
||||
"Bash(timeout 300 dotnet build Jellyfin.Plugin.JRay/Jellyfin.Plugin.JRay.csproj -v q --nologo)",
|
||||
"Bash(cp SPEC.md /tmp/claude-1000/-home-dtourolle-Development-Jray-project/de7e497c-da04-462d-a971-d0fcd36bda2a/scratchpad/SPEC.md.bak)",
|
||||
"Bash(perl -pi -e ' *)",
|
||||
"Bash(grep -vE \"AR-|DP-|IR-|GR-|VR-|SR-|PR-|H[0-9]|[0-9]{3,}\")",
|
||||
"Bash(git update-index *)",
|
||||
"WebFetch(domain:github.com)",
|
||||
"Bash(git -C /home/dtourolle/Development/Jray-worktrees/audio-signature status --short)",
|
||||
"Bash(git -C /home/dtourolle/Development/Jray-worktrees/audio-signature branch --show-current)",
|
||||
"Bash(python3 -c \"import pytest; print\\(pytest.__version__\\)\")",
|
||||
"Bash(python3 -c \"import numpy; print\\('numpy', numpy.__version__\\)\")",
|
||||
"Read(//usr/lib/cmake/**)",
|
||||
"Read(//usr/include/**)",
|
||||
"Bash(grep -n \"^#\\\\{1,3\\\\} \" JRay-public-server/SPEC.md)",
|
||||
"Bash(grep -n \"^#\\\\{1,3\\\\} \" scene-actor-extraction/docs/SPEC.md)",
|
||||
"Bash(cargo build *)",
|
||||
"Bash(python3 gen.py tone.wav)",
|
||||
"Bash(python3 ref.py tone.wav)",
|
||||
"Bash(python3 -c \"import json;d=json.load\\(open\\('package.json'\\)\\);print\\(json.dumps\\({k:v for k,v in d.get\\('scripts',{}\\).items\\(\\) if 'trace' in k.lower\\(\\)},indent=2\\)\\)\")",
|
||||
"Bash(python3 scripts/traceability/extract_traces.py --format json)",
|
||||
"Bash(python3 scripts/test_extract_traces.py)",
|
||||
"Bash(python3 -m pytest scripts/traceability/test_extract_traces.py -q)",
|
||||
"Bash(python3 scripts/traceability/extract_traces.py --format coverage)",
|
||||
"Bash(python3 scripts/extract_traces.py)",
|
||||
"Bash(python3 scripts/extract_traces.py --format json)",
|
||||
"Bash(git -C /home/dtourolle/Development/Jray-worktrees/traceability-tooling log --oneline -3)",
|
||||
"Bash(python3 /home/dtourolle/Development/Jray-worktrees/traceability-tooling/scripts/traceability/extract_traces.py --root /home/dtourolle/Development/Jray-project/JRay-public-server --requirements /home/dtourolle/Development/Jray-project/JRay-public-server/docs/requirements.md --system-spec /home/dtourolle/Development/Jray-project/SPEC.md --format coverage)",
|
||||
"Bash(python3 scripts/traceability/test_extract_traces.py)",
|
||||
"Bash(sh scripts/traceability/traceability-gate.sh)",
|
||||
"Bash(python3 -c \"import pyflakes; print\\('pyflakes', pyflakes.__version__\\)\")",
|
||||
"Bash(timeout 300 dotnet build -v q --nologo)",
|
||||
"Bash(git check-ignore *)",
|
||||
"Bash(rm -rf /tmp/claude-1000/-home-dtourolle-Development-Jray-project/de7e497c-da04-462d-a971-d0fcd36bda2a/scratchpad/build-full)",
|
||||
"Bash(cmake -S . -B /tmp/claude-1000/-home-dtourolle-Development-Jray-project/de7e497c-da04-462d-a971-d0fcd36bda2a/scratchpad/build-full -DSAE_BUILD_TESTS=ON)",
|
||||
"Bash(pkg-config --cflags opencv4)",
|
||||
"Bash(pkg-config --cflags opencv)",
|
||||
"Bash(awk '{s+=$4} END {print \"tests: \" s}')",
|
||||
"Bash(python3 -m pyflakes scripts/validation/min_face_size.py)",
|
||||
"Bash(python3 -m flake8 --select=F scripts/validation/min_face_size.py)",
|
||||
"Bash(git -c user.name=\"Duncan Tourolle\" -c user.email=\"duncan@tourolle.paris\" commit -q -F -)",
|
||||
"Bash(cmake -S . -B /tmp/claude-1000/-home-dtourolle-Development-Jray-project/de7e497c-da04-462d-a971-d0fcd36bda2a/scratchpad/build-full -DSAE_BUILD_TESTS=ON -DSAE_GEMM_BACKEND=CPU)",
|
||||
"Bash(python3 -c \"import h5py, numpy, requests, PIL; print\\('deps ok'\\)\")",
|
||||
"Bash(python3 -m py_compile scripts/validation/min_face_size.py)",
|
||||
"Bash(git remote *)",
|
||||
"Bash(python3 et.py --root /home/dtourolle/Development/Jray-project/jRay --requirements /home/dtourolle/Development/Jray-project/jRay/docs/requirements.md --system-spec /home/dtourolle/Development/Jray-project/SPEC.md --format coverage)",
|
||||
"Bash(git -C /home/dtourolle/Development/Jray-worktrees/traceability-tooling status --short)",
|
||||
"Bash(git -C /home/dtourolle/Development/Jray-worktrees/traceability-tooling log --oneline -2 -- scripts/traceability/)",
|
||||
"Bash(timeout 240 python3 scripts/stamp_gallery.py --gallery /tmp/claude-1000/-home-dtourolle-Development-Jray-project/de7e497c-da04-462d-a971-d0fcd36bda2a/scratchpad/g_plain.h5 --show)",
|
||||
"Bash(cp /tmp/claude-1000/-home-dtourolle-Development-Jray-project/de7e497c-da04-462d-a971-d0fcd36bda2a/scratchpad/g_plain.h5 /tmp/claude-1000/-home-dtourolle-Development-Jray-project/de7e497c-da04-462d-a971-d0fcd36bda2a/scratchpad/g_mig.h5)",
|
||||
"Bash(python3 scripts/stamp_gallery.py --gallery /tmp/claude-1000/-home-dtourolle-Development-Jray-project/de7e497c-da04-462d-a971-d0fcd36bda2a/scratchpad/g_mig.h5 --arcface /tmp/claude-1000/-home-dtourolle-Development-Jray-project/de7e497c-da04-462d-a971-d0fcd36bda2a/scratchpad/fake.onnx)",
|
||||
"Bash(timeout 590 cmake -S . -B /tmp/claude-1000/-home-dtourolle-Development-Jray-project/de7e497c-da04-462d-a971-d0fcd36bda2a/scratchpad/bt -DSAE_BUILD_TESTS=ON -DCMAKE_BUILD_TYPE=Release)",
|
||||
"Bash(grep -n \"^#\\\\{1,3\\\\} \\\\|^| ID \\\\|^| Requirement \\\\|^|---\" jRay/docs/requirements.md)",
|
||||
"Bash(./build-full/tests/sae_tests)",
|
||||
"Bash(ls /usr/include/catch2/catch_test_macros.hpp 2>/dev/null && echo \"catch2 system\"; ls /tmp/claude-1000/-home-dtourolle-Development-Jray-project/de7e497c-da04-462d-a971-d0fcd36bda2a/scratchpad/build-full/_deps/ 2>/dev/null | head; ls /usr/lib/libCatch2* 2>/dev/null | head)",
|
||||
"Read(//usr/lib/**)",
|
||||
"Bash(find /tmp/claude-1000/-home-dtourolle-Development-Jray-project/de7e497c-da04-462d-a971-d0fcd36bda2a/scratchpad/build-full/_deps/catch2-build -name libCatch2*)",
|
||||
"Bash(timeout 590 g++ -std=c++20 -O1 -I/home/dtourolle/Development/Jray-worktrees/gallery-model-binding/src -I/usr/include/opencv5 -I/tmp/claude-1000/-home-dtourolle-Development-Jray-project/de7e497c-da04-462d-a971-d0fcd36bda2a/scratchpad/build-full/_deps/nlohmann_json-src/single_include -I/tmp/claude-1000/-home-dtourolle-Development-Jray-project/de7e497c-da04-462d-a971-d0fcd36bda2a/scratchpad/build-full/_deps/catch2-src/src -I/tmp/claude-1000/-home-dtourolle-Development-Jray-project/de7e497c-da04-462d-a971-d0fcd36bda2a/scratchpad/build-full/_deps/catch2-build/generated-includes '-DSAE_MODELS_DIR=\"/home/dtourolle/Development/Jray-worktrees/gallery-model-binding/models\"' -o gallery_tests /home/dtourolle/Development/Jray-worktrees/gallery-model-binding/tests/test_gallery_store.cpp /home/dtourolle/Development/Jray-worktrees/gallery-model-binding/src/gallery/gallery_store.cpp /home/dtourolle/Development/Jray-worktrees/gallery-model-binding/src/gallery/embedder_stamp.cpp /tmp/claude-1000/-home-dtourolle-Development-Jray-project/de7e497c-da04-462d-a971-d0fcd36bda2a/scratchpad/build-full/_deps/catch2-build/src/libCatch2Main.a /tmp/claude-1000/-home-dtourolle-Development-Jray-project/de7e497c-da04-462d-a971-d0fcd36bda2a/scratchpad/build-full/_deps/catch2-build/src/libCatch2.a -lhdf5_cpp -lhdf5 -lopencv_core)",
|
||||
"Bash(./gallery_tests)",
|
||||
"Bash(grep -viE \"^\\\\[gallery\\\\]|^ |^$|WARNING \\\\\\(GR-004\\\\\\)|\\\\*\\\\*\\\\*\\\\*\")",
|
||||
"Bash(python3 scripts/traceability/extract_traces.py --root jRay --requirements jRay/docs/requirements.md --system-spec SPEC.md --types JR --suffixes .cs,.js --scan-roots Jellyfin.Plugin.JRay --format coverage)",
|
||||
"Bash(python3 scripts/traceability/extract_traces.py --root /home/dtourolle/Development/Jray-project/jRay --requirements /home/dtourolle/Development/Jray-project/jRay/docs/requirements.md --system-spec /home/dtourolle/Development/Jray-project/SPEC.md --types JR --suffixes .cs,.js --scan-roots Jellyfin.Plugin.JRay --format coverage)",
|
||||
"Bash(cp -r /home/dtourolle/Development/Jray-project/scene-actor-extraction/external/KPN/. external/KPN/)",
|
||||
"Bash(rm -rf external/KPN/.git)",
|
||||
"Bash(rm -rf /tmp/claude-1000/-home-dtourolle-Development-Jray-project/de7e497c-da04-462d-a971-d0fcd36bda2a/scratchpad/bt)",
|
||||
"Bash(timeout 590 cmake -S . -B /tmp/claude-1000/-home-dtourolle-Development-Jray-project/de7e497c-da04-462d-a971-d0fcd36bda2a/scratchpad/bt -DSAE_BUILD_TESTS=ON -DSAE_GEMM_BACKEND=CPU -DCMAKE_BUILD_TYPE=Release)",
|
||||
"Bash(python3 scripts/traceability/extract_traces.py --root scene-actor-extraction --requirements scene-actor-extraction/docs/requirements.md --format coverage)",
|
||||
"Bash(timeout 590 cmake --build /tmp/claude-1000/-home-dtourolle-Development-Jray-project/de7e497c-da04-462d-a971-d0fcd36bda2a/scratchpad/bt --target sae_tests -j8)",
|
||||
"Bash(sh -n scripts/traceability/traceability-gate.sh)",
|
||||
"Bash(cd /home/dtourolle/Development/Jray-project *)"
|
||||
],
|
||||
"additionalDirectories": [
|
||||
"/home/dtourolle/Development/Jray-worktrees/gallery-model-binding/src/gallery",
|
||||
"/home/dtourolle/Development/Jray-worktrees/min-face-study/scripts/validation",
|
||||
"/home/dtourolle/Development/Jray-worktrees/audio-signature/tests",
|
||||
"/home/dtourolle/Development/Jray-worktrees/traceability-tooling/scripts/traceability"
|
||||
]
|
||||
}
|
||||
}
|
||||
Executable
+1299
File diff suppressed because it is too large
Load Diff
Executable
+822
@@ -0,0 +1,822 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Tests for the traceability extractor and coverage gate.
|
||||
|
||||
Run standalone (no third-party dependencies)::
|
||||
|
||||
python3 scripts/traceability/test_extract_traces.py
|
||||
|
||||
or under pytest, which discovers the same functions::
|
||||
|
||||
pytest scripts/traceability/test_extract_traces.py
|
||||
|
||||
Almost every test runs over fixture strings rather than the live
|
||||
``docs/requirements.md``, so their meaning does not drift as requirements are
|
||||
added. The two properties they exist to pin are the ones JellyTau's gate lost
|
||||
(see JellyTau/docs/specs/traceability-gate-repair.md):
|
||||
|
||||
* the denominator is computed from the register at run time, so adding a
|
||||
requirement lowers coverage until it is traced;
|
||||
* the numerator is an intersection, so a tag naming an undefined ID cannot push
|
||||
the ratio above 100%.
|
||||
|
||||
Plus the rule specific to this repo: a requirement only verifiable on GPU
|
||||
hardware is reported as tagged-but-unexecuted and never counted as covered.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import io
|
||||
import os
|
||||
import sys
|
||||
import tempfile
|
||||
from contextlib import redirect_stdout
|
||||
from pathlib import Path
|
||||
|
||||
HERE = Path(__file__).resolve().parent
|
||||
sys.path.insert(0, str(HERE))
|
||||
|
||||
import extract_traces as et # noqa: E402
|
||||
|
||||
# The literal tag keyword is assembled at run time so that this file's fixtures
|
||||
# do not register as real traces when the extractor scans scripts/.
|
||||
TAG = "TRA" + "CES:"
|
||||
EXC = "EXCEP" + "TION:"
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# Tag parsing
|
||||
# --------------------------------------------------------------------------
|
||||
|
||||
def test_parses_a_single_requirement():
|
||||
groups, junk = et.parse_traces_tag(" AR-012")
|
||||
assert groups == [["AR-012"]]
|
||||
assert junk == []
|
||||
|
||||
|
||||
def test_parses_multiple_types_separated_by_pipe():
|
||||
# The house format: a pipe separates requirement *types*, a comma separates
|
||||
# IDs within a type. The grouping is preserved, not flattened away, so a
|
||||
# malformed grouping stays detectable.
|
||||
groups, junk = et.parse_traces_tag(" AR-012, AR-013 | SR-002")
|
||||
assert groups == [["AR-012", "AR-013"], ["SR-002"]]
|
||||
assert junk == []
|
||||
|
||||
|
||||
def test_parses_three_groups_including_test_ids():
|
||||
groups, _ = et.parse_traces_tag(" AR-012 | SR-002 | UT-003, UT-004")
|
||||
assert groups == [["AR-012"], ["SR-002"], ["UT-003", "UT-004"]]
|
||||
|
||||
|
||||
def test_strips_a_trailing_block_comment_terminator():
|
||||
groups, junk = et.parse_traces_tag(" AR-001 */")
|
||||
assert groups == [["AR-001"]]
|
||||
assert junk == []
|
||||
|
||||
|
||||
def test_does_not_harvest_ids_out_of_prose_after_the_tag():
|
||||
# A trailing sentence must not smuggle IDs into the trace set: AR-999 here
|
||||
# is discussion, not a claim that this code satisfies AR-999.
|
||||
groups, junk = et.parse_traces_tag(" AR-001 - see also AR-999 in the notes")
|
||||
assert groups == [["AR-001"]]
|
||||
assert junk and "AR-999" in junk[0]
|
||||
|
||||
|
||||
def test_ignores_a_tag_with_no_ids_at_all():
|
||||
groups, _ = et.parse_traces_tag(" see the register")
|
||||
assert groups == []
|
||||
|
||||
|
||||
def test_two_digit_id_is_not_accepted_as_a_requirement():
|
||||
# AR-12 is a typo for AR-012; silently accepting it would create a
|
||||
# phantom requirement.
|
||||
groups, junk = et.parse_traces_tag(" AR-12")
|
||||
assert groups == []
|
||||
assert junk == ["AR-12"]
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# Scanning C++ and Python sources
|
||||
# --------------------------------------------------------------------------
|
||||
|
||||
def _write_tree(root: Path) -> None:
|
||||
(root / "src").mkdir(parents=True, exist_ok=True)
|
||||
(root / "scripts").mkdir(parents=True, exist_ok=True)
|
||||
(root / "external").mkdir(parents=True, exist_ok=True)
|
||||
(root / "src" / "tracker.hpp").write_text(
|
||||
"#pragma once\n"
|
||||
f"/// {TAG} AR-012, AR-013 | SR-002\n"
|
||||
"struct TrackRegistry {\n"
|
||||
" void close_all();\n"
|
||||
"};\n",
|
||||
encoding="utf-8")
|
||||
(root / "scripts" / "gallery.py").write_text(
|
||||
"def build_gallery(cast):\n"
|
||||
' """Build a gallery from Jellyfin plus TMDB fallback.\n'
|
||||
"\n"
|
||||
f" {TAG} GR-001 | SR-005\n"
|
||||
' """\n'
|
||||
" return {}\n",
|
||||
encoding="utf-8")
|
||||
(root / "external" / "vendored.cpp").write_text(
|
||||
f"// {TAG} AR-001\n", encoding="utf-8")
|
||||
(root / "src" / "notes.txt").write_text(
|
||||
f"// {TAG} AR-002\n", encoding="utf-8")
|
||||
|
||||
|
||||
def test_scans_cpp_and_python_but_not_vendored_or_non_source():
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
root = Path(tmp)
|
||||
_write_tree(root)
|
||||
files = et.iter_source_files(root)
|
||||
names = sorted(f.name for f in files)
|
||||
assert names == ["gallery.py", "tracker.hpp"], names
|
||||
|
||||
scan = et.scan_files(files, root)
|
||||
traced = sorted({i for t in scan.traces for i in t.requirements})
|
||||
assert traced == ["AR-012", "AR-013", "GR-001", "SR-002", "SR-005"]
|
||||
|
||||
|
||||
def test_context_is_found_below_a_cpp_tag_and_above_a_python_tag():
|
||||
# The two languages put the tag on opposite sides of what it describes.
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
root = Path(tmp)
|
||||
_write_tree(root)
|
||||
scan = et.scan_files(et.iter_source_files(root), root)
|
||||
contexts = {t.file: t.context for t in scan.traces}
|
||||
assert "TrackRegistry" in contexts["src/tracker.hpp"]
|
||||
assert "build_gallery" in contexts["scripts/gallery.py"]
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# EXCEPTION tags
|
||||
# --------------------------------------------------------------------------
|
||||
|
||||
EXCEPTION_SOURCE = (
|
||||
"float outlier_score(const Refs& refs) {\n"
|
||||
f" // {EXC} AR-024 distributional check on an actor's own references,\n"
|
||||
" return spread(refs);\n"
|
||||
"}\n"
|
||||
)
|
||||
|
||||
|
||||
def test_exception_tag_is_captured_with_its_reason():
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
root = Path(tmp)
|
||||
(root / "src").mkdir(parents=True)
|
||||
(root / "src" / "outlier.cpp").write_text(EXCEPTION_SOURCE, encoding="utf-8")
|
||||
scan = et.scan_files(et.iter_source_files(root), root)
|
||||
assert len(scan.exceptions) == 1
|
||||
exc = scan.exceptions[0]
|
||||
assert exc.requirement == "AR-024"
|
||||
assert exc.reason.startswith("distributional check")
|
||||
assert exc.line == 2
|
||||
|
||||
|
||||
def test_exception_is_never_counted_as_coverage():
|
||||
# An exception is a recorded decision to depart from an invariant. Counting
|
||||
# it as evidence the requirement is met inverts its meaning.
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
root = Path(tmp)
|
||||
(root / "src").mkdir(parents=True)
|
||||
(root / "src" / "outlier.cpp").write_text(EXCEPTION_SOURCE, encoding="utf-8")
|
||||
scan = et.scan_files(et.iter_source_files(root), root)
|
||||
assert scan.traces == []
|
||||
register = et.parse_register(
|
||||
"| ID | Requirement | Status |\n|---|---|---|\n"
|
||||
"| AR-024 | Always the calibrated probability | Planned |\n")
|
||||
cov = et.compute_coverage(
|
||||
[i for t in scan.traces for i in t.requirements], register)
|
||||
assert cov.covered == []
|
||||
assert cov.percent == 0.0
|
||||
|
||||
|
||||
def test_exception_without_a_reason_is_reported():
|
||||
# CLAUDE.md: an exception is only *agreed* if the reason is recorded.
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
root = Path(tmp)
|
||||
(root / "src").mkdir(parents=True)
|
||||
(root / "src" / "bare.cpp").write_text(
|
||||
f"// {EXC} AR-024\n", encoding="utf-8")
|
||||
scan = et.scan_files(et.iter_source_files(root), root)
|
||||
assert len(scan.exceptions) == 1
|
||||
assert scan.diagnostics.exceptions_without_reason
|
||||
|
||||
|
||||
def test_mixed_type_group_is_reported():
|
||||
# `AR-001, SR-002` in one group misuses the comma; the pipe is what
|
||||
# separates types, so the tag does not say what it appears to say.
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
root = Path(tmp)
|
||||
(root / "src").mkdir(parents=True)
|
||||
(root / "src" / "a.cpp").write_text(
|
||||
f"// {TAG} AR-001, SR-002\n", encoding="utf-8")
|
||||
scan = et.scan_files(et.iter_source_files(root), root)
|
||||
assert scan.diagnostics.mixed_type_groups
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# The register: denominators from requirements.md
|
||||
# --------------------------------------------------------------------------
|
||||
|
||||
REGISTER_HEADER = "| ID | Requirement | Traces to | Priority | Status |\n|---|---|---|---|---|\n"
|
||||
|
||||
|
||||
def test_counts_a_well_formed_table_row_as_a_defined_requirement():
|
||||
md = REGISTER_HEADER + (
|
||||
"| AR-001 | Detect faces in sampled frames | SR-002 | High | Done |\n"
|
||||
"| AR-002 | Minimum face size 66x66 px | SR-002 | High | Planned |\n")
|
||||
register = et.parse_register(md)
|
||||
assert register.count("AR") == 2
|
||||
assert register.count("GR") == 0
|
||||
assert register.total == 2
|
||||
|
||||
|
||||
def test_does_not_count_ids_that_appear_only_in_the_traces_to_column():
|
||||
# The bug this parse rule avoids: a naive scan for AR-\d{3} over the whole
|
||||
# file counts every reference as a definition and inflates the denominator.
|
||||
md = REGISTER_HEADER + (
|
||||
"| GR-001 | Build gallery from library cast | SR-001, SR-005 | High | Done |\n"
|
||||
"| GR-002 | Incremental merge refresh | PR-003 | High | Done |\n")
|
||||
register = et.parse_register(md)
|
||||
assert register.count("GR") == 2
|
||||
assert register.ids == {"GR-001", "GR-002"}
|
||||
|
||||
|
||||
def test_does_not_count_ids_mentioned_in_prose():
|
||||
md = ("Some prose explaining that AR-005 relates to GR-001 and VR-003.\n\n"
|
||||
+ REGISTER_HEADER
|
||||
+ "| AR-005 | Align to 112x112 | SR-002 | High | Done |\n")
|
||||
register = et.parse_register(md)
|
||||
assert register.ids == {"AR-005"}
|
||||
|
||||
|
||||
def test_does_not_count_the_verification_plan_table_as_definitions():
|
||||
# The per-requirement verification plan is also keyed on `ID`, but it
|
||||
# assigns tiers rather than defining requirements. Counting its rows would
|
||||
# double the denominator for every requirement that has a plan entry.
|
||||
md = (REGISTER_HEADER
|
||||
+ "| AR-001 | Detect faces | SR-002 | High | Done |\n"
|
||||
+ "\n"
|
||||
+ "| ID | Tier | Test asserts | Edge cases to cover |\n|---|---|---|---|\n"
|
||||
+ "| AR-001 | T3 | Detector returns plausible boxes | smoke only |\n"
|
||||
+ "| AR-099 | T1 | Something not in the register | - |\n")
|
||||
register = et.parse_register(md)
|
||||
assert register.ids == {"AR-001"}
|
||||
assert register.total == 1
|
||||
|
||||
|
||||
def test_deduplicates_an_id_listed_in_two_definition_tables():
|
||||
md = (REGISTER_HEADER
|
||||
+ "| AR-001 | Detect faces | SR-002 | High | Done |\n"
|
||||
+ "\n"
|
||||
+ REGISTER_HEADER
|
||||
+ "| AR-001 | Detect faces | SR-002 | High | Done |\n")
|
||||
register = et.parse_register(md)
|
||||
assert register.total == 1
|
||||
|
||||
|
||||
def test_withdrawn_requirements_leave_the_denominator():
|
||||
# IDs are permanent, but a withdrawn requirement can never be implemented.
|
||||
# Leaving it in the denominator would depress coverage forever.
|
||||
md = REGISTER_HEADER + (
|
||||
"| AR-001 | Detect faces | SR-002 | High | Done |\n"
|
||||
"| AR-002 | Superseded mechanism | SR-002 | High | Withdrawn |\n")
|
||||
register = et.parse_register(md)
|
||||
assert register.ids == {"AR-001"}
|
||||
assert "AR-002" in register.withdrawn
|
||||
|
||||
|
||||
def test_the_denominator_is_live_adding_a_row_lowers_coverage():
|
||||
# The property JellyTau's frozen literals destroyed. Same traced set, one
|
||||
# more requirement defined => a lower percentage, mechanically.
|
||||
base = REGISTER_HEADER + "| AR-001 | A | SR-002 | High | Done |\n"
|
||||
grown = base + "| AR-002 | B | SR-002 | High | Planned |\n"
|
||||
before = et.compute_coverage(["AR-001"], et.parse_register(base))
|
||||
after = et.compute_coverage(["AR-001"], et.parse_register(grown))
|
||||
assert before.percent == 100.0
|
||||
assert after.percent == 50.0
|
||||
assert after.total == 2
|
||||
|
||||
|
||||
def test_register_captures_the_row_fields_not_just_the_id():
|
||||
md = REGISTER_HEADER + (
|
||||
"| AR-012 | Presence follows track extent | **SR-002** | High | Planned |\n")
|
||||
req = et.parse_register(md).requirements["AR-012"]
|
||||
assert req.text == "Presence follows track extent"
|
||||
assert req.traces_to == "**SR-002**"
|
||||
assert req.status == "Planned"
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# Verification tiers and the GPU-less CI host
|
||||
# --------------------------------------------------------------------------
|
||||
|
||||
TIER_REGISTER = REGISTER_HEADER + "".join(
|
||||
f"| AR-{n:03d} | Requirement {n} | SR-002 | High | Planned |\n"
|
||||
for n in range(1, 10)) + "".join(
|
||||
f"| VR-{n:03d} | Study {n} | PR-002 | Medium | Planned |\n"
|
||||
for n in range(1, 4))
|
||||
|
||||
|
||||
def _tier_table(rows: str) -> str:
|
||||
return "| Requirement | Tier | Note |\n|---|---|---|\n" + rows
|
||||
|
||||
|
||||
def test_tier_assignment_handles_lists_ranges_and_wildcards():
|
||||
md = TIER_REGISTER + "\n" + _tier_table(
|
||||
"| AR-001, AR-005 | T3 | smoke |\n"
|
||||
"| AR-002 … AR-004 | **T2** | replay |\n"
|
||||
"| AR-006 | T1 + T4 | mixed |\n"
|
||||
"| VR-* | Out of CI | studies |\n")
|
||||
register = et.parse_register(md)
|
||||
assert register.requirements["AR-001"].tiers == {"T3"}
|
||||
assert register.requirements["AR-003"].tiers == {"T2"}
|
||||
assert register.requirements["AR-006"].tiers == {"T1", "T4"}
|
||||
assert register.requirements["VR-002"].tiers == {"out-of-ci"}
|
||||
|
||||
|
||||
def test_a_range_cannot_invent_a_requirement_the_register_lacks():
|
||||
md = TIER_REGISTER + "\n" + _tier_table("| AR-001 … AR-050 | T2 | wide |\n")
|
||||
register = et.parse_register(md)
|
||||
assert register.total == 12
|
||||
assert "AR-050" not in register.ids
|
||||
|
||||
|
||||
def test_slash_shorthand_in_the_verification_plan_expands():
|
||||
md = TIER_REGISTER + "\n" + (
|
||||
"| ID | Tier | Test asserts | Edge cases |\n|---|---|---|---|\n"
|
||||
"| AR-009/008 | T2 | Cut shifts weighting | cut with same people |\n")
|
||||
register = et.parse_register(md)
|
||||
assert register.requirements["AR-008"].tiers == {"T2"}
|
||||
assert register.requirements["AR-009"].tiers == {"T2"}
|
||||
|
||||
|
||||
def test_tiers_from_both_tables_are_unioned_not_overwritten():
|
||||
# The summary table says AR-006 is T4; the per-requirement plan adds a T1
|
||||
# equivalence check. The T1 part does run in CI, so the requirement is
|
||||
# executable and must not be written off as GPU-only.
|
||||
md = TIER_REGISTER + "\n" + _tier_table("| AR-006 | T4 | GPU host only |\n") + "\n" + (
|
||||
"| ID | Tier | Test asserts | Edge cases |\n|---|---|---|---|\n"
|
||||
"| AR-006 | T1 + T4 | GEMM equals reference loop | small input in CI |\n")
|
||||
register = et.parse_register(md)
|
||||
assert register.requirements["AR-006"].tiers == {"T1", "T4"}
|
||||
assert register.requirements["AR-006"].ci_executable
|
||||
|
||||
|
||||
def test_a_t4_only_requirement_is_not_ci_executable():
|
||||
md = TIER_REGISTER + "\n" + _tier_table("| AR-007 | **T4** | GPU only |\n")
|
||||
register = et.parse_register(md)
|
||||
assert not register.requirements["AR-007"].ci_executable
|
||||
assert register.unexecutable_ids() == {"AR-007"}
|
||||
|
||||
|
||||
def test_a_requirement_tracing_up_to_nothing_is_reported():
|
||||
# SPEC.md section 6: a requirement citing no parent is scope creep, and it
|
||||
# is invisible unless something looks. A section reference counts as a
|
||||
# parent - what matters is that something was recorded.
|
||||
md = REGISTER_HEADER + (
|
||||
"| AR-001 | Has a parent | SR-002 | High | Done |\n"
|
||||
"| AR-002 | Parent is a section | §4 | Medium | Planned |\n"
|
||||
"| AR-003 | Serves nothing stated | - | Low | Planned |\n"
|
||||
"| AR-004 | Blank cell | | Low | Planned |\n")
|
||||
register = et.parse_register(md)
|
||||
assert register.parentless_ids() == {"AR-003", "AR-004"}
|
||||
|
||||
|
||||
def test_a_requirement_with_no_tier_is_unknown_not_unexecutable():
|
||||
register = et.parse_register(TIER_REGISTER)
|
||||
assert register.tier_unknown_ids() == register.ids
|
||||
assert register.unexecutable_ids() == set()
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# Coverage arithmetic
|
||||
# --------------------------------------------------------------------------
|
||||
|
||||
COVERAGE_REGISTER = et.parse_register(
|
||||
REGISTER_HEADER
|
||||
+ "| AR-001 | A | SR-002 | High | Done |\n"
|
||||
+ "| AR-002 | B | SR-002 | High | Done |\n"
|
||||
+ "| GR-001 | C | SR-005 | High | Done |\n"
|
||||
+ "| AR-027 | Arbitrary gallery scale | SR-001 | High | Planned |\n"
|
||||
+ "\n"
|
||||
+ _tier_table("| AR-001, AR-002 | T2 | replay |\n"
|
||||
"| GR-001 | T1 | bookkeeping |\n"
|
||||
"| AR-027 | **T4** | GPU host only |\n"))
|
||||
|
||||
|
||||
def test_coverage_is_the_intersection_of_traced_and_defined():
|
||||
cov = et.compute_coverage(["AR-001", "GR-001"], COVERAGE_REGISTER)
|
||||
assert cov.covered == ["AR-001", "GR-001"]
|
||||
assert cov.total == 4
|
||||
assert cov.percent == 50.0
|
||||
|
||||
|
||||
def test_a_traced_but_undefined_id_cannot_inflate_the_numerator():
|
||||
# This is exactly how a ratio exceeds 100%: a tag naming a renumbered or
|
||||
# mistyped requirement counted as covered.
|
||||
cov = et.compute_coverage(["AR-001", "GR-001", "AR-097"], COVERAGE_REGISTER)
|
||||
assert cov.covered == ["AR-001", "GR-001"]
|
||||
assert cov.percent == 50.0
|
||||
|
||||
|
||||
def test_orphan_tags_are_reported_so_they_get_fixed():
|
||||
cov = et.compute_coverage(["AR-001", "AR-097", "GR-404"], COVERAGE_REGISTER)
|
||||
assert cov.orphaned == ["AR-097", "GR-404"]
|
||||
|
||||
|
||||
def test_no_orphans_when_every_traced_id_is_defined():
|
||||
cov = et.compute_coverage(["AR-001", "AR-002"], COVERAGE_REGISTER)
|
||||
assert cov.orphaned == []
|
||||
|
||||
|
||||
def test_test_and_system_ids_are_a_separate_taxonomy():
|
||||
# UT/IT live in their own register section; PR/SR live in the umbrella
|
||||
# SPEC.md, which is not part of this repo's checkout. Neither counts toward
|
||||
# coverage, and flagging them as orphans would bury real typos in noise.
|
||||
cov = et.compute_coverage(
|
||||
["AR-001", "UT-003", "IT-007", "SR-002", "PR-001"], COVERAGE_REGISTER)
|
||||
assert cov.orphaned == []
|
||||
assert cov.covered == ["AR-001"]
|
||||
|
||||
|
||||
def test_a_gpu_only_requirement_is_tagged_but_unexecuted_not_covered():
|
||||
# The rule specific to this repo: CI is an Intel N100 with no dGPU. A test
|
||||
# that exists but can never run is not evidence, and counting it is the
|
||||
# same failure mode as the 158% bug.
|
||||
cov = et.compute_coverage(["AR-001", "AR-027"], COVERAGE_REGISTER)
|
||||
assert cov.unexecuted == ["AR-027"]
|
||||
assert cov.covered == ["AR-001"]
|
||||
assert cov.percent == 25.0
|
||||
assert cov.orphaned == []
|
||||
|
||||
|
||||
def test_ci_scope_percentage_excludes_unexecutable_requirements_from_both_sides():
|
||||
cov = et.compute_coverage(["AR-001"], COVERAGE_REGISTER)
|
||||
assert cov.ci_executable_total == 3
|
||||
assert round(cov.ci_percent) == 33
|
||||
|
||||
|
||||
def test_tracing_only_gpu_only_requirements_yields_zero_coverage():
|
||||
cov = et.compute_coverage(["AR-027"], COVERAGE_REGISTER)
|
||||
assert cov.covered == []
|
||||
assert cov.percent == 0.0
|
||||
assert cov.unexecuted == ["AR-027"]
|
||||
|
||||
|
||||
def test_duplicate_traced_ids_are_counted_once():
|
||||
cov = et.compute_coverage(["AR-001", "AR-001", "AR-001"], COVERAGE_REGISTER)
|
||||
assert cov.covered == ["AR-001"]
|
||||
|
||||
|
||||
def test_an_empty_trace_set_is_zero_percent_not_a_divide_by_zero():
|
||||
cov = et.compute_coverage([], COVERAGE_REGISTER)
|
||||
assert cov.covered == []
|
||||
assert cov.percent == 0.0
|
||||
|
||||
|
||||
def test_an_empty_register_reports_zero_rather_than_nan():
|
||||
cov = et.compute_coverage(["AR-001"], et.Register())
|
||||
assert cov.percent == 0.0
|
||||
assert cov.ci_percent == 0.0
|
||||
|
||||
|
||||
def test_full_coverage_reports_exactly_one_hundred_never_above():
|
||||
cov = et.compute_coverage(
|
||||
["AR-001", "AR-002", "GR-001", "AR-027"], COVERAGE_REGISTER)
|
||||
# AR-027 is GPU-only, so the honest ceiling here is 3/4.
|
||||
assert cov.percent == 75.0
|
||||
assert cov.ci_percent == 100.0
|
||||
assert cov.ci_percent <= 100.0
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# The gate: end to end, including the ways it must fail
|
||||
# --------------------------------------------------------------------------
|
||||
|
||||
FIXTURE_REGISTER = (
|
||||
"# register\n\n" + REGISTER_HEADER
|
||||
+ "| AR-001 | Detect faces | SR-002 | High | Done |\n"
|
||||
+ "| AR-002 | Minimum face size | SR-002 | High | Planned |\n"
|
||||
+ "| AR-027 | Arbitrary gallery scale | SR-001 | High | Planned |\n"
|
||||
+ "\n"
|
||||
+ _tier_table("| AR-001, AR-002 | T2 | replay |\n"
|
||||
"| AR-027 | **T4** | GPU host only |\n"))
|
||||
|
||||
|
||||
def _fixture_repo(tmp: str, source: str) -> Path:
|
||||
root = Path(tmp)
|
||||
(root / "docs").mkdir(parents=True, exist_ok=True)
|
||||
(root / "src").mkdir(parents=True, exist_ok=True)
|
||||
(root / "docs" / "requirements.md").write_text(FIXTURE_REGISTER, encoding="utf-8")
|
||||
(root / "src" / "pipeline.cpp").write_text(source, encoding="utf-8")
|
||||
return root
|
||||
|
||||
|
||||
def _run_gate(root: Path, *extra: str):
|
||||
buffer = io.StringIO()
|
||||
with redirect_stdout(buffer):
|
||||
code = et.main(["--root", str(root), "--format", "coverage", *extra])
|
||||
return code, buffer.getvalue()
|
||||
|
||||
|
||||
def test_gate_passes_at_the_default_threshold_with_no_tags_yet():
|
||||
# Near-zero coverage on a fresh tree is the correct reading, not a failure.
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
root = _fixture_repo(tmp, "int main() { return 0; }\n")
|
||||
code, out = _run_gate(root)
|
||||
assert code == 0, out
|
||||
assert "0 / 3 (0.0%)" in out
|
||||
|
||||
|
||||
def test_gate_fails_below_an_explicit_threshold():
|
||||
# Proves the gate can fail at all. A gate nobody has watched fail is not
|
||||
# known to work.
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
root = _fixture_repo(tmp, f"// {TAG} AR-001\nint main() {{ return 0; }}\n")
|
||||
code, out = _run_gate(root, "--min-coverage", "99")
|
||||
assert code == 1
|
||||
assert "below the minimum" in out
|
||||
|
||||
|
||||
def test_gate_fails_on_an_orphan_tag():
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
root = _fixture_repo(tmp, f"// {TAG} AR-001, AR-404\nint main() {{}}\n")
|
||||
code, out = _run_gate(root)
|
||||
assert code == 1
|
||||
assert "AR-404" in out
|
||||
assert "orphan tag" in out
|
||||
|
||||
|
||||
def test_gate_can_be_asked_to_report_orphans_without_failing():
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
root = _fixture_repo(tmp, f"// {TAG} AR-001, AR-404\nint main() {{}}\n")
|
||||
code, out = _run_gate(root, "--allow-orphans")
|
||||
assert code == 0
|
||||
assert "AR-404" in out
|
||||
|
||||
|
||||
def test_gate_fails_when_the_register_parses_to_nothing():
|
||||
# With a low threshold this self-check is what keeps the gate meaningful:
|
||||
# a broken parser would otherwise report 0/0 and pass.
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
root = _fixture_repo(tmp, "int main() {}\n")
|
||||
(root / "docs" / "requirements.md").write_text(
|
||||
"# register\n\nNo tables here.\n", encoding="utf-8")
|
||||
code, out = _run_gate(root)
|
||||
assert code == 1
|
||||
assert "ZERO requirements" in out
|
||||
|
||||
|
||||
def test_gate_fails_when_no_source_files_were_scanned():
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
root = Path(tmp)
|
||||
(root / "docs").mkdir(parents=True)
|
||||
(root / "docs" / "requirements.md").write_text(FIXTURE_REGISTER,
|
||||
encoding="utf-8")
|
||||
code, out = _run_gate(root)
|
||||
assert code == 1
|
||||
assert "no source files were scanned" in out
|
||||
|
||||
|
||||
def test_gate_reports_a_gpu_only_requirement_as_tagged_but_unexecuted():
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
root = _fixture_repo(tmp, f"// {TAG} AR-027\nint main() {{}}\n")
|
||||
code, out = _run_gate(root)
|
||||
assert code == 0
|
||||
assert "TAGGED BUT UNEXECUTED" in out
|
||||
assert "AR-027 (tier T4)" in out
|
||||
assert "0 / 3 (0.0%)" in out
|
||||
|
||||
|
||||
def test_gate_hard_fails_on_an_impossible_ratio():
|
||||
# Coverage above 100% cannot happen through the intersection, which is the
|
||||
# point: if it ever does, the arithmetic is broken and the run must not be
|
||||
# reported as a pass. Forced here by handing the reporter a poisoned value.
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
root = _fixture_repo(tmp, f"// {TAG} AR-001\nint main() {{}}\n")
|
||||
register = et.read_register(root / "docs" / "requirements.md")
|
||||
scan = et.scan_files(et.iter_source_files(root), root)
|
||||
report = et.build_report(root, register, scan)
|
||||
report.coverage.percent = 158.0
|
||||
text, code = et.format_coverage_report(report, 50.0)
|
||||
assert code == 1
|
||||
assert "exceeds 100%" in text
|
||||
|
||||
|
||||
def test_the_per_type_breakdown_sums_to_the_headline_figure():
|
||||
# A breakdown that does not add up to its own total is how a wrong number
|
||||
# survives review: every row looks plausible on its own.
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
root = _fixture_repo(
|
||||
tmp, f"// {TAG} AR-001, AR-027\nint main() {{}}\n")
|
||||
register = et.read_register(root / "docs" / "requirements.md")
|
||||
scan = et.scan_files(et.iter_source_files(root), root)
|
||||
report = et.build_report(root, register, scan)
|
||||
stats = et.per_type_stats(report)
|
||||
assert sum(c for c, _, _ in stats.values()) == len(report.coverage.covered)
|
||||
assert sum(u for _, u, _ in stats.values()) == len(report.coverage.unexecuted)
|
||||
assert sum(d for _, _, d in stats.values()) == report.coverage.total
|
||||
assert stats["AR"] == (1, 1, 3)
|
||||
|
||||
|
||||
def test_json_and_markdown_outputs_are_written_and_consistent():
|
||||
import json
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
root = _fixture_repo(tmp, f"// {TAG} AR-001 | SR-002\nint main() {{}}\n")
|
||||
json_out = root / "traces-report.json"
|
||||
md_out = root / "docs" / "traceability.md"
|
||||
buffer = io.StringIO()
|
||||
with redirect_stdout(buffer):
|
||||
code = et.main(["--root", str(root), "--format", "coverage",
|
||||
"--json-out", str(json_out),
|
||||
"--markdown-out", str(md_out)])
|
||||
assert code == 0
|
||||
data = json.loads(json_out.read_text(encoding="utf-8"))
|
||||
assert data["defined"]["total"] == 3
|
||||
assert data["coverage"]["covered"] == 1
|
||||
assert data["coverage"]["percent"] == round(100 / 3, 1)
|
||||
assert data["byType"]["SR"] == ["SR-002"]
|
||||
assert data["gpuOnlyRequirements"] == ["AR-027"]
|
||||
assert "AR-001" in md_out.read_text(encoding="utf-8")
|
||||
|
||||
|
||||
def test_system_spec_parsing_enables_orphan_checks_for_pr_and_sr():
|
||||
spec = ("### SR-002 - Presence is scene-scoped\n\n"
|
||||
"| ID | Goal | Why |\n|---|---|---|\n"
|
||||
"| **PR-001** | Show which actors are on screen | The product |\n")
|
||||
ids = et.parse_system_spec(spec)
|
||||
assert ids == {"SR-002", "PR-001"}
|
||||
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
root = _fixture_repo(tmp, f"// {TAG} AR-001 | SR-999\nint main() {{}}\n")
|
||||
spec_path = root / "system-spec.md"
|
||||
spec_path.write_text(spec, encoding="utf-8")
|
||||
code, out = _run_gate(root, "--system-spec", str(spec_path))
|
||||
assert "SR-999" in out
|
||||
assert code == 0 # advisory: the system register is not this repo's
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# The live register — structural assertions only, so this does not churn as
|
||||
# requirements are added.
|
||||
# --------------------------------------------------------------------------
|
||||
|
||||
#: The live-register tests below ran against this repo's own register when the
|
||||
#: tool lived inside `scene-actor-extraction`. Now that it is shared, this repo
|
||||
#: is the project home and holds no component register of its own, so they take
|
||||
#: a path from the environment and skip when it is absent.
|
||||
#:
|
||||
#: LIVE_REGISTER=../scene-actor-extraction/docs/requirements.md \
|
||||
#: python3 test_extract_traces.py
|
||||
#:
|
||||
#: They are kept rather than deleted because they assert something the synthetic
|
||||
#: fixtures cannot: that a *real* register, with all its formatting accidents,
|
||||
#: parses at all.
|
||||
LIVE_REGISTER = os.environ.get("LIVE_REGISTER")
|
||||
|
||||
|
||||
def test_the_live_register_parses_and_assigns_tiers():
|
||||
if not LIVE_REGISTER:
|
||||
return # skipped: no component register to point at
|
||||
register = et.read_register(Path(LIVE_REGISTER))
|
||||
assert register.total > 0
|
||||
assert sum(register.count(t) for t in et.LOCAL_TYPES) == register.total
|
||||
# A tier that no GPU-less host can run must be visible in the parse, not
|
||||
# merely stated in prose.
|
||||
assert register.unexecutable_ids() <= register.ids
|
||||
|
||||
|
||||
def test_the_live_register_yields_a_gate_run_that_cannot_exceed_one_hundred():
|
||||
if not LIVE_REGISTER:
|
||||
return
|
||||
register = et.read_register(Path(LIVE_REGISTER))
|
||||
root = Path(LIVE_REGISTER).resolve().parent.parent
|
||||
scan = et.scan_files(et.iter_source_files(root), root)
|
||||
cov = et.compute_coverage(
|
||||
[i for t in scan.traces for i in t.requirements], register)
|
||||
assert 0.0 <= cov.percent <= 100.0
|
||||
assert len(cov.covered) <= cov.total
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# Per-repo configuration — the flags that make this tool shareable.
|
||||
#
|
||||
# Without them a repo whose prefixes or language differ from the defaults gets
|
||||
# zero requirements and zero files, which the gate correctly refuses to report
|
||||
# as coverage. These assert the overrides actually take effect.
|
||||
# --------------------------------------------------------------------------
|
||||
|
||||
def _restore_taxonomy(fn):
|
||||
"""Run `fn` with the module defaults restored afterwards."""
|
||||
local, suffixes, roots = et.LOCAL_TYPES, set(et.SOURCE_SUFFIXES), et.SCAN_ROOTS
|
||||
try:
|
||||
fn()
|
||||
finally:
|
||||
et.configure_taxonomy(local)
|
||||
et.configure_suffixes(suffixes)
|
||||
et.configure_scan_roots(roots)
|
||||
|
||||
|
||||
def test_types_override_changes_which_prefixes_are_counted():
|
||||
def body():
|
||||
register_md = (
|
||||
"| ID | Requirement | Traces to | Priority | Status |\n"
|
||||
"|---|---|---|---|---|\n"
|
||||
"| UR-001 | A user requirement | SR-001 | High | Done |\n"
|
||||
"| DR-001 | A dev requirement | PR-004 | High | Done |\n"
|
||||
)
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
path = Path(tmp) / "requirements.md"
|
||||
path.write_text(register_md, encoding="utf-8")
|
||||
|
||||
# Under the defaults these prefixes are unknown, so nothing counts.
|
||||
et.configure_taxonomy(("AR", "DP"))
|
||||
assert et.read_register(path).total == 0
|
||||
|
||||
et.configure_taxonomy(("UR", "DR"))
|
||||
register = et.read_register(path)
|
||||
assert register.total == 2
|
||||
assert register.count("UR") == 1
|
||||
assert register.count("DR") == 1
|
||||
|
||||
_restore_taxonomy(body)
|
||||
|
||||
|
||||
def test_suffix_override_changes_which_files_are_scanned():
|
||||
def body():
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
root = Path(tmp)
|
||||
(root / "src").mkdir()
|
||||
(root / "src" / "lib.rs").write_text(
|
||||
"/// TRACES: UR-001 | SR-004\npub fn f() {}\n", encoding="utf-8")
|
||||
|
||||
# Default suffixes are C++/Python, so a Rust tree scans as empty —
|
||||
# the failure mode this override exists to fix.
|
||||
assert et.iter_source_files(root) == []
|
||||
|
||||
et.configure_suffixes([".rs"])
|
||||
files = et.iter_source_files(root)
|
||||
assert len(files) == 1
|
||||
scan = et.scan_files(files, root)
|
||||
assert [i for t in scan.traces for i in t.requirements] == [
|
||||
"UR-001", "SR-004"]
|
||||
|
||||
_restore_taxonomy(body)
|
||||
|
||||
|
||||
def test_suffix_override_accepts_extensions_with_or_without_a_dot():
|
||||
def body():
|
||||
et.configure_suffixes(["rs", ".cs"])
|
||||
assert et.SOURCE_SUFFIXES == {".rs", ".cs"}
|
||||
|
||||
_restore_taxonomy(body)
|
||||
|
||||
|
||||
def test_scan_root_override_limits_the_walk():
|
||||
def body():
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
root = Path(tmp)
|
||||
# Not "vendor" — that is in EXCLUDED_DIR_NAMES and would be
|
||||
# filtered whatever the scan roots say, testing the wrong thing.
|
||||
for d in ("src", "eval"):
|
||||
(root / d).mkdir()
|
||||
(root / d / "f.py").write_text("# TRACES: AR-001\n", encoding="utf-8")
|
||||
|
||||
et.configure_scan_roots(["src"])
|
||||
assert len(et.iter_source_files(root)) == 1
|
||||
|
||||
et.configure_scan_roots(["src", "eval"])
|
||||
assert len(et.iter_source_files(root)) == 2
|
||||
|
||||
_restore_taxonomy(body)
|
||||
|
||||
|
||||
def test_defaults_are_unchanged_by_the_overrides_existing():
|
||||
# The extraction pipeline must keep working with no flags at all, or moving
|
||||
# the tool here would have broken the repo it came from.
|
||||
assert et.LOCAL_TYPES == ("AR", "DP", "IR", "GR", "VR")
|
||||
assert ".py" in et.SOURCE_SUFFIXES and ".cpp" in et.SOURCE_SUFFIXES
|
||||
assert ".rs" not in et.SOURCE_SUFFIXES
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
|
||||
def _main() -> int:
|
||||
tests = [(name, obj) for name, obj in sorted(globals().items())
|
||||
if name.startswith("test_") and callable(obj)]
|
||||
failed = []
|
||||
for name, fn in tests:
|
||||
try:
|
||||
fn()
|
||||
except Exception as exc: # noqa: BLE001 - a test runner reports everything
|
||||
failed.append((name, exc))
|
||||
print(f"FAIL {name}: {type(exc).__name__}: {exc}")
|
||||
else:
|
||||
print(f"ok {name}")
|
||||
print(f"\n{len(tests) - len(failed)}/{len(tests)} passed")
|
||||
return 1 if failed else 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(_main())
|
||||
Executable
+89
@@ -0,0 +1,89 @@
|
||||
#!/bin/sh
|
||||
#
|
||||
# Requirement traceability gate. Run locally exactly as CI runs it:
|
||||
#
|
||||
# scripts/traceability/traceability-gate.sh
|
||||
#
|
||||
# Writes traces-report.json and docs/traceability.md, prints the coverage
|
||||
# report, and exits non-zero when the gate fails.
|
||||
#
|
||||
# Environment:
|
||||
# MIN_COVERAGE minimum overall coverage percent (default 0 - see below)
|
||||
# ALLOW_ORPHANS set to 1 to report orphan tags without failing
|
||||
# TRACES_JSON JSON report path (default traces-report.json)
|
||||
# TRACES_MD markdown matrix path (default docs/traceability.md)
|
||||
# REPO_ROOT repository to scan. Defaults to two levels above this
|
||||
# script, which is correct when the tooling lives in the repo
|
||||
# it checks. **When vendored as a submodule that default is
|
||||
# the submodule itself**, so a consuming repo must set this —
|
||||
# its wrapper does.
|
||||
# TYPES comma-separated requirement prefixes (e.g. UR,DR). Defaults
|
||||
# to the extraction set; a repo whose prefixes differ parses
|
||||
# to zero requirements without this.
|
||||
# SUFFIXES comma-separated file extensions (e.g. .rs). Defaults to the
|
||||
# C++/Python set; a repo whose language differs scans zero
|
||||
# files without this.
|
||||
# SCAN_ROOTS comma-separated directories to walk, relative to REPO_ROOT.
|
||||
# SYSTEM_SPEC optional path to the system SPEC.md, which defines the PR/SR
|
||||
# IDs; when given, PR/SR orphans are reported too. It lives in
|
||||
# the project home (jray-project) — when this tooling is
|
||||
# vendored from there, it is a sibling of this script.
|
||||
#
|
||||
# Threshold policy lives here and nowhere else. It is deliberately NOT
|
||||
# duplicated into the workflow YAML: a threshold written in two places is a
|
||||
# threshold that will disagree with itself.
|
||||
#
|
||||
# MIN_COVERAGE defaults to 0 because almost nothing is tagged yet - tags are
|
||||
# added as the pipeline is built, so a low number today is accurate rather than
|
||||
# alarming. A zero threshold does NOT mean the gate cannot fail: orphan tags,
|
||||
# a >100% ratio, a register that parses to nothing, and an empty source scan
|
||||
# are all hard failures from day one. Raise MIN_COVERAGE as tags land; treat
|
||||
# every raise as a ratchet, never a reset.
|
||||
#
|
||||
# POSIX sh, no bashisms, no jq - the extractor does its own arithmetic and
|
||||
# printing so CI needs nothing beyond python3.
|
||||
|
||||
set -eu
|
||||
|
||||
SCRIPT_DIR=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd)
|
||||
REPO_ROOT="${REPO_ROOT:-$(CDPATH= cd -- "$SCRIPT_DIR/../.." && pwd)}"
|
||||
|
||||
MIN_COVERAGE="${MIN_COVERAGE:-0}"
|
||||
TRACES_JSON="${TRACES_JSON:-$REPO_ROOT/traces-report.json}"
|
||||
TRACES_MD="${TRACES_MD:-$REPO_ROOT/docs/traceability.md}"
|
||||
|
||||
PYTHON="${PYTHON:-python3}"
|
||||
command -v "$PYTHON" >/dev/null 2>&1 || {
|
||||
echo "FAILED: $PYTHON not found. The traceability gate needs Python 3.9+" >&2
|
||||
exit 2
|
||||
}
|
||||
|
||||
set -- \
|
||||
--root "$REPO_ROOT" \
|
||||
--requirements "${REQUIREMENTS:-$REPO_ROOT/docs/requirements.md}" \
|
||||
--format coverage \
|
||||
--json-out "$TRACES_JSON" \
|
||||
--markdown-out "$TRACES_MD" \
|
||||
--min-coverage "$MIN_COVERAGE"
|
||||
|
||||
if [ "${ALLOW_ORPHANS:-0}" = "1" ]; then
|
||||
set -- "$@" --allow-orphans
|
||||
fi
|
||||
|
||||
if [ -n "${SYSTEM_SPEC:-}" ]; then
|
||||
set -- "$@" --system-spec "$SYSTEM_SPEC"
|
||||
fi
|
||||
|
||||
if [ -n "${TYPES:-}" ]; then
|
||||
set -- "$@" --types "$TYPES"
|
||||
fi
|
||||
|
||||
if [ -n "${SUFFIXES:-}" ]; then
|
||||
set -- "$@" --suffixes "$SUFFIXES"
|
||||
fi
|
||||
|
||||
if [ -n "${SCAN_ROOTS:-}" ]; then
|
||||
set -- "$@" --scan-roots "$SCAN_ROOTS"
|
||||
fi
|
||||
|
||||
exec "$PYTHON" "$SCRIPT_DIR/extract_traces.py" "$@"
|
||||
Reference in New Issue
Block a user