/** * Tests for the traceability coverage computation. * * These run over fixture strings rather than the live docs/requirements.md, so * their meaning does not drift as requirements are added. * * Background: the CI gate divided traced-requirement counts by hardcoded * denominators (UR/39, IR/24, DR/48, JA/3, total 114) that had fallen out of * date, reporting 158% coverage and making the 50% threshold unreachable. These * tests pin the parsing and arithmetic that replace those literals. * * @req-test: UT-089 - Requirement definitions parsed from requirements.md * @req-test: UT-090 - Coverage is the intersection of traced and defined IDs * @req-test: UT-202 - Generated matrix links resolve from docs/ */ import { describe, it, expect } from "vitest"; import * as fs from "fs"; import * as path from "path"; import { countDefinedRequirements, computeCoverage, findDanglingIds, formatMatrixFileLink, generateMarkdown, isTracedSourceFile, MIN_COVERAGE_PERCENT, type TracesData, } from "./extract-traces"; // import.meta.dir is Bun-only; derive from import.meta.url under vitest. const HERE = path.dirname(new URL(import.meta.url).pathname); describe("isTracedSourceFile", () => { // The extractor used to accept only .ts/.svelte/.rs under src/, src-tauri/src/ // and scripts/. Every requirement implemented by *configuration* was therefore // invisible to the matrix that measures it: eslint.config.js (DR-205), the // pre-commit hook (DR-207), rust-toolchain.toml (DR-206) and deny.toml // (DR-216) all carry TRACES comments that were never read. Each one counted // against coverage as an uncovered requirement while being, in fact, covered. it("accepts the source extensions it always did", () => { expect(isTracedSourceFile("src/lib/utils/logger.ts")).toBe(true); expect(isTracedSourceFile("src/routes/settings/+page.svelte")).toBe(true); expect(isTracedSourceFile("src-tauri/src/lib.rs")).toBe(true); }); it("accepts tooling files that implement a requirement", () => { expect(isTracedSourceFile("eslint.config.js")).toBe(true); expect(isTracedSourceFile("src-tauri/deny.toml")).toBe(true); expect(isTracedSourceFile("src-tauri/rust-toolchain.toml")).toBe(true); expect(isTracedSourceFile("scripts/hooks/pre-commit")).toBe(true); // Shell tooling is listed individually, not globbed: most scripts/*.sh // implement nothing, and adding one should be a decision. expect(isTracedSourceFile("scripts/check-release-artifacts.sh")).toBe(true); expect(isTracedSourceFile("scripts/build-desktop-linux.sh")).toBe(true); expect(isTracedSourceFile("scripts/logcat.sh")).toBe(false); }); it("does not scan CI workflows, whose comments discuss TRACES in prose", () => { // .gitea/workflows/traceability-check.yml explains the gate, so it contains // lines like "a `TRACES:` comment ... (DR-189 and UT-188 lived in three // source files, defined nowhere)". The extractor's pattern would read that // as a trace and manufacture references to IDs that do not exist, failing // traces:validate. A file that *describes* traceability is not a file that // implements a requirement. expect(isTracedSourceFile(".gitea/workflows/traceability-check.yml")).toBe(false); expect(isTracedSourceFile(".gitea/workflows/build-and-test.yml")).toBe(false); }); it("rejects files that merely mention a requirement in prose", () => { // requirements.md defines IDs; traceability.md is generated *from* traces. // Scanning either would make every requirement trace to itself. expect(isTracedSourceFile("docs/requirements.md")).toBe(false); expect(isTracedSourceFile("docs/traceability.md")).toBe(false); expect(isTracedSourceFile("README.md")).toBe(false); }); it("rejects generated and vendored trees", () => { expect(isTracedSourceFile("node_modules/foo/index.ts")).toBe(false); expect(isTracedSourceFile("src-tauri/target/debug/build/x.rs")).toBe(false); expect(isTracedSourceFile("src-tauri/gen/android/app/build.gradle.kts")).toBe(false); }); }); describe("countDefinedRequirements", () => { it("counts a well-formed table row as a defined requirement", () => { const md = ` | ID | Requirement | Priority | Status | |----|-------------|----------|--------| | UR-001 | Run the app on multiple platforms | High | In Progress | | UR-002 | Access media when online or offline | High | Done | `; const defined = countDefinedRequirements(md); expect(defined.UR).toBe(2); expect(defined.DR).toBe(0); }); it("does not count IDs that appear only in the Traces To column", () => { // The bug this rule avoids: a naive grep for /DR-\d{3}/ over the whole file // counts DR-001 here as "defined", inflating the denominator with IDs that // are merely referenced. const md = ` | DR-001 | Player state machine | Player | UR-005 | Done | | DR-002 | MediaItem struct | Player | UR-003, UR-004 | Done | `; const defined = countDefinedRequirements(md); expect(defined.DR).toBe(2); // UR-005/UR-003/UR-004 are referenced, never defined here. expect(defined.UR).toBe(0); }); it("does not count IDs mentioned in prose", () => { const md = ` Some prose explaining that UR-005 relates to DR-001 and JA-002. | UR-005 | Control media playback | High | Done | `; const defined = countDefinedRequirements(md); expect(defined.UR).toBe(1); expect(defined.DR).toBe(0); expect(defined.JA).toBe(0); }); it("deduplicates an ID listed in both the spec table and the traceability matrix", () => { // requirements.md lists every UR twice: once in §1 (definition) and again in // §3 (traceability matrix), both as a leading table cell. Counting rows // instead of unique IDs double-counts the UR denominator (121 vs 61). const md = ` | UR-005 | Control media playback | High | Done | | UR-006 | Browse the library | High | Done | ### Traceability Matrix | UR-005 | - | DR-001, DR-005, DR-009 | | UR-006 | - | DR-012 | `; const defined = countDefinedRequirements(md); expect(defined.UR).toBe(2); }); it("collects the defined ID set, not just counts", () => { const md = ` | UR-001 | A | High | Done | | DR-050 | B | Player | UR-001 | Done | `; const defined = countDefinedRequirements(md); expect(defined.ids.has("UR-001")).toBe(true); expect(defined.ids.has("DR-050")).toBe(true); expect(defined.ids.has("UR-999")).toBe(false); }); it("collects UT/IT rows separately, out of the coverage denominator", () => { // §4 defines the test taxonomy. Those rows must be known (so a TRACES // comment may name them) without ever moving the coverage ratio. const md = ` | UR-001 | A | High | Done | | UT-001 | Player state transitions | DR-001 | Pending | | IT-004 | Playback end-to-end | DR-002 | Pending | `; const defined = countDefinedRequirements(md); expect(defined.total).toBe(1); expect(defined.ids.has("UT-001")).toBe(false); expect(defined.testIds.has("UT-001")).toBe(true); expect(defined.testIds.has("IT-004")).toBe(true); }); }); describe("findDanglingIds", () => { const defined = { UR: 1, IR: 0, DR: 1, JA: 0, total: 2, ids: new Set(["UR-001", "DR-001"]), testIds: new Set(["UT-001"]), }; it("flags a requirement ID that requirements.md does not define", () => { expect(findDanglingIds(["UR-001", "DR-189"], defined)).toEqual(["DR-189"]); }); it("flags an undefined UT/IT id, which the coverage orphan list cannot", () => { // The gap this closes: computeCoverage deliberately ignores UT/IT, so // UT-188 sat in three source files, defined nowhere, entirely unreported. expect(computeCoverage(["UT-188"], defined).orphaned).toEqual([]); expect(findDanglingIds(["UT-188"], defined)).toEqual(["UT-188"]); }); it("accepts every ID that is defined, requirement or test", () => { expect(findDanglingIds(["UR-001", "DR-001", "UT-001"], defined)).toEqual([]); }); it("deduplicates and sorts, so one typo is reported once", () => { expect(findDanglingIds(["DR-189", "DR-189", "UR-999", "DR-189"], defined)).toEqual([ "DR-189", "UR-999", ]); }); it("ignores IDs whose prefix is not a known trace type", () => { // e.g. an unrelated "AB-123" caught by the loose ID regex. expect(findDanglingIds(["AB-123"], defined)).toEqual([]); }); }); describe("coverage threshold", () => { it("matches MIN_THRESHOLD in the Gitea traceability workflow", () => { // Two files must agree on the gate: the script (local `traces:coverage`) // and the workflow. Drift means the local gate and CI disagree about what // passes, which is how the 50%-while-actually-86% slack went unnoticed. const workflow = fs.readFileSync( path.resolve(HERE, "../.gitea/workflows/traceability-check.yml"), "utf-8", ); const match = workflow.match(/^\s*MIN_THRESHOLD=(\d+)\s*$/m); expect(match).not.toBeNull(); expect(Number(match![1])).toBe(MIN_COVERAGE_PERCENT); }); it("is a ratchet: never lower it to make a red build pass", () => { // Sanity bound. If coverage genuinely climbs, raise both numbers together. expect(MIN_COVERAGE_PERCENT).toBeGreaterThanOrEqual(82); expect(MIN_COVERAGE_PERCENT).toBeLessThanOrEqual(100); }); }); describe("computeCoverage", () => { const defined = { UR: 2, IR: 0, DR: 2, JA: 0, total: 4, ids: new Set(["UR-001", "UR-002", "DR-001", "DR-002"]), testIds: new Set(), }; it("computes coverage as traced ∩ defined over defined", () => { const traced = ["UR-001", "DR-001"]; const cov = computeCoverage(traced, defined); expect(cov.covered).toBe(2); expect(cov.total).toBe(4); expect(cov.percent).toBe(50); }); it("does not let a traced-but-undefined ID inflate the numerator", () => { // This is how a ratio exceeds 100%: a TRACES comment naming a typo'd or // deleted requirement counted as covered. const traced = ["UR-001", "DR-001", "DR-097"]; const cov = computeCoverage(traced, defined); expect(cov.covered).toBe(2); expect(cov.percent).toBe(50); }); it("reports traced-but-undefined IDs as orphaned so they get fixed", () => { const traced = ["UR-001", "DR-097", "JA-404"]; const cov = computeCoverage(traced, defined); expect(cov.orphaned).toEqual(["DR-097", "JA-404"]); }); it("has no orphans when every traced ID is defined", () => { const cov = computeCoverage(["UR-001", "UR-002"], defined); expect(cov.orphaned).toEqual([]); }); it("ignores UT/IT test IDs entirely — they are a separate taxonomy", () => { // UT/IT are defined in §4 of requirements.md, not among the four // requirement types. Treating them as orphans buries real typos in ~60 // lines of noise, and counting them would corrupt the ratio. const cov = computeCoverage(["UR-001", "UT-088", "IT-017"], defined); expect(cov.orphaned).toEqual([]); expect(cov.covered).toBe(1); }); it("reports 0% rather than dividing by zero for an empty trace set", () => { const cov = computeCoverage([], defined); expect(cov.covered).toBe(0); expect(cov.percent).toBe(0); }); it("reports 0% rather than NaN when nothing is defined", () => { const empty = { UR: 0, IR: 0, DR: 0, JA: 0, total: 0, ids: new Set(), testIds: new Set(), }; const cov = computeCoverage([], empty); expect(cov.percent).toBe(0); expect(Number.isNaN(cov.percent)).toBe(false); }); it("reports exactly 100% when all defined requirements are traced, never above", () => { const traced = ["UR-001", "UR-002", "DR-001", "DR-002"]; const cov = computeCoverage(traced, defined); expect(cov.percent).toBe(100); }); it("ignores duplicate traced IDs", () => { const traced = ["UR-001", "UR-001", "UR-001"]; const cov = computeCoverage(traced, defined); expect(cov.covered).toBe(1); }); }); describe("generated matrix file links", () => { // Regression: the generator emitted the repo-root-relative path as the href // (`](src-tauri/src/…)`), but writes its output to docs/traceability.md — so // every one of the ~2,800 links resolved to docs/src-tauri/… and 404'd, in // the repo browser and on the published mdBook site. The markdown generator // had no test at all, which is why it survived. UT-202. // // @req-test: UT-202 /** A minimal TracesData whose single entry points at a file that really exists. */ function fixture(file: string, line = 12): TracesData { return { timestamp: new Date().toISOString(), totalFiles: 1, totalTraces: 1, requirements: { "DR-093": [{ file, line, context: "export function x() {}" }], }, byType: { UR: [], IR: [], DR: ["DR-093"], JA: [] }, } as TracesData; } /** Pull the href out of the first `- **File:** [`x`](href)` line. */ function firstHref(md: string): string { const m = md.match(/^- \*\*File:\*\* \[`[^`]+`\]\(([^)]+)\)/m); expect(m).not.toBeNull(); return m![1]; } it("emits an href that resolves, from docs/, to a file that exists", () => { // Use a real repo file so "exists on disk" is a genuine assertion. const target = "scripts/extract-traces.ts"; const md = generateMarkdown(fixture(target)); const href = firstHref(md); const [relPath] = href.split("#"); // traceability.md is written to docs/, so links resolve from there. const resolved = path.resolve(HERE, "../docs", relPath); expect(fs.existsSync(resolved)).toBe(true); expect(resolved).toBe(path.resolve(HERE, "..", target)); }); it("keeps the repo-root-relative path as the visible link text", () => { // The text is what a developer copies into an editor or a grep; only the // href is rewritten for the docs/ location. const md = generateMarkdown(fixture("src-tauri/src/lib.rs")); expect(md).toContain("[`src-tauri/src/lib.rs`]"); expect(md).not.toContain("[`../src-tauri/src/lib.rs`]"); }); it("keeps the #Lnn line anchor on the href", () => { const link = formatMatrixFileLink("scripts/extract-traces.ts", 427); expect(link).toBe("[`scripts/extract-traces.ts`](../scripts/extract-traces.ts#L427)"); }); it("does not produce a bare repo-root href, which resolves to docs/", () => { const md = generateMarkdown(fixture("scripts/extract-traces.ts")); const href = firstHref(md); expect(href.startsWith("../")).toBe(true); // The pre-fix output — the exact shape that produced docs/scripts/…. expect(href.startsWith("scripts/")).toBe(false); }); }); describe("live requirements.md", () => { it("parses the real file into a self-consistent denominator", () => { // Guards the original regression: CI hardcoded UR/39, IR/24, DR/48, JA/3 // (total 114) while the real file had grown past 200, so the gate compared // live traces against a frozen denominator and reported 158% coverage. // // Deliberately asserts *invariants*, not exact totals. Pinning the counts // was tried and turned this test into a merge-conflict magnet: every // requirement added on any branch had to edit the numbers here too, and the // comment above them grew into a ledger of which branch contributed which // row. Worse, the pins never guarded the actual defect — a stale denominator // is caught by the sum-consistency check below, and the >100% ratio it // produced is covered directly by the computeCoverage tests, on fixtures. const md = fs.readFileSync(path.resolve(HERE, "../docs/requirements.md"), "utf-8"); const defined = countDefinedRequirements(md); // The parser found real rows of every type: a section silently failing to // parse would shrink the denominator and inflate coverage. expect(defined.UR).toBeGreaterThan(0); expect(defined.IR).toBeGreaterThan(0); expect(defined.DR).toBeGreaterThan(0); expect(defined.JA).toBeGreaterThan(0); // The denominator is the sum of its parts, and every counted id is unique — // double-counting one section is the other way a ratio breaks. expect(defined.total).toBe(defined.UR + defined.IR + defined.DR + defined.JA); expect(defined.ids.size).toBe(defined.total); // The file is live, not frozen: it is well past the 114 the stale gate used. expect(defined.total).toBeGreaterThan(200); }); });