/** * Tests for the traceability coverage computation. * * These run over fixture strings rather than the live docs/requirements.md, so * their meaning does not drift as requirements are added. * * Background: the CI gate divided traced-requirement counts by hardcoded * denominators (UR/39, IR/24, DR/48, JA/3, total 114) that had fallen out of * date, reporting 158% coverage and making the 50% threshold unreachable. These * tests pin the parsing and arithmetic that replace those literals. * * @req-test: UT-089 - Requirement definitions parsed from requirements.md * @req-test: UT-090 - Coverage is the intersection of traced and defined IDs */ import { describe, it, expect } from "vitest"; import * as fs from "fs"; import * as path from "path"; import { countDefinedRequirements, computeCoverage, findDanglingIds, MIN_COVERAGE_PERCENT, } from "./extract-traces"; // import.meta.dir is Bun-only; derive from import.meta.url under vitest. const HERE = path.dirname(new URL(import.meta.url).pathname); describe("countDefinedRequirements", () => { it("counts a well-formed table row as a defined requirement", () => { const md = ` | ID | Requirement | Priority | Status | |----|-------------|----------|--------| | UR-001 | Run the app on multiple platforms | High | In Progress | | UR-002 | Access media when online or offline | High | Done | `; const defined = countDefinedRequirements(md); expect(defined.UR).toBe(2); expect(defined.DR).toBe(0); }); it("does not count IDs that appear only in the Traces To column", () => { // The bug this rule avoids: a naive grep for /DR-\d{3}/ over the whole file // counts DR-001 here as "defined", inflating the denominator with IDs that // are merely referenced. const md = ` | DR-001 | Player state machine | Player | UR-005 | Done | | DR-002 | MediaItem struct | Player | UR-003, UR-004 | Done | `; const defined = countDefinedRequirements(md); expect(defined.DR).toBe(2); // UR-005/UR-003/UR-004 are referenced, never defined here. expect(defined.UR).toBe(0); }); it("does not count IDs mentioned in prose", () => { const md = ` Some prose explaining that UR-005 relates to DR-001 and JA-002. | UR-005 | Control media playback | High | Done | `; const defined = countDefinedRequirements(md); expect(defined.UR).toBe(1); expect(defined.DR).toBe(0); expect(defined.JA).toBe(0); }); it("deduplicates an ID listed in both the spec table and the traceability matrix", () => { // requirements.md lists every UR twice: once in §1 (definition) and again in // §3 (traceability matrix), both as a leading table cell. Counting rows // instead of unique IDs double-counts the UR denominator (121 vs 61). const md = ` | UR-005 | Control media playback | High | Done | | UR-006 | Browse the library | High | Done | ### Traceability Matrix | UR-005 | - | DR-001, DR-005, DR-009 | | UR-006 | - | DR-012 | `; const defined = countDefinedRequirements(md); expect(defined.UR).toBe(2); }); it("collects the defined ID set, not just counts", () => { const md = ` | UR-001 | A | High | Done | | DR-050 | B | Player | UR-001 | Done | `; const defined = countDefinedRequirements(md); expect(defined.ids.has("UR-001")).toBe(true); expect(defined.ids.has("DR-050")).toBe(true); expect(defined.ids.has("UR-999")).toBe(false); }); it("collects UT/IT rows separately, out of the coverage denominator", () => { // §4 defines the test taxonomy. Those rows must be known (so a TRACES // comment may name them) without ever moving the coverage ratio. const md = ` | UR-001 | A | High | Done | | UT-001 | Player state transitions | DR-001 | Pending | | IT-004 | Playback end-to-end | DR-002 | Pending | `; const defined = countDefinedRequirements(md); expect(defined.total).toBe(1); expect(defined.ids.has("UT-001")).toBe(false); expect(defined.testIds.has("UT-001")).toBe(true); expect(defined.testIds.has("IT-004")).toBe(true); }); }); describe("findDanglingIds", () => { const defined = { UR: 1, IR: 0, DR: 1, JA: 0, total: 2, ids: new Set(["UR-001", "DR-001"]), testIds: new Set(["UT-001"]), }; it("flags a requirement ID that requirements.md does not define", () => { expect(findDanglingIds(["UR-001", "DR-189"], defined)).toEqual(["DR-189"]); }); it("flags an undefined UT/IT id, which the coverage orphan list cannot", () => { // The gap this closes: computeCoverage deliberately ignores UT/IT, so // UT-188 sat in three source files, defined nowhere, entirely unreported. expect(computeCoverage(["UT-188"], defined).orphaned).toEqual([]); expect(findDanglingIds(["UT-188"], defined)).toEqual(["UT-188"]); }); it("accepts every ID that is defined, requirement or test", () => { expect(findDanglingIds(["UR-001", "DR-001", "UT-001"], defined)).toEqual([]); }); it("deduplicates and sorts, so one typo is reported once", () => { expect( findDanglingIds(["DR-189", "DR-189", "UR-999", "DR-189"], defined) ).toEqual(["DR-189", "UR-999"]); }); it("ignores IDs whose prefix is not a known trace type", () => { // e.g. an unrelated "AB-123" caught by the loose ID regex. expect(findDanglingIds(["AB-123"], defined)).toEqual([]); }); }); describe("coverage threshold", () => { it("matches MIN_THRESHOLD in the Gitea traceability workflow", () => { // Two files must agree on the gate: the script (local `traces:coverage`) // and the workflow. Drift means the local gate and CI disagree about what // passes, which is how the 50%-while-actually-86% slack went unnoticed. const workflow = fs.readFileSync( path.resolve(HERE, "../.gitea/workflows/traceability-check.yml"), "utf-8" ); const match = workflow.match(/^\s*MIN_THRESHOLD=(\d+)\s*$/m); expect(match).not.toBeNull(); expect(Number(match![1])).toBe(MIN_COVERAGE_PERCENT); }); it("is a ratchet: never lower it to make a red build pass", () => { // Sanity bound. If coverage genuinely climbs, raise both numbers together. expect(MIN_COVERAGE_PERCENT).toBeGreaterThanOrEqual(82); expect(MIN_COVERAGE_PERCENT).toBeLessThanOrEqual(100); }); }); describe("computeCoverage", () => { const defined = { UR: 2, IR: 0, DR: 2, JA: 0, total: 4, ids: new Set(["UR-001", "UR-002", "DR-001", "DR-002"]), testIds: new Set(), }; it("computes coverage as traced ∩ defined over defined", () => { const traced = ["UR-001", "DR-001"]; const cov = computeCoverage(traced, defined); expect(cov.covered).toBe(2); expect(cov.total).toBe(4); expect(cov.percent).toBe(50); }); it("does not let a traced-but-undefined ID inflate the numerator", () => { // This is how a ratio exceeds 100%: a TRACES comment naming a typo'd or // deleted requirement counted as covered. const traced = ["UR-001", "DR-001", "DR-097"]; const cov = computeCoverage(traced, defined); expect(cov.covered).toBe(2); expect(cov.percent).toBe(50); }); it("reports traced-but-undefined IDs as orphaned so they get fixed", () => { const traced = ["UR-001", "DR-097", "JA-404"]; const cov = computeCoverage(traced, defined); expect(cov.orphaned).toEqual(["DR-097", "JA-404"]); }); it("has no orphans when every traced ID is defined", () => { const cov = computeCoverage(["UR-001", "UR-002"], defined); expect(cov.orphaned).toEqual([]); }); it("ignores UT/IT test IDs entirely — they are a separate taxonomy", () => { // UT/IT are defined in §4 of requirements.md, not among the four // requirement types. Treating them as orphans buries real typos in ~60 // lines of noise, and counting them would corrupt the ratio. const cov = computeCoverage(["UR-001", "UT-088", "IT-017"], defined); expect(cov.orphaned).toEqual([]); expect(cov.covered).toBe(1); }); it("reports 0% rather than dividing by zero for an empty trace set", () => { const cov = computeCoverage([], defined); expect(cov.covered).toBe(0); expect(cov.percent).toBe(0); }); it("reports 0% rather than NaN when nothing is defined", () => { const empty = { UR: 0, IR: 0, DR: 0, JA: 0, total: 0, ids: new Set(), testIds: new Set(), }; const cov = computeCoverage([], empty); expect(cov.percent).toBe(0); expect(Number.isNaN(cov.percent)).toBe(false); }); it("reports exactly 100% when all defined requirements are traced, never above", () => { const traced = ["UR-001", "UR-002", "DR-001", "DR-002"]; const cov = computeCoverage(traced, defined); expect(cov.percent).toBe(100); }); it("ignores duplicate traced IDs", () => { const traced = ["UR-001", "UR-001", "UR-001"]; const cov = computeCoverage(traced, defined); expect(cov.covered).toBe(1); }); }); describe("live requirements.md", () => { it("parses the real file to the counts the CI gate must use", () => { // Guards the specific regression: CI hardcoded UR/39, IR/24, DR/48, JA/3 // (total 114) while the real file had grown to 211. Update these numbers // deliberately when requirements are added — that edit is the signal the // denominator is live rather than frozen. const md = fs.readFileSync( path.resolve(HERE, "../docs/requirements.md"), "utf-8" ); const defined = countDefinedRequirements(md); expect(defined.UR).toBe(75); expect(defined.IR).toBe(32); // 192 = 187 + four requirements added independently on four audit branches, // plus DR-201 (lockscreen skip resolution). Originally 191 = 187 + four // that landed together: DR-189 (control-bar auto-hide), DR-198 (asset // scope/CSP), DR-199 (webview mixed-content) and DR-200 (the // POST_NOTIFICATIONS media-session exemption; renumbered from 198 on // merge, where it collided). Each branch bumped for its own — merged, // they sum. Resolve this by summing, never by taking one side. 193 adds // DR-202 (video keeps the display awake), 194 DR-203 (the handoff // transcode refusing the player's own load-error retry). expect(defined.DR).toBe(194); expect(defined.JA).toBe(36); expect(defined.total).toBe(337); }); });