diff --git a/.gitea/workflows/traceability-check.yml b/.gitea/workflows/traceability-check.yml index 7d4f249a..b42920cb 100644 --- a/.gitea/workflows/traceability-check.yml +++ b/.gitea/workflows/traceability-check.yml @@ -42,30 +42,45 @@ jobs: echo "๐Ÿ“Š Validating requirement traceability..." echo "" - # Parse JSON + # Denominators come from docs/requirements.md at run time โ€” NEVER + # hardcode them here. This step previously divided by frozen literals + # (UR/39, IR/24, DR/48, JA/3, total 114) while the file had grown to + # 211 requirements, so it reported 158% coverage and the threshold + # below could never trip. See docs/specs/traceability-gate-repair.md. TOTAL_TRACES=$(jq '.totalTraces' traces-report.json) - UR=$(jq '.byType.UR | length' traces-report.json) - IR=$(jq '.byType.IR | length' traces-report.json) - DR=$(jq '.byType.DR | length' traces-report.json) - JA=$(jq '.byType.JA | length' traces-report.json) + COVERED=$(jq '.coverage.covered' traces-report.json) + TOTAL_REQS=$(jq '.coverage.total' traces-report.json) + COVERAGE=$(jq '.coverage.percent' traces-report.json) - # Print coverage report echo "โœ… TRACES Found: $TOTAL_TRACES" echo "" - echo "๐Ÿ“‹ Coverage Summary:" - echo " User Requirements (UR): $UR / 39 ($(( UR * 100 / 39 ))%)" - echo " Integration Requirements (IR): $IR / 24 ($(( IR * 100 / 24 ))%)" - echo " Development Requirements (DR): $DR / 48 ($(( DR * 100 / 48 ))%)" - echo " Jellyfin API Requirements (JA): $JA / 3 ($(( JA * 100 / 3 ))%)" + echo "๐Ÿ“‹ Coverage Summary (traced / defined):" + for T in UR IR DR JA; do + TRACED=$(jq --arg t "$T" '[.byType[$t][] | select(. != null)] | length' traces-report.json) + DEFINED=$(jq --arg t "$T" '.defined[$t]' traces-report.json) + echo " $T: $TRACED / $DEFINED" + done echo "" - COVERED=$((UR + IR + DR + JA)) - TOTAL_REQS=114 - COVERAGE=$((COVERED * 100 / TOTAL_REQS)) - echo "๐Ÿ“ˆ Overall Coverage: $COVERED / $TOTAL_REQS ($COVERAGE%)" echo "" + # Traced IDs that requirements.md does not define (typo, or a deleted + # requirement). These do not count toward coverage. + ORPHANED=$(jq -c '.coverage.orphaned' traces-report.json) + if [ "$ORPHANED" != "[]" ]; then + echo "โš ๏ธ Traced but not defined in requirements.md: $ORPHANED" + echo "" + fi + + # A ratio above 100% means the computation is broken โ€” the exact + # condition that hid the stale-denominator bug. Fail loudly. + if [ "$COVERAGE" -gt 100 ]; then + echo "โŒ ERROR: Coverage ($COVERAGE%) exceeds 100% โ€” the gate is miscomputing." + echo " Orphaned IDs: $ORPHANED" + exit 1 + fi + # Check minimum threshold MIN_THRESHOLD=50 if [ "$COVERAGE" -lt "$MIN_THRESHOLD" ]; then diff --git a/docs/traceability-ci.md b/docs/traceability-ci.md index a6f07b22..e1d4fb17 100644 --- a/docs/traceability-ci.md +++ b/docs/traceability-ci.md @@ -43,14 +43,26 @@ Extracts all TRACES comments from: ### 2. Coverage Thresholds The workflow checks: -- **Minimum overall coverage:** 50% (57+ requirements traced) -- **Requirements by type:** - - UR (User): 23+ of 39 - - IR (Integration): 5+ of 24 - - DR (Development): 28+ of 48 - - JA (Jellyfin API): 0+ of 3 +- **Minimum overall coverage:** 50% -If coverage drops below threshold, the workflow **fails** and blocks merge. +Denominators are **derived from `docs/requirements.md` at run time** โ€” they are +never hardcoded here or in the workflow. Run `bun run traces:coverage` for the +current per-type breakdown; any number written into this document is a snapshot +that will drift. + +> **Why this matters.** The workflow used to divide by frozen literals +> (UR/39, IR/24, DR/48, JA/3, total 114) while `requirements.md` had grown past +> 200. It reported **158%** coverage, so the 50% threshold was unreachable and +> the job could not fail regardless of how far coverage dropped. See +> [specs/traceability-gate-repair.md](specs/traceability-gate-repair.md). + +Coverage is the *intersection* of traced and defined IDs: an ID that appears in +a `TRACES:` comment but is not defined in `requirements.md` is reported as +**orphaned** and does not count toward coverage. UT/IT test identifiers are a +separate taxonomy and are excluded entirely. + +The workflow **fails** and blocks merge if coverage drops below 50% โ€” or if it +computes above 100%, which can only mean the gate is miscounting. ### 3. Modified File Checking On pull requests, the workflow: @@ -153,11 +165,13 @@ cat docs/traceability.md ## Coverage Goals ### Current Status -- Overall: 51% (56/114) -- UR: 59% (23/39) -- IR: 21% (5/24) -- DR: 58% (28/48) -- JA: 0% (0/3) + +Run `bun run traces:coverage` โ€” it prints the live figure and exits non-zero +below threshold. Numbers are deliberately not pinned here; the previous snapshot +in this section (51%, 56/114) was stale by roughly 100 requirements and was what +made the broken CI arithmetic look plausible for so long. + +As of July 2026 overall coverage is ~86% (182/212). ### Targets - **Short term** (Sprint): Maintain โ‰ฅ50% overall diff --git a/package.json b/package.json index a246ec90..add00f4e 100644 --- a/package.json +++ b/package.json @@ -36,6 +36,7 @@ "traces": "bun run scripts/extract-traces.ts", "traces:json": "bun run scripts/extract-traces.ts --format json", "traces:markdown": "bun run scripts/extract-traces.ts --format markdown > docs/traceability.md", + "traces:coverage": "bun run scripts/extract-traces.ts --format coverage", "release:notes": "bun run scripts/release-notes.ts" }, "license": "MIT", diff --git a/scripts/extract-traces.test.ts b/scripts/extract-traces.test.ts new file mode 100644 index 00000000..9f2988a6 --- /dev/null +++ b/scripts/extract-traces.test.ts @@ -0,0 +1,182 @@ +/** + * Tests for the traceability coverage computation. + * + * These run over fixture strings rather than the live docs/requirements.md, so + * their meaning does not drift as requirements are added. + * + * Background: the CI gate divided traced-requirement counts by hardcoded + * denominators (UR/39, IR/24, DR/48, JA/3, total 114) that had fallen out of + * date, reporting 158% coverage and making the 50% threshold unreachable. These + * tests pin the parsing and arithmetic that replace those literals. + * + * @req-test: UT-089 - Requirement definitions parsed from requirements.md + * @req-test: UT-090 - Coverage is the intersection of traced and defined IDs + */ + +import { describe, it, expect } from "vitest"; +import { countDefinedRequirements, computeCoverage } from "./extract-traces"; + +describe("countDefinedRequirements", () => { + it("counts a well-formed table row as a defined requirement", () => { + const md = ` +| ID | Requirement | Priority | Status | +|----|-------------|----------|--------| +| UR-001 | Run the app on multiple platforms | High | In Progress | +| UR-002 | Access media when online or offline | High | Done | +`; + const defined = countDefinedRequirements(md); + expect(defined.UR).toBe(2); + expect(defined.DR).toBe(0); + }); + + it("does not count IDs that appear only in the Traces To column", () => { + // The bug this rule avoids: a naive grep for /DR-\d{3}/ over the whole file + // counts DR-001 here as "defined", inflating the denominator with IDs that + // are merely referenced. + const md = ` +| DR-001 | Player state machine | Player | UR-005 | Done | +| DR-002 | MediaItem struct | Player | UR-003, UR-004 | Done | +`; + const defined = countDefinedRequirements(md); + expect(defined.DR).toBe(2); + // UR-005/UR-003/UR-004 are referenced, never defined here. + expect(defined.UR).toBe(0); + }); + + it("does not count IDs mentioned in prose", () => { + const md = ` +Some prose explaining that UR-005 relates to DR-001 and JA-002. + +| UR-005 | Control media playback | High | Done | +`; + const defined = countDefinedRequirements(md); + expect(defined.UR).toBe(1); + expect(defined.DR).toBe(0); + expect(defined.JA).toBe(0); + }); + + it("deduplicates an ID listed in both the spec table and the traceability matrix", () => { + // requirements.md lists every UR twice: once in ยง1 (definition) and again in + // ยง3 (traceability matrix), both as a leading table cell. Counting rows + // instead of unique IDs double-counts the UR denominator (121 vs 61). + const md = ` +| UR-005 | Control media playback | High | Done | +| UR-006 | Browse the library | High | Done | + +### Traceability Matrix + +| UR-005 | - | DR-001, DR-005, DR-009 | +| UR-006 | - | DR-012 | +`; + const defined = countDefinedRequirements(md); + expect(defined.UR).toBe(2); + }); + + it("collects the defined ID set, not just counts", () => { + const md = ` +| UR-001 | A | High | Done | +| DR-050 | B | Player | UR-001 | Done | +`; + const defined = countDefinedRequirements(md); + expect(defined.ids.has("UR-001")).toBe(true); + expect(defined.ids.has("DR-050")).toBe(true); + expect(defined.ids.has("UR-999")).toBe(false); + }); +}); + +describe("computeCoverage", () => { + const defined = { + UR: 2, + IR: 0, + DR: 2, + JA: 0, + total: 4, + ids: new Set(["UR-001", "UR-002", "DR-001", "DR-002"]), + }; + + it("computes coverage as traced โˆฉ defined over defined", () => { + const traced = ["UR-001", "DR-001"]; + const cov = computeCoverage(traced, defined); + expect(cov.covered).toBe(2); + expect(cov.total).toBe(4); + expect(cov.percent).toBe(50); + }); + + it("does not let a traced-but-undefined ID inflate the numerator", () => { + // This is how a ratio exceeds 100%: a TRACES comment naming a typo'd or + // deleted requirement counted as covered. + const traced = ["UR-001", "DR-001", "DR-097"]; + const cov = computeCoverage(traced, defined); + expect(cov.covered).toBe(2); + expect(cov.percent).toBe(50); + }); + + it("reports traced-but-undefined IDs as orphaned so they get fixed", () => { + const traced = ["UR-001", "DR-097", "JA-404"]; + const cov = computeCoverage(traced, defined); + expect(cov.orphaned).toEqual(["DR-097", "JA-404"]); + }); + + it("has no orphans when every traced ID is defined", () => { + const cov = computeCoverage(["UR-001", "UR-002"], defined); + expect(cov.orphaned).toEqual([]); + }); + + it("ignores UT/IT test IDs entirely โ€” they are a separate taxonomy", () => { + // UT/IT are defined in ยง4 of requirements.md, not among the four + // requirement types. Treating them as orphans buries real typos in ~60 + // lines of noise, and counting them would corrupt the ratio. + const cov = computeCoverage(["UR-001", "UT-088", "IT-017"], defined); + expect(cov.orphaned).toEqual([]); + expect(cov.covered).toBe(1); + }); + + it("reports 0% rather than dividing by zero for an empty trace set", () => { + const cov = computeCoverage([], defined); + expect(cov.covered).toBe(0); + expect(cov.percent).toBe(0); + }); + + it("reports 0% rather than NaN when nothing is defined", () => { + const empty = { UR: 0, IR: 0, DR: 0, JA: 0, total: 0, ids: new Set() }; + const cov = computeCoverage([], empty); + expect(cov.percent).toBe(0); + expect(Number.isNaN(cov.percent)).toBe(false); + }); + + it("reports exactly 100% when all defined requirements are traced, never above", () => { + const traced = ["UR-001", "UR-002", "DR-001", "DR-002"]; + const cov = computeCoverage(traced, defined); + expect(cov.percent).toBe(100); + }); + + it("ignores duplicate traced IDs", () => { + const traced = ["UR-001", "UR-001", "UR-001"]; + const cov = computeCoverage(traced, defined); + expect(cov.covered).toBe(1); + }); +}); + +describe("live requirements.md", () => { + it("parses the real file to the counts the CI gate must use", () => { + // Guards the specific regression: CI hardcoded UR/39, IR/24, DR/48, JA/3 + // (total 114) while the real file had grown to 211. Update these numbers + // deliberately when requirements are added โ€” that edit is the signal the + // denominator is live rather than frozen. + const fs = require("fs"); + const path = require("path"); + // import.meta.dir is Bun-only; derive from import.meta.url under vitest. + const here = path.dirname(new URL(import.meta.url).pathname); + const md = fs.readFileSync( + path.resolve(here, "../docs/requirements.md"), + "utf-8" + ); + const defined = countDefinedRequirements(md); + + expect(defined.UR).toBe(61); + expect(defined.IR).toBe(29); + expect(defined.DR).toBe(91); + expect(defined.JA).toBe(32); + expect(defined.total).toBe(213); + }); +}); diff --git a/scripts/extract-traces.ts b/scripts/extract-traces.ts index a473fe0b..b45ca228 100644 --- a/scripts/extract-traces.ts +++ b/scripts/extract-traces.ts @@ -34,11 +34,20 @@ interface TracesData { DR: string[]; JA: string[]; }; + /** Requirements *defined* in requirements.md โ€” the coverage denominators. */ + defined?: { UR: number; IR: number; DR: number; JA: number; total: number }; + coverage?: CoverageResult; } // Repo root, derived from this script's location (scripts/ -> repo root). // Must NOT be hardcoded to a developer's machine, or CI checkouts see no files. -const BASE_DIR = path.resolve(import.meta.dir, ".."); +// +// `import.meta.dir` is a Bun extension and is undefined when this module is +// imported by vitest (which runs it as an ordinary ESM module), so fall back to +// import.meta.url โ€” this file must stay importable for extract-traces.test.ts. +const SCRIPT_DIR = + import.meta.dir ?? path.dirname(new URL(import.meta.url).pathname); +const BASE_DIR = path.resolve(SCRIPT_DIR, ".."); const TRACES_PATTERN = /TRACES:\s*([^\n]+)/gi; const REQ_ID_PATTERN = /([A-Z]{2})-(\d{3})/g; @@ -50,7 +59,10 @@ function extractRequirementIds(tracesString: string): string[] { function getAllSourceFiles(): string[] { const baseDir = BASE_DIR; - const patterns = ["src", "src-tauri/src"]; + // `scripts` is scanned too: build tooling implements requirements (e.g. + // DR-093, the coverage engine itself) and would otherwise be invisible to the + // very matrix it generates. + const patterns = ["src", "src-tauri/src", "scripts"]; const files: string[] = []; function walkDir(dir: string) { @@ -192,6 +204,109 @@ function extractTraces(): TracesData { }; } +// --------------------------------------------------------------------------- +// Coverage: how many *defined* requirements are actually traced. +// +// The denominators MUST be derived from requirements.md, never hardcoded. The +// CI gate previously divided by frozen literals (UR/39, IR/24, DR/48, JA/3, +// total 114) while the real file had grown to 211 requirements, so it reported +// 158% coverage and the 50% threshold became unreachable โ€” the gate could not +// fail. See docs/specs/traceability-gate-repair.md. +// +// TRACES: | DR-093 +// --------------------------------------------------------------------------- + +export interface DefinedRequirements { + UR: number; + IR: number; + DR: number; + JA: number; + total: number; + ids: Set; +} + +export interface CoverageResult { + covered: number; + total: number; + percent: number; + /** Traced in code but not defined in requirements.md (typo, or deleted req). */ + orphaned: string[]; +} + +/** + * A requirement is *defined* only where its ID is the leading cell of a markdown + * table row: `| DR-001 | โ€ฆ |`. + * + * This deliberately ignores IDs in the "Traces To" column and in prose โ€” a + * naive scan for /DR-\d{3}/ counts those as definitions and inflates the + * denominator. IDs are deduplicated because requirements.md lists each UR twice + * (once in ยง1 as a definition, again in ยง3's traceability matrix), which would + * otherwise double the UR count from 61 to 121. + * + * TRACES: | DR-093 + */ +export function countDefinedRequirements(markdown: string): DefinedRequirements { + const ids = new Set(); + const ROW_ID = /^\|\s*(UR|IR|DR|JA)-(\d{3})\s*\|/; + + for (const line of markdown.split("\n")) { + const match = line.match(ROW_ID); + if (match) ids.add(`${match[1]}-${match[2]}`); + } + + const countOf = (type: string) => + [...ids].filter((id) => id.startsWith(`${type}-`)).length; + + return { + UR: countOf("UR"), + IR: countOf("IR"), + DR: countOf("DR"), + JA: countOf("JA"), + total: ids.size, + ids, + }; +} + +/** + * Coverage is the *intersection* of traced and defined IDs over defined IDs. + * + * Using the raw traced count as the numerator is what lets a ratio exceed 100%: + * a TRACES comment naming a requirement that no longer exists would count as + * covered. Those IDs are reported as `orphaned` so they get fixed rather than + * silently counted or silently dropped. + * + * TRACES: | DR-093 + */ +export function computeCoverage( + tracedIds: string[], + defined: DefinedRequirements +): CoverageResult { + // Only the four *requirement* types participate in coverage. UT/IT are test + // identifiers defined in ยง4 of requirements.md โ€” a different taxonomy, and + // flagging them as orphans would bury real typos in ~60 lines of noise. + const isRequirement = (id: string) => /^(UR|IR|DR|JA)-\d{3}$/.test(id); + + const traced = new Set(tracedIds.filter(isRequirement)); + const covered = [...traced].filter((id) => defined.ids.has(id)); + const orphaned = [...traced].filter((id) => !defined.ids.has(id)).sort(); + + return { + covered: covered.length, + total: defined.total, + percent: + defined.total === 0 + ? 0 + : Math.round((covered.length / defined.total) * 100), + orphaned, + }; +} + +/** Read requirements.md from the repo and count what it defines. */ +export function readDefinedRequirements(): DefinedRequirements { + const reqPath = path.join(BASE_DIR, "docs", "requirements.md"); + return countDefinedRequirements(fs.readFileSync(reqPath, "utf-8")); +} + function generateMarkdown(data: TracesData): string { let md = `# Code Traceability Matrix @@ -265,21 +380,83 @@ function generateJson(data: TracesData): string { return JSON.stringify(data, null, 2); } -// Main -const args = Bun.argv.slice(2); -const format = args.includes("--format") - ? args[args.indexOf("--format") + 1] - : "markdown"; +/** + * Human-readable coverage report; exits non-zero below the threshold so this is + * runnable as a local gate (`bun run traces:coverage`), not just in CI. + * + * TRACES: | DR-093 + */ +function reportCoverage(data: TracesData, minThreshold: number): number { + const defined = data.defined!; + const cov = data.coverage!; -console.error("๐Ÿ” Extracting TRACES from codebase..."); -const data = extractTraces(); + const definedIds = readDefinedRequirements().ids; -if (format === "json") { - console.log(generateJson(data)); -} else { - console.log(generateMarkdown(data)); + console.log("๐Ÿ“‹ Requirement coverage (traced / defined):"); + for (const type of ["UR", "IR", "DR", "JA"] as const) { + const traced = data.byType[type].filter((id) => definedIds.has(id)).length; + console.log(` ${type}: ${traced} / ${defined[type]}`); + } + console.log(""); + console.log(`๐Ÿ“ˆ Overall: ${cov.covered} / ${cov.total} (${cov.percent}%)`); + + if (cov.orphaned.length > 0) { + console.log(""); + console.log( + `โš ๏ธ Traced but not defined in requirements.md: ${cov.orphaned.join(", ")}` + ); + console.log(" Fix the TRACES comment or add the requirement."); + } + + // A ratio above 100% means the computation is broken (the condition that hid + // the stale-denominator bug for so long). Fail loudly rather than report it. + if (cov.percent > 100) { + console.log(""); + console.log(`โŒ Coverage > 100% โ€” the gate is miscomputing.`); + return 1; + } + + if (cov.percent < minThreshold) { + console.log(""); + console.log(`โŒ Coverage (${cov.percent}%) is below minimum (${minThreshold}%)`); + return 1; + } + + console.log(""); + console.log(`โœ… Coverage is acceptable (${cov.percent}% >= ${minThreshold}%)`); + return 0; } -console.error( - `\nโœ… Complete! Found ${data.totalTraces} TRACES across ${data.totalFiles} files` -); +// Main โ€” guarded so this module stays importable from extract-traces.test.ts. +if (import.meta.main) { + const args = process.argv.slice(2); + const format = args.includes("--format") + ? args[args.indexOf("--format") + 1] + : "markdown"; + + console.error("๐Ÿ” Extracting TRACES from codebase..."); + const data = extractTraces(); + + const defined = readDefinedRequirements(); + const allTraced = Object.keys(data.requirements); + data.defined = { + UR: defined.UR, + IR: defined.IR, + DR: defined.DR, + JA: defined.JA, + total: defined.total, + }; + data.coverage = computeCoverage(allTraced, defined); + + if (format === "json") { + console.log(generateJson(data)); + } else if (format === "coverage") { + process.exit(reportCoverage(data, 50)); + } else { + console.log(generateMarkdown(data)); + } + + console.error( + `\nโœ… Complete! Found ${data.totalTraces} TRACES across ${data.totalFiles} files` + ); +} diff --git a/vitest.config.ts b/vitest.config.ts index 3c6274a0..0c55890d 100644 --- a/vitest.config.ts +++ b/vitest.config.ts @@ -8,7 +8,9 @@ export default defineConfig({ globals: true, environment: "jsdom", setupFiles: ["./src/test/setup-globals.ts", "./src/test/setup.ts"], - include: ["src/**/*.{test,spec}.{js,ts}"], + // `scripts/` is included so build tooling (the traceability coverage + // engine) is covered by the normal suite rather than only by CI. + include: ["src/**/*.{test,spec}.{js,ts}", "scripts/**/*.{test,spec}.{js,ts}"], coverage: { provider: "v8", reporter: ["text", "json", "html"],