fix(ci): derive traceability denominators from requirements.md (DR-093)
The coverage gate divided traced counts by hardcoded literals (UR/39, IR/24, DR/48, JA/3, TOTAL_REQS=114) that had fallen out of date as requirements grew to 211. It reported 158% coverage — JA alone printed 800% — so the 50% threshold was mathematically unreachable and the job could not fail. Coverage could have collapsed to 30% and CI would still have printed a green tick. Real coverage is 86%. The number was fine; the gate was dead. extract-traces.ts now owns both sides of the fraction: - countDefinedRequirements() counts an ID only where it leads a markdown table row, ignoring the "Traces To" column and prose. IDs are deduplicated because requirements.md lists every UR twice (§1 definition + §3 matrix), which would otherwise report UR as 121/61. - computeCoverage() uses the intersection of traced and defined IDs, so a TRACES comment naming a deleted or typo'd requirement is reported as `orphaned` rather than inflating the ratio past 100%. UT/IT test identifiers are excluded as a separate taxonomy. - CI reads .coverage.percent and fails on <50% or >100%; a >100% reading is now a hard error rather than the condition that hid this bug. - New `bun run traces:coverage` runs the same computation locally. - scripts/ added to the scan roots — the coverage tool was invisible to the matrix it generates. Tests written first (15, over fixtures so they don't drift as requirements are added). vitest include widened to scripts/** so build tooling is covered by the normal suite. Verified empirically rather than by inspection: forcing the threshold to 99% fails; adding a requirement lowers coverage 86%→85%; a TRACES: DR-999 lands in `orphaned` without changing `covered`. traceability-ci.md documented the same stale numbers and would have let the broken arithmetic be reconstructed — replaced with a pointer to the live command.
This commit is contained in:
@@ -0,0 +1,182 @@
|
||||
/**
|
||||
* Tests for the traceability coverage computation.
|
||||
*
|
||||
* These run over fixture strings rather than the live docs/requirements.md, so
|
||||
* their meaning does not drift as requirements are added.
|
||||
*
|
||||
* Background: the CI gate divided traced-requirement counts by hardcoded
|
||||
* denominators (UR/39, IR/24, DR/48, JA/3, total 114) that had fallen out of
|
||||
* date, reporting 158% coverage and making the 50% threshold unreachable. These
|
||||
* tests pin the parsing and arithmetic that replace those literals.
|
||||
*
|
||||
* @req-test: UT-089 - Requirement definitions parsed from requirements.md
|
||||
* @req-test: UT-090 - Coverage is the intersection of traced and defined IDs
|
||||
*/
|
||||
|
||||
import { describe, it, expect } from "vitest";
|
||||
import { countDefinedRequirements, computeCoverage } from "./extract-traces";
|
||||
|
||||
describe("countDefinedRequirements", () => {
|
||||
it("counts a well-formed table row as a defined requirement", () => {
|
||||
const md = `
|
||||
| ID | Requirement | Priority | Status |
|
||||
|----|-------------|----------|--------|
|
||||
| UR-001 | Run the app on multiple platforms | High | In Progress |
|
||||
| UR-002 | Access media when online or offline | High | Done |
|
||||
`;
|
||||
const defined = countDefinedRequirements(md);
|
||||
expect(defined.UR).toBe(2);
|
||||
expect(defined.DR).toBe(0);
|
||||
});
|
||||
|
||||
it("does not count IDs that appear only in the Traces To column", () => {
|
||||
// The bug this rule avoids: a naive grep for /DR-\d{3}/ over the whole file
|
||||
// counts DR-001 here as "defined", inflating the denominator with IDs that
|
||||
// are merely referenced.
|
||||
const md = `
|
||||
| DR-001 | Player state machine | Player | UR-005 | Done |
|
||||
| DR-002 | MediaItem struct | Player | UR-003, UR-004 | Done |
|
||||
`;
|
||||
const defined = countDefinedRequirements(md);
|
||||
expect(defined.DR).toBe(2);
|
||||
// UR-005/UR-003/UR-004 are referenced, never defined here.
|
||||
expect(defined.UR).toBe(0);
|
||||
});
|
||||
|
||||
it("does not count IDs mentioned in prose", () => {
|
||||
const md = `
|
||||
Some prose explaining that UR-005 relates to DR-001 and JA-002.
|
||||
|
||||
| UR-005 | Control media playback | High | Done |
|
||||
`;
|
||||
const defined = countDefinedRequirements(md);
|
||||
expect(defined.UR).toBe(1);
|
||||
expect(defined.DR).toBe(0);
|
||||
expect(defined.JA).toBe(0);
|
||||
});
|
||||
|
||||
it("deduplicates an ID listed in both the spec table and the traceability matrix", () => {
|
||||
// requirements.md lists every UR twice: once in §1 (definition) and again in
|
||||
// §3 (traceability matrix), both as a leading table cell. Counting rows
|
||||
// instead of unique IDs double-counts the UR denominator (121 vs 61).
|
||||
const md = `
|
||||
| UR-005 | Control media playback | High | Done |
|
||||
| UR-006 | Browse the library | High | Done |
|
||||
|
||||
### Traceability Matrix
|
||||
|
||||
| UR-005 | - | DR-001, DR-005, DR-009 |
|
||||
| UR-006 | - | DR-012 |
|
||||
`;
|
||||
const defined = countDefinedRequirements(md);
|
||||
expect(defined.UR).toBe(2);
|
||||
});
|
||||
|
||||
it("collects the defined ID set, not just counts", () => {
|
||||
const md = `
|
||||
| UR-001 | A | High | Done |
|
||||
| DR-050 | B | Player | UR-001 | Done |
|
||||
`;
|
||||
const defined = countDefinedRequirements(md);
|
||||
expect(defined.ids.has("UR-001")).toBe(true);
|
||||
expect(defined.ids.has("DR-050")).toBe(true);
|
||||
expect(defined.ids.has("UR-999")).toBe(false);
|
||||
});
|
||||
});
|
||||
|
||||
describe("computeCoverage", () => {
|
||||
const defined = {
|
||||
UR: 2,
|
||||
IR: 0,
|
||||
DR: 2,
|
||||
JA: 0,
|
||||
total: 4,
|
||||
ids: new Set(["UR-001", "UR-002", "DR-001", "DR-002"]),
|
||||
};
|
||||
|
||||
it("computes coverage as traced ∩ defined over defined", () => {
|
||||
const traced = ["UR-001", "DR-001"];
|
||||
const cov = computeCoverage(traced, defined);
|
||||
expect(cov.covered).toBe(2);
|
||||
expect(cov.total).toBe(4);
|
||||
expect(cov.percent).toBe(50);
|
||||
});
|
||||
|
||||
it("does not let a traced-but-undefined ID inflate the numerator", () => {
|
||||
// This is how a ratio exceeds 100%: a TRACES comment naming a typo'd or
|
||||
// deleted requirement counted as covered.
|
||||
const traced = ["UR-001", "DR-001", "DR-097"];
|
||||
const cov = computeCoverage(traced, defined);
|
||||
expect(cov.covered).toBe(2);
|
||||
expect(cov.percent).toBe(50);
|
||||
});
|
||||
|
||||
it("reports traced-but-undefined IDs as orphaned so they get fixed", () => {
|
||||
const traced = ["UR-001", "DR-097", "JA-404"];
|
||||
const cov = computeCoverage(traced, defined);
|
||||
expect(cov.orphaned).toEqual(["DR-097", "JA-404"]);
|
||||
});
|
||||
|
||||
it("has no orphans when every traced ID is defined", () => {
|
||||
const cov = computeCoverage(["UR-001", "UR-002"], defined);
|
||||
expect(cov.orphaned).toEqual([]);
|
||||
});
|
||||
|
||||
it("ignores UT/IT test IDs entirely — they are a separate taxonomy", () => {
|
||||
// UT/IT are defined in §4 of requirements.md, not among the four
|
||||
// requirement types. Treating them as orphans buries real typos in ~60
|
||||
// lines of noise, and counting them would corrupt the ratio.
|
||||
const cov = computeCoverage(["UR-001", "UT-088", "IT-017"], defined);
|
||||
expect(cov.orphaned).toEqual([]);
|
||||
expect(cov.covered).toBe(1);
|
||||
});
|
||||
|
||||
it("reports 0% rather than dividing by zero for an empty trace set", () => {
|
||||
const cov = computeCoverage([], defined);
|
||||
expect(cov.covered).toBe(0);
|
||||
expect(cov.percent).toBe(0);
|
||||
});
|
||||
|
||||
it("reports 0% rather than NaN when nothing is defined", () => {
|
||||
const empty = { UR: 0, IR: 0, DR: 0, JA: 0, total: 0, ids: new Set<string>() };
|
||||
const cov = computeCoverage([], empty);
|
||||
expect(cov.percent).toBe(0);
|
||||
expect(Number.isNaN(cov.percent)).toBe(false);
|
||||
});
|
||||
|
||||
it("reports exactly 100% when all defined requirements are traced, never above", () => {
|
||||
const traced = ["UR-001", "UR-002", "DR-001", "DR-002"];
|
||||
const cov = computeCoverage(traced, defined);
|
||||
expect(cov.percent).toBe(100);
|
||||
});
|
||||
|
||||
it("ignores duplicate traced IDs", () => {
|
||||
const traced = ["UR-001", "UR-001", "UR-001"];
|
||||
const cov = computeCoverage(traced, defined);
|
||||
expect(cov.covered).toBe(1);
|
||||
});
|
||||
});
|
||||
|
||||
describe("live requirements.md", () => {
|
||||
it("parses the real file to the counts the CI gate must use", () => {
|
||||
// Guards the specific regression: CI hardcoded UR/39, IR/24, DR/48, JA/3
|
||||
// (total 114) while the real file had grown to 211. Update these numbers
|
||||
// deliberately when requirements are added — that edit is the signal the
|
||||
// denominator is live rather than frozen.
|
||||
const fs = require("fs");
|
||||
const path = require("path");
|
||||
// import.meta.dir is Bun-only; derive from import.meta.url under vitest.
|
||||
const here = path.dirname(new URL(import.meta.url).pathname);
|
||||
const md = fs.readFileSync(
|
||||
path.resolve(here, "../docs/requirements.md"),
|
||||
"utf-8"
|
||||
);
|
||||
const defined = countDefinedRequirements(md);
|
||||
|
||||
expect(defined.UR).toBe(61);
|
||||
expect(defined.IR).toBe(29);
|
||||
expect(defined.DR).toBe(91);
|
||||
expect(defined.JA).toBe(32);
|
||||
expect(defined.total).toBe(213);
|
||||
});
|
||||
});
|
||||
Reference in New Issue
Block a user