fix(ci): derive traceability denominators from requirements.md (DR-093)
The coverage gate divided traced counts by hardcoded literals (UR/39, IR/24, DR/48, JA/3, TOTAL_REQS=114) that had fallen out of date as requirements grew to 211. It reported 158% coverage — JA alone printed 800% — so the 50% threshold was mathematically unreachable and the job could not fail. Coverage could have collapsed to 30% and CI would still have printed a green tick. Real coverage is 86%. The number was fine; the gate was dead. extract-traces.ts now owns both sides of the fraction: - countDefinedRequirements() counts an ID only where it leads a markdown table row, ignoring the "Traces To" column and prose. IDs are deduplicated because requirements.md lists every UR twice (§1 definition + §3 matrix), which would otherwise report UR as 121/61. - computeCoverage() uses the intersection of traced and defined IDs, so a TRACES comment naming a deleted or typo'd requirement is reported as `orphaned` rather than inflating the ratio past 100%. UT/IT test identifiers are excluded as a separate taxonomy. - CI reads .coverage.percent and fails on <50% or >100%; a >100% reading is now a hard error rather than the condition that hid this bug. - New `bun run traces:coverage` runs the same computation locally. - scripts/ added to the scan roots — the coverage tool was invisible to the matrix it generates. Tests written first (15, over fixtures so they don't drift as requirements are added). vitest include widened to scripts/** so build tooling is covered by the normal suite. Verified empirically rather than by inspection: forcing the threshold to 99% fails; adding a requirement lowers coverage 86%→85%; a TRACES: DR-999 lands in `orphaned` without changing `covered`. traceability-ci.md documented the same stale numbers and would have let the broken arithmetic be reconstructed — replaced with a pointer to the live command.
This commit is contained in:
+193
-16
@@ -34,11 +34,20 @@ interface TracesData {
|
||||
DR: string[];
|
||||
JA: string[];
|
||||
};
|
||||
/** Requirements *defined* in requirements.md — the coverage denominators. */
|
||||
defined?: { UR: number; IR: number; DR: number; JA: number; total: number };
|
||||
coverage?: CoverageResult;
|
||||
}
|
||||
|
||||
// Repo root, derived from this script's location (scripts/ -> repo root).
|
||||
// Must NOT be hardcoded to a developer's machine, or CI checkouts see no files.
|
||||
const BASE_DIR = path.resolve(import.meta.dir, "..");
|
||||
//
|
||||
// `import.meta.dir` is a Bun extension and is undefined when this module is
|
||||
// imported by vitest (which runs it as an ordinary ESM module), so fall back to
|
||||
// import.meta.url — this file must stay importable for extract-traces.test.ts.
|
||||
const SCRIPT_DIR =
|
||||
import.meta.dir ?? path.dirname(new URL(import.meta.url).pathname);
|
||||
const BASE_DIR = path.resolve(SCRIPT_DIR, "..");
|
||||
|
||||
const TRACES_PATTERN = /TRACES:\s*([^\n]+)/gi;
|
||||
const REQ_ID_PATTERN = /([A-Z]{2})-(\d{3})/g;
|
||||
@@ -50,7 +59,10 @@ function extractRequirementIds(tracesString: string): string[] {
|
||||
|
||||
function getAllSourceFiles(): string[] {
|
||||
const baseDir = BASE_DIR;
|
||||
const patterns = ["src", "src-tauri/src"];
|
||||
// `scripts` is scanned too: build tooling implements requirements (e.g.
|
||||
// DR-093, the coverage engine itself) and would otherwise be invisible to the
|
||||
// very matrix it generates.
|
||||
const patterns = ["src", "src-tauri/src", "scripts"];
|
||||
const files: string[] = [];
|
||||
|
||||
function walkDir(dir: string) {
|
||||
@@ -192,6 +204,109 @@ function extractTraces(): TracesData {
|
||||
};
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Coverage: how many *defined* requirements are actually traced.
|
||||
//
|
||||
// The denominators MUST be derived from requirements.md, never hardcoded. The
|
||||
// CI gate previously divided by frozen literals (UR/39, IR/24, DR/48, JA/3,
|
||||
// total 114) while the real file had grown to 211 requirements, so it reported
|
||||
// 158% coverage and the 50% threshold became unreachable — the gate could not
|
||||
// fail. See docs/specs/traceability-gate-repair.md.
|
||||
//
|
||||
// TRACES: | DR-093
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
export interface DefinedRequirements {
|
||||
UR: number;
|
||||
IR: number;
|
||||
DR: number;
|
||||
JA: number;
|
||||
total: number;
|
||||
ids: Set<string>;
|
||||
}
|
||||
|
||||
export interface CoverageResult {
|
||||
covered: number;
|
||||
total: number;
|
||||
percent: number;
|
||||
/** Traced in code but not defined in requirements.md (typo, or deleted req). */
|
||||
orphaned: string[];
|
||||
}
|
||||
|
||||
/**
|
||||
* A requirement is *defined* only where its ID is the leading cell of a markdown
|
||||
* table row: `| DR-001 | … |`.
|
||||
*
|
||||
* This deliberately ignores IDs in the "Traces To" column and in prose — a
|
||||
* naive scan for /DR-\d{3}/ counts those as definitions and inflates the
|
||||
* denominator. IDs are deduplicated because requirements.md lists each UR twice
|
||||
* (once in §1 as a definition, again in §3's traceability matrix), which would
|
||||
* otherwise double the UR count from 61 to 121.
|
||||
*
|
||||
* TRACES: | DR-093
|
||||
*/
|
||||
export function countDefinedRequirements(markdown: string): DefinedRequirements {
|
||||
const ids = new Set<string>();
|
||||
const ROW_ID = /^\|\s*(UR|IR|DR|JA)-(\d{3})\s*\|/;
|
||||
|
||||
for (const line of markdown.split("\n")) {
|
||||
const match = line.match(ROW_ID);
|
||||
if (match) ids.add(`${match[1]}-${match[2]}`);
|
||||
}
|
||||
|
||||
const countOf = (type: string) =>
|
||||
[...ids].filter((id) => id.startsWith(`${type}-`)).length;
|
||||
|
||||
return {
|
||||
UR: countOf("UR"),
|
||||
IR: countOf("IR"),
|
||||
DR: countOf("DR"),
|
||||
JA: countOf("JA"),
|
||||
total: ids.size,
|
||||
ids,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Coverage is the *intersection* of traced and defined IDs over defined IDs.
|
||||
*
|
||||
* Using the raw traced count as the numerator is what lets a ratio exceed 100%:
|
||||
* a TRACES comment naming a requirement that no longer exists would count as
|
||||
* covered. Those IDs are reported as `orphaned` so they get fixed rather than
|
||||
* silently counted or silently dropped.
|
||||
*
|
||||
* TRACES: | DR-093
|
||||
*/
|
||||
export function computeCoverage(
|
||||
tracedIds: string[],
|
||||
defined: DefinedRequirements
|
||||
): CoverageResult {
|
||||
// Only the four *requirement* types participate in coverage. UT/IT are test
|
||||
// identifiers defined in §4 of requirements.md — a different taxonomy, and
|
||||
// flagging them as orphans would bury real typos in ~60 lines of noise.
|
||||
const isRequirement = (id: string) => /^(UR|IR|DR|JA)-\d{3}$/.test(id);
|
||||
|
||||
const traced = new Set(tracedIds.filter(isRequirement));
|
||||
const covered = [...traced].filter((id) => defined.ids.has(id));
|
||||
const orphaned = [...traced].filter((id) => !defined.ids.has(id)).sort();
|
||||
|
||||
return {
|
||||
covered: covered.length,
|
||||
total: defined.total,
|
||||
percent:
|
||||
defined.total === 0
|
||||
? 0
|
||||
: Math.round((covered.length / defined.total) * 100),
|
||||
orphaned,
|
||||
};
|
||||
}
|
||||
|
||||
/** Read requirements.md from the repo and count what it defines. */
|
||||
export function readDefinedRequirements(): DefinedRequirements {
|
||||
const reqPath = path.join(BASE_DIR, "docs", "requirements.md");
|
||||
return countDefinedRequirements(fs.readFileSync(reqPath, "utf-8"));
|
||||
}
|
||||
|
||||
function generateMarkdown(data: TracesData): string {
|
||||
let md = `# Code Traceability Matrix
|
||||
|
||||
@@ -265,21 +380,83 @@ function generateJson(data: TracesData): string {
|
||||
return JSON.stringify(data, null, 2);
|
||||
}
|
||||
|
||||
// Main
|
||||
const args = Bun.argv.slice(2);
|
||||
const format = args.includes("--format")
|
||||
? args[args.indexOf("--format") + 1]
|
||||
: "markdown";
|
||||
/**
|
||||
* Human-readable coverage report; exits non-zero below the threshold so this is
|
||||
* runnable as a local gate (`bun run traces:coverage`), not just in CI.
|
||||
*
|
||||
* TRACES: | DR-093
|
||||
*/
|
||||
function reportCoverage(data: TracesData, minThreshold: number): number {
|
||||
const defined = data.defined!;
|
||||
const cov = data.coverage!;
|
||||
|
||||
console.error("🔍 Extracting TRACES from codebase...");
|
||||
const data = extractTraces();
|
||||
const definedIds = readDefinedRequirements().ids;
|
||||
|
||||
if (format === "json") {
|
||||
console.log(generateJson(data));
|
||||
} else {
|
||||
console.log(generateMarkdown(data));
|
||||
console.log("📋 Requirement coverage (traced / defined):");
|
||||
for (const type of ["UR", "IR", "DR", "JA"] as const) {
|
||||
const traced = data.byType[type].filter((id) => definedIds.has(id)).length;
|
||||
console.log(` ${type}: ${traced} / ${defined[type]}`);
|
||||
}
|
||||
console.log("");
|
||||
console.log(`📈 Overall: ${cov.covered} / ${cov.total} (${cov.percent}%)`);
|
||||
|
||||
if (cov.orphaned.length > 0) {
|
||||
console.log("");
|
||||
console.log(
|
||||
`⚠️ Traced but not defined in requirements.md: ${cov.orphaned.join(", ")}`
|
||||
);
|
||||
console.log(" Fix the TRACES comment or add the requirement.");
|
||||
}
|
||||
|
||||
// A ratio above 100% means the computation is broken (the condition that hid
|
||||
// the stale-denominator bug for so long). Fail loudly rather than report it.
|
||||
if (cov.percent > 100) {
|
||||
console.log("");
|
||||
console.log(`❌ Coverage > 100% — the gate is miscomputing.`);
|
||||
return 1;
|
||||
}
|
||||
|
||||
if (cov.percent < minThreshold) {
|
||||
console.log("");
|
||||
console.log(`❌ Coverage (${cov.percent}%) is below minimum (${minThreshold}%)`);
|
||||
return 1;
|
||||
}
|
||||
|
||||
console.log("");
|
||||
console.log(`✅ Coverage is acceptable (${cov.percent}% >= ${minThreshold}%)`);
|
||||
return 0;
|
||||
}
|
||||
|
||||
console.error(
|
||||
`\n✅ Complete! Found ${data.totalTraces} TRACES across ${data.totalFiles} files`
|
||||
);
|
||||
// Main — guarded so this module stays importable from extract-traces.test.ts.
|
||||
if (import.meta.main) {
|
||||
const args = process.argv.slice(2);
|
||||
const format = args.includes("--format")
|
||||
? args[args.indexOf("--format") + 1]
|
||||
: "markdown";
|
||||
|
||||
console.error("🔍 Extracting TRACES from codebase...");
|
||||
const data = extractTraces();
|
||||
|
||||
const defined = readDefinedRequirements();
|
||||
const allTraced = Object.keys(data.requirements);
|
||||
data.defined = {
|
||||
UR: defined.UR,
|
||||
IR: defined.IR,
|
||||
DR: defined.DR,
|
||||
JA: defined.JA,
|
||||
total: defined.total,
|
||||
};
|
||||
data.coverage = computeCoverage(allTraced, defined);
|
||||
|
||||
if (format === "json") {
|
||||
console.log(generateJson(data));
|
||||
} else if (format === "coverage") {
|
||||
process.exit(reportCoverage(data, 50));
|
||||
} else {
|
||||
console.log(generateMarkdown(data));
|
||||
}
|
||||
|
||||
console.error(
|
||||
`\n✅ Complete! Found ${data.totalTraces} TRACES across ${data.totalFiles} files`
|
||||
);
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user