ci: enforce the checks the contributor rules already required
Four gates that were documented but unenforced, plus the flaky test that made a full-suite run untrustworthy. Rust lint/format: CLAUDE.md has required `cargo fmt` and `cargo clippy` before every commit for as long as the rule existed, yet neither ran anywhere in CI — the requirement rested on memory alone. Both now run in build-and-test.yml and build-release.yml. rustfmt and clippy are already baked into the builder image, so nothing is installed at job time. `cargo fmt --all -- --check` is strict immediately (the tree is clean). Clippy is advisory for now: ~51 pre-existing warnings mean `-D warnings` would fail on unrelated work, so the step carries a TODO to flip the flag once the backlog clears. A compile error still fails it, so it is not a no-op. Traceability threshold: MIN_THRESHOLD sat at 50 while real coverage was 86%, so nearly half the matrix could rot before the gate objected. Ratcheted to 82 with the policy written down — it only ever goes up, and is never lowered to make a red build pass. The same figure lives in MIN_COVERAGE_PERCENT so `traces:coverage` gates locally on the same bar, and a test fails if the two drift. Dangling IDs: a TRACES comment could name any well-formed ID and the extractor accepted it silently, so typos and renames that missed a call site passed unnoticed. `bun run traces:validate` cross-checks every traced ID against the table rows in requirements.md and fails with the referencing files listed. It spans UT/IT as well, which the coverage orphan list ignores by design. This currently reports DR-189 and UT-188, which are being defined separately. Flaky offlineCatalog test: the first dynamic import of the service paid ~1s to transform its dependency graph, charged to a test body against vitest's 5s default. Alone it passed; under suite-wide contention it timed out. The import is now warmed at collection time, so no test is timing the compiler — the timeout is deliberately unchanged. The store shim also drops subscribers from module instances discarded by resetModules, which previously leaked across tests.
This commit is contained in:
+103
-3
@@ -37,8 +37,27 @@ interface TracesData {
|
||||
/** Requirements *defined* in requirements.md — the coverage denominators. */
|
||||
defined?: { UR: number; IR: number; DR: number; JA: number; total: number };
|
||||
coverage?: CoverageResult;
|
||||
/** Traced IDs of any type that requirements.md does not define. */
|
||||
dangling?: string[];
|
||||
}
|
||||
|
||||
/**
|
||||
* Minimum overall requirement coverage the traceability gate accepts.
|
||||
*
|
||||
* **Ratchet policy: this number only ever goes up.** It is set a few points
|
||||
* below the coverage actually achieved, so a real regression trips it instead of
|
||||
* being absorbed by slack. It sat at 50 while true coverage was 86%, which meant
|
||||
* half the matrix could rot before CI noticed. When coverage rises durably,
|
||||
* raise this to sit just under the new figure. Do **not** lower it to make a
|
||||
* failing build pass — add the missing TRACES comments instead.
|
||||
*
|
||||
* `.gitea/workflows/traceability-check.yml` carries the same number as
|
||||
* `MIN_THRESHOLD`; `scripts/extract-traces.test.ts` fails if the two drift.
|
||||
*
|
||||
* TRACES: | DR-093
|
||||
*/
|
||||
export const MIN_COVERAGE_PERCENT = 82;
|
||||
|
||||
// Repo root, derived from this script's location (scripts/ -> repo root).
|
||||
// Must NOT be hardcoded to a developer's machine, or CI checkouts see no files.
|
||||
//
|
||||
@@ -222,7 +241,10 @@ export interface DefinedRequirements {
|
||||
DR: number;
|
||||
JA: number;
|
||||
total: number;
|
||||
/** Requirement IDs (UR/IR/DR/JA) — the coverage denominator. */
|
||||
ids: Set<string>;
|
||||
/** Test IDs (UT/IT) from §4. A separate taxonomy: never part of coverage. */
|
||||
testIds: Set<string>;
|
||||
}
|
||||
|
||||
export interface CoverageResult {
|
||||
@@ -247,11 +269,18 @@ export interface CoverageResult {
|
||||
*/
|
||||
export function countDefinedRequirements(markdown: string): DefinedRequirements {
|
||||
const ids = new Set<string>();
|
||||
const ROW_ID = /^\|\s*(UR|IR|DR|JA)-(\d{3})\s*\|/;
|
||||
const testIds = new Set<string>();
|
||||
const ROW_ID = /^\|\s*(UR|IR|DR|JA|UT|IT)-(\d{3})\s*\|/;
|
||||
|
||||
for (const line of markdown.split("\n")) {
|
||||
const match = line.match(ROW_ID);
|
||||
if (match) ids.add(`${match[1]}-${match[2]}`);
|
||||
if (!match) continue;
|
||||
const id = `${match[1]}-${match[2]}`;
|
||||
// UT/IT rows live in §4 and are collected separately: they must not enter
|
||||
// the coverage denominator, but they still need to exist for a `TRACES:`
|
||||
// comment to be allowed to name them (see findDanglingIds).
|
||||
if (match[1] === "UT" || match[1] === "IT") testIds.add(id);
|
||||
else ids.add(id);
|
||||
}
|
||||
|
||||
const countOf = (type: string) =>
|
||||
@@ -264,9 +293,39 @@ export function countDefinedRequirements(markdown: string): DefinedRequirements
|
||||
JA: countOf("JA"),
|
||||
total: ids.size,
|
||||
ids,
|
||||
testIds,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Every traced ID that requirements.md defines nowhere — a typo, a rename that
|
||||
* missed a call site, or a reference to a deleted requirement.
|
||||
*
|
||||
* This is broader than `CoverageResult.orphaned`, which only ever considers the
|
||||
* four requirement types because a UT/IT entry among the orphans would corrupt
|
||||
* the coverage ratio's reporting. Dangling detection has no such constraint, so
|
||||
* it checks all six ID types against both defined sets. Before it existed, the
|
||||
* extractor accepted any well-formed ID silently: `DR-189` and `UT-188` were
|
||||
* referenced from `controlsVisibility.ts` and `VideoPlayer.svelte` for months
|
||||
* without being defined anywhere, and nothing reported it.
|
||||
*
|
||||
* TRACES: | DR-093
|
||||
*/
|
||||
export function findDanglingIds(
|
||||
tracedIds: string[],
|
||||
defined: DefinedRequirements
|
||||
): string[] {
|
||||
const KNOWN_TYPE = /^(UR|IR|DR|JA|UT|IT)-\d{3}$/;
|
||||
|
||||
const dangling = new Set(
|
||||
tracedIds
|
||||
.filter((id) => KNOWN_TYPE.test(id))
|
||||
.filter((id) => !defined.ids.has(id) && !defined.testIds.has(id))
|
||||
);
|
||||
|
||||
return [...dangling].sort();
|
||||
}
|
||||
|
||||
/**
|
||||
* Coverage is the *intersection* of traced and defined IDs over defined IDs.
|
||||
*
|
||||
@@ -408,6 +467,13 @@ function reportCoverage(data: TracesData, minThreshold: number): number {
|
||||
console.log(" Fix the TRACES comment or add the requirement.");
|
||||
}
|
||||
|
||||
if (data.dangling && data.dangling.length > 0) {
|
||||
console.log("");
|
||||
console.log(
|
||||
`⚠️ Dangling IDs (incl. UT/IT): ${data.dangling.join(", ")} — run \`bun run traces:validate\`.`
|
||||
);
|
||||
}
|
||||
|
||||
// A ratio above 100% means the computation is broken (the condition that hid
|
||||
// the stale-denominator bug for so long). Fail loudly rather than report it.
|
||||
if (cov.percent > 100) {
|
||||
@@ -427,6 +493,37 @@ function reportCoverage(data: TracesData, minThreshold: number): number {
|
||||
return 0;
|
||||
}
|
||||
|
||||
/**
|
||||
* Hard gate on dangling IDs: a `TRACES:` comment may only name an ID that
|
||||
* requirements.md actually defines. Prints every offender with the files that
|
||||
* reference it, so the fix is mechanical.
|
||||
*
|
||||
* TRACES: | DR-093
|
||||
*/
|
||||
function reportDangling(data: TracesData): number {
|
||||
const dangling = data.dangling ?? [];
|
||||
|
||||
if (dangling.length === 0) {
|
||||
console.log("✅ All traced IDs are defined in docs/requirements.md");
|
||||
return 0;
|
||||
}
|
||||
|
||||
console.log("❌ TRACES reference IDs that docs/requirements.md does not define:");
|
||||
console.log("");
|
||||
for (const id of dangling) {
|
||||
const files = [
|
||||
...new Set((data.requirements[id] ?? []).map((e) => e.file)),
|
||||
].sort();
|
||||
console.log(` ${id}`);
|
||||
for (const file of files) console.log(` ${file}`);
|
||||
}
|
||||
console.log("");
|
||||
console.log("Fix each one by either:");
|
||||
console.log(" • correcting the ID in the TRACES comment (typo/rename), or");
|
||||
console.log(" • adding the requirement as a table row in docs/requirements.md.");
|
||||
return 1;
|
||||
}
|
||||
|
||||
// Main — guarded so this module stays importable from extract-traces.test.ts.
|
||||
if (import.meta.main) {
|
||||
const args = process.argv.slice(2);
|
||||
@@ -447,11 +544,14 @@ if (import.meta.main) {
|
||||
total: defined.total,
|
||||
};
|
||||
data.coverage = computeCoverage(allTraced, defined);
|
||||
data.dangling = findDanglingIds(allTraced, defined);
|
||||
|
||||
if (format === "json") {
|
||||
console.log(generateJson(data));
|
||||
} else if (format === "coverage") {
|
||||
process.exit(reportCoverage(data, 50));
|
||||
process.exit(reportCoverage(data, MIN_COVERAGE_PERCENT));
|
||||
} else if (format === "validate") {
|
||||
process.exit(reportDangling(data));
|
||||
} else {
|
||||
console.log(generateMarkdown(data));
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user