jellytau/scripts/extract-traces.test.ts

/**
 * Tests for the traceability coverage computation.
 *
 * These run over fixture strings rather than the live docs/requirements.md, so
 * their meaning does not drift as requirements are added.
 *
 * Background: the CI gate divided traced-requirement counts by hardcoded
 * denominators (UR/39, IR/24, DR/48, JA/3, total 114) that had fallen out of
 * date, reporting 158% coverage and making the 50% threshold unreachable. These
 * tests pin the parsing and arithmetic that replace those literals.
 *
 * @req-test: UT-089 - Requirement definitions parsed from requirements.md
 * @req-test: UT-090 - Coverage is the intersection of traced and defined IDs
 * @req-test: UT-202 - Generated matrix links resolve from docs/
 */

import { describe, it, expect } from "vitest";
import * as fs from "fs";
import * as path from "path";
import {
  countDefinedRequirements,
  computeCoverage,
  findDanglingIds,
  formatMatrixFileLink,
  generateMarkdown,
  isTracedSourceFile,
  MIN_COVERAGE_PERCENT,
  type TracesData,
} from "./extract-traces";

// import.meta.dir is Bun-only; derive from import.meta.url under vitest.
const HERE = path.dirname(new URL(import.meta.url).pathname);

describe("isTracedSourceFile", () => {
  // The extractor used to accept only .ts/.svelte/.rs under src/, src-tauri/src/
  // and scripts/. Every requirement implemented by *configuration* was therefore
  // invisible to the matrix that measures it: eslint.config.js (DR-205), the
  // pre-commit hook (DR-207), rust-toolchain.toml (DR-206) and deny.toml
  // (DR-216) all carry TRACES comments that were never read. Each one counted
  // against coverage as an uncovered requirement while being, in fact, covered.
  it("accepts the source extensions it always did", () => {
    expect(isTracedSourceFile("src/lib/utils/logger.ts")).toBe(true);
    expect(isTracedSourceFile("src/routes/settings/+page.svelte")).toBe(true);
    expect(isTracedSourceFile("src-tauri/src/lib.rs")).toBe(true);
  });

  it("accepts tooling files that implement a requirement", () => {
    expect(isTracedSourceFile("eslint.config.js")).toBe(true);
    expect(isTracedSourceFile("src-tauri/deny.toml")).toBe(true);
    expect(isTracedSourceFile("src-tauri/rust-toolchain.toml")).toBe(true);
    expect(isTracedSourceFile("scripts/hooks/pre-commit")).toBe(true);
    // Shell tooling is listed individually, not globbed: most scripts/*.sh
    // implement nothing, and adding one should be a decision.
    expect(isTracedSourceFile("scripts/check-release-artifacts.sh")).toBe(true);
    expect(isTracedSourceFile("scripts/build-desktop-linux.sh")).toBe(true);
    expect(isTracedSourceFile("scripts/logcat.sh")).toBe(false);
  });

  it("does not scan CI workflows, whose comments discuss TRACES in prose", () => {
    // .gitea/workflows/traceability-check.yml explains the gate, so it contains
    // lines like "a `TRACES:` comment ... (DR-189 and UT-188 lived in three
    // source files, defined nowhere)". The extractor's pattern would read that
    // as a trace and manufacture references to IDs that do not exist, failing
    // traces:validate. A file that *describes* traceability is not a file that
    // implements a requirement.
    expect(isTracedSourceFile(".gitea/workflows/traceability-check.yml")).toBe(false);
    expect(isTracedSourceFile(".gitea/workflows/build-and-test.yml")).toBe(false);
  });

  it("rejects files that merely mention a requirement in prose", () => {
    // requirements.md defines IDs; traceability.md is generated *from* traces.
    // Scanning either would make every requirement trace to itself.
    expect(isTracedSourceFile("docs/requirements.md")).toBe(false);
    expect(isTracedSourceFile("docs/traceability.md")).toBe(false);
    expect(isTracedSourceFile("README.md")).toBe(false);
  });

  it("rejects generated and vendored trees", () => {
    expect(isTracedSourceFile("node_modules/foo/index.ts")).toBe(false);
    expect(isTracedSourceFile("src-tauri/target/debug/build/x.rs")).toBe(false);
    expect(isTracedSourceFile("src-tauri/gen/android/app/build.gradle.kts")).toBe(false);
  });
});

describe("countDefinedRequirements", () => {
  it("counts a well-formed table row as a defined requirement", () => {
    const md = `
| ID | Requirement | Priority | Status |
|----|-------------|----------|--------|
| UR-001 | Run the app on multiple platforms | High | In Progress |
| UR-002 | Access media when online or offline | High | Done |
`;
    const defined = countDefinedRequirements(md);
    expect(defined.UR).toBe(2);
    expect(defined.DR).toBe(0);
  });

  it("does not count IDs that appear only in the Traces To column", () => {
    // The bug this rule avoids: a naive grep for /DR-\d{3}/ over the whole file
    // counts DR-001 here as "defined", inflating the denominator with IDs that
    // are merely referenced.
    const md = `
| DR-001 | Player state machine | Player | UR-005 | Done |
| DR-002 | MediaItem struct | Player | UR-003, UR-004 | Done |
`;
    const defined = countDefinedRequirements(md);
    expect(defined.DR).toBe(2);
    // UR-005/UR-003/UR-004 are referenced, never defined here.
    expect(defined.UR).toBe(0);
  });

  it("does not count IDs mentioned in prose", () => {
    const md = `
Some prose explaining that UR-005 relates to DR-001 and JA-002.

| UR-005 | Control media playback | High | Done |
`;
    const defined = countDefinedRequirements(md);
    expect(defined.UR).toBe(1);
    expect(defined.DR).toBe(0);
    expect(defined.JA).toBe(0);
  });

  it("deduplicates an ID listed in both the spec table and the traceability matrix", () => {
    // requirements.md lists every UR twice: once in §1 (definition) and again in
    // §3 (traceability matrix), both as a leading table cell. Counting rows
    // instead of unique IDs double-counts the UR denominator (121 vs 61).
    const md = `
| UR-005 | Control media playback | High | Done |
| UR-006 | Browse the library | High | Done |

### Traceability Matrix

| UR-005 | - | DR-001, DR-005, DR-009 |
| UR-006 | - | DR-012 |
`;
    const defined = countDefinedRequirements(md);
    expect(defined.UR).toBe(2);
  });

  it("collects the defined ID set, not just counts", () => {
    const md = `
| UR-001 | A | High | Done |
| DR-050 | B | Player | UR-001 | Done |
`;
    const defined = countDefinedRequirements(md);
    expect(defined.ids.has("UR-001")).toBe(true);
    expect(defined.ids.has("DR-050")).toBe(true);
    expect(defined.ids.has("UR-999")).toBe(false);
  });

  it("collects UT/IT rows separately, out of the coverage denominator", () => {
    // §4 defines the test taxonomy. Those rows must be known (so a TRACES
    // comment may name them) without ever moving the coverage ratio.
    const md = `
| UR-001 | A | High | Done |
| UT-001 | Player state transitions | DR-001 | Pending |
| IT-004 | Playback end-to-end | DR-002 | Pending |
`;
    const defined = countDefinedRequirements(md);
    expect(defined.total).toBe(1);
    expect(defined.ids.has("UT-001")).toBe(false);
    expect(defined.testIds.has("UT-001")).toBe(true);
    expect(defined.testIds.has("IT-004")).toBe(true);
  });
});

describe("findDanglingIds", () => {
  const defined = {
    UR: 1,
    IR: 0,
    DR: 1,
    JA: 0,
    total: 2,
    ids: new Set(["UR-001", "DR-001"]),
    testIds: new Set(["UT-001"]),
  };

  it("flags a requirement ID that requirements.md does not define", () => {
    expect(findDanglingIds(["UR-001", "DR-189"], defined)).toEqual(["DR-189"]);
  });

  it("flags an undefined UT/IT id, which the coverage orphan list cannot", () => {
    // The gap this closes: computeCoverage deliberately ignores UT/IT, so
    // UT-188 sat in three source files, defined nowhere, entirely unreported.
    expect(computeCoverage(["UT-188"], defined).orphaned).toEqual([]);
    expect(findDanglingIds(["UT-188"], defined)).toEqual(["UT-188"]);
  });

  it("accepts every ID that is defined, requirement or test", () => {
    expect(findDanglingIds(["UR-001", "DR-001", "UT-001"], defined)).toEqual([]);
  });

  it("deduplicates and sorts, so one typo is reported once", () => {
    expect(findDanglingIds(["DR-189", "DR-189", "UR-999", "DR-189"], defined)).toEqual([
      "DR-189",
      "UR-999",
    ]);
  });

  it("ignores IDs whose prefix is not a known trace type", () => {
    // e.g. an unrelated "AB-123" caught by the loose ID regex.
    expect(findDanglingIds(["AB-123"], defined)).toEqual([]);
  });
});

describe("coverage threshold", () => {
  it("matches MIN_THRESHOLD in the Gitea traceability workflow", () => {
    // Two files must agree on the gate: the script (local `traces:coverage`)
    // and the workflow. Drift means the local gate and CI disagree about what
    // passes, which is how the 50%-while-actually-86% slack went unnoticed.
    const workflow = fs.readFileSync(
      path.resolve(HERE, "../.gitea/workflows/traceability-check.yml"),
      "utf-8",
    );
    const match = workflow.match(/^\s*MIN_THRESHOLD=(\d+)\s*$/m);
    expect(match).not.toBeNull();
    expect(Number(match![1])).toBe(MIN_COVERAGE_PERCENT);
  });

  it("is a ratchet: never lower it to make a red build pass", () => {
    // Sanity bound. If coverage genuinely climbs, raise both numbers together.
    expect(MIN_COVERAGE_PERCENT).toBeGreaterThanOrEqual(82);
    expect(MIN_COVERAGE_PERCENT).toBeLessThanOrEqual(100);
  });
});

describe("computeCoverage", () => {
  const defined = {
    UR: 2,
    IR: 0,
    DR: 2,
    JA: 0,
    total: 4,
    ids: new Set(["UR-001", "UR-002", "DR-001", "DR-002"]),
    testIds: new Set<string>(),
  };

  it("computes coverage as traced ∩ defined over defined", () => {
    const traced = ["UR-001", "DR-001"];
    const cov = computeCoverage(traced, defined);
    expect(cov.covered).toBe(2);
    expect(cov.total).toBe(4);
    expect(cov.percent).toBe(50);
  });

  it("does not let a traced-but-undefined ID inflate the numerator", () => {
    // This is how a ratio exceeds 100%: a TRACES comment naming a typo'd or
    // deleted requirement counted as covered.
    const traced = ["UR-001", "DR-001", "DR-097"];
    const cov = computeCoverage(traced, defined);
    expect(cov.covered).toBe(2);
    expect(cov.percent).toBe(50);
  });

  it("reports traced-but-undefined IDs as orphaned so they get fixed", () => {
    const traced = ["UR-001", "DR-097", "JA-404"];
    const cov = computeCoverage(traced, defined);
    expect(cov.orphaned).toEqual(["DR-097", "JA-404"]);
  });

  it("has no orphans when every traced ID is defined", () => {
    const cov = computeCoverage(["UR-001", "UR-002"], defined);
    expect(cov.orphaned).toEqual([]);
  });

  it("ignores UT/IT test IDs entirely — they are a separate taxonomy", () => {
    // UT/IT are defined in §4 of requirements.md, not among the four
    // requirement types. Treating them as orphans buries real typos in ~60
    // lines of noise, and counting them would corrupt the ratio.
    const cov = computeCoverage(["UR-001", "UT-088", "IT-017"], defined);
    expect(cov.orphaned).toEqual([]);
    expect(cov.covered).toBe(1);
  });

  it("reports 0% rather than dividing by zero for an empty trace set", () => {
    const cov = computeCoverage([], defined);
    expect(cov.covered).toBe(0);
    expect(cov.percent).toBe(0);
  });

  it("reports 0% rather than NaN when nothing is defined", () => {
    const empty = {
      UR: 0,
      IR: 0,
      DR: 0,
      JA: 0,
      total: 0,
      ids: new Set<string>(),
      testIds: new Set<string>(),
    };
    const cov = computeCoverage([], empty);
    expect(cov.percent).toBe(0);
    expect(Number.isNaN(cov.percent)).toBe(false);
  });

  it("reports exactly 100% when all defined requirements are traced, never above", () => {
    const traced = ["UR-001", "UR-002", "DR-001", "DR-002"];
    const cov = computeCoverage(traced, defined);
    expect(cov.percent).toBe(100);
  });

  it("ignores duplicate traced IDs", () => {
    const traced = ["UR-001", "UR-001", "UR-001"];
    const cov = computeCoverage(traced, defined);
    expect(cov.covered).toBe(1);
  });
});

describe("generated matrix file links", () => {
  // Regression: the generator emitted the repo-root-relative path as the href
  // (`](src-tauri/src/…)`), but writes its output to docs/traceability.md — so
  // every one of the ~2,800 links resolved to docs/src-tauri/… and 404'd, in
  // the repo browser and on the published mdBook site. The markdown generator
  // had no test at all, which is why it survived. UT-202.
  //
  // @req-test: UT-202

  /** A minimal TracesData whose single entry points at a file that really exists. */
  function fixture(file: string, line = 12): TracesData {
    return {
      timestamp: new Date().toISOString(),
      totalFiles: 1,
      totalTraces: 1,
      requirements: {
        "DR-093": [{ file, line, context: "export function x() {}" }],
      },
      byType: { UR: [], IR: [], DR: ["DR-093"], JA: [] },
    } as TracesData;
  }

  /** Pull the href out of the first `- **File:** [`x`](href)` line. */
  function firstHref(md: string): string {
    const m = md.match(/^- \*\*File:\*\* \[`[^`]+`\]\(([^)]+)\)/m);
    expect(m).not.toBeNull();
    return m![1];
  }

  it("emits an href that resolves, from docs/, to a file that exists", () => {
    // Use a real repo file so "exists on disk" is a genuine assertion.
    const target = "scripts/extract-traces.ts";
    const md = generateMarkdown(fixture(target));

    const href = firstHref(md);
    const [relPath] = href.split("#");

    // traceability.md is written to docs/, so links resolve from there.
    const resolved = path.resolve(HERE, "../docs", relPath);
    expect(fs.existsSync(resolved)).toBe(true);
    expect(resolved).toBe(path.resolve(HERE, "..", target));
  });

  it("keeps the repo-root-relative path as the visible link text", () => {
    // The text is what a developer copies into an editor or a grep; only the
    // href is rewritten for the docs/ location.
    const md = generateMarkdown(fixture("src-tauri/src/lib.rs"));
    expect(md).toContain("[`src-tauri/src/lib.rs`]");
    expect(md).not.toContain("[`../src-tauri/src/lib.rs`]");
  });

  it("keeps the #Lnn line anchor on the href", () => {
    const link = formatMatrixFileLink("scripts/extract-traces.ts", 427);
    expect(link).toBe("[`scripts/extract-traces.ts`](../scripts/extract-traces.ts#L427)");
  });

  it("does not produce a bare repo-root href, which resolves to docs/<path>", () => {
    const md = generateMarkdown(fixture("scripts/extract-traces.ts"));
    const href = firstHref(md);
    expect(href.startsWith("../")).toBe(true);
    // The pre-fix output — the exact shape that produced docs/scripts/….
    expect(href.startsWith("scripts/")).toBe(false);
  });
});

describe("live requirements.md", () => {
  it("parses the real file into a self-consistent denominator", () => {
    // Guards the original regression: CI hardcoded UR/39, IR/24, DR/48, JA/3
    // (total 114) while the real file had grown past 200, so the gate compared
    // live traces against a frozen denominator and reported 158% coverage.
    //
    // Deliberately asserts *invariants*, not exact totals. Pinning the counts
    // was tried and turned this test into a merge-conflict magnet: every
    // requirement added on any branch had to edit the numbers here too, and the
    // comment above them grew into a ledger of which branch contributed which
    // row. Worse, the pins never guarded the actual defect — a stale denominator
    // is caught by the sum-consistency check below, and the >100% ratio it
    // produced is covered directly by the computeCoverage tests, on fixtures.
    const md = fs.readFileSync(path.resolve(HERE, "../docs/requirements.md"), "utf-8");
    const defined = countDefinedRequirements(md);

    // The parser found real rows of every type: a section silently failing to
    // parse would shrink the denominator and inflate coverage.
    expect(defined.UR).toBeGreaterThan(0);
    expect(defined.IR).toBeGreaterThan(0);
    expect(defined.DR).toBeGreaterThan(0);
    expect(defined.JA).toBeGreaterThan(0);

    // The denominator is the sum of its parts, and every counted id is unique —
    // double-counting one section is the other way a ratio breaks.
    expect(defined.total).toBe(defined.UR + defined.IR + defined.DR + defined.JA);
    expect(defined.ids.size).toBe(defined.total);

    // The file is live, not frozen: it is well past the 114 the stale gate used.
    expect(defined.total).toBeGreaterThan(200);
  });
});