jellytau/scripts/extract-traces.test.ts

/**
 * Tests for the traceability coverage computation.
 *
 * These run over fixture strings rather than the live docs/requirements.md, so
 * their meaning does not drift as requirements are added.
 *
 * Background: the CI gate divided traced-requirement counts by hardcoded
 * denominators (UR/39, IR/24, DR/48, JA/3, total 114) that had fallen out of
 * date, reporting 158% coverage and making the 50% threshold unreachable. These
 * tests pin the parsing and arithmetic that replace those literals.
 *
 * @req-test: UT-089 - Requirement definitions parsed from requirements.md
 * @req-test: UT-090 - Coverage is the intersection of traced and defined IDs
 */

import { describe, it, expect } from "vitest";
import * as fs from "fs";
import * as path from "path";
import {
  countDefinedRequirements,
  computeCoverage,
  findDanglingIds,
  MIN_COVERAGE_PERCENT,
} from "./extract-traces";

// import.meta.dir is Bun-only; derive from import.meta.url under vitest.
const HERE = path.dirname(new URL(import.meta.url).pathname);

describe("countDefinedRequirements", () => {
  it("counts a well-formed table row as a defined requirement", () => {
    const md = `
| ID | Requirement | Priority | Status |
|----|-------------|----------|--------|
| UR-001 | Run the app on multiple platforms | High | In Progress |
| UR-002 | Access media when online or offline | High | Done |
`;
    const defined = countDefinedRequirements(md);
    expect(defined.UR).toBe(2);
    expect(defined.DR).toBe(0);
  });

  it("does not count IDs that appear only in the Traces To column", () => {
    // The bug this rule avoids: a naive grep for /DR-\d{3}/ over the whole file
    // counts DR-001 here as "defined", inflating the denominator with IDs that
    // are merely referenced.
    const md = `
| DR-001 | Player state machine | Player | UR-005 | Done |
| DR-002 | MediaItem struct | Player | UR-003, UR-004 | Done |
`;
    const defined = countDefinedRequirements(md);
    expect(defined.DR).toBe(2);
    // UR-005/UR-003/UR-004 are referenced, never defined here.
    expect(defined.UR).toBe(0);
  });

  it("does not count IDs mentioned in prose", () => {
    const md = `
Some prose explaining that UR-005 relates to DR-001 and JA-002.

| UR-005 | Control media playback | High | Done |
`;
    const defined = countDefinedRequirements(md);
    expect(defined.UR).toBe(1);
    expect(defined.DR).toBe(0);
    expect(defined.JA).toBe(0);
  });

  it("deduplicates an ID listed in both the spec table and the traceability matrix", () => {
    // requirements.md lists every UR twice: once in §1 (definition) and again in
    // §3 (traceability matrix), both as a leading table cell. Counting rows
    // instead of unique IDs double-counts the UR denominator (121 vs 61).
    const md = `
| UR-005 | Control media playback | High | Done |
| UR-006 | Browse the library | High | Done |

### Traceability Matrix

| UR-005 | - | DR-001, DR-005, DR-009 |
| UR-006 | - | DR-012 |
`;
    const defined = countDefinedRequirements(md);
    expect(defined.UR).toBe(2);
  });

  it("collects the defined ID set, not just counts", () => {
    const md = `
| UR-001 | A | High | Done |
| DR-050 | B | Player | UR-001 | Done |
`;
    const defined = countDefinedRequirements(md);
    expect(defined.ids.has("UR-001")).toBe(true);
    expect(defined.ids.has("DR-050")).toBe(true);
    expect(defined.ids.has("UR-999")).toBe(false);
  });

  it("collects UT/IT rows separately, out of the coverage denominator", () => {
    // §4 defines the test taxonomy. Those rows must be known (so a TRACES
    // comment may name them) without ever moving the coverage ratio.
    const md = `
| UR-001 | A | High | Done |
| UT-001 | Player state transitions | DR-001 | Pending |
| IT-004 | Playback end-to-end | DR-002 | Pending |
`;
    const defined = countDefinedRequirements(md);
    expect(defined.total).toBe(1);
    expect(defined.ids.has("UT-001")).toBe(false);
    expect(defined.testIds.has("UT-001")).toBe(true);
    expect(defined.testIds.has("IT-004")).toBe(true);
  });
});

describe("findDanglingIds", () => {
  const defined = {
    UR: 1,
    IR: 0,
    DR: 1,
    JA: 0,
    total: 2,
    ids: new Set(["UR-001", "DR-001"]),
    testIds: new Set(["UT-001"]),
  };

  it("flags a requirement ID that requirements.md does not define", () => {
    expect(findDanglingIds(["UR-001", "DR-189"], defined)).toEqual(["DR-189"]);
  });

  it("flags an undefined UT/IT id, which the coverage orphan list cannot", () => {
    // The gap this closes: computeCoverage deliberately ignores UT/IT, so
    // UT-188 sat in three source files, defined nowhere, entirely unreported.
    expect(computeCoverage(["UT-188"], defined).orphaned).toEqual([]);
    expect(findDanglingIds(["UT-188"], defined)).toEqual(["UT-188"]);
  });

  it("accepts every ID that is defined, requirement or test", () => {
    expect(findDanglingIds(["UR-001", "DR-001", "UT-001"], defined)).toEqual([]);
  });

  it("deduplicates and sorts, so one typo is reported once", () => {
    expect(
      findDanglingIds(["DR-189", "DR-189", "UR-999", "DR-189"], defined)
    ).toEqual(["DR-189", "UR-999"]);
  });

  it("ignores IDs whose prefix is not a known trace type", () => {
    // e.g. an unrelated "AB-123" caught by the loose ID regex.
    expect(findDanglingIds(["AB-123"], defined)).toEqual([]);
  });
});

describe("coverage threshold", () => {
  it("matches MIN_THRESHOLD in the Gitea traceability workflow", () => {
    // Two files must agree on the gate: the script (local `traces:coverage`)
    // and the workflow. Drift means the local gate and CI disagree about what
    // passes, which is how the 50%-while-actually-86% slack went unnoticed.
    const workflow = fs.readFileSync(
      path.resolve(HERE, "../.gitea/workflows/traceability-check.yml"),
      "utf-8"
    );
    const match = workflow.match(/^\s*MIN_THRESHOLD=(\d+)\s*$/m);
    expect(match).not.toBeNull();
    expect(Number(match![1])).toBe(MIN_COVERAGE_PERCENT);
  });

  it("is a ratchet: never lower it to make a red build pass", () => {
    // Sanity bound. If coverage genuinely climbs, raise both numbers together.
    expect(MIN_COVERAGE_PERCENT).toBeGreaterThanOrEqual(82);
    expect(MIN_COVERAGE_PERCENT).toBeLessThanOrEqual(100);
  });
});

describe("computeCoverage", () => {
  const defined = {
    UR: 2,
    IR: 0,
    DR: 2,
    JA: 0,
    total: 4,
    ids: new Set(["UR-001", "UR-002", "DR-001", "DR-002"]),
    testIds: new Set<string>(),
  };

  it("computes coverage as traced ∩ defined over defined", () => {
    const traced = ["UR-001", "DR-001"];
    const cov = computeCoverage(traced, defined);
    expect(cov.covered).toBe(2);
    expect(cov.total).toBe(4);
    expect(cov.percent).toBe(50);
  });

  it("does not let a traced-but-undefined ID inflate the numerator", () => {
    // This is how a ratio exceeds 100%: a TRACES comment naming a typo'd or
    // deleted requirement counted as covered.
    const traced = ["UR-001", "DR-001", "DR-097"];
    const cov = computeCoverage(traced, defined);
    expect(cov.covered).toBe(2);
    expect(cov.percent).toBe(50);
  });

  it("reports traced-but-undefined IDs as orphaned so they get fixed", () => {
    const traced = ["UR-001", "DR-097", "JA-404"];
    const cov = computeCoverage(traced, defined);
    expect(cov.orphaned).toEqual(["DR-097", "JA-404"]);
  });

  it("has no orphans when every traced ID is defined", () => {
    const cov = computeCoverage(["UR-001", "UR-002"], defined);
    expect(cov.orphaned).toEqual([]);
  });

  it("ignores UT/IT test IDs entirely — they are a separate taxonomy", () => {
    // UT/IT are defined in §4 of requirements.md, not among the four
    // requirement types. Treating them as orphans buries real typos in ~60
    // lines of noise, and counting them would corrupt the ratio.
    const cov = computeCoverage(["UR-001", "UT-088", "IT-017"], defined);
    expect(cov.orphaned).toEqual([]);
    expect(cov.covered).toBe(1);
  });

  it("reports 0% rather than dividing by zero for an empty trace set", () => {
    const cov = computeCoverage([], defined);
    expect(cov.covered).toBe(0);
    expect(cov.percent).toBe(0);
  });

  it("reports 0% rather than NaN when nothing is defined", () => {
    const empty = {
      UR: 0,
      IR: 0,
      DR: 0,
      JA: 0,
      total: 0,
      ids: new Set<string>(),
      testIds: new Set<string>(),
    };
    const cov = computeCoverage([], empty);
    expect(cov.percent).toBe(0);
    expect(Number.isNaN(cov.percent)).toBe(false);
  });

  it("reports exactly 100% when all defined requirements are traced, never above", () => {
    const traced = ["UR-001", "UR-002", "DR-001", "DR-002"];
    const cov = computeCoverage(traced, defined);
    expect(cov.percent).toBe(100);
  });

  it("ignores duplicate traced IDs", () => {
    const traced = ["UR-001", "UR-001", "UR-001"];
    const cov = computeCoverage(traced, defined);
    expect(cov.covered).toBe(1);
  });
});

describe("live requirements.md", () => {
  it("parses the real file to the counts the CI gate must use", () => {
    // Guards the specific regression: CI hardcoded UR/39, IR/24, DR/48, JA/3
    // (total 114) while the real file had grown to 211. Update these numbers
    // deliberately when requirements are added — that edit is the signal the
    // denominator is live rather than frozen.
    const md = fs.readFileSync(
      path.resolve(HERE, "../docs/requirements.md"),
      "utf-8"
    );
    const defined = countDefinedRequirements(md);

    expect(defined.UR).toBe(75);
    expect(defined.IR).toBe(32);
    // 191 = 187 + four requirements added independently on four branches
    // that landed together: DR-189 (control-bar auto-hide), DR-198 (asset
    // scope/CSP), DR-199 (webview mixed-content) and DR-200 (the
    // POST_NOTIFICATIONS media-session exemption; renumbered from 198 on
    // merge, where it collided). Each branch bumped for its own — merged,
    // they sum. Resolve this by summing, never by taking one side.
    expect(defined.DR).toBe(191);
    expect(defined.JA).toBe(36);
    expect(defined.total).toBe(334);
  });
});