jellytau/scripts/extract-traces.ts

#!/usr/bin/env bun
/**
 * Extract TRACES from source code and generate requirement mapping
 *
 * Usage:
 *   bun run scripts/extract-traces.ts
 *   bun run scripts/extract-traces.ts --format json
 *   bun run scripts/extract-traces.ts --format markdown > docs/traceability.md
 */

import * as fs from "fs";
import * as path from "path";
import { execSync } from "child_process";

interface TraceEntry {
  file: string;
  line: number;
  context: string;
  requirements: string[];
}

interface RequirementMapping {
  [reqId: string]: TraceEntry[];
}

export interface TracesData {
  timestamp: string;
  totalFiles: number;
  totalTraces: number;
  requirements: RequirementMapping;
  byType: {
    UR: string[];
    IR: string[];
    DR: string[];
    JA: string[];
  };
  /** Requirements *defined* in requirements.md — the coverage denominators. */
  defined?: { UR: number; IR: number; DR: number; JA: number; total: number };
  coverage?: CoverageResult;
  /** Traced IDs of any type that requirements.md does not define. */
  dangling?: string[];
}

/**
 * Minimum overall requirement coverage the traceability gate accepts.
 *
 * **Ratchet policy: this number only ever goes up.** It is set a few points
 * below the coverage actually achieved, so a real regression trips it instead of
 * being absorbed by slack. It sat at 50 while true coverage was 86%, which meant
 * half the matrix could rot before CI noticed. When coverage rises durably,
 * raise this to sit just under the new figure. Do **not** lower it to make a
 * failing build pass — add the missing TRACES comments instead.
 *
 * `.gitea/workflows/traceability-check.yml` carries the same number as
 * `MIN_THRESHOLD`; `scripts/extract-traces.test.ts` fails if the two drift.
 *
 * TRACES: | DR-093
 */
export const MIN_COVERAGE_PERCENT = 89;

// Repo root, derived from this script's location (scripts/ -> repo root).
// Must NOT be hardcoded to a developer's machine, or CI checkouts see no files.
//
// `import.meta.dir` is a Bun extension and is undefined when this module is
// imported by vitest (which runs it as an ordinary ESM module), so fall back to
// import.meta.url — this file must stay importable for extract-traces.test.ts.
const SCRIPT_DIR = import.meta.dir ?? path.dirname(new URL(import.meta.url).pathname);
const BASE_DIR = path.resolve(SCRIPT_DIR, "..");

const TRACES_PATTERN = /TRACES:\s*([^\n]+)/gi;
const REQ_ID_PATTERN = /([A-Z]{2})-(\d{3})/g;

function extractRequirementIds(tracesString: string): string[] {
  const matches = [...tracesString.matchAll(REQ_ID_PATTERN)];
  return matches.map((m) => `${m[1]}-${m[2]}`);
}

/**
 * Tooling files that implement a requirement.
 *
 * The walker below only visits `src/`, `src-tauri/src/` and `scripts/`, and only
 * picks up `.ts`/`.svelte`/`.rs`. That made every requirement implemented by
 * *configuration* invisible to the matrix that measures it — DR-205
 * (eslint.config.js), DR-206 (rust-toolchain.toml), DR-207 (the pre-commit
 * hook) and DR-216 (deny.toml) all carry TRACES comments that nothing read, so
 * each was counted as uncovered while being covered.
 *
 * An explicit list rather than "also scan .toml/.js/.yml": most config files in
 * this repo implement nothing, and one class of file is actively dangerous to
 * scan — see `isTracedSourceFile`.
 */
const TOOLING_FILES = new Set([
  "eslint.config.js",
  "vitest.config.ts",
  "scripts/hooks/pre-commit",
  "src-tauri/deny.toml",
  "src-tauri/rust-toolchain.toml",
]);

/** Directory names that never contain hand-written traced source. */
const EXCLUDED_SEGMENTS = new Set([
  "node_modules",
  "target",
  "build",
  ".git",
  ".svelte-kit",
  "docs-site",
  // Tauri regenerates src-tauri/gen/ on every android/desktop init; the
  // canonical Android sources live in src-tauri/android/ and are synced into it.
  "gen",
]);

/**
 * Decide whether a repo-relative path should be scanned for TRACES comments.
 *
 * Exported for scripts/extract-traces.test.ts — the file-walking half needs a
 * filesystem, this half is a pure decision and is where the mistakes live.
 *
 * Deliberately excluded:
 *  - `docs/requirements.md` *defines* IDs and `docs/traceability.md` is
 *    generated from traces; scanning either would make requirements trace to
 *    themselves.
 *  - `.gitea/workflows/*.yml` — traceability-check.yml explains the gate in
 *    prose, quoting "a `TRACES:` comment" on the same line as example IDs that
 *    are deliberately undefined. The extractor would read those as real traces
 *    and then fail its own dangling-ID check.
 */
export function isTracedSourceFile(relativePath: string): boolean {
  const p = relativePath.split(path.sep).join("/");
  if (p.split("/").some((segment) => EXCLUDED_SEGMENTS.has(segment))) {
    return false;
  }
  if (TOOLING_FILES.has(p)) {
    return true;
  }
  const isSourceExtension = p.endsWith(".ts") || p.endsWith(".svelte") || p.endsWith(".rs");
  if (!isSourceExtension) {
    return false;
  }
  return p.startsWith("src/") || p.startsWith("src-tauri/src/") || p.startsWith("scripts/");
}

function getAllSourceFiles(): string[] {
  const baseDir = BASE_DIR;
  // `scripts` is scanned too: build tooling implements requirements (e.g.
  // DR-093, the coverage engine itself) and would otherwise be invisible to the
  // very matrix it generates.
  const patterns = ["src", "src-tauri/src", "scripts"];
  const files: string[] = [];

  function walkDir(dir: string) {
    try {
      const entries = fs.readdirSync(dir, { withFileTypes: true });
      for (const entry of entries) {
        const fullPath = path.join(dir, entry.name);
        const relativePath = path.relative(baseDir, fullPath);

        // Directory pruning still happens here so the walk does not descend
        // into node_modules/target at all; isTracedSourceFile repeats the rule
        // for individual files (and is the version under test).
        if (entry.isDirectory() && !isTracedSourceFile(path.join(relativePath, "x.ts"))) {
          continue;
        }

        if (entry.isDirectory()) {
          walkDir(fullPath);
        } else if (isTracedSourceFile(relativePath)) {
          files.push(fullPath);
        }
      }
    } catch (error) {
      // Skip directories we can't read
    }
  }

  for (const pattern of patterns) {
    const dir = path.join(baseDir, pattern);
    if (fs.existsSync(dir)) {
      walkDir(dir);
    }
  }

  // eslint.config.js, deny.toml and rust-toolchain.toml sit at the repo root or
  // in src-tauri/ rather than under a walked root, so they are added by name.
  for (const toolingFile of TOOLING_FILES) {
    const fullPath = path.join(baseDir, toolingFile);
    if (fs.existsSync(fullPath) && !files.includes(fullPath)) {
      files.push(fullPath);
    }
  }

  return files;
}

function extractTraces(): TracesData {
  const requirementMap: RequirementMapping = {};
  const byType: Record<string, Set<string>> = {
    UR: new Set(),
    IR: new Set(),
    DR: new Set(),
    JA: new Set(),
  };

  let totalTraces = 0;
  const baseDir = BASE_DIR;

  const files = getAllSourceFiles();

  for (const fullPath of files) {
    try {
      const content = fs.readFileSync(fullPath, "utf-8");
      const lines = content.split("\n");
      const relativePath = path.relative(baseDir, fullPath);

      let match;
      TRACES_PATTERN.lastIndex = 0;

      while ((match = TRACES_PATTERN.exec(content)) !== null) {
        const tracesStr = match[1];
        const reqIds = extractRequirementIds(tracesStr);

        if (reqIds.length === 0) continue;

        // Find line number
        const beforeMatch = content.substring(0, match.index);
        const lineNum = beforeMatch.split("\n").length - 1;

        // Get context (function/class name if available)
        let context = "Unknown";
        for (let i = lineNum; i >= Math.max(0, lineNum - 10); i--) {
          const line = lines[i];
          if (
            line.includes("function ") ||
            line.includes("export const ") ||
            line.includes("pub fn ") ||
            line.includes("pub enum ") ||
            line.includes("pub struct ") ||
            line.includes("impl ") ||
            line.includes("async function ") ||
            line.includes("class ") ||
            line.includes("export type ")
          ) {
            context = line
              .trim()
              .replace(/^\s*\/\/\s*/, "")
              .replace(/^\s*\/\*\*\s*/, "");
            break;
          }
        }

        const entry: TraceEntry = {
          file: relativePath,
          line: lineNum + 1,
          context,
          requirements: reqIds,
        };

        for (const reqId of reqIds) {
          if (!requirementMap[reqId]) {
            requirementMap[reqId] = [];
          }
          requirementMap[reqId].push(entry);

          // Track by type
          const type = reqId.substring(0, 2);
          if (byType[type]) {
            byType[type].add(reqId);
          }
        }

        totalTraces++;
      }
    } catch (error) {
      // Skip files we can't read
    }
  }

  return {
    timestamp: new Date().toISOString(),
    totalFiles: files.length,
    totalTraces,
    requirements: requirementMap,
    byType: {
      UR: Array.from(byType["UR"]).sort(),
      IR: Array.from(byType["IR"]).sort(),
      DR: Array.from(byType["DR"]).sort(),
      JA: Array.from(byType["JA"]).sort(),
    },
  };
}

// ---------------------------------------------------------------------------
// Coverage: how many *defined* requirements are actually traced.
//
// The denominators MUST be derived from requirements.md, never hardcoded. The
// CI gate previously divided by frozen literals (UR/39, IR/24, DR/48, JA/3,
// total 114) while the real file had grown to 211 requirements, so it reported
// 158% coverage and the 50% threshold became unreachable — the gate could not
// fail. See docs/traceability-ci.md, "Coverage Thresholds".
//
// TRACES: | DR-093
// ---------------------------------------------------------------------------

export interface DefinedRequirements {
  UR: number;
  IR: number;
  DR: number;
  JA: number;
  total: number;
  /** Requirement IDs (UR/IR/DR/JA) — the coverage denominator. */
  ids: Set<string>;
  /** Test IDs (UT/IT) from §4. A separate taxonomy: never part of coverage. */
  testIds: Set<string>;
}

export interface CoverageResult {
  covered: number;
  total: number;
  percent: number;
  /** Traced in code but not defined in requirements.md (typo, or deleted req). */
  orphaned: string[];
}

/**
 * A requirement is *defined* only where its ID is the leading cell of a markdown
 * table row: `| DR-001 | … |`.
 *
 * This deliberately ignores IDs in the "Traces To" column and in prose — a
 * naive scan for /DR-\d{3}/ counts those as definitions and inflates the
 * denominator. IDs are deduplicated because requirements.md lists each UR twice
 * (once in §1 as a definition, again in §3's traceability matrix), which would
 * otherwise double the UR count from 61 to 121.
 *
 * TRACES: | DR-093
 */
export function countDefinedRequirements(markdown: string): DefinedRequirements {
  const ids = new Set<string>();
  const testIds = new Set<string>();
  const ROW_ID = /^\|\s*(UR|IR|DR|JA|UT|IT)-(\d{3})\s*\|/;

  for (const line of markdown.split("\n")) {
    const match = line.match(ROW_ID);
    if (!match) continue;
    const id = `${match[1]}-${match[2]}`;
    // UT/IT rows live in §4 and are collected separately: they must not enter
    // the coverage denominator, but they still need to exist for a `TRACES:`
    // comment to be allowed to name them (see findDanglingIds).
    if (match[1] === "UT" || match[1] === "IT") testIds.add(id);
    else ids.add(id);
  }

  const countOf = (type: string) => [...ids].filter((id) => id.startsWith(`${type}-`)).length;

  return {
    UR: countOf("UR"),
    IR: countOf("IR"),
    DR: countOf("DR"),
    JA: countOf("JA"),
    total: ids.size,
    ids,
    testIds,
  };
}

/**
 * Every traced ID that requirements.md defines nowhere — a typo, a rename that
 * missed a call site, or a reference to a deleted requirement.
 *
 * This is broader than `CoverageResult.orphaned`, which only ever considers the
 * four requirement types because a UT/IT entry among the orphans would corrupt
 * the coverage ratio's reporting. Dangling detection has no such constraint, so
 * it checks all six ID types against both defined sets. Before it existed, the
 * extractor accepted any well-formed ID silently: `DR-189` and `UT-188` were
 * referenced from `controlsVisibility.ts` and `VideoPlayer.svelte` for months
 * without being defined anywhere, and nothing reported it.
 *
 * TRACES: | DR-093
 */
export function findDanglingIds(tracedIds: string[], defined: DefinedRequirements): string[] {
  const KNOWN_TYPE = /^(UR|IR|DR|JA|UT|IT)-\d{3}$/;

  const dangling = new Set(
    tracedIds
      .filter((id) => KNOWN_TYPE.test(id))
      .filter((id) => !defined.ids.has(id) && !defined.testIds.has(id)),
  );

  return [...dangling].sort();
}

/**
 * Coverage is the *intersection* of traced and defined IDs over defined IDs.
 *
 * Using the raw traced count as the numerator is what lets a ratio exceed 100%:
 * a TRACES comment naming a requirement that no longer exists would count as
 * covered. Those IDs are reported as `orphaned` so they get fixed rather than
 * silently counted or silently dropped.
 *
 * TRACES: | DR-093
 */
export function computeCoverage(tracedIds: string[], defined: DefinedRequirements): CoverageResult {
  // Only the four *requirement* types participate in coverage. UT/IT are test
  // identifiers defined in §4 of requirements.md — a different taxonomy, and
  // flagging them as orphans would bury real typos in ~60 lines of noise.
  const isRequirement = (id: string) => /^(UR|IR|DR|JA)-\d{3}$/.test(id);

  const traced = new Set(tracedIds.filter(isRequirement));
  const covered = [...traced].filter((id) => defined.ids.has(id));
  const orphaned = [...traced].filter((id) => !defined.ids.has(id)).sort();

  return {
    covered: covered.length,
    total: defined.total,
    percent: defined.total === 0 ? 0 : Math.round((covered.length / defined.total) * 100),
    orphaned,
  };
}

/** Read requirements.md from the repo and count what it defines. */
export function readDefinedRequirements(): DefinedRequirements {
  const reqPath = path.join(BASE_DIR, "docs", "requirements.md");
  return countDefinedRequirements(fs.readFileSync(reqPath, "utf-8"));
}

/**
 * Path prefix that turns a repo-root-relative file path into a link target that
 * resolves from `docs/traceability.md`, where this markdown is written.
 *
 * The generated matrix lives one directory below the repo root, so a bare
 * `src-tauri/src/player/mod.rs` href resolves to `docs/src-tauri/…` and 404s —
 * in the repo browser and on the published mdBook site alike. Every file link
 * in the matrix was dead for this reason. The *display text* stays
 * repo-root-relative (that is the path a developer types and greps for); only
 * the href is rewritten.
 *
 * TRACES: | DR-093 | UT-202
 */
export const MATRIX_LINK_PREFIX = "../";

/**
 * Build the ``[`path`](href#Lnn)`` link used for one trace entry in the matrix.
 *
 * Exported so extract-traces.test.ts can resolve a generated href against
 * `docs/` and assert the target exists on disk.
 *
 * TRACES: | DR-093 | UT-202
 */
export function formatMatrixFileLink(file: string, line: number): string {
  return `[\`${file}\`](${MATRIX_LINK_PREFIX}${file}#L${line})`;
}

export function generateMarkdown(data: TracesData): string {
  let md = `# Code Traceability Matrix

**Generated:** ${new Date(data.timestamp).toLocaleString()}

## Summary

- **Total Files Scanned:** ${data.totalFiles}
- **Total TRACES Found:** ${data.totalTraces}
- **Requirements Covered:**
  - User Requirements (UR): ${data.byType.UR.length}
  - Integration Requirements (IR): ${data.byType.IR.length}
  - Development Requirements (DR): ${data.byType.DR.length}
  - Jellyfin API Requirements (JA): ${data.byType.JA.length}

## Requirements by Type

### User Requirements (UR)
\`\`\`
${data.byType.UR.join(", ")}
\`\`\`

### Integration Requirements (IR)
\`\`\`
${data.byType.IR.join(", ")}
\`\`\`

### Development Requirements (DR)
\`\`\`
${data.byType.DR.join(", ")}
\`\`\`

### Jellyfin API Requirements (JA)
\`\`\`
${data.byType.JA.join(", ")}
\`\`\`

## Detailed Mapping

`;

  // Sort requirements by ID
  const sortedReqs = Object.keys(data.requirements).sort((a, b) => {
    const typeA = a.substring(0, 2);
    const typeB = b.substring(0, 2);
    const typeOrder = { UR: 0, IR: 1, DR: 2, JA: 3 };
    if (typeOrder[typeA] !== typeOrder[typeB]) {
      return (typeOrder[typeA] || 4) - (typeOrder[typeB] || 4);
    }
    return a.localeCompare(b);
  });

  for (const reqId of sortedReqs) {
    const entries = data.requirements[reqId];
    md += `### ${reqId}\n\n`;
    md += `**Locations:** ${entries.length} file(s)\n\n`;

    for (const entry of entries) {
      md += `- **File:** ${formatMatrixFileLink(entry.file, entry.line)}\n`;
      md += `  - **Line:** ${entry.line}\n`;
      const contextPreview = entry.context.substring(0, 70);
      md += `  - **Context:** \`${contextPreview}${entry.context.length > 70 ? "..." : ""}\`\n`;
    }
    md += "\n";
  }

  return md;
}

function generateJson(data: TracesData): string {
  return JSON.stringify(data, null, 2);
}

/**
 * Human-readable coverage report; exits non-zero below the threshold so this is
 * runnable as a local gate (`bun run traces:coverage`), not just in CI.
 *
 * TRACES: | DR-093
 */
function reportCoverage(data: TracesData, minThreshold: number): number {
  const defined = data.defined!;
  const cov = data.coverage!;

  const definedIds = readDefinedRequirements().ids;

  console.log("📋 Requirement coverage (traced / defined):");
  for (const type of ["UR", "IR", "DR", "JA"] as const) {
    const traced = data.byType[type].filter((id) => definedIds.has(id)).length;
    console.log(`   ${type}: ${traced} / ${defined[type]}`);
  }
  console.log("");
  console.log(`📈 Overall: ${cov.covered} / ${cov.total} (${cov.percent}%)`);

  if (cov.orphaned.length > 0) {
    console.log("");
    console.log(`⚠️  Traced but not defined in requirements.md: ${cov.orphaned.join(", ")}`);
    console.log("   Fix the TRACES comment or add the requirement.");
  }

  if (data.dangling && data.dangling.length > 0) {
    console.log("");
    console.log(
      `⚠️  Dangling IDs (incl. UT/IT): ${data.dangling.join(", ")} — run \`bun run traces:validate\`.`,
    );
  }

  // A ratio above 100% means the computation is broken (the condition that hid
  // the stale-denominator bug for so long). Fail loudly rather than report it.
  if (cov.percent > 100) {
    console.log("");
    console.log(`❌ Coverage > 100% — the gate is miscomputing.`);
    return 1;
  }

  if (cov.percent < minThreshold) {
    console.log("");
    console.log(`❌ Coverage (${cov.percent}%) is below minimum (${minThreshold}%)`);
    return 1;
  }

  console.log("");
  console.log(`✅ Coverage is acceptable (${cov.percent}% >= ${minThreshold}%)`);
  return 0;
}

/**
 * Hard gate on dangling IDs: a `TRACES:` comment may only name an ID that
 * requirements.md actually defines. Prints every offender with the files that
 * reference it, so the fix is mechanical.
 *
 * TRACES: | DR-093
 */
function reportDangling(data: TracesData): number {
  const dangling = data.dangling ?? [];

  if (dangling.length === 0) {
    console.log("✅ All traced IDs are defined in docs/requirements.md");
    return 0;
  }

  console.log("❌ TRACES reference IDs that docs/requirements.md does not define:");
  console.log("");
  for (const id of dangling) {
    const files = [...new Set((data.requirements[id] ?? []).map((e) => e.file))].sort();
    console.log(`   ${id}`);
    for (const file of files) console.log(`      ${file}`);
  }
  console.log("");
  console.log("Fix each one by either:");
  console.log("   • correcting the ID in the TRACES comment (typo/rename), or");
  console.log("   • adding the requirement as a table row in docs/requirements.md.");
  return 1;
}

// Main — guarded so this module stays importable from extract-traces.test.ts.
if (import.meta.main) {
  const args = process.argv.slice(2);
  const format = args.includes("--format") ? args[args.indexOf("--format") + 1] : "markdown";

  console.error("🔍 Extracting TRACES from codebase...");
  const data = extractTraces();

  const defined = readDefinedRequirements();
  const allTraced = Object.keys(data.requirements);
  data.defined = {
    UR: defined.UR,
    IR: defined.IR,
    DR: defined.DR,
    JA: defined.JA,
    total: defined.total,
  };
  data.coverage = computeCoverage(allTraced, defined);
  data.dangling = findDanglingIds(allTraced, defined);

  if (format === "json") {
    console.log(generateJson(data));
  } else if (format === "coverage") {
    process.exit(reportCoverage(data, MIN_COVERAGE_PERCENT));
  } else if (format === "validate") {
    process.exit(reportDangling(data));
  } else {
    console.log(generateMarkdown(data));
  }

  console.error(`\n✅ Complete! Found ${data.totalTraces} TRACES across ${data.totalFiles} files`);
}