jellytau/scripts/extract-traces.ts

#!/usr/bin/env bun
/**
 * Extract TRACES from source code and generate requirement mapping
 *
 * Usage:
 *   bun run scripts/extract-traces.ts
 *   bun run scripts/extract-traces.ts --format json
 *   bun run scripts/extract-traces.ts --format markdown > docs/traceability.md
 */

import * as fs from "fs";
import * as path from "path";
import { execSync } from "child_process";

interface TraceEntry {
  file: string;
  line: number;
  context: string;
  requirements: string[];
}

interface RequirementMapping {
  [reqId: string]: TraceEntry[];
}

interface TracesData {
  timestamp: string;
  totalFiles: number;
  totalTraces: number;
  requirements: RequirementMapping;
  byType: {
    UR: string[];
    IR: string[];
    DR: string[];
    JA: string[];
  };
  /** Requirements *defined* in requirements.md — the coverage denominators. */
  defined?: { UR: number; IR: number; DR: number; JA: number; total: number };
  coverage?: CoverageResult;
  /** Traced IDs of any type that requirements.md does not define. */
  dangling?: string[];
}

/**
 * Minimum overall requirement coverage the traceability gate accepts.
 *
 * **Ratchet policy: this number only ever goes up.** It is set a few points
 * below the coverage actually achieved, so a real regression trips it instead of
 * being absorbed by slack. It sat at 50 while true coverage was 86%, which meant
 * half the matrix could rot before CI noticed. When coverage rises durably,
 * raise this to sit just under the new figure. Do **not** lower it to make a
 * failing build pass — add the missing TRACES comments instead.
 *
 * `.gitea/workflows/traceability-check.yml` carries the same number as
 * `MIN_THRESHOLD`; `scripts/extract-traces.test.ts` fails if the two drift.
 *
 * TRACES: | DR-093
 */
export const MIN_COVERAGE_PERCENT = 82;

// Repo root, derived from this script's location (scripts/ -> repo root).
// Must NOT be hardcoded to a developer's machine, or CI checkouts see no files.
//
// `import.meta.dir` is a Bun extension and is undefined when this module is
// imported by vitest (which runs it as an ordinary ESM module), so fall back to
// import.meta.url — this file must stay importable for extract-traces.test.ts.
const SCRIPT_DIR =
  import.meta.dir ?? path.dirname(new URL(import.meta.url).pathname);
const BASE_DIR = path.resolve(SCRIPT_DIR, "..");

const TRACES_PATTERN = /TRACES:\s*([^\n]+)/gi;
const REQ_ID_PATTERN = /([A-Z]{2})-(\d{3})/g;

function extractRequirementIds(tracesString: string): string[] {
  const matches = [...tracesString.matchAll(REQ_ID_PATTERN)];
  return matches.map((m) => `${m[1]}-${m[2]}`);
}

function getAllSourceFiles(): string[] {
  const baseDir = BASE_DIR;
  // `scripts` is scanned too: build tooling implements requirements (e.g.
  // DR-093, the coverage engine itself) and would otherwise be invisible to the
  // very matrix it generates.
  const patterns = ["src", "src-tauri/src", "scripts"];
  const files: string[] = [];

  function walkDir(dir: string) {
    try {
      const entries = fs.readdirSync(dir, { withFileTypes: true });
      for (const entry of entries) {
        const fullPath = path.join(dir, entry.name);
        const relativePath = path.relative(baseDir, fullPath);

        // Skip node_modules, target, build
        if (
          relativePath.includes("node_modules") ||
          relativePath.includes("target") ||
          relativePath.includes("build") ||
          relativePath.includes(".git")
        ) {
          continue;
        }

        if (entry.isDirectory()) {
          walkDir(fullPath);
        } else if (
          entry.name.endsWith(".ts") ||
          entry.name.endsWith(".svelte") ||
          entry.name.endsWith(".rs")
        ) {
          files.push(fullPath);
        }
      }
    } catch (error) {
      // Skip directories we can't read
    }
  }

  for (const pattern of patterns) {
    const dir = path.join(baseDir, pattern);
    if (fs.existsSync(dir)) {
      walkDir(dir);
    }
  }

  return files;
}

function extractTraces(): TracesData {
  const requirementMap: RequirementMapping = {};
  const byType: Record<string, Set<string>> = {
    UR: new Set(),
    IR: new Set(),
    DR: new Set(),
    JA: new Set(),
  };

  let totalTraces = 0;
  const baseDir = BASE_DIR;

  const files = getAllSourceFiles();

  for (const fullPath of files) {
    try {
      const content = fs.readFileSync(fullPath, "utf-8");
      const lines = content.split("\n");
      const relativePath = path.relative(baseDir, fullPath);

      let match;
      TRACES_PATTERN.lastIndex = 0;

      while ((match = TRACES_PATTERN.exec(content)) !== null) {
        const tracesStr = match[1];
        const reqIds = extractRequirementIds(tracesStr);

        if (reqIds.length === 0) continue;

        // Find line number
        const beforeMatch = content.substring(0, match.index);
        const lineNum = beforeMatch.split("\n").length - 1;

        // Get context (function/class name if available)
        let context = "Unknown";
        for (let i = lineNum; i >= Math.max(0, lineNum - 10); i--) {
          const line = lines[i];
          if (
            line.includes("function ") ||
            line.includes("export const ") ||
            line.includes("pub fn ") ||
            line.includes("pub enum ") ||
            line.includes("pub struct ") ||
            line.includes("impl ") ||
            line.includes("async function ") ||
            line.includes("class ") ||
            line.includes("export type ")
          ) {
            context = line
              .trim()
              .replace(/^\s*\/\/\s*/, "")
              .replace(/^\s*\/\*\*\s*/, "");
            break;
          }
        }

        const entry: TraceEntry = {
          file: relativePath,
          line: lineNum + 1,
          context,
          requirements: reqIds,
        };

        for (const reqId of reqIds) {
          if (!requirementMap[reqId]) {
            requirementMap[reqId] = [];
          }
          requirementMap[reqId].push(entry);

          // Track by type
          const type = reqId.substring(0, 2);
          if (byType[type]) {
            byType[type].add(reqId);
          }
        }

        totalTraces++;
      }
    } catch (error) {
      // Skip files we can't read
    }
  }

  return {
    timestamp: new Date().toISOString(),
    totalFiles: files.length,
    totalTraces,
    requirements: requirementMap,
    byType: {
      UR: Array.from(byType["UR"]).sort(),
      IR: Array.from(byType["IR"]).sort(),
      DR: Array.from(byType["DR"]).sort(),
      JA: Array.from(byType["JA"]).sort(),
    },
  };
}

// ---------------------------------------------------------------------------
// Coverage: how many *defined* requirements are actually traced.
//
// The denominators MUST be derived from requirements.md, never hardcoded. The
// CI gate previously divided by frozen literals (UR/39, IR/24, DR/48, JA/3,
// total 114) while the real file had grown to 211 requirements, so it reported
// 158% coverage and the 50% threshold became unreachable — the gate could not
// fail. See docs/specs/traceability-gate-repair.md.
//
// TRACES: | DR-093
// ---------------------------------------------------------------------------

export interface DefinedRequirements {
  UR: number;
  IR: number;
  DR: number;
  JA: number;
  total: number;
  /** Requirement IDs (UR/IR/DR/JA) — the coverage denominator. */
  ids: Set<string>;
  /** Test IDs (UT/IT) from §4. A separate taxonomy: never part of coverage. */
  testIds: Set<string>;
}

export interface CoverageResult {
  covered: number;
  total: number;
  percent: number;
  /** Traced in code but not defined in requirements.md (typo, or deleted req). */
  orphaned: string[];
}

/**
 * A requirement is *defined* only where its ID is the leading cell of a markdown
 * table row: `| DR-001 | … |`.
 *
 * This deliberately ignores IDs in the "Traces To" column and in prose — a
 * naive scan for /DR-\d{3}/ counts those as definitions and inflates the
 * denominator. IDs are deduplicated because requirements.md lists each UR twice
 * (once in §1 as a definition, again in §3's traceability matrix), which would
 * otherwise double the UR count from 61 to 121.
 *
 * TRACES: | DR-093
 */
export function countDefinedRequirements(markdown: string): DefinedRequirements {
  const ids = new Set<string>();
  const testIds = new Set<string>();
  const ROW_ID = /^\|\s*(UR|IR|DR|JA|UT|IT)-(\d{3})\s*\|/;

  for (const line of markdown.split("\n")) {
    const match = line.match(ROW_ID);
    if (!match) continue;
    const id = `${match[1]}-${match[2]}`;
    // UT/IT rows live in §4 and are collected separately: they must not enter
    // the coverage denominator, but they still need to exist for a `TRACES:`
    // comment to be allowed to name them (see findDanglingIds).
    if (match[1] === "UT" || match[1] === "IT") testIds.add(id);
    else ids.add(id);
  }

  const countOf = (type: string) =>
    [...ids].filter((id) => id.startsWith(`${type}-`)).length;

  return {
    UR: countOf("UR"),
    IR: countOf("IR"),
    DR: countOf("DR"),
    JA: countOf("JA"),
    total: ids.size,
    ids,
    testIds,
  };
}

/**
 * Every traced ID that requirements.md defines nowhere — a typo, a rename that
 * missed a call site, or a reference to a deleted requirement.
 *
 * This is broader than `CoverageResult.orphaned`, which only ever considers the
 * four requirement types because a UT/IT entry among the orphans would corrupt
 * the coverage ratio's reporting. Dangling detection has no such constraint, so
 * it checks all six ID types against both defined sets. Before it existed, the
 * extractor accepted any well-formed ID silently: `DR-189` and `UT-188` were
 * referenced from `controlsVisibility.ts` and `VideoPlayer.svelte` for months
 * without being defined anywhere, and nothing reported it.
 *
 * TRACES: | DR-093
 */
export function findDanglingIds(
  tracedIds: string[],
  defined: DefinedRequirements
): string[] {
  const KNOWN_TYPE = /^(UR|IR|DR|JA|UT|IT)-\d{3}$/;

  const dangling = new Set(
    tracedIds
      .filter((id) => KNOWN_TYPE.test(id))
      .filter((id) => !defined.ids.has(id) && !defined.testIds.has(id))
  );

  return [...dangling].sort();
}

/**
 * Coverage is the *intersection* of traced and defined IDs over defined IDs.
 *
 * Using the raw traced count as the numerator is what lets a ratio exceed 100%:
 * a TRACES comment naming a requirement that no longer exists would count as
 * covered. Those IDs are reported as `orphaned` so they get fixed rather than
 * silently counted or silently dropped.
 *
 * TRACES: | DR-093
 */
export function computeCoverage(
  tracedIds: string[],
  defined: DefinedRequirements
): CoverageResult {
  // Only the four *requirement* types participate in coverage. UT/IT are test
  // identifiers defined in §4 of requirements.md — a different taxonomy, and
  // flagging them as orphans would bury real typos in ~60 lines of noise.
  const isRequirement = (id: string) => /^(UR|IR|DR|JA)-\d{3}$/.test(id);

  const traced = new Set(tracedIds.filter(isRequirement));
  const covered = [...traced].filter((id) => defined.ids.has(id));
  const orphaned = [...traced].filter((id) => !defined.ids.has(id)).sort();

  return {
    covered: covered.length,
    total: defined.total,
    percent:
      defined.total === 0
        ? 0
        : Math.round((covered.length / defined.total) * 100),
    orphaned,
  };
}

/** Read requirements.md from the repo and count what it defines. */
export function readDefinedRequirements(): DefinedRequirements {
  const reqPath = path.join(BASE_DIR, "docs", "requirements.md");
  return countDefinedRequirements(fs.readFileSync(reqPath, "utf-8"));
}

function generateMarkdown(data: TracesData): string {
  let md = `# Code Traceability Matrix

**Generated:** ${new Date(data.timestamp).toLocaleString()}

## Summary

- **Total Files Scanned:** ${data.totalFiles}
- **Total TRACES Found:** ${data.totalTraces}
- **Requirements Covered:**
  - User Requirements (UR): ${data.byType.UR.length}
  - Integration Requirements (IR): ${data.byType.IR.length}
  - Development Requirements (DR): ${data.byType.DR.length}
  - Jellyfin API Requirements (JA): ${data.byType.JA.length}

## Requirements by Type

### User Requirements (UR)
\`\`\`
${data.byType.UR.join(", ")}
\`\`\`

### Integration Requirements (IR)
\`\`\`
${data.byType.IR.join(", ")}
\`\`\`

### Development Requirements (DR)
\`\`\`
${data.byType.DR.join(", ")}
\`\`\`

### Jellyfin API Requirements (JA)
\`\`\`
${data.byType.JA.join(", ")}
\`\`\`

## Detailed Mapping

`;

  // Sort requirements by ID
  const sortedReqs = Object.keys(data.requirements).sort((a, b) => {
    const typeA = a.substring(0, 2);
    const typeB = b.substring(0, 2);
    const typeOrder = { UR: 0, IR: 1, DR: 2, JA: 3 };
    if (typeOrder[typeA] !== typeOrder[typeB]) {
      return (typeOrder[typeA] || 4) - (typeOrder[typeB] || 4);
    }
    return a.localeCompare(b);
  });

  for (const reqId of sortedReqs) {
    const entries = data.requirements[reqId];
    md += `### ${reqId}\n\n`;
    md += `**Locations:** ${entries.length} file(s)\n\n`;

    for (const entry of entries) {
      md += `- **File:** [\`${entry.file}\`](${entry.file}#L${entry.line})\n`;
      md += `  - **Line:** ${entry.line}\n`;
      const contextPreview = entry.context.substring(0, 70);
      md += `  - **Context:** \`${contextPreview}${entry.context.length > 70 ? "..." : ""}\`\n`;
    }
    md += "\n";
  }

  return md;
}

function generateJson(data: TracesData): string {
  return JSON.stringify(data, null, 2);
}

/**
 * Human-readable coverage report; exits non-zero below the threshold so this is
 * runnable as a local gate (`bun run traces:coverage`), not just in CI.
 *
 * TRACES: | DR-093
 */
function reportCoverage(data: TracesData, minThreshold: number): number {
  const defined = data.defined!;
  const cov = data.coverage!;

  const definedIds = readDefinedRequirements().ids;

  console.log("📋 Requirement coverage (traced / defined):");
  for (const type of ["UR", "IR", "DR", "JA"] as const) {
    const traced = data.byType[type].filter((id) => definedIds.has(id)).length;
    console.log(`   ${type}: ${traced} / ${defined[type]}`);
  }
  console.log("");
  console.log(`📈 Overall: ${cov.covered} / ${cov.total} (${cov.percent}%)`);

  if (cov.orphaned.length > 0) {
    console.log("");
    console.log(
      `⚠️  Traced but not defined in requirements.md: ${cov.orphaned.join(", ")}`
    );
    console.log("   Fix the TRACES comment or add the requirement.");
  }

  if (data.dangling && data.dangling.length > 0) {
    console.log("");
    console.log(
      `⚠️  Dangling IDs (incl. UT/IT): ${data.dangling.join(", ")} — run \`bun run traces:validate\`.`
    );
  }

  // A ratio above 100% means the computation is broken (the condition that hid
  // the stale-denominator bug for so long). Fail loudly rather than report it.
  if (cov.percent > 100) {
    console.log("");
    console.log(`❌ Coverage > 100% — the gate is miscomputing.`);
    return 1;
  }

  if (cov.percent < minThreshold) {
    console.log("");
    console.log(`❌ Coverage (${cov.percent}%) is below minimum (${minThreshold}%)`);
    return 1;
  }

  console.log("");
  console.log(`✅ Coverage is acceptable (${cov.percent}% >= ${minThreshold}%)`);
  return 0;
}

/**
 * Hard gate on dangling IDs: a `TRACES:` comment may only name an ID that
 * requirements.md actually defines. Prints every offender with the files that
 * reference it, so the fix is mechanical.
 *
 * TRACES: | DR-093
 */
function reportDangling(data: TracesData): number {
  const dangling = data.dangling ?? [];

  if (dangling.length === 0) {
    console.log("✅ All traced IDs are defined in docs/requirements.md");
    return 0;
  }

  console.log("❌ TRACES reference IDs that docs/requirements.md does not define:");
  console.log("");
  for (const id of dangling) {
    const files = [
      ...new Set((data.requirements[id] ?? []).map((e) => e.file)),
    ].sort();
    console.log(`   ${id}`);
    for (const file of files) console.log(`      ${file}`);
  }
  console.log("");
  console.log("Fix each one by either:");
  console.log("   • correcting the ID in the TRACES comment (typo/rename), or");
  console.log("   • adding the requirement as a table row in docs/requirements.md.");
  return 1;
}

// Main — guarded so this module stays importable from extract-traces.test.ts.
if (import.meta.main) {
  const args = process.argv.slice(2);
  const format = args.includes("--format")
    ? args[args.indexOf("--format") + 1]
    : "markdown";

  console.error("🔍 Extracting TRACES from codebase...");
  const data = extractTraces();

  const defined = readDefinedRequirements();
  const allTraced = Object.keys(data.requirements);
  data.defined = {
    UR: defined.UR,
    IR: defined.IR,
    DR: defined.DR,
    JA: defined.JA,
    total: defined.total,
  };
  data.coverage = computeCoverage(allTraced, defined);
  data.dangling = findDanglingIds(allTraced, defined);

  if (format === "json") {
    console.log(generateJson(data));
  } else if (format === "coverage") {
    process.exit(reportCoverage(data, MIN_COVERAGE_PERCENT));
  } else if (format === "validate") {
    process.exit(reportDangling(data));
  } else {
    console.log(generateMarkdown(data));
  }

  console.error(
    `\n✅ Complete! Found ${data.totalTraces} TRACES across ${data.totalFiles} files`
  );
}