dtourolle
2cde287440
FR-CULL-13 says evidence never writes a rating, flag, label or trash membership, and nothing enforced it. tools/traceability/src/verdicts.rs parses the shipped code with syn and enumerates every write: calls to the catalog setters and trash recorders, SQL that assigns those columns, sidecar Amendment::Judgement, and fields named rating/flag/label. Each site must be in ALLOWED with a reason, as Input (inside a Slint on_* closure, checked structurally), Relay (its callers are checked in turn), Carried (a verdict made elsewhere: sidecar and XMP pulls, sync merge, catalog mirrored to file, duplicates consolidation) or NotAVerdict. Unlisted sites and stale entries both fail `cargo test -p traceability`; `traces verdicts` prints the list. syn and proc-macro2 were already in the lockfile as proc-macro dependencies; this adds the edges, no new crate and no version change.
507 lines
18 KiB
Rust
507 lines
18 KiB
Rust
//! Traceability gate and matrix generator.
|
|
//!
|
|
//! ```text
|
|
//! traces report # write docs/dev/traceability.md
|
|
//! traces json # machine-readable, to stdout
|
|
//! traces check # gate: non-zero exit on failure
|
|
//! traces gestures # write docs/gestures.md and the app's gesture table
|
|
//! traces gestures-check # gate: non-zero exit if either has drifted
|
|
//! traces manual # write docs/manual/index.html from its README
|
|
//! traces manual-check # gate: non-zero exit if the page has drifted
|
|
//! traces verdicts # list every verdict write; non-zero exit on an unlisted one
|
|
//! ```
|
|
//!
|
|
//! The gesture half scans a different tag out of the same files — see
|
|
//! [`traceability::gestures`] for what it is and why it lives here — and holds
|
|
//! the keys the Slint handlers bind to the keys those tags name, in both
|
|
//! directions ([`traceability::keymap`]).
|
|
//!
|
|
//! The gate fails hard on a *misconfigured run* — zero requirements parsed, or
|
|
//! zero source files scanned — rather than reporting a plausible-looking 0%.
|
|
//! A gate that cannot distinguish "nothing is tagged" from "I read nothing" is
|
|
//! how JellyTau's reported 158% went unnoticed for months.
|
|
|
|
use std::collections::{BTreeMap, BTreeSet};
|
|
use std::path::{Path, PathBuf};
|
|
|
|
use anyhow::{bail, Context, Result};
|
|
use traceability::*;
|
|
|
|
/// Directories scanned for tags.
|
|
///
|
|
/// `platform` belongs here as much as the rest: it is where the display-server
|
|
/// and secret-store integrations live, so leaving it out made every
|
|
/// FR-PLAT-\* and NFR-PORT-\* tag in `dr-plat` invisible and understated the
|
|
/// matrix by exactly the requirements the platform layer exists to satisfy.
|
|
const SOURCE_ROOTS: &[&str] = &["core", "ui", "platform", "apps", "tools"];
|
|
|
|
/// Minimum coverage the gate accepts.
|
|
///
|
|
/// 0 today: the requirements register is written but the code that implements
|
|
/// it barely exists. Ratchet upward as tags land; never reset downward.
|
|
/// Structural failures below are unconditional and do not depend on this.
|
|
const MIN_COVERAGE: f64 = 0.0;
|
|
|
|
/// Where the extracted gesture vocabulary is written.
|
|
///
|
|
/// Two artefacts from one scan: the document a person reads, and the table the
|
|
/// application draws its help sheet from. Both committed, both gated, so
|
|
/// neither can quietly stop describing the code.
|
|
const GESTURE_DOC: &str = "docs/gestures.md";
|
|
const GESTURE_TABLE: &str = "ui/dr-ui/src/gesture_book.rs";
|
|
|
|
/// Directories scanned for gestures.
|
|
///
|
|
/// Narrower than `SOURCE_ROOTS`, and not merely as an optimisation. A gesture
|
|
/// is something a *user* performs, so it can only be declared where there is an
|
|
/// interface to perform it on: `ui` and the two application shells. `core` has
|
|
/// no pointer and no finger.
|
|
///
|
|
/// Excluding `tools` is what stops this scanner reading its own documentation.
|
|
/// The tag has to appear in this crate — in the doc comment that teaches the
|
|
/// format, and in the fixtures that test the parser — and every one of those
|
|
/// appearances was being extracted as a broken gesture. A scanner that indexes
|
|
/// its own examples is a scanner nobody can document.
|
|
const GESTURE_ROOTS: &[&str] = &["ui", "apps"];
|
|
|
|
/// The manual's source, and the page rendered from it that the application
|
|
/// carries. Committed and gated like the gesture book — see
|
|
/// [`traceability::manual`] for why it is not rendered at build time.
|
|
const MANUAL_SOURCE: &str = "docs/manual/README.md";
|
|
const MANUAL_PAGE: &str = "docs/manual/index.html";
|
|
|
|
fn main() -> Result<()> {
|
|
let base = repo_root()?;
|
|
let mode = std::env::args().nth(1).unwrap_or_else(|| "report".into());
|
|
|
|
if mode.starts_with("gestures") {
|
|
return run_gestures(&base, mode == "gestures-check");
|
|
}
|
|
if mode.starts_with("manual") {
|
|
return run_manual(&base, mode == "manual-check");
|
|
}
|
|
if mode == "verdicts" {
|
|
return run_verdicts(&base);
|
|
}
|
|
|
|
let req_path = base.join("docs/dev/requirements.md");
|
|
let markdown = std::fs::read_to_string(&req_path)
|
|
.with_context(|| format!("reading {}", req_path.display()))?;
|
|
let defined = parse_defined_requirements(&markdown);
|
|
|
|
let files = collect_sources(&base, SOURCE_ROOTS);
|
|
let mut entries = Vec::new();
|
|
for file in &files {
|
|
let text = std::fs::read_to_string(file).unwrap_or_default();
|
|
let rel = file
|
|
.strip_prefix(&base)
|
|
.unwrap_or(file)
|
|
.to_string_lossy()
|
|
.to_string();
|
|
entries.extend(extract_from_text(&text, &rel));
|
|
}
|
|
|
|
let traced: BTreeSet<String> = entries
|
|
.iter()
|
|
.flat_map(|e| e.requirements.iter().cloned())
|
|
.collect();
|
|
let coverage = compute_coverage(&traced, &defined);
|
|
|
|
match mode.as_str() {
|
|
"json" => println!(
|
|
"{}",
|
|
serde_json::to_string_pretty(&serde_json::json!({
|
|
"filesScanned": files.len(),
|
|
"tagsFound": entries.len(),
|
|
"defined": defined.total(),
|
|
"coverage": coverage,
|
|
}))?
|
|
),
|
|
"check" => {
|
|
print_summary(&files, &entries, &defined, &coverage);
|
|
gate(&files, &defined, &coverage)?;
|
|
println!("\ntraceability gate: PASS");
|
|
}
|
|
_ => {
|
|
let out = base.join("docs/dev/traceability.md");
|
|
let md = render(&files, &entries, &defined, &coverage);
|
|
std::fs::write(&out, md)?;
|
|
print_summary(&files, &entries, &defined, &coverage);
|
|
println!("\nwrote {}", out.display());
|
|
}
|
|
}
|
|
|
|
Ok(())
|
|
}
|
|
|
|
/// Scan the gesture vocabulary, and either write it or check it has not moved.
|
|
///
|
|
/// Writing and checking share every step but the last, so they are one function
|
|
/// with a flag rather than two that could come to disagree about what the
|
|
/// artefact should contain — which is the only way a gate like this fails
|
|
/// dishonestly.
|
|
fn run_gestures(base: &Path, check: bool) -> Result<()> {
|
|
let files = collect_sources(base, GESTURE_ROOTS);
|
|
if files.is_empty() {
|
|
bail!("no source files scanned — misconfigured, not zero gestures");
|
|
}
|
|
|
|
let mut found = Vec::new();
|
|
let mut problems = Vec::new();
|
|
let mut bindings = Vec::new();
|
|
let mut keys_file_seen = false;
|
|
for file in &files {
|
|
let text = std::fs::read_to_string(file).unwrap_or_default();
|
|
let rel = file
|
|
.strip_prefix(base)
|
|
.unwrap_or(file)
|
|
.to_string_lossy()
|
|
.to_string();
|
|
// Never the table this command writes. It quotes the tag in its own
|
|
// header, so scanning it fed every generated gesture back in as a
|
|
// malformed one — and a generator that consumes its own output cannot
|
|
// converge.
|
|
if rel == GESTURE_TABLE {
|
|
continue;
|
|
}
|
|
let (g, p) = gestures::extract_from_text(&text, &rel);
|
|
found.extend(g);
|
|
problems.extend(p);
|
|
if rel.ends_with(".slint") {
|
|
let (b, p) = keymap::extract_bindings(&text, &rel);
|
|
bindings.extend(b);
|
|
problems.extend(p);
|
|
if rel.ends_with(&format!("/{}", keymap::KEYS_FILE)) {
|
|
keys_file_seen = true;
|
|
problems.extend(keymap::check_vocabulary(
|
|
&keymap::slint_named_keys(&text),
|
|
&rel,
|
|
));
|
|
}
|
|
}
|
|
}
|
|
|
|
// A `manual:` must name a section the manual has, checked against the
|
|
// same anchors the bundled page is rendered with.
|
|
let manual_source = std::fs::read_to_string(base.join(MANUAL_SOURCE))
|
|
.with_context(|| format!("reading {MANUAL_SOURCE}"))?;
|
|
problems.extend(gestures::check_manual(
|
|
&found,
|
|
&manual::anchors(&manual_source),
|
|
));
|
|
|
|
println!("files scanned {}", files.len());
|
|
println!("gestures found {}", found.len());
|
|
println!("places {}", gestures::by_section(&found).len());
|
|
println!("keys bound {}", bindings.len());
|
|
|
|
if !problems.is_empty() {
|
|
for p in &problems {
|
|
println!(" {p}");
|
|
}
|
|
bail!("{} malformed gesture tag(s)", problems.len());
|
|
}
|
|
|
|
// The key half, after the tags parse: a key cannot be held to a tag that
|
|
// did not. A run that found no bindings, or no `keys.slint`, read the
|
|
// wrong tree — the application has keys, so zero is a misconfiguration.
|
|
if !keys_file_seen || bindings.is_empty() {
|
|
bail!(
|
|
"no {} or no key bindings found — misconfigured, not an application without keys",
|
|
keymap::KEYS_FILE
|
|
);
|
|
}
|
|
let key_problems = keymap::cross_check(&found, &bindings);
|
|
if !key_problems.is_empty() {
|
|
for p in &key_problems {
|
|
println!(" {p}");
|
|
}
|
|
bail!(
|
|
"{} key(s) bound and undocumented, or documented and unbound",
|
|
key_problems.len()
|
|
);
|
|
}
|
|
|
|
let doc = gestures::render_markdown(&found);
|
|
let table = gestures::render_rust(&found);
|
|
|
|
if check {
|
|
// Read back rather than trusting a timestamp: the artefacts are
|
|
// committed, and what matters is whether the file in the tree says what
|
|
// the source says, however it got there.
|
|
let mut stale = Vec::new();
|
|
for (path, want) in [(GESTURE_DOC, &doc), (GESTURE_TABLE, &table)] {
|
|
let have = std::fs::read_to_string(base.join(path)).unwrap_or_default();
|
|
if have != *want {
|
|
stale.push(path);
|
|
}
|
|
}
|
|
if !stale.is_empty() {
|
|
bail!(
|
|
"{} is stale — run: cargo run -p traceability -- gestures",
|
|
stale.join(" and ")
|
|
);
|
|
}
|
|
println!("\ngesture gate: PASS");
|
|
return Ok(());
|
|
}
|
|
|
|
std::fs::write(base.join(GESTURE_DOC), &doc)?;
|
|
std::fs::write(base.join(GESTURE_TABLE), &table)?;
|
|
println!("\nwrote {GESTURE_DOC} and {GESTURE_TABLE}");
|
|
Ok(())
|
|
}
|
|
|
|
/// List every verdict write with the reason it is allowed, and fail on any
|
|
/// that has none — the same check `cargo test -p traceability` runs, in a
|
|
/// form a reviewer can read ([`traceability::verdicts`]).
|
|
fn run_verdicts(base: &Path) -> Result<()> {
|
|
let files = verdicts::sources(base);
|
|
let report = verdicts::check(&files, verdicts::ALLOWED);
|
|
println!("files scanned {}", files.len());
|
|
println!("writes found {}", report.sites.len());
|
|
for s in &report.sites {
|
|
let kind = verdicts::ALLOWED
|
|
.iter()
|
|
.find(|a| a.file == s.file && a.within == s.within && a.writes == s.writes)
|
|
.map(|a| format!("{:?}", a.kind))
|
|
.unwrap_or_else(|| "UNLISTED".into());
|
|
println!(
|
|
" {kind:<11} {}:{} {} {} {}",
|
|
s.file, s.line, s.within, s.writes, s.detail
|
|
);
|
|
}
|
|
if !report.problems.is_empty() {
|
|
for p in &report.problems {
|
|
println!(" {p}");
|
|
}
|
|
bail!(
|
|
"{} verdict write problem(s) (FR-CULL-13)",
|
|
report.problems.len()
|
|
);
|
|
}
|
|
println!("\nverdict gate: PASS");
|
|
Ok(())
|
|
}
|
|
|
|
/// Render the manual to its page, or check the committed page is that render.
|
|
fn run_manual(base: &Path, check: bool) -> Result<()> {
|
|
let source = std::fs::read_to_string(base.join(MANUAL_SOURCE))
|
|
.with_context(|| format!("reading {MANUAL_SOURCE}"))?;
|
|
let page = manual::render_html(&source);
|
|
let heads = manual::headings(&source);
|
|
println!("headings {}", heads.len());
|
|
if heads.is_empty() {
|
|
bail!("no headings parsed from {MANUAL_SOURCE} — misconfigured, not an empty manual");
|
|
}
|
|
if check {
|
|
let have = std::fs::read_to_string(base.join(MANUAL_PAGE)).unwrap_or_default();
|
|
if have != page {
|
|
bail!("{MANUAL_PAGE} is stale — run: cargo run -p traceability -- manual");
|
|
}
|
|
println!("\nmanual gate: PASS");
|
|
return Ok(());
|
|
}
|
|
std::fs::write(base.join(MANUAL_PAGE), &page)?;
|
|
println!("\nwrote {MANUAL_PAGE}");
|
|
Ok(())
|
|
}
|
|
|
|
/// Structural checks that fail regardless of the coverage threshold.
|
|
fn gate(files: &[PathBuf], defined: &DefinedRequirements, cov: &Coverage) -> Result<()> {
|
|
// A run that parsed nothing is misconfigured, not passing.
|
|
if defined.total() == 0 {
|
|
bail!(
|
|
"no requirements parsed from docs/dev/requirements.md — misconfigured, not 0% coverage"
|
|
);
|
|
}
|
|
if files.is_empty() {
|
|
bail!("no source files scanned — misconfigured, not 0% coverage");
|
|
}
|
|
|
|
// Arithmetic invariant. If this ever trips, the numerator has stopped
|
|
// being an intersection — the exact JellyTau defect.
|
|
if cov.percent > 100.0 {
|
|
bail!(
|
|
"coverage {:.1}% exceeds 100% — numerator is not an intersection of traced and defined",
|
|
cov.percent
|
|
);
|
|
}
|
|
|
|
if !cov.orphaned.is_empty() {
|
|
bail!(
|
|
"{} orphan tag(s) naming requirements that do not exist: {}",
|
|
cov.orphaned.len(),
|
|
cov.orphaned.join(", ")
|
|
);
|
|
}
|
|
|
|
if cov.percent < MIN_COVERAGE {
|
|
bail!(
|
|
"coverage {:.1}% is below the {:.1}% threshold",
|
|
cov.percent,
|
|
MIN_COVERAGE
|
|
);
|
|
}
|
|
|
|
Ok(())
|
|
}
|
|
|
|
fn print_summary(
|
|
files: &[PathBuf],
|
|
entries: &[TraceEntry],
|
|
defined: &DefinedRequirements,
|
|
cov: &Coverage,
|
|
) {
|
|
println!("files scanned {}", files.len());
|
|
println!("tags found {}", entries.len());
|
|
println!("requirements {}", defined.total());
|
|
if !defined.deferred.is_empty() {
|
|
println!("deferred {} (post-v1)", defined.deferred.len());
|
|
}
|
|
println!(
|
|
"coverage {:.1}% ({}/{})",
|
|
cov.percent, cov.covered, cov.total
|
|
);
|
|
if !cov.orphaned.is_empty() {
|
|
println!("orphan tags {}", cov.orphaned.join(", "));
|
|
}
|
|
}
|
|
|
|
fn render(
|
|
files: &[PathBuf],
|
|
entries: &[TraceEntry],
|
|
defined: &DefinedRequirements,
|
|
cov: &Coverage,
|
|
) -> String {
|
|
let mut m = String::new();
|
|
m.push_str("# Requirements traceability matrix\n\n");
|
|
m.push_str("<!-- GENERATED FILE — do not edit by hand. -->\n");
|
|
m.push_str("<!-- Regenerate: cargo run -p traceability -- report -->\n\n");
|
|
m.push_str(
|
|
"Denominators are parsed from [`requirements.md`](requirements.md) at run time, \
|
|
never hardcoded. Coverage is the intersection of tagged and defined IDs over \
|
|
defined IDs, so it cannot exceed 100%. A requirement whose definition line carries \
|
|
`(post-v1)` is defined but deferred: outside the denominator, listed below, and \
|
|
never an orphan.\n\n",
|
|
);
|
|
|
|
m.push_str("## Summary\n\n| Metric | Value |\n|---|---|\n");
|
|
m.push_str(&format!("| Source files scanned | {} |\n", files.len()));
|
|
m.push_str(&format!("| TRACES tags found | {} |\n", entries.len()));
|
|
m.push_str(&format!("| Requirements defined | {} |\n", defined.total()));
|
|
m.push_str(&format!(
|
|
"| Requirements deferred (post-v1) | {} |\n",
|
|
defined.deferred.len()
|
|
));
|
|
m.push_str(&format!("| Requirements covered | {} |\n", cov.covered));
|
|
m.push_str(&format!(
|
|
"| **Coverage** | **{:.1}%** ({}/{}) |\n\n",
|
|
cov.percent, cov.covered, cov.total
|
|
));
|
|
|
|
m.push_str("### By type\n\n| Type | Covered | Defined |\n|---|---|---|\n");
|
|
let mut covered_by_type: BTreeMap<&str, usize> = BTreeMap::new();
|
|
for id in &defined.ids {
|
|
if entries.iter().any(|e| e.requirements.contains(id)) {
|
|
*covered_by_type.entry(type_of(id)).or_insert(0) += 1;
|
|
}
|
|
}
|
|
for (ty, count) in &defined.by_type {
|
|
m.push_str(&format!(
|
|
"| {} | {} | {} |\n",
|
|
ty,
|
|
covered_by_type.get(ty.as_str()).copied().unwrap_or(0),
|
|
count
|
|
));
|
|
}
|
|
m.push('\n');
|
|
|
|
m.push_str("## Orphan tags\n\n");
|
|
m.push_str(
|
|
"A tag naming an ID `requirements.md` does not define — what renumbering produces, \
|
|
and what a typo produces.\n\n",
|
|
);
|
|
if cov.orphaned.is_empty() {
|
|
m.push_str("_None._\n\n");
|
|
} else {
|
|
for id in &cov.orphaned {
|
|
m.push_str(&format!("- `{id}`\n"));
|
|
}
|
|
m.push('\n');
|
|
}
|
|
|
|
m.push_str("## Tagged requirements\n\n| ID | Tagged in |\n|---|---|\n");
|
|
let mut by_req: BTreeMap<&String, Vec<&TraceEntry>> = BTreeMap::new();
|
|
for e in entries {
|
|
for r in &e.requirements {
|
|
if defined.ids.contains(r) {
|
|
by_req.entry(r).or_default().push(e);
|
|
}
|
|
}
|
|
}
|
|
for (id, es) in &by_req {
|
|
let mut locs: Vec<String> = es
|
|
.iter()
|
|
.map(|e| format!("[`{}:{}`](../../{}#L{})", e.file, e.line, e.file, e.line))
|
|
.collect();
|
|
locs.sort();
|
|
locs.dedup();
|
|
m.push_str(&format!("| {} | {} |\n", id, locs.join(", ")));
|
|
}
|
|
m.push('\n');
|
|
|
|
m.push_str("## Deferred (post-v1)\n\n");
|
|
m.push_str(
|
|
"Defined in `requirements.md` and marked `(post-v1)` on the defining line. Not in \
|
|
the denominator; a tag naming one is recorded here rather than counted.\n\n",
|
|
);
|
|
if defined.deferred.is_empty() {
|
|
m.push_str("_None._\n\n");
|
|
} else {
|
|
for id in &defined.deferred {
|
|
if cov.deferred_tagged.contains(id) {
|
|
let mut locs: Vec<String> = entries
|
|
.iter()
|
|
.filter(|e| e.requirements.contains(id))
|
|
.map(|e| format!("[`{}:{}`](../../{}#L{})", e.file, e.line, e.file, e.line))
|
|
.collect();
|
|
locs.sort();
|
|
locs.dedup();
|
|
m.push_str(&format!("- {id} — tagged in {}\n", locs.join(", ")));
|
|
} else {
|
|
m.push_str(&format!("- {id}\n"));
|
|
}
|
|
}
|
|
m.push('\n');
|
|
}
|
|
|
|
m.push_str("## Not yet tagged\n\n");
|
|
m.push_str(&format!(
|
|
"{} of {} requirements have no implementation tag. Expected while the \
|
|
codebase is young; each should gain one as it is built.\n\n",
|
|
cov.untraced.len(),
|
|
defined.total()
|
|
));
|
|
if !cov.untraced.is_empty() {
|
|
m.push_str("<details><summary>Show untagged requirements</summary>\n\n");
|
|
for id in &cov.untraced {
|
|
m.push_str(&format!("- {id}\n"));
|
|
}
|
|
m.push_str("\n</details>\n");
|
|
}
|
|
|
|
m
|
|
}
|
|
|
|
/// The repo root, found by walking up from the executable's manifest dir.
|
|
fn repo_root() -> Result<PathBuf> {
|
|
let mut dir = Path::new(env!("CARGO_MANIFEST_DIR")).to_path_buf();
|
|
while !dir.join("docs/dev/requirements.md").exists() {
|
|
if !dir.pop() {
|
|
bail!("could not locate repo root (no docs/dev/requirements.md above the tool)");
|
|
}
|
|
}
|
|
Ok(dir)
|
|
}
|