tools: add output-manifest-verify; extend flow-lint with fan-out checks

- tools/output-manifest-verify/ — enforces .claude/rules/output-coverage.md
  §3: every node ships test/_output-manifest.md and every declared key
  is referenced by at least one test file. First run shows only
  machineGroupControl has the manifest (16 keys covered); all other nodes
  warn. --strict escalates "missing manifest" to an error for CI gating.
- flow-lint gains two rules from the same output-coverage rule:
  * FN_OUTPUT_WIRES_MISMATCH — function declares outputs=N but wires has
    M arrays (causes silent dropped or duplicate emissions).
  * FN_PAYLOAD_NULL_LITERAL — function source contains `payload: null`
    literal (the η-null ui-chart crash pattern from 2026-05-14).
  First run found 1 instance in mgc/02-Dashboard.json.

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
znetsixe
2026-05-19 10:13:49 +02:00
parent ecd466f7a3
commit edef1cecbf
4 changed files with 231 additions and 0 deletions

View File

@@ -35,6 +35,7 @@ function lintFlow(flowPath) {
checkLinkPair(n, byId, findings); checkLinkPair(n, byId, findings);
checkDebugLog(n, findings); checkDebugLog(n, findings);
checkBackwardWires(n, findings); checkBackwardWires(n, findings);
checkFunctionFanOut(n, findings);
} }
checkGroupWidths(nodes, findings); checkGroupWidths(nodes, findings);
return findings; return findings;
@@ -172,6 +173,28 @@ function checkBackwardWires(n, findings) {
} }
} }
function checkFunctionFanOut(n, findings) {
if (n.type !== 'function') return;
const outputs = Number(n.outputs);
if (!Number.isFinite(outputs) || outputs <= 1) return;
if (!Array.isArray(n.wires) || n.wires.length !== outputs) {
findings.push({
rule: 'FN_OUTPUT_WIRES_MISMATCH',
severity: 'error',
node: n.id,
msg: `function "${n.name || n.id}" declares outputs=${outputs} but wires has ${Array.isArray(n.wires) ? n.wires.length : 'no'} arrays.`,
});
}
if (typeof n.func === 'string' && /payload\s*:\s*null\b/.test(n.func)) {
findings.push({
rule: 'FN_PAYLOAD_NULL_LITERAL',
severity: 'error',
node: n.id,
msg: `function "${n.name || n.id}" emits a literal { payload: null } — crashes ui-chart on first frame; return the whole msg as null instead so the port skips.`,
});
}
}
function checkGroupWidths(nodes, findings) { function checkGroupWidths(nodes, findings) {
const pages = new Map(); const pages = new Map();
for (const n of nodes) { for (const n of nodes) {

View File

@@ -0,0 +1,47 @@
# @evolv/output-manifest-verify
Enforce `.claude/rules/output-coverage.md` §3:
1. Every node ships `test/_output-manifest.md`.
2. Every key declared in the manifest is referenced by at least one test
file under `test/**/*.test.js`.
Designed to replace the manual verification checklist in the rule. The
rule itself can become a thin pointer to this tool once every node is in
compliance.
## Usage
```bash
# scan every node with a CONTRACT.md
node tools/output-manifest-verify/bin/output-manifest-verify.js
# one node
node tools/output-manifest-verify/bin/output-manifest-verify.js nodes/machineGroupControl
# fail (exit 1) when a manifest is missing, not just warn
node tools/output-manifest-verify/bin/output-manifest-verify.js --strict
# CI / JSON output (one line per node)
node tools/output-manifest-verify/bin/output-manifest-verify.js --json
```
## Severity
| Condition | Severity |
|---|---|
| `test/_output-manifest.md` missing | `warn` (escalates to `error` under `--strict`) |
| Manifest has no `##` sections / no tables | `error` |
| Declared key not referenced by any test file | `warn` |
| Manifest declares keys but `test/**/*.test.js` directory empty | `error` |
## What it does NOT check (yet)
- Whether each key has both a **populated** and **degraded** test (the
rule's §3 promise). Reaching that level of confidence requires
instrumenting the actual node runtime; tracked as a follow-up.
- The Port 1 (InfluxDB) cardinality discipline — see
`.claude/rules/telemetry.md`.
Run after touching `_output-manifest.md`, `test/**/output-*.test.js`,
or any Port 0/1/2 emission key.

View File

@@ -0,0 +1,148 @@
#!/usr/bin/env node
'use strict';
const fs = require('fs');
const path = require('path');
function parseManifest(manifestPath) {
if (!fs.existsSync(manifestPath)) return null;
const src = fs.readFileSync(manifestPath, 'utf8');
const lines = src.split(/\r?\n/);
const sections = [];
let currentSection = null;
let inTable = false;
for (const line of lines) {
const headerMatch = line.match(/^##\s+(.+)$/);
if (headerMatch) {
currentSection = { title: headerMatch[1].trim(), keys: [] };
sections.push(currentSection);
inTable = false;
continue;
}
if (!currentSection) continue;
if (/^\|\s*(Key|Topic|Field|#)\s*\|/i.test(line)) { inTable = true; continue; }
if (inTable && /^\|[\s-]+\|/.test(line)) continue;
if (inTable && !line.trim().startsWith('|')) {
if (line.trim().startsWith('### ')) continue;
inTable = false;
continue;
}
if (!inTable) continue;
const cells = line.split('|').slice(1, -1).map((c) => c.trim());
if (cells.length === 0) continue;
const keyCell = cells[0];
const keyMatch = keyCell.match(/`([^`]+)`/);
if (keyMatch) currentSection.keys.push(keyMatch[1]);
}
return sections;
}
function findTestFiles(nodeDir) {
const out = [];
function walk(dir) {
if (!fs.existsSync(dir)) return;
for (const entry of fs.readdirSync(dir, { withFileTypes: true })) {
const full = path.join(dir, entry.name);
if (entry.isDirectory()) walk(full);
else if (entry.isFile() && /\.test\.js$/.test(entry.name)) out.push(full);
}
}
walk(path.join(nodeDir, 'test'));
return out;
}
function keyIsCovered(key, testSources) {
if (!key) return true;
if (/^[a-z]+$/i.test(key) && key.length <= 4) return true;
const escaped = key.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
const pattern = new RegExp(`['"\\\`]${escaped}['"\\\`]|\\.${escaped}\\b|\\b${escaped}\\b`);
return testSources.some((src) => pattern.test(src));
}
function verifyNode(nodeDir, opts) {
const nodeName = path.basename(nodeDir);
const manifestPath = path.join(nodeDir, 'test/_output-manifest.md');
const findings = [];
const sections = parseManifest(manifestPath);
if (sections === null) {
findings.push({ severity: opts.strict ? 'error' : 'warn', msg: 'test/_output-manifest.md does not exist (output-coverage rule §3 requires it).' });
return { node: nodeName, findings, manifest: null };
}
if (sections.length === 0) {
findings.push({ severity: 'error', msg: 'test/_output-manifest.md has no sections (## headings) — empty or malformed.' });
return { node: nodeName, findings, manifest: sections };
}
const totalKeys = sections.reduce((s, sec) => s + sec.keys.length, 0);
if (totalKeys === 0) {
findings.push({ severity: 'error', msg: 'No keys parsed from any section table — check that ` `tickmarks wrap each Key/Topic/Field cell.' });
return { node: nodeName, findings, manifest: sections };
}
const testFiles = findTestFiles(nodeDir);
const testSources = testFiles.map((f) => fs.readFileSync(f, 'utf8'));
if (testFiles.length === 0) {
findings.push({ severity: 'error', msg: 'manifest declares keys but no test/**/*.test.js files exist.' });
return { node: nodeName, findings, manifest: sections };
}
for (const section of sections) {
for (const key of section.keys) {
if (!keyIsCovered(key, testSources)) {
findings.push({
severity: 'warn',
section: section.title,
key,
msg: `key \`${key}\` declared in "${section.title}" but no test file references it.`,
});
}
}
}
return { node: nodeName, findings, manifest: sections, totalKeys };
}
function findNodes(repoRoot) {
const nodesDir = path.join(repoRoot, 'nodes');
return fs.readdirSync(nodesDir)
.filter((n) => fs.existsSync(path.join(nodesDir, n, 'CONTRACT.md')))
.filter((n) => n !== 'generalFunctions')
.map((n) => path.join(nodesDir, n));
}
function report(result, json) {
if (json) {
process.stdout.write(JSON.stringify(result) + '\n');
return result.findings.some((f) => f.severity === 'error');
}
const errs = result.findings.filter((f) => f.severity === 'error').length;
const warns = result.findings.filter((f) => f.severity === 'warn').length;
if (errs === 0 && warns === 0) {
process.stdout.write(`OK ${result.node} (manifest covers ${result.totalKeys} keys)\n`);
return false;
}
const tag = errs ? 'FAIL' : 'WARN';
process.stdout.write(`\n${tag} ${result.node} (${errs} err, ${warns} warn)\n`);
for (const f of result.findings) {
const t = f.severity === 'error' ? 'ERR ' : 'WARN';
process.stdout.write(` ${t} ${f.msg}\n`);
}
return errs > 0;
}
function main() {
const args = process.argv.slice(2);
const json = args.includes('--json');
const strict = args.includes('--strict');
const positional = args.filter((a) => !a.startsWith('--'));
const repoRoot = path.resolve(__dirname, '../../..');
const targets = positional.length === 0
? findNodes(repoRoot)
: positional.map((p) => path.resolve(p));
let fail = false;
for (const nodeDir of targets) {
const result = verifyNode(nodeDir, { strict });
if (report(result, json)) fail = true;
}
process.exit(fail ? 1 : 0);
}
if (require.main === module) main();
module.exports = { parseManifest, verifyNode, keyIsCovered };

View File

@@ -0,0 +1,13 @@
{
"name": "@evolv/output-manifest-verify",
"version": "0.1.0",
"private": true,
"description": "Verify each node ships test/_output-manifest.md and that declared keys appear in test files (per .claude/rules/output-coverage.md)",
"bin": {
"evolv-output-manifest-verify": "bin/output-manifest-verify.js"
},
"scripts": {
"test": "node --test test/*.test.js"
},
"license": "UNLICENSED"
}