Files
eip4-web/tools/modeler-fidelity-sweep.mjs
zhengwr b391ee9f16 fix(modeler): 保真扫描回归加固——refs袋+unresolved入门+规模门+stderr(评审I1/M1/M2/M5)
- I1 引用回归盲区:surface() 新增 refs 袋(sourceRef/targetRef/bpmnElement/
  processRef/attachedToRef/default,与 sanitizeIds 同覆盖面)并入比对键;normWarn
  的 unresolved reference 类 warning 计入硬门 failures(净化/往返破坏连线的直接信号)。
- M1:删除未使用的 realFail。
- M2:新增 EXPECTED_TOTAL 环境变量(默认 1512),实际语料数不符即判失败;
  decodeFailures.length 并入硬门(过滤集内本应 100% 干净 UTF8)。
- M5:崩溃分支透出子进程 e.stderr。
- M3/M4:命名空间字面前缀识别的语料前提、extElemText trim 口径各加注释声明。

重跑实测(node tools/modeler-fidelity-sweep.mjs):匹配 1512/1512 零丢失,
解析异常 0,解码失败 0,warnings=0,exit 0;EXPECTED_TOTAL=9999 时 exit 1
(规模门生效)。npx vitest run src/modeler 12/12 绿;npx tsc -b 干净(exit 0)。

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
2026-07-02 10:15:49 +08:00

326 lines
14 KiB
JavaScript
Raw Permalink Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
#!/usr/bin/env node
/**
* 全量真实流程 moddle round-trip 保真扫描(Task 1.2)
*
* 对本地 eip4 库(5433, search_path=eip3)里 act_ge_bytearray 的全部真实 BPMN
* (name_ like %.bpmn% / %.xml)逐条做 bpmn-moddle fromXML→toXML 往返,验证
* 「所有 activiti / flowable 扩展属性(含值,实体解码归一后) + 扩展元素结构
* + 全部 bpmn 元素 id 集合」在往返前后多重集一致(数据级零丢失,非字节级)。
*
* 取数:psql 子进程(不引 pg 依赖);命名空间嗅探后单载 activiti 或 flowable
* (T1.1 实证:双载会因跨包同名 extends 冲突丢 userTask)。进 moddle 前先净化
* 中文/非法 id(镜像后端 ProcessDiagramController.sanitizeBpmnIds;moddle-xml
* 拒绝非 ASCII-NCName id,1445/1512 文档 process id 为中文)。
*
* 用法:node tools/modeler-fidelity-sweep.mjs
* 环境变量:PG_HOST(localhost) PG_PORT(5433) PG_DB(eip4) PG_SCHEMA(eip3)
*/
import { execFileSync } from 'node:child_process';
import { readFileSync } from 'node:fs';
import { fileURLToPath } from 'node:url';
import { dirname, resolve } from 'node:path';
import BpmnModdle from 'bpmn-moddle';
const HERE = dirname(fileURLToPath(import.meta.url));
const activiti = JSON.parse(readFileSync(resolve(HERE, '../src/modeler/moddle/activiti.json'), 'utf8'));
const flowable = JSON.parse(readFileSync(resolve(HERE, '../src/modeler/moddle/flowable.json'), 'utf8'));
const PG_HOST = process.env.PG_HOST || 'localhost';
const PG_PORT = process.env.PG_PORT || '5433';
const PG_DB = process.env.PG_DB || 'eip4';
const PG_SCHEMA = process.env.PG_SCHEMA || 'eip3';
// ── 取数:psql 拉 id|name|base64(base64 去换行,同勘探法)───────────────────
function pullCorpus() {
const sql =
`set search_path=${PG_SCHEMA},public; ` +
`select b.id_||'|'||b.name_||'|'||translate(encode(b.bytes_,'base64'),E'\\n','') ` +
`from act_ge_bytearray b where b.name_ like '%.bpmn%' or b.name_ like '%.xml';`;
const out = execFileSync(
'psql',
['-h', PG_HOST, '-p', PG_PORT, '-d', PG_DB, '-tA', '-c', sql],
{ encoding: 'utf8', maxBuffer: 1024 * 1024 * 1024 },
);
const docs = [];
for (const line of out.split('\n')) {
if (!line) continue;
const i1 = line.indexOf('|');
const i2 = line.indexOf('|', i1 + 1);
if (i1 < 0 || i2 < 0) continue;
docs.push({ id: line.slice(0, i1), name: line.slice(i1 + 1, i2), b64: line.slice(i2 + 1) });
}
return docs;
}
// ── XML 实体解码归一(数据级比对:&quot;→" &#34;→" 全角/中文原样)────────────
function decodeEntities(s) {
return String(s)
.replace(/&#x([0-9a-fA-F]+);/g, (_, h) => String.fromCodePoint(parseInt(h, 16)))
.replace(/&#(\d+);/g, (_, d) => String.fromCodePoint(parseInt(d, 10)))
.replace(/&quot;/g, '"')
.replace(/&apos;/g, "'")
.replace(/&lt;/g, '<')
.replace(/&gt;/g, '>')
.replace(/&amp;/g, '&'); // amp 最后解,避免 &amp;#34; 被二次解码
}
// ── id 净化:镜像 ProcessDiagramController.sanitizeBpmnIds(可逆非本工具关注)─
// 安全 id = ASCII-NCName(首字符字母/下划线,其余字母/数字/_/-/.)。非法则映射为
// 稳定唯一占位符(本工具用无冲突计数器,隔离 T1.3 哈希碰撞与 moddle 保真两问题)。
const ID_ATTR = /\b(id|bpmnElement|sourceRef|targetRef|attachedToRef|processRef|default)\s*=\s*"([^"]+)"/g;
function isSafeId(v) {
return /^[A-Za-z_][A-Za-z0-9_.-]*$/.test(v);
}
function sanitizeIds(xml) {
const map = new Map();
let n = 0;
let m;
ID_ATTR.lastIndex = 0;
while ((m = ID_ATTR.exec(xml))) {
const val = m[2];
if (!isSafeId(val) && !map.has(val)) map.set(val, `sanid_${n++}`);
}
if (map.size === 0) return { xml, remapped: 0 };
ID_ATTR.lastIndex = 0;
const rewritten = xml.replace(ID_ATTR, (_, attr, val) => `${attr}="${map.get(val) ?? val}"`);
return { xml: rewritten, remapped: map.size };
}
// ── 命名空间嗅探:activiti 优先,其次 flowable,默认 activiti ─────────────────
// M3 前提:靠字面量 uri/前缀识别命名空间,依赖语料使用规范前缀 activiti/flowable。
// §0 实测本库确如此(activiti 1509 / flowable 3,无自定义前缀重映射),故成立;
// 若未来语料出现 xmlns:act="http://activiti.org/bpmn" 之类别名,此处需扩正则。
function sniffModdle(xml) {
const hasActiviti = xml.includes('activiti.org/bpmn') || /\bactiviti:/.test(xml);
const hasFlowable = xml.includes('flowable.org/bpmn') || /\bflowable:/.test(xml);
if (hasActiviti) return { pkg: { activiti }, ns: 'activiti' };
if (hasFlowable) return { pkg: { flowable }, ns: 'flowable' };
return { pkg: { activiti }, ns: 'activiti(default)' };
}
// ── 扩展面「多重集」签名:数据级零丢失的比对基础 ───────────────────────────
// 逐 tag 扫描(well-formed BPMN 下正则足够),区分扩展元素 tag / 原生元素上的
// 扩展属性;文本单独扫;id 全量。所有值实体解码后入袋(顺序无关)。
const TAG_RE = /<([/!?]?)([\w:.-]+)((?:[^>"']|"[^"]*"|'[^']*')*)>/g;
const ATTR_RE = /([\w:.-]+)\s*=\s*"([^"]*)"/g;
const EXT_TEXT_RE = /<((?:activiti|flowable):[\w.-]+)((?:[^>"']|"[^"]*"|'[^']*')*)>([^<]*)<\//g;
// I1:引用类属性单独入袋,捕捉 round-trip/净化重写破坏的连线(sourceRef/targetRef/
// bpmnElement/processRef/attachedToRef/default)。与 sanitizeIds 的 ID_ATTR 同覆盖面。
const REF_ATTRS = new Set(['sourceRef', 'targetRef', 'bpmnElement', 'processRef', 'attachedToRef', 'default']);
function surface(xml) {
const extAttrs = []; // 原生元素上的 activiti:/flowable: 属性 p:name=value
const extElemTags = []; // 扩展元素本身 tag
const extElemAttrs = [];// 扩展元素上的属性(含无前缀) tag@attr=value
const extElemText = []; // 扩展叶元素的文本 tag#text=value
const ids = []; // 全部 id="..."(含原生/DI/扩展)
const refs = []; // 引用类属性 an=value(连线/DI 指向完整性,I1)
let m;
TAG_RE.lastIndex = 0;
while ((m = TAG_RE.exec(xml))) {
const close = m[1];
const name = m[2];
const attrBlob = m[3] || '';
if (close === '/' || close === '!' || close === '?') continue; // 闭合/注释/声明
const isExt = /^(activiti|flowable):/.test(name);
let a;
ATTR_RE.lastIndex = 0;
while ((a = ATTR_RE.exec(attrBlob))) {
const an = a[1];
const av = decodeEntities(a[2]);
if (an === 'id') ids.push(av);
else if (REF_ATTRS.has(an)) refs.push(`${an}=${av}`);
if (isExt) {
extElemAttrs.push(`${name}@${an}=${av}`);
} else if (/^(activiti|flowable):/.test(an)) {
extAttrs.push(`${an}=${av}`);
}
}
if (isExt) extElemTags.push(name);
}
// M4:extElemText 对叶元素文本做 trim 归一(去两端空白),故不比对纯空白/缩进差异。
// 扩展叶文本(监听器 field 值、表单选项 name 等)非空白敏感,且前后同样 trim,对称无偏。
EXT_TEXT_RE.lastIndex = 0;
while ((m = EXT_TEXT_RE.exec(xml))) {
const t = decodeEntities(m[3]).trim();
if (t) extElemText.push(`${m[1]}#text=${t}`);
}
return { extAttrs, extElemTags, extElemAttrs, extElemText, ids, refs };
}
function bag(arr) {
const b = new Map();
for (const v of arr) b.set(v, (b.get(v) || 0) + 1);
return b;
}
// 多重集差异:返回 {missing:[...], added:[...]}(before→after)
function diffBag(before, after) {
const b = bag(before);
const a = bag(after);
const missing = [];
const added = [];
for (const [k, c] of b) {
const ac = a.get(k) || 0;
if (ac < c) missing.push(`${k} (×${c - ac})`);
}
for (const [k, c] of a) {
const bc = b.get(k) || 0;
if (bc < c) added.push(`${k} (×${c - bc})`);
}
return { missing, added };
}
// ── 主流程 ───────────────────────────────────────────────────────────────
async function main() {
process.stdout.write(`连库拉取语料 (${PG_HOST}:${PG_PORT}/${PG_DB}, schema=${PG_SCHEMA}) ...\n`);
const docs = pullCorpus();
process.stdout.write(`取到 ${docs.length} 个 .bpmn/.xml 文档,开始 round-trip 扫描 ...\n\n`);
let ok = 0;
const failures = [];
const decodeFailures = [];
const parseErrors = [];
const warnCounts = new Map(); // 归一 warning key → 次数
const nsCounts = new Map();
for (const doc of docs) {
// 1) base64 → UTF8,并校验解码无损(非法 UTF8 会 toString 回落 U+FFFD)
const buf = Buffer.from(doc.b64, 'base64');
const xml = buf.toString('utf8');
if (!Buffer.from(xml, 'utf8').equals(buf)) {
decodeFailures.push({ id: doc.id, name: doc.name, bytes: buf.length });
continue;
}
// 2) 命名空间嗅探 → 单载
const { pkg, ns } = sniffModdle(xml);
nsCounts.set(ns, (nsCounts.get(ns) || 0) + 1);
// 3) 净化非法 id(否则中文 id 文档被 moddle 整体拒收)
const { xml: sanitized } = sanitizeIds(xml);
// 4) round-trip
let out;
let warnings = [];
try {
const moddle = new BpmnModdle(pkg);
const parsed = await moddle.fromXML(sanitized);
warnings = parsed.warnings || [];
const exported = await moddle.toXML(parsed.rootElement, { format: false });
out = exported.xml;
} catch (e) {
parseErrors.push({ id: doc.id, name: doc.name, error: String(e && e.message || e) });
continue;
}
const unresolvedRefs = [];
for (const w of warnings) {
const msg = String(w && w.message || w);
const key = normWarn(msg);
warnCounts.set(key, (warnCounts.get(key) || 0) + 1);
if (key === 'unresolved reference') {
const mr = msg.match(/unresolved reference <([^>]+)>/i);
unresolvedRefs.push(mr ? mr[1] : msg.slice(0, 60));
}
}
// 5) 数据级零丢失比对:sanitized(前) vs out(后)
const before = surface(sanitized);
const after = surface(out);
const diffs = {};
let lost = false;
for (const k of ['extAttrs', 'extElemTags', 'extElemAttrs', 'extElemText', 'ids', 'refs']) {
const d = diffBag(before[k], after[k]);
if (d.missing.length || d.added.length) {
diffs[k] = d;
lost = true;
}
}
// I1②:未解析引用计入硬门(净化/往返破坏连线的直接信号,不再只当 warning)
if (lost || unresolvedRefs.length) {
failures.push({ id: doc.id, name: doc.name, ns, diffs, unresolved: unresolvedRefs });
} else {
ok++;
}
}
const expectedTotal = Number(process.env.EXPECTED_TOTAL || 1512);
report({ total: docs.length, ok, failures, decodeFailures, parseErrors, warnCounts, nsCounts, expectedTotal });
// 硬门:真实 BPMN 全部零丢失 + 无解析异常 + 无解码失败 + 语料规模符合预期。
// (M2:解码失败与规模不符均判失败——过滤集内本应 100% 干净 UTF8,且恒 EXPECTED_TOTAL 篇)
const gateOk =
failures.length === 0 &&
parseErrors.length === 0 &&
decodeFailures.length === 0 &&
docs.length === expectedTotal;
process.exit(gateOk ? 0 : 1);
}
// warning 归一:抽取属性/元素名做聚合键
function normWarn(msg) {
const mAttr = msg.match(/unknown (?:type|attribute) <([^>]+)>/i);
if (mAttr) return `unknown <${mAttr[1]}>`;
const mUn = msg.match(/unparsable content <([^>]+)>/i);
if (mUn) return `unparsable <${mUn[1]}>`;
const mRef = msg.match(/unresolved reference <([^>]+)>/i);
if (mRef) return 'unresolved reference';
return msg.replace(/<[^>]*>/g, '<…>').slice(0, 80);
}
function report(r) {
const line = '─'.repeat(60);
process.stdout.write(`${line}\n保真扫描结果\n${line}\n`);
process.stdout.write(`匹配 ${r.ok}/${r.total} 零丢失(真实 BPMN)\n`);
process.stdout.write(`总文档 ${r.total}(预期 ${r.expectedTotal})|零丢失 ${r.ok}|丢失 ${r.failures.length}|解析异常 ${r.parseErrors.length}|解码失败 ${r.decodeFailures.length}\n`);
if (r.total !== r.expectedTotal) {
process.stdout.write(`⚠ 语料规模 ${r.total} ≠ 预期 ${r.expectedTotal}(EXPECTED_TOTAL)→ 判失败\n`);
}
process.stdout.write(`\n命名空间加载分布:\n`);
for (const [ns, c] of [...r.nsCounts].sort((a, b) => b[1] - a[1])) {
process.stdout.write(` ${ns}: ${c}\n`);
}
process.stdout.write(`\nfromXML warnings(按归一键聚合,仅描述符覆盖信号,不判失败):\n`);
const ws = [...r.warnCounts].sort((a, b) => b[1] - a[1]);
if (!ws.length) process.stdout.write(' (无)\n');
for (const [k, c] of ws.slice(0, 40)) process.stdout.write(` ${c.toString().padStart(6)} ${k}\n`);
if (ws.length > 40) process.stdout.write(` … 另有 ${ws.length - 40} 类\n`);
if (r.decodeFailures.length) {
process.stdout.write(`\nbase64→UTF8 解码失败文档(单列,非 UTF8 BPMN):\n`);
for (const d of r.decodeFailures) process.stdout.write(` id=${d.id} name=${d.name} bytes=${d.bytes}\n`);
}
if (r.parseErrors.length) {
process.stdout.write(`\nmoddle 解析/序列化异常:\n`);
for (const e of r.parseErrors.slice(0, 30)) process.stdout.write(` id=${e.id} name=${e.name}\n ${e.error}\n`);
}
if (r.failures.length) {
process.stdout.write(`\n零丢失失败清单(round-trip 前后扩展面差异):\n`);
for (const f of r.failures.slice(0, 30)) {
process.stdout.write(` id=${f.id} name=${f.name} [${f.ns}]\n`);
for (const [k, d] of Object.entries(f.diffs)) {
if (d.missing.length) process.stdout.write(` ${k} 丢失: ${d.missing.slice(0, 6).join(' ; ')}${d.missing.length > 6 ? ' …' : ''}\n`);
if (d.added.length) process.stdout.write(` ${k} 新增: ${d.added.slice(0, 6).join(' ; ')}${d.added.length > 6 ? ' …' : ''}\n`);
}
if (f.unresolved && f.unresolved.length) {
process.stdout.write(` 未解析引用(I1): ${f.unresolved.slice(0, 6).join(' ; ')}${f.unresolved.length > 6 ? ' …' : ''}\n`);
}
}
if (r.failures.length > 30) process.stdout.write(` … 另有 ${r.failures.length - 30} 个失败文档\n`);
}
process.stdout.write(`${line}\n`);
}
main().catch((e) => {
// M5:psql 等子进程失败时把其 stderr 透出来(execFileSync 把 stderr 挂在 e.stderr)
if (e && e.stderr) process.stderr.write(String(e.stderr));
process.stderr.write(`扫描崩溃: ${e && e.stack || e}\n`);
process.exit(2);
});