Files
eip4-web/tools/modeler-fidelity-sweep.mjs
zhengwr 6eb856bb75 feat(modeler): 全量真实流程 moddle round-trip 保真扫描工具(实测 1512/1512 零丢失)
对本地 eip4 库(5433,search_path=eip3) act_ge_bytearray 全部真实 BPMN
(name_ like %.bpmn%/%.xml,共 1512)逐条 bpmn-moddle fromXML→toXML,
数据级(实体解码归一后)校验「activiti/flowable 扩展属性+扩展元素结构
+全部 bpmn 元素 id 集合」多重集往返一致。

- psql 子进程取数(不引 pg 依赖);命名空间嗅探后单载 activiti 或 flowable
  (T1.1 实证双载丢 userTask);进 moddle 前净化非法 id(镜像后端
  sanitizeBpmnIds,1445/1512 文档 process id 为中文,moddle 拒非 ASCII-NCName)。
- 实测: 匹配 1512/1512 零丢失(activiti 1509 / flowable 3);解析异常 0;
  解码失败 0。warnings 仅 4 例 unknown attribute <value>(formProperty
  value= 属性未声明,均 Activiti 自带 demo 流程,lax $attrs 保留,零丢失)。
- 解码守卫: 校验 base64→UTF8 无损,单列非 UTF8 文档。全表探明解码失败仅
  出现在非 BPMN 二进制资源(.png 1504 / var-* 87524),被 name 过滤器正确排除;
  真实 BPMN 语料 0 解码失败。

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
2026-07-02 09:59:54 +08:00

293 lines
12 KiB
JavaScript
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
#!/usr/bin/env node
/**
* 全量真实流程 moddle round-trip 保真扫描(Task 1.2)
*
* 对本地 eip4 库(5433, search_path=eip3)里 act_ge_bytearray 的全部真实 BPMN
* (name_ like %.bpmn% / %.xml)逐条做 bpmn-moddle fromXML→toXML 往返,验证
* 「所有 activiti / flowable 扩展属性(含值,实体解码归一后) + 扩展元素结构
* + 全部 bpmn 元素 id 集合」在往返前后多重集一致(数据级零丢失,非字节级)。
*
* 取数:psql 子进程(不引 pg 依赖);命名空间嗅探后单载 activiti 或 flowable
* (T1.1 实证:双载会因跨包同名 extends 冲突丢 userTask)。进 moddle 前先净化
* 中文/非法 id(镜像后端 ProcessDiagramController.sanitizeBpmnIds;moddle-xml
* 拒绝非 ASCII-NCName id,1445/1512 文档 process id 为中文)。
*
* 用法:node tools/modeler-fidelity-sweep.mjs
* 环境变量:PG_HOST(localhost) PG_PORT(5433) PG_DB(eip4) PG_SCHEMA(eip3)
*/
import { execFileSync } from 'node:child_process';
import { readFileSync } from 'node:fs';
import { fileURLToPath } from 'node:url';
import { dirname, resolve } from 'node:path';
import BpmnModdle from 'bpmn-moddle';
const HERE = dirname(fileURLToPath(import.meta.url));
const activiti = JSON.parse(readFileSync(resolve(HERE, '../src/modeler/moddle/activiti.json'), 'utf8'));
const flowable = JSON.parse(readFileSync(resolve(HERE, '../src/modeler/moddle/flowable.json'), 'utf8'));
const PG_HOST = process.env.PG_HOST || 'localhost';
const PG_PORT = process.env.PG_PORT || '5433';
const PG_DB = process.env.PG_DB || 'eip4';
const PG_SCHEMA = process.env.PG_SCHEMA || 'eip3';
// ── 取数:psql 拉 id|name|base64(base64 去换行,同勘探法)───────────────────
function pullCorpus() {
const sql =
`set search_path=${PG_SCHEMA},public; ` +
`select b.id_||'|'||b.name_||'|'||translate(encode(b.bytes_,'base64'),E'\\n','') ` +
`from act_ge_bytearray b where b.name_ like '%.bpmn%' or b.name_ like '%.xml';`;
const out = execFileSync(
'psql',
['-h', PG_HOST, '-p', PG_PORT, '-d', PG_DB, '-tA', '-c', sql],
{ encoding: 'utf8', maxBuffer: 1024 * 1024 * 1024 },
);
const docs = [];
for (const line of out.split('\n')) {
if (!line) continue;
const i1 = line.indexOf('|');
const i2 = line.indexOf('|', i1 + 1);
if (i1 < 0 || i2 < 0) continue;
docs.push({ id: line.slice(0, i1), name: line.slice(i1 + 1, i2), b64: line.slice(i2 + 1) });
}
return docs;
}
// ── XML 实体解码归一(数据级比对:&quot;→" &#34;→" 全角/中文原样)────────────
function decodeEntities(s) {
return String(s)
.replace(/&#x([0-9a-fA-F]+);/g, (_, h) => String.fromCodePoint(parseInt(h, 16)))
.replace(/&#(\d+);/g, (_, d) => String.fromCodePoint(parseInt(d, 10)))
.replace(/&quot;/g, '"')
.replace(/&apos;/g, "'")
.replace(/&lt;/g, '<')
.replace(/&gt;/g, '>')
.replace(/&amp;/g, '&'); // amp 最后解,避免 &amp;#34; 被二次解码
}
// ── id 净化:镜像 ProcessDiagramController.sanitizeBpmnIds(可逆非本工具关注)─
// 安全 id = ASCII-NCName(首字符字母/下划线,其余字母/数字/_/-/.)。非法则映射为
// 稳定唯一占位符(本工具用无冲突计数器,隔离 T1.3 哈希碰撞与 moddle 保真两问题)。
const ID_ATTR = /\b(id|bpmnElement|sourceRef|targetRef|attachedToRef|processRef|default)\s*=\s*"([^"]+)"/g;
function isSafeId(v) {
return /^[A-Za-z_][A-Za-z0-9_.-]*$/.test(v);
}
function sanitizeIds(xml) {
const map = new Map();
let n = 0;
let m;
ID_ATTR.lastIndex = 0;
while ((m = ID_ATTR.exec(xml))) {
const val = m[2];
if (!isSafeId(val) && !map.has(val)) map.set(val, `sanid_${n++}`);
}
if (map.size === 0) return { xml, remapped: 0 };
ID_ATTR.lastIndex = 0;
const rewritten = xml.replace(ID_ATTR, (_, attr, val) => `${attr}="${map.get(val) ?? val}"`);
return { xml: rewritten, remapped: map.size };
}
// ── 命名空间嗅探:activiti 优先,其次 flowable,默认 activiti ─────────────────
function sniffModdle(xml) {
const hasActiviti = xml.includes('activiti.org/bpmn') || /\bactiviti:/.test(xml);
const hasFlowable = xml.includes('flowable.org/bpmn') || /\bflowable:/.test(xml);
if (hasActiviti) return { pkg: { activiti }, ns: 'activiti' };
if (hasFlowable) return { pkg: { flowable }, ns: 'flowable' };
return { pkg: { activiti }, ns: 'activiti(default)' };
}
// ── 扩展面「多重集」签名:数据级零丢失的比对基础 ───────────────────────────
// 逐 tag 扫描(well-formed BPMN 下正则足够),区分扩展元素 tag / 原生元素上的
// 扩展属性;文本单独扫;id 全量。所有值实体解码后入袋(顺序无关)。
const TAG_RE = /<([/!?]?)([\w:.-]+)((?:[^>"']|"[^"]*"|'[^']*')*)>/g;
const ATTR_RE = /([\w:.-]+)\s*=\s*"([^"]*)"/g;
const EXT_TEXT_RE = /<((?:activiti|flowable):[\w.-]+)((?:[^>"']|"[^"]*"|'[^']*')*)>([^<]*)<\//g;
function surface(xml) {
const extAttrs = []; // 原生元素上的 activiti:/flowable: 属性 p:name=value
const extElemTags = []; // 扩展元素本身 tag
const extElemAttrs = [];// 扩展元素上的属性(含无前缀) tag@attr=value
const extElemText = []; // 扩展叶元素的文本 tag#text=value
const ids = []; // 全部 id="..."(含原生/DI/扩展)
let m;
TAG_RE.lastIndex = 0;
while ((m = TAG_RE.exec(xml))) {
const close = m[1];
const name = m[2];
const attrBlob = m[3] || '';
if (close === '/' || close === '!' || close === '?') continue; // 闭合/注释/声明
const isExt = /^(activiti|flowable):/.test(name);
let a;
ATTR_RE.lastIndex = 0;
while ((a = ATTR_RE.exec(attrBlob))) {
const an = a[1];
const av = decodeEntities(a[2]);
if (an === 'id') ids.push(av);
if (isExt) {
extElemAttrs.push(`${name}@${an}=${av}`);
} else if (/^(activiti|flowable):/.test(an)) {
extAttrs.push(`${an}=${av}`);
}
}
if (isExt) extElemTags.push(name);
}
EXT_TEXT_RE.lastIndex = 0;
while ((m = EXT_TEXT_RE.exec(xml))) {
const t = decodeEntities(m[3]).trim();
if (t) extElemText.push(`${m[1]}#text=${t}`);
}
return { extAttrs, extElemTags, extElemAttrs, extElemText, ids };
}
function bag(arr) {
const b = new Map();
for (const v of arr) b.set(v, (b.get(v) || 0) + 1);
return b;
}
// 多重集差异:返回 {missing:[...], added:[...]}(before→after)
function diffBag(before, after) {
const b = bag(before);
const a = bag(after);
const missing = [];
const added = [];
for (const [k, c] of b) {
const ac = a.get(k) || 0;
if (ac < c) missing.push(`${k} (×${c - ac})`);
}
for (const [k, c] of a) {
const bc = b.get(k) || 0;
if (bc < c) added.push(`${k} (×${c - bc})`);
}
return { missing, added };
}
// ── 主流程 ───────────────────────────────────────────────────────────────
async function main() {
process.stdout.write(`连库拉取语料 (${PG_HOST}:${PG_PORT}/${PG_DB}, schema=${PG_SCHEMA}) ...\n`);
const docs = pullCorpus();
process.stdout.write(`取到 ${docs.length} 个 .bpmn/.xml 文档,开始 round-trip 扫描 ...\n\n`);
let ok = 0;
const failures = [];
const decodeFailures = [];
const parseErrors = [];
const warnCounts = new Map(); // 归一 warning key → 次数
const nsCounts = new Map();
for (const doc of docs) {
// 1) base64 → UTF8,并校验解码无损(非法 UTF8 会 toString 回落 U+FFFD)
const buf = Buffer.from(doc.b64, 'base64');
const xml = buf.toString('utf8');
if (!Buffer.from(xml, 'utf8').equals(buf)) {
decodeFailures.push({ id: doc.id, name: doc.name, bytes: buf.length });
continue;
}
// 2) 命名空间嗅探 → 单载
const { pkg, ns } = sniffModdle(xml);
nsCounts.set(ns, (nsCounts.get(ns) || 0) + 1);
// 3) 净化非法 id(否则中文 id 文档被 moddle 整体拒收)
const { xml: sanitized } = sanitizeIds(xml);
// 4) round-trip
let out;
let warnings = [];
try {
const moddle = new BpmnModdle(pkg);
const parsed = await moddle.fromXML(sanitized);
warnings = parsed.warnings || [];
const exported = await moddle.toXML(parsed.rootElement, { format: false });
out = exported.xml;
} catch (e) {
parseErrors.push({ id: doc.id, name: doc.name, error: String(e && e.message || e) });
continue;
}
for (const w of warnings) {
const key = normWarn(String(w && w.message || w));
warnCounts.set(key, (warnCounts.get(key) || 0) + 1);
}
// 5) 数据级零丢失比对:sanitized(前) vs out(后)
const before = surface(sanitized);
const after = surface(out);
const diffs = {};
let lost = false;
for (const k of ['extAttrs', 'extElemTags', 'extElemAttrs', 'extElemText', 'ids']) {
const d = diffBag(before[k], after[k]);
if (d.missing.length || d.added.length) {
diffs[k] = d;
lost = true;
}
}
if (lost) failures.push({ id: doc.id, name: doc.name, ns, diffs });
else ok++;
}
report({ total: docs.length, ok, failures, decodeFailures, parseErrors, warnCounts, nsCounts });
// 硬门:真实 BPMN 全部零丢失(解码失败=非目标语料,单列不计入失败门)
process.exit(failures.length === 0 && parseErrors.length === 0 ? 0 : 1);
}
// warning 归一:抽取属性/元素名做聚合键
function normWarn(msg) {
const mAttr = msg.match(/unknown (?:type|attribute) <([^>]+)>/i);
if (mAttr) return `unknown <${mAttr[1]}>`;
const mUn = msg.match(/unparsable content <([^>]+)>/i);
if (mUn) return `unparsable <${mUn[1]}>`;
const mRef = msg.match(/unresolved reference <([^>]+)>/i);
if (mRef) return 'unresolved reference';
return msg.replace(/<[^>]*>/g, '<…>').slice(0, 80);
}
function report(r) {
const line = '─'.repeat(60);
process.stdout.write(`${line}\n保真扫描结果\n${line}\n`);
const realFail = r.failures.length + r.parseErrors.length;
const realTotal = r.total - r.decodeFailures.length;
process.stdout.write(`匹配 ${r.ok}/${realTotal}(真实 BPMN,已排除 ${r.decodeFailures.length} 解码失败)\n`);
process.stdout.write(`总文档 ${r.total}|零丢失 ${r.ok}|丢失 ${r.failures.length}|解析异常 ${r.parseErrors.length}|解码失败 ${r.decodeFailures.length}\n`);
process.stdout.write(`\n命名空间加载分布:\n`);
for (const [ns, c] of [...r.nsCounts].sort((a, b) => b[1] - a[1])) {
process.stdout.write(` ${ns}: ${c}\n`);
}
process.stdout.write(`\nfromXML warnings(按归一键聚合,仅描述符覆盖信号,不判失败):\n`);
const ws = [...r.warnCounts].sort((a, b) => b[1] - a[1]);
if (!ws.length) process.stdout.write(' (无)\n');
for (const [k, c] of ws.slice(0, 40)) process.stdout.write(` ${c.toString().padStart(6)} ${k}\n`);
if (ws.length > 40) process.stdout.write(` … 另有 ${ws.length - 40} 类\n`);
if (r.decodeFailures.length) {
process.stdout.write(`\nbase64→UTF8 解码失败文档(单列,非 UTF8 BPMN):\n`);
for (const d of r.decodeFailures) process.stdout.write(` id=${d.id} name=${d.name} bytes=${d.bytes}\n`);
}
if (r.parseErrors.length) {
process.stdout.write(`\nmoddle 解析/序列化异常:\n`);
for (const e of r.parseErrors.slice(0, 30)) process.stdout.write(` id=${e.id} name=${e.name}\n ${e.error}\n`);
}
if (r.failures.length) {
process.stdout.write(`\n零丢失失败清单(round-trip 前后扩展面差异):\n`);
for (const f of r.failures.slice(0, 30)) {
process.stdout.write(` id=${f.id} name=${f.name} [${f.ns}]\n`);
for (const [k, d] of Object.entries(f.diffs)) {
if (d.missing.length) process.stdout.write(` ${k} 丢失: ${d.missing.slice(0, 6).join(' ; ')}${d.missing.length > 6 ? ' …' : ''}\n`);
if (d.added.length) process.stdout.write(` ${k} 新增: ${d.added.slice(0, 6).join(' ; ')}${d.added.length > 6 ? ' …' : ''}\n`);
}
}
if (r.failures.length > 30) process.stdout.write(` … 另有 ${r.failures.length - 30} 个失败文档\n`);
}
process.stdout.write(`${line}\n`);
}
main().catch((e) => {
process.stderr.write(`扫描崩溃: ${e && e.stack || e}\n`);
process.exit(2);
});