#!/usr/bin/env node /** * 全量真实流程 moddle round-trip 保真扫描(Task 1.2) * * 对本地 eip4 库(5433, search_path=eip3)里 act_ge_bytearray 的全部真实 BPMN * (name_ like %.bpmn% / %.xml)逐条做 bpmn-moddle fromXML→toXML 往返,验证 * 「所有 activiti / flowable 扩展属性(含值,实体解码归一后) + 扩展元素结构 * + 全部 bpmn 元素 id 集合」在往返前后多重集一致(数据级零丢失,非字节级)。 * * 取数:psql 子进程(不引 pg 依赖);命名空间嗅探后单载 activiti 或 flowable * (T1.1 实证:双载会因跨包同名 extends 冲突丢 userTask)。进 moddle 前先净化 * 中文/非法 id(镜像后端 ProcessDiagramController.sanitizeBpmnIds;moddle-xml * 拒绝非 ASCII-NCName id,1445/1512 文档 process id 为中文)。 * * 用法:node tools/modeler-fidelity-sweep.mjs * 环境变量:PG_HOST(localhost) PG_PORT(5433) PG_DB(eip4) PG_SCHEMA(eip3) */ import { execFileSync } from 'node:child_process'; import { readFileSync } from 'node:fs'; import { fileURLToPath } from 'node:url'; import { dirname, resolve } from 'node:path'; import BpmnModdle from 'bpmn-moddle'; const HERE = dirname(fileURLToPath(import.meta.url)); const activiti = JSON.parse(readFileSync(resolve(HERE, '../src/modeler/moddle/activiti.json'), 'utf8')); const flowable = JSON.parse(readFileSync(resolve(HERE, '../src/modeler/moddle/flowable.json'), 'utf8')); const PG_HOST = process.env.PG_HOST || 'localhost'; const PG_PORT = process.env.PG_PORT || '5433'; const PG_DB = process.env.PG_DB || 'eip4'; const PG_SCHEMA = process.env.PG_SCHEMA || 'eip3'; // ── 取数:psql 拉 id|name|base64(base64 去换行,同勘探法)─────────────────── function pullCorpus() { const sql = `set search_path=${PG_SCHEMA},public; ` + `select b.id_||'|'||b.name_||'|'||translate(encode(b.bytes_,'base64'),E'\\n','') ` + `from act_ge_bytearray b where b.name_ like '%.bpmn%' or b.name_ like '%.xml';`; const out = execFileSync( 'psql', ['-h', PG_HOST, '-p', PG_PORT, '-d', PG_DB, '-tA', '-c', sql], { encoding: 'utf8', maxBuffer: 1024 * 1024 * 1024 }, ); const docs = []; for (const line of out.split('\n')) { if (!line) continue; const i1 = line.indexOf('|'); const i2 = line.indexOf('|', i1 + 1); if (i1 < 0 || i2 < 0) continue; docs.push({ id: line.slice(0, i1), name: line.slice(i1 + 1, i2), b64: line.slice(i2 + 1) }); } return docs; } // ── XML 实体解码归一(数据级比对:"→" "→" 全角/中文原样)──────────── function decodeEntities(s) { return String(s) .replace(/&#x([0-9a-fA-F]+);/g, (_, h) => String.fromCodePoint(parseInt(h, 16))) .replace(/&#(\d+);/g, (_, d) => String.fromCodePoint(parseInt(d, 10))) .replace(/"/g, '"') .replace(/'/g, "'") .replace(/</g, '<') .replace(/>/g, '>') .replace(/&/g, '&'); // amp 最后解,避免 &#34; 被二次解码 } // ── id 净化:镜像 ProcessDiagramController.sanitizeBpmnIds(可逆非本工具关注)─ // 安全 id = ASCII-NCName(首字符字母/下划线,其余字母/数字/_/-/.)。非法则映射为 // 稳定唯一占位符(本工具用无冲突计数器,隔离 T1.3 哈希碰撞与 moddle 保真两问题)。 const ID_ATTR = /\b(id|bpmnElement|sourceRef|targetRef|attachedToRef|processRef|default)\s*=\s*"([^"]+)"/g; function isSafeId(v) { return /^[A-Za-z_][A-Za-z0-9_.-]*$/.test(v); } function sanitizeIds(xml) { const map = new Map(); let n = 0; let m; ID_ATTR.lastIndex = 0; while ((m = ID_ATTR.exec(xml))) { const val = m[2]; if (!isSafeId(val) && !map.has(val)) map.set(val, `sanid_${n++}`); } if (map.size === 0) return { xml, remapped: 0 }; ID_ATTR.lastIndex = 0; const rewritten = xml.replace(ID_ATTR, (_, attr, val) => `${attr}="${map.get(val) ?? val}"`); return { xml: rewritten, remapped: map.size }; } // ── 命名空间嗅探:activiti 优先,其次 flowable,默认 activiti ───────────────── // M3 前提:靠字面量 uri/前缀识别命名空间,依赖语料使用规范前缀 activiti/flowable。 // §0 实测本库确如此(activiti 1509 / flowable 3,无自定义前缀重映射),故成立; // 若未来语料出现 xmlns:act="http://activiti.org/bpmn" 之类别名,此处需扩正则。 function sniffModdle(xml) { const hasActiviti = xml.includes('activiti.org/bpmn') || /\bactiviti:/.test(xml); const hasFlowable = xml.includes('flowable.org/bpmn') || /\bflowable:/.test(xml); if (hasActiviti) return { pkg: { activiti }, ns: 'activiti' }; if (hasFlowable) return { pkg: { flowable }, ns: 'flowable' }; return { pkg: { activiti }, ns: 'activiti(default)' }; } // ── 扩展面「多重集」签名:数据级零丢失的比对基础 ─────────────────────────── // 逐 tag 扫描(well-formed BPMN 下正则足够),区分扩展元素 tag / 原生元素上的 // 扩展属性;文本单独扫;id 全量。所有值实体解码后入袋(顺序无关)。 const TAG_RE = /<([/!?]?)([\w:.-]+)((?:[^>"']|"[^"]*"|'[^']*')*)>/g; const ATTR_RE = /([\w:.-]+)\s*=\s*"([^"]*)"/g; const EXT_TEXT_RE = /<((?:activiti|flowable):[\w.-]+)((?:[^>"']|"[^"]*"|'[^']*')*)>([^<]*)<\//g; // I1:引用类属性单独入袋,捕捉 round-trip/净化重写破坏的连线(sourceRef/targetRef/ // bpmnElement/processRef/attachedToRef/default)。与 sanitizeIds 的 ID_ATTR 同覆盖面。 const REF_ATTRS = new Set(['sourceRef', 'targetRef', 'bpmnElement', 'processRef', 'attachedToRef', 'default']); function surface(xml) { const extAttrs = []; // 原生元素上的 activiti:/flowable: 属性 p:name=value const extElemTags = []; // 扩展元素本身 tag const extElemAttrs = [];// 扩展元素上的属性(含无前缀) tag@attr=value const extElemText = []; // 扩展叶元素的文本 tag#text=value const ids = []; // 全部 id="..."(含原生/DI/扩展) const refs = []; // 引用类属性 an=value(连线/DI 指向完整性,I1) let m; TAG_RE.lastIndex = 0; while ((m = TAG_RE.exec(xml))) { const close = m[1]; const name = m[2]; const attrBlob = m[3] || ''; if (close === '/' || close === '!' || close === '?') continue; // 闭合/注释/声明 const isExt = /^(activiti|flowable):/.test(name); let a; ATTR_RE.lastIndex = 0; while ((a = ATTR_RE.exec(attrBlob))) { const an = a[1]; const av = decodeEntities(a[2]); if (an === 'id') ids.push(av); else if (REF_ATTRS.has(an)) refs.push(`${an}=${av}`); if (isExt) { extElemAttrs.push(`${name}@${an}=${av}`); } else if (/^(activiti|flowable):/.test(an)) { extAttrs.push(`${an}=${av}`); } } if (isExt) extElemTags.push(name); } // M4:extElemText 对叶元素文本做 trim 归一(去两端空白),故不比对纯空白/缩进差异。 // 扩展叶文本(监听器 field 值、表单选项 name 等)非空白敏感,且前后同样 trim,对称无偏。 EXT_TEXT_RE.lastIndex = 0; while ((m = EXT_TEXT_RE.exec(xml))) { const t = decodeEntities(m[3]).trim(); if (t) extElemText.push(`${m[1]}#text=${t}`); } return { extAttrs, extElemTags, extElemAttrs, extElemText, ids, refs }; } function bag(arr) { const b = new Map(); for (const v of arr) b.set(v, (b.get(v) || 0) + 1); return b; } // 多重集差异:返回 {missing:[...], added:[...]}(before→after) function diffBag(before, after) { const b = bag(before); const a = bag(after); const missing = []; const added = []; for (const [k, c] of b) { const ac = a.get(k) || 0; if (ac < c) missing.push(`${k} (×${c - ac})`); } for (const [k, c] of a) { const bc = b.get(k) || 0; if (bc < c) added.push(`${k} (×${c - bc})`); } return { missing, added }; } // ── 主流程 ─────────────────────────────────────────────────────────────── async function main() { process.stdout.write(`连库拉取语料 (${PG_HOST}:${PG_PORT}/${PG_DB}, schema=${PG_SCHEMA}) ...\n`); const docs = pullCorpus(); process.stdout.write(`取到 ${docs.length} 个 .bpmn/.xml 文档,开始 round-trip 扫描 ...\n\n`); let ok = 0; const failures = []; const decodeFailures = []; const parseErrors = []; const warnCounts = new Map(); // 归一 warning key → 次数 const nsCounts = new Map(); for (const doc of docs) { // 1) base64 → UTF8,并校验解码无损(非法 UTF8 会 toString 回落 U+FFFD) const buf = Buffer.from(doc.b64, 'base64'); const xml = buf.toString('utf8'); if (!Buffer.from(xml, 'utf8').equals(buf)) { decodeFailures.push({ id: doc.id, name: doc.name, bytes: buf.length }); continue; } // 2) 命名空间嗅探 → 单载 const { pkg, ns } = sniffModdle(xml); nsCounts.set(ns, (nsCounts.get(ns) || 0) + 1); // 3) 净化非法 id(否则中文 id 文档被 moddle 整体拒收) const { xml: sanitized } = sanitizeIds(xml); // 4) round-trip let out; let warnings = []; try { const moddle = new BpmnModdle(pkg); const parsed = await moddle.fromXML(sanitized); warnings = parsed.warnings || []; const exported = await moddle.toXML(parsed.rootElement, { format: false }); out = exported.xml; } catch (e) { parseErrors.push({ id: doc.id, name: doc.name, error: String(e && e.message || e) }); continue; } const unresolvedRefs = []; for (const w of warnings) { const msg = String(w && w.message || w); const key = normWarn(msg); warnCounts.set(key, (warnCounts.get(key) || 0) + 1); if (key === 'unresolved reference') { const mr = msg.match(/unresolved reference <([^>]+)>/i); unresolvedRefs.push(mr ? mr[1] : msg.slice(0, 60)); } } // 5) 数据级零丢失比对:sanitized(前) vs out(后) const before = surface(sanitized); const after = surface(out); const diffs = {}; let lost = false; for (const k of ['extAttrs', 'extElemTags', 'extElemAttrs', 'extElemText', 'ids', 'refs']) { const d = diffBag(before[k], after[k]); if (d.missing.length || d.added.length) { diffs[k] = d; lost = true; } } // I1②:未解析引用计入硬门(净化/往返破坏连线的直接信号,不再只当 warning) if (lost || unresolvedRefs.length) { failures.push({ id: doc.id, name: doc.name, ns, diffs, unresolved: unresolvedRefs }); } else { ok++; } } const expectedTotal = Number(process.env.EXPECTED_TOTAL || 1512); report({ total: docs.length, ok, failures, decodeFailures, parseErrors, warnCounts, nsCounts, expectedTotal }); // 硬门:真实 BPMN 全部零丢失 + 无解析异常 + 无解码失败 + 语料规模符合预期。 // (M2:解码失败与规模不符均判失败——过滤集内本应 100% 干净 UTF8,且恒 EXPECTED_TOTAL 篇) const gateOk = failures.length === 0 && parseErrors.length === 0 && decodeFailures.length === 0 && docs.length === expectedTotal; process.exit(gateOk ? 0 : 1); } // warning 归一:抽取属性/元素名做聚合键 function normWarn(msg) { const mAttr = msg.match(/unknown (?:type|attribute) <([^>]+)>/i); if (mAttr) return `unknown <${mAttr[1]}>`; const mUn = msg.match(/unparsable content <([^>]+)>/i); if (mUn) return `unparsable <${mUn[1]}>`; const mRef = msg.match(/unresolved reference <([^>]+)>/i); if (mRef) return 'unresolved reference'; return msg.replace(/<[^>]*>/g, '<…>').slice(0, 80); } function report(r) { const line = '─'.repeat(60); process.stdout.write(`${line}\n保真扫描结果\n${line}\n`); process.stdout.write(`匹配 ${r.ok}/${r.total} 零丢失(真实 BPMN)\n`); process.stdout.write(`总文档 ${r.total}(预期 ${r.expectedTotal})|零丢失 ${r.ok}|丢失 ${r.failures.length}|解析异常 ${r.parseErrors.length}|解码失败 ${r.decodeFailures.length}\n`); if (r.total !== r.expectedTotal) { process.stdout.write(`⚠ 语料规模 ${r.total} ≠ 预期 ${r.expectedTotal}(EXPECTED_TOTAL)→ 判失败\n`); } process.stdout.write(`\n命名空间加载分布:\n`); for (const [ns, c] of [...r.nsCounts].sort((a, b) => b[1] - a[1])) { process.stdout.write(` ${ns}: ${c}\n`); } process.stdout.write(`\nfromXML warnings(按归一键聚合,仅描述符覆盖信号,不判失败):\n`); const ws = [...r.warnCounts].sort((a, b) => b[1] - a[1]); if (!ws.length) process.stdout.write(' (无)\n'); for (const [k, c] of ws.slice(0, 40)) process.stdout.write(` ${c.toString().padStart(6)} ${k}\n`); if (ws.length > 40) process.stdout.write(` … 另有 ${ws.length - 40} 类\n`); if (r.decodeFailures.length) { process.stdout.write(`\nbase64→UTF8 解码失败文档(单列,非 UTF8 BPMN):\n`); for (const d of r.decodeFailures) process.stdout.write(` id=${d.id} name=${d.name} bytes=${d.bytes}\n`); } if (r.parseErrors.length) { process.stdout.write(`\nmoddle 解析/序列化异常:\n`); for (const e of r.parseErrors.slice(0, 30)) process.stdout.write(` id=${e.id} name=${e.name}\n ${e.error}\n`); } if (r.failures.length) { process.stdout.write(`\n零丢失失败清单(round-trip 前后扩展面差异):\n`); for (const f of r.failures.slice(0, 30)) { process.stdout.write(` id=${f.id} name=${f.name} [${f.ns}]\n`); for (const [k, d] of Object.entries(f.diffs)) { if (d.missing.length) process.stdout.write(` ${k} 丢失: ${d.missing.slice(0, 6).join(' ; ')}${d.missing.length > 6 ? ' …' : ''}\n`); if (d.added.length) process.stdout.write(` ${k} 新增: ${d.added.slice(0, 6).join(' ; ')}${d.added.length > 6 ? ' …' : ''}\n`); } if (f.unresolved && f.unresolved.length) { process.stdout.write(` 未解析引用(I1): ${f.unresolved.slice(0, 6).join(' ; ')}${f.unresolved.length > 6 ? ' …' : ''}\n`); } } if (r.failures.length > 30) process.stdout.write(` … 另有 ${r.failures.length - 30} 个失败文档\n`); } process.stdout.write(`${line}\n`); } main().catch((e) => { // M5:psql 等子进程失败时把其 stderr 透出来(execFileSync 把 stderr 挂在 e.stderr) if (e && e.stderr) process.stderr.write(String(e.stderr)); process.stderr.write(`扫描崩溃: ${e && e.stack || e}\n`); process.exit(2); });