From 6eb856bb758bb23bf0dcb7a4739738cb599faec4 Mon Sep 17 00:00:00 2001 From: zhengwr Date: Thu, 2 Jul 2026 09:59:54 +0800 Subject: [PATCH] =?UTF-8?q?feat(modeler):=20=E5=85=A8=E9=87=8F=E7=9C=9F?= =?UTF-8?q?=E5=AE=9E=E6=B5=81=E7=A8=8B=20moddle=20round-trip=20=E4=BF=9D?= =?UTF-8?q?=E7=9C=9F=E6=89=AB=E6=8F=8F=E5=B7=A5=E5=85=B7(=E5=AE=9E?= =?UTF-8?q?=E6=B5=8B=201512/1512=20=E9=9B=B6=E4=B8=A2=E5=A4=B1)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 对本地 eip4 库(5433,search_path=eip3) act_ge_bytearray 全部真实 BPMN (name_ like %.bpmn%/%.xml,共 1512)逐条 bpmn-moddle fromXML→toXML, 数据级(实体解码归一后)校验「activiti/flowable 扩展属性+扩展元素结构 +全部 bpmn 元素 id 集合」多重集往返一致。 - psql 子进程取数(不引 pg 依赖);命名空间嗅探后单载 activiti 或 flowable (T1.1 实证双载丢 userTask);进 moddle 前净化非法 id(镜像后端 sanitizeBpmnIds,1445/1512 文档 process id 为中文,moddle 拒非 ASCII-NCName)。 - 实测: 匹配 1512/1512 零丢失(activiti 1509 / flowable 3);解析异常 0; 解码失败 0。warnings 仅 4 例 unknown attribute (formProperty value= 属性未声明,均 Activiti 自带 demo 流程,lax $attrs 保留,零丢失)。 - 解码守卫: 校验 base64→UTF8 无损,单列非 UTF8 文档。全表探明解码失败仅 出现在非 BPMN 二进制资源(.png 1504 / var-* 87524),被 name 过滤器正确排除; 真实 BPMN 语料 0 解码失败。 Co-Authored-By: Claude Fable 5 --- tools/modeler-fidelity-sweep.mjs | 292 +++++++++++++++++++++++++++++++ 1 file changed, 292 insertions(+) create mode 100644 tools/modeler-fidelity-sweep.mjs diff --git a/tools/modeler-fidelity-sweep.mjs b/tools/modeler-fidelity-sweep.mjs new file mode 100644 index 0000000..6d7853d --- /dev/null +++ b/tools/modeler-fidelity-sweep.mjs @@ -0,0 +1,292 @@ +#!/usr/bin/env node +/** + * 全量真实流程 moddle round-trip 保真扫描(Task 1.2) + * + * 对本地 eip4 库(5433, search_path=eip3)里 act_ge_bytearray 的全部真实 BPMN + * (name_ like %.bpmn% / %.xml)逐条做 bpmn-moddle fromXML→toXML 往返,验证 + * 「所有 activiti / flowable 扩展属性(含值,实体解码归一后) + 扩展元素结构 + * + 全部 bpmn 元素 id 集合」在往返前后多重集一致(数据级零丢失,非字节级)。 + * + * 取数:psql 子进程(不引 pg 依赖);命名空间嗅探后单载 activiti 或 flowable + * (T1.1 实证:双载会因跨包同名 extends 冲突丢 userTask)。进 moddle 前先净化 + * 中文/非法 id(镜像后端 ProcessDiagramController.sanitizeBpmnIds;moddle-xml + * 拒绝非 ASCII-NCName id,1445/1512 文档 process id 为中文)。 + * + * 用法:node tools/modeler-fidelity-sweep.mjs + * 环境变量:PG_HOST(localhost) PG_PORT(5433) PG_DB(eip4) PG_SCHEMA(eip3) + */ +import { execFileSync } from 'node:child_process'; +import { readFileSync } from 'node:fs'; +import { fileURLToPath } from 'node:url'; +import { dirname, resolve } from 'node:path'; +import BpmnModdle from 'bpmn-moddle'; + +const HERE = dirname(fileURLToPath(import.meta.url)); +const activiti = JSON.parse(readFileSync(resolve(HERE, '../src/modeler/moddle/activiti.json'), 'utf8')); +const flowable = JSON.parse(readFileSync(resolve(HERE, '../src/modeler/moddle/flowable.json'), 'utf8')); + +const PG_HOST = process.env.PG_HOST || 'localhost'; +const PG_PORT = process.env.PG_PORT || '5433'; +const PG_DB = process.env.PG_DB || 'eip4'; +const PG_SCHEMA = process.env.PG_SCHEMA || 'eip3'; + +// ── 取数:psql 拉 id|name|base64(base64 去换行,同勘探法)─────────────────── +function pullCorpus() { + const sql = + `set search_path=${PG_SCHEMA},public; ` + + `select b.id_||'|'||b.name_||'|'||translate(encode(b.bytes_,'base64'),E'\\n','') ` + + `from act_ge_bytearray b where b.name_ like '%.bpmn%' or b.name_ like '%.xml';`; + const out = execFileSync( + 'psql', + ['-h', PG_HOST, '-p', PG_PORT, '-d', PG_DB, '-tA', '-c', sql], + { encoding: 'utf8', maxBuffer: 1024 * 1024 * 1024 }, + ); + const docs = []; + for (const line of out.split('\n')) { + if (!line) continue; + const i1 = line.indexOf('|'); + const i2 = line.indexOf('|', i1 + 1); + if (i1 < 0 || i2 < 0) continue; + docs.push({ id: line.slice(0, i1), name: line.slice(i1 + 1, i2), b64: line.slice(i2 + 1) }); + } + return docs; +} + +// ── XML 实体解码归一(数据级比对:"→" "→" 全角/中文原样)──────────── +function decodeEntities(s) { + return String(s) + .replace(/&#x([0-9a-fA-F]+);/g, (_, h) => String.fromCodePoint(parseInt(h, 16))) + .replace(/&#(\d+);/g, (_, d) => String.fromCodePoint(parseInt(d, 10))) + .replace(/"/g, '"') + .replace(/'/g, "'") + .replace(/</g, '<') + .replace(/>/g, '>') + .replace(/&/g, '&'); // amp 最后解,避免 &#34; 被二次解码 +} + +// ── id 净化:镜像 ProcessDiagramController.sanitizeBpmnIds(可逆非本工具关注)─ +// 安全 id = ASCII-NCName(首字符字母/下划线,其余字母/数字/_/-/.)。非法则映射为 +// 稳定唯一占位符(本工具用无冲突计数器,隔离 T1.3 哈希碰撞与 moddle 保真两问题)。 +const ID_ATTR = /\b(id|bpmnElement|sourceRef|targetRef|attachedToRef|processRef|default)\s*=\s*"([^"]+)"/g; +function isSafeId(v) { + return /^[A-Za-z_][A-Za-z0-9_.-]*$/.test(v); +} +function sanitizeIds(xml) { + const map = new Map(); + let n = 0; + let m; + ID_ATTR.lastIndex = 0; + while ((m = ID_ATTR.exec(xml))) { + const val = m[2]; + if (!isSafeId(val) && !map.has(val)) map.set(val, `sanid_${n++}`); + } + if (map.size === 0) return { xml, remapped: 0 }; + ID_ATTR.lastIndex = 0; + const rewritten = xml.replace(ID_ATTR, (_, attr, val) => `${attr}="${map.get(val) ?? val}"`); + return { xml: rewritten, remapped: map.size }; +} + +// ── 命名空间嗅探:activiti 优先,其次 flowable,默认 activiti ───────────────── +function sniffModdle(xml) { + const hasActiviti = xml.includes('activiti.org/bpmn') || /\bactiviti:/.test(xml); + const hasFlowable = xml.includes('flowable.org/bpmn') || /\bflowable:/.test(xml); + if (hasActiviti) return { pkg: { activiti }, ns: 'activiti' }; + if (hasFlowable) return { pkg: { flowable }, ns: 'flowable' }; + return { pkg: { activiti }, ns: 'activiti(default)' }; +} + +// ── 扩展面「多重集」签名:数据级零丢失的比对基础 ─────────────────────────── +// 逐 tag 扫描(well-formed BPMN 下正则足够),区分扩展元素 tag / 原生元素上的 +// 扩展属性;文本单独扫;id 全量。所有值实体解码后入袋(顺序无关)。 +const TAG_RE = /<([/!?]?)([\w:.-]+)((?:[^>"']|"[^"]*"|'[^']*')*)>/g; +const ATTR_RE = /([\w:.-]+)\s*=\s*"([^"]*)"/g; +const EXT_TEXT_RE = /<((?:activiti|flowable):[\w.-]+)((?:[^>"']|"[^"]*"|'[^']*')*)>([^<]*)<\//g; + +function surface(xml) { + const extAttrs = []; // 原生元素上的 activiti:/flowable: 属性 p:name=value + const extElemTags = []; // 扩展元素本身 tag + const extElemAttrs = [];// 扩展元素上的属性(含无前缀) tag@attr=value + const extElemText = []; // 扩展叶元素的文本 tag#text=value + const ids = []; // 全部 id="..."(含原生/DI/扩展) + + let m; + TAG_RE.lastIndex = 0; + while ((m = TAG_RE.exec(xml))) { + const close = m[1]; + const name = m[2]; + const attrBlob = m[3] || ''; + if (close === '/' || close === '!' || close === '?') continue; // 闭合/注释/声明 + const isExt = /^(activiti|flowable):/.test(name); + let a; + ATTR_RE.lastIndex = 0; + while ((a = ATTR_RE.exec(attrBlob))) { + const an = a[1]; + const av = decodeEntities(a[2]); + if (an === 'id') ids.push(av); + if (isExt) { + extElemAttrs.push(`${name}@${an}=${av}`); + } else if (/^(activiti|flowable):/.test(an)) { + extAttrs.push(`${an}=${av}`); + } + } + if (isExt) extElemTags.push(name); + } + + EXT_TEXT_RE.lastIndex = 0; + while ((m = EXT_TEXT_RE.exec(xml))) { + const t = decodeEntities(m[3]).trim(); + if (t) extElemText.push(`${m[1]}#text=${t}`); + } + + return { extAttrs, extElemTags, extElemAttrs, extElemText, ids }; +} + +function bag(arr) { + const b = new Map(); + for (const v of arr) b.set(v, (b.get(v) || 0) + 1); + return b; +} +// 多重集差异:返回 {missing:[...], added:[...]}(before→after) +function diffBag(before, after) { + const b = bag(before); + const a = bag(after); + const missing = []; + const added = []; + for (const [k, c] of b) { + const ac = a.get(k) || 0; + if (ac < c) missing.push(`${k} (×${c - ac})`); + } + for (const [k, c] of a) { + const bc = b.get(k) || 0; + if (bc < c) added.push(`${k} (×${c - bc})`); + } + return { missing, added }; +} + +// ── 主流程 ─────────────────────────────────────────────────────────────── +async function main() { + process.stdout.write(`连库拉取语料 (${PG_HOST}:${PG_PORT}/${PG_DB}, schema=${PG_SCHEMA}) ...\n`); + const docs = pullCorpus(); + process.stdout.write(`取到 ${docs.length} 个 .bpmn/.xml 文档,开始 round-trip 扫描 ...\n\n`); + + let ok = 0; + const failures = []; + const decodeFailures = []; + const parseErrors = []; + const warnCounts = new Map(); // 归一 warning key → 次数 + const nsCounts = new Map(); + + for (const doc of docs) { + // 1) base64 → UTF8,并校验解码无损(非法 UTF8 会 toString 回落 U+FFFD) + const buf = Buffer.from(doc.b64, 'base64'); + const xml = buf.toString('utf8'); + if (!Buffer.from(xml, 'utf8').equals(buf)) { + decodeFailures.push({ id: doc.id, name: doc.name, bytes: buf.length }); + continue; + } + + // 2) 命名空间嗅探 → 单载 + const { pkg, ns } = sniffModdle(xml); + nsCounts.set(ns, (nsCounts.get(ns) || 0) + 1); + + // 3) 净化非法 id(否则中文 id 文档被 moddle 整体拒收) + const { xml: sanitized } = sanitizeIds(xml); + + // 4) round-trip + let out; + let warnings = []; + try { + const moddle = new BpmnModdle(pkg); + const parsed = await moddle.fromXML(sanitized); + warnings = parsed.warnings || []; + const exported = await moddle.toXML(parsed.rootElement, { format: false }); + out = exported.xml; + } catch (e) { + parseErrors.push({ id: doc.id, name: doc.name, error: String(e && e.message || e) }); + continue; + } + + for (const w of warnings) { + const key = normWarn(String(w && w.message || w)); + warnCounts.set(key, (warnCounts.get(key) || 0) + 1); + } + + // 5) 数据级零丢失比对:sanitized(前) vs out(后) + const before = surface(sanitized); + const after = surface(out); + const diffs = {}; + let lost = false; + for (const k of ['extAttrs', 'extElemTags', 'extElemAttrs', 'extElemText', 'ids']) { + const d = diffBag(before[k], after[k]); + if (d.missing.length || d.added.length) { + diffs[k] = d; + lost = true; + } + } + if (lost) failures.push({ id: doc.id, name: doc.name, ns, diffs }); + else ok++; + } + + report({ total: docs.length, ok, failures, decodeFailures, parseErrors, warnCounts, nsCounts }); + // 硬门:真实 BPMN 全部零丢失(解码失败=非目标语料,单列不计入失败门) + process.exit(failures.length === 0 && parseErrors.length === 0 ? 0 : 1); +} + +// warning 归一:抽取属性/元素名做聚合键 +function normWarn(msg) { + const mAttr = msg.match(/unknown (?:type|attribute) <([^>]+)>/i); + if (mAttr) return `unknown <${mAttr[1]}>`; + const mUn = msg.match(/unparsable content <([^>]+)>/i); + if (mUn) return `unparsable <${mUn[1]}>`; + const mRef = msg.match(/unresolved reference <([^>]+)>/i); + if (mRef) return 'unresolved reference'; + return msg.replace(/<[^>]*>/g, '<…>').slice(0, 80); +} + +function report(r) { + const line = '─'.repeat(60); + process.stdout.write(`${line}\n保真扫描结果\n${line}\n`); + const realFail = r.failures.length + r.parseErrors.length; + const realTotal = r.total - r.decodeFailures.length; + process.stdout.write(`匹配 ${r.ok}/${realTotal}(真实 BPMN,已排除 ${r.decodeFailures.length} 解码失败)\n`); + process.stdout.write(`总文档 ${r.total}|零丢失 ${r.ok}|丢失 ${r.failures.length}|解析异常 ${r.parseErrors.length}|解码失败 ${r.decodeFailures.length}\n`); + + process.stdout.write(`\n命名空间加载分布:\n`); + for (const [ns, c] of [...r.nsCounts].sort((a, b) => b[1] - a[1])) { + process.stdout.write(` ${ns}: ${c}\n`); + } + + process.stdout.write(`\nfromXML warnings(按归一键聚合,仅描述符覆盖信号,不判失败):\n`); + const ws = [...r.warnCounts].sort((a, b) => b[1] - a[1]); + if (!ws.length) process.stdout.write(' (无)\n'); + for (const [k, c] of ws.slice(0, 40)) process.stdout.write(` ${c.toString().padStart(6)} ${k}\n`); + if (ws.length > 40) process.stdout.write(` … 另有 ${ws.length - 40} 类\n`); + + if (r.decodeFailures.length) { + process.stdout.write(`\nbase64→UTF8 解码失败文档(单列,非 UTF8 BPMN):\n`); + for (const d of r.decodeFailures) process.stdout.write(` id=${d.id} name=${d.name} bytes=${d.bytes}\n`); + } + + if (r.parseErrors.length) { + process.stdout.write(`\nmoddle 解析/序列化异常:\n`); + for (const e of r.parseErrors.slice(0, 30)) process.stdout.write(` id=${e.id} name=${e.name}\n ${e.error}\n`); + } + + if (r.failures.length) { + process.stdout.write(`\n零丢失失败清单(round-trip 前后扩展面差异):\n`); + for (const f of r.failures.slice(0, 30)) { + process.stdout.write(` id=${f.id} name=${f.name} [${f.ns}]\n`); + for (const [k, d] of Object.entries(f.diffs)) { + if (d.missing.length) process.stdout.write(` ${k} 丢失: ${d.missing.slice(0, 6).join(' ; ')}${d.missing.length > 6 ? ' …' : ''}\n`); + if (d.added.length) process.stdout.write(` ${k} 新增: ${d.added.slice(0, 6).join(' ; ')}${d.added.length > 6 ? ' …' : ''}\n`); + } + } + if (r.failures.length > 30) process.stdout.write(` … 另有 ${r.failures.length - 30} 个失败文档\n`); + } + process.stdout.write(`${line}\n`); +} + +main().catch((e) => { + process.stderr.write(`扫描崩溃: ${e && e.stack || e}\n`); + process.exit(2); +});