feat(modeler): 全量真实流程 moddle round-trip 保真扫描工具(实测 1512/1512 零丢失)
对本地 eip4 库(5433,search_path=eip3) act_ge_bytearray 全部真实 BPMN (name_ like %.bpmn%/%.xml,共 1512)逐条 bpmn-moddle fromXML→toXML, 数据级(实体解码归一后)校验「activiti/flowable 扩展属性+扩展元素结构 +全部 bpmn 元素 id 集合」多重集往返一致。 - psql 子进程取数(不引 pg 依赖);命名空间嗅探后单载 activiti 或 flowable (T1.1 实证双载丢 userTask);进 moddle 前净化非法 id(镜像后端 sanitizeBpmnIds,1445/1512 文档 process id 为中文,moddle 拒非 ASCII-NCName)。 - 实测: 匹配 1512/1512 零丢失(activiti 1509 / flowable 3);解析异常 0; 解码失败 0。warnings 仅 4 例 unknown attribute <value>(formProperty value= 属性未声明,均 Activiti 自带 demo 流程,lax $attrs 保留,零丢失)。 - 解码守卫: 校验 base64→UTF8 无损,单列非 UTF8 文档。全表探明解码失败仅 出现在非 BPMN 二进制资源(.png 1504 / var-* 87524),被 name 过滤器正确排除; 真实 BPMN 语料 0 解码失败。 Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
292
tools/modeler-fidelity-sweep.mjs
Normal file
292
tools/modeler-fidelity-sweep.mjs
Normal file
@@ -0,0 +1,292 @@
|
|||||||
|
#!/usr/bin/env node
|
||||||
|
/**
|
||||||
|
* 全量真实流程 moddle round-trip 保真扫描(Task 1.2)
|
||||||
|
*
|
||||||
|
* 对本地 eip4 库(5433, search_path=eip3)里 act_ge_bytearray 的全部真实 BPMN
|
||||||
|
* (name_ like %.bpmn% / %.xml)逐条做 bpmn-moddle fromXML→toXML 往返,验证
|
||||||
|
* 「所有 activiti / flowable 扩展属性(含值,实体解码归一后) + 扩展元素结构
|
||||||
|
* + 全部 bpmn 元素 id 集合」在往返前后多重集一致(数据级零丢失,非字节级)。
|
||||||
|
*
|
||||||
|
* 取数:psql 子进程(不引 pg 依赖);命名空间嗅探后单载 activiti 或 flowable
|
||||||
|
* (T1.1 实证:双载会因跨包同名 extends 冲突丢 userTask)。进 moddle 前先净化
|
||||||
|
* 中文/非法 id(镜像后端 ProcessDiagramController.sanitizeBpmnIds;moddle-xml
|
||||||
|
* 拒绝非 ASCII-NCName id,1445/1512 文档 process id 为中文)。
|
||||||
|
*
|
||||||
|
* 用法:node tools/modeler-fidelity-sweep.mjs
|
||||||
|
* 环境变量:PG_HOST(localhost) PG_PORT(5433) PG_DB(eip4) PG_SCHEMA(eip3)
|
||||||
|
*/
|
||||||
|
import { execFileSync } from 'node:child_process';
|
||||||
|
import { readFileSync } from 'node:fs';
|
||||||
|
import { fileURLToPath } from 'node:url';
|
||||||
|
import { dirname, resolve } from 'node:path';
|
||||||
|
import BpmnModdle from 'bpmn-moddle';
|
||||||
|
|
||||||
|
const HERE = dirname(fileURLToPath(import.meta.url));
|
||||||
|
const activiti = JSON.parse(readFileSync(resolve(HERE, '../src/modeler/moddle/activiti.json'), 'utf8'));
|
||||||
|
const flowable = JSON.parse(readFileSync(resolve(HERE, '../src/modeler/moddle/flowable.json'), 'utf8'));
|
||||||
|
|
||||||
|
const PG_HOST = process.env.PG_HOST || 'localhost';
|
||||||
|
const PG_PORT = process.env.PG_PORT || '5433';
|
||||||
|
const PG_DB = process.env.PG_DB || 'eip4';
|
||||||
|
const PG_SCHEMA = process.env.PG_SCHEMA || 'eip3';
|
||||||
|
|
||||||
|
// ── 取数:psql 拉 id|name|base64(base64 去换行,同勘探法)───────────────────
|
||||||
|
function pullCorpus() {
|
||||||
|
const sql =
|
||||||
|
`set search_path=${PG_SCHEMA},public; ` +
|
||||||
|
`select b.id_||'|'||b.name_||'|'||translate(encode(b.bytes_,'base64'),E'\\n','') ` +
|
||||||
|
`from act_ge_bytearray b where b.name_ like '%.bpmn%' or b.name_ like '%.xml';`;
|
||||||
|
const out = execFileSync(
|
||||||
|
'psql',
|
||||||
|
['-h', PG_HOST, '-p', PG_PORT, '-d', PG_DB, '-tA', '-c', sql],
|
||||||
|
{ encoding: 'utf8', maxBuffer: 1024 * 1024 * 1024 },
|
||||||
|
);
|
||||||
|
const docs = [];
|
||||||
|
for (const line of out.split('\n')) {
|
||||||
|
if (!line) continue;
|
||||||
|
const i1 = line.indexOf('|');
|
||||||
|
const i2 = line.indexOf('|', i1 + 1);
|
||||||
|
if (i1 < 0 || i2 < 0) continue;
|
||||||
|
docs.push({ id: line.slice(0, i1), name: line.slice(i1 + 1, i2), b64: line.slice(i2 + 1) });
|
||||||
|
}
|
||||||
|
return docs;
|
||||||
|
}
|
||||||
|
|
||||||
|
// ── XML 实体解码归一(数据级比对:"→" "→" 全角/中文原样)────────────
|
||||||
|
function decodeEntities(s) {
|
||||||
|
return String(s)
|
||||||
|
.replace(/&#x([0-9a-fA-F]+);/g, (_, h) => String.fromCodePoint(parseInt(h, 16)))
|
||||||
|
.replace(/&#(\d+);/g, (_, d) => String.fromCodePoint(parseInt(d, 10)))
|
||||||
|
.replace(/"/g, '"')
|
||||||
|
.replace(/'/g, "'")
|
||||||
|
.replace(/</g, '<')
|
||||||
|
.replace(/>/g, '>')
|
||||||
|
.replace(/&/g, '&'); // amp 最后解,避免 &#34; 被二次解码
|
||||||
|
}
|
||||||
|
|
||||||
|
// ── id 净化:镜像 ProcessDiagramController.sanitizeBpmnIds(可逆非本工具关注)─
|
||||||
|
// 安全 id = ASCII-NCName(首字符字母/下划线,其余字母/数字/_/-/.)。非法则映射为
|
||||||
|
// 稳定唯一占位符(本工具用无冲突计数器,隔离 T1.3 哈希碰撞与 moddle 保真两问题)。
|
||||||
|
const ID_ATTR = /\b(id|bpmnElement|sourceRef|targetRef|attachedToRef|processRef|default)\s*=\s*"([^"]+)"/g;
|
||||||
|
function isSafeId(v) {
|
||||||
|
return /^[A-Za-z_][A-Za-z0-9_.-]*$/.test(v);
|
||||||
|
}
|
||||||
|
function sanitizeIds(xml) {
|
||||||
|
const map = new Map();
|
||||||
|
let n = 0;
|
||||||
|
let m;
|
||||||
|
ID_ATTR.lastIndex = 0;
|
||||||
|
while ((m = ID_ATTR.exec(xml))) {
|
||||||
|
const val = m[2];
|
||||||
|
if (!isSafeId(val) && !map.has(val)) map.set(val, `sanid_${n++}`);
|
||||||
|
}
|
||||||
|
if (map.size === 0) return { xml, remapped: 0 };
|
||||||
|
ID_ATTR.lastIndex = 0;
|
||||||
|
const rewritten = xml.replace(ID_ATTR, (_, attr, val) => `${attr}="${map.get(val) ?? val}"`);
|
||||||
|
return { xml: rewritten, remapped: map.size };
|
||||||
|
}
|
||||||
|
|
||||||
|
// ── 命名空间嗅探:activiti 优先,其次 flowable,默认 activiti ─────────────────
|
||||||
|
function sniffModdle(xml) {
|
||||||
|
const hasActiviti = xml.includes('activiti.org/bpmn') || /\bactiviti:/.test(xml);
|
||||||
|
const hasFlowable = xml.includes('flowable.org/bpmn') || /\bflowable:/.test(xml);
|
||||||
|
if (hasActiviti) return { pkg: { activiti }, ns: 'activiti' };
|
||||||
|
if (hasFlowable) return { pkg: { flowable }, ns: 'flowable' };
|
||||||
|
return { pkg: { activiti }, ns: 'activiti(default)' };
|
||||||
|
}
|
||||||
|
|
||||||
|
// ── 扩展面「多重集」签名:数据级零丢失的比对基础 ───────────────────────────
|
||||||
|
// 逐 tag 扫描(well-formed BPMN 下正则足够),区分扩展元素 tag / 原生元素上的
|
||||||
|
// 扩展属性;文本单独扫;id 全量。所有值实体解码后入袋(顺序无关)。
|
||||||
|
const TAG_RE = /<([/!?]?)([\w:.-]+)((?:[^>"']|"[^"]*"|'[^']*')*)>/g;
|
||||||
|
const ATTR_RE = /([\w:.-]+)\s*=\s*"([^"]*)"/g;
|
||||||
|
const EXT_TEXT_RE = /<((?:activiti|flowable):[\w.-]+)((?:[^>"']|"[^"]*"|'[^']*')*)>([^<]*)<\//g;
|
||||||
|
|
||||||
|
function surface(xml) {
|
||||||
|
const extAttrs = []; // 原生元素上的 activiti:/flowable: 属性 p:name=value
|
||||||
|
const extElemTags = []; // 扩展元素本身 tag
|
||||||
|
const extElemAttrs = [];// 扩展元素上的属性(含无前缀) tag@attr=value
|
||||||
|
const extElemText = []; // 扩展叶元素的文本 tag#text=value
|
||||||
|
const ids = []; // 全部 id="..."(含原生/DI/扩展)
|
||||||
|
|
||||||
|
let m;
|
||||||
|
TAG_RE.lastIndex = 0;
|
||||||
|
while ((m = TAG_RE.exec(xml))) {
|
||||||
|
const close = m[1];
|
||||||
|
const name = m[2];
|
||||||
|
const attrBlob = m[3] || '';
|
||||||
|
if (close === '/' || close === '!' || close === '?') continue; // 闭合/注释/声明
|
||||||
|
const isExt = /^(activiti|flowable):/.test(name);
|
||||||
|
let a;
|
||||||
|
ATTR_RE.lastIndex = 0;
|
||||||
|
while ((a = ATTR_RE.exec(attrBlob))) {
|
||||||
|
const an = a[1];
|
||||||
|
const av = decodeEntities(a[2]);
|
||||||
|
if (an === 'id') ids.push(av);
|
||||||
|
if (isExt) {
|
||||||
|
extElemAttrs.push(`${name}@${an}=${av}`);
|
||||||
|
} else if (/^(activiti|flowable):/.test(an)) {
|
||||||
|
extAttrs.push(`${an}=${av}`);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if (isExt) extElemTags.push(name);
|
||||||
|
}
|
||||||
|
|
||||||
|
EXT_TEXT_RE.lastIndex = 0;
|
||||||
|
while ((m = EXT_TEXT_RE.exec(xml))) {
|
||||||
|
const t = decodeEntities(m[3]).trim();
|
||||||
|
if (t) extElemText.push(`${m[1]}#text=${t}`);
|
||||||
|
}
|
||||||
|
|
||||||
|
return { extAttrs, extElemTags, extElemAttrs, extElemText, ids };
|
||||||
|
}
|
||||||
|
|
||||||
|
function bag(arr) {
|
||||||
|
const b = new Map();
|
||||||
|
for (const v of arr) b.set(v, (b.get(v) || 0) + 1);
|
||||||
|
return b;
|
||||||
|
}
|
||||||
|
// 多重集差异:返回 {missing:[...], added:[...]}(before→after)
|
||||||
|
function diffBag(before, after) {
|
||||||
|
const b = bag(before);
|
||||||
|
const a = bag(after);
|
||||||
|
const missing = [];
|
||||||
|
const added = [];
|
||||||
|
for (const [k, c] of b) {
|
||||||
|
const ac = a.get(k) || 0;
|
||||||
|
if (ac < c) missing.push(`${k} (×${c - ac})`);
|
||||||
|
}
|
||||||
|
for (const [k, c] of a) {
|
||||||
|
const bc = b.get(k) || 0;
|
||||||
|
if (bc < c) added.push(`${k} (×${c - bc})`);
|
||||||
|
}
|
||||||
|
return { missing, added };
|
||||||
|
}
|
||||||
|
|
||||||
|
// ── 主流程 ───────────────────────────────────────────────────────────────
|
||||||
|
async function main() {
|
||||||
|
process.stdout.write(`连库拉取语料 (${PG_HOST}:${PG_PORT}/${PG_DB}, schema=${PG_SCHEMA}) ...\n`);
|
||||||
|
const docs = pullCorpus();
|
||||||
|
process.stdout.write(`取到 ${docs.length} 个 .bpmn/.xml 文档,开始 round-trip 扫描 ...\n\n`);
|
||||||
|
|
||||||
|
let ok = 0;
|
||||||
|
const failures = [];
|
||||||
|
const decodeFailures = [];
|
||||||
|
const parseErrors = [];
|
||||||
|
const warnCounts = new Map(); // 归一 warning key → 次数
|
||||||
|
const nsCounts = new Map();
|
||||||
|
|
||||||
|
for (const doc of docs) {
|
||||||
|
// 1) base64 → UTF8,并校验解码无损(非法 UTF8 会 toString 回落 U+FFFD)
|
||||||
|
const buf = Buffer.from(doc.b64, 'base64');
|
||||||
|
const xml = buf.toString('utf8');
|
||||||
|
if (!Buffer.from(xml, 'utf8').equals(buf)) {
|
||||||
|
decodeFailures.push({ id: doc.id, name: doc.name, bytes: buf.length });
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
// 2) 命名空间嗅探 → 单载
|
||||||
|
const { pkg, ns } = sniffModdle(xml);
|
||||||
|
nsCounts.set(ns, (nsCounts.get(ns) || 0) + 1);
|
||||||
|
|
||||||
|
// 3) 净化非法 id(否则中文 id 文档被 moddle 整体拒收)
|
||||||
|
const { xml: sanitized } = sanitizeIds(xml);
|
||||||
|
|
||||||
|
// 4) round-trip
|
||||||
|
let out;
|
||||||
|
let warnings = [];
|
||||||
|
try {
|
||||||
|
const moddle = new BpmnModdle(pkg);
|
||||||
|
const parsed = await moddle.fromXML(sanitized);
|
||||||
|
warnings = parsed.warnings || [];
|
||||||
|
const exported = await moddle.toXML(parsed.rootElement, { format: false });
|
||||||
|
out = exported.xml;
|
||||||
|
} catch (e) {
|
||||||
|
parseErrors.push({ id: doc.id, name: doc.name, error: String(e && e.message || e) });
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
for (const w of warnings) {
|
||||||
|
const key = normWarn(String(w && w.message || w));
|
||||||
|
warnCounts.set(key, (warnCounts.get(key) || 0) + 1);
|
||||||
|
}
|
||||||
|
|
||||||
|
// 5) 数据级零丢失比对:sanitized(前) vs out(后)
|
||||||
|
const before = surface(sanitized);
|
||||||
|
const after = surface(out);
|
||||||
|
const diffs = {};
|
||||||
|
let lost = false;
|
||||||
|
for (const k of ['extAttrs', 'extElemTags', 'extElemAttrs', 'extElemText', 'ids']) {
|
||||||
|
const d = diffBag(before[k], after[k]);
|
||||||
|
if (d.missing.length || d.added.length) {
|
||||||
|
diffs[k] = d;
|
||||||
|
lost = true;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if (lost) failures.push({ id: doc.id, name: doc.name, ns, diffs });
|
||||||
|
else ok++;
|
||||||
|
}
|
||||||
|
|
||||||
|
report({ total: docs.length, ok, failures, decodeFailures, parseErrors, warnCounts, nsCounts });
|
||||||
|
// 硬门:真实 BPMN 全部零丢失(解码失败=非目标语料,单列不计入失败门)
|
||||||
|
process.exit(failures.length === 0 && parseErrors.length === 0 ? 0 : 1);
|
||||||
|
}
|
||||||
|
|
||||||
|
// warning 归一:抽取属性/元素名做聚合键
|
||||||
|
function normWarn(msg) {
|
||||||
|
const mAttr = msg.match(/unknown (?:type|attribute) <([^>]+)>/i);
|
||||||
|
if (mAttr) return `unknown <${mAttr[1]}>`;
|
||||||
|
const mUn = msg.match(/unparsable content <([^>]+)>/i);
|
||||||
|
if (mUn) return `unparsable <${mUn[1]}>`;
|
||||||
|
const mRef = msg.match(/unresolved reference <([^>]+)>/i);
|
||||||
|
if (mRef) return 'unresolved reference';
|
||||||
|
return msg.replace(/<[^>]*>/g, '<…>').slice(0, 80);
|
||||||
|
}
|
||||||
|
|
||||||
|
function report(r) {
|
||||||
|
const line = '─'.repeat(60);
|
||||||
|
process.stdout.write(`${line}\n保真扫描结果\n${line}\n`);
|
||||||
|
const realFail = r.failures.length + r.parseErrors.length;
|
||||||
|
const realTotal = r.total - r.decodeFailures.length;
|
||||||
|
process.stdout.write(`匹配 ${r.ok}/${realTotal}(真实 BPMN,已排除 ${r.decodeFailures.length} 解码失败)\n`);
|
||||||
|
process.stdout.write(`总文档 ${r.total}|零丢失 ${r.ok}|丢失 ${r.failures.length}|解析异常 ${r.parseErrors.length}|解码失败 ${r.decodeFailures.length}\n`);
|
||||||
|
|
||||||
|
process.stdout.write(`\n命名空间加载分布:\n`);
|
||||||
|
for (const [ns, c] of [...r.nsCounts].sort((a, b) => b[1] - a[1])) {
|
||||||
|
process.stdout.write(` ${ns}: ${c}\n`);
|
||||||
|
}
|
||||||
|
|
||||||
|
process.stdout.write(`\nfromXML warnings(按归一键聚合,仅描述符覆盖信号,不判失败):\n`);
|
||||||
|
const ws = [...r.warnCounts].sort((a, b) => b[1] - a[1]);
|
||||||
|
if (!ws.length) process.stdout.write(' (无)\n');
|
||||||
|
for (const [k, c] of ws.slice(0, 40)) process.stdout.write(` ${c.toString().padStart(6)} ${k}\n`);
|
||||||
|
if (ws.length > 40) process.stdout.write(` … 另有 ${ws.length - 40} 类\n`);
|
||||||
|
|
||||||
|
if (r.decodeFailures.length) {
|
||||||
|
process.stdout.write(`\nbase64→UTF8 解码失败文档(单列,非 UTF8 BPMN):\n`);
|
||||||
|
for (const d of r.decodeFailures) process.stdout.write(` id=${d.id} name=${d.name} bytes=${d.bytes}\n`);
|
||||||
|
}
|
||||||
|
|
||||||
|
if (r.parseErrors.length) {
|
||||||
|
process.stdout.write(`\nmoddle 解析/序列化异常:\n`);
|
||||||
|
for (const e of r.parseErrors.slice(0, 30)) process.stdout.write(` id=${e.id} name=${e.name}\n ${e.error}\n`);
|
||||||
|
}
|
||||||
|
|
||||||
|
if (r.failures.length) {
|
||||||
|
process.stdout.write(`\n零丢失失败清单(round-trip 前后扩展面差异):\n`);
|
||||||
|
for (const f of r.failures.slice(0, 30)) {
|
||||||
|
process.stdout.write(` id=${f.id} name=${f.name} [${f.ns}]\n`);
|
||||||
|
for (const [k, d] of Object.entries(f.diffs)) {
|
||||||
|
if (d.missing.length) process.stdout.write(` ${k} 丢失: ${d.missing.slice(0, 6).join(' ; ')}${d.missing.length > 6 ? ' …' : ''}\n`);
|
||||||
|
if (d.added.length) process.stdout.write(` ${k} 新增: ${d.added.slice(0, 6).join(' ; ')}${d.added.length > 6 ? ' …' : ''}\n`);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if (r.failures.length > 30) process.stdout.write(` … 另有 ${r.failures.length - 30} 个失败文档\n`);
|
||||||
|
}
|
||||||
|
process.stdout.write(`${line}\n`);
|
||||||
|
}
|
||||||
|
|
||||||
|
main().catch((e) => {
|
||||||
|
process.stderr.write(`扫描崩溃: ${e && e.stack || e}\n`);
|
||||||
|
process.exit(2);
|
||||||
|
});
|
||||||
Reference in New Issue
Block a user