fix(modeler): codec 评审收口——stringToJson引号映射+半角content变体+item未知键透传+形状守卫
评审者跑全库 4794 条真实表达式往返对照原版序列化器,四项收口:
- I-1 build 补 stringToJson 引号契约(BasePropertiesConverter.java:235-280):字符串叶子值
'→‘、"→“(经 JSON.stringify replacer,{}→【】由 halfToFull、\\与控制字符由 stringify 覆盖)。
否则用户输入含 ' 截断外层 JUEL 单引号字面量 / 含 " 截断内层 JSON,部署后表达式解析即断。
语料 ‘/“ 均 0 命中→对存量为 no-op,不破逐字往返。
- I-2 头注释纠偏:「content 不含花括号」被 20 条真实 ${users}(半角)证伪;改为事实——
4774/4794 逐字,20 条半角 content 变体 build 归一为全角,引擎 :54 全局【→{ 还原后运行期等价。
- M-1 item 未知键 known-first 重排+按原序透传(与 meta 全量保留对称,不静默丢未来扩展键)。
- M-2 items 元素形状守卫:元素非对象或 type 非字符串→整体返 null 不抛。
加分:tools/codec-corpus-sweep.mjs(照 modeler-fidelity-sweep 风格,内联镜像 codec 跑全语料):
1512 文档 / 4794 表达式 → 逐字 4774 + 引擎运行期等价 20 + 解析失败 0 + 硬不匹配 0(门禁绿)。
测试 29 例(原 20 + 引号映射/幂等/meta 引号/半角变体运行期等价/未知键透传/形状守卫 9 例);
modeler 全量 66 例绿;tsc -b 干净。
Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
178
tools/codec-corpus-sweep.mjs
Normal file
178
tools/codec-corpus-sweep.mjs
Normal file
@@ -0,0 +1,178 @@
|
||||
#!/usr/bin/env node
|
||||
/**
|
||||
* 语料级办理人/会签表达式 codec 往返核查(T2.3 评审加分项)
|
||||
*
|
||||
* 对本地 eip4 库(5433, search_path=eip3)里 act_ge_bytearray 的全部真实 BPMN,抽出每个
|
||||
* activiti/flowable 的 candidateUsers / collection 属性值(实体解码到 businessObject 属性
|
||||
* 值层),跑与 src/modeler/candidateExpression.ts **逐字镜像**的 parse/build,统计:
|
||||
* - byteIdentical : build(parse(x)) === x 逐字一致
|
||||
* - engineEquiv : 非逐字但引擎运行期等价(整串全局 【→{ 、】→} 还原后相同)
|
||||
* —— 覆盖 content 用半角 ${users} 的 scriptUser 会签变体(build 归一为全角,
|
||||
* 引擎 CandidateUsersHandler:54 全局替换后复原,语义无损)
|
||||
* - parseNull : 前缀匹配但解析失败(硬失败)
|
||||
* - hardMismatch : 既非逐字亦非引擎等价(硬失败)
|
||||
*
|
||||
* ⚠️ 本文件内联的 parse/build 必须与 candidateExpression.ts 保持一致;单测
|
||||
* (candidateExpression.test.ts)是编解码契约的权威来源,本工具是全语料回归的旁证。
|
||||
*
|
||||
* 取数:psql 子进程(不引 pg 依赖),同 tools/modeler-fidelity-sweep.mjs。
|
||||
* 用法:node tools/codec-corpus-sweep.mjs
|
||||
* 环境变量:PG_HOST(localhost) PG_PORT(5433) PG_DB(eip4) PG_SCHEMA(eip3) EXPECTED_TOTAL(1512)
|
||||
*/
|
||||
import { execFileSync } from 'node:child_process';
|
||||
|
||||
const PG_HOST = process.env.PG_HOST || 'localhost';
|
||||
const PG_PORT = process.env.PG_PORT || '5433';
|
||||
const PG_DB = process.env.PG_DB || 'eip4';
|
||||
const PG_SCHEMA = process.env.PG_SCHEMA || 'eip3';
|
||||
const EXPECTED_TOTAL = Number(process.env.EXPECTED_TOTAL || 1512);
|
||||
|
||||
// ── 镜像 src/modeler/candidateExpression.ts(逐字)────────────────────────────
|
||||
const CANDIDATE_PREFIX = "${candidateUsersHandler.getCandidateUsers(execution,'";
|
||||
const MULTI_PREFIX = "${multiInstanceUsersHandler.getUsers(execution,'";
|
||||
const SUFFIX = "')}";
|
||||
const KNOWN_ITEM_KEYS = ['type', 'content', 'isCandidate', 'isSendMail', 'isSendPhone', 'isSendMessage'];
|
||||
const KNOWN_ITEM_KEY_SET = new Set(KNOWN_ITEM_KEYS);
|
||||
const fullToHalf = (s) => s.split('【').join('{').split('】').join('}');
|
||||
const halfToFull = (s) => s.split('{').join('【').split('}').join('】');
|
||||
const escapeLeaf = (v) => v.split("'").join('‘').split('"').join('“');
|
||||
function reorderItem(src) {
|
||||
const o = {};
|
||||
for (const k of KNOWN_ITEM_KEYS) if (src[k] !== undefined) o[k] = src[k];
|
||||
for (const k of Object.keys(src)) if (!KNOWN_ITEM_KEY_SET.has(k) && src[k] !== undefined) o[k] = src[k];
|
||||
return o;
|
||||
}
|
||||
function parse(expr) {
|
||||
if (typeof expr !== 'string' || expr.length === 0) return null;
|
||||
let variant, prefix;
|
||||
if (expr.startsWith(CANDIDATE_PREFIX)) { variant = 'candidate'; prefix = CANDIDATE_PREFIX; }
|
||||
else if (expr.startsWith(MULTI_PREFIX)) { variant = 'multiInstance'; prefix = MULTI_PREFIX; }
|
||||
else return null;
|
||||
if (!expr.endsWith(SUFFIX)) return null;
|
||||
const content = expr.slice(prefix.length, expr.length - SUFFIX.length);
|
||||
let obj;
|
||||
try { obj = JSON.parse(fullToHalf(content)); } catch { return null; }
|
||||
if (obj === null || typeof obj !== 'object') return null;
|
||||
if (!Array.isArray(obj.items)) return null;
|
||||
for (const el of obj.items) {
|
||||
if (el === null || typeof el !== 'object' || Array.isArray(el)) return null;
|
||||
if (typeof el.type !== 'string') return null;
|
||||
}
|
||||
const items = obj.items.map(reorderItem);
|
||||
const meta = { ...obj };
|
||||
delete meta.items;
|
||||
return { items, variant, meta };
|
||||
}
|
||||
function build(items, variant, meta) {
|
||||
const prefix = variant === 'multiInstance' ? MULTI_PREFIX : CANDIDATE_PREFIX;
|
||||
const outer = { items: items.map(reorderItem), ...(meta ?? {}) };
|
||||
const json = JSON.stringify(outer, (_k, v) => (typeof v === 'string' ? escapeLeaf(v) : v));
|
||||
return prefix + halfToFull(json) + SUFFIX;
|
||||
}
|
||||
// 镜像引擎 CandidateUsersHandler:54 的整串全局字面替换
|
||||
const engineRestore = (s) => s.split('【').join('{').split('】').join('}');
|
||||
// ─────────────────────────────────────────────────────────────────────────
|
||||
|
||||
function decodeEntities(s) {
|
||||
return String(s)
|
||||
.replace(/&#x([0-9a-fA-F]+);/g, (_, h) => String.fromCodePoint(parseInt(h, 16)))
|
||||
.replace(/&#(\d+);/g, (_, d) => String.fromCodePoint(parseInt(d, 10)))
|
||||
.replace(/"/g, '"')
|
||||
.replace(/'/g, "'")
|
||||
.replace(/</g, '<')
|
||||
.replace(/>/g, '>')
|
||||
.replace(/&/g, '&');
|
||||
}
|
||||
|
||||
function pullCorpus() {
|
||||
const sql =
|
||||
`set search_path=${PG_SCHEMA},public; ` +
|
||||
`select b.id_||'|'||translate(encode(b.bytes_,'base64'),E'\\n','') ` +
|
||||
`from act_ge_bytearray b where b.name_ like '%.bpmn%' or b.name_ like '%.xml';`;
|
||||
const out = execFileSync(
|
||||
'psql',
|
||||
['-h', PG_HOST, '-p', PG_PORT, '-d', PG_DB, '-tA', '-c', sql],
|
||||
{ encoding: 'utf8', maxBuffer: 1024 * 1024 * 1024 },
|
||||
);
|
||||
const docs = [];
|
||||
for (const line of out.split('\n')) {
|
||||
if (!line) continue;
|
||||
const i = line.indexOf('|');
|
||||
if (i < 0) continue;
|
||||
docs.push({ id: line.slice(0, i), b64: line.slice(i + 1) });
|
||||
}
|
||||
return docs;
|
||||
}
|
||||
|
||||
function main() {
|
||||
process.stdout.write(`连库拉取语料 (${PG_HOST}:${PG_PORT}/${PG_DB}, schema=${PG_SCHEMA}) ...\n`);
|
||||
const docs = pullCorpus();
|
||||
process.stdout.write(`取到 ${docs.length} 个 .bpmn/.xml 文档,开始 codec 往返扫描 ...\n\n`);
|
||||
|
||||
const EXPR_RE = /(?:activiti|flowable):(?:candidateUsers|collection)="([^"]*)"/g;
|
||||
let exprs = 0;
|
||||
let byteIdentical = 0;
|
||||
let engineEquiv = 0;
|
||||
let parseNull = 0;
|
||||
let hardMismatch = 0;
|
||||
const itemKeySets = new Map();
|
||||
const metaKeySets = new Map();
|
||||
const hardSamples = [];
|
||||
const parseNullSamples = [];
|
||||
|
||||
for (const doc of docs) {
|
||||
const xml = Buffer.from(doc.b64, 'base64').toString('utf8');
|
||||
let m;
|
||||
EXPR_RE.lastIndex = 0;
|
||||
while ((m = EXPR_RE.exec(xml)) !== null) {
|
||||
const val = decodeEntities(m[1]);
|
||||
if (!val.includes('Handler')) continue; // collection 也可是普通变量名,非 handler 表达式
|
||||
exprs++;
|
||||
const r = parse(val);
|
||||
if (!r) {
|
||||
parseNull++;
|
||||
if (parseNullSamples.length < 5) parseNullSamples.push(val.slice(0, 200));
|
||||
continue;
|
||||
}
|
||||
for (const it of r.items) itemKeySets.set(Object.keys(it).join(','), (itemKeySets.get(Object.keys(it).join(',')) || 0) + 1);
|
||||
metaKeySets.set(Object.keys(r.meta).join(','), (metaKeySets.get(Object.keys(r.meta).join(',')) || 0) + 1);
|
||||
const rebuilt = build(r.items, r.variant, r.meta);
|
||||
if (rebuilt === val) byteIdentical++;
|
||||
else if (engineRestore(rebuilt) === engineRestore(val)) engineEquiv++;
|
||||
else {
|
||||
hardMismatch++;
|
||||
if (hardSamples.length < 5) hardSamples.push({ val: val.slice(0, 300), rebuilt: rebuilt.slice(0, 300) });
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
const line = '─'.repeat(60);
|
||||
process.stdout.write(`${line}\ncodec 往返扫描结果\n${line}\n`);
|
||||
process.stdout.write(`总文档 ${docs.length}(预期 ${EXPECTED_TOTAL})|表达式 ${exprs}\n`);
|
||||
process.stdout.write(`逐字一致 ${byteIdentical}|引擎运行期等价 ${engineEquiv}|解析失败 ${parseNull}|硬不匹配 ${hardMismatch}\n`);
|
||||
process.stdout.write(`\nitem 键集分布:\n`);
|
||||
for (const [k, c] of [...itemKeySets].sort((a, b) => b[1] - a[1])) process.stdout.write(` ${c.toString().padStart(6)} [${k}]\n`);
|
||||
process.stdout.write(`\nmeta 键集分布:\n`);
|
||||
for (const [k, c] of [...metaKeySets].sort((a, b) => b[1] - a[1])) process.stdout.write(` ${c.toString().padStart(6)} [${k || '(空)'}]\n`);
|
||||
if (parseNullSamples.length) {
|
||||
process.stdout.write(`\n解析失败样本:\n`);
|
||||
for (const s of parseNullSamples) process.stdout.write(` ${s}\n`);
|
||||
}
|
||||
if (hardSamples.length) {
|
||||
process.stdout.write(`\n硬不匹配样本(既非逐字亦非引擎等价):\n`);
|
||||
for (const s of hardSamples) process.stdout.write(` 原文: ${s.val}\n 重建: ${s.rebuilt}\n`);
|
||||
}
|
||||
process.stdout.write(`${line}\n`);
|
||||
|
||||
// 硬门:每条表达式都能解析,且要么逐字要么引擎等价(无硬失败),且语料规模符合预期。
|
||||
const gateOk = parseNull === 0 && hardMismatch === 0 && docs.length === EXPECTED_TOTAL;
|
||||
process.exit(gateOk ? 0 : 1);
|
||||
}
|
||||
|
||||
try {
|
||||
main();
|
||||
} catch (e) {
|
||||
if (e && e.stderr) process.stderr.write(String(e.stderr));
|
||||
process.stderr.write(`扫描崩溃: ${(e && e.stack) || e}\n`);
|
||||
process.exit(2);
|
||||
}
|
||||
Reference in New Issue
Block a user