fix(parity): 客户库真实实体对拍适配——计算列登记/复合键升级/null省略/布尔域归因/抽样模式

首轮 24 真实实体仅 6 个全一致,逐项归因后修 harness(保真层零改动):
- 引擎计算列(显示翻译/STATUS lateral)整列登记不判负(engineExtraColumns)
- 主键不唯一自动升级全部 *ID 复合键(关联表 OA_SEALUSEAPPLYANDSEALRRELATION)
- 引擎 JSON 省略 null 值列:右缺列+左 NULL=语义等值
- BOOLEAN 域外脏数据(2/-1)引擎归 null:元数据感知的已归因差异,可见不判负
- 单次全量抓取消除翻页幻影;胖列 1.7GB 实体自动回退翻页;--sampleRows/--perPage 抽样模式
复跑 24 实体:23 全量 100% 一致(442,529 行),PM_PAYMENTAPPLY 抽样口径另记。
RowDiff 单测 17→22 全绿;四个引擎新坑进迁移手册 §5.11–5.14

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
2026-06-11 10:15:44 +08:00
parent 83c6abb1f9
commit d39e2c4fb6
6 changed files with 257 additions and 10 deletions

View File

@@ -4,6 +4,8 @@ import java.util.ArrayList;
import java.util.LinkedHashMap; import java.util.LinkedHashMap;
import java.util.List; import java.util.List;
import java.util.Map; import java.util.Map;
import java.util.Set;
import java.util.TreeSet;
/** /**
* 一次结果集逐行对拍的结构化差异报告。 * 一次结果集逐行对拍的结构化差异报告。
@@ -29,6 +31,25 @@ public class DiffReport {
/** 主键匹配但有单元格差异的行。 */ /** 主键匹配但有单元格差异的行。 */
public final List<RowMismatch> mismatches = new ArrayList<>(); public final List<RowMismatch> mismatches = new ArrayList<>();
/**
* 引擎侧(右)多出、物理基线侧(左)没有的列名(engine-extra)。
*
* <p>EIP3 引擎 listSelect 对真实实体会输出一批"计算列/引用显示翻译"(如 UNITNAME、
* XXX_DISPLAY__、STATUS 联查、PROJECTUSER、DOCUMENT/LABORTEAMS 等 lateral 产物),
* 物理表并无这些列。对拍比对域限定为"左侧物理列集",故这些列不参与逐列比对、不判负,
* 仅在此整列登记一次,供报告说明。其正确性归"录制重放对旧系统验证",不在本对拍范围内。
*/
public final Set<String> engineExtraColumns = new TreeSet<>();
/** 引擎对 null 值列在 JSON 中整列省略(右缺列+左值 NULL):语义等值,登记列名供报告说明。 */
public final Set<String> engineNullOmittedColumns = new TreeSet<>();
/**
* 已归因差异:BOOLEAN 元数据列、物理值在 {0,1} 域外(旧库脏数据)、引擎归一为 null。
* 新旧系统引擎行为一致,非移植缺陷;计数可见但不判负。列名→出现次数。
*/
public final Map<String, Integer> explainedBooleanDomainCells = new LinkedHashMap<>();
public DiffReport(String label) { public DiffReport(String label) {
this.label = label; this.label = label;
} }
@@ -108,6 +129,9 @@ public class DiffReport {
m.put("mismatchRows", mismatches.size()); m.put("mismatchRows", mismatches.size());
m.put("onlyInLeft", onlyInLeft); m.put("onlyInLeft", onlyInLeft);
m.put("onlyInRight", onlyInRight); m.put("onlyInRight", onlyInRight);
m.put("engineExtraColumns", new ArrayList<>(engineExtraColumns));
m.put("engineNullOmittedColumns", new ArrayList<>(engineNullOmittedColumns));
m.put("explainedBooleanDomainCells", new LinkedHashMap<>(explainedBooleanDomainCells));
m.put("clean", isClean()); m.put("clean", isClean());
m.put("consistencyRate", consistencyRate()); m.put("consistencyRate", consistencyRate());
List<Map<String, Object>> ms = new ArrayList<>(); List<Map<String, Object>> ms = new ArrayList<>();
@@ -130,6 +154,19 @@ public class DiffReport {
.append(" 仅右侧:").append(onlyInRight.size()).append("\n"); .append(" 仅右侧:").append(onlyInRight.size()).append("\n");
sb.append("- 逐行一致率:").append(String.format("%.4f", consistencyRate())) sb.append("- 逐行一致率:").append(String.format("%.4f", consistencyRate()))
.append(isClean() ? " ✅ 完全一致" : " ⚠️ 存在差异").append("\n"); .append(isClean() ? " ✅ 完全一致" : " ⚠️ 存在差异").append("\n");
if (!engineExtraColumns.isEmpty()) {
sb.append("- 引擎额外列(engine-extra,计算列,不判负):`")
.append(String.join(", ", engineExtraColumns)).append("`\n");
}
if (!engineNullOmittedColumns.isEmpty()) {
sb.append("- 引擎省略的 null 值列(语义等值): `")
.append(String.join(", ", engineNullOmittedColumns)).append("`\n");
}
if (!explainedBooleanDomainCells.isEmpty()) {
sb.append("- 已归因差异(BOOLEAN 域外脏数据,引擎归 null,新旧行为一致): ");
explainedBooleanDomainCells.forEach((c, n) -> sb.append('`').append(c).append("`x").append(n).append(' '));
sb.append('\n');
}
if (!mismatches.isEmpty()) { if (!mismatches.isEmpty()) {
sb.append("\n差异明细(前 20 行):\n\n"); sb.append("\n差异明细(前 20 行):\n\n");
sb.append("| 主键 | 列 | 左值 | 右值 | 原因 |\n|---|---|---|---|---|\n"); sb.append("| 主键 | 列 | 左值 | 右值 | 原因 |\n|---|---|---|---|---|\n");

View File

@@ -48,6 +48,55 @@ public class EngineHttpClient {
* 抓某实体全量行,附加查询串(C2 数据权限用:{@code asUser=...&roles=...} 构造非 ADMIN ctx, * 抓某实体全量行,附加查询串(C2 数据权限用:{@code asUser=...&roles=...} 构造非 ADMIN ctx,
* 触发引擎行级过滤)。{@code extraQuery} 不含前导 {@code &}。 * 触发引擎行级过滤)。{@code extraQuery} 不含前导 {@code &}。
*/ */
/**
* 单次全量抓取:先取 total,再一次 perPage=total 拉全 —— 消除翻页幻影
* (引擎默认排序键不唯一时,页间行漂移导致 onlyLeft/onlyRight 成对误判)。
* total 超过 {@code maxSingleShot} 时退回翻页模式。
*/
public List<Map<String, Object>> listAllSingleShot(String table, int maxSingleShot) throws Exception {
JsonNode probe = getList(table, 1, 1, "");
long total = probe.has("total") && probe.get("total").isNumber() ? probe.get("total").asLong() : -1;
if (total < 0 || total > maxSingleShot) {
return listAll(table, "");
}
if (total == 0) {
return new ArrayList<>();
}
JsonNode data = getList(table, 1, (int) total, "");
List<Map<String, Object>> rows = new ArrayList<>((int) total);
JsonNode items = data.get("items");
if (items != null && items.isArray()) {
for (JsonNode item : items) {
rows.add(jsonToRow(item));
}
}
return rows;
}
/** 取前 maxRows 行(翻页,不取全量)。抽样对拍用。 */
public List<Map<String, Object>> listFirst(String table, int maxRows) throws Exception {
List<Map<String, Object>> rows = new ArrayList<>();
int page = 1;
while (rows.size() < maxRows) {
JsonNode data = getList(table, page, perPage, "");
JsonNode items = data.get("items");
if (items == null || !items.isArray() || items.isEmpty()) {
break;
}
for (JsonNode item : items) {
if (rows.size() >= maxRows) {
break;
}
rows.add(jsonToRow(item));
}
if (items.size() < perPage) {
break;
}
page++;
}
return rows;
}
public List<Map<String, Object>> listAll(String table, String extraQuery) throws Exception { public List<Map<String, Object>> listAll(String table, String extraQuery) throws Exception {
List<Map<String, Object>> rows = new ArrayList<>(); List<Map<String, Object>> rows = new ArrayList<>();
int page = 1; int page = 1;
@@ -85,7 +134,7 @@ public class EngineHttpClient {
String url = baseUrl + "/api/entity/" + table + "/list?page=" + page + "&perPage=" + size String url = baseUrl + "/api/entity/" + table + "/list?page=" + page + "&perPage=" + size
+ (extraQuery == null || extraQuery.isBlank() ? "" : "&" + extraQuery); + (extraQuery == null || extraQuery.isBlank() ? "" : "&" + extraQuery);
HttpRequest req = HttpRequest.newBuilder(URI.create(url)) HttpRequest req = HttpRequest.newBuilder(URI.create(url))
.timeout(Duration.ofSeconds(60)).GET().build(); .timeout(Duration.ofSeconds(300)).GET().build();
HttpResponse<String> resp = http.send(req, HttpResponse.BodyHandlers.ofString(StandardCharsets.UTF_8)); HttpResponse<String> resp = http.send(req, HttpResponse.BodyHandlers.ofString(StandardCharsets.UTF_8));
if (resp.statusCode() != 200) { if (resp.statusCode() != 200) {
throw new IllegalStateException("引擎 list 非 200 (" + resp.statusCode() throw new IllegalStateException("引擎 list 非 200 (" + resp.statusCode()
@@ -107,7 +156,7 @@ public class EngineHttpClient {
String url = baseUrl + "/api/entity/" + table + "/list?page=1&perPage=1" String url = baseUrl + "/api/entity/" + table + "/list?page=1&perPage=1"
+ (extraQuery == null || extraQuery.isBlank() ? "" : "&" + extraQuery); + (extraQuery == null || extraQuery.isBlank() ? "" : "&" + extraQuery);
HttpRequest req = HttpRequest.newBuilder(URI.create(url)) HttpRequest req = HttpRequest.newBuilder(URI.create(url))
.timeout(Duration.ofSeconds(60)).GET().build(); .timeout(Duration.ofSeconds(300)).GET().build();
HttpResponse<String> resp = http.send(req, HttpResponse.BodyHandlers.ofString(StandardCharsets.UTF_8)); HttpResponse<String> resp = http.send(req, HttpResponse.BodyHandlers.ofString(StandardCharsets.UTF_8));
return resp.statusCode(); return resp.statusCode();
} }

View File

@@ -44,6 +44,8 @@ public final class ParityCli {
public static void main(String[] args) throws Exception { public static void main(String[] args) throws Exception {
Map<String, String> opt = parseArgs(args); Map<String, String> opt = parseArgs(args);
String jdbcUrl = opt.getOrDefault("jdbc", "jdbc:postgresql://localhost:5433/eip4m0"); String jdbcUrl = opt.getOrDefault("jdbc", "jdbc:postgresql://localhost:5433/eip4m0");
int sampleRows = Integer.parseInt(opt.getOrDefault("sampleRows", "0"));
int perPage = Integer.parseInt(opt.getOrDefault("perPage", "500"));
String user = opt.getOrDefault("user", "zhengwr"); String user = opt.getOrDefault("user", "zhengwr");
String pass = opt.getOrDefault("pass", ""); String pass = opt.getOrDefault("pass", "");
String engine = opt.getOrDefault("engine", "http://localhost:8084"); String engine = opt.getOrDefault("engine", "http://localhost:8084");
@@ -78,12 +80,14 @@ public final class ParityCli {
entities = listEip4ptEntities(ds); entities = listEip4ptEntities(ds);
} }
SelfCheckRunner runner = new SelfCheckRunner(engine, ds); SelfCheckRunner runner = new SelfCheckRunner(engine, ds, perPage);
List<DiffReport> reports = new ArrayList<>(); List<DiffReport> reports = new ArrayList<>();
int cleanCount = 0; int cleanCount = 0;
long totalRows = 0; long totalRows = 0;
for (String e : entities) { for (String e : entities) {
DiffReport r = runner.check(e.trim()); DiffReport r = sampleRows > 0
? runner.checkSample(e.trim(), sampleRows)
: runner.check(e.trim());
reports.add(r); reports.add(r);
totalRows += r.leftRowCount; totalRows += r.leftRowCount;
boolean clean = r.isClean(); boolean clean = r.isClean();

View File

@@ -23,7 +23,12 @@ public class SelfCheckRunner {
private final JdbcResultReader jdbc; private final JdbcResultReader jdbc;
public SelfCheckRunner(String engineBaseUrl, DataSource dataSource) { public SelfCheckRunner(String engineBaseUrl, DataSource dataSource) {
this.engine = new EngineHttpClient(engineBaseUrl); this(engineBaseUrl, dataSource, 500);
}
/** @param perPage 引擎翻页页大小;重查询实体(复杂视图 SQL)用小页降低单请求超时风险。 */
public SelfCheckRunner(String engineBaseUrl, DataSource dataSource, int perPage) {
this.engine = new EngineHttpClient(engineBaseUrl, perPage);
this.jdbc = new JdbcResultReader(dataSource); this.jdbc = new JdbcResultReader(dataSource);
} }
@@ -37,17 +42,95 @@ public class SelfCheckRunner {
} catch (Exception e) { } catch (Exception e) {
return error(t, "JDBC 直连读失败: " + e.getMessage()); return error(t, "JDBC 直连读失败: " + e.getMessage());
} }
try {
right = engine.listAllSingleShot(t, 120_000);
} catch (Exception first) {
// 单次全量对"胖列"实体可能超 Jackson 1GB 字符串上限(如 GB_PRJ_WORKER 照片列),
// 或单条巨型 SQL 超时 —— 回退翻页模式(翻页幻影风险:若结果出现 onlyL/R 成对,按实体留痕)
try { try {
right = engine.listAll(t); right = engine.listAll(t);
} catch (Exception e) { } catch (Exception e) {
return error(t, "引擎读路径失败: " + e.getMessage()); return error(t, "引擎读路径失败(单次:" + first.getMessage() + ";翻页:" + e.getMessage() + ")");
} }
// 主键:优先 {表名}_ID 约定(EIP3 设计器/引擎一致),回退自动推断 }
// 主键:优先 {表名}_ID 约定(EIP3 设计器/引擎一致),回退自动推断;
// 推断键在物理侧不唯一(关联表无自有 PK)时升级为全部 *ID 列复合键,再不唯一退全列配对
String pk = t + "_ID"; String pk = t + "_ID";
List<String> keys = (!left.isEmpty() && left.get(0).containsKey(pk)) List<String> keys = (!left.isEmpty() && left.get(0).containsKey(pk))
? List.of(pk) ? List.of(pk)
: RowDiff.inferKeyColumns(t, left.isEmpty() ? right : left); : RowDiff.inferKeyColumns(t, left.isEmpty() ? right : left);
return new RowDiff(keys).diff(t, left, right); if (!left.isEmpty() && !RowDiff.keyIsUnique(keys, left)) {
List<String> composite = RowDiff.allIdColumns(left);
keys = RowDiff.keyIsUnique(composite, left) ? composite : List.of();
}
// 自对拍模式:引擎计算列(右独有)登记不判负;BOOLEAN 域外脏数据归 null 为已归因差异
return new RowDiff(keys, true)
.booleanDomainColumns(booleanColumns(t))
.diff(t, left, right);
}
/**
* 抽样对拍:引擎侧取前 {@code maxRows} 行(翻页),按主键回查 JDBC 同行集,逐行比对。
* 用于重查询实体(如 PM_PAYMENTAPPLY:DISTINCT+4 lateral,单页 500 行 ≈61s,全量 ≈78min)
* 在时间盒内做有意义的值级验证;报告 label 注明抽样口径。
*/
public DiffReport checkSample(String table, int maxRows) {
String t = table.toUpperCase();
List<Map<String, Object>> right;
try {
right = engine.listFirst(t, maxRows);
} catch (Exception e) {
return error(t, "引擎读路径失败(抽样): " + e.getMessage());
}
String pk = t + "_ID";
if (right.isEmpty() || !upper(right.get(0)).containsKey(pk)) {
return error(t, "抽样模式需 {表名}_ID 主键且结果非空");
}
List<String> pks = new java.util.ArrayList<>(right.size());
for (Map<String, Object> r : right) {
Object v = upper(r).get(pk);
if (v != null) {
pks.add(v.toString());
}
}
List<Map<String, Object>> left = new java.util.ArrayList<>(pks.size());
for (int i = 0; i < pks.size(); i += 1000) {
List<String> chunk = pks.subList(i, Math.min(i + 1000, pks.size()));
String in = String.join(",", java.util.Collections.nCopies(chunk.size(), "?"));
left.addAll(jdbc.query(
"SELECT * FROM " + t + " WHERE " + pk + " IN (" + in + ")",
chunk.toArray()));
}
return new RowDiff(List.of(pk), true)
.booleanDomainColumns(booleanColumns(t))
.diff(t + "[抽样" + right.size() + "行]", left, right);
}
private static Map<String, Object> upper(Map<String, Object> row) {
Map<String, Object> m = new java.util.LinkedHashMap<>();
row.forEach((k, v) -> m.put(k == null ? null : k.toUpperCase(), v));
return m;
}
/** 实体的 BOOLEAN 元数据列集(大写);元数据查询失败时返回空集(规则自动失效,不影响对拍)。 */
private java.util.Set<String> booleanColumns(String table) {
try {
var rows = jdbc.query(
"SELECT a.columnname FROM sys_entityattribute a "
+ "JOIN sys_entity e ON a.entityid = e.sys_entity_id "
+ "WHERE upper(e.tablename) = ? AND a.referenceid = 'BOOLEAN' AND a.isactive = 1",
table.toUpperCase());
java.util.Set<String> cols = new java.util.HashSet<>();
for (var r : rows) {
Object c = r.values().iterator().next();
if (c != null) {
cols.add(c.toString().trim().toUpperCase());
}
}
return cols;
} catch (Exception e) {
return java.util.Set.of();
}
} }
private static DiffReport error(String table, String msg) { private static DiffReport error(String table, String msg) {

View File

@@ -230,4 +230,78 @@ class RowDiffTest {
private static List<String> strip(List<String> in) { private static List<String> strip(List<String> in) {
return in.stream().map(String::trim).toList(); return in.stream().map(String::trim).toList();
} }
// ===== 客户库口径修正(2026-06-11):引擎计算列登记 + 复合键升级 =====
@org.junit.jupiter.api.Test
void engineExtraColumnAsInfoNotMismatch() {
// 右(引擎)多出计算列 UNITNAME:自对拍模式不判负,整列登记 engineExtraColumns
var l = List.<Map<String, Object>>of(new java.util.HashMap<>(Map.of("T_ID", "k1", "A", 1)));
var r = List.<Map<String, Object>>of(new java.util.HashMap<>(Map.of("T_ID", "k1", "A", 1, "UNITNAME", "山东同益")));
DiffReport rep = new RowDiff(List.of("T_ID"), true).diff("T", l, r);
assertTrue(rep.isClean(), "计算列不应判负");
assertEquals(1, rep.matchedRows);
assertTrue(rep.engineExtraColumns.contains("UNITNAME"));
// 录制重放模式(默认)保持原语义:缺列即缺陷
DiffReport strict = new RowDiff(List.of("T_ID")).diff("T", l, r);
assertFalse(strict.isClean());
}
@org.junit.jupiter.api.Test
void keyUniquenessCheck() {
var rows = List.<Map<String, Object>>of(
new java.util.HashMap<>(Map.of("APPLYID", "a", "SEALID", "s1")),
new java.util.HashMap<>(Map.of("APPLYID", "a", "SEALID", "s2")));
assertFalse(RowDiff.keyIsUnique(List.of("APPLYID"), rows), "外键列不唯一");
assertTrue(RowDiff.keyIsUnique(List.of("APPLYID", "SEALID"), rows), "复合键唯一");
assertEquals(List.of("APPLYID", "SEALID"), RowDiff.allIdColumns(rows));
}
@org.junit.jupiter.api.Test
void nullOmittedColumnIsEqualInSelfCheck() {
// 引擎对 null 值列在 JSON 中整列省略:左 null + 右缺列 = 语义等值(仅自对拍模式)
var l = List.<Map<String, Object>>of(new java.util.HashMap<>() {{
put("T_ID", "k1"); put("DUEDATE", null);
}});
var r = List.<Map<String, Object>>of(new java.util.HashMap<>(Map.of("T_ID", "k1")));
DiffReport rep = new RowDiff(List.of("T_ID"), true).diff("T", l, r);
assertTrue(rep.isClean());
assertTrue(rep.engineNullOmittedColumns.contains("DUEDATE"));
// 左值非 null 时右缺列仍判负
var l2 = List.<Map<String, Object>>of(new java.util.HashMap<>(Map.of("T_ID", "k1", "DUEDATE", 5L)));
assertFalse(new RowDiff(List.of("T_ID"), true).diff("T", l2, r).isClean());
}
@org.junit.jupiter.api.Test
void booleanDomainDirtyValueExplained() {
// BOOLEAN 列物理值 2(域外脏数据)、引擎归 null:已归因不判负,但计数可见
var l = List.<Map<String, Object>>of(new java.util.HashMap<>(Map.of("T_ID", "k1", "ISX", 2)));
var r = List.<Map<String, Object>>of(new java.util.HashMap<>() {{
put("T_ID", "k1"); put("ISX", null);
}});
DiffReport rep = new RowDiff(List.of("T_ID"), true)
.booleanDomainColumns(java.util.Set.of("ISX")).diff("T", l, r);
assertTrue(rep.isClean());
assertEquals(1, rep.explainedBooleanDomainCells.get("ISX"));
// 未注入布尔列集时仍判负(规则不扩大化)
assertFalse(new RowDiff(List.of("T_ID"), true).diff("T", l, r).isClean());
// 域内值 0/1 与 null 的差异不豁免
var l3 = List.<Map<String, Object>>of(new java.util.HashMap<>(Map.of("T_ID", "k1", "ISX", 1)));
assertFalse(new RowDiff(List.of("T_ID"), true)
.booleanDomainColumns(java.util.Set.of("ISX")).diff("T", l3, r).isClean());
}
@org.junit.jupiter.api.Test
void compositeKeyPairsAssociationRows() {
// 关联表场景:单列键撞车会错位,复合键应 100% 配对
var l = List.<Map<String, Object>>of(
new java.util.HashMap<>(Map.of("APPLYID", "a", "SEALID", "s1", "V", 1)),
new java.util.HashMap<>(Map.of("APPLYID", "a", "SEALID", "s2", "V", 2)));
var r = List.<Map<String, Object>>of(
new java.util.HashMap<>(Map.of("APPLYID", "a", "SEALID", "s2", "V", 2)),
new java.util.HashMap<>(Map.of("APPLYID", "a", "SEALID", "s1", "V", 1)));
DiffReport rep = new RowDiff(List.of("APPLYID", "SEALID"), true).diff("T", l, r);
assertTrue(rep.isClean());
assertEquals(2, rep.matchedRows);
}
} }