diff --git a/platform-parity/src/main/java/com/eip4/platform/parity/diff/DiffReport.java b/platform-parity/src/main/java/com/eip4/platform/parity/diff/DiffReport.java index a6293b1..24d517d 100644 --- a/platform-parity/src/main/java/com/eip4/platform/parity/diff/DiffReport.java +++ b/platform-parity/src/main/java/com/eip4/platform/parity/diff/DiffReport.java @@ -4,6 +4,8 @@ import java.util.ArrayList; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; +import java.util.Set; +import java.util.TreeSet; /** * 一次结果集逐行对拍的结构化差异报告。 @@ -29,6 +31,25 @@ public class DiffReport { /** 主键匹配但有单元格差异的行。 */ public final List mismatches = new ArrayList<>(); + /** + * 引擎侧(右)多出、物理基线侧(左)没有的列名(engine-extra)。 + * + *

EIP3 引擎 listSelect 对真实实体会输出一批"计算列/引用显示翻译"(如 UNITNAME、 + * XXX_DISPLAY__、STATUS 联查、PROJECTUSER、DOCUMENT/LABORTEAMS 等 lateral 产物), + * 物理表并无这些列。对拍比对域限定为"左侧物理列集",故这些列不参与逐列比对、不判负, + * 仅在此整列登记一次,供报告说明。其正确性归"录制重放对旧系统验证",不在本对拍范围内。 + */ + public final Set engineExtraColumns = new TreeSet<>(); + + /** 引擎对 null 值列在 JSON 中整列省略(右缺列+左值 NULL):语义等值,登记列名供报告说明。 */ + public final Set engineNullOmittedColumns = new TreeSet<>(); + + /** + * 已归因差异:BOOLEAN 元数据列、物理值在 {0,1} 域外(旧库脏数据)、引擎归一为 null。 + * 新旧系统引擎行为一致,非移植缺陷;计数可见但不判负。列名→出现次数。 + */ + public final Map explainedBooleanDomainCells = new LinkedHashMap<>(); + public DiffReport(String label) { this.label = label; } @@ -108,6 +129,9 @@ public class DiffReport { m.put("mismatchRows", mismatches.size()); m.put("onlyInLeft", onlyInLeft); m.put("onlyInRight", onlyInRight); + m.put("engineExtraColumns", new ArrayList<>(engineExtraColumns)); + m.put("engineNullOmittedColumns", new ArrayList<>(engineNullOmittedColumns)); + m.put("explainedBooleanDomainCells", new LinkedHashMap<>(explainedBooleanDomainCells)); m.put("clean", isClean()); m.put("consistencyRate", consistencyRate()); List> ms = new ArrayList<>(); @@ -130,6 +154,19 @@ public class DiffReport { .append(" 仅右侧:").append(onlyInRight.size()).append("\n"); sb.append("- 逐行一致率:").append(String.format("%.4f", consistencyRate())) .append(isClean() ? " ✅ 完全一致" : " ⚠️ 存在差异").append("\n"); + if (!engineExtraColumns.isEmpty()) { + sb.append("- 引擎额外列(engine-extra,计算列,不判负):`") + .append(String.join(", ", engineExtraColumns)).append("`\n"); + } + if (!engineNullOmittedColumns.isEmpty()) { + sb.append("- 引擎省略的 null 值列(语义等值): `") + .append(String.join(", ", engineNullOmittedColumns)).append("`\n"); + } + if (!explainedBooleanDomainCells.isEmpty()) { + sb.append("- 已归因差异(BOOLEAN 域外脏数据,引擎归 null,新旧行为一致): "); + explainedBooleanDomainCells.forEach((c, n) -> sb.append('`').append(c).append("`x").append(n).append(' ')); + sb.append('\n'); + } if (!mismatches.isEmpty()) { sb.append("\n差异明细(前 20 行):\n\n"); sb.append("| 主键 | 列 | 左值 | 右值 | 原因 |\n|---|---|---|---|---|\n"); diff --git a/platform-parity/src/main/java/com/eip4/platform/parity/diff/RowDiff.java b/platform-parity/src/main/java/com/eip4/platform/parity/diff/RowDiff.java index 3c854d1..49da14f 100644 Binary files a/platform-parity/src/main/java/com/eip4/platform/parity/diff/RowDiff.java and b/platform-parity/src/main/java/com/eip4/platform/parity/diff/RowDiff.java differ diff --git a/platform-parity/src/main/java/com/eip4/platform/parity/selfcheck/EngineHttpClient.java b/platform-parity/src/main/java/com/eip4/platform/parity/selfcheck/EngineHttpClient.java index cb7d970..da3e542 100644 --- a/platform-parity/src/main/java/com/eip4/platform/parity/selfcheck/EngineHttpClient.java +++ b/platform-parity/src/main/java/com/eip4/platform/parity/selfcheck/EngineHttpClient.java @@ -48,6 +48,55 @@ public class EngineHttpClient { * 抓某实体全量行,附加查询串(C2 数据权限用:{@code asUser=...&roles=...} 构造非 ADMIN ctx, * 触发引擎行级过滤)。{@code extraQuery} 不含前导 {@code &}。 */ + /** + * 单次全量抓取:先取 total,再一次 perPage=total 拉全 —— 消除翻页幻影 + * (引擎默认排序键不唯一时,页间行漂移导致 onlyLeft/onlyRight 成对误判)。 + * total 超过 {@code maxSingleShot} 时退回翻页模式。 + */ + public List> listAllSingleShot(String table, int maxSingleShot) throws Exception { + JsonNode probe = getList(table, 1, 1, ""); + long total = probe.has("total") && probe.get("total").isNumber() ? probe.get("total").asLong() : -1; + if (total < 0 || total > maxSingleShot) { + return listAll(table, ""); + } + if (total == 0) { + return new ArrayList<>(); + } + JsonNode data = getList(table, 1, (int) total, ""); + List> rows = new ArrayList<>((int) total); + JsonNode items = data.get("items"); + if (items != null && items.isArray()) { + for (JsonNode item : items) { + rows.add(jsonToRow(item)); + } + } + return rows; + } + + /** 取前 maxRows 行(翻页,不取全量)。抽样对拍用。 */ + public List> listFirst(String table, int maxRows) throws Exception { + List> rows = new ArrayList<>(); + int page = 1; + while (rows.size() < maxRows) { + JsonNode data = getList(table, page, perPage, ""); + JsonNode items = data.get("items"); + if (items == null || !items.isArray() || items.isEmpty()) { + break; + } + for (JsonNode item : items) { + if (rows.size() >= maxRows) { + break; + } + rows.add(jsonToRow(item)); + } + if (items.size() < perPage) { + break; + } + page++; + } + return rows; + } + public List> listAll(String table, String extraQuery) throws Exception { List> rows = new ArrayList<>(); int page = 1; @@ -85,7 +134,7 @@ public class EngineHttpClient { String url = baseUrl + "/api/entity/" + table + "/list?page=" + page + "&perPage=" + size + (extraQuery == null || extraQuery.isBlank() ? "" : "&" + extraQuery); HttpRequest req = HttpRequest.newBuilder(URI.create(url)) - .timeout(Duration.ofSeconds(60)).GET().build(); + .timeout(Duration.ofSeconds(300)).GET().build(); HttpResponse resp = http.send(req, HttpResponse.BodyHandlers.ofString(StandardCharsets.UTF_8)); if (resp.statusCode() != 200) { throw new IllegalStateException("引擎 list 非 200 (" + resp.statusCode() @@ -107,7 +156,7 @@ public class EngineHttpClient { String url = baseUrl + "/api/entity/" + table + "/list?page=1&perPage=1" + (extraQuery == null || extraQuery.isBlank() ? "" : "&" + extraQuery); HttpRequest req = HttpRequest.newBuilder(URI.create(url)) - .timeout(Duration.ofSeconds(60)).GET().build(); + .timeout(Duration.ofSeconds(300)).GET().build(); HttpResponse resp = http.send(req, HttpResponse.BodyHandlers.ofString(StandardCharsets.UTF_8)); return resp.statusCode(); } diff --git a/platform-parity/src/main/java/com/eip4/platform/parity/selfcheck/ParityCli.java b/platform-parity/src/main/java/com/eip4/platform/parity/selfcheck/ParityCli.java index c5a63ce..30b0761 100644 --- a/platform-parity/src/main/java/com/eip4/platform/parity/selfcheck/ParityCli.java +++ b/platform-parity/src/main/java/com/eip4/platform/parity/selfcheck/ParityCli.java @@ -44,6 +44,8 @@ public final class ParityCli { public static void main(String[] args) throws Exception { Map opt = parseArgs(args); String jdbcUrl = opt.getOrDefault("jdbc", "jdbc:postgresql://localhost:5433/eip4m0"); + int sampleRows = Integer.parseInt(opt.getOrDefault("sampleRows", "0")); + int perPage = Integer.parseInt(opt.getOrDefault("perPage", "500")); String user = opt.getOrDefault("user", "zhengwr"); String pass = opt.getOrDefault("pass", ""); String engine = opt.getOrDefault("engine", "http://localhost:8084"); @@ -78,12 +80,14 @@ public final class ParityCli { entities = listEip4ptEntities(ds); } - SelfCheckRunner runner = new SelfCheckRunner(engine, ds); + SelfCheckRunner runner = new SelfCheckRunner(engine, ds, perPage); List reports = new ArrayList<>(); int cleanCount = 0; long totalRows = 0; for (String e : entities) { - DiffReport r = runner.check(e.trim()); + DiffReport r = sampleRows > 0 + ? runner.checkSample(e.trim(), sampleRows) + : runner.check(e.trim()); reports.add(r); totalRows += r.leftRowCount; boolean clean = r.isClean(); diff --git a/platform-parity/src/main/java/com/eip4/platform/parity/selfcheck/SelfCheckRunner.java b/platform-parity/src/main/java/com/eip4/platform/parity/selfcheck/SelfCheckRunner.java index 807286b..1f790dc 100644 --- a/platform-parity/src/main/java/com/eip4/platform/parity/selfcheck/SelfCheckRunner.java +++ b/platform-parity/src/main/java/com/eip4/platform/parity/selfcheck/SelfCheckRunner.java @@ -23,7 +23,12 @@ public class SelfCheckRunner { private final JdbcResultReader jdbc; public SelfCheckRunner(String engineBaseUrl, DataSource dataSource) { - this.engine = new EngineHttpClient(engineBaseUrl); + this(engineBaseUrl, dataSource, 500); + } + + /** @param perPage 引擎翻页页大小;重查询实体(复杂视图 SQL)用小页降低单请求超时风险。 */ + public SelfCheckRunner(String engineBaseUrl, DataSource dataSource, int perPage) { + this.engine = new EngineHttpClient(engineBaseUrl, perPage); this.jdbc = new JdbcResultReader(dataSource); } @@ -38,16 +43,94 @@ public class SelfCheckRunner { return error(t, "JDBC 直连读失败: " + e.getMessage()); } try { - right = engine.listAll(t); - } catch (Exception e) { - return error(t, "引擎读路径失败: " + e.getMessage()); + right = engine.listAllSingleShot(t, 120_000); + } catch (Exception first) { + // 单次全量对"胖列"实体可能超 Jackson 1GB 字符串上限(如 GB_PRJ_WORKER 照片列), + // 或单条巨型 SQL 超时 —— 回退翻页模式(翻页幻影风险:若结果出现 onlyL/R 成对,按实体留痕) + try { + right = engine.listAll(t); + } catch (Exception e) { + return error(t, "引擎读路径失败(单次:" + first.getMessage() + ";翻页:" + e.getMessage() + ")"); + } } - // 主键:优先 {表名}_ID 约定(EIP3 设计器/引擎一致),回退自动推断 + // 主键:优先 {表名}_ID 约定(EIP3 设计器/引擎一致),回退自动推断; + // 推断键在物理侧不唯一(关联表无自有 PK)时升级为全部 *ID 列复合键,再不唯一退全列配对 String pk = t + "_ID"; List keys = (!left.isEmpty() && left.get(0).containsKey(pk)) ? List.of(pk) : RowDiff.inferKeyColumns(t, left.isEmpty() ? right : left); - return new RowDiff(keys).diff(t, left, right); + if (!left.isEmpty() && !RowDiff.keyIsUnique(keys, left)) { + List composite = RowDiff.allIdColumns(left); + keys = RowDiff.keyIsUnique(composite, left) ? composite : List.of(); + } + // 自对拍模式:引擎计算列(右独有)登记不判负;BOOLEAN 域外脏数据归 null 为已归因差异 + return new RowDiff(keys, true) + .booleanDomainColumns(booleanColumns(t)) + .diff(t, left, right); + } + + /** + * 抽样对拍:引擎侧取前 {@code maxRows} 行(翻页),按主键回查 JDBC 同行集,逐行比对。 + * 用于重查询实体(如 PM_PAYMENTAPPLY:DISTINCT+4 lateral,单页 500 行 ≈61s,全量 ≈78min) + * 在时间盒内做有意义的值级验证;报告 label 注明抽样口径。 + */ + public DiffReport checkSample(String table, int maxRows) { + String t = table.toUpperCase(); + List> right; + try { + right = engine.listFirst(t, maxRows); + } catch (Exception e) { + return error(t, "引擎读路径失败(抽样): " + e.getMessage()); + } + String pk = t + "_ID"; + if (right.isEmpty() || !upper(right.get(0)).containsKey(pk)) { + return error(t, "抽样模式需 {表名}_ID 主键且结果非空"); + } + List pks = new java.util.ArrayList<>(right.size()); + for (Map r : right) { + Object v = upper(r).get(pk); + if (v != null) { + pks.add(v.toString()); + } + } + List> left = new java.util.ArrayList<>(pks.size()); + for (int i = 0; i < pks.size(); i += 1000) { + List chunk = pks.subList(i, Math.min(i + 1000, pks.size())); + String in = String.join(",", java.util.Collections.nCopies(chunk.size(), "?")); + left.addAll(jdbc.query( + "SELECT * FROM " + t + " WHERE " + pk + " IN (" + in + ")", + chunk.toArray())); + } + return new RowDiff(List.of(pk), true) + .booleanDomainColumns(booleanColumns(t)) + .diff(t + "[抽样" + right.size() + "行]", left, right); + } + + private static Map upper(Map row) { + Map m = new java.util.LinkedHashMap<>(); + row.forEach((k, v) -> m.put(k == null ? null : k.toUpperCase(), v)); + return m; + } + + /** 实体的 BOOLEAN 元数据列集(大写);元数据查询失败时返回空集(规则自动失效,不影响对拍)。 */ + private java.util.Set booleanColumns(String table) { + try { + var rows = jdbc.query( + "SELECT a.columnname FROM sys_entityattribute a " + + "JOIN sys_entity e ON a.entityid = e.sys_entity_id " + + "WHERE upper(e.tablename) = ? AND a.referenceid = 'BOOLEAN' AND a.isactive = 1", + table.toUpperCase()); + java.util.Set cols = new java.util.HashSet<>(); + for (var r : rows) { + Object c = r.values().iterator().next(); + if (c != null) { + cols.add(c.toString().trim().toUpperCase()); + } + } + return cols; + } catch (Exception e) { + return java.util.Set.of(); + } } private static DiffReport error(String table, String msg) { diff --git a/platform-parity/src/test/java/com/eip4/platform/parity/diff/RowDiffTest.java b/platform-parity/src/test/java/com/eip4/platform/parity/diff/RowDiffTest.java index da5ee82..b2d85b1 100644 --- a/platform-parity/src/test/java/com/eip4/platform/parity/diff/RowDiffTest.java +++ b/platform-parity/src/test/java/com/eip4/platform/parity/diff/RowDiffTest.java @@ -230,4 +230,78 @@ class RowDiffTest { private static List strip(List in) { return in.stream().map(String::trim).toList(); } + + // ===== 客户库口径修正(2026-06-11):引擎计算列登记 + 复合键升级 ===== + + @org.junit.jupiter.api.Test + void engineExtraColumnAsInfoNotMismatch() { + // 右(引擎)多出计算列 UNITNAME:自对拍模式不判负,整列登记 engineExtraColumns + var l = List.>of(new java.util.HashMap<>(Map.of("T_ID", "k1", "A", 1))); + var r = List.>of(new java.util.HashMap<>(Map.of("T_ID", "k1", "A", 1, "UNITNAME", "山东同益"))); + DiffReport rep = new RowDiff(List.of("T_ID"), true).diff("T", l, r); + assertTrue(rep.isClean(), "计算列不应判负"); + assertEquals(1, rep.matchedRows); + assertTrue(rep.engineExtraColumns.contains("UNITNAME")); + // 录制重放模式(默认)保持原语义:缺列即缺陷 + DiffReport strict = new RowDiff(List.of("T_ID")).diff("T", l, r); + assertFalse(strict.isClean()); + } + + @org.junit.jupiter.api.Test + void keyUniquenessCheck() { + var rows = List.>of( + new java.util.HashMap<>(Map.of("APPLYID", "a", "SEALID", "s1")), + new java.util.HashMap<>(Map.of("APPLYID", "a", "SEALID", "s2"))); + assertFalse(RowDiff.keyIsUnique(List.of("APPLYID"), rows), "外键列不唯一"); + assertTrue(RowDiff.keyIsUnique(List.of("APPLYID", "SEALID"), rows), "复合键唯一"); + assertEquals(List.of("APPLYID", "SEALID"), RowDiff.allIdColumns(rows)); + } + + @org.junit.jupiter.api.Test + void nullOmittedColumnIsEqualInSelfCheck() { + // 引擎对 null 值列在 JSON 中整列省略:左 null + 右缺列 = 语义等值(仅自对拍模式) + var l = List.>of(new java.util.HashMap<>() {{ + put("T_ID", "k1"); put("DUEDATE", null); + }}); + var r = List.>of(new java.util.HashMap<>(Map.of("T_ID", "k1"))); + DiffReport rep = new RowDiff(List.of("T_ID"), true).diff("T", l, r); + assertTrue(rep.isClean()); + assertTrue(rep.engineNullOmittedColumns.contains("DUEDATE")); + // 左值非 null 时右缺列仍判负 + var l2 = List.>of(new java.util.HashMap<>(Map.of("T_ID", "k1", "DUEDATE", 5L))); + assertFalse(new RowDiff(List.of("T_ID"), true).diff("T", l2, r).isClean()); + } + + @org.junit.jupiter.api.Test + void booleanDomainDirtyValueExplained() { + // BOOLEAN 列物理值 2(域外脏数据)、引擎归 null:已归因不判负,但计数可见 + var l = List.>of(new java.util.HashMap<>(Map.of("T_ID", "k1", "ISX", 2))); + var r = List.>of(new java.util.HashMap<>() {{ + put("T_ID", "k1"); put("ISX", null); + }}); + DiffReport rep = new RowDiff(List.of("T_ID"), true) + .booleanDomainColumns(java.util.Set.of("ISX")).diff("T", l, r); + assertTrue(rep.isClean()); + assertEquals(1, rep.explainedBooleanDomainCells.get("ISX")); + // 未注入布尔列集时仍判负(规则不扩大化) + assertFalse(new RowDiff(List.of("T_ID"), true).diff("T", l, r).isClean()); + // 域内值 0/1 与 null 的差异不豁免 + var l3 = List.>of(new java.util.HashMap<>(Map.of("T_ID", "k1", "ISX", 1))); + assertFalse(new RowDiff(List.of("T_ID"), true) + .booleanDomainColumns(java.util.Set.of("ISX")).diff("T", l3, r).isClean()); + } + + @org.junit.jupiter.api.Test + void compositeKeyPairsAssociationRows() { + // 关联表场景:单列键撞车会错位,复合键应 100% 配对 + var l = List.>of( + new java.util.HashMap<>(Map.of("APPLYID", "a", "SEALID", "s1", "V", 1)), + new java.util.HashMap<>(Map.of("APPLYID", "a", "SEALID", "s2", "V", 2))); + var r = List.>of( + new java.util.HashMap<>(Map.of("APPLYID", "a", "SEALID", "s2", "V", 2)), + new java.util.HashMap<>(Map.of("APPLYID", "a", "SEALID", "s1", "V", 1))); + DiffReport rep = new RowDiff(List.of("APPLYID", "SEALID"), true).diff("T", l, r); + assertTrue(rep.isClean()); + assertEquals(2, rep.matchedRows); + } }