diff --git a/platform-parity/pom.xml b/platform-parity/pom.xml new file mode 100644 index 0000000..d1831e5 --- /dev/null +++ b/platform-parity/pom.xml @@ -0,0 +1,22 @@ + + + 4.0.0 + + com.eip4 + eip4-platform + 0.1.0-SNAPSHOT + + platform-parity + platform-parity + C1 逐行对拍框架:旧系统 SQL 录制重放 + 结果集逐行 diff(P1 一级交付物,可 CLI 跑批) + + com.eip4platform-legacy-core + org.springframework.bootspring-boot-starter-jdbc + com.fasterxml.jackson.corejackson-databind + + org.postgresqlpostgresql + org.springframework.bootspring-boot-starter-testtest + + diff --git a/platform-parity/src/main/java/com/eip4/platform/parity/diff/DiffReport.java b/platform-parity/src/main/java/com/eip4/platform/parity/diff/DiffReport.java new file mode 100644 index 0000000..a6293b1 --- /dev/null +++ b/platform-parity/src/main/java/com/eip4/platform/parity/diff/DiffReport.java @@ -0,0 +1,153 @@ +package com.eip4.platform.parity.diff; + +import java.util.ArrayList; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; + +/** + * 一次结果集逐行对拍的结构化差异报告。 + * + *

同时承载机器可读(JSON 序列化)与人读(Markdown 摘要)两种产物: + * {@link #toSummaryMap()} 给 JSON,{@link #toMarkdown()} 给报告附录。 + */ +public class DiffReport { + + /** 对拍标签(通常是实体/表名)。 */ + public final String label; + /** 左侧(基线,旧系统/录制)行数。 */ + public int leftRowCount; + /** 右侧(新引擎)行数。 */ + public int rightRowCount; + /** 用于排序与配对的主键列(规范化后)。 */ + public final List keyColumns = new ArrayList<>(); + + /** 仅左侧存在的行(右侧缺失),记录主键。 */ + public final List onlyInLeft = new ArrayList<>(); + /** 仅右侧存在的行(左侧缺失),记录主键。 */ + public final List onlyInRight = new ArrayList<>(); + /** 主键匹配但有单元格差异的行。 */ + public final List mismatches = new ArrayList<>(); + + public DiffReport(String label) { + this.label = label; + } + + /** 主键匹配、单元格全等的行数。 */ + public int matchedRows; + + public boolean isClean() { + return onlyInLeft.isEmpty() && onlyInRight.isEmpty() && mismatches.isEmpty(); + } + + /** 总比对行数(以两侧并集计)。 */ + public int totalCompared() { + return matchedRows + mismatches.size() + onlyInLeft.size() + onlyInRight.size(); + } + + /** 一致率(matched / 并集),用于报告里"逐行一致率"。 */ + public double consistencyRate() { + int total = totalCompared(); + return total == 0 ? 1.0 : (double) matchedRows / total; + } + + /** 单元格级差异。 */ + public static class CellDiff { + public final String column; + public final Object left; + public final Object right; + /** 差异原因:VALUE(值不同)/ MISSING_LEFT(左缺列)/ MISSING_RIGHT(右缺列)。 */ + public final String reason; + + public CellDiff(String column, Object left, Object right, String reason) { + this.column = column; + this.left = left; + this.right = right; + this.reason = reason; + } + + public Map toMap() { + Map m = new LinkedHashMap<>(); + m.put("column", column); + m.put("left", left); + m.put("right", right); + m.put("reason", reason); + return m; + } + } + + /** 主键匹配但内容有差异的一行。 */ + public static class RowMismatch { + public final String key; + public final List cells = new ArrayList<>(); + + public RowMismatch(String key) { + this.key = key; + } + + public Map toMap() { + Map m = new LinkedHashMap<>(); + m.put("key", key); + List> cs = new ArrayList<>(); + for (CellDiff c : cells) { + cs.add(c.toMap()); + } + m.put("cells", cs); + return m; + } + } + + /** 机器可读摘要(JSON 序列化用)。 */ + public Map toSummaryMap() { + Map m = new LinkedHashMap<>(); + m.put("label", label); + m.put("keyColumns", keyColumns); + m.put("leftRowCount", leftRowCount); + m.put("rightRowCount", rightRowCount); + m.put("matchedRows", matchedRows); + m.put("mismatchRows", mismatches.size()); + m.put("onlyInLeft", onlyInLeft); + m.put("onlyInRight", onlyInRight); + m.put("clean", isClean()); + m.put("consistencyRate", consistencyRate()); + List> ms = new ArrayList<>(); + for (RowMismatch r : mismatches) { + ms.add(r.toMap()); + } + m.put("mismatches", ms); + return m; + } + + /** 人读 Markdown 摘要(进 C1 报告附录)。 */ + public String toMarkdown() { + StringBuilder sb = new StringBuilder(); + sb.append("### 对拍:").append(label).append("\n\n"); + sb.append("- 主键列:`").append(String.join(",", keyColumns)).append("`\n"); + sb.append("- 左/右行数:").append(leftRowCount).append(" / ").append(rightRowCount).append("\n"); + sb.append("- 匹配一致行:").append(matchedRows).append("\n"); + sb.append("- 内容差异行:").append(mismatches.size()).append("\n"); + sb.append("- 仅左侧:").append(onlyInLeft.size()) + .append(" 仅右侧:").append(onlyInRight.size()).append("\n"); + sb.append("- 逐行一致率:").append(String.format("%.4f", consistencyRate())) + .append(isClean() ? " ✅ 完全一致" : " ⚠️ 存在差异").append("\n"); + if (!mismatches.isEmpty()) { + sb.append("\n差异明细(前 20 行):\n\n"); + sb.append("| 主键 | 列 | 左值 | 右值 | 原因 |\n|---|---|---|---|---|\n"); + int shown = 0; + for (RowMismatch r : mismatches) { + for (CellDiff c : r.cells) { + if (shown++ >= 20) { + break; + } + sb.append("| ").append(r.key).append(" | ").append(c.column) + .append(" | ").append(c.left).append(" | ").append(c.right) + .append(" | ").append(c.reason).append(" |\n"); + } + if (shown >= 20) { + break; + } + } + } + return sb.toString(); + } +} diff --git a/platform-parity/src/main/java/com/eip4/platform/parity/diff/Normalizer.java b/platform-parity/src/main/java/com/eip4/platform/parity/diff/Normalizer.java new file mode 100644 index 0000000..3e219a7 --- /dev/null +++ b/platform-parity/src/main/java/com/eip4/platform/parity/diff/Normalizer.java @@ -0,0 +1,165 @@ +package com.eip4.platform.parity.diff; + +import java.math.BigDecimal; +import java.util.Objects; + +/** + * 单值规范化器 —— 把"语义等值但表示不同"的两个值归一到可直接 equals 的规范形式。 + * + *

归一化规则全部源自保真层引擎与 PG 物理类型口径(见 {@code Entity.createTableSql} + * 与《C1实测报告.md》归一化规则表),逐条对应一类"旧/新两侧表示差异但语义一致"的场景: + * + *

    + *
  1. char(36) 尾部空格:PG {@code character(n)} 定长列读出会补空格(id/外键/CREATEBY 等), + * 两侧驱动/字段长度差异会带来不同尾空格 → 统一 {@code stripTrailing()}。
  2. + *
  3. 数值 scale 归一:{@code numeric(18,6)} 读出 {@code 2.000000} 与 amis/JDBC 读出 {@code 2} + * 语义等值 → 统一用 {@link BigDecimal#compareTo} 而非 {@code equals}(后者比较 scale)。
  4. + *
  5. 整数跨类型:同一整数列在两侧可能是 Integer/Long/Short/BigInteger → 统一比较 long 值。
  6. + *
  7. 布尔/整型混用:BOOLEAN 物理列是 integer(0/1),一侧 Boolean 一侧 Integer → 归一到 0/1。
  8. + *
  9. bigint 时间戳原样:DATE/TIME 物理列是 epoch 毫秒 bigint,按整数原样比(不做时区/格式解释)。
  10. + *
  11. NULL 区分缺列:本类只负责单值;NULL-vs-缺列的区分在 {@link RowDiff} 行级处理。
  12. + *
+ * + * 设计为无状态纯函数,便于单测覆盖每条规则。 + */ +public final class Normalizer { + + private Normalizer() { + } + + /** 归一化的规范形式标记:用于报告里描述某值被按哪条规则处理。 */ + public enum Kind { + NULL, STRING, INTEGRAL, DECIMAL, BOOL_AS_INT, OTHER + } + + /** + * 把单值归一到规范对象: + * + */ + public static Object canonical(Object v) { + if (v == null) { + return null; + } + if (v instanceof Boolean b) { + return b ? 1L : 0L; + } + if (v instanceof Byte || v instanceof Short || v instanceof Integer || v instanceof Long) { + return ((Number) v).longValue(); + } + if (v instanceof java.math.BigInteger bi) { + return bi.longValueExact(); + } + if (v instanceof BigDecimal bd) { + return stripBd(bd); + } + if (v instanceof Float || v instanceof Double) { + return stripBd(BigDecimal.valueOf(((Number) v).doubleValue())); + } + if (v instanceof CharSequence || v instanceof Character) { + return stripTrailing(v.toString()); + } + // 兜底:任意对象按字符串去尾空格(覆盖 java.sql.Timestamp/joda 等) + return stripTrailing(v.toString()); + } + + /** 两值是否语义等值(经规范化后比较)。NULL == NULL 为 true(缺列区分在行级)。 */ + public static boolean equal(Object a, Object b) { + Object ca = canonical(a); + Object cb = canonical(b); + if (ca == null || cb == null) { + return ca == null && cb == null; + } + // 数值跨族比较:Long 与 BigDecimal 之间也应等值(2 vs 2.000000) + if (ca instanceof Long la && cb instanceof BigDecimal db) { + return new BigDecimal(la).compareTo(db) == 0; + } + if (ca instanceof BigDecimal da && cb instanceof Long lb) { + return da.compareTo(new BigDecimal(lb)) == 0; + } + if (ca instanceof BigDecimal da && cb instanceof BigDecimal db) { + return da.compareTo(db) == 0; + } + // 规则5b:DATE/TIME 物理列是 bigint epoch 毫秒,但引擎 REST 侧经 JodaModule 序列化为 + // ISO-8601 字符串(如 1970-01-01T00:00:00.000Z)。一侧 epoch(Long) 一侧 ISO 串时, + // 解析 ISO 为 epoch 毫秒后比较,使"同一时刻两种表示"语义等值。 + Long ea = epochOf(ca); + Long eb = epochOf(cb); + if (ea != null && eb != null) { + return ea.longValue() == eb.longValue(); + } + return Objects.equals(ca, cb); + } + + /** + * 把"已规范化的值"尝试解读为 epoch 毫秒: + * Long 原样返回;ISO-8601 instant 字符串(带 Z)解析为毫秒;否则 null(不可比为时间)。 + * 仅当对端也是时间表示时才生效,普通字符串/整数不会被误判。 + */ + private static Long epochOf(Object canonical) { + if (canonical instanceof Long l) { + return l; + } + if (canonical instanceof String s && looksLikeIsoInstant(s)) { + try { + return java.time.Instant.parse(s).toEpochMilli(); + } catch (RuntimeException ignore) { + return null; + } + } + return null; + } + + private static boolean looksLikeIsoInstant(String s) { + // 形如 1970-01-01T00:00:00.000Z;要求含 'T' 与结尾 'Z',避免把普通文本当时间 + return s.length() >= 16 && s.indexOf('T') == 10 && s.endsWith("Z"); + } + + /** 分类标记(报告/调试用)。 */ + public static Kind kindOf(Object v) { + if (v == null) { + return Kind.NULL; + } + if (v instanceof Boolean) { + return Kind.BOOL_AS_INT; + } + if (v instanceof Byte || v instanceof Short || v instanceof Integer + || v instanceof Long || v instanceof java.math.BigInteger) { + return Kind.INTEGRAL; + } + if (v instanceof BigDecimal || v instanceof Float || v instanceof Double) { + return Kind.DECIMAL; + } + if (v instanceof CharSequence || v instanceof Character) { + return Kind.STRING; + } + return Kind.OTHER; + } + + /** char(n) 定长列尾部空格归一(只去尾部,保留中间与前导)。 */ + public static String stripTrailing(String s) { + if (s == null) { + return null; + } + int end = s.length(); + while (end > 0 && s.charAt(end - 1) == ' ') { + end--; + } + return s.substring(0, end); + } + + private static BigDecimal stripBd(BigDecimal bd) { + BigDecimal stripped = bd.stripTrailingZeros(); + // stripTrailingZeros 对 0 会给出 0E-n,规范成 0 比例,避免 scale 噪声 + if (stripped.compareTo(BigDecimal.ZERO) == 0) { + return BigDecimal.ZERO; + } + return stripped; + } +} diff --git a/platform-parity/src/main/java/com/eip4/platform/parity/diff/RowDiff.java b/platform-parity/src/main/java/com/eip4/platform/parity/diff/RowDiff.java new file mode 100644 index 0000000..3c854d1 Binary files /dev/null and b/platform-parity/src/main/java/com/eip4/platform/parity/diff/RowDiff.java differ diff --git a/platform-parity/src/test/java/com/eip4/platform/parity/diff/RowDiffTest.java b/platform-parity/src/test/java/com/eip4/platform/parity/diff/RowDiffTest.java new file mode 100644 index 0000000..da5ee82 --- /dev/null +++ b/platform-parity/src/test/java/com/eip4/platform/parity/diff/RowDiffTest.java @@ -0,0 +1,233 @@ +package com.eip4.platform.parity.diff; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.math.BigDecimal; +import java.util.HashMap; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; + +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; + +/** + * RowDiff / Normalizer 单测:覆盖全部归一化规则与差异检出路径。 + * 每个用例对应 C1 报告归一化规则表里的一条。 + */ +class RowDiffTest { + + // ---- 单值归一化(Normalizer)---- + + @Test + @DisplayName("规则1:char(36) 尾部空格归一 —— 'abc ' 等值 'abc'") + void charTrailingSpace() { + assertTrue(Normalizer.equal("abc", "abc ")); + assertTrue(Normalizer.equal("ADMIN" + " ".repeat(31), "ADMIN")); + // 前导/中间空格不归一(保真:不该被误判等值) + assertFalse(Normalizer.equal(" abc", "abc")); + assertFalse(Normalizer.equal("a bc", "abc")); + } + + @Test + @DisplayName("规则2:numeric scale 归一 —— 2 等值 2.000000") + void decimalScale() { + assertTrue(Normalizer.equal(new BigDecimal("2"), new BigDecimal("2.000000"))); + assertTrue(Normalizer.equal(new BigDecimal("0"), new BigDecimal("0.0000"))); + assertTrue(Normalizer.equal(new BigDecimal("18.500000"), new BigDecimal("18.5"))); + assertFalse(Normalizer.equal(new BigDecimal("2.001"), new BigDecimal("2.000"))); + } + + @Test + @DisplayName("规则3:整数跨类型 —— Integer 等值 Long 等值 Short") + void integralCrossType() { + assertTrue(Normalizer.equal(5, 5L)); + assertTrue(Normalizer.equal((short) 7, 7)); + assertTrue(Normalizer.equal(Integer.valueOf(100), Long.valueOf(100))); + assertFalse(Normalizer.equal(5, 6L)); + } + + @Test + @DisplayName("规则3b:整数与等值小数互通 —— 2(Long) 等值 2.000000(BigDecimal)") + void integralVsDecimal() { + assertTrue(Normalizer.equal(2, new BigDecimal("2.000000"))); + assertTrue(Normalizer.equal(new BigDecimal("2.000000"), 2L)); + assertFalse(Normalizer.equal(2, new BigDecimal("2.5"))); + } + + @Test + @DisplayName("规则4:布尔与整型混用 —— BOOLEAN 列 true/1、false/0 归一") + void booleanAsInt() { + assertTrue(Normalizer.equal(Boolean.TRUE, 1)); + assertTrue(Normalizer.equal(Boolean.FALSE, 0L)); + assertTrue(Normalizer.equal(true, new BigDecimal("1"))); + assertFalse(Normalizer.equal(Boolean.TRUE, 0)); + } + + @Test + @DisplayName("规则5:bigint 时间戳原样比 —— epoch 毫秒按整数比,不做时区解释") + void bigintTimestamp() { + long t = 1717999999000L; + assertTrue(Normalizer.equal(t, Long.valueOf(t))); + assertFalse(Normalizer.equal(t, t + 1)); + } + + @Test + @DisplayName("规则5b:DATE/TIME ISO 串 与 bigint epoch 等值 —— 引擎 REST vs 直连 JDBC") + void isoInstantVsEpoch() { + // 引擎 REST 经 JodaModule 序列化 epoch 0 为 ISO;JDBC 直连返回 bigint 0 + assertTrue(Normalizer.equal("1970-01-01T00:00:00.000Z", 0L)); + assertTrue(Normalizer.equal(0, "1970-01-01T00:00:00.000Z")); + long t = 1717999999000L; + assertTrue(Normalizer.equal(java.time.Instant.ofEpochMilli(t).toString(), t)); + // 普通整数串不会被误判为时间 + assertFalse(Normalizer.equal("12345", 0L)); + // 不同时刻不等值 + assertFalse(Normalizer.equal("1970-01-01T00:00:01.000Z", 0L)); + } + + @Test + @DisplayName("规则6:NULL 与 NULL 等值,NULL 与非空不等值") + void nullSemantics() { + assertTrue(Normalizer.equal(null, null)); + assertFalse(Normalizer.equal(null, "x")); + assertFalse(Normalizer.equal("x", null)); + assertFalse(Normalizer.equal(null, 0)); + } + + @Test + @DisplayName("Normalizer.kindOf 分类正确") + void kindClassification() { + assertEquals(Normalizer.Kind.NULL, Normalizer.kindOf(null)); + assertEquals(Normalizer.Kind.STRING, Normalizer.kindOf("a")); + assertEquals(Normalizer.Kind.INTEGRAL, Normalizer.kindOf(3L)); + assertEquals(Normalizer.Kind.DECIMAL, Normalizer.kindOf(new BigDecimal("1.5"))); + assertEquals(Normalizer.Kind.BOOL_AS_INT, Normalizer.kindOf(true)); + } + + // ---- 行级 diff(RowDiff)---- + + private static Map row(Object... kv) { + Map m = new LinkedHashMap<>(); + for (int i = 0; i < kv.length; i += 2) { + m.put((String) kv[i], kv[i + 1]); + } + return m; + } + + @Test + @DisplayName("行序无关:左右行序相反但内容相同 → 完全一致") + void orderIndependent() { + List> left = List.of( + row("DEMO_ID", "id-1", "NAME", "甲"), + row("DEMO_ID", "id-2", "NAME", "乙")); + List> right = List.of( + row("DEMO_ID", "id-2", "NAME", "乙"), + row("DEMO_ID", "id-1", "NAME", "甲")); + DiffReport r = new RowDiff(List.of("DEMO_ID")).diff("DEMO", left, right); + assertTrue(r.isClean()); + assertEquals(2, r.matchedRows); + assertEquals(1.0, r.consistencyRate()); + } + + @Test + @DisplayName("综合归一:char 尾空格 + numeric scale + 整数/长整 在同一行被吸收 → 一致") + void mixedNormalizationRow() { + List> left = List.of( + row("T_ID", "k1 ", "AMOUNT", new BigDecimal("100.000000"), "QTY", 3)); + List> right = List.of( + row("T_ID", "k1", "AMOUNT", new BigDecimal("100"), "QTY", 3L)); + DiffReport r = new RowDiff(List.of("T_ID")).diff("T", left, right); + assertTrue(r.isClean(), () -> r.toMarkdown()); + } + + @Test + @DisplayName("值差异检出:同主键单元格值不同 → 报 VALUE 差异") + void valueMismatchDetected() { + List> left = List.of(row("T_ID", "k1", "NAME", "原值")); + List> right = List.of(row("T_ID", "k1", "NAME", "改值")); + DiffReport r = new RowDiff(List.of("T_ID")).diff("T", left, right); + assertFalse(r.isClean()); + assertEquals(1, r.mismatches.size()); + DiffReport.CellDiff c = r.mismatches.get(0).cells.get(0); + assertEquals("NAME", c.column); + assertEquals("VALUE", c.reason); + } + + @Test + @DisplayName("缺行检出:左多一行/右多一行 → onlyInLeft / onlyInRight") + void rowPresenceMismatch() { + List> left = List.of( + row("T_ID", "k1", "NAME", "a"), + row("T_ID", "k2", "NAME", "b")); + List> right = List.of( + row("T_ID", "k1", "NAME", "a"), + row("T_ID", "k3", "NAME", "c")); + DiffReport r = new RowDiff(List.of("T_ID")).diff("T", left, right); + assertEquals(1, r.matchedRows); + assertEquals(List.of("k2"), strip(r.onlyInLeft)); + assertEquals(List.of("k3"), strip(r.onlyInRight)); + assertFalse(r.isClean()); + } + + @Test + @DisplayName("NULL 区分缺列:键存在值NULL vs 键不存在 → 不同 reason") + void nullVsMissingColumn() { + // 左有 EXTRA 列值 null,右没有 EXTRA 键 → MISSING_RIGHT + Map l = new HashMap<>(); + l.put("T_ID", "k1"); + l.put("EXTRA", null); + Map rr = new HashMap<>(); + rr.put("T_ID", "k1"); + DiffReport r = new RowDiff(List.of("T_ID")).diff("T", List.of(l), List.of(rr)); + assertEquals(1, r.mismatches.size()); + assertEquals("MISSING_RIGHT", r.mismatches.get(0).cells.get(0).reason); + + // 两侧都键存在且值 null → 一致 + Map l2 = new HashMap<>(); + l2.put("T_ID", "k1"); + l2.put("EXTRA", null); + Map r2 = new HashMap<>(); + r2.put("T_ID", "k1"); + r2.put("EXTRA", null); + DiffReport ok = new RowDiff(List.of("T_ID")).diff("T", List.of(l2), List.of(r2)); + assertTrue(ok.isClean()); + } + + @Test + @DisplayName("布尔/整型在行级吸收:ISACTIVE true vs 1 → 一致") + void booleanRowLevel() { + List> left = List.of(row("T_ID", "k1", "ISACTIVE", true)); + List> right = List.of(row("T_ID", "k1", "ISACTIVE", 1)); + DiffReport r = new RowDiff(List.of("T_ID")).diff("T", left, right); + assertTrue(r.isClean()); + } + + @Test + @DisplayName("主键自动推断:{表名}_ID 优先,其次 _id") + void inferKey() { + List> s1 = List.of(row("DEMO_ID", "x", "NAME", "n")); + assertEquals(List.of("DEMO_ID"), RowDiff.inferKeyColumns("DEMO", s1)); + List> s2 = List.of(row("_id", "x", "NAME", "n")); + assertEquals(List.of("_id"), RowDiff.inferKeyColumns("DEMO", s2)); + } + + @Test + @DisplayName("报告序列化:toSummaryMap / toMarkdown 含关键字段") + void reportShape() { + List> left = List.of(row("T_ID", "k1", "NAME", "a")); + List> right = List.of(row("T_ID", "k1", "NAME", "b")); + DiffReport r = new RowDiff(List.of("T_ID")).diff("T", left, right); + Map m = r.toSummaryMap(); + assertEquals("T", m.get("label")); + assertEquals(false, m.get("clean")); + assertEquals(1, m.get("mismatchRows")); + assertTrue(r.toMarkdown().contains("逐行一致率")); + } + + private static List strip(List in) { + return in.stream().map(String::trim).toList(); + } +}