From 41afc7ae4c16e2d88f320a890937f94cfbd1c64d Mon Sep 17 00:00:00 2001 From: zhengwr Date: Thu, 11 Jun 2026 11:36:29 +0800 Subject: [PATCH] =?UTF-8?q?feat(parity):=20P1-1=20=E5=85=A8=E5=AE=9E?= =?UTF-8?q?=E4=BD=93=E5=8F=91=E7=8E=B0=E5=BC=8F=E5=AF=B9=E6=8B=8D=E6=A8=A1?= =?UTF-8?q?=E5=BC=8F(--discover)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 枚举 sys_entity 全部活跃实体:≤maxFull(默认2000)行全量逐行,大表抽样 discoverSample(默认300)行; 逐实体 JSONL 进度可断点续跑(--resumeFrom);分桶 OK_FULL/OK_SAMPLE/DIFF/ENGINE_ERROR/JDBC_ERROR; 内存纪律:仅留实体摘要与前 5 个差异格 Co-Authored-By: Claude Fable 5 --- .../platform/parity/selfcheck/ParityCli.java | 117 ++++++++++++++++++ 1 file changed, 117 insertions(+) diff --git a/platform-parity/src/main/java/com/eip4/platform/parity/selfcheck/ParityCli.java b/platform-parity/src/main/java/com/eip4/platform/parity/selfcheck/ParityCli.java index 30b0761..6d1b1f1 100644 --- a/platform-parity/src/main/java/com/eip4/platform/parity/selfcheck/ParityCli.java +++ b/platform-parity/src/main/java/com/eip4/platform/parity/selfcheck/ParityCli.java @@ -55,6 +55,15 @@ public final class ParityCli { SimpleDriverDataSource ds = new SimpleDriverDataSource( new org.postgresql.Driver(), jdbcUrl, user, pass == null ? "" : pass); + // P1-1 全实体发现式对拍:--discover [--maxFull=2000] [--discoverSample=300] [--resumeFrom=进度.jsonl] + if (opt.containsKey("discover")) { + int maxFull = Integer.parseInt(opt.getOrDefault("maxFull", "2000")); + int dSample = Integer.parseInt(opt.getOrDefault("discoverSample", "300")); + int code = runDiscover(engine, ds, outDir, maxFull, dSample, perPage, + opt.get("resumeFrom")); + System.exit(code); + } + // C2 数据行级权限对拍模式:--privilege if (opt.containsKey("privilege")) { int code = runPrivilege(engine, ds, outDir); @@ -305,6 +314,114 @@ public final class ParityCli { return m; } + /** + * P1-1 全实体发现式对拍:枚举 sys_entity 全部活跃实体,行数 ≤maxFull 全量逐行、 + * 否则抽样 sampleRows 行;逐实体进度写 JSONL(可 --resumeFrom 断点续跑), + * 终局汇总分桶:OK_FULL/OK_SAMPLE/DIFF/ENGINE_ERROR/JDBC_ERROR。 + * 内存纪律:只保留每实体摘要(差异明细仅前 5 格),不留全量 mismatch。 + */ + private static int runDiscover(String engine, javax.sql.DataSource ds, Path outDir, + int maxFull, int sampleRows, int perPage, String resumeFrom) throws Exception { + SelfCheckRunner runner = new SelfCheckRunner(engine, ds, perPage); + JdbcResultReader jdbc = new JdbcResultReader(ds); + java.util.Set done = new java.util.HashSet<>(); + if (resumeFrom != null && Files.exists(Path.of(resumeFrom))) { + for (String line : Files.readAllLines(Path.of(resumeFrom))) { + int i = line.indexOf("\"entity\":\""); + if (i >= 0) { + int st = i + 10, en = line.indexOf('"', st); + done.add(line.substring(st, en)); + } + } + System.out.println("[discover] 续跑,已完成 " + done.size() + " 个实体跳过"); + } + List> ents = jdbc.query( + "SELECT trim(tablename) AS t FROM sys_entity WHERE isactive=1 ORDER BY trim(tablename)"); + String ts = new java.text.SimpleDateFormat("yyyyMMdd-HHmmss").format(new java.util.Date()); + Path progress = outDir.resolve("discover-progress-" + ts + ".jsonl"); + Path summaryMd = outDir.resolve("discover-summary-" + ts + ".md"); + Map bucket = new java.util.LinkedHashMap<>(); + List diffLines = new ArrayList<>(); + long totalRows = 0; + int idx = 0; + com.fasterxml.jackson.databind.ObjectMapper om = new com.fasterxml.jackson.databind.ObjectMapper(); + for (Map row : ents) { + String t = String.valueOf(row.values().iterator().next()).trim().toUpperCase(); + idx++; + if (done.contains(t)) { + continue; + } + Map rec = new java.util.LinkedHashMap<>(); + rec.put("entity", t); + String b; + try { + long n; + try { + n = jdbc.count(t); + } catch (Exception ce) { + b = "JDBC_ERROR"; + rec.put("bucket", b); + rec.put("error", String.valueOf(ce.getMessage()).substring(0, + Math.min(200, String.valueOf(ce.getMessage()).length()))); + bucket.merge(b, 1, Integer::sum); + Files.writeString(progress, om.writeValueAsString(rec) + "\n", + java.nio.file.StandardOpenOption.CREATE, java.nio.file.StandardOpenOption.APPEND); + System.out.printf("[%d/%d] %-30s %s%n", idx, ents.size(), t, b); + continue; + } + DiffReport r = n <= maxFull ? runner.check(t) : runner.checkSample(t, sampleRows); + boolean engineErr = !r.mismatches.isEmpty() + && "RUNNER_ERROR".equals(r.mismatches.get(0).key); + if (engineErr) { + b = "ENGINE_ERROR"; + rec.put("error", String.valueOf(r.mismatches.get(0).cells.get(0).right)); + } else if (r.isClean()) { + b = n <= maxFull ? "OK_FULL" : "OK_SAMPLE"; + } else { + b = "DIFF"; + StringBuilder sb = new StringBuilder(); + int c = 0; + for (DiffReport.RowMismatch mm : r.mismatches) { + for (DiffReport.CellDiff cd : mm.cells) { + if (c++ >= 5) break; + sb.append(cd.column).append('[').append(cd.reason).append("] "); + } + if (c >= 5) break; + } + rec.put("diffCols", sb.toString()); + rec.put("onlyL", r.onlyInLeft.size()); + rec.put("onlyR", r.onlyInRight.size()); + diffLines.add(String.format("| %s | %d | %.4f | %s |", + t, r.leftRowCount, r.consistencyRate(), sb)); + } + rec.put("bucket", b); + rec.put("rows", r.leftRowCount); + rec.put("rate", Math.round(r.consistencyRate() * 10000.0) / 10000.0); + totalRows += r.leftRowCount; + } catch (Exception ex) { + b = "ENGINE_ERROR"; + rec.put("bucket", b); + rec.put("error", String.valueOf(ex.getMessage())); + } + bucket.merge(b, 1, Integer::sum); + Files.writeString(progress, om.writeValueAsString(rec) + "\n", + java.nio.file.StandardOpenOption.CREATE, java.nio.file.StandardOpenOption.APPEND); + System.out.printf("[%d/%d] %-30s %s%n", idx, ents.size(), t, b); + } + StringBuilder md = new StringBuilder("# P1-1 全实体发现式对拍汇总\n\n"); + md.append("- 实体总数: ").append(ents.size()).append("\n"); + bucket.forEach((k, v) -> md.append("- ").append(k).append(": ").append(v).append("\n")); + md.append("- 总比对行数(基线侧): ").append(totalRows).append("\n\n"); + if (!diffLines.isEmpty()) { + md.append("## DIFF 实体\n\n| 实体 | 行数 | 一致率 | 差异列(前5) |\n|---|---|---|---|\n"); + diffLines.forEach(l -> md.append(l).append('\n')); + } + Files.writeString(summaryMd, md.toString()); + System.out.println("=== discover 完成 === 进度: " + progress + " 汇总: " + summaryMd); + int diffs = bucket.getOrDefault("DIFF", 0); + return diffs == 0 ? 0 : 1; + } + private static String truncate(String s) { if (s == null) { return "";