feat(parity): P1-1 全实体发现式对拍模式(--discover)

枚举 sys_entity 全部活跃实体:≤maxFull(默认2000)行全量逐行,大表抽样 discoverSample(默认300)行;
逐实体 JSONL 进度可断点续跑(--resumeFrom);分桶 OK_FULL/OK_SAMPLE/DIFF/ENGINE_ERROR/JDBC_ERROR;
内存纪律:仅留实体摘要与前 5 个差异格

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
2026-06-11 11:36:29 +08:00
parent c2b3c5004f
commit 41afc7ae4c

View File

@@ -55,6 +55,15 @@ public final class ParityCli {
SimpleDriverDataSource ds = new SimpleDriverDataSource( SimpleDriverDataSource ds = new SimpleDriverDataSource(
new org.postgresql.Driver(), jdbcUrl, user, pass == null ? "" : pass); new org.postgresql.Driver(), jdbcUrl, user, pass == null ? "" : pass);
// P1-1 全实体发现式对拍:--discover [--maxFull=2000] [--discoverSample=300] [--resumeFrom=进度.jsonl]
if (opt.containsKey("discover")) {
int maxFull = Integer.parseInt(opt.getOrDefault("maxFull", "2000"));
int dSample = Integer.parseInt(opt.getOrDefault("discoverSample", "300"));
int code = runDiscover(engine, ds, outDir, maxFull, dSample, perPage,
opt.get("resumeFrom"));
System.exit(code);
}
// C2 数据行级权限对拍模式:--privilege // C2 数据行级权限对拍模式:--privilege
if (opt.containsKey("privilege")) { if (opt.containsKey("privilege")) {
int code = runPrivilege(engine, ds, outDir); int code = runPrivilege(engine, ds, outDir);
@@ -305,6 +314,114 @@ public final class ParityCli {
return m; return m;
} }
/**
* P1-1 全实体发现式对拍:枚举 sys_entity 全部活跃实体,行数 ≤maxFull 全量逐行、
* 否则抽样 sampleRows 行;逐实体进度写 JSONL(可 --resumeFrom 断点续跑),
* 终局汇总分桶:OK_FULL/OK_SAMPLE/DIFF/ENGINE_ERROR/JDBC_ERROR。
* 内存纪律:只保留每实体摘要(差异明细仅前 5 格),不留全量 mismatch。
*/
private static int runDiscover(String engine, javax.sql.DataSource ds, Path outDir,
int maxFull, int sampleRows, int perPage, String resumeFrom) throws Exception {
SelfCheckRunner runner = new SelfCheckRunner(engine, ds, perPage);
JdbcResultReader jdbc = new JdbcResultReader(ds);
java.util.Set<String> done = new java.util.HashSet<>();
if (resumeFrom != null && Files.exists(Path.of(resumeFrom))) {
for (String line : Files.readAllLines(Path.of(resumeFrom))) {
int i = line.indexOf("\"entity\":\"");
if (i >= 0) {
int st = i + 10, en = line.indexOf('"', st);
done.add(line.substring(st, en));
}
}
System.out.println("[discover] 续跑,已完成 " + done.size() + " 个实体跳过");
}
List<Map<String, Object>> ents = jdbc.query(
"SELECT trim(tablename) AS t FROM sys_entity WHERE isactive=1 ORDER BY trim(tablename)");
String ts = new java.text.SimpleDateFormat("yyyyMMdd-HHmmss").format(new java.util.Date());
Path progress = outDir.resolve("discover-progress-" + ts + ".jsonl");
Path summaryMd = outDir.resolve("discover-summary-" + ts + ".md");
Map<String, Integer> bucket = new java.util.LinkedHashMap<>();
List<String> diffLines = new ArrayList<>();
long totalRows = 0;
int idx = 0;
com.fasterxml.jackson.databind.ObjectMapper om = new com.fasterxml.jackson.databind.ObjectMapper();
for (Map<String, Object> row : ents) {
String t = String.valueOf(row.values().iterator().next()).trim().toUpperCase();
idx++;
if (done.contains(t)) {
continue;
}
Map<String, Object> rec = new java.util.LinkedHashMap<>();
rec.put("entity", t);
String b;
try {
long n;
try {
n = jdbc.count(t);
} catch (Exception ce) {
b = "JDBC_ERROR";
rec.put("bucket", b);
rec.put("error", String.valueOf(ce.getMessage()).substring(0,
Math.min(200, String.valueOf(ce.getMessage()).length())));
bucket.merge(b, 1, Integer::sum);
Files.writeString(progress, om.writeValueAsString(rec) + "\n",
java.nio.file.StandardOpenOption.CREATE, java.nio.file.StandardOpenOption.APPEND);
System.out.printf("[%d/%d] %-30s %s%n", idx, ents.size(), t, b);
continue;
}
DiffReport r = n <= maxFull ? runner.check(t) : runner.checkSample(t, sampleRows);
boolean engineErr = !r.mismatches.isEmpty()
&& "RUNNER_ERROR".equals(r.mismatches.get(0).key);
if (engineErr) {
b = "ENGINE_ERROR";
rec.put("error", String.valueOf(r.mismatches.get(0).cells.get(0).right));
} else if (r.isClean()) {
b = n <= maxFull ? "OK_FULL" : "OK_SAMPLE";
} else {
b = "DIFF";
StringBuilder sb = new StringBuilder();
int c = 0;
for (DiffReport.RowMismatch mm : r.mismatches) {
for (DiffReport.CellDiff cd : mm.cells) {
if (c++ >= 5) break;
sb.append(cd.column).append('[').append(cd.reason).append("] ");
}
if (c >= 5) break;
}
rec.put("diffCols", sb.toString());
rec.put("onlyL", r.onlyInLeft.size());
rec.put("onlyR", r.onlyInRight.size());
diffLines.add(String.format("| %s | %d | %.4f | %s |",
t, r.leftRowCount, r.consistencyRate(), sb));
}
rec.put("bucket", b);
rec.put("rows", r.leftRowCount);
rec.put("rate", Math.round(r.consistencyRate() * 10000.0) / 10000.0);
totalRows += r.leftRowCount;
} catch (Exception ex) {
b = "ENGINE_ERROR";
rec.put("bucket", b);
rec.put("error", String.valueOf(ex.getMessage()));
}
bucket.merge(b, 1, Integer::sum);
Files.writeString(progress, om.writeValueAsString(rec) + "\n",
java.nio.file.StandardOpenOption.CREATE, java.nio.file.StandardOpenOption.APPEND);
System.out.printf("[%d/%d] %-30s %s%n", idx, ents.size(), t, b);
}
StringBuilder md = new StringBuilder("# P1-1 全实体发现式对拍汇总\n\n");
md.append("- 实体总数: ").append(ents.size()).append("\n");
bucket.forEach((k, v) -> md.append("- ").append(k).append(": ").append(v).append("\n"));
md.append("- 总比对行数(基线侧): ").append(totalRows).append("\n\n");
if (!diffLines.isEmpty()) {
md.append("## DIFF 实体\n\n| 实体 | 行数 | 一致率 | 差异列(前5) |\n|---|---|---|---|\n");
diffLines.forEach(l -> md.append(l).append('\n'));
}
Files.writeString(summaryMd, md.toString());
System.out.println("=== discover 完成 === 进度: " + progress + " 汇总: " + summaryMd);
int diffs = bucket.getOrDefault("DIFF", 0);
return diffs == 0 ? 0 : 1;
}
private static String truncate(String s) { private static String truncate(String s) {
if (s == null) { if (s == null) {
return ""; return "";