feat(parity): P1-1 全实体发现式对拍模式(--discover)
枚举 sys_entity 全部活跃实体:≤maxFull(默认2000)行全量逐行,大表抽样 discoverSample(默认300)行; 逐实体 JSONL 进度可断点续跑(--resumeFrom);分桶 OK_FULL/OK_SAMPLE/DIFF/ENGINE_ERROR/JDBC_ERROR; 内存纪律:仅留实体摘要与前 5 个差异格 Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
@@ -55,6 +55,15 @@ public final class ParityCli {
|
||||
SimpleDriverDataSource ds = new SimpleDriverDataSource(
|
||||
new org.postgresql.Driver(), jdbcUrl, user, pass == null ? "" : pass);
|
||||
|
||||
// P1-1 全实体发现式对拍:--discover [--maxFull=2000] [--discoverSample=300] [--resumeFrom=进度.jsonl]
|
||||
if (opt.containsKey("discover")) {
|
||||
int maxFull = Integer.parseInt(opt.getOrDefault("maxFull", "2000"));
|
||||
int dSample = Integer.parseInt(opt.getOrDefault("discoverSample", "300"));
|
||||
int code = runDiscover(engine, ds, outDir, maxFull, dSample, perPage,
|
||||
opt.get("resumeFrom"));
|
||||
System.exit(code);
|
||||
}
|
||||
|
||||
// C2 数据行级权限对拍模式:--privilege
|
||||
if (opt.containsKey("privilege")) {
|
||||
int code = runPrivilege(engine, ds, outDir);
|
||||
@@ -305,6 +314,114 @@ public final class ParityCli {
|
||||
return m;
|
||||
}
|
||||
|
||||
/**
|
||||
* P1-1 全实体发现式对拍:枚举 sys_entity 全部活跃实体,行数 ≤maxFull 全量逐行、
|
||||
* 否则抽样 sampleRows 行;逐实体进度写 JSONL(可 --resumeFrom 断点续跑),
|
||||
* 终局汇总分桶:OK_FULL/OK_SAMPLE/DIFF/ENGINE_ERROR/JDBC_ERROR。
|
||||
* 内存纪律:只保留每实体摘要(差异明细仅前 5 格),不留全量 mismatch。
|
||||
*/
|
||||
private static int runDiscover(String engine, javax.sql.DataSource ds, Path outDir,
|
||||
int maxFull, int sampleRows, int perPage, String resumeFrom) throws Exception {
|
||||
SelfCheckRunner runner = new SelfCheckRunner(engine, ds, perPage);
|
||||
JdbcResultReader jdbc = new JdbcResultReader(ds);
|
||||
java.util.Set<String> done = new java.util.HashSet<>();
|
||||
if (resumeFrom != null && Files.exists(Path.of(resumeFrom))) {
|
||||
for (String line : Files.readAllLines(Path.of(resumeFrom))) {
|
||||
int i = line.indexOf("\"entity\":\"");
|
||||
if (i >= 0) {
|
||||
int st = i + 10, en = line.indexOf('"', st);
|
||||
done.add(line.substring(st, en));
|
||||
}
|
||||
}
|
||||
System.out.println("[discover] 续跑,已完成 " + done.size() + " 个实体跳过");
|
||||
}
|
||||
List<Map<String, Object>> ents = jdbc.query(
|
||||
"SELECT trim(tablename) AS t FROM sys_entity WHERE isactive=1 ORDER BY trim(tablename)");
|
||||
String ts = new java.text.SimpleDateFormat("yyyyMMdd-HHmmss").format(new java.util.Date());
|
||||
Path progress = outDir.resolve("discover-progress-" + ts + ".jsonl");
|
||||
Path summaryMd = outDir.resolve("discover-summary-" + ts + ".md");
|
||||
Map<String, Integer> bucket = new java.util.LinkedHashMap<>();
|
||||
List<String> diffLines = new ArrayList<>();
|
||||
long totalRows = 0;
|
||||
int idx = 0;
|
||||
com.fasterxml.jackson.databind.ObjectMapper om = new com.fasterxml.jackson.databind.ObjectMapper();
|
||||
for (Map<String, Object> row : ents) {
|
||||
String t = String.valueOf(row.values().iterator().next()).trim().toUpperCase();
|
||||
idx++;
|
||||
if (done.contains(t)) {
|
||||
continue;
|
||||
}
|
||||
Map<String, Object> rec = new java.util.LinkedHashMap<>();
|
||||
rec.put("entity", t);
|
||||
String b;
|
||||
try {
|
||||
long n;
|
||||
try {
|
||||
n = jdbc.count(t);
|
||||
} catch (Exception ce) {
|
||||
b = "JDBC_ERROR";
|
||||
rec.put("bucket", b);
|
||||
rec.put("error", String.valueOf(ce.getMessage()).substring(0,
|
||||
Math.min(200, String.valueOf(ce.getMessage()).length())));
|
||||
bucket.merge(b, 1, Integer::sum);
|
||||
Files.writeString(progress, om.writeValueAsString(rec) + "\n",
|
||||
java.nio.file.StandardOpenOption.CREATE, java.nio.file.StandardOpenOption.APPEND);
|
||||
System.out.printf("[%d/%d] %-30s %s%n", idx, ents.size(), t, b);
|
||||
continue;
|
||||
}
|
||||
DiffReport r = n <= maxFull ? runner.check(t) : runner.checkSample(t, sampleRows);
|
||||
boolean engineErr = !r.mismatches.isEmpty()
|
||||
&& "RUNNER_ERROR".equals(r.mismatches.get(0).key);
|
||||
if (engineErr) {
|
||||
b = "ENGINE_ERROR";
|
||||
rec.put("error", String.valueOf(r.mismatches.get(0).cells.get(0).right));
|
||||
} else if (r.isClean()) {
|
||||
b = n <= maxFull ? "OK_FULL" : "OK_SAMPLE";
|
||||
} else {
|
||||
b = "DIFF";
|
||||
StringBuilder sb = new StringBuilder();
|
||||
int c = 0;
|
||||
for (DiffReport.RowMismatch mm : r.mismatches) {
|
||||
for (DiffReport.CellDiff cd : mm.cells) {
|
||||
if (c++ >= 5) break;
|
||||
sb.append(cd.column).append('[').append(cd.reason).append("] ");
|
||||
}
|
||||
if (c >= 5) break;
|
||||
}
|
||||
rec.put("diffCols", sb.toString());
|
||||
rec.put("onlyL", r.onlyInLeft.size());
|
||||
rec.put("onlyR", r.onlyInRight.size());
|
||||
diffLines.add(String.format("| %s | %d | %.4f | %s |",
|
||||
t, r.leftRowCount, r.consistencyRate(), sb));
|
||||
}
|
||||
rec.put("bucket", b);
|
||||
rec.put("rows", r.leftRowCount);
|
||||
rec.put("rate", Math.round(r.consistencyRate() * 10000.0) / 10000.0);
|
||||
totalRows += r.leftRowCount;
|
||||
} catch (Exception ex) {
|
||||
b = "ENGINE_ERROR";
|
||||
rec.put("bucket", b);
|
||||
rec.put("error", String.valueOf(ex.getMessage()));
|
||||
}
|
||||
bucket.merge(b, 1, Integer::sum);
|
||||
Files.writeString(progress, om.writeValueAsString(rec) + "\n",
|
||||
java.nio.file.StandardOpenOption.CREATE, java.nio.file.StandardOpenOption.APPEND);
|
||||
System.out.printf("[%d/%d] %-30s %s%n", idx, ents.size(), t, b);
|
||||
}
|
||||
StringBuilder md = new StringBuilder("# P1-1 全实体发现式对拍汇总\n\n");
|
||||
md.append("- 实体总数: ").append(ents.size()).append("\n");
|
||||
bucket.forEach((k, v) -> md.append("- ").append(k).append(": ").append(v).append("\n"));
|
||||
md.append("- 总比对行数(基线侧): ").append(totalRows).append("\n\n");
|
||||
if (!diffLines.isEmpty()) {
|
||||
md.append("## DIFF 实体\n\n| 实体 | 行数 | 一致率 | 差异列(前5) |\n|---|---|---|---|\n");
|
||||
diffLines.forEach(l -> md.append(l).append('\n'));
|
||||
}
|
||||
Files.writeString(summaryMd, md.toString());
|
||||
System.out.println("=== discover 完成 === 进度: " + progress + " 汇总: " + summaryMd);
|
||||
int diffs = bucket.getOrDefault("DIFF", 0);
|
||||
return diffs == 0 ? 0 : 1;
|
||||
}
|
||||
|
||||
private static String truncate(String s) {
|
||||
if (s == null) {
|
||||
return "";
|
||||
|
||||
Reference in New Issue
Block a user