feat(parity): P1-1 全实体发现式对拍模式(--discover)
枚举 sys_entity 全部活跃实体:≤maxFull(默认2000)行全量逐行,大表抽样 discoverSample(默认300)行; 逐实体 JSONL 进度可断点续跑(--resumeFrom);分桶 OK_FULL/OK_SAMPLE/DIFF/ENGINE_ERROR/JDBC_ERROR; 内存纪律:仅留实体摘要与前 5 个差异格 Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
@@ -55,6 +55,15 @@ public final class ParityCli {
|
|||||||
SimpleDriverDataSource ds = new SimpleDriverDataSource(
|
SimpleDriverDataSource ds = new SimpleDriverDataSource(
|
||||||
new org.postgresql.Driver(), jdbcUrl, user, pass == null ? "" : pass);
|
new org.postgresql.Driver(), jdbcUrl, user, pass == null ? "" : pass);
|
||||||
|
|
||||||
|
// P1-1 全实体发现式对拍:--discover [--maxFull=2000] [--discoverSample=300] [--resumeFrom=进度.jsonl]
|
||||||
|
if (opt.containsKey("discover")) {
|
||||||
|
int maxFull = Integer.parseInt(opt.getOrDefault("maxFull", "2000"));
|
||||||
|
int dSample = Integer.parseInt(opt.getOrDefault("discoverSample", "300"));
|
||||||
|
int code = runDiscover(engine, ds, outDir, maxFull, dSample, perPage,
|
||||||
|
opt.get("resumeFrom"));
|
||||||
|
System.exit(code);
|
||||||
|
}
|
||||||
|
|
||||||
// C2 数据行级权限对拍模式:--privilege
|
// C2 数据行级权限对拍模式:--privilege
|
||||||
if (opt.containsKey("privilege")) {
|
if (opt.containsKey("privilege")) {
|
||||||
int code = runPrivilege(engine, ds, outDir);
|
int code = runPrivilege(engine, ds, outDir);
|
||||||
@@ -305,6 +314,114 @@ public final class ParityCli {
|
|||||||
return m;
|
return m;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* P1-1 全实体发现式对拍:枚举 sys_entity 全部活跃实体,行数 ≤maxFull 全量逐行、
|
||||||
|
* 否则抽样 sampleRows 行;逐实体进度写 JSONL(可 --resumeFrom 断点续跑),
|
||||||
|
* 终局汇总分桶:OK_FULL/OK_SAMPLE/DIFF/ENGINE_ERROR/JDBC_ERROR。
|
||||||
|
* 内存纪律:只保留每实体摘要(差异明细仅前 5 格),不留全量 mismatch。
|
||||||
|
*/
|
||||||
|
private static int runDiscover(String engine, javax.sql.DataSource ds, Path outDir,
|
||||||
|
int maxFull, int sampleRows, int perPage, String resumeFrom) throws Exception {
|
||||||
|
SelfCheckRunner runner = new SelfCheckRunner(engine, ds, perPage);
|
||||||
|
JdbcResultReader jdbc = new JdbcResultReader(ds);
|
||||||
|
java.util.Set<String> done = new java.util.HashSet<>();
|
||||||
|
if (resumeFrom != null && Files.exists(Path.of(resumeFrom))) {
|
||||||
|
for (String line : Files.readAllLines(Path.of(resumeFrom))) {
|
||||||
|
int i = line.indexOf("\"entity\":\"");
|
||||||
|
if (i >= 0) {
|
||||||
|
int st = i + 10, en = line.indexOf('"', st);
|
||||||
|
done.add(line.substring(st, en));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
System.out.println("[discover] 续跑,已完成 " + done.size() + " 个实体跳过");
|
||||||
|
}
|
||||||
|
List<Map<String, Object>> ents = jdbc.query(
|
||||||
|
"SELECT trim(tablename) AS t FROM sys_entity WHERE isactive=1 ORDER BY trim(tablename)");
|
||||||
|
String ts = new java.text.SimpleDateFormat("yyyyMMdd-HHmmss").format(new java.util.Date());
|
||||||
|
Path progress = outDir.resolve("discover-progress-" + ts + ".jsonl");
|
||||||
|
Path summaryMd = outDir.resolve("discover-summary-" + ts + ".md");
|
||||||
|
Map<String, Integer> bucket = new java.util.LinkedHashMap<>();
|
||||||
|
List<String> diffLines = new ArrayList<>();
|
||||||
|
long totalRows = 0;
|
||||||
|
int idx = 0;
|
||||||
|
com.fasterxml.jackson.databind.ObjectMapper om = new com.fasterxml.jackson.databind.ObjectMapper();
|
||||||
|
for (Map<String, Object> row : ents) {
|
||||||
|
String t = String.valueOf(row.values().iterator().next()).trim().toUpperCase();
|
||||||
|
idx++;
|
||||||
|
if (done.contains(t)) {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
Map<String, Object> rec = new java.util.LinkedHashMap<>();
|
||||||
|
rec.put("entity", t);
|
||||||
|
String b;
|
||||||
|
try {
|
||||||
|
long n;
|
||||||
|
try {
|
||||||
|
n = jdbc.count(t);
|
||||||
|
} catch (Exception ce) {
|
||||||
|
b = "JDBC_ERROR";
|
||||||
|
rec.put("bucket", b);
|
||||||
|
rec.put("error", String.valueOf(ce.getMessage()).substring(0,
|
||||||
|
Math.min(200, String.valueOf(ce.getMessage()).length())));
|
||||||
|
bucket.merge(b, 1, Integer::sum);
|
||||||
|
Files.writeString(progress, om.writeValueAsString(rec) + "\n",
|
||||||
|
java.nio.file.StandardOpenOption.CREATE, java.nio.file.StandardOpenOption.APPEND);
|
||||||
|
System.out.printf("[%d/%d] %-30s %s%n", idx, ents.size(), t, b);
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
DiffReport r = n <= maxFull ? runner.check(t) : runner.checkSample(t, sampleRows);
|
||||||
|
boolean engineErr = !r.mismatches.isEmpty()
|
||||||
|
&& "RUNNER_ERROR".equals(r.mismatches.get(0).key);
|
||||||
|
if (engineErr) {
|
||||||
|
b = "ENGINE_ERROR";
|
||||||
|
rec.put("error", String.valueOf(r.mismatches.get(0).cells.get(0).right));
|
||||||
|
} else if (r.isClean()) {
|
||||||
|
b = n <= maxFull ? "OK_FULL" : "OK_SAMPLE";
|
||||||
|
} else {
|
||||||
|
b = "DIFF";
|
||||||
|
StringBuilder sb = new StringBuilder();
|
||||||
|
int c = 0;
|
||||||
|
for (DiffReport.RowMismatch mm : r.mismatches) {
|
||||||
|
for (DiffReport.CellDiff cd : mm.cells) {
|
||||||
|
if (c++ >= 5) break;
|
||||||
|
sb.append(cd.column).append('[').append(cd.reason).append("] ");
|
||||||
|
}
|
||||||
|
if (c >= 5) break;
|
||||||
|
}
|
||||||
|
rec.put("diffCols", sb.toString());
|
||||||
|
rec.put("onlyL", r.onlyInLeft.size());
|
||||||
|
rec.put("onlyR", r.onlyInRight.size());
|
||||||
|
diffLines.add(String.format("| %s | %d | %.4f | %s |",
|
||||||
|
t, r.leftRowCount, r.consistencyRate(), sb));
|
||||||
|
}
|
||||||
|
rec.put("bucket", b);
|
||||||
|
rec.put("rows", r.leftRowCount);
|
||||||
|
rec.put("rate", Math.round(r.consistencyRate() * 10000.0) / 10000.0);
|
||||||
|
totalRows += r.leftRowCount;
|
||||||
|
} catch (Exception ex) {
|
||||||
|
b = "ENGINE_ERROR";
|
||||||
|
rec.put("bucket", b);
|
||||||
|
rec.put("error", String.valueOf(ex.getMessage()));
|
||||||
|
}
|
||||||
|
bucket.merge(b, 1, Integer::sum);
|
||||||
|
Files.writeString(progress, om.writeValueAsString(rec) + "\n",
|
||||||
|
java.nio.file.StandardOpenOption.CREATE, java.nio.file.StandardOpenOption.APPEND);
|
||||||
|
System.out.printf("[%d/%d] %-30s %s%n", idx, ents.size(), t, b);
|
||||||
|
}
|
||||||
|
StringBuilder md = new StringBuilder("# P1-1 全实体发现式对拍汇总\n\n");
|
||||||
|
md.append("- 实体总数: ").append(ents.size()).append("\n");
|
||||||
|
bucket.forEach((k, v) -> md.append("- ").append(k).append(": ").append(v).append("\n"));
|
||||||
|
md.append("- 总比对行数(基线侧): ").append(totalRows).append("\n\n");
|
||||||
|
if (!diffLines.isEmpty()) {
|
||||||
|
md.append("## DIFF 实体\n\n| 实体 | 行数 | 一致率 | 差异列(前5) |\n|---|---|---|---|\n");
|
||||||
|
diffLines.forEach(l -> md.append(l).append('\n'));
|
||||||
|
}
|
||||||
|
Files.writeString(summaryMd, md.toString());
|
||||||
|
System.out.println("=== discover 完成 === 进度: " + progress + " 汇总: " + summaryMd);
|
||||||
|
int diffs = bucket.getOrDefault("DIFF", 0);
|
||||||
|
return diffs == 0 ? 0 : 1;
|
||||||
|
}
|
||||||
|
|
||||||
private static String truncate(String s) {
|
private static String truncate(String s) {
|
||||||
if (s == null) {
|
if (s == null) {
|
||||||
return "";
|
return "";
|
||||||
|
|||||||
Reference in New Issue
Block a user