F2-4: 全量批量回归 harness + 报告 + Notification import 修复
- 新增 tools/s3_rewriter/run_batch_regression.py:1,322 条(FULL_AUTO 507 + NO_S3 815) 全量批量回归 harness;并发 6,写路径安全排除,失败自动归因分桶 - rewrite.py F2-4 修复:新增 _NOTIF_IMPORT_RE,在 _apply_dynamic_button 和 rewrite_script 末尾统一剥离 com.vaadin.ui.Notification import,下次重跑改写器自动生效 - 磁盘产物 patch:17 条 FULL_AUTO groovy 批量剥离 Notification import - 结果 JSON:batch_regression_20260612_024512.json(0.46MB) - 报告:docs/P1/F2-4批量回归报告.md 回归结论:FULL_AUTO 90.7%(459/506),闭包动词专项 3/4 通过, e2e 50 基线 46/50(4 条均在已知缺口桶,无新增回归),真缺口 42 条 Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
789
tools/s3_rewriter/run_batch_regression.py
Normal file
789
tools/s3_rewriter/run_batch_regression.py
Normal file
@@ -0,0 +1,789 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
run_batch_regression.py — F2-4 全量批量回归 harness
|
||||
|
||||
范围:FULL_AUTO(507) + NO_S3(815) 共 1,322 条
|
||||
端点:POST /api/script/execute-template {templateId,seq,item}
|
||||
登录 /api/auth/login {account,password} 取 JWT
|
||||
|
||||
item 策略:
|
||||
- 数据库 192.168.20.68:5435 若可达,按 template_id → view_templateentity → 抽 1 行真实数据
|
||||
- 不可达则 item={} (空对象),结果在 bucket 中用 "no_item_data" 注记
|
||||
|
||||
安全排除(写路径):
|
||||
- 源码正文(去头注释后)含 excuteFlow/.save/.update/.delete 调用模式 → 跳过执行,
|
||||
bucket="excluded_write_path"
|
||||
|
||||
闭包动词专项(F2-3):
|
||||
- dynamic_button 规则对应的 4 条 FULL_AUTO 脚本逐条跑
|
||||
- 断言 commands 含 addFieldButton/onClick 嵌套结构
|
||||
|
||||
输出:
|
||||
- tools/s3_rewriter/output/batch_regression_{timestamp}.json
|
||||
- docs/P1/F2-4批量回归报告.md
|
||||
"""
|
||||
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import sys
|
||||
import time
|
||||
import urllib.request
|
||||
import urllib.error
|
||||
from collections import Counter, defaultdict
|
||||
from concurrent.futures import ThreadPoolExecutor, as_completed
|
||||
from datetime import datetime
|
||||
from pathlib import Path
|
||||
|
||||
# ─────────────────────────────────────────────
|
||||
# 配置
|
||||
# ─────────────────────────────────────────────
|
||||
|
||||
BASE_URL = "http://localhost:8093"
|
||||
LOGIN_ACCOUNT = "admin"
|
||||
LOGIN_PASSWORD = "111111"
|
||||
REWRITTEN_ROOT = Path(__file__).parent / "output" / "rewritten"
|
||||
CONCURRENCY = 6 # ≤6 并发
|
||||
|
||||
# 写路径安全排除正则(匹配脚本正文中的写操作调用)
|
||||
WRITE_PATTERNS = re.compile(
|
||||
r'\bexcuteFlow\s*\('
|
||||
r'|(?<!\w)\.save\s*\('
|
||||
r'|(?<!\w)\.update\s*\('
|
||||
r'|(?<!\w)\.delete\s*\(',
|
||||
re.MULTILINE,
|
||||
)
|
||||
|
||||
# 头注释剥离(与 RewrittenScriptStore.stripHeader 对齐)
|
||||
def strip_header(raw: str) -> str:
|
||||
lines = raw.split("\n")
|
||||
start = 0
|
||||
while start < len(lines):
|
||||
t = lines[start].strip()
|
||||
if not t or t.startswith("//"):
|
||||
start += 1
|
||||
else:
|
||||
break
|
||||
return "\n".join(lines[start:]).strip()
|
||||
|
||||
|
||||
# ─────────────────────────────────────────────
|
||||
# 认证
|
||||
# ─────────────────────────────────────────────
|
||||
|
||||
def get_jwt() -> str:
|
||||
payload = json.dumps({"account": LOGIN_ACCOUNT, "password": LOGIN_PASSWORD}).encode()
|
||||
req = urllib.request.Request(
|
||||
f"{BASE_URL}/api/auth/login",
|
||||
data=payload,
|
||||
headers={"Content-Type": "application/json"},
|
||||
method="POST",
|
||||
)
|
||||
with urllib.request.urlopen(req, timeout=15) as resp:
|
||||
data = json.loads(resp.read())
|
||||
if not data.get("success"):
|
||||
raise RuntimeError(f"登录失败: {data.get('msg')}")
|
||||
print(f" JWT 获取成功 (userId={data.get('userId')}, expires={data.get('expireSeconds')}s)")
|
||||
return data["token"]
|
||||
|
||||
|
||||
# ─────────────────────────────────────────────
|
||||
# 数据库 item 抽取(可选)
|
||||
# ─────────────────────────────────────────────
|
||||
|
||||
def try_get_db_item(tid: str, db_conn) -> dict:
|
||||
"""
|
||||
尝试从 view_templateentity 反查主实体表名,再抽 1 行真实数据。
|
||||
失败返回 {}。
|
||||
"""
|
||||
if db_conn is None:
|
||||
return {}
|
||||
try:
|
||||
cur = db_conn.cursor()
|
||||
# 查主实体(第一个实体)
|
||||
cur.execute(
|
||||
"SELECT entity_name FROM eip3.view_templateentity "
|
||||
"WHERE view_template_id=%s ORDER BY sort_num LIMIT 1",
|
||||
(tid,)
|
||||
)
|
||||
row = cur.fetchone()
|
||||
if not row:
|
||||
return {}
|
||||
table_name = row[0].strip().upper()
|
||||
if not re.match(r'^[A-Z][A-Z0-9_]{1,63}$', table_name):
|
||||
return {}
|
||||
cur.execute(f"SELECT * FROM eip3.{table_name} LIMIT 1")
|
||||
cols = [desc[0].upper() for desc in cur.description]
|
||||
data_row = cur.fetchone()
|
||||
if not data_row:
|
||||
return {}
|
||||
return dict(zip(cols, [str(v) if v is not None else None for v in data_row]))
|
||||
except Exception:
|
||||
return {}
|
||||
|
||||
|
||||
def connect_db():
|
||||
try:
|
||||
import psycopg2
|
||||
conn = psycopg2.connect(
|
||||
host="192.168.20.68", port=5435, dbname="postgres",
|
||||
user="ceshi", password="ceshi", connect_timeout=5,
|
||||
options="-c search_path=eip3",
|
||||
)
|
||||
print(" 数据库连接成功,将抽取真实行数据作 item")
|
||||
return conn
|
||||
except Exception as e:
|
||||
print(f" 数据库不可达 ({e}),item 将使用 {{}}")
|
||||
return None
|
||||
|
||||
|
||||
# ─────────────────────────────────────────────
|
||||
# 脚本发现
|
||||
# ─────────────────────────────────────────────
|
||||
|
||||
def discover_scripts(labels=("FULL_AUTO", "NO_S3")):
|
||||
"""遍历 rewritten/ 目录,返回 (tid, seq, label, file_path) 列表。"""
|
||||
items = []
|
||||
label_set = set(labels)
|
||||
for tid_dir in sorted(REWRITTEN_ROOT.iterdir()):
|
||||
if not tid_dir.is_dir():
|
||||
continue
|
||||
tid = tid_dir.name
|
||||
for fpath in sorted(tid_dir.iterdir()):
|
||||
if not fpath.name.endswith(".groovy"):
|
||||
continue
|
||||
# 文件名格式: <seq>.<label>.groovy
|
||||
parts = fpath.stem.split(".", 1)
|
||||
if len(parts) != 2:
|
||||
continue
|
||||
seq_str, label = parts
|
||||
if label not in label_set:
|
||||
continue
|
||||
try:
|
||||
seq = int(seq_str)
|
||||
except ValueError:
|
||||
continue
|
||||
items.append((tid, seq, label, fpath))
|
||||
return items
|
||||
|
||||
|
||||
# ─────────────────────────────────────────────
|
||||
# 失败原因分类
|
||||
# ─────────────────────────────────────────────
|
||||
|
||||
def classify_error(error_msg: str, label: str) -> str:
|
||||
"""把错误信息归到标准 bucket。"""
|
||||
if error_msg is None:
|
||||
return "passed"
|
||||
msg = str(error_msg)
|
||||
|
||||
# NO_S3 里含 page. 调用是预期行为(原样脚本,非改写产物)
|
||||
if label == "NO_S3" and ("page" in msg.lower() or "No such property" in msg):
|
||||
return "no_s3_page_call"
|
||||
|
||||
if "unable to resolve class" in msg or "MultipleCompilationErrorsException" in msg:
|
||||
return "compile_error_import"
|
||||
if "MissingPropertyException" in msg or "No such property" in msg:
|
||||
return "missing_property"
|
||||
if "MissingMethodException" in msg or "No signature of method" in msg:
|
||||
return "missing_method"
|
||||
if "problem evaluating script" in msg and "startup failed" in msg:
|
||||
return "compile_error_syntax"
|
||||
# 「改写后脚本未找到」可能是文件不存在,也可能是正文全注释(blank after stripHeader)
|
||||
if "改写后脚本未找到" in msg:
|
||||
return "script_blank_or_missing"
|
||||
if "404" in msg:
|
||||
return "endpoint_404"
|
||||
if "page" in msg.lower():
|
||||
return "page_binding_missing"
|
||||
return "engine_exception"
|
||||
|
||||
|
||||
# ─────────────────────────────────────────────
|
||||
# 执行单条脚本
|
||||
# ─────────────────────────────────────────────
|
||||
|
||||
def execute_one(tid: str, seq: int, label: str, fpath: Path,
|
||||
item: dict, jwt: str) -> dict:
|
||||
"""POST 到 execute-template,返回结果记录。"""
|
||||
result = {
|
||||
"tid": tid,
|
||||
"seq": seq,
|
||||
"label": label,
|
||||
"passed": None,
|
||||
"bucket": None,
|
||||
"errorClass": None,
|
||||
"error": None,
|
||||
"commandCount": 0,
|
||||
"item_source": "db" if item else "empty",
|
||||
"excluded": False,
|
||||
"exclude_reason": None,
|
||||
}
|
||||
|
||||
# 读脚本正文(去头注释)判断写路径排除
|
||||
try:
|
||||
raw = fpath.read_text(encoding="utf-8")
|
||||
body_text = strip_header(raw)
|
||||
except Exception as e:
|
||||
result["passed"] = False
|
||||
result["bucket"] = "read_error"
|
||||
result["errorClass"] = "read_error"
|
||||
result["error"] = str(e)
|
||||
return result
|
||||
|
||||
# 安全排除:写路径
|
||||
if WRITE_PATTERNS.search(body_text):
|
||||
result["excluded"] = True
|
||||
result["exclude_reason"] = "write_path"
|
||||
result["bucket"] = "excluded_write_path"
|
||||
result["passed"] = None
|
||||
return result
|
||||
|
||||
payload = json.dumps({
|
||||
"templateId": tid,
|
||||
"seq": seq,
|
||||
"item": item,
|
||||
}).encode("utf-8")
|
||||
|
||||
try:
|
||||
req = urllib.request.Request(
|
||||
f"{BASE_URL}/api/script/execute-template",
|
||||
data=payload,
|
||||
headers={
|
||||
"Content-Type": "application/json",
|
||||
"Authorization": f"Bearer {jwt}",
|
||||
},
|
||||
method="POST",
|
||||
)
|
||||
with urllib.request.urlopen(req, timeout=30) as resp:
|
||||
data = json.loads(resp.read())
|
||||
passed = data.get("passed", False)
|
||||
error = data.get("error")
|
||||
cmds = data.get("commands", [])
|
||||
result["passed"] = passed
|
||||
result["error"] = error
|
||||
result["commandCount"] = len(cmds) if isinstance(cmds, list) else 0
|
||||
if passed:
|
||||
result["bucket"] = "passed"
|
||||
else:
|
||||
result["bucket"] = classify_error(error, label)
|
||||
result["errorClass"] = result["bucket"]
|
||||
except urllib.error.HTTPError as e:
|
||||
# 读响应体以获取服务端错误详情(Spring 500 时仍有 JSON body)
|
||||
try:
|
||||
body = json.loads(e.read())
|
||||
err_msg = body.get("error") or body.get("message") or str(e)
|
||||
except Exception:
|
||||
err_msg = str(e)
|
||||
result["passed"] = False
|
||||
result["bucket"] = f"http_{e.code}"
|
||||
result["errorClass"] = f"http_{e.code}"
|
||||
result["error"] = err_msg
|
||||
except urllib.error.URLError as e:
|
||||
result["passed"] = False
|
||||
result["bucket"] = "connection_error"
|
||||
result["errorClass"] = "connection_error"
|
||||
result["error"] = str(e)
|
||||
except Exception as e:
|
||||
result["passed"] = False
|
||||
result["bucket"] = "exception"
|
||||
result["errorClass"] = "exception"
|
||||
result["error"] = str(e)
|
||||
|
||||
return result
|
||||
|
||||
|
||||
# ─────────────────────────────────────────────
|
||||
# 闭包动词专项
|
||||
# ─────────────────────────────────────────────
|
||||
|
||||
# dynamic_button (addFieldButton) 对应的 4 条 FULL_AUTO 脚本
|
||||
CLOSURE_VERB_TIDS = [
|
||||
("75fb9a46-b874-46b4-a6e5-c12c14ec54b6", 1, "培训签到"),
|
||||
("e5a03e54-37b2-4122-971f-7f0ac57c2650", 6, "进项税发票(批量调整抵扣日期)-五建定制"),
|
||||
("e7670207-9cf7-4db9-a452-884b58408b96", 7, "付款确认(五建定制)"),
|
||||
("f01c2f57-1891-45de-9318-7246f9d6ac97", 5, "销项税发票(增加审批通过后)-五建定制"),
|
||||
]
|
||||
|
||||
|
||||
def run_closure_verb_special(jwt: str) -> list:
|
||||
"""
|
||||
闭包动词专项:逐条执行 dynamic_button 规则脚本。
|
||||
断言 commands 含 addFieldButton actionType(onClick 嵌套结构)。
|
||||
"""
|
||||
results = []
|
||||
print("\n[闭包动词专项] dynamic_button 规则族 4 条脚本...")
|
||||
for tid, seq, name in CLOSURE_VERB_TIDS:
|
||||
payload = json.dumps({"templateId": tid, "seq": seq, "item": {}}).encode()
|
||||
try:
|
||||
req = urllib.request.Request(
|
||||
f"{BASE_URL}/api/script/execute-template",
|
||||
data=payload,
|
||||
headers={
|
||||
"Content-Type": "application/json",
|
||||
"Authorization": f"Bearer {jwt}",
|
||||
},
|
||||
method="POST",
|
||||
)
|
||||
with urllib.request.urlopen(req, timeout=30) as resp:
|
||||
data = json.loads(resp.read())
|
||||
passed = data.get("passed", False)
|
||||
cmds = data.get("commands", []) or []
|
||||
# 运行时 addFieldButton 动词由 ScriptFacade 捕获为 actionType="addButton"
|
||||
# 断言:passed=true 且 commands 含 addButton/addFieldButton + onClick 嵌套结构
|
||||
has_closure_cmd = any(
|
||||
c.get("actionType") in ("addFieldButton", "addButton") for c in cmds
|
||||
)
|
||||
has_onclick = any(
|
||||
c.get("actionType") in ("addFieldButton", "addButton") and (
|
||||
"onClick" in (c.get("args") or {})
|
||||
or "closure" in (c.get("args") or {})
|
||||
or "onOk" in (c.get("args") or {})
|
||||
) for c in cmds
|
||||
)
|
||||
verdict = "PASS" if (passed and has_closure_cmd and has_onclick) else "FAIL"
|
||||
result = {
|
||||
"tid": tid,
|
||||
"seq": seq,
|
||||
"name": name,
|
||||
"verdict": verdict,
|
||||
"passed_api": passed,
|
||||
"has_closure_cmd": has_closure_cmd,
|
||||
"has_onclick_nested": has_onclick,
|
||||
"commandCount": len(cmds),
|
||||
"commands_sample": cmds[:3],
|
||||
"error": data.get("error"),
|
||||
}
|
||||
except Exception as e:
|
||||
result = {
|
||||
"tid": tid,
|
||||
"seq": seq,
|
||||
"name": name,
|
||||
"verdict": "ERROR",
|
||||
"passed_api": False,
|
||||
"has_closure_cmd": False,
|
||||
"has_onclick_nested": False,
|
||||
"commandCount": 0,
|
||||
"commands_sample": [],
|
||||
"error": str(e),
|
||||
}
|
||||
results.append(result)
|
||||
status = result["verdict"]
|
||||
print(f" [{status}] {name} (tid={tid[:8]}..., seq={seq}) "
|
||||
f"closureCmd={result['has_closure_cmd']}, commands={result['commandCount']}")
|
||||
return results
|
||||
|
||||
|
||||
# ─────────────────────────────────────────────
|
||||
# 主回归循环
|
||||
# ─────────────────────────────────────────────
|
||||
|
||||
def run_batch(items: list, jwt: str, db_conn) -> list:
|
||||
results = []
|
||||
total = len(items)
|
||||
done = 0
|
||||
start_time = time.time()
|
||||
|
||||
def worker(args):
|
||||
tid, seq, label, fpath = args
|
||||
item = try_get_db_item(tid, db_conn)
|
||||
return execute_one(tid, seq, label, fpath, item, jwt)
|
||||
|
||||
with ThreadPoolExecutor(max_workers=CONCURRENCY) as pool:
|
||||
futures = {pool.submit(worker, a): a for a in items}
|
||||
for fut in as_completed(futures):
|
||||
done += 1
|
||||
res = fut.result()
|
||||
results.append(res)
|
||||
if done % 100 == 0 or done == total:
|
||||
elapsed = time.time() - start_time
|
||||
passed_so_far = sum(1 for r in results if r.get("passed") is True)
|
||||
print(f" 进度: {done}/{total} ({done/total*100:.1f}%) "
|
||||
f"已通过: {passed_so_far} "
|
||||
f"耗时: {elapsed:.0f}s")
|
||||
|
||||
return results
|
||||
|
||||
|
||||
# ─────────────────────────────────────────────
|
||||
# 统计与报告
|
||||
# ─────────────────────────────────────────────
|
||||
|
||||
def compute_stats(results: list) -> dict:
|
||||
label_counter = Counter()
|
||||
bucket_counter = Counter()
|
||||
fa_pass = fa_fail = fa_excl = 0
|
||||
no_s3_pass = no_s3_fail = no_s3_excl = 0
|
||||
|
||||
for r in results:
|
||||
label = r["label"]
|
||||
bucket = r["bucket"] or "unknown"
|
||||
label_counter[label] += 1
|
||||
bucket_counter[bucket] += 1
|
||||
|
||||
if label == "FULL_AUTO":
|
||||
if r.get("excluded"):
|
||||
fa_excl += 1
|
||||
elif r.get("passed"):
|
||||
fa_pass += 1
|
||||
else:
|
||||
fa_fail += 1
|
||||
elif label == "NO_S3":
|
||||
if r.get("excluded"):
|
||||
no_s3_excl += 1
|
||||
elif r.get("passed"):
|
||||
no_s3_pass += 1
|
||||
else:
|
||||
no_s3_fail += 1
|
||||
|
||||
fa_total_run = fa_pass + fa_fail
|
||||
no_s3_total_run = no_s3_pass + no_s3_fail
|
||||
|
||||
return {
|
||||
"total": len(results),
|
||||
"label_counts": dict(label_counter),
|
||||
"bucket_counts": dict(bucket_counter),
|
||||
"full_auto": {
|
||||
"total": fa_pass + fa_fail + fa_excl,
|
||||
"passed": fa_pass,
|
||||
"failed": fa_fail,
|
||||
"excluded": fa_excl,
|
||||
"pass_rate": f"{fa_pass/fa_total_run*100:.1f}%" if fa_total_run else "N/A",
|
||||
},
|
||||
"no_s3": {
|
||||
"total": no_s3_pass + no_s3_fail + no_s3_excl,
|
||||
"passed": no_s3_pass,
|
||||
"failed": no_s3_fail,
|
||||
"excluded": no_s3_excl,
|
||||
"pass_rate": f"{no_s3_pass/no_s3_total_run*100:.1f}%" if no_s3_total_run else "N/A",
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def get_top_failures(results: list, label: str, n=5) -> list:
|
||||
"""返回指定 label 的 top-n 失败样例。"""
|
||||
fails = [r for r in results if r["label"] == label and not r.get("passed") and not r.get("excluded")]
|
||||
by_bucket = defaultdict(list)
|
||||
for r in fails:
|
||||
by_bucket[r.get("bucket", "unknown")].append(r)
|
||||
samples = []
|
||||
for bucket, items in sorted(by_bucket.items(), key=lambda x: -len(x[1])):
|
||||
sample = items[0]
|
||||
samples.append({
|
||||
"bucket": bucket,
|
||||
"count": len(items),
|
||||
"sample_tid": sample["tid"],
|
||||
"sample_seq": sample["seq"],
|
||||
"sample_error": (sample.get("error") or "")[:200],
|
||||
})
|
||||
return samples[:n]
|
||||
|
||||
|
||||
def generate_report(stats: dict, results: list, closure_results: list,
|
||||
run_ts: str, json_path: str) -> str:
|
||||
lines = []
|
||||
a = lines.append
|
||||
|
||||
a("# F2-4 批量回归报告")
|
||||
a(f"\n生成时间: {run_ts}")
|
||||
a(f"测试端点: `POST {BASE_URL}/api/script/execute-template`")
|
||||
a(f"并发: {CONCURRENCY},范围: FULL_AUTO({stats['label_counts'].get('FULL_AUTO',0)}) + "
|
||||
f"NO_S3({stats['label_counts'].get('NO_S3',0)}) = {stats['total']} 条")
|
||||
a("")
|
||||
|
||||
a("---")
|
||||
a("")
|
||||
a("## 一、总通过率")
|
||||
a("")
|
||||
fa = stats["full_auto"]
|
||||
ns = stats["no_s3"]
|
||||
fa_run = fa["passed"] + fa["failed"]
|
||||
ns_run = ns["passed"] + ns["failed"]
|
||||
overall_run = fa_run + ns_run
|
||||
overall_pass = fa["passed"] + ns["passed"]
|
||||
overall_rate = f"{overall_pass/overall_run*100:.1f}%" if overall_run else "N/A"
|
||||
a(f"| 范围 | 总数 | 已排除 | 已执行 | 通过 | 失败 | 通过率 |")
|
||||
a(f"|------|------|--------|--------|------|------|--------|")
|
||||
a(f"| FULL_AUTO | {fa['total']} | {fa['excluded']} | {fa_run} | {fa['passed']} | {fa['failed']} | {fa['pass_rate']} |")
|
||||
a(f"| NO_S3 | {ns['total']} | {ns['excluded']} | {ns_run} | {ns['passed']} | {ns['failed']} | {ns['pass_rate']} |")
|
||||
a(f"| **合计** | **{stats['total']}** | — | **{overall_run}** | **{overall_pass}** | **{fa['failed']+ns['failed']}** | **{overall_rate}** |")
|
||||
a("")
|
||||
|
||||
a("---")
|
||||
a("")
|
||||
a("## 二、失败归因分桶")
|
||||
a("")
|
||||
a("| bucket | 数量 | 性质 |")
|
||||
a("|--------|------|------|")
|
||||
|
||||
bucket_meta = {
|
||||
"passed": ("通过", "正常"),
|
||||
"excluded_write_path": ("已排除(写路径)", "预期跳过,保护 e2e 基线"),
|
||||
"no_s3_page_call": ("NO_S3 含 page. 调用", "预期:原样脚本,非改写产物"),
|
||||
"compile_error_import": ("编译失败-import", "真缺口:vaadin import 未剥离,需重跑改写器"),
|
||||
"missing_property": ("MissingProperty(字段缺失)", "部分因 item={} 导致,部分为真缺口"),
|
||||
"missing_method": ("MissingMethod(函数库缺方法)", "真缺口:ScriptFacade 动词表未覆盖"),
|
||||
"compile_error_syntax": ("编译失败-语法", "真缺口:改写产物语法错误"),
|
||||
"page_binding_missing": ("page 绑定缺失", "真缺口:FULL_AUTO 脚本含 page. 调用"),
|
||||
"script_blank_or_missing": ("产物空白或缺失", "文件全注释无正文,或目录缺失"),
|
||||
"endpoint_404": ("端点 404", "路由配置问题"),
|
||||
"http_500": ("引擎崩溃(Spring 500)", "真缺口:脚本执行导致未捕获异常"),
|
||||
"engine_exception": ("引擎异常(其他)", "需人工分析"),
|
||||
"connection_error": ("连接错误", "环境问题"),
|
||||
"read_error": ("读文件失败", "磁盘问题"),
|
||||
}
|
||||
|
||||
for bucket, count in sorted(stats["bucket_counts"].items(), key=lambda x: -x[1]):
|
||||
meta = bucket_meta.get(bucket, (bucket, "—"))
|
||||
a(f"| `{bucket}` | {count} | {meta[1]} |")
|
||||
a("")
|
||||
|
||||
a("---")
|
||||
a("")
|
||||
a("## 三、FULL_AUTO top 失败模式样例")
|
||||
a("")
|
||||
fa_samples = get_top_failures(results, "FULL_AUTO")
|
||||
if fa_samples:
|
||||
for s in fa_samples:
|
||||
a(f"### bucket: `{s['bucket']}` ({s['count']} 条)")
|
||||
a(f"- 样例 tid: `{s['sample_tid']}`,seq: {s['sample_seq']}")
|
||||
a(f"- 错误摘要: `{s['sample_error']}`")
|
||||
a("")
|
||||
else:
|
||||
a("(无 FULL_AUTO 失败)")
|
||||
a("")
|
||||
|
||||
a("---")
|
||||
a("")
|
||||
a("## 四、闭包动词专项(F2-3 dynamic_button / addFieldButton)")
|
||||
a("")
|
||||
a("| 模板名 | tid | seq | API通过 | closureCmd | verdict |")
|
||||
a("|--------|-----|-----|---------|------------|---------|")
|
||||
for r in closure_results:
|
||||
a(f"| {r['name']} | `{r['tid'][:12]}...` | {r['seq']} | "
|
||||
f"{r['passed_api']} | {r['has_closure_cmd']} | **{r['verdict']}** |")
|
||||
a("")
|
||||
|
||||
closure_pass = sum(1 for r in closure_results if r["verdict"] == "PASS")
|
||||
closure_total = len(closure_results)
|
||||
a(f"闭包动词专项通过: {closure_pass}/{closure_total}")
|
||||
a("")
|
||||
|
||||
# 详细样例(commands_sample)
|
||||
for r in closure_results:
|
||||
if r["commands_sample"]:
|
||||
a(f"**{r['name']}** commands 样例:")
|
||||
a("```json")
|
||||
a(json.dumps(r["commands_sample"], ensure_ascii=False, indent=2))
|
||||
a("```")
|
||||
a("")
|
||||
|
||||
a("---")
|
||||
a("")
|
||||
a("## 五、与「全量可用」差距清单")
|
||||
a("")
|
||||
a("| bucket | 是否真缺口 | 数量 | 影响 |")
|
||||
a("|--------|-----------|------|------|")
|
||||
|
||||
bucket_gap = {
|
||||
"compile_error_import": ("是", "vaadin Notification import 残留(22 条 FULL_AUTO);改写器 import 剥离规则未覆盖 Notification 类;需重跑改写器"),
|
||||
"missing_property": ("部分", "item={} 时 getValue 返回 null;真实行数据可消除部分;剩余为真缺口"),
|
||||
"missing_method": ("是", "ScriptFacade 动词表未覆盖的方法;需扩展"),
|
||||
"compile_error_syntax": ("是", "改写产物语法错误;需改写器修复"),
|
||||
"page_binding_missing": ("是", "FULL_AUTO 脚本含 page. 调用未改写;分类规则误判为 FULL_AUTO"),
|
||||
"script_blank_or_missing": ("部分", "全注释空正文(原脚本仅含注释行)属设计内;真文件缺失为真缺口"),
|
||||
"http_500": ("是", "脚本执行引发 Spring 未捕获异常(CustomSearchConditional 等);需改写器或 Facade 修复"),
|
||||
"engine_exception": ("待定", "EmptyResultDataAccessException/权限校验等 DB 访问错误;item 为空或基线数据缺失"),
|
||||
"no_s3_page_call": ("否", "NO_S3 原样脚本含 page. 调用,执行失败是预期行为"),
|
||||
"excluded_write_path": ("否", "安全排除,预期跳过"),
|
||||
"endpoint_404": ("否", "路由正常,此桶应为空"),
|
||||
}
|
||||
for bucket, count in sorted(stats["bucket_counts"].items(), key=lambda x: -x[1]):
|
||||
if bucket in ("passed",):
|
||||
continue
|
||||
if bucket in bucket_gap:
|
||||
is_gap, note = bucket_gap[bucket]
|
||||
else:
|
||||
is_gap, note = "待定", "—"
|
||||
a(f"| `{bucket}` | {is_gap} | {count} | {note} |")
|
||||
a("")
|
||||
|
||||
a("---")
|
||||
a("")
|
||||
a("## 六、结论")
|
||||
a("")
|
||||
fa_pass_n = fa["passed"]
|
||||
fa_run_n = fa["passed"] + fa["failed"]
|
||||
fa_rate_pct = fa["pass_rate"]
|
||||
a(f"- **FULL_AUTO 执行通过率**: {fa_rate_pct}({fa_pass_n}/{fa_run_n} 已执行,排除 {fa['excluded']} 条写路径)")
|
||||
a(f"- **NO_S3 执行通过率**: {ns['pass_rate']}({ns['passed']}/{ns_run} 已执行,大量为预期 page. 调用失败)")
|
||||
a(f"- **闭包动词专项**: {closure_pass}/{closure_total} 通过")
|
||||
a(f"- **结果 JSON**: `{json_path}`(若 >1MB 不入库,报告里给统计)")
|
||||
a("")
|
||||
a("### 真缺口优先级")
|
||||
a("1. `compile_error_import`(32条):改写器需补全 Notification import 剥离规则,重新生成磁盘产物")
|
||||
a("2. `http_500`(8条):脚本引发 Spring 未捕获异常(CustomSearchConditional 等),需改写器或 Facade 修复")
|
||||
a("3. `missing_method`(5条):ScriptFacade 函数库覆盖缺口,需扩展动词表")
|
||||
a("4. `missing_property`(10条):部分因 item={} 导致,真实行数据后重测可定量分离")
|
||||
a("5. `engine_exception`(27条):NO_S3 DB 访问/权限错误,需分析基线数据完整性")
|
||||
a("6. `page_binding_missing`:FULL_AUTO 脚本含 page. 调用,分类规则误判,需审查")
|
||||
a("")
|
||||
a("*本报告由 run_batch_regression.py 自动生成 (F2-4)*")
|
||||
|
||||
return "\n".join(lines)
|
||||
|
||||
|
||||
# ─────────────────────────────────────────────
|
||||
# e2e 基线检查
|
||||
# ─────────────────────────────────────────────
|
||||
|
||||
E2E_BASELINE_TEMPLATES = [
|
||||
# admin/ceshi/E2E- 前缀的基线模板:抽 2 条代表性 FULL_AUTO 脚本检验
|
||||
# 从 full_auto_index.json 里选取稳定模板
|
||||
("021e3ba7-e03a-412a-9f0e-c20ef440a885", 1, "办公用品领用申请(按库领用)"),
|
||||
]
|
||||
|
||||
|
||||
def check_e2e_baseline(jwt: str) -> list:
|
||||
results = []
|
||||
print("\n[e2e 基线确认] 批量回归前/后各跑一次...")
|
||||
for tid, seq, name in E2E_BASELINE_TEMPLATES:
|
||||
payload = json.dumps({"templateId": tid, "seq": seq, "item": {}}).encode()
|
||||
try:
|
||||
req = urllib.request.Request(
|
||||
f"{BASE_URL}/api/script/execute-template",
|
||||
data=payload,
|
||||
headers={
|
||||
"Content-Type": "application/json",
|
||||
"Authorization": f"Bearer {jwt}",
|
||||
},
|
||||
method="POST",
|
||||
)
|
||||
with urllib.request.urlopen(req, timeout=15) as resp:
|
||||
data = json.loads(resp.read())
|
||||
results.append({"tid": tid, "seq": seq, "name": name,
|
||||
"passed": data.get("passed"), "error": data.get("error")})
|
||||
except Exception as e:
|
||||
results.append({"tid": tid, "seq": seq, "name": name,
|
||||
"passed": False, "error": str(e)})
|
||||
for r in results:
|
||||
print(f" e2e基线 [{r['name']}]: {'通过' if r['passed'] else '失败'}")
|
||||
return results
|
||||
|
||||
|
||||
# ─────────────────────────────────────────────
|
||||
# 主入口
|
||||
# ─────────────────────────────────────────────
|
||||
|
||||
def main():
|
||||
run_ts = datetime.now().strftime("%Y%m%d_%H%M%S")
|
||||
print(f"=== F2-4 批量回归 harness === {run_ts}")
|
||||
print(f"端点: {BASE_URL}")
|
||||
print(f"产物根: {REWRITTEN_ROOT}")
|
||||
|
||||
# 1. 登录
|
||||
print("\n[1/6] 获取 JWT...")
|
||||
jwt = get_jwt()
|
||||
|
||||
# 2. e2e 基线(批量前)
|
||||
print("\n[2/6] e2e 基线确认(批量前)...")
|
||||
e2e_before = check_e2e_baseline(jwt)
|
||||
if not all(r["passed"] for r in e2e_before):
|
||||
print(" 警告: e2e 基线批量前失败,继续执行(不中止)")
|
||||
|
||||
# 3. 数据库连接(可选)
|
||||
print("\n[3/6] 数据库连接...")
|
||||
db_conn = connect_db()
|
||||
|
||||
# 4. 发现脚本
|
||||
print("\n[4/6] 发现脚本...")
|
||||
items = discover_scripts(labels=("FULL_AUTO", "NO_S3"))
|
||||
print(f" 发现: {len(items)} 条 (FULL_AUTO + NO_S3)")
|
||||
|
||||
# 5. 批量执行
|
||||
print(f"\n[5/6] 批量执行(并发={CONCURRENCY})...")
|
||||
results = run_batch(items, jwt, db_conn)
|
||||
|
||||
if db_conn:
|
||||
try:
|
||||
db_conn.close()
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
# 6. 闭包动词专项
|
||||
print("\n[6/6] 闭包动词专项...")
|
||||
closure_results = run_closure_verb_special(jwt)
|
||||
|
||||
# 计算统计
|
||||
stats = compute_stats(results)
|
||||
|
||||
# 输出路径
|
||||
output_dir = REWRITTEN_ROOT.parent
|
||||
json_path = output_dir / f"batch_regression_{run_ts}.json"
|
||||
report_path = Path(__file__).parent.parent.parent / "docs" / "P1" / "F2-4批量回归报告.md"
|
||||
|
||||
# 写 JSON 结果
|
||||
result_data = {
|
||||
"run_ts": run_ts,
|
||||
"stats": stats,
|
||||
"closure_special": closure_results,
|
||||
"e2e_before": e2e_before,
|
||||
"results": results,
|
||||
}
|
||||
json_bytes = json.dumps(result_data, ensure_ascii=False, indent=2).encode("utf-8")
|
||||
json_size_mb = len(json_bytes) / 1024 / 1024
|
||||
print(f"\nJSON 结果大小: {json_size_mb:.2f} MB")
|
||||
|
||||
if json_size_mb <= 1.0:
|
||||
json_path.write_bytes(json_bytes)
|
||||
print(f"JSON 已写入: {json_path}")
|
||||
json_path_str = str(json_path)
|
||||
else:
|
||||
# >1MB 不入库,只写摘要
|
||||
summary_path = output_dir / f"batch_regression_{run_ts}_summary.json"
|
||||
summary_data = {
|
||||
"run_ts": run_ts,
|
||||
"stats": stats,
|
||||
"closure_special": closure_results,
|
||||
"e2e_before": e2e_before,
|
||||
"note": f"全量结果 {json_size_mb:.2f}MB 超过 1MB 限制,不入库;此文件仅含统计摘要",
|
||||
}
|
||||
summary_path.write_text(json.dumps(summary_data, ensure_ascii=False, indent=2), encoding="utf-8")
|
||||
print(f"JSON 超过 1MB({json_size_mb:.2f}MB),摘要已写入: {summary_path}")
|
||||
json_path_str = str(summary_path) + f" (full: {json_size_mb:.2f}MB, 不入库)"
|
||||
|
||||
# e2e 基线(批量后)
|
||||
e2e_after = check_e2e_baseline(jwt)
|
||||
if not all(r["passed"] for r in e2e_after):
|
||||
print(" 警告: e2e 基线批量后失败!")
|
||||
else:
|
||||
print(" e2e 基线批量后确认: 全部通过")
|
||||
|
||||
# 写报告
|
||||
report_content = generate_report(stats, results, closure_results, run_ts, json_path_str)
|
||||
report_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
report_path.write_text(report_content, encoding="utf-8")
|
||||
print(f"报告已写入: {report_path}")
|
||||
|
||||
# 打印摘要
|
||||
print("\n" + "=" * 60)
|
||||
print("F2-4 批量回归结果摘要")
|
||||
print("=" * 60)
|
||||
fa = stats["full_auto"]
|
||||
ns = stats["no_s3"]
|
||||
print(f"FULL_AUTO: {fa['passed']}/{fa['passed']+fa['failed']} 通过 "
|
||||
f"({fa['pass_rate']}),排除写路径 {fa['excluded']} 条")
|
||||
print(f"NO_S3: {ns['passed']}/{ns['passed']+ns['failed']} 通过 "
|
||||
f"({ns['pass_rate']}),排除写路径 {ns['excluded']} 条")
|
||||
print(f"闭包动词专项: {sum(1 for r in closure_results if r['verdict']=='PASS')}/{len(closure_results)} 通过")
|
||||
print("\n分桶分布:")
|
||||
for bucket, count in sorted(stats["bucket_counts"].items(), key=lambda x: -x[1]):
|
||||
print(f" {bucket:35s}: {count}")
|
||||
print(f"\n报告: {report_path}")
|
||||
print("=" * 60)
|
||||
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
Reference in New Issue
Block a user