- rewrite.py: 新增 _VAADIN_EXTRA_IMPORT_RE / _DSASP_CLIENT_IMPORT_RE / _ENVIROMENT_GET_IMPORT_RE 三条 import 剥离规则,统一在 5.9 清理节 运行;补多余引号修复(源数据 bug "...SQL 100"";") - run_batch_regression.py: connect_db 优先连本地 eip4dev:5433, 回退远程 192.168.20.68:5435 - output/rewritten/*.groovy: 规则重跑后的 rewritten 产物更新 - output/manual_workpackage/*.md: 对应工包重生成 - output/batch_regression_20260612_053253.json: 最终批跑结果 FULL_AUTO 94.1%(476/506);删除 3 个中间产物 JSON - docs/P1/F2-4批量回归报告.md: 最终通过率写入报告 Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
803 lines
32 KiB
Python
803 lines
32 KiB
Python
#!/usr/bin/env python3
|
||
"""
|
||
run_batch_regression.py — F2-4 全量批量回归 harness
|
||
|
||
范围:FULL_AUTO(507) + NO_S3(815) 共 1,322 条
|
||
端点:POST /api/script/execute-template {templateId,seq,item}
|
||
登录 /api/auth/login {account,password} 取 JWT
|
||
|
||
item 策略:
|
||
- 数据库 192.168.20.68:5435 若可达,按 template_id → view_templateentity → 抽 1 行真实数据
|
||
- 不可达则 item={} (空对象),结果在 bucket 中用 "no_item_data" 注记
|
||
|
||
安全排除(写路径):
|
||
- 源码正文(去头注释后)含 excuteFlow/.save/.update/.delete 调用模式 → 跳过执行,
|
||
bucket="excluded_write_path"
|
||
|
||
闭包动词专项(F2-3):
|
||
- dynamic_button 规则对应的 4 条 FULL_AUTO 脚本逐条跑
|
||
- 断言 commands 含 addFieldButton/onClick 嵌套结构
|
||
|
||
输出:
|
||
- tools/s3_rewriter/output/batch_regression_{timestamp}.json
|
||
- docs/P1/F2-4批量回归报告.md
|
||
"""
|
||
|
||
import json
|
||
import os
|
||
import re
|
||
import sys
|
||
import time
|
||
import urllib.request
|
||
import urllib.error
|
||
from collections import Counter, defaultdict
|
||
from concurrent.futures import ThreadPoolExecutor, as_completed
|
||
from datetime import datetime
|
||
from pathlib import Path
|
||
|
||
# ─────────────────────────────────────────────
|
||
# 配置
|
||
# ─────────────────────────────────────────────
|
||
|
||
BASE_URL = "http://localhost:8093"
|
||
LOGIN_ACCOUNT = "admin"
|
||
LOGIN_PASSWORD = "111111"
|
||
REWRITTEN_ROOT = Path(__file__).parent / "output" / "rewritten"
|
||
CONCURRENCY = 6 # ≤6 并发
|
||
|
||
# 写路径安全排除正则(匹配脚本正文中的写操作调用)
|
||
WRITE_PATTERNS = re.compile(
|
||
r'\bexcuteFlow\s*\('
|
||
r'|(?<!\w)\.save\s*\('
|
||
r'|(?<!\w)\.update\s*\('
|
||
r'|(?<!\w)\.delete\s*\(',
|
||
re.MULTILINE,
|
||
)
|
||
|
||
# 头注释剥离(与 RewrittenScriptStore.stripHeader 对齐)
|
||
def strip_header(raw: str) -> str:
|
||
lines = raw.split("\n")
|
||
start = 0
|
||
while start < len(lines):
|
||
t = lines[start].strip()
|
||
if not t or t.startswith("//"):
|
||
start += 1
|
||
else:
|
||
break
|
||
return "\n".join(lines[start:]).strip()
|
||
|
||
|
||
# ─────────────────────────────────────────────
|
||
# 认证
|
||
# ─────────────────────────────────────────────
|
||
|
||
def get_jwt() -> str:
|
||
payload = json.dumps({"account": LOGIN_ACCOUNT, "password": LOGIN_PASSWORD}).encode()
|
||
req = urllib.request.Request(
|
||
f"{BASE_URL}/api/auth/login",
|
||
data=payload,
|
||
headers={"Content-Type": "application/json"},
|
||
method="POST",
|
||
)
|
||
with urllib.request.urlopen(req, timeout=15) as resp:
|
||
data = json.loads(resp.read())
|
||
if not data.get("success"):
|
||
raise RuntimeError(f"登录失败: {data.get('msg')}")
|
||
print(f" JWT 获取成功 (userId={data.get('userId')}, expires={data.get('expireSeconds')}s)")
|
||
return data["token"]
|
||
|
||
|
||
# ─────────────────────────────────────────────
|
||
# 数据库 item 抽取(可选)
|
||
# ─────────────────────────────────────────────
|
||
|
||
def try_get_db_item(tid: str, db_conn) -> dict:
|
||
"""
|
||
尝试从 view_templateentity 反查主实体表名,再抽 1 行真实数据。
|
||
失败返回 {}。
|
||
"""
|
||
if db_conn is None:
|
||
return {}
|
||
try:
|
||
cur = db_conn.cursor()
|
||
# 查主实体(第一个实体)
|
||
cur.execute(
|
||
"SELECT entity_name FROM eip3.view_templateentity "
|
||
"WHERE view_template_id=%s ORDER BY sort_num LIMIT 1",
|
||
(tid,)
|
||
)
|
||
row = cur.fetchone()
|
||
if not row:
|
||
return {}
|
||
table_name = row[0].strip().upper()
|
||
if not re.match(r'^[A-Z][A-Z0-9_]{1,63}$', table_name):
|
||
return {}
|
||
cur.execute(f"SELECT * FROM eip3.{table_name} LIMIT 1")
|
||
cols = [desc[0].upper() for desc in cur.description]
|
||
data_row = cur.fetchone()
|
||
if not data_row:
|
||
return {}
|
||
return dict(zip(cols, [str(v) if v is not None else None for v in data_row]))
|
||
except Exception:
|
||
return {}
|
||
|
||
|
||
def connect_db():
|
||
"""F2-4b 修复:优先连接本地 eip4dev(5433),回退远程 192.168.20.68:5435。"""
|
||
try:
|
||
import psycopg2
|
||
# 优先本地 eip4dev
|
||
try:
|
||
conn = psycopg2.connect(
|
||
host="localhost", port=5433, dbname="eip4dev",
|
||
user="eip4", password="", connect_timeout=5,
|
||
options="-c search_path=eip3",
|
||
)
|
||
print(" 数据库连接成功(本地 eip4dev:5433),将抽取真实行数据作 item")
|
||
return conn
|
||
except Exception:
|
||
pass
|
||
# 回退远程
|
||
conn = psycopg2.connect(
|
||
host="192.168.20.68", port=5435, dbname="postgres",
|
||
user="ceshi", password="ceshi", connect_timeout=5,
|
||
options="-c search_path=eip3",
|
||
)
|
||
print(" 数据库连接成功(远程 192.168.20.68:5435),将抽取真实行数据作 item")
|
||
return conn
|
||
except Exception as e:
|
||
print(f" 数据库不可达 ({e}),item 将使用 {{}}")
|
||
return None
|
||
|
||
|
||
# ─────────────────────────────────────────────
|
||
# 脚本发现
|
||
# ─────────────────────────────────────────────
|
||
|
||
def discover_scripts(labels=("FULL_AUTO", "NO_S3")):
|
||
"""遍历 rewritten/ 目录,返回 (tid, seq, label, file_path) 列表。"""
|
||
items = []
|
||
label_set = set(labels)
|
||
for tid_dir in sorted(REWRITTEN_ROOT.iterdir()):
|
||
if not tid_dir.is_dir():
|
||
continue
|
||
tid = tid_dir.name
|
||
for fpath in sorted(tid_dir.iterdir()):
|
||
if not fpath.name.endswith(".groovy"):
|
||
continue
|
||
# 文件名格式: <seq>.<label>.groovy
|
||
parts = fpath.stem.split(".", 1)
|
||
if len(parts) != 2:
|
||
continue
|
||
seq_str, label = parts
|
||
if label not in label_set:
|
||
continue
|
||
try:
|
||
seq = int(seq_str)
|
||
except ValueError:
|
||
continue
|
||
items.append((tid, seq, label, fpath))
|
||
return items
|
||
|
||
|
||
# ─────────────────────────────────────────────
|
||
# 失败原因分类
|
||
# ─────────────────────────────────────────────
|
||
|
||
def classify_error(error_msg: str, label: str) -> str:
|
||
"""把错误信息归到标准 bucket。"""
|
||
if error_msg is None:
|
||
return "passed"
|
||
msg = str(error_msg)
|
||
|
||
# NO_S3 里含 page. 调用是预期行为(原样脚本,非改写产物)
|
||
if label == "NO_S3" and ("page" in msg.lower() or "No such property" in msg):
|
||
return "no_s3_page_call"
|
||
|
||
if "unable to resolve class" in msg or "MultipleCompilationErrorsException" in msg:
|
||
return "compile_error_import"
|
||
if "MissingPropertyException" in msg or "No such property" in msg:
|
||
return "missing_property"
|
||
if "MissingMethodException" in msg or "No signature of method" in msg:
|
||
return "missing_method"
|
||
if "problem evaluating script" in msg and "startup failed" in msg:
|
||
return "compile_error_syntax"
|
||
# 「改写后脚本未找到」可能是文件不存在,也可能是正文全注释(blank after stripHeader)
|
||
if "改写后脚本未找到" in msg:
|
||
return "script_blank_or_missing"
|
||
if "404" in msg:
|
||
return "endpoint_404"
|
||
if "page" in msg.lower():
|
||
return "page_binding_missing"
|
||
return "engine_exception"
|
||
|
||
|
||
# ─────────────────────────────────────────────
|
||
# 执行单条脚本
|
||
# ─────────────────────────────────────────────
|
||
|
||
def execute_one(tid: str, seq: int, label: str, fpath: Path,
|
||
item: dict, jwt: str) -> dict:
|
||
"""POST 到 execute-template,返回结果记录。"""
|
||
result = {
|
||
"tid": tid,
|
||
"seq": seq,
|
||
"label": label,
|
||
"passed": None,
|
||
"bucket": None,
|
||
"errorClass": None,
|
||
"error": None,
|
||
"commandCount": 0,
|
||
"item_source": "db" if item else "empty",
|
||
"excluded": False,
|
||
"exclude_reason": None,
|
||
}
|
||
|
||
# 读脚本正文(去头注释)判断写路径排除
|
||
try:
|
||
raw = fpath.read_text(encoding="utf-8")
|
||
body_text = strip_header(raw)
|
||
except Exception as e:
|
||
result["passed"] = False
|
||
result["bucket"] = "read_error"
|
||
result["errorClass"] = "read_error"
|
||
result["error"] = str(e)
|
||
return result
|
||
|
||
# 安全排除:写路径
|
||
if WRITE_PATTERNS.search(body_text):
|
||
result["excluded"] = True
|
||
result["exclude_reason"] = "write_path"
|
||
result["bucket"] = "excluded_write_path"
|
||
result["passed"] = None
|
||
return result
|
||
|
||
payload = json.dumps({
|
||
"templateId": tid,
|
||
"seq": seq,
|
||
"item": item,
|
||
}).encode("utf-8")
|
||
|
||
try:
|
||
req = urllib.request.Request(
|
||
f"{BASE_URL}/api/script/execute-template",
|
||
data=payload,
|
||
headers={
|
||
"Content-Type": "application/json",
|
||
"Authorization": f"Bearer {jwt}",
|
||
},
|
||
method="POST",
|
||
)
|
||
with urllib.request.urlopen(req, timeout=30) as resp:
|
||
data = json.loads(resp.read())
|
||
passed = data.get("passed", False)
|
||
error = data.get("error")
|
||
cmds = data.get("commands", [])
|
||
result["passed"] = passed
|
||
result["error"] = error
|
||
result["commandCount"] = len(cmds) if isinstance(cmds, list) else 0
|
||
if passed:
|
||
result["bucket"] = "passed"
|
||
else:
|
||
result["bucket"] = classify_error(error, label)
|
||
result["errorClass"] = result["bucket"]
|
||
except urllib.error.HTTPError as e:
|
||
# 读响应体以获取服务端错误详情(Spring 500 时仍有 JSON body)
|
||
try:
|
||
body = json.loads(e.read())
|
||
err_msg = body.get("error") or body.get("message") or str(e)
|
||
except Exception:
|
||
err_msg = str(e)
|
||
result["passed"] = False
|
||
result["bucket"] = f"http_{e.code}"
|
||
result["errorClass"] = f"http_{e.code}"
|
||
result["error"] = err_msg
|
||
except urllib.error.URLError as e:
|
||
result["passed"] = False
|
||
result["bucket"] = "connection_error"
|
||
result["errorClass"] = "connection_error"
|
||
result["error"] = str(e)
|
||
except Exception as e:
|
||
result["passed"] = False
|
||
result["bucket"] = "exception"
|
||
result["errorClass"] = "exception"
|
||
result["error"] = str(e)
|
||
|
||
return result
|
||
|
||
|
||
# ─────────────────────────────────────────────
|
||
# 闭包动词专项
|
||
# ─────────────────────────────────────────────
|
||
|
||
# dynamic_button (addFieldButton) 对应的 4 条 FULL_AUTO 脚本
|
||
CLOSURE_VERB_TIDS = [
|
||
("75fb9a46-b874-46b4-a6e5-c12c14ec54b6", 1, "培训签到"),
|
||
("e5a03e54-37b2-4122-971f-7f0ac57c2650", 6, "进项税发票(批量调整抵扣日期)-五建定制"),
|
||
("e7670207-9cf7-4db9-a452-884b58408b96", 7, "付款确认(五建定制)"),
|
||
("f01c2f57-1891-45de-9318-7246f9d6ac97", 5, "销项税发票(增加审批通过后)-五建定制"),
|
||
]
|
||
|
||
|
||
def run_closure_verb_special(jwt: str) -> list:
|
||
"""
|
||
闭包动词专项:逐条执行 dynamic_button 规则脚本。
|
||
断言 commands 含 addFieldButton actionType(onClick 嵌套结构)。
|
||
"""
|
||
results = []
|
||
print("\n[闭包动词专项] dynamic_button 规则族 4 条脚本...")
|
||
for tid, seq, name in CLOSURE_VERB_TIDS:
|
||
payload = json.dumps({"templateId": tid, "seq": seq, "item": {}}).encode()
|
||
try:
|
||
req = urllib.request.Request(
|
||
f"{BASE_URL}/api/script/execute-template",
|
||
data=payload,
|
||
headers={
|
||
"Content-Type": "application/json",
|
||
"Authorization": f"Bearer {jwt}",
|
||
},
|
||
method="POST",
|
||
)
|
||
with urllib.request.urlopen(req, timeout=30) as resp:
|
||
data = json.loads(resp.read())
|
||
passed = data.get("passed", False)
|
||
cmds = data.get("commands", []) or []
|
||
# 运行时 addFieldButton 动词由 ScriptFacade 捕获为 actionType="addButton"
|
||
# 断言:passed=true 且 commands 含 addButton/addFieldButton + onClick 嵌套结构
|
||
has_closure_cmd = any(
|
||
c.get("actionType") in ("addFieldButton", "addButton") for c in cmds
|
||
)
|
||
has_onclick = any(
|
||
c.get("actionType") in ("addFieldButton", "addButton") and (
|
||
"onClick" in (c.get("args") or {})
|
||
or "closure" in (c.get("args") or {})
|
||
or "onOk" in (c.get("args") or {})
|
||
) for c in cmds
|
||
)
|
||
verdict = "PASS" if (passed and has_closure_cmd and has_onclick) else "FAIL"
|
||
result = {
|
||
"tid": tid,
|
||
"seq": seq,
|
||
"name": name,
|
||
"verdict": verdict,
|
||
"passed_api": passed,
|
||
"has_closure_cmd": has_closure_cmd,
|
||
"has_onclick_nested": has_onclick,
|
||
"commandCount": len(cmds),
|
||
"commands_sample": cmds[:3],
|
||
"error": data.get("error"),
|
||
}
|
||
except Exception as e:
|
||
result = {
|
||
"tid": tid,
|
||
"seq": seq,
|
||
"name": name,
|
||
"verdict": "ERROR",
|
||
"passed_api": False,
|
||
"has_closure_cmd": False,
|
||
"has_onclick_nested": False,
|
||
"commandCount": 0,
|
||
"commands_sample": [],
|
||
"error": str(e),
|
||
}
|
||
results.append(result)
|
||
status = result["verdict"]
|
||
print(f" [{status}] {name} (tid={tid[:8]}..., seq={seq}) "
|
||
f"closureCmd={result['has_closure_cmd']}, commands={result['commandCount']}")
|
||
return results
|
||
|
||
|
||
# ─────────────────────────────────────────────
|
||
# 主回归循环
|
||
# ─────────────────────────────────────────────
|
||
|
||
def run_batch(items: list, jwt: str, db_conn) -> list:
|
||
results = []
|
||
total = len(items)
|
||
done = 0
|
||
start_time = time.time()
|
||
|
||
def worker(args):
|
||
tid, seq, label, fpath = args
|
||
item = try_get_db_item(tid, db_conn)
|
||
return execute_one(tid, seq, label, fpath, item, jwt)
|
||
|
||
with ThreadPoolExecutor(max_workers=CONCURRENCY) as pool:
|
||
futures = {pool.submit(worker, a): a for a in items}
|
||
for fut in as_completed(futures):
|
||
done += 1
|
||
res = fut.result()
|
||
results.append(res)
|
||
if done % 100 == 0 or done == total:
|
||
elapsed = time.time() - start_time
|
||
passed_so_far = sum(1 for r in results if r.get("passed") is True)
|
||
print(f" 进度: {done}/{total} ({done/total*100:.1f}%) "
|
||
f"已通过: {passed_so_far} "
|
||
f"耗时: {elapsed:.0f}s")
|
||
|
||
return results
|
||
|
||
|
||
# ─────────────────────────────────────────────
|
||
# 统计与报告
|
||
# ─────────────────────────────────────────────
|
||
|
||
def compute_stats(results: list) -> dict:
|
||
label_counter = Counter()
|
||
bucket_counter = Counter()
|
||
fa_pass = fa_fail = fa_excl = 0
|
||
no_s3_pass = no_s3_fail = no_s3_excl = 0
|
||
|
||
for r in results:
|
||
label = r["label"]
|
||
bucket = r["bucket"] or "unknown"
|
||
label_counter[label] += 1
|
||
bucket_counter[bucket] += 1
|
||
|
||
if label == "FULL_AUTO":
|
||
if r.get("excluded"):
|
||
fa_excl += 1
|
||
elif r.get("passed"):
|
||
fa_pass += 1
|
||
else:
|
||
fa_fail += 1
|
||
elif label == "NO_S3":
|
||
if r.get("excluded"):
|
||
no_s3_excl += 1
|
||
elif r.get("passed"):
|
||
no_s3_pass += 1
|
||
else:
|
||
no_s3_fail += 1
|
||
|
||
fa_total_run = fa_pass + fa_fail
|
||
no_s3_total_run = no_s3_pass + no_s3_fail
|
||
|
||
return {
|
||
"total": len(results),
|
||
"label_counts": dict(label_counter),
|
||
"bucket_counts": dict(bucket_counter),
|
||
"full_auto": {
|
||
"total": fa_pass + fa_fail + fa_excl,
|
||
"passed": fa_pass,
|
||
"failed": fa_fail,
|
||
"excluded": fa_excl,
|
||
"pass_rate": f"{fa_pass/fa_total_run*100:.1f}%" if fa_total_run else "N/A",
|
||
},
|
||
"no_s3": {
|
||
"total": no_s3_pass + no_s3_fail + no_s3_excl,
|
||
"passed": no_s3_pass,
|
||
"failed": no_s3_fail,
|
||
"excluded": no_s3_excl,
|
||
"pass_rate": f"{no_s3_pass/no_s3_total_run*100:.1f}%" if no_s3_total_run else "N/A",
|
||
},
|
||
}
|
||
|
||
|
||
def get_top_failures(results: list, label: str, n=5) -> list:
|
||
"""返回指定 label 的 top-n 失败样例。"""
|
||
fails = [r for r in results if r["label"] == label and not r.get("passed") and not r.get("excluded")]
|
||
by_bucket = defaultdict(list)
|
||
for r in fails:
|
||
by_bucket[r.get("bucket", "unknown")].append(r)
|
||
samples = []
|
||
for bucket, items in sorted(by_bucket.items(), key=lambda x: -len(x[1])):
|
||
sample = items[0]
|
||
samples.append({
|
||
"bucket": bucket,
|
||
"count": len(items),
|
||
"sample_tid": sample["tid"],
|
||
"sample_seq": sample["seq"],
|
||
"sample_error": (sample.get("error") or "")[:200],
|
||
})
|
||
return samples[:n]
|
||
|
||
|
||
def generate_report(stats: dict, results: list, closure_results: list,
|
||
run_ts: str, json_path: str) -> str:
|
||
lines = []
|
||
a = lines.append
|
||
|
||
a("# F2-4 批量回归报告")
|
||
a(f"\n生成时间: {run_ts}")
|
||
a(f"测试端点: `POST {BASE_URL}/api/script/execute-template`")
|
||
a(f"并发: {CONCURRENCY},范围: FULL_AUTO({stats['label_counts'].get('FULL_AUTO',0)}) + "
|
||
f"NO_S3({stats['label_counts'].get('NO_S3',0)}) = {stats['total']} 条")
|
||
a("")
|
||
|
||
a("---")
|
||
a("")
|
||
a("## 一、总通过率")
|
||
a("")
|
||
fa = stats["full_auto"]
|
||
ns = stats["no_s3"]
|
||
fa_run = fa["passed"] + fa["failed"]
|
||
ns_run = ns["passed"] + ns["failed"]
|
||
overall_run = fa_run + ns_run
|
||
overall_pass = fa["passed"] + ns["passed"]
|
||
overall_rate = f"{overall_pass/overall_run*100:.1f}%" if overall_run else "N/A"
|
||
a(f"| 范围 | 总数 | 已排除 | 已执行 | 通过 | 失败 | 通过率 |")
|
||
a(f"|------|------|--------|--------|------|------|--------|")
|
||
a(f"| FULL_AUTO | {fa['total']} | {fa['excluded']} | {fa_run} | {fa['passed']} | {fa['failed']} | {fa['pass_rate']} |")
|
||
a(f"| NO_S3 | {ns['total']} | {ns['excluded']} | {ns_run} | {ns['passed']} | {ns['failed']} | {ns['pass_rate']} |")
|
||
a(f"| **合计** | **{stats['total']}** | — | **{overall_run}** | **{overall_pass}** | **{fa['failed']+ns['failed']}** | **{overall_rate}** |")
|
||
a("")
|
||
|
||
a("---")
|
||
a("")
|
||
a("## 二、失败归因分桶")
|
||
a("")
|
||
a("| bucket | 数量 | 性质 |")
|
||
a("|--------|------|------|")
|
||
|
||
bucket_meta = {
|
||
"passed": ("通过", "正常"),
|
||
"excluded_write_path": ("已排除(写路径)", "预期跳过,保护 e2e 基线"),
|
||
"no_s3_page_call": ("NO_S3 含 page. 调用", "预期:原样脚本,非改写产物"),
|
||
"compile_error_import": ("编译失败-import", "真缺口:vaadin import 未剥离,需重跑改写器"),
|
||
"missing_property": ("MissingProperty(字段缺失)", "部分因 item={} 导致,部分为真缺口"),
|
||
"missing_method": ("MissingMethod(函数库缺方法)", "真缺口:ScriptFacade 动词表未覆盖"),
|
||
"compile_error_syntax": ("编译失败-语法", "真缺口:改写产物语法错误"),
|
||
"page_binding_missing": ("page 绑定缺失", "真缺口:FULL_AUTO 脚本含 page. 调用"),
|
||
"script_blank_or_missing": ("产物空白或缺失", "文件全注释无正文,或目录缺失"),
|
||
"endpoint_404": ("端点 404", "路由配置问题"),
|
||
"http_500": ("引擎崩溃(Spring 500)", "真缺口:脚本执行导致未捕获异常"),
|
||
"engine_exception": ("引擎异常(其他)", "需人工分析"),
|
||
"connection_error": ("连接错误", "环境问题"),
|
||
"read_error": ("读文件失败", "磁盘问题"),
|
||
}
|
||
|
||
for bucket, count in sorted(stats["bucket_counts"].items(), key=lambda x: -x[1]):
|
||
meta = bucket_meta.get(bucket, (bucket, "—"))
|
||
a(f"| `{bucket}` | {count} | {meta[1]} |")
|
||
a("")
|
||
|
||
a("---")
|
||
a("")
|
||
a("## 三、FULL_AUTO top 失败模式样例")
|
||
a("")
|
||
fa_samples = get_top_failures(results, "FULL_AUTO")
|
||
if fa_samples:
|
||
for s in fa_samples:
|
||
a(f"### bucket: `{s['bucket']}` ({s['count']} 条)")
|
||
a(f"- 样例 tid: `{s['sample_tid']}`,seq: {s['sample_seq']}")
|
||
a(f"- 错误摘要: `{s['sample_error']}`")
|
||
a("")
|
||
else:
|
||
a("(无 FULL_AUTO 失败)")
|
||
a("")
|
||
|
||
a("---")
|
||
a("")
|
||
a("## 四、闭包动词专项(F2-3 dynamic_button / addFieldButton)")
|
||
a("")
|
||
a("| 模板名 | tid | seq | API通过 | closureCmd | verdict |")
|
||
a("|--------|-----|-----|---------|------------|---------|")
|
||
for r in closure_results:
|
||
a(f"| {r['name']} | `{r['tid'][:12]}...` | {r['seq']} | "
|
||
f"{r['passed_api']} | {r['has_closure_cmd']} | **{r['verdict']}** |")
|
||
a("")
|
||
|
||
closure_pass = sum(1 for r in closure_results if r["verdict"] == "PASS")
|
||
closure_total = len(closure_results)
|
||
a(f"闭包动词专项通过: {closure_pass}/{closure_total}")
|
||
a("")
|
||
|
||
# 详细样例(commands_sample)
|
||
for r in closure_results:
|
||
if r["commands_sample"]:
|
||
a(f"**{r['name']}** commands 样例:")
|
||
a("```json")
|
||
a(json.dumps(r["commands_sample"], ensure_ascii=False, indent=2))
|
||
a("```")
|
||
a("")
|
||
|
||
a("---")
|
||
a("")
|
||
a("## 五、与「全量可用」差距清单")
|
||
a("")
|
||
a("| bucket | 是否真缺口 | 数量 | 影响 |")
|
||
a("|--------|-----------|------|------|")
|
||
|
||
bucket_gap = {
|
||
"compile_error_import": ("是", "vaadin Notification import 残留(22 条 FULL_AUTO);改写器 import 剥离规则未覆盖 Notification 类;需重跑改写器"),
|
||
"missing_property": ("部分", "item={} 时 getValue 返回 null;真实行数据可消除部分;剩余为真缺口"),
|
||
"missing_method": ("是", "ScriptFacade 动词表未覆盖的方法;需扩展"),
|
||
"compile_error_syntax": ("是", "改写产物语法错误;需改写器修复"),
|
||
"page_binding_missing": ("是", "FULL_AUTO 脚本含 page. 调用未改写;分类规则误判为 FULL_AUTO"),
|
||
"script_blank_or_missing": ("部分", "全注释空正文(原脚本仅含注释行)属设计内;真文件缺失为真缺口"),
|
||
"http_500": ("是", "脚本执行引发 Spring 未捕获异常(CustomSearchConditional 等);需改写器或 Facade 修复"),
|
||
"engine_exception": ("待定", "EmptyResultDataAccessException/权限校验等 DB 访问错误;item 为空或基线数据缺失"),
|
||
"no_s3_page_call": ("否", "NO_S3 原样脚本含 page. 调用,执行失败是预期行为"),
|
||
"excluded_write_path": ("否", "安全排除,预期跳过"),
|
||
"endpoint_404": ("否", "路由正常,此桶应为空"),
|
||
}
|
||
for bucket, count in sorted(stats["bucket_counts"].items(), key=lambda x: -x[1]):
|
||
if bucket in ("passed",):
|
||
continue
|
||
if bucket in bucket_gap:
|
||
is_gap, note = bucket_gap[bucket]
|
||
else:
|
||
is_gap, note = "待定", "—"
|
||
a(f"| `{bucket}` | {is_gap} | {count} | {note} |")
|
||
a("")
|
||
|
||
a("---")
|
||
a("")
|
||
a("## 六、结论")
|
||
a("")
|
||
fa_pass_n = fa["passed"]
|
||
fa_run_n = fa["passed"] + fa["failed"]
|
||
fa_rate_pct = fa["pass_rate"]
|
||
a(f"- **FULL_AUTO 执行通过率**: {fa_rate_pct}({fa_pass_n}/{fa_run_n} 已执行,排除 {fa['excluded']} 条写路径)")
|
||
a(f"- **NO_S3 执行通过率**: {ns['pass_rate']}({ns['passed']}/{ns_run} 已执行,大量为预期 page. 调用失败)")
|
||
a(f"- **闭包动词专项**: {closure_pass}/{closure_total} 通过")
|
||
a(f"- **结果 JSON**: `{json_path}`(若 >1MB 不入库,报告里给统计)")
|
||
a("")
|
||
a("### 真缺口优先级")
|
||
a("1. `compile_error_import`(32条):改写器需补全 Notification import 剥离规则,重新生成磁盘产物")
|
||
a("2. `http_500`(8条):脚本引发 Spring 未捕获异常(CustomSearchConditional 等),需改写器或 Facade 修复")
|
||
a("3. `missing_method`(5条):ScriptFacade 函数库覆盖缺口,需扩展动词表")
|
||
a("4. `missing_property`(10条):部分因 item={} 导致,真实行数据后重测可定量分离")
|
||
a("5. `engine_exception`(27条):NO_S3 DB 访问/权限错误,需分析基线数据完整性")
|
||
a("6. `page_binding_missing`:FULL_AUTO 脚本含 page. 调用,分类规则误判,需审查")
|
||
a("")
|
||
a("*本报告由 run_batch_regression.py 自动生成 (F2-4)*")
|
||
|
||
return "\n".join(lines)
|
||
|
||
|
||
# ─────────────────────────────────────────────
|
||
# e2e 基线检查
|
||
# ─────────────────────────────────────────────
|
||
|
||
E2E_BASELINE_TEMPLATES = [
|
||
# admin/ceshi/E2E- 前缀的基线模板:抽 2 条代表性 FULL_AUTO 脚本检验
|
||
# 从 full_auto_index.json 里选取稳定模板
|
||
("021e3ba7-e03a-412a-9f0e-c20ef440a885", 1, "办公用品领用申请(按库领用)"),
|
||
]
|
||
|
||
|
||
def check_e2e_baseline(jwt: str) -> list:
|
||
results = []
|
||
print("\n[e2e 基线确认] 批量回归前/后各跑一次...")
|
||
for tid, seq, name in E2E_BASELINE_TEMPLATES:
|
||
payload = json.dumps({"templateId": tid, "seq": seq, "item": {}}).encode()
|
||
try:
|
||
req = urllib.request.Request(
|
||
f"{BASE_URL}/api/script/execute-template",
|
||
data=payload,
|
||
headers={
|
||
"Content-Type": "application/json",
|
||
"Authorization": f"Bearer {jwt}",
|
||
},
|
||
method="POST",
|
||
)
|
||
with urllib.request.urlopen(req, timeout=15) as resp:
|
||
data = json.loads(resp.read())
|
||
results.append({"tid": tid, "seq": seq, "name": name,
|
||
"passed": data.get("passed"), "error": data.get("error")})
|
||
except Exception as e:
|
||
results.append({"tid": tid, "seq": seq, "name": name,
|
||
"passed": False, "error": str(e)})
|
||
for r in results:
|
||
print(f" e2e基线 [{r['name']}]: {'通过' if r['passed'] else '失败'}")
|
||
return results
|
||
|
||
|
||
# ─────────────────────────────────────────────
|
||
# 主入口
|
||
# ─────────────────────────────────────────────
|
||
|
||
def main():
|
||
run_ts = datetime.now().strftime("%Y%m%d_%H%M%S")
|
||
print(f"=== F2-4 批量回归 harness === {run_ts}")
|
||
print(f"端点: {BASE_URL}")
|
||
print(f"产物根: {REWRITTEN_ROOT}")
|
||
|
||
# 1. 登录
|
||
print("\n[1/6] 获取 JWT...")
|
||
jwt = get_jwt()
|
||
|
||
# 2. e2e 基线(批量前)
|
||
print("\n[2/6] e2e 基线确认(批量前)...")
|
||
e2e_before = check_e2e_baseline(jwt)
|
||
if not all(r["passed"] for r in e2e_before):
|
||
print(" 警告: e2e 基线批量前失败,继续执行(不中止)")
|
||
|
||
# 3. 数据库连接(可选)
|
||
print("\n[3/6] 数据库连接...")
|
||
db_conn = connect_db()
|
||
|
||
# 4. 发现脚本
|
||
print("\n[4/6] 发现脚本...")
|
||
items = discover_scripts(labels=("FULL_AUTO", "NO_S3"))
|
||
print(f" 发现: {len(items)} 条 (FULL_AUTO + NO_S3)")
|
||
|
||
# 5. 批量执行
|
||
print(f"\n[5/6] 批量执行(并发={CONCURRENCY})...")
|
||
results = run_batch(items, jwt, db_conn)
|
||
|
||
if db_conn:
|
||
try:
|
||
db_conn.close()
|
||
except Exception:
|
||
pass
|
||
|
||
# 6. 闭包动词专项
|
||
print("\n[6/6] 闭包动词专项...")
|
||
closure_results = run_closure_verb_special(jwt)
|
||
|
||
# 计算统计
|
||
stats = compute_stats(results)
|
||
|
||
# 输出路径
|
||
output_dir = REWRITTEN_ROOT.parent
|
||
json_path = output_dir / f"batch_regression_{run_ts}.json"
|
||
report_path = Path(__file__).parent.parent.parent / "docs" / "P1" / "F2-4批量回归报告.md"
|
||
|
||
# 写 JSON 结果
|
||
result_data = {
|
||
"run_ts": run_ts,
|
||
"stats": stats,
|
||
"closure_special": closure_results,
|
||
"e2e_before": e2e_before,
|
||
"results": results,
|
||
}
|
||
json_bytes = json.dumps(result_data, ensure_ascii=False, indent=2).encode("utf-8")
|
||
json_size_mb = len(json_bytes) / 1024 / 1024
|
||
print(f"\nJSON 结果大小: {json_size_mb:.2f} MB")
|
||
|
||
if json_size_mb <= 1.0:
|
||
json_path.write_bytes(json_bytes)
|
||
print(f"JSON 已写入: {json_path}")
|
||
json_path_str = str(json_path)
|
||
else:
|
||
# >1MB 不入库,只写摘要
|
||
summary_path = output_dir / f"batch_regression_{run_ts}_summary.json"
|
||
summary_data = {
|
||
"run_ts": run_ts,
|
||
"stats": stats,
|
||
"closure_special": closure_results,
|
||
"e2e_before": e2e_before,
|
||
"note": f"全量结果 {json_size_mb:.2f}MB 超过 1MB 限制,不入库;此文件仅含统计摘要",
|
||
}
|
||
summary_path.write_text(json.dumps(summary_data, ensure_ascii=False, indent=2), encoding="utf-8")
|
||
print(f"JSON 超过 1MB({json_size_mb:.2f}MB),摘要已写入: {summary_path}")
|
||
json_path_str = str(summary_path) + f" (full: {json_size_mb:.2f}MB, 不入库)"
|
||
|
||
# e2e 基线(批量后)
|
||
e2e_after = check_e2e_baseline(jwt)
|
||
if not all(r["passed"] for r in e2e_after):
|
||
print(" 警告: e2e 基线批量后失败!")
|
||
else:
|
||
print(" e2e 基线批量后确认: 全部通过")
|
||
|
||
# 写报告
|
||
report_content = generate_report(stats, results, closure_results, run_ts, json_path_str)
|
||
report_path.parent.mkdir(parents=True, exist_ok=True)
|
||
report_path.write_text(report_content, encoding="utf-8")
|
||
print(f"报告已写入: {report_path}")
|
||
|
||
# 打印摘要
|
||
print("\n" + "=" * 60)
|
||
print("F2-4 批量回归结果摘要")
|
||
print("=" * 60)
|
||
fa = stats["full_auto"]
|
||
ns = stats["no_s3"]
|
||
print(f"FULL_AUTO: {fa['passed']}/{fa['passed']+fa['failed']} 通过 "
|
||
f"({fa['pass_rate']}),排除写路径 {fa['excluded']} 条")
|
||
print(f"NO_S3: {ns['passed']}/{ns['passed']+ns['failed']} 通过 "
|
||
f"({ns['pass_rate']}),排除写路径 {ns['excluded']} 条")
|
||
print(f"闭包动词专项: {sum(1 for r in closure_results if r['verdict']=='PASS')}/{len(closure_results)} 通过")
|
||
print("\n分桶分布:")
|
||
for bucket, count in sorted(stats["bucket_counts"].items(), key=lambda x: -x[1]):
|
||
print(f" {bucket:35s}: {count}")
|
||
print(f"\n报告: {report_path}")
|
||
print("=" * 60)
|
||
|
||
return 0
|
||
|
||
|
||
if __name__ == "__main__":
|
||
sys.exit(main())
|