Files
eip4-platform/tools/s3_rewriter/run_batch_regression.py
zhengwr aa576ea2c6 F2-4b: rewrite 规则扩展 + item 改进 + 最终批量结果
- rewrite.py: 新增 _VAADIN_EXTRA_IMPORT_RE / _DSASP_CLIENT_IMPORT_RE /
  _ENVIROMENT_GET_IMPORT_RE 三条 import 剥离规则,统一在 5.9 清理节
  运行;补多余引号修复(源数据 bug "...SQL 100"";")
- run_batch_regression.py: connect_db 优先连本地 eip4dev:5433,
  回退远程 192.168.20.68:5435
- output/rewritten/*.groovy: 规则重跑后的 rewritten 产物更新
- output/manual_workpackage/*.md: 对应工包重生成
- output/batch_regression_20260612_053253.json: 最终批跑结果
  FULL_AUTO 94.1%(476/506);删除 3 个中间产物 JSON
- docs/P1/F2-4批量回归报告.md: 最终通过率写入报告

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
2026-06-12 05:33:41 +08:00

803 lines
32 KiB
Python
Raw Permalink Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
#!/usr/bin/env python3
"""
run_batch_regression.py — F2-4 全量批量回归 harness
范围:FULL_AUTO(507) + NO_S3(815) 共 1,322 条
端点:POST /api/script/execute-template {templateId,seq,item}
登录 /api/auth/login {account,password} 取 JWT
item 策略:
- 数据库 192.168.20.68:5435 若可达,按 template_id → view_templateentity → 抽 1 行真实数据
- 不可达则 item={} (空对象),结果在 bucket 中用 "no_item_data" 注记
安全排除(写路径):
- 源码正文(去头注释后)含 excuteFlow/.save/.update/.delete 调用模式 → 跳过执行,
bucket="excluded_write_path"
闭包动词专项(F2-3):
- dynamic_button 规则对应的 4 条 FULL_AUTO 脚本逐条跑
- 断言 commands 含 addFieldButton/onClick 嵌套结构
输出:
- tools/s3_rewriter/output/batch_regression_{timestamp}.json
- docs/P1/F2-4批量回归报告.md
"""
import json
import os
import re
import sys
import time
import urllib.request
import urllib.error
from collections import Counter, defaultdict
from concurrent.futures import ThreadPoolExecutor, as_completed
from datetime import datetime
from pathlib import Path
# ─────────────────────────────────────────────
# 配置
# ─────────────────────────────────────────────
BASE_URL = "http://localhost:8093"
LOGIN_ACCOUNT = "admin"
LOGIN_PASSWORD = "111111"
REWRITTEN_ROOT = Path(__file__).parent / "output" / "rewritten"
CONCURRENCY = 6 # ≤6 并发
# 写路径安全排除正则(匹配脚本正文中的写操作调用)
WRITE_PATTERNS = re.compile(
r'\bexcuteFlow\s*\('
r'|(?<!\w)\.save\s*\('
r'|(?<!\w)\.update\s*\('
r'|(?<!\w)\.delete\s*\(',
re.MULTILINE,
)
# 头注释剥离(与 RewrittenScriptStore.stripHeader 对齐)
def strip_header(raw: str) -> str:
lines = raw.split("\n")
start = 0
while start < len(lines):
t = lines[start].strip()
if not t or t.startswith("//"):
start += 1
else:
break
return "\n".join(lines[start:]).strip()
# ─────────────────────────────────────────────
# 认证
# ─────────────────────────────────────────────
def get_jwt() -> str:
payload = json.dumps({"account": LOGIN_ACCOUNT, "password": LOGIN_PASSWORD}).encode()
req = urllib.request.Request(
f"{BASE_URL}/api/auth/login",
data=payload,
headers={"Content-Type": "application/json"},
method="POST",
)
with urllib.request.urlopen(req, timeout=15) as resp:
data = json.loads(resp.read())
if not data.get("success"):
raise RuntimeError(f"登录失败: {data.get('msg')}")
print(f" JWT 获取成功 (userId={data.get('userId')}, expires={data.get('expireSeconds')}s)")
return data["token"]
# ─────────────────────────────────────────────
# 数据库 item 抽取(可选)
# ─────────────────────────────────────────────
def try_get_db_item(tid: str, db_conn) -> dict:
"""
尝试从 view_templateentity 反查主实体表名,再抽 1 行真实数据。
失败返回 {}。
"""
if db_conn is None:
return {}
try:
cur = db_conn.cursor()
# 查主实体(第一个实体)
cur.execute(
"SELECT entity_name FROM eip3.view_templateentity "
"WHERE view_template_id=%s ORDER BY sort_num LIMIT 1",
(tid,)
)
row = cur.fetchone()
if not row:
return {}
table_name = row[0].strip().upper()
if not re.match(r'^[A-Z][A-Z0-9_]{1,63}$', table_name):
return {}
cur.execute(f"SELECT * FROM eip3.{table_name} LIMIT 1")
cols = [desc[0].upper() for desc in cur.description]
data_row = cur.fetchone()
if not data_row:
return {}
return dict(zip(cols, [str(v) if v is not None else None for v in data_row]))
except Exception:
return {}
def connect_db():
"""F2-4b 修复:优先连接本地 eip4dev(5433),回退远程 192.168.20.68:5435。"""
try:
import psycopg2
# 优先本地 eip4dev
try:
conn = psycopg2.connect(
host="localhost", port=5433, dbname="eip4dev",
user="eip4", password="", connect_timeout=5,
options="-c search_path=eip3",
)
print(" 数据库连接成功(本地 eip4dev:5433),将抽取真实行数据作 item")
return conn
except Exception:
pass
# 回退远程
conn = psycopg2.connect(
host="192.168.20.68", port=5435, dbname="postgres",
user="ceshi", password="ceshi", connect_timeout=5,
options="-c search_path=eip3",
)
print(" 数据库连接成功(远程 192.168.20.68:5435),将抽取真实行数据作 item")
return conn
except Exception as e:
print(f" 数据库不可达 ({e}),item 将使用 {{}}")
return None
# ─────────────────────────────────────────────
# 脚本发现
# ─────────────────────────────────────────────
def discover_scripts(labels=("FULL_AUTO", "NO_S3")):
"""遍历 rewritten/ 目录,返回 (tid, seq, label, file_path) 列表。"""
items = []
label_set = set(labels)
for tid_dir in sorted(REWRITTEN_ROOT.iterdir()):
if not tid_dir.is_dir():
continue
tid = tid_dir.name
for fpath in sorted(tid_dir.iterdir()):
if not fpath.name.endswith(".groovy"):
continue
# 文件名格式: <seq>.<label>.groovy
parts = fpath.stem.split(".", 1)
if len(parts) != 2:
continue
seq_str, label = parts
if label not in label_set:
continue
try:
seq = int(seq_str)
except ValueError:
continue
items.append((tid, seq, label, fpath))
return items
# ─────────────────────────────────────────────
# 失败原因分类
# ─────────────────────────────────────────────
def classify_error(error_msg: str, label: str) -> str:
"""把错误信息归到标准 bucket。"""
if error_msg is None:
return "passed"
msg = str(error_msg)
# NO_S3 里含 page. 调用是预期行为(原样脚本,非改写产物)
if label == "NO_S3" and ("page" in msg.lower() or "No such property" in msg):
return "no_s3_page_call"
if "unable to resolve class" in msg or "MultipleCompilationErrorsException" in msg:
return "compile_error_import"
if "MissingPropertyException" in msg or "No such property" in msg:
return "missing_property"
if "MissingMethodException" in msg or "No signature of method" in msg:
return "missing_method"
if "problem evaluating script" in msg and "startup failed" in msg:
return "compile_error_syntax"
# 「改写后脚本未找到」可能是文件不存在,也可能是正文全注释(blank after stripHeader)
if "改写后脚本未找到" in msg:
return "script_blank_or_missing"
if "404" in msg:
return "endpoint_404"
if "page" in msg.lower():
return "page_binding_missing"
return "engine_exception"
# ─────────────────────────────────────────────
# 执行单条脚本
# ─────────────────────────────────────────────
def execute_one(tid: str, seq: int, label: str, fpath: Path,
item: dict, jwt: str) -> dict:
"""POST 到 execute-template,返回结果记录。"""
result = {
"tid": tid,
"seq": seq,
"label": label,
"passed": None,
"bucket": None,
"errorClass": None,
"error": None,
"commandCount": 0,
"item_source": "db" if item else "empty",
"excluded": False,
"exclude_reason": None,
}
# 读脚本正文(去头注释)判断写路径排除
try:
raw = fpath.read_text(encoding="utf-8")
body_text = strip_header(raw)
except Exception as e:
result["passed"] = False
result["bucket"] = "read_error"
result["errorClass"] = "read_error"
result["error"] = str(e)
return result
# 安全排除:写路径
if WRITE_PATTERNS.search(body_text):
result["excluded"] = True
result["exclude_reason"] = "write_path"
result["bucket"] = "excluded_write_path"
result["passed"] = None
return result
payload = json.dumps({
"templateId": tid,
"seq": seq,
"item": item,
}).encode("utf-8")
try:
req = urllib.request.Request(
f"{BASE_URL}/api/script/execute-template",
data=payload,
headers={
"Content-Type": "application/json",
"Authorization": f"Bearer {jwt}",
},
method="POST",
)
with urllib.request.urlopen(req, timeout=30) as resp:
data = json.loads(resp.read())
passed = data.get("passed", False)
error = data.get("error")
cmds = data.get("commands", [])
result["passed"] = passed
result["error"] = error
result["commandCount"] = len(cmds) if isinstance(cmds, list) else 0
if passed:
result["bucket"] = "passed"
else:
result["bucket"] = classify_error(error, label)
result["errorClass"] = result["bucket"]
except urllib.error.HTTPError as e:
# 读响应体以获取服务端错误详情(Spring 500 时仍有 JSON body)
try:
body = json.loads(e.read())
err_msg = body.get("error") or body.get("message") or str(e)
except Exception:
err_msg = str(e)
result["passed"] = False
result["bucket"] = f"http_{e.code}"
result["errorClass"] = f"http_{e.code}"
result["error"] = err_msg
except urllib.error.URLError as e:
result["passed"] = False
result["bucket"] = "connection_error"
result["errorClass"] = "connection_error"
result["error"] = str(e)
except Exception as e:
result["passed"] = False
result["bucket"] = "exception"
result["errorClass"] = "exception"
result["error"] = str(e)
return result
# ─────────────────────────────────────────────
# 闭包动词专项
# ─────────────────────────────────────────────
# dynamic_button (addFieldButton) 对应的 4 条 FULL_AUTO 脚本
CLOSURE_VERB_TIDS = [
("75fb9a46-b874-46b4-a6e5-c12c14ec54b6", 1, "培训签到"),
("e5a03e54-37b2-4122-971f-7f0ac57c2650", 6, "进项税发票(批量调整抵扣日期)-五建定制"),
("e7670207-9cf7-4db9-a452-884b58408b96", 7, "付款确认(五建定制)"),
("f01c2f57-1891-45de-9318-7246f9d6ac97", 5, "销项税发票(增加审批通过后)-五建定制"),
]
def run_closure_verb_special(jwt: str) -> list:
"""
闭包动词专项:逐条执行 dynamic_button 规则脚本。
断言 commands 含 addFieldButton actionType(onClick 嵌套结构)。
"""
results = []
print("\n[闭包动词专项] dynamic_button 规则族 4 条脚本...")
for tid, seq, name in CLOSURE_VERB_TIDS:
payload = json.dumps({"templateId": tid, "seq": seq, "item": {}}).encode()
try:
req = urllib.request.Request(
f"{BASE_URL}/api/script/execute-template",
data=payload,
headers={
"Content-Type": "application/json",
"Authorization": f"Bearer {jwt}",
},
method="POST",
)
with urllib.request.urlopen(req, timeout=30) as resp:
data = json.loads(resp.read())
passed = data.get("passed", False)
cmds = data.get("commands", []) or []
# 运行时 addFieldButton 动词由 ScriptFacade 捕获为 actionType="addButton"
# 断言:passed=true 且 commands 含 addButton/addFieldButton + onClick 嵌套结构
has_closure_cmd = any(
c.get("actionType") in ("addFieldButton", "addButton") for c in cmds
)
has_onclick = any(
c.get("actionType") in ("addFieldButton", "addButton") and (
"onClick" in (c.get("args") or {})
or "closure" in (c.get("args") or {})
or "onOk" in (c.get("args") or {})
) for c in cmds
)
verdict = "PASS" if (passed and has_closure_cmd and has_onclick) else "FAIL"
result = {
"tid": tid,
"seq": seq,
"name": name,
"verdict": verdict,
"passed_api": passed,
"has_closure_cmd": has_closure_cmd,
"has_onclick_nested": has_onclick,
"commandCount": len(cmds),
"commands_sample": cmds[:3],
"error": data.get("error"),
}
except Exception as e:
result = {
"tid": tid,
"seq": seq,
"name": name,
"verdict": "ERROR",
"passed_api": False,
"has_closure_cmd": False,
"has_onclick_nested": False,
"commandCount": 0,
"commands_sample": [],
"error": str(e),
}
results.append(result)
status = result["verdict"]
print(f" [{status}] {name} (tid={tid[:8]}..., seq={seq}) "
f"closureCmd={result['has_closure_cmd']}, commands={result['commandCount']}")
return results
# ─────────────────────────────────────────────
# 主回归循环
# ─────────────────────────────────────────────
def run_batch(items: list, jwt: str, db_conn) -> list:
results = []
total = len(items)
done = 0
start_time = time.time()
def worker(args):
tid, seq, label, fpath = args
item = try_get_db_item(tid, db_conn)
return execute_one(tid, seq, label, fpath, item, jwt)
with ThreadPoolExecutor(max_workers=CONCURRENCY) as pool:
futures = {pool.submit(worker, a): a for a in items}
for fut in as_completed(futures):
done += 1
res = fut.result()
results.append(res)
if done % 100 == 0 or done == total:
elapsed = time.time() - start_time
passed_so_far = sum(1 for r in results if r.get("passed") is True)
print(f" 进度: {done}/{total} ({done/total*100:.1f}%) "
f"已通过: {passed_so_far} "
f"耗时: {elapsed:.0f}s")
return results
# ─────────────────────────────────────────────
# 统计与报告
# ─────────────────────────────────────────────
def compute_stats(results: list) -> dict:
label_counter = Counter()
bucket_counter = Counter()
fa_pass = fa_fail = fa_excl = 0
no_s3_pass = no_s3_fail = no_s3_excl = 0
for r in results:
label = r["label"]
bucket = r["bucket"] or "unknown"
label_counter[label] += 1
bucket_counter[bucket] += 1
if label == "FULL_AUTO":
if r.get("excluded"):
fa_excl += 1
elif r.get("passed"):
fa_pass += 1
else:
fa_fail += 1
elif label == "NO_S3":
if r.get("excluded"):
no_s3_excl += 1
elif r.get("passed"):
no_s3_pass += 1
else:
no_s3_fail += 1
fa_total_run = fa_pass + fa_fail
no_s3_total_run = no_s3_pass + no_s3_fail
return {
"total": len(results),
"label_counts": dict(label_counter),
"bucket_counts": dict(bucket_counter),
"full_auto": {
"total": fa_pass + fa_fail + fa_excl,
"passed": fa_pass,
"failed": fa_fail,
"excluded": fa_excl,
"pass_rate": f"{fa_pass/fa_total_run*100:.1f}%" if fa_total_run else "N/A",
},
"no_s3": {
"total": no_s3_pass + no_s3_fail + no_s3_excl,
"passed": no_s3_pass,
"failed": no_s3_fail,
"excluded": no_s3_excl,
"pass_rate": f"{no_s3_pass/no_s3_total_run*100:.1f}%" if no_s3_total_run else "N/A",
},
}
def get_top_failures(results: list, label: str, n=5) -> list:
"""返回指定 label 的 top-n 失败样例。"""
fails = [r for r in results if r["label"] == label and not r.get("passed") and not r.get("excluded")]
by_bucket = defaultdict(list)
for r in fails:
by_bucket[r.get("bucket", "unknown")].append(r)
samples = []
for bucket, items in sorted(by_bucket.items(), key=lambda x: -len(x[1])):
sample = items[0]
samples.append({
"bucket": bucket,
"count": len(items),
"sample_tid": sample["tid"],
"sample_seq": sample["seq"],
"sample_error": (sample.get("error") or "")[:200],
})
return samples[:n]
def generate_report(stats: dict, results: list, closure_results: list,
run_ts: str, json_path: str) -> str:
lines = []
a = lines.append
a("# F2-4 批量回归报告")
a(f"\n生成时间: {run_ts}")
a(f"测试端点: `POST {BASE_URL}/api/script/execute-template`")
a(f"并发: {CONCURRENCY},范围: FULL_AUTO({stats['label_counts'].get('FULL_AUTO',0)}) + "
f"NO_S3({stats['label_counts'].get('NO_S3',0)}) = {stats['total']} 条")
a("")
a("---")
a("")
a("## 一、总通过率")
a("")
fa = stats["full_auto"]
ns = stats["no_s3"]
fa_run = fa["passed"] + fa["failed"]
ns_run = ns["passed"] + ns["failed"]
overall_run = fa_run + ns_run
overall_pass = fa["passed"] + ns["passed"]
overall_rate = f"{overall_pass/overall_run*100:.1f}%" if overall_run else "N/A"
a(f"| 范围 | 总数 | 已排除 | 已执行 | 通过 | 失败 | 通过率 |")
a(f"|------|------|--------|--------|------|------|--------|")
a(f"| FULL_AUTO | {fa['total']} | {fa['excluded']} | {fa_run} | {fa['passed']} | {fa['failed']} | {fa['pass_rate']} |")
a(f"| NO_S3 | {ns['total']} | {ns['excluded']} | {ns_run} | {ns['passed']} | {ns['failed']} | {ns['pass_rate']} |")
a(f"| **合计** | **{stats['total']}** | — | **{overall_run}** | **{overall_pass}** | **{fa['failed']+ns['failed']}** | **{overall_rate}** |")
a("")
a("---")
a("")
a("## 二、失败归因分桶")
a("")
a("| bucket | 数量 | 性质 |")
a("|--------|------|------|")
bucket_meta = {
"passed": ("通过", "正常"),
"excluded_write_path": ("已排除(写路径)", "预期跳过,保护 e2e 基线"),
"no_s3_page_call": ("NO_S3 含 page. 调用", "预期:原样脚本,非改写产物"),
"compile_error_import": ("编译失败-import", "真缺口:vaadin import 未剥离,需重跑改写器"),
"missing_property": ("MissingProperty(字段缺失)", "部分因 item={} 导致,部分为真缺口"),
"missing_method": ("MissingMethod(函数库缺方法)", "真缺口:ScriptFacade 动词表未覆盖"),
"compile_error_syntax": ("编译失败-语法", "真缺口:改写产物语法错误"),
"page_binding_missing": ("page 绑定缺失", "真缺口:FULL_AUTO 脚本含 page. 调用"),
"script_blank_or_missing": ("产物空白或缺失", "文件全注释无正文,或目录缺失"),
"endpoint_404": ("端点 404", "路由配置问题"),
"http_500": ("引擎崩溃(Spring 500)", "真缺口:脚本执行导致未捕获异常"),
"engine_exception": ("引擎异常(其他)", "需人工分析"),
"connection_error": ("连接错误", "环境问题"),
"read_error": ("读文件失败", "磁盘问题"),
}
for bucket, count in sorted(stats["bucket_counts"].items(), key=lambda x: -x[1]):
meta = bucket_meta.get(bucket, (bucket, "—"))
a(f"| `{bucket}` | {count} | {meta[1]} |")
a("")
a("---")
a("")
a("## 三、FULL_AUTO top 失败模式样例")
a("")
fa_samples = get_top_failures(results, "FULL_AUTO")
if fa_samples:
for s in fa_samples:
a(f"### bucket: `{s['bucket']}` ({s['count']} 条)")
a(f"- 样例 tid: `{s['sample_tid']}`,seq: {s['sample_seq']}")
a(f"- 错误摘要: `{s['sample_error']}`")
a("")
else:
a("(无 FULL_AUTO 失败)")
a("")
a("---")
a("")
a("## 四、闭包动词专项(F2-3 dynamic_button / addFieldButton)")
a("")
a("| 模板名 | tid | seq | API通过 | closureCmd | verdict |")
a("|--------|-----|-----|---------|------------|---------|")
for r in closure_results:
a(f"| {r['name']} | `{r['tid'][:12]}...` | {r['seq']} | "
f"{r['passed_api']} | {r['has_closure_cmd']} | **{r['verdict']}** |")
a("")
closure_pass = sum(1 for r in closure_results if r["verdict"] == "PASS")
closure_total = len(closure_results)
a(f"闭包动词专项通过: {closure_pass}/{closure_total}")
a("")
# 详细样例(commands_sample)
for r in closure_results:
if r["commands_sample"]:
a(f"**{r['name']}** commands 样例:")
a("```json")
a(json.dumps(r["commands_sample"], ensure_ascii=False, indent=2))
a("```")
a("")
a("---")
a("")
a("## 五、与「全量可用」差距清单")
a("")
a("| bucket | 是否真缺口 | 数量 | 影响 |")
a("|--------|-----------|------|------|")
bucket_gap = {
"compile_error_import": ("是", "vaadin Notification import 残留(22 条 FULL_AUTO);改写器 import 剥离规则未覆盖 Notification 类;需重跑改写器"),
"missing_property": ("部分", "item={} 时 getValue 返回 null;真实行数据可消除部分;剩余为真缺口"),
"missing_method": ("是", "ScriptFacade 动词表未覆盖的方法;需扩展"),
"compile_error_syntax": ("是", "改写产物语法错误;需改写器修复"),
"page_binding_missing": ("是", "FULL_AUTO 脚本含 page. 调用未改写;分类规则误判为 FULL_AUTO"),
"script_blank_or_missing": ("部分", "全注释空正文(原脚本仅含注释行)属设计内;真文件缺失为真缺口"),
"http_500": ("是", "脚本执行引发 Spring 未捕获异常(CustomSearchConditional 等);需改写器或 Facade 修复"),
"engine_exception": ("待定", "EmptyResultDataAccessException/权限校验等 DB 访问错误;item 为空或基线数据缺失"),
"no_s3_page_call": ("否", "NO_S3 原样脚本含 page. 调用,执行失败是预期行为"),
"excluded_write_path": ("否", "安全排除,预期跳过"),
"endpoint_404": ("否", "路由正常,此桶应为空"),
}
for bucket, count in sorted(stats["bucket_counts"].items(), key=lambda x: -x[1]):
if bucket in ("passed",):
continue
if bucket in bucket_gap:
is_gap, note = bucket_gap[bucket]
else:
is_gap, note = "待定", "—"
a(f"| `{bucket}` | {is_gap} | {count} | {note} |")
a("")
a("---")
a("")
a("## 六、结论")
a("")
fa_pass_n = fa["passed"]
fa_run_n = fa["passed"] + fa["failed"]
fa_rate_pct = fa["pass_rate"]
a(f"- **FULL_AUTO 执行通过率**: {fa_rate_pct}({fa_pass_n}/{fa_run_n} 已执行,排除 {fa['excluded']} 条写路径)")
a(f"- **NO_S3 执行通过率**: {ns['pass_rate']}({ns['passed']}/{ns_run} 已执行,大量为预期 page. 调用失败)")
a(f"- **闭包动词专项**: {closure_pass}/{closure_total} 通过")
a(f"- **结果 JSON**: `{json_path}`(若 >1MB 不入库,报告里给统计)")
a("")
a("### 真缺口优先级")
a("1. `compile_error_import`(32条):改写器需补全 Notification import 剥离规则,重新生成磁盘产物")
a("2. `http_500`(8条):脚本引发 Spring 未捕获异常(CustomSearchConditional 等),需改写器或 Facade 修复")
a("3. `missing_method`(5条):ScriptFacade 函数库覆盖缺口,需扩展动词表")
a("4. `missing_property`(10条):部分因 item={} 导致,真实行数据后重测可定量分离")
a("5. `engine_exception`(27条):NO_S3 DB 访问/权限错误,需分析基线数据完整性")
a("6. `page_binding_missing`:FULL_AUTO 脚本含 page. 调用,分类规则误判,需审查")
a("")
a("*本报告由 run_batch_regression.py 自动生成 (F2-4)*")
return "\n".join(lines)
# ─────────────────────────────────────────────
# e2e 基线检查
# ─────────────────────────────────────────────
E2E_BASELINE_TEMPLATES = [
# admin/ceshi/E2E- 前缀的基线模板:抽 2 条代表性 FULL_AUTO 脚本检验
# 从 full_auto_index.json 里选取稳定模板
("021e3ba7-e03a-412a-9f0e-c20ef440a885", 1, "办公用品领用申请(按库领用)"),
]
def check_e2e_baseline(jwt: str) -> list:
results = []
print("\n[e2e 基线确认] 批量回归前/后各跑一次...")
for tid, seq, name in E2E_BASELINE_TEMPLATES:
payload = json.dumps({"templateId": tid, "seq": seq, "item": {}}).encode()
try:
req = urllib.request.Request(
f"{BASE_URL}/api/script/execute-template",
data=payload,
headers={
"Content-Type": "application/json",
"Authorization": f"Bearer {jwt}",
},
method="POST",
)
with urllib.request.urlopen(req, timeout=15) as resp:
data = json.loads(resp.read())
results.append({"tid": tid, "seq": seq, "name": name,
"passed": data.get("passed"), "error": data.get("error")})
except Exception as e:
results.append({"tid": tid, "seq": seq, "name": name,
"passed": False, "error": str(e)})
for r in results:
print(f" e2e基线 [{r['name']}]: {'通过' if r['passed'] else '失败'}")
return results
# ─────────────────────────────────────────────
# 主入口
# ─────────────────────────────────────────────
def main():
run_ts = datetime.now().strftime("%Y%m%d_%H%M%S")
print(f"=== F2-4 批量回归 harness === {run_ts}")
print(f"端点: {BASE_URL}")
print(f"产物根: {REWRITTEN_ROOT}")
# 1. 登录
print("\n[1/6] 获取 JWT...")
jwt = get_jwt()
# 2. e2e 基线(批量前)
print("\n[2/6] e2e 基线确认(批量前)...")
e2e_before = check_e2e_baseline(jwt)
if not all(r["passed"] for r in e2e_before):
print(" 警告: e2e 基线批量前失败,继续执行(不中止)")
# 3. 数据库连接(可选)
print("\n[3/6] 数据库连接...")
db_conn = connect_db()
# 4. 发现脚本
print("\n[4/6] 发现脚本...")
items = discover_scripts(labels=("FULL_AUTO", "NO_S3"))
print(f" 发现: {len(items)} 条 (FULL_AUTO + NO_S3)")
# 5. 批量执行
print(f"\n[5/6] 批量执行(并发={CONCURRENCY})...")
results = run_batch(items, jwt, db_conn)
if db_conn:
try:
db_conn.close()
except Exception:
pass
# 6. 闭包动词专项
print("\n[6/6] 闭包动词专项...")
closure_results = run_closure_verb_special(jwt)
# 计算统计
stats = compute_stats(results)
# 输出路径
output_dir = REWRITTEN_ROOT.parent
json_path = output_dir / f"batch_regression_{run_ts}.json"
report_path = Path(__file__).parent.parent.parent / "docs" / "P1" / "F2-4批量回归报告.md"
# 写 JSON 结果
result_data = {
"run_ts": run_ts,
"stats": stats,
"closure_special": closure_results,
"e2e_before": e2e_before,
"results": results,
}
json_bytes = json.dumps(result_data, ensure_ascii=False, indent=2).encode("utf-8")
json_size_mb = len(json_bytes) / 1024 / 1024
print(f"\nJSON 结果大小: {json_size_mb:.2f} MB")
if json_size_mb <= 1.0:
json_path.write_bytes(json_bytes)
print(f"JSON 已写入: {json_path}")
json_path_str = str(json_path)
else:
# >1MB 不入库,只写摘要
summary_path = output_dir / f"batch_regression_{run_ts}_summary.json"
summary_data = {
"run_ts": run_ts,
"stats": stats,
"closure_special": closure_results,
"e2e_before": e2e_before,
"note": f"全量结果 {json_size_mb:.2f}MB 超过 1MB 限制,不入库;此文件仅含统计摘要",
}
summary_path.write_text(json.dumps(summary_data, ensure_ascii=False, indent=2), encoding="utf-8")
print(f"JSON 超过 1MB({json_size_mb:.2f}MB),摘要已写入: {summary_path}")
json_path_str = str(summary_path) + f" (full: {json_size_mb:.2f}MB, 不入库)"
# e2e 基线(批量后)
e2e_after = check_e2e_baseline(jwt)
if not all(r["passed"] for r in e2e_after):
print(" 警告: e2e 基线批量后失败!")
else:
print(" e2e 基线批量后确认: 全部通过")
# 写报告
report_content = generate_report(stats, results, closure_results, run_ts, json_path_str)
report_path.parent.mkdir(parents=True, exist_ok=True)
report_path.write_text(report_content, encoding="utf-8")
print(f"报告已写入: {report_path}")
# 打印摘要
print("\n" + "=" * 60)
print("F2-4 批量回归结果摘要")
print("=" * 60)
fa = stats["full_auto"]
ns = stats["no_s3"]
print(f"FULL_AUTO: {fa['passed']}/{fa['passed']+fa['failed']} 通过 "
f"({fa['pass_rate']}),排除写路径 {fa['excluded']} 条")
print(f"NO_S3: {ns['passed']}/{ns['passed']+ns['failed']} 通过 "
f"({ns['pass_rate']}),排除写路径 {ns['excluded']} 条")
print(f"闭包动词专项: {sum(1 for r in closure_results if r['verdict']=='PASS')}/{len(closure_results)} 通过")
print("\n分桶分布:")
for bucket, count in sorted(stats["bucket_counts"].items(), key=lambda x: -x[1]):
print(f" {bucket:35s}: {count}")
print(f"\n报告: {report_path}")
print("=" * 60)
return 0
if __name__ == "__main__":
sys.exit(main())