- run_eval: PG xpath(...)::text 保留实体转义(< → <),抽取后补 XML 预定义实体反转义 (等价手册 §5.3 dom4j getText() 语义);293 个污染产物清零,Groovy 编译失败根因消除 - 缓存重建(源=eip4dev,1428 条:1 模板改名+1 新增脚本属数据演进);全量重跑 497/30/86/815,自动化率 81.1% 守住 ≥80% - 重大发现:同模板内脚本顺序在两次抽取间会交换((templateId,seq) 非稳定键, welfare 模板 seq1↔seq2 实锤)——重批后 onEvent seq 已与磁盘对齐,长期需内容哈希键(登记 F2-4) - tools/page-schema-loader/load_page_schema.py:幂等标准入库器(etag=sha256,version 递增, 仅允许 localhost 双库);本次双库各更新 756/幂等跳过 733,e2e 28/28 保持全绿 - 坑:mvn exec:java 的 -Dexec.args 引号空参会被吞导致参数错位(密码用占位符代替空串) Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
392 lines
14 KiB
Python
392 lines
14 KiB
Python
"""
|
||
run_eval.py — S3 改写器全语料评估
|
||
|
||
用法:
|
||
python run_eval.py # 跑 corpus/ 目录下的合成语料
|
||
python run_eval.py --corpus ./corpus # 指定语料目录
|
||
python run_eval.py --from-db # 从数据库抽取真实脚本(需 psycopg2)
|
||
|
||
--from-db 模式依赖可选依赖 psycopg2,连接参数从环境变量读取:
|
||
DB_HOST (默认 192.168.20.68)
|
||
DB_PORT (默认 5435)
|
||
DB_NAME (默认 postgres)
|
||
DB_USER (默认 readonly)
|
||
DB_PASS (默认 "")
|
||
|
||
输出指标:
|
||
- 总条数 / 各分类条数
|
||
- 自动化率 = FULL_AUTO / (FULL_AUTO + PARTIAL + HARD) [S3 条数为分母]
|
||
- 每规则命中统计
|
||
- HARD 条目清单(供人工排期)
|
||
"""
|
||
|
||
import argparse
|
||
import json
|
||
import os
|
||
import sys
|
||
from collections import Counter, defaultdict
|
||
from typing import List, Dict, Any
|
||
|
||
# 确保同目录的 rewrite.py 可导入
|
||
sys.path.insert(0, os.path.dirname(__file__))
|
||
from rewrite import rewrite_script, REWRITE_RULES, RewriteDetail
|
||
|
||
# ──────────────────────────────────────────────
|
||
# 1. 语料加载
|
||
# ──────────────────────────────────────────────
|
||
|
||
def load_corpus_dir(corpus_dir: str) -> List[Dict[str, Any]]:
|
||
"""从 corpus/ 目录加载脚本,读取 index.json 获取期望标签。"""
|
||
index_path = os.path.join(corpus_dir, "index.json")
|
||
if not os.path.exists(index_path):
|
||
raise FileNotFoundError(f"未找到 index.json: {index_path}")
|
||
|
||
with open(index_path, encoding="utf-8") as f:
|
||
index = json.load(f)
|
||
|
||
records = []
|
||
for entry in index:
|
||
filepath = os.path.join(corpus_dir, entry["filename"])
|
||
if not os.path.exists(filepath):
|
||
print(f" [警告] 文件不存在,跳过: {filepath}")
|
||
continue
|
||
with open(filepath, encoding="utf-8") as f:
|
||
src = f.read()
|
||
records.append({
|
||
"id": entry["id"],
|
||
"name": entry["name"],
|
||
"description": entry.get("description", ""),
|
||
"expected_label": entry["expected_label"],
|
||
"source": src,
|
||
"origin": "corpus",
|
||
})
|
||
return records
|
||
|
||
|
||
def _xml_unescape(s: str) -> str:
|
||
"""XML 预定义实体反转义(amp 必须最后解,避免 &lt; 双解)。含十进制/十六进制数字引用。"""
|
||
import re as _re
|
||
s = _re.sub(r"&#x([0-9a-fA-F]+);", lambda m: chr(int(m.group(1), 16)), s)
|
||
s = _re.sub(r"&#([0-9]+);", lambda m: chr(int(m.group(1))), s)
|
||
for ent, ch in (("<", "<"), (">", ">"), (""", '"'), ("'", "'"), ("&", "&")):
|
||
s = s.replace(ent, ch)
|
||
return s
|
||
|
||
|
||
def load_corpus_db(db_params: Dict[str, str]) -> List[Dict[str, Any]]:
|
||
"""
|
||
从数据库 view_template 表抽取脚本(--from-db 模式)。
|
||
SQL 口径参照盘点报告 WS3 节,抽取含实质脚本的模板。
|
||
|
||
需要 psycopg2(可选依赖)。
|
||
"""
|
||
try:
|
||
import psycopg2
|
||
import psycopg2.extras
|
||
except ImportError:
|
||
print("错误: --from-db 模式需要 psycopg2,请先安装: pip install psycopg2-binary")
|
||
sys.exit(1)
|
||
|
||
host = db_params.get("host", "192.168.20.68")
|
||
port = int(db_params.get("port", "5435"))
|
||
dbname = db_params.get("dbname", "postgres")
|
||
user = db_params.get("user", "readonly")
|
||
password = db_params.get("password", "")
|
||
|
||
print(f"连接数据库: {host}:{port}/{dbname} (用户: {user}) ...")
|
||
|
||
try:
|
||
conn = psycopg2.connect(
|
||
host=host, port=port, dbname=dbname,
|
||
user=user, password=password,
|
||
connect_timeout=10,
|
||
options="-c search_path=eip3",
|
||
)
|
||
except Exception as e:
|
||
print(f"数据库连接失败: {e}")
|
||
print("提示: 确认 192.168.20.68:5435 网络可达后重试")
|
||
sys.exit(1)
|
||
|
||
# 抽取有脚本的模板:参照盘点口径,从 view_template 的 template_content XML
|
||
# 中提取 <script> 节点文本(去重后约 1,427 条)。
|
||
# 此 SQL 直接从 XML 列提取脚本文本,兼容 PG 9.4+。
|
||
# 实测列名(2026-06-11 information_schema 核实):主键 view_template_id(character),
|
||
# 模板内容列 xml(PG xml 类型)。脚本节点取 //script 文本。
|
||
SQL = """
|
||
SELECT
|
||
trim(vt.view_template_id) AS template_id,
|
||
vt.name AS template_name,
|
||
unnest(
|
||
xpath('//script/text()', vt.xml)
|
||
)::text AS script_text
|
||
FROM eip3.view_template vt
|
||
WHERE vt.xml IS NOT NULL
|
||
ORDER BY vt.view_template_id
|
||
"""
|
||
|
||
records = []
|
||
try:
|
||
with conn.cursor(cursor_factory=psycopg2.extras.DictCursor) as cur:
|
||
cur.execute(SQL)
|
||
rows = cur.fetchall()
|
||
for i, row in enumerate(rows):
|
||
script_text = row["script_text"] or ""
|
||
# F2-4 修复:PG xpath(...)::text 把文本节点按 XML 序列化返回,
|
||
# 预定义实体保持转义(< → <),不反转义则 Groovy 编译失败
|
||
# (等价于迁移手册 §5.3 dom4j getText() 的解码语义;只解 XML
|
||
# 五个预定义实体+数字引用,不引入 HTML 实体集)。
|
||
script_text = _xml_unescape(script_text)
|
||
if len(script_text.strip()) < 5:
|
||
continue
|
||
records.append({
|
||
"id": i + 1,
|
||
"name": f"{row['template_id']}_{row['template_name']}",
|
||
"description": f"真实脚本 from view_template id={row['template_id']}",
|
||
"expected_label": None, # 真实脚本无预设期望标签
|
||
"source": script_text,
|
||
"origin": "database",
|
||
})
|
||
finally:
|
||
conn.close()
|
||
|
||
print(f"从数据库抽取脚本: {len(records)} 条")
|
||
return records
|
||
|
||
|
||
# ──────────────────────────────────────────────
|
||
# 2. 评估核心
|
||
# ──────────────────────────────────────────────
|
||
|
||
def evaluate(records: List[Dict[str, Any]]) -> Dict[str, Any]:
|
||
"""对所有脚本跑改写器,汇总统计。"""
|
||
label_counts = Counter()
|
||
rule_hit_counts = Counter() # 规则名 → 命中总次数
|
||
hard_items = [] # HARD 条目列表(供人工排期)
|
||
partial_items = [] # PARTIAL 条目列表
|
||
wrong_items = [] # 期望标签 != 实际标签(仅合成语料)
|
||
total_s3 = 0 # S3 脚本总数(FULL_AUTO+PARTIAL+HARD)
|
||
|
||
for rec in records:
|
||
src = rec["source"]
|
||
expected = rec.get("expected_label")
|
||
|
||
result_text, details, label = rewrite_script(src)
|
||
label_counts[label] += 1
|
||
|
||
# 统计规则命中
|
||
for d in details:
|
||
if d.rule_name not in ("HARD_DETECTION", "inline_field_handle",
|
||
"inline_field_handle_pass2"):
|
||
rule_name_base = d.rule_name.replace("_pass2", "")
|
||
rule_hit_counts[rule_name_base] += d.count
|
||
|
||
if label in ("FULL_AUTO", "PARTIAL", "HARD"):
|
||
total_s3 += 1
|
||
|
||
if label == "HARD":
|
||
# 提取硬 S3 原因
|
||
hard_reasons = [d.original for d in details
|
||
if d.rule_name == "HARD_DETECTION"]
|
||
hard_items.append({
|
||
"id": rec["id"],
|
||
"name": rec["name"],
|
||
"description": rec["description"],
|
||
"hard_reasons": hard_reasons,
|
||
})
|
||
|
||
if label == "PARTIAL":
|
||
partial_items.append({
|
||
"id": rec["id"],
|
||
"name": rec["name"],
|
||
"description": rec["description"],
|
||
})
|
||
|
||
# 合成语料:验证标签正确性
|
||
if expected is not None and expected != label:
|
||
wrong_items.append({
|
||
"id": rec["id"],
|
||
"name": rec["name"],
|
||
"expected": expected,
|
||
"actual": label,
|
||
})
|
||
|
||
# 自动化率 = FULL_AUTO / 所有 S3 条目
|
||
auto_count = label_counts["FULL_AUTO"]
|
||
automation_rate = (auto_count / total_s3 * 100) if total_s3 > 0 else 0.0
|
||
|
||
return {
|
||
"total": len(records),
|
||
"label_counts": dict(label_counts),
|
||
"total_s3": total_s3,
|
||
"automation_rate": automation_rate,
|
||
"rule_hit_counts": dict(rule_hit_counts),
|
||
"hard_items": hard_items,
|
||
"partial_items": partial_items,
|
||
"wrong_items": wrong_items,
|
||
}
|
||
|
||
|
||
# ──────────────────────────────────────────────
|
||
# 3. 报告输出
|
||
# ──────────────────────────────────────────────
|
||
|
||
def print_report(result: Dict[str, Any], origin: str = "corpus"):
|
||
total = result["total"]
|
||
lc = result["label_counts"]
|
||
rate = result["automation_rate"]
|
||
total_s3 = result["total_s3"]
|
||
|
||
print("\n" + "=" * 60)
|
||
print("S3 改写器评估报告")
|
||
print("=" * 60)
|
||
|
||
if origin == "corpus":
|
||
print("数据源: 合成语料(corpus/)")
|
||
print("⚠️ 注意: 合成语料实测,真实库全量实测待 5435 网络恢复后")
|
||
print(" 用 run_eval.py --from-db 重跑")
|
||
else:
|
||
print("数据源: 真实数据库(view_template)")
|
||
|
||
print(f"\n总脚本数: {total}")
|
||
print(f"S3 脚本数: {total_s3} (FULL_AUTO + PARTIAL + HARD)")
|
||
print()
|
||
|
||
labels = ["FULL_AUTO", "PARTIAL", "HARD", "NO_S3"]
|
||
label_desc = {
|
||
"FULL_AUTO": "全自动改写成功",
|
||
"PARTIAL": "部分改写,余留人工",
|
||
"HARD": "硬 S3,整条转人工",
|
||
"NO_S3": "无 S3 模式,原样",
|
||
}
|
||
for lbl in labels:
|
||
cnt = lc.get(lbl, 0)
|
||
pct = cnt / total * 100 if total > 0 else 0
|
||
bar = "█" * int(pct / 2)
|
||
print(f" {lbl:12s}: {cnt:3d} 条 ({pct:5.1f}%) {bar}")
|
||
|
||
print()
|
||
print(f"★ 自动化率 = FULL_AUTO / S3总数 = {lc.get('FULL_AUTO',0)} / {total_s3}"
|
||
f" = {rate:.1f}%")
|
||
|
||
if rate >= 80:
|
||
print(f" → 超过 80% 预算线 ✓")
|
||
else:
|
||
print(f" → 未达 80% 预算线,需扩展规则表")
|
||
|
||
# 规则命中统计
|
||
print("\n规则命中统计:")
|
||
rule_hits = result["rule_hit_counts"]
|
||
if rule_hits:
|
||
max_cnt = max(rule_hits.values()) if rule_hits else 1
|
||
for rule_name in sorted(rule_hits, key=lambda k: -rule_hits[k]):
|
||
cnt = rule_hits[rule_name]
|
||
bar = "▪" * min(int(cnt / max_cnt * 20), 20)
|
||
print(f" {rule_name:30s}: {cnt:4d} 次 {bar}")
|
||
else:
|
||
print(" (无规则命中)")
|
||
|
||
# HARD 清单
|
||
hard_items = result["hard_items"]
|
||
if hard_items:
|
||
print(f"\nHARD 条目清单(共 {len(hard_items)} 条,需人工处理):")
|
||
for item in hard_items:
|
||
reasons = "; ".join(item["hard_reasons"])
|
||
print(f" [{item['id']:03d}] {item['name']}")
|
||
print(f" 原因: {reasons}")
|
||
|
||
# PARTIAL 清单
|
||
partial_items = result["partial_items"]
|
||
if partial_items:
|
||
print(f"\nPARTIAL 条目(共 {len(partial_items)} 条,部分改写待人工):")
|
||
for item in partial_items:
|
||
print(f" [{item['id']:03d}] {item['name']} — {item['description']}")
|
||
|
||
# 标签对比(合成语料)
|
||
wrong_items = result["wrong_items"]
|
||
if wrong_items:
|
||
print(f"\n⚠️ 标签不符(预期 != 实际): {len(wrong_items)} 条")
|
||
for w in wrong_items:
|
||
print(f" [{w['id']:03d}] {w['name']}: 预期={w['expected']}, 实际={w['actual']}")
|
||
elif origin == "corpus":
|
||
print(f"\n✓ 合成语料标签全部符合预期(0 条不符)")
|
||
|
||
print("\n" + "=" * 60)
|
||
|
||
|
||
# ──────────────────────────────────────────────
|
||
# 4. 主入口
|
||
# ──────────────────────────────────────────────
|
||
|
||
def main():
|
||
parser = argparse.ArgumentParser(description="S3 改写器全语料评估")
|
||
parser.add_argument(
|
||
"--corpus",
|
||
default=os.path.join(os.path.dirname(__file__), "corpus"),
|
||
help="语料目录路径(默认: ./corpus)",
|
||
)
|
||
parser.add_argument(
|
||
"--from-db",
|
||
action="store_true",
|
||
help="从数据库抽取真实脚本(需 psycopg2,需 192.168.20.68:5435 可达)",
|
||
)
|
||
parser.add_argument(
|
||
"--db-host", default=os.environ.get("DB_HOST", "192.168.20.68"),
|
||
)
|
||
parser.add_argument(
|
||
"--db-port", default=os.environ.get("DB_PORT", "5435"),
|
||
)
|
||
parser.add_argument(
|
||
"--db-name", default=os.environ.get("DB_NAME", "postgres"),
|
||
)
|
||
parser.add_argument(
|
||
"--db-user", default=os.environ.get("DB_USER", "readonly"),
|
||
)
|
||
parser.add_argument(
|
||
"--db-pass", default=os.environ.get("DB_PASS", ""),
|
||
)
|
||
parser.add_argument(
|
||
"--output-json",
|
||
default=None,
|
||
help="将评估结果以 JSON 格式写入指定文件",
|
||
)
|
||
args = parser.parse_args()
|
||
|
||
# 加载语料
|
||
if args.from_db:
|
||
records = load_corpus_db({
|
||
"host": args.db_host,
|
||
"port": args.db_port,
|
||
"dbname": args.db_name,
|
||
"user": args.db_user,
|
||
"password": args.db_pass,
|
||
})
|
||
origin = "database"
|
||
else:
|
||
records = load_corpus_dir(args.corpus)
|
||
origin = "corpus"
|
||
|
||
if not records:
|
||
print("错误: 未加载到任何脚本,退出")
|
||
sys.exit(1)
|
||
|
||
print(f"已加载 {len(records)} 条脚本,开始评估...")
|
||
|
||
# 执行评估
|
||
result = evaluate(records)
|
||
|
||
# 输出报告
|
||
print_report(result, origin=origin)
|
||
|
||
# 可选 JSON 输出
|
||
if args.output_json:
|
||
with open(args.output_json, "w", encoding="utf-8") as f:
|
||
json.dump(result, f, ensure_ascii=False, indent=2)
|
||
print(f"\nJSON 评估结果已写入: {args.output_json}")
|
||
|
||
# 返回自动化率(供 CI 检查)
|
||
return 0 if result["automation_rate"] >= 80.0 else 1
|
||
|
||
|
||
if __name__ == "__main__":
|
||
sys.exit(main())
|