Files
eip4-platform/tools/s3_rewriter/run_eval.py
zhengwr a711ecd8f0 F2-4(一): 修 XML 实体污染并重批重入库,固化标准入库器
- run_eval: PG xpath(...)::text 保留实体转义(< → &lt;),抽取后补 XML 预定义实体反转义
  (等价手册 §5.3 dom4j getText() 语义);293 个污染产物清零,Groovy 编译失败根因消除
- 缓存重建(源=eip4dev,1428 条:1 模板改名+1 新增脚本属数据演进);全量重跑
  497/30/86/815,自动化率 81.1% 守住 ≥80%
- 重大发现:同模板内脚本顺序在两次抽取间会交换((templateId,seq) 非稳定键,
  welfare 模板 seq1↔seq2 实锤)——重批后 onEvent seq 已与磁盘对齐,长期需内容哈希键(登记 F2-4)
- tools/page-schema-loader/load_page_schema.py:幂等标准入库器(etag=sha256,version 递增,
  仅允许 localhost 双库);本次双库各更新 756/幂等跳过 733,e2e 28/28 保持全绿
- 坑:mvn exec:java 的 -Dexec.args 引号空参会被吞导致参数错位(密码用占位符代替空串)

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
2026-06-11 21:27:38 +08:00

392 lines
14 KiB
Python
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
"""
run_eval.py — S3 改写器全语料评估
用法:
python run_eval.py # 跑 corpus/ 目录下的合成语料
python run_eval.py --corpus ./corpus # 指定语料目录
python run_eval.py --from-db # 从数据库抽取真实脚本(需 psycopg2)
--from-db 模式依赖可选依赖 psycopg2,连接参数从环境变量读取:
DB_HOST (默认 192.168.20.68)
DB_PORT (默认 5435)
DB_NAME (默认 postgres)
DB_USER (默认 readonly)
DB_PASS (默认 "")
输出指标:
- 总条数 / 各分类条数
- 自动化率 = FULL_AUTO / (FULL_AUTO + PARTIAL + HARD) [S3 条数为分母]
- 每规则命中统计
- HARD 条目清单(供人工排期)
"""
import argparse
import json
import os
import sys
from collections import Counter, defaultdict
from typing import List, Dict, Any
# 确保同目录的 rewrite.py 可导入
sys.path.insert(0, os.path.dirname(__file__))
from rewrite import rewrite_script, REWRITE_RULES, RewriteDetail
# ──────────────────────────────────────────────
# 1. 语料加载
# ──────────────────────────────────────────────
def load_corpus_dir(corpus_dir: str) -> List[Dict[str, Any]]:
"""从 corpus/ 目录加载脚本,读取 index.json 获取期望标签。"""
index_path = os.path.join(corpus_dir, "index.json")
if not os.path.exists(index_path):
raise FileNotFoundError(f"未找到 index.json: {index_path}")
with open(index_path, encoding="utf-8") as f:
index = json.load(f)
records = []
for entry in index:
filepath = os.path.join(corpus_dir, entry["filename"])
if not os.path.exists(filepath):
print(f" [警告] 文件不存在,跳过: {filepath}")
continue
with open(filepath, encoding="utf-8") as f:
src = f.read()
records.append({
"id": entry["id"],
"name": entry["name"],
"description": entry.get("description", ""),
"expected_label": entry["expected_label"],
"source": src,
"origin": "corpus",
})
return records
def _xml_unescape(s: str) -> str:
"""XML 预定义实体反转义(amp 必须最后解,避免 &amp;lt; 双解)。含十进制/十六进制数字引用。"""
import re as _re
s = _re.sub(r"&#x([0-9a-fA-F]+);", lambda m: chr(int(m.group(1), 16)), s)
s = _re.sub(r"&#([0-9]+);", lambda m: chr(int(m.group(1))), s)
for ent, ch in (("&lt;", "<"), ("&gt;", ">"), ("&quot;", '"'), ("&apos;", "'"), ("&amp;", "&")):
s = s.replace(ent, ch)
return s
def load_corpus_db(db_params: Dict[str, str]) -> List[Dict[str, Any]]:
"""
从数据库 view_template 表抽取脚本(--from-db 模式)。
SQL 口径参照盘点报告 WS3 节,抽取含实质脚本的模板。
需要 psycopg2(可选依赖)。
"""
try:
import psycopg2
import psycopg2.extras
except ImportError:
print("错误: --from-db 模式需要 psycopg2,请先安装: pip install psycopg2-binary")
sys.exit(1)
host = db_params.get("host", "192.168.20.68")
port = int(db_params.get("port", "5435"))
dbname = db_params.get("dbname", "postgres")
user = db_params.get("user", "readonly")
password = db_params.get("password", "")
print(f"连接数据库: {host}:{port}/{dbname} (用户: {user}) ...")
try:
conn = psycopg2.connect(
host=host, port=port, dbname=dbname,
user=user, password=password,
connect_timeout=10,
options="-c search_path=eip3",
)
except Exception as e:
print(f"数据库连接失败: {e}")
print("提示: 确认 192.168.20.68:5435 网络可达后重试")
sys.exit(1)
# 抽取有脚本的模板:参照盘点口径,从 view_template 的 template_content XML
# 中提取 <script> 节点文本(去重后约 1,427 条)。
# 此 SQL 直接从 XML 列提取脚本文本,兼容 PG 9.4+。
# 实测列名(2026-06-11 information_schema 核实):主键 view_template_id(character),
# 模板内容列 xml(PG xml 类型)。脚本节点取 //script 文本。
SQL = """
SELECT
trim(vt.view_template_id) AS template_id,
vt.name AS template_name,
unnest(
xpath('//script/text()', vt.xml)
)::text AS script_text
FROM eip3.view_template vt
WHERE vt.xml IS NOT NULL
ORDER BY vt.view_template_id
"""
records = []
try:
with conn.cursor(cursor_factory=psycopg2.extras.DictCursor) as cur:
cur.execute(SQL)
rows = cur.fetchall()
for i, row in enumerate(rows):
script_text = row["script_text"] or ""
# F2-4 修复:PG xpath(...)::text 把文本节点按 XML 序列化返回,
# 预定义实体保持转义(< → &lt;),不反转义则 Groovy 编译失败
# (等价于迁移手册 §5.3 dom4j getText() 的解码语义;只解 XML
# 五个预定义实体+数字引用,不引入 HTML 实体集)。
script_text = _xml_unescape(script_text)
if len(script_text.strip()) < 5:
continue
records.append({
"id": i + 1,
"name": f"{row['template_id']}_{row['template_name']}",
"description": f"真实脚本 from view_template id={row['template_id']}",
"expected_label": None, # 真实脚本无预设期望标签
"source": script_text,
"origin": "database",
})
finally:
conn.close()
print(f"从数据库抽取脚本: {len(records)} 条")
return records
# ──────────────────────────────────────────────
# 2. 评估核心
# ──────────────────────────────────────────────
def evaluate(records: List[Dict[str, Any]]) -> Dict[str, Any]:
"""对所有脚本跑改写器,汇总统计。"""
label_counts = Counter()
rule_hit_counts = Counter() # 规则名 → 命中总次数
hard_items = [] # HARD 条目列表(供人工排期)
partial_items = [] # PARTIAL 条目列表
wrong_items = [] # 期望标签 != 实际标签(仅合成语料)
total_s3 = 0 # S3 脚本总数(FULL_AUTO+PARTIAL+HARD)
for rec in records:
src = rec["source"]
expected = rec.get("expected_label")
result_text, details, label = rewrite_script(src)
label_counts[label] += 1
# 统计规则命中
for d in details:
if d.rule_name not in ("HARD_DETECTION", "inline_field_handle",
"inline_field_handle_pass2"):
rule_name_base = d.rule_name.replace("_pass2", "")
rule_hit_counts[rule_name_base] += d.count
if label in ("FULL_AUTO", "PARTIAL", "HARD"):
total_s3 += 1
if label == "HARD":
# 提取硬 S3 原因
hard_reasons = [d.original for d in details
if d.rule_name == "HARD_DETECTION"]
hard_items.append({
"id": rec["id"],
"name": rec["name"],
"description": rec["description"],
"hard_reasons": hard_reasons,
})
if label == "PARTIAL":
partial_items.append({
"id": rec["id"],
"name": rec["name"],
"description": rec["description"],
})
# 合成语料:验证标签正确性
if expected is not None and expected != label:
wrong_items.append({
"id": rec["id"],
"name": rec["name"],
"expected": expected,
"actual": label,
})
# 自动化率 = FULL_AUTO / 所有 S3 条目
auto_count = label_counts["FULL_AUTO"]
automation_rate = (auto_count / total_s3 * 100) if total_s3 > 0 else 0.0
return {
"total": len(records),
"label_counts": dict(label_counts),
"total_s3": total_s3,
"automation_rate": automation_rate,
"rule_hit_counts": dict(rule_hit_counts),
"hard_items": hard_items,
"partial_items": partial_items,
"wrong_items": wrong_items,
}
# ──────────────────────────────────────────────
# 3. 报告输出
# ──────────────────────────────────────────────
def print_report(result: Dict[str, Any], origin: str = "corpus"):
total = result["total"]
lc = result["label_counts"]
rate = result["automation_rate"]
total_s3 = result["total_s3"]
print("\n" + "=" * 60)
print("S3 改写器评估报告")
print("=" * 60)
if origin == "corpus":
print("数据源: 合成语料(corpus/)")
print("⚠️ 注意: 合成语料实测,真实库全量实测待 5435 网络恢复后")
print(" 用 run_eval.py --from-db 重跑")
else:
print("数据源: 真实数据库(view_template)")
print(f"\n总脚本数: {total}")
print(f"S3 脚本数: {total_s3} (FULL_AUTO + PARTIAL + HARD)")
print()
labels = ["FULL_AUTO", "PARTIAL", "HARD", "NO_S3"]
label_desc = {
"FULL_AUTO": "全自动改写成功",
"PARTIAL": "部分改写,余留人工",
"HARD": "硬 S3,整条转人工",
"NO_S3": "无 S3 模式,原样",
}
for lbl in labels:
cnt = lc.get(lbl, 0)
pct = cnt / total * 100 if total > 0 else 0
bar = "█" * int(pct / 2)
print(f" {lbl:12s}: {cnt:3d} 条 ({pct:5.1f}%) {bar}")
print()
print(f"★ 自动化率 = FULL_AUTO / S3总数 = {lc.get('FULL_AUTO',0)} / {total_s3}"
f" = {rate:.1f}%")
if rate >= 80:
print(f" → 超过 80% 预算线 ✓")
else:
print(f" → 未达 80% 预算线,需扩展规则表")
# 规则命中统计
print("\n规则命中统计:")
rule_hits = result["rule_hit_counts"]
if rule_hits:
max_cnt = max(rule_hits.values()) if rule_hits else 1
for rule_name in sorted(rule_hits, key=lambda k: -rule_hits[k]):
cnt = rule_hits[rule_name]
bar = "▪" * min(int(cnt / max_cnt * 20), 20)
print(f" {rule_name:30s}: {cnt:4d} 次 {bar}")
else:
print(" (无规则命中)")
# HARD 清单
hard_items = result["hard_items"]
if hard_items:
print(f"\nHARD 条目清单(共 {len(hard_items)} 条,需人工处理):")
for item in hard_items:
reasons = "; ".join(item["hard_reasons"])
print(f" [{item['id']:03d}] {item['name']}")
print(f" 原因: {reasons}")
# PARTIAL 清单
partial_items = result["partial_items"]
if partial_items:
print(f"\nPARTIAL 条目(共 {len(partial_items)} 条,部分改写待人工):")
for item in partial_items:
print(f" [{item['id']:03d}] {item['name']} — {item['description']}")
# 标签对比(合成语料)
wrong_items = result["wrong_items"]
if wrong_items:
print(f"\n⚠️ 标签不符(预期 != 实际): {len(wrong_items)} 条")
for w in wrong_items:
print(f" [{w['id']:03d}] {w['name']}: 预期={w['expected']}, 实际={w['actual']}")
elif origin == "corpus":
print(f"\n✓ 合成语料标签全部符合预期(0 条不符)")
print("\n" + "=" * 60)
# ──────────────────────────────────────────────
# 4. 主入口
# ──────────────────────────────────────────────
def main():
parser = argparse.ArgumentParser(description="S3 改写器全语料评估")
parser.add_argument(
"--corpus",
default=os.path.join(os.path.dirname(__file__), "corpus"),
help="语料目录路径(默认: ./corpus)",
)
parser.add_argument(
"--from-db",
action="store_true",
help="从数据库抽取真实脚本(需 psycopg2,需 192.168.20.68:5435 可达)",
)
parser.add_argument(
"--db-host", default=os.environ.get("DB_HOST", "192.168.20.68"),
)
parser.add_argument(
"--db-port", default=os.environ.get("DB_PORT", "5435"),
)
parser.add_argument(
"--db-name", default=os.environ.get("DB_NAME", "postgres"),
)
parser.add_argument(
"--db-user", default=os.environ.get("DB_USER", "readonly"),
)
parser.add_argument(
"--db-pass", default=os.environ.get("DB_PASS", ""),
)
parser.add_argument(
"--output-json",
default=None,
help="将评估结果以 JSON 格式写入指定文件",
)
args = parser.parse_args()
# 加载语料
if args.from_db:
records = load_corpus_db({
"host": args.db_host,
"port": args.db_port,
"dbname": args.db_name,
"user": args.db_user,
"password": args.db_pass,
})
origin = "database"
else:
records = load_corpus_dir(args.corpus)
origin = "corpus"
if not records:
print("错误: 未加载到任何脚本,退出")
sys.exit(1)
print(f"已加载 {len(records)} 条脚本,开始评估...")
# 执行评估
result = evaluate(records)
# 输出报告
print_report(result, origin=origin)
# 可选 JSON 输出
if args.output_json:
with open(args.output_json, "w", encoding="utf-8") as f:
json.dump(result, f, ensure_ascii=False, indent=2)
print(f"\nJSON 评估结果已写入: {args.output_json}")
# 返回自动化率(供 CI 检查)
return 0 if result["automation_rate"] >= 80.0 else 1
if __name__ == "__main__":
sys.exit(main())