Files
eip4-platform/tools/s3_rewriter/run_eval.py
zhengwr c174fbdf83 chore: 工具迁入正式工程 + 工程 README + 根目录草稿报告入 ignore
- eip4-poc/tools/{s3_rewriter,loadtest} → tools/(P4 改写器种子与压测 harness 随工程演进,PoC 冻结)
- 新增工程 README(模块图/纪律/常用命令/自举说明)
- 根目录 parity-reports/ 为 CLI 默认输出草稿,入 .gitignore(正式报告在 platform-parity/parity-reports/)

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
2026-06-11 11:20:01 +08:00

377 lines
14 KiB
Python
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
"""
run_eval.py — S3 改写器全语料评估
用法:
python run_eval.py # 跑 corpus/ 目录下的合成语料
python run_eval.py --corpus ./corpus # 指定语料目录
python run_eval.py --from-db # 从数据库抽取真实脚本(需 psycopg2)
--from-db 模式依赖可选依赖 psycopg2,连接参数从环境变量读取:
DB_HOST (默认 192.168.20.68)
DB_PORT (默认 5435)
DB_NAME (默认 postgres)
DB_USER (默认 readonly)
DB_PASS (默认 "")
输出指标:
- 总条数 / 各分类条数
- 自动化率 = FULL_AUTO / (FULL_AUTO + PARTIAL + HARD) [S3 条数为分母]
- 每规则命中统计
- HARD 条目清单(供人工排期)
"""
import argparse
import json
import os
import sys
from collections import Counter, defaultdict
from typing import List, Dict, Any
# 确保同目录的 rewrite.py 可导入
sys.path.insert(0, os.path.dirname(__file__))
from rewrite import rewrite_script, REWRITE_RULES, RewriteDetail
# ──────────────────────────────────────────────
# 1. 语料加载
# ──────────────────────────────────────────────
def load_corpus_dir(corpus_dir: str) -> List[Dict[str, Any]]:
"""从 corpus/ 目录加载脚本,读取 index.json 获取期望标签。"""
index_path = os.path.join(corpus_dir, "index.json")
if not os.path.exists(index_path):
raise FileNotFoundError(f"未找到 index.json: {index_path}")
with open(index_path, encoding="utf-8") as f:
index = json.load(f)
records = []
for entry in index:
filepath = os.path.join(corpus_dir, entry["filename"])
if not os.path.exists(filepath):
print(f" [警告] 文件不存在,跳过: {filepath}")
continue
with open(filepath, encoding="utf-8") as f:
src = f.read()
records.append({
"id": entry["id"],
"name": entry["name"],
"description": entry.get("description", ""),
"expected_label": entry["expected_label"],
"source": src,
"origin": "corpus",
})
return records
def load_corpus_db(db_params: Dict[str, str]) -> List[Dict[str, Any]]:
"""
从数据库 view_template 表抽取脚本(--from-db 模式)。
SQL 口径参照盘点报告 WS3 节,抽取含实质脚本的模板。
需要 psycopg2(可选依赖)。
"""
try:
import psycopg2
import psycopg2.extras
except ImportError:
print("错误: --from-db 模式需要 psycopg2,请先安装: pip install psycopg2-binary")
sys.exit(1)
host = db_params.get("host", "192.168.20.68")
port = int(db_params.get("port", "5435"))
dbname = db_params.get("dbname", "postgres")
user = db_params.get("user", "readonly")
password = db_params.get("password", "")
print(f"连接数据库: {host}:{port}/{dbname} (用户: {user}) ...")
try:
conn = psycopg2.connect(
host=host, port=port, dbname=dbname,
user=user, password=password,
connect_timeout=10,
options="-c search_path=eip3",
)
except Exception as e:
print(f"数据库连接失败: {e}")
print("提示: 确认 192.168.20.68:5435 网络可达后重试")
sys.exit(1)
# 抽取有脚本的模板:参照盘点口径,从 view_template 的 template_content XML
# 中提取 <script> 节点文本(去重后约 1,427 条)。
# 此 SQL 直接从 XML 列提取脚本文本,兼容 PG 9.4+。
# 实测列名(2026-06-11 information_schema 核实):主键 view_template_id(character),
# 模板内容列 xml(PG xml 类型)。脚本节点取 //script 文本。
SQL = """
SELECT
trim(vt.view_template_id) AS template_id,
vt.name AS template_name,
unnest(
xpath('//script/text()', vt.xml)
)::text AS script_text
FROM eip3.view_template vt
WHERE vt.xml IS NOT NULL
ORDER BY vt.view_template_id
"""
records = []
try:
with conn.cursor(cursor_factory=psycopg2.extras.DictCursor) as cur:
cur.execute(SQL)
rows = cur.fetchall()
for i, row in enumerate(rows):
script_text = row["script_text"] or ""
if len(script_text.strip()) < 5:
continue
records.append({
"id": i + 1,
"name": f"{row['template_id']}_{row['template_name']}",
"description": f"真实脚本 from view_template id={row['template_id']}",
"expected_label": None, # 真实脚本无预设期望标签
"source": script_text,
"origin": "database",
})
finally:
conn.close()
print(f"从数据库抽取脚本: {len(records)} 条")
return records
# ──────────────────────────────────────────────
# 2. 评估核心
# ──────────────────────────────────────────────
def evaluate(records: List[Dict[str, Any]]) -> Dict[str, Any]:
"""对所有脚本跑改写器,汇总统计。"""
label_counts = Counter()
rule_hit_counts = Counter() # 规则名 → 命中总次数
hard_items = [] # HARD 条目列表(供人工排期)
partial_items = [] # PARTIAL 条目列表
wrong_items = [] # 期望标签 != 实际标签(仅合成语料)
total_s3 = 0 # S3 脚本总数(FULL_AUTO+PARTIAL+HARD)
for rec in records:
src = rec["source"]
expected = rec.get("expected_label")
result_text, details, label = rewrite_script(src)
label_counts[label] += 1
# 统计规则命中
for d in details:
if d.rule_name not in ("HARD_DETECTION", "inline_field_handle",
"inline_field_handle_pass2"):
rule_name_base = d.rule_name.replace("_pass2", "")
rule_hit_counts[rule_name_base] += d.count
if label in ("FULL_AUTO", "PARTIAL", "HARD"):
total_s3 += 1
if label == "HARD":
# 提取硬 S3 原因
hard_reasons = [d.original for d in details
if d.rule_name == "HARD_DETECTION"]
hard_items.append({
"id": rec["id"],
"name": rec["name"],
"description": rec["description"],
"hard_reasons": hard_reasons,
})
if label == "PARTIAL":
partial_items.append({
"id": rec["id"],
"name": rec["name"],
"description": rec["description"],
})
# 合成语料:验证标签正确性
if expected is not None and expected != label:
wrong_items.append({
"id": rec["id"],
"name": rec["name"],
"expected": expected,
"actual": label,
})
# 自动化率 = FULL_AUTO / 所有 S3 条目
auto_count = label_counts["FULL_AUTO"]
automation_rate = (auto_count / total_s3 * 100) if total_s3 > 0 else 0.0
return {
"total": len(records),
"label_counts": dict(label_counts),
"total_s3": total_s3,
"automation_rate": automation_rate,
"rule_hit_counts": dict(rule_hit_counts),
"hard_items": hard_items,
"partial_items": partial_items,
"wrong_items": wrong_items,
}
# ──────────────────────────────────────────────
# 3. 报告输出
# ──────────────────────────────────────────────
def print_report(result: Dict[str, Any], origin: str = "corpus"):
total = result["total"]
lc = result["label_counts"]
rate = result["automation_rate"]
total_s3 = result["total_s3"]
print("\n" + "=" * 60)
print("S3 改写器评估报告")
print("=" * 60)
if origin == "corpus":
print("数据源: 合成语料(corpus/)")
print("⚠️ 注意: 合成语料实测,真实库全量实测待 5435 网络恢复后")
print(" 用 run_eval.py --from-db 重跑")
else:
print("数据源: 真实数据库(view_template)")
print(f"\n总脚本数: {total}")
print(f"S3 脚本数: {total_s3} (FULL_AUTO + PARTIAL + HARD)")
print()
labels = ["FULL_AUTO", "PARTIAL", "HARD", "NO_S3"]
label_desc = {
"FULL_AUTO": "全自动改写成功",
"PARTIAL": "部分改写,余留人工",
"HARD": "硬 S3,整条转人工",
"NO_S3": "无 S3 模式,原样",
}
for lbl in labels:
cnt = lc.get(lbl, 0)
pct = cnt / total * 100 if total > 0 else 0
bar = "█" * int(pct / 2)
print(f" {lbl:12s}: {cnt:3d} 条 ({pct:5.1f}%) {bar}")
print()
print(f"★ 自动化率 = FULL_AUTO / S3总数 = {lc.get('FULL_AUTO',0)} / {total_s3}"
f" = {rate:.1f}%")
if rate >= 80:
print(f" → 超过 80% 预算线 ✓")
else:
print(f" → 未达 80% 预算线,需扩展规则表")
# 规则命中统计
print("\n规则命中统计:")
rule_hits = result["rule_hit_counts"]
if rule_hits:
max_cnt = max(rule_hits.values()) if rule_hits else 1
for rule_name in sorted(rule_hits, key=lambda k: -rule_hits[k]):
cnt = rule_hits[rule_name]
bar = "▪" * min(int(cnt / max_cnt * 20), 20)
print(f" {rule_name:30s}: {cnt:4d} 次 {bar}")
else:
print(" (无规则命中)")
# HARD 清单
hard_items = result["hard_items"]
if hard_items:
print(f"\nHARD 条目清单(共 {len(hard_items)} 条,需人工处理):")
for item in hard_items:
reasons = "; ".join(item["hard_reasons"])
print(f" [{item['id']:03d}] {item['name']}")
print(f" 原因: {reasons}")
# PARTIAL 清单
partial_items = result["partial_items"]
if partial_items:
print(f"\nPARTIAL 条目(共 {len(partial_items)} 条,部分改写待人工):")
for item in partial_items:
print(f" [{item['id']:03d}] {item['name']} — {item['description']}")
# 标签对比(合成语料)
wrong_items = result["wrong_items"]
if wrong_items:
print(f"\n⚠️ 标签不符(预期 != 实际): {len(wrong_items)} 条")
for w in wrong_items:
print(f" [{w['id']:03d}] {w['name']}: 预期={w['expected']}, 实际={w['actual']}")
elif origin == "corpus":
print(f"\n✓ 合成语料标签全部符合预期(0 条不符)")
print("\n" + "=" * 60)
# ──────────────────────────────────────────────
# 4. 主入口
# ──────────────────────────────────────────────
def main():
parser = argparse.ArgumentParser(description="S3 改写器全语料评估")
parser.add_argument(
"--corpus",
default=os.path.join(os.path.dirname(__file__), "corpus"),
help="语料目录路径(默认: ./corpus)",
)
parser.add_argument(
"--from-db",
action="store_true",
help="从数据库抽取真实脚本(需 psycopg2,需 192.168.20.68:5435 可达)",
)
parser.add_argument(
"--db-host", default=os.environ.get("DB_HOST", "192.168.20.68"),
)
parser.add_argument(
"--db-port", default=os.environ.get("DB_PORT", "5435"),
)
parser.add_argument(
"--db-name", default=os.environ.get("DB_NAME", "postgres"),
)
parser.add_argument(
"--db-user", default=os.environ.get("DB_USER", "readonly"),
)
parser.add_argument(
"--db-pass", default=os.environ.get("DB_PASS", ""),
)
parser.add_argument(
"--output-json",
default=None,
help="将评估结果以 JSON 格式写入指定文件",
)
args = parser.parse_args()
# 加载语料
if args.from_db:
records = load_corpus_db({
"host": args.db_host,
"port": args.db_port,
"dbname": args.db_name,
"user": args.db_user,
"password": args.db_pass,
})
origin = "database"
else:
records = load_corpus_dir(args.corpus)
origin = "corpus"
if not records:
print("错误: 未加载到任何脚本,退出")
sys.exit(1)
print(f"已加载 {len(records)} 条脚本,开始评估...")
# 执行评估
result = evaluate(records)
# 输出报告
print_report(result, origin=origin)
# 可选 JSON 输出
if args.output_json:
with open(args.output_json, "w", encoding="utf-8") as f:
json.dump(result, f, ensure_ascii=False, indent=2)
print(f"\nJSON 评估结果已写入: {args.output_json}")
# 返回自动化率(供 CI 检查)
return 0 if result["automation_rate"] >= 80.0 else 1
if __name__ == "__main__":
sys.exit(main())