chore: 工具迁入正式工程 + 工程 README + 根目录草稿报告入 ignore
- eip4-poc/tools/{s3_rewriter,loadtest} → tools/(P4 改写器种子与压测 harness 随工程演进,PoC 冻结)
- 新增工程 README(模块图/纪律/常用命令/自举说明)
- 根目录 parity-reports/ 为 CLI 默认输出草稿,入 .gitignore(正式报告在 platform-parity/parity-reports/)
Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
376
tools/s3_rewriter/run_eval.py
Normal file
376
tools/s3_rewriter/run_eval.py
Normal file
@@ -0,0 +1,376 @@
|
||||
"""
|
||||
run_eval.py — S3 改写器全语料评估
|
||||
|
||||
用法:
|
||||
python run_eval.py # 跑 corpus/ 目录下的合成语料
|
||||
python run_eval.py --corpus ./corpus # 指定语料目录
|
||||
python run_eval.py --from-db # 从数据库抽取真实脚本(需 psycopg2)
|
||||
|
||||
--from-db 模式依赖可选依赖 psycopg2,连接参数从环境变量读取:
|
||||
DB_HOST (默认 192.168.20.68)
|
||||
DB_PORT (默认 5435)
|
||||
DB_NAME (默认 postgres)
|
||||
DB_USER (默认 readonly)
|
||||
DB_PASS (默认 "")
|
||||
|
||||
输出指标:
|
||||
- 总条数 / 各分类条数
|
||||
- 自动化率 = FULL_AUTO / (FULL_AUTO + PARTIAL + HARD) [S3 条数为分母]
|
||||
- 每规则命中统计
|
||||
- HARD 条目清单(供人工排期)
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
from collections import Counter, defaultdict
|
||||
from typing import List, Dict, Any
|
||||
|
||||
# 确保同目录的 rewrite.py 可导入
|
||||
sys.path.insert(0, os.path.dirname(__file__))
|
||||
from rewrite import rewrite_script, REWRITE_RULES, RewriteDetail
|
||||
|
||||
# ──────────────────────────────────────────────
|
||||
# 1. 语料加载
|
||||
# ──────────────────────────────────────────────
|
||||
|
||||
def load_corpus_dir(corpus_dir: str) -> List[Dict[str, Any]]:
|
||||
"""从 corpus/ 目录加载脚本,读取 index.json 获取期望标签。"""
|
||||
index_path = os.path.join(corpus_dir, "index.json")
|
||||
if not os.path.exists(index_path):
|
||||
raise FileNotFoundError(f"未找到 index.json: {index_path}")
|
||||
|
||||
with open(index_path, encoding="utf-8") as f:
|
||||
index = json.load(f)
|
||||
|
||||
records = []
|
||||
for entry in index:
|
||||
filepath = os.path.join(corpus_dir, entry["filename"])
|
||||
if not os.path.exists(filepath):
|
||||
print(f" [警告] 文件不存在,跳过: {filepath}")
|
||||
continue
|
||||
with open(filepath, encoding="utf-8") as f:
|
||||
src = f.read()
|
||||
records.append({
|
||||
"id": entry["id"],
|
||||
"name": entry["name"],
|
||||
"description": entry.get("description", ""),
|
||||
"expected_label": entry["expected_label"],
|
||||
"source": src,
|
||||
"origin": "corpus",
|
||||
})
|
||||
return records
|
||||
|
||||
|
||||
def load_corpus_db(db_params: Dict[str, str]) -> List[Dict[str, Any]]:
|
||||
"""
|
||||
从数据库 view_template 表抽取脚本(--from-db 模式)。
|
||||
SQL 口径参照盘点报告 WS3 节,抽取含实质脚本的模板。
|
||||
|
||||
需要 psycopg2(可选依赖)。
|
||||
"""
|
||||
try:
|
||||
import psycopg2
|
||||
import psycopg2.extras
|
||||
except ImportError:
|
||||
print("错误: --from-db 模式需要 psycopg2,请先安装: pip install psycopg2-binary")
|
||||
sys.exit(1)
|
||||
|
||||
host = db_params.get("host", "192.168.20.68")
|
||||
port = int(db_params.get("port", "5435"))
|
||||
dbname = db_params.get("dbname", "postgres")
|
||||
user = db_params.get("user", "readonly")
|
||||
password = db_params.get("password", "")
|
||||
|
||||
print(f"连接数据库: {host}:{port}/{dbname} (用户: {user}) ...")
|
||||
|
||||
try:
|
||||
conn = psycopg2.connect(
|
||||
host=host, port=port, dbname=dbname,
|
||||
user=user, password=password,
|
||||
connect_timeout=10,
|
||||
options="-c search_path=eip3",
|
||||
)
|
||||
except Exception as e:
|
||||
print(f"数据库连接失败: {e}")
|
||||
print("提示: 确认 192.168.20.68:5435 网络可达后重试")
|
||||
sys.exit(1)
|
||||
|
||||
# 抽取有脚本的模板:参照盘点口径,从 view_template 的 template_content XML
|
||||
# 中提取 <script> 节点文本(去重后约 1,427 条)。
|
||||
# 此 SQL 直接从 XML 列提取脚本文本,兼容 PG 9.4+。
|
||||
# 实测列名(2026-06-11 information_schema 核实):主键 view_template_id(character),
|
||||
# 模板内容列 xml(PG xml 类型)。脚本节点取 //script 文本。
|
||||
SQL = """
|
||||
SELECT
|
||||
trim(vt.view_template_id) AS template_id,
|
||||
vt.name AS template_name,
|
||||
unnest(
|
||||
xpath('//script/text()', vt.xml)
|
||||
)::text AS script_text
|
||||
FROM eip3.view_template vt
|
||||
WHERE vt.xml IS NOT NULL
|
||||
ORDER BY vt.view_template_id
|
||||
"""
|
||||
|
||||
records = []
|
||||
try:
|
||||
with conn.cursor(cursor_factory=psycopg2.extras.DictCursor) as cur:
|
||||
cur.execute(SQL)
|
||||
rows = cur.fetchall()
|
||||
for i, row in enumerate(rows):
|
||||
script_text = row["script_text"] or ""
|
||||
if len(script_text.strip()) < 5:
|
||||
continue
|
||||
records.append({
|
||||
"id": i + 1,
|
||||
"name": f"{row['template_id']}_{row['template_name']}",
|
||||
"description": f"真实脚本 from view_template id={row['template_id']}",
|
||||
"expected_label": None, # 真实脚本无预设期望标签
|
||||
"source": script_text,
|
||||
"origin": "database",
|
||||
})
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
print(f"从数据库抽取脚本: {len(records)} 条")
|
||||
return records
|
||||
|
||||
|
||||
# ──────────────────────────────────────────────
|
||||
# 2. 评估核心
|
||||
# ──────────────────────────────────────────────
|
||||
|
||||
def evaluate(records: List[Dict[str, Any]]) -> Dict[str, Any]:
|
||||
"""对所有脚本跑改写器,汇总统计。"""
|
||||
label_counts = Counter()
|
||||
rule_hit_counts = Counter() # 规则名 → 命中总次数
|
||||
hard_items = [] # HARD 条目列表(供人工排期)
|
||||
partial_items = [] # PARTIAL 条目列表
|
||||
wrong_items = [] # 期望标签 != 实际标签(仅合成语料)
|
||||
total_s3 = 0 # S3 脚本总数(FULL_AUTO+PARTIAL+HARD)
|
||||
|
||||
for rec in records:
|
||||
src = rec["source"]
|
||||
expected = rec.get("expected_label")
|
||||
|
||||
result_text, details, label = rewrite_script(src)
|
||||
label_counts[label] += 1
|
||||
|
||||
# 统计规则命中
|
||||
for d in details:
|
||||
if d.rule_name not in ("HARD_DETECTION", "inline_field_handle",
|
||||
"inline_field_handle_pass2"):
|
||||
rule_name_base = d.rule_name.replace("_pass2", "")
|
||||
rule_hit_counts[rule_name_base] += d.count
|
||||
|
||||
if label in ("FULL_AUTO", "PARTIAL", "HARD"):
|
||||
total_s3 += 1
|
||||
|
||||
if label == "HARD":
|
||||
# 提取硬 S3 原因
|
||||
hard_reasons = [d.original for d in details
|
||||
if d.rule_name == "HARD_DETECTION"]
|
||||
hard_items.append({
|
||||
"id": rec["id"],
|
||||
"name": rec["name"],
|
||||
"description": rec["description"],
|
||||
"hard_reasons": hard_reasons,
|
||||
})
|
||||
|
||||
if label == "PARTIAL":
|
||||
partial_items.append({
|
||||
"id": rec["id"],
|
||||
"name": rec["name"],
|
||||
"description": rec["description"],
|
||||
})
|
||||
|
||||
# 合成语料:验证标签正确性
|
||||
if expected is not None and expected != label:
|
||||
wrong_items.append({
|
||||
"id": rec["id"],
|
||||
"name": rec["name"],
|
||||
"expected": expected,
|
||||
"actual": label,
|
||||
})
|
||||
|
||||
# 自动化率 = FULL_AUTO / 所有 S3 条目
|
||||
auto_count = label_counts["FULL_AUTO"]
|
||||
automation_rate = (auto_count / total_s3 * 100) if total_s3 > 0 else 0.0
|
||||
|
||||
return {
|
||||
"total": len(records),
|
||||
"label_counts": dict(label_counts),
|
||||
"total_s3": total_s3,
|
||||
"automation_rate": automation_rate,
|
||||
"rule_hit_counts": dict(rule_hit_counts),
|
||||
"hard_items": hard_items,
|
||||
"partial_items": partial_items,
|
||||
"wrong_items": wrong_items,
|
||||
}
|
||||
|
||||
|
||||
# ──────────────────────────────────────────────
|
||||
# 3. 报告输出
|
||||
# ──────────────────────────────────────────────
|
||||
|
||||
def print_report(result: Dict[str, Any], origin: str = "corpus"):
|
||||
total = result["total"]
|
||||
lc = result["label_counts"]
|
||||
rate = result["automation_rate"]
|
||||
total_s3 = result["total_s3"]
|
||||
|
||||
print("\n" + "=" * 60)
|
||||
print("S3 改写器评估报告")
|
||||
print("=" * 60)
|
||||
|
||||
if origin == "corpus":
|
||||
print("数据源: 合成语料(corpus/)")
|
||||
print("⚠️ 注意: 合成语料实测,真实库全量实测待 5435 网络恢复后")
|
||||
print(" 用 run_eval.py --from-db 重跑")
|
||||
else:
|
||||
print("数据源: 真实数据库(view_template)")
|
||||
|
||||
print(f"\n总脚本数: {total}")
|
||||
print(f"S3 脚本数: {total_s3} (FULL_AUTO + PARTIAL + HARD)")
|
||||
print()
|
||||
|
||||
labels = ["FULL_AUTO", "PARTIAL", "HARD", "NO_S3"]
|
||||
label_desc = {
|
||||
"FULL_AUTO": "全自动改写成功",
|
||||
"PARTIAL": "部分改写,余留人工",
|
||||
"HARD": "硬 S3,整条转人工",
|
||||
"NO_S3": "无 S3 模式,原样",
|
||||
}
|
||||
for lbl in labels:
|
||||
cnt = lc.get(lbl, 0)
|
||||
pct = cnt / total * 100 if total > 0 else 0
|
||||
bar = "█" * int(pct / 2)
|
||||
print(f" {lbl:12s}: {cnt:3d} 条 ({pct:5.1f}%) {bar}")
|
||||
|
||||
print()
|
||||
print(f"★ 自动化率 = FULL_AUTO / S3总数 = {lc.get('FULL_AUTO',0)} / {total_s3}"
|
||||
f" = {rate:.1f}%")
|
||||
|
||||
if rate >= 80:
|
||||
print(f" → 超过 80% 预算线 ✓")
|
||||
else:
|
||||
print(f" → 未达 80% 预算线,需扩展规则表")
|
||||
|
||||
# 规则命中统计
|
||||
print("\n规则命中统计:")
|
||||
rule_hits = result["rule_hit_counts"]
|
||||
if rule_hits:
|
||||
max_cnt = max(rule_hits.values()) if rule_hits else 1
|
||||
for rule_name in sorted(rule_hits, key=lambda k: -rule_hits[k]):
|
||||
cnt = rule_hits[rule_name]
|
||||
bar = "▪" * min(int(cnt / max_cnt * 20), 20)
|
||||
print(f" {rule_name:30s}: {cnt:4d} 次 {bar}")
|
||||
else:
|
||||
print(" (无规则命中)")
|
||||
|
||||
# HARD 清单
|
||||
hard_items = result["hard_items"]
|
||||
if hard_items:
|
||||
print(f"\nHARD 条目清单(共 {len(hard_items)} 条,需人工处理):")
|
||||
for item in hard_items:
|
||||
reasons = "; ".join(item["hard_reasons"])
|
||||
print(f" [{item['id']:03d}] {item['name']}")
|
||||
print(f" 原因: {reasons}")
|
||||
|
||||
# PARTIAL 清单
|
||||
partial_items = result["partial_items"]
|
||||
if partial_items:
|
||||
print(f"\nPARTIAL 条目(共 {len(partial_items)} 条,部分改写待人工):")
|
||||
for item in partial_items:
|
||||
print(f" [{item['id']:03d}] {item['name']} — {item['description']}")
|
||||
|
||||
# 标签对比(合成语料)
|
||||
wrong_items = result["wrong_items"]
|
||||
if wrong_items:
|
||||
print(f"\n⚠️ 标签不符(预期 != 实际): {len(wrong_items)} 条")
|
||||
for w in wrong_items:
|
||||
print(f" [{w['id']:03d}] {w['name']}: 预期={w['expected']}, 实际={w['actual']}")
|
||||
elif origin == "corpus":
|
||||
print(f"\n✓ 合成语料标签全部符合预期(0 条不符)")
|
||||
|
||||
print("\n" + "=" * 60)
|
||||
|
||||
|
||||
# ──────────────────────────────────────────────
|
||||
# 4. 主入口
|
||||
# ──────────────────────────────────────────────
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description="S3 改写器全语料评估")
|
||||
parser.add_argument(
|
||||
"--corpus",
|
||||
default=os.path.join(os.path.dirname(__file__), "corpus"),
|
||||
help="语料目录路径(默认: ./corpus)",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--from-db",
|
||||
action="store_true",
|
||||
help="从数据库抽取真实脚本(需 psycopg2,需 192.168.20.68:5435 可达)",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--db-host", default=os.environ.get("DB_HOST", "192.168.20.68"),
|
||||
)
|
||||
parser.add_argument(
|
||||
"--db-port", default=os.environ.get("DB_PORT", "5435"),
|
||||
)
|
||||
parser.add_argument(
|
||||
"--db-name", default=os.environ.get("DB_NAME", "postgres"),
|
||||
)
|
||||
parser.add_argument(
|
||||
"--db-user", default=os.environ.get("DB_USER", "readonly"),
|
||||
)
|
||||
parser.add_argument(
|
||||
"--db-pass", default=os.environ.get("DB_PASS", ""),
|
||||
)
|
||||
parser.add_argument(
|
||||
"--output-json",
|
||||
default=None,
|
||||
help="将评估结果以 JSON 格式写入指定文件",
|
||||
)
|
||||
args = parser.parse_args()
|
||||
|
||||
# 加载语料
|
||||
if args.from_db:
|
||||
records = load_corpus_db({
|
||||
"host": args.db_host,
|
||||
"port": args.db_port,
|
||||
"dbname": args.db_name,
|
||||
"user": args.db_user,
|
||||
"password": args.db_pass,
|
||||
})
|
||||
origin = "database"
|
||||
else:
|
||||
records = load_corpus_dir(args.corpus)
|
||||
origin = "corpus"
|
||||
|
||||
if not records:
|
||||
print("错误: 未加载到任何脚本,退出")
|
||||
sys.exit(1)
|
||||
|
||||
print(f"已加载 {len(records)} 条脚本,开始评估...")
|
||||
|
||||
# 执行评估
|
||||
result = evaluate(records)
|
||||
|
||||
# 输出报告
|
||||
print_report(result, origin=origin)
|
||||
|
||||
# 可选 JSON 输出
|
||||
if args.output_json:
|
||||
with open(args.output_json, "w", encoding="utf-8") as f:
|
||||
json.dump(result, f, ensure_ascii=False, indent=2)
|
||||
print(f"\nJSON 评估结果已写入: {args.output_json}")
|
||||
|
||||
# 返回自动化率(供 CI 检查)
|
||||
return 0 if result["automation_rate"] >= 80.0 else 1
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
Reference in New Issue
Block a user