chore: 工具迁入正式工程 + 工程 README + 根目录草稿报告入 ignore

- eip4-poc/tools/{s3_rewriter,loadtest} → tools/(P4 改写器种子与压测 harness 随工程演进,PoC 冻结)
- 新增工程 README(模块图/纪律/常用命令/自举说明)
- 根目录 parity-reports/ 为 CLI 默认输出草稿,入 .gitignore(正式报告在 platform-parity/parity-reports/)

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
2026-06-11 11:20:01 +08:00
parent f10d11c958
commit c174fbdf83
94 changed files with 17746 additions and 0 deletions

View File

@@ -0,0 +1,376 @@
"""
run_eval.py — S3 改写器全语料评估
用法:
python run_eval.py # 跑 corpus/ 目录下的合成语料
python run_eval.py --corpus ./corpus # 指定语料目录
python run_eval.py --from-db # 从数据库抽取真实脚本(需 psycopg2)
--from-db 模式依赖可选依赖 psycopg2,连接参数从环境变量读取:
DB_HOST (默认 192.168.20.68)
DB_PORT (默认 5435)
DB_NAME (默认 postgres)
DB_USER (默认 readonly)
DB_PASS (默认 "")
输出指标:
- 总条数 / 各分类条数
- 自动化率 = FULL_AUTO / (FULL_AUTO + PARTIAL + HARD) [S3 条数为分母]
- 每规则命中统计
- HARD 条目清单(供人工排期)
"""
import argparse
import json
import os
import sys
from collections import Counter, defaultdict
from typing import List, Dict, Any
# 确保同目录的 rewrite.py 可导入
sys.path.insert(0, os.path.dirname(__file__))
from rewrite import rewrite_script, REWRITE_RULES, RewriteDetail
# ──────────────────────────────────────────────
# 1. 语料加载
# ──────────────────────────────────────────────
def load_corpus_dir(corpus_dir: str) -> List[Dict[str, Any]]:
"""从 corpus/ 目录加载脚本,读取 index.json 获取期望标签。"""
index_path = os.path.join(corpus_dir, "index.json")
if not os.path.exists(index_path):
raise FileNotFoundError(f"未找到 index.json: {index_path}")
with open(index_path, encoding="utf-8") as f:
index = json.load(f)
records = []
for entry in index:
filepath = os.path.join(corpus_dir, entry["filename"])
if not os.path.exists(filepath):
print(f" [警告] 文件不存在,跳过: {filepath}")
continue
with open(filepath, encoding="utf-8") as f:
src = f.read()
records.append({
"id": entry["id"],
"name": entry["name"],
"description": entry.get("description", ""),
"expected_label": entry["expected_label"],
"source": src,
"origin": "corpus",
})
return records
def load_corpus_db(db_params: Dict[str, str]) -> List[Dict[str, Any]]:
"""
从数据库 view_template 表抽取脚本(--from-db 模式)。
SQL 口径参照盘点报告 WS3 节,抽取含实质脚本的模板。
需要 psycopg2(可选依赖)。
"""
try:
import psycopg2
import psycopg2.extras
except ImportError:
print("错误: --from-db 模式需要 psycopg2,请先安装: pip install psycopg2-binary")
sys.exit(1)
host = db_params.get("host", "192.168.20.68")
port = int(db_params.get("port", "5435"))
dbname = db_params.get("dbname", "postgres")
user = db_params.get("user", "readonly")
password = db_params.get("password", "")
print(f"连接数据库: {host}:{port}/{dbname} (用户: {user}) ...")
try:
conn = psycopg2.connect(
host=host, port=port, dbname=dbname,
user=user, password=password,
connect_timeout=10,
options="-c search_path=eip3",
)
except Exception as e:
print(f"数据库连接失败: {e}")
print("提示: 确认 192.168.20.68:5435 网络可达后重试")
sys.exit(1)
# 抽取有脚本的模板:参照盘点口径,从 view_template 的 template_content XML
# 中提取 <script> 节点文本(去重后约 1,427 条)。
# 此 SQL 直接从 XML 列提取脚本文本,兼容 PG 9.4+。
# 实测列名(2026-06-11 information_schema 核实):主键 view_template_id(character),
# 模板内容列 xml(PG xml 类型)。脚本节点取 //script 文本。
SQL = """
SELECT
trim(vt.view_template_id) AS template_id,
vt.name AS template_name,
unnest(
xpath('//script/text()', vt.xml)
)::text AS script_text
FROM eip3.view_template vt
WHERE vt.xml IS NOT NULL
ORDER BY vt.view_template_id
"""
records = []
try:
with conn.cursor(cursor_factory=psycopg2.extras.DictCursor) as cur:
cur.execute(SQL)
rows = cur.fetchall()
for i, row in enumerate(rows):
script_text = row["script_text"] or ""
if len(script_text.strip()) < 5:
continue
records.append({
"id": i + 1,
"name": f"{row['template_id']}_{row['template_name']}",
"description": f"真实脚本 from view_template id={row['template_id']}",
"expected_label": None, # 真实脚本无预设期望标签
"source": script_text,
"origin": "database",
})
finally:
conn.close()
print(f"从数据库抽取脚本: {len(records)} 条")
return records
# ──────────────────────────────────────────────
# 2. 评估核心
# ──────────────────────────────────────────────
def evaluate(records: List[Dict[str, Any]]) -> Dict[str, Any]:
"""对所有脚本跑改写器,汇总统计。"""
label_counts = Counter()
rule_hit_counts = Counter() # 规则名 → 命中总次数
hard_items = [] # HARD 条目列表(供人工排期)
partial_items = [] # PARTIAL 条目列表
wrong_items = [] # 期望标签 != 实际标签(仅合成语料)
total_s3 = 0 # S3 脚本总数(FULL_AUTO+PARTIAL+HARD)
for rec in records:
src = rec["source"]
expected = rec.get("expected_label")
result_text, details, label = rewrite_script(src)
label_counts[label] += 1
# 统计规则命中
for d in details:
if d.rule_name not in ("HARD_DETECTION", "inline_field_handle",
"inline_field_handle_pass2"):
rule_name_base = d.rule_name.replace("_pass2", "")
rule_hit_counts[rule_name_base] += d.count
if label in ("FULL_AUTO", "PARTIAL", "HARD"):
total_s3 += 1
if label == "HARD":
# 提取硬 S3 原因
hard_reasons = [d.original for d in details
if d.rule_name == "HARD_DETECTION"]
hard_items.append({
"id": rec["id"],
"name": rec["name"],
"description": rec["description"],
"hard_reasons": hard_reasons,
})
if label == "PARTIAL":
partial_items.append({
"id": rec["id"],
"name": rec["name"],
"description": rec["description"],
})
# 合成语料:验证标签正确性
if expected is not None and expected != label:
wrong_items.append({
"id": rec["id"],
"name": rec["name"],
"expected": expected,
"actual": label,
})
# 自动化率 = FULL_AUTO / 所有 S3 条目
auto_count = label_counts["FULL_AUTO"]
automation_rate = (auto_count / total_s3 * 100) if total_s3 > 0 else 0.0
return {
"total": len(records),
"label_counts": dict(label_counts),
"total_s3": total_s3,
"automation_rate": automation_rate,
"rule_hit_counts": dict(rule_hit_counts),
"hard_items": hard_items,
"partial_items": partial_items,
"wrong_items": wrong_items,
}
# ──────────────────────────────────────────────
# 3. 报告输出
# ──────────────────────────────────────────────
def print_report(result: Dict[str, Any], origin: str = "corpus"):
total = result["total"]
lc = result["label_counts"]
rate = result["automation_rate"]
total_s3 = result["total_s3"]
print("\n" + "=" * 60)
print("S3 改写器评估报告")
print("=" * 60)
if origin == "corpus":
print("数据源: 合成语料(corpus/)")
print("⚠️ 注意: 合成语料实测,真实库全量实测待 5435 网络恢复后")
print(" 用 run_eval.py --from-db 重跑")
else:
print("数据源: 真实数据库(view_template)")
print(f"\n总脚本数: {total}")
print(f"S3 脚本数: {total_s3} (FULL_AUTO + PARTIAL + HARD)")
print()
labels = ["FULL_AUTO", "PARTIAL", "HARD", "NO_S3"]
label_desc = {
"FULL_AUTO": "全自动改写成功",
"PARTIAL": "部分改写,余留人工",
"HARD": "硬 S3,整条转人工",
"NO_S3": "无 S3 模式,原样",
}
for lbl in labels:
cnt = lc.get(lbl, 0)
pct = cnt / total * 100 if total > 0 else 0
bar = "█" * int(pct / 2)
print(f" {lbl:12s}: {cnt:3d} 条 ({pct:5.1f}%) {bar}")
print()
print(f"★ 自动化率 = FULL_AUTO / S3总数 = {lc.get('FULL_AUTO',0)} / {total_s3}"
f" = {rate:.1f}%")
if rate >= 80:
print(f" → 超过 80% 预算线 ✓")
else:
print(f" → 未达 80% 预算线,需扩展规则表")
# 规则命中统计
print("\n规则命中统计:")
rule_hits = result["rule_hit_counts"]
if rule_hits:
max_cnt = max(rule_hits.values()) if rule_hits else 1
for rule_name in sorted(rule_hits, key=lambda k: -rule_hits[k]):
cnt = rule_hits[rule_name]
bar = "▪" * min(int(cnt / max_cnt * 20), 20)
print(f" {rule_name:30s}: {cnt:4d} 次 {bar}")
else:
print(" (无规则命中)")
# HARD 清单
hard_items = result["hard_items"]
if hard_items:
print(f"\nHARD 条目清单(共 {len(hard_items)} 条,需人工处理):")
for item in hard_items:
reasons = "; ".join(item["hard_reasons"])
print(f" [{item['id']:03d}] {item['name']}")
print(f" 原因: {reasons}")
# PARTIAL 清单
partial_items = result["partial_items"]
if partial_items:
print(f"\nPARTIAL 条目(共 {len(partial_items)} 条,部分改写待人工):")
for item in partial_items:
print(f" [{item['id']:03d}] {item['name']} — {item['description']}")
# 标签对比(合成语料)
wrong_items = result["wrong_items"]
if wrong_items:
print(f"\n⚠️ 标签不符(预期 != 实际): {len(wrong_items)} 条")
for w in wrong_items:
print(f" [{w['id']:03d}] {w['name']}: 预期={w['expected']}, 实际={w['actual']}")
elif origin == "corpus":
print(f"\n✓ 合成语料标签全部符合预期(0 条不符)")
print("\n" + "=" * 60)
# ──────────────────────────────────────────────
# 4. 主入口
# ──────────────────────────────────────────────
def main():
parser = argparse.ArgumentParser(description="S3 改写器全语料评估")
parser.add_argument(
"--corpus",
default=os.path.join(os.path.dirname(__file__), "corpus"),
help="语料目录路径(默认: ./corpus)",
)
parser.add_argument(
"--from-db",
action="store_true",
help="从数据库抽取真实脚本(需 psycopg2,需 192.168.20.68:5435 可达)",
)
parser.add_argument(
"--db-host", default=os.environ.get("DB_HOST", "192.168.20.68"),
)
parser.add_argument(
"--db-port", default=os.environ.get("DB_PORT", "5435"),
)
parser.add_argument(
"--db-name", default=os.environ.get("DB_NAME", "postgres"),
)
parser.add_argument(
"--db-user", default=os.environ.get("DB_USER", "readonly"),
)
parser.add_argument(
"--db-pass", default=os.environ.get("DB_PASS", ""),
)
parser.add_argument(
"--output-json",
default=None,
help="将评估结果以 JSON 格式写入指定文件",
)
args = parser.parse_args()
# 加载语料
if args.from_db:
records = load_corpus_db({
"host": args.db_host,
"port": args.db_port,
"dbname": args.db_name,
"user": args.db_user,
"password": args.db_pass,
})
origin = "database"
else:
records = load_corpus_dir(args.corpus)
origin = "corpus"
if not records:
print("错误: 未加载到任何脚本,退出")
sys.exit(1)
print(f"已加载 {len(records)} 条脚本,开始评估...")
# 执行评估
result = evaluate(records)
# 输出报告
print_report(result, origin=origin)
# 可选 JSON 输出
if args.output_json:
with open(args.output_json, "w", encoding="utf-8") as f:
json.dump(result, f, ensure_ascii=False, indent=2)
print(f"\nJSON 评估结果已写入: {args.output_json}")
# 返回自动化率(供 CI 检查)
return 0 if result["automation_rate"] >= 80.0 else 1
if __name__ == "__main__":
sys.exit(main())