Files
eip4-platform/tools/s3_rewriter/mine_patterns.py
zhengwr c174fbdf83 chore: 工具迁入正式工程 + 工程 README + 根目录草稿报告入 ignore
- eip4-poc/tools/{s3_rewriter,loadtest} → tools/(P4 改写器种子与压测 harness 随工程演进,PoC 冻结)
- 新增工程 README(模块图/纪律/常用命令/自举说明)
- 根目录 parity-reports/ 为 CLI 默认输出草稿,入 .gitignore(正式报告在 platform-parity/parity-reports/)

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
2026-06-11 11:20:01 +08:00

176 lines
6.4 KiB
Python
Raw Permalink Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
"""
mine_patterns.py — 真实 S3 语料模式挖掘
目的:
对全部 S3 脚本逐条提取 Vaadin API 调用模式并归一化形态,输出 Top 模式表
(模式形态 × 出现次数 × 覆盖脚本数),驱动 rewrite.py 规则扩展。
归一化策略(把字面量抹平,只看"形状"):
- getField("X") → getField(F) 字段句柄取数
- 变量名 / 字面串 → 抹平为 _
- 安全导航 ?. → 保留(区分 . 与 ?. 是关键发现)
- 链尾 .method(...) → .method(...) 只保留方法名与"有无参数"
输出:
1) 动词频率表: <持有者>.<动词> → 出现次数 / 覆盖脚本数
2) 形态分布: getField 前缀 vs 裸 / ?. vs . / 链长
3) 残留模式: 改写后仍含 S3 的脚本里,剩下的 Top 调用形态
用法:
python mine_patterns.py # 读 scripts_cache.json
python mine_patterns.py --top 50
python mine_patterns.py --residual # 只看改写后残留模式
"""
import argparse
import json
import os
import re
import sys
from collections import Counter, defaultdict
sys.path.insert(0, os.path.dirname(__file__))
from rewrite import rewrite_script, HARD_S3_PATTERNS
CACHE = os.path.join(os.path.dirname(__file__), "scripts_cache.json")
# ──────────────────────────────────────────────
# 调用形态提取
# ──────────────────────────────────────────────
# 匹配 group.getField("X") 或 group.getField('X') 或 group.getField(var)
_GETFIELD = re.compile(r'group\.getField\(\s*(["\'][^"\']*["\']|[\w.]+)\s*\)')
# 匹配任意 .method( 或 ?.method( (持有者不限)
_METHOD_CALL = re.compile(r'(\?\.|\.)\s*(\w+)\s*\(')
# 字段句柄链: group.getField(...) [?.|.] method(...) 捕获导航符 + 方法名
_FIELD_CHAIN = re.compile(
r'group\.getField\(\s*(["\'][^"\']*["\']|[\w.]+)\s*\)\s*(\?\.|\.)\s*(\w+)\s*\('
)
# 组级调用: group.<method>( (排除 getField)
_GROUP_CALL = re.compile(r'\bgroup\s*(\?\.|\.)\s*(\w+)\s*\(')
# 静态调用: Identifier.method( 例如 Notification.show
_STATIC_CALL = re.compile(r'\b([A-Z]\w+)\.(\w+)\s*\(')
def _is_string_literal(arg: str) -> bool:
return bool(arg) and arg[0] in '"\''
def normalize_field_chain(nav: str, method: str) -> str:
"""归一: getField(F)<nav><method>(...)"""
return f'getField(F){nav}{method}(...)'
def mine(scripts, only_residual=False):
verb_count = Counter() # 形态 → 出现总次数
verb_scripts = defaultdict(set) # 形态 → 覆盖脚本 id 集合
nonliteral_field = 0 # getField(非字面量) 命中脚本数
nonliteral_scripts = set()
# 形态细分计数
nav_dot = Counter() # 安全导航 vs 普通点
group_verbs = Counter()
group_verb_scripts = defaultdict(set)
static_verbs = Counter()
static_verb_scripts = defaultdict(set)
for r in scripts:
src = r["source"]
rid = r["id"]
if only_residual:
rewritten, _, label = rewrite_script(src)
if label != "PARTIAL":
continue
scan = rewritten
else:
# 跳过 HARD(不该被自动化,不参与挖掘)
_, _, label = rewrite_script(src)
if label == "HARD":
continue
scan = src
# 字段句柄链
for m in _FIELD_CHAIN.finditer(scan):
arg, nav, method = m.group(1), m.group(2), m.group(3)
shape = normalize_field_chain(nav, method)
verb_count[shape] += 1
verb_scripts[shape].add(rid)
nav_dot[nav] += 1
if not _is_string_literal(arg):
nonliteral_field += 1
nonliteral_scripts.add(rid)
# 组级调用
for m in _GROUP_CALL.finditer(scan):
nav, method = m.group(1), m.group(2)
if method == "getField":
continue
shape = f'group{nav}{method}(...)'
group_verbs[shape] += 1
group_verb_scripts[shape].add(rid)
# 静态调用
for m in _STATIC_CALL.finditer(scan):
cls, method = m.group(1), m.group(2)
shape = f'{cls}.{method}(...)'
static_verbs[shape] += 1
static_verb_scripts[shape].add(rid)
return {
"verb_count": verb_count,
"verb_scripts": verb_scripts,
"nav_dot": nav_dot,
"nonliteral_field": nonliteral_field,
"nonliteral_scripts": nonliteral_scripts,
"group_verbs": group_verbs,
"group_verb_scripts": group_verb_scripts,
"static_verbs": static_verbs,
"static_verb_scripts": static_verb_scripts,
}
def print_table(title, counter, script_map, top):
print(f"\n{'='*72}\n{title}\n{'='*72}")
print(f"{'形态':<42}{'次数':>8}{'覆盖脚本':>10}")
print("-" * 72)
for shape in sorted(counter, key=lambda k: -counter[k])[:top]:
cov = len(script_map[shape]) if shape in script_map else 0
print(f"{shape:<42}{counter[shape]:>8}{cov:>10}")
def main():
p = argparse.ArgumentParser()
p.add_argument("--cache", default=CACHE)
p.add_argument("--top", type=int, default=40)
p.add_argument("--residual", action="store_true",
help="只对 PARTIAL 脚本改写后的残留模式做挖掘")
args = p.parse_args()
scripts = json.load(open(args.cache, encoding="utf-8"))
print(f"加载脚本 {len(scripts)} 条 from {args.cache}")
mode = "改写后残留模式(PARTIAL)" if args.residual else "全量 S3 模式(剔除 HARD)"
print(f"挖掘模式: {mode}")
res = mine(scripts, only_residual=args.residual)
print_table("字段句柄链形态 getField(F).<动词>(...)",
res["verb_count"], res["verb_scripts"], args.top)
print_table("组级调用形态 group.<动词>(...)",
res["group_verbs"], res["group_verb_scripts"], args.top)
print_table("静态调用形态 Class.<方法>(...)",
res["static_verbs"], res["static_verb_scripts"], args.top)
print(f"\n{'='*72}\n形态细分\n{'='*72}")
print(f"导航符分布: {dict(res['nav_dot'])}")
print(f"getField(非字面量字段名) 命中次数: {res['nonliteral_field']}"
f",覆盖脚本 {len(res['nonliteral_scripts'])} 条")
if __name__ == "__main__":
main()