- eip4-poc/tools/{s3_rewriter,loadtest} → tools/(P4 改写器种子与压测 harness 随工程演进,PoC 冻结)
- 新增工程 README(模块图/纪律/常用命令/自举说明)
- 根目录 parity-reports/ 为 CLI 默认输出草稿,入 .gitignore(正式报告在 platform-parity/parity-reports/)
Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
176 lines
6.4 KiB
Python
176 lines
6.4 KiB
Python
"""
|
||
mine_patterns.py — 真实 S3 语料模式挖掘
|
||
|
||
目的:
|
||
对全部 S3 脚本逐条提取 Vaadin API 调用模式并归一化形态,输出 Top 模式表
|
||
(模式形态 × 出现次数 × 覆盖脚本数),驱动 rewrite.py 规则扩展。
|
||
|
||
归一化策略(把字面量抹平,只看"形状"):
|
||
- getField("X") → getField(F) 字段句柄取数
|
||
- 变量名 / 字面串 → 抹平为 _
|
||
- 安全导航 ?. → 保留(区分 . 与 ?. 是关键发现)
|
||
- 链尾 .method(...) → .method(...) 只保留方法名与"有无参数"
|
||
|
||
输出:
|
||
1) 动词频率表: <持有者>.<动词> → 出现次数 / 覆盖脚本数
|
||
2) 形态分布: getField 前缀 vs 裸 / ?. vs . / 链长
|
||
3) 残留模式: 改写后仍含 S3 的脚本里,剩下的 Top 调用形态
|
||
|
||
用法:
|
||
python mine_patterns.py # 读 scripts_cache.json
|
||
python mine_patterns.py --top 50
|
||
python mine_patterns.py --residual # 只看改写后残留模式
|
||
"""
|
||
|
||
import argparse
|
||
import json
|
||
import os
|
||
import re
|
||
import sys
|
||
from collections import Counter, defaultdict
|
||
|
||
sys.path.insert(0, os.path.dirname(__file__))
|
||
from rewrite import rewrite_script, HARD_S3_PATTERNS
|
||
|
||
CACHE = os.path.join(os.path.dirname(__file__), "scripts_cache.json")
|
||
|
||
# ──────────────────────────────────────────────
|
||
# 调用形态提取
|
||
# ──────────────────────────────────────────────
|
||
|
||
# 匹配 group.getField("X") 或 group.getField('X') 或 group.getField(var)
|
||
_GETFIELD = re.compile(r'group\.getField\(\s*(["\'][^"\']*["\']|[\w.]+)\s*\)')
|
||
|
||
# 匹配任意 .method( 或 ?.method( (持有者不限)
|
||
_METHOD_CALL = re.compile(r'(\?\.|\.)\s*(\w+)\s*\(')
|
||
|
||
# 字段句柄链: group.getField(...) [?.|.] method(...) 捕获导航符 + 方法名
|
||
_FIELD_CHAIN = re.compile(
|
||
r'group\.getField\(\s*(["\'][^"\']*["\']|[\w.]+)\s*\)\s*(\?\.|\.)\s*(\w+)\s*\('
|
||
)
|
||
|
||
# 组级调用: group.<method>( (排除 getField)
|
||
_GROUP_CALL = re.compile(r'\bgroup\s*(\?\.|\.)\s*(\w+)\s*\(')
|
||
|
||
# 静态调用: Identifier.method( 例如 Notification.show
|
||
_STATIC_CALL = re.compile(r'\b([A-Z]\w+)\.(\w+)\s*\(')
|
||
|
||
|
||
def _is_string_literal(arg: str) -> bool:
|
||
return bool(arg) and arg[0] in '"\''
|
||
|
||
|
||
def normalize_field_chain(nav: str, method: str) -> str:
|
||
"""归一: getField(F)<nav><method>(...)"""
|
||
return f'getField(F){nav}{method}(...)'
|
||
|
||
|
||
def mine(scripts, only_residual=False):
|
||
verb_count = Counter() # 形态 → 出现总次数
|
||
verb_scripts = defaultdict(set) # 形态 → 覆盖脚本 id 集合
|
||
nonliteral_field = 0 # getField(非字面量) 命中脚本数
|
||
nonliteral_scripts = set()
|
||
|
||
# 形态细分计数
|
||
nav_dot = Counter() # 安全导航 vs 普通点
|
||
group_verbs = Counter()
|
||
group_verb_scripts = defaultdict(set)
|
||
static_verbs = Counter()
|
||
static_verb_scripts = defaultdict(set)
|
||
|
||
for r in scripts:
|
||
src = r["source"]
|
||
rid = r["id"]
|
||
|
||
if only_residual:
|
||
rewritten, _, label = rewrite_script(src)
|
||
if label != "PARTIAL":
|
||
continue
|
||
scan = rewritten
|
||
else:
|
||
# 跳过 HARD(不该被自动化,不参与挖掘)
|
||
_, _, label = rewrite_script(src)
|
||
if label == "HARD":
|
||
continue
|
||
scan = src
|
||
|
||
# 字段句柄链
|
||
for m in _FIELD_CHAIN.finditer(scan):
|
||
arg, nav, method = m.group(1), m.group(2), m.group(3)
|
||
shape = normalize_field_chain(nav, method)
|
||
verb_count[shape] += 1
|
||
verb_scripts[shape].add(rid)
|
||
nav_dot[nav] += 1
|
||
if not _is_string_literal(arg):
|
||
nonliteral_field += 1
|
||
nonliteral_scripts.add(rid)
|
||
|
||
# 组级调用
|
||
for m in _GROUP_CALL.finditer(scan):
|
||
nav, method = m.group(1), m.group(2)
|
||
if method == "getField":
|
||
continue
|
||
shape = f'group{nav}{method}(...)'
|
||
group_verbs[shape] += 1
|
||
group_verb_scripts[shape].add(rid)
|
||
|
||
# 静态调用
|
||
for m in _STATIC_CALL.finditer(scan):
|
||
cls, method = m.group(1), m.group(2)
|
||
shape = f'{cls}.{method}(...)'
|
||
static_verbs[shape] += 1
|
||
static_verb_scripts[shape].add(rid)
|
||
|
||
return {
|
||
"verb_count": verb_count,
|
||
"verb_scripts": verb_scripts,
|
||
"nav_dot": nav_dot,
|
||
"nonliteral_field": nonliteral_field,
|
||
"nonliteral_scripts": nonliteral_scripts,
|
||
"group_verbs": group_verbs,
|
||
"group_verb_scripts": group_verb_scripts,
|
||
"static_verbs": static_verbs,
|
||
"static_verb_scripts": static_verb_scripts,
|
||
}
|
||
|
||
|
||
def print_table(title, counter, script_map, top):
|
||
print(f"\n{'='*72}\n{title}\n{'='*72}")
|
||
print(f"{'形态':<42}{'次数':>8}{'覆盖脚本':>10}")
|
||
print("-" * 72)
|
||
for shape in sorted(counter, key=lambda k: -counter[k])[:top]:
|
||
cov = len(script_map[shape]) if shape in script_map else 0
|
||
print(f"{shape:<42}{counter[shape]:>8}{cov:>10}")
|
||
|
||
|
||
def main():
|
||
p = argparse.ArgumentParser()
|
||
p.add_argument("--cache", default=CACHE)
|
||
p.add_argument("--top", type=int, default=40)
|
||
p.add_argument("--residual", action="store_true",
|
||
help="只对 PARTIAL 脚本改写后的残留模式做挖掘")
|
||
args = p.parse_args()
|
||
|
||
scripts = json.load(open(args.cache, encoding="utf-8"))
|
||
print(f"加载脚本 {len(scripts)} 条 from {args.cache}")
|
||
mode = "改写后残留模式(PARTIAL)" if args.residual else "全量 S3 模式(剔除 HARD)"
|
||
print(f"挖掘模式: {mode}")
|
||
|
||
res = mine(scripts, only_residual=args.residual)
|
||
|
||
print_table("字段句柄链形态 getField(F).<动词>(...)",
|
||
res["verb_count"], res["verb_scripts"], args.top)
|
||
print_table("组级调用形态 group.<动词>(...)",
|
||
res["group_verbs"], res["group_verb_scripts"], args.top)
|
||
print_table("静态调用形态 Class.<方法>(...)",
|
||
res["static_verbs"], res["static_verb_scripts"], args.top)
|
||
|
||
print(f"\n{'='*72}\n形态细分\n{'='*72}")
|
||
print(f"导航符分布: {dict(res['nav_dot'])}")
|
||
print(f"getField(非字面量字段名) 命中次数: {res['nonliteral_field']}"
|
||
f",覆盖脚本 {len(res['nonliteral_scripts'])} 条")
|
||
|
||
|
||
if __name__ == "__main__":
|
||
main()
|