chore: 工具迁入正式工程 + 工程 README + 根目录草稿报告入 ignore
- eip4-poc/tools/{s3_rewriter,loadtest} → tools/(P4 改写器种子与压测 harness 随工程演进,PoC 冻结)
- 新增工程 README(模块图/纪律/常用命令/自举说明)
- 根目录 parity-reports/ 为 CLI 默认输出草稿,入 .gitignore(正式报告在 platform-parity/parity-reports/)
Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
175
tools/s3_rewriter/mine_patterns.py
Normal file
175
tools/s3_rewriter/mine_patterns.py
Normal file
@@ -0,0 +1,175 @@
|
||||
"""
|
||||
mine_patterns.py — 真实 S3 语料模式挖掘
|
||||
|
||||
目的:
|
||||
对全部 S3 脚本逐条提取 Vaadin API 调用模式并归一化形态,输出 Top 模式表
|
||||
(模式形态 × 出现次数 × 覆盖脚本数),驱动 rewrite.py 规则扩展。
|
||||
|
||||
归一化策略(把字面量抹平,只看"形状"):
|
||||
- getField("X") → getField(F) 字段句柄取数
|
||||
- 变量名 / 字面串 → 抹平为 _
|
||||
- 安全导航 ?. → 保留(区分 . 与 ?. 是关键发现)
|
||||
- 链尾 .method(...) → .method(...) 只保留方法名与"有无参数"
|
||||
|
||||
输出:
|
||||
1) 动词频率表: <持有者>.<动词> → 出现次数 / 覆盖脚本数
|
||||
2) 形态分布: getField 前缀 vs 裸 / ?. vs . / 链长
|
||||
3) 残留模式: 改写后仍含 S3 的脚本里,剩下的 Top 调用形态
|
||||
|
||||
用法:
|
||||
python mine_patterns.py # 读 scripts_cache.json
|
||||
python mine_patterns.py --top 50
|
||||
python mine_patterns.py --residual # 只看改写后残留模式
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import sys
|
||||
from collections import Counter, defaultdict
|
||||
|
||||
sys.path.insert(0, os.path.dirname(__file__))
|
||||
from rewrite import rewrite_script, HARD_S3_PATTERNS
|
||||
|
||||
CACHE = os.path.join(os.path.dirname(__file__), "scripts_cache.json")
|
||||
|
||||
# ──────────────────────────────────────────────
|
||||
# 调用形态提取
|
||||
# ──────────────────────────────────────────────
|
||||
|
||||
# 匹配 group.getField("X") 或 group.getField('X') 或 group.getField(var)
|
||||
_GETFIELD = re.compile(r'group\.getField\(\s*(["\'][^"\']*["\']|[\w.]+)\s*\)')
|
||||
|
||||
# 匹配任意 .method( 或 ?.method( (持有者不限)
|
||||
_METHOD_CALL = re.compile(r'(\?\.|\.)\s*(\w+)\s*\(')
|
||||
|
||||
# 字段句柄链: group.getField(...) [?.|.] method(...) 捕获导航符 + 方法名
|
||||
_FIELD_CHAIN = re.compile(
|
||||
r'group\.getField\(\s*(["\'][^"\']*["\']|[\w.]+)\s*\)\s*(\?\.|\.)\s*(\w+)\s*\('
|
||||
)
|
||||
|
||||
# 组级调用: group.<method>( (排除 getField)
|
||||
_GROUP_CALL = re.compile(r'\bgroup\s*(\?\.|\.)\s*(\w+)\s*\(')
|
||||
|
||||
# 静态调用: Identifier.method( 例如 Notification.show
|
||||
_STATIC_CALL = re.compile(r'\b([A-Z]\w+)\.(\w+)\s*\(')
|
||||
|
||||
|
||||
def _is_string_literal(arg: str) -> bool:
|
||||
return bool(arg) and arg[0] in '"\''
|
||||
|
||||
|
||||
def normalize_field_chain(nav: str, method: str) -> str:
|
||||
"""归一: getField(F)<nav><method>(...)"""
|
||||
return f'getField(F){nav}{method}(...)'
|
||||
|
||||
|
||||
def mine(scripts, only_residual=False):
|
||||
verb_count = Counter() # 形态 → 出现总次数
|
||||
verb_scripts = defaultdict(set) # 形态 → 覆盖脚本 id 集合
|
||||
nonliteral_field = 0 # getField(非字面量) 命中脚本数
|
||||
nonliteral_scripts = set()
|
||||
|
||||
# 形态细分计数
|
||||
nav_dot = Counter() # 安全导航 vs 普通点
|
||||
group_verbs = Counter()
|
||||
group_verb_scripts = defaultdict(set)
|
||||
static_verbs = Counter()
|
||||
static_verb_scripts = defaultdict(set)
|
||||
|
||||
for r in scripts:
|
||||
src = r["source"]
|
||||
rid = r["id"]
|
||||
|
||||
if only_residual:
|
||||
rewritten, _, label = rewrite_script(src)
|
||||
if label != "PARTIAL":
|
||||
continue
|
||||
scan = rewritten
|
||||
else:
|
||||
# 跳过 HARD(不该被自动化,不参与挖掘)
|
||||
_, _, label = rewrite_script(src)
|
||||
if label == "HARD":
|
||||
continue
|
||||
scan = src
|
||||
|
||||
# 字段句柄链
|
||||
for m in _FIELD_CHAIN.finditer(scan):
|
||||
arg, nav, method = m.group(1), m.group(2), m.group(3)
|
||||
shape = normalize_field_chain(nav, method)
|
||||
verb_count[shape] += 1
|
||||
verb_scripts[shape].add(rid)
|
||||
nav_dot[nav] += 1
|
||||
if not _is_string_literal(arg):
|
||||
nonliteral_field += 1
|
||||
nonliteral_scripts.add(rid)
|
||||
|
||||
# 组级调用
|
||||
for m in _GROUP_CALL.finditer(scan):
|
||||
nav, method = m.group(1), m.group(2)
|
||||
if method == "getField":
|
||||
continue
|
||||
shape = f'group{nav}{method}(...)'
|
||||
group_verbs[shape] += 1
|
||||
group_verb_scripts[shape].add(rid)
|
||||
|
||||
# 静态调用
|
||||
for m in _STATIC_CALL.finditer(scan):
|
||||
cls, method = m.group(1), m.group(2)
|
||||
shape = f'{cls}.{method}(...)'
|
||||
static_verbs[shape] += 1
|
||||
static_verb_scripts[shape].add(rid)
|
||||
|
||||
return {
|
||||
"verb_count": verb_count,
|
||||
"verb_scripts": verb_scripts,
|
||||
"nav_dot": nav_dot,
|
||||
"nonliteral_field": nonliteral_field,
|
||||
"nonliteral_scripts": nonliteral_scripts,
|
||||
"group_verbs": group_verbs,
|
||||
"group_verb_scripts": group_verb_scripts,
|
||||
"static_verbs": static_verbs,
|
||||
"static_verb_scripts": static_verb_scripts,
|
||||
}
|
||||
|
||||
|
||||
def print_table(title, counter, script_map, top):
|
||||
print(f"\n{'='*72}\n{title}\n{'='*72}")
|
||||
print(f"{'形态':<42}{'次数':>8}{'覆盖脚本':>10}")
|
||||
print("-" * 72)
|
||||
for shape in sorted(counter, key=lambda k: -counter[k])[:top]:
|
||||
cov = len(script_map[shape]) if shape in script_map else 0
|
||||
print(f"{shape:<42}{counter[shape]:>8}{cov:>10}")
|
||||
|
||||
|
||||
def main():
|
||||
p = argparse.ArgumentParser()
|
||||
p.add_argument("--cache", default=CACHE)
|
||||
p.add_argument("--top", type=int, default=40)
|
||||
p.add_argument("--residual", action="store_true",
|
||||
help="只对 PARTIAL 脚本改写后的残留模式做挖掘")
|
||||
args = p.parse_args()
|
||||
|
||||
scripts = json.load(open(args.cache, encoding="utf-8"))
|
||||
print(f"加载脚本 {len(scripts)} 条 from {args.cache}")
|
||||
mode = "改写后残留模式(PARTIAL)" if args.residual else "全量 S3 模式(剔除 HARD)"
|
||||
print(f"挖掘模式: {mode}")
|
||||
|
||||
res = mine(scripts, only_residual=args.residual)
|
||||
|
||||
print_table("字段句柄链形态 getField(F).<动词>(...)",
|
||||
res["verb_count"], res["verb_scripts"], args.top)
|
||||
print_table("组级调用形态 group.<动词>(...)",
|
||||
res["group_verbs"], res["group_verb_scripts"], args.top)
|
||||
print_table("静态调用形态 Class.<方法>(...)",
|
||||
res["static_verbs"], res["static_verb_scripts"], args.top)
|
||||
|
||||
print(f"\n{'='*72}\n形态细分\n{'='*72}")
|
||||
print(f"导航符分布: {dict(res['nav_dot'])}")
|
||||
print(f"getField(非字面量字段名) 命中次数: {res['nonliteral_field']}"
|
||||
f",覆盖脚本 {len(res['nonliteral_scripts'])} 条")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in New Issue
Block a user