F2-4(一): 修 XML 实体污染并重批重入库,固化标准入库器
- run_eval: PG xpath(...)::text 保留实体转义(< → <),抽取后补 XML 预定义实体反转义 (等价手册 §5.3 dom4j getText() 语义);293 个污染产物清零,Groovy 编译失败根因消除 - 缓存重建(源=eip4dev,1428 条:1 模板改名+1 新增脚本属数据演进);全量重跑 497/30/86/815,自动化率 81.1% 守住 ≥80% - 重大发现:同模板内脚本顺序在两次抽取间会交换((templateId,seq) 非稳定键, welfare 模板 seq1↔seq2 实锤)——重批后 onEvent seq 已与磁盘对齐,长期需内容哈希键(登记 F2-4) - tools/page-schema-loader/load_page_schema.py:幂等标准入库器(etag=sha256,version 递增, 仅允许 localhost 双库);本次双库各更新 756/幂等跳过 733,e2e 28/28 保持全绿 - 坑:mvn exec:java 的 -Dexec.args 引号空参会被吞导致参数错位(密码用占位符代替空串) Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
99
tools/page-schema-loader/load_page_schema.py
Normal file
99
tools/page-schema-loader/load_page_schema.py
Normal file
@@ -0,0 +1,99 @@
|
||||
"""
|
||||
load_page_schema.py — 把批量转换产物(转换产物/schema/*.json)幂等入库 sys_page_schema。
|
||||
|
||||
P4-4 首次入库用的是一次性手段未固化;本脚本(F2-4)成为唯一标准入库器:
|
||||
- etag = 文件内容 sha-256 hex(与 PageSchemaController 的 ETag/304 口径一致)
|
||||
- 已存在且 etag 不同 → 更新 schema_json/schema_etag,version+1
|
||||
- 不存在 → 插入 version=1,name 从同库 view_template 反查
|
||||
- etag 相同 → 跳过(幂等)
|
||||
|
||||
用法:
|
||||
python3 load_page_schema.py --schema-dir <dir> \
|
||||
--target host:port:dbname:schema:user [--target ...]
|
||||
默认双库:localhost:5433:eip4dev:eip3:zhengwr 与 localhost:5433:eip4m0:public:zhengwr
|
||||
仅允许 localhost(本地双库是仅有的合法目标;客户库副本 5435 只读红线)。
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
import time
|
||||
|
||||
|
||||
def upsert_target(schema_dir: str, host: str, port: str, dbname: str, schema: str, user: str):
|
||||
import psycopg2
|
||||
|
||||
if host not in ("localhost", "127.0.0.1"):
|
||||
raise SystemExit(f"拒绝目标 {host}:仅允许 localhost(5435 客户库只读红线)")
|
||||
|
||||
conn = psycopg2.connect(host=host, port=int(port), dbname=dbname, user=user,
|
||||
options=f"-c search_path={schema}")
|
||||
conn.autocommit = False
|
||||
inserted = updated = skipped = 0
|
||||
now_ms = int(time.time() * 1000)
|
||||
try:
|
||||
with conn.cursor() as cur:
|
||||
for fn in sorted(os.listdir(schema_dir)):
|
||||
if not fn.endswith(".json"):
|
||||
continue
|
||||
tid = fn[:-5]
|
||||
with open(os.path.join(schema_dir, fn), "rb") as f:
|
||||
raw = f.read()
|
||||
# 入库内容 = 文件原文(jsonb 规范化由 PG 负责);etag 对原文计算
|
||||
etag = hashlib.sha256(raw).hexdigest()
|
||||
text = raw.decode("utf-8")
|
||||
json.loads(text) # 入库前自检:JSON 必须合法
|
||||
|
||||
cur.execute("SELECT schema_etag FROM sys_page_schema WHERE template_id=%s", (tid,))
|
||||
row = cur.fetchone()
|
||||
if row is None:
|
||||
cur.execute("SELECT name FROM view_template WHERE trim(view_template_id)=%s", (tid,))
|
||||
nrow = cur.fetchone()
|
||||
name = nrow[0] if nrow else tid
|
||||
cur.execute(
|
||||
"INSERT INTO sys_page_schema(template_id,name,schema_json,schema_etag,version,update_time) "
|
||||
"VALUES (%s,%s,%s::jsonb,%s,1,%s)",
|
||||
(tid, name, text, etag, now_ms))
|
||||
inserted += 1
|
||||
elif row[0] != etag:
|
||||
cur.execute(
|
||||
"UPDATE sys_page_schema SET schema_json=%s::jsonb, schema_etag=%s, "
|
||||
"version=version+1, update_time=%s WHERE template_id=%s",
|
||||
(text, etag, now_ms, tid))
|
||||
updated += 1
|
||||
else:
|
||||
skipped += 1
|
||||
conn.commit()
|
||||
except Exception:
|
||||
conn.rollback()
|
||||
raise
|
||||
finally:
|
||||
conn.close()
|
||||
print(f"[{dbname}.{schema}] 插入 {inserted} / 更新 {updated} / 幂等跳过 {skipped}")
|
||||
|
||||
|
||||
def main():
|
||||
here = os.path.dirname(os.path.abspath(__file__))
|
||||
default_dir = os.path.normpath(os.path.join(
|
||||
here, "..", "..", "platform-converter", "转换产物", "schema"))
|
||||
p = argparse.ArgumentParser()
|
||||
p.add_argument("--schema-dir", default=default_dir)
|
||||
p.add_argument("--target", action="append",
|
||||
help="host:port:dbname:schema:user,可多次;缺省=eip4dev+eip4m0 双库")
|
||||
args = p.parse_args()
|
||||
|
||||
targets = args.target or [
|
||||
"localhost:5433:eip4dev:eip3:zhengwr",
|
||||
"localhost:5433:eip4m0:public:zhengwr",
|
||||
]
|
||||
if not os.path.isdir(args.schema_dir):
|
||||
raise SystemExit(f"产物目录不存在: {args.schema_dir}")
|
||||
print(f"产物目录: {args.schema_dir} ({len([f for f in os.listdir(args.schema_dir) if f.endswith('.json')])} 文件)")
|
||||
for t in targets:
|
||||
upsert_target(args.schema_dir, *t.split(":"))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in New Issue
Block a user