diff --git a/references/generalization_test_log.md b/references/generalization_test_log.md new file mode 100644 index 0000000..6a433b0 --- /dev/null +++ b/references/generalization_test_log.md @@ -0,0 +1,92 @@ +# Skill 泛化测试记录(test3d-7 三项目,2026-07-02 起) + +目的:验证 cad-building-showcase skill 在不同 CAD 图纸上能否达到与贝昂案例 +同等的 3D 模型质量。判定标尺 = One-Shot Definition of Done 七条门禁 + +量化指标 + 固定目检清单,任何门禁失败 → 修 skill(不绕过)→ 重跑。 + +## 测试对象 + +| 项目 | 形态 | 导出 | 考验点 | +|---|---|---|---| +| 黄桥文体中心 | 单体大公建 | 天正 _t3 | 多图框单文件(平面图 A-01-101~107 共 7 框)、钢结构、大跨 | +| 13号地块 | 双塔 + 地库 + 总图 | 天正 _t3 | 厂区装配、住宅塔楼标准层、地库 | +| 27号地块 | 1#综合楼A/B + 2# + 配电房 + 门卫 + 地库 | _t3 与 _t8 混用 | 混合导出惯例、小体量构筑物、总平面定位 | + +## 质量对比表(随进度填充) + +| 门禁/指标 | 贝昂(基线) | 黄桥 | 13号 | 27号 | +|---|---|---|---|---| +| 读图 Warnings 清零 | PASS | PASS(13/13 ok,ODA 兜底×3) | | | +| 图框网格对齐 votes≥3 | PASS | PASS(7框全对齐 x=14 y≥11) | | | +| audit exit 0(三硬门禁) | PASS | PASS | | | +| 无 topless_group / 三角化失败 | PASS | PASS(补 roof_specs 后) | | | +| 邻栋间距门禁 | PASS(0.19m 贴缝) | n/a 单体 | | | +| 裁剪口径覆盖率 | ~40%(多图框正常) | 56.9% | | | +| 幕墙连续率 | 100% | 100%(6/6 层) | | | +| GLB(Draco) | 0.57MB/121节点 | 0.37MB/42节点 | | | +| 目检(渲染+viewer) | PASS | PASS(console 零消息) | | | + +黄桥备注:柱仅 5 段(天正柱为 HATCH 填充,架构图不可提取——SKILL.md 已知 +限制,需结构图补柱网);F4 幕墙延伸出墙体范围属实(高大厅通高幕墙出现在 +多层平面),按图提取;主屋面暂合并为 20.1m 单板(25.0m 高体量顶待按 +剖面证据细分 roof_specs);层高取自立面剖面 PUB_TEXT 标高 +(B1=-5.5/F1=0/F2=5.6/F3=11.0/F4=15.5/RF=20.1)。 + +## 泛化缺口与修复清单 + +1. **dwg2dxf GBK 输出崩报告**(发现于黄桥首跑):libredwg 在中文图纸上 + stdout/stderr 是 GBK 字节,`subprocess text=True` 按 UTF-8 解码直接崩。 + 修复:改 bytes 捕获 + `errors="replace"` 解码。 +2. **消毒步骤毁 CJK 文本**(发现于黄桥二跑):`sanitize_dxf_comments` 用 + UTF-8 `errors="ignore"` round-trip 重写 ANSI_936/GBK 的 DXF, + "一层平面图"→"一平图",下游 detect_sheet_frames 的平面图标题匹配整体失效; + 结构破损还让 ezdxf 落入 recover 的 surrogateescape 路径,报告写盘 + `surrogates not allowed` 崩溃。修复:消毒改**字节级**处理(保持原编码), + 另加 `scrub_surrogates` 纵深防御(recover 泄漏的代理字符按 GBK 还原, + 还原不了就替换,单个坏图不再拖死整批)。回归测试 +4 项(GBK 合成 DXF + 全链路)。 + +(待续) + +3. **消毒步骤误删 XRECORD 的 999 数值行**(发现于苏州博物馆/test3d-8 首跑): + `sanitize_dxf_comments` 逐行匹配 `999`,但 DXF 是严格 码/值 行对;ODA 转出的 + XRECORD 里整数值恰为 999 时被当成注释码连删两行,后续组码整体错位, + strict 读取抛 `ValueError: Invalid handle {ACAD_REACTORS.`(非 DXFError, + read_doc 不兜底)→ 整文件误判 failed。libredwg 输出同样中招——首跑 111 个 + "失败"文件中相当一部分是消毒自伤。修复:按码/值对遍历(只在码位上匹配 999); + read_doc 兜底 `(DXFError, ValueError)` 走 recover。 +4. **MIF 图名解码落地**(黄桥 TDD 存量):`\M+5XXXX` 转义在 cad_dxf_report 的 + count_plan_titles 与 detect_sheet_frames 的 entity_text 统一解码 + (\M+1..5 → cp932/big5/cp949/johab/gbk),多帧警告与图框识别恢复。 + 回归测试 10 项全绿。 +5. **ODA 自动兜底进代码**(黄桥主平面图 0 实体触发):libredwg 转大型天正 + DWG 产坏件(重复句柄、缺 ENDBLK/EOF)或转出空 modelspace——黄桥 + A-01-101~107 经 dwg2dxf 读 0 实体、经 ODA 读 21,171 实体。 + cad_dxf_report 现在对"转换失败 / inspect failed / modelspace 0 实体"的 + DWG 自动走 ODA File Converter 重转(ASCII 暂存名规避编码坑、1800s 超时、 + `oda_fallback_ok/failed` 留痕;env `ODA_CONVERTER` 可覆盖路径)。 + 回归测试用假目录转换器锁住兜底管线(3 项,沙盒无 ODA 也可跑)。 +7. **图签窗图例进模型,footprint 拉出"每层一个尖角"**(黄桥,用户目检发现): + 每张图纸图签区画的窗样符号在真实 CURTWALL 层上(每框 6 段共 4.6m,距主体 + 18–40m,逐层同位),图层排除逮不住;footprint 凹包被拉过去生成细三角飘 + 板。修复:derive 新增 `drop_isolated_clusters`(默认开)——分类后对全部 + 段做端点邻近聚簇(union-find + 网格桶,linkage 3m),保留最大簇,丢弃 + "距主体 >12m 且总长 <15m"的孤立微簇并大声警告;bounds 随过滤重算。真实 + 的脱开构件(连廊/门头,总长大或贴近主体)不受影响;F4 西侧高大厅通高幕 + 墙保留验证通过。回归 6 项(默认丢弃+警告+bounds 收缩+开关保留)。 +8. **无支撑屋面线挤成悬空飘带**(黄桥,用户在 Blender 里发现 F3__roof 悬浮): + 平面图会画低矮雨篷/坡屋面轮廓(A-ROOF),其支撑柱是天正 HATCH 填充提不出 + 来——黄桥 F3 西南角坡屋面下方 F1/F2 均无一段墙,挤出后 15 段线带悬在 + 11m 空中;B1 平面上的地面雨篷投影线更是埋在 -5.85m。修复:derive 新增 + `drop_unsupported_roof_lines`(默认开)——roof 段中点在"全部楼层 + footprint+caps 并集 buffer 2m"之外即丢弃并大声警告(黄桥 F3 丢 10 段/ + 24.1m、B1 丢 1 段/82m;贴主体 2m 内的附墙雨篷边保留);要保留可显式加 + roof_specs 板或关开关。回归 5 项(域内保留/域外丢弃/警告/开关)。 +9. **轴网藏在匿名块里,图框对齐全 WEAK**(黄桥):detect_sheet_frames 只扫 + modelspace 线,而这套图把每框轴网包在 A$C... 匿名块(DOTE 线 296/203/197 + 条在块内、msp 仅 8 条),投票 x=0 y=0 → 全部落到 clip 角点兜底。修复: + 检测器递归展开 INSERT(深度≤3、按块名 memo 只展开含轴/墙层曲线的块), + 黄桥 7 框全部恢复 grid-aligned(votes x=14 y≥11),B1 的独立偏移被正确 + 纠出。回归用例构造"轴网在块内 + 两框墙体 bbox 不同"场景,clip 角点兜底 + 必错 20m、网格投票必对;教训:合成轴网要用不均匀开间,等距网格的子集 + 相关天然多解。 diff --git a/scripts/cad_dxf_report.py b/scripts/cad_dxf_report.py index 4d96579..0296d61 100755 --- a/scripts/cad_dxf_report.py +++ b/scripts/cad_dxf_report.py @@ -5,6 +5,7 @@ from __future__ import annotations import argparse import json +import os import shutil import subprocess import sys @@ -68,32 +69,94 @@ def run_dwg2dxf(source: Path, dest: Path) -> dict: } dest.parent.mkdir(parents=True, exist_ok=True) cmd = [exe, "-y", "-o", str(dest), str(source)] - proc = subprocess.run(cmd, text=True, capture_output=True) + # libredwg echoes CJK layer/file names in the platform encoding (GBK on + # Chinese drawings), so strict UTF-8 text mode crashes; decode leniently. + proc = subprocess.run(cmd, capture_output=True) + stdout = proc.stdout.decode("utf-8", errors="replace") + stderr = proc.stderr.decode("utf-8", errors="replace") return { "source": str(source), "dest": str(dest), "status": "ok" if proc.returncode == 0 and dest.exists() else "failed", "returncode": proc.returncode, - "stdout": proc.stdout[-2000:], - "stderr": proc.stderr[-2000:], + "stdout": stdout[-2000:], + "stderr": stderr[-2000:], } +def oda_converter() -> str | None: + """ODA File Converter path (env ODA_CONVERTER overrides the macOS default).""" + override = os.environ.get("ODA_CONVERTER") + if override: + return override if Path(override).exists() else None + default = Path("/Applications/ODAFileConverter.app/Contents/MacOS/ODAFileConverter") + return str(default) if default.exists() else None + + +def run_oda_batch(sources: list[Path], work_dir: Path) -> dict[Path, Path]: + """Convert DWGs via ODA File Converter; returns {source: produced_dxf}. + + ODA converts whole directories with absolute paths and reports bad + arguments via GUI dialogs (silent exit-0), so sources are staged under + ASCII names in a scratch in-dir and results matched back by stem. + """ + exe = oda_converter() + if not exe or not sources: + return {} + in_dir = work_dir / "oda_in" + out_dir = work_dir / "oda_out" + shutil.rmtree(in_dir, ignore_errors=True) + shutil.rmtree(out_dir, ignore_errors=True) + in_dir.mkdir(parents=True) + out_dir.mkdir(parents=True) + mapping: dict[str, Path] = {} + for index, src in enumerate(sources): + stem = f"oda{index:03d}" + shutil.copy2(src, in_dir / f"{stem}.dwg") + mapping[stem] = src + try: + subprocess.run( + [exe, str(in_dir.resolve()), str(out_dir.resolve()), "ACAD2018", "DXF", "0", "1", "*.dwg"], + capture_output=True, timeout=1800, + ) + except (subprocess.TimeoutExpired, OSError): + return {} + produced: dict[Path, Path] = {} + for stem, src in mapping.items(): + out = out_dir / f"{stem}.dxf" + if out.exists() and out.stat().st_size > 0: + produced[src] = out + return produced + + def sanitize_dxf_comments(source: Path, dest: Path) -> int: - """Remove DXF group code 999 comment pairs that can break brittle importers.""" - lines = source.read_text(errors="ignore").splitlines(keepends=True) + """Remove DXF group code 999 comment pairs that can break brittle importers. + + Operates on BYTES: pre-2007 DXF from libredwg/TArch stores CJK text in the + $DWGCODEPAGE encoding (ANSI_936/GBK). A UTF-8 round-trip with + errors="ignore" silently drops those bytes, mangling every CJK layer/text + ("一层平面图" -> "一平图") and breaking plan-title detection downstream, so + the file content must pass through unchanged apart from the comment pairs. + """ + lines = source.read_bytes().splitlines(keepends=True) dest.parent.mkdir(parents=True, exist_ok=True) - cleaned: list[str] = [] + cleaned: list[bytes] = [] removed = 0 index = 0 - while index < len(lines): - if lines[index].strip() == "999": + # DXF is a strict alternation of group-code line / value line, so walk in + # pairs: a bare "999" VALUE (e.g. an integer inside an ODA XRECORD) must + # not be mistaken for a comment group code, or the pair removal shifts + # every following tag and corrupts the file. + while index + 1 < len(lines): + if lines[index].strip() == b"999": removed += 1 index += 2 continue cleaned.append(lines[index]) - index += 1 - dest.write_text("".join(cleaned), encoding="utf-8", errors="ignore") + cleaned.append(lines[index + 1]) + index += 2 + cleaned.extend(lines[index:]) + dest.write_bytes(b"".join(cleaned)) return removed @@ -130,7 +193,10 @@ def guess_unit_scale(bbox_data: dict | None, insunits: int) -> dict: def read_doc(path: Path, ezdxf, recover): try: return ezdxf.readfile(path), [] - except ezdxf.DXFError: + except (ezdxf.DXFError, ValueError): + # Strict reads can also raise plain ValueError on quirky-but- + # recoverable files (e.g. an XRECORD holding "{ACAD_REACTORS" where a + # handle is expected); recover skips those records. doc, auditor = recover.readfile(path) return doc, [str(error) for error in auditor.errors] @@ -167,6 +233,27 @@ import re as _re PLAN_TITLE_RE = _re.compile(r"平面图$") PLAN_TITLE_EXCLUDE_RE = _re.compile(r"(示意|疏散|分区|索引|定位|总平面|放大|详图|布置|大样)") +# AutoCAD MIF escapes (\M+5XXXX): CJK text in pre-Unicode DWGs survives +# DXF conversion as literal escape sequences, so title matching must decode +# them first. The digit selects the codepage; XXXX is the hex byte pair. +MIF_RE = _re.compile(r"\\M\+([1-5])([0-9A-Fa-f]{4})") +MIF_CODEPAGES = {"1": "cp932", "2": "big5", "3": "cp949", "4": "johab", "5": "gbk"} + + +def decode_mif(text: str) -> str: + if "\\M+" not in text: + return text + + def repl(match: "_re.Match[str]") -> str: + value = int(match.group(2), 16) + raw = value.to_bytes(2, "big") if value > 0xFF else value.to_bytes(1, "big") + try: + return raw.decode(MIF_CODEPAGES.get(match.group(1), "gbk")) + except (UnicodeDecodeError, LookupError): + return match.group(0) + + return MIF_RE.sub(repl, text) + def count_plan_titles(entities) -> int: """Distinct plan-frame titles in modelspace (multi-frame sheet detector).""" @@ -179,6 +266,7 @@ def count_plan_titles(entities) -> int: text = str(entity.dxf.text if kind == "TEXT" else entity.plain_text() or "").strip() except Exception: continue + text = decode_mif(text) if not text or len(text) > 40: continue if PLAN_TITLE_RE.search(text) and not PLAN_TITLE_EXCLUDE_RE.search(text): @@ -335,6 +423,28 @@ def output_paths(output: Path) -> tuple[Path, Path]: return output / "cad_read_report.json", output / "cad_read_report.md" +def scrub_surrogates(value): + """Make report data JSON-writable when ezdxf's recover mode leaks + surrogate-escaped bytes (broken files read with errors="surrogateescape"). + CJK bytes are recovered via GBK where possible; the batch never dies on + one corrupt drawing.""" + if isinstance(value, str): + try: + value.encode("utf-8") + return value + except UnicodeEncodeError: + raw = value.encode("utf-8", errors="surrogateescape") + try: + return raw.decode("gbk") + except UnicodeDecodeError: + return raw.decode("utf-8", errors="replace") + if isinstance(value, dict): + return {scrub_surrogates(k): scrub_surrogates(v) for k, v in value.items()} + if isinstance(value, list): + return [scrub_surrogates(item) for item in value] + return value + + def main() -> int: args = parse_args() input_dir = Path(args.input) @@ -376,6 +486,48 @@ def main() -> int: conversions.append({"source": str(raw), "dest": str(dest), "status": f"sanitized_{removed}_comments"}) results = [inspect_file(path, ezdxf, bbox, recover) for path in sorted(sanitized)] + + # ODA auto-fallback: libredwg chokes on some TArch DWGs (duplicate handles, + # missing ENDBLK/EOF) or yields an empty modelspace even when the drawing + # is full — the 黄桥 plans sheet read 0 entities via dwg2dxf but 21k via + # ODA. Reconvert those sources through ODA when it is available. + dwg_dest_to_source = { + Path(c["dest"]).name: Path(c["source"]) + for c in conversions + if str(c.get("source", "")).lower().endswith(".dwg") and c.get("dest") + } + retry: dict[str, Path] = {} + for item in results: + src = dwg_dest_to_source.get(item.get("file", "")) + if src and (item.get("status") == "failed" or item.get("modelspace_entities") == 0): + retry[item["file"]] = src + for c in list(conversions): + if c.get("status") in {"failed", "missing_dwg2dxf"} and str(c.get("source", "")).lower().endswith(".dwg"): + name = Path(c["dest"]).name if c.get("dest") else Path(c["source"]).with_suffix(".dxf").name + retry.setdefault(name, Path(c["source"])) + if retry: + produced = run_oda_batch(list(retry.values()), work_dir) + for name, src in retry.items(): + out = produced.get(src) + if not out: + conversions.append({"source": str(src), "dest": "", "status": "oda_fallback_failed"}) + continue + raw_dest = raw_dir / name + shutil.copy2(out, raw_dest) + clean_dest = clean_dir / name + sanitize_dxf_comments(raw_dest, clean_dest) + item = inspect_file(clean_dest, ezdxf, bbox, recover) + replaced = False + for i, existing in enumerate(results): + if existing.get("file") == name: + results[i] = item + replaced = True + break + if not replaced: + results.append(item) + conversions.append({"source": str(src), "dest": str(raw_dest), "status": "oda_fallback_ok"}) + results.sort(key=lambda r: r.get("file", "")) + json_path, md_path = output_paths(Path(args.output)) json_path.parent.mkdir(parents=True, exist_ok=True) md_path.parent.mkdir(parents=True, exist_ok=True) @@ -386,6 +538,8 @@ def main() -> int: "conversions": conversions, "files": results, } + payload = scrub_surrogates(payload) + results = payload["files"] json_path.write_text(json.dumps(payload, ensure_ascii=False, indent=2), encoding="utf-8") md_path.write_text(markdown_report(results, conversions, input_dir), encoding="utf-8") print(json.dumps({"json": str(json_path), "markdown": str(md_path), "files": len(results)}, ensure_ascii=False)) diff --git a/scripts/derive_showcase_spec.py b/scripts/derive_showcase_spec.py index a197aa8..98f9460 100755 --- a/scripts/derive_showcase_spec.py +++ b/scripts/derive_showcase_spec.py @@ -771,6 +771,163 @@ def compute_caps(floors: dict[str, Any], config: dict[str, Any]) -> None: fl.setdefault("caps", []).append({"z": round(ftop, 6), "polygons": polys}) +def drop_isolated_clusters(segments: dict[str, list[dict[str, Any]]], config: dict[str, Any], floor_key: str) -> dict[str, list[dict[str, Any]]]: + """Drop tiny segment clusters far away from the floor's main mass. + + Sheet legends and title blocks draw window/door symbols with REAL geometry + layers (黄桥: four 1 m CURTWALL strips 30 m east of the building on every + frame), so layer exclusion cannot catch them — and the generated footprint + hull then stretches into sliver fins ("每层一个尖角"). Cluster all + classified segments by endpoint proximity, keep the largest cluster plus + anything near it or substantial, and WARN about what was dropped. + Config: {"drop_isolated_clusters": false} disables; tuning knobs + cluster_linkage (m, 3.0), isolation_distance (m, 12.0), + isolation_min_length (m, 15.0). + """ + if not config.get("drop_isolated_clusters", True): + return segments + linkage = float(config.get("cluster_linkage", 3.0)) + iso_dist = float(config.get("isolation_distance", 12.0)) + min_len = float(config.get("isolation_min_length", 15.0)) + + items: list[tuple[str, int, dict[str, Any]]] = [] + for category, segs in segments.items(): + for i, seg in enumerate(segs): + items.append((category, i, seg)) + if len(items) < 2: + return segments + + parent = list(range(len(items))) + + def find(a: int) -> int: + while parent[a] != a: + parent[a] = parent[parent[a]] + a = parent[a] + return a + + def union(a: int, b: int) -> None: + ra, rb = find(a), find(b) + if ra != rb: + parent[rb] = ra + + cell = max(linkage, 1e-6) + buckets: dict[tuple[int, int], list[int]] = {} + for idx, (_c, _i, seg) in enumerate(items): + for px, py in (seg["start"], seg["end"]): + buckets.setdefault((int(px // cell), int(py // cell)), []).append(idx) + for (bx, by), members in buckets.items(): + for gx in (bx - 1, bx, bx + 1): + for gy in (by - 1, by, by + 1): + for other in buckets.get((gx, gy), []): + union(members[0], other) + for other in members[1:]: + union(members[0], other) + + clusters: dict[int, list[int]] = {} + for idx in range(len(items)): + clusters.setdefault(find(idx), []).append(idx) + + def cluster_stats(indices: list[int]) -> tuple[float, list[float]]: + total = 0.0 + bx = [math.inf, math.inf, -math.inf, -math.inf] + for idx in indices: + seg = items[idx][2] + (x1, y1), (x2, y2) = seg["start"], seg["end"] + total += math.hypot(x2 - x1, y2 - y1) + bx[0] = min(bx[0], x1, x2); bx[1] = min(bx[1], y1, y2) + bx[2] = max(bx[2], x1, x2); bx[3] = max(bx[3], y1, y2) + return total, bx + + stats = {root: cluster_stats(idx_list) for root, idx_list in clusters.items()} + main_root = max(stats, key=lambda r: stats[r][0]) + main_box = stats[main_root][1] + + def box_gap(a: list[float], b: list[float]) -> float: + dx = max(0.0, max(a[0] - b[2], b[0] - a[2])) + dy = max(0.0, max(a[1] - b[3], b[1] - a[3])) + return math.hypot(dx, dy) + + dropped: set[int] = set() + for root, idx_list in clusters.items(): + if root == main_root: + continue + total, box = stats[root] + gap = box_gap(main_box, box) + if gap > iso_dist and total < min_len: + dropped.update(idx_list) + print( + f"WARNING isolated_cluster {floor_key}: dropped {len(idx_list)} segments " + f"({total:.1f}m total, {gap:.0f}m from main mass) — likely sheet legend/" + "title-block content; set drop_isolated_clusters=false to keep", + file=sys.stderr, + ) + if not dropped: + return segments + dropped_keys = {(items[d][0], items[d][1]) for d in dropped} + return { + category: [seg for i, seg in enumerate(segs) if (category, i) not in dropped_keys] + for category, segs in segments.items() + } + + +def drop_unsupported_roof_lines(floors: dict[str, Any], config: dict[str, Any]) -> None: + """Drop roof outlines that would extrude into ribbons floating in mid-air. + + Plans draw low canopies / porch roofs whose supports are hatch-only + columns the extractor cannot capture (黄桥: 坡屋面 outlines on the F3 + sheet with zero walls beneath on any floor). A roof segment is kept only + if some plate exists under it — any floor's footprint or cap polygon + within roof_support_distance (m, default 2.0). Dropped segments are + reported loudly; keep them via {"drop_unsupported_roof_lines": false} or + model the canopy explicitly with a roof_specs plate. + """ + if not config.get("drop_unsupported_roof_lines", True): + return + try: + from shapely.geometry import Point, Polygon + from shapely.ops import unary_union + except ImportError: + return + plates = [] + for fl in floors.values(): + polys = list(fl.get("footprint", {}).get("polygons", [])) + for cap in fl.get("caps", []): + polys.extend(cap.get("polygons", [])) + for coords in polys: + try: + poly = Polygon([(float(x), float(y)) for x, y in coords]) + if not poly.is_valid: + poly = poly.buffer(0) + if not poly.is_empty: + plates.append(poly) + except Exception: + continue + if not plates: + return # no footprint evidence at all — nothing to judge against + support = unary_union(plates).buffer(float(config.get("roof_support_distance", 2.0))) + for key, fl in floors.items(): + roof = fl.get("segments", {}).get("roof") + if not roof: + continue + kept, dropped_len, dropped = [], 0.0, 0 + for seg in roof: + mid = Point((seg["start"][0] + seg["end"][0]) / 2.0, (seg["start"][1] + seg["end"][1]) / 2.0) + if support.contains(mid): + kept.append(seg) + else: + dropped += 1 + dropped_len += math.hypot(seg["end"][0] - seg["start"][0], seg["end"][1] - seg["start"][1]) + if dropped: + print( + f"WARNING unsupported_roof {key}: dropped {dropped} roof segments " + f"({dropped_len:.1f}m) with no slab/cap beneath — likely a canopy whose " + "supports are hatch-only; set drop_unsupported_roof_lines=false or add a " + "roof_specs plate to keep them", + file=sys.stderr, + ) + fl["segments"]["roof"] = kept + + def finalize_spec( floor_data: dict[str, dict[str, Any]], config: dict[str, Any], @@ -804,6 +961,9 @@ def finalize_spec( "source_type": raw["source_type"], } ) + segments = drop_isolated_clusters(segments, config, key) + xs = [p for segs in segments.values() for seg in segs for p in (seg["start"][0], seg["end"][0])] + ys = [p for segs in segments.values() for seg in segs for p in (seg["start"][1], seg["end"][1])] bounds = [ round(min(xs), 6) if xs else 0.0, round(min(ys), 6) if ys else 0.0, @@ -834,6 +994,7 @@ def finalize_spec( gap_tol=float(config.get("glazing_gap_tol", 3.6)), ) compute_caps(floors, config) + drop_unsupported_roof_lines(floors, config) # Topless-roof guard: the top above-ground floor of each group must end in # a cap or a config roof_specs entry, otherwise the model renders open-top diff --git a/scripts/detect_sheet_frames.py b/scripts/detect_sheet_frames.py index 5e4f803..66dfc79 100644 --- a/scripts/detect_sheet_frames.py +++ b/scripts/detect_sheet_frames.py @@ -40,6 +40,26 @@ from pathlib import Path PLAN_TITLE = re.compile(r"平面图$") PLAN_EXCLUDE = re.compile(r"(示意|疏散|分区|索引|定位|总平面|放大|详图|布置|大样)") OTHER_TITLE = re.compile(r"(立面图|剖面图)$") + +# AutoCAD MIF escapes (\M+5XXXX): CJK titles in pre-Unicode DWGs arrive as +# literal escape sequences, invisible to the 平面图 regexes until decoded. +MIF_RE = re.compile(r"\\M\+([1-5])([0-9A-Fa-f]{4})") +MIF_CODEPAGES = {"1": "cp932", "2": "big5", "3": "cp949", "4": "johab", "5": "gbk"} + + +def decode_mif(text: str) -> str: + if "\\M+" not in text: + return text + + def repl(match: "re.Match[str]") -> str: + value = int(match.group(2), 16) + raw = value.to_bytes(2, "big") if value > 0xFF else value.to_bytes(1, "big") + try: + return raw.decode(MIF_CODEPAGES.get(match.group(1), "gbk")) + except (UnicodeDecodeError, LookupError): + return match.group(0) + + return MIF_RE.sub(repl, text) CN_NUM = {"一": 1, "二": 2, "三": 3, "四": 4, "五": 5, "六": 6, "七": 7, "八": 8, "九": 9, "十": 10} FLOOR_NAME_KEY = [ (re.compile(r"地下(?P[一二三四五六七八九十\d]+)层夹层"), lambda m: f"B{_cn(m.group('n'))}M"), @@ -100,9 +120,9 @@ def match_any(value: str, patterns: list[str]) -> bool: def entity_text(entity) -> str: try: if entity.dxftype() == "TEXT": - return str(entity.dxf.text or "").strip() + return decode_mif(str(entity.dxf.text or "").strip()) if entity.dxftype() == "MTEXT": - return str(entity.plain_text() or "").strip() + return decode_mif(str(entity.plain_text() or "").strip()) except Exception: pass return "" @@ -184,8 +204,65 @@ def main() -> int: plan_exclude = re.compile(args.title_exclude_regex) if args.title_exclude_regex else None titles, others = [], [] lines = [] # (layer, sx, sy, ex, ey) for LINE and polyline edges + + # Plan grids are not always modelspace lines: TArch exports may wrap each + # frame's axis system in an anonymous block (黄桥: A$C... wrappers holding + # hundreds of DOTE lines while msp has ~8). Expand INSERTs — but only + # blocks that actually contain axis/wall-layer curves (memoized per block + # name), or a sheet full of door/furniture blocks slows the scan for nothing. + expand_patterns = axis_layers + wall_layers + block_memo: dict[str, bool] = {} + + def block_has_expandable(name: str, depth: int = 0) -> bool: + if name in block_memo: + return block_memo[name] + if depth > 3 or name not in doc.blocks: + return False + block_memo[name] = False # cycle guard + found = False + for be in doc.blocks.get(name): + bkind = be.dxftype() + if bkind in {"LINE", "LWPOLYLINE"} and match_any(str(be.dxf.layer), expand_patterns): + found = True + break + if bkind == "INSERT" and block_has_expandable(str(be.dxf.name), depth + 1): + found = True + break + block_memo[name] = found + return found + + def collect_insert_lines(insert, depth: int = 0) -> None: + if depth > 3: + return + try: + virtual = list(insert.virtual_entities()) + except Exception: + return + for ve in virtual: + vkind = ve.dxftype() + if vkind == "LINE": + d = ve.dxf + if match_any(str(d.layer), expand_patterns): + lines.append((str(d.layer), float(d.start.x), float(d.start.y), float(d.end.x), float(d.end.y))) + elif vkind == "LWPOLYLINE": + layer = str(ve.dxf.layer) + if not match_any(layer, expand_patterns): + continue + try: + pts = [(float(px), float(py)) for px, py in ve.get_points("xy")] + except Exception: + continue + for a, b in zip(pts, pts[1:]): + lines.append((layer, a[0], a[1], b[0], b[1])) + elif vkind == "INSERT": + collect_insert_lines(ve, depth + 1) + for entity in msp: kind = entity.dxftype() + if kind == "INSERT": + if block_has_expandable(str(entity.dxf.name)): + collect_insert_lines(entity) + continue if kind in {"TEXT", "MTEXT"}: content = entity_text(entity) if not content or len(content) > 40: diff --git a/tests/test_extraction_pipeline.py b/tests/test_extraction_pipeline.py index 166c698..c57570a 100644 --- a/tests/test_extraction_pipeline.py +++ b/tests/test_extraction_pipeline.py @@ -95,12 +95,24 @@ def make_xref_shell_dxf(path: Path) -> None: doc.saveas(path) -def make_multiframe_dxf(path: Path) -> None: +def to_mif(text: str) -> str: + """Encode CJK chars as AutoCAD MIF escapes (\\M+5XXXX, codepage 936).""" + out = [] + for ch in text: + if ord(ch) > 127: + out.append("\\M+5%04X" % int.from_bytes(ch.encode("gbk"), "big")) + else: + out.append(ch) + return "".join(out) + + +def make_multiframe_dxf(path: Path, mif_titles: bool = False) -> None: """Chinese sheet-wall pattern: two floor-plan frames stacked 98100 apart. Reproduces the 贝昂 failure mode: one DWG holds every floor; frames keep their sheet positions, so extraction without per-frame offsets scatters - floors diagonally. + floors diagonally. mif_titles=True writes the plan titles as MIF escapes + (\\M+5....), the 黄桥文体中心 pattern that plain text matching misses. """ doc = ezdxf.new("R2010", setup=True) for layer in ("WALL", "DOTE", "PUB_TEXT"): @@ -123,6 +135,8 @@ def make_multiframe_dxf(path: Path) -> None: for j in range(3): # horizontal grid lines y = j * 10000 msp.add_line((-3000, y + oy), (46000, y + oy), dxfattribs={"layer": "DOTE"}) + if mif_titles: + title = to_mif(title) msp.add_text(title, height=800, dxfattribs={"layer": "PUB_TEXT"}).set_placement((18000, oy - 6000)) draw_frame(0.0, "一层平面图") @@ -130,6 +144,43 @@ def make_multiframe_dxf(path: Path) -> None: doc.saveas(path) +def make_blockgrid_dxf(path: Path) -> None: + """黄桥 pattern: plan grids live inside anonymous wrapper blocks (A$C...), + NOT in modelspace, and floor wall bboxes differ (F2 covers half the + building). Clip-corner fallback then mis-aligns by 20 m in x; only + grid-line correlation via INSERT expansion recovers the true offset.""" + doc = ezdxf.new("R2010", setup=True) + for layer in ("WALL", "DOTE", "PUB_TEXT"): + doc.layers.add(layer) + msp = doc.modelspace() + pitch = 98100.0 + + def draw_frame(index: int, oy: float, title: str, full_walls: bool) -> None: + wall_x0 = 0 if full_walls else 20000 + outline = [((wall_x0, 0), (40000, 0)), ((40000, 0), (40000, 20000)), + ((40000, 20000), (wall_x0, 20000)), ((wall_x0, 20000), (wall_x0, 0))] + interior = [((8000 * i, 0), (8000 * i, 20000)) for i in range(3 if full_walls else 4, 5)] + interior += [((wall_x0, 10000), (40000, 10000)), ((30000, 10000), (30000, 20000)), + ((wall_x0 + 2000, 5000), (wall_x0 + 8000, 5000)), + ((wall_x0 + 5000, 0), (wall_x0 + 5000, 10000)), + ((34000, 0), (34000, 20000)), ((wall_x0 + 1000, 15000), (38000, 15000))] + for (sx, sy), (ex, ey) in outline + interior: + msp.add_line((sx, sy + oy), (ex, ey + oy), dxfattribs={"layer": "WALL"}) + block = doc.blocks.new(name=f"A$CGRID{index}") + # real structural grids have varied bays; a perfectly uniform grid makes + # subset correlation ambiguous (any multiple of the pitch ties) + for x in (0, 8000, 19000, 26000, 37000, 40000): + block.add_line((x, -3000), (x, 26000), dxfattribs={"layer": "DOTE"}) + for y in (0, 6000, 10000, 20000): + block.add_line((-3000, y), (46000, y), dxfattribs={"layer": "DOTE"}) + msp.add_blockref(f"A$CGRID{index}", (0, oy)) + msp.add_text(title, height=800, dxfattribs={"layer": "PUB_TEXT"}).set_placement((18000, oy - 6000)) + + draw_frame(1, 0.0, "一层平面图", True) + draw_frame(2, -pitch, "二层平面图", False) + doc.saveas(path) + + def make_rowframe_dxf(path: Path) -> None: """Row layout (3#辅房 pattern): frames side by side, pitch on X, titles same row.""" doc = ezdxf.new("R2010", setup=True) @@ -466,6 +517,224 @@ def main() -> int: f"default={gdefault_info} on={gon_info}", ) + # --- GBK/ANSI_936 DXF survives sanitize + report (libredwg dwg2dxf output) --- + gbk_dir = WORK / "dxf_gbk" + gbk_dir.mkdir(parents=True, exist_ok=True) + gbk_lines = [ + "999", "天正注释行 should be stripped", + "0", "SECTION", "2", "HEADER", + "9", "$ACADVER", "1", "AC1009", + "9", "$DWGCODEPAGE", "3", "ANSI_936", + "0", "ENDSEC", + "0", "SECTION", "2", "ENTITIES", + "0", "TEXT", "8", "PUB_TEXT", + "10", "0.0", "20", "0.0", "40", "500.0", + "1", "一层平面图", + "0", "ENDSEC", + "0", "EOF", "", + ] + (gbk_dir / "gbk_plan.dxf").write_bytes("\r\n".join(gbk_lines).encode("gbk")) + proc = run([ + sys.executable, "scripts/cad_dxf_report.py", + "--input", str(gbk_dir), "--work", str(WORK / "gbk_work"), + "--output", str(WORK / "gbk_report"), "--vendor-dir", str(VENDOR), + ]) + check("report survives GBK/ANSI_936 DXF", proc.returncode == 0, proc.stderr[-600:]) + gbk_report = WORK / "gbk_report" / "cad_read_report.json" + check("GBK report json parses", gbk_report.exists() and bool(json.loads(gbk_report.read_text(encoding="utf-8")))) + sanitized_gbk = WORK / "gbk_work" / "dxf_sanitized" / "gbk_plan.dxf" + if sanitized_gbk.exists(): + doc = ezdxf.readfile(sanitized_gbk) + texts = [e.dxf.text for e in doc.modelspace() if e.dxftype() == "TEXT"] + check("sanitize preserves GBK CJK text", texts == ["一层平面图"], str(texts)) + raw_bytes = sanitized_gbk.read_bytes() + check("sanitize still strips 999 comments", b"999" not in raw_bytes.split(b"ENTITIES")[0] or b"should be stripped" not in raw_bytes) + else: + check("sanitized GBK dxf exists", False, str(sanitized_gbk)) + + # --- MIF-escaped plan titles (\M+5....): 黄桥文体中心 pattern --- + mif_dir = WORK / "dxf_mif" + mif_dir.mkdir(parents=True, exist_ok=True) + make_multiframe_dxf(mif_dir / "mifframes.dxf", mif_titles=True) + proc = run([ + sys.executable, "scripts/detect_sheet_frames.py", + "--dxf", str(mif_dir / "mifframes.dxf"), "--out-config", str(WORK / "frames_mif.json"), + "--out-report", str(WORK / "frames_mif.md"), "--vendor-dir", str(VENDOR), + ]) + check("detect handles MIF titles", proc.returncode == 0, proc.stderr[-600:]) + if proc.returncode == 0: + frames_mif = json.loads((WORK / "frames_mif.json").read_text(encoding="utf-8")) + mif_keys = sorted(f["key"] for f in frames_mif["drawings"][0]["floors"]) + check("MIF titles decoded to two plan frames", mif_keys == ["F1", "F2"], str(mif_keys)) + proc = run([ + sys.executable, "scripts/cad_dxf_report.py", + "--input", str(mif_dir), "--work", str(WORK / "mif_work"), + "--output", str(WORK / "mif_report"), "--vendor-dir", str(VENDOR), + ]) + mif_report = json.loads((WORK / "mif_report" / "cad_read_report.json").read_text(encoding="utf-8")) + mif_item = {i["file"]: i for i in mif_report["files"]}.get("mifframes.dxf", {}) + check( + "multi_frame_sheet warning fires on MIF titles", + any("multi_frame_sheet" in w for w in mif_item.get("warnings", [])), + str(mif_item.get("warnings")), + ) + + # --- block-wrapped plan grids (黄桥 A$C... wrapper blocks) --- + bg_dir = WORK / "dxf_blockgrid" + bg_dir.mkdir(parents=True, exist_ok=True) + make_blockgrid_dxf(bg_dir / "blockgrid.dxf") + proc = run([ + sys.executable, "scripts/detect_sheet_frames.py", + "--dxf", str(bg_dir / "blockgrid.dxf"), "--out-config", str(WORK / "frames_bg.json"), + "--out-report", str(WORK / "frames_bg.md"), "--vendor-dir", str(VENDOR), + ]) + check("detect handles block-wrapped grids", proc.returncode == 0, proc.stderr[-600:]) + if proc.returncode == 0: + frames_bg = json.loads((WORK / "frames_bg.json").read_text(encoding="utf-8")) + bg_off = {f["key"]: f["offset"] for f in frames_bg["drawings"][0]["floors"]} + ok_bg = "F2" in bg_off and abs(bg_off["F2"][0]) <= 120 and abs(bg_off["F2"][1] - 98100.0) <= 120 + check("block-grid correlation beats clip-corner fallback", ok_bg, str(bg_off)) + + # --- isolated legend clusters (黄桥图签窗图例 -> footprint 细三角飘板) --- + iso_dir = WORK / "dxf_isolated" + iso_dir.mkdir(parents=True, exist_ok=True) + iso_doc = ezdxf.new("R2010", setup=True) + for layer in ("I-WALL", "A-GLAZ-CURT"): + iso_doc.layers.add(layer) + iso_msp = iso_doc.modelspace() + for start, end in [ + ((0, 0), (40000, 0)), ((40000, 0), (40000, 20000)), + ((40000, 20000), (0, 20000)), ((0, 20000), (0, 0)), + ]: + iso_msp.add_line(start, end, dxfattribs={"layer": "I-WALL"}) + iso_msp.add_line((0, 0), (20000, 0), dxfattribs={"layer": "A-GLAZ-CURT"}) # real facade band + for i in range(3): # window legend in the title block, 60 m east of the building + iso_msp.add_line((100000, 30000 + i * 400), (101000, 30000 + i * 400), dxfattribs={"layer": "A-GLAZ-CURT"}) + iso_doc.saveas(iso_dir / "plan.dxf") + iso_config = { + "scale": 0.001, + "drawings": [{"file": "plan.dxf", "floors": [{"key": "1F", "name": "首层", "z": 0.0, "height": 4.2}]}], + } + + def derive_iso(tag: str, extra: dict) -> tuple[dict, str]: + cfg = dict(iso_config, **extra) + cfg_path = WORK / f"iso_config_{tag}.json" + cfg_path.write_text(json.dumps(cfg, ensure_ascii=False), encoding="utf-8") + out_path = WORK / f"iso_spec_{tag}.json" + proc = run([ + sys.executable, "scripts/derive_showcase_spec.py", + "--dxf-dir", str(iso_dir), "--config", str(cfg_path), + "--out", str(out_path), "--vendor-dir", str(VENDOR), + ]) + check(f"derive isolated ({tag}) runs", proc.returncode == 0, proc.stderr[-400:]) + return json.loads(out_path.read_text(encoding="utf-8")), proc.stderr + + iso_spec, iso_err = derive_iso("default", {}) + glz = iso_spec["floors"]["1F"]["segments"].get("glazing", []) + far = [s for s in glz if max(s["start"][0], s["end"][0]) > 60] + check("isolated legend cluster dropped by default", len(glz) > 0 and not far, f"glazing={len(glz)} far={len(far)}") + check("isolated-cluster drop warns loudly", "isolated" in iso_err.lower(), iso_err[-200:]) + check( + "bounds shrink to the real building", + iso_spec["floors"]["1F"]["bounds"][2] <= 45.0, + str(iso_spec["floors"]["1F"]["bounds"]), + ) + iso_spec_off, _ = derive_iso("off", {"drop_isolated_clusters": False}) + glz_off = iso_spec_off["floors"]["1F"]["segments"].get("glazing", []) + check( + "legend kept when drop_isolated_clusters=false", + any(max(s["start"][0], s["end"][0]) > 60 for s in glz_off), + f"glazing={len(glz_off)}", + ) + + # --- unsupported roof outlines (黄桥坡屋面: canopy with hatch-only posts) --- + uns_dir = WORK / "dxf_unsupported" + uns_dir.mkdir(parents=True, exist_ok=True) + uns_doc = ezdxf.new("R2010", setup=True) + for layer in ("I-WALL", "A-ROOF"): + uns_doc.layers.add(layer) + uns_msp = uns_doc.modelspace() + for start, end in [ + ((0, 0), (40000, 0)), ((40000, 0), (40000, 20000)), + ((40000, 20000), (0, 20000)), ((0, 20000), (0, 0)), + ]: + uns_msp.add_line(start, end, dxfattribs={"layer": "I-WALL"}) + uns_msp.add_line((5000, 10000), (35000, 10000), dxfattribs={"layer": "A-ROOF"}) # over the building + uns_msp.add_line((80000, 50000), (95000, 50000), dxfattribs={"layer": "A-ROOF"}) # over nothing + uns_doc.saveas(uns_dir / "plan.dxf") + uns_config = { + "scale": 0.001, + "generate_footprints": True, + "drop_isolated_clusters": False, # keep the far roof line so THIS rule is what's tested + "drawings": [{"file": "plan.dxf", "floors": [{"key": "1F", "name": "首层", "z": 0.0, "height": 4.2}]}], + } + + def derive_uns(tag: str, extra: dict) -> tuple[dict, str]: + cfg = dict(uns_config, **extra) + cfg_path = WORK / f"uns_config_{tag}.json" + cfg_path.write_text(json.dumps(cfg, ensure_ascii=False), encoding="utf-8") + out_path = WORK / f"uns_spec_{tag}.json" + proc = run([ + sys.executable, "scripts/derive_showcase_spec.py", + "--dxf-dir", str(uns_dir), "--config", str(cfg_path), + "--out", str(out_path), "--vendor-dir", str(VENDOR), + ]) + check(f"derive unsupported-roof ({tag}) runs", proc.returncode == 0, proc.stderr[-400:]) + return json.loads(out_path.read_text(encoding="utf-8")), proc.stderr + + uns_spec, uns_err = derive_uns("default", {}) + roof_segs = uns_spec["floors"]["1F"]["segments"].get("roof", []) + check( + "unsupported roof line dropped, supported kept", + len(roof_segs) == 1 and max(roof_segs[0]["start"][0], roof_segs[0]["end"][0]) <= 45, + f"roof={[(s['start'], s['end']) for s in roof_segs]}", + ) + check("unsupported-roof drop warns loudly", "unsupported_roof" in uns_err, uns_err[-200:]) + uns_spec_off, _ = derive_uns("off", {"drop_unsupported_roof_lines": False}) + check( + "roof lines kept when drop_unsupported_roof_lines=false", + len(uns_spec_off["floors"]["1F"]["segments"].get("roof", [])) == 2, + ) + + # --- ODA auto-fallback: broken libredwg conversion reconverts via ODA --- + # A garbage .dwg makes dwg2dxf fail (or be absent); with ODA_CONVERTER + # pointing at a fake directory-converter the report must recover the file + # instead of shipping a failed/empty record (黄桥: libredwg emits broken + # DXF for large TArch files while ODA reads back 21k entities). + oda_dir = WORK / "dxf_oda" + oda_dir.mkdir(parents=True, exist_ok=True) + (oda_dir / "broken.dwg").write_bytes(b"NOT-A-REAL-DWG" * 64) + good_template = WORK / "oda_template.dxf" + shutil.copy2(dxf_dir / "plan.dxf", good_template) + fake_conv = WORK / "fake_oda.sh" + fake_conv.write_text( + "#!/bin/sh\n" + "for f in \"$1\"/*.dwg; do\n" + " name=$(basename \"$f\" .dwg)\n" + f" cp \"{good_template}\" \"$2/$name.dxf\"\n" + "done\n", + encoding="utf-8", + ) + fake_conv.chmod(0o755) + import os as _os + env = dict(_os.environ, ODA_CONVERTER=str(fake_conv)) + proc = subprocess.run( + [sys.executable, "scripts/cad_dxf_report.py", + "--input", str(oda_dir), "--work", str(WORK / "oda_work"), + "--output", str(WORK / "oda_report"), "--vendor-dir", str(VENDOR)], + cwd=ROOT, text=True, capture_output=True, env=env, + ) + check("report with ODA fallback runs", proc.returncode == 0, proc.stderr[-600:]) + oda_json = json.loads((WORK / "oda_report" / "cad_read_report.json").read_text(encoding="utf-8")) + oda_convs = [c for c in oda_json["conversions"] if str(c.get("status", "")).startswith("oda_fallback")] + check("oda_fallback conversion recorded", any(c["status"] == "oda_fallback_ok" for c in oda_convs), str(oda_convs)) + broken_item = {i["file"]: i for i in oda_json["files"]}.get("broken.dxf", {}) + check( + "broken DWG recovered via ODA fallback", + broken_item.get("status") != "failed" and broken_item.get("modelspace_entities", 0) > 0, + str({k: broken_item.get(k) for k in ("status", "modelspace_entities")}), + ) + print() if FAILURES: print(f"{len(FAILURES)} check(s) failed: {FAILURES}")