Generalization hardening from test3d-7 round 1 (黄桥文体中心): 8 gap fixes

Conversion/encoding chain (libredwg on Chinese TArch DWGs):
- run_dwg2dxf: capture bytes, decode leniently (GBK console output crashed
  the batch under subprocess text=True)
- sanitize_dxf_comments: byte-level, code/value pair walk — the UTF-8
  errors="ignore" round-trip mangled every GBK CJK string ("一层平面图"
  -> "一平图") and eating XRECORD "999" VALUE lines corrupted files
- read_doc: recover fallback also on ValueError; scrub_surrogates keeps one
  corrupt drawing from killing the whole report
- decode_mif in report + frame detector: \M+5XXXX escapes (whole 黄桥 set)
  restored so plan-title detection works at all
- ODA auto-fallback: reconvert DWGs whose libredwg DXF fails inspection or
  reads an empty modelspace (黄桥 plans: 0 entities via dwg2dxf, 21k via
  ODA); env ODA_CONVERTER override, fake-converter regression test

Frame detection:
- detect_sheet_frames expands INSERTs (memoized, depth<=3) when collecting
  axis/wall lines — 黄桥 wraps each frame's grid in anonymous A$C... blocks
  (msp had 8 DOTE lines, blocks held 700); all 7 frames went from WEAK
  clip-corner fallback to grid-aligned votes x=14

Extraction judgment rules (both loud, both default-on, both switchable):
- drop_isolated_clusters: tiny segment clusters far from the main mass are
  sheet-legend content drawn on real layers (title-block window symbols made
  every floor's footprint grow a sliver fin)
- drop_unsupported_roof_lines: roof outlines with no footprint/cap plate
  within 2m beneath are hatch-only-support canopies that extrude into
  floating ribbons (F3 SW 坡屋面, B1 ground canopy at -5.85m)

Tests: 64 -> 75 checks, incl. GBK/MIF synthetic DXFs, block-wrapped grid
alignment (non-uniform bays — uniform grids make subset correlation
ambiguous), legend-cluster and unsupported-roof fixtures.
references/generalization_test_log.md tracks the test protocol, per-project
gate table, and every gap with its regression coverage.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
2026-07-02 21:23:20 +08:00
parent 6ace84ffba
commit 221626b1c7
5 changed files with 768 additions and 15 deletions

View File

@@ -5,6 +5,7 @@ from __future__ import annotations
import argparse
import json
import os
import shutil
import subprocess
import sys
@@ -68,32 +69,94 @@ def run_dwg2dxf(source: Path, dest: Path) -> dict:
}
dest.parent.mkdir(parents=True, exist_ok=True)
cmd = [exe, "-y", "-o", str(dest), str(source)]
proc = subprocess.run(cmd, text=True, capture_output=True)
# libredwg echoes CJK layer/file names in the platform encoding (GBK on
# Chinese drawings), so strict UTF-8 text mode crashes; decode leniently.
proc = subprocess.run(cmd, capture_output=True)
stdout = proc.stdout.decode("utf-8", errors="replace")
stderr = proc.stderr.decode("utf-8", errors="replace")
return {
"source": str(source),
"dest": str(dest),
"status": "ok" if proc.returncode == 0 and dest.exists() else "failed",
"returncode": proc.returncode,
"stdout": proc.stdout[-2000:],
"stderr": proc.stderr[-2000:],
"stdout": stdout[-2000:],
"stderr": stderr[-2000:],
}
def oda_converter() -> str | None:
"""ODA File Converter path (env ODA_CONVERTER overrides the macOS default)."""
override = os.environ.get("ODA_CONVERTER")
if override:
return override if Path(override).exists() else None
default = Path("/Applications/ODAFileConverter.app/Contents/MacOS/ODAFileConverter")
return str(default) if default.exists() else None
def run_oda_batch(sources: list[Path], work_dir: Path) -> dict[Path, Path]:
"""Convert DWGs via ODA File Converter; returns {source: produced_dxf}.
ODA converts whole directories with absolute paths and reports bad
arguments via GUI dialogs (silent exit-0), so sources are staged under
ASCII names in a scratch in-dir and results matched back by stem.
"""
exe = oda_converter()
if not exe or not sources:
return {}
in_dir = work_dir / "oda_in"
out_dir = work_dir / "oda_out"
shutil.rmtree(in_dir, ignore_errors=True)
shutil.rmtree(out_dir, ignore_errors=True)
in_dir.mkdir(parents=True)
out_dir.mkdir(parents=True)
mapping: dict[str, Path] = {}
for index, src in enumerate(sources):
stem = f"oda{index:03d}"
shutil.copy2(src, in_dir / f"{stem}.dwg")
mapping[stem] = src
try:
subprocess.run(
[exe, str(in_dir.resolve()), str(out_dir.resolve()), "ACAD2018", "DXF", "0", "1", "*.dwg"],
capture_output=True, timeout=1800,
)
except (subprocess.TimeoutExpired, OSError):
return {}
produced: dict[Path, Path] = {}
for stem, src in mapping.items():
out = out_dir / f"{stem}.dxf"
if out.exists() and out.stat().st_size > 0:
produced[src] = out
return produced
def sanitize_dxf_comments(source: Path, dest: Path) -> int:
"""Remove DXF group code 999 comment pairs that can break brittle importers."""
lines = source.read_text(errors="ignore").splitlines(keepends=True)
"""Remove DXF group code 999 comment pairs that can break brittle importers.
Operates on BYTES: pre-2007 DXF from libredwg/TArch stores CJK text in the
$DWGCODEPAGE encoding (ANSI_936/GBK). A UTF-8 round-trip with
errors="ignore" silently drops those bytes, mangling every CJK layer/text
("一层平面图" -> "一平图") and breaking plan-title detection downstream, so
the file content must pass through unchanged apart from the comment pairs.
"""
lines = source.read_bytes().splitlines(keepends=True)
dest.parent.mkdir(parents=True, exist_ok=True)
cleaned: list[str] = []
cleaned: list[bytes] = []
removed = 0
index = 0
while index < len(lines):
if lines[index].strip() == "999":
# DXF is a strict alternation of group-code line / value line, so walk in
# pairs: a bare "999" VALUE (e.g. an integer inside an ODA XRECORD) must
# not be mistaken for a comment group code, or the pair removal shifts
# every following tag and corrupts the file.
while index + 1 < len(lines):
if lines[index].strip() == b"999":
removed += 1
index += 2
continue
cleaned.append(lines[index])
index += 1
dest.write_text("".join(cleaned), encoding="utf-8", errors="ignore")
cleaned.append(lines[index + 1])
index += 2
cleaned.extend(lines[index:])
dest.write_bytes(b"".join(cleaned))
return removed
@@ -130,7 +193,10 @@ def guess_unit_scale(bbox_data: dict | None, insunits: int) -> dict:
def read_doc(path: Path, ezdxf, recover):
try:
return ezdxf.readfile(path), []
except ezdxf.DXFError:
except (ezdxf.DXFError, ValueError):
# Strict reads can also raise plain ValueError on quirky-but-
# recoverable files (e.g. an XRECORD holding "{ACAD_REACTORS" where a
# handle is expected); recover skips those records.
doc, auditor = recover.readfile(path)
return doc, [str(error) for error in auditor.errors]
@@ -167,6 +233,27 @@ import re as _re
PLAN_TITLE_RE = _re.compile(r"平面图$")
PLAN_TITLE_EXCLUDE_RE = _re.compile(r"(示意|疏散|分区|索引|定位|总平面|放大|详图|布置|大样)")
# AutoCAD MIF escapes (\M+5XXXX): CJK text in pre-Unicode DWGs survives
# DXF conversion as literal escape sequences, so title matching must decode
# them first. The digit selects the codepage; XXXX is the hex byte pair.
MIF_RE = _re.compile(r"\\M\+([1-5])([0-9A-Fa-f]{4})")
MIF_CODEPAGES = {"1": "cp932", "2": "big5", "3": "cp949", "4": "johab", "5": "gbk"}
def decode_mif(text: str) -> str:
if "\\M+" not in text:
return text
def repl(match: "_re.Match[str]") -> str:
value = int(match.group(2), 16)
raw = value.to_bytes(2, "big") if value > 0xFF else value.to_bytes(1, "big")
try:
return raw.decode(MIF_CODEPAGES.get(match.group(1), "gbk"))
except (UnicodeDecodeError, LookupError):
return match.group(0)
return MIF_RE.sub(repl, text)
def count_plan_titles(entities) -> int:
"""Distinct plan-frame titles in modelspace (multi-frame sheet detector)."""
@@ -179,6 +266,7 @@ def count_plan_titles(entities) -> int:
text = str(entity.dxf.text if kind == "TEXT" else entity.plain_text() or "").strip()
except Exception:
continue
text = decode_mif(text)
if not text or len(text) > 40:
continue
if PLAN_TITLE_RE.search(text) and not PLAN_TITLE_EXCLUDE_RE.search(text):
@@ -335,6 +423,28 @@ def output_paths(output: Path) -> tuple[Path, Path]:
return output / "cad_read_report.json", output / "cad_read_report.md"
def scrub_surrogates(value):
"""Make report data JSON-writable when ezdxf's recover mode leaks
surrogate-escaped bytes (broken files read with errors="surrogateescape").
CJK bytes are recovered via GBK where possible; the batch never dies on
one corrupt drawing."""
if isinstance(value, str):
try:
value.encode("utf-8")
return value
except UnicodeEncodeError:
raw = value.encode("utf-8", errors="surrogateescape")
try:
return raw.decode("gbk")
except UnicodeDecodeError:
return raw.decode("utf-8", errors="replace")
if isinstance(value, dict):
return {scrub_surrogates(k): scrub_surrogates(v) for k, v in value.items()}
if isinstance(value, list):
return [scrub_surrogates(item) for item in value]
return value
def main() -> int:
args = parse_args()
input_dir = Path(args.input)
@@ -376,6 +486,48 @@ def main() -> int:
conversions.append({"source": str(raw), "dest": str(dest), "status": f"sanitized_{removed}_comments"})
results = [inspect_file(path, ezdxf, bbox, recover) for path in sorted(sanitized)]
# ODA auto-fallback: libredwg chokes on some TArch DWGs (duplicate handles,
# missing ENDBLK/EOF) or yields an empty modelspace even when the drawing
# is full — the 黄桥 plans sheet read 0 entities via dwg2dxf but 21k via
# ODA. Reconvert those sources through ODA when it is available.
dwg_dest_to_source = {
Path(c["dest"]).name: Path(c["source"])
for c in conversions
if str(c.get("source", "")).lower().endswith(".dwg") and c.get("dest")
}
retry: dict[str, Path] = {}
for item in results:
src = dwg_dest_to_source.get(item.get("file", ""))
if src and (item.get("status") == "failed" or item.get("modelspace_entities") == 0):
retry[item["file"]] = src
for c in list(conversions):
if c.get("status") in {"failed", "missing_dwg2dxf"} and str(c.get("source", "")).lower().endswith(".dwg"):
name = Path(c["dest"]).name if c.get("dest") else Path(c["source"]).with_suffix(".dxf").name
retry.setdefault(name, Path(c["source"]))
if retry:
produced = run_oda_batch(list(retry.values()), work_dir)
for name, src in retry.items():
out = produced.get(src)
if not out:
conversions.append({"source": str(src), "dest": "", "status": "oda_fallback_failed"})
continue
raw_dest = raw_dir / name
shutil.copy2(out, raw_dest)
clean_dest = clean_dir / name
sanitize_dxf_comments(raw_dest, clean_dest)
item = inspect_file(clean_dest, ezdxf, bbox, recover)
replaced = False
for i, existing in enumerate(results):
if existing.get("file") == name:
results[i] = item
replaced = True
break
if not replaced:
results.append(item)
conversions.append({"source": str(src), "dest": str(raw_dest), "status": "oda_fallback_ok"})
results.sort(key=lambda r: r.get("file", ""))
json_path, md_path = output_paths(Path(args.output))
json_path.parent.mkdir(parents=True, exist_ok=True)
md_path.parent.mkdir(parents=True, exist_ok=True)
@@ -386,6 +538,8 @@ def main() -> int:
"conversions": conversions,
"files": results,
}
payload = scrub_surrogates(payload)
results = payload["files"]
json_path.write_text(json.dumps(payload, ensure_ascii=False, indent=2), encoding="utf-8")
md_path.write_text(markdown_report(results, conversions, input_dir), encoding="utf-8")
print(json.dumps({"json": str(json_path), "markdown": str(md_path), "files": len(results)}, ensure_ascii=False))

View File

@@ -771,6 +771,163 @@ def compute_caps(floors: dict[str, Any], config: dict[str, Any]) -> None:
fl.setdefault("caps", []).append({"z": round(ftop, 6), "polygons": polys})
def drop_isolated_clusters(segments: dict[str, list[dict[str, Any]]], config: dict[str, Any], floor_key: str) -> dict[str, list[dict[str, Any]]]:
"""Drop tiny segment clusters far away from the floor's main mass.
Sheet legends and title blocks draw window/door symbols with REAL geometry
layers (黄桥: four 1 m CURTWALL strips 30 m east of the building on every
frame), so layer exclusion cannot catch them — and the generated footprint
hull then stretches into sliver fins ("每层一个尖角"). Cluster all
classified segments by endpoint proximity, keep the largest cluster plus
anything near it or substantial, and WARN about what was dropped.
Config: {"drop_isolated_clusters": false} disables; tuning knobs
cluster_linkage (m, 3.0), isolation_distance (m, 12.0),
isolation_min_length (m, 15.0).
"""
if not config.get("drop_isolated_clusters", True):
return segments
linkage = float(config.get("cluster_linkage", 3.0))
iso_dist = float(config.get("isolation_distance", 12.0))
min_len = float(config.get("isolation_min_length", 15.0))
items: list[tuple[str, int, dict[str, Any]]] = []
for category, segs in segments.items():
for i, seg in enumerate(segs):
items.append((category, i, seg))
if len(items) < 2:
return segments
parent = list(range(len(items)))
def find(a: int) -> int:
while parent[a] != a:
parent[a] = parent[parent[a]]
a = parent[a]
return a
def union(a: int, b: int) -> None:
ra, rb = find(a), find(b)
if ra != rb:
parent[rb] = ra
cell = max(linkage, 1e-6)
buckets: dict[tuple[int, int], list[int]] = {}
for idx, (_c, _i, seg) in enumerate(items):
for px, py in (seg["start"], seg["end"]):
buckets.setdefault((int(px // cell), int(py // cell)), []).append(idx)
for (bx, by), members in buckets.items():
for gx in (bx - 1, bx, bx + 1):
for gy in (by - 1, by, by + 1):
for other in buckets.get((gx, gy), []):
union(members[0], other)
for other in members[1:]:
union(members[0], other)
clusters: dict[int, list[int]] = {}
for idx in range(len(items)):
clusters.setdefault(find(idx), []).append(idx)
def cluster_stats(indices: list[int]) -> tuple[float, list[float]]:
total = 0.0
bx = [math.inf, math.inf, -math.inf, -math.inf]
for idx in indices:
seg = items[idx][2]
(x1, y1), (x2, y2) = seg["start"], seg["end"]
total += math.hypot(x2 - x1, y2 - y1)
bx[0] = min(bx[0], x1, x2); bx[1] = min(bx[1], y1, y2)
bx[2] = max(bx[2], x1, x2); bx[3] = max(bx[3], y1, y2)
return total, bx
stats = {root: cluster_stats(idx_list) for root, idx_list in clusters.items()}
main_root = max(stats, key=lambda r: stats[r][0])
main_box = stats[main_root][1]
def box_gap(a: list[float], b: list[float]) -> float:
dx = max(0.0, max(a[0] - b[2], b[0] - a[2]))
dy = max(0.0, max(a[1] - b[3], b[1] - a[3]))
return math.hypot(dx, dy)
dropped: set[int] = set()
for root, idx_list in clusters.items():
if root == main_root:
continue
total, box = stats[root]
gap = box_gap(main_box, box)
if gap > iso_dist and total < min_len:
dropped.update(idx_list)
print(
f"WARNING isolated_cluster {floor_key}: dropped {len(idx_list)} segments "
f"({total:.1f}m total, {gap:.0f}m from main mass) — likely sheet legend/"
"title-block content; set drop_isolated_clusters=false to keep",
file=sys.stderr,
)
if not dropped:
return segments
dropped_keys = {(items[d][0], items[d][1]) for d in dropped}
return {
category: [seg for i, seg in enumerate(segs) if (category, i) not in dropped_keys]
for category, segs in segments.items()
}
def drop_unsupported_roof_lines(floors: dict[str, Any], config: dict[str, Any]) -> None:
"""Drop roof outlines that would extrude into ribbons floating in mid-air.
Plans draw low canopies / porch roofs whose supports are hatch-only
columns the extractor cannot capture (黄桥: 坡屋面 outlines on the F3
sheet with zero walls beneath on any floor). A roof segment is kept only
if some plate exists under it — any floor's footprint or cap polygon
within roof_support_distance (m, default 2.0). Dropped segments are
reported loudly; keep them via {"drop_unsupported_roof_lines": false} or
model the canopy explicitly with a roof_specs plate.
"""
if not config.get("drop_unsupported_roof_lines", True):
return
try:
from shapely.geometry import Point, Polygon
from shapely.ops import unary_union
except ImportError:
return
plates = []
for fl in floors.values():
polys = list(fl.get("footprint", {}).get("polygons", []))
for cap in fl.get("caps", []):
polys.extend(cap.get("polygons", []))
for coords in polys:
try:
poly = Polygon([(float(x), float(y)) for x, y in coords])
if not poly.is_valid:
poly = poly.buffer(0)
if not poly.is_empty:
plates.append(poly)
except Exception:
continue
if not plates:
return # no footprint evidence at all — nothing to judge against
support = unary_union(plates).buffer(float(config.get("roof_support_distance", 2.0)))
for key, fl in floors.items():
roof = fl.get("segments", {}).get("roof")
if not roof:
continue
kept, dropped_len, dropped = [], 0.0, 0
for seg in roof:
mid = Point((seg["start"][0] + seg["end"][0]) / 2.0, (seg["start"][1] + seg["end"][1]) / 2.0)
if support.contains(mid):
kept.append(seg)
else:
dropped += 1
dropped_len += math.hypot(seg["end"][0] - seg["start"][0], seg["end"][1] - seg["start"][1])
if dropped:
print(
f"WARNING unsupported_roof {key}: dropped {dropped} roof segments "
f"({dropped_len:.1f}m) with no slab/cap beneath — likely a canopy whose "
"supports are hatch-only; set drop_unsupported_roof_lines=false or add a "
"roof_specs plate to keep them",
file=sys.stderr,
)
fl["segments"]["roof"] = kept
def finalize_spec(
floor_data: dict[str, dict[str, Any]],
config: dict[str, Any],
@@ -804,6 +961,9 @@ def finalize_spec(
"source_type": raw["source_type"],
}
)
segments = drop_isolated_clusters(segments, config, key)
xs = [p for segs in segments.values() for seg in segs for p in (seg["start"][0], seg["end"][0])]
ys = [p for segs in segments.values() for seg in segs for p in (seg["start"][1], seg["end"][1])]
bounds = [
round(min(xs), 6) if xs else 0.0,
round(min(ys), 6) if ys else 0.0,
@@ -834,6 +994,7 @@ def finalize_spec(
gap_tol=float(config.get("glazing_gap_tol", 3.6)),
)
compute_caps(floors, config)
drop_unsupported_roof_lines(floors, config)
# Topless-roof guard: the top above-ground floor of each group must end in
# a cap or a config roof_specs entry, otherwise the model renders open-top

View File

@@ -40,6 +40,26 @@ from pathlib import Path
PLAN_TITLE = re.compile(r"平面图$")
PLAN_EXCLUDE = re.compile(r"(示意|疏散|分区|索引|定位|总平面|放大|详图|布置|大样)")
OTHER_TITLE = re.compile(r"(立面图|剖面图)$")
# AutoCAD MIF escapes (\M+5XXXX): CJK titles in pre-Unicode DWGs arrive as
# literal escape sequences, invisible to the 平面图 regexes until decoded.
MIF_RE = re.compile(r"\\M\+([1-5])([0-9A-Fa-f]{4})")
MIF_CODEPAGES = {"1": "cp932", "2": "big5", "3": "cp949", "4": "johab", "5": "gbk"}
def decode_mif(text: str) -> str:
if "\\M+" not in text:
return text
def repl(match: "re.Match[str]") -> str:
value = int(match.group(2), 16)
raw = value.to_bytes(2, "big") if value > 0xFF else value.to_bytes(1, "big")
try:
return raw.decode(MIF_CODEPAGES.get(match.group(1), "gbk"))
except (UnicodeDecodeError, LookupError):
return match.group(0)
return MIF_RE.sub(repl, text)
CN_NUM = {"一": 1, "二": 2, "三": 3, "四": 4, "五": 5, "六": 6, "七": 7, "八": 8, "九": 9, "十": 10}
FLOOR_NAME_KEY = [
(re.compile(r"地下(?P<n>[一二三四五六七八九十\d]+)层夹层"), lambda m: f"B{_cn(m.group('n'))}M"),
@@ -100,9 +120,9 @@ def match_any(value: str, patterns: list[str]) -> bool:
def entity_text(entity) -> str:
try:
if entity.dxftype() == "TEXT":
return str(entity.dxf.text or "").strip()
return decode_mif(str(entity.dxf.text or "").strip())
if entity.dxftype() == "MTEXT":
return str(entity.plain_text() or "").strip()
return decode_mif(str(entity.plain_text() or "").strip())
except Exception:
pass
return ""
@@ -184,8 +204,65 @@ def main() -> int:
plan_exclude = re.compile(args.title_exclude_regex) if args.title_exclude_regex else None
titles, others = [], []
lines = [] # (layer, sx, sy, ex, ey) for LINE and polyline edges
# Plan grids are not always modelspace lines: TArch exports may wrap each
# frame's axis system in an anonymous block (黄桥: A$C... wrappers holding
# hundreds of DOTE lines while msp has ~8). Expand INSERTs — but only
# blocks that actually contain axis/wall-layer curves (memoized per block
# name), or a sheet full of door/furniture blocks slows the scan for nothing.
expand_patterns = axis_layers + wall_layers
block_memo: dict[str, bool] = {}
def block_has_expandable(name: str, depth: int = 0) -> bool:
if name in block_memo:
return block_memo[name]
if depth > 3 or name not in doc.blocks:
return False
block_memo[name] = False # cycle guard
found = False
for be in doc.blocks.get(name):
bkind = be.dxftype()
if bkind in {"LINE", "LWPOLYLINE"} and match_any(str(be.dxf.layer), expand_patterns):
found = True
break
if bkind == "INSERT" and block_has_expandable(str(be.dxf.name), depth + 1):
found = True
break
block_memo[name] = found
return found
def collect_insert_lines(insert, depth: int = 0) -> None:
if depth > 3:
return
try:
virtual = list(insert.virtual_entities())
except Exception:
return
for ve in virtual:
vkind = ve.dxftype()
if vkind == "LINE":
d = ve.dxf
if match_any(str(d.layer), expand_patterns):
lines.append((str(d.layer), float(d.start.x), float(d.start.y), float(d.end.x), float(d.end.y)))
elif vkind == "LWPOLYLINE":
layer = str(ve.dxf.layer)
if not match_any(layer, expand_patterns):
continue
try:
pts = [(float(px), float(py)) for px, py in ve.get_points("xy")]
except Exception:
continue
for a, b in zip(pts, pts[1:]):
lines.append((layer, a[0], a[1], b[0], b[1]))
elif vkind == "INSERT":
collect_insert_lines(ve, depth + 1)
for entity in msp:
kind = entity.dxftype()
if kind == "INSERT":
if block_has_expandable(str(entity.dxf.name)):
collect_insert_lines(entity)
continue
if kind in {"TEXT", "MTEXT"}:
content = entity_text(entity)
if not content or len(content) > 40: