Generalization hardening from test3d-7 round 1 (黄桥文体中心): 8 gap fixes

Conversion/encoding chain (libredwg on Chinese TArch DWGs):
- run_dwg2dxf: capture bytes, decode leniently (GBK console output crashed
  the batch under subprocess text=True)
- sanitize_dxf_comments: byte-level, code/value pair walk — the UTF-8
  errors="ignore" round-trip mangled every GBK CJK string ("一层平面图"
  -> "一平图") and eating XRECORD "999" VALUE lines corrupted files
- read_doc: recover fallback also on ValueError; scrub_surrogates keeps one
  corrupt drawing from killing the whole report
- decode_mif in report + frame detector: \M+5XXXX escapes (whole 黄桥 set)
  restored so plan-title detection works at all
- ODA auto-fallback: reconvert DWGs whose libredwg DXF fails inspection or
  reads an empty modelspace (黄桥 plans: 0 entities via dwg2dxf, 21k via
  ODA); env ODA_CONVERTER override, fake-converter regression test

Frame detection:
- detect_sheet_frames expands INSERTs (memoized, depth<=3) when collecting
  axis/wall lines — 黄桥 wraps each frame's grid in anonymous A$C... blocks
  (msp had 8 DOTE lines, blocks held 700); all 7 frames went from WEAK
  clip-corner fallback to grid-aligned votes x=14

Extraction judgment rules (both loud, both default-on, both switchable):
- drop_isolated_clusters: tiny segment clusters far from the main mass are
  sheet-legend content drawn on real layers (title-block window symbols made
  every floor's footprint grow a sliver fin)
- drop_unsupported_roof_lines: roof outlines with no footprint/cap plate
  within 2m beneath are hatch-only-support canopies that extrude into
  floating ribbons (F3 SW 坡屋面, B1 ground canopy at -5.85m)

Tests: 64 -> 75 checks, incl. GBK/MIF synthetic DXFs, block-wrapped grid
alignment (non-uniform bays — uniform grids make subset correlation
ambiguous), legend-cluster and unsupported-roof fixtures.
references/generalization_test_log.md tracks the test protocol, per-project
gate table, and every gap with its regression coverage.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
2026-07-02 21:23:20 +08:00
parent 6ace84ffba
commit 221626b1c7
5 changed files with 768 additions and 15 deletions

View File

@@ -5,6 +5,7 @@ from __future__ import annotations
import argparse
import json
import os
import shutil
import subprocess
import sys
@@ -68,32 +69,94 @@ def run_dwg2dxf(source: Path, dest: Path) -> dict:
}
dest.parent.mkdir(parents=True, exist_ok=True)
cmd = [exe, "-y", "-o", str(dest), str(source)]
proc = subprocess.run(cmd, text=True, capture_output=True)
# libredwg echoes CJK layer/file names in the platform encoding (GBK on
# Chinese drawings), so strict UTF-8 text mode crashes; decode leniently.
proc = subprocess.run(cmd, capture_output=True)
stdout = proc.stdout.decode("utf-8", errors="replace")
stderr = proc.stderr.decode("utf-8", errors="replace")
return {
"source": str(source),
"dest": str(dest),
"status": "ok" if proc.returncode == 0 and dest.exists() else "failed",
"returncode": proc.returncode,
"stdout": proc.stdout[-2000:],
"stderr": proc.stderr[-2000:],
"stdout": stdout[-2000:],
"stderr": stderr[-2000:],
}
def oda_converter() -> str | None:
"""ODA File Converter path (env ODA_CONVERTER overrides the macOS default)."""
override = os.environ.get("ODA_CONVERTER")
if override:
return override if Path(override).exists() else None
default = Path("/Applications/ODAFileConverter.app/Contents/MacOS/ODAFileConverter")
return str(default) if default.exists() else None
def run_oda_batch(sources: list[Path], work_dir: Path) -> dict[Path, Path]:
"""Convert DWGs via ODA File Converter; returns {source: produced_dxf}.
ODA converts whole directories with absolute paths and reports bad
arguments via GUI dialogs (silent exit-0), so sources are staged under
ASCII names in a scratch in-dir and results matched back by stem.
"""
exe = oda_converter()
if not exe or not sources:
return {}
in_dir = work_dir / "oda_in"
out_dir = work_dir / "oda_out"
shutil.rmtree(in_dir, ignore_errors=True)
shutil.rmtree(out_dir, ignore_errors=True)
in_dir.mkdir(parents=True)
out_dir.mkdir(parents=True)
mapping: dict[str, Path] = {}
for index, src in enumerate(sources):
stem = f"oda{index:03d}"
shutil.copy2(src, in_dir / f"{stem}.dwg")
mapping[stem] = src
try:
subprocess.run(
[exe, str(in_dir.resolve()), str(out_dir.resolve()), "ACAD2018", "DXF", "0", "1", "*.dwg"],
capture_output=True, timeout=1800,
)
except (subprocess.TimeoutExpired, OSError):
return {}
produced: dict[Path, Path] = {}
for stem, src in mapping.items():
out = out_dir / f"{stem}.dxf"
if out.exists() and out.stat().st_size > 0:
produced[src] = out
return produced
def sanitize_dxf_comments(source: Path, dest: Path) -> int:
"""Remove DXF group code 999 comment pairs that can break brittle importers."""
lines = source.read_text(errors="ignore").splitlines(keepends=True)
"""Remove DXF group code 999 comment pairs that can break brittle importers.
Operates on BYTES: pre-2007 DXF from libredwg/TArch stores CJK text in the
$DWGCODEPAGE encoding (ANSI_936/GBK). A UTF-8 round-trip with
errors="ignore" silently drops those bytes, mangling every CJK layer/text
("一层平面图" -> "一平图") and breaking plan-title detection downstream, so
the file content must pass through unchanged apart from the comment pairs.
"""
lines = source.read_bytes().splitlines(keepends=True)
dest.parent.mkdir(parents=True, exist_ok=True)
cleaned: list[str] = []
cleaned: list[bytes] = []
removed = 0
index = 0
while index < len(lines):
if lines[index].strip() == "999":
# DXF is a strict alternation of group-code line / value line, so walk in
# pairs: a bare "999" VALUE (e.g. an integer inside an ODA XRECORD) must
# not be mistaken for a comment group code, or the pair removal shifts
# every following tag and corrupts the file.
while index + 1 < len(lines):
if lines[index].strip() == b"999":
removed += 1
index += 2
continue
cleaned.append(lines[index])
index += 1
dest.write_text("".join(cleaned), encoding="utf-8", errors="ignore")
cleaned.append(lines[index + 1])
index += 2
cleaned.extend(lines[index:])
dest.write_bytes(b"".join(cleaned))
return removed
@@ -130,7 +193,10 @@ def guess_unit_scale(bbox_data: dict | None, insunits: int) -> dict:
def read_doc(path: Path, ezdxf, recover):
try:
return ezdxf.readfile(path), []
except ezdxf.DXFError:
except (ezdxf.DXFError, ValueError):
# Strict reads can also raise plain ValueError on quirky-but-
# recoverable files (e.g. an XRECORD holding "{ACAD_REACTORS" where a
# handle is expected); recover skips those records.
doc, auditor = recover.readfile(path)
return doc, [str(error) for error in auditor.errors]
@@ -167,6 +233,27 @@ import re as _re
PLAN_TITLE_RE = _re.compile(r"平面图$")
PLAN_TITLE_EXCLUDE_RE = _re.compile(r"(示意|疏散|分区|索引|定位|总平面|放大|详图|布置|大样)")
# AutoCAD MIF escapes (\M+5XXXX): CJK text in pre-Unicode DWGs survives
# DXF conversion as literal escape sequences, so title matching must decode
# them first. The digit selects the codepage; XXXX is the hex byte pair.
MIF_RE = _re.compile(r"\\M\+([1-5])([0-9A-Fa-f]{4})")
MIF_CODEPAGES = {"1": "cp932", "2": "big5", "3": "cp949", "4": "johab", "5": "gbk"}
def decode_mif(text: str) -> str:
if "\\M+" not in text:
return text
def repl(match: "_re.Match[str]") -> str:
value = int(match.group(2), 16)
raw = value.to_bytes(2, "big") if value > 0xFF else value.to_bytes(1, "big")
try:
return raw.decode(MIF_CODEPAGES.get(match.group(1), "gbk"))
except (UnicodeDecodeError, LookupError):
return match.group(0)
return MIF_RE.sub(repl, text)
def count_plan_titles(entities) -> int:
"""Distinct plan-frame titles in modelspace (multi-frame sheet detector)."""
@@ -179,6 +266,7 @@ def count_plan_titles(entities) -> int:
text = str(entity.dxf.text if kind == "TEXT" else entity.plain_text() or "").strip()
except Exception:
continue
text = decode_mif(text)
if not text or len(text) > 40:
continue
if PLAN_TITLE_RE.search(text) and not PLAN_TITLE_EXCLUDE_RE.search(text):
@@ -335,6 +423,28 @@ def output_paths(output: Path) -> tuple[Path, Path]:
return output / "cad_read_report.json", output / "cad_read_report.md"
def scrub_surrogates(value):
"""Make report data JSON-writable when ezdxf's recover mode leaks
surrogate-escaped bytes (broken files read with errors="surrogateescape").
CJK bytes are recovered via GBK where possible; the batch never dies on
one corrupt drawing."""
if isinstance(value, str):
try:
value.encode("utf-8")
return value
except UnicodeEncodeError:
raw = value.encode("utf-8", errors="surrogateescape")
try:
return raw.decode("gbk")
except UnicodeDecodeError:
return raw.decode("utf-8", errors="replace")
if isinstance(value, dict):
return {scrub_surrogates(k): scrub_surrogates(v) for k, v in value.items()}
if isinstance(value, list):
return [scrub_surrogates(item) for item in value]
return value
def main() -> int:
args = parse_args()
input_dir = Path(args.input)
@@ -376,6 +486,48 @@ def main() -> int:
conversions.append({"source": str(raw), "dest": str(dest), "status": f"sanitized_{removed}_comments"})
results = [inspect_file(path, ezdxf, bbox, recover) for path in sorted(sanitized)]
# ODA auto-fallback: libredwg chokes on some TArch DWGs (duplicate handles,
# missing ENDBLK/EOF) or yields an empty modelspace even when the drawing
# is full — the 黄桥 plans sheet read 0 entities via dwg2dxf but 21k via
# ODA. Reconvert those sources through ODA when it is available.
dwg_dest_to_source = {
Path(c["dest"]).name: Path(c["source"])
for c in conversions
if str(c.get("source", "")).lower().endswith(".dwg") and c.get("dest")
}
retry: dict[str, Path] = {}
for item in results:
src = dwg_dest_to_source.get(item.get("file", ""))
if src and (item.get("status") == "failed" or item.get("modelspace_entities") == 0):
retry[item["file"]] = src
for c in list(conversions):
if c.get("status") in {"failed", "missing_dwg2dxf"} and str(c.get("source", "")).lower().endswith(".dwg"):
name = Path(c["dest"]).name if c.get("dest") else Path(c["source"]).with_suffix(".dxf").name
retry.setdefault(name, Path(c["source"]))
if retry:
produced = run_oda_batch(list(retry.values()), work_dir)
for name, src in retry.items():
out = produced.get(src)
if not out:
conversions.append({"source": str(src), "dest": "", "status": "oda_fallback_failed"})
continue
raw_dest = raw_dir / name
shutil.copy2(out, raw_dest)
clean_dest = clean_dir / name
sanitize_dxf_comments(raw_dest, clean_dest)
item = inspect_file(clean_dest, ezdxf, bbox, recover)
replaced = False
for i, existing in enumerate(results):
if existing.get("file") == name:
results[i] = item
replaced = True
break
if not replaced:
results.append(item)
conversions.append({"source": str(src), "dest": str(raw_dest), "status": "oda_fallback_ok"})
results.sort(key=lambda r: r.get("file", ""))
json_path, md_path = output_paths(Path(args.output))
json_path.parent.mkdir(parents=True, exist_ok=True)
md_path.parent.mkdir(parents=True, exist_ok=True)
@@ -386,6 +538,8 @@ def main() -> int:
"conversions": conversions,
"files": results,
}
payload = scrub_surrogates(payload)
results = payload["files"]
json_path.write_text(json.dumps(payload, ensure_ascii=False, indent=2), encoding="utf-8")
md_path.write_text(markdown_report(results, conversions, input_dir), encoding="utf-8")
print(json.dumps({"json": str(json_path), "markdown": str(md_path), "files": len(results)}, ensure_ascii=False))