Generalization hardening from test3d-7 round 1 (黄桥文体中心): 8 gap fixes
Conversion/encoding chain (libredwg on Chinese TArch DWGs):
- run_dwg2dxf: capture bytes, decode leniently (GBK console output crashed
the batch under subprocess text=True)
- sanitize_dxf_comments: byte-level, code/value pair walk — the UTF-8
errors="ignore" round-trip mangled every GBK CJK string ("一层平面图"
-> "一平图") and eating XRECORD "999" VALUE lines corrupted files
- read_doc: recover fallback also on ValueError; scrub_surrogates keeps one
corrupt drawing from killing the whole report
- decode_mif in report + frame detector: \M+5XXXX escapes (whole 黄桥 set)
restored so plan-title detection works at all
- ODA auto-fallback: reconvert DWGs whose libredwg DXF fails inspection or
reads an empty modelspace (黄桥 plans: 0 entities via dwg2dxf, 21k via
ODA); env ODA_CONVERTER override, fake-converter regression test
Frame detection:
- detect_sheet_frames expands INSERTs (memoized, depth<=3) when collecting
axis/wall lines — 黄桥 wraps each frame's grid in anonymous A$C... blocks
(msp had 8 DOTE lines, blocks held 700); all 7 frames went from WEAK
clip-corner fallback to grid-aligned votes x=14
Extraction judgment rules (both loud, both default-on, both switchable):
- drop_isolated_clusters: tiny segment clusters far from the main mass are
sheet-legend content drawn on real layers (title-block window symbols made
every floor's footprint grow a sliver fin)
- drop_unsupported_roof_lines: roof outlines with no footprint/cap plate
within 2m beneath are hatch-only-support canopies that extrude into
floating ribbons (F3 SW 坡屋面, B1 ground canopy at -5.85m)
Tests: 64 -> 75 checks, incl. GBK/MIF synthetic DXFs, block-wrapped grid
alignment (non-uniform bays — uniform grids make subset correlation
ambiguous), legend-cluster and unsupported-roof fixtures.
references/generalization_test_log.md tracks the test protocol, per-project
gate table, and every gap with its regression coverage.
Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
@@ -5,6 +5,7 @@ from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import os
|
||||
import shutil
|
||||
import subprocess
|
||||
import sys
|
||||
@@ -68,32 +69,94 @@ def run_dwg2dxf(source: Path, dest: Path) -> dict:
|
||||
}
|
||||
dest.parent.mkdir(parents=True, exist_ok=True)
|
||||
cmd = [exe, "-y", "-o", str(dest), str(source)]
|
||||
proc = subprocess.run(cmd, text=True, capture_output=True)
|
||||
# libredwg echoes CJK layer/file names in the platform encoding (GBK on
|
||||
# Chinese drawings), so strict UTF-8 text mode crashes; decode leniently.
|
||||
proc = subprocess.run(cmd, capture_output=True)
|
||||
stdout = proc.stdout.decode("utf-8", errors="replace")
|
||||
stderr = proc.stderr.decode("utf-8", errors="replace")
|
||||
return {
|
||||
"source": str(source),
|
||||
"dest": str(dest),
|
||||
"status": "ok" if proc.returncode == 0 and dest.exists() else "failed",
|
||||
"returncode": proc.returncode,
|
||||
"stdout": proc.stdout[-2000:],
|
||||
"stderr": proc.stderr[-2000:],
|
||||
"stdout": stdout[-2000:],
|
||||
"stderr": stderr[-2000:],
|
||||
}
|
||||
|
||||
|
||||
def oda_converter() -> str | None:
|
||||
"""ODA File Converter path (env ODA_CONVERTER overrides the macOS default)."""
|
||||
override = os.environ.get("ODA_CONVERTER")
|
||||
if override:
|
||||
return override if Path(override).exists() else None
|
||||
default = Path("/Applications/ODAFileConverter.app/Contents/MacOS/ODAFileConverter")
|
||||
return str(default) if default.exists() else None
|
||||
|
||||
|
||||
def run_oda_batch(sources: list[Path], work_dir: Path) -> dict[Path, Path]:
|
||||
"""Convert DWGs via ODA File Converter; returns {source: produced_dxf}.
|
||||
|
||||
ODA converts whole directories with absolute paths and reports bad
|
||||
arguments via GUI dialogs (silent exit-0), so sources are staged under
|
||||
ASCII names in a scratch in-dir and results matched back by stem.
|
||||
"""
|
||||
exe = oda_converter()
|
||||
if not exe or not sources:
|
||||
return {}
|
||||
in_dir = work_dir / "oda_in"
|
||||
out_dir = work_dir / "oda_out"
|
||||
shutil.rmtree(in_dir, ignore_errors=True)
|
||||
shutil.rmtree(out_dir, ignore_errors=True)
|
||||
in_dir.mkdir(parents=True)
|
||||
out_dir.mkdir(parents=True)
|
||||
mapping: dict[str, Path] = {}
|
||||
for index, src in enumerate(sources):
|
||||
stem = f"oda{index:03d}"
|
||||
shutil.copy2(src, in_dir / f"{stem}.dwg")
|
||||
mapping[stem] = src
|
||||
try:
|
||||
subprocess.run(
|
||||
[exe, str(in_dir.resolve()), str(out_dir.resolve()), "ACAD2018", "DXF", "0", "1", "*.dwg"],
|
||||
capture_output=True, timeout=1800,
|
||||
)
|
||||
except (subprocess.TimeoutExpired, OSError):
|
||||
return {}
|
||||
produced: dict[Path, Path] = {}
|
||||
for stem, src in mapping.items():
|
||||
out = out_dir / f"{stem}.dxf"
|
||||
if out.exists() and out.stat().st_size > 0:
|
||||
produced[src] = out
|
||||
return produced
|
||||
|
||||
|
||||
def sanitize_dxf_comments(source: Path, dest: Path) -> int:
|
||||
"""Remove DXF group code 999 comment pairs that can break brittle importers."""
|
||||
lines = source.read_text(errors="ignore").splitlines(keepends=True)
|
||||
"""Remove DXF group code 999 comment pairs that can break brittle importers.
|
||||
|
||||
Operates on BYTES: pre-2007 DXF from libredwg/TArch stores CJK text in the
|
||||
$DWGCODEPAGE encoding (ANSI_936/GBK). A UTF-8 round-trip with
|
||||
errors="ignore" silently drops those bytes, mangling every CJK layer/text
|
||||
("一层平面图" -> "一平图") and breaking plan-title detection downstream, so
|
||||
the file content must pass through unchanged apart from the comment pairs.
|
||||
"""
|
||||
lines = source.read_bytes().splitlines(keepends=True)
|
||||
dest.parent.mkdir(parents=True, exist_ok=True)
|
||||
cleaned: list[str] = []
|
||||
cleaned: list[bytes] = []
|
||||
removed = 0
|
||||
index = 0
|
||||
while index < len(lines):
|
||||
if lines[index].strip() == "999":
|
||||
# DXF is a strict alternation of group-code line / value line, so walk in
|
||||
# pairs: a bare "999" VALUE (e.g. an integer inside an ODA XRECORD) must
|
||||
# not be mistaken for a comment group code, or the pair removal shifts
|
||||
# every following tag and corrupts the file.
|
||||
while index + 1 < len(lines):
|
||||
if lines[index].strip() == b"999":
|
||||
removed += 1
|
||||
index += 2
|
||||
continue
|
||||
cleaned.append(lines[index])
|
||||
index += 1
|
||||
dest.write_text("".join(cleaned), encoding="utf-8", errors="ignore")
|
||||
cleaned.append(lines[index + 1])
|
||||
index += 2
|
||||
cleaned.extend(lines[index:])
|
||||
dest.write_bytes(b"".join(cleaned))
|
||||
return removed
|
||||
|
||||
|
||||
@@ -130,7 +193,10 @@ def guess_unit_scale(bbox_data: dict | None, insunits: int) -> dict:
|
||||
def read_doc(path: Path, ezdxf, recover):
|
||||
try:
|
||||
return ezdxf.readfile(path), []
|
||||
except ezdxf.DXFError:
|
||||
except (ezdxf.DXFError, ValueError):
|
||||
# Strict reads can also raise plain ValueError on quirky-but-
|
||||
# recoverable files (e.g. an XRECORD holding "{ACAD_REACTORS" where a
|
||||
# handle is expected); recover skips those records.
|
||||
doc, auditor = recover.readfile(path)
|
||||
return doc, [str(error) for error in auditor.errors]
|
||||
|
||||
@@ -167,6 +233,27 @@ import re as _re
|
||||
PLAN_TITLE_RE = _re.compile(r"平面图$")
|
||||
PLAN_TITLE_EXCLUDE_RE = _re.compile(r"(示意|疏散|分区|索引|定位|总平面|放大|详图|布置|大样)")
|
||||
|
||||
# AutoCAD MIF escapes (\M+5XXXX): CJK text in pre-Unicode DWGs survives
|
||||
# DXF conversion as literal escape sequences, so title matching must decode
|
||||
# them first. The digit selects the codepage; XXXX is the hex byte pair.
|
||||
MIF_RE = _re.compile(r"\\M\+([1-5])([0-9A-Fa-f]{4})")
|
||||
MIF_CODEPAGES = {"1": "cp932", "2": "big5", "3": "cp949", "4": "johab", "5": "gbk"}
|
||||
|
||||
|
||||
def decode_mif(text: str) -> str:
|
||||
if "\\M+" not in text:
|
||||
return text
|
||||
|
||||
def repl(match: "_re.Match[str]") -> str:
|
||||
value = int(match.group(2), 16)
|
||||
raw = value.to_bytes(2, "big") if value > 0xFF else value.to_bytes(1, "big")
|
||||
try:
|
||||
return raw.decode(MIF_CODEPAGES.get(match.group(1), "gbk"))
|
||||
except (UnicodeDecodeError, LookupError):
|
||||
return match.group(0)
|
||||
|
||||
return MIF_RE.sub(repl, text)
|
||||
|
||||
|
||||
def count_plan_titles(entities) -> int:
|
||||
"""Distinct plan-frame titles in modelspace (multi-frame sheet detector)."""
|
||||
@@ -179,6 +266,7 @@ def count_plan_titles(entities) -> int:
|
||||
text = str(entity.dxf.text if kind == "TEXT" else entity.plain_text() or "").strip()
|
||||
except Exception:
|
||||
continue
|
||||
text = decode_mif(text)
|
||||
if not text or len(text) > 40:
|
||||
continue
|
||||
if PLAN_TITLE_RE.search(text) and not PLAN_TITLE_EXCLUDE_RE.search(text):
|
||||
@@ -335,6 +423,28 @@ def output_paths(output: Path) -> tuple[Path, Path]:
|
||||
return output / "cad_read_report.json", output / "cad_read_report.md"
|
||||
|
||||
|
||||
def scrub_surrogates(value):
|
||||
"""Make report data JSON-writable when ezdxf's recover mode leaks
|
||||
surrogate-escaped bytes (broken files read with errors="surrogateescape").
|
||||
CJK bytes are recovered via GBK where possible; the batch never dies on
|
||||
one corrupt drawing."""
|
||||
if isinstance(value, str):
|
||||
try:
|
||||
value.encode("utf-8")
|
||||
return value
|
||||
except UnicodeEncodeError:
|
||||
raw = value.encode("utf-8", errors="surrogateescape")
|
||||
try:
|
||||
return raw.decode("gbk")
|
||||
except UnicodeDecodeError:
|
||||
return raw.decode("utf-8", errors="replace")
|
||||
if isinstance(value, dict):
|
||||
return {scrub_surrogates(k): scrub_surrogates(v) for k, v in value.items()}
|
||||
if isinstance(value, list):
|
||||
return [scrub_surrogates(item) for item in value]
|
||||
return value
|
||||
|
||||
|
||||
def main() -> int:
|
||||
args = parse_args()
|
||||
input_dir = Path(args.input)
|
||||
@@ -376,6 +486,48 @@ def main() -> int:
|
||||
conversions.append({"source": str(raw), "dest": str(dest), "status": f"sanitized_{removed}_comments"})
|
||||
|
||||
results = [inspect_file(path, ezdxf, bbox, recover) for path in sorted(sanitized)]
|
||||
|
||||
# ODA auto-fallback: libredwg chokes on some TArch DWGs (duplicate handles,
|
||||
# missing ENDBLK/EOF) or yields an empty modelspace even when the drawing
|
||||
# is full — the 黄桥 plans sheet read 0 entities via dwg2dxf but 21k via
|
||||
# ODA. Reconvert those sources through ODA when it is available.
|
||||
dwg_dest_to_source = {
|
||||
Path(c["dest"]).name: Path(c["source"])
|
||||
for c in conversions
|
||||
if str(c.get("source", "")).lower().endswith(".dwg") and c.get("dest")
|
||||
}
|
||||
retry: dict[str, Path] = {}
|
||||
for item in results:
|
||||
src = dwg_dest_to_source.get(item.get("file", ""))
|
||||
if src and (item.get("status") == "failed" or item.get("modelspace_entities") == 0):
|
||||
retry[item["file"]] = src
|
||||
for c in list(conversions):
|
||||
if c.get("status") in {"failed", "missing_dwg2dxf"} and str(c.get("source", "")).lower().endswith(".dwg"):
|
||||
name = Path(c["dest"]).name if c.get("dest") else Path(c["source"]).with_suffix(".dxf").name
|
||||
retry.setdefault(name, Path(c["source"]))
|
||||
if retry:
|
||||
produced = run_oda_batch(list(retry.values()), work_dir)
|
||||
for name, src in retry.items():
|
||||
out = produced.get(src)
|
||||
if not out:
|
||||
conversions.append({"source": str(src), "dest": "", "status": "oda_fallback_failed"})
|
||||
continue
|
||||
raw_dest = raw_dir / name
|
||||
shutil.copy2(out, raw_dest)
|
||||
clean_dest = clean_dir / name
|
||||
sanitize_dxf_comments(raw_dest, clean_dest)
|
||||
item = inspect_file(clean_dest, ezdxf, bbox, recover)
|
||||
replaced = False
|
||||
for i, existing in enumerate(results):
|
||||
if existing.get("file") == name:
|
||||
results[i] = item
|
||||
replaced = True
|
||||
break
|
||||
if not replaced:
|
||||
results.append(item)
|
||||
conversions.append({"source": str(src), "dest": str(raw_dest), "status": "oda_fallback_ok"})
|
||||
results.sort(key=lambda r: r.get("file", ""))
|
||||
|
||||
json_path, md_path = output_paths(Path(args.output))
|
||||
json_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
md_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
@@ -386,6 +538,8 @@ def main() -> int:
|
||||
"conversions": conversions,
|
||||
"files": results,
|
||||
}
|
||||
payload = scrub_surrogates(payload)
|
||||
results = payload["files"]
|
||||
json_path.write_text(json.dumps(payload, ensure_ascii=False, indent=2), encoding="utf-8")
|
||||
md_path.write_text(markdown_report(results, conversions, input_dir), encoding="utf-8")
|
||||
print(json.dumps({"json": str(json_path), "markdown": str(md_path), "files": len(results)}, ensure_ascii=False))
|
||||
|
||||
Reference in New Issue
Block a user