Files
cad-building-showcase/scripts/cad_showcase.py

709 lines
24 KiB
Python

#!/usr/bin/env python3
"""Resumable command-line orchestrator for the deterministic showcase pipeline."""
from __future__ import annotations
import argparse
import hashlib
import importlib.metadata
import json
import os
import shutil
import socket
import subprocess
import sys
import time
import urllib.request
from datetime import datetime, timezone
from pathlib import Path
from typing import Any
import showcase_schema
from build_showcase_variants import default_blender
from cad_dxf_report import oda_converter
ROOT = Path(__file__).resolve().parent.parent
SCRIPTS = ROOT / "scripts"
def now() -> str:
return datetime.now(timezone.utc).isoformat()
DERIVE_FORBIDDEN_WARNINGS = (
"WARNING isolated_cluster",
"WARNING unsupported_roof",
"WARNING floor_order",
"WARNING topless_group",
)
def fingerprint(
config: Path,
source_report: Path,
dxf_dir: Path,
extra_inputs: tuple[Path, ...] = (),
) -> str:
"""Hash source data and pipeline code so resume cannot reuse stale work."""
digest = hashlib.sha256()
tracked = [config, source_report, *extra_inputs, *sorted(dxf_dir.glob("*.dxf"))]
tracked.extend(sorted(SCRIPTS.glob("*.py")))
tracked.extend(sorted(SCRIPTS.glob("*.mjs")))
tracked.extend(sorted((ROOT / "schemas").glob("*.json")))
tracked.extend(sorted(path for path in (ROOT / "tools").rglob("*") if path.is_file()))
tracked.extend(sorted(path for path in (ROOT / "assets" / "web-showcase-template").rglob("*") if path.is_file()))
tracked.extend(sorted(path for path in (ROOT / "assets" / "offline-viewer-libs").rglob("*") if path.is_file()))
tracked.extend(path for path in (ROOT / "package.json", ROOT / "package-lock.json") if path.exists())
for path in sorted(set(tracked), key=lambda item: str(item)):
if not path.is_file():
continue
digest.update(str(path.relative_to(ROOT) if path.is_relative_to(ROOT) else path).encode("utf-8"))
with path.open("rb") as stream:
for chunk in iter(lambda: stream.read(1024 * 1024), b""):
digest.update(chunk)
return digest.hexdigest()
def output_ready(path: Path) -> bool:
if not path.exists():
return False
if path.is_file():
return path.stat().st_size > 0
return any(path.iterdir())
class Pipeline:
def __init__(self, state_path: Path, input_fingerprint: str, resume: bool):
self.path = state_path
self.resume = resume
if state_path.exists():
try:
current = json.loads(state_path.read_text(encoding="utf-8"))
except Exception as exc:
raise SystemExit(f"Pipeline state is corrupt and was not overwritten: {state_path}: {exc}") from exc
else:
current = {}
if current.get("input_fingerprint") != input_fingerprint:
if resume and current:
print(
json.dumps(
{
"status": "INVALIDATED",
"reason": "input fingerprint changed",
"state": str(state_path),
},
ensure_ascii=False,
)
)
current = {}
self.state: dict[str, Any] = {
"state_version": 1,
"input_fingerprint": input_fingerprint,
"started_at": current.get("started_at", now()),
"updated_at": now(),
"steps": current.get("steps", {}),
}
self.save()
def save(self) -> None:
self.state["updated_at"] = now()
self.path.parent.mkdir(parents=True, exist_ok=True)
temporary = self.path.with_name(f".{self.path.name}.{os.getpid()}.tmp")
try:
with temporary.open("w", encoding="utf-8") as stream:
json.dump(self.state, stream, ensure_ascii=False, indent=2)
stream.flush()
os.fsync(stream.fileno())
os.replace(temporary, self.path)
finally:
temporary.unlink(missing_ok=True)
def run(
self,
name: str,
command: list[str],
outputs: list[Path],
forbidden_stderr: tuple[str, ...] = (),
) -> None:
old = self.state["steps"].get(name, {})
if (
self.resume
and old.get("status") == "PASS"
and old.get("command") == command
and all(output_ready(path) for path in outputs)
):
print(json.dumps({"step": name, "status": "SKIP", "reason": "resume"}, ensure_ascii=False))
return
started = now()
proc = subprocess.run(command, cwd=ROOT, text=True, capture_output=True)
forbidden_hits = [pattern for pattern in forbidden_stderr if pattern in proc.stderr]
effective_code = proc.returncode or (3 if forbidden_hits else 0)
entry = {
"status": "PASS" if effective_code == 0 else "FAIL",
"started_at": started,
"finished_at": now(),
"returncode": effective_code,
"command": command,
"outputs": [str(path) for path in outputs],
"stdout_tail": proc.stdout[-4000:],
"stderr_tail": proc.stderr[-4000:],
"forbidden_stderr_hits": forbidden_hits,
}
self.state["steps"][name] = entry
self.save()
if proc.stdout:
print(proc.stdout, end="")
if proc.stderr:
print(proc.stderr, file=sys.stderr, end="")
if forbidden_hits:
print(f"GATE FAIL [{name}]: forbidden warning(s): {', '.join(forbidden_hits)}", file=sys.stderr)
if effective_code != 0:
raise SystemExit(effective_code)
missing = [str(path) for path in outputs if not output_ready(path)]
if missing:
entry["status"] = "FAIL"
entry["stderr_tail"] += "\nMissing expected outputs: " + ", ".join(missing)
self.save()
raise SystemExit(f"Step {name} did not create expected outputs: {missing}")
def python_script(name: str, *args: str) -> list[str]:
return [sys.executable, str(SCRIPTS / name), *args]
def resolve_executable(name: str) -> str:
"""Resolve command shims portably (notably npm.cmd on Windows)."""
candidates = [name]
if sys.platform == "win32":
candidates = [f"{name}.cmd", f"{name}.exe", name]
for candidate in candidates:
found = shutil.which(candidate)
if found:
return found
raise SystemExit(f"Required executable not found on PATH: {name}")
def npm_command(*args: str) -> list[str]:
return [resolve_executable("npm"), *args]
def node_script(path: Path, *args: str) -> list[str]:
return [resolve_executable("node"), str(path), *args]
def parse_waivers(values: list[str]) -> dict[str, str]:
waivers: dict[str, str] = {}
for value in values:
name, separator, reason = value.partition("=")
name = name.strip()
reason = reason.strip()
if not separator or not name or not reason:
raise ValueError(f"Invalid waiver {value!r}; expected CHECK=REASON")
if name in waivers:
raise ValueError(f"Duplicate waiver for {name!r}")
waivers[name] = reason
return waivers
def enforce_audit_gates(pipeline: Pipeline, audit_path: Path, waivers: dict[str, str]) -> bool:
report = json.loads(audit_path.read_text(encoding="utf-8"))
blocked = []
waived = []
for gate in report.get("gates", []):
status = str(gate.get("status", "FAIL"))
if status in {"PASS", "NOT_APPLICABLE"}:
continue
name = f"audit:{gate.get('name', 'unnamed')}"
if status in {"WARN", "SKIP"} and name in waivers:
waived.append({"name": name, "original_status": status, "reason": waivers[name]})
else:
blocked.append({"name": name, "status": status, "detail": gate.get("detail", "")})
pipeline.state["audit_waivers"] = waived
if blocked:
pipeline.state["status"] = "BLOCKED"
pipeline.state["blocked_gates"] = blocked
pipeline.save()
for gate in blocked:
print(f"GATE BLOCKED [{gate['name']}]: {gate['detail']}", file=sys.stderr)
return False
pipeline.save()
return True
def available_port() -> int:
with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as stream:
stream.bind(("127.0.0.1", 0))
return int(stream.getsockname()[1])
def wait_for_http(url: str, timeout: float = 20.0) -> None:
deadline = time.monotonic() + timeout
last_error: Exception | None = None
while time.monotonic() < deadline:
try:
with urllib.request.urlopen(url, timeout=1) as response:
if response.status < 500:
return
except Exception as exc: # pragma: no cover - timing depends on host
last_error = exc
time.sleep(0.2)
raise RuntimeError(f"Local Web server did not become ready: {last_error}")
def ensure_node_runtime(pipeline: Pipeline, *, browser: bool) -> None:
pipeline.run(
"node_dependencies",
npm_command("install", "--prefix", str(ROOT), "--ignore-scripts"),
[
ROOT / "node_modules" / "playwright" / "package.json",
ROOT / "node_modules" / "@gltf-transform" / "core" / "package.json",
],
)
if browser:
pipeline.run(
"browser_runtime",
node_script(ROOT / "node_modules" / "playwright" / "cli.js", "install", "chromium"),
[],
)
def run_browser_smoke(pipeline: Pipeline, web_dir: Path, report: Path, screenshots: Path) -> None:
"""Serve the formal Web bundle and mechanically prove desktop/mobile loading."""
report.parent.mkdir(parents=True, exist_ok=True)
port = available_port()
url = f"http://127.0.0.1:{port}/"
server = subprocess.Popen(
[
sys.executable,
"-m",
"http.server",
str(port),
"--bind",
"127.0.0.1",
"--directory",
str(web_dir),
],
cwd=ROOT,
stdout=subprocess.DEVNULL,
stderr=subprocess.DEVNULL,
)
try:
wait_for_http(url)
pipeline.run(
"browser_smoke",
node_script(
SCRIPTS / "web_smoke_test.mjs",
"--url",
url,
"--out",
str(report),
"--screenshots-dir",
str(screenshots),
),
[report, screenshots],
)
finally:
server.terminate()
try:
server.wait(timeout=5)
except subprocess.TimeoutExpired: # pragma: no cover - defensive cleanup
server.kill()
server.wait(timeout=5)
def run_offline_browser_smoke(pipeline: Pipeline, viewer: Path, report: Path, screenshot: Path) -> None:
"""Open the single-file viewer through file:// and prove that its model renders."""
pipeline.run(
"offline_browser_smoke",
node_script(
SCRIPTS / "offline_viewer_smoke.mjs",
"--viewer",
str(viewer),
"--out",
str(report),
"--screenshot",
str(screenshot),
),
[report, screenshot],
)
def doctor() -> int:
modules = ["ezdxf", "jsonschema", "numpy", "scipy", "shapely", "trimesh", "matplotlib", "PIL", "pygltflib"]
checks = []
failed = False
for module in modules:
try:
version = importlib.metadata.version("pillow" if module == "PIL" else module)
checks.append({"name": module, "status": "PASS", "version": version})
except Exception as exc:
checks.append({"name": module, "status": "FAIL", "detail": str(exc)})
failed = True
for executable in ("node", "npm", "dwg2dxf"):
try:
path = resolve_executable(executable)
except SystemExit:
path = None
checks.append({"name": executable, "status": "PASS" if path else "OPTIONAL", "path": path})
for executable, resolver in (("ODAFileConverter", oda_converter), ("blender", default_blender)):
candidate = resolver()
path = candidate if candidate and Path(candidate).exists() else None
checks.append({"name": executable, "status": "PASS" if path else "OPTIONAL", "path": path})
print(json.dumps({"status": "FAIL" if failed else "PASS", "checks": checks}, ensure_ascii=False, indent=2))
return 3 if failed else 0
def inspect_command(args: argparse.Namespace) -> int:
command = python_script(
"cad_dxf_report.py",
"--input",
args.input,
"--work",
args.work,
"--output",
args.output,
"--vendor-dir",
args.vendor_dir,
"--converter",
args.converter,
)
if args.allow_partial:
command.extend(["--allow-partial", "--partial-reason", args.partial_reason])
if args.no_convert:
command.append("--no-convert")
return subprocess.run(command, cwd=ROOT).returncode
def build_with_trimesh(pipeline: Pipeline, bundle: Path) -> list[Path]:
glbs = []
for key in ("overview", "standard", "detail"):
spec = bundle / "variant_specs" / f"{key}.json"
out_dir = bundle / "variants" / key
raw_glb = out_dir / f"{key}.raw.glb"
glb = out_dir / f"{key}.glb"
metadata = out_dir / "model_metadata.json"
pipeline.run(
f"glb_{key}",
python_script(
"spec_to_glb.py",
"--spec",
str(spec),
"--out",
str(raw_glb),
"--metadata-out",
str(metadata),
"--name",
key,
),
[raw_glb, metadata],
forbidden_stderr=("WARNING mesh_from_polygons",),
)
pipeline.run(
f"compress_{key}",
node_script(ROOT / "tools" / "compress.mjs", str(raw_glb), str(glb)),
[glb],
)
glbs.append(glb)
return glbs
def run_pipeline(args: argparse.Namespace) -> int:
dxf_dir = Path(args.dxf_dir).resolve()
config = Path(args.config).resolve()
source_report = Path(args.source_report).resolve()
scope_manifest = Path(args.scope_manifest).resolve()
frame_inputs = tuple(Path(path).resolve() for path in args.frames)
try:
waivers = parse_waivers(args.waiver)
except ValueError as exc:
print(f"GATE FAIL [waiver]: {exc}", file=sys.stderr)
return 3
try:
report = json.loads(source_report.read_text(encoding="utf-8"))
showcase_schema.validate_source_report(report, dxf_dir)
except (OSError, json.JSONDecodeError, showcase_schema.ValidationFailure) as exc:
print(f"GATE FAIL [source_report]: {exc}", file=sys.stderr)
return 3
source_status = str((report.get("gate") or {}).get("status", "FAIL"))
if source_status == "WARN" and "cad_source_report" not in waivers:
print(
"GATE BLOCKED [cad_source_report]: partial source report requires "
"--waiver 'cad_source_report=<user-approved reason>'",
file=sys.stderr,
)
return 3
out_dir = Path(args.out_dir).resolve()
out_dir.mkdir(parents=True, exist_ok=True)
pipeline = Pipeline(
out_dir / "pipeline_state.json",
fingerprint(config, source_report, dxf_dir, (scope_manifest, *frame_inputs)),
args.resume,
)
if source_status == "WARN":
pipeline.state["source_waiver"] = {
"check": "cad_source_report",
"reason": waivers["cad_source_report"],
}
pipeline.save()
spec = out_dir / "building_spec.json"
audit = out_dir / "extraction_audit.json"
audit_md = out_dir / "extraction_audit.md"
overlays = out_dir / "overlays"
pipeline.run(
"derive",
python_script(
"derive_showcase_spec.py",
"--dxf-dir",
str(dxf_dir),
"--config",
str(config),
"--out",
str(spec),
"--vendor-dir",
args.vendor_dir,
),
[spec],
forbidden_stderr=DERIVE_FORBIDDEN_WARNINGS,
)
audit_cmd = python_script(
"audit_showcase_extraction.py",
"--dxf-dir",
str(dxf_dir),
"--config",
str(config),
"--spec",
str(spec),
"--out",
str(audit),
"--vendor-dir",
args.vendor_dir,
)
if args.frames:
audit_cmd.extend(["--frames", *args.frames])
pipeline.run("audit", audit_cmd, [audit, audit_md])
if not enforce_audit_gates(pipeline, audit, waivers):
return 3
pipeline.run(
"overlays",
python_script(
"draw_coverage_overlay.py",
"--dxf-dir",
str(dxf_dir),
"--config",
str(config),
"--spec",
str(spec),
"--out-dir",
str(overlays),
"--vendor-dir",
args.vendor_dir,
),
[overlays],
)
bundle = out_dir / "delivery"
manifest = bundle / "variants_manifest.json"
variant_cmd = python_script(
"build_showcase_variants.py",
"--spec",
str(spec),
"--out-dir",
str(bundle),
"--name",
args.name,
)
if args.backend == "blender":
variant_cmd.append("--build")
if args.blender:
variant_cmd.extend(["--blender", args.blender])
pipeline.run("variants", variant_cmd, [manifest, bundle / "variant_specs" / "standard.json"])
if args.backend == "trimesh" or args.web or args.offline:
ensure_node_runtime(pipeline, browser=args.web or args.offline)
if args.backend == "trimesh":
glbs = build_with_trimesh(pipeline, bundle)
else:
glbs = [bundle / "variants" / key / f"{key}.glb" for key in ("overview", "standard", "detail")]
web_dir = out_dir / "web"
if args.web:
pipeline.run(
"web",
python_script(
"write_web_showcase.py",
"--variants-manifest",
str(manifest),
"--out-dir",
str(web_dir),
"--title",
args.name,
),
[web_dir / "index.html", web_dir / "model.glb", web_dir / "model_metadata.json"],
)
if not args.skip_web_deps:
pipeline.run(
"web_dependencies",
npm_command("install", "--prefix", str(web_dir), "--ignore-scripts"),
[web_dir / "node_modules" / "three" / "build" / "three.module.js"],
)
browser_report = (
Path(args.browser_report).resolve() if args.browser_report else out_dir / "browser_smoke_report.json"
)
run_browser_smoke(pipeline, web_dir, browser_report, out_dir / "browser_smoke")
standard = bundle / "variants" / "standard" / "standard.glb"
offline = out_dir / "viewer.html"
offline_browser_report = out_dir / "offline_browser_smoke_report.json"
if args.offline:
offline_source = standard.with_name("standard.raw.glb")
if not offline_source.exists():
offline_source = standard
pipeline.run(
"offline",
python_script(
"write_offline_viewer.py",
"--spec",
str(spec),
"--glb",
str(offline_source),
"--out",
str(offline),
"--title",
args.name,
),
[offline],
)
run_offline_browser_smoke(
pipeline,
offline,
offline_browser_report,
out_dir / "offline_browser_smoke.png",
)
preview = out_dir / "verify_previews.png"
pipeline.run(
"preview",
python_script(
"verify_showcase.py",
"--spec",
str(spec),
"--glb",
*[str(path) for path in glbs],
"--out",
str(preview),
"--budget-mb",
str(args.budget_mb),
"--strict",
),
[preview],
)
evidence = out_dir / "evidence_manifest.json"
verify_cmd = python_script(
"verify_delivery.py",
"--spec",
str(spec),
"--source-report",
str(source_report),
"--variants-manifest",
str(manifest),
"--render",
str(preview),
"--audit",
str(audit),
"--require-audit",
"--budget-mb",
str(args.budget_mb),
"--evidence-out",
str(evidence),
"--scope-manifest",
str(scope_manifest),
"--require-scope-manifest",
)
if args.web:
verify_cmd.extend(
["--web-dir", str(web_dir), "--browser-report", str(browser_report), "--require-browser-report"]
)
if args.offline:
verify_cmd.extend(
[
"--offline-viewer",
str(offline),
"--offline-browser-report",
str(offline_browser_report),
]
)
if args.require_browser_report and not args.web:
verify_cmd.append("--require-browser-report")
for value in args.waiver:
verify_cmd.extend(["--waiver", value])
pipeline.run("delivery_verification", verify_cmd, [evidence])
verification = json.loads(evidence.read_text(encoding="utf-8"))
if verification.get("status") != "PASS":
pipeline.state["status"] = "BLOCKED"
pipeline.save()
print(f"GATE BLOCKED [delivery_verification]: {verification.get('status')}", file=sys.stderr)
return 3
pipeline.state["status"] = "PASS"
pipeline.state["finished_at"] = now()
pipeline.save()
print(json.dumps({"status": "PASS", "out_dir": str(out_dir), "evidence": str(evidence)}, ensure_ascii=False))
return 0
def parser() -> argparse.ArgumentParser:
root = argparse.ArgumentParser(description=__doc__)
sub = root.add_subparsers(dest="command", required=True)
sub.add_parser("doctor", help="Check Python, Node, CAD converter, and Blender prerequisites")
inspect = sub.add_parser("inspect", help="Convert/read a CAD folder and enforce the source gate")
inspect.add_argument("--input", required=True)
inspect.add_argument("--work", required=True)
inspect.add_argument("--output", required=True)
inspect.add_argument("--vendor-dir", default="work/vendor")
inspect.add_argument("--converter", choices=("auto", "libredwg", "oda"), default="auto")
inspect.add_argument("--no-convert", action="store_true")
inspect.add_argument("--allow-partial", action="store_true")
inspect.add_argument("--partial-reason", default="")
run = sub.add_parser("run", help="Run extract -> audit -> overlays -> variants -> verify from an approved config")
run.add_argument("--dxf-dir", required=True)
run.add_argument("--config", required=True)
run.add_argument("--source-report", required=True, help="Passing cad_read_report.json for this sanitized DXF dir")
run.add_argument("--out-dir", required=True)
run.add_argument("--name", default="cad_building_showcase")
run.add_argument("--vendor-dir", default="work/vendor")
run.add_argument("--frames", nargs="*", default=[])
run.add_argument("--backend", choices=("trimesh", "blender"), default="trimesh")
run.add_argument("--blender")
run.add_argument("--web", action="store_true")
run.add_argument("--skip-web-deps", action="store_true")
run.add_argument("--offline", action="store_true")
run.add_argument("--browser-report")
run.add_argument("--require-browser-report", action="store_true")
run.add_argument(
"--scope-manifest",
required=True,
help="Required single-building/campus scope ledger included in final verification",
)
run.add_argument(
"--waiver",
action="append",
default=[],
metavar="CHECK=REASON",
help="Explicitly acknowledge one WARN/SKIP audit or delivery check",
)
run.add_argument("--budget-mb", type=float, default=25.0)
run.add_argument("--resume", action="store_true")
return root
def main() -> int:
args = parser().parse_args()
if args.command == "doctor":
return doctor()
if args.command == "inspect":
return inspect_command(args)
return run_pipeline(args)
if __name__ == "__main__":
raise SystemExit(main())