570 lines
20 KiB
Python
570 lines
20 KiB
Python
#!/usr/bin/env python3
|
|
"""Resumable command-line orchestrator for the deterministic showcase pipeline."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
import hashlib
|
|
import importlib.metadata
|
|
import json
|
|
import shutil
|
|
import socket
|
|
import subprocess
|
|
import sys
|
|
import time
|
|
import urllib.request
|
|
from datetime import datetime, timezone
|
|
from pathlib import Path
|
|
from typing import Any
|
|
|
|
import showcase_schema
|
|
from build_showcase_variants import default_blender
|
|
from cad_dxf_report import oda_converter
|
|
|
|
|
|
ROOT = Path(__file__).resolve().parent.parent
|
|
SCRIPTS = ROOT / "scripts"
|
|
|
|
|
|
def now() -> str:
|
|
return datetime.now(timezone.utc).isoformat()
|
|
|
|
|
|
def fingerprint(config: Path, source_report: Path, dxf_dir: Path) -> str:
|
|
"""Hash source data and pipeline code so resume cannot reuse stale work."""
|
|
digest = hashlib.sha256()
|
|
tracked = [config, source_report, *sorted(dxf_dir.glob("*.dxf"))]
|
|
tracked.extend(sorted(SCRIPTS.glob("*.py")))
|
|
tracked.extend(sorted(SCRIPTS.glob("*.mjs")))
|
|
tracked.extend(sorted((ROOT / "schemas").glob("*.json")))
|
|
tracked.extend(sorted((ROOT / "assets" / "web-showcase-template").glob("*")))
|
|
tracked.extend(path for path in (ROOT / "package.json", ROOT / "package-lock.json") if path.exists())
|
|
for path in tracked:
|
|
if not path.is_file():
|
|
continue
|
|
digest.update(str(path.relative_to(ROOT) if path.is_relative_to(ROOT) else path).encode("utf-8"))
|
|
with path.open("rb") as stream:
|
|
for chunk in iter(lambda: stream.read(1024 * 1024), b""):
|
|
digest.update(chunk)
|
|
return digest.hexdigest()
|
|
|
|
|
|
def output_ready(path: Path) -> bool:
|
|
if not path.exists():
|
|
return False
|
|
if path.is_file():
|
|
return path.stat().st_size > 0
|
|
return any(path.iterdir())
|
|
|
|
|
|
class Pipeline:
|
|
def __init__(self, state_path: Path, input_fingerprint: str, resume: bool):
|
|
self.path = state_path
|
|
self.resume = resume
|
|
if state_path.exists():
|
|
try:
|
|
current = json.loads(state_path.read_text(encoding="utf-8"))
|
|
except Exception:
|
|
current = {}
|
|
else:
|
|
current = {}
|
|
if current.get("input_fingerprint") != input_fingerprint:
|
|
current = {}
|
|
self.state: dict[str, Any] = {
|
|
"state_version": 1,
|
|
"input_fingerprint": input_fingerprint,
|
|
"started_at": current.get("started_at", now()),
|
|
"updated_at": now(),
|
|
"steps": current.get("steps", {}),
|
|
}
|
|
self.save()
|
|
|
|
def save(self) -> None:
|
|
self.state["updated_at"] = now()
|
|
self.path.parent.mkdir(parents=True, exist_ok=True)
|
|
self.path.write_text(json.dumps(self.state, ensure_ascii=False, indent=2), encoding="utf-8")
|
|
|
|
def run(
|
|
self,
|
|
name: str,
|
|
command: list[str],
|
|
outputs: list[Path],
|
|
forbidden_stderr: tuple[str, ...] = (),
|
|
) -> None:
|
|
old = self.state["steps"].get(name, {})
|
|
if (
|
|
self.resume
|
|
and old.get("status") == "PASS"
|
|
and old.get("command") == command
|
|
and all(output_ready(path) for path in outputs)
|
|
):
|
|
print(json.dumps({"step": name, "status": "SKIP", "reason": "resume"}, ensure_ascii=False))
|
|
return
|
|
started = now()
|
|
proc = subprocess.run(command, cwd=ROOT, text=True, capture_output=True)
|
|
forbidden_hits = [pattern for pattern in forbidden_stderr if pattern in proc.stderr]
|
|
effective_code = proc.returncode or (3 if forbidden_hits else 0)
|
|
entry = {
|
|
"status": "PASS" if effective_code == 0 else "FAIL",
|
|
"started_at": started,
|
|
"finished_at": now(),
|
|
"returncode": effective_code,
|
|
"command": command,
|
|
"outputs": [str(path) for path in outputs],
|
|
"stdout_tail": proc.stdout[-4000:],
|
|
"stderr_tail": proc.stderr[-4000:],
|
|
"forbidden_stderr_hits": forbidden_hits,
|
|
}
|
|
self.state["steps"][name] = entry
|
|
self.save()
|
|
if proc.stdout:
|
|
print(proc.stdout, end="")
|
|
if proc.stderr:
|
|
print(proc.stderr, file=sys.stderr, end="")
|
|
if forbidden_hits:
|
|
print(f"GATE FAIL [{name}]: forbidden warning(s): {', '.join(forbidden_hits)}", file=sys.stderr)
|
|
if effective_code != 0:
|
|
raise SystemExit(effective_code)
|
|
missing = [str(path) for path in outputs if not output_ready(path)]
|
|
if missing:
|
|
entry["status"] = "FAIL"
|
|
entry["stderr_tail"] += "\nMissing expected outputs: " + ", ".join(missing)
|
|
self.save()
|
|
raise SystemExit(f"Step {name} did not create expected outputs: {missing}")
|
|
|
|
|
|
def python_script(name: str, *args: str) -> list[str]:
|
|
return [sys.executable, str(SCRIPTS / name), *args]
|
|
|
|
|
|
def parse_waivers(values: list[str]) -> dict[str, str]:
|
|
waivers: dict[str, str] = {}
|
|
for value in values:
|
|
name, separator, reason = value.partition("=")
|
|
name = name.strip()
|
|
reason = reason.strip()
|
|
if not separator or not name or not reason:
|
|
raise ValueError(f"Invalid waiver {value!r}; expected CHECK=REASON")
|
|
if name in waivers:
|
|
raise ValueError(f"Duplicate waiver for {name!r}")
|
|
waivers[name] = reason
|
|
return waivers
|
|
|
|
|
|
def enforce_audit_gates(pipeline: Pipeline, audit_path: Path, waivers: dict[str, str]) -> bool:
|
|
report = json.loads(audit_path.read_text(encoding="utf-8"))
|
|
blocked = []
|
|
waived = []
|
|
for gate in report.get("gates", []):
|
|
status = str(gate.get("status", "FAIL"))
|
|
if status == "PASS":
|
|
continue
|
|
name = f"audit:{gate.get('name', 'unnamed')}"
|
|
if status in {"WARN", "SKIP"} and name in waivers:
|
|
waived.append({"name": name, "original_status": status, "reason": waivers[name]})
|
|
else:
|
|
blocked.append({"name": name, "status": status, "detail": gate.get("detail", "")})
|
|
pipeline.state["audit_waivers"] = waived
|
|
if blocked:
|
|
pipeline.state["status"] = "BLOCKED"
|
|
pipeline.state["blocked_gates"] = blocked
|
|
pipeline.save()
|
|
for gate in blocked:
|
|
print(f"GATE BLOCKED [{gate['name']}]: {gate['detail']}", file=sys.stderr)
|
|
return False
|
|
pipeline.save()
|
|
return True
|
|
|
|
|
|
def available_port() -> int:
|
|
with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as stream:
|
|
stream.bind(("127.0.0.1", 0))
|
|
return int(stream.getsockname()[1])
|
|
|
|
|
|
def wait_for_http(url: str, timeout: float = 20.0) -> None:
|
|
deadline = time.monotonic() + timeout
|
|
last_error: Exception | None = None
|
|
while time.monotonic() < deadline:
|
|
try:
|
|
with urllib.request.urlopen(url, timeout=1) as response:
|
|
if response.status < 500:
|
|
return
|
|
except Exception as exc: # pragma: no cover - timing depends on host
|
|
last_error = exc
|
|
time.sleep(0.2)
|
|
raise RuntimeError(f"Local Web server did not become ready: {last_error}")
|
|
|
|
|
|
def run_browser_smoke(pipeline: Pipeline, web_dir: Path, report: Path, screenshots: Path) -> None:
|
|
"""Serve the formal Web bundle and mechanically prove desktop/mobile loading."""
|
|
pipeline.run(
|
|
"browser_dependencies",
|
|
["npm", "install", "--prefix", str(ROOT), "--ignore-scripts"],
|
|
[ROOT / "node_modules" / "playwright" / "package.json"],
|
|
)
|
|
pipeline.run(
|
|
"browser_runtime",
|
|
[str(ROOT / "node_modules" / ".bin" / "playwright"), "install", "chromium"],
|
|
[],
|
|
)
|
|
report.parent.mkdir(parents=True, exist_ok=True)
|
|
port = available_port()
|
|
url = f"http://127.0.0.1:{port}/"
|
|
server = subprocess.Popen(
|
|
[
|
|
sys.executable,
|
|
"-m",
|
|
"http.server",
|
|
str(port),
|
|
"--bind",
|
|
"127.0.0.1",
|
|
"--directory",
|
|
str(web_dir),
|
|
],
|
|
cwd=ROOT,
|
|
stdout=subprocess.DEVNULL,
|
|
stderr=subprocess.DEVNULL,
|
|
)
|
|
try:
|
|
wait_for_http(url)
|
|
pipeline.run(
|
|
"browser_smoke",
|
|
[
|
|
"node",
|
|
str(SCRIPTS / "web_smoke_test.mjs"),
|
|
"--url",
|
|
url,
|
|
"--out",
|
|
str(report),
|
|
"--screenshots-dir",
|
|
str(screenshots),
|
|
],
|
|
[report, screenshots],
|
|
)
|
|
finally:
|
|
server.terminate()
|
|
try:
|
|
server.wait(timeout=5)
|
|
except subprocess.TimeoutExpired: # pragma: no cover - defensive cleanup
|
|
server.kill()
|
|
server.wait(timeout=5)
|
|
|
|
|
|
def doctor() -> int:
|
|
modules = ["ezdxf", "jsonschema", "numpy", "scipy", "shapely", "trimesh", "matplotlib", "PIL", "pygltflib"]
|
|
checks = []
|
|
failed = False
|
|
for module in modules:
|
|
try:
|
|
version = importlib.metadata.version("pillow" if module == "PIL" else module)
|
|
checks.append({"name": module, "status": "PASS", "version": version})
|
|
except Exception as exc:
|
|
checks.append({"name": module, "status": "FAIL", "detail": str(exc)})
|
|
failed = True
|
|
for executable in ("node", "npm", "dwg2dxf"):
|
|
path = shutil.which(executable)
|
|
checks.append({"name": executable, "status": "PASS" if path else "OPTIONAL", "path": path})
|
|
for executable, resolver in (("ODAFileConverter", oda_converter), ("blender", default_blender)):
|
|
candidate = resolver()
|
|
path = candidate if candidate and Path(candidate).exists() else None
|
|
checks.append({"name": executable, "status": "PASS" if path else "OPTIONAL", "path": path})
|
|
print(json.dumps({"status": "FAIL" if failed else "PASS", "checks": checks}, ensure_ascii=False, indent=2))
|
|
return 3 if failed else 0
|
|
|
|
|
|
def inspect_command(args: argparse.Namespace) -> int:
|
|
command = python_script(
|
|
"cad_dxf_report.py",
|
|
"--input",
|
|
args.input,
|
|
"--work",
|
|
args.work,
|
|
"--output",
|
|
args.output,
|
|
"--vendor-dir",
|
|
args.vendor_dir,
|
|
"--converter",
|
|
args.converter,
|
|
)
|
|
return subprocess.run(command, cwd=ROOT).returncode
|
|
|
|
|
|
def build_with_trimesh(pipeline: Pipeline, bundle: Path) -> list[Path]:
|
|
glbs = []
|
|
for key in ("overview", "standard", "detail"):
|
|
spec = bundle / "variant_specs" / f"{key}.json"
|
|
out_dir = bundle / "variants" / key
|
|
glb = out_dir / f"{key}.glb"
|
|
metadata = out_dir / "model_metadata.json"
|
|
pipeline.run(
|
|
f"glb_{key}",
|
|
python_script(
|
|
"spec_to_glb.py",
|
|
"--spec",
|
|
str(spec),
|
|
"--out",
|
|
str(glb),
|
|
"--metadata-out",
|
|
str(metadata),
|
|
"--name",
|
|
key,
|
|
),
|
|
[glb, metadata],
|
|
forbidden_stderr=("WARNING mesh_from_polygons",),
|
|
)
|
|
glbs.append(glb)
|
|
return glbs
|
|
|
|
|
|
def run_pipeline(args: argparse.Namespace) -> int:
|
|
dxf_dir = Path(args.dxf_dir).resolve()
|
|
config = Path(args.config).resolve()
|
|
source_report = Path(args.source_report).resolve()
|
|
try:
|
|
report = json.loads(source_report.read_text(encoding="utf-8"))
|
|
showcase_schema.validate_source_report(report, dxf_dir)
|
|
except (OSError, json.JSONDecodeError, showcase_schema.ValidationFailure) as exc:
|
|
print(f"GATE FAIL [source_report]: {exc}", file=sys.stderr)
|
|
return 3
|
|
out_dir = Path(args.out_dir).resolve()
|
|
out_dir.mkdir(parents=True, exist_ok=True)
|
|
try:
|
|
waivers = parse_waivers(args.waiver)
|
|
except ValueError as exc:
|
|
print(f"GATE FAIL [waiver]: {exc}", file=sys.stderr)
|
|
return 3
|
|
pipeline = Pipeline(out_dir / "pipeline_state.json", fingerprint(config, source_report, dxf_dir), args.resume)
|
|
spec = out_dir / "building_spec.json"
|
|
audit = out_dir / "extraction_audit.json"
|
|
audit_md = out_dir / "extraction_audit.md"
|
|
overlays = out_dir / "overlays"
|
|
pipeline.run(
|
|
"derive",
|
|
python_script(
|
|
"derive_showcase_spec.py",
|
|
"--dxf-dir",
|
|
str(dxf_dir),
|
|
"--config",
|
|
str(config),
|
|
"--out",
|
|
str(spec),
|
|
"--vendor-dir",
|
|
args.vendor_dir,
|
|
),
|
|
[spec],
|
|
forbidden_stderr=("WARNING topless_group",),
|
|
)
|
|
audit_cmd = python_script(
|
|
"audit_showcase_extraction.py",
|
|
"--dxf-dir",
|
|
str(dxf_dir),
|
|
"--config",
|
|
str(config),
|
|
"--spec",
|
|
str(spec),
|
|
"--out",
|
|
str(audit),
|
|
"--vendor-dir",
|
|
args.vendor_dir,
|
|
)
|
|
if args.frames:
|
|
audit_cmd.extend(["--frames", *args.frames])
|
|
pipeline.run("audit", audit_cmd, [audit, audit_md])
|
|
if not enforce_audit_gates(pipeline, audit, waivers):
|
|
return 3
|
|
pipeline.run(
|
|
"overlays",
|
|
python_script(
|
|
"draw_coverage_overlay.py",
|
|
"--dxf-dir",
|
|
str(dxf_dir),
|
|
"--config",
|
|
str(config),
|
|
"--spec",
|
|
str(spec),
|
|
"--out-dir",
|
|
str(overlays),
|
|
"--vendor-dir",
|
|
args.vendor_dir,
|
|
),
|
|
[overlays],
|
|
)
|
|
|
|
bundle = out_dir / "delivery"
|
|
manifest = bundle / "variants_manifest.json"
|
|
variant_cmd = python_script(
|
|
"build_showcase_variants.py",
|
|
"--spec",
|
|
str(spec),
|
|
"--out-dir",
|
|
str(bundle),
|
|
"--name",
|
|
args.name,
|
|
)
|
|
if args.backend == "blender":
|
|
variant_cmd.append("--build")
|
|
if args.blender:
|
|
variant_cmd.extend(["--blender", args.blender])
|
|
pipeline.run("variants", variant_cmd, [manifest, bundle / "variant_specs" / "standard.json"])
|
|
if args.backend == "trimesh":
|
|
glbs = build_with_trimesh(pipeline, bundle)
|
|
else:
|
|
glbs = [bundle / "variants" / key / f"{key}.glb" for key in ("overview", "standard", "detail")]
|
|
|
|
web_dir = out_dir / "web"
|
|
if args.web:
|
|
pipeline.run(
|
|
"web",
|
|
python_script(
|
|
"write_web_showcase.py",
|
|
"--variants-manifest",
|
|
str(manifest),
|
|
"--out-dir",
|
|
str(web_dir),
|
|
"--title",
|
|
args.name,
|
|
),
|
|
[web_dir / "index.html", web_dir / "model.glb", web_dir / "model_metadata.json"],
|
|
)
|
|
if not args.skip_web_deps:
|
|
pipeline.run(
|
|
"web_dependencies",
|
|
["npm", "install", "--prefix", str(web_dir), "--ignore-scripts"],
|
|
[web_dir / "node_modules" / "three" / "build" / "three.module.js"],
|
|
)
|
|
browser_report = (
|
|
Path(args.browser_report).resolve() if args.browser_report else out_dir / "browser_smoke_report.json"
|
|
)
|
|
run_browser_smoke(pipeline, web_dir, browser_report, out_dir / "browser_smoke")
|
|
|
|
standard = bundle / "variants" / "standard" / "standard.glb"
|
|
offline = out_dir / "viewer.html"
|
|
if args.offline:
|
|
pipeline.run(
|
|
"offline",
|
|
python_script(
|
|
"write_offline_viewer.py",
|
|
"--spec",
|
|
str(spec),
|
|
"--glb",
|
|
str(standard),
|
|
"--out",
|
|
str(offline),
|
|
"--title",
|
|
args.name,
|
|
),
|
|
[offline],
|
|
)
|
|
|
|
preview = out_dir / "verify_previews.png"
|
|
pipeline.run(
|
|
"preview",
|
|
python_script(
|
|
"verify_showcase.py",
|
|
"--spec",
|
|
str(spec),
|
|
"--glb",
|
|
*[str(path) for path in glbs],
|
|
"--out",
|
|
str(preview),
|
|
"--budget-mb",
|
|
str(args.budget_mb),
|
|
"--strict",
|
|
),
|
|
[preview],
|
|
)
|
|
evidence = out_dir / "evidence_manifest.json"
|
|
verify_cmd = python_script(
|
|
"verify_delivery.py",
|
|
"--spec",
|
|
str(spec),
|
|
"--source-report",
|
|
str(source_report),
|
|
"--variants-manifest",
|
|
str(manifest),
|
|
"--render",
|
|
str(preview),
|
|
"--audit",
|
|
str(audit),
|
|
"--budget-mb",
|
|
str(args.budget_mb),
|
|
"--evidence-out",
|
|
str(evidence),
|
|
)
|
|
if args.web:
|
|
verify_cmd.extend(
|
|
["--web-dir", str(web_dir), "--browser-report", str(browser_report), "--require-browser-report"]
|
|
)
|
|
if args.offline:
|
|
verify_cmd.extend(["--offline-viewer", str(offline)])
|
|
if args.require_browser_report and not args.web:
|
|
verify_cmd.append("--require-browser-report")
|
|
if args.scope_manifest:
|
|
verify_cmd.extend(["--scope-manifest", str(Path(args.scope_manifest).resolve())])
|
|
for value in args.waiver:
|
|
verify_cmd.extend(["--waiver", value])
|
|
pipeline.run("delivery_verification", verify_cmd, [evidence])
|
|
verification = json.loads(evidence.read_text(encoding="utf-8"))
|
|
if verification.get("status") != "PASS":
|
|
pipeline.state["status"] = "BLOCKED"
|
|
pipeline.save()
|
|
print(f"GATE BLOCKED [delivery_verification]: {verification.get('status')}", file=sys.stderr)
|
|
return 3
|
|
pipeline.state["status"] = "PASS"
|
|
pipeline.state["finished_at"] = now()
|
|
pipeline.save()
|
|
print(json.dumps({"status": "PASS", "out_dir": str(out_dir), "evidence": str(evidence)}, ensure_ascii=False))
|
|
return 0
|
|
|
|
|
|
def parser() -> argparse.ArgumentParser:
|
|
root = argparse.ArgumentParser(description=__doc__)
|
|
sub = root.add_subparsers(dest="command", required=True)
|
|
sub.add_parser("doctor", help="Check Python, Node, CAD converter, and Blender prerequisites")
|
|
inspect = sub.add_parser("inspect", help="Convert/read a CAD folder and enforce the source gate")
|
|
inspect.add_argument("--input", required=True)
|
|
inspect.add_argument("--work", required=True)
|
|
inspect.add_argument("--output", required=True)
|
|
inspect.add_argument("--vendor-dir", default="work/vendor")
|
|
inspect.add_argument("--converter", choices=("auto", "libredwg", "oda"), default="auto")
|
|
run = sub.add_parser("run", help="Run extract -> audit -> overlays -> variants -> verify from an approved config")
|
|
run.add_argument("--dxf-dir", required=True)
|
|
run.add_argument("--config", required=True)
|
|
run.add_argument("--source-report", required=True, help="Passing cad_read_report.json for this sanitized DXF dir")
|
|
run.add_argument("--out-dir", required=True)
|
|
run.add_argument("--name", default="cad_building_showcase")
|
|
run.add_argument("--vendor-dir", default="work/vendor")
|
|
run.add_argument("--frames", nargs="*", default=[])
|
|
run.add_argument("--backend", choices=("trimesh", "blender"), default="trimesh")
|
|
run.add_argument("--blender")
|
|
run.add_argument("--web", action="store_true")
|
|
run.add_argument("--skip-web-deps", action="store_true")
|
|
run.add_argument("--offline", action="store_true")
|
|
run.add_argument("--browser-report")
|
|
run.add_argument("--require-browser-report", action="store_true")
|
|
run.add_argument("--scope-manifest", help="Single-building/campus scope ledger included in final verification")
|
|
run.add_argument(
|
|
"--waiver",
|
|
action="append",
|
|
default=[],
|
|
metavar="CHECK=REASON",
|
|
help="Explicitly acknowledge one WARN/SKIP audit or delivery check",
|
|
)
|
|
run.add_argument("--budget-mb", type=float, default=25.0)
|
|
run.add_argument("--resume", action="store_true")
|
|
return root
|
|
|
|
|
|
def main() -> int:
|
|
args = parser().parse_args()
|
|
if args.command == "doctor":
|
|
return doctor()
|
|
if args.command == "inspect":
|
|
return inspect_command(args)
|
|
return run_pipeline(args)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
raise SystemExit(main())
|