From 78ec1fbdcccf88ce776faa4b42b6652da3e88975 Mon Sep 17 00:00:00 2001 From: zhengwr Date: Fri, 10 Jul 2026 12:38:12 +0800 Subject: [PATCH] fix: enforce strict CAD showcase delivery gates --- SKILL.md | 19 +-- references/campus_workflow.md | 28 ++++- references/web_delivery.md | 8 +- requirements-core.txt | 1 + scripts/cad_showcase.py | 160 ++++++++++++++++++++++++- scripts/verify_delivery.py | 214 ++++++++++++++++++++++++++++++++-- tests/test_contracts.py | 8 ++ tests/test_delivery.py | 127 ++++++++++++++++++++ 8 files changed, 537 insertions(+), 28 deletions(-) diff --git a/SKILL.md b/SKILL.md index 919969d..6508f1d 100644 --- a/SKILL.md +++ b/SKILL.md @@ -15,6 +15,7 @@ Produce an evidence-backed architectural showcase, not a blind CAD conversion. D - overview/standard/detail GLBs plus metadata; - Blender scene/preview when Blender is selected; - served Web viewer and/or offline single-file viewer; +- `scope_manifest.json` accounting for every requested building and the site; - `evidence_manifest.json` with machine-readable final checks; - notes stating assumptions, missing geometry, source limitations, and confidence level. @@ -44,6 +45,7 @@ Read only the branches needed for the current task: - Use standard line extraction for architectural DWG/DXF plans. - Use `detect_sheet_frames.py` for multi-frame sheets. - Model each building separately before campus assembly. + - For multi-building/campus requests, write `scope_manifest.json` before modeling. Copy every requested building ID from the user/source inventory; never shrink scope to the easiest drawing set. - Use `--converter oda` plus `extract_3d_solids.py` for known 3DSOLID/ACIS drawings. - Stop and request DWG/DXF/vector conversion for PDF-only input. @@ -57,12 +59,13 @@ Read only the branches needed for the current task: `python3 scripts/cad_showcase.py run --dxf-dir --config --source-report --out-dir --name --frames --backend trimesh`. - Use `--backend blender` when `.blend` and Blender preview are required. - Never pass `--no-strict-gates` to proceed toward delivery. + - Treat audit `WARN` and `SKIP` as `BLOCKED`, not success. Continue only with an explicit `--waiver 'audit:='`; retain it in final evidence. - Open the SVG overlays. Iterate if red/unclassified geometry forms walls, glazing, columns, beams, grids, stairs, slabs, roofs, or whole building wings. 5. Verify actual deliverables. - Require `verify_showcase.py --strict` and `verify_delivery.py` to pass. - Inspect at least one iso/front/side/plan render personally. - - For served Web output, run `web_smoke_test.mjs` at desktop and 390px widths and feed its report to `verify_delivery.py --browser-report`. + - For served Web output, use `cad_showcase.py run --web`; it starts a temporary local server, runs `web_smoke_test.mjs` at desktop and 390px widths, and feeds the report to final verification automatically. - Confirm idle `window.__renders` does not grow, GLBs decode, variants exist, controls work, and console errors are zero. 6. Report confidence and limits. @@ -77,12 +80,13 @@ Do not hand over a model unless every applicable item has evidence: 2. Config and spec pass their versioned JSON Schemas. 3. Every configured source exists, every floor key is unique, every floor is nonempty, and `floor_order` has no gaps/ghosts. 4. Multi-frame projects pass `frame_ledger`; every detected plan frame is used or appears in `discarded_frames` with a reason. -5. Audit exits 0 and all required gates pass: source presence, floor uniqueness/nonempty, floor alignment/cross-floor datum, glazing capture, annotation exclusion, declared areas, and storey count when armed. +5. Audit exits 0 and every gate is `PASS`; `WARN`/`SKIP` block delivery unless a named, reasoned waiver is recorded in evidence. 6. Derive emits no `WARNING topless_group`; GLB build emits no `WARNING mesh_from_polygons`. -7. Campus output has proven transforms, no unexplained overlap, numeric neighbour gaps, boundary containment, and a top-view placement image. -8. `verify_delivery.py` writes `evidence_manifest.json` with overall `PASS`; a human has looked at the render. -9. Served Web delivery includes a passing browser smoke report when the environment permits browser testing. -10. After any script/schema/template change, run: +7. Campus output has a `scope_manifest.json` containing every requested building. Every building and the site is `PASS`, with existing artifacts; a `BLOCKED` entry means the campus request is not complete. +8. Campus output has proven transforms, no unexplained overlap, numeric neighbour gaps, boundary containment, and a top-view placement image. +9. `verify_delivery.py` writes `evidence_manifest.json` with overall `PASS`; `BLOCKED`/`FAIL` must never be described as complete, and a human has looked at the render. +10. Served Web delivery includes a passing desktop/mobile browser smoke report. If Chromium cannot run, report `BLOCKED`; do not substitute the offline viewer for the requested Web template. +11. After any script/schema/template change, run: - `pytest tests/test_contracts.py tests/test_delivery.py` - `python3 tests/test_extraction_pipeline.py` - `ruff format --check scripts tests && ruff check scripts tests` @@ -113,7 +117,7 @@ python3 scripts/cad_showcase.py run \ --source-report outputs//cad_read_report/cad_read_report.json \ --frames work//frames_config.json \ --out-dir outputs/ --name \ - --backend trimesh --web --offline --resume + --backend trimesh --web --offline --scope-manifest work//scope_manifest.json --resume ``` For a known ACIS/3DSOLID source, force ODA during inspection; do not let an apparently nonempty libredwg conversion hide dropped solids: @@ -131,6 +135,7 @@ python3 scripts/extract_3d_solids.py \ - Prefer stable layered geometry over overfitted CAD noise. - Preserve floor/category naming as `__` and glTF Y-up. - Keep source originals unchanged; write intermediates under `work/` and deliveries under `outputs/`. +- Start each independent attempt in a new project directory. Reusing a prior directory requires an explicit user request and a recorded provenance note. - Use CAD-derived footprint polygons for slabs/roofs when evidence is sufficient; do not present generated caps, hulls, mullions, wall unions, or massings as BIM-verified geometry. - Keep slabs visible for presentation, grids/details toggleable, and in-scene labels off by default. - Never claim rooms, openings, quantities, materials, MEP coordination, code compliance, or production BIM unless the IFC branch explicitly generates and validates them. diff --git a/references/campus_workflow.md b/references/campus_workflow.md index 8ea05e5..5950d0d 100644 --- a/references/campus_workflow.md +++ b/references/campus_workflow.md @@ -3,10 +3,11 @@ ## Contents 1. Branch selection -2. Placement evidence -3. Site outlines and missing buildings -4. Assembly gates -5. Site context +2. Scope contract +3. Placement evidence +4. Site outlines and missing buildings +5. Assembly gates +6. Site context ## Branch Selection @@ -14,6 +15,25 @@ Treat folders such as `1#厂房 / 2#厂房 / 3#辅房 / 总图` as a campus, not Building plans are usually millimeters; total/site plans may be meters at surveying coordinates. Never combine both in one extraction config. Convert everything to site-local meters before GLB export; raw coordinates hundreds of kilometers from origin lose float32 precision. +## Scope Contract + +Before modeling, write `scope_manifest.json` from the user request and source inventory. Do not derive requested scope from whichever config or GLB happened to succeed. Minimum shape: + +```json +{ + "scope_version": 1, + "mode": "campus", + "requested": ["1#", "2#", "3#"], + "buildings": [ + {"id": "1#", "status": "PASS", "artifacts": ["outputs/1/model.glb"]}, + {"id": "2#", "status": "BLOCKED", "reason": "unresolved XREF", "artifacts": []} + ], + "site": {"status": "PASS", "artifacts": ["outputs/site_top.png", "outputs/site_manifest.json"]} +} +``` + +Every `requested` ID must have exactly one ledger entry. `PASS` requires at least one existing non-empty artifact. `BLOCKED` requires a concrete reason and makes the overall campus result `BLOCKED`, not complete. The campus site must likewise be `PASS` with artifacts. Feed the ledger to `cad_showcase.py run --scope-manifest ...` or `verify_delivery.py --scope-manifest ...`. + ## Placement Evidence Prefer anchors in this order: diff --git a/references/web_delivery.md b/references/web_delivery.md index 9d39b65..4e5d6c6 100644 --- a/references/web_delivery.md +++ b/references/web_delivery.md @@ -36,7 +36,7 @@ The offline viewer intentionally follows an older Three API. Keep its vendored l ## Browser Smoke Test -Serve the Web directory locally, then run: +`cad_showcase.py run --web` performs this gate automatically. For a manual/custom delivery, serve the Web directory locally, then run: ```bash node scripts/web_smoke_test.mjs \ @@ -45,7 +45,9 @@ node scripts/web_smoke_test.mjs \ --screenshots-dir outputs//browser_smoke ``` -Install Playwright/Chromium first when needed. The script checks desktop/mobile control visibility, console/page/request failures, and idle render behavior. Feed the report to final verification with `--browser-report`; use `--require-browser-report` for formal Web delivery. +Install Playwright/Chromium first when needed. The script checks desktop/mobile control visibility, console/page/request failures, and idle render behavior. Feed the report to final verification with `--browser-report`. Supplying `--web-dir` makes the report mandatory: no report is `BLOCKED`, never PASS. + +When an execution tool already has a `run_in_background` option, launch `python3 -m http.server ...` as the foreground command of that background process. Do not append shell `&`; the child server can die with the wrapper. Before presenting the viewer, probe the URL and keep that server process alive. Present the formal `web/index.html` URL when Web was requested; `viewer.html` is a separate offline fallback, not a substitute. ## Strict Evidence Manifest @@ -64,4 +66,4 @@ python3 scripts/verify_delivery.py \ --evidence-out outputs//evidence_manifest.json ``` -The verifier checks schema versions, GLB headers/mesh structure/naming/budgets, render dimensions/nonblank variance, Web assets/placeholders/features/variants, offline payload, and browser report. Treat overall FAIL as a delivery blocker. +The verifier checks schema versions, audit gates, optional campus scope, GLB headers/mesh structure/naming/budgets, render dimensions/nonblank variance, Web assets/placeholders/features/variants, offline payload, and browser report. Only overall `PASS` is complete. `WARN`/`SKIP` produce overall `BLOCKED` unless the exact check has an explicit `--waiver 'CHECK=REASON'`; `FAIL` cannot be waived. diff --git a/requirements-core.txt b/requirements-core.txt index 5426894..8c81f9a 100644 --- a/requirements-core.txt +++ b/requirements-core.txt @@ -2,5 +2,6 @@ ezdxf>=1.4,<2 jsonschema>=4.20,<5 mapbox-earcut>=1.0,<2 numpy>=1.26,<3 +scipy>=1.13,<2 shapely>=2.0,<3 trimesh>=4.5,<5 diff --git a/scripts/cad_showcase.py b/scripts/cad_showcase.py index 03312ba..12c55c4 100644 --- a/scripts/cad_showcase.py +++ b/scripts/cad_showcase.py @@ -8,8 +8,11 @@ import hashlib import importlib.metadata import json import shutil +import socket import subprocess import sys +import time +import urllib.request from datetime import datetime, timezone from pathlib import Path from typing import Any @@ -32,8 +35,10 @@ def fingerprint(config: Path, source_report: Path, dxf_dir: Path) -> str: digest = hashlib.sha256() tracked = [config, source_report, *sorted(dxf_dir.glob("*.dxf"))] tracked.extend(sorted(SCRIPTS.glob("*.py"))) + tracked.extend(sorted(SCRIPTS.glob("*.mjs"))) tracked.extend(sorted((ROOT / "schemas").glob("*.json"))) tracked.extend(sorted((ROOT / "assets" / "web-showcase-template").glob("*"))) + tracked.extend(path for path in (ROOT / "package.json", ROOT / "package-lock.json") if path.exists()) for path in tracked: if not path.is_file(): continue @@ -132,8 +137,122 @@ def python_script(name: str, *args: str) -> list[str]: return [sys.executable, str(SCRIPTS / name), *args] +def parse_waivers(values: list[str]) -> dict[str, str]: + waivers: dict[str, str] = {} + for value in values: + name, separator, reason = value.partition("=") + name = name.strip() + reason = reason.strip() + if not separator or not name or not reason: + raise ValueError(f"Invalid waiver {value!r}; expected CHECK=REASON") + if name in waivers: + raise ValueError(f"Duplicate waiver for {name!r}") + waivers[name] = reason + return waivers + + +def enforce_audit_gates(pipeline: Pipeline, audit_path: Path, waivers: dict[str, str]) -> bool: + report = json.loads(audit_path.read_text(encoding="utf-8")) + blocked = [] + waived = [] + for gate in report.get("gates", []): + status = str(gate.get("status", "FAIL")) + if status == "PASS": + continue + name = f"audit:{gate.get('name', 'unnamed')}" + if status in {"WARN", "SKIP"} and name in waivers: + waived.append({"name": name, "original_status": status, "reason": waivers[name]}) + else: + blocked.append({"name": name, "status": status, "detail": gate.get("detail", "")}) + pipeline.state["audit_waivers"] = waived + if blocked: + pipeline.state["status"] = "BLOCKED" + pipeline.state["blocked_gates"] = blocked + pipeline.save() + for gate in blocked: + print(f"GATE BLOCKED [{gate['name']}]: {gate['detail']}", file=sys.stderr) + return False + pipeline.save() + return True + + +def available_port() -> int: + with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as stream: + stream.bind(("127.0.0.1", 0)) + return int(stream.getsockname()[1]) + + +def wait_for_http(url: str, timeout: float = 20.0) -> None: + deadline = time.monotonic() + timeout + last_error: Exception | None = None + while time.monotonic() < deadline: + try: + with urllib.request.urlopen(url, timeout=1) as response: + if response.status < 500: + return + except Exception as exc: # pragma: no cover - timing depends on host + last_error = exc + time.sleep(0.2) + raise RuntimeError(f"Local Web server did not become ready: {last_error}") + + +def run_browser_smoke(pipeline: Pipeline, web_dir: Path, report: Path, screenshots: Path) -> None: + """Serve the formal Web bundle and mechanically prove desktop/mobile loading.""" + pipeline.run( + "browser_dependencies", + ["npm", "install", "--prefix", str(ROOT), "--ignore-scripts"], + [ROOT / "node_modules" / "playwright" / "package.json"], + ) + pipeline.run( + "browser_runtime", + [str(ROOT / "node_modules" / ".bin" / "playwright"), "install", "chromium"], + [], + ) + report.parent.mkdir(parents=True, exist_ok=True) + port = available_port() + url = f"http://127.0.0.1:{port}/" + server = subprocess.Popen( + [ + sys.executable, + "-m", + "http.server", + str(port), + "--bind", + "127.0.0.1", + "--directory", + str(web_dir), + ], + cwd=ROOT, + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + ) + try: + wait_for_http(url) + pipeline.run( + "browser_smoke", + [ + "node", + str(SCRIPTS / "web_smoke_test.mjs"), + "--url", + url, + "--out", + str(report), + "--screenshots-dir", + str(screenshots), + ], + [report, screenshots], + ) + finally: + server.terminate() + try: + server.wait(timeout=5) + except subprocess.TimeoutExpired: # pragma: no cover - defensive cleanup + server.kill() + server.wait(timeout=5) + + def doctor() -> int: - modules = ["ezdxf", "jsonschema", "numpy", "shapely", "trimesh", "matplotlib", "PIL", "pygltflib"] + modules = ["ezdxf", "jsonschema", "numpy", "scipy", "shapely", "trimesh", "matplotlib", "PIL", "pygltflib"] checks = [] failed = False for module in modules: @@ -210,6 +329,11 @@ def run_pipeline(args: argparse.Namespace) -> int: return 3 out_dir = Path(args.out_dir).resolve() out_dir.mkdir(parents=True, exist_ok=True) + try: + waivers = parse_waivers(args.waiver) + except ValueError as exc: + print(f"GATE FAIL [waiver]: {exc}", file=sys.stderr) + return 3 pipeline = Pipeline(out_dir / "pipeline_state.json", fingerprint(config, source_report, dxf_dir), args.resume) spec = out_dir / "building_spec.json" audit = out_dir / "extraction_audit.json" @@ -247,6 +371,8 @@ def run_pipeline(args: argparse.Namespace) -> int: if args.frames: audit_cmd.extend(["--frames", *args.frames]) pipeline.run("audit", audit_cmd, [audit, audit_md]) + if not enforce_audit_gates(pipeline, audit, waivers): + return 3 pipeline.run( "overlays", python_script( @@ -307,6 +433,10 @@ def run_pipeline(args: argparse.Namespace) -> int: ["npm", "install", "--prefix", str(web_dir), "--ignore-scripts"], [web_dir / "node_modules" / "three" / "build" / "three.module.js"], ) + browser_report = ( + Path(args.browser_report).resolve() if args.browser_report else out_dir / "browser_smoke_report.json" + ) + run_browser_smoke(pipeline, web_dir, browser_report, out_dir / "browser_smoke") standard = bundle / "variants" / "standard" / "standard.glb" offline = out_dir / "viewer.html" @@ -355,20 +485,32 @@ def run_pipeline(args: argparse.Namespace) -> int: str(manifest), "--render", str(preview), + "--audit", + str(audit), "--budget-mb", str(args.budget_mb), "--evidence-out", str(evidence), ) if args.web: - verify_cmd.extend(["--web-dir", str(web_dir)]) + verify_cmd.extend( + ["--web-dir", str(web_dir), "--browser-report", str(browser_report), "--require-browser-report"] + ) if args.offline: verify_cmd.extend(["--offline-viewer", str(offline)]) - if args.browser_report: - verify_cmd.extend(["--browser-report", args.browser_report]) - if args.require_browser_report: + if args.require_browser_report and not args.web: verify_cmd.append("--require-browser-report") + if args.scope_manifest: + verify_cmd.extend(["--scope-manifest", str(Path(args.scope_manifest).resolve())]) + for value in args.waiver: + verify_cmd.extend(["--waiver", value]) pipeline.run("delivery_verification", verify_cmd, [evidence]) + verification = json.loads(evidence.read_text(encoding="utf-8")) + if verification.get("status") != "PASS": + pipeline.state["status"] = "BLOCKED" + pipeline.save() + print(f"GATE BLOCKED [delivery_verification]: {verification.get('status')}", file=sys.stderr) + return 3 pipeline.state["status"] = "PASS" pipeline.state["finished_at"] = now() pipeline.save() @@ -401,6 +543,14 @@ def parser() -> argparse.ArgumentParser: run.add_argument("--offline", action="store_true") run.add_argument("--browser-report") run.add_argument("--require-browser-report", action="store_true") + run.add_argument("--scope-manifest", help="Single-building/campus scope ledger included in final verification") + run.add_argument( + "--waiver", + action="append", + default=[], + metavar="CHECK=REASON", + help="Explicitly acknowledge one WARN/SKIP audit or delivery check", + ) run.add_argument("--budget-mb", type=float, default=25.0) run.add_argument("--resume", action="store_true") return root diff --git a/scripts/verify_delivery.py b/scripts/verify_delivery.py index d362f30..a441234 100644 --- a/scripts/verify_delivery.py +++ b/scripts/verify_delivery.py @@ -29,6 +29,48 @@ def add_check(checks: list[dict[str, Any]], name: str, status: str, detail: str, checks.append({"name": name, "status": status, "detail": detail, **data}) +def parse_waivers(values: list[str]) -> dict[str, str]: + """Parse explicit CHECK=REASON waivers. + + Waivers are deliberately verbose: an Agent cannot silently turn WARN/SKIP + into PASS, and a misspelled or unused waiver becomes a hard failure. + """ + waivers: dict[str, str] = {} + for value in values: + name, separator, reason = value.partition("=") + name = name.strip() + reason = reason.strip() + if not separator or not name or not reason: + raise ValueError(f"Invalid waiver {value!r}; expected CHECK=REASON") + if name in waivers: + raise ValueError(f"Duplicate waiver for {name!r}") + waivers[name] = reason + return waivers + + +def apply_waivers(checks: list[dict[str, Any]], waivers: dict[str, str]) -> None: + used: set[str] = set() + for check in checks: + name = str(check["name"]) + if name not in waivers: + continue + used.add(name) + if check["status"] not in {"WARN", "SKIP"}: + add_check( + checks, + f"waiver:{name}", + "FAIL", + f"Waivers may only acknowledge WARN/SKIP checks; {name} is {check['status']}", + ) + continue + check["original_status"] = check["status"] + check["status"] = "PASS" + check["waiver_reason"] = waivers[name] + check["detail"] = f"WAIVED ({waivers[name]}): {check['detail']}" + for name in sorted(set(waivers) - used): + add_check(checks, f"waiver:{name}", "FAIL", f"Unused or misspelled waiver: {name}") + + def inspect_glb(path: Path, budget_mb: float, checks: list[dict[str, Any]]) -> dict[str, Any] | None: if not path.exists() or path.stat().st_size == 0: add_check(checks, f"glb:{path.name}", "FAIL", f"Missing or empty GLB: {path}") @@ -162,13 +204,144 @@ def inspect_offline(path: Path, checks: list[dict[str, Any]]) -> None: def inspect_browser_report(path: Path, checks: list[dict[str, Any]]) -> None: try: report = json.loads(path.read_text(encoding="utf-8")) - passed = report.get("status") == "PASS" and not report.get("console_errors") + passed = ( + report.get("status") == "PASS" + and not report.get("console_errors") + and not report.get("page_errors") + and not report.get("failed_requests") + ) detail = report.get("detail") or json.dumps(report, ensure_ascii=False)[:500] add_check(checks, "browser:smoke", "PASS" if passed else "FAIL", detail) except Exception as exc: add_check(checks, "browser:smoke", "FAIL", f"Browser report invalid: {exc}") +def inspect_audit(path: Path, checks: list[dict[str, Any]]) -> None: + try: + report = json.loads(path.read_text(encoding="utf-8")) + gates = report.get("gates") + if not isinstance(gates, list) or not gates: + raise ValueError("audit has no gates") + for gate in gates: + status = str(gate.get("status", "FAIL")) + if status not in {"PASS", "WARN", "SKIP", "FAIL"}: + status = "FAIL" + add_check( + checks, + f"audit:{gate.get('name', 'unnamed')}", + status, + str(gate.get("detail", "Audit gate has no detail")), + ) + except Exception as exc: + add_check(checks, "audit:report", "FAIL", f"Extraction audit invalid: {exc}") + + +def inspect_scope(path: Path, checks: list[dict[str, Any]]) -> None: + """Validate a single-building/campus scope ledger against real artifacts.""" + try: + scope = json.loads(path.read_text(encoding="utf-8")) + if scope.get("scope_version") != 1: + raise ValueError("scope_version must be 1") + mode = scope.get("mode") + if mode not in {"single", "campus"}: + raise ValueError("mode must be single or campus") + requested = scope.get("requested") + buildings = scope.get("buildings") + if ( + not isinstance(requested, list) + or not requested + or not all(isinstance(item, str) and item for item in requested) + ): + raise ValueError("requested must be a non-empty list of building IDs") + if len(set(requested)) != len(requested): + raise ValueError("requested contains duplicate building IDs") + if not isinstance(buildings, list): + raise ValueError("buildings must be a list") + by_id: dict[str, dict[str, Any]] = {} + for item in buildings: + building_id = str(item.get("id", "")) + if not building_id or building_id in by_id: + raise ValueError(f"missing or duplicate building id: {building_id!r}") + by_id[building_id] = item + missing = [item for item in requested if item not in by_id] + if missing: + add_check( + checks, "scope:inventory", "FAIL", "Requested buildings missing from ledger: " + ", ".join(missing) + ) + else: + add_check( + checks, "scope:inventory", "PASS", f"All {len(requested)} requested building(s) are in the ledger" + ) + + for building_id in requested: + item = by_id.get(building_id) + if not item: + continue + status = item.get("status") + reason = str(item.get("reason", "")).strip() + artifacts = item.get("artifacts", []) + if status == "BLOCKED": + add_check( + checks, + f"scope:building:{building_id}", + "WARN" if reason else "FAIL", + reason or "BLOCKED building has no reason", + ) + continue + if status != "PASS": + add_check(checks, f"scope:building:{building_id}", "FAIL", f"Invalid status: {status!r}") + continue + if not isinstance(artifacts, list) or not artifacts: + add_check(checks, f"scope:building:{building_id}", "FAIL", "PASS building has no artifacts") + continue + missing_artifacts = [] + for artifact in artifacts: + artifact_path = Path(str(artifact)) + if not artifact_path.is_absolute(): + artifact_path = path.parent / artifact_path + if not artifact_path.exists() or (artifact_path.is_file() and artifact_path.stat().st_size == 0): + missing_artifacts.append(str(artifact)) + add_check( + checks, + f"scope:building:{building_id}", + "FAIL" if missing_artifacts else "PASS", + "Missing artifacts: " + ", ".join(missing_artifacts) + if missing_artifacts + else f"{len(artifacts)} declared artifact(s) exist", + ) + + site = scope.get("site", {}) + if mode == "campus": + site_status = site.get("status") if isinstance(site, dict) else None + if site_status == "BLOCKED": + reason = str(site.get("reason", "")).strip() + add_check(checks, "scope:site", "WARN" if reason else "FAIL", reason or "Campus site is blocked") + elif site_status != "PASS": + add_check(checks, "scope:site", "FAIL", "Campus scope requires site.status=PASS or BLOCKED") + else: + artifacts = site.get("artifacts", []) + missing_site = [] + for artifact in artifacts if isinstance(artifacts, list) else []: + artifact_path = Path(str(artifact)) + if not artifact_path.is_absolute(): + artifact_path = path.parent / artifact_path + if not artifact_path.exists() or (artifact_path.is_file() and artifact_path.stat().st_size == 0): + missing_site.append(str(artifact)) + if not artifacts or missing_site: + add_check( + checks, + "scope:site", + "FAIL", + "Campus site has no artifacts" + if not artifacts + else "Missing site artifacts: " + ", ".join(missing_site), + ) + else: + add_check(checks, "scope:site", "PASS", f"{len(artifacts)} site artifact(s) exist") + except Exception as exc: + add_check(checks, "scope:manifest", "FAIL", f"Scope manifest invalid: {exc}") + + def main() -> int: parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("--spec", required=True) @@ -178,14 +351,28 @@ def main() -> int: parser.add_argument("--web-dir") parser.add_argument("--offline-viewer") parser.add_argument("--render", nargs="*", default=[]) + parser.add_argument("--audit") + parser.add_argument("--scope-manifest") parser.add_argument("--browser-report") parser.add_argument("--require-browser-report", action="store_true") + parser.add_argument( + "--waiver", + action="append", + default=[], + metavar="CHECK=REASON", + help="Explicitly acknowledge one WARN/SKIP check with a non-empty reason", + ) parser.add_argument("--budget-mb", type=float, default=25.0) parser.add_argument("--evidence-out", default="evidence_manifest.json") args = parser.parse_args() checks: list[dict[str, Any]] = [] artifacts: list[dict[str, Any]] = [] + try: + waivers = parse_waivers(args.waiver) + except ValueError as exc: + print(f"GATE FAIL [waiver]: {exc}", file=sys.stderr) + return 3 try: source_report = json.loads(Path(args.source_report).read_text(encoding="utf-8")) showcase_schema.validate_source_report(source_report) @@ -250,16 +437,25 @@ def main() -> int: add_check(checks, "render", "FAIL", "At least one inspected render is required") if args.web_dir: inspect_web(Path(args.web_dir), checks) + if args.audit: + inspect_audit(Path(args.audit), checks) + if args.scope_manifest: + inspect_scope(Path(args.scope_manifest), checks) if args.offline_viewer: inspect_offline(Path(args.offline_viewer), checks) if args.browser_report: inspect_browser_report(Path(args.browser_report), checks) - elif args.require_browser_report: - add_check(checks, "browser:smoke", "FAIL", "Browser smoke report required but not supplied") - else: - add_check(checks, "browser:smoke", "WARN", "No browser smoke report supplied") + elif args.web_dir or args.require_browser_report: + add_check(checks, "browser:smoke", "SKIP", "Formal Web delivery has no browser smoke report") - status = "FAIL" if any(item["status"] == "FAIL" for item in checks) else "PASS" + apply_waivers(checks, waivers) + status = ( + "FAIL" + if any(item["status"] == "FAIL" for item in checks) + else "BLOCKED" + if any(item["status"] in {"WARN", "SKIP"} for item in checks) + else "PASS" + ) evidence = { "evidence_version": 1, "generated_at": datetime.now(timezone.utc).isoformat(), @@ -271,10 +467,10 @@ def main() -> int: out.parent.mkdir(parents=True, exist_ok=True) out.write_text(json.dumps(evidence, ensure_ascii=False, indent=2), encoding="utf-8") print(json.dumps({"status": status, "evidence": str(out), "checks": len(checks)}, ensure_ascii=False)) - if status == "FAIL": + if status != "PASS": for item in checks: - if item["status"] == "FAIL": - print(f"GATE FAIL [{item['name']}]: {item['detail']}", file=sys.stderr) + if item["status"] in {"FAIL", "WARN", "SKIP"}: + print(f"GATE {status} [{item['name']}]: {item['detail']}", file=sys.stderr) return 3 return 0 diff --git a/tests/test_contracts.py b/tests/test_contracts.py index b808a0a..e2c7880 100644 --- a/tests/test_contracts.py +++ b/tests/test_contracts.py @@ -293,6 +293,14 @@ def test_orchestrator_runs_resumable_trimesh_pipeline(tmp_path: Path) -> None: str(out), "--name", "orchestrator-test", + "--waiver", + "audit:glazing_captured=Synthetic fixture contains no glazing evidence", + "--waiver", + "audit:frame_ledger=Synthetic fixture does not exercise sheet-frame detection", + "--waiver", + "audit:area_declared=Synthetic fixture has no independent declared-area evidence", + "--waiver", + "audit:storey_count=Synthetic fixture has no independent storey-count evidence", "--budget-mb", "5", ] diff --git a/tests/test_delivery.py b/tests/test_delivery.py index 936b0f9..eca29b9 100644 --- a/tests/test_delivery.py +++ b/tests/test_delivery.py @@ -156,9 +156,136 @@ def test_web_writer_and_static_delivery_checks(tmp_path: Path) -> None: "--evidence-out", str(tmp_path / "web_evidence.json"), ) + assert proc.returncode == 3 + assert "browser:smoke" in proc.stderr + browser_report = tmp_path / "browser_smoke_report.json" + browser_report.write_text( + json.dumps( + { + "status": "PASS", + "detail": "12/12 checks passed", + "console_errors": [], + "page_errors": [], + "failed_requests": [], + } + ), + encoding="utf-8", + ) + proc = run( + sys.executable, + "scripts/verify_delivery.py", + "--spec", + str(spec), + "--source-report", + str(source_report), + "--glb", + str(glb), + "--render", + str(render), + "--web-dir", + str(web), + "--browser-report", + str(browser_report), + "--evidence-out", + str(tmp_path / "web_evidence.json"), + ) assert proc.returncode == 0, proc.stderr +def test_warn_or_skip_blocks_without_explicit_reasoned_waiver(tmp_path: Path) -> None: + spec, glb, _metadata = build_fixture(tmp_path) + render = tmp_path / "render.png" + render_fixture(render) + source_report = source_report_fixture(tmp_path) + evidence = tmp_path / "evidence.json" + proc = run( + sys.executable, + "scripts/verify_delivery.py", + "--spec", + str(spec), + "--source-report", + str(source_report), + "--glb", + str(glb), + "--render", + str(render), + "--require-browser-report", + "--evidence-out", + str(evidence), + ) + assert proc.returncode == 3 + assert json.loads(evidence.read_text(encoding="utf-8"))["status"] == "BLOCKED" + proc = run( + sys.executable, + "scripts/verify_delivery.py", + "--spec", + str(spec), + "--source-report", + str(source_report), + "--glb", + str(glb), + "--render", + str(render), + "--require-browser-report", + "--waiver", + "browser:smoke=No browser is available in this offline-only benchmark", + "--evidence-out", + str(evidence), + ) + assert proc.returncode == 0, proc.stderr + report = json.loads(evidence.read_text(encoding="utf-8")) + smoke = next(item for item in report["checks"] if item["name"] == "browser:smoke") + assert smoke["status"] == "PASS" + assert smoke["original_status"] == "SKIP" + assert smoke["waiver_reason"] + + +def test_campus_scope_cannot_pass_with_a_missing_requested_building(tmp_path: Path) -> None: + spec, glb, _metadata = build_fixture(tmp_path) + render = tmp_path / "render.png" + render_fixture(render) + source_report = source_report_fixture(tmp_path) + site = tmp_path / "site.png" + render_fixture(site) + scope = tmp_path / "scope_manifest.json" + scope.write_text( + json.dumps( + { + "scope_version": 1, + "mode": "campus", + "requested": ["1#", "2#"], + "buildings": [ + {"id": "1#", "status": "PASS", "artifacts": [glb.name]}, + {"id": "2#", "status": "BLOCKED", "reason": "No readable source drawing", "artifacts": []}, + ], + "site": {"status": "PASS", "artifacts": [site.name]}, + } + ), + encoding="utf-8", + ) + evidence = tmp_path / "evidence.json" + proc = run( + sys.executable, + "scripts/verify_delivery.py", + "--spec", + str(spec), + "--source-report", + str(source_report), + "--glb", + str(glb), + "--render", + str(render), + "--scope-manifest", + str(scope), + "--evidence-out", + str(evidence), + ) + assert proc.returncode == 3 + report = json.loads(evidence.read_text(encoding="utf-8")) + assert report["status"] == "BLOCKED" + assert any(item["name"] == "scope:building:2#" and item["status"] == "WARN" for item in report["checks"]) + + def test_variants_manifest_requires_metadata_artifact(tmp_path: Path) -> None: spec, glb, _metadata = build_fixture(tmp_path) render = tmp_path / "render.png"