Files
cad-building-showcase/scripts/verify_delivery.py

676 lines
28 KiB
Python

#!/usr/bin/env python3
"""Strict, machine-readable verification for final CAD showcase delivery."""
from __future__ import annotations
import argparse
import base64
import gzip
import hashlib
import json
import re
import statistics
import sys
from datetime import datetime, timezone
from pathlib import Path
from typing import Any
import showcase_schema
def sha256(path: Path) -> str:
digest = hashlib.sha256()
with path.open("rb") as stream:
for chunk in iter(lambda: stream.read(1024 * 1024), b""):
digest.update(chunk)
return digest.hexdigest()
def add_check(checks: list[dict[str, Any]], name: str, status: str, detail: str, **data) -> None:
checks.append({"name": name, "status": status, "detail": detail, **data})
def parse_waivers(values: list[str]) -> dict[str, str]:
"""Parse explicit CHECK=REASON waivers.
Waivers are deliberately verbose: an Agent cannot silently turn WARN/SKIP
into PASS, and a misspelled or unused waiver becomes a hard failure.
"""
waivers: dict[str, str] = {}
for value in values:
name, separator, reason = value.partition("=")
name = name.strip()
reason = reason.strip()
if not separator or not name or not reason:
raise ValueError(f"Invalid waiver {value!r}; expected CHECK=REASON")
if name in waivers:
raise ValueError(f"Duplicate waiver for {name!r}")
waivers[name] = reason
return waivers
def apply_waivers(checks: list[dict[str, Any]], waivers: dict[str, str]) -> None:
used: set[str] = set()
for check in checks:
name = str(check["name"])
if name not in waivers:
continue
used.add(name)
if check["status"] not in {"WARN", "SKIP"}:
add_check(
checks,
f"waiver:{name}",
"FAIL",
f"Waivers may only acknowledge WARN/SKIP checks; {name} is {check['status']}",
)
continue
check["original_status"] = check["status"]
check["status"] = "PASS"
check["waiver_reason"] = waivers[name]
check["detail"] = f"WAIVED ({waivers[name]}): {check['detail']}"
for name in sorted(set(waivers) - used):
add_check(checks, f"waiver:{name}", "FAIL", f"Unused or misspelled waiver: {name}")
def inspect_glb(path: Path, budget_mb: float, checks: list[dict[str, Any]]) -> dict[str, Any] | None:
if not path.exists() or path.stat().st_size == 0:
add_check(checks, f"glb:{path.name}", "FAIL", f"Missing or empty GLB: {path}")
return None
with path.open("rb") as stream:
header = stream.read(4)
if header != b"glTF":
add_check(checks, f"glb:{path.name}", "FAIL", "File does not have a GLB header")
return None
try:
from pygltflib import GLTF2
model = GLTF2().load(str(path))
nodes = [node.name or "" for node in model.nodes]
named = [name for name in nodes if "__" in name]
meshes = len(model.meshes or [])
primitives = sum(len(mesh.primitives) for mesh in model.meshes or [])
if meshes == 0 or primitives == 0:
add_check(checks, f"glb:{path.name}", "FAIL", "GLB contains no mesh primitives")
return None
if not named:
add_check(checks, f"glb:{path.name}:names", "FAIL", "No <floor>__<category> node names found")
else:
add_check(checks, f"glb:{path.name}:names", "PASS", f"{len(named)} named node(s)")
extensions = list(model.extensionsUsed or [])
detail = f"{meshes} meshes, {primitives} primitives, {path.stat().st_size / 1024 / 1024:.2f} MB"
status = "PASS" if path.stat().st_size <= budget_mb * 1024 * 1024 else "FAIL"
if status == "FAIL":
detail += f"; exceeds {budget_mb:.2f} MB budget"
add_check(checks, f"glb:{path.name}:structure", status, detail)
return {
"path": str(path),
"bytes": path.stat().st_size,
"sha256": sha256(path),
"meshes": meshes,
"primitives": primitives,
"nodes": named,
"extensions": extensions,
}
except Exception as exc:
add_check(checks, f"glb:{path.name}", "FAIL", f"GLB parser failed: {exc}")
return None
def inspect_render(path: Path, checks: list[dict[str, Any]]) -> None:
if not path.exists() or path.stat().st_size == 0:
add_check(checks, f"render:{path.name}", "FAIL", f"Missing render: {path}")
return
try:
from PIL import Image, ImageStat
image = Image.open(path).convert("RGB")
sample = image.resize((128, 128))
stat = ImageStat.Stat(sample)
spread = max(stat.var)
pixels = list(sample.getdata())
border = []
for index in range(128):
border.extend((sample.getpixel((index, 0)), sample.getpixel((index, 127))))
for index in range(1, 127):
border.extend((sample.getpixel((0, index)), sample.getpixel((127, index))))
background = tuple(int(statistics.median(pixel[channel] for pixel in border)) for channel in range(3))
foreground = [max(abs(pixel[channel] - background[channel]) for channel in range(3)) > 12 for pixel in pixels]
occupancy = sum(foreground) / len(foreground)
xs = [index % 128 for index, present in enumerate(foreground) if present]
ys = [index // 128 for index, present in enumerate(foreground) if present]
bbox_w = max(xs) - min(xs) + 1 if xs else 0
bbox_h = max(ys) - min(ys) + 1 if ys else 0
def luminance(pixel: tuple[int, int, int]) -> float:
return pixel[0] * 0.2126 + pixel[1] * 0.7152 + pixel[2] * 0.0722
transitions = 0
comparisons = 0
for y in range(128):
for x in range(128):
here = luminance(sample.getpixel((x, y)))
if x + 1 < 128:
transitions += abs(here - luminance(sample.getpixel((x + 1, y)))) > 45
comparisons += 1
if y + 1 < 128:
transitions += abs(here - luminance(sample.getpixel((x, y + 1)))) > 45
comparisons += 1
edge_ratio = transitions / comparisons
status = (
"PASS"
if image.width >= 320
and image.height >= 240
and spread > 2.0
and 0.01 <= occupancy <= 0.95
and bbox_w >= 24
and bbox_h >= 24
and edge_ratio < 0.45
else "FAIL"
)
add_check(
checks,
f"render:{path.name}",
status,
(
f"{image.width}x{image.height}, variance {spread:.2f}, "
f"foreground {occupancy:.1%}, bbox {bbox_w}x{bbox_h}/128, edge ratio {edge_ratio:.3f}"
),
)
except Exception as exc:
add_check(checks, f"render:{path.name}", "FAIL", f"Image inspection failed: {exc}")
def inspect_web(web_dir: Path, checks: list[dict[str, Any]]) -> None:
required = [
"index.html",
"app.js",
"styles.css",
"model.glb",
"model_metadata.json",
"node_modules/three/build/three.module.js",
"node_modules/lucide/dist/umd/lucide.js",
]
missing = [name for name in required if not (web_dir / name).exists()]
if missing:
add_check(checks, "web:files", "FAIL", "Missing Web files: " + ", ".join(missing))
return
text = (web_dir / "index.html").read_text(encoding="utf-8") + (web_dir / "app.js").read_text(encoding="utf-8")
placeholders = sorted(set(re.findall(r"\{\{[^}]+\}\}", text)))
if placeholders:
add_check(checks, "web:placeholders", "FAIL", f"Unexpanded placeholders: {placeholders}")
else:
add_check(checks, "web:placeholders", "PASS", "Template placeholders expanded")
features = ("window.__renders", "invalidate(", "DRACOLoader", "restoreStateFromHash", "switchVariant")
absent = [feature for feature in features if feature not in text]
add_check(
checks,
"web:features",
"FAIL" if absent else "PASS",
"Missing features: " + ", ".join(absent) if absent else "Required viewer features present",
)
try:
metadata = json.loads((web_dir / "model_metadata.json").read_text(encoding="utf-8"))
missing_variants = [
item.get("file", "?")
for item in metadata.get("model_variants", [])
if not (web_dir / item.get("file", "")).exists()
]
add_check(
checks,
"web:variants",
"FAIL" if missing_variants else "PASS",
"Missing variant assets: " + ", ".join(missing_variants)
if missing_variants
else "Variant assets accounted for",
)
except Exception as exc:
add_check(checks, "web:metadata", "FAIL", f"Metadata parse failed: {exc}")
def inspect_offline(path: Path, checks: list[dict[str, Any]]) -> None:
if not path.exists():
add_check(checks, "offline:viewer", "FAIL", f"Offline viewer missing: {path}")
return
text = path.read_text(encoding="utf-8")
match = re.search(r'const GLB_B64="([A-Za-z0-9+/=]+)";', text)
if not match:
add_check(checks, "offline:payload", "FAIL", "Embedded GLB payload not found")
return
try:
payload = gzip.decompress(base64.b64decode(match.group(1)))
if payload[:4] != b"glTF":
raise ValueError("decoded payload is not a GLB")
add_check(checks, "offline:payload", "PASS", f"Decoded {len(payload)} GLB bytes")
except Exception as exc:
add_check(checks, "offline:payload", "FAIL", f"Embedded payload decode failed: {exc}")
WEB_BROWSER_CHECKS = {
"desktop:viewport",
"desktop:#floor-controls",
"desktop:#category-controls",
"desktop:#explode",
"desktop:#section",
"desktop:#snapshot",
"desktop:idle-render",
"mobile:viewport",
"mobile:#floor-controls",
"mobile:#category-controls",
"mobile:#explode",
"mobile:#section",
"mobile:#snapshot",
"mobile:idle-render",
}
OFFLINE_BROWSER_CHECKS = {
"offline:viewport",
"offline:model-ready",
"offline:canvas",
"offline:render",
}
def inspect_browser_report(path: Path, checks: list[dict[str, Any]], name: str = "browser:smoke") -> None:
try:
report = json.loads(path.read_text(encoding="utf-8"))
report_checks = report.get("checks")
if not isinstance(report_checks, list) or not report_checks:
raise ValueError("checks must be a non-empty list")
by_name: dict[str, dict[str, Any]] = {}
for item in report_checks:
if not isinstance(item, dict) or not isinstance(item.get("name"), str):
raise ValueError("every browser check must be an object with a name")
if item["name"] in by_name:
raise ValueError(f"duplicate browser check: {item['name']}")
by_name[item["name"]] = item
required = OFFLINE_BROWSER_CHECKS if name == "offline:browser-smoke" else WEB_BROWSER_CHECKS
missing = sorted(required - set(by_name))
failed = sorted(item_name for item_name, item in by_name.items() if item.get("status") != "PASS")
if missing:
raise ValueError("missing required checks: " + ", ".join(missing))
if failed:
raise ValueError("non-PASS checks: " + ", ".join(failed))
expected_viewports = (
{"offline:viewport": "1440x900 expected 1440x900"}
if name == "offline:browser-smoke"
else {
"desktop:viewport": "1440x900 expected 1440x900",
"mobile:viewport": "390x844 expected 390x844",
}
)
for check_name, expected_detail in expected_viewports.items():
if by_name[check_name].get("detail") != expected_detail:
raise ValueError(f"{check_name} has untrusted viewport evidence: {by_name[check_name].get('detail')!r}")
for error_key in ("console_errors", "page_errors", "failed_requests"):
if not isinstance(report.get(error_key), list):
raise ValueError(f"{error_key} must be a list")
passed = (
report.get("status") == "PASS"
and not report.get("console_errors")
and not report.get("page_errors")
and not report.get("failed_requests")
)
detail = f"validated {len(required)}/{len(required)} required checks; " + (
report.get("detail") or json.dumps(report, ensure_ascii=False)[:300]
)
add_check(checks, name, "PASS" if passed else "FAIL", detail)
except Exception as exc:
add_check(checks, name, "FAIL", f"Browser report invalid: {exc}")
def inspect_audit(path: Path, checks: list[dict[str, Any]]) -> None:
try:
report = json.loads(path.read_text(encoding="utf-8"))
gates = report.get("gates")
if not isinstance(gates, list) or not gates:
raise ValueError("audit has no gates")
for gate in gates:
status = str(gate.get("status", "FAIL"))
if status not in {"PASS", "WARN", "SKIP", "FAIL", "NOT_APPLICABLE"}:
status = "FAIL"
add_check(
checks,
f"audit:{gate.get('name', 'unnamed')}",
status,
str(gate.get("detail", "Audit gate has no detail")),
)
except Exception as exc:
add_check(checks, "audit:report", "FAIL", f"Extraction audit invalid: {exc}")
def verifiable_model_artifact(path: Path) -> tuple[bool, str]:
"""Reject scope ledgers backed only by arbitrary non-empty files."""
try:
suffix = path.suffix.lower()
if suffix == ".glb":
from pygltflib import GLTF2
model = GLTF2().load(str(path))
primitives = sum(len(mesh.primitives) for mesh in model.meshes or [])
if primitives <= 0:
return False, "GLB has no mesh primitives"
return True, f"GLB with {primitives} primitive(s)"
if suffix == ".blend":
if path.read_bytes()[:7] != b"BLENDER":
return False, "invalid Blender header"
return True, "Blender scene"
if suffix == ".json":
document = json.loads(path.read_text(encoding="utf-8"))
showcase_schema.validate_spec(document)
return True, f"building spec with {len(document['floors'])} floor(s)"
return False, f"unsupported model artifact type {suffix or '<none>'}"
except Exception as exc:
return False, str(exc)
def verifiable_site_artifact(path: Path) -> tuple[bool, str]:
model_ok, model_detail = verifiable_model_artifact(path)
if model_ok:
return True, model_detail
try:
from PIL import Image
with Image.open(path) as image:
image.verify()
if image.width < 320 or image.height < 240:
return False, f"site image too small: {image.width}x{image.height}"
return True, f"site image {image.width}x{image.height}"
except Exception:
return False, model_detail
def inspect_scope(path: Path, checks: list[dict[str, Any]]) -> None:
"""Validate a single-building/campus scope ledger against real artifacts."""
try:
scope = json.loads(path.read_text(encoding="utf-8"))
if scope.get("scope_version") != 1:
raise ValueError("scope_version must be 1")
mode = scope.get("mode")
if mode not in {"single", "campus"}:
raise ValueError("mode must be single or campus")
requested = scope.get("requested")
buildings = scope.get("buildings")
if (
not isinstance(requested, list)
or not requested
or not all(isinstance(item, str) and item for item in requested)
):
raise ValueError("requested must be a non-empty list of building IDs")
if len(set(requested)) != len(requested):
raise ValueError("requested contains duplicate building IDs")
if mode == "single" and len(requested) != 1:
raise ValueError("single mode requires exactly one requested building ID")
if not isinstance(buildings, list):
raise ValueError("buildings must be a list")
by_id: dict[str, dict[str, Any]] = {}
for item in buildings:
building_id = str(item.get("id", ""))
if not building_id or building_id in by_id:
raise ValueError(f"missing or duplicate building id: {building_id!r}")
by_id[building_id] = item
missing = [item for item in requested if item not in by_id]
unexpected = sorted(set(by_id) - set(requested))
if missing or unexpected:
add_check(
checks,
"scope:inventory",
"FAIL",
(
("Requested buildings missing from ledger: " + ", ".join(missing) if missing else "")
+ ("; " if missing and unexpected else "")
+ ("Unrequested buildings in ledger: " + ", ".join(unexpected) if unexpected else "")
),
)
else:
add_check(
checks, "scope:inventory", "PASS", f"All {len(requested)} requested building(s) are in the ledger"
)
for building_id in requested:
item = by_id.get(building_id)
if not item:
continue
status = item.get("status")
reason = str(item.get("reason", "")).strip()
artifacts = item.get("artifacts", [])
if status == "BLOCKED":
add_check(
checks,
f"scope:building:{building_id}",
"WARN" if reason else "FAIL",
reason or "BLOCKED building has no reason",
)
continue
if status != "PASS":
add_check(checks, f"scope:building:{building_id}", "FAIL", f"Invalid status: {status!r}")
continue
if not isinstance(artifacts, list) or not artifacts:
add_check(checks, f"scope:building:{building_id}", "FAIL", "PASS building has no artifacts")
continue
missing_artifacts = []
verified_models = []
invalid_models = []
for artifact in artifacts:
artifact_path = Path(str(artifact))
if not artifact_path.is_absolute():
artifact_path = path.parent / artifact_path
if not artifact_path.exists() or (artifact_path.is_file() and artifact_path.stat().st_size == 0):
missing_artifacts.append(str(artifact))
continue
verified, detail = verifiable_model_artifact(artifact_path)
if verified:
verified_models.append(f"{artifact}: {detail}")
else:
invalid_models.append(f"{artifact}: {detail}")
failed = bool(missing_artifacts or not verified_models)
add_check(
checks,
f"scope:building:{building_id}",
"FAIL" if failed else "PASS",
"Missing artifacts: " + ", ".join(missing_artifacts)
if missing_artifacts
else (
f"{len(verified_models)} verifiable model artifact(s)"
if verified_models
else "No verifiable GLB/BLEND/building-spec artifact; rejected: " + "; ".join(invalid_models)
),
)
site = scope.get("site", {})
if mode == "campus":
site_status = site.get("status") if isinstance(site, dict) else None
if site_status == "BLOCKED":
reason = str(site.get("reason", "")).strip()
add_check(checks, "scope:site", "WARN" if reason else "FAIL", reason or "Campus site is blocked")
elif site_status != "PASS":
add_check(checks, "scope:site", "FAIL", "Campus scope requires site.status=PASS or BLOCKED")
else:
artifacts = site.get("artifacts", [])
missing_site = []
verified_site = []
for artifact in artifacts if isinstance(artifacts, list) else []:
artifact_path = Path(str(artifact))
if not artifact_path.is_absolute():
artifact_path = path.parent / artifact_path
if not artifact_path.exists() or (artifact_path.is_file() and artifact_path.stat().st_size == 0):
missing_site.append(str(artifact))
continue
verified, detail = verifiable_site_artifact(artifact_path)
if verified:
verified_site.append(f"{artifact}: {detail}")
if not artifacts or missing_site or not verified_site:
add_check(
checks,
"scope:site",
"FAIL",
"Campus site has no artifacts"
if not artifacts
else (
"Missing site artifacts: " + ", ".join(missing_site)
if missing_site
else "Campus site has no verifiable model or image artifact"
),
)
else:
add_check(checks, "scope:site", "PASS", f"{len(verified_site)} verified site artifact(s)")
except Exception as exc:
add_check(checks, "scope:manifest", "FAIL", f"Scope manifest invalid: {exc}")
def main() -> int:
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--spec", required=True)
parser.add_argument("--source-report", required=True)
parser.add_argument("--glb", nargs="*", default=[])
parser.add_argument("--variants-manifest")
parser.add_argument("--web-dir")
parser.add_argument("--offline-viewer")
parser.add_argument("--offline-browser-report")
parser.add_argument("--render", nargs="*", default=[])
parser.add_argument("--audit")
parser.add_argument("--require-audit", action="store_true")
parser.add_argument("--scope-manifest")
parser.add_argument("--require-scope-manifest", action="store_true")
parser.add_argument("--browser-report")
parser.add_argument("--require-browser-report", action="store_true")
parser.add_argument(
"--waiver",
action="append",
default=[],
metavar="CHECK=REASON",
help="Explicitly acknowledge one WARN/SKIP check with a non-empty reason",
)
parser.add_argument("--budget-mb", type=float, default=25.0)
parser.add_argument("--evidence-out", default="evidence_manifest.json")
args = parser.parse_args()
checks: list[dict[str, Any]] = []
artifacts: list[dict[str, Any]] = []
try:
waivers = parse_waivers(args.waiver)
except ValueError as exc:
print(f"GATE FAIL [waiver]: {exc}", file=sys.stderr)
return 3
try:
source_report = json.loads(Path(args.source_report).read_text(encoding="utf-8"))
showcase_schema.validate_source_report(source_report)
source_gate = source_report["gate"]
source_status = str(source_gate.get("status", "FAIL"))
add_check(
checks,
"cad_source_report",
source_status,
(
f"Source gate {source_status}; {len(source_report['files'])} inspected DXF file(s)"
+ (f"; partial reason: {source_gate.get('partial_reason')}" if source_status == "WARN" else "")
),
)
except Exception as exc:
add_check(checks, "cad_source_report", "FAIL", str(exc))
spec_path = Path(args.spec)
try:
spec = json.loads(spec_path.read_text(encoding="utf-8"))
showcase_schema.validate_spec(spec)
add_check(checks, "building_spec", "PASS", f"Schema v{spec['schema_version']}; {len(spec['floors'])} floor(s)")
except Exception as exc:
add_check(checks, "building_spec", "FAIL", str(exc))
glb_paths = [Path(item) for item in args.glb]
if args.variants_manifest:
manifest_path = Path(args.variants_manifest)
try:
manifest = json.loads(manifest_path.read_text(encoding="utf-8"))
showcase_schema.validate_manifest(manifest)
add_check(checks, "variants_manifest", "PASS", f"{len(manifest['variants'])} variant(s)")
for variant in manifest["variants"]:
source = Path(variant["source_glb"])
if not source.is_absolute():
source = manifest_path.parent / source
glb_paths.append(source)
spec_file = Path(variant["spec"])
if not spec_file.is_absolute():
spec_file = manifest_path.parent / spec_file
variant_spec = json.loads(spec_file.read_text(encoding="utf-8"))
showcase_schema.validate_spec(variant_spec)
metadata_file = Path(variant["metadata"])
if not metadata_file.is_absolute():
metadata_file = manifest_path.parent / metadata_file
metadata = json.loads(metadata_file.read_text(encoding="utf-8"))
if metadata.get("metadata_version") != 1:
raise ValueError(f"{metadata_file}: metadata_version must be 1")
if metadata.get("schema_version") != variant_spec.get("schema_version"):
raise ValueError(f"{metadata_file}: schema_version does not match {spec_file}")
except Exception as exc:
add_check(checks, "variants_manifest", "FAIL", str(exc))
seen = set()
for path in glb_paths:
resolved = path.resolve()
if resolved in seen:
continue
seen.add(resolved)
artifact = inspect_glb(path, args.budget_mb, checks)
if artifact:
artifacts.append(artifact)
if not glb_paths:
add_check(checks, "glb", "FAIL", "No GLB supplied")
for path in args.render:
inspect_render(Path(path), checks)
if not args.render:
add_check(checks, "render", "FAIL", "At least one inspected render is required")
if args.web_dir:
inspect_web(Path(args.web_dir), checks)
if args.audit:
inspect_audit(Path(args.audit), checks)
elif args.require_audit:
add_check(checks, "audit:report", "FAIL", "Required extraction audit was not supplied")
if args.scope_manifest:
inspect_scope(Path(args.scope_manifest), checks)
elif args.require_scope_manifest:
add_check(checks, "scope:manifest", "FAIL", "Required scope manifest was not supplied")
if args.offline_viewer:
inspect_offline(Path(args.offline_viewer), checks)
if args.offline_browser_report:
inspect_browser_report(Path(args.offline_browser_report), checks, "offline:browser-smoke")
else:
add_check(
checks,
"offline:browser-smoke",
"SKIP",
"Offline viewer has no real file:// browser smoke report",
)
if args.browser_report:
inspect_browser_report(Path(args.browser_report), checks)
elif args.web_dir or args.require_browser_report:
add_check(checks, "browser:smoke", "SKIP", "Formal Web delivery has no browser smoke report")
apply_waivers(checks, waivers)
status = (
"FAIL"
if any(item["status"] == "FAIL" for item in checks)
else "BLOCKED"
if any(item["status"] in {"WARN", "SKIP"} for item in checks)
else "PASS"
)
evidence = {
"evidence_version": 1,
"generated_at": datetime.now(timezone.utc).isoformat(),
"status": status,
"waivers": [{"check": name, "reason": reason} for name, reason in sorted(waivers.items())],
"checks": checks,
"artifacts": artifacts,
}
out = Path(args.evidence_out)
out.parent.mkdir(parents=True, exist_ok=True)
out.write_text(json.dumps(evidence, ensure_ascii=False, indent=2), encoding="utf-8")
print(json.dumps({"status": status, "evidence": str(out), "checks": len(checks)}, ensure_ascii=False))
if status != "PASS":
for item in checks:
if item["status"] in {"FAIL", "WARN", "SKIP"}:
print(f"GATE {status} [{item['name']}]: {item['detail']}", file=sys.stderr)
return 3
return 0
if __name__ == "__main__":
raise SystemExit(main())