fix: enforce strict CAD showcase delivery gates

This commit is contained in:
2026-07-10 12:38:12 +08:00
parent 8b1b1d5e89
commit 78ec1fbdcc
8 changed files with 537 additions and 28 deletions

View File

@@ -29,6 +29,48 @@ def add_check(checks: list[dict[str, Any]], name: str, status: str, detail: str,
checks.append({"name": name, "status": status, "detail": detail, **data})
def parse_waivers(values: list[str]) -> dict[str, str]:
"""Parse explicit CHECK=REASON waivers.
Waivers are deliberately verbose: an Agent cannot silently turn WARN/SKIP
into PASS, and a misspelled or unused waiver becomes a hard failure.
"""
waivers: dict[str, str] = {}
for value in values:
name, separator, reason = value.partition("=")
name = name.strip()
reason = reason.strip()
if not separator or not name or not reason:
raise ValueError(f"Invalid waiver {value!r}; expected CHECK=REASON")
if name in waivers:
raise ValueError(f"Duplicate waiver for {name!r}")
waivers[name] = reason
return waivers
def apply_waivers(checks: list[dict[str, Any]], waivers: dict[str, str]) -> None:
used: set[str] = set()
for check in checks:
name = str(check["name"])
if name not in waivers:
continue
used.add(name)
if check["status"] not in {"WARN", "SKIP"}:
add_check(
checks,
f"waiver:{name}",
"FAIL",
f"Waivers may only acknowledge WARN/SKIP checks; {name} is {check['status']}",
)
continue
check["original_status"] = check["status"]
check["status"] = "PASS"
check["waiver_reason"] = waivers[name]
check["detail"] = f"WAIVED ({waivers[name]}): {check['detail']}"
for name in sorted(set(waivers) - used):
add_check(checks, f"waiver:{name}", "FAIL", f"Unused or misspelled waiver: {name}")
def inspect_glb(path: Path, budget_mb: float, checks: list[dict[str, Any]]) -> dict[str, Any] | None:
if not path.exists() or path.stat().st_size == 0:
add_check(checks, f"glb:{path.name}", "FAIL", f"Missing or empty GLB: {path}")
@@ -162,13 +204,144 @@ def inspect_offline(path: Path, checks: list[dict[str, Any]]) -> None:
def inspect_browser_report(path: Path, checks: list[dict[str, Any]]) -> None:
try:
report = json.loads(path.read_text(encoding="utf-8"))
passed = report.get("status") == "PASS" and not report.get("console_errors")
passed = (
report.get("status") == "PASS"
and not report.get("console_errors")
and not report.get("page_errors")
and not report.get("failed_requests")
)
detail = report.get("detail") or json.dumps(report, ensure_ascii=False)[:500]
add_check(checks, "browser:smoke", "PASS" if passed else "FAIL", detail)
except Exception as exc:
add_check(checks, "browser:smoke", "FAIL", f"Browser report invalid: {exc}")
def inspect_audit(path: Path, checks: list[dict[str, Any]]) -> None:
try:
report = json.loads(path.read_text(encoding="utf-8"))
gates = report.get("gates")
if not isinstance(gates, list) or not gates:
raise ValueError("audit has no gates")
for gate in gates:
status = str(gate.get("status", "FAIL"))
if status not in {"PASS", "WARN", "SKIP", "FAIL"}:
status = "FAIL"
add_check(
checks,
f"audit:{gate.get('name', 'unnamed')}",
status,
str(gate.get("detail", "Audit gate has no detail")),
)
except Exception as exc:
add_check(checks, "audit:report", "FAIL", f"Extraction audit invalid: {exc}")
def inspect_scope(path: Path, checks: list[dict[str, Any]]) -> None:
"""Validate a single-building/campus scope ledger against real artifacts."""
try:
scope = json.loads(path.read_text(encoding="utf-8"))
if scope.get("scope_version") != 1:
raise ValueError("scope_version must be 1")
mode = scope.get("mode")
if mode not in {"single", "campus"}:
raise ValueError("mode must be single or campus")
requested = scope.get("requested")
buildings = scope.get("buildings")
if (
not isinstance(requested, list)
or not requested
or not all(isinstance(item, str) and item for item in requested)
):
raise ValueError("requested must be a non-empty list of building IDs")
if len(set(requested)) != len(requested):
raise ValueError("requested contains duplicate building IDs")
if not isinstance(buildings, list):
raise ValueError("buildings must be a list")
by_id: dict[str, dict[str, Any]] = {}
for item in buildings:
building_id = str(item.get("id", ""))
if not building_id or building_id in by_id:
raise ValueError(f"missing or duplicate building id: {building_id!r}")
by_id[building_id] = item
missing = [item for item in requested if item not in by_id]
if missing:
add_check(
checks, "scope:inventory", "FAIL", "Requested buildings missing from ledger: " + ", ".join(missing)
)
else:
add_check(
checks, "scope:inventory", "PASS", f"All {len(requested)} requested building(s) are in the ledger"
)
for building_id in requested:
item = by_id.get(building_id)
if not item:
continue
status = item.get("status")
reason = str(item.get("reason", "")).strip()
artifacts = item.get("artifacts", [])
if status == "BLOCKED":
add_check(
checks,
f"scope:building:{building_id}",
"WARN" if reason else "FAIL",
reason or "BLOCKED building has no reason",
)
continue
if status != "PASS":
add_check(checks, f"scope:building:{building_id}", "FAIL", f"Invalid status: {status!r}")
continue
if not isinstance(artifacts, list) or not artifacts:
add_check(checks, f"scope:building:{building_id}", "FAIL", "PASS building has no artifacts")
continue
missing_artifacts = []
for artifact in artifacts:
artifact_path = Path(str(artifact))
if not artifact_path.is_absolute():
artifact_path = path.parent / artifact_path
if not artifact_path.exists() or (artifact_path.is_file() and artifact_path.stat().st_size == 0):
missing_artifacts.append(str(artifact))
add_check(
checks,
f"scope:building:{building_id}",
"FAIL" if missing_artifacts else "PASS",
"Missing artifacts: " + ", ".join(missing_artifacts)
if missing_artifacts
else f"{len(artifacts)} declared artifact(s) exist",
)
site = scope.get("site", {})
if mode == "campus":
site_status = site.get("status") if isinstance(site, dict) else None
if site_status == "BLOCKED":
reason = str(site.get("reason", "")).strip()
add_check(checks, "scope:site", "WARN" if reason else "FAIL", reason or "Campus site is blocked")
elif site_status != "PASS":
add_check(checks, "scope:site", "FAIL", "Campus scope requires site.status=PASS or BLOCKED")
else:
artifacts = site.get("artifacts", [])
missing_site = []
for artifact in artifacts if isinstance(artifacts, list) else []:
artifact_path = Path(str(artifact))
if not artifact_path.is_absolute():
artifact_path = path.parent / artifact_path
if not artifact_path.exists() or (artifact_path.is_file() and artifact_path.stat().st_size == 0):
missing_site.append(str(artifact))
if not artifacts or missing_site:
add_check(
checks,
"scope:site",
"FAIL",
"Campus site has no artifacts"
if not artifacts
else "Missing site artifacts: " + ", ".join(missing_site),
)
else:
add_check(checks, "scope:site", "PASS", f"{len(artifacts)} site artifact(s) exist")
except Exception as exc:
add_check(checks, "scope:manifest", "FAIL", f"Scope manifest invalid: {exc}")
def main() -> int:
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--spec", required=True)
@@ -178,14 +351,28 @@ def main() -> int:
parser.add_argument("--web-dir")
parser.add_argument("--offline-viewer")
parser.add_argument("--render", nargs="*", default=[])
parser.add_argument("--audit")
parser.add_argument("--scope-manifest")
parser.add_argument("--browser-report")
parser.add_argument("--require-browser-report", action="store_true")
parser.add_argument(
"--waiver",
action="append",
default=[],
metavar="CHECK=REASON",
help="Explicitly acknowledge one WARN/SKIP check with a non-empty reason",
)
parser.add_argument("--budget-mb", type=float, default=25.0)
parser.add_argument("--evidence-out", default="evidence_manifest.json")
args = parser.parse_args()
checks: list[dict[str, Any]] = []
artifacts: list[dict[str, Any]] = []
try:
waivers = parse_waivers(args.waiver)
except ValueError as exc:
print(f"GATE FAIL [waiver]: {exc}", file=sys.stderr)
return 3
try:
source_report = json.loads(Path(args.source_report).read_text(encoding="utf-8"))
showcase_schema.validate_source_report(source_report)
@@ -250,16 +437,25 @@ def main() -> int:
add_check(checks, "render", "FAIL", "At least one inspected render is required")
if args.web_dir:
inspect_web(Path(args.web_dir), checks)
if args.audit:
inspect_audit(Path(args.audit), checks)
if args.scope_manifest:
inspect_scope(Path(args.scope_manifest), checks)
if args.offline_viewer:
inspect_offline(Path(args.offline_viewer), checks)
if args.browser_report:
inspect_browser_report(Path(args.browser_report), checks)
elif args.require_browser_report:
add_check(checks, "browser:smoke", "FAIL", "Browser smoke report required but not supplied")
else:
add_check(checks, "browser:smoke", "WARN", "No browser smoke report supplied")
elif args.web_dir or args.require_browser_report:
add_check(checks, "browser:smoke", "SKIP", "Formal Web delivery has no browser smoke report")
status = "FAIL" if any(item["status"] == "FAIL" for item in checks) else "PASS"
apply_waivers(checks, waivers)
status = (
"FAIL"
if any(item["status"] == "FAIL" for item in checks)
else "BLOCKED"
if any(item["status"] in {"WARN", "SKIP"} for item in checks)
else "PASS"
)
evidence = {
"evidence_version": 1,
"generated_at": datetime.now(timezone.utc).isoformat(),
@@ -271,10 +467,10 @@ def main() -> int:
out.parent.mkdir(parents=True, exist_ok=True)
out.write_text(json.dumps(evidence, ensure_ascii=False, indent=2), encoding="utf-8")
print(json.dumps({"status": status, "evidence": str(out), "checks": len(checks)}, ensure_ascii=False))
if status == "FAIL":
if status != "PASS":
for item in checks:
if item["status"] == "FAIL":
print(f"GATE FAIL [{item['name']}]: {item['detail']}", file=sys.stderr)
if item["status"] in {"FAIL", "WARN", "SKIP"}:
print(f"GATE {status} [{item['name']}]: {item['detail']}", file=sys.stderr)
return 3
return 0