Files
cad-building-showcase/tests/test_delivery.py

609 lines
19 KiB
Python

from __future__ import annotations
import json
import subprocess
import sys
from pathlib import Path
from PIL import Image, ImageDraw
ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(ROOT / "scripts"))
import showcase_schema # noqa: E402
from test_contracts import minimal_spec # noqa: E402
def run(*args: str) -> subprocess.CompletedProcess[str]:
return subprocess.run(list(args), cwd=ROOT, text=True, capture_output=True)
def build_fixture(tmp_path: Path) -> tuple[Path, Path, Path]:
spec = tmp_path / "spec.json"
glb = tmp_path / "model.glb"
metadata = tmp_path / "model_metadata.json"
spec.write_text(json.dumps(minimal_spec()), encoding="utf-8")
proc = run(
sys.executable,
"scripts/spec_to_glb.py",
"--spec",
str(spec),
"--out",
str(glb),
"--metadata-out",
str(metadata),
)
assert proc.returncode == 0, proc.stderr
return spec, glb, metadata
def render_fixture(path: Path) -> None:
image = Image.new("RGB", (640, 480), "white")
draw = ImageDraw.Draw(image)
draw.rectangle((100, 120, 540, 400), fill="#4477aa", outline="black", width=4)
image.save(path)
def source_report_fixture(tmp_path: Path) -> Path:
path = tmp_path / "cad_read_report.json"
path.write_text(
json.dumps(
{
"report_version": 1,
"dxf_sanitized": str(tmp_path),
"gate": {"name": "all_sources_converted_and_readable", "status": "PASS"},
"files": [{"file": "fixture.dxf", "status": "ok", "modelspace_entities": 1}],
}
),
encoding="utf-8",
)
return path
def browser_report_payload(*, offline: bool = False) -> dict:
if offline:
checks = [
{"name": "offline:viewport", "status": "PASS", "detail": "1440x900 expected 1440x900"},
{"name": "offline:model-ready", "status": "PASS", "detail": "1 mesh(es)"},
{"name": "offline:canvas", "status": "PASS"},
{"name": "offline:render", "status": "PASS", "detail": "1 render(s)"},
]
else:
checks = []
for viewport, detail in (
("desktop", "1440x900 expected 1440x900"),
("mobile", "390x844 expected 390x844"),
):
checks.append({"name": f"{viewport}:viewport", "status": "PASS", "detail": detail})
for selector in ("#floor-controls", "#category-controls", "#explode", "#section", "#snapshot"):
checks.append({"name": f"{viewport}:{selector}", "status": "PASS"})
checks.append({"name": f"{viewport}:idle-render", "status": "PASS", "detail": "3->3"})
return {
"status": "PASS",
"detail": f"{len(checks)}/{len(checks)} checks passed",
"checks": checks,
"console_errors": [],
"page_errors": [],
"failed_requests": [],
}
def test_strict_delivery_passes_for_glb_render_and_offline_viewer(tmp_path: Path) -> None:
spec, glb, _metadata = build_fixture(tmp_path)
render = tmp_path / "render.png"
render_fixture(render)
offline = tmp_path / "viewer.html"
proc = run(
sys.executable,
"scripts/write_offline_viewer.py",
"--spec",
str(spec),
"--glb",
str(glb),
"--out",
str(offline),
)
assert proc.returncode == 0, proc.stderr
offline_browser_report = tmp_path / "offline_browser_smoke_report.json"
offline_browser_report.write_text(
json.dumps(browser_report_payload(offline=True)),
encoding="utf-8",
)
evidence = tmp_path / "evidence.json"
source_report = source_report_fixture(tmp_path)
proc = run(
sys.executable,
"scripts/verify_delivery.py",
"--spec",
str(spec),
"--source-report",
str(source_report),
"--glb",
str(glb),
"--render",
str(render),
"--offline-viewer",
str(offline),
"--offline-browser-report",
str(offline_browser_report),
"--evidence-out",
str(evidence),
)
assert proc.returncode == 0, proc.stderr
report = json.loads(evidence.read_text(encoding="utf-8"))
assert report["status"] == "PASS"
assert any(item["name"] == "offline:payload" and item["status"] == "PASS" for item in report["checks"])
assert any(item["name"] == "offline:browser-smoke" and item["status"] == "PASS" for item in report["checks"])
def test_browser_report_cannot_forge_pass_without_required_checks(tmp_path: Path) -> None:
import verify_delivery
path = tmp_path / "forged.json"
path.write_text(
json.dumps(
{
"status": "PASS",
"checks": [],
"console_errors": [],
"page_errors": [],
"failed_requests": [],
}
),
encoding="utf-8",
)
checks: list[dict] = []
verify_delivery.inspect_browser_report(path, checks)
assert checks[0]["status"] == "FAIL"
assert "non-empty list" in checks[0]["detail"]
def test_render_gate_rejects_high_variance_noise(tmp_path: Path) -> None:
import verify_delivery
noisy = Image.new("RGB", (640, 480))
pixels = noisy.load()
for y in range(noisy.height):
for x in range(noisy.width):
pixels[x, y] = (255, 255, 255) if (x + y) % 2 else (0, 0, 0)
path = tmp_path / "noise.png"
noisy.save(path)
checks: list[dict] = []
verify_delivery.inspect_render(path, checks)
assert checks[0]["status"] == "FAIL"
def test_offline_viewer_without_real_browser_report_is_blocked(tmp_path: Path) -> None:
spec, glb, _metadata = build_fixture(tmp_path)
render = tmp_path / "render.png"
render_fixture(render)
offline = tmp_path / "viewer.html"
proc = run(
sys.executable,
"scripts/write_offline_viewer.py",
"--spec",
str(spec),
"--glb",
str(glb),
"--out",
str(offline),
)
assert proc.returncode == 0, proc.stderr
source_report = source_report_fixture(tmp_path)
evidence = tmp_path / "evidence.json"
proc = run(
sys.executable,
"scripts/verify_delivery.py",
"--spec",
str(spec),
"--source-report",
str(source_report),
"--glb",
str(glb),
"--render",
str(render),
"--offline-viewer",
str(offline),
"--evidence-out",
str(evidence),
)
assert proc.returncode == 3
report = json.loads(evidence.read_text(encoding="utf-8"))
assert report["status"] == "BLOCKED"
assert any(item["name"] == "offline:browser-smoke" and item["status"] == "SKIP" for item in report["checks"])
def test_offline_viewer_escapes_html_and_inline_script_data(tmp_path: Path) -> None:
spec, glb, _metadata = build_fixture(tmp_path)
document = json.loads(spec.read_text(encoding="utf-8"))
document["project"]["name"] = '<Tower & Co></script><script>alert("title")</script>'
document["floors"]["F1"]["name"] = '首层</script><script>alert("floor")</script>'
document["categories"]["walls"]["label"] = "墙<&>"
spec.write_text(json.dumps(document), encoding="utf-8")
offline = tmp_path / "viewer.html"
proc = run(
sys.executable,
"scripts/write_offline_viewer.py",
"--spec",
str(spec),
"--glb",
str(glb),
"--out",
str(offline),
"--subtitle",
"<unsafe & subtitle>",
)
assert proc.returncode == 0, proc.stderr
text = offline.read_text(encoding="utf-8")
assert "&lt;Tower &amp; Co&gt;&lt;/script&gt;" in text
assert "&lt;unsafe &amp; subtitle&gt;" in text
assert '首层<\\/script><script>alert(\\"floor\\")<\\/script>' in text
assert '首层</script><script>alert("floor")</script>' not in text
def test_web_variant_entry_requires_a_nonempty_file(tmp_path: Path) -> None:
import verify_delivery
web = tmp_path / "web"
for relative in (
"index.html",
"app.js",
"styles.css",
"model.glb",
"node_modules/three/build/three.module.js",
"node_modules/lucide/dist/umd/lucide.js",
):
target = web / relative
target.parent.mkdir(parents=True, exist_ok=True)
target.write_text(
"window.__renders; invalidate(); DRACOLoader; restoreStateFromHash; switchVariant", encoding="utf-8"
)
(web / "model_metadata.json").write_text(json.dumps({"model_variants": [{"key": "standard", "file": ""}]}))
checks: list[dict] = []
verify_delivery.inspect_web(web, checks)
variant_check = next(item for item in checks if item["name"] == "web:variants")
assert variant_check["status"] == "FAIL"
assert "<missing file>" in variant_check["detail"]
def test_strict_delivery_requires_a_render(tmp_path: Path) -> None:
spec, glb, _metadata = build_fixture(tmp_path)
source_report = source_report_fixture(tmp_path)
proc = run(
sys.executable,
"scripts/verify_delivery.py",
"--spec",
str(spec),
"--source-report",
str(source_report),
"--glb",
str(glb),
"--evidence-out",
str(tmp_path / "evidence.json"),
)
assert proc.returncode == 3
assert "At least one inspected render is required" in proc.stderr
def test_web_writer_and_static_delivery_checks(tmp_path: Path) -> None:
spec, glb, metadata = build_fixture(tmp_path)
web = tmp_path / "web"
proc = run(
sys.executable,
"scripts/write_web_showcase.py",
"--glb",
str(glb),
"--metadata",
str(metadata),
"--out-dir",
str(web),
)
assert proc.returncode == 0, proc.stderr
for relative in ("node_modules/three/build/three.module.js", "node_modules/lucide/dist/umd/lucide.js"):
target = web / relative
target.parent.mkdir(parents=True, exist_ok=True)
target.write_text("// test dependency", encoding="utf-8")
render = tmp_path / "render.png"
render_fixture(render)
source_report = source_report_fixture(tmp_path)
proc = run(
sys.executable,
"scripts/verify_delivery.py",
"--spec",
str(spec),
"--source-report",
str(source_report),
"--glb",
str(glb),
"--render",
str(render),
"--web-dir",
str(web),
"--evidence-out",
str(tmp_path / "web_evidence.json"),
)
assert proc.returncode == 3
assert "browser:smoke" in proc.stderr
browser_report = tmp_path / "browser_smoke_report.json"
browser_report.write_text(
json.dumps(browser_report_payload()),
encoding="utf-8",
)
proc = run(
sys.executable,
"scripts/verify_delivery.py",
"--spec",
str(spec),
"--source-report",
str(source_report),
"--glb",
str(glb),
"--render",
str(render),
"--web-dir",
str(web),
"--browser-report",
str(browser_report),
"--evidence-out",
str(tmp_path / "web_evidence.json"),
)
assert proc.returncode == 0, proc.stderr
def test_reasoned_partial_source_can_only_pass_with_named_waiver(tmp_path: Path) -> None:
spec, glb, _metadata = build_fixture(tmp_path)
render = tmp_path / "render.png"
render_fixture(render)
source_report = source_report_fixture(tmp_path)
source = json.loads(source_report.read_text(encoding="utf-8"))
source["gate"] = {
"name": "all_sources_converted_and_readable",
"status": "WARN",
"detail": "one optional elevation is missing",
"partial_reason": "User approved plan-only delivery",
}
source_report.write_text(json.dumps(source), encoding="utf-8")
evidence = tmp_path / "evidence.json"
base = [
sys.executable,
"scripts/verify_delivery.py",
"--spec",
str(spec),
"--source-report",
str(source_report),
"--glb",
str(glb),
"--render",
str(render),
"--evidence-out",
str(evidence),
]
proc = run(*base)
assert proc.returncode == 3
assert json.loads(evidence.read_text(encoding="utf-8"))["status"] == "BLOCKED"
proc = run(*base, "--waiver", "cad_source_report=User approved plan-only delivery")
assert proc.returncode == 0, proc.stderr
report = json.loads(evidence.read_text(encoding="utf-8"))
source_check = next(item for item in report["checks"] if item["name"] == "cad_source_report")
assert source_check["original_status"] == "WARN"
def test_warn_or_skip_blocks_without_explicit_reasoned_waiver(tmp_path: Path) -> None:
spec, glb, _metadata = build_fixture(tmp_path)
render = tmp_path / "render.png"
render_fixture(render)
source_report = source_report_fixture(tmp_path)
evidence = tmp_path / "evidence.json"
proc = run(
sys.executable,
"scripts/verify_delivery.py",
"--spec",
str(spec),
"--source-report",
str(source_report),
"--glb",
str(glb),
"--render",
str(render),
"--require-browser-report",
"--evidence-out",
str(evidence),
)
assert proc.returncode == 3
assert json.loads(evidence.read_text(encoding="utf-8"))["status"] == "BLOCKED"
proc = run(
sys.executable,
"scripts/verify_delivery.py",
"--spec",
str(spec),
"--source-report",
str(source_report),
"--glb",
str(glb),
"--render",
str(render),
"--require-browser-report",
"--waiver",
"browser:smoke=No browser is available in this offline-only benchmark",
"--evidence-out",
str(evidence),
)
assert proc.returncode == 0, proc.stderr
report = json.loads(evidence.read_text(encoding="utf-8"))
smoke = next(item for item in report["checks"] if item["name"] == "browser:smoke")
assert smoke["status"] == "PASS"
assert smoke["original_status"] == "SKIP"
assert smoke["waiver_reason"]
def test_campus_scope_cannot_pass_with_a_missing_requested_building(tmp_path: Path) -> None:
spec, glb, _metadata = build_fixture(tmp_path)
render = tmp_path / "render.png"
render_fixture(render)
source_report = source_report_fixture(tmp_path)
site = tmp_path / "site.png"
render_fixture(site)
scope = tmp_path / "scope_manifest.json"
scope.write_text(
json.dumps(
{
"scope_version": 1,
"mode": "campus",
"requested": ["1#", "2#"],
"buildings": [
{"id": "1#", "status": "PASS", "artifacts": [glb.name]},
{"id": "2#", "status": "BLOCKED", "reason": "No readable source drawing", "artifacts": []},
],
"site": {"status": "PASS", "artifacts": [site.name]},
}
),
encoding="utf-8",
)
evidence = tmp_path / "evidence.json"
proc = run(
sys.executable,
"scripts/verify_delivery.py",
"--spec",
str(spec),
"--source-report",
str(source_report),
"--glb",
str(glb),
"--render",
str(render),
"--scope-manifest",
str(scope),
"--evidence-out",
str(evidence),
)
assert proc.returncode == 3
report = json.loads(evidence.read_text(encoding="utf-8"))
assert report["status"] == "BLOCKED"
assert any(item["name"] == "scope:building:2#" and item["status"] == "WARN" for item in report["checks"])
def test_scope_rejects_arbitrary_nonempty_artifact(tmp_path: Path) -> None:
spec, glb, _metadata = build_fixture(tmp_path)
render = tmp_path / "render.png"
render_fixture(render)
source_report = source_report_fixture(tmp_path)
fake = tmp_path / "looks-done.txt"
fake.write_text("not a model", encoding="utf-8")
scope = tmp_path / "scope_manifest.json"
scope.write_text(
json.dumps(
{
"scope_version": 1,
"mode": "single",
"requested": ["11#"],
"buildings": [{"id": "11#", "status": "PASS", "artifacts": [fake.name]}],
}
),
encoding="utf-8",
)
evidence = tmp_path / "evidence.json"
proc = run(
sys.executable,
"scripts/verify_delivery.py",
"--spec",
str(spec),
"--source-report",
str(source_report),
"--glb",
str(glb),
"--render",
str(render),
"--scope-manifest",
str(scope),
"--evidence-out",
str(evidence),
)
assert proc.returncode == 3
report = json.loads(evidence.read_text(encoding="utf-8"))
check = next(item for item in report["checks"] if item["name"] == "scope:building:11#")
assert check["status"] == "FAIL"
assert "No verifiable" in check["detail"]
def test_required_audit_and_scope_cannot_be_omitted(tmp_path: Path) -> None:
spec, glb, _metadata = build_fixture(tmp_path)
render = tmp_path / "render.png"
render_fixture(render)
source_report = source_report_fixture(tmp_path)
evidence = tmp_path / "evidence.json"
proc = run(
sys.executable,
"scripts/verify_delivery.py",
"--spec",
str(spec),
"--source-report",
str(source_report),
"--glb",
str(glb),
"--render",
str(render),
"--require-audit",
"--require-scope-manifest",
"--evidence-out",
str(evidence),
)
assert proc.returncode == 3
failed = {
item["name"] for item in json.loads(evidence.read_text(encoding="utf-8"))["checks"] if item["status"] == "FAIL"
}
assert {"audit:report", "scope:manifest"} <= failed
def test_variants_manifest_requires_metadata_artifact(tmp_path: Path) -> None:
spec, glb, _metadata = build_fixture(tmp_path)
render = tmp_path / "render.png"
render_fixture(render)
source_report = source_report_fixture(tmp_path)
manifest = tmp_path / "variants_manifest.json"
manifest.write_text(
json.dumps(
{
"manifest_version": 1,
"name": "missing-metadata",
"default_key": "standard",
"variants": [
{
"key": "standard",
"label": "standard",
"file": glb.name,
"source_glb": glb.name,
"metadata": "missing_metadata.json",
"spec": spec.name,
}
],
}
),
encoding="utf-8",
)
proc = run(
sys.executable,
"scripts/verify_delivery.py",
"--source-report",
str(source_report),
"--spec",
str(spec),
"--variants-manifest",
str(manifest),
"--render",
str(render),
"--evidence-out",
str(tmp_path / "evidence.json"),
)
assert proc.returncode == 3
assert "missing_metadata.json" in proc.stderr
def test_schema_validator_rejects_legacy_spec_in_strict_mode() -> None:
legacy = minimal_spec()
legacy.pop("schema_version")
try:
showcase_schema.validate_spec(legacy)
except showcase_schema.ValidationFailure:
return
raise AssertionError("strict validator accepted a spec without schema_version")