609 lines
19 KiB
Python
609 lines
19 KiB
Python
from __future__ import annotations
|
|
|
|
import json
|
|
import subprocess
|
|
import sys
|
|
from pathlib import Path
|
|
|
|
from PIL import Image, ImageDraw
|
|
|
|
ROOT = Path(__file__).resolve().parents[1]
|
|
sys.path.insert(0, str(ROOT / "scripts"))
|
|
|
|
import showcase_schema # noqa: E402
|
|
from test_contracts import minimal_spec # noqa: E402
|
|
|
|
|
|
def run(*args: str) -> subprocess.CompletedProcess[str]:
|
|
return subprocess.run(list(args), cwd=ROOT, text=True, capture_output=True)
|
|
|
|
|
|
def build_fixture(tmp_path: Path) -> tuple[Path, Path, Path]:
|
|
spec = tmp_path / "spec.json"
|
|
glb = tmp_path / "model.glb"
|
|
metadata = tmp_path / "model_metadata.json"
|
|
spec.write_text(json.dumps(minimal_spec()), encoding="utf-8")
|
|
proc = run(
|
|
sys.executable,
|
|
"scripts/spec_to_glb.py",
|
|
"--spec",
|
|
str(spec),
|
|
"--out",
|
|
str(glb),
|
|
"--metadata-out",
|
|
str(metadata),
|
|
)
|
|
assert proc.returncode == 0, proc.stderr
|
|
return spec, glb, metadata
|
|
|
|
|
|
def render_fixture(path: Path) -> None:
|
|
image = Image.new("RGB", (640, 480), "white")
|
|
draw = ImageDraw.Draw(image)
|
|
draw.rectangle((100, 120, 540, 400), fill="#4477aa", outline="black", width=4)
|
|
image.save(path)
|
|
|
|
|
|
def source_report_fixture(tmp_path: Path) -> Path:
|
|
path = tmp_path / "cad_read_report.json"
|
|
path.write_text(
|
|
json.dumps(
|
|
{
|
|
"report_version": 1,
|
|
"dxf_sanitized": str(tmp_path),
|
|
"gate": {"name": "all_sources_converted_and_readable", "status": "PASS"},
|
|
"files": [{"file": "fixture.dxf", "status": "ok", "modelspace_entities": 1}],
|
|
}
|
|
),
|
|
encoding="utf-8",
|
|
)
|
|
return path
|
|
|
|
|
|
def browser_report_payload(*, offline: bool = False) -> dict:
|
|
if offline:
|
|
checks = [
|
|
{"name": "offline:viewport", "status": "PASS", "detail": "1440x900 expected 1440x900"},
|
|
{"name": "offline:model-ready", "status": "PASS", "detail": "1 mesh(es)"},
|
|
{"name": "offline:canvas", "status": "PASS"},
|
|
{"name": "offline:render", "status": "PASS", "detail": "1 render(s)"},
|
|
]
|
|
else:
|
|
checks = []
|
|
for viewport, detail in (
|
|
("desktop", "1440x900 expected 1440x900"),
|
|
("mobile", "390x844 expected 390x844"),
|
|
):
|
|
checks.append({"name": f"{viewport}:viewport", "status": "PASS", "detail": detail})
|
|
for selector in ("#floor-controls", "#category-controls", "#explode", "#section", "#snapshot"):
|
|
checks.append({"name": f"{viewport}:{selector}", "status": "PASS"})
|
|
checks.append({"name": f"{viewport}:idle-render", "status": "PASS", "detail": "3->3"})
|
|
return {
|
|
"status": "PASS",
|
|
"detail": f"{len(checks)}/{len(checks)} checks passed",
|
|
"checks": checks,
|
|
"console_errors": [],
|
|
"page_errors": [],
|
|
"failed_requests": [],
|
|
}
|
|
|
|
|
|
def test_strict_delivery_passes_for_glb_render_and_offline_viewer(tmp_path: Path) -> None:
|
|
spec, glb, _metadata = build_fixture(tmp_path)
|
|
render = tmp_path / "render.png"
|
|
render_fixture(render)
|
|
offline = tmp_path / "viewer.html"
|
|
proc = run(
|
|
sys.executable,
|
|
"scripts/write_offline_viewer.py",
|
|
"--spec",
|
|
str(spec),
|
|
"--glb",
|
|
str(glb),
|
|
"--out",
|
|
str(offline),
|
|
)
|
|
assert proc.returncode == 0, proc.stderr
|
|
offline_browser_report = tmp_path / "offline_browser_smoke_report.json"
|
|
offline_browser_report.write_text(
|
|
json.dumps(browser_report_payload(offline=True)),
|
|
encoding="utf-8",
|
|
)
|
|
evidence = tmp_path / "evidence.json"
|
|
source_report = source_report_fixture(tmp_path)
|
|
proc = run(
|
|
sys.executable,
|
|
"scripts/verify_delivery.py",
|
|
"--spec",
|
|
str(spec),
|
|
"--source-report",
|
|
str(source_report),
|
|
"--glb",
|
|
str(glb),
|
|
"--render",
|
|
str(render),
|
|
"--offline-viewer",
|
|
str(offline),
|
|
"--offline-browser-report",
|
|
str(offline_browser_report),
|
|
"--evidence-out",
|
|
str(evidence),
|
|
)
|
|
assert proc.returncode == 0, proc.stderr
|
|
report = json.loads(evidence.read_text(encoding="utf-8"))
|
|
assert report["status"] == "PASS"
|
|
assert any(item["name"] == "offline:payload" and item["status"] == "PASS" for item in report["checks"])
|
|
assert any(item["name"] == "offline:browser-smoke" and item["status"] == "PASS" for item in report["checks"])
|
|
|
|
|
|
def test_browser_report_cannot_forge_pass_without_required_checks(tmp_path: Path) -> None:
|
|
import verify_delivery
|
|
|
|
path = tmp_path / "forged.json"
|
|
path.write_text(
|
|
json.dumps(
|
|
{
|
|
"status": "PASS",
|
|
"checks": [],
|
|
"console_errors": [],
|
|
"page_errors": [],
|
|
"failed_requests": [],
|
|
}
|
|
),
|
|
encoding="utf-8",
|
|
)
|
|
checks: list[dict] = []
|
|
verify_delivery.inspect_browser_report(path, checks)
|
|
assert checks[0]["status"] == "FAIL"
|
|
assert "non-empty list" in checks[0]["detail"]
|
|
|
|
|
|
def test_render_gate_rejects_high_variance_noise(tmp_path: Path) -> None:
|
|
import verify_delivery
|
|
|
|
noisy = Image.new("RGB", (640, 480))
|
|
pixels = noisy.load()
|
|
for y in range(noisy.height):
|
|
for x in range(noisy.width):
|
|
pixels[x, y] = (255, 255, 255) if (x + y) % 2 else (0, 0, 0)
|
|
path = tmp_path / "noise.png"
|
|
noisy.save(path)
|
|
checks: list[dict] = []
|
|
verify_delivery.inspect_render(path, checks)
|
|
assert checks[0]["status"] == "FAIL"
|
|
|
|
|
|
def test_offline_viewer_without_real_browser_report_is_blocked(tmp_path: Path) -> None:
|
|
spec, glb, _metadata = build_fixture(tmp_path)
|
|
render = tmp_path / "render.png"
|
|
render_fixture(render)
|
|
offline = tmp_path / "viewer.html"
|
|
proc = run(
|
|
sys.executable,
|
|
"scripts/write_offline_viewer.py",
|
|
"--spec",
|
|
str(spec),
|
|
"--glb",
|
|
str(glb),
|
|
"--out",
|
|
str(offline),
|
|
)
|
|
assert proc.returncode == 0, proc.stderr
|
|
source_report = source_report_fixture(tmp_path)
|
|
evidence = tmp_path / "evidence.json"
|
|
proc = run(
|
|
sys.executable,
|
|
"scripts/verify_delivery.py",
|
|
"--spec",
|
|
str(spec),
|
|
"--source-report",
|
|
str(source_report),
|
|
"--glb",
|
|
str(glb),
|
|
"--render",
|
|
str(render),
|
|
"--offline-viewer",
|
|
str(offline),
|
|
"--evidence-out",
|
|
str(evidence),
|
|
)
|
|
assert proc.returncode == 3
|
|
report = json.loads(evidence.read_text(encoding="utf-8"))
|
|
assert report["status"] == "BLOCKED"
|
|
assert any(item["name"] == "offline:browser-smoke" and item["status"] == "SKIP" for item in report["checks"])
|
|
|
|
|
|
def test_offline_viewer_escapes_html_and_inline_script_data(tmp_path: Path) -> None:
|
|
spec, glb, _metadata = build_fixture(tmp_path)
|
|
document = json.loads(spec.read_text(encoding="utf-8"))
|
|
document["project"]["name"] = '<Tower & Co></script><script>alert("title")</script>'
|
|
document["floors"]["F1"]["name"] = '首层</script><script>alert("floor")</script>'
|
|
document["categories"]["walls"]["label"] = "墙<&>"
|
|
spec.write_text(json.dumps(document), encoding="utf-8")
|
|
offline = tmp_path / "viewer.html"
|
|
proc = run(
|
|
sys.executable,
|
|
"scripts/write_offline_viewer.py",
|
|
"--spec",
|
|
str(spec),
|
|
"--glb",
|
|
str(glb),
|
|
"--out",
|
|
str(offline),
|
|
"--subtitle",
|
|
"<unsafe & subtitle>",
|
|
)
|
|
assert proc.returncode == 0, proc.stderr
|
|
text = offline.read_text(encoding="utf-8")
|
|
assert "<Tower & Co></script>" in text
|
|
assert "<unsafe & subtitle>" in text
|
|
assert '首层<\\/script><script>alert(\\"floor\\")<\\/script>' in text
|
|
assert '首层</script><script>alert("floor")</script>' not in text
|
|
|
|
|
|
def test_web_variant_entry_requires_a_nonempty_file(tmp_path: Path) -> None:
|
|
import verify_delivery
|
|
|
|
web = tmp_path / "web"
|
|
for relative in (
|
|
"index.html",
|
|
"app.js",
|
|
"styles.css",
|
|
"model.glb",
|
|
"node_modules/three/build/three.module.js",
|
|
"node_modules/lucide/dist/umd/lucide.js",
|
|
):
|
|
target = web / relative
|
|
target.parent.mkdir(parents=True, exist_ok=True)
|
|
target.write_text(
|
|
"window.__renders; invalidate(); DRACOLoader; restoreStateFromHash; switchVariant", encoding="utf-8"
|
|
)
|
|
(web / "model_metadata.json").write_text(json.dumps({"model_variants": [{"key": "standard", "file": ""}]}))
|
|
checks: list[dict] = []
|
|
verify_delivery.inspect_web(web, checks)
|
|
variant_check = next(item for item in checks if item["name"] == "web:variants")
|
|
assert variant_check["status"] == "FAIL"
|
|
assert "<missing file>" in variant_check["detail"]
|
|
|
|
|
|
def test_strict_delivery_requires_a_render(tmp_path: Path) -> None:
|
|
spec, glb, _metadata = build_fixture(tmp_path)
|
|
source_report = source_report_fixture(tmp_path)
|
|
proc = run(
|
|
sys.executable,
|
|
"scripts/verify_delivery.py",
|
|
"--spec",
|
|
str(spec),
|
|
"--source-report",
|
|
str(source_report),
|
|
"--glb",
|
|
str(glb),
|
|
"--evidence-out",
|
|
str(tmp_path / "evidence.json"),
|
|
)
|
|
assert proc.returncode == 3
|
|
assert "At least one inspected render is required" in proc.stderr
|
|
|
|
|
|
def test_web_writer_and_static_delivery_checks(tmp_path: Path) -> None:
|
|
spec, glb, metadata = build_fixture(tmp_path)
|
|
web = tmp_path / "web"
|
|
proc = run(
|
|
sys.executable,
|
|
"scripts/write_web_showcase.py",
|
|
"--glb",
|
|
str(glb),
|
|
"--metadata",
|
|
str(metadata),
|
|
"--out-dir",
|
|
str(web),
|
|
)
|
|
assert proc.returncode == 0, proc.stderr
|
|
for relative in ("node_modules/three/build/three.module.js", "node_modules/lucide/dist/umd/lucide.js"):
|
|
target = web / relative
|
|
target.parent.mkdir(parents=True, exist_ok=True)
|
|
target.write_text("// test dependency", encoding="utf-8")
|
|
render = tmp_path / "render.png"
|
|
render_fixture(render)
|
|
source_report = source_report_fixture(tmp_path)
|
|
proc = run(
|
|
sys.executable,
|
|
"scripts/verify_delivery.py",
|
|
"--spec",
|
|
str(spec),
|
|
"--source-report",
|
|
str(source_report),
|
|
"--glb",
|
|
str(glb),
|
|
"--render",
|
|
str(render),
|
|
"--web-dir",
|
|
str(web),
|
|
"--evidence-out",
|
|
str(tmp_path / "web_evidence.json"),
|
|
)
|
|
assert proc.returncode == 3
|
|
assert "browser:smoke" in proc.stderr
|
|
browser_report = tmp_path / "browser_smoke_report.json"
|
|
browser_report.write_text(
|
|
json.dumps(browser_report_payload()),
|
|
encoding="utf-8",
|
|
)
|
|
proc = run(
|
|
sys.executable,
|
|
"scripts/verify_delivery.py",
|
|
"--spec",
|
|
str(spec),
|
|
"--source-report",
|
|
str(source_report),
|
|
"--glb",
|
|
str(glb),
|
|
"--render",
|
|
str(render),
|
|
"--web-dir",
|
|
str(web),
|
|
"--browser-report",
|
|
str(browser_report),
|
|
"--evidence-out",
|
|
str(tmp_path / "web_evidence.json"),
|
|
)
|
|
assert proc.returncode == 0, proc.stderr
|
|
|
|
|
|
def test_reasoned_partial_source_can_only_pass_with_named_waiver(tmp_path: Path) -> None:
|
|
spec, glb, _metadata = build_fixture(tmp_path)
|
|
render = tmp_path / "render.png"
|
|
render_fixture(render)
|
|
source_report = source_report_fixture(tmp_path)
|
|
source = json.loads(source_report.read_text(encoding="utf-8"))
|
|
source["gate"] = {
|
|
"name": "all_sources_converted_and_readable",
|
|
"status": "WARN",
|
|
"detail": "one optional elevation is missing",
|
|
"partial_reason": "User approved plan-only delivery",
|
|
}
|
|
source_report.write_text(json.dumps(source), encoding="utf-8")
|
|
evidence = tmp_path / "evidence.json"
|
|
base = [
|
|
sys.executable,
|
|
"scripts/verify_delivery.py",
|
|
"--spec",
|
|
str(spec),
|
|
"--source-report",
|
|
str(source_report),
|
|
"--glb",
|
|
str(glb),
|
|
"--render",
|
|
str(render),
|
|
"--evidence-out",
|
|
str(evidence),
|
|
]
|
|
proc = run(*base)
|
|
assert proc.returncode == 3
|
|
assert json.loads(evidence.read_text(encoding="utf-8"))["status"] == "BLOCKED"
|
|
proc = run(*base, "--waiver", "cad_source_report=User approved plan-only delivery")
|
|
assert proc.returncode == 0, proc.stderr
|
|
report = json.loads(evidence.read_text(encoding="utf-8"))
|
|
source_check = next(item for item in report["checks"] if item["name"] == "cad_source_report")
|
|
assert source_check["original_status"] == "WARN"
|
|
|
|
|
|
def test_warn_or_skip_blocks_without_explicit_reasoned_waiver(tmp_path: Path) -> None:
|
|
spec, glb, _metadata = build_fixture(tmp_path)
|
|
render = tmp_path / "render.png"
|
|
render_fixture(render)
|
|
source_report = source_report_fixture(tmp_path)
|
|
evidence = tmp_path / "evidence.json"
|
|
proc = run(
|
|
sys.executable,
|
|
"scripts/verify_delivery.py",
|
|
"--spec",
|
|
str(spec),
|
|
"--source-report",
|
|
str(source_report),
|
|
"--glb",
|
|
str(glb),
|
|
"--render",
|
|
str(render),
|
|
"--require-browser-report",
|
|
"--evidence-out",
|
|
str(evidence),
|
|
)
|
|
assert proc.returncode == 3
|
|
assert json.loads(evidence.read_text(encoding="utf-8"))["status"] == "BLOCKED"
|
|
proc = run(
|
|
sys.executable,
|
|
"scripts/verify_delivery.py",
|
|
"--spec",
|
|
str(spec),
|
|
"--source-report",
|
|
str(source_report),
|
|
"--glb",
|
|
str(glb),
|
|
"--render",
|
|
str(render),
|
|
"--require-browser-report",
|
|
"--waiver",
|
|
"browser:smoke=No browser is available in this offline-only benchmark",
|
|
"--evidence-out",
|
|
str(evidence),
|
|
)
|
|
assert proc.returncode == 0, proc.stderr
|
|
report = json.loads(evidence.read_text(encoding="utf-8"))
|
|
smoke = next(item for item in report["checks"] if item["name"] == "browser:smoke")
|
|
assert smoke["status"] == "PASS"
|
|
assert smoke["original_status"] == "SKIP"
|
|
assert smoke["waiver_reason"]
|
|
|
|
|
|
def test_campus_scope_cannot_pass_with_a_missing_requested_building(tmp_path: Path) -> None:
|
|
spec, glb, _metadata = build_fixture(tmp_path)
|
|
render = tmp_path / "render.png"
|
|
render_fixture(render)
|
|
source_report = source_report_fixture(tmp_path)
|
|
site = tmp_path / "site.png"
|
|
render_fixture(site)
|
|
scope = tmp_path / "scope_manifest.json"
|
|
scope.write_text(
|
|
json.dumps(
|
|
{
|
|
"scope_version": 1,
|
|
"mode": "campus",
|
|
"requested": ["1#", "2#"],
|
|
"buildings": [
|
|
{"id": "1#", "status": "PASS", "artifacts": [glb.name]},
|
|
{"id": "2#", "status": "BLOCKED", "reason": "No readable source drawing", "artifacts": []},
|
|
],
|
|
"site": {"status": "PASS", "artifacts": [site.name]},
|
|
}
|
|
),
|
|
encoding="utf-8",
|
|
)
|
|
evidence = tmp_path / "evidence.json"
|
|
proc = run(
|
|
sys.executable,
|
|
"scripts/verify_delivery.py",
|
|
"--spec",
|
|
str(spec),
|
|
"--source-report",
|
|
str(source_report),
|
|
"--glb",
|
|
str(glb),
|
|
"--render",
|
|
str(render),
|
|
"--scope-manifest",
|
|
str(scope),
|
|
"--evidence-out",
|
|
str(evidence),
|
|
)
|
|
assert proc.returncode == 3
|
|
report = json.loads(evidence.read_text(encoding="utf-8"))
|
|
assert report["status"] == "BLOCKED"
|
|
assert any(item["name"] == "scope:building:2#" and item["status"] == "WARN" for item in report["checks"])
|
|
|
|
|
|
def test_scope_rejects_arbitrary_nonempty_artifact(tmp_path: Path) -> None:
|
|
spec, glb, _metadata = build_fixture(tmp_path)
|
|
render = tmp_path / "render.png"
|
|
render_fixture(render)
|
|
source_report = source_report_fixture(tmp_path)
|
|
fake = tmp_path / "looks-done.txt"
|
|
fake.write_text("not a model", encoding="utf-8")
|
|
scope = tmp_path / "scope_manifest.json"
|
|
scope.write_text(
|
|
json.dumps(
|
|
{
|
|
"scope_version": 1,
|
|
"mode": "single",
|
|
"requested": ["11#"],
|
|
"buildings": [{"id": "11#", "status": "PASS", "artifacts": [fake.name]}],
|
|
}
|
|
),
|
|
encoding="utf-8",
|
|
)
|
|
evidence = tmp_path / "evidence.json"
|
|
proc = run(
|
|
sys.executable,
|
|
"scripts/verify_delivery.py",
|
|
"--spec",
|
|
str(spec),
|
|
"--source-report",
|
|
str(source_report),
|
|
"--glb",
|
|
str(glb),
|
|
"--render",
|
|
str(render),
|
|
"--scope-manifest",
|
|
str(scope),
|
|
"--evidence-out",
|
|
str(evidence),
|
|
)
|
|
assert proc.returncode == 3
|
|
report = json.loads(evidence.read_text(encoding="utf-8"))
|
|
check = next(item for item in report["checks"] if item["name"] == "scope:building:11#")
|
|
assert check["status"] == "FAIL"
|
|
assert "No verifiable" in check["detail"]
|
|
|
|
|
|
def test_required_audit_and_scope_cannot_be_omitted(tmp_path: Path) -> None:
|
|
spec, glb, _metadata = build_fixture(tmp_path)
|
|
render = tmp_path / "render.png"
|
|
render_fixture(render)
|
|
source_report = source_report_fixture(tmp_path)
|
|
evidence = tmp_path / "evidence.json"
|
|
proc = run(
|
|
sys.executable,
|
|
"scripts/verify_delivery.py",
|
|
"--spec",
|
|
str(spec),
|
|
"--source-report",
|
|
str(source_report),
|
|
"--glb",
|
|
str(glb),
|
|
"--render",
|
|
str(render),
|
|
"--require-audit",
|
|
"--require-scope-manifest",
|
|
"--evidence-out",
|
|
str(evidence),
|
|
)
|
|
assert proc.returncode == 3
|
|
failed = {
|
|
item["name"] for item in json.loads(evidence.read_text(encoding="utf-8"))["checks"] if item["status"] == "FAIL"
|
|
}
|
|
assert {"audit:report", "scope:manifest"} <= failed
|
|
|
|
|
|
def test_variants_manifest_requires_metadata_artifact(tmp_path: Path) -> None:
|
|
spec, glb, _metadata = build_fixture(tmp_path)
|
|
render = tmp_path / "render.png"
|
|
render_fixture(render)
|
|
source_report = source_report_fixture(tmp_path)
|
|
manifest = tmp_path / "variants_manifest.json"
|
|
manifest.write_text(
|
|
json.dumps(
|
|
{
|
|
"manifest_version": 1,
|
|
"name": "missing-metadata",
|
|
"default_key": "standard",
|
|
"variants": [
|
|
{
|
|
"key": "standard",
|
|
"label": "standard",
|
|
"file": glb.name,
|
|
"source_glb": glb.name,
|
|
"metadata": "missing_metadata.json",
|
|
"spec": spec.name,
|
|
}
|
|
],
|
|
}
|
|
),
|
|
encoding="utf-8",
|
|
)
|
|
proc = run(
|
|
sys.executable,
|
|
"scripts/verify_delivery.py",
|
|
"--source-report",
|
|
str(source_report),
|
|
"--spec",
|
|
str(spec),
|
|
"--variants-manifest",
|
|
str(manifest),
|
|
"--render",
|
|
str(render),
|
|
"--evidence-out",
|
|
str(tmp_path / "evidence.json"),
|
|
)
|
|
assert proc.returncode == 3
|
|
assert "missing_metadata.json" in proc.stderr
|
|
|
|
|
|
def test_schema_validator_rejects_legacy_spec_in_strict_mode() -> None:
|
|
legacy = minimal_spec()
|
|
legacy.pop("schema_version")
|
|
try:
|
|
showcase_schema.validate_spec(legacy)
|
|
except showcase_schema.ValidationFailure:
|
|
return
|
|
raise AssertionError("strict validator accepted a spec without schema_version")
|