fix: close audited CAD delivery gate gaps
This commit is contained in:
@@ -60,6 +60,34 @@ def source_report_fixture(tmp_path: Path) -> Path:
|
||||
return path
|
||||
|
||||
|
||||
def browser_report_payload(*, offline: bool = False) -> dict:
|
||||
if offline:
|
||||
checks = [
|
||||
{"name": "offline:viewport", "status": "PASS", "detail": "1440x900 expected 1440x900"},
|
||||
{"name": "offline:model-ready", "status": "PASS", "detail": "1 mesh(es)"},
|
||||
{"name": "offline:canvas", "status": "PASS"},
|
||||
{"name": "offline:render", "status": "PASS", "detail": "1 render(s)"},
|
||||
]
|
||||
else:
|
||||
checks = []
|
||||
for viewport, detail in (
|
||||
("desktop", "1440x900 expected 1440x900"),
|
||||
("mobile", "390x844 expected 390x844"),
|
||||
):
|
||||
checks.append({"name": f"{viewport}:viewport", "status": "PASS", "detail": detail})
|
||||
for selector in ("#floor-controls", "#category-controls", "#explode", "#section", "#snapshot"):
|
||||
checks.append({"name": f"{viewport}:{selector}", "status": "PASS"})
|
||||
checks.append({"name": f"{viewport}:idle-render", "status": "PASS", "detail": "3->3"})
|
||||
return {
|
||||
"status": "PASS",
|
||||
"detail": f"{len(checks)}/{len(checks)} checks passed",
|
||||
"checks": checks,
|
||||
"console_errors": [],
|
||||
"page_errors": [],
|
||||
"failed_requests": [],
|
||||
}
|
||||
|
||||
|
||||
def test_strict_delivery_passes_for_glb_render_and_offline_viewer(tmp_path: Path) -> None:
|
||||
spec, glb, _metadata = build_fixture(tmp_path)
|
||||
render = tmp_path / "render.png"
|
||||
@@ -78,15 +106,7 @@ def test_strict_delivery_passes_for_glb_render_and_offline_viewer(tmp_path: Path
|
||||
assert proc.returncode == 0, proc.stderr
|
||||
offline_browser_report = tmp_path / "offline_browser_smoke_report.json"
|
||||
offline_browser_report.write_text(
|
||||
json.dumps(
|
||||
{
|
||||
"status": "PASS",
|
||||
"detail": "4/4 checks passed",
|
||||
"console_errors": [],
|
||||
"page_errors": [],
|
||||
"failed_requests": [],
|
||||
}
|
||||
),
|
||||
json.dumps(browser_report_payload(offline=True)),
|
||||
encoding="utf-8",
|
||||
)
|
||||
evidence = tmp_path / "evidence.json"
|
||||
@@ -116,6 +136,43 @@ def test_strict_delivery_passes_for_glb_render_and_offline_viewer(tmp_path: Path
|
||||
assert any(item["name"] == "offline:browser-smoke" and item["status"] == "PASS" for item in report["checks"])
|
||||
|
||||
|
||||
def test_browser_report_cannot_forge_pass_without_required_checks(tmp_path: Path) -> None:
|
||||
import verify_delivery
|
||||
|
||||
path = tmp_path / "forged.json"
|
||||
path.write_text(
|
||||
json.dumps(
|
||||
{
|
||||
"status": "PASS",
|
||||
"checks": [],
|
||||
"console_errors": [],
|
||||
"page_errors": [],
|
||||
"failed_requests": [],
|
||||
}
|
||||
),
|
||||
encoding="utf-8",
|
||||
)
|
||||
checks: list[dict] = []
|
||||
verify_delivery.inspect_browser_report(path, checks)
|
||||
assert checks[0]["status"] == "FAIL"
|
||||
assert "non-empty list" in checks[0]["detail"]
|
||||
|
||||
|
||||
def test_render_gate_rejects_high_variance_noise(tmp_path: Path) -> None:
|
||||
import verify_delivery
|
||||
|
||||
noisy = Image.new("RGB", (640, 480))
|
||||
pixels = noisy.load()
|
||||
for y in range(noisy.height):
|
||||
for x in range(noisy.width):
|
||||
pixels[x, y] = (255, 255, 255) if (x + y) % 2 else (0, 0, 0)
|
||||
path = tmp_path / "noise.png"
|
||||
noisy.save(path)
|
||||
checks: list[dict] = []
|
||||
verify_delivery.inspect_render(path, checks)
|
||||
assert checks[0]["status"] == "FAIL"
|
||||
|
||||
|
||||
def test_offline_viewer_without_real_browser_report_is_blocked(tmp_path: Path) -> None:
|
||||
spec, glb, _metadata = build_fixture(tmp_path)
|
||||
render = tmp_path / "render.png"
|
||||
@@ -216,15 +273,7 @@ def test_web_writer_and_static_delivery_checks(tmp_path: Path) -> None:
|
||||
assert "browser:smoke" in proc.stderr
|
||||
browser_report = tmp_path / "browser_smoke_report.json"
|
||||
browser_report.write_text(
|
||||
json.dumps(
|
||||
{
|
||||
"status": "PASS",
|
||||
"detail": "12/12 checks passed",
|
||||
"console_errors": [],
|
||||
"page_errors": [],
|
||||
"failed_requests": [],
|
||||
}
|
||||
),
|
||||
json.dumps(browser_report_payload()),
|
||||
encoding="utf-8",
|
||||
)
|
||||
proc = run(
|
||||
@@ -248,6 +297,44 @@ def test_web_writer_and_static_delivery_checks(tmp_path: Path) -> None:
|
||||
assert proc.returncode == 0, proc.stderr
|
||||
|
||||
|
||||
def test_reasoned_partial_source_can_only_pass_with_named_waiver(tmp_path: Path) -> None:
|
||||
spec, glb, _metadata = build_fixture(tmp_path)
|
||||
render = tmp_path / "render.png"
|
||||
render_fixture(render)
|
||||
source_report = source_report_fixture(tmp_path)
|
||||
source = json.loads(source_report.read_text(encoding="utf-8"))
|
||||
source["gate"] = {
|
||||
"name": "all_sources_converted_and_readable",
|
||||
"status": "WARN",
|
||||
"detail": "one optional elevation is missing",
|
||||
"partial_reason": "User approved plan-only delivery",
|
||||
}
|
||||
source_report.write_text(json.dumps(source), encoding="utf-8")
|
||||
evidence = tmp_path / "evidence.json"
|
||||
base = [
|
||||
sys.executable,
|
||||
"scripts/verify_delivery.py",
|
||||
"--spec",
|
||||
str(spec),
|
||||
"--source-report",
|
||||
str(source_report),
|
||||
"--glb",
|
||||
str(glb),
|
||||
"--render",
|
||||
str(render),
|
||||
"--evidence-out",
|
||||
str(evidence),
|
||||
]
|
||||
proc = run(*base)
|
||||
assert proc.returncode == 3
|
||||
assert json.loads(evidence.read_text(encoding="utf-8"))["status"] == "BLOCKED"
|
||||
proc = run(*base, "--waiver", "cad_source_report=User approved plan-only delivery")
|
||||
assert proc.returncode == 0, proc.stderr
|
||||
report = json.loads(evidence.read_text(encoding="utf-8"))
|
||||
source_check = next(item for item in report["checks"] if item["name"] == "cad_source_report")
|
||||
assert source_check["original_status"] == "WARN"
|
||||
|
||||
|
||||
def test_warn_or_skip_blocks_without_explicit_reasoned_waiver(tmp_path: Path) -> None:
|
||||
spec, glb, _metadata = build_fixture(tmp_path)
|
||||
render = tmp_path / "render.png"
|
||||
@@ -342,6 +429,78 @@ def test_campus_scope_cannot_pass_with_a_missing_requested_building(tmp_path: Pa
|
||||
assert any(item["name"] == "scope:building:2#" and item["status"] == "WARN" for item in report["checks"])
|
||||
|
||||
|
||||
def test_scope_rejects_arbitrary_nonempty_artifact(tmp_path: Path) -> None:
|
||||
spec, glb, _metadata = build_fixture(tmp_path)
|
||||
render = tmp_path / "render.png"
|
||||
render_fixture(render)
|
||||
source_report = source_report_fixture(tmp_path)
|
||||
fake = tmp_path / "looks-done.txt"
|
||||
fake.write_text("not a model", encoding="utf-8")
|
||||
scope = tmp_path / "scope_manifest.json"
|
||||
scope.write_text(
|
||||
json.dumps(
|
||||
{
|
||||
"scope_version": 1,
|
||||
"mode": "single",
|
||||
"requested": ["11#"],
|
||||
"buildings": [{"id": "11#", "status": "PASS", "artifacts": [fake.name]}],
|
||||
}
|
||||
),
|
||||
encoding="utf-8",
|
||||
)
|
||||
evidence = tmp_path / "evidence.json"
|
||||
proc = run(
|
||||
sys.executable,
|
||||
"scripts/verify_delivery.py",
|
||||
"--spec",
|
||||
str(spec),
|
||||
"--source-report",
|
||||
str(source_report),
|
||||
"--glb",
|
||||
str(glb),
|
||||
"--render",
|
||||
str(render),
|
||||
"--scope-manifest",
|
||||
str(scope),
|
||||
"--evidence-out",
|
||||
str(evidence),
|
||||
)
|
||||
assert proc.returncode == 3
|
||||
report = json.loads(evidence.read_text(encoding="utf-8"))
|
||||
check = next(item for item in report["checks"] if item["name"] == "scope:building:11#")
|
||||
assert check["status"] == "FAIL"
|
||||
assert "No verifiable" in check["detail"]
|
||||
|
||||
|
||||
def test_required_audit_and_scope_cannot_be_omitted(tmp_path: Path) -> None:
|
||||
spec, glb, _metadata = build_fixture(tmp_path)
|
||||
render = tmp_path / "render.png"
|
||||
render_fixture(render)
|
||||
source_report = source_report_fixture(tmp_path)
|
||||
evidence = tmp_path / "evidence.json"
|
||||
proc = run(
|
||||
sys.executable,
|
||||
"scripts/verify_delivery.py",
|
||||
"--spec",
|
||||
str(spec),
|
||||
"--source-report",
|
||||
str(source_report),
|
||||
"--glb",
|
||||
str(glb),
|
||||
"--render",
|
||||
str(render),
|
||||
"--require-audit",
|
||||
"--require-scope-manifest",
|
||||
"--evidence-out",
|
||||
str(evidence),
|
||||
)
|
||||
assert proc.returncode == 3
|
||||
failed = {
|
||||
item["name"] for item in json.loads(evidence.read_text(encoding="utf-8"))["checks"] if item["status"] == "FAIL"
|
||||
}
|
||||
assert {"audit:report", "scope:manifest"} <= failed
|
||||
|
||||
|
||||
def test_variants_manifest_requires_metadata_artifact(tmp_path: Path) -> None:
|
||||
spec, glb, _metadata = build_fixture(tmp_path)
|
||||
render = tmp_path / "render.png"
|
||||
|
||||
Reference in New Issue
Block a user