fix: close audited CAD delivery gate gaps

This commit is contained in:
2026-07-10 15:58:51 +08:00
parent a761175c09
commit 6c1336fa59
15 changed files with 713 additions and 124 deletions

View File

@@ -60,6 +60,34 @@ def source_report_fixture(tmp_path: Path) -> Path:
return path
def browser_report_payload(*, offline: bool = False) -> dict:
if offline:
checks = [
{"name": "offline:viewport", "status": "PASS", "detail": "1440x900 expected 1440x900"},
{"name": "offline:model-ready", "status": "PASS", "detail": "1 mesh(es)"},
{"name": "offline:canvas", "status": "PASS"},
{"name": "offline:render", "status": "PASS", "detail": "1 render(s)"},
]
else:
checks = []
for viewport, detail in (
("desktop", "1440x900 expected 1440x900"),
("mobile", "390x844 expected 390x844"),
):
checks.append({"name": f"{viewport}:viewport", "status": "PASS", "detail": detail})
for selector in ("#floor-controls", "#category-controls", "#explode", "#section", "#snapshot"):
checks.append({"name": f"{viewport}:{selector}", "status": "PASS"})
checks.append({"name": f"{viewport}:idle-render", "status": "PASS", "detail": "3->3"})
return {
"status": "PASS",
"detail": f"{len(checks)}/{len(checks)} checks passed",
"checks": checks,
"console_errors": [],
"page_errors": [],
"failed_requests": [],
}
def test_strict_delivery_passes_for_glb_render_and_offline_viewer(tmp_path: Path) -> None:
spec, glb, _metadata = build_fixture(tmp_path)
render = tmp_path / "render.png"
@@ -78,15 +106,7 @@ def test_strict_delivery_passes_for_glb_render_and_offline_viewer(tmp_path: Path
assert proc.returncode == 0, proc.stderr
offline_browser_report = tmp_path / "offline_browser_smoke_report.json"
offline_browser_report.write_text(
json.dumps(
{
"status": "PASS",
"detail": "4/4 checks passed",
"console_errors": [],
"page_errors": [],
"failed_requests": [],
}
),
json.dumps(browser_report_payload(offline=True)),
encoding="utf-8",
)
evidence = tmp_path / "evidence.json"
@@ -116,6 +136,43 @@ def test_strict_delivery_passes_for_glb_render_and_offline_viewer(tmp_path: Path
assert any(item["name"] == "offline:browser-smoke" and item["status"] == "PASS" for item in report["checks"])
def test_browser_report_cannot_forge_pass_without_required_checks(tmp_path: Path) -> None:
import verify_delivery
path = tmp_path / "forged.json"
path.write_text(
json.dumps(
{
"status": "PASS",
"checks": [],
"console_errors": [],
"page_errors": [],
"failed_requests": [],
}
),
encoding="utf-8",
)
checks: list[dict] = []
verify_delivery.inspect_browser_report(path, checks)
assert checks[0]["status"] == "FAIL"
assert "non-empty list" in checks[0]["detail"]
def test_render_gate_rejects_high_variance_noise(tmp_path: Path) -> None:
import verify_delivery
noisy = Image.new("RGB", (640, 480))
pixels = noisy.load()
for y in range(noisy.height):
for x in range(noisy.width):
pixels[x, y] = (255, 255, 255) if (x + y) % 2 else (0, 0, 0)
path = tmp_path / "noise.png"
noisy.save(path)
checks: list[dict] = []
verify_delivery.inspect_render(path, checks)
assert checks[0]["status"] == "FAIL"
def test_offline_viewer_without_real_browser_report_is_blocked(tmp_path: Path) -> None:
spec, glb, _metadata = build_fixture(tmp_path)
render = tmp_path / "render.png"
@@ -216,15 +273,7 @@ def test_web_writer_and_static_delivery_checks(tmp_path: Path) -> None:
assert "browser:smoke" in proc.stderr
browser_report = tmp_path / "browser_smoke_report.json"
browser_report.write_text(
json.dumps(
{
"status": "PASS",
"detail": "12/12 checks passed",
"console_errors": [],
"page_errors": [],
"failed_requests": [],
}
),
json.dumps(browser_report_payload()),
encoding="utf-8",
)
proc = run(
@@ -248,6 +297,44 @@ def test_web_writer_and_static_delivery_checks(tmp_path: Path) -> None:
assert proc.returncode == 0, proc.stderr
def test_reasoned_partial_source_can_only_pass_with_named_waiver(tmp_path: Path) -> None:
spec, glb, _metadata = build_fixture(tmp_path)
render = tmp_path / "render.png"
render_fixture(render)
source_report = source_report_fixture(tmp_path)
source = json.loads(source_report.read_text(encoding="utf-8"))
source["gate"] = {
"name": "all_sources_converted_and_readable",
"status": "WARN",
"detail": "one optional elevation is missing",
"partial_reason": "User approved plan-only delivery",
}
source_report.write_text(json.dumps(source), encoding="utf-8")
evidence = tmp_path / "evidence.json"
base = [
sys.executable,
"scripts/verify_delivery.py",
"--spec",
str(spec),
"--source-report",
str(source_report),
"--glb",
str(glb),
"--render",
str(render),
"--evidence-out",
str(evidence),
]
proc = run(*base)
assert proc.returncode == 3
assert json.loads(evidence.read_text(encoding="utf-8"))["status"] == "BLOCKED"
proc = run(*base, "--waiver", "cad_source_report=User approved plan-only delivery")
assert proc.returncode == 0, proc.stderr
report = json.loads(evidence.read_text(encoding="utf-8"))
source_check = next(item for item in report["checks"] if item["name"] == "cad_source_report")
assert source_check["original_status"] == "WARN"
def test_warn_or_skip_blocks_without_explicit_reasoned_waiver(tmp_path: Path) -> None:
spec, glb, _metadata = build_fixture(tmp_path)
render = tmp_path / "render.png"
@@ -342,6 +429,78 @@ def test_campus_scope_cannot_pass_with_a_missing_requested_building(tmp_path: Pa
assert any(item["name"] == "scope:building:2#" and item["status"] == "WARN" for item in report["checks"])
def test_scope_rejects_arbitrary_nonempty_artifact(tmp_path: Path) -> None:
spec, glb, _metadata = build_fixture(tmp_path)
render = tmp_path / "render.png"
render_fixture(render)
source_report = source_report_fixture(tmp_path)
fake = tmp_path / "looks-done.txt"
fake.write_text("not a model", encoding="utf-8")
scope = tmp_path / "scope_manifest.json"
scope.write_text(
json.dumps(
{
"scope_version": 1,
"mode": "single",
"requested": ["11#"],
"buildings": [{"id": "11#", "status": "PASS", "artifacts": [fake.name]}],
}
),
encoding="utf-8",
)
evidence = tmp_path / "evidence.json"
proc = run(
sys.executable,
"scripts/verify_delivery.py",
"--spec",
str(spec),
"--source-report",
str(source_report),
"--glb",
str(glb),
"--render",
str(render),
"--scope-manifest",
str(scope),
"--evidence-out",
str(evidence),
)
assert proc.returncode == 3
report = json.loads(evidence.read_text(encoding="utf-8"))
check = next(item for item in report["checks"] if item["name"] == "scope:building:11#")
assert check["status"] == "FAIL"
assert "No verifiable" in check["detail"]
def test_required_audit_and_scope_cannot_be_omitted(tmp_path: Path) -> None:
spec, glb, _metadata = build_fixture(tmp_path)
render = tmp_path / "render.png"
render_fixture(render)
source_report = source_report_fixture(tmp_path)
evidence = tmp_path / "evidence.json"
proc = run(
sys.executable,
"scripts/verify_delivery.py",
"--spec",
str(spec),
"--source-report",
str(source_report),
"--glb",
str(glb),
"--render",
str(render),
"--require-audit",
"--require-scope-manifest",
"--evidence-out",
str(evidence),
)
assert proc.returncode == 3
failed = {
item["name"] for item in json.loads(evidence.read_text(encoding="utf-8"))["checks"] if item["status"] == "FAIL"
}
assert {"audit:report", "scope:manifest"} <= failed
def test_variants_manifest_requires_metadata_artifact(tmp_path: Path) -> None:
spec, glb, _metadata = build_fixture(tmp_path)
render = tmp_path / "render.png"