fix: close audited CAD delivery gate gaps
This commit is contained in:
@@ -2,6 +2,7 @@ from __future__ import annotations
|
||||
|
||||
import json
|
||||
import os
|
||||
import shutil
|
||||
import socket
|
||||
import subprocess
|
||||
import sys
|
||||
@@ -50,7 +51,9 @@ def test_served_viewer_in_desktop_and_mobile(tmp_path: Path) -> None:
|
||||
],
|
||||
check=True,
|
||||
)
|
||||
subprocess.run(["npm", "install", "--prefix", str(web), "--ignore-scripts"], check=True)
|
||||
npm = shutil.which("npm.cmd" if sys.platform == "win32" else "npm") or shutil.which("npm")
|
||||
assert npm, "npm executable not found"
|
||||
subprocess.run([npm, "install", "--prefix", str(web), "--ignore-scripts"], check=True)
|
||||
with socket.socket() as sock:
|
||||
sock.bind(("127.0.0.1", 0))
|
||||
port = sock.getsockname()[1]
|
||||
|
||||
@@ -11,8 +11,8 @@ import ezdxf
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
sys.path.insert(0, str(ROOT / "scripts"))
|
||||
|
||||
import build_blender_showcase # noqa: E402
|
||||
import audit_showcase_extraction # noqa: E402
|
||||
import build_blender_showcase # noqa: E402
|
||||
import cad_showcase # noqa: E402
|
||||
import showcase_schema # noqa: E402
|
||||
import spec_to_glb # noqa: E402
|
||||
@@ -60,7 +60,7 @@ def test_vertical_storey_continuity_rejects_representative_floor_islands() -> No
|
||||
spec["floors"]["F2"] = {**spec["floors"]["F1"], "name": "二层", "z": 5.0}
|
||||
spec["floors"]["F16"] = {**spec["floors"]["F1"], "name": "十六层", "z": 59.6}
|
||||
gate = audit_showcase_extraction.vertical_storey_continuity_gate(spec)
|
||||
assert gate["status"] == "FAIL"
|
||||
assert gate["status"] == "WARN"
|
||||
assert "F2->F16" in gate["detail"]
|
||||
|
||||
|
||||
@@ -97,16 +97,151 @@ def test_config_rejects_missing_source(tmp_path: Path) -> None:
|
||||
showcase_schema.validate_config(config, tmp_path)
|
||||
|
||||
|
||||
def test_source_report_rejects_partial_gate() -> None:
|
||||
def test_source_report_requires_reason_for_partial_gate() -> None:
|
||||
report = {
|
||||
"report_version": 1,
|
||||
"gate": {"name": "all_sources_converted_and_readable", "status": "WARN"},
|
||||
"files": [{"file": "partial.dxf", "status": "ok", "modelspace_entities": 1}],
|
||||
}
|
||||
with pytest.raises(showcase_schema.ValidationFailure, match="must be PASS"):
|
||||
with pytest.raises(showcase_schema.ValidationFailure, match="partial reason"):
|
||||
showcase_schema.validate_source_report(report)
|
||||
|
||||
|
||||
def test_source_report_accepts_reasoned_partial_gate() -> None:
|
||||
report = {
|
||||
"report_version": 1,
|
||||
"gate": {
|
||||
"name": "all_sources_converted_and_readable",
|
||||
"status": "WARN",
|
||||
"partial_reason": "One optional elevation DWG is unavailable",
|
||||
},
|
||||
"files": [{"file": "partial.dxf", "status": "ok", "modelspace_entities": 1}],
|
||||
}
|
||||
assert showcase_schema.validate_source_report(report) == report
|
||||
|
||||
|
||||
def test_unarmed_cross_evidence_gates_are_not_applicable() -> None:
|
||||
spec = minimal_spec()
|
||||
assert audit_showcase_extraction.frame_ledger_gate({}, None)["status"] == "NOT_APPLICABLE"
|
||||
assert audit_showcase_extraction.declared_area_gate(spec, {})["status"] == "NOT_APPLICABLE"
|
||||
assert audit_showcase_extraction.storey_count_gate(spec, {})["status"] == "NOT_APPLICABLE"
|
||||
|
||||
|
||||
def test_annotation_is_waivable_warn_and_absent_glazing_is_not_applicable() -> None:
|
||||
report = {
|
||||
"spec_counts": {"F1": {"glazing": 0}},
|
||||
"block_locked_layers": [],
|
||||
"top_insert_names": [],
|
||||
"classified_layers": [{"layer": "A-ANNO-DIMS"}],
|
||||
"coverage": {
|
||||
"block_locked_classifiable_length_meters": 0,
|
||||
"block_captured_classified_length_meters": 0,
|
||||
},
|
||||
}
|
||||
gates = {
|
||||
gate["name"]: gate["status"]
|
||||
for gate in audit_showcase_extraction.evaluate_gates(report, minimal_spec(), {}, None)
|
||||
}
|
||||
assert gates["no_annotation_layers_classified"] == "WARN"
|
||||
assert gates["glazing_captured"] == "NOT_APPLICABLE"
|
||||
|
||||
|
||||
def test_config_rejects_malformed_category_overrides() -> None:
|
||||
config = {
|
||||
"config_version": 1,
|
||||
"drawings": [{"file": "fixture.dxf", "floors": [{"key": "F1"}]}],
|
||||
"categories": {"walls": "not-an-object"},
|
||||
}
|
||||
with pytest.raises(showcase_schema.ValidationFailure, match="categories.walls"):
|
||||
showcase_schema.validate_config(config)
|
||||
|
||||
|
||||
def test_all_derive_geometry_loss_warnings_are_forbidden() -> None:
|
||||
assert set(cad_showcase.DERIVE_FORBIDDEN_WARNINGS) == {
|
||||
"WARNING isolated_cluster",
|
||||
"WARNING unsupported_roof",
|
||||
"WARNING floor_order",
|
||||
"WARNING topless_group",
|
||||
}
|
||||
|
||||
|
||||
def test_pipeline_fails_on_derive_geometry_loss_warning(tmp_path: Path) -> None:
|
||||
pipeline = cad_showcase.Pipeline(tmp_path / "state.json", "fingerprint", resume=False)
|
||||
command = [
|
||||
sys.executable,
|
||||
"-c",
|
||||
"import sys; print('WARNING isolated_cluster F1: dropped geometry', file=sys.stderr)",
|
||||
]
|
||||
with pytest.raises(SystemExit):
|
||||
pipeline.run("derive", command, [], forbidden_stderr=cad_showcase.DERIVE_FORBIDDEN_WARNINGS)
|
||||
state = json.loads((tmp_path / "state.json").read_text(encoding="utf-8"))
|
||||
assert state["steps"]["derive"]["status"] == "FAIL"
|
||||
assert state["steps"]["derive"]["forbidden_stderr_hits"] == ["WARNING isolated_cluster"]
|
||||
|
||||
|
||||
def test_windows_command_resolution_uses_cmd_shims(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
monkeypatch.setattr(cad_showcase.sys, "platform", "win32")
|
||||
monkeypatch.setattr(
|
||||
cad_showcase.shutil,
|
||||
"which",
|
||||
lambda candidate: f"C:/tools/{candidate}" if candidate in {"npm.cmd", "node.exe"} else None,
|
||||
)
|
||||
assert cad_showcase.npm_command("install")[0] == "C:/tools/npm.cmd"
|
||||
assert cad_showcase.node_script(Path("smoke.mjs"))[0] == "C:/tools/node.exe"
|
||||
|
||||
|
||||
def test_resume_fingerprint_tracks_tools_offline_libs_and_extra_inputs(
|
||||
monkeypatch: pytest.MonkeyPatch, tmp_path: Path
|
||||
) -> None:
|
||||
root = tmp_path / "skill"
|
||||
scripts = root / "scripts"
|
||||
dxf_dir = tmp_path / "dxf"
|
||||
for directory in (
|
||||
scripts,
|
||||
root / "schemas",
|
||||
root / "tools",
|
||||
root / "assets" / "web-showcase-template",
|
||||
root / "assets" / "offline-viewer-libs",
|
||||
dxf_dir,
|
||||
):
|
||||
directory.mkdir(parents=True, exist_ok=True)
|
||||
config = tmp_path / "config.json"
|
||||
source = tmp_path / "source.json"
|
||||
frames = tmp_path / "frames.json"
|
||||
config.write_text("{}", encoding="utf-8")
|
||||
source.write_text("{}", encoding="utf-8")
|
||||
frames.write_text("{}", encoding="utf-8")
|
||||
(dxf_dir / "plan.dxf").write_text("DXF", encoding="utf-8")
|
||||
(scripts / "step.py").write_text("pass", encoding="utf-8")
|
||||
compressor = root / "tools" / "compress.mjs"
|
||||
compressor.write_text("v1", encoding="utf-8")
|
||||
offline_lib = root / "assets" / "offline-viewer-libs" / "three.min.js"
|
||||
offline_lib.write_text("v1", encoding="utf-8")
|
||||
monkeypatch.setattr(cad_showcase, "ROOT", root)
|
||||
monkeypatch.setattr(cad_showcase, "SCRIPTS", scripts)
|
||||
first = cad_showcase.fingerprint(config, source, dxf_dir, (frames,))
|
||||
compressor.write_text("v2", encoding="utf-8")
|
||||
second = cad_showcase.fingerprint(config, source, dxf_dir, (frames,))
|
||||
offline_lib.write_text("v2", encoding="utf-8")
|
||||
third = cad_showcase.fingerprint(config, source, dxf_dir, (frames,))
|
||||
frames.write_text('{"changed": true}', encoding="utf-8")
|
||||
fourth = cad_showcase.fingerprint(config, source, dxf_dir, (frames,))
|
||||
assert len({first, second, third, fourth}) == 4
|
||||
|
||||
|
||||
def test_pipeline_state_is_atomic_and_corruption_is_loud(tmp_path: Path) -> None:
|
||||
state = tmp_path / "pipeline_state.json"
|
||||
pipeline = cad_showcase.Pipeline(state, "fingerprint", resume=False)
|
||||
pipeline.state["proof"] = "saved"
|
||||
pipeline.save()
|
||||
assert json.loads(state.read_text(encoding="utf-8"))["proof"] == "saved"
|
||||
assert not list(tmp_path.glob(".pipeline_state.json.*.tmp"))
|
||||
state.write_text("{truncated", encoding="utf-8")
|
||||
with pytest.raises(SystemExit, match="corrupt and was not overwritten"):
|
||||
cad_showcase.Pipeline(state, "fingerprint", resume=True)
|
||||
assert state.read_text(encoding="utf-8") == "{truncated"
|
||||
|
||||
|
||||
def test_doctor_uses_platform_tool_resolvers(monkeypatch: pytest.MonkeyPatch, tmp_path: Path, capsys) -> None:
|
||||
oda = tmp_path / "ODAFileConverter"
|
||||
blender = tmp_path / "blender"
|
||||
@@ -143,7 +278,8 @@ def test_report_empty_input_is_a_hard_failure(tmp_path: Path) -> None:
|
||||
proc = subprocess.run(
|
||||
[
|
||||
sys.executable,
|
||||
str(ROOT / "scripts" / "cad_dxf_report.py"),
|
||||
str(ROOT / "scripts" / "cad_showcase.py"),
|
||||
"inspect",
|
||||
"--input",
|
||||
str(tmp_path),
|
||||
"--work",
|
||||
@@ -162,7 +298,8 @@ def test_partial_report_requires_and_records_a_reason(tmp_path: Path) -> None:
|
||||
proc = subprocess.run(
|
||||
[
|
||||
sys.executable,
|
||||
str(ROOT / "scripts" / "cad_dxf_report.py"),
|
||||
str(ROOT / "scripts" / "cad_showcase.py"),
|
||||
"inspect",
|
||||
"--input",
|
||||
str(tmp_path),
|
||||
"--work",
|
||||
@@ -331,14 +468,6 @@ def test_orchestrator_runs_resumable_trimesh_pipeline(tmp_path: Path) -> None:
|
||||
str(out),
|
||||
"--name",
|
||||
"orchestrator-test",
|
||||
"--waiver",
|
||||
"audit:glazing_captured=Synthetic fixture contains no glazing evidence",
|
||||
"--waiver",
|
||||
"audit:frame_ledger=Synthetic fixture does not exercise sheet-frame detection",
|
||||
"--waiver",
|
||||
"audit:area_declared=Synthetic fixture has no independent declared-area evidence",
|
||||
"--waiver",
|
||||
"audit:storey_count=Synthetic fixture has no independent storey-count evidence",
|
||||
"--budget-mb",
|
||||
"5",
|
||||
]
|
||||
|
||||
@@ -60,6 +60,34 @@ def source_report_fixture(tmp_path: Path) -> Path:
|
||||
return path
|
||||
|
||||
|
||||
def browser_report_payload(*, offline: bool = False) -> dict:
|
||||
if offline:
|
||||
checks = [
|
||||
{"name": "offline:viewport", "status": "PASS", "detail": "1440x900 expected 1440x900"},
|
||||
{"name": "offline:model-ready", "status": "PASS", "detail": "1 mesh(es)"},
|
||||
{"name": "offline:canvas", "status": "PASS"},
|
||||
{"name": "offline:render", "status": "PASS", "detail": "1 render(s)"},
|
||||
]
|
||||
else:
|
||||
checks = []
|
||||
for viewport, detail in (
|
||||
("desktop", "1440x900 expected 1440x900"),
|
||||
("mobile", "390x844 expected 390x844"),
|
||||
):
|
||||
checks.append({"name": f"{viewport}:viewport", "status": "PASS", "detail": detail})
|
||||
for selector in ("#floor-controls", "#category-controls", "#explode", "#section", "#snapshot"):
|
||||
checks.append({"name": f"{viewport}:{selector}", "status": "PASS"})
|
||||
checks.append({"name": f"{viewport}:idle-render", "status": "PASS", "detail": "3->3"})
|
||||
return {
|
||||
"status": "PASS",
|
||||
"detail": f"{len(checks)}/{len(checks)} checks passed",
|
||||
"checks": checks,
|
||||
"console_errors": [],
|
||||
"page_errors": [],
|
||||
"failed_requests": [],
|
||||
}
|
||||
|
||||
|
||||
def test_strict_delivery_passes_for_glb_render_and_offline_viewer(tmp_path: Path) -> None:
|
||||
spec, glb, _metadata = build_fixture(tmp_path)
|
||||
render = tmp_path / "render.png"
|
||||
@@ -78,15 +106,7 @@ def test_strict_delivery_passes_for_glb_render_and_offline_viewer(tmp_path: Path
|
||||
assert proc.returncode == 0, proc.stderr
|
||||
offline_browser_report = tmp_path / "offline_browser_smoke_report.json"
|
||||
offline_browser_report.write_text(
|
||||
json.dumps(
|
||||
{
|
||||
"status": "PASS",
|
||||
"detail": "4/4 checks passed",
|
||||
"console_errors": [],
|
||||
"page_errors": [],
|
||||
"failed_requests": [],
|
||||
}
|
||||
),
|
||||
json.dumps(browser_report_payload(offline=True)),
|
||||
encoding="utf-8",
|
||||
)
|
||||
evidence = tmp_path / "evidence.json"
|
||||
@@ -116,6 +136,43 @@ def test_strict_delivery_passes_for_glb_render_and_offline_viewer(tmp_path: Path
|
||||
assert any(item["name"] == "offline:browser-smoke" and item["status"] == "PASS" for item in report["checks"])
|
||||
|
||||
|
||||
def test_browser_report_cannot_forge_pass_without_required_checks(tmp_path: Path) -> None:
|
||||
import verify_delivery
|
||||
|
||||
path = tmp_path / "forged.json"
|
||||
path.write_text(
|
||||
json.dumps(
|
||||
{
|
||||
"status": "PASS",
|
||||
"checks": [],
|
||||
"console_errors": [],
|
||||
"page_errors": [],
|
||||
"failed_requests": [],
|
||||
}
|
||||
),
|
||||
encoding="utf-8",
|
||||
)
|
||||
checks: list[dict] = []
|
||||
verify_delivery.inspect_browser_report(path, checks)
|
||||
assert checks[0]["status"] == "FAIL"
|
||||
assert "non-empty list" in checks[0]["detail"]
|
||||
|
||||
|
||||
def test_render_gate_rejects_high_variance_noise(tmp_path: Path) -> None:
|
||||
import verify_delivery
|
||||
|
||||
noisy = Image.new("RGB", (640, 480))
|
||||
pixels = noisy.load()
|
||||
for y in range(noisy.height):
|
||||
for x in range(noisy.width):
|
||||
pixels[x, y] = (255, 255, 255) if (x + y) % 2 else (0, 0, 0)
|
||||
path = tmp_path / "noise.png"
|
||||
noisy.save(path)
|
||||
checks: list[dict] = []
|
||||
verify_delivery.inspect_render(path, checks)
|
||||
assert checks[0]["status"] == "FAIL"
|
||||
|
||||
|
||||
def test_offline_viewer_without_real_browser_report_is_blocked(tmp_path: Path) -> None:
|
||||
spec, glb, _metadata = build_fixture(tmp_path)
|
||||
render = tmp_path / "render.png"
|
||||
@@ -216,15 +273,7 @@ def test_web_writer_and_static_delivery_checks(tmp_path: Path) -> None:
|
||||
assert "browser:smoke" in proc.stderr
|
||||
browser_report = tmp_path / "browser_smoke_report.json"
|
||||
browser_report.write_text(
|
||||
json.dumps(
|
||||
{
|
||||
"status": "PASS",
|
||||
"detail": "12/12 checks passed",
|
||||
"console_errors": [],
|
||||
"page_errors": [],
|
||||
"failed_requests": [],
|
||||
}
|
||||
),
|
||||
json.dumps(browser_report_payload()),
|
||||
encoding="utf-8",
|
||||
)
|
||||
proc = run(
|
||||
@@ -248,6 +297,44 @@ def test_web_writer_and_static_delivery_checks(tmp_path: Path) -> None:
|
||||
assert proc.returncode == 0, proc.stderr
|
||||
|
||||
|
||||
def test_reasoned_partial_source_can_only_pass_with_named_waiver(tmp_path: Path) -> None:
|
||||
spec, glb, _metadata = build_fixture(tmp_path)
|
||||
render = tmp_path / "render.png"
|
||||
render_fixture(render)
|
||||
source_report = source_report_fixture(tmp_path)
|
||||
source = json.loads(source_report.read_text(encoding="utf-8"))
|
||||
source["gate"] = {
|
||||
"name": "all_sources_converted_and_readable",
|
||||
"status": "WARN",
|
||||
"detail": "one optional elevation is missing",
|
||||
"partial_reason": "User approved plan-only delivery",
|
||||
}
|
||||
source_report.write_text(json.dumps(source), encoding="utf-8")
|
||||
evidence = tmp_path / "evidence.json"
|
||||
base = [
|
||||
sys.executable,
|
||||
"scripts/verify_delivery.py",
|
||||
"--spec",
|
||||
str(spec),
|
||||
"--source-report",
|
||||
str(source_report),
|
||||
"--glb",
|
||||
str(glb),
|
||||
"--render",
|
||||
str(render),
|
||||
"--evidence-out",
|
||||
str(evidence),
|
||||
]
|
||||
proc = run(*base)
|
||||
assert proc.returncode == 3
|
||||
assert json.loads(evidence.read_text(encoding="utf-8"))["status"] == "BLOCKED"
|
||||
proc = run(*base, "--waiver", "cad_source_report=User approved plan-only delivery")
|
||||
assert proc.returncode == 0, proc.stderr
|
||||
report = json.loads(evidence.read_text(encoding="utf-8"))
|
||||
source_check = next(item for item in report["checks"] if item["name"] == "cad_source_report")
|
||||
assert source_check["original_status"] == "WARN"
|
||||
|
||||
|
||||
def test_warn_or_skip_blocks_without_explicit_reasoned_waiver(tmp_path: Path) -> None:
|
||||
spec, glb, _metadata = build_fixture(tmp_path)
|
||||
render = tmp_path / "render.png"
|
||||
@@ -342,6 +429,78 @@ def test_campus_scope_cannot_pass_with_a_missing_requested_building(tmp_path: Pa
|
||||
assert any(item["name"] == "scope:building:2#" and item["status"] == "WARN" for item in report["checks"])
|
||||
|
||||
|
||||
def test_scope_rejects_arbitrary_nonempty_artifact(tmp_path: Path) -> None:
|
||||
spec, glb, _metadata = build_fixture(tmp_path)
|
||||
render = tmp_path / "render.png"
|
||||
render_fixture(render)
|
||||
source_report = source_report_fixture(tmp_path)
|
||||
fake = tmp_path / "looks-done.txt"
|
||||
fake.write_text("not a model", encoding="utf-8")
|
||||
scope = tmp_path / "scope_manifest.json"
|
||||
scope.write_text(
|
||||
json.dumps(
|
||||
{
|
||||
"scope_version": 1,
|
||||
"mode": "single",
|
||||
"requested": ["11#"],
|
||||
"buildings": [{"id": "11#", "status": "PASS", "artifacts": [fake.name]}],
|
||||
}
|
||||
),
|
||||
encoding="utf-8",
|
||||
)
|
||||
evidence = tmp_path / "evidence.json"
|
||||
proc = run(
|
||||
sys.executable,
|
||||
"scripts/verify_delivery.py",
|
||||
"--spec",
|
||||
str(spec),
|
||||
"--source-report",
|
||||
str(source_report),
|
||||
"--glb",
|
||||
str(glb),
|
||||
"--render",
|
||||
str(render),
|
||||
"--scope-manifest",
|
||||
str(scope),
|
||||
"--evidence-out",
|
||||
str(evidence),
|
||||
)
|
||||
assert proc.returncode == 3
|
||||
report = json.loads(evidence.read_text(encoding="utf-8"))
|
||||
check = next(item for item in report["checks"] if item["name"] == "scope:building:11#")
|
||||
assert check["status"] == "FAIL"
|
||||
assert "No verifiable" in check["detail"]
|
||||
|
||||
|
||||
def test_required_audit_and_scope_cannot_be_omitted(tmp_path: Path) -> None:
|
||||
spec, glb, _metadata = build_fixture(tmp_path)
|
||||
render = tmp_path / "render.png"
|
||||
render_fixture(render)
|
||||
source_report = source_report_fixture(tmp_path)
|
||||
evidence = tmp_path / "evidence.json"
|
||||
proc = run(
|
||||
sys.executable,
|
||||
"scripts/verify_delivery.py",
|
||||
"--spec",
|
||||
str(spec),
|
||||
"--source-report",
|
||||
str(source_report),
|
||||
"--glb",
|
||||
str(glb),
|
||||
"--render",
|
||||
str(render),
|
||||
"--require-audit",
|
||||
"--require-scope-manifest",
|
||||
"--evidence-out",
|
||||
str(evidence),
|
||||
)
|
||||
assert proc.returncode == 3
|
||||
failed = {
|
||||
item["name"] for item in json.loads(evidence.read_text(encoding="utf-8"))["checks"] if item["status"] == "FAIL"
|
||||
}
|
||||
assert {"audit:report", "scope:manifest"} <= failed
|
||||
|
||||
|
||||
def test_variants_manifest_requires_metadata_artifact(tmp_path: Path) -> None:
|
||||
spec, glb, _metadata = build_fixture(tmp_path)
|
||||
render = tmp_path / "render.png"
|
||||
|
||||
Reference in New Issue
Block a user