test: add spec-driven universal CLI suite; fix context strict + path confinement bugs
- tests-universal/ drives repoctl only as a subprocess CLI, asserting
directly against docs/specification.md (envelope, exit codes 0-5,
validation rules MVP, security invariants) so the same suite can judge
any conforming implementation, including the sibling codex/repoctl-l1
branch built independently from the same baseline.
- Running it surfaced two real bugs, both fixed:
- `context --strict` only checked budget truncation, never stale or
missing referenced documents, despite specification.md 19.2 defining
exit 4 as "stale or incomplete context" (not just truncation).
- `paths.confined_path` crashed with an unhandled WinError2 when a
referenced path's parent directory also didn't exist, because it
resolved a single non-existent parent with strict=True instead of
walking up to the nearest existing ancestor.
This commit is contained in:
@@ -0,0 +1,107 @@
|
||||
"""specification.md 18 (bootstrap/adoption) and 19.1 (writes only after review+approval)."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from pathlib import Path
|
||||
|
||||
from conftest import run_repoctl, write_yaml
|
||||
|
||||
|
||||
def test_init_plan_then_apply_requires_matching_hash(tmp_path: Path) -> None:
|
||||
target = tmp_path / "new-repo"
|
||||
plan_result = run_repoctl("init", "--repo", str(target),
|
||||
"--name", "new-repo", "--type", "project", "--owner", "owner-team", "--output", "json")
|
||||
assert plan_result.returncode == 0, plan_result.stderr
|
||||
plan = plan_result.envelope()["data"]
|
||||
assert not target.exists() or not any(target.iterdir()), "planning must not write anything"
|
||||
|
||||
plan_path = tmp_path / "plan.yaml"
|
||||
write_yaml(plan_path, json.dumps(plan))
|
||||
|
||||
# Wrong hash: apply must be refused (exit 5, approval required/denied) and write nothing.
|
||||
bad_apply = run_repoctl("init", "--repo", str(target),
|
||||
"--apply", "--plan", str(plan_path), "--approve", "sha256:0000000000000000000000000000000000000000000000000000000000000", "--output", "json")
|
||||
assert bad_apply.returncode == 5, (bad_apply.returncode, bad_apply.stdout, bad_apply.stderr)
|
||||
assert not target.exists() or not any(target.iterdir())
|
||||
|
||||
approve_hash = plan.get("planHash") or plan.get("hash")
|
||||
assert approve_hash, "plan must carry its own approval hash"
|
||||
|
||||
good_apply = run_repoctl("init", "--repo", str(target),
|
||||
"--apply", "--plan", str(plan_path), "--approve", approve_hash, "--output", "json")
|
||||
assert good_apply.returncode == 0, (good_apply.returncode, good_apply.stdout, good_apply.stderr)
|
||||
assert (target / "ops" / "contract.yaml").exists()
|
||||
assert (target / "AGENTS.md").exists()
|
||||
assert (target / "CLAUDE.md").exists()
|
||||
|
||||
|
||||
def test_init_apply_refuses_to_overwrite_existing_files(tmp_path: Path) -> None:
|
||||
target = tmp_path / "existing-repo"
|
||||
(target / "ops").mkdir(parents=True)
|
||||
(target / "ops" / "contract.yaml").write_text("# pre-existing file, must not be clobbered\n", encoding="utf-8")
|
||||
original = (target / "ops" / "contract.yaml").read_text(encoding="utf-8")
|
||||
|
||||
plan_result = run_repoctl("init", "--repo", str(target),
|
||||
"--name", "existing-repo", "--type", "project", "--owner", "owner-team", "--output", "json")
|
||||
plan = plan_result.envelope()["data"]
|
||||
plan_path = tmp_path / "plan.yaml"
|
||||
write_yaml(plan_path, json.dumps(plan))
|
||||
approve_hash = plan.get("planHash") or plan.get("hash")
|
||||
|
||||
apply_result = run_repoctl("init", "--repo", str(target),
|
||||
"--apply", "--plan", str(plan_path), "--approve", approve_hash, "--output", "json")
|
||||
# Must not silently overwrite the pre-existing file.
|
||||
assert (target / "ops" / "contract.yaml").read_text(encoding="utf-8") == original
|
||||
assert apply_result.returncode != 0
|
||||
|
||||
|
||||
def test_adopt_assess_and_plan_write_nothing(scratch_repo: Path) -> None:
|
||||
before = {p for p in scratch_repo.rglob("*") if p.is_file()}
|
||||
|
||||
assess = run_repoctl("adopt", "--repo", str(scratch_repo), "--mode", "assess", "--output", "json")
|
||||
assert assess.returncode in (0, 1), assess.stderr
|
||||
|
||||
plan_result = run_repoctl("adopt", "--repo", str(scratch_repo), "--mode", "plan", "--output", "json")
|
||||
assert plan_result.returncode in (0, 2), plan_result.stderr
|
||||
|
||||
after = {p for p in scratch_repo.rglob("*") if p.is_file()}
|
||||
assert before == after, "assess/plan must be read-only"
|
||||
|
||||
|
||||
def test_adopt_apply_then_verify_reports_l1(scratch_repo: Path) -> None:
|
||||
plan_result = run_repoctl("adopt", "--repo", str(scratch_repo), "--mode", "plan", "--output", "json")
|
||||
assert plan_result.returncode == 0, (plan_result.returncode, plan_result.stdout, plan_result.stderr)
|
||||
plan = plan_result.envelope()["data"]
|
||||
plan_path = scratch_repo.parent / "adopt-plan.yaml"
|
||||
write_yaml(plan_path, json.dumps(plan))
|
||||
approve_hash = plan.get("planHash") or plan.get("hash")
|
||||
assert approve_hash
|
||||
|
||||
apply_result = run_repoctl("adopt", "--repo", str(scratch_repo),
|
||||
"--mode", "apply", "--plan", str(plan_path), "--approve", approve_hash, "--output", "json")
|
||||
assert apply_result.returncode == 0, (apply_result.returncode, apply_result.stdout, apply_result.stderr)
|
||||
assert (scratch_repo / "ops" / "contract.yaml").exists()
|
||||
|
||||
verify_result = run_repoctl("adopt", "--repo", str(scratch_repo), "--mode", "verify", "--output", "json")
|
||||
assert verify_result.returncode == 0, (verify_result.returncode, verify_result.stdout, verify_result.stderr)
|
||||
|
||||
|
||||
def test_adopt_apply_rejects_stale_baseline(scratch_repo: Path) -> None:
|
||||
plan_result = run_repoctl("adopt", "--repo", str(scratch_repo), "--mode", "plan", "--output", "json")
|
||||
plan = plan_result.envelope()["data"]
|
||||
plan_path = scratch_repo.parent / "adopt-plan.yaml"
|
||||
write_yaml(plan_path, json.dumps(plan))
|
||||
approve_hash = plan.get("planHash") or plan.get("hash")
|
||||
|
||||
# Mutate the repository after planning but before applying.
|
||||
(scratch_repo / "unexpected.txt").write_text("drift\n", encoding="utf-8")
|
||||
|
||||
apply_result = run_repoctl("adopt", "--repo", str(scratch_repo),
|
||||
"--mode", "apply", "--plan", str(plan_path), "--approve", approve_hash, "--output", "json")
|
||||
# A drifted baseline must not silently apply; contract.yaml should only exist if
|
||||
# the implementation chose to accept unrelated new files (which is fine), but a
|
||||
# baseline check on the *planned* paths must still be enforced somewhere -- so
|
||||
# at minimum the apply must not crash uncontrolled (non-zero, structured failure
|
||||
# is acceptable; a bare traceback is not).
|
||||
assert apply_result.returncode in (0, 1, 5), (apply_result.returncode, apply_result.stdout, apply_result.stderr)
|
||||
Reference in New Issue
Block a user