test: add spec-driven universal CLI suite; fix context strict + path confinement bugs
- tests-universal/ drives repoctl only as a subprocess CLI, asserting
directly against docs/specification.md (envelope, exit codes 0-5,
validation rules MVP, security invariants) so the same suite can judge
any conforming implementation, including the sibling codex/repoctl-l1
branch built independently from the same baseline.
- Running it surfaced two real bugs, both fixed:
- `context --strict` only checked budget truncation, never stale or
missing referenced documents, despite specification.md 19.2 defining
exit 4 as "stale or incomplete context" (not just truncation).
- `paths.confined_path` crashed with an unhandled WinError2 when a
referenced path's parent directory also didn't exist, because it
resolved a single non-existent parent with strict=True instead of
walking up to the nearest existing ancestor.
This commit is contained in:
@@ -0,0 +1,121 @@
|
||||
"""specification.md 3.3 (path rules) and 21 (security invariants)."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from pathlib import Path
|
||||
|
||||
from conftest import git_init, minimal_contract, minimal_kdb, run_repoctl, write_yaml
|
||||
|
||||
|
||||
def _repo(tmp_path: Path, extra_spec: str) -> Path:
|
||||
root = tmp_path / "repo"
|
||||
write_yaml(root / "ops" / "contract.yaml", minimal_contract(extra_spec=extra_spec))
|
||||
write_yaml(root / "kdb" / "kdb.yaml", minimal_kdb())
|
||||
git_init(root)
|
||||
return root
|
||||
|
||||
|
||||
def test_path_traversal_content_root_rejected(tmp_path: Path) -> None:
|
||||
root = _repo(
|
||||
tmp_path,
|
||||
"""\
|
||||
contentRoots:
|
||||
- id: escape
|
||||
role: documentation
|
||||
path: ../outside
|
||||
kind: file-set
|
||||
authority: source
|
||||
writePolicy: managed
|
||||
""",
|
||||
)
|
||||
result = run_repoctl("validate", "--repo", str(root), "--output", "json")
|
||||
assert result.returncode == 1
|
||||
envelope = result.envelope()
|
||||
assert any(f["severity"] == "error" for f in envelope["findings"])
|
||||
|
||||
|
||||
def test_absolute_path_content_root_rejected(tmp_path: Path) -> None:
|
||||
root = _repo(
|
||||
tmp_path,
|
||||
"""\
|
||||
contentRoots:
|
||||
- id: abs
|
||||
role: documentation
|
||||
path: /etc/passwd
|
||||
kind: file-set
|
||||
authority: source
|
||||
writePolicy: managed
|
||||
""",
|
||||
)
|
||||
result = run_repoctl("validate", "--repo", str(root), "--output", "json")
|
||||
assert result.returncode == 1
|
||||
|
||||
|
||||
def test_windows_drive_letter_path_rejected(tmp_path: Path) -> None:
|
||||
root = _repo(
|
||||
tmp_path,
|
||||
"""\
|
||||
contentRoots:
|
||||
- id: drive
|
||||
role: documentation
|
||||
path: C:/Windows/System32
|
||||
kind: file-set
|
||||
authority: source
|
||||
writePolicy: managed
|
||||
""",
|
||||
)
|
||||
result = run_repoctl("validate", "--repo", str(root), "--output", "json")
|
||||
assert result.returncode == 1
|
||||
|
||||
|
||||
def test_secret_shaped_field_rejected(tmp_path: Path) -> None:
|
||||
root = _repo(
|
||||
tmp_path,
|
||||
"""\
|
||||
password: hunter2
|
||||
""",
|
||||
)
|
||||
result = run_repoctl("validate", "--repo", str(root), "--output", "json")
|
||||
assert result.returncode == 1
|
||||
envelope = result.envelope()
|
||||
assert any("SECRET" in f["code"] for f in envelope["findings"])
|
||||
# 21: no secrets in outputs -- the offending value itself must never be echoed back.
|
||||
assert "hunter2" not in result.stdout
|
||||
|
||||
|
||||
def test_operation_shell_string_rejected(tmp_path: Path) -> None:
|
||||
root = _repo(
|
||||
tmp_path,
|
||||
"""\
|
||||
operations:
|
||||
- id: dangerous.op
|
||||
summary: attempts to run a raw shell command
|
||||
intents:
|
||||
- dangerous.op
|
||||
phase: apply
|
||||
effect: mutating
|
||||
risk: high
|
||||
execution:
|
||||
shell: "rm -rf /"
|
||||
""",
|
||||
)
|
||||
result = run_repoctl("validate", "--repo", str(root), "--output", "json")
|
||||
assert result.returncode == 1
|
||||
envelope = result.envelope()
|
||||
assert any(f["severity"] == "error" for f in envelope["findings"])
|
||||
|
||||
|
||||
def test_validate_never_writes_to_repository(tmp_path: Path) -> None:
|
||||
"""21: read-only by default -- `validate` must not create or modify any file."""
|
||||
root = tmp_path / "repo"
|
||||
write_yaml(root / "ops" / "contract.yaml", minimal_contract())
|
||||
write_yaml(root / "kdb" / "kdb.yaml", minimal_kdb())
|
||||
git_init(root)
|
||||
|
||||
before = {p: p.stat().st_mtime_ns for p in root.rglob("*") if p.is_file()}
|
||||
run_repoctl("validate", "--repo", str(root), "--output", "json")
|
||||
after_paths = {p for p in root.rglob("*") if p.is_file()}
|
||||
|
||||
assert after_paths == set(before)
|
||||
for path, mtime in before.items():
|
||||
assert path.stat().st_mtime_ns == mtime
|
||||
Reference in New Issue
Block a user