test: add spec-driven universal CLI suite; fix context strict + path confinement bugs
- tests-universal/ drives repoctl only as a subprocess CLI, asserting
directly against docs/specification.md (envelope, exit codes 0-5,
validation rules MVP, security invariants) so the same suite can judge
any conforming implementation, including the sibling codex/repoctl-l1
branch built independently from the same baseline.
- Running it surfaced two real bugs, both fixed:
- `context --strict` only checked budget truncation, never stale or
missing referenced documents, despite specification.md 19.2 defining
exit 4 as "stale or incomplete context" (not just truncation).
- `paths.confined_path` crashed with an unhandled WinError2 when a
referenced path's parent directory also didn't exist, because it
resolved a single non-existent parent with strict=True instead of
walking up to the nearest existing ancestor.
This commit is contained in:
@@ -0,0 +1,80 @@
|
||||
"""specification.md 4.1, 4.2, 19.2: envelope shape and exit codes."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from pathlib import Path
|
||||
|
||||
from conftest import RESULT_SCHEMA, git_init, run_repoctl
|
||||
|
||||
|
||||
def test_validate_success_envelope_and_exit_0(repo: Path) -> None:
|
||||
result = run_repoctl("validate", "--repo", str(repo), "--output", "json")
|
||||
assert result.returncode == 0, result.stderr
|
||||
envelope = result.envelope()
|
||||
assert envelope["schema"] == RESULT_SCHEMA
|
||||
assert envelope["command"] == "validate"
|
||||
assert envelope["outcome"] == "success"
|
||||
assert envelope["findings"] == []
|
||||
assert "data" in envelope
|
||||
|
||||
|
||||
def test_validate_missing_repository_exits_3(tmp_path: Path) -> None:
|
||||
empty = tmp_path / "not-a-repo"
|
||||
empty.mkdir()
|
||||
# No .git ancestor at all: repository cannot be resolved.
|
||||
result = run_repoctl("validate", "--repo", str(empty), "--output", "json")
|
||||
assert result.returncode == 3, (result.returncode, result.stderr)
|
||||
|
||||
|
||||
def test_unknown_command_exits_2() -> None:
|
||||
result = run_repoctl("not-a-real-command", "--output", "json")
|
||||
assert result.returncode == 2
|
||||
|
||||
|
||||
def test_validate_broken_contract_exits_1(tmp_path: Path) -> None:
|
||||
root = tmp_path / "broken-repo"
|
||||
(root / "ops").mkdir(parents=True)
|
||||
(root / "ops" / "contract.yaml").write_text("kind: RepositoryContract\n", encoding="utf-8")
|
||||
git_init(root)
|
||||
|
||||
result = run_repoctl("validate", "--repo", str(root), "--output", "json")
|
||||
assert result.returncode == 1, (result.returncode, result.stdout, result.stderr)
|
||||
envelope = result.envelope()
|
||||
assert envelope["outcome"] == "failure"
|
||||
assert any(f["severity"] == "error" for f in envelope["findings"])
|
||||
|
||||
|
||||
def test_findings_have_required_shape(tmp_path: Path) -> None:
|
||||
"""4.3 / 19.3: every finding carries severity in {info, warning, error}, a code, a message."""
|
||||
root = tmp_path / "broken-repo-2"
|
||||
(root / "ops").mkdir(parents=True)
|
||||
(root / "ops" / "contract.yaml").write_text("not: a-valid-contract\n", encoding="utf-8")
|
||||
git_init(root)
|
||||
|
||||
result = run_repoctl("validate", "--repo", str(root), "--output", "json")
|
||||
envelope = result.envelope()
|
||||
for finding in envelope["findings"]:
|
||||
assert finding["severity"] in {"info", "warning", "error"}
|
||||
assert isinstance(finding.get("code"), str) and finding["code"]
|
||||
assert isinstance(finding.get("message"), str) and finding["message"]
|
||||
|
||||
|
||||
def test_output_modes_yaml_and_json_both_parseable(repo: Path) -> None:
|
||||
import json
|
||||
|
||||
from ruamel.yaml import YAML
|
||||
|
||||
json_result = run_repoctl("validate", "--repo", str(repo), "--output", "json")
|
||||
json.loads(json_result.stdout)
|
||||
|
||||
yaml_result = run_repoctl("validate", "--repo", str(repo), "--output", "yaml")
|
||||
YAML(typ="safe").load(yaml_result.stdout)
|
||||
|
||||
|
||||
def test_human_output_writes_diagnostics_to_stderr_not_stdout(repo: Path) -> None:
|
||||
"""PLAN.md: 'Machine output идёт только в stdout, диагностика процесса — в stderr.'"""
|
||||
result = run_repoctl("validate", "--repo", str(repo), "--output", "human")
|
||||
assert result.returncode == 0
|
||||
# A clean validate has no findings to print, but stdout must not carry a JSON/YAML
|
||||
# machine envelope in human mode either.
|
||||
assert "schema:" not in result.stdout and '"schema"' not in result.stdout
|
||||
Reference in New Issue
Block a user