- tests-universal/ drives repoctl only as a subprocess CLI, asserting
directly against docs/specification.md (envelope, exit codes 0-5,
validation rules MVP, security invariants) so the same suite can judge
any conforming implementation, including the sibling codex/repoctl-l1
branch built independently from the same baseline.
- Running it surfaced two real bugs, both fixed:
- `context --strict` only checked budget truncation, never stale or
missing referenced documents, despite specification.md 19.2 defining
exit 4 as "stale or incomplete context" (not just truncation).
- `paths.confined_path` crashed with an unhandled WinError2 when a
referenced path's parent directory also didn't exist, because it
resolved a single non-existent parent with strict=True instead of
walking up to the nearest existing ancestor.
81 lines
3.2 KiB
Python
81 lines
3.2 KiB
Python
"""specification.md 4.1, 4.2, 19.2: envelope shape and exit codes."""
|
|
|
|
from __future__ import annotations
|
|
|
|
from pathlib import Path
|
|
|
|
from conftest import RESULT_SCHEMA, git_init, run_repoctl
|
|
|
|
|
|
def test_validate_success_envelope_and_exit_0(repo: Path) -> None:
|
|
result = run_repoctl("validate", "--repo", str(repo), "--output", "json")
|
|
assert result.returncode == 0, result.stderr
|
|
envelope = result.envelope()
|
|
assert envelope["schema"] == RESULT_SCHEMA
|
|
assert envelope["command"] == "validate"
|
|
assert envelope["outcome"] == "success"
|
|
assert envelope["findings"] == []
|
|
assert "data" in envelope
|
|
|
|
|
|
def test_validate_missing_repository_exits_3(tmp_path: Path) -> None:
|
|
empty = tmp_path / "not-a-repo"
|
|
empty.mkdir()
|
|
# No .git ancestor at all: repository cannot be resolved.
|
|
result = run_repoctl("validate", "--repo", str(empty), "--output", "json")
|
|
assert result.returncode == 3, (result.returncode, result.stderr)
|
|
|
|
|
|
def test_unknown_command_exits_2() -> None:
|
|
result = run_repoctl("not-a-real-command", "--output", "json")
|
|
assert result.returncode == 2
|
|
|
|
|
|
def test_validate_broken_contract_exits_1(tmp_path: Path) -> None:
|
|
root = tmp_path / "broken-repo"
|
|
(root / "ops").mkdir(parents=True)
|
|
(root / "ops" / "contract.yaml").write_text("kind: RepositoryContract\n", encoding="utf-8")
|
|
git_init(root)
|
|
|
|
result = run_repoctl("validate", "--repo", str(root), "--output", "json")
|
|
assert result.returncode == 1, (result.returncode, result.stdout, result.stderr)
|
|
envelope = result.envelope()
|
|
assert envelope["outcome"] == "failure"
|
|
assert any(f["severity"] == "error" for f in envelope["findings"])
|
|
|
|
|
|
def test_findings_have_required_shape(tmp_path: Path) -> None:
|
|
"""4.3 / 19.3: every finding carries severity in {info, warning, error}, a code, a message."""
|
|
root = tmp_path / "broken-repo-2"
|
|
(root / "ops").mkdir(parents=True)
|
|
(root / "ops" / "contract.yaml").write_text("not: a-valid-contract\n", encoding="utf-8")
|
|
git_init(root)
|
|
|
|
result = run_repoctl("validate", "--repo", str(root), "--output", "json")
|
|
envelope = result.envelope()
|
|
for finding in envelope["findings"]:
|
|
assert finding["severity"] in {"info", "warning", "error"}
|
|
assert isinstance(finding.get("code"), str) and finding["code"]
|
|
assert isinstance(finding.get("message"), str) and finding["message"]
|
|
|
|
|
|
def test_output_modes_yaml_and_json_both_parseable(repo: Path) -> None:
|
|
import json
|
|
|
|
from ruamel.yaml import YAML
|
|
|
|
json_result = run_repoctl("validate", "--repo", str(repo), "--output", "json")
|
|
json.loads(json_result.stdout)
|
|
|
|
yaml_result = run_repoctl("validate", "--repo", str(repo), "--output", "yaml")
|
|
YAML(typ="safe").load(yaml_result.stdout)
|
|
|
|
|
|
def test_human_output_writes_diagnostics_to_stderr_not_stdout(repo: Path) -> None:
|
|
"""PLAN.md: 'Machine output идёт только в stdout, диагностика процесса — в stderr.'"""
|
|
result = run_repoctl("validate", "--repo", str(repo), "--output", "human")
|
|
assert result.returncode == 0
|
|
# A clean validate has no findings to print, but stdout must not carry a JSON/YAML
|
|
# machine envelope in human mode either.
|
|
assert "schema:" not in result.stdout and '"schema"' not in result.stdout
|