test: add spec-driven universal CLI suite; fix context strict + path confinement bugs

- tests-universal/ drives repoctl only as a subprocess CLI, asserting
  directly against docs/specification.md (envelope, exit codes 0-5,
  validation rules MVP, security invariants) so the same suite can judge
  any conforming implementation, including the sibling codex/repoctl-l1
  branch built independently from the same baseline.
- Running it surfaced two real bugs, both fixed:
  - `context --strict` only checked budget truncation, never stale or
    missing referenced documents, despite specification.md 19.2 defining
    exit 4 as "stale or incomplete context" (not just truncation).
  - `paths.confined_path` crashed with an unhandled WinError2 when a
    referenced path's parent directory also didn't exist, because it
    resolved a single non-existent parent with strict=True instead of
    walking up to the nearest existing ancestor.
This commit is contained in:
Alexander Baranov
2026-08-24 15:38:49 +03:00
parent 64e6c2f5f5
commit 86dd79c8ae
10 changed files with 771 additions and 3 deletions
@@ -0,0 +1,80 @@
"""specification.md 4.1, 4.2, 19.2: envelope shape and exit codes."""
from __future__ import annotations
from pathlib import Path
from conftest import RESULT_SCHEMA, git_init, run_repoctl
def test_validate_success_envelope_and_exit_0(repo: Path) -> None:
result = run_repoctl("validate", "--repo", str(repo), "--output", "json")
assert result.returncode == 0, result.stderr
envelope = result.envelope()
assert envelope["schema"] == RESULT_SCHEMA
assert envelope["command"] == "validate"
assert envelope["outcome"] == "success"
assert envelope["findings"] == []
assert "data" in envelope
def test_validate_missing_repository_exits_3(tmp_path: Path) -> None:
empty = tmp_path / "not-a-repo"
empty.mkdir()
# No .git ancestor at all: repository cannot be resolved.
result = run_repoctl("validate", "--repo", str(empty), "--output", "json")
assert result.returncode == 3, (result.returncode, result.stderr)
def test_unknown_command_exits_2() -> None:
result = run_repoctl("not-a-real-command", "--output", "json")
assert result.returncode == 2
def test_validate_broken_contract_exits_1(tmp_path: Path) -> None:
root = tmp_path / "broken-repo"
(root / "ops").mkdir(parents=True)
(root / "ops" / "contract.yaml").write_text("kind: RepositoryContract\n", encoding="utf-8")
git_init(root)
result = run_repoctl("validate", "--repo", str(root), "--output", "json")
assert result.returncode == 1, (result.returncode, result.stdout, result.stderr)
envelope = result.envelope()
assert envelope["outcome"] == "failure"
assert any(f["severity"] == "error" for f in envelope["findings"])
def test_findings_have_required_shape(tmp_path: Path) -> None:
"""4.3 / 19.3: every finding carries severity in {info, warning, error}, a code, a message."""
root = tmp_path / "broken-repo-2"
(root / "ops").mkdir(parents=True)
(root / "ops" / "contract.yaml").write_text("not: a-valid-contract\n", encoding="utf-8")
git_init(root)
result = run_repoctl("validate", "--repo", str(root), "--output", "json")
envelope = result.envelope()
for finding in envelope["findings"]:
assert finding["severity"] in {"info", "warning", "error"}
assert isinstance(finding.get("code"), str) and finding["code"]
assert isinstance(finding.get("message"), str) and finding["message"]
def test_output_modes_yaml_and_json_both_parseable(repo: Path) -> None:
import json
from ruamel.yaml import YAML
json_result = run_repoctl("validate", "--repo", str(repo), "--output", "json")
json.loads(json_result.stdout)
yaml_result = run_repoctl("validate", "--repo", str(repo), "--output", "yaml")
YAML(typ="safe").load(yaml_result.stdout)
def test_human_output_writes_diagnostics_to_stderr_not_stdout(repo: Path) -> None:
"""PLAN.md: 'Machine output идёт только в stdout, диагностика процесса — в stderr.'"""
result = run_repoctl("validate", "--repo", str(repo), "--output", "human")
assert result.returncode == 0
# A clean validate has no findings to print, but stdout must not carry a JSON/YAML
# machine envelope in human mode either.
assert "schema:" not in result.stdout and '"schema"' not in result.stdout