test: add spec-driven universal CLI suite; fix context strict + path confinement bugs

- tests-universal/ drives repoctl only as a subprocess CLI, asserting
  directly against docs/specification.md (envelope, exit codes 0-5,
  validation rules MVP, security invariants) so the same suite can judge
  any conforming implementation, including the sibling codex/repoctl-l1
  branch built independently from the same baseline.
- Running it surfaced two real bugs, both fixed:
  - `context --strict` only checked budget truncation, never stale or
    missing referenced documents, despite specification.md 19.2 defining
    exit 4 as "stale or incomplete context" (not just truncation).
  - `paths.confined_path` crashed with an unhandled WinError2 when a
    referenced path's parent directory also didn't exist, because it
    resolved a single non-existent parent with strict=True instead of
    walking up to the nearest existing ancestor.
This commit is contained in:
Alexander Baranov
2026-08-24 15:38:49 +03:00
parent 64e6c2f5f5
commit 86dd79c8ae
10 changed files with 771 additions and 3 deletions
+147
View File
@@ -0,0 +1,147 @@
"""Spec-driven, implementation-agnostic contract tests for `repoctl`.
These tests never import `stewardship` internals. They drive `repoctl` purely
as a subprocess CLI and assert only what `docs/specification.md` requires, so
the same suite can be pointed at any conforming implementation.
Target the CLI under test with one of:
- `REPOCTL_BIN="uv run --project <path> repoctl"` (space-separated argv prefix), or
- `REPOCTL_PROJECT=<path>` to reuse the default `uv run --project <path> repoctl` form.
Defaults to the implementation checked out at the repository root.
"""
from __future__ import annotations
import json
import os
import shutil
import subprocess
from pathlib import Path
from typing import Any
import pytest
API_VERSION = "stewardship.corsus.ru/v1alpha1"
RESULT_SCHEMA = "stewardship/repoctl-result@0.1"
REPO_ROOT = Path(__file__).resolve().parents[1]
def _default_cmd() -> list[str]:
explicit = os.environ.get("REPOCTL_BIN")
if explicit:
return explicit.split()
project = os.environ.get("REPOCTL_PROJECT", str(REPO_ROOT))
return ["uv", "run", "--project", project, "repoctl"]
REPOCTL_CMD = _default_cmd()
class Result:
def __init__(self, proc: subprocess.CompletedProcess[str]) -> None:
self.proc = proc
self.returncode = proc.returncode
self.stdout = proc.stdout
self.stderr = proc.stderr
def envelope(self) -> dict[str, Any]:
try:
return json.loads(self.stdout)
except json.JSONDecodeError as exc: # pragma: no cover - assertion aid
raise AssertionError(
f"stdout is not valid JSON: {exc}\nstdout={self.stdout!r}\nstderr={self.stderr!r}"
) from exc
def run_repoctl(*args: str, cwd: Path | None = None) -> Result:
"""Run `repoctl <args>`.
`--output` is a per-subcommand option in this CLI (it must follow the
subcommand, not precede it), so callers pass it as part of `args` after
the subcommand name -- this helper does not inject it.
"""
cmd = [*REPOCTL_CMD, *args]
proc = subprocess.run(
cmd,
cwd=str(cwd) if cwd else None,
capture_output=True,
text=True,
encoding="utf-8",
)
return Result(proc)
def git_init(repo: Path) -> None:
subprocess.run(["git", "init", "-q"], cwd=repo, check=True)
subprocess.run(["git", "config", "user.email", "universal-test@example.com"], cwd=repo, check=True)
subprocess.run(["git", "config", "user.name", "Universal Test"], cwd=repo, check=True)
subprocess.run(["git", "add", "-A"], cwd=repo, check=True)
subprocess.run(["git", "commit", "-q", "-m", "baseline"], cwd=repo, check=True)
def write_yaml(path: Path, text: str) -> None:
path.parent.mkdir(parents=True, exist_ok=True)
path.write_text(text, encoding="utf-8", newline="\n")
def minimal_contract(name: str = "sample-repo", *, extra_spec: str = "") -> str:
return f"""\
apiVersion: {API_VERSION}
kind: RepositoryContract
metadata:
name: {name}
title: Sample repository
owners:
- owner-team
spec:
summary: Minimal spec-conformant repository used by the universal test suite
repository:
type: project
lifecycle: active
agent:
primary: CLAUDE.md
portable: AGENTS.md
knowledge:
entrypoint: kdb/kdb.yaml
{extra_spec}"""
def minimal_kdb(name: str = "sample-repo", *, extra_spec: str = "") -> str:
return f"""\
apiVersion: {API_VERSION}
kind: KnowledgeBase
metadata:
name: {name}-kdb
spec:
repository: repo:{name}
defaultLanguage: en
topics: []
documents: []
{extra_spec}"""
@pytest.fixture
def repo(tmp_path: Path) -> Path:
"""A minimal, spec-valid L1 repository: contract + KDB + agent entrypoints + git."""
root = tmp_path / "sample-repo"
write_yaml(root / "ops" / "contract.yaml", minimal_contract())
write_yaml(root / "kdb" / "kdb.yaml", minimal_kdb())
(root / "AGENTS.md").write_text("# Repository instructions\n", encoding="utf-8")
(root / "CLAUDE.md").write_text("@AGENTS.md\n", encoding="utf-8")
git_init(root)
return root
@pytest.fixture
def scratch_repo(tmp_path: Path) -> Path:
"""An empty git-initialized directory with no contract yet, for init/adopt tests."""
root = tmp_path / "scratch-repo"
root.mkdir()
(root / "README.md").write_text("# scratch\n", encoding="utf-8")
git_init(root)
return root