test: add spec-driven universal CLI suite; fix context strict + path confinement bugs
- tests-universal/ drives repoctl only as a subprocess CLI, asserting
directly against docs/specification.md (envelope, exit codes 0-5,
validation rules MVP, security invariants) so the same suite can judge
any conforming implementation, including the sibling codex/repoctl-l1
branch built independently from the same baseline.
- Running it surfaced two real bugs, both fixed:
- `context --strict` only checked budget truncation, never stale or
missing referenced documents, despite specification.md 19.2 defining
exit 4 as "stale or incomplete context" (not just truncation).
- `paths.confined_path` crashed with an unhandled WinError2 when a
referenced path's parent directory also didn't exist, because it
resolved a single non-existent parent with strict=True instead of
walking up to the nearest existing ancestor.
This commit is contained in:
@@ -0,0 +1,147 @@
|
||||
"""Spec-driven, implementation-agnostic contract tests for `repoctl`.
|
||||
|
||||
These tests never import `stewardship` internals. They drive `repoctl` purely
|
||||
as a subprocess CLI and assert only what `docs/specification.md` requires, so
|
||||
the same suite can be pointed at any conforming implementation.
|
||||
|
||||
Target the CLI under test with one of:
|
||||
|
||||
- `REPOCTL_BIN="uv run --project <path> repoctl"` (space-separated argv prefix), or
|
||||
- `REPOCTL_PROJECT=<path>` to reuse the default `uv run --project <path> repoctl` form.
|
||||
|
||||
Defaults to the implementation checked out at the repository root.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import os
|
||||
import shutil
|
||||
import subprocess
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
import pytest
|
||||
|
||||
API_VERSION = "stewardship.corsus.ru/v1alpha1"
|
||||
RESULT_SCHEMA = "stewardship/repoctl-result@0.1"
|
||||
|
||||
REPO_ROOT = Path(__file__).resolve().parents[1]
|
||||
|
||||
|
||||
def _default_cmd() -> list[str]:
|
||||
explicit = os.environ.get("REPOCTL_BIN")
|
||||
if explicit:
|
||||
return explicit.split()
|
||||
project = os.environ.get("REPOCTL_PROJECT", str(REPO_ROOT))
|
||||
return ["uv", "run", "--project", project, "repoctl"]
|
||||
|
||||
|
||||
REPOCTL_CMD = _default_cmd()
|
||||
|
||||
|
||||
class Result:
|
||||
def __init__(self, proc: subprocess.CompletedProcess[str]) -> None:
|
||||
self.proc = proc
|
||||
self.returncode = proc.returncode
|
||||
self.stdout = proc.stdout
|
||||
self.stderr = proc.stderr
|
||||
|
||||
def envelope(self) -> dict[str, Any]:
|
||||
try:
|
||||
return json.loads(self.stdout)
|
||||
except json.JSONDecodeError as exc: # pragma: no cover - assertion aid
|
||||
raise AssertionError(
|
||||
f"stdout is not valid JSON: {exc}\nstdout={self.stdout!r}\nstderr={self.stderr!r}"
|
||||
) from exc
|
||||
|
||||
|
||||
def run_repoctl(*args: str, cwd: Path | None = None) -> Result:
|
||||
"""Run `repoctl <args>`.
|
||||
|
||||
`--output` is a per-subcommand option in this CLI (it must follow the
|
||||
subcommand, not precede it), so callers pass it as part of `args` after
|
||||
the subcommand name -- this helper does not inject it.
|
||||
"""
|
||||
cmd = [*REPOCTL_CMD, *args]
|
||||
proc = subprocess.run(
|
||||
cmd,
|
||||
cwd=str(cwd) if cwd else None,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
encoding="utf-8",
|
||||
)
|
||||
return Result(proc)
|
||||
|
||||
|
||||
def git_init(repo: Path) -> None:
|
||||
subprocess.run(["git", "init", "-q"], cwd=repo, check=True)
|
||||
subprocess.run(["git", "config", "user.email", "universal-test@example.com"], cwd=repo, check=True)
|
||||
subprocess.run(["git", "config", "user.name", "Universal Test"], cwd=repo, check=True)
|
||||
subprocess.run(["git", "add", "-A"], cwd=repo, check=True)
|
||||
subprocess.run(["git", "commit", "-q", "-m", "baseline"], cwd=repo, check=True)
|
||||
|
||||
|
||||
def write_yaml(path: Path, text: str) -> None:
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
path.write_text(text, encoding="utf-8", newline="\n")
|
||||
|
||||
|
||||
def minimal_contract(name: str = "sample-repo", *, extra_spec: str = "") -> str:
|
||||
return f"""\
|
||||
apiVersion: {API_VERSION}
|
||||
kind: RepositoryContract
|
||||
metadata:
|
||||
name: {name}
|
||||
title: Sample repository
|
||||
owners:
|
||||
- owner-team
|
||||
spec:
|
||||
summary: Minimal spec-conformant repository used by the universal test suite
|
||||
repository:
|
||||
type: project
|
||||
lifecycle: active
|
||||
agent:
|
||||
primary: CLAUDE.md
|
||||
portable: AGENTS.md
|
||||
knowledge:
|
||||
entrypoint: kdb/kdb.yaml
|
||||
{extra_spec}"""
|
||||
|
||||
|
||||
def minimal_kdb(name: str = "sample-repo", *, extra_spec: str = "") -> str:
|
||||
return f"""\
|
||||
apiVersion: {API_VERSION}
|
||||
kind: KnowledgeBase
|
||||
metadata:
|
||||
name: {name}-kdb
|
||||
spec:
|
||||
repository: repo:{name}
|
||||
defaultLanguage: en
|
||||
topics: []
|
||||
documents: []
|
||||
{extra_spec}"""
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def repo(tmp_path: Path) -> Path:
|
||||
"""A minimal, spec-valid L1 repository: contract + KDB + agent entrypoints + git."""
|
||||
|
||||
root = tmp_path / "sample-repo"
|
||||
write_yaml(root / "ops" / "contract.yaml", minimal_contract())
|
||||
write_yaml(root / "kdb" / "kdb.yaml", minimal_kdb())
|
||||
(root / "AGENTS.md").write_text("# Repository instructions\n", encoding="utf-8")
|
||||
(root / "CLAUDE.md").write_text("@AGENTS.md\n", encoding="utf-8")
|
||||
git_init(root)
|
||||
return root
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def scratch_repo(tmp_path: Path) -> Path:
|
||||
"""An empty git-initialized directory with no contract yet, for init/adopt tests."""
|
||||
|
||||
root = tmp_path / "scratch-repo"
|
||||
root.mkdir()
|
||||
(root / "README.md").write_text("# scratch\n", encoding="utf-8")
|
||||
git_init(root)
|
||||
return root
|
||||
Reference in New Issue
Block a user