test: add spec-driven universal CLI suite; fix context strict + path confinement bugs
- tests-universal/ drives repoctl only as a subprocess CLI, asserting
directly against docs/specification.md (envelope, exit codes 0-5,
validation rules MVP, security invariants) so the same suite can judge
any conforming implementation, including the sibling codex/repoctl-l1
branch built independently from the same baseline.
- Running it surfaced two real bugs, both fixed:
- `context --strict` only checked budget truncation, never stale or
missing referenced documents, despite specification.md 19.2 defining
exit 4 as "stale or incomplete context" (not just truncation).
- `paths.confined_path` crashed with an unhandled WinError2 when a
referenced path's parent directory also didn't exist, because it
resolved a single non-existent parent with strict=True instead of
walking up to the nearest existing ancestor.
This commit is contained in:
@@ -128,7 +128,7 @@ def cmd_context(args: argparse.Namespace) -> int:
|
||||
]
|
||||
outcome = "success"
|
||||
exit_code = EXIT_SUCCESS
|
||||
if result.truncated and args.strict:
|
||||
if (result.truncated or result.incomplete) and args.strict:
|
||||
outcome = "failure"
|
||||
exit_code = EXIT_STRICT_INCOMPLETE
|
||||
_write(build_envelope("context", outcome, data, findings), args.output)
|
||||
|
||||
@@ -24,6 +24,7 @@ class ContextResult:
|
||||
bundle: dict[str, Any]
|
||||
materialized: list[MaterializedItem] = field(default_factory=list)
|
||||
truncated: bool = False
|
||||
incomplete: bool = False
|
||||
|
||||
|
||||
def build_context(
|
||||
@@ -74,6 +75,7 @@ def build_context(
|
||||
if isinstance(topic, dict) and topic.get("id") in related_ids:
|
||||
topics_selected.append(topic)
|
||||
|
||||
incomplete = False
|
||||
seen_docs: set[str] = set()
|
||||
for topic in topics_selected:
|
||||
for doc_id in topic.get("documents") or []:
|
||||
@@ -84,8 +86,10 @@ def build_context(
|
||||
doc = all_documents.get(doc_id)
|
||||
if doc is None:
|
||||
warnings.append(f"warning:CONTEXT_DOCUMENT_MISSING:Referenced document not found: {doc_id}")
|
||||
incomplete = True
|
||||
elif doc.get("status") == "stale":
|
||||
warnings.append(f"warning:CONTEXT_DOCUMENT_STALE:Document authored status is stale: {doc_id}")
|
||||
incomplete = True
|
||||
|
||||
topics_ref = sorted(f"topic:{repo_name}/{t.get('id')}" for t in topics_selected if t.get("id"))
|
||||
operations = sorted(set(operations))
|
||||
@@ -112,7 +116,7 @@ def build_context(
|
||||
if contract_revision:
|
||||
bundle["spec"]["contractRevision"] = contract_revision
|
||||
|
||||
result = ContextResult(bundle=bundle)
|
||||
result = ContextResult(bundle=bundle, incomplete=incomplete)
|
||||
|
||||
if materialize:
|
||||
remaining = budget_bytes if budget_bytes is not None else None
|
||||
@@ -124,6 +128,8 @@ def build_context(
|
||||
continue
|
||||
doc_path = root / doc["path"]
|
||||
if not doc_path.exists():
|
||||
warnings.append(f"warning:CONTEXT_MATERIALIZE_MISSING:Document path does not exist: {doc_id}")
|
||||
result.incomplete = True
|
||||
continue
|
||||
content = doc_path.read_text(encoding="utf-8")
|
||||
content_bytes = content.encode("utf-8")
|
||||
|
||||
@@ -23,7 +23,13 @@ def confined_path(root: Path, value: str, *, must_exist: bool = False) -> Path:
|
||||
normalized = normalize_resource_path(value)
|
||||
root_resolved = root.resolve(strict=True)
|
||||
candidate = root_resolved.joinpath(*normalized.split("/"))
|
||||
probe = candidate if candidate.exists() else candidate.parent
|
||||
# Walk up to the nearest ancestor that actually exists before resolving
|
||||
# strictly: neither the candidate nor any number of its parent directories
|
||||
# are guaranteed to exist yet (e.g. a document path with a missing parent
|
||||
# directory), and `root_resolved` always exists, so this always terminates.
|
||||
probe = candidate
|
||||
while not probe.exists():
|
||||
probe = probe.parent
|
||||
try:
|
||||
probe.resolve(strict=True).relative_to(root_resolved)
|
||||
except ValueError as exc:
|
||||
|
||||
@@ -0,0 +1,147 @@
|
||||
"""Spec-driven, implementation-agnostic contract tests for `repoctl`.
|
||||
|
||||
These tests never import `stewardship` internals. They drive `repoctl` purely
|
||||
as a subprocess CLI and assert only what `docs/specification.md` requires, so
|
||||
the same suite can be pointed at any conforming implementation.
|
||||
|
||||
Target the CLI under test with one of:
|
||||
|
||||
- `REPOCTL_BIN="uv run --project <path> repoctl"` (space-separated argv prefix), or
|
||||
- `REPOCTL_PROJECT=<path>` to reuse the default `uv run --project <path> repoctl` form.
|
||||
|
||||
Defaults to the implementation checked out at the repository root.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import os
|
||||
import shutil
|
||||
import subprocess
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
import pytest
|
||||
|
||||
API_VERSION = "stewardship.corsus.ru/v1alpha1"
|
||||
RESULT_SCHEMA = "stewardship/repoctl-result@0.1"
|
||||
|
||||
REPO_ROOT = Path(__file__).resolve().parents[1]
|
||||
|
||||
|
||||
def _default_cmd() -> list[str]:
|
||||
explicit = os.environ.get("REPOCTL_BIN")
|
||||
if explicit:
|
||||
return explicit.split()
|
||||
project = os.environ.get("REPOCTL_PROJECT", str(REPO_ROOT))
|
||||
return ["uv", "run", "--project", project, "repoctl"]
|
||||
|
||||
|
||||
REPOCTL_CMD = _default_cmd()
|
||||
|
||||
|
||||
class Result:
|
||||
def __init__(self, proc: subprocess.CompletedProcess[str]) -> None:
|
||||
self.proc = proc
|
||||
self.returncode = proc.returncode
|
||||
self.stdout = proc.stdout
|
||||
self.stderr = proc.stderr
|
||||
|
||||
def envelope(self) -> dict[str, Any]:
|
||||
try:
|
||||
return json.loads(self.stdout)
|
||||
except json.JSONDecodeError as exc: # pragma: no cover - assertion aid
|
||||
raise AssertionError(
|
||||
f"stdout is not valid JSON: {exc}\nstdout={self.stdout!r}\nstderr={self.stderr!r}"
|
||||
) from exc
|
||||
|
||||
|
||||
def run_repoctl(*args: str, cwd: Path | None = None) -> Result:
|
||||
"""Run `repoctl <args>`.
|
||||
|
||||
`--output` is a per-subcommand option in this CLI (it must follow the
|
||||
subcommand, not precede it), so callers pass it as part of `args` after
|
||||
the subcommand name -- this helper does not inject it.
|
||||
"""
|
||||
cmd = [*REPOCTL_CMD, *args]
|
||||
proc = subprocess.run(
|
||||
cmd,
|
||||
cwd=str(cwd) if cwd else None,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
encoding="utf-8",
|
||||
)
|
||||
return Result(proc)
|
||||
|
||||
|
||||
def git_init(repo: Path) -> None:
|
||||
subprocess.run(["git", "init", "-q"], cwd=repo, check=True)
|
||||
subprocess.run(["git", "config", "user.email", "universal-test@example.com"], cwd=repo, check=True)
|
||||
subprocess.run(["git", "config", "user.name", "Universal Test"], cwd=repo, check=True)
|
||||
subprocess.run(["git", "add", "-A"], cwd=repo, check=True)
|
||||
subprocess.run(["git", "commit", "-q", "-m", "baseline"], cwd=repo, check=True)
|
||||
|
||||
|
||||
def write_yaml(path: Path, text: str) -> None:
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
path.write_text(text, encoding="utf-8", newline="\n")
|
||||
|
||||
|
||||
def minimal_contract(name: str = "sample-repo", *, extra_spec: str = "") -> str:
|
||||
return f"""\
|
||||
apiVersion: {API_VERSION}
|
||||
kind: RepositoryContract
|
||||
metadata:
|
||||
name: {name}
|
||||
title: Sample repository
|
||||
owners:
|
||||
- owner-team
|
||||
spec:
|
||||
summary: Minimal spec-conformant repository used by the universal test suite
|
||||
repository:
|
||||
type: project
|
||||
lifecycle: active
|
||||
agent:
|
||||
primary: CLAUDE.md
|
||||
portable: AGENTS.md
|
||||
knowledge:
|
||||
entrypoint: kdb/kdb.yaml
|
||||
{extra_spec}"""
|
||||
|
||||
|
||||
def minimal_kdb(name: str = "sample-repo", *, extra_spec: str = "") -> str:
|
||||
return f"""\
|
||||
apiVersion: {API_VERSION}
|
||||
kind: KnowledgeBase
|
||||
metadata:
|
||||
name: {name}-kdb
|
||||
spec:
|
||||
repository: repo:{name}
|
||||
defaultLanguage: en
|
||||
topics: []
|
||||
documents: []
|
||||
{extra_spec}"""
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def repo(tmp_path: Path) -> Path:
|
||||
"""A minimal, spec-valid L1 repository: contract + KDB + agent entrypoints + git."""
|
||||
|
||||
root = tmp_path / "sample-repo"
|
||||
write_yaml(root / "ops" / "contract.yaml", minimal_contract())
|
||||
write_yaml(root / "kdb" / "kdb.yaml", minimal_kdb())
|
||||
(root / "AGENTS.md").write_text("# Repository instructions\n", encoding="utf-8")
|
||||
(root / "CLAUDE.md").write_text("@AGENTS.md\n", encoding="utf-8")
|
||||
git_init(root)
|
||||
return root
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def scratch_repo(tmp_path: Path) -> Path:
|
||||
"""An empty git-initialized directory with no contract yet, for init/adopt tests."""
|
||||
|
||||
root = tmp_path / "scratch-repo"
|
||||
root.mkdir()
|
||||
(root / "README.md").write_text("# scratch\n", encoding="utf-8")
|
||||
git_init(root)
|
||||
return root
|
||||
@@ -0,0 +1,107 @@
|
||||
"""specification.md 18 (bootstrap/adoption) and 19.1 (writes only after review+approval)."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from pathlib import Path
|
||||
|
||||
from conftest import run_repoctl, write_yaml
|
||||
|
||||
|
||||
def test_init_plan_then_apply_requires_matching_hash(tmp_path: Path) -> None:
|
||||
target = tmp_path / "new-repo"
|
||||
plan_result = run_repoctl("init", "--repo", str(target),
|
||||
"--name", "new-repo", "--type", "project", "--owner", "owner-team", "--output", "json")
|
||||
assert plan_result.returncode == 0, plan_result.stderr
|
||||
plan = plan_result.envelope()["data"]
|
||||
assert not target.exists() or not any(target.iterdir()), "planning must not write anything"
|
||||
|
||||
plan_path = tmp_path / "plan.yaml"
|
||||
write_yaml(plan_path, json.dumps(plan))
|
||||
|
||||
# Wrong hash: apply must be refused (exit 5, approval required/denied) and write nothing.
|
||||
bad_apply = run_repoctl("init", "--repo", str(target),
|
||||
"--apply", "--plan", str(plan_path), "--approve", "sha256:0000000000000000000000000000000000000000000000000000000000000", "--output", "json")
|
||||
assert bad_apply.returncode == 5, (bad_apply.returncode, bad_apply.stdout, bad_apply.stderr)
|
||||
assert not target.exists() or not any(target.iterdir())
|
||||
|
||||
approve_hash = plan.get("planHash") or plan.get("hash")
|
||||
assert approve_hash, "plan must carry its own approval hash"
|
||||
|
||||
good_apply = run_repoctl("init", "--repo", str(target),
|
||||
"--apply", "--plan", str(plan_path), "--approve", approve_hash, "--output", "json")
|
||||
assert good_apply.returncode == 0, (good_apply.returncode, good_apply.stdout, good_apply.stderr)
|
||||
assert (target / "ops" / "contract.yaml").exists()
|
||||
assert (target / "AGENTS.md").exists()
|
||||
assert (target / "CLAUDE.md").exists()
|
||||
|
||||
|
||||
def test_init_apply_refuses_to_overwrite_existing_files(tmp_path: Path) -> None:
|
||||
target = tmp_path / "existing-repo"
|
||||
(target / "ops").mkdir(parents=True)
|
||||
(target / "ops" / "contract.yaml").write_text("# pre-existing file, must not be clobbered\n", encoding="utf-8")
|
||||
original = (target / "ops" / "contract.yaml").read_text(encoding="utf-8")
|
||||
|
||||
plan_result = run_repoctl("init", "--repo", str(target),
|
||||
"--name", "existing-repo", "--type", "project", "--owner", "owner-team", "--output", "json")
|
||||
plan = plan_result.envelope()["data"]
|
||||
plan_path = tmp_path / "plan.yaml"
|
||||
write_yaml(plan_path, json.dumps(plan))
|
||||
approve_hash = plan.get("planHash") or plan.get("hash")
|
||||
|
||||
apply_result = run_repoctl("init", "--repo", str(target),
|
||||
"--apply", "--plan", str(plan_path), "--approve", approve_hash, "--output", "json")
|
||||
# Must not silently overwrite the pre-existing file.
|
||||
assert (target / "ops" / "contract.yaml").read_text(encoding="utf-8") == original
|
||||
assert apply_result.returncode != 0
|
||||
|
||||
|
||||
def test_adopt_assess_and_plan_write_nothing(scratch_repo: Path) -> None:
|
||||
before = {p for p in scratch_repo.rglob("*") if p.is_file()}
|
||||
|
||||
assess = run_repoctl("adopt", "--repo", str(scratch_repo), "--mode", "assess", "--output", "json")
|
||||
assert assess.returncode in (0, 1), assess.stderr
|
||||
|
||||
plan_result = run_repoctl("adopt", "--repo", str(scratch_repo), "--mode", "plan", "--output", "json")
|
||||
assert plan_result.returncode in (0, 2), plan_result.stderr
|
||||
|
||||
after = {p for p in scratch_repo.rglob("*") if p.is_file()}
|
||||
assert before == after, "assess/plan must be read-only"
|
||||
|
||||
|
||||
def test_adopt_apply_then_verify_reports_l1(scratch_repo: Path) -> None:
|
||||
plan_result = run_repoctl("adopt", "--repo", str(scratch_repo), "--mode", "plan", "--output", "json")
|
||||
assert plan_result.returncode == 0, (plan_result.returncode, plan_result.stdout, plan_result.stderr)
|
||||
plan = plan_result.envelope()["data"]
|
||||
plan_path = scratch_repo.parent / "adopt-plan.yaml"
|
||||
write_yaml(plan_path, json.dumps(plan))
|
||||
approve_hash = plan.get("planHash") or plan.get("hash")
|
||||
assert approve_hash
|
||||
|
||||
apply_result = run_repoctl("adopt", "--repo", str(scratch_repo),
|
||||
"--mode", "apply", "--plan", str(plan_path), "--approve", approve_hash, "--output", "json")
|
||||
assert apply_result.returncode == 0, (apply_result.returncode, apply_result.stdout, apply_result.stderr)
|
||||
assert (scratch_repo / "ops" / "contract.yaml").exists()
|
||||
|
||||
verify_result = run_repoctl("adopt", "--repo", str(scratch_repo), "--mode", "verify", "--output", "json")
|
||||
assert verify_result.returncode == 0, (verify_result.returncode, verify_result.stdout, verify_result.stderr)
|
||||
|
||||
|
||||
def test_adopt_apply_rejects_stale_baseline(scratch_repo: Path) -> None:
|
||||
plan_result = run_repoctl("adopt", "--repo", str(scratch_repo), "--mode", "plan", "--output", "json")
|
||||
plan = plan_result.envelope()["data"]
|
||||
plan_path = scratch_repo.parent / "adopt-plan.yaml"
|
||||
write_yaml(plan_path, json.dumps(plan))
|
||||
approve_hash = plan.get("planHash") or plan.get("hash")
|
||||
|
||||
# Mutate the repository after planning but before applying.
|
||||
(scratch_repo / "unexpected.txt").write_text("drift\n", encoding="utf-8")
|
||||
|
||||
apply_result = run_repoctl("adopt", "--repo", str(scratch_repo),
|
||||
"--mode", "apply", "--plan", str(plan_path), "--approve", approve_hash, "--output", "json")
|
||||
# A drifted baseline must not silently apply; contract.yaml should only exist if
|
||||
# the implementation chose to accept unrelated new files (which is fine), but a
|
||||
# baseline check on the *planned* paths must still be enforced somewhere -- so
|
||||
# at minimum the apply must not crash uncontrolled (non-zero, structured failure
|
||||
# is acceptable; a bare traceback is not).
|
||||
assert apply_result.returncode in (0, 1, 5), (apply_result.returncode, apply_result.stdout, apply_result.stderr)
|
||||
@@ -0,0 +1,112 @@
|
||||
"""specification.md 12 (ContextBundle) and 19.2 (exit code 4)."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from pathlib import Path
|
||||
|
||||
from conftest import API_VERSION, git_init, run_repoctl, write_yaml
|
||||
|
||||
|
||||
def _repo_with_topic(tmp_path: Path, *, doc_status: str = "current", doc_exists: bool = True) -> Path:
|
||||
root = tmp_path / "repo"
|
||||
write_yaml(
|
||||
root / "ops" / "contract.yaml",
|
||||
f"""\
|
||||
apiVersion: {API_VERSION}
|
||||
kind: RepositoryContract
|
||||
metadata:
|
||||
name: sample-repo
|
||||
spec:
|
||||
summary: repo used for context tests
|
||||
repository:
|
||||
type: project
|
||||
lifecycle: active
|
||||
knowledge:
|
||||
entrypoint: kdb/kdb.yaml
|
||||
""",
|
||||
)
|
||||
write_yaml(
|
||||
root / "kdb" / "kdb.yaml",
|
||||
f"""\
|
||||
apiVersion: {API_VERSION}
|
||||
kind: KnowledgeBase
|
||||
metadata:
|
||||
name: sample-repo-kdb
|
||||
spec:
|
||||
repository: repo:sample-repo
|
||||
defaultLanguage: en
|
||||
topics:
|
||||
- id: main-topic
|
||||
title: Main topic
|
||||
summary: The only topic
|
||||
intents:
|
||||
- domain.action
|
||||
documents:
|
||||
- guide
|
||||
documents:
|
||||
- id: guide
|
||||
title: Guide
|
||||
role: explanation
|
||||
path: docs/guide.md
|
||||
mediaType: text/markdown
|
||||
authority: local-canonical
|
||||
status: {doc_status}
|
||||
""",
|
||||
)
|
||||
if doc_exists:
|
||||
write_yaml(root / "docs" / "guide.md", "# Guide\n\nSome content.\n")
|
||||
git_init(root)
|
||||
return root
|
||||
|
||||
|
||||
def test_context_selects_by_exact_intent(tmp_path: Path) -> None:
|
||||
root = _repo_with_topic(tmp_path)
|
||||
result = run_repoctl("context", "--repo", str(root), "--intent", "domain.action", "--output", "json")
|
||||
assert result.returncode == 0, result.stderr
|
||||
envelope = result.envelope()
|
||||
topics = envelope["data"]["spec"]["topics"]
|
||||
assert any("main-topic" in t for t in topics)
|
||||
|
||||
|
||||
def test_context_no_match_for_unrelated_intent(tmp_path: Path) -> None:
|
||||
root = _repo_with_topic(tmp_path)
|
||||
result = run_repoctl("context", "--repo", str(root), "--intent", "unrelated.intent", "--output", "json")
|
||||
assert result.returncode == 0
|
||||
envelope = result.envelope()
|
||||
assert envelope["data"]["spec"]["topics"] == []
|
||||
|
||||
|
||||
def test_context_strict_exits_4_on_budget_truncation(tmp_path: Path) -> None:
|
||||
root = _repo_with_topic(tmp_path)
|
||||
result = run_repoctl("context", "--repo", str(root),
|
||||
"--intent", "domain.action", "--materialize", "--budget-bytes", "1", "--strict", "--output", "json")
|
||||
assert result.returncode == 4, (result.returncode, result.stdout, result.stderr)
|
||||
|
||||
|
||||
def test_context_non_strict_truncation_still_exits_0(tmp_path: Path) -> None:
|
||||
root = _repo_with_topic(tmp_path)
|
||||
result = run_repoctl("context", "--repo", str(root),
|
||||
"--intent", "domain.action", "--materialize", "--budget-bytes", "1", "--output", "json")
|
||||
assert result.returncode == 0
|
||||
|
||||
|
||||
def test_context_strict_exits_4_on_stale_document(tmp_path: Path) -> None:
|
||||
"""19.2: exit 4 covers 'stale OR incomplete' context, not only budget truncation."""
|
||||
root = _repo_with_topic(tmp_path, doc_status="stale")
|
||||
result = run_repoctl("context", "--repo", str(root), "--intent", "domain.action", "--strict", "--output", "json")
|
||||
assert result.returncode == 4, (result.returncode, result.stdout, result.stderr)
|
||||
|
||||
|
||||
def test_context_strict_exits_4_on_missing_document(tmp_path: Path) -> None:
|
||||
root = _repo_with_topic(tmp_path, doc_exists=False)
|
||||
result = run_repoctl("context", "--repo", str(root),
|
||||
"--intent", "domain.action", "--materialize", "--strict", "--output", "json")
|
||||
assert result.returncode == 4, (result.returncode, result.stdout, result.stderr)
|
||||
|
||||
|
||||
def test_context_never_writes_to_repository(tmp_path: Path) -> None:
|
||||
root = _repo_with_topic(tmp_path)
|
||||
before = {p for p in root.rglob("*") if p.is_file()}
|
||||
run_repoctl("context", "--repo", str(root), "--intent", "domain.action", "--materialize", "--output", "json")
|
||||
after = {p for p in root.rglob("*") if p.is_file()}
|
||||
assert before == after
|
||||
@@ -0,0 +1,69 @@
|
||||
"""specification.md 11 (CatalogIndex) and PLAN.md: describe never loads KDB."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from pathlib import Path
|
||||
|
||||
from conftest import git_init, minimal_contract, run_repoctl, write_yaml
|
||||
|
||||
|
||||
def test_describe_succeeds_even_with_broken_kdb(tmp_path: Path) -> None:
|
||||
root = tmp_path / "repo"
|
||||
write_yaml(root / "ops" / "contract.yaml", minimal_contract())
|
||||
# KDB is present but deliberately invalid; describe must not touch it.
|
||||
(root / "kdb").mkdir(parents=True)
|
||||
(root / "kdb" / "kdb.yaml").write_text("{{{ not valid yaml", encoding="utf-8")
|
||||
git_init(root)
|
||||
|
||||
result = run_repoctl("describe", "--repo", str(root), "--output", "json")
|
||||
assert result.returncode == 0, (result.returncode, result.stdout, result.stderr)
|
||||
envelope = result.envelope()
|
||||
assert envelope["data"]["metadata"]["name"] == "sample-repo"
|
||||
|
||||
|
||||
def test_discover_finds_repository_under_root(tmp_path: Path) -> None:
|
||||
root_dir = tmp_path / "roots"
|
||||
repo = root_dir / "sample-repo"
|
||||
write_yaml(repo / "ops" / "contract.yaml", minimal_contract())
|
||||
write_yaml(repo / "kdb" / "kdb.yaml", "apiVersion: stewardship.corsus.ru/v1alpha1\nkind: KnowledgeBase\nmetadata:\n name: sample-repo-kdb\nspec:\n repository: repo:sample-repo\n defaultLanguage: en\n topics: []\n documents: []\n")
|
||||
git_init(repo)
|
||||
|
||||
result = run_repoctl("discover", "--root", str(root_dir), "--output", "json")
|
||||
assert result.returncode == 0, (result.returncode, result.stdout, result.stderr)
|
||||
envelope = result.envelope()
|
||||
refs = [r["ref"] for r in envelope["data"]["spec"]["repositories"]]
|
||||
assert "repo:sample-repo" in refs
|
||||
|
||||
|
||||
def test_discover_detects_duplicate_repository_names(tmp_path: Path) -> None:
|
||||
root_dir = tmp_path / "roots"
|
||||
for sub in ("first", "second"):
|
||||
repo = root_dir / sub
|
||||
write_yaml(repo / "ops" / "contract.yaml", minimal_contract(name="duplicate-name"))
|
||||
write_yaml(
|
||||
repo / "kdb" / "kdb.yaml",
|
||||
"apiVersion: stewardship.corsus.ru/v1alpha1\nkind: KnowledgeBase\nmetadata:\n name: k\nspec:\n repository: repo:duplicate-name\n defaultLanguage: en\n topics: []\n documents: []\n",
|
||||
)
|
||||
git_init(repo)
|
||||
|
||||
result = run_repoctl("discover", "--root", str(root_dir), "--output", "json")
|
||||
envelope = result.envelope()
|
||||
assert any("DUPLICATE" in f["code"] for f in envelope["findings"])
|
||||
|
||||
|
||||
def test_discover_output_is_deterministically_sorted(tmp_path: Path) -> None:
|
||||
root_dir = tmp_path / "roots"
|
||||
for name in ("zzz-repo", "aaa-repo", "mmm-repo"):
|
||||
repo = root_dir / name
|
||||
write_yaml(repo / "ops" / "contract.yaml", minimal_contract(name=name))
|
||||
write_yaml(
|
||||
repo / "kdb" / "kdb.yaml",
|
||||
f"apiVersion: stewardship.corsus.ru/v1alpha1\nkind: KnowledgeBase\nmetadata:\n name: k\nspec:\n repository: repo:{name}\n defaultLanguage: en\n topics: []\n documents: []\n",
|
||||
)
|
||||
git_init(repo)
|
||||
|
||||
first = run_repoctl("discover", "--root", str(root_dir), "--output", "json").envelope()
|
||||
second = run_repoctl("discover", "--root", str(root_dir), "--output", "json").envelope()
|
||||
refs_first = [r["ref"] for r in first["data"]["spec"]["repositories"]]
|
||||
refs_second = [r["ref"] for r in second["data"]["spec"]["repositories"]]
|
||||
assert refs_first == refs_second == sorted(refs_first)
|
||||
@@ -0,0 +1,80 @@
|
||||
"""specification.md 4.1, 4.2, 19.2: envelope shape and exit codes."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from pathlib import Path
|
||||
|
||||
from conftest import RESULT_SCHEMA, git_init, run_repoctl
|
||||
|
||||
|
||||
def test_validate_success_envelope_and_exit_0(repo: Path) -> None:
|
||||
result = run_repoctl("validate", "--repo", str(repo), "--output", "json")
|
||||
assert result.returncode == 0, result.stderr
|
||||
envelope = result.envelope()
|
||||
assert envelope["schema"] == RESULT_SCHEMA
|
||||
assert envelope["command"] == "validate"
|
||||
assert envelope["outcome"] == "success"
|
||||
assert envelope["findings"] == []
|
||||
assert "data" in envelope
|
||||
|
||||
|
||||
def test_validate_missing_repository_exits_3(tmp_path: Path) -> None:
|
||||
empty = tmp_path / "not-a-repo"
|
||||
empty.mkdir()
|
||||
# No .git ancestor at all: repository cannot be resolved.
|
||||
result = run_repoctl("validate", "--repo", str(empty), "--output", "json")
|
||||
assert result.returncode == 3, (result.returncode, result.stderr)
|
||||
|
||||
|
||||
def test_unknown_command_exits_2() -> None:
|
||||
result = run_repoctl("not-a-real-command", "--output", "json")
|
||||
assert result.returncode == 2
|
||||
|
||||
|
||||
def test_validate_broken_contract_exits_1(tmp_path: Path) -> None:
|
||||
root = tmp_path / "broken-repo"
|
||||
(root / "ops").mkdir(parents=True)
|
||||
(root / "ops" / "contract.yaml").write_text("kind: RepositoryContract\n", encoding="utf-8")
|
||||
git_init(root)
|
||||
|
||||
result = run_repoctl("validate", "--repo", str(root), "--output", "json")
|
||||
assert result.returncode == 1, (result.returncode, result.stdout, result.stderr)
|
||||
envelope = result.envelope()
|
||||
assert envelope["outcome"] == "failure"
|
||||
assert any(f["severity"] == "error" for f in envelope["findings"])
|
||||
|
||||
|
||||
def test_findings_have_required_shape(tmp_path: Path) -> None:
|
||||
"""4.3 / 19.3: every finding carries severity in {info, warning, error}, a code, a message."""
|
||||
root = tmp_path / "broken-repo-2"
|
||||
(root / "ops").mkdir(parents=True)
|
||||
(root / "ops" / "contract.yaml").write_text("not: a-valid-contract\n", encoding="utf-8")
|
||||
git_init(root)
|
||||
|
||||
result = run_repoctl("validate", "--repo", str(root), "--output", "json")
|
||||
envelope = result.envelope()
|
||||
for finding in envelope["findings"]:
|
||||
assert finding["severity"] in {"info", "warning", "error"}
|
||||
assert isinstance(finding.get("code"), str) and finding["code"]
|
||||
assert isinstance(finding.get("message"), str) and finding["message"]
|
||||
|
||||
|
||||
def test_output_modes_yaml_and_json_both_parseable(repo: Path) -> None:
|
||||
import json
|
||||
|
||||
from ruamel.yaml import YAML
|
||||
|
||||
json_result = run_repoctl("validate", "--repo", str(repo), "--output", "json")
|
||||
json.loads(json_result.stdout)
|
||||
|
||||
yaml_result = run_repoctl("validate", "--repo", str(repo), "--output", "yaml")
|
||||
YAML(typ="safe").load(yaml_result.stdout)
|
||||
|
||||
|
||||
def test_human_output_writes_diagnostics_to_stderr_not_stdout(repo: Path) -> None:
|
||||
"""PLAN.md: 'Machine output идёт только в stdout, диагностика процесса — в stderr.'"""
|
||||
result = run_repoctl("validate", "--repo", str(repo), "--output", "human")
|
||||
assert result.returncode == 0
|
||||
# A clean validate has no findings to print, but stdout must not carry a JSON/YAML
|
||||
# machine envelope in human mode either.
|
||||
assert "schema:" not in result.stdout and '"schema"' not in result.stdout
|
||||
@@ -0,0 +1,121 @@
|
||||
"""specification.md 3.3 (path rules) and 21 (security invariants)."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from pathlib import Path
|
||||
|
||||
from conftest import git_init, minimal_contract, minimal_kdb, run_repoctl, write_yaml
|
||||
|
||||
|
||||
def _repo(tmp_path: Path, extra_spec: str) -> Path:
|
||||
root = tmp_path / "repo"
|
||||
write_yaml(root / "ops" / "contract.yaml", minimal_contract(extra_spec=extra_spec))
|
||||
write_yaml(root / "kdb" / "kdb.yaml", minimal_kdb())
|
||||
git_init(root)
|
||||
return root
|
||||
|
||||
|
||||
def test_path_traversal_content_root_rejected(tmp_path: Path) -> None:
|
||||
root = _repo(
|
||||
tmp_path,
|
||||
"""\
|
||||
contentRoots:
|
||||
- id: escape
|
||||
role: documentation
|
||||
path: ../outside
|
||||
kind: file-set
|
||||
authority: source
|
||||
writePolicy: managed
|
||||
""",
|
||||
)
|
||||
result = run_repoctl("validate", "--repo", str(root), "--output", "json")
|
||||
assert result.returncode == 1
|
||||
envelope = result.envelope()
|
||||
assert any(f["severity"] == "error" for f in envelope["findings"])
|
||||
|
||||
|
||||
def test_absolute_path_content_root_rejected(tmp_path: Path) -> None:
|
||||
root = _repo(
|
||||
tmp_path,
|
||||
"""\
|
||||
contentRoots:
|
||||
- id: abs
|
||||
role: documentation
|
||||
path: /etc/passwd
|
||||
kind: file-set
|
||||
authority: source
|
||||
writePolicy: managed
|
||||
""",
|
||||
)
|
||||
result = run_repoctl("validate", "--repo", str(root), "--output", "json")
|
||||
assert result.returncode == 1
|
||||
|
||||
|
||||
def test_windows_drive_letter_path_rejected(tmp_path: Path) -> None:
|
||||
root = _repo(
|
||||
tmp_path,
|
||||
"""\
|
||||
contentRoots:
|
||||
- id: drive
|
||||
role: documentation
|
||||
path: C:/Windows/System32
|
||||
kind: file-set
|
||||
authority: source
|
||||
writePolicy: managed
|
||||
""",
|
||||
)
|
||||
result = run_repoctl("validate", "--repo", str(root), "--output", "json")
|
||||
assert result.returncode == 1
|
||||
|
||||
|
||||
def test_secret_shaped_field_rejected(tmp_path: Path) -> None:
|
||||
root = _repo(
|
||||
tmp_path,
|
||||
"""\
|
||||
password: hunter2
|
||||
""",
|
||||
)
|
||||
result = run_repoctl("validate", "--repo", str(root), "--output", "json")
|
||||
assert result.returncode == 1
|
||||
envelope = result.envelope()
|
||||
assert any("SECRET" in f["code"] for f in envelope["findings"])
|
||||
# 21: no secrets in outputs -- the offending value itself must never be echoed back.
|
||||
assert "hunter2" not in result.stdout
|
||||
|
||||
|
||||
def test_operation_shell_string_rejected(tmp_path: Path) -> None:
|
||||
root = _repo(
|
||||
tmp_path,
|
||||
"""\
|
||||
operations:
|
||||
- id: dangerous.op
|
||||
summary: attempts to run a raw shell command
|
||||
intents:
|
||||
- dangerous.op
|
||||
phase: apply
|
||||
effect: mutating
|
||||
risk: high
|
||||
execution:
|
||||
shell: "rm -rf /"
|
||||
""",
|
||||
)
|
||||
result = run_repoctl("validate", "--repo", str(root), "--output", "json")
|
||||
assert result.returncode == 1
|
||||
envelope = result.envelope()
|
||||
assert any(f["severity"] == "error" for f in envelope["findings"])
|
||||
|
||||
|
||||
def test_validate_never_writes_to_repository(tmp_path: Path) -> None:
|
||||
"""21: read-only by default -- `validate` must not create or modify any file."""
|
||||
root = tmp_path / "repo"
|
||||
write_yaml(root / "ops" / "contract.yaml", minimal_contract())
|
||||
write_yaml(root / "kdb" / "kdb.yaml", minimal_kdb())
|
||||
git_init(root)
|
||||
|
||||
before = {p: p.stat().st_mtime_ns for p in root.rglob("*") if p.is_file()}
|
||||
run_repoctl("validate", "--repo", str(root), "--output", "json")
|
||||
after_paths = {p for p in root.rglob("*") if p.is_file()}
|
||||
|
||||
assert after_paths == set(before)
|
||||
for path, mtime in before.items():
|
||||
assert path.stat().st_mtime_ns == mtime
|
||||
@@ -0,0 +1,120 @@
|
||||
"""specification.md 20: Validation rules MVP."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from pathlib import Path
|
||||
|
||||
from conftest import API_VERSION, git_init, minimal_contract, minimal_kdb, run_repoctl, write_yaml
|
||||
|
||||
|
||||
def _repo_with_contract(tmp_path: Path, contract_text: str, *, name: str = "sample-repo") -> Path:
|
||||
root = tmp_path / "repo"
|
||||
write_yaml(root / "ops" / "contract.yaml", contract_text)
|
||||
write_yaml(root / "kdb" / "kdb.yaml", minimal_kdb(name))
|
||||
git_init(root)
|
||||
return root
|
||||
|
||||
|
||||
def test_duplicate_content_root_ids_rejected(tmp_path: Path) -> None:
|
||||
contract = minimal_contract(
|
||||
extra_spec="""\
|
||||
contentRoots:
|
||||
- id: docs
|
||||
role: documentation
|
||||
path: docs
|
||||
kind: file-set
|
||||
authority: source
|
||||
writePolicy: managed
|
||||
- id: docs
|
||||
role: documentation
|
||||
path: docs2
|
||||
kind: file-set
|
||||
authority: source
|
||||
writePolicy: managed
|
||||
""",
|
||||
)
|
||||
root = _repo_with_contract(tmp_path, contract)
|
||||
(root / "docs").mkdir()
|
||||
(root / "docs2").mkdir()
|
||||
|
||||
result = run_repoctl("validate", "--repo", str(root), "--output", "json")
|
||||
envelope = result.envelope()
|
||||
assert result.returncode == 1
|
||||
assert any("DUPLICATE" in f["code"] for f in envelope["findings"])
|
||||
|
||||
|
||||
def test_invalid_id_format_rejected(tmp_path: Path) -> None:
|
||||
contract = minimal_contract(name="Not-A-Valid-ID!!")
|
||||
root = _repo_with_contract(tmp_path, contract, name="Not-A-Valid-ID!!")
|
||||
|
||||
result = run_repoctl("validate", "--repo", str(root), "--output", "json")
|
||||
assert result.returncode == 1
|
||||
envelope = result.envelope()
|
||||
assert any(f["severity"] == "error" for f in envelope["findings"])
|
||||
|
||||
|
||||
def test_unknown_repository_type_rejected(tmp_path: Path) -> None:
|
||||
contract = f"""\
|
||||
apiVersion: {API_VERSION}
|
||||
kind: RepositoryContract
|
||||
metadata:
|
||||
name: sample-repo
|
||||
spec:
|
||||
summary: repo with a bogus type
|
||||
repository:
|
||||
type: not-a-real-type
|
||||
lifecycle: active
|
||||
"""
|
||||
root = _repo_with_contract(tmp_path, contract)
|
||||
result = run_repoctl("validate", "--repo", str(root), "--output", "json")
|
||||
assert result.returncode == 1
|
||||
envelope = result.envelope()
|
||||
assert any(f["severity"] == "error" for f in envelope["findings"])
|
||||
|
||||
|
||||
def test_unknown_apiversion_rejected(tmp_path: Path) -> None:
|
||||
contract = """\
|
||||
apiVersion: stewardship.corsus.ru/v99
|
||||
kind: RepositoryContract
|
||||
metadata:
|
||||
name: sample-repo
|
||||
spec:
|
||||
summary: repo declaring an unsupported apiVersion
|
||||
repository:
|
||||
type: project
|
||||
lifecycle: active
|
||||
"""
|
||||
root = _repo_with_contract(tmp_path, contract)
|
||||
result = run_repoctl("validate", "--repo", str(root), "--output", "json")
|
||||
assert result.returncode == 1
|
||||
|
||||
|
||||
def test_kdb_topic_references_missing_document_rejected(tmp_path: Path) -> None:
|
||||
contract = minimal_contract()
|
||||
kdb = f"""\
|
||||
apiVersion: {API_VERSION}
|
||||
kind: KnowledgeBase
|
||||
metadata:
|
||||
name: sample-repo-kdb
|
||||
spec:
|
||||
repository: repo:sample-repo
|
||||
defaultLanguage: en
|
||||
topics:
|
||||
- id: some-topic
|
||||
title: Some topic
|
||||
summary: Refers to a document that was never declared
|
||||
intents:
|
||||
- some.intent
|
||||
documents:
|
||||
- missing-document
|
||||
documents: []
|
||||
"""
|
||||
root = tmp_path / "repo"
|
||||
write_yaml(root / "ops" / "contract.yaml", contract)
|
||||
write_yaml(root / "kdb" / "kdb.yaml", kdb)
|
||||
git_init(root)
|
||||
|
||||
result = run_repoctl("validate", "--repo", str(root), "--output", "json")
|
||||
assert result.returncode == 1
|
||||
envelope = result.envelope()
|
||||
assert any(f["severity"] == "error" for f in envelope["findings"])
|
||||
Reference in New Issue
Block a user