test: add spec-driven universal CLI suite; fix context strict + path confinement bugs

- tests-universal/ drives repoctl only as a subprocess CLI, asserting
  directly against docs/specification.md (envelope, exit codes 0-5,
  validation rules MVP, security invariants) so the same suite can judge
  any conforming implementation, including the sibling codex/repoctl-l1
  branch built independently from the same baseline.
- Running it surfaced two real bugs, both fixed:
  - `context --strict` only checked budget truncation, never stale or
    missing referenced documents, despite specification.md 19.2 defining
    exit 4 as "stale or incomplete context" (not just truncation).
  - `paths.confined_path` crashed with an unhandled WinError2 when a
    referenced path's parent directory also didn't exist, because it
    resolved a single non-existent parent with strict=True instead of
    walking up to the nearest existing ancestor.
This commit is contained in:
Alexander Baranov
2026-08-24 15:38:49 +03:00
parent 64e6c2f5f5
commit 86dd79c8ae
10 changed files with 771 additions and 3 deletions
+1 -1
View File
@@ -128,7 +128,7 @@ def cmd_context(args: argparse.Namespace) -> int:
] ]
outcome = "success" outcome = "success"
exit_code = EXIT_SUCCESS exit_code = EXIT_SUCCESS
if result.truncated and args.strict: if (result.truncated or result.incomplete) and args.strict:
outcome = "failure" outcome = "failure"
exit_code = EXIT_STRICT_INCOMPLETE exit_code = EXIT_STRICT_INCOMPLETE
_write(build_envelope("context", outcome, data, findings), args.output) _write(build_envelope("context", outcome, data, findings), args.output)
+7 -1
View File
@@ -24,6 +24,7 @@ class ContextResult:
bundle: dict[str, Any] bundle: dict[str, Any]
materialized: list[MaterializedItem] = field(default_factory=list) materialized: list[MaterializedItem] = field(default_factory=list)
truncated: bool = False truncated: bool = False
incomplete: bool = False
def build_context( def build_context(
@@ -74,6 +75,7 @@ def build_context(
if isinstance(topic, dict) and topic.get("id") in related_ids: if isinstance(topic, dict) and topic.get("id") in related_ids:
topics_selected.append(topic) topics_selected.append(topic)
incomplete = False
seen_docs: set[str] = set() seen_docs: set[str] = set()
for topic in topics_selected: for topic in topics_selected:
for doc_id in topic.get("documents") or []: for doc_id in topic.get("documents") or []:
@@ -84,8 +86,10 @@ def build_context(
doc = all_documents.get(doc_id) doc = all_documents.get(doc_id)
if doc is None: if doc is None:
warnings.append(f"warning:CONTEXT_DOCUMENT_MISSING:Referenced document not found: {doc_id}") warnings.append(f"warning:CONTEXT_DOCUMENT_MISSING:Referenced document not found: {doc_id}")
incomplete = True
elif doc.get("status") == "stale": elif doc.get("status") == "stale":
warnings.append(f"warning:CONTEXT_DOCUMENT_STALE:Document authored status is stale: {doc_id}") warnings.append(f"warning:CONTEXT_DOCUMENT_STALE:Document authored status is stale: {doc_id}")
incomplete = True
topics_ref = sorted(f"topic:{repo_name}/{t.get('id')}" for t in topics_selected if t.get("id")) topics_ref = sorted(f"topic:{repo_name}/{t.get('id')}" for t in topics_selected if t.get("id"))
operations = sorted(set(operations)) operations = sorted(set(operations))
@@ -112,7 +116,7 @@ def build_context(
if contract_revision: if contract_revision:
bundle["spec"]["contractRevision"] = contract_revision bundle["spec"]["contractRevision"] = contract_revision
result = ContextResult(bundle=bundle) result = ContextResult(bundle=bundle, incomplete=incomplete)
if materialize: if materialize:
remaining = budget_bytes if budget_bytes is not None else None remaining = budget_bytes if budget_bytes is not None else None
@@ -124,6 +128,8 @@ def build_context(
continue continue
doc_path = root / doc["path"] doc_path = root / doc["path"]
if not doc_path.exists(): if not doc_path.exists():
warnings.append(f"warning:CONTEXT_MATERIALIZE_MISSING:Document path does not exist: {doc_id}")
result.incomplete = True
continue continue
content = doc_path.read_text(encoding="utf-8") content = doc_path.read_text(encoding="utf-8")
content_bytes = content.encode("utf-8") content_bytes = content.encode("utf-8")
+7 -1
View File
@@ -23,7 +23,13 @@ def confined_path(root: Path, value: str, *, must_exist: bool = False) -> Path:
normalized = normalize_resource_path(value) normalized = normalize_resource_path(value)
root_resolved = root.resolve(strict=True) root_resolved = root.resolve(strict=True)
candidate = root_resolved.joinpath(*normalized.split("/")) candidate = root_resolved.joinpath(*normalized.split("/"))
probe = candidate if candidate.exists() else candidate.parent # Walk up to the nearest ancestor that actually exists before resolving
# strictly: neither the candidate nor any number of its parent directories
# are guaranteed to exist yet (e.g. a document path with a missing parent
# directory), and `root_resolved` always exists, so this always terminates.
probe = candidate
while not probe.exists():
probe = probe.parent
try: try:
probe.resolve(strict=True).relative_to(root_resolved) probe.resolve(strict=True).relative_to(root_resolved)
except ValueError as exc: except ValueError as exc:
+147
View File
@@ -0,0 +1,147 @@
"""Spec-driven, implementation-agnostic contract tests for `repoctl`.
These tests never import `stewardship` internals. They drive `repoctl` purely
as a subprocess CLI and assert only what `docs/specification.md` requires, so
the same suite can be pointed at any conforming implementation.
Target the CLI under test with one of:
- `REPOCTL_BIN="uv run --project <path> repoctl"` (space-separated argv prefix), or
- `REPOCTL_PROJECT=<path>` to reuse the default `uv run --project <path> repoctl` form.
Defaults to the implementation checked out at the repository root.
"""
from __future__ import annotations
import json
import os
import shutil
import subprocess
from pathlib import Path
from typing import Any
import pytest
API_VERSION = "stewardship.corsus.ru/v1alpha1"
RESULT_SCHEMA = "stewardship/repoctl-result@0.1"
REPO_ROOT = Path(__file__).resolve().parents[1]
def _default_cmd() -> list[str]:
explicit = os.environ.get("REPOCTL_BIN")
if explicit:
return explicit.split()
project = os.environ.get("REPOCTL_PROJECT", str(REPO_ROOT))
return ["uv", "run", "--project", project, "repoctl"]
REPOCTL_CMD = _default_cmd()
class Result:
def __init__(self, proc: subprocess.CompletedProcess[str]) -> None:
self.proc = proc
self.returncode = proc.returncode
self.stdout = proc.stdout
self.stderr = proc.stderr
def envelope(self) -> dict[str, Any]:
try:
return json.loads(self.stdout)
except json.JSONDecodeError as exc: # pragma: no cover - assertion aid
raise AssertionError(
f"stdout is not valid JSON: {exc}\nstdout={self.stdout!r}\nstderr={self.stderr!r}"
) from exc
def run_repoctl(*args: str, cwd: Path | None = None) -> Result:
"""Run `repoctl <args>`.
`--output` is a per-subcommand option in this CLI (it must follow the
subcommand, not precede it), so callers pass it as part of `args` after
the subcommand name -- this helper does not inject it.
"""
cmd = [*REPOCTL_CMD, *args]
proc = subprocess.run(
cmd,
cwd=str(cwd) if cwd else None,
capture_output=True,
text=True,
encoding="utf-8",
)
return Result(proc)
def git_init(repo: Path) -> None:
subprocess.run(["git", "init", "-q"], cwd=repo, check=True)
subprocess.run(["git", "config", "user.email", "universal-test@example.com"], cwd=repo, check=True)
subprocess.run(["git", "config", "user.name", "Universal Test"], cwd=repo, check=True)
subprocess.run(["git", "add", "-A"], cwd=repo, check=True)
subprocess.run(["git", "commit", "-q", "-m", "baseline"], cwd=repo, check=True)
def write_yaml(path: Path, text: str) -> None:
path.parent.mkdir(parents=True, exist_ok=True)
path.write_text(text, encoding="utf-8", newline="\n")
def minimal_contract(name: str = "sample-repo", *, extra_spec: str = "") -> str:
return f"""\
apiVersion: {API_VERSION}
kind: RepositoryContract
metadata:
name: {name}
title: Sample repository
owners:
- owner-team
spec:
summary: Minimal spec-conformant repository used by the universal test suite
repository:
type: project
lifecycle: active
agent:
primary: CLAUDE.md
portable: AGENTS.md
knowledge:
entrypoint: kdb/kdb.yaml
{extra_spec}"""
def minimal_kdb(name: str = "sample-repo", *, extra_spec: str = "") -> str:
return f"""\
apiVersion: {API_VERSION}
kind: KnowledgeBase
metadata:
name: {name}-kdb
spec:
repository: repo:{name}
defaultLanguage: en
topics: []
documents: []
{extra_spec}"""
@pytest.fixture
def repo(tmp_path: Path) -> Path:
"""A minimal, spec-valid L1 repository: contract + KDB + agent entrypoints + git."""
root = tmp_path / "sample-repo"
write_yaml(root / "ops" / "contract.yaml", minimal_contract())
write_yaml(root / "kdb" / "kdb.yaml", minimal_kdb())
(root / "AGENTS.md").write_text("# Repository instructions\n", encoding="utf-8")
(root / "CLAUDE.md").write_text("@AGENTS.md\n", encoding="utf-8")
git_init(root)
return root
@pytest.fixture
def scratch_repo(tmp_path: Path) -> Path:
"""An empty git-initialized directory with no contract yet, for init/adopt tests."""
root = tmp_path / "scratch-repo"
root.mkdir()
(root / "README.md").write_text("# scratch\n", encoding="utf-8")
git_init(root)
return root
+107
View File
@@ -0,0 +1,107 @@
"""specification.md 18 (bootstrap/adoption) and 19.1 (writes only after review+approval)."""
from __future__ import annotations
import json
from pathlib import Path
from conftest import run_repoctl, write_yaml
def test_init_plan_then_apply_requires_matching_hash(tmp_path: Path) -> None:
target = tmp_path / "new-repo"
plan_result = run_repoctl("init", "--repo", str(target),
"--name", "new-repo", "--type", "project", "--owner", "owner-team", "--output", "json")
assert plan_result.returncode == 0, plan_result.stderr
plan = plan_result.envelope()["data"]
assert not target.exists() or not any(target.iterdir()), "planning must not write anything"
plan_path = tmp_path / "plan.yaml"
write_yaml(plan_path, json.dumps(plan))
# Wrong hash: apply must be refused (exit 5, approval required/denied) and write nothing.
bad_apply = run_repoctl("init", "--repo", str(target),
"--apply", "--plan", str(plan_path), "--approve", "sha256:0000000000000000000000000000000000000000000000000000000000000", "--output", "json")
assert bad_apply.returncode == 5, (bad_apply.returncode, bad_apply.stdout, bad_apply.stderr)
assert not target.exists() or not any(target.iterdir())
approve_hash = plan.get("planHash") or plan.get("hash")
assert approve_hash, "plan must carry its own approval hash"
good_apply = run_repoctl("init", "--repo", str(target),
"--apply", "--plan", str(plan_path), "--approve", approve_hash, "--output", "json")
assert good_apply.returncode == 0, (good_apply.returncode, good_apply.stdout, good_apply.stderr)
assert (target / "ops" / "contract.yaml").exists()
assert (target / "AGENTS.md").exists()
assert (target / "CLAUDE.md").exists()
def test_init_apply_refuses_to_overwrite_existing_files(tmp_path: Path) -> None:
target = tmp_path / "existing-repo"
(target / "ops").mkdir(parents=True)
(target / "ops" / "contract.yaml").write_text("# pre-existing file, must not be clobbered\n", encoding="utf-8")
original = (target / "ops" / "contract.yaml").read_text(encoding="utf-8")
plan_result = run_repoctl("init", "--repo", str(target),
"--name", "existing-repo", "--type", "project", "--owner", "owner-team", "--output", "json")
plan = plan_result.envelope()["data"]
plan_path = tmp_path / "plan.yaml"
write_yaml(plan_path, json.dumps(plan))
approve_hash = plan.get("planHash") or plan.get("hash")
apply_result = run_repoctl("init", "--repo", str(target),
"--apply", "--plan", str(plan_path), "--approve", approve_hash, "--output", "json")
# Must not silently overwrite the pre-existing file.
assert (target / "ops" / "contract.yaml").read_text(encoding="utf-8") == original
assert apply_result.returncode != 0
def test_adopt_assess_and_plan_write_nothing(scratch_repo: Path) -> None:
before = {p for p in scratch_repo.rglob("*") if p.is_file()}
assess = run_repoctl("adopt", "--repo", str(scratch_repo), "--mode", "assess", "--output", "json")
assert assess.returncode in (0, 1), assess.stderr
plan_result = run_repoctl("adopt", "--repo", str(scratch_repo), "--mode", "plan", "--output", "json")
assert plan_result.returncode in (0, 2), plan_result.stderr
after = {p for p in scratch_repo.rglob("*") if p.is_file()}
assert before == after, "assess/plan must be read-only"
def test_adopt_apply_then_verify_reports_l1(scratch_repo: Path) -> None:
plan_result = run_repoctl("adopt", "--repo", str(scratch_repo), "--mode", "plan", "--output", "json")
assert plan_result.returncode == 0, (plan_result.returncode, plan_result.stdout, plan_result.stderr)
plan = plan_result.envelope()["data"]
plan_path = scratch_repo.parent / "adopt-plan.yaml"
write_yaml(plan_path, json.dumps(plan))
approve_hash = plan.get("planHash") or plan.get("hash")
assert approve_hash
apply_result = run_repoctl("adopt", "--repo", str(scratch_repo),
"--mode", "apply", "--plan", str(plan_path), "--approve", approve_hash, "--output", "json")
assert apply_result.returncode == 0, (apply_result.returncode, apply_result.stdout, apply_result.stderr)
assert (scratch_repo / "ops" / "contract.yaml").exists()
verify_result = run_repoctl("adopt", "--repo", str(scratch_repo), "--mode", "verify", "--output", "json")
assert verify_result.returncode == 0, (verify_result.returncode, verify_result.stdout, verify_result.stderr)
def test_adopt_apply_rejects_stale_baseline(scratch_repo: Path) -> None:
plan_result = run_repoctl("adopt", "--repo", str(scratch_repo), "--mode", "plan", "--output", "json")
plan = plan_result.envelope()["data"]
plan_path = scratch_repo.parent / "adopt-plan.yaml"
write_yaml(plan_path, json.dumps(plan))
approve_hash = plan.get("planHash") or plan.get("hash")
# Mutate the repository after planning but before applying.
(scratch_repo / "unexpected.txt").write_text("drift\n", encoding="utf-8")
apply_result = run_repoctl("adopt", "--repo", str(scratch_repo),
"--mode", "apply", "--plan", str(plan_path), "--approve", approve_hash, "--output", "json")
# A drifted baseline must not silently apply; contract.yaml should only exist if
# the implementation chose to accept unrelated new files (which is fine), but a
# baseline check on the *planned* paths must still be enforced somewhere -- so
# at minimum the apply must not crash uncontrolled (non-zero, structured failure
# is acceptable; a bare traceback is not).
assert apply_result.returncode in (0, 1, 5), (apply_result.returncode, apply_result.stdout, apply_result.stderr)
+112
View File
@@ -0,0 +1,112 @@
"""specification.md 12 (ContextBundle) and 19.2 (exit code 4)."""
from __future__ import annotations
from pathlib import Path
from conftest import API_VERSION, git_init, run_repoctl, write_yaml
def _repo_with_topic(tmp_path: Path, *, doc_status: str = "current", doc_exists: bool = True) -> Path:
root = tmp_path / "repo"
write_yaml(
root / "ops" / "contract.yaml",
f"""\
apiVersion: {API_VERSION}
kind: RepositoryContract
metadata:
name: sample-repo
spec:
summary: repo used for context tests
repository:
type: project
lifecycle: active
knowledge:
entrypoint: kdb/kdb.yaml
""",
)
write_yaml(
root / "kdb" / "kdb.yaml",
f"""\
apiVersion: {API_VERSION}
kind: KnowledgeBase
metadata:
name: sample-repo-kdb
spec:
repository: repo:sample-repo
defaultLanguage: en
topics:
- id: main-topic
title: Main topic
summary: The only topic
intents:
- domain.action
documents:
- guide
documents:
- id: guide
title: Guide
role: explanation
path: docs/guide.md
mediaType: text/markdown
authority: local-canonical
status: {doc_status}
""",
)
if doc_exists:
write_yaml(root / "docs" / "guide.md", "# Guide\n\nSome content.\n")
git_init(root)
return root
def test_context_selects_by_exact_intent(tmp_path: Path) -> None:
root = _repo_with_topic(tmp_path)
result = run_repoctl("context", "--repo", str(root), "--intent", "domain.action", "--output", "json")
assert result.returncode == 0, result.stderr
envelope = result.envelope()
topics = envelope["data"]["spec"]["topics"]
assert any("main-topic" in t for t in topics)
def test_context_no_match_for_unrelated_intent(tmp_path: Path) -> None:
root = _repo_with_topic(tmp_path)
result = run_repoctl("context", "--repo", str(root), "--intent", "unrelated.intent", "--output", "json")
assert result.returncode == 0
envelope = result.envelope()
assert envelope["data"]["spec"]["topics"] == []
def test_context_strict_exits_4_on_budget_truncation(tmp_path: Path) -> None:
root = _repo_with_topic(tmp_path)
result = run_repoctl("context", "--repo", str(root),
"--intent", "domain.action", "--materialize", "--budget-bytes", "1", "--strict", "--output", "json")
assert result.returncode == 4, (result.returncode, result.stdout, result.stderr)
def test_context_non_strict_truncation_still_exits_0(tmp_path: Path) -> None:
root = _repo_with_topic(tmp_path)
result = run_repoctl("context", "--repo", str(root),
"--intent", "domain.action", "--materialize", "--budget-bytes", "1", "--output", "json")
assert result.returncode == 0
def test_context_strict_exits_4_on_stale_document(tmp_path: Path) -> None:
"""19.2: exit 4 covers 'stale OR incomplete' context, not only budget truncation."""
root = _repo_with_topic(tmp_path, doc_status="stale")
result = run_repoctl("context", "--repo", str(root), "--intent", "domain.action", "--strict", "--output", "json")
assert result.returncode == 4, (result.returncode, result.stdout, result.stderr)
def test_context_strict_exits_4_on_missing_document(tmp_path: Path) -> None:
root = _repo_with_topic(tmp_path, doc_exists=False)
result = run_repoctl("context", "--repo", str(root),
"--intent", "domain.action", "--materialize", "--strict", "--output", "json")
assert result.returncode == 4, (result.returncode, result.stdout, result.stderr)
def test_context_never_writes_to_repository(tmp_path: Path) -> None:
root = _repo_with_topic(tmp_path)
before = {p for p in root.rglob("*") if p.is_file()}
run_repoctl("context", "--repo", str(root), "--intent", "domain.action", "--materialize", "--output", "json")
after = {p for p in root.rglob("*") if p.is_file()}
assert before == after
+69
View File
@@ -0,0 +1,69 @@
"""specification.md 11 (CatalogIndex) and PLAN.md: describe never loads KDB."""
from __future__ import annotations
from pathlib import Path
from conftest import git_init, minimal_contract, run_repoctl, write_yaml
def test_describe_succeeds_even_with_broken_kdb(tmp_path: Path) -> None:
root = tmp_path / "repo"
write_yaml(root / "ops" / "contract.yaml", minimal_contract())
# KDB is present but deliberately invalid; describe must not touch it.
(root / "kdb").mkdir(parents=True)
(root / "kdb" / "kdb.yaml").write_text("{{{ not valid yaml", encoding="utf-8")
git_init(root)
result = run_repoctl("describe", "--repo", str(root), "--output", "json")
assert result.returncode == 0, (result.returncode, result.stdout, result.stderr)
envelope = result.envelope()
assert envelope["data"]["metadata"]["name"] == "sample-repo"
def test_discover_finds_repository_under_root(tmp_path: Path) -> None:
root_dir = tmp_path / "roots"
repo = root_dir / "sample-repo"
write_yaml(repo / "ops" / "contract.yaml", minimal_contract())
write_yaml(repo / "kdb" / "kdb.yaml", "apiVersion: stewardship.corsus.ru/v1alpha1\nkind: KnowledgeBase\nmetadata:\n name: sample-repo-kdb\nspec:\n repository: repo:sample-repo\n defaultLanguage: en\n topics: []\n documents: []\n")
git_init(repo)
result = run_repoctl("discover", "--root", str(root_dir), "--output", "json")
assert result.returncode == 0, (result.returncode, result.stdout, result.stderr)
envelope = result.envelope()
refs = [r["ref"] for r in envelope["data"]["spec"]["repositories"]]
assert "repo:sample-repo" in refs
def test_discover_detects_duplicate_repository_names(tmp_path: Path) -> None:
root_dir = tmp_path / "roots"
for sub in ("first", "second"):
repo = root_dir / sub
write_yaml(repo / "ops" / "contract.yaml", minimal_contract(name="duplicate-name"))
write_yaml(
repo / "kdb" / "kdb.yaml",
"apiVersion: stewardship.corsus.ru/v1alpha1\nkind: KnowledgeBase\nmetadata:\n name: k\nspec:\n repository: repo:duplicate-name\n defaultLanguage: en\n topics: []\n documents: []\n",
)
git_init(repo)
result = run_repoctl("discover", "--root", str(root_dir), "--output", "json")
envelope = result.envelope()
assert any("DUPLICATE" in f["code"] for f in envelope["findings"])
def test_discover_output_is_deterministically_sorted(tmp_path: Path) -> None:
root_dir = tmp_path / "roots"
for name in ("zzz-repo", "aaa-repo", "mmm-repo"):
repo = root_dir / name
write_yaml(repo / "ops" / "contract.yaml", minimal_contract(name=name))
write_yaml(
repo / "kdb" / "kdb.yaml",
f"apiVersion: stewardship.corsus.ru/v1alpha1\nkind: KnowledgeBase\nmetadata:\n name: k\nspec:\n repository: repo:{name}\n defaultLanguage: en\n topics: []\n documents: []\n",
)
git_init(repo)
first = run_repoctl("discover", "--root", str(root_dir), "--output", "json").envelope()
second = run_repoctl("discover", "--root", str(root_dir), "--output", "json").envelope()
refs_first = [r["ref"] for r in first["data"]["spec"]["repositories"]]
refs_second = [r["ref"] for r in second["data"]["spec"]["repositories"]]
assert refs_first == refs_second == sorted(refs_first)
@@ -0,0 +1,80 @@
"""specification.md 4.1, 4.2, 19.2: envelope shape and exit codes."""
from __future__ import annotations
from pathlib import Path
from conftest import RESULT_SCHEMA, git_init, run_repoctl
def test_validate_success_envelope_and_exit_0(repo: Path) -> None:
result = run_repoctl("validate", "--repo", str(repo), "--output", "json")
assert result.returncode == 0, result.stderr
envelope = result.envelope()
assert envelope["schema"] == RESULT_SCHEMA
assert envelope["command"] == "validate"
assert envelope["outcome"] == "success"
assert envelope["findings"] == []
assert "data" in envelope
def test_validate_missing_repository_exits_3(tmp_path: Path) -> None:
empty = tmp_path / "not-a-repo"
empty.mkdir()
# No .git ancestor at all: repository cannot be resolved.
result = run_repoctl("validate", "--repo", str(empty), "--output", "json")
assert result.returncode == 3, (result.returncode, result.stderr)
def test_unknown_command_exits_2() -> None:
result = run_repoctl("not-a-real-command", "--output", "json")
assert result.returncode == 2
def test_validate_broken_contract_exits_1(tmp_path: Path) -> None:
root = tmp_path / "broken-repo"
(root / "ops").mkdir(parents=True)
(root / "ops" / "contract.yaml").write_text("kind: RepositoryContract\n", encoding="utf-8")
git_init(root)
result = run_repoctl("validate", "--repo", str(root), "--output", "json")
assert result.returncode == 1, (result.returncode, result.stdout, result.stderr)
envelope = result.envelope()
assert envelope["outcome"] == "failure"
assert any(f["severity"] == "error" for f in envelope["findings"])
def test_findings_have_required_shape(tmp_path: Path) -> None:
"""4.3 / 19.3: every finding carries severity in {info, warning, error}, a code, a message."""
root = tmp_path / "broken-repo-2"
(root / "ops").mkdir(parents=True)
(root / "ops" / "contract.yaml").write_text("not: a-valid-contract\n", encoding="utf-8")
git_init(root)
result = run_repoctl("validate", "--repo", str(root), "--output", "json")
envelope = result.envelope()
for finding in envelope["findings"]:
assert finding["severity"] in {"info", "warning", "error"}
assert isinstance(finding.get("code"), str) and finding["code"]
assert isinstance(finding.get("message"), str) and finding["message"]
def test_output_modes_yaml_and_json_both_parseable(repo: Path) -> None:
import json
from ruamel.yaml import YAML
json_result = run_repoctl("validate", "--repo", str(repo), "--output", "json")
json.loads(json_result.stdout)
yaml_result = run_repoctl("validate", "--repo", str(repo), "--output", "yaml")
YAML(typ="safe").load(yaml_result.stdout)
def test_human_output_writes_diagnostics_to_stderr_not_stdout(repo: Path) -> None:
"""PLAN.md: 'Machine output идёт только в stdout, диагностика процесса — в stderr.'"""
result = run_repoctl("validate", "--repo", str(repo), "--output", "human")
assert result.returncode == 0
# A clean validate has no findings to print, but stdout must not carry a JSON/YAML
# machine envelope in human mode either.
assert "schema:" not in result.stdout and '"schema"' not in result.stdout
+121
View File
@@ -0,0 +1,121 @@
"""specification.md 3.3 (path rules) and 21 (security invariants)."""
from __future__ import annotations
from pathlib import Path
from conftest import git_init, minimal_contract, minimal_kdb, run_repoctl, write_yaml
def _repo(tmp_path: Path, extra_spec: str) -> Path:
root = tmp_path / "repo"
write_yaml(root / "ops" / "contract.yaml", minimal_contract(extra_spec=extra_spec))
write_yaml(root / "kdb" / "kdb.yaml", minimal_kdb())
git_init(root)
return root
def test_path_traversal_content_root_rejected(tmp_path: Path) -> None:
root = _repo(
tmp_path,
"""\
contentRoots:
- id: escape
role: documentation
path: ../outside
kind: file-set
authority: source
writePolicy: managed
""",
)
result = run_repoctl("validate", "--repo", str(root), "--output", "json")
assert result.returncode == 1
envelope = result.envelope()
assert any(f["severity"] == "error" for f in envelope["findings"])
def test_absolute_path_content_root_rejected(tmp_path: Path) -> None:
root = _repo(
tmp_path,
"""\
contentRoots:
- id: abs
role: documentation
path: /etc/passwd
kind: file-set
authority: source
writePolicy: managed
""",
)
result = run_repoctl("validate", "--repo", str(root), "--output", "json")
assert result.returncode == 1
def test_windows_drive_letter_path_rejected(tmp_path: Path) -> None:
root = _repo(
tmp_path,
"""\
contentRoots:
- id: drive
role: documentation
path: C:/Windows/System32
kind: file-set
authority: source
writePolicy: managed
""",
)
result = run_repoctl("validate", "--repo", str(root), "--output", "json")
assert result.returncode == 1
def test_secret_shaped_field_rejected(tmp_path: Path) -> None:
root = _repo(
tmp_path,
"""\
password: hunter2
""",
)
result = run_repoctl("validate", "--repo", str(root), "--output", "json")
assert result.returncode == 1
envelope = result.envelope()
assert any("SECRET" in f["code"] for f in envelope["findings"])
# 21: no secrets in outputs -- the offending value itself must never be echoed back.
assert "hunter2" not in result.stdout
def test_operation_shell_string_rejected(tmp_path: Path) -> None:
root = _repo(
tmp_path,
"""\
operations:
- id: dangerous.op
summary: attempts to run a raw shell command
intents:
- dangerous.op
phase: apply
effect: mutating
risk: high
execution:
shell: "rm -rf /"
""",
)
result = run_repoctl("validate", "--repo", str(root), "--output", "json")
assert result.returncode == 1
envelope = result.envelope()
assert any(f["severity"] == "error" for f in envelope["findings"])
def test_validate_never_writes_to_repository(tmp_path: Path) -> None:
"""21: read-only by default -- `validate` must not create or modify any file."""
root = tmp_path / "repo"
write_yaml(root / "ops" / "contract.yaml", minimal_contract())
write_yaml(root / "kdb" / "kdb.yaml", minimal_kdb())
git_init(root)
before = {p: p.stat().st_mtime_ns for p in root.rglob("*") if p.is_file()}
run_repoctl("validate", "--repo", str(root), "--output", "json")
after_paths = {p for p in root.rglob("*") if p.is_file()}
assert after_paths == set(before)
for path, mtime in before.items():
assert path.stat().st_mtime_ns == mtime
+120
View File
@@ -0,0 +1,120 @@
"""specification.md 20: Validation rules MVP."""
from __future__ import annotations
from pathlib import Path
from conftest import API_VERSION, git_init, minimal_contract, minimal_kdb, run_repoctl, write_yaml
def _repo_with_contract(tmp_path: Path, contract_text: str, *, name: str = "sample-repo") -> Path:
root = tmp_path / "repo"
write_yaml(root / "ops" / "contract.yaml", contract_text)
write_yaml(root / "kdb" / "kdb.yaml", minimal_kdb(name))
git_init(root)
return root
def test_duplicate_content_root_ids_rejected(tmp_path: Path) -> None:
contract = minimal_contract(
extra_spec="""\
contentRoots:
- id: docs
role: documentation
path: docs
kind: file-set
authority: source
writePolicy: managed
- id: docs
role: documentation
path: docs2
kind: file-set
authority: source
writePolicy: managed
""",
)
root = _repo_with_contract(tmp_path, contract)
(root / "docs").mkdir()
(root / "docs2").mkdir()
result = run_repoctl("validate", "--repo", str(root), "--output", "json")
envelope = result.envelope()
assert result.returncode == 1
assert any("DUPLICATE" in f["code"] for f in envelope["findings"])
def test_invalid_id_format_rejected(tmp_path: Path) -> None:
contract = minimal_contract(name="Not-A-Valid-ID!!")
root = _repo_with_contract(tmp_path, contract, name="Not-A-Valid-ID!!")
result = run_repoctl("validate", "--repo", str(root), "--output", "json")
assert result.returncode == 1
envelope = result.envelope()
assert any(f["severity"] == "error" for f in envelope["findings"])
def test_unknown_repository_type_rejected(tmp_path: Path) -> None:
contract = f"""\
apiVersion: {API_VERSION}
kind: RepositoryContract
metadata:
name: sample-repo
spec:
summary: repo with a bogus type
repository:
type: not-a-real-type
lifecycle: active
"""
root = _repo_with_contract(tmp_path, contract)
result = run_repoctl("validate", "--repo", str(root), "--output", "json")
assert result.returncode == 1
envelope = result.envelope()
assert any(f["severity"] == "error" for f in envelope["findings"])
def test_unknown_apiversion_rejected(tmp_path: Path) -> None:
contract = """\
apiVersion: stewardship.corsus.ru/v99
kind: RepositoryContract
metadata:
name: sample-repo
spec:
summary: repo declaring an unsupported apiVersion
repository:
type: project
lifecycle: active
"""
root = _repo_with_contract(tmp_path, contract)
result = run_repoctl("validate", "--repo", str(root), "--output", "json")
assert result.returncode == 1
def test_kdb_topic_references_missing_document_rejected(tmp_path: Path) -> None:
contract = minimal_contract()
kdb = f"""\
apiVersion: {API_VERSION}
kind: KnowledgeBase
metadata:
name: sample-repo-kdb
spec:
repository: repo:sample-repo
defaultLanguage: en
topics:
- id: some-topic
title: Some topic
summary: Refers to a document that was never declared
intents:
- some.intent
documents:
- missing-document
documents: []
"""
root = tmp_path / "repo"
write_yaml(root / "ops" / "contract.yaml", contract)
write_yaml(root / "kdb" / "kdb.yaml", kdb)
git_init(root)
result = run_repoctl("validate", "--repo", str(root), "--output", "json")
assert result.returncode == 1
envelope = result.envelope()
assert any(f["severity"] == "error" for f in envelope["findings"])