From 86dd79c8ae819357e1173e1eab0cab70fe5cc051 Mon Sep 17 00:00:00 2001 From: Alexander Baranov Date: Mon, 24 Aug 2026 15:38:49 +0300 Subject: [PATCH] test: add spec-driven universal CLI suite; fix context strict + path confinement bugs - tests-universal/ drives repoctl only as a subprocess CLI, asserting directly against docs/specification.md (envelope, exit codes 0-5, validation rules MVP, security invariants) so the same suite can judge any conforming implementation, including the sibling codex/repoctl-l1 branch built independently from the same baseline. - Running it surfaced two real bugs, both fixed: - `context --strict` only checked budget truncation, never stale or missing referenced documents, despite specification.md 19.2 defining exit 4 as "stale or incomplete context" (not just truncation). - `paths.confined_path` crashed with an unhandled WinError2 when a referenced path's parent directory also didn't exist, because it resolved a single non-existent parent with strict=True instead of walking up to the nearest existing ancestor. --- src/stewardship/cli.py | 2 +- src/stewardship/context.py | 8 +- src/stewardship/paths.py | 8 +- tests-universal/conftest.py | 147 ++++++++++++++++++ tests-universal/test_bootstrap_adopt.py | 107 +++++++++++++ tests-universal/test_context.py | 112 +++++++++++++ tests-universal/test_describe_discover.py | 69 ++++++++ .../test_envelope_and_exit_codes.py | 80 ++++++++++ tests-universal/test_security_invariants.py | 121 ++++++++++++++ tests-universal/test_validation_rules.py | 120 ++++++++++++++ 10 files changed, 771 insertions(+), 3 deletions(-) create mode 100644 tests-universal/conftest.py create mode 100644 tests-universal/test_bootstrap_adopt.py create mode 100644 tests-universal/test_context.py create mode 100644 tests-universal/test_describe_discover.py create mode 100644 tests-universal/test_envelope_and_exit_codes.py create mode 100644 tests-universal/test_security_invariants.py create mode 100644 tests-universal/test_validation_rules.py diff --git a/src/stewardship/cli.py b/src/stewardship/cli.py index 50648b8..e76d602 100644 --- a/src/stewardship/cli.py +++ b/src/stewardship/cli.py @@ -128,7 +128,7 @@ def cmd_context(args: argparse.Namespace) -> int: ] outcome = "success" exit_code = EXIT_SUCCESS - if result.truncated and args.strict: + if (result.truncated or result.incomplete) and args.strict: outcome = "failure" exit_code = EXIT_STRICT_INCOMPLETE _write(build_envelope("context", outcome, data, findings), args.output) diff --git a/src/stewardship/context.py b/src/stewardship/context.py index c750c16..f07e664 100644 --- a/src/stewardship/context.py +++ b/src/stewardship/context.py @@ -24,6 +24,7 @@ class ContextResult: bundle: dict[str, Any] materialized: list[MaterializedItem] = field(default_factory=list) truncated: bool = False + incomplete: bool = False def build_context( @@ -74,6 +75,7 @@ def build_context( if isinstance(topic, dict) and topic.get("id") in related_ids: topics_selected.append(topic) + incomplete = False seen_docs: set[str] = set() for topic in topics_selected: for doc_id in topic.get("documents") or []: @@ -84,8 +86,10 @@ def build_context( doc = all_documents.get(doc_id) if doc is None: warnings.append(f"warning:CONTEXT_DOCUMENT_MISSING:Referenced document not found: {doc_id}") + incomplete = True elif doc.get("status") == "stale": warnings.append(f"warning:CONTEXT_DOCUMENT_STALE:Document authored status is stale: {doc_id}") + incomplete = True topics_ref = sorted(f"topic:{repo_name}/{t.get('id')}" for t in topics_selected if t.get("id")) operations = sorted(set(operations)) @@ -112,7 +116,7 @@ def build_context( if contract_revision: bundle["spec"]["contractRevision"] = contract_revision - result = ContextResult(bundle=bundle) + result = ContextResult(bundle=bundle, incomplete=incomplete) if materialize: remaining = budget_bytes if budget_bytes is not None else None @@ -124,6 +128,8 @@ def build_context( continue doc_path = root / doc["path"] if not doc_path.exists(): + warnings.append(f"warning:CONTEXT_MATERIALIZE_MISSING:Document path does not exist: {doc_id}") + result.incomplete = True continue content = doc_path.read_text(encoding="utf-8") content_bytes = content.encode("utf-8") diff --git a/src/stewardship/paths.py b/src/stewardship/paths.py index 0f3bbaf..f182122 100644 --- a/src/stewardship/paths.py +++ b/src/stewardship/paths.py @@ -23,7 +23,13 @@ def confined_path(root: Path, value: str, *, must_exist: bool = False) -> Path: normalized = normalize_resource_path(value) root_resolved = root.resolve(strict=True) candidate = root_resolved.joinpath(*normalized.split("/")) - probe = candidate if candidate.exists() else candidate.parent + # Walk up to the nearest ancestor that actually exists before resolving + # strictly: neither the candidate nor any number of its parent directories + # are guaranteed to exist yet (e.g. a document path with a missing parent + # directory), and `root_resolved` always exists, so this always terminates. + probe = candidate + while not probe.exists(): + probe = probe.parent try: probe.resolve(strict=True).relative_to(root_resolved) except ValueError as exc: diff --git a/tests-universal/conftest.py b/tests-universal/conftest.py new file mode 100644 index 0000000..4755c6d --- /dev/null +++ b/tests-universal/conftest.py @@ -0,0 +1,147 @@ +"""Spec-driven, implementation-agnostic contract tests for `repoctl`. + +These tests never import `stewardship` internals. They drive `repoctl` purely +as a subprocess CLI and assert only what `docs/specification.md` requires, so +the same suite can be pointed at any conforming implementation. + +Target the CLI under test with one of: + +- `REPOCTL_BIN="uv run --project repoctl"` (space-separated argv prefix), or +- `REPOCTL_PROJECT=` to reuse the default `uv run --project repoctl` form. + +Defaults to the implementation checked out at the repository root. +""" + +from __future__ import annotations + +import json +import os +import shutil +import subprocess +from pathlib import Path +from typing import Any + +import pytest + +API_VERSION = "stewardship.corsus.ru/v1alpha1" +RESULT_SCHEMA = "stewardship/repoctl-result@0.1" + +REPO_ROOT = Path(__file__).resolve().parents[1] + + +def _default_cmd() -> list[str]: + explicit = os.environ.get("REPOCTL_BIN") + if explicit: + return explicit.split() + project = os.environ.get("REPOCTL_PROJECT", str(REPO_ROOT)) + return ["uv", "run", "--project", project, "repoctl"] + + +REPOCTL_CMD = _default_cmd() + + +class Result: + def __init__(self, proc: subprocess.CompletedProcess[str]) -> None: + self.proc = proc + self.returncode = proc.returncode + self.stdout = proc.stdout + self.stderr = proc.stderr + + def envelope(self) -> dict[str, Any]: + try: + return json.loads(self.stdout) + except json.JSONDecodeError as exc: # pragma: no cover - assertion aid + raise AssertionError( + f"stdout is not valid JSON: {exc}\nstdout={self.stdout!r}\nstderr={self.stderr!r}" + ) from exc + + +def run_repoctl(*args: str, cwd: Path | None = None) -> Result: + """Run `repoctl `. + + `--output` is a per-subcommand option in this CLI (it must follow the + subcommand, not precede it), so callers pass it as part of `args` after + the subcommand name -- this helper does not inject it. + """ + cmd = [*REPOCTL_CMD, *args] + proc = subprocess.run( + cmd, + cwd=str(cwd) if cwd else None, + capture_output=True, + text=True, + encoding="utf-8", + ) + return Result(proc) + + +def git_init(repo: Path) -> None: + subprocess.run(["git", "init", "-q"], cwd=repo, check=True) + subprocess.run(["git", "config", "user.email", "universal-test@example.com"], cwd=repo, check=True) + subprocess.run(["git", "config", "user.name", "Universal Test"], cwd=repo, check=True) + subprocess.run(["git", "add", "-A"], cwd=repo, check=True) + subprocess.run(["git", "commit", "-q", "-m", "baseline"], cwd=repo, check=True) + + +def write_yaml(path: Path, text: str) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(text, encoding="utf-8", newline="\n") + + +def minimal_contract(name: str = "sample-repo", *, extra_spec: str = "") -> str: + return f"""\ +apiVersion: {API_VERSION} +kind: RepositoryContract +metadata: + name: {name} + title: Sample repository + owners: + - owner-team +spec: + summary: Minimal spec-conformant repository used by the universal test suite + repository: + type: project + lifecycle: active + agent: + primary: CLAUDE.md + portable: AGENTS.md + knowledge: + entrypoint: kdb/kdb.yaml +{extra_spec}""" + + +def minimal_kdb(name: str = "sample-repo", *, extra_spec: str = "") -> str: + return f"""\ +apiVersion: {API_VERSION} +kind: KnowledgeBase +metadata: + name: {name}-kdb +spec: + repository: repo:{name} + defaultLanguage: en + topics: [] + documents: [] +{extra_spec}""" + + +@pytest.fixture +def repo(tmp_path: Path) -> Path: + """A minimal, spec-valid L1 repository: contract + KDB + agent entrypoints + git.""" + + root = tmp_path / "sample-repo" + write_yaml(root / "ops" / "contract.yaml", minimal_contract()) + write_yaml(root / "kdb" / "kdb.yaml", minimal_kdb()) + (root / "AGENTS.md").write_text("# Repository instructions\n", encoding="utf-8") + (root / "CLAUDE.md").write_text("@AGENTS.md\n", encoding="utf-8") + git_init(root) + return root + + +@pytest.fixture +def scratch_repo(tmp_path: Path) -> Path: + """An empty git-initialized directory with no contract yet, for init/adopt tests.""" + + root = tmp_path / "scratch-repo" + root.mkdir() + (root / "README.md").write_text("# scratch\n", encoding="utf-8") + git_init(root) + return root diff --git a/tests-universal/test_bootstrap_adopt.py b/tests-universal/test_bootstrap_adopt.py new file mode 100644 index 0000000..8f83d73 --- /dev/null +++ b/tests-universal/test_bootstrap_adopt.py @@ -0,0 +1,107 @@ +"""specification.md 18 (bootstrap/adoption) and 19.1 (writes only after review+approval).""" + +from __future__ import annotations + +import json +from pathlib import Path + +from conftest import run_repoctl, write_yaml + + +def test_init_plan_then_apply_requires_matching_hash(tmp_path: Path) -> None: + target = tmp_path / "new-repo" + plan_result = run_repoctl("init", "--repo", str(target), + "--name", "new-repo", "--type", "project", "--owner", "owner-team", "--output", "json") + assert plan_result.returncode == 0, plan_result.stderr + plan = plan_result.envelope()["data"] + assert not target.exists() or not any(target.iterdir()), "planning must not write anything" + + plan_path = tmp_path / "plan.yaml" + write_yaml(plan_path, json.dumps(plan)) + + # Wrong hash: apply must be refused (exit 5, approval required/denied) and write nothing. + bad_apply = run_repoctl("init", "--repo", str(target), + "--apply", "--plan", str(plan_path), "--approve", "sha256:0000000000000000000000000000000000000000000000000000000000000", "--output", "json") + assert bad_apply.returncode == 5, (bad_apply.returncode, bad_apply.stdout, bad_apply.stderr) + assert not target.exists() or not any(target.iterdir()) + + approve_hash = plan.get("planHash") or plan.get("hash") + assert approve_hash, "plan must carry its own approval hash" + + good_apply = run_repoctl("init", "--repo", str(target), + "--apply", "--plan", str(plan_path), "--approve", approve_hash, "--output", "json") + assert good_apply.returncode == 0, (good_apply.returncode, good_apply.stdout, good_apply.stderr) + assert (target / "ops" / "contract.yaml").exists() + assert (target / "AGENTS.md").exists() + assert (target / "CLAUDE.md").exists() + + +def test_init_apply_refuses_to_overwrite_existing_files(tmp_path: Path) -> None: + target = tmp_path / "existing-repo" + (target / "ops").mkdir(parents=True) + (target / "ops" / "contract.yaml").write_text("# pre-existing file, must not be clobbered\n", encoding="utf-8") + original = (target / "ops" / "contract.yaml").read_text(encoding="utf-8") + + plan_result = run_repoctl("init", "--repo", str(target), + "--name", "existing-repo", "--type", "project", "--owner", "owner-team", "--output", "json") + plan = plan_result.envelope()["data"] + plan_path = tmp_path / "plan.yaml" + write_yaml(plan_path, json.dumps(plan)) + approve_hash = plan.get("planHash") or plan.get("hash") + + apply_result = run_repoctl("init", "--repo", str(target), + "--apply", "--plan", str(plan_path), "--approve", approve_hash, "--output", "json") + # Must not silently overwrite the pre-existing file. + assert (target / "ops" / "contract.yaml").read_text(encoding="utf-8") == original + assert apply_result.returncode != 0 + + +def test_adopt_assess_and_plan_write_nothing(scratch_repo: Path) -> None: + before = {p for p in scratch_repo.rglob("*") if p.is_file()} + + assess = run_repoctl("adopt", "--repo", str(scratch_repo), "--mode", "assess", "--output", "json") + assert assess.returncode in (0, 1), assess.stderr + + plan_result = run_repoctl("adopt", "--repo", str(scratch_repo), "--mode", "plan", "--output", "json") + assert plan_result.returncode in (0, 2), plan_result.stderr + + after = {p for p in scratch_repo.rglob("*") if p.is_file()} + assert before == after, "assess/plan must be read-only" + + +def test_adopt_apply_then_verify_reports_l1(scratch_repo: Path) -> None: + plan_result = run_repoctl("adopt", "--repo", str(scratch_repo), "--mode", "plan", "--output", "json") + assert plan_result.returncode == 0, (plan_result.returncode, plan_result.stdout, plan_result.stderr) + plan = plan_result.envelope()["data"] + plan_path = scratch_repo.parent / "adopt-plan.yaml" + write_yaml(plan_path, json.dumps(plan)) + approve_hash = plan.get("planHash") or plan.get("hash") + assert approve_hash + + apply_result = run_repoctl("adopt", "--repo", str(scratch_repo), + "--mode", "apply", "--plan", str(plan_path), "--approve", approve_hash, "--output", "json") + assert apply_result.returncode == 0, (apply_result.returncode, apply_result.stdout, apply_result.stderr) + assert (scratch_repo / "ops" / "contract.yaml").exists() + + verify_result = run_repoctl("adopt", "--repo", str(scratch_repo), "--mode", "verify", "--output", "json") + assert verify_result.returncode == 0, (verify_result.returncode, verify_result.stdout, verify_result.stderr) + + +def test_adopt_apply_rejects_stale_baseline(scratch_repo: Path) -> None: + plan_result = run_repoctl("adopt", "--repo", str(scratch_repo), "--mode", "plan", "--output", "json") + plan = plan_result.envelope()["data"] + plan_path = scratch_repo.parent / "adopt-plan.yaml" + write_yaml(plan_path, json.dumps(plan)) + approve_hash = plan.get("planHash") or plan.get("hash") + + # Mutate the repository after planning but before applying. + (scratch_repo / "unexpected.txt").write_text("drift\n", encoding="utf-8") + + apply_result = run_repoctl("adopt", "--repo", str(scratch_repo), + "--mode", "apply", "--plan", str(plan_path), "--approve", approve_hash, "--output", "json") + # A drifted baseline must not silently apply; contract.yaml should only exist if + # the implementation chose to accept unrelated new files (which is fine), but a + # baseline check on the *planned* paths must still be enforced somewhere -- so + # at minimum the apply must not crash uncontrolled (non-zero, structured failure + # is acceptable; a bare traceback is not). + assert apply_result.returncode in (0, 1, 5), (apply_result.returncode, apply_result.stdout, apply_result.stderr) diff --git a/tests-universal/test_context.py b/tests-universal/test_context.py new file mode 100644 index 0000000..0b90434 --- /dev/null +++ b/tests-universal/test_context.py @@ -0,0 +1,112 @@ +"""specification.md 12 (ContextBundle) and 19.2 (exit code 4).""" + +from __future__ import annotations + +from pathlib import Path + +from conftest import API_VERSION, git_init, run_repoctl, write_yaml + + +def _repo_with_topic(tmp_path: Path, *, doc_status: str = "current", doc_exists: bool = True) -> Path: + root = tmp_path / "repo" + write_yaml( + root / "ops" / "contract.yaml", + f"""\ +apiVersion: {API_VERSION} +kind: RepositoryContract +metadata: + name: sample-repo +spec: + summary: repo used for context tests + repository: + type: project + lifecycle: active + knowledge: + entrypoint: kdb/kdb.yaml +""", + ) + write_yaml( + root / "kdb" / "kdb.yaml", + f"""\ +apiVersion: {API_VERSION} +kind: KnowledgeBase +metadata: + name: sample-repo-kdb +spec: + repository: repo:sample-repo + defaultLanguage: en + topics: + - id: main-topic + title: Main topic + summary: The only topic + intents: + - domain.action + documents: + - guide + documents: + - id: guide + title: Guide + role: explanation + path: docs/guide.md + mediaType: text/markdown + authority: local-canonical + status: {doc_status} +""", + ) + if doc_exists: + write_yaml(root / "docs" / "guide.md", "# Guide\n\nSome content.\n") + git_init(root) + return root + + +def test_context_selects_by_exact_intent(tmp_path: Path) -> None: + root = _repo_with_topic(tmp_path) + result = run_repoctl("context", "--repo", str(root), "--intent", "domain.action", "--output", "json") + assert result.returncode == 0, result.stderr + envelope = result.envelope() + topics = envelope["data"]["spec"]["topics"] + assert any("main-topic" in t for t in topics) + + +def test_context_no_match_for_unrelated_intent(tmp_path: Path) -> None: + root = _repo_with_topic(tmp_path) + result = run_repoctl("context", "--repo", str(root), "--intent", "unrelated.intent", "--output", "json") + assert result.returncode == 0 + envelope = result.envelope() + assert envelope["data"]["spec"]["topics"] == [] + + +def test_context_strict_exits_4_on_budget_truncation(tmp_path: Path) -> None: + root = _repo_with_topic(tmp_path) + result = run_repoctl("context", "--repo", str(root), + "--intent", "domain.action", "--materialize", "--budget-bytes", "1", "--strict", "--output", "json") + assert result.returncode == 4, (result.returncode, result.stdout, result.stderr) + + +def test_context_non_strict_truncation_still_exits_0(tmp_path: Path) -> None: + root = _repo_with_topic(tmp_path) + result = run_repoctl("context", "--repo", str(root), + "--intent", "domain.action", "--materialize", "--budget-bytes", "1", "--output", "json") + assert result.returncode == 0 + + +def test_context_strict_exits_4_on_stale_document(tmp_path: Path) -> None: + """19.2: exit 4 covers 'stale OR incomplete' context, not only budget truncation.""" + root = _repo_with_topic(tmp_path, doc_status="stale") + result = run_repoctl("context", "--repo", str(root), "--intent", "domain.action", "--strict", "--output", "json") + assert result.returncode == 4, (result.returncode, result.stdout, result.stderr) + + +def test_context_strict_exits_4_on_missing_document(tmp_path: Path) -> None: + root = _repo_with_topic(tmp_path, doc_exists=False) + result = run_repoctl("context", "--repo", str(root), + "--intent", "domain.action", "--materialize", "--strict", "--output", "json") + assert result.returncode == 4, (result.returncode, result.stdout, result.stderr) + + +def test_context_never_writes_to_repository(tmp_path: Path) -> None: + root = _repo_with_topic(tmp_path) + before = {p for p in root.rglob("*") if p.is_file()} + run_repoctl("context", "--repo", str(root), "--intent", "domain.action", "--materialize", "--output", "json") + after = {p for p in root.rglob("*") if p.is_file()} + assert before == after diff --git a/tests-universal/test_describe_discover.py b/tests-universal/test_describe_discover.py new file mode 100644 index 0000000..f818ec6 --- /dev/null +++ b/tests-universal/test_describe_discover.py @@ -0,0 +1,69 @@ +"""specification.md 11 (CatalogIndex) and PLAN.md: describe never loads KDB.""" + +from __future__ import annotations + +from pathlib import Path + +from conftest import git_init, minimal_contract, run_repoctl, write_yaml + + +def test_describe_succeeds_even_with_broken_kdb(tmp_path: Path) -> None: + root = tmp_path / "repo" + write_yaml(root / "ops" / "contract.yaml", minimal_contract()) + # KDB is present but deliberately invalid; describe must not touch it. + (root / "kdb").mkdir(parents=True) + (root / "kdb" / "kdb.yaml").write_text("{{{ not valid yaml", encoding="utf-8") + git_init(root) + + result = run_repoctl("describe", "--repo", str(root), "--output", "json") + assert result.returncode == 0, (result.returncode, result.stdout, result.stderr) + envelope = result.envelope() + assert envelope["data"]["metadata"]["name"] == "sample-repo" + + +def test_discover_finds_repository_under_root(tmp_path: Path) -> None: + root_dir = tmp_path / "roots" + repo = root_dir / "sample-repo" + write_yaml(repo / "ops" / "contract.yaml", minimal_contract()) + write_yaml(repo / "kdb" / "kdb.yaml", "apiVersion: stewardship.corsus.ru/v1alpha1\nkind: KnowledgeBase\nmetadata:\n name: sample-repo-kdb\nspec:\n repository: repo:sample-repo\n defaultLanguage: en\n topics: []\n documents: []\n") + git_init(repo) + + result = run_repoctl("discover", "--root", str(root_dir), "--output", "json") + assert result.returncode == 0, (result.returncode, result.stdout, result.stderr) + envelope = result.envelope() + refs = [r["ref"] for r in envelope["data"]["spec"]["repositories"]] + assert "repo:sample-repo" in refs + + +def test_discover_detects_duplicate_repository_names(tmp_path: Path) -> None: + root_dir = tmp_path / "roots" + for sub in ("first", "second"): + repo = root_dir / sub + write_yaml(repo / "ops" / "contract.yaml", minimal_contract(name="duplicate-name")) + write_yaml( + repo / "kdb" / "kdb.yaml", + "apiVersion: stewardship.corsus.ru/v1alpha1\nkind: KnowledgeBase\nmetadata:\n name: k\nspec:\n repository: repo:duplicate-name\n defaultLanguage: en\n topics: []\n documents: []\n", + ) + git_init(repo) + + result = run_repoctl("discover", "--root", str(root_dir), "--output", "json") + envelope = result.envelope() + assert any("DUPLICATE" in f["code"] for f in envelope["findings"]) + + +def test_discover_output_is_deterministically_sorted(tmp_path: Path) -> None: + root_dir = tmp_path / "roots" + for name in ("zzz-repo", "aaa-repo", "mmm-repo"): + repo = root_dir / name + write_yaml(repo / "ops" / "contract.yaml", minimal_contract(name=name)) + write_yaml( + repo / "kdb" / "kdb.yaml", + f"apiVersion: stewardship.corsus.ru/v1alpha1\nkind: KnowledgeBase\nmetadata:\n name: k\nspec:\n repository: repo:{name}\n defaultLanguage: en\n topics: []\n documents: []\n", + ) + git_init(repo) + + first = run_repoctl("discover", "--root", str(root_dir), "--output", "json").envelope() + second = run_repoctl("discover", "--root", str(root_dir), "--output", "json").envelope() + refs_first = [r["ref"] for r in first["data"]["spec"]["repositories"]] + refs_second = [r["ref"] for r in second["data"]["spec"]["repositories"]] + assert refs_first == refs_second == sorted(refs_first) diff --git a/tests-universal/test_envelope_and_exit_codes.py b/tests-universal/test_envelope_and_exit_codes.py new file mode 100644 index 0000000..903813a --- /dev/null +++ b/tests-universal/test_envelope_and_exit_codes.py @@ -0,0 +1,80 @@ +"""specification.md 4.1, 4.2, 19.2: envelope shape and exit codes.""" + +from __future__ import annotations + +from pathlib import Path + +from conftest import RESULT_SCHEMA, git_init, run_repoctl + + +def test_validate_success_envelope_and_exit_0(repo: Path) -> None: + result = run_repoctl("validate", "--repo", str(repo), "--output", "json") + assert result.returncode == 0, result.stderr + envelope = result.envelope() + assert envelope["schema"] == RESULT_SCHEMA + assert envelope["command"] == "validate" + assert envelope["outcome"] == "success" + assert envelope["findings"] == [] + assert "data" in envelope + + +def test_validate_missing_repository_exits_3(tmp_path: Path) -> None: + empty = tmp_path / "not-a-repo" + empty.mkdir() + # No .git ancestor at all: repository cannot be resolved. + result = run_repoctl("validate", "--repo", str(empty), "--output", "json") + assert result.returncode == 3, (result.returncode, result.stderr) + + +def test_unknown_command_exits_2() -> None: + result = run_repoctl("not-a-real-command", "--output", "json") + assert result.returncode == 2 + + +def test_validate_broken_contract_exits_1(tmp_path: Path) -> None: + root = tmp_path / "broken-repo" + (root / "ops").mkdir(parents=True) + (root / "ops" / "contract.yaml").write_text("kind: RepositoryContract\n", encoding="utf-8") + git_init(root) + + result = run_repoctl("validate", "--repo", str(root), "--output", "json") + assert result.returncode == 1, (result.returncode, result.stdout, result.stderr) + envelope = result.envelope() + assert envelope["outcome"] == "failure" + assert any(f["severity"] == "error" for f in envelope["findings"]) + + +def test_findings_have_required_shape(tmp_path: Path) -> None: + """4.3 / 19.3: every finding carries severity in {info, warning, error}, a code, a message.""" + root = tmp_path / "broken-repo-2" + (root / "ops").mkdir(parents=True) + (root / "ops" / "contract.yaml").write_text("not: a-valid-contract\n", encoding="utf-8") + git_init(root) + + result = run_repoctl("validate", "--repo", str(root), "--output", "json") + envelope = result.envelope() + for finding in envelope["findings"]: + assert finding["severity"] in {"info", "warning", "error"} + assert isinstance(finding.get("code"), str) and finding["code"] + assert isinstance(finding.get("message"), str) and finding["message"] + + +def test_output_modes_yaml_and_json_both_parseable(repo: Path) -> None: + import json + + from ruamel.yaml import YAML + + json_result = run_repoctl("validate", "--repo", str(repo), "--output", "json") + json.loads(json_result.stdout) + + yaml_result = run_repoctl("validate", "--repo", str(repo), "--output", "yaml") + YAML(typ="safe").load(yaml_result.stdout) + + +def test_human_output_writes_diagnostics_to_stderr_not_stdout(repo: Path) -> None: + """PLAN.md: 'Machine output идёт только в stdout, диагностика процесса — в stderr.'""" + result = run_repoctl("validate", "--repo", str(repo), "--output", "human") + assert result.returncode == 0 + # A clean validate has no findings to print, but stdout must not carry a JSON/YAML + # machine envelope in human mode either. + assert "schema:" not in result.stdout and '"schema"' not in result.stdout diff --git a/tests-universal/test_security_invariants.py b/tests-universal/test_security_invariants.py new file mode 100644 index 0000000..41d5dc7 --- /dev/null +++ b/tests-universal/test_security_invariants.py @@ -0,0 +1,121 @@ +"""specification.md 3.3 (path rules) and 21 (security invariants).""" + +from __future__ import annotations + +from pathlib import Path + +from conftest import git_init, minimal_contract, minimal_kdb, run_repoctl, write_yaml + + +def _repo(tmp_path: Path, extra_spec: str) -> Path: + root = tmp_path / "repo" + write_yaml(root / "ops" / "contract.yaml", minimal_contract(extra_spec=extra_spec)) + write_yaml(root / "kdb" / "kdb.yaml", minimal_kdb()) + git_init(root) + return root + + +def test_path_traversal_content_root_rejected(tmp_path: Path) -> None: + root = _repo( + tmp_path, + """\ + contentRoots: + - id: escape + role: documentation + path: ../outside + kind: file-set + authority: source + writePolicy: managed +""", + ) + result = run_repoctl("validate", "--repo", str(root), "--output", "json") + assert result.returncode == 1 + envelope = result.envelope() + assert any(f["severity"] == "error" for f in envelope["findings"]) + + +def test_absolute_path_content_root_rejected(tmp_path: Path) -> None: + root = _repo( + tmp_path, + """\ + contentRoots: + - id: abs + role: documentation + path: /etc/passwd + kind: file-set + authority: source + writePolicy: managed +""", + ) + result = run_repoctl("validate", "--repo", str(root), "--output", "json") + assert result.returncode == 1 + + +def test_windows_drive_letter_path_rejected(tmp_path: Path) -> None: + root = _repo( + tmp_path, + """\ + contentRoots: + - id: drive + role: documentation + path: C:/Windows/System32 + kind: file-set + authority: source + writePolicy: managed +""", + ) + result = run_repoctl("validate", "--repo", str(root), "--output", "json") + assert result.returncode == 1 + + +def test_secret_shaped_field_rejected(tmp_path: Path) -> None: + root = _repo( + tmp_path, + """\ + password: hunter2 +""", + ) + result = run_repoctl("validate", "--repo", str(root), "--output", "json") + assert result.returncode == 1 + envelope = result.envelope() + assert any("SECRET" in f["code"] for f in envelope["findings"]) + # 21: no secrets in outputs -- the offending value itself must never be echoed back. + assert "hunter2" not in result.stdout + + +def test_operation_shell_string_rejected(tmp_path: Path) -> None: + root = _repo( + tmp_path, + """\ + operations: + - id: dangerous.op + summary: attempts to run a raw shell command + intents: + - dangerous.op + phase: apply + effect: mutating + risk: high + execution: + shell: "rm -rf /" +""", + ) + result = run_repoctl("validate", "--repo", str(root), "--output", "json") + assert result.returncode == 1 + envelope = result.envelope() + assert any(f["severity"] == "error" for f in envelope["findings"]) + + +def test_validate_never_writes_to_repository(tmp_path: Path) -> None: + """21: read-only by default -- `validate` must not create or modify any file.""" + root = tmp_path / "repo" + write_yaml(root / "ops" / "contract.yaml", minimal_contract()) + write_yaml(root / "kdb" / "kdb.yaml", minimal_kdb()) + git_init(root) + + before = {p: p.stat().st_mtime_ns for p in root.rglob("*") if p.is_file()} + run_repoctl("validate", "--repo", str(root), "--output", "json") + after_paths = {p for p in root.rglob("*") if p.is_file()} + + assert after_paths == set(before) + for path, mtime in before.items(): + assert path.stat().st_mtime_ns == mtime diff --git a/tests-universal/test_validation_rules.py b/tests-universal/test_validation_rules.py new file mode 100644 index 0000000..c123ec4 --- /dev/null +++ b/tests-universal/test_validation_rules.py @@ -0,0 +1,120 @@ +"""specification.md 20: Validation rules MVP.""" + +from __future__ import annotations + +from pathlib import Path + +from conftest import API_VERSION, git_init, minimal_contract, minimal_kdb, run_repoctl, write_yaml + + +def _repo_with_contract(tmp_path: Path, contract_text: str, *, name: str = "sample-repo") -> Path: + root = tmp_path / "repo" + write_yaml(root / "ops" / "contract.yaml", contract_text) + write_yaml(root / "kdb" / "kdb.yaml", minimal_kdb(name)) + git_init(root) + return root + + +def test_duplicate_content_root_ids_rejected(tmp_path: Path) -> None: + contract = minimal_contract( + extra_spec="""\ + contentRoots: + - id: docs + role: documentation + path: docs + kind: file-set + authority: source + writePolicy: managed + - id: docs + role: documentation + path: docs2 + kind: file-set + authority: source + writePolicy: managed +""", + ) + root = _repo_with_contract(tmp_path, contract) + (root / "docs").mkdir() + (root / "docs2").mkdir() + + result = run_repoctl("validate", "--repo", str(root), "--output", "json") + envelope = result.envelope() + assert result.returncode == 1 + assert any("DUPLICATE" in f["code"] for f in envelope["findings"]) + + +def test_invalid_id_format_rejected(tmp_path: Path) -> None: + contract = minimal_contract(name="Not-A-Valid-ID!!") + root = _repo_with_contract(tmp_path, contract, name="Not-A-Valid-ID!!") + + result = run_repoctl("validate", "--repo", str(root), "--output", "json") + assert result.returncode == 1 + envelope = result.envelope() + assert any(f["severity"] == "error" for f in envelope["findings"]) + + +def test_unknown_repository_type_rejected(tmp_path: Path) -> None: + contract = f"""\ +apiVersion: {API_VERSION} +kind: RepositoryContract +metadata: + name: sample-repo +spec: + summary: repo with a bogus type + repository: + type: not-a-real-type + lifecycle: active +""" + root = _repo_with_contract(tmp_path, contract) + result = run_repoctl("validate", "--repo", str(root), "--output", "json") + assert result.returncode == 1 + envelope = result.envelope() + assert any(f["severity"] == "error" for f in envelope["findings"]) + + +def test_unknown_apiversion_rejected(tmp_path: Path) -> None: + contract = """\ +apiVersion: stewardship.corsus.ru/v99 +kind: RepositoryContract +metadata: + name: sample-repo +spec: + summary: repo declaring an unsupported apiVersion + repository: + type: project + lifecycle: active +""" + root = _repo_with_contract(tmp_path, contract) + result = run_repoctl("validate", "--repo", str(root), "--output", "json") + assert result.returncode == 1 + + +def test_kdb_topic_references_missing_document_rejected(tmp_path: Path) -> None: + contract = minimal_contract() + kdb = f"""\ +apiVersion: {API_VERSION} +kind: KnowledgeBase +metadata: + name: sample-repo-kdb +spec: + repository: repo:sample-repo + defaultLanguage: en + topics: + - id: some-topic + title: Some topic + summary: Refers to a document that was never declared + intents: + - some.intent + documents: + - missing-document + documents: [] +""" + root = tmp_path / "repo" + write_yaml(root / "ops" / "contract.yaml", contract) + write_yaml(root / "kdb" / "kdb.yaml", kdb) + git_init(root) + + result = run_repoctl("validate", "--repo", str(root), "--output", "json") + assert result.returncode == 1 + envelope = result.envelope() + assert any(f["severity"] == "error" for f in envelope["findings"])