test: add spec-driven universal CLI suite; fix context strict + path confinement bugs

- tests-universal/ drives repoctl only as a subprocess CLI, asserting
  directly against docs/specification.md (envelope, exit codes 0-5,
  validation rules MVP, security invariants) so the same suite can judge
  any conforming implementation, including the sibling codex/repoctl-l1
  branch built independently from the same baseline.
- Running it surfaced two real bugs, both fixed:
  - `context --strict` only checked budget truncation, never stale or
    missing referenced documents, despite specification.md 19.2 defining
    exit 4 as "stale or incomplete context" (not just truncation).
  - `paths.confined_path` crashed with an unhandled WinError2 when a
    referenced path's parent directory also didn't exist, because it
    resolved a single non-existent parent with strict=True instead of
    walking up to the nearest existing ancestor.
This commit is contained in:
Alexander Baranov
2026-08-24 15:38:49 +03:00
parent 64e6c2f5f5
commit 86dd79c8ae
10 changed files with 771 additions and 3 deletions
+112
View File
@@ -0,0 +1,112 @@
"""specification.md 12 (ContextBundle) and 19.2 (exit code 4)."""
from __future__ import annotations
from pathlib import Path
from conftest import API_VERSION, git_init, run_repoctl, write_yaml
def _repo_with_topic(tmp_path: Path, *, doc_status: str = "current", doc_exists: bool = True) -> Path:
root = tmp_path / "repo"
write_yaml(
root / "ops" / "contract.yaml",
f"""\
apiVersion: {API_VERSION}
kind: RepositoryContract
metadata:
name: sample-repo
spec:
summary: repo used for context tests
repository:
type: project
lifecycle: active
knowledge:
entrypoint: kdb/kdb.yaml
""",
)
write_yaml(
root / "kdb" / "kdb.yaml",
f"""\
apiVersion: {API_VERSION}
kind: KnowledgeBase
metadata:
name: sample-repo-kdb
spec:
repository: repo:sample-repo
defaultLanguage: en
topics:
- id: main-topic
title: Main topic
summary: The only topic
intents:
- domain.action
documents:
- guide
documents:
- id: guide
title: Guide
role: explanation
path: docs/guide.md
mediaType: text/markdown
authority: local-canonical
status: {doc_status}
""",
)
if doc_exists:
write_yaml(root / "docs" / "guide.md", "# Guide\n\nSome content.\n")
git_init(root)
return root
def test_context_selects_by_exact_intent(tmp_path: Path) -> None:
root = _repo_with_topic(tmp_path)
result = run_repoctl("context", "--repo", str(root), "--intent", "domain.action", "--output", "json")
assert result.returncode == 0, result.stderr
envelope = result.envelope()
topics = envelope["data"]["spec"]["topics"]
assert any("main-topic" in t for t in topics)
def test_context_no_match_for_unrelated_intent(tmp_path: Path) -> None:
root = _repo_with_topic(tmp_path)
result = run_repoctl("context", "--repo", str(root), "--intent", "unrelated.intent", "--output", "json")
assert result.returncode == 0
envelope = result.envelope()
assert envelope["data"]["spec"]["topics"] == []
def test_context_strict_exits_4_on_budget_truncation(tmp_path: Path) -> None:
root = _repo_with_topic(tmp_path)
result = run_repoctl("context", "--repo", str(root),
"--intent", "domain.action", "--materialize", "--budget-bytes", "1", "--strict", "--output", "json")
assert result.returncode == 4, (result.returncode, result.stdout, result.stderr)
def test_context_non_strict_truncation_still_exits_0(tmp_path: Path) -> None:
root = _repo_with_topic(tmp_path)
result = run_repoctl("context", "--repo", str(root),
"--intent", "domain.action", "--materialize", "--budget-bytes", "1", "--output", "json")
assert result.returncode == 0
def test_context_strict_exits_4_on_stale_document(tmp_path: Path) -> None:
"""19.2: exit 4 covers 'stale OR incomplete' context, not only budget truncation."""
root = _repo_with_topic(tmp_path, doc_status="stale")
result = run_repoctl("context", "--repo", str(root), "--intent", "domain.action", "--strict", "--output", "json")
assert result.returncode == 4, (result.returncode, result.stdout, result.stderr)
def test_context_strict_exits_4_on_missing_document(tmp_path: Path) -> None:
root = _repo_with_topic(tmp_path, doc_exists=False)
result = run_repoctl("context", "--repo", str(root),
"--intent", "domain.action", "--materialize", "--strict", "--output", "json")
assert result.returncode == 4, (result.returncode, result.stdout, result.stderr)
def test_context_never_writes_to_repository(tmp_path: Path) -> None:
root = _repo_with_topic(tmp_path)
before = {p for p in root.rglob("*") if p.is_file()}
run_repoctl("context", "--repo", str(root), "--intent", "domain.action", "--materialize", "--output", "json")
after = {p for p in root.rglob("*") if p.is_file()}
assert before == after