test: add spec-driven universal CLI suite; fix context strict + path confinement bugs
- tests-universal/ drives repoctl only as a subprocess CLI, asserting
directly against docs/specification.md (envelope, exit codes 0-5,
validation rules MVP, security invariants) so the same suite can judge
any conforming implementation, including the sibling codex/repoctl-l1
branch built independently from the same baseline.
- Running it surfaced two real bugs, both fixed:
- `context --strict` only checked budget truncation, never stale or
missing referenced documents, despite specification.md 19.2 defining
exit 4 as "stale or incomplete context" (not just truncation).
- `paths.confined_path` crashed with an unhandled WinError2 when a
referenced path's parent directory also didn't exist, because it
resolved a single non-existent parent with strict=True instead of
walking up to the nearest existing ancestor.
This commit is contained in:
@@ -0,0 +1,112 @@
|
||||
"""specification.md 12 (ContextBundle) and 19.2 (exit code 4)."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from pathlib import Path
|
||||
|
||||
from conftest import API_VERSION, git_init, run_repoctl, write_yaml
|
||||
|
||||
|
||||
def _repo_with_topic(tmp_path: Path, *, doc_status: str = "current", doc_exists: bool = True) -> Path:
|
||||
root = tmp_path / "repo"
|
||||
write_yaml(
|
||||
root / "ops" / "contract.yaml",
|
||||
f"""\
|
||||
apiVersion: {API_VERSION}
|
||||
kind: RepositoryContract
|
||||
metadata:
|
||||
name: sample-repo
|
||||
spec:
|
||||
summary: repo used for context tests
|
||||
repository:
|
||||
type: project
|
||||
lifecycle: active
|
||||
knowledge:
|
||||
entrypoint: kdb/kdb.yaml
|
||||
""",
|
||||
)
|
||||
write_yaml(
|
||||
root / "kdb" / "kdb.yaml",
|
||||
f"""\
|
||||
apiVersion: {API_VERSION}
|
||||
kind: KnowledgeBase
|
||||
metadata:
|
||||
name: sample-repo-kdb
|
||||
spec:
|
||||
repository: repo:sample-repo
|
||||
defaultLanguage: en
|
||||
topics:
|
||||
- id: main-topic
|
||||
title: Main topic
|
||||
summary: The only topic
|
||||
intents:
|
||||
- domain.action
|
||||
documents:
|
||||
- guide
|
||||
documents:
|
||||
- id: guide
|
||||
title: Guide
|
||||
role: explanation
|
||||
path: docs/guide.md
|
||||
mediaType: text/markdown
|
||||
authority: local-canonical
|
||||
status: {doc_status}
|
||||
""",
|
||||
)
|
||||
if doc_exists:
|
||||
write_yaml(root / "docs" / "guide.md", "# Guide\n\nSome content.\n")
|
||||
git_init(root)
|
||||
return root
|
||||
|
||||
|
||||
def test_context_selects_by_exact_intent(tmp_path: Path) -> None:
|
||||
root = _repo_with_topic(tmp_path)
|
||||
result = run_repoctl("context", "--repo", str(root), "--intent", "domain.action", "--output", "json")
|
||||
assert result.returncode == 0, result.stderr
|
||||
envelope = result.envelope()
|
||||
topics = envelope["data"]["spec"]["topics"]
|
||||
assert any("main-topic" in t for t in topics)
|
||||
|
||||
|
||||
def test_context_no_match_for_unrelated_intent(tmp_path: Path) -> None:
|
||||
root = _repo_with_topic(tmp_path)
|
||||
result = run_repoctl("context", "--repo", str(root), "--intent", "unrelated.intent", "--output", "json")
|
||||
assert result.returncode == 0
|
||||
envelope = result.envelope()
|
||||
assert envelope["data"]["spec"]["topics"] == []
|
||||
|
||||
|
||||
def test_context_strict_exits_4_on_budget_truncation(tmp_path: Path) -> None:
|
||||
root = _repo_with_topic(tmp_path)
|
||||
result = run_repoctl("context", "--repo", str(root),
|
||||
"--intent", "domain.action", "--materialize", "--budget-bytes", "1", "--strict", "--output", "json")
|
||||
assert result.returncode == 4, (result.returncode, result.stdout, result.stderr)
|
||||
|
||||
|
||||
def test_context_non_strict_truncation_still_exits_0(tmp_path: Path) -> None:
|
||||
root = _repo_with_topic(tmp_path)
|
||||
result = run_repoctl("context", "--repo", str(root),
|
||||
"--intent", "domain.action", "--materialize", "--budget-bytes", "1", "--output", "json")
|
||||
assert result.returncode == 0
|
||||
|
||||
|
||||
def test_context_strict_exits_4_on_stale_document(tmp_path: Path) -> None:
|
||||
"""19.2: exit 4 covers 'stale OR incomplete' context, not only budget truncation."""
|
||||
root = _repo_with_topic(tmp_path, doc_status="stale")
|
||||
result = run_repoctl("context", "--repo", str(root), "--intent", "domain.action", "--strict", "--output", "json")
|
||||
assert result.returncode == 4, (result.returncode, result.stdout, result.stderr)
|
||||
|
||||
|
||||
def test_context_strict_exits_4_on_missing_document(tmp_path: Path) -> None:
|
||||
root = _repo_with_topic(tmp_path, doc_exists=False)
|
||||
result = run_repoctl("context", "--repo", str(root),
|
||||
"--intent", "domain.action", "--materialize", "--strict", "--output", "json")
|
||||
assert result.returncode == 4, (result.returncode, result.stdout, result.stderr)
|
||||
|
||||
|
||||
def test_context_never_writes_to_repository(tmp_path: Path) -> None:
|
||||
root = _repo_with_topic(tmp_path)
|
||||
before = {p for p in root.rglob("*") if p.is_file()}
|
||||
run_repoctl("context", "--repo", str(root), "--intent", "domain.action", "--materialize", "--output", "json")
|
||||
after = {p for p in root.rglob("*") if p.is_file()}
|
||||
assert before == after
|
||||
Reference in New Issue
Block a user