test: add spec-driven universal CLI suite; fix context strict + path confinement bugs
- tests-universal/ drives repoctl only as a subprocess CLI, asserting
directly against docs/specification.md (envelope, exit codes 0-5,
validation rules MVP, security invariants) so the same suite can judge
any conforming implementation, including the sibling codex/repoctl-l1
branch built independently from the same baseline.
- Running it surfaced two real bugs, both fixed:
- `context --strict` only checked budget truncation, never stale or
missing referenced documents, despite specification.md 19.2 defining
exit 4 as "stale or incomplete context" (not just truncation).
- `paths.confined_path` crashed with an unhandled WinError2 when a
referenced path's parent directory also didn't exist, because it
resolved a single non-existent parent with strict=True instead of
walking up to the nearest existing ancestor.
This commit is contained in:
@@ -0,0 +1,120 @@
|
||||
"""specification.md 20: Validation rules MVP."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from pathlib import Path
|
||||
|
||||
from conftest import API_VERSION, git_init, minimal_contract, minimal_kdb, run_repoctl, write_yaml
|
||||
|
||||
|
||||
def _repo_with_contract(tmp_path: Path, contract_text: str, *, name: str = "sample-repo") -> Path:
|
||||
root = tmp_path / "repo"
|
||||
write_yaml(root / "ops" / "contract.yaml", contract_text)
|
||||
write_yaml(root / "kdb" / "kdb.yaml", minimal_kdb(name))
|
||||
git_init(root)
|
||||
return root
|
||||
|
||||
|
||||
def test_duplicate_content_root_ids_rejected(tmp_path: Path) -> None:
|
||||
contract = minimal_contract(
|
||||
extra_spec="""\
|
||||
contentRoots:
|
||||
- id: docs
|
||||
role: documentation
|
||||
path: docs
|
||||
kind: file-set
|
||||
authority: source
|
||||
writePolicy: managed
|
||||
- id: docs
|
||||
role: documentation
|
||||
path: docs2
|
||||
kind: file-set
|
||||
authority: source
|
||||
writePolicy: managed
|
||||
""",
|
||||
)
|
||||
root = _repo_with_contract(tmp_path, contract)
|
||||
(root / "docs").mkdir()
|
||||
(root / "docs2").mkdir()
|
||||
|
||||
result = run_repoctl("validate", "--repo", str(root), "--output", "json")
|
||||
envelope = result.envelope()
|
||||
assert result.returncode == 1
|
||||
assert any("DUPLICATE" in f["code"] for f in envelope["findings"])
|
||||
|
||||
|
||||
def test_invalid_id_format_rejected(tmp_path: Path) -> None:
|
||||
contract = minimal_contract(name="Not-A-Valid-ID!!")
|
||||
root = _repo_with_contract(tmp_path, contract, name="Not-A-Valid-ID!!")
|
||||
|
||||
result = run_repoctl("validate", "--repo", str(root), "--output", "json")
|
||||
assert result.returncode == 1
|
||||
envelope = result.envelope()
|
||||
assert any(f["severity"] == "error" for f in envelope["findings"])
|
||||
|
||||
|
||||
def test_unknown_repository_type_rejected(tmp_path: Path) -> None:
|
||||
contract = f"""\
|
||||
apiVersion: {API_VERSION}
|
||||
kind: RepositoryContract
|
||||
metadata:
|
||||
name: sample-repo
|
||||
spec:
|
||||
summary: repo with a bogus type
|
||||
repository:
|
||||
type: not-a-real-type
|
||||
lifecycle: active
|
||||
"""
|
||||
root = _repo_with_contract(tmp_path, contract)
|
||||
result = run_repoctl("validate", "--repo", str(root), "--output", "json")
|
||||
assert result.returncode == 1
|
||||
envelope = result.envelope()
|
||||
assert any(f["severity"] == "error" for f in envelope["findings"])
|
||||
|
||||
|
||||
def test_unknown_apiversion_rejected(tmp_path: Path) -> None:
|
||||
contract = """\
|
||||
apiVersion: stewardship.corsus.ru/v99
|
||||
kind: RepositoryContract
|
||||
metadata:
|
||||
name: sample-repo
|
||||
spec:
|
||||
summary: repo declaring an unsupported apiVersion
|
||||
repository:
|
||||
type: project
|
||||
lifecycle: active
|
||||
"""
|
||||
root = _repo_with_contract(tmp_path, contract)
|
||||
result = run_repoctl("validate", "--repo", str(root), "--output", "json")
|
||||
assert result.returncode == 1
|
||||
|
||||
|
||||
def test_kdb_topic_references_missing_document_rejected(tmp_path: Path) -> None:
|
||||
contract = minimal_contract()
|
||||
kdb = f"""\
|
||||
apiVersion: {API_VERSION}
|
||||
kind: KnowledgeBase
|
||||
metadata:
|
||||
name: sample-repo-kdb
|
||||
spec:
|
||||
repository: repo:sample-repo
|
||||
defaultLanguage: en
|
||||
topics:
|
||||
- id: some-topic
|
||||
title: Some topic
|
||||
summary: Refers to a document that was never declared
|
||||
intents:
|
||||
- some.intent
|
||||
documents:
|
||||
- missing-document
|
||||
documents: []
|
||||
"""
|
||||
root = tmp_path / "repo"
|
||||
write_yaml(root / "ops" / "contract.yaml", contract)
|
||||
write_yaml(root / "kdb" / "kdb.yaml", kdb)
|
||||
git_init(root)
|
||||
|
||||
result = run_repoctl("validate", "--repo", str(root), "--output", "json")
|
||||
assert result.returncode == 1
|
||||
envelope = result.envelope()
|
||||
assert any(f["severity"] == "error" for f in envelope["findings"])
|
||||
Reference in New Issue
Block a user