test: add spec-driven universal CLI suite; fix context strict + path confinement bugs

- tests-universal/ drives repoctl only as a subprocess CLI, asserting
  directly against docs/specification.md (envelope, exit codes 0-5,
  validation rules MVP, security invariants) so the same suite can judge
  any conforming implementation, including the sibling codex/repoctl-l1
  branch built independently from the same baseline.
- Running it surfaced two real bugs, both fixed:
  - `context --strict` only checked budget truncation, never stale or
    missing referenced documents, despite specification.md 19.2 defining
    exit 4 as "stale or incomplete context" (not just truncation).
  - `paths.confined_path` crashed with an unhandled WinError2 when a
    referenced path's parent directory also didn't exist, because it
    resolved a single non-existent parent with strict=True instead of
    walking up to the nearest existing ancestor.
This commit is contained in:
Alexander Baranov
2026-08-24 15:38:49 +03:00
parent 64e6c2f5f5
commit 86dd79c8ae
10 changed files with 771 additions and 3 deletions
+1 -1
View File
@@ -128,7 +128,7 @@ def cmd_context(args: argparse.Namespace) -> int:
]
outcome = "success"
exit_code = EXIT_SUCCESS
if result.truncated and args.strict:
if (result.truncated or result.incomplete) and args.strict:
outcome = "failure"
exit_code = EXIT_STRICT_INCOMPLETE
_write(build_envelope("context", outcome, data, findings), args.output)
+7 -1
View File
@@ -24,6 +24,7 @@ class ContextResult:
bundle: dict[str, Any]
materialized: list[MaterializedItem] = field(default_factory=list)
truncated: bool = False
incomplete: bool = False
def build_context(
@@ -74,6 +75,7 @@ def build_context(
if isinstance(topic, dict) and topic.get("id") in related_ids:
topics_selected.append(topic)
incomplete = False
seen_docs: set[str] = set()
for topic in topics_selected:
for doc_id in topic.get("documents") or []:
@@ -84,8 +86,10 @@ def build_context(
doc = all_documents.get(doc_id)
if doc is None:
warnings.append(f"warning:CONTEXT_DOCUMENT_MISSING:Referenced document not found: {doc_id}")
incomplete = True
elif doc.get("status") == "stale":
warnings.append(f"warning:CONTEXT_DOCUMENT_STALE:Document authored status is stale: {doc_id}")
incomplete = True
topics_ref = sorted(f"topic:{repo_name}/{t.get('id')}" for t in topics_selected if t.get("id"))
operations = sorted(set(operations))
@@ -112,7 +116,7 @@ def build_context(
if contract_revision:
bundle["spec"]["contractRevision"] = contract_revision
result = ContextResult(bundle=bundle)
result = ContextResult(bundle=bundle, incomplete=incomplete)
if materialize:
remaining = budget_bytes if budget_bytes is not None else None
@@ -124,6 +128,8 @@ def build_context(
continue
doc_path = root / doc["path"]
if not doc_path.exists():
warnings.append(f"warning:CONTEXT_MATERIALIZE_MISSING:Document path does not exist: {doc_id}")
result.incomplete = True
continue
content = doc_path.read_text(encoding="utf-8")
content_bytes = content.encode("utf-8")
+7 -1
View File
@@ -23,7 +23,13 @@ def confined_path(root: Path, value: str, *, must_exist: bool = False) -> Path:
normalized = normalize_resource_path(value)
root_resolved = root.resolve(strict=True)
candidate = root_resolved.joinpath(*normalized.split("/"))
probe = candidate if candidate.exists() else candidate.parent
# Walk up to the nearest ancestor that actually exists before resolving
# strictly: neither the candidate nor any number of its parent directories
# are guaranteed to exist yet (e.g. a document path with a missing parent
# directory), and `root_resolved` always exists, so this always terminates.
probe = candidate
while not probe.exists():
probe = probe.parent
try:
probe.resolve(strict=True).relative_to(root_resolved)
except ValueError as exc: