feat: add reviewer-validated claim evidence matrix

README.md

@@ -257,6 +257,44 @@ counterproof share-witness REGRESSION_WITNESS.json \
 
 The note includes the exact HEAD / BASE commits, exit codes, executed changed-test command, evidence digest, links, and the scope limit that a regression witness proves the tested before/after delta — not every claimed production cause.
 
+### When one PR contains multiple review claims
+
+A real PR can have one genuinely witnessed regression and several adjacent concerns that its tests do not exercise.
+
+Counterproof's experimental claim matrix keeps those claims separate:
+
+```bash
+counterproof claim-matrix examples/claim_matrix/codex-plugin-cc-731.yml
+```
+
+The manifest is explicit. Counterproof does **not** use an LLM to invent claims or decide which product behavior is authoritative.
+
+Each row has two mechanical evidence axes:
+
+```text
+submitted-test evidence
+  WITNESSED / NOT_WITNESSED / UNPROVEN
+
+oracle alignment
+  ALIGNED / CONTRADICTED / UNVERIFIED
+```
+
+The overall claim is derived conservatively:
+
+```text
+WITNESSED + ALIGNED      -> PROVEN
+anything + CONTRADICTED  -> CONTRADICTED
+WITNESSED + UNVERIFIED   -> WITNESSED (submitted judge)
+otherwise                -> UNPROVEN
+```
+
+That means a red→green regression can stay useful without silently becoming a product-correctness claim.
+
+The first two acceptance fixtures come directly from public reviewer feedback:
+
+- `examples/claim_matrix/codex-plugin-cc-731.yml` — one witnessed submitted regression, later review concerns still unproven;
+- `examples/claim_matrix/claude-code-89404.yml` — product-oracle contradiction stays stronger than an internally green submitted judge.
+
 ---
 
 ## It also checks whether the PR changed the judge

examples/claim_matrix/claude-code-89404.yml

@@ -0,0 +1,16 @@
+schema_version: 1
+title: "Oracle Alignment Matrix — anthropics/claude-code #89404"
+source_pr: "https://github.com/anthropics/claude-code/pull/89404"
+claims:
+  - id: fixture-validity
+    claim: "Agent fixtures labeled valid by the submitted validator are valid to Claude Code"
+    submitted_test_evidence: UNPROVEN
+    oracle_alignment: CONTRADICTED
+    oracle_probe: "claude plugin validate rejects fixtures that the submitted validator treats as valid"
+    oracle_source_url: "https://github.com/anthropics/claude-code/pull/89404"
+    note: "Reviewer evidence establishes product-oracle contradiction; CounterProof has not independently replayed BASE/HEAD for this row."
+  - id: multiline-description-regression
+    claim: "The submitted suite protects the multi-line description extraction behavior"
+    submitted_test_evidence: UNPROVEN
+    oracle_alignment: UNVERIFIED
+    note: "Reviewer reports the suite stays green when the extraction fix is reverted."

examples/claim_matrix/codex-plugin-cc-731.yml

@@ -0,0 +1,29 @@
+schema_version: 1
+title: "Claim / Evidence Matrix — openai/codex-plugin-cc #731"
+source_pr: "https://github.com/openai/codex-plugin-cc/pull/731"
+base_sha: "db52e28f4d9ded852ab3942cea316258ae4ef346"
+head_sha: "a81f52f9c2c366d7d5e095179d2473a9f7e12798"
+runner_url: "https://github.com/hippoley/CounterProof/actions/runs/35952687043"
+evidence_digest: "sha256:01c447fbfca700a6dcbfcec2753ca3c55a0760b5b6200a751d44a948bdbf5c97"
+claims:
+  - id: authority-boundary
+    claim: "Cross-CLAUDE_PLUGIN_DATA durable review state must not fail open"
+    tests:
+      - "tests/runtime.test.mjs"
+      - "tests/state.test.mjs"
+    base_result: "FAIL"
+    head_result: "PASS"
+    submitted_test_evidence: WITNESSED
+    oracle_alignment: UNVERIFIED
+    assertion_excerpt: "The submitted changed tests fail on BASE and pass on HEAD for the authority-boundary behavior."
+    note: "This witness is scoped to the submitted judge; no independent product oracle was run."
+  - id: restrictive-permissions
+    claim: "Durable config should use restrictive file permissions such as 0600"
+    submitted_test_evidence: UNPROVEN
+    oracle_alignment: UNVERIFIED
+    note: "No changed test exercises file mode bits."
+  - id: atomic-replacement
+    claim: "Failed or interrupted replacement should preserve the previous valid config"
+    submitted_test_evidence: UNPROVEN
+    oracle_alignment: UNVERIFIED
+    note: "No changed test injects replacement/write failure."

skill_factory/evolution/claim_matrix.py

@@ -0,0 +1,235 @@
+"""Reviewer-facing claim/evidence matrices with mechanical proof semantics."""
+from __future__ import annotations
+
+import json
+from enum import Enum
+from pathlib import Path
+from typing import Any
+
+import yaml
+from pydantic import BaseModel, Field, ValidationError, model_validator
+
+
+class SubmittedTestEvidence(str, Enum):
+    WITNESSED = "WITNESSED"
+    NOT_WITNESSED = "NOT_WITNESSED"
+    UNPROVEN = "UNPROVEN"
+
+
+class OracleAlignment(str, Enum):
+    ALIGNED = "ALIGNED"
+    CONTRADICTED = "CONTRADICTED"
+    UNVERIFIED = "UNVERIFIED"
+
+
+class OverallClaim(str, Enum):
+    PROVEN = "PROVEN"
+    CONTRADICTED = "CONTRADICTED"
+    WITNESSED_SUBMITTED_JUDGE = "WITNESSED (submitted judge)"
+    UNPROVEN = "UNPROVEN"
+
+
+class ClaimEvidence(BaseModel):
+    id: str = Field(min_length=1)
+    claim: str = Field(min_length=1)
+    source_url: str | None = None
+    tests: list[str] = Field(default_factory=list)
+    base_result: str | None = None
+    head_result: str | None = None
+    submitted_test_evidence: SubmittedTestEvidence
+    oracle_alignment: OracleAlignment = OracleAlignment.UNVERIFIED
+    oracle_probe: str | None = None
+    oracle_source_url: str | None = None
+    assertion_excerpt: str | None = None
+    note: str | None = None
+
+    @model_validator(mode="after")
+    def validate_evidence_contract(self) -> ClaimEvidence:
+        if self.submitted_test_evidence in {
+            SubmittedTestEvidence.WITNESSED,
+            SubmittedTestEvidence.NOT_WITNESSED,
+        }:
+            if not self.tests:
+                raise ValueError(
+                    "WITNESSED / NOT_WITNESSED claims require at least one exact test"
+                )
+            if not self.base_result or not self.head_result:
+                raise ValueError(
+                    "WITNESSED / NOT_WITNESSED claims require BASE and HEAD results"
+                )
+
+        if self.oracle_alignment in {
+            OracleAlignment.ALIGNED,
+            OracleAlignment.CONTRADICTED,
+        } and not self.oracle_probe:
+            raise ValueError(
+                "ALIGNED / CONTRADICTED oracle status requires an explicit oracle_probe"
+            )
+        return self
+
+    @property
+    def overall_claim(self) -> OverallClaim:
+        if self.oracle_alignment is OracleAlignment.CONTRADICTED:
+            return OverallClaim.CONTRADICTED
+        if (
+            self.submitted_test_evidence is SubmittedTestEvidence.WITNESSED
+            and self.oracle_alignment is OracleAlignment.ALIGNED
+        ):
+            return OverallClaim.PROVEN
+        if (
+            self.submitted_test_evidence is SubmittedTestEvidence.WITNESSED
+            and self.oracle_alignment is OracleAlignment.UNVERIFIED
+        ):
+            return OverallClaim.WITNESSED_SUBMITTED_JUDGE
+        return OverallClaim.UNPROVEN
+
+
+class ClaimMatrixManifest(BaseModel):
+    schema_version: int = 1
+    title: str = Field(min_length=1)
+    source_pr: str | None = None
+    base_sha: str | None = None
+    head_sha: str | None = None
+    runner_url: str | None = None
+    evidence_digest: str | None = None
+    claims: list[ClaimEvidence] = Field(min_length=1)
+
+
+def load_claim_matrix(path: Path) -> ClaimMatrixManifest:
+    text = path.read_text(encoding="utf-8")
+    try:
+        if path.suffix.lower() == ".json":
+            raw: Any = json.loads(text)
+        else:
+            raw = yaml.safe_load(text)
+        return ClaimMatrixManifest.model_validate(raw)
+    except (json.JSONDecodeError, yaml.YAMLError, ValidationError, TypeError) as exc:
+        raise ValueError(f"invalid claim matrix manifest: {exc}") from exc
+
+
+def claim_matrix_to_dict(manifest: ClaimMatrixManifest) -> dict[str, Any]:
+    payload = manifest.model_dump(mode="json")
+    for claim, raw in zip(manifest.claims, payload["claims"], strict=True):
+        raw["overall_claim"] = claim.overall_claim.value
+    return payload
+
+
+def _cell(value: str | None) -> str:
+    if not value:
+        return "—"
+    return value.replace("|", "\\|").replace("\n", " ")
+
+
+def render_claim_matrix_markdown(manifest: ClaimMatrixManifest) -> str:
+    lines = [
+        f"# {manifest.title}",
+        "",
+        "> Evidence is scoped per claim. A green submitted test is not product-level proof",
+        "> unless the relevant oracle is independently aligned.",
+        "",
+    ]
+
+    if manifest.source_pr:
+        lines.append(f"- Source PR: {manifest.source_pr}")
+    if manifest.base_sha:
+        lines.append(f"- BASE: `{manifest.base_sha}`")
+    if manifest.head_sha:
+        lines.append(f"- HEAD: `{manifest.head_sha}`")
+    if manifest.runner_url:
+        lines.append(f"- Runner: {manifest.runner_url}")
+    if manifest.evidence_digest:
+        lines.append(f"- Evidence digest: `{manifest.evidence_digest}`")
+    if (
+        manifest.source_pr
+        or manifest.base_sha
+        or manifest.head_sha
+        or manifest.runner_url
+        or manifest.evidence_digest
+    ):
+        lines.append("")
+
+    lines.extend(
+        [
+            "## Claim / evidence matrix",
+            "",
+            (
+                "| Claim | Exact test(s) | BASE | HEAD | Submitted-test evidence | "
+                "Oracle alignment | Overall claim |"
+            ),
+            "|---|---|---|---|---|---|---|",
+        ]
+    )
+
+    for claim in manifest.claims:
+        tests = "<br>".join(f"`{item}`" for item in claim.tests) if claim.tests else "—"
+        lines.append(
+            "| "
+            + " | ".join(
+                [
+                    _cell(claim.claim),
+                    tests,
+                    _cell(claim.base_result),
+                    _cell(claim.head_result),
+                    f"**{claim.submitted_test_evidence.value}**",
+                    f"**{claim.oracle_alignment.value}**",
+                    f"**{claim.overall_claim.value}**",
+                ]
+            )
+            + " |"
+        )
+
+    details = [
+        claim
+        for claim in manifest.claims
+        if claim.assertion_excerpt
+        or claim.oracle_probe
+        or claim.note
+        or claim.source_url
+        or claim.oracle_source_url
+    ]
+    if details:
+        lines.extend(["", "## Evidence details", ""])
+        for claim in details:
+            lines.append(f"### {claim.id}")
+            lines.append("")
+            if claim.source_url:
+                lines.append(f"- Claim source: {claim.source_url}")
+            if claim.oracle_probe:
+                lines.append(f"- Oracle probe: {_cell(claim.oracle_probe)}")
+            if claim.oracle_source_url:
+                lines.append(f"- Oracle source: {claim.oracle_source_url}")
+            if claim.note:
+                lines.append(f"- Note: {_cell(claim.note)}")
+            if claim.assertion_excerpt:
+                lines.extend(
+                    [
+                        "",
+                        "Assertion / failure excerpt:",
+                        "",
+                        "```text",
+                        claim.assertion_excerpt.rstrip(),
+                        "```",
+                    ]
+                )
+            lines.append("")
+
+    lines.extend(
+        [
+            "## Vocabulary",
+            "",
+            "- Submitted-test evidence: `WITNESSED / NOT_WITNESSED / UNPROVEN`",
+            "- Oracle alignment: `ALIGNED / CONTRADICTED / UNVERIFIED`",
+            (
+                "- `WITNESSED (submitted judge)` means the submitted test distinguishes "
+                "BASE from HEAD while product-oracle alignment remains unverified."
+            ),
+            (
+                "- `PROVEN` requires both a witnessed submitted regression and an aligned "
+                "authoritative oracle."
+            ),
+            "- `CONTRADICTED` wins whenever the authoritative oracle contradicts the claim.",
+            "- Otherwise the overall claim remains `UNPROVEN`.",
+            "",
+        ]
+    )
+    return "\n".join(lines)

skill_factory/evolution/cli.py

@@ -12,6 +12,11 @@
 
 from .adapter_binding import bind_probe_adapter
 from .capabilities import capability_report
+from .claim_matrix import (
+    claim_matrix_to_dict,
+    load_claim_matrix,
+    render_claim_matrix_markdown,
+)
 from .discriminate import (
     discrimination_to_dict,
     render_discrimination_markdown,
@@ -94,6 +99,39 @@ def cli() -> None:
     """Counterproof: falsifiable change control for self-modifying agents."""
 
 
+@cli.command("claim-matrix")
+@click.argument("manifest_file", type=click.Path(exists=True, dir_okay=False))
+@click.option("--out", "out_file", default="CLAIM_EVIDENCE_MATRIX.md", show_default=True)
+@click.option(
+    "--json-out",
+    default="CLAIM_EVIDENCE_MATRIX.json",
+    show_default=True,
+    type=click.Path(dir_okay=False),
+)
+def claim_matrix(manifest_file: str, out_file: str, json_out: str) -> None:
+    """Render an explicit reviewer claim/evidence manifest.
+
+    Claims are supplied by a human or upstream review system. CounterProof does
+    not infer claims or oracle authority in this command.
+    """
+    try:
+        manifest = load_claim_matrix(Path(manifest_file))
+    except ValueError as exc:
+        raise click.ClickException(str(exc)) from exc
+
+    Path(out_file).write_text(
+        render_claim_matrix_markdown(manifest),
+        encoding="utf-8",
+    )
+    Path(json_out).write_text(
+        json.dumps(claim_matrix_to_dict(manifest), indent=2, ensure_ascii=False),
+        encoding="utf-8",
+    )
+    click.echo(f"Claims: {len(manifest.claims)}")
+    click.echo(f"Wrote {out_file}")
+    click.echo(f"Wrote {json_out}")
+
+
 @cli.command("build")
 @click.argument("packet_file", type=click.Path(exists=True, dir_okay=False))
 @click.option("--out", "out_file", default="EVOLUTION_PR.md", show_default=True)