agentic-ai-engineering/pipeline/confidence_ladder.py

"""
Confidence Ladder — 5-level Verdict system.

Replaces binary pass/fail gates with a graduated confidence scale.
Each verdict carries a human-readable summary and optional evidence.

Levels:
  PERFECT   — All assertions passed, all paths verified
  VERIFIED  — Core assertions passed, edge cases known
  PARTIAL   — Nothing failed, but significant gaps were unverifiable
  FEEDBACK  — Assertions failed OR verification revealed blocking issues
  FAILED    — Assertions failed AND blocker will require rework

PARTIAL is the critical addition: it catches "green but dead" — cases where
the system ran without errors but couldn't prove correctness.
"""

from __future__ import annotations

import json
from dataclasses import dataclass, field, asdict
from datetime import datetime, timezone
from enum import Enum
from typing import Any, Optional


class VerdictLevel(Enum):
    """Five-level confidence ladder.

    Ordered from most to least confident. Comparisons are meaningful:
    ``PERFECT > VERIFIED > PARTIAL > FEEDBACK > FAILED``.
    """

    PERFECT = 5
    VERIFIED = 4
    PARTIAL = 3
    FEEDBACK = 2
    FAILED = 1

    def __lt__(self, other: Any) -> bool:
        if not isinstance(other, VerdictLevel):
            return NotImplemented
        return self.value < other.value

    def __le__(self, other: Any) -> bool:
        if not isinstance(other, VerdictLevel):
            return NotImplemented
        return self.value <= other.value

    def __gt__(self, other: Any) -> bool:
        if not isinstance(other, VerdictLevel):
            return NotImplemented
        return self.value > other.value

    def __ge__(self, other: Any) -> bool:
        if not isinstance(other, VerdictLevel):
            return NotImplemented
        return self.value >= other.value

    @classmethod
    def from_str(cls, label: str) -> VerdictLevel:
        """Parse a case-insensitive string to a VerdictLevel."""
        return cls[label.upper()]

    @property
    def label(self) -> str:
        return self.name

    @property
    def is_pass(self) -> bool:
        """``True`` for PERFECT, VERIFIED, PARTIAL — the system should continue."""
        return self.value >= VerdictLevel.PARTIAL.value

    @property
    def is_blocking(self) -> bool:
        """``True`` for PERFECT, VERIFIED — may proceed without intervention."""
        return self.value >= VerdictLevel.VERIFIED.value


@dataclass
class Verdict:
    """A single verdict from one verifier check."""

    level: VerdictLevel
    summary: str
    evidence: list[str] = field(default_factory=list)
    source: Optional[str] = None  # e.g. "critic:style", "test:unit"

    def dict(self) -> dict[str, Any]:
        return {
            "level": self.level.name,
            "summary": self.summary,
            "evidence": self.evidence,
            "source": self.source,
        }


@dataclass
class VerdictReport:
    """Aggregate report from one or more verifier checks."""

    verdicts: list[Verdict] = field(default_factory=list)
    pipeline_id: Optional[str] = None  # e.g. ADW ID or cron run ID
    timestamp: str = field(default_factory=lambda: datetime.now(timezone.utc).isoformat())

    @property
    def aggregate(self) -> VerdictLevel:
        """The lowest individual verdict sets the aggregate level.

        The chain is only as strong as its weakest gate.
        """
        if not self.verdicts:
            return VerdictLevel.FAILED
        return min(v.level for v in self.verdicts)

    @property
    def aggregate_label(self) -> str:
        return self.aggregate.label

    @property
    def passed(self) -> bool:
        """``True`` if aggregate is PERFECT, VERIFIED, or PARTIAL."""
        return self.aggregate.is_pass

    @property
    def blocked(self) -> bool:
        """``True`` if aggregate is FEEDBACK or FAILED."""
        return not self.aggregate.is_pass

    def add(self, verdict: Verdict) -> None:
        self.verdicts.append(verdict)

    def dict(self) -> dict[str, Any]:
        return {
            "pipeline_id": self.pipeline_id,
            "timestamp": self.timestamp,
            "aggregate": self.aggregate_label,
            "passed": self.passed,
            "blocked": self.blocked,
            "verdicts": [v.dict() for v in self.verdicts],
        }

    def to_json(self, indent: int = 2) -> str:
        return json.dumps(self.dict(), indent=indent)

    @classmethod
    def from_json(cls, raw: str) -> VerdictReport:
        data = json.loads(raw)
        report = cls(pipeline_id=data.get("pipeline_id"), timestamp=data.get("timestamp", ""))
        for vd in data.get("verdicts", []):
            report.add(
                Verdict(
                    level=VerdictLevel.from_str(vd["level"]),
                    summary=vd["summary"],
                    evidence=vd.get("evidence", []),
                    source=vd.get("source"),
                )
            )
        return report

    def threshhold_met(self, minimum: VerdictLevel = VerdictLevel.PARTIAL) -> bool:
        """Check if aggregate meets a minimum confidence threshold."""
        return self.aggregate >= minimum