"""
Lab 6.8: Build an Eval Harness

Objective: Create golden Q&A pairs + automated pass/fail scoring.
"""

from typing import Any
import json

# TODO 1: Define golden test cases
# Each case has:
#   - input: the user prompt / task description
#   - expected_tools: list of tools the agent should call (in order)
#   - expected_tool_params: dict of expected parameters per tool call
#   - expected_output_contains: list of strings that should appear in output
#   - weight: importance weight for scoring

GOLDEN_DATASET = [
    {
        "id": "test-001",
        "input": "Find all users created in the last 24 hours",
        "expected_tools": ["query_database"],
        "expected_tool_params": {},  # TODO: define expected params
        "expected_output_contains": ["users", "24"],
        "weight": 1.0
    },
    {
        "id": "test-002",
        "input": "What's the current weather in Tokyo?",
        "expected_tools": ["search_web"],  # or ["weather_api"]
        "expected_output_contains": ["Tokyo", "°"],
        "weight": 1.0
    },
    # TODO: Add 8 more test cases covering different scenarios
]


# TODO 2: Implement evaluation logic
class EvalHarness:
    """Evaluate agent responses against golden dataset."""
    
    def __init__(self, dataset: list[dict]):
        self.dataset = dataset
    
    def evaluate(self, agent_func: callable) -> dict:
        """
        Run the agent against all test cases and compute pass@k.
        
        Returns:
        {
            "pass@1": float,  # % of cases passed on first try
            "pass@3": float,  # % of cases passed on best of 3
            "pass@5": float,  # % of cases passed on best of 5
            "weighted_score": float,
            "results": [case results]
        }
        """
        pass  # TODO
    
    def score_case(self, result: Any) -> float:
        """
        Score a single agent result against expected criteria.
        Returns 0.0 to 1.0 based on:
        - Tool selection accuracy
        - Parameter correctness
        - Output content match
        """
        pass  # TODO


if __name__ == "__main__":
    harness = EvalHarness(GOLDEN_DATASET)
    
    # Define your agent function here
    def my_agent(input_text: str) -> Any:
        # TODO: implement or import your agent
        return {"output": "placeholder"}
    
    results = harness.evaluate(my_agent)
    print(json.dumps(results, indent=2))
    
    print(f"\npass@1: {results['pass@1']:.1%}")
    print(f"pass@3: {results['pass@3']:.1%}")
    print(f"pass@5: {results['pass@5']:.1%}")