85 lines
2.5 KiB
Python
85 lines
2.5 KiB
Python
"""
|
|
Lab 6.8: Build an Eval Harness
|
|
|
|
Objective: Create golden Q&A pairs + automated pass/fail scoring.
|
|
"""
|
|
|
|
from typing import Any
|
|
import json
|
|
|
|
# TODO 1: Define golden test cases
|
|
# Each case has:
|
|
# - input: the user prompt / task description
|
|
# - expected_tools: list of tools the agent should call (in order)
|
|
# - expected_tool_params: dict of expected parameters per tool call
|
|
# - expected_output_contains: list of strings that should appear in output
|
|
# - weight: importance weight for scoring
|
|
|
|
GOLDEN_DATASET = [
|
|
{
|
|
"id": "test-001",
|
|
"input": "Find all users created in the last 24 hours",
|
|
"expected_tools": ["query_database"],
|
|
"expected_tool_params": {}, # TODO: define expected params
|
|
"expected_output_contains": ["users", "24"],
|
|
"weight": 1.0
|
|
},
|
|
{
|
|
"id": "test-002",
|
|
"input": "What's the current weather in Tokyo?",
|
|
"expected_tools": ["search_web"], # or ["weather_api"]
|
|
"expected_output_contains": ["Tokyo", "°"],
|
|
"weight": 1.0
|
|
},
|
|
# TODO: Add 8 more test cases covering different scenarios
|
|
]
|
|
|
|
|
|
# TODO 2: Implement evaluation logic
|
|
class EvalHarness:
|
|
"""Evaluate agent responses against golden dataset."""
|
|
|
|
def __init__(self, dataset: list[dict]):
|
|
self.dataset = dataset
|
|
|
|
def evaluate(self, agent_func: callable) -> dict:
|
|
"""
|
|
Run the agent against all test cases and compute pass@k.
|
|
|
|
Returns:
|
|
{
|
|
"pass@1": float, # % of cases passed on first try
|
|
"pass@3": float, # % of cases passed on best of 3
|
|
"pass@5": float, # % of cases passed on best of 5
|
|
"weighted_score": float,
|
|
"results": [case results]
|
|
}
|
|
"""
|
|
pass # TODO
|
|
|
|
def score_case(self, result: Any) -> float:
|
|
"""
|
|
Score a single agent result against expected criteria.
|
|
Returns 0.0 to 1.0 based on:
|
|
- Tool selection accuracy
|
|
- Parameter correctness
|
|
- Output content match
|
|
"""
|
|
pass # TODO
|
|
|
|
|
|
if __name__ == "__main__":
|
|
harness = EvalHarness(GOLDEN_DATASET)
|
|
|
|
# Define your agent function here
|
|
def my_agent(input_text: str) -> Any:
|
|
# TODO: implement or import your agent
|
|
return {"output": "placeholder"}
|
|
|
|
results = harness.evaluate(my_agent)
|
|
print(json.dumps(results, indent=2))
|
|
|
|
print(f"\npass@1: {results['pass@1']:.1%}")
|
|
print(f"pass@3: {results['pass@3']:.1%}")
|
|
print(f"pass@5: {results['pass@5']:.1%}")
|