""" Lab 3.9: Build a Verifier Agent -- SOLUTION """ import json import os VERIFIER_TOOLS = [ { "name": "read_file", "description": "Read the contents of a file", "input_schema": { "type": "object", "properties": { "path": {"type": "string"}, "reasoning": {"type": "string"} }, "required": ["path", "reasoning"] } }, { "name": "grep_search", "description": "Search for a pattern in files", "input_schema": { "type": "object", "properties": { "pattern": {"type": "string"}, "path": {"type": "string"}, "reasoning": {"type": "string"} }, "required": ["pattern", "path", "reasoning"] } }, { "name": "list_files", "description": "List files in a directory", "input_schema": { "type": "object", "properties": { "path": {"type": "string"}, "reasoning": {"type": "string"} }, "required": ["path", "reasoning"] } } ] def verify_work(claims: list[dict], workspace_path: str) -> dict: verified = [] failed = [] unverifiable = [] for claim in claims: check_type = claim.get("check_type", "file_exists") evidence = claim["expected_evidence"] try: if check_type == "file_exists": target = os.path.join(workspace_path, evidence) if os.path.exists(target): verified.append(claim) else: failed.append({**claim, "reason": f"File not found: {target}"}) elif check_type == "file_contains": target = os.path.join(workspace_path, evidence) contains = claim.get("contains", "") if os.path.exists(target): with open(target) as f: content = f.read() if contains in content: verified.append(claim) else: failed.append({**claim, "reason": f"'{contains}' not found in {target}"}) else: failed.append({**claim, "reason": f"File not found: {target}"}) elif check_type == "file_not_exists": target = os.path.join(workspace_path, evidence) if not os.path.exists(target): verified.append(claim) else: failed.append({**claim, "reason": f"File should not exist: {target}"}) else: unverifiable.append({**claim, "reason": f"Unknown check type: {check_type}"}) except Exception as e: failed.append({**claim, "reason": str(e)}) # Determine confidence level total = len(verified) + len(failed) + len(unverifiable) if total == 0: status = "FAILED" elif len(failed) == 0 and len(unverifiable) == 0: status = "PERFECT" elif len(failed) == 0 and len(unverifiable) > 0: status = "PARTIAL" elif len(failed) > 0 and len(failed) < total: status = "FEEDBACK" else: status = "FAILED" return { "status": status, "verified_claims": [c["claim"] for c in verified], "failed_claims": [{"claim": c["claim"], "reason": c["reason"]} for c in failed], "unverifiable_claims": [{"claim": c["claim"], "reason": c["reason"]} for c in unverifiable], "summary": f"{len(verified)}/{total} claims verified" } if __name__ == "__main__": test_claims = [ {"claim": "Created the data processing script", "expected_evidence": "process.py", "check_type": "file_exists"}, {"claim": "Script reads CSV files", "expected_evidence": "process.py", "contains": "read_csv", "check_type": "file_contains"}, ] result = verify_work(test_claims, "./") print(json.dumps(result, indent=2))