127 lines
4.1 KiB
Python
127 lines
4.1 KiB
Python
"""
|
|
Lab 3.9: Build a Verifier Agent -- SOLUTION
|
|
"""
|
|
|
|
import json
|
|
import os
|
|
|
|
VERIFIER_TOOLS = [
|
|
{
|
|
"name": "read_file",
|
|
"description": "Read the contents of a file",
|
|
"input_schema": {
|
|
"type": "object",
|
|
"properties": {
|
|
"path": {"type": "string"},
|
|
"reasoning": {"type": "string"}
|
|
},
|
|
"required": ["path", "reasoning"]
|
|
}
|
|
},
|
|
{
|
|
"name": "grep_search",
|
|
"description": "Search for a pattern in files",
|
|
"input_schema": {
|
|
"type": "object",
|
|
"properties": {
|
|
"pattern": {"type": "string"},
|
|
"path": {"type": "string"},
|
|
"reasoning": {"type": "string"}
|
|
},
|
|
"required": ["pattern", "path", "reasoning"]
|
|
}
|
|
},
|
|
{
|
|
"name": "list_files",
|
|
"description": "List files in a directory",
|
|
"input_schema": {
|
|
"type": "object",
|
|
"properties": {
|
|
"path": {"type": "string"},
|
|
"reasoning": {"type": "string"}
|
|
},
|
|
"required": ["path", "reasoning"]
|
|
}
|
|
}
|
|
]
|
|
|
|
|
|
def verify_work(claims: list[dict], workspace_path: str) -> dict:
|
|
verified = []
|
|
failed = []
|
|
unverifiable = []
|
|
|
|
for claim in claims:
|
|
check_type = claim.get("check_type", "file_exists")
|
|
evidence = claim["expected_evidence"]
|
|
|
|
try:
|
|
if check_type == "file_exists":
|
|
target = os.path.join(workspace_path, evidence)
|
|
if os.path.exists(target):
|
|
verified.append(claim)
|
|
else:
|
|
failed.append({**claim, "reason": f"File not found: {target}"})
|
|
|
|
elif check_type == "file_contains":
|
|
target = os.path.join(workspace_path, evidence)
|
|
contains = claim.get("contains", "")
|
|
if os.path.exists(target):
|
|
with open(target) as f:
|
|
content = f.read()
|
|
if contains in content:
|
|
verified.append(claim)
|
|
else:
|
|
failed.append({**claim, "reason": f"'{contains}' not found in {target}"})
|
|
else:
|
|
failed.append({**claim, "reason": f"File not found: {target}"})
|
|
|
|
elif check_type == "file_not_exists":
|
|
target = os.path.join(workspace_path, evidence)
|
|
if not os.path.exists(target):
|
|
verified.append(claim)
|
|
else:
|
|
failed.append({**claim, "reason": f"File should not exist: {target}"})
|
|
|
|
else:
|
|
unverifiable.append({**claim, "reason": f"Unknown check type: {check_type}"})
|
|
|
|
except Exception as e:
|
|
failed.append({**claim, "reason": str(e)})
|
|
|
|
# Determine confidence level
|
|
total = len(verified) + len(failed) + len(unverifiable)
|
|
if total == 0:
|
|
status = "FAILED"
|
|
elif len(failed) == 0 and len(unverifiable) == 0:
|
|
status = "PERFECT"
|
|
elif len(failed) == 0 and len(unverifiable) > 0:
|
|
status = "PARTIAL"
|
|
elif len(failed) > 0 and len(failed) < total:
|
|
status = "FEEDBACK"
|
|
else:
|
|
status = "FAILED"
|
|
|
|
return {
|
|
"status": status,
|
|
"verified_claims": [c["claim"] for c in verified],
|
|
"failed_claims": [{"claim": c["claim"], "reason": c["reason"]} for c in failed],
|
|
"unverifiable_claims": [{"claim": c["claim"], "reason": c["reason"]} for c in unverifiable],
|
|
"summary": f"{len(verified)}/{total} claims verified"
|
|
}
|
|
|
|
|
|
if __name__ == "__main__":
|
|
test_claims = [
|
|
{"claim": "Created the data processing script",
|
|
"expected_evidence": "process.py",
|
|
"check_type": "file_exists"},
|
|
{"claim": "Script reads CSV files",
|
|
"expected_evidence": "process.py",
|
|
"contains": "read_csv",
|
|
"check_type": "file_contains"},
|
|
]
|
|
|
|
result = verify_work(test_claims, "./")
|
|
print(json.dumps(result, indent=2))
|