62 lines
2.0 KiB
Python
62 lines
2.0 KiB
Python
"""
|
|
Lab 3.9: Build a Verifier Agent
|
|
|
|
Objective: Create a read-only agent that checks the builder's work.
|
|
|
|
The verifier should:
|
|
1. Read the builder's work (files changed)
|
|
2. Verify factual claims (grep/search for evidence)
|
|
3. Report a confidence level (PERFECT, VERIFIED, PARTIAL, FEEDBACK, FAILED)
|
|
4. Have NO write/edit/bash tools
|
|
"""
|
|
|
|
# TODO: Define the verifier's tool surface
|
|
# The verifier should ONLY have read-only tools:
|
|
# - read_file: read files
|
|
# - grep_search: search for patterns in files
|
|
# - list_files: list files in a directory
|
|
# NO write, edit, bash, or destructive tools
|
|
|
|
VERIFIER_TOOLS = [
|
|
# TODO: Define a read_file tool (name, description, input_schema)
|
|
# TODO: Define a grep_search tool
|
|
# TODO: Define a list_files tool
|
|
]
|
|
|
|
|
|
# TODO: Implement the verification logic
|
|
def verify_work(claims: list[dict], workspace_path: str) -> dict:
|
|
"""
|
|
Verify a set of claims against the actual workspace.
|
|
|
|
Each claim has:
|
|
- claim: str (what the builder claims)
|
|
- expected_evidence: str (what to look for)
|
|
- check_type: str ("file_exists", "file_contains", "file_not_exists")
|
|
|
|
Returns a verification report with:
|
|
- status: PERFECT, VERIFIED, PARTIAL, FEEDBACK, FAILED
|
|
- verified_claims: list of passed claims
|
|
- failed_claims: list of failed claims with reasons
|
|
- unverifiable_claims: list of claims that couldn't be checked
|
|
"""
|
|
pass # TODO
|
|
|
|
|
|
if __name__ == "__main__":
|
|
# Test case
|
|
test_claims = [
|
|
{"claim": "Created the data processing script",
|
|
"expected_evidence": "process.py",
|
|
"check_type": "file_exists"},
|
|
{"claim": "Script reads CSV files",
|
|
"expected_evidence": "read_csv",
|
|
"check_type": "file_contains"},
|
|
{"claim": "Output is written to results/ directory",
|
|
"expected_evidence": "results",
|
|
"check_type": "file_contains"},
|
|
]
|
|
|
|
result = verify_work(test_claims, "./")
|
|
print(json.dumps(result, indent=2))
|