135 lines
4.1 KiB
Python
135 lines
4.1 KiB
Python
"""
|
|
Lab Auto-Grader: validates student lab outputs against expected results.
|
|
|
|
Usage:
|
|
python grade.py # Grade all labs
|
|
python grade.py L1-first-agent # Grade specific lab
|
|
"""
|
|
|
|
import os, sys, subprocess
|
|
from pathlib import Path
|
|
|
|
LABS_DIR = Path(__file__).resolve().parent
|
|
sys.path.insert(0, str(LABS_DIR / '_shared'))
|
|
|
|
LAB_TESTS = {
|
|
'L1-first-agent': {
|
|
'file': 'solution.py',
|
|
'args': [str(LABS_DIR / 'L1-first-agent' / 'test.txt'), 'what is in this file'],
|
|
'assert_output_contains': ['Final answer'],
|
|
},
|
|
'L2-multi-tool': {
|
|
'file': 'solution.py',
|
|
'args': ['test query'],
|
|
'assert_output_contains': [],
|
|
'allow_exit_code': [0, 1],
|
|
},
|
|
'L2-context': {
|
|
'file': 'solution.py',
|
|
'args': [],
|
|
'assert_output_contains': ['messages'],
|
|
},
|
|
'L3-whitelist-hook': {
|
|
'file': 'solution.py',
|
|
'args': [],
|
|
'assert_output_contains': ['ALLOW', 'BLOCK'],
|
|
},
|
|
'L3-verifier': {
|
|
'file': 'solution.py',
|
|
'args': [],
|
|
'assert_output_contains': ['status'],
|
|
},
|
|
'L5-cicd': {
|
|
'file': 'solution.py',
|
|
'args': [],
|
|
'assert_output_contains': ['PIPELINE'],
|
|
'allow_exit_code': [0, 1], # Pipeline exits 1 when gate fails (expected)
|
|
},
|
|
'L5-observability': {
|
|
'file': 'solution.py',
|
|
'args': [],
|
|
'assert_output_contains': ['Session'],
|
|
},
|
|
'L6-cost-optimization': {
|
|
'file': 'solution.py',
|
|
'args': [],
|
|
'assert_output_contains': ['OPTIMIZATION'],
|
|
'allow_exit_code': [0, 1],
|
|
},
|
|
'L6-eval-harness': {
|
|
'file': 'solution.py',
|
|
'args': [],
|
|
'assert_output_contains': ['HARNESS'],
|
|
'allow_exit_code': [0, 1],
|
|
},
|
|
'L7-autoresearch': {
|
|
'file': 'solution.py',
|
|
'args': [],
|
|
'assert_output_contains': ['Baseline'],
|
|
'allow_exit_code': [0, 1],
|
|
},
|
|
'L7-meta-agent': {
|
|
'file': 'solution.py',
|
|
'args': [],
|
|
'assert_output_contains': ['Building'],
|
|
'allow_exit_code': [0, 1],
|
|
},
|
|
}
|
|
|
|
|
|
def grade_lab(lab_name: str) -> dict:
|
|
config = LAB_TESTS.get(lab_name)
|
|
if not config:
|
|
return {'lab': lab_name, 'status': 'SKIP', 'reason': 'No test'}
|
|
|
|
lab_dir = LABS_DIR / lab_name
|
|
script = lab_dir / config['file']
|
|
if not script.exists():
|
|
return {'lab': lab_name, 'status': 'FAIL', 'reason': f'{config["file"]} not found'}
|
|
|
|
cmd = [sys.executable, str(script)] + config.get('args', [])
|
|
try:
|
|
result = subprocess.run(cmd, capture_output=True, text=True, timeout=config.get('timeout', 15), cwd=str(lab_dir))
|
|
except subprocess.TimeoutExpired:
|
|
return {'lab': lab_name, 'status': 'FAIL', 'reason': 'Timed out'}
|
|
|
|
output = result.stdout + result.stderr
|
|
allowed = config.get('allow_exit_code', [0])
|
|
if result.returncode not in allowed:
|
|
err = result.stderr[:200] if result.stderr else f'Exit code {result.returncode}'
|
|
return {'lab': lab_name, 'status': 'FAIL', 'reason': err}
|
|
|
|
errors = []
|
|
for pattern in config.get('assert_output_contains', []):
|
|
if pattern.lower() not in output.lower():
|
|
errors.append(f'Missing: "{pattern}"')
|
|
if errors:
|
|
return {'lab': lab_name, 'status': 'FAIL', 'reason': '; '.join(errors)}
|
|
|
|
return {'lab': lab_name, 'status': 'PASS'}
|
|
|
|
|
|
def grade_all():
|
|
results = []
|
|
for lab_name in sorted(LAB_TESTS.keys()):
|
|
result = grade_lab(lab_name)
|
|
results.append(result)
|
|
icon = '+' if result['status'] == 'PASS' else '-'
|
|
line = f' [{icon}] {lab_name:25s} {result["status"]}'
|
|
if 'reason' in result:
|
|
line += f' - {result["reason"][:100]}'
|
|
print(line)
|
|
|
|
passed = sum(1 for r in results if r['status'] == 'PASS')
|
|
total = len(results)
|
|
print(f'\n {passed}/{total} passing')
|
|
return results
|
|
|
|
|
|
if __name__ == '__main__':
|
|
if len(sys.argv) > 1:
|
|
result = grade_lab(sys.argv[1])
|
|
print(f'[{result["status"]}] {result["lab"]}' + (f' - {result.get("reason","")}' if 'reason' in result else ''))
|
|
else:
|
|
grade_all()
|