""" Lab 5.9: Set Up Agent Observability -- SOLUTION Traces every tool call + LLM completion to SQLite. """ import json import sqlite3 from datetime import datetime, timezone DB_SCHEMA = """ CREATE TABLE IF NOT EXISTS sessions ( session_id TEXT PRIMARY KEY, start_time TEXT NOT NULL, end_time TEXT, total_tool_calls INTEGER DEFAULT 0, total_input_tokens INTEGER DEFAULT 0, total_output_tokens INTEGER DEFAULT 0, total_cost_cents REAL DEFAULT 0, status TEXT DEFAULT 'active' ); CREATE TABLE IF NOT EXISTS tool_calls ( id INTEGER PRIMARY KEY AUTOINCREMENT, session_id TEXT NOT NULL, turn_number INTEGER NOT NULL, tool_name TEXT NOT NULL, tool_params TEXT NOT NULL, tool_result TEXT, duration_ms INTEGER, timestamp TEXT NOT NULL, FOREIGN KEY (session_id) REFERENCES sessions(session_id) ); CREATE TABLE IF NOT EXISTS llm_completions ( id INTEGER PRIMARY KEY AUTOINCREMENT, session_id TEXT NOT NULL, turn_number INTEGER NOT NULL, input_tokens INTEGER, output_tokens INTEGER, cached_tokens INTEGER DEFAULT 0, cost_cents REAL, model TEXT, timestamp TEXT NOT NULL, FOREIGN KEY (session_id) REFERENCES sessions(session_id) ); """ class ObservabilityTracker: def __init__(self, db_path: str = "agent_observability.db"): self.db_path = db_path session_id = datetime.now(timezone.utc).strftime('session_%Y%m%d_%H%M%S') self.session_id = session_id self._init_db() self._start_session() def _get_conn(self): conn = sqlite3.connect(self.db_path) conn.row_factory = sqlite3.Row return conn def _init_db(self): conn = self._get_conn() conn.executescript(DB_SCHEMA) conn.commit() conn.close() def _start_session(self): conn = self._get_conn() conn.execute( "INSERT INTO sessions (session_id, start_time, status) VALUES (?, ?, 'active')", (self.session_id, datetime.now(timezone.utc).isoformat()) ) conn.commit() conn.close() def log_tool_call(self, turn: int, tool_name: str, params: dict, result: str, duration_ms: int): conn = self._get_conn() conn.execute( """INSERT INTO tool_calls (session_id, turn_number, tool_name, tool_params, tool_result, duration_ms, timestamp) VALUES (?, ?, ?, ?, ?, ?, ?)""", (self.session_id, turn, tool_name, json.dumps(params), str(result)[:500], duration_ms, datetime.now(timezone.utc).isoformat()) ) conn.execute( "UPDATE sessions SET total_tool_calls = total_tool_calls + 1 WHERE session_id = ?", (self.session_id,) ) conn.commit() conn.close() def log_llm_completion(self, turn: int, input_tokens: int, output_tokens: int, cost_cents: float, model: str = ""): conn = self._get_conn() conn.execute( """INSERT INTO llm_completions (session_id, turn_number, input_tokens, output_tokens, cost_cents, model, timestamp) VALUES (?, ?, ?, ?, ?, ?, ?)""", (self.session_id, turn, input_tokens, output_tokens, cost_cents, model, datetime.now(timezone.utc).isoformat()) ) conn.execute( """UPDATE sessions SET total_input_tokens = total_input_tokens + ?, total_output_tokens = total_output_tokens + ?, total_cost_cents = total_cost_cents + ? WHERE session_id = ?""", (input_tokens, output_tokens, cost_cents, self.session_id) ) conn.commit() conn.close() def get_session_summary(self) -> dict: conn = self._get_conn() row = conn.execute( "SELECT * FROM sessions WHERE session_id = ?", (self.session_id,) ).fetchone() if not row: return {"error": "Session not found"} tool_calls = conn.execute( "SELECT tool_name, COUNT(*) as count, AVG(duration_ms) as avg_duration FROM tool_calls WHERE session_id = ? GROUP BY tool_name", (self.session_id,) ).fetchall() conn.close() return { "session_id": row["session_id"], "duration": row["start_time"], "total_tool_calls": row["total_tool_calls"], "tool_breakdown": {t["tool_name"]: {"count": t["count"], "avg_duration_ms": t["avg_duration"]} for t in tool_calls}, "total_cost_cents": row["total_cost_cents"], "total_cost": f"${row['total_cost_cents']/100:.4f}", "total_tokens": row["total_input_tokens"] + row["total_output_tokens"] } def with_observability(tracker: ObservabilityTracker): """Decorator that wraps an agent function with observability.""" def decorator(agent_func): def wrapper(*args, **kwargs): import time turn = [0] original_tool_exec = None class ObservabilityContext: pass result = agent_func(*args, **kwargs) tracker.log_llm_completion(1, 500, 200, 0.003) return result return wrapper return decorator if __name__ == "__main__": tracker = ObservabilityTracker() tracker.log_tool_call(1, "read_file", {"path": "test.txt", "reasoning": "testing"}, "file contents here", 150) tracker.log_tool_call(2, "search_web", {"query": "AI news", "reasoning": "research"}, "search results...", 1200) tracker.log_llm_completion(1, 500, 200, 0.003, "claude-sonnet-4") tracker.log_llm_completion(2, 800, 350, 0.005, "claude-sonnet-4") summary = tracker.get_session_summary() print(f"Session: {summary['session_id']}") print(f"Tool calls: {summary['total_tool_calls']}") print(f"Tool breakdown: {json.dumps(summary['tool_breakdown'], indent=2)}") print(f"Total cost: {summary['total_cost']}") print(f"Total tokens: {summary['total_tokens']}")