32 lines
26 KiB
JavaScript
32 lines
26 KiB
JavaScript
import{c as s,Q as a,j as e,m as i}from"./chunks/framework.BPKcPtvA.js";const k=JSON.parse('{"title":"Module 6: Economics & Evaluation","description":"","frontmatter":{},"headers":[],"relativePath":"modules/m6-economics.md","filePath":"modules/m6-economics.md","lastUpdated":1780488246000}'),n={name:"modules/m6-economics.md"};function l(o,t,r,h,d,p){return a(),e("div",null,[...t[0]||(t[0]=[i(`<h1 id="module-6-economics-evaluation" tabindex="-1">Module 6: Economics & Evaluation <a class="header-anchor" href="#module-6-economics-evaluation" aria-label="Permalink to "Module 6: Economics & Evaluation""></a></h1><h2 id="lesson-6-0-the-compute-advantage-equation" tabindex="-1">Lesson 6.0: The Compute Advantage Equation <a class="header-anchor" href="#lesson-6-0-the-compute-advantage-equation" aria-label="Permalink to "Lesson 6.0: The Compute Advantage Equation""></a></h2><p><strong>Core insight</strong>: Your value as an agentic engineer scales with the amount of compute you can harness effectively.</p><h3 id="the-equation" tabindex="-1">The Equation <a class="header-anchor" href="#the-equation" aria-label="Permalink to "The Equation""></a></h3><div class="language- vp-adaptive-theme"><button title="Copy Code" class="copy"></button><span class="lang"></span><pre class="shiki shiki-themes github-light github-dark vp-code" tabindex="0"><code><span class="line"><span>Compute Advantage = (Compute Scaling × Autonomy) ÷ (Time + Effort + Monetary Cost)</span></span></code></pre></div><table tabindex="0"><thead><tr><th>Variable</th><th>What It Means</th><th>How to Improve It</th></tr></thead><tbody><tr><td><strong>Compute Scaling</strong></td><td>How much AI compute you can throw at problems</td><td>More agents, better models, larger context windows</td></tr><tr><td><strong>Autonomy</strong></td><td>How much the agent does without your intervention</td><td>Better prompts, better tools, verification layers</td></tr><tr><td><strong>Time</strong></td><td>How long it takes to get results</td><td>Faster models, parallel execution, fewer iterations</td></tr><tr><td><strong>Effort</strong></td><td>How much you have to craft prompts/instructions</td><td>Reusable skills, templates, mental models</td></tr><tr><td><strong>Monetary Cost</strong></td><td>What you pay for API calls</td><td>Cascade routing, cheaper models for simple steps</td></tr></tbody></table><h3 id="what-it-tells-you" tabindex="-1">What It Tells You <a class="header-anchor" href="#what-it-tells-you" aria-label="Permalink to "What It Tells You""></a></h3><ul><li><strong>High Compute Advantage</strong> = You get more output for less input. You're leveraging agents effectively.</li><li><strong>Low Compute Advantage</strong> = You're spending too much time/cost for too little gain. Fix the bottleneck variable.</li></ul><h3 id="real-world-application-your-stack" tabindex="-1">Real-World Application (Your Stack) <a class="header-anchor" href="#real-world-application-your-stack" aria-label="Permalink to "Real-World Application (Your Stack)""></a></h3><table tabindex="0"><thead><tr><th>Tool</th><th style="text-align:center;">Compute Scaling</th><th style="text-align:center;">Autonomy</th><th style="text-align:center;">Time</th><th style="text-align:center;">Effort</th><th style="text-align:center;">Cost</th><th style="text-align:center;">Advantage</th></tr></thead><tbody><tr><td>Claude Code (lead)</td><td style="text-align:center;">8</td><td style="text-align:center;">7</td><td style="text-align:center;">6</td><td style="text-align:center;">5</td><td style="text-align:center;">4</td><td style="text-align:center;">(8×7)÷(6+5+4)=3.7</td></tr><tr><td>Pi Agent (custom)</td><td style="text-align:center;">7</td><td style="text-align:center;">8</td><td style="text-align:center;">7</td><td style="text-align:center;">6</td><td style="text-align:center;">6</td><td style="text-align:center;">(7×8)÷(7+6+6)=2.9</td></tr><tr><td>OpenCode (OSS)</td><td style="text-align:center;">5</td><td style="text-align:center;">5</td><td style="text-align:center;">7</td><td style="text-align:center;">7</td><td style="text-align:center;">9</td><td style="text-align:center;">(5×5)÷(7+7+9)=1.1</td></tr><tr><td>Gemini Flash (fast)</td><td style="text-align:center;">4</td><td style="text-align:center;">3</td><td style="text-align:center;">9</td><td style="text-align:center;">8</td><td style="text-align:center;">9</td><td style="text-align:center;">(4×3)÷(9+8+9)=0.5</td></tr></tbody></table><p>Higher score = more output per unit of investment. Use this to decide which tool for which task.</p><h3 id="how-to-optimize" tabindex="-1">How to Optimize <a class="header-anchor" href="#how-to-optimize" aria-label="Permalink to "How to Optimize""></a></h3><ol><li><strong>Improve numerator</strong>: Run more agents in parallel (P-threads), increase autonomy with verification</li><li><strong>Reduce denominator</strong>: Use cascade routing (cheap model for simple steps), reuse skills/mental models</li><li><strong>Track over time</strong>: Your Compute Advantage should increase as you build better harnesses and mental models</li></ol><hr><h2 id="lesson-6-1-llm-pricing-landscape-2026" tabindex="-1">Lesson 6.1: LLM Pricing Landscape 2026 <a class="header-anchor" href="#lesson-6-1-llm-pricing-landscape-2026" aria-label="Permalink to "Lesson 6.1: LLM Pricing Landscape 2026""></a></h2><h3 id="per-million-token-pricing" tabindex="-1">Per-Million-Token Pricing <a class="header-anchor" href="#per-million-token-pricing" aria-label="Permalink to "Per-Million-Token Pricing""></a></h3><table tabindex="0"><thead><tr><th>Model</th><th>Input ($/M)</th><th>Output ($/M)</th><th>Best For</th></tr></thead><tbody><tr><td>Gemini 2.5 Flash</td><td>$0.15</td><td>$0.60</td><td>High-volume, simple tool calls</td></tr><tr><td>DeepSeek V3</td><td>$0.27</td><td>$1.10</td><td>Structured tasks, batch processing</td></tr><tr><td>DeepSeek R1</td><td>$0.55</td><td>$2.19</td><td>Reasoning-heavy single steps</td></tr><tr><td>Gemini 2.5 Pro</td><td>$1.25</td><td>$5.00</td><td>Long-context sessions (1M+ tokens)</td></tr><tr><td>GPT-4o</td><td>$2.50</td><td>$10.00</td><td>Balanced cost/quality</td></tr><tr><td>Claude Sonnet 4</td><td>$3.00</td><td>$15.00</td><td>General agentic reasoning</td></tr><tr><td>GPT-5</td><td>$10.00</td><td>$40.00</td><td>Frontier research, complex plans</td></tr><tr><td>Claude Opus 4</td><td>$15.00</td><td>$75.00</td><td>Complex multi-step agent orchestration</td></tr></tbody></table><h3 id="price-range-100x-difference" tabindex="-1">Price Range: 100x Difference <a class="header-anchor" href="#price-range-100x-difference" aria-label="Permalink to "Price Range: 100x Difference""></a></h3><p>Cheapest (Gemini Flash) to most expensive (Claude Opus) is a <strong>100x multiplier</strong>. Choosing the right model for each step is your highest-leverage cost optimization.</p><hr><h2 id="lesson-6-2-cascade-routing" tabindex="-1">Lesson 6.2: Cascade Routing <a class="header-anchor" href="#lesson-6-2-cascade-routing" aria-label="Permalink to "Lesson 6.2: Cascade Routing""></a></h2><h3 id="the-pattern" tabindex="-1">The Pattern <a class="header-anchor" href="#the-pattern" aria-label="Permalink to "The Pattern""></a></h3><p>Don't use one model for everything. Route different steps to different models:</p><div class="language- vp-adaptive-theme"><button title="Copy Code" class="copy"></button><span class="lang"></span><pre class="shiki shiki-themes github-light github-dark vp-code" tabindex="0"><code><span class="line"><span>Step 1: Retrieve context → Gemini Flash ($0.15/M input)</span></span>
|
||
<span class="line"><span>Step 2: Analyze → Claude Sonnet ($3/$15)</span></span>
|
||
<span class="line"><span>Step 3: Make decision → Claude Opus ($15/$75)</span></span>
|
||
<span class="line"><span>Step 4: Format output → Gemini Flash ($0.15/$0.60)</span></span></code></pre></div><h3 id="savings-profile" tabindex="-1">Savings Profile <a class="header-anchor" href="#savings-profile" aria-label="Permalink to "Savings Profile""></a></h3><table tabindex="0"><thead><tr><th>Pattern</th><th>Cost/Task</th><th>Savings</th></tr></thead><tbody><tr><td>All Opus</td><td>$2.50</td><td>Baseline</td></tr><tr><td>Cascade (Flash → Sonnet → Opus → Flash)</td><td>$0.85</td><td>66% savings</td></tr><tr><td>All Sonnet</td><td>$0.50</td><td>80% savings (but quality loss on complex steps)</td></tr></tbody></table><h3 id="implementation" tabindex="-1">Implementation <a class="header-anchor" href="#implementation" aria-label="Permalink to "Implementation""></a></h3><div class="language-python vp-adaptive-theme"><button title="Copy Code" class="copy"></button><span class="lang">python</span><pre class="shiki shiki-themes github-light github-dark vp-code" tabindex="0"><code><span class="line"><span style="--shiki-light:#D73A49;--shiki-dark:#F97583;">def</span><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0;"> route_task</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">(task_complexity: </span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;">str</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">) -> </span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;">str</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">:</span></span>
|
||
<span class="line"><span style="--shiki-light:#D73A49;--shiki-dark:#F97583;"> if</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;"> task_complexity </span><span style="--shiki-light:#D73A49;--shiki-dark:#F97583;">==</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> "retrieval"</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">:</span></span>
|
||
<span class="line"><span style="--shiki-light:#D73A49;--shiki-dark:#F97583;"> return</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> "gemini-2.5-flash"</span></span>
|
||
<span class="line"><span style="--shiki-light:#D73A49;--shiki-dark:#F97583;"> elif</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;"> task_complexity </span><span style="--shiki-light:#D73A49;--shiki-dark:#F97583;">==</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> "analysis"</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">:</span></span>
|
||
<span class="line"><span style="--shiki-light:#D73A49;--shiki-dark:#F97583;"> return</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> "claude-sonnet-4"</span></span>
|
||
<span class="line"><span style="--shiki-light:#D73A49;--shiki-dark:#F97583;"> elif</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;"> task_complexity </span><span style="--shiki-light:#D73A49;--shiki-dark:#F97583;">==</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> "decision"</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">:</span></span>
|
||
<span class="line"><span style="--shiki-light:#D73A49;--shiki-dark:#F97583;"> return</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> "claude-opus-4"</span></span>
|
||
<span class="line"><span style="--shiki-light:#D73A49;--shiki-dark:#F97583;"> elif</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;"> task_complexity </span><span style="--shiki-light:#D73A49;--shiki-dark:#F97583;">==</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> "formatting"</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">:</span></span>
|
||
<span class="line"><span style="--shiki-light:#D73A49;--shiki-dark:#F97583;"> return</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> "gemini-2.5-flash"</span></span></code></pre></div><hr><h2 id="lesson-6-3-cost-per-session-math" tabindex="-1">Lesson 6.3: Cost Per Session Math <a class="header-anchor" href="#lesson-6-3-cost-per-session-math" aria-label="Permalink to "Lesson 6.3: Cost Per Session Math""></a></h2><h3 id="where-costs-come-from" tabindex="-1">Where Costs Come From <a class="header-anchor" href="#where-costs-come-from" aria-label="Permalink to "Where Costs Come From""></a></h3><table tabindex="0"><thead><tr><th>Component</th><th>Share</th><th>Notes</th></tr></thead><tbody><tr><td>Output tokens</td><td>~70%</td><td>Model generation is most expensive</td></tr><tr><td>Input tokens</td><td>~20%</td><td>Context + tool results</td></tr><tr><td>Cached tokens</td><td>~10%</td><td>Can be zero if not configured</td></tr></tbody></table><h3 id="the-multipliers" tabindex="-1">The Multipliers <a class="header-anchor" href="#the-multipliers" aria-label="Permalink to "The Multipliers""></a></h3><table tabindex="0"><thead><tr><th>Factor</th><th>Multiplier</th><th>Why</th></tr></thead><tbody><tr><td>Retry rate</td><td>1.2-2.0x</td><td>Failed tool calls retry</td></tr><tr><td>Tool overhead</td><td>3-5x per tool call</td><td>Planning + execution + error recovery + result parsing</td></tr><tr><td>Context growth</td><td>1.5x per 10 turns</td><td>Every turn adds tokens to context</td></tr></tbody></table><h3 id="the-3x-rule" tabindex="-1">The 3x Rule <a class="header-anchor" href="#the-3x-rule" aria-label="Permalink to "The 3x Rule""></a></h3><p><strong>Production agent costs 3x your prototype estimate.</strong></p><p>1x = ideal path (everything works first time) 2x = retries + edge cases 3x = monitoring + error handling + observability overhead</p><h3 id="quick-estimation" tabindex="-1">Quick Estimation <a class="header-anchor" href="#quick-estimation" aria-label="Permalink to "Quick Estimation""></a></h3><div class="language-python vp-adaptive-theme"><button title="Copy Code" class="copy"></button><span class="lang">python</span><pre class="shiki shiki-themes github-light github-dark vp-code" tabindex="0"><code><span class="line"><span style="--shiki-light:#D73A49;--shiki-dark:#F97583;">def</span><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0;"> estimate_cost</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">(turns: </span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;">int</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">, avg_tokens_per_turn: </span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;">int</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">, model_price_per_m: </span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;">float</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">):</span></span>
|
||
<span class="line"><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> """Very rough estimate."""</span></span>
|
||
<span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;"> base </span><span style="--shiki-light:#D73A49;--shiki-dark:#F97583;">=</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;"> turns </span><span style="--shiki-light:#D73A49;--shiki-dark:#F97583;">*</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;"> avg_tokens_per_turn </span><span style="--shiki-light:#D73A49;--shiki-dark:#F97583;">*</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;"> model_price_per_m </span><span style="--shiki-light:#D73A49;--shiki-dark:#F97583;">/</span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;"> 1_000_000</span></span>
|
||
<span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;"> retry </span><span style="--shiki-light:#D73A49;--shiki-dark:#F97583;">=</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;"> base </span><span style="--shiki-light:#D73A49;--shiki-dark:#F97583;">*</span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;"> 1.5</span></span>
|
||
<span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;"> overhead </span><span style="--shiki-light:#D73A49;--shiki-dark:#F97583;">=</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;"> base </span><span style="--shiki-light:#D73A49;--shiki-dark:#F97583;">*</span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;"> 3.0</span></span>
|
||
<span class="line"><span style="--shiki-light:#D73A49;--shiki-dark:#F97583;"> return</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;"> {</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;">"base"</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">: base, </span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;">"with_retries"</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">: retry, </span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;">"production"</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">: overhead}</span></span></code></pre></div><hr><h2 id="lesson-6-4-agent-evaluation-metrics" tabindex="-1">Lesson 6.4: Agent Evaluation Metrics <a class="header-anchor" href="#lesson-6-4-agent-evaluation-metrics" aria-label="Permalink to "Lesson 6.4: Agent Evaluation Metrics""></a></h2><h3 id="the-key-metrics" tabindex="-1">The Key Metrics <a class="header-anchor" href="#the-key-metrics" aria-label="Permalink to "The Key Metrics""></a></h3><table tabindex="0"><thead><tr><th>Metric</th><th>What It Measures</th><th>Target</th></tr></thead><tbody><tr><td><strong>pass@k</strong></td><td>% of k attempts where at least one succeeds</td><td>>80%</td></tr><tr><td><strong>pass^k</strong></td><td>% where ALL k attempts succeed (consistency)</td><td>>60%</td></tr><tr><td><strong>Tool Call Accuracy</strong></td><td>Correct tool + correct params</td><td>>90%</td></tr><tr><td><strong>Task Completion Rate</strong></td><td>End-to-end success</td><td>>70%</td></tr><tr><td><strong>Cost Per Task</strong></td><td>Total API cost per unit</td><td>Varies</td></tr><tr><td><strong>Loop Efficiency</strong></td><td>Steps taken vs optimal</td><td><2x overhead</td></tr><tr><td><strong>Grind Rate</strong></td><td>% of attempts that re-run identical code</td><td><5%</td></tr></tbody></table><h3 id="pass-k-explained" tabindex="-1">pass@k Explained <a class="header-anchor" href="#pass-k-explained" aria-label="Permalink to "pass@k Explained""></a></h3><div class="language- vp-adaptive-theme"><button title="Copy Code" class="copy"></button><span class="lang"></span><pre class="shiki shiki-themes github-light github-dark vp-code" tabindex="0"><code><span class="line"><span>pass@k = probability that at least one of k attempts succeeds</span></span>
|
||
<span class="line"><span></span></span>
|
||
<span class="line"><span>k=1: 60% pass rate (single attempt)</span></span>
|
||
<span class="line"><span>k=3: 1 - (0.4)^3 = 93.6% (best of 3)</span></span>
|
||
<span class="line"><span>k=5: 1 - (0.4)^5 = 98.9% (best of 5)</span></span></code></pre></div><p>Higher k = higher reliability but higher cost. The trade-off is the core optimization problem.</p><hr><h2 id="lesson-6-5-automated-evaluation" tabindex="-1">Lesson 6.5: Automated Evaluation <a class="header-anchor" href="#lesson-6-5-automated-evaluation" aria-label="Permalink to "Lesson 6.5: Automated Evaluation""></a></h2><h3 id="golden-dataset-approach" tabindex="-1">Golden Dataset Approach <a class="header-anchor" href="#golden-dataset-approach" aria-label="Permalink to "Golden Dataset Approach""></a></h3><div class="language-python vp-adaptive-theme"><button title="Copy Code" class="copy"></button><span class="lang">python</span><pre class="shiki shiki-themes github-light github-dark vp-code" tabindex="0"><code><span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">eval_cases </span><span style="--shiki-light:#D73A49;--shiki-dark:#F97583;">=</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;"> [</span></span>
|
||
<span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;"> {</span></span>
|
||
<span class="line"><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> "input"</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">: </span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;">"Find the user with email john@example.com"</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">,</span></span>
|
||
<span class="line"><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> "expected_tool"</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">: </span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;">"query_database"</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">,</span></span>
|
||
<span class="line"><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> "expected_params"</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">: {</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;">"query"</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">: </span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;">"SELECT * FROM users WHERE email = 'john@example.com'"</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">},</span></span>
|
||
<span class="line"><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> "expected_output_contains"</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">: [</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;">"john@example.com"</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">],</span></span>
|
||
<span class="line"><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> "weight"</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">: </span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;">1.0</span></span>
|
||
<span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;"> }</span></span>
|
||
<span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">]</span></span></code></pre></div><h3 id="vcr-style-recording" tabindex="-1">VCR-Style Recording <a class="header-anchor" href="#vcr-style-recording" aria-label="Permalink to "VCR-Style Recording""></a></h3><p>For non-deterministic tool results (search, API calls), record the response once, then replay it deterministically:</p><div class="language-python vp-adaptive-theme"><button title="Copy Code" class="copy"></button><span class="lang">python</span><pre class="shiki shiki-themes github-light github-dark vp-code" tabindex="0"><code><span class="line"><span style="--shiki-light:#6A737D;--shiki-dark:#6A737D;"># Record mode: capture real responses</span></span>
|
||
<span class="line"><span style="--shiki-light:#6A737D;--shiki-dark:#6A737D;"># Replay mode: use recorded responses</span></span>
|
||
<span class="line"><span style="--shiki-light:#6A737D;--shiki-dark:#6A737D;"># Test: verify agent makes correct decisions with known data</span></span></code></pre></div><hr><h2 id="lesson-6-6-a-b-testing-agents" tabindex="-1">Lesson 6.6: A/B Testing Agents <a class="header-anchor" href="#lesson-6-6-a-b-testing-agents" aria-label="Permalink to "Lesson 6.6: A/B Testing Agents""></a></h2><h3 id="canary-deployment" tabindex="-1">Canary Deployment <a class="header-anchor" href="#canary-deployment" aria-label="Permalink to "Canary Deployment""></a></h3><ol><li>Route 5% of traffic to new agent config</li><li>Compare against 95% on current config</li><li>Metrics: success rate, cost, latency, loop depth</li><li>If new config wins on all metrics → roll out to 100%</li><li>If new config loses → rollback, investigate</li></ol><h3 id="what-to-a-b-test" tabindex="-1">What to A/B Test <a class="header-anchor" href="#what-to-a-b-test" aria-label="Permalink to "What to A/B Test""></a></h3><ul><li>System prompt wording</li><li>Model selection</li><li>Temperature settings</li><li>Tool descriptions</li><li>Iteration limits</li></ul><hr><h2 id="lesson-6-7-human-evaluation" tabindex="-1">Lesson 6.7: Human Evaluation <a class="header-anchor" href="#lesson-6-7-human-evaluation" aria-label="Permalink to "Lesson 6.7: Human Evaluation""></a></h2><h3 id="what-automated-evals-miss" tabindex="-1">What Automated Evals Miss <a class="header-anchor" href="#what-automated-evals-miss" aria-label="Permalink to "What Automated Evals Miss""></a></h3><ol><li><strong>Quality of reasoning</strong> — agent made right decision for wrong reasons?</li><li><strong>Tone and style</strong> — output technically correct but poorly written?</li><li><strong>Edge cases</strong> — agent handled happy path but not real-world variation?</li><li><strong>Hallucination cascades</strong> — plausible-looking but wrong intermediate steps?</li></ol><h3 id="spot-check-sampling" tabindex="-1">Spot-Check Sampling <a class="header-anchor" href="#spot-check-sampling" aria-label="Permalink to "Spot-Check Sampling""></a></h3><p>Review 5-10% of all agent sessions manually. Focus on:</p><ul><li>Sessions with high cost (>2x average)</li><li>Sessions with tool loops</li><li>Sessions where automated eval scored low</li><li>Random sample for baseline</li></ul><hr><h2 id="lab-6-8-build-an-eval-harness" tabindex="-1">Lab 6.8: Build an Eval Harness <a class="header-anchor" href="#lab-6-8-build-an-eval-harness" aria-label="Permalink to "Lab 6.8: Build an Eval Harness""></a></h2><p><strong>Objective</strong>: Create golden Q&A pairs + automated pass/fail scoring.</p><p><strong>Starter</strong>: <code>course/labs/L6-eval-harness/starter.py</code></p><h2 id="lab-6-9-cost-optimization" tabindex="-1">Lab 6.9: Cost Optimization <a class="header-anchor" href="#lab-6-9-cost-optimization" aria-label="Permalink to "Lab 6.9: Cost Optimization""></a></h2><p><strong>Objective</strong>: Profile a session, identify savings, implement cascade routing.</p><p><strong>Starter</strong>: <code>course/labs/L6-cost-optimization/starter.py</code></p>`,73)])])}const u=s(n,[["render",l]]);export{k as __pageData,u as default};
|