134 lines
41 KiB
JavaScript
134 lines
41 KiB
JavaScript
import{c as a,Q as n,j as i,m as t}from"./chunks/framework.BPKcPtvA.js";const k=JSON.parse('{"title":"Module 5: Production Patterns","description":"","frontmatter":{},"headers":[],"relativePath":"modules/m5-production.md","filePath":"modules/m5-production.md","lastUpdated":null}'),e={name:"modules/m5-production.md"};function l(o,s,p,r,h,d){return n(),i("div",null,[...s[0]||(s[0]=[t(`<h1 id="module-5-production-patterns" tabindex="-1">Module 5: Production Patterns <a class="header-anchor" href="#module-5-production-patterns" aria-label="Permalink to "Module 5: Production Patterns""></a></h1><h2 id="lesson-5-1-what-production-means-for-agents" tabindex="-1">Lesson 5.1: What Production Means for Agents <a class="header-anchor" href="#lesson-5-1-what-production-means-for-agents" aria-label="Permalink to "Lesson 5.1: What Production Means for Agents""></a></h2><p>Production for agents is fundamentally different from traditional software:</p><table tabindex="0"><thead><tr><th>Traditional Software</th><th>Agent Systems</th></tr></thead><tbody><tr><td>Deterministic output</td><td>Non-deterministic behavior</td></tr><tr><td>Fixed cost per operation</td><td>Variable cost per session</td></tr><tr><td>Error = known exception</td><td>Error = unexpected behavior</td></tr><tr><td>Rollback = revert code</td><td>Rollback = revert prompt + pin model</td></tr><tr><td>Monitoring = latency + errors</td><td>Monitoring = token usage + loop depth</td></tr><tr><td>Testing = unit + integration</td><td>Testing = golden datasets + evals</td></tr></tbody></table><h3 id="production-readiness-checklist" tabindex="-1">Production Readiness Checklist <a class="header-anchor" href="#production-readiness-checklist" aria-label="Permalink to "Production Readiness Checklist""></a></h3><div class="language- vp-adaptive-theme"><button title="Copy Code" class="copy"></button><span class="lang"></span><pre class="shiki shiki-themes github-light github-dark vp-code" tabindex="0"><code><span class="line"><span>[ ] Version-locked prompts (hashed, diffed, reviewed)</span></span>
|
||
<span class="line"><span>[ ] Model pinning (not "latest", specific version)</span></span>
|
||
<span class="line"><span>[ ] Cost budgets per session/task/user</span></span>
|
||
<span class="line"><span>[ ] Iteration limits (max tool calls per session)</span></span>
|
||
<span class="line"><span>[ ] Observability (every tool call logged)</span></span>
|
||
<span class="line"><span>[ ] Shadow deployment capability</span></span>
|
||
<span class="line"><span>[ ] Rollback plan (prompt + model + env)</span></span>
|
||
<span class="line"><span>[ ] Security review (L3+ minimum)</span></span></code></pre></div><hr><h2 id="lesson-5-2-ci-cd-for-agents" tabindex="-1">Lesson 5.2: CI/CD for Agents <a class="header-anchor" href="#lesson-5-2-ci-cd-for-agents" aria-label="Permalink to "Lesson 5.2: CI/CD for Agents""></a></h2><h3 id="golden-dataset" tabindex="-1">Golden Dataset <a class="header-anchor" href="#golden-dataset" aria-label="Permalink to "Golden Dataset""></a></h3><p>A curated set of input/output/behavior pairs that define correct agent behavior:</p><div class="language-json vp-adaptive-theme"><button title="Copy Code" class="copy"></button><span class="lang">json</span><pre class="shiki shiki-themes github-light github-dark vp-code" tabindex="0"><code><span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">[</span></span>
|
||
<span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;"> {</span></span>
|
||
<span class="line"><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;"> "input"</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">: </span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;">"Find all users created in the last 24 hours"</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">,</span></span>
|
||
<span class="line"><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;"> "expected_tools"</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">: [</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;">"query_database"</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">],</span></span>
|
||
<span class="line"><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;"> "expected_tool_params"</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">: {</span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;">"query"</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">: </span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;">"SELECT * FROM users WHERE created_at > now() - interval '24 hours'"</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">},</span></span>
|
||
<span class="line"><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;"> "expected_output_contains"</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">: [</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;">"users"</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">, </span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;">"24 hours"</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">]</span></span>
|
||
<span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;"> }</span></span>
|
||
<span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">]</span></span></code></pre></div><h3 id="pipeline" tabindex="-1">Pipeline <a class="header-anchor" href="#pipeline" aria-label="Permalink to "Pipeline""></a></h3><div class="language- vp-adaptive-theme"><button title="Copy Code" class="copy"></button><span class="lang"></span><pre class="shiki shiki-themes github-light github-dark vp-code" tabindex="0"><code><span class="line"><span>1. Agent runs against golden dataset</span></span>
|
||
<span class="line"><span>2. Compare: tool selections match expected?</span></span>
|
||
<span class="line"><span> params match expected?</span></span>
|
||
<span class="line"><span> output contains expected strings?</span></span>
|
||
<span class="line"><span>3. Calculate pass@k</span></span>
|
||
<span class="line"><span>4. If pass@k < threshold (e.g., 80%), FAIL build</span></span>
|
||
<span class="line"><span>5. If pass, deploy new prompt/config</span></span></code></pre></div><hr><h2 id="lesson-5-2b-case-study-—-the-5-tool-production-stack" tabindex="-1">Lesson 5.2b: Case Study — The 5-Tool Production Stack <a class="header-anchor" href="#lesson-5-2b-case-study-—-the-5-tool-production-stack" aria-label="Permalink to "Lesson 5.2b: Case Study — The 5-Tool Production Stack""></a></h2><p>A real production multi-agent deployment uses multiple agent tools together, each for its strength. See <code>TOOL-REFERENCE.md</code> for full command references.</p><h3 id="the-stack" tabindex="-1">The Stack <a class="header-anchor" href="#the-stack" aria-label="Permalink to "The Stack""></a></h3><div class="language- vp-adaptive-theme"><button title="Copy Code" class="copy"></button><span class="lang"></span><pre class="shiki shiki-themes github-light github-dark vp-code" tabindex="0"><code><span class="line"><span>agent-mux (Tauri UI) —— Meta-agent control plane</span></span>
|
||
<span class="line"><span> │ └── cc agent SDK, pi-coding-agent SDK, opencode SDK via sidecar</span></span>
|
||
<span class="line"><span> ▼</span></span>
|
||
<span class="line"><span>mprocs (process monitor) —— Launches all agents</span></span>
|
||
<span class="line"><span> │ mprocs -c ~/mprocs-teams.yaml</span></span>
|
||
<span class="line"><span> │</span></span>
|
||
<span class="line"><span> ├── claude-lead (Claude Code)</span></span>
|
||
<span class="line"><span> │ └── psmux → tmux.exe → --teammate-mode → split panes</span></span>
|
||
<span class="line"><span> │ └── Each teammate = separate Claude session</span></span>
|
||
<span class="line"><span> │</span></span>
|
||
<span class="line"><span> ├── pi-agent (Pi Coding Agent)</span></span>
|
||
<span class="line"><span> │ └── Extensions: damage-control, tilldone, coms</span></span>
|
||
<span class="line"><span> │</span></span>
|
||
<span class="line"><span> ├── opencode (OpenCode CLI)</span></span>
|
||
<span class="line"><span> │ └── Model: opencode-go/deepseek-v4-flash (via proxy)</span></span>
|
||
<span class="line"><span> │</span></span>
|
||
<span class="line"><span> ├── hermes (Hermes Agent)</span></span>
|
||
<span class="line"><span> │ └── TypeScript-native, MCP-first workflows</span></span>
|
||
<span class="line"><span> │</span></span>
|
||
<span class="line"><span> ├── openclaw (OpenClaw daemon)</span></span>
|
||
<span class="line"><span> │ └── Always-on employee, heartbeat-driven</span></span>
|
||
<span class="line"><span> │</span></span>
|
||
<span class="line"><span> ├── gemini (Gemini fallback)</span></span>
|
||
<span class="line"><span> │ └── Fast/cheap tasks, cascade routing</span></span>
|
||
<span class="line"><span> │</span></span>
|
||
<span class="line"><span> ├── qwen (Qwen specialist)</span></span>
|
||
<span class="line"><span> │ └── Chinese + structured tasks</span></span>
|
||
<span class="line"><span> │</span></span>
|
||
<span class="line"><span> └── sidecar (agent-mux IPC proxy)</span></span>
|
||
<span class="line"><span> └── RPC bridge between Tauri UI and agent processes</span></span>
|
||
<span class="line"><span> │</span></span>
|
||
<span class="line"><span> ▼</span></span>
|
||
<span class="line"><span> psmux (tmux session manager)</span></span>
|
||
<span class="line"><span> │ tmux.exe at ~/.cargo/bin/tmux</span></span>
|
||
<span class="line"><span> │ Requires: start agent-teams first, then dmux inside it</span></span>
|
||
<span class="line"><span> │</span></span>
|
||
<span class="line"><span> ▼</span></span>
|
||
<span class="line"><span> dmux (git worktree isolation)</span></span>
|
||
<span class="line"><span> Each task gets an isolated worktree</span></span>
|
||
<span class="line"><span> Rollback = delete worktree</span></span></code></pre></div><h3 id="tool-roles-and-selection-logic" tabindex="-1">Tool Roles and Selection Logic <a class="header-anchor" href="#tool-roles-and-selection-logic" aria-label="Permalink to "Tool Roles and Selection Logic""></a></h3><table tabindex="0"><thead><tr><th>Tool</th><th>Role</th><th>When to Use</th><th>Stack Position</th></tr></thead><tbody><tr><td><strong>Claude Code</strong></td><td>Primary coding agent</td><td>Complex multi-step tasks, general development</td><td><code>claude-lead</code> in mprocs</td></tr><tr><td><strong>Pi Agent</strong></td><td>Customizable harness</td><td>Custom workflows, safety-critical ops, P2P</td><td>Side agent with extensions</td></tr><tr><td><strong>OpenCode</strong></td><td>OSS alternative</td><td>Budget tasks, CI/CD, when license matters</td><td>Backup in mprocs</td></tr><tr><td><strong>Hermes</strong></td><td>TypeScript pipelines</td><td>MCP-native workflows, structured output</td><td>Specialist in mprocs</td></tr><tr><td><strong>OpenClaw</strong></td><td>Always-on employee</td><td>Scheduled tasks, heartbeats, recurring</td><td>Daemon (always running)</td></tr><tr><td><strong>Gemini</strong></td><td>Fast/cheap fallback</td><td>High-volume simple tasks</td><td>Cascade routing tier 1</td></tr><tr><td><strong>Qwen</strong></td><td>Specialist model</td><td>Chinese content, structured generation</td><td>Cascade routing tier 2</td></tr></tbody></table><h3 id="how-they-work-together-real-session-flow" tabindex="-1">How They Work Together (Real Session Flow) <a class="header-anchor" href="#how-they-work-together-real-session-flow" aria-label="Permalink to "How They Work Together (Real Session Flow)""></a></h3><div class="language- vp-adaptive-theme"><button title="Copy Code" class="copy"></button><span class="lang"></span><pre class="shiki shiki-themes github-light github-dark vp-code" tabindex="0"><code><span class="line"><span>1. Human opens agent-mux Tauri UI</span></span>
|
||
<span class="line"><span>2. mprocs launches all agents from mprocs-teams.yaml</span></span>
|
||
<span class="line"><span>3. Claude Code (lead) runs in tmux via psmux</span></span>
|
||
<span class="line"><span> └── --teammate-mode creates split panes:</span></span>
|
||
<span class="line"><span> ├── pane 1: lead (primary coder)</span></span>
|
||
<span class="line"><span> ├── pane 2: worker (sub-tasks)</span></span>
|
||
<span class="line"><span> ├── pane 3: reviewer (code review)</span></span>
|
||
<span class="line"><span> └── pane 4: verifier (read-only checks)</span></span>
|
||
<span class="line"><span>4. OpenCode runs alongside as budget-aware backup</span></span>
|
||
<span class="line"><span>5. OpenClaw daemon handles scheduled background tasks</span></span>
|
||
<span class="line"><span>6. dmux isolates each task in its own git worktree</span></span>
|
||
<span class="line"><span>7. agent-mux sidecar collects status from all agents</span></span>
|
||
<span class="line"><span>8. Human monitors via Tauri UI, intervenes when needed</span></span></code></pre></div><h3 id="key-production-patterns" tabindex="-1">Key Production Patterns <a class="header-anchor" href="#key-production-patterns" aria-label="Permalink to "Key Production Patterns""></a></h3><ol><li><strong>Model heterogeneity</strong> — Different models for different roles. Cascade routing in practice (M6).</li><li><strong>Tool heterogeneity</strong> — Five CLIs, each with different strengths. No single point of failure.</li><li><strong>Process management</strong> — mprocs supervises. If one agent crashes, the stack keeps running.</li><li><strong>Session isolation</strong> — psmux (terminal sessions) + dmux (git worktrees) = two layers.</li><li><strong>Meta-control plane</strong> — agent-mux Tauri UI. Human watches and intervenes, not drives.</li><li><strong>Defense in depth</strong> — tool-level (damage-control), session-level (psmux), filesystem-level (dmux).</li></ol><hr><h2 id="lesson-5-2c-the-agent-manager-role" tabindex="-1">Lesson 5.2c: The Agent Manager Role <a class="header-anchor" href="#lesson-5-2c-the-agent-manager-role" aria-label="Permalink to "Lesson 5.2c: The Agent Manager Role""></a></h2><p>In enterprise deployments, someone owns the agent harness. This is the <strong>Agent Manager</strong> (or DevEx Lead for AI).</p><h3 id="responsibilities" tabindex="-1">Responsibilities <a class="header-anchor" href="#responsibilities" aria-label="Permalink to "Responsibilities""></a></h3><div class="language- vp-adaptive-theme"><button title="Copy Code" class="copy"></button><span class="lang"></span><pre class="shiki shiki-themes github-light github-dark vp-code" tabindex="0"><code><span class="line"><span>Agent Manager</span></span>
|
||
<span class="line"><span>├── Harness design (CLAUDE.md, skills, hooks, MCPs)</span></span>
|
||
<span class="line"><span>├── Tool selection (which agent CLIs, which models)</span></span>
|
||
<span class="line"><span>├── Security policy (damage-control rules, access levels)</span></span>
|
||
<span class="line"><span>├── Cost management (budgets per agent/task, optimization)</span></span>
|
||
<span class="line"><span>├── Quality gates (golden datasets, regression testing)</span></span>
|
||
<span class="line"><span>├── Update cadence (prompt versioning, model pinning)</span></span>
|
||
<span class="line"><span>└── Incident response (tool loops, cost spikes, failures)</span></span></code></pre></div><h3 id="_90-day-setup-playbook" tabindex="-1">90-Day Setup Playbook <a class="header-anchor" href="#_90-day-setup-playbook" aria-label="Permalink to "90-Day Setup Playbook""></a></h3><p><strong>Month 1</strong>: Foundation</p><ul><li>Set up agent CLI (Claude Code, Pi, or OpenCode)</li><li>Create CLAUDE.md with project context</li><li>Install damage-control with 3 access levels</li><li>Set up basic observability (tool call logging)</li></ul><p><strong>Month 2</strong>: Scale</p><ul><li>Add multi-agent teams (lead + workers)</li><li>Create golden dataset (10+ test cases)</li><li>Implement CI/CD gate</li><li>Set up cost tracking and budgets</li></ul><p><strong>Month 3</strong>: Production</p><ul><li>Shadow deployment pipeline</li><li>Rollback procedures documented</li><li>Monitoring dashboard live</li><li>Team trained on agent interaction patterns</li></ul><hr><h2 id="lesson-5-3-shadow-deployments" tabindex="-1">Lesson 5.3: Shadow Deployments <a class="header-anchor" href="#lesson-5-3-shadow-deployments" aria-label="Permalink to "Lesson 5.3: Shadow Deployments""></a></h2><h3 id="how-it-works" tabindex="-1">How It Works <a class="header-anchor" href="#how-it-works" aria-label="Permalink to "How It Works""></a></h3><div class="language- vp-adaptive-theme"><button title="Copy Code" class="copy"></button><span class="lang"></span><pre class="shiki shiki-themes github-light github-dark vp-code" tabindex="0"><code><span class="line"><span>Production agent: serves user traffic</span></span>
|
||
<span class="line"><span>Shadow agent: runs IDENTICAL inputs, but outputs are NOT served</span></span>
|
||
<span class="line"><span>Compare: did shadow make same decisions as production?</span></span>
|
||
<span class="line"><span> did shadow cost more/less?</span></span>
|
||
<span class="line"><span> did shadow hit any errors?</span></span>
|
||
<span class="line"><span>Decision: if shadow improves on all metrics, swap them</span></span></code></pre></div><h3 id="when-to-shadow-deploy" tabindex="-1">When to Shadow Deploy <a class="header-anchor" href="#when-to-shadow-deploy" aria-label="Permalink to "When to Shadow Deploy""></a></h3><ul><li>New prompt version</li><li>New model version</li><li>New tool addition</li><li>Agent architecture change</li></ul><hr><h2 id="lesson-5-4-rollback-strategies" tabindex="-1">Lesson 5.4: Rollback Strategies <a class="header-anchor" href="#lesson-5-4-rollback-strategies" aria-label="Permalink to "Lesson 5.4: Rollback Strategies""></a></h2><h3 id="what-rollback-means-for-agents" tabindex="-1">What Rollback Means for Agents <a class="header-anchor" href="#what-rollback-means-for-agents" aria-label="Permalink to "What Rollback Means for Agents""></a></h3><p>You can't just revert a Git commit. Agent behavior depends on:</p><ol><li><strong>Prompt</strong> — the text of the system prompt + tools</li><li><strong>Model</strong> — which model version</li><li><strong>Parameters</strong> — temperature, top_p, etc.</li><li><strong>Configuration</strong> — tool list, iteration limits, budget</li></ol><p>A proper rollback restores ALL four.</p><h3 id="implementation" tabindex="-1">Implementation <a class="header-anchor" href="#implementation" aria-label="Permalink to "Implementation""></a></h3><div class="language-yaml vp-adaptive-theme"><button title="Copy Code" class="copy"></button><span class="lang">yaml</span><pre class="shiki shiki-themes github-light github-dark vp-code" tabindex="0"><code><span class="line"><span style="--shiki-light:#6A737D;--shiki-dark:#6A737D;"># agent-config-v42.yaml</span></span>
|
||
<span class="line"><span style="--shiki-light:#22863A;--shiki-dark:#85E89D;">prompt_hash</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">: </span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;">"a1b2c3d4"</span></span>
|
||
<span class="line"><span style="--shiki-light:#22863A;--shiki-dark:#85E89D;">model</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">: </span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;">"claude-sonnet-4-20260501"</span><span style="--shiki-light:#6A737D;--shiki-dark:#6A737D;"> # pinned, not "latest"</span></span>
|
||
<span class="line"><span style="--shiki-light:#22863A;--shiki-dark:#85E89D;">temperature</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">: </span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;">0.0</span></span>
|
||
<span class="line"><span style="--shiki-light:#22863A;--shiki-dark:#85E89D;">max_iterations</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">: </span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;">25</span></span>
|
||
<span class="line"><span style="--shiki-light:#22863A;--shiki-dark:#85E89D;">tools</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">: [</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;">"read"</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">, </span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;">"write"</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">, </span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;">"bash"</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">, </span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;">"search"</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">]</span></span>
|
||
<span class="line"><span style="--shiki-light:#22863A;--shiki-dark:#85E89D;">budget_per_session</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">: </span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;">0.50</span></span></code></pre></div><p>Rollback = <code>cp agent-config-v41.yaml agent-config.yaml</code> + reload.</p><hr><h2 id="lesson-5-5-observability-monitoring" tabindex="-1">Lesson 5.5: Observability & Monitoring <a class="header-anchor" href="#lesson-5-5-observability-monitoring" aria-label="Permalink to "Lesson 5.5: Observability & Monitoring""></a></h2><h3 id="what-to-trace-every-single-turn" tabindex="-1">What to Trace (Every Single Turn) <a class="header-anchor" href="#what-to-trace-every-single-turn" aria-label="Permalink to "What to Trace (Every Single Turn)""></a></h3><ol><li><strong>Input prompt</strong> (full, including system prompt)</li><li><strong>LLM response</strong> (including tool call choices)</li><li><strong>Tool calls</strong> (name, params, timestamp)</li><li><strong>Tool results</strong> (output, error status, duration)</li><li><strong>Token counts</strong> (input, output, cached)</li><li><strong>Cost</strong> (per-call and running total)</li><li><strong>Loop depth</strong> (current turn number)</li></ol><h3 id="decision-tracing" tabindex="-1">Decision Tracing <a class="header-anchor" href="#decision-tracing" aria-label="Permalink to "Decision Tracing""></a></h3><p>Standard APM (Datadog, Grafana) captures latency and errors. Agents need <strong>decision tracing</strong> — the full chain of reasoning and actions:</p><div class="language-json vp-adaptive-theme"><button title="Copy Code" class="copy"></button><span class="lang">json</span><pre class="shiki shiki-themes github-light github-dark vp-code" tabindex="0"><code><span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">{</span></span>
|
||
<span class="line"><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;"> "session_id"</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">: </span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;">"sess_abc123"</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">,</span></span>
|
||
<span class="line"><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;"> "turn"</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">: </span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;">5</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">,</span></span>
|
||
<span class="line"><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;"> "input_tokens"</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">: </span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;">12400</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">,</span></span>
|
||
<span class="line"><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;"> "output_tokens"</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">: </span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;">350</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">,</span></span>
|
||
<span class="line"><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;"> "tool_calls"</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">: [</span></span>
|
||
<span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;"> {</span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;">"tool"</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">: </span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;">"search_web"</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">, </span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;">"params"</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">: {</span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;">"query"</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">: </span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;">"latest pricing"</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">}, </span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;">"duration_ms"</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">: </span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;">1200</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">}</span></span>
|
||
<span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;"> ],</span></span>
|
||
<span class="line"><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;"> "decision"</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">: </span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;">"Found pricing page, will extract"</span></span>
|
||
<span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">}</span></span></code></pre></div><h3 id="key-metrics" tabindex="-1">Key Metrics <a class="header-anchor" href="#key-metrics" aria-label="Permalink to "Key Metrics""></a></h3><table tabindex="0"><thead><tr><th>Metric</th><th>Warning</th><th>Critical</th></tr></thead><tbody><tr><td>Tool calls per session</td><td>>20</td><td>>50</td></tr><tr><td>Cost per session</td><td>>$0.50</td><td>>$2.00</td></tr><tr><td>Loop depth</td><td>>15</td><td>>30</td></tr><tr><td>Same tool >5x in row</td><td>Investigate loop</td><td>Kill session</td></tr><tr><td>Context utilization</td><td>>80%</td><td>>95%</td></tr></tbody></table><hr><h2 id="lesson-5-6-alerting-on-agent-specific-signals" tabindex="-1">Lesson 5.6: Alerting on Agent-Specific Signals <a class="header-anchor" href="#lesson-5-6-alerting-on-agent-specific-signals" aria-label="Permalink to "Lesson 5.6: Alerting on Agent-Specific Signals""></a></h2><h3 id="what-to-alert-on" tabindex="-1">What to Alert On <a class="header-anchor" href="#what-to-alert-on" aria-label="Permalink to "What to Alert On""></a></h3><ol><li><strong>Tool loop detected</strong> — same tool called 5+ times with same params</li><li><strong>Cost spike</strong> — session cost > 3x average</li><li><strong>Context overflow imminent</strong> — token count within 10% of limit</li><li><strong>Permission escalation</strong> — agent attempting blocked operations</li><li><strong>Error cascade</strong> — 3+ tool failures in a row</li><li><strong>Grinding detected</strong> — identical code rerun without changes</li></ol><h3 id="alert-routing" tabindex="-1">Alert Routing <a class="header-anchor" href="#alert-routing" aria-label="Permalink to "Alert Routing""></a></h3><div class="language- vp-adaptive-theme"><button title="Copy Code" class="copy"></button><span class="lang"></span><pre class="shiki shiki-themes github-light github-dark vp-code" tabindex="0"><code><span class="line"><span>P0 (immediate): Cost spike > $10, permission escalation, data exfil attempt</span></span>
|
||
<span class="line"><span>P1 (within 5 min): Tool loop, error cascade, grinding</span></span>
|
||
<span class="line"><span>P2 (within 1 hour): Context utilization high, cost trending up</span></span>
|
||
<span class="line"><span>P3 (daily report): Average session cost, success rate, failure modes</span></span></code></pre></div><h2 id="lesson-5-6b-cross-provider-session-search" tabindex="-1">Lesson 5.6b: Cross-Provider Session Search <a class="header-anchor" href="#lesson-5-6b-cross-provider-session-search" aria-label="Permalink to "Lesson 5.6b: Cross-Provider Session Search""></a></h2><p>When you run agents across 5+ tools (Claude Code, Pi, OpenCode, Gemini, OpenClaw), session history is scattered across different directories and formats.</p><h3 id="the-problem" tabindex="-1">The Problem <a class="header-anchor" href="#the-problem" aria-label="Permalink to "The Problem""></a></h3><div class="language-bash vp-adaptive-theme"><button title="Copy Code" class="copy"></button><span class="lang">bash</span><pre class="shiki shiki-themes github-light github-dark vp-code" tabindex="0"><code><span class="line"><span style="--shiki-light:#D73A49;--shiki-dark:#F97583;">~</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">/.claude/sessions/</span><span style="--shiki-light:#D73A49;--shiki-dark:#F97583;">*</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">.jsonl </span><span style="--shiki-light:#6A737D;--shiki-dark:#6A737D;"># Claude Code format</span></span>
|
||
<span class="line"><span style="--shiki-light:#D73A49;--shiki-dark:#F97583;">~</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">/.pi/sessions/</span><span style="--shiki-light:#D73A49;--shiki-dark:#F97583;">*</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">.jsonl </span><span style="--shiki-light:#6A737D;--shiki-dark:#6A737D;"># Pi format</span></span>
|
||
<span class="line"><span style="--shiki-light:#D73A49;--shiki-dark:#F97583;">~</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">/.opencode/sessions/</span><span style="--shiki-light:#D73A49;--shiki-dark:#F97583;">*</span><span style="--shiki-light:#6A737D;--shiki-dark:#6A737D;"> # OpenCode format</span></span>
|
||
<span class="line"><span style="--shiki-light:#D73A49;--shiki-dark:#F97583;">~</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">/.gemini/sessions/</span><span style="--shiki-light:#D73A49;--shiki-dark:#F97583;">*</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">.jsonl </span><span style="--shiki-light:#6A737D;--shiki-dark:#6A737D;"># Gemini format</span></span>
|
||
<span class="line"><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0;">mprocs-logs/*.log</span><span style="--shiki-light:#6A737D;--shiki-dark:#6A737D;"> # mprocs supervisor logs</span></span></code></pre></div><p>Searching across all of them is impossible without a unified index.</p><h3 id="the-solution" tabindex="-1">The Solution <a class="header-anchor" href="#the-solution" aria-label="Permalink to "The Solution""></a></h3><p><strong>Reference implementation</strong>: Jeff Emanuel's <code>coding_agent_session_search</code> (783★)</p><div class="language- vp-adaptive-theme"><button title="Copy Code" class="copy"></button><span class="lang"></span><pre class="shiki shiki-themes github-light github-dark vp-code" tabindex="0"><code><span class="line"><span>Indexer:</span></span>
|
||
<span class="line"><span> ├── Watches all session directories</span></span>
|
||
<span class="line"><span> ├── Normalizes into canonical format</span></span>
|
||
<span class="line"><span> ├── Full-text indexes prompts, responses, tool calls</span></span>
|
||
<span class="line"><span> └── Stores in SQLite with FTS5</span></span>
|
||
<span class="line"><span></span></span>
|
||
<span class="line"><span>Search CLI:</span></span>
|
||
<span class="line"><span> ├── Search across ALL providers from one command</span></span>
|
||
<span class="line"><span> ├── Filter by: provider, date, model, tool, token count</span></span>
|
||
<span class="line"><span> ├── Replay any session from any provider</span></span>
|
||
<span class="line"><span> └── Export sessions as markdown or JSON</span></span></code></pre></div><h3 id="why-this-matters-for-production" tabindex="-1">Why This Matters for Production <a class="header-anchor" href="#why-this-matters-for-production" aria-label="Permalink to "Why This Matters for Production""></a></h3><ol><li><strong>Debugging</strong>: Find what went wrong across all agents in one search</li><li><strong>Audit</strong>: Full history of every agent action, unified format</li><li><strong>Learning</strong>: Search past solutions instead of re-solving problems</li><li><strong>Cost analysis</strong>: Aggregate costs across all providers</li></ol><h3 id="integration-with-our-stack" tabindex="-1">Integration with Our Stack <a class="header-anchor" href="#integration-with-our-stack" aria-label="Permalink to "Integration with Our Stack""></a></h3><div class="language-bash vp-adaptive-theme"><button title="Copy Code" class="copy"></button><span class="lang">bash</span><pre class="shiki shiki-themes github-light github-dark vp-code" tabindex="0"><code><span class="line"><span style="--shiki-light:#6A737D;--shiki-dark:#6A737D;"># Add to mprocs-teams.yaml as a sidecar service</span></span>
|
||
<span class="line"><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0;">session-indexer:</span></span>
|
||
<span class="line"><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0;"> cmd:</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;"> [</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;">"python"</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">, </span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;">"session-indexer.py",</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> "--watch-dirs",</span></span>
|
||
<span class="line"><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0;"> "~/.claude/sessions/"</span><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0;">,</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> "~/.pi/sessions/",</span></span>
|
||
<span class="line"><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0;"> "~/.opencode/sessions/"</span><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0;">,</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> "logs/"]</span></span>
|
||
<span class="line"></span>
|
||
<span class="line"><span style="--shiki-light:#6A737D;--shiki-dark:#6A737D;"># Search across everything</span></span>
|
||
<span class="line"><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0;">session-search</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> "deployment error"</span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;"> --providers</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> claude,pi,opencode</span></span>
|
||
<span class="line"><span style="--shiki-light:#6A737D;--shiki-dark:#6A737D;"># Returns results from all providers in unified format</span></span></code></pre></div><hr><h2 id="lesson-5-7-deployment-modes" tabindex="-1">Lesson 5.7: Deployment Modes <a class="header-anchor" href="#lesson-5-7-deployment-modes" aria-label="Permalink to "Lesson 5.7: Deployment Modes""></a></h2><p>From Paperclip's deployment model:</p><table tabindex="0"><thead><tr><th>Mode</th><th>Auth</th><th>Exposure</th><th>Use Case</th></tr></thead><tbody><tr><td>Local trusted</td><td>None</td><td>localhost only</td><td>Single dev machine</td></tr><tr><td>Authenticated private</td><td>Login required</td><td>LAN/Tailscale/VPN</td><td>Team on private network</td></tr><tr><td>Authenticated public</td><td>Login required</td><td>Internet (behind reverse proxy)</td><td>Production cloud deployment</td></tr></tbody></table><h3 id="reachability" tabindex="-1">Reachability <a class="header-anchor" href="#reachability" aria-label="Permalink to "Reachability""></a></h3><table tabindex="0"><thead><tr><th>Bind</th><th>What It Means</th></tr></thead><tbody><tr><td><code>loopback</code></td><td>localhost only (default)</td></tr><tr><td><code>lan</code></td><td>All interfaces (0.0.0.0)</td></tr><tr><td><code>tailnet</code></td><td>Tailscale IP only</td></tr><tr><td><code>custom</code></td><td>Specific host/IP</td></tr></tbody></table><hr><h2 id="lesson-5-8-cost-control" tabindex="-1">Lesson 5.8: Cost Control <a class="header-anchor" href="#lesson-5-8-cost-control" aria-label="Permalink to "Lesson 5.8: Cost Control""></a></h2><h3 id="budget-architecture" tabindex="-1">Budget Architecture <a class="header-anchor" href="#budget-architecture" aria-label="Permalink to "Budget Architecture""></a></h3><div class="language- vp-adaptive-theme"><button title="Copy Code" class="copy"></button><span class="lang"></span><pre class="shiki shiki-themes github-light github-dark vp-code" tabindex="0"><code><span class="line"><span>Company budget → Agent budgets → Session budgets → Per-call tracking</span></span></code></pre></div><h3 id="budget-policies" tabindex="-1">Budget Policies <a class="header-anchor" href="#budget-policies" aria-label="Permalink to "Budget Policies""></a></h3><div class="language-yaml vp-adaptive-theme"><button title="Copy Code" class="copy"></button><span class="lang">yaml</span><pre class="shiki shiki-themes github-light github-dark vp-code" tabindex="0"><code><span class="line"><span style="--shiki-light:#22863A;--shiki-dark:#85E89D;">budget_policies</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">:</span></span>
|
||
<span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;"> - </span><span style="--shiki-light:#22863A;--shiki-dark:#85E89D;">metric</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">: </span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;">"monthly_cost_cents"</span></span>
|
||
<span class="line"><span style="--shiki-light:#22863A;--shiki-dark:#85E89D;"> scope</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">: </span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;">"agent:backend-dev"</span></span>
|
||
<span class="line"><span style="--shiki-light:#22863A;--shiki-dark:#85E89D;"> amount</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">: </span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;">50000</span><span style="--shiki-light:#6A737D;--shiki-dark:#6A737D;"> # $500/month</span></span>
|
||
<span class="line"><span style="--shiki-light:#22863A;--shiki-dark:#85E89D;"> warn_at</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">: </span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;">80%</span><span style="--shiki-light:#6A737D;--shiki-dark:#6A737D;"> # warn at $400</span></span>
|
||
<span class="line"><span style="--shiki-light:#22863A;--shiki-dark:#85E89D;"> hard_stop</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">: </span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;">true</span><span style="--shiki-light:#6A737D;--shiki-dark:#6A737D;"> # kill at $500</span></span>
|
||
<span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;"> - </span><span style="--shiki-light:#22863A;--shiki-dark:#85E89D;">metric</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">: </span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;">"session_cost_cents"</span></span>
|
||
<span class="line"><span style="--shiki-light:#22863A;--shiki-dark:#85E89D;"> scope</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">: </span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;">"global"</span></span>
|
||
<span class="line"><span style="--shiki-light:#22863A;--shiki-dark:#85E89D;"> amount</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">: </span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;">200</span><span style="--shiki-light:#6A737D;--shiki-dark:#6A737D;"> # $2/session max</span></span>
|
||
<span class="line"><span style="--shiki-light:#22863A;--shiki-dark:#85E89D;"> hard_stop</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">: </span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;">true</span></span></code></pre></div><h3 id="warning-vs-hard-stop" tabindex="-1">Warning vs Hard Stop <a class="header-anchor" href="#warning-vs-hard-stop" aria-label="Permalink to "Warning vs Hard Stop""></a></h3><ul><li><strong>Warning</strong> (80%) — Notify operator, agent keeps running</li><li><strong>Hard Stop</strong> (100%) — Agent paused, new tasks queued, running task cancelled</li></ul><hr><h2 id="lab-5-9-set-up-agent-observability" tabindex="-1">Lab 5.9: Set Up Agent Observability <a class="header-anchor" href="#lab-5-9-set-up-agent-observability" aria-label="Permalink to "Lab 5.9: Set Up Agent Observability""></a></h2><p><strong>Objective</strong>: Trace every tool call + LLM completion to a local SQLite database.</p><p><strong>Starter</strong>: <code>course/labs/L5-observability/starter/</code></p><h2 id="lab-5-10-ci-cd-pipeline" tabindex="-1">Lab 5.10: CI/CD Pipeline <a class="header-anchor" href="#lab-5-10-ci-cd-pipeline" aria-label="Permalink to "Lab 5.10: CI/CD Pipeline""></a></h2><p><strong>Objective</strong>: Create a golden dataset and automated regression gate.</p><p><strong>Starter</strong>: <code>course/labs/L5-cicd/starter/</code></p>`,99)])])}const g=a(e,[["render",l]]);export{k as __pageData,g as default};
|