import{c as a,Q as n,j as i,m as t}from"./chunks/framework.BPKcPtvA.js";const k=JSON.parse('{"title":"Module 5: Production Patterns","description":"","frontmatter":{},"headers":[],"relativePath":"modules/m5-production.md","filePath":"modules/m5-production.md","lastUpdated":null}'),e={name:"modules/m5-production.md"};function l(o,s,p,r,h,d){return n(),i("div",null,[...s[0]||(s[0]=[t(`<h1 id="module-5-production-patterns" tabindex="-1">Module 5: Production Patterns <a class="header-anchor" href="#module-5-production-patterns" aria-label="Permalink to &quot;Module 5: Production Patterns&quot;">​</a></h1><h2 id="lesson-5-1-what-production-means-for-agents" tabindex="-1">Lesson 5.1: What Production Means for Agents <a class="header-anchor" href="#lesson-5-1-what-production-means-for-agents" aria-label="Permalink to &quot;Lesson 5.1: What Production Means for Agents&quot;">​</a></h2><p>Production for agents is fundamentally different from traditional software:</p><table tabindex="0"><thead><tr><th>Traditional Software</th><th>Agent Systems</th></tr></thead><tbody><tr><td>Deterministic output</td><td>Non-deterministic behavior</td></tr><tr><td>Fixed cost per operation</td><td>Variable cost per session</td></tr><tr><td>Error = known exception</td><td>Error = unexpected behavior</td></tr><tr><td>Rollback = revert code</td><td>Rollback = revert prompt + pin model</td></tr><tr><td>Monitoring = latency + errors</td><td>Monitoring = token usage + loop depth</td></tr><tr><td>Testing = unit + integration</td><td>Testing = golden datasets + evals</td></tr></tbody></table><h3 id="production-readiness-checklist" tabindex="-1">Production Readiness Checklist <a class="header-anchor" href="#production-readiness-checklist" aria-label="Permalink to &quot;Production Readiness Checklist&quot;">​</a></h3><div class="language- vp-adaptive-theme"><button title="Copy Code" class="copy"></button><span class="lang"></span><pre class="shiki shiki-themes github-light github-dark vp-code" tabindex="0"><code><span class="line"><span>[ ] Version-locked prompts (hashed, diffed, reviewed)</span></span>
<span class="line"><span>[ ] Model pinning (not &quot;latest&quot;, specific version)</span></span>
<span class="line"><span>[ ] Cost budgets per session/task/user</span></span>
<span class="line"><span>[ ] Iteration limits (max tool calls per session)</span></span>
<span class="line"><span>[ ] Observability (every tool call logged)</span></span>
<span class="line"><span>[ ] Shadow deployment capability</span></span>
<span class="line"><span>[ ] Rollback plan (prompt + model + env)</span></span>
<span class="line"><span>[ ] Security review (L3+ minimum)</span></span></code></pre></div><hr><h2 id="lesson-5-2-ci-cd-for-agents" tabindex="-1">Lesson 5.2: CI/CD for Agents <a class="header-anchor" href="#lesson-5-2-ci-cd-for-agents" aria-label="Permalink to &quot;Lesson 5.2: CI/CD for Agents&quot;">​</a></h2><h3 id="golden-dataset" tabindex="-1">Golden Dataset <a class="header-anchor" href="#golden-dataset" aria-label="Permalink to &quot;Golden Dataset&quot;">​</a></h3><p>A curated set of input/output/behavior pairs that define correct agent behavior:</p><div class="language-json vp-adaptive-theme"><button title="Copy Code" class="copy"></button><span class="lang">json</span><pre class="shiki shiki-themes github-light github-dark vp-code" tabindex="0"><code><span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">[</span></span>
<span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">  {</span></span>
<span class="line"><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;">    &quot;input&quot;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">: </span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;">&quot;Find all users created in the last 24 hours&quot;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">,</span></span>
<span class="line"><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;">    &quot;expected_tools&quot;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">: [</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;">&quot;query_database&quot;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">],</span></span>
<span class="line"><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;">    &quot;expected_tool_params&quot;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">: {</span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;">&quot;query&quot;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">: </span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;">&quot;SELECT * FROM users WHERE created_at &gt; now() - interval &#39;24 hours&#39;&quot;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">},</span></span>
<span class="line"><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;">    &quot;expected_output_contains&quot;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">: [</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;">&quot;users&quot;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">, </span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;">&quot;24 hours&quot;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">]</span></span>
<span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">  }</span></span>
<span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">]</span></span></code></pre></div><h3 id="pipeline" tabindex="-1">Pipeline <a class="header-anchor" href="#pipeline" aria-label="Permalink to &quot;Pipeline&quot;">​</a></h3><div class="language- vp-adaptive-theme"><button title="Copy Code" class="copy"></button><span class="lang"></span><pre class="shiki shiki-themes github-light github-dark vp-code" tabindex="0"><code><span class="line"><span>1. Agent runs against golden dataset</span></span>
<span class="line"><span>2. Compare: tool selections match expected?</span></span>
<span class="line"><span>            params match expected?</span></span>
<span class="line"><span>            output contains expected strings?</span></span>
<span class="line"><span>3. Calculate pass@k</span></span>
<span class="line"><span>4. If pass@k &lt; threshold (e.g., 80%), FAIL build</span></span>
<span class="line"><span>5. If pass, deploy new prompt/config</span></span></code></pre></div><hr><h2 id="lesson-5-2b-case-study-—-the-5-tool-production-stack" tabindex="-1">Lesson 5.2b: Case Study — The 5-Tool Production Stack <a class="header-anchor" href="#lesson-5-2b-case-study-—-the-5-tool-production-stack" aria-label="Permalink to &quot;Lesson 5.2b: Case Study — The 5-Tool Production Stack&quot;">​</a></h2><p>A real production multi-agent deployment uses multiple agent tools together, each for its strength. See <code>TOOL-REFERENCE.md</code> for full command references.</p><h3 id="the-stack" tabindex="-1">The Stack <a class="header-anchor" href="#the-stack" aria-label="Permalink to &quot;The Stack&quot;">​</a></h3><div class="language- vp-adaptive-theme"><button title="Copy Code" class="copy"></button><span class="lang"></span><pre class="shiki shiki-themes github-light github-dark vp-code" tabindex="0"><code><span class="line"><span>agent-mux (Tauri UI) —— Meta-agent control plane</span></span>
<span class="line"><span>    │   └── cc agent SDK, pi-coding-agent SDK, opencode SDK via sidecar</span></span>
<span class="line"><span>    ▼</span></span>
<span class="line"><span>mprocs (process monitor) —— Launches all agents</span></span>
<span class="line"><span>    │   mprocs -c ~/mprocs-teams.yaml</span></span>
<span class="line"><span>    │</span></span>
<span class="line"><span>    ├── claude-lead (Claude Code)</span></span>
<span class="line"><span>    │     └── psmux → tmux.exe → --teammate-mode → split panes</span></span>
<span class="line"><span>    │           └── Each teammate = separate Claude session</span></span>
<span class="line"><span>    │</span></span>
<span class="line"><span>    ├── pi-agent (Pi Coding Agent)</span></span>
<span class="line"><span>    │     └── Extensions: damage-control, tilldone, coms</span></span>
<span class="line"><span>    │</span></span>
<span class="line"><span>    ├── opencode (OpenCode CLI)</span></span>
<span class="line"><span>    │     └── Model: opencode-go/deepseek-v4-flash (via proxy)</span></span>
<span class="line"><span>    │</span></span>
<span class="line"><span>    ├── hermes (Hermes Agent)</span></span>
<span class="line"><span>    │     └── TypeScript-native, MCP-first workflows</span></span>
<span class="line"><span>    │</span></span>
<span class="line"><span>    ├── openclaw (OpenClaw daemon)</span></span>
<span class="line"><span>    │     └── Always-on employee, heartbeat-driven</span></span>
<span class="line"><span>    │</span></span>
<span class="line"><span>    ├── gemini (Gemini fallback)</span></span>
<span class="line"><span>    │     └── Fast/cheap tasks, cascade routing</span></span>
<span class="line"><span>    │</span></span>
<span class="line"><span>    ├── qwen (Qwen specialist)</span></span>
<span class="line"><span>    │     └── Chinese + structured tasks</span></span>
<span class="line"><span>    │</span></span>
<span class="line"><span>    └── sidecar (agent-mux IPC proxy)</span></span>
<span class="line"><span>          └── RPC bridge between Tauri UI and agent processes</span></span>
<span class="line"><span>          │</span></span>
<span class="line"><span>          ▼</span></span>
<span class="line"><span>    psmux (tmux session manager)</span></span>
<span class="line"><span>    │   tmux.exe at ~/.cargo/bin/tmux</span></span>
<span class="line"><span>    │   Requires: start agent-teams first, then dmux inside it</span></span>
<span class="line"><span>    │</span></span>
<span class="line"><span>    ▼</span></span>
<span class="line"><span>    dmux (git worktree isolation)</span></span>
<span class="line"><span>        Each task gets an isolated worktree</span></span>
<span class="line"><span>        Rollback = delete worktree</span></span></code></pre></div><h3 id="tool-roles-and-selection-logic" tabindex="-1">Tool Roles and Selection Logic <a class="header-anchor" href="#tool-roles-and-selection-logic" aria-label="Permalink to &quot;Tool Roles and Selection Logic&quot;">​</a></h3><table tabindex="0"><thead><tr><th>Tool</th><th>Role</th><th>When to Use</th><th>Stack Position</th></tr></thead><tbody><tr><td><strong>Claude Code</strong></td><td>Primary coding agent</td><td>Complex multi-step tasks, general development</td><td><code>claude-lead</code> in mprocs</td></tr><tr><td><strong>Pi Agent</strong></td><td>Customizable harness</td><td>Custom workflows, safety-critical ops, P2P</td><td>Side agent with extensions</td></tr><tr><td><strong>OpenCode</strong></td><td>OSS alternative</td><td>Budget tasks, CI/CD, when license matters</td><td>Backup in mprocs</td></tr><tr><td><strong>Hermes</strong></td><td>TypeScript pipelines</td><td>MCP-native workflows, structured output</td><td>Specialist in mprocs</td></tr><tr><td><strong>OpenClaw</strong></td><td>Always-on employee</td><td>Scheduled tasks, heartbeats, recurring</td><td>Daemon (always running)</td></tr><tr><td><strong>Gemini</strong></td><td>Fast/cheap fallback</td><td>High-volume simple tasks</td><td>Cascade routing tier 1</td></tr><tr><td><strong>Qwen</strong></td><td>Specialist model</td><td>Chinese content, structured generation</td><td>Cascade routing tier 2</td></tr></tbody></table><h3 id="how-they-work-together-real-session-flow" tabindex="-1">How They Work Together (Real Session Flow) <a class="header-anchor" href="#how-they-work-together-real-session-flow" aria-label="Permalink to &quot;How They Work Together (Real Session Flow)&quot;">​</a></h3><div class="language- vp-adaptive-theme"><button title="Copy Code" class="copy"></button><span class="lang"></span><pre class="shiki shiki-themes github-light github-dark vp-code" tabindex="0"><code><span class="line"><span>1. Human opens agent-mux Tauri UI</span></span>
<span class="line"><span>2. mprocs launches all agents from mprocs-teams.yaml</span></span>
<span class="line"><span>3. Claude Code (lead) runs in tmux via psmux</span></span>
<span class="line"><span>     └── --teammate-mode creates split panes:</span></span>
<span class="line"><span>           ├── pane 1: lead (primary coder)</span></span>
<span class="line"><span>           ├── pane 2: worker (sub-tasks)</span></span>
<span class="line"><span>           ├── pane 3: reviewer (code review)</span></span>
<span class="line"><span>           └── pane 4: verifier (read-only checks)</span></span>
<span class="line"><span>4. OpenCode runs alongside as budget-aware backup</span></span>
<span class="line"><span>5. OpenClaw daemon handles scheduled background tasks</span></span>
<span class="line"><span>6. dmux isolates each task in its own git worktree</span></span>
<span class="line"><span>7. agent-mux sidecar collects status from all agents</span></span>
<span class="line"><span>8. Human monitors via Tauri UI, intervenes when needed</span></span></code></pre></div><h3 id="key-production-patterns" tabindex="-1">Key Production Patterns <a class="header-anchor" href="#key-production-patterns" aria-label="Permalink to &quot;Key Production Patterns&quot;">​</a></h3><ol><li><strong>Model heterogeneity</strong> — Different models for different roles. Cascade routing in practice (M6).</li><li><strong>Tool heterogeneity</strong> — Five CLIs, each with different strengths. No single point of failure.</li><li><strong>Process management</strong> — mprocs supervises. If one agent crashes, the stack keeps running.</li><li><strong>Session isolation</strong> — psmux (terminal sessions) + dmux (git worktrees) = two layers.</li><li><strong>Meta-control plane</strong> — agent-mux Tauri UI. Human watches and intervenes, not drives.</li><li><strong>Defense in depth</strong> — tool-level (damage-control), session-level (psmux), filesystem-level (dmux).</li></ol><hr><h2 id="lesson-5-2c-the-agent-manager-role" tabindex="-1">Lesson 5.2c: The Agent Manager Role <a class="header-anchor" href="#lesson-5-2c-the-agent-manager-role" aria-label="Permalink to &quot;Lesson 5.2c: The Agent Manager Role&quot;">​</a></h2><p>In enterprise deployments, someone owns the agent harness. This is the <strong>Agent Manager</strong> (or DevEx Lead for AI).</p><h3 id="responsibilities" tabindex="-1">Responsibilities <a class="header-anchor" href="#responsibilities" aria-label="Permalink to &quot;Responsibilities&quot;">​</a></h3><div class="language- vp-adaptive-theme"><button title="Copy Code" class="copy"></button><span class="lang"></span><pre class="shiki shiki-themes github-light github-dark vp-code" tabindex="0"><code><span class="line"><span>Agent Manager</span></span>
<span class="line"><span>├── Harness design (CLAUDE.md, skills, hooks, MCPs)</span></span>
<span class="line"><span>├── Tool selection (which agent CLIs, which models)</span></span>
<span class="line"><span>├── Security policy (damage-control rules, access levels)</span></span>
<span class="line"><span>├── Cost management (budgets per agent/task, optimization)</span></span>
<span class="line"><span>├── Quality gates (golden datasets, regression testing)</span></span>
<span class="line"><span>├── Update cadence (prompt versioning, model pinning)</span></span>
<span class="line"><span>└── Incident response (tool loops, cost spikes, failures)</span></span></code></pre></div><h3 id="_90-day-setup-playbook" tabindex="-1">90-Day Setup Playbook <a class="header-anchor" href="#_90-day-setup-playbook" aria-label="Permalink to &quot;90-Day Setup Playbook&quot;">​</a></h3><p><strong>Month 1</strong>: Foundation</p><ul><li>Set up agent CLI (Claude Code, Pi, or OpenCode)</li><li>Create CLAUDE.md with project context</li><li>Install damage-control with 3 access levels</li><li>Set up basic observability (tool call logging)</li></ul><p><strong>Month 2</strong>: Scale</p><ul><li>Add multi-agent teams (lead + workers)</li><li>Create golden dataset (10+ test cases)</li><li>Implement CI/CD gate</li><li>Set up cost tracking and budgets</li></ul><p><strong>Month 3</strong>: Production</p><ul><li>Shadow deployment pipeline</li><li>Rollback procedures documented</li><li>Monitoring dashboard live</li><li>Team trained on agent interaction patterns</li></ul><hr><h2 id="lesson-5-3-shadow-deployments" tabindex="-1">Lesson 5.3: Shadow Deployments <a class="header-anchor" href="#lesson-5-3-shadow-deployments" aria-label="Permalink to &quot;Lesson 5.3: Shadow Deployments&quot;">​</a></h2><h3 id="how-it-works" tabindex="-1">How It Works <a class="header-anchor" href="#how-it-works" aria-label="Permalink to &quot;How It Works&quot;">​</a></h3><div class="language- vp-adaptive-theme"><button title="Copy Code" class="copy"></button><span class="lang"></span><pre class="shiki shiki-themes github-light github-dark vp-code" tabindex="0"><code><span class="line"><span>Production agent: serves user traffic</span></span>
<span class="line"><span>Shadow agent: runs IDENTICAL inputs, but outputs are NOT served</span></span>
<span class="line"><span>Compare: did shadow make same decisions as production?</span></span>
<span class="line"><span>         did shadow cost more/less?</span></span>
<span class="line"><span>         did shadow hit any errors?</span></span>
<span class="line"><span>Decision: if shadow improves on all metrics, swap them</span></span></code></pre></div><h3 id="when-to-shadow-deploy" tabindex="-1">When to Shadow Deploy <a class="header-anchor" href="#when-to-shadow-deploy" aria-label="Permalink to &quot;When to Shadow Deploy&quot;">​</a></h3><ul><li>New prompt version</li><li>New model version</li><li>New tool addition</li><li>Agent architecture change</li></ul><hr><h2 id="lesson-5-4-rollback-strategies" tabindex="-1">Lesson 5.4: Rollback Strategies <a class="header-anchor" href="#lesson-5-4-rollback-strategies" aria-label="Permalink to &quot;Lesson 5.4: Rollback Strategies&quot;">​</a></h2><h3 id="what-rollback-means-for-agents" tabindex="-1">What Rollback Means for Agents <a class="header-anchor" href="#what-rollback-means-for-agents" aria-label="Permalink to &quot;What Rollback Means for Agents&quot;">​</a></h3><p>You can&#39;t just revert a Git commit. Agent behavior depends on:</p><ol><li><strong>Prompt</strong> — the text of the system prompt + tools</li><li><strong>Model</strong> — which model version</li><li><strong>Parameters</strong> — temperature, top_p, etc.</li><li><strong>Configuration</strong> — tool list, iteration limits, budget</li></ol><p>A proper rollback restores ALL four.</p><h3 id="implementation" tabindex="-1">Implementation <a class="header-anchor" href="#implementation" aria-label="Permalink to &quot;Implementation&quot;">​</a></h3><div class="language-yaml vp-adaptive-theme"><button title="Copy Code" class="copy"></button><span class="lang">yaml</span><pre class="shiki shiki-themes github-light github-dark vp-code" tabindex="0"><code><span class="line"><span style="--shiki-light:#6A737D;--shiki-dark:#6A737D;"># agent-config-v42.yaml</span></span>
<span class="line"><span style="--shiki-light:#22863A;--shiki-dark:#85E89D;">prompt_hash</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">: </span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;">&quot;a1b2c3d4&quot;</span></span>
<span class="line"><span style="--shiki-light:#22863A;--shiki-dark:#85E89D;">model</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">: </span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;">&quot;claude-sonnet-4-20260501&quot;</span><span style="--shiki-light:#6A737D;--shiki-dark:#6A737D;">  # pinned, not &quot;latest&quot;</span></span>
<span class="line"><span style="--shiki-light:#22863A;--shiki-dark:#85E89D;">temperature</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">: </span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;">0.0</span></span>
<span class="line"><span style="--shiki-light:#22863A;--shiki-dark:#85E89D;">max_iterations</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">: </span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;">25</span></span>
<span class="line"><span style="--shiki-light:#22863A;--shiki-dark:#85E89D;">tools</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">: [</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;">&quot;read&quot;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">, </span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;">&quot;write&quot;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">, </span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;">&quot;bash&quot;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">, </span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;">&quot;search&quot;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">]</span></span>
<span class="line"><span style="--shiki-light:#22863A;--shiki-dark:#85E89D;">budget_per_session</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">: </span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;">0.50</span></span></code></pre></div><p>Rollback = <code>cp agent-config-v41.yaml agent-config.yaml</code> + reload.</p><hr><h2 id="lesson-5-5-observability-monitoring" tabindex="-1">Lesson 5.5: Observability &amp; Monitoring <a class="header-anchor" href="#lesson-5-5-observability-monitoring" aria-label="Permalink to &quot;Lesson 5.5: Observability &amp; Monitoring&quot;">​</a></h2><h3 id="what-to-trace-every-single-turn" tabindex="-1">What to Trace (Every Single Turn) <a class="header-anchor" href="#what-to-trace-every-single-turn" aria-label="Permalink to &quot;What to Trace (Every Single Turn)&quot;">​</a></h3><ol><li><strong>Input prompt</strong> (full, including system prompt)</li><li><strong>LLM response</strong> (including tool call choices)</li><li><strong>Tool calls</strong> (name, params, timestamp)</li><li><strong>Tool results</strong> (output, error status, duration)</li><li><strong>Token counts</strong> (input, output, cached)</li><li><strong>Cost</strong> (per-call and running total)</li><li><strong>Loop depth</strong> (current turn number)</li></ol><h3 id="decision-tracing" tabindex="-1">Decision Tracing <a class="header-anchor" href="#decision-tracing" aria-label="Permalink to &quot;Decision Tracing&quot;">​</a></h3><p>Standard APM (Datadog, Grafana) captures latency and errors. Agents need <strong>decision tracing</strong> — the full chain of reasoning and actions:</p><div class="language-json vp-adaptive-theme"><button title="Copy Code" class="copy"></button><span class="lang">json</span><pre class="shiki shiki-themes github-light github-dark vp-code" tabindex="0"><code><span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">{</span></span>
<span class="line"><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;">  &quot;session_id&quot;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">: </span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;">&quot;sess_abc123&quot;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">,</span></span>
<span class="line"><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;">  &quot;turn&quot;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">: </span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;">5</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">,</span></span>
<span class="line"><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;">  &quot;input_tokens&quot;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">: </span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;">12400</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">,</span></span>
<span class="line"><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;">  &quot;output_tokens&quot;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">: </span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;">350</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">,</span></span>
<span class="line"><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;">  &quot;tool_calls&quot;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">: [</span></span>
<span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">    {</span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;">&quot;tool&quot;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">: </span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;">&quot;search_web&quot;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">, </span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;">&quot;params&quot;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">: {</span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;">&quot;query&quot;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">: </span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;">&quot;latest pricing&quot;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">}, </span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;">&quot;duration_ms&quot;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">: </span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;">1200</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">}</span></span>
<span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">  ],</span></span>
<span class="line"><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;">  &quot;decision&quot;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">: </span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;">&quot;Found pricing page, will extract&quot;</span></span>
<span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">}</span></span></code></pre></div><h3 id="key-metrics" tabindex="-1">Key Metrics <a class="header-anchor" href="#key-metrics" aria-label="Permalink to &quot;Key Metrics&quot;">​</a></h3><table tabindex="0"><thead><tr><th>Metric</th><th>Warning</th><th>Critical</th></tr></thead><tbody><tr><td>Tool calls per session</td><td>&gt;20</td><td>&gt;50</td></tr><tr><td>Cost per session</td><td>&gt;$0.50</td><td>&gt;$2.00</td></tr><tr><td>Loop depth</td><td>&gt;15</td><td>&gt;30</td></tr><tr><td>Same tool &gt;5x in row</td><td>Investigate loop</td><td>Kill session</td></tr><tr><td>Context utilization</td><td>&gt;80%</td><td>&gt;95%</td></tr></tbody></table><hr><h2 id="lesson-5-6-alerting-on-agent-specific-signals" tabindex="-1">Lesson 5.6: Alerting on Agent-Specific Signals <a class="header-anchor" href="#lesson-5-6-alerting-on-agent-specific-signals" aria-label="Permalink to &quot;Lesson 5.6: Alerting on Agent-Specific Signals&quot;">​</a></h2><h3 id="what-to-alert-on" tabindex="-1">What to Alert On <a class="header-anchor" href="#what-to-alert-on" aria-label="Permalink to &quot;What to Alert On&quot;">​</a></h3><ol><li><strong>Tool loop detected</strong> — same tool called 5+ times with same params</li><li><strong>Cost spike</strong> — session cost &gt; 3x average</li><li><strong>Context overflow imminent</strong> — token count within 10% of limit</li><li><strong>Permission escalation</strong> — agent attempting blocked operations</li><li><strong>Error cascade</strong> — 3+ tool failures in a row</li><li><strong>Grinding detected</strong> — identical code rerun without changes</li></ol><h3 id="alert-routing" tabindex="-1">Alert Routing <a class="header-anchor" href="#alert-routing" aria-label="Permalink to &quot;Alert Routing&quot;">​</a></h3><div class="language- vp-adaptive-theme"><button title="Copy Code" class="copy"></button><span class="lang"></span><pre class="shiki shiki-themes github-light github-dark vp-code" tabindex="0"><code><span class="line"><span>P0 (immediate): Cost spike &gt; $10, permission escalation, data exfil attempt</span></span>
<span class="line"><span>P1 (within 5 min): Tool loop, error cascade, grinding</span></span>
<span class="line"><span>P2 (within 1 hour): Context utilization high, cost trending up</span></span>
<span class="line"><span>P3 (daily report): Average session cost, success rate, failure modes</span></span></code></pre></div><h2 id="lesson-5-6b-cross-provider-session-search" tabindex="-1">Lesson 5.6b: Cross-Provider Session Search <a class="header-anchor" href="#lesson-5-6b-cross-provider-session-search" aria-label="Permalink to &quot;Lesson 5.6b: Cross-Provider Session Search&quot;">​</a></h2><p>When you run agents across 5+ tools (Claude Code, Pi, OpenCode, Gemini, OpenClaw), session history is scattered across different directories and formats.</p><h3 id="the-problem" tabindex="-1">The Problem <a class="header-anchor" href="#the-problem" aria-label="Permalink to &quot;The Problem&quot;">​</a></h3><div class="language-bash vp-adaptive-theme"><button title="Copy Code" class="copy"></button><span class="lang">bash</span><pre class="shiki shiki-themes github-light github-dark vp-code" tabindex="0"><code><span class="line"><span style="--shiki-light:#D73A49;--shiki-dark:#F97583;">~</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">/.claude/sessions/</span><span style="--shiki-light:#D73A49;--shiki-dark:#F97583;">*</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">.jsonl      </span><span style="--shiki-light:#6A737D;--shiki-dark:#6A737D;"># Claude Code format</span></span>
<span class="line"><span style="--shiki-light:#D73A49;--shiki-dark:#F97583;">~</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">/.pi/sessions/</span><span style="--shiki-light:#D73A49;--shiki-dark:#F97583;">*</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">.jsonl          </span><span style="--shiki-light:#6A737D;--shiki-dark:#6A737D;"># Pi format</span></span>
<span class="line"><span style="--shiki-light:#D73A49;--shiki-dark:#F97583;">~</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">/.opencode/sessions/</span><span style="--shiki-light:#D73A49;--shiki-dark:#F97583;">*</span><span style="--shiki-light:#6A737D;--shiki-dark:#6A737D;">          # OpenCode format</span></span>
<span class="line"><span style="--shiki-light:#D73A49;--shiki-dark:#F97583;">~</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">/.gemini/sessions/</span><span style="--shiki-light:#D73A49;--shiki-dark:#F97583;">*</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">.jsonl      </span><span style="--shiki-light:#6A737D;--shiki-dark:#6A737D;"># Gemini format</span></span>
<span class="line"><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0;">mprocs-logs/*.log</span><span style="--shiki-light:#6A737D;--shiki-dark:#6A737D;">               # mprocs supervisor logs</span></span></code></pre></div><p>Searching across all of them is impossible without a unified index.</p><h3 id="the-solution" tabindex="-1">The Solution <a class="header-anchor" href="#the-solution" aria-label="Permalink to &quot;The Solution&quot;">​</a></h3><p><strong>Reference implementation</strong>: Jeff Emanuel&#39;s <code>coding_agent_session_search</code> (783★)</p><div class="language- vp-adaptive-theme"><button title="Copy Code" class="copy"></button><span class="lang"></span><pre class="shiki shiki-themes github-light github-dark vp-code" tabindex="0"><code><span class="line"><span>Indexer:</span></span>
<span class="line"><span>  ├── Watches all session directories</span></span>
<span class="line"><span>  ├── Normalizes into canonical format</span></span>
<span class="line"><span>  ├── Full-text indexes prompts, responses, tool calls</span></span>
<span class="line"><span>  └── Stores in SQLite with FTS5</span></span>
<span class="line"><span></span></span>
<span class="line"><span>Search CLI:</span></span>
<span class="line"><span>  ├── Search across ALL providers from one command</span></span>
<span class="line"><span>  ├── Filter by: provider, date, model, tool, token count</span></span>
<span class="line"><span>  ├── Replay any session from any provider</span></span>
<span class="line"><span>  └── Export sessions as markdown or JSON</span></span></code></pre></div><h3 id="why-this-matters-for-production" tabindex="-1">Why This Matters for Production <a class="header-anchor" href="#why-this-matters-for-production" aria-label="Permalink to &quot;Why This Matters for Production&quot;">​</a></h3><ol><li><strong>Debugging</strong>: Find what went wrong across all agents in one search</li><li><strong>Audit</strong>: Full history of every agent action, unified format</li><li><strong>Learning</strong>: Search past solutions instead of re-solving problems</li><li><strong>Cost analysis</strong>: Aggregate costs across all providers</li></ol><h3 id="integration-with-our-stack" tabindex="-1">Integration with Our Stack <a class="header-anchor" href="#integration-with-our-stack" aria-label="Permalink to &quot;Integration with Our Stack&quot;">​</a></h3><div class="language-bash vp-adaptive-theme"><button title="Copy Code" class="copy"></button><span class="lang">bash</span><pre class="shiki shiki-themes github-light github-dark vp-code" tabindex="0"><code><span class="line"><span style="--shiki-light:#6A737D;--shiki-dark:#6A737D;"># Add to mprocs-teams.yaml as a sidecar service</span></span>
<span class="line"><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0;">session-indexer:</span></span>
<span class="line"><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0;">  cmd:</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;"> [</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;">&quot;python&quot;</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">, </span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;">&quot;session-indexer.py&quot;,</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> &quot;--watch-dirs&quot;,</span></span>
<span class="line"><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0;">        &quot;~/.claude/sessions/&quot;</span><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0;">,</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> &quot;~/.pi/sessions/&quot;,</span></span>
<span class="line"><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0;">        &quot;~/.opencode/sessions/&quot;</span><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0;">,</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> &quot;logs/&quot;]</span></span>
<span class="line"></span>
<span class="line"><span style="--shiki-light:#6A737D;--shiki-dark:#6A737D;"># Search across everything</span></span>
<span class="line"><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0;">session-search</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> &quot;deployment error&quot;</span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;"> --providers</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> claude,pi,opencode</span></span>
<span class="line"><span style="--shiki-light:#6A737D;--shiki-dark:#6A737D;"># Returns results from all providers in unified format</span></span></code></pre></div><hr><h2 id="lesson-5-7-deployment-modes" tabindex="-1">Lesson 5.7: Deployment Modes <a class="header-anchor" href="#lesson-5-7-deployment-modes" aria-label="Permalink to &quot;Lesson 5.7: Deployment Modes&quot;">​</a></h2><p>From Paperclip&#39;s deployment model:</p><table tabindex="0"><thead><tr><th>Mode</th><th>Auth</th><th>Exposure</th><th>Use Case</th></tr></thead><tbody><tr><td>Local trusted</td><td>None</td><td>localhost only</td><td>Single dev machine</td></tr><tr><td>Authenticated private</td><td>Login required</td><td>LAN/Tailscale/VPN</td><td>Team on private network</td></tr><tr><td>Authenticated public</td><td>Login required</td><td>Internet (behind reverse proxy)</td><td>Production cloud deployment</td></tr></tbody></table><h3 id="reachability" tabindex="-1">Reachability <a class="header-anchor" href="#reachability" aria-label="Permalink to &quot;Reachability&quot;">​</a></h3><table tabindex="0"><thead><tr><th>Bind</th><th>What It Means</th></tr></thead><tbody><tr><td><code>loopback</code></td><td>localhost only (default)</td></tr><tr><td><code>lan</code></td><td>All interfaces (0.0.0.0)</td></tr><tr><td><code>tailnet</code></td><td>Tailscale IP only</td></tr><tr><td><code>custom</code></td><td>Specific host/IP</td></tr></tbody></table><hr><h2 id="lesson-5-8-cost-control" tabindex="-1">Lesson 5.8: Cost Control <a class="header-anchor" href="#lesson-5-8-cost-control" aria-label="Permalink to &quot;Lesson 5.8: Cost Control&quot;">​</a></h2><h3 id="budget-architecture" tabindex="-1">Budget Architecture <a class="header-anchor" href="#budget-architecture" aria-label="Permalink to &quot;Budget Architecture&quot;">​</a></h3><div class="language- vp-adaptive-theme"><button title="Copy Code" class="copy"></button><span class="lang"></span><pre class="shiki shiki-themes github-light github-dark vp-code" tabindex="0"><code><span class="line"><span>Company budget → Agent budgets → Session budgets → Per-call tracking</span></span></code></pre></div><h3 id="budget-policies" tabindex="-1">Budget Policies <a class="header-anchor" href="#budget-policies" aria-label="Permalink to &quot;Budget Policies&quot;">​</a></h3><div class="language-yaml vp-adaptive-theme"><button title="Copy Code" class="copy"></button><span class="lang">yaml</span><pre class="shiki shiki-themes github-light github-dark vp-code" tabindex="0"><code><span class="line"><span style="--shiki-light:#22863A;--shiki-dark:#85E89D;">budget_policies</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">:</span></span>
<span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">  - </span><span style="--shiki-light:#22863A;--shiki-dark:#85E89D;">metric</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">: </span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;">&quot;monthly_cost_cents&quot;</span></span>
<span class="line"><span style="--shiki-light:#22863A;--shiki-dark:#85E89D;">    scope</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">: </span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;">&quot;agent:backend-dev&quot;</span></span>
<span class="line"><span style="--shiki-light:#22863A;--shiki-dark:#85E89D;">    amount</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">: </span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;">50000</span><span style="--shiki-light:#6A737D;--shiki-dark:#6A737D;">  # $500/month</span></span>
<span class="line"><span style="--shiki-light:#22863A;--shiki-dark:#85E89D;">    warn_at</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">: </span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;">80%</span><span style="--shiki-light:#6A737D;--shiki-dark:#6A737D;">   # warn at $400</span></span>
<span class="line"><span style="--shiki-light:#22863A;--shiki-dark:#85E89D;">    hard_stop</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">: </span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;">true</span><span style="--shiki-light:#6A737D;--shiki-dark:#6A737D;"> # kill at $500</span></span>
<span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">  - </span><span style="--shiki-light:#22863A;--shiki-dark:#85E89D;">metric</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">: </span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;">&quot;session_cost_cents&quot;</span></span>
<span class="line"><span style="--shiki-light:#22863A;--shiki-dark:#85E89D;">    scope</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">: </span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;">&quot;global&quot;</span></span>
<span class="line"><span style="--shiki-light:#22863A;--shiki-dark:#85E89D;">    amount</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">: </span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;">200</span><span style="--shiki-light:#6A737D;--shiki-dark:#6A737D;">     # $2/session max</span></span>
<span class="line"><span style="--shiki-light:#22863A;--shiki-dark:#85E89D;">    hard_stop</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">: </span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;">true</span></span></code></pre></div><h3 id="warning-vs-hard-stop" tabindex="-1">Warning vs Hard Stop <a class="header-anchor" href="#warning-vs-hard-stop" aria-label="Permalink to &quot;Warning vs Hard Stop&quot;">​</a></h3><ul><li><strong>Warning</strong> (80%) — Notify operator, agent keeps running</li><li><strong>Hard Stop</strong> (100%) — Agent paused, new tasks queued, running task cancelled</li></ul><hr><h2 id="lab-5-9-set-up-agent-observability" tabindex="-1">Lab 5.9: Set Up Agent Observability <a class="header-anchor" href="#lab-5-9-set-up-agent-observability" aria-label="Permalink to &quot;Lab 5.9: Set Up Agent Observability&quot;">​</a></h2><p><strong>Objective</strong>: Trace every tool call + LLM completion to a local SQLite database.</p><p><strong>Starter</strong>: <code>course/labs/L5-observability/starter/</code></p><h2 id="lab-5-10-ci-cd-pipeline" tabindex="-1">Lab 5.10: CI/CD Pipeline <a class="header-anchor" href="#lab-5-10-ci-cd-pipeline" aria-label="Permalink to &quot;Lab 5.10: CI/CD Pipeline&quot;">​</a></h2><p><strong>Objective</strong>: Create a golden dataset and automated regression gate.</p><p><strong>Starter</strong>: <code>course/labs/L5-cicd/starter/</code></p>`,99)])])}const g=a(e,[["render",l]]);export{k as __pageData,g as default};
