3 lines
2.4 KiB
JavaScript
3 lines
2.4 KiB
JavaScript
import{c as e,Q as s,j as t,m as i}from"./chunks/framework.BPKcPtvA.js";const u=JSON.parse('{"title":"L6a: Eval Harness","description":"","frontmatter":{},"headers":[],"relativePath":"labs/l6-eval-harness.md","filePath":"labs/l6-eval-harness.md","lastUpdated":1780488246000}'),l={name:"labs/l6-eval-harness.md"};function n(o,a,r,c,p,h){return s(),t("div",null,[...a[0]||(a[0]=[i(`<h1 id="l6a-eval-harness" tabindex="-1">L6a: Eval Harness <a class="header-anchor" href="#l6a-eval-harness" aria-label="Permalink to "L6a: Eval Harness""></a></h1><p><strong>Module</strong>: M6 Economics<br><strong>Files</strong>: starter.py, solution.py</p><h2 id="objective" tabindex="-1">Objective <a class="header-anchor" href="#objective" aria-label="Permalink to "Objective""></a></h2><p>Build golden Q&A pairs + pass@k scoring system.</p><h2 id="concepts" tabindex="-1">Concepts <a class="header-anchor" href="#concepts" aria-label="Permalink to "Concepts""></a></h2><p>Refer to the corresponding module for full concept explanations.</p><h2 id="starter" tabindex="-1">Starter <a class="header-anchor" href="#starter" aria-label="Permalink to "Starter""></a></h2><div class="language-bash vp-adaptive-theme"><button title="Copy Code" class="copy"></button><span class="lang">bash</span><pre class="shiki shiki-themes github-light github-dark vp-code" tabindex="0"><code><span class="line"><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;">cd</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> course/labs/l6-eval-harness/</span></span>
|
||
<span class="line"><span style="--shiki-light:#6A737D;--shiki-dark:#6A737D;"># Open starter file and fill in the TODOs</span></span></code></pre></div><h2 id="checkpoints" tabindex="-1">Checkpoints <a class="header-anchor" href="#checkpoints" aria-label="Permalink to "Checkpoints""></a></h2><ol><li>Define 10+ test cases with tool and output expectations</li><li>Implement EvalHarness class with score_case</li><li>Compute pass@1, pass@3, pass@5 metrics</li><li>Support weighted scoring by case importance</li></ol><h2 id="solution" tabindex="-1">Solution <a class="header-anchor" href="#solution" aria-label="Permalink to "Solution""></a></h2><p>Compare against the solution file after attempting the starter.</p>`,12)])])}const b=e(l,[["render",n]]);export{u as __pageData,b as default};
|