49 lines
3.0 KiB
JSON
49 lines
3.0 KiB
JSON
{
|
|
"schema": "fable.research_map.receipt.v1",
|
|
"createdAt": "2026-07-04T22:05:00.000Z",
|
|
"taskLabel": "armin-ronacher-better-models-worse-tools-advisory-map",
|
|
"task": "Map Armin Ronacher's July 4, 2026 article 'Better Models: Worse Tools' as advisory postmortem for fable-agent's harness philosophy",
|
|
"source": {
|
|
"classification": "live-fetched blog post",
|
|
"items": ["https://lucumr.pocoo.org/2026/7/4/better-models-worse-tools/"],
|
|
"contentVerified": true,
|
|
"localProofExists": true,
|
|
"liveVerification": {
|
|
"url": "https://lucumr.pocoo.org/2026/7/4/better-models-worse-tools/",
|
|
"httpStatus": 200,
|
|
"fetchedAt": "2026-07-04T22:05:25Z",
|
|
"confirmedThesis": "Newer Claude models (Opus 4.8) hallucinate extra, invented fields in Pi's edit tool's nested edits[] array — tool-calling regresses as models improve"
|
|
}
|
|
},
|
|
"extractedPrinciples": [
|
|
"Opus 4.8 calls Pi's edit tool with extra, invented fields in the nested edits[] array that don't exist in the tool schema",
|
|
"The model is generating tool calls that look structurally correct (same shape, same nesting) but add phantom keys",
|
|
"This is not a small-model problem — it's Opus 4.8, Anthropic's frontier model",
|
|
"The regression is intermittent — same prompt sometimes works, sometimes adds junk fields",
|
|
"Pi's tool schema is stricter than what the model chooses to send; the model is 'over-generating'",
|
|
"The practical consequence: better models produce more believable but subtly wrong tool invocations",
|
|
"This validates the Dario blueprint's hard verification gate approach — don't trust the model's output, verify every tool call against schema"
|
|
],
|
|
"constraints": [
|
|
"Article is advisory; Armin's specific Pi bug may not reproduce in fable-agent's tool harness",
|
|
"The principle generalizes: verify agent tool calls against schema, don't assume correct shape",
|
|
"fable-agent's verify-gate (build-test-lint) catches code errors but does not validate tool call schema compliance",
|
|
"The flat-ledger (JSONL state) and repo-mapper are structural mitigations — agent reads state, doesn't remember it",
|
|
"No deploy, vendoring, dashboards, or new packages were performed for this mapping"
|
|
],
|
|
"fableAgentAlignment": [
|
|
"fable-agent's verify-gate already rejects builds that fail; tool-call schema validation would be a new layer",
|
|
"The flat-ledger pattern (state outside LLM context) reduces hallucination surface area",
|
|
"The interrupt-gate (human approval before live actions) catches bad tool invocations before they execute",
|
|
"Worktree isolation (WorktreeManager) means bad tool calls only damage disposable worktrees, not main branches",
|
|
"Dario blueprint's 'hard automated verification' is directly validated by this article — models get smarter, tool-calling gets sloppier, so verify harder"
|
|
],
|
|
"skipped": [
|
|
"deploy actions or route checks",
|
|
"new dependency changes"
|
|
],
|
|
"decision": "advisory",
|
|
"deployAttempted": false,
|
|
"autoPatchProven": false
|
|
}
|