diff --git a/src/upgrades/rsi-guard.ts b/src/upgrades/rsi-guard.ts new file mode 100644 index 0000000..4b409ab --- /dev/null +++ b/src/upgrades/rsi-guard.ts @@ -0,0 +1,287 @@ +/** + * RSI Guard — monitors for recursive self-improvement failure modes. + * + * Sources: + * Sakana AI — RSI Lab (rsi-lab.sakana.ai) + * Anthropic — "When AI builds itself" (anthropic.com/institute/recursive-self-improvement) + * Stephen Wolfram — "Games between Programs" (writings.stephenwolfram.com) + * + * Three RSI bottlenecks this module addresses: + * + * 1. KNOWLEDGE COLLAPSE — retraining on synthetic data makes AI "confidently wrong." + * Solution: Track source entropy. If >60% of recent inputs are self-generated, + * flag knowledge collapse risk and inject external data from verified sources. + * + * 2. THE TASTE GAP — AI can run experiments but struggles with "scientific taste" + * (intuition for which ideas are breakthroughs). + * Solution: Rubric-based idea selection with novelty + feasibility + impact criteria. + * The rubric is the "scientific taste" function. + * + * 3. PLANNING FAILURES — long-horizon tasks break down due to catastrophic forgetting + * of original goals. + * Solution: Goal preservation — snapshot the original goal at plan time and check + * each iteration's output against it (not just the rubric). + */ + +import { StateStore } from "../core/state-store.js"; +import type { LoopIteration } from "../core/types.js"; + +// ── Knowledge Collapse Detection ───────────────────────── + +export interface SourceEntry { + timestamp: string; + sourceType: "external" | "self_generated" | "verified_fact"; + content: string; + sourceUrl?: string; +} + +export interface KnowledgeCollapseRisk { + risk: "low" | "medium" | "high" | "critical"; + selfGeneratedRatio: number; + totalSources: number; + recommendation: string; +} + +export class KnowledgeCollapseDetector { + private store: StateStore; + + constructor(store?: StateStore) { + this.store = store ?? new StateStore(); + this.store.ensureSubDir("rsi"); + } + + /** Record a source of information */ + recordSource(entry: SourceEntry): void { + this.store.append("rsi", "sources.jsonl", entry); + } + + /** Assess knowledge collapse risk */ + assess(): KnowledgeCollapseRisk { + const sources = this.store.readLines("rsi", "sources.jsonl") + .slice(-100); // Last 100 sources + + if (sources.length === 0) { + return { risk: "low", selfGeneratedRatio: 0, totalSources: 0, recommendation: "Not enough data" }; + } + + const selfGenerated = sources.filter((s) => s.sourceType === "self_generated").length; + const ratio = selfGenerated / sources.length; + + let risk: KnowledgeCollapseRisk["risk"]; + let recommendation: string; + + if (ratio > 0.8) { + risk = "critical"; + recommendation = "CRITICAL: >80% self-generated data. Inject external verified facts immediately. Run Exa search to diversify sources."; + } else if (ratio > 0.6) { + risk = "high"; + recommendation = "High risk: >60% self-generated data. Reduce dreaming cycles, increase external research."; + } else if (ratio > 0.4) { + risk = "medium"; + recommendation = "Moderate risk. Balance self-generated iterations with external data sources."; + } else { + risk = "low"; + recommendation = "Healthy source diversity. Continue maintaining external input ratio."; + } + + return { risk, selfGeneratedRatio: ratio, totalSources: sources.length, recommendation }; + } + + /** Log a dream cycle result (counts as self-generated) */ + logDreamCycle(insights: string[]): void { + for (const insight of insights) { + this.recordSource({ + timestamp: new Date().toISOString(), + sourceType: "self_generated", + content: insight, + }); + } + } + + /** Get a human-readable report */ + getReport(): string { + const assessment = this.assess(); + return [ + "Knowledge Collapse Risk Assessment", + "", + ` Risk level: ${assessment.risk.toUpperCase()}`, + ` Self-generated ratio: ${(assessment.selfGeneratedRatio * 100).toFixed(0)}%`, + ` Total sources tracked: ${assessment.totalSources}`, + "", + ` Recommendation: ${assessment.recommendation}`, + "", + " Mitigation:", + " - Use Exa search to inject external data", + " - Prioritize verified facts over generated insights", + " - Reduce dreaming frequency when ratio exceeds 60%", + ].join("\n"); + } +} + +// ── Scientific Taste Filter ───────────────────────────── + +export interface IdeaCandidate { + id: string; + title: string; + description: string; + novelty: number; // 0-1: how new is this idea + feasibility: number; // 0-1: how buildable is it + impact: number; // 0-1: how impactful if successful + evidenceLevel: number; // 0-1: how much evidence supports it +} + +export interface TasteVerdict { + candidate: IdeaCandidate; + tasteScore: number; + selected: boolean; + reason: string; +} + +export class ScientificTasteFilter { + /** Evaluate an idea candidate using "scientific taste" criteria */ + evaluate(candidate: IdeaCandidate): TasteVerdict { + const tasteScore = ( + candidate.novelty * 0.25 + + candidate.feasibility * 0.20 + + candidate.impact * 0.35 + + candidate.evidenceLevel * 0.20 + ); + + const selected = tasteScore > 0.6; + + const strengths: string[] = []; + const weaknesses: string[] = []; + + if (candidate.novelty > 0.7) strengths.push("high novelty — not tried before"); + else weaknesses.push("low novelty — similar approaches may already exist"); + + if (candidate.feasibility > 0.6) strengths.push("feasible with current architecture"); + else weaknesses.push("may require architectural changes"); + + if (candidate.impact > 0.7) strengths.push("high potential impact"); + else weaknesses.push("limited impact if successful"); + + if (candidate.evidenceLevel > 0.5) strengths.push("supported by evidence"); + else weaknesses.push("speculative — limited evidence"); + + const reason = selected + ? `SELECTED (score: ${(tasteScore * 100).toFixed(0)}). ${strengths.join("; ")}.` + : `REJECTED (score: ${(tasteScore * 100).toFixed(0)}). ${weaknesses.join("; ")}. Improve feasibility or gather evidence.`; + + return { candidate, tasteScore, selected, reason }; + } + + /** Evaluate multiple candidates, return the best */ + selectBest(candidates: IdeaCandidate[], maxResults: number = 1): TasteVerdict[] { + const evaluated = candidates.map((c) => this.evaluate(c)); + return evaluated + .filter((v) => v.selected) + .sort((a, b) => b.tasteScore - a.tasteScore) + .slice(0, maxResults); + } +} + +// ── Goal Persistence ──────────────────────────────────── + +export interface GoalSnapshot { + id: string; + originalGoal: string; + rubricCriteria: string[]; + createdAt: string; + iterations: number; + status: "tracking" | "drifted" | "completed" | "abandoned"; + lastCheckAt: string; + driftSignals: string[]; +} + +export class GoalPersistenceGuard { + private store: StateStore; + + constructor(store?: StateStore) { + this.store = store ?? new StateStore(); + this.store.ensureSubDir("rsi"); + } + + /** Snapshot the original goal at the start of a loop */ + snapshot(goal: string, rubricCriteria: string[]): GoalSnapshot { + const snap: GoalSnapshot = { + id: StateStore.uid().slice(0, 8), + originalGoal: goal, + rubricCriteria, + createdAt: new Date().toISOString(), + iterations: 0, + status: "tracking", + lastCheckAt: new Date().toISOString(), + driftSignals: [], + }; + this.store.write("rsi", `goal-${snap.id}.json`, snap); + return snap; + } + + /** Check an iteration's output against the original goal */ + checkForDrift(snapshotId: string, iteration: LoopIteration): GoalSnapshot { + const snap = this.store.read("rsi", `goal-${snapshotId}.json`); + if (!snap) throw new Error(`Goal snapshot not found: ${snapshotId}`); + + snap.iterations++; + snap.lastCheckAt = new Date().toISOString(); + + // Check if the latest output still references the original goal + const originalLower = snap.originalGoal.toLowerCase(); + const planLower = (iteration.plan ?? "").toLowerCase(); + const refinementLower = (iteration.refinement ?? "").toLowerCase(); + + // If the plan no longer mentions key terms from the original goal, flag drift + const goalWords = originalLower.split(/\s+/).filter((w) => w.length > 4); + const planMentions = goalWords.filter((w) => planLower.includes(w)).length; + const refinementMentions = goalWords.filter((w) => refinementLower.includes(w)).length; + + const mentionRatio = Math.max(planMentions, refinementMentions) / Math.max(goalWords.length, 1); + + if (mentionRatio < 0.2 && snap.iterations > 2) { + snap.driftSignals.push( + `Iteration ${snap.iterations}: Goal term mention dropped to ${(mentionRatio * 100).toFixed(0)}%` + ); + } + + if (snap.driftSignals.length >= 2) { + snap.status = "drifted"; + } + + this.store.write("rsi", `goal-${snap.id}.json`, snap); + return snap; + } + + /** Check if a goal has drifted */ + hasDrifted(snapshotId: string): boolean { + const snap = this.store.read("rsi", `goal-${snapshotId}.json`); + return snap?.status === "drifted"; + } + + /** Get drift report */ + getDriftReport(): string { + const files = this.store.list("rsi").filter((f) => f.startsWith("goal-")); + const drifted: GoalSnapshot[] = []; + + for (const f of files) { + const snap = this.store.read("rsi", f); + if (snap && snap.status === "drifted") drifted.push(snap); + } + + if (drifted.length === 0) { + return "No goal drift detected. All goals on track."; + } + + const lines = ["Goal Drift Report", ""]; + for (const d of drifted) { + lines.push(` GOAL: ${d.originalGoal.slice(0, 80)}`); + lines.push(` Status: ${d.status}`); + lines.push(` Iterations: ${d.iterations}`); + for (const signal of d.driftSignals) { + lines.push(` ⚠ ${signal}`); + } + lines.push(""); + } + return lines.join("\n"); + } +}