add rsi-guard: knowledge collapse detection, scientific taste filter, goal persistence

src/upgrades/rsi-guard.ts

@@ -0,0 +1,287 @@
+/**
+ * RSI Guard — monitors for recursive self-improvement failure modes.
+ *
+ * Sources:
+ *   Sakana AI — RSI Lab (rsi-lab.sakana.ai)
+ *   Anthropic — "When AI builds itself" (anthropic.com/institute/recursive-self-improvement)
+ *   Stephen Wolfram — "Games between Programs" (writings.stephenwolfram.com)
+ *
+ * Three RSI bottlenecks this module addresses:
+ *
+ *   1. KNOWLEDGE COLLAPSE — retraining on synthetic data makes AI "confidently wrong."
+ *      Solution: Track source entropy. If >60% of recent inputs are self-generated,
+ *      flag knowledge collapse risk and inject external data from verified sources.
+ *
+ *   2. THE TASTE GAP — AI can run experiments but struggles with "scientific taste"
+ *      (intuition for which ideas are breakthroughs).
+ *      Solution: Rubric-based idea selection with novelty + feasibility + impact criteria.
+ *      The rubric is the "scientific taste" function.
+ *
+ *   3. PLANNING FAILURES — long-horizon tasks break down due to catastrophic forgetting
+ *      of original goals.
+ *      Solution: Goal preservation — snapshot the original goal at plan time and check
+ *      each iteration's output against it (not just the rubric).
+ */
+
+import { StateStore } from "../core/state-store.js";
+import type { LoopIteration } from "../core/types.js";
+
+// ── Knowledge Collapse Detection ─────────────────────────
+
+export interface SourceEntry {
+  timestamp: string;
+  sourceType: "external" | "self_generated" | "verified_fact";
+  content: string;
+  sourceUrl?: string;
+}
+
+export interface KnowledgeCollapseRisk {
+  risk: "low" | "medium" | "high" | "critical";
+  selfGeneratedRatio: number;
+  totalSources: number;
+  recommendation: string;
+}
+
+export class KnowledgeCollapseDetector {
+  private store: StateStore;
+
+  constructor(store?: StateStore) {
+    this.store = store ?? new StateStore();
+    this.store.ensureSubDir("rsi");
+  }
+
+  /** Record a source of information */
+  recordSource(entry: SourceEntry): void {
+    this.store.append("rsi", "sources.jsonl", entry);
+  }
+
+  /** Assess knowledge collapse risk */
+  assess(): KnowledgeCollapseRisk {
+    const sources = this.store.readLines<SourceEntry>("rsi", "sources.jsonl")
+      .slice(-100); // Last 100 sources
+
+    if (sources.length === 0) {
+      return { risk: "low", selfGeneratedRatio: 0, totalSources: 0, recommendation: "Not enough data" };
+    }
+
+    const selfGenerated = sources.filter((s) => s.sourceType === "self_generated").length;
+    const ratio = selfGenerated / sources.length;
+
+    let risk: KnowledgeCollapseRisk["risk"];
+    let recommendation: string;
+
+    if (ratio > 0.8) {
+      risk = "critical";
+      recommendation = "CRITICAL: >80% self-generated data. Inject external verified facts immediately. Run Exa search to diversify sources.";
+    } else if (ratio > 0.6) {
+      risk = "high";
+      recommendation = "High risk: >60% self-generated data. Reduce dreaming cycles, increase external research.";
+    } else if (ratio > 0.4) {
+      risk = "medium";
+      recommendation = "Moderate risk. Balance self-generated iterations with external data sources.";
+    } else {
+      risk = "low";
+      recommendation = "Healthy source diversity. Continue maintaining external input ratio.";
+    }
+
+    return { risk, selfGeneratedRatio: ratio, totalSources: sources.length, recommendation };
+  }
+
+  /** Log a dream cycle result (counts as self-generated) */
+  logDreamCycle(insights: string[]): void {
+    for (const insight of insights) {
+      this.recordSource({
+        timestamp: new Date().toISOString(),
+        sourceType: "self_generated",
+        content: insight,
+      });
+    }
+  }
+
+  /** Get a human-readable report */
+  getReport(): string {
+    const assessment = this.assess();
+    return [
+      "Knowledge Collapse Risk Assessment",
+      "",
+      `  Risk level: ${assessment.risk.toUpperCase()}`,
+      `  Self-generated ratio: ${(assessment.selfGeneratedRatio * 100).toFixed(0)}%`,
+      `  Total sources tracked: ${assessment.totalSources}`,
+      "",
+      `  Recommendation: ${assessment.recommendation}`,
+      "",
+      "  Mitigation:",
+      "    - Use Exa search to inject external data",
+      "    - Prioritize verified facts over generated insights",
+      "    - Reduce dreaming frequency when ratio exceeds 60%",
+    ].join("\n");
+  }
+}
+
+// ── Scientific Taste Filter ─────────────────────────────
+
+export interface IdeaCandidate {
+  id: string;
+  title: string;
+  description: string;
+  novelty: number;       // 0-1: how new is this idea
+  feasibility: number;   // 0-1: how buildable is it
+  impact: number;        // 0-1: how impactful if successful
+  evidenceLevel: number; // 0-1: how much evidence supports it
+}
+
+export interface TasteVerdict {
+  candidate: IdeaCandidate;
+  tasteScore: number;
+  selected: boolean;
+  reason: string;
+}
+
+export class ScientificTasteFilter {
+  /** Evaluate an idea candidate using "scientific taste" criteria */
+  evaluate(candidate: IdeaCandidate): TasteVerdict {
+    const tasteScore = (
+      candidate.novelty * 0.25 +
+      candidate.feasibility * 0.20 +
+      candidate.impact * 0.35 +
+      candidate.evidenceLevel * 0.20
+    );
+
+    const selected = tasteScore > 0.6;
+
+    const strengths: string[] = [];
+    const weaknesses: string[] = [];
+
+    if (candidate.novelty > 0.7) strengths.push("high novelty — not tried before");
+    else weaknesses.push("low novelty — similar approaches may already exist");
+
+    if (candidate.feasibility > 0.6) strengths.push("feasible with current architecture");
+    else weaknesses.push("may require architectural changes");
+
+    if (candidate.impact > 0.7) strengths.push("high potential impact");
+    else weaknesses.push("limited impact if successful");
+
+    if (candidate.evidenceLevel > 0.5) strengths.push("supported by evidence");
+    else weaknesses.push("speculative — limited evidence");
+
+    const reason = selected
+      ? `SELECTED (score: ${(tasteScore * 100).toFixed(0)}). ${strengths.join("; ")}.`
+      : `REJECTED (score: ${(tasteScore * 100).toFixed(0)}). ${weaknesses.join("; ")}. Improve feasibility or gather evidence.`;
+
+    return { candidate, tasteScore, selected, reason };
+  }
+
+  /** Evaluate multiple candidates, return the best */
+  selectBest(candidates: IdeaCandidate[], maxResults: number = 1): TasteVerdict[] {
+    const evaluated = candidates.map((c) => this.evaluate(c));
+    return evaluated
+      .filter((v) => v.selected)
+      .sort((a, b) => b.tasteScore - a.tasteScore)
+      .slice(0, maxResults);
+  }
+}
+
+// ── Goal Persistence ────────────────────────────────────
+
+export interface GoalSnapshot {
+  id: string;
+  originalGoal: string;
+  rubricCriteria: string[];
+  createdAt: string;
+  iterations: number;
+  status: "tracking" | "drifted" | "completed" | "abandoned";
+  lastCheckAt: string;
+  driftSignals: string[];
+}
+
+export class GoalPersistenceGuard {
+  private store: StateStore;
+
+  constructor(store?: StateStore) {
+    this.store = store ?? new StateStore();
+    this.store.ensureSubDir("rsi");
+  }
+
+  /** Snapshot the original goal at the start of a loop */
+  snapshot(goal: string, rubricCriteria: string[]): GoalSnapshot {
+    const snap: GoalSnapshot = {
+      id: StateStore.uid().slice(0, 8),
+      originalGoal: goal,
+      rubricCriteria,
+      createdAt: new Date().toISOString(),
+      iterations: 0,
+      status: "tracking",
+      lastCheckAt: new Date().toISOString(),
+      driftSignals: [],
+    };
+    this.store.write("rsi", `goal-${snap.id}.json`, snap);
+    return snap;
+  }
+
+  /** Check an iteration's output against the original goal */
+  checkForDrift(snapshotId: string, iteration: LoopIteration): GoalSnapshot {
+    const snap = this.store.read<GoalSnapshot>("rsi", `goal-${snapshotId}.json`);
+    if (!snap) throw new Error(`Goal snapshot not found: ${snapshotId}`);
+
+    snap.iterations++;
+    snap.lastCheckAt = new Date().toISOString();
+
+    // Check if the latest output still references the original goal
+    const originalLower = snap.originalGoal.toLowerCase();
+    const planLower = (iteration.plan ?? "").toLowerCase();
+    const refinementLower = (iteration.refinement ?? "").toLowerCase();
+
+    // If the plan no longer mentions key terms from the original goal, flag drift
+    const goalWords = originalLower.split(/\s+/).filter((w) => w.length > 4);
+    const planMentions = goalWords.filter((w) => planLower.includes(w)).length;
+    const refinementMentions = goalWords.filter((w) => refinementLower.includes(w)).length;
+
+    const mentionRatio = Math.max(planMentions, refinementMentions) / Math.max(goalWords.length, 1);
+
+    if (mentionRatio < 0.2 && snap.iterations > 2) {
+      snap.driftSignals.push(
+        `Iteration ${snap.iterations}: Goal term mention dropped to ${(mentionRatio * 100).toFixed(0)}%`
+      );
+    }
+
+    if (snap.driftSignals.length >= 2) {
+      snap.status = "drifted";
+    }
+
+    this.store.write("rsi", `goal-${snap.id}.json`, snap);
+    return snap;
+  }
+
+  /** Check if a goal has drifted */
+  hasDrifted(snapshotId: string): boolean {
+    const snap = this.store.read<GoalSnapshot>("rsi", `goal-${snapshotId}.json`);
+    return snap?.status === "drifted";
+  }
+
+  /** Get drift report */
+  getDriftReport(): string {
+    const files = this.store.list("rsi").filter((f) => f.startsWith("goal-"));
+    const drifted: GoalSnapshot[] = [];
+
+    for (const f of files) {
+      const snap = this.store.read<GoalSnapshot>("rsi", f);
+      if (snap && snap.status === "drifted") drifted.push(snap);
+    }
+
+    if (drifted.length === 0) {
+      return "No goal drift detected. All goals on track.";
+    }
+
+    const lines = ["Goal Drift Report", ""];
+    for (const d of drifted) {
+      lines.push(`  GOAL: ${d.originalGoal.slice(0, 80)}`);
+      lines.push(`  Status: ${d.status}`);
+      lines.push(`  Iterations: ${d.iterations}`);
+      for (const signal of d.driftSignals) {
+        lines.push(`    ⚠ ${signal}`);
+      }
+      lines.push("");
+    }
+    return lines.join("\n");
+  }
+}