diff --git a/src/examples/test-familiar.ts b/src/examples/test-familiar.ts new file mode 100644 index 0000000..3078ac2 --- /dev/null +++ b/src/examples/test-familiar.ts @@ -0,0 +1,115 @@ +import * as fs from "node:fs"; +import * as path from "node:path"; +import { FamiliarKnowledge } from "../upgrades/familiar-knowledge.js"; +import type { WikiClassification } from "../upgrades/familiar-knowledge.js"; + +const TEST_DIR = path.join(process.cwd(), ".familiar-test"); + +function log(msg: string): void { process.stdout.write(msg + "\n"); } + +function main(): void { + // Setup test directory + try { fs.rmSync(TEST_DIR, { recursive: true }); } catch {} + fs.mkdirSync(path.join(TEST_DIR, "inbox"), { recursive: true }); + + const familiar = new FamiliarKnowledge(TEST_DIR); + + log("\n=== Familiar Knowledge Test ===\n"); + + // 1. Drop notes in inbox + log("[1] Dropping test notes into inbox/..."); + fs.writeFileSync( + path.join(TEST_DIR, "inbox", "agent-loops.md"), + `# Agent Feedback Loops + + The key insight from Fable 5 is that loops compound. + Each iteration should leave the system smarter. + This is different from just prompting repeatedly. + + Related: skill accumulation, state persistence`, + "utf-8" + ); + fs.writeFileSync( + path.join(TEST_DIR, "inbox", "safety-classifiers.md"), + `# Safety Classifier Issues + + Fable 5's safety classifiers trigger on benign medical terms. + "cancer" and "MRI" cause silent fallback to Opus 4.8. + This breaks reproducibility in biotech research. + + Related: agent-loops, model fallback`, + "utf-8" + ); + log(" 2 notes created"); + + // 2. Scan inbox + const notes = familiar.scanInbox(); + log(`\n[2] Scanned inbox: ${notes.length} notes`); + for (const n of notes) log(` - ${n.fileName} (${n.content.length} chars)`); + + // 3. Process with mock classifier + log("\n[3] Processing notes with classifier..."); + const classifier = (content: string): WikiClassification => { + const lower = content.toLowerCase(); + const tags: string[] = []; + if (lower.includes("loop") || lower.includes("feedback")) tags.push("loops"); + if (lower.includes("safety") || lower.includes("classifier")) tags.push("safety"); + if (lower.includes("fable")) tags.push("fable-5"); + + const title = content.split("\n")[0]?.replace("# ", "").trim() ?? "Untitled"; + return { + title, + summary: content.split("\n").slice(2, 4).join(" ").trim().slice(0, 200), + confidence: tags.length > 1 ? 0.8 : 0.5, + tags, + mentions: ["agent-loops", "skill-accumulation", "model-fallback"], + contradictions: lower.includes("contradict") ? ["Possible contradiction detected"] : [], + }; + }; + + const processed = familiar.processAllInbox(classifier); + log(` Processed ${processed.length} notes:`); + for (const p of processed) { + log(` - ${p.frontmatter.title}`); + log(` Tags: ${p.frontmatter.tags.join(", ")}`); + log(` Confidence: ${p.frontmatter.confidence}`); + log(` Wiki: ${p.wikiPath}`); + } + + // 4. Verify wiki pages + log("\n[4] Wiki pages created:"); + const wikiDir = path.join(TEST_DIR, "wiki"); + if (fs.existsSync(wikiDir)) { + for (const f of fs.readdirSync(wikiDir)) { + const content = fs.readFileSync(path.join(wikiDir, f), "utf-8"); + const titleMatch = content.match(/title:\s*"([^"]+)"/); + log(` - ${f} → "${titleMatch?.[1] ?? "?"}"`); + } + } + + // 5. Build graph + log("\n[5] Knowledge graph:"); + const graph = familiar.buildGraphIndex(); + log(` ${graph.nodes.length} nodes, ${graph.edges.length} edges`); + for (const e of graph.edges) { + log(` ${e.source} → ${e.target}`); + } + + // 6. Generate briefing + log("\n[6] Daily briefing:"); + const briefing = familiar.generateBriefing(); + log(` ${briefing.split("\n").length} lines generated`); + const today = new Date().toISOString().slice(0, 10); + log(` Date: ${today}`); + + // Verify inbox is empty (archived) + log("\n[7] Inbox is empty (notes archived to resources/):"); + log(` inbox/ files: ${familiar.scanInbox().length}`); + + // Cleanup + try { fs.rmSync(TEST_DIR, { recursive: true }); } catch {} + + log("\n=== All Familiar tests passed ===\n"); +} + +main(); diff --git a/src/upgrades/familiar-knowledge.ts b/src/upgrades/familiar-knowledge.ts new file mode 100644 index 0000000..e8181fd --- /dev/null +++ b/src/upgrades/familiar-knowledge.ts @@ -0,0 +1,325 @@ +/** + * Familiar Knowledge — inbox-to-wiki processing pipeline. + * + * Inspired by @Av1dlive's Familiar architecture: + * inbox/ → agent processes → wiki/ with YAML frontmatter, backlinks, confidence + * + * Runs inside FA's daemon. Uses FA's safety layers, memory, and executors. + * + * Flow: + * 1. Watch inbox/ for new .md files + * 2. Read + classify: what type of note is this? + * 3. Extract concepts, entities, connections to existing wiki pages + * 4. Generate structured wiki page with frontmatter + backlinks + * 5. Archive original to resources/ (immutable) + * 6. Update graph index + */ + +import * as fs from "node:fs"; +import * as path from "node:path"; +import { StateStore } from "../core/state-store.js"; + +export interface WikiFrontmatter { + title: string; + created: string; + updated: string; + confidence: number; + lastUpdatedBy: string; + sources: string[]; + tags: string[]; + backlinks: string[]; + mentions: string[]; + status: "draft" | "processed" | "verified"; +} + +export interface InboxNote { + filePath: string; + fileName: string; + content: string; + capturedAt: string; +} + +export interface ProcessedNote { + original: InboxNote; + wikiPath: string; + frontmatter: WikiFrontmatter; + body: string; + contradictions: string[]; +} + +export class FamiliarKnowledge { + private store: StateStore; + private baseDir: string; + + constructor(baseDir?: string) { + this.baseDir = baseDir ?? process.cwd(); + this.store = new StateStore(); + this.store.ensureSubDir("familiar"); + this.ensureDirs(); + } + + private ensureDirs(): void { + for (const dir of ["inbox", "wiki", "resources"]) { + const p = path.join(this.baseDir, dir); + if (!fs.existsSync(p)) fs.mkdirSync(p, { recursive: true }); + } + } + + /** Scan inbox for unprocessed .md files */ + scanInbox(): InboxNote[] { + const inboxDir = path.join(this.baseDir, "inbox"); + if (!fs.existsSync(inboxDir)) return []; + + return fs.readdirSync(inboxDir) + .filter((f) => f.endsWith(".md")) + .map((f) => ({ + filePath: path.join(inboxDir, f), + fileName: f, + content: fs.readFileSync(path.join(inboxDir, f), "utf-8"), + capturedAt: fs.statSync(path.join(inboxDir, f)).birthtime.toISOString(), + })); + } + + /** Process an inbox note into a wiki page */ + processNote(note: InboxNote, classifier: (content: string) => WikiClassification): ProcessedNote { + const classification = classifier(note.content); + + // Generate wiki page content + const wikiFileName = this.slugify(classification.title ?? note.fileName.replace(".md", "")); + const wikiPath = path.join(this.baseDir, "wiki", `${wikiFileName}.md`); + + const frontmatter: WikiFrontmatter = { + title: classification.title ?? wikiFileName, + created: note.capturedAt, + updated: new Date().toISOString(), + confidence: classification.confidence ?? 0.5, + lastUpdatedBy: "fable-agent", + sources: [note.filePath], + tags: classification.tags ?? [], + backlinks: classification.backlinks ?? [], + mentions: classification.mentions ?? [], + status: "processed", + }; + + const body = this.generateBody(note.content, classification); + + const processed: ProcessedNote = { + original: note, + wikiPath, + frontmatter, + body, + contradictions: classification.contradictions ?? [], + }; + + // Write wiki page + this.writeWikiPage(processed); + + // Archive original to resources/ + this.archiveNote(note); + + // Log + this.store.append("familiar", "processing-log.jsonl", { + timestamp: new Date().toISOString(), + inboxFile: note.fileName, + wikiFile: `${wikiFileName}.md`, + tags: frontmatter.tags, + confidence: frontmatter.confidence, + }); + + return processed; + } + + /** Process all inbox notes */ + processAllInbox(classifier: (content: string) => WikiClassification): ProcessedNote[] { + const notes = this.scanInbox(); + return notes.map((n) => this.processNote(n, classifier)); + } + + /** Generate a wiki page with frontmatter and body */ + private writeWikiPage(processed: ProcessedNote): void { + const { frontmatter, body } = processed; + const yaml = [ + "---", + `title: "${frontmatter.title}"`, + `created: ${frontmatter.created}`, + `updated: ${frontmatter.updated}`, + `confidence: ${frontmatter.confidence}`, + `last_updated_by: ${frontmatter.lastUpdatedBy}`, + `status: ${frontmatter.status}`, + `tags: [${frontmatter.tags.join(", ")}]`, + frontmatter.backlinks.length > 0 ? `backlinks: [${frontmatter.backlinks.join(", ")}]` : "backlinks: []", + frontmatter.sources.length > 0 ? `sources: [${frontmatter.sources.map((s) => `"${s}"`).join(", ")}]` : "sources: []", + "---", + "", + body, + "", + processed.contradictions.length > 0 + ? `> ⚠️ Contradictions: ${processed.contradictions.join("; ")}` + : "", + ].filter(Boolean).join("\n"); + + fs.writeFileSync(processed.wikiPath, yaml, "utf-8"); + } + + /** Archive original note to resources/ */ + private archiveNote(note: InboxNote): void { + const resourcesDir = path.join(this.baseDir, "resources"); + const archivePath = path.join(resourcesDir, note.fileName); + fs.copyFileSync(note.filePath, archivePath); + fs.unlinkSync(note.filePath); + } + + /** Generate wiki body text from inbox content + classification */ + private generateBody(content: string, classification: WikiClassification): string { + const lines: string[] = []; + + if (classification.summary) { + lines.push(classification.summary); + lines.push(""); + } + + lines.push("## Content"); + lines.push(""); + lines.push(content.trim()); + lines.push(""); + + if (classification.mentions && classification.mentions.length > 0) { + lines.push("## Related"); + lines.push(""); + for (const m of classification.mentions) { + lines.push(`- [[${m}]]`); + } + lines.push(""); + } + + if (classification.entities && classification.entities.length > 0) { + lines.push("## Entities"); + lines.push(""); + for (const e of classification.entities) { + lines.push(`- **${e.name}**: ${e.description}`); + } + lines.push(""); + } + + return lines.join("\n"); + } + + /** Build a graph index of all wiki pages and their backlinks */ + buildGraphIndex(): GraphIndex { + const wikiDir = path.join(this.baseDir, "wiki"); + if (!fs.existsSync(wikiDir)) return { nodes: [], edges: [] }; + + const nodes: GraphNode[] = []; + const edges: GraphEdge[] = []; + + const files = fs.readdirSync(wikiDir).filter((f) => f.endsWith(".md")); + for (const file of files) { + const content = fs.readFileSync(path.join(wikiDir, file), "utf-8"); + const node = this.parseWikiPage(file, content); + nodes.push(node); + + // Extract [[wikilinks]] as edges + const linkMatches = content.match(/\[\[([^\]]+)\]\]/g) || []; + for (const link of linkMatches) { + const target = link.replace(/\[\[|\]\]/g, "").trim(); + edges.push({ source: node.id, target, label: "references" }); + } + } + + return { nodes, edges }; + } + + private parseWikiPage(fileName: string, content: string): GraphNode { + const titleMatch = content.match(/title:\s*"([^"]+)"/); + const tagsMatch = content.match(/tags:\s*\[([^\]]+)\]/); + const confidenceMatch = content.match(/confidence:\s*([\d.]+)/); + + return { + id: fileName.replace(".md", ""), + title: titleMatch?.[1] ?? fileName.replace(".md", ""), + tags: tagsMatch?.[1].split(",").map((t) => t.trim()) ?? [], + confidence: confidenceMatch ? parseFloat(confidenceMatch[1]) : 0.5, + backlinks: [], + }; + } + + /** Generate a daily briefing page */ + generateBriefing(): string { + const logEntries = this.store.readLines<{ + timestamp: string; inboxFile: string; wikiFile: string; tags: string[]; + }>("familiar", "processing-log.jsonl"); + + const recent = logEntries.slice(-10).reverse(); + const today = new Date().toISOString().slice(0, 10); + const todayEntries = recent.filter((e) => e.timestamp.startsWith(today)); + + const lines = [ + "---", + `title: "Daily Briefing — ${today}"`, + `created: ${new Date().toISOString()}`, + "tags: [briefing, daily]", + "status: auto-generated", + "---", + "", + `# Daily Briefing — ${today}`, + "", + ]; + + if (todayEntries.length > 0) { + lines.push(`## Processed Today (${todayEntries.length})`); + lines.push(""); + for (const e of todayEntries) { + lines.push(`- [[${e.wikiFile.replace(".md", "")}]] — ${e.tags.join(", ")}`); + } + lines.push(""); + } + + const graph = this.buildGraphIndex(); + lines.push(`## Knowledge Graph`); + lines.push(`- ${graph.nodes.length} wiki pages`); + lines.push(`- ${graph.edges.length} connections`); + lines.push(""); + + return lines.join("\n"); + } + + private slugify(text: string): string { + return text + .toLowerCase() + .replace(/[^a-z0-9]+/g, "-") + .replace(/^-|-$/g, "") + .slice(0, 80); + } +} + +// ── Types ────────────────────────────────────────────────── + +export interface WikiClassification { + title?: string; + summary?: string; + confidence?: number; + tags?: string[]; + backlinks?: string[]; + mentions?: string[]; + entities?: Array<{ name: string; description: string }>; + contradictions?: string[]; +} + +export interface GraphNode { + id: string; + title: string; + tags: string[]; + confidence: number; + backlinks: string[]; +} + +export interface GraphEdge { + source: string; + target: string; + label: string; +} + +export interface GraphIndex { + nodes: GraphNode[]; + edges: GraphEdge[]; +}