add familiar-knowledge: inbox-to-wiki processing with frontmatter, backlinks, knowledge graph
This commit is contained in:
artale
parent
17ab7f5459
commit
a8a88a4553
|
|
@ -0,0 +1,115 @@
|
|||
import * as fs from "node:fs";
|
||||
import * as path from "node:path";
|
||||
import { FamiliarKnowledge } from "../upgrades/familiar-knowledge.js";
|
||||
import type { WikiClassification } from "../upgrades/familiar-knowledge.js";
|
||||
|
||||
const TEST_DIR = path.join(process.cwd(), ".familiar-test");
|
||||
|
||||
function log(msg: string): void { process.stdout.write(msg + "\n"); }
|
||||
|
||||
function main(): void {
|
||||
// Setup test directory
|
||||
try { fs.rmSync(TEST_DIR, { recursive: true }); } catch {}
|
||||
fs.mkdirSync(path.join(TEST_DIR, "inbox"), { recursive: true });
|
||||
|
||||
const familiar = new FamiliarKnowledge(TEST_DIR);
|
||||
|
||||
log("\n=== Familiar Knowledge Test ===\n");
|
||||
|
||||
// 1. Drop notes in inbox
|
||||
log("[1] Dropping test notes into inbox/...");
|
||||
fs.writeFileSync(
|
||||
path.join(TEST_DIR, "inbox", "agent-loops.md"),
|
||||
`# Agent Feedback Loops
|
||||
|
||||
The key insight from Fable 5 is that loops compound.
|
||||
Each iteration should leave the system smarter.
|
||||
This is different from just prompting repeatedly.
|
||||
|
||||
Related: skill accumulation, state persistence`,
|
||||
"utf-8"
|
||||
);
|
||||
fs.writeFileSync(
|
||||
path.join(TEST_DIR, "inbox", "safety-classifiers.md"),
|
||||
`# Safety Classifier Issues
|
||||
|
||||
Fable 5's safety classifiers trigger on benign medical terms.
|
||||
"cancer" and "MRI" cause silent fallback to Opus 4.8.
|
||||
This breaks reproducibility in biotech research.
|
||||
|
||||
Related: agent-loops, model fallback`,
|
||||
"utf-8"
|
||||
);
|
||||
log(" 2 notes created");
|
||||
|
||||
// 2. Scan inbox
|
||||
const notes = familiar.scanInbox();
|
||||
log(`\n[2] Scanned inbox: ${notes.length} notes`);
|
||||
for (const n of notes) log(` - ${n.fileName} (${n.content.length} chars)`);
|
||||
|
||||
// 3. Process with mock classifier
|
||||
log("\n[3] Processing notes with classifier...");
|
||||
const classifier = (content: string): WikiClassification => {
|
||||
const lower = content.toLowerCase();
|
||||
const tags: string[] = [];
|
||||
if (lower.includes("loop") || lower.includes("feedback")) tags.push("loops");
|
||||
if (lower.includes("safety") || lower.includes("classifier")) tags.push("safety");
|
||||
if (lower.includes("fable")) tags.push("fable-5");
|
||||
|
||||
const title = content.split("\n")[0]?.replace("# ", "").trim() ?? "Untitled";
|
||||
return {
|
||||
title,
|
||||
summary: content.split("\n").slice(2, 4).join(" ").trim().slice(0, 200),
|
||||
confidence: tags.length > 1 ? 0.8 : 0.5,
|
||||
tags,
|
||||
mentions: ["agent-loops", "skill-accumulation", "model-fallback"],
|
||||
contradictions: lower.includes("contradict") ? ["Possible contradiction detected"] : [],
|
||||
};
|
||||
};
|
||||
|
||||
const processed = familiar.processAllInbox(classifier);
|
||||
log(` Processed ${processed.length} notes:`);
|
||||
for (const p of processed) {
|
||||
log(` - ${p.frontmatter.title}`);
|
||||
log(` Tags: ${p.frontmatter.tags.join(", ")}`);
|
||||
log(` Confidence: ${p.frontmatter.confidence}`);
|
||||
log(` Wiki: ${p.wikiPath}`);
|
||||
}
|
||||
|
||||
// 4. Verify wiki pages
|
||||
log("\n[4] Wiki pages created:");
|
||||
const wikiDir = path.join(TEST_DIR, "wiki");
|
||||
if (fs.existsSync(wikiDir)) {
|
||||
for (const f of fs.readdirSync(wikiDir)) {
|
||||
const content = fs.readFileSync(path.join(wikiDir, f), "utf-8");
|
||||
const titleMatch = content.match(/title:\s*"([^"]+)"/);
|
||||
log(` - ${f} → "${titleMatch?.[1] ?? "?"}"`);
|
||||
}
|
||||
}
|
||||
|
||||
// 5. Build graph
|
||||
log("\n[5] Knowledge graph:");
|
||||
const graph = familiar.buildGraphIndex();
|
||||
log(` ${graph.nodes.length} nodes, ${graph.edges.length} edges`);
|
||||
for (const e of graph.edges) {
|
||||
log(` ${e.source} → ${e.target}`);
|
||||
}
|
||||
|
||||
// 6. Generate briefing
|
||||
log("\n[6] Daily briefing:");
|
||||
const briefing = familiar.generateBriefing();
|
||||
log(` ${briefing.split("\n").length} lines generated`);
|
||||
const today = new Date().toISOString().slice(0, 10);
|
||||
log(` Date: ${today}`);
|
||||
|
||||
// Verify inbox is empty (archived)
|
||||
log("\n[7] Inbox is empty (notes archived to resources/):");
|
||||
log(` inbox/ files: ${familiar.scanInbox().length}`);
|
||||
|
||||
// Cleanup
|
||||
try { fs.rmSync(TEST_DIR, { recursive: true }); } catch {}
|
||||
|
||||
log("\n=== All Familiar tests passed ===\n");
|
||||
}
|
||||
|
||||
main();
|
||||
|
|
@ -0,0 +1,325 @@
|
|||
/**
|
||||
* Familiar Knowledge — inbox-to-wiki processing pipeline.
|
||||
*
|
||||
* Inspired by @Av1dlive's Familiar architecture:
|
||||
* inbox/ → agent processes → wiki/ with YAML frontmatter, backlinks, confidence
|
||||
*
|
||||
* Runs inside FA's daemon. Uses FA's safety layers, memory, and executors.
|
||||
*
|
||||
* Flow:
|
||||
* 1. Watch inbox/ for new .md files
|
||||
* 2. Read + classify: what type of note is this?
|
||||
* 3. Extract concepts, entities, connections to existing wiki pages
|
||||
* 4. Generate structured wiki page with frontmatter + backlinks
|
||||
* 5. Archive original to resources/ (immutable)
|
||||
* 6. Update graph index
|
||||
*/
|
||||
|
||||
import * as fs from "node:fs";
|
||||
import * as path from "node:path";
|
||||
import { StateStore } from "../core/state-store.js";
|
||||
|
||||
export interface WikiFrontmatter {
|
||||
title: string;
|
||||
created: string;
|
||||
updated: string;
|
||||
confidence: number;
|
||||
lastUpdatedBy: string;
|
||||
sources: string[];
|
||||
tags: string[];
|
||||
backlinks: string[];
|
||||
mentions: string[];
|
||||
status: "draft" | "processed" | "verified";
|
||||
}
|
||||
|
||||
export interface InboxNote {
|
||||
filePath: string;
|
||||
fileName: string;
|
||||
content: string;
|
||||
capturedAt: string;
|
||||
}
|
||||
|
||||
export interface ProcessedNote {
|
||||
original: InboxNote;
|
||||
wikiPath: string;
|
||||
frontmatter: WikiFrontmatter;
|
||||
body: string;
|
||||
contradictions: string[];
|
||||
}
|
||||
|
||||
export class FamiliarKnowledge {
|
||||
private store: StateStore;
|
||||
private baseDir: string;
|
||||
|
||||
constructor(baseDir?: string) {
|
||||
this.baseDir = baseDir ?? process.cwd();
|
||||
this.store = new StateStore();
|
||||
this.store.ensureSubDir("familiar");
|
||||
this.ensureDirs();
|
||||
}
|
||||
|
||||
private ensureDirs(): void {
|
||||
for (const dir of ["inbox", "wiki", "resources"]) {
|
||||
const p = path.join(this.baseDir, dir);
|
||||
if (!fs.existsSync(p)) fs.mkdirSync(p, { recursive: true });
|
||||
}
|
||||
}
|
||||
|
||||
/** Scan inbox for unprocessed .md files */
|
||||
scanInbox(): InboxNote[] {
|
||||
const inboxDir = path.join(this.baseDir, "inbox");
|
||||
if (!fs.existsSync(inboxDir)) return [];
|
||||
|
||||
return fs.readdirSync(inboxDir)
|
||||
.filter((f) => f.endsWith(".md"))
|
||||
.map((f) => ({
|
||||
filePath: path.join(inboxDir, f),
|
||||
fileName: f,
|
||||
content: fs.readFileSync(path.join(inboxDir, f), "utf-8"),
|
||||
capturedAt: fs.statSync(path.join(inboxDir, f)).birthtime.toISOString(),
|
||||
}));
|
||||
}
|
||||
|
||||
/** Process an inbox note into a wiki page */
|
||||
processNote(note: InboxNote, classifier: (content: string) => WikiClassification): ProcessedNote {
|
||||
const classification = classifier(note.content);
|
||||
|
||||
// Generate wiki page content
|
||||
const wikiFileName = this.slugify(classification.title ?? note.fileName.replace(".md", ""));
|
||||
const wikiPath = path.join(this.baseDir, "wiki", `${wikiFileName}.md`);
|
||||
|
||||
const frontmatter: WikiFrontmatter = {
|
||||
title: classification.title ?? wikiFileName,
|
||||
created: note.capturedAt,
|
||||
updated: new Date().toISOString(),
|
||||
confidence: classification.confidence ?? 0.5,
|
||||
lastUpdatedBy: "fable-agent",
|
||||
sources: [note.filePath],
|
||||
tags: classification.tags ?? [],
|
||||
backlinks: classification.backlinks ?? [],
|
||||
mentions: classification.mentions ?? [],
|
||||
status: "processed",
|
||||
};
|
||||
|
||||
const body = this.generateBody(note.content, classification);
|
||||
|
||||
const processed: ProcessedNote = {
|
||||
original: note,
|
||||
wikiPath,
|
||||
frontmatter,
|
||||
body,
|
||||
contradictions: classification.contradictions ?? [],
|
||||
};
|
||||
|
||||
// Write wiki page
|
||||
this.writeWikiPage(processed);
|
||||
|
||||
// Archive original to resources/
|
||||
this.archiveNote(note);
|
||||
|
||||
// Log
|
||||
this.store.append("familiar", "processing-log.jsonl", {
|
||||
timestamp: new Date().toISOString(),
|
||||
inboxFile: note.fileName,
|
||||
wikiFile: `${wikiFileName}.md`,
|
||||
tags: frontmatter.tags,
|
||||
confidence: frontmatter.confidence,
|
||||
});
|
||||
|
||||
return processed;
|
||||
}
|
||||
|
||||
/** Process all inbox notes */
|
||||
processAllInbox(classifier: (content: string) => WikiClassification): ProcessedNote[] {
|
||||
const notes = this.scanInbox();
|
||||
return notes.map((n) => this.processNote(n, classifier));
|
||||
}
|
||||
|
||||
/** Generate a wiki page with frontmatter and body */
|
||||
private writeWikiPage(processed: ProcessedNote): void {
|
||||
const { frontmatter, body } = processed;
|
||||
const yaml = [
|
||||
"---",
|
||||
`title: "${frontmatter.title}"`,
|
||||
`created: ${frontmatter.created}`,
|
||||
`updated: ${frontmatter.updated}`,
|
||||
`confidence: ${frontmatter.confidence}`,
|
||||
`last_updated_by: ${frontmatter.lastUpdatedBy}`,
|
||||
`status: ${frontmatter.status}`,
|
||||
`tags: [${frontmatter.tags.join(", ")}]`,
|
||||
frontmatter.backlinks.length > 0 ? `backlinks: [${frontmatter.backlinks.join(", ")}]` : "backlinks: []",
|
||||
frontmatter.sources.length > 0 ? `sources: [${frontmatter.sources.map((s) => `"${s}"`).join(", ")}]` : "sources: []",
|
||||
"---",
|
||||
"",
|
||||
body,
|
||||
"",
|
||||
processed.contradictions.length > 0
|
||||
? `> ⚠️ Contradictions: ${processed.contradictions.join("; ")}`
|
||||
: "",
|
||||
].filter(Boolean).join("\n");
|
||||
|
||||
fs.writeFileSync(processed.wikiPath, yaml, "utf-8");
|
||||
}
|
||||
|
||||
/** Archive original note to resources/ */
|
||||
private archiveNote(note: InboxNote): void {
|
||||
const resourcesDir = path.join(this.baseDir, "resources");
|
||||
const archivePath = path.join(resourcesDir, note.fileName);
|
||||
fs.copyFileSync(note.filePath, archivePath);
|
||||
fs.unlinkSync(note.filePath);
|
||||
}
|
||||
|
||||
/** Generate wiki body text from inbox content + classification */
|
||||
private generateBody(content: string, classification: WikiClassification): string {
|
||||
const lines: string[] = [];
|
||||
|
||||
if (classification.summary) {
|
||||
lines.push(classification.summary);
|
||||
lines.push("");
|
||||
}
|
||||
|
||||
lines.push("## Content");
|
||||
lines.push("");
|
||||
lines.push(content.trim());
|
||||
lines.push("");
|
||||
|
||||
if (classification.mentions && classification.mentions.length > 0) {
|
||||
lines.push("## Related");
|
||||
lines.push("");
|
||||
for (const m of classification.mentions) {
|
||||
lines.push(`- [[${m}]]`);
|
||||
}
|
||||
lines.push("");
|
||||
}
|
||||
|
||||
if (classification.entities && classification.entities.length > 0) {
|
||||
lines.push("## Entities");
|
||||
lines.push("");
|
||||
for (const e of classification.entities) {
|
||||
lines.push(`- **${e.name}**: ${e.description}`);
|
||||
}
|
||||
lines.push("");
|
||||
}
|
||||
|
||||
return lines.join("\n");
|
||||
}
|
||||
|
||||
/** Build a graph index of all wiki pages and their backlinks */
|
||||
buildGraphIndex(): GraphIndex {
|
||||
const wikiDir = path.join(this.baseDir, "wiki");
|
||||
if (!fs.existsSync(wikiDir)) return { nodes: [], edges: [] };
|
||||
|
||||
const nodes: GraphNode[] = [];
|
||||
const edges: GraphEdge[] = [];
|
||||
|
||||
const files = fs.readdirSync(wikiDir).filter((f) => f.endsWith(".md"));
|
||||
for (const file of files) {
|
||||
const content = fs.readFileSync(path.join(wikiDir, file), "utf-8");
|
||||
const node = this.parseWikiPage(file, content);
|
||||
nodes.push(node);
|
||||
|
||||
// Extract [[wikilinks]] as edges
|
||||
const linkMatches = content.match(/\[\[([^\]]+)\]\]/g) || [];
|
||||
for (const link of linkMatches) {
|
||||
const target = link.replace(/\[\[|\]\]/g, "").trim();
|
||||
edges.push({ source: node.id, target, label: "references" });
|
||||
}
|
||||
}
|
||||
|
||||
return { nodes, edges };
|
||||
}
|
||||
|
||||
private parseWikiPage(fileName: string, content: string): GraphNode {
|
||||
const titleMatch = content.match(/title:\s*"([^"]+)"/);
|
||||
const tagsMatch = content.match(/tags:\s*\[([^\]]+)\]/);
|
||||
const confidenceMatch = content.match(/confidence:\s*([\d.]+)/);
|
||||
|
||||
return {
|
||||
id: fileName.replace(".md", ""),
|
||||
title: titleMatch?.[1] ?? fileName.replace(".md", ""),
|
||||
tags: tagsMatch?.[1].split(",").map((t) => t.trim()) ?? [],
|
||||
confidence: confidenceMatch ? parseFloat(confidenceMatch[1]) : 0.5,
|
||||
backlinks: [],
|
||||
};
|
||||
}
|
||||
|
||||
/** Generate a daily briefing page */
|
||||
generateBriefing(): string {
|
||||
const logEntries = this.store.readLines<{
|
||||
timestamp: string; inboxFile: string; wikiFile: string; tags: string[];
|
||||
}>("familiar", "processing-log.jsonl");
|
||||
|
||||
const recent = logEntries.slice(-10).reverse();
|
||||
const today = new Date().toISOString().slice(0, 10);
|
||||
const todayEntries = recent.filter((e) => e.timestamp.startsWith(today));
|
||||
|
||||
const lines = [
|
||||
"---",
|
||||
`title: "Daily Briefing — ${today}"`,
|
||||
`created: ${new Date().toISOString()}`,
|
||||
"tags: [briefing, daily]",
|
||||
"status: auto-generated",
|
||||
"---",
|
||||
"",
|
||||
`# Daily Briefing — ${today}`,
|
||||
"",
|
||||
];
|
||||
|
||||
if (todayEntries.length > 0) {
|
||||
lines.push(`## Processed Today (${todayEntries.length})`);
|
||||
lines.push("");
|
||||
for (const e of todayEntries) {
|
||||
lines.push(`- [[${e.wikiFile.replace(".md", "")}]] — ${e.tags.join(", ")}`);
|
||||
}
|
||||
lines.push("");
|
||||
}
|
||||
|
||||
const graph = this.buildGraphIndex();
|
||||
lines.push(`## Knowledge Graph`);
|
||||
lines.push(`- ${graph.nodes.length} wiki pages`);
|
||||
lines.push(`- ${graph.edges.length} connections`);
|
||||
lines.push("");
|
||||
|
||||
return lines.join("\n");
|
||||
}
|
||||
|
||||
private slugify(text: string): string {
|
||||
return text
|
||||
.toLowerCase()
|
||||
.replace(/[^a-z0-9]+/g, "-")
|
||||
.replace(/^-|-$/g, "")
|
||||
.slice(0, 80);
|
||||
}
|
||||
}
|
||||
|
||||
// ── Types ──────────────────────────────────────────────────
|
||||
|
||||
export interface WikiClassification {
|
||||
title?: string;
|
||||
summary?: string;
|
||||
confidence?: number;
|
||||
tags?: string[];
|
||||
backlinks?: string[];
|
||||
mentions?: string[];
|
||||
entities?: Array<{ name: string; description: string }>;
|
||||
contradictions?: string[];
|
||||
}
|
||||
|
||||
export interface GraphNode {
|
||||
id: string;
|
||||
title: string;
|
||||
tags: string[];
|
||||
confidence: number;
|
||||
backlinks: string[];
|
||||
}
|
||||
|
||||
export interface GraphEdge {
|
||||
source: string;
|
||||
target: string;
|
||||
label: string;
|
||||
}
|
||||
|
||||
export interface GraphIndex {
|
||||
nodes: GraphNode[];
|
||||
edges: GraphEdge[];
|
||||
}
|
||||
Loading…
Reference in New Issue