Improved memory management
This commit is contained in:
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "@ztimson/ai-utils",
|
||||
"version": "1.2.3",
|
||||
"version": "1.2.4",
|
||||
"description": "AI Utility library",
|
||||
"author": "Zak Timson",
|
||||
"license": "MIT",
|
||||
|
||||
605
src/memory.ts
605
src/memory.ts
@@ -7,8 +7,6 @@ export type Memory = {
|
||||
description: string;
|
||||
content: string;
|
||||
embedding: number[];
|
||||
links: string[];
|
||||
backlinks: string[];
|
||||
}
|
||||
|
||||
type MemoryRef = {
|
||||
@@ -19,6 +17,7 @@ type MemoryRef = {
|
||||
type FactBucket = {
|
||||
subject: string;
|
||||
facts: string[];
|
||||
isNew: boolean;
|
||||
}
|
||||
|
||||
export type MemoryNode = {
|
||||
@@ -33,26 +32,31 @@ export function buildMemoryGraph(memories: Memory[] | MemoryCache): MemoryNode[]
|
||||
const nameSet = new Set(mems.map(m => m.name));
|
||||
const ghosts = new Set<string>();
|
||||
|
||||
for (const m of mems) {
|
||||
for (const link of m.links) {
|
||||
const nodes: MemoryNode[] = mems.map(m => {
|
||||
const {links, backlinks} = extractMetadata(m.content);
|
||||
return {
|
||||
name: m.name,
|
||||
missing: false,
|
||||
links,
|
||||
backlinks,
|
||||
};
|
||||
});
|
||||
|
||||
for (const node of nodes) {
|
||||
for (const link of node.links) {
|
||||
if (!nameSet.has(link)) ghosts.add(link);
|
||||
}
|
||||
}
|
||||
|
||||
return [
|
||||
...mems.map(m => ({
|
||||
name: m.name,
|
||||
missing: false,
|
||||
links: m.links,
|
||||
backlinks: m.backlinks,
|
||||
})),
|
||||
...nodes,
|
||||
...[...ghosts].map(name => ({
|
||||
name,
|
||||
missing: true,
|
||||
links: [],
|
||||
backlinks: mems
|
||||
.filter(m => m.links.includes(name))
|
||||
.map(m => m.name),
|
||||
backlinks: nodes
|
||||
.filter(n => n.links.includes(name))
|
||||
.map(n => n.name),
|
||||
}))
|
||||
];
|
||||
}
|
||||
@@ -62,14 +66,21 @@ function extractLinks(content: string): string[] {
|
||||
return [...new Set([...matches].map(m => m[1].trim()))];
|
||||
}
|
||||
|
||||
function rebuildBacklinks(memories: Memory[]): void {
|
||||
for (const m of memories) m.backlinks = [];
|
||||
for (const m of memories) {
|
||||
for (const link of m.links) {
|
||||
const target = memories.find(t => t.name === link);
|
||||
if (target) target.backlinks.push(m.name);
|
||||
}
|
||||
}
|
||||
export function extractMetadata(content: string): {links: string[], backlinks: string[]} {
|
||||
const match = content.match(/^---\n([\s\S]*?)\n---/);
|
||||
if (!match) return {links: [], backlinks: []};
|
||||
|
||||
const fm = match[1];
|
||||
const getList = (key: string): string[] => {
|
||||
const m = fm.match(new RegExp(`^${key}:\\s*\\[(.*)\\]$`, 'm'));
|
||||
if (!m || !m[1].trim()) return [];
|
||||
return m[1].split(',').map(s => s.trim().replace(/^"|"$/g, '')).filter(Boolean);
|
||||
};
|
||||
|
||||
return {
|
||||
links: getList('links'),
|
||||
backlinks: getList('backlinks'),
|
||||
};
|
||||
}
|
||||
|
||||
function cosineDistance(a: number[], b: number[]): number {
|
||||
@@ -83,9 +94,53 @@ function cosineDistance(a: number[], b: number[]): number {
|
||||
return denom === 0 ? 1 : 1 - dot / denom;
|
||||
}
|
||||
|
||||
function getWeekMonday(date: Date = new Date()): string {
|
||||
const d = new Date(Date.UTC(date.getFullYear(), date.getMonth(), date.getDate()));
|
||||
const day = d.getUTCDay();
|
||||
const diff = day === 0 ? -6 : 1 - day;
|
||||
d.setUTCDate(d.getUTCDate() + diff);
|
||||
return d.toISOString().slice(0, 10);
|
||||
}
|
||||
|
||||
function getWeekSunday(monday: string): string {
|
||||
const d = new Date(`${monday}T00:00:00Z`);
|
||||
d.setUTCDate(d.getUTCDate() + 6);
|
||||
return d.toISOString().slice(0, 10);
|
||||
}
|
||||
|
||||
function tagsFromName(name: string): string[] {
|
||||
const prefix = name.split('/')[0];
|
||||
return prefix ? [prefix.toLowerCase()] : [];
|
||||
}
|
||||
|
||||
export function serializeMemory(mem: Memory, week?: {monday: string, sunday: string}): string {
|
||||
return mem.content;
|
||||
}
|
||||
|
||||
export function deserializeMemory(raw: string, embedding: number[] = []): Memory {
|
||||
const match = raw.match(/^---\n([\s\S]*?)\n---\n\n?([\s\S]*)$/);
|
||||
if (!match) {
|
||||
return {name: '', description: '', content: raw.trim(), embedding};
|
||||
}
|
||||
|
||||
const [, fm] = match;
|
||||
const get = (key: string): string => {
|
||||
const m = fm.match(new RegExp(`^${key}:\\s*(.+)$`, 'm'));
|
||||
return m ? m[1].trim() : '';
|
||||
};
|
||||
|
||||
return {
|
||||
name: get('name'),
|
||||
description: get('description'),
|
||||
content: raw.trim(),
|
||||
embedding,
|
||||
};
|
||||
}
|
||||
|
||||
export class MemoryCache {
|
||||
private tree: KDTree<MemoryRef>;
|
||||
public memories: Memory[];
|
||||
private locks = new Map<string, Promise<void>>();
|
||||
|
||||
constructor(memories: Memory[]) {
|
||||
this.memories = memories;
|
||||
@@ -125,16 +180,38 @@ export class MemoryCache {
|
||||
this.rebuild();
|
||||
}
|
||||
|
||||
remove(name: string): void {
|
||||
const idx = this.memories.findIndex(m => m.name === name);
|
||||
if (idx !== -1) {
|
||||
this.memories.splice(idx, 1);
|
||||
this.rebuild();
|
||||
}
|
||||
}
|
||||
|
||||
rebuild(): void {
|
||||
this.tree = this.buildTree();
|
||||
}
|
||||
|
||||
rebuildLinks(): void {
|
||||
rebuildBacklinks(this.memories);
|
||||
lock<T>(name: string, fn: () => Promise<T>): Promise<T> {
|
||||
const prev = this.locks.get(name) ?? Promise.resolve();
|
||||
let resolveLock!: () => void;
|
||||
const next = new Promise<void>(r => { resolveLock = r; });
|
||||
this.locks.set(name, next);
|
||||
|
||||
const result = prev.then(fn).finally(resolveLock);
|
||||
result.finally(() => {
|
||||
if (this.locks.get(name) === next) this.locks.delete(name);
|
||||
});
|
||||
return result;
|
||||
}
|
||||
}
|
||||
|
||||
export class MemoryManager {
|
||||
private pendingMemorizations = new Map<string, {
|
||||
memories: Memory[] | MemoryCache,
|
||||
tempMemoryName: string,
|
||||
timestamp: number,
|
||||
}>();
|
||||
|
||||
tools = {
|
||||
read: (memories: Memory[] | MemoryCache): AiTool => ({
|
||||
@@ -147,19 +224,78 @@ export class MemoryManager {
|
||||
const mems = memories instanceof MemoryCache ? memories.memories : memories;
|
||||
const mem = mems.find(m => m.name === args.name);
|
||||
if (!mem) return 'Document not found';
|
||||
return this.formatMemory(mem);
|
||||
}
|
||||
return mem.content;
|
||||
},
|
||||
}),
|
||||
|
||||
forget: (memories: Memory[] | MemoryCache): AiTool => ({
|
||||
name: 'forget_memory',
|
||||
description: 'Permanently delete a memory document and clean up all references to it',
|
||||
args: {
|
||||
name: {type: 'string', description: 'Exact memory name to forget', required: true},
|
||||
reason: {type: 'string', description: 'Why this memory is being deleted', required: true},
|
||||
},
|
||||
fn: (args: any) => {
|
||||
const result = this.forget(args.name, memories);
|
||||
return result ? `Forgotten: ${args.name}` : `Not found: ${args.name}`;
|
||||
},
|
||||
}),
|
||||
};
|
||||
|
||||
constructor(private llm: any) {}
|
||||
|
||||
private async createTempMemory(conversation: string): Promise<Memory> {
|
||||
const [e] = await this.llm.embedding(conversation);
|
||||
const timestamp = Date.now();
|
||||
return {
|
||||
name: `_temp_${timestamp}`,
|
||||
description: 'Temporary memory - processing in background',
|
||||
content: `---
|
||||
name: _temp_${timestamp}
|
||||
description: Temporary memory - processing in background
|
||||
tags: [_temporary]
|
||||
links: []
|
||||
backlinks: []
|
||||
modified: ${new Date().toISOString()}
|
||||
---
|
||||
|
||||
# Recent Conversation (Processing)
|
||||
|
||||
${conversation}`,
|
||||
embedding: e?.embedding || [],
|
||||
};
|
||||
}
|
||||
|
||||
forget(name: string, memories: Memory[] | MemoryCache): boolean {
|
||||
const mem = memories instanceof MemoryCache ? memories.memories : memories;
|
||||
const idx = mem.findIndex(m => m.name === name);
|
||||
if (idx === -1) return false;
|
||||
|
||||
for (const node of mem) {
|
||||
const {links, backlinks} = extractMetadata(node.content);
|
||||
const newBacklinks = backlinks.filter(b => b !== name);
|
||||
const newLinks = links.filter(l => l !== name);
|
||||
|
||||
if (newBacklinks.length !== backlinks.length || newLinks.length !== links.length) {
|
||||
node.content = this.updateFrontmatter(node.content, {
|
||||
links: newLinks,
|
||||
backlinks: newBacklinks,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
mem.splice(idx, 1);
|
||||
|
||||
if (memories instanceof MemoryCache) memories.rebuild();
|
||||
return true;
|
||||
}
|
||||
|
||||
private cosineSearch(query: number[], memories: Memory[], limit: number): MemoryRef[] {
|
||||
const scored = memories
|
||||
.filter(m => m.embedding?.length)
|
||||
.map(m => ({
|
||||
ref: {name: m.name, description: m.description},
|
||||
distance: cosineDistance(query, m.embedding)
|
||||
distance: cosineDistance(query, m.embedding),
|
||||
}))
|
||||
.sort((a, b) => a.distance - b.distance)
|
||||
.slice(0, limit);
|
||||
@@ -174,22 +310,9 @@ export class MemoryManager {
|
||||
description: '',
|
||||
content: '',
|
||||
embedding: [],
|
||||
links: [],
|
||||
backlinks: [],
|
||||
};
|
||||
}
|
||||
|
||||
private formatMemory(mem: Memory): string {
|
||||
return [
|
||||
`# ${mem.name}`,
|
||||
mem.description ? `> ${mem.description}` : '',
|
||||
mem.links.length ? `**Links:** ${mem.links.map(l => `[[${l}]]`).join(', ')}` : '',
|
||||
mem.backlinks.length ? `**Referenced by:** ${mem.backlinks.map(l => `[[${l}]]`).join(', ')}` : '',
|
||||
'',
|
||||
mem.content,
|
||||
].filter(l => l !== undefined).join('\n');
|
||||
}
|
||||
|
||||
private listNodes(memories: Memory[]): MemoryRef[] {
|
||||
return memories.map(m => ({name: m.name, description: m.description}));
|
||||
}
|
||||
@@ -197,6 +320,7 @@ export class MemoryManager {
|
||||
async recollect(query: string, memories: Memory[] | MemoryCache, limit = 5, graphDepth = 1): Promise<Memory[]> {
|
||||
const mem: Memory[] = memories instanceof MemoryCache ? memories.memories : memories;
|
||||
if (!mem.length) return [];
|
||||
|
||||
const [e] = await this.llm.embedding(query);
|
||||
if (!e) return [];
|
||||
|
||||
@@ -212,7 +336,8 @@ export class MemoryManager {
|
||||
for (const name of frontier) {
|
||||
const node = mem.find(m => m.name === name);
|
||||
if (!node) continue;
|
||||
for(const link of node.links) {
|
||||
const {links} = extractMetadata(node.content);
|
||||
for (const link of links) {
|
||||
if (!found.has(link) && mem.find(m => m.name === link)) {
|
||||
found.add(link);
|
||||
next.push(link);
|
||||
@@ -231,97 +356,160 @@ export class MemoryManager {
|
||||
}
|
||||
|
||||
async memorize(history: LLMMessage[], memories: Memory[] | MemoryCache, options: LLMRequest): Promise<void> {
|
||||
const mem = memories instanceof MemoryCache ? memories.memories : memories;
|
||||
const conversation = history
|
||||
.filter(h => h.role === 'user' || h.role === 'assistant')
|
||||
.map(h => `[${h.role}]: ${h.content}`).join('\n\n').trim();
|
||||
|
||||
if(conversation) {
|
||||
const buckets = await this.factAgent(conversation, mem, options);
|
||||
if(buckets.length) {
|
||||
await Promise.all(buckets.map(async bucket => {
|
||||
const node = await this.organizingAgent(bucket, mem, options);
|
||||
if(!mem.find(m => m.name === node.name)) mem.push(node);
|
||||
await this.docAgent(node, bucket, mem, options);
|
||||
}));
|
||||
}
|
||||
}
|
||||
if (!conversation) return;
|
||||
|
||||
// Auto-compress old journals
|
||||
const weekAgo = Date.now() - (7 * 24 * 60 * 60 * 1000);
|
||||
const oldDailies = mem.filter(m => {
|
||||
const journal = /^Journal\/(\d{4}-\d{2}-\d{2}$)/.exec(m.name);
|
||||
return journal && new Date(journal[1]).getTime() < weekAgo;
|
||||
});
|
||||
const trackingId = `${Date.now()}_${Math.random()}`;
|
||||
|
||||
if(oldDailies.length) {
|
||||
const byMonth = new Map<string, Memory[]>();
|
||||
for(const daily of oldDailies) {
|
||||
const match = daily.name.match(/^Journal\/(\d{4}-\d{2})-\d{2}$/);
|
||||
if(!match) continue;
|
||||
const monthKey = match[1];
|
||||
if(!byMonth.has(monthKey)) byMonth.set(monthKey, []);
|
||||
byMonth.get(monthKey)!.push(daily);
|
||||
}
|
||||
|
||||
for(const [monthKey, entries] of byMonth) {
|
||||
const monthlyPath = `Journal/${monthKey}`;
|
||||
let monthly = mem.find(m => m.name === monthlyPath);
|
||||
if(!monthly) {
|
||||
monthly = this.createNode(monthlyPath, mem);
|
||||
mem.push(monthly);
|
||||
}
|
||||
|
||||
const bucket: FactBucket = {
|
||||
subject: monthlyPath,
|
||||
facts: entries.flatMap(e => e.content.split('\n').filter(line => line.trim())),
|
||||
};
|
||||
|
||||
await this.docAgent(monthly, bucket, mem, options);
|
||||
for(const daily of entries) {
|
||||
const idx = mem.indexOf(daily);
|
||||
if(idx !== -1) mem.splice(idx, 1);
|
||||
}
|
||||
}
|
||||
}
|
||||
// Create and insert temp memory immediately
|
||||
const tempMemory = await this.createTempMemory(conversation);
|
||||
const mem = memories instanceof MemoryCache ? memories.memories : memories;
|
||||
mem.push(tempMemory);
|
||||
|
||||
if (memories instanceof MemoryCache) {
|
||||
memories.rebuildLinks();
|
||||
memories.rebuild();
|
||||
} else {
|
||||
rebuildBacklinks(mem);
|
||||
}
|
||||
|
||||
this.pendingMemorizations.set(trackingId, {
|
||||
memories,
|
||||
tempMemoryName: tempMemory.name,
|
||||
timestamp: Date.now(),
|
||||
});
|
||||
|
||||
this._memorizeBackground(conversation, memories, options, trackingId)
|
||||
.catch(err => {
|
||||
console.error('[memorize] Background memorization failed:', err);
|
||||
})
|
||||
.finally(() => {
|
||||
// Remove temp memory from the exact same memory array/cache
|
||||
const pending = this.pendingMemorizations.get(trackingId);
|
||||
if (pending) {
|
||||
const cleanMem = pending.memories instanceof MemoryCache
|
||||
? pending.memories.memories
|
||||
: pending.memories;
|
||||
const idx = cleanMem.findIndex(m => m.name === pending.tempMemoryName);
|
||||
if (idx !== -1) {
|
||||
cleanMem.splice(idx, 1);
|
||||
console.log(`[memorize] Removed temp memory: ${pending.tempMemoryName}`);
|
||||
}
|
||||
if (pending.memories instanceof MemoryCache) {
|
||||
pending.memories.rebuild();
|
||||
}
|
||||
}
|
||||
this.pendingMemorizations.delete(trackingId);
|
||||
});
|
||||
}
|
||||
|
||||
private async _memorizeBackground(conversation: string, memories: Memory[] | MemoryCache, options: LLMRequest, trackingId: string): Promise<void> {
|
||||
const mem = memories instanceof MemoryCache ? memories.memories : memories;
|
||||
const monday = getWeekMonday();
|
||||
const sunday = getWeekSunday(monday);
|
||||
const weekKey = monday;
|
||||
|
||||
console.log('[memorize] Starting fact extraction...');
|
||||
const buckets = await this.factAgent(conversation, mem, options, weekKey);
|
||||
console.log(`[memorize] Extracted ${buckets.length} buckets:`, buckets);
|
||||
|
||||
if (!buckets.length) {
|
||||
console.log('[memorize] No facts extracted, exiting');
|
||||
return;
|
||||
}
|
||||
|
||||
const runDocAgent = (node: Memory, bucket: FactBucket, embedding?: number[], week?: {monday: string, sunday: string}) => {
|
||||
if (memories instanceof MemoryCache) {
|
||||
return memories.lock(node.name, () => this.docAgent(node, bucket, mem, options, embedding, week));
|
||||
}
|
||||
return this.docAgent(node, bucket, mem, options, embedding, week);
|
||||
};
|
||||
|
||||
await Promise.all(buckets.map(async bucket => {
|
||||
let node = mem.find(m => m.name === bucket.subject && !m.name.startsWith('_temp_'));
|
||||
let embedding: number[] | undefined;
|
||||
|
||||
if (!node || bucket.isNew) {
|
||||
const [e] = await this.llm.embedding(`${bucket.subject}\n${bucket.facts.join('\n')}`);
|
||||
embedding = e?.embedding;
|
||||
|
||||
if (!node) {
|
||||
node = this.createNode(bucket.subject, mem);
|
||||
mem.push(node);
|
||||
}
|
||||
}
|
||||
|
||||
private async docAgent(node: Memory, bucket: FactBucket, memories: Memory[], options: LLMRequest): Promise<void> {
|
||||
const week = bucket.subject.startsWith('Journal/') ? {monday, sunday} : undefined;
|
||||
await runDocAgent(node, bucket, embedding, week);
|
||||
}));
|
||||
|
||||
if (memories instanceof MemoryCache) {
|
||||
memories.rebuild();
|
||||
}
|
||||
|
||||
console.log('[memorize] Completed successfully');
|
||||
}
|
||||
|
||||
private buildHeader(node: Memory, week?: {monday: string, sunday: string}, links: string[] = [], backlinks: string[] = []): string {
|
||||
const tags = node.name.split('/')[0]?.toLowerCase();
|
||||
const lines = [
|
||||
'---',
|
||||
`name: ${node.name}`,
|
||||
`description: ${node.description || ''}`,
|
||||
tags ? `tags: [${tags}]` : '',
|
||||
links.length ? `links: [${links.map(l => `"${l}"`).join(', ')}]` : 'links: []',
|
||||
backlinks.length ? `backlinks: [${backlinks.map(l => `"${l}"`).join(', ')}]` : 'backlinks: []',
|
||||
week ? `week: ${week.monday} – ${week.sunday}` : '',
|
||||
`modified: ${new Date().toISOString()}`,
|
||||
'---',
|
||||
].filter(Boolean);
|
||||
return lines.join('\n');
|
||||
}
|
||||
|
||||
private applyHeader(content: string, header: string): string {
|
||||
const hasFrontmatter = content.trimStart().startsWith('---');
|
||||
if (hasFrontmatter) {
|
||||
return content.replace(/^---[\s\S]*?---\n?/, `${header}\n`);
|
||||
}
|
||||
return `${header}\n\n${content}`;
|
||||
}
|
||||
|
||||
private updateFrontmatter(content: string, updates: {links?: string[], backlinks?: string[]}): string {
|
||||
const match = content.match(/^---\n([\s\S]*?)\n---\n\n?([\s\S]*)$/);
|
||||
if (!match) return content;
|
||||
|
||||
const [, fm, body] = match;
|
||||
let newFm = fm;
|
||||
|
||||
if (updates.links !== undefined) {
|
||||
const linksList = updates.links.length ? `[${updates.links.map(l => `"${l}"`).join(', ')}]` : '[]';
|
||||
newFm = newFm.replace(/^links:.*$/m, `links: ${linksList}`);
|
||||
}
|
||||
|
||||
if (updates.backlinks !== undefined) {
|
||||
const backlinksList = updates.backlinks.length ? `[${updates.backlinks.map(l => `"${l}"`).join(', ')}]` : '[]';
|
||||
newFm = newFm.replace(/^backlinks:.*$/m, `backlinks: ${backlinksList}`);
|
||||
}
|
||||
|
||||
newFm = newFm.replace(/^modified:.*$/m, `modified: ${new Date().toISOString()}`);
|
||||
|
||||
return `---\n${newFm}\n---\n\n${body}`;
|
||||
}
|
||||
|
||||
private stripHeader(content: string): string {
|
||||
return content.replace(/^---[\s\S]*?---\n?/, '').trimStart();
|
||||
}
|
||||
|
||||
private async docAgent(node: Memory, bucket: FactBucket, memories: Memory[], options: LLMRequest, precomputedEmbedding?: number[], week?: {monday: string, sunday: string}): Promise<void> {
|
||||
const {links: oldLinks} = extractMetadata(node.content);
|
||||
let finalContent = node.content;
|
||||
const isJournalCompression = node.name.match(/^Journal\/\d{4}-\d{2}$/);
|
||||
const systemPrompt = isJournalCompression
|
||||
? `You are a journal compressor. Condense the daily entries below into a monthly summary.
|
||||
|
||||
Format:
|
||||
# ${node.name}
|
||||
|
||||
## Themes
|
||||
(Recurring topics, moods, patterns)
|
||||
|
||||
## Key Events
|
||||
(Important moments, decisions, milestones)
|
||||
|
||||
## Notable Conversations
|
||||
(Significant discussions or revelations)
|
||||
|
||||
Rules:
|
||||
- Use [[WikiLinks]] to reference permanent notes using full paths like [[People/Sarah]] or [[Projects/Website]]
|
||||
- Keep it concise but preserve emotional/temporal context
|
||||
- Discard filler but keep things the user vented about or cared about
|
||||
- If a fact belongs in a permanent note, link to it instead of duplicating
|
||||
|
||||
Current monthly summary:
|
||||
\`\`\`markdown
|
||||
${node.content || '(empty — first compression for this month)'}
|
||||
\`\`\``
|
||||
: `You are a knowledge base editor. Integrate the provided facts into the document below.
|
||||
await this.llm.ask(
|
||||
`New facts to integrate:\n${bucket.facts.map(f => `- ${f}`).join('\n')}`,
|
||||
{
|
||||
model: options.model,
|
||||
temperature: 0.3,
|
||||
system: `You are a knowledge base editor. Integrate the provided facts into the document below.
|
||||
|
||||
Formatting rules:
|
||||
- Use Obsidian-style markdown: # headings, **bold** for key terms, bullet lists for facts
|
||||
@@ -330,49 +518,77 @@ Formatting rules:
|
||||
- Keep the document concise, factual, and human-readable
|
||||
- Resolve any contradictions between old content and new facts (new facts win)
|
||||
- Do not add filler, preamble, or AI commentary — just clean knowledge documents
|
||||
|
||||
- The document begins with a YAML frontmatter block (between --- markers) — do not remove or rewrite it, it is maintained automatically
|
||||
${week ? '- This is a weekly journal entry. The frontmatter contains the week date range.\n' : ''}
|
||||
All nodes:
|
||||
${this.listNodes(memories).map(n => n.name).join(', ') || 'none'}
|
||||
|
||||
Current document:
|
||||
\`\`\`markdown
|
||||
${node.content || '(empty — this is a new document)'}
|
||||
\`\`\``;
|
||||
|
||||
await this.llm.ask(
|
||||
`New facts to integrate:\n${bucket.facts.map(f => `- ${f}`).join('\n')}`,
|
||||
{
|
||||
model: options.model,
|
||||
temperature: 0.3,
|
||||
system: systemPrompt,
|
||||
\`\`\``,
|
||||
tools: [{
|
||||
name: 'update_document',
|
||||
description: 'Write the complete updated document content',
|
||||
description: 'Write the complete updated document content. Include everything after the frontmatter block — the frontmatter will be recalculated automatically.',
|
||||
args: {
|
||||
description: {type: 'string', description: 'One-line description of what this document covers, no formatting or emojis', required: true},
|
||||
content: {type: 'string', description: 'Fully updated document in markdown', required: true},
|
||||
content: {type: 'string', description: 'Document body in markdown, without the frontmatter block', required: true},
|
||||
},
|
||||
fn: (args: any) => {
|
||||
node.description = args.description;
|
||||
finalContent = args.content;
|
||||
return 'Saved';
|
||||
}
|
||||
}]
|
||||
},
|
||||
}],
|
||||
}
|
||||
);
|
||||
|
||||
node.content = finalContent;
|
||||
node.links = extractLinks(finalContent);
|
||||
const needsEmbed = !node.embedding?.length || node.description !== memories.find(m => m.name === node.name)?.description;
|
||||
if (needsEmbed) {
|
||||
const [e] = await this.llm.embedding(node.description);
|
||||
const newLinks = extractLinks(finalContent).filter(l => l !== node.name);
|
||||
const newLinkSet = new Set(newLinks);
|
||||
const oldLinkSet = new Set(oldLinks);
|
||||
|
||||
for (const added of newLinkSet) {
|
||||
if (!oldLinkSet.has(added)) {
|
||||
const target = memories.find(m => m.name === added);
|
||||
if (target) {
|
||||
const {backlinks} = extractMetadata(target.content);
|
||||
if (!backlinks.includes(node.name)) {
|
||||
target.content = this.updateFrontmatter(target.content, {
|
||||
backlinks: [...backlinks, node.name],
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
for (const removed of oldLinkSet) {
|
||||
if (!newLinkSet.has(removed)) {
|
||||
const target = memories.find(m => m.name === removed);
|
||||
if (target) {
|
||||
const {backlinks} = extractMetadata(target.content);
|
||||
target.content = this.updateFrontmatter(target.content, {
|
||||
backlinks: backlinks.filter(b => b !== node.name),
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
const {backlinks} = extractMetadata(node.content);
|
||||
const header = this.buildHeader(node, week, newLinks, backlinks);
|
||||
node.content = this.applyHeader(finalContent, header);
|
||||
|
||||
if (precomputedEmbedding) {
|
||||
node.embedding = precomputedEmbedding;
|
||||
} else {
|
||||
const embedInput = `${node.description}\n\n${this.stripHeader(node.content)}`.trim();
|
||||
const [e] = await this.llm.embedding(embedInput);
|
||||
if (e) node.embedding = e.embedding;
|
||||
}
|
||||
}
|
||||
|
||||
private async factAgent(conversation: string, memories: Memory[], options: LLMRequest): Promise<FactBucket[]> {
|
||||
private async factAgent(conversation: string, memories: Memory[], options: LLMRequest, weekKey: string): Promise<FactBucket[]> {
|
||||
const buckets: FactBucket[] = [];
|
||||
const today = new Date().toISOString().split('T')[0];
|
||||
|
||||
console.log('[factAgent] Starting extraction...');
|
||||
|
||||
await this.llm.ask(conversation, {
|
||||
model: options.model,
|
||||
@@ -386,116 +602,37 @@ Rules:
|
||||
- DO NOT extract greetings, pleasantries, or generic exchanges
|
||||
- If nothing worth remembering was said, do not call any tools
|
||||
|
||||
**Organizational patterns:**
|
||||
- Journal entries use paths like: Journal/${today}
|
||||
- People use paths like: People/Name
|
||||
- Projects use paths like: Projects/Name
|
||||
- Personal info uses paths like: Personal/Goals, Personal/Tasks, etc.
|
||||
- General knowledge uses paths like: Biology/Topic, History/Topic, etc.
|
||||
When extracting facts, you MUST also decide the exact destination path:
|
||||
- Use an existing node name if the facts clearly belong there
|
||||
- Create a new path following collection/subject format if needed (e.g., People/Sarah, Projects/Oxide)
|
||||
- For journal entries, use "journal" (will auto-route to Journal/${weekKey})
|
||||
|
||||
Learn from existing nodes and follow the same pattern when extracting.
|
||||
|
||||
Group facts by subject. For each group call \`extract_facts\` once with the FULL PATH.
|
||||
|
||||
Known nodes (name: description):
|
||||
Available nodes:
|
||||
${this.listNodes(memories).map(n => `- ${n.name}: ${n.description}`).join('\n') || 'None yet.'}`,
|
||||
tools: [{
|
||||
name: 'extract_facts',
|
||||
description: 'Submit a group of related facts for a specific subject',
|
||||
description: 'Submit facts with their destination',
|
||||
args: {
|
||||
subject: {type: 'string', description: 'Full path for the subject (e.g., "Journal/2025-01-27", "People/Sarah", "Projects/Website")', required: true},
|
||||
facts: {type: 'string', description: 'Comma-separated list of extracted facts', required: true},
|
||||
destination: {type: 'string', description: 'Exact existing node name OR new path (e.g. "People/Sarah", "Projects/Oxide")', required: true},
|
||||
facts: {type: 'string', description: 'Comma-separated facts', required: true},
|
||||
create_new: {type: 'boolean', description: 'True if this is a new node that doesn\'t exist yet', required: true},
|
||||
},
|
||||
fn: (args: any) => {
|
||||
console.log('[factAgent] Tool called with:', args);
|
||||
const subject = args.destination.trim().toLowerCase() === 'journal'
|
||||
? `Journal/${weekKey}`
|
||||
: args.destination;
|
||||
buckets.push({
|
||||
subject: args.subject,
|
||||
subject,
|
||||
facts: args.facts.split(',').map((f: string) => f.trim()).filter(Boolean),
|
||||
isNew: args.create_new,
|
||||
});
|
||||
return 'Recorded';
|
||||
}
|
||||
}]
|
||||
},
|
||||
}],
|
||||
});
|
||||
|
||||
console.log(`[factAgent] Extracted ${buckets.length} buckets:`, buckets);
|
||||
return buckets;
|
||||
}
|
||||
|
||||
private async organizingAgent(bucket: FactBucket, memories: Memory[], options: LLMRequest): Promise<Memory> {
|
||||
let candidates = this.listNodes(memories);
|
||||
let attempts = 0;
|
||||
const maxAttempts = 3;
|
||||
|
||||
while (attempts++ < maxAttempts) {
|
||||
let home = '', mode: string | null = null;
|
||||
|
||||
const resp = await this.llm.ask(`Subject: ${bucket.subject}\n\nFacts:\n${bucket.facts.map(f => `- ${f}`).join('\n')}`, {
|
||||
model: options.model,
|
||||
temperature: 0.1,
|
||||
system: `You are a knowledge organizer. Your job is to find the correct home for the supplied facts.
|
||||
|
||||
1. Review the facts and the node list below. Pick the most likely match or decide if a new node is needed.
|
||||
2. If you picked an existing node, use \`read\` to verify it's the right place.
|
||||
- After reading, call either \`confirm\` (correct node) or \`mismatched\` (wrong node).
|
||||
3. If none of the nodes match, call \`create\` to make a new node.
|
||||
|
||||
**Organizational patterns:**
|
||||
- Journal entries: Journal/YYYY-MM-DD
|
||||
- People: People/Name
|
||||
- Projects: Projects/Name
|
||||
- Personal: Personal/Goals, Personal/Tasks, etc.
|
||||
- Knowledge: Biology/Topic, History/Topic, etc.
|
||||
|
||||
Available nodes:
|
||||
${candidates.map(n => `- ${n.name}: ${n.description}`).join('\n') || 'None — create a new node.'}`,
|
||||
tools: [{
|
||||
name: 'read',
|
||||
description: 'Read a node file to verify it is the right home for these facts',
|
||||
args: {name: {type: 'string', description: 'Exact node name (full path)', required: true}},
|
||||
fn: ({name}) => {
|
||||
const mem = memories.find(m => m.name === name);
|
||||
if (!mem) return 'Node not found';
|
||||
home = name;
|
||||
return this.formatMemory(mem);
|
||||
}
|
||||
}, {
|
||||
name: 'confirm',
|
||||
description: 'Confirm this is the correct node for the facts',
|
||||
args: {},
|
||||
fn: () => {
|
||||
mode = 'success';
|
||||
resp.abort();
|
||||
}
|
||||
}, {
|
||||
name: 'mismatched',
|
||||
description: 'This is not the node you are looking for',
|
||||
args: {},
|
||||
fn: () => {
|
||||
mode = 'failed';
|
||||
resp.abort();
|
||||
}
|
||||
}, {
|
||||
name: 'create',
|
||||
description: 'No existing node fits — create a new one',
|
||||
args: {
|
||||
name: {type: 'string', description: 'Full path for the new node (e.g., "People/Sarah", "Journal/2025-01-27")', required: true}
|
||||
},
|
||||
fn: ({name}) => {
|
||||
home = name;
|
||||
mode = 'create';
|
||||
resp.abort();
|
||||
}
|
||||
}]
|
||||
});
|
||||
|
||||
if(mode === 'create') {
|
||||
return this.createNode(home, memories);
|
||||
} else if (mode === 'failed') {
|
||||
candidates = candidates.filter(c => c.name !== home);
|
||||
if(!candidates.length) return this.createNode(bucket.subject, memories);
|
||||
} else if (mode === 'success') {
|
||||
const existing = memories.find(m => m.name === home);
|
||||
return existing || this.createNode(home, memories);
|
||||
}
|
||||
}
|
||||
return this.createNode(bucket.subject, memories);
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user