Improved memory management

package.json

@@ -1,6 +1,6 @@
 {
 	"name": "@ztimson/ai-utils",
-	"version": "1.2.2",
+	"version": "1.2.3",
 	"description": "AI Utility library",
 	"author": "Zak Timson",
 	"license": "MIT",

src/memory.ts

@@ -20,7 +20,6 @@ type FactBucket = {
 	subject: string;
 	facts: string[];
 }
-// In memory.ts - replace findGhostNodes with this:
 
 export type MemoryNode = {
 	name: string;
@@ -34,14 +33,12 @@ export function buildMemoryGraph(memories: Memory[] | MemoryCache): MemoryNode[]
 	const nameSet = new Set(mems.map(m => m.name));
 	const ghosts = new Set<string>();
 
-	// Collect all ghost references
 	for (const m of mems) {
 		for (const link of m.links) {
 			if (!nameSet.has(link)) ghosts.add(link);
 		}
 	}
 
-	// Build node list: real nodes + ghost nodes
 	return [
 		...mems.map(m => ({
 			name: m.name,
@@ -208,7 +205,6 @@ export class MemoryManager {
 		else vectorResults = this.cosineSearch(e.embedding, mem, limit);
 		const found = new Set<string>(vectorResults.map(r => r.name));
 
-		// Graph expansion
 		if(graphDepth > 0) {
 			const frontier = [...found];
 			for(let depth = 0; depth < graphDepth; depth++) {
@@ -239,17 +235,56 @@ export class MemoryManager {
 		const conversation = history
 			.filter(h => h.role === 'user' || h.role === 'assistant')
 			.map(h => `[${h.role}]: ${h.content}`).join('\n\n').trim();
-		if(!conversation) return;
 
+		if(conversation) {
 			const buckets = await this.factAgent(conversation, mem, options);
-		if(!buckets.length) return;
+			if(buckets.length) {
 				await Promise.all(buckets.map(async bucket => {
 					const node = await this.organizingAgent(bucket, mem, options);
 					if(!mem.find(m => m.name === node.name)) mem.push(node);
 					await this.docAgent(node, bucket, mem, options);
 				}));
+			}
+		}
+
+		// Auto-compress old journals
+		const weekAgo = Date.now() - (7 * 24 * 60 * 60 * 1000);
+		const oldDailies = mem.filter(m => {
+			const journal = /^Journal\/(\d{4}-\d{2}-\d{2}$)/.exec(m.name);
+			return journal && new Date(journal[1]).getTime() < weekAgo;
+		});
+
+		if(oldDailies.length) {
+			const byMonth = new Map<string, Memory[]>();
+			for(const daily of oldDailies) {
+				const match = daily.name.match(/^Journal\/(\d{4}-\d{2})-\d{2}$/);
+				if(!match) continue;
+				const monthKey = match[1];
+				if(!byMonth.has(monthKey)) byMonth.set(monthKey, []);
+				byMonth.get(monthKey)!.push(daily);
+			}
+
+			for(const [monthKey, entries] of byMonth) {
+				const monthlyPath = `Journal/${monthKey}`;
+				let monthly = mem.find(m => m.name === monthlyPath);
+				if(!monthly) {
+					monthly = this.createNode(monthlyPath, mem);
+					mem.push(monthly);
+				}
+
+				const bucket: FactBucket = {
+					subject: monthlyPath,
+					facts: entries.flatMap(e => e.content.split('\n').filter(line => line.trim())),
+				};
+
+				await this.docAgent(monthly, bucket, mem, options);
+				for(const daily of entries) {
+					const idx = mem.indexOf(daily);
+					if(idx !== -1) mem.splice(idx, 1);
+				}
+			}
+		}
 
-		// Rebuild indexes
 		if (memories instanceof MemoryCache) {
 			memories.rebuildLinks();
 			memories.rebuild();
@@ -260,17 +295,37 @@ export class MemoryManager {
 
 	private async docAgent(node: Memory, bucket: FactBucket, memories: Memory[], options: LLMRequest): Promise<void> {
 		let finalContent = node.content;
+		const isJournalCompression = node.name.match(/^Journal\/\d{4}-\d{2}$/);
+		const systemPrompt = isJournalCompression
+			? `You are a journal compressor. Condense the daily entries below into a monthly summary.
 
-		await this.llm.ask(
+Format:
-			`New facts to integrate:\n${bucket.facts.map(f => `- ${f}`).join('\n')}`,
+# ${node.name}
-			{
+
-				model: options.model,
+## Themes
-				temperature: 0.3,
+(Recurring topics, moods, patterns)
-				system: `You are a knowledge base editor. Integrate the provided facts into the document below.
+
+## Key Events
+(Important moments, decisions, milestones)
+
+## Notable Conversations
+(Significant discussions or revelations)
+
+Rules:
+- Use [[WikiLinks]] to reference permanent notes using full paths like [[People/Sarah]] or [[Projects/Website]]
+- Keep it concise but preserve emotional/temporal context
+- Discard filler but keep things the user vented about or cared about
+- If a fact belongs in a permanent note, link to it instead of duplicating
+
+Current monthly summary:
+\`\`\`markdown
+${node.content || '(empty — first compression for this month)'}
+\`\`\``
+			: `You are a knowledge base editor. Integrate the provided facts into the document below.
 
 Formatting rules:
 - Use Obsidian-style markdown: # headings, **bold** for key terms, bullet lists for facts
-- Link related concepts with [[WikiLink]] notation — only link things that are genuinely related
+- Link related concepts with [[WikiLink]] notation using full paths like [[People/Sarah]] or [[Projects/Website]]
 - You may create links to nodes that don't exist yet if the concept is important
 - Keep the document concise, factual, and human-readable
 - Resolve any contradictions between old content and new facts (new facts win)
@@ -282,7 +337,14 @@ ${this.listNodes(memories).map(n => n.name).join(', ') || 'none'}
 Current document:
 \`\`\`markdown
 ${node.content || '(empty — this is a new document)'}
-\`\`\``,
+\`\`\``;
+
+		await this.llm.ask(
+			`New facts to integrate:\n${bucket.facts.map(f => `- ${f}`).join('\n')}`,
+			{
+				model: options.model,
+				temperature: 0.3,
+				system: systemPrompt,
 				tools: [{
 					name: 'update_document',
 					description: 'Write the complete updated document content',
@@ -310,6 +372,7 @@ ${node.content || '(empty — this is a new document)'}
 
 	private async factAgent(conversation: string, memories: Memory[], options: LLMRequest): Promise<FactBucket[]> {
 		const buckets: FactBucket[] = [];
+		const today = new Date().toISOString().split('T')[0];
 
 		await this.llm.ask(conversation, {
 			model: options.model,
@@ -323,7 +386,16 @@ Rules:
 - DO NOT extract greetings, pleasantries, or generic exchanges
 - If nothing worth remembering was said, do not call any tools
 
-Group facts by subject. For each group call \`extract_facts\` once.
+**Organizational patterns:**
+- Journal entries use paths like: Journal/${today}
+- People use paths like: People/Name
+- Projects use paths like: Projects/Name
+- Personal info uses paths like: Personal/Goals, Personal/Tasks, etc.
+- General knowledge uses paths like: Biology/Topic, History/Topic, etc.
+
+Learn from existing nodes and follow the same pattern when extracting.
+
+Group facts by subject. For each group call \`extract_facts\` once with the FULL PATH.
 
 Known nodes (name: description):
 ${this.listNodes(memories).map(n => `- ${n.name}: ${n.description}`).join('\n') || 'None yet.'}`,
@@ -331,7 +403,7 @@ ${this.listNodes(memories).map(n => `- ${n.name}: ${n.description}`).join('\n')
 				name: 'extract_facts',
 				description: 'Submit a group of related facts for a specific subject',
 				args: {
-					subject: {type: 'string', description: 'Subject matter facts regard', required: true},
+					subject: {type: 'string', description: 'Full path for the subject (e.g., "Journal/2025-01-27", "People/Sarah", "Projects/Website")', required: true},
 					facts: {type: 'string', description: 'Comma-separated list of extracted facts', required: true},
 				},
 				fn: (args: any) => {
@@ -365,12 +437,19 @@ ${this.listNodes(memories).map(n => `- ${n.name}: ${n.description}`).join('\n')
 - After reading, call either \`confirm\` (correct node) or \`mismatched\` (wrong node).
 3. If none of the nodes match, call \`create\` to make a new node.
 
+**Organizational patterns:**
+- Journal entries: Journal/YYYY-MM-DD
+- People: People/Name
+- Projects: Projects/Name
+- Personal: Personal/Goals, Personal/Tasks, etc.
+- Knowledge: Biology/Topic, History/Topic, etc.
+
 Available nodes:
 ${candidates.map(n => `- ${n.name}: ${n.description}`).join('\n') || 'None — create a new node.'}`,
 				tools: [{
 					name: 'read',
 					description: 'Read a node file to verify it is the right home for these facts',
-					args: {name: {type: 'string', description: 'Exact node name', required: true}},
+					args: {name: {type: 'string', description: 'Exact node name (full path)', required: true}},
 					fn: ({name}) => {
 						const mem = memories.find(m => m.name === name);
 						if (!mem) return 'Node not found';
@@ -396,7 +475,9 @@ ${candidates.map(n => `- ${n.name}: ${n.description}`).join('\n') || 'None — c
 				}, {
 					name: 'create',
 					description: 'No existing node fits — create a new one',
-					args: {name: {type: 'string', description: 'Canonical name for the new node', required: true}},
+					args: {
+						name: {type: 'string', description: 'Full path for the new node (e.g., "People/Sarah", "Journal/2025-01-27")', required: true}
+					},
 					fn: ({name}) => {
 						home = name;
 						mode = 'create';

src/tools.ts

@@ -100,16 +100,11 @@ export const CliTool: AiTool = {
 
 export const DateTimeTool: AiTool = {
 	name: 'get_datetime',
-	description: 'Get local date / time',
+	description: 'Get local/UTC date/time',
-	args: {},
+	args: {
-	fn: async () => new Date().toString()
+		timezone: {type: 'string', description: 'Which timezone to return, defaults to local', enum: ['local', 'utc'], default: 'local'}
-}
+	},
-
+	fn: ({timezone}) => new Date()[timezone === 'local' ? 'toString' : 'toUTCString']()
-export const DateTimeUTCTool: AiTool = {
-	name: 'get_datetime_utc',
-	description: 'Get current UTC date / time',
-	args: {},
-	fn: async () => new Date().toUTCString()
 }
 
 export const ExecTool: AiTool = {
@@ -168,7 +163,7 @@ export const JSTool: AiTool = {
 }
 
 export const PythonTool: AiTool = {
-	name: 'exec_javascript',
+	name: 'exec_python',
 	description: 'Execute commonjs javascript',
 	args: {
 		code: {type: 'string', description: 'CommonJS javascript', required: true}