Compare commits
2 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| 3b5c71de7c | |||
| 8229e02a52 |
@@ -1,6 +1,6 @@
|
|||||||
{
|
{
|
||||||
"name": "@ztimson/ai-utils",
|
"name": "@ztimson/ai-utils",
|
||||||
"version": "1.2.2",
|
"version": "1.2.4",
|
||||||
"description": "AI Utility library",
|
"description": "AI Utility library",
|
||||||
"author": "Zak Timson",
|
"author": "Zak Timson",
|
||||||
"license": "MIT",
|
"license": "MIT",
|
||||||
|
|||||||
512
src/memory.ts
512
src/memory.ts
@@ -7,8 +7,6 @@ export type Memory = {
|
|||||||
description: string;
|
description: string;
|
||||||
content: string;
|
content: string;
|
||||||
embedding: number[];
|
embedding: number[];
|
||||||
links: string[];
|
|
||||||
backlinks: string[];
|
|
||||||
}
|
}
|
||||||
|
|
||||||
type MemoryRef = {
|
type MemoryRef = {
|
||||||
@@ -19,8 +17,8 @@ type MemoryRef = {
|
|||||||
type FactBucket = {
|
type FactBucket = {
|
||||||
subject: string;
|
subject: string;
|
||||||
facts: string[];
|
facts: string[];
|
||||||
|
isNew: boolean;
|
||||||
}
|
}
|
||||||
// In memory.ts - replace findGhostNodes with this:
|
|
||||||
|
|
||||||
export type MemoryNode = {
|
export type MemoryNode = {
|
||||||
name: string;
|
name: string;
|
||||||
@@ -34,28 +32,31 @@ export function buildMemoryGraph(memories: Memory[] | MemoryCache): MemoryNode[]
|
|||||||
const nameSet = new Set(mems.map(m => m.name));
|
const nameSet = new Set(mems.map(m => m.name));
|
||||||
const ghosts = new Set<string>();
|
const ghosts = new Set<string>();
|
||||||
|
|
||||||
// Collect all ghost references
|
const nodes: MemoryNode[] = mems.map(m => {
|
||||||
for (const m of mems) {
|
const {links, backlinks} = extractMetadata(m.content);
|
||||||
for (const link of m.links) {
|
return {
|
||||||
|
name: m.name,
|
||||||
|
missing: false,
|
||||||
|
links,
|
||||||
|
backlinks,
|
||||||
|
};
|
||||||
|
});
|
||||||
|
|
||||||
|
for (const node of nodes) {
|
||||||
|
for (const link of node.links) {
|
||||||
if (!nameSet.has(link)) ghosts.add(link);
|
if (!nameSet.has(link)) ghosts.add(link);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// Build node list: real nodes + ghost nodes
|
|
||||||
return [
|
return [
|
||||||
...mems.map(m => ({
|
...nodes,
|
||||||
name: m.name,
|
|
||||||
missing: false,
|
|
||||||
links: m.links,
|
|
||||||
backlinks: m.backlinks,
|
|
||||||
})),
|
|
||||||
...[...ghosts].map(name => ({
|
...[...ghosts].map(name => ({
|
||||||
name,
|
name,
|
||||||
missing: true,
|
missing: true,
|
||||||
links: [],
|
links: [],
|
||||||
backlinks: mems
|
backlinks: nodes
|
||||||
.filter(m => m.links.includes(name))
|
.filter(n => n.links.includes(name))
|
||||||
.map(m => m.name),
|
.map(n => n.name),
|
||||||
}))
|
}))
|
||||||
];
|
];
|
||||||
}
|
}
|
||||||
@@ -65,14 +66,21 @@ function extractLinks(content: string): string[] {
|
|||||||
return [...new Set([...matches].map(m => m[1].trim()))];
|
return [...new Set([...matches].map(m => m[1].trim()))];
|
||||||
}
|
}
|
||||||
|
|
||||||
function rebuildBacklinks(memories: Memory[]): void {
|
export function extractMetadata(content: string): {links: string[], backlinks: string[]} {
|
||||||
for (const m of memories) m.backlinks = [];
|
const match = content.match(/^---\n([\s\S]*?)\n---/);
|
||||||
for (const m of memories) {
|
if (!match) return {links: [], backlinks: []};
|
||||||
for (const link of m.links) {
|
|
||||||
const target = memories.find(t => t.name === link);
|
const fm = match[1];
|
||||||
if (target) target.backlinks.push(m.name);
|
const getList = (key: string): string[] => {
|
||||||
}
|
const m = fm.match(new RegExp(`^${key}:\\s*\\[(.*)\\]$`, 'm'));
|
||||||
}
|
if (!m || !m[1].trim()) return [];
|
||||||
|
return m[1].split(',').map(s => s.trim().replace(/^"|"$/g, '')).filter(Boolean);
|
||||||
|
};
|
||||||
|
|
||||||
|
return {
|
||||||
|
links: getList('links'),
|
||||||
|
backlinks: getList('backlinks'),
|
||||||
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
function cosineDistance(a: number[], b: number[]): number {
|
function cosineDistance(a: number[], b: number[]): number {
|
||||||
@@ -86,9 +94,53 @@ function cosineDistance(a: number[], b: number[]): number {
|
|||||||
return denom === 0 ? 1 : 1 - dot / denom;
|
return denom === 0 ? 1 : 1 - dot / denom;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
function getWeekMonday(date: Date = new Date()): string {
|
||||||
|
const d = new Date(Date.UTC(date.getFullYear(), date.getMonth(), date.getDate()));
|
||||||
|
const day = d.getUTCDay();
|
||||||
|
const diff = day === 0 ? -6 : 1 - day;
|
||||||
|
d.setUTCDate(d.getUTCDate() + diff);
|
||||||
|
return d.toISOString().slice(0, 10);
|
||||||
|
}
|
||||||
|
|
||||||
|
function getWeekSunday(monday: string): string {
|
||||||
|
const d = new Date(`${monday}T00:00:00Z`);
|
||||||
|
d.setUTCDate(d.getUTCDate() + 6);
|
||||||
|
return d.toISOString().slice(0, 10);
|
||||||
|
}
|
||||||
|
|
||||||
|
function tagsFromName(name: string): string[] {
|
||||||
|
const prefix = name.split('/')[0];
|
||||||
|
return prefix ? [prefix.toLowerCase()] : [];
|
||||||
|
}
|
||||||
|
|
||||||
|
export function serializeMemory(mem: Memory, week?: {monday: string, sunday: string}): string {
|
||||||
|
return mem.content;
|
||||||
|
}
|
||||||
|
|
||||||
|
export function deserializeMemory(raw: string, embedding: number[] = []): Memory {
|
||||||
|
const match = raw.match(/^---\n([\s\S]*?)\n---\n\n?([\s\S]*)$/);
|
||||||
|
if (!match) {
|
||||||
|
return {name: '', description: '', content: raw.trim(), embedding};
|
||||||
|
}
|
||||||
|
|
||||||
|
const [, fm] = match;
|
||||||
|
const get = (key: string): string => {
|
||||||
|
const m = fm.match(new RegExp(`^${key}:\\s*(.+)$`, 'm'));
|
||||||
|
return m ? m[1].trim() : '';
|
||||||
|
};
|
||||||
|
|
||||||
|
return {
|
||||||
|
name: get('name'),
|
||||||
|
description: get('description'),
|
||||||
|
content: raw.trim(),
|
||||||
|
embedding,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
export class MemoryCache {
|
export class MemoryCache {
|
||||||
private tree: KDTree<MemoryRef>;
|
private tree: KDTree<MemoryRef>;
|
||||||
public memories: Memory[];
|
public memories: Memory[];
|
||||||
|
private locks = new Map<string, Promise<void>>();
|
||||||
|
|
||||||
constructor(memories: Memory[]) {
|
constructor(memories: Memory[]) {
|
||||||
this.memories = memories;
|
this.memories = memories;
|
||||||
@@ -128,16 +180,38 @@ export class MemoryCache {
|
|||||||
this.rebuild();
|
this.rebuild();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
remove(name: string): void {
|
||||||
|
const idx = this.memories.findIndex(m => m.name === name);
|
||||||
|
if (idx !== -1) {
|
||||||
|
this.memories.splice(idx, 1);
|
||||||
|
this.rebuild();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
rebuild(): void {
|
rebuild(): void {
|
||||||
this.tree = this.buildTree();
|
this.tree = this.buildTree();
|
||||||
}
|
}
|
||||||
|
|
||||||
rebuildLinks(): void {
|
lock<T>(name: string, fn: () => Promise<T>): Promise<T> {
|
||||||
rebuildBacklinks(this.memories);
|
const prev = this.locks.get(name) ?? Promise.resolve();
|
||||||
|
let resolveLock!: () => void;
|
||||||
|
const next = new Promise<void>(r => { resolveLock = r; });
|
||||||
|
this.locks.set(name, next);
|
||||||
|
|
||||||
|
const result = prev.then(fn).finally(resolveLock);
|
||||||
|
result.finally(() => {
|
||||||
|
if (this.locks.get(name) === next) this.locks.delete(name);
|
||||||
|
});
|
||||||
|
return result;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
export class MemoryManager {
|
export class MemoryManager {
|
||||||
|
private pendingMemorizations = new Map<string, {
|
||||||
|
memories: Memory[] | MemoryCache,
|
||||||
|
tempMemoryName: string,
|
||||||
|
timestamp: number,
|
||||||
|
}>();
|
||||||
|
|
||||||
tools = {
|
tools = {
|
||||||
read: (memories: Memory[] | MemoryCache): AiTool => ({
|
read: (memories: Memory[] | MemoryCache): AiTool => ({
|
||||||
@@ -150,19 +224,78 @@ export class MemoryManager {
|
|||||||
const mems = memories instanceof MemoryCache ? memories.memories : memories;
|
const mems = memories instanceof MemoryCache ? memories.memories : memories;
|
||||||
const mem = mems.find(m => m.name === args.name);
|
const mem = mems.find(m => m.name === args.name);
|
||||||
if (!mem) return 'Document not found';
|
if (!mem) return 'Document not found';
|
||||||
return this.formatMemory(mem);
|
return mem.content;
|
||||||
}
|
},
|
||||||
|
}),
|
||||||
|
|
||||||
|
forget: (memories: Memory[] | MemoryCache): AiTool => ({
|
||||||
|
name: 'forget_memory',
|
||||||
|
description: 'Permanently delete a memory document and clean up all references to it',
|
||||||
|
args: {
|
||||||
|
name: {type: 'string', description: 'Exact memory name to forget', required: true},
|
||||||
|
reason: {type: 'string', description: 'Why this memory is being deleted', required: true},
|
||||||
|
},
|
||||||
|
fn: (args: any) => {
|
||||||
|
const result = this.forget(args.name, memories);
|
||||||
|
return result ? `Forgotten: ${args.name}` : `Not found: ${args.name}`;
|
||||||
|
},
|
||||||
}),
|
}),
|
||||||
};
|
};
|
||||||
|
|
||||||
constructor(private llm: any) {}
|
constructor(private llm: any) {}
|
||||||
|
|
||||||
|
private async createTempMemory(conversation: string): Promise<Memory> {
|
||||||
|
const [e] = await this.llm.embedding(conversation);
|
||||||
|
const timestamp = Date.now();
|
||||||
|
return {
|
||||||
|
name: `_temp_${timestamp}`,
|
||||||
|
description: 'Temporary memory - processing in background',
|
||||||
|
content: `---
|
||||||
|
name: _temp_${timestamp}
|
||||||
|
description: Temporary memory - processing in background
|
||||||
|
tags: [_temporary]
|
||||||
|
links: []
|
||||||
|
backlinks: []
|
||||||
|
modified: ${new Date().toISOString()}
|
||||||
|
---
|
||||||
|
|
||||||
|
# Recent Conversation (Processing)
|
||||||
|
|
||||||
|
${conversation}`,
|
||||||
|
embedding: e?.embedding || [],
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
forget(name: string, memories: Memory[] | MemoryCache): boolean {
|
||||||
|
const mem = memories instanceof MemoryCache ? memories.memories : memories;
|
||||||
|
const idx = mem.findIndex(m => m.name === name);
|
||||||
|
if (idx === -1) return false;
|
||||||
|
|
||||||
|
for (const node of mem) {
|
||||||
|
const {links, backlinks} = extractMetadata(node.content);
|
||||||
|
const newBacklinks = backlinks.filter(b => b !== name);
|
||||||
|
const newLinks = links.filter(l => l !== name);
|
||||||
|
|
||||||
|
if (newBacklinks.length !== backlinks.length || newLinks.length !== links.length) {
|
||||||
|
node.content = this.updateFrontmatter(node.content, {
|
||||||
|
links: newLinks,
|
||||||
|
backlinks: newBacklinks,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
mem.splice(idx, 1);
|
||||||
|
|
||||||
|
if (memories instanceof MemoryCache) memories.rebuild();
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
private cosineSearch(query: number[], memories: Memory[], limit: number): MemoryRef[] {
|
private cosineSearch(query: number[], memories: Memory[], limit: number): MemoryRef[] {
|
||||||
const scored = memories
|
const scored = memories
|
||||||
.filter(m => m.embedding?.length)
|
.filter(m => m.embedding?.length)
|
||||||
.map(m => ({
|
.map(m => ({
|
||||||
ref: {name: m.name, description: m.description},
|
ref: {name: m.name, description: m.description},
|
||||||
distance: cosineDistance(query, m.embedding)
|
distance: cosineDistance(query, m.embedding),
|
||||||
}))
|
}))
|
||||||
.sort((a, b) => a.distance - b.distance)
|
.sort((a, b) => a.distance - b.distance)
|
||||||
.slice(0, limit);
|
.slice(0, limit);
|
||||||
@@ -177,22 +310,9 @@ export class MemoryManager {
|
|||||||
description: '',
|
description: '',
|
||||||
content: '',
|
content: '',
|
||||||
embedding: [],
|
embedding: [],
|
||||||
links: [],
|
|
||||||
backlinks: [],
|
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
private formatMemory(mem: Memory): string {
|
|
||||||
return [
|
|
||||||
`# ${mem.name}`,
|
|
||||||
mem.description ? `> ${mem.description}` : '',
|
|
||||||
mem.links.length ? `**Links:** ${mem.links.map(l => `[[${l}]]`).join(', ')}` : '',
|
|
||||||
mem.backlinks.length ? `**Referenced by:** ${mem.backlinks.map(l => `[[${l}]]`).join(', ')}` : '',
|
|
||||||
'',
|
|
||||||
mem.content,
|
|
||||||
].filter(l => l !== undefined).join('\n');
|
|
||||||
}
|
|
||||||
|
|
||||||
private listNodes(memories: Memory[]): MemoryRef[] {
|
private listNodes(memories: Memory[]): MemoryRef[] {
|
||||||
return memories.map(m => ({name: m.name, description: m.description}));
|
return memories.map(m => ({name: m.name, description: m.description}));
|
||||||
}
|
}
|
||||||
@@ -200,6 +320,7 @@ export class MemoryManager {
|
|||||||
async recollect(query: string, memories: Memory[] | MemoryCache, limit = 5, graphDepth = 1): Promise<Memory[]> {
|
async recollect(query: string, memories: Memory[] | MemoryCache, limit = 5, graphDepth = 1): Promise<Memory[]> {
|
||||||
const mem: Memory[] = memories instanceof MemoryCache ? memories.memories : memories;
|
const mem: Memory[] = memories instanceof MemoryCache ? memories.memories : memories;
|
||||||
if (!mem.length) return [];
|
if (!mem.length) return [];
|
||||||
|
|
||||||
const [e] = await this.llm.embedding(query);
|
const [e] = await this.llm.embedding(query);
|
||||||
if (!e) return [];
|
if (!e) return [];
|
||||||
|
|
||||||
@@ -208,7 +329,6 @@ export class MemoryManager {
|
|||||||
else vectorResults = this.cosineSearch(e.embedding, mem, limit);
|
else vectorResults = this.cosineSearch(e.embedding, mem, limit);
|
||||||
const found = new Set<string>(vectorResults.map(r => r.name));
|
const found = new Set<string>(vectorResults.map(r => r.name));
|
||||||
|
|
||||||
// Graph expansion
|
|
||||||
if (graphDepth > 0) {
|
if (graphDepth > 0) {
|
||||||
const frontier = [...found];
|
const frontier = [...found];
|
||||||
for (let depth = 0; depth < graphDepth; depth++) {
|
for (let depth = 0; depth < graphDepth; depth++) {
|
||||||
@@ -216,7 +336,8 @@ export class MemoryManager {
|
|||||||
for (const name of frontier) {
|
for (const name of frontier) {
|
||||||
const node = mem.find(m => m.name === name);
|
const node = mem.find(m => m.name === name);
|
||||||
if (!node) continue;
|
if (!node) continue;
|
||||||
for(const link of node.links) {
|
const {links} = extractMetadata(node.content);
|
||||||
|
for (const link of links) {
|
||||||
if (!found.has(link) && mem.find(m => m.name === link)) {
|
if (!found.has(link) && mem.find(m => m.name === link)) {
|
||||||
found.add(link);
|
found.add(link);
|
||||||
next.push(link);
|
next.push(link);
|
||||||
@@ -235,30 +356,152 @@ export class MemoryManager {
|
|||||||
}
|
}
|
||||||
|
|
||||||
async memorize(history: LLMMessage[], memories: Memory[] | MemoryCache, options: LLMRequest): Promise<void> {
|
async memorize(history: LLMMessage[], memories: Memory[] | MemoryCache, options: LLMRequest): Promise<void> {
|
||||||
const mem = memories instanceof MemoryCache ? memories.memories : memories;
|
|
||||||
const conversation = history
|
const conversation = history
|
||||||
.filter(h => h.role === 'user' || h.role === 'assistant')
|
.filter(h => h.role === 'user' || h.role === 'assistant')
|
||||||
.map(h => `[${h.role}]: ${h.content}`).join('\n\n').trim();
|
.map(h => `[${h.role}]: ${h.content}`).join('\n\n').trim();
|
||||||
|
|
||||||
if (!conversation) return;
|
if (!conversation) return;
|
||||||
|
|
||||||
const buckets = await this.factAgent(conversation, mem, options);
|
const trackingId = `${Date.now()}_${Math.random()}`;
|
||||||
if(!buckets.length) return;
|
|
||||||
|
// Create and insert temp memory immediately
|
||||||
|
const tempMemory = await this.createTempMemory(conversation);
|
||||||
|
const mem = memories instanceof MemoryCache ? memories.memories : memories;
|
||||||
|
mem.push(tempMemory);
|
||||||
|
|
||||||
|
if (memories instanceof MemoryCache) {
|
||||||
|
memories.rebuild();
|
||||||
|
}
|
||||||
|
|
||||||
|
this.pendingMemorizations.set(trackingId, {
|
||||||
|
memories,
|
||||||
|
tempMemoryName: tempMemory.name,
|
||||||
|
timestamp: Date.now(),
|
||||||
|
});
|
||||||
|
|
||||||
|
this._memorizeBackground(conversation, memories, options, trackingId)
|
||||||
|
.catch(err => {
|
||||||
|
console.error('[memorize] Background memorization failed:', err);
|
||||||
|
})
|
||||||
|
.finally(() => {
|
||||||
|
// Remove temp memory from the exact same memory array/cache
|
||||||
|
const pending = this.pendingMemorizations.get(trackingId);
|
||||||
|
if (pending) {
|
||||||
|
const cleanMem = pending.memories instanceof MemoryCache
|
||||||
|
? pending.memories.memories
|
||||||
|
: pending.memories;
|
||||||
|
const idx = cleanMem.findIndex(m => m.name === pending.tempMemoryName);
|
||||||
|
if (idx !== -1) {
|
||||||
|
cleanMem.splice(idx, 1);
|
||||||
|
console.log(`[memorize] Removed temp memory: ${pending.tempMemoryName}`);
|
||||||
|
}
|
||||||
|
if (pending.memories instanceof MemoryCache) {
|
||||||
|
pending.memories.rebuild();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
this.pendingMemorizations.delete(trackingId);
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
private async _memorizeBackground(conversation: string, memories: Memory[] | MemoryCache, options: LLMRequest, trackingId: string): Promise<void> {
|
||||||
|
const mem = memories instanceof MemoryCache ? memories.memories : memories;
|
||||||
|
const monday = getWeekMonday();
|
||||||
|
const sunday = getWeekSunday(monday);
|
||||||
|
const weekKey = monday;
|
||||||
|
|
||||||
|
console.log('[memorize] Starting fact extraction...');
|
||||||
|
const buckets = await this.factAgent(conversation, mem, options, weekKey);
|
||||||
|
console.log(`[memorize] Extracted ${buckets.length} buckets:`, buckets);
|
||||||
|
|
||||||
|
if (!buckets.length) {
|
||||||
|
console.log('[memorize] No facts extracted, exiting');
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
const runDocAgent = (node: Memory, bucket: FactBucket, embedding?: number[], week?: {monday: string, sunday: string}) => {
|
||||||
|
if (memories instanceof MemoryCache) {
|
||||||
|
return memories.lock(node.name, () => this.docAgent(node, bucket, mem, options, embedding, week));
|
||||||
|
}
|
||||||
|
return this.docAgent(node, bucket, mem, options, embedding, week);
|
||||||
|
};
|
||||||
|
|
||||||
await Promise.all(buckets.map(async bucket => {
|
await Promise.all(buckets.map(async bucket => {
|
||||||
const node = await this.organizingAgent(bucket, mem, options);
|
let node = mem.find(m => m.name === bucket.subject && !m.name.startsWith('_temp_'));
|
||||||
if(!mem.find(m => m.name === node.name)) mem.push(node);
|
let embedding: number[] | undefined;
|
||||||
await this.docAgent(node, bucket, mem, options);
|
|
||||||
|
if (!node || bucket.isNew) {
|
||||||
|
const [e] = await this.llm.embedding(`${bucket.subject}\n${bucket.facts.join('\n')}`);
|
||||||
|
embedding = e?.embedding;
|
||||||
|
|
||||||
|
if (!node) {
|
||||||
|
node = this.createNode(bucket.subject, mem);
|
||||||
|
mem.push(node);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
const week = bucket.subject.startsWith('Journal/') ? {monday, sunday} : undefined;
|
||||||
|
await runDocAgent(node, bucket, embedding, week);
|
||||||
}));
|
}));
|
||||||
|
|
||||||
// Rebuild indexes
|
|
||||||
if (memories instanceof MemoryCache) {
|
if (memories instanceof MemoryCache) {
|
||||||
memories.rebuildLinks();
|
|
||||||
memories.rebuild();
|
memories.rebuild();
|
||||||
} else {
|
|
||||||
rebuildBacklinks(mem);
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
private async docAgent(node: Memory, bucket: FactBucket, memories: Memory[], options: LLMRequest): Promise<void> {
|
console.log('[memorize] Completed successfully');
|
||||||
|
}
|
||||||
|
|
||||||
|
private buildHeader(node: Memory, week?: {monday: string, sunday: string}, links: string[] = [], backlinks: string[] = []): string {
|
||||||
|
const tags = node.name.split('/')[0]?.toLowerCase();
|
||||||
|
const lines = [
|
||||||
|
'---',
|
||||||
|
`name: ${node.name}`,
|
||||||
|
`description: ${node.description || ''}`,
|
||||||
|
tags ? `tags: [${tags}]` : '',
|
||||||
|
links.length ? `links: [${links.map(l => `"${l}"`).join(', ')}]` : 'links: []',
|
||||||
|
backlinks.length ? `backlinks: [${backlinks.map(l => `"${l}"`).join(', ')}]` : 'backlinks: []',
|
||||||
|
week ? `week: ${week.monday} – ${week.sunday}` : '',
|
||||||
|
`modified: ${new Date().toISOString()}`,
|
||||||
|
'---',
|
||||||
|
].filter(Boolean);
|
||||||
|
return lines.join('\n');
|
||||||
|
}
|
||||||
|
|
||||||
|
private applyHeader(content: string, header: string): string {
|
||||||
|
const hasFrontmatter = content.trimStart().startsWith('---');
|
||||||
|
if (hasFrontmatter) {
|
||||||
|
return content.replace(/^---[\s\S]*?---\n?/, `${header}\n`);
|
||||||
|
}
|
||||||
|
return `${header}\n\n${content}`;
|
||||||
|
}
|
||||||
|
|
||||||
|
private updateFrontmatter(content: string, updates: {links?: string[], backlinks?: string[]}): string {
|
||||||
|
const match = content.match(/^---\n([\s\S]*?)\n---\n\n?([\s\S]*)$/);
|
||||||
|
if (!match) return content;
|
||||||
|
|
||||||
|
const [, fm, body] = match;
|
||||||
|
let newFm = fm;
|
||||||
|
|
||||||
|
if (updates.links !== undefined) {
|
||||||
|
const linksList = updates.links.length ? `[${updates.links.map(l => `"${l}"`).join(', ')}]` : '[]';
|
||||||
|
newFm = newFm.replace(/^links:.*$/m, `links: ${linksList}`);
|
||||||
|
}
|
||||||
|
|
||||||
|
if (updates.backlinks !== undefined) {
|
||||||
|
const backlinksList = updates.backlinks.length ? `[${updates.backlinks.map(l => `"${l}"`).join(', ')}]` : '[]';
|
||||||
|
newFm = newFm.replace(/^backlinks:.*$/m, `backlinks: ${backlinksList}`);
|
||||||
|
}
|
||||||
|
|
||||||
|
newFm = newFm.replace(/^modified:.*$/m, `modified: ${new Date().toISOString()}`);
|
||||||
|
|
||||||
|
return `---\n${newFm}\n---\n\n${body}`;
|
||||||
|
}
|
||||||
|
|
||||||
|
private stripHeader(content: string): string {
|
||||||
|
return content.replace(/^---[\s\S]*?---\n?/, '').trimStart();
|
||||||
|
}
|
||||||
|
|
||||||
|
private async docAgent(node: Memory, bucket: FactBucket, memories: Memory[], options: LLMRequest, precomputedEmbedding?: number[], week?: {monday: string, sunday: string}): Promise<void> {
|
||||||
|
const {links: oldLinks} = extractMetadata(node.content);
|
||||||
let finalContent = node.content;
|
let finalContent = node.content;
|
||||||
|
|
||||||
await this.llm.ask(
|
await this.llm.ask(
|
||||||
@@ -270,12 +513,13 @@ export class MemoryManager {
|
|||||||
|
|
||||||
Formatting rules:
|
Formatting rules:
|
||||||
- Use Obsidian-style markdown: # headings, **bold** for key terms, bullet lists for facts
|
- Use Obsidian-style markdown: # headings, **bold** for key terms, bullet lists for facts
|
||||||
- Link related concepts with [[WikiLink]] notation — only link things that are genuinely related
|
- Link related concepts with [[WikiLink]] notation using full paths like [[People/Sarah]] or [[Projects/Website]]
|
||||||
- You may create links to nodes that don't exist yet if the concept is important
|
- You may create links to nodes that don't exist yet if the concept is important
|
||||||
- Keep the document concise, factual, and human-readable
|
- Keep the document concise, factual, and human-readable
|
||||||
- Resolve any contradictions between old content and new facts (new facts win)
|
- Resolve any contradictions between old content and new facts (new facts win)
|
||||||
- Do not add filler, preamble, or AI commentary — just clean knowledge documents
|
- Do not add filler, preamble, or AI commentary — just clean knowledge documents
|
||||||
|
- The document begins with a YAML frontmatter block (between --- markers) — do not remove or rewrite it, it is maintained automatically
|
||||||
|
${week ? '- This is a weekly journal entry. The frontmatter contains the week date range.\n' : ''}
|
||||||
All nodes:
|
All nodes:
|
||||||
${this.listNodes(memories).map(n => n.name).join(', ') || 'none'}
|
${this.listNodes(memories).map(n => n.name).join(', ') || 'none'}
|
||||||
|
|
||||||
@@ -285,32 +529,67 @@ ${node.content || '(empty — this is a new document)'}
|
|||||||
\`\`\``,
|
\`\`\``,
|
||||||
tools: [{
|
tools: [{
|
||||||
name: 'update_document',
|
name: 'update_document',
|
||||||
description: 'Write the complete updated document content',
|
description: 'Write the complete updated document content. Include everything after the frontmatter block — the frontmatter will be recalculated automatically.',
|
||||||
args: {
|
args: {
|
||||||
description: {type: 'string', description: 'One-line description of what this document covers, no formatting or emojis', required: true},
|
description: {type: 'string', description: 'One-line description of what this document covers, no formatting or emojis', required: true},
|
||||||
content: {type: 'string', description: 'Fully updated document in markdown', required: true},
|
content: {type: 'string', description: 'Document body in markdown, without the frontmatter block', required: true},
|
||||||
},
|
},
|
||||||
fn: (args: any) => {
|
fn: (args: any) => {
|
||||||
node.description = args.description;
|
node.description = args.description;
|
||||||
finalContent = args.content;
|
finalContent = args.content;
|
||||||
return 'Saved';
|
return 'Saved';
|
||||||
}
|
},
|
||||||
}]
|
}],
|
||||||
}
|
}
|
||||||
);
|
);
|
||||||
|
|
||||||
node.content = finalContent;
|
const newLinks = extractLinks(finalContent).filter(l => l !== node.name);
|
||||||
node.links = extractLinks(finalContent);
|
const newLinkSet = new Set(newLinks);
|
||||||
const needsEmbed = !node.embedding?.length || node.description !== memories.find(m => m.name === node.name)?.description;
|
const oldLinkSet = new Set(oldLinks);
|
||||||
if (needsEmbed) {
|
|
||||||
const [e] = await this.llm.embedding(node.description);
|
for (const added of newLinkSet) {
|
||||||
|
if (!oldLinkSet.has(added)) {
|
||||||
|
const target = memories.find(m => m.name === added);
|
||||||
|
if (target) {
|
||||||
|
const {backlinks} = extractMetadata(target.content);
|
||||||
|
if (!backlinks.includes(node.name)) {
|
||||||
|
target.content = this.updateFrontmatter(target.content, {
|
||||||
|
backlinks: [...backlinks, node.name],
|
||||||
|
});
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
for (const removed of oldLinkSet) {
|
||||||
|
if (!newLinkSet.has(removed)) {
|
||||||
|
const target = memories.find(m => m.name === removed);
|
||||||
|
if (target) {
|
||||||
|
const {backlinks} = extractMetadata(target.content);
|
||||||
|
target.content = this.updateFrontmatter(target.content, {
|
||||||
|
backlinks: backlinks.filter(b => b !== node.name),
|
||||||
|
});
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
const {backlinks} = extractMetadata(node.content);
|
||||||
|
const header = this.buildHeader(node, week, newLinks, backlinks);
|
||||||
|
node.content = this.applyHeader(finalContent, header);
|
||||||
|
|
||||||
|
if (precomputedEmbedding) {
|
||||||
|
node.embedding = precomputedEmbedding;
|
||||||
|
} else {
|
||||||
|
const embedInput = `${node.description}\n\n${this.stripHeader(node.content)}`.trim();
|
||||||
|
const [e] = await this.llm.embedding(embedInput);
|
||||||
if (e) node.embedding = e.embedding;
|
if (e) node.embedding = e.embedding;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
private async factAgent(conversation: string, memories: Memory[], options: LLMRequest): Promise<FactBucket[]> {
|
private async factAgent(conversation: string, memories: Memory[], options: LLMRequest, weekKey: string): Promise<FactBucket[]> {
|
||||||
const buckets: FactBucket[] = [];
|
const buckets: FactBucket[] = [];
|
||||||
|
|
||||||
|
console.log('[factAgent] Starting extraction...');
|
||||||
|
|
||||||
await this.llm.ask(conversation, {
|
await this.llm.ask(conversation, {
|
||||||
model: options.model,
|
model: options.model,
|
||||||
temperature: 0.2,
|
temperature: 0.2,
|
||||||
@@ -323,98 +602,37 @@ Rules:
|
|||||||
- DO NOT extract greetings, pleasantries, or generic exchanges
|
- DO NOT extract greetings, pleasantries, or generic exchanges
|
||||||
- If nothing worth remembering was said, do not call any tools
|
- If nothing worth remembering was said, do not call any tools
|
||||||
|
|
||||||
Group facts by subject. For each group call \`extract_facts\` once.
|
When extracting facts, you MUST also decide the exact destination path:
|
||||||
|
- Use an existing node name if the facts clearly belong there
|
||||||
|
- Create a new path following collection/subject format if needed (e.g., People/Sarah, Projects/Oxide)
|
||||||
|
- For journal entries, use "journal" (will auto-route to Journal/${weekKey})
|
||||||
|
|
||||||
Known nodes (name: description):
|
Available nodes:
|
||||||
${this.listNodes(memories).map(n => `- ${n.name}: ${n.description}`).join('\n') || 'None yet.'}`,
|
${this.listNodes(memories).map(n => `- ${n.name}: ${n.description}`).join('\n') || 'None yet.'}`,
|
||||||
tools: [{
|
tools: [{
|
||||||
name: 'extract_facts',
|
name: 'extract_facts',
|
||||||
description: 'Submit a group of related facts for a specific subject',
|
description: 'Submit facts with their destination',
|
||||||
args: {
|
args: {
|
||||||
subject: {type: 'string', description: 'Subject matter facts regard', required: true},
|
destination: {type: 'string', description: 'Exact existing node name OR new path (e.g. "People/Sarah", "Projects/Oxide")', required: true},
|
||||||
facts: {type: 'string', description: 'Comma-separated list of extracted facts', required: true},
|
facts: {type: 'string', description: 'Comma-separated facts', required: true},
|
||||||
|
create_new: {type: 'boolean', description: 'True if this is a new node that doesn\'t exist yet', required: true},
|
||||||
},
|
},
|
||||||
fn: (args: any) => {
|
fn: (args: any) => {
|
||||||
|
console.log('[factAgent] Tool called with:', args);
|
||||||
|
const subject = args.destination.trim().toLowerCase() === 'journal'
|
||||||
|
? `Journal/${weekKey}`
|
||||||
|
: args.destination;
|
||||||
buckets.push({
|
buckets.push({
|
||||||
subject: args.subject,
|
subject,
|
||||||
facts: args.facts.split(',').map((f: string) => f.trim()).filter(Boolean),
|
facts: args.facts.split(',').map((f: string) => f.trim()).filter(Boolean),
|
||||||
|
isNew: args.create_new,
|
||||||
});
|
});
|
||||||
return 'Recorded';
|
return 'Recorded';
|
||||||
}
|
},
|
||||||
}]
|
}],
|
||||||
});
|
});
|
||||||
|
|
||||||
|
console.log(`[factAgent] Extracted ${buckets.length} buckets:`, buckets);
|
||||||
return buckets;
|
return buckets;
|
||||||
}
|
}
|
||||||
|
|
||||||
private async organizingAgent(bucket: FactBucket, memories: Memory[], options: LLMRequest): Promise<Memory> {
|
|
||||||
let candidates = this.listNodes(memories);
|
|
||||||
let attempts = 0;
|
|
||||||
const maxAttempts = 3;
|
|
||||||
|
|
||||||
while (attempts++ < maxAttempts) {
|
|
||||||
let home = '', mode: string | null = null;
|
|
||||||
|
|
||||||
const resp = await this.llm.ask(`Subject: ${bucket.subject}\n\nFacts:\n${bucket.facts.map(f => `- ${f}`).join('\n')}`, {
|
|
||||||
model: options.model,
|
|
||||||
temperature: 0.1,
|
|
||||||
system: `You are a knowledge organizer. Your job is to find the correct home for the supplied facts.
|
|
||||||
|
|
||||||
1. Review the facts and the node list below. Pick the most likely match or decide if a new node is needed.
|
|
||||||
2. If you picked an existing node, use \`read\` to verify it's the right place.
|
|
||||||
- After reading, call either \`confirm\` (correct node) or \`mismatched\` (wrong node).
|
|
||||||
3. If none of the nodes match, call \`create\` to make a new node.
|
|
||||||
|
|
||||||
Available nodes:
|
|
||||||
${candidates.map(n => `- ${n.name}: ${n.description}`).join('\n') || 'None — create a new node.'}`,
|
|
||||||
tools: [{
|
|
||||||
name: 'read',
|
|
||||||
description: 'Read a node file to verify it is the right home for these facts',
|
|
||||||
args: {name: {type: 'string', description: 'Exact node name', required: true}},
|
|
||||||
fn: ({name}) => {
|
|
||||||
const mem = memories.find(m => m.name === name);
|
|
||||||
if (!mem) return 'Node not found';
|
|
||||||
home = name;
|
|
||||||
return this.formatMemory(mem);
|
|
||||||
}
|
|
||||||
}, {
|
|
||||||
name: 'confirm',
|
|
||||||
description: 'Confirm this is the correct node for the facts',
|
|
||||||
args: {},
|
|
||||||
fn: () => {
|
|
||||||
mode = 'success';
|
|
||||||
resp.abort();
|
|
||||||
}
|
|
||||||
}, {
|
|
||||||
name: 'mismatched',
|
|
||||||
description: 'This is not the node you are looking for',
|
|
||||||
args: {},
|
|
||||||
fn: () => {
|
|
||||||
mode = 'failed';
|
|
||||||
resp.abort();
|
|
||||||
}
|
|
||||||
}, {
|
|
||||||
name: 'create',
|
|
||||||
description: 'No existing node fits — create a new one',
|
|
||||||
args: {name: {type: 'string', description: 'Canonical name for the new node', required: true}},
|
|
||||||
fn: ({name}) => {
|
|
||||||
home = name;
|
|
||||||
mode = 'create';
|
|
||||||
resp.abort();
|
|
||||||
}
|
|
||||||
}]
|
|
||||||
});
|
|
||||||
|
|
||||||
if(mode === 'create') {
|
|
||||||
return this.createNode(home, memories);
|
|
||||||
} else if (mode === 'failed') {
|
|
||||||
candidates = candidates.filter(c => c.name !== home);
|
|
||||||
if(!candidates.length) return this.createNode(bucket.subject, memories);
|
|
||||||
} else if (mode === 'success') {
|
|
||||||
const existing = memories.find(m => m.name === home);
|
|
||||||
return existing || this.createNode(home, memories);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return this.createNode(bucket.subject, memories);
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|||||||
17
src/tools.ts
17
src/tools.ts
@@ -100,16 +100,11 @@ export const CliTool: AiTool = {
|
|||||||
|
|
||||||
export const DateTimeTool: AiTool = {
|
export const DateTimeTool: AiTool = {
|
||||||
name: 'get_datetime',
|
name: 'get_datetime',
|
||||||
description: 'Get local date / time',
|
description: 'Get local/UTC date/time',
|
||||||
args: {},
|
args: {
|
||||||
fn: async () => new Date().toString()
|
timezone: {type: 'string', description: 'Which timezone to return, defaults to local', enum: ['local', 'utc'], default: 'local'}
|
||||||
}
|
},
|
||||||
|
fn: ({timezone}) => new Date()[timezone === 'local' ? 'toString' : 'toUTCString']()
|
||||||
export const DateTimeUTCTool: AiTool = {
|
|
||||||
name: 'get_datetime_utc',
|
|
||||||
description: 'Get current UTC date / time',
|
|
||||||
args: {},
|
|
||||||
fn: async () => new Date().toUTCString()
|
|
||||||
}
|
}
|
||||||
|
|
||||||
export const ExecTool: AiTool = {
|
export const ExecTool: AiTool = {
|
||||||
@@ -168,7 +163,7 @@ export const JSTool: AiTool = {
|
|||||||
}
|
}
|
||||||
|
|
||||||
export const PythonTool: AiTool = {
|
export const PythonTool: AiTool = {
|
||||||
name: 'exec_javascript',
|
name: 'exec_python',
|
||||||
description: 'Execute commonjs javascript',
|
description: 'Execute commonjs javascript',
|
||||||
args: {
|
args: {
|
||||||
code: {type: 'string', description: 'CommonJS javascript', required: true}
|
code: {type: 'string', description: 'CommonJS javascript', required: true}
|
||||||
|
|||||||
Reference in New Issue
Block a user