|
|
|
|
@@ -1,11 +1,13 @@
|
|
|
|
|
import {MemoryNode, rebuildGraph} from './helpers.ts';
|
|
|
|
|
import {MemoryNode, patchGraph, rebuildGraph} from './helpers.ts';
|
|
|
|
|
import {LLMRequest, LLMMessage} from './llm.ts';
|
|
|
|
|
import {AiTool} from './tools.ts';
|
|
|
|
|
import {KDPoint, KDTree} from './kd-tree.ts';
|
|
|
|
|
import {KDTree} from './kd-tree.ts';
|
|
|
|
|
import {escapeRegex} from '@ztimson/utils';
|
|
|
|
|
|
|
|
|
|
const MERGE_THRESHOLD = 0.12;
|
|
|
|
|
const PENDING_HEADING = '## Pending';
|
|
|
|
|
const TREE_TOMBSTONE_LIMIT = 0.25;
|
|
|
|
|
const ALIAS_MATCH_THRESHOLD = 0.55;
|
|
|
|
|
const GENERIC_TEMPLATE = `# {{Title}}
|
|
|
|
|
|
|
|
|
|
## Summary
|
|
|
|
|
@@ -18,7 +20,12 @@ export type Memory = {
|
|
|
|
|
name: string;
|
|
|
|
|
description: string;
|
|
|
|
|
content: string;
|
|
|
|
|
/** Description embedding — indexed in the KD tree, used for merge/ANN candidate lookup */
|
|
|
|
|
embedding: number[];
|
|
|
|
|
/** Title-only embedding, weighted heaviest during recall ranking */
|
|
|
|
|
titleEmbedding?: number[];
|
|
|
|
|
/** Chunked body embeddings, best-chunk match used during recall ranking */
|
|
|
|
|
bodyEmbeddings?: number[][];
|
|
|
|
|
links: string[];
|
|
|
|
|
backlinks: string[];
|
|
|
|
|
}
|
|
|
|
|
@@ -26,6 +33,8 @@ export type Memory = {
|
|
|
|
|
type MemoryRef = {
|
|
|
|
|
name: string;
|
|
|
|
|
description: string;
|
|
|
|
|
/** Cosine distance from the query, present when returned from a search */
|
|
|
|
|
distance?: number;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
type FactBucket = {
|
|
|
|
|
@@ -61,10 +70,20 @@ function cosineDistance(a: number[], b: number[]): number {
|
|
|
|
|
function cosineSearch(query: number[], memories: Memory[], limit: number): MemoryRef[] {
|
|
|
|
|
return memories
|
|
|
|
|
.filter(m => m.embedding?.length)
|
|
|
|
|
.map(m => ({ref: {name: m.name, description: m.description}, distance: cosineDistance(query, m.embedding)}))
|
|
|
|
|
.map(m => ({name: m.name, description: m.description, distance: cosineDistance(query, m.embedding)}))
|
|
|
|
|
.sort((a, b) => a.distance - b.distance)
|
|
|
|
|
.slice(0, limit)
|
|
|
|
|
.map(s => s.ref);
|
|
|
|
|
.slice(0, limit);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/** Re-embed a node's title / description / body fields. Description embedding stays the KD-tree index key. */
|
|
|
|
|
async function embedMemoryFields(node: Memory, llm: any): Promise<void> {
|
|
|
|
|
const body = stripHeader(node.content);
|
|
|
|
|
const [titleE] = await llm.embedding(node.name.split('/').pop() || node.name);
|
|
|
|
|
const [descE] = await llm.embedding(node.description || '');
|
|
|
|
|
const bodyChunks = body ? await llm.embedding(body) : [];
|
|
|
|
|
if (titleE) node.titleEmbedding = titleE.embedding;
|
|
|
|
|
if (descE) node.embedding = descE.embedding;
|
|
|
|
|
node.bodyEmbeddings = bodyChunks.map((c: any) => c.embedding).filter(Boolean);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
export function stripHeader(content: string): string {
|
|
|
|
|
@@ -73,6 +92,8 @@ export function stripHeader(content: string): string {
|
|
|
|
|
|
|
|
|
|
export class MemoryCache {
|
|
|
|
|
private tree!: KDTree<MemoryRef>;
|
|
|
|
|
/** Tracks which memories are currently indexed in the tree, keyed by name -> embedding reference */
|
|
|
|
|
private indexed = new Map<string, number[]>();
|
|
|
|
|
public memories: Memory[];
|
|
|
|
|
public nodes: MemoryNode[] = [];
|
|
|
|
|
|
|
|
|
|
@@ -80,37 +101,48 @@ export class MemoryCache {
|
|
|
|
|
|
|
|
|
|
constructor(memories: Memory[]) {
|
|
|
|
|
this.memories = memories;
|
|
|
|
|
this.tree = new KDTree<MemoryRef>(0);
|
|
|
|
|
this.rebuild();
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
private buildTree(): KDTree<MemoryRef> {
|
|
|
|
|
const embedded = this.memories.filter(m => m.embedding?.length);
|
|
|
|
|
if (!embedded.length) return new KDTree<MemoryRef>(0);
|
|
|
|
|
/** Incrementally sync the KD tree against `this.memories` instead of rebuilding from scratch */
|
|
|
|
|
private syncTree(): void {
|
|
|
|
|
const current = new Set(this.memories.map(m => m.name));
|
|
|
|
|
|
|
|
|
|
const dims = embedded[0].embedding.length;
|
|
|
|
|
const points: KDPoint<MemoryRef>[] = embedded.map(m => ({
|
|
|
|
|
vector: m.embedding,
|
|
|
|
|
payload: {name: m.name, description: m.description},
|
|
|
|
|
}));
|
|
|
|
|
for (const [name, emb] of [...this.indexed]) {
|
|
|
|
|
const mem = this.memories.find(m => m.name === name);
|
|
|
|
|
if (!mem || !current.has(name) || mem.embedding !== emb) {
|
|
|
|
|
this.tree.remove(p => p.name === name);
|
|
|
|
|
this.indexed.delete(name);
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
return new KDTree<MemoryRef>(dims, 'cosine', points);
|
|
|
|
|
for (const mem of this.memories) {
|
|
|
|
|
if (!mem.embedding?.length || this.indexed.has(mem.name)) continue;
|
|
|
|
|
if (this.tree.dims === 0) this.tree = new KDTree<MemoryRef>(mem.embedding.length, 'cosine');
|
|
|
|
|
if (mem.embedding.length !== this.tree.dims) continue; // guard against embedding model/dim drift
|
|
|
|
|
this.tree.insert({vector: mem.embedding, payload: {name: mem.name, description: mem.description}});
|
|
|
|
|
this.indexed.set(mem.name, mem.embedding);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
if (this.tree.tombstoneRatio > TREE_TOMBSTONE_LIMIT) this.tree.rebalance();
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
search(query: number[], limit: number): MemoryRef[] {
|
|
|
|
|
if (!this.tree || this.tree.dims === 0) return [];
|
|
|
|
|
return this.tree.knn(query, limit).map(r => r.point.payload);
|
|
|
|
|
return this.tree.knn(query, limit).map(r => ({...r.point.payload, distance: r.distance}));
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
add(memory: Memory): void {
|
|
|
|
|
this.memories.push(memory);
|
|
|
|
|
this.rebuild();
|
|
|
|
|
this.rebuild([memory]);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
update(memory: Memory): void {
|
|
|
|
|
const existing = this.memories.find(m => m.name === memory.name);
|
|
|
|
|
if (existing) Object.assign(existing, memory);
|
|
|
|
|
else this.memories.push(memory);
|
|
|
|
|
this.rebuild();
|
|
|
|
|
this.rebuild([existing ?? memory]);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
remove(name: string): void {
|
|
|
|
|
@@ -121,9 +153,11 @@ export class MemoryCache {
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
rebuild(): void {
|
|
|
|
|
this.nodes = rebuildGraph(this.memories);
|
|
|
|
|
this.tree = this.buildTree();
|
|
|
|
|
rebuild(changed?: Memory[]): void {
|
|
|
|
|
this.nodes = (changed?.length && this.nodes.length)
|
|
|
|
|
? patchGraph(this.memories, this.nodes, changed)
|
|
|
|
|
: rebuildGraph(this.memories);
|
|
|
|
|
this.syncTree();
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
@@ -140,9 +174,9 @@ class MemoryAccessor {
|
|
|
|
|
return this.list.find(m => m.name === name);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
commit(): MemoryNode[] {
|
|
|
|
|
commit(changed?: Memory[]): MemoryNode[] {
|
|
|
|
|
if (this.cache) {
|
|
|
|
|
this.cache.rebuild();
|
|
|
|
|
this.cache.rebuild(changed);
|
|
|
|
|
return this.cache.nodes;
|
|
|
|
|
}
|
|
|
|
|
return rebuildGraph(this.list);
|
|
|
|
|
@@ -153,6 +187,7 @@ class MemoryAccessor {
|
|
|
|
|
return nodes.filter(n => n.missing).map(n => n.name);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/** Cache path uses the KD tree's knn(); raw-array path (no cache available) falls back to a linear cosine scan */
|
|
|
|
|
search(vector: number[], limit: number): MemoryRef[] {
|
|
|
|
|
return this.cache ? this.cache.search(vector, limit) : cosineSearch(vector, this.list, limit);
|
|
|
|
|
}
|
|
|
|
|
@@ -168,10 +203,7 @@ class MemoryAccessor {
|
|
|
|
|
async backfillEmbeddings(llm: any): Promise<number> {
|
|
|
|
|
const missing = this.list.filter(m => !m.embedding?.length);
|
|
|
|
|
if (!missing.length) return 0;
|
|
|
|
|
await Promise.all(missing.map(async node => {
|
|
|
|
|
const [e] = await llm.embedding(`${node.description}\n\n${stripHeader(node.content)}`.trim());
|
|
|
|
|
if (e) node.embedding = e.embedding;
|
|
|
|
|
}));
|
|
|
|
|
await Promise.all(missing.map(node => embedMemoryFields(node, llm)));
|
|
|
|
|
this.commit();
|
|
|
|
|
return missing.length;
|
|
|
|
|
}
|
|
|
|
|
@@ -282,6 +314,39 @@ ${m.content}
|
|
|
|
|
for (const m of memories) if (pattern.test(m.content)) m.content = m.content.replace(pattern, `[[${to}]]`);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
private normalizeLeaf(name: string): string {
|
|
|
|
|
return name.trim().toLowerCase().replace(/\s+/g, ' ');
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/**
|
|
|
|
|
* Resolve a fact-agent proposed subject to an existing node when it's an alias/rename of one.
|
|
|
|
|
* Exact match is checked first (cheap, and covers the common case since node names are
|
|
|
|
|
* already normalized at creation time). Only falls through to fuzzy alias matching against
|
|
|
|
|
* same-root candidates when there's no existing hit — i.e. only on likely-new-doc creation.
|
|
|
|
|
*/
|
|
|
|
|
private resolveSubject(subject: string, store: MemoryAccessor): string {
|
|
|
|
|
const trimmed = subject.trim();
|
|
|
|
|
const exact = store.find(trimmed);
|
|
|
|
|
if (exact) return exact.name;
|
|
|
|
|
|
|
|
|
|
const normalized = this.normalizeLeaf(trimmed);
|
|
|
|
|
const caseInsensitive = store.list.find(m => this.normalizeLeaf(m.name) === normalized);
|
|
|
|
|
if (caseInsensitive) return caseInsensitive.name;
|
|
|
|
|
|
|
|
|
|
const root = trimmed.split('/')[0];
|
|
|
|
|
const leaf = trimmed.split('/').slice(1).join('/') || trimmed;
|
|
|
|
|
const candidates = store.list.filter(m => m.name.split('/')[0] === root && m.name !== trimmed);
|
|
|
|
|
if (!candidates.length) return trimmed;
|
|
|
|
|
|
|
|
|
|
// fuzzyMatch requires >=2 terms; pad with an empty string when there's only one candidate
|
|
|
|
|
const leaves = candidates.map(m => m.name.split('/').slice(1).join('/') || m.name);
|
|
|
|
|
const probe = leaves.length > 1 ? leaves : [...leaves, ''];
|
|
|
|
|
const {max, similarities} = this.llm.fuzzyMatch(leaf, ...probe);
|
|
|
|
|
if (max >= ALIAS_MATCH_THRESHOLD) return candidates[similarities.indexOf(max)].name;
|
|
|
|
|
|
|
|
|
|
return trimmed;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
private async factAgent(conversation: string, store: MemoryAccessor, options: LLMRequest): Promise<FactAgentResult> {
|
|
|
|
|
const ghosts = store.ghosts();
|
|
|
|
|
|
|
|
|
|
@@ -300,15 +365,23 @@ ${m.content}
|
|
|
|
|
- NEVER extract greetings, pleasantries, or anything the assistant itself said
|
|
|
|
|
- Extract the final/end state, not deltas
|
|
|
|
|
|
|
|
|
|
Path assignment rules:
|
|
|
|
|
- Reuse existing node names whenever possible
|
|
|
|
|
- Documents should be grouped and named by the root subject
|
|
|
|
|
- Person → People/Name
|
|
|
|
|
- Project → Projects/Name
|
|
|
|
|
- Concept → Concepts/Name
|
|
|
|
|
- A bug report, its investigation, should be nested and attached to the same root subject node
|
|
|
|
|
- Tickets/one-off tasks → file under the project/name/component they belong to
|
|
|
|
|
- Only create a new top-level node when the fact belongs to a genuinely new subject (person/project/concept)\`
|
|
|
|
|
Path assignment (entity) rules:
|
|
|
|
|
- Use the owning entity of the fact (even if implied): "New bug on project 51 -> Projects/51"
|
|
|
|
|
- When multiple facts relate to the same entity, pick a primary owner and wikilink related entities
|
|
|
|
|
- Reuse existing entities when the owner already has a node
|
|
|
|
|
- Always group under consistent entity roots (always plural):
|
|
|
|
|
- Projects/[Name] for all initiatives
|
|
|
|
|
- People/[Name] for all individuals
|
|
|
|
|
- History/[Name] for all historical figures/events
|
|
|
|
|
- Science/[Name] for all scientific concepts
|
|
|
|
|
- Child entities nest under their parent entity:
|
|
|
|
|
- Projects/51/Memory System, Projects/51/Bug-XYZ, not Bugs/51
|
|
|
|
|
- Science/AI/Model-X, not Model-X/AI
|
|
|
|
|
|
|
|
|
|
Wikilink rules:
|
|
|
|
|
- Use [[WikiLinks]] to connect related entities (e.g., [[Projects/51]], [[People/Robert]])
|
|
|
|
|
- Only link specific, existing or implied entity paths — skip generic terms
|
|
|
|
|
- Don't over-link: each link should add clarity or context, not noise
|
|
|
|
|
|
|
|
|
|
Available nodes:
|
|
|
|
|
${this.listNodes(store.list).map(n => `- ${n.name}: ${n.description}`).join('\n') || 'None yet.'}
|
|
|
|
|
@@ -351,23 +424,21 @@ ${ghosts.length ? `${ghosts.map(g => `- ${g}: (Ghost)`).join('\n')}` : ''}`,
|
|
|
|
|
return memories.map(m => ({name: m.name, description: m.description}));
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/** Finds the closest merge candidate via the KD tree's knn() instead of a manual O(n) cosine scan */
|
|
|
|
|
private async checkMerge(node: Memory, memories: Memory[] | MemoryCache, options: LLMRequest, threshold = MERGE_THRESHOLD): Promise<Memory | null> {
|
|
|
|
|
if (!node.embedding?.length || node.name.startsWith('Journal/')) return null;
|
|
|
|
|
const store = this.access(memories);
|
|
|
|
|
|
|
|
|
|
let closest: Memory | null = null, closestDist = Infinity;
|
|
|
|
|
for (const other of store.list) {
|
|
|
|
|
if (other.name === node.name || other.name.startsWith('Journal/') || !other.embedding?.length) continue;
|
|
|
|
|
const d = cosineDistance(node.embedding, other.embedding);
|
|
|
|
|
if (d < closestDist) { closestDist = d; closest = other; }
|
|
|
|
|
}
|
|
|
|
|
if (!closest || closestDist > threshold) return null;
|
|
|
|
|
const candidate = store.search(node.embedding, 5)
|
|
|
|
|
.find(r => r.name !== node.name && !r.name.startsWith('Journal/') && r.distance !== undefined && r.distance <= threshold);
|
|
|
|
|
if (!candidate) return null;
|
|
|
|
|
const closest = store.find(candidate.name);
|
|
|
|
|
if (!closest) return null;
|
|
|
|
|
|
|
|
|
|
const result = await this.mergeAgent(node, closest, options);
|
|
|
|
|
const merged: Memory = {name: result.name, description: this.sanitizeDescription(result.description), content: '', embedding: [], links: [], backlinks: []};
|
|
|
|
|
merged.content = this.touchHeader(merged, result.content);
|
|
|
|
|
const [e] = await this.llm.embedding(`${merged.description}\n\n${result.content}`.trim());
|
|
|
|
|
if (e) merged.embedding = e.embedding;
|
|
|
|
|
await embedMemoryFields(merged, this.llm);
|
|
|
|
|
|
|
|
|
|
this.relink(store.list, node.name, merged.name);
|
|
|
|
|
this.relink(store.list, closest.name, merged.name);
|
|
|
|
|
@@ -396,18 +467,20 @@ ${ghosts.length ? `${ghosts.map(g => `- ${g}: (Ghost)`).join('\n')}` : ''}`,
|
|
|
|
|
this.queues.set(key, entry);
|
|
|
|
|
const store = this.access(memories);
|
|
|
|
|
entry.task = (async () => {
|
|
|
|
|
let current = node;
|
|
|
|
|
do {
|
|
|
|
|
entry.dirty = false;
|
|
|
|
|
await this.docAgent(current, store.list, options, entry);
|
|
|
|
|
this.mergeLock = this.mergeLock.then(() => this.checkMerge(current, memories, options));
|
|
|
|
|
const merged = await this.mergeLock;
|
|
|
|
|
if(merged) current = merged;
|
|
|
|
|
} while (entry.dirty);
|
|
|
|
|
})().finally(() => {
|
|
|
|
|
this.queues.delete(key);
|
|
|
|
|
store.commit();
|
|
|
|
|
});
|
|
|
|
|
let current = node, merged = false;
|
|
|
|
|
try {
|
|
|
|
|
do {
|
|
|
|
|
entry.dirty = false;
|
|
|
|
|
await this.docAgent(current, store.list, options, entry);
|
|
|
|
|
this.mergeLock = this.mergeLock.then(() => this.checkMerge(current, memories, options));
|
|
|
|
|
const result = await this.mergeLock;
|
|
|
|
|
if (result) { current = result; merged = true; }
|
|
|
|
|
} while (entry.dirty);
|
|
|
|
|
} finally {
|
|
|
|
|
store.commit(merged ? undefined : [node]);
|
|
|
|
|
this.queues.delete(key);
|
|
|
|
|
}
|
|
|
|
|
})();
|
|
|
|
|
return entry.task;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
@@ -434,7 +507,7 @@ ${GENERIC_TEMPLATE}
|
|
|
|
|
\`\`\`
|
|
|
|
|
|
|
|
|
|
Rules:
|
|
|
|
|
- Contradictions: newer facts always win — delete outdated statements entirely
|
|
|
|
|
- Contradictions: "## Pending" holds the newest information — bias toward it. Fold it in as the standing fact and drop the outdated statement, unless the old context adds meaningful nuance (e.g. "previously X, now Y"). This document should read as a source of truth, not an audit log
|
|
|
|
|
- Journals (Journal/...): keep entries as a chronological timeline; clean up grammar within entries but never delete history
|
|
|
|
|
- Use Obsidian markdown: # headings, **bold**, bullet/numbered lists, tables for 2D data
|
|
|
|
|
- Link specific entities and concepts with [[WikiLink]] (e.g., [[Projects/KiwixServer]]); skip generics
|
|
|
|
|
@@ -462,11 +535,12 @@ ${currentBody}
|
|
|
|
|
if (!update?.content) return;
|
|
|
|
|
node.description = node.name !== 'People/User' ? this.sanitizeDescription(update.description) : 'All information about the current user';
|
|
|
|
|
node.content = this.touchHeader(node, update.content);
|
|
|
|
|
const [e] = await this.llm.embedding(`${node.description}\n\n${update.content}`.trim());
|
|
|
|
|
if (e) node.embedding = e.embedding;
|
|
|
|
|
await embedMemoryFields(node, this.llm);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
private async mergeAgent(a: Memory, b: Memory, options: LLMRequest): Promise<{name: string, description: string, content: string}> {
|
|
|
|
|
const modifiedOf = (m: Memory) => this.parseFrontmatter(m.content).fm.get('modified') || 'unknown';
|
|
|
|
|
|
|
|
|
|
return this.llm.ask('', {
|
|
|
|
|
model: options.model,
|
|
|
|
|
temperature: 0.3,
|
|
|
|
|
@@ -475,21 +549,21 @@ ${currentBody}
|
|
|
|
|
description: {type: 'string', description: 'One factual sentence describing the merged document\'s subject matter', required: true},
|
|
|
|
|
content: {type: 'string', description: 'Fully reconciled body in markdown, without frontmatter', required: true},
|
|
|
|
|
},
|
|
|
|
|
system: `You are a knowledge base editor merging two overlapping Obsidian documents into one. Newer facts win on contradiction.
|
|
|
|
|
system: `You are a knowledge base editor merging two overlapping Obsidian documents into one.
|
|
|
|
|
|
|
|
|
|
Structure loosely:
|
|
|
|
|
\`\`\`markdown
|
|
|
|
|
${GENERIC_TEMPLATE}
|
|
|
|
|
\`\`\`
|
|
|
|
|
|
|
|
|
|
Combine both documents, resolve duplication and contradictions.
|
|
|
|
|
Combine both documents, resolve duplication. On contradictions, bias toward whichever document was modified more recently; drop the outdated statement unless the old context adds meaningful nuance.
|
|
|
|
|
|
|
|
|
|
Document A ("${a.name}"):
|
|
|
|
|
Document A ("${a.name}", last modified ${modifiedOf(a)}):
|
|
|
|
|
\`\`\`markdown
|
|
|
|
|
${stripHeader(a.content)}
|
|
|
|
|
\`\`\`
|
|
|
|
|
|
|
|
|
|
Document B ("${b.name}"):
|
|
|
|
|
Document B ("${b.name}", last modified ${modifiedOf(b)}):
|
|
|
|
|
\`\`\`markdown
|
|
|
|
|
${stripHeader(b.content)}
|
|
|
|
|
\`\`\``,
|
|
|
|
|
@@ -512,12 +586,17 @@ ${stripHeader(b.content)}
|
|
|
|
|
return {fm, body: match[2]};
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/**
|
|
|
|
|
* Writes the code-owned frontmatter block. `body` is passed through stripHeader() first so a
|
|
|
|
|
* model that ignores instructions and hallucinates its own `---` block can never corrupt or
|
|
|
|
|
* duplicate the real frontmatter — the LLM only ever gets to influence the body.
|
|
|
|
|
*/
|
|
|
|
|
private touchHeader(node: Memory, body: string): string {
|
|
|
|
|
const {fm} = this.parseFrontmatter(node.content);
|
|
|
|
|
fm.set('name', node.name);
|
|
|
|
|
fm.set('description', node.description || '');
|
|
|
|
|
fm.set('modified', new Date().toISOString());
|
|
|
|
|
return this.writeFrontmatter(fm, body);
|
|
|
|
|
return this.writeFrontmatter(fm, stripHeader(body));
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
private writeFrontmatter(fm: Map<string, string>, body: string): string {
|
|
|
|
|
@@ -540,6 +619,19 @@ ${stripHeader(b.content)}
|
|
|
|
|
return this.access(memories).forget(name);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/** Ranks a candidate pool by weighted title/description/body similarity against the query embedding */
|
|
|
|
|
private rankByFields(query: number[], candidates: Memory[], limit: number): Memory[] {
|
|
|
|
|
const scored = candidates.map(m => {
|
|
|
|
|
const titleSim = m.titleEmbedding?.length ? 1 - cosineDistance(query, m.titleEmbedding) : 0;
|
|
|
|
|
const descSim = m.embedding?.length ? 1 - cosineDistance(query, m.embedding) : 0;
|
|
|
|
|
const bodySim = m.bodyEmbeddings?.length
|
|
|
|
|
? Math.max(...m.bodyEmbeddings.map(b => 1 - cosineDistance(query, b)))
|
|
|
|
|
: 0;
|
|
|
|
|
return {memory: m, score: titleSim * 0.5 + descSim * 0.35 + bodySim * 0.15};
|
|
|
|
|
});
|
|
|
|
|
return scored.sort((a, b) => b.score - a.score).slice(0, limit).map(s => s.memory);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
async recollect(query: string, memories: Memory[] | MemoryCache, limit = 5, graphDepth = 1): Promise<Memory[]> {
|
|
|
|
|
const store = this.access(memories);
|
|
|
|
|
if (!store.list.length) return [];
|
|
|
|
|
@@ -549,8 +641,11 @@ ${stripHeader(b.content)}
|
|
|
|
|
const [e] = await this.llm.embedding(query);
|
|
|
|
|
if (!e) return [];
|
|
|
|
|
|
|
|
|
|
const vectorResults = store.search(e.embedding, limit);
|
|
|
|
|
const found = new Set<string>(vectorResults.map(r => r.name));
|
|
|
|
|
// Description embedding is the cheap ANN index key; pull a wider pool then re-rank by field weight
|
|
|
|
|
const pool = store.search(e.embedding, Math.max(limit * 3, limit));
|
|
|
|
|
const poolMemories = pool.map(r => store.find(r.name)).filter((m): m is Memory => !!m);
|
|
|
|
|
const ranked = this.rankByFields(e.embedding, poolMemories, limit);
|
|
|
|
|
const found = new Set<string>(ranked.map(m => m.name));
|
|
|
|
|
|
|
|
|
|
if (graphDepth > 0) {
|
|
|
|
|
let frontier = [...found];
|
|
|
|
|
@@ -570,9 +665,9 @@ ${stripHeader(b.content)}
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
const vectorOrder = vectorResults.map(r => r.name);
|
|
|
|
|
const graphExpansions = [...found].filter(n => !vectorOrder.includes(n));
|
|
|
|
|
return [...vectorOrder, ...graphExpansions].map(n => store.find(n)!).filter(Boolean);
|
|
|
|
|
const rankedOrder = ranked.map(m => m.name);
|
|
|
|
|
const graphExpansions = [...found].filter(n => !rankedOrder.includes(n));
|
|
|
|
|
return [...rankedOrder, ...graphExpansions].map(n => store.find(n)!).filter(Boolean);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
async memorize(history: LLMMessage[], memories: Memory[] | MemoryCache, options: LLMRequest): Promise<Memory[]> {
|
|
|
|
|
@@ -601,9 +696,10 @@ ${stripHeader(b.content)}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
for (const {subject, facts} of buckets) {
|
|
|
|
|
let node = store.find(subject);
|
|
|
|
|
const resolved = this.resolveSubject(subject, store);
|
|
|
|
|
let node = store.find(resolved);
|
|
|
|
|
if (!node) {
|
|
|
|
|
node = {name: subject, description: '', content: '', embedding: [], links: [], backlinks: []};
|
|
|
|
|
node = {name: resolved, description: '', content: '', embedding: [], links: [], backlinks: []};
|
|
|
|
|
store.list.push(node);
|
|
|
|
|
}
|
|
|
|
|
this.stage(node, facts.map(f => `- ${f}`).join('\n'));
|
|
|
|
|
@@ -611,13 +707,12 @@ ${stripHeader(b.content)}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
await Promise.all(touched.map(async node => {
|
|
|
|
|
const [e] = await this.llm.embedding(`${node.description}\n\n${stripHeader(node.content)}`.trim());
|
|
|
|
|
if (e) node.embedding = e.embedding;
|
|
|
|
|
await embedMemoryFields(node, this.llm);
|
|
|
|
|
this.touch(node.name);
|
|
|
|
|
}));
|
|
|
|
|
|
|
|
|
|
if (touched.length) {
|
|
|
|
|
store.commit();
|
|
|
|
|
store.commit(touched);
|
|
|
|
|
(pending as any).content = `Saved to ${touched.map(n => `[[${n.name}]]`).join(', ')}`;
|
|
|
|
|
Promise.all(touched.map(node => this.reconcile(node, memories, options).catch(() => {})));
|
|
|
|
|
} else {
|
|
|
|
|
|
