ztimson/ai-utils
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Pulled chunking out into its own exported function for easy access
All checks were successful
Publish Library / Build NPM Project (push) Successful in 41s
Details
Publish Library / Tag Version (push) Successful in 7s
Details

Browse Source
This commit is contained in:
Zakary Timson
2026-01-30 10:38:51 -05:00
parent cb60a0b0c5
commit d5bf1ec47e
2 changed files with 25 additions and 32 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
package.json
View File

@@ -1,6 +1,6 @@
{ {
"name": "@ztimson/ai-utils", "name": "@ztimson/ai-utils",
"version": "0.2.6", "version": "0.2.7",
"description": "AI Utility library", "description": "AI Utility library",
"author": "Zak Timson", "author": "Zak Timson",
"license": "MIT", "license": "MIT",

55
src/llm.ts
View File

@@ -160,17 +160,35 @@ export class LLM {
return denominator === 0 ? 0 : dotProduct / denominator; return denominator === 0 ? 0 : dotProduct / denominator;
} }
embedding(target: object | string, maxTokens = 500, overlapTokens = 50) { chunk(target: object | string, maxTokens = 500, overlapTokens = 50): string[] {
const objString = (obj: any, path = ''): string[] => { const objString = (obj: any, path = ''): string[] => {
if(obj === null || obj === undefined) return []; if(!obj) return [];
return Object.entries(obj).flatMap(([key, value]) => { return Object.entries(obj).flatMap(([key, value]) => {
const p = path ? `${path}${isNaN(+key) ? `.${key}` : `[${key}]`}` : key; const p = path ? `${path}${isNaN(+key) ? `.${key}` : `[${key}]`}` : key;
if(typeof value === 'object' && value !== null && !Array.isArray(value)) return objString(value, p); if(typeof value === 'object' && !Array.isArray(value)) return objString(value, p);
const valueStr = Array.isArray(value) ? value.join(', ') : String(value); return `${p}: ${Array.isArray(value) ? value.join(', ') : value}`;
return `${p}: ${valueStr}`;
}); });
}; };
const lines = typeof target === 'object' ? objString(target) : target.split('\n');
const tokens = lines.flatMap(l => [...l.split(/\s+/).filter(Boolean), '\n']);
const chunks: string[] = [];
for(let i = 0; i < tokens.length;) {
let text = '', j = i;
while(j < tokens.length) {
const next = text + (text ? ' ' : '') + tokens[j];
if(this.estimateTokens(next.replace(/\s*\n\s*/g, '\n')) > maxTokens && text) break;
text = next;
j++;
}
const clean = text.replace(/\s*\n\s*/g, '\n').trim();
if(clean) chunks.push(clean);
i = Math.max(j - overlapTokens, j === i ? i + 1 : j);
}
return chunks;
}
embedding(target: object | string, maxTokens = 500, overlapTokens = 50) {
const embed = (text: string): Promise<number[]> => { const embed = (text: string): Promise<number[]> => {
return new Promise((resolve, reject) => { return new Promise((resolve, reject) => {
const id = this.embedId++; const id = this.embedId++;
@@ -179,32 +197,7 @@ export class LLM {
}); });
}; };
// Tokenize const chunks = this.chunk(target, maxTokens, overlapTokens);
const lines = typeof target === 'object' ? objString(target) : target.split('\n');
const tokens = lines.flatMap(line => [...line.split(/\s+/).filter(w => w.trim()), '\n']);
// Chunk
const chunks: string[] = [];
let start = 0;
while (start < tokens.length) {
let end = start;
let text = '';
// Build chunk
while (end < tokens.length) {
const nextToken = tokens[end];
const testText = text + (text ? ' ' : '') + nextToken;
const testTokens = this.estimateTokens(testText.replace(/\s*\n\s*/g, '\n'));
if (testTokens > maxTokens && text) break;
text = testText;
end++;
}
// Save chunk
const cleanText = text.replace(/\s*\n\s*/g, '\n').trim();
if(cleanText) chunks.push(cleanText);
start = end - overlapTokens;
if(start <= end - tokens.length + end) start = end;
}
return Promise.all(chunks.map(async (text, index) => ({ return Promise.all(chunks.map(async (text, index) => ({
index, index,
embedding: await embed(text), embedding: await embed(text),