diff --git a/package.json b/package.json index d837476..6c4456b 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@ztimson/ai-utils", - "version": "1.3.5", + "version": "1.3.6", "description": "AI Utility library", "author": "Zak Timson", "license": "MIT", diff --git a/src/llm.ts b/src/llm.ts index 1ddebc8..2a3ef9a 100644 --- a/src/llm.ts +++ b/src/llm.ts @@ -266,7 +266,10 @@ ${a.system}`, nestedAborts.forEach(a => a()); }; - const promise = (async () => { + let promise: any; + const requestStart = Date.now(); + + promise = (async () => { let tools: AiTool[] = options.tools || this.ai.options.llm?.tools || []; const prompts: string[] = []; let history = options.history || []; @@ -336,6 +339,8 @@ Also relevant but not preloaded (use \`memory_recall\`): ${listed.map(r => r.nam const toolTimings = new Map(); tools = this.wrapToolTiming(tools, toolTimings); + if(aborted) throw Object.assign(new Error('Aborted'), {name: 'AbortError'}); + prompts.unshift(options.system || this.ai.options.llm?.system || ''); request = this.models[m].ask(message, {...options, tools, system: prompts.filter(Boolean).join('\n\n')}); let resp = await request; @@ -374,6 +379,13 @@ Also relevant but not preloaded (use \`memory_recall\`): ${listed.map(r => r.nam if(options.history) options.history.splice(0, options.history.length, ...compressed); } + const requestDuration = Date.now() - requestStart; + const totalTokens = history + .filter((h: any) => h.role === 'assistant' && h.duration && h.tps) + .reduce((sum: number, h: any) => sum + h.tps * (h.duration / 1000), 0); + const requestTps = requestDuration > 0 ? totalTokens / (requestDuration / 1000) : 0; + Object.assign(promise, {duration: requestDuration, tps: requestTps}); + return resp; })();