Compare commits
1 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| be08db8e2c |
@@ -1,6 +1,6 @@
|
|||||||
{
|
{
|
||||||
"name": "@ztimson/ai-utils",
|
"name": "@ztimson/ai-utils",
|
||||||
"version": "1.3.5",
|
"version": "1.3.6",
|
||||||
"description": "AI Utility library",
|
"description": "AI Utility library",
|
||||||
"author": "Zak Timson",
|
"author": "Zak Timson",
|
||||||
"license": "MIT",
|
"license": "MIT",
|
||||||
|
|||||||
14
src/llm.ts
14
src/llm.ts
@@ -266,7 +266,10 @@ ${a.system}`,
|
|||||||
nestedAborts.forEach(a => a());
|
nestedAborts.forEach(a => a());
|
||||||
};
|
};
|
||||||
|
|
||||||
const promise = (async () => {
|
let promise: any;
|
||||||
|
const requestStart = Date.now();
|
||||||
|
|
||||||
|
promise = (async () => {
|
||||||
let tools: AiTool[] = options.tools || this.ai.options.llm?.tools || [];
|
let tools: AiTool[] = options.tools || this.ai.options.llm?.tools || [];
|
||||||
const prompts: string[] = [];
|
const prompts: string[] = [];
|
||||||
let history = options.history || [];
|
let history = options.history || [];
|
||||||
@@ -336,6 +339,8 @@ Also relevant but not preloaded (use \`memory_recall\`): ${listed.map(r => r.nam
|
|||||||
const toolTimings = new Map<string, {duration: number, tps: number}>();
|
const toolTimings = new Map<string, {duration: number, tps: number}>();
|
||||||
tools = this.wrapToolTiming(tools, toolTimings);
|
tools = this.wrapToolTiming(tools, toolTimings);
|
||||||
|
|
||||||
|
if(aborted) throw Object.assign(new Error('Aborted'), {name: 'AbortError'});
|
||||||
|
|
||||||
prompts.unshift(options.system || this.ai.options.llm?.system || '');
|
prompts.unshift(options.system || this.ai.options.llm?.system || '');
|
||||||
request = this.models[m].ask(message, {...options, tools, system: prompts.filter(Boolean).join('\n\n')});
|
request = this.models[m].ask(message, {...options, tools, system: prompts.filter(Boolean).join('\n\n')});
|
||||||
let resp = await request;
|
let resp = await request;
|
||||||
@@ -374,6 +379,13 @@ Also relevant but not preloaded (use \`memory_recall\`): ${listed.map(r => r.nam
|
|||||||
if(options.history) options.history.splice(0, options.history.length, ...compressed);
|
if(options.history) options.history.splice(0, options.history.length, ...compressed);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
const requestDuration = Date.now() - requestStart;
|
||||||
|
const totalTokens = history
|
||||||
|
.filter((h: any) => h.role === 'assistant' && h.duration && h.tps)
|
||||||
|
.reduce((sum: number, h: any) => sum + h.tps * (h.duration / 1000), 0);
|
||||||
|
const requestTps = requestDuration > 0 ? totalTokens / (requestDuration / 1000) : 0;
|
||||||
|
Object.assign(promise, {duration: requestDuration, tps: requestTps});
|
||||||
|
|
||||||
return resp;
|
return resp;
|
||||||
})();
|
})();
|
||||||
|
|
||||||
|
|||||||
Reference in New Issue
Block a user