token pools

src/antrhopic.ts

@@ -1,16 +1,27 @@
 import {Anthropic as anthropic} from '@anthropic-ai/sdk';
-import {findByProp, objectMap, JSONSanitize, JSONAttemptParse} from '@ztimson/utils';
+import {findByProp, objectMap, JSONSanitize, JSONAttemptParse, makeArray} from '@ztimson/utils';
 import {AbortablePromise, Ai} from './ai.ts';
 import {LLMMessage, LLMRequest} from './llm.ts';
 import {LLMProvider} from './provider.ts';
+import {TokenPool} from './token-pool.ts';
 import {convertSchema} from './tools.ts';
 
 export class Anthropic extends LLMProvider {
-	client!: anthropic;
+	private clients = new Map<string, anthropic>();
+	tokenPool!: TokenPool;
 
-	constructor(public readonly ai: Ai, public readonly apiToken: string, public model: string) {
+	constructor(public readonly ai: Ai, public readonly apiToken: string | string[], public model: string) {
 		super();
-		this.client = new anthropic({apiKey: apiToken});
+		this.tokenPool = new TokenPool(...makeArray(apiToken).filter(Boolean));
+	}
+
+	private getClient(token: string): anthropic {
+		let client = this.clients.get(token);
+		if(!client) {
+			client = new anthropic({apiKey: token});
+			this.clients.set(token, client);
+		}
+		return client;
 	}
 
 	private toStandard(history: any[]): LLMMessage[] {
@@ -90,7 +101,7 @@ export class Anthropic extends LLMProvider {
 			do {
 				requestParams.messages = history.map(({timestamp, ...m}) => m);
 				const callStart = Date.now();
-				resp = await this.client.messages.create(requestParams).catch(err => {
+				resp = await this.tokenPool.run(token => this.getClient(token).messages.create(requestParams)).catch(err => {
 					err.message += `\n\nMessages:\n${JSON.stringify(history, null, 2)}`;
 					throw err;
 				});