diff --git a/package-lock.json b/package-lock.json index 5da5c32..9714f00 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "@ztimson/ai-utils", - "version": "1.2.6", + "version": "1.5.0", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "@ztimson/ai-utils", - "version": "1.2.6", + "version": "1.5.0", "license": "MIT", "dependencies": { "@anthropic-ai/sdk": "^0.102.0", @@ -16,6 +16,7 @@ "@ztimson/utils": "^0.29.4", "cheerio": "^1.2.0", "openai": "^6.42.0", + "pdf-parse": "^2.4.5", "tesseract.js": "^7.0.0" }, "devDependencies": { @@ -56,39 +57,12 @@ "node": ">=6.9.0" } }, - "node_modules/@emnapi/core": { - "version": "2.0.0-alpha.3", - "resolved": "https://registry.npmjs.org/@emnapi/core/-/core-2.0.0-alpha.3.tgz", - "integrity": "sha512-AZypUeJ/yByuxyS7BlSNRDOMLMlROYtjYdIAuBmJssVz1UJDSeYxLrdizhXCFYhedC5bqd/ASy8EuNXbVVXp9g==", - "dev": true, - "license": "MIT", - "optional": true, - "peer": true, - "dependencies": { - "@emnapi/wasi-threads": "2.0.1", - "tslib": "^2.4.0" - } - }, "node_modules/@emnapi/runtime": { - "version": "2.0.0-alpha.3", - "resolved": "https://registry.npmjs.org/@emnapi/runtime/-/runtime-2.0.0-alpha.3.tgz", - "integrity": "sha512-hFPAhMUjJD9BSyCANEISPOogeXC9Zo9ZQl7L6vKnaVsMkCtzznaW/naYypeyl0Gv5rYfWYsZbpixTMpjDJzQeA==", - "dev": true, + "version": "1.11.3", + "resolved": "https://registry.npmjs.org/@emnapi/runtime/-/runtime-1.11.3.tgz", + "integrity": "sha512-Xz4Tpyki7XyrpbUK1jR1AhdAdaXyhhY4lZ3neLodmhpuWfy2PAQN5B46sAiU4liOXGLkHypn/qU+jvfWSCYYLA==", "license": "MIT", "optional": true, - "peer": true, - "dependencies": { - "tslib": "^2.4.0" - } - }, - "node_modules/@emnapi/wasi-threads": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/@emnapi/wasi-threads/-/wasi-threads-2.0.1.tgz", - "integrity": "sha512-9DsSk+o5NBX0CCJT8s0EROGSGxjR/tKu6aBTaVyq+SjAEQH4XcdcRxPBRzsBLizTTJ49MJjF+jgu3qnO9GLQcQ==", - "dev": true, - "license": "MIT", - "optional": true, - "peer": true, "dependencies": { "tslib": "^2.4.0" } @@ -577,16 +551,6 @@ "url": "https://opencollective.com/libvips" } }, - "node_modules/@img/sharp-wasm32/node_modules/@emnapi/runtime": { - "version": "1.11.3", - "resolved": "https://registry.npmjs.org/@emnapi/runtime/-/runtime-1.11.3.tgz", - "integrity": "sha512-Xz4Tpyki7XyrpbUK1jR1AhdAdaXyhhY4lZ3neLodmhpuWfy2PAQN5B46sAiU4liOXGLkHypn/qU+jvfWSCYYLA==", - "license": "MIT", - "optional": true, - "dependencies": { - "tslib": "^2.4.0" - } - }, "node_modules/@img/sharp-win32-arm64": { "version": "0.34.5", "resolved": "https://registry.npmjs.org/@img/sharp-win32-arm64/-/sharp-win32-arm64-0.34.5.tgz", @@ -694,32 +658,209 @@ "@jridgewell/sourcemap-codec": "^1.4.14" } }, - "node_modules/@napi-rs/wasm-runtime": { - "version": "1.2.0", - "resolved": "https://registry.npmjs.org/@napi-rs/wasm-runtime/-/wasm-runtime-1.2.0.tgz", - "integrity": "sha512-kDoONqMa+VnZ4vvvu/ZUurpJ4gkZU57e7g69qpNgWhYcZFPUHZM2CEMKm+cG6ufDVALbjMvfmMjFVqaK7uEMnA==", - "dev": true, + "node_modules/@napi-rs/canvas": { + "version": "0.1.80", + "resolved": "https://registry.npmjs.org/@napi-rs/canvas/-/canvas-0.1.80.tgz", + "integrity": "sha512-DxuT1ClnIPts1kQx8FBmkk4BQDTfI5kIzywAaMjQSXfNnra5UFU9PwurXrl+Je3bJ6BGsp/zmshVVFbCmyI+ww==", + "license": "MIT", + "workspaces": [ + "e2e/*" + ], + "engines": { + "node": ">= 10" + }, + "optionalDependencies": { + "@napi-rs/canvas-android-arm64": "0.1.80", + "@napi-rs/canvas-darwin-arm64": "0.1.80", + "@napi-rs/canvas-darwin-x64": "0.1.80", + "@napi-rs/canvas-linux-arm-gnueabihf": "0.1.80", + "@napi-rs/canvas-linux-arm64-gnu": "0.1.80", + "@napi-rs/canvas-linux-arm64-musl": "0.1.80", + "@napi-rs/canvas-linux-riscv64-gnu": "0.1.80", + "@napi-rs/canvas-linux-x64-gnu": "0.1.80", + "@napi-rs/canvas-linux-x64-musl": "0.1.80", + "@napi-rs/canvas-win32-x64-msvc": "0.1.80" + } + }, + "node_modules/@napi-rs/canvas-android-arm64": { + "version": "0.1.80", + "resolved": "https://registry.npmjs.org/@napi-rs/canvas-android-arm64/-/canvas-android-arm64-0.1.80.tgz", + "integrity": "sha512-sk7xhN/MoXeuExlggf91pNziBxLPVUqF2CAVnB57KLG/pz7+U5TKG8eXdc3pm0d7Od0WreB6ZKLj37sX9muGOQ==", + "cpu": [ + "arm64" + ], "license": "MIT", "optional": true, - "dependencies": { - "@tybys/wasm-util": "^0.10.3" - }, + "os": [ + "android" + ], "engines": { - "node": "^20.19.0 || ^22.13.0 || >=23.5.0" - }, - "funding": { - "type": "github", - "url": "https://github.com/sponsors/Brooooooklyn" - }, - "peerDependencies": { - "@emnapi/core": "^2.0.0-alpha.3", - "@emnapi/runtime": "^2.0.0-alpha.3" + "node": ">= 10" + } + }, + "node_modules/@napi-rs/canvas-darwin-arm64": { + "version": "0.1.80", + "resolved": "https://registry.npmjs.org/@napi-rs/canvas-darwin-arm64/-/canvas-darwin-arm64-0.1.80.tgz", + "integrity": "sha512-O64APRTXRUiAz0P8gErkfEr3lipLJgM6pjATwavZ22ebhjYl/SUbpgM0xcWPQBNMP1n29afAC/Us5PX1vg+JNQ==", + "cpu": [ + "arm64" + ], + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">= 10" + } + }, + "node_modules/@napi-rs/canvas-darwin-x64": { + "version": "0.1.80", + "resolved": "https://registry.npmjs.org/@napi-rs/canvas-darwin-x64/-/canvas-darwin-x64-0.1.80.tgz", + "integrity": "sha512-FqqSU7qFce0Cp3pwnTjVkKjjOtxMqRe6lmINxpIZYaZNnVI0H5FtsaraZJ36SiTHNjZlUB69/HhxNDT1Aaa9vA==", + "cpu": [ + "x64" + ], + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">= 10" + } + }, + "node_modules/@napi-rs/canvas-linux-arm-gnueabihf": { + "version": "0.1.80", + "resolved": "https://registry.npmjs.org/@napi-rs/canvas-linux-arm-gnueabihf/-/canvas-linux-arm-gnueabihf-0.1.80.tgz", + "integrity": "sha512-eyWz0ddBDQc7/JbAtY4OtZ5SpK8tR4JsCYEZjCE3dI8pqoWUC8oMwYSBGCYfsx2w47cQgQCgMVRVTFiiO38hHQ==", + "cpu": [ + "arm" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 10" + } + }, + "node_modules/@napi-rs/canvas-linux-arm64-gnu": { + "version": "0.1.80", + "resolved": "https://registry.npmjs.org/@napi-rs/canvas-linux-arm64-gnu/-/canvas-linux-arm64-gnu-0.1.80.tgz", + "integrity": "sha512-qwA63t8A86bnxhuA/GwOkK3jvb+XTQaTiVML0vAWoHyoZYTjNs7BzoOONDgTnNtr8/yHrq64XXzUoLqDzU+Uuw==", + "cpu": [ + "arm64" + ], + "libc": [ + "glibc" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 10" + } + }, + "node_modules/@napi-rs/canvas-linux-arm64-musl": { + "version": "0.1.80", + "resolved": "https://registry.npmjs.org/@napi-rs/canvas-linux-arm64-musl/-/canvas-linux-arm64-musl-0.1.80.tgz", + "integrity": "sha512-1XbCOz/ymhj24lFaIXtWnwv/6eFHXDrjP0jYkc6iHQ9q8oXKzUX1Lc6bu+wuGiLhGh2GS/2JlfORC5ZcXimRcg==", + "cpu": [ + "arm64" + ], + "libc": [ + "musl" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 10" + } + }, + "node_modules/@napi-rs/canvas-linux-riscv64-gnu": { + "version": "0.1.80", + "resolved": "https://registry.npmjs.org/@napi-rs/canvas-linux-riscv64-gnu/-/canvas-linux-riscv64-gnu-0.1.80.tgz", + "integrity": "sha512-XTzR125w5ZMs0lJcxRlS1K3P5RaZ9RmUsPtd1uGt+EfDyYMu4c6SEROYsxyatbbu/2+lPe7MPHOO/0a0x7L/gw==", + "cpu": [ + "riscv64" + ], + "libc": [ + "glibc" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 10" + } + }, + "node_modules/@napi-rs/canvas-linux-x64-gnu": { + "version": "0.1.80", + "resolved": "https://registry.npmjs.org/@napi-rs/canvas-linux-x64-gnu/-/canvas-linux-x64-gnu-0.1.80.tgz", + "integrity": "sha512-BeXAmhKg1kX3UCrJsYbdQd3hIMDH/K6HnP/pG2LuITaXhXBiNdh//TVVVVCBbJzVQaV5gK/4ZOCMrQW9mvuTqA==", + "cpu": [ + "x64" + ], + "libc": [ + "glibc" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 10" + } + }, + "node_modules/@napi-rs/canvas-linux-x64-musl": { + "version": "0.1.80", + "resolved": "https://registry.npmjs.org/@napi-rs/canvas-linux-x64-musl/-/canvas-linux-x64-musl-0.1.80.tgz", + "integrity": "sha512-x0XvZWdHbkgdgucJsRxprX/4o4sEed7qo9rCQA9ugiS9qE2QvP0RIiEugtZhfLH3cyI+jIRFJHV4Fuz+1BHHMg==", + "cpu": [ + "x64" + ], + "libc": [ + "musl" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 10" + } + }, + "node_modules/@napi-rs/canvas-win32-x64-msvc": { + "version": "0.1.80", + "resolved": "https://registry.npmjs.org/@napi-rs/canvas-win32-x64-msvc/-/canvas-win32-x64-msvc-0.1.80.tgz", + "integrity": "sha512-Z8jPsM6df5V8B1HrCHB05+bDiCxjE9QA//3YrkKIdVDEwn5RKaqOxCJDRJkl48cJbylcrJbW4HxZbTte8juuPg==", + "cpu": [ + "x64" + ], + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">= 10" } }, "node_modules/@oxc-project/types": { - "version": "0.139.0", - "resolved": "https://registry.npmjs.org/@oxc-project/types/-/types-0.139.0.tgz", - "integrity": "sha512-r9gHphtCs+1M7J0pw6Sn/hh/Wpa/iQrOOkrNAlVLF/gHq+/CJmHIWKKUUhdWjcD6CIa8idarspCsASiXCXvFUw==", + "version": "0.144.0", + "resolved": "https://registry.npmjs.org/@oxc-project/types/-/types-0.144.0.tgz", + "integrity": "sha512-nuhZIOLuI6TFQ32I/WnUx+SCPY7SdSKwgnFHydAuoS1+Z4BRcaP+RRJmGzl9lw+0OFF7UmaESf7KQRXaNLHypg==", "dev": true, "license": "MIT", "funding": { @@ -784,9 +925,9 @@ "license": "BSD-3-Clause" }, "node_modules/@rolldown/binding-android-arm64": { - "version": "1.1.5", - "resolved": "https://registry.npmjs.org/@rolldown/binding-android-arm64/-/binding-android-arm64-1.1.5.tgz", - "integrity": "sha512-lZg8fqIv2v7FF237bwMgzGZEJvGL79/s5knJ/i6FmsGF4XXlzccZ4jb+TrFIxtSSxFtIpdsgrPZeMk1I9AFcyQ==", + "version": "1.2.4", + "resolved": "https://registry.npmjs.org/@rolldown/binding-android-arm64/-/binding-android-arm64-1.2.4.tgz", + "integrity": "sha512-jHC2cnyKz5xU2fhECtFl8OZ83cYNt13GZQD+0uMJ/X3o+ijmd56okHhTUwxVSHPx1IRVIJEZ1/1pPzeLCU6XKA==", "cpu": [ "arm64" ], @@ -801,9 +942,9 @@ } }, "node_modules/@rolldown/binding-darwin-arm64": { - "version": "1.1.5", - "resolved": "https://registry.npmjs.org/@rolldown/binding-darwin-arm64/-/binding-darwin-arm64-1.1.5.tgz", - "integrity": "sha512-51Bnx9pNiMRKSUNtBfySkNJ9vMU9Hh3I1ozDd6gyPPYzaXCfnptUcEZxXGYFn+ul2dtcMUiqGR1Yai2K10uoTw==", + "version": "1.2.4", + "resolved": "https://registry.npmjs.org/@rolldown/binding-darwin-arm64/-/binding-darwin-arm64-1.2.4.tgz", + "integrity": "sha512-Dc5mPD8F5F/FS8i01syd7FTF6yB2fVthH/TRkjwJkzUK6EpoxHtqvZQP5Zwq80/5z19TWYHIg1KOHboCgVx/aQ==", "cpu": [ "arm64" ], @@ -818,9 +959,9 @@ } }, "node_modules/@rolldown/binding-darwin-x64": { - "version": "1.1.5", - "resolved": "https://registry.npmjs.org/@rolldown/binding-darwin-x64/-/binding-darwin-x64-1.1.5.tgz", - "integrity": "sha512-Tm+gbfC0aHu1tBA/JvKQh32S0K6YgCHkiAF4/W6xX0K0RmNuc94VeK419dJoE65R5aRxmo+noZQSWrAMF6yb6g==", + "version": "1.2.4", + "resolved": "https://registry.npmjs.org/@rolldown/binding-darwin-x64/-/binding-darwin-x64-1.2.4.tgz", + "integrity": "sha512-fpDm4oBo6SqLvWUYCmFhdde3U9KH2fRNNMeAnAPAIwxRL345xutL0EtEUcuoxsoazdJGv/MuDBQHlCDrtbvqOg==", "cpu": [ "x64" ], @@ -835,9 +976,9 @@ } }, "node_modules/@rolldown/binding-freebsd-x64": { - "version": "1.1.5", - "resolved": "https://registry.npmjs.org/@rolldown/binding-freebsd-x64/-/binding-freebsd-x64-1.1.5.tgz", - "integrity": "sha512-JMzDKCCXq93YccG5gz3hvOs1oXRKAf0XYpfOS88e+wZrC8Iugj6j68867vrYZkvpDDpKn/KoKORThmchMpF6TA==", + "version": "1.2.4", + "resolved": "https://registry.npmjs.org/@rolldown/binding-freebsd-x64/-/binding-freebsd-x64-1.2.4.tgz", + "integrity": "sha512-rSJoreDE/HoIzoaib6MTp5jQtCTdMHKIvItAKT/ImS6Y6Ww76oUaeMyp4Vc/fAgd/ehji068IxetHXAnqUwN9A==", "cpu": [ "x64" ], @@ -852,9 +993,9 @@ } }, "node_modules/@rolldown/binding-linux-arm-gnueabihf": { - "version": "1.1.5", - "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-arm-gnueabihf/-/binding-linux-arm-gnueabihf-1.1.5.tgz", - "integrity": "sha512-uML21j2K5TfPGutKxub+M+nLjZIrWjXQ5Grx4lCe/nimTj9B4L63zHpjXLl4y0L3mcm2htEQIb06oCG/szerNw==", + "version": "1.2.4", + "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-arm-gnueabihf/-/binding-linux-arm-gnueabihf-1.2.4.tgz", + "integrity": "sha512-/jm8OGHgn7oGaJu3i/qZI9spUGcJ+y/lk43ttQ/iO1tOd9NissG6o97bighBCiL+BKRngmcDuR6ikfwYdJmVuQ==", "cpu": [ "arm" ], @@ -869,9 +1010,9 @@ } }, "node_modules/@rolldown/binding-linux-arm64-gnu": { - "version": "1.1.5", - "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-arm64-gnu/-/binding-linux-arm64-gnu-1.1.5.tgz", - "integrity": "sha512-navSiuTMogvnQoZoM/v+l3ZWo50/NTwSHSzheABx/RCnmUPaKwq9qSo4Br2OYRs21+Fz8uFqITZM3H4opOB0/Q==", + "version": "1.2.4", + "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-arm64-gnu/-/binding-linux-arm64-gnu-1.2.4.tgz", + "integrity": "sha512-tIP06BeD9EqvECBrPZ+sqdPlYrT+aYaAiu1wYziVx5elRK/ftm33JxVDy2bXGbr6J0CrtirCkR87/X5a2euEng==", "cpu": [ "arm64" ], @@ -889,9 +1030,9 @@ } }, "node_modules/@rolldown/binding-linux-arm64-musl": { - "version": "1.1.5", - "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-arm64-musl/-/binding-linux-arm64-musl-1.1.5.tgz", - "integrity": "sha512-lAryqH7IteztmCXQXk0etKj4wBQ7Gx5S6LjKhsgp9zb8I5bsuvU/2llH1hDQcjsFeqIsovMVN339/8pUDDBXxA==", + "version": "1.2.4", + "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-arm64-musl/-/binding-linux-arm64-musl-1.2.4.tgz", + "integrity": "sha512-Ql1Q0EQqVThvn9VAVlwNzsUvbSFtCMGjLpRRi4pk5i7NZZ4n5ISiLMjHYtus4VQ2PvkSw24zyaCVsiS+sXPj1w==", "cpu": [ "arm64" ], @@ -909,9 +1050,9 @@ } }, "node_modules/@rolldown/binding-linux-ppc64-gnu": { - "version": "1.1.5", - "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-ppc64-gnu/-/binding-linux-ppc64-gnu-1.1.5.tgz", - "integrity": "sha512-fsK/sNBnxzBlL4O1JNrZakVQxPspqpED5dLtNsZS9oOKmtSpdNIzxH2kkol5HYTWJN47sE20ztMJPxfZ89qGOg==", + "version": "1.2.4", + "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-ppc64-gnu/-/binding-linux-ppc64-gnu-1.2.4.tgz", + "integrity": "sha512-GjbjXD4XXfN19D0LZNbmiCBUoDiRACsYHr0yaIbbn8aFsXjHZifcYqu/W5Er5X2X990WjHXFrxarn5chzItorQ==", "cpu": [ "ppc64" ], @@ -929,9 +1070,9 @@ } }, "node_modules/@rolldown/binding-linux-s390x-gnu": { - "version": "1.1.5", - "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-s390x-gnu/-/binding-linux-s390x-gnu-1.1.5.tgz", - "integrity": "sha512-gLYb4BIadlfTOYT5gO503n8zQjXflgzpD0FcyKh0Mzx3rqCZKnHoJWV9xe1KXUJ5lx2JfcSHr/mhzS0PC/McAA==", + "version": "1.2.4", + "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-s390x-gnu/-/binding-linux-s390x-gnu-1.2.4.tgz", + "integrity": "sha512-p5WR0NOwaRmJ/B1b6IjEFLLivwEsf3PrdBIhRbhTCQisbo2SvHHpG4ELB/+FgQNnB88LTOF86upmJmbvZdQ2lw==", "cpu": [ "s390x" ], @@ -949,9 +1090,9 @@ } }, "node_modules/@rolldown/binding-linux-x64-gnu": { - "version": "1.1.5", - "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-x64-gnu/-/binding-linux-x64-gnu-1.1.5.tgz", - "integrity": "sha512-FjcpEKUyJygHgs1o50VYNvkt5+7Le/VEdYt0AkRpkL33MnyQfwr8l5mXwMmfmTbyMPr5vJLC+8/Gd9gXnwU1QQ==", + "version": "1.2.4", + "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-x64-gnu/-/binding-linux-x64-gnu-1.2.4.tgz", + "integrity": "sha512-4/GyVjmhR+Tc6HLJvwc1sOhPqAZtySiSMesOZyX6JQ5XBxoTDEMKQzvo07NIK6nTon/SivlZqvhzvuVBNQhObQ==", "cpu": [ "x64" ], @@ -969,9 +1110,9 @@ } }, "node_modules/@rolldown/binding-linux-x64-musl": { - "version": "1.1.5", - "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-x64-musl/-/binding-linux-x64-musl-1.1.5.tgz", - "integrity": "sha512-Me+PfPI2TMeOQk0gYWfLQZtTktrmzbr8cDboqX83XKc7UrgAi55gF+2dUkWdxd19n55Essp2yeca+O9N5rBxHg==", + "version": "1.2.4", + "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-x64-musl/-/binding-linux-x64-musl-1.2.4.tgz", + "integrity": "sha512-l9eeLsCNvPpmSXUej0etw/J1eqV0Jj1D5G/xG6YTijmE6dkv6E2QezgWbTfQk63v952DPqrjOCoiqxq7Bw0YUQ==", "cpu": [ "x64" ], @@ -989,9 +1130,9 @@ } }, "node_modules/@rolldown/binding-openharmony-arm64": { - "version": "1.1.5", - "resolved": "https://registry.npmjs.org/@rolldown/binding-openharmony-arm64/-/binding-openharmony-arm64-1.1.5.tgz", - "integrity": "sha512-yc5WrLzXks6zCQfn9Oxr8pORKyl/pF+QjHmW/Qx3qu0oyrrNC+y2JLTU1E2rcWYAmzlnqngWXHQjy51VzW70Vw==", + "version": "1.2.4", + "resolved": "https://registry.npmjs.org/@rolldown/binding-openharmony-arm64/-/binding-openharmony-arm64-1.2.4.tgz", + "integrity": "sha512-e0F355MSTMm3+UOqtV3L24gFUp2N5m1f8L/7d56deik6va+AXdrt9F8LbzGpeWGWRbZEDq4m8NVnJDeBtf9DZg==", "cpu": [ "arm64" ], @@ -1005,63 +1146,10 @@ "node": "^20.19.0 || >=22.12.0" } }, - "node_modules/@rolldown/binding-wasm32-wasi": { - "version": "1.1.5", - "resolved": "https://registry.npmjs.org/@rolldown/binding-wasm32-wasi/-/binding-wasm32-wasi-1.1.5.tgz", - "integrity": "sha512-VbQGPX2b4r48TAMIM2cjgluIM1HYutm4pcTEJsle7iEP7sB1dFqtPLBVbdLAZCxy1txCcPxf4QFf4v8uvltPqA==", - "cpu": [ - "wasm32" - ], - "dev": true, - "license": "MIT", - "optional": true, - "dependencies": { - "@emnapi/core": "1.11.1", - "@emnapi/runtime": "1.11.1", - "@napi-rs/wasm-runtime": "^1.1.6" - }, - "engines": { - "node": "^20.19.0 || >=22.12.0" - } - }, - "node_modules/@rolldown/binding-wasm32-wasi/node_modules/@emnapi/core": { - "version": "1.11.1", - "resolved": "https://registry.npmjs.org/@emnapi/core/-/core-1.11.1.tgz", - "integrity": "sha512-RSvbQmHzdKzNsLYa/wHrbc3KN4sYLKAdPZxqiM2HATqv/SBk2/ENSHpvXGaLOMcsAyz0poEGqkmmKYG3OWiJEQ==", - "dev": true, - "license": "MIT", - "optional": true, - "dependencies": { - "@emnapi/wasi-threads": "1.2.2", - "tslib": "^2.4.0" - } - }, - "node_modules/@rolldown/binding-wasm32-wasi/node_modules/@emnapi/runtime": { - "version": "1.11.1", - "resolved": "https://registry.npmjs.org/@emnapi/runtime/-/runtime-1.11.1.tgz", - "integrity": "sha512-vgj7R3y3Wgx24IQaGPA/R6YFXLHVMOZ0uVEyIQPaWs+rd1AzfEMXlAC22FYwO1XkKR6NPsq7mUandH8oIRdZFw==", - "dev": true, - "license": "MIT", - "optional": true, - "dependencies": { - "tslib": "^2.4.0" - } - }, - "node_modules/@rolldown/binding-wasm32-wasi/node_modules/@emnapi/wasi-threads": { - "version": "1.2.2", - "resolved": "https://registry.npmjs.org/@emnapi/wasi-threads/-/wasi-threads-1.2.2.tgz", - "integrity": "sha512-c95qOXkHdydNKhscBTebqEC1CVAZpyqOfVfBzQ1qgzyl3gfeldUjIggDbIZgDKsHLgnsM+igH7TJ/eAasaVuMA==", - "dev": true, - "license": "MIT", - "optional": true, - "dependencies": { - "tslib": "^2.4.0" - } - }, "node_modules/@rolldown/binding-win32-arm64-msvc": { - "version": "1.1.5", - "resolved": "https://registry.npmjs.org/@rolldown/binding-win32-arm64-msvc/-/binding-win32-arm64-msvc-1.1.5.tgz", - "integrity": "sha512-gHv82k63z4qpV5+Q1y/12KrK0ltWBukVDI8nZcbT7Tt/ZlOIVwppazneq0F93oDxTo3IgAMEDIoQh3E2n6mVsw==", + "version": "1.2.4", + "resolved": "https://registry.npmjs.org/@rolldown/binding-win32-arm64-msvc/-/binding-win32-arm64-msvc-1.2.4.tgz", + "integrity": "sha512-AWLi0uBRYh6QlE7OKhiz+phZC0qwtij2QZmhmOdsLdFn64m7oMpooE9ICE3lhm9xMb4SpDo2WbHcxX1iFLFtqw==", "cpu": [ "arm64" ], @@ -1076,9 +1164,9 @@ } }, "node_modules/@rolldown/binding-win32-x64-msvc": { - "version": "1.1.5", - "resolved": "https://registry.npmjs.org/@rolldown/binding-win32-x64-msvc/-/binding-win32-x64-msvc-1.1.5.tgz", - "integrity": "sha512-tTZuDBPw85tEN5PQi1pnEBzDy0Z49HtScLAbD5t6hyeU92A95pRWaSMw1GZZi/RwgSgUIl0xrSlXIT/9QzvYSA==", + "version": "1.2.4", + "resolved": "https://registry.npmjs.org/@rolldown/binding-win32-x64-msvc/-/binding-win32-x64-msvc-1.2.4.tgz", + "integrity": "sha512-UwSDJOg3dqCAejWdxclJjCsh3Qq4vLYMDxmyHqo1btz3stK2VqgwNd3mm5tuIwzSlGIQ/1H9Hr+Zn09mrezNqQ==", "cpu": [ "x64" ], @@ -1317,17 +1405,6 @@ "@tensorflow/tfjs-core": "4.22.0" } }, - "node_modules/@tybys/wasm-util": { - "version": "0.10.3", - "resolved": "https://registry.npmjs.org/@tybys/wasm-util/-/wasm-util-0.10.3.tgz", - "integrity": "sha512-F3fo1MYrRJYL3zER0OUOmkutjr1Vp23m7OsSgp7nq4SP6OqX6C/56XFIPAl5bt3zaBRjmW7SGz3u/6LwFpYcOg==", - "dev": true, - "license": "MIT", - "optional": true, - "dependencies": { - "tslib": "^2.4.0" - } - }, "node_modules/@types/estree": { "version": "1.0.9", "resolved": "https://registry.npmjs.org/@types/estree/-/estree-1.0.9.tgz", @@ -1544,9 +1621,9 @@ "license": "MIT" }, "node_modules/brace-expansion": { - "version": "2.1.3", - "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-2.1.3.tgz", - "integrity": "sha512-DRdx5neNsG/QXbniLFWi2YmC/68oeOOmKz6zOjVk6ZS1ZLXgLIKqVEc6hWsmkjBbgii0SwaBTcJ5XKj5gzY/4A==", + "version": "2.1.4", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-2.1.4.tgz", + "integrity": "sha512-hGfVzPxthbf3+2yjg/RBs60cB0FhqBS/zvdV/4wn4/BmN0bNMMHPc4V/BbFieqf1TKAGGAHnY4eSjajCl0f2Xg==", "dev": true, "license": "MIT", "dependencies": { @@ -3011,9 +3088,9 @@ "license": "MIT" }, "node_modules/nanoid": { - "version": "3.3.16", - "resolved": "https://registry.npmjs.org/nanoid/-/nanoid-3.3.16.tgz", - "integrity": "sha512-bzlKTyNJ7+LdGIIwy8ijFpIqEQIvafahV7eYykJ8Cvh42EdJeODoJ6gUJXpQJvej1BddH8OqTXZNE/KfbWAu8Q==", + "version": "3.3.18", + "resolved": "https://registry.npmjs.org/nanoid/-/nanoid-3.3.18.tgz", + "integrity": "sha512-DTg4MJbGMWkfi6VZFdNt2/caMbQy4Ou+Op/hJQvGEWcnVfoA1QA+xzRKAzw9jD6+GVOOeYr/mIcuDSdug6F6+w==", "dev": true, "funding": [ { @@ -3233,6 +3310,38 @@ "dev": true, "license": "MIT" }, + "node_modules/pdf-parse": { + "version": "2.4.5", + "resolved": "https://registry.npmjs.org/pdf-parse/-/pdf-parse-2.4.5.tgz", + "integrity": "sha512-mHU89HGh7v+4u2ubfnevJ03lmPgQ5WU4CxAVmTSh/sxVTEDYd1er/dKS/A6vg77NX47KTEoihq8jZBLr8Cxuwg==", + "license": "Apache-2.0", + "dependencies": { + "@napi-rs/canvas": "0.1.80", + "pdfjs-dist": "5.4.296" + }, + "bin": { + "pdf-parse": "bin/cli.mjs" + }, + "engines": { + "node": ">=20.16.0 <21 || >=22.3.0" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/mehmet-kozan" + } + }, + "node_modules/pdfjs-dist": { + "version": "5.4.296", + "resolved": "https://registry.npmjs.org/pdfjs-dist/-/pdfjs-dist-5.4.296.tgz", + "integrity": "sha512-DlOzet0HO7OEnmUmB6wWGJrrdvbyJKftI1bhMitK7O2N8W2gc757yyYBbINy9IDafXAV9wmKr9t7xsTaNKRG5Q==", + "license": "Apache-2.0", + "engines": { + "node": ">=20.16.0 || >=22.3.0" + }, + "optionalDependencies": { + "@napi-rs/canvas": "^0.1.80" + } + }, "node_modules/picocolors": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/picocolors/-/picocolors-1.1.1.tgz", @@ -3272,9 +3381,9 @@ "license": "MIT" }, "node_modules/postcss": { - "version": "8.5.25", - "resolved": "https://registry.npmjs.org/postcss/-/postcss-8.5.25.tgz", - "integrity": "sha512-DTPx3RWSSnWyzLxQnlH0rJP+EW5ekl16ZU4/psbIhA0e53kJfdgaN5vKM+xP7yJtXVu+nfdVFmlgFDEKAe4Pyw==", + "version": "8.5.26", + "resolved": "https://registry.npmjs.org/postcss/-/postcss-8.5.26.tgz", + "integrity": "sha512-u82N74LFzG8ca+dD8puPnplTXoGH4fTPpVGuIbt36G3qvNlkvfD0lEAZSxaly3KX8TS/L1A1gsCEmvKmBcVbkQ==", "dev": true, "funding": [ { @@ -3292,7 +3401,7 @@ ], "license": "MIT", "dependencies": { - "nanoid": "^3.3.16", + "nanoid": "^3.3.17", "picocolors": "^1.1.1", "source-map-js": "^1.2.1" }, @@ -3434,13 +3543,13 @@ "license": "BSD-3-Clause" }, "node_modules/rolldown": { - "version": "1.1.5", - "resolved": "https://registry.npmjs.org/rolldown/-/rolldown-1.1.5.tgz", - "integrity": "sha512-t9z29cJjXf/vxQ8dyhCSpt6H6aSwHTk8cT5I3iy6SMXuFpk5mB6PL6XfC8PCwrPTx93udwKUm9HRteAlTGBLiA==", + "version": "1.2.4", + "resolved": "https://registry.npmjs.org/rolldown/-/rolldown-1.2.4.tgz", + "integrity": "sha512-rSr7irW0K7QRWzjdJXqZowkcRdDtjRduh43rBltnVKd0VFq839l1lJoDvGJb6gl7+4rTTCrPWu+YfujUL8Ug7w==", "dev": true, "license": "MIT", "dependencies": { - "@oxc-project/types": "=0.139.0", + "@oxc-project/types": "=0.144.0", "@rolldown/pluginutils": "^1.0.0" }, "bin": { @@ -3450,21 +3559,20 @@ "node": "^20.19.0 || >=22.12.0" }, "optionalDependencies": { - "@rolldown/binding-android-arm64": "1.1.5", - "@rolldown/binding-darwin-arm64": "1.1.5", - "@rolldown/binding-darwin-x64": "1.1.5", - "@rolldown/binding-freebsd-x64": "1.1.5", - "@rolldown/binding-linux-arm-gnueabihf": "1.1.5", - "@rolldown/binding-linux-arm64-gnu": "1.1.5", - "@rolldown/binding-linux-arm64-musl": "1.1.5", - "@rolldown/binding-linux-ppc64-gnu": "1.1.5", - "@rolldown/binding-linux-s390x-gnu": "1.1.5", - "@rolldown/binding-linux-x64-gnu": "1.1.5", - "@rolldown/binding-linux-x64-musl": "1.1.5", - "@rolldown/binding-openharmony-arm64": "1.1.5", - "@rolldown/binding-wasm32-wasi": "1.1.5", - "@rolldown/binding-win32-arm64-msvc": "1.1.5", - "@rolldown/binding-win32-x64-msvc": "1.1.5" + "@rolldown/binding-android-arm64": "1.2.4", + "@rolldown/binding-darwin-arm64": "1.2.4", + "@rolldown/binding-darwin-x64": "1.2.4", + "@rolldown/binding-freebsd-x64": "1.2.4", + "@rolldown/binding-linux-arm-gnueabihf": "1.2.4", + "@rolldown/binding-linux-arm64-gnu": "1.2.4", + "@rolldown/binding-linux-arm64-musl": "1.2.4", + "@rolldown/binding-linux-ppc64-gnu": "1.2.4", + "@rolldown/binding-linux-s390x-gnu": "1.2.4", + "@rolldown/binding-linux-x64-gnu": "1.2.4", + "@rolldown/binding-linux-x64-musl": "1.2.4", + "@rolldown/binding-openharmony-arm64": "1.2.4", + "@rolldown/binding-win32-arm64-msvc": "1.2.4", + "@rolldown/binding-win32-x64-msvc": "1.2.4" } }, "node_modules/safe-buffer": { @@ -4021,16 +4129,16 @@ } }, "node_modules/vite": { - "version": "8.1.5", - "resolved": "https://registry.npmjs.org/vite/-/vite-8.1.5.tgz", - "integrity": "sha512-7ULLwsCdYx/nRyrpiEwvqb5TFHrMVZyBt+rg/OAXT7rgj/z+DtTDyKFeLAdDkubDVDKD8jOsndmy7m55XcfUsw==", + "version": "8.2.1", + "resolved": "https://registry.npmjs.org/vite/-/vite-8.2.1.tgz", + "integrity": "sha512-EU/eS7BH3XROHh2YnBefjM6DBKA6ZeMZEYQbj7NLWg5wHYlhB8B/Mayd5XsgWq+NFYccDOTemRpdETWR6Ka/lw==", "dev": true, "license": "MIT", "dependencies": { - "lightningcss": "^1.32.0", + "lightningcss": "^1.33.0", "picomatch": "^4.0.5", - "postcss": "^8.5.17", - "rolldown": "~1.1.5", + "postcss": "^8.5.25", + "rolldown": "~1.2.1", "tinyglobby": "^0.2.17" }, "bin": { @@ -4047,7 +4155,7 @@ }, "peerDependencies": { "@types/node": "^20.19.0 || >=22.12.0", - "@vitejs/devtools": "^0.3.0", + "@vitejs/devtools": "^0.4.0", "esbuild": "^0.27.0 || ^0.28.0", "jiti": ">=1.21.0", "less": "^4.0.0", @@ -4132,9 +4240,9 @@ "license": "MIT" }, "node_modules/wasm-feature-detect": { - "version": "1.8.0", - "resolved": "https://registry.npmjs.org/wasm-feature-detect/-/wasm-feature-detect-1.8.0.tgz", - "integrity": "sha512-zksaLKM2fVlnB5jQQDqKXXwYHLQUVH9es+5TOOHwGOVJOCeRBCiPjwSg+3tN2AdTCzjgli4jijCH290kXb/zWQ==", + "version": "1.9.0", + "resolved": "https://registry.npmjs.org/wasm-feature-detect/-/wasm-feature-detect-1.9.0.tgz", + "integrity": "sha512-zonE+xlIIYtxPy++L24ow0hAD8CICb4+FgPyROd3buyXIqsJvUEDkBgfCCoXOd1Hu3DUr0GOfnPIdcGV+YpNaA==", "license": "Apache-2.0" }, "node_modules/webidl-conversions": { diff --git a/package.json b/package.json index 429b3d7..23fb7ce 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@ztimson/ai-utils", - "version": "1.5.0", + "version": "1.6.0", "description": "AI Utility library", "author": "Zak Timson", "license": "MIT", @@ -26,12 +26,13 @@ }, "dependencies": { "@anthropic-ai/sdk": "^0.102.0", - "@tensorflow/tfjs": "^4.22.0", "@huggingface/transformers": "^4.2.0", + "@tensorflow/tfjs": "^4.22.0", "@ztimson/node-utils": "^1.0.7", "@ztimson/utils": "^0.29.4", "cheerio": "^1.2.0", "openai": "^6.42.0", + "pdf-parse": "^2.4.5", "tesseract.js": "^7.0.0" }, "devDependencies": { diff --git a/src/antrhopic.ts b/src/antrhopic.ts index 46c73fa..bbd9611 100644 --- a/src/antrhopic.ts +++ b/src/antrhopic.ts @@ -24,6 +24,13 @@ export class Anthropic extends LLMProvider { return client; } + private toWireContent(content: any): any { + if(!Array.isArray(content)) return content; + return content.map(c => c.type === 'image' + ? {type: 'image', source: {type: 'base64', media_type: c.mime, data: c.data}} + : {type: 'text', text: c.text}); + } + /** Convert standard history -> Anthropic wire format */ private toWire(history: LLMMessage[]): any[] { const wire: any[] = []; @@ -34,7 +41,7 @@ export class Anthropic extends LLMProvider { {role: 'user', content: [{type: 'tool_result', tool_use_id: h.id, is_error: !!h.error, content: h.error || h.content || ''}]} ); } else { - wire.push({role: h.role, content: h.content}); + wire.push({role: h.role, content: this.toWireContent(h.content)}); } } return wire; diff --git a/src/llm.ts b/src/llm.ts index dd31c31..26a38a8 100644 --- a/src/llm.ts +++ b/src/llm.ts @@ -5,11 +5,16 @@ import {OpenAi} from './open-ai.ts'; import {LLMProvider} from './provider.ts'; import {AiTool, AiToolArg} from './tools.ts'; import {fileURLToPath} from 'url'; -import {dirname, join} from 'path'; import {spawn} from 'node:child_process'; import {Memory, MemoryCache, MemoryManager, MemoryOptions, stripHeader} from './memory.ts'; +import {mkdtempSync} from 'node:fs'; +import fs from 'node:fs/promises'; +import {tmpdir} from 'node:os'; +import {dirname, join, basename, extname} from 'path'; +import { PDFParse } from 'pdf-parse'; const MAX_AGENT_DEPTH = 5; +const PDF_OCR_PAGE_THRESHOLD = 12; // above this many pages, OCR scanned pages instead of feeding images to the model export type AnthropicConfig = {proto: 'anthropic', token: string | string[]}; export type OpenAiConfig = {proto: 'openai', host?: string, token: string | string[]}; @@ -27,6 +32,19 @@ export type Agent = { agents?: string[] | null; } +export type LLMFile = { + /** Path to file on disk */ + path?: string; + /** File content: raw text, base64-encoded binary, or a Buffer */ + content?: string | Buffer; + /** Original filename, used to infer type from extension */ + name?: string; + /** Mime type override, inferred from extension if omitted */ + mime?: string; + /** @internal set once extraction has run, skips re-processing next turn */ + extracted?: boolean; +}; + export type LLMMessage = { /** Message originator */ role: 'assistant' | 'system' | 'user'; @@ -88,6 +106,8 @@ export type LLMRequest = { mcp?: McpServer[]; /** Subagents exposed as delegatable/wrapped tools */ agents?: Agent[]; + /** Attach files to request */ + files?: LLMFile[]; /** @internal recursion guard for nested agent delegation */ _agentDepth?: number; } @@ -111,6 +131,11 @@ export type Skill = { } class LLM { + private static AUDIO_EXT = ['wav','mp3','m4a','flac','ogg','aac','wma']; + private static IMAGE_EXT = ['png','jpg','jpeg','bmp','gif','tiff','webp']; + private static TEXT_EXT = ['txt','md','csv','json','xml','html','js','ts','py','yaml','yml','log']; + private static PDF_EXT = ['pdf']; + private memoryManager!: MemoryManager; defaultModel!: string; @@ -126,6 +151,120 @@ class LLM { this.memoryManager = new MemoryManager(this); } + private async loadBuffer(file: LLMFile, asText: boolean): Promise { + if(file.path) return fs.readFile(file.path); + if(Buffer.isBuffer(file.content)) return file.content; + if(typeof file.content === 'string') return Buffer.from(file.content, asText ? 'utf-8' : 'base64'); + throw new Error('No path or content provided'); + } + + private async writeTemp(name: string, buffer: Buffer): Promise { + const path = join(mkdtempSync(join(tmpdir(), 'ai-file-')), name); + await fs.writeFile(path, buffer); + return path; + } + + /** + * Extract text from a PDF. Pages with no text layer (scanned/image-only) are handled as either: + * - Rendered to images and returned alongside the text so the (vision-capable) model can read them directly + * - OCR'd via Tesseract when the doc is too large to reasonably pass as images + */ + private async resolvePdf(buffer: Buffer): Promise<{text: string, images: {mime: string, data: string}[]}> { + const parser = new PDFParse({data: buffer}); + try { + const {text, pages} = await parser.getText(); + const scanned = (pages || []).filter(p => !p.text?.trim()); + if(!scanned.length) return {text: text.trim() || '[Empty PDF]', images: []}; + const total = pages.length; + const pageNums = scanned.map(p => p.num); + const {pages: shots} = await parser.getScreenshot({partial: pageNums}); + if(total <= PDF_OCR_PAGE_THRESHOLD) { + return { + text: text.trim(), + images: shots.map(s => ({mime: 'image/png', data: Buffer.from(s.data).toString('base64')})) + }; + } + const ocrText = await Promise.all(shots.map(async (s, i) => { + const path = await this.writeTemp(`page-${pageNums[i]}.png`, Buffer.from(s.data)); + try { + return await this.ai.vision.ocr(path) || ''; + } finally { + fs.rm(dirname(path), {recursive: true, force: true}).catch(() => {}); + } + })); + return {text: [text.trim(), ...ocrText].filter(Boolean).join('\n\n'), images: []}; + } finally { + await parser.destroy(); + } + } + + private async resolveFile(file: LLMFile): Promise<{text?: string, images?: {mime: string, data: string}[]}> { + const name = file.name || (file.path ? basename(file.path) : 'file'); + + // Already resolved on a previous turn, reuse cached text + if(file.extracted) return {text: `\n${file.content}\n`}; + + const ext = extname(name).slice(1).toLowerCase(); + const mime = file.mime || ''; + const isAudio = mime.startsWith('audio/') || LLM.AUDIO_EXT.includes(ext); + const isImage = mime.startsWith('image/') || LLM.IMAGE_EXT.includes(ext); + const isPdf = mime === 'application/pdf' || LLM.PDF_EXT.includes(ext); + const isText = mime.startsWith('text/') || LLM.TEXT_EXT.includes(ext); + + let tmpDir: string | null = null; + try { + if(isImage) { + const data = (await this.loadBuffer(file, false)).toString('base64'); + return {images: [{mime: mime || `image/${ext === 'jpg' ? 'jpeg' : ext}`, data}]}; + } + + if(isPdf) { + const {text, images} = await this.resolvePdf(await this.loadBuffer(file, false)); + // Only cache/skip re-processing when we didn't need to hand off images (OCR'd or fully text-based) + if(!images.length) { + file.content = text; + file.extracted = true; + delete file.path; + } + return {text: `\n${text || '[Scanned PDF - see attached page images]'}\n`, images}; + } + + let text: string; + if(isAudio) { + let path = file.path; + if(!path) { + const buffer = await this.loadBuffer(file, false); + path = await this.writeTemp(name, buffer); + tmpDir = dirname(path); + } + text = await this.ai.audio.asr(path) || ''; + } else if(isText) { + text = (await this.loadBuffer(file, true)).toString('utf-8'); + } else { + text = `Unsupported file type: ${ext || mime}`; + } + + // Cache result, skip re-extraction on future turns of the same conversation + file.content = text; + file.extracted = true; + delete file.path; + + return {text: `\n${text}\n`}; + } catch(err: any) { + return {text: `Failed to process: ${err.message}`}; + } finally { + if(tmpDir) fs.rm(tmpDir, {recursive: true, force: true}).catch(() => {}); + } + } + + private async resolveFiles(files: LLMFile[]): Promise<{text: string, images: {mime: string, data: string}[]}> { + const resolved = await Promise.all(files.map(f => this.resolveFile(f))); + return { + text: resolved.filter(r => r.text).map(r => r.text).join('\n\n'), + images: resolved.flatMap(r => r.images || []) + }; + } + private setupAgent(agents: Agent[] = [], allAgents: Agent[], history: LLMMessage[], aborts: (() => void)[], depth = 0, delegateState: {resp: string | null}): AiTool[] { return agents.map(a => { const toolName = `${a.delegate ? '' : 'sub'}agent_${snakeCase(a.name)}`; @@ -343,6 +482,18 @@ Linked: ${makeUnique([...r.links, ...r.backlinks]).join(', ')} if(aborted) throw Object.assign(new Error('Aborted'), {name: 'AbortError'}); + // Files + const files = options.files || []; + const lastMsg = history[history.length - 1]; + const originalContent = lastMsg?.content; + if(files.length && lastMsg?.role === 'user') { + const {text, images} = await this.resolveFiles(files); + const merged = text ? `${originalContent}\n\n${text}` : originalContent; + lastMsg.content = images.length + ? [...images.map(i => ({type: 'image', mime: i.mime, data: i.data})), {type: 'text', text: merged}] + : merged; + } + const toolTimings = new Map(); tools = this.wrapToolTiming(tools, toolTimings); @@ -352,6 +503,8 @@ Linked: ${makeUnique([...r.links, ...r.backlinks]).join(', ')} request = this.models[m].ask('', {...options, tools, system: prompts.filter(Boolean).join('\n\n')}); let resp = await request; + if(files.length && lastMsg?.role === 'user') lastMsg.content = originalContent; + // Capture meta (duration / tps) for(const h of history) { if(h.role === 'tool' && toolTimings.has(h.id)) Object.assign(h, toolTimings.get(h.id)); diff --git a/src/open-ai.ts b/src/open-ai.ts index b8df409..97baeb6 100644 --- a/src/open-ai.ts +++ b/src/open-ai.ts @@ -25,6 +25,13 @@ export class OpenAi extends LLMProvider { return client; } + private toWireContent(content: any): any { + if(!Array.isArray(content)) return content; + return content.map(c => c.type === 'image' + ? {type: 'image_url', image_url: {url: `data:${c.mime};base64,${c.data}`}} + : {type: 'text', text: c.text}); + } + /** Convert standard history -> OpenAI wire format */ private toWire(history: LLMMessage[], system?: string): any[] { const wire: any[] = []; @@ -41,7 +48,7 @@ export class OpenAi extends LLMProvider { content: h.error || h.content || '', }); } else { - wire.push({role: h.role, content: h.content}); + wire.push({role: h.role, content: this.toWireContent(h.content)}); } } return wire;