Compare commits
7 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| 0a6f1e4d62 | |||
| 08a351e028 | |||
| 85c01d3ef1 | |||
| 5826573d5c | |||
| 797a40a566 | |||
| 7308927a3c | |||
| 04f038ba65 |
@@ -119,7 +119,7 @@ const ai = new Ai({
|
|||||||
system: 'You are a helpful assistant.',
|
system: 'You are a helpful assistant.',
|
||||||
compress: {max: 90_000, min: 50_000}, // Compress chat history to min tokens when max is reached
|
compress: {max: 90_000, min: 50_000}, // Compress chat history to min tokens when max is reached
|
||||||
temperature: 0.8,
|
temperature: 0.8,
|
||||||
max_tokens: 100_000,
|
maxTokens: 100_000,
|
||||||
memoryModel: 'gpt-4o', // Cheap model for managing memories in background, defaults to current model
|
memoryModel: 'gpt-4o', // Cheap model for managing memories in background, defaults to current model
|
||||||
models: {
|
models: {
|
||||||
'claude-3-5-sonnet': {proto: 'anthropic', token: process.env.ANTHROPIC_TOKEN},
|
'claude-3-5-sonnet': {proto: 'anthropic', token: process.env.ANTHROPIC_TOKEN},
|
||||||
|
|||||||
518
package-lock.json
generated
518
package-lock.json
generated
@@ -1,12 +1,12 @@
|
|||||||
{
|
{
|
||||||
"name": "@ztimson/ai-utils",
|
"name": "@ztimson/ai-utils",
|
||||||
"version": "1.2.6",
|
"version": "1.5.0",
|
||||||
"lockfileVersion": 3,
|
"lockfileVersion": 3,
|
||||||
"requires": true,
|
"requires": true,
|
||||||
"packages": {
|
"packages": {
|
||||||
"": {
|
"": {
|
||||||
"name": "@ztimson/ai-utils",
|
"name": "@ztimson/ai-utils",
|
||||||
"version": "1.2.6",
|
"version": "1.5.0",
|
||||||
"license": "MIT",
|
"license": "MIT",
|
||||||
"dependencies": {
|
"dependencies": {
|
||||||
"@anthropic-ai/sdk": "^0.102.0",
|
"@anthropic-ai/sdk": "^0.102.0",
|
||||||
@@ -16,6 +16,7 @@
|
|||||||
"@ztimson/utils": "^0.29.4",
|
"@ztimson/utils": "^0.29.4",
|
||||||
"cheerio": "^1.2.0",
|
"cheerio": "^1.2.0",
|
||||||
"openai": "^6.42.0",
|
"openai": "^6.42.0",
|
||||||
|
"pdf-parse": "^2.4.5",
|
||||||
"tesseract.js": "^7.0.0"
|
"tesseract.js": "^7.0.0"
|
||||||
},
|
},
|
||||||
"devDependencies": {
|
"devDependencies": {
|
||||||
@@ -56,39 +57,12 @@
|
|||||||
"node": ">=6.9.0"
|
"node": ">=6.9.0"
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
"node_modules/@emnapi/core": {
|
|
||||||
"version": "2.0.0-alpha.3",
|
|
||||||
"resolved": "https://registry.npmjs.org/@emnapi/core/-/core-2.0.0-alpha.3.tgz",
|
|
||||||
"integrity": "sha512-AZypUeJ/yByuxyS7BlSNRDOMLMlROYtjYdIAuBmJssVz1UJDSeYxLrdizhXCFYhedC5bqd/ASy8EuNXbVVXp9g==",
|
|
||||||
"dev": true,
|
|
||||||
"license": "MIT",
|
|
||||||
"optional": true,
|
|
||||||
"peer": true,
|
|
||||||
"dependencies": {
|
|
||||||
"@emnapi/wasi-threads": "2.0.1",
|
|
||||||
"tslib": "^2.4.0"
|
|
||||||
}
|
|
||||||
},
|
|
||||||
"node_modules/@emnapi/runtime": {
|
"node_modules/@emnapi/runtime": {
|
||||||
"version": "2.0.0-alpha.3",
|
"version": "1.11.3",
|
||||||
"resolved": "https://registry.npmjs.org/@emnapi/runtime/-/runtime-2.0.0-alpha.3.tgz",
|
"resolved": "https://registry.npmjs.org/@emnapi/runtime/-/runtime-1.11.3.tgz",
|
||||||
"integrity": "sha512-hFPAhMUjJD9BSyCANEISPOogeXC9Zo9ZQl7L6vKnaVsMkCtzznaW/naYypeyl0Gv5rYfWYsZbpixTMpjDJzQeA==",
|
"integrity": "sha512-Xz4Tpyki7XyrpbUK1jR1AhdAdaXyhhY4lZ3neLodmhpuWfy2PAQN5B46sAiU4liOXGLkHypn/qU+jvfWSCYYLA==",
|
||||||
"dev": true,
|
|
||||||
"license": "MIT",
|
"license": "MIT",
|
||||||
"optional": true,
|
"optional": true,
|
||||||
"peer": true,
|
|
||||||
"dependencies": {
|
|
||||||
"tslib": "^2.4.0"
|
|
||||||
}
|
|
||||||
},
|
|
||||||
"node_modules/@emnapi/wasi-threads": {
|
|
||||||
"version": "2.0.1",
|
|
||||||
"resolved": "https://registry.npmjs.org/@emnapi/wasi-threads/-/wasi-threads-2.0.1.tgz",
|
|
||||||
"integrity": "sha512-9DsSk+o5NBX0CCJT8s0EROGSGxjR/tKu6aBTaVyq+SjAEQH4XcdcRxPBRzsBLizTTJ49MJjF+jgu3qnO9GLQcQ==",
|
|
||||||
"dev": true,
|
|
||||||
"license": "MIT",
|
|
||||||
"optional": true,
|
|
||||||
"peer": true,
|
|
||||||
"dependencies": {
|
"dependencies": {
|
||||||
"tslib": "^2.4.0"
|
"tslib": "^2.4.0"
|
||||||
}
|
}
|
||||||
@@ -577,16 +551,6 @@
|
|||||||
"url": "https://opencollective.com/libvips"
|
"url": "https://opencollective.com/libvips"
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
"node_modules/@img/sharp-wasm32/node_modules/@emnapi/runtime": {
|
|
||||||
"version": "1.11.3",
|
|
||||||
"resolved": "https://registry.npmjs.org/@emnapi/runtime/-/runtime-1.11.3.tgz",
|
|
||||||
"integrity": "sha512-Xz4Tpyki7XyrpbUK1jR1AhdAdaXyhhY4lZ3neLodmhpuWfy2PAQN5B46sAiU4liOXGLkHypn/qU+jvfWSCYYLA==",
|
|
||||||
"license": "MIT",
|
|
||||||
"optional": true,
|
|
||||||
"dependencies": {
|
|
||||||
"tslib": "^2.4.0"
|
|
||||||
}
|
|
||||||
},
|
|
||||||
"node_modules/@img/sharp-win32-arm64": {
|
"node_modules/@img/sharp-win32-arm64": {
|
||||||
"version": "0.34.5",
|
"version": "0.34.5",
|
||||||
"resolved": "https://registry.npmjs.org/@img/sharp-win32-arm64/-/sharp-win32-arm64-0.34.5.tgz",
|
"resolved": "https://registry.npmjs.org/@img/sharp-win32-arm64/-/sharp-win32-arm64-0.34.5.tgz",
|
||||||
@@ -694,32 +658,209 @@
|
|||||||
"@jridgewell/sourcemap-codec": "^1.4.14"
|
"@jridgewell/sourcemap-codec": "^1.4.14"
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
"node_modules/@napi-rs/wasm-runtime": {
|
"node_modules/@napi-rs/canvas": {
|
||||||
"version": "1.2.0",
|
"version": "0.1.80",
|
||||||
"resolved": "https://registry.npmjs.org/@napi-rs/wasm-runtime/-/wasm-runtime-1.2.0.tgz",
|
"resolved": "https://registry.npmjs.org/@napi-rs/canvas/-/canvas-0.1.80.tgz",
|
||||||
"integrity": "sha512-kDoONqMa+VnZ4vvvu/ZUurpJ4gkZU57e7g69qpNgWhYcZFPUHZM2CEMKm+cG6ufDVALbjMvfmMjFVqaK7uEMnA==",
|
"integrity": "sha512-DxuT1ClnIPts1kQx8FBmkk4BQDTfI5kIzywAaMjQSXfNnra5UFU9PwurXrl+Je3bJ6BGsp/zmshVVFbCmyI+ww==",
|
||||||
"dev": true,
|
"license": "MIT",
|
||||||
|
"workspaces": [
|
||||||
|
"e2e/*"
|
||||||
|
],
|
||||||
|
"engines": {
|
||||||
|
"node": ">= 10"
|
||||||
|
},
|
||||||
|
"optionalDependencies": {
|
||||||
|
"@napi-rs/canvas-android-arm64": "0.1.80",
|
||||||
|
"@napi-rs/canvas-darwin-arm64": "0.1.80",
|
||||||
|
"@napi-rs/canvas-darwin-x64": "0.1.80",
|
||||||
|
"@napi-rs/canvas-linux-arm-gnueabihf": "0.1.80",
|
||||||
|
"@napi-rs/canvas-linux-arm64-gnu": "0.1.80",
|
||||||
|
"@napi-rs/canvas-linux-arm64-musl": "0.1.80",
|
||||||
|
"@napi-rs/canvas-linux-riscv64-gnu": "0.1.80",
|
||||||
|
"@napi-rs/canvas-linux-x64-gnu": "0.1.80",
|
||||||
|
"@napi-rs/canvas-linux-x64-musl": "0.1.80",
|
||||||
|
"@napi-rs/canvas-win32-x64-msvc": "0.1.80"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"node_modules/@napi-rs/canvas-android-arm64": {
|
||||||
|
"version": "0.1.80",
|
||||||
|
"resolved": "https://registry.npmjs.org/@napi-rs/canvas-android-arm64/-/canvas-android-arm64-0.1.80.tgz",
|
||||||
|
"integrity": "sha512-sk7xhN/MoXeuExlggf91pNziBxLPVUqF2CAVnB57KLG/pz7+U5TKG8eXdc3pm0d7Od0WreB6ZKLj37sX9muGOQ==",
|
||||||
|
"cpu": [
|
||||||
|
"arm64"
|
||||||
|
],
|
||||||
"license": "MIT",
|
"license": "MIT",
|
||||||
"optional": true,
|
"optional": true,
|
||||||
"dependencies": {
|
"os": [
|
||||||
"@tybys/wasm-util": "^0.10.3"
|
"android"
|
||||||
},
|
],
|
||||||
"engines": {
|
"engines": {
|
||||||
"node": "^20.19.0 || ^22.13.0 || >=23.5.0"
|
"node": ">= 10"
|
||||||
|
}
|
||||||
},
|
},
|
||||||
"funding": {
|
"node_modules/@napi-rs/canvas-darwin-arm64": {
|
||||||
"type": "github",
|
"version": "0.1.80",
|
||||||
"url": "https://github.com/sponsors/Brooooooklyn"
|
"resolved": "https://registry.npmjs.org/@napi-rs/canvas-darwin-arm64/-/canvas-darwin-arm64-0.1.80.tgz",
|
||||||
|
"integrity": "sha512-O64APRTXRUiAz0P8gErkfEr3lipLJgM6pjATwavZ22ebhjYl/SUbpgM0xcWPQBNMP1n29afAC/Us5PX1vg+JNQ==",
|
||||||
|
"cpu": [
|
||||||
|
"arm64"
|
||||||
|
],
|
||||||
|
"license": "MIT",
|
||||||
|
"optional": true,
|
||||||
|
"os": [
|
||||||
|
"darwin"
|
||||||
|
],
|
||||||
|
"engines": {
|
||||||
|
"node": ">= 10"
|
||||||
|
}
|
||||||
},
|
},
|
||||||
"peerDependencies": {
|
"node_modules/@napi-rs/canvas-darwin-x64": {
|
||||||
"@emnapi/core": "^2.0.0-alpha.3",
|
"version": "0.1.80",
|
||||||
"@emnapi/runtime": "^2.0.0-alpha.3"
|
"resolved": "https://registry.npmjs.org/@napi-rs/canvas-darwin-x64/-/canvas-darwin-x64-0.1.80.tgz",
|
||||||
|
"integrity": "sha512-FqqSU7qFce0Cp3pwnTjVkKjjOtxMqRe6lmINxpIZYaZNnVI0H5FtsaraZJ36SiTHNjZlUB69/HhxNDT1Aaa9vA==",
|
||||||
|
"cpu": [
|
||||||
|
"x64"
|
||||||
|
],
|
||||||
|
"license": "MIT",
|
||||||
|
"optional": true,
|
||||||
|
"os": [
|
||||||
|
"darwin"
|
||||||
|
],
|
||||||
|
"engines": {
|
||||||
|
"node": ">= 10"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"node_modules/@napi-rs/canvas-linux-arm-gnueabihf": {
|
||||||
|
"version": "0.1.80",
|
||||||
|
"resolved": "https://registry.npmjs.org/@napi-rs/canvas-linux-arm-gnueabihf/-/canvas-linux-arm-gnueabihf-0.1.80.tgz",
|
||||||
|
"integrity": "sha512-eyWz0ddBDQc7/JbAtY4OtZ5SpK8tR4JsCYEZjCE3dI8pqoWUC8oMwYSBGCYfsx2w47cQgQCgMVRVTFiiO38hHQ==",
|
||||||
|
"cpu": [
|
||||||
|
"arm"
|
||||||
|
],
|
||||||
|
"license": "MIT",
|
||||||
|
"optional": true,
|
||||||
|
"os": [
|
||||||
|
"linux"
|
||||||
|
],
|
||||||
|
"engines": {
|
||||||
|
"node": ">= 10"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"node_modules/@napi-rs/canvas-linux-arm64-gnu": {
|
||||||
|
"version": "0.1.80",
|
||||||
|
"resolved": "https://registry.npmjs.org/@napi-rs/canvas-linux-arm64-gnu/-/canvas-linux-arm64-gnu-0.1.80.tgz",
|
||||||
|
"integrity": "sha512-qwA63t8A86bnxhuA/GwOkK3jvb+XTQaTiVML0vAWoHyoZYTjNs7BzoOONDgTnNtr8/yHrq64XXzUoLqDzU+Uuw==",
|
||||||
|
"cpu": [
|
||||||
|
"arm64"
|
||||||
|
],
|
||||||
|
"libc": [
|
||||||
|
"glibc"
|
||||||
|
],
|
||||||
|
"license": "MIT",
|
||||||
|
"optional": true,
|
||||||
|
"os": [
|
||||||
|
"linux"
|
||||||
|
],
|
||||||
|
"engines": {
|
||||||
|
"node": ">= 10"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"node_modules/@napi-rs/canvas-linux-arm64-musl": {
|
||||||
|
"version": "0.1.80",
|
||||||
|
"resolved": "https://registry.npmjs.org/@napi-rs/canvas-linux-arm64-musl/-/canvas-linux-arm64-musl-0.1.80.tgz",
|
||||||
|
"integrity": "sha512-1XbCOz/ymhj24lFaIXtWnwv/6eFHXDrjP0jYkc6iHQ9q8oXKzUX1Lc6bu+wuGiLhGh2GS/2JlfORC5ZcXimRcg==",
|
||||||
|
"cpu": [
|
||||||
|
"arm64"
|
||||||
|
],
|
||||||
|
"libc": [
|
||||||
|
"musl"
|
||||||
|
],
|
||||||
|
"license": "MIT",
|
||||||
|
"optional": true,
|
||||||
|
"os": [
|
||||||
|
"linux"
|
||||||
|
],
|
||||||
|
"engines": {
|
||||||
|
"node": ">= 10"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"node_modules/@napi-rs/canvas-linux-riscv64-gnu": {
|
||||||
|
"version": "0.1.80",
|
||||||
|
"resolved": "https://registry.npmjs.org/@napi-rs/canvas-linux-riscv64-gnu/-/canvas-linux-riscv64-gnu-0.1.80.tgz",
|
||||||
|
"integrity": "sha512-XTzR125w5ZMs0lJcxRlS1K3P5RaZ9RmUsPtd1uGt+EfDyYMu4c6SEROYsxyatbbu/2+lPe7MPHOO/0a0x7L/gw==",
|
||||||
|
"cpu": [
|
||||||
|
"riscv64"
|
||||||
|
],
|
||||||
|
"libc": [
|
||||||
|
"glibc"
|
||||||
|
],
|
||||||
|
"license": "MIT",
|
||||||
|
"optional": true,
|
||||||
|
"os": [
|
||||||
|
"linux"
|
||||||
|
],
|
||||||
|
"engines": {
|
||||||
|
"node": ">= 10"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"node_modules/@napi-rs/canvas-linux-x64-gnu": {
|
||||||
|
"version": "0.1.80",
|
||||||
|
"resolved": "https://registry.npmjs.org/@napi-rs/canvas-linux-x64-gnu/-/canvas-linux-x64-gnu-0.1.80.tgz",
|
||||||
|
"integrity": "sha512-BeXAmhKg1kX3UCrJsYbdQd3hIMDH/K6HnP/pG2LuITaXhXBiNdh//TVVVVCBbJzVQaV5gK/4ZOCMrQW9mvuTqA==",
|
||||||
|
"cpu": [
|
||||||
|
"x64"
|
||||||
|
],
|
||||||
|
"libc": [
|
||||||
|
"glibc"
|
||||||
|
],
|
||||||
|
"license": "MIT",
|
||||||
|
"optional": true,
|
||||||
|
"os": [
|
||||||
|
"linux"
|
||||||
|
],
|
||||||
|
"engines": {
|
||||||
|
"node": ">= 10"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"node_modules/@napi-rs/canvas-linux-x64-musl": {
|
||||||
|
"version": "0.1.80",
|
||||||
|
"resolved": "https://registry.npmjs.org/@napi-rs/canvas-linux-x64-musl/-/canvas-linux-x64-musl-0.1.80.tgz",
|
||||||
|
"integrity": "sha512-x0XvZWdHbkgdgucJsRxprX/4o4sEed7qo9rCQA9ugiS9qE2QvP0RIiEugtZhfLH3cyI+jIRFJHV4Fuz+1BHHMg==",
|
||||||
|
"cpu": [
|
||||||
|
"x64"
|
||||||
|
],
|
||||||
|
"libc": [
|
||||||
|
"musl"
|
||||||
|
],
|
||||||
|
"license": "MIT",
|
||||||
|
"optional": true,
|
||||||
|
"os": [
|
||||||
|
"linux"
|
||||||
|
],
|
||||||
|
"engines": {
|
||||||
|
"node": ">= 10"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"node_modules/@napi-rs/canvas-win32-x64-msvc": {
|
||||||
|
"version": "0.1.80",
|
||||||
|
"resolved": "https://registry.npmjs.org/@napi-rs/canvas-win32-x64-msvc/-/canvas-win32-x64-msvc-0.1.80.tgz",
|
||||||
|
"integrity": "sha512-Z8jPsM6df5V8B1HrCHB05+bDiCxjE9QA//3YrkKIdVDEwn5RKaqOxCJDRJkl48cJbylcrJbW4HxZbTte8juuPg==",
|
||||||
|
"cpu": [
|
||||||
|
"x64"
|
||||||
|
],
|
||||||
|
"license": "MIT",
|
||||||
|
"optional": true,
|
||||||
|
"os": [
|
||||||
|
"win32"
|
||||||
|
],
|
||||||
|
"engines": {
|
||||||
|
"node": ">= 10"
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
"node_modules/@oxc-project/types": {
|
"node_modules/@oxc-project/types": {
|
||||||
"version": "0.139.0",
|
"version": "0.144.0",
|
||||||
"resolved": "https://registry.npmjs.org/@oxc-project/types/-/types-0.139.0.tgz",
|
"resolved": "https://registry.npmjs.org/@oxc-project/types/-/types-0.144.0.tgz",
|
||||||
"integrity": "sha512-r9gHphtCs+1M7J0pw6Sn/hh/Wpa/iQrOOkrNAlVLF/gHq+/CJmHIWKKUUhdWjcD6CIa8idarspCsASiXCXvFUw==",
|
"integrity": "sha512-nuhZIOLuI6TFQ32I/WnUx+SCPY7SdSKwgnFHydAuoS1+Z4BRcaP+RRJmGzl9lw+0OFF7UmaESf7KQRXaNLHypg==",
|
||||||
"dev": true,
|
"dev": true,
|
||||||
"license": "MIT",
|
"license": "MIT",
|
||||||
"funding": {
|
"funding": {
|
||||||
@@ -784,9 +925,9 @@
|
|||||||
"license": "BSD-3-Clause"
|
"license": "BSD-3-Clause"
|
||||||
},
|
},
|
||||||
"node_modules/@rolldown/binding-android-arm64": {
|
"node_modules/@rolldown/binding-android-arm64": {
|
||||||
"version": "1.1.5",
|
"version": "1.2.4",
|
||||||
"resolved": "https://registry.npmjs.org/@rolldown/binding-android-arm64/-/binding-android-arm64-1.1.5.tgz",
|
"resolved": "https://registry.npmjs.org/@rolldown/binding-android-arm64/-/binding-android-arm64-1.2.4.tgz",
|
||||||
"integrity": "sha512-lZg8fqIv2v7FF237bwMgzGZEJvGL79/s5knJ/i6FmsGF4XXlzccZ4jb+TrFIxtSSxFtIpdsgrPZeMk1I9AFcyQ==",
|
"integrity": "sha512-jHC2cnyKz5xU2fhECtFl8OZ83cYNt13GZQD+0uMJ/X3o+ijmd56okHhTUwxVSHPx1IRVIJEZ1/1pPzeLCU6XKA==",
|
||||||
"cpu": [
|
"cpu": [
|
||||||
"arm64"
|
"arm64"
|
||||||
],
|
],
|
||||||
@@ -801,9 +942,9 @@
|
|||||||
}
|
}
|
||||||
},
|
},
|
||||||
"node_modules/@rolldown/binding-darwin-arm64": {
|
"node_modules/@rolldown/binding-darwin-arm64": {
|
||||||
"version": "1.1.5",
|
"version": "1.2.4",
|
||||||
"resolved": "https://registry.npmjs.org/@rolldown/binding-darwin-arm64/-/binding-darwin-arm64-1.1.5.tgz",
|
"resolved": "https://registry.npmjs.org/@rolldown/binding-darwin-arm64/-/binding-darwin-arm64-1.2.4.tgz",
|
||||||
"integrity": "sha512-51Bnx9pNiMRKSUNtBfySkNJ9vMU9Hh3I1ozDd6gyPPYzaXCfnptUcEZxXGYFn+ul2dtcMUiqGR1Yai2K10uoTw==",
|
"integrity": "sha512-Dc5mPD8F5F/FS8i01syd7FTF6yB2fVthH/TRkjwJkzUK6EpoxHtqvZQP5Zwq80/5z19TWYHIg1KOHboCgVx/aQ==",
|
||||||
"cpu": [
|
"cpu": [
|
||||||
"arm64"
|
"arm64"
|
||||||
],
|
],
|
||||||
@@ -818,9 +959,9 @@
|
|||||||
}
|
}
|
||||||
},
|
},
|
||||||
"node_modules/@rolldown/binding-darwin-x64": {
|
"node_modules/@rolldown/binding-darwin-x64": {
|
||||||
"version": "1.1.5",
|
"version": "1.2.4",
|
||||||
"resolved": "https://registry.npmjs.org/@rolldown/binding-darwin-x64/-/binding-darwin-x64-1.1.5.tgz",
|
"resolved": "https://registry.npmjs.org/@rolldown/binding-darwin-x64/-/binding-darwin-x64-1.2.4.tgz",
|
||||||
"integrity": "sha512-Tm+gbfC0aHu1tBA/JvKQh32S0K6YgCHkiAF4/W6xX0K0RmNuc94VeK419dJoE65R5aRxmo+noZQSWrAMF6yb6g==",
|
"integrity": "sha512-fpDm4oBo6SqLvWUYCmFhdde3U9KH2fRNNMeAnAPAIwxRL345xutL0EtEUcuoxsoazdJGv/MuDBQHlCDrtbvqOg==",
|
||||||
"cpu": [
|
"cpu": [
|
||||||
"x64"
|
"x64"
|
||||||
],
|
],
|
||||||
@@ -835,9 +976,9 @@
|
|||||||
}
|
}
|
||||||
},
|
},
|
||||||
"node_modules/@rolldown/binding-freebsd-x64": {
|
"node_modules/@rolldown/binding-freebsd-x64": {
|
||||||
"version": "1.1.5",
|
"version": "1.2.4",
|
||||||
"resolved": "https://registry.npmjs.org/@rolldown/binding-freebsd-x64/-/binding-freebsd-x64-1.1.5.tgz",
|
"resolved": "https://registry.npmjs.org/@rolldown/binding-freebsd-x64/-/binding-freebsd-x64-1.2.4.tgz",
|
||||||
"integrity": "sha512-JMzDKCCXq93YccG5gz3hvOs1oXRKAf0XYpfOS88e+wZrC8Iugj6j68867vrYZkvpDDpKn/KoKORThmchMpF6TA==",
|
"integrity": "sha512-rSJoreDE/HoIzoaib6MTp5jQtCTdMHKIvItAKT/ImS6Y6Ww76oUaeMyp4Vc/fAgd/ehji068IxetHXAnqUwN9A==",
|
||||||
"cpu": [
|
"cpu": [
|
||||||
"x64"
|
"x64"
|
||||||
],
|
],
|
||||||
@@ -852,9 +993,9 @@
|
|||||||
}
|
}
|
||||||
},
|
},
|
||||||
"node_modules/@rolldown/binding-linux-arm-gnueabihf": {
|
"node_modules/@rolldown/binding-linux-arm-gnueabihf": {
|
||||||
"version": "1.1.5",
|
"version": "1.2.4",
|
||||||
"resolved": "https://registry.npmjs.org/@rolldown/binding-linux-arm-gnueabihf/-/binding-linux-arm-gnueabihf-1.1.5.tgz",
|
"resolved": "https://registry.npmjs.org/@rolldown/binding-linux-arm-gnueabihf/-/binding-linux-arm-gnueabihf-1.2.4.tgz",
|
||||||
"integrity": "sha512-uML21j2K5TfPGutKxub+M+nLjZIrWjXQ5Grx4lCe/nimTj9B4L63zHpjXLl4y0L3mcm2htEQIb06oCG/szerNw==",
|
"integrity": "sha512-/jm8OGHgn7oGaJu3i/qZI9spUGcJ+y/lk43ttQ/iO1tOd9NissG6o97bighBCiL+BKRngmcDuR6ikfwYdJmVuQ==",
|
||||||
"cpu": [
|
"cpu": [
|
||||||
"arm"
|
"arm"
|
||||||
],
|
],
|
||||||
@@ -869,9 +1010,9 @@
|
|||||||
}
|
}
|
||||||
},
|
},
|
||||||
"node_modules/@rolldown/binding-linux-arm64-gnu": {
|
"node_modules/@rolldown/binding-linux-arm64-gnu": {
|
||||||
"version": "1.1.5",
|
"version": "1.2.4",
|
||||||
"resolved": "https://registry.npmjs.org/@rolldown/binding-linux-arm64-gnu/-/binding-linux-arm64-gnu-1.1.5.tgz",
|
"resolved": "https://registry.npmjs.org/@rolldown/binding-linux-arm64-gnu/-/binding-linux-arm64-gnu-1.2.4.tgz",
|
||||||
"integrity": "sha512-navSiuTMogvnQoZoM/v+l3ZWo50/NTwSHSzheABx/RCnmUPaKwq9qSo4Br2OYRs21+Fz8uFqITZM3H4opOB0/Q==",
|
"integrity": "sha512-tIP06BeD9EqvECBrPZ+sqdPlYrT+aYaAiu1wYziVx5elRK/ftm33JxVDy2bXGbr6J0CrtirCkR87/X5a2euEng==",
|
||||||
"cpu": [
|
"cpu": [
|
||||||
"arm64"
|
"arm64"
|
||||||
],
|
],
|
||||||
@@ -889,9 +1030,9 @@
|
|||||||
}
|
}
|
||||||
},
|
},
|
||||||
"node_modules/@rolldown/binding-linux-arm64-musl": {
|
"node_modules/@rolldown/binding-linux-arm64-musl": {
|
||||||
"version": "1.1.5",
|
"version": "1.2.4",
|
||||||
"resolved": "https://registry.npmjs.org/@rolldown/binding-linux-arm64-musl/-/binding-linux-arm64-musl-1.1.5.tgz",
|
"resolved": "https://registry.npmjs.org/@rolldown/binding-linux-arm64-musl/-/binding-linux-arm64-musl-1.2.4.tgz",
|
||||||
"integrity": "sha512-lAryqH7IteztmCXQXk0etKj4wBQ7Gx5S6LjKhsgp9zb8I5bsuvU/2llH1hDQcjsFeqIsovMVN339/8pUDDBXxA==",
|
"integrity": "sha512-Ql1Q0EQqVThvn9VAVlwNzsUvbSFtCMGjLpRRi4pk5i7NZZ4n5ISiLMjHYtus4VQ2PvkSw24zyaCVsiS+sXPj1w==",
|
||||||
"cpu": [
|
"cpu": [
|
||||||
"arm64"
|
"arm64"
|
||||||
],
|
],
|
||||||
@@ -909,9 +1050,9 @@
|
|||||||
}
|
}
|
||||||
},
|
},
|
||||||
"node_modules/@rolldown/binding-linux-ppc64-gnu": {
|
"node_modules/@rolldown/binding-linux-ppc64-gnu": {
|
||||||
"version": "1.1.5",
|
"version": "1.2.4",
|
||||||
"resolved": "https://registry.npmjs.org/@rolldown/binding-linux-ppc64-gnu/-/binding-linux-ppc64-gnu-1.1.5.tgz",
|
"resolved": "https://registry.npmjs.org/@rolldown/binding-linux-ppc64-gnu/-/binding-linux-ppc64-gnu-1.2.4.tgz",
|
||||||
"integrity": "sha512-fsK/sNBnxzBlL4O1JNrZakVQxPspqpED5dLtNsZS9oOKmtSpdNIzxH2kkol5HYTWJN47sE20ztMJPxfZ89qGOg==",
|
"integrity": "sha512-GjbjXD4XXfN19D0LZNbmiCBUoDiRACsYHr0yaIbbn8aFsXjHZifcYqu/W5Er5X2X990WjHXFrxarn5chzItorQ==",
|
||||||
"cpu": [
|
"cpu": [
|
||||||
"ppc64"
|
"ppc64"
|
||||||
],
|
],
|
||||||
@@ -929,9 +1070,9 @@
|
|||||||
}
|
}
|
||||||
},
|
},
|
||||||
"node_modules/@rolldown/binding-linux-s390x-gnu": {
|
"node_modules/@rolldown/binding-linux-s390x-gnu": {
|
||||||
"version": "1.1.5",
|
"version": "1.2.4",
|
||||||
"resolved": "https://registry.npmjs.org/@rolldown/binding-linux-s390x-gnu/-/binding-linux-s390x-gnu-1.1.5.tgz",
|
"resolved": "https://registry.npmjs.org/@rolldown/binding-linux-s390x-gnu/-/binding-linux-s390x-gnu-1.2.4.tgz",
|
||||||
"integrity": "sha512-gLYb4BIadlfTOYT5gO503n8zQjXflgzpD0FcyKh0Mzx3rqCZKnHoJWV9xe1KXUJ5lx2JfcSHr/mhzS0PC/McAA==",
|
"integrity": "sha512-p5WR0NOwaRmJ/B1b6IjEFLLivwEsf3PrdBIhRbhTCQisbo2SvHHpG4ELB/+FgQNnB88LTOF86upmJmbvZdQ2lw==",
|
||||||
"cpu": [
|
"cpu": [
|
||||||
"s390x"
|
"s390x"
|
||||||
],
|
],
|
||||||
@@ -949,9 +1090,9 @@
|
|||||||
}
|
}
|
||||||
},
|
},
|
||||||
"node_modules/@rolldown/binding-linux-x64-gnu": {
|
"node_modules/@rolldown/binding-linux-x64-gnu": {
|
||||||
"version": "1.1.5",
|
"version": "1.2.4",
|
||||||
"resolved": "https://registry.npmjs.org/@rolldown/binding-linux-x64-gnu/-/binding-linux-x64-gnu-1.1.5.tgz",
|
"resolved": "https://registry.npmjs.org/@rolldown/binding-linux-x64-gnu/-/binding-linux-x64-gnu-1.2.4.tgz",
|
||||||
"integrity": "sha512-FjcpEKUyJygHgs1o50VYNvkt5+7Le/VEdYt0AkRpkL33MnyQfwr8l5mXwMmfmTbyMPr5vJLC+8/Gd9gXnwU1QQ==",
|
"integrity": "sha512-4/GyVjmhR+Tc6HLJvwc1sOhPqAZtySiSMesOZyX6JQ5XBxoTDEMKQzvo07NIK6nTon/SivlZqvhzvuVBNQhObQ==",
|
||||||
"cpu": [
|
"cpu": [
|
||||||
"x64"
|
"x64"
|
||||||
],
|
],
|
||||||
@@ -969,9 +1110,9 @@
|
|||||||
}
|
}
|
||||||
},
|
},
|
||||||
"node_modules/@rolldown/binding-linux-x64-musl": {
|
"node_modules/@rolldown/binding-linux-x64-musl": {
|
||||||
"version": "1.1.5",
|
"version": "1.2.4",
|
||||||
"resolved": "https://registry.npmjs.org/@rolldown/binding-linux-x64-musl/-/binding-linux-x64-musl-1.1.5.tgz",
|
"resolved": "https://registry.npmjs.org/@rolldown/binding-linux-x64-musl/-/binding-linux-x64-musl-1.2.4.tgz",
|
||||||
"integrity": "sha512-Me+PfPI2TMeOQk0gYWfLQZtTktrmzbr8cDboqX83XKc7UrgAi55gF+2dUkWdxd19n55Essp2yeca+O9N5rBxHg==",
|
"integrity": "sha512-l9eeLsCNvPpmSXUej0etw/J1eqV0Jj1D5G/xG6YTijmE6dkv6E2QezgWbTfQk63v952DPqrjOCoiqxq7Bw0YUQ==",
|
||||||
"cpu": [
|
"cpu": [
|
||||||
"x64"
|
"x64"
|
||||||
],
|
],
|
||||||
@@ -989,9 +1130,9 @@
|
|||||||
}
|
}
|
||||||
},
|
},
|
||||||
"node_modules/@rolldown/binding-openharmony-arm64": {
|
"node_modules/@rolldown/binding-openharmony-arm64": {
|
||||||
"version": "1.1.5",
|
"version": "1.2.4",
|
||||||
"resolved": "https://registry.npmjs.org/@rolldown/binding-openharmony-arm64/-/binding-openharmony-arm64-1.1.5.tgz",
|
"resolved": "https://registry.npmjs.org/@rolldown/binding-openharmony-arm64/-/binding-openharmony-arm64-1.2.4.tgz",
|
||||||
"integrity": "sha512-yc5WrLzXks6zCQfn9Oxr8pORKyl/pF+QjHmW/Qx3qu0oyrrNC+y2JLTU1E2rcWYAmzlnqngWXHQjy51VzW70Vw==",
|
"integrity": "sha512-e0F355MSTMm3+UOqtV3L24gFUp2N5m1f8L/7d56deik6va+AXdrt9F8LbzGpeWGWRbZEDq4m8NVnJDeBtf9DZg==",
|
||||||
"cpu": [
|
"cpu": [
|
||||||
"arm64"
|
"arm64"
|
||||||
],
|
],
|
||||||
@@ -1005,63 +1146,10 @@
|
|||||||
"node": "^20.19.0 || >=22.12.0"
|
"node": "^20.19.0 || >=22.12.0"
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
"node_modules/@rolldown/binding-wasm32-wasi": {
|
|
||||||
"version": "1.1.5",
|
|
||||||
"resolved": "https://registry.npmjs.org/@rolldown/binding-wasm32-wasi/-/binding-wasm32-wasi-1.1.5.tgz",
|
|
||||||
"integrity": "sha512-VbQGPX2b4r48TAMIM2cjgluIM1HYutm4pcTEJsle7iEP7sB1dFqtPLBVbdLAZCxy1txCcPxf4QFf4v8uvltPqA==",
|
|
||||||
"cpu": [
|
|
||||||
"wasm32"
|
|
||||||
],
|
|
||||||
"dev": true,
|
|
||||||
"license": "MIT",
|
|
||||||
"optional": true,
|
|
||||||
"dependencies": {
|
|
||||||
"@emnapi/core": "1.11.1",
|
|
||||||
"@emnapi/runtime": "1.11.1",
|
|
||||||
"@napi-rs/wasm-runtime": "^1.1.6"
|
|
||||||
},
|
|
||||||
"engines": {
|
|
||||||
"node": "^20.19.0 || >=22.12.0"
|
|
||||||
}
|
|
||||||
},
|
|
||||||
"node_modules/@rolldown/binding-wasm32-wasi/node_modules/@emnapi/core": {
|
|
||||||
"version": "1.11.1",
|
|
||||||
"resolved": "https://registry.npmjs.org/@emnapi/core/-/core-1.11.1.tgz",
|
|
||||||
"integrity": "sha512-RSvbQmHzdKzNsLYa/wHrbc3KN4sYLKAdPZxqiM2HATqv/SBk2/ENSHpvXGaLOMcsAyz0poEGqkmmKYG3OWiJEQ==",
|
|
||||||
"dev": true,
|
|
||||||
"license": "MIT",
|
|
||||||
"optional": true,
|
|
||||||
"dependencies": {
|
|
||||||
"@emnapi/wasi-threads": "1.2.2",
|
|
||||||
"tslib": "^2.4.0"
|
|
||||||
}
|
|
||||||
},
|
|
||||||
"node_modules/@rolldown/binding-wasm32-wasi/node_modules/@emnapi/runtime": {
|
|
||||||
"version": "1.11.1",
|
|
||||||
"resolved": "https://registry.npmjs.org/@emnapi/runtime/-/runtime-1.11.1.tgz",
|
|
||||||
"integrity": "sha512-vgj7R3y3Wgx24IQaGPA/R6YFXLHVMOZ0uVEyIQPaWs+rd1AzfEMXlAC22FYwO1XkKR6NPsq7mUandH8oIRdZFw==",
|
|
||||||
"dev": true,
|
|
||||||
"license": "MIT",
|
|
||||||
"optional": true,
|
|
||||||
"dependencies": {
|
|
||||||
"tslib": "^2.4.0"
|
|
||||||
}
|
|
||||||
},
|
|
||||||
"node_modules/@rolldown/binding-wasm32-wasi/node_modules/@emnapi/wasi-threads": {
|
|
||||||
"version": "1.2.2",
|
|
||||||
"resolved": "https://registry.npmjs.org/@emnapi/wasi-threads/-/wasi-threads-1.2.2.tgz",
|
|
||||||
"integrity": "sha512-c95qOXkHdydNKhscBTebqEC1CVAZpyqOfVfBzQ1qgzyl3gfeldUjIggDbIZgDKsHLgnsM+igH7TJ/eAasaVuMA==",
|
|
||||||
"dev": true,
|
|
||||||
"license": "MIT",
|
|
||||||
"optional": true,
|
|
||||||
"dependencies": {
|
|
||||||
"tslib": "^2.4.0"
|
|
||||||
}
|
|
||||||
},
|
|
||||||
"node_modules/@rolldown/binding-win32-arm64-msvc": {
|
"node_modules/@rolldown/binding-win32-arm64-msvc": {
|
||||||
"version": "1.1.5",
|
"version": "1.2.4",
|
||||||
"resolved": "https://registry.npmjs.org/@rolldown/binding-win32-arm64-msvc/-/binding-win32-arm64-msvc-1.1.5.tgz",
|
"resolved": "https://registry.npmjs.org/@rolldown/binding-win32-arm64-msvc/-/binding-win32-arm64-msvc-1.2.4.tgz",
|
||||||
"integrity": "sha512-gHv82k63z4qpV5+Q1y/12KrK0ltWBukVDI8nZcbT7Tt/ZlOIVwppazneq0F93oDxTo3IgAMEDIoQh3E2n6mVsw==",
|
"integrity": "sha512-AWLi0uBRYh6QlE7OKhiz+phZC0qwtij2QZmhmOdsLdFn64m7oMpooE9ICE3lhm9xMb4SpDo2WbHcxX1iFLFtqw==",
|
||||||
"cpu": [
|
"cpu": [
|
||||||
"arm64"
|
"arm64"
|
||||||
],
|
],
|
||||||
@@ -1076,9 +1164,9 @@
|
|||||||
}
|
}
|
||||||
},
|
},
|
||||||
"node_modules/@rolldown/binding-win32-x64-msvc": {
|
"node_modules/@rolldown/binding-win32-x64-msvc": {
|
||||||
"version": "1.1.5",
|
"version": "1.2.4",
|
||||||
"resolved": "https://registry.npmjs.org/@rolldown/binding-win32-x64-msvc/-/binding-win32-x64-msvc-1.1.5.tgz",
|
"resolved": "https://registry.npmjs.org/@rolldown/binding-win32-x64-msvc/-/binding-win32-x64-msvc-1.2.4.tgz",
|
||||||
"integrity": "sha512-tTZuDBPw85tEN5PQi1pnEBzDy0Z49HtScLAbD5t6hyeU92A95pRWaSMw1GZZi/RwgSgUIl0xrSlXIT/9QzvYSA==",
|
"integrity": "sha512-UwSDJOg3dqCAejWdxclJjCsh3Qq4vLYMDxmyHqo1btz3stK2VqgwNd3mm5tuIwzSlGIQ/1H9Hr+Zn09mrezNqQ==",
|
||||||
"cpu": [
|
"cpu": [
|
||||||
"x64"
|
"x64"
|
||||||
],
|
],
|
||||||
@@ -1317,17 +1405,6 @@
|
|||||||
"@tensorflow/tfjs-core": "4.22.0"
|
"@tensorflow/tfjs-core": "4.22.0"
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
"node_modules/@tybys/wasm-util": {
|
|
||||||
"version": "0.10.3",
|
|
||||||
"resolved": "https://registry.npmjs.org/@tybys/wasm-util/-/wasm-util-0.10.3.tgz",
|
|
||||||
"integrity": "sha512-F3fo1MYrRJYL3zER0OUOmkutjr1Vp23m7OsSgp7nq4SP6OqX6C/56XFIPAl5bt3zaBRjmW7SGz3u/6LwFpYcOg==",
|
|
||||||
"dev": true,
|
|
||||||
"license": "MIT",
|
|
||||||
"optional": true,
|
|
||||||
"dependencies": {
|
|
||||||
"tslib": "^2.4.0"
|
|
||||||
}
|
|
||||||
},
|
|
||||||
"node_modules/@types/estree": {
|
"node_modules/@types/estree": {
|
||||||
"version": "1.0.9",
|
"version": "1.0.9",
|
||||||
"resolved": "https://registry.npmjs.org/@types/estree/-/estree-1.0.9.tgz",
|
"resolved": "https://registry.npmjs.org/@types/estree/-/estree-1.0.9.tgz",
|
||||||
@@ -1544,9 +1621,9 @@
|
|||||||
"license": "MIT"
|
"license": "MIT"
|
||||||
},
|
},
|
||||||
"node_modules/brace-expansion": {
|
"node_modules/brace-expansion": {
|
||||||
"version": "2.1.3",
|
"version": "2.1.4",
|
||||||
"resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-2.1.3.tgz",
|
"resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-2.1.4.tgz",
|
||||||
"integrity": "sha512-DRdx5neNsG/QXbniLFWi2YmC/68oeOOmKz6zOjVk6ZS1ZLXgLIKqVEc6hWsmkjBbgii0SwaBTcJ5XKj5gzY/4A==",
|
"integrity": "sha512-hGfVzPxthbf3+2yjg/RBs60cB0FhqBS/zvdV/4wn4/BmN0bNMMHPc4V/BbFieqf1TKAGGAHnY4eSjajCl0f2Xg==",
|
||||||
"dev": true,
|
"dev": true,
|
||||||
"license": "MIT",
|
"license": "MIT",
|
||||||
"dependencies": {
|
"dependencies": {
|
||||||
@@ -3011,9 +3088,9 @@
|
|||||||
"license": "MIT"
|
"license": "MIT"
|
||||||
},
|
},
|
||||||
"node_modules/nanoid": {
|
"node_modules/nanoid": {
|
||||||
"version": "3.3.16",
|
"version": "3.3.18",
|
||||||
"resolved": "https://registry.npmjs.org/nanoid/-/nanoid-3.3.16.tgz",
|
"resolved": "https://registry.npmjs.org/nanoid/-/nanoid-3.3.18.tgz",
|
||||||
"integrity": "sha512-bzlKTyNJ7+LdGIIwy8ijFpIqEQIvafahV7eYykJ8Cvh42EdJeODoJ6gUJXpQJvej1BddH8OqTXZNE/KfbWAu8Q==",
|
"integrity": "sha512-DTg4MJbGMWkfi6VZFdNt2/caMbQy4Ou+Op/hJQvGEWcnVfoA1QA+xzRKAzw9jD6+GVOOeYr/mIcuDSdug6F6+w==",
|
||||||
"dev": true,
|
"dev": true,
|
||||||
"funding": [
|
"funding": [
|
||||||
{
|
{
|
||||||
@@ -3233,6 +3310,38 @@
|
|||||||
"dev": true,
|
"dev": true,
|
||||||
"license": "MIT"
|
"license": "MIT"
|
||||||
},
|
},
|
||||||
|
"node_modules/pdf-parse": {
|
||||||
|
"version": "2.4.5",
|
||||||
|
"resolved": "https://registry.npmjs.org/pdf-parse/-/pdf-parse-2.4.5.tgz",
|
||||||
|
"integrity": "sha512-mHU89HGh7v+4u2ubfnevJ03lmPgQ5WU4CxAVmTSh/sxVTEDYd1er/dKS/A6vg77NX47KTEoihq8jZBLr8Cxuwg==",
|
||||||
|
"license": "Apache-2.0",
|
||||||
|
"dependencies": {
|
||||||
|
"@napi-rs/canvas": "0.1.80",
|
||||||
|
"pdfjs-dist": "5.4.296"
|
||||||
|
},
|
||||||
|
"bin": {
|
||||||
|
"pdf-parse": "bin/cli.mjs"
|
||||||
|
},
|
||||||
|
"engines": {
|
||||||
|
"node": ">=20.16.0 <21 || >=22.3.0"
|
||||||
|
},
|
||||||
|
"funding": {
|
||||||
|
"type": "github",
|
||||||
|
"url": "https://github.com/sponsors/mehmet-kozan"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"node_modules/pdfjs-dist": {
|
||||||
|
"version": "5.4.296",
|
||||||
|
"resolved": "https://registry.npmjs.org/pdfjs-dist/-/pdfjs-dist-5.4.296.tgz",
|
||||||
|
"integrity": "sha512-DlOzet0HO7OEnmUmB6wWGJrrdvbyJKftI1bhMitK7O2N8W2gc757yyYBbINy9IDafXAV9wmKr9t7xsTaNKRG5Q==",
|
||||||
|
"license": "Apache-2.0",
|
||||||
|
"engines": {
|
||||||
|
"node": ">=20.16.0 || >=22.3.0"
|
||||||
|
},
|
||||||
|
"optionalDependencies": {
|
||||||
|
"@napi-rs/canvas": "^0.1.80"
|
||||||
|
}
|
||||||
|
},
|
||||||
"node_modules/picocolors": {
|
"node_modules/picocolors": {
|
||||||
"version": "1.1.1",
|
"version": "1.1.1",
|
||||||
"resolved": "https://registry.npmjs.org/picocolors/-/picocolors-1.1.1.tgz",
|
"resolved": "https://registry.npmjs.org/picocolors/-/picocolors-1.1.1.tgz",
|
||||||
@@ -3272,9 +3381,9 @@
|
|||||||
"license": "MIT"
|
"license": "MIT"
|
||||||
},
|
},
|
||||||
"node_modules/postcss": {
|
"node_modules/postcss": {
|
||||||
"version": "8.5.25",
|
"version": "8.5.26",
|
||||||
"resolved": "https://registry.npmjs.org/postcss/-/postcss-8.5.25.tgz",
|
"resolved": "https://registry.npmjs.org/postcss/-/postcss-8.5.26.tgz",
|
||||||
"integrity": "sha512-DTPx3RWSSnWyzLxQnlH0rJP+EW5ekl16ZU4/psbIhA0e53kJfdgaN5vKM+xP7yJtXVu+nfdVFmlgFDEKAe4Pyw==",
|
"integrity": "sha512-u82N74LFzG8ca+dD8puPnplTXoGH4fTPpVGuIbt36G3qvNlkvfD0lEAZSxaly3KX8TS/L1A1gsCEmvKmBcVbkQ==",
|
||||||
"dev": true,
|
"dev": true,
|
||||||
"funding": [
|
"funding": [
|
||||||
{
|
{
|
||||||
@@ -3292,7 +3401,7 @@
|
|||||||
],
|
],
|
||||||
"license": "MIT",
|
"license": "MIT",
|
||||||
"dependencies": {
|
"dependencies": {
|
||||||
"nanoid": "^3.3.16",
|
"nanoid": "^3.3.17",
|
||||||
"picocolors": "^1.1.1",
|
"picocolors": "^1.1.1",
|
||||||
"source-map-js": "^1.2.1"
|
"source-map-js": "^1.2.1"
|
||||||
},
|
},
|
||||||
@@ -3434,13 +3543,13 @@
|
|||||||
"license": "BSD-3-Clause"
|
"license": "BSD-3-Clause"
|
||||||
},
|
},
|
||||||
"node_modules/rolldown": {
|
"node_modules/rolldown": {
|
||||||
"version": "1.1.5",
|
"version": "1.2.4",
|
||||||
"resolved": "https://registry.npmjs.org/rolldown/-/rolldown-1.1.5.tgz",
|
"resolved": "https://registry.npmjs.org/rolldown/-/rolldown-1.2.4.tgz",
|
||||||
"integrity": "sha512-t9z29cJjXf/vxQ8dyhCSpt6H6aSwHTk8cT5I3iy6SMXuFpk5mB6PL6XfC8PCwrPTx93udwKUm9HRteAlTGBLiA==",
|
"integrity": "sha512-rSr7irW0K7QRWzjdJXqZowkcRdDtjRduh43rBltnVKd0VFq839l1lJoDvGJb6gl7+4rTTCrPWu+YfujUL8Ug7w==",
|
||||||
"dev": true,
|
"dev": true,
|
||||||
"license": "MIT",
|
"license": "MIT",
|
||||||
"dependencies": {
|
"dependencies": {
|
||||||
"@oxc-project/types": "=0.139.0",
|
"@oxc-project/types": "=0.144.0",
|
||||||
"@rolldown/pluginutils": "^1.0.0"
|
"@rolldown/pluginutils": "^1.0.0"
|
||||||
},
|
},
|
||||||
"bin": {
|
"bin": {
|
||||||
@@ -3450,21 +3559,20 @@
|
|||||||
"node": "^20.19.0 || >=22.12.0"
|
"node": "^20.19.0 || >=22.12.0"
|
||||||
},
|
},
|
||||||
"optionalDependencies": {
|
"optionalDependencies": {
|
||||||
"@rolldown/binding-android-arm64": "1.1.5",
|
"@rolldown/binding-android-arm64": "1.2.4",
|
||||||
"@rolldown/binding-darwin-arm64": "1.1.5",
|
"@rolldown/binding-darwin-arm64": "1.2.4",
|
||||||
"@rolldown/binding-darwin-x64": "1.1.5",
|
"@rolldown/binding-darwin-x64": "1.2.4",
|
||||||
"@rolldown/binding-freebsd-x64": "1.1.5",
|
"@rolldown/binding-freebsd-x64": "1.2.4",
|
||||||
"@rolldown/binding-linux-arm-gnueabihf": "1.1.5",
|
"@rolldown/binding-linux-arm-gnueabihf": "1.2.4",
|
||||||
"@rolldown/binding-linux-arm64-gnu": "1.1.5",
|
"@rolldown/binding-linux-arm64-gnu": "1.2.4",
|
||||||
"@rolldown/binding-linux-arm64-musl": "1.1.5",
|
"@rolldown/binding-linux-arm64-musl": "1.2.4",
|
||||||
"@rolldown/binding-linux-ppc64-gnu": "1.1.5",
|
"@rolldown/binding-linux-ppc64-gnu": "1.2.4",
|
||||||
"@rolldown/binding-linux-s390x-gnu": "1.1.5",
|
"@rolldown/binding-linux-s390x-gnu": "1.2.4",
|
||||||
"@rolldown/binding-linux-x64-gnu": "1.1.5",
|
"@rolldown/binding-linux-x64-gnu": "1.2.4",
|
||||||
"@rolldown/binding-linux-x64-musl": "1.1.5",
|
"@rolldown/binding-linux-x64-musl": "1.2.4",
|
||||||
"@rolldown/binding-openharmony-arm64": "1.1.5",
|
"@rolldown/binding-openharmony-arm64": "1.2.4",
|
||||||
"@rolldown/binding-wasm32-wasi": "1.1.5",
|
"@rolldown/binding-win32-arm64-msvc": "1.2.4",
|
||||||
"@rolldown/binding-win32-arm64-msvc": "1.1.5",
|
"@rolldown/binding-win32-x64-msvc": "1.2.4"
|
||||||
"@rolldown/binding-win32-x64-msvc": "1.1.5"
|
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
"node_modules/safe-buffer": {
|
"node_modules/safe-buffer": {
|
||||||
@@ -4021,16 +4129,16 @@
|
|||||||
}
|
}
|
||||||
},
|
},
|
||||||
"node_modules/vite": {
|
"node_modules/vite": {
|
||||||
"version": "8.1.5",
|
"version": "8.2.1",
|
||||||
"resolved": "https://registry.npmjs.org/vite/-/vite-8.1.5.tgz",
|
"resolved": "https://registry.npmjs.org/vite/-/vite-8.2.1.tgz",
|
||||||
"integrity": "sha512-7ULLwsCdYx/nRyrpiEwvqb5TFHrMVZyBt+rg/OAXT7rgj/z+DtTDyKFeLAdDkubDVDKD8jOsndmy7m55XcfUsw==",
|
"integrity": "sha512-EU/eS7BH3XROHh2YnBefjM6DBKA6ZeMZEYQbj7NLWg5wHYlhB8B/Mayd5XsgWq+NFYccDOTemRpdETWR6Ka/lw==",
|
||||||
"dev": true,
|
"dev": true,
|
||||||
"license": "MIT",
|
"license": "MIT",
|
||||||
"dependencies": {
|
"dependencies": {
|
||||||
"lightningcss": "^1.32.0",
|
"lightningcss": "^1.33.0",
|
||||||
"picomatch": "^4.0.5",
|
"picomatch": "^4.0.5",
|
||||||
"postcss": "^8.5.17",
|
"postcss": "^8.5.25",
|
||||||
"rolldown": "~1.1.5",
|
"rolldown": "~1.2.1",
|
||||||
"tinyglobby": "^0.2.17"
|
"tinyglobby": "^0.2.17"
|
||||||
},
|
},
|
||||||
"bin": {
|
"bin": {
|
||||||
@@ -4047,7 +4155,7 @@
|
|||||||
},
|
},
|
||||||
"peerDependencies": {
|
"peerDependencies": {
|
||||||
"@types/node": "^20.19.0 || >=22.12.0",
|
"@types/node": "^20.19.0 || >=22.12.0",
|
||||||
"@vitejs/devtools": "^0.3.0",
|
"@vitejs/devtools": "^0.4.0",
|
||||||
"esbuild": "^0.27.0 || ^0.28.0",
|
"esbuild": "^0.27.0 || ^0.28.0",
|
||||||
"jiti": ">=1.21.0",
|
"jiti": ">=1.21.0",
|
||||||
"less": "^4.0.0",
|
"less": "^4.0.0",
|
||||||
@@ -4132,9 +4240,9 @@
|
|||||||
"license": "MIT"
|
"license": "MIT"
|
||||||
},
|
},
|
||||||
"node_modules/wasm-feature-detect": {
|
"node_modules/wasm-feature-detect": {
|
||||||
"version": "1.8.0",
|
"version": "1.9.0",
|
||||||
"resolved": "https://registry.npmjs.org/wasm-feature-detect/-/wasm-feature-detect-1.8.0.tgz",
|
"resolved": "https://registry.npmjs.org/wasm-feature-detect/-/wasm-feature-detect-1.9.0.tgz",
|
||||||
"integrity": "sha512-zksaLKM2fVlnB5jQQDqKXXwYHLQUVH9es+5TOOHwGOVJOCeRBCiPjwSg+3tN2AdTCzjgli4jijCH290kXb/zWQ==",
|
"integrity": "sha512-zonE+xlIIYtxPy++L24ow0hAD8CICb4+FgPyROd3buyXIqsJvUEDkBgfCCoXOd1Hu3DUr0GOfnPIdcGV+YpNaA==",
|
||||||
"license": "Apache-2.0"
|
"license": "Apache-2.0"
|
||||||
},
|
},
|
||||||
"node_modules/webidl-conversions": {
|
"node_modules/webidl-conversions": {
|
||||||
|
|||||||
@@ -1,6 +1,6 @@
|
|||||||
{
|
{
|
||||||
"name": "@ztimson/ai-utils",
|
"name": "@ztimson/ai-utils",
|
||||||
"version": "1.4.5",
|
"version": "1.6.4",
|
||||||
"description": "AI Utility library",
|
"description": "AI Utility library",
|
||||||
"author": "Zak Timson",
|
"author": "Zak Timson",
|
||||||
"license": "MIT",
|
"license": "MIT",
|
||||||
@@ -26,12 +26,13 @@
|
|||||||
},
|
},
|
||||||
"dependencies": {
|
"dependencies": {
|
||||||
"@anthropic-ai/sdk": "^0.102.0",
|
"@anthropic-ai/sdk": "^0.102.0",
|
||||||
"@tensorflow/tfjs": "^4.22.0",
|
|
||||||
"@huggingface/transformers": "^4.2.0",
|
"@huggingface/transformers": "^4.2.0",
|
||||||
|
"@tensorflow/tfjs": "^4.22.0",
|
||||||
"@ztimson/node-utils": "^1.0.7",
|
"@ztimson/node-utils": "^1.0.7",
|
||||||
"@ztimson/utils": "^0.29.4",
|
"@ztimson/utils": "^0.29.4",
|
||||||
"cheerio": "^1.2.0",
|
"cheerio": "^1.2.0",
|
||||||
"openai": "^6.42.0",
|
"openai": "^6.42.0",
|
||||||
|
"pdf-parse": "^2.4.5",
|
||||||
"tesseract.js": "^7.0.0"
|
"tesseract.js": "^7.0.0"
|
||||||
},
|
},
|
||||||
"devDependencies": {
|
"devDependencies": {
|
||||||
|
|||||||
@@ -24,6 +24,13 @@ export class Anthropic extends LLMProvider {
|
|||||||
return client;
|
return client;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
private toWireContent(content: any): any {
|
||||||
|
if(!Array.isArray(content)) return content;
|
||||||
|
return content.map(c => c.type === 'image'
|
||||||
|
? {type: 'image', source: {type: 'base64', media_type: c.mime, data: c.data}}
|
||||||
|
: {type: 'text', text: c.text});
|
||||||
|
}
|
||||||
|
|
||||||
/** Convert standard history -> Anthropic wire format */
|
/** Convert standard history -> Anthropic wire format */
|
||||||
private toWire(history: LLMMessage[]): any[] {
|
private toWire(history: LLMMessage[]): any[] {
|
||||||
const wire: any[] = [];
|
const wire: any[] = [];
|
||||||
@@ -34,7 +41,7 @@ export class Anthropic extends LLMProvider {
|
|||||||
{role: 'user', content: [{type: 'tool_result', tool_use_id: h.id, is_error: !!h.error, content: h.error || h.content || ''}]}
|
{role: 'user', content: [{type: 'tool_result', tool_use_id: h.id, is_error: !!h.error, content: h.error || h.content || ''}]}
|
||||||
);
|
);
|
||||||
} else {
|
} else {
|
||||||
wire.push({role: h.role, content: h.content});
|
wire.push({role: h.role, content: this.toWireContent(h.content)});
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
return wire;
|
return wire;
|
||||||
@@ -50,7 +57,7 @@ export class Anthropic extends LLMProvider {
|
|||||||
const tools = options.tools || this.ai.options.llm?.tools || [];
|
const tools = options.tools || this.ai.options.llm?.tools || [];
|
||||||
const requestParams: any = {
|
const requestParams: any = {
|
||||||
model: options.model || this.model,
|
model: options.model || this.model,
|
||||||
max_tokens: options.max_tokens || this.ai.options.llm?.max_tokens || 4096,
|
max_tokens: options.maxTokens || this.ai.options.llm?.maxTokens || 4096,
|
||||||
system: options.system || this.ai.options.llm?.system || '',
|
system: options.system || this.ai.options.llm?.system || '',
|
||||||
temperature: options.temperature || this.ai.options.llm?.temperature || undefined,
|
temperature: options.temperature || this.ai.options.llm?.temperature || undefined,
|
||||||
tools: tools.map(t => ({
|
tools: tools.map(t => ({
|
||||||
|
|||||||
193
src/llm.ts
193
src/llm.ts
@@ -1,15 +1,20 @@
|
|||||||
import {clean, snakeCase} from '@ztimson/utils';
|
import {clean, makeUnique, snakeCase} from '@ztimson/utils';
|
||||||
import {AbortablePromise, Ai} from './ai.ts';
|
import {AbortablePromise, Ai} from './ai.ts';
|
||||||
import {Anthropic} from './antrhopic.ts';
|
import {Anthropic} from './antrhopic.ts';
|
||||||
import {OpenAi} from './open-ai.ts';
|
import {OpenAi} from './open-ai.ts';
|
||||||
import {LLMProvider} from './provider.ts';
|
import {LLMProvider} from './provider.ts';
|
||||||
import {AiTool, AiToolArg} from './tools.ts';
|
import {AiTool, AiToolArg} from './tools.ts';
|
||||||
import {fileURLToPath} from 'url';
|
import {fileURLToPath} from 'url';
|
||||||
import {dirname, join} from 'path';
|
|
||||||
import {spawn} from 'node:child_process';
|
import {spawn} from 'node:child_process';
|
||||||
import {Memory, MemoryCache, MemoryManager, MemoryOptions} from './memory.ts';
|
import {Memory, MemoryCache, MemoryManager, MemoryOptions, stripHeader} from './memory.ts';
|
||||||
|
import {mkdtempSync} from 'node:fs';
|
||||||
|
import fs from 'node:fs/promises';
|
||||||
|
import {tmpdir} from 'node:os';
|
||||||
|
import {dirname, join, basename, extname} from 'path';
|
||||||
|
import { PDFParse } from 'pdf-parse';
|
||||||
|
|
||||||
const MAX_AGENT_DEPTH = 5;
|
const MAX_AGENT_DEPTH = 5;
|
||||||
|
const PDF_OCR_PAGE_THRESHOLD = 12; // above this many pages, OCR scanned pages instead of feeding images to the model
|
||||||
|
|
||||||
export type AnthropicConfig = {proto: 'anthropic', token: string | string[]};
|
export type AnthropicConfig = {proto: 'anthropic', token: string | string[]};
|
||||||
export type OpenAiConfig = {proto: 'openai', host?: string, token: string | string[]};
|
export type OpenAiConfig = {proto: 'openai', host?: string, token: string | string[]};
|
||||||
@@ -27,11 +32,26 @@ export type Agent = {
|
|||||||
agents?: string[] | null;
|
agents?: string[] | null;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
export type LLMFile = {
|
||||||
|
/** Path to file on disk */
|
||||||
|
path?: string;
|
||||||
|
/** File content: raw text, base64-encoded binary, or a Buffer */
|
||||||
|
content?: string | Buffer;
|
||||||
|
/** Original filename, used to infer type from extension */
|
||||||
|
name?: string;
|
||||||
|
/** Mime type override, inferred from extension if omitted */
|
||||||
|
mime?: string;
|
||||||
|
/** @internal set once extraction has run, skips re-processing next turn */
|
||||||
|
extracted?: boolean;
|
||||||
|
};
|
||||||
|
|
||||||
export type LLMMessage = {
|
export type LLMMessage = {
|
||||||
/** Message originator */
|
/** Message originator */
|
||||||
role: 'assistant' | 'system' | 'user';
|
role: 'assistant' | 'system' | 'user';
|
||||||
/** Message content */
|
/** Message content */
|
||||||
content: string | any;
|
content: string | any;
|
||||||
|
/** Files attached to request */
|
||||||
|
files?: LLMFile[];
|
||||||
/** Timestamp */
|
/** Timestamp */
|
||||||
timestamp?: number;
|
timestamp?: number;
|
||||||
/** Response duration in ms */
|
/** Response duration in ms */
|
||||||
@@ -67,7 +87,7 @@ export type LLMRequest = {
|
|||||||
/** Message history */
|
/** Message history */
|
||||||
history?: LLMMessage[];
|
history?: LLMMessage[];
|
||||||
/** Max tokens for request */
|
/** Max tokens for request */
|
||||||
max_tokens?: number;
|
maxTokens?: number;
|
||||||
/** 0 = Rigid Logic, 1 = Balanced, 2 = Hyper Creative **/
|
/** 0 = Rigid Logic, 1 = Balanced, 2 = Hyper Creative **/
|
||||||
temperature?: number;
|
temperature?: number;
|
||||||
/** Available tools */
|
/** Available tools */
|
||||||
@@ -88,6 +108,8 @@ export type LLMRequest = {
|
|||||||
mcp?: McpServer[];
|
mcp?: McpServer[];
|
||||||
/** Subagents exposed as delegatable/wrapped tools */
|
/** Subagents exposed as delegatable/wrapped tools */
|
||||||
agents?: Agent[];
|
agents?: Agent[];
|
||||||
|
/** Attach files to request */
|
||||||
|
files?: LLMFile[];
|
||||||
/** @internal recursion guard for nested agent delegation */
|
/** @internal recursion guard for nested agent delegation */
|
||||||
_agentDepth?: number;
|
_agentDepth?: number;
|
||||||
}
|
}
|
||||||
@@ -111,6 +133,11 @@ export type Skill = {
|
|||||||
}
|
}
|
||||||
|
|
||||||
class LLM {
|
class LLM {
|
||||||
|
private static AUDIO_EXT = ['wav','mp3','m4a','flac','ogg','aac','wma'];
|
||||||
|
private static IMAGE_EXT = ['png','jpg','jpeg','bmp','gif','tiff','webp'];
|
||||||
|
private static TEXT_EXT = ['txt','md','csv','json','xml','html','js','ts','py','yaml','yml','log'];
|
||||||
|
private static PDF_EXT = ['pdf'];
|
||||||
|
|
||||||
private memoryManager!: MemoryManager;
|
private memoryManager!: MemoryManager;
|
||||||
|
|
||||||
defaultModel!: string;
|
defaultModel!: string;
|
||||||
@@ -126,6 +153,118 @@ class LLM {
|
|||||||
this.memoryManager = new MemoryManager(this);
|
this.memoryManager = new MemoryManager(this);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
private async loadBuffer(file: LLMFile, asText: boolean): Promise<Buffer> {
|
||||||
|
if(file.path) return fs.readFile(file.path);
|
||||||
|
if(Buffer.isBuffer(file.content)) return file.content;
|
||||||
|
if(typeof file.content === 'string') return Buffer.from(file.content, asText ? 'utf-8' : 'base64');
|
||||||
|
throw new Error('No path or content provided');
|
||||||
|
}
|
||||||
|
|
||||||
|
private async writeTemp(name: string, buffer: Buffer): Promise<string> {
|
||||||
|
const path = join(mkdtempSync(join(tmpdir(), 'ai-file-')), name);
|
||||||
|
await fs.writeFile(path, buffer);
|
||||||
|
return path;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Extract text from a PDF. Pages with no text layer (scanned/image-only) are handled as either:
|
||||||
|
* - Rendered to images and returned alongside the text so the (vision-capable) model can read them directly
|
||||||
|
* - OCR'd via Tesseract when the doc is too large to reasonably pass as images
|
||||||
|
*/
|
||||||
|
private async resolvePdf(buffer: Buffer): Promise<{text: string, images: {mime: string, data: string}[]}> {
|
||||||
|
const parser = new PDFParse({data: buffer});
|
||||||
|
try {
|
||||||
|
const {text, pages} = await parser.getText();
|
||||||
|
const scanned = (pages || []).filter(p => !p.text?.trim());
|
||||||
|
if(!scanned.length) return {text: text.trim() || '[Empty PDF]', images: []};
|
||||||
|
const total = pages.length;
|
||||||
|
const pageNums = scanned.map(p => p.num);
|
||||||
|
const {pages: shots} = await parser.getScreenshot({partial: pageNums});
|
||||||
|
if(total <= PDF_OCR_PAGE_THRESHOLD) {
|
||||||
|
return {
|
||||||
|
text: text.trim(),
|
||||||
|
images: shots.map(s => ({mime: 'image/png', data: Buffer.from(s.data).toString('base64')}))
|
||||||
|
};
|
||||||
|
}
|
||||||
|
const ocrText = await Promise.all(shots.map(async (s, i) => {
|
||||||
|
const path = await this.writeTemp(`page-${pageNums[i]}.png`, Buffer.from(s.data));
|
||||||
|
try {
|
||||||
|
return await this.ai.vision.ocr(path) || '';
|
||||||
|
} finally {
|
||||||
|
fs.rm(dirname(path), {recursive: true, force: true}).catch(() => {});
|
||||||
|
}
|
||||||
|
}));
|
||||||
|
return {text: [text.trim(), ...ocrText].filter(Boolean).join('\n\n'), images: []};
|
||||||
|
} finally {
|
||||||
|
await parser.destroy();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private async resolveFile(file: LLMFile): Promise<{text?: string, images?: {mime: string, data: string}[]}> {
|
||||||
|
const name = file.name || (file.path ? basename(file.path) : 'file');
|
||||||
|
|
||||||
|
// Already resolved on a previous turn, reuse cached text
|
||||||
|
if(file.extracted) return {text: `<file name="${name}">\n${file.content}\n</file>`};
|
||||||
|
|
||||||
|
const ext = extname(name).slice(1).toLowerCase();
|
||||||
|
const mime = file.mime || '';
|
||||||
|
const isAudio = mime.startsWith('audio/') || LLM.AUDIO_EXT.includes(ext);
|
||||||
|
const isImage = mime.startsWith('image/') || LLM.IMAGE_EXT.includes(ext);
|
||||||
|
const isPdf = mime === 'application/pdf' || LLM.PDF_EXT.includes(ext);
|
||||||
|
const isText = mime.startsWith('text/') || LLM.TEXT_EXT.includes(ext);
|
||||||
|
|
||||||
|
let tmpDir: string | null = null;
|
||||||
|
try {
|
||||||
|
if(isImage) {
|
||||||
|
const data = (await this.loadBuffer(file, false)).toString('base64');
|
||||||
|
return {images: [{mime: mime || `image/${ext === 'jpg' ? 'jpeg' : ext}`, data}]};
|
||||||
|
}
|
||||||
|
|
||||||
|
if(isPdf) {
|
||||||
|
const {text, images} = await this.resolvePdf(await this.loadBuffer(file, false));
|
||||||
|
// Only cache/skip re-processing when we didn't need to hand off images (OCR'd or fully text-based)
|
||||||
|
if(!images.length) {
|
||||||
|
file.content = text;
|
||||||
|
file.extracted = true;
|
||||||
|
delete file.path;
|
||||||
|
}
|
||||||
|
return {text: `<file name="${name}">\n${text || '[Scanned PDF - see attached page images]'}\n</file>`, images};
|
||||||
|
}
|
||||||
|
|
||||||
|
let text: string;
|
||||||
|
if(isAudio) {
|
||||||
|
let path = file.path;
|
||||||
|
if(!path) {
|
||||||
|
const buffer = await this.loadBuffer(file, false);
|
||||||
|
path = await this.writeTemp(name, buffer);
|
||||||
|
tmpDir = dirname(path);
|
||||||
|
}
|
||||||
|
text = await this.ai.audio.asr(path) || '';
|
||||||
|
} else if(isText) {
|
||||||
|
text = (await this.loadBuffer(file, true)).toString('utf-8');
|
||||||
|
} else {
|
||||||
|
text = typeof file.content === 'string' ? file.content : `[Binary file, unable to extract: ${name}]`;
|
||||||
|
}
|
||||||
|
file.content = text;
|
||||||
|
file.extracted = true;
|
||||||
|
delete file.path;
|
||||||
|
|
||||||
|
return {text: `<file name="${name}">\n${text}\n</file>`};
|
||||||
|
} catch(err: any) {
|
||||||
|
return {text: `<file name="${name}">Failed to process: ${err.message}</file>`};
|
||||||
|
} finally {
|
||||||
|
if(tmpDir) fs.rm(tmpDir, {recursive: true, force: true}).catch(() => {});
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private async resolveFiles(files: LLMFile[]): Promise<{text: string, images: {mime: string, data: string}[]}> {
|
||||||
|
const resolved = await Promise.all(files.map(f => this.resolveFile(f)));
|
||||||
|
return {
|
||||||
|
text: resolved.filter(r => r.text).map(r => r.text).join('\n\n'),
|
||||||
|
images: resolved.flatMap(r => r.images || [])
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
private setupAgent(agents: Agent[] = [], allAgents: Agent[], history: LLMMessage[], aborts: (() => void)[], depth = 0, delegateState: {resp: string | null}): AiTool[] {
|
private setupAgent(agents: Agent[] = [], allAgents: Agent[], history: LLMMessage[], aborts: (() => void)[], depth = 0, delegateState: {resp: string | null}): AiTool[] {
|
||||||
return agents.map(a => {
|
return agents.map(a => {
|
||||||
const toolName = `${a.delegate ? '' : 'sub'}agent_${snakeCase(a.name)}`;
|
const toolName = `${a.delegate ? '' : 'sub'}agent_${snakeCase(a.name)}`;
|
||||||
@@ -207,7 +346,7 @@ ${a.system}`,
|
|||||||
|
|
||||||
const list = allTools.map(t => `- ${t.name}: ${t.description}`).join('\n');
|
const list = allTools.map(t => `- ${t.name}: ${t.description}`).join('\n');
|
||||||
return {
|
return {
|
||||||
prompt: `You have access to the following MCP tools:\n${list}`,
|
prompt: `## MCP\nYou have access to the following MCP tools:\n${list}`,
|
||||||
tools: allTools
|
tools: allTools
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
@@ -216,7 +355,7 @@ ${a.system}`,
|
|||||||
if(!skills?.length) return {prompt: '', tools: []};
|
if(!skills?.length) return {prompt: '', tools: []};
|
||||||
const list = skills.map(s => `- ${s.name}: ${s.description}`).join('\n');
|
const list = skills.map(s => `- ${s.name}: ${s.description}`).join('\n');
|
||||||
return {
|
return {
|
||||||
prompt: `You have access to the following skill documents, whenever there is overlap between a question and a skill file, use \`skill_read\` to get instructions and background knowledge:\n${list}`,
|
prompt: `## Skills\nYou have access to the following skill documents, whenever there is overlap between a question and a skill file, use \`skill_read\` to get instructions and background knowledge:\n${list}`,
|
||||||
tools: [{
|
tools: [{
|
||||||
name: 'skill_read',
|
name: 'skill_read',
|
||||||
description: 'Read the full content of a skill/knowledge document',
|
description: 'Read the full content of a skill/knowledge document',
|
||||||
@@ -272,7 +411,8 @@ ${a.system}`,
|
|||||||
let tools: AiTool[] = options.tools || this.ai.options.llm?.tools || [];
|
let tools: AiTool[] = options.tools || this.ai.options.llm?.tools || [];
|
||||||
const prompts: string[] = [];
|
const prompts: string[] = [];
|
||||||
let history = options.history || [];
|
let history = options.history || [];
|
||||||
if(message) history.push({role: 'user', content: message, timestamp: Date.now()});
|
const files = options.files || [];
|
||||||
|
if(message || files.length) history.push({role: 'user', content: message || '', timestamp: Date.now()});
|
||||||
|
|
||||||
// MCP
|
// MCP
|
||||||
const mcp = options.mcp || this.ai.options?.llm?.mcp;
|
const mcp = options.mcp || this.ai.options?.llm?.mcp;
|
||||||
@@ -316,30 +456,26 @@ ${a.system}`,
|
|||||||
} else listed.push(r);
|
} else listed.push(r);
|
||||||
}
|
}
|
||||||
|
|
||||||
prompts.unshift(`You have a background memory process which has prefetched relevant information${mem.update ? ' and will create new memories from this conversation' : ''} for you
|
prompts.unshift(`## Memory
|
||||||
|
You have a background memory process which has prefetched relevant information${mem.update ? ' and will create new memories from this conversation' : ''} for you
|
||||||
Assume it is perfect and never mention this process to anyone ever
|
Assume it is perfect and never mention this process to anyone ever
|
||||||
Always use your memories to craft a personalized response, they contain links / [[wiki links]] which you use navigate between them
|
Always use your memories to craft a personalized response, they contain links / [[wiki links]] which you use navigate between them
|
||||||
${mem.tool ? `You can access memory files via the \`memory_search\` and \`memory_recall\` tools
|
${mem.tool ? `You can access memory files via the \`memory_search\` and \`memory_recall\` tools
|
||||||
When you need information about the user, \`memory_recall\` \`People/User\` before asking (fetch if not included bellow)
|
When you need information about the user, \`memory_recall\` \`People/User\` before asking (fetch if not included bellow)
|
||||||
When you need information not provided, attempt 1-3 \`memory_search\` calls with unique queries before asking` : ''}
|
When you need information not provided, attempt 1-3 \`memory_search\` calls with distinct queries before asking` : ''}
|
||||||
|
|
||||||
${preloaded.length ? `
|
${preloaded.length ? `### Prefetched Memories (Most relevant first):
|
||||||
Prefetched Memories (Most relevant first):
|
|
||||||
|
|
||||||
${preloaded.map(r => `Memory: ${r.name}
|
${preloaded.map(r => `Memory: ${r.name}
|
||||||
Description: ${r.description}
|
Description: ${r.description}
|
||||||
Linked: ${[r.links, ...r.backlinks].join(', ')}
|
Linked: ${makeUnique([...r.links, ...r.backlinks]).join(', ')}
|
||||||
\`\`\`
|
\`\`\`
|
||||||
${r.content}
|
${stripHeader(r.content)}
|
||||||
\`\`\``).join('\n\n')}` : ''}
|
\`\`\``).join('\n\n')}` : ''}
|
||||||
|
${mem.tool && listed.length ? '\n' + listed.map(r => `Memory: ${r.name}
|
||||||
${mem.tool && listed.length ? listed.map(r => `Memory: ${r.name}
|
|
||||||
Description: ${r.description}
|
Description: ${r.description}
|
||||||
Linked: ${[r.links, ...r.backlinks].join(', ')}
|
Linked: ${makeUnique([...r.links, ...r.backlinks]).join(', ')}
|
||||||
<!-- Truncated -->`).join('\n\n') : ''}
|
<!-- Truncated -->`).join('\n\n') : ''}`.trim())
|
||||||
|
|
||||||
${mem.tool ? `Full memory list:
|
|
||||||
${mems.map(m => `- ${m.name}: ${m.description}`).join('\n')}` : ''}`.trim())
|
|
||||||
}
|
}
|
||||||
if(mem.tool) tools.push(this.memoryManager.tools.read(mem.memory));
|
if(mem.tool) tools.push(this.memoryManager.tools.read(mem.memory));
|
||||||
}
|
}
|
||||||
@@ -347,6 +483,20 @@ ${mems.map(m => `- ${m.name}: ${m.description}`).join('\n')}` : ''}`.trim())
|
|||||||
|
|
||||||
if(aborted) throw Object.assign(new Error('Aborted'), {name: 'AbortError'});
|
if(aborted) throw Object.assign(new Error('Aborted'), {name: 'AbortError'});
|
||||||
|
|
||||||
|
const lastMsg = history[history.length - 1];
|
||||||
|
if(files.length && lastMsg?.role === 'user') lastMsg.files = files;
|
||||||
|
const restores: {msg: LLMMessage, content: any}[] = [];
|
||||||
|
for(const msg of history) {
|
||||||
|
if(msg.role !== 'user' || !msg.files?.length) continue;
|
||||||
|
const {text, images} = await this.resolveFiles(msg.files);
|
||||||
|
if(!text && !images.length) continue;
|
||||||
|
restores.push({msg, content: msg.content});
|
||||||
|
const merged = text ? [msg.content, text].filter(Boolean).join('\n\n') : msg.content;
|
||||||
|
msg.content = images.length
|
||||||
|
? [...images.map(i => ({type: 'image', mime: i.mime, data: i.data})), {type: 'text', text: merged}]
|
||||||
|
: merged;
|
||||||
|
}
|
||||||
|
|
||||||
const toolTimings = new Map<string, {duration: number, tps: number}>();
|
const toolTimings = new Map<string, {duration: number, tps: number}>();
|
||||||
tools = this.wrapToolTiming(tools, toolTimings);
|
tools = this.wrapToolTiming(tools, toolTimings);
|
||||||
|
|
||||||
@@ -356,6 +506,9 @@ ${mems.map(m => `- ${m.name}: ${m.description}`).join('\n')}` : ''}`.trim())
|
|||||||
request = this.models[m].ask('', {...options, tools, system: prompts.filter(Boolean).join('\n\n')});
|
request = this.models[m].ask('', {...options, tools, system: prompts.filter(Boolean).join('\n\n')});
|
||||||
let resp = await request;
|
let resp = await request;
|
||||||
|
|
||||||
|
// Strip the file injection shim
|
||||||
|
restores.forEach(({msg, content}) => msg.content = content);
|
||||||
|
|
||||||
// Capture meta (duration / tps)
|
// Capture meta (duration / tps)
|
||||||
for(const h of history) {
|
for(const h of history) {
|
||||||
if(h.role === 'tool' && toolTimings.has(h.id)) Object.assign(h, toolTimings.get(h.id));
|
if(h.role === 'tool' && toolTimings.has(h.id)) Object.assign(h, toolTimings.get(h.id));
|
||||||
|
|||||||
249
src/memory.ts
249
src/memory.ts
@@ -2,9 +2,10 @@ import {MemoryNode, rebuildGraph} from './helpers.ts';
|
|||||||
import {LLMRequest, LLMMessage} from './llm.ts';
|
import {LLMRequest, LLMMessage} from './llm.ts';
|
||||||
import {AiTool} from './tools.ts';
|
import {AiTool} from './tools.ts';
|
||||||
import {KDPoint, KDTree} from './kd-tree.ts';
|
import {KDPoint, KDTree} from './kd-tree.ts';
|
||||||
|
import {escapeRegex} from '@ztimson/utils';
|
||||||
|
|
||||||
const FACTS_HEADING = '## Facts';
|
const MERGE_THRESHOLD = 0.88;
|
||||||
|
const PENDING_HEADING = '## Pending';
|
||||||
const GENERIC_TEMPLATE = `# {{Title}}
|
const GENERIC_TEMPLATE = `# {{Title}}
|
||||||
|
|
||||||
## Summary
|
## Summary
|
||||||
@@ -32,6 +33,11 @@ type FactBucket = {
|
|||||||
facts: string[];
|
facts: string[];
|
||||||
}
|
}
|
||||||
|
|
||||||
|
type FactAgentResult = {
|
||||||
|
buckets: FactBucket[];
|
||||||
|
journal: string;
|
||||||
|
}
|
||||||
|
|
||||||
function dedupeFacts(facts: string[]): string[] {
|
function dedupeFacts(facts: string[]): string[] {
|
||||||
const seen = new Map<string, string>();
|
const seen = new Map<string, string>();
|
||||||
for (const f of facts) {
|
for (const f of facts) {
|
||||||
@@ -61,6 +67,10 @@ function cosineSearch(query: number[], memories: Memory[], limit: number): Memor
|
|||||||
.map(s => s.ref);
|
.map(s => s.ref);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
export function stripHeader(content: string): string {
|
||||||
|
return content.replace(/^---[\s\S]*?\n---\n?/, '').trimStart();
|
||||||
|
}
|
||||||
|
|
||||||
export class MemoryCache {
|
export class MemoryCache {
|
||||||
private tree!: KDTree<MemoryRef>;
|
private tree!: KDTree<MemoryRef>;
|
||||||
public memories: Memory[];
|
public memories: Memory[];
|
||||||
@@ -97,8 +107,8 @@ export class MemoryCache {
|
|||||||
}
|
}
|
||||||
|
|
||||||
update(memory: Memory): void {
|
update(memory: Memory): void {
|
||||||
const idx = this.memories.findIndex(m => m.name === memory.name);
|
const existing = this.memories.find(m => m.name === memory.name);
|
||||||
if (idx !== -1) this.memories[idx] = memory;
|
if (existing) Object.assign(existing, memory);
|
||||||
else this.memories.push(memory);
|
else this.memories.push(memory);
|
||||||
this.rebuild();
|
this.rebuild();
|
||||||
}
|
}
|
||||||
@@ -159,7 +169,7 @@ class MemoryAccessor {
|
|||||||
const missing = this.list.filter(m => !m.embedding?.length);
|
const missing = this.list.filter(m => !m.embedding?.length);
|
||||||
if (!missing.length) return 0;
|
if (!missing.length) return 0;
|
||||||
await Promise.all(missing.map(async node => {
|
await Promise.all(missing.map(async node => {
|
||||||
const [e] = await llm.embedding(node.content);
|
const [e] = await llm.embedding(`${node.description}\n\n${stripHeader(node.content)}`.trim());
|
||||||
if (e) node.embedding = e.embedding;
|
if (e) node.embedding = e.embedding;
|
||||||
}));
|
}));
|
||||||
this.commit();
|
this.commit();
|
||||||
@@ -247,14 +257,13 @@ ${m.content}
|
|||||||
return new MemoryAccessor(memories);
|
return new MemoryAccessor(memories);
|
||||||
}
|
}
|
||||||
|
|
||||||
private appendFacts(node: Memory, facts: string[]): void {
|
private stage(node: Memory, block: string): void {
|
||||||
this.ensureDoc(node);
|
this.ensureDoc(node);
|
||||||
const body = this.stripHeader(node.content);
|
const body = stripHeader(node.content);
|
||||||
const bullets = facts.map(f => `- ${f}`).join('\n');
|
const idx = body.indexOf(PENDING_HEADING);
|
||||||
const idx = body.indexOf(FACTS_HEADING);
|
|
||||||
const newBody = idx === -1
|
const newBody = idx === -1
|
||||||
? `${body.trimEnd()}\n\n${FACTS_HEADING}\n${bullets}\n`
|
? `${body.trimEnd()}\n\n${PENDING_HEADING}\n${block}\n`
|
||||||
: `${body.slice(0, idx + FACTS_HEADING.length)}\n${bullets}${body.slice(idx + FACTS_HEADING.length)}`;
|
: `${body.slice(0, idx + PENDING_HEADING.length)}\n${block}${body.slice(idx + PENDING_HEADING.length)}`;
|
||||||
node.content = this.touchHeader(node, newBody);
|
node.content = this.touchHeader(node, newBody);
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -264,42 +273,52 @@ ${m.content}
|
|||||||
node.content = this.touchHeader(node, `# ${title}\n`);
|
node.content = this.touchHeader(node, `# ${title}\n`);
|
||||||
}
|
}
|
||||||
|
|
||||||
private async factAgent(conversation: string, store: MemoryAccessor, options: LLMRequest, weekKey: string): Promise<FactBucket[]> {
|
private sanitizeDescription(text: string): string {
|
||||||
|
return (text ?? '').replace(/\s+/g, ' ').trim().slice(0, 240);
|
||||||
|
}
|
||||||
|
|
||||||
|
private relink(memories: Memory[], from: string, to: string): void {
|
||||||
|
const pattern = new RegExp(`\\[\\[${escapeRegex(from)}\\]\\]`, 'g');
|
||||||
|
for (const m of memories) if (pattern.test(m.content)) m.content = m.content.replace(pattern, `[[${to}]]`);
|
||||||
|
}
|
||||||
|
|
||||||
|
private async factAgent(conversation: string, store: MemoryAccessor, options: LLMRequest): Promise<FactAgentResult> {
|
||||||
const ghosts = store.ghosts();
|
const ghosts = store.ghosts();
|
||||||
|
|
||||||
const response = await this.llm.ask(conversation, {
|
const response = await this.llm.ask(conversation, {
|
||||||
model: options.model,
|
model: options.model,
|
||||||
temperature: 0.2,
|
temperature: 0.2,
|
||||||
system: `You are a fact extractor to build obsidian knowledge vaults.
|
system: `You are a fact extractor for Obsidian-style knowledge vaults. Analyze the conversation and produce:
|
||||||
Analyze this conversation and extract facts worth remembering long-term.
|
|
||||||
|
|
||||||
Rules:
|
1. Journal recap (single paragraph)
|
||||||
- Always extract facts that the user explicitly told you to remember
|
- "Captains Log" style record keeping
|
||||||
- ONLY extract current facts the USER explicitly stated about themselves, their work, projects or decisions that were MADE during this conversation
|
- What was discussed/worked on, decisions, user's events/state/mood, general context
|
||||||
- DO NOT extract greetings, pleasantries, or generic exchanges
|
- Leave empty only for trivial/empty exchanges/small talk
|
||||||
- DO NOT extract deltas or changes in facts; ONLY the end fact
|
|
||||||
- DO NOT extract anything the AI/assistant itself said
|
|
||||||
- If nothing worth remembering was said, return an empty buckets array
|
|
||||||
|
|
||||||
When extracting facts, you MUST also decide the exact destination path:
|
2. Fact buckets
|
||||||
- Reuse node names (including ghost) as much as possible IF the facts belongs there
|
- ONLY facts the USER explicitly stated about themselves, their work, projects, or decisions made during this conversation
|
||||||
- All information primarily about the user should go under "People/User"
|
- NEVER extract greetings, pleasantries, or anything the assistant itself said
|
||||||
- When required, create a new path following collection/subject format (e.g., People/Sarah, Projects/Oxide) — you are not limited to any fixed list of collections, use whatever fits
|
- Extract the final/end state, not deltas
|
||||||
- For journal entries, use "Journal"
|
|
||||||
|
Path assignment rules:
|
||||||
|
- Reuse existing node names whenever possible
|
||||||
|
- Documents should be grouped and named by the root subject
|
||||||
|
- Person → People/Name
|
||||||
|
- Project → Projects/Name
|
||||||
|
- Concept → Concepts/Name
|
||||||
|
- A bug report, its investigation, should be nested and attached to the same root subject node
|
||||||
|
- Tickets/one-off tasks → file under the project/name/component they belong to
|
||||||
|
- Only create a new top-level node when the fact belongs to a genuinely new subject (person/project/concept)\`
|
||||||
|
|
||||||
Available nodes:
|
Available nodes:
|
||||||
- Journal
|
${this.listNodes(store.list).map(n => `- ${n.name}: ${n.description}`).join('\n') || 'None yet.'}
|
||||||
${this.listNodes(store.list).filter(n => !n.name.includes('Journal')).map(n => `- ${n.name}: ${n.description}`).join('\n') || 'None yet.'}
|
|
||||||
${ghosts.length ? `${ghosts.map(g => `- ${g}: (Ghost)`).join('\n')}` : ''}`,
|
${ghosts.length ? `${ghosts.map(g => `- ${g}: (Ghost)`).join('\n')}` : ''}`,
|
||||||
schema: {
|
schema: {
|
||||||
buckets: {type: 'array', description: 'Groups of facts to remember, each assigned to a different node. Return an empty array if there is nothing worth storing in an obsidian vault', items: {
|
journal: {type: 'string', description: 'Short day-to-day recap; empty if nothing happened.', required: false},
|
||||||
|
buckets: {type: 'array', description: 'Groups of facts to remember; empty array if nothing worth storing.', items: {
|
||||||
type: 'object', items: {
|
type: 'object', items: {
|
||||||
subject: {type: 'string', description: 'Exact existing node name OR new path (e.g. "People/Sarah", "Projects/Oxide"), or "Journal"', required: true},
|
subject: {type: 'string', description: 'Exact node name or new path (e.g. "People/Sarah", "Projects/Oxide")', required: true},
|
||||||
facts: {
|
facts: {type: 'array', description: 'Facts to store here', items: {type: 'string'}},
|
||||||
type: 'array',
|
|
||||||
description: 'Facts to store at this destination',
|
|
||||||
items: {type: 'string', description: 'A single fact'},
|
|
||||||
},
|
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
@@ -307,15 +326,17 @@ ${ghosts.length ? `${ghosts.map(g => `- ${g}: (Ghost)`).join('\n')}` : ''}`,
|
|||||||
});
|
});
|
||||||
|
|
||||||
const buckets = new Map<string, string[]>();
|
const buckets = new Map<string, string[]>();
|
||||||
for(const bucket of response.buckets ?? []) {
|
for (const bucket of response.buckets ?? []) {
|
||||||
const subject = bucket.subject.trim().toLowerCase() === 'journal'
|
const subject = bucket.subject.trim();
|
||||||
? `Journal/${weekKey}` : bucket.subject.trim();
|
|
||||||
const facts = buckets.get(subject) ?? [];
|
const facts = buckets.get(subject) ?? [];
|
||||||
facts.push(...dedupeFacts(bucket.facts));
|
facts.push(...dedupeFacts(bucket.facts));
|
||||||
buckets.set(subject, facts);
|
buckets.set(subject, facts);
|
||||||
}
|
}
|
||||||
|
|
||||||
return buckets.entries().toArray().map(([subject, facts]) => ({subject, facts}));
|
return {
|
||||||
|
buckets: buckets.entries().toArray().map(([subject, facts]) => ({subject, facts})),
|
||||||
|
journal: (response.journal ?? '').trim(),
|
||||||
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
private getWeekMonday(date: Date = new Date()): string {
|
private getWeekMonday(date: Date = new Date()): string {
|
||||||
@@ -330,6 +351,42 @@ ${ghosts.length ? `${ghosts.map(g => `- ${g}: (Ghost)`).join('\n')}` : ''}`,
|
|||||||
return memories.map(m => ({name: m.name, description: m.description}));
|
return memories.map(m => ({name: m.name, description: m.description}));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/** Find the nearest node above the similarity threshold and fold the smaller/less-connected one into
|
||||||
|
* the other. Journals are exempt — they're partitioned by date, not topic, and merging across weeks
|
||||||
|
* would wreck the timeline. Returns 'merged' if `node` absorbed another (caller should re-run the doc
|
||||||
|
* agent), 'absorbed' if `node` itself got folded away (caller should stop touching it), or null. */
|
||||||
|
private async checkMerge(node: Memory, memories: Memory[] | MemoryCache, options: LLMRequest, threshold = MERGE_THRESHOLD): Promise<Memory | null> {
|
||||||
|
if (!node.embedding?.length || node.name.startsWith('Journal/')) return null;
|
||||||
|
const store = this.access(memories);
|
||||||
|
|
||||||
|
let closest: Memory | null = null, closestDist = Infinity;
|
||||||
|
for (const other of store.list) {
|
||||||
|
if (other.name === node.name || other.name.startsWith('Journal/') || !other.embedding?.length) continue;
|
||||||
|
const d = cosineDistance(node.embedding, other.embedding);
|
||||||
|
if (d < closestDist) { closestDist = d; closest = other; }
|
||||||
|
}
|
||||||
|
if (!closest || closestDist > threshold) return null;
|
||||||
|
|
||||||
|
const result = await this.mergeAgent(node, closest, options);
|
||||||
|
const merged: Memory = {name: result.name, description: this.sanitizeDescription(result.description), content: '', embedding: [], links: [], backlinks: []};
|
||||||
|
merged.content = this.touchHeader(merged, result.content);
|
||||||
|
const [e] = await this.llm.embedding(`${merged.description}\n\n${result.content}`.trim());
|
||||||
|
if (e) merged.embedding = e.embedding;
|
||||||
|
|
||||||
|
this.relink(store.list, node.name, merged.name);
|
||||||
|
this.relink(store.list, closest.name, merged.name);
|
||||||
|
|
||||||
|
this.queues.get(closest.name)?.request?.abort?.();
|
||||||
|
this.queues.delete(closest.name);
|
||||||
|
|
||||||
|
store.forget(node.name);
|
||||||
|
store.forget(closest.name);
|
||||||
|
store.list.push(merged);
|
||||||
|
store.commit();
|
||||||
|
|
||||||
|
return merged;
|
||||||
|
}
|
||||||
|
|
||||||
private reconcile(node: Memory, memories: Memory[] | MemoryCache, options: LLMRequest): Promise<void> {
|
private reconcile(node: Memory, memories: Memory[] | MemoryCache, options: LLMRequest): Promise<void> {
|
||||||
const key = node.name;
|
const key = node.name;
|
||||||
const existing = this.queues.get(key);
|
const existing = this.queues.get(key);
|
||||||
@@ -343,9 +400,12 @@ ${ghosts.length ? `${ghosts.map(g => `- ${g}: (Ghost)`).join('\n')}` : ''}`,
|
|||||||
this.queues.set(key, entry);
|
this.queues.set(key, entry);
|
||||||
const store = this.access(memories);
|
const store = this.access(memories);
|
||||||
entry.task = (async () => {
|
entry.task = (async () => {
|
||||||
|
let current = node;
|
||||||
do {
|
do {
|
||||||
entry.dirty = false;
|
entry.dirty = false;
|
||||||
await this.docAgent(node, store.list, options, entry);
|
await this.docAgent(current, store.list, options, entry);
|
||||||
|
const merged = await this.checkMerge(current, memories, options);
|
||||||
|
if (merged) { current = merged; entry.dirty = true; }
|
||||||
} while (entry.dirty);
|
} while (entry.dirty);
|
||||||
})().finally(() => {
|
})().finally(() => {
|
||||||
this.queues.delete(key);
|
this.queues.delete(key);
|
||||||
@@ -355,7 +415,8 @@ ${ghosts.length ? `${ghosts.map(g => `- ${g}: (Ghost)`).join('\n')}` : ''}`,
|
|||||||
}
|
}
|
||||||
|
|
||||||
private async docAgent(node: Memory, memories: Memory[], options: LLMRequest, entry: {request: {abort?: () => void} | null}): Promise<void> {
|
private async docAgent(node: Memory, memories: Memory[], options: LLMRequest, entry: {request: {abort?: () => void} | null}): Promise<void> {
|
||||||
const currentBody = this.stripHeader(node.content);
|
if(!memories.includes(node)) return;
|
||||||
|
const currentBody = stripHeader(node.content);
|
||||||
let update;
|
let update;
|
||||||
try {
|
try {
|
||||||
for (let i = 0; i < 2 && !update?.content; i++) {
|
for (let i = 0; i < 2 && !update?.content; i++) {
|
||||||
@@ -363,27 +424,29 @@ ${ghosts.length ? `${ghosts.map(g => `- ${g}: (Ghost)`).join('\n')}` : ''}`,
|
|||||||
model: options.model,
|
model: options.model,
|
||||||
temperature: 0.3,
|
temperature: 0.3,
|
||||||
schema: {
|
schema: {
|
||||||
description: {type: 'string', description: 'One-line description of what this document covers, no formatting or emojis', required: true},
|
description: {type: 'string', description: 'One factual sentence describing the document\'s ENTIRE SUBJECT MATTER — for use as a search/merge fingerprint', required: true},
|
||||||
content: {type: 'string', description: 'Rewritten document body in markdown, without the frontmatter block', required: true},
|
content: {type: 'string', description: 'Rewritten document body in markdown, without the frontmatter block', required: true},
|
||||||
},
|
},
|
||||||
system: `You are a knowledge base editor maintaining one document in an Obsidian-style vault.
|
system: `You are a knowledge base editor maintaining one Obsidian-style document.
|
||||||
|
|
||||||
If the document has a "${FACTS_HEADING}" section, integrate every bullet under it into the appropriate part of the document, then remove the "${FACTS_HEADING}" section entirely. If there is no such section, just tidy the document per the rules below.
|
If it has a "## Pending" section, fold all new material into the appropriate part, resolve overlap, then remove the section entirely. If no section, just tidy per the rules below.
|
||||||
|
|
||||||
Structure: follow this generic shape loosely, adapting section names/order to what the content actually needs (e.g. journal-style docs may want a timeline instead of "Details"):
|
Use this loose structure, adapting headings to what the content needs:
|
||||||
\`\`\`markdown
|
|
||||||
${GENERIC_TEMPLATE}
|
|
||||||
\`\`\`
|
|
||||||
|
|
||||||
Formatting rules:
|
# Title
|
||||||
- Use Obsidian-style markdown: # headings, **bold** for emphasis, bullet & numbered lists for grouped 1D data, tables for 2D data
|
## Summary
|
||||||
- Link related concepts with [[WikiLink]] notation using full paths like [[People/Sarah]] or [[Projects/Website]]
|
## Details
|
||||||
- Create links for specific entities (person, place, project, program) and abstract concepts, but skip generics (car, red, dog)
|
## Related
|
||||||
- Keep the document concise, factual, and human-readable
|
|
||||||
- Resolve contradictions: newer facts always win — delete the outdated statement entirely, never keep both
|
|
||||||
- Do not add frontmatter blocks, filler, preamble, or AI commentary
|
|
||||||
|
|
||||||
Other nodes in the vault (link to these instead of duplicating their content):
|
Rules:
|
||||||
|
- Contradictions: newer facts always win — delete outdated statements entirely
|
||||||
|
- Journals (Journal/...): keep entries as a chronological timeline; clean up grammar within entries but never delete history
|
||||||
|
- Use Obsidian markdown: # headings, **bold**, bullet/numbered lists, tables for 2D data
|
||||||
|
- Link specific entities and concepts with [[WikiLink]] (e.g., [[Projects/KiwixServer]]); skip generics
|
||||||
|
- Keep concise, factual, human-readable
|
||||||
|
- NO frontmatter, filler, preamble, or AI commentary
|
||||||
|
|
||||||
|
Available nodes to link to (don't duplicate their content):
|
||||||
${this.listNodes(memories).filter(n => n.name !== node.name).map(n => n.name).join(', ') || 'none'}
|
${this.listNodes(memories).filter(n => n.name !== node.name).map(n => n.name).join(', ') || 'none'}
|
||||||
|
|
||||||
Current document:
|
Current document:
|
||||||
@@ -402,12 +465,44 @@ ${currentBody}
|
|||||||
}
|
}
|
||||||
|
|
||||||
if (!update?.content) return;
|
if (!update?.content) return;
|
||||||
node.description = node.name !== 'People/User' ? update.description : 'All information about the current user';
|
node.description = node.name !== 'People/User' ? this.sanitizeDescription(update.description) : 'All information about the current user';
|
||||||
node.content = this.touchHeader(node, update.content);
|
node.content = this.touchHeader(node, update.content);
|
||||||
const [e] = await this.llm.embedding(node.content);
|
const [e] = await this.llm.embedding(`${node.description}\n\n${update.content}`.trim());
|
||||||
if (e) node.embedding = e.embedding;
|
if (e) node.embedding = e.embedding;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
private async mergeAgent(a: Memory, b: Memory, options: LLMRequest): Promise<{name: string, description: string, content: string}> {
|
||||||
|
return this.llm.ask('', {
|
||||||
|
model: options.model,
|
||||||
|
temperature: 0.3,
|
||||||
|
schema: {
|
||||||
|
name: {type: 'string', description: 'New path for the merged doc, collection/subject format (e.g. Projects/Oxide) — only reuse an old title if it\'s genuinely the best fit', required: true},
|
||||||
|
description: {type: 'string', description: 'One factual sentence describing the merged document\'s subject matter', required: true},
|
||||||
|
content: {type: 'string', description: 'Fully reconciled body in markdown, without frontmatter', required: true},
|
||||||
|
},
|
||||||
|
system: `You are a knowledge base editor merging two overlapping Obsidian documents into one. Newer facts win on contradiction.
|
||||||
|
|
||||||
|
Structure loosely:
|
||||||
|
|
||||||
|
# Title
|
||||||
|
## Summary
|
||||||
|
## Details
|
||||||
|
## Related
|
||||||
|
|
||||||
|
Combine both documents, resolve duplication and contradictions.
|
||||||
|
|
||||||
|
Document A ("${a.name}"):
|
||||||
|
\`\`\`markdown
|
||||||
|
${stripHeader(a.content)}
|
||||||
|
\`\`\`
|
||||||
|
|
||||||
|
Document B ("${b.name}"):
|
||||||
|
\`\`\`markdown
|
||||||
|
${stripHeader(b.content)}
|
||||||
|
\`\`\``,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
private parseFrontmatter(content: string): {fm: Map<string, string>, body: string} {
|
private parseFrontmatter(content: string): {fm: Map<string, string>, body: string} {
|
||||||
const match = content.match(/^---\n([\s\S]*?)\n---\n?([\s\S]*)$/);
|
const match = content.match(/^---\n([\s\S]*?)\n---\n?([\s\S]*)$/);
|
||||||
if (!match) return {fm: new Map(), body: content};
|
if (!match) return {fm: new Map(), body: content};
|
||||||
@@ -415,15 +510,15 @@ ${currentBody}
|
|||||||
for (const line of match[1].split('\n')) {
|
for (const line of match[1].split('\n')) {
|
||||||
const i = line.indexOf(':');
|
const i = line.indexOf(':');
|
||||||
if (i === -1) continue;
|
if (i === -1) continue;
|
||||||
fm.set(line.slice(0, i).trim(), line.slice(i + 1).trim());
|
const key = line.slice(0, i).trim();
|
||||||
|
const raw = line.slice(i + 1).trim();
|
||||||
|
let value = raw;
|
||||||
|
try { value = JSON.parse(raw); } catch { /* legacy unquoted value, keep raw */ }
|
||||||
|
fm.set(key, value);
|
||||||
}
|
}
|
||||||
return {fm, body: match[2]};
|
return {fm, body: match[2]};
|
||||||
}
|
}
|
||||||
|
|
||||||
private stripHeader(content: string): string {
|
|
||||||
return content.replace(/^---[\s\S]*?\n---\n?/, '').trimStart();
|
|
||||||
}
|
|
||||||
|
|
||||||
private touchHeader(node: Memory, body: string): string {
|
private touchHeader(node: Memory, body: string): string {
|
||||||
const {fm} = this.parseFrontmatter(node.content);
|
const {fm} = this.parseFrontmatter(node.content);
|
||||||
fm.set('name', node.name);
|
fm.set('name', node.name);
|
||||||
@@ -433,7 +528,7 @@ ${currentBody}
|
|||||||
}
|
}
|
||||||
|
|
||||||
private writeFrontmatter(fm: Map<string, string>, body: string): string {
|
private writeFrontmatter(fm: Map<string, string>, body: string): string {
|
||||||
const lines = [...fm.entries()].map(([k, v]) => `${k}: ${v}`);
|
const lines = [...fm.entries()].map(([k, v]) => `${k}: ${JSON.stringify(String(v).replace(/\s+/g, ' ').trim())}`);
|
||||||
return `---\n${lines.join('\n')}\n---\n\n${body.trimStart()}`;
|
return `---\n${lines.join('\n')}\n---\n\n${body.trimStart()}`;
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -498,20 +593,34 @@ ${currentBody}
|
|||||||
history.push(pending);
|
history.push(pending);
|
||||||
|
|
||||||
const store = this.access(memories);
|
const store = this.access(memories);
|
||||||
const buckets = await this.factAgent(conversation, store, options, this.getWeekMonday());
|
const {buckets, journal} = await this.factAgent(conversation, store, options);
|
||||||
const touched: Memory[] = [];
|
const touched: Memory[] = [];
|
||||||
|
|
||||||
|
if (journal) {
|
||||||
|
const journalName = `Journal/${this.getWeekMonday()}`;
|
||||||
|
let jnode = store.find(journalName);
|
||||||
|
if (!jnode) {
|
||||||
|
jnode = {name: journalName, description: '', content: '', embedding: [], links: [], backlinks: []};
|
||||||
|
store.list.push(jnode);
|
||||||
|
}
|
||||||
|
this.stage(jnode, `### ${new Date().toISOString().slice(0, 10)}\n${journal}`);
|
||||||
|
touched.push(jnode);
|
||||||
|
}
|
||||||
|
|
||||||
for (const {subject, facts} of buckets) {
|
for (const {subject, facts} of buckets) {
|
||||||
let node = store.find(subject);
|
let node = store.find(subject);
|
||||||
if (!node) {
|
if (!node) {
|
||||||
node = {name: subject, description: '', content: '', embedding: [], links: [], backlinks: []};
|
node = {name: subject, description: '', content: '', embedding: [], links: [], backlinks: []};
|
||||||
store.list.push(node);
|
store.list.push(node);
|
||||||
}
|
}
|
||||||
this.appendFacts(node, facts);
|
this.stage(node, facts.map(f => `- ${f}`).join('\n'));
|
||||||
const [e] = await this.llm.embedding(node.content);
|
touched.push(node);
|
||||||
|
}
|
||||||
|
|
||||||
|
for (const node of touched) {
|
||||||
|
const [e] = await this.llm.embedding(`${node.description}\n\n${stripHeader(node.content)}`.trim());
|
||||||
if (e) node.embedding = e.embedding;
|
if (e) node.embedding = e.embedding;
|
||||||
this.touch(node.name);
|
this.touch(node.name);
|
||||||
touched.push(node);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
if (touched.length) {
|
if (touched.length) {
|
||||||
@@ -526,9 +635,9 @@ ${currentBody}
|
|||||||
return touched;
|
return touched;
|
||||||
}
|
}
|
||||||
|
|
||||||
async reconcileVault(memories: Memory[] | MemoryCache, options: LLMRequest, scope: 'touched' | 'all' = 'touched'): Promise<void> {
|
async reconcileAll(memories: Memory[] | MemoryCache, options: LLMRequest, scope: 'touched' | 'all' = 'touched'): Promise<void> {
|
||||||
const store = this.access(memories);
|
const store = this.access(memories);
|
||||||
const targets = scope === 'all' ? store.list : store.list.filter(m => m.content.includes(FACTS_HEADING));
|
const targets = scope === 'all' ? store.list : store.list.filter(m => m.content.includes(PENDING_HEADING));
|
||||||
await Promise.all(targets.map(node => this.reconcile(node, memories, options)));
|
await Promise.all(targets.map(node => this.reconcile(node, memories, options)));
|
||||||
store.commit();
|
store.commit();
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -25,6 +25,13 @@ export class OpenAi extends LLMProvider {
|
|||||||
return client;
|
return client;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
private toWireContent(content: any): any {
|
||||||
|
if(!Array.isArray(content)) return content;
|
||||||
|
return content.map(c => c.type === 'image'
|
||||||
|
? {type: 'image_url', image_url: {url: `data:${c.mime};base64,${c.data}`}}
|
||||||
|
: {type: 'text', text: c.text});
|
||||||
|
}
|
||||||
|
|
||||||
/** Convert standard history -> OpenAI wire format */
|
/** Convert standard history -> OpenAI wire format */
|
||||||
private toWire(history: LLMMessage[], system?: string): any[] {
|
private toWire(history: LLMMessage[], system?: string): any[] {
|
||||||
const wire: any[] = [];
|
const wire: any[] = [];
|
||||||
@@ -41,7 +48,7 @@ export class OpenAi extends LLMProvider {
|
|||||||
content: h.error || h.content || '',
|
content: h.error || h.content || '',
|
||||||
});
|
});
|
||||||
} else {
|
} else {
|
||||||
wire.push({role: h.role, content: h.content});
|
wire.push({role: h.role, content: this.toWireContent(h.content)});
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
return wire;
|
return wire;
|
||||||
@@ -58,7 +65,7 @@ export class OpenAi extends LLMProvider {
|
|||||||
const requestParams: any = {
|
const requestParams: any = {
|
||||||
model: options.model || this.model,
|
model: options.model || this.model,
|
||||||
stream: !!options.stream,
|
stream: !!options.stream,
|
||||||
max_completion_tokens: options.max_tokens || this.ai.options.llm?.max_tokens || undefined,
|
max_completion_tokens: options.maxTokens || this.ai.options.llm?.maxTokens || undefined,
|
||||||
temperature: options.temperature || this.ai.options.llm?.temperature || undefined,
|
temperature: options.temperature || this.ai.options.llm?.temperature || undefined,
|
||||||
tools: tools.map(t => ({
|
tools: tools.map(t => ({
|
||||||
type: 'function',
|
type: 'function',
|
||||||
|
|||||||
Reference in New Issue
Block a user