From 18dbd98e1967b3df8c8783d6ea026628c0f017aa Mon Sep 17 00:00:00 2001 From: ztimson Date: Thu, 20 Aug 2026 11:17:45 -0400 Subject: [PATCH] New Zim utilties --- LICENSE | 11 --- README.md | 187 +++++++++++++++++++----------------- package-lock.json | 52 ++++++++++ package.json | 19 ++++ src/catalog.js | 73 ++++++++++++++ src/index.js | 4 + src/manager.js | 163 ++++++++++++++++++++++++++++++++ src/reader.js | 235 ++++++++++++++++++++++++++++++++++++++++++++++ src/utils.js | 32 +++++++ 9 files changed, 681 insertions(+), 95 deletions(-) delete mode 100644 LICENSE create mode 100644 package-lock.json create mode 100644 package.json create mode 100644 src/catalog.js create mode 100644 src/index.js create mode 100644 src/manager.js create mode 100644 src/reader.js create mode 100644 src/utils.js diff --git a/LICENSE b/LICENSE deleted file mode 100644 index fc50bed..0000000 --- a/LICENSE +++ /dev/null @@ -1,11 +0,0 @@ -Copyright (c) 2023 Zakary Timson - -All Rights Reserved. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -THE SOFTWARE. diff --git a/README.md b/README.md index 62d4043..471df75 100644 --- a/README.md +++ b/README.md @@ -1,96 +1,59 @@

- + Logo - ### Template - +### Zim Utils + - Simple repository template +Native, dependency-light ZIM archive reader/searcher and Kiwix catalog downloader for Node.js - [![Version](https://img.shields.io/badge/dynamic/json.svg?label=Version&style=for-the-badge&url=https://git.zakscode.com/api/v1/repos/ztimson/template/tags&query=$[0].name)](https://git.zakscode.com/ztimson/template/tags) - [![Pull Requests](https://img.shields.io/badge/dynamic/json.svg?label=Pull%20Requests&style=for-the-badge&url=https://git.zakscode.com/api/v1/repos/ztimson/template&query=open_pr_counter)](https://git.zakscode.com/ztimson/template/pulls) - [![Issues](https://img.shields.io/badge/dynamic/json.svg?label=Issues&style=for-the-badge&url=https://git.zakscode.com/api/v1/repos/ztimson/template&query=open_issues_count)](https://git.zakscode.com/ztimson/template/issues) +[![Version](https://img.shields.io/badge/dynamic/json.svg?label=Version&style=for-the-badge&url=https://git.zakscode.com/api/v1/repos/ztimson/zim-utils/tags&query=$[0].name)](https://git.zakscode.com/ztimson/zim-utils/tags) +[![Pull Requests](https://img.shields.io/badge/dynamic/json.svg?label=Pull%20Requests&style=for-the-badge&url=https://git.zakscode.com/api/v1/repos/ztimson/zim-utils&query=open_pr_counter)](https://git.zakscode.com/ztimson/zim-utils/pulls) +[![Issues](https://img.shields.io/badge/dynamic/json.svg?label=Issues&style=for-the-badge&url=https://git.zakscode.com/api/v1/repos/ztimson/zim-utils&query=open_issues_count)](https://git.zakscode.com/ztimson/zim-utils/issues) ---
- Documentation - • Release Notes - • Report a Bug - • Request a Feature + Documentation + • Release Notes + • Report a Bug + • Request a Feature
---
## Table of Contents -- [Template](#top) - - [About](#about) - - [Demo](#demo) - - [Built With](#built-with) - - [Setup](#setup) - - [Production](#production) - - [Development](#development) - - [License](#license) +- [Zim Utils](#top) + - [About](#about) + - [Built With](#built-with) + - [Setup](#setup) + - [Production](#production) + - [Usage](#usage) + - [ZimManager](#zimmanager) + - [ZimReader](#zimreader) + - [Catalog](#catalog) + - [License](#license) ## About -Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum. +`@ztimson/zim-utils` is a native, dependency-light toolkit for working with [ZIM](https://wiki.openzim.org/wiki/ZIM_file_format) archives and the [Kiwix](https://kiwix.org/) catalog in Node.js -### Demo +It comes with the following helpers: -Website: https://git.zakscode.com +- **`ZimManager`** — Local library manager: listing, updating/downloading, and running searches accross the entire library +- **`ZimReader`** — A `.zim` file reader for reading pages, metadata and running searches +- **Catalog** (`zimCatalog`/`zimCatalogInfo`) — Helper functions to search the Kiwix OPDS catalog ### Built With -[![Angular](https://img.shields.io/badge/Angular-DD0031?style=for-the-badge&logo=angular)](https://angular.io/) -[![Android](https://img.shields.io/badge/android-34A853?style=for-the-badge&logo=android&logoColor=ffffff)](https://www.android.com/) -[![Arduino](https://img.shields.io/badge/Arduino-00878F?style=for-the-badge&logo=arduino&logoColor=white)](https://www.arduino.cc/) -[![Bootstrap](https://img.shields.io/badge/Bootstrap-563D7C?style=for-the-badge&logo=bootstrap&logoColor=white)](https://getbootstrap.com) -[![C](https://img.shields.io/badge/C-A8B9CC?style=for-the-badge&logo=c&logoColor=ffffff)](https://en.cppreference.com/w/c/language) -[![C++](https://img.shields.io/badge/C%2B%2B-00599C?style=for-the-badge&logo=cplusplus)](https://cplusplus.com/) -[![C#](https://img.shields.io/badge/C%23-239120?style=for-the-badge&logo=csharp)](https://dotnet.microsoft.com/) -[![CSS](https://img.shields.io/badge/CSS-1572B6?style=for-the-badge&logo=css3)](https://www.w3.org/Style/CSS/Overview.en.html) -[![Django](https://img.shields.io/badge/django-0C4B33?style=for-the-badge&logo=django)](https://www.djangoproject.com/) -[![Docker](https://img.shields.io/badge/Docker-384d54?style=for-the-badge&logo=docker)](https://docker.com/) -[![Electron](https://img.shields.io/badge/Electron-47848F?style=for-the-badge&logo=electron&logoColor=white)](https://www.electronjs.org/) -[![Firebase](https://img.shields.io/badge/Firebase-FFFFFF?style=for-the-badge&logo=firebase)](https://firebase.google.com/) -[![Go](https://img.shields.io/badge/Go-00ADD8?style=for-the-badge&logo=go&logoColor=ffffff)](https://go.dev/) -[![GraphQL](https://img.shields.io/badge/GraphQL-E10098?style=for-the-badge&logo=graphql)](https://graphql.org/) -[![HTML](https://img.shields.io/badge/HTML-FFFFFF?style=for-the-badge&logo=html5)](https://developer.mozilla.org/en-US/docs/Glossary/HTML) -[![Java](https://img.shields.io/badge/Java-5382A1?style=for-the-badge&logo=coffeescript&logoColor=F8981D)](https://java.com/) -[![JavaScript](https://img.shields.io/badge/JavaScript-000000?style=for-the-badge&logo=javascript)](https://javascript.com/) -[![JQuery](https://img.shields.io/badge/jQuery-0769AD?style=for-the-badge&logo=jquery)](https://jquery.com ) -[![Laravel](https://img.shields.io/badge/Laravel-6C6C6C?style=for-the-badge&logo=laravel)](https://laravel.com) -[![Linux](https://img.shields.io/badge/Linux-eeeeee?style=for-the-badge&logo=linux&logoColor=000000)](https://www.linux.org/) -[![Momentum](https://img.shields.io/badge/Momentum-000000?style=for-the-badge&logo=data:image/svg%2bxml;base64,PD94bWwgdmVyc2lvbj0iMS4wIiBzdGFuZGFsb25lPSJubyI/Pgo8IURPQ1RZUEUgc3ZnIFBVQkxJQyAiLS8vVzNDLy9EVEQgU1ZHIDIwMDEwOTA0Ly9FTiIgImh0dHA6Ly93d3cudzMub3JnL1RSLzIwMDEvUkVDLVNWRy0yMDAxMDkwNC9EVEQvc3ZnMTAuZHRkIj4KPHN2ZyB2ZXJzaW9uPSIxLjAiIHhtbG5zPSJodHRwOi8vd3d3LnczLm9yZy8yMDAwL3N2ZyIgd2lkdGg9IjI1MHB0IiBoZWlnaHQ9IjI1MHB0IiB2aWV3Qm94PSIwIDAgMjUwIDI1MCIgIHByZXNlcnZlQXNwZWN0UmF0aW89InhNaWRZTWlkIG1lZXQiPgoJPHN0eWxlPgoJCUBtZWRpYSAocHJlZmVycy1jb2xvci1zY2hlbWU6IGxpZ2h0KSB7CgkJZyB7IGZpbGw6ICMwMDAwMDA7IH0KCQl9CgkJQG1lZGlhIChwcmVmZXJzLWNvbG9yLXNjaGVtZTogZGFyaykgewoJCWcgeyBmaWxsOiAjZmZmZmZmOyB9CgkJfQoJPC9zdHlsZT4KCTxnIHRyYW5zZm9ybT0idHJhbnNsYXRlKDAsMjUwKSBzY2FsZSgwLjEsLTAuMSkiPgoJCTxwYXRoIGQ9Ik0xMTY1IDIyOTkgYy0yNDUgLTE4IC00NzAgLTEyMyAtNjUxIC0zMDMgLTExNyAtMTE4IC0yMDAgLTI1MCAtMjUyIC00MDEgLTcxIC0yMDkgLTcxIC00NjggLTEgLTY3OSAzNyAtMTA4IDExOCAtMjU0IDE3OCAtMzE5IGwyNSAtMjggLTEzMiAtMTMyIC0xMzIgLTEzMiA1MiAtNTMgNTMgLTUyIDEzMiAxMzIgMTMyIDEzMiAyOCAtMjUgYzQxIC0zOCAxMDkgLTgyIDE4OCAtMTIxIDE1MyAtNzcgMjkyIC0xMDkgNDcwIC0xMDkgMjkxIDAgNTM2IDEwMSA3NDEgMzA1IDE1MyAxNTQgMjQ0IDMyNCAyODkgNTQwIDIxIDEwMyAyMSAyOTkgLTEgNDAyIC0zMiAxNTkgLTEwNSAzMjQgLTE4NyA0MjYgbC00OSA2MSAxMjIgMTIyIDEyMyAxMjMgLTIyIDMxIGMtMTIgMTYgLTM2IDQwIC01MiA1MiBsLTMxIDIyIC0xMjEgLTEyMSAtMTIyIC0xMjEgLTYwIDQ2IGMtODEgNjEgLTI2MSAxNDcgLTM2NSAxNzMgLTEwNCAyNyAtMjM1IDM3IC0zNTUgMjl6IG0yNzQgLTE2NCBjMTAyIC0yMSAyMTcgLTY4IDI5NyAtMTIxIDEwOSAtNzIgMTA2IC02NiA0OSAtMTI0IGwtNTAgLTUwIC01MCA5IGMtODMgMTYgLTE2OSAtNSAtMzA1IC03NCBsLTc1IC0zOCA0NSAtMzggYzI1IC0yMiA1MyAtNDIgNjMgLTQ1IDExIC0zIDQ1IDYgNzcgMjAgNjggMzEgMTM1IDQ1IDE3MCAzNiAyMiAtNSAyNSAtMTIgMjggLTU2IDEyIC0yMDQgLTM2MCAtNjQ1IC02NzQgLTgwMCAtMTA3IC01MyAtMTgwIC02NiAtMjA4IC0zOCAtMjcgMjcgLTE4IDExMyAyMiAxOTggMTggMzcgMjkgNzMgMjUgODAgLTE0IDI0IC03NiA5NiAtODIgOTYgLTEyIDAgLTc4IC0xMzIgLTEwMSAtMTk5IC0xNiAtNDkgLTIxIC04NCAtMTkgLTE0OCBsMyAtODQgLTQxIC00MSBjLTUwIC01MCAtNTIgLTQ5IC0xMTkgNTggLTU1IDg4IC05OCAxOTYgLTExOSAyOTQgLTE5IDkxIC0xOSAyNzkgMCAzNjkgNzIgMzQzIDM0NyA2MjEgNjg4IDY5NSA4NSAxOSAyODkgMTkgMzc2IDF6IG01NjkgLTM5MyBjNjMgLTk1IDEwNSAtMTk1IDEyOCAtMzA1IDE4IC04OSAxOCAtMjc3IC0xIC0zNjcgLTczIC0zNDYgLTM0OSAtNjIyIC02OTUgLTY5NSAtOTEgLTE5IC0yNzggLTE5IC0zNjggMCAtMTMwIDI3IC0yNzggOTUgLTM2NyAxNjkgbC0zMCAyNSA0NSA0NSBjNDQgNDUgNDUgNDYgMTEwIDQ2IDE3MyAwIDM5MiAxMjQgNjI1IDM1NSAxMzAgMTI5IDIwNCAyMjIgMjc4IDM1MCA2MyAxMDkgODkgMTg1IDk0IDI4MCBsNSA4MCA1MSA1MyBjMjggMjggNTUgNTIgNTkgNTIgNSAwIDM1IC00MCA2NiAtODh6Ii8+CgkJPHBhdGggZD0iTTc1MiAxODM5IGMtMTk1IC05NyAtOTMgLTQzMyAyMzggLTc5MSA3MSAtNzcgNzIgLTc4IDEwMCAtNjMgMTUgOCA0MiAyNiA2MCA0MSBsMzEgMjcgLTY5IDcxIGMtMTc2IDE4MSAtMjk3IDM3NCAtMzE3IDUwOCAtMTQgOTIgMzIgMTEwIDE1OCA2NSAxMjAgLTQ0IDI2OCAtMTQ3IDQwOSAtMjg3IGw4OCAtODcgMTkgMjEgYzEwIDEyIDI4IDM2IDM5IDU1IGwyMSAzNCAtMjYgMzEgYy00NyA1NyAtMTk3IDE4NiAtMjg1IDI0NyAtMTk2IDEzNCAtMzYxIDE4MCAtNDY2IDEyOHoiLz4KCQk8cGF0aCBkPSJNMTAxMiAxNDk3IGMtMjkgLTI5IC01MiAtNTggLTUyIC02NiAwIC0xMyA1NCAtOTYgNjggLTEwNSA1IC0zIDQ0IDMwIDg2IDczIGw3OCA3OSAtNTMgMzYgYy0zMCAyMCAtNTggMzYgLTY0IDM2IC02IDAgLTM0IC0yNCAtNjMgLTUzeiIvPgoJCTxwYXRoIGQ9Ik0xNjcyIDExMzkgYy00NiAtNTYgLTQ5IC02OSAtMjggLTEwMiAyMyAtMzUgNTYgLTE0MyA1NiAtMTgxIDAgLTc4IC01OSAtODMgLTIwNiAtMjAgbC04MyAzNiAtNTEgLTQ0IGMtMjcgLTI0IC01MCAtNDYgLTUwIC01MCAwIC0xNCAxODYgLTk5IDI1MiAtMTE0IDIwMCAtNDkgMzE3IDc4IDI2MyAyODYgLTE2IDYzIC03MCAxODYgLTk3IDIyMyAtMTMgMTcgLTE3IDE0IC01NiAtMzR6Ii8+Cgk8L2c+Cjwvc3ZnPgo=)](https://git.zakscode.com/ztimson/momentum) -[![MongoDB](https://img.shields.io/badge/mongodb-000000?style=for-the-badge&logo=mongodb)](https://www.mongodb.com/) -[![MySQL](https://img.shields.io/badge/MySQL-4479A1?style=for-the-badge&logo=mysql&logoColor=ffffff)](https://www.mysql.com/) -[![Nest](https://img.shields.io/badge/nestjs-E0234E?style=for-the-badge&logo=nestjs)](https://nestjs.com/) -[![.NET](https://img.shields.io/badge/.NET-512BD4?style=for-the-badge&logo=dotnet)](https://dotnet.microsoft.com/) -[![Next](https://img.shields.io/badge/next.js-000000?style=for-the-badge&logo=nextdotjs)](https://nextjs.org/) -[![NGINX](https://img.shields.io/badge/NGINX-009639?style=for-the-badge&logo=nginx)](https://www.nginx.com/) [![Node](https://img.shields.io/badge/Node.js-000000?style=for-the-badge&logo=nodedotjs)](https://nodejs.org/) -[![p5.js](https://img.shields.io/badge/p5.js-ed225d?style=for-the-badge&logo=p5dotjs&logoColor=white)](https://p5js.org/) -[![PHP](https://img.shields.io/badge/PHP-474A8A?style=for-the-badge&logo=php&logoColor=white)](https://www.php.net/) -[![PostgreSQL](https://img.shields.io/badge/PostgreSQl-212121?style=for-the-badge&logo=postgresql)](https://www.postgresql.org/) -[![Python](https://img.shields.io/badge/Python-FFD43B?style=for-the-badge&logo=python)](https://www.python.org/) -[![React](https://img.shields.io/badge/React-20232A?style=for-the-badge&logo=react)](https://reactjs.org/) -[![Redis](https://img.shields.io/badge/Redis-ffffff?style=for-the-badge&logo=redis)](https://redis.com/) -[![SASS](https://img.shields.io/badge/SASS-CC6699?style=for-the-badge&logo=sass&logoColor=ffffff)](https://sass-lang.com/) -[![Shell](https://img.shields.io/badge/Shell-000000?style=for-the-badge&logo=windowsterminal&logoColor=00ff00)](https://en.wikipedia.org/wiki/Shell_script) -[![SQL Server](https://img.shields.io/badge/SQL%20Server-CC2927?style=for-the-badge&logo=microsoftsqlserver)](https://www.microsoft.com/en-ca/sql-server) -[![SQLite](https://img.shields.io/badge/SQLITE-003B57?style=for-the-badge&logo=sqlite)](https://www.sqlite.org/index.html) -[![Svelte](https://img.shields.io/badge/Svelte-4A4A55?style=for-the-badge&logo=svelte)](https://svelte.dev/) -[![TypeScript](https://img.shields.io/badge/TypeScript-3178C6?style=for-the-badge&logo=typescript&logoColor=white)](https://typescriptlang.org/) -[![Windows](https://img.shields.io/badge/Windows-0078D4?style=for-the-badge&logo=windows)](https://microsoft.com/windows) -[![Vite](https://img.shields.io/badge/vite-1b1b1b?style=for-the-badge&logo=vite)](https://vitejs.dev/) -[![Vue](https://img.shields.io/badge/Vue.js-35495E?style=for-the-badge&logo=vuedotjs)](https://vuejs.org/) +[![JavaScript](https://img.shields.io/badge/JavaScript-000000?style=for-the-badge&logo=javascript)](https://javascript.com/) ## Setup @@ -101,32 +64,88 @@ Website: https://git.zakscode.com -#### Prerequisites -- [Docker](https://docs.docker.com/install/) - -#### Instructions -1. Run the docker image: `docker run -p 80:80 git.zakscode.com/ztimson/template:latest` -2. Open [http://localhost](http://localhost) - - -
- -

- Development -

- - #### Prerequisites - [Node.js](https://nodejs.org/en/download) #### Instructions 1. Install the dependencies: `npm install` -2. Start the Angular server: `npm run start` -3. Open [http://localhost:4200](http://localhost:4200) +2. Install the package: `npm install @ztimson/zim-utils`
-## License -Copyright © 2023 Zakary Timson | All Rights Reserved | Available under MIT Licensing +## Usage + +### ZimManager + +`ZimManager` owns a directory of `.zim` files and handles everything from downloading to cross-archive search. + +```js +import {ZimManager} from '@ztimson/zim-utils'; + +const manager = new ZimManager('./zims'); // optional 2nd arg: custom catalog URL + +// Search the Kiwix catalog & download the top hit +const [entry] = await manager.catalog('wikipedia,medicine'); +await manager.download(entry.href); + +// List local archives with their parsed metadata +const local = await manager.list(); +// [{file: './zims/wikipedia_en_medicine.zim', meta: {name, date, title}}, ...] + +// Check a single file for updates without downloading +const status = await manager.isOutdated(local[0].file); + +// Update every local ZIM that has a newer catalog version +await manager.updateAll({force: false}); + +// Fuzzy-search titles across ALL local archives at once +const hits = await manager.search('diabetes treatment', {limit: 10}); + +// Open a reader by file path OR by catalog name +const reader = await manager.open('wikipedia_en_medicine'); +const page = await reader.readPage('A/Diabetes'); +await reader.close(); +``` + +### ZimReader +Everything `ZimManager` does to a single archive is just a thin wrapper around `ZimReader`. Use it directly when you don't need a whole managed library: + +```js +import {ZimReader} from '@ztimson/zim-utils'; + +const reader = await new ZimReader('./zims/wikipedia_en_medicine.zim').open(); + +// Metadata (what manager.#readMeta / isOutdated rely on) +const name = await reader.metadata('Name'); +const date = await reader.metadata('Date'); +const title = await reader.metadata('Title'); + +// Landing page +const home = await reader.mainPage(); + +// Direct page lookup by URL +const page = await reader.readPage('A/Diabetes'); +console.log(page.mimetype, page.data.toString('utf8')); + +// Fuzzy title search within just this archive (what manager.search fans out over) +const results = await reader.search('diabetes,insulin', {limit: 20, htmlOnly: true}); + +await reader.close(); +``` + +### Catalog +`ZimManager.catalog()` and its update checks are backed directly by these two functions: + +```js +import {zimCatalog, zimCatalogInfo, CATALOG_URL} from '@ztimson/zim-utils'; + +// Ranked search across the Kiwix catalog (comma-separated terms, like ZimReader.search) +const results = await zimCatalog('history,rome', {lang: 'eng', count: 20, url: CATALOG_URL}); + +// Exact lookup by catalog `name`, used to check if a local copy is outdated +const entry = await zimCatalogInfo('wikipedia_en_medicine'); +``` + +## License +Copyright © 2026 Zakary Timson | Available under MIT Licensing -See the [license](./LICENSE) for more information. diff --git a/package-lock.json b/package-lock.json new file mode 100644 index 0000000..cba0faf --- /dev/null +++ b/package-lock.json @@ -0,0 +1,52 @@ +{ + "name": "@ztimson/zim-utils", + "version": "0.1.0", + "lockfileVersion": 3, + "requires": true, + "packages": { + "": { + "name": "@ztimson/zim-utils", + "version": "0.1.0", + "license": "MIT", + "dependencies": { + "@ztimson/utils": "^0.30.7", + "lzma1": "^0.3.0", + "zstd-codec": "^0.1.5" + } + }, + "node_modules/@ztimson/utils": { + "version": "0.30.7", + "resolved": "https://registry.npmjs.org/@ztimson/utils/-/utils-0.30.7.tgz", + "integrity": "sha512-0TkjQFVe0edTqlezIfUtwTXA9pkS1rVYw0xzFycKmtcoldh6urQEKQ/T6Z0Id1+w53ubomSdVfj6GB3u29+AOQ==", + "license": "MIT", + "dependencies": { + "var-persist": "^1.0.1" + } + }, + "node_modules/lzma1": { + "version": "0.3.0", + "resolved": "https://registry.npmjs.org/lzma1/-/lzma1-0.3.0.tgz", + "integrity": "sha512-ETg5CT6Ojybzjx2YG48NuPcN7kGzhLO6FfKbkvUwt0bxIEOq+wXykCZBoNBrkGq66zwJT1ZdUI71F0wC6pP9TQ==", + "license": "MIT", + "engines": { + "bun": ">=1.0.0", + "node": ">=18.0.0" + }, + "funding": { + "url": "https://github.com/sponsors/xseman" + } + }, + "node_modules/var-persist": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/var-persist/-/var-persist-1.0.1.tgz", + "integrity": "sha512-Zon+pwvEpb0dEQCVShoMQQWV1JbWi4P2knW3h2sfSZS3pLecgbFig76tMSHEECQuEQ3KfYhMXgRDDIybtHTyZw==", + "license": "MIT" + }, + "node_modules/zstd-codec": { + "version": "0.1.5", + "resolved": "https://registry.npmjs.org/zstd-codec/-/zstd-codec-0.1.5.tgz", + "integrity": "sha512-v3fyjpK8S/dpY/X5WxqTK3IoCnp/ZOLxn144GZVlNUjtwAchzrVo03h+oMATFhCIiJ5KTr4V3vDQQYz4RU684g==", + "license": "MIT" + } + } +} diff --git a/package.json b/package.json new file mode 100644 index 0000000..fb7bd6c --- /dev/null +++ b/package.json @@ -0,0 +1,19 @@ +{ + "name": "@ztimson/zim-utils", + "version": "0.1.0", + "description": "Native, dependency-light ZIM archive reader/searcher and Kiwix catalog downloader for Node.js", + "author": "Zak Timson", + "license": "MIT", + "private": false, + "type": "module", + "repository": { + "type": "git", + "url": "https://git.zakscode.com/ztimson/zim-utils" + }, + "main": "index.js", + "dependencies": { + "@ztimson/utils": "^0.30.7", + "lzma1": "^0.3.0", + "zstd-codec": "^0.1.5" + } +} diff --git a/src/catalog.js b/src/catalog.js new file mode 100644 index 0000000..66c27dd --- /dev/null +++ b/src/catalog.js @@ -0,0 +1,73 @@ +import {fuzzyMatch} from './utils.js'; +import {decodeHtml} from '@ztimson/utils'; + +export const CATALOG_URL = 'https://library.kiwix.org/catalog/v2/entries'; + +const PAGE_SIZE = 100; + +/** Parses `` blocks out of a Kiwix catalog OPDS XML response. */ +function parseEntries(xml) { + const blocks = xml.match(/[\s\S]*?<\/entry>/g) || []; + return blocks.map(b => { + const grab = re => (b.match(re) || [])[1] || ''; + const linkMatch = b.match(/]*type=["']application\/x-zim[^"']*["'][^>]*href=["']([^"']+)["']/); + let tags = grab(/([^<]*)<\/tags>/); + if(tags) tags = tags.split(';'); + return { + id: grab(/([^<]*)<\/id>/), + title: decodeHtml(grab(/([^<]*)<\/title>/)), + updated: new Date(grab(/<updated>([^<]*)<\/updated>/)), + summary: decodeHtml(grab(/<summary>([^<]*)<\/summary>/)), + language: grab(/<language>([^<]*)<\/language>/), + name: grab(/<name>([^<]*)<\/name>/), + category: grab(/<category>([^<]*)<\/category>/), + tags, + mediaCount: Number(grab(/<mediaCount>([^<]*)<\/mediaCount>/)) || 0, + author: grab(/<author>\s*<name>([^<]*)<\/name>\s*<\/author>/m), + publisher: grab(/<publisher>\s*<name>([^<]*)<\/name>\s*<\/publisher>/m), + articleCount: Number(grab(/<articleCount>([^<]*)<\/articleCount>/)) || 0, + sizeMb: linkMatch ? (Number((b.match(/length=["'](\d+)["']/) || [])[1] || 0) / 1024 / 1024).toFixed(1) : '?', + href: linkMatch ? linkMatch[1] : null, + }; + }); +} + +async function fetchEntries(term, lang, url = CATALOG_URL) { + const params = new URLSearchParams({q: term, count: String(PAGE_SIZE), lang: lang || 'eng'}); + const res = await fetch(`${url}?${params}`); + if (!res.ok) throw new Error(`${res.status} ${res.statusText}`); + return parseEntries(await res.text()); +} + +/** Looks up a single catalog entry by exact `name` (used to check for available updates). */ +export async function zimCatalogInfo(name, url = CATALOG_URL) { + const params = new URLSearchParams({name, count: '5'}); + const res = await fetch(`${url}?${params}`); + if (!res.ok) return null; + return parseEntries(await res.text()).find(e => e.name === name) || null; +} + +/** Searches the Kiwix catalog for ZIMs matching `terms` within `category`, ranked by term coverage then fuzzy similarity. */ +export async function zimCatalog(terms, opts = {lang: 'eng', count: 20, url: CATALOG_URL}) { + opts = Object.assign({lang: 'eng', count: 20, url: CATALOG_URL}, opts) + const termList = [...String(terms).split(',')].filter(Boolean).map(t => t.trim().toLowerCase()); + const results = await Promise.allSettled(termList.map(t => fetchEntries(t, opts.lang, opts.url))); + const byName = new Map(); // name -> {entry, hitTerms:Set} + results.forEach((r, i) => { + if (r.status !== 'fulfilled') return; + const term = termList[i]; + for (const entry of r.value) { + if (!entry.name) continue; + if (!byName.has(entry.name)) byName.set(entry.name, {entry, hitTerms: new Set()}); + byName.get(entry.name).hitTerms.add(term); + } + }); + if (!byName.size) return []; + + return [...byName.values()].map(({entry, hitTerms}) => { + const text = `${entry.title} ${entry.summary}`.trim(); + return {entry, hits: hitTerms.size, fuzzy: fuzzyMatch(text, ...termList).max}; + }).toSorted((a, b) => + b.hits - a.hits || b.fuzzy - a.fuzzy || b.entry.articleCount - a.entry.articleCount + ).slice(0, opts.count).map(r => r.entry); +} diff --git a/src/index.js b/src/index.js new file mode 100644 index 0000000..4ca7a76 --- /dev/null +++ b/src/index.js @@ -0,0 +1,4 @@ +export * from './catalog.js'; +export * from './manager.js'; +export * from './reader.js'; +export * from './utils.js'; diff --git a/src/manager.js b/src/manager.js new file mode 100644 index 0000000..47b3091 --- /dev/null +++ b/src/manager.js @@ -0,0 +1,163 @@ +import fs from 'node:fs'; +import path from 'node:path'; +import {pipeline} from 'node:stream/promises'; +import {Readable} from 'node:stream'; +import {ZimReader} from './reader.js'; +import {zimCatalog, zimCatalogInfo, CATALOG_URL} from './catalog.js'; + +/** Manages a local directory of ZIM archives: listing, update checks, downloads, and reading. */ +export class ZimManager { + #catalog; + #dir; + + constructor(dir, catalog = CATALOG_URL) { + this.#catalog = catalog; + this.#dir = dir; + } + + async #download(url, destPath) { + const {res} = await this.#resolveUrl(url); + if (!res.ok) throw new Error(`${res.status} ${res.statusText}`); + const tmpPath = `${destPath}.part`; + await pipeline(Readable.fromWeb(res.body), fs.createWriteStream(tmpPath)); + await fs.promises.rename(tmpPath, destPath); + } + + async #ensureDir() { + await fs.promises.mkdir(this.#dir, {recursive: true}); + } + + /** Resolves a `.meta4` metalink URL down to the real mirror `.zim` download URL. */ + async #resolveUrl(url) { + const head = await fetch(url); + const ct = head.headers.get('content-type') || ''; + if (!ct.includes('metalink') && !url.endsWith('.meta4')) return {res: head, url}; + const meta = await head.text(); + const m = meta.match(/<url[^>]*>([^<]+\.zim)<\/url>/); + if (!m) throw new Error('Could not resolve metalink mirror'); + const res = await fetch(m[1]); + return {res, url: m[1]}; + } + + /** Reads Name/Date/Title metadata from a local ZIM file. Returns null if unreadable. */ + async #readMeta(filepath) { + let reader; + try { + reader = await new ZimReader(filepath).open(); + return { + name: await reader.metadata('Name'), + date: await reader.metadata('Date'), + title: await reader.metadata('Title'), + }; + } catch { + return null; + } finally { + await reader?.close(); + } + } + + async #update(name, catalogEntry, localMatch, force) { + const remoteDate = catalogEntry.updated ? new Date(catalogEntry.updated) : null; + const localDate = localMatch?.meta?.date ? new Date(localMatch.meta.date) : null; + if (!force && localMatch && remoteDate && localDate && remoteDate <= localDate) + return {name, status: 'skipped', reason: 'up to date'}; + if (!catalogEntry.href) return {name, status: 'skipped', reason: 'missing download link'}; + + const filename = path.basename(new URL(catalogEntry.href).pathname).replace(/\.meta4$/i, ''); + const destPath = path.join(this.#dir, filename); + await this.#download(catalogEntry.href, destPath); + if (localMatch && localMatch.file !== destPath) await fs.promises.unlink(localMatch.file).catch(() => {}); + return {name, status: 'updated', file: destPath}; + } + + /** Resolves a file path or catalog `name` to a local file path. */ + async #resolveFile(fileOrName) { + if (fs.existsSync(fileOrName)) return fileOrName; + const local = await this.list(); + const match = local.find(l => l.meta?.name === fileOrName); + if (!match) throw new Error(`ZIM not found locally: ${fileOrName}`); + return match.file; + } + + catalog(search, opts) { + return zimCatalog(search, opts); + } + + /** Checks whether a local ZIM has a newer version in the catalog, without downloading. */ + async isOutdated(file) { + const meta = await this.#readMeta(file); + if (!meta?.name) return {file, upToDate: null, reason: 'no metadata'}; + const catalogEntry = await zimCatalog(meta.name, this.#catalog); + if (!catalogEntry) return {file, upToDate: null, reason: 'missing from catalog'}; + const remoteDate = catalogEntry.updated ? new Date(catalogEntry.updated) : null; + const localDate = meta.date ? new Date(meta.date) : null; + return {file, name: meta.name, upToDate: !!(remoteDate && localDate && remoteDate <= localDate), localDate, remoteDate}; + } + + /** Lists local `.zim` files with their parsed metadata (or `null` if unreadable). */ + async list() { + await this.#ensureDir(); + const files = (await fs.promises.readdir(this.#dir)).filter(f => f.endsWith('.zim')); + return Promise.all(files.map(async f => { + const file = path.join(this.#dir, f); + return {file, meta: await this.#readMeta(file)}; + })); + } + + /** Downloads/updates a single ZIM by direct href, matching against any existing local copy by name. */ + async download(href, {force = false} = {}) { + await this.#ensureDir(); + const {url: finalUrl} = await this.#resolveUrl(href); + const filename = path.basename(new URL(finalUrl).pathname).replace(/\.meta4$/i, ''); + const name = filename.replace(/\.zim$/i, '').replace(/_\d{4}-\d{2}(?:_\d+)?$/, ''); + + const local = await this.list(); + const localMatch = local.find(l => l.meta?.name === name) ?? null; + const catalogEntry = await zimCatalogInfo(name, this.#catalog) || {name, updated: null, href}; + return this.#update(name, catalogEntry, localMatch, force); + } + + /** Checks all local ZIMs against the catalog and updates any that are outdated. */ + async updateAll({force = false}) { + const local = await this.list(); + if (!local.length) return []; + + const results = []; + for (const {file, meta} of local) { + if (!meta?.name) { results.push({file, status: 'skipped', reason: 'no metadata'}); continue; } + const catalogEntry = await zimCatalogInfo(meta.name, this.#catalog); + if (!catalogEntry) { results.push({name: meta.name, status: 'skipped', reason: 'missing from catalog'}); continue; } + try { + results.push(await this.#update(meta.name, catalogEntry, {file, meta}, force)); + } catch (e) { + results.push({name: meta.name, status: 'error', reason: e.message}); + } + } + return results; + } + + /** Opens a `ZimReader` for a local ZIM, resolved by file path or catalog `name`. Caller must `.close()` it. */ + async open(fileOrName) { + await this.#ensureDir(); + const file = await this.#resolveFile(fileOrName); + return new ZimReader(file).open(); + } + + /** Fuzzy-searches titles across every local ZIM in the library, merging & re-ranking hits by score. */ + async search(terms, {limit = 20, htmlOnly = true} = {}) { + const local = await this.list(); + const perZim = await Promise.all(local.map(async ({file, meta}) => { + let reader; + try { + reader = await new ZimReader(file).open(); + const hits = await reader.search(terms, {limit, htmlOnly}); + return hits.map(h => ({...h, file, name: meta?.name})); + } catch { + return []; + } finally { + await reader?.close(); + } + })); + return perZim.flat().toSorted((a, b) => b.score - a.score).slice(0, limit); + } +} diff --git a/src/reader.js b/src/reader.js new file mode 100644 index 0000000..764a461 --- /dev/null +++ b/src/reader.js @@ -0,0 +1,235 @@ +'use strict'; + +import fs from 'node:fs'; +import {decompress as lzmaDecompress} from 'lzma1'; +import {ZstdCodec} from 'zstd-codec'; +import {fuzzyMatch} from './utils.js'; + +const HEADER_SIZE = 80; +const NS_CONTENT = 'C'; +const NS_METADATA = 'M'; +const TITLE_SENTINEL = 0xffffffffffffffffn; // Indicator -> ZIM v6+ archives with no title + +let zstdStreamingPromise = null; + +/** Lazily initialised, shared Zstd streaming decompressor (handles unknown-size frames). */ +function getZstd() { + if (!zstdStreamingPromise) { + zstdStreamingPromise = new Promise(resolve => ZstdCodec.run(zstd => resolve(new zstd.Streaming()))); + } + return zstdStreamingPromise; +} + +/** Rebuilds a full 13-byte "alone" LZMA header from libzim's truncated 5-byte one (unknown size). */ +function toLzmaAloneStream(body) { + const header = Buffer.concat([body.subarray(0, 5), Buffer.alloc(8, 0xff)]); + return Buffer.concat([header, body.subarray(5)]); +} + +/** Native, dependency-light reader for .zim archives. Supports zstd & LZMA cluster compression. */ +export class ZimReader { + #fd = null; + #header = null; + #mimeTypes = []; + #hasTitleListing = false; + + get articleCount() { return this.#header?.articleCount ?? 0; } + + constructor(path) { + this.path = path; + } + + /** Full O(n) scan over the URL pointer list, used when there's no title index. */ + async #allDirents() { + const dirents = []; + for (let i = 0; i < this.#header.articleCount; i++) { + dirents.push(await this.#readDirent(await this.#ptr64(this.#header.urlPtrPos, i))); + } + return dirents; + } + + /** Binary search the URL pointer list for namespace+url. */ + async #findByUrl(url, namespace) { + const key = namespace + url; + let lo = 0, hi = this.#header.articleCount - 1; + while (lo <= hi) { + const mid = (lo + hi) >> 1; + const dirent = await this.#readDirent(await this.#ptr64(this.#header.urlPtrPos, mid)); + const dirKey = dirent.namespace + dirent.url; + const cmp = key < dirKey ? -1 : key > dirKey ? 1 : 0; + if (cmp === 0) return dirent; + if (cmp < 0) hi = mid - 1; else lo = mid + 1; + } + return null; + } + + async #getBlob(clusterNumber, blobNumber) { + const start = await this.#ptr64(this.#header.clusterPtrPos, clusterNumber); + const isLast = clusterNumber === this.#header.clusterCount - 1; + const end = isLast + ? (await fs.promises.stat(this.path)).size + : await this.#ptr64(this.#header.clusterPtrPos, clusterNumber + 1); + + const raw = await this.#read(start, end - start); + const compType = raw[0] & 0x0f; + const extended = (raw[0] & 0x10) !== 0; + const body = raw.subarray(1); + + let data; + if (compType <= 1) data = body; + else if (compType === 4) data = Buffer.from(lzmaDecompress(toLzmaAloneStream(body))); + else if (compType === 5) data = Buffer.from((await getZstd()).decompress(new Uint8Array(body))); + else throw new Error(`Unsupported cluster compression type: ${compType}`); + if (!data) throw new Error(`Cluster ${clusterNumber} failed to decompress (compType ${compType})`); + + const readPtr = i => extended ? Number(data.readBigUInt64LE(i * 8)) : data.readUInt32LE(i * 4); + return data.subarray(readPtr(blobNumber), readPtr(blobNumber + 1)); + } + + async #ptr64(base, index) { + return Number((await this.#read(base + index * 8, 8)).readBigUInt64LE(0)); + } + + async #read(pos, length) { + const buf = Buffer.alloc(length); + await this.#fd.read(buf, 0, length, pos); + return buf; + } + + async #readHeader() { + const b = await this.#read(0, HEADER_SIZE); + const titlePtrRaw = b.readBigUInt64LE(40); + this.#hasTitleListing = titlePtrRaw !== TITLE_SENTINEL; + this.#header = { + articleCount: b.readUInt32LE(24), + clusterCount: b.readUInt32LE(28), + urlPtrPos: Number(b.readBigUInt64LE(32)), + titlePtrPos: this.#hasTitleListing ? Number(titlePtrRaw) : null, + clusterPtrPos: Number(b.readBigUInt64LE(48)), + mimeListPos: Number(b.readBigUInt64LE(56)), + mainPage: b.readUInt32LE(64), + }; + } + + async #readMimeTypes() { + let pos = this.#header.mimeListPos, str = ''; + for (;;) { + str += (await this.#read(pos, 1024)).toString('binary'); + const end = str.indexOf('\0\0'); + if (end !== -1) { str = str.slice(0, end + 1); break; } + pos += 1024; + } + this.#mimeTypes = str.split('\0').filter(Boolean); + } + + /** Directory entry (article record) at byte `offset`, growing the read window as needed. */ + async #readDirent(offset) { + for (let size = 512; ; size *= 2) { + const buf = await this.#read(offset, size); + let o = 0; + const mimetype = buf.readUInt16LE(o); o += 2; + o += 1; // extraLen, unused + const namespace = String.fromCharCode(buf.readUInt8(o)); o += 1; + o += 4; // revision, unused + + let redirectIndex = null, cluster = null, blob = null; + if (mimetype === 0xffff) { redirectIndex = buf.readUInt32LE(o); o += 4; } + else { cluster = buf.readUInt32LE(o); o += 4; blob = buf.readUInt32LE(o); o += 4; } + + const urlEnd = buf.indexOf(0, o); + if (urlEnd === -1) continue; + const titleEnd = buf.indexOf(0, urlEnd + 1); + if (titleEnd === -1) continue; + + const url = buf.toString('utf8', o, urlEnd); + const title = buf.toString('utf8', urlEnd + 1, titleEnd) || url; + return {mimetype, namespace, redirectIndex, cluster, blob, url, title}; + } + } + + async #resolveRedirect(dirent) { + if (dirent.mimetype !== 0xffff) return dirent; + return this.#readDirent(await this.#ptr64(this.#header.urlPtrPos, dirent.redirectIndex)); + } + + /** Narrows to dirents near the term's alphabetical position in the title index. */ + async #titleIndexCandidates(term) { + const q = term.toLowerCase(); + let lo = 0, hi = this.#header.articleCount - 1; + while (lo < hi) { + const mid = (lo + hi) >> 1; + const urlIdx = (await this.#read(this.#header.titlePtrPos + mid * 4, 4)).readUInt32LE(0); + const dirent = await this.#readDirent(await this.#ptr64(this.#header.urlPtrPos, urlIdx)); + if (dirent.title.toLowerCase() < q) lo = mid + 1; else hi = mid; + } + // Widen around the prefix match since fuzzy scoring isn't purely alphabetical. + const start = Math.max(0, lo - 50), end = Math.min(this.#header.articleCount, lo + 200); + const dirents = []; + for (let i = start; i < end; i++) { + const urlIdx = (await this.#read(this.#header.titlePtrPos + i * 4, 4)).readUInt32LE(0); + dirents.push(await this.#readDirent(await this.#ptr64(this.#header.urlPtrPos, urlIdx))); + } + return dirents; + } + + async close() { + if (this.#fd) await this.#fd.close(); + this.#fd = null; + } + + /** Reads an 'M' namespace metadata value (e.g. Name, Date, Title). Returns null if missing. */ + async metadata(key) { + const page = await this.readPage(key, NS_METADATA); + return page ? page.data.toString('utf8') : null; + } + + /** Opens the archive and parses its header + mimetype list. */ + async open() { + this.#fd = await fs.promises.open(this.path, 'r'); + await this.#readHeader(); + await this.#readMimeTypes(); + return this; + } + + /** Read a page's content by URL. Returns `{mimetype, data}` or `null` if not found. */ + async readPage(url, namespace = NS_CONTENT) { + let dirent = await this.#findByUrl(url, namespace); + if (!dirent) return null; + dirent = await this.#resolveRedirect(dirent); + const data = await this.#getBlob(dirent.cluster, dirent.blob); + return {mimetype: this.#mimeTypes[dirent.mimetype] || 'application/octet-stream', data}; + } + + /** Reads the archive's designated main/landing page, if one is set. */ + async mainPage() { + if (this.#header.mainPage === 0xffffffff) return null; + let dirent = await this.#readDirent(await this.#ptr64(this.#header.urlPtrPos, this.#header.mainPage)); + dirent = await this.#resolveRedirect(dirent); + const data = await this.#getBlob(dirent.cluster, dirent.blob); + return {mimetype: this.#mimeTypes[dirent.mimetype] || 'application/octet-stream', data, url: dirent.url}; + } + + /** + * Fuzzy-ranked title search. Accepts comma-separated `terms` the same way the + * catalog search does. Uses the sorted title index when present (binary search + * narrows the candidate window); falls back to a full linear scan otherwise + * (common on ZIM v6+/zimit-generated archives with no title index). + */ + async search(terms, {limit = 20, htmlOnly = true} = {}) { + const termList = String(terms).split(',').map(t => t.trim()).filter(Boolean); + if (!termList.length) return []; + + const candidates = this.#hasTitleListing + ? await this.#titleIndexCandidates(termList[0]) + : await this.#allDirents(); + + const scored = []; + for (const dirent of candidates) { + if (dirent.namespace !== NS_CONTENT) continue; + if (htmlOnly && !(this.#mimeTypes[dirent.mimetype] || '').startsWith('text/html')) continue; + const {max} = fuzzyMatch(dirent.title, ...termList); + scored.push({url: dirent.url, title: dirent.title, namespace: dirent.namespace, score: max}); + } + return scored.toSorted((a, b) => b.score - a.score).slice(0, limit); + } +} diff --git a/src/utils.js b/src/utils.js new file mode 100644 index 0000000..dd6b566 --- /dev/null +++ b/src/utils.js @@ -0,0 +1,32 @@ +export function levenshtein(a, b) { + const m = a.length, n = b.length; + if (!m) return n; + if (!n) return m; + const dp = Array.from({length: m + 1}, (_, i) => [i, ...Array(n).fill(0)]); + for (let j = 0; j <= n; j++) dp[0][j] = j; + for (let i = 1; i <= m; i++) { + for (let j = 1; j <= n; j++) { + dp[i][j] = a[i - 1] === b[j - 1] + ? dp[i - 1][j - 1] + : 1 + Math.min(dp[i - 1][j - 1], dp[i - 1][j], dp[i][j - 1]); + } + } + return dp[m][n]; +} + +/** Normalized similarity in [0,1]: 1 - editDistance / maxLength. */ +export function similarity(a, b) { + a = a.toLowerCase(); b = b.toLowerCase(); + return 1 - levenshtein(a, b) / Math.max(a.length, b.length, 1); +} + +/** Compares `target` against one or more search terms; returns avg/max/per-term similarity. */ +export function fuzzyMatch(target, ...terms) { + if (!terms.length) throw new Error('Requires at least 1 term to compare'); + const similarities = terms.map(t => similarity(target, t)); + return { + avg: similarities.reduce((acc, s) => acc + s, 0) / similarities.length, + max: Math.max(...similarities), + similarities, + }; +}