17 Commits
0.1.2 ... 0.3.3

Author SHA1 Message Date
7a2f17bfe9 Use kiwix search completely, no fuzzy ranking
All checks were successful
Publish Library / Build NPM Project (push) Successful in 16s
Publish Library / Tag Version (push) Successful in 10s
2026-08-25 00:42:59 -04:00
a2bde0b5cd Fixed server getter
All checks were successful
Publish Library / Build NPM Project (push) Successful in 14s
Publish Library / Tag Version (push) Successful in 11s
2026-08-24 21:18:24 -04:00
cab2571160 Accenpt kiwix server URL and patched local zim icon to be dataURL
All checks were successful
Publish Library / Build NPM Project (push) Successful in 15s
Publish Library / Tag Version (push) Successful in 10s
2026-08-24 21:06:23 -04:00
7d85031181 fixed install
All checks were successful
Publish Library / Build NPM Project (push) Successful in 16s
Publish Library / Tag Version (push) Successful in 8s
2026-08-24 20:29:31 -04:00
db2b334ed6 Migrated to kiwix-serve
Some checks failed
Publish Library / Build NPM Project (push) Failing after 21s
Publish Library / Tag Version (push) Has been skipped
2026-08-24 20:05:56 -04:00
7d376c90b4 Create title index for zims missing them
All checks were successful
Publish Library / Build NPM Project (push) Successful in 17s
Publish Library / Tag Version (push) Successful in 10s
2026-08-24 13:08:17 -04:00
bfb3f0efd3 Fixed post install step
All checks were successful
Publish Library / Build NPM Project (push) Successful in 13s
Publish Library / Tag Version (push) Successful in 8s
2026-08-24 11:12:03 -04:00
4ef022aa0e Fixed post install step
Some checks failed
Publish Library / Build NPM Project (push) Failing after 7s
Publish Library / Tag Version (push) Has been skipped
2026-08-24 11:10:57 -04:00
fb7ae55d49 Fixed post install step
Some checks failed
Publish Library / Build NPM Project (push) Failing after 7s
Publish Library / Tag Version (push) Has been skipped
2026-08-24 11:09:50 -04:00
bc90557de7 use kiwix-serach as prefilter before fuzzy ranking instead of building index
Some checks failed
Publish Library / Build NPM Project (push) Failing after 21s
Publish Library / Tag Version (push) Has been skipped
2026-08-24 11:02:47 -04:00
96ddcbe67a Garbage colleciton fix
All checks were successful
Publish Library / Build NPM Project (push) Successful in 12s
Publish Library / Tag Version (push) Successful in 10s
2026-08-22 17:57:19 -04:00
d10ee1d686 Auto caching
All checks were successful
Publish Library / Build NPM Project (push) Successful in 9s
Publish Library / Tag Version (push) Successful in 7s
2026-08-22 17:31:50 -04:00
ffdca81c78 Decompression caching and preview links
All checks were successful
Publish Library / Build NPM Project (push) Successful in 10s
Publish Library / Tag Version (push) Successful in 8s
2026-08-22 16:52:07 -04:00
aedb117002 Create search index cache and return more meta
All checks were successful
Publish Library / Build NPM Project (push) Successful in 10s
Publish Library / Tag Version (push) Successful in 8s
2026-08-22 14:24:22 -04:00
4278ea9df9 Added delete
All checks were successful
Publish Library / Build NPM Project (push) Successful in 9s
Publish Library / Tag Version (push) Successful in 8s
2026-08-22 13:01:19 -04:00
86730f1de8 Better searching
All checks were successful
Publish Library / Build NPM Project (push) Successful in 10s
Publish Library / Tag Version (push) Successful in 8s
2026-08-22 12:27:54 -04:00
c5207261fa Added namespace to main page
All checks were successful
Publish Library / Build NPM Project (push) Successful in 9s
Publish Library / Tag Version (push) Successful in 7s
2026-08-22 10:48:40 -04:00
13 changed files with 724 additions and 390 deletions

10
.gitignore vendored Normal file
View File

@@ -0,0 +1,10 @@
.idea
.vscode
logs
*.log
bin/*.dll
bin/kiwix*
node_modles
zims

156
README.md
View File

@@ -1,4 +1,3 @@
<!-- Header -->
<div id="top" align="center">
<br />
@@ -9,7 +8,7 @@
### Zim Utils
<!-- Description -->
Native, dependency-light ZIM archive reader/searcher and Kiwix catalog downloader for Node.js
Node.js wrapper around [kiwix-tools](https://github.com/kiwix/kiwix-tools) for managing a local ZIM library, serving content, and searching both locally and against the Kiwix catalog
<!-- Repo badges -->
[![Version](https://img.shields.io/badge/dynamic/json.svg?label=Version&style=for-the-badge&url=https://git.zakscode.com/api/v1/repos/ztimson/zim-utils/tags&query=$[0].name)](https://git.zakscode.com/ztimson/zim-utils/tags)
@@ -36,24 +35,23 @@ Native, dependency-light ZIM archive reader/searcher and Kiwix catalog downloade
- [Setup](#setup)
- [Production](#production)
- [Usage](#usage)
- [ZimManager](#zimmanager)
- [ZimReader](#zimreader)
- [KiwixServer](#kiwixserver)
- [Catalog](#catalog)
- [License](#license)
## About
`@ztimson/zim-utils` is a native, dependency-light toolkit for working with [ZIM](https://wiki.openzim.org/wiki/ZIM_file_format) archives and the [Kiwix](https://kiwix.org/) catalog in Node.js
`@ztimson/zim-utils` manages a local [ZIM](https://wiki.openzim.org/wiki/ZIM_file_format) library by driving the official [kiwix-tools](https://github.com/kiwix/kiwix-tools) binaries (`kiwix-serve`, `kiwix-manage`) as child processes, rather than re-implementing ZIM reading from scratch. This means content serving, indexing, and fulltext search all come straight from Kiwix's own battle-tested implementation.
It comes with the following helpers:
It comes with:
- **`ZimManager`** — Local library manager: listing, updating/downloading, and running searches accross the entire library
- **`ZimReader`** — A `.zim` file reader for reading pages, metadata and running searches
- **Catalog** (`zimCatalog`/`zimCatalogInfo`) — Helper functions to search the Kiwix OPDS catalog
- **`KiwixServer`** — Owns a directory of `.zim` files: builds/rebuilds `library.xml`, starts/stops `kiwix-serve`, lists the local library, runs fulltext search, and resolves content URLs/bytes for any archive+page
- **Catalog** (`zimCatalog`/`zimCatalogInfo`) — Helper functions to search the remote Kiwix OPDS catalog, for discovering and downloading new ZIMs
### Built With
[![Node](https://img.shields.io/badge/Node.js-000000?style=for-the-badge&logo=nodedotjs)](https://nodejs.org/)
[![JavaScript](https://img.shields.io/badge/JavaScript-000000?style=for-the-badge&logo=javascript)](https://javascript.com/)
[![Kiwix](https://img.shields.io/badge/kiwix--tools-000000?style=for-the-badge&logo=kiwix)](https://github.com/kiwix/kiwix-tools)
## Setup
@@ -68,84 +66,122 @@ It comes with the following helpers:
- [Node.js](https://nodejs.org/en/download)
#### Instructions
1. Install the dependencies: `npm install`
2. Install the package: `npm install @ztimson/zim-utils`
1. Install the package: `npm install @ztimson/zim-utils`
2. On first install, `kiwix-serve`/`kiwix-manage` binaries are fetched into `./bin` (or wherever `binDir` points) - no separate Kiwix install needed.
</details>
## Usage
### ZimManager
### KiwixServer
`ZimManager` owns a directory of `.zim` files and handles everything from downloading to cross-archive search.
`KiwixServer` owns a directory of `.zim` files and manages the whole `kiwix-serve` lifecycle for you.
```js
import {ZimManager} from '@ztimson/zim-utils';
import {KiwixServer} from '@ztimson/zim-utils';
const manager = new ZimManager('./zims'); // optional 2nd arg: custom catalog URL
// Search the Kiwix catalog & download the top hit
const [entry] = await manager.catalog('wikipedia,medicine');
await manager.download(entry.href);
// List local archives with their parsed metadata
const local = await manager.list();
// [{file: './zims/wikipedia_en_medicine.zim', meta: {name, date, title}}, ...]
// Check a single file for updates without downloading
const status = await manager.isOutdated(local[0].file);
// Update every local ZIM that has a newer catalog version
await manager.updateAll({force: false});
// Fuzzy-search titles across ALL local archives at once
const hits = await manager.search('diabetes treatment', {limit: 10});
// Open a reader by file path OR by catalog name
const reader = await manager.open('wikipedia_en_medicine');
const page = await reader.readPage('A/Diabetes');
await reader.close();
const server = new KiwixServer('./zims'); // optional 2nd arg: {port, host, binDir}
await server.start(); // rebuilds library.xml, spawns kiwix-serve, waits until ready
await server.reload(); // pick up newly added/removed .zim files, no downtime
await server.stop(); // Kill server
```
### ZimReader
Everything `ZimManager` does to a single archive is just a thin wrapper around `ZimReader`. Use it directly when you don't need a whole managed library:
#### List Local ZIMs
```js
import {ZimReader} from '@ztimson/zim-utils';
const local = await server.list();
[
{
id: '37a99758-43a0-6ba3-cd54-1af556369eee',
title: 'FOSS cooking',
updated: 2026-05-06T00:00:00.000Z,
summary: 'Making cooking fast, easy, foss',
language: 'eng',
name: 'foss.cooking_en_all',
category: 'other',
tags: ['_category:other', '_ftindex:yes', 'preppers,food', '_pictures:yes', '_videos:yes', '_details:yes'],
mediaCount: 154,
author: '-',
publisher: 'openZIM',
articleCount: 719,
sizeMb: 23.1,
href: 'foss.cooking_en_all_2026-05',
icon: undefined,
viewer: 'http://127.0.0.1:51992/content/foss.cooking_en_all_2026-05',
},
// ...
]
```
const reader = await new ZimReader('./zims/wikipedia_en_medicine.zim').open();
#### Search Local ZIMs
// Metadata (what manager.#readMeta / isOutdated rely on)
const name = await reader.metadata('Name');
const date = await reader.metadata('Date');
const title = await reader.metadata('Title');
```js
// Fulltext search across every local archive: xapian prefilter, then fuzzy re-ranked
const hits = await server.search('chocolate', 5);
[
{
id: '37a99758-43a0-6ba3-cd54-1af556369eee',
title: 'Chocolate Chip Cookies',
page: 'foss.cooking/recipe/mfed3/chocolate-chip-cookies',
name: 'foss.cooking_en_all',
publisher: 'openZIM',
href: 'foss.cooking_en_all_2026-05/foss.cooking/recipe/mfed3/chocolate-chip-cookies',
icon: undefined,
viewer: 'http://127.0.0.1:51992/content/foss.cooking_en_all_2026-05/foss.cooking/recipe/mfed3/chocolate-chip-cookies',
summary: '...cream scooper or spoon to make uniform balls of cookie dough...',
score: 0.67,
},
// ...
]
```
// Landing page
const home = await reader.mainPage();
#### View ZIM Content
// Direct page lookup by URL
const page = await reader.readPage('A/Diabetes');
console.log(page.mimetype, page.data.toString('utf8'));
```js
// Build a content URL from any href (list()/search() output, or a full viewer URL) without hitting the network
const url = server.fetch(hits[0].href);
'http://127.0.0.1:51992/content/foss.cooking_en_all_2026-05/foss.cooking/recipe/mfed3/chocolate-chip-cookies'
// Fuzzy title search within just this archive (what manager.search fans out over)
const results = await reader.search('diabetes,insulin', {limit: 20, htmlOnly: true});
await reader.close();
// Or fetch the bytes directly, proxied straight from kiwix-serve
const {mimetype, data} = await server.raw(hits[0].href);
{mimetype: 'text/html; charset=utf-8', data: <Buffer ...>}
```
### Catalog
`ZimManager.catalog()` and its update checks are backed directly by these two functions:
```js
import {zimCatalog, zimCatalogInfo, CATALOG_URL} from '@ztimson/zim-utils';
// Ranked search across the Kiwix catalog (comma-separated terms, like ZimReader.search)
const results = await zimCatalog('history,rome', {lang: 'eng', count: 20, url: CATALOG_URL});
// Search the remote Kiwix catalog to discover new ZIMs
const search = await zimCatalog('knots', {lang: 'eng', count: 20, url: CATALOG_URL});
// OR Exact lookup by catalog `name`, useful for checking if a local copy is outdated
const entry = await zimCatalogInfo('wikipedia_en_knots');
// Exact lookup by catalog `name`, used to check if a local copy is outdated
const entry = await zimCatalogInfo('wikipedia_en_medicine');
[
{
id: 'urn:uuid:3a4fe0d0-0bd0-7583-ada8-c52d173ae44d',
title: 'Knots by Wikipedia',
updated: 2026-07-20T00:00:00.000Z,
summary: 'A subset of Wikipedia encyclopedia dedicated to knots',
language: 'eng',
name: 'wikipedia_en_knots',
category: 'wikipedia',
tags: ['wikipedia', '_category:wikipedia', '_pictures:yes', '_videos:no', '_details:yes', '_ftindex:yes'],
mediaCount: 3501,
author: 'Wikipedia',
publisher: 'openZIM',
articleCount: 1730,
sizeMb: 17.6,
href: 'https://lb.download.kiwix.org/zim/wikipedia/wikipedia_en_knots_maxi_2026-07.zim.meta4',
icon: 'https://library.kiwix.org/catalog/v2/illustration/3a4fe0d0-0bd0-7583-ada8-c52d173ae44d/?size=48',
viewer: 'https://library.kiwix.org/viewer#wikipedia_en_knots',
},
// ...
]
// Download catalog ZIM if out of date (unless forced) and reload automatically
await mgr.download(search[0].href, force);
```
## License
Copyright © 2026 Zakary Timson | Available under MIT Licensing

81
bin/install-kwix.js Normal file
View File

@@ -0,0 +1,81 @@
import fs from 'node:fs';
import os from 'node:os';
import path from 'node:path';
import https from 'node:https';
import {execFileSync} from 'node:child_process';
import {fileURLToPath} from 'node:url';
const __dirname = path.dirname(fileURLToPath(import.meta.url));
const VERSION = '3.8.1';
const BASE_URL = 'https://download.kiwix.org/release/kiwix-tools';
const PLATFORM_MAP = {linux: 'linux', darwin: 'macos', win32: 'win'};
const BIN_DIR = path.join(__dirname, '..', 'bin'); // project root/bin
/** Download a file, following redirects. */
function download(url, dest) {
return new Promise((resolve, reject) => {
const file = fs.createWriteStream(dest);
https.get(url, res => {
if (res.statusCode >= 300 && res.statusCode < 400 && res.headers.location) {
file.close();
return resolve(download(res.headers.location, dest));
}
if (!res.statusCode || res.statusCode >= 400) {
return reject(new Error(`Download failed: ${res.statusCode} ${res.statusMessage}`));
}
res.pipe(file);
file.on('finish', () => file.close(resolve));
}).on('error', err => fs.unlink(dest, () => reject(err)));
});
}
/** Recursively chmod all files under a directory. */
function chmodRecursive(dir, mode) {
for (const entry of fs.readdirSync(dir, {withFileTypes: true})) {
const full = path.join(dir, entry.name);
if (entry.isDirectory()) chmodRecursive(full, mode);
else fs.chmodSync(full, mode);
}
}
async function downloadAndExtract() {
const platformName = PLATFORM_MAP[process.platform];
if (!platformName) throw new Error(`Unsupported platform: ${process.platform}`);
const ext = platformName === 'win' ? 'zip' : 'tar.gz';
const archiveName = `kiwix-tools_${platformName}-x86_64-${VERSION}.${ext}`;
const url = `${BASE_URL}/${archiveName}`;
const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'kiwix-tools-'));
const archivePath = path.join(tmpDir, archiveName);
console.log('Downloading kiwix-tools from:', url);
await download(url, archivePath);
await fs.promises.mkdir(BIN_DIR, {recursive: true});
console.log('Extracting...');
execFileSync('tar', ['-xf', archivePath, '-C', BIN_DIR]); // bsdtar handles zip too
// Archives (tar.gz) may wrap contents in a subdirectory - flatten it into BIN_DIR
const wrapperDir = (await fs.promises.readdir(BIN_DIR, {withFileTypes: true}))
.find(e => e.isDirectory() && e.name.startsWith('kiwix-tools_'));
if (wrapperDir) {
const wrapperPath = path.join(BIN_DIR, wrapperDir.name);
for (const entry of await fs.promises.readdir(wrapperPath)) {
await fs.promises.rename(path.join(wrapperPath, entry), path.join(BIN_DIR, entry));
}
await fs.promises.rmdir(wrapperPath);
}
if (process.platform !== 'win32') chmodRecursive(BIN_DIR, 0o755);
await fs.promises.rm(tmpDir, {recursive: true, force: true});
console.log('Installed to:', BIN_DIR);
}
downloadAndExtract().catch(err => {
console.error(err);
process.exit(1)
}).finally(() => process.exit());

13
package-lock.json generated
View File

@@ -1,23 +1,24 @@
{
"name": "@ztimson/zim-utils",
"version": "0.1.0",
"version": "0.2.5",
"lockfileVersion": 3,
"requires": true,
"packages": {
"": {
"name": "@ztimson/zim-utils",
"version": "0.1.0",
"version": "0.2.5",
"hasInstallScript": true,
"license": "MIT",
"dependencies": {
"@ztimson/utils": "^0.30.7",
"@ztimson/utils": "^0.30.8",
"lzma1": "^0.3.0",
"zstd-codec": "^0.1.5"
}
},
"node_modules/@ztimson/utils": {
"version": "0.30.7",
"resolved": "https://registry.npmjs.org/@ztimson/utils/-/utils-0.30.7.tgz",
"integrity": "sha512-0TkjQFVe0edTqlezIfUtwTXA9pkS1rVYw0xzFycKmtcoldh6urQEKQ/T6Z0Id1+w53ubomSdVfj6GB3u29+AOQ==",
"version": "0.30.8",
"resolved": "https://registry.npmjs.org/@ztimson/utils/-/utils-0.30.8.tgz",
"integrity": "sha512-+vBjcinqckqMHkP95xWiQeQz2E7Q1oS0b+Odjp+F9rvQ4z0US4JdodjqhEaqh+RAO/yP77x4Eu0a04yBB4HNdw==",
"license": "MIT",
"dependencies": {
"var-persist": "^1.0.1"

View File

@@ -1,6 +1,6 @@
{
"name": "@ztimson/zim-utils",
"version": "0.1.2",
"version": "0.3.3",
"description": "Native, dependency-light ZIM archive reader/searcher and Kiwix catalog downloader for Node.js",
"author": "Zak Timson",
"license": "MIT",
@@ -11,8 +11,11 @@
"url": "https://git.zakscode.com/ztimson/zim-utils"
},
"main": "src/index.js",
"scripts": {
"postinstall": "node ./bin/install-kwix.js"
},
"dependencies": {
"@ztimson/utils": "^0.30.7",
"@ztimson/utils": "^0.30.8",
"lzma1": "^0.3.0",
"zstd-codec": "^0.1.5"
}

View File

@@ -1,40 +1,44 @@
import {fuzzyMatch} from './utils.js';
import {decodeHtml} from '@ztimson/utils';
export const CATALOG_URL = 'https://library.kiwix.org/catalog/v2/entries';
export const CATALOG_URL = 'https://library.kiwix.org';
const PAGE_SIZE = 100;
/** Parses `<entry>` blocks out of a Kiwix catalog OPDS XML response. */
function parseEntries(xml) {
export function parseEntries(xml, catalog = CATALOG_URL) {
const blocks = xml.match(/<entry>[\s\S]*?<\/entry>/g) || [];
return blocks.map(b => {
const grab = re => (b.match(re) || [])[1] || '';
const linkMatch = b.match(/<link[^>]*type=["']application\/x-zim[^"']*["'][^>]*href=["']([^"']+)["']/);
const iconMatch = b.match(/<link[^>]*rel=["']http:\/\/opds-spec\.org\/image\/thumbnail["'][^>]*href=["']([^"']+)["']/)
|| b.match(/<link[^>]*type=["']image\/[^"']*["'][^>]*href=["']([^"']+)["']/);
let tags = grab(/<tags>([^<]*)<\/tags>/);
if(tags) tags = tags.split(';');
const name = grab(/<name>([^<]*)<\/name>/);
return {
id: grab(/<id>([^<]*)<\/id>/),
title: decodeHtml(grab(/<title>([^<]*)<\/title>/)),
updated: new Date(grab(/<updated>([^<]*)<\/updated>/)),
summary: decodeHtml(grab(/<summary>([^<]*)<\/summary>/)),
language: grab(/<language>([^<]*)<\/language>/),
name: grab(/<name>([^<]*)<\/name>/),
name,
category: grab(/<category>([^<]*)<\/category>/),
tags,
mediaCount: Number(grab(/<mediaCount>([^<]*)<\/mediaCount>/)) || 0,
author: grab(/<author>\s*<name>([^<]*)<\/name>\s*<\/author>/m),
publisher: grab(/<publisher>\s*<name>([^<]*)<\/name>\s*<\/publisher>/m),
articleCount: Number(grab(/<articleCount>([^<]*)<\/articleCount>/)) || 0,
sizeMb: linkMatch ? (Number((b.match(/length=["'](\d+)["']/) || [])[1] || 0) / 1024 / 1024).toFixed(1) : '?',
sizeMb: linkMatch ? +(Number((b.match(/length=["'](\d+)["']/) || [])[1] || 0) / 1024 / 1024).toFixed(1) : '?',
href: linkMatch ? linkMatch[1] : null,
icon: iconMatch ? new URL(iconMatch[1], catalog).href : null,
viewer: name ? new URL(`viewer#${name}`, catalog).href : null,
};
});
}
async function fetchEntries(term, lang, url = CATALOG_URL) {
const params = new URLSearchParams({q: term, count: String(PAGE_SIZE), lang: lang || 'eng'});
const res = await fetch(`${url}?${params}`);
const res = await fetch(`${url}/catalog/v2/entries?${params}`);
if (!res.ok) throw new Error(`${res.status} ${res.statusText}`);
return parseEntries(await res.text());
}
@@ -42,7 +46,7 @@ async function fetchEntries(term, lang, url = CATALOG_URL) {
/** Looks up a single catalog entry by exact `name` (used to check for available updates). */
export async function zimCatalogInfo(name, url = CATALOG_URL) {
const params = new URLSearchParams({name, count: '5'});
const res = await fetch(`${url}?${params}`);
const res = await fetch(`${url}/catalog/v2/entries?${params}`);
if (!res.ok) return null;
return parseEntries(await res.text()).find(e => e.name === name) || null;
}
@@ -52,7 +56,7 @@ export async function zimCatalog(terms, opts = {lang: 'eng', count: 20, url: CAT
opts = Object.assign({lang: 'eng', count: 20, url: CATALOG_URL}, opts)
const termList = [...String(terms).split(',')].filter(Boolean).map(t => t.trim().toLowerCase());
const results = await Promise.allSettled(termList.map(t => fetchEntries(t, opts.lang, opts.url)));
const byName = new Map(); // name -> {entry, hitTerms:Set}
const byName = new Map();
results.forEach((r, i) => {
if (r.status !== 'fulfilled') return;
const term = termList[i];

32
src/decompress-worker.js Normal file
View File

@@ -0,0 +1,32 @@
'use strict';
import {parentPort} from 'node:worker_threads';
import {decompress as lzmaDecompress} from 'lzma1';
import {ZstdCodec} from 'zstd-codec';
let zstdStreamingPromise = null;
function getZstd() {
if (!zstdStreamingPromise) {
zstdStreamingPromise = new Promise(resolve => ZstdCodec.run(zstd => resolve(new zstd.Streaming())));
}
return zstdStreamingPromise;
}
/** Rebuilds a full 13-byte "alone" LZMA header from libzim's truncated 5-byte one (unknown size). */
function toLzmaAloneStream(body) {
const header = Buffer.concat([body.subarray(0, 5), Buffer.alloc(8, 0xff)]);
return Buffer.concat([header, body.subarray(5)]);
}
parentPort.on('message', async ({id, compType, body}) => {
try {
const buf = Buffer.from(body);
const data = compType === 4
? Buffer.from(lzmaDecompress(toLzmaAloneStream(buf)))
: Buffer.from((await getZstd()).decompress(new Uint8Array(buf)));
parentPort.postMessage({id, data}, [data.buffer]);
} catch (e) {
parentPort.postMessage({id, error: e.message});
}
});

50
src/decompress.js Normal file
View File

@@ -0,0 +1,50 @@
'use strict';
import {Worker} from 'node:worker_threads';
import path from 'node:path';
import os from 'node:os';
import {fileURLToPath} from 'node:url';
const __dirname = path.dirname(fileURLToPath(import.meta.url));
/** Round-robin pool of worker threads for off-main-thread cluster decompression. */
class DecompressPool {
#workers = [];
#next = 0;
#pending = new Map(); // id -> {resolve, reject}
#nextId = 0;
#active = 0;
constructor(size = Math.max(1, os.cpus().length - 1)) {
for (let i = 0; i < size; i++) this.#spawn();
}
#spawn() {
const worker = new Worker(path.join(__dirname, 'decompress-worker.js'));
worker.on('message', ({id, data, error}) => {
const task = this.#pending.get(id);
if (!task) return;
this.#pending.delete(id);
// No work left in flight -> safe to let the process exit again.
if (--this.#active === 0) for (const w of this.#workers) w.unref();
error ? task.reject(new Error(error)) : task.resolve(Buffer.from(data));
});
worker.unref(); // idle workers must never keep a short script/server alive
this.#workers.push(worker);
}
/** Decompresses `{compType, body}` on the next worker in rotation. `body` must be a Uint8Array/Buffer view. */
run({compType, body}) {
const worker = this.#workers[this.#next];
this.#next = (this.#next + 1) % this.#workers.length;
const id = this.#nextId++;
const owned = new Uint8Array(body);
if (this.#active++ === 0) for (const w of this.#workers) w.ref();
return new Promise((resolve, reject) => {
this.#pending.set(id, {resolve, reject});
worker.postMessage({id, compType, body: owned}, [owned.buffer]);
});
}
}
export const decompressPool = new DecompressPool();

View File

@@ -1,4 +1,5 @@
export * from './catalog.js';
export * from './manager.js';
export * from './reader.js';
export * from './server.js';
export * from './utils.js';

View File

@@ -2,17 +2,34 @@ import fs from 'node:fs';
import path from 'node:path';
import {pipeline} from 'node:stream/promises';
import {Readable} from 'node:stream';
import {ZimReader} from './reader.js';
import {KiwixServer} from './server.js';
import {zimCatalog, zimCatalogInfo, CATALOG_URL} from './catalog.js';
/** Manages a local directory of ZIM archives: listing, update checks, downloads, and reading. */
/** Manages a local directory of ZIM archives: catalog search, downloads, update checks, deletion.
* Reuses (or owns & lazily starts) a KiwixServer for local listing/search, so `list()` and `catalog()` return the same shape. */
export class ZimManager {
#catalog;
#catalogUrl;
#dir;
#server;
#ownsServer;
constructor(dir, catalog = CATALOG_URL) {
this.#catalog = catalog;
/** @param {{catalog?: string, server?: KiwixServer, port?: number, host?: string, binDir?: string, url?: string}} [opts]
* server: reuse an existing KiwixServer instance directly.
* url: attach to an already-running kiwix-serve by address instead (e.g. started elsewhere in your codebase
* with no shared reference) - mutually exclusive with `server`, ignored if `server` is given. */
constructor(dir, {catalog = CATALOG_URL, server, port, host, binDir, url} = {}) {
this.#catalogUrl = catalog;
this.#dir = dir;
this.#ownsServer = !server;
this.#server = server ?? new KiwixServer(dir, {port, host, binDir, url});
}
/** The KiwixServer backing this manager - reuse it directly for content/search access, or pass into another ZimManager. */
get server() { return this.#server; }
async #ensureServer() {
if (!this.#server.running) await this.#server.start();
return this.#server;
}
async #download(url, destPath) {
@@ -23,10 +40,6 @@ export class ZimManager {
await fs.promises.rename(tmpPath, destPath);
}
async #ensureDir() {
await fs.promises.mkdir(this.#dir, {recursive: true});
}
/** Resolves a `.meta4` metalink URL down to the real mirror `.zim` download URL. */
async #resolveUrl(url) {
const head = await fetch(url);
@@ -39,127 +52,102 @@ export class ZimManager {
return {res, url: m[1]};
}
/** Reads Name/Date/Title metadata from a local ZIM file. Returns null if unreadable. */
async #readMeta(filepath) {
let reader;
try {
reader = await new ZimReader(filepath).open();
return {
name: await reader.metadata('Name'),
date: await reader.metadata('Date'),
title: await reader.metadata('Title'),
};
} catch {
return null;
} finally {
await reader?.close();
}
}
async #update(name, catalogEntry, localMatch, force) {
const remoteDate = catalogEntry.updated ? new Date(catalogEntry.updated) : null;
const localDate = localMatch?.meta?.date ? new Date(localMatch.meta.date) : null;
const localDate = localMatch?.updated ?? null;
if (!force && localMatch && remoteDate && localDate && remoteDate <= localDate)
return {name, status: 'skipped', reason: 'up to date'};
if (!catalogEntry.href) return {name, status: 'skipped', reason: 'missing download link'};
if (!catalogEntry.href) return {name, status: 'skipped', reason: 'missing href'};
const filename = path.basename(new URL(catalogEntry.href).pathname).replace(/\.meta4$/i, '');
const destPath = path.join(this.#dir, filename);
await this.#download(catalogEntry.href, destPath);
if (localMatch && localMatch.file !== destPath) await fs.promises.unlink(localMatch.file).catch(() => {});
return {name, status: 'updated', file: destPath};
if (localMatch && localMatch.file !== filename) await fs.promises.rm(path.join(this.#dir, localMatch.file), {force: true});
const server = await this.#ensureServer();
await server.reload();
return {name, status: 'updated', file: filename};
}
/** Resolves a file path or catalog `name` to a local file path. */
async #resolveFile(fileOrName) {
if (fs.existsSync(fileOrName)) return fileOrName;
const joined = path.join(this.#dir, fileOrName);
if (fs.existsSync(joined)) return joined;
catalog(search, opts = {}) {
return zimCatalog(search, {url: this.#catalogUrl, ...opts});
}
/** Stops the internally-owned KiwixServer, if this manager created its own (no-op if one was passed in). */
async close() {
if (this.#ownsServer) await this.#server.stop();
}
async delete(nameOrFile) {
const local = await this.list();
const match = local.find(l => l.meta?.name === fileOrName || path.basename(l.file) === fileOrName);
if (!match) throw new Error(`ZIM not found locally: ${fileOrName}`);
return match.file;
}
catalog(search, opts) {
return zimCatalog(search, opts);
}
/** Checks whether a local ZIM has a newer version in the catalog, without downloading. */
async isOutdated(file) {
const meta = await this.#readMeta(file);
if (!meta?.name) return {file, upToDate: null, reason: 'no metadata'};
const catalogEntry = await zimCatalog(meta.name, this.#catalog);
if (!catalogEntry) return {file, upToDate: null, reason: 'missing from catalog'};
const remoteDate = catalogEntry.updated ? new Date(catalogEntry.updated) : null;
const localDate = meta.date ? new Date(meta.date) : null;
return {file, name: meta.name, upToDate: !!(remoteDate && localDate && remoteDate <= localDate), localDate, remoteDate};
}
/** Lists local `.zim` files with their parsed metadata (or `null` if unreadable). */
async list() {
await this.#ensureDir();
const files = (await fs.promises.readdir(this.#dir)).filter(f => f.endsWith('.zim'));
return Promise.all(files.map(async f => {
const file = path.join(this.#dir, f);
return {file, meta: await this.#readMeta(file)};
}));
const match = local.find(l => l.name === nameOrFile || l.file === nameOrFile);
if (!match) throw new Error(`ZIM not found locally: ${nameOrFile}`);
await fs.promises.rm(path.join(this.#dir, match.file), {force: true});
const server = await this.#ensureServer();
await server.reload();
return {file: match.file, status: 'deleted'};
}
/** Downloads/updates a single ZIM by direct href, matching against any existing local copy by name. */
async download(href, {force = false} = {}) {
await this.#ensureDir();
await fs.promises.mkdir(this.#dir, {recursive: true});
const {url: finalUrl} = await this.#resolveUrl(href);
const filename = path.basename(new URL(finalUrl).pathname).replace(/\.meta4$/i, '');
const name = filename.replace(/\.zim$/i, '').replace(/_\d{4}-\d{2}(?:_\d+)?$/, '');
const local = await this.list();
const localMatch = local.find(l => l.meta?.name === name) ?? null;
const catalogEntry = await zimCatalogInfo(name, this.#catalog) || {name, updated: null, href};
const localMatch = local.find(l => l.name === name) ?? null;
const catalogEntry = await zimCatalogInfo(name, this.#catalogUrl) || {name, updated: null, href: href};
return this.#update(name, catalogEntry, localMatch, force);
}
/** Checks all local ZIMs against the catalog and updates any that are outdated. */
async updateAll({force = false}) {
const local = await this.list();
if (!local.length) return [];
async link(href) {
const server = await this.#ensureServer();
return server.link(href);
}
/** Checks whether a local ZIM has a newer version in the catalog, without downloading. */
async isOutdated(nameOrFile) {
const local = await this.list();
const match = local.find(l => l.name === nameOrFile || l.file === nameOrFile);
if (!match?.name) return {file: nameOrFile, upToDate: null, reason: 'no metadata'};
const catalogEntry = await zimCatalogInfo(match.name, this.#catalogUrl);
if (!catalogEntry) return {file: match.file, upToDate: null, reason: 'missing from catalog'};
const remoteDate = catalogEntry.updated ? new Date(catalogEntry.updated) : null;
return {file: match.file, name: match.name, upToDate: !!(remoteDate && match.updated && remoteDate <= match.updated), localDate: match.updated, remoteDate};
}
/** Lists local ZIMs - same flat shape as `catalog()` (plus `file`), sourced from the KiwixServer's own library.xml. */
async list() {
const server = await this.#ensureServer();
return server.list();
}
/** Fetches a single asset's raw bytes. */
async raw(href) {
const server = await this.#ensureServer();
return server.raw(href);
}
/** Two-pass fulltext search across every local ZIM, returns enriched results matching catalog/list shape. */
async search(terms, limit = 20) {
const server = await this.#ensureServer();
return server.search(terms, limit);
}
/** Checks all local ZIMs against the catalog and updates any that are outdated. */
async updateAll({force = false} = {}) {
const local = await this.list();
const results = [];
for (const {file, meta} of local) {
if (!meta?.name) { results.push({file, status: 'skipped', reason: 'no metadata'}); continue; }
const catalogEntry = await zimCatalogInfo(meta.name, this.#catalog);
if (!catalogEntry) { results.push({name: meta.name, status: 'skipped', reason: 'missing from catalog'}); continue; }
for (const entry of local) {
if (!entry.name) { results.push({file: entry.file, status: 'skipped', reason: 'no metadata'}); continue; }
const catalogEntry = await zimCatalogInfo(entry.name, this.#catalogUrl);
if (!catalogEntry) { results.push({name: entry.name, status: 'skipped', reason: 'missing from catalog'}); continue; }
try {
results.push(await this.#update(meta.name, catalogEntry, {file, meta}, force));
results.push(await this.#update(entry.name, catalogEntry, entry, force));
} catch (e) {
results.push({name: meta.name, status: 'error', reason: e.message});
results.push({name: entry.name, status: 'error', reason: e.message});
}
}
return results;
}
/** Opens a `ZimReader` for a local ZIM, resolved by file path or catalog `name`. Caller must `.close()` it. */
async open(fileOrName) {
await this.#ensureDir();
const file = await this.#resolveFile(fileOrName);
return new ZimReader(file).open();
}
/** Fuzzy-searches titles across every local ZIM in the library, merging & re-ranking hits by score. */
async search(terms, {limit = 20, htmlOnly = true} = {}) {
const local = await this.list();
const perZim = await Promise.all(local.map(async ({file, meta}) => {
let reader;
try {
reader = await new ZimReader(file).open();
const hits = await reader.search(terms, {limit, htmlOnly});
return hits.map(h => ({...h, file, name: meta?.name}));
} catch {
return [];
} finally {
await reader?.close();
}
}));
return perZim.flat().toSorted((a, b) => b.score - a.score).slice(0, limit);
}
}

View File

@@ -1,235 +1,138 @@
'use strict';
import fs from 'node:fs';
import {decompress as lzmaDecompress} from 'lzma1';
import {ZstdCodec} from 'zstd-codec';
import {fuzzyMatch} from './utils.js';
import {decompressPool} from './decompress.js';
const HEADER_SIZE = 80;
const NS_CONTENT = 'C';
const NS_METADATA = 'M';
const TITLE_SENTINEL = 0xffffffffffffffffn; // Indicator -> ZIM v6+ archives with no title
let zstdStreamingPromise = null;
/** Lazily initialised, shared Zstd streaming decompressor (handles unknown-size frames). */
function getZstd() {
if (!zstdStreamingPromise) {
zstdStreamingPromise = new Promise(resolve => ZstdCodec.run(zstd => resolve(new zstd.Streaming())));
}
return zstdStreamingPromise;
async function readAt(fd, pos, length) {
const buf = Buffer.alloc(length);
await fd.read(buf, 0, length, pos);
return buf;
}
/** Rebuilds a full 13-byte "alone" LZMA header from libzim's truncated 5-byte one (unknown size). */
function toLzmaAloneStream(body) {
const header = Buffer.concat([body.subarray(0, 5), Buffer.alloc(8, 0xff)]);
return Buffer.concat([header, body.subarray(5)]);
async function ptr64(fd, base, index) {
return Number((await readAt(fd, base + index * 8, 8)).readBigUInt64LE(0));
}
/** Native, dependency-light reader for .zim archives. Supports zstd & LZMA cluster compression. */
export class ZimReader {
#fd = null;
#header = null;
#mimeTypes = [];
#hasTitleListing = false;
async function readHeader(fd) {
const b = await readAt(fd, 0, HEADER_SIZE);
return {
articleCount: b.readUInt32LE(24),
clusterCount: b.readUInt32LE(28),
urlPtrPos: Number(b.readBigUInt64LE(32)),
clusterPtrPos: Number(b.readBigUInt64LE(48)),
mimeListPos: Number(b.readBigUInt64LE(56)),
};
}
get articleCount() { return this.#header?.articleCount ?? 0; }
constructor(path) {
this.path = path;
async function readMimeTypes(fd, mimeListPos) {
let pos = mimeListPos, str = '';
for (;;) {
str += (await readAt(fd, pos, 1024)).toString('binary');
const end = str.indexOf('\0\0');
if (end !== -1) { str = str.slice(0, end + 1); break; }
pos += 1024;
}
return str.split('\0').filter(Boolean);
}
/** Full O(n) scan over the URL pointer list, used when there's no title index. */
async #allDirents() {
const dirents = [];
for (let i = 0; i < this.#header.articleCount; i++) {
dirents.push(await this.#readDirent(await this.#ptr64(this.#header.urlPtrPos, i)));
}
return dirents;
/** Directory entry (article record) at byte `offset`, growing the read window as needed. */
async function readDirent(fd, offset) {
for (let size = 512; ; size *= 2) {
const buf = await readAt(fd, offset, size);
let o = 0;
const mimetype = buf.readUInt16LE(o); o += 2;
o += 1; // extraLen, unused
const namespace = String.fromCharCode(buf.readUInt8(o)); o += 1;
o += 4; // revision, unused
let redirectIndex = null, cluster = null, blob = null;
if (mimetype === 0xffff) { redirectIndex = buf.readUInt32LE(o); o += 4; }
else { cluster = buf.readUInt32LE(o); o += 4; blob = buf.readUInt32LE(o); o += 4; }
const urlEnd = buf.indexOf(0, o);
if (urlEnd === -1) continue;
const titleEnd = buf.indexOf(0, urlEnd + 1);
if (titleEnd === -1) continue;
const url = buf.toString('utf8', o, urlEnd);
const title = buf.toString('utf8', urlEnd + 1, titleEnd) || url;
return {mimetype, namespace, redirectIndex, cluster, blob, url, title};
}
}
/** Binary search the URL pointer list for namespace+url. */
async #findByUrl(url, namespace) {
const key = namespace + url;
let lo = 0, hi = this.#header.articleCount - 1;
while (lo <= hi) {
const mid = (lo + hi) >> 1;
const dirent = await this.#readDirent(await this.#ptr64(this.#header.urlPtrPos, mid));
const dirKey = dirent.namespace + dirent.url;
const cmp = key < dirKey ? -1 : key > dirKey ? 1 : 0;
if (cmp === 0) return dirent;
if (cmp < 0) hi = mid - 1; else lo = mid + 1;
}
return null;
async function findByUrl(fd, header, url, namespace) {
const key = namespace + url;
let lo = 0, hi = header.articleCount - 1;
while (lo <= hi) {
const mid = (lo + hi) >> 1;
const dirent = await readDirent(fd, await ptr64(fd, header.urlPtrPos, mid));
const dirKey = dirent.namespace + dirent.url;
const cmp = key < dirKey ? -1 : key > dirKey ? 1 : 0;
if (cmp === 0) return dirent;
if (cmp < 0) hi = mid - 1; else lo = mid + 1;
}
return null;
}
async #getBlob(clusterNumber, blobNumber) {
const start = await this.#ptr64(this.#header.clusterPtrPos, clusterNumber);
const isLast = clusterNumber === this.#header.clusterCount - 1;
const end = isLast
? (await fs.promises.stat(this.path)).size
: await this.#ptr64(this.#header.clusterPtrPos, clusterNumber + 1);
async function resolveRedirect(fd, header, dirent) {
if (dirent.mimetype !== 0xffff) return dirent;
return readDirent(fd, await ptr64(fd, header.urlPtrPos, dirent.redirectIndex));
}
const raw = await this.#read(start, end - start);
const compType = raw[0] & 0x0f;
const extended = (raw[0] & 0x10) !== 0;
const body = raw.subarray(1);
async function getBlob(fd, header, filePath, clusterNumber, blobNumber) {
const start = await ptr64(fd, header.clusterPtrPos, clusterNumber);
const isLast = clusterNumber === header.clusterCount - 1;
const end = isLast
? (await fs.promises.stat(filePath)).size
: await ptr64(fd, header.clusterPtrPos, clusterNumber + 1);
let data;
if (compType <= 1) data = body;
else if (compType === 4) data = Buffer.from(lzmaDecompress(toLzmaAloneStream(body)));
else if (compType === 5) data = Buffer.from((await getZstd()).decompress(new Uint8Array(body)));
else throw new Error(`Unsupported cluster compression type: ${compType}`);
if (!data) throw new Error(`Cluster ${clusterNumber} failed to decompress (compType ${compType})`);
const raw = await readAt(fd, start, end - start);
const compType = raw[0] & 0x0f;
const extended = (raw[0] & 0x10) !== 0;
const body = raw.subarray(1);
const readPtr = i => extended ? Number(data.readBigUInt64LE(i * 8)) : data.readUInt32LE(i * 4);
return data.subarray(readPtr(blobNumber), readPtr(blobNumber + 1));
}
let data;
if (compType <= 1) data = Buffer.from(body);
else if (compType === 4 || compType === 5) data = await decompressPool.run({compType, body});
else throw new Error(`Unsupported cluster compression type: ${compType}`);
async #ptr64(base, index) {
return Number((await this.#read(base + index * 8, 8)).readBigUInt64LE(0));
}
const readPtr = i => extended ? Number(data.readBigUInt64LE(i * 8)) : data.readUInt32LE(i * 4);
return data.subarray(readPtr(blobNumber), readPtr(blobNumber + 1));
}
async #read(pos, length) {
const buf = Buffer.alloc(length);
await this.#fd.read(buf, 0, length, pos);
return buf;
}
async #readHeader() {
const b = await this.#read(0, HEADER_SIZE);
const titlePtrRaw = b.readBigUInt64LE(40);
this.#hasTitleListing = titlePtrRaw !== TITLE_SENTINEL;
this.#header = {
articleCount: b.readUInt32LE(24),
clusterCount: b.readUInt32LE(28),
urlPtrPos: Number(b.readBigUInt64LE(32)),
titlePtrPos: this.#hasTitleListing ? Number(titlePtrRaw) : null,
clusterPtrPos: Number(b.readBigUInt64LE(48)),
mimeListPos: Number(b.readBigUInt64LE(56)),
mainPage: b.readUInt32LE(64),
};
}
async #readMimeTypes() {
let pos = this.#header.mimeListPos, str = '';
for (;;) {
str += (await this.#read(pos, 1024)).toString('binary');
const end = str.indexOf('\0\0');
if (end !== -1) { str = str.slice(0, end + 1); break; }
pos += 1024;
}
this.#mimeTypes = str.split('\0').filter(Boolean);
}
/** Directory entry (article record) at byte `offset`, growing the read window as needed. */
async #readDirent(offset) {
for (let size = 512; ; size *= 2) {
const buf = await this.#read(offset, size);
let o = 0;
const mimetype = buf.readUInt16LE(o); o += 2;
o += 1; // extraLen, unused
const namespace = String.fromCharCode(buf.readUInt8(o)); o += 1;
o += 4; // revision, unused
let redirectIndex = null, cluster = null, blob = null;
if (mimetype === 0xffff) { redirectIndex = buf.readUInt32LE(o); o += 4; }
else { cluster = buf.readUInt32LE(o); o += 4; blob = buf.readUInt32LE(o); o += 4; }
const urlEnd = buf.indexOf(0, o);
if (urlEnd === -1) continue;
const titleEnd = buf.indexOf(0, urlEnd + 1);
if (titleEnd === -1) continue;
const url = buf.toString('utf8', o, urlEnd);
const title = buf.toString('utf8', urlEnd + 1, titleEnd) || url;
return {mimetype, namespace, redirectIndex, cluster, blob, url, title};
}
}
async #resolveRedirect(dirent) {
if (dirent.mimetype !== 0xffff) return dirent;
return this.#readDirent(await this.#ptr64(this.#header.urlPtrPos, dirent.redirectIndex));
}
/** Narrows to dirents near the term's alphabetical position in the title index. */
async #titleIndexCandidates(term) {
const q = term.toLowerCase();
let lo = 0, hi = this.#header.articleCount - 1;
while (lo < hi) {
const mid = (lo + hi) >> 1;
const urlIdx = (await this.#read(this.#header.titlePtrPos + mid * 4, 4)).readUInt32LE(0);
const dirent = await this.#readDirent(await this.#ptr64(this.#header.urlPtrPos, urlIdx));
if (dirent.title.toLowerCase() < q) lo = mid + 1; else hi = mid;
}
// Widen around the prefix match since fuzzy scoring isn't purely alphabetical.
const start = Math.max(0, lo - 50), end = Math.min(this.#header.articleCount, lo + 200);
const dirents = [];
for (let i = start; i < end; i++) {
const urlIdx = (await this.#read(this.#header.titlePtrPos + i * 4, 4)).readUInt32LE(0);
dirents.push(await this.#readDirent(await this.#ptr64(this.#header.urlPtrPos, urlIdx)));
}
return dirents;
}
async close() {
if (this.#fd) await this.#fd.close();
this.#fd = null;
}
/** Reads an 'M' namespace metadata value (e.g. Name, Date, Title). Returns null if missing. */
async metadata(key) {
const page = await this.readPage(key, NS_METADATA);
return page ? page.data.toString('utf8') : null;
}
/** Opens the archive and parses its header + mimetype list. */
async open() {
this.#fd = await fs.promises.open(this.path, 'r');
await this.#readHeader();
await this.#readMimeTypes();
return this;
}
/** Read a page's content by URL. Returns `{mimetype, data}` or `null` if not found. */
async readPage(url, namespace = NS_CONTENT) {
let dirent = await this.#findByUrl(url, namespace);
/**
* One-shot read of a single entry from a .zim archive - no server required.
* Opt-in convenience for callers who don't want to run kiwix-serve for a single
* lookup. Re-parses the header/mimetype list on every call; fine for occasional
* reads, not meant for high-volume access (use KiwixServer for that).
*/
export async function readZimEntry(zimPath, url, namespace = NS_CONTENT) {
const fd = await fs.promises.open(zimPath, 'r');
try {
const header = await readHeader(fd);
const mimeTypes = await readMimeTypes(fd, header.mimeListPos);
let dirent = await findByUrl(fd, header, url, namespace);
if (!dirent) return null;
dirent = await this.#resolveRedirect(dirent);
const data = await this.#getBlob(dirent.cluster, dirent.blob);
return {mimetype: this.#mimeTypes[dirent.mimetype] || 'application/octet-stream', data};
}
/** Reads the archive's designated main/landing page, if one is set. */
async mainPage() {
if (this.#header.mainPage === 0xffffffff) return null;
let dirent = await this.#readDirent(await this.#ptr64(this.#header.urlPtrPos, this.#header.mainPage));
dirent = await this.#resolveRedirect(dirent);
const data = await this.#getBlob(dirent.cluster, dirent.blob);
return {mimetype: this.#mimeTypes[dirent.mimetype] || 'application/octet-stream', data, url: dirent.url};
}
/**
* Fuzzy-ranked title search. Accepts comma-separated `terms` the same way the
* catalog search does. Uses the sorted title index when present (binary search
* narrows the candidate window); falls back to a full linear scan otherwise
* (common on ZIM v6+/zimit-generated archives with no title index).
*/
async search(terms, {limit = 20, htmlOnly = true} = {}) {
const termList = String(terms).split(',').map(t => t.trim()).filter(Boolean);
if (!termList.length) return [];
const candidates = this.#hasTitleListing
? await this.#titleIndexCandidates(termList[0])
: await this.#allDirents();
const scored = [];
for (const dirent of candidates) {
if (dirent.namespace !== NS_CONTENT) continue;
if (htmlOnly && !(this.#mimeTypes[dirent.mimetype] || '').startsWith('text/html')) continue;
const {max} = fuzzyMatch(dirent.title, ...termList);
scored.push({url: dirent.url, title: dirent.title, namespace: dirent.namespace, score: max});
}
return scored.toSorted((a, b) => b.score - a.score).slice(0, limit);
dirent = await resolveRedirect(fd, header, dirent);
const data = await getBlob(fd, header, zimPath, dirent.cluster, dirent.blob);
return {mimetype: mimeTypes[dirent.mimetype] || 'application/octet-stream', data};
} finally {
await fd.close();
}
}
/** Reads 'M' namespace metadata (Title, Creator, Date, etc.) without a server. Minimal by design - no icon/size/counts. */
export async function readZimMetadata(zimPath) {
const keys = ['Title', 'Creator', 'Publisher', 'Date', 'Description', 'Language', 'Name', 'Tags'];
const entries = await Promise.all(keys.map(k => readZimEntry(zimPath, k, NS_METADATA)));
const [title, creator, publisher, date, description, language, name, tags] = entries.map(e => e?.data.toString('utf8') ?? null);
return {
title, updated: date ? new Date(date) : null, summary: description, language, name,
category: tags ? tags.split(';')[0] || '' : '', tags: tags ? tags.split(';') : [],
author: creator, publisher,
};
}

215
src/server.js Normal file
View File

@@ -0,0 +1,215 @@
'use strict';
import {spawn, execFile} from 'node:child_process';
import {promisify} from 'node:util';
import net from 'node:net';
import fs from 'node:fs';
import path from 'node:path';
import {fileURLToPath} from 'node:url';
import {fromXml} from '@ztimson/utils';
const execFileAsync = promisify(execFile);
const __dirname = path.dirname(fileURLToPath(import.meta.url));
const DEFAULT_BIN_DIR = path.join(__dirname, '..', 'bin'); // npm package root/bin - where bin/install.js drops the kiwix-tools binaries
const READY_TIMEOUT = 10_000;
const READY_POLL_INTERVAL = 100;
function findFreePort() {
return new Promise((resolve, reject) => {
const srv = net.createServer();
srv.on('error', reject);
srv.listen(0, () => {
const {port} = srv.address();
srv.close(() => resolve(port));
});
});
}
/** Owns a kiwix-serve process's full lifecycle: library.xml, start/stop/reload, content + search access. */
export class KiwixServer {
#dir;
#host;
#port;
#binDir;
#libraryPath;
#child = null;
#remote; // baseUrl string if attached to an externally-managed kiwix-serve, else null
get port() { return this.#port; }
get running() { return !!this.#remote || !!this.#child; }
get baseUrl() { return this.#remote || (this.#child ? `http://${this.#host}:${this.#port}` : null); }
/** @param {{port?: number, host?: string, binDir?: string, url?: string}} [opts]
* url: attach to an already-running kiwix-serve (e.g. one started elsewhere in your codebase) instead of
* spawning/owning one - start/stop/reload become no-ops, and library.xml is read over HTTP instead of disk. */
constructor(dir, {port, host = '127.0.0.1', binDir = DEFAULT_BIN_DIR, url} = {}) {
this.#dir = dir;
this.#host = host;
this.#port = port;
this.#binDir = binDir;
this.#libraryPath = path.join(dir, 'library.xml');
this.#remote = url ? url.replace(/\/$/, '') : null;
}
#assertRunning() {
if (!this.running) throw new Error('KiwixServer is not running - call start() first');
}
#bin(name) {
return path.join(this.#binDir, process.platform === 'win32' ? `${name}.exe` : name);
}
/** Reads library.xml from disk if we own the server, or over HTTP if attached to a remote one. */
async #fetchLibraryXml() {
if (this.#remote) return (await fetch(`${this.#remote}/library.xml`).catch(() => null))?.text?.() ?? '';
return fs.promises.readFile(this.#libraryPath, 'utf8').catch(() => '');
}
/** Rebuilds library.xml from scratch by scanning `dir` for .zim files - no-op if attached to a remote server. */
async #rebuildLibrary() {
if (this.#remote) return;
await fs.promises.rm(this.#libraryPath, {force: true});
for (const f of await this.#zimFiles()) await execFileAsync(this.#bin('kiwix-manage'), [this.#libraryPath, 'add', path.join(this.#dir, f)]);
}
async #waitUntilReady() {
const deadline = Date.now() + READY_TIMEOUT;
while (Date.now() < deadline) {
try {
await fetch(`http://${this.#host}:${this.#port}/`);
return;
} catch {}
await new Promise(r => setTimeout(r, READY_POLL_INTERVAL));
}
throw new Error('kiwix-serve did not become ready in time');
}
async #zimFiles() {
return (await fs.promises.readdir(this.#dir).catch(() => [])).filter(f => f.endsWith('.zim'));
}
/** Rebuilds library.xml and starts kiwix-serve. Resolves once the server is responding. */
async start() {
if (this.#remote || this.#child) return;
await fs.promises.mkdir(this.#dir, {recursive: true});
await this.#rebuildLibrary();
this.#port ??= await findFreePort();
this.#child = spawn(this.#bin('kiwix-serve'), ['--library', '-i', this.#host, '-p', String(this.#port), this.#libraryPath], {stdio: 'ignore'});
this.#child.on('exit', () => { this.#child = null; });
try {
await this.#waitUntilReady();
} catch (e) {
await this.stop();
throw e;
}
}
/** Gracefully stops kiwix-serve, if we own it. No-op if attached to a remote instance. */
async stop() {
if (this.#remote || !this.#child) return;
const child = this.#child;
await new Promise(resolve => {
child.once('exit', resolve);
child.kill('SIGTERM');
});
this.#child = null;
}
async restart() {
await this.stop();
await this.start();
}
/** Rebuilds library.xml from disk and restarts kiwix-serve. No-op if attached to a remote instance -
* whoever owns that process is responsible for reloading it. */
async reload() {
if (this.#remote || !this.#child) return;
await this.restart();
}
/** Local catalog listing - same flat shape as the online catalog (catalog.js), plus a `file` field. */
async list() {
this.#assertRunning();
const xml = await this.#fetchLibraryXml();
const entries = fromXml(xml);
return (entries?.library?.book || []).map(e => {
const tags = e.tags.split(';');
const name = e.path.replaceAll('.zim', '');
return {
id: e.id,
title: e.title,
updated: new Date(e.date),
summary: e.description,
language: e.language,
name: e.name,
category: tags.find(t => t.startsWith('_category'))?.slice(10) || '',
tags,
mediaCount: +e.mediaCount || 0,
author: e.creator,
publisher: e.publisher,
articleCount: +e.articleCount || 0,
sizeMb: +(Number(e.size) / 1024).toFixed(1) || 0,
href: name,
icon: `data:${e.faviconMimetype || 'image/png'};base64,${e.favicon}`,
viewer: `${this.baseUrl}/content/${name}`,
};
});
}
/** Splits a href ("zim/path/to/page") or a full content/viewer URL into {zim, path}. */
#splitHref(href) {
const clean = href.replace(`${this.baseUrl}/content/`, '').replace(/^\/+/, '');
const [zim, ...rest] = clean.split('/');
return {zim, path: rest.join('/')};
}
/** Builds a kiwix-serve content URL from a href (as returned by list()/search()), or from an already-built content/viewer URL. */
link(href) {
this.#assertRunning();
const {zim, path} = this.#splitHref(href);
return `${this.baseUrl}/content/${zim}${path ? '/' + path : ''}`;
}
/** Fetches a single asset's raw bytes straight from kiwix-serve. */
async raw(href) {
const res = await fetch(this.link(href));
if(!res.ok) return null;
return {mimetype: res.headers.get('content-type'), data: Buffer.from(await res.arrayBuffer())};
}
/** Fulltext search across every local ZIM via kiwix-serve's own xapian index */
async search(terms, limit = 20) {
this.#assertRunning();
const termList = String(terms).split(/[,\s]+/).map(t => t.trim()).filter(Boolean);
if (!termList.length) return [];
const params = new URLSearchParams({pattern: termList.join(' '), format: 'xml', pageLength: String(limit)});
const res = await fetch(`${this.baseUrl}/search?${params}`);
if (!res.ok) return [];
const found = fromXml(await res.text())?.rss?.channel?.item || [];
const books = await this.list();
const bookMap = new Map(books.map(b => [b.title, b]));
return found.map(hit => {
const book = bookMap.get(hit.book.title);
if (!book) return null;
const prefix = `/content/${book.href}/`;
const page = hit.link.startsWith(prefix) ? hit.link.slice(prefix.length) : hit.link.replace(/^\/+/, '');
return {
id: book.id,
title: hit.title,
page,
name: book.name,
publisher: book.publisher,
href: `${book.href}/${page}`,
icon: book.icon,
viewer: this.baseUrl + hit.link,
summary: hit.description,
score: +hit.score || 0,
};
}).filter(Boolean);
}
}

View File

@@ -14,16 +14,26 @@ export function levenshtein(a, b) {
return dp[m][n];
}
/** Normalized similarity in [0,1]: 1 - editDistance / maxLength. */
export function similarity(a, b) {
a = a.toLowerCase(); b = b.toLowerCase();
return 1 - levenshtein(a, b) / Math.max(a.length, b.length, 1);
function scoreAgainst(text, term) {
if (text.includes(term)) return 1 - (text.length - term.length) / text.length * 0.3;
if (!text.length || !term.length || text[0] !== term[0]) return 0;
const dist = levenshtein(text, term);
const maxAllowed = Math.max(1, Math.ceil(term.length * 0.34));
if (dist > maxAllowed) return 0;
return 1 - dist / Math.max(text.length, term.length);
}
/** Compares `target` against one or more search terms; returns avg/max/per-term similarity. */
export function fuzzyMatch(target, ...terms) {
if (!terms.length) throw new Error('Requires at least 1 term to compare');
const similarities = terms.map(t => similarity(target, t));
const lowerTarget = String(target).toLowerCase();
const words = lowerTarget.split(/\W+/).filter(Boolean);
const similarities = terms.map(term => {
const t = term.toLowerCase();
return Math.max(scoreAgainst(lowerTarget, t), ...words.map(w => scoreAgainst(w, t)));
});
return {
avg: similarities.reduce((acc, s) => acc + s, 0) / similarities.length,
max: Math.max(...similarities),