diff --git a/README.md b/README.md index 2bd3546..5e271a5 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,3 @@ -

@@ -9,7 +8,7 @@ ### Zim Utils -Native, dependency-light ZIM archive reader/searcher and Kiwix catalog downloader for Node.js +Node.js wrapper around [kiwix-tools](https://github.com/kiwix/kiwix-tools) for managing a local ZIM library, serving content, and searching both locally and against the Kiwix catalog [![Version](https://img.shields.io/badge/dynamic/json.svg?label=Version&style=for-the-badge&url=https://git.zakscode.com/api/v1/repos/ztimson/zim-utils/tags&query=$[0].name)](https://git.zakscode.com/ztimson/zim-utils/tags) @@ -36,24 +35,23 @@ Native, dependency-light ZIM archive reader/searcher and Kiwix catalog downloade - [Setup](#setup) - [Production](#production) - [Usage](#usage) - - [ZimManager](#zimmanager) - - [ZimReader](#zimreader) + - [KiwixServer](#kiwixserver) - [Catalog](#catalog) - [License](#license) ## About -`@ztimson/zim-utils` is a native, dependency-light toolkit for working with [ZIM](https://wiki.openzim.org/wiki/ZIM_file_format) archives and the [Kiwix](https://kiwix.org/) catalog in Node.js +`@ztimson/zim-utils` manages a local [ZIM](https://wiki.openzim.org/wiki/ZIM_file_format) library by driving the official [kiwix-tools](https://github.com/kiwix/kiwix-tools) binaries (`kiwix-serve`, `kiwix-manage`) as child processes, rather than re-implementing ZIM reading from scratch. This means content serving, indexing, and fulltext search all come straight from Kiwix's own battle-tested implementation. -It comes with the following helpers: +It comes with: -- **`ZimManager`** — Local library manager: listing, updating/downloading, and running searches accross the entire library -- **`ZimReader`** — A `.zim` file reader for reading pages, metadata and running searches -- **Catalog** (`zimCatalog`/`zimCatalogInfo`) — Helper functions to search the Kiwix OPDS catalog +- **`KiwixServer`** — Owns a directory of `.zim` files: builds/rebuilds `library.xml`, starts/stops `kiwix-serve`, lists the local library, runs fulltext search, and resolves content URLs/bytes for any archive+page +- **Catalog** (`zimCatalog`/`zimCatalogInfo`) — Helper functions to search the remote Kiwix OPDS catalog, for discovering and downloading new ZIMs ### Built With [![Node](https://img.shields.io/badge/Node.js-000000?style=for-the-badge&logo=nodedotjs)](https://nodejs.org/) [![JavaScript](https://img.shields.io/badge/JavaScript-000000?style=for-the-badge&logo=javascript)](https://javascript.com/) +[![Kiwix](https://img.shields.io/badge/kiwix--tools-000000?style=for-the-badge&logo=kiwix)](https://github.com/kiwix/kiwix-tools) ## Setup @@ -68,84 +66,121 @@ It comes with the following helpers: - [Node.js](https://nodejs.org/en/download) #### Instructions -1. Install the dependencies: `npm install` -2. Install the package: `npm install @ztimson/zim-utils` +1. Install the package: `npm install @ztimson/zim-utils` +2. On first install, `kiwix-serve`/`kiwix-manage` binaries are fetched into `./bin` (or wherever `binDir` points) - no separate Kiwix install needed. ## Usage -### ZimManager +### KiwixServer -`ZimManager` owns a directory of `.zim` files and handles everything from downloading to cross-archive search. +`KiwixServer` owns a directory of `.zim` files and manages the whole `kiwix-serve` lifecycle for you. ```js -import {ZimManager} from '@ztimson/zim-utils'; +import {KiwixServer} from '@ztimson/zim-utils'; -const manager = new ZimManager('./zims'); // optional 2nd arg: custom catalog URL - -// Search the Kiwix catalog & download the top hit -const [entry] = await manager.catalog('wikipedia,medicine'); -await manager.download(entry.href); - -// List local archives with their parsed metadata -const local = await manager.list(); -// [{file: './zims/wikipedia_en_medicine.zim', meta: {name, date, title}}, ...] - -// Check a single file for updates without downloading -const status = await manager.isOutdated(local[0].file); - -// Update every local ZIM that has a newer catalog version -await manager.updateAll({force: false}); - -// Fuzzy-search titles across ALL local archives at once -const hits = await manager.search('diabetes treatment', {limit: 10}); - -// Open a reader by file path OR by catalog name -const reader = await manager.open('wikipedia_en_medicine'); -const page = await reader.readPage('A/Diabetes'); -await reader.close(); +const server = new KiwixServer('./zims'); // optional 2nd arg: {port, host, binDir} +await server.start(); // rebuilds library.xml, spawns kiwix-serve, waits until ready +await server.reload(); // pick up newly added/removed .zim files, no downtime +await server.stop(); // Kill server ``` -### ZimReader -Everything `ZimManager` does to a single archive is just a thin wrapper around `ZimReader`. Use it directly when you don't need a whole managed library: +#### List Local ZIMs +```js +const local = await server.list(); +[ + { + id: '37a99758-43a0-6ba3-cd54-1af556369eee', + title: 'FOSS cooking', + updated: 2026-05-06T00:00:00.000Z, + summary: 'Making cooking fast, easy, foss', + language: 'eng', + name: 'foss.cooking_en_all', + category: 'other', + tags: ['_category:other', '_ftindex:yes', 'preppers,food', '_pictures:yes', '_videos:yes', '_details:yes'], + mediaCount: 154, + author: '-', + publisher: 'openZIM', + articleCount: 719, + sizeMb: 23.1, + href: 'foss.cooking_en_all_2026-05', + icon: undefined, + viewer: 'http://127.0.0.1:51992/content/foss.cooking_en_all_2026-05', + }, + // ... +] +``` + +#### Search Local ZIMs ```js -import {ZimReader} from '@ztimson/zim-utils'; +// Fulltext search across every local archive: xapian prefilter, then fuzzy re-ranked +const hits = await server.search('chocolate', 5); +[ + { + id: '37a99758-43a0-6ba3-cd54-1af556369eee', + title: 'Chocolate Chip Cookies', + page: 'foss.cooking/recipe/mfed3/chocolate-chip-cookies', + name: 'foss.cooking_en_all', + publisher: 'openZIM', + href: 'foss.cooking_en_all_2026-05/foss.cooking/recipe/mfed3/chocolate-chip-cookies', + icon: undefined, + viewer: 'http://127.0.0.1:51992/content/foss.cooking_en_all_2026-05/foss.cooking/recipe/mfed3/chocolate-chip-cookies', + summary: '...cream scooper or spoon to make uniform balls of cookie dough...', + score: 0.67, + }, + // ... +] +``` -const reader = await new ZimReader('./zims/wikipedia_en_medicine.zim').open(); +#### View ZIM Content -// Metadata (what manager.#readMeta / isOutdated rely on) -const name = await reader.metadata('Name'); -const date = await reader.metadata('Date'); -const title = await reader.metadata('Title'); +```js +// Build a content URL from any href (list()/search() output, or a full viewer URL) without hitting the network +const url = server.fetch(hits[0].href); +'http://127.0.0.1:51992/content/foss.cooking_en_all_2026-05/foss.cooking/recipe/mfed3/chocolate-chip-cookies' -// Landing page -const home = await reader.mainPage(); - -// Direct page lookup by URL -const page = await reader.readPage('A/Diabetes'); -console.log(page.mimetype, page.data.toString('utf8')); - -// Fuzzy title search within just this archive (what manager.search fans out over) -const results = await reader.search('diabetes,insulin', {limit: 20, htmlOnly: true}); - -await reader.close(); +// Or fetch the bytes directly, proxied straight from kiwix-serve +const {mimetype, data} = await server.raw(hits[0].href); +{mimetype: 'text/html; charset=utf-8', data: } ``` ### Catalog -`ZimManager.catalog()` and its update checks are backed directly by these two functions: ```js import {zimCatalog, zimCatalogInfo, CATALOG_URL} from '@ztimson/zim-utils'; -// Ranked search across the Kiwix catalog (comma-separated terms, like ZimReader.search) -const results = await zimCatalog('history,rome', {lang: 'eng', count: 20, url: CATALOG_URL}); +// Search the remote Kiwix catalog to discover new ZIMs +const search = await zimCatalog('knots', {lang: 'eng', count: 20, url: CATALOG_URL}); +// OR Exact lookup by catalog `name`, useful for checking if a local copy is outdated +const entry = await zimCatalogInfo('wikipedia_en_knots'); -// Exact lookup by catalog `name`, used to check if a local copy is outdated -const entry = await zimCatalogInfo('wikipedia_en_medicine'); +[ + { + id: 'urn:uuid:3a4fe0d0-0bd0-7583-ada8-c52d173ae44d', + title: 'Knots by Wikipedia', + updated: 2026-07-20T00:00:00.000Z, + summary: 'A subset of Wikipedia encyclopedia dedicated to knots', + language: 'eng', + name: 'wikipedia_en_knots', + category: 'wikipedia', + tags: ['wikipedia', '_category:wikipedia', '_pictures:yes', '_videos:no', '_details:yes', '_ftindex:yes'], + mediaCount: 3501, + author: 'Wikipedia', + publisher: 'openZIM', + articleCount: 1730, + sizeMb: 17.6, + href: 'https://lb.download.kiwix.org/zim/wikipedia/wikipedia_en_knots_maxi_2026-07.zim.meta4', + icon: 'https://library.kiwix.org/catalog/v2/illustration/3a4fe0d0-0bd0-7583-ada8-c52d173ae44d/?size=48', + viewer: 'https://library.kiwix.org/viewer#wikipedia_en_knots', + }, + // ... +] + +// Download catalog ZIM if out of date (unless forced) and reload automatically +await mgr.download(search[0].href, force); ``` ## License Copyright © 2026 Zakary Timson | Available under MIT Licensing - diff --git a/bin/install-kwix.js b/bin/install-kwix.js index 2e78fe5..9cbf139 100644 --- a/bin/install-kwix.js +++ b/bin/install-kwix.js @@ -11,9 +11,7 @@ const VERSION = '3.8.1'; const BASE_URL = 'https://download.kiwix.org/release/kiwix-tools'; const PLATFORM_MAP = {linux: 'linux', darwin: 'macos', win32: 'win'}; -const BIN_NAME = process.platform === 'win32' ? 'kiwix-search.exe' : 'kiwix-search'; const BIN_DIR = path.join(__dirname, '..', 'bin'); // project root/bin -const BIN_PATH = path.join(BIN_DIR, BIN_NAME); /** Download a file, following redirects. */ function download(url, dest) { @@ -63,22 +61,16 @@ async function downloadAndExtract() { console.log('Extracting...'); execFileSync('tar', ['-xf', archivePath, '-C', tmpDir]); // bsdtar handles zip too - const extractedBin = findFile(tmpDir, BIN_NAME); - if (!extractedBin) throw new Error(`Could not find ${BIN_NAME} in extracted archive`); - const extractedDir = path.dirname(extractedBin); - await fs.promises.mkdir(BIN_DIR, {recursive: true}); // Copy the binary plus any DLLs sitting alongside it (Windows deps) - for (const entry of await fs.promises.readdir(extractedDir)) { - if (entry === BIN_NAME || entry.toLowerCase().endsWith('.dll')) { - await fs.promises.copyFile(path.join(extractedDir, entry), path.join(BIN_DIR, entry)); - } + for (const entry of await fs.promises.readdir(tmpDir)) { + if(!/\.(tar|gz|zip)$/.test(entry)) await fs.promises.copyFile(path.join(tmpDir, entry), path.join(BIN_DIR, entry)); + if(process.platform !== 'win32') await fs.promises.chmod(BIN_DIR, 0o755, {recursive: true}); } - if (process.platform !== 'win32') await fs.promises.chmod(BIN_PATH, 0o755); await fs.promises.rm(tmpDir, {recursive: true, force: true}); - console.log('Installed to:', BIN_PATH); + console.log('Installed to:', BIN_DIR); } downloadAndExtract().catch(err => { diff --git a/package-lock.json b/package-lock.json index cba0faf..d70aa8b 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,23 +1,24 @@ { "name": "@ztimson/zim-utils", - "version": "0.1.0", + "version": "0.2.5", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "@ztimson/zim-utils", - "version": "0.1.0", + "version": "0.2.5", + "hasInstallScript": true, "license": "MIT", "dependencies": { - "@ztimson/utils": "^0.30.7", + "@ztimson/utils": "^0.30.8", "lzma1": "^0.3.0", "zstd-codec": "^0.1.5" } }, "node_modules/@ztimson/utils": { - "version": "0.30.7", - "resolved": "https://registry.npmjs.org/@ztimson/utils/-/utils-0.30.7.tgz", - "integrity": "sha512-0TkjQFVe0edTqlezIfUtwTXA9pkS1rVYw0xzFycKmtcoldh6urQEKQ/T6Z0Id1+w53ubomSdVfj6GB3u29+AOQ==", + "version": "0.30.8", + "resolved": "https://registry.npmjs.org/@ztimson/utils/-/utils-0.30.8.tgz", + "integrity": "sha512-+vBjcinqckqMHkP95xWiQeQz2E7Q1oS0b+Odjp+F9rvQ4z0US4JdodjqhEaqh+RAO/yP77x4Eu0a04yBB4HNdw==", "license": "MIT", "dependencies": { "var-persist": "^1.0.1" diff --git a/package.json b/package.json index f4bcb14..e8ab8b6 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@ztimson/zim-utils", - "version": "0.2.5", + "version": "0.3.0", "description": "Native, dependency-light ZIM archive reader/searcher and Kiwix catalog downloader for Node.js", "author": "Zak Timson", "license": "MIT", @@ -15,7 +15,7 @@ "postinstall": "node ./bin/install-kwix.js" }, "dependencies": { - "@ztimson/utils": "^0.30.7", + "@ztimson/utils": "^0.30.8", "lzma1": "^0.3.0", "zstd-codec": "^0.1.5" } diff --git a/src/catalog.js b/src/catalog.js index bdd7508..45249e5 100644 --- a/src/catalog.js +++ b/src/catalog.js @@ -5,7 +5,7 @@ export const CATALOG_URL = 'https://library.kiwix.org'; const PAGE_SIZE = 100; -function parseEntries(xml, catalog = CATALOG_URL) { +export function parseEntries(xml, catalog = CATALOG_URL) { const blocks = xml.match(/[\s\S]*?<\/entry>/g) || []; return blocks.map(b => { const grab = re => (b.match(re) || [])[1] || ''; @@ -29,7 +29,7 @@ function parseEntries(xml, catalog = CATALOG_URL) { publisher: grab(/\s*([^<]*)<\/name>\s*<\/publisher>/m), articleCount: Number(grab(/([^<]*)<\/articleCount>/)) || 0, sizeMb: linkMatch ? +(Number((b.match(/length=["'](\d+)["']/) || [])[1] || 0) / 1024 / 1024).toFixed(1) : '?', - download: linkMatch ? linkMatch[1] : null, + href: linkMatch ? linkMatch[1] : null, icon: iconMatch ? new URL(iconMatch[1], catalog).href : null, viewer: name ? new URL(`viewer#${name}`, catalog).href : null, }; @@ -56,7 +56,7 @@ export async function zimCatalog(terms, opts = {lang: 'eng', count: 20, url: CAT opts = Object.assign({lang: 'eng', count: 20, url: CATALOG_URL}, opts) const termList = [...String(terms).split(',')].filter(Boolean).map(t => t.trim().toLowerCase()); const results = await Promise.allSettled(termList.map(t => fetchEntries(t, opts.lang, opts.url))); - const byName = new Map(); // name -> {entry, hitTerms:Set} + const byName = new Map(); results.forEach((r, i) => { if (r.status !== 'fulfilled') return; const term = termList[i]; diff --git a/src/index.js b/src/index.js index 4ca7a76..4a7f450 100644 --- a/src/index.js +++ b/src/index.js @@ -1,4 +1,5 @@ export * from './catalog.js'; export * from './manager.js'; export * from './reader.js'; +export * from './server.js'; export * from './utils.js'; diff --git a/src/manager.js b/src/manager.js index e2b1c24..1b4cfca 100644 --- a/src/manager.js +++ b/src/manager.js @@ -2,26 +2,32 @@ import fs from 'node:fs'; import path from 'node:path'; import {pipeline} from 'node:stream/promises'; import {Readable} from 'node:stream'; -import {ZimReader} from './reader.js'; +import {KiwixServer} from './server.js'; import {zimCatalog, zimCatalogInfo, CATALOG_URL} from './catalog.js'; -const DEFAULT_READER_TTL = 60_000; - -/** Manages a local directory of ZIM archives: listing, update checks, downloads, and reading. */ +/** Manages a local directory of ZIM archives: catalog search, downloads, update checks, deletion. + * Reuses (or owns & lazily starts) a KiwixServer for local listing/search, so `list()` and `catalog()` return the same shape. */ export class ZimManager { - #catalog; + #catalogUrl; #dir; - #readerTTL; - #clusterTTL; - #readers = new Map(); // file -> {reader, timer} - #opening = new Map(); // file -> Promise, dedupes concurrent first-open races + server; + #ownsServer; - /** @param {{catalog?: string, readerTTL?: number, clusterTTL?: number}} [opts] */ - constructor(dir, catalog = CATALOG_URL, {readerTTL = DEFAULT_READER_TTL, clusterTTL} = {}) { - this.#catalog = catalog; + /** @param {{catalog?: string, server?: KiwixServer, port?: number, host?: string, binDir?: string}} [opts] + * server: reuse an existing KiwixServer; if omitted, one is created (not yet started) and exposed via `.server` for reuse elsewhere. */ + constructor(dir, {catalog = CATALOG_URL, server, port, host, binDir} = {}) { + this.#catalogUrl = catalog; this.#dir = dir; - this.#readerTTL = readerTTL; - this.#clusterTTL = clusterTTL; // undefined -> ZimReader's own default + this.#ownsServer = !server; + this.server = server ?? new KiwixServer(dir, {port, host, binDir}); + } + + /** The KiwixServer backing this manager - reuse it directly for content/search access, or pass into another ZimManager. */ + get server() { return this.server; } + + async #ensureServer() { + if (!this.server.running) await this.server.start(); + return this.server; } async #download(url, destPath) { @@ -32,10 +38,6 @@ export class ZimManager { await fs.promises.rename(tmpPath, destPath); } - async #ensureDir() { - await fs.promises.mkdir(this.#dir, {recursive: true}); - } - /** Resolves a `.meta4` metalink URL down to the real mirror `.zim` download URL. */ async #resolveUrl(url) { const head = await fetch(url); @@ -48,159 +50,102 @@ export class ZimManager { return {res, url: m[1]}; } - /** Reads Name/Date/Title metadata from a local ZIM file. Returns null if unreadable. */ - async #readMeta(filepath) { - let reader; - try { - reader = await new ZimReader(filepath).open(); - return await reader.metadata(); - } catch { - return null; - } finally { - await reader?.close(); - } - } - async #update(name, catalogEntry, localMatch, force) { const remoteDate = catalogEntry.updated ? new Date(catalogEntry.updated) : null; - const localDate = localMatch?.meta?.updated ?? null; + const localDate = localMatch?.updated ?? null; if (!force && localMatch && remoteDate && localDate && remoteDate <= localDate) return {name, status: 'skipped', reason: 'up to date'}; - if (!catalogEntry.download) return {name, status: 'skipped', reason: 'missing download link'}; + if (!catalogEntry.href) return {name, status: 'skipped', reason: 'missing href'}; - const filename = path.basename(new URL(catalogEntry.download).pathname).replace(/\.meta4$/i, ''); + const filename = path.basename(new URL(catalogEntry.href).pathname).replace(/\.meta4$/i, ''); const destPath = path.join(this.#dir, filename); - await this.#download(catalogEntry.download, destPath); - if (localMatch && localMatch.file !== filename) await this.#evict(path.join(this.#dir, localMatch.file)); + await this.#download(catalogEntry.href, destPath); + if (localMatch && localMatch.file !== filename) await fs.promises.rm(path.join(this.#dir, localMatch.file), {force: true}); + const server = await this.#ensureServer(); + await server.reload(); return {name, status: 'updated', file: filename}; } - /** Resolves a file path or catalog `name` to a local file path. */ - async #resolveFile(fileOrName) { - if (fs.existsSync(fileOrName)) return fileOrName; - const joined = path.join(this.#dir, fileOrName); - if (fs.existsSync(joined)) return joined; + catalog(search, opts = {}) { + return zimCatalog(search, {url: this.#catalogUrl, ...opts}); + } + + /** Stops the internally-owned KiwixServer, if this manager created its own (no-op if one was passed in). */ + async close() { + if (this.#ownsServer) await this.server.stop(); + } + + async delete(nameOrFile) { const local = await this.list(); - const match = local.find(l => l.meta?.name === fileOrName || path.basename(l.file) === fileOrName); - if (!match) throw new Error(`ZIM not found locally: ${fileOrName}`); - return path.join(this.#dir, match.file); - } - - /** Closes and drops a cached reader for `file`, if any (used before delete/replace). */ - async #evict(file) { - const entry = this.#readers.get(file); - if (!entry) return new ZimReader(file).delete().catch(() => {}); - clearTimeout(entry.timer); - this.#readers.delete(file); - await entry.reader.delete(); - } - - catalog(search, opts) { - return zimCatalog(search, opts); - } - - async delete(fileOrName) { - const file = await this.#resolveFile(fileOrName); - await this.#evict(file); - return {file: path.basename(file), status: 'deleted'}; - } - - /** Checks whether a local ZIM has a newer version in the catalog, without downloading. */ - async isOutdated(file) { - const meta = await this.#readMeta(file); - if (!meta?.name) return {file, upToDate: null, reason: 'no metadata'}; - const catalogEntry = await zimCatalog(meta.name, this.#catalog); - if (!catalogEntry) return {file, upToDate: null, reason: 'missing from catalog'}; - const remoteDate = catalogEntry.updated ? new Date(catalogEntry.updated) : null; - const localDate = meta.updated ?? null; - return {file, name: meta.name, upToDate: !!(remoteDate && localDate && remoteDate <= localDate), localDate, remoteDate}; - } - - /** Lists local `.zim` files with their parsed metadata (or `null` if unreadable). */ - async list() { - await this.#ensureDir(); - const files = (await fs.promises.readdir(this.#dir)).filter(f => f.endsWith('.zim')); - return Promise.all(files.map(async f => { - const file = path.join(this.#dir, f); - return {file: f, ...(await this.#readMeta(file))}; - })); + const match = local.find(l => l.name === nameOrFile || l.file === nameOrFile); + if (!match) throw new Error(`ZIM not found locally: ${nameOrFile}`); + await fs.promises.rm(path.join(this.#dir, match.file), {force: true}); + const server = await this.#ensureServer(); + await server.reload(); + return {file: match.file, status: 'deleted'}; } /** Downloads/updates a single ZIM by direct href, matching against any existing local copy by name. */ async download(href, {force = false} = {}) { - await this.#ensureDir(); + await fs.promises.mkdir(this.#dir, {recursive: true}); const {url: finalUrl} = await this.#resolveUrl(href); const filename = path.basename(new URL(finalUrl).pathname).replace(/\.meta4$/i, ''); const name = filename.replace(/\.zim$/i, '').replace(/_\d{4}-\d{2}(?:_\d+)?$/, ''); const local = await this.list(); - const localMatch = local.find(l => l.meta?.name === name) ?? null; - const catalogEntry = await zimCatalogInfo(name, this.#catalog) || {name, updated: null, download: href}; + const localMatch = local.find(l => l.name === name) ?? null; + const catalogEntry = await zimCatalogInfo(name, this.#catalogUrl) || {name, updated: null, href: href}; return this.#update(name, catalogEntry, localMatch, force); } - /** Checks all local ZIMs against the catalog and updates any that are outdated. */ - async updateAll({force = false}) { - const local = await this.list(); - if (!local.length) return []; + async link(href) { + const server = await this.#ensureServer(); + return server.link(href); + } + /** Checks whether a local ZIM has a newer version in the catalog, without downloading. */ + async isOutdated(nameOrFile) { + const local = await this.list(); + const match = local.find(l => l.name === nameOrFile || l.file === nameOrFile); + if (!match?.name) return {file: nameOrFile, upToDate: null, reason: 'no metadata'}; + const catalogEntry = await zimCatalogInfo(match.name, this.#catalogUrl); + if (!catalogEntry) return {file: match.file, upToDate: null, reason: 'missing from catalog'}; + const remoteDate = catalogEntry.updated ? new Date(catalogEntry.updated) : null; + return {file: match.file, name: match.name, upToDate: !!(remoteDate && match.updated && remoteDate <= match.updated), localDate: match.updated, remoteDate}; + } + + /** Lists local ZIMs - same flat shape as `catalog()` (plus `file`), sourced from the KiwixServer's own library.xml. */ + async list() { + const server = await this.#ensureServer(); + return server.list(); + } + + /** Fetches a single asset's raw bytes. */ + async raw(href) { + const server = await this.#ensureServer(); + return server.raw(href); + } + + /** Two-pass fulltext search across every local ZIM, returns enriched results matching catalog/list shape. */ + async search(terms, limit = 20) { + const server = await this.#ensureServer(); + return server.search(terms, limit); + } + + /** Checks all local ZIMs against the catalog and updates any that are outdated. */ + async updateAll({force = false} = {}) { + const local = await this.list(); const results = []; - for (const {file, meta} of local) { - if (!meta?.name) { results.push({file, status: 'skipped', reason: 'no metadata'}); continue; } - const catalogEntry = await zimCatalogInfo(meta.name, this.#catalog); - if (!catalogEntry) { results.push({name: meta.name, status: 'skipped', reason: 'missing from catalog'}); continue; } + for (const entry of local) { + if (!entry.name) { results.push({file: entry.file, status: 'skipped', reason: 'no metadata'}); continue; } + const catalogEntry = await zimCatalogInfo(entry.name, this.#catalogUrl); + if (!catalogEntry) { results.push({name: entry.name, status: 'skipped', reason: 'missing from catalog'}); continue; } try { - results.push(await this.#update(meta.name, catalogEntry, {file, meta}, force)); + results.push(await this.#update(entry.name, catalogEntry, entry, force)); } catch (e) { - results.push({name: meta.name, status: 'error', reason: e.message}); + results.push({name: entry.name, status: 'error', reason: e.message}); } } return results; } - - /** Opens a fresh `ZimReader` for a local ZIM, resolved by file path or catalog `name`. Caller must `.close()` it. */ - async open(fileOrName) { - await this.#ensureDir(); - const file = await this.#resolveFile(fileOrName); - let entry = this.#readers.get(file); - if (!entry) { - let pending = this.#opening.get(file); - if (!pending) { - pending = new ZimReader(file, {clusterTTL: this.#clusterTTL}).open(); - this.#opening.set(file, pending); - } - let reader; - try { - reader = await pending; - } finally { - this.#opening.delete(file); - } - entry = this.#readers.get(file) ?? {reader}; - this.#readers.set(file, entry); - } - clearTimeout(entry.timer); - entry.timer = setTimeout(() => { - this.#readers.delete(file); - entry.reader.close(); - }, this.#readerTTL).unref(); - return entry.reader; - } - - /** Fuzzy-searches titles across every local ZIM in the library, merging & re-ranking hits by score. */ - async search(terms, {limit = 20, htmlOnly = true} = {}) { - const local = await this.list(); - const perZim = await Promise.all(local.map(async ({file, meta}) => { - let reader; - try { - reader = await new ZimReader(path.join(this.#dir, file)).open(); - const hits = await reader.search(terms, {limit, htmlOnly}); - return hits.map(h => ({...h, file})); - } catch { - return []; - } finally { - await reader?.close(); - } - })); - return perZim.flat().filter(a => a.score > 0).toSorted((a, b) => b.score - a.score).slice(0, limit); - } } diff --git a/src/reader.js b/src/reader.js index 65c24fd..81ecd4f 100644 --- a/src/reader.js +++ b/src/reader.js @@ -1,474 +1,138 @@ 'use strict'; import fs from 'node:fs'; -import {createHash} from 'node:crypto'; import {decompressPool} from './decompress.js'; -import {fuzzyMatch, titleFromUrl, kiwixSearch} from './utils.js'; const HEADER_SIZE = 80; const NS_CONTENT = 'C'; const NS_METADATA = 'M'; -const TITLE_SENTINEL = 0xffffffffffffffffn; // Indicator -> ZIM v6+ archives with no title -const DEFAULT_CLUSTER_CACHE_MAX = 32; -const DEFAULT_CLUSTER_TTL = 60_000; +async function readAt(fd, pos, length) { + const buf = Buffer.alloc(length); + await fd.read(buf, 0, length, pos); + return buf; +} -// --- Fallback search index (only used for archives with no native title listing) --- -const INDEX_SUFFIX = '.searchidx.bin'; -const INDEX_MAGIC = 'ZXI1'; -const INDEX_HEADER_SIZE = 24; // magic(4) + staleness key(16) + recordCount(4) -const INDEX_RECORD_SIZE = 20; // keyOff(4) keyLen(2) urlOff(4) urlLen(2) titleOff(4) titleLen(2) mimetype(2) +async function ptr64(fd, base, index) { + return Number((await readAt(fd, base + index * 8, 8)).readBigUInt64LE(0)); +} -/** Native, dependency-light reader for .zim archives. Supports zstd & LZMA cluster compression. */ -export class ZimReader { - #fd = null; - #header = null; - #mimeTypes = []; - #hasTitleListing = false; - #clusterCache = new Map(); // clusterNumber -> {data, extended, timer} - #pending = new Map(); // clusterNumber -> Promise, dedupes concurrent misses - #clusterCacheMax; - #clusterTTL; +async function readHeader(fd) { + const b = await readAt(fd, 0, HEADER_SIZE); + return { + articleCount: b.readUInt32LE(24), + clusterCount: b.readUInt32LE(28), + urlPtrPos: Number(b.readBigUInt64LE(32)), + clusterPtrPos: Number(b.readBigUInt64LE(48)), + mimeListPos: Number(b.readBigUInt64LE(56)), + }; +} - #indexPath; - #indexReady = null; // Promise, awaited by search() before using the fallback index - #indexFd = null; // open fd for the fallback index, once loaded/built - #indexRecordCount = 0; - #indexTableStart = 0; - - get articleCount() { return this.#header?.articleCount ?? 0; } - get mediaCount() { return this.#header?.clusterCount ?? 0; } - - /** @param {{clusterCacheMax?: number, clusterTTL?: number}} [opts] clusterTTL in ms; 0/null disables idle eviction. */ - constructor(path, {clusterCacheMax = DEFAULT_CLUSTER_CACHE_MAX, clusterTTL = DEFAULT_CLUSTER_TTL} = {}) { - this.path = path; - this.#clusterCacheMax = clusterCacheMax; - this.#clusterTTL = clusterTTL; - this.#indexPath = `${path}${INDEX_SUFFIX}`; +async function readMimeTypes(fd, mimeListPos) { + let pos = mimeListPos, str = ''; + for (;;) { + str += (await readAt(fd, pos, 1024)).toString('binary'); + const end = str.indexOf('\0\0'); + if (end !== -1) { str = str.slice(0, end + 1); break; } + pos += 1024; } + return str.split('\0').filter(Boolean); +} - /** Binary search the URL pointer list for namespace+url. For exact-key lookups (readPage, metadata, icons). */ - async #findByUrl(url, namespace) { - const key = namespace + url; - let lo = 0, hi = this.#header.articleCount - 1; - while (lo <= hi) { - const mid = (lo + hi) >> 1; - const dirent = await this.#readDirent(await this.#ptr64(this.#header.urlPtrPos, mid)); - const dirKey = dirent.namespace + dirent.url; - const cmp = key < dirKey ? -1 : key > dirKey ? 1 : 0; - if (cmp === 0) return dirent; - if (cmp < 0) hi = mid - 1; else lo = mid + 1; - } - return null; - } +/** Directory entry (article record) at byte `offset`, growing the read window as needed. */ +async function readDirent(fd, offset) { + for (let size = 512; ; size *= 2) { + const buf = await readAt(fd, offset, size); + let o = 0; + const mimetype = buf.readUInt16LE(o); o += 2; + o += 1; // extraLen, unused + const namespace = String.fromCharCode(buf.readUInt8(o)); o += 1; + o += 4; // revision, unused - /** Binary search the ZIM's own title pointer list. O(log n), zero extra storage - the happy path. */ - async #findByTitleBuiltin(title) { - let lo = 0, hi = this.#header.articleCount - 1; - while (lo <= hi) { - const mid = (lo + hi) >> 1; - const urlIdx = (await this.#read(this.#header.titlePtrPos + mid * 4, 4)).readUInt32LE(0); - const d = await this.#readDirent(await this.#ptr64(this.#header.urlPtrPos, urlIdx)); - if (d.title === title) return d; - if (d.title < title) lo = mid + 1; else hi = mid - 1; - } - return null; - } + let redirectIndex = null, cluster = null, blob = null; + if (mimetype === 0xffff) { redirectIndex = buf.readUInt32LE(o); o += 4; } + else { cluster = buf.readUInt32LE(o); o += 4; blob = buf.readUInt32LE(o); o += 4; } - /** - * Resolves a kiwix-search result (a title, not necessarily a url) to - * {url, title, mimetype}. kiwix-search's fulltext index returns titles, and - * a title isn't guaranteed to equal its url (unicode normalization, - * disambiguation suffixes, punctuation stripping), so: - * 1. URL binary search - matches when title happens to equal url (free to check) - * 2. Title pointer list - when the archive ships one (Wikipedia etc. do) - * 3. Persisted fallback index / linear scan - only for archives without (2) - */ - async #resolveSearchEntry(name) { - let dirent = await this.#findByUrl(name, NS_CONTENT); - if (dirent) return dirent; + const urlEnd = buf.indexOf(0, o); + if (urlEnd === -1) continue; + const titleEnd = buf.indexOf(0, urlEnd + 1); + if (titleEnd === -1) continue; - if (this.#hasTitleListing) return this.#findByTitleBuiltin(name); - - if (this.#indexReady) await this.#indexReady; - const hit = await this.#lookupFallbackIndex(name); - if (hit) return hit; - - // Index unavailable (build failed - unwritable disk, etc.) or genuinely no match. - for (let i = 0; i < this.#header.articleCount; i++) { - const d = await this.#readDirent(await this.#ptr64(this.#header.urlPtrPos, i)); - if (d.namespace === NS_CONTENT && (d.title === name || titleFromUrl(d.url) === name)) return d; - } - return null; - } - - /** - * A cheap, stable fingerprint for "is the cached index still valid for this file". - * ZIM archives end with a 16-byte MD5 of their own contents (the same trailer - * `zimcheck` validates against), so this is a single 16-byte read regardless of - * archive size - no need to hash a multi-hundred-GB file. It's also - * content-based rather than path/mtime-based, so moving or redownloading an - * identical archive doesn't invalidate the cache. Falls back to a tiny - * size+mtime hash only if the file is too short to have a real trailer. - */ - async #stalenessKey() { - const {size} = await fs.promises.stat(this.path); - if (size >= 16) return this.#read(size - 16, 16); - const stat = await fs.promises.stat(this.path); - return createHash('md5').update(`${stat.size}:${stat.mtimeMs}`).digest(); - } - - /** Walks every content dirent once, resolving redirects, keyed by both its title and its url-derived title. */ - async #collectFallbackEntries() { - const map = new Map(); // key -> {url, title, mimetype} - for (let i = 0; i < this.#header.articleCount; i++) { - let dirent = await this.#readDirent(await this.#ptr64(this.#header.urlPtrPos, i)); - if (dirent.namespace !== NS_CONTENT) continue; - dirent = await this.#resolveRedirect(dirent); - if (dirent.namespace !== NS_CONTENT) continue; // redirected outside content namespace - - const entry = {url: dirent.url, title: dirent.title, mimetype: dirent.mimetype}; - if (!map.has(dirent.title)) map.set(dirent.title, entry); - const urlTitle = titleFromUrl(dirent.url); - if (urlTitle !== dirent.title && !map.has(urlTitle)) map.set(urlTitle, entry); - } - return map; - } - - /** Serializes the fallback index to disk: header, sorted fixed-width records, then a string table. */ - async #buildFallbackIndex(stalenessKey) { - const map = await this.#collectFallbackEntries(); - const keys = [...map.keys()].sort(); - - const records = Buffer.alloc(keys.length * INDEX_RECORD_SIZE); - const strings = []; - let tableOffset = 0; - keys.forEach((key, i) => { - const {url, title, mimetype} = map.get(key); - const keyBuf = Buffer.from(key, 'utf8'); - const urlBuf = Buffer.from(url, 'utf8'); - const titleBuf = Buffer.from(title, 'utf8'); - const base = i * INDEX_RECORD_SIZE; - - records.writeUInt32LE(tableOffset, base); records.writeUInt16LE(keyBuf.length, base + 4); - tableOffset += keyBuf.length; - records.writeUInt32LE(tableOffset, base + 6); records.writeUInt16LE(urlBuf.length, base + 10); - tableOffset += urlBuf.length; - records.writeUInt32LE(tableOffset, base + 12); records.writeUInt16LE(titleBuf.length, base + 16); - tableOffset += titleBuf.length; - records.writeUInt16LE(mimetype, base + 18); - - strings.push(keyBuf, urlBuf, titleBuf); - }); - - const header = Buffer.alloc(INDEX_HEADER_SIZE); - header.write(INDEX_MAGIC, 0, 'ascii'); - stalenessKey.copy(header, 4); - header.writeUInt32LE(keys.length, 20); - - const tmpPath = `${this.#indexPath}.tmp-${process.pid}`; - await fs.promises.writeFile(tmpPath, Buffer.concat([header, records, ...strings])); - await fs.promises.rename(tmpPath, this.#indexPath); - } - - /** Opens the fallback index file and caches its header fields for querying. */ - async #openFallbackIndex() { - const fd = await fs.promises.open(this.#indexPath, 'r'); - const header = Buffer.alloc(INDEX_HEADER_SIZE); - await fd.read(header, 0, INDEX_HEADER_SIZE, 0); - this.#indexFd = fd; - this.#indexRecordCount = header.readUInt32LE(20); - this.#indexTableStart = INDEX_HEADER_SIZE + this.#indexRecordCount * INDEX_RECORD_SIZE; - } - - /** - * Validates the on-disk fallback index against the archive's current staleness - * key and (re)builds it if missing/stale. Failures are swallowed - callers see - * #indexFd stay null and fall through to the linear-scan last resort. - */ - async #loadOrBuildFallbackIndex() { - const stalenessKey = await this.#stalenessKey(); - - try { - const fd = await fs.promises.open(this.#indexPath, 'r'); - const header = Buffer.alloc(INDEX_HEADER_SIZE); - await fd.read(header, 0, INDEX_HEADER_SIZE, 0); - const fresh = header.toString('ascii', 0, 4) === INDEX_MAGIC && header.subarray(4, 20).equals(stalenessKey); - if (fresh) { - this.#indexFd = fd; - this.#indexRecordCount = header.readUInt32LE(20); - this.#indexTableStart = INDEX_HEADER_SIZE + this.#indexRecordCount * INDEX_RECORD_SIZE; - return; - } - await fd.close(); - } catch { /* missing, corrupt, or unreadable -> rebuild below */ } - - await this.#buildFallbackIndex(stalenessKey); - await this.#openFallbackIndex(); - } - - async #indexRead(pos, length) { - const buf = Buffer.alloc(length); - await this.#indexFd.read(buf, 0, length, pos); - return buf; - } - - async #indexString(offset, length) { - if (!length) return ''; - return (await this.#indexRead(this.#indexTableStart + offset, length)).toString('utf8'); - } - - /** Binary search over the on-disk record array. Never loads the full index into memory. */ - async #lookupFallbackIndex(name) { - if (!this.#indexFd) return null; - let lo = 0, hi = this.#indexRecordCount - 1; - while (lo <= hi) { - const mid = (lo + hi) >> 1; - const rec = await this.#indexRead(INDEX_HEADER_SIZE + mid * INDEX_RECORD_SIZE, INDEX_RECORD_SIZE); - const keyOff = rec.readUInt32LE(0), keyLen = rec.readUInt16LE(4); - const key = await this.#indexString(keyOff, keyLen); - if (key === name) { - const urlOff = rec.readUInt32LE(6), urlLen = rec.readUInt16LE(10); - const titleOff = rec.readUInt32LE(12), titleLen = rec.readUInt16LE(16); - const [url, title] = await Promise.all([this.#indexString(urlOff, urlLen), this.#indexString(titleOff, titleLen)]); - return {url, title, mimetype: rec.readUInt16LE(18)}; - } - if (key < name) lo = mid + 1; else hi = mid - 1; - } - return null; - } - - /** Resets a cluster's idle-eviction timer. No-op when TTL disabled. */ - #touch(clusterNumber, entry) { - if (!this.#clusterTTL) return; - clearTimeout(entry.timer); - entry.timer = setTimeout(() => this.#clusterCache.delete(clusterNumber), this.#clusterTTL).unref(); - } - - /** Fetches + decompresses a cluster exactly once, offloading decompression to the worker pool. */ - async #loadCluster(clusterNumber) { - const start = await this.#ptr64(this.#header.clusterPtrPos, clusterNumber); - const isLast = clusterNumber === this.#header.clusterCount - 1; - const end = isLast - ? (await fs.promises.stat(this.path)).size - : await this.#ptr64(this.#header.clusterPtrPos, clusterNumber + 1); - - const raw = await this.#read(start, end - start); - const compType = raw[0] & 0x0f; - const extended = (raw[0] & 0x10) !== 0; - const body = raw.subarray(1); - - let data; - if (compType <= 1) data = Buffer.from(body); - else if (compType === 4 || compType === 5) data = await decompressPool.run({compType, body}); - else throw new Error(`Unsupported cluster compression type: ${compType}`); - if (!data) throw new Error(`Cluster ${clusterNumber} failed to decompress (compType ${compType})`); - - return {data, extended}; - } - - async #getBlob(clusterNumber, blobNumber) { - let entry = this.#clusterCache.get(clusterNumber); - if (!entry) { - let pending = this.#pending.get(clusterNumber); - if (!pending) { - pending = this.#loadCluster(clusterNumber); - this.#pending.set(clusterNumber, pending); - } - entry = await pending; - this.#pending.delete(clusterNumber); - this.#clusterCache.set(clusterNumber, entry); - if (this.#clusterCache.size > this.#clusterCacheMax) { - const oldestKey = this.#clusterCache.keys().next().value; - clearTimeout(this.#clusterCache.get(oldestKey)?.timer); - this.#clusterCache.delete(oldestKey); - } - } - this.#touch(clusterNumber, entry); - - const readPtr = i => entry.extended ? Number(entry.data.readBigUInt64LE(i * 8)) : entry.data.readUInt32LE(i * 4); - return entry.data.subarray(readPtr(blobNumber), readPtr(blobNumber + 1)); - } - - async #icon(size = 48) { - const page = await this.readPage(`Illustration_${size}x${size}@1`, NS_METADATA) - || await this.readPage('Favicon', NS_METADATA); - if (!page) return null; - return `data:${page.mimetype};base64,${page.data.toString('base64')}`; - } - - async #ptr64(base, index) { - return Number((await this.#read(base + index * 8, 8)).readBigUInt64LE(0)); - } - - async #read(pos, length) { - const buf = Buffer.alloc(length); - await this.#fd.read(buf, 0, length, pos); - return buf; - } - - async #readHeader() { - const b = await this.#read(0, HEADER_SIZE); - const titlePtrRaw = b.readBigUInt64LE(40); - this.#hasTitleListing = titlePtrRaw !== TITLE_SENTINEL; - this.#header = { - articleCount: b.readUInt32LE(24), - clusterCount: b.readUInt32LE(28), - urlPtrPos: Number(b.readBigUInt64LE(32)), - titlePtrPos: this.#hasTitleListing ? Number(titlePtrRaw) : null, - clusterPtrPos: Number(b.readBigUInt64LE(48)), - mimeListPos: Number(b.readBigUInt64LE(56)), - mainPage: b.readUInt32LE(64), - }; - } - - async #readMimeTypes() { - let pos = this.#header.mimeListPos, str = ''; - for (;;) { - str += (await this.#read(pos, 1024)).toString('binary'); - const end = str.indexOf('\0\0'); - if (end !== -1) { str = str.slice(0, end + 1); break; } - pos += 1024; - } - this.#mimeTypes = str.split('\0').filter(Boolean); - } - - /** Directory entry (article record) at byte `offset`, growing the read window as needed. */ - async #readDirent(offset) { - for (let size = 512; ; size *= 2) { - const buf = await this.#read(offset, size); - let o = 0; - const mimetype = buf.readUInt16LE(o); o += 2; - o += 1; // extraLen, unused - const namespace = String.fromCharCode(buf.readUInt8(o)); o += 1; - o += 4; // revision, unused - - let redirectIndex = null, cluster = null, blob = null; - if (mimetype === 0xffff) { redirectIndex = buf.readUInt32LE(o); o += 4; } - else { cluster = buf.readUInt32LE(o); o += 4; blob = buf.readUInt32LE(o); o += 4; } - - const urlEnd = buf.indexOf(0, o); - if (urlEnd === -1) continue; - const titleEnd = buf.indexOf(0, urlEnd + 1); - if (titleEnd === -1) continue; - - const url = buf.toString('utf8', o, urlEnd); - const title = buf.toString('utf8', urlEnd + 1, titleEnd) || url; - return {mimetype, namespace, redirectIndex, cluster, blob, url, title}; - } - } - - async #resolveRedirect(dirent) { - if (dirent.mimetype !== 0xffff) return dirent; - return this.#readDirent(await this.#ptr64(this.#header.urlPtrPos, dirent.redirectIndex)); - } - - async close() { - if (this.#fd) await this.#fd.close(); - this.#fd = null; - if (this.#indexFd) await this.#indexFd.close(); - this.#indexFd = null; - for (const entry of this.#clusterCache.values()) clearTimeout(entry.timer); - this.#clusterCache.clear(); - this.#pending.clear(); - } - - /** Deletes the zim archive and its cached fallback index (if any). Safe to call on unopened readers. */ - async delete() { - await this.close(); - await fs.promises.rm(this.path, {force: true}); - await fs.promises.rm(this.#indexPath, {force: true}); - } - - /** Reads an 'M' namespace metadata value (e.g. Name, Date, Title). Returns null if missing. */ - async metadata(key) { - const get = async key => { - const page = await this.readPage(key, NS_METADATA); - return page ? page.data.toString('utf8') : null; - }; - if(key) return get(key); - - const [title, creator, publisher, date, description, language, name, tags] = await Promise.all( - ['Title', 'Creator', 'Publisher', 'Date', 'Description', 'Language', 'Name', 'Tags'].map(get) - ); - return { - title, - updated: date ? new Date(date) : null, - summary: description, - language, - name, - category: tags ? tags.split(';')[0] || '' : '', - tags: tags ? tags.split(';') : [], - author: creator, - publisher, - articleCount: this.articleCount, - mediaCount: this.mediaCount, - sizeMb: +((await fs.promises.stat(this.path)).size / 1024 / 1024).toFixed(1), - icon: await this.#icon(), - }; - } - - /** Opens the archive and parses its header + mimetype list. */ - async open() { - this.#fd = await fs.promises.open(this.path, 'r'); - try { - await this.#readHeader(); - await this.#readMimeTypes(); - } catch (e) { - await this.#fd.close(); - this.#fd = null; - throw e; - } - - // The fallback index is only needed when the archive has no native title - // pointer list - #findByTitleBuiltin already covers that case in O(log n) - // with zero extra storage. Well-maintained archives (Wikipedia etc.) - // ship a title listing, so this path is expected to be rare in practice. - if (!this.#hasTitleListing) { - this.#indexReady = this.#loadOrBuildFallbackIndex().catch(() => {}); - } - return this; - } - - /** Read a page's content by URL. Returns `{mimetype, data}` or `null` if not found. */ - async readPage(url, namespace = NS_CONTENT) { - let dirent = await this.#findByUrl(url, namespace); - if (!dirent) return null; - dirent = await this.#resolveRedirect(dirent); - const data = await this.#getBlob(dirent.cluster, dirent.blob); - return {mimetype: this.#mimeTypes[dirent.mimetype] || 'application/octet-stream', data}; - } - - /** Reads the archive's designated main/landing page, if one is set. */ - async mainPage() { - if (this.#header.mainPage === 0xffffffff) return null; - let dirent = await this.#readDirent(await this.#ptr64(this.#header.urlPtrPos, this.#header.mainPage)); - dirent = await this.#resolveRedirect(dirent); - const data = await this.#getBlob(dirent.cluster, dirent.blob); - return {mimetype: this.#mimeTypes[dirent.mimetype] || 'application/octet-stream', data, url: dirent.url, namespace: dirent.namespace}; - } - - /** - * Fuzzy-ranked title search. Accepts comma-separated `terms` the same way the - * catalog search does. Uses kiwix-search's embedded fulltext index as a prefilter - * to narrow candidates before fuzzy scoring. - */ - async search(terms, {limit = 20, htmlOnly = true} = {}) { - const termList = String(terms).split(',').map(t => t.trim()).filter(Boolean); - if (!termList.length) return []; - - const names = await kiwixSearch(this.path, termList.join(' ')); - const candidates = (await Promise.all(names.map(n => this.#resolveSearchEntry(n)))).filter(Boolean); - - const scored = []; - for (const dirent of candidates) { - if (htmlOnly && !(this.#mimeTypes[dirent.mimetype] || '').startsWith('text/html')) continue; - const urlTitle = titleFromUrl(dirent.url); - const titleScore = fuzzyMatch(dirent.title, ...termList).max; - const urlScore = fuzzyMatch(urlTitle, ...termList).max; - scored.push({url: dirent.url, title: dirent.title.length > urlTitle.length ? dirent.title : urlTitle, namespace: NS_CONTENT, score: Math.max(titleScore, urlScore)}); - } - const {summary, mediaCount, articleCount, sizeMb, ...meta} = await this.metadata(); - return scored.filter(a => a.score > 0).toSorted((a, b) => b.score - a.score).slice(0, limit).map(a => ({...meta, ...a})); + const url = buf.toString('utf8', o, urlEnd); + const title = buf.toString('utf8', urlEnd + 1, titleEnd) || url; + return {mimetype, namespace, redirectIndex, cluster, blob, url, title}; } } + +async function findByUrl(fd, header, url, namespace) { + const key = namespace + url; + let lo = 0, hi = header.articleCount - 1; + while (lo <= hi) { + const mid = (lo + hi) >> 1; + const dirent = await readDirent(fd, await ptr64(fd, header.urlPtrPos, mid)); + const dirKey = dirent.namespace + dirent.url; + const cmp = key < dirKey ? -1 : key > dirKey ? 1 : 0; + if (cmp === 0) return dirent; + if (cmp < 0) hi = mid - 1; else lo = mid + 1; + } + return null; +} + +async function resolveRedirect(fd, header, dirent) { + if (dirent.mimetype !== 0xffff) return dirent; + return readDirent(fd, await ptr64(fd, header.urlPtrPos, dirent.redirectIndex)); +} + +async function getBlob(fd, header, filePath, clusterNumber, blobNumber) { + const start = await ptr64(fd, header.clusterPtrPos, clusterNumber); + const isLast = clusterNumber === header.clusterCount - 1; + const end = isLast + ? (await fs.promises.stat(filePath)).size + : await ptr64(fd, header.clusterPtrPos, clusterNumber + 1); + + const raw = await readAt(fd, start, end - start); + const compType = raw[0] & 0x0f; + const extended = (raw[0] & 0x10) !== 0; + const body = raw.subarray(1); + + let data; + if (compType <= 1) data = Buffer.from(body); + else if (compType === 4 || compType === 5) data = await decompressPool.run({compType, body}); + else throw new Error(`Unsupported cluster compression type: ${compType}`); + + const readPtr = i => extended ? Number(data.readBigUInt64LE(i * 8)) : data.readUInt32LE(i * 4); + return data.subarray(readPtr(blobNumber), readPtr(blobNumber + 1)); +} + +/** + * One-shot read of a single entry from a .zim archive - no server required. + * Opt-in convenience for callers who don't want to run kiwix-serve for a single + * lookup. Re-parses the header/mimetype list on every call; fine for occasional + * reads, not meant for high-volume access (use KiwixServer for that). + */ +export async function readZimEntry(zimPath, url, namespace = NS_CONTENT) { + const fd = await fs.promises.open(zimPath, 'r'); + try { + const header = await readHeader(fd); + const mimeTypes = await readMimeTypes(fd, header.mimeListPos); + let dirent = await findByUrl(fd, header, url, namespace); + if (!dirent) return null; + dirent = await resolveRedirect(fd, header, dirent); + const data = await getBlob(fd, header, zimPath, dirent.cluster, dirent.blob); + return {mimetype: mimeTypes[dirent.mimetype] || 'application/octet-stream', data}; + } finally { + await fd.close(); + } +} + +/** Reads 'M' namespace metadata (Title, Creator, Date, etc.) without a server. Minimal by design - no icon/size/counts. */ +export async function readZimMetadata(zimPath) { + const keys = ['Title', 'Creator', 'Publisher', 'Date', 'Description', 'Language', 'Name', 'Tags']; + const entries = await Promise.all(keys.map(k => readZimEntry(zimPath, k, NS_METADATA))); + const [title, creator, publisher, date, description, language, name, tags] = entries.map(e => e?.data.toString('utf8') ?? null); + return { + title, updated: date ? new Date(date) : null, summary: description, language, name, + category: tags ? tags.split(';')[0] || '' : '', tags: tags ? tags.split(';') : [], + author: creator, publisher, + }; +} diff --git a/src/server.js b/src/server.js new file mode 100644 index 0000000..f0d3005 --- /dev/null +++ b/src/server.js @@ -0,0 +1,219 @@ +'use strict'; + +import {spawn, execFile} from 'node:child_process'; +import {promisify} from 'node:util'; +import net from 'node:net'; +import fs from 'node:fs'; +import path from 'node:path'; +import {fileURLToPath} from 'node:url'; +import {decodeHtml, fromXml} from '@ztimson/utils'; +import {fuzzyMatch, weightedScore} from './utils.js'; + +const execFileAsync = promisify(execFile); +const __dirname = path.dirname(fileURLToPath(import.meta.url)); +const DEFAULT_BIN_DIR = path.join(__dirname, '..', 'bin'); // npm package root/bin - where bin/install.js drops the kiwix-tools binaries +const READY_TIMEOUT = 10_000; +const READY_POLL_INTERVAL = 100; + +function findFreePort() { + return new Promise((resolve, reject) => { + const srv = net.createServer(); + srv.on('error', reject); + srv.listen(0, () => { + const {port} = srv.address(); + srv.close(() => resolve(port)); + }); + }); +} + +/** Owns a kiwix-serve process's full lifecycle: library.xml, start/stop/reload, content + search access. */ +export class KiwixServer { + #dir; + #host; + #port; + #binDir; + #libraryPath; + #child = null; + + get port() { return this.#port; } + get running() { return !!this.#child; } + get baseUrl() { return this.#child ? `http://${this.#host}:${this.#port}` : null; } + + /** @param {{port?: number, host?: string, binDir?: string}} [opts] port defaults to an auto-picked free port; host defaults to localhost-only; binDir defaults to the bundled ./bin next to this package. */ + constructor(dir, {port, host = '127.0.0.1', binDir = DEFAULT_BIN_DIR} = {}) { + this.#dir = dir; + this.#host = host; + this.#port = port; + this.#binDir = binDir; + this.#libraryPath = path.join(dir, 'library.xml'); + } + + #assertRunning() { + if (!this.#child) throw new Error('KiwixServer is not running - call start() first'); + } + + #bin(name) { + return path.join(this.#binDir, process.platform === 'win32' ? `${name}.exe` : name); + } + + /** Rebuilds library.xml from scratch by scanning `dir` for .zim files - a full rebuild is simpler and less bug-prone than tracking incremental add/remove. */ + async #rebuildLibrary() { + await fs.promises.rm(this.#libraryPath, {force: true}); + for (const f of await this.#zimFiles()) await execFileAsync(this.#bin('kiwix-manage'), [this.#libraryPath, 'add', path.join(this.#dir, f)]); + } + + async #waitUntilReady() { + const deadline = Date.now() + READY_TIMEOUT; + while (Date.now() < deadline) { + try { + await fetch(`http://${this.#host}:${this.#port}/`); + return; + } catch {} + await new Promise(r => setTimeout(r, READY_POLL_INTERVAL)); + } + throw new Error('kiwix-serve did not become ready in time'); + } + + async #zimFiles() { + return (await fs.promises.readdir(this.#dir).catch(() => [])).filter(f => f.endsWith('.zim')); + } + + /** URL for a single asset, e.g. for pulling one html/image file when you already know the path (from search). */ + contentUrl(book, pathInZim = '') { + this.#assertRunning(); + return `${this.baseUrl}/content/${book}/${pathInZim}`; + } + + /** Rebuilds library.xml and starts kiwix-serve. Resolves once the server is responding. */ + async start() { + if (this.#child) return; + await fs.promises.mkdir(this.#dir, {recursive: true}); + await this.#rebuildLibrary(); + this.#port ??= await findFreePort(); + + this.#child = spawn(this.#bin('kiwix-serve'), ['--library', '-i', this.#host, '-p', String(this.#port), this.#libraryPath], {stdio: 'ignore'}); + this.#child.on('exit', () => { this.#child = null; }); + + try { + await this.#waitUntilReady(); + } catch (e) { + await this.stop(); + throw e; + } + } + + /** Gracefully stops kiwix-serve, if running. */ + async stop() { + if (!this.#child) return; + const child = this.#child; + await new Promise(resolve => { + child.once('exit', resolve); + child.kill('SIGTERM'); + }); + this.#child = null; + } + + async restart() { + await this.stop(); + await this.start(); + } + + /** Rebuilds library.xml from disk and hot-reloads kiwix-serve via SIGHUP - no downtime/restart needed. */ + async reload() { + if(!this.#child) return; + await this.restart(); + } + + /** Local catalog listing - same flat shape as the online catalog (catalog.js), plus a `file` field since these already live on disk. */ + async list() { + this.#assertRunning(); + const xml = await fs.promises.readFile(this.#libraryPath, 'utf8').catch(() => ''); + const entries = fromXml(xml); + return (entries?.library?.book || []).map(e => { + const tags = e.tags.split(';'); + const name = e.path.replaceAll('.zim', ''); + return { + id: e.id, + title: e.title, + updated: new Date(e.date), + summary: e.description, + language: e.language, + name: e.name, + category: tags.find(t => t.startsWith('_category'))?.slice(10) || '', + tags, + mediaCount: +e.mediaCount || 0, + author: e.creator, + publisher: e.publisher, + articleCount: +e.articleCount || 0, + sizeMb: +(Number(e.size) / 1024).toFixed(1) || 0, + href: name, + icon: e.favicon, + viewer: `${this.baseUrl}/content/${name}`, + }; + }); + } + + /** Splits a href ("zim/path/to/page") or a full content/viewer URL into {zim, path}. */ + #splitHref(href) { + const clean = href.replace(`${this.baseUrl}/content/`, '').replace(/^\/+/, ''); + const [zim, ...rest] = clean.split('/'); + return {zim, path: rest.join('/')}; + } + + /** Builds a kiwix-serve content URL from a href (as returned by list()/search()), or from an already-built content/viewer URL. */ + link(href) { + this.#assertRunning(); + const {zim, path} = this.#splitHref(href); + return `${this.baseUrl}/content/${zim}${path ? '/' + path : ''}`; + } + + /** Fetches a single asset's raw bytes straight from kiwix-serve. */ + async raw(href) { + const res = await fetch(this.fetch(href)); + if (!res.ok) return null; + return {mimetype: res.headers.get('content-type'), data: Buffer.from(await res.arrayBuffer())}; + } + + /** + * Two-pass fulltext search across every local ZIM: xapian prefilter (kiwix-serve's + * own index), then fuzzy-reranked by title so the strongest matches surface first. + * Returns a flat array matching the catalog/list shape: {id, title, name, category, ..., href, score} + */ + async search(terms, limit = 20) { + this.#assertRunning(); + const termList = String(terms).split(',').map(t => t.trim()).filter(Boolean); + if (!termList.length) return []; + + const params = new URLSearchParams({pattern: termList.join(' '), format: 'xml', pageLength: String(limit)}); + const res = await fetch(`${this.baseUrl}/search?${params}`); + if (!res.ok) return []; + + const xml = await res.text(); + let found = fromXml(xml); + found = found?.rss?.channel?.item || []; + + const books = await this.list(); + const bookMap = new Map(books.map(b => [b.title, b])); + + const enriched = found.map(hit => { + const book = bookMap.get(hit.book.title); + if(!book) return null; + const prefix = `/content/${book.href}/`; + const page = hit.link.startsWith(prefix) ? hit.link.slice(prefix.length) : hit.link.replace(/^\/+/, ''); + return { + id: book.id, + title: hit.title, + page, + name: book.name, + publisher: book.publisher, + href: `${book.href}/${page}`, + icon: book.icon, + viewer: this.baseUrl + hit.link, + summary: hit.description, + score: weightedScore(hit.title, termList) + weightedScore(hit.description, termList), + }; + }).filter(hit => !!hit && hit.score > 0); + + return enriched.toSorted((a, b) => b.score - a.score).slice(0, limit); + } +} diff --git a/src/utils.js b/src/utils.js index 3fe59da..d58ca33 100644 --- a/src/utils.js +++ b/src/utils.js @@ -1,32 +1,3 @@ -import {execFile} from 'node:child_process'; -import path from 'node:path'; -import {fileURLToPath} from 'node:url'; -import {promisify} from 'node:util'; - -/** - * Search a ZIM file's fulltext index. - * @param {string} zimPath - Path to the .zim file - * @param {string} pattern - Search terms - * @param {object} [opts] - * @param {boolean} [opts.suggestion] - Suggest titles from partial pattern (completion-style) - * @param {boolean} [opts.spelling] - Suggest spelling-corrected titles - * @returns {Promise} Matching article/tag titles - */ -export async function kiwixSearch(zimPath, pattern, opts = {}) { - const execFileAsync = promisify(execFile); - const __dirname = path.dirname(fileURLToPath(import.meta.url)); - const BIN_NAME = process.platform === 'win32' ? 'kiwix-search.exe' : 'kiwix-search'; - const BIN_PATH = path.join(__dirname, '..', 'bin', BIN_NAME); - - const args = []; - if (opts.suggestion) args.push('-s'); - if (opts.spelling) args.push('--spelling'); - args.push(zimPath, pattern); - - const {stdout} = await execFileAsync(BIN_PATH, args); - return stdout.split('\n').map(line => line.trim()).filter(Boolean); -} - export function levenshtein(a, b) { const m = a.length, n = b.length; if (!m) return n; @@ -43,12 +14,6 @@ export function levenshtein(a, b) { return dp[m][n]; } -/** Normalized similarity in [0,1]: 1 - editDistance / maxLength. */ -export function similarity(a, b) { - a = a.toLowerCase(); b = b.toLowerCase(); - return 1 - levenshtein(a, b) / Math.max(a.length, b.length, 1); -} - function scoreAgainst(text, term) { if (text.includes(term)) return 1 - (text.length - term.length) / text.length * 0.3; if (!text.length || !term.length || text[0] !== term[0]) return 0; @@ -76,9 +41,9 @@ export function fuzzyMatch(target, ...terms) { }; } -/** Derives a readable pseudo-title from a URL's last path segment, e.g. ".../diannes-southwest-salad/" -> "Diannes Southwest Salad". */ -export function titleFromUrl(url) { - const slug = String(url).replace(/\/$/, '').split('/').pop() || url; - const clean = slug.replace(/\.(zim|meta4|html?|md)$/i, '').replace(/[-_.]+/g, ' '); - return clean.replace(/\b\w/g, c => c.toUpperCase()); +export function weightedScore(text, termList) { + if (!text) return 0; + const covered = termList.reduce((sum, t) => sum + t.length, 0); + const coverage = Math.min(1, covered / text.length); + return fuzzyMatch(text, ...termList).max * coverage; }