Create title index for zims missing them

.gitignore

@@ -0,0 +1,10 @@
+.idea
+.vscode
+
+logs
+*.log
+
+bin/*.dll
+bin/kiwix*
+node_modles
+zims

bin/install-kwix.js

@@ -0,0 +1,87 @@
+import fs from 'node:fs';
+import os from 'node:os';
+import path from 'node:path';
+import https from 'node:https';
+import {execFileSync} from 'node:child_process';
+import {fileURLToPath} from 'node:url';
+
+const __dirname = path.dirname(fileURLToPath(import.meta.url));
+
+const VERSION = '3.8.1';
+const BASE_URL = 'https://download.kiwix.org/release/kiwix-tools';
+const PLATFORM_MAP = {linux: 'linux', darwin: 'macos', win32: 'win'};
+
+const BIN_NAME = process.platform === 'win32' ? 'kiwix-search.exe' : 'kiwix-search';
+const BIN_DIR = path.join(__dirname, '..', 'bin'); // project root/bin
+const BIN_PATH = path.join(BIN_DIR, BIN_NAME);
+
+/** Download a file, following redirects. */
+function download(url, dest) {
+	return new Promise((resolve, reject) => {
+		const file = fs.createWriteStream(dest);
+		https.get(url, res => {
+			if (res.statusCode >= 300 && res.statusCode < 400 && res.headers.location) {
+				file.close();
+				return resolve(download(res.headers.location, dest));
+			}
+			if (!res.statusCode || res.statusCode >= 400) {
+				return reject(new Error(`Download failed: ${res.statusCode} ${res.statusMessage}`));
+			}
+			res.pipe(file);
+			file.on('finish', () => file.close(resolve));
+		}).on('error', err => fs.unlink(dest, () => reject(err)));
+	});
+}
+
+/** Recursively search a directory for a file by name. */
+function findFile(dir, name) {
+	for (const entry of fs.readdirSync(dir, {withFileTypes: true})) {
+		const full = path.join(dir, entry.name);
+		if (entry.isDirectory()) {
+			const found = findFile(full, name);
+			if (found) return found;
+		} else if (entry.name === name) {
+			return full;
+		}
+	}
+	return null;
+}
+
+async function downloadAndExtract() {
+	const platformName = PLATFORM_MAP[process.platform];
+	if (!platformName) throw new Error(`Unsupported platform: ${process.platform}`);
+
+	const ext = platformName === 'win' ? 'zip' : 'tar.gz';
+	const archiveName = `kiwix-tools_${platformName}-x86_64-${VERSION}.${ext}`;
+	const url = `${BASE_URL}/${archiveName}`;
+	const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'kiwix-tools-'));
+	const archivePath = path.join(tmpDir, archiveName);
+
+	console.log('Downloading kiwix-tools from:', url);
+	await download(url, archivePath);
+
+	console.log('Extracting...');
+	execFileSync('tar', ['-xf', archivePath, '-C', tmpDir]); // bsdtar handles zip too
+
+	const extractedBin = findFile(tmpDir, BIN_NAME);
+	if (!extractedBin) throw new Error(`Could not find ${BIN_NAME} in extracted archive`);
+	const extractedDir = path.dirname(extractedBin);
+
+	await fs.promises.mkdir(BIN_DIR, {recursive: true});
+
+	// Copy the binary plus any DLLs sitting alongside it (Windows deps)
+	for (const entry of await fs.promises.readdir(extractedDir)) {
+		if (entry === BIN_NAME || entry.toLowerCase().endsWith('.dll')) {
+			await fs.promises.copyFile(path.join(extractedDir, entry), path.join(BIN_DIR, entry));
+		}
+	}
+	if (process.platform !== 'win32') await fs.promises.chmod(BIN_PATH, 0o755);
+
+	await fs.promises.rm(tmpDir, {recursive: true, force: true});
+	console.log('Installed to:', BIN_PATH);
+}
+
+downloadAndExtract().catch(err => {
+	console.error(err);
+	process.exit(1)
+}).finally(() => process.exit());

package.json

@@ -1,6 +1,6 @@
 {
 	"name": "@ztimson/zim-utils",
-	"version": "0.1.1",
+	"version": "0.2.5",
 	"description": "Native, dependency-light ZIM archive reader/searcher and Kiwix catalog downloader for Node.js",
 	"author": "Zak Timson",
 	"license": "MIT",
@@ -11,6 +11,9 @@
 		"url": "https://git.zakscode.com/ztimson/zim-utils"
 	},
 	"main": "src/index.js",
+	"scripts": {
+		"postinstall": "node ./bin/install-kwix.js"
+	},
 	"dependencies": {
 		"@ztimson/utils": "^0.30.7",
 		"lzma1": "^0.3.0",

src/catalog.js

@@ -1,40 +1,44 @@
 import {fuzzyMatch} from './utils.js';
 import {decodeHtml} from '@ztimson/utils';
 
-export const CATALOG_URL = 'https://library.kiwix.org/catalog/v2/entries';
+export const CATALOG_URL = 'https://library.kiwix.org';
 
 const PAGE_SIZE = 100;
 
-/** Parses `<entry>` blocks out of a Kiwix catalog OPDS XML response. */
-function parseEntries(xml) {
+function parseEntries(xml, catalog = CATALOG_URL) {
 	const blocks = xml.match(/<entry>[\s\S]*?<\/entry>/g) || [];
 	return blocks.map(b => {
 		const grab = re => (b.match(re) || [])[1] || '';
 		const linkMatch = b.match(/<link[^>]*type=["']application\/x-zim[^"']*["'][^>]*href=["']([^"']+)["']/);
+		const iconMatch = b.match(/<link[^>]*rel=["']http:\/\/opds-spec\.org\/image\/thumbnail["'][^>]*href=["']([^"']+)["']/)
+			|| b.match(/<link[^>]*type=["']image\/[^"']*["'][^>]*href=["']([^"']+)["']/);
 		let tags = grab(/<tags>([^<]*)<\/tags>/);
 		if(tags) tags = tags.split(';');
+		const name = grab(/<name>([^<]*)<\/name>/);
 		return {
 			id: grab(/<id>([^<]*)<\/id>/),
 			title: decodeHtml(grab(/<title>([^<]*)<\/title>/)),
 			updated: new Date(grab(/<updated>([^<]*)<\/updated>/)),
 			summary: decodeHtml(grab(/<summary>([^<]*)<\/summary>/)),
 			language: grab(/<language>([^<]*)<\/language>/),
-			name: grab(/<name>([^<]*)<\/name>/),
+			name,
 			category: grab(/<category>([^<]*)<\/category>/),
 			tags,
 			mediaCount: Number(grab(/<mediaCount>([^<]*)<\/mediaCount>/)) || 0,
 			author: grab(/<author>\s*<name>([^<]*)<\/name>\s*<\/author>/m),
 			publisher: grab(/<publisher>\s*<name>([^<]*)<\/name>\s*<\/publisher>/m),
 			articleCount: Number(grab(/<articleCount>([^<]*)<\/articleCount>/)) || 0,
-			sizeMb: linkMatch ? (Number((b.match(/length=["'](\d+)["']/) || [])[1] || 0) / 1024 / 1024).toFixed(1) : '?',
-			href: linkMatch ? linkMatch[1] : null,
+			sizeMb: linkMatch ? +(Number((b.match(/length=["'](\d+)["']/) || [])[1] || 0) / 1024 / 1024).toFixed(1) : '?',
+			download: linkMatch ? linkMatch[1] : null,
+			icon: iconMatch ? new URL(iconMatch[1], catalog).href : null,
+			viewer: name ? new URL(`viewer#${name}`, catalog).href : null,
 		};
 	});
 }
 
 async function fetchEntries(term, lang, url = CATALOG_URL) {
 	const params = new URLSearchParams({q: term, count: String(PAGE_SIZE), lang: lang || 'eng'});
-	const res = await fetch(`${url}?${params}`);
+	const res = await fetch(`${url}/catalog/v2/entries?${params}`);
 	if (!res.ok) throw new Error(`${res.status} ${res.statusText}`);
 	return parseEntries(await res.text());
 }
@@ -42,7 +46,7 @@ async function fetchEntries(term, lang, url = CATALOG_URL) {
 /** Looks up a single catalog entry by exact `name` (used to check for available updates). */
 export async function zimCatalogInfo(name, url = CATALOG_URL) {
 	const params = new URLSearchParams({name, count: '5'});
-	const res = await fetch(`${url}?${params}`);
+	const res = await fetch(`${url}/catalog/v2/entries?${params}`);
 	if (!res.ok) return null;
 	return parseEntries(await res.text()).find(e => e.name === name) || null;
 }

src/decompress-worker.js

@@ -0,0 +1,32 @@
+'use strict';
+
+import {parentPort} from 'node:worker_threads';
+import {decompress as lzmaDecompress} from 'lzma1';
+import {ZstdCodec} from 'zstd-codec';
+
+let zstdStreamingPromise = null;
+
+function getZstd() {
+	if (!zstdStreamingPromise) {
+		zstdStreamingPromise = new Promise(resolve => ZstdCodec.run(zstd => resolve(new zstd.Streaming())));
+	}
+	return zstdStreamingPromise;
+}
+
+/** Rebuilds a full 13-byte "alone" LZMA header from libzim's truncated 5-byte one (unknown size). */
+function toLzmaAloneStream(body) {
+	const header = Buffer.concat([body.subarray(0, 5), Buffer.alloc(8, 0xff)]);
+	return Buffer.concat([header, body.subarray(5)]);
+}
+
+parentPort.on('message', async ({id, compType, body}) => {
+	try {
+		const buf = Buffer.from(body);
+		const data = compType === 4
+			? Buffer.from(lzmaDecompress(toLzmaAloneStream(buf)))
+			: Buffer.from((await getZstd()).decompress(new Uint8Array(buf)));
+		parentPort.postMessage({id, data}, [data.buffer]);
+	} catch (e) {
+		parentPort.postMessage({id, error: e.message});
+	}
+});

src/decompress.js

@@ -0,0 +1,50 @@
+'use strict';
+
+import {Worker} from 'node:worker_threads';
+import path from 'node:path';
+import os from 'node:os';
+import {fileURLToPath} from 'node:url';
+
+const __dirname = path.dirname(fileURLToPath(import.meta.url));
+
+/** Round-robin pool of worker threads for off-main-thread cluster decompression. */
+class DecompressPool {
+	#workers = [];
+	#next = 0;
+	#pending = new Map(); // id -> {resolve, reject}
+	#nextId = 0;
+	#active = 0;
+
+	constructor(size = Math.max(1, os.cpus().length - 1)) {
+		for (let i = 0; i < size; i++) this.#spawn();
+	}
+
+	#spawn() {
+		const worker = new Worker(path.join(__dirname, 'decompress-worker.js'));
+		worker.on('message', ({id, data, error}) => {
+			const task = this.#pending.get(id);
+			if (!task) return;
+			this.#pending.delete(id);
+			// No work left in flight -> safe to let the process exit again.
+			if (--this.#active === 0) for (const w of this.#workers) w.unref();
+			error ? task.reject(new Error(error)) : task.resolve(Buffer.from(data));
+		});
+		worker.unref(); // idle workers must never keep a short script/server alive
+		this.#workers.push(worker);
+	}
+
+	/** Decompresses `{compType, body}` on the next worker in rotation. `body` must be a Uint8Array/Buffer view. */
+	run({compType, body}) {
+		const worker = this.#workers[this.#next];
+		this.#next = (this.#next + 1) % this.#workers.length;
+		const id = this.#nextId++;
+		const owned = new Uint8Array(body);
+		if (this.#active++ === 0) for (const w of this.#workers) w.ref();
+		return new Promise((resolve, reject) => {
+			this.#pending.set(id, {resolve, reject});
+			worker.postMessage({id, compType, body: owned}, [owned.buffer]);
+		});
+	}
+}
+
+export const decompressPool = new DecompressPool();

src/manager.js

@@ -5,14 +5,23 @@ import {Readable} from 'node:stream';
 import {ZimReader} from './reader.js';
 import {zimCatalog, zimCatalogInfo, CATALOG_URL} from './catalog.js';
 
+const DEFAULT_READER_TTL = 60_000;
+
 /** Manages a local directory of ZIM archives: listing, update checks, downloads, and reading. */
 export class ZimManager {
 	#catalog;
 	#dir;
+	#readerTTL;
+	#clusterTTL;
+	#readers = new Map(); // file -> {reader, timer}
+	#opening = new Map(); // file -> Promise<ZimReader>, dedupes concurrent first-open races
 
-	constructor(dir, catalog = CATALOG_URL) {
+	/** @param {{catalog?: string, readerTTL?: number, clusterTTL?: number}} [opts] */
+	constructor(dir, catalog = CATALOG_URL, {readerTTL = DEFAULT_READER_TTL, clusterTTL} = {}) {
 		this.#catalog = catalog;
 		this.#dir = dir;
+		this.#readerTTL = readerTTL;
+		this.#clusterTTL = clusterTTL; // undefined -> ZimReader's own default
 	}
 
 	async #download(url, destPath) {
@@ -44,11 +53,7 @@ export class ZimManager {
 		let reader;
 		try {
 			reader = await new ZimReader(filepath).open();
-			return {
-				name: await reader.metadata('Name'),
-				date: await reader.metadata('Date'),
-				title: await reader.metadata('Title'),
-			};
+			return await reader.metadata();
 		} catch {
 			return null;
 		} finally {
@@ -58,31 +63,48 @@ export class ZimManager {
 
 	async #update(name, catalogEntry, localMatch, force) {
 		const remoteDate = catalogEntry.updated ? new Date(catalogEntry.updated) : null;
-		const localDate = localMatch?.meta?.date ? new Date(localMatch.meta.date) : null;
+		const localDate = localMatch?.meta?.updated ?? null;
 		if (!force && localMatch && remoteDate && localDate && remoteDate <= localDate)
 			return {name, status: 'skipped', reason: 'up to date'};
-		if (!catalogEntry.href) return {name, status: 'skipped', reason: 'missing download link'};
+		if (!catalogEntry.download) return {name, status: 'skipped', reason: 'missing download link'};
 
-		const filename = path.basename(new URL(catalogEntry.href).pathname).replace(/\.meta4$/i, '');
+		const filename = path.basename(new URL(catalogEntry.download).pathname).replace(/\.meta4$/i, '');
 		const destPath = path.join(this.#dir, filename);
-		await this.#download(catalogEntry.href, destPath);
-		if (localMatch && localMatch.file !== destPath) await fs.promises.unlink(localMatch.file).catch(() => {});
-		return {name, status: 'updated', file: destPath};
+		await this.#download(catalogEntry.download, destPath);
+		if (localMatch && localMatch.file !== filename) await this.#evict(path.join(this.#dir, localMatch.file));
+		return {name, status: 'updated', file: filename};
 	}
 
 	/** Resolves a file path or catalog `name` to a local file path. */
 	async #resolveFile(fileOrName) {
 		if (fs.existsSync(fileOrName)) return fileOrName;
+		const joined = path.join(this.#dir, fileOrName);
+		if (fs.existsSync(joined)) return joined;
 		const local = await this.list();
-		const match = local.find(l => l.meta?.name === fileOrName);
+		const match = local.find(l => l.meta?.name === fileOrName || path.basename(l.file) === fileOrName);
 		if (!match) throw new Error(`ZIM not found locally: ${fileOrName}`);
-		return match.file;
+		return path.join(this.#dir, match.file);
+	}
+
+	/** Closes and drops a cached reader for `file`, if any (used before delete/replace). */
+	async #evict(file) {
+		const entry = this.#readers.get(file);
+		if (!entry) return new ZimReader(file).delete().catch(() => {});
+		clearTimeout(entry.timer);
+		this.#readers.delete(file);
+		await entry.reader.delete();
 	}
 
 	catalog(search, opts) {
 		return zimCatalog(search, opts);
 	}
 
+	async delete(fileOrName) {
+		const file = await this.#resolveFile(fileOrName);
+		await this.#evict(file);
+		return {file: path.basename(file), status: 'deleted'};
+	}
+
 	/** Checks whether a local ZIM has a newer version in the catalog, without downloading. */
 	async isOutdated(file) {
 		const meta = await this.#readMeta(file);
@@ -90,7 +112,7 @@ export class ZimManager {
 		const catalogEntry = await zimCatalog(meta.name, this.#catalog);
 		if (!catalogEntry) return {file, upToDate: null, reason: 'missing from catalog'};
 		const remoteDate = catalogEntry.updated ? new Date(catalogEntry.updated) : null;
-		const localDate = meta.date ? new Date(meta.date) : null;
+		const localDate = meta.updated ?? null;
 		return {file, name: meta.name, upToDate: !!(remoteDate && localDate && remoteDate <= localDate), localDate, remoteDate};
 	}
 
@@ -100,7 +122,7 @@ export class ZimManager {
 		const files = (await fs.promises.readdir(this.#dir)).filter(f => f.endsWith('.zim'));
 		return Promise.all(files.map(async f => {
 			const file = path.join(this.#dir, f);
-			return {file, meta: await this.#readMeta(file)};
+			return {file: f, ...(await this.#readMeta(file))};
 		}));
 	}
 
@@ -113,7 +135,7 @@ export class ZimManager {
 
 		const local = await this.list();
 		const localMatch = local.find(l => l.meta?.name === name) ?? null;
-		const catalogEntry = await zimCatalogInfo(name, this.#catalog) || {name, updated: null, href};
+		const catalogEntry = await zimCatalogInfo(name, this.#catalog) || {name, updated: null, download: href};
 		return this.#update(name, catalogEntry, localMatch, force);
 	}
 
@@ -136,11 +158,32 @@ export class ZimManager {
 		return results;
 	}
 
-	/** Opens a `ZimReader` for a local ZIM, resolved by file path or catalog `name`. Caller must `.close()` it. */
+	/** Opens a fresh `ZimReader` for a local ZIM, resolved by file path or catalog `name`. Caller must `.close()` it. */
 	async open(fileOrName) {
 		await this.#ensureDir();
 		const file = await this.#resolveFile(fileOrName);
-		return new ZimReader(file).open();
+		let entry = this.#readers.get(file);
+		if (!entry) {
+			let pending = this.#opening.get(file);
+			if (!pending) {
+				pending = new ZimReader(file, {clusterTTL: this.#clusterTTL}).open();
+				this.#opening.set(file, pending);
+			}
+			let reader;
+			try {
+				reader = await pending;
+			} finally {
+				this.#opening.delete(file);
+			}
+			entry = this.#readers.get(file) ?? {reader};
+			this.#readers.set(file, entry);
+		}
+		clearTimeout(entry.timer);
+		entry.timer = setTimeout(() => {
+			this.#readers.delete(file);
+			entry.reader.close();
+		}, this.#readerTTL).unref();
+		return entry.reader;
 	}
 
 	/** Fuzzy-searches titles across every local ZIM in the library, merging & re-ranking hits by score. */
@@ -149,15 +192,15 @@ export class ZimManager {
 		const perZim = await Promise.all(local.map(async ({file, meta}) => {
 			let reader;
 			try {
-				reader = await new ZimReader(file).open();
+				reader = await new ZimReader(path.join(this.#dir, file)).open();
 				const hits = await reader.search(terms, {limit, htmlOnly});
-				return hits.map(h => ({...h, file, name: meta?.name}));
+				return hits.map(h => ({...h, file}));
 			} catch {
 				return [];
 			} finally {
 				await reader?.close();
 			}
 		}));
-		return perZim.flat().toSorted((a, b) => b.score - a.score).slice(0, limit);
+		return perZim.flat().filter(a => a.score > 0).toSorted((a, b) => b.score - a.score).slice(0, limit);
 	}
 }

src/reader.js

@@ -1,30 +1,23 @@
 'use strict';
 
 import fs from 'node:fs';
-import {decompress as lzmaDecompress} from 'lzma1';
-import {ZstdCodec} from 'zstd-codec';
-import {fuzzyMatch} from './utils.js';
+import {createHash} from 'node:crypto';
+import {decompressPool} from './decompress.js';
+import {fuzzyMatch, titleFromUrl, kiwixSearch} from './utils.js';
 
 const HEADER_SIZE = 80;
 const NS_CONTENT = 'C';
 const NS_METADATA = 'M';
 const TITLE_SENTINEL = 0xffffffffffffffffn; // Indicator -> ZIM v6+ archives with no title
 
-let zstdStreamingPromise = null;
+const DEFAULT_CLUSTER_CACHE_MAX = 32;
+const DEFAULT_CLUSTER_TTL = 60_000;
 
-/** Lazily initialised, shared Zstd streaming decompressor (handles unknown-size frames). */
-function getZstd() {
-	if (!zstdStreamingPromise) {
-		zstdStreamingPromise = new Promise(resolve => ZstdCodec.run(zstd => resolve(new zstd.Streaming())));
-	}
-	return zstdStreamingPromise;
-}
-
-/** Rebuilds a full 13-byte "alone" LZMA header from libzim's truncated 5-byte one (unknown size). */
-function toLzmaAloneStream(body) {
-	const header = Buffer.concat([body.subarray(0, 5), Buffer.alloc(8, 0xff)]);
-	return Buffer.concat([header, body.subarray(5)]);
-}
+// --- Fallback search index (only used for archives with no native title listing) ---
+const INDEX_SUFFIX = '.searchidx.bin';
+const INDEX_MAGIC = 'ZXI1';
+const INDEX_HEADER_SIZE = 24;     // magic(4) + staleness key(16) + recordCount(4)
+const INDEX_RECORD_SIZE = 20;     // keyOff(4) keyLen(2) urlOff(4) urlLen(2) titleOff(4) titleLen(2) mimetype(2)
 
 /** Native, dependency-light reader for .zim archives. Supports zstd & LZMA cluster compression. */
 export class ZimReader {
@@ -32,23 +25,29 @@ export class ZimReader {
 	#header = null;
 	#mimeTypes = [];
 	#hasTitleListing = false;
+	#clusterCache = new Map(); // clusterNumber -> {data, extended, timer}
+	#pending = new Map();      // clusterNumber -> Promise, dedupes concurrent misses
+	#clusterCacheMax;
+	#clusterTTL;
+
+	#indexPath;
+	#indexReady = null;   // Promise, awaited by search() before using the fallback index
+	#indexFd = null;      // open fd for the fallback index, once loaded/built
+	#indexRecordCount = 0;
+	#indexTableStart = 0;
 
 	get articleCount() { return this.#header?.articleCount ?? 0; }
+	get mediaCount() { return this.#header?.clusterCount ?? 0; }
 
-	constructor(path) {
+	/** @param {{clusterCacheMax?: number, clusterTTL?: number}} [opts] clusterTTL in ms; 0/null disables idle eviction. */
+	constructor(path, {clusterCacheMax = DEFAULT_CLUSTER_CACHE_MAX, clusterTTL = DEFAULT_CLUSTER_TTL} = {}) {
 		this.path = path;
+		this.#clusterCacheMax = clusterCacheMax;
+		this.#clusterTTL = clusterTTL;
+		this.#indexPath = `${path}${INDEX_SUFFIX}`;
 	}
 
-	/** Full O(n) scan over the URL pointer list, used when there's no title index. */
-	async #allDirents() {
-		const dirents = [];
-		for (let i = 0; i < this.#header.articleCount; i++) {
-			dirents.push(await this.#readDirent(await this.#ptr64(this.#header.urlPtrPos, i)));
-		}
-		return dirents;
-	}
-
-	/** Binary search the URL pointer list for namespace+url. */
+	/** Binary search the URL pointer list for namespace+url. For exact-key lookups (readPage, metadata, icons). */
 	async #findByUrl(url, namespace) {
 		const key = namespace + url;
 		let lo = 0, hi = this.#header.articleCount - 1;
@@ -63,7 +62,191 @@ export class ZimReader {
 		return null;
 	}
 
-	async #getBlob(clusterNumber, blobNumber) {
+	/** Binary search the ZIM's own title pointer list. O(log n), zero extra storage - the happy path. */
+	async #findByTitleBuiltin(title) {
+		let lo = 0, hi = this.#header.articleCount - 1;
+		while (lo <= hi) {
+			const mid = (lo + hi) >> 1;
+			const urlIdx = (await this.#read(this.#header.titlePtrPos + mid * 4, 4)).readUInt32LE(0);
+			const d = await this.#readDirent(await this.#ptr64(this.#header.urlPtrPos, urlIdx));
+			if (d.title === title) return d;
+			if (d.title < title) lo = mid + 1; else hi = mid - 1;
+		}
+		return null;
+	}
+
+	/**
+	 * Resolves a kiwix-search result (a title, not necessarily a url) to
+	 * {url, title, mimetype}. kiwix-search's fulltext index returns titles, and
+	 * a title isn't guaranteed to equal its url (unicode normalization,
+	 * disambiguation suffixes, punctuation stripping), so:
+	 *   1. URL binary search    - matches when title happens to equal url (free to check)
+	 *   2. Title pointer list   - when the archive ships one (Wikipedia etc. do)
+	 *   3. Persisted fallback index / linear scan - only for archives without (2)
+	 */
+	async #resolveSearchEntry(name) {
+		let dirent = await this.#findByUrl(name, NS_CONTENT);
+		if (dirent) return dirent;
+
+		if (this.#hasTitleListing) return this.#findByTitleBuiltin(name);
+
+		if (this.#indexReady) await this.#indexReady;
+		const hit = await this.#lookupFallbackIndex(name);
+		if (hit) return hit;
+
+		// Index unavailable (build failed - unwritable disk, etc.) or genuinely no match.
+		for (let i = 0; i < this.#header.articleCount; i++) {
+			const d = await this.#readDirent(await this.#ptr64(this.#header.urlPtrPos, i));
+			if (d.namespace === NS_CONTENT && (d.title === name || titleFromUrl(d.url) === name)) return d;
+		}
+		return null;
+	}
+
+	/**
+	 * A cheap, stable fingerprint for "is the cached index still valid for this file".
+	 * ZIM archives end with a 16-byte MD5 of their own contents (the same trailer
+	 * `zimcheck` validates against), so this is a single 16-byte read regardless of
+	 * archive size - no need to hash a multi-hundred-GB file. It's also
+	 * content-based rather than path/mtime-based, so moving or redownloading an
+	 * identical archive doesn't invalidate the cache. Falls back to a tiny
+	 * size+mtime hash only if the file is too short to have a real trailer.
+	 */
+	async #stalenessKey() {
+		const {size} = await fs.promises.stat(this.path);
+		if (size >= 16) return this.#read(size - 16, 16);
+		const stat = await fs.promises.stat(this.path);
+		return createHash('md5').update(`${stat.size}:${stat.mtimeMs}`).digest();
+	}
+
+	/** Walks every content dirent once, resolving redirects, keyed by both its title and its url-derived title. */
+	async #collectFallbackEntries() {
+		const map = new Map(); // key -> {url, title, mimetype}
+		for (let i = 0; i < this.#header.articleCount; i++) {
+			let dirent = await this.#readDirent(await this.#ptr64(this.#header.urlPtrPos, i));
+			if (dirent.namespace !== NS_CONTENT) continue;
+			dirent = await this.#resolveRedirect(dirent);
+			if (dirent.namespace !== NS_CONTENT) continue; // redirected outside content namespace
+
+			const entry = {url: dirent.url, title: dirent.title, mimetype: dirent.mimetype};
+			if (!map.has(dirent.title)) map.set(dirent.title, entry);
+			const urlTitle = titleFromUrl(dirent.url);
+			if (urlTitle !== dirent.title && !map.has(urlTitle)) map.set(urlTitle, entry);
+		}
+		return map;
+	}
+
+	/** Serializes the fallback index to disk: header, sorted fixed-width records, then a string table. */
+	async #buildFallbackIndex(stalenessKey) {
+		const map = await this.#collectFallbackEntries();
+		const keys = [...map.keys()].sort();
+
+		const records = Buffer.alloc(keys.length * INDEX_RECORD_SIZE);
+		const strings = [];
+		let tableOffset = 0;
+		keys.forEach((key, i) => {
+			const {url, title, mimetype} = map.get(key);
+			const keyBuf = Buffer.from(key, 'utf8');
+			const urlBuf = Buffer.from(url, 'utf8');
+			const titleBuf = Buffer.from(title, 'utf8');
+			const base = i * INDEX_RECORD_SIZE;
+
+			records.writeUInt32LE(tableOffset, base); records.writeUInt16LE(keyBuf.length, base + 4);
+			tableOffset += keyBuf.length;
+			records.writeUInt32LE(tableOffset, base + 6); records.writeUInt16LE(urlBuf.length, base + 10);
+			tableOffset += urlBuf.length;
+			records.writeUInt32LE(tableOffset, base + 12); records.writeUInt16LE(titleBuf.length, base + 16);
+			tableOffset += titleBuf.length;
+			records.writeUInt16LE(mimetype, base + 18);
+
+			strings.push(keyBuf, urlBuf, titleBuf);
+		});
+
+		const header = Buffer.alloc(INDEX_HEADER_SIZE);
+		header.write(INDEX_MAGIC, 0, 'ascii');
+		stalenessKey.copy(header, 4);
+		header.writeUInt32LE(keys.length, 20);
+
+		const tmpPath = `${this.#indexPath}.tmp-${process.pid}`;
+		await fs.promises.writeFile(tmpPath, Buffer.concat([header, records, ...strings]));
+		await fs.promises.rename(tmpPath, this.#indexPath);
+	}
+
+	/** Opens the fallback index file and caches its header fields for querying. */
+	async #openFallbackIndex() {
+		const fd = await fs.promises.open(this.#indexPath, 'r');
+		const header = Buffer.alloc(INDEX_HEADER_SIZE);
+		await fd.read(header, 0, INDEX_HEADER_SIZE, 0);
+		this.#indexFd = fd;
+		this.#indexRecordCount = header.readUInt32LE(20);
+		this.#indexTableStart = INDEX_HEADER_SIZE + this.#indexRecordCount * INDEX_RECORD_SIZE;
+	}
+
+	/**
+	 * Validates the on-disk fallback index against the archive's current staleness
+	 * key and (re)builds it if missing/stale. Failures are swallowed - callers see
+	 * #indexFd stay null and fall through to the linear-scan last resort.
+	 */
+	async #loadOrBuildFallbackIndex() {
+		const stalenessKey = await this.#stalenessKey();
+
+		try {
+			const fd = await fs.promises.open(this.#indexPath, 'r');
+			const header = Buffer.alloc(INDEX_HEADER_SIZE);
+			await fd.read(header, 0, INDEX_HEADER_SIZE, 0);
+			const fresh = header.toString('ascii', 0, 4) === INDEX_MAGIC && header.subarray(4, 20).equals(stalenessKey);
+			if (fresh) {
+				this.#indexFd = fd;
+				this.#indexRecordCount = header.readUInt32LE(20);
+				this.#indexTableStart = INDEX_HEADER_SIZE + this.#indexRecordCount * INDEX_RECORD_SIZE;
+				return;
+			}
+			await fd.close();
+		} catch { /* missing, corrupt, or unreadable -> rebuild below */ }
+
+		await this.#buildFallbackIndex(stalenessKey);
+		await this.#openFallbackIndex();
+	}
+
+	async #indexRead(pos, length) {
+		const buf = Buffer.alloc(length);
+		await this.#indexFd.read(buf, 0, length, pos);
+		return buf;
+	}
+
+	async #indexString(offset, length) {
+		if (!length) return '';
+		return (await this.#indexRead(this.#indexTableStart + offset, length)).toString('utf8');
+	}
+
+	/** Binary search over the on-disk record array. Never loads the full index into memory. */
+	async #lookupFallbackIndex(name) {
+		if (!this.#indexFd) return null;
+		let lo = 0, hi = this.#indexRecordCount - 1;
+		while (lo <= hi) {
+			const mid = (lo + hi) >> 1;
+			const rec = await this.#indexRead(INDEX_HEADER_SIZE + mid * INDEX_RECORD_SIZE, INDEX_RECORD_SIZE);
+			const keyOff = rec.readUInt32LE(0), keyLen = rec.readUInt16LE(4);
+			const key = await this.#indexString(keyOff, keyLen);
+			if (key === name) {
+				const urlOff = rec.readUInt32LE(6), urlLen = rec.readUInt16LE(10);
+				const titleOff = rec.readUInt32LE(12), titleLen = rec.readUInt16LE(16);
+				const [url, title] = await Promise.all([this.#indexString(urlOff, urlLen), this.#indexString(titleOff, titleLen)]);
+				return {url, title, mimetype: rec.readUInt16LE(18)};
+			}
+			if (key < name) lo = mid + 1; else hi = mid - 1;
+		}
+		return null;
+	}
+
+	/** Resets a cluster's idle-eviction timer. No-op when TTL disabled. */
+	#touch(clusterNumber, entry) {
+		if (!this.#clusterTTL) return;
+		clearTimeout(entry.timer);
+		entry.timer = setTimeout(() => this.#clusterCache.delete(clusterNumber), this.#clusterTTL).unref();
+	}
+
+	/** Fetches + decompresses a cluster exactly once, offloading decompression to the worker pool. */
+	async #loadCluster(clusterNumber) {
 		const start = await this.#ptr64(this.#header.clusterPtrPos, clusterNumber);
 		const isLast = clusterNumber === this.#header.clusterCount - 1;
 		const end = isLast
@@ -76,14 +259,42 @@ export class ZimReader {
 		const body = raw.subarray(1);
 
 		let data;
-		if (compType <= 1) data = body;
-		else if (compType === 4) data = Buffer.from(lzmaDecompress(toLzmaAloneStream(body)));
-		else if (compType === 5) data = Buffer.from((await getZstd()).decompress(new Uint8Array(body)));
+		if (compType <= 1) data = Buffer.from(body);
+		else if (compType === 4 || compType === 5) data = await decompressPool.run({compType, body});
 		else throw new Error(`Unsupported cluster compression type: ${compType}`);
 		if (!data) throw new Error(`Cluster ${clusterNumber} failed to decompress (compType ${compType})`);
 
-		const readPtr = i => extended ? Number(data.readBigUInt64LE(i * 8)) : data.readUInt32LE(i * 4);
-		return data.subarray(readPtr(blobNumber), readPtr(blobNumber + 1));
+		return {data, extended};
+	}
+
+	async #getBlob(clusterNumber, blobNumber) {
+		let entry = this.#clusterCache.get(clusterNumber);
+		if (!entry) {
+			let pending = this.#pending.get(clusterNumber);
+			if (!pending) {
+				pending = this.#loadCluster(clusterNumber);
+				this.#pending.set(clusterNumber, pending);
+			}
+			entry = await pending;
+			this.#pending.delete(clusterNumber);
+			this.#clusterCache.set(clusterNumber, entry);
+			if (this.#clusterCache.size > this.#clusterCacheMax) {
+				const oldestKey = this.#clusterCache.keys().next().value;
+				clearTimeout(this.#clusterCache.get(oldestKey)?.timer);
+				this.#clusterCache.delete(oldestKey);
+			}
+		}
+		this.#touch(clusterNumber, entry);
+
+		const readPtr = i => entry.extended ? Number(entry.data.readBigUInt64LE(i * 8)) : entry.data.readUInt32LE(i * 4);
+		return entry.data.subarray(readPtr(blobNumber), readPtr(blobNumber + 1));
+	}
+
+	async #icon(size = 48) {
+		const page = await this.readPage(`Illustration_${size}x${size}@1`, NS_METADATA)
+			|| await this.readPage('Favicon', NS_METADATA);
+		if (!page) return null;
+		return `data:${page.mimetype};base64,${page.data.toString('base64')}`;
 	}
 
 	async #ptr64(base, index) {
@@ -152,42 +363,70 @@ export class ZimReader {
 		return this.#readDirent(await this.#ptr64(this.#header.urlPtrPos, dirent.redirectIndex));
 	}
 
-	/** Narrows to dirents near the term's alphabetical position in the title index. */
-	async #titleIndexCandidates(term) {
-		const q = term.toLowerCase();
-		let lo = 0, hi = this.#header.articleCount - 1;
-		while (lo < hi) {
-			const mid = (lo + hi) >> 1;
-			const urlIdx = (await this.#read(this.#header.titlePtrPos + mid * 4, 4)).readUInt32LE(0);
-			const dirent = await this.#readDirent(await this.#ptr64(this.#header.urlPtrPos, urlIdx));
-			if (dirent.title.toLowerCase() < q) lo = mid + 1; else hi = mid;
-		}
-		// Widen around the prefix match since fuzzy scoring isn't purely alphabetical.
-		const start = Math.max(0, lo - 50), end = Math.min(this.#header.articleCount, lo + 200);
-		const dirents = [];
-		for (let i = start; i < end; i++) {
-			const urlIdx = (await this.#read(this.#header.titlePtrPos + i * 4, 4)).readUInt32LE(0);
-			dirents.push(await this.#readDirent(await this.#ptr64(this.#header.urlPtrPos, urlIdx)));
-		}
-		return dirents;
-	}
-
 	async close() {
 		if (this.#fd) await this.#fd.close();
 		this.#fd = null;
+		if (this.#indexFd) await this.#indexFd.close();
+		this.#indexFd = null;
+		for (const entry of this.#clusterCache.values()) clearTimeout(entry.timer);
+		this.#clusterCache.clear();
+		this.#pending.clear();
+	}
+
+	/** Deletes the zim archive and its cached fallback index (if any). Safe to call on unopened readers. */
+	async delete() {
+		await this.close();
+		await fs.promises.rm(this.path, {force: true});
+		await fs.promises.rm(this.#indexPath, {force: true});
 	}
 
 	/** Reads an 'M' namespace metadata value (e.g. Name, Date, Title). Returns null if missing. */
 	async metadata(key) {
-		const page = await this.readPage(key, NS_METADATA);
-		return page ? page.data.toString('utf8') : null;
+		const get = async key => {
+			const page = await this.readPage(key, NS_METADATA);
+			return page ? page.data.toString('utf8') : null;
+		};
+		if(key) return get(key);
+
+		const [title, creator, publisher, date, description, language, name, tags] = await Promise.all(
+			['Title', 'Creator', 'Publisher', 'Date', 'Description', 'Language', 'Name', 'Tags'].map(get)
+		);
+		return {
+			title,
+			updated: date ? new Date(date) : null,
+			summary: description,
+			language,
+			name,
+			category: tags ? tags.split(';')[0] || '' : '',
+			tags: tags ? tags.split(';') : [],
+			author: creator,
+			publisher,
+			articleCount: this.articleCount,
+			mediaCount: this.mediaCount,
+			sizeMb: +((await fs.promises.stat(this.path)).size / 1024 / 1024).toFixed(1),
+			icon: await this.#icon(),
+		};
 	}
 
 	/** Opens the archive and parses its header + mimetype list. */
 	async open() {
 		this.#fd = await fs.promises.open(this.path, 'r');
-		await this.#readHeader();
-		await this.#readMimeTypes();
+		try {
+			await this.#readHeader();
+			await this.#readMimeTypes();
+		} catch (e) {
+			await this.#fd.close();
+			this.#fd = null;
+			throw e;
+		}
+
+		// The fallback index is only needed when the archive has no native title
+		// pointer list - #findByTitleBuiltin already covers that case in O(log n)
+		// with zero extra storage. Well-maintained archives (Wikipedia etc.)
+		// ship a title listing, so this path is expected to be rare in practice.
+		if (!this.#hasTitleListing) {
+			this.#indexReady = this.#loadOrBuildFallbackIndex().catch(() => {});
+		}
 		return this;
 	}
 
@@ -206,30 +445,30 @@ export class ZimReader {
 		let dirent = await this.#readDirent(await this.#ptr64(this.#header.urlPtrPos, this.#header.mainPage));
 		dirent = await this.#resolveRedirect(dirent);
 		const data = await this.#getBlob(dirent.cluster, dirent.blob);
-		return {mimetype: this.#mimeTypes[dirent.mimetype] || 'application/octet-stream', data, url: dirent.url};
+		return {mimetype: this.#mimeTypes[dirent.mimetype] || 'application/octet-stream', data, url: dirent.url, namespace: dirent.namespace};
 	}
 
 	/**
 	 * Fuzzy-ranked title search. Accepts comma-separated `terms` the same way the
-	 * catalog search does. Uses the sorted title index when present (binary search
-	 * narrows the candidate window); falls back to a full linear scan otherwise
-	 * (common on ZIM v6+/zimit-generated archives with no title index).
+	 * catalog search does. Uses kiwix-search's embedded fulltext index as a prefilter
+	 * to narrow candidates before fuzzy scoring.
 	 */
 	async search(terms, {limit = 20, htmlOnly = true} = {}) {
 		const termList = String(terms).split(',').map(t => t.trim()).filter(Boolean);
 		if (!termList.length) return [];
 
-		const candidates = this.#hasTitleListing
-			? await this.#titleIndexCandidates(termList[0])
-			: await this.#allDirents();
+		const names = await kiwixSearch(this.path, termList.join(' '));
+		const candidates = (await Promise.all(names.map(n => this.#resolveSearchEntry(n)))).filter(Boolean);
 
 		const scored = [];
 		for (const dirent of candidates) {
-			if (dirent.namespace !== NS_CONTENT) continue;
 			if (htmlOnly && !(this.#mimeTypes[dirent.mimetype] || '').startsWith('text/html')) continue;
-			const {max} = fuzzyMatch(dirent.title, ...termList);
-			scored.push({url: dirent.url, title: dirent.title, namespace: dirent.namespace, score: max});
+			const urlTitle = titleFromUrl(dirent.url);
+			const titleScore = fuzzyMatch(dirent.title, ...termList).max;
+			const urlScore = fuzzyMatch(urlTitle, ...termList).max;
+			scored.push({url: dirent.url, title: dirent.title.length > urlTitle.length ? dirent.title : urlTitle, namespace: NS_CONTENT, score: Math.max(titleScore, urlScore)});
 		}
-		return scored.toSorted((a, b) => b.score - a.score).slice(0, limit);
+		const {summary, mediaCount, articleCount, sizeMb, ...meta} = await this.metadata();
+		return scored.filter(a => a.score > 0).toSorted((a, b) => b.score - a.score).slice(0, limit).map(a => ({...meta, ...a}));
 	}
 }

src/utils.js

@@ -1,3 +1,32 @@
+import {execFile} from 'node:child_process';
+import path from 'node:path';
+import {fileURLToPath} from 'node:url';
+import {promisify} from 'node:util';
+
+/**
+ * Search a ZIM file's fulltext index.
+ * @param {string} zimPath - Path to the .zim file
+ * @param {string} pattern - Search terms
+ * @param {object} [opts]
+ * @param {boolean} [opts.suggestion] - Suggest titles from partial pattern (completion-style)
+ * @param {boolean} [opts.spelling] - Suggest spelling-corrected titles
+ * @returns {Promise<string[]>} Matching article/tag titles
+ */
+export async function kiwixSearch(zimPath, pattern, opts = {}) {
+	const execFileAsync = promisify(execFile);
+	const __dirname = path.dirname(fileURLToPath(import.meta.url));
+	const BIN_NAME = process.platform === 'win32' ? 'kiwix-search.exe' : 'kiwix-search';
+	const BIN_PATH = path.join(__dirname, '..', 'bin', BIN_NAME);
+
+	const args = [];
+	if (opts.suggestion) args.push('-s');
+	if (opts.spelling) args.push('--spelling');
+	args.push(zimPath, pattern);
+
+	const {stdout} = await execFileAsync(BIN_PATH, args);
+	return stdout.split('\n').map(line => line.trim()).filter(Boolean);
+}
+
 export function levenshtein(a, b) {
 	const m = a.length, n = b.length;
 	if (!m) return n;
@@ -20,13 +49,36 @@ export function similarity(a, b) {
 	return 1 - levenshtein(a, b) / Math.max(a.length, b.length, 1);
 }
 
+function scoreAgainst(text, term) {
+	if (text.includes(term)) return 1 - (text.length - term.length) / text.length * 0.3;
+	if (!text.length || !term.length || text[0] !== term[0]) return 0;
+	const dist = levenshtein(text, term);
+	const maxAllowed = Math.max(1, Math.ceil(term.length * 0.34));
+	if (dist > maxAllowed) return 0;
+	return 1 - dist / Math.max(text.length, term.length);
+}
+
 /** Compares `target` against one or more search terms; returns avg/max/per-term similarity. */
 export function fuzzyMatch(target, ...terms) {
 	if (!terms.length) throw new Error('Requires at least 1 term to compare');
-	const similarities = terms.map(t => similarity(target, t));
+	const lowerTarget = String(target).toLowerCase();
+	const words = lowerTarget.split(/\W+/).filter(Boolean);
+
+	const similarities = terms.map(term => {
+		const t = term.toLowerCase();
+		return Math.max(scoreAgainst(lowerTarget, t), ...words.map(w => scoreAgainst(w, t)));
+	});
+
 	return {
 		avg: similarities.reduce((acc, s) => acc + s, 0) / similarities.length,
 		max: Math.max(...similarities),
 		similarities,
 	};
 }
+
+/** Derives a readable pseudo-title from a URL's last path segment, e.g. ".../diannes-southwest-salad/" -> "Diannes Southwest Salad". */
+export function titleFromUrl(url) {
+	const slug = String(url).replace(/\/$/, '').split('/').pop() || url;
+	const clean = slug.replace(/\.(zim|meta4|html?|md)$/i, '').replace(/[-_.]+/g, ' ');
+	return clean.replace(/\b\w/g, c => c.toUpperCase());
+}