New Zim utilties

2026-08-20 11:17:45 -04:00
parent 7e95b337e3
commit 18dbd98e19
9 changed files with 681 additions and 95 deletions
--- a/src/catalog.js
+++ b/src/catalog.js
@@ -0,0 +1,73 @@
+import {fuzzyMatch} from './utils.js';
+import {decodeHtml} from '@ztimson/utils';
+
+export const CATALOG_URL = 'https://library.kiwix.org/catalog/v2/entries';
+
+const PAGE_SIZE = 100;
+
+/** Parses `<entry>` blocks out of a Kiwix catalog OPDS XML response. */
+function parseEntries(xml) {
+	const blocks = xml.match(/<entry>[\s\S]*?<\/entry>/g) || [];
+	return blocks.map(b => {
+		const grab = re => (b.match(re) || [])[1] || '';
+		const linkMatch = b.match(/<link[^>]*type=["']application\/x-zim[^"']*["'][^>]*href=["']([^"']+)["']/);
+		let tags = grab(/<tags>([^<]*)<\/tags>/);
+		if(tags) tags = tags.split(';');
+		return {
+			id: grab(/<id>([^<]*)<\/id>/),
+			title: decodeHtml(grab(/<title>([^<]*)<\/title>/)),
+			updated: new Date(grab(/<updated>([^<]*)<\/updated>/)),
+			summary: decodeHtml(grab(/<summary>([^<]*)<\/summary>/)),
+			language: grab(/<language>([^<]*)<\/language>/),
+			name: grab(/<name>([^<]*)<\/name>/),
+			category: grab(/<category>([^<]*)<\/category>/),
+			tags,
+			mediaCount: Number(grab(/<mediaCount>([^<]*)<\/mediaCount>/)) || 0,
+			author: grab(/<author>\s*<name>([^<]*)<\/name>\s*<\/author>/m),
+			publisher: grab(/<publisher>\s*<name>([^<]*)<\/name>\s*<\/publisher>/m),
+			articleCount: Number(grab(/<articleCount>([^<]*)<\/articleCount>/)) || 0,
+			sizeMb: linkMatch ? (Number((b.match(/length=["'](\d+)["']/) || [])[1] || 0) / 1024 / 1024).toFixed(1) : '?',
+			href: linkMatch ? linkMatch[1] : null,
+		};
+	});
+}
+
+async function fetchEntries(term, lang, url = CATALOG_URL) {
+	const params = new URLSearchParams({q: term, count: String(PAGE_SIZE), lang: lang || 'eng'});
+	const res = await fetch(`${url}?${params}`);
+	if (!res.ok) throw new Error(`${res.status} ${res.statusText}`);
+	return parseEntries(await res.text());
+}
+
+/** Looks up a single catalog entry by exact `name` (used to check for available updates). */
+export async function zimCatalogInfo(name, url = CATALOG_URL) {
+	const params = new URLSearchParams({name, count: '5'});
+	const res = await fetch(`${url}?${params}`);
+	if (!res.ok) return null;
+	return parseEntries(await res.text()).find(e => e.name === name) || null;
+}
+
+/** Searches the Kiwix catalog for ZIMs matching `terms` within `category`, ranked by term coverage then fuzzy similarity. */
+export async function zimCatalog(terms, opts = {lang: 'eng', count: 20, url: CATALOG_URL}) {
+	opts = Object.assign({lang: 'eng', count: 20, url: CATALOG_URL}, opts)
+	const termList = [...String(terms).split(',')].filter(Boolean).map(t => t.trim().toLowerCase());
+	const results = await Promise.allSettled(termList.map(t => fetchEntries(t, opts.lang, opts.url)));
+	const byName = new Map(); // name -> {entry, hitTerms:Set}
+	results.forEach((r, i) => {
+		if (r.status !== 'fulfilled') return;
+		const term = termList[i];
+		for (const entry of r.value) {
+			if (!entry.name) continue;
+			if (!byName.has(entry.name)) byName.set(entry.name, {entry, hitTerms: new Set()});
+			byName.get(entry.name).hitTerms.add(term);
+		}
+	});
+	if (!byName.size) return [];
+
+	return [...byName.values()].map(({entry, hitTerms}) => {
+		const text = `${entry.title} ${entry.summary}`.trim();
+		return {entry, hits: hitTerms.size, fuzzy: fuzzyMatch(text, ...termList).max};
+	}).toSorted((a, b) =>
+		b.hits - a.hits || b.fuzzy - a.fuzzy || b.entry.articleCount - a.entry.articleCount
+	).slice(0, opts.count).map(r => r.entry);
+}
--- a/src/index.js
+++ b/src/index.js
@@ -0,0 +1,4 @@
+export * from './catalog.js';
+export * from './manager.js';
+export * from './reader.js';
+export * from './utils.js';
--- a/src/manager.js
+++ b/src/manager.js
@@ -0,0 +1,163 @@
+import fs from 'node:fs';
+import path from 'node:path';
+import {pipeline} from 'node:stream/promises';
+import {Readable} from 'node:stream';
+import {ZimReader} from './reader.js';
+import {zimCatalog, zimCatalogInfo, CATALOG_URL} from './catalog.js';
+
+/** Manages a local directory of ZIM archives: listing, update checks, downloads, and reading. */
+export class ZimManager {
+	#catalog;
+	#dir;
+
+	constructor(dir, catalog = CATALOG_URL) {
+		this.#catalog = catalog;
+		this.#dir = dir;
+	}
+
+	async #download(url, destPath) {
+		const {res} = await this.#resolveUrl(url);
+		if (!res.ok) throw new Error(`${res.status} ${res.statusText}`);
+		const tmpPath = `${destPath}.part`;
+		await pipeline(Readable.fromWeb(res.body), fs.createWriteStream(tmpPath));
+		await fs.promises.rename(tmpPath, destPath);
+	}
+
+	async #ensureDir() {
+		await fs.promises.mkdir(this.#dir, {recursive: true});
+	}
+
+	/** Resolves a `.meta4` metalink URL down to the real mirror `.zim` download URL. */
+	async #resolveUrl(url) {
+		const head = await fetch(url);
+		const ct = head.headers.get('content-type') || '';
+		if (!ct.includes('metalink') && !url.endsWith('.meta4')) return {res: head, url};
+		const meta = await head.text();
+		const m = meta.match(/<url[^>]*>([^<]+\.zim)<\/url>/);
+		if (!m) throw new Error('Could not resolve metalink mirror');
+		const res = await fetch(m[1]);
+		return {res, url: m[1]};
+	}
+
+	/** Reads Name/Date/Title metadata from a local ZIM file. Returns null if unreadable. */
+	async #readMeta(filepath) {
+		let reader;
+		try {
+			reader = await new ZimReader(filepath).open();
+			return {
+				name: await reader.metadata('Name'),
+				date: await reader.metadata('Date'),
+				title: await reader.metadata('Title'),
+			};
+		} catch {
+			return null;
+		} finally {
+			await reader?.close();
+		}
+	}
+
+	async #update(name, catalogEntry, localMatch, force) {
+		const remoteDate = catalogEntry.updated ? new Date(catalogEntry.updated) : null;
+		const localDate = localMatch?.meta?.date ? new Date(localMatch.meta.date) : null;
+		if (!force && localMatch && remoteDate && localDate && remoteDate <= localDate)
+			return {name, status: 'skipped', reason: 'up to date'};
+		if (!catalogEntry.href) return {name, status: 'skipped', reason: 'missing download link'};
+
+		const filename = path.basename(new URL(catalogEntry.href).pathname).replace(/\.meta4$/i, '');
+		const destPath = path.join(this.#dir, filename);
+		await this.#download(catalogEntry.href, destPath);
+		if (localMatch && localMatch.file !== destPath) await fs.promises.unlink(localMatch.file).catch(() => {});
+		return {name, status: 'updated', file: destPath};
+	}
+
+	/** Resolves a file path or catalog `name` to a local file path. */
+	async #resolveFile(fileOrName) {
+		if (fs.existsSync(fileOrName)) return fileOrName;
+		const local = await this.list();
+		const match = local.find(l => l.meta?.name === fileOrName);
+		if (!match) throw new Error(`ZIM not found locally: ${fileOrName}`);
+		return match.file;
+	}
+
+	catalog(search, opts) {
+		return zimCatalog(search, opts);
+	}
+
+	/** Checks whether a local ZIM has a newer version in the catalog, without downloading. */
+	async isOutdated(file) {
+		const meta = await this.#readMeta(file);
+		if (!meta?.name) return {file, upToDate: null, reason: 'no metadata'};
+		const catalogEntry = await zimCatalog(meta.name, this.#catalog);
+		if (!catalogEntry) return {file, upToDate: null, reason: 'missing from catalog'};
+		const remoteDate = catalogEntry.updated ? new Date(catalogEntry.updated) : null;
+		const localDate = meta.date ? new Date(meta.date) : null;
+		return {file, name: meta.name, upToDate: !!(remoteDate && localDate && remoteDate <= localDate), localDate, remoteDate};
+	}
+
+	/** Lists local `.zim` files with their parsed metadata (or `null` if unreadable). */
+	async list() {
+		await this.#ensureDir();
+		const files = (await fs.promises.readdir(this.#dir)).filter(f => f.endsWith('.zim'));
+		return Promise.all(files.map(async f => {
+			const file = path.join(this.#dir, f);
+			return {file, meta: await this.#readMeta(file)};
+		}));
+	}
+
+	/** Downloads/updates a single ZIM by direct href, matching against any existing local copy by name. */
+	async download(href, {force = false} = {}) {
+		await this.#ensureDir();
+		const {url: finalUrl} = await this.#resolveUrl(href);
+		const filename = path.basename(new URL(finalUrl).pathname).replace(/\.meta4$/i, '');
+		const name = filename.replace(/\.zim$/i, '').replace(/_\d{4}-\d{2}(?:_\d+)?$/, '');
+
+		const local = await this.list();
+		const localMatch = local.find(l => l.meta?.name === name) ?? null;
+		const catalogEntry = await zimCatalogInfo(name, this.#catalog) || {name, updated: null, href};
+		return this.#update(name, catalogEntry, localMatch, force);
+	}
+
+	/** Checks all local ZIMs against the catalog and updates any that are outdated. */
+	async updateAll({force = false}) {
+		const local = await this.list();
+		if (!local.length) return [];
+
+		const results = [];
+		for (const {file, meta} of local) {
+			if (!meta?.name) { results.push({file, status: 'skipped', reason: 'no metadata'}); continue; }
+			const catalogEntry = await zimCatalogInfo(meta.name, this.#catalog);
+			if (!catalogEntry) { results.push({name: meta.name, status: 'skipped', reason: 'missing from catalog'}); continue; }
+			try {
+				results.push(await this.#update(meta.name, catalogEntry, {file, meta}, force));
+			} catch (e) {
+				results.push({name: meta.name, status: 'error', reason: e.message});
+			}
+		}
+		return results;
+	}
+
+	/** Opens a `ZimReader` for a local ZIM, resolved by file path or catalog `name`. Caller must `.close()` it. */
+	async open(fileOrName) {
+		await this.#ensureDir();
+		const file = await this.#resolveFile(fileOrName);
+		return new ZimReader(file).open();
+	}
+
+	/** Fuzzy-searches titles across every local ZIM in the library, merging & re-ranking hits by score. */
+	async search(terms, {limit = 20, htmlOnly = true} = {}) {
+		const local = await this.list();
+		const perZim = await Promise.all(local.map(async ({file, meta}) => {
+			let reader;
+			try {
+				reader = await new ZimReader(file).open();
+				const hits = await reader.search(terms, {limit, htmlOnly});
+				return hits.map(h => ({...h, file, name: meta?.name}));
+			} catch {
+				return [];
+			} finally {
+				await reader?.close();
+			}
+		}));
+		return perZim.flat().toSorted((a, b) => b.score - a.score).slice(0, limit);
+	}
+}
--- a/src/reader.js
+++ b/src/reader.js
@@ -0,0 +1,235 @@
+'use strict';
+
+import fs from 'node:fs';
+import {decompress as lzmaDecompress} from 'lzma1';
+import {ZstdCodec} from 'zstd-codec';
+import {fuzzyMatch} from './utils.js';
+
+const HEADER_SIZE = 80;
+const NS_CONTENT = 'C';
+const NS_METADATA = 'M';
+const TITLE_SENTINEL = 0xffffffffffffffffn; // Indicator -> ZIM v6+ archives with no title
+
+let zstdStreamingPromise = null;
+
+/** Lazily initialised, shared Zstd streaming decompressor (handles unknown-size frames). */
+function getZstd() {
+	if (!zstdStreamingPromise) {
+		zstdStreamingPromise = new Promise(resolve => ZstdCodec.run(zstd => resolve(new zstd.Streaming())));
+	}
+	return zstdStreamingPromise;
+}
+
+/** Rebuilds a full 13-byte "alone" LZMA header from libzim's truncated 5-byte one (unknown size). */
+function toLzmaAloneStream(body) {
+	const header = Buffer.concat([body.subarray(0, 5), Buffer.alloc(8, 0xff)]);
+	return Buffer.concat([header, body.subarray(5)]);
+}
+
+/** Native, dependency-light reader for .zim archives. Supports zstd & LZMA cluster compression. */
+export class ZimReader {
+	#fd = null;
+	#header = null;
+	#mimeTypes = [];
+	#hasTitleListing = false;
+
+	get articleCount() { return this.#header?.articleCount ?? 0; }
+
+	constructor(path) {
+		this.path = path;
+	}
+
+	/** Full O(n) scan over the URL pointer list, used when there's no title index. */
+	async #allDirents() {
+		const dirents = [];
+		for (let i = 0; i < this.#header.articleCount; i++) {
+			dirents.push(await this.#readDirent(await this.#ptr64(this.#header.urlPtrPos, i)));
+		}
+		return dirents;
+	}
+
+	/** Binary search the URL pointer list for namespace+url. */
+	async #findByUrl(url, namespace) {
+		const key = namespace + url;
+		let lo = 0, hi = this.#header.articleCount - 1;
+		while (lo <= hi) {
+			const mid = (lo + hi) >> 1;
+			const dirent = await this.#readDirent(await this.#ptr64(this.#header.urlPtrPos, mid));
+			const dirKey = dirent.namespace + dirent.url;
+			const cmp = key < dirKey ? -1 : key > dirKey ? 1 : 0;
+			if (cmp === 0) return dirent;
+			if (cmp < 0) hi = mid - 1; else lo = mid + 1;
+		}
+		return null;
+	}
+
+	async #getBlob(clusterNumber, blobNumber) {
+		const start = await this.#ptr64(this.#header.clusterPtrPos, clusterNumber);
+		const isLast = clusterNumber === this.#header.clusterCount - 1;
+		const end = isLast
+			? (await fs.promises.stat(this.path)).size
+			: await this.#ptr64(this.#header.clusterPtrPos, clusterNumber + 1);
+
+		const raw = await this.#read(start, end - start);
+		const compType = raw[0] & 0x0f;
+		const extended = (raw[0] & 0x10) !== 0;
+		const body = raw.subarray(1);
+
+		let data;
+		if (compType <= 1) data = body;
+		else if (compType === 4) data = Buffer.from(lzmaDecompress(toLzmaAloneStream(body)));
+		else if (compType === 5) data = Buffer.from((await getZstd()).decompress(new Uint8Array(body)));
+		else throw new Error(`Unsupported cluster compression type: ${compType}`);
+		if (!data) throw new Error(`Cluster ${clusterNumber} failed to decompress (compType ${compType})`);
+
+		const readPtr = i => extended ? Number(data.readBigUInt64LE(i * 8)) : data.readUInt32LE(i * 4);
+		return data.subarray(readPtr(blobNumber), readPtr(blobNumber + 1));
+	}
+
+	async #ptr64(base, index) {
+		return Number((await this.#read(base + index * 8, 8)).readBigUInt64LE(0));
+	}
+
+	async #read(pos, length) {
+		const buf = Buffer.alloc(length);
+		await this.#fd.read(buf, 0, length, pos);
+		return buf;
+	}
+
+	async #readHeader() {
+		const b = await this.#read(0, HEADER_SIZE);
+		const titlePtrRaw = b.readBigUInt64LE(40);
+		this.#hasTitleListing = titlePtrRaw !== TITLE_SENTINEL;
+		this.#header = {
+			articleCount: b.readUInt32LE(24),
+			clusterCount: b.readUInt32LE(28),
+			urlPtrPos: Number(b.readBigUInt64LE(32)),
+			titlePtrPos: this.#hasTitleListing ? Number(titlePtrRaw) : null,
+			clusterPtrPos: Number(b.readBigUInt64LE(48)),
+			mimeListPos: Number(b.readBigUInt64LE(56)),
+			mainPage: b.readUInt32LE(64),
+		};
+	}
+
+	async #readMimeTypes() {
+		let pos = this.#header.mimeListPos, str = '';
+		for (;;) {
+			str += (await this.#read(pos, 1024)).toString('binary');
+			const end = str.indexOf('\0\0');
+			if (end !== -1) { str = str.slice(0, end + 1); break; }
+			pos += 1024;
+		}
+		this.#mimeTypes = str.split('\0').filter(Boolean);
+	}
+
+	/** Directory entry (article record) at byte `offset`, growing the read window as needed. */
+	async #readDirent(offset) {
+		for (let size = 512; ; size *= 2) {
+			const buf = await this.#read(offset, size);
+			let o = 0;
+			const mimetype = buf.readUInt16LE(o); o += 2;
+			o += 1; // extraLen, unused
+			const namespace = String.fromCharCode(buf.readUInt8(o)); o += 1;
+			o += 4; // revision, unused
+
+			let redirectIndex = null, cluster = null, blob = null;
+			if (mimetype === 0xffff) { redirectIndex = buf.readUInt32LE(o); o += 4; }
+			else { cluster = buf.readUInt32LE(o); o += 4; blob = buf.readUInt32LE(o); o += 4; }
+
+			const urlEnd = buf.indexOf(0, o);
+			if (urlEnd === -1) continue;
+			const titleEnd = buf.indexOf(0, urlEnd + 1);
+			if (titleEnd === -1) continue;
+
+			const url = buf.toString('utf8', o, urlEnd);
+			const title = buf.toString('utf8', urlEnd + 1, titleEnd) || url;
+			return {mimetype, namespace, redirectIndex, cluster, blob, url, title};
+		}
+	}
+
+	async #resolveRedirect(dirent) {
+		if (dirent.mimetype !== 0xffff) return dirent;
+		return this.#readDirent(await this.#ptr64(this.#header.urlPtrPos, dirent.redirectIndex));
+	}
+
+	/** Narrows to dirents near the term's alphabetical position in the title index. */
+	async #titleIndexCandidates(term) {
+		const q = term.toLowerCase();
+		let lo = 0, hi = this.#header.articleCount - 1;
+		while (lo < hi) {
+			const mid = (lo + hi) >> 1;
+			const urlIdx = (await this.#read(this.#header.titlePtrPos + mid * 4, 4)).readUInt32LE(0);
+			const dirent = await this.#readDirent(await this.#ptr64(this.#header.urlPtrPos, urlIdx));
+			if (dirent.title.toLowerCase() < q) lo = mid + 1; else hi = mid;
+		}
+		// Widen around the prefix match since fuzzy scoring isn't purely alphabetical.
+		const start = Math.max(0, lo - 50), end = Math.min(this.#header.articleCount, lo + 200);
+		const dirents = [];
+		for (let i = start; i < end; i++) {
+			const urlIdx = (await this.#read(this.#header.titlePtrPos + i * 4, 4)).readUInt32LE(0);
+			dirents.push(await this.#readDirent(await this.#ptr64(this.#header.urlPtrPos, urlIdx)));
+		}
+		return dirents;
+	}
+
+	async close() {
+		if (this.#fd) await this.#fd.close();
+		this.#fd = null;
+	}
+
+	/** Reads an 'M' namespace metadata value (e.g. Name, Date, Title). Returns null if missing. */
+	async metadata(key) {
+		const page = await this.readPage(key, NS_METADATA);
+		return page ? page.data.toString('utf8') : null;
+	}
+
+	/** Opens the archive and parses its header + mimetype list. */
+	async open() {
+		this.#fd = await fs.promises.open(this.path, 'r');
+		await this.#readHeader();
+		await this.#readMimeTypes();
+		return this;
+	}
+
+	/** Read a page's content by URL. Returns `{mimetype, data}` or `null` if not found. */
+	async readPage(url, namespace = NS_CONTENT) {
+		let dirent = await this.#findByUrl(url, namespace);
+		if (!dirent) return null;
+		dirent = await this.#resolveRedirect(dirent);
+		const data = await this.#getBlob(dirent.cluster, dirent.blob);
+		return {mimetype: this.#mimeTypes[dirent.mimetype] || 'application/octet-stream', data};
+	}
+
+	/** Reads the archive's designated main/landing page, if one is set. */
+	async mainPage() {
+		if (this.#header.mainPage === 0xffffffff) return null;
+		let dirent = await this.#readDirent(await this.#ptr64(this.#header.urlPtrPos, this.#header.mainPage));
+		dirent = await this.#resolveRedirect(dirent);
+		const data = await this.#getBlob(dirent.cluster, dirent.blob);
+		return {mimetype: this.#mimeTypes[dirent.mimetype] || 'application/octet-stream', data, url: dirent.url};
+	}
+
+	/**
+	 * Fuzzy-ranked title search. Accepts comma-separated `terms` the same way the
+	 * catalog search does. Uses the sorted title index when present (binary search
+	 * narrows the candidate window); falls back to a full linear scan otherwise
+	 * (common on ZIM v6+/zimit-generated archives with no title index).
+	 */
+	async search(terms, {limit = 20, htmlOnly = true} = {}) {
+		const termList = String(terms).split(',').map(t => t.trim()).filter(Boolean);
+		if (!termList.length) return [];
+
+		const candidates = this.#hasTitleListing
+			? await this.#titleIndexCandidates(termList[0])
+			: await this.#allDirents();
+
+		const scored = [];
+		for (const dirent of candidates) {
+			if (dirent.namespace !== NS_CONTENT) continue;
+			if (htmlOnly && !(this.#mimeTypes[dirent.mimetype] || '').startsWith('text/html')) continue;
+			const {max} = fuzzyMatch(dirent.title, ...termList);
+			scored.push({url: dirent.url, title: dirent.title, namespace: dirent.namespace, score: max});
+		}
+		return scored.toSorted((a, b) => b.score - a.score).slice(0, limit);
+	}
+}
--- a/src/utils.js
+++ b/src/utils.js
@@ -0,0 +1,32 @@
+export function levenshtein(a, b) {
+	const m = a.length, n = b.length;
+	if (!m) return n;
+	if (!n) return m;
+	const dp = Array.from({length: m + 1}, (_, i) => [i, ...Array(n).fill(0)]);
+	for (let j = 0; j <= n; j++) dp[0][j] = j;
+	for (let i = 1; i <= m; i++) {
+		for (let j = 1; j <= n; j++) {
+			dp[i][j] = a[i - 1] === b[j - 1]
+				? dp[i - 1][j - 1]
+				: 1 + Math.min(dp[i - 1][j - 1], dp[i - 1][j], dp[i][j - 1]);
+		}
+	}
+	return dp[m][n];
+}
+
+/** Normalized similarity in [0,1]: 1 - editDistance / maxLength. */
+export function similarity(a, b) {
+	a = a.toLowerCase(); b = b.toLowerCase();
+	return 1 - levenshtein(a, b) / Math.max(a.length, b.length, 1);
+}
+
+/** Compares `target` against one or more search terms; returns avg/max/per-term similarity. */
+export function fuzzyMatch(target, ...terms) {
+	if (!terms.length) throw new Error('Requires at least 1 term to compare');
+	const similarities = terms.map(t => similarity(target, t));
+	return {
+		avg: similarities.reduce((acc, s) => acc + s, 0) / similarities.length,
+		max: Math.max(...similarities),
+		similarities,
+	};
+}