Create search index cache and return more meta

package.json

@@ -1,6 +1,6 @@
 {
 	"name": "@ztimson/zim-utils",
-	"version": "0.1.1",
+	"version": "0.2.0",
 	"description": "Native, dependency-light ZIM archive reader/searcher and Kiwix catalog downloader for Node.js",
 	"author": "Zak Timson",
 	"license": "MIT",

src/catalog.js

@@ -5,12 +5,13 @@ export const CATALOG_URL = 'https://library.kiwix.org/catalog/v2/entries';
 
 const PAGE_SIZE = 100;
 
-/** Parses `<entry>` blocks out of a Kiwix catalog OPDS XML response. */
-function parseEntries(xml) {
+function parseEntries(xml, baseUrl = 'https://library.kiwix.org') {
 	const blocks = xml.match(/<entry>[\s\S]*?<\/entry>/g) || [];
 	return blocks.map(b => {
 		const grab = re => (b.match(re) || [])[1] || '';
 		const linkMatch = b.match(/<link[^>]*type=["']application\/x-zim[^"']*["'][^>]*href=["']([^"']+)["']/);
+		const iconMatch = b.match(/<link[^>]*rel=["']http:\/\/opds-spec\.org\/image\/thumbnail["'][^>]*href=["']([^"']+)["']/)
+			|| b.match(/<link[^>]*type=["']image\/[^"']*["'][^>]*href=["']([^"']+)["']/);
 		let tags = grab(/<tags>([^<]*)<\/tags>/);
 		if(tags) tags = tags.split(';');
 		return {
@@ -28,6 +29,7 @@ function parseEntries(xml) {
 			articleCount: Number(grab(/<articleCount>([^<]*)<\/articleCount>/)) || 0,
 			sizeMb: linkMatch ? (Number((b.match(/length=["'](\d+)["']/) || [])[1] || 0) / 1024 / 1024).toFixed(1) : '?',
 			href: linkMatch ? linkMatch[1] : null,
+			icon: iconMatch ? new URL(iconMatch[1], baseUrl).href : null,
 		};
 	});
 }

src/manager.js

@@ -4,6 +4,7 @@ import {pipeline} from 'node:stream/promises';
 import {Readable} from 'node:stream';
 import {ZimReader} from './reader.js';
 import {zimCatalog, zimCatalogInfo, CATALOG_URL} from './catalog.js';
+import {fuzzyMatch, titleFromUrl} from './utils.js';
 
 /** Manages a local directory of ZIM archives: listing, update checks, downloads, and reading. */
 export class ZimManager {
@@ -44,11 +45,7 @@ export class ZimManager {
 		let reader;
 		try {
 			reader = await new ZimReader(filepath).open();
-			return {
-				name: await reader.metadata('Name'),
-				date: await reader.metadata('Date'),
-				title: await reader.metadata('Title'),
-			};
+			return await reader.metadata();
 		} catch {
 			return null;
 		} finally {
@@ -58,7 +55,7 @@ export class ZimManager {
 
 	async #update(name, catalogEntry, localMatch, force) {
 		const remoteDate = catalogEntry.updated ? new Date(catalogEntry.updated) : null;
-		const localDate = localMatch?.meta?.date ? new Date(localMatch.meta.date) : null;
+		const localDate = localMatch?.meta?.updated ?? null;
 		if (!force && localMatch && remoteDate && localDate && remoteDate <= localDate)
 			return {name, status: 'skipped', reason: 'up to date'};
 		if (!catalogEntry.href) return {name, status: 'skipped', reason: 'missing download link'};
@@ -66,15 +63,17 @@ export class ZimManager {
 		const filename = path.basename(new URL(catalogEntry.href).pathname).replace(/\.meta4$/i, '');
 		const destPath = path.join(this.#dir, filename);
 		await this.#download(catalogEntry.href, destPath);
-		if (localMatch && localMatch.file !== destPath) await fs.promises.unlink(localMatch.file).catch(() => {});
+		if (localMatch && localMatch.file !== destPath) await new ZimReader(localMatch.file).delete().catch(() => {});
 		return {name, status: 'updated', file: destPath};
 	}
 
 	/** Resolves a file path or catalog `name` to a local file path. */
 	async #resolveFile(fileOrName) {
 		if (fs.existsSync(fileOrName)) return fileOrName;
+		const joined = path.join(this.#dir, fileOrName);
+		if (fs.existsSync(joined)) return joined;
 		const local = await this.list();
-		const match = local.find(l => l.meta?.name === fileOrName);
+		const match = local.find(l => l.meta?.name === fileOrName || path.basename(l.file) === fileOrName);
 		if (!match) throw new Error(`ZIM not found locally: ${fileOrName}`);
 		return match.file;
 	}
@@ -83,6 +82,12 @@ export class ZimManager {
 		return zimCatalog(search, opts);
 	}
 
+	async delete(fileOrName) {
+		const file = await this.#resolveFile(fileOrName);
+		await new ZimReader(file).delete();
+		return {file, status: 'deleted'};
+	}
+
 	/** Checks whether a local ZIM has a newer version in the catalog, without downloading. */
 	async isOutdated(file) {
 		const meta = await this.#readMeta(file);
@@ -90,7 +95,7 @@ export class ZimManager {
 		const catalogEntry = await zimCatalog(meta.name, this.#catalog);
 		if (!catalogEntry) return {file, upToDate: null, reason: 'missing from catalog'};
 		const remoteDate = catalogEntry.updated ? new Date(catalogEntry.updated) : null;
-		const localDate = meta.date ? new Date(meta.date) : null;
+		const localDate = meta.updated ?? null;
 		return {file, name: meta.name, upToDate: !!(remoteDate && localDate && remoteDate <= localDate), localDate, remoteDate};
 	}
 
@@ -158,6 +163,6 @@ export class ZimManager {
 				await reader?.close();
 			}
 		}));
-		return perZim.flat().toSorted((a, b) => b.score - a.score).slice(0, limit);
+		return perZim.flat().filter(a => a.score > 0).toSorted((a, b) => b.score - a.score).slice(0, limit);
 	}
 }

src/reader.js

@@ -1,10 +1,12 @@
 'use strict';
 
 import fs from 'node:fs';
+import path from 'node:path';
 import {decompress as lzmaDecompress} from 'lzma1';
 import {ZstdCodec} from 'zstd-codec';
-import {fuzzyMatch} from './utils.js';
+import {fuzzyMatch, titleFromUrl} from './utils.js';
 
+const INDEX_VERSION = 1;
 const HEADER_SIZE = 80;
 const NS_CONTENT = 'C';
 const NS_METADATA = 'M';
@@ -32,6 +34,7 @@ export class ZimReader {
 	#header = null;
 	#mimeTypes = [];
 	#hasTitleListing = false;
+	#index;
 
 	get articleCount() { return this.#header?.articleCount ?? 0; }
 
@@ -86,6 +89,13 @@ export class ZimReader {
 		return data.subarray(readPtr(blobNumber), readPtr(blobNumber + 1));
 	}
 
+	async #icon(size = 48) {
+		const page = await this.readPage(`Illustration_${size}x${size}@1`, NS_METADATA)
+			|| await this.readPage('Favicon', NS_METADATA);
+		if (!page) return null;
+		return `data:${page.mimetype};base64,${page.data.toString('base64')}`;
+	}
+
 	async #ptr64(base, index) {
 		return Number((await this.#read(base + index * 8, 8)).readBigUInt64LE(0));
 	}
@@ -172,15 +182,89 @@ export class ZimReader {
 		return dirents;
 	}
 
+	/** Path of the cached word index, kept in an `index/` subdirectory next to the archive. */
+	#indexPath() {
+		return path.join(path.dirname(this.path), 'index', `${path.basename(this.path)}.idx.json`);
+	}
+
+	/** Builds (or loads a cached) word -> entry index, avoiding a re-scan on every search. */
+	async #wordIndex() {
+		if (this.#index) return this.#index;
+		const cachePath = this.#indexPath();
+		const stat = await fs.promises.stat(this.path);
+
+		if (fs.existsSync(cachePath)) {
+			try {
+				const cached = JSON.parse(await fs.promises.readFile(cachePath, 'utf8'));
+				// Rebuild if stale: index predates the archive's current mtime (e.g. a re-download).
+				if (cached.version === INDEX_VERSION && cached.mtimeMs >= stat.mtimeMs) {
+					return (this.#index = cached);
+				}
+			} catch {}
+		}
+
+		// Expensive full scan — only happens once per archive (or after it changes).
+		const dirents = await this.#allDirents();
+		const entries = [];
+		const words = new Map(); // word -> [entryIndex, ...]
+		for (const d of dirents) {
+			if (d.namespace !== NS_CONTENT) continue;
+			const idx = entries.length;
+			entries.push({url: d.url, title: d.title, mimetype: d.mimetype});
+			const text = `${d.title} ${titleFromUrl(d.url)}`.toLowerCase();
+			for (const w of text.split(/\W+/).filter(Boolean)) {
+				if (!words.has(w)) words.set(w, []);
+				words.get(w).push(idx);
+			}
+		}
+
+		const index = {version: INDEX_VERSION, size: stat.size, mtimeMs: stat.mtimeMs, entries, words: Object.fromEntries(words)};
+		await fs.promises.mkdir(path.dirname(cachePath), {recursive: true});
+		await fs.promises.writeFile(cachePath, JSON.stringify(index));
+		return (this.#index = index);
+	}
+
 	async close() {
 		if (this.#fd) await this.#fd.close();
 		this.#fd = null;
 	}
 
+	/** Deletes the zim archive and its cached index (if any). Safe to call on unopened readers. */
+	async delete() {
+		await this.close();
+		this.#index = null;
+		await fs.promises.rm(this.path, {force: true});
+		await fs.promises.rm(this.#indexPath(), {force: true});
+	}
+
 	/** Reads an 'M' namespace metadata value (e.g. Name, Date, Title). Returns null if missing. */
 	async metadata(key) {
-		const page = await this.readPage(key, NS_METADATA);
-		return page ? page.data.toString('utf8') : null;
+		const get = async key => {
+			const page = await this.readPage(key, NS_METADATA);
+			return page ? page.data.toString('utf8') : null;
+		};
+		if(key) return get(key);
+
+		const [title, creator, publisher, date, description, language, name, tags] = await Promise.all(
+			['Title', 'Creator', 'Publisher', 'Date', 'Description', 'Language', 'Name', 'Tags'].map(get)
+		);
+		return {
+			id: name,
+			title,
+			updated: date ? new Date(date) : null,
+			summary: description,
+			language,
+			name,
+			category: tags ? tags.split(';')[0] || '' : '',
+			tags: tags ? tags.split(';') : [],
+			mediaCount: 0,
+			author: creator,
+			publisher,
+			articleCount: this.articleCount,
+			sizeMb: ((await fs.promises.stat(this.path)).size / 1024 / 1024).toFixed(1),
+			href: null,
+			icon: await this.#icon(),
+		};
 	}
 
 	/** Opens the archive and parses its header + mimetype list. */
@@ -206,7 +290,7 @@ export class ZimReader {
 		let dirent = await this.#readDirent(await this.#ptr64(this.#header.urlPtrPos, this.#header.mainPage));
 		dirent = await this.#resolveRedirect(dirent);
 		const data = await this.#getBlob(dirent.cluster, dirent.blob);
-		return {mimetype: this.#mimeTypes[dirent.mimetype] || 'application/octet-stream', data, url: dirent.url};
+		return {mimetype: this.#mimeTypes[dirent.mimetype] || 'application/octet-stream', data, url: dirent.url, namespace: dirent.namespace};
 	}
 
 	/**
@@ -219,17 +303,29 @@ export class ZimReader {
 		const termList = String(terms).split(',').map(t => t.trim()).filter(Boolean);
 		if (!termList.length) return [];
 
-		const candidates = this.#hasTitleListing
-			? await this.#titleIndexCandidates(termList[0])
-			: await this.#allDirents();
+		let candidates;
+		if (this.#hasTitleListing && this.#header.articleCount > 5000) {
+			candidates = await this.#titleIndexCandidates(termList[0]);
+		} else {
+			const {entries, words} = await this.#wordIndex();
+			// Pull candidates from postings of any word that starts with (or contains) the search term.
+			const q = termList[0].toLowerCase();
+			const idxSet = new Set();
+			for (const [word, postings] of Object.entries(words)) {
+				if (word.includes(q) || q.includes(word)) postings.forEach(i => idxSet.add(i));
+			}
+			candidates = [...idxSet].map(i => ({...entries[i], namespace: NS_CONTENT}));
+		}
 
 		const scored = [];
 		for (const dirent of candidates) {
-			if (dirent.namespace !== NS_CONTENT) continue;
 			if (htmlOnly && !(this.#mimeTypes[dirent.mimetype] || '').startsWith('text/html')) continue;
-			const {max} = fuzzyMatch(dirent.title, ...termList);
-			scored.push({url: dirent.url, title: dirent.title, namespace: dirent.namespace, score: max});
+			const urlTitle = titleFromUrl(dirent.url);
+			const titleScore = fuzzyMatch(dirent.title, ...termList).max;
+			const urlScore = fuzzyMatch(urlTitle, ...termList).max;
+			scored.push({url: dirent.url, title: dirent.title.length > urlTitle.length ? dirent.title : urlTitle, namespace: NS_CONTENT, score: Math.max(titleScore, urlScore)});
 		}
-		return scored.toSorted((a, b) => b.score - a.score).slice(0, limit);
+		const {id, summary, mediaCount, articleCount, sizeMb, href, ...meta} = await this.metadata();
+		return scored.filter(a => a.score > 0).toSorted((a, b) => b.score - a.score).slice(0, limit).map(a => ({...meta, ...a}));
 	}
 }

src/utils.js

@@ -20,13 +20,36 @@ export function similarity(a, b) {
 	return 1 - levenshtein(a, b) / Math.max(a.length, b.length, 1);
 }
 
+function scoreAgainst(text, term) {
+	if (text.includes(term)) return 1 - (text.length - term.length) / text.length * 0.3;
+	if (!text.length || !term.length || text[0] !== term[0]) return 0;
+	const dist = levenshtein(text, term);
+	const maxAllowed = Math.max(1, Math.ceil(term.length * 0.34));
+	if (dist > maxAllowed) return 0;
+	return 1 - dist / Math.max(text.length, term.length);
+}
+
 /** Compares `target` against one or more search terms; returns avg/max/per-term similarity. */
 export function fuzzyMatch(target, ...terms) {
 	if (!terms.length) throw new Error('Requires at least 1 term to compare');
-	const similarities = terms.map(t => similarity(target, t));
+	const lowerTarget = String(target).toLowerCase();
+	const words = lowerTarget.split(/\W+/).filter(Boolean);
+
+	const similarities = terms.map(term => {
+		const t = term.toLowerCase();
+		return Math.max(scoreAgainst(lowerTarget, t), ...words.map(w => scoreAgainst(w, t)));
+	});
+
 	return {
 		avg: similarities.reduce((acc, s) => acc + s, 0) / similarities.length,
 		max: Math.max(...similarities),
 		similarities,
 	};
 }
+
+/** Derives a readable pseudo-title from a URL's last path segment, e.g. ".../diannes-southwest-salad/" -> "Diannes Southwest Salad". */
+export function titleFromUrl(url) {
+	const slug = String(url).replace(/\/$/, '').split('/').pop() || url;
+	const clean = slug.replace(/\.(zim|meta4|html?|md)$/i, '').replace(/[-_.]+/g, ' ');
+	return clean.replace(/\b\w/g, c => c.toUpperCase());
+}