Better search results

src/server.js

@@ -7,10 +7,12 @@ import fs from 'node:fs';
 import path from 'node:path';
 import {fileURLToPath} from 'node:url';
 import {fromXml, makeArray} from '@ztimson/utils';
+import {titleVocabulary} from './reader.js';
+import {diversify, rerank, rrfMerge, suggestCorrection} from './search.js';
 
 const execFileAsync = promisify(execFile);
 const __dirname = path.dirname(fileURLToPath(import.meta.url));
-const DEFAULT_BIN_DIR = path.join(__dirname, '..', 'bin'); // npm package root/bin - where bin/install.js drops the kiwix-tools binaries
+const DEFAULT_BIN_DIR = path.join(__dirname, '..', 'bin');
 const READY_TIMEOUT = 10_000;
 const READY_POLL_INTERVAL = 100;
 const WATCH_DEBOUNCE = 300;
@@ -26,9 +28,10 @@ function findFreePort() {
 	});
 }
 
-/** Owns a kiwix-serve process's full lifecycle: library.xml, start/stop/reload, content + search access.
- * Watches its own directory for .zim files appearing/disappearing (from *any* writer - itself, a remote-attached
- * ZimManager, a script, whatever) and keeps library.xml + the running kiwix-serve process in sync automatically. */
+function tokenize(terms) {
+	return String(terms).split(/[,\s]+/).map(t => t.trim()).filter(Boolean);
+}
+
 export class KiwixServer {
 	static #empty = '<?xml version="1.0" encoding="UTF-8" ?>\n<library version="20110515"></library>\n';
 
@@ -38,7 +41,7 @@ export class KiwixServer {
 	#binDir;
 	#libraryPath;
 	#child = null;
-	#remote; // baseUrl string if attached to an externally-managed kiwix-serve, else null
+	#remote;
 	#watcher = null;
 	#watchTimer = null;
 
@@ -46,11 +49,6 @@ export class KiwixServer {
 	get running() { return !!this.#remote || !!this.#child; }
 	get baseUrl() { return this.#remote || (this.#child ? `http://${this.#host}:${this.#port}` : null); }
 
-	/** @param {{port?: number, host?: string, binDir?: string, url?: string}} [opts]
-	 * url: attach to an already-running kiwix-serve (e.g. one started elsewhere in your codebase) instead of
-	 * spawning/owning one - start/stop become no-ops, and library.xml is read over HTTP instead of disk.
-	 * Whichever instance *does* own the process is responsible for watching the shared directory; an attached
-	 * instance doesn't need its own watcher; it just reads whatever the owner already reloaded. */
 	constructor(dir, {port, host = '127.0.0.1', binDir = DEFAULT_BIN_DIR, url} = {}) {
 		this.#dir = dir;
 		this.#host = host;
@@ -58,25 +56,29 @@ export class KiwixServer {
 		this.#binDir = binDir;
 		this.#libraryPath = path.join(dir, 'library.xml');
 		this.#remote = url ? url.replace(/\/$/, '') : null;
-		if (!this.#remote) this.#ensureLocalStore();
+
+		if(!this.#remote) this.#ensureLocalStore();
 	}
 
 	#ensureLocalStore() {
 		fs.mkdirSync(this.#dir, {recursive: true});
-		if (!fs.existsSync(this.#libraryPath)) fs.writeFileSync(this.#libraryPath, KiwixServer.#empty);
+
+		if(!fs.existsSync(this.#libraryPath))
+			fs.writeFileSync(this.#libraryPath, KiwixServer.#empty);
 	}
 
 	#assertRunning() {
-		if (!this.running) throw new Error('KiwixServer is not running - call start() first');
+		if(!this.running) throw new Error('KiwixServer is not running - call start() first');
 	}
 
 	#bin(name) {
 		return path.join(this.#binDir, process.platform === 'win32' ? `${name}.exe` : name);
 	}
 
-	/** Reads library.xml from disk if we own the server, or over HTTP if attached to a remote one. */
 	async #fetchLibraryXml() {
-		if (!this.#remote) return fs.promises.readFile(this.#libraryPath, 'utf8').catch(() => '');
+		if(!this.#remote)
+			return fs.promises.readFile(this.#libraryPath, 'utf8').catch(() => '');
+
 		try {
 			const res = await fetch(`${this.#remote}/library.xml`);
 			return res.ok ? await res.text() : '';
@@ -85,44 +87,55 @@ export class KiwixServer {
 		}
 	}
 
-	/** Rebuilds library.xml from scratch by scanning `dir` for .zim files - no-op if attached to a remote server.
-	 * Rewriting the file bumps its mtime, which kiwix-serve (started with --monitorLibrary) picks up on its own
-	 * and hot-reloads without needing a restart. */
 	async #rebuildLibrary() {
-		if (this.#remote) return;
+		if(this.#remote) return;
+
 		await fs.promises.rm(this.#libraryPath, {force: true});
+
 		const files = await this.#zimFiles();
-		if (!files.length) return fs.promises.writeFile(this.#libraryPath, KiwixServer.#empty);
-		for (const f of files) await execFileAsync(this.#bin('kiwix-manage'), [this.#libraryPath, 'add', path.join(this.#dir, f)]);
+
+		if(!files.length)
+			return fs.promises.writeFile(this.#libraryPath, KiwixServer.#empty);
+
+		for(const file of files)
+			await execFileAsync(this.#bin('kiwix-manage'), [
+				this.#libraryPath,
+				'add',
+				path.join(this.#dir, file),
+			]);
 	}
 
 	async #waitUntilReady() {
 		const deadline = Date.now() + READY_TIMEOUT;
-		while (Date.now() < deadline) {
+
+		while(Date.now() < deadline) {
 			try {
 				await fetch(`http://${this.#host}:${this.#port}/`);
 				return;
 			} catch {}
+
 			await new Promise(r => setTimeout(r, READY_POLL_INTERVAL));
 		}
+
 		throw new Error('kiwix-serve did not become ready in time');
 	}
 
 	async #zimFiles() {
-		return (await fs.promises.readdir(this.#dir).catch(() => [])).filter(f => f.endsWith('.zim'));
+		return (await fs.promises.readdir(this.#dir).catch(() => []))
+			.filter(f => f.endsWith('.zim'));
 	}
 
-	/** Watches `dir` for .zim files being added/removed by anyone (this process, a remote-attached ZimManager,
-	 * a manual copy) and debounces a library rebuild so kiwix-serve's --monitorLibrary picks it up. Only runs
-	 * while we own the process - an attached instance has nothing local to watch on behalf of. */
 	#watchDir() {
 		this.#watcher?.close();
+
 		this.#watcher = fs.watch(this.#dir, (_event, filename) => {
-			if (!filename?.endsWith('.zim')) return;
+			if(!filename?.endsWith('.zim')) return;
+
 			clearTimeout(this.#watchTimer);
 			this.#watchTimer = setTimeout(() => this.#rebuildLibrary().catch(() => {}), WATCH_DEBOUNCE);
 		});
-		this.#watcher.on('error', () => {}); // e.g. dir removed out from under us - just stop watching, don't crash the process
+
+		this.#watcher.on('error', () => {});
 	}
 
 	#unwatchDir() {
@@ -132,23 +145,31 @@ export class KiwixServer {
 		this.#watcher = null;
 	}
 
-	/** Rebuilds library.xml and starts kiwix-serve. Resolves once the server is responding.
-	 * Starts with --monitorLibrary so the process reloads itself whenever library.xml's mtime changes -
-	 * no restart needed for updates after this. */
 	async start() {
-		if (this.#remote || this.#child) return;
+		if(this.#remote || this.#child) return;
+
 		await fs.promises.mkdir(this.#dir, {recursive: true});
 		await this.#rebuildLibrary();
 		this.#port ??= await findFreePort();
 
-		this.#child = spawn(this.#bin('kiwix-serve'),
-			['--library', '--monitorLibrary', '-i', this.#host, '-p', String(this.#port), this.#libraryPath],
-			{stdio: 'ignore'});
-		this.#child.on('exit', () => { this.#child = null; this.#unwatchDir(); });
+		this.#child = spawn(this.#bin('kiwix-serve'), [
+			'--library',
+			'--monitorLibrary',
+			'-i',
+			this.#host,
+			'-p',
+			String(this.#port),
+			this.#libraryPath,
+		], {stdio: 'ignore'});
+
+		this.#child.on('exit', () => {
+			this.#child = null;
+			this.#unwatchDir();
+		});
 
 		try {
 			await this.#waitUntilReady();
-		} catch (e) {
+		} catch(e) {
 			await this.stop();
 			throw e;
 		}
@@ -156,15 +177,18 @@ export class KiwixServer {
 		this.#watchDir();
 	}
 
-	/** Gracefully stops kiwix-serve, if we own it. No-op if attached to a remote instance. */
 	async stop() {
 		this.#unwatchDir();
-		if (this.#remote || !this.#child) return;
+
+		if(this.#remote || !this.#child) return;
+
 		const child = this.#child;
+
 		await new Promise(resolve => {
 			child.once('exit', resolve);
 			child.kill('SIGTERM');
 		});
+
 		this.#child = null;
 	}
 
@@ -173,24 +197,23 @@ export class KiwixServer {
 		await this.start();
 	}
 
-	/** Forces an immediate library rebuild rather than waiting for the directory watcher's debounce to fire.
-	 * Not required for correctness (the watcher does this automatically for any writer), just a manual way to
-	 * skip the ~300ms wait when you already know something changed. No-op if attached to a remote instance -
-	 * whoever owns that process already reloads itself. */
 	async reload() {
-		if (this.#remote || !this.#child) return;
+		if(this.#remote || !this.#child) return;
 		await this.#rebuildLibrary();
 	}
 
-	/** Local catalog listing - same flat shape as the online catalog (catalog.js), plus a `file` field. */
 	async list() {
 		this.#assertRunning();
+
 		const xml = await this.#fetchLibraryXml();
-		if (!xml) return [];
+		if(!xml) return [];
+
 		const entries = fromXml(xml);
+
 		return makeArray(entries?.library?.book || []).map(e => {
 			const tags = e.tags.split(';');
 			const name = e.path.replace('.zim', '');
+
 			return {
 				id: e.id,
 				title: e.title,
@@ -205,6 +228,7 @@ export class KiwixServer {
 				publisher: e.publisher,
 				articleCount: +e.articleCount || 0,
 				sizeMb: +(Number(e.size) / 1024).toFixed(1) || 0,
+				file: e.path,
 				href: name,
 				icon: `data:${e.faviconMimetype || 'image/png'};base64,${e.favicon}`,
 				viewer: `${this.baseUrl}/content/${name}`,
@@ -212,58 +236,133 @@ export class KiwixServer {
 		});
 	}
 
-	/** Splits a href ("zim/path/to/page") or a full content/viewer URL into {zim, path}. */
 	#splitHref(href) {
 		const clean = href.replace(`${this.baseUrl}/content/`, '').replace(/^\/+/, '');
 		const [zim, ...rest] = clean.split('/');
 		return {zim, path: rest.join('/')};
 	}
 
-	/** Builds a kiwix-serve content URL from a href (as returned by list()/search()), or from an already-built content/viewer URL. */
 	link(href) {
 		this.#assertRunning();
+
 		const {zim, path} = this.#splitHref(href);
+
 		return `${this.baseUrl}/content/${zim}${path ? '/' + path : ''}`;
 	}
 
-	/** Fetches a single asset's raw bytes straight from kiwix-serve. */
 	async raw(href) {
 		const res = await fetch(this.link(href));
+
 		if(!res.ok) return null;
-		return {mimetype: res.headers.get('content-type'), data: Buffer.from(await res.arrayBuffer())};
+
+		return {
+			mimetype: res.headers.get('content-type'),
+			data: Buffer.from(await res.arrayBuffer()),
+		};
 	}
 
-	/** Fulltext search across every local ZIM via kiwix-serve's own xapian index */
-	async search(terms, limit = 20) {
+	async #rawSearch(termList, scoped, bookMap, limit) {
+		const groups = new Map();
+
+		for(const book of scoped) {
+			const lang = book.language || '';
+			(groups.get(lang) ?? groups.set(lang, []).get(lang)).push(book);
+		}
+
+		const perGroup = await Promise.all([...groups.values()].map(async group => {
+			const params = new URLSearchParams({
+				pattern: termList.join(' '),
+				format: 'xml',
+				pageLength: String(limit),
+			});
+
+			for(const book of group)
+				params.append('books.name', book.name);
+
+			const res = await fetch(`${this.baseUrl}/search?${params}`);
+			if(!res.ok) return [];
+
+			const found = fromXml(await res.text())?.rss?.channel?.item || [];
+
+			return found.map(hit => {
+				const book = bookMap.get(hit.book.title);
+				if(!book) return null;
+
+				const prefix = `/content/${book.href}/`;
+				const page = hit.link.startsWith(prefix)
+					? hit.link.slice(prefix.length)
+					: hit.link.replace(/^\/+/, '');
+
+				return {
+					id: book.id,
+					title: hit.title,
+					page,
+					name: book.name,
+					publisher: book.publisher,
+					href: `${book.href}/${page}`,
+					icon: book.icon,
+					viewer: this.baseUrl + hit.link,
+					summary: hit.description,
+					xapianScore: +hit.score || 0,
+				};
+			}).filter(Boolean).sort((a, b) => b.xapianScore - a.xapianScore);
+		}));
+
+		return rrfMerge(perGroup);
+	}
+
+	/** Fulltext search across local ZIMs with ranking, diversification and spelling correction. */
+	async search(terms, {limit = 20, sources = null} = {}) {
 		this.#assertRunning();
-		const termList = String(terms).split(/[,\s]+/).map(t => t.trim()).filter(Boolean);
-		if (!termList.length) return [];
 
-		const params = new URLSearchParams({pattern: termList.join(' '), format: 'xml', pageLength: String(limit)});
-		const res = await fetch(`${this.baseUrl}/search?${params}`);
-		if (!res.ok) return [];
+		const termList = tokenize(terms);
+		if(!termList.length) return {results: [], spellcheck: null};
 
-		const found = fromXml(await res.text())?.rss?.channel?.item || [];
 		const books = await this.list();
 		const bookMap = new Map(books.map(b => [b.title, b]));
+		const scoped = sources?.length
+			? books.filter(b => sources.includes(b.name) || sources.includes(b.href))
+			: books;
 
-		return found.map(hit => {
-			const book = bookMap.get(hit.book.title);
-			if (!book) return null;
-			const prefix = `/content/${book.href}/`;
-			const page = hit.link.startsWith(prefix) ? hit.link.slice(prefix.length) : hit.link.replace(/^\/+/, '');
-			return {
-				id: book.id,
-				title: hit.title,
-				page,
-				name: book.name,
-				publisher: book.publisher,
-				href: `${book.href}/${page}`,
-				icon: book.icon,
-				viewer: this.baseUrl + hit.link,
-				summary: hit.description,
-				score: +hit.score || 0,
-			};
-		}).filter(Boolean);
+		if(!scoped.length) return {results: [], spellcheck: null};
+
+		let activeTerms = termList;
+		let hits = await this.#rawSearch(activeTerms, scoped, bookMap, limit);
+		let spellcheck = null;
+
+		if(!hits.length && !this.#remote) {
+			const vocabulary = new Set();
+
+			for(const book of scoped) {
+				if(!book.file) continue;
+
+				try {
+					for(const word of await titleVocabulary(path.join(this.#dir, book.file)))
+						vocabulary.add(word);
+				} catch {
+					// Unreadable ZIM - skip it, don't fail the whole search.
+				}
+			}
+
+			const corrected = suggestCorrection(termList, vocabulary);
+
+			if(corrected) {
+				const retry = await this.#rawSearch(corrected, scoped, bookMap, limit);
+
+				if(retry.length) {
+					hits = retry;
+					activeTerms = corrected;
+					spellcheck = {
+						from: termList.join(' '),
+						to: corrected.join(' '),
+					};
+				}
+			}
+		}
+
+		return {
+			results: diversify(rerank(hits, activeTerms), limit),
+			spellcheck,
+		};
 	}
 }