zim-utils/src/server.js

'use strict';

import {spawn, execFile} from 'node:child_process';
import {promisify} from 'node:util';
import net from 'node:net';
import fs from 'node:fs';
import path from 'node:path';
import {fileURLToPath} from 'node:url';
import {fromXml, makeArray} from '@ztimson/utils';
import {titleVocabulary} from './reader.js';
import {diversify, rerank, rrfMerge, suggestCorrection} from './search.js';

const execFileAsync = promisify(execFile);
const __dirname = path.dirname(fileURLToPath(import.meta.url));
const DEFAULT_BIN_DIR = path.join(__dirname, '..', 'bin');
const READY_TIMEOUT = 10_000;
const READY_POLL_INTERVAL = 100;
const WATCH_DEBOUNCE = 300;

function findFreePort() {
	return new Promise((resolve, reject) => {
		const srv = net.createServer();
		srv.on('error', reject);
		srv.listen(0, () => {
			const {port} = srv.address();
			srv.close(() => resolve(port));
		});
	});
}

function tokenize(terms) {
	return String(terms).split(/[,\s]+/).map(t => t.trim()).filter(Boolean);
}

export class KiwixServer {
	static #empty = '<?xml version="1.0" encoding="UTF-8" ?>\n<library version="20110515"></library>\n';

	#dir;
	#host;
	#port;
	#binDir;
	#libraryPath;
	#child = null;
	#remote;
	#watcher = null;
	#watchTimer = null;

	get port() { return this.#port; }
	get running() { return !!this.#remote || !!this.#child; }
	get baseUrl() { return this.#remote || (this.#child ? `http://${this.#host}:${this.#port}` : null); }

	constructor(dir, {port, host = '127.0.0.1', binDir = DEFAULT_BIN_DIR, url} = {}) {
		this.#dir = dir;
		this.#host = host;
		this.#port = port;
		this.#binDir = binDir;
		this.#libraryPath = path.join(dir, 'library.xml');
		this.#remote = url ? url.replace(/\/$/, '') : null;

		if(!this.#remote) this.#ensureLocalStore();
	}

	#ensureLocalStore() {
		fs.mkdirSync(this.#dir, {recursive: true});

		if(!fs.existsSync(this.#libraryPath))
			fs.writeFileSync(this.#libraryPath, KiwixServer.#empty);
	}

	#assertRunning() {
		if(!this.running) throw new Error('KiwixServer is not running - call start() first');
	}

	#bin(name) {
		return path.join(this.#binDir, process.platform === 'win32' ? `${name}.exe` : name);
	}

	async #fetchLibraryXml() {
		if(!this.#remote)
			return fs.promises.readFile(this.#libraryPath, 'utf8').catch(() => '');

		try {
			const res = await fetch(`${this.#remote}/library.xml`);
			return res.ok ? await res.text() : '';
		} catch {
			return '';
		}
	}

	async #rebuildLibrary() {
		if(this.#remote) return;

		await fs.promises.rm(this.#libraryPath, {force: true});

		const files = await this.#zimFiles();

		if(!files.length)
			return fs.promises.writeFile(this.#libraryPath, KiwixServer.#empty);

		for(const file of files)
			await execFileAsync(this.#bin('kiwix-manage'), [
				this.#libraryPath,
				'add',
				path.join(this.#dir, file),
			]);
	}

	async #waitUntilReady() {
		const deadline = Date.now() + READY_TIMEOUT;

		while(Date.now() < deadline) {
			try {
				await fetch(`http://${this.#host}:${this.#port}/`);
				return;
			} catch {}

			await new Promise(r => setTimeout(r, READY_POLL_INTERVAL));
		}

		throw new Error('kiwix-serve did not become ready in time');
	}

	async #zimFiles() {
		return (await fs.promises.readdir(this.#dir).catch(() => []))
			.filter(f => f.endsWith('.zim'));
	}

	#watchDir() {
		this.#watcher?.close();

		this.#watcher = fs.watch(this.#dir, (_event, filename) => {
			if(!filename?.endsWith('.zim')) return;

			clearTimeout(this.#watchTimer);
			this.#watchTimer = setTimeout(() => this.#rebuildLibrary().catch(() => {}), WATCH_DEBOUNCE);
		});

		this.#watcher.on('error', () => {});
	}

	#unwatchDir() {
		clearTimeout(this.#watchTimer);
		this.#watchTimer = null;
		this.#watcher?.close();
		this.#watcher = null;
	}

	async start() {
		if(this.#remote || this.#child) return;

		await fs.promises.mkdir(this.#dir, {recursive: true});
		await this.#rebuildLibrary();
		this.#port ??= await findFreePort();

		this.#child = spawn(this.#bin('kiwix-serve'), [
			'--library',
			'--monitorLibrary',
			'-i',
			this.#host,
			'-p',
			String(this.#port),
			this.#libraryPath,
		], {stdio: 'ignore'});

		this.#child.on('exit', () => {
			this.#child = null;
			this.#unwatchDir();
		});

		try {
			await this.#waitUntilReady();
		} catch(e) {
			await this.stop();
			throw e;
		}

		this.#watchDir();
	}

	async stop() {
		this.#unwatchDir();

		if(this.#remote || !this.#child) return;

		const child = this.#child;

		await new Promise(resolve => {
			child.once('exit', resolve);
			child.kill('SIGTERM');
		});

		this.#child = null;
	}

	async restart() {
		await this.stop();
		await this.start();
	}

	async reload() {
		if(this.#remote || !this.#child) return;
		await this.#rebuildLibrary();
	}

	async list() {
		this.#assertRunning();

		const xml = await this.#fetchLibraryXml();
		if(!xml) return [];

		const entries = fromXml(xml);

		return makeArray(entries?.library?.book || []).map(e => {
			const tags = e.tags.split(';');
			const name = e.path.replace(/\.zim$/i, '');

			return {
				id: e.id,
				title: e.title,
				updated: new Date(e.date),
				summary: e.description,
				language: e.language,
				name: e.name,
				category: tags.find(t => t.startsWith('_category'))?.slice(10) || '',
				tags,
				mediaCount: +e.mediaCount || 0,
				author: e.creator,
				publisher: e.publisher,
				articleCount: +e.articleCount || 0,
				sizeMb: +(Number(e.size) / 1024).toFixed(1) || 0,
				file: e.path,
				href: name,
				icon: `data:${e.faviconMimetype || 'image/png'};base64,${e.favicon}`,
				viewer: `${this.baseUrl}/content/${name}`,
			};
		});
	}

	#splitHref(href) {
		const clean = href.replace(`${this.baseUrl}/content/`, '').replace(/^\/+/, '');
		const [zim, ...rest] = clean.split('/');
		return {zim, path: rest.join('/')};
	}

	link(href) {
		this.#assertRunning();

		const {zim, path} = this.#splitHref(href);

		return `${this.baseUrl}/content/${zim}${path ? '/' + path : ''}`;
	}

	async raw(href) {
		const res = await fetch(this.link(href));

		if(!res.ok) return null;

		return {
			mimetype: res.headers.get('content-type'),
			data: Buffer.from(await res.arrayBuffer()),
		};
	}

	async #rawSearch(termList, scoped, bookMap, limit) {
		const perBook = await Promise.all(scoped.map(async book => {
			const params = new URLSearchParams({
				pattern: termList.join(' '),
				format: 'xml',
				pageLength: String(limit),
				'books.name': book.href,
			});

			const res = await fetch(`${this.baseUrl}/search?${params}`);
			if(!res.ok) return [];

			const found = fromXml(await res.text())?.rss?.channel?.item || [];

			return makeArray(found).map(hit => {
				const resultBook = bookMap.get(hit.book?.title) || book;

				const prefix = `/content/${resultBook.href}/`;
				const page = hit.link.startsWith(prefix)
					? hit.link.slice(prefix.length)
					: hit.link.replace(/^\/+/, '');

				return {
					id: resultBook.id,
					title: hit.title,
					page,
					name: resultBook.name,
					publisher: resultBook.publisher,
					href: `${resultBook.href}/${page}`,
					icon: resultBook.icon,
					viewer: this.baseUrl + hit.link,
					summary: hit.description,
					xapianScore: +hit.score || 0,
				};
			}).filter(Boolean);
		}));

		return rrfMerge(perBook);
	}

	/** Fulltext search across local ZIMs with ranking, diversification and spelling correction. */
	async search(terms, {limit = 20, sources = null} = {}) {
		this.#assertRunning();

		const termList = tokenize(terms);
		if(!termList.length) return {results: [], spellcheck: null};

		const books = await this.list();
		const bookMap = new Map(books.map(b => [b.title, b]));

		const scoped = sources?.length
			? books.filter(b => sources.includes(b.name) || sources.includes(b.href))
			: books;

		if(!scoped.length) return {results: [], spellcheck: null};

		let activeTerms = termList;
		let hits = await this.#rawSearch(activeTerms, scoped, bookMap, limit);
		let spellcheck = null;

		if(!hits.length && !this.#remote) {
			const vocabulary = new Set();

			for(const book of scoped) {
				if(!book.file) continue;

				try {
					for(const word of await titleVocabulary(path.join(this.#dir, book.file)))
						vocabulary.add(word);
				} catch {
					// Unreadable ZIM - skip it, don't fail the whole search.
				}
			}

			const corrected = suggestCorrection(termList, vocabulary);

			if(corrected) {
				const retry = await this.#rawSearch(corrected, scoped, bookMap, limit);

				if(retry.length) {
					hits = retry;
					activeTerms = corrected;
					spellcheck = {
						from: termList.join(' '),
						to: corrected.join(' '),
					};
				}
			}
		}

		const ranked = rerank(hits, activeTerms);

		return {
			results: diversify(ranked, limit),
			spellcheck,
		};
	}
}