'use strict'; import {spawn, execFile} from 'node:child_process'; import {promisify} from 'node:util'; import net from 'node:net'; import fs from 'node:fs'; import path from 'node:path'; import {fileURLToPath} from 'node:url'; import {fromXml, makeArray} from '@ztimson/utils'; import {titleVocabulary} from './reader.js'; import {diversify, rerank, rrfMerge, suggestCorrection} from './search.js'; const execFileAsync = promisify(execFile); const __dirname = path.dirname(fileURLToPath(import.meta.url)); const DEFAULT_BIN_DIR = path.join(__dirname, '..', 'bin'); const READY_TIMEOUT = 10_000; const READY_POLL_INTERVAL = 100; const WATCH_DEBOUNCE = 300; function findFreePort() { return new Promise((resolve, reject) => { const srv = net.createServer(); srv.on('error', reject); srv.listen(0, () => { const {port} = srv.address(); srv.close(() => resolve(port)); }); }); } function tokenize(terms) { return String(terms).split(/[,\s]+/).map(t => t.trim()).filter(Boolean); } export class KiwixServer { static #empty = '\n\n'; #dir; #host; #port; #binDir; #libraryPath; #child = null; #remote; #watcher = null; #watchTimer = null; get port() { return this.#port; } get running() { return !!this.#remote || !!this.#child; } get baseUrl() { return this.#remote || (this.#child ? `http://${this.#host}:${this.#port}` : null); } constructor(dir, {port, host = '127.0.0.1', binDir = DEFAULT_BIN_DIR, url} = {}) { this.#dir = dir; this.#host = host; this.#port = port; this.#binDir = binDir; this.#libraryPath = path.join(dir, 'library.xml'); this.#remote = url ? url.replace(/\/$/, '') : null; if(!this.#remote) this.#ensureLocalStore(); } #ensureLocalStore() { fs.mkdirSync(this.#dir, {recursive: true}); if(!fs.existsSync(this.#libraryPath)) fs.writeFileSync(this.#libraryPath, KiwixServer.#empty); } #assertRunning() { if(!this.running) throw new Error('KiwixServer is not running - call start() first'); } #bin(name) { return path.join(this.#binDir, process.platform === 'win32' ? `${name}.exe` : name); } async #fetchLibraryXml() { if(!this.#remote) return fs.promises.readFile(this.#libraryPath, 'utf8').catch(() => ''); try { const res = await fetch(`${this.#remote}/library.xml`); return res.ok ? await res.text() : ''; } catch { return ''; } } async #rebuildLibrary() { if(this.#remote) return; await fs.promises.rm(this.#libraryPath, {force: true}); const files = await this.#zimFiles(); if(!files.length) return fs.promises.writeFile(this.#libraryPath, KiwixServer.#empty); for(const file of files) await execFileAsync(this.#bin('kiwix-manage'), [ this.#libraryPath, 'add', path.join(this.#dir, file), ]); } async #waitUntilReady() { const deadline = Date.now() + READY_TIMEOUT; while(Date.now() < deadline) { try { await fetch(`http://${this.#host}:${this.#port}/`); return; } catch {} await new Promise(r => setTimeout(r, READY_POLL_INTERVAL)); } throw new Error('kiwix-serve did not become ready in time'); } async #zimFiles() { return (await fs.promises.readdir(this.#dir).catch(() => [])) .filter(f => f.endsWith('.zim')); } #watchDir() { this.#watcher?.close(); this.#watcher = fs.watch(this.#dir, (_event, filename) => { if(!filename?.endsWith('.zim')) return; clearTimeout(this.#watchTimer); this.#watchTimer = setTimeout(() => this.#rebuildLibrary().catch(() => {}), WATCH_DEBOUNCE); }); this.#watcher.on('error', () => {}); } #unwatchDir() { clearTimeout(this.#watchTimer); this.#watchTimer = null; this.#watcher?.close(); this.#watcher = null; } async start() { if(this.#remote || this.#child) return; await fs.promises.mkdir(this.#dir, {recursive: true}); await this.#rebuildLibrary(); this.#port ??= await findFreePort(); this.#child = spawn(this.#bin('kiwix-serve'), [ '--library', '--monitorLibrary', '-i', this.#host, '-p', String(this.#port), this.#libraryPath, ], {stdio: 'ignore'}); this.#child.on('exit', () => { this.#child = null; this.#unwatchDir(); }); try { await this.#waitUntilReady(); } catch(e) { await this.stop(); throw e; } this.#watchDir(); } async stop() { this.#unwatchDir(); if(this.#remote || !this.#child) return; const child = this.#child; await new Promise(resolve => { child.once('exit', resolve); child.kill('SIGTERM'); }); this.#child = null; } async restart() { await this.stop(); await this.start(); } async reload() { if(this.#remote || !this.#child) return; await this.#rebuildLibrary(); } async list() { this.#assertRunning(); const xml = await this.#fetchLibraryXml(); if(!xml) return []; const entries = fromXml(xml); return makeArray(entries?.library?.book || []).map(e => { const tags = e.tags.split(';'); const name = e.path.replace(/\.zim$/i, ''); return { id: e.id, title: e.title, updated: new Date(e.date), summary: e.description, language: e.language, name: e.name, category: tags.find(t => t.startsWith('_category'))?.slice(10) || '', tags, mediaCount: +e.mediaCount || 0, author: e.creator, publisher: e.publisher, articleCount: +e.articleCount || 0, sizeMb: +(Number(e.size) / 1024).toFixed(1) || 0, file: e.path, href: name, icon: `data:${e.faviconMimetype || 'image/png'};base64,${e.favicon}`, viewer: `${this.baseUrl}/content/${name}`, }; }); } #splitHref(href) { const clean = href.replace(`${this.baseUrl}/content/`, '').replace(/^\/+/, ''); const [zim, ...rest] = clean.split('/'); return {zim, path: rest.join('/')}; } link(href) { this.#assertRunning(); const {zim, path} = this.#splitHref(href); return `${this.baseUrl}/content/${zim}${path ? '/' + path : ''}`; } async raw(href) { const res = await fetch(this.link(href)); if(!res.ok) return null; return { mimetype: res.headers.get('content-type'), data: Buffer.from(await res.arrayBuffer()), }; } async #rawSearch(termList, scoped, bookMap, limit) { const perBook = await Promise.all(scoped.map(async book => { const params = new URLSearchParams({ pattern: termList.join(' '), format: 'xml', pageLength: String(limit), 'books.name': book.href, }); const res = await fetch(`${this.baseUrl}/search?${params}`); if(!res.ok) return []; const found = fromXml(await res.text())?.rss?.channel?.item || []; return makeArray(found).map(hit => { const resultBook = bookMap.get(hit.book?.title) || book; const prefix = `/content/${resultBook.href}/`; const page = hit.link.startsWith(prefix) ? hit.link.slice(prefix.length) : hit.link.replace(/^\/+/, ''); return { id: resultBook.id, title: hit.title, page, name: resultBook.name, publisher: resultBook.publisher, href: `${resultBook.href}/${page}`, icon: resultBook.icon, viewer: this.baseUrl + hit.link, summary: hit.description, xapianScore: +hit.score || 0, }; }).filter(Boolean); })); return rrfMerge(perBook); } /** Fulltext search across local ZIMs with ranking, diversification and spelling correction. */ async search(terms, {limit = 20, sources = null} = {}) { this.#assertRunning(); const termList = tokenize(terms); if(!termList.length) return {results: [], spellcheck: null}; const books = await this.list(); const bookMap = new Map(books.map(b => [b.title, b])); const scoped = sources?.length ? books.filter(b => sources.includes(b.name) || sources.includes(b.href)) : books; if(!scoped.length) return {results: [], spellcheck: null}; let activeTerms = termList; let hits = await this.#rawSearch(activeTerms, scoped, bookMap, limit); let spellcheck = null; if(!hits.length && !this.#remote) { const vocabulary = new Set(); for(const book of scoped) { if(!book.file) continue; try { for(const word of await titleVocabulary(path.join(this.#dir, book.file))) vocabulary.add(word); } catch { // Unreadable ZIM - skip it, don't fail the whole search. } } const corrected = suggestCorrection(termList, vocabulary); if(corrected) { const retry = await this.#rawSearch(corrected, scoped, bookMap, limit); if(retry.length) { hits = retry; activeTerms = corrected; spellcheck = { from: termList.join(' '), to: corrected.join(' '), }; } } } const ranked = rerank(hits, activeTerms); return { results: diversify(ranked, limit), spellcheck, }; } }