From bc90557de77ce17e14a24fa3d9504f51b6de15aa Mon Sep 17 00:00:00 2001 From: ztimson Date: Mon, 24 Aug 2026 11:02:47 -0400 Subject: [PATCH] use kiwix-serach as prefilter before fuzzy ranking instead of building index --- .gitignore | 10 ++++ bin/install-kwix.js | 87 +++++++++++++++++++++++++++++++++ package.json | 5 +- src/reader.js | 116 +++++++++----------------------------------- src/utils.js | 29 +++++++++++ 5 files changed, 152 insertions(+), 95 deletions(-) create mode 100644 .gitignore create mode 100644 bin/install-kwix.js diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..4bedc79 --- /dev/null +++ b/.gitignore @@ -0,0 +1,10 @@ +.idea +.vscode + +logs +*.log + +bin/*.dll +bin/kiwix* +node_modles +zims diff --git a/bin/install-kwix.js b/bin/install-kwix.js new file mode 100644 index 0000000..2e78fe5 --- /dev/null +++ b/bin/install-kwix.js @@ -0,0 +1,87 @@ +import fs from 'node:fs'; +import os from 'node:os'; +import path from 'node:path'; +import https from 'node:https'; +import {execFileSync} from 'node:child_process'; +import {fileURLToPath} from 'node:url'; + +const __dirname = path.dirname(fileURLToPath(import.meta.url)); + +const VERSION = '3.8.1'; +const BASE_URL = 'https://download.kiwix.org/release/kiwix-tools'; +const PLATFORM_MAP = {linux: 'linux', darwin: 'macos', win32: 'win'}; + +const BIN_NAME = process.platform === 'win32' ? 'kiwix-search.exe' : 'kiwix-search'; +const BIN_DIR = path.join(__dirname, '..', 'bin'); // project root/bin +const BIN_PATH = path.join(BIN_DIR, BIN_NAME); + +/** Download a file, following redirects. */ +function download(url, dest) { + return new Promise((resolve, reject) => { + const file = fs.createWriteStream(dest); + https.get(url, res => { + if (res.statusCode >= 300 && res.statusCode < 400 && res.headers.location) { + file.close(); + return resolve(download(res.headers.location, dest)); + } + if (!res.statusCode || res.statusCode >= 400) { + return reject(new Error(`Download failed: ${res.statusCode} ${res.statusMessage}`)); + } + res.pipe(file); + file.on('finish', () => file.close(resolve)); + }).on('error', err => fs.unlink(dest, () => reject(err))); + }); +} + +/** Recursively search a directory for a file by name. */ +function findFile(dir, name) { + for (const entry of fs.readdirSync(dir, {withFileTypes: true})) { + const full = path.join(dir, entry.name); + if (entry.isDirectory()) { + const found = findFile(full, name); + if (found) return found; + } else if (entry.name === name) { + return full; + } + } + return null; +} + +async function downloadAndExtract() { + const platformName = PLATFORM_MAP[process.platform]; + if (!platformName) throw new Error(`Unsupported platform: ${process.platform}`); + + const ext = platformName === 'win' ? 'zip' : 'tar.gz'; + const archiveName = `kiwix-tools_${platformName}-x86_64-${VERSION}.${ext}`; + const url = `${BASE_URL}/${archiveName}`; + const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'kiwix-tools-')); + const archivePath = path.join(tmpDir, archiveName); + + console.log('Downloading kiwix-tools from:', url); + await download(url, archivePath); + + console.log('Extracting...'); + execFileSync('tar', ['-xf', archivePath, '-C', tmpDir]); // bsdtar handles zip too + + const extractedBin = findFile(tmpDir, BIN_NAME); + if (!extractedBin) throw new Error(`Could not find ${BIN_NAME} in extracted archive`); + const extractedDir = path.dirname(extractedBin); + + await fs.promises.mkdir(BIN_DIR, {recursive: true}); + + // Copy the binary plus any DLLs sitting alongside it (Windows deps) + for (const entry of await fs.promises.readdir(extractedDir)) { + if (entry === BIN_NAME || entry.toLowerCase().endsWith('.dll')) { + await fs.promises.copyFile(path.join(extractedDir, entry), path.join(BIN_DIR, entry)); + } + } + if (process.platform !== 'win32') await fs.promises.chmod(BIN_PATH, 0o755); + + await fs.promises.rm(tmpDir, {recursive: true, force: true}); + console.log('Installed to:', BIN_PATH); +} + +downloadAndExtract().catch(err => { + console.error(err); + process.exit(1) +}).finally(() => process.exit()); diff --git a/package.json b/package.json index 5ba296b..f464cca 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@ztimson/zim-utils", - "version": "0.2.3", + "version": "0.2.4", "description": "Native, dependency-light ZIM archive reader/searcher and Kiwix catalog downloader for Node.js", "author": "Zak Timson", "license": "MIT", @@ -11,6 +11,9 @@ "url": "https://git.zakscode.com/ztimson/zim-utils" }, "main": "src/index.js", + "scripts": { + "postinstall": "node scripts/install-kiwix.mjs" + }, "dependencies": { "@ztimson/utils": "^0.30.7", "lzma1": "^0.3.0", diff --git a/src/reader.js b/src/reader.js index 905e29c..012da08 100644 --- a/src/reader.js +++ b/src/reader.js @@ -1,11 +1,9 @@ 'use strict'; import fs from 'node:fs'; -import path from 'node:path'; import {decompressPool} from './decompress.js'; -import {fuzzyMatch, titleFromUrl} from './utils.js'; +import {fuzzyMatch, titleFromUrl, kiwixSearch} from './utils.js'; -const INDEX_VERSION = 1; const HEADER_SIZE = 80; const NS_CONTENT = 'C'; const NS_METADATA = 'M'; @@ -20,7 +18,6 @@ export class ZimReader { #header = null; #mimeTypes = []; #hasTitleListing = false; - #index; #clusterCache = new Map(); // clusterNumber -> {data, extended, timer} #pending = new Map(); // clusterNumber -> Promise, dedupes concurrent misses #clusterCacheMax; @@ -36,15 +33,6 @@ export class ZimReader { this.#clusterTTL = clusterTTL; } - /** Full O(n) scan over the URL pointer list, used when there's no title index. */ - async #allDirents() { - const dirents = []; - for (let i = 0; i < this.#header.articleCount; i++) { - dirents.push(await this.#readDirent(await this.#ptr64(this.#header.urlPtrPos, i))); - } - return dirents; - } - /** Binary search the URL pointer list for namespace+url. */ async #findByUrl(url, namespace) { const key = namespace + url; @@ -60,6 +48,22 @@ export class ZimReader { return null; } + /** Binary search the title index for an exact title match. */ + async #findByTitle(title) { + if (!this.#hasTitleListing) return null; + const q = title.toLowerCase(); + let lo = 0, hi = this.#header.articleCount - 1; + while (lo <= hi) { + const mid = (lo + hi) >> 1; + const urlIdx = (await this.#read(this.#header.titlePtrPos + mid * 4, 4)).readUInt32LE(0); + const dirent = await this.#readDirent(await this.#ptr64(this.#header.urlPtrPos, urlIdx)); + const t = dirent.title.toLowerCase(); + if (t === q) return dirent; + if (t < q) lo = mid + 1; else hi = mid - 1; + } + return null; + } + /** Resets a cluster's idle-eviction timer. No-op when TTL disabled. */ #touch(clusterNumber, entry) { if (!this.#clusterTTL) return; @@ -185,68 +189,6 @@ export class ZimReader { return this.#readDirent(await this.#ptr64(this.#header.urlPtrPos, dirent.redirectIndex)); } - /** Narrows to dirents near the term's alphabetical position in the title index. */ - async #titleIndexCandidates(term) { - const q = term.toLowerCase(); - let lo = 0, hi = this.#header.articleCount - 1; - while (lo < hi) { - const mid = (lo + hi) >> 1; - const urlIdx = (await this.#read(this.#header.titlePtrPos + mid * 4, 4)).readUInt32LE(0); - const dirent = await this.#readDirent(await this.#ptr64(this.#header.urlPtrPos, urlIdx)); - if (dirent.title.toLowerCase() < q) lo = mid + 1; else hi = mid; - } - // Widen around the prefix match since fuzzy scoring isn't purely alphabetical. - const start = Math.max(0, lo - 50), end = Math.min(this.#header.articleCount, lo + 200); - const dirents = []; - for (let i = start; i < end; i++) { - const urlIdx = (await this.#read(this.#header.titlePtrPos + i * 4, 4)).readUInt32LE(0); - dirents.push(await this.#readDirent(await this.#ptr64(this.#header.urlPtrPos, urlIdx))); - } - return dirents; - } - - /** Path of the cached word index, kept in an `index/` subdirectory next to the archive. */ - #indexPath() { - return path.join(path.dirname(this.path), 'index', `${path.basename(this.path)}.idx.json`); - } - - /** Builds (or loads a cached) word -> entry index, avoiding a re-scan on every search. */ - async #wordIndex() { - if (this.#index) return this.#index; - const cachePath = this.#indexPath(); - const stat = await fs.promises.stat(this.path); - - if (fs.existsSync(cachePath)) { - try { - const cached = JSON.parse(await fs.promises.readFile(cachePath, 'utf8')); - // Rebuild if stale: index predates the archive's current mtime (e.g. a re-download). - if (cached.version === INDEX_VERSION && cached.mtimeMs >= stat.mtimeMs) { - return (this.#index = cached); - } - } catch {} - } - - // Expensive full scan — only happens once per archive (or after it changes). - const dirents = await this.#allDirents(); - const entries = []; - const words = new Map(); // word -> [entryIndex, ...] - for (const d of dirents) { - if (d.namespace !== NS_CONTENT) continue; - const idx = entries.length; - entries.push({url: d.url, title: d.title, mimetype: d.mimetype}); - const text = `${d.title} ${titleFromUrl(d.url)}`.toLowerCase(); - for (const w of text.split(/\W+/).filter(Boolean)) { - if (!words.has(w)) words.set(w, []); - words.get(w).push(idx); - } - } - - const index = {version: INDEX_VERSION, size: stat.size, mtimeMs: stat.mtimeMs, entries, words: Object.fromEntries(words)}; - await fs.promises.mkdir(path.dirname(cachePath), {recursive: true}); - await fs.promises.writeFile(cachePath, JSON.stringify(index)); - return (this.#index = index); - } - async close() { if (this.#fd) await this.#fd.close(); this.#fd = null; @@ -255,12 +197,10 @@ export class ZimReader { this.#pending.clear(); } - /** Deletes the zim archive and its cached index (if any). Safe to call on unopened readers. */ + /** Deletes the zim archive. Safe to call on unopened readers. */ async delete() { await this.close(); - this.#index = null; await fs.promises.rm(this.path, {force: true}); - await fs.promises.rm(this.#indexPath(), {force: true}); } /** Reads an 'M' namespace metadata value (e.g. Name, Date, Title). Returns null if missing. */ @@ -325,27 +265,15 @@ export class ZimReader { /** * Fuzzy-ranked title search. Accepts comma-separated `terms` the same way the - * catalog search does. Uses the sorted title index when present (binary search - * narrows the candidate window); falls back to a full linear scan otherwise - * (common on ZIM v6+/zimit-generated archives with no title index). + * catalog search does. Uses kiwix-search's embedded fulltext index as a prefilter + * to narrow candidates before fuzzy scoring. */ async search(terms, {limit = 20, htmlOnly = true} = {}) { const termList = String(terms).split(',').map(t => t.trim()).filter(Boolean); if (!termList.length) return []; - let candidates; - if (this.#hasTitleListing && this.#header.articleCount > 5000) { - candidates = await this.#titleIndexCandidates(termList[0]); - } else { - const {entries, words} = await this.#wordIndex(); - // Pull candidates from postings of any word that starts with (or contains) the search term. - const q = termList[0].toLowerCase(); - const idxSet = new Set(); - for (const [word, postings] of Object.entries(words)) { - if (word.includes(q) || q.includes(word)) postings.forEach(i => idxSet.add(i)); - } - candidates = [...idxSet].map(i => ({...entries[i], namespace: NS_CONTENT})); - } + const titles = await kiwixSearch(this.path, termList.join(' ')); + const candidates = (await Promise.all(titles.map(t => this.#findByTitle(t)))).filter(Boolean); const scored = []; for (const dirent of candidates) { diff --git a/src/utils.js b/src/utils.js index 9802c32..3fe59da 100644 --- a/src/utils.js +++ b/src/utils.js @@ -1,3 +1,32 @@ +import {execFile} from 'node:child_process'; +import path from 'node:path'; +import {fileURLToPath} from 'node:url'; +import {promisify} from 'node:util'; + +/** + * Search a ZIM file's fulltext index. + * @param {string} zimPath - Path to the .zim file + * @param {string} pattern - Search terms + * @param {object} [opts] + * @param {boolean} [opts.suggestion] - Suggest titles from partial pattern (completion-style) + * @param {boolean} [opts.spelling] - Suggest spelling-corrected titles + * @returns {Promise} Matching article/tag titles + */ +export async function kiwixSearch(zimPath, pattern, opts = {}) { + const execFileAsync = promisify(execFile); + const __dirname = path.dirname(fileURLToPath(import.meta.url)); + const BIN_NAME = process.platform === 'win32' ? 'kiwix-search.exe' : 'kiwix-search'; + const BIN_PATH = path.join(__dirname, '..', 'bin', BIN_NAME); + + const args = []; + if (opts.suggestion) args.push('-s'); + if (opts.spelling) args.push('--spelling'); + args.push(zimPath, pattern); + + const {stdout} = await execFileAsync(BIN_PATH, args); + return stdout.split('\n').map(line => line.trim()).filter(Boolean); +} + export function levenshtein(a, b) { const m = a.length, n = b.length; if (!m) return n;