diff --git a/package.json b/package.json index c0af0ac..0988f78 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@ztimson/zim-utils", - "version": "0.1.5", + "version": "0.2.0", "description": "Native, dependency-light ZIM archive reader/searcher and Kiwix catalog downloader for Node.js", "author": "Zak Timson", "license": "MIT", diff --git a/src/catalog.js b/src/catalog.js index 66c27dd..e57d496 100644 --- a/src/catalog.js +++ b/src/catalog.js @@ -5,12 +5,13 @@ export const CATALOG_URL = 'https://library.kiwix.org/catalog/v2/entries'; const PAGE_SIZE = 100; -/** Parses `` blocks out of a Kiwix catalog OPDS XML response. */ -function parseEntries(xml) { +function parseEntries(xml, baseUrl = 'https://library.kiwix.org') { const blocks = xml.match(/[\s\S]*?<\/entry>/g) || []; return blocks.map(b => { const grab = re => (b.match(re) || [])[1] || ''; const linkMatch = b.match(/]*type=["']application\/x-zim[^"']*["'][^>]*href=["']([^"']+)["']/); + const iconMatch = b.match(/]*rel=["']http:\/\/opds-spec\.org\/image\/thumbnail["'][^>]*href=["']([^"']+)["']/) + || b.match(/]*type=["']image\/[^"']*["'][^>]*href=["']([^"']+)["']/); let tags = grab(/([^<]*)<\/tags>/); if(tags) tags = tags.split(';'); return { @@ -28,6 +29,7 @@ function parseEntries(xml) { articleCount: Number(grab(/([^<]*)<\/articleCount>/)) || 0, sizeMb: linkMatch ? (Number((b.match(/length=["'](\d+)["']/) || [])[1] || 0) / 1024 / 1024).toFixed(1) : '?', href: linkMatch ? linkMatch[1] : null, + icon: iconMatch ? new URL(iconMatch[1], baseUrl).href : null, }; }); } diff --git a/src/manager.js b/src/manager.js index c0f2256..d9cf273 100644 --- a/src/manager.js +++ b/src/manager.js @@ -45,11 +45,7 @@ export class ZimManager { let reader; try { reader = await new ZimReader(filepath).open(); - return { - name: await reader.metadata('Name'), - date: await reader.metadata('Date'), - title: await reader.metadata('Title'), - }; + return await reader.metadata(); } catch { return null; } finally { @@ -59,7 +55,7 @@ export class ZimManager { async #update(name, catalogEntry, localMatch, force) { const remoteDate = catalogEntry.updated ? new Date(catalogEntry.updated) : null; - const localDate = localMatch?.meta?.date ? new Date(localMatch.meta.date) : null; + const localDate = localMatch?.meta?.updated ?? null; if (!force && localMatch && remoteDate && localDate && remoteDate <= localDate) return {name, status: 'skipped', reason: 'up to date'}; if (!catalogEntry.href) return {name, status: 'skipped', reason: 'missing download link'}; @@ -67,7 +63,7 @@ export class ZimManager { const filename = path.basename(new URL(catalogEntry.href).pathname).replace(/\.meta4$/i, ''); const destPath = path.join(this.#dir, filename); await this.#download(catalogEntry.href, destPath); - if (localMatch && localMatch.file !== destPath) await fs.promises.unlink(localMatch.file).catch(() => {}); + if (localMatch && localMatch.file !== destPath) await new ZimReader(localMatch.file).delete().catch(() => {}); return {name, status: 'updated', file: destPath}; } @@ -88,7 +84,7 @@ export class ZimManager { async delete(fileOrName) { const file = await this.#resolveFile(fileOrName); - await fs.promises.unlink(file); + await new ZimReader(file).delete(); return {file, status: 'deleted'}; } @@ -99,7 +95,7 @@ export class ZimManager { const catalogEntry = await zimCatalog(meta.name, this.#catalog); if (!catalogEntry) return {file, upToDate: null, reason: 'missing from catalog'}; const remoteDate = catalogEntry.updated ? new Date(catalogEntry.updated) : null; - const localDate = meta.date ? new Date(meta.date) : null; + const localDate = meta.updated ?? null; return {file, name: meta.name, upToDate: !!(remoteDate && localDate && remoteDate <= localDate), localDate, remoteDate}; } @@ -153,7 +149,7 @@ export class ZimManager { } /** Fuzzy-searches titles across every local ZIM in the library, merging & re-ranking hits by score. */ - async search(terms, {limit = 10, htmlOnly = true} = {}) { + async search(terms, {limit = 20, htmlOnly = true} = {}) { const local = await this.list(); const perZim = await Promise.all(local.map(async ({file, meta}) => { let reader; diff --git a/src/reader.js b/src/reader.js index edbb6fa..19960af 100644 --- a/src/reader.js +++ b/src/reader.js @@ -1,10 +1,12 @@ 'use strict'; import fs from 'node:fs'; +import path from 'node:path'; import {decompress as lzmaDecompress} from 'lzma1'; import {ZstdCodec} from 'zstd-codec'; import {fuzzyMatch, titleFromUrl} from './utils.js'; +const INDEX_VERSION = 1; const HEADER_SIZE = 80; const NS_CONTENT = 'C'; const NS_METADATA = 'M'; @@ -32,6 +34,7 @@ export class ZimReader { #header = null; #mimeTypes = []; #hasTitleListing = false; + #index; get articleCount() { return this.#header?.articleCount ?? 0; } @@ -86,6 +89,13 @@ export class ZimReader { return data.subarray(readPtr(blobNumber), readPtr(blobNumber + 1)); } + async #icon(size = 48) { + const page = await this.readPage(`Illustration_${size}x${size}@1`, NS_METADATA) + || await this.readPage('Favicon', NS_METADATA); + if (!page) return null; + return `data:${page.mimetype};base64,${page.data.toString('base64')}`; + } + async #ptr64(base, index) { return Number((await this.#read(base + index * 8, 8)).readBigUInt64LE(0)); } @@ -172,15 +182,89 @@ export class ZimReader { return dirents; } + /** Path of the cached word index, kept in an `index/` subdirectory next to the archive. */ + #indexPath() { + return path.join(path.dirname(this.path), 'index', `${path.basename(this.path)}.idx.json`); + } + + /** Builds (or loads a cached) word -> entry index, avoiding a re-scan on every search. */ + async #wordIndex() { + if (this.#index) return this.#index; + const cachePath = this.#indexPath(); + const stat = await fs.promises.stat(this.path); + + if (fs.existsSync(cachePath)) { + try { + const cached = JSON.parse(await fs.promises.readFile(cachePath, 'utf8')); + // Rebuild if stale: index predates the archive's current mtime (e.g. a re-download). + if (cached.version === INDEX_VERSION && cached.mtimeMs >= stat.mtimeMs) { + return (this.#index = cached); + } + } catch {} + } + + // Expensive full scan — only happens once per archive (or after it changes). + const dirents = await this.#allDirents(); + const entries = []; + const words = new Map(); // word -> [entryIndex, ...] + for (const d of dirents) { + if (d.namespace !== NS_CONTENT) continue; + const idx = entries.length; + entries.push({url: d.url, title: d.title, mimetype: d.mimetype}); + const text = `${d.title} ${titleFromUrl(d.url)}`.toLowerCase(); + for (const w of text.split(/\W+/).filter(Boolean)) { + if (!words.has(w)) words.set(w, []); + words.get(w).push(idx); + } + } + + const index = {version: INDEX_VERSION, size: stat.size, mtimeMs: stat.mtimeMs, entries, words: Object.fromEntries(words)}; + await fs.promises.mkdir(path.dirname(cachePath), {recursive: true}); + await fs.promises.writeFile(cachePath, JSON.stringify(index)); + return (this.#index = index); + } + async close() { if (this.#fd) await this.#fd.close(); this.#fd = null; } + /** Deletes the zim archive and its cached index (if any). Safe to call on unopened readers. */ + async delete() { + await this.close(); + this.#index = null; + await fs.promises.rm(this.path, {force: true}); + await fs.promises.rm(this.#indexPath(), {force: true}); + } + /** Reads an 'M' namespace metadata value (e.g. Name, Date, Title). Returns null if missing. */ async metadata(key) { - const page = await this.readPage(key, NS_METADATA); - return page ? page.data.toString('utf8') : null; + const get = async key => { + const page = await this.readPage(key, NS_METADATA); + return page ? page.data.toString('utf8') : null; + }; + if(key) return get(key); + + const [title, creator, publisher, date, description, language, name, tags] = await Promise.all( + ['Title', 'Creator', 'Publisher', 'Date', 'Description', 'Language', 'Name', 'Tags'].map(get) + ); + return { + id: name, + title, + updated: date ? new Date(date) : null, + summary: description, + language, + name, + category: tags ? tags.split(';')[0] || '' : '', + tags: tags ? tags.split(';') : [], + mediaCount: 0, + author: creator, + publisher, + articleCount: this.articleCount, + sizeMb: ((await fs.promises.stat(this.path)).size / 1024 / 1024).toFixed(1), + href: null, + icon: await this.#icon(), + }; } /** Opens the archive and parses its header + mimetype list. */ @@ -219,24 +303,29 @@ export class ZimReader { const termList = String(terms).split(',').map(t => t.trim()).filter(Boolean); if (!termList.length) return []; - const candidates = (this.#hasTitleListing && this.#header.articleCount > 5000) - ? await this.#titleIndexCandidates(termList[0]) - : await this.#allDirents(); + let candidates; + if (this.#hasTitleListing && this.#header.articleCount > 5000) { + candidates = await this.#titleIndexCandidates(termList[0]); + } else { + const {entries, words} = await this.#wordIndex(); + // Pull candidates from postings of any word that starts with (or contains) the search term. + const q = termList[0].toLowerCase(); + const idxSet = new Set(); + for (const [word, postings] of Object.entries(words)) { + if (word.includes(q) || q.includes(word)) postings.forEach(i => idxSet.add(i)); + } + candidates = [...idxSet].map(i => ({...entries[i], namespace: NS_CONTENT})); + } const scored = []; for (const dirent of candidates) { - if (dirent.namespace !== NS_CONTENT) continue; if (htmlOnly && !(this.#mimeTypes[dirent.mimetype] || '').startsWith('text/html')) continue; const urlTitle = titleFromUrl(dirent.url); const titleScore = fuzzyMatch(dirent.title, ...termList).max; const urlScore = fuzzyMatch(urlTitle, ...termList).max; - scored.push({ - url: dirent.url, - title: dirent.title.length > urlTitle.length ? dirent.title : urlTitle, - namespace: dirent.namespace, - score: Math.max(titleScore, urlScore) - }); + scored.push({url: dirent.url, title: dirent.title.length > urlTitle.length ? dirent.title : urlTitle, namespace: NS_CONTENT, score: Math.max(titleScore, urlScore)}); } - return scored.filter(a => a.score > 0).toSorted((a, b) => b.score - a.score).slice(0, limit); + const {id, summary, mediaCount, articleCount, sizeMb, href, ...meta} = await this.metadata(); + return scored.filter(a => a.score > 0).toSorted((a, b) => b.score - a.score).slice(0, limit).map(a => ({...meta, ...a})); } } diff --git a/src/utils.js b/src/utils.js index a0dc334..9802c32 100644 --- a/src/utils.js +++ b/src/utils.js @@ -20,12 +20,6 @@ export function similarity(a, b) { return 1 - levenshtein(a, b) / Math.max(a.length, b.length, 1); } -/** - * Scores `text` against a single lowercased `term`. Substring containment always wins. - * Otherwise, only rescues genuine typos: requires the same leading letter and a tight - * absolute edit-distance cap, so unrelated words can't win purely on coincidental - * letter overlap (e.g. "Viennese" vs "diannes"). - */ function scoreAgainst(text, term) { if (text.includes(term)) return 1 - (text.length - term.length) / text.length * 0.3; if (!text.length || !term.length || text[0] !== term[0]) return 0;