New Zim utilties

src/catalog.js

@@ -0,0 +1,73 @@
+import {fuzzyMatch} from './utils.js';
+import {decodeHtml} from '@ztimson/utils';
+
+export const CATALOG_URL = 'https://library.kiwix.org/catalog/v2/entries';
+
+const PAGE_SIZE = 100;
+
+/** Parses `<entry>` blocks out of a Kiwix catalog OPDS XML response. */
+function parseEntries(xml) {
+	const blocks = xml.match(/<entry>[\s\S]*?<\/entry>/g) || [];
+	return blocks.map(b => {
+		const grab = re => (b.match(re) || [])[1] || '';
+		const linkMatch = b.match(/<link[^>]*type=["']application\/x-zim[^"']*["'][^>]*href=["']([^"']+)["']/);
+		let tags = grab(/<tags>([^<]*)<\/tags>/);
+		if(tags) tags = tags.split(';');
+		return {
+			id: grab(/<id>([^<]*)<\/id>/),
+			title: decodeHtml(grab(/<title>([^<]*)<\/title>/)),
+			updated: new Date(grab(/<updated>([^<]*)<\/updated>/)),
+			summary: decodeHtml(grab(/<summary>([^<]*)<\/summary>/)),
+			language: grab(/<language>([^<]*)<\/language>/),
+			name: grab(/<name>([^<]*)<\/name>/),
+			category: grab(/<category>([^<]*)<\/category>/),
+			tags,
+			mediaCount: Number(grab(/<mediaCount>([^<]*)<\/mediaCount>/)) || 0,
+			author: grab(/<author>\s*<name>([^<]*)<\/name>\s*<\/author>/m),
+			publisher: grab(/<publisher>\s*<name>([^<]*)<\/name>\s*<\/publisher>/m),
+			articleCount: Number(grab(/<articleCount>([^<]*)<\/articleCount>/)) || 0,
+			sizeMb: linkMatch ? (Number((b.match(/length=["'](\d+)["']/) || [])[1] || 0) / 1024 / 1024).toFixed(1) : '?',
+			href: linkMatch ? linkMatch[1] : null,
+		};
+	});
+}
+
+async function fetchEntries(term, lang, url = CATALOG_URL) {
+	const params = new URLSearchParams({q: term, count: String(PAGE_SIZE), lang: lang || 'eng'});
+	const res = await fetch(`${url}?${params}`);
+	if (!res.ok) throw new Error(`${res.status} ${res.statusText}`);
+	return parseEntries(await res.text());
+}
+
+/** Looks up a single catalog entry by exact `name` (used to check for available updates). */
+export async function zimCatalogInfo(name, url = CATALOG_URL) {
+	const params = new URLSearchParams({name, count: '5'});
+	const res = await fetch(`${url}?${params}`);
+	if (!res.ok) return null;
+	return parseEntries(await res.text()).find(e => e.name === name) || null;
+}
+
+/** Searches the Kiwix catalog for ZIMs matching `terms` within `category`, ranked by term coverage then fuzzy similarity. */
+export async function zimCatalog(terms, opts = {lang: 'eng', count: 20, url: CATALOG_URL}) {
+	opts = Object.assign({lang: 'eng', count: 20, url: CATALOG_URL}, opts)
+	const termList = [...String(terms).split(',')].filter(Boolean).map(t => t.trim().toLowerCase());
+	const results = await Promise.allSettled(termList.map(t => fetchEntries(t, opts.lang, opts.url)));
+	const byName = new Map(); // name -> {entry, hitTerms:Set}
+	results.forEach((r, i) => {
+		if (r.status !== 'fulfilled') return;
+		const term = termList[i];
+		for (const entry of r.value) {
+			if (!entry.name) continue;
+			if (!byName.has(entry.name)) byName.set(entry.name, {entry, hitTerms: new Set()});
+			byName.get(entry.name).hitTerms.add(term);
+		}
+	});
+	if (!byName.size) return [];
+
+	return [...byName.values()].map(({entry, hitTerms}) => {
+		const text = `${entry.title} ${entry.summary}`.trim();
+		return {entry, hits: hitTerms.size, fuzzy: fuzzyMatch(text, ...termList).max};
+	}).toSorted((a, b) =>
+		b.hits - a.hits || b.fuzzy - a.fuzzy || b.entry.articleCount - a.entry.articleCount
+	).slice(0, opts.count).map(r => r.entry);
+}