Better searching

package.json

@@ -1,6 +1,6 @@
 {
 	"name": "@ztimson/zim-utils",
-	"version": "0.1.3",
+	"version": "0.1.4",
 	"description": "Native, dependency-light ZIM archive reader/searcher and Kiwix catalog downloader for Node.js",
 	"author": "Zak Timson",
 	"license": "MIT",

src/manager.js

@@ -4,6 +4,7 @@ import {pipeline} from 'node:stream/promises';
 import {Readable} from 'node:stream';
 import {ZimReader} from './reader.js';
 import {zimCatalog, zimCatalogInfo, CATALOG_URL} from './catalog.js';
+import {fuzzyMatch, titleFromUrl} from './utils.js';
 
 /** Manages a local directory of ZIM archives: listing, update checks, downloads, and reading. */
 export class ZimManager {
@@ -146,7 +147,7 @@ export class ZimManager {
 	}
 
 	/** Fuzzy-searches titles across every local ZIM in the library, merging & re-ranking hits by score. */
-	async search(terms, {limit = 20, htmlOnly = true} = {}) {
+	async search(terms, {limit = 10, htmlOnly = true} = {}) {
 		const local = await this.list();
 		const perZim = await Promise.all(local.map(async ({file, meta}) => {
 			let reader;
@@ -160,6 +161,6 @@ export class ZimManager {
 				await reader?.close();
 			}
 		}));
-		return perZim.flat().toSorted((a, b) => b.score - a.score).slice(0, limit);
+		return perZim.flat().filter(a => a.score > 0).toSorted((a, b) => b.score - a.score).slice(0, limit);
 	}
 }

src/reader.js

@@ -3,7 +3,7 @@
 import fs from 'node:fs';
 import {decompress as lzmaDecompress} from 'lzma1';
 import {ZstdCodec} from 'zstd-codec';
-import {fuzzyMatch} from './utils.js';
+import {fuzzyMatch, titleFromUrl} from './utils.js';
 
 const HEADER_SIZE = 80;
 const NS_CONTENT = 'C';
@@ -219,7 +219,7 @@ export class ZimReader {
 		const termList = String(terms).split(',').map(t => t.trim()).filter(Boolean);
 		if (!termList.length) return [];
 
-		const candidates = this.#hasTitleListing
+		const candidates = (this.#hasTitleListing && this.#header.articleCount > 5000)
 			? await this.#titleIndexCandidates(termList[0])
 			: await this.#allDirents();
 
@@ -227,9 +227,16 @@ export class ZimReader {
 		for (const dirent of candidates) {
 			if (dirent.namespace !== NS_CONTENT) continue;
 			if (htmlOnly && !(this.#mimeTypes[dirent.mimetype] || '').startsWith('text/html')) continue;
-			const {max} = fuzzyMatch(dirent.title, ...termList);
+			const urlTitle = titleFromUrl(dirent.url);
-			scored.push({url: dirent.url, title: dirent.title, namespace: dirent.namespace, score: max});
+			const titleScore = fuzzyMatch(dirent.title, ...termList).max;
+			const urlScore = fuzzyMatch(urlTitle, ...termList).max;
+			scored.push({
+				url: dirent.url,
+				title: dirent.title.length > urlTitle.length ? dirent.title : urlTitle,
+				namespace: dirent.namespace,
+				score: Math.max(titleScore, urlScore)
+			});
 		}
-		return scored.toSorted((a, b) => b.score - a.score).slice(0, limit);
+		return scored.filter(a => a.score > 0).toSorted((a, b) => b.score - a.score).slice(0, limit);
 	}
 }

src/utils.js

@@ -20,13 +20,42 @@ export function similarity(a, b) {
 	return 1 - levenshtein(a, b) / Math.max(a.length, b.length, 1);
 }
 
+/**
+ * Scores `text` against a single lowercased `term`. Substring containment always wins.
+ * Otherwise, only rescues genuine typos: requires the same leading letter and a tight
+ * absolute edit-distance cap, so unrelated words can't win purely on coincidental
+ * letter overlap (e.g. "Viennese" vs "diannes").
+ */
+function scoreAgainst(text, term) {
+	if (text.includes(term)) return 1 - (text.length - term.length) / text.length * 0.3;
+	if (!text.length || !term.length || text[0] !== term[0]) return 0;
+	const dist = levenshtein(text, term);
+	const maxAllowed = Math.max(1, Math.ceil(term.length * 0.34));
+	if (dist > maxAllowed) return 0;
+	return 1 - dist / Math.max(text.length, term.length);
+}
+
 /** Compares `target` against one or more search terms; returns avg/max/per-term similarity. */
 export function fuzzyMatch(target, ...terms) {
 	if (!terms.length) throw new Error('Requires at least 1 term to compare');
-	const similarities = terms.map(t => similarity(target, t));
+	const lowerTarget = String(target).toLowerCase();
+	const words = lowerTarget.split(/\W+/).filter(Boolean);
+
+	const similarities = terms.map(term => {
+		const t = term.toLowerCase();
+		return Math.max(scoreAgainst(lowerTarget, t), ...words.map(w => scoreAgainst(w, t)));
+	});
+
 	return {
 		avg: similarities.reduce((acc, s) => acc + s, 0) / similarities.length,
 		max: Math.max(...similarities),
 		similarities,
 	};
 }
+
+/** Derives a readable pseudo-title from a URL's last path segment, e.g. ".../diannes-southwest-salad/" -> "Diannes Southwest Salad". */
+export function titleFromUrl(url) {
+	const slug = String(url).replace(/\/$/, '').split('/').pop() || url;
+	const clean = slug.replace(/\.(zim|meta4|html?|md)$/i, '').replace(/[-_.]+/g, ' ');
+	return clean.replace(/\b\w/g, c => c.toUpperCase());
+}