diff --git a/package.json b/package.json index d3cb771..608a248 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@ztimson/zim-utils", - "version": "0.1.3", + "version": "0.1.4", "description": "Native, dependency-light ZIM archive reader/searcher and Kiwix catalog downloader for Node.js", "author": "Zak Timson", "license": "MIT", diff --git a/src/manager.js b/src/manager.js index b19e59c..43549ea 100644 --- a/src/manager.js +++ b/src/manager.js @@ -4,6 +4,7 @@ import {pipeline} from 'node:stream/promises'; import {Readable} from 'node:stream'; import {ZimReader} from './reader.js'; import {zimCatalog, zimCatalogInfo, CATALOG_URL} from './catalog.js'; +import {fuzzyMatch, titleFromUrl} from './utils.js'; /** Manages a local directory of ZIM archives: listing, update checks, downloads, and reading. */ export class ZimManager { @@ -146,7 +147,7 @@ export class ZimManager { } /** Fuzzy-searches titles across every local ZIM in the library, merging & re-ranking hits by score. */ - async search(terms, {limit = 20, htmlOnly = true} = {}) { + async search(terms, {limit = 10, htmlOnly = true} = {}) { const local = await this.list(); const perZim = await Promise.all(local.map(async ({file, meta}) => { let reader; @@ -160,6 +161,6 @@ export class ZimManager { await reader?.close(); } })); - return perZim.flat().toSorted((a, b) => b.score - a.score).slice(0, limit); + return perZim.flat().filter(a => a.score > 0).toSorted((a, b) => b.score - a.score).slice(0, limit); } } diff --git a/src/reader.js b/src/reader.js index 5b3e1e9..edbb6fa 100644 --- a/src/reader.js +++ b/src/reader.js @@ -3,7 +3,7 @@ import fs from 'node:fs'; import {decompress as lzmaDecompress} from 'lzma1'; import {ZstdCodec} from 'zstd-codec'; -import {fuzzyMatch} from './utils.js'; +import {fuzzyMatch, titleFromUrl} from './utils.js'; const HEADER_SIZE = 80; const NS_CONTENT = 'C'; @@ -219,7 +219,7 @@ export class ZimReader { const termList = String(terms).split(',').map(t => t.trim()).filter(Boolean); if (!termList.length) return []; - const candidates = this.#hasTitleListing + const candidates = (this.#hasTitleListing && this.#header.articleCount > 5000) ? await this.#titleIndexCandidates(termList[0]) : await this.#allDirents(); @@ -227,9 +227,16 @@ export class ZimReader { for (const dirent of candidates) { if (dirent.namespace !== NS_CONTENT) continue; if (htmlOnly && !(this.#mimeTypes[dirent.mimetype] || '').startsWith('text/html')) continue; - const {max} = fuzzyMatch(dirent.title, ...termList); - scored.push({url: dirent.url, title: dirent.title, namespace: dirent.namespace, score: max}); + const urlTitle = titleFromUrl(dirent.url); + const titleScore = fuzzyMatch(dirent.title, ...termList).max; + const urlScore = fuzzyMatch(urlTitle, ...termList).max; + scored.push({ + url: dirent.url, + title: dirent.title.length > urlTitle.length ? dirent.title : urlTitle, + namespace: dirent.namespace, + score: Math.max(titleScore, urlScore) + }); } - return scored.toSorted((a, b) => b.score - a.score).slice(0, limit); + return scored.filter(a => a.score > 0).toSorted((a, b) => b.score - a.score).slice(0, limit); } } diff --git a/src/utils.js b/src/utils.js index dd6b566..a0dc334 100644 --- a/src/utils.js +++ b/src/utils.js @@ -20,13 +20,42 @@ export function similarity(a, b) { return 1 - levenshtein(a, b) / Math.max(a.length, b.length, 1); } +/** + * Scores `text` against a single lowercased `term`. Substring containment always wins. + * Otherwise, only rescues genuine typos: requires the same leading letter and a tight + * absolute edit-distance cap, so unrelated words can't win purely on coincidental + * letter overlap (e.g. "Viennese" vs "diannes"). + */ +function scoreAgainst(text, term) { + if (text.includes(term)) return 1 - (text.length - term.length) / text.length * 0.3; + if (!text.length || !term.length || text[0] !== term[0]) return 0; + const dist = levenshtein(text, term); + const maxAllowed = Math.max(1, Math.ceil(term.length * 0.34)); + if (dist > maxAllowed) return 0; + return 1 - dist / Math.max(text.length, term.length); +} + /** Compares `target` against one or more search terms; returns avg/max/per-term similarity. */ export function fuzzyMatch(target, ...terms) { if (!terms.length) throw new Error('Requires at least 1 term to compare'); - const similarities = terms.map(t => similarity(target, t)); + const lowerTarget = String(target).toLowerCase(); + const words = lowerTarget.split(/\W+/).filter(Boolean); + + const similarities = terms.map(term => { + const t = term.toLowerCase(); + return Math.max(scoreAgainst(lowerTarget, t), ...words.map(w => scoreAgainst(w, t))); + }); + return { avg: similarities.reduce((acc, s) => acc + s, 0) / similarities.length, max: Math.max(...similarities), similarities, }; } + +/** Derives a readable pseudo-title from a URL's last path segment, e.g. ".../diannes-southwest-salad/" -> "Diannes Southwest Salad". */ +export function titleFromUrl(url) { + const slug = String(url).replace(/\/$/, '').split('/').pop() || url; + const clean = slug.replace(/\.(zim|meta4|html?|md)$/i, '').replace(/[-_.]+/g, ' '); + return clean.replace(/\b\w/g, c => c.toUpperCase()); +}