3 Commits
0.3.2 ... 0.3.5

Author SHA1 Message Date
0f8637f913 Ensure library.xml is setup
All checks were successful
Publish Library / Build NPM Project (push) Successful in 45s
Publish Library / Tag Version (push) Successful in 8s
2026-08-26 18:41:15 -04:00
57cb9c8bd1 Fixed delete by href
All checks were successful
Publish Library / Build NPM Project (push) Successful in 44s
Publish Library / Tag Version (push) Successful in 26s
2026-08-25 01:00:28 -04:00
7a2f17bfe9 Use kiwix search completely, no fuzzy ranking
All checks were successful
Publish Library / Build NPM Project (push) Successful in 16s
Publish Library / Tag Version (push) Successful in 10s
2026-08-25 00:42:59 -04:00
4 changed files with 32 additions and 32 deletions

View File

@@ -1,6 +1,6 @@
{ {
"name": "@ztimson/zim-utils", "name": "@ztimson/zim-utils",
"version": "0.3.2", "version": "0.3.5",
"description": "Native, dependency-light ZIM archive reader/searcher and Kiwix catalog downloader for Node.js", "description": "Native, dependency-light ZIM archive reader/searcher and Kiwix catalog downloader for Node.js",
"author": "Zak Timson", "author": "Zak Timson",
"license": "MIT", "license": "MIT",

View File

@@ -79,9 +79,10 @@ export class ZimManager {
async delete(nameOrFile) { async delete(nameOrFile) {
const local = await this.list(); const local = await this.list();
const match = local.find(l => l.name === nameOrFile || l.file === nameOrFile); const match = local.find(l => l.name === nameOrFile || l.href === nameOrFile);
if (!match) throw new Error(`ZIM not found locally: ${nameOrFile}`); if (!match) throw new Error(`ZIM not found locally: ${nameOrFile}`);
await fs.promises.rm(path.join(this.#dir, match.file), {force: true}); const file = match.href + (match.href.endsWith('.zim') ? '' : '.zim');
await fs.promises.rm(path.join(this.#dir, file), {force: true});
const server = await this.#ensureServer(); const server = await this.#ensureServer();
await server.reload(); await server.reload();
return {file: match.file, status: 'deleted'}; return {file: match.file, status: 'deleted'};
@@ -89,9 +90,9 @@ export class ZimManager {
/** Downloads/updates a single ZIM by direct href, matching against any existing local copy by name. */ /** Downloads/updates a single ZIM by direct href, matching against any existing local copy by name. */
async download(href, {force = false} = {}) { async download(href, {force = false} = {}) {
await fs.promises.mkdir(this.#dir, {recursive: true});
const {url: finalUrl} = await this.#resolveUrl(href); const {url: finalUrl} = await this.#resolveUrl(href);
const filename = path.basename(new URL(finalUrl).pathname).replace(/\.meta4$/i, ''); const filename = path.basename(new URL(finalUrl).pathname).replace(/\.meta4$/i, '');
await fs.promises.mkdir(this.#dir, {recursive: true});
const name = filename.replace(/\.zim$/i, '').replace(/_\d{4}-\d{2}(?:_\d+)?$/, ''); const name = filename.replace(/\.zim$/i, '').replace(/_\d{4}-\d{2}(?:_\d+)?$/, '');
const local = await this.list(); const local = await this.list();

View File

@@ -6,8 +6,7 @@ import net from 'node:net';
import fs from 'node:fs'; import fs from 'node:fs';
import path from 'node:path'; import path from 'node:path';
import {fileURLToPath} from 'node:url'; import {fileURLToPath} from 'node:url';
import {decodeHtml, fromXml} from '@ztimson/utils'; import {fromXml} from '@ztimson/utils';
import {fuzzyMatch, weightedScore} from './utils.js';
const execFileAsync = promisify(execFile); const execFileAsync = promisify(execFile);
const __dirname = path.dirname(fileURLToPath(import.meta.url)); const __dirname = path.dirname(fileURLToPath(import.meta.url));
@@ -28,6 +27,8 @@ function findFreePort() {
/** Owns a kiwix-serve process's full lifecycle: library.xml, start/stop/reload, content + search access. */ /** Owns a kiwix-serve process's full lifecycle: library.xml, start/stop/reload, content + search access. */
export class KiwixServer { export class KiwixServer {
static #empty = '<?xml version="1.0" encoding="UTF-8" ?>\n<library version="20110515"></library>\n';
#dir; #dir;
#host; #host;
#port; #port;
@@ -50,6 +51,12 @@ export class KiwixServer {
this.#binDir = binDir; this.#binDir = binDir;
this.#libraryPath = path.join(dir, 'library.xml'); this.#libraryPath = path.join(dir, 'library.xml');
this.#remote = url ? url.replace(/\/$/, '') : null; this.#remote = url ? url.replace(/\/$/, '') : null;
if (!this.#remote) this.#ensureLocalStore();
}
#ensureLocalStore() {
fs.mkdirSync(this.#dir, {recursive: true});
if (!fs.existsSync(this.#libraryPath)) fs.writeFileSync(this.#libraryPath, KiwixServer.#empty);
} }
#assertRunning() { #assertRunning() {
@@ -62,15 +69,22 @@ export class KiwixServer {
/** Reads library.xml from disk if we own the server, or over HTTP if attached to a remote one. */ /** Reads library.xml from disk if we own the server, or over HTTP if attached to a remote one. */
async #fetchLibraryXml() { async #fetchLibraryXml() {
if (this.#remote) return (await fetch(`${this.#remote}/library.xml`).catch(() => null))?.text?.() ?? ''; if (!this.#remote) return fs.promises.readFile(this.#libraryPath, 'utf8').catch(() => '');
return fs.promises.readFile(this.#libraryPath, 'utf8').catch(() => ''); try {
const res = await fetch(`${this.#remote}/library.xml`);
return res.ok ? await res.text() : '';
} catch {
return '';
}
} }
/** Rebuilds library.xml from scratch by scanning `dir` for .zim files - no-op if attached to a remote server. */ /** Rebuilds library.xml from scratch by scanning `dir` for .zim files - no-op if attached to a remote server. */
async #rebuildLibrary() { async #rebuildLibrary() {
if (this.#remote) return; if (this.#remote) return;
await fs.promises.rm(this.#libraryPath, {force: true}); await fs.promises.rm(this.#libraryPath, {force: true});
for (const f of await this.#zimFiles()) await execFileAsync(this.#bin('kiwix-manage'), [this.#libraryPath, 'add', path.join(this.#dir, f)]); const files = await this.#zimFiles();
if (!files.length) return fs.promises.writeFile(this.#libraryPath, KiwixServer.#empty);
for (const f of files) await execFileAsync(this.#bin('kiwix-manage'), [this.#libraryPath, 'add', path.join(this.#dir, f)]);
} }
async #waitUntilReady() { async #waitUntilReady() {
@@ -134,6 +148,7 @@ export class KiwixServer {
async list() { async list() {
this.#assertRunning(); this.#assertRunning();
const xml = await this.#fetchLibraryXml(); const xml = await this.#fetchLibraryXml();
if (!xml) return [];
const entries = fromXml(xml); const entries = fromXml(xml);
return (entries?.library?.book || []).map(e => { return (entries?.library?.book || []).map(e => {
const tags = e.tags.split(';'); const tags = e.tags.split(';');
@@ -180,30 +195,23 @@ export class KiwixServer {
return {mimetype: res.headers.get('content-type'), data: Buffer.from(await res.arrayBuffer())}; return {mimetype: res.headers.get('content-type'), data: Buffer.from(await res.arrayBuffer())};
} }
/** /** Fulltext search across every local ZIM via kiwix-serve's own xapian index */
* Two-pass fulltext search across every local ZIM: xapian prefilter (kiwix-serve's
* own index), then fuzzy-reranked by title so the strongest matches surface first.
* Returns a flat array matching the catalog/list shape: {id, title, name, category, ..., href, score}
*/
async search(terms, limit = 20) { async search(terms, limit = 20) {
this.#assertRunning(); this.#assertRunning();
const termList = String(terms).split(',').map(t => t.trim()).filter(Boolean); const termList = String(terms).split(/[,\s]+/).map(t => t.trim()).filter(Boolean);
if (!termList.length) return []; if (!termList.length) return [];
const params = new URLSearchParams({pattern: termList.join(' '), format: 'xml', pageLength: String(limit)}); const params = new URLSearchParams({pattern: termList.join(' '), format: 'xml', pageLength: String(limit)});
const res = await fetch(`${this.baseUrl}/search?${params}`); const res = await fetch(`${this.baseUrl}/search?${params}`);
if (!res.ok) return []; if (!res.ok) return [];
const xml = await res.text(); const found = fromXml(await res.text())?.rss?.channel?.item || [];
let found = fromXml(xml);
found = found?.rss?.channel?.item || [];
const books = await this.list(); const books = await this.list();
const bookMap = new Map(books.map(b => [b.title, b])); const bookMap = new Map(books.map(b => [b.title, b]));
const enriched = found.map(hit => { return found.map(hit => {
const book = bookMap.get(hit.book.title); const book = bookMap.get(hit.book.title);
if(!book) return null; if (!book) return null;
const prefix = `/content/${book.href}/`; const prefix = `/content/${book.href}/`;
const page = hit.link.startsWith(prefix) ? hit.link.slice(prefix.length) : hit.link.replace(/^\/+/, ''); const page = hit.link.startsWith(prefix) ? hit.link.slice(prefix.length) : hit.link.replace(/^\/+/, '');
return { return {
@@ -216,10 +224,8 @@ export class KiwixServer {
icon: book.icon, icon: book.icon,
viewer: this.baseUrl + hit.link, viewer: this.baseUrl + hit.link,
summary: hit.description, summary: hit.description,
score: weightedScore(hit.title, termList) + weightedScore(hit.description, termList), score: +hit.score || 0,
}; };
}).filter(hit => !!hit && hit.score > 0); }).filter(Boolean);
return enriched.toSorted((a, b) => b.score - a.score).slice(0, limit);
} }
} }

View File

@@ -40,10 +40,3 @@ export function fuzzyMatch(target, ...terms) {
similarities, similarities,
}; };
} }
export function weightedScore(text, termList) {
if (!text) return 0;
const covered = termList.reduce((sum, t) => sum + t.length, 0);
const coverage = Math.min(1, covered / text.length);
return fuzzyMatch(text, ...termList).max * coverage;
}