zim-utils/src/manager.js

import fs from 'node:fs';
import path from 'node:path';
import {pipeline} from 'node:stream/promises';
import {Readable} from 'node:stream';
import {ZimReader} from './reader.js';
import {zimCatalog, zimCatalogInfo, CATALOG_URL} from './catalog.js';

const DEFAULT_READER_TTL = 60_000;

/** Manages a local directory of ZIM archives: listing, update checks, downloads, and reading. */
export class ZimManager {
	#catalog;
	#dir;
	#readerTTL;
	#clusterTTL;
	#readers = new Map(); // file -> {reader, timer}
	#opening = new Map(); // file -> Promise<ZimReader>, dedupes concurrent first-open races

	/** @param {{catalog?: string, readerTTL?: number, clusterTTL?: number}} [opts] */
	constructor(dir, catalog = CATALOG_URL, {readerTTL = DEFAULT_READER_TTL, clusterTTL} = {}) {
		this.#catalog = catalog;
		this.#dir = dir;
		this.#readerTTL = readerTTL;
		this.#clusterTTL = clusterTTL; // undefined -> ZimReader's own default
	}

	async #download(url, destPath) {
		const {res} = await this.#resolveUrl(url);
		if (!res.ok) throw new Error(`${res.status} ${res.statusText}`);
		const tmpPath = `${destPath}.part`;
		await pipeline(Readable.fromWeb(res.body), fs.createWriteStream(tmpPath));
		await fs.promises.rename(tmpPath, destPath);
	}

	async #ensureDir() {
		await fs.promises.mkdir(this.#dir, {recursive: true});
	}

	/** Resolves a `.meta4` metalink URL down to the real mirror `.zim` download URL. */
	async #resolveUrl(url) {
		const head = await fetch(url);
		const ct = head.headers.get('content-type') || '';
		if (!ct.includes('metalink') && !url.endsWith('.meta4')) return {res: head, url};
		const meta = await head.text();
		const m = meta.match(/<url[^>]*>([^<]+\.zim)<\/url>/);
		if (!m) throw new Error('Could not resolve metalink mirror');
		const res = await fetch(m[1]);
		return {res, url: m[1]};
	}

	/** Reads Name/Date/Title metadata from a local ZIM file. Returns null if unreadable. */
	async #readMeta(filepath) {
		let reader;
		try {
			reader = await new ZimReader(filepath).open();
			return await reader.metadata();
		} catch {
			return null;
		} finally {
			await reader?.close();
		}
	}

	async #update(name, catalogEntry, localMatch, force) {
		const remoteDate = catalogEntry.updated ? new Date(catalogEntry.updated) : null;
		const localDate = localMatch?.meta?.updated ?? null;
		if (!force && localMatch && remoteDate && localDate && remoteDate <= localDate)
			return {name, status: 'skipped', reason: 'up to date'};
		if (!catalogEntry.download) return {name, status: 'skipped', reason: 'missing download link'};

		const filename = path.basename(new URL(catalogEntry.download).pathname).replace(/\.meta4$/i, '');
		const destPath = path.join(this.#dir, filename);
		await this.#download(catalogEntry.download, destPath);
		if (localMatch && localMatch.file !== filename) await this.#evict(path.join(this.#dir, localMatch.file));
		return {name, status: 'updated', file: filename};
	}

	/** Resolves a file path or catalog `name` to a local file path. */
	async #resolveFile(fileOrName) {
		if (fs.existsSync(fileOrName)) return fileOrName;
		const joined = path.join(this.#dir, fileOrName);
		if (fs.existsSync(joined)) return joined;
		const local = await this.list();
		const match = local.find(l => l.meta?.name === fileOrName || path.basename(l.file) === fileOrName);
		if (!match) throw new Error(`ZIM not found locally: ${fileOrName}`);
		return path.join(this.#dir, match.file);
	}

	/** Closes and drops a cached reader for `file`, if any (used before delete/replace). */
	async #evict(file) {
		const entry = this.#readers.get(file);
		if (!entry) return new ZimReader(file).delete().catch(() => {});
		clearTimeout(entry.timer);
		this.#readers.delete(file);
		await entry.reader.delete();
	}

	catalog(search, opts) {
		return zimCatalog(search, opts);
	}

	async delete(fileOrName) {
		const file = await this.#resolveFile(fileOrName);
		await this.#evict(file);
		return {file: path.basename(file), status: 'deleted'};
	}

	/** Checks whether a local ZIM has a newer version in the catalog, without downloading. */
	async isOutdated(file) {
		const meta = await this.#readMeta(file);
		if (!meta?.name) return {file, upToDate: null, reason: 'no metadata'};
		const catalogEntry = await zimCatalog(meta.name, this.#catalog);
		if (!catalogEntry) return {file, upToDate: null, reason: 'missing from catalog'};
		const remoteDate = catalogEntry.updated ? new Date(catalogEntry.updated) : null;
		const localDate = meta.updated ?? null;
		return {file, name: meta.name, upToDate: !!(remoteDate && localDate && remoteDate <= localDate), localDate, remoteDate};
	}

	/** Lists local `.zim` files with their parsed metadata (or `null` if unreadable). */
	async list() {
		await this.#ensureDir();
		const files = (await fs.promises.readdir(this.#dir)).filter(f => f.endsWith('.zim'));
		return Promise.all(files.map(async f => {
			const file = path.join(this.#dir, f);
			return {file: f, ...(await this.#readMeta(file))};
		}));
	}

	/** Downloads/updates a single ZIM by direct href, matching against any existing local copy by name. */
	async download(href, {force = false} = {}) {
		await this.#ensureDir();
		const {url: finalUrl} = await this.#resolveUrl(href);
		const filename = path.basename(new URL(finalUrl).pathname).replace(/\.meta4$/i, '');
		const name = filename.replace(/\.zim$/i, '').replace(/_\d{4}-\d{2}(?:_\d+)?$/, '');

		const local = await this.list();
		const localMatch = local.find(l => l.meta?.name === name) ?? null;
		const catalogEntry = await zimCatalogInfo(name, this.#catalog) || {name, updated: null, download: href};
		return this.#update(name, catalogEntry, localMatch, force);
	}

	/** Checks all local ZIMs against the catalog and updates any that are outdated. */
	async updateAll({force = false}) {
		const local = await this.list();
		if (!local.length) return [];

		const results = [];
		for (const {file, meta} of local) {
			if (!meta?.name) { results.push({file, status: 'skipped', reason: 'no metadata'}); continue; }
			const catalogEntry = await zimCatalogInfo(meta.name, this.#catalog);
			if (!catalogEntry) { results.push({name: meta.name, status: 'skipped', reason: 'missing from catalog'}); continue; }
			try {
				results.push(await this.#update(meta.name, catalogEntry, {file, meta}, force));
			} catch (e) {
				results.push({name: meta.name, status: 'error', reason: e.message});
			}
		}
		return results;
	}

	/** Opens a fresh `ZimReader` for a local ZIM, resolved by file path or catalog `name`. Caller must `.close()` it. */
	async open(fileOrName) {
		await this.#ensureDir();
		const file = await this.#resolveFile(fileOrName);
		let entry = this.#readers.get(file);
		if (!entry) {
			let pending = this.#opening.get(file);
			if (!pending) {
				pending = new ZimReader(file, {clusterTTL: this.#clusterTTL}).open();
				this.#opening.set(file, pending);
			}
			let reader;
			try {
				reader = await pending;
			} finally {
				this.#opening.delete(file);
			}
			entry = this.#readers.get(file) ?? {reader};
			this.#readers.set(file, entry);
		}
		clearTimeout(entry.timer);
		entry.timer = setTimeout(() => {
			this.#readers.delete(file);
			entry.reader.close();
		}, this.#readerTTL).unref();
		return entry.reader;
	}

	/** Fuzzy-searches titles across every local ZIM in the library, merging & re-ranking hits by score. */
	async search(terms, {limit = 20, htmlOnly = true} = {}) {
		const local = await this.list();
		const perZim = await Promise.all(local.map(async ({file, meta}) => {
			let reader;
			try {
				reader = await new ZimReader(path.join(this.#dir, file)).open();
				const hits = await reader.search(terms, {limit, htmlOnly});
				return hits.map(h => ({...h, file}));
			} catch {
				return [];
			} finally {
				await reader?.close();
			}
		}));
		return perZim.flat().filter(a => a.score > 0).toSorted((a, b) => b.score - a.score).slice(0, limit);
	}
}