import fs from 'node:fs'; import path from 'node:path'; import {pipeline} from 'node:stream/promises'; import {Readable} from 'node:stream'; import {ZimReader} from './reader.js'; import {zimCatalog, zimCatalogInfo, CATALOG_URL} from './catalog.js'; const DEFAULT_READER_TTL = 60_000; /** Manages a local directory of ZIM archives: listing, update checks, downloads, and reading. */ export class ZimManager { #catalog; #dir; #readerTTL; #clusterTTL; #readers = new Map(); // file -> {reader, timer} #opening = new Map(); // file -> Promise, dedupes concurrent first-open races /** @param {{catalog?: string, readerTTL?: number, clusterTTL?: number}} [opts] */ constructor(dir, catalog = CATALOG_URL, {readerTTL = DEFAULT_READER_TTL, clusterTTL} = {}) { this.#catalog = catalog; this.#dir = dir; this.#readerTTL = readerTTL; this.#clusterTTL = clusterTTL; // undefined -> ZimReader's own default } async #download(url, destPath) { const {res} = await this.#resolveUrl(url); if (!res.ok) throw new Error(`${res.status} ${res.statusText}`); const tmpPath = `${destPath}.part`; await pipeline(Readable.fromWeb(res.body), fs.createWriteStream(tmpPath)); await fs.promises.rename(tmpPath, destPath); } async #ensureDir() { await fs.promises.mkdir(this.#dir, {recursive: true}); } /** Resolves a `.meta4` metalink URL down to the real mirror `.zim` download URL. */ async #resolveUrl(url) { const head = await fetch(url); const ct = head.headers.get('content-type') || ''; if (!ct.includes('metalink') && !url.endsWith('.meta4')) return {res: head, url}; const meta = await head.text(); const m = meta.match(/]*>([^<]+\.zim)<\/url>/); if (!m) throw new Error('Could not resolve metalink mirror'); const res = await fetch(m[1]); return {res, url: m[1]}; } /** Reads Name/Date/Title metadata from a local ZIM file. Returns null if unreadable. */ async #readMeta(filepath) { let reader; try { reader = await new ZimReader(filepath).open(); return await reader.metadata(); } catch { return null; } finally { await reader?.close(); } } async #update(name, catalogEntry, localMatch, force) { const remoteDate = catalogEntry.updated ? new Date(catalogEntry.updated) : null; const localDate = localMatch?.meta?.updated ?? null; if (!force && localMatch && remoteDate && localDate && remoteDate <= localDate) return {name, status: 'skipped', reason: 'up to date'}; if (!catalogEntry.download) return {name, status: 'skipped', reason: 'missing download link'}; const filename = path.basename(new URL(catalogEntry.download).pathname).replace(/\.meta4$/i, ''); const destPath = path.join(this.#dir, filename); await this.#download(catalogEntry.download, destPath); if (localMatch && localMatch.file !== filename) await this.#evict(path.join(this.#dir, localMatch.file)); return {name, status: 'updated', file: filename}; } /** Resolves a file path or catalog `name` to a local file path. */ async #resolveFile(fileOrName) { if (fs.existsSync(fileOrName)) return fileOrName; const joined = path.join(this.#dir, fileOrName); if (fs.existsSync(joined)) return joined; const local = await this.list(); const match = local.find(l => l.meta?.name === fileOrName || path.basename(l.file) === fileOrName); if (!match) throw new Error(`ZIM not found locally: ${fileOrName}`); return path.join(this.#dir, match.file); } /** Closes and drops a cached reader for `file`, if any (used before delete/replace). */ async #evict(file) { const entry = this.#readers.get(file); if (!entry) return new ZimReader(file).delete().catch(() => {}); clearTimeout(entry.timer); this.#readers.delete(file); await entry.reader.delete(); } catalog(search, opts) { return zimCatalog(search, opts); } async delete(fileOrName) { const file = await this.#resolveFile(fileOrName); await this.#evict(file); return {file: path.basename(file), status: 'deleted'}; } /** Checks whether a local ZIM has a newer version in the catalog, without downloading. */ async isOutdated(file) { const meta = await this.#readMeta(file); if (!meta?.name) return {file, upToDate: null, reason: 'no metadata'}; const catalogEntry = await zimCatalog(meta.name, this.#catalog); if (!catalogEntry) return {file, upToDate: null, reason: 'missing from catalog'}; const remoteDate = catalogEntry.updated ? new Date(catalogEntry.updated) : null; const localDate = meta.updated ?? null; return {file, name: meta.name, upToDate: !!(remoteDate && localDate && remoteDate <= localDate), localDate, remoteDate}; } /** Lists local `.zim` files with their parsed metadata (or `null` if unreadable). */ async list() { await this.#ensureDir(); const files = (await fs.promises.readdir(this.#dir)).filter(f => f.endsWith('.zim')); return Promise.all(files.map(async f => { const file = path.join(this.#dir, f); return {file: f, ...(await this.#readMeta(file))}; })); } /** Downloads/updates a single ZIM by direct href, matching against any existing local copy by name. */ async download(href, {force = false} = {}) { await this.#ensureDir(); const {url: finalUrl} = await this.#resolveUrl(href); const filename = path.basename(new URL(finalUrl).pathname).replace(/\.meta4$/i, ''); const name = filename.replace(/\.zim$/i, '').replace(/_\d{4}-\d{2}(?:_\d+)?$/, ''); const local = await this.list(); const localMatch = local.find(l => l.meta?.name === name) ?? null; const catalogEntry = await zimCatalogInfo(name, this.#catalog) || {name, updated: null, download: href}; return this.#update(name, catalogEntry, localMatch, force); } /** Checks all local ZIMs against the catalog and updates any that are outdated. */ async updateAll({force = false}) { const local = await this.list(); if (!local.length) return []; const results = []; for (const {file, meta} of local) { if (!meta?.name) { results.push({file, status: 'skipped', reason: 'no metadata'}); continue; } const catalogEntry = await zimCatalogInfo(meta.name, this.#catalog); if (!catalogEntry) { results.push({name: meta.name, status: 'skipped', reason: 'missing from catalog'}); continue; } try { results.push(await this.#update(meta.name, catalogEntry, {file, meta}, force)); } catch (e) { results.push({name: meta.name, status: 'error', reason: e.message}); } } return results; } /** Opens a fresh `ZimReader` for a local ZIM, resolved by file path or catalog `name`. Caller must `.close()` it. */ async open(fileOrName) { await this.#ensureDir(); const file = await this.#resolveFile(fileOrName); let entry = this.#readers.get(file); if (!entry) { let pending = this.#opening.get(file); if (!pending) { pending = new ZimReader(file, {clusterTTL: this.#clusterTTL}).open(); this.#opening.set(file, pending); } let reader; try { reader = await pending; } finally { this.#opening.delete(file); } entry = this.#readers.get(file) ?? {reader}; this.#readers.set(file, entry); } clearTimeout(entry.timer); entry.timer = setTimeout(() => { this.#readers.delete(file); entry.reader.close(); }, this.#readerTTL).unref(); return entry.reader; } /** Fuzzy-searches titles across every local ZIM in the library, merging & re-ranking hits by score. */ async search(terms, {limit = 20, htmlOnly = true} = {}) { const local = await this.list(); const perZim = await Promise.all(local.map(async ({file, meta}) => { let reader; try { reader = await new ZimReader(path.join(this.#dir, file)).open(); const hits = await reader.search(terms, {limit, htmlOnly}); return hits.map(h => ({...h, file})); } catch { return []; } finally { await reader?.close(); } })); return perZim.flat().filter(a => a.score > 0).toSorted((a, b) => b.score - a.score).slice(0, limit); } }