import fs from 'node:fs'; import path from 'node:path'; import {pipeline} from 'node:stream/promises'; import {Readable} from 'node:stream'; import {PromiseProgress} from '@ztimson/utils'; import {KiwixServer} from './server.js'; import {zimCatalog, zimCatalogInfo, CATALOG_URL} from './catalog.js'; const VISIBLE_TIMEOUT = 5_000; const VISIBLE_POLL_INTERVAL = 200; /** Manages a local directory of ZIM archives: catalog search, downloads, update checks, deletion. * Reuses (or owns & lazily starts) a KiwixServer for local listing/search, so `list()` and `catalog()` return the same shape. * Doesn't drive library.xml/kiwix-serve reloads itself - the owning KiwixServer watches its directory and * reloads automatically whenever .zim files change, no matter which ZimManager (local or remote-attached) wrote them. */ export class ZimManager { #catalogUrl; #dir; #server; #ownsServer; #active = {}; // name -> in-flight PromiseProgress, for on-demand status lookups /** @param {{catalog?: string, server?: KiwixServer, port?: number, host?: string, binDir?: string, url?: string}} [opts] * server: reuse an existing KiwixServer instance directly. * url: attach to an already-running kiwix-serve by address instead (e.g. started elsewhere in your codebase * with no shared reference) - mutually exclusive with `server`, ignored if `server` is given. */ constructor(dir, {catalog = CATALOG_URL, server, port, host, binDir, url} = {}) { this.#catalogUrl = catalog; this.#dir = dir; this.#ownsServer = !server; this.#server = server ?? new KiwixServer(dir, {port, host, binDir, url}); } #stripDate(filename) { return filename.replace(/\.zim$/i, '').replace(/_\d{4}-\d{2}(?:_\d+)?$/, ''); } /** The KiwixServer backing this manager - reuse it directly for content/search access, or pass into another ZimManager. */ get server() { return this.#server; } async #ensureServer() { if (!this.#server.running) await this.#server.start(); return this.#server; } /** Streams `url` to `destPath`, reporting raw bytes/total via `onBytes(size, total)` as chunks arrive. */ async #download(url, destPath, onBytes) { const {res} = await this.#resolveUrl(url); if (!res.ok) throw new Error(`${res.status} ${res.statusText}`); const total = Number(res.headers.get('content-length')) || 0; const tmpPath = `${destPath}.part`; let size = 0; const readable = Readable.fromWeb(res.body); if (onBytes) readable.on('data', chunk => onBytes(size += chunk.length, total)); await pipeline(readable, fs.createWriteStream(tmpPath)); await fs.promises.rename(tmpPath, destPath); } /** Resolves a `.meta4` metalink URL down to the real mirror `.zim` download URL. */ async #resolveUrl(url) { const head = await fetch(url); const ct = head.headers.get('content-type') || ''; if (!ct.includes('metalink') && !url.endsWith('.meta4')) return {res: head, url}; const meta = await head.text(); const m = meta.match(/]*>([^<]+\.zim)<\/url>/); if (!m) throw new Error('Could not resolve metalink mirror'); const res = await fetch(m[1]); return {res, url: m[1]}; } /** Polls list() until `filename` shows up (or disappears, if `expect: false`), so callers get an accurate * status back rather than one that's ahead of what the server has actually picked up yet. The owning * KiwixServer's directory watcher does the real reload work in the background; this just waits for it. */ async #waitUntilVisible(filename, {expect = true, timeout = VISIBLE_TIMEOUT} = {}) { const deadline = Date.now() + timeout; while (Date.now() < deadline) { const local = await this.list(); const present = local.some(l => l.file === filename); if (present === expect) return true; await new Promise(r => setTimeout(r, VISIBLE_POLL_INTERVAL)); } return false; } /** Performs the actual download + post-processing for a resolved catalog entry, forwarding byte progress via `onBytes`. */ async #update(name, catalogEntry, localMatch, force, onBytes) { const remoteDate = catalogEntry.updated ? new Date(catalogEntry.updated) : null; const localDate = localMatch?.updated ?? null; if (!force && localMatch && remoteDate && localDate && remoteDate <= localDate) return {name, status: 'skipped', reason: 'up to date'}; if (!catalogEntry.href) return {name, status: 'skipped', reason: 'missing href'}; const filename = path.basename(new URL(catalogEntry.href).pathname).replace(/\.meta4$/i, ''); const destPath = path.join(this.#dir, filename); await this.#download(catalogEntry.href, destPath, onBytes); if (localMatch && localMatch.file !== filename) await fs.promises.rm(path.join(this.#dir, localMatch.file), {force: true}); await this.#ensureServer(); await this.#waitUntilVisible(filename, {expect: true}); return {name, status: 'updated', file: filename}; } /** Wraps `#update` in a tracked `PromiseProgress`, registered in `#active` under `name` until it settles. */ #trackedUpdate(name, catalogEntry, localMatch, force) { const dl = new PromiseProgress((resolve, reject, prog) => this.#update(name, catalogEntry, localMatch, force, prog).then(resolve, reject)); this.#active[name] = dl; dl.catch(() => {}).finally(() => { if (this.#active[name] === dl) delete this.#active[name]; }); return dl; } catalog(search, opts = {}) { return zimCatalog(search, {url: this.#catalogUrl, ...opts}); } /** Stops the internally-owned KiwixServer, if this manager created its own (no-op if one was passed in). */ async close() { if (this.#ownsServer) await this.#server.stop(); } async delete(nameOrFile) { const local = await this.list(); const match = local.find(l => l.name === nameOrFile || l.href === nameOrFile); if (!match) throw new Error(`ZIM not found locally: ${nameOrFile}`); const file = match.href + (match.href.endsWith('.zim') ? '' : '.zim'); await fs.promises.rm(path.join(this.#dir, file), {force: true}); await this.#ensureServer(); await this.#waitUntilVisible(file, {expect: false}); return {file: match.file, status: 'deleted'}; } /** Downloads/updates a single ZIM by direct href, matching against any existing local copy by name. * Returns a `PromiseProgress` with live `.progress`/`.speed`/`.eta`/etc getters & `onProgress` callback, * while also tracking it internally (see `progress()`/`downloads()`) in case the reference is dropped. */ download(href, {force = false} = {}) { let dl; dl = new PromiseProgress(async (resolve, reject, prog) => { try { const {url: finalUrl} = await this.#resolveUrl(href); const filename = path.basename(new URL(finalUrl).pathname).replace(/\.meta4$/i, ''); await fs.promises.mkdir(this.#dir, {recursive: true}); const local = await this.list(); const exact = local.find(l => l.file === filename); if (exact && !force) return resolve({name: exact.name ?? this.#stripDate(filename), status: 'skipped', reason: 'already downloaded', file: filename}); const name = this.#stripDate(filename); const localMatch = local.find(l => l.name === name) ?? null; const catalogEntry = await zimCatalogInfo(name, this.#catalogUrl) || {name, updated: null, href}; this.#active[name] = dl; dl.catch(() => {}).finally(() => { if (this.#active[name] === dl) delete this.#active[name]; }); resolve(await this.#update(name, catalogEntry, localMatch, force, prog)); } catch (e) { reject(e); } }); return dl; } async link(href) { const server = await this.#ensureServer(); return server.link(href); } /** Checks whether a local ZIM has a newer version in the catalog, without downloading. */ async isOutdated(nameOrFile) { const local = await this.list(); const match = local.find(l => l.name === nameOrFile || l.file === nameOrFile); if (!match?.name) return {file: nameOrFile, upToDate: null, reason: 'no metadata'}; const catalogEntry = await zimCatalogInfo(match.name, this.#catalogUrl); if (!catalogEntry) return {file: match.file, upToDate: null, reason: 'missing from catalog'}; const remoteDate = catalogEntry.updated ? new Date(catalogEntry.updated) : null; return {file: match.file, name: match.name, upToDate: !!(remoteDate && match.updated && remoteDate <= match.updated), localDate: match.updated, remoteDate}; } /** Lists local ZIMs - same flat shape as `catalog()` (plus `file`), sourced from the KiwixServer's own library.xml. */ async list() { const server = await this.#ensureServer(); return server.list(); } /** Fetches a single asset's raw bytes. */ async raw(href) { const server = await this.#ensureServer(); return server.raw(href); } /** Two-pass fulltext search across every local ZIM, returns enriched results matching catalog/list shape. */ async search(terms, {limit = 20, sources = null} = {}) { const server = await this.#ensureServer(); return server.search(terms, {limit, sources}); } /** Checks all local ZIMs against the catalog and updates any that are outdated. * Each in-flight update is individually tracked in `#active` - use `progress(name)`/`downloads()` to poll them mid-run. */ async updateAll({force = false} = {}) { const local = await this.list(); const results = []; for (const entry of local) { if (!entry.name) { results.push({file: entry.file, status: 'skipped', reason: 'no metadata'}); continue; } const catalogEntry = await zimCatalogInfo(entry.name, this.#catalogUrl); if (!catalogEntry) { results.push({name: entry.name, status: 'skipped', reason: 'missing from catalog'}); continue; } try { results.push(await this.#trackedUpdate(entry.name, catalogEntry, entry, force)); } catch (e) { results.push({name: entry.name, status: 'error', reason: e.message}); } } return results; } /** Live progress snapshot (speed/eta/size/total/...) for a single named download, or `null` if nothing's in flight. */ progress(name) { return this.#active[name] ? {name, ...this.#active[name].meta} : null; } /** Live progress snapshots for every currently active download. */ downloads() { return Object.entries(this.#active).map(([name, dl]) => ({name, ...dl.meta})); } }