From 3515968929c7d5d26122c31ce502319cb0b9caa7 Mon Sep 17 00:00:00 2001 From: ztimson Date: Mon, 5 Oct 2026 18:42:22 -0400 Subject: [PATCH] Merge download progress into local catalog info --- package.json | 2 +- src/manager.js | 85 ++++++++++++++++++++++++++++---------------------- 2 files changed, 49 insertions(+), 38 deletions(-) diff --git a/package.json b/package.json index 729505e..a3e2f6c 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@ztimson/zim-utils", - "version": "0.4.8", + "version": "0.4.9", "description": "Native, dependency-light ZIM archive reader/searcher and Kiwix catalog downloader for Node.js", "author": "Zak Timson", "license": "MIT", diff --git a/src/manager.js b/src/manager.js index 391ce83..2cda52f 100644 --- a/src/manager.js +++ b/src/manager.js @@ -11,14 +11,15 @@ const VISIBLE_POLL_INTERVAL = 200; /** Manages a local directory of ZIM archives: catalog search, downloads, update checks, deletion. * Reuses (or owns & lazily starts) a KiwixServer for local listing/search, so `list()` and `catalog()` return the same shape. - * Doesn't drive library.xml/kiwix-serve reloads itself - the owning KiwixServer watches its directory and - * reloads automatically whenever .zim files change, no matter which ZimManager (local or remote-attached) wrote them. */ + * In-flight downloads/updates are tracked by `href` (unique per catalog entry/flavor - unlike `name`, which collides + * across flavors) and merged directly into `list()` (as synthetic pending entries) & `catalog()` (overlaid onto the + * matching row), so callers never need a separate progress endpoint - just re-poll list()/catalog() like normal. */ export class ZimManager { #catalogUrl; #dir; #server; #ownsServer; - #active = {}; // name -> in-flight PromiseProgress, for on-demand status lookups + #active = {}; // href -> {entry: catalogEntry, dl: PromiseProgress} /** @param {{catalog?: string, server?: KiwixServer, port?: number, host?: string, binDir?: string, url?: string}} [opts] * server: reuse an existing KiwixServer instance directly. @@ -43,6 +44,12 @@ export class ZimManager { return this.#server; } + /** Raw local listing, no pending downloads merged in - used internally for matching against real files on disk. */ + async #rawList() { + const server = await this.#ensureServer(); + return server.list(); + } + /** Streams `url` to `destPath`, reporting raw bytes/total via `onBytes(size, total)` as chunks arrive. */ async #download(url, destPath, onBytes) { const {res} = await this.#resolveUrl(url); @@ -68,13 +75,13 @@ export class ZimManager { return {res, url: m[1]}; } - /** Polls list() until `filename` shows up (or disappears, if `expect: false`), so callers get an accurate + /** Polls #rawList() until `filename` shows up (or disappears, if `expect: false`), so callers get an accurate * status back rather than one that's ahead of what the server has actually picked up yet. The owning * KiwixServer's directory watcher does the real reload work in the background; this just waits for it. */ async #waitUntilVisible(filename, {expect = true, timeout = VISIBLE_TIMEOUT} = {}) { const deadline = Date.now() + timeout; while (Date.now() < deadline) { - const local = await this.list(); + const local = await this.#rawList(); const present = local.some(l => l.file === filename); if (present === expect) return true; await new Promise(r => setTimeout(r, VISIBLE_POLL_INTERVAL)); @@ -83,7 +90,8 @@ export class ZimManager { } /** Performs the actual download + post-processing for a resolved catalog entry, forwarding byte progress via `onBytes`. */ - async #update(name, catalogEntry, localMatch, force, onBytes) { + async #update(catalogEntry, localMatch, force, onBytes) { + const name = catalogEntry.name; const remoteDate = catalogEntry.updated ? new Date(catalogEntry.updated) : null; const localDate = localMatch?.updated ?? null; if (!force && localMatch && remoteDate && localDate && remoteDate <= localDate) @@ -99,17 +107,23 @@ export class ZimManager { return {name, status: 'updated', file: filename}; } - /** Wraps `#update` in a tracked `PromiseProgress`, registered in `#active` under `name` until it settles. */ - #trackedUpdate(name, catalogEntry, localMatch, force) { + /** Wraps `#update` in a tracked `PromiseProgress`, registered in `#active` under the catalog entry's `href`. */ + #trackedUpdate(catalogEntry, localMatch, force) { const dl = new PromiseProgress((resolve, reject, prog) => - this.#update(name, catalogEntry, localMatch, force, prog).then(resolve, reject)); - this.#active[name] = dl; - dl.catch(() => {}).finally(() => { if (this.#active[name] === dl) delete this.#active[name]; }); + this.#update(catalogEntry, localMatch, force, prog).then(resolve, reject)); + this.#active[catalogEntry.href] = {entry: catalogEntry, dl}; + dl.catch(() => {}).finally(() => { if (this.#active[catalogEntry.href]?.dl === dl) delete this.#active[catalogEntry.href]; }); return dl; } - catalog(search, opts = {}) { - return zimCatalog(search, {url: this.#catalogUrl, ...opts}); + /** Remote catalog search - any entry with a matching in-flight download/update gets live progress fields + * (`progress`/`speed`/`eta`/`size`/`total`/`downloading: true`) merged directly onto it by `href`, so distinct + * flavors of the same `name` never cross-light each other. */ + async catalog(search, opts = {}) { + const results = await zimCatalog(search, {url: this.#catalogUrl, ...opts}); + return results.map(entry => this.#active[entry.href] + ? {...entry, ...this.#active[entry.href].dl.meta, downloading: true} + : entry); } /** Stops the internally-owned KiwixServer, if this manager created its own (no-op if one was passed in). */ @@ -118,7 +132,7 @@ export class ZimManager { } async delete(nameOrFile) { - const local = await this.list(); + const local = await this.#rawList(); const match = local.find(l => l.name === nameOrFile || l.href === nameOrFile); if (!match) throw new Error(`ZIM not found locally: ${nameOrFile}`); const file = match.href + (match.href.endsWith('.zim') ? '' : '.zim'); @@ -129,8 +143,8 @@ export class ZimManager { } /** Downloads/updates a single ZIM by direct href, matching against any existing local copy by name. - * Returns a `PromiseProgress` with live `.progress`/`.speed`/`.eta`/etc getters & `onProgress` callback, - * while also tracking it internally (see `progress()`/`downloads()`) in case the reference is dropped. */ + * Returns a `PromiseProgress` with live `.progress`/`.speed`/`.eta`/etc getters & `onProgress` callback; + * also tracked internally under `href` so `list()`/`catalog()` reflect it even if the reference is dropped. */ download(href, {force = false} = {}) { let dl; dl = new PromiseProgress(async (resolve, reject, prog) => { @@ -138,15 +152,15 @@ export class ZimManager { const {url: finalUrl} = await this.#resolveUrl(href); const filename = path.basename(new URL(finalUrl).pathname).replace(/\.meta4$/i, ''); await fs.promises.mkdir(this.#dir, {recursive: true}); - const local = await this.list(); + const local = await this.#rawList(); const exact = local.find(l => l.file === filename); if (exact && !force) return resolve({name: exact.name ?? this.#stripDate(filename), status: 'skipped', reason: 'already downloaded', file: filename}); const name = this.#stripDate(filename); const localMatch = local.find(l => l.name === name) ?? null; const catalogEntry = await zimCatalogInfo(name, this.#catalogUrl) || {name, updated: null, href}; - this.#active[name] = dl; - dl.catch(() => {}).finally(() => { if (this.#active[name] === dl) delete this.#active[name]; }); - resolve(await this.#update(name, catalogEntry, localMatch, force, prog)); + this.#active[href] = {entry: catalogEntry, dl}; + dl.catch(() => {}).finally(() => { if (this.#active[href]?.dl === dl) delete this.#active[href]; }); + resolve(await this.#update(catalogEntry, localMatch, force, prog)); } catch (e) { reject(e); } }); return dl; @@ -159,7 +173,7 @@ export class ZimManager { /** Checks whether a local ZIM has a newer version in the catalog, without downloading. */ async isOutdated(nameOrFile) { - const local = await this.list(); + const local = await this.#rawList(); const match = local.find(l => l.name === nameOrFile || l.file === nameOrFile); if (!match?.name) return {file: nameOrFile, upToDate: null, reason: 'no metadata'}; const catalogEntry = await zimCatalogInfo(match.name, this.#catalogUrl); @@ -168,10 +182,17 @@ export class ZimManager { return {file: match.file, name: match.name, upToDate: !!(remoteDate && match.updated && remoteDate <= match.updated), localDate: match.updated, remoteDate}; } - /** Lists local ZIMs - same flat shape as `catalog()` (plus `file`), sourced from the KiwixServer's own library.xml. */ + /** Local ZIMs - same flat shape as `catalog()` (plus `file`) - with any in-flight downloads/updates appended as + * synthetic pending entries (catalog metadata + live progress fields + `downloading: true`) so the UI always has + * a card to render progress against, even before the file exists on disk. Resolves itself: once the real file + * lands, its name matches the local entry and the synthetic one drops out automatically. */ async list() { - const server = await this.#ensureServer(); - return server.list(); + const local = await this.#rawList(); + const localNames = new Set(local.map(l => l.name)); + const pending = Object.values(this.#active) + .filter(({entry}) => !localNames.has(entry.name)) + .map(({entry, dl}) => ({...entry, ...dl.meta, downloading: true})); + return [...local, ...pending]; } /** Fetches a single asset's raw bytes. */ @@ -187,30 +208,20 @@ export class ZimManager { } /** Checks all local ZIMs against the catalog and updates any that are outdated. - * Each in-flight update is individually tracked in `#active` - use `progress(name)`/`downloads()` to poll them mid-run. */ + * Each in-flight update is tracked in `#active` by href - reflected live via `list()`/`catalog()` mid-run. */ async updateAll({force = false} = {}) { - const local = await this.list(); + const local = await this.#rawList(); const results = []; for (const entry of local) { if (!entry.name) { results.push({file: entry.file, status: 'skipped', reason: 'no metadata'}); continue; } const catalogEntry = await zimCatalogInfo(entry.name, this.#catalogUrl); if (!catalogEntry) { results.push({name: entry.name, status: 'skipped', reason: 'missing from catalog'}); continue; } try { - results.push(await this.#trackedUpdate(entry.name, catalogEntry, entry, force)); + results.push(await this.#trackedUpdate(catalogEntry, entry, force)); } catch (e) { results.push({name: entry.name, status: 'error', reason: e.message}); } } return results; } - - /** Live progress snapshot (speed/eta/size/total/...) for a single named download, or `null` if nothing's in flight. */ - progress(name) { - return this.#active[name] ? {name, ...this.#active[name].meta} : null; - } - - /** Live progress snapshots for every currently active download. */ - downloads() { - return Object.entries(this.#active).map(([name, dl]) => ({name, ...dl.meta})); - } }