Merge download progress into local catalog info

bin/install-kwix.js

@@ -146,6 +146,9 @@ function resolveArchiveName() {
 }
 
 async function install() {
+	if (fs.existsSync(BIN_DIR) && (await fs.promises.readdir(BIN_DIR)).some(f => f.startsWith('kiwix-')))
+		return console.log(`Already installed: ${BIN_DIR}`);
+
 	const {archiveName, ext} = resolveArchiveName();
 	const url = `${BASE_URL}/${archiveName}`;
 	const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'kiwix-install-'));

package-lock.json

@@ -1,17 +1,17 @@
 {
 	"name": "@ztimson/zim-utils",
-	"version": "0.3.6",
+	"version": "0.4.8",
 	"lockfileVersion": 3,
 	"requires": true,
 	"packages": {
 		"": {
 			"name": "@ztimson/zim-utils",
-			"version": "0.3.6",
+			"version": "0.4.8",
 			"hasInstallScript": true,
 			"license": "MIT",
 			"dependencies": {
-				"@ztimson/utils": "^0.30.8",
+				"@ztimson/utils": "^0.30.9",
-				"adm-zip": "^0.5.16",
+				"adm-zip": "^0.5.18",
 				"lzma1": "^0.3.0",
 				"tar": "^7.4.3",
 				"zstd-codec": "^0.1.5"
@@ -30,9 +30,9 @@
 			}
 		},
 		"node_modules/@ztimson/utils": {
-			"version": "0.30.8",
+			"version": "0.30.9",
-			"resolved": "https://registry.npmjs.org/@ztimson/utils/-/utils-0.30.8.tgz",
+			"resolved": "https://registry.npmjs.org/@ztimson/utils/-/utils-0.30.9.tgz",
-			"integrity": "sha512-+vBjcinqckqMHkP95xWiQeQz2E7Q1oS0b+Odjp+F9rvQ4z0US4JdodjqhEaqh+RAO/yP77x4Eu0a04yBB4HNdw==",
+			"integrity": "sha512-th/6qBaRJFnxoQk+01ESXjFqhYzsiXziOzd3gPgFG6XrusA+oHtfgxBKJ77ygPq//0cI85srWgAzvQcAmKUAQw==",
 			"license": "MIT",
 			"dependencies": {
 				"var-persist": "^1.0.1"

package.json

@@ -1,6 +1,6 @@
 {
 	"name": "@ztimson/zim-utils",
-	"version": "0.4.7",
+	"version": "0.4.9",
 	"description": "Native, dependency-light ZIM archive reader/searcher and Kiwix catalog downloader for Node.js",
 	"author": "Zak Timson",
 	"license": "MIT",
@@ -15,10 +15,10 @@
 		"postinstall": "node ./bin/install-kwix.js"
 	},
 	"dependencies": {
-		"@ztimson/utils": "^0.30.8",
+		"@ztimson/utils": "^0.30.9",
 		"lzma1": "^0.3.0",
 		"tar": "^7.4.3",
 		"zstd-codec": "^0.1.5",
-		"adm-zip": "^0.5.16"
+		"adm-zip": "^0.5.18"
 	}
 }

src/manager.js

@@ -11,14 +11,15 @@ const VISIBLE_POLL_INTERVAL = 200;
 
 /** Manages a local directory of ZIM archives: catalog search, downloads, update checks, deletion.
  * Reuses (or owns & lazily starts) a KiwixServer for local listing/search, so `list()` and `catalog()` return the same shape.
- * Doesn't drive library.xml/kiwix-serve reloads itself - the owning KiwixServer watches its directory and
+ * In-flight downloads/updates are tracked by `href` (unique per catalog entry/flavor - unlike `name`, which collides
- * reloads automatically whenever .zim files change, no matter which ZimManager (local or remote-attached) wrote them. */
+ * across flavors) and merged directly into `list()` (as synthetic pending entries) & `catalog()` (overlaid onto the
+ * matching row), so callers never need a separate progress endpoint - just re-poll list()/catalog() like normal. */
 export class ZimManager {
 	#catalogUrl;
 	#dir;
 	#server;
 	#ownsServer;
-	#active = {}; // name -> in-flight PromiseProgress, for on-demand status lookups
+	#active = {}; // href -> {entry: catalogEntry, dl: PromiseProgress}
 
 	/** @param {{catalog?: string, server?: KiwixServer, port?: number, host?: string, binDir?: string, url?: string}} [opts]
 	 * server: reuse an existing KiwixServer instance directly.
@@ -43,6 +44,12 @@ export class ZimManager {
 		return this.#server;
 	}
 
+	/** Raw local listing, no pending downloads merged in - used internally for matching against real files on disk. */
+	async #rawList() {
+		const server = await this.#ensureServer();
+		return server.list();
+	}
+
 	/** Streams `url` to `destPath`, reporting raw bytes/total via `onBytes(size, total)` as chunks arrive. */
 	async #download(url, destPath, onBytes) {
 		const {res} = await this.#resolveUrl(url);
@@ -68,13 +75,13 @@ export class ZimManager {
 		return {res, url: m[1]};
 	}
 
-	/** Polls list() until `filename` shows up (or disappears, if `expect: false`), so callers get an accurate
+	/** Polls #rawList() until `filename` shows up (or disappears, if `expect: false`), so callers get an accurate
 	 * status back rather than one that's ahead of what the server has actually picked up yet. The owning
 	 * KiwixServer's directory watcher does the real reload work in the background; this just waits for it. */
 	async #waitUntilVisible(filename, {expect = true, timeout = VISIBLE_TIMEOUT} = {}) {
 		const deadline = Date.now() + timeout;
 		while (Date.now() < deadline) {
-			const local = await this.list();
+			const local = await this.#rawList();
 			const present = local.some(l => l.file === filename);
 			if (present === expect) return true;
 			await new Promise(r => setTimeout(r, VISIBLE_POLL_INTERVAL));
@@ -83,7 +90,8 @@ export class ZimManager {
 	}
 
 	/** Performs the actual download + post-processing for a resolved catalog entry, forwarding byte progress via `onBytes`. */
-	async #update(name, catalogEntry, localMatch, force, onBytes) {
+	async #update(catalogEntry, localMatch, force, onBytes) {
+		const name = catalogEntry.name;
 		const remoteDate = catalogEntry.updated ? new Date(catalogEntry.updated) : null;
 		const localDate = localMatch?.updated ?? null;
 		if (!force && localMatch && remoteDate && localDate && remoteDate <= localDate)
@@ -99,17 +107,23 @@ export class ZimManager {
 		return {name, status: 'updated', file: filename};
 	}
 
-	/** Wraps `#update` in a tracked `PromiseProgress`, registered in `#active` under `name` until it settles. */
+	/** Wraps `#update` in a tracked `PromiseProgress`, registered in `#active` under the catalog entry's `href`. */
-	#trackedUpdate(name, catalogEntry, localMatch, force) {
+	#trackedUpdate(catalogEntry, localMatch, force) {
 		const dl = new PromiseProgress((resolve, reject, prog) =>
-			this.#update(name, catalogEntry, localMatch, force, prog).then(resolve, reject));
+			this.#update(catalogEntry, localMatch, force, prog).then(resolve, reject));
-		this.#active[name] = dl;
+		this.#active[catalogEntry.href] = {entry: catalogEntry, dl};
-		dl.catch(() => {}).finally(() => { if (this.#active[name] === dl) delete this.#active[name]; });
+		dl.catch(() => {}).finally(() => { if (this.#active[catalogEntry.href]?.dl === dl) delete this.#active[catalogEntry.href]; });
 		return dl;
 	}
 
-	catalog(search, opts = {}) {
+	/** Remote catalog search - any entry with a matching in-flight download/update gets live progress fields
-		return zimCatalog(search, {url: this.#catalogUrl, ...opts});
+	 * (`progress`/`speed`/`eta`/`size`/`total`/`downloading: true`) merged directly onto it by `href`, so distinct
+	 * flavors of the same `name` never cross-light each other. */
+	async catalog(search, opts = {}) {
+		const results = await zimCatalog(search, {url: this.#catalogUrl, ...opts});
+		return results.map(entry => this.#active[entry.href]
+			? {...entry, ...this.#active[entry.href].dl.meta, downloading: true}
+			: entry);
 	}
 
 	/** Stops the internally-owned KiwixServer, if this manager created its own (no-op if one was passed in). */
@@ -118,7 +132,7 @@ export class ZimManager {
 	}
 
 	async delete(nameOrFile) {
-		const local = await this.list();
+		const local = await this.#rawList();
 		const match = local.find(l => l.name === nameOrFile || l.href === nameOrFile);
 		if (!match) throw new Error(`ZIM not found locally: ${nameOrFile}`);
 		const file = match.href + (match.href.endsWith('.zim') ? '' : '.zim');
@@ -129,8 +143,8 @@ export class ZimManager {
 	}
 
 	/** Downloads/updates a single ZIM by direct href, matching against any existing local copy by name.
-	 * Returns a `PromiseProgress` with live `.progress`/`.speed`/`.eta`/etc getters & `onProgress` callback,
+	 * Returns a `PromiseProgress` with live `.progress`/`.speed`/`.eta`/etc getters & `onProgress` callback;
-	 * while also tracking it internally (see `progress()`/`downloads()`) in case the reference is dropped. */
+	 * also tracked internally under `href` so `list()`/`catalog()` reflect it even if the reference is dropped. */
 	download(href, {force = false} = {}) {
 		let dl;
 		dl = new PromiseProgress(async (resolve, reject, prog) => {
@@ -138,15 +152,15 @@ export class ZimManager {
 				const {url: finalUrl} = await this.#resolveUrl(href);
 				const filename = path.basename(new URL(finalUrl).pathname).replace(/\.meta4$/i, '');
 				await fs.promises.mkdir(this.#dir, {recursive: true});
-				const local = await this.list();
+				const local = await this.#rawList();
 				const exact = local.find(l => l.file === filename);
 				if (exact && !force) return resolve({name: exact.name ?? this.#stripDate(filename), status: 'skipped', reason: 'already downloaded', file: filename});
 				const name = this.#stripDate(filename);
 				const localMatch = local.find(l => l.name === name) ?? null;
 				const catalogEntry = await zimCatalogInfo(name, this.#catalogUrl) || {name, updated: null, href};
-				this.#active[name] = dl;
+				this.#active[href] = {entry: catalogEntry, dl};
-				dl.catch(() => {}).finally(() => { if (this.#active[name] === dl) delete this.#active[name]; });
+				dl.catch(() => {}).finally(() => { if (this.#active[href]?.dl === dl) delete this.#active[href]; });
-				resolve(await this.#update(name, catalogEntry, localMatch, force, prog));
+				resolve(await this.#update(catalogEntry, localMatch, force, prog));
 			} catch (e) { reject(e); }
 		});
 		return dl;
@@ -159,7 +173,7 @@ export class ZimManager {
 
 	/** Checks whether a local ZIM has a newer version in the catalog, without downloading. */
 	async isOutdated(nameOrFile) {
-		const local = await this.list();
+		const local = await this.#rawList();
 		const match = local.find(l => l.name === nameOrFile || l.file === nameOrFile);
 		if (!match?.name) return {file: nameOrFile, upToDate: null, reason: 'no metadata'};
 		const catalogEntry = await zimCatalogInfo(match.name, this.#catalogUrl);
@@ -168,10 +182,17 @@ export class ZimManager {
 		return {file: match.file, name: match.name, upToDate: !!(remoteDate && match.updated && remoteDate <= match.updated), localDate: match.updated, remoteDate};
 	}
 
-	/** Lists local ZIMs - same flat shape as `catalog()` (plus `file`), sourced from the KiwixServer's own library.xml. */
+	/** Local ZIMs - same flat shape as `catalog()` (plus `file`) - with any in-flight downloads/updates appended as
+	 * synthetic pending entries (catalog metadata + live progress fields + `downloading: true`) so the UI always has
+	 * a card to render progress against, even before the file exists on disk. Resolves itself: once the real file
+	 * lands, its name matches the local entry and the synthetic one drops out automatically. */
 	async list() {
-		const server = await this.#ensureServer();
+		const local = await this.#rawList();
-		return server.list();
+		const localNames = new Set(local.map(l => l.name));
+		const pending = Object.values(this.#active)
+			.filter(({entry}) => !localNames.has(entry.name))
+			.map(({entry, dl}) => ({...entry, ...dl.meta, downloading: true}));
+		return [...local, ...pending];
 	}
 
 	/** Fetches a single asset's raw bytes. */
@@ -187,30 +208,20 @@ export class ZimManager {
 	}
 
 	/** Checks all local ZIMs against the catalog and updates any that are outdated.
-	 * Each in-flight update is individually tracked in `#active` - use `progress(name)`/`downloads()` to poll them mid-run. */
+	 * Each in-flight update is tracked in `#active` by href - reflected live via `list()`/`catalog()` mid-run. */
 	async updateAll({force = false} = {}) {
-		const local = await this.list();
+		const local = await this.#rawList();
 		const results = [];
 		for (const entry of local) {
 			if (!entry.name) { results.push({file: entry.file, status: 'skipped', reason: 'no metadata'}); continue; }
 			const catalogEntry = await zimCatalogInfo(entry.name, this.#catalogUrl);
 			if (!catalogEntry) { results.push({name: entry.name, status: 'skipped', reason: 'missing from catalog'}); continue; }
 			try {
-				results.push(await this.#trackedUpdate(entry.name, catalogEntry, entry, force));
+				results.push(await this.#trackedUpdate(catalogEntry, entry, force));
 			} catch (e) {
 				results.push({name: entry.name, status: 'error', reason: e.message});
 			}
 		}
 		return results;
 	}
-
-	/** Live progress snapshot (speed/eta/size/total/...) for a single named download, or `null` if nothing's in flight. */
-	progress(name) {
-		return this.#active[name] ? {name, ...this.#active[name].meta} : null;
-	}
-
-	/** Live progress snapshots for every currently active download. */
-	downloads() {
-		return Object.entries(this.#active).map(([name, dl]) => ({name, ...dl.meta}));
-	}
 }