generated from ztimson/template
217 lines
9.9 KiB
JavaScript
217 lines
9.9 KiB
JavaScript
import fs from 'node:fs';
|
|
import path from 'node:path';
|
|
import {pipeline} from 'node:stream/promises';
|
|
import {Readable} from 'node:stream';
|
|
import {PromiseProgress} from '@ztimson/utils';
|
|
import {KiwixServer} from './server.js';
|
|
import {zimCatalog, zimCatalogInfo, CATALOG_URL} from './catalog.js';
|
|
|
|
const VISIBLE_TIMEOUT = 5_000;
|
|
const VISIBLE_POLL_INTERVAL = 200;
|
|
|
|
/** Manages a local directory of ZIM archives: catalog search, downloads, update checks, deletion.
|
|
* Reuses (or owns & lazily starts) a KiwixServer for local listing/search, so `list()` and `catalog()` return the same shape.
|
|
* Doesn't drive library.xml/kiwix-serve reloads itself - the owning KiwixServer watches its directory and
|
|
* reloads automatically whenever .zim files change, no matter which ZimManager (local or remote-attached) wrote them. */
|
|
export class ZimManager {
|
|
#catalogUrl;
|
|
#dir;
|
|
#server;
|
|
#ownsServer;
|
|
#active = {}; // name -> in-flight PromiseProgress, for on-demand status lookups
|
|
|
|
/** @param {{catalog?: string, server?: KiwixServer, port?: number, host?: string, binDir?: string, url?: string}} [opts]
|
|
* server: reuse an existing KiwixServer instance directly.
|
|
* url: attach to an already-running kiwix-serve by address instead (e.g. started elsewhere in your codebase
|
|
* with no shared reference) - mutually exclusive with `server`, ignored if `server` is given. */
|
|
constructor(dir, {catalog = CATALOG_URL, server, port, host, binDir, url} = {}) {
|
|
this.#catalogUrl = catalog;
|
|
this.#dir = dir;
|
|
this.#ownsServer = !server;
|
|
this.#server = server ?? new KiwixServer(dir, {port, host, binDir, url});
|
|
}
|
|
|
|
#stripDate(filename) {
|
|
return filename.replace(/\.zim$/i, '').replace(/_\d{4}-\d{2}(?:_\d+)?$/, '');
|
|
}
|
|
|
|
/** The KiwixServer backing this manager - reuse it directly for content/search access, or pass into another ZimManager. */
|
|
get server() { return this.#server; }
|
|
|
|
async #ensureServer() {
|
|
if (!this.#server.running) await this.#server.start();
|
|
return this.#server;
|
|
}
|
|
|
|
/** Streams `url` to `destPath`, reporting raw bytes/total via `onBytes(size, total)` as chunks arrive. */
|
|
async #download(url, destPath, onBytes) {
|
|
const {res} = await this.#resolveUrl(url);
|
|
if (!res.ok) throw new Error(`${res.status} ${res.statusText}`);
|
|
const total = Number(res.headers.get('content-length')) || 0;
|
|
const tmpPath = `${destPath}.part`;
|
|
let size = 0;
|
|
const readable = Readable.fromWeb(res.body);
|
|
if (onBytes) readable.on('data', chunk => onBytes(size += chunk.length, total));
|
|
await pipeline(readable, fs.createWriteStream(tmpPath));
|
|
await fs.promises.rename(tmpPath, destPath);
|
|
}
|
|
|
|
/** Resolves a `.meta4` metalink URL down to the real mirror `.zim` download URL. */
|
|
async #resolveUrl(url) {
|
|
const head = await fetch(url);
|
|
const ct = head.headers.get('content-type') || '';
|
|
if (!ct.includes('metalink') && !url.endsWith('.meta4')) return {res: head, url};
|
|
const meta = await head.text();
|
|
const m = meta.match(/<url[^>]*>([^<]+\.zim)<\/url>/);
|
|
if (!m) throw new Error('Could not resolve metalink mirror');
|
|
const res = await fetch(m[1]);
|
|
return {res, url: m[1]};
|
|
}
|
|
|
|
/** Polls list() until `filename` shows up (or disappears, if `expect: false`), so callers get an accurate
|
|
* status back rather than one that's ahead of what the server has actually picked up yet. The owning
|
|
* KiwixServer's directory watcher does the real reload work in the background; this just waits for it. */
|
|
async #waitUntilVisible(filename, {expect = true, timeout = VISIBLE_TIMEOUT} = {}) {
|
|
const deadline = Date.now() + timeout;
|
|
while (Date.now() < deadline) {
|
|
const local = await this.list();
|
|
const present = local.some(l => l.file === filename);
|
|
if (present === expect) return true;
|
|
await new Promise(r => setTimeout(r, VISIBLE_POLL_INTERVAL));
|
|
}
|
|
return false;
|
|
}
|
|
|
|
/** Performs the actual download + post-processing for a resolved catalog entry, forwarding byte progress via `onBytes`. */
|
|
async #update(name, catalogEntry, localMatch, force, onBytes) {
|
|
const remoteDate = catalogEntry.updated ? new Date(catalogEntry.updated) : null;
|
|
const localDate = localMatch?.updated ?? null;
|
|
if (!force && localMatch && remoteDate && localDate && remoteDate <= localDate)
|
|
return {name, status: 'skipped', reason: 'up to date'};
|
|
if (!catalogEntry.href) return {name, status: 'skipped', reason: 'missing href'};
|
|
|
|
const filename = path.basename(new URL(catalogEntry.href).pathname).replace(/\.meta4$/i, '');
|
|
const destPath = path.join(this.#dir, filename);
|
|
await this.#download(catalogEntry.href, destPath, onBytes);
|
|
if (localMatch && localMatch.file !== filename) await fs.promises.rm(path.join(this.#dir, localMatch.file), {force: true});
|
|
await this.#ensureServer();
|
|
await this.#waitUntilVisible(filename, {expect: true});
|
|
return {name, status: 'updated', file: filename};
|
|
}
|
|
|
|
/** Wraps `#update` in a tracked `PromiseProgress`, registered in `#active` under `name` until it settles. */
|
|
#trackedUpdate(name, catalogEntry, localMatch, force) {
|
|
const dl = new PromiseProgress((resolve, reject, prog) =>
|
|
this.#update(name, catalogEntry, localMatch, force, prog).then(resolve, reject));
|
|
this.#active[name] = dl;
|
|
dl.catch(() => {}).finally(() => { if (this.#active[name] === dl) delete this.#active[name]; });
|
|
return dl;
|
|
}
|
|
|
|
catalog(search, opts = {}) {
|
|
return zimCatalog(search, {url: this.#catalogUrl, ...opts});
|
|
}
|
|
|
|
/** Stops the internally-owned KiwixServer, if this manager created its own (no-op if one was passed in). */
|
|
async close() {
|
|
if (this.#ownsServer) await this.#server.stop();
|
|
}
|
|
|
|
async delete(nameOrFile) {
|
|
const local = await this.list();
|
|
const match = local.find(l => l.name === nameOrFile || l.href === nameOrFile);
|
|
if (!match) throw new Error(`ZIM not found locally: ${nameOrFile}`);
|
|
const file = match.href + (match.href.endsWith('.zim') ? '' : '.zim');
|
|
await fs.promises.rm(path.join(this.#dir, file), {force: true});
|
|
await this.#ensureServer();
|
|
await this.#waitUntilVisible(file, {expect: false});
|
|
return {file: match.file, status: 'deleted'};
|
|
}
|
|
|
|
/** Downloads/updates a single ZIM by direct href, matching against any existing local copy by name.
|
|
* Returns a `PromiseProgress` with live `.progress`/`.speed`/`.eta`/etc getters & `onProgress` callback,
|
|
* while also tracking it internally (see `progress()`/`downloads()`) in case the reference is dropped. */
|
|
download(href, {force = false} = {}) {
|
|
let dl;
|
|
dl = new PromiseProgress(async (resolve, reject, prog) => {
|
|
try {
|
|
const {url: finalUrl} = await this.#resolveUrl(href);
|
|
const filename = path.basename(new URL(finalUrl).pathname).replace(/\.meta4$/i, '');
|
|
await fs.promises.mkdir(this.#dir, {recursive: true});
|
|
const local = await this.list();
|
|
const exact = local.find(l => l.file === filename);
|
|
if (exact && !force) return resolve({name: exact.name ?? this.#stripDate(filename), status: 'skipped', reason: 'already downloaded', file: filename});
|
|
const name = this.#stripDate(filename);
|
|
const localMatch = local.find(l => l.name === name) ?? null;
|
|
const catalogEntry = await zimCatalogInfo(name, this.#catalogUrl) || {name, updated: null, href};
|
|
this.#active[name] = dl;
|
|
dl.catch(() => {}).finally(() => { if (this.#active[name] === dl) delete this.#active[name]; });
|
|
resolve(await this.#update(name, catalogEntry, localMatch, force, prog));
|
|
} catch (e) { reject(e); }
|
|
});
|
|
return dl;
|
|
}
|
|
|
|
async link(href) {
|
|
const server = await this.#ensureServer();
|
|
return server.link(href);
|
|
}
|
|
|
|
/** Checks whether a local ZIM has a newer version in the catalog, without downloading. */
|
|
async isOutdated(nameOrFile) {
|
|
const local = await this.list();
|
|
const match = local.find(l => l.name === nameOrFile || l.file === nameOrFile);
|
|
if (!match?.name) return {file: nameOrFile, upToDate: null, reason: 'no metadata'};
|
|
const catalogEntry = await zimCatalogInfo(match.name, this.#catalogUrl);
|
|
if (!catalogEntry) return {file: match.file, upToDate: null, reason: 'missing from catalog'};
|
|
const remoteDate = catalogEntry.updated ? new Date(catalogEntry.updated) : null;
|
|
return {file: match.file, name: match.name, upToDate: !!(remoteDate && match.updated && remoteDate <= match.updated), localDate: match.updated, remoteDate};
|
|
}
|
|
|
|
/** Lists local ZIMs - same flat shape as `catalog()` (plus `file`), sourced from the KiwixServer's own library.xml. */
|
|
async list() {
|
|
const server = await this.#ensureServer();
|
|
return server.list();
|
|
}
|
|
|
|
/** Fetches a single asset's raw bytes. */
|
|
async raw(href) {
|
|
const server = await this.#ensureServer();
|
|
return server.raw(href);
|
|
}
|
|
|
|
/** Two-pass fulltext search across every local ZIM, returns enriched results matching catalog/list shape. */
|
|
async search(terms, {limit = 20, sources = null} = {}) {
|
|
const server = await this.#ensureServer();
|
|
return server.search(terms, {limit, sources});
|
|
}
|
|
|
|
/** Checks all local ZIMs against the catalog and updates any that are outdated.
|
|
* Each in-flight update is individually tracked in `#active` - use `progress(name)`/`downloads()` to poll them mid-run. */
|
|
async updateAll({force = false} = {}) {
|
|
const local = await this.list();
|
|
const results = [];
|
|
for (const entry of local) {
|
|
if (!entry.name) { results.push({file: entry.file, status: 'skipped', reason: 'no metadata'}); continue; }
|
|
const catalogEntry = await zimCatalogInfo(entry.name, this.#catalogUrl);
|
|
if (!catalogEntry) { results.push({name: entry.name, status: 'skipped', reason: 'missing from catalog'}); continue; }
|
|
try {
|
|
results.push(await this.#trackedUpdate(entry.name, catalogEntry, entry, force));
|
|
} catch (e) {
|
|
results.push({name: entry.name, status: 'error', reason: e.message});
|
|
}
|
|
}
|
|
return results;
|
|
}
|
|
|
|
/** Live progress snapshot (speed/eta/size/total/...) for a single named download, or `null` if nothing's in flight. */
|
|
progress(name) {
|
|
return this.#active[name] ? {name, ...this.#active[name].meta} : null;
|
|
}
|
|
|
|
/** Live progress snapshots for every currently active download. */
|
|
downloads() {
|
|
return Object.entries(this.#active).map(([name, dl]) => ({name, ...dl.meta}));
|
|
}
|
|
}
|