generated from ztimson/template
Migrated to kiwix-serve
This commit is contained in:
+88
-143
@@ -2,26 +2,32 @@ import fs from 'node:fs';
|
||||
import path from 'node:path';
|
||||
import {pipeline} from 'node:stream/promises';
|
||||
import {Readable} from 'node:stream';
|
||||
import {ZimReader} from './reader.js';
|
||||
import {KiwixServer} from './server.js';
|
||||
import {zimCatalog, zimCatalogInfo, CATALOG_URL} from './catalog.js';
|
||||
|
||||
const DEFAULT_READER_TTL = 60_000;
|
||||
|
||||
/** Manages a local directory of ZIM archives: listing, update checks, downloads, and reading. */
|
||||
/** Manages a local directory of ZIM archives: catalog search, downloads, update checks, deletion.
|
||||
* Reuses (or owns & lazily starts) a KiwixServer for local listing/search, so `list()` and `catalog()` return the same shape. */
|
||||
export class ZimManager {
|
||||
#catalog;
|
||||
#catalogUrl;
|
||||
#dir;
|
||||
#readerTTL;
|
||||
#clusterTTL;
|
||||
#readers = new Map(); // file -> {reader, timer}
|
||||
#opening = new Map(); // file -> Promise<ZimReader>, dedupes concurrent first-open races
|
||||
server;
|
||||
#ownsServer;
|
||||
|
||||
/** @param {{catalog?: string, readerTTL?: number, clusterTTL?: number}} [opts] */
|
||||
constructor(dir, catalog = CATALOG_URL, {readerTTL = DEFAULT_READER_TTL, clusterTTL} = {}) {
|
||||
this.#catalog = catalog;
|
||||
/** @param {{catalog?: string, server?: KiwixServer, port?: number, host?: string, binDir?: string}} [opts]
|
||||
* server: reuse an existing KiwixServer; if omitted, one is created (not yet started) and exposed via `.server` for reuse elsewhere. */
|
||||
constructor(dir, {catalog = CATALOG_URL, server, port, host, binDir} = {}) {
|
||||
this.#catalogUrl = catalog;
|
||||
this.#dir = dir;
|
||||
this.#readerTTL = readerTTL;
|
||||
this.#clusterTTL = clusterTTL; // undefined -> ZimReader's own default
|
||||
this.#ownsServer = !server;
|
||||
this.server = server ?? new KiwixServer(dir, {port, host, binDir});
|
||||
}
|
||||
|
||||
/** The KiwixServer backing this manager - reuse it directly for content/search access, or pass into another ZimManager. */
|
||||
get server() { return this.server; }
|
||||
|
||||
async #ensureServer() {
|
||||
if (!this.server.running) await this.server.start();
|
||||
return this.server;
|
||||
}
|
||||
|
||||
async #download(url, destPath) {
|
||||
@@ -32,10 +38,6 @@ export class ZimManager {
|
||||
await fs.promises.rename(tmpPath, destPath);
|
||||
}
|
||||
|
||||
async #ensureDir() {
|
||||
await fs.promises.mkdir(this.#dir, {recursive: true});
|
||||
}
|
||||
|
||||
/** Resolves a `.meta4` metalink URL down to the real mirror `.zim` download URL. */
|
||||
async #resolveUrl(url) {
|
||||
const head = await fetch(url);
|
||||
@@ -48,159 +50,102 @@ export class ZimManager {
|
||||
return {res, url: m[1]};
|
||||
}
|
||||
|
||||
/** Reads Name/Date/Title metadata from a local ZIM file. Returns null if unreadable. */
|
||||
async #readMeta(filepath) {
|
||||
let reader;
|
||||
try {
|
||||
reader = await new ZimReader(filepath).open();
|
||||
return await reader.metadata();
|
||||
} catch {
|
||||
return null;
|
||||
} finally {
|
||||
await reader?.close();
|
||||
}
|
||||
}
|
||||
|
||||
async #update(name, catalogEntry, localMatch, force) {
|
||||
const remoteDate = catalogEntry.updated ? new Date(catalogEntry.updated) : null;
|
||||
const localDate = localMatch?.meta?.updated ?? null;
|
||||
const localDate = localMatch?.updated ?? null;
|
||||
if (!force && localMatch && remoteDate && localDate && remoteDate <= localDate)
|
||||
return {name, status: 'skipped', reason: 'up to date'};
|
||||
if (!catalogEntry.download) return {name, status: 'skipped', reason: 'missing download link'};
|
||||
if (!catalogEntry.href) return {name, status: 'skipped', reason: 'missing href'};
|
||||
|
||||
const filename = path.basename(new URL(catalogEntry.download).pathname).replace(/\.meta4$/i, '');
|
||||
const filename = path.basename(new URL(catalogEntry.href).pathname).replace(/\.meta4$/i, '');
|
||||
const destPath = path.join(this.#dir, filename);
|
||||
await this.#download(catalogEntry.download, destPath);
|
||||
if (localMatch && localMatch.file !== filename) await this.#evict(path.join(this.#dir, localMatch.file));
|
||||
await this.#download(catalogEntry.href, destPath);
|
||||
if (localMatch && localMatch.file !== filename) await fs.promises.rm(path.join(this.#dir, localMatch.file), {force: true});
|
||||
const server = await this.#ensureServer();
|
||||
await server.reload();
|
||||
return {name, status: 'updated', file: filename};
|
||||
}
|
||||
|
||||
/** Resolves a file path or catalog `name` to a local file path. */
|
||||
async #resolveFile(fileOrName) {
|
||||
if (fs.existsSync(fileOrName)) return fileOrName;
|
||||
const joined = path.join(this.#dir, fileOrName);
|
||||
if (fs.existsSync(joined)) return joined;
|
||||
catalog(search, opts = {}) {
|
||||
return zimCatalog(search, {url: this.#catalogUrl, ...opts});
|
||||
}
|
||||
|
||||
/** Stops the internally-owned KiwixServer, if this manager created its own (no-op if one was passed in). */
|
||||
async close() {
|
||||
if (this.#ownsServer) await this.server.stop();
|
||||
}
|
||||
|
||||
async delete(nameOrFile) {
|
||||
const local = await this.list();
|
||||
const match = local.find(l => l.meta?.name === fileOrName || path.basename(l.file) === fileOrName);
|
||||
if (!match) throw new Error(`ZIM not found locally: ${fileOrName}`);
|
||||
return path.join(this.#dir, match.file);
|
||||
}
|
||||
|
||||
/** Closes and drops a cached reader for `file`, if any (used before delete/replace). */
|
||||
async #evict(file) {
|
||||
const entry = this.#readers.get(file);
|
||||
if (!entry) return new ZimReader(file).delete().catch(() => {});
|
||||
clearTimeout(entry.timer);
|
||||
this.#readers.delete(file);
|
||||
await entry.reader.delete();
|
||||
}
|
||||
|
||||
catalog(search, opts) {
|
||||
return zimCatalog(search, opts);
|
||||
}
|
||||
|
||||
async delete(fileOrName) {
|
||||
const file = await this.#resolveFile(fileOrName);
|
||||
await this.#evict(file);
|
||||
return {file: path.basename(file), status: 'deleted'};
|
||||
}
|
||||
|
||||
/** Checks whether a local ZIM has a newer version in the catalog, without downloading. */
|
||||
async isOutdated(file) {
|
||||
const meta = await this.#readMeta(file);
|
||||
if (!meta?.name) return {file, upToDate: null, reason: 'no metadata'};
|
||||
const catalogEntry = await zimCatalog(meta.name, this.#catalog);
|
||||
if (!catalogEntry) return {file, upToDate: null, reason: 'missing from catalog'};
|
||||
const remoteDate = catalogEntry.updated ? new Date(catalogEntry.updated) : null;
|
||||
const localDate = meta.updated ?? null;
|
||||
return {file, name: meta.name, upToDate: !!(remoteDate && localDate && remoteDate <= localDate), localDate, remoteDate};
|
||||
}
|
||||
|
||||
/** Lists local `.zim` files with their parsed metadata (or `null` if unreadable). */
|
||||
async list() {
|
||||
await this.#ensureDir();
|
||||
const files = (await fs.promises.readdir(this.#dir)).filter(f => f.endsWith('.zim'));
|
||||
return Promise.all(files.map(async f => {
|
||||
const file = path.join(this.#dir, f);
|
||||
return {file: f, ...(await this.#readMeta(file))};
|
||||
}));
|
||||
const match = local.find(l => l.name === nameOrFile || l.file === nameOrFile);
|
||||
if (!match) throw new Error(`ZIM not found locally: ${nameOrFile}`);
|
||||
await fs.promises.rm(path.join(this.#dir, match.file), {force: true});
|
||||
const server = await this.#ensureServer();
|
||||
await server.reload();
|
||||
return {file: match.file, status: 'deleted'};
|
||||
}
|
||||
|
||||
/** Downloads/updates a single ZIM by direct href, matching against any existing local copy by name. */
|
||||
async download(href, {force = false} = {}) {
|
||||
await this.#ensureDir();
|
||||
await fs.promises.mkdir(this.#dir, {recursive: true});
|
||||
const {url: finalUrl} = await this.#resolveUrl(href);
|
||||
const filename = path.basename(new URL(finalUrl).pathname).replace(/\.meta4$/i, '');
|
||||
const name = filename.replace(/\.zim$/i, '').replace(/_\d{4}-\d{2}(?:_\d+)?$/, '');
|
||||
|
||||
const local = await this.list();
|
||||
const localMatch = local.find(l => l.meta?.name === name) ?? null;
|
||||
const catalogEntry = await zimCatalogInfo(name, this.#catalog) || {name, updated: null, download: href};
|
||||
const localMatch = local.find(l => l.name === name) ?? null;
|
||||
const catalogEntry = await zimCatalogInfo(name, this.#catalogUrl) || {name, updated: null, href: href};
|
||||
return this.#update(name, catalogEntry, localMatch, force);
|
||||
}
|
||||
|
||||
/** Checks all local ZIMs against the catalog and updates any that are outdated. */
|
||||
async updateAll({force = false}) {
|
||||
const local = await this.list();
|
||||
if (!local.length) return [];
|
||||
async link(href) {
|
||||
const server = await this.#ensureServer();
|
||||
return server.link(href);
|
||||
}
|
||||
|
||||
/** Checks whether a local ZIM has a newer version in the catalog, without downloading. */
|
||||
async isOutdated(nameOrFile) {
|
||||
const local = await this.list();
|
||||
const match = local.find(l => l.name === nameOrFile || l.file === nameOrFile);
|
||||
if (!match?.name) return {file: nameOrFile, upToDate: null, reason: 'no metadata'};
|
||||
const catalogEntry = await zimCatalogInfo(match.name, this.#catalogUrl);
|
||||
if (!catalogEntry) return {file: match.file, upToDate: null, reason: 'missing from catalog'};
|
||||
const remoteDate = catalogEntry.updated ? new Date(catalogEntry.updated) : null;
|
||||
return {file: match.file, name: match.name, upToDate: !!(remoteDate && match.updated && remoteDate <= match.updated), localDate: match.updated, remoteDate};
|
||||
}
|
||||
|
||||
/** Lists local ZIMs - same flat shape as `catalog()` (plus `file`), sourced from the KiwixServer's own library.xml. */
|
||||
async list() {
|
||||
const server = await this.#ensureServer();
|
||||
return server.list();
|
||||
}
|
||||
|
||||
/** Fetches a single asset's raw bytes. */
|
||||
async raw(href) {
|
||||
const server = await this.#ensureServer();
|
||||
return server.raw(href);
|
||||
}
|
||||
|
||||
/** Two-pass fulltext search across every local ZIM, returns enriched results matching catalog/list shape. */
|
||||
async search(terms, limit = 20) {
|
||||
const server = await this.#ensureServer();
|
||||
return server.search(terms, limit);
|
||||
}
|
||||
|
||||
/** Checks all local ZIMs against the catalog and updates any that are outdated. */
|
||||
async updateAll({force = false} = {}) {
|
||||
const local = await this.list();
|
||||
const results = [];
|
||||
for (const {file, meta} of local) {
|
||||
if (!meta?.name) { results.push({file, status: 'skipped', reason: 'no metadata'}); continue; }
|
||||
const catalogEntry = await zimCatalogInfo(meta.name, this.#catalog);
|
||||
if (!catalogEntry) { results.push({name: meta.name, status: 'skipped', reason: 'missing from catalog'}); continue; }
|
||||
for (const entry of local) {
|
||||
if (!entry.name) { results.push({file: entry.file, status: 'skipped', reason: 'no metadata'}); continue; }
|
||||
const catalogEntry = await zimCatalogInfo(entry.name, this.#catalogUrl);
|
||||
if (!catalogEntry) { results.push({name: entry.name, status: 'skipped', reason: 'missing from catalog'}); continue; }
|
||||
try {
|
||||
results.push(await this.#update(meta.name, catalogEntry, {file, meta}, force));
|
||||
results.push(await this.#update(entry.name, catalogEntry, entry, force));
|
||||
} catch (e) {
|
||||
results.push({name: meta.name, status: 'error', reason: e.message});
|
||||
results.push({name: entry.name, status: 'error', reason: e.message});
|
||||
}
|
||||
}
|
||||
return results;
|
||||
}
|
||||
|
||||
/** Opens a fresh `ZimReader` for a local ZIM, resolved by file path or catalog `name`. Caller must `.close()` it. */
|
||||
async open(fileOrName) {
|
||||
await this.#ensureDir();
|
||||
const file = await this.#resolveFile(fileOrName);
|
||||
let entry = this.#readers.get(file);
|
||||
if (!entry) {
|
||||
let pending = this.#opening.get(file);
|
||||
if (!pending) {
|
||||
pending = new ZimReader(file, {clusterTTL: this.#clusterTTL}).open();
|
||||
this.#opening.set(file, pending);
|
||||
}
|
||||
let reader;
|
||||
try {
|
||||
reader = await pending;
|
||||
} finally {
|
||||
this.#opening.delete(file);
|
||||
}
|
||||
entry = this.#readers.get(file) ?? {reader};
|
||||
this.#readers.set(file, entry);
|
||||
}
|
||||
clearTimeout(entry.timer);
|
||||
entry.timer = setTimeout(() => {
|
||||
this.#readers.delete(file);
|
||||
entry.reader.close();
|
||||
}, this.#readerTTL).unref();
|
||||
return entry.reader;
|
||||
}
|
||||
|
||||
/** Fuzzy-searches titles across every local ZIM in the library, merging & re-ranking hits by score. */
|
||||
async search(terms, {limit = 20, htmlOnly = true} = {}) {
|
||||
const local = await this.list();
|
||||
const perZim = await Promise.all(local.map(async ({file, meta}) => {
|
||||
let reader;
|
||||
try {
|
||||
reader = await new ZimReader(path.join(this.#dir, file)).open();
|
||||
const hits = await reader.search(terms, {limit, htmlOnly});
|
||||
return hits.map(h => ({...h, file}));
|
||||
} catch {
|
||||
return [];
|
||||
} finally {
|
||||
await reader?.close();
|
||||
}
|
||||
}));
|
||||
return perZim.flat().filter(a => a.score > 0).toSorted((a, b) => b.score - a.score).slice(0, limit);
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user