generated from ztimson/template
Migrated to kiwix-serve
This commit is contained in:
+119
-455
@@ -1,474 +1,138 @@
|
||||
'use strict';
|
||||
|
||||
import fs from 'node:fs';
|
||||
import {createHash} from 'node:crypto';
|
||||
import {decompressPool} from './decompress.js';
|
||||
import {fuzzyMatch, titleFromUrl, kiwixSearch} from './utils.js';
|
||||
|
||||
const HEADER_SIZE = 80;
|
||||
const NS_CONTENT = 'C';
|
||||
const NS_METADATA = 'M';
|
||||
const TITLE_SENTINEL = 0xffffffffffffffffn; // Indicator -> ZIM v6+ archives with no title
|
||||
|
||||
const DEFAULT_CLUSTER_CACHE_MAX = 32;
|
||||
const DEFAULT_CLUSTER_TTL = 60_000;
|
||||
async function readAt(fd, pos, length) {
|
||||
const buf = Buffer.alloc(length);
|
||||
await fd.read(buf, 0, length, pos);
|
||||
return buf;
|
||||
}
|
||||
|
||||
// --- Fallback search index (only used for archives with no native title listing) ---
|
||||
const INDEX_SUFFIX = '.searchidx.bin';
|
||||
const INDEX_MAGIC = 'ZXI1';
|
||||
const INDEX_HEADER_SIZE = 24; // magic(4) + staleness key(16) + recordCount(4)
|
||||
const INDEX_RECORD_SIZE = 20; // keyOff(4) keyLen(2) urlOff(4) urlLen(2) titleOff(4) titleLen(2) mimetype(2)
|
||||
async function ptr64(fd, base, index) {
|
||||
return Number((await readAt(fd, base + index * 8, 8)).readBigUInt64LE(0));
|
||||
}
|
||||
|
||||
/** Native, dependency-light reader for .zim archives. Supports zstd & LZMA cluster compression. */
|
||||
export class ZimReader {
|
||||
#fd = null;
|
||||
#header = null;
|
||||
#mimeTypes = [];
|
||||
#hasTitleListing = false;
|
||||
#clusterCache = new Map(); // clusterNumber -> {data, extended, timer}
|
||||
#pending = new Map(); // clusterNumber -> Promise, dedupes concurrent misses
|
||||
#clusterCacheMax;
|
||||
#clusterTTL;
|
||||
async function readHeader(fd) {
|
||||
const b = await readAt(fd, 0, HEADER_SIZE);
|
||||
return {
|
||||
articleCount: b.readUInt32LE(24),
|
||||
clusterCount: b.readUInt32LE(28),
|
||||
urlPtrPos: Number(b.readBigUInt64LE(32)),
|
||||
clusterPtrPos: Number(b.readBigUInt64LE(48)),
|
||||
mimeListPos: Number(b.readBigUInt64LE(56)),
|
||||
};
|
||||
}
|
||||
|
||||
#indexPath;
|
||||
#indexReady = null; // Promise, awaited by search() before using the fallback index
|
||||
#indexFd = null; // open fd for the fallback index, once loaded/built
|
||||
#indexRecordCount = 0;
|
||||
#indexTableStart = 0;
|
||||
|
||||
get articleCount() { return this.#header?.articleCount ?? 0; }
|
||||
get mediaCount() { return this.#header?.clusterCount ?? 0; }
|
||||
|
||||
/** @param {{clusterCacheMax?: number, clusterTTL?: number}} [opts] clusterTTL in ms; 0/null disables idle eviction. */
|
||||
constructor(path, {clusterCacheMax = DEFAULT_CLUSTER_CACHE_MAX, clusterTTL = DEFAULT_CLUSTER_TTL} = {}) {
|
||||
this.path = path;
|
||||
this.#clusterCacheMax = clusterCacheMax;
|
||||
this.#clusterTTL = clusterTTL;
|
||||
this.#indexPath = `${path}${INDEX_SUFFIX}`;
|
||||
async function readMimeTypes(fd, mimeListPos) {
|
||||
let pos = mimeListPos, str = '';
|
||||
for (;;) {
|
||||
str += (await readAt(fd, pos, 1024)).toString('binary');
|
||||
const end = str.indexOf('\0\0');
|
||||
if (end !== -1) { str = str.slice(0, end + 1); break; }
|
||||
pos += 1024;
|
||||
}
|
||||
return str.split('\0').filter(Boolean);
|
||||
}
|
||||
|
||||
/** Binary search the URL pointer list for namespace+url. For exact-key lookups (readPage, metadata, icons). */
|
||||
async #findByUrl(url, namespace) {
|
||||
const key = namespace + url;
|
||||
let lo = 0, hi = this.#header.articleCount - 1;
|
||||
while (lo <= hi) {
|
||||
const mid = (lo + hi) >> 1;
|
||||
const dirent = await this.#readDirent(await this.#ptr64(this.#header.urlPtrPos, mid));
|
||||
const dirKey = dirent.namespace + dirent.url;
|
||||
const cmp = key < dirKey ? -1 : key > dirKey ? 1 : 0;
|
||||
if (cmp === 0) return dirent;
|
||||
if (cmp < 0) hi = mid - 1; else lo = mid + 1;
|
||||
}
|
||||
return null;
|
||||
}
|
||||
/** Directory entry (article record) at byte `offset`, growing the read window as needed. */
|
||||
async function readDirent(fd, offset) {
|
||||
for (let size = 512; ; size *= 2) {
|
||||
const buf = await readAt(fd, offset, size);
|
||||
let o = 0;
|
||||
const mimetype = buf.readUInt16LE(o); o += 2;
|
||||
o += 1; // extraLen, unused
|
||||
const namespace = String.fromCharCode(buf.readUInt8(o)); o += 1;
|
||||
o += 4; // revision, unused
|
||||
|
||||
/** Binary search the ZIM's own title pointer list. O(log n), zero extra storage - the happy path. */
|
||||
async #findByTitleBuiltin(title) {
|
||||
let lo = 0, hi = this.#header.articleCount - 1;
|
||||
while (lo <= hi) {
|
||||
const mid = (lo + hi) >> 1;
|
||||
const urlIdx = (await this.#read(this.#header.titlePtrPos + mid * 4, 4)).readUInt32LE(0);
|
||||
const d = await this.#readDirent(await this.#ptr64(this.#header.urlPtrPos, urlIdx));
|
||||
if (d.title === title) return d;
|
||||
if (d.title < title) lo = mid + 1; else hi = mid - 1;
|
||||
}
|
||||
return null;
|
||||
}
|
||||
let redirectIndex = null, cluster = null, blob = null;
|
||||
if (mimetype === 0xffff) { redirectIndex = buf.readUInt32LE(o); o += 4; }
|
||||
else { cluster = buf.readUInt32LE(o); o += 4; blob = buf.readUInt32LE(o); o += 4; }
|
||||
|
||||
/**
|
||||
* Resolves a kiwix-search result (a title, not necessarily a url) to
|
||||
* {url, title, mimetype}. kiwix-search's fulltext index returns titles, and
|
||||
* a title isn't guaranteed to equal its url (unicode normalization,
|
||||
* disambiguation suffixes, punctuation stripping), so:
|
||||
* 1. URL binary search - matches when title happens to equal url (free to check)
|
||||
* 2. Title pointer list - when the archive ships one (Wikipedia etc. do)
|
||||
* 3. Persisted fallback index / linear scan - only for archives without (2)
|
||||
*/
|
||||
async #resolveSearchEntry(name) {
|
||||
let dirent = await this.#findByUrl(name, NS_CONTENT);
|
||||
if (dirent) return dirent;
|
||||
const urlEnd = buf.indexOf(0, o);
|
||||
if (urlEnd === -1) continue;
|
||||
const titleEnd = buf.indexOf(0, urlEnd + 1);
|
||||
if (titleEnd === -1) continue;
|
||||
|
||||
if (this.#hasTitleListing) return this.#findByTitleBuiltin(name);
|
||||
|
||||
if (this.#indexReady) await this.#indexReady;
|
||||
const hit = await this.#lookupFallbackIndex(name);
|
||||
if (hit) return hit;
|
||||
|
||||
// Index unavailable (build failed - unwritable disk, etc.) or genuinely no match.
|
||||
for (let i = 0; i < this.#header.articleCount; i++) {
|
||||
const d = await this.#readDirent(await this.#ptr64(this.#header.urlPtrPos, i));
|
||||
if (d.namespace === NS_CONTENT && (d.title === name || titleFromUrl(d.url) === name)) return d;
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
/**
|
||||
* A cheap, stable fingerprint for "is the cached index still valid for this file".
|
||||
* ZIM archives end with a 16-byte MD5 of their own contents (the same trailer
|
||||
* `zimcheck` validates against), so this is a single 16-byte read regardless of
|
||||
* archive size - no need to hash a multi-hundred-GB file. It's also
|
||||
* content-based rather than path/mtime-based, so moving or redownloading an
|
||||
* identical archive doesn't invalidate the cache. Falls back to a tiny
|
||||
* size+mtime hash only if the file is too short to have a real trailer.
|
||||
*/
|
||||
async #stalenessKey() {
|
||||
const {size} = await fs.promises.stat(this.path);
|
||||
if (size >= 16) return this.#read(size - 16, 16);
|
||||
const stat = await fs.promises.stat(this.path);
|
||||
return createHash('md5').update(`${stat.size}:${stat.mtimeMs}`).digest();
|
||||
}
|
||||
|
||||
/** Walks every content dirent once, resolving redirects, keyed by both its title and its url-derived title. */
|
||||
async #collectFallbackEntries() {
|
||||
const map = new Map(); // key -> {url, title, mimetype}
|
||||
for (let i = 0; i < this.#header.articleCount; i++) {
|
||||
let dirent = await this.#readDirent(await this.#ptr64(this.#header.urlPtrPos, i));
|
||||
if (dirent.namespace !== NS_CONTENT) continue;
|
||||
dirent = await this.#resolveRedirect(dirent);
|
||||
if (dirent.namespace !== NS_CONTENT) continue; // redirected outside content namespace
|
||||
|
||||
const entry = {url: dirent.url, title: dirent.title, mimetype: dirent.mimetype};
|
||||
if (!map.has(dirent.title)) map.set(dirent.title, entry);
|
||||
const urlTitle = titleFromUrl(dirent.url);
|
||||
if (urlTitle !== dirent.title && !map.has(urlTitle)) map.set(urlTitle, entry);
|
||||
}
|
||||
return map;
|
||||
}
|
||||
|
||||
/** Serializes the fallback index to disk: header, sorted fixed-width records, then a string table. */
|
||||
async #buildFallbackIndex(stalenessKey) {
|
||||
const map = await this.#collectFallbackEntries();
|
||||
const keys = [...map.keys()].sort();
|
||||
|
||||
const records = Buffer.alloc(keys.length * INDEX_RECORD_SIZE);
|
||||
const strings = [];
|
||||
let tableOffset = 0;
|
||||
keys.forEach((key, i) => {
|
||||
const {url, title, mimetype} = map.get(key);
|
||||
const keyBuf = Buffer.from(key, 'utf8');
|
||||
const urlBuf = Buffer.from(url, 'utf8');
|
||||
const titleBuf = Buffer.from(title, 'utf8');
|
||||
const base = i * INDEX_RECORD_SIZE;
|
||||
|
||||
records.writeUInt32LE(tableOffset, base); records.writeUInt16LE(keyBuf.length, base + 4);
|
||||
tableOffset += keyBuf.length;
|
||||
records.writeUInt32LE(tableOffset, base + 6); records.writeUInt16LE(urlBuf.length, base + 10);
|
||||
tableOffset += urlBuf.length;
|
||||
records.writeUInt32LE(tableOffset, base + 12); records.writeUInt16LE(titleBuf.length, base + 16);
|
||||
tableOffset += titleBuf.length;
|
||||
records.writeUInt16LE(mimetype, base + 18);
|
||||
|
||||
strings.push(keyBuf, urlBuf, titleBuf);
|
||||
});
|
||||
|
||||
const header = Buffer.alloc(INDEX_HEADER_SIZE);
|
||||
header.write(INDEX_MAGIC, 0, 'ascii');
|
||||
stalenessKey.copy(header, 4);
|
||||
header.writeUInt32LE(keys.length, 20);
|
||||
|
||||
const tmpPath = `${this.#indexPath}.tmp-${process.pid}`;
|
||||
await fs.promises.writeFile(tmpPath, Buffer.concat([header, records, ...strings]));
|
||||
await fs.promises.rename(tmpPath, this.#indexPath);
|
||||
}
|
||||
|
||||
/** Opens the fallback index file and caches its header fields for querying. */
|
||||
async #openFallbackIndex() {
|
||||
const fd = await fs.promises.open(this.#indexPath, 'r');
|
||||
const header = Buffer.alloc(INDEX_HEADER_SIZE);
|
||||
await fd.read(header, 0, INDEX_HEADER_SIZE, 0);
|
||||
this.#indexFd = fd;
|
||||
this.#indexRecordCount = header.readUInt32LE(20);
|
||||
this.#indexTableStart = INDEX_HEADER_SIZE + this.#indexRecordCount * INDEX_RECORD_SIZE;
|
||||
}
|
||||
|
||||
/**
|
||||
* Validates the on-disk fallback index against the archive's current staleness
|
||||
* key and (re)builds it if missing/stale. Failures are swallowed - callers see
|
||||
* #indexFd stay null and fall through to the linear-scan last resort.
|
||||
*/
|
||||
async #loadOrBuildFallbackIndex() {
|
||||
const stalenessKey = await this.#stalenessKey();
|
||||
|
||||
try {
|
||||
const fd = await fs.promises.open(this.#indexPath, 'r');
|
||||
const header = Buffer.alloc(INDEX_HEADER_SIZE);
|
||||
await fd.read(header, 0, INDEX_HEADER_SIZE, 0);
|
||||
const fresh = header.toString('ascii', 0, 4) === INDEX_MAGIC && header.subarray(4, 20).equals(stalenessKey);
|
||||
if (fresh) {
|
||||
this.#indexFd = fd;
|
||||
this.#indexRecordCount = header.readUInt32LE(20);
|
||||
this.#indexTableStart = INDEX_HEADER_SIZE + this.#indexRecordCount * INDEX_RECORD_SIZE;
|
||||
return;
|
||||
}
|
||||
await fd.close();
|
||||
} catch { /* missing, corrupt, or unreadable -> rebuild below */ }
|
||||
|
||||
await this.#buildFallbackIndex(stalenessKey);
|
||||
await this.#openFallbackIndex();
|
||||
}
|
||||
|
||||
async #indexRead(pos, length) {
|
||||
const buf = Buffer.alloc(length);
|
||||
await this.#indexFd.read(buf, 0, length, pos);
|
||||
return buf;
|
||||
}
|
||||
|
||||
async #indexString(offset, length) {
|
||||
if (!length) return '';
|
||||
return (await this.#indexRead(this.#indexTableStart + offset, length)).toString('utf8');
|
||||
}
|
||||
|
||||
/** Binary search over the on-disk record array. Never loads the full index into memory. */
|
||||
async #lookupFallbackIndex(name) {
|
||||
if (!this.#indexFd) return null;
|
||||
let lo = 0, hi = this.#indexRecordCount - 1;
|
||||
while (lo <= hi) {
|
||||
const mid = (lo + hi) >> 1;
|
||||
const rec = await this.#indexRead(INDEX_HEADER_SIZE + mid * INDEX_RECORD_SIZE, INDEX_RECORD_SIZE);
|
||||
const keyOff = rec.readUInt32LE(0), keyLen = rec.readUInt16LE(4);
|
||||
const key = await this.#indexString(keyOff, keyLen);
|
||||
if (key === name) {
|
||||
const urlOff = rec.readUInt32LE(6), urlLen = rec.readUInt16LE(10);
|
||||
const titleOff = rec.readUInt32LE(12), titleLen = rec.readUInt16LE(16);
|
||||
const [url, title] = await Promise.all([this.#indexString(urlOff, urlLen), this.#indexString(titleOff, titleLen)]);
|
||||
return {url, title, mimetype: rec.readUInt16LE(18)};
|
||||
}
|
||||
if (key < name) lo = mid + 1; else hi = mid - 1;
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
/** Resets a cluster's idle-eviction timer. No-op when TTL disabled. */
|
||||
#touch(clusterNumber, entry) {
|
||||
if (!this.#clusterTTL) return;
|
||||
clearTimeout(entry.timer);
|
||||
entry.timer = setTimeout(() => this.#clusterCache.delete(clusterNumber), this.#clusterTTL).unref();
|
||||
}
|
||||
|
||||
/** Fetches + decompresses a cluster exactly once, offloading decompression to the worker pool. */
|
||||
async #loadCluster(clusterNumber) {
|
||||
const start = await this.#ptr64(this.#header.clusterPtrPos, clusterNumber);
|
||||
const isLast = clusterNumber === this.#header.clusterCount - 1;
|
||||
const end = isLast
|
||||
? (await fs.promises.stat(this.path)).size
|
||||
: await this.#ptr64(this.#header.clusterPtrPos, clusterNumber + 1);
|
||||
|
||||
const raw = await this.#read(start, end - start);
|
||||
const compType = raw[0] & 0x0f;
|
||||
const extended = (raw[0] & 0x10) !== 0;
|
||||
const body = raw.subarray(1);
|
||||
|
||||
let data;
|
||||
if (compType <= 1) data = Buffer.from(body);
|
||||
else if (compType === 4 || compType === 5) data = await decompressPool.run({compType, body});
|
||||
else throw new Error(`Unsupported cluster compression type: ${compType}`);
|
||||
if (!data) throw new Error(`Cluster ${clusterNumber} failed to decompress (compType ${compType})`);
|
||||
|
||||
return {data, extended};
|
||||
}
|
||||
|
||||
async #getBlob(clusterNumber, blobNumber) {
|
||||
let entry = this.#clusterCache.get(clusterNumber);
|
||||
if (!entry) {
|
||||
let pending = this.#pending.get(clusterNumber);
|
||||
if (!pending) {
|
||||
pending = this.#loadCluster(clusterNumber);
|
||||
this.#pending.set(clusterNumber, pending);
|
||||
}
|
||||
entry = await pending;
|
||||
this.#pending.delete(clusterNumber);
|
||||
this.#clusterCache.set(clusterNumber, entry);
|
||||
if (this.#clusterCache.size > this.#clusterCacheMax) {
|
||||
const oldestKey = this.#clusterCache.keys().next().value;
|
||||
clearTimeout(this.#clusterCache.get(oldestKey)?.timer);
|
||||
this.#clusterCache.delete(oldestKey);
|
||||
}
|
||||
}
|
||||
this.#touch(clusterNumber, entry);
|
||||
|
||||
const readPtr = i => entry.extended ? Number(entry.data.readBigUInt64LE(i * 8)) : entry.data.readUInt32LE(i * 4);
|
||||
return entry.data.subarray(readPtr(blobNumber), readPtr(blobNumber + 1));
|
||||
}
|
||||
|
||||
async #icon(size = 48) {
|
||||
const page = await this.readPage(`Illustration_${size}x${size}@1`, NS_METADATA)
|
||||
|| await this.readPage('Favicon', NS_METADATA);
|
||||
if (!page) return null;
|
||||
return `data:${page.mimetype};base64,${page.data.toString('base64')}`;
|
||||
}
|
||||
|
||||
async #ptr64(base, index) {
|
||||
return Number((await this.#read(base + index * 8, 8)).readBigUInt64LE(0));
|
||||
}
|
||||
|
||||
async #read(pos, length) {
|
||||
const buf = Buffer.alloc(length);
|
||||
await this.#fd.read(buf, 0, length, pos);
|
||||
return buf;
|
||||
}
|
||||
|
||||
async #readHeader() {
|
||||
const b = await this.#read(0, HEADER_SIZE);
|
||||
const titlePtrRaw = b.readBigUInt64LE(40);
|
||||
this.#hasTitleListing = titlePtrRaw !== TITLE_SENTINEL;
|
||||
this.#header = {
|
||||
articleCount: b.readUInt32LE(24),
|
||||
clusterCount: b.readUInt32LE(28),
|
||||
urlPtrPos: Number(b.readBigUInt64LE(32)),
|
||||
titlePtrPos: this.#hasTitleListing ? Number(titlePtrRaw) : null,
|
||||
clusterPtrPos: Number(b.readBigUInt64LE(48)),
|
||||
mimeListPos: Number(b.readBigUInt64LE(56)),
|
||||
mainPage: b.readUInt32LE(64),
|
||||
};
|
||||
}
|
||||
|
||||
async #readMimeTypes() {
|
||||
let pos = this.#header.mimeListPos, str = '';
|
||||
for (;;) {
|
||||
str += (await this.#read(pos, 1024)).toString('binary');
|
||||
const end = str.indexOf('\0\0');
|
||||
if (end !== -1) { str = str.slice(0, end + 1); break; }
|
||||
pos += 1024;
|
||||
}
|
||||
this.#mimeTypes = str.split('\0').filter(Boolean);
|
||||
}
|
||||
|
||||
/** Directory entry (article record) at byte `offset`, growing the read window as needed. */
|
||||
async #readDirent(offset) {
|
||||
for (let size = 512; ; size *= 2) {
|
||||
const buf = await this.#read(offset, size);
|
||||
let o = 0;
|
||||
const mimetype = buf.readUInt16LE(o); o += 2;
|
||||
o += 1; // extraLen, unused
|
||||
const namespace = String.fromCharCode(buf.readUInt8(o)); o += 1;
|
||||
o += 4; // revision, unused
|
||||
|
||||
let redirectIndex = null, cluster = null, blob = null;
|
||||
if (mimetype === 0xffff) { redirectIndex = buf.readUInt32LE(o); o += 4; }
|
||||
else { cluster = buf.readUInt32LE(o); o += 4; blob = buf.readUInt32LE(o); o += 4; }
|
||||
|
||||
const urlEnd = buf.indexOf(0, o);
|
||||
if (urlEnd === -1) continue;
|
||||
const titleEnd = buf.indexOf(0, urlEnd + 1);
|
||||
if (titleEnd === -1) continue;
|
||||
|
||||
const url = buf.toString('utf8', o, urlEnd);
|
||||
const title = buf.toString('utf8', urlEnd + 1, titleEnd) || url;
|
||||
return {mimetype, namespace, redirectIndex, cluster, blob, url, title};
|
||||
}
|
||||
}
|
||||
|
||||
async #resolveRedirect(dirent) {
|
||||
if (dirent.mimetype !== 0xffff) return dirent;
|
||||
return this.#readDirent(await this.#ptr64(this.#header.urlPtrPos, dirent.redirectIndex));
|
||||
}
|
||||
|
||||
async close() {
|
||||
if (this.#fd) await this.#fd.close();
|
||||
this.#fd = null;
|
||||
if (this.#indexFd) await this.#indexFd.close();
|
||||
this.#indexFd = null;
|
||||
for (const entry of this.#clusterCache.values()) clearTimeout(entry.timer);
|
||||
this.#clusterCache.clear();
|
||||
this.#pending.clear();
|
||||
}
|
||||
|
||||
/** Deletes the zim archive and its cached fallback index (if any). Safe to call on unopened readers. */
|
||||
async delete() {
|
||||
await this.close();
|
||||
await fs.promises.rm(this.path, {force: true});
|
||||
await fs.promises.rm(this.#indexPath, {force: true});
|
||||
}
|
||||
|
||||
/** Reads an 'M' namespace metadata value (e.g. Name, Date, Title). Returns null if missing. */
|
||||
async metadata(key) {
|
||||
const get = async key => {
|
||||
const page = await this.readPage(key, NS_METADATA);
|
||||
return page ? page.data.toString('utf8') : null;
|
||||
};
|
||||
if(key) return get(key);
|
||||
|
||||
const [title, creator, publisher, date, description, language, name, tags] = await Promise.all(
|
||||
['Title', 'Creator', 'Publisher', 'Date', 'Description', 'Language', 'Name', 'Tags'].map(get)
|
||||
);
|
||||
return {
|
||||
title,
|
||||
updated: date ? new Date(date) : null,
|
||||
summary: description,
|
||||
language,
|
||||
name,
|
||||
category: tags ? tags.split(';')[0] || '' : '',
|
||||
tags: tags ? tags.split(';') : [],
|
||||
author: creator,
|
||||
publisher,
|
||||
articleCount: this.articleCount,
|
||||
mediaCount: this.mediaCount,
|
||||
sizeMb: +((await fs.promises.stat(this.path)).size / 1024 / 1024).toFixed(1),
|
||||
icon: await this.#icon(),
|
||||
};
|
||||
}
|
||||
|
||||
/** Opens the archive and parses its header + mimetype list. */
|
||||
async open() {
|
||||
this.#fd = await fs.promises.open(this.path, 'r');
|
||||
try {
|
||||
await this.#readHeader();
|
||||
await this.#readMimeTypes();
|
||||
} catch (e) {
|
||||
await this.#fd.close();
|
||||
this.#fd = null;
|
||||
throw e;
|
||||
}
|
||||
|
||||
// The fallback index is only needed when the archive has no native title
|
||||
// pointer list - #findByTitleBuiltin already covers that case in O(log n)
|
||||
// with zero extra storage. Well-maintained archives (Wikipedia etc.)
|
||||
// ship a title listing, so this path is expected to be rare in practice.
|
||||
if (!this.#hasTitleListing) {
|
||||
this.#indexReady = this.#loadOrBuildFallbackIndex().catch(() => {});
|
||||
}
|
||||
return this;
|
||||
}
|
||||
|
||||
/** Read a page's content by URL. Returns `{mimetype, data}` or `null` if not found. */
|
||||
async readPage(url, namespace = NS_CONTENT) {
|
||||
let dirent = await this.#findByUrl(url, namespace);
|
||||
if (!dirent) return null;
|
||||
dirent = await this.#resolveRedirect(dirent);
|
||||
const data = await this.#getBlob(dirent.cluster, dirent.blob);
|
||||
return {mimetype: this.#mimeTypes[dirent.mimetype] || 'application/octet-stream', data};
|
||||
}
|
||||
|
||||
/** Reads the archive's designated main/landing page, if one is set. */
|
||||
async mainPage() {
|
||||
if (this.#header.mainPage === 0xffffffff) return null;
|
||||
let dirent = await this.#readDirent(await this.#ptr64(this.#header.urlPtrPos, this.#header.mainPage));
|
||||
dirent = await this.#resolveRedirect(dirent);
|
||||
const data = await this.#getBlob(dirent.cluster, dirent.blob);
|
||||
return {mimetype: this.#mimeTypes[dirent.mimetype] || 'application/octet-stream', data, url: dirent.url, namespace: dirent.namespace};
|
||||
}
|
||||
|
||||
/**
|
||||
* Fuzzy-ranked title search. Accepts comma-separated `terms` the same way the
|
||||
* catalog search does. Uses kiwix-search's embedded fulltext index as a prefilter
|
||||
* to narrow candidates before fuzzy scoring.
|
||||
*/
|
||||
async search(terms, {limit = 20, htmlOnly = true} = {}) {
|
||||
const termList = String(terms).split(',').map(t => t.trim()).filter(Boolean);
|
||||
if (!termList.length) return [];
|
||||
|
||||
const names = await kiwixSearch(this.path, termList.join(' '));
|
||||
const candidates = (await Promise.all(names.map(n => this.#resolveSearchEntry(n)))).filter(Boolean);
|
||||
|
||||
const scored = [];
|
||||
for (const dirent of candidates) {
|
||||
if (htmlOnly && !(this.#mimeTypes[dirent.mimetype] || '').startsWith('text/html')) continue;
|
||||
const urlTitle = titleFromUrl(dirent.url);
|
||||
const titleScore = fuzzyMatch(dirent.title, ...termList).max;
|
||||
const urlScore = fuzzyMatch(urlTitle, ...termList).max;
|
||||
scored.push({url: dirent.url, title: dirent.title.length > urlTitle.length ? dirent.title : urlTitle, namespace: NS_CONTENT, score: Math.max(titleScore, urlScore)});
|
||||
}
|
||||
const {summary, mediaCount, articleCount, sizeMb, ...meta} = await this.metadata();
|
||||
return scored.filter(a => a.score > 0).toSorted((a, b) => b.score - a.score).slice(0, limit).map(a => ({...meta, ...a}));
|
||||
const url = buf.toString('utf8', o, urlEnd);
|
||||
const title = buf.toString('utf8', urlEnd + 1, titleEnd) || url;
|
||||
return {mimetype, namespace, redirectIndex, cluster, blob, url, title};
|
||||
}
|
||||
}
|
||||
|
||||
async function findByUrl(fd, header, url, namespace) {
|
||||
const key = namespace + url;
|
||||
let lo = 0, hi = header.articleCount - 1;
|
||||
while (lo <= hi) {
|
||||
const mid = (lo + hi) >> 1;
|
||||
const dirent = await readDirent(fd, await ptr64(fd, header.urlPtrPos, mid));
|
||||
const dirKey = dirent.namespace + dirent.url;
|
||||
const cmp = key < dirKey ? -1 : key > dirKey ? 1 : 0;
|
||||
if (cmp === 0) return dirent;
|
||||
if (cmp < 0) hi = mid - 1; else lo = mid + 1;
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
async function resolveRedirect(fd, header, dirent) {
|
||||
if (dirent.mimetype !== 0xffff) return dirent;
|
||||
return readDirent(fd, await ptr64(fd, header.urlPtrPos, dirent.redirectIndex));
|
||||
}
|
||||
|
||||
async function getBlob(fd, header, filePath, clusterNumber, blobNumber) {
|
||||
const start = await ptr64(fd, header.clusterPtrPos, clusterNumber);
|
||||
const isLast = clusterNumber === header.clusterCount - 1;
|
||||
const end = isLast
|
||||
? (await fs.promises.stat(filePath)).size
|
||||
: await ptr64(fd, header.clusterPtrPos, clusterNumber + 1);
|
||||
|
||||
const raw = await readAt(fd, start, end - start);
|
||||
const compType = raw[0] & 0x0f;
|
||||
const extended = (raw[0] & 0x10) !== 0;
|
||||
const body = raw.subarray(1);
|
||||
|
||||
let data;
|
||||
if (compType <= 1) data = Buffer.from(body);
|
||||
else if (compType === 4 || compType === 5) data = await decompressPool.run({compType, body});
|
||||
else throw new Error(`Unsupported cluster compression type: ${compType}`);
|
||||
|
||||
const readPtr = i => extended ? Number(data.readBigUInt64LE(i * 8)) : data.readUInt32LE(i * 4);
|
||||
return data.subarray(readPtr(blobNumber), readPtr(blobNumber + 1));
|
||||
}
|
||||
|
||||
/**
|
||||
* One-shot read of a single entry from a .zim archive - no server required.
|
||||
* Opt-in convenience for callers who don't want to run kiwix-serve for a single
|
||||
* lookup. Re-parses the header/mimetype list on every call; fine for occasional
|
||||
* reads, not meant for high-volume access (use KiwixServer for that).
|
||||
*/
|
||||
export async function readZimEntry(zimPath, url, namespace = NS_CONTENT) {
|
||||
const fd = await fs.promises.open(zimPath, 'r');
|
||||
try {
|
||||
const header = await readHeader(fd);
|
||||
const mimeTypes = await readMimeTypes(fd, header.mimeListPos);
|
||||
let dirent = await findByUrl(fd, header, url, namespace);
|
||||
if (!dirent) return null;
|
||||
dirent = await resolveRedirect(fd, header, dirent);
|
||||
const data = await getBlob(fd, header, zimPath, dirent.cluster, dirent.blob);
|
||||
return {mimetype: mimeTypes[dirent.mimetype] || 'application/octet-stream', data};
|
||||
} finally {
|
||||
await fd.close();
|
||||
}
|
||||
}
|
||||
|
||||
/** Reads 'M' namespace metadata (Title, Creator, Date, etc.) without a server. Minimal by design - no icon/size/counts. */
|
||||
export async function readZimMetadata(zimPath) {
|
||||
const keys = ['Title', 'Creator', 'Publisher', 'Date', 'Description', 'Language', 'Name', 'Tags'];
|
||||
const entries = await Promise.all(keys.map(k => readZimEntry(zimPath, k, NS_METADATA)));
|
||||
const [title, creator, publisher, date, description, language, name, tags] = entries.map(e => e?.data.toString('utf8') ?? null);
|
||||
return {
|
||||
title, updated: date ? new Date(date) : null, summary: description, language, name,
|
||||
category: tags ? tags.split(';')[0] || '' : '', tags: tags ? tags.split(';') : [],
|
||||
author: creator, publisher,
|
||||
};
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user