generated from ztimson/template
New Zim utilties
This commit is contained in:
235
src/reader.js
Normal file
235
src/reader.js
Normal file
@@ -0,0 +1,235 @@
|
||||
'use strict';
|
||||
|
||||
import fs from 'node:fs';
|
||||
import {decompress as lzmaDecompress} from 'lzma1';
|
||||
import {ZstdCodec} from 'zstd-codec';
|
||||
import {fuzzyMatch} from './utils.js';
|
||||
|
||||
const HEADER_SIZE = 80;
|
||||
const NS_CONTENT = 'C';
|
||||
const NS_METADATA = 'M';
|
||||
const TITLE_SENTINEL = 0xffffffffffffffffn; // Indicator -> ZIM v6+ archives with no title
|
||||
|
||||
let zstdStreamingPromise = null;
|
||||
|
||||
/** Lazily initialised, shared Zstd streaming decompressor (handles unknown-size frames). */
|
||||
function getZstd() {
|
||||
if (!zstdStreamingPromise) {
|
||||
zstdStreamingPromise = new Promise(resolve => ZstdCodec.run(zstd => resolve(new zstd.Streaming())));
|
||||
}
|
||||
return zstdStreamingPromise;
|
||||
}
|
||||
|
||||
/** Rebuilds a full 13-byte "alone" LZMA header from libzim's truncated 5-byte one (unknown size). */
|
||||
function toLzmaAloneStream(body) {
|
||||
const header = Buffer.concat([body.subarray(0, 5), Buffer.alloc(8, 0xff)]);
|
||||
return Buffer.concat([header, body.subarray(5)]);
|
||||
}
|
||||
|
||||
/** Native, dependency-light reader for .zim archives. Supports zstd & LZMA cluster compression. */
|
||||
export class ZimReader {
|
||||
#fd = null;
|
||||
#header = null;
|
||||
#mimeTypes = [];
|
||||
#hasTitleListing = false;
|
||||
|
||||
get articleCount() { return this.#header?.articleCount ?? 0; }
|
||||
|
||||
constructor(path) {
|
||||
this.path = path;
|
||||
}
|
||||
|
||||
/** Full O(n) scan over the URL pointer list, used when there's no title index. */
|
||||
async #allDirents() {
|
||||
const dirents = [];
|
||||
for (let i = 0; i < this.#header.articleCount; i++) {
|
||||
dirents.push(await this.#readDirent(await this.#ptr64(this.#header.urlPtrPos, i)));
|
||||
}
|
||||
return dirents;
|
||||
}
|
||||
|
||||
/** Binary search the URL pointer list for namespace+url. */
|
||||
async #findByUrl(url, namespace) {
|
||||
const key = namespace + url;
|
||||
let lo = 0, hi = this.#header.articleCount - 1;
|
||||
while (lo <= hi) {
|
||||
const mid = (lo + hi) >> 1;
|
||||
const dirent = await this.#readDirent(await this.#ptr64(this.#header.urlPtrPos, mid));
|
||||
const dirKey = dirent.namespace + dirent.url;
|
||||
const cmp = key < dirKey ? -1 : key > dirKey ? 1 : 0;
|
||||
if (cmp === 0) return dirent;
|
||||
if (cmp < 0) hi = mid - 1; else lo = mid + 1;
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
async #getBlob(clusterNumber, blobNumber) {
|
||||
const start = await this.#ptr64(this.#header.clusterPtrPos, clusterNumber);
|
||||
const isLast = clusterNumber === this.#header.clusterCount - 1;
|
||||
const end = isLast
|
||||
? (await fs.promises.stat(this.path)).size
|
||||
: await this.#ptr64(this.#header.clusterPtrPos, clusterNumber + 1);
|
||||
|
||||
const raw = await this.#read(start, end - start);
|
||||
const compType = raw[0] & 0x0f;
|
||||
const extended = (raw[0] & 0x10) !== 0;
|
||||
const body = raw.subarray(1);
|
||||
|
||||
let data;
|
||||
if (compType <= 1) data = body;
|
||||
else if (compType === 4) data = Buffer.from(lzmaDecompress(toLzmaAloneStream(body)));
|
||||
else if (compType === 5) data = Buffer.from((await getZstd()).decompress(new Uint8Array(body)));
|
||||
else throw new Error(`Unsupported cluster compression type: ${compType}`);
|
||||
if (!data) throw new Error(`Cluster ${clusterNumber} failed to decompress (compType ${compType})`);
|
||||
|
||||
const readPtr = i => extended ? Number(data.readBigUInt64LE(i * 8)) : data.readUInt32LE(i * 4);
|
||||
return data.subarray(readPtr(blobNumber), readPtr(blobNumber + 1));
|
||||
}
|
||||
|
||||
async #ptr64(base, index) {
|
||||
return Number((await this.#read(base + index * 8, 8)).readBigUInt64LE(0));
|
||||
}
|
||||
|
||||
async #read(pos, length) {
|
||||
const buf = Buffer.alloc(length);
|
||||
await this.#fd.read(buf, 0, length, pos);
|
||||
return buf;
|
||||
}
|
||||
|
||||
async #readHeader() {
|
||||
const b = await this.#read(0, HEADER_SIZE);
|
||||
const titlePtrRaw = b.readBigUInt64LE(40);
|
||||
this.#hasTitleListing = titlePtrRaw !== TITLE_SENTINEL;
|
||||
this.#header = {
|
||||
articleCount: b.readUInt32LE(24),
|
||||
clusterCount: b.readUInt32LE(28),
|
||||
urlPtrPos: Number(b.readBigUInt64LE(32)),
|
||||
titlePtrPos: this.#hasTitleListing ? Number(titlePtrRaw) : null,
|
||||
clusterPtrPos: Number(b.readBigUInt64LE(48)),
|
||||
mimeListPos: Number(b.readBigUInt64LE(56)),
|
||||
mainPage: b.readUInt32LE(64),
|
||||
};
|
||||
}
|
||||
|
||||
async #readMimeTypes() {
|
||||
let pos = this.#header.mimeListPos, str = '';
|
||||
for (;;) {
|
||||
str += (await this.#read(pos, 1024)).toString('binary');
|
||||
const end = str.indexOf('\0\0');
|
||||
if (end !== -1) { str = str.slice(0, end + 1); break; }
|
||||
pos += 1024;
|
||||
}
|
||||
this.#mimeTypes = str.split('\0').filter(Boolean);
|
||||
}
|
||||
|
||||
/** Directory entry (article record) at byte `offset`, growing the read window as needed. */
|
||||
async #readDirent(offset) {
|
||||
for (let size = 512; ; size *= 2) {
|
||||
const buf = await this.#read(offset, size);
|
||||
let o = 0;
|
||||
const mimetype = buf.readUInt16LE(o); o += 2;
|
||||
o += 1; // extraLen, unused
|
||||
const namespace = String.fromCharCode(buf.readUInt8(o)); o += 1;
|
||||
o += 4; // revision, unused
|
||||
|
||||
let redirectIndex = null, cluster = null, blob = null;
|
||||
if (mimetype === 0xffff) { redirectIndex = buf.readUInt32LE(o); o += 4; }
|
||||
else { cluster = buf.readUInt32LE(o); o += 4; blob = buf.readUInt32LE(o); o += 4; }
|
||||
|
||||
const urlEnd = buf.indexOf(0, o);
|
||||
if (urlEnd === -1) continue;
|
||||
const titleEnd = buf.indexOf(0, urlEnd + 1);
|
||||
if (titleEnd === -1) continue;
|
||||
|
||||
const url = buf.toString('utf8', o, urlEnd);
|
||||
const title = buf.toString('utf8', urlEnd + 1, titleEnd) || url;
|
||||
return {mimetype, namespace, redirectIndex, cluster, blob, url, title};
|
||||
}
|
||||
}
|
||||
|
||||
async #resolveRedirect(dirent) {
|
||||
if (dirent.mimetype !== 0xffff) return dirent;
|
||||
return this.#readDirent(await this.#ptr64(this.#header.urlPtrPos, dirent.redirectIndex));
|
||||
}
|
||||
|
||||
/** Narrows to dirents near the term's alphabetical position in the title index. */
|
||||
async #titleIndexCandidates(term) {
|
||||
const q = term.toLowerCase();
|
||||
let lo = 0, hi = this.#header.articleCount - 1;
|
||||
while (lo < hi) {
|
||||
const mid = (lo + hi) >> 1;
|
||||
const urlIdx = (await this.#read(this.#header.titlePtrPos + mid * 4, 4)).readUInt32LE(0);
|
||||
const dirent = await this.#readDirent(await this.#ptr64(this.#header.urlPtrPos, urlIdx));
|
||||
if (dirent.title.toLowerCase() < q) lo = mid + 1; else hi = mid;
|
||||
}
|
||||
// Widen around the prefix match since fuzzy scoring isn't purely alphabetical.
|
||||
const start = Math.max(0, lo - 50), end = Math.min(this.#header.articleCount, lo + 200);
|
||||
const dirents = [];
|
||||
for (let i = start; i < end; i++) {
|
||||
const urlIdx = (await this.#read(this.#header.titlePtrPos + i * 4, 4)).readUInt32LE(0);
|
||||
dirents.push(await this.#readDirent(await this.#ptr64(this.#header.urlPtrPos, urlIdx)));
|
||||
}
|
||||
return dirents;
|
||||
}
|
||||
|
||||
async close() {
|
||||
if (this.#fd) await this.#fd.close();
|
||||
this.#fd = null;
|
||||
}
|
||||
|
||||
/** Reads an 'M' namespace metadata value (e.g. Name, Date, Title). Returns null if missing. */
|
||||
async metadata(key) {
|
||||
const page = await this.readPage(key, NS_METADATA);
|
||||
return page ? page.data.toString('utf8') : null;
|
||||
}
|
||||
|
||||
/** Opens the archive and parses its header + mimetype list. */
|
||||
async open() {
|
||||
this.#fd = await fs.promises.open(this.path, 'r');
|
||||
await this.#readHeader();
|
||||
await this.#readMimeTypes();
|
||||
return this;
|
||||
}
|
||||
|
||||
/** Read a page's content by URL. Returns `{mimetype, data}` or `null` if not found. */
|
||||
async readPage(url, namespace = NS_CONTENT) {
|
||||
let dirent = await this.#findByUrl(url, namespace);
|
||||
if (!dirent) return null;
|
||||
dirent = await this.#resolveRedirect(dirent);
|
||||
const data = await this.#getBlob(dirent.cluster, dirent.blob);
|
||||
return {mimetype: this.#mimeTypes[dirent.mimetype] || 'application/octet-stream', data};
|
||||
}
|
||||
|
||||
/** Reads the archive's designated main/landing page, if one is set. */
|
||||
async mainPage() {
|
||||
if (this.#header.mainPage === 0xffffffff) return null;
|
||||
let dirent = await this.#readDirent(await this.#ptr64(this.#header.urlPtrPos, this.#header.mainPage));
|
||||
dirent = await this.#resolveRedirect(dirent);
|
||||
const data = await this.#getBlob(dirent.cluster, dirent.blob);
|
||||
return {mimetype: this.#mimeTypes[dirent.mimetype] || 'application/octet-stream', data, url: dirent.url};
|
||||
}
|
||||
|
||||
/**
|
||||
* Fuzzy-ranked title search. Accepts comma-separated `terms` the same way the
|
||||
* catalog search does. Uses the sorted title index when present (binary search
|
||||
* narrows the candidate window); falls back to a full linear scan otherwise
|
||||
* (common on ZIM v6+/zimit-generated archives with no title index).
|
||||
*/
|
||||
async search(terms, {limit = 20, htmlOnly = true} = {}) {
|
||||
const termList = String(terms).split(',').map(t => t.trim()).filter(Boolean);
|
||||
if (!termList.length) return [];
|
||||
|
||||
const candidates = this.#hasTitleListing
|
||||
? await this.#titleIndexCandidates(termList[0])
|
||||
: await this.#allDirents();
|
||||
|
||||
const scored = [];
|
||||
for (const dirent of candidates) {
|
||||
if (dirent.namespace !== NS_CONTENT) continue;
|
||||
if (htmlOnly && !(this.#mimeTypes[dirent.mimetype] || '').startsWith('text/html')) continue;
|
||||
const {max} = fuzzyMatch(dirent.title, ...termList);
|
||||
scored.push({url: dirent.url, title: dirent.title, namespace: dirent.namespace, score: max});
|
||||
}
|
||||
return scored.toSorted((a, b) => b.score - a.score).slice(0, limit);
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user