ztimson/zim-utils
1
0
Fork 0
generated from ztimson/template
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Decompression caching and preview links
All checks were successful
Publish Library / Build NPM Project (push) Successful in 10s
Details
Publish Library / Tag Version (push) Successful in 8s
Details

Browse Source
This commit is contained in:
Zakary Timson
2026-08-22 16:52:07 -04:00
parent aedb117002
commit ffdca81c78
6 changed files with 206 additions and 54 deletions
Download Patch File Download Diff File Expand all files Collapse all files

82
src/reader.js
View File

@@ -2,8 +2,7 @@
import fs from 'node:fs';
import path from 'node:path';
import {decompress as lzmaDecompress} from 'lzma1';
import {ZstdCodec} from 'zstd-codec';
import {decompressPool} from './decompress.js';
import {fuzzyMatch, titleFromUrl} from './utils.js';
const INDEX_VERSION = 1;
@@ -12,21 +11,8 @@ const NS_CONTENT = 'C';
const NS_METADATA = 'M';
const TITLE_SENTINEL = 0xffffffffffffffffn; // Indicator -> ZIM v6+ archives with no title
let zstdStreamingPromise = null;
/** Lazily initialised, shared Zstd streaming decompressor (handles unknown-size frames). */
function getZstd() {
if (!zstdStreamingPromise) {
zstdStreamingPromise = new Promise(resolve => ZstdCodec.run(zstd => resolve(new zstd.Streaming())));
}
return zstdStreamingPromise;
}
/** Rebuilds a full 13-byte "alone" LZMA header from libzim's truncated 5-byte one (unknown size). */
function toLzmaAloneStream(body) {
const header = Buffer.concat([body.subarray(0, 5), Buffer.alloc(8, 0xff)]);
return Buffer.concat([header, body.subarray(5)]);
}
const DEFAULT_CLUSTER_CACHE_MAX = 32;
const DEFAULT_CLUSTER_TTL = 60_000;
/** Native, dependency-light reader for .zim archives. Supports zstd & LZMA cluster compression. */
export class ZimReader {
@@ -35,11 +21,19 @@ export class ZimReader {
#mimeTypes = [];
#hasTitleListing = false;
#index;
#clusterCache = new Map(); // clusterNumber -> {data, extended, timer}
#pending = new Map(); // clusterNumber -> Promise, dedupes concurrent misses
#clusterCacheMax;
#clusterTTL;
get articleCount() { return this.#header?.articleCount ?? 0; }
get mediaCount() { return this.#header?.clusterCount ?? 0; }
constructor(path) {
/** @param {{clusterCacheMax?: number, clusterTTL?: number}} [opts] clusterTTL in ms; 0/null disables idle eviction. */
constructor(path, {clusterCacheMax = DEFAULT_CLUSTER_CACHE_MAX, clusterTTL = DEFAULT_CLUSTER_TTL} = {}) {
this.path = path;
this.#clusterCacheMax = clusterCacheMax;
this.#clusterTTL = clusterTTL;
}
/** Full O(n) scan over the URL pointer list, used when there's no title index. */
@@ -66,7 +60,15 @@ export class ZimReader {
return null;
}
async #getBlob(clusterNumber, blobNumber) {
/** Resets a cluster's idle-eviction timer. No-op when TTL disabled. */
#touch(clusterNumber, entry) {
if (!this.#clusterTTL) return;
clearTimeout(entry.timer);
entry.timer = setTimeout(() => this.#clusterCache.delete(clusterNumber), this.#clusterTTL).unref();
}
/** Fetches + decompresses a cluster exactly once, offloading decompression to the worker pool. */
async #loadCluster(clusterNumber) {
const start = await this.#ptr64(this.#header.clusterPtrPos, clusterNumber);
const isLast = clusterNumber === this.#header.clusterCount - 1;
const end = isLast
@@ -79,14 +81,35 @@ export class ZimReader {
const body = raw.subarray(1);
let data;
if (compType <= 1) data = body;
else if (compType === 4) data = Buffer.from(lzmaDecompress(toLzmaAloneStream(body)));
else if (compType === 5) data = Buffer.from((await getZstd()).decompress(new Uint8Array(body)));
if (compType <= 1) data = Buffer.from(body);
else if (compType === 4 || compType === 5) data = await decompressPool.run({compType, body});
else throw new Error(`Unsupported cluster compression type: ${compType}`);
if (!data) throw new Error(`Cluster ${clusterNumber} failed to decompress (compType ${compType})`);
const readPtr = i => extended ? Number(data.readBigUInt64LE(i * 8)) : data.readUInt32LE(i * 4);
return data.subarray(readPtr(blobNumber), readPtr(blobNumber + 1));
return {data, extended};
}
async #getBlob(clusterNumber, blobNumber) {
let entry = this.#clusterCache.get(clusterNumber);
if (!entry) {
let pending = this.#pending.get(clusterNumber);
if (!pending) {
pending = this.#loadCluster(clusterNumber);
this.#pending.set(clusterNumber, pending);
}
entry = await pending;
this.#pending.delete(clusterNumber);
this.#clusterCache.set(clusterNumber, entry);
if (this.#clusterCache.size > this.#clusterCacheMax) {
const oldestKey = this.#clusterCache.keys().next().value;
clearTimeout(this.#clusterCache.get(oldestKey)?.timer);
this.#clusterCache.delete(oldestKey);
}
}
this.#touch(clusterNumber, entry);
const readPtr = i => entry.extended ? Number(entry.data.readBigUInt64LE(i * 8)) : entry.data.readUInt32LE(i * 4);
return entry.data.subarray(readPtr(blobNumber), readPtr(blobNumber + 1));
}
async #icon(size = 48) {
@@ -227,6 +250,9 @@ export class ZimReader {
async close() {
if (this.#fd) await this.#fd.close();
this.#fd = null;
for (const entry of this.#clusterCache.values()) clearTimeout(entry.timer);
this.#clusterCache.clear();
this.#pending.clear();
}
/** Deletes the zim archive and its cached index (if any). Safe to call on unopened readers. */
@@ -249,7 +275,6 @@ export class ZimReader {
['Title', 'Creator', 'Publisher', 'Date', 'Description', 'Language', 'Name', 'Tags'].map(get)
);
return {
id: name,
title,
updated: date ? new Date(date) : null,
summary: description,
@@ -257,12 +282,11 @@ export class ZimReader {
name,
category: tags ? tags.split(';')[0] || '' : '',
tags: tags ? tags.split(';') : [],
mediaCount: 0,
author: creator,
publisher,
articleCount: this.articleCount,
sizeMb: ((await fs.promises.stat(this.path)).size / 1024 / 1024).toFixed(1),
href: null,
mediaCount: this.mediaCount,
sizeMb: +((await fs.promises.stat(this.path)).size / 1024 / 1024).toFixed(1),
icon: await this.#icon(),
};
}
@@ -325,7 +349,7 @@ export class ZimReader {
const urlScore = fuzzyMatch(urlTitle, ...termList).max;
scored.push({url: dirent.url, title: dirent.title.length > urlTitle.length ? dirent.title : urlTitle, namespace: NS_CONTENT, score: Math.max(titleScore, urlScore)});
}
const {id, summary, mediaCount, articleCount, sizeMb, href, ...meta} = await this.metadata();
const {summary, mediaCount, articleCount, sizeMb, ...meta} = await this.metadata();
return scored.filter(a => a.score > 0).toSorted((a, b) => b.score - a.score).slice(0, limit).map(a => ({...meta, ...a}));
}
}