Compare commits

...
4 Commits
Author SHA1 Message Date
ztimson 60bceb195f Fix name
Publish Library / Build NPM Project (push) Successful in 1m35s
Publish Library / Tag Version (push) Successful in 8s
2026-10-05 23:14:14 -04:00
ztimson 33e4ee35c7 Download timeout fix
Publish Library / Build NPM Project (push) Successful in 1m52s
Publish Library / Tag Version (push) Successful in 9s
2026-10-05 23:06:08 -04:00
ztimson 36f954bb6b Fix delete locks
Publish Library / Build NPM Project (push) Successful in 52s
Publish Library / Tag Version (push) Successful in 6s
2026-10-05 19:29:47 -04:00
ztimson a1aa158264 Fix new zim reload
Publish Library / Build NPM Project (push) Successful in 1m25s
Publish Library / Tag Version (push) Successful in 7s
2026-10-05 19:20:08 -04:00
4 changed files with 41 additions and 24 deletions
+12 -2
View File
@@ -1,12 +1,12 @@
{ {
"name": "@ztimson/zim-utils", "name": "@ztimson/zim-utils",
"version": "0.4.8", "version": "0.4.11",
"lockfileVersion": 3, "lockfileVersion": 3,
"requires": true, "requires": true,
"packages": { "packages": {
"": { "": {
"name": "@ztimson/zim-utils", "name": "@ztimson/zim-utils",
"version": "0.4.8", "version": "0.4.11",
"hasInstallScript": true, "hasInstallScript": true,
"license": "MIT", "license": "MIT",
"dependencies": { "dependencies": {
@@ -14,6 +14,7 @@
"adm-zip": "^0.5.18", "adm-zip": "^0.5.18",
"lzma1": "^0.3.0", "lzma1": "^0.3.0",
"tar": "^7.4.3", "tar": "^7.4.3",
"undici": "^8.11.2",
"zstd-codec": "^0.1.5" "zstd-codec": "^0.1.5"
} }
}, },
@@ -106,6 +107,15 @@
"node": ">=18" "node": ">=18"
} }
}, },
"node_modules/undici": {
"version": "8.11.2",
"resolved": "https://registry.npmjs.org/undici/-/undici-8.11.2.tgz",
"integrity": "sha512-u4UB2/IrKdU6lFxumHmmo1a3fCQO5tzQllRorfoRS63txhrB7xTpSn1PftwC4qEHkOaqP95fCWW4lJzwErwzhQ==",
"license": "MIT",
"engines": {
"node": ">=22.19.0"
}
},
"node_modules/var-persist": { "node_modules/var-persist": {
"version": "1.0.1", "version": "1.0.1",
"resolved": "https://registry.npmjs.org/var-persist/-/var-persist-1.0.1.tgz", "resolved": "https://registry.npmjs.org/var-persist/-/var-persist-1.0.1.tgz",
+4 -3
View File
@@ -1,6 +1,6 @@
{ {
"name": "@ztimson/zim-utils", "name": "@ztimson/zim-utils",
"version": "0.4.9", "version": "0.4.13",
"description": "Native, dependency-light ZIM archive reader/searcher and Kiwix catalog downloader for Node.js", "description": "Native, dependency-light ZIM archive reader/searcher and Kiwix catalog downloader for Node.js",
"author": "Zak Timson", "author": "Zak Timson",
"license": "MIT", "license": "MIT",
@@ -16,9 +16,10 @@
}, },
"dependencies": { "dependencies": {
"@ztimson/utils": "^0.30.9", "@ztimson/utils": "^0.30.9",
"adm-zip": "^0.5.18",
"lzma1": "^0.3.0", "lzma1": "^0.3.0",
"tar": "^7.4.3", "tar": "^7.4.3",
"zstd-codec": "^0.1.5", "undici": "^8.11.2",
"adm-zip": "^0.5.18" "zstd-codec": "^0.1.5"
} }
} }
+16 -6
View File
@@ -5,6 +5,9 @@ import {Readable} from 'node:stream';
import {PromiseProgress} from '@ztimson/utils'; import {PromiseProgress} from '@ztimson/utils';
import {KiwixServer} from './server.js'; import {KiwixServer} from './server.js';
import {zimCatalog, zimCatalogInfo, CATALOG_URL} from './catalog.js'; import {zimCatalog, zimCatalogInfo, CATALOG_URL} from './catalog.js';
import {Agent} from 'undici';
const agent = new Agent({headersTimeout: 0, bodyTimeout: 0});
const VISIBLE_TIMEOUT = 5_000; const VISIBLE_TIMEOUT = 5_000;
const VISIBLE_POLL_INTERVAL = 200; const VISIBLE_POLL_INTERVAL = 200;
@@ -65,13 +68,13 @@ export class ZimManager {
/** Resolves a `.meta4` metalink URL down to the real mirror `.zim` download URL. */ /** Resolves a `.meta4` metalink URL down to the real mirror `.zim` download URL. */
async #resolveUrl(url) { async #resolveUrl(url) {
const head = await fetch(url); const head = await fetch(url, {dispatcher: agent});
const ct = head.headers.get('content-type') || ''; const ct = head.headers.get('content-type') || '';
if (!ct.includes('metalink') && !url.endsWith('.meta4')) return {res: head, url}; if (!ct.includes('metalink') && !url.endsWith('.meta4')) return {res: head, url};
const meta = await head.text(); const meta = await head.text();
const m = meta.match(/<url[^>]*>([^<]+\.zim)<\/url>/); const m = meta.match(/<url[^>]*>([^<]+\.zim)<\/url>/);
if (!m) throw new Error('Could not resolve metalink mirror'); if (!m) throw new Error('Could not resolve metalink mirror');
const res = await fetch(m[1]); const res = await fetch(m[1], {dispatcher: agent});
return {res, url: m[1]}; return {res, url: m[1]};
} }
@@ -80,10 +83,16 @@ export class ZimManager {
* KiwixServer's directory watcher does the real reload work in the background; this just waits for it. */ * KiwixServer's directory watcher does the real reload work in the background; this just waits for it. */
async #waitUntilVisible(filename, {expect = true, timeout = VISIBLE_TIMEOUT} = {}) { async #waitUntilVisible(filename, {expect = true, timeout = VISIBLE_TIMEOUT} = {}) {
const deadline = Date.now() + timeout; const deadline = Date.now() + timeout;
const server = await this.#ensureServer();
while (Date.now() < deadline) { while (Date.now() < deadline) {
const local = await this.#rawList(); const local = await this.#rawList();
const present = local.some(l => l.file === filename); const match = local.find(l => l.file === filename);
if (present === expect) return true; if (!expect) {
if (!match) return true;
} else if (match?.name) {
const probe = await fetch(`${server.baseUrl}/content/${encodeURIComponent(match.name)}/`).catch(() => null);
if (probe?.ok) return true;
}
await new Promise(r => setTimeout(r, VISIBLE_POLL_INTERVAL)); await new Promise(r => setTimeout(r, VISIBLE_POLL_INTERVAL));
} }
return false; return false;
@@ -122,7 +131,7 @@ export class ZimManager {
async catalog(search, opts = {}) { async catalog(search, opts = {}) {
const results = await zimCatalog(search, {url: this.#catalogUrl, ...opts}); const results = await zimCatalog(search, {url: this.#catalogUrl, ...opts});
return results.map(entry => this.#active[entry.href] return results.map(entry => this.#active[entry.href]
? {...entry, ...this.#active[entry.href].dl.meta, downloading: true} ? {...this.#active[entry.href].dl.meta, ...entry, downloading: true}
: entry); : entry);
} }
@@ -136,6 +145,7 @@ export class ZimManager {
const match = local.find(l => l.name === nameOrFile || l.href === nameOrFile); const match = local.find(l => l.name === nameOrFile || l.href === nameOrFile);
if (!match) throw new Error(`ZIM not found locally: ${nameOrFile}`); if (!match) throw new Error(`ZIM not found locally: ${nameOrFile}`);
const file = match.href + (match.href.endsWith('.zim') ? '' : '.zim'); const file = match.href + (match.href.endsWith('.zim') ? '' : '.zim');
await this.#server.stop();
await fs.promises.rm(path.join(this.#dir, file), {force: true}); await fs.promises.rm(path.join(this.#dir, file), {force: true});
await this.#ensureServer(); await this.#ensureServer();
await this.#waitUntilVisible(file, {expect: false}); await this.#waitUntilVisible(file, {expect: false});
@@ -191,7 +201,7 @@ export class ZimManager {
const localNames = new Set(local.map(l => l.name)); const localNames = new Set(local.map(l => l.name));
const pending = Object.values(this.#active) const pending = Object.values(this.#active)
.filter(({entry}) => !localNames.has(entry.name)) .filter(({entry}) => !localNames.has(entry.name))
.map(({entry, dl}) => ({...entry, ...dl.meta, downloading: true})); .map(({entry, dl}) => ({...dl.meta, ...entry, downloading: true}));
return [...local, ...pending]; return [...local, ...pending];
} }
+8 -12
View File
@@ -95,20 +95,16 @@ export class KiwixServer {
async #rebuildLibrary() { async #rebuildLibrary() {
if(this.#remote) return; if(this.#remote) return;
await fs.promises.rm(this.#libraryPath, {force: true});
const files = await this.#zimFiles(); const files = await this.#zimFiles();
const tmpPath = `${this.#libraryPath}.tmp-${process.pid}`;
if(!files.length) await fs.promises.rm(tmpPath, {force: true});
return fs.promises.writeFile(this.#libraryPath, KiwixServer.#empty); if(!files.length) {
await fs.promises.writeFile(tmpPath, KiwixServer.#empty);
} else {
for(const file of files) for(const file of files)
await execFileAsync(this.#bin('kiwix-manage'), [ await execFileAsync(this.#bin('kiwix-manage'), [tmpPath, 'add', path.join(this.#dir, file)]);
this.#libraryPath, }
'add', await fs.promises.rename(tmpPath, this.#libraryPath);
path.join(this.#dir, file),
]);
} }
async #waitUntilReady() { async #waitUntilReady() {