Merge pull request 'Fix/install kwix arm64' (#1) from fix/install-kwix-arm64 into master

Reviewed-on: #1

README.md

@@ -87,6 +87,7 @@ await server.stop(); // Kill server
 ```
 
 #### List Local ZIMs
+
 ```js
 const local = await server.list();
 [

bin/install-kwix.js

@@ -2,80 +2,185 @@ import fs from 'node:fs';
 import os from 'node:os';
 import path from 'node:path';
 import https from 'node:https';
-import {execFileSync} from 'node:child_process';
 import {fileURLToPath} from 'node:url';
+import AdmZip from 'adm-zip';
+import * as tar from 'tar';
 
 const __dirname = path.dirname(fileURLToPath(import.meta.url));
-
 const VERSION = '3.8.1';
 const BASE_URL = 'https://download.kiwix.org/release/kiwix-tools';
-const PLATFORM_MAP = {linux: 'linux', darwin: 'macos', win32: 'win'};
+const MAX_REDIRECTS = 5;
+const BIN_DIR = path.join(__dirname, '..', 'bin');
+const PLATFORM_MAP = {
+	linux: {name: 'linux', ext: 'tar.gz'},
+	darwin: {name: 'macos', ext: 'tar.gz'},
+	win32: {name: 'win', ext: 'zip'}
+};
+const ARCH_MAP = {
+	linux: {
+		x64: 'x86_64',
+		arm64: 'aarch64',
+		arm: 'armhf',
+		ia32: 'i586'
+	},
+	darwin: {
+		x64: 'x86_64',
+		arm64: 'arm64'
+	},
+	win32: {
+		x64: 'x86_64',
+		ia32: 'i686'
+	}
+};
 
-const BIN_DIR = path.join(__dirname, '..', 'bin'); // project root/bin
 
-/** Download a file, following redirects. */
+function download(url, dest, redirectsLeft = MAX_REDIRECTS) {
-function download(url, dest) {
 	return new Promise((resolve, reject) => {
 		const file = fs.createWriteStream(dest);
-		https.get(url, res => {
+		const agent = new https.Agent({maxFreeSockets: 20, maxTotalSockets: 50});
+
+		const cleanupAndReject = err => {
+			file.close();
+			fs.unlink(dest, () => reject(err));
+		};
+
+		const req = https.get(url, {agent}, res => {
 			if (res.statusCode >= 300 && res.statusCode < 400 && res.headers.location) {
+				res.resume();
 				file.close();
-				return resolve(download(res.headers.location, dest));
+				if (redirectsLeft <= 0) {
+					reject(new Error(`Too many redirects while downloading ${url}`));
+					return;
+				}
+
+				const nextUrl = new URL(res.headers.location, url).toString();
+				resolve(download(nextUrl, dest, redirectsLeft - 1));
+				return;
 			}
+
 			if (!res.statusCode || res.statusCode >= 400) {
-				return reject(new Error(`Download failed: ${res.statusCode} ${res.statusMessage}`));
+				res.resume();
+				cleanupAndReject(new Error(`HTTP ${res.statusCode} - ${res.statusMessage}`));
+				return;
 			}
+
 			res.pipe(file);
-			file.on('finish', () => file.close(resolve));
+			file.on('finish', () => {
-		}).on('error', err => fs.unlink(dest, () => reject(err)));
+				file.close(resolve);
+			});
+			file.on('error', cleanupAndReject);
+		});
+
+		req.on('error', cleanupAndReject);
 	});
 }
 
-/** Recursively chmod all files under a directory. */
+async function flattenExtractedDir(srcDir, destDir) {
-function chmodRecursive(dir, mode) {
+	let entries = await fs.promises.readdir(srcDir, {withFileTypes: true});
-	for (const entry of fs.readdirSync(dir, {withFileTypes: true})) {
+
-		const full = path.join(dir, entry.name);
+	let sourceDir = srcDir;
-		if (entry.isDirectory()) chmodRecursive(full, mode);
+	if (entries.length === 1 && entries[0].isDirectory()) {
-		else fs.chmodSync(full, mode);
+		sourceDir = path.join(srcDir, entries[0].name);
+		entries = await fs.promises.readdir(sourceDir, {withFileTypes: true});
+	}
+
+	for (const entry of entries) {
+		const src = path.join(sourceDir, entry.name);
+		const dest = path.join(destDir, entry.name);
+
+		await fs.promises.cp(src, dest, {
+			recursive: true,
+			force: true
+		});
+
+		await fs.promises.rm(src, {recursive: true, force: true});
 	}
 }
 
-async function downloadAndExtract() {
+async function extractTarGz(archivePath, destDir) {
-	const platformName = PLATFORM_MAP[process.platform];
+	const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'kiwix-extract-'));
-	if (!platformName) throw new Error(`Unsupported platform: ${process.platform}`);
 
-	const ext = platformName === 'win' ? 'zip' : 'tar.gz';
+	await tar.extract({
-	const archiveName = `kiwix-tools_${platformName}-x86_64-${VERSION}.${ext}`;
+		file: archivePath,
+		cwd: tmpDir
+	});
+
+	await flattenExtractedDir(tmpDir, destDir);
+	await fs.promises.rm(tmpDir, {recursive: true, force: true});
+}
+
+async function extractZip(archivePath, destDir) {
+	const zip = new AdmZip(archivePath);
+	const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'kiwix-extract-'));
+
+	await new Promise((resolve, reject) => {
+		zip.extractAllToAsync(tmpDir, true, false, err => {
+			if (err) reject(err);
+			else resolve();
+		});
+	});
+
+	await flattenExtractedDir(tmpDir, destDir);
+	await fs.promises.rm(tmpDir, {recursive: true, force: true});
+}
+
+function resolveArchiveName() {
+	const platformInfo = PLATFORM_MAP[process.platform];
+	if (!platformInfo) {
+		throw new Error(`Unsupported platform: ${process.platform}`);
+	}
+
+	const archName = ARCH_MAP[process.platform]?.[process.arch];
+	if (!archName) {
+		const supported = Object.keys(ARCH_MAP[process.platform] || {}).join(', ');
+		throw new Error(
+			`Unsupported architecture "${process.arch}" for platform "${process.platform}". ` +
+			`Supported architectures: ${supported}`
+		);
+	}
+
+	return {
+		archiveName: `kiwix-tools_${platformInfo.name}-${archName}-${VERSION}.${platformInfo.ext}`,
+		ext: platformInfo.ext
+	};
+}
+
+async function install() {
+	const {archiveName, ext} = resolveArchiveName();
 	const url = `${BASE_URL}/${archiveName}`;
-	const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'kiwix-tools-'));
+	const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'kiwix-install-'));
 	const archivePath = path.join(tmpDir, archiveName);
 
-	console.log('Downloading kiwix-tools from:', url);
+	console.log(`Platform: ${process.platform}/${process.arch}`);
+	console.log(`Downloading kiwix-tools: ${url}`);
 	await download(url, archivePath);
-
+	console.log('Download complete!');
-	await fs.promises.mkdir(BIN_DIR, {recursive: true});
 
 	console.log('Extracting...');
-	execFileSync('tar', ['-xf', archivePath, '-C', BIN_DIR]); // bsdtar handles zip too
+	await fs.promises.mkdir(BIN_DIR, {recursive: true});
 
-	// Archives (tar.gz) may wrap contents in a subdirectory - flatten it into BIN_DIR
+	if (ext === 'zip') {
-	const wrapperDir = (await fs.promises.readdir(BIN_DIR, {withFileTypes: true}))
+		await extractZip(archivePath, BIN_DIR);
-		.find(e => e.isDirectory() && e.name.startsWith('kiwix-tools_'));
+	} else {
-	if (wrapperDir) {
+		await extractTarGz(archivePath, BIN_DIR);
-		const wrapperPath = path.join(BIN_DIR, wrapperDir.name);
-		for (const entry of await fs.promises.readdir(wrapperPath)) {
-			await fs.promises.rename(path.join(wrapperPath, entry), path.join(BIN_DIR, entry));
-		}
-		await fs.promises.rmdir(wrapperPath);
 	}
 
-	if (process.platform !== 'win32') chmodRecursive(BIN_DIR, 0o755);
+	for (const entry of await fs.promises.readdir(BIN_DIR)) {
+		if (entry.startsWith('kiwix-')) {
+			const fullPath = path.join(BIN_DIR, entry);
+			const stat = await fs.promises.stat(fullPath);
 
+			if (stat.isFile()) {
+				await fs.promises.chmod(fullPath, 0o755);
+			}
+		}
+	}
+
+	console.log(`Installed to: ${BIN_DIR}`);
 	await fs.promises.rm(tmpDir, {recursive: true, force: true});
-	console.log('Installed to:', BIN_DIR);
 }
 
-downloadAndExtract().catch(err => {
+install().catch(err => {
-	console.error(err);
+	console.error('Install failed:', err.message);
-	process.exit(1)
+	process.exit(1);
-}).finally(() => process.exit());
+});

package-lock.json

@@ -1,20 +1,34 @@
 {
 	"name": "@ztimson/zim-utils",
-	"version": "0.2.5",
+	"version": "0.3.6",
 	"lockfileVersion": 3,
 	"requires": true,
 	"packages": {
 		"": {
 			"name": "@ztimson/zim-utils",
-			"version": "0.2.5",
+			"version": "0.3.6",
 			"hasInstallScript": true,
 			"license": "MIT",
 			"dependencies": {
 				"@ztimson/utils": "^0.30.8",
+				"adm-zip": "^0.5.16",
 				"lzma1": "^0.3.0",
+				"tar": "^7.4.3",
 				"zstd-codec": "^0.1.5"
 			}
 		},
+		"node_modules/@isaacs/fs-minipass": {
+			"version": "4.0.1",
+			"resolved": "https://registry.npmjs.org/@isaacs/fs-minipass/-/fs-minipass-4.0.1.tgz",
+			"integrity": "sha512-wgm9Ehl2jpeqP3zw/7mo3kRHFp5MEDhqAdwy1fTGkHAwnkGOVsgpvQhL8B5n1qlb01jV3n/bI0ZfZp5lWA1k4w==",
+			"license": "ISC",
+			"dependencies": {
+				"minipass": "^7.0.4"
+			},
+			"engines": {
+				"node": ">=18.0.0"
+			}
+		},
 		"node_modules/@ztimson/utils": {
 			"version": "0.30.8",
 			"resolved": "https://registry.npmjs.org/@ztimson/utils/-/utils-0.30.8.tgz",
@@ -24,6 +38,24 @@
 				"var-persist": "^1.0.1"
 			}
 		},
+		"node_modules/adm-zip": {
+			"version": "0.5.18",
+			"resolved": "https://registry.npmjs.org/adm-zip/-/adm-zip-0.5.18.tgz",
+			"integrity": "sha512-ufJnssQGbxzLNS1Ho9bCtX4rQKCCvoVuDLHoJyc3F9dOGDB4BkWs2Ci0kv53lqocAEQ/Cbi+I2XCsNYGqVYqng==",
+			"license": "MIT",
+			"engines": {
+				"node": ">=12.0"
+			}
+		},
+		"node_modules/chownr": {
+			"version": "3.0.0",
+			"resolved": "https://registry.npmjs.org/chownr/-/chownr-3.0.0.tgz",
+			"integrity": "sha512-+IxzY9BZOQd/XuYPRmrvEVjF/nqj5kgT4kEq7VofrDoM1MxoRjEWkrCC3EtLi59TVawxTAn+orJwFQcrqEN1+g==",
+			"license": "BlueOak-1.0.0",
+			"engines": {
+				"node": ">=18"
+			}
+		},
 		"node_modules/lzma1": {
 			"version": "0.3.0",
 			"resolved": "https://registry.npmjs.org/lzma1/-/lzma1-0.3.0.tgz",
@@ -37,12 +69,58 @@
 				"url": "https://github.com/sponsors/xseman"
 			}
 		},
+		"node_modules/minipass": {
+			"version": "7.1.3",
+			"resolved": "https://registry.npmjs.org/minipass/-/minipass-7.1.3.tgz",
+			"integrity": "sha512-tEBHqDnIoM/1rXME1zgka9g6Q2lcoCkxHLuc7ODJ5BxbP5d4c2Z5cGgtXAku59200Cx7diuHTOYfSBD8n6mm8A==",
+			"license": "BlueOak-1.0.0",
+			"engines": {
+				"node": ">=16 || 14 >=14.17"
+			}
+		},
+		"node_modules/minizlib": {
+			"version": "3.1.0",
+			"resolved": "https://registry.npmjs.org/minizlib/-/minizlib-3.1.0.tgz",
+			"integrity": "sha512-KZxYo1BUkWD2TVFLr0MQoM8vUUigWD3LlD83a/75BqC+4qE0Hb1Vo5v1FgcfaNXvfXzr+5EhQ6ing/CaBijTlw==",
+			"license": "MIT",
+			"dependencies": {
+				"minipass": "^7.1.2"
+			},
+			"engines": {
+				"node": ">= 18"
+			}
+		},
+		"node_modules/tar": {
+			"version": "7.5.22",
+			"resolved": "https://registry.npmjs.org/tar/-/tar-7.5.22.tgz",
+			"integrity": "sha512-MFO/QzvtAOmJbkhOaCTvbGcFN9L9b+JunIsDwaKljSOdcLMea3NJ1k9Usz/rjdfSXTq4dfzfeS7W4p4YOAAHeA==",
+			"license": "BlueOak-1.0.0",
+			"dependencies": {
+				"@isaacs/fs-minipass": "^4.0.0",
+				"chownr": "^3.0.0",
+				"minipass": "^7.1.2",
+				"minizlib": "^3.1.0",
+				"yallist": "^5.0.0"
+			},
+			"engines": {
+				"node": ">=18"
+			}
+		},
 		"node_modules/var-persist": {
 			"version": "1.0.1",
 			"resolved": "https://registry.npmjs.org/var-persist/-/var-persist-1.0.1.tgz",
 			"integrity": "sha512-Zon+pwvEpb0dEQCVShoMQQWV1JbWi4P2knW3h2sfSZS3pLecgbFig76tMSHEECQuEQ3KfYhMXgRDDIybtHTyZw==",
 			"license": "MIT"
 		},
+		"node_modules/yallist": {
+			"version": "5.0.0",
+			"resolved": "https://registry.npmjs.org/yallist/-/yallist-5.0.0.tgz",
+			"integrity": "sha512-YgvUTfwqyc7UXVMrB+SImsVYSmTS8X/tSrtdNZMImM+n7+QTriRXyXim0mBrTXNeqzVF0KWGgHPeiyViFFrNDw==",
+			"license": "BlueOak-1.0.0",
+			"engines": {
+				"node": ">=18"
+			}
+		},
 		"node_modules/zstd-codec": {
 			"version": "0.1.5",
 			"resolved": "https://registry.npmjs.org/zstd-codec/-/zstd-codec-0.1.5.tgz",

package.json

@@ -1,6 +1,6 @@
 {
 	"name": "@ztimson/zim-utils",
-	"version": "0.3.1",
+	"version": "0.3.6",
 	"description": "Native, dependency-light ZIM archive reader/searcher and Kiwix catalog downloader for Node.js",
 	"author": "Zak Timson",
 	"license": "MIT",
@@ -17,6 +17,8 @@
 	"dependencies": {
 		"@ztimson/utils": "^0.30.8",
 		"lzma1": "^0.3.0",
-		"zstd-codec": "^0.1.5"
+		"tar": "^7.4.3",
+		"zstd-codec": "^0.1.5",
+		"adm-zip": "^0.5.16"
 	}
-}
+}

src/manager.js

@@ -10,7 +10,7 @@ import {zimCatalog, zimCatalogInfo, CATALOG_URL} from './catalog.js';
 export class ZimManager {
 	#catalogUrl;
 	#dir;
-	server;
+	#server;
 	#ownsServer;
 
 	/** @param {{catalog?: string, server?: KiwixServer, port?: number, host?: string, binDir?: string, url?: string}} [opts]
@@ -21,15 +21,15 @@ export class ZimManager {
 		this.#catalogUrl = catalog;
 		this.#dir = dir;
 		this.#ownsServer = !server;
-		this.server = server ?? new KiwixServer(dir, {port, host, binDir, url});
+		this.#server = server ?? new KiwixServer(dir, {port, host, binDir, url});
 	}
 
 	/** The KiwixServer backing this manager - reuse it directly for content/search access, or pass into another ZimManager. */
-	get server() { return this.server; }
+	get server() { return this.#server; }
 
 	async #ensureServer() {
-		if (!this.server.running) await this.server.start();
+		if (!this.#server.running) await this.#server.start();
-		return this.server;
+		return this.#server;
 	}
 
 	async #download(url, destPath) {
@@ -74,14 +74,15 @@ export class ZimManager {
 
 	/** Stops the internally-owned KiwixServer, if this manager created its own (no-op if one was passed in). */
 	async close() {
-		if (this.#ownsServer) await this.server.stop();
+		if (this.#ownsServer) await this.#server.stop();
 	}
 
 	async delete(nameOrFile) {
 		const local = await this.list();
-		const match = local.find(l => l.name === nameOrFile || l.file === nameOrFile);
+		const match = local.find(l => l.name === nameOrFile || l.href === nameOrFile);
 		if (!match) throw new Error(`ZIM not found locally: ${nameOrFile}`);
-		await fs.promises.rm(path.join(this.#dir, match.file), {force: true});
+		const file = match.href + (match.href.endsWith('.zim') ? '' : '.zim');
+		await fs.promises.rm(path.join(this.#dir, file), {force: true});
 		const server = await this.#ensureServer();
 		await server.reload();
 		return {file: match.file, status: 'deleted'};
@@ -89,9 +90,9 @@ export class ZimManager {
 
 	/** Downloads/updates a single ZIM by direct href, matching against any existing local copy by name. */
 	async download(href, {force = false} = {}) {
-		await fs.promises.mkdir(this.#dir, {recursive: true});
 		const {url: finalUrl} = await this.#resolveUrl(href);
 		const filename = path.basename(new URL(finalUrl).pathname).replace(/\.meta4$/i, '');
+		await fs.promises.mkdir(this.#dir, {recursive: true});
 		const name = filename.replace(/\.zim$/i, '').replace(/_\d{4}-\d{2}(?:_\d+)?$/, '');
 
 		const local = await this.list();

src/server.js

@@ -6,8 +6,7 @@ import net from 'node:net';
 import fs from 'node:fs';
 import path from 'node:path';
 import {fileURLToPath} from 'node:url';
-import {decodeHtml, fromXml} from '@ztimson/utils';
+import {fromXml} from '@ztimson/utils';
-import {fuzzyMatch, weightedScore} from './utils.js';
 
 const execFileAsync = promisify(execFile);
 const __dirname = path.dirname(fileURLToPath(import.meta.url));
@@ -28,6 +27,8 @@ function findFreePort() {
 
 /** Owns a kiwix-serve process's full lifecycle: library.xml, start/stop/reload, content + search access. */
 export class KiwixServer {
+	static #empty = '<?xml version="1.0" encoding="UTF-8" ?>\n<library version="20110515"></library>\n';
+
 	#dir;
 	#host;
 	#port;
@@ -50,6 +51,12 @@ export class KiwixServer {
 		this.#binDir = binDir;
 		this.#libraryPath = path.join(dir, 'library.xml');
 		this.#remote = url ? url.replace(/\/$/, '') : null;
+		if (!this.#remote) this.#ensureLocalStore();
+	}
+
+	#ensureLocalStore() {
+		fs.mkdirSync(this.#dir, {recursive: true});
+		if (!fs.existsSync(this.#libraryPath)) fs.writeFileSync(this.#libraryPath, KiwixServer.#empty);
 	}
 
 	#assertRunning() {
@@ -62,15 +69,22 @@ export class KiwixServer {
 
 	/** Reads library.xml from disk if we own the server, or over HTTP if attached to a remote one. */
 	async #fetchLibraryXml() {
-		if (this.#remote) return (await fetch(`${this.#remote}/library.xml`).catch(() => null))?.text?.() ?? '';
+		if (!this.#remote) return fs.promises.readFile(this.#libraryPath, 'utf8').catch(() => '');
-		return fs.promises.readFile(this.#libraryPath, 'utf8').catch(() => '');
+		try {
+			const res = await fetch(`${this.#remote}/library.xml`);
+			return res.ok ? await res.text() : '';
+		} catch {
+			return '';
+		}
 	}
 
 	/** Rebuilds library.xml from scratch by scanning `dir` for .zim files - no-op if attached to a remote server. */
 	async #rebuildLibrary() {
 		if (this.#remote) return;
 		await fs.promises.rm(this.#libraryPath, {force: true});
-		for (const f of await this.#zimFiles()) await execFileAsync(this.#bin('kiwix-manage'), [this.#libraryPath, 'add', path.join(this.#dir, f)]);
+		const files = await this.#zimFiles();
+		if (!files.length) return fs.promises.writeFile(this.#libraryPath, KiwixServer.#empty);
+		for (const f of files) await execFileAsync(this.#bin('kiwix-manage'), [this.#libraryPath, 'add', path.join(this.#dir, f)]);
 	}
 
 	async #waitUntilReady() {
@@ -134,6 +148,7 @@ export class KiwixServer {
 	async list() {
 		this.#assertRunning();
 		const xml = await this.#fetchLibraryXml();
+		if (!xml) return [];
 		const entries = fromXml(xml);
 		return (entries?.library?.book || []).map(e => {
 			const tags = e.tags.split(';');
@@ -180,30 +195,23 @@ export class KiwixServer {
 		return {mimetype: res.headers.get('content-type'), data: Buffer.from(await res.arrayBuffer())};
 	}
 
-	/**
+	/** Fulltext search across every local ZIM via kiwix-serve's own xapian index */
-	 * Two-pass fulltext search across every local ZIM: xapian prefilter (kiwix-serve's
-	 * own index), then fuzzy-reranked by title so the strongest matches surface first.
-	 * Returns a flat array matching the catalog/list shape: {id, title, name, category, ..., href, score}
-	 */
 	async search(terms, limit = 20) {
 		this.#assertRunning();
-		const termList = String(terms).split(',').map(t => t.trim()).filter(Boolean);
+		const termList = String(terms).split(/[,\s]+/).map(t => t.trim()).filter(Boolean);
 		if (!termList.length) return [];
 
 		const params = new URLSearchParams({pattern: termList.join(' '), format: 'xml', pageLength: String(limit)});
 		const res = await fetch(`${this.baseUrl}/search?${params}`);
 		if (!res.ok) return [];
 
-		const xml = await res.text();
+		const found = fromXml(await res.text())?.rss?.channel?.item || [];
-		let found = fromXml(xml);
-		found = found?.rss?.channel?.item || [];
-
 		const books = await this.list();
 		const bookMap = new Map(books.map(b => [b.title, b]));
 
-		const enriched = found.map(hit => {
+		return found.map(hit => {
 			const book = bookMap.get(hit.book.title);
-			if(!book) return null;
+			if (!book) return null;
 			const prefix = `/content/${book.href}/`;
 			const page = hit.link.startsWith(prefix) ? hit.link.slice(prefix.length) : hit.link.replace(/^\/+/, '');
 			return {
@@ -216,10 +224,8 @@ export class KiwixServer {
 				icon: book.icon,
 				viewer: this.baseUrl + hit.link,
 				summary: hit.description,
-				score: weightedScore(hit.title, termList) + weightedScore(hit.description, termList),
+				score: +hit.score || 0,
 			};
-		}).filter(hit => !!hit && hit.score > 0);
+		}).filter(Boolean);
-
-		return enriched.toSorted((a, b) => b.score - a.score).slice(0, limit);
 	}
 }

src/utils.js

@@ -40,10 +40,3 @@ export function fuzzyMatch(target, ...terms) {
 		similarities,
 	};
 }
-
-export function weightedScore(text, termList) {
-	if (!text) return 0;
-	const covered = termList.reduce((sum, t) => sum + t.length, 0);
-	const coverage = Math.min(1, covered / text.length);
-	return fuzzyMatch(text, ...termList).max * coverage;
-}