diff --git a/bin/install-kwix.js b/bin/install-kwix.js index b0d6cd5..16ea2ce 100644 --- a/bin/install-kwix.js +++ b/bin/install-kwix.js @@ -3,111 +3,184 @@ import os from 'node:os'; import path from 'node:path'; import https from 'node:https'; import {fileURLToPath} from 'node:url'; -import {createWriteStream} from 'node:fs'; import AdmZip from 'adm-zip'; import * as tar from 'tar'; const __dirname = path.dirname(fileURLToPath(import.meta.url)); - const VERSION = '3.8.1'; const BASE_URL = 'https://download.kiwix.org/release/kiwix-tools'; -const PLATFORM_MAP = {linux: 'linux', darwin: 'macos', win32: 'win'}; - +const MAX_REDIRECTS = 5; const BIN_DIR = path.join(__dirname, '..', 'bin'); +const PLATFORM_MAP = { + linux: {name: 'linux', ext: 'tar.gz'}, + darwin: {name: 'macos', ext: 'tar.gz'}, + win32: {name: 'win', ext: 'zip'} +}; +const ARCH_MAP = { + linux: { + x64: 'x86_64', + arm64: 'aarch64', + arm: 'armhf', + ia32: 'i586' + }, + darwin: { + x64: 'x86_64', + arm64: 'arm64' + }, + win32: { + x64: 'x86_64', + ia32: 'i686' + } +}; -function download(url, dest) { + +function download(url, dest, redirectsLeft = MAX_REDIRECTS) { return new Promise((resolve, reject) => { const file = fs.createWriteStream(dest); - https.get(url, res => { + const agent = new https.Agent({maxFreeSockets: 20, maxTotalSockets: 50}); + + const cleanupAndReject = err => { + file.close(); + fs.unlink(dest, () => reject(err)); + }; + + const req = https.get(url, {agent}, res => { if (res.statusCode >= 300 && res.statusCode < 400 && res.headers.location) { + res.resume(); file.close(); - return resolve(download(res.headers.location, dest)); + if (redirectsLeft <= 0) { + reject(new Error(`Too many redirects while downloading ${url}`)); + return; + } + + const nextUrl = new URL(res.headers.location, url).toString(); + resolve(download(nextUrl, dest, redirectsLeft - 1)); + return; } + if (!res.statusCode || res.statusCode >= 400) { - return reject(new Error(`Download failed: ${res.statusCode} ${res.statusMessage}`)); + res.resume(); + cleanupAndReject(new Error(`HTTP ${res.statusCode} - ${res.statusMessage}`)); + return; } + res.pipe(file); - file.on('finish', () => file.close(resolve)); - }).on('error', err => fs.unlink(dest, () => reject(err))); + file.on('finish', () => { + file.close(resolve); + }); + file.on('error', cleanupAndReject); + }); + + req.on('error', cleanupAndReject); }); } -function chmodRecursive(dir, mode) { - for (const entry of fs.readdirSync(dir, {withFileTypes: true})) { - const full = path.join(dir, entry.name); - if (entry.isDirectory()) chmodRecursive(full, mode); - else fs.chmodSync(full, mode); +async function flattenExtractedDir(srcDir, destDir) { + let entries = await fs.promises.readdir(srcDir, {withFileTypes: true}); + + let sourceDir = srcDir; + if (entries.length === 1 && entries[0].isDirectory()) { + sourceDir = path.join(srcDir, entries[0].name); + entries = await fs.promises.readdir(sourceDir, {withFileTypes: true}); } -} -async function downloadAndExtract() { - const platformName = PLATFORM_MAP[process.platform]; - if (!platformName) throw new Error(`Unsupported platform: ${process.platform}`); + for (const entry of entries) { + const src = path.join(sourceDir, entry.name); + const dest = path.join(destDir, entry.name); - const ext = platformName === 'win' ? 'zip' : 'tar.gz'; - const archiveName = `kiwix-tools_${platformName}-x86_64-${VERSION}.${ext}`; - const url = `${BASE_URL}/${archiveName}`; - const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'kiwix-tools-')); - const archivePath = path.join(tmpDir, archiveName); - - console.log('Downloading kiwix-tools from:', url); - await download(url, archivePath); - - await fs.promises.mkdir(BIN_DIR, {recursive: true}); - - console.log('Extracting...'); - - // Use npm-installable libraries for extraction - if (archiveName.endsWith('.zip')) { - const zip = new AdmZip(archivePath); - const zipName = zip.getZipName(); // e.g. "kiwix-tools_linux-x86_64-3.8.1" - const extractDir = path.join(BIN_DIR, zipName); - - await zip.extractAllToAsync(extractDir, true); - - // Flatten the extracted directory into BIN_DIR - if (fs.existsSync(extractDir)) { - for (const entry of await fs.promises.readdir(extractDir)) { - const src = path.join(extractDir, entry); - const dest = path.join(BIN_DIR, entry); - await fs.promises.rename(src, dest); - } - await fs.promises.rmdir(extractDir); - } - } else if (archiveName.endsWith('.tar.gz')) { - // Extract to a temp dir first to check for subdirectory wrapper - const extractDir = path.join(tmpDir, 'extracted'); - await fs.promises.mkdir(extractDir, {recursive: true}); - - await tar.extract({ - file: archivePath, - cwd: extractDir, - silent: false + await fs.promises.cp(src, dest, { + recursive: true, + force: true }); - const list = await fs.promises.readdir(extractDir, {withFileTypes: true}); - const wrapper = list.find(e => e.isDirectory() && e.name.startsWith('kiwix-tools_')); - const finalDir = wrapper ? path.join(extractDir, wrapper.name) : extractDir; - - // Flatten into BIN_DIR - for (const entry of await fs.promises.readdir(finalDir)) { - const src = path.join(finalDir, entry); - const dest = path.join(BIN_DIR, entry); - await fs.promises.rename(src, dest); - } - - await fs.promises.rmdir(finalDir); + await fs.promises.rm(src, {recursive: true, force: true}); } - - if (process.platform !== 'win32') { - chmodRecursive(BIN_DIR, 0o755); - } - - await fs.promises.rm(tmpDir, {recursive: true, force: true}); - console.log('Installed to:', BIN_DIR); } -downloadAndExtract().catch(err => { - console.error(err); - process.exit(1) -}).finally(() => process.exit()); +async function extractTarGz(archivePath, destDir) { + const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'kiwix-extract-')); + + await tar.extract({ + file: archivePath, + cwd: tmpDir + }); + + await flattenExtractedDir(tmpDir, destDir); + await fs.promises.rm(tmpDir, {recursive: true, force: true}); +} + +async function extractZip(archivePath, destDir) { + const zip = new AdmZip(archivePath); + const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'kiwix-extract-')); + + await new Promise((resolve, reject) => { + zip.extractAllToAsync(tmpDir, true, false, err => { + if (err) reject(err); + else resolve(); + }); + }); + + await flattenExtractedDir(tmpDir, destDir); + await fs.promises.rm(tmpDir, {recursive: true, force: true}); +} + +function resolveArchiveName() { + const platformInfo = PLATFORM_MAP[process.platform]; + if (!platformInfo) { + throw new Error(`Unsupported platform: ${process.platform}`); + } + + const archName = ARCH_MAP[process.platform]?.[process.arch]; + if (!archName) { + const supported = Object.keys(ARCH_MAP[process.platform] || {}).join(', '); + throw new Error( + `Unsupported architecture "${process.arch}" for platform "${process.platform}". ` + + `Supported architectures: ${supported}` + ); + } + + return { + archiveName: `kiwix-tools_${platformInfo.name}-${archName}-${VERSION}.${platformInfo.ext}`, + ext: platformInfo.ext + }; +} + +async function install() { + const {archiveName, ext} = resolveArchiveName(); + const url = `${BASE_URL}/${archiveName}`; + const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'kiwix-install-')); + const archivePath = path.join(tmpDir, archiveName); + + console.log(`Platform: ${process.platform}/${process.arch}`); + console.log(`Downloading kiwix-tools: ${url}`); + await download(url, archivePath); + console.log('Download complete!'); + + console.log('Extracting...'); + await fs.promises.mkdir(BIN_DIR, {recursive: true}); + + if (ext === 'zip') { + await extractZip(archivePath, BIN_DIR); + } else { + await extractTarGz(archivePath, BIN_DIR); + } + + for (const entry of await fs.promises.readdir(BIN_DIR)) { + if (entry.startsWith('kiwix-')) { + const fullPath = path.join(BIN_DIR, entry); + const stat = await fs.promises.stat(fullPath); + + if (stat.isFile()) { + await fs.promises.chmod(fullPath, 0o755); + } + } + } + + console.log(`Installed to: ${BIN_DIR}`); + await fs.promises.rm(tmpDir, {recursive: true, force: true}); +} + +install().catch(err => { + console.error('Install failed:', err.message); + process.exit(1); +}); diff --git a/package-lock.json b/package-lock.json index d70aa8b..8a9a33c 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,20 +1,34 @@ { "name": "@ztimson/zim-utils", - "version": "0.2.5", + "version": "0.3.6", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "@ztimson/zim-utils", - "version": "0.2.5", + "version": "0.3.6", "hasInstallScript": true, "license": "MIT", "dependencies": { "@ztimson/utils": "^0.30.8", + "adm-zip": "^0.5.16", "lzma1": "^0.3.0", + "tar": "^7.4.3", "zstd-codec": "^0.1.5" } }, + "node_modules/@isaacs/fs-minipass": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/@isaacs/fs-minipass/-/fs-minipass-4.0.1.tgz", + "integrity": "sha512-wgm9Ehl2jpeqP3zw/7mo3kRHFp5MEDhqAdwy1fTGkHAwnkGOVsgpvQhL8B5n1qlb01jV3n/bI0ZfZp5lWA1k4w==", + "license": "ISC", + "dependencies": { + "minipass": "^7.0.4" + }, + "engines": { + "node": ">=18.0.0" + } + }, "node_modules/@ztimson/utils": { "version": "0.30.8", "resolved": "https://registry.npmjs.org/@ztimson/utils/-/utils-0.30.8.tgz", @@ -24,6 +38,24 @@ "var-persist": "^1.0.1" } }, + "node_modules/adm-zip": { + "version": "0.5.18", + "resolved": "https://registry.npmjs.org/adm-zip/-/adm-zip-0.5.18.tgz", + "integrity": "sha512-ufJnssQGbxzLNS1Ho9bCtX4rQKCCvoVuDLHoJyc3F9dOGDB4BkWs2Ci0kv53lqocAEQ/Cbi+I2XCsNYGqVYqng==", + "license": "MIT", + "engines": { + "node": ">=12.0" + } + }, + "node_modules/chownr": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/chownr/-/chownr-3.0.0.tgz", + "integrity": "sha512-+IxzY9BZOQd/XuYPRmrvEVjF/nqj5kgT4kEq7VofrDoM1MxoRjEWkrCC3EtLi59TVawxTAn+orJwFQcrqEN1+g==", + "license": "BlueOak-1.0.0", + "engines": { + "node": ">=18" + } + }, "node_modules/lzma1": { "version": "0.3.0", "resolved": "https://registry.npmjs.org/lzma1/-/lzma1-0.3.0.tgz", @@ -37,12 +69,58 @@ "url": "https://github.com/sponsors/xseman" } }, + "node_modules/minipass": { + "version": "7.1.3", + "resolved": "https://registry.npmjs.org/minipass/-/minipass-7.1.3.tgz", + "integrity": "sha512-tEBHqDnIoM/1rXME1zgka9g6Q2lcoCkxHLuc7ODJ5BxbP5d4c2Z5cGgtXAku59200Cx7diuHTOYfSBD8n6mm8A==", + "license": "BlueOak-1.0.0", + "engines": { + "node": ">=16 || 14 >=14.17" + } + }, + "node_modules/minizlib": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/minizlib/-/minizlib-3.1.0.tgz", + "integrity": "sha512-KZxYo1BUkWD2TVFLr0MQoM8vUUigWD3LlD83a/75BqC+4qE0Hb1Vo5v1FgcfaNXvfXzr+5EhQ6ing/CaBijTlw==", + "license": "MIT", + "dependencies": { + "minipass": "^7.1.2" + }, + "engines": { + "node": ">= 18" + } + }, + "node_modules/tar": { + "version": "7.5.22", + "resolved": "https://registry.npmjs.org/tar/-/tar-7.5.22.tgz", + "integrity": "sha512-MFO/QzvtAOmJbkhOaCTvbGcFN9L9b+JunIsDwaKljSOdcLMea3NJ1k9Usz/rjdfSXTq4dfzfeS7W4p4YOAAHeA==", + "license": "BlueOak-1.0.0", + "dependencies": { + "@isaacs/fs-minipass": "^4.0.0", + "chownr": "^3.0.0", + "minipass": "^7.1.2", + "minizlib": "^3.1.0", + "yallist": "^5.0.0" + }, + "engines": { + "node": ">=18" + } + }, "node_modules/var-persist": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/var-persist/-/var-persist-1.0.1.tgz", "integrity": "sha512-Zon+pwvEpb0dEQCVShoMQQWV1JbWi4P2knW3h2sfSZS3pLecgbFig76tMSHEECQuEQ3KfYhMXgRDDIybtHTyZw==", "license": "MIT" }, + "node_modules/yallist": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/yallist/-/yallist-5.0.0.tgz", + "integrity": "sha512-YgvUTfwqyc7UXVMrB+SImsVYSmTS8X/tSrtdNZMImM+n7+QTriRXyXim0mBrTXNeqzVF0KWGgHPeiyViFFrNDw==", + "license": "BlueOak-1.0.0", + "engines": { + "node": ">=18" + } + }, "node_modules/zstd-codec": { "version": "0.1.5", "resolved": "https://registry.npmjs.org/zstd-codec/-/zstd-codec-0.1.5.tgz", diff --git a/package.json b/package.json index c45cb06..73a655f 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@ztimson/zim-utils", - "version": "0.3.5", + "version": "0.3.6", "description": "Native, dependency-light ZIM archive reader/searcher and Kiwix catalog downloader for Node.js", "author": "Zak Timson", "license": "MIT",