commit f08a4a0808df0af17613ace74b673e42747be94d Author: ztimson Date: Sat Sep 27 00:40:24 2025 -0400 init diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..33a1d34 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,10 @@ +.idea +.vscode +.git +.gitignore +.dockerignore +DOckerfile +docker-compose.yml +docker-compose.yaml +node_modules +archive diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml new file mode 100644 index 0000000..d65db90 --- /dev/null +++ b/.github/workflows/build.yaml @@ -0,0 +1,55 @@ +name: Build Website +run-name: Build Website + +on: + push: + +jobs: + build: + name: Build NPM Project + runs-on: ubuntu-latest + container: node + steps: + - name: Clone Repository + uses: ztimson/actions/clone@develop + + - name: Install Dependencies + run: npm i + + - name: Build Project + run: npm run build + + tag: + name: Tag Version + needs: build + runs-on: ubuntu-latest + container: node + steps: + - name: Clone Repository + uses: ztimson/actions/clone@develop + + - name: Get Version Number + run: echo "VERSION=$(cat package.json | grep version | grep -Eo ':.+' | grep -Eo '[[:alnum:]\.\/\-]+')" >> $GITHUB_ENV + + - name: Tag Version + uses: ztimson/actions/tag@develop + with: + tag: ${{env.VERSION}} + + publish: + name: Build & Push Dockerfile + needs: build + uses: ztimson/actions/.github/workflows/docker.yaml@develop + with: + name: ztimson/kiwixm + repository: ${{github.server_url}}/${{github.repository}}.git + pass: ${{secrets.DEPLOY_TOKEN}} + + publish-docker: + name: Build & Push Dockerfile + needs: build + uses: ztimson/actions/.github/workflows/docker.yaml@develop + with: + name: ztimson/kiwixm + repository: hub.docker.com + pass: ${{secrets.DOCKER_TOKEN}} diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..db5f2ff --- /dev/null +++ b/.gitignore @@ -0,0 +1,4 @@ +.idea +.vscode +node_modules +dist diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..7a48623 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,24 @@ +FROM public.ecr.aws/docker/library/node + +# Environment +ENV PORT=3000 \ + KIWIX_MANAGE=/usr/bin/kiwix-manage \ + KIWIX_SERVE=/usr/bin/kiwix-serve \ + ARCHIVE_DIR=/data/archive \ + ZIM_DIR=/data/zim + +# Install dependencies +RUN apt update && apt upgrade -y && \ + apt install -y --no-install-recommends \ + curl docker.io kiwix-tools unzip zim-tools && \ + rm -rf /var/lib/apt/lists/* + +# Setup app +WORKDIR /app +COPY . . +RUN mkdir -p $ARCHIVE_DIR && mkdir -p $ZIM_DIR && npm i + +# Startup +EXPOSE $PORT +HEALTHCHECK --interval=30s --timeout=5s --retries=3 CMD curl -f http://localhost:$PORT || exit 1 +CMD ["npm", "run", "start:prod"] diff --git a/README.md b/README.md new file mode 100644 index 0000000..d401ba2 --- /dev/null +++ b/README.md @@ -0,0 +1,104 @@ + +

+
+ + +

+ + +### Kiwix Manager + + +A web-based Kiwix management tool to create & upload zims as well as manage non zim files + + +[![Version](https://img.shields.io/badge/dynamic/json.svg?label=Version&style=for-the-badge&url=https://git.zakscode.com/api/v1/repos/ztimson/kiwixm/tags&query=$[0].name)](https://git.zakscode.com/ztimson/kiwixm/tags) +[![Pull Requests](https://img.shields.io/badge/dynamic/json.svg?label=Pull%20Requests&style=for-the-badge&url=https://git.zakscode.com/api/v1/repos/ztimson/kiwixm&query=open_pr_counter)](https://git.zakscode.com/ztimson/kiwixm/pulls) +[![Issues](https://img.shields.io/badge/dynamic/json.svg?label=Issues&style=for-the-badge&url=https://git.zakscode.com/api/v1/repos/ztimson/kiwixm&query=open_issues_count)](https://git.zakscode.com/ztimson/kiwixm/issues) + + + + --- +

+ Documentation + • Release Notes + • Report a Bug + • Request a Feature +

+ + --- +

+ +## Table of Contents +- [Kiwix Manager](#top) + - [About](#about) + - [Features](#features) + - [Built With](#built-with) + - [Setup](#setup) + - [Production](#production) + - [Development](#development) + - [License](#license) + +## About + +Kiwix Manager is a comprehensive web-based management interface for Kiwix servers that extends beyond traditional ZIM file management. It provides both a powerful Kiwix library management system and an integrated file browser for non-ZIM content, making it a complete solution for offline content libraries. + +### Features + +- **Non-ZIM library browser + uploader integrated into Kiwix** - Browse and upload any file type through a custom file browser that works alongside your Kiwix content +- **Kiwix management interface for updating library content** - Full administrative control over your Kiwix server and ZIM library +- **Upload ZIMs from computer or download them directly from Kiwix library** - Flexible ZIM file management with local uploads and remote downloads +- **Create ZIMs from URLs (zimmit)** - Convert websites directly to ZIM files with customizable scraping options +- **Create ZIMs from zips / files** - Generate ZIM archives from your existing file collections + +### Built With +[![Docker](https://img.shields.io/badge/Docker-384d54?style=for-the-badge&logo=docker)](https://docker.com/) +[![Express](https://img.shields.io/badge/Express-000000?style=for-the-badge&logo=express)](https://expressjs.com/) +[![JavaScript](https://img.shields.io/badge/JavaScript-000000?style=for-the-badge&logo=javascript)](https://javascript.com/) +[![Node](https://img.shields.io/badge/Node.js-000000?style=for-the-badge&logo=nodedotjs)](https://nodejs.org/) +[![TailwindCSS](https://img.shields.io/badge/Tailwind_CSS-38B2AC?style=for-the-badge&logo=tailwind-css&logoColor=white)](https://tailwindcss.com/) + +## Setup + +

+ Production +

+ +#### Prerequisites +- [Docker](https://docs.docker.com/install/) + +#### Instructions +1. Run the docker image: `docker run -p 3000:3000 -v /path/to/zim/files:/data/zim -v /path/to/archive/files:/data/archive git.zakscode.com/ztimson/kiwixm:latest` +2. Open [http://localhost:3000](http://localhost:3000) +3. Access the admin panel at [http://localhost:3000/admin](http://localhost:3000/admin) +4. Browse non-ZIM files at [http://localhost:3000/archive](http://localhost:3000/archive) +

+ +

+ Development +

+ +#### Prerequisites +- [Node.js](https://nodejs.org/en/download) +- [Kiwix Tools](https://kiwix.org/en/downloads/kiwix-tools/) (for ZIM file management) + +#### Instructions +1. Clone the repository: `git clone https://git.zakscode.com/ztimson/kiwixm.git` +2. Navigate to the project directory: `cd kiwixm` +3. Install the dependencies: `npm install` +4. Create the required directories: `mkdir -p data/zim data/archive` +5. Start the server: `npm run start` +6. Open [http://localhost:3000](http://localhost:3000) + +

+ +## License +Copyright © 2023 Zakary Timson | All Rights Reserved | Available under MIT Licensing + +See the [license](./LICENSE) for more information. diff --git a/archive/index.html b/archive/index.html new file mode 100644 index 0000000..cfd35fd --- /dev/null +++ b/archive/index.html @@ -0,0 +1,13 @@ + + + + Kiwix Archive + + + +

Kiwix Archive

+ + + diff --git a/archive/logo.png b/archive/logo.png new file mode 100644 index 0000000..55ad032 Binary files /dev/null and b/archive/logo.png differ diff --git a/assets/archive.zim b/assets/archive.zim new file mode 100644 index 0000000..3aaad76 Binary files /dev/null and b/assets/archive.zim differ diff --git a/favicon.png b/favicon.png new file mode 100644 index 0000000..32aea5e Binary files /dev/null and b/favicon.png differ diff --git a/package.json b/package.json new file mode 100644 index 0000000..74af391 --- /dev/null +++ b/package.json @@ -0,0 +1,20 @@ +{ + "name": "kiwixm", + "version": "1.0.0", + "description": "Kiwix Library Manager", + "main": "src/main.mjs", + "type": "module", + "scripts": { + "start": "node src/main.mjs", + "start:prod": "mv assets/* $ZIM_DIR/ && npm run start", + "archive-zim": "zimwriterfs --welcome=index.html --illustration=logo.png --language=eng --name=Archive --title=Archive --description=\"Your personal space for saving and browsing uploaded files\" --creator=ZaksCode --publisher=ZaksCode ./archive assets/archive.zim" + }, + "keywords": [], + "author": "Zak Timson", + "license": "MIT", + "dependencies": { + "dotenv": "^17.2.2", + "express": "^5.1.0", + "multer": "^2.0.2" + } +} diff --git a/src/controllers/admin.mjs b/src/controllers/admin.mjs new file mode 100644 index 0000000..fbc4a19 --- /dev/null +++ b/src/controllers/admin.mjs @@ -0,0 +1,139 @@ +import express from 'express'; +import path from 'path'; +import fs from 'fs/promises'; +import {deleteFile, downloadFile, ensureDirectoryExists, getZimFiles, upload} from '../services/files.mjs'; +import {restartKiwix} from '../services/kiwix.mjs'; +import {environment} from '../services/environment.mjs'; +import {createZimFromDirectory, createZimFromUrl} from '../services/zim.mjs'; +import {promisify} from 'util'; +import {exec} from 'child_process'; + +const execAsync = promisify(exec); +const router = express.Router(); + +export default function adminController(config) { + router.get('/download', (req, res) => { + const filePath = req.query.path; + if (!filePath || !filePath.endsWith('.zim')) return res.status(400).send('Invalid ZIM file path'); + res.download(filePath, (err) => { + if(err) { + console.error('Download error:', err.message); + res.status(500).send('Download failed'); + } + }); + }); + + // Download endpoint + router.post('/download', async (req, res, next) => { + try { + const { url, filename, restartKiwix: shouldRestart = true } = req.body; + if (!url) return res.status(400).json({ error: 'URL required' }); + + await ensureDirectoryExists(config.paths.zim); + const finalFilename = filename || path.basename(new URL(url).pathname) || 'download.zim'; + const destination = path.join(config.paths.zim, finalFilename); + + await downloadFile(url, destination); + if (shouldRestart) await restartKiwix(); + + res.json({ message: 'Downloaded successfully', path: destination }); + } catch (err) { + next(err); + } + }); + + // Upload endpoint + router.post('/upload', upload(environment.paths.zim, true).single('file'), async (req, res, next) => { + try { + if (!req.file) return res.status(400).json({ message: 'No file uploaded' }); + await restartKiwix(); + res.json({ message: 'File uploaded successfully', files: [req.file.filename] }); + } catch (err) { + next(err); + } + }); + + router.post('/zim-zip', upload('/tmp/').single('zipFile'), async (req, res, next) => { + if (!req.file) return res.status(400).json({ error: 'No ZIP file uploaded.' }); + + const zipPath = req.file.path; + const tempDir = path.join('/tmp/', `unzipped-${Date.now()}`); + + try { + await ensureDirectoryExists(tempDir); + await execAsync(`unzip -o "${zipPath}" -d "${tempDir}"`); + + const { name, welcome, language, description, tags, favicon, creator } = req.body; + if(!name) throw new Error('ZIM name is are required.'); + + const entries = await fs.readdir(tempDir, { withFileTypes: true }); + let contentPath = tempDir; + const directories = entries.filter(e => e.isDirectory()); + if (directories.length === 1 && entries.length === 1) { + contentPath = path.join(tempDir, directories[0].name); + } + + await createZimFromDirectory({ name, welcome, language, description, tags, favicon, creator }, contentPath); + await restartKiwix(); + + res.json({ message: 'ZIM from ZIP created successfully' }); + } catch (err) { + next(err); + } finally { + if (tempDir) await fs.rm(tempDir, { recursive: true, force: true }).catch(e => console.error(`Failed to cleanup temp dir: ${e.message}`)); + if (zipPath) await fs.unlink(zipPath).catch(e => console.error(`Failed to cleanup zip file: ${e.message}`)); + } + }); + + router.post('/zim-url', async (req, res, next) => { + try { + const { url, name, language, description, tags, regex } = req.body; + if (!url || !name) { + return res.status(400).json({ error: 'URL and ZIM Name are required.' }); + } + + const zimFile = await createZimFromUrl({ url, name, language, description, tags, regex }); + await restartKiwix(); + + res.json({ message: 'ZIM created successfully from URL.', file: zimFile }); + } catch (err) { + next(err); + } + }); + + // Delete file endpoint + router.post('/delete', async (req, res, next) => { + try { + await deleteFile(req.body.path); + await restartKiwix(); + res.json({ message: 'File deleted + Kiwix restarted' }); + } catch (err) { + next(err); + } + }); + + // Restart endpoint + router.get('/restart', async (req, res, next) => { + try { + const ok = await restartKiwix(); + ok ? res.json({ message: 'Restarted' }) : res.status(500).json({ error: 'Failed restart' }); + } catch (err) { + next(err); + } + }); + + // Info + router.get('/info', (req, res) => res.json(config)); + + // Health + router.get('/health', async (req, res, next) => { + try { + const zimFiles = await getZimFiles(config.paths.zim); + res.json({ status: 'healthy', zimFileCount: zimFiles.length }); + } catch (err) { + res.status(500).json({ status: 'unhealthy', error: err.message }); + } + }); + + return router; +} diff --git a/src/controllers/archive.mjs b/src/controllers/archive.mjs new file mode 100644 index 0000000..359d792 --- /dev/null +++ b/src/controllers/archive.mjs @@ -0,0 +1,167 @@ +import express from 'express'; +import path from 'path'; +import fs from 'fs/promises'; +import {ensureDirectoryExists, upload} from '../services/files.mjs'; +import {environment} from '../services/environment.mjs'; + +const router = express.Router(); + +export default function archiveController() { + const archiveRoot = environment.paths.archive; + + // Security helper to validate paths + function validatePath(requestedPath) { + const normalizedArchiveRoot = path.resolve(archiveRoot); + const normalizedPath = path.resolve(path.join(archiveRoot, requestedPath)); + + if (!normalizedPath.startsWith(normalizedArchiveRoot)) { + throw new Error('Access denied: Path outside archive directory'); + } + + return normalizedPath; + } + + // Create folder endpoint + router.post('/create-folder', async (req, res, next) => { + try { + const { path: requestedPath, name } = req.body; + if (!name || !name.trim()) { + return res.status(400).json({ error: 'Folder name is required' }); + } + + const parentPath = validatePath(requestedPath || ''); + const newFolderPath = path.join(parentPath, name.trim()); + + // Check if folder already exists + try { + await fs.access(newFolderPath); + return res.status(409).json({ error: 'Folder already exists' }); + } catch { + // Folder doesn't exist, which is what we want + } + + await fs.mkdir(newFolderPath, { recursive: true }); + res.json({ message: 'Folder created successfully', path: newFolderPath }); + } catch (err) { + next(err); + } + }); + + // Upload files endpoint + router.post('/upload', upload('/tmp/', false).array('files'), async (req, res, next) => { + try { + if (!req.files || req.files.length === 0) { + return res.status(400).json({ error: 'No files uploaded' }); + } + + const uploadPath = req.body.uploadPath || ''; + const targetDir = validatePath(uploadPath); + + // Ensure target directory exists + await ensureDirectoryExists(targetDir); + + const uploadedFiles = []; + for (const file of req.files) { + const targetPath = path.join(targetDir, file.originalname); + + // Use copyFile + unlink instead of rename for cross-device moves + try { + await fs.copyFile(file.path, targetPath); + await fs.unlink(file.path); + uploadedFiles.push(file.originalname); + } catch (err) { + // If copy fails, clean up the temp file + try { + await fs.unlink(file.path); + } catch { + // Ignore cleanup errors + } + throw err; + } + } + + res.json({ + message: `${uploadedFiles.length} file(s) uploaded successfully`, + files: uploadedFiles + }); + } catch (err) { + // Clean up any uploaded files if there's an error + if (req.files) { + for (const file of req.files) { + try { + await fs.unlink(file.path); + } catch { + // Ignore cleanup errors + } + } + } + next(err); + } + }); + + // Rename file/folder endpoint + router.post('/rename', async (req, res, next) => { + try { + const { path: requestedPath, oldName, newName } = req.body; + if (!oldName || !newName || !newName.trim()) { + return res.status(400).json({ error: 'Old name and new name are required' }); + } + + const parentPath = validatePath(requestedPath || ''); + const oldPath = path.join(parentPath, oldName); + const newPath = path.join(parentPath, newName.trim()); + + // Check if old path exists + try { + await fs.access(oldPath); + } catch { + return res.status(404).json({ error: 'File or folder not found' }); + } + + // Check if new path already exists + try { + await fs.access(newPath); + return res.status(409).json({ error: 'A file or folder with that name already exists' }); + } catch { + // New path doesn't exist, which is what we want + } + + await fs.rename(oldPath, newPath); + res.json({ message: 'Renamed successfully' }); + } catch (err) { + next(err); + } + }); + + // Delete file/folder endpoint + router.post('/delete', async (req, res, next) => { + try { + const { path: requestedPath, name, isDirectory } = req.body; + if (!name) { + return res.status(400).json({ error: 'Name is required' }); + } + + const parentPath = validatePath(requestedPath || ''); + const targetPath = path.join(parentPath, name); + + // Check if target exists + try { + await fs.access(targetPath); + } catch { + return res.status(404).json({ error: 'File or folder not found' }); + } + + if (isDirectory === true || isDirectory === 'true') { + await fs.rm(targetPath, { recursive: true, force: true }); + res.json({ message: 'Folder deleted successfully' }); + } else { + await fs.unlink(targetPath); + res.json({ message: 'File deleted successfully' }); + } + } catch (err) { + next(err); + } + }); + + return router; +} diff --git a/src/controllers/views.mjs b/src/controllers/views.mjs new file mode 100644 index 0000000..9cb7375 --- /dev/null +++ b/src/controllers/views.mjs @@ -0,0 +1,108 @@ +import { renderAdminPanel } from '../views/admin.mjs'; +import {getDirectoryContents, getZimFiles} from '../services/files.mjs'; +import { environment } from '../services/environment.mjs'; +import path from 'path'; +import fs from 'fs/promises'; +import {renderArchive} from '../views/archive.mjs'; + +export const viewController = { + adminPanel: async (req, res, next) => { + try { + const zimFiles = await getZimFiles(environment.paths.zim); + res.send(renderAdminPanel(zimFiles)); + } catch (err) { + next(err); + } + }, + + fileBrowser: async (req, res, next) => { + try { + const requestedPath = req.query.path || ''; + const currentPath = path.join(environment.paths.archive, requestedPath); + const normalizedArchiveRoot = path.resolve(environment.paths.archive); + const normalizedCurrentPath = path.resolve(currentPath); + + if(!normalizedCurrentPath.startsWith(normalizedArchiveRoot)) + return res.status(403).send('Forbidden'); + + const files = await getDirectoryContents(currentPath); + res.send(renderArchive(currentPath, files, environment.paths.archive)); + } catch (err) { + next(err); + } + }, + + fileViewer: async (req, res, next) => { + try { + const filePath = req.query.file; + if (!filePath) { + return res.status(400).send('File path required'); + } + + const archiveRoot = environment.paths.archive; + const normalizedArchiveRoot = path.resolve(archiveRoot); + const normalizedFilePath = path.resolve(filePath); + + // Security check + if (!normalizedFilePath.startsWith(normalizedArchiveRoot)) { + return res.status(403).send('Access denied: File outside archive directory'); + } + + // Check if file exists + try { + await fs.access(filePath); + } catch { + return res.status(404).send('File not found'); + } + + const stats = await fs.stat(filePath); + if (stats.isDirectory()) { + // Redirect to directory browser + const relativePath = path.relative(archiveRoot, filePath); + return res.redirect(`/files?path=${encodeURIComponent(relativePath)}`); + } + + // Determine how to serve the file based on its extension + const ext = path.extname(filePath).toLowerCase(); + + // For PDFs, images, and other web-viewable files, serve them directly + if (['.pdf', '.jpg', '.jpeg', '.png', '.gif', '.txt', '.html', '.css', '.js'].includes(ext)) { + res.sendFile(normalizedFilePath); + } else { + // For other files, trigger download + res.download(normalizedFilePath); + } + } catch (err) { + next(err); + } + }, + + fileDownload: async (req, res, next) => { + try { + const filePath = req.query.file; + if (!filePath) { + return res.status(400).send('File path required'); + } + + const archiveRoot = environment.paths.archive; + const normalizedArchiveRoot = path.resolve(archiveRoot); + const normalizedFilePath = path.resolve(filePath); + + // Security check + if (!normalizedFilePath.startsWith(normalizedArchiveRoot)) { + return res.status(403).send('Access denied: File outside archive directory'); + } + + // Check if file exists + try { + await fs.access(filePath); + } catch { + return res.status(404).send('File not found'); + } + + res.download(normalizedFilePath); + } catch (err) { + next(err); + } + } +}; diff --git a/src/main.mjs b/src/main.mjs new file mode 100644 index 0000000..3495767 --- /dev/null +++ b/src/main.mjs @@ -0,0 +1,65 @@ +import express from 'express'; +import adminController from './controllers/admin.mjs'; +import {environment} from './services/environment.mjs'; +import {ensureDirectoryExists} from './services/files.mjs'; +import path from 'path'; +import {startKiwix} from './services/kiwix.mjs'; +import http from 'http'; +import archiveController from './controllers/archive.mjs'; +import {viewController} from './controllers/views.mjs'; + +const app = express(); + +// Middleware +app.use(express.json()); +app.use(express.urlencoded({extended: true})); +app.use(express.static('public')); + +// Routes +app.get('/admin', viewController.adminPanel); +app.use('/api', adminController(environment)); +app.use('/api/files', archiveController()); +app.get('/archive', viewController.fileBrowser); +app.get('/archive/view', viewController.fileViewer); +app.get('/archive/download', viewController.fileDownload); +app.get('/favicon.:ext', (req, res) => res.sendFile(path.join(environment.root, "favicon.png"))); + +// Kiwix proxy +app.use((req, res) => { + const proxyReq = http.request({ + hostname: '127.0.0.1', + port: environment.kiwix.port, + path: req.url, + method: req.method, + headers: { + ...req.headers, + host: `127.0.0.1:${environment.kiwix.port}`, + }, + }, (proxyRes) => { + res.writeHead(proxyRes.statusCode, proxyRes.headers); + proxyRes.pipe(res, { end: true }); + }); + + proxyReq.on('error', (err) => { + console.error('Proxy error:', err); + if(!res.headersSent) res.status(502).send('Bad Gateway'); + }); + + req.pipe(proxyReq, { end: true }); +}); + +// Error handler +app.use((err, req, res, next) => { + console.error('Unhandled error:', err); + res.status(500).json({ error: err.message || 'Internal server error' }); +}); + +// Start server +app.listen(environment.port, async () => { + console.log(`\n🥝 Kiwix Manager started on http://localhost:${environment.port}\n`); + await ensureDirectoryExists(environment.paths.archive); + await ensureDirectoryExists(environment.paths.zim); + await startKiwix(); +}); + +export default app; diff --git a/src/services/environment.mjs b/src/services/environment.mjs new file mode 100644 index 0000000..ad24051 --- /dev/null +++ b/src/services/environment.mjs @@ -0,0 +1,19 @@ +import { dirname } from 'path'; +import { fileURLToPath } from 'url'; +import dotenv from 'dotenv'; + +dotenv.config(); + +export const environment = { + paths: { + archive: process.env.ARCHIVE_DIR || '/data/archive', + zim: process.env.ZIM_DIR || '/data/zim', + }, + port: process.env.PORT || 3001, + root: dirname(dirname(dirname(fileURLToPath(import.meta.url)))), + kiwix: { + manage: process.env.KIWIX_MANAGE || '/usr/bin/kiwix-manage', + serve: process.env.KIWIX_SERVE || '/usr/bin/kiwix-serve', + port: process.env.KIWIX_PORT || 8000, + }, +} diff --git a/src/services/files.mjs b/src/services/files.mjs new file mode 100644 index 0000000..acc1960 --- /dev/null +++ b/src/services/files.mjs @@ -0,0 +1,133 @@ +import path from 'path'; +import { createWriteStream } from 'fs'; +import { pipeline } from 'stream'; +import { promisify } from 'util'; +import multer from 'multer'; +import { promises as fs } from 'fs'; + +export async function getDirectoryContents(dirPath) { + try { + await fs.access(dirPath); + } catch { + // Create directory if it doesn't exist + await fs.mkdir(dirPath, { recursive: true }); + } + + const items = await fs.readdir(dirPath, { withFileTypes: true }); + const files = []; + + console.log(dirPath, items); + + for (const item of items) { + const itemPath = path.join(dirPath, item.name); + const stats = await fs.stat(itemPath); + + files.push({ + name: item.name, + path: itemPath, + isDirectory: item.isDirectory(), + size: item.isDirectory() ? 0 : stats.size, + modified: stats.mtime.toISOString() + }); + } + + // Sort: directories first, then files, both alphabetically + return files.sort((a, b) => { + if (a.isDirectory && !b.isDirectory) return -1; + if (!a.isDirectory && b.isDirectory) return 1; + return a.name.localeCompare(b.name); + }); +} + +export function upload(path, zimOnly = false) { + const storage = multer.diskStorage({ + destination: async (req, file, cb) => { + const uploadPath = req.body.uploadPath || path; + try { + await fs.mkdir(uploadPath, { recursive: true }); + cb(null, uploadPath); + } catch (err) { + cb(err); + } + }, + filename: (req, file, cb) => { + let finalFilename = file.originalname; + if(req.body.filename && req.body.filename.trim() !== '') { + finalFilename = req.body.filename.trim(); + if(!finalFilename.toLowerCase().endsWith('.zim')) { + finalFilename += '.zim'; + } + } + cb(null, finalFilename); + } + }); + + return multer({ + storage, + fileFilter: (req, file, cb) => { + if(!zimOnly) cb(null, true); + else if(/\.zim$/i.test(file.originalname)) cb(null, true); + else cb(new Error('Only ZIM files allowed')); + } + }); +} +const pipelineAsync = promisify(pipeline); + +export async function downloadFile(url, destination) { + if(!url.endsWith('.zim')) throw new Error('Only ZIM files allowed'); + if(!destination.endsWith('.zim')) destination += '.zim'; + const response = await fetch(url); + if (!response.ok) throw new Error(`HTTP error! ${response.status}`); + const writer = createWriteStream(destination); + await pipelineAsync(response.body, writer); + return destination; +} + +export function parseSize(sizeStr) { + if (sizeStr.toLowerCase() === 'infinity') return Infinity; + const units = { B: 1, KB: 1024, MB: 1024 ** 2, GB: 1024 ** 3 }; + const match = sizeStr.match(/^(\d+)(B|KB|MB|GB)$/i); + if (!match) return 100 * 1024 * 1024; // default 100MB + return parseInt(match[1]) * units[match[2].toUpperCase()]; +} + +export async function ensureDirectoryExists(dirPath) { + try { + await fs.mkdir(dirPath, { recursive: true }); + } catch (err) { + if (err.code !== 'EEXIST') throw err; + } +} + +export function formatFileSize(bytes) { + if (bytes === 0) return '0 Bytes'; + const k = 1024; + const sizes = ['Bytes', 'KB', 'MB', 'GB']; + const i = Math.floor(Math.log(bytes) / Math.log(k)); + return parseFloat((bytes / Math.pow(k, i)).toFixed(2)) + ' ' + sizes[i]; +} + +export async function getZimFiles(archiveDir) { + await ensureDirectoryExists(archiveDir); + const items = await fs.readdir(archiveDir, { withFileTypes: true }); + + const zimFiles = []; + for (const item of items) { + if (item.isFile() && path.extname(item.name).toLowerCase() === '.zim') { + const fullPath = path.join(archiveDir, item.name); + const stats = await fs.stat(fullPath); + zimFiles.push({ + name: item.name, + path: fullPath, + size: stats.size, + modified: stats.mtime.toISOString() + }); + } + } + + return zimFiles.sort((a, b) => new Date(b.modified) - new Date(a.modified)); +} + +export async function deleteFile(filePath) { + await fs.unlink(filePath); +} diff --git a/src/services/kiwix.mjs b/src/services/kiwix.mjs new file mode 100644 index 0000000..7a109ca --- /dev/null +++ b/src/services/kiwix.mjs @@ -0,0 +1,96 @@ +import {exec, spawn} from 'child_process'; +import { promisify } from 'util'; +import { environment } from './environment.mjs'; +import { getZimFiles } from './files.mjs'; +import { resolve } from 'path'; +import fs from 'fs'; + +const execAsync = promisify(exec); +let kiwixProcess = null; + +export async function startKiwix() { + if(kiwixProcess) return; + await synchronizeLibrary(); + const libraryPath = resolve(environment.paths.zim, 'library.xml'); + return new Promise((resolvePromise, rejectPromise) => { + kiwixProcess = spawn(environment.kiwix.serve, [`--port=${environment.kiwix.port}`, `--library`, libraryPath]); + kiwixProcess.stdout.on('data', (data) => console.log(data.toString().trim())); + kiwixProcess.stderr.on('data', (data) => console.error(`Kiwix Error: ${data.toString().trim()}`)); + kiwixProcess.on('error', (err) => { + console.error('Failed to start Kiwix process.', err); + kiwixProcess = null; + rejectPromise(err); + }); + kiwixProcess.on('close', (code) => { + console.log(`Kiwix Serve exited with code ${code}`); + kiwixProcess = null; + }); + setTimeout(() => resolvePromise(), 2000); + }); +} + +export function stopKiwix() { + if (kiwixProcess) { + kiwixProcess.kill('SIGTERM'); + console.log('Kiwix server stopped.'); + kiwixProcess = null; + return true; + } + console.log('Kiwix server is not running.'); + return false; +} + +export async function restartKiwix() { + try { + if(kiwixProcess) { + stopKiwix(); + await new Promise(resolve => setTimeout(resolve, 1000)); + } + await startKiwix(); + console.log('Kiwix service restarted'); + return true; + } catch (err) { + console.error('Error restarting Kiwix:', err.message); + return false; + } +} + +export async function synchronizeLibrary() { + const libraryPath = resolve(environment.paths.zim, 'library.xml'); + try { + const { stdout: libraryOutput } = await execAsync(`${environment.kiwix.manage} ${libraryPath} show || echo`); + const libraryBooks = new Set((libraryOutput.match(/ID: \w+, Path: (.*?),/g) || []).map(line => line.substring(line.indexOf('Path: ') + 6, line.length - 1))); + + const zimFiles = await getZimFiles(environment.paths.zim); + const directoryFiles = new Set(zimFiles.map(file => file.path)); + + const filesToAdd = [...directoryFiles].filter(file => file.endsWith('.zim') && !libraryBooks.has(file)); + const filesToRemove = [...libraryBooks].filter(file => !directoryFiles.has(file)); + + for (const file of filesToAdd) { + console.log(`Adding file to library: ${file}`); + await execAsync(`${environment.kiwix.manage} ${libraryPath} add ${file}`); + } + + for (const file of filesToRemove) { + console.log(`Removing file from library: ${file}`); + const { stdout } = await execAsync(`${environment.kiwix.manage} ${libraryPath} show`); + const match = stdout.match(new RegExp(`ID: (\\w+), Path: ${file.replace(/[.*+?^${}()|[\]\\]/g, '\\$&')},`)); + if (match && match[1]) { + const bookId = match[1]; + await execAsync(`${environment.kiwix.manage} ${libraryPath} remove ${bookId}`); + } + } + } catch (error) { + if (error.stderr && (error.stderr.includes('No such file or directory') || error.stderr.includes('cannot open library'))) { + console.log('Library file not found or is invalid, creating a new one.'); + const zimFiles = await getZimFiles(environment.paths.zim); + for (const file of zimFiles.map(f => f.path)) { + await execAsync(`${environment.kiwix.manage} ${libraryPath} add ${file}`); + } + } else { + console.error('Error synchronizing library:', error); + throw error; + } + } +} diff --git a/src/services/zim.mjs b/src/services/zim.mjs new file mode 100644 index 0000000..a45ac9e --- /dev/null +++ b/src/services/zim.mjs @@ -0,0 +1,81 @@ +import { exec } from 'child_process'; +import { promisify } from 'util'; +import { environment } from './environment.mjs'; +import path from 'path'; +import { promises as fs } from 'fs'; + +const execAsync = promisify(exec, { timeout: 3600000 }); + +function escapeDockerArg(arg) { + if (arg === null || arg === undefined || arg === '') return '""'; + return `"${String(arg).replace(/\\/g, '\\\\').replace(/"/g, '\\"')}"`; +} + +function escapeShellArg(arg) { + if (arg === null || arg === undefined || arg === '') return "''"; + return `'${String(arg).replace(/'/g, "'\\''")}'`; +} + +export async function createZimFromDirectory(options, contentPath) { + let { name, welcome, language, description, tags, favicon, creator } = options; + const zimFile = name.endsWith('.zim') ? name : `${name}.zim`; + const zimFilePath = path.join(environment.paths.zim, zimFile); + if(!favicon) { + fs.copyFile('/app/favicon.png', path.join(contentPath, 'favicon.png')); + favicon = 'favicon.png'; + } + + let command = `zimwriterfs -w ${escapeShellArg(welcome || 'index.html')} -I ${escapeShellArg(favicon)} -l ${escapeShellArg(language || 'eng')} -t ${escapeShellArg(name)} -n ${escapeShellArg(name)} -d ${escapeShellArg(description || 'ZIM Archive')} -c ${escapeShellArg(creator || 'Unknown')} -p KiwixM`; + if(tags) command += ` --tags=${escapeShellArg(tags)}`; + command += ` "${contentPath}" "${zimFilePath}"`; + + console.log('Executing:', command); + const { stdout, stderr } = await execAsync(command); + if(stderr) console.error(stderr); + return zimFile; +} + +export async function createZimFromUrl(options) { + const { url, name, language, description, tags, regex, maxHops, waitEvent } = options; + const sanitizedName = name.replace(/[^a-zA-Z0-9\-_]/g, '_'); + const zimFile = `${sanitizedName}.zim`; + const zimFilePath = path.join(environment.paths.zim, zimFile); + const tempDir = `/tmp/zimit-${Date.now()}-${Math.random().toString(36).substr(2, 9)}`; + + let zimitArgs = []; + zimitArgs.push(`--seeds ${escapeDockerArg(url)}`); + zimitArgs.push(`--name ${escapeDockerArg(name)}`); + zimitArgs.push(`--headless`); + if(description) zimitArgs.push(`--description ${escapeDockerArg(description)}`); + if(language) zimitArgs.push(`--lang ${escapeDockerArg(language || 'en-US')}`); + if(tags) zimitArgs.push(`--tags ${escapeDockerArg(tags)}`); + if(regex) zimitArgs.push(`--include ${escapeDockerArg(regex)}`); + if(maxHops && maxHops !== '') zimitArgs.push(`--max-hops ${parseInt(maxHops)}`); + if(waitEvent && waitEvent !== '') zimitArgs.push(`--wait-until ${escapeDockerArg(waitEvent)}`); + zimitArgs.push(`--output ${tempDir}`); + + const containerName = `zimit-${Date.now()}-${Math.random().toString(36).substr(2, 9)}`; + + try { + await execAsync(`mkdir -p ${tempDir}`); + const dockerCommand = `docker run --name ${containerName} --rm -v ${tempDir}:${tempDir}:rw --shm-size=2g --cap-add=SYS_ADMIN --security-opt seccomp=unconfined ghcr.io/openzim/zimit zimit ${zimitArgs.join(' ')}`; + + console.log('Executing:', dockerCommand); + let { stdout, stderr } = await execAsync(dockerCommand); + if(stderr) console.log(stderr); + + try { + const resp = await execAsync(`find ${tempDir} -name "*.zim" -type f`); + const tempZimPath = resp.stdout?.split('\n')?.[0]?.trim(); + await fs.copyFile(tempZimPath, zimFilePath); + return zimFile; + } catch (fileError) { + throw new Error(fileError.message); + } + } catch (err) { + await execAsync(`docker rm -f ${containerName}`).catch(() => {}); + throw err; + } finally { + execAsync(`rm -rf ${tempDir}`).catch(() => {}); + } +} diff --git a/src/views/admin.mjs b/src/views/admin.mjs new file mode 100644 index 0000000..f5ac0d8 --- /dev/null +++ b/src/views/admin.mjs @@ -0,0 +1,452 @@ +import {formatFileSize} from '../services/files.mjs'; + +export function renderAdminPanel(zimFiles) { + const totalSize = zimFiles.reduce((sum, file) => sum + file.size, 0); + + return ` + + + + Kiwix Manager + + + + + +

Kiwix Manager

Manage your offline content library

+ Browse + +

+ +

+ ${zimFiles.length} ZIM files + • + ${formatFileSize(totalSize)} total +

+ 📚 Kiwix Library +

+ + + + +

+ 📤 Upload ZIM▼ +

+ + +

+ +

+ + +

+ +

+ 🌐 URL to ZIM▼ +

+ URL + +

+ ZIM Name + +

+ Language + +

+ Description + +

+ Tags (semi-colon separator) + +

+ Include Regex + +

+ Max Hop Count + +

Maximum number of link hops to follow (0-10)

+ Wait Event + +

When to consider page loading complete

+ +

+ 📦 ZIP to ZIM▼ +

+ Select ZIP File + +

+ ZIM Name (required) + +

+ Welcome Page (path inside zip) + +

+ Language + +

+ Description + +

+ Tags (semicolon-separated) + +

+ Favicon (path inside zip) + +

+ Creator + +

+ +

ZIM Library

+ +

+ ${zimFiles.length === 0 + ? `

📚

No ZIM files found

Upload or download to get started

` + : zimFiles.map(file => ` +

📚

${file.name}

+ ${formatFileSize(file.size)} + Modified: ${new Date(file.modified).toLocaleDateString()} +

+ ${!file.path.endsWith('archive.zim') ? ` +

+ Download + +

+ ` : ''} +

`).join('')} +

+ + + + +`; +} diff --git a/src/views/archive.mjs b/src/views/archive.mjs new file mode 100644 index 0000000..b21d1c6 --- /dev/null +++ b/src/views/archive.mjs @@ -0,0 +1,330 @@ + +import { formatFileSize } from '../services/files.mjs'; +import path from 'path'; + +// Move getFileIcon function to server-side +function getFileIcon(fileName) { + const ext = fileName.toLowerCase().split('.').pop(); + const icons = { + pdf: '📄', txt: '📝', doc: '📄', jpg: '🖼️', png: '🖼️', gif: '🖼️', + mp4: '🎥', mp3: '🎵', zip: '📦', html: '🌐', css: '🎨', js: '⚡' + }; + return icons[ext] || '📄'; +} + +export function renderArchive(currentPath, files, archiveRoot) { + const relativePath = path.relative(archiveRoot, currentPath); + const pathSegments = relativePath === '' ? [] : relativePath.split(path.sep); + + // Updated breadcrumbs: 🏠 / Archive / + const breadcrumbs = [ + { name: '🏠', path: '/', isHome: true }, + { name: 'Archive', path: '', isArchiveRoot: true } + ]; + + let accumulatedPath = ''; + pathSegments.forEach(segment => { + accumulatedPath = accumulatedPath ? path.join(accumulatedPath, segment) : segment; + breadcrumbs.push({ name: segment, path: accumulatedPath }); + }); + + return ` + + + Kiwix Archive + + + + + +

+ +