This commit is contained in:
2023-08-14 14:36:45 -04:00
commit b5966f98b2
94 changed files with 21124 additions and 0 deletions

View File

@ -0,0 +1,20 @@
import {KeyVal} from '@transmute/common';
import {app, config} from '../main';
import {ErrorHandler} from '../middleware/error-handler.middleware';
import {CrudApiController} from '../modules/crud-api/controller';
export class ConfigController extends CrudApiController<string, KeyVal>{
protected readonly baseUrl = '/api/config';
constructor() {
super(config);
super.setup({
update: true
});
// Fetch entire config
app.get('/api/config', ErrorHandler(async (req, res) => {
res.json(await config.formatted());
}));
}
}

View File

@ -0,0 +1,38 @@
import {Library} from '@transmute/common';
import {app, libraries} from '../main';
import {ErrorHandler} from '../middleware/error-handler.middleware';
import {CrudApiController} from '../modules/crud-api/controller';
export class LibraryController extends CrudApiController<number, Library>{
protected readonly baseUrl = '/api/library';
constructor() {
super(libraries);
// List all videos (must come before super)
app.get(`${this.baseUrl}/videos`, ErrorHandler(async (req, res) =>
res.json(await libraries.videos())));
app.get(`${this.baseUrl}/scan`, ErrorHandler(async (req, res) =>
res.json({length: await libraries.scanAll(true)})));
// All stats (must come before super)
app.get(`${this.baseUrl}/metrics`, ErrorHandler(async (req, res) =>
res.json(await libraries.metrics())));
// Library CRUD operations
super.setup();
// Scan library for videos
app.get(`${this.baseUrl}/:id/scan`, ErrorHandler(async (req, res) => {
res.json({length: await libraries.scan(Number(req.params['id']), true)})}));
// Library stats
app.get(`${this.baseUrl}/:id/metrics`, ErrorHandler(async (req, res) =>
res.json(await libraries.metrics(Number(req.params['id'])))));
// Library videos
app.get(`${this.baseUrl}/:library/videos`, ErrorHandler(async (req, res) =>
res.json(await libraries.videos(Number(req.params['library'])))));
}
}

View File

@ -0,0 +1,7 @@
import {join} from 'path';
import * as process from 'process';
export const environment = {
clientPath: process.env['CLIENT_PATH'] ?? join(process.cwd(), '../client/dist'),
port: process.env['PORT'] ?? 5000
}

52
server/src/main.ts Normal file
View File

@ -0,0 +1,52 @@
import express from 'express';
import http from 'http';
import {ConfigController} from './controllers/config.controller';
import {LibraryController} from './controllers/library.controller';
import {environment} from './environments/environment';
import {ErrorMiddleware} from './middleware/error-handler.middleware';
import {LoggerMiddleware} from './middleware/logger.middleware';
import {ConfigService} from './services/config.service';
import {LibraryService} from './services/library.service';
import {QueueService} from './services/queue.service';
import {SocketService} from './services/socket.service';
import * as SourceMap from 'source-map-support';
import {VideoService} from './services/video.service';
import * as cors from 'cors';
SourceMap.install();
// Globals
export const app = express();
export const server = http.createServer(app);
// Singleton services
// await connectAndMigrate();
export const config = new ConfigService();
export const socket = new SocketService(server);
export const queue = new QueueService();
export const videos = new VideoService();
export const libraries = new LibraryService(videos);
// Express setup
(async () => {
// Middleware
app.use(cors.default());
app.use(express.json());
app.use(LoggerMiddleware);
// API
new ConfigController();
new LibraryController();
// Client/WebUI
console.log(environment.clientPath);
app.get('*', express.static(environment.clientPath));
// Error handling
app.use(ErrorMiddleware);
process.on('unhandledRejection', ErrorMiddleware);
// Start
server.listen(environment.port, () => console.log(`Listening on http://localhost:${environment.port}`));
})();

View File

@ -0,0 +1,29 @@
import {Logger} from '@transmute/common';
import {Request, Response} from 'express';
import {BadRequestError, CustomError, InternalServerError} from '../utils/errors';
const logger = new Logger('ErrorMiddleware');
export const ErrorHandler = fn => (req, res, next) => {
return Promise
.resolve(fn(req, res, next))
.catch(next);
};
export const ErrorMiddleware = (err: Error | CustomError, req: Request, res: Response, next) => {
const e: CustomError = CustomError.instanceof(err) ? err :
err.stack.includes('SqliteError') ? BadRequestError.from(err) : InternalServerError.from(err);
const code = (<any>e).constructor?.code;
const userError = code >= 400 && code < 500;
logger[userError ? 'verbose' : 'error'](userError ? `${code} ${err.message}` : err.stack);
if(res) {
res.status(code).json({
code,
message: err.message,
stack: userError ? undefined : err.stack,
timestamp: (new Date()).toISOString()
})
}
}

View File

@ -0,0 +1,8 @@
import {Logger} from '@transmute/common';
const logger = new Logger('LoggerMiddleware');
export function LoggerMiddleware(req, res, next) {
logger.verbose(`${req.method} ${decodeURI(req.url)}`);
next();
}

View File

@ -0,0 +1,37 @@
import {app} from '../../main';
import {ErrorHandler} from '../../middleware/error-handler.middleware';
import {CrudApiService} from './service';
export abstract class CrudApiController<K, T> {
protected readonly baseUrl: string;
protected constructor(private service: CrudApiService<K, T>) { }
setup(methods?: {list?: boolean, read?: boolean, create?: boolean, update?: boolean, delete?: boolean}): void {
if(!methods || methods?.create)
app.post(this.baseUrl, ErrorHandler(async (req, res) => {
res.json(await this.service.create(req.body));
}));
if(!methods || methods?.list)
app.get(this.baseUrl, ErrorHandler(async (req, res) => {
const pagination = {offset: Number(req.query['offset']), limit: Number(req.query['limit'])}
res.json(await this.service.list(Object.keys(req.params).length > 0 ? <Partial<T>>req.params : undefined, pagination));
}));
if(!methods || methods?.read)
app.get(`${this.baseUrl}/:${this.service.pk.toString()}`, ErrorHandler(async (req, res) => {
res.json(await this.service.read(<K>req.params[this.service.pk]));
}));
if(!methods || methods?.update)
app.patch(`${this.baseUrl}/:id?`, ErrorHandler(async (req, res) => {
res.json(await this.service.update(req.body));
}));
if(!methods || methods?.delete)
app.delete(`${this.baseUrl}/:${this.service.pk.toString()}`, ErrorHandler(async (req, res) => {
res.json(await this.service.delete(<K>req.params[this.service.pk]));
}));
}
}

View File

@ -0,0 +1,63 @@
import {db} from '../../services/sqlite.service';
import {NotFoundError} from '../../utils/errors';
import {whereBuilder} from '../../utils/sql.utils';
export abstract class CrudApiService<K, T> {
protected constructor(protected readonly table: string, public readonly pk: keyof T, private autoCreate: boolean = false) {}
abstract afterRead(value: T, list: boolean): Promise<T>;
abstract beforeWrite(value: Partial<T>, original: T | null): Promise<T>;
abstract afterWrite(value: T, original: T | null): void | Promise<void>;
async list(filter?: Partial<T>, paginate?: {offset?: number, limit?: number}): Promise<T[]> {
let qb = db.selectFrom(<any>this.table).selectAll();
if(filter != null) qb = whereBuilder(qb, filter);
return Promise.all((await qb.execute())
// .filter((el, i) => !paginate || ((paginate?.offset == null || i >= paginate.offset) && (paginate?.limit == null || i < (paginate?.offset || 0) + paginate.limit)))
.map((f: T) => this.afterRead(f, true)));
}
async create(value: Partial<T>): Promise<T> {
const row = await db.insertInto(<any>this.table)
.values(await this.beforeWrite(value, null))
.returning(this.pk.toString())
.executeTakeFirst();
const newVal = await this.read((<any>row)[this.pk]);
await this.afterWrite(newVal, null);
return newVal;
}
async read(filter: K | Partial<T>): Promise<T> {
const found = await whereBuilder(
db.selectFrom(<any>this.table).selectAll(),
typeof filter == 'object' ? filter : {[this.pk]: filter}
).executeTakeFirst();
if(!found) throw new NotFoundError();
return this.afterRead(<T>found, false);
}
async update(value: Partial<T>): Promise<T> {
const original = await this.read(<any>value[this.pk]);
if(!original) {
if(!this.autoCreate) throw new NotFoundError();
return this.create(value);
}
return whereBuilder(
db.updateTable(<any>this.table).set({...(await this.beforeWrite(value, original)), [this.pk]: undefined}),
{[this.pk]: value[this.pk]}
).execute().then(async () => {
const newVal = await this.read(<any>value[this.pk]);
await this.afterWrite(newVal, original);
return newVal;
});
}
async delete(filter?: K | Partial<T>): Promise<void> {
return whereBuilder(
db.deleteFrom(<any>this.table),
typeof filter == 'object' ? filter : {[this.pk]: filter}
).execute().then(() => {});
}
}

View File

@ -0,0 +1,36 @@
import {Config, ConfigDefaults, KeyVal} from '@transmute/common';
import {CrudApiService} from '../modules/crud-api/service';
import {BadRequestError} from '../utils/errors';
export class ConfigService extends CrudApiService<string, KeyVal> {
readonly options: Partial<Config> = ConfigDefaults;
constructor() {
super('config', 'key', true);
// Load config values
(async () => {
(await this.list()).forEach(({key, value}) => this.options[key] = value);
})();
}
async afterRead(value: KeyVal, list: boolean): Promise<KeyVal> {
value.value = JSON.parse(value.value);
return value;
}
async beforeWrite(value: KeyVal, original: KeyVal): Promise<KeyVal> {
if(value.key && this.options[value.key] === undefined)
throw new BadRequestError(`Invalid config key: ${value.key}`);
this.options[value.key] = value.value;
value.value = JSON.stringify(value.value);
return value;
}
afterWrite(value, original: | null): void { }
async formatted(): Promise<Config> {
return <Config>(await super.list())
.reduce((acc, {key, value}) => ({...acc, [key]: value}), {});
}
}

View File

@ -0,0 +1,153 @@
import {Library, Metrics} from '@transmute/common';
import {CrudApiService} from '../modules/crud-api/service';
import {existsSync, statSync} from 'fs';
import {BadRequestError} from '../utils/errors';
import {createChecksum, deepSearch, parseFilePath} from '../utils/file.utils';
import {VideoService} from './video.service';
export class LibraryService extends CrudApiService<number, Library> {
constructor(private videoService: VideoService) {
super('library', 'id');
}
async afterRead(value: Library, list: boolean): Promise<Library> {
value.watch = (<any>value).watch == 1;
return value;
}
async beforeWrite(value: Library, original: Library | null): Promise<Library> {
if(!existsSync(value.path))
throw new BadRequestError(`Library path is invalid: ${value.path}`);
value.watch = <any>(value.watch ? 1 : 0);
return value;
}
afterWrite(value: Library, original: Library): Promise<void> {
return this.scan(value.id).then(() => {});
}
async delete(filter: Partial<Library> | number): Promise<void> {
const id = typeof filter == 'object' ? filter.id : filter;
await this.videoService.delete({library: id});
return super.delete(filter);
}
async scan(library: number, force: boolean = false): Promise<number> {
// Check if library exists
const lib = await this.read(library);
if(!existsSync(lib.path)) throw new BadRequestError(`Library path is invalid: ${lib.path}`)
// Fetch list of previously scanned videos & current videos on disk
const files = deepSearch(lib.path).filter(file => /\.(avi|mp4|mkv)$/gmi.test(file));
const videos = await this.videoService.list({library});
// Initial save to DB
const saved = await Promise.all(files.map(async file => {
const exists = videos.findIndex(v => v.path == file);
if(exists != -1) return videos.splice(exists, 1)[0];
const fileInfo = parseFilePath(file);
return await this.videoService.create({
name: fileInfo.fileName,
path: file,
library,
container: fileInfo.extension,
size: statSync(file).size
});
}));
// Scan each file asynchronously
Promise.all(saved.map(async video => {
const checksum = await createChecksum(video.path);
if(!force && video.checksum == checksum && !Object.values(video).includes(null)) return video;
return this.videoService.update({
...video,
checksum,
...(await this.videoService.scan(video.path)),
});
})).then(resp => {
// Delete scans for files that no longer exist on disk
videos.forEach(v => this.videoService.delete(v.id));
return resp;
});
// Return number of discovered files
return saved.length;
}
async scanAll(force: boolean = false): Promise<number> {
const libraries = await this.list();
return (await Promise.all(libraries.map(l => this.scan(l.id, force))))
.reduce((acc, n) => acc + n, 0);
}
async metrics(library?: number): Promise<Metrics> {
// Check if library exists
if(library) await this.read(library);
// Iterate over all video files & add up stats
const stats: Metrics = {
resolution: {},
container: {},
videoCodec: {},
audioCodec: {},
health: {healthy: 0, unhealthy: 0, unknown: 0},
audioLang: {},
subLang: {},
size: 0,
videos: 0
};
(await this.videoService.list(library != null ? {library} : undefined)).forEach(f => {
// Resolution
if(f.resolution) {
if(!stats.resolution[f.resolution]) stats.resolution[f.resolution] = 0;
stats.resolution[f.resolution]++;
}
// Container
if(f.container) {
if(!stats.container[f.container]) stats.container[f.container] = 0;
stats.container[f.container]++;
}
// Video codec
if(f.videoCodec) {
if(!stats.videoCodec[f.videoCodec]) stats.videoCodec[f.videoCodec] = 0;
stats.videoCodec[f.videoCodec]++;
}
// Audio codec
if(f.audioCodec) {
if(!stats.audioCodec[f.audioCodec]) stats.audioCodec[f.audioCodec] = 0;
stats.audioCodec[f.audioCodec]++;
}
// Audio tracks
f.audioTracks?.forEach(at => {
if(!stats.audioLang[at]) stats.audioLang[at] = 0;
stats.audioLang[at]++;
});
// Subtitles
f.subtitleTracks?.forEach(st => {
if(!stats.subLang[st]) stats.subLang[st] = 0;
stats.subLang[st]++;
});
// Health
stats.health[f.healthy == null ? 'unknown' : f.healthy ? 'healthy' : 'unhealthy']++;
// Filesize
stats.size += f.size;
// Length
stats.videos++;
});
return stats;
}
async videos(library?: number) {
// Check if library exists
if(library != null) await this.read(library);
return this.videoService.list(library != null ? {library} : undefined);
}
}

View File

@ -0,0 +1,27 @@
import {videos} from '../main';
export class QueueService {
private healthcheckJobs: number[] = [];
private transcodeJobs: number[] = [];
mode: 'auto' | 'healthcheck' | 'transcode' = 'auto';
async getJob(type?: 'healthcheck' | 'transcode'): Promise<['healthcheck' | 'transcode', number]> {
if((type || this.mode) == 'healthcheck')
return ['healthcheck', this.healthcheckJobs.pop()];
if((type || this.mode) == 'transcode')
return ['transcode', this.transcodeJobs.pop()];
// Auto - Get next transcode job or it's healthcheck if it's needed still required
const id = this.transcodeJobs.pop();
const video = await videos.read(id);
if(video.healthy != null)
return [video.healthy ? 'transcode' : 'healthcheck', id]
}
createJob(id: number, type: 'healthcheck' | 'transcode') {
(type == 'healthcheck' ?
this.healthcheckJobs :
this.transcodeJobs).push(id);
}
}

View File

@ -0,0 +1,25 @@
import {Server as HTTP} from 'http';
import {Server} from 'socket.io';
export class SocketService {
private socket !: any;
constructor(server: HTTP) {
this.socket = new Server(server);
this.socket.on('connection', (socket) => {
console.log('a user connected');
});
this.socket.on('disconnect', () => {
console.log('user disconnected');
});
}
scanStatus(library: string, complete: number, pending: number) {
this.socket.emit('scan', {
library,
complete,
pending,
});
}
}

View File

@ -0,0 +1,41 @@
import {Library, Video} from '@transmute/common';
import {Kysely, SqliteDialect} from 'kysely';
import Database from 'better-sqlite3'
interface Schema {
library: Library,
video: Video
}
export const db = new Kysely<Schema>({
dialect: new SqliteDialect({
database: new Database('db.sqlite3')
})
});
// export async function connectAndMigrate() {
// const migrator = new Migrator({
// db,
// provider: new FileMigrationProvider({
// fs,
// path,
// migrationFolder: 'dist/migrations',
// })
// });
//
// const { error, results } = await migrator.migrateToLatest()
//
// results?.forEach((it) => {
// if (it.status === 'Success') {
// console.log(`migration "${it.migrationName}" was executed successfully`);
// } else if (it.status === 'Error') {
// console.error(`failed to execute migration "${it.migrationName}"`);
// }
// });
//
// if (error) {
// console.error('failed to migrate');
// console.error(error);
// process.exit(1);
// }
// }

View File

@ -0,0 +1,35 @@
import {Video} from '@transmute/common';
import fs from 'fs';
import {CrudApiService} from '../modules/crud-api/service';
import {getCodec, getResolution, getStreams} from '../utils/ffmpeg.utils';
export class VideoService extends CrudApiService<number, Video> {
constructor() {
super('video', 'id');
}
async afterRead(value: Video, list: boolean): Promise<Video> {
value.audioTracks = JSON.parse(<any>value.audioTracks);
value.subtitleTracks = JSON.parse(<any>value.subtitleTracks);
return value;
}
async beforeWrite(value: Video, original: Video | null): Promise<Video> {
value.audioTracks = <any>JSON.stringify(value.audioTracks);
value.subtitleTracks = <any>JSON.stringify(value.subtitleTracks);
return value;
}
async afterWrite(value: Video, original: Video | null): Promise<void> { }
async scan(file: string): Promise<{resolution: string, videoCodec: string, audioCodec: string, audioTracks: string[], subtitleTracks: string[]}> {
if(!fs.existsSync(file)) throw new Error(`File could not be found: ${file}`);
return {
resolution: getResolution(file),
videoCodec: getCodec(file, 'video', 0),
audioCodec: getCodec(file, 'audio', 0),
audioTracks: getStreams(file, 'audio'),
subtitleTracks: getStreams(file, 'subtitle')
}
}
}

View File

@ -0,0 +1,3 @@
export class WorkerService {
workers = [];
}

View File

@ -0,0 +1,77 @@
export class CustomError extends Error {
static code = 500;
constructor(message?: string) {
super(message);
}
static from(err: Error) {
const newErr = new this(err.message);
return Object.assign(newErr, err);
}
static instanceof(err: Error) {
return (<any>err).constructor.code != undefined;
}
}
export class BadRequestError extends CustomError {
static code = 400;
constructor(message: string = 'Bad Request') {
super(message);
}
static instanceof(err: Error) {
return (<any>err).constructor.code == this.code;
}
}
export class UnauthorizedError extends CustomError {
static code = 401;
constructor(message: string = 'Unauthorized') {
super(message);
}
static instanceof(err: Error) {
return (<any>err).constructor.code == this.code;
}
}
export class ForbiddenError extends CustomError {
static code = 403;
constructor(message: string = 'Forbidden') {
super(message);
}
static instanceof(err: Error) {
return (<any>err).constructor.code == this.code;
}
}
export class NotFoundError extends CustomError {
static code = 404;
constructor(message: string = 'Not Found') {
super(message);
}
static instanceof(err: Error) {
return (<any>err).constructor.code == this.code;
}
}
export class InternalServerError extends CustomError {
static code = 500;
constructor(message: string = 'Internal Server Error') {
super(message);
}
static instanceof(err: Error) {
return (<any>err).constructor.code == this.code;
}
}

View File

@ -0,0 +1,67 @@
import {Resolution} from '@transmute/common';
import {execSync} from 'child_process';
/**
* Use FFProbe to look up the encoding of a specific stream track
*
* @example
* ```ts
* console.log(getEncoding('./sample.mp4', 'video'));
* // Output: 'h264'
*
* console.log(getEncoding('./sample.mp4', 'audio', 1));
* // Output: 'ACC'
* ```
*
* @param {string} file Absolute or relative path to video file
* @param {"audio" | "subtitle" | "video"} stream Type of stream to inspect
* @param {number} track Index of stream (if multiple are available; for example, bilingual audio tracks)
* @returns {string} The encoding algorithm used
*/
export function getCodec(file: string, stream: 'audio' | 'subtitle' | 'video', track: number = 0): string {
return execSync(
`ffprobe -v error -select_streams "${stream[0]}:${track}" -show_entries stream=codec_name -of default=noprint_wrappers=1:nokey=1 "${file}"`,
{encoding: 'utf-8', shell: 'cmd'}).trim();
}
/**
* Fetch resolution of video. Can either provide the real resolution or the closest standard resolution.
*
* @param {string} file Absolute or relative path to file
* @param {boolean} real Return the real resolution or the closest standard
* @returns {string} If real enabled, will return XY (1920x1020), if disabled, will return closes resolution tag (1080p)
*/
export function getResolution(file: string, real?: boolean): string {
const fileRes = execSync(
`ffprobe -v error -select_streams v:0 -show_entries stream=width,height -of csv=p=0 "${file}"`,
{encoding: 'utf-8', shell: 'cmd'});
if(real) return fileRes;
const y = Number(fileRes.split(',')[1]);
return <string>Object.keys(Resolution).reduce((best:[string, number], res: string) => {
const diff = Math.abs(y - Resolution[res]);
return (best == null || best[1] > diff) ? [res, diff] : best;
}, null)[0];
}
/**
* Use FFProbe to get an ordered list of available tracks for a given stream, identified by the tagged language
*
* @example
* ```ts
* console.log(getStreams('./sample.mp4', 'audio'));
* // Output: ['eng', 'fr', 'unk']
* ```
*
* @param {string} file Absolute or relative path to video file
* @param {"audio" | "subtitle" | "video"} stream Type of stream to inspect
* @returns {string[]} Ordered list of available tracks & their respective tagged language
*/
export function getStreams(file: string, stream: 'audio' | 'subtitle' | 'video'): string[] {
const resp = execSync(
`ffprobe -v error -select_streams ${stream[0]} -show_entries stream=index:stream_tags=language -of csv=p=0 "${file}"`,
{encoding: 'utf-8', shell: 'cmd'});
return !resp ? [] : resp.trim().split('\n').map(text => {
const split = text.split(',');
return !!split[1] ? split[1].trim() : 'unk'
});
}

View File

@ -0,0 +1,55 @@
import * as crypto from 'crypto';
import * as fs from 'fs';
import {join} from 'path';
/**
* Calculate checksum of file
*
* @param {string} path Path to file
* @returns {string} md5 checksum
*/
export function createChecksum(path: string): Promise<string> {
return new Promise(function (res, rej) {
const hash = crypto.createHash('md5');
const input = fs.createReadStream(path);
input.on('error', rej);
input.on('data', (chunk) => hash.update(chunk));
input.on('close', () => res(hash.digest('hex')));
});
}
/**
* Break apart a relative or absolute path to a file into it's individual parts
*
* @example
* ```ts
* console.log(parseFilename('/some/path/sample.mp4'));
* // Output: {path: '/some/path/', fileName: 'sample.mp4', baseName: 'sample', extension: 'mp4'}
* ```
*
* @param {string} file Absolute or relative path to a file
* @returns {{path: any, fileName: any, extension: any, baseName: any}} The components that makeup the given file path
*/
export function parseFilePath(file: string): {path: string, fileName: string, baseName: string, extension: string} {
const matches: any = /^(?<path>.*?)(?<fileName>(?<baseName>[a-zA-Z0-9_\-\.\(\)\s]+)\.(?<extension>[a-zA-Z0-9]+))$/.exec(file);
return {
path: matches?.groups?.['path'],
baseName: matches?.groups?.['baseName'],
fileName: matches?.groups?.['fileName'],
extension: matches?.groups?.['extension'],
}
}
/**
* Recursively search a directory for files
*
* @param {string} path Starting path
* @returns {string[]} List of discovered files
*/
export function deepSearch(path: string): string[] {
return fs.readdirSync(path).reduce((found, file) => {
const filePath = join(path, file), isFile = fs.lstatSync(filePath).isFile();
return [...found, ...(isFile ? [filePath] : deepSearch(filePath))];
}, []);
}

View File

@ -0,0 +1,4 @@
export function whereBuilder<T>(query: T, where: object): T {
return !where ? query :
Object.entries(where).reduce((qb, [key, val]) => (<any>query).where(key, '=', val), query);
}