This commit is contained in:
2023-08-14 14:36:45 -04:00
commit b5966f98b2
94 changed files with 21124 additions and 0 deletions

9
common/.gitignore vendored Normal file
View File

@ -0,0 +1,9 @@
# IDEs
.idea
.vscode
# Artifacts
coverage
dist
junit.xml
node_modules

View File

@ -0,0 +1,105 @@
image: node:18
npm:
stage: build
cache:
- key:
files:
- package-lock.json
paths:
- node_modules
policy: pull-push
- key: $CI_PIPELINE_ID
paths:
- dist
policy: push
script:
- npm install
- npm run build
artifacts:
paths:
- dist
expire_in: 1 week
rules:
- if: $CI_COMMIT_BRANCH
audit:
stage: test
cache:
- key:
files:
- package-lock.json
paths:
- node_modules
policy: pull
script:
- echo "vulnerabilities_high $(npm audit | grep -oE '[0-9]+ high' | grep -oE '[0-9]+' || echo 0)" > metrics.txt
- echo "vulnerabilities_medium $(npm audit | grep -oE '[0-9]+ moderate' | grep -oE '[0-9]+' || echo 0)" >> metrics.txt
- echo "vulnerabilities_low $(npm audit | grep -oE '[0-9]+ low' | grep -oE '[0-9]+' || echo 0)" >> metrics.txt
artifacts:
reports:
metrics: metrics.txt
rules:
- if: $CI_COMMIT_BRANCH
jest:
stage: test
cache:
- key:
files:
- package-lock.json
paths:
- node_modules
policy: pull
script:
- npm run test:coverage
coverage: /All\sfiles.*?\s+(\d+.\d+)/
artifacts:
when: always
reports:
junit: junit.xml
rules:
- if: $CI_COMMIT_BRANCH
registry:
stage: deploy
cache:
- key:
files:
- package.json
paths:
- node_modules
policy: pull
- key: $CI_PIPELINE_ID
paths:
- dist
policy: pull
before_script:
- VERSION=$(cat package.json | grep version | grep -Eo ':.+' | grep -Eo '[[:alnum:]\.\/\-]+')
- if [ "$CI_COMMIT_BRANCH" != "$CI_DEFAULT_BRANCH" ] && [ "$VERSION" != *"-$CI_COMMIT_BRANCH" ]; then VERSION="$VERSION-$(echo "$CI_COMMIT_BRANCH" | sed -E "s/[_/]/-/g")"; npm version --no-git-tag-version $VERSION; fi
script:
- PACKAGES=$(curl -s -H "PRIVATE-TOKEN:$DEPLOY_TOKEN" https://$CI_SERVER_HOST/api/v4/projects/$CI_PROJECT_ID/packages)
- ID=$(node -pe "JSON.parse(process.argv[1]).find(p => p['version'] == process.argv[2])?.id || ''" $PACKAGES $VERSION)
- if [ -n "$ID" ]; then curl -s -X DELETE -H "PRIVATE-TOKEN:$DEPLOY_TOKEN" https://$CI_SERVER_HOST/api/v4/projects/$CI_PROJECT_ID/packages/$ID; fi
- printf "@transmute:registry=https://$CI_SERVER_HOST/api/v4/projects/$CI_PROJECT_ID/packages/npm/\n//$CI_SERVER_HOST/api/v4/projects/$CI_PROJECT_ID/packages/npm/:_authToken=$DEPLOY_TOKEN" > .npmrc
- npm publish
rules:
- if: '$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH'
- if: $CI_COMMIT_BRANCH
when: manual
allow_failure: true
tag:
stage: deploy
image:
name: alpine/git
entrypoint: [""]
cache: []
before_script:
- git remote set-url origin "https://ReleaseBot:$DEPLOY_TOKEN@$CI_SERVER_HOST/$CI_PROJECT_PATH.git"
script:
- VERSION=$(cat package.json | grep version | grep -Eo ':.+' | grep -Eo '[[:alnum:]\.\/\-]+')
- git tag -f $VERSION $CI_COMMIT_SHA
- git push -f origin $VERSION
rules:
- if: '$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH'

65
common/README.md Normal file
View File

@ -0,0 +1,65 @@
# Transmute - Common
Provides common types & utilities used throughout the Transmute stack.
Please check out the [Transmute repository](https://gitlab.zakscode.com/zakscode/transmute/transmute) for more info.
## Table of Contents
<!-- TOC -->
* [Transmute - Common](#transmute---common)
* [Table of Contents](#table-of-contents)
* [Prerequisites](#prerequisites)
* [Setup](#setup)
* [Cheatsheet](#cheatsheet)
<!-- TOC -->
## Prerequisites
- [Git](https://git-scm.com/book/en/v2/Getting-Started-Installing-Git)
- [NodeJS 18](https://nodejs.org/en/)
## Setup
<details>
<summary>NPM Install</summary>
This will install the [prebuilt library](https://gitlab.zakscode.com/zakscode/transmute/transmute/-/packages) from GitLab:
1. Create a `.npmrc` file & add the GitLab's package registry URL':
```
@transmute:registry=https://gitlab.zakscode.com/api/v4/projects/85/packages/npm/
//gitlab.zakscode.com/api/v4/projects/85/packages/npm/:_authToken=tvNAnPtzjy59xFrHBJ2J
```
2. Install as normal: `npm install --save @transmute/common`
If you would like to use your local source code instead of the prebuilt library, continue to the <ins>NPM Link</ins> section.
</details>
<details>
<summary>NPM Link</summary>
Make sure you have completed the <ins>NPM Install</ins> section before continuing.
A local copy of common can be used to test changes using [npm link](https://docs.npmjs.com/cli/v8/commands/npm-link). After cloning:
1. Install the dependencies: `npm install`
2. Build or watch the common library: `npm run build` or `npm run watch`
3. link the library to npm from common's root directory: `npm link`
4. Link the library to a project: `cd ../project && npm link @transmute/common`
**Warning:** Step 4 will need to be re-run when ever an `npm install` is completed.
This will only work on your local machine. Make sure you have completed the __NPM Install__ & `@cwb/common` is part of `package.json`.
</details>
## Cheatsheet
```bash
# Build JS
npm run build
# Watch for changes
npm run watch
# Run unit tests
npm test
# Re-run tests on changes
npm run test:watch
```

16
common/jest.config.js Normal file
View File

@ -0,0 +1,16 @@
module.exports = {
"reporters": ["default", "jest-junit"],
"roots": [
"<rootDir>/tests"
],
"testMatch": [
"**/?(*.)+(spec|test).+(ts|tsx|js)"
],
"transform": {
".+\\.(ts)$": "ts-jest"
},
collectCoverageFrom: [
'src/**/utils/**/*.ts',
'!src/**/utils/**/*.d.ts'
],
};

3605
common/package-lock.json generated Normal file

File diff suppressed because it is too large Load Diff

27
common/package.json Normal file
View File

@ -0,0 +1,27 @@
{
"name": "@transmute/common",
"version": "0.0.0",
"description": "Transmute dependencies",
"author": "ztimson",
"main": "dist/index.js",
"types": "dist/index.d.ts",
"scripts": {
"build": "npx tsc",
"test": "npx jest --verbose",
"test:watch": "npx jest --watch",
"test:coverage": "npx jest --verbose --coverage",
"watch": "npm run build && npx tsc --watch"
},
"dependencies": { },
"devDependencies": {
"@types/jest": "^29.2.3",
"@types/node": "^18.15.3",
"jest": "^29.3.1",
"jest-junit": "^15.0.0",
"ts-jest": "^29.0.3",
"typescript": "^4.9.3"
},
"files": [
"dist"
]
}

11
common/src/index.ts Normal file
View File

@ -0,0 +1,11 @@
// Models
export * from './models/config';
export * from './models/job';
export * from './models/languages';
export * from './models/library';
export * from './models/metrics';
export * from './models/video';
// Utilities
export * from './utils/logger.utils';
export * from './utils/object.utils';

View File

@ -0,0 +1,45 @@
export type Config = {
/** What type of job type should be prioritized, leaving blank will automatically both queues */
priority: 'healthcheck' | 'transcode' | null;
/** Enable healthchecks using the given method, leaving blank disables */
healthcheck: 'quick' | 'verbose' | null;
/** Automatically delete unhealthy files */
deleteUnhealthy: boolean;
/** Require videos pass a healthcheck before being transcoded */
onlyTranscodeHealthy: boolean;
/** Delete original video file after it's been successfully transcoded */
deleteOriginal: boolean;
/** Desired video container/extension, leaving blank will accept any container type */
targetContainer: string | null;
/** Desired video codec, leaving blank will accept any codec */
targetVideoCodec: string | null;
/** Desired audio codec, leaving blank will accept any codec */
targetAudioCodec: string | null;
/** Only keep 1 audio track if multiple match */
singleAudioTrack: boolean;
/** Accepted audio track languages, leaving blank removes all */
audioWhitelist: string[];
/** Accepted subtitle languages, leaving blank removes all */
subtitleWhitelist: string[];
};
export const ConfigDefaults: Config = {
priority: null,
healthcheck: null,
deleteUnhealthy: false,
onlyTranscodeHealthy: false,
deleteOriginal: false,
targetContainer: null,
targetVideoCodec: null,
targetAudioCodec: null,
singleAudioTrack: false,
audioWhitelist: ['eng', 'unk'],
subtitleWhitelist: [],
}
export type KeyVal = {
/** Configuration key */
key: string;
/** Configuration value */
value: any;
}

8
common/src/models/job.ts Normal file
View File

@ -0,0 +1,8 @@
import {File} from 'buffer';
export type JobType = 'healthcheck' | 'transcode'
export type Job = {
type: JobType,
file: File
}

View File

@ -0,0 +1,6 @@
export enum Languages {
eng = 'English',
fre = 'French',
spa = 'Spanish',
unk = 'Unknown'
}

View File

@ -0,0 +1,10 @@
export type Library = {
/** Primary Key */
id?: number;
/** Human-readable name */
name: string;
/** Path to library folder */
path: string;
/** Monitor directory for changes */
watch: boolean;
}

View File

@ -0,0 +1,15 @@
export type Metrics = {
resolution: {[key: string]: number},
container: {[key: string]: number},
videoCodec: {[key: string]: number},
audioCodec: {[key: string]: number},
health: {
healthy: number,
unhealthy: number,
unknown: number
},
audioLang: {[key: string]: number},
subLang: {[key: string]: number},
size: number,
videos: number
}

View File

@ -0,0 +1,63 @@
export const Resolution = {
'240p': 240,
'360p': 360,
'480p': 480,
'720p': 720,
'1080p': 1080,
'4k': 2160,
'8k': 4320,
}
export enum Container {
avi = 'AVI',
mkv = 'MKV',
mp4 = 'MP4',
webm = 'WebM'
}
export enum VideoCodec {
h264 = 'h.264 (AVC)',
h265 = 'h.265 (HEVC)',
h266 = 'h.266 (VVC)',
mpeg2 = 'MPEG-2',
mpeg4 = 'MPEG-4'
}
export enum AudioCodec {
aac = 'AAC',
ac3 = 'AC3',
mp3 = 'MP3',
vorbis = 'Ogg Vorbis',
wav = 'WAV'
}
export type VideoMeta = {
/** Closest standard (NOT actual) video resolution (420p, 720p, 1080p, etc..) */
resolution: string;
/** Algorithm used to encode video */
videoCodec?: keyof VideoCodec;
/** Algorithm used to encode audio */
audioCodec?: keyof AudioCodec;
/** List of available audio tracks */
audioTracks?: string[];
/** List of available subtitle languages */
subtitleTracks?: string[];
}
export type Video = VideoMeta & {
id?: number;
/** Name of the file (extension included, path omitted: "sample.mp4") */
name: string;
/** Path to file */
path: string;
/** Library foreign key */
library: number;
/** Video container/File extension (Binds everything together) */
container?: keyof Container;
/** Whether the file is healthy or not; null if unchecked */
checksum?: string;
/** Size of file in bytes */
healthy?: boolean;
/** Checksum of file - useful for seeing if a file has changed */
size: number;
}

View File

@ -0,0 +1,65 @@
export const CliEffects = {
CLEAR: "\x1b[0m",
BRIGHT: "\x1b[1m",
DIM: "\x1b[2m",
UNDERSCORE: "\x1b[4m",
BLINK: "\x1b[5m",
REVERSE: "\x1b[7m",
HIDDEN: "\x1b[8m",
}
export const CliForeground = {
BLACK: "\x1b[30m",
RED: "\x1b[31m",
GREEN: "\x1b[32m",
YELLOW: "\x1b[33m",
BLUE: "\x1b[34m",
MAGENTA: "\x1b[35m",
CYAN: "\x1b[36m",
WHITE: "\x1b[37m",
GREY: "\x1b[90m",
}
export const CliBackground = {
BLACK: "\x1b[40m",
RED: "\x1b[41m",
GREEN: "\x1b[42m",
YELLOW: "\x1b[43m",
BLUE: "\x1b[44m",
MAGENTA: "\x1b[45m",
CYAN: "\x1b[46m",
WHITE: "\x1b[47m",
GREY: "\x1b[100m",
}
export class Logger {
constructor(public readonly namespace: string) { }
private format(...text: string[]): string {
return `${new Date().toISOString()} [${this.namespace}] ${text.join(' ')}`;
}
debug(...args: string[]) {
console.log(CliForeground.MAGENTA + this.format(...args) + CliEffects.CLEAR);
}
error(...args: string[]) {
console.log(CliForeground.RED + this.format(...args) + CliEffects.CLEAR);
}
info(...args: string[]) {
console.log(CliForeground.CYAN + this.format(...args) + CliEffects.CLEAR);
}
log(...args: string[]) {
console.log(CliEffects.CLEAR + this.format(...args));
}
warn(...args: string[]) {
console.log(CliForeground.YELLOW + this.format(...args) + CliEffects.CLEAR);
}
verbose(...args: string[]) {
console.log(CliForeground.WHITE + this.format(...args) + CliEffects.CLEAR);
}
}

View File

@ -0,0 +1,109 @@
/**
* Removes any null values from an object in-place
*
* @example
* ```ts
* let test = {a: 0, b: false, c: null, d: 'abc'}
* console.log(clean(test)); // Output: {a: 0, b: false, d: 'abc'}
* ```
*
* @param {T} obj Object reference that will be cleaned
* @returns {Partial<T>} Cleaned object
*/
export function clean<T>(obj: T): Partial<T> {
if(obj == null) throw new Error("Cannot clean a NULL value");
Object.entries(obj).forEach(([key, value]) => {
if(value == null) delete (<any>obj)[key];
});
return <any>obj;
}
/**
* Create a deep copy of an object (vs. a shallow copy of references)
*
* Should be replaced by `structuredClone` once released.
*
* @param {T} value Object to copy
* @returns {T} Type
*/
export function deepCopy<T>(value: T): T {
return JSON.parse(JSON.stringify(value));
}
/**
* Get/set a property of an object using dot notation
*
* @example
* ```ts
* // Get a value
* const name = dotNotation<string>(person, 'firstName');
* const familyCarMake = dotNotation(family, 'cars[0].make');
* // Set a value
* dotNotation(family, 'cars[0].make', 'toyota');
* ```
*
* @type T Return type
* @param {Object} obj source object to search
* @param {string} prop property name (Dot notation & indexing allowed)
* @param {any} set Set object property to value, omit to fetch value instead
* @return {T} property value
*/
export function dotNotation<T>(obj: any, prop: string, set: T): T;
export function dotNotation<T>(obj: any, prop: string): T | undefined;
export function dotNotation<T>(obj: any, prop: string, set?: T): T | undefined {
if(obj == null || !prop) return undefined;
// Split property string by '.' or [index]
return <T>prop.split(/[.[\]]/g).filter(prop => prop.length).reduce((obj, prop, i, arr) => {
if(prop[0] == '"' || prop[0] == "'") prop = prop.slice(1, -1); // Take quotes out
if(!obj?.hasOwnProperty(prop)) {
if(set == undefined) return undefined;
obj[prop] = {};
}
if(set !== undefined && i == arr.length - 1)
return obj[prop] = set;
return obj[prop];
}, obj);
}
/**
* Check that an object has the following values
*
* @example
* ```ts
* const test = {a: 2, b: 2};
* includes(test, {a: 1}); // true
* includes(test, {b: 1, c: 3}); // false
* ```
*
* @param target Object to search
* @param values Criteria to check against
* @param allowMissing Only check the keys that are available on the target
* @returns {boolean} Does target include all the values
*/
export function includes(target: any, values: any, allowMissing = false): boolean {
if(target == undefined) return allowMissing;
if(Array.isArray(values)) return values.findIndex((e, i) => !includes(target[i], values[i], allowMissing)) == -1;
const type = typeof values;
if(type != typeof target) return false;
if(type == 'object') {
return Object.keys(values).find(key => !includes(target[key], values[key], allowMissing)) == null;
}
if(type == 'function') return target.toString() == values.toString();
return target == values;
}
/**
* Deep check if two objects are equal
*
* @param {any} a - first item to compare
* @param {any} b - second item to compare
* @returns {boolean} True if they match
*/
export function isEqual(a: any, b: any): boolean {
const ta = typeof a, tb = typeof b;
if((ta != 'object' || a == null) || (tb != 'object' || b == null))
return ta == 'function' && tb == 'function' ? a.toString() == b.toString() : a === b;
const keys = Object.keys(a);
if(keys.length != Object.keys(b).length) return false;
return Object.keys(a).every(key => isEqual(a[key], b[key]));
}

View File

@ -0,0 +1,98 @@
import {clean, deepCopy, dotNotation, includes, isEqual} from "../../src";
describe('Object Utilities', () => {
const TEST_OBJECT = {
a: 1,
b: [
[2, 3],
[4, 5]
],
c: {
d: [
[{e: 6, f: 7}]
],
},
g: {h: 8},
i: () => 9
};
describe('clean', () => {
test('remove null properties', () => {
const a = {a: 1, b: 2, c: null};
const final = {a: 1, b: 2};
expect(clean(a)).toEqual(final);
});
});
describe('deepCopy', () => {
const copy = deepCopy(TEST_OBJECT);
test('Array of arrays', () => {
const a = [[1, 2], [3, 4]];
const b = deepCopy(a);
b[0][1] = 5;
expect(a).not.toEqual(b);
});
test('Change array inside object', () => {
copy.b[1] = [1, 1, 1];
expect(copy.b[1]).not.toEqual(TEST_OBJECT.b[1]);
});
test('Change object inside object', () => {
copy.g = {h: Math.random()};
expect(copy.g).not.toEqual(TEST_OBJECT.g);
});
test('Change object property inside nested array', () => {
copy.c.d[0][0].e = -1;
expect(copy.c.d[0][0].e).not.toEqual(TEST_OBJECT.c.d[0][0].e);
});
});
describe('dotNotation', () => {
test('no object or properties', () => {
expect(dotNotation(undefined, 'z')).toStrictEqual(undefined);
expect(dotNotation(TEST_OBJECT, '')).toStrictEqual(undefined);
});
test('invalid property', () => expect(dotNotation(TEST_OBJECT, 'z')).toBeUndefined());
test('by property', () => expect(dotNotation(TEST_OBJECT, 'a')).toBe(TEST_OBJECT.a));
test('by key', () => expect(dotNotation(TEST_OBJECT, '["a"]')).toBe(TEST_OBJECT['a']));
test('by key (single quote)', () => expect(dotNotation(TEST_OBJECT, '[\'a\']')).toBe(TEST_OBJECT['a']));
test('by key (double quote)', () => expect(dotNotation(TEST_OBJECT, '["a"]')).toBe(TEST_OBJECT['a']));
test('by index', () => expect(dotNotation(TEST_OBJECT, 'b[0]')).toBe(TEST_OBJECT.b[0]));
test('by index (2d)', () => expect(dotNotation(TEST_OBJECT, 'b[1][1]')).toBe(TEST_OBJECT.b[1][1]));
test('everything combined', () => expect(dotNotation(TEST_OBJECT, 'c["d"][0][0].e'))
.toBe(TEST_OBJECT.c['d'][0][0].e));
test('set value', () => {
const COPY = JSON.parse(JSON.stringify(TEST_OBJECT));
dotNotation(COPY, 'c["d"][0][0].e', 'test');
expect(COPY['c']['d'][0][0]['e']).toBe('test');
});
test('set new value', () => {
const COPY = JSON.parse(JSON.stringify(TEST_OBJECT));
dotNotation(COPY, 'c.x.y.z', 'test');
expect(COPY['c']['x']['y']['z']).toBe('test');
});
});
describe('includes', () => {
test('simple', () => expect(includes(TEST_OBJECT, {a: 1})).toBeTruthy());
test('nested', () => expect(includes(TEST_OBJECT, {g: {h: 8}})).toBeTruthy());
test('array', () => expect(includes(TEST_OBJECT, {b: [[2]]})).toBeTruthy());
test('nested array', () => expect(includes(TEST_OBJECT, {a: 1, c: {d: [[{e: 6}]]}})).toBeTruthy());
test('wong nested array', () => expect(includes(TEST_OBJECT, {a: 1, c: {d: [{e: 7}]}})).toBeFalsy());
test('wrong value', () => expect(includes(TEST_OBJECT, {a: 1, b: 2})).toBeFalsy());
test('missing value', () => expect(includes(TEST_OBJECT, {a: 1, i: 10})).toBeFalsy());
});
describe('isEqual', () => {
test('boolean equal', () => expect(isEqual(true, true)).toBeTruthy());
test('boolean not-equal', () => expect(isEqual(true, false)).toBeFalsy());
test('number equal', () => expect(isEqual(1, 1)).toBeTruthy());
test('number not-equal', () => expect(isEqual(1, 0)).toBeFalsy());
test('string equal', () => expect(isEqual('abc', 'abc')).toBeTruthy());
test('string not-equal', () => expect(isEqual('abc', '')).toBeFalsy());
test('array equal', () => expect(isEqual([true, 1, 'a'], [true, 1, 'a'])).toBeTruthy());
test('array not-equal', () => expect(isEqual([true, 1, 'a'], [1])).toBeFalsy());
test('object equal', () => expect(isEqual({a: 1, b: 2}, {a: 1, b: 2})).toBeTruthy());
test('object not-equal', () => expect(isEqual({a: 1, b: 2}, {a: 1})).toBeFalsy());
test('complex', () => expect(isEqual(TEST_OBJECT, TEST_OBJECT)).toBeTruthy());
});
});

14
common/tsconfig.json Normal file
View File

@ -0,0 +1,14 @@
{
"compilerOptions": {
"declaration": true,
"declarationMap": true,
"lib": ["ESNext"],
"module": "commonjs",
"outDir": "./dist",
"strict": true,
"target": "es2015"
},
"include": [
"src/**/*"
]
}